feat(backend): initial data export support

obviously it's missing things that haven't been added yet

Foxnouns.Backend/Controllers/ExportsController.cs

@@ -0,0 +1,74 @@
+using Coravel.Queuing.Interfaces;
+using Foxnouns.Backend.Database;
+using Foxnouns.Backend.Database.Models;
+using Foxnouns.Backend.Jobs;
+using Foxnouns.Backend.Middleware;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.EntityFrameworkCore;
+using NodaTime;
+
+namespace Foxnouns.Backend.Controllers;
+
+[Route("/api/internal/data-exports")]
+[Authorize("identify")]
+public class ExportsController(
+    ILogger logger,
+    Config config,
+    IClock clock,
+    DatabaseContext db,
+    IQueue queue
+) : ApiControllerBase
+{
+    private static readonly Duration MinimumTimeBetween = Duration.FromDays(1);
+    private readonly ILogger _logger = logger.ForContext<ExportsController>();
+
+    [HttpGet]
+    public async Task<IActionResult> GetDataExportsAsync()
+    {
+        var export = await db
+            .DataExports.Where(d => d.UserId == CurrentUser!.Id)
+            .OrderByDescending(d => d.Id)
+            .FirstOrDefaultAsync();
+        if (export == null)
+            return Ok(new DataExportResponse(null, null));
+
+        return Ok(
+            new DataExportResponse(
+                ExportUrl(CurrentUser!.Id, export.Filename),
+                export.Id.Time + DataExport.Expiration
+            )
+        );
+    }
+
+    private string ExportUrl(Snowflake userId, string filename) =>
+        $"{config.MediaBaseUrl}/data-exports/{userId}/{filename}.zip";
+
+    private record DataExportResponse(string? Url, Instant? ExpiresAt);
+
+    [HttpPost]
+    public async Task<IActionResult> QueueDataExportAsync()
+    {
+        var snowflakeToCheck = Snowflake.FromInstant(
+            clock.GetCurrentInstant() - MinimumTimeBetween
+        );
+        _logger.Debug(
+            "Checking if user {UserId} has data exports newer than {Snowflake}",
+            CurrentUser!.Id,
+            snowflakeToCheck
+        );
+        if (
+            await db.DataExports.AnyAsync(d =>
+                d.UserId == CurrentUser.Id && d.Id > snowflakeToCheck
+            )
+        )
+        {
+            throw new ApiError.BadRequest("You can't request a new data export so soon.");
+        }
+
+        queue.QueueInvocableWithPayload<CreateDataExportInvocable, CreateDataExportPayload>(
+            new CreateDataExportPayload(CurrentUser.Id)
+        );
+
+        return NoContent();
+    }
+}

Foxnouns.Backend/Database/DatabaseContext.cs

@@ -46,17 +46,18 @@ public class DatabaseContext(DbContextOptions options) : DbContext(options)
             .UseSnakeCaseNamingConvention()
             .UseExceptionProcessor();
 
-    public DbSet<User> Users { get; set; }
+    public DbSet<User> Users { get; init; }
-    public DbSet<Member> Members { get; set; }
+    public DbSet<Member> Members { get; init; }
-    public DbSet<AuthMethod> AuthMethods { get; set; }
+    public DbSet<AuthMethod> AuthMethods { get; init; }
-    public DbSet<FediverseApplication> FediverseApplications { get; set; }
+    public DbSet<FediverseApplication> FediverseApplications { get; init; }
-    public DbSet<Token> Tokens { get; set; }
+    public DbSet<Token> Tokens { get; init; }
-    public DbSet<Application> Applications { get; set; }
+    public DbSet<Application> Applications { get; init; }
-    public DbSet<TemporaryKey> TemporaryKeys { get; set; }
+    public DbSet<TemporaryKey> TemporaryKeys { get; init; }
+    public DbSet<DataExport> DataExports { get; init; }
 
-    public DbSet<PrideFlag> PrideFlags { get; set; }
+    public DbSet<PrideFlag> PrideFlags { get; init; }
-    public DbSet<UserFlag> UserFlags { get; set; }
+    public DbSet<UserFlag> UserFlags { get; init; }
-    public DbSet<MemberFlag> MemberFlags { get; set; }
+    public DbSet<MemberFlag> MemberFlags { get; init; }
 
     protected override void ConfigureConventions(ModelConfigurationBuilder configurationBuilder)
     {
@@ -81,6 +82,7 @@ public class DatabaseContext(DbContextOptions options) : DbContext(options)
             })
             .HasFilter("fediverse_application_id IS NOT NULL")
             .IsUnique();
+        modelBuilder.Entity<DataExport>().HasIndex(d => d.Filename).IsUnique();
 
         modelBuilder
             .Entity<AuthMethod>()

Foxnouns.Backend/Database/Migrations/20241202153736_AddDataExports.cs

@@ -0,0 +1,57 @@
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Foxnouns.Backend.Database.Migrations
+{
+    /// <inheritdoc />
+    [DbContext(typeof(DatabaseContext))]
+    [Migration("20241202153736_AddDataExports")]
+    public partial class AddDataExports : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                name: "data_exports",
+                columns: table => new
+                {
+                    id = table.Column<long>(type: "bigint", nullable: false),
+                    user_id = table.Column<long>(type: "bigint", nullable: false),
+                    filename = table.Column<string>(type: "text", nullable: false),
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("pk_data_exports", x => x.id);
+                    table.ForeignKey(
+                        name: "fk_data_exports_users_user_id",
+                        column: x => x.user_id,
+                        principalTable: "users",
+                        principalColumn: "id",
+                        onDelete: ReferentialAction.Cascade
+                    );
+                }
+            );
+
+            migrationBuilder.CreateIndex(
+                name: "ix_data_exports_filename",
+                table: "data_exports",
+                column: "filename",
+                unique: true
+            );
+
+            migrationBuilder.CreateIndex(
+                name: "ix_data_exports_user_id",
+                table: "data_exports",
+                column: "user_id"
+            );
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(name: "data_exports");
+        }
+    }
+}

Foxnouns.Backend/Database/Migrations/DatabaseContextModelSnapshot.cs

@@ -111,6 +111,34 @@ namespace Foxnouns.Backend.Database.Migrations
                     b.ToTable("auth_methods", (string)null);
                 });
 
+            modelBuilder.Entity("Foxnouns.Backend.Database.Models.DataExport", b =>
+                {
+                    b.Property<long>("Id")
+                        .HasColumnType("bigint")
+                        .HasColumnName("id");
+
+                    b.Property<string>("Filename")
+                        .IsRequired()
+                        .HasColumnType("text")
+                        .HasColumnName("filename");
+
+                    b.Property<long>("UserId")
+                        .HasColumnType("bigint")
+                        .HasColumnName("user_id");
+
+                    b.HasKey("Id")
+                        .HasName("pk_data_exports");
+
+                    b.HasIndex("Filename")
+                        .IsUnique()
+                        .HasDatabaseName("ix_data_exports_filename");
+
+                    b.HasIndex("UserId")
+                        .HasDatabaseName("ix_data_exports_user_id");
+
+                    b.ToTable("data_exports", (string)null);
+                });
+
             modelBuilder.Entity("Foxnouns.Backend.Database.Models.FediverseApplication", b =>
                 {
                     b.Property<long>("Id")
@@ -515,6 +543,18 @@ namespace Foxnouns.Backend.Database.Migrations
                     b.Navigation("User");
                 });
 
+            modelBuilder.Entity("Foxnouns.Backend.Database.Models.DataExport", b =>
+                {
+                    b.HasOne("Foxnouns.Backend.Database.Models.User", "User")
+                        .WithMany()
+                        .HasForeignKey("UserId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_data_exports_users_user_id");
+
+                    b.Navigation("User");
+                });
+
             modelBuilder.Entity("Foxnouns.Backend.Database.Models.Member", b =>
                 {
                     b.HasOne("Foxnouns.Backend.Database.Models.User", "User")

Foxnouns.Backend/Database/Models/DataExport.cs

@@ -0,0 +1,14 @@
+using System.ComponentModel.DataAnnotations.Schema;
+using NodaTime;
+
+namespace Foxnouns.Backend.Database.Models;
+
+public class DataExport : BaseModel
+{
+    public Snowflake UserId { get; init; }
+    public User User { get; init; } = null!;
+    public required string Filename { get; init; }
+
+    [NotMapped]
+    public static readonly Duration Expiration = Duration.FromDays(15);
+}

Foxnouns.Backend/Database/Snowflake.cs

@@ -65,6 +65,9 @@ public readonly struct Snowflake(ulong value) : IEquatable<Snowflake>
         return true;
     }
 
+    public static Snowflake FromInstant(Instant instant) =>
+        new((ulong)(instant.ToUnixTimeMilliseconds() - Epoch) << 22);
+
     public override bool Equals(object? obj) => obj is Snowflake other && Value == other.Value;
 
     public bool Equals(Snowflake other)

Foxnouns.Backend/Extensions/WebApplicationExtensions.cs

@@ -106,12 +106,14 @@ public static class WebApplicationExtensions
                     .AddScoped<RemoteAuthService>()
                     .AddScoped<FediverseAuthService>()
                     .AddScoped<ObjectStorageService>()
+                    .AddTransient<DataCleanupService>()
                     // Background services
                     .AddHostedService<PeriodicTasksService>()
                     // Transient jobs
                     .AddTransient<MemberAvatarUpdateInvocable>()
                     .AddTransient<UserAvatarUpdateInvocable>()
-                    .AddTransient<CreateFlagInvocable>();
+                    .AddTransient<CreateFlagInvocable>()
+                    .AddTransient<CreateDataExportInvocable>();
 
                 if (!config.Logging.EnableMetrics)
                     services.AddHostedService<BackgroundMetricsCollectionService>();

Foxnouns.Backend/Jobs/CreateDataExportInvocable.cs

@@ -0,0 +1,209 @@
+using System.IO.Compression;
+using System.Net;
+using Coravel.Invocable;
+using Foxnouns.Backend.Database;
+using Foxnouns.Backend.Database.Models;
+using Foxnouns.Backend.Services;
+using Foxnouns.Backend.Utils;
+using Microsoft.EntityFrameworkCore;
+using Newtonsoft.Json;
+using NodaTime;
+using NodaTime.Text;
+
+namespace Foxnouns.Backend.Jobs;
+
+public class CreateDataExportInvocable(
+    DatabaseContext db,
+    IClock clock,
+    UserRendererService userRenderer,
+    MemberRendererService memberRenderer,
+    ObjectStorageService objectStorageService,
+    ISnowflakeGenerator snowflakeGenerator,
+    ILogger logger
+) : IInvocable, IInvocableWithPayload<CreateDataExportPayload>
+{
+    private static readonly HttpClient Client = new();
+    private readonly ILogger _logger = logger.ForContext<CreateDataExportInvocable>();
+    public required CreateDataExportPayload Payload { get; set; }
+
+    public async Task Invoke()
+    {
+        try
+        {
+            await InvokeAsync();
+        }
+        catch (Exception e)
+        {
+            _logger.Error(e, "Error generating data export for user {UserId}", Payload.UserId);
+        }
+    }
+
+    private async Task InvokeAsync()
+    {
+        var user = await db
+            .Users.Include(u => u.AuthMethods)
+            .Include(u => u.Flags)
+            .Include(u => u.ProfileFlags)
+            .FirstOrDefaultAsync(u => u.Id == Payload.UserId);
+        if (user == null)
+        {
+            _logger.Warning(
+                "Received create data export request for user {UserId} but no such user exists, deleted or otherwise. Ignoring request",
+                Payload.UserId
+            );
+            return;
+        }
+
+        _logger.Information("Generating data export for user {UserId}", user.Id);
+
+        using var stream = new MemoryStream();
+        using var zip = new ZipArchive(stream, ZipArchiveMode.Create, true);
+        zip.Comment =
+            $"This archive for {user.Username} ({user.Id}) was generated at  {InstantPattern.General.Format(clock.GetCurrentInstant())}";
+
+        // Write the user's info and avatar
+        WriteJson(
+            zip,
+            "user.json",
+            await userRenderer.RenderUserInnerAsync(
+                user,
+                true,
+                ["*"],
+                renderMembers: false,
+                renderAuthMethods: true
+            )
+        );
+        await WriteS3Object(zip, "user-avatar.webp", userRenderer.AvatarUrlFor(user));
+
+        foreach (var flag in user.Flags)
+            await WritePrideFlag(zip, flag);
+
+        var members = await db
+            .Members.Include(m => m.User)
+            .Include(m => m.ProfileFlags)
+            .Where(m => m.UserId == user.Id)
+            .ToListAsync();
+        foreach (var member in members)
+            await WriteMember(zip, member);
+
+        // We want to dispose the ZipArchive on an error, but we need to dispose it manually to upload to object storage.
+        // Calling Dispose() multiple times is fine for this class, though.
+        // ReSharper disable once DisposeOnUsingVariable
+        zip.Dispose();
+        stream.Seek(0, SeekOrigin.Begin);
+
+        // Upload the file!
+        var filename = AuthUtils.RandomToken().Replace('+', '-').Replace('/', '_');
+        await objectStorageService.PutObjectAsync(
+            ExportPath(user.Id, filename),
+            stream,
+            "application/zip"
+        );
+
+        db.Add(
+            new DataExport
+            {
+                Id = snowflakeGenerator.GenerateSnowflake(),
+                UserId = user.Id,
+                Filename = filename,
+            }
+        );
+        await db.SaveChangesAsync();
+    }
+
+    private async Task WritePrideFlag(ZipArchive zip, PrideFlag flag)
+    {
+        _logger.Debug("Writing flag {FlagId}", flag.Id);
+
+        var flagData = $"""
+            {flag.Name}
+            ----
+            {flag.Description ?? "<no description>"}
+            """;
+
+        try
+        {
+            await WriteS3Object(zip, $"flag-{flag.Id}/flag.webp", userRenderer.ImageUrlFor(flag));
+        }
+        catch (Exception e)
+        {
+            _logger.Warning(e, "Could not write image for flag {FlagId}", flag.Id);
+            return;
+        }
+
+        var entry = zip.CreateEntry($"flag-{flag.Id}/flag.txt");
+        await using var stream = entry.Open();
+        await using var writer = new StreamWriter(stream);
+        await writer.WriteAsync(flagData);
+    }
+
+    private async Task WriteMember(ZipArchive zip, Member member)
+    {
+        _logger.Debug("Writing member {MemberId}", member.Id);
+
+        WriteJson(
+            zip,
+            $"members/{member.Name} ({member.Id}).json",
+            memberRenderer.RenderMember(member)
+        );
+
+        try
+        {
+            await WriteS3Object(
+                zip,
+                $"members/{member.Name} ({member.Id}).webp",
+                memberRenderer.AvatarUrlFor(member)
+            );
+        }
+        catch (Exception e)
+        {
+            _logger.Warning(e, "Error writing avatar for member {MemberId}", member.Id);
+        }
+    }
+
+    private void WriteJson(ZipArchive zip, string filename, object data)
+    {
+        var json = JsonConvert.SerializeObject(data, Formatting.Indented);
+
+        _logger.Debug(
+            "Writing file {Filename} to archive with size {Length}",
+            filename,
+            json.Length
+        );
+
+        var entry = zip.CreateEntry(filename);
+        using var stream = entry.Open();
+        using var writer = new StreamWriter(stream);
+        writer.Write(json);
+    }
+
+    private async Task WriteS3Object(ZipArchive zip, string filename, string? s3Path)
+    {
+        if (s3Path == null)
+            return;
+
+        var resp = await Client.GetAsync(s3Path);
+        if (resp.StatusCode != HttpStatusCode.OK)
+        {
+            _logger.Warning("S3 path {S3Path} returned a non-200 status, not saving file", s3Path);
+            return;
+        }
+
+        await using var respStream = await resp.Content.ReadAsStreamAsync();
+
+        _logger.Debug(
+            "Writing file {Filename} to archive with size {Length}",
+            filename,
+            respStream.Length
+        );
+
+        var entry = zip.CreateEntry(filename);
+        await using var entryStream = entry.Open();
+
+        respStream.Seek(0, SeekOrigin.Begin);
+        await respStream.CopyToAsync(entryStream);
+    }
+
+    private static string ExportPath(Snowflake userId, string b64) =>
+        $"data-exports/{userId}/{b64}.zip";
+}

Foxnouns.Backend/Jobs/Payloads.cs

@@ -11,3 +11,5 @@ public record CreateFlagPayload(
     string ImageData,
     string? Description
 );
+
+public record CreateDataExportPayload(Snowflake UserId);

Foxnouns.Backend/Services/DataCleanupService.cs

@@ -0,0 +1,47 @@
+using Foxnouns.Backend.Database;
+using Foxnouns.Backend.Database.Models;
+using Microsoft.EntityFrameworkCore;
+using NodaTime;
+
+namespace Foxnouns.Backend.Services;
+
+public class DataCleanupService(
+    DatabaseContext db,
+    IClock clock,
+    ILogger logger,
+    ObjectStorageService objectStorageService
+)
+{
+    private readonly ILogger _logger = logger.ForContext<DataCleanupService>();
+
+    public async Task InvokeAsync(CancellationToken ct = default)
+    {
+        _logger.Information("Cleaning up expired data exports");
+        await CleanExportsAsync(ct);
+    }
+
+    private async Task CleanExportsAsync(CancellationToken ct = default)
+    {
+        var minExpiredId = Snowflake.FromInstant(clock.GetCurrentInstant() - DataExport.Expiration);
+        var exports = await db.DataExports.Where(d => d.Id < minExpiredId).ToListAsync(ct);
+        if (exports.Count == 0)
+            return;
+
+        _logger.Debug("There are {Count} expired exports", exports.Count);
+
+        foreach (var export in exports)
+        {
+            _logger.Debug("Deleting export {ExportId}", export.Id);
+            await objectStorageService.RemoveObjectAsync(
+                ExportPath(export.UserId, export.Filename),
+                ct
+            );
+            db.Remove(export);
+        }
+
+        await db.SaveChangesAsync(ct);
+    }
+
+    private static string ExportPath(Snowflake userId, string b64) =>
+        $"data-exports/{userId}/{b64}.zip";
+}

Foxnouns.Backend/Services/MemberRendererService.cs

@@ -72,7 +72,7 @@ public class MemberRendererService(DatabaseContext db, Config config)
             renderUnlisted ? member.Unlisted : null
         );
 
-    private string? AvatarUrlFor(Member member) =>
+    public string? AvatarUrlFor(Member member) =>
         member.Avatar != null
             ? $"{config.MediaBaseUrl}/members/{member.Id}/avatars/{member.Avatar}.webp"
             : null;

Foxnouns.Backend/Services/MetricsCollectionService.cs

@@ -62,7 +62,7 @@ public class BackgroundMetricsCollectionService(
         using var timer = new PeriodicTimer(TimeSpan.FromMinutes(1));
         while (await timer.WaitForNextTickAsync(ct))
         {
-            _logger.Debug("Collecting metrics");
+            _logger.Debug("Collecting metrics manually");
             await metricsCollectionService.CollectMetricsAsync(ct);
         }
     }

Foxnouns.Backend/Services/ObjectStorageService.cs

@@ -48,4 +48,13 @@ public class ObjectStorageService(ILogger logger, Config config, IMinioClient mi
             ct
         );
     }
+
+    public async Task GetObjectAsync(string path, CancellationToken ct = default)
+    {
+        var stream = new MemoryStream();
+        var resp = await minioClient.GetObjectAsync(
+            new GetObjectArgs().WithBucket(config.Storage.Bucket).WithObject(path),
+            ct
+        );
+    }
 }

Foxnouns.Backend/Services/PeriodicTasksService.cs

@@ -8,10 +8,7 @@ public class PeriodicTasksService(ILogger logger, IServiceProvider services) : B
     {
         using var timer = new PeriodicTimer(TimeSpan.FromMinutes(1));
         while (await timer.WaitForNextTickAsync(ct))
-        {
-            _logger.Debug("Collecting metrics");
             await RunPeriodicTasksAsync(ct);
-        }
     }
 
     private async Task RunPeriodicTasksAsync(CancellationToken ct)
@@ -20,7 +17,10 @@ public class PeriodicTasksService(ILogger logger, IServiceProvider services) : B
 
         await using var scope = services.CreateAsyncScope();
 
-        var keyCacheSvc = scope.ServiceProvider.GetRequiredService<KeyCacheService>();
+        var keyCacheService = scope.ServiceProvider.GetRequiredService<KeyCacheService>();
-        await keyCacheSvc.DeleteExpiredKeysAsync(ct);
+        var dataCleanupService = scope.ServiceProvider.GetRequiredService<DataCleanupService>();
+
+        await keyCacheService.DeleteExpiredKeysAsync(ct);
+        await dataCleanupService.InvokeAsync(ct);
     }
 }

Foxnouns.Backend/Services/UserRendererService.cs

@@ -22,12 +22,31 @@ public class UserRendererService(
         bool renderAuthMethods = false,
         string? overrideSid = null,
         CancellationToken ct = default
+    ) =>
+        await RenderUserInnerAsync(
+            user,
+            selfUser != null && selfUser.Id == user.Id,
+            token?.Scopes ?? [],
+            renderMembers,
+            renderAuthMethods,
+            overrideSid,
+            ct
+        );
+
+    public async Task<UserResponse> RenderUserInnerAsync(
+        User user,
+        bool isSelfUser,
+        string[] scopes,
+        bool renderMembers = true,
+        bool renderAuthMethods = false,
+        string? overrideSid = null,
+        CancellationToken ct = default
     )
     {
-        var isSelfUser = selfUser?.Id == user.Id;
+        scopes = scopes.ExpandScopes();
-        var tokenCanReadHiddenMembers = token.HasScope("member.read") && isSelfUser;
+        var tokenCanReadHiddenMembers = scopes.Contains("member.read") && isSelfUser;
-        var tokenHidden = token.HasScope("user.read_hidden") && isSelfUser;
+        var tokenHidden = scopes.Contains("user.read_hidden") && isSelfUser;
-        var tokenPrivileged = token.HasScope("user.read_privileged") && isSelfUser;
+        var tokenPrivileged = scopes.Contains("user.read_privileged") && isSelfUser;
 
         renderMembers = renderMembers && (!user.ListHidden || tokenCanReadHiddenMembers);
         renderAuthMethods = renderAuthMethods && tokenPrivileged;
@@ -105,12 +124,12 @@ public class UserRendererService(
             user.CustomPreferences
         );
 
-    private string? AvatarUrlFor(User user) =>
+    public string? AvatarUrlFor(User user) =>
         user.Avatar != null
             ? $"{config.MediaBaseUrl}/users/{user.Id}/avatars/{user.Avatar}.webp"
             : null;
 
-    private string ImageUrlFor(PrideFlag flag) => $"{config.MediaBaseUrl}/flags/{flag.Hash}.webp";
+    public string ImageUrlFor(PrideFlag flag) => $"{config.MediaBaseUrl}/flags/{flag.Hash}.webp";
 
     public record UserResponse(
         Snowflake Id,