diff --git a/Foxnouns.Backend/Controllers/ExportsController.cs b/Foxnouns.Backend/Controllers/ExportsController.cs new file mode 100644 index 0000000..30ecaaa --- /dev/null +++ b/Foxnouns.Backend/Controllers/ExportsController.cs @@ -0,0 +1,74 @@ +using Coravel.Queuing.Interfaces; +using Foxnouns.Backend.Database; +using Foxnouns.Backend.Database.Models; +using Foxnouns.Backend.Jobs; +using Foxnouns.Backend.Middleware; +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; +using NodaTime; + +namespace Foxnouns.Backend.Controllers; + +[Route("/api/internal/data-exports")] +[Authorize("identify")] +public class ExportsController( + ILogger logger, + Config config, + IClock clock, + DatabaseContext db, + IQueue queue +) : ApiControllerBase +{ + private static readonly Duration MinimumTimeBetween = Duration.FromDays(1); + private readonly ILogger _logger = logger.ForContext(); + + [HttpGet] + public async Task GetDataExportsAsync() + { + var export = await db + .DataExports.Where(d => d.UserId == CurrentUser!.Id) + .OrderByDescending(d => d.Id) + .FirstOrDefaultAsync(); + if (export == null) + return Ok(new DataExportResponse(null, null)); + + return Ok( + new DataExportResponse( + ExportUrl(CurrentUser!.Id, export.Filename), + export.Id.Time + DataExport.Expiration + ) + ); + } + + private string ExportUrl(Snowflake userId, string filename) => + $"{config.MediaBaseUrl}/data-exports/{userId}/{filename}.zip"; + + private record DataExportResponse(string? Url, Instant? ExpiresAt); + + [HttpPost] + public async Task QueueDataExportAsync() + { + var snowflakeToCheck = Snowflake.FromInstant( + clock.GetCurrentInstant() - MinimumTimeBetween + ); + _logger.Debug( + "Checking if user {UserId} has data exports newer than {Snowflake}", + CurrentUser!.Id, + snowflakeToCheck + ); + if ( + await db.DataExports.AnyAsync(d => + d.UserId == CurrentUser.Id && d.Id > snowflakeToCheck + ) + ) + { + throw new ApiError.BadRequest("You can't request a new data export so soon."); + } + + queue.QueueInvocableWithPayload( + new CreateDataExportPayload(CurrentUser.Id) + ); + + return NoContent(); + } +} diff --git a/Foxnouns.Backend/Database/DatabaseContext.cs b/Foxnouns.Backend/Database/DatabaseContext.cs index dae8b28..b3d4d76 100644 --- a/Foxnouns.Backend/Database/DatabaseContext.cs +++ b/Foxnouns.Backend/Database/DatabaseContext.cs @@ -46,17 +46,18 @@ public class DatabaseContext(DbContextOptions options) : DbContext(options) .UseSnakeCaseNamingConvention() .UseExceptionProcessor(); - public DbSet Users { get; set; } - public DbSet Members { get; set; } - public DbSet AuthMethods { get; set; } - public DbSet FediverseApplications { get; set; } - public DbSet Tokens { get; set; } - public DbSet Applications { get; set; } - public DbSet TemporaryKeys { get; set; } + public DbSet Users { get; init; } + public DbSet Members { get; init; } + public DbSet AuthMethods { get; init; } + public DbSet FediverseApplications { get; init; } + public DbSet Tokens { get; init; } + public DbSet Applications { get; init; } + public DbSet TemporaryKeys { get; init; } + public DbSet DataExports { get; init; } - public DbSet PrideFlags { get; set; } - public DbSet UserFlags { get; set; } - public DbSet MemberFlags { get; set; } + public DbSet PrideFlags { get; init; } + public DbSet UserFlags { get; init; } + public DbSet MemberFlags { get; init; } protected override void ConfigureConventions(ModelConfigurationBuilder configurationBuilder) { @@ -81,6 +82,7 @@ public class DatabaseContext(DbContextOptions options) : DbContext(options) }) .HasFilter("fediverse_application_id IS NOT NULL") .IsUnique(); + modelBuilder.Entity().HasIndex(d => d.Filename).IsUnique(); modelBuilder .Entity() diff --git a/Foxnouns.Backend/Database/Migrations/20241202153736_AddDataExports.cs b/Foxnouns.Backend/Database/Migrations/20241202153736_AddDataExports.cs new file mode 100644 index 0000000..8a4347f --- /dev/null +++ b/Foxnouns.Backend/Database/Migrations/20241202153736_AddDataExports.cs @@ -0,0 +1,57 @@ +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Foxnouns.Backend.Database.Migrations +{ + /// + [DbContext(typeof(DatabaseContext))] + [Migration("20241202153736_AddDataExports")] + public partial class AddDataExports : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "data_exports", + columns: table => new + { + id = table.Column(type: "bigint", nullable: false), + user_id = table.Column(type: "bigint", nullable: false), + filename = table.Column(type: "text", nullable: false), + }, + constraints: table => + { + table.PrimaryKey("pk_data_exports", x => x.id); + table.ForeignKey( + name: "fk_data_exports_users_user_id", + column: x => x.user_id, + principalTable: "users", + principalColumn: "id", + onDelete: ReferentialAction.Cascade + ); + } + ); + + migrationBuilder.CreateIndex( + name: "ix_data_exports_filename", + table: "data_exports", + column: "filename", + unique: true + ); + + migrationBuilder.CreateIndex( + name: "ix_data_exports_user_id", + table: "data_exports", + column: "user_id" + ); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable(name: "data_exports"); + } + } +} diff --git a/Foxnouns.Backend/Database/Migrations/DatabaseContextModelSnapshot.cs b/Foxnouns.Backend/Database/Migrations/DatabaseContextModelSnapshot.cs index f9f1609..79a9855 100644 --- a/Foxnouns.Backend/Database/Migrations/DatabaseContextModelSnapshot.cs +++ b/Foxnouns.Backend/Database/Migrations/DatabaseContextModelSnapshot.cs @@ -111,6 +111,34 @@ namespace Foxnouns.Backend.Database.Migrations b.ToTable("auth_methods", (string)null); }); + modelBuilder.Entity("Foxnouns.Backend.Database.Models.DataExport", b => + { + b.Property("Id") + .HasColumnType("bigint") + .HasColumnName("id"); + + b.Property("Filename") + .IsRequired() + .HasColumnType("text") + .HasColumnName("filename"); + + b.Property("UserId") + .HasColumnType("bigint") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("pk_data_exports"); + + b.HasIndex("Filename") + .IsUnique() + .HasDatabaseName("ix_data_exports_filename"); + + b.HasIndex("UserId") + .HasDatabaseName("ix_data_exports_user_id"); + + b.ToTable("data_exports", (string)null); + }); + modelBuilder.Entity("Foxnouns.Backend.Database.Models.FediverseApplication", b => { b.Property("Id") @@ -515,6 +543,18 @@ namespace Foxnouns.Backend.Database.Migrations b.Navigation("User"); }); + modelBuilder.Entity("Foxnouns.Backend.Database.Models.DataExport", b => + { + b.HasOne("Foxnouns.Backend.Database.Models.User", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_data_exports_users_user_id"); + + b.Navigation("User"); + }); + modelBuilder.Entity("Foxnouns.Backend.Database.Models.Member", b => { b.HasOne("Foxnouns.Backend.Database.Models.User", "User") diff --git a/Foxnouns.Backend/Database/Models/DataExport.cs b/Foxnouns.Backend/Database/Models/DataExport.cs new file mode 100644 index 0000000..6e5a719 --- /dev/null +++ b/Foxnouns.Backend/Database/Models/DataExport.cs @@ -0,0 +1,14 @@ +using System.ComponentModel.DataAnnotations.Schema; +using NodaTime; + +namespace Foxnouns.Backend.Database.Models; + +public class DataExport : BaseModel +{ + public Snowflake UserId { get; init; } + public User User { get; init; } = null!; + public required string Filename { get; init; } + + [NotMapped] + public static readonly Duration Expiration = Duration.FromDays(15); +} diff --git a/Foxnouns.Backend/Database/Snowflake.cs b/Foxnouns.Backend/Database/Snowflake.cs index 37b3c31..1294dd4 100644 --- a/Foxnouns.Backend/Database/Snowflake.cs +++ b/Foxnouns.Backend/Database/Snowflake.cs @@ -65,6 +65,9 @@ public readonly struct Snowflake(ulong value) : IEquatable return true; } + public static Snowflake FromInstant(Instant instant) => + new((ulong)(instant.ToUnixTimeMilliseconds() - Epoch) << 22); + public override bool Equals(object? obj) => obj is Snowflake other && Value == other.Value; public bool Equals(Snowflake other) diff --git a/Foxnouns.Backend/Extensions/WebApplicationExtensions.cs b/Foxnouns.Backend/Extensions/WebApplicationExtensions.cs index da6f377..f71cbc2 100644 --- a/Foxnouns.Backend/Extensions/WebApplicationExtensions.cs +++ b/Foxnouns.Backend/Extensions/WebApplicationExtensions.cs @@ -106,12 +106,14 @@ public static class WebApplicationExtensions .AddScoped() .AddScoped() .AddScoped() + .AddTransient() // Background services .AddHostedService() // Transient jobs .AddTransient() .AddTransient() - .AddTransient(); + .AddTransient() + .AddTransient(); if (!config.Logging.EnableMetrics) services.AddHostedService(); diff --git a/Foxnouns.Backend/Jobs/CreateDataExportInvocable.cs b/Foxnouns.Backend/Jobs/CreateDataExportInvocable.cs new file mode 100644 index 0000000..2ce02ae --- /dev/null +++ b/Foxnouns.Backend/Jobs/CreateDataExportInvocable.cs @@ -0,0 +1,209 @@ +using System.IO.Compression; +using System.Net; +using Coravel.Invocable; +using Foxnouns.Backend.Database; +using Foxnouns.Backend.Database.Models; +using Foxnouns.Backend.Services; +using Foxnouns.Backend.Utils; +using Microsoft.EntityFrameworkCore; +using Newtonsoft.Json; +using NodaTime; +using NodaTime.Text; + +namespace Foxnouns.Backend.Jobs; + +public class CreateDataExportInvocable( + DatabaseContext db, + IClock clock, + UserRendererService userRenderer, + MemberRendererService memberRenderer, + ObjectStorageService objectStorageService, + ISnowflakeGenerator snowflakeGenerator, + ILogger logger +) : IInvocable, IInvocableWithPayload +{ + private static readonly HttpClient Client = new(); + private readonly ILogger _logger = logger.ForContext(); + public required CreateDataExportPayload Payload { get; set; } + + public async Task Invoke() + { + try + { + await InvokeAsync(); + } + catch (Exception e) + { + _logger.Error(e, "Error generating data export for user {UserId}", Payload.UserId); + } + } + + private async Task InvokeAsync() + { + var user = await db + .Users.Include(u => u.AuthMethods) + .Include(u => u.Flags) + .Include(u => u.ProfileFlags) + .FirstOrDefaultAsync(u => u.Id == Payload.UserId); + if (user == null) + { + _logger.Warning( + "Received create data export request for user {UserId} but no such user exists, deleted or otherwise. Ignoring request", + Payload.UserId + ); + return; + } + + _logger.Information("Generating data export for user {UserId}", user.Id); + + using var stream = new MemoryStream(); + using var zip = new ZipArchive(stream, ZipArchiveMode.Create, true); + zip.Comment = + $"This archive for {user.Username} ({user.Id}) was generated at {InstantPattern.General.Format(clock.GetCurrentInstant())}"; + + // Write the user's info and avatar + WriteJson( + zip, + "user.json", + await userRenderer.RenderUserInnerAsync( + user, + true, + ["*"], + renderMembers: false, + renderAuthMethods: true + ) + ); + await WriteS3Object(zip, "user-avatar.webp", userRenderer.AvatarUrlFor(user)); + + foreach (var flag in user.Flags) + await WritePrideFlag(zip, flag); + + var members = await db + .Members.Include(m => m.User) + .Include(m => m.ProfileFlags) + .Where(m => m.UserId == user.Id) + .ToListAsync(); + foreach (var member in members) + await WriteMember(zip, member); + + // We want to dispose the ZipArchive on an error, but we need to dispose it manually to upload to object storage. + // Calling Dispose() multiple times is fine for this class, though. + // ReSharper disable once DisposeOnUsingVariable + zip.Dispose(); + stream.Seek(0, SeekOrigin.Begin); + + // Upload the file! + var filename = AuthUtils.RandomToken().Replace('+', '-').Replace('/', '_'); + await objectStorageService.PutObjectAsync( + ExportPath(user.Id, filename), + stream, + "application/zip" + ); + + db.Add( + new DataExport + { + Id = snowflakeGenerator.GenerateSnowflake(), + UserId = user.Id, + Filename = filename, + } + ); + await db.SaveChangesAsync(); + } + + private async Task WritePrideFlag(ZipArchive zip, PrideFlag flag) + { + _logger.Debug("Writing flag {FlagId}", flag.Id); + + var flagData = $""" + {flag.Name} + ---- + {flag.Description ?? ""} + """; + + try + { + await WriteS3Object(zip, $"flag-{flag.Id}/flag.webp", userRenderer.ImageUrlFor(flag)); + } + catch (Exception e) + { + _logger.Warning(e, "Could not write image for flag {FlagId}", flag.Id); + return; + } + + var entry = zip.CreateEntry($"flag-{flag.Id}/flag.txt"); + await using var stream = entry.Open(); + await using var writer = new StreamWriter(stream); + await writer.WriteAsync(flagData); + } + + private async Task WriteMember(ZipArchive zip, Member member) + { + _logger.Debug("Writing member {MemberId}", member.Id); + + WriteJson( + zip, + $"members/{member.Name} ({member.Id}).json", + memberRenderer.RenderMember(member) + ); + + try + { + await WriteS3Object( + zip, + $"members/{member.Name} ({member.Id}).webp", + memberRenderer.AvatarUrlFor(member) + ); + } + catch (Exception e) + { + _logger.Warning(e, "Error writing avatar for member {MemberId}", member.Id); + } + } + + private void WriteJson(ZipArchive zip, string filename, object data) + { + var json = JsonConvert.SerializeObject(data, Formatting.Indented); + + _logger.Debug( + "Writing file {Filename} to archive with size {Length}", + filename, + json.Length + ); + + var entry = zip.CreateEntry(filename); + using var stream = entry.Open(); + using var writer = new StreamWriter(stream); + writer.Write(json); + } + + private async Task WriteS3Object(ZipArchive zip, string filename, string? s3Path) + { + if (s3Path == null) + return; + + var resp = await Client.GetAsync(s3Path); + if (resp.StatusCode != HttpStatusCode.OK) + { + _logger.Warning("S3 path {S3Path} returned a non-200 status, not saving file", s3Path); + return; + } + + await using var respStream = await resp.Content.ReadAsStreamAsync(); + + _logger.Debug( + "Writing file {Filename} to archive with size {Length}", + filename, + respStream.Length + ); + + var entry = zip.CreateEntry(filename); + await using var entryStream = entry.Open(); + + respStream.Seek(0, SeekOrigin.Begin); + await respStream.CopyToAsync(entryStream); + } + + private static string ExportPath(Snowflake userId, string b64) => + $"data-exports/{userId}/{b64}.zip"; +} diff --git a/Foxnouns.Backend/Jobs/Payloads.cs b/Foxnouns.Backend/Jobs/Payloads.cs index 759235c..d67f1b2 100644 --- a/Foxnouns.Backend/Jobs/Payloads.cs +++ b/Foxnouns.Backend/Jobs/Payloads.cs @@ -11,3 +11,5 @@ public record CreateFlagPayload( string ImageData, string? Description ); + +public record CreateDataExportPayload(Snowflake UserId); diff --git a/Foxnouns.Backend/Services/DataCleanupService.cs b/Foxnouns.Backend/Services/DataCleanupService.cs new file mode 100644 index 0000000..a4ca9c8 --- /dev/null +++ b/Foxnouns.Backend/Services/DataCleanupService.cs @@ -0,0 +1,47 @@ +using Foxnouns.Backend.Database; +using Foxnouns.Backend.Database.Models; +using Microsoft.EntityFrameworkCore; +using NodaTime; + +namespace Foxnouns.Backend.Services; + +public class DataCleanupService( + DatabaseContext db, + IClock clock, + ILogger logger, + ObjectStorageService objectStorageService +) +{ + private readonly ILogger _logger = logger.ForContext(); + + public async Task InvokeAsync(CancellationToken ct = default) + { + _logger.Information("Cleaning up expired data exports"); + await CleanExportsAsync(ct); + } + + private async Task CleanExportsAsync(CancellationToken ct = default) + { + var minExpiredId = Snowflake.FromInstant(clock.GetCurrentInstant() - DataExport.Expiration); + var exports = await db.DataExports.Where(d => d.Id < minExpiredId).ToListAsync(ct); + if (exports.Count == 0) + return; + + _logger.Debug("There are {Count} expired exports", exports.Count); + + foreach (var export in exports) + { + _logger.Debug("Deleting export {ExportId}", export.Id); + await objectStorageService.RemoveObjectAsync( + ExportPath(export.UserId, export.Filename), + ct + ); + db.Remove(export); + } + + await db.SaveChangesAsync(ct); + } + + private static string ExportPath(Snowflake userId, string b64) => + $"data-exports/{userId}/{b64}.zip"; +} diff --git a/Foxnouns.Backend/Services/MemberRendererService.cs b/Foxnouns.Backend/Services/MemberRendererService.cs index 717f06c..336d189 100644 --- a/Foxnouns.Backend/Services/MemberRendererService.cs +++ b/Foxnouns.Backend/Services/MemberRendererService.cs @@ -72,7 +72,7 @@ public class MemberRendererService(DatabaseContext db, Config config) renderUnlisted ? member.Unlisted : null ); - private string? AvatarUrlFor(Member member) => + public string? AvatarUrlFor(Member member) => member.Avatar != null ? $"{config.MediaBaseUrl}/members/{member.Id}/avatars/{member.Avatar}.webp" : null; diff --git a/Foxnouns.Backend/Services/MetricsCollectionService.cs b/Foxnouns.Backend/Services/MetricsCollectionService.cs index 573c1a2..66172a6 100644 --- a/Foxnouns.Backend/Services/MetricsCollectionService.cs +++ b/Foxnouns.Backend/Services/MetricsCollectionService.cs @@ -62,7 +62,7 @@ public class BackgroundMetricsCollectionService( using var timer = new PeriodicTimer(TimeSpan.FromMinutes(1)); while (await timer.WaitForNextTickAsync(ct)) { - _logger.Debug("Collecting metrics"); + _logger.Debug("Collecting metrics manually"); await metricsCollectionService.CollectMetricsAsync(ct); } } diff --git a/Foxnouns.Backend/Services/ObjectStorageService.cs b/Foxnouns.Backend/Services/ObjectStorageService.cs index df9709c..08d3f9b 100644 --- a/Foxnouns.Backend/Services/ObjectStorageService.cs +++ b/Foxnouns.Backend/Services/ObjectStorageService.cs @@ -48,4 +48,13 @@ public class ObjectStorageService(ILogger logger, Config config, IMinioClient mi ct ); } + + public async Task GetObjectAsync(string path, CancellationToken ct = default) + { + var stream = new MemoryStream(); + var resp = await minioClient.GetObjectAsync( + new GetObjectArgs().WithBucket(config.Storage.Bucket).WithObject(path), + ct + ); + } } diff --git a/Foxnouns.Backend/Services/PeriodicTasksService.cs b/Foxnouns.Backend/Services/PeriodicTasksService.cs index 757ebcf..1e0e756 100644 --- a/Foxnouns.Backend/Services/PeriodicTasksService.cs +++ b/Foxnouns.Backend/Services/PeriodicTasksService.cs @@ -8,10 +8,7 @@ public class PeriodicTasksService(ILogger logger, IServiceProvider services) : B { using var timer = new PeriodicTimer(TimeSpan.FromMinutes(1)); while (await timer.WaitForNextTickAsync(ct)) - { - _logger.Debug("Collecting metrics"); await RunPeriodicTasksAsync(ct); - } } private async Task RunPeriodicTasksAsync(CancellationToken ct) @@ -20,7 +17,10 @@ public class PeriodicTasksService(ILogger logger, IServiceProvider services) : B await using var scope = services.CreateAsyncScope(); - var keyCacheSvc = scope.ServiceProvider.GetRequiredService(); - await keyCacheSvc.DeleteExpiredKeysAsync(ct); + var keyCacheService = scope.ServiceProvider.GetRequiredService(); + var dataCleanupService = scope.ServiceProvider.GetRequiredService(); + + await keyCacheService.DeleteExpiredKeysAsync(ct); + await dataCleanupService.InvokeAsync(ct); } } diff --git a/Foxnouns.Backend/Services/UserRendererService.cs b/Foxnouns.Backend/Services/UserRendererService.cs index 2911dd3..f073cda 100644 --- a/Foxnouns.Backend/Services/UserRendererService.cs +++ b/Foxnouns.Backend/Services/UserRendererService.cs @@ -22,12 +22,31 @@ public class UserRendererService( bool renderAuthMethods = false, string? overrideSid = null, CancellationToken ct = default + ) => + await RenderUserInnerAsync( + user, + selfUser != null && selfUser.Id == user.Id, + token?.Scopes ?? [], + renderMembers, + renderAuthMethods, + overrideSid, + ct + ); + + public async Task RenderUserInnerAsync( + User user, + bool isSelfUser, + string[] scopes, + bool renderMembers = true, + bool renderAuthMethods = false, + string? overrideSid = null, + CancellationToken ct = default ) { - var isSelfUser = selfUser?.Id == user.Id; - var tokenCanReadHiddenMembers = token.HasScope("member.read") && isSelfUser; - var tokenHidden = token.HasScope("user.read_hidden") && isSelfUser; - var tokenPrivileged = token.HasScope("user.read_privileged") && isSelfUser; + scopes = scopes.ExpandScopes(); + var tokenCanReadHiddenMembers = scopes.Contains("member.read") && isSelfUser; + var tokenHidden = scopes.Contains("user.read_hidden") && isSelfUser; + var tokenPrivileged = scopes.Contains("user.read_privileged") && isSelfUser; renderMembers = renderMembers && (!user.ListHidden || tokenCanReadHiddenMembers); renderAuthMethods = renderAuthMethods && tokenPrivileged; @@ -105,12 +124,12 @@ public class UserRendererService( user.CustomPreferences ); - private string? AvatarUrlFor(User user) => + public string? AvatarUrlFor(User user) => user.Avatar != null ? $"{config.MediaBaseUrl}/users/{user.Id}/avatars/{user.Avatar}.webp" : null; - private string ImageUrlFor(PrideFlag flag) => $"{config.MediaBaseUrl}/flags/{flag.Hash}.webp"; + public string ImageUrlFor(PrideFlag flag) => $"{config.MediaBaseUrl}/flags/{flag.Hash}.webp"; public record UserResponse( Snowflake Id,