sam/Foxnouns.NET
sam/Foxnouns.NET
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

feat(backend): initial data export support

Browse source 
obviously it's missing things that haven't been added yet
This commit is contained in:
sam 2024-12-02 18:06:19 +01:00
parent f0ae648492
commit 903be2709c
Signed by: sam
GPG key ID: B4EF20DDE721CAA1
15 changed files with 502 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

74
Foxnouns.Backend/Controllers/ExportsController.cs Normal file
View file

@ -0,0 +1,74 @@
using Coravel.Queuing.Interfaces;
using Foxnouns.Backend.Database;
using Foxnouns.Backend.Database.Models;
using Foxnouns.Backend.Jobs;
using Foxnouns.Backend.Middleware;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
using NodaTime;
namespace Foxnouns.Backend.Controllers;
[Route("/api/internal/data-exports")]
[Authorize("identify")]
public class ExportsController(
ILogger logger,
Config config,
IClock clock,
DatabaseContext db,
IQueue queue
) : ApiControllerBase
{
private static readonly Duration MinimumTimeBetween = Duration.FromDays(1);
private readonly ILogger _logger = logger.ForContext<ExportsController>();
[HttpGet]
public async Task<IActionResult> GetDataExportsAsync()
{
var export = await db
.DataExports.Where(d => d.UserId == CurrentUser!.Id)
.OrderByDescending(d => d.Id)
.FirstOrDefaultAsync();
if (export == null)
return Ok(new DataExportResponse(null, null));
return Ok(
new DataExportResponse(
ExportUrl(CurrentUser!.Id, export.Filename),
export.Id.Time + DataExport.Expiration
)
);
}
private string ExportUrl(Snowflake userId, string filename) =>
$"{config.MediaBaseUrl}/data-exports/{userId}/{filename}.zip";
private record DataExportResponse(string? Url, Instant? ExpiresAt);
[HttpPost]
public async Task<IActionResult> QueueDataExportAsync()
{
var snowflakeToCheck = Snowflake.FromInstant(
clock.GetCurrentInstant() - MinimumTimeBetween
);
_logger.Debug(
"Checking if user {UserId} has data exports newer than {Snowflake}",
CurrentUser!.Id,
snowflakeToCheck
);
if (
await db.DataExports.AnyAsync(d =>
d.UserId == CurrentUser.Id && d.Id > snowflakeToCheck
)
)
{
throw new ApiError.BadRequest("You can't request a new data export so soon.");
}
queue.QueueInvocableWithPayload<CreateDataExportInvocable, CreateDataExportPayload>(
new CreateDataExportPayload(CurrentUser.Id)
);
return NoContent();
}
}

22
Foxnouns.Backend/Database/DatabaseContext.cs
View file

@ -46,17 +46,18 @@ public class DatabaseContext(DbContextOptions options) : DbContext(options)
.UseSnakeCaseNamingConvention()
.UseExceptionProcessor();
public DbSet<User> Users { get; set; }
public DbSet<Member> Members { get; set; }
public DbSet<AuthMethod> AuthMethods { get; set; }
public DbSet<FediverseApplication> FediverseApplications { get; set; }
public DbSet<Token> Tokens { get; set; }
public DbSet<Application> Applications { get; set; }
public DbSet<TemporaryKey> TemporaryKeys { get; set; }
public DbSet<User> Users { get; init; }
public DbSet<Member> Members { get; init; }
public DbSet<AuthMethod> AuthMethods { get; init; }
public DbSet<FediverseApplication> FediverseApplications { get; init; }
public DbSet<Token> Tokens { get; init; }
public DbSet<Application> Applications { get; init; }
public DbSet<TemporaryKey> TemporaryKeys { get; init; }
public DbSet<DataExport> DataExports { get; init; }
public DbSet<PrideFlag> PrideFlags { get; set; }
public DbSet<UserFlag> UserFlags { get; set; }
public DbSet<MemberFlag> MemberFlags { get; set; }
public DbSet<PrideFlag> PrideFlags { get; init; }
public DbSet<UserFlag> UserFlags { get; init; }
public DbSet<MemberFlag> MemberFlags { get; init; }
protected override void ConfigureConventions(ModelConfigurationBuilder configurationBuilder)
{
@ -81,6 +82,7 @@ public class DatabaseContext(DbContextOptions options) : DbContext(options)
})
.HasFilter("fediverse_application_id IS NOT NULL")
.IsUnique();
modelBuilder.Entity<DataExport>().HasIndex(d => d.Filename).IsUnique();
modelBuilder
.Entity<AuthMethod>()

57
Foxnouns.Backend/Database/Migrations/20241202153736_AddDataExports.cs Normal file
View file

@ -0,0 +1,57 @@
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Foxnouns.Backend.Database.Migrations
{
/// <inheritdoc />
[DbContext(typeof(DatabaseContext))]
[Migration("20241202153736_AddDataExports")]
public partial class AddDataExports : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "data_exports",
columns: table => new
{
id = table.Column<long>(type: "bigint", nullable: false),
user_id = table.Column<long>(type: "bigint", nullable: false),
filename = table.Column<string>(type: "text", nullable: false),
},
constraints: table =>
{
table.PrimaryKey("pk_data_exports", x => x.id);
table.ForeignKey(
name: "fk_data_exports_users_user_id",
column: x => x.user_id,
principalTable: "users",
principalColumn: "id",
onDelete: ReferentialAction.Cascade
);
}
);
migrationBuilder.CreateIndex(
name: "ix_data_exports_filename",
table: "data_exports",
column: "filename",
unique: true
);
migrationBuilder.CreateIndex(
name: "ix_data_exports_user_id",
table: "data_exports",
column: "user_id"
);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(name: "data_exports");
}
}
}

40
Foxnouns.Backend/Database/Migrations/DatabaseContextModelSnapshot.cs
View file

@ -111,6 +111,34 @@ namespace Foxnouns.Backend.Database.Migrations
b.ToTable("auth_methods", (string)null);
});
modelBuilder.Entity("Foxnouns.Backend.Database.Models.DataExport", b =>
{
b.Property<long>("Id")
.HasColumnType("bigint")
.HasColumnName("id");
b.Property<string>("Filename")
.IsRequired()
.HasColumnType("text")
.HasColumnName("filename");
b.Property<long>("UserId")
.HasColumnType("bigint")
.HasColumnName("user_id");
b.HasKey("Id")
.HasName("pk_data_exports");
b.HasIndex("Filename")
.IsUnique()
.HasDatabaseName("ix_data_exports_filename");
b.HasIndex("UserId")
.HasDatabaseName("ix_data_exports_user_id");
b.ToTable("data_exports", (string)null);
});
modelBuilder.Entity("Foxnouns.Backend.Database.Models.FediverseApplication", b =>
{
b.Property<long>("Id")
@ -515,6 +543,18 @@ namespace Foxnouns.Backend.Database.Migrations
b.Navigation("User");
});
modelBuilder.Entity("Foxnouns.Backend.Database.Models.DataExport", b =>
{
b.HasOne("Foxnouns.Backend.Database.Models.User", "User")
.WithMany()
.HasForeignKey("UserId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_data_exports_users_user_id");
b.Navigation("User");
});
modelBuilder.Entity("Foxnouns.Backend.Database.Models.Member", b =>
{
b.HasOne("Foxnouns.Backend.Database.Models.User", "User")

14
Foxnouns.Backend/Database/Models/DataExport.cs Normal file
View file

@ -0,0 +1,14 @@
using System.ComponentModel.DataAnnotations.Schema;
using NodaTime;
namespace Foxnouns.Backend.Database.Models;
public class DataExport : BaseModel
{
public Snowflake UserId { get; init; }
public User User { get; init; } = null!;
public required string Filename { get; init; }
[NotMapped]
public static readonly Duration Expiration = Duration.FromDays(15);
}

3
Foxnouns.Backend/Database/Snowflake.cs
View file

@ -65,6 +65,9 @@ public readonly struct Snowflake(ulong value) : IEquatable<Snowflake>
return true;
}
public static Snowflake FromInstant(Instant instant) =>
new((ulong)(instant.ToUnixTimeMilliseconds() - Epoch) << 22);
public override bool Equals(object? obj) => obj is Snowflake other && Value == other.Value;
public bool Equals(Snowflake other)

4
Foxnouns.Backend/Extensions/WebApplicationExtensions.cs
View file

@ -106,12 +106,14 @@ public static class WebApplicationExtensions
.AddScoped<RemoteAuthService>()
.AddScoped<FediverseAuthService>()
.AddScoped<ObjectStorageService>()
.AddTransient<DataCleanupService>()
// Background services
.AddHostedService<PeriodicTasksService>()
// Transient jobs
.AddTransient<MemberAvatarUpdateInvocable>()
.AddTransient<UserAvatarUpdateInvocable>()
.AddTransient<CreateFlagInvocable>();
.AddTransient<CreateFlagInvocable>()
.AddTransient<CreateDataExportInvocable>();
if (!config.Logging.EnableMetrics)
services.AddHostedService<BackgroundMetricsCollectionService>();

209
Foxnouns.Backend/Jobs/CreateDataExportInvocable.cs Normal file
View file

@ -0,0 +1,209 @@
using System.IO.Compression;
using System.Net;
using Coravel.Invocable;
using Foxnouns.Backend.Database;
using Foxnouns.Backend.Database.Models;
using Foxnouns.Backend.Services;
using Foxnouns.Backend.Utils;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
using NodaTime;
using NodaTime.Text;
namespace Foxnouns.Backend.Jobs;
public class CreateDataExportInvocable(
DatabaseContext db,
IClock clock,
UserRendererService userRenderer,
MemberRendererService memberRenderer,
ObjectStorageService objectStorageService,
ISnowflakeGenerator snowflakeGenerator,
ILogger logger
) : IInvocable, IInvocableWithPayload<CreateDataExportPayload>
{
private static readonly HttpClient Client = new();
private readonly ILogger _logger = logger.ForContext<CreateDataExportInvocable>();
public required CreateDataExportPayload Payload { get; set; }
public async Task Invoke()
{
try
{
await InvokeAsync();
}
catch (Exception e)
{
_logger.Error(e, "Error generating data export for user {UserId}", Payload.UserId);
}
}
private async Task InvokeAsync()
{
var user = await db
.Users.Include(u => u.AuthMethods)
.Include(u => u.Flags)
.Include(u => u.ProfileFlags)
.FirstOrDefaultAsync(u => u.Id == Payload.UserId);
if (user == null)
{
_logger.Warning(
"Received create data export request for user {UserId} but no such user exists, deleted or otherwise. Ignoring request",
Payload.UserId
);
return;
}
_logger.Information("Generating data export for user {UserId}", user.Id);
using var stream = new MemoryStream();
using var zip = new ZipArchive(stream, ZipArchiveMode.Create, true);
zip.Comment =
$"This archive for {user.Username} ({user.Id}) was generated at {InstantPattern.General.Format(clock.GetCurrentInstant())}";
// Write the user's info and avatar
WriteJson(
zip,
"user.json",
await userRenderer.RenderUserInnerAsync(
user,
true,
["*"],
renderMembers: false,
renderAuthMethods: true
)
);
await WriteS3Object(zip, "user-avatar.webp", userRenderer.AvatarUrlFor(user));
foreach (var flag in user.Flags)
await WritePrideFlag(zip, flag);
var members = await db
.Members.Include(m => m.User)
.Include(m => m.ProfileFlags)
.Where(m => m.UserId == user.Id)
.ToListAsync();
foreach (var member in members)
await WriteMember(zip, member);
// We want to dispose the ZipArchive on an error, but we need to dispose it manually to upload to object storage.
// Calling Dispose() multiple times is fine for this class, though.
// ReSharper disable once DisposeOnUsingVariable
zip.Dispose();
stream.Seek(0, SeekOrigin.Begin);
// Upload the file!
var filename = AuthUtils.RandomToken().Replace('+', '-').Replace('/', '_');
await objectStorageService.PutObjectAsync(
ExportPath(user.Id, filename),
stream,
"application/zip"
);
db.Add(
new DataExport
{
Id = snowflakeGenerator.GenerateSnowflake(),
UserId = user.Id,
Filename = filename,
}
);
await db.SaveChangesAsync();
}
private async Task WritePrideFlag(ZipArchive zip, PrideFlag flag)
{
_logger.Debug("Writing flag {FlagId}", flag.Id);
var flagData = $"""
{flag.Name}
----
{flag.Description ?? "<no description>"}
""";
try
{
await WriteS3Object(zip, $"flag-{flag.Id}/flag.webp", userRenderer.ImageUrlFor(flag));
}
catch (Exception e)
{
_logger.Warning(e, "Could not write image for flag {FlagId}", flag.Id);
return;
}
var entry = zip.CreateEntry($"flag-{flag.Id}/flag.txt");
await using var stream = entry.Open();
await using var writer = new StreamWriter(stream);
await writer.WriteAsync(flagData);
}
private async Task WriteMember(ZipArchive zip, Member member)
{
_logger.Debug("Writing member {MemberId}", member.Id);
WriteJson(
zip,
$"members/{member.Name} ({member.Id}).json",
memberRenderer.RenderMember(member)
);
try
{
await WriteS3Object(
zip,
$"members/{member.Name} ({member.Id}).webp",
memberRenderer.AvatarUrlFor(member)
);
}
catch (Exception e)
{
_logger.Warning(e, "Error writing avatar for member {MemberId}", member.Id);
}
}
private void WriteJson(ZipArchive zip, string filename, object data)
{
var json = JsonConvert.SerializeObject(data, Formatting.Indented);
_logger.Debug(
"Writing file {Filename} to archive with size {Length}",
filename,
json.Length
);
var entry = zip.CreateEntry(filename);
using var stream = entry.Open();
using var writer = new StreamWriter(stream);
writer.Write(json);
}
private async Task WriteS3Object(ZipArchive zip, string filename, string? s3Path)
{
if (s3Path == null)
return;
var resp = await Client.GetAsync(s3Path);
if (resp.StatusCode != HttpStatusCode.OK)
{
_logger.Warning("S3 path {S3Path} returned a non-200 status, not saving file", s3Path);
return;
}
await using var respStream = await resp.Content.ReadAsStreamAsync();
_logger.Debug(
"Writing file {Filename} to archive with size {Length}",
filename,
respStream.Length
);
var entry = zip.CreateEntry(filename);
await using var entryStream = entry.Open();
respStream.Seek(0, SeekOrigin.Begin);
await respStream.CopyToAsync(entryStream);
}
private static string ExportPath(Snowflake userId, string b64) =>
$"data-exports/{userId}/{b64}.zip";
}

2
Foxnouns.Backend/Jobs/Payloads.cs
View file

@ -11,3 +11,5 @@ public record CreateFlagPayload(
string ImageData,
string? Description
);
public record CreateDataExportPayload(Snowflake UserId);

47
Foxnouns.Backend/Services/DataCleanupService.cs Normal file
View file

@ -0,0 +1,47 @@
using Foxnouns.Backend.Database;
using Foxnouns.Backend.Database.Models;
using Microsoft.EntityFrameworkCore;
using NodaTime;
namespace Foxnouns.Backend.Services;
public class DataCleanupService(
DatabaseContext db,
IClock clock,
ILogger logger,
ObjectStorageService objectStorageService
)
{
private readonly ILogger _logger = logger.ForContext<DataCleanupService>();
public async Task InvokeAsync(CancellationToken ct = default)
{
_logger.Information("Cleaning up expired data exports");
await CleanExportsAsync(ct);
}
private async Task CleanExportsAsync(CancellationToken ct = default)
{
var minExpiredId = Snowflake.FromInstant(clock.GetCurrentInstant() - DataExport.Expiration);
var exports = await db.DataExports.Where(d => d.Id < minExpiredId).ToListAsync(ct);
if (exports.Count == 0)
return;
_logger.Debug("There are {Count} expired exports", exports.Count);
foreach (var export in exports)
{
_logger.Debug("Deleting export {ExportId}", export.Id);
await objectStorageService.RemoveObjectAsync(
ExportPath(export.UserId, export.Filename),
ct
);
db.Remove(export);
}
await db.SaveChangesAsync(ct);
}
private static string ExportPath(Snowflake userId, string b64) =>
$"data-exports/{userId}/{b64}.zip";
}

2
Foxnouns.Backend/Services/MemberRendererService.cs
View file

@ -72,7 +72,7 @@ public class MemberRendererService(DatabaseContext db, Config config)
renderUnlisted ? member.Unlisted : null
);
private string? AvatarUrlFor(Member member) =>
public string? AvatarUrlFor(Member member) =>
member.Avatar != null
? $"{config.MediaBaseUrl}/members/{member.Id}/avatars/{member.Avatar}.webp"
: null;

2
Foxnouns.Backend/Services/MetricsCollectionService.cs
View file

@ -62,7 +62,7 @@ public class BackgroundMetricsCollectionService(
using var timer = new PeriodicTimer(TimeSpan.FromMinutes(1));
while (await timer.WaitForNextTickAsync(ct))
{
_logger.Debug("Collecting metrics");
_logger.Debug("Collecting metrics manually");
await metricsCollectionService.CollectMetricsAsync(ct);
}
}

9
Foxnouns.Backend/Services/ObjectStorageService.cs
View file

@ -48,4 +48,13 @@ public class ObjectStorageService(ILogger logger, Config config, IMinioClient mi
ct
);
}
public async Task GetObjectAsync(string path, CancellationToken ct = default)
{
var stream = new MemoryStream();
var resp = await minioClient.GetObjectAsync(
new GetObjectArgs().WithBucket(config.Storage.Bucket).WithObject(path),
ct
);
}
}

10
Foxnouns.Backend/Services/PeriodicTasksService.cs
View file

@ -8,11 +8,8 @@ public class PeriodicTasksService(ILogger logger, IServiceProvider services) : B
{
using var timer = new PeriodicTimer(TimeSpan.FromMinutes(1));
while (await timer.WaitForNextTickAsync(ct))
{
_logger.Debug("Collecting metrics");
await RunPeriodicTasksAsync(ct);
}
}
private async Task RunPeriodicTasksAsync(CancellationToken ct)
{
@ -20,7 +17,10 @@ public class PeriodicTasksService(ILogger logger, IServiceProvider services) : B
await using var scope = services.CreateAsyncScope();
var keyCacheSvc = scope.ServiceProvider.GetRequiredService<KeyCacheService>();
await keyCacheSvc.DeleteExpiredKeysAsync(ct);
var keyCacheService = scope.ServiceProvider.GetRequiredService<KeyCacheService>();
var dataCleanupService = scope.ServiceProvider.GetRequiredService<DataCleanupService>();
await keyCacheService.DeleteExpiredKeysAsync(ct);
await dataCleanupService.InvokeAsync(ct);
}
}

31
Foxnouns.Backend/Services/UserRendererService.cs
View file

@ -22,12 +22,31 @@ public class UserRendererService(
bool renderAuthMethods = false,
string? overrideSid = null,
CancellationToken ct = default
) =>
await RenderUserInnerAsync(
user,
selfUser != null && selfUser.Id == user.Id,
token?.Scopes ?? [],
renderMembers,
renderAuthMethods,
overrideSid,
ct
);
public async Task<UserResponse> RenderUserInnerAsync(
User user,
bool isSelfUser,
string[] scopes,
bool renderMembers = true,
bool renderAuthMethods = false,
string? overrideSid = null,
CancellationToken ct = default
)
{
var isSelfUser = selfUser?.Id == user.Id;
var tokenCanReadHiddenMembers = token.HasScope("member.read") && isSelfUser;
var tokenHidden = token.HasScope("user.read_hidden") && isSelfUser;
var tokenPrivileged = token.HasScope("user.read_privileged") && isSelfUser;
scopes = scopes.ExpandScopes();
var tokenCanReadHiddenMembers = scopes.Contains("member.read") && isSelfUser;
var tokenHidden = scopes.Contains("user.read_hidden") && isSelfUser;
var tokenPrivileged = scopes.Contains("user.read_privileged") && isSelfUser;
renderMembers = renderMembers && (!user.ListHidden || tokenCanReadHiddenMembers);
renderAuthMethods = renderAuthMethods && tokenPrivileged;
@ -105,12 +124,12 @@ public class UserRendererService(
user.CustomPreferences
);
private string? AvatarUrlFor(User user) =>
public string? AvatarUrlFor(User user) =>
user.Avatar != null
? $"{config.MediaBaseUrl}/users/{user.Id}/avatars/{user.Avatar}.webp"
: null;
private string ImageUrlFor(PrideFlag flag) => $"{config.MediaBaseUrl}/flags/{flag.Hash}.webp";
public string ImageUrlFor(PrideFlag flag) => $"{config.MediaBaseUrl}/flags/{flag.Hash}.webp";
public record UserResponse(
Snowflake Id,