Compare commits

..

3 commits

Author SHA1 Message Date
sam
71d3b42330
fix(frontend): don't throw a 500 error if a user or member doesn't exist 2024-12-03 14:55:41 +01:00
sam
18bdbc0745
feat(backend): clean deleted users 2024-12-03 14:55:19 +01:00
sam
903be2709c
feat(backend): initial data export support
obviously it's missing things that haven't been added yet
2024-12-02 18:06:19 +01:00
18 changed files with 617 additions and 38 deletions

View file

@ -0,0 +1,74 @@
using Coravel.Queuing.Interfaces;
using Foxnouns.Backend.Database;
using Foxnouns.Backend.Database.Models;
using Foxnouns.Backend.Jobs;
using Foxnouns.Backend.Middleware;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
using NodaTime;
namespace Foxnouns.Backend.Controllers;
[Route("/api/internal/data-exports")]
[Authorize("identify")]
public class ExportsController(
ILogger logger,
Config config,
IClock clock,
DatabaseContext db,
IQueue queue
) : ApiControllerBase
{
private static readonly Duration MinimumTimeBetween = Duration.FromDays(1);
private readonly ILogger _logger = logger.ForContext<ExportsController>();
[HttpGet]
public async Task<IActionResult> GetDataExportsAsync()
{
var export = await db
.DataExports.Where(d => d.UserId == CurrentUser!.Id)
.OrderByDescending(d => d.Id)
.FirstOrDefaultAsync();
if (export == null)
return Ok(new DataExportResponse(null, null));
return Ok(
new DataExportResponse(
ExportUrl(CurrentUser!.Id, export.Filename),
export.Id.Time + DataExport.Expiration
)
);
}
private string ExportUrl(Snowflake userId, string filename) =>
$"{config.MediaBaseUrl}/data-exports/{userId}/{filename}.zip";
private record DataExportResponse(string? Url, Instant? ExpiresAt);
[HttpPost]
public async Task<IActionResult> QueueDataExportAsync()
{
var snowflakeToCheck = Snowflake.FromInstant(
clock.GetCurrentInstant() - MinimumTimeBetween
);
_logger.Debug(
"Checking if user {UserId} has data exports newer than {Snowflake}",
CurrentUser!.Id,
snowflakeToCheck
);
if (
await db.DataExports.AnyAsync(d =>
d.UserId == CurrentUser.Id && d.Id > snowflakeToCheck
)
)
{
throw new ApiError.BadRequest("You can't request a new data export so soon.");
}
queue.QueueInvocableWithPayload<CreateDataExportInvocable, CreateDataExportPayload>(
new CreateDataExportPayload(CurrentUser.Id)
);
return NoContent();
}
}

View file

@ -46,17 +46,18 @@ public class DatabaseContext(DbContextOptions options) : DbContext(options)
.UseSnakeCaseNamingConvention()
.UseExceptionProcessor();
public DbSet<User> Users { get; set; }
public DbSet<Member> Members { get; set; }
public DbSet<AuthMethod> AuthMethods { get; set; }
public DbSet<FediverseApplication> FediverseApplications { get; set; }
public DbSet<Token> Tokens { get; set; }
public DbSet<Application> Applications { get; set; }
public DbSet<TemporaryKey> TemporaryKeys { get; set; }
public DbSet<User> Users { get; init; }
public DbSet<Member> Members { get; init; }
public DbSet<AuthMethod> AuthMethods { get; init; }
public DbSet<FediverseApplication> FediverseApplications { get; init; }
public DbSet<Token> Tokens { get; init; }
public DbSet<Application> Applications { get; init; }
public DbSet<TemporaryKey> TemporaryKeys { get; init; }
public DbSet<DataExport> DataExports { get; init; }
public DbSet<PrideFlag> PrideFlags { get; set; }
public DbSet<UserFlag> UserFlags { get; set; }
public DbSet<MemberFlag> MemberFlags { get; set; }
public DbSet<PrideFlag> PrideFlags { get; init; }
public DbSet<UserFlag> UserFlags { get; init; }
public DbSet<MemberFlag> MemberFlags { get; init; }
protected override void ConfigureConventions(ModelConfigurationBuilder configurationBuilder)
{
@ -81,6 +82,7 @@ public class DatabaseContext(DbContextOptions options) : DbContext(options)
})
.HasFilter("fediverse_application_id IS NOT NULL")
.IsUnique();
modelBuilder.Entity<DataExport>().HasIndex(d => d.Filename).IsUnique();
modelBuilder
.Entity<AuthMethod>()

View file

@ -0,0 +1,57 @@
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Foxnouns.Backend.Database.Migrations
{
/// <inheritdoc />
[DbContext(typeof(DatabaseContext))]
[Migration("20241202153736_AddDataExports")]
public partial class AddDataExports : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "data_exports",
columns: table => new
{
id = table.Column<long>(type: "bigint", nullable: false),
user_id = table.Column<long>(type: "bigint", nullable: false),
filename = table.Column<string>(type: "text", nullable: false),
},
constraints: table =>
{
table.PrimaryKey("pk_data_exports", x => x.id);
table.ForeignKey(
name: "fk_data_exports_users_user_id",
column: x => x.user_id,
principalTable: "users",
principalColumn: "id",
onDelete: ReferentialAction.Cascade
);
}
);
migrationBuilder.CreateIndex(
name: "ix_data_exports_filename",
table: "data_exports",
column: "filename",
unique: true
);
migrationBuilder.CreateIndex(
name: "ix_data_exports_user_id",
table: "data_exports",
column: "user_id"
);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(name: "data_exports");
}
}
}

View file

@ -111,6 +111,34 @@ namespace Foxnouns.Backend.Database.Migrations
b.ToTable("auth_methods", (string)null);
});
modelBuilder.Entity("Foxnouns.Backend.Database.Models.DataExport", b =>
{
b.Property<long>("Id")
.HasColumnType("bigint")
.HasColumnName("id");
b.Property<string>("Filename")
.IsRequired()
.HasColumnType("text")
.HasColumnName("filename");
b.Property<long>("UserId")
.HasColumnType("bigint")
.HasColumnName("user_id");
b.HasKey("Id")
.HasName("pk_data_exports");
b.HasIndex("Filename")
.IsUnique()
.HasDatabaseName("ix_data_exports_filename");
b.HasIndex("UserId")
.HasDatabaseName("ix_data_exports_user_id");
b.ToTable("data_exports", (string)null);
});
modelBuilder.Entity("Foxnouns.Backend.Database.Models.FediverseApplication", b =>
{
b.Property<long>("Id")
@ -515,6 +543,18 @@ namespace Foxnouns.Backend.Database.Migrations
b.Navigation("User");
});
modelBuilder.Entity("Foxnouns.Backend.Database.Models.DataExport", b =>
{
b.HasOne("Foxnouns.Backend.Database.Models.User", "User")
.WithMany()
.HasForeignKey("UserId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_data_exports_users_user_id");
b.Navigation("User");
});
modelBuilder.Entity("Foxnouns.Backend.Database.Models.Member", b =>
{
b.HasOne("Foxnouns.Backend.Database.Models.User", "User")

View file

@ -0,0 +1,14 @@
using System.ComponentModel.DataAnnotations.Schema;
using NodaTime;
namespace Foxnouns.Backend.Database.Models;
public class DataExport : BaseModel
{
public Snowflake UserId { get; init; }
public User User { get; init; } = null!;
public required string Filename { get; init; }
[NotMapped]
public static readonly Duration Expiration = Duration.FromDays(15);
}

View file

@ -30,6 +30,7 @@ public class User : BaseModel
public List<Member> Members { get; } = [];
public List<AuthMethod> AuthMethods { get; } = [];
public List<DataExport> DataExports { get; } = [];
public UserSettings Settings { get; set; } = new();
public required Instant LastActive { get; set; }
@ -53,6 +54,12 @@ public class User : BaseModel
[JsonConverter(typeof(ScreamingSnakeCaseEnumConverter))]
public PreferenceSize Size { get; set; }
}
[NotMapped]
public static readonly Duration DeleteAfter = Duration.FromDays(30);
[NotMapped]
public static readonly Duration DeleteSuspendedAfter = Duration.FromDays(180);
}
public enum UserRole

View file

@ -65,6 +65,9 @@ public readonly struct Snowflake(ulong value) : IEquatable<Snowflake>
return true;
}
public static Snowflake FromInstant(Instant instant) =>
new((ulong)(instant.ToUnixTimeMilliseconds() - Epoch) << 22);
public override bool Equals(object? obj) => obj is Snowflake other && Value == other.Value;
public bool Equals(Snowflake other)

View file

@ -106,12 +106,14 @@ public static class WebApplicationExtensions
.AddScoped<RemoteAuthService>()
.AddScoped<FediverseAuthService>()
.AddScoped<ObjectStorageService>()
.AddTransient<DataCleanupService>()
// Background services
.AddHostedService<PeriodicTasksService>()
// Transient jobs
.AddTransient<MemberAvatarUpdateInvocable>()
.AddTransient<UserAvatarUpdateInvocable>()
.AddTransient<CreateFlagInvocable>();
.AddTransient<CreateFlagInvocable>()
.AddTransient<CreateDataExportInvocable>();
if (!config.Logging.EnableMetrics)
services.AddHostedService<BackgroundMetricsCollectionService>();

View file

@ -0,0 +1,210 @@
using System.IO.Compression;
using System.Net;
using Coravel.Invocable;
using Foxnouns.Backend.Database;
using Foxnouns.Backend.Database.Models;
using Foxnouns.Backend.Services;
using Foxnouns.Backend.Utils;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
using NodaTime;
using NodaTime.Text;
namespace Foxnouns.Backend.Jobs;
public class CreateDataExportInvocable(
DatabaseContext db,
IClock clock,
UserRendererService userRenderer,
MemberRendererService memberRenderer,
ObjectStorageService objectStorageService,
ISnowflakeGenerator snowflakeGenerator,
ILogger logger
) : IInvocable, IInvocableWithPayload<CreateDataExportPayload>
{
private static readonly HttpClient Client = new();
private readonly ILogger _logger = logger.ForContext<CreateDataExportInvocable>();
public required CreateDataExportPayload Payload { get; set; }
public async Task Invoke()
{
try
{
await InvokeAsync();
}
catch (Exception e)
{
_logger.Error(e, "Error generating data export for user {UserId}", Payload.UserId);
}
}
private async Task InvokeAsync()
{
var user = await db
.Users.Include(u => u.AuthMethods)
.Include(u => u.Flags)
.Include(u => u.ProfileFlags)
.AsSplitQuery()
.FirstOrDefaultAsync(u => u.Id == Payload.UserId);
if (user == null)
{
_logger.Warning(
"Received create data export request for user {UserId} but no such user exists, deleted or otherwise. Ignoring request",
Payload.UserId
);
return;
}
_logger.Information("Generating data export for user {UserId}", user.Id);
using var stream = new MemoryStream();
using var zip = new ZipArchive(stream, ZipArchiveMode.Create, true);
zip.Comment =
$"This archive for {user.Username} ({user.Id}) was generated at {InstantPattern.General.Format(clock.GetCurrentInstant())}";
// Write the user's info and avatar
WriteJson(
zip,
"user.json",
await userRenderer.RenderUserInnerAsync(
user,
true,
["*"],
renderMembers: false,
renderAuthMethods: true
)
);
await WriteS3Object(zip, "user-avatar.webp", userRenderer.AvatarUrlFor(user));
foreach (var flag in user.Flags)
await WritePrideFlag(zip, flag);
var members = await db
.Members.Include(m => m.User)
.Include(m => m.ProfileFlags)
.Where(m => m.UserId == user.Id)
.ToListAsync();
foreach (var member in members)
await WriteMember(zip, member);
// We want to dispose the ZipArchive on an error, but we need to dispose it manually to upload to object storage.
// Calling Dispose() multiple times is fine for this class, though.
// ReSharper disable once DisposeOnUsingVariable
zip.Dispose();
stream.Seek(0, SeekOrigin.Begin);
// Upload the file!
var filename = AuthUtils.RandomToken().Replace('+', '-').Replace('/', '_');
await objectStorageService.PutObjectAsync(
ExportPath(user.Id, filename),
stream,
"application/zip"
);
db.Add(
new DataExport
{
Id = snowflakeGenerator.GenerateSnowflake(),
UserId = user.Id,
Filename = filename,
}
);
await db.SaveChangesAsync();
}
private async Task WritePrideFlag(ZipArchive zip, PrideFlag flag)
{
_logger.Debug("Writing flag {FlagId}", flag.Id);
var flagData = $"""
{flag.Name}
----
{flag.Description ?? "<no description>"}
""";
try
{
await WriteS3Object(zip, $"flag-{flag.Id}/flag.webp", userRenderer.ImageUrlFor(flag));
}
catch (Exception e)
{
_logger.Warning(e, "Could not write image for flag {FlagId}", flag.Id);
return;
}
var entry = zip.CreateEntry($"flag-{flag.Id}/flag.txt");
await using var stream = entry.Open();
await using var writer = new StreamWriter(stream);
await writer.WriteAsync(flagData);
}
private async Task WriteMember(ZipArchive zip, Member member)
{
_logger.Debug("Writing member {MemberId}", member.Id);
WriteJson(
zip,
$"members/{member.Name} ({member.Id}).json",
memberRenderer.RenderMember(member)
);
try
{
await WriteS3Object(
zip,
$"members/{member.Name} ({member.Id}).webp",
memberRenderer.AvatarUrlFor(member)
);
}
catch (Exception e)
{
_logger.Warning(e, "Error writing avatar for member {MemberId}", member.Id);
}
}
private void WriteJson(ZipArchive zip, string filename, object data)
{
var json = JsonConvert.SerializeObject(data, Formatting.Indented);
_logger.Debug(
"Writing file {Filename} to archive with size {Length}",
filename,
json.Length
);
var entry = zip.CreateEntry(filename);
using var stream = entry.Open();
using var writer = new StreamWriter(stream);
writer.Write(json);
}
private async Task WriteS3Object(ZipArchive zip, string filename, string? s3Path)
{
if (s3Path == null)
return;
var resp = await Client.GetAsync(s3Path);
if (resp.StatusCode != HttpStatusCode.OK)
{
_logger.Warning("S3 path {S3Path} returned a non-200 status, not saving file", s3Path);
return;
}
await using var respStream = await resp.Content.ReadAsStreamAsync();
_logger.Debug(
"Writing file {Filename} to archive with size {Length}",
filename,
respStream.Length
);
var entry = zip.CreateEntry(filename);
await using var entryStream = entry.Open();
respStream.Seek(0, SeekOrigin.Begin);
await respStream.CopyToAsync(entryStream);
}
private static string ExportPath(Snowflake userId, string b64) =>
$"data-exports/{userId}/{b64}.zip";
}

View file

@ -11,3 +11,5 @@ public record CreateFlagPayload(
string ImageData,
string? Description
);
public record CreateDataExportPayload(Snowflake UserId);

View file

@ -0,0 +1,116 @@
using System.Diagnostics;
using Foxnouns.Backend.Database;
using Foxnouns.Backend.Database.Models;
using Foxnouns.Backend.Extensions;
using Microsoft.EntityFrameworkCore;
using NodaTime;
using NodaTime.Extensions;
namespace Foxnouns.Backend.Services;
public class DataCleanupService(
DatabaseContext db,
IClock clock,
ILogger logger,
ObjectStorageService objectStorageService
)
{
private readonly ILogger _logger = logger.ForContext<DataCleanupService>();
public async Task InvokeAsync(CancellationToken ct = default)
{
_logger.Information("Cleaning up expired users");
await CleanUsersAsync(ct);
_logger.Information("Cleaning up expired data exports");
await CleanExportsAsync(ct);
}
private async Task CleanUsersAsync(CancellationToken ct = default)
{
var selfDeleteExpires = clock.GetCurrentInstant() - User.DeleteAfter;
var suspendExpires = clock.GetCurrentInstant() - User.DeleteSuspendedAfter;
var users = await db
.Users.Include(u => u.Members)
.Include(u => u.DataExports)
.Where(u =>
u.Deleted
&& (
(u.DeletedBy != null && u.DeletedAt < suspendExpires)
|| (u.DeletedBy == null && u.DeletedAt < selfDeleteExpires)
)
)
.OrderBy(u => u.Id)
.AsSplitQuery()
.ToListAsync(ct);
if (users.Count == 0)
return;
_logger.Debug(
"Deleting {Count} users that have been deleted for over 30 days or suspended for over 180 days",
users.Count
);
var sw = new Stopwatch();
await Task.WhenAll(users.Select(u => CleanUserAsync(u, ct)));
await db.SaveChangesAsync(ct);
_logger.Information(
"Deleted {Count} users, their members, and their exports in {Time}",
users.Count,
sw.ElapsedDuration()
);
}
private Task CleanUserAsync(User user, CancellationToken ct = default)
{
var tasks = new List<Task>();
if (user.Avatar != null)
tasks.Add(objectStorageService.DeleteUserAvatarAsync(user.Id, user.Avatar, ct));
tasks.AddRange(
user.Members.Select(member =>
objectStorageService.DeleteMemberAvatarAsync(member.Id, member.Avatar!, ct)
)
);
tasks.AddRange(
user.DataExports.Select(export =>
objectStorageService.RemoveObjectAsync(
ExportPath(export.UserId, export.Filename),
ct
)
)
);
db.Remove(user);
return Task.WhenAll(tasks);
}
private async Task CleanExportsAsync(CancellationToken ct = default)
{
var minExpiredId = Snowflake.FromInstant(clock.GetCurrentInstant() - DataExport.Expiration);
var exports = await db.DataExports.Where(d => d.Id < minExpiredId).ToListAsync(ct);
if (exports.Count == 0)
return;
_logger.Debug("Deleting {Count} expired exports", exports.Count);
foreach (var export in exports)
{
_logger.Debug("Deleting export {ExportId}", export.Id);
await objectStorageService.RemoveObjectAsync(
ExportPath(export.UserId, export.Filename),
ct
);
db.Remove(export);
}
await db.SaveChangesAsync(ct);
}
private static string ExportPath(Snowflake userId, string b64) =>
$"data-exports/{userId}/{b64}.zip";
}

View file

@ -72,7 +72,7 @@ public class MemberRendererService(DatabaseContext db, Config config)
renderUnlisted ? member.Unlisted : null
);
private string? AvatarUrlFor(Member member) =>
public string? AvatarUrlFor(Member member) =>
member.Avatar != null
? $"{config.MediaBaseUrl}/members/{member.Id}/avatars/{member.Avatar}.webp"
: null;

View file

@ -62,7 +62,7 @@ public class BackgroundMetricsCollectionService(
using var timer = new PeriodicTimer(TimeSpan.FromMinutes(1));
while (await timer.WaitForNextTickAsync(ct))
{
_logger.Debug("Collecting metrics");
_logger.Debug("Collecting metrics manually");
await metricsCollectionService.CollectMetricsAsync(ct);
}
}

View file

@ -48,4 +48,13 @@ public class ObjectStorageService(ILogger logger, Config config, IMinioClient mi
ct
);
}
public async Task GetObjectAsync(string path, CancellationToken ct = default)
{
var stream = new MemoryStream();
var resp = await minioClient.GetObjectAsync(
new GetObjectArgs().WithBucket(config.Storage.Bucket).WithObject(path),
ct
);
}
}

View file

@ -8,10 +8,7 @@ public class PeriodicTasksService(ILogger logger, IServiceProvider services) : B
{
using var timer = new PeriodicTimer(TimeSpan.FromMinutes(1));
while (await timer.WaitForNextTickAsync(ct))
{
_logger.Debug("Collecting metrics");
await RunPeriodicTasksAsync(ct);
}
}
private async Task RunPeriodicTasksAsync(CancellationToken ct)
@ -20,7 +17,10 @@ public class PeriodicTasksService(ILogger logger, IServiceProvider services) : B
await using var scope = services.CreateAsyncScope();
var keyCacheSvc = scope.ServiceProvider.GetRequiredService<KeyCacheService>();
await keyCacheSvc.DeleteExpiredKeysAsync(ct);
var keyCacheService = scope.ServiceProvider.GetRequiredService<KeyCacheService>();
var dataCleanupService = scope.ServiceProvider.GetRequiredService<DataCleanupService>();
await keyCacheService.DeleteExpiredKeysAsync(ct);
await dataCleanupService.InvokeAsync(ct);
}
}

View file

@ -22,12 +22,31 @@ public class UserRendererService(
bool renderAuthMethods = false,
string? overrideSid = null,
CancellationToken ct = default
) =>
await RenderUserInnerAsync(
user,
selfUser != null && selfUser.Id == user.Id,
token?.Scopes ?? [],
renderMembers,
renderAuthMethods,
overrideSid,
ct
);
public async Task<UserResponse> RenderUserInnerAsync(
User user,
bool isSelfUser,
string[] scopes,
bool renderMembers = true,
bool renderAuthMethods = false,
string? overrideSid = null,
CancellationToken ct = default
)
{
var isSelfUser = selfUser?.Id == user.Id;
var tokenCanReadHiddenMembers = token.HasScope("member.read") && isSelfUser;
var tokenHidden = token.HasScope("user.read_hidden") && isSelfUser;
var tokenPrivileged = token.HasScope("user.read_privileged") && isSelfUser;
scopes = scopes.ExpandScopes();
var tokenCanReadHiddenMembers = scopes.Contains("member.read") && isSelfUser;
var tokenHidden = scopes.Contains("user.read_hidden") && isSelfUser;
var tokenPrivileged = scopes.Contains("user.read_privileged") && isSelfUser;
renderMembers = renderMembers && (!user.ListHidden || tokenCanReadHiddenMembers);
renderAuthMethods = renderAuthMethods && tokenPrivileged;
@ -105,12 +124,12 @@ public class UserRendererService(
user.CustomPreferences
);
private string? AvatarUrlFor(User user) =>
public string? AvatarUrlFor(User user) =>
user.Avatar != null
? $"{config.MediaBaseUrl}/users/{user.Id}/avatars/{user.Avatar}.webp"
: null;
private string ImageUrlFor(PrideFlag flag) => $"{config.MediaBaseUrl}/flags/{flag.Hash}.webp";
public string ImageUrlFor(PrideFlag flag) => $"{config.MediaBaseUrl}/flags/{flag.Hash}.webp";
public record UserResponse(
Snowflake Id,

View file

@ -1,13 +1,24 @@
import { apiRequest } from "$api";
import type { PartialMember, UserWithMembers } from "$api/models";
import ApiError, { ErrorCode } from "$api/error.js";
import type { PartialMember, User, UserWithMembers } from "$api/models";
import log from "$lib/log.js";
import { error } from "@sveltejs/kit";
const MEMBERS_PER_PAGE = 20;
export const load = async ({ params, fetch, cookies, url }) => {
const user = await apiRequest<UserWithMembers>("GET", `/users/${params.username}`, {
fetch,
cookies,
});
let user: UserWithMembers;
try {
user = await apiRequest<UserWithMembers>("GET", `/users/${params.username}`, {
fetch,
cookies,
});
} catch (e) {
if (e instanceof ApiError && e.code === ErrorCode.UserNotFound) error(404, "User not found");
log.error("Error fetching user %s:", params.username, e);
throw e;
}
// Paginate members on the server side
let currentPage = 0;

View file

@ -1,15 +1,28 @@
import { apiRequest } from "$api";
import ApiError, { ErrorCode } from "$api/error.js";
import type { Member } from "$api/models/member";
import log from "$lib/log.js";
import { error } from "@sveltejs/kit";
export const load = async ({ params, fetch, cookies }) => {
const member = await apiRequest<Member>(
"GET",
`/users/${params.username}/members/${params.memberName}`,
{
fetch,
cookies,
},
);
try {
const member = await apiRequest<Member>(
"GET",
`/users/${params.username}/members/${params.memberName}`,
{
fetch,
cookies,
},
);
return { member };
return { member };
} catch (e) {
if (e instanceof ApiError) {
if (e.code === ErrorCode.UserNotFound) error(404, "User not found");
if (e.code === ErrorCode.MemberNotFound) error(404, "Member not found");
}
log.error("Error fetching user %s/member %s:", params.username, params.memberName, e);
throw e;
}
};