Compare commits
3 commits
f0ae648492
...
71d3b42330
Author | SHA1 | Date | |
---|---|---|---|
71d3b42330 | |||
18bdbc0745 | |||
903be2709c |
18 changed files with 617 additions and 38 deletions
74
Foxnouns.Backend/Controllers/ExportsController.cs
Normal file
74
Foxnouns.Backend/Controllers/ExportsController.cs
Normal file
|
@ -0,0 +1,74 @@
|
||||||
|
using Coravel.Queuing.Interfaces;
|
||||||
|
using Foxnouns.Backend.Database;
|
||||||
|
using Foxnouns.Backend.Database.Models;
|
||||||
|
using Foxnouns.Backend.Jobs;
|
||||||
|
using Foxnouns.Backend.Middleware;
|
||||||
|
using Microsoft.AspNetCore.Mvc;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using NodaTime;
|
||||||
|
|
||||||
|
namespace Foxnouns.Backend.Controllers;
|
||||||
|
|
||||||
|
[Route("/api/internal/data-exports")]
|
||||||
|
[Authorize("identify")]
|
||||||
|
public class ExportsController(
|
||||||
|
ILogger logger,
|
||||||
|
Config config,
|
||||||
|
IClock clock,
|
||||||
|
DatabaseContext db,
|
||||||
|
IQueue queue
|
||||||
|
) : ApiControllerBase
|
||||||
|
{
|
||||||
|
private static readonly Duration MinimumTimeBetween = Duration.FromDays(1);
|
||||||
|
private readonly ILogger _logger = logger.ForContext<ExportsController>();
|
||||||
|
|
||||||
|
[HttpGet]
|
||||||
|
public async Task<IActionResult> GetDataExportsAsync()
|
||||||
|
{
|
||||||
|
var export = await db
|
||||||
|
.DataExports.Where(d => d.UserId == CurrentUser!.Id)
|
||||||
|
.OrderByDescending(d => d.Id)
|
||||||
|
.FirstOrDefaultAsync();
|
||||||
|
if (export == null)
|
||||||
|
return Ok(new DataExportResponse(null, null));
|
||||||
|
|
||||||
|
return Ok(
|
||||||
|
new DataExportResponse(
|
||||||
|
ExportUrl(CurrentUser!.Id, export.Filename),
|
||||||
|
export.Id.Time + DataExport.Expiration
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private string ExportUrl(Snowflake userId, string filename) =>
|
||||||
|
$"{config.MediaBaseUrl}/data-exports/{userId}/{filename}.zip";
|
||||||
|
|
||||||
|
private record DataExportResponse(string? Url, Instant? ExpiresAt);
|
||||||
|
|
||||||
|
[HttpPost]
|
||||||
|
public async Task<IActionResult> QueueDataExportAsync()
|
||||||
|
{
|
||||||
|
var snowflakeToCheck = Snowflake.FromInstant(
|
||||||
|
clock.GetCurrentInstant() - MinimumTimeBetween
|
||||||
|
);
|
||||||
|
_logger.Debug(
|
||||||
|
"Checking if user {UserId} has data exports newer than {Snowflake}",
|
||||||
|
CurrentUser!.Id,
|
||||||
|
snowflakeToCheck
|
||||||
|
);
|
||||||
|
if (
|
||||||
|
await db.DataExports.AnyAsync(d =>
|
||||||
|
d.UserId == CurrentUser.Id && d.Id > snowflakeToCheck
|
||||||
|
)
|
||||||
|
)
|
||||||
|
{
|
||||||
|
throw new ApiError.BadRequest("You can't request a new data export so soon.");
|
||||||
|
}
|
||||||
|
|
||||||
|
queue.QueueInvocableWithPayload<CreateDataExportInvocable, CreateDataExportPayload>(
|
||||||
|
new CreateDataExportPayload(CurrentUser.Id)
|
||||||
|
);
|
||||||
|
|
||||||
|
return NoContent();
|
||||||
|
}
|
||||||
|
}
|
|
@ -46,17 +46,18 @@ public class DatabaseContext(DbContextOptions options) : DbContext(options)
|
||||||
.UseSnakeCaseNamingConvention()
|
.UseSnakeCaseNamingConvention()
|
||||||
.UseExceptionProcessor();
|
.UseExceptionProcessor();
|
||||||
|
|
||||||
public DbSet<User> Users { get; set; }
|
public DbSet<User> Users { get; init; }
|
||||||
public DbSet<Member> Members { get; set; }
|
public DbSet<Member> Members { get; init; }
|
||||||
public DbSet<AuthMethod> AuthMethods { get; set; }
|
public DbSet<AuthMethod> AuthMethods { get; init; }
|
||||||
public DbSet<FediverseApplication> FediverseApplications { get; set; }
|
public DbSet<FediverseApplication> FediverseApplications { get; init; }
|
||||||
public DbSet<Token> Tokens { get; set; }
|
public DbSet<Token> Tokens { get; init; }
|
||||||
public DbSet<Application> Applications { get; set; }
|
public DbSet<Application> Applications { get; init; }
|
||||||
public DbSet<TemporaryKey> TemporaryKeys { get; set; }
|
public DbSet<TemporaryKey> TemporaryKeys { get; init; }
|
||||||
|
public DbSet<DataExport> DataExports { get; init; }
|
||||||
|
|
||||||
public DbSet<PrideFlag> PrideFlags { get; set; }
|
public DbSet<PrideFlag> PrideFlags { get; init; }
|
||||||
public DbSet<UserFlag> UserFlags { get; set; }
|
public DbSet<UserFlag> UserFlags { get; init; }
|
||||||
public DbSet<MemberFlag> MemberFlags { get; set; }
|
public DbSet<MemberFlag> MemberFlags { get; init; }
|
||||||
|
|
||||||
protected override void ConfigureConventions(ModelConfigurationBuilder configurationBuilder)
|
protected override void ConfigureConventions(ModelConfigurationBuilder configurationBuilder)
|
||||||
{
|
{
|
||||||
|
@ -81,6 +82,7 @@ public class DatabaseContext(DbContextOptions options) : DbContext(options)
|
||||||
})
|
})
|
||||||
.HasFilter("fediverse_application_id IS NOT NULL")
|
.HasFilter("fediverse_application_id IS NOT NULL")
|
||||||
.IsUnique();
|
.IsUnique();
|
||||||
|
modelBuilder.Entity<DataExport>().HasIndex(d => d.Filename).IsUnique();
|
||||||
|
|
||||||
modelBuilder
|
modelBuilder
|
||||||
.Entity<AuthMethod>()
|
.Entity<AuthMethod>()
|
||||||
|
|
|
@ -0,0 +1,57 @@
|
||||||
|
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||||
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
|
||||||
|
#nullable disable
|
||||||
|
|
||||||
|
namespace Foxnouns.Backend.Database.Migrations
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
[DbContext(typeof(DatabaseContext))]
|
||||||
|
[Migration("20241202153736_AddDataExports")]
|
||||||
|
public partial class AddDataExports : Migration
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void Up(MigrationBuilder migrationBuilder)
|
||||||
|
{
|
||||||
|
migrationBuilder.CreateTable(
|
||||||
|
name: "data_exports",
|
||||||
|
columns: table => new
|
||||||
|
{
|
||||||
|
id = table.Column<long>(type: "bigint", nullable: false),
|
||||||
|
user_id = table.Column<long>(type: "bigint", nullable: false),
|
||||||
|
filename = table.Column<string>(type: "text", nullable: false),
|
||||||
|
},
|
||||||
|
constraints: table =>
|
||||||
|
{
|
||||||
|
table.PrimaryKey("pk_data_exports", x => x.id);
|
||||||
|
table.ForeignKey(
|
||||||
|
name: "fk_data_exports_users_user_id",
|
||||||
|
column: x => x.user_id,
|
||||||
|
principalTable: "users",
|
||||||
|
principalColumn: "id",
|
||||||
|
onDelete: ReferentialAction.Cascade
|
||||||
|
);
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
migrationBuilder.CreateIndex(
|
||||||
|
name: "ix_data_exports_filename",
|
||||||
|
table: "data_exports",
|
||||||
|
column: "filename",
|
||||||
|
unique: true
|
||||||
|
);
|
||||||
|
|
||||||
|
migrationBuilder.CreateIndex(
|
||||||
|
name: "ix_data_exports_user_id",
|
||||||
|
table: "data_exports",
|
||||||
|
column: "user_id"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void Down(MigrationBuilder migrationBuilder)
|
||||||
|
{
|
||||||
|
migrationBuilder.DropTable(name: "data_exports");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -111,6 +111,34 @@ namespace Foxnouns.Backend.Database.Migrations
|
||||||
b.ToTable("auth_methods", (string)null);
|
b.ToTable("auth_methods", (string)null);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("Foxnouns.Backend.Database.Models.DataExport", b =>
|
||||||
|
{
|
||||||
|
b.Property<long>("Id")
|
||||||
|
.HasColumnType("bigint")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
b.Property<string>("Filename")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("filename");
|
||||||
|
|
||||||
|
b.Property<long>("UserId")
|
||||||
|
.HasColumnType("bigint")
|
||||||
|
.HasColumnName("user_id");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_data_exports");
|
||||||
|
|
||||||
|
b.HasIndex("Filename")
|
||||||
|
.IsUnique()
|
||||||
|
.HasDatabaseName("ix_data_exports_filename");
|
||||||
|
|
||||||
|
b.HasIndex("UserId")
|
||||||
|
.HasDatabaseName("ix_data_exports_user_id");
|
||||||
|
|
||||||
|
b.ToTable("data_exports", (string)null);
|
||||||
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("Foxnouns.Backend.Database.Models.FediverseApplication", b =>
|
modelBuilder.Entity("Foxnouns.Backend.Database.Models.FediverseApplication", b =>
|
||||||
{
|
{
|
||||||
b.Property<long>("Id")
|
b.Property<long>("Id")
|
||||||
|
@ -515,6 +543,18 @@ namespace Foxnouns.Backend.Database.Migrations
|
||||||
b.Navigation("User");
|
b.Navigation("User");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("Foxnouns.Backend.Database.Models.DataExport", b =>
|
||||||
|
{
|
||||||
|
b.HasOne("Foxnouns.Backend.Database.Models.User", "User")
|
||||||
|
.WithMany()
|
||||||
|
.HasForeignKey("UserId")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired()
|
||||||
|
.HasConstraintName("fk_data_exports_users_user_id");
|
||||||
|
|
||||||
|
b.Navigation("User");
|
||||||
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("Foxnouns.Backend.Database.Models.Member", b =>
|
modelBuilder.Entity("Foxnouns.Backend.Database.Models.Member", b =>
|
||||||
{
|
{
|
||||||
b.HasOne("Foxnouns.Backend.Database.Models.User", "User")
|
b.HasOne("Foxnouns.Backend.Database.Models.User", "User")
|
||||||
|
|
14
Foxnouns.Backend/Database/Models/DataExport.cs
Normal file
14
Foxnouns.Backend/Database/Models/DataExport.cs
Normal file
|
@ -0,0 +1,14 @@
|
||||||
|
using System.ComponentModel.DataAnnotations.Schema;
|
||||||
|
using NodaTime;
|
||||||
|
|
||||||
|
namespace Foxnouns.Backend.Database.Models;
|
||||||
|
|
||||||
|
public class DataExport : BaseModel
|
||||||
|
{
|
||||||
|
public Snowflake UserId { get; init; }
|
||||||
|
public User User { get; init; } = null!;
|
||||||
|
public required string Filename { get; init; }
|
||||||
|
|
||||||
|
[NotMapped]
|
||||||
|
public static readonly Duration Expiration = Duration.FromDays(15);
|
||||||
|
}
|
|
@ -30,6 +30,7 @@ public class User : BaseModel
|
||||||
|
|
||||||
public List<Member> Members { get; } = [];
|
public List<Member> Members { get; } = [];
|
||||||
public List<AuthMethod> AuthMethods { get; } = [];
|
public List<AuthMethod> AuthMethods { get; } = [];
|
||||||
|
public List<DataExport> DataExports { get; } = [];
|
||||||
public UserSettings Settings { get; set; } = new();
|
public UserSettings Settings { get; set; } = new();
|
||||||
|
|
||||||
public required Instant LastActive { get; set; }
|
public required Instant LastActive { get; set; }
|
||||||
|
@ -53,6 +54,12 @@ public class User : BaseModel
|
||||||
[JsonConverter(typeof(ScreamingSnakeCaseEnumConverter))]
|
[JsonConverter(typeof(ScreamingSnakeCaseEnumConverter))]
|
||||||
public PreferenceSize Size { get; set; }
|
public PreferenceSize Size { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[NotMapped]
|
||||||
|
public static readonly Duration DeleteAfter = Duration.FromDays(30);
|
||||||
|
|
||||||
|
[NotMapped]
|
||||||
|
public static readonly Duration DeleteSuspendedAfter = Duration.FromDays(180);
|
||||||
}
|
}
|
||||||
|
|
||||||
public enum UserRole
|
public enum UserRole
|
||||||
|
|
|
@ -65,6 +65,9 @@ public readonly struct Snowflake(ulong value) : IEquatable<Snowflake>
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static Snowflake FromInstant(Instant instant) =>
|
||||||
|
new((ulong)(instant.ToUnixTimeMilliseconds() - Epoch) << 22);
|
||||||
|
|
||||||
public override bool Equals(object? obj) => obj is Snowflake other && Value == other.Value;
|
public override bool Equals(object? obj) => obj is Snowflake other && Value == other.Value;
|
||||||
|
|
||||||
public bool Equals(Snowflake other)
|
public bool Equals(Snowflake other)
|
||||||
|
|
|
@ -106,12 +106,14 @@ public static class WebApplicationExtensions
|
||||||
.AddScoped<RemoteAuthService>()
|
.AddScoped<RemoteAuthService>()
|
||||||
.AddScoped<FediverseAuthService>()
|
.AddScoped<FediverseAuthService>()
|
||||||
.AddScoped<ObjectStorageService>()
|
.AddScoped<ObjectStorageService>()
|
||||||
|
.AddTransient<DataCleanupService>()
|
||||||
// Background services
|
// Background services
|
||||||
.AddHostedService<PeriodicTasksService>()
|
.AddHostedService<PeriodicTasksService>()
|
||||||
// Transient jobs
|
// Transient jobs
|
||||||
.AddTransient<MemberAvatarUpdateInvocable>()
|
.AddTransient<MemberAvatarUpdateInvocable>()
|
||||||
.AddTransient<UserAvatarUpdateInvocable>()
|
.AddTransient<UserAvatarUpdateInvocable>()
|
||||||
.AddTransient<CreateFlagInvocable>();
|
.AddTransient<CreateFlagInvocable>()
|
||||||
|
.AddTransient<CreateDataExportInvocable>();
|
||||||
|
|
||||||
if (!config.Logging.EnableMetrics)
|
if (!config.Logging.EnableMetrics)
|
||||||
services.AddHostedService<BackgroundMetricsCollectionService>();
|
services.AddHostedService<BackgroundMetricsCollectionService>();
|
||||||
|
|
210
Foxnouns.Backend/Jobs/CreateDataExportInvocable.cs
Normal file
210
Foxnouns.Backend/Jobs/CreateDataExportInvocable.cs
Normal file
|
@ -0,0 +1,210 @@
|
||||||
|
using System.IO.Compression;
|
||||||
|
using System.Net;
|
||||||
|
using Coravel.Invocable;
|
||||||
|
using Foxnouns.Backend.Database;
|
||||||
|
using Foxnouns.Backend.Database.Models;
|
||||||
|
using Foxnouns.Backend.Services;
|
||||||
|
using Foxnouns.Backend.Utils;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Newtonsoft.Json;
|
||||||
|
using NodaTime;
|
||||||
|
using NodaTime.Text;
|
||||||
|
|
||||||
|
namespace Foxnouns.Backend.Jobs;
|
||||||
|
|
||||||
|
public class CreateDataExportInvocable(
|
||||||
|
DatabaseContext db,
|
||||||
|
IClock clock,
|
||||||
|
UserRendererService userRenderer,
|
||||||
|
MemberRendererService memberRenderer,
|
||||||
|
ObjectStorageService objectStorageService,
|
||||||
|
ISnowflakeGenerator snowflakeGenerator,
|
||||||
|
ILogger logger
|
||||||
|
) : IInvocable, IInvocableWithPayload<CreateDataExportPayload>
|
||||||
|
{
|
||||||
|
private static readonly HttpClient Client = new();
|
||||||
|
private readonly ILogger _logger = logger.ForContext<CreateDataExportInvocable>();
|
||||||
|
public required CreateDataExportPayload Payload { get; set; }
|
||||||
|
|
||||||
|
public async Task Invoke()
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await InvokeAsync();
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.Error(e, "Error generating data export for user {UserId}", Payload.UserId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task InvokeAsync()
|
||||||
|
{
|
||||||
|
var user = await db
|
||||||
|
.Users.Include(u => u.AuthMethods)
|
||||||
|
.Include(u => u.Flags)
|
||||||
|
.Include(u => u.ProfileFlags)
|
||||||
|
.AsSplitQuery()
|
||||||
|
.FirstOrDefaultAsync(u => u.Id == Payload.UserId);
|
||||||
|
if (user == null)
|
||||||
|
{
|
||||||
|
_logger.Warning(
|
||||||
|
"Received create data export request for user {UserId} but no such user exists, deleted or otherwise. Ignoring request",
|
||||||
|
Payload.UserId
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.Information("Generating data export for user {UserId}", user.Id);
|
||||||
|
|
||||||
|
using var stream = new MemoryStream();
|
||||||
|
using var zip = new ZipArchive(stream, ZipArchiveMode.Create, true);
|
||||||
|
zip.Comment =
|
||||||
|
$"This archive for {user.Username} ({user.Id}) was generated at {InstantPattern.General.Format(clock.GetCurrentInstant())}";
|
||||||
|
|
||||||
|
// Write the user's info and avatar
|
||||||
|
WriteJson(
|
||||||
|
zip,
|
||||||
|
"user.json",
|
||||||
|
await userRenderer.RenderUserInnerAsync(
|
||||||
|
user,
|
||||||
|
true,
|
||||||
|
["*"],
|
||||||
|
renderMembers: false,
|
||||||
|
renderAuthMethods: true
|
||||||
|
)
|
||||||
|
);
|
||||||
|
await WriteS3Object(zip, "user-avatar.webp", userRenderer.AvatarUrlFor(user));
|
||||||
|
|
||||||
|
foreach (var flag in user.Flags)
|
||||||
|
await WritePrideFlag(zip, flag);
|
||||||
|
|
||||||
|
var members = await db
|
||||||
|
.Members.Include(m => m.User)
|
||||||
|
.Include(m => m.ProfileFlags)
|
||||||
|
.Where(m => m.UserId == user.Id)
|
||||||
|
.ToListAsync();
|
||||||
|
foreach (var member in members)
|
||||||
|
await WriteMember(zip, member);
|
||||||
|
|
||||||
|
// We want to dispose the ZipArchive on an error, but we need to dispose it manually to upload to object storage.
|
||||||
|
// Calling Dispose() multiple times is fine for this class, though.
|
||||||
|
// ReSharper disable once DisposeOnUsingVariable
|
||||||
|
zip.Dispose();
|
||||||
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
|
|
||||||
|
// Upload the file!
|
||||||
|
var filename = AuthUtils.RandomToken().Replace('+', '-').Replace('/', '_');
|
||||||
|
await objectStorageService.PutObjectAsync(
|
||||||
|
ExportPath(user.Id, filename),
|
||||||
|
stream,
|
||||||
|
"application/zip"
|
||||||
|
);
|
||||||
|
|
||||||
|
db.Add(
|
||||||
|
new DataExport
|
||||||
|
{
|
||||||
|
Id = snowflakeGenerator.GenerateSnowflake(),
|
||||||
|
UserId = user.Id,
|
||||||
|
Filename = filename,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
await db.SaveChangesAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task WritePrideFlag(ZipArchive zip, PrideFlag flag)
|
||||||
|
{
|
||||||
|
_logger.Debug("Writing flag {FlagId}", flag.Id);
|
||||||
|
|
||||||
|
var flagData = $"""
|
||||||
|
{flag.Name}
|
||||||
|
----
|
||||||
|
{flag.Description ?? "<no description>"}
|
||||||
|
""";
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await WriteS3Object(zip, $"flag-{flag.Id}/flag.webp", userRenderer.ImageUrlFor(flag));
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.Warning(e, "Could not write image for flag {FlagId}", flag.Id);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var entry = zip.CreateEntry($"flag-{flag.Id}/flag.txt");
|
||||||
|
await using var stream = entry.Open();
|
||||||
|
await using var writer = new StreamWriter(stream);
|
||||||
|
await writer.WriteAsync(flagData);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task WriteMember(ZipArchive zip, Member member)
|
||||||
|
{
|
||||||
|
_logger.Debug("Writing member {MemberId}", member.Id);
|
||||||
|
|
||||||
|
WriteJson(
|
||||||
|
zip,
|
||||||
|
$"members/{member.Name} ({member.Id}).json",
|
||||||
|
memberRenderer.RenderMember(member)
|
||||||
|
);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await WriteS3Object(
|
||||||
|
zip,
|
||||||
|
$"members/{member.Name} ({member.Id}).webp",
|
||||||
|
memberRenderer.AvatarUrlFor(member)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.Warning(e, "Error writing avatar for member {MemberId}", member.Id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void WriteJson(ZipArchive zip, string filename, object data)
|
||||||
|
{
|
||||||
|
var json = JsonConvert.SerializeObject(data, Formatting.Indented);
|
||||||
|
|
||||||
|
_logger.Debug(
|
||||||
|
"Writing file {Filename} to archive with size {Length}",
|
||||||
|
filename,
|
||||||
|
json.Length
|
||||||
|
);
|
||||||
|
|
||||||
|
var entry = zip.CreateEntry(filename);
|
||||||
|
using var stream = entry.Open();
|
||||||
|
using var writer = new StreamWriter(stream);
|
||||||
|
writer.Write(json);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task WriteS3Object(ZipArchive zip, string filename, string? s3Path)
|
||||||
|
{
|
||||||
|
if (s3Path == null)
|
||||||
|
return;
|
||||||
|
|
||||||
|
var resp = await Client.GetAsync(s3Path);
|
||||||
|
if (resp.StatusCode != HttpStatusCode.OK)
|
||||||
|
{
|
||||||
|
_logger.Warning("S3 path {S3Path} returned a non-200 status, not saving file", s3Path);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await using var respStream = await resp.Content.ReadAsStreamAsync();
|
||||||
|
|
||||||
|
_logger.Debug(
|
||||||
|
"Writing file {Filename} to archive with size {Length}",
|
||||||
|
filename,
|
||||||
|
respStream.Length
|
||||||
|
);
|
||||||
|
|
||||||
|
var entry = zip.CreateEntry(filename);
|
||||||
|
await using var entryStream = entry.Open();
|
||||||
|
|
||||||
|
respStream.Seek(0, SeekOrigin.Begin);
|
||||||
|
await respStream.CopyToAsync(entryStream);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string ExportPath(Snowflake userId, string b64) =>
|
||||||
|
$"data-exports/{userId}/{b64}.zip";
|
||||||
|
}
|
|
@ -11,3 +11,5 @@ public record CreateFlagPayload(
|
||||||
string ImageData,
|
string ImageData,
|
||||||
string? Description
|
string? Description
|
||||||
);
|
);
|
||||||
|
|
||||||
|
public record CreateDataExportPayload(Snowflake UserId);
|
||||||
|
|
116
Foxnouns.Backend/Services/DataCleanupService.cs
Normal file
116
Foxnouns.Backend/Services/DataCleanupService.cs
Normal file
|
@ -0,0 +1,116 @@
|
||||||
|
using System.Diagnostics;
|
||||||
|
using Foxnouns.Backend.Database;
|
||||||
|
using Foxnouns.Backend.Database.Models;
|
||||||
|
using Foxnouns.Backend.Extensions;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using NodaTime;
|
||||||
|
using NodaTime.Extensions;
|
||||||
|
|
||||||
|
namespace Foxnouns.Backend.Services;
|
||||||
|
|
||||||
|
public class DataCleanupService(
|
||||||
|
DatabaseContext db,
|
||||||
|
IClock clock,
|
||||||
|
ILogger logger,
|
||||||
|
ObjectStorageService objectStorageService
|
||||||
|
)
|
||||||
|
{
|
||||||
|
private readonly ILogger _logger = logger.ForContext<DataCleanupService>();
|
||||||
|
|
||||||
|
public async Task InvokeAsync(CancellationToken ct = default)
|
||||||
|
{
|
||||||
|
_logger.Information("Cleaning up expired users");
|
||||||
|
await CleanUsersAsync(ct);
|
||||||
|
|
||||||
|
_logger.Information("Cleaning up expired data exports");
|
||||||
|
await CleanExportsAsync(ct);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task CleanUsersAsync(CancellationToken ct = default)
|
||||||
|
{
|
||||||
|
var selfDeleteExpires = clock.GetCurrentInstant() - User.DeleteAfter;
|
||||||
|
var suspendExpires = clock.GetCurrentInstant() - User.DeleteSuspendedAfter;
|
||||||
|
var users = await db
|
||||||
|
.Users.Include(u => u.Members)
|
||||||
|
.Include(u => u.DataExports)
|
||||||
|
.Where(u =>
|
||||||
|
u.Deleted
|
||||||
|
&& (
|
||||||
|
(u.DeletedBy != null && u.DeletedAt < suspendExpires)
|
||||||
|
|| (u.DeletedBy == null && u.DeletedAt < selfDeleteExpires)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.OrderBy(u => u.Id)
|
||||||
|
.AsSplitQuery()
|
||||||
|
.ToListAsync(ct);
|
||||||
|
if (users.Count == 0)
|
||||||
|
return;
|
||||||
|
|
||||||
|
_logger.Debug(
|
||||||
|
"Deleting {Count} users that have been deleted for over 30 days or suspended for over 180 days",
|
||||||
|
users.Count
|
||||||
|
);
|
||||||
|
|
||||||
|
var sw = new Stopwatch();
|
||||||
|
|
||||||
|
await Task.WhenAll(users.Select(u => CleanUserAsync(u, ct)));
|
||||||
|
|
||||||
|
await db.SaveChangesAsync(ct);
|
||||||
|
_logger.Information(
|
||||||
|
"Deleted {Count} users, their members, and their exports in {Time}",
|
||||||
|
users.Count,
|
||||||
|
sw.ElapsedDuration()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Task CleanUserAsync(User user, CancellationToken ct = default)
|
||||||
|
{
|
||||||
|
var tasks = new List<Task>();
|
||||||
|
|
||||||
|
if (user.Avatar != null)
|
||||||
|
tasks.Add(objectStorageService.DeleteUserAvatarAsync(user.Id, user.Avatar, ct));
|
||||||
|
|
||||||
|
tasks.AddRange(
|
||||||
|
user.Members.Select(member =>
|
||||||
|
objectStorageService.DeleteMemberAvatarAsync(member.Id, member.Avatar!, ct)
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
tasks.AddRange(
|
||||||
|
user.DataExports.Select(export =>
|
||||||
|
objectStorageService.RemoveObjectAsync(
|
||||||
|
ExportPath(export.UserId, export.Filename),
|
||||||
|
ct
|
||||||
|
)
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
db.Remove(user);
|
||||||
|
return Task.WhenAll(tasks);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task CleanExportsAsync(CancellationToken ct = default)
|
||||||
|
{
|
||||||
|
var minExpiredId = Snowflake.FromInstant(clock.GetCurrentInstant() - DataExport.Expiration);
|
||||||
|
var exports = await db.DataExports.Where(d => d.Id < minExpiredId).ToListAsync(ct);
|
||||||
|
if (exports.Count == 0)
|
||||||
|
return;
|
||||||
|
|
||||||
|
_logger.Debug("Deleting {Count} expired exports", exports.Count);
|
||||||
|
|
||||||
|
foreach (var export in exports)
|
||||||
|
{
|
||||||
|
_logger.Debug("Deleting export {ExportId}", export.Id);
|
||||||
|
await objectStorageService.RemoveObjectAsync(
|
||||||
|
ExportPath(export.UserId, export.Filename),
|
||||||
|
ct
|
||||||
|
);
|
||||||
|
db.Remove(export);
|
||||||
|
}
|
||||||
|
|
||||||
|
await db.SaveChangesAsync(ct);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string ExportPath(Snowflake userId, string b64) =>
|
||||||
|
$"data-exports/{userId}/{b64}.zip";
|
||||||
|
}
|
|
@ -72,7 +72,7 @@ public class MemberRendererService(DatabaseContext db, Config config)
|
||||||
renderUnlisted ? member.Unlisted : null
|
renderUnlisted ? member.Unlisted : null
|
||||||
);
|
);
|
||||||
|
|
||||||
private string? AvatarUrlFor(Member member) =>
|
public string? AvatarUrlFor(Member member) =>
|
||||||
member.Avatar != null
|
member.Avatar != null
|
||||||
? $"{config.MediaBaseUrl}/members/{member.Id}/avatars/{member.Avatar}.webp"
|
? $"{config.MediaBaseUrl}/members/{member.Id}/avatars/{member.Avatar}.webp"
|
||||||
: null;
|
: null;
|
||||||
|
|
|
@ -62,7 +62,7 @@ public class BackgroundMetricsCollectionService(
|
||||||
using var timer = new PeriodicTimer(TimeSpan.FromMinutes(1));
|
using var timer = new PeriodicTimer(TimeSpan.FromMinutes(1));
|
||||||
while (await timer.WaitForNextTickAsync(ct))
|
while (await timer.WaitForNextTickAsync(ct))
|
||||||
{
|
{
|
||||||
_logger.Debug("Collecting metrics");
|
_logger.Debug("Collecting metrics manually");
|
||||||
await metricsCollectionService.CollectMetricsAsync(ct);
|
await metricsCollectionService.CollectMetricsAsync(ct);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -48,4 +48,13 @@ public class ObjectStorageService(ILogger logger, Config config, IMinioClient mi
|
||||||
ct
|
ct
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async Task GetObjectAsync(string path, CancellationToken ct = default)
|
||||||
|
{
|
||||||
|
var stream = new MemoryStream();
|
||||||
|
var resp = await minioClient.GetObjectAsync(
|
||||||
|
new GetObjectArgs().WithBucket(config.Storage.Bucket).WithObject(path),
|
||||||
|
ct
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,10 +8,7 @@ public class PeriodicTasksService(ILogger logger, IServiceProvider services) : B
|
||||||
{
|
{
|
||||||
using var timer = new PeriodicTimer(TimeSpan.FromMinutes(1));
|
using var timer = new PeriodicTimer(TimeSpan.FromMinutes(1));
|
||||||
while (await timer.WaitForNextTickAsync(ct))
|
while (await timer.WaitForNextTickAsync(ct))
|
||||||
{
|
|
||||||
_logger.Debug("Collecting metrics");
|
|
||||||
await RunPeriodicTasksAsync(ct);
|
await RunPeriodicTasksAsync(ct);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task RunPeriodicTasksAsync(CancellationToken ct)
|
private async Task RunPeriodicTasksAsync(CancellationToken ct)
|
||||||
|
@ -20,7 +17,10 @@ public class PeriodicTasksService(ILogger logger, IServiceProvider services) : B
|
||||||
|
|
||||||
await using var scope = services.CreateAsyncScope();
|
await using var scope = services.CreateAsyncScope();
|
||||||
|
|
||||||
var keyCacheSvc = scope.ServiceProvider.GetRequiredService<KeyCacheService>();
|
var keyCacheService = scope.ServiceProvider.GetRequiredService<KeyCacheService>();
|
||||||
await keyCacheSvc.DeleteExpiredKeysAsync(ct);
|
var dataCleanupService = scope.ServiceProvider.GetRequiredService<DataCleanupService>();
|
||||||
|
|
||||||
|
await keyCacheService.DeleteExpiredKeysAsync(ct);
|
||||||
|
await dataCleanupService.InvokeAsync(ct);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -22,12 +22,31 @@ public class UserRendererService(
|
||||||
bool renderAuthMethods = false,
|
bool renderAuthMethods = false,
|
||||||
string? overrideSid = null,
|
string? overrideSid = null,
|
||||||
CancellationToken ct = default
|
CancellationToken ct = default
|
||||||
|
) =>
|
||||||
|
await RenderUserInnerAsync(
|
||||||
|
user,
|
||||||
|
selfUser != null && selfUser.Id == user.Id,
|
||||||
|
token?.Scopes ?? [],
|
||||||
|
renderMembers,
|
||||||
|
renderAuthMethods,
|
||||||
|
overrideSid,
|
||||||
|
ct
|
||||||
|
);
|
||||||
|
|
||||||
|
public async Task<UserResponse> RenderUserInnerAsync(
|
||||||
|
User user,
|
||||||
|
bool isSelfUser,
|
||||||
|
string[] scopes,
|
||||||
|
bool renderMembers = true,
|
||||||
|
bool renderAuthMethods = false,
|
||||||
|
string? overrideSid = null,
|
||||||
|
CancellationToken ct = default
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
var isSelfUser = selfUser?.Id == user.Id;
|
scopes = scopes.ExpandScopes();
|
||||||
var tokenCanReadHiddenMembers = token.HasScope("member.read") && isSelfUser;
|
var tokenCanReadHiddenMembers = scopes.Contains("member.read") && isSelfUser;
|
||||||
var tokenHidden = token.HasScope("user.read_hidden") && isSelfUser;
|
var tokenHidden = scopes.Contains("user.read_hidden") && isSelfUser;
|
||||||
var tokenPrivileged = token.HasScope("user.read_privileged") && isSelfUser;
|
var tokenPrivileged = scopes.Contains("user.read_privileged") && isSelfUser;
|
||||||
|
|
||||||
renderMembers = renderMembers && (!user.ListHidden || tokenCanReadHiddenMembers);
|
renderMembers = renderMembers && (!user.ListHidden || tokenCanReadHiddenMembers);
|
||||||
renderAuthMethods = renderAuthMethods && tokenPrivileged;
|
renderAuthMethods = renderAuthMethods && tokenPrivileged;
|
||||||
|
@ -105,12 +124,12 @@ public class UserRendererService(
|
||||||
user.CustomPreferences
|
user.CustomPreferences
|
||||||
);
|
);
|
||||||
|
|
||||||
private string? AvatarUrlFor(User user) =>
|
public string? AvatarUrlFor(User user) =>
|
||||||
user.Avatar != null
|
user.Avatar != null
|
||||||
? $"{config.MediaBaseUrl}/users/{user.Id}/avatars/{user.Avatar}.webp"
|
? $"{config.MediaBaseUrl}/users/{user.Id}/avatars/{user.Avatar}.webp"
|
||||||
: null;
|
: null;
|
||||||
|
|
||||||
private string ImageUrlFor(PrideFlag flag) => $"{config.MediaBaseUrl}/flags/{flag.Hash}.webp";
|
public string ImageUrlFor(PrideFlag flag) => $"{config.MediaBaseUrl}/flags/{flag.Hash}.webp";
|
||||||
|
|
||||||
public record UserResponse(
|
public record UserResponse(
|
||||||
Snowflake Id,
|
Snowflake Id,
|
||||||
|
|
|
@ -1,13 +1,24 @@
|
||||||
import { apiRequest } from "$api";
|
import { apiRequest } from "$api";
|
||||||
import type { PartialMember, UserWithMembers } from "$api/models";
|
import ApiError, { ErrorCode } from "$api/error.js";
|
||||||
|
import type { PartialMember, User, UserWithMembers } from "$api/models";
|
||||||
|
import log from "$lib/log.js";
|
||||||
|
import { error } from "@sveltejs/kit";
|
||||||
|
|
||||||
const MEMBERS_PER_PAGE = 20;
|
const MEMBERS_PER_PAGE = 20;
|
||||||
|
|
||||||
export const load = async ({ params, fetch, cookies, url }) => {
|
export const load = async ({ params, fetch, cookies, url }) => {
|
||||||
const user = await apiRequest<UserWithMembers>("GET", `/users/${params.username}`, {
|
let user: UserWithMembers;
|
||||||
fetch,
|
|
||||||
cookies,
|
try {
|
||||||
});
|
user = await apiRequest<UserWithMembers>("GET", `/users/${params.username}`, {
|
||||||
|
fetch,
|
||||||
|
cookies,
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
if (e instanceof ApiError && e.code === ErrorCode.UserNotFound) error(404, "User not found");
|
||||||
|
log.error("Error fetching user %s:", params.username, e);
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
|
||||||
// Paginate members on the server side
|
// Paginate members on the server side
|
||||||
let currentPage = 0;
|
let currentPage = 0;
|
||||||
|
|
|
@ -1,15 +1,28 @@
|
||||||
import { apiRequest } from "$api";
|
import { apiRequest } from "$api";
|
||||||
|
import ApiError, { ErrorCode } from "$api/error.js";
|
||||||
import type { Member } from "$api/models/member";
|
import type { Member } from "$api/models/member";
|
||||||
|
import log from "$lib/log.js";
|
||||||
|
import { error } from "@sveltejs/kit";
|
||||||
|
|
||||||
export const load = async ({ params, fetch, cookies }) => {
|
export const load = async ({ params, fetch, cookies }) => {
|
||||||
const member = await apiRequest<Member>(
|
try {
|
||||||
"GET",
|
const member = await apiRequest<Member>(
|
||||||
`/users/${params.username}/members/${params.memberName}`,
|
"GET",
|
||||||
{
|
`/users/${params.username}/members/${params.memberName}`,
|
||||||
fetch,
|
{
|
||||||
cookies,
|
fetch,
|
||||||
},
|
cookies,
|
||||||
);
|
},
|
||||||
|
);
|
||||||
|
|
||||||
return { member };
|
return { member };
|
||||||
|
} catch (e) {
|
||||||
|
if (e instanceof ApiError) {
|
||||||
|
if (e.code === ErrorCode.UserNotFound) error(404, "User not found");
|
||||||
|
if (e.code === ErrorCode.MemberNotFound) error(404, "Member not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
log.error("Error fetching user %s/member %s:", params.username, params.memberName, e);
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
Loading…
Reference in a new issue