Compare commits
No commits in common. "71d3b42330244987ee147a9ddf3b8cea5e0d9e8b" and "f0ae648492e22e89bb42f0e9ae5dfbd8b5e0e8b9" have entirely different histories.
71d3b42330
...
f0ae648492
18 changed files with 38 additions and 617 deletions
|
|
@ -1,74 +0,0 @@
|
|||
using Coravel.Queuing.Interfaces;
|
||||
using Foxnouns.Backend.Database;
|
||||
using Foxnouns.Backend.Database.Models;
|
||||
using Foxnouns.Backend.Jobs;
|
||||
using Foxnouns.Backend.Middleware;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using NodaTime;
|
||||
|
||||
namespace Foxnouns.Backend.Controllers;
|
||||
|
||||
[Route("/api/internal/data-exports")]
|
||||
[Authorize("identify")]
|
||||
public class ExportsController(
|
||||
ILogger logger,
|
||||
Config config,
|
||||
IClock clock,
|
||||
DatabaseContext db,
|
||||
IQueue queue
|
||||
) : ApiControllerBase
|
||||
{
|
||||
private static readonly Duration MinimumTimeBetween = Duration.FromDays(1);
|
||||
private readonly ILogger _logger = logger.ForContext<ExportsController>();
|
||||
|
||||
[HttpGet]
|
||||
public async Task<IActionResult> GetDataExportsAsync()
|
||||
{
|
||||
var export = await db
|
||||
.DataExports.Where(d => d.UserId == CurrentUser!.Id)
|
||||
.OrderByDescending(d => d.Id)
|
||||
.FirstOrDefaultAsync();
|
||||
if (export == null)
|
||||
return Ok(new DataExportResponse(null, null));
|
||||
|
||||
return Ok(
|
||||
new DataExportResponse(
|
||||
ExportUrl(CurrentUser!.Id, export.Filename),
|
||||
export.Id.Time + DataExport.Expiration
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
private string ExportUrl(Snowflake userId, string filename) =>
|
||||
$"{config.MediaBaseUrl}/data-exports/{userId}/{filename}.zip";
|
||||
|
||||
private record DataExportResponse(string? Url, Instant? ExpiresAt);
|
||||
|
||||
[HttpPost]
|
||||
public async Task<IActionResult> QueueDataExportAsync()
|
||||
{
|
||||
var snowflakeToCheck = Snowflake.FromInstant(
|
||||
clock.GetCurrentInstant() - MinimumTimeBetween
|
||||
);
|
||||
_logger.Debug(
|
||||
"Checking if user {UserId} has data exports newer than {Snowflake}",
|
||||
CurrentUser!.Id,
|
||||
snowflakeToCheck
|
||||
);
|
||||
if (
|
||||
await db.DataExports.AnyAsync(d =>
|
||||
d.UserId == CurrentUser.Id && d.Id > snowflakeToCheck
|
||||
)
|
||||
)
|
||||
{
|
||||
throw new ApiError.BadRequest("You can't request a new data export so soon.");
|
||||
}
|
||||
|
||||
queue.QueueInvocableWithPayload<CreateDataExportInvocable, CreateDataExportPayload>(
|
||||
new CreateDataExportPayload(CurrentUser.Id)
|
||||
);
|
||||
|
||||
return NoContent();
|
||||
}
|
||||
}
|
||||
|
|
@ -46,18 +46,17 @@ public class DatabaseContext(DbContextOptions options) : DbContext(options)
|
|||
.UseSnakeCaseNamingConvention()
|
||||
.UseExceptionProcessor();
|
||||
|
||||
public DbSet<User> Users { get; init; }
|
||||
public DbSet<Member> Members { get; init; }
|
||||
public DbSet<AuthMethod> AuthMethods { get; init; }
|
||||
public DbSet<FediverseApplication> FediverseApplications { get; init; }
|
||||
public DbSet<Token> Tokens { get; init; }
|
||||
public DbSet<Application> Applications { get; init; }
|
||||
public DbSet<TemporaryKey> TemporaryKeys { get; init; }
|
||||
public DbSet<DataExport> DataExports { get; init; }
|
||||
public DbSet<User> Users { get; set; }
|
||||
public DbSet<Member> Members { get; set; }
|
||||
public DbSet<AuthMethod> AuthMethods { get; set; }
|
||||
public DbSet<FediverseApplication> FediverseApplications { get; set; }
|
||||
public DbSet<Token> Tokens { get; set; }
|
||||
public DbSet<Application> Applications { get; set; }
|
||||
public DbSet<TemporaryKey> TemporaryKeys { get; set; }
|
||||
|
||||
public DbSet<PrideFlag> PrideFlags { get; init; }
|
||||
public DbSet<UserFlag> UserFlags { get; init; }
|
||||
public DbSet<MemberFlag> MemberFlags { get; init; }
|
||||
public DbSet<PrideFlag> PrideFlags { get; set; }
|
||||
public DbSet<UserFlag> UserFlags { get; set; }
|
||||
public DbSet<MemberFlag> MemberFlags { get; set; }
|
||||
|
||||
protected override void ConfigureConventions(ModelConfigurationBuilder configurationBuilder)
|
||||
{
|
||||
|
|
@ -82,7 +81,6 @@ public class DatabaseContext(DbContextOptions options) : DbContext(options)
|
|||
})
|
||||
.HasFilter("fediverse_application_id IS NOT NULL")
|
||||
.IsUnique();
|
||||
modelBuilder.Entity<DataExport>().HasIndex(d => d.Filename).IsUnique();
|
||||
|
||||
modelBuilder
|
||||
.Entity<AuthMethod>()
|
||||
|
|
|
|||
|
|
@ -1,57 +0,0 @@
|
|||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace Foxnouns.Backend.Database.Migrations
|
||||
{
|
||||
/// <inheritdoc />
|
||||
[DbContext(typeof(DatabaseContext))]
|
||||
[Migration("20241202153736_AddDataExports")]
|
||||
public partial class AddDataExports : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.CreateTable(
|
||||
name: "data_exports",
|
||||
columns: table => new
|
||||
{
|
||||
id = table.Column<long>(type: "bigint", nullable: false),
|
||||
user_id = table.Column<long>(type: "bigint", nullable: false),
|
||||
filename = table.Column<string>(type: "text", nullable: false),
|
||||
},
|
||||
constraints: table =>
|
||||
{
|
||||
table.PrimaryKey("pk_data_exports", x => x.id);
|
||||
table.ForeignKey(
|
||||
name: "fk_data_exports_users_user_id",
|
||||
column: x => x.user_id,
|
||||
principalTable: "users",
|
||||
principalColumn: "id",
|
||||
onDelete: ReferentialAction.Cascade
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "ix_data_exports_filename",
|
||||
table: "data_exports",
|
||||
column: "filename",
|
||||
unique: true
|
||||
);
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "ix_data_exports_user_id",
|
||||
table: "data_exports",
|
||||
column: "user_id"
|
||||
);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropTable(name: "data_exports");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -111,34 +111,6 @@ namespace Foxnouns.Backend.Database.Migrations
|
|||
b.ToTable("auth_methods", (string)null);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("Foxnouns.Backend.Database.Models.DataExport", b =>
|
||||
{
|
||||
b.Property<long>("Id")
|
||||
.HasColumnType("bigint")
|
||||
.HasColumnName("id");
|
||||
|
||||
b.Property<string>("Filename")
|
||||
.IsRequired()
|
||||
.HasColumnType("text")
|
||||
.HasColumnName("filename");
|
||||
|
||||
b.Property<long>("UserId")
|
||||
.HasColumnType("bigint")
|
||||
.HasColumnName("user_id");
|
||||
|
||||
b.HasKey("Id")
|
||||
.HasName("pk_data_exports");
|
||||
|
||||
b.HasIndex("Filename")
|
||||
.IsUnique()
|
||||
.HasDatabaseName("ix_data_exports_filename");
|
||||
|
||||
b.HasIndex("UserId")
|
||||
.HasDatabaseName("ix_data_exports_user_id");
|
||||
|
||||
b.ToTable("data_exports", (string)null);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("Foxnouns.Backend.Database.Models.FediverseApplication", b =>
|
||||
{
|
||||
b.Property<long>("Id")
|
||||
|
|
@ -543,18 +515,6 @@ namespace Foxnouns.Backend.Database.Migrations
|
|||
b.Navigation("User");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("Foxnouns.Backend.Database.Models.DataExport", b =>
|
||||
{
|
||||
b.HasOne("Foxnouns.Backend.Database.Models.User", "User")
|
||||
.WithMany()
|
||||
.HasForeignKey("UserId")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired()
|
||||
.HasConstraintName("fk_data_exports_users_user_id");
|
||||
|
||||
b.Navigation("User");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("Foxnouns.Backend.Database.Models.Member", b =>
|
||||
{
|
||||
b.HasOne("Foxnouns.Backend.Database.Models.User", "User")
|
||||
|
|
|
|||
|
|
@ -1,14 +0,0 @@
|
|||
using System.ComponentModel.DataAnnotations.Schema;
|
||||
using NodaTime;
|
||||
|
||||
namespace Foxnouns.Backend.Database.Models;
|
||||
|
||||
public class DataExport : BaseModel
|
||||
{
|
||||
public Snowflake UserId { get; init; }
|
||||
public User User { get; init; } = null!;
|
||||
public required string Filename { get; init; }
|
||||
|
||||
[NotMapped]
|
||||
public static readonly Duration Expiration = Duration.FromDays(15);
|
||||
}
|
||||
|
|
@ -30,7 +30,6 @@ public class User : BaseModel
|
|||
|
||||
public List<Member> Members { get; } = [];
|
||||
public List<AuthMethod> AuthMethods { get; } = [];
|
||||
public List<DataExport> DataExports { get; } = [];
|
||||
public UserSettings Settings { get; set; } = new();
|
||||
|
||||
public required Instant LastActive { get; set; }
|
||||
|
|
@ -54,12 +53,6 @@ public class User : BaseModel
|
|||
[JsonConverter(typeof(ScreamingSnakeCaseEnumConverter))]
|
||||
public PreferenceSize Size { get; set; }
|
||||
}
|
||||
|
||||
[NotMapped]
|
||||
public static readonly Duration DeleteAfter = Duration.FromDays(30);
|
||||
|
||||
[NotMapped]
|
||||
public static readonly Duration DeleteSuspendedAfter = Duration.FromDays(180);
|
||||
}
|
||||
|
||||
public enum UserRole
|
||||
|
|
|
|||
|
|
@ -65,9 +65,6 @@ public readonly struct Snowflake(ulong value) : IEquatable<Snowflake>
|
|||
return true;
|
||||
}
|
||||
|
||||
public static Snowflake FromInstant(Instant instant) =>
|
||||
new((ulong)(instant.ToUnixTimeMilliseconds() - Epoch) << 22);
|
||||
|
||||
public override bool Equals(object? obj) => obj is Snowflake other && Value == other.Value;
|
||||
|
||||
public bool Equals(Snowflake other)
|
||||
|
|
|
|||
|
|
@ -106,14 +106,12 @@ public static class WebApplicationExtensions
|
|||
.AddScoped<RemoteAuthService>()
|
||||
.AddScoped<FediverseAuthService>()
|
||||
.AddScoped<ObjectStorageService>()
|
||||
.AddTransient<DataCleanupService>()
|
||||
// Background services
|
||||
.AddHostedService<PeriodicTasksService>()
|
||||
// Transient jobs
|
||||
.AddTransient<MemberAvatarUpdateInvocable>()
|
||||
.AddTransient<UserAvatarUpdateInvocable>()
|
||||
.AddTransient<CreateFlagInvocable>()
|
||||
.AddTransient<CreateDataExportInvocable>();
|
||||
.AddTransient<CreateFlagInvocable>();
|
||||
|
||||
if (!config.Logging.EnableMetrics)
|
||||
services.AddHostedService<BackgroundMetricsCollectionService>();
|
||||
|
|
|
|||
|
|
@ -1,210 +0,0 @@
|
|||
using System.IO.Compression;
|
||||
using System.Net;
|
||||
using Coravel.Invocable;
|
||||
using Foxnouns.Backend.Database;
|
||||
using Foxnouns.Backend.Database.Models;
|
||||
using Foxnouns.Backend.Services;
|
||||
using Foxnouns.Backend.Utils;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Newtonsoft.Json;
|
||||
using NodaTime;
|
||||
using NodaTime.Text;
|
||||
|
||||
namespace Foxnouns.Backend.Jobs;
|
||||
|
||||
public class CreateDataExportInvocable(
|
||||
DatabaseContext db,
|
||||
IClock clock,
|
||||
UserRendererService userRenderer,
|
||||
MemberRendererService memberRenderer,
|
||||
ObjectStorageService objectStorageService,
|
||||
ISnowflakeGenerator snowflakeGenerator,
|
||||
ILogger logger
|
||||
) : IInvocable, IInvocableWithPayload<CreateDataExportPayload>
|
||||
{
|
||||
private static readonly HttpClient Client = new();
|
||||
private readonly ILogger _logger = logger.ForContext<CreateDataExportInvocable>();
|
||||
public required CreateDataExportPayload Payload { get; set; }
|
||||
|
||||
public async Task Invoke()
|
||||
{
|
||||
try
|
||||
{
|
||||
await InvokeAsync();
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.Error(e, "Error generating data export for user {UserId}", Payload.UserId);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task InvokeAsync()
|
||||
{
|
||||
var user = await db
|
||||
.Users.Include(u => u.AuthMethods)
|
||||
.Include(u => u.Flags)
|
||||
.Include(u => u.ProfileFlags)
|
||||
.AsSplitQuery()
|
||||
.FirstOrDefaultAsync(u => u.Id == Payload.UserId);
|
||||
if (user == null)
|
||||
{
|
||||
_logger.Warning(
|
||||
"Received create data export request for user {UserId} but no such user exists, deleted or otherwise. Ignoring request",
|
||||
Payload.UserId
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.Information("Generating data export for user {UserId}", user.Id);
|
||||
|
||||
using var stream = new MemoryStream();
|
||||
using var zip = new ZipArchive(stream, ZipArchiveMode.Create, true);
|
||||
zip.Comment =
|
||||
$"This archive for {user.Username} ({user.Id}) was generated at {InstantPattern.General.Format(clock.GetCurrentInstant())}";
|
||||
|
||||
// Write the user's info and avatar
|
||||
WriteJson(
|
||||
zip,
|
||||
"user.json",
|
||||
await userRenderer.RenderUserInnerAsync(
|
||||
user,
|
||||
true,
|
||||
["*"],
|
||||
renderMembers: false,
|
||||
renderAuthMethods: true
|
||||
)
|
||||
);
|
||||
await WriteS3Object(zip, "user-avatar.webp", userRenderer.AvatarUrlFor(user));
|
||||
|
||||
foreach (var flag in user.Flags)
|
||||
await WritePrideFlag(zip, flag);
|
||||
|
||||
var members = await db
|
||||
.Members.Include(m => m.User)
|
||||
.Include(m => m.ProfileFlags)
|
||||
.Where(m => m.UserId == user.Id)
|
||||
.ToListAsync();
|
||||
foreach (var member in members)
|
||||
await WriteMember(zip, member);
|
||||
|
||||
// We want to dispose the ZipArchive on an error, but we need to dispose it manually to upload to object storage.
|
||||
// Calling Dispose() multiple times is fine for this class, though.
|
||||
// ReSharper disable once DisposeOnUsingVariable
|
||||
zip.Dispose();
|
||||
stream.Seek(0, SeekOrigin.Begin);
|
||||
|
||||
// Upload the file!
|
||||
var filename = AuthUtils.RandomToken().Replace('+', '-').Replace('/', '_');
|
||||
await objectStorageService.PutObjectAsync(
|
||||
ExportPath(user.Id, filename),
|
||||
stream,
|
||||
"application/zip"
|
||||
);
|
||||
|
||||
db.Add(
|
||||
new DataExport
|
||||
{
|
||||
Id = snowflakeGenerator.GenerateSnowflake(),
|
||||
UserId = user.Id,
|
||||
Filename = filename,
|
||||
}
|
||||
);
|
||||
await db.SaveChangesAsync();
|
||||
}
|
||||
|
||||
private async Task WritePrideFlag(ZipArchive zip, PrideFlag flag)
|
||||
{
|
||||
_logger.Debug("Writing flag {FlagId}", flag.Id);
|
||||
|
||||
var flagData = $"""
|
||||
{flag.Name}
|
||||
----
|
||||
{flag.Description ?? "<no description>"}
|
||||
""";
|
||||
|
||||
try
|
||||
{
|
||||
await WriteS3Object(zip, $"flag-{flag.Id}/flag.webp", userRenderer.ImageUrlFor(flag));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.Warning(e, "Could not write image for flag {FlagId}", flag.Id);
|
||||
return;
|
||||
}
|
||||
|
||||
var entry = zip.CreateEntry($"flag-{flag.Id}/flag.txt");
|
||||
await using var stream = entry.Open();
|
||||
await using var writer = new StreamWriter(stream);
|
||||
await writer.WriteAsync(flagData);
|
||||
}
|
||||
|
||||
private async Task WriteMember(ZipArchive zip, Member member)
|
||||
{
|
||||
_logger.Debug("Writing member {MemberId}", member.Id);
|
||||
|
||||
WriteJson(
|
||||
zip,
|
||||
$"members/{member.Name} ({member.Id}).json",
|
||||
memberRenderer.RenderMember(member)
|
||||
);
|
||||
|
||||
try
|
||||
{
|
||||
await WriteS3Object(
|
||||
zip,
|
||||
$"members/{member.Name} ({member.Id}).webp",
|
||||
memberRenderer.AvatarUrlFor(member)
|
||||
);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.Warning(e, "Error writing avatar for member {MemberId}", member.Id);
|
||||
}
|
||||
}
|
||||
|
||||
private void WriteJson(ZipArchive zip, string filename, object data)
|
||||
{
|
||||
var json = JsonConvert.SerializeObject(data, Formatting.Indented);
|
||||
|
||||
_logger.Debug(
|
||||
"Writing file {Filename} to archive with size {Length}",
|
||||
filename,
|
||||
json.Length
|
||||
);
|
||||
|
||||
var entry = zip.CreateEntry(filename);
|
||||
using var stream = entry.Open();
|
||||
using var writer = new StreamWriter(stream);
|
||||
writer.Write(json);
|
||||
}
|
||||
|
||||
private async Task WriteS3Object(ZipArchive zip, string filename, string? s3Path)
|
||||
{
|
||||
if (s3Path == null)
|
||||
return;
|
||||
|
||||
var resp = await Client.GetAsync(s3Path);
|
||||
if (resp.StatusCode != HttpStatusCode.OK)
|
||||
{
|
||||
_logger.Warning("S3 path {S3Path} returned a non-200 status, not saving file", s3Path);
|
||||
return;
|
||||
}
|
||||
|
||||
await using var respStream = await resp.Content.ReadAsStreamAsync();
|
||||
|
||||
_logger.Debug(
|
||||
"Writing file {Filename} to archive with size {Length}",
|
||||
filename,
|
||||
respStream.Length
|
||||
);
|
||||
|
||||
var entry = zip.CreateEntry(filename);
|
||||
await using var entryStream = entry.Open();
|
||||
|
||||
respStream.Seek(0, SeekOrigin.Begin);
|
||||
await respStream.CopyToAsync(entryStream);
|
||||
}
|
||||
|
||||
private static string ExportPath(Snowflake userId, string b64) =>
|
||||
$"data-exports/{userId}/{b64}.zip";
|
||||
}
|
||||
|
|
@ -11,5 +11,3 @@ public record CreateFlagPayload(
|
|||
string ImageData,
|
||||
string? Description
|
||||
);
|
||||
|
||||
public record CreateDataExportPayload(Snowflake UserId);
|
||||
|
|
|
|||
|
|
@ -1,116 +0,0 @@
|
|||
using System.Diagnostics;
|
||||
using Foxnouns.Backend.Database;
|
||||
using Foxnouns.Backend.Database.Models;
|
||||
using Foxnouns.Backend.Extensions;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using NodaTime;
|
||||
using NodaTime.Extensions;
|
||||
|
||||
namespace Foxnouns.Backend.Services;
|
||||
|
||||
public class DataCleanupService(
|
||||
DatabaseContext db,
|
||||
IClock clock,
|
||||
ILogger logger,
|
||||
ObjectStorageService objectStorageService
|
||||
)
|
||||
{
|
||||
private readonly ILogger _logger = logger.ForContext<DataCleanupService>();
|
||||
|
||||
public async Task InvokeAsync(CancellationToken ct = default)
|
||||
{
|
||||
_logger.Information("Cleaning up expired users");
|
||||
await CleanUsersAsync(ct);
|
||||
|
||||
_logger.Information("Cleaning up expired data exports");
|
||||
await CleanExportsAsync(ct);
|
||||
}
|
||||
|
||||
private async Task CleanUsersAsync(CancellationToken ct = default)
|
||||
{
|
||||
var selfDeleteExpires = clock.GetCurrentInstant() - User.DeleteAfter;
|
||||
var suspendExpires = clock.GetCurrentInstant() - User.DeleteSuspendedAfter;
|
||||
var users = await db
|
||||
.Users.Include(u => u.Members)
|
||||
.Include(u => u.DataExports)
|
||||
.Where(u =>
|
||||
u.Deleted
|
||||
&& (
|
||||
(u.DeletedBy != null && u.DeletedAt < suspendExpires)
|
||||
|| (u.DeletedBy == null && u.DeletedAt < selfDeleteExpires)
|
||||
)
|
||||
)
|
||||
.OrderBy(u => u.Id)
|
||||
.AsSplitQuery()
|
||||
.ToListAsync(ct);
|
||||
if (users.Count == 0)
|
||||
return;
|
||||
|
||||
_logger.Debug(
|
||||
"Deleting {Count} users that have been deleted for over 30 days or suspended for over 180 days",
|
||||
users.Count
|
||||
);
|
||||
|
||||
var sw = new Stopwatch();
|
||||
|
||||
await Task.WhenAll(users.Select(u => CleanUserAsync(u, ct)));
|
||||
|
||||
await db.SaveChangesAsync(ct);
|
||||
_logger.Information(
|
||||
"Deleted {Count} users, their members, and their exports in {Time}",
|
||||
users.Count,
|
||||
sw.ElapsedDuration()
|
||||
);
|
||||
}
|
||||
|
||||
private Task CleanUserAsync(User user, CancellationToken ct = default)
|
||||
{
|
||||
var tasks = new List<Task>();
|
||||
|
||||
if (user.Avatar != null)
|
||||
tasks.Add(objectStorageService.DeleteUserAvatarAsync(user.Id, user.Avatar, ct));
|
||||
|
||||
tasks.AddRange(
|
||||
user.Members.Select(member =>
|
||||
objectStorageService.DeleteMemberAvatarAsync(member.Id, member.Avatar!, ct)
|
||||
)
|
||||
);
|
||||
|
||||
tasks.AddRange(
|
||||
user.DataExports.Select(export =>
|
||||
objectStorageService.RemoveObjectAsync(
|
||||
ExportPath(export.UserId, export.Filename),
|
||||
ct
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
db.Remove(user);
|
||||
return Task.WhenAll(tasks);
|
||||
}
|
||||
|
||||
private async Task CleanExportsAsync(CancellationToken ct = default)
|
||||
{
|
||||
var minExpiredId = Snowflake.FromInstant(clock.GetCurrentInstant() - DataExport.Expiration);
|
||||
var exports = await db.DataExports.Where(d => d.Id < minExpiredId).ToListAsync(ct);
|
||||
if (exports.Count == 0)
|
||||
return;
|
||||
|
||||
_logger.Debug("Deleting {Count} expired exports", exports.Count);
|
||||
|
||||
foreach (var export in exports)
|
||||
{
|
||||
_logger.Debug("Deleting export {ExportId}", export.Id);
|
||||
await objectStorageService.RemoveObjectAsync(
|
||||
ExportPath(export.UserId, export.Filename),
|
||||
ct
|
||||
);
|
||||
db.Remove(export);
|
||||
}
|
||||
|
||||
await db.SaveChangesAsync(ct);
|
||||
}
|
||||
|
||||
private static string ExportPath(Snowflake userId, string b64) =>
|
||||
$"data-exports/{userId}/{b64}.zip";
|
||||
}
|
||||
|
|
@ -72,7 +72,7 @@ public class MemberRendererService(DatabaseContext db, Config config)
|
|||
renderUnlisted ? member.Unlisted : null
|
||||
);
|
||||
|
||||
public string? AvatarUrlFor(Member member) =>
|
||||
private string? AvatarUrlFor(Member member) =>
|
||||
member.Avatar != null
|
||||
? $"{config.MediaBaseUrl}/members/{member.Id}/avatars/{member.Avatar}.webp"
|
||||
: null;
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ public class BackgroundMetricsCollectionService(
|
|||
using var timer = new PeriodicTimer(TimeSpan.FromMinutes(1));
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
_logger.Debug("Collecting metrics manually");
|
||||
_logger.Debug("Collecting metrics");
|
||||
await metricsCollectionService.CollectMetricsAsync(ct);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,13 +48,4 @@ public class ObjectStorageService(ILogger logger, Config config, IMinioClient mi
|
|||
ct
|
||||
);
|
||||
}
|
||||
|
||||
public async Task GetObjectAsync(string path, CancellationToken ct = default)
|
||||
{
|
||||
var stream = new MemoryStream();
|
||||
var resp = await minioClient.GetObjectAsync(
|
||||
new GetObjectArgs().WithBucket(config.Storage.Bucket).WithObject(path),
|
||||
ct
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,10 @@ public class PeriodicTasksService(ILogger logger, IServiceProvider services) : B
|
|||
{
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromMinutes(1));
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
_logger.Debug("Collecting metrics");
|
||||
await RunPeriodicTasksAsync(ct);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RunPeriodicTasksAsync(CancellationToken ct)
|
||||
|
|
@ -17,10 +20,7 @@ public class PeriodicTasksService(ILogger logger, IServiceProvider services) : B
|
|||
|
||||
await using var scope = services.CreateAsyncScope();
|
||||
|
||||
var keyCacheService = scope.ServiceProvider.GetRequiredService<KeyCacheService>();
|
||||
var dataCleanupService = scope.ServiceProvider.GetRequiredService<DataCleanupService>();
|
||||
|
||||
await keyCacheService.DeleteExpiredKeysAsync(ct);
|
||||
await dataCleanupService.InvokeAsync(ct);
|
||||
var keyCacheSvc = scope.ServiceProvider.GetRequiredService<KeyCacheService>();
|
||||
await keyCacheSvc.DeleteExpiredKeysAsync(ct);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,31 +22,12 @@ public class UserRendererService(
|
|||
bool renderAuthMethods = false,
|
||||
string? overrideSid = null,
|
||||
CancellationToken ct = default
|
||||
) =>
|
||||
await RenderUserInnerAsync(
|
||||
user,
|
||||
selfUser != null && selfUser.Id == user.Id,
|
||||
token?.Scopes ?? [],
|
||||
renderMembers,
|
||||
renderAuthMethods,
|
||||
overrideSid,
|
||||
ct
|
||||
);
|
||||
|
||||
public async Task<UserResponse> RenderUserInnerAsync(
|
||||
User user,
|
||||
bool isSelfUser,
|
||||
string[] scopes,
|
||||
bool renderMembers = true,
|
||||
bool renderAuthMethods = false,
|
||||
string? overrideSid = null,
|
||||
CancellationToken ct = default
|
||||
)
|
||||
{
|
||||
scopes = scopes.ExpandScopes();
|
||||
var tokenCanReadHiddenMembers = scopes.Contains("member.read") && isSelfUser;
|
||||
var tokenHidden = scopes.Contains("user.read_hidden") && isSelfUser;
|
||||
var tokenPrivileged = scopes.Contains("user.read_privileged") && isSelfUser;
|
||||
var isSelfUser = selfUser?.Id == user.Id;
|
||||
var tokenCanReadHiddenMembers = token.HasScope("member.read") && isSelfUser;
|
||||
var tokenHidden = token.HasScope("user.read_hidden") && isSelfUser;
|
||||
var tokenPrivileged = token.HasScope("user.read_privileged") && isSelfUser;
|
||||
|
||||
renderMembers = renderMembers && (!user.ListHidden || tokenCanReadHiddenMembers);
|
||||
renderAuthMethods = renderAuthMethods && tokenPrivileged;
|
||||
|
|
@ -124,12 +105,12 @@ public class UserRendererService(
|
|||
user.CustomPreferences
|
||||
);
|
||||
|
||||
public string? AvatarUrlFor(User user) =>
|
||||
private string? AvatarUrlFor(User user) =>
|
||||
user.Avatar != null
|
||||
? $"{config.MediaBaseUrl}/users/{user.Id}/avatars/{user.Avatar}.webp"
|
||||
: null;
|
||||
|
||||
public string ImageUrlFor(PrideFlag flag) => $"{config.MediaBaseUrl}/flags/{flag.Hash}.webp";
|
||||
private string ImageUrlFor(PrideFlag flag) => $"{config.MediaBaseUrl}/flags/{flag.Hash}.webp";
|
||||
|
||||
public record UserResponse(
|
||||
Snowflake Id,
|
||||
|
|
|
|||
|
|
@ -1,24 +1,13 @@
|
|||
import { apiRequest } from "$api";
|
||||
import ApiError, { ErrorCode } from "$api/error.js";
|
||||
import type { PartialMember, User, UserWithMembers } from "$api/models";
|
||||
import log from "$lib/log.js";
|
||||
import { error } from "@sveltejs/kit";
|
||||
import type { PartialMember, UserWithMembers } from "$api/models";
|
||||
|
||||
const MEMBERS_PER_PAGE = 20;
|
||||
|
||||
export const load = async ({ params, fetch, cookies, url }) => {
|
||||
let user: UserWithMembers;
|
||||
|
||||
try {
|
||||
user = await apiRequest<UserWithMembers>("GET", `/users/${params.username}`, {
|
||||
fetch,
|
||||
cookies,
|
||||
});
|
||||
} catch (e) {
|
||||
if (e instanceof ApiError && e.code === ErrorCode.UserNotFound) error(404, "User not found");
|
||||
log.error("Error fetching user %s:", params.username, e);
|
||||
throw e;
|
||||
}
|
||||
const user = await apiRequest<UserWithMembers>("GET", `/users/${params.username}`, {
|
||||
fetch,
|
||||
cookies,
|
||||
});
|
||||
|
||||
// Paginate members on the server side
|
||||
let currentPage = 0;
|
||||
|
|
|
|||
|
|
@ -1,28 +1,15 @@
|
|||
import { apiRequest } from "$api";
|
||||
import ApiError, { ErrorCode } from "$api/error.js";
|
||||
import type { Member } from "$api/models/member";
|
||||
import log from "$lib/log.js";
|
||||
import { error } from "@sveltejs/kit";
|
||||
|
||||
export const load = async ({ params, fetch, cookies }) => {
|
||||
try {
|
||||
const member = await apiRequest<Member>(
|
||||
"GET",
|
||||
`/users/${params.username}/members/${params.memberName}`,
|
||||
{
|
||||
fetch,
|
||||
cookies,
|
||||
},
|
||||
);
|
||||
const member = await apiRequest<Member>(
|
||||
"GET",
|
||||
`/users/${params.username}/members/${params.memberName}`,
|
||||
{
|
||||
fetch,
|
||||
cookies,
|
||||
},
|
||||
);
|
||||
|
||||
return { member };
|
||||
} catch (e) {
|
||||
if (e instanceof ApiError) {
|
||||
if (e.code === ErrorCode.UserNotFound) error(404, "User not found");
|
||||
if (e.code === ErrorCode.MemberNotFound) error(404, "Member not found");
|
||||
}
|
||||
|
||||
log.error("Error fetching user %s/member %s:", params.username, params.memberName, e);
|
||||
throw e;
|
||||
}
|
||||
return { member };
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in a new issue