From 0aadc5fb47f06f3cf2a80b45e0b70efc201e5b8e Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 3 Sep 2024 16:29:51 +0200 Subject: [PATCH] feat: replace Hangfire with Coravel --- .gitignore | 1 + Foxnouns.Backend/Config.cs | 7 - .../Authentication/AuthController.cs | 2 +- .../Controllers/MembersController.cs | 11 +- .../Controllers/UsersController.cs | 7 +- .../Extensions/AvatarObjectExtensions.cs | 51 ++++ .../Extensions/WebApplicationExtensions.cs | 11 +- Foxnouns.Backend/Foxnouns.Backend.csproj | 5 +- Foxnouns.Backend/Jobs/AvatarUpdateJob.cs | 220 ------------------ .../Jobs/MemberAvatarUpdateInvocable.cs | 79 +++++++ Foxnouns.Backend/Jobs/Payloads.cs | 5 + .../Jobs/UserAvatarUpdateInvocable.cs | 79 +++++++ .../Middleware/AuthenticationMiddleware.cs | 32 +-- Foxnouns.Backend/Program.cs | 18 +- .../Services/ObjectStorageService.cs | 33 +++ Foxnouns.Backend/Utils/Limits.cs | 9 + Foxnouns.Backend/Utils/ValidationUtils.cs | 31 ++- Foxnouns.Backend/appSettings.json | 7 + Foxnouns.Backend/config.example.ini | 6 - 19 files changed, 305 insertions(+), 309 deletions(-) create mode 100644 Foxnouns.Backend/Extensions/AvatarObjectExtensions.cs delete mode 100644 Foxnouns.Backend/Jobs/AvatarUpdateJob.cs create mode 100644 Foxnouns.Backend/Jobs/MemberAvatarUpdateInvocable.cs create mode 100644 Foxnouns.Backend/Jobs/Payloads.cs create mode 100644 Foxnouns.Backend/Jobs/UserAvatarUpdateInvocable.cs create mode 100644 Foxnouns.Backend/Services/ObjectStorageService.cs create mode 100644 Foxnouns.Backend/Utils/Limits.cs create mode 100644 Foxnouns.Backend/appSettings.json diff --git a/.gitignore b/.gitignore index 56d5d08..8e84a9d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ bin/ obj/ .version config.ini +*.DotSettings.user diff --git a/Foxnouns.Backend/Config.cs b/Foxnouns.Backend/Config.cs index 62a107f..140214d 100644 --- a/Foxnouns.Backend/Config.cs +++ b/Foxnouns.Backend/Config.cs @@ -14,7 +14,6 @@ public class Config public LoggingConfig Logging { get; init; } = new(); public DatabaseConfig Database { get; init; } = new(); - public JobsConfig Jobs { get; init; } = new(); public StorageConfig Storage { get; init; } = new(); public DiscordAuthConfig DiscordAuth { get; init; } = new(); public GoogleAuthConfig GoogleAuth { get; init; } = new(); @@ -38,12 +37,6 @@ public class Config public int? MaxPoolSize { get; init; } } - public class JobsConfig - { - public string Redis { get; init; } = string.Empty; - public int Workers { get; init; } = 5; - } - public class StorageConfig { public string Endpoint { get; init; } = string.Empty; diff --git a/Foxnouns.Backend/Controllers/Authentication/AuthController.cs b/Foxnouns.Backend/Controllers/Authentication/AuthController.cs index d944dd8..6565fba 100644 --- a/Foxnouns.Backend/Controllers/Authentication/AuthController.cs +++ b/Foxnouns.Backend/Controllers/Authentication/AuthController.cs @@ -19,7 +19,7 @@ public class AuthController(Config config, KeyCacheService keyCacheSvc, ILogger config.TumblrAuth.Enabled); var state = HttpUtility.UrlEncode(await keyCacheSvc.GenerateAuthStateAsync()); string? discord = null; - if (config.DiscordAuth.ClientId != null && config.DiscordAuth.ClientSecret != null) + if (config.DiscordAuth is { ClientId: not null, ClientSecret: not null }) discord = $"https://discord.com/oauth2/authorize?response_type=code" + $"&client_id={config.DiscordAuth.ClientId}&scope=identify" + diff --git a/Foxnouns.Backend/Controllers/MembersController.cs b/Foxnouns.Backend/Controllers/MembersController.cs index ec28ee5..19a9569 100644 --- a/Foxnouns.Backend/Controllers/MembersController.cs +++ b/Foxnouns.Backend/Controllers/MembersController.cs @@ -1,6 +1,8 @@ +using Coravel.Queuing.Interfaces; using EntityFramework.Exceptions.Common; using Foxnouns.Backend.Database; using Foxnouns.Backend.Database.Models; +using Foxnouns.Backend.Extensions; using Foxnouns.Backend.Jobs; using Foxnouns.Backend.Middleware; using Foxnouns.Backend.Services; @@ -16,7 +18,8 @@ public class MembersController( DatabaseContext db, MemberRendererService memberRendererService, ISnowflakeGenerator snowflakeGenerator, - AvatarUpdateJob avatarUpdate) : ApiControllerBase + ObjectStorageService objectStorage, + IQueue queue) : ApiControllerBase { private readonly ILogger _logger = logger.ForContext(); @@ -76,7 +79,9 @@ public class MembersController( throw new ApiError.BadRequest("A member with that name already exists", "name", req.Name); } - if (req.Avatar != null) AvatarUpdateJob.QueueUpdateMemberAvatar(member.Id, req.Avatar); + if (req.Avatar != null) + queue.QueueInvocableWithPayload( + new AvatarUpdatePayload(member.Id, req.Avatar)); return Ok(memberRendererService.RenderMember(member, CurrentToken)); } @@ -96,7 +101,7 @@ public class MembersController( await db.SaveChangesAsync(); - if (member.Avatar != null) await avatarUpdate.DeleteMemberAvatar(member.Id, member.Avatar); + if (member.Avatar != null) await objectStorage.DeleteMemberAvatarAsync(member.Id, member.Avatar); return NoContent(); } diff --git a/Foxnouns.Backend/Controllers/UsersController.cs b/Foxnouns.Backend/Controllers/UsersController.cs index 19bb88a..0b8f2f2 100644 --- a/Foxnouns.Backend/Controllers/UsersController.cs +++ b/Foxnouns.Backend/Controllers/UsersController.cs @@ -1,4 +1,5 @@ using System.Diagnostics.CodeAnalysis; +using Coravel.Queuing.Interfaces; using Foxnouns.Backend.Database; using Foxnouns.Backend.Database.Models; using Foxnouns.Backend.Jobs; @@ -14,7 +15,8 @@ namespace Foxnouns.Backend.Controllers; public class UsersController( DatabaseContext db, UserRendererService userRendererService, - ISnowflakeGenerator snowflakeGenerator) : ApiControllerBase + ISnowflakeGenerator snowflakeGenerator, + IQueue queue) : ApiControllerBase { [HttpGet("{userRef}")] [ProducesResponseType(statusCode: StatusCodes.Status200OK)] @@ -70,7 +72,8 @@ public class UsersController( // (atomic operations are hard when combined with background jobs) // so it's in a separate block to the validation above. if (req.HasProperty(nameof(req.Avatar))) - AvatarUpdateJob.QueueUpdateUserAvatar(CurrentUser!.Id, req.Avatar); + queue.QueueInvocableWithPayload( + new AvatarUpdatePayload(CurrentUser!.Id, req.Avatar)); await db.SaveChangesAsync(); await tx.CommitAsync(); diff --git a/Foxnouns.Backend/Extensions/AvatarObjectExtensions.cs b/Foxnouns.Backend/Extensions/AvatarObjectExtensions.cs new file mode 100644 index 0000000..f7c2b6f --- /dev/null +++ b/Foxnouns.Backend/Extensions/AvatarObjectExtensions.cs @@ -0,0 +1,51 @@ +using Foxnouns.Backend.Database; +using Foxnouns.Backend.Jobs; +using Foxnouns.Backend.Services; +using Foxnouns.Backend.Utils; +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.Formats.Webp; +using SixLabors.ImageSharp.Processing; +using SixLabors.ImageSharp.Processing.Processors.Transforms; + +namespace Foxnouns.Backend.Extensions; + +public static class AvatarObjectExtensions +{ + private static readonly string[] ValidContentTypes = ["image/png", "image/webp", "image/jpeg"]; + + public static async Task + DeleteMemberAvatarAsync(this ObjectStorageService objectStorage, Snowflake id, string hash) => + await objectStorage.RemoveObjectAsync(MemberAvatarUpdateInvocable.Path(id, hash)); + + public static async Task + DeleteUserAvatarAsync(this ObjectStorageService objectStorage, Snowflake id, string hash) => + await objectStorage.RemoveObjectAsync(UserAvatarUpdateInvocable.Path(id, hash)); + + public static async Task ConvertBase64UriToAvatar(this string uri) + { + if (!uri.StartsWith("data:image/")) + throw new ArgumentException("Not a data URI", nameof(uri)); + + var split = uri.Remove(0, "data:".Length).Split(";base64,"); + var contentType = split[0]; + var encoded = split[1]; + if (!ValidContentTypes.Contains(contentType)) + throw new ArgumentException("Invalid content type for image", nameof(uri)); + + if (!AuthUtils.TryFromBase64String(encoded, out var rawImage)) + throw new ArgumentException("Invalid base64 string", nameof(uri)); + + var image = Image.Load(rawImage); + + var processor = new ResizeProcessor( + new ResizeOptions { Size = new Size(512), Mode = ResizeMode.Crop, Position = AnchorPositionMode.Center }, + image.Size + ); + + image.Mutate(x => x.ApplyProcessor(processor)); + + var stream = new MemoryStream(64 * 1024); + await image.SaveAsync(stream, new WebpEncoder { Quality = 95, NearLossless = false }); + return stream; + } +} \ No newline at end of file diff --git a/Foxnouns.Backend/Extensions/WebApplicationExtensions.cs b/Foxnouns.Backend/Extensions/WebApplicationExtensions.cs index d1ee8fc..1915eae 100644 --- a/Foxnouns.Backend/Extensions/WebApplicationExtensions.cs +++ b/Foxnouns.Backend/Extensions/WebApplicationExtensions.cs @@ -1,6 +1,8 @@ using App.Metrics; using App.Metrics.AspNetCore; using App.Metrics.Formatters.Prometheus; +using Coravel; +using Coravel.Queuing.Interfaces; using Foxnouns.Backend.Database; using Foxnouns.Backend.Jobs; using Foxnouns.Backend.Middleware; @@ -93,6 +95,7 @@ public static class WebApplicationExtensions return builder .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appSettings.json", true) .AddIniFile(file, optional: false, reloadOnChange: true) .AddEnvironmentVariables(); } @@ -105,8 +108,10 @@ public static class WebApplicationExtensions .AddScoped() .AddScoped() .AddScoped() - // Background job classes - .AddTransient(); + .AddScoped() + // Transient jobs + .AddTransient() + .AddTransient(); public static IServiceCollection AddCustomMiddleware(this IServiceCollection services) => services .AddScoped() @@ -122,6 +127,8 @@ public static class WebApplicationExtensions { await BuildInfo.ReadBuildInfo(); + app.Services.ConfigureQueue().LogQueuedTaskProgress(app.Services.GetRequiredService>()); + await using var scope = app.Services.CreateAsyncScope(); var logger = scope.ServiceProvider.GetRequiredService().ForContext(); var db = scope.ServiceProvider.GetRequiredService(); diff --git a/Foxnouns.Backend/Foxnouns.Backend.csproj b/Foxnouns.Backend/Foxnouns.Backend.csproj index 29c91d7..711e620 100644 --- a/Foxnouns.Backend/Foxnouns.Backend.csproj +++ b/Foxnouns.Backend/Foxnouns.Backend.csproj @@ -9,11 +9,9 @@ + - - - @@ -28,7 +26,6 @@ - diff --git a/Foxnouns.Backend/Jobs/AvatarUpdateJob.cs b/Foxnouns.Backend/Jobs/AvatarUpdateJob.cs deleted file mode 100644 index bf3d35d..0000000 --- a/Foxnouns.Backend/Jobs/AvatarUpdateJob.cs +++ /dev/null @@ -1,220 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using System.Security.Cryptography; -using Foxnouns.Backend.Database; -using Foxnouns.Backend.Utils; -using Hangfire; -using Minio; -using Minio.DataModel.Args; -using Minio.Exceptions; -using SixLabors.ImageSharp; -using SixLabors.ImageSharp.Formats.Webp; -using SixLabors.ImageSharp.Processing; -using SixLabors.ImageSharp.Processing.Processors.Transforms; - -namespace Foxnouns.Backend.Jobs; - -[SuppressMessage("ReSharper", "MemberCanBePrivate.Global", Justification = "Hangfire jobs need to be public")] -public class AvatarUpdateJob(DatabaseContext db, IMinioClient minio, Config config, ILogger logger) -{ - private readonly string[] _validContentTypes = ["image/png", "image/webp", "image/jpeg"]; - - public static void QueueUpdateUserAvatar(Snowflake id, string? newAvatar) - { - if (newAvatar != null) - BackgroundJob.Enqueue(job => job.UpdateUserAvatar(id, newAvatar)); - else - BackgroundJob.Enqueue(job => job.ClearUserAvatar(id)); - } - - public static void QueueUpdateMemberAvatar(Snowflake id, string? newAvatar) - { - if (newAvatar != null) - BackgroundJob.Enqueue(job => job.UpdateMemberAvatar(id, newAvatar)); - else - BackgroundJob.Enqueue(job => job.ClearMemberAvatar(id)); - } - - public async Task UpdateUserAvatar(Snowflake id, string newAvatar) - { - var user = await db.Users.FindAsync(id); - if (user == null) - { - logger.Warning("Update avatar job queued for {UserId} but no user with that ID exists", id); - return; - } - - try - { - var image = await ConvertAvatar(newAvatar); - var hash = Convert.ToHexString(await SHA256.HashDataAsync(image)).ToLower(); - image.Seek(0, SeekOrigin.Begin); - var prevHash = user.Avatar; - - await minio.PutObjectAsync(new PutObjectArgs() - .WithBucket(config.Storage.Bucket) - .WithObject(UserAvatarPath(id, hash)) - .WithObjectSize(image.Length) - .WithStreamData(image) - .WithContentType("image/webp") - ); - - user.Avatar = hash; - await db.SaveChangesAsync(); - - if (prevHash != null && prevHash != hash) - await minio.RemoveObjectAsync(new RemoveObjectArgs() - .WithBucket(config.Storage.Bucket) - .WithObject(UserAvatarPath(id, prevHash)) - ); - - logger.Information("Updated avatar for user {UserId}", id); - } - catch (ArgumentException ae) - { - logger.Warning("Invalid data URI for new avatar for user {UserId}: {Reason}", id, ae.Message); - } - } - - public async Task ClearUserAvatar(Snowflake id) - { - var user = await db.Users.FindAsync(id); - if (user == null) - { - logger.Warning("Clear avatar job queued for {UserId} but no user with that ID exists", id); - return; - } - - if (user.Avatar == null) - { - logger.Warning("Clear avatar job queued for {UserId} with null avatar", id); - return; - } - - await minio.RemoveObjectAsync(new RemoveObjectArgs() - .WithBucket(config.Storage.Bucket) - .WithObject(UserAvatarPath(user.Id, user.Avatar)) - ); - - user.Avatar = null; - await db.SaveChangesAsync(); - } - - public async Task UpdateMemberAvatar(Snowflake id, string newAvatar) - { - var member = await db.Members.FindAsync(id); - if (member == null) - { - logger.Warning("Update avatar job queued for {MemberId} but no member with that ID exists", id); - return; - } - - try - { - var image = await ConvertAvatar(newAvatar); - var hash = Convert.ToHexString(await SHA256.HashDataAsync(image)).ToLower(); - image.Seek(0, SeekOrigin.Begin); - var prevHash = member.Avatar; - - await minio.PutObjectAsync(new PutObjectArgs() - .WithBucket(config.Storage.Bucket) - .WithObject(MemberAvatarPath(id, hash)) - .WithObjectSize(image.Length) - .WithStreamData(image) - .WithContentType("image/webp") - ); - - member.Avatar = hash; - await db.SaveChangesAsync(); - - if (prevHash != null && prevHash != hash) - await minio.RemoveObjectAsync(new RemoveObjectArgs() - .WithBucket(config.Storage.Bucket) - .WithObject(MemberAvatarPath(id, prevHash)) - ); - - logger.Information("Updated avatar for member {MemberId}", id); - } - catch (ArgumentException ae) - { - logger.Warning("Invalid data URI for new avatar for member {MemberId}: {Reason}", id, ae.Message); - } - } - - public async Task ClearMemberAvatar(Snowflake id) - { - var member = await db.Members.FindAsync(id); - if (member == null) - { - logger.Warning("Clear avatar job queued for {MemberId} but no member with that ID exists", id); - return; - } - - if (member.Avatar == null) - { - logger.Warning("Clear avatar job queued for {MemberId} with null avatar", id); - return; - } - - await minio.RemoveObjectAsync(new RemoveObjectArgs() - .WithBucket(config.Storage.Bucket) - .WithObject(MemberAvatarPath(member.Id, member.Avatar)) - ); - - member.Avatar = null; - await db.SaveChangesAsync(); - } - - /// - /// Deletes a member's avatar. This should only be used when a member is in the process of being deleted, otherwise, - /// with a null avatar should be used instead. - /// - public async Task DeleteMemberAvatar(Snowflake id, string hash) => await DeleteAvatar(MemberAvatarPath(id, hash)); - /// - /// Deletes a user's avatar. This should only be used when a user is in the process of being deleted, otherwise, - /// with a null avatar should be used instead. - /// - public async Task DeleteUserAvatar(Snowflake id, string hash) => await DeleteAvatar(UserAvatarPath(id, hash)); - - private async Task DeleteAvatar(string path) - { - logger.Debug("Deleting avatar at path {Path}", path); - try - { - await minio.RemoveObjectAsync(new RemoveObjectArgs().WithBucket(config.Storage.Bucket).WithObject(path)); - } - catch (InvalidObjectNameException) - { - } - } - - private async Task ConvertAvatar(string uri) - { - if (!uri.StartsWith("data:image/")) - throw new ArgumentException("Not a data URI", nameof(uri)); - - var split = uri.Remove(0, "data:".Length).Split(";base64,"); - var contentType = split[0]; - var encoded = split[1]; - if (!_validContentTypes.Contains(contentType)) - throw new ArgumentException("Invalid content type for image", nameof(uri)); - - if (!AuthUtils.TryFromBase64String(encoded, out var rawImage)) - throw new ArgumentException("Invalid base64 string", nameof(uri)); - - var image = Image.Load(rawImage); - - var processor = new ResizeProcessor( - new ResizeOptions { Size = new Size(512), Mode = ResizeMode.Crop, Position = AnchorPositionMode.Center }, - image.Size - ); - - image.Mutate(x => x.ApplyProcessor(processor)); - - var stream = new MemoryStream(64 * 1024); - await image.SaveAsync(stream, new WebpEncoder { Quality = 95, NearLossless = false }); - return stream; - } - - private static string UserAvatarPath(Snowflake id, string hash) => $"users/{id}/avatars/{hash}.webp"; - private static string MemberAvatarPath(Snowflake id, string hash) => $"members/{id}/avatars/{hash}.webp"; -} \ No newline at end of file diff --git a/Foxnouns.Backend/Jobs/MemberAvatarUpdateInvocable.cs b/Foxnouns.Backend/Jobs/MemberAvatarUpdateInvocable.cs new file mode 100644 index 0000000..56d5077 --- /dev/null +++ b/Foxnouns.Backend/Jobs/MemberAvatarUpdateInvocable.cs @@ -0,0 +1,79 @@ +using System.Security.Cryptography; +using Coravel.Invocable; +using Foxnouns.Backend.Database; +using Foxnouns.Backend.Extensions; +using Foxnouns.Backend.Services; + +namespace Foxnouns.Backend.Jobs; + +public class MemberAvatarUpdateInvocable(DatabaseContext db, ObjectStorageService objectStorage, ILogger logger) + : IInvocable, IInvocableWithPayload +{ + private readonly ILogger _logger = logger.ForContext(); + public required AvatarUpdatePayload Payload { get; set; } + + public async Task Invoke() + { + if (Payload.NewAvatar != null) await UpdateMemberAvatarAsync(Payload.Id, Payload.NewAvatar); + else await ClearMemberAvatarAsync(Payload.Id); + } + + private async Task UpdateMemberAvatarAsync(Snowflake id, string newAvatar) + { + _logger.Debug("Updating avatar for member {MemberId}", id); + + var member = await db.Members.FindAsync(id); + if (member == null) + { + _logger.Warning("Update avatar job queued for {MemberId} but no member with that ID exists", id); + return; + } + + try + { + var image = await newAvatar.ConvertBase64UriToAvatar(); + var hash = Convert.ToHexString(await SHA256.HashDataAsync(image)).ToLower(); + image.Seek(0, SeekOrigin.Begin); + var prevHash = member.Avatar; + + await objectStorage.PutObjectAsync(Path(id, hash), image, "image/webp"); + + member.Avatar = hash; + await db.SaveChangesAsync(); + + if (prevHash != null && prevHash != hash) + await objectStorage.RemoveObjectAsync(Path(id, prevHash)); + + _logger.Information("Updated avatar for member {MemberId}", id); + } + catch (ArgumentException ae) + { + _logger.Warning("Invalid data URI for new avatar for member {MemberId}: {Reason}", id, ae.Message); + } + } + + private async Task ClearMemberAvatarAsync(Snowflake id) + { + _logger.Debug("Clearing avatar for member {MemberId}", id); + + var member = await db.Members.FindAsync(id); + if (member == null) + { + _logger.Warning("Clear avatar job queued for {MemberId} but no member with that ID exists", id); + return; + } + + if (member.Avatar == null) + { + _logger.Warning("Clear avatar job queued for {MemberId} with null avatar", id); + return; + } + + await objectStorage.RemoveObjectAsync(Path(member.Id, member.Avatar)); + + member.Avatar = null; + await db.SaveChangesAsync(); + } + + public static string Path(Snowflake id, string hash) => $"members/{id}/avatars/{hash}.webp"; +} \ No newline at end of file diff --git a/Foxnouns.Backend/Jobs/Payloads.cs b/Foxnouns.Backend/Jobs/Payloads.cs new file mode 100644 index 0000000..f28254a --- /dev/null +++ b/Foxnouns.Backend/Jobs/Payloads.cs @@ -0,0 +1,5 @@ +using Foxnouns.Backend.Database; + +namespace Foxnouns.Backend.Jobs; + +public record AvatarUpdatePayload(Snowflake Id, string? NewAvatar); \ No newline at end of file diff --git a/Foxnouns.Backend/Jobs/UserAvatarUpdateInvocable.cs b/Foxnouns.Backend/Jobs/UserAvatarUpdateInvocable.cs new file mode 100644 index 0000000..cbec277 --- /dev/null +++ b/Foxnouns.Backend/Jobs/UserAvatarUpdateInvocable.cs @@ -0,0 +1,79 @@ +using System.Security.Cryptography; +using Coravel.Invocable; +using Foxnouns.Backend.Database; +using Foxnouns.Backend.Extensions; +using Foxnouns.Backend.Services; + +namespace Foxnouns.Backend.Jobs; + +public class UserAvatarUpdateInvocable(DatabaseContext db, ObjectStorageService objectStorage, ILogger logger) + : IInvocable, IInvocableWithPayload +{ + private readonly ILogger _logger = logger.ForContext(); + public required AvatarUpdatePayload Payload { get; set; } + + public async Task Invoke() + { + if (Payload.NewAvatar != null) await UpdateUserAvatarAsync(Payload.Id, Payload.NewAvatar); + else await ClearUserAvatarAsync(Payload.Id); + } + + private async Task UpdateUserAvatarAsync(Snowflake id, string newAvatar) + { + _logger.Debug("Updating avatar for user {MemberId}", id); + + var user = await db.Users.FindAsync(id); + if (user == null) + { + _logger.Warning("Update avatar job queued for {UserId} but no user with that ID exists", id); + return; + } + + try + { + var image = await newAvatar.ConvertBase64UriToAvatar(); + var hash = Convert.ToHexString(await SHA256.HashDataAsync(image)).ToLower(); + image.Seek(0, SeekOrigin.Begin); + var prevHash = user.Avatar; + + await objectStorage.PutObjectAsync(Path(id, hash), image, "image/webp"); + + user.Avatar = hash; + await db.SaveChangesAsync(); + + if (prevHash != null && prevHash != hash) + await objectStorage.RemoveObjectAsync(Path(id, prevHash)); + + _logger.Information("Updated avatar for user {UserId}", id); + } + catch (ArgumentException ae) + { + _logger.Warning("Invalid data URI for new avatar for user {UserId}: {Reason}", id, ae.Message); + } + } + + private async Task ClearUserAvatarAsync(Snowflake id) + { + _logger.Debug("Clearing avatar for user {MemberId}", id); + + var user = await db.Users.FindAsync(id); + if (user == null) + { + _logger.Warning("Clear avatar job queued for {UserId} but no user with that ID exists", id); + return; + } + + if (user.Avatar == null) + { + _logger.Warning("Clear avatar job queued for {UserId} with null avatar", id); + return; + } + + await objectStorage.RemoveObjectAsync(Path(user.Id, user.Avatar)); + + user.Avatar = null; + await db.SaveChangesAsync(); + } + + public static string Path(Snowflake id, string hash) => $"users/{id}/avatars/{hash}.webp"; +} \ No newline at end of file diff --git a/Foxnouns.Backend/Middleware/AuthenticationMiddleware.cs b/Foxnouns.Backend/Middleware/AuthenticationMiddleware.cs index 8ad5df7..516813b 100644 --- a/Foxnouns.Backend/Middleware/AuthenticationMiddleware.cs +++ b/Foxnouns.Backend/Middleware/AuthenticationMiddleware.cs @@ -2,7 +2,6 @@ using System.Security.Cryptography; using Foxnouns.Backend.Database; using Foxnouns.Backend.Database.Models; using Foxnouns.Backend.Utils; -using Hangfire.Dashboard; using Microsoft.EntityFrameworkCore; using NodaTime; @@ -64,33 +63,4 @@ public static class HttpContextExtensions } [AttributeUsage(AttributeTargets.Class | AttributeTargets.Method)] -public class AuthenticateAttribute : Attribute; - -/// -/// Authentication filter for the Hangfire dashboard. Uses the cookie created by the frontend -/// (and otherwise only read by the frontend) to only allow admins to use it. -/// -public class HangfireDashboardAuthorizationFilter(IServiceProvider services) : IDashboardAsyncAuthorizationFilter -{ - public async Task AuthorizeAsync(DashboardContext context) - { - await using var scope = services.CreateAsyncScope(); - - await using var db = scope.ServiceProvider.GetRequiredService(); - var clock = scope.ServiceProvider.GetRequiredService(); - - var httpContext = context.GetHttpContext(); - - if (!httpContext.Request.Cookies.TryGetValue("pronounscc-token", out var cookie)) return false; - - if (!AuthUtils.TryFromBase64String(cookie!, out var rawToken)) return false; - - var hash = SHA512.HashData(rawToken); - var oauthToken = await db.Tokens - .Include(t => t.Application) - .Include(t => t.User) - .FirstOrDefaultAsync(t => t.Hash == hash && t.ExpiresAt > clock.GetCurrentInstant() && !t.ManuallyExpired); - - return oauthToken?.User.Role == UserRole.Admin; - } -} \ No newline at end of file +public class AuthenticateAttribute : Attribute; \ No newline at end of file diff --git a/Foxnouns.Backend/Program.cs b/Foxnouns.Backend/Program.cs index 7230598..e1f201e 100644 --- a/Foxnouns.Backend/Program.cs +++ b/Foxnouns.Backend/Program.cs @@ -1,18 +1,15 @@ +using Coravel; using Foxnouns.Backend; using Foxnouns.Backend.Database; using Serilog; using Foxnouns.Backend.Extensions; -using Foxnouns.Backend.Middleware; using Foxnouns.Backend.Services; using Foxnouns.Backend.Utils; -using Hangfire; -using Hangfire.Redis.StackExchange; using Microsoft.AspNetCore.Mvc; using Minio; using Newtonsoft.Json; using Newtonsoft.Json.Serialization; using Sentry.Extensibility; -using Sentry.Hangfire; // Read version information from .version in the repository root await BuildInfo.ReadBuildInfo(); @@ -64,6 +61,7 @@ JsonConvert.DefaultSettings = () => new JsonSerializerSettings }; builder.Services + .AddQueue() .AddDbContext() .AddCustomServices() .AddCustomMiddleware() @@ -74,12 +72,6 @@ builder.Services .WithCredentials(config.Storage.AccessKey, config.Storage.SecretKey) .Build()); -builder.Services.AddHangfire(c => c.UseSentry().UseRedisStorage(config.Jobs.Redis, new RedisStorageOptions - { - Prefix = "foxnouns_" - })) - .AddHangfireServer(options => { options.WorkerCount = config.Jobs.Workers; }); - var app = builder.Build(); await app.Initialize(args); @@ -95,12 +87,6 @@ app.UseCors(); app.UseCustomMiddleware(); app.MapControllers(); -app.UseHangfireDashboard("/hangfire", new DashboardOptions -{ - AppPath = null, - AsyncAuthorization = [new HangfireDashboardAuthorizationFilter(app.Services)] -}); - app.Urls.Clear(); app.Urls.Add(config.Address); if (config.MetricsAddress != null) app.Urls.Add(config.MetricsAddress); diff --git a/Foxnouns.Backend/Services/ObjectStorageService.cs b/Foxnouns.Backend/Services/ObjectStorageService.cs new file mode 100644 index 0000000..2180b90 --- /dev/null +++ b/Foxnouns.Backend/Services/ObjectStorageService.cs @@ -0,0 +1,33 @@ +using Minio; +using Minio.DataModel.Args; +using Minio.Exceptions; + +namespace Foxnouns.Backend.Services; + +public class ObjectStorageService(ILogger logger, Config config, IMinioClient minio) +{ + private readonly ILogger _logger = logger.ForContext(); + + public async Task RemoveObjectAsync(string path) + { + logger.Debug("Deleting object at path {Path}", path); + try + { + await minio.RemoveObjectAsync(new RemoveObjectArgs().WithBucket(config.Storage.Bucket).WithObject(path)); + } + catch (InvalidObjectNameException) + { + } + } + + public async Task PutObjectAsync(string path, Stream data, string contentType) + { + await minio.PutObjectAsync(new PutObjectArgs() + .WithBucket(config.Storage.Bucket) + .WithObject(path) + .WithObjectSize(data.Length) + .WithStreamData(data) + .WithContentType(contentType) + ); + } +} \ No newline at end of file diff --git a/Foxnouns.Backend/Utils/Limits.cs b/Foxnouns.Backend/Utils/Limits.cs new file mode 100644 index 0000000..c86df0b --- /dev/null +++ b/Foxnouns.Backend/Utils/Limits.cs @@ -0,0 +1,9 @@ +namespace Foxnouns.Backend.Utils; + +public static class Limits +{ + public const int FieldLimit = 25; + public const int FieldNameLimit = 100; + public const int FieldEntryTextLimit = 100; + public const int FieldEntriesLimit = 100; +} \ No newline at end of file diff --git a/Foxnouns.Backend/Utils/ValidationUtils.cs b/Foxnouns.Backend/Utils/ValidationUtils.cs index 276395f..5c3c591 100644 --- a/Foxnouns.Backend/Utils/ValidationUtils.cs +++ b/Foxnouns.Backend/Utils/ValidationUtils.cs @@ -57,11 +57,11 @@ public static class ValidationUtils public static ValidationError? ValidateMemberName(string memberName) { - if (!UsernameRegex.IsMatch(memberName)) + if (!MemberRegex.IsMatch(memberName)) return memberName.Length switch { - < 2 => ValidationError.LengthError("Name is too short", 1, 100, memberName.Length), - > 40 => ValidationError.LengthError("Name is too long", 1, 100, memberName.Length), + < 1 => ValidationError.LengthError("Name is too short", 1, 100, memberName.Length), + > 100 => ValidationError.LengthError("Name is too long", 1, 100, memberName.Length), _ => ValidationError.GenericValidationError( "Member name cannot contain any of the following: " + " @, ?, !, #, /, \\, [, ], \", ', $, %, &, (, ), {, }, +, <, =, >, ^, |, ~, `, , " + @@ -119,10 +119,7 @@ public static class ValidationUtils }; } - private const int FieldLimit = 25; - private const int FieldNameLimit = 100; - private const int FieldEntryTextLimit = 100; - private const int FieldEntriesLimit = 100; + private static readonly string[] DefaultStatusOptions = [ @@ -140,7 +137,7 @@ public static class ValidationUtils var errors = new List<(string, ValidationError?)>(); if (fields.Count > 25) - errors.Add(("fields", ValidationError.LengthError("Too many fields", 0, FieldLimit, fields.Count))); + errors.Add(("fields", ValidationError.LengthError("Too many fields", 0, Limits.FieldLimit, fields.Count))); // No overwhelming this function, thank you if (fields.Count > 100) return errors; @@ -148,13 +145,13 @@ public static class ValidationUtils { switch (field.Name.Length) { - case > FieldNameLimit: + case > Limits.FieldNameLimit: errors.Add(($"fields.{index}.name", - ValidationError.LengthError("Field name is too long", 1, FieldNameLimit, field.Name.Length))); + ValidationError.LengthError("Field name is too long", 1, Limits.FieldNameLimit, field.Name.Length))); break; case < 1: errors.Add(($"fields.{index}.name", - ValidationError.LengthError("Field name is too short", 1, FieldNameLimit, field.Name.Length))); + ValidationError.LengthError("Field name is too short", 1, Limits.FieldNameLimit, field.Name.Length))); break; } @@ -170,26 +167,26 @@ public static class ValidationUtils if (entries == null || entries.Length == 0) return []; var errors = new List<(string, ValidationError?)>(); - if (entries.Length > FieldEntriesLimit) + if (entries.Length > Limits.FieldEntriesLimit) errors.Add(($"{errorPrefix}.entries", - ValidationError.LengthError("Field has too many entries", 0, FieldEntriesLimit, + ValidationError.LengthError("Field has too many entries", 0, Limits.FieldEntriesLimit, entries.Length))); // Same as above, no overwhelming this function with a ridiculous amount of entries - if (entries.Length > FieldEntriesLimit + 50) return errors; + if (entries.Length > Limits.FieldEntriesLimit + 50) return errors; foreach (var (entry, entryIdx) in entries.Select((entry, entryIdx) => (entry, entryIdx))) { switch (entry.Value.Length) { - case > FieldEntryTextLimit: + case > Limits.FieldEntryTextLimit: errors.Add(($"{errorPrefix}.entries.{entryIdx}.value", - ValidationError.LengthError("Field value is too long", 1, FieldEntryTextLimit, + ValidationError.LengthError("Field value is too long", 1, Limits.FieldEntryTextLimit, entry.Value.Length))); break; case < 1: errors.Add(($"{errorPrefix}.entries.{entryIdx}.value", - ValidationError.LengthError("Field value is too short", 1, FieldEntryTextLimit, + ValidationError.LengthError("Field value is too short", 1, Limits.FieldEntryTextLimit, entry.Value.Length))); break; } diff --git a/Foxnouns.Backend/appSettings.json b/Foxnouns.Backend/appSettings.json new file mode 100644 index 0000000..ae6b417 --- /dev/null +++ b/Foxnouns.Backend/appSettings.json @@ -0,0 +1,7 @@ +{ + "Coravel": { + "Queue": { + "ConsummationDelay": 1 + } + } +} \ No newline at end of file diff --git a/Foxnouns.Backend/config.example.ini b/Foxnouns.Backend/config.example.ini index e316c8e..0b80a7a 100644 --- a/Foxnouns.Backend/config.example.ini +++ b/Foxnouns.Backend/config.example.ini @@ -31,12 +31,6 @@ Timeout = 5 ; The maximum number of open connections. Defaults to 50. MaxPoolSize = 50 -[Jobs] -; The connection string for the Redis server. -Redis = localhost:6379 -; The number of workers to use for background jobs. Defaults to 5. -Workers = 5 - [Storage] Endpoint = AccessKey =