From ded4f4db2690802d33eacfeea68700d58bbed7a1 Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 13 Aug 2024 13:08:50 +0200 Subject: [PATCH] init --- .editorconfig | 9 + .gitignore | 6 + .idea/.idea.catalogger/.idea/.gitignore | 13 ++ .idea/.idea.catalogger/.idea/discord.xml | 7 + .idea/.idea.catalogger/.idea/encodings.xml | 4 + .idea/.idea.catalogger/.idea/indexLayout.xml | 8 + .idea/.idea.catalogger/.idea/vcs.xml | 6 + .../Bot/Commands/MetaCommands.cs | 32 +++ Catalogger.Backend/Bot/DiscordUtils.cs | 9 + .../Bot/Responders/GuildCreateResponder.cs | 49 +++++ .../Bot/Responders/MessageCreateResponder.cs | 157 +++++++++++++++ .../Bot/Responders/MessageDeleteResponder.cs | 102 ++++++++++ .../Bot/Responders/ReadyResponder.cs | 27 +++ .../Cache/ChannelCacheService.cs | 56 ++++++ Catalogger.Backend/Cache/UserCacheService.cs | 24 +++ Catalogger.Backend/Catalogger.Backend.csproj | 33 ++++ Catalogger.Backend/CataloggerError.cs | 5 + Catalogger.Backend/Config.cs | 42 ++++ .../Database/DatabaseContext.cs | 97 +++++++++ .../Database/EncryptionService.cs | 36 ++++ .../Database/IEncryptionService.cs | 7 + .../20240803132306_Init.Designer.cs | 180 +++++++++++++++++ .../Migrations/20240803132306_Init.cs | 115 +++++++++++ .../DatabaseContextModelSnapshot.cs | 177 +++++++++++++++++ Catalogger.Backend/Database/Models/Guild.cs | 60 ++++++ Catalogger.Backend/Database/Models/Invite.cs | 8 + Catalogger.Backend/Database/Models/Message.cs | 27 +++ .../Database/Models/Watchlist.cs | 13 ++ .../Database/Queries/MessageRepository.cs | 99 ++++++++++ .../Database/Queries/QueryExtensions.cs | 22 +++ Catalogger.Backend/Database/QueryUtils.cs | 3 + .../Extensions/DiscordExtensions.cs | 44 +++++ .../Extensions/StartupExtensions.cs | 121 ++++++++++++ Catalogger.Backend/GlobalUsing.cs | 1 + Catalogger.Backend/Program.cs | 71 +++++++ .../Properties/launchSettings.json | 41 ++++ Catalogger.Backend/Services/IWebhookCache.cs | 29 +++ .../Services/InMemoryWebhookCache.cs | 21 ++ .../Services/PluralkitApiService.cs | 73 +++++++ .../Services/WebhookExecutorService.cs | 186 ++++++++++++++++++ Catalogger.Backend/config.example.ini | 14 ++ catalogger.sln | 16 ++ catalogger.sln.DotSettings | 2 + 43 files changed, 2052 insertions(+) create mode 100644 .editorconfig create mode 100644 .gitignore create mode 100644 .idea/.idea.catalogger/.idea/.gitignore create mode 100644 .idea/.idea.catalogger/.idea/discord.xml create mode 100644 .idea/.idea.catalogger/.idea/encodings.xml create mode 100644 .idea/.idea.catalogger/.idea/indexLayout.xml create mode 100644 .idea/.idea.catalogger/.idea/vcs.xml create mode 100644 Catalogger.Backend/Bot/Commands/MetaCommands.cs create mode 100644 Catalogger.Backend/Bot/DiscordUtils.cs create mode 100644 Catalogger.Backend/Bot/Responders/GuildCreateResponder.cs create mode 100644 Catalogger.Backend/Bot/Responders/MessageCreateResponder.cs create mode 100644 Catalogger.Backend/Bot/Responders/MessageDeleteResponder.cs create mode 100644 Catalogger.Backend/Bot/Responders/ReadyResponder.cs create mode 100644 Catalogger.Backend/Cache/ChannelCacheService.cs create mode 100644 Catalogger.Backend/Cache/UserCacheService.cs create mode 100644 Catalogger.Backend/Catalogger.Backend.csproj create mode 100644 Catalogger.Backend/CataloggerError.cs create mode 100644 Catalogger.Backend/Config.cs create mode 100644 Catalogger.Backend/Database/DatabaseContext.cs create mode 100644 Catalogger.Backend/Database/EncryptionService.cs create mode 100644 Catalogger.Backend/Database/IEncryptionService.cs create mode 100644 Catalogger.Backend/Database/Migrations/20240803132306_Init.Designer.cs create mode 100644 Catalogger.Backend/Database/Migrations/20240803132306_Init.cs create mode 100644 Catalogger.Backend/Database/Migrations/DatabaseContextModelSnapshot.cs create mode 100644 Catalogger.Backend/Database/Models/Guild.cs create mode 100644 Catalogger.Backend/Database/Models/Invite.cs create mode 100644 Catalogger.Backend/Database/Models/Message.cs create mode 100644 Catalogger.Backend/Database/Models/Watchlist.cs create mode 100644 Catalogger.Backend/Database/Queries/MessageRepository.cs create mode 100644 Catalogger.Backend/Database/Queries/QueryExtensions.cs create mode 100644 Catalogger.Backend/Database/QueryUtils.cs create mode 100644 Catalogger.Backend/Extensions/DiscordExtensions.cs create mode 100644 Catalogger.Backend/Extensions/StartupExtensions.cs create mode 100644 Catalogger.Backend/GlobalUsing.cs create mode 100644 Catalogger.Backend/Program.cs create mode 100644 Catalogger.Backend/Properties/launchSettings.json create mode 100644 Catalogger.Backend/Services/IWebhookCache.cs create mode 100644 Catalogger.Backend/Services/InMemoryWebhookCache.cs create mode 100644 Catalogger.Backend/Services/PluralkitApiService.cs create mode 100644 Catalogger.Backend/Services/WebhookExecutorService.cs create mode 100644 Catalogger.Backend/config.example.ini create mode 100644 catalogger.sln create mode 100644 catalogger.sln.DotSettings diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..22852f7 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +root = true + +[*.cs] +# Responder classes are considered "unused" by ReSharper because they're only loaded through reflection. +resharper_unused_type_global_highlighting = none +# Command methods are also considered unused, for the same reason. +resharper_unused_member_global_highlighting = none +# Command classes are generally only referred to in type parameters. +resharper_class_never_instantiated_global_highlighting = none diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..746db10 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +bin/ +obj/ +/packages/ +riderModule.iml +/_ReSharper.Caches/ +config.ini diff --git a/.idea/.idea.catalogger/.idea/.gitignore b/.idea/.idea.catalogger/.idea/.gitignore new file mode 100644 index 0000000..bf2dbe0 --- /dev/null +++ b/.idea/.idea.catalogger/.idea/.gitignore @@ -0,0 +1,13 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Rider ignored files +/modules.xml +/.idea.catalogger.iml +/contentModel.xml +/projectSettingsUpdater.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/.idea.catalogger/.idea/discord.xml b/.idea/.idea.catalogger/.idea/discord.xml new file mode 100644 index 0000000..d8e9561 --- /dev/null +++ b/.idea/.idea.catalogger/.idea/discord.xml @@ -0,0 +1,7 @@ + + + + + \ No newline at end of file diff --git a/.idea/.idea.catalogger/.idea/encodings.xml b/.idea/.idea.catalogger/.idea/encodings.xml new file mode 100644 index 0000000..df87cf9 --- /dev/null +++ b/.idea/.idea.catalogger/.idea/encodings.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/.idea.catalogger/.idea/indexLayout.xml b/.idea/.idea.catalogger/.idea/indexLayout.xml new file mode 100644 index 0000000..7b08163 --- /dev/null +++ b/.idea/.idea.catalogger/.idea/indexLayout.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/.idea.catalogger/.idea/vcs.xml b/.idea/.idea.catalogger/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/.idea.catalogger/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Catalogger.Backend/Bot/Commands/MetaCommands.cs b/Catalogger.Backend/Bot/Commands/MetaCommands.cs new file mode 100644 index 0000000..24ec530 --- /dev/null +++ b/Catalogger.Backend/Bot/Commands/MetaCommands.cs @@ -0,0 +1,32 @@ +using System.ComponentModel; +using Catalogger.Backend.Extensions; +using NodaTime; +using Remora.Commands.Attributes; +using Remora.Commands.Groups; +using Remora.Discord.API.Abstractions.Rest; +using Remora.Discord.Commands.Feedback.Services; +using Remora.Discord.Gateway; +using Remora.Results; +using IResult = Remora.Results.IResult; + +namespace Catalogger.Backend.Bot.Commands; + +[Group("catalogger")] +public class MetaCommands( + IClock clock, + DiscordGatewayClient client, + IFeedbackService feedbackService, + IDiscordRestChannelAPI channelApi) : CommandGroup +{ + [Command("ping")] + [Description("Ping pong! See the bot's latency")] + public async Task PingAsync() + { + var t1 = clock.GetCurrentInstant(); + var msg = await feedbackService.SendContextualAsync("...").GetOrThrow(); + var elapsed = clock.GetCurrentInstant() - t1; + + return (Result)await channelApi.EditMessageAsync(msg.ChannelID, msg.ID, + content: $"Pong! API: {elapsed.TotalMilliseconds:N0}ms | Gateway: {client.Latency.TotalMilliseconds:N0}ms"); + } +} \ No newline at end of file diff --git a/Catalogger.Backend/Bot/DiscordUtils.cs b/Catalogger.Backend/Bot/DiscordUtils.cs new file mode 100644 index 0000000..806fa39 --- /dev/null +++ b/Catalogger.Backend/Bot/DiscordUtils.cs @@ -0,0 +1,9 @@ +using System.Drawing; + +namespace Catalogger.Backend.Bot; + +public static class DiscordUtils +{ + public static readonly Color Red = Color.FromArgb(231, 76, 60); + public static readonly Color Blue = Color.FromArgb(155, 89, 182); +} \ No newline at end of file diff --git a/Catalogger.Backend/Bot/Responders/GuildCreateResponder.cs b/Catalogger.Backend/Bot/Responders/GuildCreateResponder.cs new file mode 100644 index 0000000..c2c676a --- /dev/null +++ b/Catalogger.Backend/Bot/Responders/GuildCreateResponder.cs @@ -0,0 +1,49 @@ +using System.Diagnostics; +using Catalogger.Backend.Cache; +using Catalogger.Backend.Database; +using Catalogger.Backend.Database.Models; +using Catalogger.Backend.Extensions; +using Remora.Discord.API.Abstractions.Gateway.Events; +using Remora.Discord.Gateway.Responders; +using Remora.Results; + +namespace Catalogger.Backend.Bot.Responders; + +public class GuildCreateResponder(ILogger logger, DatabaseContext db, ChannelCacheService channelCache) + : IResponder +{ + private readonly ILogger _logger = logger.ForContext(); + + public async Task RespondAsync(IGuildCreate evt, CancellationToken ct = default) + { + ulong guildId; + string? guildName = null; + if (evt.Guild.TryPickT0(out var guild, out _)) + { + _logger.Verbose("Received guild create for available guild {GuildId} ({Name})", guild.ID, guild.Name); + guildId = guild.ID.ToUlong(); + guildName = guild.Name; + + foreach (var c in guild.Channels) channelCache.AddChannel(c, guild.ID); + } + else if (evt.Guild.TryPickT1(out var unavailableGuild, out _)) + { + _logger.Verbose("Received guild create for unavailable guild {GuildId}", unavailableGuild.ID); + guildId = unavailableGuild.ID.ToUlong(); + } + else throw new UnreachableException(); + + var tx = await db.Database.BeginTransactionAsync(ct); + if (await db.Guilds.FindAsync([guildId], ct) != null) return Result.Success; + + db.Add(new Guild + { + Id = guildId + }); + await db.SaveChangesAsync(ct); + await tx.CommitAsync(ct); + + _logger.Information("Joined new guild {GuildId} ({Name})", guildId, guildName); + return Result.Success; + } +} \ No newline at end of file diff --git a/Catalogger.Backend/Bot/Responders/MessageCreateResponder.cs b/Catalogger.Backend/Bot/Responders/MessageCreateResponder.cs new file mode 100644 index 0000000..1fe6a20 --- /dev/null +++ b/Catalogger.Backend/Bot/Responders/MessageCreateResponder.cs @@ -0,0 +1,157 @@ +using System.Text.RegularExpressions; +using Catalogger.Backend.Cache; +using Catalogger.Backend.Database; +using Catalogger.Backend.Database.Models; +using Catalogger.Backend.Database.Queries; +using Catalogger.Backend.Extensions; +using Catalogger.Backend.Services; +using Humanizer; +using Remora.Discord.API; +using Remora.Discord.API.Abstractions.Gateway.Events; +using Remora.Discord.Gateway.Responders; +using Remora.Rest.Core; +using Remora.Results; + +namespace Catalogger.Backend.Bot.Responders; + +public class MessageCreateResponder( + ILogger logger, + DatabaseContext db, + MessageRepository messageRepository, + UserCacheService userCache, + PkMessageHandler pkMessageHandler) + : IResponder +{ + private readonly ILogger _logger = logger.ForContext(); + private static readonly Snowflake PkUserId = DiscordSnowflake.New(466378653216014359); + + public async Task RespondAsync(IMessageCreate msg, CancellationToken ct = default) + { + userCache.UpdateUser(msg.Author); + + if (!msg.GuildID.IsDefined()) + { + _logger.Debug("Received message create event for message {MessageId} despite it not being in a guild", + msg.ID); + return Result.Success; + } + + var guild = await db.GetGuildAsync(msg.GuildID, ct); + // The guild needs to have enabled at least one of the message logging events, + // and the channel must not be ignored, to store the message. + if (guild.IsMessageIgnored(msg.ChannelID, msg.Author.ID)) + { + db.IgnoredMessages.Add(new IgnoredMessage(msg.ID.ToUlong())); + await db.SaveChangesAsync(ct); + return Result.Success; + } + + if (msg.Author.ID == PkUserId) + _ = pkMessageHandler.HandlePkMessageAsync(msg); + if (msg.ApplicationID.IsDefined(out var appId) && appId == PkUserId) + _ = pkMessageHandler.HandleProxiedMessageAsync(msg.ID.Value); + + await messageRepository.SaveMessageAsync(msg, ct); + return Result.Success; + } +} + +public partial class PkMessageHandler(ILogger logger, IServiceProvider services) +{ + private readonly ILogger _logger = logger.ForContext(); + + [GeneratedRegex( + @"^System ID: (\w{5,6}) \| Member ID: (\w{5,6}) \| Sender: .+ \((\d+)\) \| Message ID: (\d+) \| Original Message ID: (\d+)$")] + private static partial Regex FooterRegex(); + + [GeneratedRegex(@"^https:\/\/discord.com\/channels\/\d+\/(\d+)\/\d+$")] + private static partial Regex LinkRegex(); + + public async Task HandlePkMessageAsync(IMessageCreate msg) + { + _logger.Debug("Received PluralKit message"); + + await Task.Delay(500.Milliseconds()); + + _logger.Debug("Starting handling PluralKit message"); + + // Check if the content matches a Discord link--if not, it's not a log message (we already check if this is a PluralKit message earlier) + if (!LinkRegex().IsMatch(msg.Content)) + { + _logger.Debug("PluralKit message is not a log message because content is not a link"); + return; + } + + // The first (only, I think always?) embed's footer must match the expected format + var firstEmbed = msg.Embeds.FirstOrDefault(); + if (firstEmbed == null || !firstEmbed.Footer.TryGet(out var footer) || + !FooterRegex().IsMatch(footer.Text)) + { + _logger.Debug( + "PK message is not a log message because there is no first embed or its footer doesn't match the regex"); + return; + } + + var match = FooterRegex().Match(footer.Text); + + if (!ulong.TryParse(match.Groups[3].Value, out var authorId)) + { + _logger.Debug("Author ID in PluralKit log {LogMessageId} was not a valid snowflake", msg.ID); + return; + } + + if (!ulong.TryParse(match.Groups[4].Value, out var msgId)) + { + _logger.Debug("Message ID in PluralKit log {LogMessageId} was not a valid snowflake", msg.ID); + return; + } + + if (!ulong.TryParse(match.Groups[5].Value, out var originalId)) + { + _logger.Debug("Original ID in PluralKit log {LogMessageId} was not a valid snowflake", msg.ID); + return; + } + + await using var scope = services.CreateAsyncScope(); + await using var db = scope.ServiceProvider.GetRequiredService(); + var messageRepository = scope.ServiceProvider.GetRequiredService(); + + await messageRepository.SetProxiedMessageDataAsync(msgId, originalId, authorId, + systemId: match.Groups[1].Value, memberId: match.Groups[2].Value); + + db.IgnoredMessages.Add(new IgnoredMessage(originalId)); + await db.SaveChangesAsync(); + } + + public async Task HandleProxiedMessageAsync(ulong msgId) + { + await Task.Delay(3.Seconds()); + + await using var scope = services.CreateAsyncScope(); + await using var db = scope.ServiceProvider.GetRequiredService(); + var messageRepository = scope.ServiceProvider.GetRequiredService(); + var pluralkitApi = scope.ServiceProvider.GetRequiredService(); + + var (isStored, hasProxyInfo) = await messageRepository.HasProxyInfoAsync(msgId); + if (!isStored) + { + _logger.Debug("Message with ID {MessageId} is not stored in the database", msgId); + return; + } + + if (hasProxyInfo) return; + + var pkMessage = await pluralkitApi.GetPluralKitMessageAsync(msgId); + if (pkMessage == null) + { + _logger.Debug("Message with ID {MessageId} was proxied by PluralKit, but API returned 404", msgId); + return; + } + + await messageRepository.SetProxiedMessageDataAsync(msgId, pkMessage.Original, pkMessage.Sender, + pkMessage.System?.Id, pkMessage.Member?.Id); + + db.IgnoredMessages.Add(new IgnoredMessage(pkMessage.Original)); + await db.SaveChangesAsync(); + } +} \ No newline at end of file diff --git a/Catalogger.Backend/Bot/Responders/MessageDeleteResponder.cs b/Catalogger.Backend/Bot/Responders/MessageDeleteResponder.cs new file mode 100644 index 0000000..c2634c3 --- /dev/null +++ b/Catalogger.Backend/Bot/Responders/MessageDeleteResponder.cs @@ -0,0 +1,102 @@ +using Catalogger.Backend.Cache; +using Catalogger.Backend.Database; +using Catalogger.Backend.Database.Queries; +using Catalogger.Backend.Extensions; +using Catalogger.Backend.Services; +using Humanizer; +using NodaTime; +using Remora.Discord.API; +using Remora.Discord.API.Abstractions.Gateway.Events; +using Remora.Discord.API.Abstractions.Objects; +using Remora.Discord.API.Objects; +using Remora.Discord.Extensions.Embeds; +using Remora.Discord.Gateway.Responders; +using Remora.Results; + +namespace Catalogger.Backend.Bot.Responders; + +public class MessageDeleteResponder( + ILogger logger, + DatabaseContext db, + MessageRepository messageRepository, + WebhookExecutorService webhookExecutor, + ChannelCacheService channelCache, + UserCacheService userCache, + IClock clock) : IResponder +{ + private readonly ILogger _logger = logger.ForContext(); + + public async Task RespondAsync(IMessageDelete ev, CancellationToken ct = default) + { + if (!ev.GuildID.IsDefined()) return Result.Success; + + if (ev.ID.Timestamp < DateTimeOffset.Now - 1.Minutes()) + { + _logger.Debug( + "Deleted message {MessageId} is less than 1 minute old, delaying 5 seconds to give PK time to catch up", + ev.ID); + await Task.Delay(5.Seconds(), ct); + } + + if (await messageRepository.IsMessageIgnoredAsync(ev.ID.Value, ct)) return Result.Success; + + var guild = await db.GetGuildAsync(ev.GuildID, ct); + if (guild.IsMessageIgnored(ev.ChannelID, ev.ID)) return Result.Success; + + var logChannel = webhookExecutor.GetLogChannel(guild, LogChannelType.MessageDelete, ev.ChannelID); + var msg = await messageRepository.GetMessageAsync(ev.ID.Value, ct); + // Sometimes a message that *should* be logged isn't stored in the database, notify the user of that + if (msg == null) + { + if (logChannel == null) return Result.Success; + await webhookExecutor.QueueLogAsync(logChannel.Value, new Embed( + Title: "Message deleted", + Description: $"A message not found in the database was deleted in <#{ev.ChannelID}> ({ev.ChannelID}).", + Footer: new EmbedFooter(Text: $"ID: {ev.ID}"), + Timestamp: clock.GetCurrentInstant().ToDateTimeOffset() + )); + + return Result.Success; + } + + logChannel = webhookExecutor.GetLogChannel(guild, LogChannelType.MessageDelete, ev.ChannelID, msg.UserId); + if (logChannel == null) return Result.Success; + + var user = await userCache.GetUserAsync(DiscordSnowflake.New(msg.UserId)); + var builder = new EmbedBuilder() + .WithTitle("Message deleted") + .WithDescription(msg.Content) + .WithColour(DiscordUtils.Red) + .WithFooter($"ID: {msg.Id}") + .WithTimestamp(ev.ID); + + if (user != null) + builder.WithAuthor(user.Tag(), url: null, iconUrl: user.AvatarUrl()); + if (msg.Member != null) builder.WithTitle($"Message by {msg.Username} deleted"); + + string channelMention; + if (!channelCache.GetChannel(ev.ChannelID, out var channel)) + channelMention = $"<#{msg.ChannelId}>"; + else if (channel.Type is ChannelType.AnnouncementThread or ChannelType.PrivateThread + or ChannelType.PublicThread) + channelMention = + $"<#{channel.ParentID.Value}>\nID: {channel.ParentID.Value}\n\nThread: {channel.Name} (<#{channel.ID}>)"; + else channelMention = $"<#{channel.ID}>\nID: {channel.ID}"; + + var userMention = user != null + ? $"<@{user.ID}>\n{user.Tag()}\nID: {user.ID}" + : $"<@{msg.UserId}>\nID: {msg.UserId}"; + + builder.AddField("Channel", channelMention, true); + builder.AddField(msg.System != null ? "Linked Discord account" : "Sender", userMention, true); + if (msg is { System: not null, Member: not null }) + { + builder.AddField("\u200b", "**PluralKit information**", false); + builder.AddField("System ID", msg.System, true); + builder.AddField("Member ID", msg.Member, true); + } + + await webhookExecutor.QueueLogAsync(logChannel.Value, builder.Build().GetOrThrow()); + return Result.Success; + } +} \ No newline at end of file diff --git a/Catalogger.Backend/Bot/Responders/ReadyResponder.cs b/Catalogger.Backend/Bot/Responders/ReadyResponder.cs new file mode 100644 index 0000000..e3341fa --- /dev/null +++ b/Catalogger.Backend/Bot/Responders/ReadyResponder.cs @@ -0,0 +1,27 @@ +using Catalogger.Backend.Extensions; +using Catalogger.Backend.Services; +using Remora.Discord.API.Abstractions.Gateway.Events; +using Remora.Discord.Gateway.Responders; +using Remora.Results; + +namespace Catalogger.Backend.Bot.Responders; + +public class ReadyResponder(ILogger logger, WebhookExecutorService webhookExecutorService, Config config) + : IResponder +{ + private readonly ILogger _logger = logger.ForContext(); + + public Task RespondAsync(IReady gatewayEvent, CancellationToken ct = default) + { + var shardId = gatewayEvent.Shard.TryGet(out var shard) ? (shard.ShardID, shard.ShardCount) : (0, 1); + _logger.Information("Ready as {User} on shard {ShardId} / {ShardCount}", gatewayEvent.User.Tag(), shardId.Item1, + shardId.Item2); + if (shardId.Item1 == 0) webhookExecutorService.SetSelfUser(gatewayEvent.User); + // Sanity check + var appId = gatewayEvent.Application.ID.ToUlong(); + _logger.Debug("Application ID is {ApplicationId}, is same as config? {SameAsConfig}", appId, + appId == config.Discord.ApplicationId); + + return Task.FromResult(Result.Success); + } +} \ No newline at end of file diff --git a/Catalogger.Backend/Cache/ChannelCacheService.cs b/Catalogger.Backend/Cache/ChannelCacheService.cs new file mode 100644 index 0000000..4d1d51e --- /dev/null +++ b/Catalogger.Backend/Cache/ChannelCacheService.cs @@ -0,0 +1,56 @@ +using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; +using Remora.Discord.API.Abstractions.Objects; +using Remora.Rest.Core; + +namespace Catalogger.Backend.Cache; + +public class ChannelCacheService +{ + private readonly ConcurrentDictionary _channels = new(); + private readonly ConcurrentDictionary> _guildChannels = new(); + + public void AddChannel(IChannel channel, Snowflake? guildId = null) + { + _channels[channel.ID] = channel; + if (guildId == null) + { + if (!channel.GuildID.TryGet(out var snowflake)) return; + guildId = snowflake; + } + + // Add to set of guild channels + _guildChannels.AddOrUpdate(guildId.Value, + _ => [channel.ID], + (_, l) => + { + l.Add(channel.ID); + return l; + }); + } + + public bool GetChannel(Snowflake id, [NotNullWhen(true)] out IChannel? channel) => _channels.TryGetValue(id, out channel); + + public void RemoveChannel(Snowflake? guildId, Snowflake id, out IChannel? channel) + { + _channels.Remove(id, out channel); + if (guildId == null) return; + // Remove from set of guild channels + _guildChannels.AddOrUpdate(guildId.Value, _ => [], (_, s) => + { + s.Remove(id); + return s; + }); + } + + /// + /// Gets all of a guild's cached channels. + /// + /// The guild to get the channels of + /// A list of cached channels + public IEnumerable GuildChannels(Snowflake guildId) => + !_guildChannels.TryGetValue(guildId, out var channelIds) + ? [] + : channelIds.Select(id => _channels.GetValueOrDefault(id)) + .Where(c => c != null).Select(c => c!); +} \ No newline at end of file diff --git a/Catalogger.Backend/Cache/UserCacheService.cs b/Catalogger.Backend/Cache/UserCacheService.cs new file mode 100644 index 0000000..7053e66 --- /dev/null +++ b/Catalogger.Backend/Cache/UserCacheService.cs @@ -0,0 +1,24 @@ +using System.Collections.Concurrent; +using Remora.Discord.API.Abstractions.Objects; +using Remora.Discord.API.Abstractions.Rest; +using Remora.Rest.Core; + +namespace Catalogger.Backend.Cache; + +public class UserCacheService(IDiscordRestUserAPI userApi) +{ + private readonly ConcurrentDictionary _cache = new(); + + public async Task GetUserAsync(Snowflake userId) + { + if (_cache.TryGetValue(userId, out var user)) return user; + + var res = await userApi.GetUserAsync(userId); + if (!res.IsSuccess) return null; + + _cache[userId] = res.Entity; + return res.Entity; + } + + public void UpdateUser(IUser user) => _cache[user.ID] = user; +} \ No newline at end of file diff --git a/Catalogger.Backend/Catalogger.Backend.csproj b/Catalogger.Backend/Catalogger.Backend.csproj new file mode 100644 index 0000000..8967295 --- /dev/null +++ b/Catalogger.Backend/Catalogger.Backend.csproj @@ -0,0 +1,33 @@ + + + + net8.0 + enable + enable + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + diff --git a/Catalogger.Backend/CataloggerError.cs b/Catalogger.Backend/CataloggerError.cs new file mode 100644 index 0000000..f4c021d --- /dev/null +++ b/Catalogger.Backend/CataloggerError.cs @@ -0,0 +1,5 @@ +namespace Catalogger.Backend; + +public class CataloggerError(string message) : Exception(message) +{ +} \ No newline at end of file diff --git a/Catalogger.Backend/Config.cs b/Catalogger.Backend/Config.cs new file mode 100644 index 0000000..ee1a062 --- /dev/null +++ b/Catalogger.Backend/Config.cs @@ -0,0 +1,42 @@ +using Serilog.Events; + +namespace Catalogger.Backend; + +public class Config +{ + public LoggingConfig Logging { get; init; } = new(); + public DatabaseConfig Database { get; init; } = new(); + public DiscordConfig Discord { get; init; } = new(); + public WebConfig Web { get; init; } = new(); + + public class LoggingConfig + { + public LogEventLevel LogEventLevel { get; init; } = LogEventLevel.Debug; + public bool LogQueries { get; init; } = false; + } + + public class DatabaseConfig + { + public string Url { get; init; } = string.Empty; + public string Redis { get; init; } = string.Empty; + public int? Timeout { get; init; } + public int? MaxPoolSize { get; init; } + public string EncryptionKey { get; init; } = string.Empty; + } + + public class DiscordConfig + { + public ulong ApplicationId { get; set; } + public string Token { get; init; } = string.Empty; + public bool SyncCommands { get; init; } + public ulong? CommandsGuildId { get; init; } + } + + public class WebConfig + { + public string Host { get; init; } = "localhost"; + public int Port { get; init; } = 5000; + public string BaseUrl { get; init; } = null!; + public string Address => $"http://{Host}:{Port}"; + } +} \ No newline at end of file diff --git a/Catalogger.Backend/Database/DatabaseContext.cs b/Catalogger.Backend/Database/DatabaseContext.cs new file mode 100644 index 0000000..3126782 --- /dev/null +++ b/Catalogger.Backend/Database/DatabaseContext.cs @@ -0,0 +1,97 @@ +using Catalogger.Backend.Database.Models; +using Catalogger.Backend.Extensions; +using EntityFramework.Exceptions.PostgreSQL; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.ChangeTracking; +using Microsoft.EntityFrameworkCore.Design; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql; + +namespace Catalogger.Backend.Database; + +public class DatabaseContext : DbContext +{ + private readonly NpgsqlDataSource _dataSource; + private readonly ILoggerFactory? _loggerFactory; + + public DbSet Guilds { get; set; } + public DbSet Messages { get; set; } + public DbSet IgnoredMessages { get; set; } + public DbSet Invites { get; set; } + public DbSet Watchlists { get; set; } + + public DatabaseContext(Config config, ILoggerFactory? loggerFactory) + { + var connString = new NpgsqlConnectionStringBuilder(config.Database.Url) + { + Timeout = config.Database.Timeout ?? 5, + MaxPoolSize = config.Database.MaxPoolSize ?? 50, + }.ConnectionString; + + var dataSourceBuilder = new NpgsqlDataSourceBuilder(connString); + dataSourceBuilder + .EnableDynamicJson() + .UseNodaTime(); + _dataSource = dataSourceBuilder.Build(); + _loggerFactory = loggerFactory; + } + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder + .ConfigureWarnings(c => + c.Ignore(CoreEventId.ManyServiceProvidersCreatedWarning) + .Ignore(CoreEventId.SaveChangesFailed)) + .UseNpgsql(_dataSource, o => o.UseNodaTime()) + .UseSnakeCaseNamingConvention() + .UseLoggerFactory(_loggerFactory) + .UseExceptionProcessor(); + + protected override void ConfigureConventions(ModelConfigurationBuilder configurationBuilder) + { + configurationBuilder.Properties().HaveConversion(); + configurationBuilder.Properties>().HaveConversion(); + } + + private static readonly ValueComparer> UlongListValueComparer = new( + (c1, c2) => c1 != null && c2 != null && c1.SequenceEqual(c2), + c => c.Aggregate(0, (a, v) => HashCode.Combine(a, v.GetHashCode())) + ); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity().Property(g => g.KeyRoles) + .Metadata.SetValueComparer(UlongListValueComparer); + + modelBuilder.Entity().HasKey(i => i.Code); + modelBuilder.Entity().HasIndex(i => i.GuildId); + + modelBuilder.Entity().HasKey(w => new { w.GuildId, w.UserId }); + modelBuilder.Entity().Property(w => w.AddedAt).HasDefaultValueSql("now()"); + } +} + +public class DesignTimeDatabaseContextFactory : IDesignTimeDbContextFactory +{ + public DatabaseContext CreateDbContext(string[] args) + { + // Read the configuration file + var config = new ConfigurationBuilder() + .AddConfiguration() + .Build() + // Get the configuration as our config class + .Get() ?? new(); + + return new DatabaseContext(config, null); + } +} + +public class UlongValueConverter() : ValueConverter( + convertToProviderExpression: x => (long)x, + convertFromProviderExpression: x => (ulong)x +); + +public class UlongArrayValueConverter() : ValueConverter, List>( + convertToProviderExpression: x => x.Select(i => (long)i).ToList(), + convertFromProviderExpression: x => x.Select(i => (ulong)i).ToList() +); \ No newline at end of file diff --git a/Catalogger.Backend/Database/EncryptionService.cs b/Catalogger.Backend/Database/EncryptionService.cs new file mode 100644 index 0000000..7cfd908 --- /dev/null +++ b/Catalogger.Backend/Database/EncryptionService.cs @@ -0,0 +1,36 @@ +using System.Security.Cryptography; +using System.Text; + +namespace Catalogger.Backend.Database; + +public class EncryptionService(Config config) : IEncryptionService +{ + private readonly byte[] _secretKey = Convert.FromBase64String(config.Database.EncryptionKey); + + public byte[] Encrypt(string data) + { + using var aes = Aes.Create(); + aes.Key = _secretKey; + + var output = new List(); + output.AddRange(aes.IV); + + var plaintext = Encoding.UTF8.GetBytes(data); + var ciphertext = aes.EncryptCbc(plaintext, aes.IV); + + output.AddRange(ciphertext); + return output.ToArray(); + } + + public string Decrypt(byte[] input) + { + using var aes = Aes.Create(); + aes.Key = _secretKey; + + var iv = input.Take(aes.IV.Length).ToArray(); + var ciphertext = input.Skip(aes.IV.Length).ToArray(); + var plaintext = aes.DecryptCbc(ciphertext, iv); + + return Encoding.UTF8.GetString(plaintext); + } +} \ No newline at end of file diff --git a/Catalogger.Backend/Database/IEncryptionService.cs b/Catalogger.Backend/Database/IEncryptionService.cs new file mode 100644 index 0000000..df13477 --- /dev/null +++ b/Catalogger.Backend/Database/IEncryptionService.cs @@ -0,0 +1,7 @@ +namespace Catalogger.Backend.Database; + +public interface IEncryptionService +{ + public byte[] Encrypt(string data); + public string Decrypt(byte[] input); +} \ No newline at end of file diff --git a/Catalogger.Backend/Database/Migrations/20240803132306_Init.Designer.cs b/Catalogger.Backend/Database/Migrations/20240803132306_Init.Designer.cs new file mode 100644 index 0000000..9167588 --- /dev/null +++ b/Catalogger.Backend/Database/Migrations/20240803132306_Init.Designer.cs @@ -0,0 +1,180 @@ +// +using System.Collections.Generic; +using Catalogger.Backend.Database; +using Catalogger.Backend.Database.Models; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using NodaTime; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace Catalogger.Backend.Database.Migrations +{ + [DbContext(typeof(DatabaseContext))] + [Migration("20240803132306_Init")] + partial class Init + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Catalogger.Backend.Database.Models.Guild", b => + { + b.Property("Id") + .HasColumnType("bigint") + .HasColumnName("id"); + + b.Property>("BannedSystems") + .IsRequired() + .HasColumnType("text[]") + .HasColumnName("banned_systems"); + + b.Property("Channels") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("channels"); + + b.Property>("KeyRoles") + .IsRequired() + .HasColumnType("bigint[]") + .HasColumnName("key_roles"); + + b.HasKey("Id") + .HasName("pk_guilds"); + + b.ToTable("guilds", (string)null); + }); + + modelBuilder.Entity("Catalogger.Backend.Database.Models.IgnoredMessage", b => + { + b.Property("Id") + .HasColumnType("bigint") + .HasColumnName("id"); + + b.HasKey("Id") + .HasName("pk_ignored_messages"); + + b.ToTable("ignored_messages", (string)null); + }); + + modelBuilder.Entity("Catalogger.Backend.Database.Models.Invite", b => + { + b.Property("Code") + .HasColumnType("text") + .HasColumnName("code"); + + b.Property("GuildId") + .HasColumnType("bigint") + .HasColumnName("guild_id"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Code") + .HasName("pk_invites"); + + b.HasIndex("GuildId") + .HasDatabaseName("ix_invites_guild_id"); + + b.ToTable("invites", (string)null); + }); + + modelBuilder.Entity("Catalogger.Backend.Database.Models.Message", b => + { + b.Property("Id") + .HasColumnType("bigint") + .HasColumnName("id"); + + b.Property("AttachmentSize") + .HasColumnType("integer") + .HasColumnName("attachment_size"); + + b.Property("ChannelId") + .HasColumnType("bigint") + .HasColumnName("channel_id"); + + b.Property("EncryptedContent") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("content"); + + b.Property("EncryptedMetadata") + .HasColumnType("bytea") + .HasColumnName("metadata"); + + b.Property("EncryptedUsername") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("username"); + + b.Property("GuildId") + .HasColumnType("bigint") + .HasColumnName("guild_id"); + + b.Property("Member") + .HasColumnType("text") + .HasColumnName("member"); + + b.Property("OriginalId") + .HasColumnType("bigint") + .HasColumnName("original_id"); + + b.Property("System") + .HasColumnType("text") + .HasColumnName("system"); + + b.Property("UserId") + .HasColumnType("bigint") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("pk_messages"); + + b.ToTable("messages", (string)null); + }); + + modelBuilder.Entity("Catalogger.Backend.Database.Models.Watchlist", b => + { + b.Property("GuildId") + .HasColumnType("bigint") + .HasColumnName("guild_id"); + + b.Property("UserId") + .HasColumnType("bigint") + .HasColumnName("user_id"); + + b.Property("AddedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("added_at") + .HasDefaultValueSql("now()"); + + b.Property("ModeratorId") + .HasColumnType("bigint") + .HasColumnName("moderator_id"); + + b.Property("Reason") + .IsRequired() + .HasColumnType("text") + .HasColumnName("reason"); + + b.HasKey("GuildId", "UserId") + .HasName("pk_watchlists"); + + b.ToTable("watchlists", (string)null); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Catalogger.Backend/Database/Migrations/20240803132306_Init.cs b/Catalogger.Backend/Database/Migrations/20240803132306_Init.cs new file mode 100644 index 0000000..7867acd --- /dev/null +++ b/Catalogger.Backend/Database/Migrations/20240803132306_Init.cs @@ -0,0 +1,115 @@ +using System.Collections.Generic; +using Catalogger.Backend.Database.Models; +using Microsoft.EntityFrameworkCore.Migrations; +using NodaTime; + +#nullable disable + +namespace Catalogger.Backend.Database.Migrations +{ + /// + public partial class Init : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "guilds", + columns: table => new + { + id = table.Column(type: "bigint", nullable: false), + channels = table.Column(type: "jsonb", nullable: false), + banned_systems = table.Column>(type: "text[]", nullable: false), + key_roles = table.Column>(type: "bigint[]", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_guilds", x => x.id); + }); + + migrationBuilder.CreateTable( + name: "ignored_messages", + columns: table => new + { + id = table.Column(type: "bigint", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_ignored_messages", x => x.id); + }); + + migrationBuilder.CreateTable( + name: "invites", + columns: table => new + { + code = table.Column(type: "text", nullable: false), + guild_id = table.Column(type: "bigint", nullable: false), + name = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_invites", x => x.code); + }); + + migrationBuilder.CreateTable( + name: "messages", + columns: table => new + { + id = table.Column(type: "bigint", nullable: false), + original_id = table.Column(type: "bigint", nullable: true), + user_id = table.Column(type: "bigint", nullable: false), + channel_id = table.Column(type: "bigint", nullable: false), + guild_id = table.Column(type: "bigint", nullable: false), + member = table.Column(type: "text", nullable: true), + system = table.Column(type: "text", nullable: true), + username = table.Column(type: "bytea", nullable: false), + content = table.Column(type: "bytea", nullable: false), + metadata = table.Column(type: "bytea", nullable: true), + attachment_size = table.Column(type: "integer", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_messages", x => x.id); + }); + + migrationBuilder.CreateTable( + name: "watchlists", + columns: table => new + { + guild_id = table.Column(type: "bigint", nullable: false), + user_id = table.Column(type: "bigint", nullable: false), + added_at = table.Column(type: "timestamp with time zone", nullable: false, defaultValueSql: "now()"), + moderator_id = table.Column(type: "bigint", nullable: false), + reason = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_watchlists", x => new { x.guild_id, x.user_id }); + }); + + migrationBuilder.CreateIndex( + name: "ix_invites_guild_id", + table: "invites", + column: "guild_id"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "guilds"); + + migrationBuilder.DropTable( + name: "ignored_messages"); + + migrationBuilder.DropTable( + name: "invites"); + + migrationBuilder.DropTable( + name: "messages"); + + migrationBuilder.DropTable( + name: "watchlists"); + } + } +} diff --git a/Catalogger.Backend/Database/Migrations/DatabaseContextModelSnapshot.cs b/Catalogger.Backend/Database/Migrations/DatabaseContextModelSnapshot.cs new file mode 100644 index 0000000..c209770 --- /dev/null +++ b/Catalogger.Backend/Database/Migrations/DatabaseContextModelSnapshot.cs @@ -0,0 +1,177 @@ +// +using System.Collections.Generic; +using Catalogger.Backend.Database; +using Catalogger.Backend.Database.Models; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using NodaTime; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace Catalogger.Backend.Database.Migrations +{ + [DbContext(typeof(DatabaseContext))] + partial class DatabaseContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Catalogger.Backend.Database.Models.Guild", b => + { + b.Property("Id") + .HasColumnType("bigint") + .HasColumnName("id"); + + b.Property>("BannedSystems") + .IsRequired() + .HasColumnType("text[]") + .HasColumnName("banned_systems"); + + b.Property("Channels") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("channels"); + + b.Property>("KeyRoles") + .IsRequired() + .HasColumnType("bigint[]") + .HasColumnName("key_roles"); + + b.HasKey("Id") + .HasName("pk_guilds"); + + b.ToTable("guilds", (string)null); + }); + + modelBuilder.Entity("Catalogger.Backend.Database.Models.IgnoredMessage", b => + { + b.Property("Id") + .HasColumnType("bigint") + .HasColumnName("id"); + + b.HasKey("Id") + .HasName("pk_ignored_messages"); + + b.ToTable("ignored_messages", (string)null); + }); + + modelBuilder.Entity("Catalogger.Backend.Database.Models.Invite", b => + { + b.Property("Code") + .HasColumnType("text") + .HasColumnName("code"); + + b.Property("GuildId") + .HasColumnType("bigint") + .HasColumnName("guild_id"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Code") + .HasName("pk_invites"); + + b.HasIndex("GuildId") + .HasDatabaseName("ix_invites_guild_id"); + + b.ToTable("invites", (string)null); + }); + + modelBuilder.Entity("Catalogger.Backend.Database.Models.Message", b => + { + b.Property("Id") + .HasColumnType("bigint") + .HasColumnName("id"); + + b.Property("AttachmentSize") + .HasColumnType("integer") + .HasColumnName("attachment_size"); + + b.Property("ChannelId") + .HasColumnType("bigint") + .HasColumnName("channel_id"); + + b.Property("EncryptedContent") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("content"); + + b.Property("EncryptedMetadata") + .HasColumnType("bytea") + .HasColumnName("metadata"); + + b.Property("EncryptedUsername") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("username"); + + b.Property("GuildId") + .HasColumnType("bigint") + .HasColumnName("guild_id"); + + b.Property("Member") + .HasColumnType("text") + .HasColumnName("member"); + + b.Property("OriginalId") + .HasColumnType("bigint") + .HasColumnName("original_id"); + + b.Property("System") + .HasColumnType("text") + .HasColumnName("system"); + + b.Property("UserId") + .HasColumnType("bigint") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("pk_messages"); + + b.ToTable("messages", (string)null); + }); + + modelBuilder.Entity("Catalogger.Backend.Database.Models.Watchlist", b => + { + b.Property("GuildId") + .HasColumnType("bigint") + .HasColumnName("guild_id"); + + b.Property("UserId") + .HasColumnType("bigint") + .HasColumnName("user_id"); + + b.Property("AddedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("added_at") + .HasDefaultValueSql("now()"); + + b.Property("ModeratorId") + .HasColumnType("bigint") + .HasColumnName("moderator_id"); + + b.Property("Reason") + .IsRequired() + .HasColumnType("text") + .HasColumnName("reason"); + + b.HasKey("GuildId", "UserId") + .HasName("pk_watchlists"); + + b.ToTable("watchlists", (string)null); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Catalogger.Backend/Database/Models/Guild.cs b/Catalogger.Backend/Database/Models/Guild.cs new file mode 100644 index 0000000..1c0d11f --- /dev/null +++ b/Catalogger.Backend/Database/Models/Guild.cs @@ -0,0 +1,60 @@ +using System.ComponentModel.DataAnnotations.Schema; +using Catalogger.Backend.Extensions; +using Remora.Rest.Core; + +namespace Catalogger.Backend.Database.Models; + +public class Guild +{ + [DatabaseGenerated(DatabaseGeneratedOption.None)] + public required ulong Id { get; init; } + + [Column(TypeName = "jsonb")] + public ChannelConfig Channels { get; init; } = new(); + public List BannedSystems { get; init; } = []; + public List KeyRoles { get; init; } = []; + + public bool IsMessageIgnored(Snowflake channelId, Snowflake userId) + { + if (Channels is { MessageDelete: 0, MessageUpdate: 0, MessageDeleteBulk: 0 } || + Channels.IgnoredChannels.Contains(channelId.ToUlong()) || + Channels.IgnoredUsers.Contains(userId.ToUlong())) return true; + + if (Channels.IgnoredUsersPerChannel.TryGetValue(channelId.ToUlong(), + out var thisChannelIgnoredUsers)) + return thisChannelIgnoredUsers.Contains(userId.ToUlong()); + + return false; + } + + public class ChannelConfig + { + public List IgnoredChannels { get; init; } = []; + public List IgnoredUsers { get; init; } = []; + public Dictionary> IgnoredUsersPerChannel { get; init; } = []; + public Dictionary Redirects { get; init; } = []; + + public ulong GuildUpdate { get; init; } + public ulong GuildEmojisUpdate { get; init; } + public ulong GuildRoleCreate { get; init; } + public ulong GuildRoleUpdate { get; init; } + public ulong GuildRoleDelete { get; init; } + public ulong ChannelCreate { get; init; } + public ulong ChannelUpdate { get; init; } + public ulong ChannelDelete { get; init; } + public ulong GuildMemberAdd { get; init; } + public ulong GuildMemberUpdate { get; init; } + public ulong GuildKeyRoleUpdate { get; init; } + public ulong GuildMemberNickUpdate { get; init; } + public ulong GuildMemberAvatarUpdate { get; init; } + public ulong GuildMemberRemove { get; init; } + public ulong GuildMemberKick { get; init; } + public ulong GuildBanAdd { get; init; } + public ulong GuildBanRemove { get; init; } + public ulong InviteCreate { get; init; } + public ulong InviteDelete { get; init; } + public ulong MessageUpdate { get; init; } + public ulong MessageDelete { get; init; } + public ulong MessageDeleteBulk { get; init; } + } +} \ No newline at end of file diff --git a/Catalogger.Backend/Database/Models/Invite.cs b/Catalogger.Backend/Database/Models/Invite.cs new file mode 100644 index 0000000..9f76b2c --- /dev/null +++ b/Catalogger.Backend/Database/Models/Invite.cs @@ -0,0 +1,8 @@ +namespace Catalogger.Backend.Database.Models; + +public class Invite +{ + public required ulong GuildId { get; init; } + public required string Code { get; init; } + public required string Name { get; set; } +} \ No newline at end of file diff --git a/Catalogger.Backend/Database/Models/Message.cs b/Catalogger.Backend/Database/Models/Message.cs new file mode 100644 index 0000000..40b55e8 --- /dev/null +++ b/Catalogger.Backend/Database/Models/Message.cs @@ -0,0 +1,27 @@ +using System.ComponentModel.DataAnnotations.Schema; + +namespace Catalogger.Backend.Database.Models; + +public class Message +{ + [DatabaseGenerated(DatabaseGeneratedOption.None)] + public required ulong Id { get; init; } + + public ulong? OriginalId { get; set; } + public required ulong UserId { get; set; } + public required ulong ChannelId { get; init; } + public required ulong GuildId { get; init; } + + public string? Member { get; set; } + public string? System { get; set; } + + [Column("username")] public byte[] EncryptedUsername { get; set; } = []; + [Column("content")] public byte[] EncryptedContent { get; set; } = []; + [Column("metadata")] public byte[]? EncryptedMetadata { get; set; } + + public int AttachmentSize { get; set; } = 0; +} + +public record IgnoredMessage( + [property: DatabaseGenerated(DatabaseGeneratedOption.None)] + ulong Id); \ No newline at end of file diff --git a/Catalogger.Backend/Database/Models/Watchlist.cs b/Catalogger.Backend/Database/Models/Watchlist.cs new file mode 100644 index 0000000..61bff48 --- /dev/null +++ b/Catalogger.Backend/Database/Models/Watchlist.cs @@ -0,0 +1,13 @@ +using NodaTime; + +namespace Catalogger.Backend.Database.Models; + +public class Watchlist +{ + public required ulong GuildId { get; init; } + public required ulong UserId { get; init; } + public Instant AddedAt { get; init; } + + public required ulong ModeratorId { get; set; } + public required string Reason { get; set; } +} \ No newline at end of file diff --git a/Catalogger.Backend/Database/Queries/MessageRepository.cs b/Catalogger.Backend/Database/Queries/MessageRepository.cs new file mode 100644 index 0000000..b140a46 --- /dev/null +++ b/Catalogger.Backend/Database/Queries/MessageRepository.cs @@ -0,0 +1,99 @@ +using Catalogger.Backend.Database.Models; +using Catalogger.Backend.Extensions; +using Microsoft.EntityFrameworkCore; +using Remora.Discord.API.Abstractions.Gateway.Events; +using DbMessage = Catalogger.Backend.Database.Models.Message; + +namespace Catalogger.Backend.Database.Queries; + +public class MessageRepository(ILogger logger, DatabaseContext db, IEncryptionService encryptionService) +{ + private readonly ILogger _logger = logger.ForContext(); + + public async Task SaveMessageAsync(IMessageCreate msg, CancellationToken ct = default) + { + _logger.Debug("Saving message {MessageId}", msg.ID); + + var dbMessage = new DbMessage + { + Id = msg.ID.ToUlong(), + UserId = msg.Author.ID.ToUlong(), + ChannelId = msg.ChannelID.ToUlong(), + GuildId = msg.GuildID.ToUlong(), + + EncryptedContent = await Task.Run(() => encryptionService.Encrypt(msg.Content), ct), + EncryptedUsername = await Task.Run(() => encryptionService.Encrypt(msg.Author.Tag()), ct), + AttachmentSize = msg.Attachments.Select(a => a.Size).Sum() + }; + + db.Add(dbMessage); + await db.SaveChangesAsync(ct); + } + + public async Task GetMessageAsync(ulong id, CancellationToken ct = default) + { + _logger.Debug("Retrieving message {MessageId}", id); + + var dbMsg = await db.Messages.FindAsync(id); + if (dbMsg == null) return null; + + return new Message(dbMsg.Id, dbMsg.OriginalId, dbMsg.UserId, dbMsg.ChannelId, dbMsg.GuildId, dbMsg.Member, + dbMsg.System, await Task.Run(() => encryptionService.Decrypt(dbMsg.EncryptedUsername), ct), + await Task.Run(() => encryptionService.Decrypt(dbMsg.EncryptedContent), ct), null, dbMsg.AttachmentSize); + } + + /// + /// Checks if a message has proxy information. + /// If yes, returns (true, true). If no, returns (true, false). If the message isn't saved at all, returns (false, false). + /// + public async Task<(bool, bool)> HasProxyInfoAsync(ulong id) + { + _logger.Debug("Checking if message {MessageId} has proxy information", id); + + var msg = await db.Messages.Select(m => new { m.Id, m.OriginalId }).FirstOrDefaultAsync(m => m.Id == id); + return (msg != null, msg?.OriginalId != null); + } + + public async Task SetProxiedMessageDataAsync(ulong id, ulong originalId, ulong authorId, string? systemId, + string? memberId) + { + _logger.Debug("Setting proxy information for message {MessageId}", id); + + var message = await db.Messages.FirstOrDefaultAsync(m => m.Id == id); + if (message == null) + { + _logger.Debug("Message {MessageId} not found", id); + return; + } + + _logger.Debug("Updating message {MessageId}", id); + + message.OriginalId = originalId; + message.UserId = authorId; + message.System = systemId; + message.Member = memberId; + + db.Update(message); + await db.SaveChangesAsync(); + } + + public async Task IsMessageIgnoredAsync(ulong id, CancellationToken ct = default) + { + _logger.Debug("Checking if message {MessageId} is ignored", id); + return await db.IgnoredMessages.FirstOrDefaultAsync(m => m.Id == id, ct) != null; + } + + public record Message( + ulong Id, + ulong? OriginalId, + ulong UserId, + ulong ChannelId, + ulong GuildId, + string? Member, + string? System, + string Username, + string Content, + string? Metadata, + int AttachmentSize + ); +} \ No newline at end of file diff --git a/Catalogger.Backend/Database/Queries/QueryExtensions.cs b/Catalogger.Backend/Database/Queries/QueryExtensions.cs new file mode 100644 index 0000000..9180010 --- /dev/null +++ b/Catalogger.Backend/Database/Queries/QueryExtensions.cs @@ -0,0 +1,22 @@ +using Catalogger.Backend.Database.Models; +using Catalogger.Backend.Extensions; +using Remora.Rest.Core; + +namespace Catalogger.Backend.Database.Queries; + +public static class QueryExtensions +{ + public static async ValueTask GetGuildAsync(this DatabaseContext db, Snowflake id, + CancellationToken ct = default) => await db.GetGuildAsync(id.ToUlong(), ct); + + public static async ValueTask GetGuildAsync(this DatabaseContext db, Optional id, + CancellationToken ct = default) => await db.GetGuildAsync(id.ToUlong(), ct); + + public static async ValueTask GetGuildAsync(this DatabaseContext db, ulong id, + CancellationToken ct = default) + { + var guild = await db.Guilds.FindAsync(id); + if (guild == null) throw new Exception("oh"); + return guild; + } +} \ No newline at end of file diff --git a/Catalogger.Backend/Database/QueryUtils.cs b/Catalogger.Backend/Database/QueryUtils.cs new file mode 100644 index 0000000..5459fdc --- /dev/null +++ b/Catalogger.Backend/Database/QueryUtils.cs @@ -0,0 +1,3 @@ +using NodaTime; + +namespace Catalogger.Backend.Database; diff --git a/Catalogger.Backend/Extensions/DiscordExtensions.cs b/Catalogger.Backend/Extensions/DiscordExtensions.cs new file mode 100644 index 0000000..bb01284 --- /dev/null +++ b/Catalogger.Backend/Extensions/DiscordExtensions.cs @@ -0,0 +1,44 @@ +using Remora.Discord.API.Abstractions.Objects; +using Remora.Rest.Core; +using Remora.Results; + +namespace Catalogger.Backend.Extensions; + +public static class DiscordExtensions +{ + public static string Tag(this IPartialUser user) + { + var discriminator = user.Discriminator.OrDefault(); + return discriminator == 0 ? user.Username.Value : $"{user.Username.Value}#{discriminator:0000}"; + } + + public static string AvatarUrl(this IUser user, int size = 256) + { + if (user.Avatar != null) + { + var ext = user.Avatar.HasGif ? ".gif" : ".webp"; + return $"https://cdn.discordapp.com/avatars/{user.ID}/{user.Avatar.Value}{ext}?size={size}"; + } + + var avatarIndex = user.Discriminator == 0 ? (int)((user.ID.Value >> 22) % 6) : user.Discriminator % 5; + return $"https://cdn.discordapp.com/embed/avatars/{avatarIndex}.png?size={size}"; + } + + public static ulong ToUlong(this Snowflake snowflake) => snowflake.Value; + + public static ulong ToUlong(this Optional snowflake) + { + if (!snowflake.IsDefined()) throw new Exception("ToUlong called on an undefined Snowflake"); + return snowflake.Value.Value; + } + + public static T GetOrThrow(this Result result) + { + if (result.Error != null) throw new DiscordRestException(result.Error.Message); + return result.Entity; + } + + public static async Task GetOrThrow(this Task> result) => (await result).GetOrThrow(); + + public class DiscordRestException(string message) : Exception(message); +} \ No newline at end of file diff --git a/Catalogger.Backend/Extensions/StartupExtensions.cs b/Catalogger.Backend/Extensions/StartupExtensions.cs new file mode 100644 index 0000000..ddd5504 --- /dev/null +++ b/Catalogger.Backend/Extensions/StartupExtensions.cs @@ -0,0 +1,121 @@ +using Catalogger.Backend.Bot.Responders; +using Catalogger.Backend.Cache; +using Catalogger.Backend.Database; +using Catalogger.Backend.Database.Queries; +using Catalogger.Backend.Services; +using Microsoft.EntityFrameworkCore; +using NodaTime; +using Remora.Discord.API; +using Remora.Discord.API.Abstractions.Rest; +using Remora.Discord.Commands.Services; +using Serilog; +using Serilog.Events; + +namespace Catalogger.Backend.Extensions; + +public static class StartupExtensions +{ + /// + /// Adds Serilog to this service collection. This method also initializes Serilog, so it should be called as early as possible, before any log calls. + /// + public static WebApplicationBuilder AddSerilog(this WebApplicationBuilder builder) + { + var config = builder.Configuration.Get() ?? new(); + + var logCfg = new LoggerConfiguration() + .Enrich.FromLogContext() + .MinimumLevel.Is(config.Logging.LogEventLevel) + // ASP.NET's built in request logs are extremely verbose, so we use Serilog's instead. + // Serilog doesn't disable the built-in logs, so we do it here. + .MinimumLevel.Override("Microsoft", LogEventLevel.Information) + .MinimumLevel.Override("Microsoft.EntityFrameworkCore.Database.Command", + config.Logging.LogQueries ? LogEventLevel.Information : LogEventLevel.Fatal) + .MinimumLevel.Override("Microsoft.AspNetCore.Hosting", LogEventLevel.Warning) + .MinimumLevel.Override("Microsoft.AspNetCore.Mvc", LogEventLevel.Warning) + .MinimumLevel.Override("Microsoft.AspNetCore.Routing", LogEventLevel.Warning) + .WriteTo.Console(); + + // AddSerilog doesn't seem to add an ILogger to the service collection, so add that manually. + builder.Services.AddSerilog().AddSingleton(Log.Logger = logCfg.CreateLogger()); + + return builder; + } + + public static Config AddConfiguration(this WebApplicationBuilder builder) + { + builder.Configuration.Sources.Clear(); + builder.Configuration.AddConfiguration(); + + var config = builder.Configuration.Get() ?? new(); + builder.Services.AddSingleton(config); + return config; + } + + public static IConfigurationBuilder AddConfiguration(this IConfigurationBuilder builder) + { + var file = Environment.GetEnvironmentVariable("CATALOGGER_CONFIG_FILE") ?? "config.ini"; + + return builder + .SetBasePath(Directory.GetCurrentDirectory()) + .AddIniFile(file, optional: false, reloadOnChange: false) + .AddEnvironmentVariables(); + } + + public static IServiceCollection AddCustomServices(this IServiceCollection services) => services + .AddSingleton(SystemClock.Instance) + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddScoped() + .AddScoped() + .AddSingleton() + .AddSingleton(); + + public static async Task Initialize(this WebApplication app) + { + await using var scope = app.Services.CreateAsyncScope(); + var logger = scope.ServiceProvider.GetRequiredService().ForContext(); + logger.Information("Starting Catalogger.NET"); + + await using (var db = scope.ServiceProvider.GetRequiredService()) + { + var migrationCount = (await db.Database.GetPendingMigrationsAsync()).Count(); + if (migrationCount != 0) + { + logger.Information("Applying {Count} database migrations", migrationCount); + await db.Database.MigrateAsync(); + } + else logger.Information("There are no pending migrations"); + } + + var config = scope.ServiceProvider.GetRequiredService(); + var slashService = scope.ServiceProvider.GetRequiredService(); + + if (config.Discord.ApplicationId == 0) + { + logger.Warning( + "Application ID not set in config. Fetching and setting it now, but for future restarts, please add it to config.ini as Discord.ApplicationId."); + var restApi = scope.ServiceProvider.GetRequiredService(); + var application = await restApi.GetCurrentApplicationAsync().GetOrThrow(); + config.Discord.ApplicationId = application.ID.ToUlong(); + logger.Information("Current application ID is {ApplicationId}", config.Discord.ApplicationId); + } + + if (config.Discord.SyncCommands) + { + if (config.Discord.CommandsGuildId != null) + { + logger.Information("Syncing application commands with guild {GuildId}", config.Discord.CommandsGuildId); + await slashService.UpdateSlashCommandsAsync( + guildID: DiscordSnowflake.New(config.Discord.CommandsGuildId.Value)); + } + else + { + logger.Information("Syncing application commands globally"); + await slashService.UpdateSlashCommandsAsync(); + } + } + else logger.Information("Not syncing slash commands, Discord.SyncCommands is false or unset"); + } +} \ No newline at end of file diff --git a/Catalogger.Backend/GlobalUsing.cs b/Catalogger.Backend/GlobalUsing.cs new file mode 100644 index 0000000..dd1fe48 --- /dev/null +++ b/Catalogger.Backend/GlobalUsing.cs @@ -0,0 +1 @@ +global using ILogger = Serilog.ILogger; diff --git a/Catalogger.Backend/Program.cs b/Catalogger.Backend/Program.cs new file mode 100644 index 0000000..71b8e98 --- /dev/null +++ b/Catalogger.Backend/Program.cs @@ -0,0 +1,71 @@ +using Catalogger.Backend.Bot.Commands; +using Catalogger.Backend.Database; +using Catalogger.Backend.Extensions; +using Newtonsoft.Json.Serialization; +using Remora.Commands.Extensions; +using Remora.Discord.API.Abstractions.Gateway.Commands; +using Remora.Discord.Commands.Extensions; +using Remora.Discord.Extensions.Extensions; +using Remora.Discord.Gateway; +using Remora.Discord.Hosting.Extensions; +using Remora.Discord.Interactivity.Extensions; +using Remora.Discord.Pagination.Extensions; +using Serilog; + +var builder = WebApplication.CreateBuilder(args); +var config = builder.AddConfiguration(); +builder.AddSerilog(); + +builder.Services + .AddControllers() + .AddNewtonsoftJson(o => o.SerializerSettings.ContractResolver = + new DefaultContractResolver + { + NamingStrategy = new SnakeCaseNamingStrategy() + }); + +builder.Host + .AddDiscordService(_ => config.Discord.Token) + .ConfigureServices(s => + s.AddRespondersFromAssembly(typeof(Program).Assembly) + .Configure(g => + g.Intents = GatewayIntents.Guilds | + GatewayIntents.GuildBans | + GatewayIntents.GuildInvites | + GatewayIntents.GuildMembers | + GatewayIntents.GuildMessages | + GatewayIntents.GuildWebhooks | + GatewayIntents.MessageContents | + GatewayIntents.GuildEmojisAndStickers) + .AddDiscordCommands(enableSlash: true) + .AddCommandTree() + // Start command tree + .WithCommandGroup() + // End command tree + .Finish() + .AddPagination() + .AddInteractivity() + ); + +builder.Services + .AddDbContext() + .AddCustomServices() + .AddEndpointsApiExplorer() + .AddSwaggerGen(); + +var app = builder.Build(); + +await app.Initialize(); + +app.UseSerilogRequestLogging(); +app.UseRouting(); +app.UseSwagger(); +app.UseSwaggerUI(); +app.UseCors(); +app.MapControllers(); + +app.Urls.Clear(); +app.Urls.Add(config.Web.Address); + +app.Run(); +Log.CloseAndFlush(); \ No newline at end of file diff --git a/Catalogger.Backend/Properties/launchSettings.json b/Catalogger.Backend/Properties/launchSettings.json new file mode 100644 index 0000000..5ed7c4b --- /dev/null +++ b/Catalogger.Backend/Properties/launchSettings.json @@ -0,0 +1,41 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:35403", + "sslPort": 44334 + } + }, + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "launchUrl": "swagger", + "applicationUrl": "http://localhost:5088", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "launchUrl": "swagger", + "applicationUrl": "https://localhost:7170;http://localhost:5088", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "launchUrl": "swagger", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/Catalogger.Backend/Services/IWebhookCache.cs b/Catalogger.Backend/Services/IWebhookCache.cs new file mode 100644 index 0000000..c3033b1 --- /dev/null +++ b/Catalogger.Backend/Services/IWebhookCache.cs @@ -0,0 +1,29 @@ +using Catalogger.Backend.Extensions; +using Remora.Discord.API; +using Remora.Discord.API.Abstractions.Objects; +using Remora.Rest.Core; + +namespace Catalogger.Backend.Services; + +public interface IWebhookCache +{ + Task GetWebhookAsync(ulong channelId); + Task SetWebhookAsync(ulong channelId, Webhook webhook); + + public async Task GetOrFetchWebhookAsync(ulong channelId, Func> fetch) + { + var webhook = await GetWebhookAsync(channelId); + if (webhook != null) return webhook.Value; + + var discordWebhook = await fetch(DiscordSnowflake.New(channelId)); + webhook = new Webhook { Id = discordWebhook.ID.ToUlong(), Token = discordWebhook.Token.Value}; + await SetWebhookAsync(channelId, webhook.Value); + return webhook.Value; + } +} + +public struct Webhook +{ + public required ulong Id { get; init; } + public required string Token { get; init; } +} \ No newline at end of file diff --git a/Catalogger.Backend/Services/InMemoryWebhookCache.cs b/Catalogger.Backend/Services/InMemoryWebhookCache.cs new file mode 100644 index 0000000..5d5d5af --- /dev/null +++ b/Catalogger.Backend/Services/InMemoryWebhookCache.cs @@ -0,0 +1,21 @@ +using System.Collections.Concurrent; + +namespace Catalogger.Backend.Services; + +public class InMemoryWebhookCache : IWebhookCache +{ + private readonly ConcurrentDictionary _cache = new(); + + public Task GetWebhookAsync(ulong channelId) + { + return _cache.TryGetValue(channelId, out var webhook) + ? Task.FromResult(webhook) + : Task.FromResult(null); + } + + public Task SetWebhookAsync(ulong channelId, Webhook webhook) + { + _cache[channelId] = webhook; + return Task.CompletedTask; + } +} \ No newline at end of file diff --git a/Catalogger.Backend/Services/PluralkitApiService.cs b/Catalogger.Backend/Services/PluralkitApiService.cs new file mode 100644 index 0000000..79e342f --- /dev/null +++ b/Catalogger.Backend/Services/PluralkitApiService.cs @@ -0,0 +1,73 @@ +using System.Net; +using System.Text.Json; +using System.Threading.RateLimiting; +using Humanizer; +using NodaTime; +using Polly; +using Remora.Rest.Json.Policies; + +namespace Catalogger.Backend.Services; + +public class PluralkitApiService(ILogger logger) +{ + private const string UserAgent = "Catalogger.NET (https://codeberg.org/starshine/catalogger)"; + private const string ApiBaseUrl = "https://api.pluralkit.me/v2"; + private readonly HttpClient _client = new(); + private readonly ILogger _logger = logger.ForContext(); + + private readonly ResiliencePipeline _pipeline = new ResiliencePipelineBuilder() + .AddRateLimiter(new FixedWindowRateLimiter(new FixedWindowRateLimiterOptions() + { + Window = 1.Seconds(), + PermitLimit = 2, + QueueLimit = 64, + })) + .AddTimeout(20.Seconds()) + .Build(); + + private async Task DoRequestAsync(string path, bool allowNotFound = false, + CancellationToken ct = default) where T : class + { + var req = new HttpRequestMessage(HttpMethod.Get, $"{ApiBaseUrl}{path}"); + req.Headers.Add("User-Agent", UserAgent); + + _logger.Debug("Requesting {Path} from PluralKit API", path); + + var resp = await _client.SendAsync(req, ct); + if (resp.StatusCode == HttpStatusCode.NotFound && allowNotFound) + { + _logger.Debug("PluralKit API path {Path} returned 404 but 404 response is valid", path); + return null; + } + + if (!resp.IsSuccessStatusCode) + { + _logger.Error("Received non-200 status code {StatusCode} from PluralKit API path {Path}", resp.StatusCode, + req); + throw new CataloggerError("Non-200 status code from PluralKit API"); + } + + return await resp.Content.ReadFromJsonAsync(new JsonSerializerOptions + { PropertyNamingPolicy = new SnakeCaseNamingPolicy() }, ct) ?? + throw new CataloggerError("JSON response from PluralKit API was null"); + } + + public async Task GetPluralKitMessageAsync(ulong id, CancellationToken ct = default) => + await DoRequestAsync($"/messages/{id}", allowNotFound: true, ct); + + public async Task GetPluralKitSystemAsync(ulong id, CancellationToken ct = default) => + (await DoRequestAsync($"/systems/{id}", allowNotFound: false, ct))!; + + public record PkMessage( + ulong Id, + ulong Original, + ulong Sender, + ulong Channel, + ulong Guild, + PkSystem? System, + PkMember? Member); + + public record PkSystem(string Id, Guid Uuid, string? Name, string? Tag, Instant? Created); + + public record PkMember(string Id, Guid Uuid, string Name, string? DisplayName); +} \ No newline at end of file diff --git a/Catalogger.Backend/Services/WebhookExecutorService.cs b/Catalogger.Backend/Services/WebhookExecutorService.cs new file mode 100644 index 0000000..881e719 --- /dev/null +++ b/Catalogger.Backend/Services/WebhookExecutorService.cs @@ -0,0 +1,186 @@ +using System.Collections.Concurrent; +using Catalogger.Backend.Cache; +using Catalogger.Backend.Database.Models; +using Catalogger.Backend.Extensions; +using Humanizer; +using Remora.Discord.API; +using Remora.Discord.API.Abstractions.Objects; +using Remora.Discord.API.Abstractions.Rest; +using Remora.Rest.Core; + +namespace Catalogger.Backend.Services; + +public class WebhookExecutorService( + Config config, + ILogger logger, + IWebhookCache webhookCache, + ChannelCacheService channelCache, + IDiscordRestWebhookAPI webhookApi) +{ + private readonly ILogger _logger = logger.ForContext(); + private readonly Snowflake _applicationId = DiscordSnowflake.New(config.Discord.ApplicationId); + private readonly ConcurrentDictionary> _cache = new(); + private readonly ConcurrentDictionary _timers = new(); + private IUser? _selfUser; + + public void SetSelfUser(IUser user) => _selfUser = user; + + public async Task QueueLogAsync(Guild guild, LogChannelType logChannelType, IEmbed embed) + { + var logChannel = GetLogChannel(guild, logChannelType, channelId: null, userId: null); + if (logChannel == null) return; + + await QueueLogAsync(logChannel.Value, embed); + } + + public async Task QueueLogAsync(ulong channelId, IEmbed embed) + { + _logger.Debug("Queueing embed for channel {ChannelId}", channelId); + var webhook = await webhookCache.GetOrFetchWebhookAsync(channelId, id => FetchWebhookAsync(id)); + + var queue = _cache.GetOrAdd(channelId, []); + if (queue.Count >= 5) + await SendLogsAsync(channelId); + queue.Enqueue(embed); + + if (_timers.TryGetValue(channelId, out var existingTimer)) await existingTimer.DisposeAsync(); + + _timers[channelId] = new Timer(_ => + { + var __ = SendLogsAsync(channelId); + }, null, 3000, Timeout.Infinite); + } + + private async Task SendLogsAsync(ulong channelId) + { + var queue = _cache.GetValueOrDefault(channelId); + if (queue == null) return; + var embeds = queue.Take(5).ToList(); + + var webhook = await webhookCache.GetOrFetchWebhookAsync(channelId, id => FetchWebhookAsync(id)); + + await webhookApi.ExecuteWebhookAsync(DiscordSnowflake.New(webhook.Id), webhook.Token, shouldWait: false, + embeds: embeds, username: _selfUser!.Username, avatarUrl: _selfUser.AvatarUrl()); + } + + private async Task FetchWebhookAsync(Snowflake channelId, CancellationToken ct = default) + { + var channelWebhooks = + await webhookApi.GetChannelWebhooksAsync(channelId, ct).GetOrThrow(); + var webhook = channelWebhooks.FirstOrDefault(w => w.ApplicationID == _applicationId && w.Token.IsDefined()); + if (webhook != null) return webhook; + + return await webhookApi.CreateWebhookAsync(channelId, "Catalogger", default, reason: "Creating logging webhook", + ct: ct).GetOrThrow(); + } + + public ulong? GetLogChannel(Guild guild, LogChannelType logChannelType, Snowflake? channelId = null, + ulong? userId = null) + { + if (channelId == null) return GetDefaultLogChannel(guild, logChannelType); + if (!channelCache.GetChannel(channelId.Value, out var channel)) return null; + + Snowflake? categoryId; + if (channel.Type is ChannelType.AnnouncementThread or ChannelType.PrivateThread or ChannelType.PublicThread) + { + // parent_id should always have a value for threads + channelId = channel.ParentID.Value!.Value; + if (!channelCache.GetChannel(channelId.Value, out var parentChannel)) + return GetDefaultLogChannel(guild, logChannelType); + categoryId = parentChannel.ParentID.Value; + } + else + { + channelId = channel.ID; + categoryId = channel.ParentID.Value; + } + + // Check if the channel, or its category, or the user is ignored + if (guild.Channels.IgnoredChannels.Contains(channelId.Value.Value) || + categoryId != null && guild.Channels.IgnoredChannels.Contains(categoryId.Value.Value)) return null; + if (userId != null) + { + if (guild.Channels.IgnoredUsers.Contains(userId.Value)) return null; + + // Check the channel-local and category-local ignored users + var channelIgnoredUsers = + guild.Channels.IgnoredUsersPerChannel.GetValueOrDefault(channelId.Value.Value) ?? []; + var categoryIgnoredUsers = (categoryId != null + ? guild.Channels.IgnoredUsersPerChannel.GetValueOrDefault(categoryId.Value.Value) + : []) ?? []; + if (channelIgnoredUsers.Concat(categoryIgnoredUsers).Contains(userId.Value)) return null; + } + + // These three events can be redirected to other channels. Redirects can be on a channel or category level. + // Obviously, the events are only redirected if they're supposed to be logged in the first place. + if (logChannelType is LogChannelType.MessageUpdate or LogChannelType.MessageDelete + or LogChannelType.MessageDeleteBulk) + { + if (GetDefaultLogChannel(guild, logChannelType) == null) return null; + + ulong categoryRedirect = categoryId != null + ? guild.Channels.Redirects.GetValueOrDefault(categoryId.Value.Value) + : 0; + + if (guild.Channels.Redirects.TryGetValue(channelId.Value.Value, out var channelRedirect)) + return channelRedirect; + if (categoryRedirect != 0) return categoryRedirect; + return GetDefaultLogChannel(guild, logChannelType); + } + + return GetDefaultLogChannel(guild, logChannelType); + } + + private ulong? GetDefaultLogChannel(Guild guild, LogChannelType channelType) => channelType switch + { + LogChannelType.GuildUpdate => guild.Channels.GuildUpdate, + LogChannelType.GuildEmojisUpdate => guild.Channels.GuildEmojisUpdate, + LogChannelType.GuildRoleCreate => guild.Channels.GuildRoleCreate, + LogChannelType.GuildRoleUpdate => guild.Channels.GuildRoleUpdate, + LogChannelType.GuildRoleDelete => guild.Channels.GuildRoleDelete, + LogChannelType.ChannelCreate => guild.Channels.ChannelCreate, + LogChannelType.ChannelUpdate => guild.Channels.ChannelUpdate, + LogChannelType.ChannelDelete => guild.Channels.ChannelDelete, + LogChannelType.GuildMemberAdd => guild.Channels.GuildMemberAdd, + LogChannelType.GuildMemberUpdate => guild.Channels.GuildMemberUpdate, + LogChannelType.GuildKeyRoleUpdate => guild.Channels.GuildKeyRoleUpdate, + LogChannelType.GuildMemberNickUpdate => guild.Channels.GuildMemberNickUpdate, + LogChannelType.GuildMemberAvatarUpdate => guild.Channels.GuildMemberAvatarUpdate, + LogChannelType.GuildMemberRemove => guild.Channels.GuildMemberRemove, + LogChannelType.GuildMemberKick => guild.Channels.GuildMemberKick, + LogChannelType.GuildBanAdd => guild.Channels.GuildBanAdd, + LogChannelType.GuildBanRemove => guild.Channels.GuildBanRemove, + LogChannelType.InviteCreate => guild.Channels.InviteCreate, + LogChannelType.InviteDelete => guild.Channels.InviteDelete, + LogChannelType.MessageUpdate => guild.Channels.MessageUpdate, + LogChannelType.MessageDelete => guild.Channels.MessageDelete, + LogChannelType.MessageDeleteBulk => guild.Channels.MessageDeleteBulk, + _ => throw new ArgumentOutOfRangeException(nameof(channelType)) + }; +} + +public enum LogChannelType +{ + GuildUpdate, + GuildEmojisUpdate, + GuildRoleCreate, + GuildRoleUpdate, + GuildRoleDelete, + ChannelCreate, + ChannelUpdate, + ChannelDelete, + GuildMemberAdd, + GuildMemberUpdate, + GuildKeyRoleUpdate, + GuildMemberNickUpdate, + GuildMemberAvatarUpdate, + GuildMemberRemove, + GuildMemberKick, + GuildBanAdd, + GuildBanRemove, + InviteCreate, + InviteDelete, + MessageUpdate, + MessageDelete, + MessageDeleteBulk +} \ No newline at end of file diff --git a/Catalogger.Backend/config.example.ini b/Catalogger.Backend/config.example.ini new file mode 100644 index 0000000..77c6f9b --- /dev/null +++ b/Catalogger.Backend/config.example.ini @@ -0,0 +1,14 @@ +[Logging] +LogEventLevel = Debug +LogQueries = false + +[Database] +Url = Host=localhost;Database=postgres;Username=postgres;Password=postgres +Redis = localhost:6379 +EncryptionKey = changeMe!FNmZbotJnAAJ7grWHDluCoKIwj6NcUagKE= # base64 key + +[Discord] +ApplicationId = +Token = +CommandsGuildId = +SyncCommands = true \ No newline at end of file diff --git a/catalogger.sln b/catalogger.sln new file mode 100644 index 0000000..fbeceb5 --- /dev/null +++ b/catalogger.sln @@ -0,0 +1,16 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Catalogger.Backend", "Catalogger.Backend\Catalogger.Backend.csproj", "{1C63F4B5-6BFE-4F45-9244-B76B36CF712B}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {1C63F4B5-6BFE-4F45-9244-B76B36CF712B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1C63F4B5-6BFE-4F45-9244-B76B36CF712B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1C63F4B5-6BFE-4F45-9244-B76B36CF712B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1C63F4B5-6BFE-4F45-9244-B76B36CF712B}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/catalogger.sln.DotSettings b/catalogger.sln.DotSettings new file mode 100644 index 0000000..cda3670 --- /dev/null +++ b/catalogger.sln.DotSettings @@ -0,0 +1,2 @@ + + True \ No newline at end of file