diff --git a/Catalogger.Backend/Bot/Responders/Guilds/GuildCreateResponder.cs b/Catalogger.Backend/Bot/Responders/Guilds/GuildCreateResponder.cs index 8e124bd..89865ba 100644 --- a/Catalogger.Backend/Bot/Responders/Guilds/GuildCreateResponder.cs +++ b/Catalogger.Backend/Bot/Responders/Guilds/GuildCreateResponder.cs @@ -93,6 +93,8 @@ public class GuildCreateResponder( return Task.FromResult(Result.Success); } + guildCache.Remove(evt.ID, out _); + if (!guildCache.TryGet(evt.ID, out var guild)) { _logger.Information("Left uncached guild {GuildId}", evt.ID); diff --git a/Catalogger.Backend/Bot/Responders/Messages/MessageDeleteBulkResponder.cs b/Catalogger.Backend/Bot/Responders/Messages/MessageDeleteBulkResponder.cs new file mode 100644 index 0000000..b03a6ae --- /dev/null +++ b/Catalogger.Backend/Bot/Responders/Messages/MessageDeleteBulkResponder.cs @@ -0,0 +1,142 @@ +using System.Text; +using Catalogger.Backend.Cache.InMemoryCache; +using Catalogger.Backend.Database; +using Catalogger.Backend.Database.Queries; +using Catalogger.Backend.Extensions; +using Catalogger.Backend.Services; +using NodaTime.Extensions; +using Remora.Discord.API.Abstractions.Gateway.Events; +using Remora.Discord.API.Abstractions.Objects; +using Remora.Discord.API.Abstractions.Rest; +using Remora.Discord.Extensions.Embeds; +using Remora.Discord.Gateway.Responders; +using Remora.Rest.Core; +using Remora.Results; + +namespace Catalogger.Backend.Bot.Responders.Messages; + +public class MessageDeleteBulkResponder( + ILogger logger, + DatabaseContext db, + MessageRepository messageRepository, + WebhookExecutorService webhookExecutor, + ChannelCache channelCache +) : IResponder +{ + private readonly ILogger _logger = logger.ForContext(); + + public async Task RespondAsync(IMessageDeleteBulk evt, CancellationToken ct = default) + { + var guild = await db.GetGuildAsync(evt.GuildID, ct); + if (guild.IsMessageIgnored(evt.ChannelID, null)) + return Result.Success; + + var logChannel = webhookExecutor.GetLogChannel( + guild, + LogChannelType.MessageDeleteBulk, + evt.ChannelID + ); + if (logChannel == null) + { + return Result.Success; + } + + IChannel? rootChannel = null; + channelCache.TryGet(evt.ChannelID, out var channel); + if ( + channel is + { + Type: ChannelType.AnnouncementThread + or ChannelType.PrivateThread + or ChannelType.PublicThread + } + ) + { + if (channel.ParentID.TryGet(out var parentId) && parentId != null) + channelCache.TryGet(parentId.Value, out rootChannel); + } + + List renderedMessages = []; + var notFoundMessages = 0; + var ignoredMessages = 0; + + foreach (var msgId in evt.IDs.Order()) + { + if (await messageRepository.IsMessageIgnoredAsync(msgId.Value, ct)) + { + ignoredMessages++; + continue; + } + + var msg = await messageRepository.GetMessageAsync(msgId.Value, ct); + renderedMessages.Add(RenderMessage(msgId, msg)); + + if (msg == null) + notFoundMessages++; + } + + var output = "Bulk message delete in"; + if (channel != null) + { + output += $" #{channel.Name} ({channel.ID})"; + if (rootChannel != null) + output += $" (thread in #{rootChannel.Name} ({rootChannel.ID})"; + } + else + { + output += $" unknown channel {evt.ChannelID}"; + } + + output += $"\nwith {renderedMessages.Count} messages\n\n"; + output += string.Join("\n", renderedMessages); + + var embed = new EmbedBuilder() + .WithTitle("Bulk message delete") + .WithDescription( + $""" + {evt.IDs.Count} messages were deleted in <#{evt.ChannelID}> + ({notFoundMessages} messages not found, {ignoredMessages} messages ignored) + """ + ) + .WithColour(DiscordUtils.Red) + .WithCurrentTimestamp(); + + await webhookExecutor.SendLogAsync( + logChannel.Value, + [embed.Build().GetOrThrow()], + [ + new FileData( + $"bulk-delete-{evt.ChannelID}.txt", + new MemoryStream(Encoding.UTF8.GetBytes(output)) + ), + ] + ); + return Result.Success; + } + + private string RenderMessage(Snowflake messageId, MessageRepository.Message? message) + { + var timestamp = messageId.Timestamp.ToOffsetDateTime().ToString(); + + if (message == null) + { + return $""" + [{timestamp}] Unknown message {messageId} + -------------------------------------------- + """; + } + + var builder = new StringBuilder(); + builder.Append($"[{timestamp}] {message.Username} ({message.UserId})\n"); + if (message is { System: not null, Member: not null }) + { + builder.Append($"PK system: {message.System} | PK member: {message.Member}\n"); + } + + builder.Append("--------------------------------------------\n"); + builder.Append(message.Content); + builder.Append("\n--------------------------------------------\n"); + + return builder.ToString(); + } +} diff --git a/Catalogger.Backend/Bot/ShardedGatewayClient.cs b/Catalogger.Backend/Bot/ShardedGatewayClient.cs index ec8a28b..9d98e68 100644 --- a/Catalogger.Backend/Bot/ShardedGatewayClient.cs +++ b/Catalogger.Backend/Bot/ShardedGatewayClient.cs @@ -19,7 +19,8 @@ public class ShardedGatewayClient( Config config ) : IDisposable { - private int _shardCount = config.Discord.ShardCount ?? 0; + public int TotalShards { get; private set; } = config.Discord.ShardCount ?? 0; + private readonly ILogger _logger = logger.ForContext(); private readonly ConcurrentDictionary _gatewayClients = new(); @@ -33,6 +34,9 @@ public class ShardedGatewayClient( GatewayConnectionStatus > GetConnectionStatus = client => (GatewayConnectionStatus)Field.GetValue(client)!; + public static bool IsConnected(DiscordGatewayClient client) => + GetConnectionStatus(client) == GatewayConnectionStatus.Connected; + public IReadOnlyDictionary Shards => _gatewayClients; public async Task RunAsync(CancellationToken ct = default) @@ -46,19 +50,19 @@ public class ShardedGatewayClient( if (gatewayResult.Entity.Shards.IsDefined(out var discordShardCount)) { - if (_shardCount < discordShardCount && _shardCount != 0) + if (TotalShards < discordShardCount && TotalShards != 0) _logger.Warning( "Discord recommends {DiscordShardCount} for this bot, but only {ConfigShardCount} shards are requested. This may cause issues later", discordShardCount, - _shardCount + TotalShards ); - if (_shardCount == 0) - _shardCount = discordShardCount; + if (TotalShards == 0) + TotalShards = discordShardCount; } var clients = Enumerable - .Range(0, _shardCount) + .Range(0, TotalShards) .Select(s => { var client = ActivatorUtilities.CreateInstance( @@ -74,7 +78,7 @@ public class ShardedGatewayClient( for (var shardIndex = 0; shardIndex < clients.Length; shardIndex++) { - _logger.Debug("Starting shard {ShardId}/{ShardCount}", shardIndex, _shardCount); + _logger.Debug("Starting shard {ShardId}/{ShardCount}", shardIndex, TotalShards); var client = clients[shardIndex]; var res = client.RunAsync(ct); @@ -93,13 +97,13 @@ public class ShardedGatewayClient( return res.Result; } - _logger.Information("Started shard {ShardId}/{ShardCount}", shardIndex, _shardCount); + _logger.Information("Started shard {ShardId}/{ShardCount}", shardIndex, TotalShards); } return await await Task.WhenAny(tasks); } - public int ShardIdFor(ulong guildId) => (int)((guildId >> 22) % (ulong)_shardCount); + public int ShardIdFor(ulong guildId) => (int)((guildId >> 22) % (ulong)TotalShards); public DiscordGatewayClient ClientFor(Snowflake guildId) => ClientFor(guildId.Value); @@ -112,6 +116,7 @@ public class ShardedGatewayClient( public void Dispose() { + GC.SuppressFinalize(this); foreach (var client in _gatewayClients.Values) client.Dispose(); } @@ -123,7 +128,7 @@ public class ShardedGatewayClient( { var ret = new DiscordGatewayClientOptions { - ShardIdentification = new ShardIdentification(shardId, _shardCount), + ShardIdentification = new ShardIdentification(shardId, TotalShards), Intents = options.Intents, Presence = options.Presence, ConnectionProperties = options.ConnectionProperties, diff --git a/Catalogger.Backend/Database/Models/Guild.cs b/Catalogger.Backend/Database/Models/Guild.cs index 6ab1053..8a1cd90 100644 --- a/Catalogger.Backend/Database/Models/Guild.cs +++ b/Catalogger.Backend/Database/Models/Guild.cs @@ -18,22 +18,25 @@ public class Guild public bool IsSystemBanned(PluralkitApiService.PkSystem system) => BannedSystems.Contains(system.Id) || BannedSystems.Contains(system.Uuid.ToString()); - public bool IsMessageIgnored(Snowflake channelId, Snowflake userId) + public bool IsMessageIgnored(Snowflake channelId, Snowflake? userId) { if ( Channels is { MessageDelete: 0, MessageUpdate: 0, MessageDeleteBulk: 0 } || Channels.IgnoredChannels.Contains(channelId.ToUlong()) - || Channels.IgnoredUsers.Contains(userId.ToUlong()) + || (userId != null && Channels.IgnoredUsers.Contains(userId.Value.ToUlong())) ) return true; + if (userId == null) + return false; + if ( Channels.IgnoredUsersPerChannel.TryGetValue( channelId.ToUlong(), out var thisChannelIgnoredUsers ) ) - return thisChannelIgnoredUsers.Contains(userId.ToUlong()); + return thisChannelIgnoredUsers.Contains(userId.Value.ToUlong()); return false; } diff --git a/Catalogger.Backend/Database/Queries/MessageRepository.cs b/Catalogger.Backend/Database/Queries/MessageRepository.cs index 9234660..6b1526e 100644 --- a/Catalogger.Backend/Database/Queries/MessageRepository.cs +++ b/Catalogger.Backend/Database/Queries/MessageRepository.cs @@ -1,7 +1,9 @@ using System.Text.Json; using Catalogger.Backend.Extensions; using Microsoft.EntityFrameworkCore; +using Remora.Discord.API; using Remora.Discord.API.Abstractions.Gateway.Events; +using Remora.Rest.Core; using DbMessage = Catalogger.Backend.Database.Models.Message; namespace Catalogger.Backend.Database.Queries; @@ -181,6 +183,21 @@ public class MessageRepository( return await db.IgnoredMessages.FirstOrDefaultAsync(m => m.Id == id, ct) != null; } + public const int MaxMessageAgeDays = 15; + + public async Task<(int Messages, int IgnoredMessages)> DeleteExpiredMessagesAsync() + { + var cutoff = DateTimeOffset.UtcNow - TimeSpan.FromDays(MaxMessageAgeDays); + var cutoffId = Snowflake.CreateTimestampSnowflake(cutoff, Constants.DiscordEpoch); + + var msgCount = await db.Messages.Where(m => m.Id < cutoffId.Value).ExecuteDeleteAsync(); + var ignoredMsgCount = await db + .IgnoredMessages.Where(m => m.Id < cutoffId.Value) + .ExecuteDeleteAsync(); + + return (msgCount, ignoredMsgCount); + } + public record Message( ulong Id, ulong? OriginalId, diff --git a/Catalogger.Backend/Extensions/StartupExtensions.cs b/Catalogger.Backend/Extensions/StartupExtensions.cs index 0f2a47f..1c44038 100644 --- a/Catalogger.Backend/Extensions/StartupExtensions.cs +++ b/Catalogger.Backend/Extensions/StartupExtensions.cs @@ -98,9 +98,12 @@ public static class StartupExtensions .AddSingleton() .AddSingleton(InMemoryDataService.Instance) .AddSingleton() + // GuildFetchService is added as a separate singleton as it's also injected into other services. .AddHostedService(serviceProvider => serviceProvider.GetRequiredService() - ); + ) + .AddHostedService() + .AddHostedService(); public static IHostBuilder AddShardedDiscordService( this IHostBuilder builder, diff --git a/Catalogger.Backend/Program.cs b/Catalogger.Backend/Program.cs index 014dd9e..89bf6fd 100644 --- a/Catalogger.Backend/Program.cs +++ b/Catalogger.Backend/Program.cs @@ -6,6 +6,9 @@ using Newtonsoft.Json.Serialization; using Prometheus; using Remora.Commands.Extensions; using Remora.Discord.API.Abstractions.Gateway.Commands; +using Remora.Discord.API.Abstractions.Objects; +using Remora.Discord.API.Gateway.Commands; +using Remora.Discord.API.Objects; using Remora.Discord.Caching.Extensions; using Remora.Discord.Commands.Extensions; using Remora.Discord.Extensions.Extensions; @@ -33,16 +36,34 @@ builder .ConfigureServices(s => s.AddRespondersFromAssembly(typeof(Program).Assembly) .Configure(g => + { g.Intents = GatewayIntents.Guilds - | GatewayIntents.GuildBans // Actually GUILD_MODERATION + // Actually GUILD_MODERATION + | GatewayIntents.GuildBans | GatewayIntents.GuildInvites | GatewayIntents.GuildMembers | GatewayIntents.GuildMessages | GatewayIntents.GuildWebhooks | GatewayIntents.MessageContents - | GatewayIntents.GuildEmojisAndStickers // Actually GUILD_EXPRESSIONS - ) + // Actually GUILD_EXPRESSIONS + | GatewayIntents.GuildEmojisAndStickers; + + // Set a default status for all shards. This is updated to a shard-specific one in StatusUpdateService. + g.Presence = new UpdatePresence( + Status: UserStatus.Online, + IsAFK: false, + Since: null, + Activities: + [ + new Activity( + Name: "Beep", + Type: ActivityType.Custom, + State: "/catalogger help" + ), + ] + ); + }) .AddDiscordCaching() .AddDiscordCommands(enableSlash: true, useDefaultCommandResponder: false) .AddCommandTree() diff --git a/Catalogger.Backend/Services/BackgroundTasksService.cs b/Catalogger.Backend/Services/BackgroundTasksService.cs new file mode 100644 index 0000000..9265b0b --- /dev/null +++ b/Catalogger.Backend/Services/BackgroundTasksService.cs @@ -0,0 +1,35 @@ +using Catalogger.Backend.Database; +using Catalogger.Backend.Database.Queries; + +namespace Catalogger.Backend.Services; + +public class BackgroundTasksService(ILogger logger, IServiceProvider services) : BackgroundService +{ + private readonly ILogger _logger = logger.ForContext(); + + protected override async Task ExecuteAsync(CancellationToken ct) + { + using var timer = new PeriodicTimer(TimeSpan.FromMinutes(1)); + while (await timer.WaitForNextTickAsync(ct)) + await HandlePeriodicTasksAsync(ct); + } + + private async Task HandlePeriodicTasksAsync(CancellationToken ct = default) + { + _logger.Information("Running once per minute periodic tasks"); + + await using var scope = services.CreateAsyncScope(); + var messageRepository = scope.ServiceProvider.GetRequiredService(); + + var (msgCount, ignoredCount) = await messageRepository.DeleteExpiredMessagesAsync(); + if (msgCount != 0 || ignoredCount != 0) + { + _logger.Information( + "Deleted {Count} messages and {IgnoredCount} ignored message IDs older than {MaxDays} days old", + msgCount, + ignoredCount, + MessageRepository.MaxMessageAgeDays + ); + } + } +} diff --git a/Catalogger.Backend/Services/StatusUpdateService.cs b/Catalogger.Backend/Services/StatusUpdateService.cs new file mode 100644 index 0000000..3cd2782 --- /dev/null +++ b/Catalogger.Backend/Services/StatusUpdateService.cs @@ -0,0 +1,57 @@ +using Catalogger.Backend.Bot; +using Remora.Discord.API.Abstractions.Objects; +using Remora.Discord.API.Gateway.Commands; +using Remora.Discord.API.Objects; + +namespace Catalogger.Backend.Services; + +public class StatusUpdateService(ILogger logger, ShardedGatewayClient shardedClient) + : BackgroundService +{ + private readonly ILogger _logger = logger.ForContext(); + + protected override async Task ExecuteAsync(CancellationToken ct) + { + using var timer = new PeriodicTimer(TimeSpan.FromMinutes(3)); + while (await timer.WaitForNextTickAsync(ct)) + UpdateShardStatuses(ct); + } + + private void UpdateShardStatuses(CancellationToken ct = default) + { + _logger.Information( + "Updating status for {TotalShards} shards. Guild count is {GuildCount}", + shardedClient.TotalShards, + CataloggerMetrics.GuildsCached.Value + ); + + foreach (var (shardId, client) in shardedClient.Shards) + { + if (!ShardedGatewayClient.IsConnected(client)) + { + _logger.Warning( + "Cannot update status for shard {ShardId} as it is not connected", + shardId + ); + continue; + } + + client.SubmitCommand(PresenceFor(shardId)); + } + } + + private UpdatePresence PresenceFor(int shardId) + { + var status = $"/catalogger help | in {CataloggerMetrics.GuildsCached.Value} servers"; + + if (shardedClient.TotalShards != 1) + status += $" | shard {shardId + 1}/{shardedClient.TotalShards}"; + + return new UpdatePresence( + Status: UserStatus.Online, + IsAFK: false, + Since: null, + Activities: [new Activity(Name: "Beep", Type: ActivityType.Custom, State: status)] + ); + } +}