feat: shard status update, delete old messages when they expire

Catalogger.Backend/Bot/Responders/Guilds/GuildCreateResponder.cs

@@ -93,6 +93,8 @@ public class GuildCreateResponder(
             return Task.FromResult(Result.Success);
         }
 
+        guildCache.Remove(evt.ID, out _);
+
         if (!guildCache.TryGet(evt.ID, out var guild))
         {
             _logger.Information("Left uncached guild {GuildId}", evt.ID);

Catalogger.Backend/Bot/Responders/Messages/MessageDeleteBulkResponder.cs

@@ -0,0 +1,142 @@
+using System.Text;
+using Catalogger.Backend.Cache.InMemoryCache;
+using Catalogger.Backend.Database;
+using Catalogger.Backend.Database.Queries;
+using Catalogger.Backend.Extensions;
+using Catalogger.Backend.Services;
+using NodaTime.Extensions;
+using Remora.Discord.API.Abstractions.Gateway.Events;
+using Remora.Discord.API.Abstractions.Objects;
+using Remora.Discord.API.Abstractions.Rest;
+using Remora.Discord.Extensions.Embeds;
+using Remora.Discord.Gateway.Responders;
+using Remora.Rest.Core;
+using Remora.Results;
+
+namespace Catalogger.Backend.Bot.Responders.Messages;
+
+public class MessageDeleteBulkResponder(
+    ILogger logger,
+    DatabaseContext db,
+    MessageRepository messageRepository,
+    WebhookExecutorService webhookExecutor,
+    ChannelCache channelCache
+) : IResponder<IMessageDeleteBulk>
+{
+    private readonly ILogger _logger = logger.ForContext<MessageDeleteBulkResponder>();
+
+    public async Task<Result> RespondAsync(IMessageDeleteBulk evt, CancellationToken ct = default)
+    {
+        var guild = await db.GetGuildAsync(evt.GuildID, ct);
+        if (guild.IsMessageIgnored(evt.ChannelID, null))
+            return Result.Success;
+
+        var logChannel = webhookExecutor.GetLogChannel(
+            guild,
+            LogChannelType.MessageDeleteBulk,
+            evt.ChannelID
+        );
+        if (logChannel == null)
+        {
+            return Result.Success;
+        }
+
+        IChannel? rootChannel = null;
+        channelCache.TryGet(evt.ChannelID, out var channel);
+        if (
+            channel is
+            {
+                Type: ChannelType.AnnouncementThread
+                    or ChannelType.PrivateThread
+                    or ChannelType.PublicThread
+            }
+        )
+        {
+            if (channel.ParentID.TryGet(out var parentId) && parentId != null)
+                channelCache.TryGet(parentId.Value, out rootChannel);
+        }
+
+        List<string> renderedMessages = [];
+        var notFoundMessages = 0;
+        var ignoredMessages = 0;
+
+        foreach (var msgId in evt.IDs.Order())
+        {
+            if (await messageRepository.IsMessageIgnoredAsync(msgId.Value, ct))
+            {
+                ignoredMessages++;
+                continue;
+            }
+
+            var msg = await messageRepository.GetMessageAsync(msgId.Value, ct);
+            renderedMessages.Add(RenderMessage(msgId, msg));
+
+            if (msg == null)
+                notFoundMessages++;
+        }
+
+        var output = "Bulk message delete in";
+        if (channel != null)
+        {
+            output += $" #{channel.Name} ({channel.ID})";
+            if (rootChannel != null)
+                output += $" (thread in #{rootChannel.Name} ({rootChannel.ID})";
+        }
+        else
+        {
+            output += $" unknown channel {evt.ChannelID}";
+        }
+
+        output += $"\nwith {renderedMessages.Count} messages\n\n";
+        output += string.Join("\n", renderedMessages);
+
+        var embed = new EmbedBuilder()
+            .WithTitle("Bulk message delete")
+            .WithDescription(
+                $"""
+                {evt.IDs.Count} messages were deleted in <#{evt.ChannelID}>
+                ({notFoundMessages} messages not found, {ignoredMessages} messages ignored)
+                """
+            )
+            .WithColour(DiscordUtils.Red)
+            .WithCurrentTimestamp();
+
+        await webhookExecutor.SendLogAsync(
+            logChannel.Value,
+            [embed.Build().GetOrThrow()],
+            [
+                new FileData(
+                    $"bulk-delete-{evt.ChannelID}.txt",
+                    new MemoryStream(Encoding.UTF8.GetBytes(output))
+                ),
+            ]
+        );
+        return Result.Success;
+    }
+
+    private string RenderMessage(Snowflake messageId, MessageRepository.Message? message)
+    {
+        var timestamp = messageId.Timestamp.ToOffsetDateTime().ToString();
+
+        if (message == null)
+        {
+            return $"""
+                [{timestamp}] Unknown message {messageId}
+                --------------------------------------------
+                """;
+        }
+
+        var builder = new StringBuilder();
+        builder.Append($"[{timestamp}] {message.Username} ({message.UserId})\n");
+        if (message is { System: not null, Member: not null })
+        {
+            builder.Append($"PK system: {message.System} | PK member: {message.Member}\n");
+        }
+
+        builder.Append("--------------------------------------------\n");
+        builder.Append(message.Content);
+        builder.Append("\n--------------------------------------------\n");
+
+        return builder.ToString();
+    }
+}

Catalogger.Backend/Bot/ShardedGatewayClient.cs

@@ -19,7 +19,8 @@ public class ShardedGatewayClient(
     Config config
 ) : IDisposable
 {
-    private int _shardCount = config.Discord.ShardCount ?? 0;
+    public int TotalShards { get; private set; } = config.Discord.ShardCount ?? 0;
+
     private readonly ILogger _logger = logger.ForContext<ShardedGatewayClient>();
     private readonly ConcurrentDictionary<int, DiscordGatewayClient> _gatewayClients = new();
 
@@ -33,6 +34,9 @@ public class ShardedGatewayClient(
         GatewayConnectionStatus
     > GetConnectionStatus = client => (GatewayConnectionStatus)Field.GetValue(client)!;
 
+    public static bool IsConnected(DiscordGatewayClient client) =>
+        GetConnectionStatus(client) == GatewayConnectionStatus.Connected;
+
     public IReadOnlyDictionary<int, DiscordGatewayClient> Shards => _gatewayClients;
 
     public async Task<Result> RunAsync(CancellationToken ct = default)
@@ -46,19 +50,19 @@ public class ShardedGatewayClient(
 
         if (gatewayResult.Entity.Shards.IsDefined(out var discordShardCount))
         {
-            if (_shardCount < discordShardCount && _shardCount != 0)
+            if (TotalShards < discordShardCount && TotalShards != 0)
                 _logger.Warning(
                     "Discord recommends {DiscordShardCount} for this bot, but only {ConfigShardCount} shards are requested. This may cause issues later",
                     discordShardCount,
-                    _shardCount
+                    TotalShards
                 );
 
-            if (_shardCount == 0)
+            if (TotalShards == 0)
-                _shardCount = discordShardCount;
+                TotalShards = discordShardCount;
         }
 
         var clients = Enumerable
-            .Range(0, _shardCount)
+            .Range(0, TotalShards)
             .Select(s =>
             {
                 var client = ActivatorUtilities.CreateInstance<DiscordGatewayClient>(
@@ -74,7 +78,7 @@ public class ShardedGatewayClient(
 
         for (var shardIndex = 0; shardIndex < clients.Length; shardIndex++)
         {
-            _logger.Debug("Starting shard {ShardId}/{ShardCount}", shardIndex, _shardCount);
+            _logger.Debug("Starting shard {ShardId}/{ShardCount}", shardIndex, TotalShards);
 
             var client = clients[shardIndex];
             var res = client.RunAsync(ct);
@@ -93,13 +97,13 @@ public class ShardedGatewayClient(
                 return res.Result;
             }
 
-            _logger.Information("Started shard {ShardId}/{ShardCount}", shardIndex, _shardCount);
+            _logger.Information("Started shard {ShardId}/{ShardCount}", shardIndex, TotalShards);
         }
 
         return await await Task.WhenAny(tasks);
     }
 
-    public int ShardIdFor(ulong guildId) => (int)((guildId >> 22) % (ulong)_shardCount);
+    public int ShardIdFor(ulong guildId) => (int)((guildId >> 22) % (ulong)TotalShards);
 
     public DiscordGatewayClient ClientFor(Snowflake guildId) => ClientFor(guildId.Value);
 
@@ -112,6 +116,7 @@ public class ShardedGatewayClient(
 
     public void Dispose()
     {
+        GC.SuppressFinalize(this);
         foreach (var client in _gatewayClients.Values)
             client.Dispose();
     }
@@ -123,7 +128,7 @@ public class ShardedGatewayClient(
     {
         var ret = new DiscordGatewayClientOptions
         {
-            ShardIdentification = new ShardIdentification(shardId, _shardCount),
+            ShardIdentification = new ShardIdentification(shardId, TotalShards),
             Intents = options.Intents,
             Presence = options.Presence,
             ConnectionProperties = options.ConnectionProperties,

Catalogger.Backend/Database/Models/Guild.cs

@@ -18,22 +18,25 @@ public class Guild
     public bool IsSystemBanned(PluralkitApiService.PkSystem system) =>
         BannedSystems.Contains(system.Id) || BannedSystems.Contains(system.Uuid.ToString());
 
-    public bool IsMessageIgnored(Snowflake channelId, Snowflake userId)
+    public bool IsMessageIgnored(Snowflake channelId, Snowflake? userId)
     {
         if (
             Channels is { MessageDelete: 0, MessageUpdate: 0, MessageDeleteBulk: 0 }
             || Channels.IgnoredChannels.Contains(channelId.ToUlong())
-            || Channels.IgnoredUsers.Contains(userId.ToUlong())
+            || (userId != null && Channels.IgnoredUsers.Contains(userId.Value.ToUlong()))
         )
             return true;
 
+        if (userId == null)
+            return false;
+
         if (
             Channels.IgnoredUsersPerChannel.TryGetValue(
                 channelId.ToUlong(),
                 out var thisChannelIgnoredUsers
             )
         )
-            return thisChannelIgnoredUsers.Contains(userId.ToUlong());
+            return thisChannelIgnoredUsers.Contains(userId.Value.ToUlong());
 
         return false;
     }

Catalogger.Backend/Database/Queries/MessageRepository.cs

@@ -1,7 +1,9 @@
 using System.Text.Json;
 using Catalogger.Backend.Extensions;
 using Microsoft.EntityFrameworkCore;
+using Remora.Discord.API;
 using Remora.Discord.API.Abstractions.Gateway.Events;
+using Remora.Rest.Core;
 using DbMessage = Catalogger.Backend.Database.Models.Message;
 
 namespace Catalogger.Backend.Database.Queries;
@@ -181,6 +183,21 @@ public class MessageRepository(
         return await db.IgnoredMessages.FirstOrDefaultAsync(m => m.Id == id, ct) != null;
     }
 
+    public const int MaxMessageAgeDays = 15;
+
+    public async Task<(int Messages, int IgnoredMessages)> DeleteExpiredMessagesAsync()
+    {
+        var cutoff = DateTimeOffset.UtcNow - TimeSpan.FromDays(MaxMessageAgeDays);
+        var cutoffId = Snowflake.CreateTimestampSnowflake(cutoff, Constants.DiscordEpoch);
+
+        var msgCount = await db.Messages.Where(m => m.Id < cutoffId.Value).ExecuteDeleteAsync();
+        var ignoredMsgCount = await db
+            .IgnoredMessages.Where(m => m.Id < cutoffId.Value)
+            .ExecuteDeleteAsync();
+
+        return (msgCount, ignoredMsgCount);
+    }
+
     public record Message(
         ulong Id,
         ulong? OriginalId,

Catalogger.Backend/Extensions/StartupExtensions.cs

@@ -98,9 +98,12 @@ public static class StartupExtensions
             .AddSingleton<PkMessageHandler>()
             .AddSingleton(InMemoryDataService<Snowflake, ChannelCommandData>.Instance)
             .AddSingleton<GuildFetchService>()
+            // GuildFetchService is added as a separate singleton as it's also injected into other services.
             .AddHostedService(serviceProvider =>
                 serviceProvider.GetRequiredService<GuildFetchService>()
-            );
+            )
+            .AddHostedService<StatusUpdateService>()
+            .AddHostedService<BackgroundTasksService>();
 
     public static IHostBuilder AddShardedDiscordService(
         this IHostBuilder builder,

Catalogger.Backend/Program.cs

@@ -6,6 +6,9 @@ using Newtonsoft.Json.Serialization;
 using Prometheus;
 using Remora.Commands.Extensions;
 using Remora.Discord.API.Abstractions.Gateway.Commands;
+using Remora.Discord.API.Abstractions.Objects;
+using Remora.Discord.API.Gateway.Commands;
+using Remora.Discord.API.Objects;
 using Remora.Discord.Caching.Extensions;
 using Remora.Discord.Commands.Extensions;
 using Remora.Discord.Extensions.Extensions;
@@ -33,16 +36,34 @@ builder
     .ConfigureServices(s =>
         s.AddRespondersFromAssembly(typeof(Program).Assembly)
             .Configure<DiscordGatewayClientOptions>(g =>
+            {
                 g.Intents =
                     GatewayIntents.Guilds
-                    | GatewayIntents.GuildBans // Actually GUILD_MODERATION
+                    // Actually GUILD_MODERATION
+                    | GatewayIntents.GuildBans
                     | GatewayIntents.GuildInvites
                     | GatewayIntents.GuildMembers
                     | GatewayIntents.GuildMessages
                     | GatewayIntents.GuildWebhooks
                     | GatewayIntents.MessageContents
-                    | GatewayIntents.GuildEmojisAndStickers // Actually GUILD_EXPRESSIONS
+                    // Actually GUILD_EXPRESSIONS
-            )
+                    | GatewayIntents.GuildEmojisAndStickers;
+
+                // Set a default status for all shards. This is updated to a shard-specific one in StatusUpdateService.
+                g.Presence = new UpdatePresence(
+                    Status: UserStatus.Online,
+                    IsAFK: false,
+                    Since: null,
+                    Activities:
+                    [
+                        new Activity(
+                            Name: "Beep",
+                            Type: ActivityType.Custom,
+                            State: "/catalogger help"
+                        ),
+                    ]
+                );
+            })
             .AddDiscordCaching()
             .AddDiscordCommands(enableSlash: true, useDefaultCommandResponder: false)
             .AddCommandTree()

Catalogger.Backend/Services/BackgroundTasksService.cs

@@ -0,0 +1,35 @@
+using Catalogger.Backend.Database;
+using Catalogger.Backend.Database.Queries;
+
+namespace Catalogger.Backend.Services;
+
+public class BackgroundTasksService(ILogger logger, IServiceProvider services) : BackgroundService
+{
+    private readonly ILogger _logger = logger.ForContext<BackgroundTasksService>();
+
+    protected override async Task ExecuteAsync(CancellationToken ct)
+    {
+        using var timer = new PeriodicTimer(TimeSpan.FromMinutes(1));
+        while (await timer.WaitForNextTickAsync(ct))
+            await HandlePeriodicTasksAsync(ct);
+    }
+
+    private async Task HandlePeriodicTasksAsync(CancellationToken ct = default)
+    {
+        _logger.Information("Running once per minute periodic tasks");
+
+        await using var scope = services.CreateAsyncScope();
+        var messageRepository = scope.ServiceProvider.GetRequiredService<MessageRepository>();
+
+        var (msgCount, ignoredCount) = await messageRepository.DeleteExpiredMessagesAsync();
+        if (msgCount != 0 || ignoredCount != 0)
+        {
+            _logger.Information(
+                "Deleted {Count} messages and {IgnoredCount} ignored message IDs older than {MaxDays} days old",
+                msgCount,
+                ignoredCount,
+                MessageRepository.MaxMessageAgeDays
+            );
+        }
+    }
+}

Catalogger.Backend/Services/StatusUpdateService.cs

@@ -0,0 +1,57 @@
+using Catalogger.Backend.Bot;
+using Remora.Discord.API.Abstractions.Objects;
+using Remora.Discord.API.Gateway.Commands;
+using Remora.Discord.API.Objects;
+
+namespace Catalogger.Backend.Services;
+
+public class StatusUpdateService(ILogger logger, ShardedGatewayClient shardedClient)
+    : BackgroundService
+{
+    private readonly ILogger _logger = logger.ForContext<StatusUpdateService>();
+
+    protected override async Task ExecuteAsync(CancellationToken ct)
+    {
+        using var timer = new PeriodicTimer(TimeSpan.FromMinutes(3));
+        while (await timer.WaitForNextTickAsync(ct))
+            UpdateShardStatuses(ct);
+    }
+
+    private void UpdateShardStatuses(CancellationToken ct = default)
+    {
+        _logger.Information(
+            "Updating status for {TotalShards} shards. Guild count is {GuildCount}",
+            shardedClient.TotalShards,
+            CataloggerMetrics.GuildsCached.Value
+        );
+
+        foreach (var (shardId, client) in shardedClient.Shards)
+        {
+            if (!ShardedGatewayClient.IsConnected(client))
+            {
+                _logger.Warning(
+                    "Cannot update status for shard {ShardId} as it is not connected",
+                    shardId
+                );
+                continue;
+            }
+
+            client.SubmitCommand(PresenceFor(shardId));
+        }
+    }
+
+    private UpdatePresence PresenceFor(int shardId)
+    {
+        var status = $"/catalogger help | in {CataloggerMetrics.GuildsCached.Value} servers";
+
+        if (shardedClient.TotalShards != 1)
+            status += $" | shard {shardId + 1}/{shardedClient.TotalShards}";
+
+        return new UpdatePresence(
+            Status: UserStatus.Online,
+            IsAFK: false,
+            Since: null,
+            Activities: [new Activity(Name: "Beep", Type: ActivityType.Custom, State: status)]
+        );
+    }
+}