sam/Catalogger.NET
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: shard status update, delete old messages when they expire

Browse source
This commit is contained in:
sam 2024-10-13 17:08:32 +02:00
parent 8e030acaf3
commit 32732d74d0
9 changed files with 302 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Catalogger.Backend/Bot/Responders/Guilds/GuildCreateResponder.cs
View file

@ -93,6 +93,8 @@ public class GuildCreateResponder(
return Task.FromResult(Result.Success);
}
guildCache.Remove(evt.ID, out _);
if (!guildCache.TryGet(evt.ID, out var guild))
{
_logger.Information("Left uncached guild {GuildId}", evt.ID);

142
Catalogger.Backend/Bot/Responders/Messages/MessageDeleteBulkResponder.cs Normal file
View file

@ -0,0 +1,142 @@
using System.Text;
using Catalogger.Backend.Cache.InMemoryCache;
using Catalogger.Backend.Database;
using Catalogger.Backend.Database.Queries;
using Catalogger.Backend.Extensions;
using Catalogger.Backend.Services;
using NodaTime.Extensions;
using Remora.Discord.API.Abstractions.Gateway.Events;
using Remora.Discord.API.Abstractions.Objects;
using Remora.Discord.API.Abstractions.Rest;
using Remora.Discord.Extensions.Embeds;
using Remora.Discord.Gateway.Responders;
using Remora.Rest.Core;
using Remora.Results;
namespace Catalogger.Backend.Bot.Responders.Messages;
public class MessageDeleteBulkResponder(
ILogger logger,
DatabaseContext db,
MessageRepository messageRepository,
WebhookExecutorService webhookExecutor,
ChannelCache channelCache
) : IResponder<IMessageDeleteBulk>
{
private readonly ILogger _logger = logger.ForContext<MessageDeleteBulkResponder>();
public async Task<Result> RespondAsync(IMessageDeleteBulk evt, CancellationToken ct = default)
{
var guild = await db.GetGuildAsync(evt.GuildID, ct);
if (guild.IsMessageIgnored(evt.ChannelID, null))
return Result.Success;
var logChannel = webhookExecutor.GetLogChannel(
guild,
LogChannelType.MessageDeleteBulk,
evt.ChannelID
);
if (logChannel == null)
{
return Result.Success;
}
IChannel? rootChannel = null;
channelCache.TryGet(evt.ChannelID, out var channel);
if (
channel is
{
Type: ChannelType.AnnouncementThread
or ChannelType.PrivateThread
or ChannelType.PublicThread
}
)
{
if (channel.ParentID.TryGet(out var parentId) && parentId != null)
channelCache.TryGet(parentId.Value, out rootChannel);
}
List<string> renderedMessages = [];
var notFoundMessages = 0;
var ignoredMessages = 0;
foreach (var msgId in evt.IDs.Order())
{
if (await messageRepository.IsMessageIgnoredAsync(msgId.Value, ct))
{
ignoredMessages++;
continue;
}
var msg = await messageRepository.GetMessageAsync(msgId.Value, ct);
renderedMessages.Add(RenderMessage(msgId, msg));
if (msg == null)
notFoundMessages++;
}
var output = "Bulk message delete in";
if (channel != null)
{
output += $" #{channel.Name} ({channel.ID})";
if (rootChannel != null)
output += $" (thread in #{rootChannel.Name} ({rootChannel.ID})";
}
else
{
output += $" unknown channel {evt.ChannelID}";
}
output += $"\nwith {renderedMessages.Count} messages\n\n";
output += string.Join("\n", renderedMessages);
var embed = new EmbedBuilder()
.WithTitle("Bulk message delete")
.WithDescription(
$"""
{evt.IDs.Count} messages were deleted in <#{evt.ChannelID}>
({notFoundMessages} messages not found, {ignoredMessages} messages ignored)
"""
)
.WithColour(DiscordUtils.Red)
.WithCurrentTimestamp();
await webhookExecutor.SendLogAsync(
logChannel.Value,
[embed.Build().GetOrThrow()],
[
new FileData(
$"bulk-delete-{evt.ChannelID}.txt",
new MemoryStream(Encoding.UTF8.GetBytes(output))
),
]
);
return Result.Success;
}
private string RenderMessage(Snowflake messageId, MessageRepository.Message? message)
{
var timestamp = messageId.Timestamp.ToOffsetDateTime().ToString();
if (message == null)
{
return $"""
[{timestamp}] Unknown message {messageId}
--------------------------------------------
""";
}
var builder = new StringBuilder();
builder.Append($"[{timestamp}] {message.Username} ({message.UserId})\n");
if (message is { System: not null, Member: not null })
{
builder.Append($"PK system: {message.System} | PK member: {message.Member}\n");
}
builder.Append("--------------------------------------------\n");
builder.Append(message.Content);
builder.Append("\n--------------------------------------------\n");
return builder.ToString();
}
}

25
Catalogger.Backend/Bot/ShardedGatewayClient.cs
View file

@ -19,7 +19,8 @@ public class ShardedGatewayClient(
Config config
) : IDisposable
{
private int _shardCount = config.Discord.ShardCount ?? 0;
public int TotalShards { get; private set; } = config.Discord.ShardCount ?? 0;
private readonly ILogger _logger = logger.ForContext<ShardedGatewayClient>();
private readonly ConcurrentDictionary<int, DiscordGatewayClient> _gatewayClients = new();
@ -33,6 +34,9 @@ public class ShardedGatewayClient(
GatewayConnectionStatus
> GetConnectionStatus = client => (GatewayConnectionStatus)Field.GetValue(client)!;
public static bool IsConnected(DiscordGatewayClient client) =>
GetConnectionStatus(client) == GatewayConnectionStatus.Connected;
public IReadOnlyDictionary<int, DiscordGatewayClient> Shards => _gatewayClients;
public async Task<Result> RunAsync(CancellationToken ct = default)
@ -46,19 +50,19 @@ public class ShardedGatewayClient(
if (gatewayResult.Entity.Shards.IsDefined(out var discordShardCount))
{
if (_shardCount < discordShardCount && _shardCount != 0)
if (TotalShards < discordShardCount && TotalShards != 0)
_logger.Warning(
"Discord recommends {DiscordShardCount} for this bot, but only {ConfigShardCount} shards are requested. This may cause issues later",
discordShardCount,
_shardCount
TotalShards
);
if (_shardCount == 0)
_shardCount = discordShardCount;
if (TotalShards == 0)
TotalShards = discordShardCount;
}
var clients = Enumerable
.Range(0, _shardCount)
.Range(0, TotalShards)
.Select(s =>
{
var client = ActivatorUtilities.CreateInstance<DiscordGatewayClient>(
@ -74,7 +78,7 @@ public class ShardedGatewayClient(
for (var shardIndex = 0; shardIndex < clients.Length; shardIndex++)
{
_logger.Debug("Starting shard {ShardId}/{ShardCount}", shardIndex, _shardCount);
_logger.Debug("Starting shard {ShardId}/{ShardCount}", shardIndex, TotalShards);
var client = clients[shardIndex];
var res = client.RunAsync(ct);
@ -93,13 +97,13 @@ public class ShardedGatewayClient(
return res.Result;
}
_logger.Information("Started shard {ShardId}/{ShardCount}", shardIndex, _shardCount);
_logger.Information("Started shard {ShardId}/{ShardCount}", shardIndex, TotalShards);
}
return await await Task.WhenAny(tasks);
}
public int ShardIdFor(ulong guildId) => (int)((guildId >> 22) % (ulong)_shardCount);
public int ShardIdFor(ulong guildId) => (int)((guildId >> 22) % (ulong)TotalShards);
public DiscordGatewayClient ClientFor(Snowflake guildId) => ClientFor(guildId.Value);
@ -112,6 +116,7 @@ public class ShardedGatewayClient(
public void Dispose()
{
GC.SuppressFinalize(this);
foreach (var client in _gatewayClients.Values)
client.Dispose();
}
@ -123,7 +128,7 @@ public class ShardedGatewayClient(
{
var ret = new DiscordGatewayClientOptions
{
ShardIdentification = new ShardIdentification(shardId, _shardCount),
ShardIdentification = new ShardIdentification(shardId, TotalShards),
Intents = options.Intents,
Presence = options.Presence,
ConnectionProperties = options.ConnectionProperties,

9
Catalogger.Backend/Database/Models/Guild.cs
View file

@ -18,22 +18,25 @@ public class Guild
public bool IsSystemBanned(PluralkitApiService.PkSystem system) =>
BannedSystems.Contains(system.Id) || BannedSystems.Contains(system.Uuid.ToString());
public bool IsMessageIgnored(Snowflake channelId, Snowflake userId)
public bool IsMessageIgnored(Snowflake channelId, Snowflake? userId)
{
if (
Channels is { MessageDelete: 0, MessageUpdate: 0, MessageDeleteBulk: 0 }
|| Channels.IgnoredChannels.Contains(channelId.ToUlong())
|| Channels.IgnoredUsers.Contains(userId.ToUlong())
|| (userId != null && Channels.IgnoredUsers.Contains(userId.Value.ToUlong()))
)
return true;
if (userId == null)
return false;
if (
Channels.IgnoredUsersPerChannel.TryGetValue(
channelId.ToUlong(),
out var thisChannelIgnoredUsers
)
)
return thisChannelIgnoredUsers.Contains(userId.ToUlong());
return thisChannelIgnoredUsers.Contains(userId.Value.ToUlong());
return false;
}

17
Catalogger.Backend/Database/Queries/MessageRepository.cs
View file

@ -1,7 +1,9 @@
using System.Text.Json;
using Catalogger.Backend.Extensions;
using Microsoft.EntityFrameworkCore;
using Remora.Discord.API;
using Remora.Discord.API.Abstractions.Gateway.Events;
using Remora.Rest.Core;
using DbMessage = Catalogger.Backend.Database.Models.Message;
namespace Catalogger.Backend.Database.Queries;
@ -181,6 +183,21 @@ public class MessageRepository(
return await db.IgnoredMessages.FirstOrDefaultAsync(m => m.Id == id, ct) != null;
}
public const int MaxMessageAgeDays = 15;
public async Task<(int Messages, int IgnoredMessages)> DeleteExpiredMessagesAsync()
{
var cutoff = DateTimeOffset.UtcNow - TimeSpan.FromDays(MaxMessageAgeDays);
var cutoffId = Snowflake.CreateTimestampSnowflake(cutoff, Constants.DiscordEpoch);
var msgCount = await db.Messages.Where(m => m.Id < cutoffId.Value).ExecuteDeleteAsync();
var ignoredMsgCount = await db
.IgnoredMessages.Where(m => m.Id < cutoffId.Value)
.ExecuteDeleteAsync();
return (msgCount, ignoredMsgCount);
}
public record Message(
ulong Id,
ulong? OriginalId,

5
Catalogger.Backend/Extensions/StartupExtensions.cs
View file

@ -98,9 +98,12 @@ public static class StartupExtensions
.AddSingleton<PkMessageHandler>()
.AddSingleton(InMemoryDataService<Snowflake, ChannelCommandData>.Instance)
.AddSingleton<GuildFetchService>()
// GuildFetchService is added as a separate singleton as it's also injected into other services.
.AddHostedService(serviceProvider =>
serviceProvider.GetRequiredService<GuildFetchService>()
);
)
.AddHostedService<StatusUpdateService>()
.AddHostedService<BackgroundTasksService>();
public static IHostBuilder AddShardedDiscordService(
this IHostBuilder builder,

27
Catalogger.Backend/Program.cs
View file

@ -6,6 +6,9 @@ using Newtonsoft.Json.Serialization;
using Prometheus;
using Remora.Commands.Extensions;
using Remora.Discord.API.Abstractions.Gateway.Commands;
using Remora.Discord.API.Abstractions.Objects;
using Remora.Discord.API.Gateway.Commands;
using Remora.Discord.API.Objects;
using Remora.Discord.Caching.Extensions;
using Remora.Discord.Commands.Extensions;
using Remora.Discord.Extensions.Extensions;
@ -33,16 +36,34 @@ builder
.ConfigureServices(s =>
s.AddRespondersFromAssembly(typeof(Program).Assembly)
.Configure<DiscordGatewayClientOptions>(g =>
{
g.Intents =
GatewayIntents.Guilds
| GatewayIntents.GuildBans // Actually GUILD_MODERATION
// Actually GUILD_MODERATION
| GatewayIntents.GuildBans
| GatewayIntents.GuildInvites
| GatewayIntents.GuildMembers
| GatewayIntents.GuildMessages
| GatewayIntents.GuildWebhooks
| GatewayIntents.MessageContents
| GatewayIntents.GuildEmojisAndStickers // Actually GUILD_EXPRESSIONS
)
// Actually GUILD_EXPRESSIONS
| GatewayIntents.GuildEmojisAndStickers;
// Set a default status for all shards. This is updated to a shard-specific one in StatusUpdateService.
g.Presence = new UpdatePresence(
Status: UserStatus.Online,
IsAFK: false,
Since: null,
Activities:
[
new Activity(
Name: "Beep",
Type: ActivityType.Custom,
State: "/catalogger help"
),
]
);
})
.AddDiscordCaching()
.AddDiscordCommands(enableSlash: true, useDefaultCommandResponder: false)
.AddCommandTree()

35
Catalogger.Backend/Services/BackgroundTasksService.cs Normal file
View file

@ -0,0 +1,35 @@
using Catalogger.Backend.Database;
using Catalogger.Backend.Database.Queries;
namespace Catalogger.Backend.Services;
public class BackgroundTasksService(ILogger logger, IServiceProvider services) : BackgroundService
{
private readonly ILogger _logger = logger.ForContext<BackgroundTasksService>();
protected override async Task ExecuteAsync(CancellationToken ct)
{
using var timer = new PeriodicTimer(TimeSpan.FromMinutes(1));
while (await timer.WaitForNextTickAsync(ct))
await HandlePeriodicTasksAsync(ct);
}
private async Task HandlePeriodicTasksAsync(CancellationToken ct = default)
{
_logger.Information("Running once per minute periodic tasks");
await using var scope = services.CreateAsyncScope();
var messageRepository = scope.ServiceProvider.GetRequiredService<MessageRepository>();
var (msgCount, ignoredCount) = await messageRepository.DeleteExpiredMessagesAsync();
if (msgCount != 0 || ignoredCount != 0)
{
_logger.Information(
"Deleted {Count} messages and {IgnoredCount} ignored message IDs older than {MaxDays} days old",
msgCount,
ignoredCount,
MessageRepository.MaxMessageAgeDays
);
}
}
}

57
Catalogger.Backend/Services/StatusUpdateService.cs Normal file
View file

@ -0,0 +1,57 @@
using Catalogger.Backend.Bot;
using Remora.Discord.API.Abstractions.Objects;
using Remora.Discord.API.Gateway.Commands;
using Remora.Discord.API.Objects;
namespace Catalogger.Backend.Services;
public class StatusUpdateService(ILogger logger, ShardedGatewayClient shardedClient)
: BackgroundService
{
private readonly ILogger _logger = logger.ForContext<StatusUpdateService>();
protected override async Task ExecuteAsync(CancellationToken ct)
{
using var timer = new PeriodicTimer(TimeSpan.FromMinutes(3));
while (await timer.WaitForNextTickAsync(ct))
UpdateShardStatuses(ct);
}
private void UpdateShardStatuses(CancellationToken ct = default)
{
_logger.Information(
"Updating status for {TotalShards} shards. Guild count is {GuildCount}",
shardedClient.TotalShards,
CataloggerMetrics.GuildsCached.Value
);
foreach (var (shardId, client) in shardedClient.Shards)
{
if (!ShardedGatewayClient.IsConnected(client))
{
_logger.Warning(
"Cannot update status for shard {ShardId} as it is not connected",
shardId
);
continue;
}
client.SubmitCommand(PresenceFor(shardId));
}
}
private UpdatePresence PresenceFor(int shardId)
{
var status = $"/catalogger help | in {CataloggerMetrics.GuildsCached.Value} servers";
if (shardedClient.TotalShards != 1)
status += $" | shard {shardId + 1}/{shardedClient.TotalShards}";
return new UpdatePresence(
Status: UserStatus.Online,
IsAFK: false,
Since: null,
Activities: [new Activity(Name: "Beep", Type: ActivityType.Custom, State: status)]
);
}
}