diff --git a/.idea/.idea.catalogger/.idea/inspectionProfiles/Project_Default.xml b/.idea/.idea.catalogger/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..67139c5 --- /dev/null +++ b/.idea/.idea.catalogger/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/Catalogger.Backend/Bot/Commands/ChannelCommands.cs b/Catalogger.Backend/Bot/Commands/ChannelCommands.cs index fab35c1..9875bca 100644 --- a/Catalogger.Backend/Bot/Commands/ChannelCommands.cs +++ b/Catalogger.Backend/Bot/Commands/ChannelCommands.cs @@ -1,5 +1,6 @@ using System.ComponentModel; using Catalogger.Backend.Cache; +using Catalogger.Backend.Cache.InMemoryCache; using Catalogger.Backend.Database; using Catalogger.Backend.Database.Queries; using Catalogger.Backend.Extensions; @@ -26,8 +27,8 @@ namespace Catalogger.Backend.Bot.Commands; public class ChannelCommands( ILogger logger, DatabaseContext db, - GuildCacheService guildCache, - ChannelCacheService channelCache, + GuildCache guildCache, + ChannelCache channelCache, IFeedbackService feedbackService, ContextInjectionService contextInjection, InMemoryDataService dataService) : CommandGroup diff --git a/Catalogger.Backend/Bot/Commands/ChannelCommandsComponents.cs b/Catalogger.Backend/Bot/Commands/ChannelCommandsComponents.cs index 18f6c99..c9d1c3d 100644 --- a/Catalogger.Backend/Bot/Commands/ChannelCommandsComponents.cs +++ b/Catalogger.Backend/Bot/Commands/ChannelCommandsComponents.cs @@ -1,4 +1,5 @@ using Catalogger.Backend.Cache; +using Catalogger.Backend.Cache.InMemoryCache; using Catalogger.Backend.Database; using Catalogger.Backend.Database.Queries; using Catalogger.Backend.Extensions; @@ -22,8 +23,8 @@ namespace Catalogger.Backend.Bot.Commands; public class ChannelCommandsComponents( ILogger logger, DatabaseContext db, - GuildCacheService guildCache, - ChannelCacheService channelCache, + GuildCache guildCache, + ChannelCache channelCache, ContextInjectionService contextInjection, IFeedbackService feedbackService, IDiscordRestInteractionAPI interactionApi, diff --git a/Catalogger.Backend/Bot/Commands/MetaCommands.cs b/Catalogger.Backend/Bot/Commands/MetaCommands.cs index 11ec957..905dd52 100644 --- a/Catalogger.Backend/Bot/Commands/MetaCommands.cs +++ b/Catalogger.Backend/Bot/Commands/MetaCommands.cs @@ -1,8 +1,10 @@ using System.ComponentModel; using System.Diagnostics; using System.Runtime.InteropServices; +using System.Text.Json; using App.Metrics; using Catalogger.Backend.Cache; +using Catalogger.Backend.Cache.InMemoryCache; using Catalogger.Backend.Database; using Catalogger.Backend.Extensions; using Humanizer; @@ -11,7 +13,11 @@ using Remora.Commands.Attributes; using Remora.Commands.Groups; using Remora.Discord.API.Abstractions.Objects; using Remora.Discord.API.Abstractions.Rest; +using Remora.Discord.API.Objects; +using Remora.Discord.Commands.Contexts; +using Remora.Discord.Commands.Extensions; using Remora.Discord.Commands.Feedback.Services; +using Remora.Discord.Commands.Services; using Remora.Discord.Extensions.Embeds; using Remora.Discord.Gateway; using Remora.Results; @@ -26,8 +32,10 @@ public class MetaCommands( IMetrics metrics, DiscordGatewayClient client, IFeedbackService feedbackService, - GuildCacheService guildCache, - ChannelCacheService channelCache, + ContextInjectionService contextInjection, + IInviteCache inviteCache, + GuildCache guildCache, + ChannelCache channelCache, IDiscordRestChannelAPI channelApi) : CommandGroup { [Command("ping")] @@ -65,4 +73,17 @@ public class MetaCommands( return (Result)await channelApi.EditMessageAsync(msg.ChannelID, msg.ID, content: "", embeds: embeds); } + + [Command("debug-invites")] + [Description("Show a representation of this server's invites")] + public async Task DebugInvitesAsync() + { + if (contextInjection.Context is not IInteractionCommandContext ctx) throw new CataloggerError("No context"); + if (!ctx.TryGetGuildID(out var guildId)) throw new CataloggerError("No guild ID in context"); + + var invites = await inviteCache.TryGetAsync(guildId); + var text = invites.Select(i => $"{i.Code} in {i.Channel?.ID.Value}"); + + return await feedbackService.SendContextualAsync(string.Join("\n", text)); + } } \ No newline at end of file diff --git a/Catalogger.Backend/Bot/Responders/Channels/ChannelCreateResponder.cs b/Catalogger.Backend/Bot/Responders/Channels/ChannelCreateResponder.cs index b1ba94a..d8a746e 100644 --- a/Catalogger.Backend/Bot/Responders/Channels/ChannelCreateResponder.cs +++ b/Catalogger.Backend/Bot/Responders/Channels/ChannelCreateResponder.cs @@ -1,4 +1,5 @@ using Catalogger.Backend.Cache; +using Catalogger.Backend.Cache.InMemoryCache; using Catalogger.Backend.Database; using Catalogger.Backend.Database.Queries; using Catalogger.Backend.Extensions; @@ -13,8 +14,8 @@ namespace Catalogger.Backend.Bot.Responders.Channels; public class ChannelCreateResponder( DatabaseContext db, - RoleCacheService roleCache, - ChannelCacheService channelCache, + RoleCache roleCache, + ChannelCache channelCache, WebhookExecutorService webhookExecutor) : IResponder { public async Task RespondAsync(IChannelCreate ch, CancellationToken ct = default) diff --git a/Catalogger.Backend/Bot/Responders/GuildCreateResponder.cs b/Catalogger.Backend/Bot/Responders/Guilds/GuildCreateResponder.cs similarity index 85% rename from Catalogger.Backend/Bot/Responders/GuildCreateResponder.cs rename to Catalogger.Backend/Bot/Responders/Guilds/GuildCreateResponder.cs index e84f2ae..a0dd422 100644 --- a/Catalogger.Backend/Bot/Responders/GuildCreateResponder.cs +++ b/Catalogger.Backend/Bot/Responders/Guilds/GuildCreateResponder.cs @@ -1,26 +1,27 @@ -using System.Diagnostics; using Catalogger.Backend.Cache; +using Catalogger.Backend.Cache.InMemoryCache; using Catalogger.Backend.Database; using Catalogger.Backend.Database.Models; using Catalogger.Backend.Extensions; using Catalogger.Backend.Services; using Remora.Discord.API.Abstractions.Gateway.Events; -using Remora.Discord.API.Gateway.Events; using Remora.Discord.Extensions.Embeds; using Remora.Discord.Gateway.Responders; using Remora.Results; -namespace Catalogger.Backend.Bot.Responders; +namespace Catalogger.Backend.Bot.Responders.Guilds; public class GuildCreateResponder( Config config, ILogger logger, DatabaseContext db, - GuildCacheService guildCache, - RoleCacheService roleCache, - ChannelCacheService channelCache, - WebhookExecutorService webhookExecutor) - : IResponder, IResponder + GuildCache guildCache, + RoleCache roleCache, + ChannelCache channelCache, + WebhookExecutorService webhookExecutor, + IMemberCache memberCache, + GuildFetchService guildFetchService) + : IResponder, IResponder { private readonly ILogger _logger = logger.ForContext(); @@ -37,6 +38,9 @@ public class GuildCreateResponder( guildCache.Set(guild); foreach (var c in guild.Channels) channelCache.Set(c, guild.ID); foreach (var r in guild.Roles) roleCache.Set(r, guild.ID); + + if (!await memberCache.IsGuildCachedAsync(guild.ID)) + guildFetchService.EnqueueGuild(guild.ID); } else { @@ -71,7 +75,7 @@ public class GuildCreateResponder( return Result.Success; } - public async Task RespondAsync(GuildDelete evt, CancellationToken ct = default) + public async Task RespondAsync(IGuildDelete evt, CancellationToken ct = default) { if (evt.IsUnavailable.OrDefault(false)) { diff --git a/Catalogger.Backend/Bot/Responders/Guilds/GuildMembersChunkResponder.cs b/Catalogger.Backend/Bot/Responders/Guilds/GuildMembersChunkResponder.cs new file mode 100644 index 0000000..be85fa8 --- /dev/null +++ b/Catalogger.Backend/Bot/Responders/Guilds/GuildMembersChunkResponder.cs @@ -0,0 +1,27 @@ +using Catalogger.Backend.Cache; +using Remora.Discord.API.Abstractions.Gateway.Events; +using Remora.Discord.Gateway.Responders; +using Remora.Results; + +namespace Catalogger.Backend.Bot.Responders.Guilds; + +public class GuildMembersChunkResponder(ILogger logger, IMemberCache memberCache) : IResponder +{ + private readonly ILogger _logger = logger.ForContext(); + + public async Task RespondAsync(IGuildMembersChunk evt, CancellationToken ct = default) + { + _logger.Debug("Received chunk {ChunkIndex} / {ChunkCount} for guild {GuildId}", evt.ChunkIndex + 1, + evt.ChunkCount, evt.GuildID); + + await memberCache.SetManyAsync(evt.GuildID, evt.Members); + + if (evt.ChunkIndex == evt.ChunkCount - 1) + { + _logger.Debug("Final chunk for guild {GuildId} received, marking as cached", evt.GuildID); + await memberCache.MarkAsCachedAsync(evt.GuildID); + } + + return Result.Success; + } +} \ No newline at end of file diff --git a/Catalogger.Backend/Bot/Responders/MessageCreateResponder.cs b/Catalogger.Backend/Bot/Responders/MessageCreateResponder.cs index c608fb8..4ba968f 100644 --- a/Catalogger.Backend/Bot/Responders/MessageCreateResponder.cs +++ b/Catalogger.Backend/Bot/Responders/MessageCreateResponder.cs @@ -1,16 +1,15 @@ using System.Text.RegularExpressions; using App.Metrics; using Catalogger.Backend.Cache; +using Catalogger.Backend.Cache.InMemoryCache; using Catalogger.Backend.Database; using Catalogger.Backend.Database.Models; using Catalogger.Backend.Database.Queries; using Catalogger.Backend.Extensions; using Catalogger.Backend.Services; using Humanizer; -using Remora.Discord.API; using Remora.Discord.API.Abstractions.Gateway.Events; using Remora.Discord.Gateway.Responders; -using Remora.Rest.Core; using Remora.Results; namespace Catalogger.Backend.Bot.Responders; @@ -20,7 +19,7 @@ public class MessageCreateResponder( Config config, DatabaseContext db, MessageRepository messageRepository, - UserCacheService userCache, + UserCache userCache, PkMessageHandler pkMessageHandler, IMetrics metrics) : IResponder diff --git a/Catalogger.Backend/Bot/Responders/MessageDeleteResponder.cs b/Catalogger.Backend/Bot/Responders/MessageDeleteResponder.cs index bc8ebb0..0d93efd 100644 --- a/Catalogger.Backend/Bot/Responders/MessageDeleteResponder.cs +++ b/Catalogger.Backend/Bot/Responders/MessageDeleteResponder.cs @@ -1,4 +1,5 @@ using Catalogger.Backend.Cache; +using Catalogger.Backend.Cache.InMemoryCache; using Catalogger.Backend.Database; using Catalogger.Backend.Database.Queries; using Catalogger.Backend.Extensions; @@ -22,8 +23,8 @@ public class MessageDeleteResponder( DatabaseContext db, MessageRepository messageRepository, WebhookExecutorService webhookExecutor, - ChannelCacheService channelCache, - UserCacheService userCache, + ChannelCache channelCache, + UserCache userCache, IClock clock, PluralkitApiService pluralkitApi) : IResponder { diff --git a/Catalogger.Backend/Bot/Responders/MessageUpdateResponder.cs b/Catalogger.Backend/Bot/Responders/MessageUpdateResponder.cs index 9d309e1..f969b05 100644 --- a/Catalogger.Backend/Bot/Responders/MessageUpdateResponder.cs +++ b/Catalogger.Backend/Bot/Responders/MessageUpdateResponder.cs @@ -1,4 +1,5 @@ using Catalogger.Backend.Cache; +using Catalogger.Backend.Cache.InMemoryCache; using Catalogger.Backend.Database; using Catalogger.Backend.Database.Queries; using Catalogger.Backend.Extensions; @@ -17,8 +18,8 @@ namespace Catalogger.Backend.Bot.Responders; public class MessageUpdateResponder( ILogger logger, DatabaseContext db, - ChannelCacheService channelCache, - UserCacheService userCache, + ChannelCache channelCache, + UserCache userCache, MessageRepository messageRepository, WebhookExecutorService webhookExecutor, PluralkitApiService pluralkitApi) : IResponder @@ -60,7 +61,8 @@ public class MessageUpdateResponder( return Result.Success; } - if (oldMessage.Content == msg.Content) return Result.Success; + if (oldMessage.Content == msg.Content || + (oldMessage.Content == "None" && string.IsNullOrEmpty(msg.Content))) return Result.Success; var user = msg.Author; if (msg.Author.ID != oldMessage.UserId) diff --git a/Catalogger.Backend/Cache/IInviteCache.cs b/Catalogger.Backend/Cache/IInviteCache.cs new file mode 100644 index 0000000..8ee4912 --- /dev/null +++ b/Catalogger.Backend/Cache/IInviteCache.cs @@ -0,0 +1,10 @@ +using Remora.Discord.API.Abstractions.Objects; +using Remora.Rest.Core; + +namespace Catalogger.Backend.Cache; + +public interface IInviteCache +{ + public Task> TryGetAsync(Snowflake guildId); + public Task SetAsync(Snowflake guildId, IEnumerable invites); +} \ No newline at end of file diff --git a/Catalogger.Backend/Cache/IMemberCache.cs b/Catalogger.Backend/Cache/IMemberCache.cs new file mode 100644 index 0000000..17edc57 --- /dev/null +++ b/Catalogger.Backend/Cache/IMemberCache.cs @@ -0,0 +1,14 @@ +using Remora.Discord.API.Abstractions.Objects; +using Remora.Rest.Core; + +namespace Catalogger.Backend.Cache; + +public interface IMemberCache +{ + public Task TryGetAsync(Snowflake guildId, Snowflake userId); + public Task SetAsync(Snowflake guildId, IGuildMember member); + public Task SetManyAsync(Snowflake guildId, IReadOnlyList members); + public Task IsGuildCachedAsync(Snowflake guildId); + public Task MarkAsCachedAsync(Snowflake guildId); + public Task MarkAsUncachedAsync(Snowflake guildId); +} \ No newline at end of file diff --git a/Catalogger.Backend/Services/IWebhookCache.cs b/Catalogger.Backend/Cache/IWebhookCache.cs similarity index 95% rename from Catalogger.Backend/Services/IWebhookCache.cs rename to Catalogger.Backend/Cache/IWebhookCache.cs index 84c4881..6d855f3 100644 --- a/Catalogger.Backend/Services/IWebhookCache.cs +++ b/Catalogger.Backend/Cache/IWebhookCache.cs @@ -3,7 +3,7 @@ using Remora.Discord.API; using Remora.Discord.API.Abstractions.Objects; using Remora.Rest.Core; -namespace Catalogger.Backend.Services; +namespace Catalogger.Backend.Cache; public interface IWebhookCache { diff --git a/Catalogger.Backend/Cache/ChannelCacheService.cs b/Catalogger.Backend/Cache/InMemoryCache/ChannelCache.cs similarity index 96% rename from Catalogger.Backend/Cache/ChannelCacheService.cs rename to Catalogger.Backend/Cache/InMemoryCache/ChannelCache.cs index 4423b5b..dfc20de 100644 --- a/Catalogger.Backend/Cache/ChannelCacheService.cs +++ b/Catalogger.Backend/Cache/InMemoryCache/ChannelCache.cs @@ -3,9 +3,9 @@ using System.Diagnostics.CodeAnalysis; using Remora.Discord.API.Abstractions.Objects; using Remora.Rest.Core; -namespace Catalogger.Backend.Cache; +namespace Catalogger.Backend.Cache.InMemoryCache; -public class ChannelCacheService +public class ChannelCache { private readonly ConcurrentDictionary _channels = new(); private readonly ConcurrentDictionary> _guildChannels = new(); diff --git a/Catalogger.Backend/Cache/GuildCacheService.cs b/Catalogger.Backend/Cache/InMemoryCache/GuildCache.cs similarity index 88% rename from Catalogger.Backend/Cache/GuildCacheService.cs rename to Catalogger.Backend/Cache/InMemoryCache/GuildCache.cs index a97eed1..17d4e60 100644 --- a/Catalogger.Backend/Cache/GuildCacheService.cs +++ b/Catalogger.Backend/Cache/InMemoryCache/GuildCache.cs @@ -3,9 +3,9 @@ using System.Diagnostics.CodeAnalysis; using Remora.Discord.API.Abstractions.Objects; using Remora.Rest.Core; -namespace Catalogger.Backend.Cache; +namespace Catalogger.Backend.Cache.InMemoryCache; -public class GuildCacheService +public class GuildCache { private readonly ConcurrentDictionary _guilds = new(); diff --git a/Catalogger.Backend/Cache/InMemoryCache/InMemoryInviteCache.cs b/Catalogger.Backend/Cache/InMemoryCache/InMemoryInviteCache.cs new file mode 100644 index 0000000..603c499 --- /dev/null +++ b/Catalogger.Backend/Cache/InMemoryCache/InMemoryInviteCache.cs @@ -0,0 +1,20 @@ +using System.Collections.Concurrent; +using Remora.Discord.API.Abstractions.Objects; +using Remora.Rest.Core; + +namespace Catalogger.Backend.Cache.InMemoryCache; + +public class InMemoryInviteCache : IInviteCache +{ + private readonly ConcurrentDictionary> _invites = new(); + + public Task> TryGetAsync(Snowflake guildId) => _invites.TryGetValue(guildId, out var invites) + ? Task.FromResult(invites) + : Task.FromResult>([]); + + public Task SetAsync(Snowflake guildId, IEnumerable invites) + { + _invites[guildId] = invites; + return Task.CompletedTask; + } +} \ No newline at end of file diff --git a/Catalogger.Backend/Cache/InMemoryCache/InMemoryMemberCache.cs b/Catalogger.Backend/Cache/InMemoryCache/InMemoryMemberCache.cs new file mode 100644 index 0000000..a3100d4 --- /dev/null +++ b/Catalogger.Backend/Cache/InMemoryCache/InMemoryMemberCache.cs @@ -0,0 +1,46 @@ +using System.Collections.Concurrent; +using Remora.Discord.API.Abstractions.Objects; +using Remora.Rest.Core; + +namespace Catalogger.Backend.Cache.InMemoryCache; + +public class InMemoryMemberCache : IMemberCache +{ + private readonly ConcurrentDictionary<(Snowflake, Snowflake), IGuildMember> _members = new(); + private readonly ConcurrentDictionary _guilds = new(); + +#pragma warning disable CS8619 // Nullability of reference types in value doesn't match target type. + public Task TryGetAsync(Snowflake guildId, Snowflake userId) => + _members.TryGetValue((guildId, userId), out var member) + ? Task.FromResult(member) + : Task.FromResult(null); +#pragma warning restore CS8619 // Nullability of reference types in value doesn't match target type. + + public Task SetAsync(Snowflake guildId, IGuildMember member) + { + if (!member.User.IsDefined()) + throw new CataloggerError("Member with undefined User passed to RedisMemberCache.SetAsync"); + _members[(guildId, member.User.Value.ID)] = member; + return Task.CompletedTask; + } + + public async Task SetManyAsync(Snowflake guildId, IReadOnlyList members) + { + foreach (var member in members) + await SetAsync(guildId, member); + } + + public Task IsGuildCachedAsync(Snowflake guildId) => Task.FromResult(_guilds.ContainsKey(guildId)); + + public Task MarkAsCachedAsync(Snowflake guildId) + { + _guilds[guildId] = 1; + return Task.CompletedTask; + } + + public Task MarkAsUncachedAsync(Snowflake guildId) + { + _guilds.Remove(guildId, out _); + return Task.CompletedTask; + } +} \ No newline at end of file diff --git a/Catalogger.Backend/Services/InMemoryWebhookCache.cs b/Catalogger.Backend/Cache/InMemoryCache/InMemoryWebhookCache.cs similarity index 91% rename from Catalogger.Backend/Services/InMemoryWebhookCache.cs rename to Catalogger.Backend/Cache/InMemoryCache/InMemoryWebhookCache.cs index 5d5d5af..d88d595 100644 --- a/Catalogger.Backend/Services/InMemoryWebhookCache.cs +++ b/Catalogger.Backend/Cache/InMemoryCache/InMemoryWebhookCache.cs @@ -1,6 +1,6 @@ using System.Collections.Concurrent; -namespace Catalogger.Backend.Services; +namespace Catalogger.Backend.Cache.InMemoryCache; public class InMemoryWebhookCache : IWebhookCache { diff --git a/Catalogger.Backend/Cache/RoleCacheService.cs b/Catalogger.Backend/Cache/InMemoryCache/RoleCache.cs similarity index 96% rename from Catalogger.Backend/Cache/RoleCacheService.cs rename to Catalogger.Backend/Cache/InMemoryCache/RoleCache.cs index a51cf94..de8732e 100644 --- a/Catalogger.Backend/Cache/RoleCacheService.cs +++ b/Catalogger.Backend/Cache/InMemoryCache/RoleCache.cs @@ -3,9 +3,9 @@ using System.Diagnostics.CodeAnalysis; using Remora.Discord.API.Abstractions.Objects; using Remora.Rest.Core; -namespace Catalogger.Backend.Cache; +namespace Catalogger.Backend.Cache.InMemoryCache; -public class RoleCacheService +public class RoleCache { private readonly ConcurrentDictionary _roles = new(); private readonly ConcurrentDictionary> _guildRoles = new(); diff --git a/Catalogger.Backend/Cache/UserCacheService.cs b/Catalogger.Backend/Cache/InMemoryCache/UserCache.cs similarity index 86% rename from Catalogger.Backend/Cache/UserCacheService.cs rename to Catalogger.Backend/Cache/InMemoryCache/UserCache.cs index 867fb5d..24c6f77 100644 --- a/Catalogger.Backend/Cache/UserCacheService.cs +++ b/Catalogger.Backend/Cache/InMemoryCache/UserCache.cs @@ -3,9 +3,9 @@ using Remora.Discord.API.Abstractions.Objects; using Remora.Discord.API.Abstractions.Rest; using Remora.Rest.Core; -namespace Catalogger.Backend.Cache; +namespace Catalogger.Backend.Cache.InMemoryCache; -public class UserCacheService(IDiscordRestUserAPI userApi) +public class UserCache(IDiscordRestUserAPI userApi) { private readonly ConcurrentDictionary _cache = new(); diff --git a/Catalogger.Backend/Cache/RedisCache/RedisInviteCache.cs b/Catalogger.Backend/Cache/RedisCache/RedisInviteCache.cs new file mode 100644 index 0000000..38c4814 --- /dev/null +++ b/Catalogger.Backend/Cache/RedisCache/RedisInviteCache.cs @@ -0,0 +1,53 @@ +using Catalogger.Backend.Database.Redis; +using Remora.Discord.API; +using Remora.Discord.API.Abstractions.Objects; +using Remora.Discord.API.Objects; +using Remora.Rest.Core; + +namespace Catalogger.Backend.Cache.RedisCache; + +public class RedisInviteCache(RedisService redisService) : IInviteCache +{ + public async Task> TryGetAsync(Snowflake guildId) + { + var redisInvites = await redisService.GetAsync>(InvitesKey(guildId)) ?? []; + return redisInvites.Select(r => r.ToRemoraInvite()); + } + + public async Task SetAsync(Snowflake guildId, IEnumerable invites) => + await redisService.SetAsync(InvitesKey(guildId), invites.Select(RedisInvite.FromIInvite)); + + private static string InvitesKey(Snowflake guildId) => $"guild-invites:{guildId}"; +} + +internal record RedisInvite( + string Code, + RedisPartialGuild? Guild, + RedisPartialChannel? Channel, + RedisUser? Inviter, + DateTimeOffset? ExpiresAt) +{ + public static RedisInvite FromIInvite(IInvite invite) => new(invite.Code, + invite.Guild.Map(RedisPartialGuild.FromIPartialGuild).OrDefault(), + invite.Channel != null ? RedisPartialChannel.FromIPartialChannel(invite.Channel) : null, + invite.Inviter.Map(RedisUser.FromIUser).OrDefault(), invite.ExpiresAt.OrDefault()); + + public Invite ToRemoraInvite() => new(Code, Guild?.ToRemoraPartialGuild() ?? new Optional(), + Channel?.ToRemoraPartialChannel(), Inviter?.ToRemoraUser() ?? new Optional(), ExpiresAt: ExpiresAt); +} + +internal record RedisPartialGuild(ulong Id, string? Name) +{ + public static RedisPartialGuild FromIPartialGuild(IPartialGuild guild) => + new(guild.ID.Value.Value, guild.Name.OrDefault(null)); + + public PartialGuild ToRemoraPartialGuild() => new(DiscordSnowflake.New(Id), Name ?? new Optional()); +} + +internal record RedisPartialChannel(ulong Id, string? Name) +{ + public static RedisPartialChannel FromIPartialChannel(IPartialChannel channel) => + new(channel.ID.Value.Value, channel.Name.OrDefault(null)); + + public PartialChannel ToRemoraPartialChannel() => new(DiscordSnowflake.New(Id), Name: Name); +} \ No newline at end of file diff --git a/Catalogger.Backend/Cache/RedisCache/RedisMemberCache.cs b/Catalogger.Backend/Cache/RedisCache/RedisMemberCache.cs new file mode 100644 index 0000000..d97c0c8 --- /dev/null +++ b/Catalogger.Backend/Cache/RedisCache/RedisMemberCache.cs @@ -0,0 +1,85 @@ +using Catalogger.Backend.Database.Redis; +using Remora.Discord.API; +using Remora.Discord.API.Abstractions.Objects; +using Remora.Discord.API.Objects; +using Remora.Rest.Core; + +namespace Catalogger.Backend.Cache.RedisCache; + +public class RedisMemberCache(RedisService redisService) : IMemberCache +{ + public async Task TryGetAsync(Snowflake guildId, Snowflake userId) + { + var redisMember = await redisService.GetHashAsync(GuildMembersKey(guildId), userId.ToString()); + return redisMember?.ToRemoraMember(); + } + + public async Task SetAsync(Snowflake guildId, IGuildMember member) + { + if (!member.User.IsDefined()) + throw new CataloggerError("Member with undefined User passed to RedisMemberCache.SetAsync"); + await redisService.SetHashAsync(GuildMembersKey(guildId), member.User.Value.ID.ToString(), + RedisMember.FromIGuildMember(member)); + } + + public async Task SetManyAsync(Snowflake guildId, IReadOnlyList members) + { + if (members.Any(m => !m.User.IsDefined())) + throw new CataloggerError("Member with undefined User passed to RedisMemberCache.SetAsync"); + var redisMembers = members.Select(RedisMember.FromIGuildMember).ToList(); + + await redisService.SetHashAsync(GuildMembersKey(guildId), redisMembers, m => m.User.Id.ToString()); + } + + public async Task IsGuildCachedAsync(Snowflake guildId) => + await redisService.GetDatabase().SetContainsAsync(GuildCacheKey, guildId.ToString()); + + public async Task MarkAsCachedAsync(Snowflake guildId) => + await redisService.GetDatabase().SetAddAsync(GuildCacheKey, guildId.ToString()); + + public async Task MarkAsUncachedAsync(Snowflake guildId) => + await redisService.GetDatabase().SetRemoveAsync(GuildCacheKey, guildId.ToString()); + + private const string GuildCacheKey = "cached-guilds"; + private static string GuildMembersKey(Snowflake guildId) => $"guild-members:{guildId}"; +} + +internal record RedisMember( + RedisUser User, + string? Nickname, + string? Avatar, + Snowflake[] Roles, + DateTimeOffset JoinedAt, + DateTimeOffset? PremiumSince, + GuildMemberFlags Flags, + bool? IsPending, + DateTimeOffset? CommunicationDisabledUntil) +{ + public static RedisMember FromIGuildMember(IGuildMember member) => new( + RedisUser.FromIUser(member.User.Value), member.Nickname.OrDefault(null), member.Avatar.OrDefault(null)?.Value, + member.Roles.ToArray(), member.JoinedAt, member.PremiumSince.OrDefault(null), member.Flags, + member.IsPending.OrDefault(null), member.CommunicationDisabledUntil.OrDefault(null)); + + public GuildMember ToRemoraMember() => new(User.ToRemoraUser(), Nickname, + Avatar != null ? new ImageHash(Avatar) : null, Roles, JoinedAt, PremiumSince, false, false, Flags, + IsPending, default, CommunicationDisabledUntil); +} + +internal record RedisUser( + ulong Id, + string Username, + ushort Discriminator, + string? GlobalName, + string? Avatar, + bool IsBot, + bool IsSystem, + string? Banner) +{ + public static RedisUser FromIUser(IUser user) => new(user.ID.Value, user.Username, user.Discriminator, + user.GlobalName.OrDefault(null), user.Avatar?.Value, user.IsBot.OrDefault(false), + user.IsSystem.OrDefault(false), user.Banner.OrDefault(null)?.Value); + + public User ToRemoraUser() => new(DiscordSnowflake.New(Id), Username, Discriminator, GlobalName, + Avatar != null ? new ImageHash(Avatar) : null, IsBot, IsSystem, + Banner: Banner != null ? new ImageHash(Banner) : null); +} \ No newline at end of file diff --git a/Catalogger.Backend/Services/RedisWebhookCache.cs b/Catalogger.Backend/Cache/RedisCache/RedisWebhookCache.cs similarity index 91% rename from Catalogger.Backend/Services/RedisWebhookCache.cs rename to Catalogger.Backend/Cache/RedisCache/RedisWebhookCache.cs index 4e13da6..b11d9db 100644 --- a/Catalogger.Backend/Services/RedisWebhookCache.cs +++ b/Catalogger.Backend/Cache/RedisCache/RedisWebhookCache.cs @@ -1,7 +1,7 @@ using Catalogger.Backend.Database.Redis; using Humanizer; -namespace Catalogger.Backend.Services; +namespace Catalogger.Backend.Cache.RedisCache; public class RedisWebhookCache(RedisService redisService) : IWebhookCache { diff --git a/Catalogger.Backend/Catalogger.Backend.csproj b/Catalogger.Backend/Catalogger.Backend.csproj index d0bd5c2..d585be1 100644 --- a/Catalogger.Backend/Catalogger.Backend.csproj +++ b/Catalogger.Backend/Catalogger.Backend.csproj @@ -30,6 +30,20 @@ + + + + + + + + + + + + + + diff --git a/Catalogger.Backend/Database/Redis/RedisService.cs b/Catalogger.Backend/Database/Redis/RedisService.cs index d3b3a11..099c1c8 100644 --- a/Catalogger.Backend/Database/Redis/RedisService.cs +++ b/Catalogger.Backend/Database/Redis/RedisService.cs @@ -1,5 +1,8 @@ using System.Text.Json; -using Humanizer; +using Remora.Discord.API; +using Remora.Discord.API.Abstractions.Objects; +using Remora.Discord.API.Objects; +using Remora.Rest.Json; using StackExchange.Redis; namespace Catalogger.Backend.Database.Redis; @@ -8,17 +11,42 @@ public class RedisService(Config config) { private readonly ConnectionMultiplexer _multiplexer = ConnectionMultiplexer.Connect(config.Database.Redis!); + private readonly JsonSerializerOptions _options = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower + }; + public IDatabase GetDatabase(int db = -1) => _multiplexer.GetDatabase(db); public async Task SetAsync(string key, T value, TimeSpan? expiry = null) { - var json = JsonSerializer.Serialize(value); + var json = JsonSerializer.Serialize(value, _options); await GetDatabase().StringSetAsync(key, json, expiry); } public async Task GetAsync(string key) { var value = await GetDatabase().StringGetAsync(key); - return value.IsNull ? default : JsonSerializer.Deserialize(value!); + return value.IsNull ? default : JsonSerializer.Deserialize(value!, _options); + } + + public async Task SetHashAsync(string hashKey, string fieldKey, T value) + { + var json = JsonSerializer.Serialize(value, _options); + await GetDatabase().HashSetAsync(hashKey, fieldKey, json); + } + + public async Task SetHashAsync(string hashKey, IEnumerable values, Func keySelector) + { + var hashEntries = values + .Select(v => new { Key = keySelector(v), Value = JsonSerializer.Serialize(v, _options) }) + .Select(v => new HashEntry(v.Key, v.Value)); + await GetDatabase().HashSetAsync(hashKey, hashEntries.ToArray()); + } + + public async Task GetHashAsync(string hashKey, string fieldKey) + { + var value = await GetDatabase().HashGetAsync(hashKey, fieldKey); + return value.IsNull ? default : JsonSerializer.Deserialize(value!, _options); } } \ No newline at end of file diff --git a/Catalogger.Backend/Extensions/StartupExtensions.cs b/Catalogger.Backend/Extensions/StartupExtensions.cs index 14c7d86..2792230 100644 --- a/Catalogger.Backend/Extensions/StartupExtensions.cs +++ b/Catalogger.Backend/Extensions/StartupExtensions.cs @@ -1,6 +1,8 @@ using Catalogger.Backend.Bot.Commands; using Catalogger.Backend.Bot.Responders; using Catalogger.Backend.Cache; +using Catalogger.Backend.Cache.InMemoryCache; +using Catalogger.Backend.Cache.RedisCache; using Catalogger.Backend.Database; using Catalogger.Backend.Database.Queries; using Catalogger.Backend.Database.Redis; @@ -65,27 +67,34 @@ public static class StartupExtensions public static IServiceCollection AddCustomServices(this IServiceCollection services) => services .AddSingleton(SystemClock.Instance) - .AddSingleton() - .AddSingleton() - .AddSingleton() - .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() .AddSingleton() .AddScoped() .AddScoped() .AddSingleton() .AddSingleton() .AddSingleton(InMemoryDataService.Instance) - .AddHostedService(); + .AddHostedService() + .AddSingleton() + .AddHostedService(serviceProvider => serviceProvider.GetRequiredService()); public static IServiceCollection MaybeAddRedisCaches(this IServiceCollection services, Config config) { if (config.Database.Redis == null) { - return services.AddSingleton(); + return services + .AddSingleton() + .AddSingleton() + .AddSingleton(); } return services.AddSingleton() - .AddScoped(); + .AddSingleton() + .AddSingleton() + .AddSingleton(); } public static async Task Initialize(this WebApplication app) diff --git a/Catalogger.Backend/Services/GuildFetchService.cs b/Catalogger.Backend/Services/GuildFetchService.cs new file mode 100644 index 0000000..afb43dc --- /dev/null +++ b/Catalogger.Backend/Services/GuildFetchService.cs @@ -0,0 +1,47 @@ +using System.Collections.Concurrent; +using Catalogger.Backend.Cache; +using Humanizer; +using Remora.Discord.API.Abstractions.Rest; +using Remora.Discord.API.Gateway.Commands; +using Remora.Discord.Gateway; +using Remora.Rest.Core; + +namespace Catalogger.Backend.Services; + +public class GuildFetchService( + ILogger logger, + DiscordGatewayClient gatewayClient, + IDiscordRestGuildAPI guildApi, + IInviteCache inviteCache) : BackgroundService +{ + private readonly ILogger _logger = logger.ForContext(); + private readonly ConcurrentQueue _guilds = new(); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + using var timer = new PeriodicTimer(500.Milliseconds()); + while (await timer.WaitForNextTickAsync(stoppingToken)) + { + if (!_guilds.TryPeek(out var guildId)) continue; + + _logger.Debug("Fetching members and invites for guild {GuildId}", guildId); + gatewayClient.SubmitCommand(new RequestGuildMembers(guildId, "", 0)); + var res = await guildApi.GetGuildInvitesAsync(guildId, stoppingToken); + if (res.Error != null) + { + _logger.Error("Fetching invites for guild {GuildId}: {Error}", guildId, res.Error); + } + else + { + await inviteCache.SetAsync(guildId, res.Entity); + } + + _guilds.TryDequeue(out _); + } + } + + public void EnqueueGuild(Snowflake guildId) + { + if (!_guilds.Contains(guildId)) _guilds.Enqueue(guildId); + } +} \ No newline at end of file diff --git a/Catalogger.Backend/Services/MetricsCollectionService.cs b/Catalogger.Backend/Services/MetricsCollectionService.cs index b6d6040..2d6da0d 100644 --- a/Catalogger.Backend/Services/MetricsCollectionService.cs +++ b/Catalogger.Backend/Services/MetricsCollectionService.cs @@ -1,6 +1,7 @@ using System.Diagnostics; using App.Metrics; using Catalogger.Backend.Cache; +using Catalogger.Backend.Cache.InMemoryCache; using Catalogger.Backend.Database; using Humanizer; using Microsoft.EntityFrameworkCore; @@ -10,9 +11,9 @@ namespace Catalogger.Backend.Services; public class MetricsCollectionService( ILogger logger, - GuildCacheService guildCache, - ChannelCacheService channelCache, - UserCacheService userCache, + GuildCache guildCache, + ChannelCache channelCache, + UserCache userCache, IMetrics metrics, IServiceProvider services) : BackgroundService { diff --git a/Catalogger.Backend/Services/WebhookExecutorService.cs b/Catalogger.Backend/Services/WebhookExecutorService.cs index bb5e6c8..56e3b3e 100644 --- a/Catalogger.Backend/Services/WebhookExecutorService.cs +++ b/Catalogger.Backend/Services/WebhookExecutorService.cs @@ -1,5 +1,6 @@ using System.Collections.Concurrent; using Catalogger.Backend.Cache; +using Catalogger.Backend.Cache.InMemoryCache; using Catalogger.Backend.Extensions; using Remora.Discord.API; using Remora.Discord.API.Abstractions.Objects; @@ -14,7 +15,7 @@ public class WebhookExecutorService( Config config, ILogger logger, IWebhookCache webhookCache, - ChannelCacheService channelCache, + ChannelCache channelCache, IDiscordRestWebhookAPI webhookApi) { private readonly ILogger _logger = logger.ForContext(); diff --git a/catalogger.sln.DotSettings b/catalogger.sln.DotSettings index cda3670..b3d1d34 100644 --- a/catalogger.sln.DotSettings +++ b/catalogger.sln.DotSettings @@ -1,2 +1,3 @@  - True \ No newline at end of file + True + True \ No newline at end of file