sam/Catalogger.NET
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

init

Browse source
This commit is contained in:
sam 2024-08-13 13:08:50 +02:00
commit ded4f4db26
Signed by: sam
GPG key ID: 5F3C3C1B3166639D
43 changed files with 2052 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

32
Catalogger.Backend/Bot/Commands/MetaCommands.cs Normal file
View file

@ -0,0 +1,32 @@
using System.ComponentModel;
using Catalogger.Backend.Extensions;
using NodaTime;
using Remora.Commands.Attributes;
using Remora.Commands.Groups;
using Remora.Discord.API.Abstractions.Rest;
using Remora.Discord.Commands.Feedback.Services;
using Remora.Discord.Gateway;
using Remora.Results;
using IResult = Remora.Results.IResult;
namespace Catalogger.Backend.Bot.Commands;
[Group("catalogger")]
public class MetaCommands(
IClock clock,
DiscordGatewayClient client,
IFeedbackService feedbackService,
IDiscordRestChannelAPI channelApi) : CommandGroup
{
[Command("ping")]
[Description("Ping pong! See the bot's latency")]
public async Task<IResult> PingAsync()
{
var t1 = clock.GetCurrentInstant();
var msg = await feedbackService.SendContextualAsync("...").GetOrThrow();
var elapsed = clock.GetCurrentInstant() - t1;
return (Result)await channelApi.EditMessageAsync(msg.ChannelID, msg.ID,
content: $"Pong! API: {elapsed.TotalMilliseconds:N0}ms | Gateway: {client.Latency.TotalMilliseconds:N0}ms");
}
}

9
Catalogger.Backend/Bot/DiscordUtils.cs Normal file
View file

@ -0,0 +1,9 @@
using System.Drawing;
namespace Catalogger.Backend.Bot;
public static class DiscordUtils
{
public static readonly Color Red = Color.FromArgb(231, 76, 60);
public static readonly Color Blue = Color.FromArgb(155, 89, 182);
}

49
Catalogger.Backend/Bot/Responders/GuildCreateResponder.cs Normal file
View file

@ -0,0 +1,49 @@
using System.Diagnostics;
using Catalogger.Backend.Cache;
using Catalogger.Backend.Database;
using Catalogger.Backend.Database.Models;
using Catalogger.Backend.Extensions;
using Remora.Discord.API.Abstractions.Gateway.Events;
using Remora.Discord.Gateway.Responders;
using Remora.Results;
namespace Catalogger.Backend.Bot.Responders;
public class GuildCreateResponder(ILogger logger, DatabaseContext db, ChannelCacheService channelCache)
: IResponder<IGuildCreate>
{
private readonly ILogger _logger = logger.ForContext<GuildCreateResponder>();
public async Task<Result> RespondAsync(IGuildCreate evt, CancellationToken ct = default)
{
ulong guildId;
string? guildName = null;
if (evt.Guild.TryPickT0(out var guild, out _))
{
_logger.Verbose("Received guild create for available guild {GuildId} ({Name})", guild.ID, guild.Name);
guildId = guild.ID.ToUlong();
guildName = guild.Name;
foreach (var c in guild.Channels) channelCache.AddChannel(c, guild.ID);
}
else if (evt.Guild.TryPickT1(out var unavailableGuild, out _))
{
_logger.Verbose("Received guild create for unavailable guild {GuildId}", unavailableGuild.ID);
guildId = unavailableGuild.ID.ToUlong();
}
else throw new UnreachableException();
var tx = await db.Database.BeginTransactionAsync(ct);
if (await db.Guilds.FindAsync([guildId], ct) != null) return Result.Success;
db.Add(new Guild
{
Id = guildId
});
await db.SaveChangesAsync(ct);
await tx.CommitAsync(ct);
_logger.Information("Joined new guild {GuildId} ({Name})", guildId, guildName);
return Result.Success;
}
}

157
Catalogger.Backend/Bot/Responders/MessageCreateResponder.cs Normal file
View file

@ -0,0 +1,157 @@
using System.Text.RegularExpressions;
using Catalogger.Backend.Cache;
using Catalogger.Backend.Database;
using Catalogger.Backend.Database.Models;
using Catalogger.Backend.Database.Queries;
using Catalogger.Backend.Extensions;
using Catalogger.Backend.Services;
using Humanizer;
using Remora.Discord.API;
using Remora.Discord.API.Abstractions.Gateway.Events;
using Remora.Discord.Gateway.Responders;
using Remora.Rest.Core;
using Remora.Results;
namespace Catalogger.Backend.Bot.Responders;
public class MessageCreateResponder(
ILogger logger,
DatabaseContext db,
MessageRepository messageRepository,
UserCacheService userCache,
PkMessageHandler pkMessageHandler)
: IResponder<IMessageCreate>
{
private readonly ILogger _logger = logger.ForContext<MessageCreateResponder>();
private static readonly Snowflake PkUserId = DiscordSnowflake.New(466378653216014359);
public async Task<Result> RespondAsync(IMessageCreate msg, CancellationToken ct = default)
{
userCache.UpdateUser(msg.Author);
if (!msg.GuildID.IsDefined())
{
_logger.Debug("Received message create event for message {MessageId} despite it not being in a guild",
msg.ID);
return Result.Success;
}
var guild = await db.GetGuildAsync(msg.GuildID, ct);
// The guild needs to have enabled at least one of the message logging events,
// and the channel must not be ignored, to store the message.
if (guild.IsMessageIgnored(msg.ChannelID, msg.Author.ID))
{
db.IgnoredMessages.Add(new IgnoredMessage(msg.ID.ToUlong()));
await db.SaveChangesAsync(ct);
return Result.Success;
}
if (msg.Author.ID == PkUserId)
_ = pkMessageHandler.HandlePkMessageAsync(msg);
if (msg.ApplicationID.IsDefined(out var appId) && appId == PkUserId)
_ = pkMessageHandler.HandleProxiedMessageAsync(msg.ID.Value);
await messageRepository.SaveMessageAsync(msg, ct);
return Result.Success;
}
}
public partial class PkMessageHandler(ILogger logger, IServiceProvider services)
{
private readonly ILogger _logger = logger.ForContext<PkMessageHandler>();
[GeneratedRegex(
@"^System ID: (\w{5,6}) \| Member ID: (\w{5,6}) \| Sender: .+ \((\d+)\) \| Message ID: (\d+) \| Original Message ID: (\d+)$")]
private static partial Regex FooterRegex();
[GeneratedRegex(@"^https:\/\/discord.com\/channels\/\d+\/(\d+)\/\d+$")]
private static partial Regex LinkRegex();
public async Task HandlePkMessageAsync(IMessageCreate msg)
{
_logger.Debug("Received PluralKit message");
await Task.Delay(500.Milliseconds());
_logger.Debug("Starting handling PluralKit message");
// Check if the content matches a Discord link--if not, it's not a log message (we already check if this is a PluralKit message earlier)
if (!LinkRegex().IsMatch(msg.Content))
{
_logger.Debug("PluralKit message is not a log message because content is not a link");
return;
}
// The first (only, I think always?) embed's footer must match the expected format
var firstEmbed = msg.Embeds.FirstOrDefault();
if (firstEmbed == null || !firstEmbed.Footer.TryGet(out var footer) ||
!FooterRegex().IsMatch(footer.Text))
{
_logger.Debug(
"PK message is not a log message because there is no first embed or its footer doesn't match the regex");
return;
}
var match = FooterRegex().Match(footer.Text);
if (!ulong.TryParse(match.Groups[3].Value, out var authorId))
{
_logger.Debug("Author ID in PluralKit log {LogMessageId} was not a valid snowflake", msg.ID);
return;
}
if (!ulong.TryParse(match.Groups[4].Value, out var msgId))
{
_logger.Debug("Message ID in PluralKit log {LogMessageId} was not a valid snowflake", msg.ID);
return;
}
if (!ulong.TryParse(match.Groups[5].Value, out var originalId))
{
_logger.Debug("Original ID in PluralKit log {LogMessageId} was not a valid snowflake", msg.ID);
return;
}
await using var scope = services.CreateAsyncScope();
await using var db = scope.ServiceProvider.GetRequiredService<DatabaseContext>();
var messageRepository = scope.ServiceProvider.GetRequiredService<MessageRepository>();
await messageRepository.SetProxiedMessageDataAsync(msgId, originalId, authorId,
systemId: match.Groups[1].Value, memberId: match.Groups[2].Value);
db.IgnoredMessages.Add(new IgnoredMessage(originalId));
await db.SaveChangesAsync();
}
public async Task HandleProxiedMessageAsync(ulong msgId)
{
await Task.Delay(3.Seconds());
await using var scope = services.CreateAsyncScope();
await using var db = scope.ServiceProvider.GetRequiredService<DatabaseContext>();
var messageRepository = scope.ServiceProvider.GetRequiredService<MessageRepository>();
var pluralkitApi = scope.ServiceProvider.GetRequiredService<PluralkitApiService>();
var (isStored, hasProxyInfo) = await messageRepository.HasProxyInfoAsync(msgId);
if (!isStored)
{
_logger.Debug("Message with ID {MessageId} is not stored in the database", msgId);
return;
}
if (hasProxyInfo) return;
var pkMessage = await pluralkitApi.GetPluralKitMessageAsync(msgId);
if (pkMessage == null)
{
_logger.Debug("Message with ID {MessageId} was proxied by PluralKit, but API returned 404", msgId);
return;
}
await messageRepository.SetProxiedMessageDataAsync(msgId, pkMessage.Original, pkMessage.Sender,
pkMessage.System?.Id, pkMessage.Member?.Id);
db.IgnoredMessages.Add(new IgnoredMessage(pkMessage.Original));
await db.SaveChangesAsync();
}
}

102
Catalogger.Backend/Bot/Responders/MessageDeleteResponder.cs Normal file
View file

@ -0,0 +1,102 @@
using Catalogger.Backend.Cache;
using Catalogger.Backend.Database;
using Catalogger.Backend.Database.Queries;
using Catalogger.Backend.Extensions;
using Catalogger.Backend.Services;
using Humanizer;
using NodaTime;
using Remora.Discord.API;
using Remora.Discord.API.Abstractions.Gateway.Events;
using Remora.Discord.API.Abstractions.Objects;
using Remora.Discord.API.Objects;
using Remora.Discord.Extensions.Embeds;
using Remora.Discord.Gateway.Responders;
using Remora.Results;
namespace Catalogger.Backend.Bot.Responders;
public class MessageDeleteResponder(
ILogger logger,
DatabaseContext db,
MessageRepository messageRepository,
WebhookExecutorService webhookExecutor,
ChannelCacheService channelCache,
UserCacheService userCache,
IClock clock) : IResponder<IMessageDelete>
{
private readonly ILogger _logger = logger.ForContext<MessageDeleteResponder>();
public async Task<Result> RespondAsync(IMessageDelete ev, CancellationToken ct = default)
{
if (!ev.GuildID.IsDefined()) return Result.Success;
if (ev.ID.Timestamp < DateTimeOffset.Now - 1.Minutes())
{
_logger.Debug(
"Deleted message {MessageId} is less than 1 minute old, delaying 5 seconds to give PK time to catch up",
ev.ID);
await Task.Delay(5.Seconds(), ct);
}
if (await messageRepository.IsMessageIgnoredAsync(ev.ID.Value, ct)) return Result.Success;
var guild = await db.GetGuildAsync(ev.GuildID, ct);
if (guild.IsMessageIgnored(ev.ChannelID, ev.ID)) return Result.Success;
var logChannel = webhookExecutor.GetLogChannel(guild, LogChannelType.MessageDelete, ev.ChannelID);
var msg = await messageRepository.GetMessageAsync(ev.ID.Value, ct);
// Sometimes a message that *should* be logged isn't stored in the database, notify the user of that
if (msg == null)
{
if (logChannel == null) return Result.Success;
await webhookExecutor.QueueLogAsync(logChannel.Value, new Embed(
Title: "Message deleted",
Description: $"A message not found in the database was deleted in <#{ev.ChannelID}> ({ev.ChannelID}).",
Footer: new EmbedFooter(Text: $"ID: {ev.ID}"),
Timestamp: clock.GetCurrentInstant().ToDateTimeOffset()
));
return Result.Success;
}
logChannel = webhookExecutor.GetLogChannel(guild, LogChannelType.MessageDelete, ev.ChannelID, msg.UserId);
if (logChannel == null) return Result.Success;
var user = await userCache.GetUserAsync(DiscordSnowflake.New(msg.UserId));
var builder = new EmbedBuilder()
.WithTitle("Message deleted")
.WithDescription(msg.Content)
.WithColour(DiscordUtils.Red)
.WithFooter($"ID: {msg.Id}")
.WithTimestamp(ev.ID);
if (user != null)
builder.WithAuthor(user.Tag(), url: null, iconUrl: user.AvatarUrl());
if (msg.Member != null) builder.WithTitle($"Message by {msg.Username} deleted");
string channelMention;
if (!channelCache.GetChannel(ev.ChannelID, out var channel))
channelMention = $"<#{msg.ChannelId}>";
else if (channel.Type is ChannelType.AnnouncementThread or ChannelType.PrivateThread
or ChannelType.PublicThread)
channelMention =
$"<#{channel.ParentID.Value}>\nID: {channel.ParentID.Value}\n\nThread: {channel.Name} (<#{channel.ID}>)";
else channelMention = $"<#{channel.ID}>\nID: {channel.ID}";
var userMention = user != null
? $"<@{user.ID}>\n{user.Tag()}\nID: {user.ID}"
: $"<@{msg.UserId}>\nID: {msg.UserId}";
builder.AddField("Channel", channelMention, true);
builder.AddField(msg.System != null ? "Linked Discord account" : "Sender", userMention, true);
if (msg is { System: not null, Member: not null })
{
builder.AddField("\u200b", "**PluralKit information**", false);
builder.AddField("System ID", msg.System, true);
builder.AddField("Member ID", msg.Member, true);
}
await webhookExecutor.QueueLogAsync(logChannel.Value, builder.Build().GetOrThrow());
return Result.Success;
}
}

27
Catalogger.Backend/Bot/Responders/ReadyResponder.cs Normal file
View file

@ -0,0 +1,27 @@
using Catalogger.Backend.Extensions;
using Catalogger.Backend.Services;
using Remora.Discord.API.Abstractions.Gateway.Events;
using Remora.Discord.Gateway.Responders;
using Remora.Results;
namespace Catalogger.Backend.Bot.Responders;
public class ReadyResponder(ILogger logger, WebhookExecutorService webhookExecutorService, Config config)
: IResponder<IReady>
{
private readonly ILogger _logger = logger.ForContext<ReadyResponder>();
public Task<Result> RespondAsync(IReady gatewayEvent, CancellationToken ct = default)
{
var shardId = gatewayEvent.Shard.TryGet(out var shard) ? (shard.ShardID, shard.ShardCount) : (0, 1);
_logger.Information("Ready as {User} on shard {ShardId} / {ShardCount}", gatewayEvent.User.Tag(), shardId.Item1,
shardId.Item2);
if (shardId.Item1 == 0) webhookExecutorService.SetSelfUser(gatewayEvent.User);
// Sanity check
var appId = gatewayEvent.Application.ID.ToUlong();
_logger.Debug("Application ID is {ApplicationId}, is same as config? {SameAsConfig}", appId,
appId == config.Discord.ApplicationId);
return Task.FromResult(Result.Success);
}
}

56
Catalogger.Backend/Cache/ChannelCacheService.cs Normal file
View file

@ -0,0 +1,56 @@
using System.Collections.Concurrent;
using System.Diagnostics.CodeAnalysis;
using Remora.Discord.API.Abstractions.Objects;
using Remora.Rest.Core;
namespace Catalogger.Backend.Cache;
public class ChannelCacheService
{
private readonly ConcurrentDictionary<Snowflake, IChannel> _channels = new();
private readonly ConcurrentDictionary<Snowflake, HashSet<Snowflake>> _guildChannels = new();
public void AddChannel(IChannel channel, Snowflake? guildId = null)
{
_channels[channel.ID] = channel;
if (guildId == null)
{
if (!channel.GuildID.TryGet(out var snowflake)) return;
guildId = snowflake;
}
// Add to set of guild channels
_guildChannels.AddOrUpdate(guildId.Value,
_ => [channel.ID],
(_, l) =>
{
l.Add(channel.ID);
return l;
});
}
public bool GetChannel(Snowflake id, [NotNullWhen(true)] out IChannel? channel) => _channels.TryGetValue(id, out channel);
public void RemoveChannel(Snowflake? guildId, Snowflake id, out IChannel? channel)
{
_channels.Remove(id, out channel);
if (guildId == null) return;
// Remove from set of guild channels
_guildChannels.AddOrUpdate(guildId.Value, _ => [], (_, s) =>
{
s.Remove(id);
return s;
});
}
/// <summary>
/// Gets all of a guild's cached channels.
/// </summary>
/// <param name="guildId">The guild to get the channels of</param>
/// <returns>A list of cached channels</returns>
public IEnumerable<IChannel> GuildChannels(Snowflake guildId) =>
!_guildChannels.TryGetValue(guildId, out var channelIds)
? []
: channelIds.Select(id => _channels.GetValueOrDefault(id))
.Where(c => c != null).Select(c => c!);
}

24
Catalogger.Backend/Cache/UserCacheService.cs Normal file
View file

@ -0,0 +1,24 @@
using System.Collections.Concurrent;
using Remora.Discord.API.Abstractions.Objects;
using Remora.Discord.API.Abstractions.Rest;
using Remora.Rest.Core;
namespace Catalogger.Backend.Cache;
public class UserCacheService(IDiscordRestUserAPI userApi)
{
private readonly ConcurrentDictionary<Snowflake, IUser> _cache = new();
public async Task<IUser?> GetUserAsync(Snowflake userId)
{
if (_cache.TryGetValue(userId, out var user)) return user;
var res = await userApi.GetUserAsync(userId);
if (!res.IsSuccess) return null;
_cache[userId] = res.Entity;
return res.Entity;
}
public void UpdateUser(IUser user) => _cache[user.ID] = user;
}

33
Catalogger.Backend/Catalogger.Backend.csproj Normal file
View file

@ -0,0 +1,33 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="EFCore.NamingConventions" Version="8.0.3" />
<PackageReference Include="EntityFrameworkCore.Exceptions.PostgreSQL" Version="8.1.2" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.7" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.6"/>
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.7" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.7">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="NodaTime" Version="3.1.11" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.4" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL.NodaTime" Version="8.0.4" />
<PackageReference Include="Polly.Core" Version="8.4.1" />
<PackageReference Include="Polly.RateLimiting" Version="8.4.1" />
<PackageReference Include="Remora.Discord" Version="2024.2.0" />
<PackageReference Include="Serilog" Version="4.0.1" />
<PackageReference Include="Serilog.AspNetCore" Version="8.0.1" />
<PackageReference Include="Serilog.Extensions.Hosting" Version="8.0.0" />
<PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0"/>
</ItemGroup>
</Project>

5
Catalogger.Backend/CataloggerError.cs Normal file
View file

@ -0,0 +1,5 @@
namespace Catalogger.Backend;
public class CataloggerError(string message) : Exception(message)
{
}

42
Catalogger.Backend/Config.cs Normal file
View file

@ -0,0 +1,42 @@
using Serilog.Events;
namespace Catalogger.Backend;
public class Config
{
public LoggingConfig Logging { get; init; } = new();
public DatabaseConfig Database { get; init; } = new();
public DiscordConfig Discord { get; init; } = new();
public WebConfig Web { get; init; } = new();
public class LoggingConfig
{
public LogEventLevel LogEventLevel { get; init; } = LogEventLevel.Debug;
public bool LogQueries { get; init; } = false;
}
public class DatabaseConfig
{
public string Url { get; init; } = string.Empty;
public string Redis { get; init; } = string.Empty;
public int? Timeout { get; init; }
public int? MaxPoolSize { get; init; }
public string EncryptionKey { get; init; } = string.Empty;
}
public class DiscordConfig
{
public ulong ApplicationId { get; set; }
public string Token { get; init; } = string.Empty;
public bool SyncCommands { get; init; }
public ulong? CommandsGuildId { get; init; }
}
public class WebConfig
{
public string Host { get; init; } = "localhost";
public int Port { get; init; } = 5000;
public string BaseUrl { get; init; } = null!;
public string Address => $"http://{Host}:{Port}";
}
}

97
Catalogger.Backend/Database/DatabaseContext.cs Normal file
View file

@ -0,0 +1,97 @@
using Catalogger.Backend.Database.Models;
using Catalogger.Backend.Extensions;
using EntityFramework.Exceptions.PostgreSQL;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.ChangeTracking;
using Microsoft.EntityFrameworkCore.Design;
using Microsoft.EntityFrameworkCore.Diagnostics;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql;
namespace Catalogger.Backend.Database;
public class DatabaseContext : DbContext
{
private readonly NpgsqlDataSource _dataSource;
private readonly ILoggerFactory? _loggerFactory;
public DbSet<Guild> Guilds { get; set; }
public DbSet<Message> Messages { get; set; }
public DbSet<IgnoredMessage> IgnoredMessages { get; set; }
public DbSet<Invite> Invites { get; set; }
public DbSet<Watchlist> Watchlists { get; set; }
public DatabaseContext(Config config, ILoggerFactory? loggerFactory)
{
var connString = new NpgsqlConnectionStringBuilder(config.Database.Url)
{
Timeout = config.Database.Timeout ?? 5,
MaxPoolSize = config.Database.MaxPoolSize ?? 50,
}.ConnectionString;
var dataSourceBuilder = new NpgsqlDataSourceBuilder(connString);
dataSourceBuilder
.EnableDynamicJson()
.UseNodaTime();
_dataSource = dataSourceBuilder.Build();
_loggerFactory = loggerFactory;
}
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
=> optionsBuilder
.ConfigureWarnings(c =>
c.Ignore(CoreEventId.ManyServiceProvidersCreatedWarning)
.Ignore(CoreEventId.SaveChangesFailed))
.UseNpgsql(_dataSource, o => o.UseNodaTime())
.UseSnakeCaseNamingConvention()
.UseLoggerFactory(_loggerFactory)
.UseExceptionProcessor();
protected override void ConfigureConventions(ModelConfigurationBuilder configurationBuilder)
{
configurationBuilder.Properties<ulong>().HaveConversion<UlongValueConverter>();
configurationBuilder.Properties<List<ulong>>().HaveConversion<UlongArrayValueConverter>();
}
private static readonly ValueComparer<List<ulong>> UlongListValueComparer = new(
(c1, c2) => c1 != null && c2 != null && c1.SequenceEqual(c2),
c => c.Aggregate(0, (a, v) => HashCode.Combine(a, v.GetHashCode()))
);
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
modelBuilder.Entity<Guild>().Property(g => g.KeyRoles)
.Metadata.SetValueComparer(UlongListValueComparer);
modelBuilder.Entity<Invite>().HasKey(i => i.Code);
modelBuilder.Entity<Invite>().HasIndex(i => i.GuildId);
modelBuilder.Entity<Watchlist>().HasKey(w => new { w.GuildId, w.UserId });
modelBuilder.Entity<Watchlist>().Property(w => w.AddedAt).HasDefaultValueSql("now()");
}
}
public class DesignTimeDatabaseContextFactory : IDesignTimeDbContextFactory<DatabaseContext>
{
public DatabaseContext CreateDbContext(string[] args)
{
// Read the configuration file
var config = new ConfigurationBuilder()
.AddConfiguration()
.Build()
// Get the configuration as our config class
.Get<Config>() ?? new();
return new DatabaseContext(config, null);
}
}
public class UlongValueConverter() : ValueConverter<ulong, long>(
convertToProviderExpression: x => (long)x,
convertFromProviderExpression: x => (ulong)x
);
public class UlongArrayValueConverter() : ValueConverter<List<ulong>, List<long>>(
convertToProviderExpression: x => x.Select(i => (long)i).ToList(),
convertFromProviderExpression: x => x.Select(i => (ulong)i).ToList()
);

36
Catalogger.Backend/Database/EncryptionService.cs Normal file
View file

@ -0,0 +1,36 @@
using System.Security.Cryptography;
using System.Text;
namespace Catalogger.Backend.Database;
public class EncryptionService(Config config) : IEncryptionService
{
private readonly byte[] _secretKey = Convert.FromBase64String(config.Database.EncryptionKey);
public byte[] Encrypt(string data)
{
using var aes = Aes.Create();
aes.Key = _secretKey;
var output = new List<byte>();
output.AddRange(aes.IV);
var plaintext = Encoding.UTF8.GetBytes(data);
var ciphertext = aes.EncryptCbc(plaintext, aes.IV);
output.AddRange(ciphertext);
return output.ToArray();
}
public string Decrypt(byte[] input)
{
using var aes = Aes.Create();
aes.Key = _secretKey;
var iv = input.Take(aes.IV.Length).ToArray();
var ciphertext = input.Skip(aes.IV.Length).ToArray();
var plaintext = aes.DecryptCbc(ciphertext, iv);
return Encoding.UTF8.GetString(plaintext);
}
}

7
Catalogger.Backend/Database/IEncryptionService.cs Normal file
View file

@ -0,0 +1,7 @@
namespace Catalogger.Backend.Database;
public interface IEncryptionService
{
public byte[] Encrypt(string data);
public string Decrypt(byte[] input);
}

180
Catalogger.Backend/Database/Migrations/20240803132306_Init.Designer.cs generated Normal file
View file

@ -0,0 +1,180 @@
// <auto-generated />
using System.Collections.Generic;
using Catalogger.Backend.Database;
using Catalogger.Backend.Database.Models;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using NodaTime;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace Catalogger.Backend.Database.Migrations
{
[DbContext(typeof(DatabaseContext))]
[Migration("20240803132306_Init")]
partial class Init
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "8.0.7")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("Catalogger.Backend.Database.Models.Guild", b =>
{
b.Property<long>("Id")
.HasColumnType("bigint")
.HasColumnName("id");
b.Property<List<string>>("BannedSystems")
.IsRequired()
.HasColumnType("text[]")
.HasColumnName("banned_systems");
b.Property<Guild.ChannelConfig>("Channels")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("channels");
b.Property<List<long>>("KeyRoles")
.IsRequired()
.HasColumnType("bigint[]")
.HasColumnName("key_roles");
b.HasKey("Id")
.HasName("pk_guilds");
b.ToTable("guilds", (string)null);
});
modelBuilder.Entity("Catalogger.Backend.Database.Models.IgnoredMessage", b =>
{
b.Property<long>("Id")
.HasColumnType("bigint")
.HasColumnName("id");
b.HasKey("Id")
.HasName("pk_ignored_messages");
b.ToTable("ignored_messages", (string)null);
});
modelBuilder.Entity("Catalogger.Backend.Database.Models.Invite", b =>
{
b.Property<string>("Code")
.HasColumnType("text")
.HasColumnName("code");
b.Property<long>("GuildId")
.HasColumnType("bigint")
.HasColumnName("guild_id");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text")
.HasColumnName("name");
b.HasKey("Code")
.HasName("pk_invites");
b.HasIndex("GuildId")
.HasDatabaseName("ix_invites_guild_id");
b.ToTable("invites", (string)null);
});
modelBuilder.Entity("Catalogger.Backend.Database.Models.Message", b =>
{
b.Property<long>("Id")
.HasColumnType("bigint")
.HasColumnName("id");
b.Property<int>("AttachmentSize")
.HasColumnType("integer")
.HasColumnName("attachment_size");
b.Property<long>("ChannelId")
.HasColumnType("bigint")
.HasColumnName("channel_id");
b.Property<byte[]>("EncryptedContent")
.IsRequired()
.HasColumnType("bytea")
.HasColumnName("content");
b.Property<byte[]>("EncryptedMetadata")
.HasColumnType("bytea")
.HasColumnName("metadata");
b.Property<byte[]>("EncryptedUsername")
.IsRequired()
.HasColumnType("bytea")
.HasColumnName("username");
b.Property<long>("GuildId")
.HasColumnType("bigint")
.HasColumnName("guild_id");
b.Property<string>("Member")
.HasColumnType("text")
.HasColumnName("member");
b.Property<long?>("OriginalId")
.HasColumnType("bigint")
.HasColumnName("original_id");
b.Property<string>("System")
.HasColumnType("text")
.HasColumnName("system");
b.Property<long>("UserId")
.HasColumnType("bigint")
.HasColumnName("user_id");
b.HasKey("Id")
.HasName("pk_messages");
b.ToTable("messages", (string)null);
});
modelBuilder.Entity("Catalogger.Backend.Database.Models.Watchlist", b =>
{
b.Property<long>("GuildId")
.HasColumnType("bigint")
.HasColumnName("guild_id");
b.Property<long>("UserId")
.HasColumnType("bigint")
.HasColumnName("user_id");
b.Property<Instant>("AddedAt")
.ValueGeneratedOnAdd()
.HasColumnType("timestamp with time zone")
.HasColumnName("added_at")
.HasDefaultValueSql("now()");
b.Property<long>("ModeratorId")
.HasColumnType("bigint")
.HasColumnName("moderator_id");
b.Property<string>("Reason")
.IsRequired()
.HasColumnType("text")
.HasColumnName("reason");
b.HasKey("GuildId", "UserId")
.HasName("pk_watchlists");
b.ToTable("watchlists", (string)null);
});
#pragma warning restore 612, 618
}
}
}

115
Catalogger.Backend/Database/Migrations/20240803132306_Init.cs Normal file
View file

@ -0,0 +1,115 @@
using System.Collections.Generic;
using Catalogger.Backend.Database.Models;
using Microsoft.EntityFrameworkCore.Migrations;
using NodaTime;
#nullable disable
namespace Catalogger.Backend.Database.Migrations
{
/// <inheritdoc />
public partial class Init : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "guilds",
columns: table => new
{
id = table.Column<long>(type: "bigint", nullable: false),
channels = table.Column<Guild.ChannelConfig>(type: "jsonb", nullable: false),
banned_systems = table.Column<List<string>>(type: "text[]", nullable: false),
key_roles = table.Column<List<long>>(type: "bigint[]", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("pk_guilds", x => x.id);
});
migrationBuilder.CreateTable(
name: "ignored_messages",
columns: table => new
{
id = table.Column<long>(type: "bigint", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("pk_ignored_messages", x => x.id);
});
migrationBuilder.CreateTable(
name: "invites",
columns: table => new
{
code = table.Column<string>(type: "text", nullable: false),
guild_id = table.Column<long>(type: "bigint", nullable: false),
name = table.Column<string>(type: "text", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("pk_invites", x => x.code);
});
migrationBuilder.CreateTable(
name: "messages",
columns: table => new
{
id = table.Column<long>(type: "bigint", nullable: false),
original_id = table.Column<long>(type: "bigint", nullable: true),
user_id = table.Column<long>(type: "bigint", nullable: false),
channel_id = table.Column<long>(type: "bigint", nullable: false),
guild_id = table.Column<long>(type: "bigint", nullable: false),
member = table.Column<string>(type: "text", nullable: true),
system = table.Column<string>(type: "text", nullable: true),
username = table.Column<byte[]>(type: "bytea", nullable: false),
content = table.Column<byte[]>(type: "bytea", nullable: false),
metadata = table.Column<byte[]>(type: "bytea", nullable: true),
attachment_size = table.Column<int>(type: "integer", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("pk_messages", x => x.id);
});
migrationBuilder.CreateTable(
name: "watchlists",
columns: table => new
{
guild_id = table.Column<long>(type: "bigint", nullable: false),
user_id = table.Column<long>(type: "bigint", nullable: false),
added_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false, defaultValueSql: "now()"),
moderator_id = table.Column<long>(type: "bigint", nullable: false),
reason = table.Column<string>(type: "text", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("pk_watchlists", x => new { x.guild_id, x.user_id });
});
migrationBuilder.CreateIndex(
name: "ix_invites_guild_id",
table: "invites",
column: "guild_id");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "guilds");
migrationBuilder.DropTable(
name: "ignored_messages");
migrationBuilder.DropTable(
name: "invites");
migrationBuilder.DropTable(
name: "messages");
migrationBuilder.DropTable(
name: "watchlists");
}
}
}

177
Catalogger.Backend/Database/Migrations/DatabaseContextModelSnapshot.cs Normal file
View file

@ -0,0 +1,177 @@
// <auto-generated />
using System.Collections.Generic;
using Catalogger.Backend.Database;
using Catalogger.Backend.Database.Models;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using NodaTime;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace Catalogger.Backend.Database.Migrations
{
[DbContext(typeof(DatabaseContext))]
partial class DatabaseContextModelSnapshot : ModelSnapshot
{
protected override void BuildModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "8.0.7")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("Catalogger.Backend.Database.Models.Guild", b =>
{
b.Property<long>("Id")
.HasColumnType("bigint")
.HasColumnName("id");
b.Property<List<string>>("BannedSystems")
.IsRequired()
.HasColumnType("text[]")
.HasColumnName("banned_systems");
b.Property<Guild.ChannelConfig>("Channels")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("channels");
b.Property<List<long>>("KeyRoles")
.IsRequired()
.HasColumnType("bigint[]")
.HasColumnName("key_roles");
b.HasKey("Id")
.HasName("pk_guilds");
b.ToTable("guilds", (string)null);
});
modelBuilder.Entity("Catalogger.Backend.Database.Models.IgnoredMessage", b =>
{
b.Property<long>("Id")
.HasColumnType("bigint")
.HasColumnName("id");
b.HasKey("Id")
.HasName("pk_ignored_messages");
b.ToTable("ignored_messages", (string)null);
});
modelBuilder.Entity("Catalogger.Backend.Database.Models.Invite", b =>
{
b.Property<string>("Code")
.HasColumnType("text")
.HasColumnName("code");
b.Property<long>("GuildId")
.HasColumnType("bigint")
.HasColumnName("guild_id");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text")
.HasColumnName("name");
b.HasKey("Code")
.HasName("pk_invites");
b.HasIndex("GuildId")
.HasDatabaseName("ix_invites_guild_id");
b.ToTable("invites", (string)null);
});
modelBuilder.Entity("Catalogger.Backend.Database.Models.Message", b =>
{
b.Property<long>("Id")
.HasColumnType("bigint")
.HasColumnName("id");
b.Property<int>("AttachmentSize")
.HasColumnType("integer")
.HasColumnName("attachment_size");
b.Property<long>("ChannelId")
.HasColumnType("bigint")
.HasColumnName("channel_id");
b.Property<byte[]>("EncryptedContent")
.IsRequired()
.HasColumnType("bytea")
.HasColumnName("content");
b.Property<byte[]>("EncryptedMetadata")
.HasColumnType("bytea")
.HasColumnName("metadata");
b.Property<byte[]>("EncryptedUsername")
.IsRequired()
.HasColumnType("bytea")
.HasColumnName("username");
b.Property<long>("GuildId")
.HasColumnType("bigint")
.HasColumnName("guild_id");
b.Property<string>("Member")
.HasColumnType("text")
.HasColumnName("member");
b.Property<long?>("OriginalId")
.HasColumnType("bigint")
.HasColumnName("original_id");
b.Property<string>("System")
.HasColumnType("text")
.HasColumnName("system");
b.Property<long>("UserId")
.HasColumnType("bigint")
.HasColumnName("user_id");
b.HasKey("Id")
.HasName("pk_messages");
b.ToTable("messages", (string)null);
});
modelBuilder.Entity("Catalogger.Backend.Database.Models.Watchlist", b =>
{
b.Property<long>("GuildId")
.HasColumnType("bigint")
.HasColumnName("guild_id");
b.Property<long>("UserId")
.HasColumnType("bigint")
.HasColumnName("user_id");
b.Property<Instant>("AddedAt")
.ValueGeneratedOnAdd()
.HasColumnType("timestamp with time zone")
.HasColumnName("added_at")
.HasDefaultValueSql("now()");
b.Property<long>("ModeratorId")
.HasColumnType("bigint")
.HasColumnName("moderator_id");
b.Property<string>("Reason")
.IsRequired()
.HasColumnType("text")
.HasColumnName("reason");
b.HasKey("GuildId", "UserId")
.HasName("pk_watchlists");
b.ToTable("watchlists", (string)null);
});
#pragma warning restore 612, 618
}
}
}

60
Catalogger.Backend/Database/Models/Guild.cs Normal file
View file

@ -0,0 +1,60 @@
using System.ComponentModel.DataAnnotations.Schema;
using Catalogger.Backend.Extensions;
using Remora.Rest.Core;
namespace Catalogger.Backend.Database.Models;
public class Guild
{
[DatabaseGenerated(DatabaseGeneratedOption.None)]
public required ulong Id { get; init; }
[Column(TypeName = "jsonb")]
public ChannelConfig Channels { get; init; } = new();
public List<string> BannedSystems { get; init; } = [];
public List<ulong> KeyRoles { get; init; } = [];
public bool IsMessageIgnored(Snowflake channelId, Snowflake userId)
{
if (Channels is { MessageDelete: 0, MessageUpdate: 0, MessageDeleteBulk: 0 } ||
Channels.IgnoredChannels.Contains(channelId.ToUlong()) ||
Channels.IgnoredUsers.Contains(userId.ToUlong())) return true;
if (Channels.IgnoredUsersPerChannel.TryGetValue(channelId.ToUlong(),
out var thisChannelIgnoredUsers))
return thisChannelIgnoredUsers.Contains(userId.ToUlong());
return false;
}
public class ChannelConfig
{
public List<ulong> IgnoredChannels { get; init; } = [];
public List<ulong> IgnoredUsers { get; init; } = [];
public Dictionary<ulong, List<ulong>> IgnoredUsersPerChannel { get; init; } = [];
public Dictionary<ulong, ulong> Redirects { get; init; } = [];
public ulong GuildUpdate { get; init; }
public ulong GuildEmojisUpdate { get; init; }
public ulong GuildRoleCreate { get; init; }
public ulong GuildRoleUpdate { get; init; }
public ulong GuildRoleDelete { get; init; }
public ulong ChannelCreate { get; init; }
public ulong ChannelUpdate { get; init; }
public ulong ChannelDelete { get; init; }
public ulong GuildMemberAdd { get; init; }
public ulong GuildMemberUpdate { get; init; }
public ulong GuildKeyRoleUpdate { get; init; }
public ulong GuildMemberNickUpdate { get; init; }
public ulong GuildMemberAvatarUpdate { get; init; }
public ulong GuildMemberRemove { get; init; }
public ulong GuildMemberKick { get; init; }
public ulong GuildBanAdd { get; init; }
public ulong GuildBanRemove { get; init; }
public ulong InviteCreate { get; init; }
public ulong InviteDelete { get; init; }
public ulong MessageUpdate { get; init; }
public ulong MessageDelete { get; init; }
public ulong MessageDeleteBulk { get; init; }
}
}

8
Catalogger.Backend/Database/Models/Invite.cs Normal file
View file

@ -0,0 +1,8 @@
namespace Catalogger.Backend.Database.Models;
public class Invite
{
public required ulong GuildId { get; init; }
public required string Code { get; init; }
public required string Name { get; set; }
}

27
Catalogger.Backend/Database/Models/Message.cs Normal file
View file

@ -0,0 +1,27 @@
using System.ComponentModel.DataAnnotations.Schema;
namespace Catalogger.Backend.Database.Models;
public class Message
{
[DatabaseGenerated(DatabaseGeneratedOption.None)]
public required ulong Id { get; init; }
public ulong? OriginalId { get; set; }
public required ulong UserId { get; set; }
public required ulong ChannelId { get; init; }
public required ulong GuildId { get; init; }
public string? Member { get; set; }
public string? System { get; set; }
[Column("username")] public byte[] EncryptedUsername { get; set; } = [];
[Column("content")] public byte[] EncryptedContent { get; set; } = [];
[Column("metadata")] public byte[]? EncryptedMetadata { get; set; }
public int AttachmentSize { get; set; } = 0;
}
public record IgnoredMessage(
[property: DatabaseGenerated(DatabaseGeneratedOption.None)]
ulong Id);

13
Catalogger.Backend/Database/Models/Watchlist.cs Normal file
View file

@ -0,0 +1,13 @@
using NodaTime;
namespace Catalogger.Backend.Database.Models;
public class Watchlist
{
public required ulong GuildId { get; init; }
public required ulong UserId { get; init; }
public Instant AddedAt { get; init; }
public required ulong ModeratorId { get; set; }
public required string Reason { get; set; }
}

99
Catalogger.Backend/Database/Queries/MessageRepository.cs Normal file
View file

@ -0,0 +1,99 @@
using Catalogger.Backend.Database.Models;
using Catalogger.Backend.Extensions;
using Microsoft.EntityFrameworkCore;
using Remora.Discord.API.Abstractions.Gateway.Events;
using DbMessage = Catalogger.Backend.Database.Models.Message;
namespace Catalogger.Backend.Database.Queries;
public class MessageRepository(ILogger logger, DatabaseContext db, IEncryptionService encryptionService)
{
private readonly ILogger _logger = logger.ForContext<MessageRepository>();
public async Task SaveMessageAsync(IMessageCreate msg, CancellationToken ct = default)
{
_logger.Debug("Saving message {MessageId}", msg.ID);
var dbMessage = new DbMessage
{
Id = msg.ID.ToUlong(),
UserId = msg.Author.ID.ToUlong(),
ChannelId = msg.ChannelID.ToUlong(),
GuildId = msg.GuildID.ToUlong(),
EncryptedContent = await Task.Run(() => encryptionService.Encrypt(msg.Content), ct),
EncryptedUsername = await Task.Run(() => encryptionService.Encrypt(msg.Author.Tag()), ct),
AttachmentSize = msg.Attachments.Select(a => a.Size).Sum()
};
db.Add(dbMessage);
await db.SaveChangesAsync(ct);
}
public async Task<Message?> GetMessageAsync(ulong id, CancellationToken ct = default)
{
_logger.Debug("Retrieving message {MessageId}", id);
var dbMsg = await db.Messages.FindAsync(id);
if (dbMsg == null) return null;
return new Message(dbMsg.Id, dbMsg.OriginalId, dbMsg.UserId, dbMsg.ChannelId, dbMsg.GuildId, dbMsg.Member,
dbMsg.System, await Task.Run(() => encryptionService.Decrypt(dbMsg.EncryptedUsername), ct),
await Task.Run(() => encryptionService.Decrypt(dbMsg.EncryptedContent), ct), null, dbMsg.AttachmentSize);
}
/// <summary>
/// Checks if a message has proxy information.
/// If yes, returns (true, true). If no, returns (true, false). If the message isn't saved at all, returns (false, false).
/// </summary>
public async Task<(bool, bool)> HasProxyInfoAsync(ulong id)
{
_logger.Debug("Checking if message {MessageId} has proxy information", id);
var msg = await db.Messages.Select(m => new { m.Id, m.OriginalId }).FirstOrDefaultAsync(m => m.Id == id);
return (msg != null, msg?.OriginalId != null);
}
public async Task SetProxiedMessageDataAsync(ulong id, ulong originalId, ulong authorId, string? systemId,
string? memberId)
{
_logger.Debug("Setting proxy information for message {MessageId}", id);
var message = await db.Messages.FirstOrDefaultAsync(m => m.Id == id);
if (message == null)
{
_logger.Debug("Message {MessageId} not found", id);
return;
}
_logger.Debug("Updating message {MessageId}", id);
message.OriginalId = originalId;
message.UserId = authorId;
message.System = systemId;
message.Member = memberId;
db.Update(message);
await db.SaveChangesAsync();
}
public async Task<bool> IsMessageIgnoredAsync(ulong id, CancellationToken ct = default)
{
_logger.Debug("Checking if message {MessageId} is ignored", id);
return await db.IgnoredMessages.FirstOrDefaultAsync(m => m.Id == id, ct) != null;
}
public record Message(
ulong Id,
ulong? OriginalId,
ulong UserId,
ulong ChannelId,
ulong GuildId,
string? Member,
string? System,
string Username,
string Content,
string? Metadata,
int AttachmentSize
);
}

22
Catalogger.Backend/Database/Queries/QueryExtensions.cs Normal file
View file

@ -0,0 +1,22 @@
using Catalogger.Backend.Database.Models;
using Catalogger.Backend.Extensions;
using Remora.Rest.Core;
namespace Catalogger.Backend.Database.Queries;
public static class QueryExtensions
{
public static async ValueTask<Guild> GetGuildAsync(this DatabaseContext db, Snowflake id,
CancellationToken ct = default) => await db.GetGuildAsync(id.ToUlong(), ct);
public static async ValueTask<Guild> GetGuildAsync(this DatabaseContext db, Optional<Snowflake> id,
CancellationToken ct = default) => await db.GetGuildAsync(id.ToUlong(), ct);
public static async ValueTask<Guild> GetGuildAsync(this DatabaseContext db, ulong id,
CancellationToken ct = default)
{
var guild = await db.Guilds.FindAsync(id);
if (guild == null) throw new Exception("oh");
return guild;
}
}

3
Catalogger.Backend/Database/QueryUtils.cs Normal file
View file

@ -0,0 +1,3 @@
using NodaTime;
namespace Catalogger.Backend.Database;

44
Catalogger.Backend/Extensions/DiscordExtensions.cs Normal file
View file

@ -0,0 +1,44 @@
using Remora.Discord.API.Abstractions.Objects;
using Remora.Rest.Core;
using Remora.Results;
namespace Catalogger.Backend.Extensions;
public static class DiscordExtensions
{
public static string Tag(this IPartialUser user)
{
var discriminator = user.Discriminator.OrDefault();
return discriminator == 0 ? user.Username.Value : $"{user.Username.Value}#{discriminator:0000}";
}
public static string AvatarUrl(this IUser user, int size = 256)
{
if (user.Avatar != null)
{
var ext = user.Avatar.HasGif ? ".gif" : ".webp";
return $"https://cdn.discordapp.com/avatars/{user.ID}/{user.Avatar.Value}{ext}?size={size}";
}
var avatarIndex = user.Discriminator == 0 ? (int)((user.ID.Value >> 22) % 6) : user.Discriminator % 5;
return $"https://cdn.discordapp.com/embed/avatars/{avatarIndex}.png?size={size}";
}
public static ulong ToUlong(this Snowflake snowflake) => snowflake.Value;
public static ulong ToUlong(this Optional<Snowflake> snowflake)
{
if (!snowflake.IsDefined()) throw new Exception("ToUlong called on an undefined Snowflake");
return snowflake.Value.Value;
}
public static T GetOrThrow<T>(this Result<T> result)
{
if (result.Error != null) throw new DiscordRestException(result.Error.Message);
return result.Entity;
}
public static async Task<T> GetOrThrow<T>(this Task<Result<T>> result) => (await result).GetOrThrow();
public class DiscordRestException(string message) : Exception(message);
}

121
Catalogger.Backend/Extensions/StartupExtensions.cs Normal file
View file

@ -0,0 +1,121 @@
using Catalogger.Backend.Bot.Responders;
using Catalogger.Backend.Cache;
using Catalogger.Backend.Database;
using Catalogger.Backend.Database.Queries;
using Catalogger.Backend.Services;
using Microsoft.EntityFrameworkCore;
using NodaTime;
using Remora.Discord.API;
using Remora.Discord.API.Abstractions.Rest;
using Remora.Discord.Commands.Services;
using Serilog;
using Serilog.Events;
namespace Catalogger.Backend.Extensions;
public static class StartupExtensions
{
/// <summary>
/// Adds Serilog to this service collection. This method also initializes Serilog, so it should be called as early as possible, before any log calls.
/// </summary>
public static WebApplicationBuilder AddSerilog(this WebApplicationBuilder builder)
{
var config = builder.Configuration.Get<Config>() ?? new();
var logCfg = new LoggerConfiguration()
.Enrich.FromLogContext()
.MinimumLevel.Is(config.Logging.LogEventLevel)
// ASP.NET's built in request logs are extremely verbose, so we use Serilog's instead.
// Serilog doesn't disable the built-in logs, so we do it here.
.MinimumLevel.Override("Microsoft", LogEventLevel.Information)
.MinimumLevel.Override("Microsoft.EntityFrameworkCore.Database.Command",
config.Logging.LogQueries ? LogEventLevel.Information : LogEventLevel.Fatal)
.MinimumLevel.Override("Microsoft.AspNetCore.Hosting", LogEventLevel.Warning)
.MinimumLevel.Override("Microsoft.AspNetCore.Mvc", LogEventLevel.Warning)
.MinimumLevel.Override("Microsoft.AspNetCore.Routing", LogEventLevel.Warning)
.WriteTo.Console();
// AddSerilog doesn't seem to add an ILogger to the service collection, so add that manually.
builder.Services.AddSerilog().AddSingleton(Log.Logger = logCfg.CreateLogger());
return builder;
}
public static Config AddConfiguration(this WebApplicationBuilder builder)
{
builder.Configuration.Sources.Clear();
builder.Configuration.AddConfiguration();
var config = builder.Configuration.Get<Config>() ?? new();
builder.Services.AddSingleton(config);
return config;
}
public static IConfigurationBuilder AddConfiguration(this IConfigurationBuilder builder)
{
var file = Environment.GetEnvironmentVariable("CATALOGGER_CONFIG_FILE") ?? "config.ini";
return builder
.SetBasePath(Directory.GetCurrentDirectory())
.AddIniFile(file, optional: false, reloadOnChange: false)
.AddEnvironmentVariables();
}
public static IServiceCollection AddCustomServices(this IServiceCollection services) => services
.AddSingleton<IClock>(SystemClock.Instance)
.AddSingleton<ChannelCacheService>()
.AddSingleton<UserCacheService>()
.AddSingleton<PluralkitApiService>()
.AddSingleton<IWebhookCache, InMemoryWebhookCache>()
.AddScoped<IEncryptionService, EncryptionService>()
.AddScoped<MessageRepository>()
.AddSingleton<WebhookExecutorService>()
.AddSingleton<PkMessageHandler>();
public static async Task Initialize(this WebApplication app)
{
await using var scope = app.Services.CreateAsyncScope();
var logger = scope.ServiceProvider.GetRequiredService<ILogger>().ForContext<Program>();
logger.Information("Starting Catalogger.NET");
await using (var db = scope.ServiceProvider.GetRequiredService<DatabaseContext>())
{
var migrationCount = (await db.Database.GetPendingMigrationsAsync()).Count();
if (migrationCount != 0)
{
logger.Information("Applying {Count} database migrations", migrationCount);
await db.Database.MigrateAsync();
}
else logger.Information("There are no pending migrations");
}
var config = scope.ServiceProvider.GetRequiredService<Config>();
var slashService = scope.ServiceProvider.GetRequiredService<SlashService>();
if (config.Discord.ApplicationId == 0)
{
logger.Warning(
"Application ID not set in config. Fetching and setting it now, but for future restarts, please add it to config.ini as Discord.ApplicationId.");
var restApi = scope.ServiceProvider.GetRequiredService<IDiscordRestApplicationAPI>();
var application = await restApi.GetCurrentApplicationAsync().GetOrThrow();
config.Discord.ApplicationId = application.ID.ToUlong();
logger.Information("Current application ID is {ApplicationId}", config.Discord.ApplicationId);
}
if (config.Discord.SyncCommands)
{
if (config.Discord.CommandsGuildId != null)
{
logger.Information("Syncing application commands with guild {GuildId}", config.Discord.CommandsGuildId);
await slashService.UpdateSlashCommandsAsync(
guildID: DiscordSnowflake.New(config.Discord.CommandsGuildId.Value));
}
else
{
logger.Information("Syncing application commands globally");
await slashService.UpdateSlashCommandsAsync();
}
}
else logger.Information("Not syncing slash commands, Discord.SyncCommands is false or unset");
}
}

1
Catalogger.Backend/GlobalUsing.cs Normal file
View file

@ -0,0 +1 @@
global using ILogger = Serilog.ILogger;

71
Catalogger.Backend/Program.cs Normal file
View file

@ -0,0 +1,71 @@
using Catalogger.Backend.Bot.Commands;
using Catalogger.Backend.Database;
using Catalogger.Backend.Extensions;
using Newtonsoft.Json.Serialization;
using Remora.Commands.Extensions;
using Remora.Discord.API.Abstractions.Gateway.Commands;
using Remora.Discord.Commands.Extensions;
using Remora.Discord.Extensions.Extensions;
using Remora.Discord.Gateway;
using Remora.Discord.Hosting.Extensions;
using Remora.Discord.Interactivity.Extensions;
using Remora.Discord.Pagination.Extensions;
using Serilog;
var builder = WebApplication.CreateBuilder(args);
var config = builder.AddConfiguration();
builder.AddSerilog();
builder.Services
.AddControllers()
.AddNewtonsoftJson(o => o.SerializerSettings.ContractResolver =
new DefaultContractResolver
{
NamingStrategy = new SnakeCaseNamingStrategy()
});
builder.Host
.AddDiscordService(_ => config.Discord.Token)
.ConfigureServices(s =>
s.AddRespondersFromAssembly(typeof(Program).Assembly)
.Configure<DiscordGatewayClientOptions>(g =>
g.Intents = GatewayIntents.Guilds |
GatewayIntents.GuildBans |
GatewayIntents.GuildInvites |
GatewayIntents.GuildMembers |
GatewayIntents.GuildMessages |
GatewayIntents.GuildWebhooks |
GatewayIntents.MessageContents |
GatewayIntents.GuildEmojisAndStickers)
.AddDiscordCommands(enableSlash: true)
.AddCommandTree()
// Start command tree
.WithCommandGroup<MetaCommands>()
// End command tree
.Finish()
.AddPagination()
.AddInteractivity()
);
builder.Services
.AddDbContext<DatabaseContext>()
.AddCustomServices()
.AddEndpointsApiExplorer()
.AddSwaggerGen();
var app = builder.Build();
await app.Initialize();
app.UseSerilogRequestLogging();
app.UseRouting();
app.UseSwagger();
app.UseSwaggerUI();
app.UseCors();
app.MapControllers();
app.Urls.Clear();
app.Urls.Add(config.Web.Address);
app.Run();
Log.CloseAndFlush();

41
Catalogger.Backend/Properties/launchSettings.json Normal file
View file

@ -0,0 +1,41 @@
{
"$schema": "http://json.schemastore.org/launchsettings.json",
"iisSettings": {
"windowsAuthentication": false,
"anonymousAuthentication": true,
"iisExpress": {
"applicationUrl": "http://localhost:35403",
"sslPort": 44334
}
},
"profiles": {
"http": {
"commandName": "Project",
"dotnetRunMessages": true,
"launchBrowser": true,
"launchUrl": "swagger",
"applicationUrl": "http://localhost:5088",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
},
"https": {
"commandName": "Project",
"dotnetRunMessages": true,
"launchBrowser": true,
"launchUrl": "swagger",
"applicationUrl": "https://localhost:7170;http://localhost:5088",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
},
"IIS Express": {
"commandName": "IISExpress",
"launchBrowser": true,
"launchUrl": "swagger",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
}
}
}

29
Catalogger.Backend/Services/IWebhookCache.cs Normal file
View file

@ -0,0 +1,29 @@
using Catalogger.Backend.Extensions;
using Remora.Discord.API;
using Remora.Discord.API.Abstractions.Objects;
using Remora.Rest.Core;
namespace Catalogger.Backend.Services;
public interface IWebhookCache
{
Task<Webhook?> GetWebhookAsync(ulong channelId);
Task SetWebhookAsync(ulong channelId, Webhook webhook);
public async Task<Webhook> GetOrFetchWebhookAsync(ulong channelId, Func<Snowflake, Task<IWebhook>> fetch)
{
var webhook = await GetWebhookAsync(channelId);
if (webhook != null) return webhook.Value;
var discordWebhook = await fetch(DiscordSnowflake.New(channelId));
webhook = new Webhook { Id = discordWebhook.ID.ToUlong(), Token = discordWebhook.Token.Value};
await SetWebhookAsync(channelId, webhook.Value);
return webhook.Value;
}
}
public struct Webhook
{
public required ulong Id { get; init; }
public required string Token { get; init; }
}

21
Catalogger.Backend/Services/InMemoryWebhookCache.cs Normal file
View file

@ -0,0 +1,21 @@
using System.Collections.Concurrent;
namespace Catalogger.Backend.Services;
public class InMemoryWebhookCache : IWebhookCache
{
private readonly ConcurrentDictionary<ulong, Webhook> _cache = new();
public Task<Webhook?> GetWebhookAsync(ulong channelId)
{
return _cache.TryGetValue(channelId, out var webhook)
? Task.FromResult<Webhook?>(webhook)
: Task.FromResult<Webhook?>(null);
}
public Task SetWebhookAsync(ulong channelId, Webhook webhook)
{
_cache[channelId] = webhook;
return Task.CompletedTask;
}
}

73
Catalogger.Backend/Services/PluralkitApiService.cs Normal file
View file

@ -0,0 +1,73 @@
using System.Net;
using System.Text.Json;
using System.Threading.RateLimiting;
using Humanizer;
using NodaTime;
using Polly;
using Remora.Rest.Json.Policies;
namespace Catalogger.Backend.Services;
public class PluralkitApiService(ILogger logger)
{
private const string UserAgent = "Catalogger.NET (https://codeberg.org/starshine/catalogger)";
private const string ApiBaseUrl = "https://api.pluralkit.me/v2";
private readonly HttpClient _client = new();
private readonly ILogger _logger = logger.ForContext<PluralkitApiService>();
private readonly ResiliencePipeline _pipeline = new ResiliencePipelineBuilder()
.AddRateLimiter(new FixedWindowRateLimiter(new FixedWindowRateLimiterOptions()
{
Window = 1.Seconds(),
PermitLimit = 2,
QueueLimit = 64,
}))
.AddTimeout(20.Seconds())
.Build();
private async Task<T?> DoRequestAsync<T>(string path, bool allowNotFound = false,
CancellationToken ct = default) where T : class
{
var req = new HttpRequestMessage(HttpMethod.Get, $"{ApiBaseUrl}{path}");
req.Headers.Add("User-Agent", UserAgent);
_logger.Debug("Requesting {Path} from PluralKit API", path);
var resp = await _client.SendAsync(req, ct);
if (resp.StatusCode == HttpStatusCode.NotFound && allowNotFound)
{
_logger.Debug("PluralKit API path {Path} returned 404 but 404 response is valid", path);
return null;
}
if (!resp.IsSuccessStatusCode)
{
_logger.Error("Received non-200 status code {StatusCode} from PluralKit API path {Path}", resp.StatusCode,
req);
throw new CataloggerError("Non-200 status code from PluralKit API");
}
return await resp.Content.ReadFromJsonAsync<T>(new JsonSerializerOptions
{ PropertyNamingPolicy = new SnakeCaseNamingPolicy() }, ct) ??
throw new CataloggerError("JSON response from PluralKit API was null");
}
public async Task<PkMessage?> GetPluralKitMessageAsync(ulong id, CancellationToken ct = default) =>
await DoRequestAsync<PkMessage>($"/messages/{id}", allowNotFound: true, ct);
public async Task<PkSystem> GetPluralKitSystemAsync(ulong id, CancellationToken ct = default) =>
(await DoRequestAsync<PkSystem>($"/systems/{id}", allowNotFound: false, ct))!;
public record PkMessage(
ulong Id,
ulong Original,
ulong Sender,
ulong Channel,
ulong Guild,
PkSystem? System,
PkMember? Member);
public record PkSystem(string Id, Guid Uuid, string? Name, string? Tag, Instant? Created);
public record PkMember(string Id, Guid Uuid, string Name, string? DisplayName);
}

186
Catalogger.Backend/Services/WebhookExecutorService.cs Normal file
View file

@ -0,0 +1,186 @@
using System.Collections.Concurrent;
using Catalogger.Backend.Cache;
using Catalogger.Backend.Database.Models;
using Catalogger.Backend.Extensions;
using Humanizer;
using Remora.Discord.API;
using Remora.Discord.API.Abstractions.Objects;
using Remora.Discord.API.Abstractions.Rest;
using Remora.Rest.Core;
namespace Catalogger.Backend.Services;
public class WebhookExecutorService(
Config config,
ILogger logger,
IWebhookCache webhookCache,
ChannelCacheService channelCache,
IDiscordRestWebhookAPI webhookApi)
{
private readonly ILogger _logger = logger.ForContext<WebhookExecutorService>();
private readonly Snowflake _applicationId = DiscordSnowflake.New(config.Discord.ApplicationId);
private readonly ConcurrentDictionary<ulong, ConcurrentQueue<IEmbed>> _cache = new();
private readonly ConcurrentDictionary<ulong, Timer> _timers = new();
private IUser? _selfUser;
public void SetSelfUser(IUser user) => _selfUser = user;
public async Task QueueLogAsync(Guild guild, LogChannelType logChannelType, IEmbed embed)
{
var logChannel = GetLogChannel(guild, logChannelType, channelId: null, userId: null);
if (logChannel == null) return;
await QueueLogAsync(logChannel.Value, embed);
}
public async Task QueueLogAsync(ulong channelId, IEmbed embed)
{
_logger.Debug("Queueing embed for channel {ChannelId}", channelId);
var webhook = await webhookCache.GetOrFetchWebhookAsync(channelId, id => FetchWebhookAsync(id));
var queue = _cache.GetOrAdd(channelId, []);
if (queue.Count >= 5)
await SendLogsAsync(channelId);
queue.Enqueue(embed);
if (_timers.TryGetValue(channelId, out var existingTimer)) await existingTimer.DisposeAsync();
_timers[channelId] = new Timer(_ =>
{
var __ = SendLogsAsync(channelId);
}, null, 3000, Timeout.Infinite);
}
private async Task SendLogsAsync(ulong channelId)
{
var queue = _cache.GetValueOrDefault(channelId);
if (queue == null) return;
var embeds = queue.Take(5).ToList();
var webhook = await webhookCache.GetOrFetchWebhookAsync(channelId, id => FetchWebhookAsync(id));
await webhookApi.ExecuteWebhookAsync(DiscordSnowflake.New(webhook.Id), webhook.Token, shouldWait: false,
embeds: embeds, username: _selfUser!.Username, avatarUrl: _selfUser.AvatarUrl());
}
private async Task<IWebhook> FetchWebhookAsync(Snowflake channelId, CancellationToken ct = default)
{
var channelWebhooks =
await webhookApi.GetChannelWebhooksAsync(channelId, ct).GetOrThrow();
var webhook = channelWebhooks.FirstOrDefault(w => w.ApplicationID == _applicationId && w.Token.IsDefined());
if (webhook != null) return webhook;
return await webhookApi.CreateWebhookAsync(channelId, "Catalogger", default, reason: "Creating logging webhook",
ct: ct).GetOrThrow();
}
public ulong? GetLogChannel(Guild guild, LogChannelType logChannelType, Snowflake? channelId = null,
ulong? userId = null)
{
if (channelId == null) return GetDefaultLogChannel(guild, logChannelType);
if (!channelCache.GetChannel(channelId.Value, out var channel)) return null;
Snowflake? categoryId;
if (channel.Type is ChannelType.AnnouncementThread or ChannelType.PrivateThread or ChannelType.PublicThread)
{
// parent_id should always have a value for threads
channelId = channel.ParentID.Value!.Value;
if (!channelCache.GetChannel(channelId.Value, out var parentChannel))
return GetDefaultLogChannel(guild, logChannelType);
categoryId = parentChannel.ParentID.Value;
}
else
{
channelId = channel.ID;
categoryId = channel.ParentID.Value;
}
// Check if the channel, or its category, or the user is ignored
if (guild.Channels.IgnoredChannels.Contains(channelId.Value.Value) ||
categoryId != null && guild.Channels.IgnoredChannels.Contains(categoryId.Value.Value)) return null;
if (userId != null)
{
if (guild.Channels.IgnoredUsers.Contains(userId.Value)) return null;
// Check the channel-local and category-local ignored users
var channelIgnoredUsers =
guild.Channels.IgnoredUsersPerChannel.GetValueOrDefault(channelId.Value.Value) ?? [];
var categoryIgnoredUsers = (categoryId != null
? guild.Channels.IgnoredUsersPerChannel.GetValueOrDefault(categoryId.Value.Value)
: []) ?? [];
if (channelIgnoredUsers.Concat(categoryIgnoredUsers).Contains(userId.Value)) return null;
}
// These three events can be redirected to other channels. Redirects can be on a channel or category level.
// Obviously, the events are only redirected if they're supposed to be logged in the first place.
if (logChannelType is LogChannelType.MessageUpdate or LogChannelType.MessageDelete
or LogChannelType.MessageDeleteBulk)
{
if (GetDefaultLogChannel(guild, logChannelType) == null) return null;
ulong categoryRedirect = categoryId != null
? guild.Channels.Redirects.GetValueOrDefault(categoryId.Value.Value)
: 0;
if (guild.Channels.Redirects.TryGetValue(channelId.Value.Value, out var channelRedirect))
return channelRedirect;
if (categoryRedirect != 0) return categoryRedirect;
return GetDefaultLogChannel(guild, logChannelType);
}
return GetDefaultLogChannel(guild, logChannelType);
}
private ulong? GetDefaultLogChannel(Guild guild, LogChannelType channelType) => channelType switch
{
LogChannelType.GuildUpdate => guild.Channels.GuildUpdate,
LogChannelType.GuildEmojisUpdate => guild.Channels.GuildEmojisUpdate,
LogChannelType.GuildRoleCreate => guild.Channels.GuildRoleCreate,
LogChannelType.GuildRoleUpdate => guild.Channels.GuildRoleUpdate,
LogChannelType.GuildRoleDelete => guild.Channels.GuildRoleDelete,
LogChannelType.ChannelCreate => guild.Channels.ChannelCreate,
LogChannelType.ChannelUpdate => guild.Channels.ChannelUpdate,
LogChannelType.ChannelDelete => guild.Channels.ChannelDelete,
LogChannelType.GuildMemberAdd => guild.Channels.GuildMemberAdd,
LogChannelType.GuildMemberUpdate => guild.Channels.GuildMemberUpdate,
LogChannelType.GuildKeyRoleUpdate => guild.Channels.GuildKeyRoleUpdate,
LogChannelType.GuildMemberNickUpdate => guild.Channels.GuildMemberNickUpdate,
LogChannelType.GuildMemberAvatarUpdate => guild.Channels.GuildMemberAvatarUpdate,
LogChannelType.GuildMemberRemove => guild.Channels.GuildMemberRemove,
LogChannelType.GuildMemberKick => guild.Channels.GuildMemberKick,
LogChannelType.GuildBanAdd => guild.Channels.GuildBanAdd,
LogChannelType.GuildBanRemove => guild.Channels.GuildBanRemove,
LogChannelType.InviteCreate => guild.Channels.InviteCreate,
LogChannelType.InviteDelete => guild.Channels.InviteDelete,
LogChannelType.MessageUpdate => guild.Channels.MessageUpdate,
LogChannelType.MessageDelete => guild.Channels.MessageDelete,
LogChannelType.MessageDeleteBulk => guild.Channels.MessageDeleteBulk,
_ => throw new ArgumentOutOfRangeException(nameof(channelType))
};
}
public enum LogChannelType
{
GuildUpdate,
GuildEmojisUpdate,
GuildRoleCreate,
GuildRoleUpdate,
GuildRoleDelete,
ChannelCreate,
ChannelUpdate,
ChannelDelete,
GuildMemberAdd,
GuildMemberUpdate,
GuildKeyRoleUpdate,
GuildMemberNickUpdate,
GuildMemberAvatarUpdate,
GuildMemberRemove,
GuildMemberKick,
GuildBanAdd,
GuildBanRemove,
InviteCreate,
InviteDelete,
MessageUpdate,
MessageDelete,
MessageDeleteBulk
}

14
Catalogger.Backend/config.example.ini Normal file
View file

@ -0,0 +1,14 @@
[Logging]
LogEventLevel = Debug
LogQueries = false
[Database]
Url = Host=localhost;Database=postgres;Username=postgres;Password=postgres
Redis = localhost:6379
EncryptionKey = changeMe!FNmZbotJnAAJ7grWHDluCoKIwj6NcUagKE= # base64 key
[Discord]
ApplicationId = <applicationIdHere>
Token = <discordTokenHere>
CommandsGuildId = <testGuildIdHere>
SyncCommands = true