feat: import messages from go version

2024-10-28 23:42:57 +01:00 · 2024-10-28 23:42:57 +01:00 · a50a8567dd
commit a50a8567dd
parent b56a71e105
15 changed files with 503 additions and 769 deletions
--- a/Catalogger.GoImporter/MessageImport.cs
+++ b/Catalogger.GoImporter/MessageImport.cs
@ -0,0 +1,114 @@
+// Copyright (C) 2021-present sam (starshines.gay)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+using System.Text.Json;
+using Catalogger.Backend;
+using Catalogger.Backend.Database;
+using Catalogger.Backend.Database.Repositories;
+using Dapper;
+using NpgsqlTypes;
+using Serilog;
+
+namespace Catalogger.GoImporter;
+
+public class MessageImport
+{
+    public static async Task DoImportAsync(Config config, DatabaseConnection conn, string dirname)
+    {
+        var encryptionService = new EncryptionService(config);
+
+        var files = Directory.GetFiles(dirname);
+        var ignoredFile = files.First(n => n[dirname.Length..] == "ignored.json");
+        var messageFiles = files.Where(n => n != ignoredFile).Order();
+
+        var ignoredMessages = await ParseFileAsync<ulong[]>(ignoredFile);
+        await using (
+            var writer = await conn.Inner.BeginBinaryImportAsync(
+                "COPY ignored_messages (id) FROM STDIN (FORMAT BINARY)"
+            )
+        )
+        {
+            foreach (var id in ignoredMessages)
+            {
+                await writer.StartRowAsync();
+                await writer.WriteAsync((long)id, NpgsqlDbType.Bigint);
+            }
+
+            await writer.CompleteAsync();
+        }
+
+        await using var tx = await conn.BeginTransactionAsync();
+
+        // Metadata isn't convertible, sadly, so just generate a dummy one.
+        var metadata = encryptionService.Encrypt(
+            JsonSerializer.Serialize(
+                new MessageRepository.Metadata(IsWebhook: false, Attachments: [])
+            )
+        );
+
+        foreach (var filename in messageFiles)
+        {
+            var messages = await ParseFileAsync<List<GoMessage>>(filename);
+
+            Log.Debug(
+                "Starting import of message file, starting with {FirstMessageId}, ending with {LastMessageId}",
+                messages.First().Id,
+                messages.Last().Id
+            );
+
+            foreach (var msg in messages)
+            {
+                await conn.ExecuteAsync(
+                    """
+                    insert into messages (id, original_id, user_id, channel_id, guild_id, member, system, username, content, metadata, attachment_size)
+                    values (@Id, @OriginalId, @UserId, @ChannelId, @GuildId, @Member, @System, @Username, @Content, @Metadata, 0)
+                    on conflict do nothing
+                    """,
+                    new
+                    {
+                        msg.Id,
+                        OriginalId = (ulong?)(msg.Member != null ? msg.Id : null),
+                        msg.UserId,
+                        msg.ChannelId,
+                        msg.GuildId,
+                        msg.Member,
+                        msg.System,
+                        Content = await Task.Run(() => encryptionService.Encrypt(msg.Content)),
+                        Username = await Task.Run(() => encryptionService.Encrypt(msg.Username)),
+                        Metadata = metadata,
+                    },
+                    tx
+                );
+            }
+
+            Log.Debug(
+                "Finished importing message file ending with {LastMessageId}",
+                messages.Last().Id
+            );
+        }
+
+        await tx.CommitAsync();
+
+        Log.Debug("Finished importing files!");
+    }
+
+    private static async Task<T> ParseFileAsync<T>(string filename)
+    {
+        var rawData = await File.OpenText(filename).ReadToEndAsync();
+        return JsonSerializer.Deserialize<T>(rawData, Program.JsonOptions)
+            ?? throw new CataloggerError("Message file deserialized as null");
+        ;
+    }
+}