Catalogger.NET/Catalogger.GoImporter/MessageImport.cs

// Copyright (C) 2021-present sam (starshines.gay)
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

using System.Text.Json;
using Catalogger.Backend;
using Catalogger.Backend.Database;
using Catalogger.Backend.Database.Repositories;
using Dapper;
using NpgsqlTypes;
using Serilog;

namespace Catalogger.GoImporter;

public class MessageImport
{
    public static async Task DoImportAsync(Config config, DatabaseConnection conn, string dirname)
    {
        var encryptionService = new EncryptionService(config);

        var files = Directory.GetFiles(dirname);
        var ignoredFile = files.First(n => n[dirname.Length..] == "ignored.json");
        var messageFiles = files.Where(n => n != ignoredFile).Order();

        var ignoredMessages = await ParseFileAsync<ulong[]>(ignoredFile);
        await using (
            var writer = await conn.Inner.BeginBinaryImportAsync(
                "COPY ignored_messages (id) FROM STDIN (FORMAT BINARY)"
            )
        )
        {
            foreach (var id in ignoredMessages)
            {
                await writer.StartRowAsync();
                await writer.WriteAsync((long)id, NpgsqlDbType.Bigint);
            }

            await writer.CompleteAsync();
        }

        await using var tx = await conn.BeginTransactionAsync();

        // Metadata isn't convertible, sadly, so just generate a dummy one.
        var metadata = encryptionService.Encrypt(
            JsonSerializer.Serialize(
                new MessageRepository.Metadata(IsWebhook: false, Attachments: [])
            )
        );

        foreach (var filename in messageFiles)
        {
            var messages = await ParseFileAsync<List<GoMessage>>(filename);

            Log.Debug(
                "Starting import of message file, starting with {FirstMessageId}, ending with {LastMessageId}",
                messages.First().Id,
                messages.Last().Id
            );

            foreach (var msg in messages)
            {
                await conn.ExecuteAsync(
                    """
                    insert into messages (id, original_id, user_id, channel_id, guild_id, member, system, username, content, metadata, attachment_size)
                    values (@Id, @OriginalId, @UserId, @ChannelId, @GuildId, @Member, @System, @Username, @Content, @Metadata, 0)
                    on conflict do nothing
                    """,
                    new
                    {
                        msg.Id,
                        OriginalId = (ulong?)(msg.Member != null ? msg.Id : null),
                        msg.UserId,
                        msg.ChannelId,
                        msg.GuildId,
                        msg.Member,
                        msg.System,
                        Content = await Task.Run(() => encryptionService.Encrypt(msg.Content)),
                        Username = await Task.Run(() => encryptionService.Encrypt(msg.Username)),
                        Metadata = metadata,
                    },
                    tx
                );
            }

            Log.Debug(
                "Finished importing message file ending with {LastMessageId}",
                messages.Last().Id
            );
        }

        await tx.CommitAsync();

        Log.Debug("Finished importing files!");
    }

    private static async Task<T> ParseFileAsync<T>(string filename)
    {
        var rawData = await File.OpenText(filename).ReadToEndAsync();
        return JsonSerializer.Deserialize<T>(rawData, Program.JsonOptions)
            ?? throw new CataloggerError("Message file deserialized as null");
        ;
    }
}