// Copyright (C) 2021-present sam (starshines.gay) // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published // by the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . using System.Text.Json; using Catalogger.Backend; using Catalogger.Backend.Database; using Catalogger.Backend.Database.Repositories; using Dapper; using NpgsqlTypes; using Serilog; namespace Catalogger.GoImporter; public class MessageImport { public static async Task DoImportAsync(Config config, DatabaseConnection conn, string dirname) { var encryptionService = new EncryptionService(config); var files = Directory.GetFiles(dirname); var ignoredFile = files.First(n => n[dirname.Length..] == "ignored.json"); var messageFiles = files.Where(n => n != ignoredFile).Order(); var ignoredMessages = await ParseFileAsync(ignoredFile); await using ( var writer = await conn.Inner.BeginBinaryImportAsync( "COPY ignored_messages (id) FROM STDIN (FORMAT BINARY)" ) ) { foreach (var id in ignoredMessages) { await writer.StartRowAsync(); await writer.WriteAsync((long)id, NpgsqlDbType.Bigint); } await writer.CompleteAsync(); } await using var tx = await conn.BeginTransactionAsync(); // Metadata isn't convertible, sadly, so just generate a dummy one. var metadata = encryptionService.Encrypt( JsonSerializer.Serialize( new MessageRepository.Metadata(IsWebhook: false, Attachments: []) ) ); foreach (var filename in messageFiles) { var messages = await ParseFileAsync>(filename); Log.Debug( "Starting import of message file, starting with {FirstMessageId}, ending with {LastMessageId}", messages.First().Id, messages.Last().Id ); foreach (var msg in messages) { await conn.ExecuteAsync( """ insert into messages (id, original_id, user_id, channel_id, guild_id, member, system, username, content, metadata, attachment_size) values (@Id, @OriginalId, @UserId, @ChannelId, @GuildId, @Member, @System, @Username, @Content, @Metadata, 0) on conflict do nothing """, new { msg.Id, OriginalId = (ulong?)(msg.Member != null ? msg.Id : null), msg.UserId, msg.ChannelId, msg.GuildId, msg.Member, msg.System, Content = await Task.Run(() => encryptionService.Encrypt(msg.Content)), Username = await Task.Run(() => encryptionService.Encrypt(msg.Username)), Metadata = metadata, }, tx ); } Log.Debug( "Finished importing message file ending with {LastMessageId}", messages.Last().Id ); } await tx.CommitAsync(); Log.Debug("Finished importing files!"); } private static async Task ParseFileAsync(string filename) { var rawData = await File.OpenText(filename).ReadToEndAsync(); return JsonSerializer.Deserialize(rawData, Program.JsonOptions) ?? throw new CataloggerError("Message file deserialized as null"); ; } }