114 lines
4.2 KiB
C#
114 lines
4.2 KiB
C#
// Copyright (C) 2021-present sam (starshines.gay)
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Affero General Public License as published
|
|
// by the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Affero General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
|
|
using System.Text.Json;
|
|
using Catalogger.Backend;
|
|
using Catalogger.Backend.Database;
|
|
using Catalogger.Backend.Database.Repositories;
|
|
using Dapper;
|
|
using NpgsqlTypes;
|
|
using Serilog;
|
|
|
|
namespace Catalogger.GoImporter;
|
|
|
|
public class MessageImport
|
|
{
|
|
public static async Task DoImportAsync(Config config, DatabaseConnection conn, string dirname)
|
|
{
|
|
var encryptionService = new EncryptionService(config);
|
|
|
|
var files = Directory.GetFiles(dirname);
|
|
var ignoredFile = files.First(n => n[dirname.Length..] == "ignored.json");
|
|
var messageFiles = files.Where(n => n != ignoredFile).Order();
|
|
|
|
var ignoredMessages = await ParseFileAsync<ulong[]>(ignoredFile);
|
|
await using (
|
|
var writer = await conn.Inner.BeginBinaryImportAsync(
|
|
"COPY ignored_messages (id) FROM STDIN (FORMAT BINARY)"
|
|
)
|
|
)
|
|
{
|
|
foreach (var id in ignoredMessages)
|
|
{
|
|
await writer.StartRowAsync();
|
|
await writer.WriteAsync((long)id, NpgsqlDbType.Bigint);
|
|
}
|
|
|
|
await writer.CompleteAsync();
|
|
}
|
|
|
|
await using var tx = await conn.BeginTransactionAsync();
|
|
|
|
// Metadata isn't convertible, sadly, so just generate a dummy one.
|
|
var metadata = encryptionService.Encrypt(
|
|
JsonSerializer.Serialize(
|
|
new MessageRepository.Metadata(IsWebhook: false, Attachments: [])
|
|
)
|
|
);
|
|
|
|
foreach (var filename in messageFiles)
|
|
{
|
|
var messages = await ParseFileAsync<List<GoMessage>>(filename);
|
|
|
|
Log.Debug(
|
|
"Starting import of message file, starting with {FirstMessageId}, ending with {LastMessageId}",
|
|
messages.First().Id,
|
|
messages.Last().Id
|
|
);
|
|
|
|
foreach (var msg in messages)
|
|
{
|
|
await conn.ExecuteAsync(
|
|
"""
|
|
insert into messages (id, original_id, user_id, channel_id, guild_id, member, system, username, content, metadata, attachment_size)
|
|
values (@Id, @OriginalId, @UserId, @ChannelId, @GuildId, @Member, @System, @Username, @Content, @Metadata, 0)
|
|
on conflict do nothing
|
|
""",
|
|
new
|
|
{
|
|
msg.Id,
|
|
OriginalId = (ulong?)(msg.Member != null ? msg.Id : null),
|
|
msg.UserId,
|
|
msg.ChannelId,
|
|
msg.GuildId,
|
|
msg.Member,
|
|
msg.System,
|
|
Content = await Task.Run(() => encryptionService.Encrypt(msg.Content)),
|
|
Username = await Task.Run(() => encryptionService.Encrypt(msg.Username)),
|
|
Metadata = metadata,
|
|
},
|
|
tx
|
|
);
|
|
}
|
|
|
|
Log.Debug(
|
|
"Finished importing message file ending with {LastMessageId}",
|
|
messages.Last().Id
|
|
);
|
|
}
|
|
|
|
await tx.CommitAsync();
|
|
|
|
Log.Debug("Finished importing files!");
|
|
}
|
|
|
|
private static async Task<T> ParseFileAsync<T>(string filename)
|
|
{
|
|
var rawData = await File.OpenText(filename).ReadToEndAsync();
|
|
return JsonSerializer.Deserialize<T>(rawData, Program.JsonOptions)
|
|
?? throw new CataloggerError("Message file deserialized as null");
|
|
;
|
|
}
|
|
}
|