sam/Catalogger.NET
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: import messages from go version

Browse source
This commit is contained in:
sam 2024-10-28 23:42:57 +01:00
parent b56a71e105
commit a50a8567dd
Signed by: sam
GPG key ID: 5F3C3C1B3166639D
15 changed files with 503 additions and 769 deletions
Download patch file Download diff file Expand all files Collapse all files

94
Catalogger.GoImporter/GuildImport.cs
View file

@ -13,10 +13,12 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
using System.Data.Common;
using System.Diagnostics;
using System.Text.Json;
using Catalogger.Backend.Database;
using Catalogger.Backend.Database.Models;
using Dapper;
using NodaTime.Extensions;
using Serilog;
@ -24,7 +26,7 @@ namespace Catalogger.GoImporter;
public static class GuildImport
{
public static async Task DoImportAsync(DatabaseContext db, string filename)
public static async Task DoImportAsync(DatabaseConnection conn, string filename)
{
var rawData = await File.OpenText(filename).ReadToEndAsync();
var data = JsonSerializer.Deserialize<GuildsExport>(rawData, Program.JsonOptions);
@ -37,16 +39,15 @@ public static class GuildImport
var watch = new Stopwatch();
watch.Start();
await using var tx = await db.Database.BeginTransactionAsync();
await using var tx = await conn.BeginTransactionAsync();
foreach (var g in data.Guilds)
ImportGuild(db, g);
await ImportGuildAsync(conn, tx, g);
foreach (var i in data.Invites)
ImportInvite(db, i);
await ImportInviteAsync(conn, tx, i);
foreach (var w in data.Watchlist)
ImportWatchlistEntry(db, w);
await ImportWatchlistEntryAsync(conn, tx, w);
await db.SaveChangesAsync();
await tx.CommitAsync();
Log.Information(
@ -55,7 +56,11 @@ public static class GuildImport
);
}
private static void ImportGuild(DatabaseContext db, GoGuild guild)
private static async Task ImportGuildAsync(
DatabaseConnection conn,
DbTransaction tx,
GoGuild guild
)
{
var channels = new Guild.ChannelConfig
{
@ -90,38 +95,61 @@ public static class GuildImport
if (ulong.TryParse(key, out var fromId) && ulong.TryParse(value, out var toId))
channels.Redirects[fromId] = toId;
var dbGuild = new Guild
{
Id = guild.Id,
BannedSystems = guild.BannedSystems,
KeyRoles = guild.KeyRoles,
Channels = channels,
};
db.Guilds.Add(dbGuild);
await conn.ExecuteAsync(
"""
insert into guilds (id, channels, banned_systems, key_roles)
values (@Id, @Channels::jsonb, @BannedSystems, @KeyRoles)
""",
new
{
guild.Id,
Channels = channels,
guild.BannedSystems,
guild.KeyRoles,
},
tx
);
}
private static void ImportInvite(DatabaseContext db, GoInvite invite)
private static async Task ImportInviteAsync(
DatabaseConnection conn,
DbTransaction tx,
GoInvite invite
)
{
var dbInvite = new Invite
{
GuildId = invite.GuildId,
Code = invite.Code,
Name = invite.Name,
};
db.Invites.Add(dbInvite);
await conn.ExecuteAsync(
"insert into invites (code, guild_id, name) values (@Code, @GuildId, @Name)",
new
{
invite.GuildId,
invite.Code,
invite.Name,
},
tx
);
}
private static void ImportWatchlistEntry(DatabaseContext db, GoWatchlistEntry watchlistEntry)
private static async Task ImportWatchlistEntryAsync(
DatabaseConnection conn,
DbTransaction tx,
GoWatchlistEntry watchlistEntry
)
{
var dbWatchlist = new Watchlist
{
GuildId = watchlistEntry.GuildId,
UserId = watchlistEntry.UserId,
ModeratorId = watchlistEntry.ModeratorId,
AddedAt = watchlistEntry.AddedAt,
Reason = watchlistEntry.Reason,
};
await conn.ExecuteAsync(
"""
insert into watchlists (guild_id, user_id, added_at, moderator_id, reason)
values (@GuildId, @UserId, @AddedAt, @ModeratorId, @Reason)
""",
new
{
watchlistEntry.GuildId,
watchlistEntry.UserId,
watchlistEntry.AddedAt,
watchlistEntry.ModeratorId,
watchlistEntry.Reason,
},
tx
);
}
private static ulong TryParse(this Dictionary<string, string> dict, string key) =>

114
Catalogger.GoImporter/MessageImport.cs Normal file
View file

@ -0,0 +1,114 @@
// Copyright (C) 2021-present sam (starshines.gay)
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
using System.Text.Json;
using Catalogger.Backend;
using Catalogger.Backend.Database;
using Catalogger.Backend.Database.Repositories;
using Dapper;
using NpgsqlTypes;
using Serilog;
namespace Catalogger.GoImporter;
public class MessageImport
{
public static async Task DoImportAsync(Config config, DatabaseConnection conn, string dirname)
{
var encryptionService = new EncryptionService(config);
var files = Directory.GetFiles(dirname);
var ignoredFile = files.First(n => n[dirname.Length..] == "ignored.json");
var messageFiles = files.Where(n => n != ignoredFile).Order();
var ignoredMessages = await ParseFileAsync<ulong[]>(ignoredFile);
await using (
var writer = await conn.Inner.BeginBinaryImportAsync(
"COPY ignored_messages (id) FROM STDIN (FORMAT BINARY)"
)
)
{
foreach (var id in ignoredMessages)
{
await writer.StartRowAsync();
await writer.WriteAsync((long)id, NpgsqlDbType.Bigint);
}
await writer.CompleteAsync();
}
await using var tx = await conn.BeginTransactionAsync();
// Metadata isn't convertible, sadly, so just generate a dummy one.
var metadata = encryptionService.Encrypt(
JsonSerializer.Serialize(
new MessageRepository.Metadata(IsWebhook: false, Attachments: [])
)
);
foreach (var filename in messageFiles)
{
var messages = await ParseFileAsync<List<GoMessage>>(filename);
Log.Debug(
"Starting import of message file, starting with {FirstMessageId}, ending with {LastMessageId}",
messages.First().Id,
messages.Last().Id
);
foreach (var msg in messages)
{
await conn.ExecuteAsync(
"""
insert into messages (id, original_id, user_id, channel_id, guild_id, member, system, username, content, metadata, attachment_size)
values (@Id, @OriginalId, @UserId, @ChannelId, @GuildId, @Member, @System, @Username, @Content, @Metadata, 0)
on conflict do nothing
""",
new
{
msg.Id,
OriginalId = (ulong?)(msg.Member != null ? msg.Id : null),
msg.UserId,
msg.ChannelId,
msg.GuildId,
msg.Member,
msg.System,
Content = await Task.Run(() => encryptionService.Encrypt(msg.Content)),
Username = await Task.Run(() => encryptionService.Encrypt(msg.Username)),
Metadata = metadata,
},
tx
);
}
Log.Debug(
"Finished importing message file ending with {LastMessageId}",
messages.Last().Id
);
}
await tx.CommitAsync();
Log.Debug("Finished importing files!");
}
private static async Task<T> ParseFileAsync<T>(string filename)
{
var rawData = await File.OpenText(filename).ReadToEndAsync();
return JsonSerializer.Deserialize<T>(rawData, Program.JsonOptions)
?? throw new CataloggerError("Message file deserialized as null");
;
}
}

20
Catalogger.GoImporter/Models.cs
View file

@ -14,6 +14,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
using NodaTime;
using Remora.Discord.API.Objects;
using J = System.Text.Json.Serialization.JsonPropertyNameAttribute;
// ReSharper disable NotAccessedPositionalProperty.Global
@ -45,3 +46,22 @@ public record GuildsExport(
List<GoInvite> Invites,
List<GoWatchlistEntry> Watchlist
);
public record GoMessage(
[property: J("MsgID")] ulong Id,
[property: J("UserID")] ulong UserId,
[property: J("ChannelID")] ulong ChannelId,
[property: J("ServerID")] ulong GuildId,
string Content,
string Username,
string? Member,
string? System,
GoMetadata? Metadata
);
public record GoMetadata(
[property: J("UserID")] ulong? UserId,
string? Username,
string? Avatar,
EmbedField[]? Embeds
);

39
Catalogger.GoImporter/Program.cs
View file

@ -1,10 +1,10 @@
// See https://aka.ms/new-console-template for more information
using System.Text.Json;
using System.Text.Json.Serialization;
using Catalogger.Backend;
using Catalogger.Backend.Database;
using Catalogger.Backend.Extensions;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using NodaTime;
using NodaTime.Serialization.SystemTextJson;
@ -17,13 +17,15 @@ namespace Catalogger.GoImporter;
internal class Program
{
public static JsonSerializerOptions JsonOptions =
new JsonSerializerOptions().ConfigureForNodaTime(
new NodaJsonSettings
{
InstantConverter = new NodaPatternConverter<Instant>(InstantPattern.ExtendedIso),
}
);
public static readonly JsonSerializerOptions JsonOptions = new JsonSerializerOptions
{
NumberHandling = JsonNumberHandling.AllowReadingFromString,
}.ConfigureForNodaTime(
new NodaJsonSettings
{
InstantConverter = new NodaPatternConverter<Instant>(InstantPattern.ExtendedIso),
}
);
public static async Task Main(string[] args)
{
@ -41,8 +43,18 @@ internal class Program
return;
}
var db = new DatabaseContext(config, null);
await db.Database.MigrateAsync();
var db = new DatabasePool(config, Log.Logger, null);
DatabasePool.ConfigureDapper();
if (Environment.GetEnvironmentVariable("MIGRATE") == "true")
{
var migrator = new DatabaseMigrator(
Log.Logger,
SystemClock.Instance,
await db.AcquireAsync()
);
await migrator.MigrateUp();
}
var type = args[0].ToLowerInvariant();
var file = args[1];
@ -50,7 +62,12 @@ internal class Program
if (type == "guilds")
{
Log.Information("Importing guilds from {File}", file);
await GuildImport.DoImportAsync(db, file);
await GuildImport.DoImportAsync(await db.AcquireAsync(), file);
}
else if (type == "messages")
{
Log.Information("Importing messages from {Path}", file);
await MessageImport.DoImportAsync(config, await db.AcquireAsync(), file);
}
}
}