feat: new migrator

This commit is contained in:
sam 2024-12-16 21:38:38 +01:00
parent b36b54f9e6
commit 79b8c4799e
Signed by: sam
GPG key ID: B4EF20DDE721CAA1
17 changed files with 621 additions and 917 deletions

View file

@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Foxnouns.Backend\Foxnouns.Backend.csproj"/>
</ItemGroup>
<ItemGroup>
<PackageReference Include="Dapper" Version="2.1.35"/>
<PackageReference Include="Npgsql" Version="9.0.2"/>
<PackageReference Include="Npgsql.NodaTime" Version="9.0.2"/>
</ItemGroup>
</Project>

View file

@ -0,0 +1,71 @@
using System.Data;
using Dapper;
using Foxnouns.DataMigrator.Models;
using Newtonsoft.Json;
using Npgsql;
namespace Foxnouns.DataMigrator;
public static class GoDatabase
{
private static NpgsqlDataSource? _dataSource;
public static async Task<NpgsqlConnection> GetConnectionAsync()
{
if (_dataSource != null)
return await _dataSource.OpenConnectionAsync();
DefaultTypeMap.MatchNamesWithUnderscores = true;
SqlMapper.RemoveTypeMap(typeof(ulong));
SqlMapper.AddTypeHandler(new UlongEncodeAsLongHandler());
SqlMapper.AddTypeHandler(new JsonTypeHandler<GoFieldEntry[]>());
SqlMapper.AddTypeHandler(new JsonTypeHandler<Dictionary<string, GoCustomPreference>>());
SqlMapper.AddTypeHandler(new JsonTypeHandler<GoPronounEntry[]>());
SqlMapper.AddTypeHandler(new UlongListHandler());
string dsn =
Environment.GetEnvironmentVariable("GO_DATABASE")
?? throw new Exception("$GO_DATABASE is not set");
var dataSourceBuilder = new NpgsqlDataSourceBuilder(dsn);
dataSourceBuilder.UseJsonNet();
_dataSource = dataSourceBuilder.Build();
return await _dataSource.OpenConnectionAsync();
}
// dapper why
// taken from https://codeberg.org/starshine/catalogger/src/branch/main/Catalogger.Backend/Database/DatabasePool.cs
private class UlongEncodeAsLongHandler : SqlMapper.TypeHandler<ulong>
{
public override void SetValue(IDbDataParameter parameter, ulong value) =>
parameter.Value = (long)value;
public override ulong Parse(object value) =>
// Cast to long to unbox, then to ulong (???)
(ulong)(long)value;
}
private class UlongListHandler : SqlMapper.TypeHandler<List<ulong>>
{
public override void SetValue(IDbDataParameter parameter, List<ulong>? value) =>
parameter.Value = value?.Select(i => (long)i).ToArray();
public override List<ulong>? Parse(object value) =>
((long[])value).Select(i => (ulong)i).ToList();
}
private class JsonTypeHandler<T> : SqlMapper.TypeHandler<T>
{
public override void SetValue(IDbDataParameter parameter, T? value) =>
parameter.Value = JsonConvert.SerializeObject(value);
public override T? Parse(object value)
{
var json = (string)value;
return JsonConvert.DeserializeObject<T>(json) ?? default;
}
}
}

View file

@ -0,0 +1,26 @@
using Foxnouns.Backend.Database;
namespace Foxnouns.DataMigrator.Models;
public class GoMember
{
public required string Id { get; init; }
public required string Name { get; init; }
public string? Bio { get; init; }
public string[]? Links { get; init; }
public string? DisplayName { get; init; }
public GoFieldEntry[] Names { get; init; } = [];
public GoPronounEntry[] Pronouns { get; init; } = [];
public string? Avatar { get; init; }
public required bool Unlisted { get; init; }
public required string Sid { get; init; }
public required Snowflake SnowflakeId { get; init; }
}
public class GoMemberField
{
public required string MemberId { get; init; }
public required long Id { get; init; }
public required string Name { get; init; }
public required GoFieldEntry[] Entries { get; init; }
}

View file

@ -0,0 +1,102 @@
using Foxnouns.Backend.Database;
using Foxnouns.Backend.Database.Models;
namespace Foxnouns.DataMigrator.Models;
public class GoUser
{
public required string Id { get; init; }
public required string Username { get; init; }
public string? DisplayName { get; init; }
public string? Bio { get; init; }
public string[]? Links { get; init; }
public string? Discord { get; init; }
public string? DiscordUsername { get; init; }
public DateTimeOffset? DeletedAt { get; init; }
public bool? SelfDelete { get; init; }
public string? DeleteReason { get; init; }
public GoFieldEntry[] Names { get; init; } = [];
public GoPronounEntry[] Pronouns { get; init; } = [];
public string? Avatar { get; init; }
public string? Fediverse { get; init; }
public string? FediverseUsername { get; init; }
public int? FediverseAppId { get; init; }
public bool IsAdmin { get; init; }
public string? MemberTitle { get; init; }
public bool ListPrivate { get; init; }
public string? Tumblr { get; init; }
public string? TumblrUsername { get; init; }
public string? Google { get; init; }
public string? GoogleUsername { get; init; }
public Dictionary<string, GoCustomPreference> CustomPreferences { get; init; } = [];
public DateTimeOffset LastActive { get; init; }
public required string Sid { get; init; }
public DateTimeOffset LastSidReroll { get; init; }
public string? Timezone { get; init; }
public Snowflake SnowflakeId { get; init; }
}
public class GoUserField
{
public required string UserId { get; init; }
public required long Id { get; init; }
public required string Name { get; init; }
public required GoFieldEntry[] Entries { get; init; }
}
public class GoPrideFlag
{
public required string Id { get; init; }
public required string UserId { get; init; }
public required string Hash { get; init; }
public required string Name { get; init; }
public string? Description { get; init; }
public required Snowflake SnowflakeId { get; init; }
}
public class GoProfileFlag
{
public string? UserId { get; init; }
public string? MemberId { get; init; }
public required long Id { get; init; }
public required string FlagId { get; init; }
}
public class GoFediverseApp
{
public required int Id { get; init; }
public required string Instance { get; init; }
public required string ClientId { get; init; }
public required string ClientSecret { get; init; }
public required string InstanceType { get; init; }
public FediverseInstanceType TypeToEnum() =>
InstanceType switch
{
"sharkey" => FediverseInstanceType.MisskeyApi,
"firefish" => FediverseInstanceType.MisskeyApi,
"pixelfed" => FediverseInstanceType.MastodonApi,
"mastodon" => FediverseInstanceType.MastodonApi,
"pleroma" => FediverseInstanceType.MastodonApi,
"akkoma" => FediverseInstanceType.MastodonApi,
"misskey" => FediverseInstanceType.MastodonApi,
"gotosocial" => FediverseInstanceType.MastodonApi,
"calckey" => FediverseInstanceType.MisskeyApi,
"foundkey" => FediverseInstanceType.MisskeyApi,
// this should never happen but if it does we just fall back to the mastodon api
// basically everything but misskey forks implement it anyway :3
_ => FediverseInstanceType.MastodonApi,
};
}
public record GoFieldEntry(string Value, string Status);
public record GoPronounEntry(string Pronouns, string? DisplayText, string Status);
public record GoCustomPreference(
string Icon,
string Tooltip,
string PreferenceSize,
bool Muted,
bool Favourite
);

View file

@ -0,0 +1,101 @@
using System.Globalization;
using Foxnouns.Backend;
using Foxnouns.Backend.Database;
using Foxnouns.Backend.Database.Models;
using Foxnouns.Backend.Extensions;
using Foxnouns.DataMigrator.Models;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Npgsql;
using Serilog;
using Serilog.Sinks.SystemConsole.Themes;
namespace Foxnouns.DataMigrator;
internal class Program
{
public static async Task Main(string[] args)
{
// Create logger and get configuration
Log.Logger = new LoggerConfiguration()
.MinimumLevel.Debug()
.WriteTo.Console(theme: AnsiConsoleTheme.Sixteen)
.CreateLogger();
Config config =
new ConfigurationBuilder()
.AddConfiguration()
.Build()
// Get the configuration as our config class
.Get<Config>() ?? new Config();
NpgsqlConnection conn = await GoDatabase.GetConnectionAsync();
// just reuse the design time factory so we don't have to copy this
DatabaseContext context = new DesignTimeDatabaseContextFactory().CreateDbContext(args);
await context.Database.MigrateAsync();
Log.Information("Migrating applications");
Dictionary<int, Snowflake> appIds = await MigrateAppsAsync(conn, context);
Log.Information("Migrating users");
List<GoUser> users = await Queries.GetUsersAsync(conn);
List<GoUserField> userFields = await Queries.GetUserFieldsAsync(conn);
List<GoMemberField> memberFields = await Queries.GetMemberFieldsAsync(conn);
List<GoPrideFlag> prideFlags = await Queries.GetUserFlagsAsync(conn);
List<GoProfileFlag> userFlags = await Queries.GetUserProfileFlagsAsync(conn);
List<GoProfileFlag> memberFlags = await Queries.GetMemberProfileFlagsAsync(conn);
Log.Information("Migrating {Count} users", users.Count);
foreach ((GoUser user, int i) in users.Select((user, i) => (user, i)))
{
Log.Debug(
"Migrating user #{Index}/{Count}: {Id}/{SnowflakeId}",
i,
users.Count,
user.Id,
user.SnowflakeId
);
await new UserMigrator(
conn,
context,
user,
appIds,
userFields,
memberFields,
prideFlags,
userFlags,
memberFlags
).MigrateAsync();
}
await context.SaveChangesAsync();
Log.Information("Migration complete!");
}
private static async Task<Dictionary<int, Snowflake>> MigrateAppsAsync(
NpgsqlConnection conn,
DatabaseContext context
)
{
List<GoFediverseApp> goApps = await Queries.GetFediverseAppsAsync(conn);
var appIds = new Dictionary<int, Snowflake>();
foreach (GoFediverseApp app in goApps)
{
Log.Debug("Migrating application for {Domain}", app.Instance);
Snowflake id = SnowflakeGenerator.Instance.GenerateSnowflake();
appIds[app.Id] = id;
context.FediverseApplications.Add(
new FediverseApplication
{
Id = id,
Domain = app.Instance.ToLower(CultureInfo.InvariantCulture),
ClientId = app.ClientId,
ClientSecret = app.ClientSecret,
InstanceType = app.TypeToEnum(),
}
);
}
return appIds;
}
}

View file

@ -0,0 +1,34 @@
using Coravel.Mailer.Mail.Helpers;
using Dapper;
using Foxnouns.Backend.Database;
using Foxnouns.Backend.Database.Models;
using Foxnouns.DataMigrator.Models;
using NodaTime.Extensions;
using Npgsql;
namespace Foxnouns.DataMigrator;
public static class Queries
{
public static async Task<List<GoFediverseApp>> GetFediverseAppsAsync(NpgsqlConnection conn) =>
(await conn.QueryAsync<GoFediverseApp>("select * from fediverse_apps")).ToList();
public static async Task<List<GoUser>> GetUsersAsync(NpgsqlConnection conn) =>
(await conn.QueryAsync<GoUser>("select * from users order by id")).ToList();
public static async Task<List<GoUserField>> GetUserFieldsAsync(NpgsqlConnection conn) =>
(await conn.QueryAsync<GoUserField>("select * from user_fields order by id")).ToList();
public static async Task<List<GoMemberField>> GetMemberFieldsAsync(NpgsqlConnection conn) =>
(await conn.QueryAsync<GoMemberField>("select * from member_fields order by id")).ToList();
public static async Task<List<GoProfileFlag>> GetUserProfileFlagsAsync(NpgsqlConnection conn) =>
(await conn.QueryAsync<GoProfileFlag>("select * from user_flags order by id")).ToList();
public static async Task<List<GoProfileFlag>> GetMemberProfileFlagsAsync(
NpgsqlConnection conn
) => (await conn.QueryAsync<GoProfileFlag>("select * from member_flags order by id")).ToList();
public static async Task<List<GoPrideFlag>> GetUserFlagsAsync(NpgsqlConnection conn) =>
(await conn.QueryAsync<GoPrideFlag>("select * from pride_flags order by id")).ToList();
}

View file

@ -0,0 +1,261 @@
using System.Diagnostics.CodeAnalysis;
using Dapper;
using Foxnouns.Backend.Database;
using Foxnouns.Backend.Database.Models;
using Foxnouns.Backend.Utils;
using Foxnouns.DataMigrator.Models;
using NodaTime.Extensions;
using Npgsql;
using Serilog;
namespace Foxnouns.DataMigrator;
public class UserMigrator(
NpgsqlConnection conn,
DatabaseContext context,
GoUser goUser,
Dictionary<int, Snowflake> fediverseApplicationIds,
List<GoUserField> userFields,
List<GoMemberField> memberFields,
List<GoPrideFlag> prideFlags,
List<GoProfileFlag> userFlags,
List<GoProfileFlag> memberFlags
)
{
private readonly Dictionary<string, Snowflake> _preferenceIds = new();
private readonly Dictionary<string, Snowflake> _flagIds = new();
private User? _user;
public async Task MigrateAsync()
{
CreateNewUser();
MigrateFlags();
await MigrateMembersAsync();
}
[MemberNotNull(nameof(_user))]
private void CreateNewUser()
{
_user = new User
{
Id = goUser.SnowflakeId,
Username = goUser.Username,
DisplayName = goUser.DisplayName,
Bio = goUser.Bio,
Links = goUser.Links ?? [],
Deleted = goUser.DeletedAt != null,
DeletedAt = goUser.DeletedAt?.ToInstant(),
DeletedBy = goUser.SelfDelete == true ? null : goUser.SnowflakeId,
Names = goUser.Names.Select(ConvertFieldEntry).ToList(),
Pronouns = goUser.Pronouns.Select(ConvertPronoun).ToList(),
Avatar = goUser.Avatar,
Role = goUser.IsAdmin ? UserRole.Admin : UserRole.User,
MemberTitle = goUser.MemberTitle,
ListHidden = goUser.ListPrivate,
CustomPreferences = ConvertPreferences(),
LastActive = goUser.LastActive.ToInstant(),
Sid = goUser.Sid,
LastSidReroll = goUser.LastSidReroll.ToInstant(),
Timezone = goUser.Timezone,
Fields = userFields
.Where(f => f.UserId == goUser.Id)
.Select(f => new Field
{
Name = f.Name,
Entries = f.Entries.Select(ConvertFieldEntry).ToArray(),
})
.ToList(),
};
context.Users.Add(_user);
// Create the user's auth methods
if (goUser.Discord != null)
{
context.AuthMethods.Add(
new AuthMethod
{
Id = SnowflakeGenerator.Instance.GenerateSnowflake(_user.Id.Time),
UserId = _user.Id,
AuthType = AuthType.Discord,
RemoteId = goUser.Discord,
RemoteUsername = goUser.DiscordUsername ?? "(unknown)",
}
);
}
if (goUser.Google != null)
{
context.AuthMethods.Add(
new AuthMethod
{
Id = SnowflakeGenerator.Instance.GenerateSnowflake(_user.Id.Time),
UserId = _user.Id,
AuthType = AuthType.Google,
RemoteId = goUser.Google,
RemoteUsername = goUser.GoogleUsername ?? "(unknown)",
}
);
}
if (goUser.Tumblr != null)
{
context.AuthMethods.Add(
new AuthMethod
{
Id = SnowflakeGenerator.Instance.GenerateSnowflake(_user.Id.Time),
UserId = _user.Id,
AuthType = AuthType.Tumblr,
RemoteId = goUser.Tumblr,
RemoteUsername = goUser.TumblrUsername ?? "(unknown)",
}
);
}
if (goUser.Fediverse != null)
{
context.AuthMethods.Add(
new AuthMethod
{
Id = SnowflakeGenerator.Instance.GenerateSnowflake(_user.Id.Time),
UserId = _user.Id,
AuthType = AuthType.Fediverse,
RemoteId = goUser.Fediverse,
RemoteUsername = goUser.FediverseUsername ?? "(unknown)",
FediverseApplicationId = fediverseApplicationIds[goUser.FediverseAppId!.Value],
}
);
}
}
private void MigrateFlags()
{
foreach (GoPrideFlag flag in prideFlags.Where(f => f.UserId == goUser.Id))
{
_flagIds[flag.Id] = flag.SnowflakeId;
context.PrideFlags.Add(
new PrideFlag
{
Id = flag.SnowflakeId,
UserId = _user!.Id,
Hash = flag.Hash,
Name = flag.Name,
Description = flag.Description,
}
);
}
context.UserFlags.AddRange(
userFlags
.Where(f => f.UserId == goUser.Id)
.Where(f => _flagIds.ContainsKey(f.FlagId))
.Select(f => new UserFlag { UserId = _user!.Id, PrideFlagId = _flagIds[f.FlagId] })
);
}
private async Task MigrateMembersAsync()
{
List<GoMember> members = (
await conn.QueryAsync<GoMember>(
"select * from members where user_id = @Id",
new { goUser.Id }
)
).ToList();
foreach (GoMember member in members)
{
Log.Debug("Migrating member {Id}/{SnowflakeId}", member.Id, member.SnowflakeId);
MigrateMember(member);
}
}
private void MigrateMember(GoMember goMember)
{
var names = goMember.Names.Select(ConvertFieldEntry).ToList();
var pronouns = goMember.Pronouns.Select(ConvertPronoun).ToList();
var fields = memberFields
.Where(f => f.MemberId == goMember.Id)
.Select(f => new Field
{
Name = f.Name,
Entries = f.Entries.Select(ConvertFieldEntry).ToArray(),
})
.ToList();
var member = new Member
{
Id = goMember.SnowflakeId,
UserId = _user!.Id,
Name = goMember.Name,
Sid = goMember.Sid,
DisplayName = goMember.DisplayName,
Bio = goMember.Bio,
Avatar = goMember.Avatar,
Links = goMember.Links ?? [],
Unlisted = goMember.Unlisted,
Names = names,
Pronouns = pronouns,
Fields = fields,
};
context.Members.Add(member);
context.MemberFlags.AddRange(
memberFlags
.Where(f => f.MemberId == goMember.Id)
.Select(f => new MemberFlag
{
MemberId = member.Id,
PrideFlagId = _flagIds[f.FlagId],
})
);
}
private Dictionary<Snowflake, User.CustomPreference> ConvertPreferences()
{
var prefs = new Dictionary<Snowflake, User.CustomPreference>();
foreach ((string id, GoCustomPreference goPref) in goUser.CustomPreferences)
{
Snowflake newId = SnowflakeGenerator.Instance.GenerateSnowflake(
goUser.SnowflakeId.Time
);
_preferenceIds[id] = newId;
prefs[newId] = new User.CustomPreference
{
Icon = goPref.Icon,
Tooltip = goPref.Tooltip,
Muted = goPref.Muted,
Favourite = goPref.Favourite,
Size = goPref.PreferenceSize switch
{
"large" => PreferenceSize.Large,
"normal" => PreferenceSize.Normal,
"small" => PreferenceSize.Small,
_ => PreferenceSize.Normal,
},
};
}
return prefs;
}
private FieldEntry ConvertFieldEntry(GoFieldEntry entry) =>
new() { Value = entry.Value, Status = ConvertPreferenceId(entry.Status) };
private Pronoun ConvertPronoun(GoPronounEntry entry) =>
new()
{
Value = entry.Pronouns,
DisplayText = entry.DisplayText,
Status = ConvertPreferenceId(entry.Status),
};
private string ConvertPreferenceId(string id)
{
if (_preferenceIds.TryGetValue(id, out Snowflake preferenceId))
return preferenceId.ToString();
return ValidationUtils.DefaultStatusOptions.Contains(id) ? id : "okay";
}
}