feat: add .net user importer

.idea/.idea.Foxnouns.NET/.idea/indexLayout.xml

@@ -3,6 +3,7 @@
   <component name="UserContentModel">
     <attachedFolders>
       <Path>Foxnouns.Frontend</Path>
+      <Path>migrators/go-exporter</Path>
     </attachedFolders>
     <explicitIncludes />
     <explicitExcludes />

Foxnouns.Backend/Database/Snowflake.cs

@@ -4,6 +4,7 @@ using System.Globalization;
 using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
 using Newtonsoft.Json;
 using NodaTime;
+using JsonSerializer = Newtonsoft.Json.JsonSerializer;
 
 namespace Foxnouns.Backend.Database;
 

Foxnouns.Backend/Extensions/WebApplicationExtensions.cs

@@ -34,7 +34,6 @@ public static class WebApplicationExtensions
             .MinimumLevel.Override("Microsoft.AspNetCore.Hosting", LogEventLevel.Warning)
             .MinimumLevel.Override("Microsoft.AspNetCore.Mvc", LogEventLevel.Warning)
             .MinimumLevel.Override("Microsoft.AspNetCore.Routing", LogEventLevel.Warning)
-            .MinimumLevel.Override("Hangfire", LogEventLevel.Information)
             .WriteTo.Console();
 
         if (config.Logging.SeqLogUrl != null)

Foxnouns.Frontend/package.json

@@ -56,7 +56,7 @@
 		"eslint-plugin-react-hooks": "^4.6.0",
 		"i18next-parser": "^9.0.2",
 		"prettier": "^3.3.3",
-		"sass": "^1.78.0",
+		"sass": "1.77.6",
 		"typescript": "^5.1.6",
 		"vite": "^5.1.0",
 		"vite-tsconfig-paths": "^4.2.1"

Foxnouns.Frontend/yarn.lock

@@ -6097,10 +6097,10 @@ safe-regex-test@^1.0.3:
   resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
   integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==
 
-sass@^1.78.0:
+sass@1.77.6:
-  version "1.78.0"
+  version "1.77.6"
-  resolved "https://registry.yarnpkg.com/sass/-/sass-1.78.0.tgz#cef369b2f9dc21ea1d2cf22c979f52365da60841"
+  resolved "https://registry.yarnpkg.com/sass/-/sass-1.77.6.tgz#898845c1348078c2e6d1b64f9ee06b3f8bd489e4"
-  integrity sha512-AaIqGSrjo5lA2Yg7RvFZrlXDBCp3nV4XP73GrLGvdRWWwk+8H3l0SDvq/5bA4eF+0RFPLuWUk3E+P1U/YqnpsQ==
+  integrity sha512-ByXE1oLD79GVq9Ht1PeHWCPMPB8XHpBuz1r85oByKHjZY6qV6rWnQovQzXJXuQ/XyE1Oj3iPk3lo28uzaRA2/Q==
   dependencies:
     chokidar ">=3.0.0 <4.0.0"
     immutable "^4.0.0"

Foxnouns.NET.sln

@@ -5,6 +5,8 @@ VisualStudioVersion = 17.0.31903.59
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Foxnouns.Backend", "Foxnouns.Backend\Foxnouns.Backend.csproj", "{439E3E38-5AEF-4F73-AD57-E32057B3FC7F}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetImporter", "migrators\NetImporter\NetImporter.csproj", "{FBCF80EE-624F-43AF-8122-230B5447940C}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -18,5 +20,9 @@ Global
 		{439E3E38-5AEF-4F73-AD57-E32057B3FC7F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{439E3E38-5AEF-4F73-AD57-E32057B3FC7F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{439E3E38-5AEF-4F73-AD57-E32057B3FC7F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FBCF80EE-624F-43AF-8122-230B5447940C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FBCF80EE-624F-43AF-8122-230B5447940C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FBCF80EE-624F-43AF-8122-230B5447940C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FBCF80EE-624F-43AF-8122-230B5447940C}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

migrators/NetImporter/ImportUser.cs

@@ -0,0 +1,174 @@
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+using Foxnouns.Backend.Database;
+using Foxnouns.Backend.Database.Models;
+using Microsoft.EntityFrameworkCore;
+using NodaTime;
+using NodaTime.Extensions;
+using Serilog;
+
+namespace NetImporter;
+
+public static class Users
+{
+    public static async Task ImportUsers(string filename)
+    {
+        await using var db = await NetImporter.GetContextAsync();
+        await db.Database.ExecuteSqlRawAsync("SELECT 1");
+
+        var stopwatch = new Stopwatch();
+        stopwatch.Start();
+
+        var users = NetImporter.ReadFromFile<ImportUser>(filename).Output.Select(ConvertUser).ToList();
+        db.AddRange(users);
+        await db.SaveChangesAsync();
+
+        stopwatch.Stop();
+        Log.Information("Imported {Count} users in {Duration}", users.Count, stopwatch.ElapsedDuration());
+    }
+
+    private static User ConvertUser(ImportUser oldUser)
+    {
+        var user = new User
+        {
+            Id = oldUser.Id,
+            Username = oldUser.Username,
+            DisplayName = oldUser.DisplayName,
+            Bio = oldUser.Bio,
+            MemberTitle = oldUser.MemberTitle,
+            LastActive = oldUser.LastActive.ToInstant(),
+            Avatar = oldUser.AvatarHash,
+            Links = oldUser.Links ?? [],
+
+            Role = oldUser.ParseRole(),
+            Deleted = oldUser.Deleted,
+            DeletedAt = oldUser.DeletedAt?.ToInstant(),
+            DeletedBy = null
+        };
+
+        if (oldUser is { DiscordId: not null, DiscordUsername: not null })
+        {
+            user.AuthMethods.Add(new AuthMethod
+            {
+                Id = SnowflakeGenerator.Instance.GenerateSnowflake(),
+                AuthType = AuthType.Discord,
+                RemoteId = oldUser.DiscordId,
+                RemoteUsername = oldUser.DiscordUsername
+            });
+        }
+
+        if (oldUser is { TumblrId: not null, TumblrUsername: not null })
+        {
+            user.AuthMethods.Add(new AuthMethod
+            {
+                Id = SnowflakeGenerator.Instance.GenerateSnowflake(),
+                AuthType = AuthType.Tumblr,
+                RemoteId = oldUser.TumblrId,
+                RemoteUsername = oldUser.TumblrUsername
+            });
+        }
+
+        if (oldUser is { GoogleId: not null, GoogleUsername: not null })
+        {
+            user.AuthMethods.Add(new AuthMethod
+            {
+                Id = SnowflakeGenerator.Instance.GenerateSnowflake(),
+                AuthType = AuthType.Google,
+                RemoteId = oldUser.GoogleId,
+                RemoteUsername = oldUser.GoogleUsername
+            });
+        }
+
+        // Convert all custom preference UUIDs to snowflakes
+        var prefMapping = new Dictionary<string, Snowflake>();
+        foreach (var (key, value) in oldUser.CustomPreferences)
+        {
+            var newKey = SnowflakeGenerator.Instance.GenerateSnowflake();
+            prefMapping[key] = newKey;
+            user.CustomPreferences[newKey] = value;
+        }
+
+        foreach (var name in oldUser.Names ?? [])
+        {
+            user.Names.Add(new FieldEntry
+            {
+                Value = name.Value,
+                Status = prefMapping.TryGetValue(name.Status, out var newStatus) ? newStatus.ToString() : name.Status,
+            });
+        }
+
+        foreach (var pronoun in oldUser.Pronouns ?? [])
+        {
+            user.Pronouns.Add(new Pronoun
+            {
+                Value = pronoun.Value,
+                DisplayText = pronoun.DisplayText,
+                Status = prefMapping.TryGetValue(pronoun.Status, out var newStatus)
+                    ? newStatus.ToString()
+                    : pronoun.Status,
+            });
+        }
+
+        foreach (var field in oldUser.Fields ?? [])
+        {
+            var entries = field.Entries.Select(entry => new FieldEntry
+                {
+                    Value = entry.Value,
+                    Status = prefMapping.TryGetValue(entry.Status, out var newStatus)
+                        ? newStatus.ToString()
+                        : entry.Status,
+                })
+                .ToList();
+
+            user.Fields.Add(new Field
+            {
+                Name = field.Name,
+                Entries = entries.ToArray()
+            });
+        }
+
+        Log.Debug("Converted user {UserId}", oldUser.Id);
+
+        return user;
+    }
+
+    [SuppressMessage("ReSharper", "ClassNeverInstantiated.Local")]
+    private record ImportUser(
+        Snowflake Id,
+        string Sid,
+        string Username,
+        string? DisplayName,
+        string? Bio,
+        string? MemberTitle,
+        OffsetDateTime LastActive,
+        string? AvatarHash,
+        string[]? Links,
+        FieldEntry[]? Names,
+        Pronoun[]? Pronouns,
+        Field[]? Fields,
+        string? DiscordId,
+        string? DiscordUsername,
+        string? FediverseId,
+        string? FediverseUsername,
+        long? FediverseAppId,
+        string? TumblrId,
+        string? TumblrUsername,
+        string? GoogleId,
+        string? GoogleUsername,
+        bool MemberListHidden,
+        string? Timezone,
+        string Role,
+        bool Deleted,
+        OffsetDateTime? DeletedAt,
+        string? DeleteReason,
+        Dictionary<string, User.CustomPreference> CustomPreferences)
+    {
+        public UserRole ParseRole() => Role switch
+        {
+            "USER" => UserRole.User,
+            "MODERATOR" => UserRole.Moderator,
+            "ADMIN" => UserRole.Admin,
+            _ => UserRole.User
+        };
+    }
+}

migrators/NetImporter/NetImporter.cs

@@ -0,0 +1,83 @@
+using Foxnouns.Backend;
+using Foxnouns.Backend.Database;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Serialization;
+using NodaTime;
+using NodaTime.Serialization.JsonNet;
+using Serilog;
+using Serilog.Events;
+
+namespace NetImporter;
+
+internal static class NetImporter
+{
+    public static async Task Main(string[] args)
+    {
+        Log.Logger = new LoggerConfiguration()
+            .Enrich.FromLogContext()
+            .MinimumLevel.Debug()
+            .MinimumLevel.Override("Microsoft", LogEventLevel.Information)
+            .MinimumLevel.Override("Microsoft.EntityFrameworkCore.Database.Command", LogEventLevel.Information)
+            .WriteTo.Console()
+            .CreateLogger();
+
+        switch (args.Length)
+        {
+            case < 2:
+                Console.WriteLine("Not enough arguments. Usage: <type> <file>");
+                return;
+            case > 2:
+                Console.WriteLine("Too many arguments. Usage: <type> <file>");
+                return;
+        }
+
+        switch (args[0].ToLowerInvariant())
+        {
+            case "users":
+                await Users.ImportUsers(args[1]);
+                break;
+            default:
+                Console.WriteLine("Invalid type. Valid types are: users");
+                break;
+        }
+    }
+
+    internal static async Task<DatabaseContext> GetContextAsync()
+    {
+        var connString = Environment.GetEnvironmentVariable("DATABASE");
+        if (connString == null) throw new Exception("$DATABASE not set, must be an ADO.NET connection string");
+
+        var loggerFactory = new LoggerFactory().AddSerilog(Log.Logger);
+        var config = new Config
+        {
+            Database = new Config.DatabaseConfig
+            {
+                Url = connString
+            }
+        };
+
+        var db = new DatabaseContext(config, loggerFactory);
+
+        if ((await db.Database.GetPendingMigrationsAsync()).Any())
+        {
+            Log.Fatal("Database needs to be migrated first");
+        }
+
+        return db;
+    }
+
+    private static readonly JsonSerializerSettings Settings = new JsonSerializerSettings
+    {
+        ContractResolver = new DefaultContractResolver { NamingStrategy = new SnakeCaseNamingStrategy() }
+    }.ConfigureForNodaTime(DateTimeZoneProviders.Tzdb);
+
+    internal static Input<T> ReadFromFile<T>(string path)
+    {
+        var data = File.ReadAllText(path);
+        return JsonConvert.DeserializeObject<Input<T>>(data, Settings) ?? throw new Exception("Invalid input file");
+    }
+}
+
+internal record Input<T>(List<T> Output, List<string> Skipped);

migrators/NetImporter/NetImporter.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Foxnouns.Backend\Foxnouns.Backend.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="NodaTime.Serialization.JsonNet" Version="3.1.0" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.7" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.7">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+    <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.4" />
+    <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL.NodaTime" Version="8.0.4" />
+  </ItemGroup>
+
+</Project>

migrators/go-exporter/user.go

@@ -48,6 +48,16 @@ func userToNewUser(u db.User) (NewUser, error) {
 		Links:       u.Links,
 		Names:       u.Names,
 
+		Discord:           u.Discord,
+		DiscordUsername:   u.DiscordUsername,
+		Fediverse:         u.Fediverse,
+		FediverseUsername: u.FediverseUsername,
+		FediverseAppID:    u.FediverseAppID,
+		Tumblr:            u.Tumblr,
+		TumblrUsername:    u.TumblrUsername,
+		Google:            u.Google,
+		GoogleUsername:    u.GoogleUsername,
+
 		MemberListHidden: u.ListPrivate,
 		Timezone:         u.Timezone,
 		Role:             "USER",
@@ -88,6 +98,19 @@ type NewUser struct {
 	Pronouns    []NewPronoun    `json:"pronouns"`
 	Fields      []NewField      `json:"fields"`
 
+	Discord         *string `json:"discord_id"`
+	DiscordUsername *string `json:"discord_username"`
+
+	Fediverse         *string `json:"fediverse_id"`
+	FediverseUsername *string `json:"fediverse_username"`
+	FediverseAppID    *int64  `json:"fediverse_app_id"`
+
+	Tumblr         *string `json:"tumblr_id"`
+	TumblrUsername *string `json:"tumblr_username"`
+
+	Google         *string `json:"google_id"`
+	GoogleUsername *string `json:"google_username"`
+
 	MemberListHidden bool    `json:"member_list_hidden"`
 	Timezone         *string `json:"timezone"`
 	Role             string  `json:"role"` // one of USER or ADMIN