diff --git a/.idea/.idea.Foxnouns.NET/.idea/indexLayout.xml b/.idea/.idea.Foxnouns.NET/.idea/indexLayout.xml index a797372..ea4646c 100644 --- a/.idea/.idea.Foxnouns.NET/.idea/indexLayout.xml +++ b/.idea/.idea.Foxnouns.NET/.idea/indexLayout.xml @@ -3,6 +3,7 @@ Foxnouns.Frontend + migrators/go-exporter diff --git a/Foxnouns.Backend/Database/Snowflake.cs b/Foxnouns.Backend/Database/Snowflake.cs index 78efee6..5c1c4bb 100644 --- a/Foxnouns.Backend/Database/Snowflake.cs +++ b/Foxnouns.Backend/Database/Snowflake.cs @@ -4,6 +4,7 @@ using System.Globalization; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using Newtonsoft.Json; using NodaTime; +using JsonSerializer = Newtonsoft.Json.JsonSerializer; namespace Foxnouns.Backend.Database; diff --git a/Foxnouns.Backend/Extensions/WebApplicationExtensions.cs b/Foxnouns.Backend/Extensions/WebApplicationExtensions.cs index 77a8a7e..132b4d0 100644 --- a/Foxnouns.Backend/Extensions/WebApplicationExtensions.cs +++ b/Foxnouns.Backend/Extensions/WebApplicationExtensions.cs @@ -34,7 +34,6 @@ public static class WebApplicationExtensions .MinimumLevel.Override("Microsoft.AspNetCore.Hosting", LogEventLevel.Warning) .MinimumLevel.Override("Microsoft.AspNetCore.Mvc", LogEventLevel.Warning) .MinimumLevel.Override("Microsoft.AspNetCore.Routing", LogEventLevel.Warning) - .MinimumLevel.Override("Hangfire", LogEventLevel.Information) .WriteTo.Console(); if (config.Logging.SeqLogUrl != null) diff --git a/Foxnouns.Frontend/package.json b/Foxnouns.Frontend/package.json index cc0c993..839ad14 100644 --- a/Foxnouns.Frontend/package.json +++ b/Foxnouns.Frontend/package.json @@ -56,7 +56,7 @@ "eslint-plugin-react-hooks": "^4.6.0", "i18next-parser": "^9.0.2", "prettier": "^3.3.3", - "sass": "^1.78.0", + "sass": "1.77.6", "typescript": "^5.1.6", "vite": "^5.1.0", "vite-tsconfig-paths": "^4.2.1" diff --git a/Foxnouns.Frontend/yarn.lock b/Foxnouns.Frontend/yarn.lock index 1d7c038..c55dc2e 100644 --- a/Foxnouns.Frontend/yarn.lock +++ b/Foxnouns.Frontend/yarn.lock @@ -6097,10 +6097,10 @@ safe-regex-test@^1.0.3: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== -sass@^1.78.0: - version "1.78.0" - resolved "https://registry.yarnpkg.com/sass/-/sass-1.78.0.tgz#cef369b2f9dc21ea1d2cf22c979f52365da60841" - integrity sha512-AaIqGSrjo5lA2Yg7RvFZrlXDBCp3nV4XP73GrLGvdRWWwk+8H3l0SDvq/5bA4eF+0RFPLuWUk3E+P1U/YqnpsQ== +sass@1.77.6: + version "1.77.6" + resolved "https://registry.yarnpkg.com/sass/-/sass-1.77.6.tgz#898845c1348078c2e6d1b64f9ee06b3f8bd489e4" + integrity sha512-ByXE1oLD79GVq9Ht1PeHWCPMPB8XHpBuz1r85oByKHjZY6qV6rWnQovQzXJXuQ/XyE1Oj3iPk3lo28uzaRA2/Q== dependencies: chokidar ">=3.0.0 <4.0.0" immutable "^4.0.0" diff --git a/Foxnouns.NET.sln b/Foxnouns.NET.sln index 3b119c5..aec8ae7 100644 --- a/Foxnouns.NET.sln +++ b/Foxnouns.NET.sln @@ -5,6 +5,8 @@ VisualStudioVersion = 17.0.31903.59 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Foxnouns.Backend", "Foxnouns.Backend\Foxnouns.Backend.csproj", "{439E3E38-5AEF-4F73-AD57-E32057B3FC7F}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetImporter", "migrators\NetImporter\NetImporter.csproj", "{FBCF80EE-624F-43AF-8122-230B5447940C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -18,5 +20,9 @@ Global {439E3E38-5AEF-4F73-AD57-E32057B3FC7F}.Debug|Any CPU.Build.0 = Debug|Any CPU {439E3E38-5AEF-4F73-AD57-E32057B3FC7F}.Release|Any CPU.ActiveCfg = Release|Any CPU {439E3E38-5AEF-4F73-AD57-E32057B3FC7F}.Release|Any CPU.Build.0 = Release|Any CPU + {FBCF80EE-624F-43AF-8122-230B5447940C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FBCF80EE-624F-43AF-8122-230B5447940C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FBCF80EE-624F-43AF-8122-230B5447940C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FBCF80EE-624F-43AF-8122-230B5447940C}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/migrators/NetImporter/ImportUser.cs b/migrators/NetImporter/ImportUser.cs new file mode 100644 index 0000000..3524fad --- /dev/null +++ b/migrators/NetImporter/ImportUser.cs @@ -0,0 +1,174 @@ +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using Foxnouns.Backend.Database; +using Foxnouns.Backend.Database.Models; +using Microsoft.EntityFrameworkCore; +using NodaTime; +using NodaTime.Extensions; +using Serilog; + +namespace NetImporter; + +public static class Users +{ + public static async Task ImportUsers(string filename) + { + await using var db = await NetImporter.GetContextAsync(); + await db.Database.ExecuteSqlRawAsync("SELECT 1"); + + var stopwatch = new Stopwatch(); + stopwatch.Start(); + + var users = NetImporter.ReadFromFile(filename).Output.Select(ConvertUser).ToList(); + db.AddRange(users); + await db.SaveChangesAsync(); + + stopwatch.Stop(); + Log.Information("Imported {Count} users in {Duration}", users.Count, stopwatch.ElapsedDuration()); + } + + private static User ConvertUser(ImportUser oldUser) + { + var user = new User + { + Id = oldUser.Id, + Username = oldUser.Username, + DisplayName = oldUser.DisplayName, + Bio = oldUser.Bio, + MemberTitle = oldUser.MemberTitle, + LastActive = oldUser.LastActive.ToInstant(), + Avatar = oldUser.AvatarHash, + Links = oldUser.Links ?? [], + + Role = oldUser.ParseRole(), + Deleted = oldUser.Deleted, + DeletedAt = oldUser.DeletedAt?.ToInstant(), + DeletedBy = null + }; + + if (oldUser is { DiscordId: not null, DiscordUsername: not null }) + { + user.AuthMethods.Add(new AuthMethod + { + Id = SnowflakeGenerator.Instance.GenerateSnowflake(), + AuthType = AuthType.Discord, + RemoteId = oldUser.DiscordId, + RemoteUsername = oldUser.DiscordUsername + }); + } + + if (oldUser is { TumblrId: not null, TumblrUsername: not null }) + { + user.AuthMethods.Add(new AuthMethod + { + Id = SnowflakeGenerator.Instance.GenerateSnowflake(), + AuthType = AuthType.Tumblr, + RemoteId = oldUser.TumblrId, + RemoteUsername = oldUser.TumblrUsername + }); + } + + if (oldUser is { GoogleId: not null, GoogleUsername: not null }) + { + user.AuthMethods.Add(new AuthMethod + { + Id = SnowflakeGenerator.Instance.GenerateSnowflake(), + AuthType = AuthType.Google, + RemoteId = oldUser.GoogleId, + RemoteUsername = oldUser.GoogleUsername + }); + } + + // Convert all custom preference UUIDs to snowflakes + var prefMapping = new Dictionary(); + foreach (var (key, value) in oldUser.CustomPreferences) + { + var newKey = SnowflakeGenerator.Instance.GenerateSnowflake(); + prefMapping[key] = newKey; + user.CustomPreferences[newKey] = value; + } + + foreach (var name in oldUser.Names ?? []) + { + user.Names.Add(new FieldEntry + { + Value = name.Value, + Status = prefMapping.TryGetValue(name.Status, out var newStatus) ? newStatus.ToString() : name.Status, + }); + } + + foreach (var pronoun in oldUser.Pronouns ?? []) + { + user.Pronouns.Add(new Pronoun + { + Value = pronoun.Value, + DisplayText = pronoun.DisplayText, + Status = prefMapping.TryGetValue(pronoun.Status, out var newStatus) + ? newStatus.ToString() + : pronoun.Status, + }); + } + + foreach (var field in oldUser.Fields ?? []) + { + var entries = field.Entries.Select(entry => new FieldEntry + { + Value = entry.Value, + Status = prefMapping.TryGetValue(entry.Status, out var newStatus) + ? newStatus.ToString() + : entry.Status, + }) + .ToList(); + + user.Fields.Add(new Field + { + Name = field.Name, + Entries = entries.ToArray() + }); + } + + Log.Debug("Converted user {UserId}", oldUser.Id); + + return user; + } + + [SuppressMessage("ReSharper", "ClassNeverInstantiated.Local")] + private record ImportUser( + Snowflake Id, + string Sid, + string Username, + string? DisplayName, + string? Bio, + string? MemberTitle, + OffsetDateTime LastActive, + string? AvatarHash, + string[]? Links, + FieldEntry[]? Names, + Pronoun[]? Pronouns, + Field[]? Fields, + string? DiscordId, + string? DiscordUsername, + string? FediverseId, + string? FediverseUsername, + long? FediverseAppId, + string? TumblrId, + string? TumblrUsername, + string? GoogleId, + string? GoogleUsername, + bool MemberListHidden, + string? Timezone, + string Role, + bool Deleted, + OffsetDateTime? DeletedAt, + string? DeleteReason, + Dictionary CustomPreferences) + { + public UserRole ParseRole() => Role switch + { + "USER" => UserRole.User, + "MODERATOR" => UserRole.Moderator, + "ADMIN" => UserRole.Admin, + _ => UserRole.User + }; + } +} \ No newline at end of file diff --git a/migrators/NetImporter/NetImporter.cs b/migrators/NetImporter/NetImporter.cs new file mode 100644 index 0000000..05cc42d --- /dev/null +++ b/migrators/NetImporter/NetImporter.cs @@ -0,0 +1,83 @@ +using Foxnouns.Backend; +using Foxnouns.Backend.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; +using NodaTime; +using NodaTime.Serialization.JsonNet; +using Serilog; +using Serilog.Events; + +namespace NetImporter; + +internal static class NetImporter +{ + public static async Task Main(string[] args) + { + Log.Logger = new LoggerConfiguration() + .Enrich.FromLogContext() + .MinimumLevel.Debug() + .MinimumLevel.Override("Microsoft", LogEventLevel.Information) + .MinimumLevel.Override("Microsoft.EntityFrameworkCore.Database.Command", LogEventLevel.Information) + .WriteTo.Console() + .CreateLogger(); + + switch (args.Length) + { + case < 2: + Console.WriteLine("Not enough arguments. Usage: "); + return; + case > 2: + Console.WriteLine("Too many arguments. Usage: "); + return; + } + + switch (args[0].ToLowerInvariant()) + { + case "users": + await Users.ImportUsers(args[1]); + break; + default: + Console.WriteLine("Invalid type. Valid types are: users"); + break; + } + } + + internal static async Task GetContextAsync() + { + var connString = Environment.GetEnvironmentVariable("DATABASE"); + if (connString == null) throw new Exception("$DATABASE not set, must be an ADO.NET connection string"); + + var loggerFactory = new LoggerFactory().AddSerilog(Log.Logger); + var config = new Config + { + Database = new Config.DatabaseConfig + { + Url = connString + } + }; + + var db = new DatabaseContext(config, loggerFactory); + + if ((await db.Database.GetPendingMigrationsAsync()).Any()) + { + Log.Fatal("Database needs to be migrated first"); + } + + return db; + } + + private static readonly JsonSerializerSettings Settings = new JsonSerializerSettings + { + ContractResolver = new DefaultContractResolver { NamingStrategy = new SnakeCaseNamingStrategy() } + }.ConfigureForNodaTime(DateTimeZoneProviders.Tzdb); + + internal static Input ReadFromFile(string path) + { + var data = File.ReadAllText(path); + return JsonConvert.DeserializeObject>(data, Settings) ?? throw new Exception("Invalid input file"); + } +} + +internal record Input(List Output, List Skipped); \ No newline at end of file diff --git a/migrators/NetImporter/NetImporter.csproj b/migrators/NetImporter/NetImporter.csproj new file mode 100644 index 0000000..e62f921 --- /dev/null +++ b/migrators/NetImporter/NetImporter.csproj @@ -0,0 +1,25 @@ + + + + Exe + net8.0 + enable + enable + + + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + diff --git a/migrators/go-exporter/user.go b/migrators/go-exporter/user.go index 216c665..1af246e 100644 --- a/migrators/go-exporter/user.go +++ b/migrators/go-exporter/user.go @@ -48,6 +48,16 @@ func userToNewUser(u db.User) (NewUser, error) { Links: u.Links, Names: u.Names, + Discord: u.Discord, + DiscordUsername: u.DiscordUsername, + Fediverse: u.Fediverse, + FediverseUsername: u.FediverseUsername, + FediverseAppID: u.FediverseAppID, + Tumblr: u.Tumblr, + TumblrUsername: u.TumblrUsername, + Google: u.Google, + GoogleUsername: u.GoogleUsername, + MemberListHidden: u.ListPrivate, Timezone: u.Timezone, Role: "USER", @@ -88,6 +98,19 @@ type NewUser struct { Pronouns []NewPronoun `json:"pronouns"` Fields []NewField `json:"fields"` + Discord *string `json:"discord_id"` + DiscordUsername *string `json:"discord_username"` + + Fediverse *string `json:"fediverse_id"` + FediverseUsername *string `json:"fediverse_username"` + FediverseAppID *int64 `json:"fediverse_app_id"` + + Tumblr *string `json:"tumblr_id"` + TumblrUsername *string `json:"tumblr_username"` + + Google *string `json:"google_id"` + GoogleUsername *string `json:"google_username"` + MemberListHidden bool `json:"member_list_hidden"` Timezone *string `json:"timezone"` Role string `json:"role"` // one of USER or ADMIN