sam/Foxnouns.NET
sam/Foxnouns.NET
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: add .net user importer

Browse source
This commit is contained in:
sam 2024-09-18 21:44:47 +02:00
parent 41e620ad03
commit 412d720abc
Signed by: sam
GPG key ID: B4EF20DDE721CAA1
10 changed files with 318 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.idea/.idea.Foxnouns.NET/.idea/indexLayout.xml
View file

@ -3,6 +3,7 @@
<component name="UserContentModel">
<attachedFolders>
<Path>Foxnouns.Frontend</Path>
<Path>migrators/go-exporter</Path>
</attachedFolders>
<explicitIncludes />
<explicitExcludes />

1
Foxnouns.Backend/Database/Snowflake.cs
View file

@ -4,6 +4,7 @@ using System.Globalization;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Newtonsoft.Json;
using NodaTime;
using JsonSerializer = Newtonsoft.Json.JsonSerializer;
namespace Foxnouns.Backend.Database;

1
Foxnouns.Backend/Extensions/WebApplicationExtensions.cs
View file

@ -34,7 +34,6 @@ public static class WebApplicationExtensions
.MinimumLevel.Override("Microsoft.AspNetCore.Hosting", LogEventLevel.Warning)
.MinimumLevel.Override("Microsoft.AspNetCore.Mvc", LogEventLevel.Warning)
.MinimumLevel.Override("Microsoft.AspNetCore.Routing", LogEventLevel.Warning)
.MinimumLevel.Override("Hangfire", LogEventLevel.Information)
.WriteTo.Console();
if (config.Logging.SeqLogUrl != null)

2
Foxnouns.Frontend/package.json
View file

@ -56,7 +56,7 @@
"eslint-plugin-react-hooks": "^4.6.0",
"i18next-parser": "^9.0.2",
"prettier": "^3.3.3",
"sass": "^1.78.0",
"sass": "1.77.6",
"typescript": "^5.1.6",
"vite": "^5.1.0",
"vite-tsconfig-paths": "^4.2.1"

8
Foxnouns.Frontend/yarn.lock
View file

@ -6097,10 +6097,10 @@ safe-regex-test@^1.0.3:
resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==
sass@^1.78.0:
version "1.78.0"
resolved "https://registry.yarnpkg.com/sass/-/sass-1.78.0.tgz#cef369b2f9dc21ea1d2cf22c979f52365da60841"
integrity sha512-AaIqGSrjo5lA2Yg7RvFZrlXDBCp3nV4XP73GrLGvdRWWwk+8H3l0SDvq/5bA4eF+0RFPLuWUk3E+P1U/YqnpsQ==
sass@1.77.6:
version "1.77.6"
resolved "https://registry.yarnpkg.com/sass/-/sass-1.77.6.tgz#898845c1348078c2e6d1b64f9ee06b3f8bd489e4"
integrity sha512-ByXE1oLD79GVq9Ht1PeHWCPMPB8XHpBuz1r85oByKHjZY6qV6rWnQovQzXJXuQ/XyE1Oj3iPk3lo28uzaRA2/Q==
dependencies:
chokidar ">=3.0.0 <4.0.0"
immutable "^4.0.0"

6
Foxnouns.NET.sln
View file

@ -5,6 +5,8 @@ VisualStudioVersion = 17.0.31903.59
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Foxnouns.Backend", "Foxnouns.Backend\Foxnouns.Backend.csproj", "{439E3E38-5AEF-4F73-AD57-E32057B3FC7F}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetImporter", "migrators\NetImporter\NetImporter.csproj", "{FBCF80EE-624F-43AF-8122-230B5447940C}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -18,5 +20,9 @@ Global
{439E3E38-5AEF-4F73-AD57-E32057B3FC7F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{439E3E38-5AEF-4F73-AD57-E32057B3FC7F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{439E3E38-5AEF-4F73-AD57-E32057B3FC7F}.Release|Any CPU.Build.0 = Release|Any CPU
{FBCF80EE-624F-43AF-8122-230B5447940C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{FBCF80EE-624F-43AF-8122-230B5447940C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{FBCF80EE-624F-43AF-8122-230B5447940C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{FBCF80EE-624F-43AF-8122-230B5447940C}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
EndGlobal

174
migrators/NetImporter/ImportUser.cs Normal file
View file

@ -0,0 +1,174 @@
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using Foxnouns.Backend.Database;
using Foxnouns.Backend.Database.Models;
using Microsoft.EntityFrameworkCore;
using NodaTime;
using NodaTime.Extensions;
using Serilog;
namespace NetImporter;
public static class Users
{
public static async Task ImportUsers(string filename)
{
await using var db = await NetImporter.GetContextAsync();
await db.Database.ExecuteSqlRawAsync("SELECT 1");
var stopwatch = new Stopwatch();
stopwatch.Start();
var users = NetImporter.ReadFromFile<ImportUser>(filename).Output.Select(ConvertUser).ToList();
db.AddRange(users);
await db.SaveChangesAsync();
stopwatch.Stop();
Log.Information("Imported {Count} users in {Duration}", users.Count, stopwatch.ElapsedDuration());
}
private static User ConvertUser(ImportUser oldUser)
{
var user = new User
{
Id = oldUser.Id,
Username = oldUser.Username,
DisplayName = oldUser.DisplayName,
Bio = oldUser.Bio,
MemberTitle = oldUser.MemberTitle,
LastActive = oldUser.LastActive.ToInstant(),
Avatar = oldUser.AvatarHash,
Links = oldUser.Links ?? [],
Role = oldUser.ParseRole(),
Deleted = oldUser.Deleted,
DeletedAt = oldUser.DeletedAt?.ToInstant(),
DeletedBy = null
};
if (oldUser is { DiscordId: not null, DiscordUsername: not null })
{
user.AuthMethods.Add(new AuthMethod
{
Id = SnowflakeGenerator.Instance.GenerateSnowflake(),
AuthType = AuthType.Discord,
RemoteId = oldUser.DiscordId,
RemoteUsername = oldUser.DiscordUsername
});
}
if (oldUser is { TumblrId: not null, TumblrUsername: not null })
{
user.AuthMethods.Add(new AuthMethod
{
Id = SnowflakeGenerator.Instance.GenerateSnowflake(),
AuthType = AuthType.Tumblr,
RemoteId = oldUser.TumblrId,
RemoteUsername = oldUser.TumblrUsername
});
}
if (oldUser is { GoogleId: not null, GoogleUsername: not null })
{
user.AuthMethods.Add(new AuthMethod
{
Id = SnowflakeGenerator.Instance.GenerateSnowflake(),
AuthType = AuthType.Google,
RemoteId = oldUser.GoogleId,
RemoteUsername = oldUser.GoogleUsername
});
}
// Convert all custom preference UUIDs to snowflakes
var prefMapping = new Dictionary<string, Snowflake>();
foreach (var (key, value) in oldUser.CustomPreferences)
{
var newKey = SnowflakeGenerator.Instance.GenerateSnowflake();
prefMapping[key] = newKey;
user.CustomPreferences[newKey] = value;
}
foreach (var name in oldUser.Names ?? [])
{
user.Names.Add(new FieldEntry
{
Value = name.Value,
Status = prefMapping.TryGetValue(name.Status, out var newStatus) ? newStatus.ToString() : name.Status,
});
}
foreach (var pronoun in oldUser.Pronouns ?? [])
{
user.Pronouns.Add(new Pronoun
{
Value = pronoun.Value,
DisplayText = pronoun.DisplayText,
Status = prefMapping.TryGetValue(pronoun.Status, out var newStatus)
? newStatus.ToString()
: pronoun.Status,
});
}
foreach (var field in oldUser.Fields ?? [])
{
var entries = field.Entries.Select(entry => new FieldEntry
{
Value = entry.Value,
Status = prefMapping.TryGetValue(entry.Status, out var newStatus)
? newStatus.ToString()
: entry.Status,
})
.ToList();
user.Fields.Add(new Field
{
Name = field.Name,
Entries = entries.ToArray()
});
}
Log.Debug("Converted user {UserId}", oldUser.Id);
return user;
}
[SuppressMessage("ReSharper", "ClassNeverInstantiated.Local")]
private record ImportUser(
Snowflake Id,
string Sid,
string Username,
string? DisplayName,
string? Bio,
string? MemberTitle,
OffsetDateTime LastActive,
string? AvatarHash,
string[]? Links,
FieldEntry[]? Names,
Pronoun[]? Pronouns,
Field[]? Fields,
string? DiscordId,
string? DiscordUsername,
string? FediverseId,
string? FediverseUsername,
long? FediverseAppId,
string? TumblrId,
string? TumblrUsername,
string? GoogleId,
string? GoogleUsername,
bool MemberListHidden,
string? Timezone,
string Role,
bool Deleted,
OffsetDateTime? DeletedAt,
string? DeleteReason,
Dictionary<string, User.CustomPreference> CustomPreferences)
{
public UserRole ParseRole() => Role switch
{
"USER" => UserRole.User,
"MODERATOR" => UserRole.Moderator,
"ADMIN" => UserRole.Admin,
_ => UserRole.User
};
}
}

83
migrators/NetImporter/NetImporter.cs Normal file
View file

@ -0,0 +1,83 @@
using Foxnouns.Backend;
using Foxnouns.Backend.Database;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Newtonsoft.Json.Serialization;
using NodaTime;
using NodaTime.Serialization.JsonNet;
using Serilog;
using Serilog.Events;
namespace NetImporter;
internal static class NetImporter
{
public static async Task Main(string[] args)
{
Log.Logger = new LoggerConfiguration()
.Enrich.FromLogContext()
.MinimumLevel.Debug()
.MinimumLevel.Override("Microsoft", LogEventLevel.Information)
.MinimumLevel.Override("Microsoft.EntityFrameworkCore.Database.Command", LogEventLevel.Information)
.WriteTo.Console()
.CreateLogger();
switch (args.Length)
{
case < 2:
Console.WriteLine("Not enough arguments. Usage: <type> <file>");
return;
case > 2:
Console.WriteLine("Too many arguments. Usage: <type> <file>");
return;
}
switch (args[0].ToLowerInvariant())
{
case "users":
await Users.ImportUsers(args[1]);
break;
default:
Console.WriteLine("Invalid type. Valid types are: users");
break;
}
}
internal static async Task<DatabaseContext> GetContextAsync()
{
var connString = Environment.GetEnvironmentVariable("DATABASE");
if (connString == null) throw new Exception("$DATABASE not set, must be an ADO.NET connection string");
var loggerFactory = new LoggerFactory().AddSerilog(Log.Logger);
var config = new Config
{
Database = new Config.DatabaseConfig
{
Url = connString
}
};
var db = new DatabaseContext(config, loggerFactory);
if ((await db.Database.GetPendingMigrationsAsync()).Any())
{
Log.Fatal("Database needs to be migrated first");
}
return db;
}
private static readonly JsonSerializerSettings Settings = new JsonSerializerSettings
{
ContractResolver = new DefaultContractResolver { NamingStrategy = new SnakeCaseNamingStrategy() }
}.ConfigureForNodaTime(DateTimeZoneProviders.Tzdb);
internal static Input<T> ReadFromFile<T>(string path)
{
var data = File.ReadAllText(path);
return JsonConvert.DeserializeObject<Input<T>>(data, Settings) ?? throw new Exception("Invalid input file");
}
}
internal record Input<T>(List<T> Output, List<string> Skipped);

25
migrators/NetImporter/NetImporter.csproj Normal file
View file

@ -0,0 +1,25 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\Foxnouns.Backend\Foxnouns.Backend.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="NodaTime.Serialization.JsonNet" Version="3.1.0" />
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.7" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.7">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.4" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL.NodaTime" Version="8.0.4" />
</ItemGroup>
</Project>

23
migrators/go-exporter/user.go
View file

@ -48,6 +48,16 @@ func userToNewUser(u db.User) (NewUser, error) {
Links: u.Links,
Names: u.Names,
Discord: u.Discord,
DiscordUsername: u.DiscordUsername,
Fediverse: u.Fediverse,
FediverseUsername: u.FediverseUsername,
FediverseAppID: u.FediverseAppID,
Tumblr: u.Tumblr,
TumblrUsername: u.TumblrUsername,
Google: u.Google,
GoogleUsername: u.GoogleUsername,
MemberListHidden: u.ListPrivate,
Timezone: u.Timezone,
Role: "USER",
@ -88,6 +98,19 @@ type NewUser struct {
Pronouns []NewPronoun `json:"pronouns"`
Fields []NewField `json:"fields"`
Discord *string `json:"discord_id"`
DiscordUsername *string `json:"discord_username"`
Fediverse *string `json:"fediverse_id"`
FediverseUsername *string `json:"fediverse_username"`
FediverseAppID *int64 `json:"fediverse_app_id"`
Tumblr *string `json:"tumblr_id"`
TumblrUsername *string `json:"tumblr_username"`
Google *string `json:"google_id"`
GoogleUsername *string `json:"google_username"`
MemberListHidden bool `json:"member_list_hidden"`
Timezone *string `json:"timezone"`
Role string `json:"role"` // one of USER or ADMIN