diff --git a/DOCKER.md b/DOCKER.md index b485eb7..5cbb6e6 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -8,6 +8,3 @@ The Caddy server will listen on `localhost:5004` for the frontend and API, and on `localhost:5005` for the profile URL shortener. - -The backend server listens on `localhost:5006` for unproxied API access, -and `localhost:5007` for metrics. \ No newline at end of file diff --git a/Foxnouns.Backend/Utils/ValidationUtils.Fields.cs b/Foxnouns.Backend/Utils/ValidationUtils.Fields.cs index 0235eb6..8a7cad5 100644 --- a/Foxnouns.Backend/Utils/ValidationUtils.Fields.cs +++ b/Foxnouns.Backend/Utils/ValidationUtils.Fields.cs @@ -19,7 +19,7 @@ namespace Foxnouns.Backend.Utils; public static partial class ValidationUtils { - public static readonly string[] DefaultStatusOptions = + private static readonly string[] DefaultStatusOptions = [ "favourite", "okay", diff --git a/Foxnouns.DataMigrator/Foxnouns.DataMigrator.csproj b/Foxnouns.DataMigrator/Foxnouns.DataMigrator.csproj deleted file mode 100644 index 5fde110..0000000 --- a/Foxnouns.DataMigrator/Foxnouns.DataMigrator.csproj +++ /dev/null @@ -1,20 +0,0 @@ - - - - Exe - net9.0 - enable - enable - - - - - - - - - - - - - diff --git a/Foxnouns.DataMigrator/GoDatabase.cs b/Foxnouns.DataMigrator/GoDatabase.cs deleted file mode 100644 index a2f7720..0000000 --- a/Foxnouns.DataMigrator/GoDatabase.cs +++ /dev/null @@ -1,71 +0,0 @@ -using System.Data; -using Dapper; -using Foxnouns.DataMigrator.Models; -using Newtonsoft.Json; -using Npgsql; - -namespace Foxnouns.DataMigrator; - -public static class GoDatabase -{ - private static NpgsqlDataSource? _dataSource; - - public static async Task GetConnectionAsync() - { - if (_dataSource != null) - return await _dataSource.OpenConnectionAsync(); - - DefaultTypeMap.MatchNamesWithUnderscores = true; - - SqlMapper.RemoveTypeMap(typeof(ulong)); - SqlMapper.AddTypeHandler(new UlongEncodeAsLongHandler()); - SqlMapper.AddTypeHandler(new JsonTypeHandler()); - SqlMapper.AddTypeHandler(new JsonTypeHandler>()); - SqlMapper.AddTypeHandler(new JsonTypeHandler()); - SqlMapper.AddTypeHandler(new UlongListHandler()); - - string dsn = - Environment.GetEnvironmentVariable("GO_DATABASE") - ?? throw new Exception("$GO_DATABASE is not set"); - - var dataSourceBuilder = new NpgsqlDataSourceBuilder(dsn); - dataSourceBuilder.UseJsonNet(); - - _dataSource = dataSourceBuilder.Build(); - - return await _dataSource.OpenConnectionAsync(); - } - - // dapper why - // taken from https://codeberg.org/starshine/catalogger/src/branch/main/Catalogger.Backend/Database/DatabasePool.cs - private class UlongEncodeAsLongHandler : SqlMapper.TypeHandler - { - public override void SetValue(IDbDataParameter parameter, ulong value) => - parameter.Value = (long)value; - - public override ulong Parse(object value) => - // Cast to long to unbox, then to ulong (???) - (ulong)(long)value; - } - - private class UlongListHandler : SqlMapper.TypeHandler> - { - public override void SetValue(IDbDataParameter parameter, List? value) => - parameter.Value = value?.Select(i => (long)i).ToArray(); - - public override List? Parse(object value) => - ((long[])value).Select(i => (ulong)i).ToList(); - } - - private class JsonTypeHandler : SqlMapper.TypeHandler - { - public override void SetValue(IDbDataParameter parameter, T? value) => - parameter.Value = JsonConvert.SerializeObject(value); - - public override T? Parse(object value) - { - var json = (string)value; - return JsonConvert.DeserializeObject(json) ?? default; - } - } -} diff --git a/Foxnouns.DataMigrator/Models/GoMember.cs b/Foxnouns.DataMigrator/Models/GoMember.cs deleted file mode 100644 index d1c52fe..0000000 --- a/Foxnouns.DataMigrator/Models/GoMember.cs +++ /dev/null @@ -1,26 +0,0 @@ -using Foxnouns.Backend.Database; - -namespace Foxnouns.DataMigrator.Models; - -public class GoMember -{ - public required string Id { get; init; } - public required string Name { get; init; } - public string? Bio { get; init; } - public string[]? Links { get; init; } - public string? DisplayName { get; init; } - public GoFieldEntry[] Names { get; init; } = []; - public GoPronounEntry[] Pronouns { get; init; } = []; - public string? Avatar { get; init; } - public required bool Unlisted { get; init; } - public required string Sid { get; init; } - public required Snowflake SnowflakeId { get; init; } -} - -public class GoMemberField -{ - public required string MemberId { get; init; } - public required long Id { get; init; } - public required string Name { get; init; } - public required GoFieldEntry[] Entries { get; init; } -} diff --git a/Foxnouns.DataMigrator/Models/GoUser.cs b/Foxnouns.DataMigrator/Models/GoUser.cs deleted file mode 100644 index 20f5d51..0000000 --- a/Foxnouns.DataMigrator/Models/GoUser.cs +++ /dev/null @@ -1,102 +0,0 @@ -using Foxnouns.Backend.Database; -using Foxnouns.Backend.Database.Models; - -namespace Foxnouns.DataMigrator.Models; - -public class GoUser -{ - public required string Id { get; init; } - public required string Username { get; init; } - public string? DisplayName { get; init; } - public string? Bio { get; init; } - public string[]? Links { get; init; } - public string? Discord { get; init; } - public string? DiscordUsername { get; init; } - public DateTimeOffset? DeletedAt { get; init; } - public bool? SelfDelete { get; init; } - public string? DeleteReason { get; init; } - public GoFieldEntry[] Names { get; init; } = []; - public GoPronounEntry[] Pronouns { get; init; } = []; - public string? Avatar { get; init; } - public string? Fediverse { get; init; } - public string? FediverseUsername { get; init; } - public int? FediverseAppId { get; init; } - public bool IsAdmin { get; init; } - public string? MemberTitle { get; init; } - public bool ListPrivate { get; init; } - public string? Tumblr { get; init; } - public string? TumblrUsername { get; init; } - public string? Google { get; init; } - public string? GoogleUsername { get; init; } - public Dictionary CustomPreferences { get; init; } = []; - public DateTimeOffset LastActive { get; init; } - public required string Sid { get; init; } - public DateTimeOffset LastSidReroll { get; init; } - public string? Timezone { get; init; } - public Snowflake SnowflakeId { get; init; } -} - -public class GoUserField -{ - public required string UserId { get; init; } - public required long Id { get; init; } - public required string Name { get; init; } - public required GoFieldEntry[] Entries { get; init; } -} - -public class GoPrideFlag -{ - public required string Id { get; init; } - public required string UserId { get; init; } - public required string Hash { get; init; } - public required string Name { get; init; } - public string? Description { get; init; } - public required Snowflake SnowflakeId { get; init; } -} - -public class GoProfileFlag -{ - public string? UserId { get; init; } - public string? MemberId { get; init; } - public required long Id { get; init; } - public required string FlagId { get; init; } -} - -public class GoFediverseApp -{ - public required int Id { get; init; } - public required string Instance { get; init; } - public required string ClientId { get; init; } - public required string ClientSecret { get; init; } - public required string InstanceType { get; init; } - - public FediverseInstanceType TypeToEnum() => - InstanceType switch - { - "sharkey" => FediverseInstanceType.MisskeyApi, - "firefish" => FediverseInstanceType.MisskeyApi, - "pixelfed" => FediverseInstanceType.MastodonApi, - "mastodon" => FediverseInstanceType.MastodonApi, - "pleroma" => FediverseInstanceType.MastodonApi, - "akkoma" => FediverseInstanceType.MastodonApi, - "misskey" => FediverseInstanceType.MastodonApi, - "gotosocial" => FediverseInstanceType.MastodonApi, - "calckey" => FediverseInstanceType.MisskeyApi, - "foundkey" => FediverseInstanceType.MisskeyApi, - // this should never happen but if it does we just fall back to the mastodon api - // basically everything but misskey forks implement it anyway :3 - _ => FediverseInstanceType.MastodonApi, - }; -} - -public record GoFieldEntry(string Value, string Status); - -public record GoPronounEntry(string Pronouns, string? DisplayText, string Status); - -public record GoCustomPreference( - string Icon, - string Tooltip, - string PreferenceSize, - bool Muted, - bool Favourite -); diff --git a/Foxnouns.DataMigrator/Program.cs b/Foxnouns.DataMigrator/Program.cs deleted file mode 100644 index 307cda5..0000000 --- a/Foxnouns.DataMigrator/Program.cs +++ /dev/null @@ -1,101 +0,0 @@ -using System.Globalization; -using Foxnouns.Backend; -using Foxnouns.Backend.Database; -using Foxnouns.Backend.Database.Models; -using Foxnouns.Backend.Extensions; -using Foxnouns.DataMigrator.Models; -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Configuration; -using Npgsql; -using Serilog; -using Serilog.Sinks.SystemConsole.Themes; - -namespace Foxnouns.DataMigrator; - -internal class Program -{ - public static async Task Main(string[] args) - { - // Create logger and get configuration - Log.Logger = new LoggerConfiguration() - .MinimumLevel.Debug() - .WriteTo.Console(theme: AnsiConsoleTheme.Sixteen) - .CreateLogger(); - - Config config = - new ConfigurationBuilder() - .AddConfiguration() - .Build() - // Get the configuration as our config class - .Get() ?? new Config(); - - NpgsqlConnection conn = await GoDatabase.GetConnectionAsync(); - // just reuse the design time factory so we don't have to copy this - DatabaseContext context = new DesignTimeDatabaseContextFactory().CreateDbContext(args); - - await context.Database.MigrateAsync(); - - Log.Information("Migrating applications"); - Dictionary appIds = await MigrateAppsAsync(conn, context); - - Log.Information("Migrating users"); - List users = await Queries.GetUsersAsync(conn); - List userFields = await Queries.GetUserFieldsAsync(conn); - List memberFields = await Queries.GetMemberFieldsAsync(conn); - List prideFlags = await Queries.GetUserFlagsAsync(conn); - List userFlags = await Queries.GetUserProfileFlagsAsync(conn); - List memberFlags = await Queries.GetMemberProfileFlagsAsync(conn); - Log.Information("Migrating {Count} users", users.Count); - foreach ((GoUser user, int i) in users.Select((user, i) => (user, i))) - { - Log.Debug( - "Migrating user #{Index}/{Count}: {Id}/{SnowflakeId}", - i, - users.Count, - user.Id, - user.SnowflakeId - ); - await new UserMigrator( - conn, - context, - user, - appIds, - userFields, - memberFields, - prideFlags, - userFlags, - memberFlags - ).MigrateAsync(); - } - - await context.SaveChangesAsync(); - Log.Information("Migration complete!"); - } - - private static async Task> MigrateAppsAsync( - NpgsqlConnection conn, - DatabaseContext context - ) - { - List goApps = await Queries.GetFediverseAppsAsync(conn); - var appIds = new Dictionary(); - foreach (GoFediverseApp app in goApps) - { - Log.Debug("Migrating application for {Domain}", app.Instance); - Snowflake id = SnowflakeGenerator.Instance.GenerateSnowflake(); - appIds[app.Id] = id; - context.FediverseApplications.Add( - new FediverseApplication - { - Id = id, - Domain = app.Instance.ToLower(CultureInfo.InvariantCulture), - ClientId = app.ClientId, - ClientSecret = app.ClientSecret, - InstanceType = app.TypeToEnum(), - } - ); - } - - return appIds; - } -} diff --git a/Foxnouns.DataMigrator/Queries.cs b/Foxnouns.DataMigrator/Queries.cs deleted file mode 100644 index 0d6e71f..0000000 --- a/Foxnouns.DataMigrator/Queries.cs +++ /dev/null @@ -1,34 +0,0 @@ -using Coravel.Mailer.Mail.Helpers; -using Dapper; -using Foxnouns.Backend.Database; -using Foxnouns.Backend.Database.Models; -using Foxnouns.DataMigrator.Models; -using NodaTime.Extensions; -using Npgsql; - -namespace Foxnouns.DataMigrator; - -public static class Queries -{ - public static async Task> GetFediverseAppsAsync(NpgsqlConnection conn) => - (await conn.QueryAsync("select * from fediverse_apps")).ToList(); - - public static async Task> GetUsersAsync(NpgsqlConnection conn) => - (await conn.QueryAsync("select * from users order by id")).ToList(); - - public static async Task> GetUserFieldsAsync(NpgsqlConnection conn) => - (await conn.QueryAsync("select * from user_fields order by id")).ToList(); - - public static async Task> GetMemberFieldsAsync(NpgsqlConnection conn) => - (await conn.QueryAsync("select * from member_fields order by id")).ToList(); - - public static async Task> GetUserProfileFlagsAsync(NpgsqlConnection conn) => - (await conn.QueryAsync("select * from user_flags order by id")).ToList(); - - public static async Task> GetMemberProfileFlagsAsync( - NpgsqlConnection conn - ) => (await conn.QueryAsync("select * from member_flags order by id")).ToList(); - - public static async Task> GetUserFlagsAsync(NpgsqlConnection conn) => - (await conn.QueryAsync("select * from pride_flags order by id")).ToList(); -} diff --git a/Foxnouns.DataMigrator/UserMigrator.cs b/Foxnouns.DataMigrator/UserMigrator.cs deleted file mode 100644 index 0263c47..0000000 --- a/Foxnouns.DataMigrator/UserMigrator.cs +++ /dev/null @@ -1,261 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using Dapper; -using Foxnouns.Backend.Database; -using Foxnouns.Backend.Database.Models; -using Foxnouns.Backend.Utils; -using Foxnouns.DataMigrator.Models; -using NodaTime.Extensions; -using Npgsql; -using Serilog; - -namespace Foxnouns.DataMigrator; - -public class UserMigrator( - NpgsqlConnection conn, - DatabaseContext context, - GoUser goUser, - Dictionary fediverseApplicationIds, - List userFields, - List memberFields, - List prideFlags, - List userFlags, - List memberFlags -) -{ - private readonly Dictionary _preferenceIds = new(); - private readonly Dictionary _flagIds = new(); - private User? _user; - - public async Task MigrateAsync() - { - CreateNewUser(); - MigrateFlags(); - await MigrateMembersAsync(); - } - - [MemberNotNull(nameof(_user))] - private void CreateNewUser() - { - _user = new User - { - Id = goUser.SnowflakeId, - Username = goUser.Username, - DisplayName = goUser.DisplayName, - Bio = goUser.Bio, - Links = goUser.Links ?? [], - - Deleted = goUser.DeletedAt != null, - DeletedAt = goUser.DeletedAt?.ToInstant(), - DeletedBy = goUser.SelfDelete == true ? null : goUser.SnowflakeId, - - Names = goUser.Names.Select(ConvertFieldEntry).ToList(), - Pronouns = goUser.Pronouns.Select(ConvertPronoun).ToList(), - Avatar = goUser.Avatar, - - Role = goUser.IsAdmin ? UserRole.Admin : UserRole.User, - MemberTitle = goUser.MemberTitle, - ListHidden = goUser.ListPrivate, - CustomPreferences = ConvertPreferences(), - LastActive = goUser.LastActive.ToInstant(), - Sid = goUser.Sid, - LastSidReroll = goUser.LastSidReroll.ToInstant(), - Timezone = goUser.Timezone, - Fields = userFields - .Where(f => f.UserId == goUser.Id) - .Select(f => new Field - { - Name = f.Name, - Entries = f.Entries.Select(ConvertFieldEntry).ToArray(), - }) - .ToList(), - }; - context.Users.Add(_user); - - // Create the user's auth methods - if (goUser.Discord != null) - { - context.AuthMethods.Add( - new AuthMethod - { - Id = SnowflakeGenerator.Instance.GenerateSnowflake(_user.Id.Time), - UserId = _user.Id, - AuthType = AuthType.Discord, - RemoteId = goUser.Discord, - RemoteUsername = goUser.DiscordUsername ?? "(unknown)", - } - ); - } - - if (goUser.Google != null) - { - context.AuthMethods.Add( - new AuthMethod - { - Id = SnowflakeGenerator.Instance.GenerateSnowflake(_user.Id.Time), - UserId = _user.Id, - AuthType = AuthType.Google, - RemoteId = goUser.Google, - RemoteUsername = goUser.GoogleUsername ?? "(unknown)", - } - ); - } - - if (goUser.Tumblr != null) - { - context.AuthMethods.Add( - new AuthMethod - { - Id = SnowflakeGenerator.Instance.GenerateSnowflake(_user.Id.Time), - UserId = _user.Id, - AuthType = AuthType.Tumblr, - RemoteId = goUser.Tumblr, - RemoteUsername = goUser.TumblrUsername ?? "(unknown)", - } - ); - } - - if (goUser.Fediverse != null) - { - context.AuthMethods.Add( - new AuthMethod - { - Id = SnowflakeGenerator.Instance.GenerateSnowflake(_user.Id.Time), - UserId = _user.Id, - AuthType = AuthType.Fediverse, - RemoteId = goUser.Fediverse, - RemoteUsername = goUser.FediverseUsername ?? "(unknown)", - FediverseApplicationId = fediverseApplicationIds[goUser.FediverseAppId!.Value], - } - ); - } - } - - private void MigrateFlags() - { - foreach (GoPrideFlag flag in prideFlags.Where(f => f.UserId == goUser.Id)) - { - _flagIds[flag.Id] = flag.SnowflakeId; - context.PrideFlags.Add( - new PrideFlag - { - Id = flag.SnowflakeId, - UserId = _user!.Id, - Hash = flag.Hash, - Name = flag.Name, - Description = flag.Description, - } - ); - } - - context.UserFlags.AddRange( - userFlags - .Where(f => f.UserId == goUser.Id) - .Where(f => _flagIds.ContainsKey(f.FlagId)) - .Select(f => new UserFlag { UserId = _user!.Id, PrideFlagId = _flagIds[f.FlagId] }) - ); - } - - private async Task MigrateMembersAsync() - { - List members = ( - await conn.QueryAsync( - "select * from members where user_id = @Id", - new { goUser.Id } - ) - ).ToList(); - - foreach (GoMember member in members) - { - Log.Debug("Migrating member {Id}/{SnowflakeId}", member.Id, member.SnowflakeId); - MigrateMember(member); - } - } - - private void MigrateMember(GoMember goMember) - { - var names = goMember.Names.Select(ConvertFieldEntry).ToList(); - var pronouns = goMember.Pronouns.Select(ConvertPronoun).ToList(); - var fields = memberFields - .Where(f => f.MemberId == goMember.Id) - .Select(f => new Field - { - Name = f.Name, - Entries = f.Entries.Select(ConvertFieldEntry).ToArray(), - }) - .ToList(); - - var member = new Member - { - Id = goMember.SnowflakeId, - UserId = _user!.Id, - Name = goMember.Name, - Sid = goMember.Sid, - DisplayName = goMember.DisplayName, - Bio = goMember.Bio, - Avatar = goMember.Avatar, - Links = goMember.Links ?? [], - Unlisted = goMember.Unlisted, - Names = names, - Pronouns = pronouns, - Fields = fields, - }; - context.Members.Add(member); - - context.MemberFlags.AddRange( - memberFlags - .Where(f => f.MemberId == goMember.Id) - .Select(f => new MemberFlag - { - MemberId = member.Id, - PrideFlagId = _flagIds[f.FlagId], - }) - ); - } - - private Dictionary ConvertPreferences() - { - var prefs = new Dictionary(); - - foreach ((string id, GoCustomPreference goPref) in goUser.CustomPreferences) - { - Snowflake newId = SnowflakeGenerator.Instance.GenerateSnowflake( - goUser.SnowflakeId.Time - ); - _preferenceIds[id] = newId; - prefs[newId] = new User.CustomPreference - { - Icon = goPref.Icon, - Tooltip = goPref.Tooltip, - Muted = goPref.Muted, - Favourite = goPref.Favourite, - Size = goPref.PreferenceSize switch - { - "large" => PreferenceSize.Large, - "normal" => PreferenceSize.Normal, - "small" => PreferenceSize.Small, - _ => PreferenceSize.Normal, - }, - }; - } - - return prefs; - } - - private FieldEntry ConvertFieldEntry(GoFieldEntry entry) => - new() { Value = entry.Value, Status = ConvertPreferenceId(entry.Status) }; - - private Pronoun ConvertPronoun(GoPronounEntry entry) => - new() - { - Value = entry.Pronouns, - DisplayText = entry.DisplayText, - Status = ConvertPreferenceId(entry.Status), - }; - - private string ConvertPreferenceId(string id) - { - if (_preferenceIds.TryGetValue(id, out Snowflake preferenceId)) - return preferenceId.ToString(); - return ValidationUtils.DefaultStatusOptions.Contains(id) ? id : "okay"; - } -} diff --git a/Foxnouns.NET.sln b/Foxnouns.NET.sln index de091c2..aec8ae7 100644 --- a/Foxnouns.NET.sln +++ b/Foxnouns.NET.sln @@ -5,7 +5,7 @@ VisualStudioVersion = 17.0.31903.59 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Foxnouns.Backend", "Foxnouns.Backend\Foxnouns.Backend.csproj", "{439E3E38-5AEF-4F73-AD57-E32057B3FC7F}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Foxnouns.DataMigrator", "Foxnouns.DataMigrator\Foxnouns.DataMigrator.csproj", "{1909CCB6-F9B0-4C36-9618-82CC3A41C5B0}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetImporter", "migrators\NetImporter\NetImporter.csproj", "{FBCF80EE-624F-43AF-8122-230B5447940C}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -20,9 +20,9 @@ Global {439E3E38-5AEF-4F73-AD57-E32057B3FC7F}.Debug|Any CPU.Build.0 = Debug|Any CPU {439E3E38-5AEF-4F73-AD57-E32057B3FC7F}.Release|Any CPU.ActiveCfg = Release|Any CPU {439E3E38-5AEF-4F73-AD57-E32057B3FC7F}.Release|Any CPU.Build.0 = Release|Any CPU - {1909CCB6-F9B0-4C36-9618-82CC3A41C5B0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {1909CCB6-F9B0-4C36-9618-82CC3A41C5B0}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1909CCB6-F9B0-4C36-9618-82CC3A41C5B0}.Release|Any CPU.ActiveCfg = Release|Any CPU - {1909CCB6-F9B0-4C36-9618-82CC3A41C5B0}.Release|Any CPU.Build.0 = Release|Any CPU + {FBCF80EE-624F-43AF-8122-230B5447940C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FBCF80EE-624F-43AF-8122-230B5447940C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FBCF80EE-624F-43AF-8122-230B5447940C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FBCF80EE-624F-43AF-8122-230B5447940C}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/docker-compose.yml b/docker-compose.yml index 4fc94bb..5606760 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,11 +9,7 @@ services: - "Database:EnablePooling=true" - "Host=0.0.0.0" - "Port=5000" - - "Logging:MetricsPort=5001" restart: unless-stopped - ports: - - "5006:5000" - - "5007:5001" volumes: - ./docker/config.ini:/app/config.ini diff --git a/migrators/NetImporter/ImportUser.cs b/migrators/NetImporter/ImportUser.cs new file mode 100644 index 0000000..d1c264a --- /dev/null +++ b/migrators/NetImporter/ImportUser.cs @@ -0,0 +1,192 @@ +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using Foxnouns.Backend.Database; +using Foxnouns.Backend.Database.Models; +using Microsoft.EntityFrameworkCore; +using NodaTime; +using NodaTime.Extensions; +using Serilog; + +namespace NetImporter; + +public static class Users +{ + public static async Task ImportUsers(string filename) + { + await using var db = await NetImporter.GetContextAsync(); + await db.Database.ExecuteSqlRawAsync("SELECT 1"); + + var stopwatch = new Stopwatch(); + stopwatch.Start(); + + var users = NetImporter + .ReadFromFile(filename) + .Output.Select(ConvertUser) + .ToList(); + db.AddRange(users); + await db.SaveChangesAsync(); + + stopwatch.Stop(); + Log.Information( + "Imported {Count} users in {Duration}", + users.Count, + stopwatch.ElapsedDuration() + ); + } + + private static User ConvertUser(ImportUser oldUser) + { + var user = new User + { + Id = oldUser.Id, + Username = oldUser.Username, + DisplayName = oldUser.DisplayName, + Bio = oldUser.Bio, + MemberTitle = oldUser.MemberTitle, + LastActive = oldUser.LastActive.ToInstant(), + Avatar = oldUser.AvatarHash, + Links = oldUser.Links ?? [], + + Role = oldUser.ParseRole(), + Deleted = oldUser.Deleted, + DeletedAt = oldUser.DeletedAt?.ToInstant(), + DeletedBy = null, + }; + + if (oldUser is { DiscordId: not null, DiscordUsername: not null }) + { + user.AuthMethods.Add( + new AuthMethod + { + Id = SnowflakeGenerator.Instance.GenerateSnowflake(), + AuthType = AuthType.Discord, + RemoteId = oldUser.DiscordId, + RemoteUsername = oldUser.DiscordUsername, + } + ); + } + + if (oldUser is { TumblrId: not null, TumblrUsername: not null }) + { + user.AuthMethods.Add( + new AuthMethod + { + Id = SnowflakeGenerator.Instance.GenerateSnowflake(), + AuthType = AuthType.Tumblr, + RemoteId = oldUser.TumblrId, + RemoteUsername = oldUser.TumblrUsername, + } + ); + } + + if (oldUser is { GoogleId: not null, GoogleUsername: not null }) + { + user.AuthMethods.Add( + new AuthMethod + { + Id = SnowflakeGenerator.Instance.GenerateSnowflake(), + AuthType = AuthType.Google, + RemoteId = oldUser.GoogleId, + RemoteUsername = oldUser.GoogleUsername, + } + ); + } + + // Convert all custom preference UUIDs to snowflakes + var prefMapping = new Dictionary(); + foreach (var (key, value) in oldUser.CustomPreferences) + { + var newKey = SnowflakeGenerator.Instance.GenerateSnowflake(); + prefMapping[key] = newKey; + user.CustomPreferences[newKey] = value; + } + + foreach (var name in oldUser.Names ?? []) + { + user.Names.Add( + new FieldEntry + { + Value = name.Value, + Status = prefMapping.TryGetValue(name.Status, out var newStatus) + ? newStatus.ToString() + : name.Status, + } + ); + } + + foreach (var pronoun in oldUser.Pronouns ?? []) + { + user.Pronouns.Add( + new Pronoun + { + Value = pronoun.Value, + DisplayText = pronoun.DisplayText, + Status = prefMapping.TryGetValue(pronoun.Status, out var newStatus) + ? newStatus.ToString() + : pronoun.Status, + } + ); + } + + foreach (var field in oldUser.Fields ?? []) + { + var entries = field + .Entries.Select(entry => new FieldEntry + { + Value = entry.Value, + Status = prefMapping.TryGetValue(entry.Status, out var newStatus) + ? newStatus.ToString() + : entry.Status, + }) + .ToList(); + + user.Fields.Add(new Field { Name = field.Name, Entries = entries.ToArray() }); + } + + Log.Debug("Converted user {UserId}", oldUser.Id); + + return user; + } + + [SuppressMessage("ReSharper", "ClassNeverInstantiated.Local")] + private record ImportUser( + Snowflake Id, + string Sid, + string Username, + string? DisplayName, + string? Bio, + string? MemberTitle, + OffsetDateTime LastActive, + string? AvatarHash, + string[]? Links, + FieldEntry[]? Names, + Pronoun[]? Pronouns, + Field[]? Fields, + string? DiscordId, + string? DiscordUsername, + string? FediverseId, + string? FediverseUsername, + long? FediverseAppId, + string? TumblrId, + string? TumblrUsername, + string? GoogleId, + string? GoogleUsername, + bool MemberListHidden, + string? Timezone, + string Role, + bool Deleted, + OffsetDateTime? DeletedAt, + string? DeleteReason, + Dictionary CustomPreferences + ) + { + public UserRole ParseRole() => + Role switch + { + "USER" => UserRole.User, + "MODERATOR" => UserRole.Moderator, + "ADMIN" => UserRole.Admin, + _ => UserRole.User, + }; + } +} diff --git a/migrators/NetImporter/NetImporter.cs b/migrators/NetImporter/NetImporter.cs new file mode 100644 index 0000000..7a0ddf6 --- /dev/null +++ b/migrators/NetImporter/NetImporter.cs @@ -0,0 +1,89 @@ +using Foxnouns.Backend; +using Foxnouns.Backend.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; +using NodaTime; +using NodaTime.Serialization.JsonNet; +using Serilog; +using Serilog.Events; + +namespace NetImporter; + +internal static class NetImporter +{ + public static async Task Main(string[] args) + { + Log.Logger = new LoggerConfiguration() + .Enrich.FromLogContext() + .MinimumLevel.Debug() + .MinimumLevel.Override("Microsoft", LogEventLevel.Information) + .MinimumLevel.Override( + "Microsoft.EntityFrameworkCore.Database.Command", + LogEventLevel.Information + ) + .WriteTo.Console() + .CreateLogger(); + + switch (args.Length) + { + case < 2: + Console.WriteLine("Not enough arguments. Usage: "); + return; + case > 2: + Console.WriteLine("Too many arguments. Usage: "); + return; + } + + switch (args[0].ToLowerInvariant()) + { + case "users": + await Users.ImportUsers(args[1]); + break; + default: + Console.WriteLine("Invalid type. Valid types are: users"); + break; + } + } + + internal static async Task GetContextAsync() + { + var connString = Environment.GetEnvironmentVariable("DATABASE"); + if (connString == null) + throw new Exception("$DATABASE not set, must be an ADO.NET connection string"); + + var loggerFactory = new LoggerFactory().AddSerilog(Log.Logger); + var config = new Config { Database = new Config.DatabaseConfig { Url = connString } }; + + var dataSource = DatabaseContext.BuildDataSource(config); + var options = DatabaseContext + .BuildOptions(new DbContextOptionsBuilder(), dataSource, loggerFactory) + .Options; + var db = new DatabaseContext(options); + + if ((await db.Database.GetPendingMigrationsAsync()).Any()) + { + Log.Fatal("Database needs to be migrated first"); + } + + return db; + } + + private static readonly JsonSerializerSettings Settings = new JsonSerializerSettings + { + ContractResolver = new DefaultContractResolver + { + NamingStrategy = new SnakeCaseNamingStrategy(), + }, + }.ConfigureForNodaTime(DateTimeZoneProviders.Tzdb); + + internal static Input ReadFromFile(string path) + { + var data = File.ReadAllText(path); + return JsonConvert.DeserializeObject>(data, Settings) + ?? throw new Exception("Invalid input file"); + } +} + +internal record Input(List Output, List Skipped); diff --git a/migrators/NetImporter/NetImporter.csproj b/migrators/NetImporter/NetImporter.csproj new file mode 100644 index 0000000..d27f214 --- /dev/null +++ b/migrators/NetImporter/NetImporter.csproj @@ -0,0 +1,26 @@ + + + + Exe + net9.0 + enable + enable + + + + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + diff --git a/migrators/go-exporter/.gitignore b/migrators/go-exporter/.gitignore new file mode 100644 index 0000000..94a2dd1 --- /dev/null +++ b/migrators/go-exporter/.gitignore @@ -0,0 +1 @@ +*.json \ No newline at end of file diff --git a/migrators/go-exporter/go.mod b/migrators/go-exporter/go.mod new file mode 100644 index 0000000..aaadec0 --- /dev/null +++ b/migrators/go-exporter/go.mod @@ -0,0 +1,82 @@ +module code.vulpine.solutions/sam/Foxnouns.NET/go-exporter + +go 1.22.6 + +require ( + cloud.google.com/go/compute v1.23.2 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + codeberg.org/pronounscc/pronouns.cc v0.6.5-0.20240213162950-5fcd87a94a07 // indirect + emperror.dev/errors v0.8.1 // indirect + github.com/Masterminds/squirrel v1.5.4 // indirect + github.com/aarondl/json v0.0.0-20221020222930-8b0db17ef1bf // indirect + github.com/aarondl/opt v0.0.0-20230313190023-85d93d668fec // indirect + github.com/ajg/form v1.5.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/bwmarrin/discordgo v0.27.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect + github.com/davidbyttow/govips/v2 v2.13.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/georgysavva/scany/v2 v2.0.0 // indirect + github.com/getsentry/sentry-go v0.25.0 // indirect + github.com/go-chi/chi/v5 v5.0.10 // indirect + github.com/go-chi/cors v1.2.1 // indirect + github.com/go-chi/httprate v0.7.4 // indirect + github.com/go-chi/render v1.0.3 // indirect + github.com/go-gorp/gorp/v3 v3.1.0 // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/google/uuid v1.4.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/gorilla/websocket v1.5.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgx/v5 v5.4.3 // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/joho/godotenv v1.5.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect + github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/mediocregopher/radix/v4 v4.1.4 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.63 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.17.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/rs/xid v1.5.0 // indirect + github.com/rubenv/sql-migrate v1.5.2 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/tilinna/clock v1.1.0 // indirect + github.com/toshi0607/chi-prometheus v0.1.4 // indirect + github.com/urfave/cli/v2 v2.25.7 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + go.opencensus.io v0.24.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.26.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/image v0.13.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/oauth2 v0.13.0 // indirect + golang.org/x/sync v0.4.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + google.golang.org/api v0.148.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a // indirect + google.golang.org/grpc v1.59.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/migrators/go-exporter/go.sum b/migrators/go-exporter/go.sum new file mode 100644 index 0000000..933780c --- /dev/null +++ b/migrators/go-exporter/go.sum @@ -0,0 +1,286 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go/compute v1.23.2 h1:nWEMDhgbBkBJjfpVySqU4jgWdc22PLR0o4vEexZHers= +cloud.google.com/go/compute v1.23.2/go.mod h1:JJ0atRC0J/oWYiiVBmsSsrRnh92DhZPG4hFDcR04Rns= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +codeberg.org/pronounscc/pronouns.cc v0.6.5-0.20240213162950-5fcd87a94a07 h1:Vsf1xjT8msCuNj3KbDsqs7QR614WJgYFYc6+cMsx8zM= +codeberg.org/pronounscc/pronouns.cc v0.6.5-0.20240213162950-5fcd87a94a07/go.mod h1:Y4HFkHFeIGoK+CAkig0bUxRTUm5Eprr7oP3mpVizUi0= +emperror.dev/errors v0.8.1 h1:UavXZ5cSX/4u9iyvH6aDcuGkVjeexUGJ7Ij7G4VfQT0= +emperror.dev/errors v0.8.1/go.mod h1:YcRvLPh626Ubn2xqtoprejnA5nFha+TJ+2vew48kWuE= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= +github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= +github.com/aarondl/json v0.0.0-20221020222930-8b0db17ef1bf h1:+edM69bH/X6JpYPmJYBRLanAMe1V5yRXYU3hHUovGcE= +github.com/aarondl/json v0.0.0-20221020222930-8b0db17ef1bf/go.mod h1:FZqLhJSj2tg0ZN48GB1zvj00+ZYcHPqgsC7yzcgCq6k= +github.com/aarondl/opt v0.0.0-20230313190023-85d93d668fec h1:2NFk5fe52cHyRcUnXSs4CSEAqm+rL/hr3AdflBE3VPU= +github.com/aarondl/opt v0.0.0-20230313190023-85d93d668fec/go.mod h1:l4/5NZtYd/SIohsFhaJQQe+sPOTG22furpZ5FvcYOzk= +github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bwmarrin/discordgo v0.27.1 h1:ib9AIc/dom1E/fSIulrBwnez0CToJE113ZGt4HoliGY= +github.com/bwmarrin/discordgo v0.27.1/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davidbyttow/govips/v2 v2.13.0 h1:5MK9ZcXZC5GzUR9Ca8fJwOYqMgll/H096ec0PJP59QM= +github.com/davidbyttow/govips/v2 v2.13.0/go.mod h1:LPTrwWtNa5n4yl9UC52YBOEGdZcY5hDTP4Ms2QWasTw= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/georgysavva/scany/v2 v2.0.0 h1:RGXqxDv4row7/FYoK8MRXAZXqoWF/NM+NP0q50k3DKU= +github.com/georgysavva/scany/v2 v2.0.0/go.mod h1:sigOdh+0qb/+aOs3TVhehVT10p8qJL7K/Zhyz8vWo38= +github.com/getsentry/sentry-go v0.25.0 h1:q6Eo+hS+yoJlTO3uu/azhQadsD8V+jQn2D8VvX1eOyI= +github.com/getsentry/sentry-go v0.25.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/go-chi/chi/v5 v5.0.10 h1:rLz5avzKpjqxrYwXNfmjkrYYXOyLJd37pz53UFHC6vk= +github.com/go-chi/chi/v5 v5.0.10/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4= +github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= +github.com/go-chi/httprate v0.7.4 h1:a2GIjv8he9LRf3712zxxnRdckQCm7I8y8yQhkJ84V6M= +github.com/go-chi/httprate v0.7.4/go.mod h1:6GOYBSwnpra4CQfAKXu8sQZg+nZ0M1g9QnyFvxrAB8A= +github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4= +github.com/go-chi/render v1.0.3/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0= +github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= +github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY= +github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= +github.com/mediocregopher/radix/v4 v4.1.4 h1:Uze6DEbEAvL+VHXUEu/EDBTkUk5CLct5h3nVSGpc6Ts= +github.com/mediocregopher/radix/v4 v4.1.4/go.mod h1:ajchozX/6ELmydxWeWM6xCFHVpZ4+67LXHOTOVR0nCE= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ= +github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rubenv/sql-migrate v1.5.2 h1:bMDqOnrJVV/6JQgQ/MxOpU+AdO8uzYYA/TxFUBzFtS0= +github.com/rubenv/sql-migrate v1.5.2/go.mod h1:H38GW8Vqf8F0Su5XignRyaRcbXbJunSWxs+kmzlg0Is= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/tilinna/clock v1.0.2/go.mod h1:ZsP7BcY7sEEz7ktc0IVy8Us6boDrK8VradlKRUGfOao= +github.com/tilinna/clock v1.1.0 h1:6IQQQCo6KoBxVudv6gwtY8o4eDfhHo8ojA5dP0MfhSs= +github.com/tilinna/clock v1.1.0/go.mod h1:ZsP7BcY7sEEz7ktc0IVy8Us6boDrK8VradlKRUGfOao= +github.com/toshi0607/chi-prometheus v0.1.4 h1:5KpqJrmdvMvbfU0JiL9ghOTbe8S9sgHDCCQvXgnyoJo= +github.com/toshi0607/chi-prometheus v0.1.4/go.mod h1:E++tBjqpDsvGWjLYdcFd5rvqJ7HG8wwBux+M6gyIL/Q= +github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= +github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= +golang.org/x/image v0.13.0 h1:3cge/F/QTkNLauhf2QoE9zp+7sr+ZcL4HnoZmdwg9sg= +golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.148.0 h1:HBq4TZlN4/1pNcu0geJZ/Q50vIwIXT532UIMYoo0vOs= +google.golang.org/api v0.148.0/go.mod h1:8/TBgwaKjfqTdacOJrOv2+2Q6fBDU1uHKK06oGSkxzU= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a h1:a2MQQVoTo96JC9PMGtGBymLp7+/RzpFc2yX/9WfFg1c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/migrators/go-exporter/main.go b/migrators/go-exporter/main.go new file mode 100644 index 0000000..ac749ac --- /dev/null +++ b/migrators/go-exporter/main.go @@ -0,0 +1,101 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "log" + "os" + "time" + + "codeberg.org/pronounscc/pronouns.cc/backend/db" + "github.com/Masterminds/squirrel" + "github.com/jackc/pgx/v5/pgxpool" +) + +var exportWhat = flag.String("export", "", "What to export") +var exportLimit = flag.Int64("limit", 0, "Number of items to export for testing") + +var sq = squirrel.StatementBuilder.PlaceholderFormat(squirrel.Dollar) +var oldDB *db.DB + +type Output struct { + Output any `json:"output"` // The output array + Skipped []string `json:"skipped"` // IDs of skipped items +} + +func main() { + flag.Parse() + + dbUrl := os.Getenv("DATABASE") + + pool, err := pgxpool.New(context.Background(), dbUrl) + if err != nil { + log.Fatalf("creating database pool: %v\n", err) + } + + oldDB = &db.DB{ + Pool: pool, + } + + switch *exportWhat { + case "users": + exportUsers() + default: + fmt.Printf("invalid export type %q\nvalid export types are: users\n", *exportWhat) + } +} + +func exportUsers() { + filename := fmt.Sprintf("users-output-%v.json", time.Now().Unix()) + f, err := os.Create(filename) + if err != nil { + log.Fatalf("error opening output file %q: %v\n", filename, err) + } + defer f.Close() + + users, err := getUsers() + if err != nil { + log.Fatalf("getting users from database: %v\n", err) + } + + log.Println("converting users") + + start := time.Now() + + var ( + newUsers []NewUser + skipped []string + ) + for i, u := range users { + newUser, err := userToNewUser(u) + if err != nil { + log.Printf("error converting user %v: %v\n", u.SnowflakeID, err) + skipped = append(skipped, u.SnowflakeID.String()) + continue + } + + newUsers = append(newUsers, newUser) + log.Printf("converted user %7d (%v)\n", i+1, u.SnowflakeID) + } + + log.Printf("converted users in %v (skipped: %v)\n", time.Since(start).Round(time.Millisecond), len(skipped)) + + b, err := json.MarshalIndent(Output{Output: newUsers, Skipped: skipped}, "", " ") + if err != nil { + log.Fatalf("error marshaling json: %v\n", err) + } + + _, err = f.Write(b) + if err != nil { + log.Fatalf("writing file: %v\n", err) + } + + err = f.Sync() + if err != nil { + log.Fatalf("syncing file: %v\n", err) + } + + fmt.Printf("\n\nexported %v users! filename: %q\n", len(newUsers), filename) +} diff --git a/migrators/go-exporter/user.go b/migrators/go-exporter/user.go new file mode 100644 index 0000000..1af246e --- /dev/null +++ b/migrators/go-exporter/user.go @@ -0,0 +1,134 @@ +package main + +import ( + "context" + "time" + + "codeberg.org/pronounscc/pronouns.cc/backend/db" + "emperror.dev/errors" + "github.com/georgysavva/scany/v2/pgxscan" +) + +func getUsers() (u []db.User, err error) { + q := sq.Select("*", "(SELECT instance FROM fediverse_apps WHERE id = users.fediverse_app_id) AS fediverse_instance"). + From("users").OrderBy("snowflake_id ASC") + + if *exportLimit != 0 { + q = q.Limit(uint64(*exportLimit)) + } + + sql, args, err := q.ToSql() + if err != nil { + return u, errors.Wrap(err, "building sql") + } + + err = pgxscan.Select(context.Background(), oldDB, &u, sql, args...) + if err != nil { + return u, errors.Wrap(err, "getting users from db") + } + + return u, nil +} + +func userToNewUser(u db.User) (NewUser, error) { + fields, err := oldDB.UserFields(context.Background(), u.ID) + if err != nil { + return NewUser{}, errors.Wrap(err, "getting fields") + } + + new := NewUser{ + ID: u.SnowflakeID.String(), + SID: u.SID, + Username: u.Username, + DisplayName: u.DisplayName, + Bio: u.Bio, + MemberTitle: u.MemberTitle, + LastActive: u.LastActive, + Avatar: u.Avatar, + Links: u.Links, + Names: u.Names, + + Discord: u.Discord, + DiscordUsername: u.DiscordUsername, + Fediverse: u.Fediverse, + FediverseUsername: u.FediverseUsername, + FediverseAppID: u.FediverseAppID, + Tumblr: u.Tumblr, + TumblrUsername: u.TumblrUsername, + Google: u.Google, + GoogleUsername: u.GoogleUsername, + + MemberListHidden: u.ListPrivate, + Timezone: u.Timezone, + Role: "USER", + + Deleted: u.DeletedAt != nil, + DeletedAt: u.DeletedAt, + DeleteReason: u.DeleteReason, + + CustomPreferences: u.CustomPreferences, + } + + if u.IsAdmin { + new.Role = "ADMIN" + } + + for _, p := range u.Pronouns { + new.Pronouns = append(new.Pronouns, NewPronoun{Value: p.Pronouns, Status: string(p.Status), DisplayText: p.DisplayText}) + } + + for _, f := range fields { + new.Fields = append(new.Fields, NewField{Name: f.Name, Entries: f.Entries}) + } + + return new, nil +} + +type NewUser struct { + ID string `json:"id"` + SID string `json:"sid"` + Username string `json:"username"` + DisplayName *string `json:"display_name"` + Bio *string `json:"bio"` + MemberTitle *string `json:"member_title"` + LastActive time.Time `json:"last_active"` + Avatar *string `json:"avatar_hash"` + Links []string `json:"links"` + Names []db.FieldEntry `json:"names"` + Pronouns []NewPronoun `json:"pronouns"` + Fields []NewField `json:"fields"` + + Discord *string `json:"discord_id"` + DiscordUsername *string `json:"discord_username"` + + Fediverse *string `json:"fediverse_id"` + FediverseUsername *string `json:"fediverse_username"` + FediverseAppID *int64 `json:"fediverse_app_id"` + + Tumblr *string `json:"tumblr_id"` + TumblrUsername *string `json:"tumblr_username"` + + Google *string `json:"google_id"` + GoogleUsername *string `json:"google_username"` + + MemberListHidden bool `json:"member_list_hidden"` + Timezone *string `json:"timezone"` + Role string `json:"role"` // one of USER or ADMIN + + Deleted bool `json:"deleted"` + DeletedAt *time.Time `json:"deleted_at"` + DeleteReason *string `json:"delete_reason"` // TODO: this should be imported as a warning + + CustomPreferences db.CustomPreferences `json:"custom_preferences"` +} + +type NewPronoun struct { + Value string `json:"value"` + Status string `json:"status"` + DisplayText *string `json:"display_text"` +} + +type NewField struct { + Name string `json:"name"` + Entries []db.FieldEntry `json:"entries"` +}