From 80385893c7de4b6574c5b2041709a95b73507fe6 Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 17 Dec 2024 21:23:02 +0100 Subject: [PATCH] feat: split migration into batches --- .gitignore | 2 + .../20241217195351_AddFediAppForceRefresh.cs | 51 +++++++++++++++++++ .../DatabaseContextModelSnapshot.cs | 7 ++- .../Database/Models/FediverseApplication.cs | 1 + .../Services/Auth/FediverseAuthService.cs | 2 +- Foxnouns.DataMigrator/Program.cs | 39 ++++++++++++-- Foxnouns.DataMigrator/Queries.cs | 9 +++- 7 files changed, 104 insertions(+), 7 deletions(-) create mode 100644 Foxnouns.Backend/Database/Migrations/20241217195351_AddFediAppForceRefresh.cs diff --git a/.gitignore b/.gitignore index c61154d..5a2b908 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ proxy-config.json docker/config.ini docker/proxy-config.json docker/frontend.env + +Foxnouns.DataMigrator/apps.json diff --git a/Foxnouns.Backend/Database/Migrations/20241217195351_AddFediAppForceRefresh.cs b/Foxnouns.Backend/Database/Migrations/20241217195351_AddFediAppForceRefresh.cs new file mode 100644 index 0000000..8340273 --- /dev/null +++ b/Foxnouns.Backend/Database/Migrations/20241217195351_AddFediAppForceRefresh.cs @@ -0,0 +1,51 @@ +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Foxnouns.Backend.Database.Migrations +{ + /// + [DbContext(typeof(DatabaseContext))] + [Migration("20241217195351_AddFediAppForceRefresh")] + public partial class AddFediAppForceRefresh : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn>( + name: "localization_params", + table: "notifications", + type: "hstore", + nullable: false, + oldClrType: typeof(Dictionary), + oldType: "hstore", + oldNullable: true + ); + + migrationBuilder.AddColumn( + name: "force_refresh", + table: "fediverse_applications", + type: "boolean", + nullable: false, + defaultValue: false + ); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn(name: "force_refresh", table: "fediverse_applications"); + + migrationBuilder.AlterColumn>( + name: "localization_params", + table: "notifications", + type: "hstore", + nullable: true, + oldClrType: typeof(Dictionary), + oldType: "hstore" + ); + } + } +} diff --git a/Foxnouns.Backend/Database/Migrations/DatabaseContextModelSnapshot.cs b/Foxnouns.Backend/Database/Migrations/DatabaseContextModelSnapshot.cs index 83a90fd..79a0232 100644 --- a/Foxnouns.Backend/Database/Migrations/DatabaseContextModelSnapshot.cs +++ b/Foxnouns.Backend/Database/Migrations/DatabaseContextModelSnapshot.cs @@ -216,6 +216,10 @@ namespace Foxnouns.Backend.Database.Migrations .HasColumnType("text") .HasColumnName("domain"); + b.Property("ForceRefresh") + .HasColumnType("boolean") + .HasColumnName("force_refresh"); + b.Property("InstanceType") .HasColumnType("integer") .HasColumnName("instance_type"); @@ -342,6 +346,7 @@ namespace Foxnouns.Backend.Database.Migrations .HasColumnName("localization_key"); b.Property>("LocalizationParams") + .IsRequired() .HasColumnType("hstore") .HasColumnName("localization_params"); @@ -411,7 +416,7 @@ namespace Foxnouns.Backend.Database.Migrations b.Property("ReporterId") .HasColumnType("bigint") .HasColumnName("reporter_id"); - + b.Property("Status") .HasColumnType("integer") .HasColumnName("status"); diff --git a/Foxnouns.Backend/Database/Models/FediverseApplication.cs b/Foxnouns.Backend/Database/Models/FediverseApplication.cs index 13e1318..9c61937 100644 --- a/Foxnouns.Backend/Database/Models/FediverseApplication.cs +++ b/Foxnouns.Backend/Database/Models/FediverseApplication.cs @@ -20,6 +20,7 @@ public class FediverseApplication : BaseModel public required string ClientId { get; set; } public required string ClientSecret { get; set; } public required FediverseInstanceType InstanceType { get; set; } + public bool ForceRefresh { get; set; } } public enum FediverseInstanceType diff --git a/Foxnouns.Backend/Services/Auth/FediverseAuthService.cs b/Foxnouns.Backend/Services/Auth/FediverseAuthService.cs index 9ca7290..49afe1d 100644 --- a/Foxnouns.Backend/Services/Auth/FediverseAuthService.cs +++ b/Foxnouns.Backend/Services/Auth/FediverseAuthService.cs @@ -58,7 +58,7 @@ public partial class FediverseAuthService ) { FediverseApplication app = await GetApplicationAsync(instance); - return await GenerateAuthUrlAsync(app, forceRefresh, state); + return await GenerateAuthUrlAsync(app, forceRefresh || app.ForceRefresh, state); } // thank you, gargron and syuilo, for agreeing on a name for *once* in your lives, diff --git a/Foxnouns.DataMigrator/Program.cs b/Foxnouns.DataMigrator/Program.cs index 307cda5..e79977b 100644 --- a/Foxnouns.DataMigrator/Program.cs +++ b/Foxnouns.DataMigrator/Program.cs @@ -6,6 +6,7 @@ using Foxnouns.Backend.Extensions; using Foxnouns.DataMigrator.Models; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; +using Newtonsoft.Json; using Npgsql; using Serilog; using Serilog.Sinks.SystemConsole.Themes; @@ -22,6 +23,12 @@ internal class Program .WriteTo.Console(theme: AnsiConsoleTheme.Sixteen) .CreateLogger(); + var minUserId = new Snowflake(0); + if (args.Length > 0) + minUserId = ulong.Parse(args[0]); + + Log.Information("Starting migration from user ID {MinUserId}", minUserId); + Config config = new ConfigurationBuilder() .AddConfiguration() @@ -35,11 +42,30 @@ internal class Program await context.Database.MigrateAsync(); - Log.Information("Migrating applications"); - Dictionary appIds = await MigrateAppsAsync(conn, context); + Dictionary appIds; + if (minUserId == new Snowflake(0)) + { + Log.Information("Migrating applications"); + appIds = await MigrateAppsAsync(conn, context); + + string appJson = JsonConvert.SerializeObject(appIds); + await File.WriteAllTextAsync("apps.json", appJson); + } + else + { + Log.Information( + "Not the first migration, reading application IDs from {Filename}", + "apps.json" + ); + + string appJson = await File.ReadAllTextAsync("apps.json"); + appIds = + JsonConvert.DeserializeObject>(appJson) + ?? throw new Exception("invalid apps.json file"); + } Log.Information("Migrating users"); - List users = await Queries.GetUsersAsync(conn); + List users = await Queries.GetUsersAsync(conn, minUserId); List userFields = await Queries.GetUserFieldsAsync(conn); List memberFields = await Queries.GetMemberFieldsAsync(conn); List prideFlags = await Queries.GetUserFlagsAsync(conn); @@ -70,6 +96,12 @@ internal class Program await context.SaveChangesAsync(); Log.Information("Migration complete!"); + Log.Information( + "Migrated {Count} users, last user was {UserId}. Complete? {Complete}", + users.Count, + users.Last().SnowflakeId, + users.Count != 1000 + ); } private static async Task> MigrateAppsAsync( @@ -92,6 +124,7 @@ internal class Program ClientId = app.ClientId, ClientSecret = app.ClientSecret, InstanceType = app.TypeToEnum(), + ForceRefresh = true, } ); } diff --git a/Foxnouns.DataMigrator/Queries.cs b/Foxnouns.DataMigrator/Queries.cs index 0d6e71f..0bc14a2 100644 --- a/Foxnouns.DataMigrator/Queries.cs +++ b/Foxnouns.DataMigrator/Queries.cs @@ -13,8 +13,13 @@ public static class Queries public static async Task> GetFediverseAppsAsync(NpgsqlConnection conn) => (await conn.QueryAsync("select * from fediverse_apps")).ToList(); - public static async Task> GetUsersAsync(NpgsqlConnection conn) => - (await conn.QueryAsync("select * from users order by id")).ToList(); + public static async Task> GetUsersAsync(NpgsqlConnection conn, Snowflake minId) => + ( + await conn.QueryAsync( + "select * from users where snowflake_id > @Id order by snowflake_id limit 1000", + new { Id = minId.Value } + ) + ).ToList(); public static async Task> GetUserFieldsAsync(NpgsqlConnection conn) => (await conn.QueryAsync("select * from user_fields order by id")).ToList();