feat: split migration into batches

.gitignore

@@ -10,3 +10,5 @@ proxy-config.json
 docker/config.ini
 docker/proxy-config.json
 docker/frontend.env
+
+Foxnouns.DataMigrator/apps.json

Foxnouns.Backend/Database/Migrations/20241217195351_AddFediAppForceRefresh.cs

@@ -0,0 +1,51 @@
+using System.Collections.Generic;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Foxnouns.Backend.Database.Migrations
+{
+    /// <inheritdoc />
+    [DbContext(typeof(DatabaseContext))]
+    [Migration("20241217195351_AddFediAppForceRefresh")]
+    public partial class AddFediAppForceRefresh : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AlterColumn<Dictionary<string, string>>(
+                name: "localization_params",
+                table: "notifications",
+                type: "hstore",
+                nullable: false,
+                oldClrType: typeof(Dictionary<string, string>),
+                oldType: "hstore",
+                oldNullable: true
+            );
+
+            migrationBuilder.AddColumn<bool>(
+                name: "force_refresh",
+                table: "fediverse_applications",
+                type: "boolean",
+                nullable: false,
+                defaultValue: false
+            );
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropColumn(name: "force_refresh", table: "fediverse_applications");
+
+            migrationBuilder.AlterColumn<Dictionary<string, string>>(
+                name: "localization_params",
+                table: "notifications",
+                type: "hstore",
+                nullable: true,
+                oldClrType: typeof(Dictionary<string, string>),
+                oldType: "hstore"
+            );
+        }
+    }
+}

Foxnouns.Backend/Database/Migrations/DatabaseContextModelSnapshot.cs

@@ -216,6 +216,10 @@ namespace Foxnouns.Backend.Database.Migrations
                         .HasColumnType("text")
                         .HasColumnName("domain");
 
+                    b.Property<bool>("ForceRefresh")
+                        .HasColumnType("boolean")
+                        .HasColumnName("force_refresh");
+
                     b.Property<int>("InstanceType")
                         .HasColumnType("integer")
                         .HasColumnName("instance_type");
@@ -342,6 +346,7 @@ namespace Foxnouns.Backend.Database.Migrations
                         .HasColumnName("localization_key");
 
                     b.Property<Dictionary<string, string>>("LocalizationParams")
+                        .IsRequired()
                         .HasColumnType("hstore")
                         .HasColumnName("localization_params");
 
@@ -411,7 +416,7 @@ namespace Foxnouns.Backend.Database.Migrations
                     b.Property<long>("ReporterId")
                         .HasColumnType("bigint")
                         .HasColumnName("reporter_id");
-                    
+
                     b.Property<int>("Status")
                         .HasColumnType("integer")
                         .HasColumnName("status");

Foxnouns.Backend/Database/Models/FediverseApplication.cs

@@ -20,6 +20,7 @@ public class FediverseApplication : BaseModel
     public required string ClientId { get; set; }
     public required string ClientSecret { get; set; }
     public required FediverseInstanceType InstanceType { get; set; }
+    public bool ForceRefresh { get; set; }
 }
 
 public enum FediverseInstanceType

Foxnouns.Backend/Services/Auth/FediverseAuthService.cs

@@ -58,7 +58,7 @@ public partial class FediverseAuthService
     )
     {
         FediverseApplication app = await GetApplicationAsync(instance);
-        return await GenerateAuthUrlAsync(app, forceRefresh, state);
+        return await GenerateAuthUrlAsync(app, forceRefresh || app.ForceRefresh, state);
     }
 
     // thank you, gargron and syuilo, for agreeing on a name for *once* in your lives,

Foxnouns.DataMigrator/Program.cs

@@ -6,6 +6,7 @@ using Foxnouns.Backend.Extensions;
 using Foxnouns.DataMigrator.Models;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Configuration;
+using Newtonsoft.Json;
 using Npgsql;
 using Serilog;
 using Serilog.Sinks.SystemConsole.Themes;
@@ -22,6 +23,12 @@ internal class Program
             .WriteTo.Console(theme: AnsiConsoleTheme.Sixteen)
             .CreateLogger();
 
+        var minUserId = new Snowflake(0);
+        if (args.Length > 0)
+            minUserId = ulong.Parse(args[0]);
+
+        Log.Information("Starting migration from user ID {MinUserId}", minUserId);
+
         Config config =
             new ConfigurationBuilder()
                 .AddConfiguration()
@@ -35,11 +42,30 @@ internal class Program
 
         await context.Database.MigrateAsync();
 
-        Log.Information("Migrating applications");
-        Dictionary<int, Snowflake> appIds = await MigrateAppsAsync(conn, context);
+        Dictionary<int, Snowflake> appIds;
+        if (minUserId == new Snowflake(0))
+        {
+            Log.Information("Migrating applications");
+            appIds = await MigrateAppsAsync(conn, context);
+
+            string appJson = JsonConvert.SerializeObject(appIds);
+            await File.WriteAllTextAsync("apps.json", appJson);
+        }
+        else
+        {
+            Log.Information(
+                "Not the first migration, reading application IDs from {Filename}",
+                "apps.json"
+            );
+
+            string appJson = await File.ReadAllTextAsync("apps.json");
+            appIds =
+                JsonConvert.DeserializeObject<Dictionary<int, Snowflake>>(appJson)
+                ?? throw new Exception("invalid apps.json file");
+        }
 
         Log.Information("Migrating users");
-        List<GoUser> users = await Queries.GetUsersAsync(conn);
+        List<GoUser> users = await Queries.GetUsersAsync(conn, minUserId);
         List<GoUserField> userFields = await Queries.GetUserFieldsAsync(conn);
         List<GoMemberField> memberFields = await Queries.GetMemberFieldsAsync(conn);
         List<GoPrideFlag> prideFlags = await Queries.GetUserFlagsAsync(conn);
@@ -70,6 +96,12 @@ internal class Program
 
         await context.SaveChangesAsync();
         Log.Information("Migration complete!");
+        Log.Information(
+            "Migrated {Count} users, last user was {UserId}. Complete? {Complete}",
+            users.Count,
+            users.Last().SnowflakeId,
+            users.Count != 1000
+        );
     }
 
     private static async Task<Dictionary<int, Snowflake>> MigrateAppsAsync(
@@ -92,6 +124,7 @@ internal class Program
                     ClientId = app.ClientId,
                     ClientSecret = app.ClientSecret,
                     InstanceType = app.TypeToEnum(),
+                    ForceRefresh = true,
                 }
             );
         }

Foxnouns.DataMigrator/Queries.cs

@@ -13,8 +13,13 @@ public static class Queries
     public static async Task<List<GoFediverseApp>> GetFediverseAppsAsync(NpgsqlConnection conn) =>
         (await conn.QueryAsync<GoFediverseApp>("select * from fediverse_apps")).ToList();
 
-    public static async Task<List<GoUser>> GetUsersAsync(NpgsqlConnection conn) =>
-        (await conn.QueryAsync<GoUser>("select * from users order by id")).ToList();
+    public static async Task<List<GoUser>> GetUsersAsync(NpgsqlConnection conn, Snowflake minId) =>
+        (
+            await conn.QueryAsync<GoUser>(
+                "select * from users where snowflake_id > @Id order by snowflake_id limit 1000",
+                new { Id = minId.Value }
+            )
+        ).ToList();
 
     public static async Task<List<GoUserField>> GetUserFieldsAsync(NpgsqlConnection conn) =>
         (await conn.QueryAsync<GoUserField>("select * from user_fields order by id")).ToList();