sam/Catalogger.NET
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: import/export settings, send backup of settings when leaving guild

Browse source
This commit is contained in:
sam 2024-11-08 17:12:00 +01:00
parent e6d68338db
commit db5d7bb4f8
Signed by: sam
GPG key ID: 5F3C3C1B3166639D
18 changed files with 392 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

117
Catalogger.Backend/Api/GuildsController.Backup.cs Normal file
View file

@ -0,0 +1,117 @@
// Copyright (C) 2021-present sam (starshines.gay)
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
using System.Net;
using Catalogger.Backend.Api.Middleware;
using Catalogger.Backend.Database.Models;
using Microsoft.AspNetCore.Mvc;
using NodaTime;
using Remora.Discord.API;
namespace Catalogger.Backend.Api;
public partial class GuildsController
{
[Authorize]
[HttpGet("config")]
public async Task<IActionResult> ExportConfigAsync(string id)
{
var (guildId, _) = await ParseGuildAsync(id);
var guildConfig = await guildRepository.GetAsync(guildId);
return Ok(await ToExport(guildConfig));
}
[Authorize]
[HttpPost("config")]
public async Task<IActionResult> ImportConfigAsync(
string id,
[FromBody] GuildConfigExport export
)
{
var (guildId, _) = await ParseGuildAsync(id);
if (export.Id != guildId.Value)
throw new ApiError(
HttpStatusCode.BadRequest,
ErrorCode.BadRequest,
"This backup is not from this server."
);
// Filter invites to *only* those that exist for this guild.
// Blame past me for not making (code, guild_id) a unique index >:|
var cachedInvites = (await inviteCache.TryGetAsync(guildId)).ToList();
var invites = export.Invites.Where(i => cachedInvites.Any(ci => i.Code == ci.Code));
await guildRepository.ImportConfigAsync(
guildId.Value,
export.Channels,
export.BannedSystems,
export.KeyRoles
);
await inviteRepository.ImportInvitesAsync(
guildId,
invites.Select(i => new Invite
{
Code = i.Code,
Name = i.Name,
GuildId = guildId.Value,
})
);
await watchlistRepository.ImportWatchlistAsync(
guildId,
export.Watchlist.Select(w => new Watchlist
{
GuildId = guildId.Value,
UserId = w.UserId,
ModeratorId = w.ModeratorId,
AddedAt = w.AddedAt,
Reason = w.Reason,
})
);
return NoContent();
}
private async Task<GuildConfigExport> ToExport(Database.Models.Guild config)
{
var id = DiscordSnowflake.New(config.Id);
var invites = await inviteRepository.GetGuildInvitesAsync(id);
var watchlist = await watchlistRepository.GetGuildWatchlistAsync(id);
return new GuildConfigExport(
config.Id,
config.Channels,
config.BannedSystems,
config.KeyRoles,
invites.Select(i => new InviteExport(i.Code, i.Name)),
watchlist.Select(w => new WatchlistExport(w.UserId, w.AddedAt, w.ModeratorId, w.Reason))
);
}
public record GuildConfigExport(
ulong Id,
Database.Models.Guild.ChannelConfig Channels,
string[] BannedSystems,
ulong[] KeyRoles,
IEnumerable<InviteExport> Invites,
IEnumerable<WatchlistExport> Watchlist
);
public record InviteExport(string Code, string Name);
public record WatchlistExport(ulong UserId, Instant AddedAt, ulong ModeratorId, string Reason);
}

23
Catalogger.Backend/Api/GuildsController.Remove.cs
View file

@ -14,12 +14,15 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
using System.Net;
using System.Text;
using System.Text.Json;
using Catalogger.Backend.Api.Middleware;
using Catalogger.Backend.Bot;
using Catalogger.Backend.Extensions;
using Catalogger.Backend.Services;
using Dapper;
using Microsoft.AspNetCore.Mvc;
using Remora.Discord.API.Abstractions.Rest;
using Remora.Discord.Extensions.Embeds;
namespace Catalogger.Backend.Api;
@ -40,6 +43,8 @@ public partial class GuildsController
}
var guildConfig = await guildRepository.GetAsync(guildId);
var export = await ToExport(guildConfig);
var logChannelId =
webhookExecutor.GetLogChannel(guildConfig, LogChannelType.GuildUpdate)
?? webhookExecutor.GetLogChannel(guildConfig, LogChannelType.GuildMemberRemove);
@ -50,15 +55,25 @@ public partial class GuildsController
var embed = new EmbedBuilder()
.WithTitle("Catalogger is leaving this server")
.WithDescription(
$"A moderator in this server ({currentUser.Tag}, <@{currentUser.Id}>) requested that Catalogger leave. "
+ "All data related to this server will be deleted."
$"""
A moderator in this server ({currentUser.Tag}, <@{currentUser.Id}>) requested that Catalogger leave.
All data related to this server will be deleted.
A backup of this server's configuration is attached to this message,
in case you want to use the bot again later.
"""
)
.WithColour(DiscordUtils.Red)
.WithCurrentTimestamp()
.Build()
.GetOrThrow();
await webhookExecutor.SendLogAsync(logChannelId.Value, [embed], []);
var exportData = JsonSerializer.Serialize(export, JsonUtils.ApiJsonOptions);
var file = new FileData(
"config-backup.json",
new MemoryStream(Encoding.UTF8.GetBytes(exportData))
);
await webhookExecutor.SendLogAsync(logChannelId.Value, [embed], [file]);
}
else
{
@ -125,7 +140,7 @@ public partial class GuildsController
_logger.Information("Left guild {GuildId} and removed all data for it", guildId);
return NoContent();
return Ok(export);
}
public record LeaveGuildRequest(string Name);

2
Catalogger.Backend/Api/GuildsController.cs
View file

@ -34,6 +34,8 @@ public partial class GuildsController(
ILogger logger,
DatabaseConnection dbConn,
GuildRepository guildRepository,
InviteRepository inviteRepository,
WatchlistRepository watchlistRepository,
GuildCache guildCache,
EmojiCache emojiCache,
ChannelCache channelCache,

1
Catalogger.Backend/Cache/IWebhookCache.cs
View file

@ -24,6 +24,7 @@ public interface IWebhookCache
{
Task<Webhook?> GetWebhookAsync(ulong channelId);
Task SetWebhookAsync(ulong channelId, Webhook webhook);
Task RemoveWebhooksAsync(ulong[] channelIds);
public async Task<Webhook> GetOrFetchWebhookAsync(
ulong channelId,

7
Catalogger.Backend/Cache/InMemoryCache/InMemoryWebhookCache.cs
View file

@ -33,4 +33,11 @@ public class InMemoryWebhookCache : IWebhookCache
_cache[channelId] = webhook;
return Task.CompletedTask;
}
public Task RemoveWebhooksAsync(ulong[] channelIds)
{
foreach (var id in channelIds)
_cache.TryRemove(id, out _);
return Task.CompletedTask;
}
}

3
Catalogger.Backend/Cache/RedisCache/RedisWebhookCache.cs
View file

@ -26,5 +26,8 @@ public class RedisWebhookCache(RedisService redisService) : IWebhookCache
public async Task SetWebhookAsync(ulong channelId, Webhook webhook) =>
await redisService.SetAsync(WebhookKey(channelId), webhook, 24.Hours());
public async Task RemoveWebhooksAsync(ulong[] channelIds) =>
await redisService.DeleteAsync(channelIds.Select(WebhookKey).ToArray());
private static string WebhookKey(ulong channelId) => $"webhook:{channelId}";
}

3
Catalogger.Backend/Database/Redis/RedisService.cs
View file

@ -44,6 +44,9 @@ public class RedisService(Config config)
await GetDatabase().StringSetAsync(key, json, expiry);
}
public async Task DeleteAsync(string[] keys) =>
await GetDatabase().KeyDeleteAsync(keys.Select(k => new RedisKey(k)).ToArray());
public async Task<T?> GetAsync<T>(string key)
{
var value = await GetDatabase().StringGetAsync(key);

17
Catalogger.Backend/Database/Repositories/GuildRepository.cs
View file

@ -137,6 +137,23 @@ public class GuildRepository(ILogger logger, DatabaseConnection conn)
new { Id = id.Value, Channels = config }
);
public async Task ImportConfigAsync(
ulong id,
Guild.ChannelConfig channels,
string[] bannedSystems,
ulong[] keyRoles
) =>
await conn.ExecuteAsync(
"update guilds set channels = @channels::jsonb, banned_systems = @bannedSystems, key_roles = @keyRoles where id = @id",
new
{
id,
channels,
bannedSystems,
keyRoles,
}
);
public void Dispose()
{
conn.Dispose();

28
Catalogger.Backend/Database/Repositories/InviteRepository.cs
View file

@ -65,6 +65,34 @@ public class InviteRepository(ILogger logger, DatabaseConnection conn)
new { GuildId = guildId.Value, Code = code }
);
/// <summary>
/// Bulk imports an array of invite codes and names.
/// The GuildId property in the Invite object is ignored.
/// </summary>
public async Task ImportInvitesAsync(Snowflake guildId, IEnumerable<Invite> invites)
{
await using var tx = await conn.BeginTransactionAsync();
foreach (var invite in invites)
{
await conn.ExecuteAsync(
"""
insert into invites (code, guild_id, name)
values (@Code, @GuildId, @Name) on conflict (code)
do update set name = @Name
""",
new
{
GuildId = guildId.Value,
invite.Code,
invite.Name,
},
transaction: tx
);
}
await tx.CommitAsync();
}
public void Dispose()
{
conn.Dispose();

33
Catalogger.Backend/Database/Repositories/WatchlistRepository.cs
View file

@ -70,6 +70,39 @@ public class WatchlistRepository(ILogger logger, DatabaseConnection conn)
)
) != 0;
/// <summary>
/// Bulk imports an array of watchlist entries.
/// The GuildId property in the Watchlist object is ignored.
/// </summary>
public async Task ImportWatchlistAsync(Snowflake guildId, IEnumerable<Watchlist> watchlist)
{
await using var tx = await conn.BeginTransactionAsync();
foreach (var entry in watchlist)
{
await conn.ExecuteAsync(
"""
insert into watchlists (guild_id, user_id, added_at, moderator_id, reason)
values (@GuildId, @UserId, @AddedAt, @ModeratorId, @Reason)
on conflict (guild_id, user_id) do update
set added_at = @AddedAt,
moderator_id = @ModeratorId,
reason = @Reason
""",
new
{
GuildId = guildId.Value,
entry.UserId,
entry.AddedAt,
entry.ModeratorId,
entry.Reason,
},
transaction: tx
);
}
await tx.CommitAsync();
}
public void Dispose()
{
conn.Dispose();

27
Catalogger.Backend/JsonUtils.cs Normal file
View file

@ -0,0 +1,27 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using NodaTime;
using NodaTime.Serialization.SystemTextJson;
using NodaTime.Text;
namespace Catalogger.Backend;
public static class JsonUtils
{
public static readonly NodaJsonSettings NodaTimeSettings = new NodaJsonSettings
{
InstantConverter = new NodaPatternConverter<Instant>(InstantPattern.ExtendedIso),
};
public static readonly JsonSerializerOptions BaseJsonOptions = new JsonSerializerOptions
{
NumberHandling = JsonNumberHandling.AllowReadingFromString,
}.ConfigureForNodaTime(NodaTimeSettings);
public static readonly JsonSerializerOptions ApiJsonOptions = new JsonSerializerOptions
{
NumberHandling =
JsonNumberHandling.WriteAsString | JsonNumberHandling.AllowReadingFromString,
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
}.ConfigureForNodaTime(NodaTimeSettings);
}

3
Catalogger.Backend/Program.cs
View file

@ -15,9 +15,11 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using Catalogger.Backend;
using Catalogger.Backend.Bot.Commands;
using Catalogger.Backend.Extensions;
using Catalogger.Backend.Services;
using NodaTime.Serialization.SystemTextJson;
using Prometheus;
using Remora.Commands.Extensions;
using Remora.Discord.API.Abstractions.Gateway.Commands;
@ -45,6 +47,7 @@ builder
options.JsonSerializerOptions.IncludeFields = true;
options.JsonSerializerOptions.NumberHandling =
JsonNumberHandling.WriteAsString | JsonNumberHandling.AllowReadingFromString;
options.JsonSerializerOptions.ConfigureForNodaTime(JsonUtils.NodaTimeSettings);
});
builder

17
Catalogger.Backend/Services/PluralkitApiService.cs
View file

@ -14,13 +14,9 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
using System.Net;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading.RateLimiting;
using Humanizer;
using NodaTime;
using NodaTime.Serialization.SystemTextJson;
using NodaTime.Text;
using Polly;
namespace Catalogger.Backend.Services;
@ -32,17 +28,6 @@ public class PluralkitApiService(ILogger logger)
private readonly HttpClient _client = new();
private readonly ILogger _logger = logger.ForContext<PluralkitApiService>();
private readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
NumberHandling = JsonNumberHandling.AllowReadingFromString,
}.ConfigureForNodaTime(
new NodaJsonSettings
{
InstantConverter = new NodaPatternConverter<Instant>(InstantPattern.ExtendedIso),
}
);
private readonly ResiliencePipeline _pipeline = new ResiliencePipelineBuilder()
.AddRateLimiter(
new FixedWindowRateLimiter(
@ -86,7 +71,7 @@ public class PluralkitApiService(ILogger logger)
throw new CataloggerError("Non-200 status code from PluralKit API");
}
return await resp.Content.ReadFromJsonAsync<T>(_jsonOptions, ct)
return await resp.Content.ReadFromJsonAsync<T>(JsonUtils.ApiJsonOptions, ct)
?? throw new CataloggerError("JSON response from PluralKit API was null");
}

4
Catalogger.Frontend/package.json
View file

@ -17,11 +17,13 @@
"@sveltejs/vite-plugin-svelte": "^3.0.0",
"@sveltestrap/sveltestrap": "^6.2.7",
"@types/eslint": "^9.6.0",
"@types/file-saver": "^2.0.7",
"@types/luxon": "^3.4.2",
"bootstrap": "^5.3.3",
"eslint": "^9.0.0",
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-svelte": "^2.36.0",
"file-saver": "^2.0.5",
"globals": "^15.0.0",
"luxon": "^3.5.0",
"marked": "^14.1.3",
@ -38,4 +40,4 @@
"vite-plugin-markdown": "^2.2.0"
},
"type": "module"
}
}

6
Catalogger.Frontend/src/routes/dash/[guildId]/+layout.svelte
View file

@ -68,6 +68,12 @@
>
Key roles
</NavLink>
<NavLink
href="/dash/{data.guild.id}/import"
active={$page.url.pathname === `/dash/${data.guild.id}/import`}
>
Import/export settings
</NavLink>
<NavLink
href="/dash/{data.guild.id}/delete"
active={$page.url.pathname === `/dash/${data.guild.id}/delete`}

47
Catalogger.Frontend/src/routes/dash/[guildId]/delete/+page.svelte
View file

@ -1,9 +1,10 @@
<script lang="ts">
import { Alert, Button, Input, InputGroup } from "@sveltestrap/sveltestrap";
import { Button, Input, InputGroup } from "@sveltestrap/sveltestrap";
import type { PageData } from "./$types";
import { fastFetch, type ApiError } from "$lib/api";
import apiFetch, { type ApiError } from "$lib/api";
import { addToast } from "$lib/toast";
import { goto } from "$app/navigation";
import { saveAs } from "file-saver";
export let data: PageData;
@ -11,9 +12,16 @@
const deleteData = async () => {
try {
await fastFetch("POST", `/api/guilds/${data.guild.id}/leave`, {
name: guildName,
});
const backup = await apiFetch<any>(
"POST",
`/api/guilds/${data.guild.id}/leave`,
{
name: guildName,
},
);
downloadBackup(backup);
addToast({
header: "Left server",
body: `Successfully left ${data.guild.name} and deleted all data related to it.`,
@ -27,28 +35,31 @@
});
}
};
const downloadBackup = (data: any) => {
const backup = JSON.stringify(data);
const blob = new Blob([backup], { type: "text/plain;charset=utf-8" });
saveAs(blob, "server-backup.json");
};
</script>
<h3>Delete this server's data</h3>
<p>
To make Catalogger leave your server and delete all data from your server,
fill its name in below and press "Delete".
fill its name (<strong>{data.guild.name}</strong>) in below and press
"Delete".
<br />
<strong>
This is irreversible. If you change your mind later, your data cannot be
restored.
</strong>
<br />
If you just want to make Catalogger leave your server but not delete data, simply
kick it via Discord.
You will get a backup of your server's settings which you can restore later if
you change your mind.
</p>
<Alert color="danger">
<h4 class="alert-heading">This is irreversible!</h4>
We <strong>cannot</strong> help you recover data deleted in this way.
</Alert>
<p>
<strong>
Message data is not backed up. If you change your mind, we cannot restore it
for you.
</strong>
</p>
<p>
<InputGroup>

83
Catalogger.Frontend/src/routes/dash/[guildId]/import/+page.svelte Normal file
View file

@ -0,0 +1,83 @@
<script lang="ts">
import { Button, ButtonGroup, Input } from "@sveltestrap/sveltestrap";
import type { PageData } from "./$types";
import apiFetch, { fastFetch, type ApiError } from "$lib/api";
import saveAs from "file-saver";
import { addToast } from "$lib/toast";
import { goto } from "$app/navigation";
export let data: PageData;
const exportData = async () => {
try {
const config = await apiFetch<any>(
"GET",
`/api/guilds/${data.guild.id}/config`,
);
downloadBackup(config);
} catch (e) {
addToast({
header: "Error downloading export",
body: (e as ApiError).message || "Unknown error",
});
}
};
let importFiles: FileList | undefined;
const importData = async () => {
if (!importFiles || importFiles.length === 0) return;
const fileData = await importFiles[0].text();
const body = JSON.parse(fileData);
try {
await fastFetch("POST", `/api/guilds/${data.guild.id}/config`, body);
addToast({
header: "Imported data",
body: `Successfully imported the settings for ${data.guild.name}!`,
});
importFiles = undefined;
await goto(`/dash/${data.guild.id}`, { invalidateAll: true });
} catch (e) {
addToast({
header: "Error importing data",
body: (e as ApiError).message || "Unknown error",
});
}
};
const downloadBackup = (data: any) => {
const backup = JSON.stringify(data);
const blob = new Blob([backup], { type: "text/plain;charset=utf-8" });
saveAs(blob, "server-backup.json");
};
</script>
<h3>Import and export settings</h3>
<p>
You can create a backup of your server's configuration here. If you removed
the bot from your server before, you can import the backup you got then here
too.
</p>
<p>
<Input
type="file"
id="import"
bind:files={importFiles}
accept="text/plain, application/json"
/>
</p>
<ButtonGroup>
<Button color="primary" on:click={exportData}>Export settings</Button>
<Button
color="secondary"
on:click={importData}
disabled={!importFiles || importFiles.length === 0}>Import settings</Button
>
</ButtonGroup>

10
Catalogger.Frontend/yarn.lock
View file

@ -500,6 +500,11 @@
resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.6.tgz#628effeeae2064a1b4e79f78e81d87b7e5fc7b50"
integrity sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==
"@types/file-saver@^2.0.7":
version "2.0.7"
resolved "https://registry.yarnpkg.com/@types/file-saver/-/file-saver-2.0.7.tgz#8dbb2f24bdc7486c54aa854eb414940bbd056f7d"
integrity sha512-dNKVfHd/jk0SkR/exKGj2ggkB45MAkzvWCaqLUUgkyjITkGNzH8H+yUwr+BLJUBjZOe9w8X3wgmXhZDRg1ED6A==
"@types/json-schema@*", "@types/json-schema@^7.0.15":
version "7.0.15"
resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841"
@ -1048,6 +1053,11 @@ file-entry-cache@^8.0.0:
dependencies:
flat-cache "^4.0.0"
file-saver@^2.0.5:
version "2.0.5"
resolved "https://registry.yarnpkg.com/file-saver/-/file-saver-2.0.5.tgz#d61cfe2ce059f414d899e9dd6d4107ee25670c38"
integrity sha512-P9bmyZ3h/PRG+Nzga+rbdI4OEpNDzAVyy74uVO9ATgzLK6VtAsYybF/+TOCvrc0MO793d6+42lLyZTw7/ArVzA==
fill-range@^7.1.1:
version "7.1.1"
resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292"