[feature] Allow import/export/creation of domain allows via admin panel (#2264)

* it's happening!

* aaa

* fix silly whoopsie

* it's working pa! it's working ma!

* model report parameters

* shuffle some more stuff around

* getting there

* oo hoo

* finish tidying up for now

* aaa

* fix use form submit errors

* peepee poo poo

* aaaaa

* ffff

* they see me typin', they hatin'

* boop

* aaa

* oooo

* typing typing tappa tappa

* almost done typing

* weee

* alright

* push it push it real good doo doo doo doo doo doo

* thingy no worky

* almost done

* mutation modifers not quite right

* hmm

* it works

* view blocks + allows nicely

* it works!

* typia install

* the old linterino

* linter plz

web/source/settings/lib/query/admin/domain-permissions/export.ts

@@ -0,0 +1,155 @@
+/*
+	GoToSocial
+	Copyright (C) GoToSocial Authors admin@gotosocial.org
+	SPDX-License-Identifier: AGPL-3.0-or-later
+
+	This program is free software: you can redistribute it and/or modify
+	it under the terms of the GNU Affero General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU Affero General Public License for more details.
+
+	You should have received a copy of the GNU Affero General Public License
+	along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import fileDownload from "js-file-download";
+import { unparse as csvUnparse } from "papaparse";
+
+import { gtsApi } from "../../gts-api";
+import { RootState } from "../../../../redux/store";
+import { FetchBaseQueryError } from "@reduxjs/toolkit/query";
+import { DomainPerm, ExportDomainPermsParams } from "../../../types/domain-permission";
+
+interface _exportProcess {
+	transformEntry: (_entry: DomainPerm) => any;
+	stringify: (_list: any[]) => string;
+	extension: string;
+	mime: string;
+}
+
+/**
+ * Derive process functions and metadata
+ * from provided export request form.
+ * 
+ * @param formData 
+ * @returns 
+ */
+function exportProcess(formData: ExportDomainPermsParams): _exportProcess {
+	if (formData.exportType == "json") {
+		return {
+			transformEntry: (entry) => ({
+				domain: entry.domain,
+				public_comment: entry.public_comment,
+				obfuscate: entry.obfuscate
+			}),
+			stringify: (list) => JSON.stringify(list),
+			extension: ".json",
+			mime: "application/json"
+		};
+	}
+	
+	if (formData.exportType == "csv") {
+		return {
+			transformEntry: (entry) => [
+				entry.domain,            // #domain
+				"suspend",               // #severity
+				false,                   // #reject_media
+				false,                   // #reject_reports
+				entry.public_comment,    // #public_comment
+				entry.obfuscate ?? false // #obfuscate
+			],
+			stringify: (list) => csvUnparse({
+				fields: [
+					"#domain",
+					"#severity",
+					"#reject_media",
+					"#reject_reports",
+					"#public_comment",
+					"#obfuscate",
+				],
+				data: list
+			}),
+			extension: ".csv",
+			mime: "text/csv"
+		};
+	}
+
+	// Fall back to plain text export.
+	return {
+		transformEntry: (entry) => entry.domain,
+		stringify: (list) => list.join("\n"),
+		extension: ".txt",
+		mime: "text/plain"
+	};
+}
+
+const extended = gtsApi.injectEndpoints({
+	endpoints: (build) => ({		
+		exportDomainList: build.mutation<string | null, ExportDomainPermsParams>({
+			async queryFn(formData, api, _extraOpts, fetchWithBQ) {
+				// Fetch domain perms from relevant endpoint.
+				// We could have used 'useDomainBlocksQuery'
+				// or 'useDomainAllowsQuery' for this, but
+				// we want the untransformed array version.
+				const permsRes = await fetchWithBQ({ url: `/api/v1/admin/domain_${formData.permType}s` });
+				if (permsRes.error) {
+					return { error: permsRes.error as FetchBaseQueryError };
+				}
+
+				// Process perms into desired export format.
+				const process = exportProcess(formData); 
+				const transformed = (permsRes.data as DomainPerm[]).map(process.transformEntry);
+				const exportAsString = process.stringify(transformed);
+
+				if (formData.action == "export") {
+					// Data will just be exported
+					// to the domains text field.
+					return { data: exportAsString };
+				}
+
+				// File export has been requested.
+				// Parse filename to something like:
+				// `example.org-blocklist-2023-10-09.json`.
+				const state = api.getState() as RootState;
+				const instanceUrl = state.oauth.instanceUrl?? "unknown";
+				const domain = new URL(instanceUrl).host;
+				const date = new Date();
+				const filename = [
+					domain,
+					"blocklist",
+					date.getFullYear(),
+					(date.getMonth() + 1).toString().padStart(2, "0"),
+					date.getDate().toString().padStart(2, "0"),
+				].join("-");
+
+				fileDownload(
+					exportAsString,
+					filename + process.extension,
+					process.mime
+				);
+
+				// js-file-download handles the
+				// nitty gritty for us, so we can
+				// just return null data.
+				return { data: null };
+			}
+		}),
+	})
+});
+
+/**
+ * Makes a GET to `/api/v1/admin/domain_{perm_type}s`
+ * and exports the result in the requested format.
+ * 
+ * Return type will be string if `action` is "export",
+ * else it will be null, since the file downloader handles
+ * the rest of the request then.
+ */
+const useExportDomainListMutation = extended.useExportDomainListMutation;
+
+export { useExportDomainListMutation };

web/source/settings/lib/query/admin/domain-permissions/get.ts

@@ -0,0 +1,56 @@
+/*
+	GoToSocial
+	Copyright (C) GoToSocial Authors admin@gotosocial.org
+	SPDX-License-Identifier: AGPL-3.0-or-later
+
+	This program is free software: you can redistribute it and/or modify
+	it under the terms of the GNU Affero General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU Affero General Public License for more details.
+
+	You should have received a copy of the GNU Affero General Public License
+	along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { gtsApi } from "../../gts-api";
+
+import type { DomainPerm, MappedDomainPerms } from "../../../types/domain-permission";
+import { listToKeyedObject } from "../../transforms";
+
+const extended = gtsApi.injectEndpoints({
+	endpoints: (build) => ({
+		domainBlocks: build.query<MappedDomainPerms, void>({
+			query: () => ({
+				url: `/api/v1/admin/domain_blocks`
+			}),
+			transformResponse: listToKeyedObject<DomainPerm>("domain"),
+		}),
+
+		domainAllows: build.query<MappedDomainPerms, void>({
+			query: () => ({
+				url: `/api/v1/admin/domain_allows`
+			}),
+			transformResponse: listToKeyedObject<DomainPerm>("domain"),
+		}),
+	}),
+});
+
+/**
+ * Get admin view of all explicitly blocked domains.
+ */
+const useDomainBlocksQuery = extended.useDomainBlocksQuery;
+
+/**
+ * Get admin view of all explicitly allowed domains.
+ */
+const useDomainAllowsQuery = extended.useDomainAllowsQuery;
+
+export {
+	useDomainBlocksQuery,
+	useDomainAllowsQuery,
+};

web/source/settings/lib/query/admin/domain-permissions/import.ts

@@ -0,0 +1,140 @@
+/*
+	GoToSocial
+	Copyright (C) GoToSocial Authors admin@gotosocial.org
+	SPDX-License-Identifier: AGPL-3.0-or-later
+
+	This program is free software: you can redistribute it and/or modify
+	it under the terms of the GNU Affero General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU Affero General Public License for more details.
+
+	You should have received a copy of the GNU Affero General Public License
+	along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { replaceCacheOnMutation } from "../../query-modifiers";
+import { gtsApi } from "../../gts-api";
+
+import {
+	type DomainPerm,
+	type ImportDomainPermsParams,
+	type MappedDomainPerms,
+	isDomainPermInternalKey,
+} from "../../../types/domain-permission";
+import { listToKeyedObject } from "../../transforms";
+
+/**
+ * Builds up a map function that can be applied to a
+ * list of DomainPermission entries in order to normalize
+ * them before submission to the API.
+ * @param formData 
+ * @returns 
+ */
+function importEntriesProcessor(formData: ImportDomainPermsParams): (_entry: DomainPerm) => DomainPerm {
+	let processingFuncs: { (_entry: DomainPerm): void; }[] = [];
+
+	// Override each obfuscate entry if necessary.
+	if (formData.obfuscate !== undefined) {
+		const obfuscateEntry = (entry: DomainPerm) => {
+			entry.obfuscate = formData.obfuscate;
+		};
+		processingFuncs.push(obfuscateEntry);
+	}
+
+	// Check whether we need to append or replace
+	// private_comment and public_comment.
+	["private_comment","public_comment"].forEach((commentType) => {
+		let text = formData.commentType?.trim();
+		if (!text) {
+			return;
+		}
+
+		switch(formData[`${commentType}_behavior`]) {
+			case "append":
+				const appendComment = (entry: DomainPerm) => {
+					if (entry.commentType == undefined) {
+						entry.commentType = text;
+					} else {
+						entry.commentType = [entry.commentType, text].join("\n");
+					}
+				};
+
+				processingFuncs.push(appendComment);
+				break;
+			case "replace":
+				const replaceComment = (entry: DomainPerm) => {
+					entry.commentType = text;
+				};
+
+				processingFuncs.push(replaceComment);
+				break;
+		}
+	});
+
+	return function process(entry) {
+		// Call all the assembled processing functions.
+		processingFuncs.forEach((f) => f(entry));
+
+		// Unset all internal processing keys
+		// and any undefined keys on this entry.
+		Object.entries(entry).forEach(([key, val]: [keyof DomainPerm, any]) => {			
+			if (val == undefined || isDomainPermInternalKey(key)) {
+				delete entry[key];
+			}
+		});
+
+		return entry;
+	};
+}
+
+const extended = gtsApi.injectEndpoints({
+	endpoints: (build) => ({		
+		importDomainPerms: build.mutation<MappedDomainPerms, ImportDomainPermsParams>({
+			query: (formData) => {
+				// Add/replace comments, remove internal keys.
+				const process = importEntriesProcessor(formData);
+				const domains = formData.domains.map(process);
+
+				return {
+					method: "POST",
+					url: `/api/v1/admin/domain_${formData.permType}s?import=true`,
+					asForm: true,
+					discardEmpty: true,
+					body: {
+						import: true,
+						domains: new Blob(
+							[JSON.stringify(domains)],
+							{ type: "application/json" },
+						),
+					}
+				};
+			},
+			transformResponse: listToKeyedObject<DomainPerm>("domain"),
+			...replaceCacheOnMutation((formData: ImportDomainPermsParams) => {
+				// Query names for blocks and allows are like
+				// `domainBlocks` and `domainAllows`, so we need
+				// to convert `block` -> `Block` or `allow` -> `Allow`
+				// to do proper cache invalidation.
+				const permType =
+					formData.permType.charAt(0).toUpperCase() +
+					formData.permType.slice(1); 
+				return `domain${permType}s`;
+			}),
+		})
+	})
+});
+
+/**
+ * POST domain permissions to /api/v1/admin/domain_{permType}s.
+ * Returns the newly created permissions.
+ */
+const useImportDomainPermsMutation = extended.useImportDomainPermsMutation;
+
+export {
+	useImportDomainPermsMutation,
+};

web/source/settings/lib/query/admin/domain-permissions/process.ts

@@ -0,0 +1,163 @@
+/*
+	GoToSocial
+	Copyright (C) GoToSocial Authors admin@gotosocial.org
+	SPDX-License-Identifier: AGPL-3.0-or-later
+
+	This program is free software: you can redistribute it and/or modify
+	it under the terms of the GNU Affero General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU Affero General Public License for more details.
+
+	You should have received a copy of the GNU Affero General Public License
+	along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import {
+	ParseConfig as CSVParseConfig,
+	parse as csvParse
+} from "papaparse";
+import { nanoid } from "nanoid";
+
+import { isValidDomainPermission, hasBetterScope } from "../../../util/domain-permission";
+import { gtsApi } from "../../gts-api";
+
+import {
+	isDomainPerms,
+	type DomainPerm,
+} from "../../../types/domain-permission";
+
+/**
+ * Parse the given string of domain permissions and return it as an array.
+ * Accepts input as a JSON array string, a CSV, or newline-separated domain names.
+ * Will throw an error if input is invalid.
+ * @param list 
+ * @returns
+ * @throws
+ */
+function parseDomainList(list: string): DomainPerm[] {	
+	if (list.startsWith("[")) {
+		// Assume JSON array.
+		const data = JSON.parse(list);
+		if (!isDomainPerms(data)) {
+			throw "parsed JSON was not array of DomainPermission";
+		}
+
+		return data;
+	} else if (list.startsWith("#domain") || list.startsWith("domain,severity")) {
+		// Assume Mastodon-style CSV.
+		const csvParseCfg: CSVParseConfig = {
+			header: true,
+			// Remove leading '#' if present.
+			transformHeader: (header) => header.startsWith("#") ? header.slice(1) : header,
+			skipEmptyLines: true,
+			dynamicTyping: true
+		};
+		
+		const { data, errors } = csvParse(list, csvParseCfg);
+		if (errors.length > 0) {
+			let error = "";
+			errors.forEach((err) => {
+				error += `${err.message} (line ${err.row})`;
+			});
+			throw error;
+		} 
+
+		if (!isDomainPerms(data)) {
+			throw "parsed CSV was not array of DomainPermission";
+		}
+
+		return data;
+	} else {
+		// Fallback: assume newline-separated
+		// list of simple domain strings.
+		const data: DomainPerm[] = [];
+		list.split("\n").forEach((line) => {
+			let domain = line.trim();
+			let valid = true;
+
+			if (domain.startsWith("http")) {
+				try {
+					domain = new URL(domain).hostname;
+				} catch (e) {
+					valid = false;
+				}
+			}
+
+			if (domain.length > 0) {
+				data.push({ domain, valid });
+			}
+		});
+
+		return data;
+	}
+}
+
+function deduplicateDomainList(list: DomainPerm[]): DomainPerm[] {
+	let domains = new Set();
+	return list.filter((entry) => {
+		if (domains.has(entry.domain)) {
+			return false;
+		} else {
+			domains.add(entry.domain);
+			return true;
+		}
+	});
+}
+
+function validateDomainList(list: DomainPerm[]) {
+	list.forEach((entry) => {		
+		if (entry.domain.startsWith("*.")) {
+			// A domain permission always includes
+			// all subdomains, wildcard is meaningless here
+			entry.domain = entry.domain.slice(2);
+		}
+
+		entry.valid = (entry.valid !== false) && isValidDomainPermission(entry.domain);
+		if (entry.valid) {
+			entry.suggest = hasBetterScope(entry.domain);
+		}
+		entry.checked = entry.valid;
+	});
+
+	return list;
+}
+
+const extended = gtsApi.injectEndpoints({
+	endpoints: (build) => ({
+		processDomainPermissions: build.mutation<DomainPerm[], any>({
+			async queryFn(formData, _api, _extraOpts, _fetchWithBQ) {
+				if (formData.domains == undefined || formData.domains.length == 0) {
+					throw "No domains entered";
+				}
+				
+				// Parse + tidy up the form data.
+				const permissions = parseDomainList(formData.domains);
+				const deduped = deduplicateDomainList(permissions);
+				const validated = validateDomainList(deduped);
+				
+				validated.forEach((entry) => {
+					// Set unique key that stays stable
+					// even if domain gets modified by user.
+					entry.key = nanoid();
+				});
+				
+				return { data: validated };
+			}
+		})
+	})
+});
+
+/**
+ * useProcessDomainPermissionsMutation uses the RTK Query API without actually
+ * hitting the GtS API, it's purely an internal function for our own convenience.
+ * 
+ * It returns the validated and deduplicated domain permission list.
+ */
+const useProcessDomainPermissionsMutation = extended.useProcessDomainPermissionsMutation;
+
+export { useProcessDomainPermissionsMutation };

web/source/settings/lib/query/admin/domain-permissions/update.ts

@@ -0,0 +1,109 @@
+/*
+	GoToSocial
+	Copyright (C) GoToSocial Authors admin@gotosocial.org
+	SPDX-License-Identifier: AGPL-3.0-or-later
+
+	This program is free software: you can redistribute it and/or modify
+	it under the terms of the GNU Affero General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU Affero General Public License for more details.
+
+	You should have received a copy of the GNU Affero General Public License
+	along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { gtsApi } from "../../gts-api";
+
+import {
+	replaceCacheOnMutation,
+	removeFromCacheOnMutation,
+} from "../../query-modifiers";
+import { listToKeyedObject } from "../../transforms";
+import type {
+	DomainPerm,
+	MappedDomainPerms
+} from "../../../types/domain-permission";
+
+const extended = gtsApi.injectEndpoints({
+	endpoints: (build) => ({
+		addDomainBlock: build.mutation<MappedDomainPerms, any>({
+			query: (formData) => ({
+				method: "POST",
+				url: `/api/v1/admin/domain_blocks`,
+				asForm: true,
+				body: formData,
+				discardEmpty: true
+			}),
+			transformResponse: listToKeyedObject<DomainPerm>("domain"),
+			...replaceCacheOnMutation("domainBlocks"),
+		}),
+
+		addDomainAllow: build.mutation<MappedDomainPerms, any>({
+			query: (formData) => ({
+				method: "POST",
+				url: `/api/v1/admin/domain_allows`,
+				asForm: true,
+				body: formData,
+				discardEmpty: true
+			}),
+			transformResponse: listToKeyedObject<DomainPerm>("domain"),
+			...replaceCacheOnMutation("domainAllows")
+		}),
+
+		removeDomainBlock: build.mutation<DomainPerm, string>({
+			query: (id) => ({
+				method: "DELETE",
+				url: `/api/v1/admin/domain_blocks/${id}`,
+			}),
+			...removeFromCacheOnMutation("domainBlocks", {
+				key: (_draft, newData) => {
+					return newData.domain;
+				}
+			})
+		}),
+
+		removeDomainAllow: build.mutation<DomainPerm, string>({
+			query: (id) => ({
+				method: "DELETE",
+				url: `/api/v1/admin/domain_allows/${id}`,
+			}),
+			...removeFromCacheOnMutation("domainAllows", {
+				key: (_draft, newData) => {
+					return newData.domain;
+				}
+			})
+		}),
+	}),
+});
+
+/**
+ * Add a single domain permission (block) by POSTing to `/api/v1/admin/domain_blocks`.
+ */
+const useAddDomainBlockMutation = extended.useAddDomainBlockMutation;
+
+/**
+ * Add a single domain permission (allow) by POSTing to `/api/v1/admin/domain_allows`.
+ */
+const useAddDomainAllowMutation = extended.useAddDomainAllowMutation;
+
+/**
+ * Remove a single domain permission (block) by DELETEing to `/api/v1/admin/domain_blocks/{id}`.
+ */
+const useRemoveDomainBlockMutation = extended.useRemoveDomainBlockMutation;
+
+/**
+ * Remove a single domain permission (allow) by DELETEing to `/api/v1/admin/domain_allows/{id}`.
+ */
+const useRemoveDomainAllowMutation = extended.useRemoveDomainAllowMutation;
+
+export {
+	useAddDomainBlockMutation,
+	useAddDomainAllowMutation,
+	useRemoveDomainBlockMutation,
+	useRemoveDomainAllowMutation
+};