mirror of
https://github.com/superseriousbusiness/gotosocial
synced 2025-06-05 21:59:39 +02:00
[feature/frogend] (Mastodon) domain block CSV import (#1390)
* checkbox-list styling with taller <p> element * CSV import/export, UI/UX improvements to import-export interface * minor styling tweaks * csv export, clean up export type branching * abstract domain block entry validation * foundation for PSL check + suggestions * Squashed commit of the following: commite3655ba4fbAuthor: f0x <f0x@cthu.lu> Date: Tue Jan 31 15:19:10 2023 +0100 let debug depend on env (prod/debug) again commit79c792b832Author: f0x <f0x@cthu.lu> Date: Tue Jan 31 00:34:01 2023 +0100 update checklist components commit4367960fe4Author: f0x <f0x@cthu.lu> Date: Mon Jan 30 23:46:20 2023 +0100 checklist performance improvements commit204a4c02d1Author: f0x <f0x@cthu.lu> Date: Mon Jan 30 20:05:34 2023 +0100 checklist field: use reducer for state * remove debug logging * show and use domain block suggestion * restructure import/export buttons * updating suggestions * suggestion overview * restructure check-list behavior, domain import/export
This commit is contained in:
f0x52
GitHub
50
web/source/settings/lib/domain-block.js
Normal file
50
web/source/settings/lib/domain-block.js
Normal file
@ -0,0 +1,50 @@
|
||||
/*
|
||||
GoToSocial
|
||||
Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
"use strict";
|
||||
|
||||
const isValidDomain = require("is-valid-domain");
|
||||
const psl = require("psl");
|
||||
|
||||
function isValidDomainBlock(domain) {
|
||||
return isValidDomain(domain, {
|
||||
/*
|
||||
Wildcard prefix *. can be stripped since it's equivalent to not having it,
|
||||
but wildcard anywhere else in the domain is not handled by the backend so it's invalid.
|
||||
*/
|
||||
wildcard: false,
|
||||
allowUnicode: true
|
||||
});
|
||||
}
|
||||
|
||||
/*
|
||||
Still can't think of a better function name for this,
|
||||
but we're checking a domain against the Public Suffix List <https://publicsuffix.org/>
|
||||
to see if we should suggest removing subdomain(s) since they're likely owned/ran by the same party
|
||||
social.example.com -> suggests example.com
|
||||
*/
|
||||
function hasBetterScope(domain) {
|
||||
const lookup = psl.get(domain);
|
||||
if (lookup && lookup != domain) {
|
||||
return lookup;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { isValidDomainBlock, hasBetterScope };
|
||||
@ -20,128 +20,163 @@
|
||||
|
||||
const React = require("react");
|
||||
const syncpipe = require("syncpipe");
|
||||
const { createSlice } = require("@reduxjs/toolkit");
|
||||
const { enableMapSet } = require("immer");
|
||||
|
||||
function createState(entries, uniqueKey, oldState, defaultValue) {
|
||||
return syncpipe(entries, [
|
||||
(_) => _.map((entry) => {
|
||||
let key = entry[uniqueKey];
|
||||
return [
|
||||
key,
|
||||
{
|
||||
...entry,
|
||||
key,
|
||||
checked: oldState[key]?.checked ?? entry.checked ?? defaultValue
|
||||
enableMapSet(); // for use in reducers
|
||||
|
||||
const { reducer, actions } = createSlice({
|
||||
name: "checklist",
|
||||
initialState: {}, // not handled by slice itself
|
||||
reducers: {
|
||||
updateAll: (state, { payload: checked }) => {
|
||||
const selectedEntries = new Set();
|
||||
return {
|
||||
entries: syncpipe(state.entries, [
|
||||
(_) => Object.values(_),
|
||||
(_) => _.map((entry) => {
|
||||
if (checked) {
|
||||
selectedEntries.add(entry.key);
|
||||
}
|
||||
return [entry.key, {
|
||||
...entry,
|
||||
checked
|
||||
}];
|
||||
}),
|
||||
(_) => Object.fromEntries(_)
|
||||
]),
|
||||
selectedEntries
|
||||
};
|
||||
},
|
||||
update: (state, { payload: { key, value } }) => {
|
||||
if (value.checked !== undefined) {
|
||||
if (value.checked === true) {
|
||||
state.selectedEntries.add(key);
|
||||
} else {
|
||||
state.selectedEntries.delete(key);
|
||||
}
|
||||
];
|
||||
}),
|
||||
(_) => Object.fromEntries(_)
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
function updateAllState(state, newValue) {
|
||||
return syncpipe(state, [
|
||||
(_) => Object.values(_),
|
||||
(_) => _.map((entry) => [entry.key, {
|
||||
...entry,
|
||||
checked: newValue
|
||||
}]),
|
||||
(_) => Object.fromEntries(_)
|
||||
]);
|
||||
}
|
||||
state.entries[key] = {
|
||||
...state.entries[key],
|
||||
...value
|
||||
};
|
||||
},
|
||||
updateMultiple: (state, { payload }) => {
|
||||
payload.forEach(([key, value]) => {
|
||||
if (value.checked !== undefined) {
|
||||
if (value.checked === true) {
|
||||
state.selectedEntries.add(key);
|
||||
} else {
|
||||
state.selectedEntries.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
function updateState(state, key, newValue) {
|
||||
return {
|
||||
...state,
|
||||
[key]: {
|
||||
...state[key],
|
||||
...newValue
|
||||
state.entries[key] = {
|
||||
...state.entries[key],
|
||||
...value
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
function initialState({ entries, uniqueKey, defaultValue }) {
|
||||
const selectedEntries = new Set();
|
||||
return {
|
||||
entries: syncpipe(entries, [
|
||||
(_) => _.map((entry) => {
|
||||
let key = entry[uniqueKey];
|
||||
let checked = entry.checked ?? defaultValue;
|
||||
|
||||
if (checked) {
|
||||
selectedEntries.add(key);
|
||||
} else {
|
||||
selectedEntries.delete(key);
|
||||
}
|
||||
|
||||
return [
|
||||
key,
|
||||
{
|
||||
...entry,
|
||||
key,
|
||||
checked
|
||||
}
|
||||
];
|
||||
}),
|
||||
(_) => Object.fromEntries(_)
|
||||
]),
|
||||
selectedEntries
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = function useCheckListInput({ name }, { entries, uniqueKey = "key", defaultValue = false }) {
|
||||
const [state, setState] = React.useState({});
|
||||
const [state, dispatch] = React.useReducer(reducer, null,
|
||||
() => initialState({ entries, uniqueKey, defaultValue }) // initial state
|
||||
);
|
||||
|
||||
const [someSelected, setSomeSelected] = React.useState(false);
|
||||
const [toggleAllState, setToggleAllState] = React.useState(0);
|
||||
const toggleAllRef = React.useRef(null);
|
||||
|
||||
React.useEffect(() => {
|
||||
/*
|
||||
entries changed, update state,
|
||||
re-using old state if available for key
|
||||
*/
|
||||
setState(createState(entries, uniqueKey, state, defaultValue));
|
||||
if (toggleAllRef.current != null) {
|
||||
let some = state.selectedEntries.size > 0;
|
||||
let all = false;
|
||||
if (some) {
|
||||
all = state.selectedEntries.size == Object.values(state.entries).length;
|
||||
}
|
||||
toggleAllRef.current.checked = all;
|
||||
toggleAllRef.current.indeterminate = some && !all;
|
||||
}
|
||||
// only needs to update when state.selectedEntries changes, not state.entries
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [state.selectedEntries]);
|
||||
|
||||
/* eslint-disable-next-line react-hooks/exhaustive-deps */
|
||||
}, [entries]);
|
||||
const reset = React.useCallback(
|
||||
() => dispatch(actions.updateAll(defaultValue)),
|
||||
[defaultValue]
|
||||
);
|
||||
|
||||
React.useEffect(() => {
|
||||
/* Updates (un)check all checkbox, based on shortcode checkboxes
|
||||
Can be 0 (not checked), 1 (checked) or 2 (indeterminate)
|
||||
*/
|
||||
if (toggleAllRef.current == null) {
|
||||
return;
|
||||
const onChange = React.useCallback(
|
||||
(key, value) => dispatch(actions.update({ key, value })),
|
||||
[]
|
||||
);
|
||||
|
||||
const updateMultiple = React.useCallback(
|
||||
(entries) => dispatch(actions.updateMultiple(entries)),
|
||||
[]
|
||||
);
|
||||
|
||||
return React.useMemo(() => {
|
||||
function toggleAll(e) {
|
||||
let checked = e.target.checked;
|
||||
if (e.target.indeterminate) {
|
||||
checked = false;
|
||||
}
|
||||
dispatch(actions.updateAll(checked));
|
||||
}
|
||||
|
||||
let values = Object.values(state);
|
||||
/* one or more boxes are checked */
|
||||
let some = values.some((v) => v.checked);
|
||||
|
||||
let all = false;
|
||||
if (some) {
|
||||
/* there's not at least one unchecked box */
|
||||
all = !values.some((v) => v.checked == false);
|
||||
function selectedValues() {
|
||||
return Array.from((state.selectedEntries)).map((key) => ({
|
||||
...state.entries[key] // returned as new object, because reducer state is immutable
|
||||
}));
|
||||
}
|
||||
|
||||
setSomeSelected(some);
|
||||
|
||||
if (some && !all) {
|
||||
setToggleAllState(2);
|
||||
toggleAllRef.current.indeterminate = true;
|
||||
} else {
|
||||
setToggleAllState(all ? 1 : 0);
|
||||
toggleAllRef.current.indeterminate = false;
|
||||
}
|
||||
}, [state, toggleAllRef]);
|
||||
|
||||
function toggleAll(e) {
|
||||
let selectAll = e.target.checked;
|
||||
|
||||
if (toggleAllState == 2) { // indeterminate
|
||||
selectAll = false;
|
||||
}
|
||||
|
||||
setState(updateAllState(state, selectAll));
|
||||
setToggleAllState(selectAll);
|
||||
}
|
||||
|
||||
function reset() {
|
||||
setState(updateAllState(state, defaultValue));
|
||||
}
|
||||
|
||||
function selectedValues() {
|
||||
return syncpipe(state, [
|
||||
(_) => Object.values(_),
|
||||
(_) => _.filter((entry) => entry.checked)
|
||||
]);
|
||||
}
|
||||
|
||||
return Object.assign([
|
||||
state,
|
||||
reset,
|
||||
{ name }
|
||||
], {
|
||||
name,
|
||||
value: state,
|
||||
onChange: (key, newValue) => setState(updateState(state, key, newValue)),
|
||||
selectedValues,
|
||||
reset,
|
||||
someSelected,
|
||||
toggleAll: {
|
||||
ref: toggleAllRef,
|
||||
value: toggleAllState,
|
||||
onChange: toggleAll
|
||||
}
|
||||
});
|
||||
return Object.assign([
|
||||
state,
|
||||
reset,
|
||||
{ name }
|
||||
], {
|
||||
name,
|
||||
value: state.entries,
|
||||
onChange,
|
||||
selectedValues,
|
||||
reset,
|
||||
someSelected: state.someChecked,
|
||||
updateMultiple,
|
||||
toggleAll: {
|
||||
ref: toggleAllRef,
|
||||
onChange: toggleAll
|
||||
}
|
||||
});
|
||||
}, [state, reset, name, onChange, updateMultiple]);
|
||||
};
|
||||
@ -20,14 +20,30 @@
|
||||
|
||||
const React = require("react");
|
||||
|
||||
module.exports = function useTextInput({ name, Name }, { validator, defaultValue = "", dontReset = false } = {}) {
|
||||
module.exports = function useTextInput({ name, Name }, {
|
||||
defaultValue = "",
|
||||
dontReset = false,
|
||||
validator,
|
||||
showValidation = true,
|
||||
initValidation
|
||||
} = {}) {
|
||||
|
||||
const [text, setText] = React.useState(defaultValue);
|
||||
const [valid, setValid] = React.useState(true);
|
||||
const textRef = React.useRef(null);
|
||||
|
||||
const [validation, setValidation] = React.useState(initValidation ?? "");
|
||||
const [_isValidating, startValidation] = React.useTransition();
|
||||
let valid = validation == "";
|
||||
|
||||
function onChange(e) {
|
||||
let input = e.target.value;
|
||||
setText(input);
|
||||
|
||||
if (validator) {
|
||||
startValidation(() => {
|
||||
setValidation(validator(input));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function reset() {
|
||||
@ -38,11 +54,13 @@ module.exports = function useTextInput({ name, Name }, { validator, defaultValue
|
||||
|
||||
React.useEffect(() => {
|
||||
if (validator && textRef.current) {
|
||||
let res = validator(text);
|
||||
setValid(res == "");
|
||||
textRef.current.setCustomValidity(res);
|
||||
if (showValidation) {
|
||||
textRef.current.setCustomValidity(validation);
|
||||
} else {
|
||||
textRef.current.setCustomValidity("");
|
||||
}
|
||||
}
|
||||
}, [text, textRef, validator]);
|
||||
}, [validation, validator, showValidation]);
|
||||
|
||||
// Array / Object hybrid, for easier access in different contexts
|
||||
return Object.assign([
|
||||
@ -62,6 +80,7 @@ module.exports = function useTextInput({ name, Name }, { validator, defaultValue
|
||||
ref: textRef,
|
||||
setter: setText,
|
||||
valid,
|
||||
validate: () => setValidation(validator(text)),
|
||||
hasChanged: () => text != defaultValue
|
||||
});
|
||||
};
|
||||
@ -19,8 +19,11 @@
|
||||
"use strict";
|
||||
|
||||
const Promise = require("bluebird");
|
||||
const isValidDomain = require("is-valid-domain");
|
||||
const fileDownload = require("js-file-download");
|
||||
const csv = require("papaparse");
|
||||
const { nanoid } = require("nanoid");
|
||||
|
||||
const { isValidDomainBlock, hasBetterScope } = require("../../domain-block");
|
||||
|
||||
const {
|
||||
replaceCacheOnMutation,
|
||||
@ -31,6 +34,23 @@ const {
|
||||
function parseDomainList(list) {
|
||||
if (list[0] == "[") {
|
||||
return JSON.parse(list);
|
||||
} else if (list.startsWith("#domain")) { // Mastodon CSV
|
||||
const { data, errors } = csv.parse(list, {
|
||||
header: true,
|
||||
transformHeader: (header) => header.slice(1), // removes starting '#'
|
||||
skipEmptyLines: true,
|
||||
dynamicTyping: true
|
||||
});
|
||||
|
||||
if (errors.length > 0) {
|
||||
let error = "";
|
||||
errors.forEach((err) => {
|
||||
error += `${err.message} (line ${err.row})`;
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
|
||||
return data;
|
||||
} else {
|
||||
return list.split("\n").map((line) => {
|
||||
let domain = line.trim();
|
||||
@ -51,7 +71,15 @@ function parseDomainList(list) {
|
||||
|
||||
function validateDomainList(list) {
|
||||
list.forEach((entry) => {
|
||||
entry.valid = (entry.valid !== false) && isValidDomain(entry.domain, { wildcard: true, allowUnicode: true });
|
||||
if (entry.domain.startsWith("*.")) {
|
||||
// domain block always includes all subdomains, wildcard is meaningless here
|
||||
entry.domain = entry.domain.slice(2);
|
||||
}
|
||||
|
||||
entry.valid = (entry.valid !== false) && isValidDomainBlock(entry.domain);
|
||||
if (entry.valid) {
|
||||
entry.suggest = hasBetterScope(entry.domain);
|
||||
}
|
||||
entry.checked = entry.valid;
|
||||
});
|
||||
|
||||
@ -83,6 +111,9 @@ module.exports = (build) => ({
|
||||
}).then((deduped) => {
|
||||
return validateDomainList(deduped);
|
||||
}).then((data) => {
|
||||
data.forEach((entry) => {
|
||||
entry.key = nanoid(); // unique id that stays stable even if domain gets modified by user
|
||||
});
|
||||
return { data };
|
||||
}).catch((e) => {
|
||||
return { error: e.toString() };
|
||||
@ -91,27 +122,53 @@ module.exports = (build) => ({
|
||||
}),
|
||||
exportDomainList: build.mutation({
|
||||
queryFn: (formData, api, _extraOpts, baseQuery) => {
|
||||
let process;
|
||||
|
||||
if (formData.exportType == "json") {
|
||||
process = {
|
||||
transformEntry: (entry) => ({
|
||||
domain: entry.domain,
|
||||
public_comment: entry.public_comment,
|
||||
obfuscate: entry.obfuscate
|
||||
}),
|
||||
stringify: (list) => JSON.stringify(list),
|
||||
extension: ".json",
|
||||
mime: "application/json"
|
||||
};
|
||||
} else if (formData.exportType == "csv") {
|
||||
process = {
|
||||
transformEntry: (entry) => [
|
||||
entry.domain,
|
||||
"suspend", // severity
|
||||
false, // reject_media
|
||||
false, // reject_reports
|
||||
entry.public_comment,
|
||||
entry.obfuscate ?? false
|
||||
],
|
||||
stringify: (list) => csv.unparse({
|
||||
fields: "#domain,#severity,#reject_media,#reject_reports,#public_comment,#obfuscate".split(","),
|
||||
data: list
|
||||
}),
|
||||
extension: ".csv",
|
||||
mime: "text/csv"
|
||||
};
|
||||
} else {
|
||||
process = {
|
||||
transformEntry: (entry) => entry.domain,
|
||||
stringify: (list) => list.join("\n"),
|
||||
extension: ".txt",
|
||||
mime: "text/plain"
|
||||
};
|
||||
}
|
||||
|
||||
return Promise.try(() => {
|
||||
return baseQuery({
|
||||
url: `/api/v1/admin/domain_blocks`
|
||||
});
|
||||
}).then(unwrapRes).then((blockedInstances) => {
|
||||
return blockedInstances.map((entry) => {
|
||||
if (formData.exportType == "json") {
|
||||
return {
|
||||
domain: entry.domain,
|
||||
public_comment: entry.public_comment
|
||||
};
|
||||
} else {
|
||||
return entry.domain;
|
||||
}
|
||||
});
|
||||
return blockedInstances.map(process.transformEntry);
|
||||
}).then((exportList) => {
|
||||
if (formData.exportType == "json") {
|
||||
return JSON.stringify(exportList);
|
||||
} else {
|
||||
return exportList.join("\n");
|
||||
}
|
||||
return process.stringify(exportList);
|
||||
}).then((exportAsString) => {
|
||||
if (formData.action == "export") {
|
||||
return {
|
||||
@ -120,7 +177,6 @@ module.exports = (build) => ({
|
||||
} else if (formData.action == "export-file") {
|
||||
let domain = new URL(api.getState().oauth.instance).host;
|
||||
let date = new Date();
|
||||
let mime;
|
||||
|
||||
let filename = [
|
||||
domain,
|
||||
@ -130,15 +186,11 @@ module.exports = (build) => ({
|
||||
date.getDate().toString().padStart(2, "0"),
|
||||
].join("-");
|
||||
|
||||
if (formData.exportType == "json") {
|
||||
filename += ".json";
|
||||
mime = "application/json";
|
||||
} else {
|
||||
filename += ".txt";
|
||||
mime = "text/plain";
|
||||
}
|
||||
|
||||
fileDownload(exportAsString, filename, mime);
|
||||
fileDownload(
|
||||
exportAsString,
|
||||
filename + process.extension,
|
||||
process.mime
|
||||
);
|
||||
}
|
||||
return { data: null };
|
||||
}).catch((e) => {
|
||||
@ -171,6 +223,7 @@ module.exports = (build) => ({
|
||||
})
|
||||
});
|
||||
|
||||
const internalKeys = new Set("key,suggest,valid,checked".split(","));
|
||||
function entryProcessor(formData) {
|
||||
let funcs = [];
|
||||
|
||||
@ -204,7 +257,7 @@ function entryProcessor(formData) {
|
||||
entry.obfuscate = formData.obfuscate;
|
||||
|
||||
Object.entries(entry).forEach(([key, val]) => {
|
||||
if (val == undefined) {
|
||||
if (internalKeys.has(key) || val == undefined) {
|
||||
delete entry[key];
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user