rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Minimal support for importing chat histories in Chub format 

Fix crasher in script.js:displayChats if user has directly put a Chub chat file into their user data

Let eslint tidy a few things
This commit is contained in:
ceruleandeep 2024-09-19 14:18:15 +10:00
parent 83c3f6d1bf
commit a2d9526cdf
2 changed files with 64 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
public/script.js
View File

@ -6923,15 +6923,13 @@ export async function displayPastChats() {
}
// Check whether `text` {string} includes all of the `fragments` {string[]}.
function matchFragments(fragments, text) {
if (!text) {
return false;
}
return fragments.every(item => text.includes(item));
if (!text || !text.toLowerCase) return false;
return fragments.every(item => text.toLowerCase().includes(item));
}
const fragments = makeQueryFragments(searchQuery);
// At least one chat message must match *all* the fragments.
// Currently, this doesn't match if the fragment matches are distributed across several chat messages.
return chatContent && Object.values(chatContent).some(message => matchFragments(fragments, message?.mes?.toLowerCase()));
return chatContent && Object.values(chatContent).some(message => matchFragments(fragments, message?.mes));
});
console.debug(filteredData);

79
src/endpoints/chats.js
View File

@ -92,8 +92,7 @@ function importOobaChat(userName, characterName, jsonData) {
}
}
const chatContent = chat.map(obj => JSON.stringify(obj)).join('\n');
return chatContent;
return chat.map(obj => JSON.stringify(obj)).join('\n');
}
/**
@ -121,8 +120,7 @@ function importAgnaiChat(userName, characterName, jsonData) {
});
}
const chatContent = chat.map(obj => JSON.stringify(obj)).join('\n');
return chatContent;
return chat.map(obj => JSON.stringify(obj)).join('\n');
}
/**
@ -159,6 +157,37 @@ function importCAIChat(userName, characterName, jsonData) {
return newChats;
}
/**
* Flattens `msg` and `swipes` data from Chub Chat format.
* Only changes enough to make it compatible with the standard chat serialization format.
* @param {string} userName User name
* @param {string} characterName Character name
* @param {string[]} lines serialised JSONL data
* @returns {string} Converted data
*/
function flattenChubChat(userName, characterName, lines) {
function flattenSwipe(swipe) {
return swipe.message ? swipe.message : swipe;
}
function convert(line) {
const lineData = tryParse(line);
if (!lineData) return line;
if (lineData.mes && lineData.mes.message) {
lineData.mes = lineData?.mes.message;
}
if (lineData?.swipes && Array.isArray(lineData.swipes)) {
lineData.swipes = lineData.swipes.map(swipe => flattenSwipe(swipe));
}
return JSON.stringify(lineData);
}
return (lines ?? []).map(convert).join('\n');
}
const router = express.Router();
router.post('/save', jsonParser, function (request, response) {
@ -273,7 +302,7 @@ router.post('/export', jsonParser, async function (request, response) {
}
try {
// Short path for JSONL files
if (request.body.format == 'jsonl') {
if (request.body.format === 'jsonl') {
try {
const rawFile = fs.readFileSync(filename, 'utf8');
const successMessage = {
@ -283,8 +312,7 @@ router.post('/export', jsonParser, async function (request, response) {
console.log(`Chat exported as ${exportfilename}`);
return response.status(200).json(successMessage);
}
catch (err) {
} catch (err) {
console.error(err);
const errorMessage = {
message: `Could not read JSONL file to export. Source chat file: ${filename}.`,
@ -319,8 +347,7 @@ router.post('/export', jsonParser, async function (request, response) {
console.log(`Chat exported as ${exportfilename}`);
return response.status(200).json(successMessage);
});
}
catch (err) {
} catch (err) {
console.log('chat export failed.');
console.log(err);
return response.sendStatus(400);
@ -396,20 +423,36 @@ router.post('/import', urlencodedParser, function (request, response) {
}
if (format === 'jsonl') {
const line = data.split('\n')[0];
let lines = data.split('\n');
const header = lines[0];
const jsonData = JSON.parse(line);
const jsonData = JSON.parse(header);
if (jsonData.user_name !== undefined || jsonData.name !== undefined) {
const fileName = `${characterName} - ${humanizedISO8601DateTime()} imported.jsonl`;
const filePath = path.join(request.user.directories.chats, avatarUrl, fileName);
fs.copyFileSync(pathToUpload, filePath);
fs.unlinkSync(pathToUpload);
response.send({ res: true });
} else {
if (!(jsonData.user_name !== undefined || jsonData.name !== undefined)) {
console.log('Incorrect chat format .jsonl');
return response.send({ error: true });
}
// Do a tiny bit of work to import Chub Chat data
// Processing the entire file is so fast that it's not worth checking if it's a Chub chat first
let flattenedChat;
try {
// flattening is unlikely to break, but it's not worth failing to
// import normal chats in an attempt to import a Chub chat
flattenedChat = flattenChubChat(userName, characterName, lines);
} catch (error) {
console.warn('Failed to flatten Chub Chat data: ', error);
}
const fileName = `${characterName} - ${humanizedISO8601DateTime()} imported.jsonl`;
const filePath = path.join(request.user.directories.chats, avatarUrl, fileName);
if (flattenedChat !== data) {
writeFileAtomicSync(filePath, flattenedChat, 'utf8');
} else {
fs.copyFileSync(pathToUpload, filePath);
}
fs.unlinkSync(pathToUpload);
response.send({ res: true });
}
} catch (error) {
console.error(error);