mirror of
https://github.com/SillyTavern/SillyTavern.git
synced 2025-06-05 21:59:27 +02:00
7b87227cd2
1) Character files now represent images, all data about the characters is located within the image file. This makes it easy to exchange characters. You do not need to redesign your old characters in the new format, the conversion will happen automatically, old character files will not be deleted 2) Import function for characters is available, you can also import characters in the old TavernAI .json format, and you can also import characters from the Pygmalion notebook format 3) Export function for characters is available 4) The chat structure has been completely redesigned, now chats are located in the public/chats folder, chats now represent jsonl files, messages can be read one line at a time from such a file, this will help for future optimization when there are many messages 5) A new menu for selecting saved TavernAI chats is now available Importing TavernAI chats is now available Importing chats from CAI is now available 6) Additional advanced settings have been added for characters. A "Scenario" field has been added, as well as a "Dialogue Examples" field 7) Full support for the Pygmalion format has been added A indicator now appears when special formatting for Pygmalion is enabled 8) Many other minor bugs have been fixed To update an old version to version 1.2.0, you simply need to copy files with replacement.
1292 lines
46 KiB
JavaScript
1292 lines
46 KiB
JavaScript
var express = require('express');
|
||
var app = express();
|
||
var fs = require('fs');
|
||
const readline = require('readline');
|
||
|
||
var rimraf = require("rimraf");
|
||
const multer = require("multer");
|
||
const https = require('https');
|
||
//const PNG = require('pngjs').PNG;
|
||
const extract = require('png-chunks-extract');
|
||
const encode = require('png-chunks-encode');
|
||
const PNGtext = require('png-chunk-text');
|
||
|
||
const sharp = require('sharp');
|
||
sharp.cache(false);
|
||
const path = require('path');
|
||
|
||
var Client = require('node-rest-client').Client;
|
||
var client = new Client();
|
||
|
||
var api_server = "";//"http://127.0.0.1:5000";
|
||
var server_port = 8000;
|
||
|
||
var api_novelai = "https://api.novelai.net";
|
||
|
||
var response_get_story;
|
||
var response_generate;
|
||
var response_generate_novel;
|
||
var request_promt;
|
||
var response_promt;
|
||
var characters = {};
|
||
var character_i = 0;
|
||
var response_create;
|
||
var response_edit;
|
||
var response_dw_bg;
|
||
var response_getstatus;
|
||
var response_getstatus_novel;
|
||
var response_getlastversion;
|
||
var api_key_novel;
|
||
|
||
var is_colab = false;
|
||
|
||
const jsonParser = express.json();
|
||
const urlencodedParser = express.urlencoded({extended: false});
|
||
|
||
|
||
|
||
|
||
app.post("/getlastversion", jsonParser, function(request, response_getlastversion = response){
|
||
if(!request.body) return response_getlastversion.sendStatus(400);
|
||
|
||
const repo = 'TavernAI/TavernAI';
|
||
|
||
https.request({
|
||
hostname: 'github.com',
|
||
path: `/${repo}/releases/latest`,
|
||
method: 'HEAD'
|
||
}, (res) => {
|
||
if(res.statusCode === 302) {
|
||
const glocation = res.headers.location;
|
||
const versionStartIndex = glocation.lastIndexOf('@')+1;
|
||
const version = glocation.substring(versionStartIndex);
|
||
//console.log(version);
|
||
response_getlastversion.send({version: version});
|
||
}else{
|
||
response_getlastversion.send({version: 'error'});
|
||
}
|
||
}).end();
|
||
|
||
|
||
});
|
||
app.use(express.static(__dirname + "/public"));
|
||
|
||
app.use(multer({dest:"uploads"}).single("avatar"));
|
||
app.get("/", function(request, response){
|
||
response.sendFile(__dirname + "/public/index.html");
|
||
//response.send("<h1>Главная страница</h1>");
|
||
});
|
||
app.get("/notes/*", function(request, response){
|
||
response.sendFile(__dirname + "/public"+request.url+".html");
|
||
//response.send("<h1>Главная страница</h1>");
|
||
});
|
||
//**************Kobold api
|
||
app.post("/generate", jsonParser, function(request, response_generate = response){
|
||
if(!request.body) return response_generate.sendStatus(400);
|
||
//console.log(request.body.prompt);
|
||
//const dataJson = JSON.parse(request.body);
|
||
request_promt = request.body.prompt;
|
||
|
||
//console.log(request.body);
|
||
var this_settings = { prompt: request_promt,
|
||
use_story:false,
|
||
use_memory:false,
|
||
use_authors_note:false,
|
||
use_world_info:false,
|
||
max_context_length: request.body.max_context_length
|
||
//temperature: request.body.temperature,
|
||
//max_length: request.body.max_length
|
||
};
|
||
|
||
if(request.body.gui_settings == false){
|
||
var sampler_order = [request.body.s1,request.body.s2,request.body.s3,request.body.s4,request.body.s5,request.body.s6,request.body.s7];
|
||
this_settings = { prompt: request_promt,
|
||
use_story:false,
|
||
use_memory:false,
|
||
use_authors_note:false,
|
||
use_world_info:false,
|
||
max_context_length: request.body.max_context_length,
|
||
max_length: request.body.max_length,
|
||
rep_pen: request.body.rep_pen,
|
||
rep_pen_range: request.body.rep_pen_range,
|
||
rep_pen_slope: request.body.rep_pen_slope,
|
||
temperature: request.body.temperature,
|
||
tfs: request.body.tfs,
|
||
top_a: request.body.top_a,
|
||
top_k: request.body.top_k,
|
||
top_p: request.body.top_p,
|
||
typical: request.body.typical,
|
||
sampler_order: sampler_order
|
||
};
|
||
}
|
||
|
||
console.log(this_settings);
|
||
var args = {
|
||
data: this_settings,
|
||
headers: { "Content-Type": "application/json" }
|
||
};
|
||
client.post(api_server+"/v1/generate",args, function (data, response) {
|
||
if(response.statusCode == 200){
|
||
console.log(data);
|
||
response_generate.send(data);
|
||
}
|
||
if(response.statusCode == 422){
|
||
console.log('Validation error');
|
||
response_generate.send({error: true});
|
||
}
|
||
if(response.statusCode == 501 || response.statusCode == 503 || response.statusCode == 507){
|
||
console.log(data);
|
||
response_generate.send({error: true});
|
||
}
|
||
}).on('error', function (err) {
|
||
//console.log('');
|
||
//console.log('something went wrong on the request', err.request.options);
|
||
response_generate.send({error: true});
|
||
});
|
||
});
|
||
app.post("/savechat", jsonParser, function(request, response){
|
||
//console.log(request.data);
|
||
//console.log(request.body.bg);
|
||
//const data = request.body;
|
||
//console.log(request);
|
||
//console.log(request.body.chat);
|
||
//var bg = "body {background-image: linear-gradient(rgba(19,21,44,0.75), rgba(19,21,44,0.75)), url(../backgrounds/"+request.body.bg+");}";
|
||
var dir_name = String(request.body.avatar_url).replace('.png','');
|
||
let chat_data = request.body.chat;
|
||
let jsonlData = chat_data.map(JSON.stringify).join('\n');
|
||
fs.writeFile('public/chats/'+dir_name+"/"+request.body.file_name+'.jsonl', jsonlData, 'utf8', function(err) {
|
||
if(err) {
|
||
response.send(err);
|
||
return console.log(err);
|
||
//response.send(err);
|
||
}else{
|
||
//response.redirect("/");
|
||
response.send({result: "ok"});
|
||
}
|
||
});
|
||
|
||
});
|
||
app.post("/getchat", jsonParser, function(request, response){
|
||
//console.log(request.data);
|
||
//console.log(request.body.bg);
|
||
//const data = request.body;
|
||
//console.log(request);
|
||
//console.log(request.body.chat);
|
||
//var bg = "body {background-image: linear-gradient(rgba(19,21,44,0.75), rgba(19,21,44,0.75)), url(../backgrounds/"+request.body.bg+");}";
|
||
var dir_name = String(request.body.avatar_url).replace('.png','');
|
||
|
||
fs.stat('public/chats/'+dir_name, function(err, stat) {
|
||
|
||
if(stat === undefined){
|
||
|
||
fs.mkdirSync('public/chats/'+dir_name);
|
||
response.send({});
|
||
return;
|
||
}else{
|
||
|
||
if(err === null){
|
||
|
||
fs.stat('public/chats/'+dir_name+"/"+request.body.file_name+".jsonl", function(err, stat) {
|
||
|
||
if (err === null) {
|
||
|
||
if(stat !== undefined){
|
||
fs.readFile('public/chats/'+dir_name+"/"+request.body.file_name+".jsonl", 'utf8', (err, data) => {
|
||
if (err) {
|
||
console.error(err);
|
||
response.send(err);
|
||
return;
|
||
}
|
||
//console.log(data);
|
||
const lines = data.split('\n');
|
||
|
||
// Iterate through the array of strings and parse each line as JSON
|
||
const jsonData = lines.map(JSON.parse);
|
||
response.send(jsonData);
|
||
|
||
|
||
});
|
||
}
|
||
}else{
|
||
response.send({});
|
||
//return console.log(err);
|
||
return;
|
||
}
|
||
});
|
||
}else{
|
||
console.error(err);
|
||
response.send({});
|
||
return;
|
||
}
|
||
}
|
||
|
||
|
||
});
|
||
|
||
|
||
});
|
||
app.post("/getstatus", jsonParser, function(request, response_getstatus = response){
|
||
if(!request.body) return response_getstatus.sendStatus(400);
|
||
if(is_colab === true){
|
||
api_server = '127.0.0.1:5000';
|
||
}else{
|
||
api_server = request.body.api_server;
|
||
}
|
||
if(api_server.indexOf('localhost') != -1){
|
||
api_server = api_server.replace('localhost','127.0.0.1');
|
||
}
|
||
var args = {
|
||
headers: { "Content-Type": "application/json" }
|
||
};
|
||
client.get(api_server+"/v1/model",args, function (data, response) {
|
||
if(response.statusCode == 200){
|
||
if(data.result != "ReadOnly"){
|
||
|
||
//response_getstatus.send(data.result);
|
||
}else{
|
||
data.result = "no_connection";
|
||
}
|
||
}else{
|
||
data.result = "no_connection";
|
||
}
|
||
response_getstatus.send(data);
|
||
//console.log(response.statusCode);
|
||
//console.log(data);
|
||
//response_getstatus.send(data);
|
||
//data.results[0].text
|
||
}).on('error', function (err) {
|
||
//console.log('');
|
||
//console.log('something went wrong on the request', err.request.options);
|
||
response_getstatus.send({result: "no_connection"});
|
||
});
|
||
});
|
||
function checkServer(){
|
||
//console.log('Check run###################################################');
|
||
api_server = 'http://127.0.0.1:5000';
|
||
var args = {
|
||
headers: { "Content-Type": "application/json" }
|
||
};
|
||
client.get(api_server+"/v1/model",args, function (data, response) {
|
||
console.log(data.result);
|
||
//console.log('###################################################');
|
||
console.log(data);
|
||
}).on('error', function (err) {
|
||
console.log(err);
|
||
//console.log('');
|
||
//console.log('something went wrong on the request', err.request.options);
|
||
//console.log('errorrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr');
|
||
});
|
||
}
|
||
|
||
//***************** Main functions
|
||
function charaFormatData(data){
|
||
var char = {"name": data.ch_name, "description": data.description, "personality": data.personality, "first_mes": data.first_mes, "avatar": 'none', "chat": Date.now(), "mes_example": data.mes_example, "scenario": data.scenario, "create_date": Date.now()};
|
||
return char;
|
||
}
|
||
app.post("/createcharacter", urlencodedParser, function(request, response){
|
||
if(!request.body) return response.sendStatus(400);
|
||
if (!fs.existsSync('public/characters/'+request.body.ch_name+'.png')){
|
||
if(!fs.existsSync('public/chats/'+request.body.ch_name) )fs.mkdirSync('public/chats/'+request.body.ch_name);
|
||
//if(!fs.existsSync('public/characters/'+request.body.ch_name+'/chats')) fs.mkdirSync('public/characters/'+request.body.ch_name+'/chats');
|
||
//if(!fs.existsSync('public/characters/'+request.body.ch_name+'/avatars')) fs.mkdirSync('public/characters/'+request.body.ch_name+'/avatars');
|
||
|
||
let filedata = request.file;
|
||
//console.log(filedata.mimetype);
|
||
var fileType = ".png";
|
||
var img_file = "ai";
|
||
var img_path = "public/img/";
|
||
|
||
var char = charaFormatData(request.body);//{"name": request.body.ch_name, "description": request.body.description, "personality": request.body.personality, "first_mes": request.body.first_mes, "avatar": 'none', "chat": Date.now(), "last_mes": '', "mes_example": ''};
|
||
char = JSON.stringify(char);
|
||
if(!filedata){
|
||
|
||
charaWrite('./public/img/fluffy.png', char, request.body.ch_name, response);
|
||
|
||
//fs.writeFile('public/characters/'+request.body.ch_name+"/"+request.body.ch_name+".json", char, 'utf8', function(err) {
|
||
//if(err) {
|
||
//response.send(err);
|
||
//return console.log(err);
|
||
//}else{
|
||
|
||
//}
|
||
//});
|
||
}else{
|
||
|
||
img_path = "./uploads/";
|
||
img_file = filedata.filename
|
||
if (filedata.mimetype == "image/jpeg") fileType = ".jpeg";
|
||
if (filedata.mimetype == "image/png") fileType = ".png";
|
||
if (filedata.mimetype == "image/gif") fileType = ".gif";
|
||
if (filedata.mimetype == "image/bmp") fileType = ".bmp";
|
||
charaWrite(img_path+img_file, char, request.body.ch_name, response);
|
||
}
|
||
//console.log("The file was saved.");
|
||
|
||
}else{
|
||
response.send("Error: A character with that name already exists.");
|
||
}
|
||
//console.log(request.body);
|
||
//response.send(request.body.ch_name);
|
||
|
||
//response.redirect("https://metanit.com")
|
||
});
|
||
|
||
|
||
app.post("/editcharacter", urlencodedParser, function(request, response){
|
||
if(!request.body) return response.sendStatus(400);
|
||
let filedata = request.file;
|
||
//console.log(filedata.mimetype);
|
||
var fileType = ".png";
|
||
var img_file = "ai";
|
||
var img_path = "./public/characters/";
|
||
|
||
var char = charaFormatData(request.body);//{"name": request.body.ch_name, "description": request.body.description, "personality": request.body.personality, "first_mes": request.body.first_mes, "avatar": request.body.avatar_url, "chat": request.body.chat, "last_mes": request.body.last_mes, "mes_example": ''};
|
||
char.chat = request.body.chat;
|
||
char.create_date = request.body.create_date;
|
||
|
||
char = JSON.stringify(char);
|
||
if(!filedata){
|
||
|
||
charaWrite(img_path+request.body.avatar_url, char, request.body.ch_name, response, 'Character saved');
|
||
}else{
|
||
//console.log(filedata.filename);
|
||
img_path = "uploads/";
|
||
img_file = filedata.filename;
|
||
|
||
charaWrite(img_path+img_file, char, request.body.ch_name, response, 'Character saved');
|
||
//response.send('Character saved');
|
||
}
|
||
});
|
||
app.post("/deletecharacter", urlencodedParser, function(request, response){
|
||
if(!request.body) return response.sendStatus(400);
|
||
rimraf('public/characters/'+request.body.avatar_url, (err) => {
|
||
if(err) {
|
||
response.send(err);
|
||
return console.log(err);
|
||
}else{
|
||
//response.redirect("/");
|
||
let dir_name = request.body.avatar_url;
|
||
rimraf('public/chats/'+dir_name.replace('.png',''), (err) => {
|
||
if(err) {
|
||
response.send(err);
|
||
return console.log(err);
|
||
}else{
|
||
//response.redirect("/");
|
||
|
||
response.send('ok');
|
||
}
|
||
});
|
||
}
|
||
});
|
||
});
|
||
|
||
async function charaWrite(img_url, data, name, response = undefined, mes = 'ok'){
|
||
try {
|
||
// Load the image in any format
|
||
sharp.cache(false);
|
||
var image = await sharp(img_url).resize(170, 234).toFormat('png').toBuffer();
|
||
// Convert the image to PNG format
|
||
//const pngImage = image.toFormat('png');
|
||
|
||
// Resize the image to 100x100
|
||
//const resizedImage = pngImage.resize(100, 100);
|
||
|
||
// Get the chunks
|
||
var chunks = extract(image);
|
||
var tEXtChunks = chunks.filter(chunk => chunk.name === 'tEXt');
|
||
|
||
// Remove all existing tEXt chunks
|
||
for (var tEXtChunk of tEXtChunks) {
|
||
chunks.splice(chunks.indexOf(tEXtChunk), 1);
|
||
}
|
||
// Add new chunks before the IEND chunk
|
||
var base64EncodedData = Buffer.from(data, 'utf8').toString('base64');
|
||
chunks.splice(-1, 0, PNGtext.encode('chara', base64EncodedData));
|
||
//chunks.splice(-1, 0, text.encode('lorem', 'ipsum'));
|
||
|
||
fs.writeFileSync('public/characters/'+name+'.png', new Buffer.from(encode(chunks)));
|
||
if(response !== undefined) response.send(mes);
|
||
|
||
|
||
} catch (err) {
|
||
console.log(err);
|
||
if(response !== undefined) response.send(err);
|
||
}
|
||
}
|
||
|
||
|
||
|
||
|
||
|
||
function charaRead(img_url){
|
||
sharp.cache(false);
|
||
const buffer = fs.readFileSync(img_url);
|
||
const chunks = extract(buffer);
|
||
|
||
const textChunks = chunks.filter(function (chunk) {
|
||
return chunk.name === 'tEXt';
|
||
}).map(function (chunk) {
|
||
//console.log(text.decode(chunk.data));
|
||
return PNGtext.decode(chunk.data);
|
||
});
|
||
var base64DecodedData = Buffer.from(textChunks[0].text, 'base64').toString('utf8');
|
||
return base64DecodedData;//textChunks[0].text;
|
||
//console.log(textChunks[0].keyword); // 'hello'
|
||
//console.log(textChunks[0].text); // 'world'
|
||
}
|
||
|
||
app.post("/getcharacters", jsonParser, function(request, response){
|
||
fs.readdir("public/characters", (err, files) => {
|
||
if (err) {
|
||
console.error(err);
|
||
return;
|
||
}
|
||
|
||
const pngFiles = files.filter(file => file.endsWith('.png'));
|
||
|
||
//console.log(pngFiles);
|
||
characters = {};
|
||
var i = 0;
|
||
pngFiles.forEach(item => {
|
||
//console.log(item);
|
||
var img_data = charaRead('./public/characters/'+item);
|
||
try {
|
||
let jsonObject = JSON.parse(img_data);
|
||
jsonObject.avatar = item;
|
||
//console.log(jsonObject);
|
||
characters[i] = {};
|
||
characters[i] = jsonObject;
|
||
i++;
|
||
} catch (error) {
|
||
if (error instanceof SyntaxError) {
|
||
console.log("String [" + (i) + "] is not valid JSON!");
|
||
} else {
|
||
console.log("An unexpected error occurred: ", error);
|
||
}
|
||
}
|
||
});
|
||
//console.log(characters);
|
||
response.send(JSON.stringify(characters));
|
||
});
|
||
//var directories = getDirectories("public/characters");
|
||
//console.log(directories[0]);
|
||
//characters = {};
|
||
//character_i = 0;
|
||
//getCharaterFile(directories, response,0);
|
||
|
||
});
|
||
app.post("/getbackgrounds", jsonParser, function(request, response){
|
||
var images = getImages("public/backgrounds");
|
||
if(is_colab === true){
|
||
images = ['tavern.png'];
|
||
}
|
||
response.send(JSON.stringify(images));
|
||
|
||
});
|
||
app.post("/iscolab", jsonParser, function(request, response){
|
||
|
||
response.send({is_colab:is_colab});
|
||
|
||
});
|
||
app.post("/getuseravatars", jsonParser, function(request, response){
|
||
var images = getImages("public/User Avatars");
|
||
response.send(JSON.stringify(images));
|
||
|
||
});
|
||
app.post("/setbackground", jsonParser, function(request, response){
|
||
//console.log(request.data);
|
||
//console.log(request.body.bg);
|
||
//const data = request.body;
|
||
//console.log(request);
|
||
//console.log(1);
|
||
var bg = "#bg1 {background-image: linear-gradient(rgba(19,21,44,0.75), rgba(19,21,44,0.75)), url(../backgrounds/"+request.body.bg+");}";
|
||
fs.writeFile('public/css/bg_load.css', bg, 'utf8', function(err) {
|
||
if(err) {
|
||
response.send(err);
|
||
return console.log(err);
|
||
}else{
|
||
//response.redirect("/");
|
||
response.send({result:'ok'});
|
||
}
|
||
});
|
||
|
||
});
|
||
app.post("/delbackground", jsonParser, function(request, response){
|
||
if(!request.body) return response.sendStatus(400);
|
||
rimraf('public/backgrounds/'+request.body.bg, (err) => {
|
||
if(err) {
|
||
response.send(err);
|
||
return console.log(err);
|
||
}else{
|
||
//response.redirect("/");
|
||
response.send('ok');
|
||
}
|
||
});
|
||
|
||
});
|
||
app.post("/downloadbackground", urlencodedParser, function(request, response){
|
||
response_dw_bg = response;
|
||
if(!request.body) return response.sendStatus(400);
|
||
|
||
let filedata = request.file;
|
||
//console.log(filedata.mimetype);
|
||
var fileType = ".png";
|
||
var img_file = "ai";
|
||
var img_path = "public/img/";
|
||
|
||
img_path = "uploads/";
|
||
img_file = filedata.filename;
|
||
if (filedata.mimetype == "image/jpeg") fileType = ".jpeg";
|
||
if (filedata.mimetype == "image/png") fileType = ".png";
|
||
if (filedata.mimetype == "image/gif") fileType = ".gif";
|
||
if (filedata.mimetype == "image/bmp") fileType = ".bmp";
|
||
fs.copyFile(img_path+img_file, 'public/backgrounds/'+img_file+fileType, (err) => {
|
||
if(err) {
|
||
|
||
return console.log(err);
|
||
}else{
|
||
//console.log(img_file+fileType);
|
||
response_dw_bg.send(img_file+fileType);
|
||
}
|
||
//console.log('The image was copied from temp directory.');
|
||
});
|
||
|
||
|
||
});
|
||
|
||
app.post("/savesettings", jsonParser, function(request, response){
|
||
|
||
|
||
fs.writeFile('public/settings.json', JSON.stringify(request.body), 'utf8', function(err) {
|
||
if(err) {
|
||
response.send(err);
|
||
return console.log(err);
|
||
//response.send(err);
|
||
}else{
|
||
//response.redirect("/");
|
||
response.send({result: "ok"});
|
||
}
|
||
});
|
||
|
||
});
|
||
app.post('/getsettings', jsonParser, (request, response) => { //Wintermute's code
|
||
const koboldai_settings = [];
|
||
const koboldai_setting_names = [];
|
||
const novelai_settings = [];
|
||
const novelai_setting_names = [];
|
||
const settings = fs.readFileSync('public/settings.json', 'utf8', (err, data) => {
|
||
if (err) return response.sendStatus(500);
|
||
|
||
return data;
|
||
});
|
||
//Kobold
|
||
const files = fs
|
||
.readdirSync('public/KoboldAI Settings')
|
||
.sort(
|
||
(a, b) =>
|
||
new Date(fs.statSync(`public/KoboldAI Settings/${b}`).mtime) -
|
||
new Date(fs.statSync(`public/KoboldAI Settings/${a}`).mtime)
|
||
);
|
||
|
||
files.forEach(item => {
|
||
const file = fs.readFileSync(
|
||
`public/KoboldAI Settings/${item}`,
|
||
'utf8',
|
||
(err, data) => {
|
||
if (err) return response.sendStatus(500)
|
||
|
||
return data;
|
||
}
|
||
);
|
||
koboldai_settings.push(file);
|
||
koboldai_setting_names.push(item.replace(/\.[^/.]+$/, ''));
|
||
});
|
||
|
||
//Novel
|
||
const files2 = fs
|
||
.readdirSync('public/NovelAI Settings')
|
||
.sort(
|
||
(a, b) =>
|
||
new Date(fs.statSync(`public/NovelAI Settings/${b}`).mtime) -
|
||
new Date(fs.statSync(`public/NovelAI Settings/${a}`).mtime)
|
||
);
|
||
|
||
files2.forEach(item => {
|
||
const file2 = fs.readFileSync(
|
||
`public/NovelAI Settings/${item}`,
|
||
'utf8',
|
||
(err, data) => {
|
||
if (err) return response.sendStatus(500);
|
||
|
||
return data;
|
||
}
|
||
);
|
||
|
||
novelai_settings.push(file2);
|
||
novelai_setting_names.push(item.replace(/\.[^/.]+$/, ''));
|
||
});
|
||
|
||
response.send({
|
||
settings,
|
||
koboldai_settings,
|
||
koboldai_setting_names,
|
||
novelai_settings,
|
||
novelai_setting_names
|
||
});
|
||
});
|
||
|
||
|
||
function getCharaterFile(directories,response,i){ //old need del
|
||
if(directories.length > i){
|
||
|
||
fs.stat('public/characters/'+directories[i]+'/'+directories[i]+".json", function(err, stat) {
|
||
if (err == null) {
|
||
fs.readFile('public/characters/'+directories[i]+'/'+directories[i]+".json", 'utf8', (err, data) => {
|
||
if (err) {
|
||
console.error(err);
|
||
return;
|
||
}
|
||
//console.log(data);
|
||
|
||
characters[character_i] = {};
|
||
characters[character_i] = data;
|
||
i++;
|
||
character_i++;
|
||
getCharaterFile(directories,response,i);
|
||
});
|
||
}else{
|
||
i++;
|
||
getCharaterFile(directories,response,i);
|
||
}
|
||
});
|
||
|
||
}else{
|
||
response.send(JSON.stringify(characters));
|
||
}
|
||
}
|
||
function getImages(path) {
|
||
return fs.readdirSync(path).sort(function (a, b) {
|
||
return new Date(fs.statSync(path + '/' + a).mtime) - new Date(fs.statSync(path + '/' + b).mtime);
|
||
}).reverse();
|
||
}
|
||
function getKoboldSettingFiles(path) {
|
||
return fs.readdirSync(path).sort(function (a, b) {
|
||
return new Date(fs.statSync(path + '/' + a).mtime) - new Date(fs.statSync(path + '/' + b).mtime);
|
||
}).reverse();
|
||
}
|
||
function getDirectories(path) {
|
||
return fs.readdirSync(path).sort(function (a, b) {
|
||
return new Date(fs.statSync(path + '/' + a).mtime) - new Date(fs.statSync(path + '/' + b).mtime);
|
||
}).reverse();
|
||
}
|
||
|
||
//***********Novel.ai API
|
||
|
||
app.post("/getstatus_novelai", jsonParser, function(request, response_getstatus_novel =response){
|
||
|
||
if(!request.body) return response_getstatus_novel.sendStatus(400);
|
||
api_key_novel = request.body.key;
|
||
var data = {};
|
||
var args = {
|
||
data: data,
|
||
|
||
headers: { "Content-Type": "application/json", "Authorization": "Bearer "+api_key_novel}
|
||
};
|
||
client.get(api_novelai+"/user/subscription",args, function (data, response) {
|
||
if(response.statusCode == 200){
|
||
//console.log(data);
|
||
response_getstatus_novel.send(data);//data);
|
||
}
|
||
if(response.statusCode == 401){
|
||
console.log('Access Token is incorrect.');
|
||
response_getstatus_novel.send({error: true});
|
||
}
|
||
if(response.statusCode == 500 || response.statusCode == 501 || response.statusCode == 501 || response.statusCode == 503 || response.statusCode == 507){
|
||
console.log(data);
|
||
response_getstatus_novel.send({error: true});
|
||
}
|
||
}).on('error', function (err) {
|
||
//console.log('');
|
||
//console.log('something went wrong on the request', err.request.options);
|
||
response_getstatus_novel.send({error: true});
|
||
});
|
||
});
|
||
|
||
|
||
|
||
app.post("/generate_novelai", jsonParser, function(request, response_generate_novel = response){
|
||
if(!request.body) return response_generate_novel.sendStatus(400);
|
||
|
||
console.log(request.body);
|
||
var data = {
|
||
"input": request.body.input,
|
||
"model": request.body.model,
|
||
"parameters": {
|
||
"use_string": request.body.use_string,
|
||
"temperature": request.body.temperature,
|
||
"max_length": request.body.max_length,
|
||
"min_length": request.body.min_length,
|
||
"tail_free_sampling": request.body.tail_free_sampling,
|
||
"repetition_penalty": request.body.repetition_penalty,
|
||
"repetition_penalty_range": request.body.repetition_penalty_range,
|
||
"repetition_penalty_frequency": request.body.repetition_penalty_frequency,
|
||
"repetition_penalty_presence": request.body.repetition_penalty_presence,
|
||
//"stop_sequences": {{187}},
|
||
//bad_words_ids = {{50256}, {0}, {1}};
|
||
//generate_until_sentence = true;
|
||
"use_cache": request.body.use_cache,
|
||
//use_string = true;
|
||
"return_full_text": request.body.return_full_text,
|
||
"prefix": request.body.prefix,
|
||
"order": request.body.order
|
||
}
|
||
};
|
||
|
||
var args = {
|
||
data: data,
|
||
|
||
headers: { "Content-Type": "application/json", "Authorization": "Bearer "+api_key_novel}
|
||
};
|
||
client.post(api_novelai+"/ai/generate",args, function (data, response) {
|
||
if(response.statusCode == 201){
|
||
console.log(data);
|
||
response_generate_novel.send(data);
|
||
}
|
||
if(response.statusCode == 400){
|
||
console.log('Validation error');
|
||
response_generate_novel.send({error: true});
|
||
}
|
||
if(response.statusCode == 401){
|
||
console.log('Access Token is incorrect');
|
||
response_generate_novel.send({error: true});
|
||
}
|
||
if(response.statusCode == 402){
|
||
console.log('An active subscription is required to access this endpoint');
|
||
response_generate_novel.send({error: true});
|
||
}
|
||
if(response.statusCode == 500 || response.statusCode == 409){
|
||
console.log(data);
|
||
response_generate_novel.send({error: true});
|
||
}
|
||
}).on('error', function (err) {
|
||
//console.log('');
|
||
//console.log('something went wrong on the request', err.request.options);
|
||
response_getstatus.send({error: true});
|
||
});
|
||
});
|
||
|
||
app.post("/getallchatsofchatacter", jsonParser, function(request, response){
|
||
if(!request.body) return response.sendStatus(400);
|
||
|
||
var char_dir = (request.body.avatar_url).replace('.png','')
|
||
fs.readdir('public/chats/'+char_dir, (err, files) => {
|
||
if (err) {
|
||
console.error(err);
|
||
response.send({error: true});
|
||
return;
|
||
}
|
||
|
||
// filter for JSON files
|
||
const jsonFiles = files.filter(file => path.extname(file) === '.jsonl');
|
||
|
||
// sort the files by name
|
||
//jsonFiles.sort().reverse();
|
||
|
||
// print the sorted file names
|
||
var chatData = {};
|
||
let ii = jsonFiles.length;
|
||
for(let i = jsonFiles.length-1; i >= 0; i--){
|
||
const file = jsonFiles[i];
|
||
|
||
const fileStream = fs.createReadStream('public/chats/'+char_dir+'/'+file);
|
||
const rl = readline.createInterface({
|
||
input: fileStream,
|
||
crlfDelay: Infinity
|
||
});
|
||
|
||
let lastLine;
|
||
|
||
rl.on('line', (line) => {
|
||
lastLine = line;
|
||
});
|
||
|
||
rl.on('close', () => {
|
||
if(lastLine){
|
||
let jsonData = JSON.parse(lastLine);
|
||
if(jsonData.name !== undefined){
|
||
chatData[i] = {};
|
||
chatData[i]['file_name'] = file;
|
||
chatData[i]['mes'] = jsonData['mes'];
|
||
ii--;
|
||
if(ii === 0){
|
||
response.send(chatData);
|
||
}
|
||
}else{
|
||
return;
|
||
}
|
||
}
|
||
rl.close();
|
||
});
|
||
}
|
||
});
|
||
|
||
});
|
||
function getPngName(file){
|
||
if (fs.existsSync('./public/characters/'+file+'.png')) {
|
||
file = file+'1';
|
||
}
|
||
return file;
|
||
}
|
||
app.post("/importcharacter", urlencodedParser, function(request, response){
|
||
if(!request.body) return response.sendStatus(400);
|
||
|
||
let png_name = '';
|
||
let filedata = request.file;
|
||
//console.log(filedata.filename);
|
||
var format = request.body.file_type;
|
||
//console.log(format);
|
||
if(filedata){
|
||
if(format == 'json'){
|
||
fs.readFile('./uploads/'+filedata.filename, 'utf8', (err, data) => {
|
||
if (err){
|
||
console.log(err);
|
||
response.send({error:true});
|
||
}
|
||
const jsonData = JSON.parse(data);
|
||
|
||
if(jsonData.char_name !== undefined){//json Pygmalion notepad
|
||
png_name = getPngName(jsonData.char_name);
|
||
var char = {"name": jsonData.char_name, "description": jsonData.char_persona, "personality": '', "first_mes": jsonData.char_greeting, "avatar": 'none', "chat": Date.now(), "mes_example": jsonData.example_dialogue, "scenario": jsonData.world_scenario, "create_date": Date.now()};
|
||
char = JSON.stringify(char);
|
||
charaWrite('./public/img/fluffy.png', char, png_name, response, {file_name: png_name});
|
||
}else if(jsonData.name !== undefined){
|
||
png_name = getPngName(jsonData.name);
|
||
var char = {"name": jsonData.name, "description": jsonData.description, "personality": jsonData.personality, "first_mes": jsonData.first_mes, "avatar": 'none', "chat": Date.now(), "mes_example": '', "scenario": '', "create_date": Date.now()};
|
||
char = JSON.stringify(char);
|
||
charaWrite('./public/img/fluffy.png', char, png_name, response, {file_name: png_name});
|
||
}else{
|
||
console.log('Incorrect character format .json');
|
||
response.send({error:true});
|
||
}
|
||
});
|
||
}else{
|
||
try{
|
||
|
||
var img_data = charaRead('./uploads/'+filedata.filename);
|
||
let jsonObject = JSON.parse(img_data);
|
||
png_name = getPngName(jsonObject.name);
|
||
if(jsonObject.name !== undefined){
|
||
fs.copyFile('./uploads/'+filedata.filename, 'public/characters/'+png_name+'.png', (err) => {
|
||
if(err) {
|
||
response.send({error:true});
|
||
return console.log(err);
|
||
}else{
|
||
//console.log(img_file+fileType);
|
||
response.send({file_name: png_name});
|
||
}
|
||
//console.log('The image was copied from temp directory.');
|
||
});
|
||
}
|
||
}catch(err){
|
||
console.log(err);
|
||
response.send({error:true});
|
||
}
|
||
}
|
||
//charaWrite(img_path+img_file, char, request.body.ch_name, response);
|
||
}
|
||
//console.log("The file was saved.");
|
||
|
||
|
||
//console.log(request.body);
|
||
//response.send(request.body.ch_name);
|
||
|
||
//response.redirect("https://metanit.com")
|
||
});
|
||
|
||
app.post("/importchat", urlencodedParser, function(request, response){
|
||
if(!request.body) return response.sendStatus(400);
|
||
|
||
var format = request.body.file_type;
|
||
let filedata = request.file;
|
||
let avatar_url = (request.body.avatar_url).replace('.png', '');
|
||
let ch_name = request.body.character_name;
|
||
//console.log(filedata.filename);
|
||
//var format = request.body.file_type;
|
||
//console.log(format);
|
||
//console.log(1);
|
||
if(filedata){
|
||
|
||
if(format === 'json'){
|
||
fs.readFile('./uploads/'+filedata.filename, 'utf8', (err, data) => {
|
||
|
||
if (err){
|
||
console.log(err);
|
||
response.send({error:true});
|
||
}
|
||
|
||
const jsonData = JSON.parse(data);
|
||
var new_chat = [];
|
||
if(jsonData.histories !== undefined){
|
||
let i = 0;
|
||
new_chat[i] = {};
|
||
new_chat[0]['user_name'] = 'You';
|
||
new_chat[0]['character_name'] = ch_name;
|
||
new_chat[0]['create_date'] = Date.now();
|
||
i++;
|
||
jsonData.histories.histories[0].msgs.forEach(function(item) {
|
||
new_chat[i] = {};
|
||
if(item.src.is_human == true){
|
||
new_chat[i]['name'] = 'You';
|
||
}else{
|
||
new_chat[i]['name'] = ch_name;
|
||
}
|
||
new_chat[i]['is_user'] = item.src.is_human;
|
||
new_chat[i]['send_date'] = Date.now();
|
||
new_chat[i]['mes'] = item.text;
|
||
i++;
|
||
});
|
||
const chatJsonlData = new_chat.map(JSON.stringify).join('\n');
|
||
fs.writeFile('public/chats/'+avatar_url+'/'+Date.now()+'.jsonl', chatJsonlData, 'utf8', function(err) {
|
||
if(err) {
|
||
response.send(err);
|
||
return console.log(err);
|
||
//response.send(err);
|
||
}else{
|
||
//response.redirect("/");
|
||
response.send({res:true});
|
||
}
|
||
});
|
||
|
||
}else{
|
||
response.send({error:true});
|
||
return;
|
||
}
|
||
|
||
});
|
||
}
|
||
if(format === 'jsonl'){
|
||
const fileStream = fs.createReadStream('./uploads/'+filedata.filename);
|
||
const rl = readline.createInterface({
|
||
input: fileStream,
|
||
crlfDelay: Infinity
|
||
});
|
||
|
||
rl.once('line', (line) => {
|
||
let jsonData = JSON.parse(line);
|
||
|
||
if(jsonData.user_name !== undefined){
|
||
fs.copyFile('./uploads/'+filedata.filename, 'public/chats/'+avatar_url+'/'+Date.now()+'.jsonl', (err) => {
|
||
if(err) {
|
||
response.send({error:true});
|
||
return console.log(err);
|
||
}else{
|
||
response.send({res:true});
|
||
return;
|
||
}
|
||
});
|
||
}else{
|
||
//response.send({error:true});
|
||
return;
|
||
}
|
||
rl.close();
|
||
});
|
||
}
|
||
|
||
}
|
||
|
||
});
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
app.listen(server_port, function() {
|
||
if(process.env.colab !== undefined){
|
||
if(process.env.colab == 2){
|
||
is_colab = true;
|
||
}
|
||
}
|
||
console.log('TavernAI started: http://127.0.0.1:'+server_port);
|
||
if (fs.existsSync('public/characters/update.txt')) { //&& !is_colab <- this need to put again
|
||
convertStage1();
|
||
}
|
||
|
||
});
|
||
|
||
//#####################CONVERTING IN NEW FORMAT########################
|
||
|
||
var charactersB = {};//B - Backup
|
||
var character_ib = 0;
|
||
|
||
var directoriesB = {};
|
||
|
||
|
||
function convertStage1(){
|
||
//if (!fs.existsSync('public/charactersBackup')) {
|
||
//fs.mkdirSync('public/charactersBackup');
|
||
//copyFolder('public/characters/', 'public/charactersBackup');
|
||
//}
|
||
|
||
var directories = getDirectories2("public/characters");
|
||
//console.log(directories[0]);
|
||
charactersB = {};
|
||
character_ib = 0;
|
||
var folderForDel = {};
|
||
getCharaterFile2(directories, 0);
|
||
}
|
||
function convertStage2(){
|
||
//directoriesB = JSON.parse(directoriesB);
|
||
//console.log(directoriesB);
|
||
var mes = true;
|
||
for (const key in directoriesB) {
|
||
if(mes){
|
||
console.log('***');
|
||
console.log('The update of the character format has begun...');
|
||
console.log('***');
|
||
mes = false;
|
||
}
|
||
//console.log(`${key}: ${directoriesB[key]}`);
|
||
//console.log(JSON.parse(charactersB[key]));
|
||
//console.log(directoriesB[key]);
|
||
|
||
var char = JSON.parse(charactersB[key]);
|
||
char.create_date = Date.now();
|
||
charactersB[key] = JSON.stringify(char);
|
||
var avatar = 'public/img/fluffy.png';
|
||
if(char.avatar !== 'none'){
|
||
avatar = 'public/characters/'+char.name+'/avatars/'+char.avatar;
|
||
}
|
||
|
||
charaWrite(avatar, charactersB[key], directoriesB[key]);
|
||
|
||
const files = fs.readdirSync('public/characters/'+directoriesB[key]+'/chats');
|
||
if (!fs.existsSync('public/chats/'+char.name)) {
|
||
fs.mkdirSync('public/chats/'+char.name);
|
||
}
|
||
files.forEach(function(file) {
|
||
// Read the contents of the file
|
||
|
||
const fileContents = fs.readFileSync('public/characters/'+directoriesB[key]+'/chats/' + file, 'utf8');
|
||
|
||
|
||
// Iterate through the array of strings and parse each line as JSON
|
||
let chat_data = JSON.parse(fileContents);
|
||
let new_chat_data = [];
|
||
let this_chat_user_name = 'You';
|
||
let is_pass_0 = false;
|
||
if(chat_data[0].indexOf('<username-holder>') !== -1){
|
||
this_chat_user_name = chat_data[0].substr('<username-holder>'.length, chat_data[0].length);
|
||
is_pass_0 = true;
|
||
}
|
||
let i = 0;
|
||
let ii = 0;
|
||
new_chat_data[i] = {user_name:'You', character_name:char.name, create_date: Date.now()};
|
||
i++;
|
||
ii++;
|
||
chat_data.forEach(function(mes) {
|
||
if(!(i === 1 && is_pass_0)){
|
||
if(mes.indexOf('<username-holder>') === -1 && mes.indexOf('<username-idkey>') === -1){
|
||
new_chat_data[ii] = {};
|
||
let is_name = false;
|
||
if(mes.trim().indexOf(this_chat_user_name+':') !== 0){
|
||
if(mes.trim().indexOf(char.name+':') === 0){
|
||
mes = mes.replace(char.name+':','');
|
||
is_name = true;
|
||
}
|
||
new_chat_data[ii]['name'] = char.name;
|
||
new_chat_data[ii]['is_user'] = false;
|
||
new_chat_data[ii]['is_name'] = is_name;
|
||
new_chat_data[ii]['send_date'] = Date.now();
|
||
|
||
}else{
|
||
mes = mes.replace(this_chat_user_name+':','');
|
||
new_chat_data[ii]['name'] = 'You';
|
||
new_chat_data[ii]['is_user'] = true;
|
||
new_chat_data[ii]['is_name'] = true;
|
||
new_chat_data[ii]['send_date'] = Date.now();
|
||
|
||
}
|
||
new_chat_data[ii]['mes'] = mes.trim();
|
||
ii++;
|
||
}
|
||
}
|
||
i++;
|
||
|
||
});
|
||
const jsonlData = new_chat_data.map(JSON.stringify).join('\n');
|
||
// Write the contents to the destination folder
|
||
fs.writeFileSync('public/chats/'+char.name+'/' + file+'l', jsonlData);
|
||
});
|
||
//fs.rmSync('public/characters/'+directoriesB[key],{ recursive: true });
|
||
console.log(char.name+' update!');
|
||
}
|
||
//removeFolders('public/characters');
|
||
fs.unlinkSync('public/characters/update.txt');
|
||
if(mes == false){
|
||
console.log('***');
|
||
console.log('Сharacter format update completed successfully!');
|
||
console.log('***');
|
||
console.log('Now you can delete these folders, they are no longer used by TavernAI:');
|
||
}
|
||
for (const key in directoriesB) {
|
||
console.log('public/characters/'+directoriesB[key]);
|
||
}
|
||
}
|
||
function removeFolders(folder) {
|
||
const files = fs.readdirSync(folder);
|
||
files.forEach(function(file) {
|
||
const filePath = folder + '/' + file;
|
||
const stat = fs.statSync(filePath);
|
||
if (stat.isDirectory()) {
|
||
removeFolders(filePath);
|
||
fs.rmdirSync(filePath);
|
||
}
|
||
});
|
||
}
|
||
|
||
function copyFolder(src, dest) {
|
||
const files = fs.readdirSync(src);
|
||
files.forEach(function(file) {
|
||
const filePath = src + '/' + file;
|
||
const stat = fs.statSync(filePath);
|
||
if (stat.isFile()) {
|
||
fs.copyFileSync(filePath, dest + '/' + file);
|
||
} else if (stat.isDirectory()) {
|
||
fs.mkdirSync(dest + '/' + file);
|
||
copyFolder(filePath, dest + '/' + file);
|
||
}
|
||
});
|
||
}
|
||
|
||
|
||
function getDirectories2(path) {
|
||
return fs.readdirSync(path)
|
||
.filter(function (file) {
|
||
return fs.statSync(path + '/' + file).isDirectory();
|
||
})
|
||
.sort(function (a, b) {
|
||
return new Date(fs.statSync(path + '/' + a).mtime) - new Date(fs.statSync(path + '/' + b).mtime);
|
||
})
|
||
.reverse();
|
||
}
|
||
function getCharaterFile2(directories,i){
|
||
if(directories.length > i){
|
||
fs.stat('public/characters/'+directories[i]+'/'+directories[i]+".json", function(err, stat) {
|
||
if (err == null) {
|
||
fs.readFile('public/characters/'+directories[i]+'/'+directories[i]+".json", 'utf8', (err, data) => {
|
||
if (err) {
|
||
console.error(err);
|
||
return;
|
||
}
|
||
//console.log(data);
|
||
if (!fs.existsSync('public/characters/'+directories[i]+'.png')) {
|
||
charactersB[character_ib] = {};
|
||
charactersB[character_ib] = data;
|
||
directoriesB[character_ib] = directories[i];
|
||
character_ib++;
|
||
}
|
||
i++;
|
||
getCharaterFile2(directories,i);
|
||
});
|
||
}else{
|
||
i++;
|
||
getCharaterFile2(directories,i);
|
||
}
|
||
});
|
||
}else{
|
||
convertStage2();
|
||
}
|
||
}
|
||
/*
|
||
* async function aaa2(){
|
||
try {
|
||
// Load the image in any format
|
||
|
||
const image = await sharp('./original.jpg').resize(100, 100).toFormat('png');
|
||
|
||
image.metadata((err, metadata) => {
|
||
if (err) throw err;
|
||
if (!metadata.chunks) {
|
||
metadata.chunks = [];
|
||
}
|
||
const textData = text.encode('hello', 'world');
|
||
const textBuffer = Buffer.from(`${textData.keyword}\0${textData.text}`);
|
||
metadata.chunks.push({
|
||
type: 'tEXt',
|
||
data: textBuffer
|
||
});
|
||
return metadata;
|
||
})
|
||
.toFile('test-out.png')
|
||
.then(() => {
|
||
console.log('PNG image with tEXt chunks has been saved.');
|
||
})
|
||
.catch((err) => {
|
||
console.log(err);
|
||
});
|
||
} catch (err) {
|
||
console.log(err);
|
||
}
|
||
}
|
||
* function writePNG(){
|
||
|
||
const buffer = fs.readFileSync('test-out.png');
|
||
const chunks = extract(buffer);
|
||
const tEXtChunks = chunks.filter(chunk => chunk.name === 'tEXt');
|
||
|
||
// Remove all existing tEXt chunks
|
||
for (const tEXtChunk of tEXtChunks) {
|
||
chunks.splice(chunks.indexOf(tEXtChunk), 1);
|
||
}
|
||
// Add new chunks before the IEND chunk
|
||
chunks.splice(-1, 0, text.encode('hello', 'world'));
|
||
chunks.splice(-1, 0, text.encode('lorem', 'ipsum'));
|
||
|
||
fs.writeFileSync(
|
||
'test-out.png',
|
||
new Buffer.from(encode(chunks))
|
||
);
|
||
}
|
||
* function readPNG2(){
|
||
sharp('./test-out.png')
|
||
.metadata()
|
||
.then((metadata) => {
|
||
console.log(metadata);
|
||
if (!metadata.chunks) {
|
||
console.log("No tEXt chunks found in the image file");
|
||
}
|
||
const textChunks = metadata.chunks.filter((chunk) => chunk.type === 'tEXt');
|
||
textChunks.forEach((textChunk) => {
|
||
const textData = JSON.parse(textChunk.data.toString());
|
||
console.log(textData);
|
||
});
|
||
})
|
||
.catch((err) => {
|
||
console.log(err);
|
||
});
|
||
}
|
||
const requestListener = function (req, res) {
|
||
fs.readFile(__dirname + "/index.html")
|
||
.then(contents => {
|
||
res.setHeader("Content-Type", "text/html");
|
||
res.writeHead(200);
|
||
res.end(contents);
|
||
})
|
||
.catch(err => {
|
||
res.writeHead(500);
|
||
res.end(err);
|
||
return;
|
||
});
|
||
};
|
||
|
||
|
||
|
||
const server = http.createServer(requestListener);
|
||
server.listen(port, host, () => {
|
||
console.log(`Server is running on http://${host}:${port}`);
|
||
});
|
||
*/
|
||
|