Add OpenAI vector source.

This commit is contained in:
Cohee
2023-09-08 13:57:27 +03:00
parent 02bdd56e20
commit a5acc7872d
8 changed files with 345 additions and 159 deletions

120
server.js
View File

@@ -68,6 +68,7 @@ const characterCardParser = require('./src/character-card-parser.js');
const contentManager = require('./src/content-manager');
const novelai = require('./src/novelai');
const statsHelpers = require('./statsHelpers.js');
const { writeSecret, readSecret, readSecretState, migrateSecrets, SECRET_KEYS, getAllSecrets } = require('./src/secrets');
function createDefaultFiles() {
const files = {
@@ -325,6 +326,7 @@ function humanizedISO8601DateTime(date) {
var charactersPath = 'public/characters/';
var chatsPath = 'public/chats/';
const UPLOADS_PATH = './uploads';
const SETTINGS_FILE = './public/settings.json';
const AVATAR_WIDTH = 400;
const AVATAR_HEIGHT = 600;
const jsonParser = express.json({ limit: '100mb' });
@@ -4070,7 +4072,7 @@ const setupTasks = async function () {
console.log(`SillyTavern ${version.pkgVersion}` + (version.gitBranch ? ` '${version.gitBranch}' (${version.gitRevision})` : ''));
backupSettings();
migrateSecrets();
migrateSecrets(SETTINGS_FILE);
ensurePublicDirectoriesExist();
await ensureThumbnailCache();
contentManager.checkForNewContent();
@@ -4221,69 +4223,6 @@ function ensurePublicDirectoriesExist() {
}
}
const SECRETS_FILE = './secrets.json';
const SETTINGS_FILE = './public/settings.json';
const SECRET_KEYS = {
HORDE: 'api_key_horde',
MANCER: 'api_key_mancer',
OPENAI: 'api_key_openai',
NOVEL: 'api_key_novel',
CLAUDE: 'api_key_claude',
DEEPL: 'deepl',
LIBRE: 'libre',
LIBRE_URL: 'libre_url',
OPENROUTER: 'api_key_openrouter',
SCALE: 'api_key_scale',
AI21: 'api_key_ai21',
SCALE_COOKIE: 'scale_cookie',
}
function migrateSecrets() {
if (!fs.existsSync(SETTINGS_FILE)) {
console.log('Settings file does not exist');
return;
}
try {
let modified = false;
const fileContents = fs.readFileSync(SETTINGS_FILE, 'utf8');
const settings = JSON.parse(fileContents);
const oaiKey = settings?.api_key_openai;
const hordeKey = settings?.horde_settings?.api_key;
const novelKey = settings?.api_key_novel;
if (typeof oaiKey === 'string') {
console.log('Migrating OpenAI key...');
writeSecret(SECRET_KEYS.OPENAI, oaiKey);
delete settings.api_key_openai;
modified = true;
}
if (typeof hordeKey === 'string') {
console.log('Migrating Horde key...');
writeSecret(SECRET_KEYS.HORDE, hordeKey);
delete settings.horde_settings.api_key;
modified = true;
}
if (typeof novelKey === 'string') {
console.log('Migrating Novel key...');
writeSecret(SECRET_KEYS.NOVEL, novelKey);
delete settings.api_key_novel;
modified = true;
}
if (modified) {
console.log('Writing updated settings.json...');
const settingsContent = JSON.stringify(settings);
writeFileAtomicSync(SETTINGS_FILE, settingsContent, "utf-8");
}
}
catch (error) {
console.error('Could not migrate secrets file. Proceed with caution.');
}
}
app.post('/writesecret', jsonParser, (request, response) => {
const key = request.body.key;
const value = request.body.value;
@@ -4293,19 +4232,9 @@ app.post('/writesecret', jsonParser, (request, response) => {
});
app.post('/readsecretstate', jsonParser, (_, response) => {
if (!fs.existsSync(SECRETS_FILE)) {
return response.send({});
}
try {
const fileContents = fs.readFileSync(SECRETS_FILE, 'utf8');
const secrets = JSON.parse(fileContents);
const state = {};
for (const key of Object.values(SECRET_KEYS)) {
state[key] = !!secrets[key]; // convert to boolean
}
const state = readSecretState();
return response.send(state);
} catch (error) {
console.error(error);
@@ -4351,14 +4280,13 @@ app.post('/viewsecrets', jsonParser, async (_, response) => {
return response.sendStatus(403);
}
if (!fs.existsSync(SECRETS_FILE)) {
console.error('secrets.json does not exist');
return response.sendStatus(404);
}
try {
const fileContents = fs.readFileSync(SECRETS_FILE, 'utf-8');
const secrets = JSON.parse(fileContents);
const secrets = getAllSecrets();
if (!secrets) {
return response.sendStatus(404);
}
return response.send(secrets);
} catch (error) {
console.error(error);
@@ -4797,6 +4725,11 @@ app.post('/libre_translate', jsonParser, async (request, response) => {
const key = readSecret(SECRET_KEYS.LIBRE);
const url = readSecret(SECRET_KEYS.LIBRE_URL);
if (!url) {
console.log('LibreTranslate URL is not configured.');
return response.sendStatus(401);
}
const text = request.body.text;
const lang = request.body.lang;
@@ -5292,27 +5225,6 @@ function importRisuSprites(data) {
}
}
function writeSecret(key, value) {
if (!fs.existsSync(SECRETS_FILE)) {
const emptyFile = JSON.stringify({});
writeFileAtomicSync(SECRETS_FILE, emptyFile, "utf-8");
}
const fileContents = fs.readFileSync(SECRETS_FILE, 'utf-8');
const secrets = JSON.parse(fileContents);
secrets[key] = value;
writeFileAtomicSync(SECRETS_FILE, JSON.stringify(secrets), "utf-8");
}
function readSecret(key) {
if (!fs.existsSync(SECRETS_FILE)) {
return undefined;
}
const fileContents = fs.readFileSync(SECRETS_FILE, 'utf-8');
const secrets = JSON.parse(fileContents);
return secrets[key];
}
async function readAllChunks(readableStream) {
return new Promise((resolve, reject) => {
@@ -5385,8 +5297,6 @@ async function getImageBuffers(zipFilePath) {
});
}
/**
* This function extracts the extension information from the manifest file.
* @param {string} extensionPath - The path of the extension folder