Compare commits

...

10 Commits

Author SHA1 Message Date
Himickoff a6b1efae84
Merge 95d3304146 into e6e8d7726b 2024-07-26 15:50:46 +03:00
Cohee e6e8d7726b Generate random seed for HF endpoint 2024-07-26 12:50:39 +00:00
Himickoff 95d3304146 Translate comments to en 2024-07-14 17:27:09 +07:00
Himickoff 378be40210 Add more ru comments 2024-07-14 02:35:20 +07:00
Himickoff 00350df82d Fix for other languages
But again small problem with decodig
2024-07-14 01:29:33 +07:00
Himickoff 9cdf2216a2 Now formating apply only for russian language 2024-07-14 00:58:59 +07:00
Himickoff c0d05040e6 Fix syntax 2024-07-14 00:42:05 +07:00
Himickoff 898657dde4 Improved rus google translate 2024-07-14 00:32:19 +07:00
Himickoff 8e17117854 Fix quartes
Google translator, when translating direct speech, for some reason adds an additional quotation mark, which breaks the syntax
1.Before submitting text for translation, we replace all quotation marks with special markers (__OPEN_QUOTE__ and __CLOSE_QUOTE__).
This ensures that quotes are not changed or deleted during translation.
2.Similarly, we replace asterisks with markers (__OPEN_STAR__ and __CLOSE_STAR__).
3.After receiving the translated text, we replace all markers back to the corresponding characters.
2024-07-13 19:13:08 +07:00
Himickoff bf682be17a Translate fix
Fix problem when Googole Transtale replace "" on «» during translation, what broke the syntax
2024-07-13 18:42:30 +07:00
2 changed files with 64 additions and 9 deletions

View File

@ -1184,7 +1184,7 @@ export function getTextGenGenerationData(finalPrompt, maxTokens, isImpersonate,
if (settings.type === HUGGINGFACE) {
params.top_p = Math.min(Math.max(Number(params.top_p), 0.0), 0.999);
params.stop = Array.isArray(params.stop) ? params.stop.slice(0, 4) : [];
nonAphroditeParams.seed = settings.seed >= 0 ? settings.seed : undefined;
nonAphroditeParams.seed = settings.seed >= 0 ? settings.seed : Math.floor(Math.random() * Math.pow(2, 32));
}
if (settings.type === MANCER) {

View File

@ -4,6 +4,9 @@ const express = require('express');
const { readSecret, SECRET_KEYS } = require('./secrets');
const { getConfigValue, uuidv4 } = require('../util');
const { jsonParser } = require('../express-common');
const iconv = require('iconv-lite');
const { generateRequestUrl, normaliseResponse } = require('google-translate-api-browser');
const bodyParser = require('body-parser');
const DEEPLX_URL_DEFAULT = 'http://127.0.0.1:1188/translate';
const ONERING_URL_DEFAULT = 'http://127.0.0.1:4990/translate';
@ -65,39 +68,88 @@ router.post('/libre', jsonParser, async (request, response) => {
}
});
router.post('/google', jsonParser, async (request, response) => {
try {
const { generateRequestUrl, normaliseResponse } = require('google-translate-api-browser');
const text = request.body.text;
let text = request.body.text;
const lang = request.body.lang;
if (!text || !lang) {
return response.sendStatus(400);
}
console.log('Input text: ' + text);
console.log('Input text:', text);
//console.log('----------');
// Apply formatting only if the translation language is Russian
if (lang === 'ru') {
// Replace quotation marks and asterisks with special markers before translating
const openQuote = '__OPEN_QUOTE__ ';
const closeQuote = ' __CLOSE_QUOTE__';
const openStar = '__OPEN_STAR__ ';
const closeStar = ' __CLOSE_STAR__';
// A very rare case
const very_rare_execption = '__OPEN_STAR__, ';
// Use counters to alternate between opening and closing markers
let quoteCounter = 0;
let starCounter = 0;
text = text.replace(/"/g, () => (quoteCounter++ % 2 === 0 ? openQuote : closeQuote));
text = text.replace(/\*/g, () => (starCounter++ % 2 === 0 ? openStar : closeStar));
//console.log('Pre-Input text:', text);
//console.log('----------');
}
const url = generateRequestUrl(text, { to: lang });
https.get(url, (resp) => {
let data = '';
let data = [];
resp.on('data', (chunk) => {
data += chunk;
data.push(chunk);
});
resp.on('end', () => {
try {
const result = normaliseResponse(JSON.parse(data));
console.log('Translated text: ' + result.text);
return response.send(result.text);
let result;
if (lang === 'ru') {
// Decoding for Russian language
const decodedData = iconv.decode(Buffer.concat(data), 'utf-8');
result = normaliseResponse(JSON.parse(decodedData));
} else {
// For other languages, we use the data as is
result = normaliseResponse(JSON.parse(Buffer.concat(data).toString()));
}
//console.log('Pre-Translated text:', result.text);
//console.log('----------');
let fixedText = result.text;
// Restore formatting only if the target language is Russian
if (lang === 'ru') {
fixedText = result.text
.replace(new RegExp('__OPEN_QUOTE__ ', 'g'), '"')
.replace(new RegExp(' __CLOSE_QUOTE__', 'g'), '"')
.replace(new RegExp('__OPEN_STAR__ ', 'g'), '*')
.replace(new RegExp(' __CLOSE_STAR__', 'g'), '*')
.replace(new RegExp('__OPEN_STAR__, ', 'g'), '*');
}
console.log('Translated text:', fixedText);
response.setHeader('Content-Type', 'text/plain; charset=utf-8');
return response.send(fixedText);
} catch (error) {
console.log('Translation error', error);
return response.sendStatus(500);
}
});
}).on('error', (err) => {
console.log('Translation error: ' + err.message);
console.log('Translation error:', err.message);
return response.sendStatus(500);
});
} catch (error) {
@ -106,6 +158,9 @@ router.post('/google', jsonParser, async (request, response) => {
}
});
router.post('/yandex', jsonParser, async (request, response) => {
const chunks = request.body.chunks;
const lang = request.body.lang;