Merge pull request #996 from kingbased/scale
Alternative method for scale generations
This commit is contained in:
Cohee
GitHub
commit
0ffb3a8dda
|
|
@ -635,7 +635,7 @@
|
|||
Max prompt cost: <span id="openrouter_max_prompt_cost">Unknown</span>
|
||||
</div>
|
||||
<hr>
|
||||
<div class="range-block" data-source="openai,claude,windowai,openrouter,ai21">
|
||||
<div class="range-block" data-source="openai,claude,windowai,openrouter,ai21,scale">
|
||||
<div class="range-block-title" data-i18n="Temperature">
|
||||
Temperature
|
||||
</div>
|
||||
|
|
@ -710,7 +710,7 @@
|
|||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="range-block" data-source="openai,claude,openrouter,ai21">
|
||||
<div class="range-block" data-source="openai,claude,openrouter,ai21,scale">
|
||||
<div class="range-block-title" data-i18n="Top-p">
|
||||
Top P
|
||||
</div>
|
||||
|
|
@ -1578,7 +1578,7 @@
|
|||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="range-block m-t-1" data-source="openai,openrouter">
|
||||
<div class="range-block m-t-1" data-source="openai,openrouter,scale">
|
||||
<div class="range-block-title openai_restorable" data-i18n="Logit Bias">
|
||||
Logit Bias
|
||||
</div>
|
||||
|
|
@ -1976,18 +1976,34 @@
|
|||
</form>
|
||||
|
||||
<form id="scale_form" data-source="scale" action="javascript:void(null);" method="post" enctype="multipart/form-data">
|
||||
<h4>Scale API Key</h4>
|
||||
<div class="flex-container">
|
||||
<input id="api_key_scale" name="api_key_scale" class="text_pole flex1" maxlength="500" value="" autocomplete="off">
|
||||
<div title="Clear your API key" data-i18n="[title]Clear your API key" class="menu_button fa-solid fa-circle-xmark clear-api-key" data-key="api_key_scale"></div>
|
||||
<div id="normal_scale_form">
|
||||
<h4>Scale API Key</h4>
|
||||
<div class="flex-container">
|
||||
<input id="api_key_scale" name="api_key_scale" class="text_pole flex1" maxlength="500" value="" autocomplete="off">
|
||||
<div title="Clear your API key" data-i18n="[title]Clear your API key" class="menu_button fa-solid fa-circle-xmark clear-api-key" data-key="api_key_scale"></div>
|
||||
</div>
|
||||
<div data-for="api_key_scale" class="neutral_warning">
|
||||
For privacy reasons, your API key will be hidden after you reload the page.
|
||||
</div>
|
||||
<h4>Scale API URL</h4>
|
||||
<input id="api_url_scale" name="api_url_scale" class="text_pole" maxlength="500" value="" autocomplete="off" placeholder="https://dashboard.scale.com/spellbook/api/v2/deploy/xxxxxxx">
|
||||
</div>
|
||||
<div data-for="api_key_scale" class="neutral_warning">
|
||||
For privacy reasons, your API key will be hidden after you reload the page.
|
||||
<div id="alt_scale_form">
|
||||
<h4>Scale Cookie (_jwt)</h4>
|
||||
<div class="flex-container">
|
||||
<input id="scale_cookie" name="scale_cookie" class="text_pole flex1" maxlength="500" value="" autocomplete="off">
|
||||
<div title="Clear your cookie" data-i18n="[title]Clear your cookie" class="menu_button fa-solid fa-circle-xmark clear-api-key" data-key="scale_cookie"></div>
|
||||
</div>
|
||||
<div data-for="scale_cookie" class="neutral_warning">
|
||||
For privacy reasons, your cookie will be hidden after you reload the page.
|
||||
</div>
|
||||
</div>
|
||||
<h4>Scale API URL</h4>
|
||||
<input id="api_url_scale" name="api_url_scale" class="text_pole" maxlength="500" value="" autocomplete="off" placeholder="https://dashboard.scale.com/spellbook/api/v2/deploy/xxxxxxx">
|
||||
<!-- Its only purpose is to trigger max context size check -->
|
||||
<select id="model_scale_select" class="displayNone"></select>
|
||||
<label for="scale-alt" class="checkbox_label">
|
||||
<input id="scale-alt" type="checkbox" checked>
|
||||
<span data-i18n="Alt Method">Alt Method</span>
|
||||
</label>
|
||||
</form>
|
||||
|
||||
<form id="ai21_form" data-source="ai21" action="javascript:void(null);" method="post" enctype="multipart/form-data">
|
||||
|
|
|
|||
|
|
@ -365,7 +365,7 @@ function RA_autoconnect(PrevApi) {
|
|||
case 'openai':
|
||||
if (((secret_state[SECRET_KEYS.OPENAI] || oai_settings.reverse_proxy) && oai_settings.chat_completion_source == chat_completion_sources.OPENAI)
|
||||
|| ((secret_state[SECRET_KEYS.CLAUDE] || oai_settings.reverse_proxy) && oai_settings.chat_completion_source == chat_completion_sources.CLAUDE)
|
||||
|| (secret_state[SECRET_KEYS.SCALE] && oai_settings.chat_completion_source == chat_completion_sources.SCALE)
|
||||
|| ((secret_state[SECRET_KEYS.SCALE] || secret_state[SECRET_KEYS.SCALE_COOKIE]) && oai_settings.chat_completion_source == chat_completion_sources.SCALE)
|
||||
|| (oai_settings.chat_completion_source == chat_completion_sources.WINDOWAI)
|
||||
|| (secret_state[SECRET_KEYS.OPENROUTER] && oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER)
|
||||
|| (secret_state[SECRET_KEYS.AI21] && oai_settings.chat_completion_source == chat_completion_sources.AI21)
|
||||
|
|
|
|||
|
|
@ -1386,7 +1386,7 @@ export async function createNewGroupChat(groupId) {
|
|||
group.chat_metadata = {};
|
||||
updateChatMetadata(group.chat_metadata, true);
|
||||
|
||||
await editGroup(group.id, true);
|
||||
await editGroup(group.id, true, false);
|
||||
await getGroupChat(group.id);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -109,8 +109,8 @@ const max_4k = 4095;
|
|||
const max_8k = 8191;
|
||||
const max_16k = 16383;
|
||||
const max_32k = 32767;
|
||||
const scale_max = 7900; // Probably more. Save some for the system prompt defined on Scale site.
|
||||
const claude_max = 8000; // We have a proper tokenizer, so theoretically could be larger (up to 9k)
|
||||
const scale_max = 8191;
|
||||
const claude_max = 9000; // We have a proper tokenizer, so theoretically could be larger (up to 9k)
|
||||
const palm2_max = 7500; // The real context window is 8192, spare some for padding due to using turbo tokenizer
|
||||
const claude_100k_max = 99000;
|
||||
let ai21_max = 9200; //can easily fit 9k gpt tokens because j2's tokenizer is efficient af
|
||||
|
|
@ -219,6 +219,7 @@ const default_settings = {
|
|||
assistant_prefill: '',
|
||||
use_ai21_tokenizer: false,
|
||||
exclude_assistant: false,
|
||||
use_alt_scale: false,
|
||||
};
|
||||
|
||||
const oai_settings = {
|
||||
|
|
@ -261,6 +262,7 @@ const oai_settings = {
|
|||
assistant_prefill: '',
|
||||
use_ai21_tokenizer: false,
|
||||
exclude_assistant: false,
|
||||
use_alt_scale: false,
|
||||
};
|
||||
|
||||
let openai_setting_names;
|
||||
|
|
@ -1082,6 +1084,47 @@ function saveModelList(data) {
|
|||
}
|
||||
}
|
||||
|
||||
async function sendAltScaleRequest(openai_msgs_tosend, logit_bias, signal) {
|
||||
const generate_url = '/generate_altscale';
|
||||
|
||||
let firstSysMsgs = []
|
||||
for(let msg of openai_msgs_tosend){
|
||||
if(msg.role === 'system') {
|
||||
firstSysMsgs.push(substituteParams(msg.name ? msg.name + ": " + msg.content : msg.content));
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let subsequentMsgs = openai_msgs_tosend.slice(firstSysMsgs.length);
|
||||
|
||||
const joinedSysMsgs = substituteParams(firstSysMsgs.join("\n"));
|
||||
const joinedSubsequentMsgs = subsequentMsgs.reduce((acc, obj) => {
|
||||
return acc + obj.role + ": " + obj.content + "\n";
|
||||
}, "");
|
||||
|
||||
openai_msgs_tosend = substituteParams(joinedSubsequentMsgs);
|
||||
|
||||
const generate_data = {
|
||||
sysprompt: joinedSysMsgs,
|
||||
prompt: openai_msgs_tosend,
|
||||
temp: parseFloat(oai_settings.temp_openai),
|
||||
top_p: parseFloat(oai_settings.top_p_openai),
|
||||
max_tokens: parseFloat(oai_settings.openai_max_tokens),
|
||||
logit_bias: logit_bias,
|
||||
}
|
||||
|
||||
const response = await fetch(generate_url, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(generate_data),
|
||||
headers: getRequestHeaders(),
|
||||
signal: signal
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
return data.output;
|
||||
}
|
||||
|
||||
async function sendOpenAIRequest(type, openai_msgs_tosend, signal) {
|
||||
// Provide default abort signal
|
||||
if (!signal) {
|
||||
|
|
@ -1118,7 +1161,7 @@ async function sendOpenAIRequest(type, openai_msgs_tosend, signal) {
|
|||
return sendWindowAIRequest(openai_msgs_tosend, signal, stream);
|
||||
}
|
||||
|
||||
const logitBiasSources = [chat_completion_sources.OPENAI, chat_completion_sources.OPENROUTER];
|
||||
const logitBiasSources = [chat_completion_sources.OPENAI, chat_completion_sources.OPENROUTER, chat_completion_sources.SCALE];
|
||||
if (oai_settings.bias_preset_selected
|
||||
&& logitBiasSources.includes(oai_settings.chat_completion_source)
|
||||
&& Array.isArray(oai_settings.bias_presets[oai_settings.bias_preset_selected])
|
||||
|
|
@ -1127,6 +1170,10 @@ async function sendOpenAIRequest(type, openai_msgs_tosend, signal) {
|
|||
biasCache = logit_bias;
|
||||
}
|
||||
|
||||
if (isScale && oai_settings.use_alt_scale) {
|
||||
return sendAltScaleRequest(openai_msgs_tosend, logit_bias, signal)
|
||||
}
|
||||
|
||||
const model = getChatCompletionModel();
|
||||
const generate_data = {
|
||||
"messages": openai_msgs_tosend,
|
||||
|
|
@ -1971,6 +2018,7 @@ function loadOpenAISettings(data, settings) {
|
|||
if (settings.openai_model !== undefined) oai_settings.openai_model = settings.openai_model;
|
||||
if (settings.use_ai21_tokenizer !== undefined) { oai_settings.use_ai21_tokenizer = !!settings.use_ai21_tokenizer; oai_settings.use_ai21_tokenizer ? ai21_max = 8191 : ai21_max = 9200; }
|
||||
if (settings.exclude_assistant !== undefined) oai_settings.exclude_assistant = !!settings.exclude_assistant;
|
||||
if (settings.use_alt_scale !== undefined) { oai_settings.use_alt_scale = !!settings.use_alt_scale; updateScaleForm(); }
|
||||
$('#stream_toggle').prop('checked', oai_settings.stream_openai);
|
||||
$('#api_url_scale').val(oai_settings.api_url_scale);
|
||||
$('#openai_proxy_password').val(oai_settings.proxy_password);
|
||||
|
|
@ -2001,6 +2049,7 @@ function loadOpenAISettings(data, settings) {
|
|||
$('#openai_external_category').toggle(oai_settings.show_external_models);
|
||||
$('#use_ai21_tokenizer').prop('checked', oai_settings.use_ai21_tokenizer);
|
||||
$('#exclude_assistant').prop('checked', oai_settings.exclude_assistant);
|
||||
$('#scale-alt').prop('checked', oai_settings.use_alt_scale);
|
||||
if (settings.impersonation_prompt !== undefined) oai_settings.impersonation_prompt = settings.impersonation_prompt;
|
||||
|
||||
$('#impersonation_prompt_textarea').val(oai_settings.impersonation_prompt);
|
||||
|
|
@ -2199,6 +2248,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
|
|||
assistant_prefill: settings.assistant_prefill,
|
||||
use_ai21_tokenizer: settings.use_ai21_tokenizer,
|
||||
exclude_assistant: settings.exclude_assistant,
|
||||
use_alt_scale: settings.use_alt_scale,
|
||||
};
|
||||
|
||||
const savePresetSettings = await fetch(`/savepreset_openai?name=${name}`, {
|
||||
|
|
@ -2536,7 +2586,8 @@ function onSettingsPresetChange() {
|
|||
assistant_prefill: ['#claude_assistant_prefill', 'assistant_prefill', false],
|
||||
use_ai21_tokenizer: ['#use_ai21_tokenizer', 'use_ai21_tokenizer', false],
|
||||
exclude_assistant: ['#exclude_assistant', 'exclude_assistant', false],
|
||||
};
|
||||
use_alt_scale: ['#use_alt_scale', 'use_alt_scale', false],
|
||||
};
|
||||
|
||||
const presetName = $('#settings_perset_openai').find(":selected").text();
|
||||
oai_settings.preset_settings_openai = presetName;
|
||||
|
|
@ -2831,20 +2882,31 @@ async function onConnectButtonClick(e) {
|
|||
|
||||
if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) {
|
||||
const api_key_scale = $('#api_key_scale').val().trim();
|
||||
const scale_cookie = $('#scale_cookie').val().trim();
|
||||
|
||||
if (api_key_scale.length) {
|
||||
await writeSecret(SECRET_KEYS.SCALE, api_key_scale);
|
||||
}
|
||||
|
||||
if (!oai_settings.api_url_scale) {
|
||||
if (scale_cookie.length) {
|
||||
await writeSecret(SECRET_KEYS.SCALE_COOKIE, scale_cookie);
|
||||
}
|
||||
|
||||
if (!oai_settings.api_url_scale && !oai_settings.use_alt_scale) {
|
||||
console.log('No API URL saved for Scale');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!secret_state[SECRET_KEYS.SCALE]) {
|
||||
if (!secret_state[SECRET_KEYS.SCALE] && !oai_settings.use_alt_scale) {
|
||||
console.log('No secret key saved for Scale');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!secret_state[SECRET_KEYS.SCALE_COOKIE] && oai_settings.use_alt_scale) {
|
||||
console.log("No cookie set for Scale");
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (oai_settings.chat_completion_source == chat_completion_sources.CLAUDE) {
|
||||
|
|
@ -2958,11 +3020,27 @@ function onProxyPasswordShowClick() {
|
|||
$(this).toggleClass('fa-eye-slash fa-eye');
|
||||
}
|
||||
|
||||
function updateScaleForm() {
|
||||
if (oai_settings.use_alt_scale) {
|
||||
$('#normal_scale_form').css('display', 'none');
|
||||
$('#alt_scale_form').css('display', '');
|
||||
} else {
|
||||
$('#normal_scale_form').css('display', '');
|
||||
$('#alt_scale_form').css('display', 'none');
|
||||
}
|
||||
}
|
||||
|
||||
$(document).ready(async function () {
|
||||
await loadTokenCache();
|
||||
|
||||
$('#test_api_button').on('click', testApiConnection);
|
||||
|
||||
$('#scale-alt').on('change', function () {
|
||||
oai_settings.use_alt_scale = !!$('#scale-alt').prop('checked');
|
||||
saveSettingsDebounced();
|
||||
updateScaleForm();
|
||||
});
|
||||
|
||||
$(document).on('input', '#temp_openai', function () {
|
||||
oai_settings.temp_openai = Number($(this).val());
|
||||
$('#temp_counter_openai').text(Number($(this).val()).toFixed(2));
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ export const SECRET_KEYS = {
|
|||
OPENROUTER: 'api_key_openrouter',
|
||||
SCALE: 'api_key_scale',
|
||||
AI21: 'api_key_ai21',
|
||||
SCALE_COOKIE: 'scale_cookie',
|
||||
}
|
||||
|
||||
const INPUT_MAP = {
|
||||
|
|
@ -20,6 +21,7 @@ const INPUT_MAP = {
|
|||
[SECRET_KEYS.OPENROUTER]: '#api_key_openrouter',
|
||||
[SECRET_KEYS.SCALE]: '#api_key_scale',
|
||||
[SECRET_KEYS.AI21]: '#api_key_ai21',
|
||||
[SECRET_KEYS.SCALE_COOKIE]: '#scale_cookie',
|
||||
}
|
||||
|
||||
async function clearSecret() {
|
||||
|
|
|
|||
69
server.js
69
server.js
|
|
@ -3300,6 +3300,72 @@ async function sendScaleRequest(request, response) {
|
|||
}
|
||||
}
|
||||
|
||||
app.post("/generate_altscale", jsonParser, function (request, response_generate_scale) {
|
||||
if(!request.body) return response_generate_scale.sendStatus(400);
|
||||
|
||||
fetch('https://dashboard.scale.com/spellbook/api/trpc/v2.variant.run', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'cookie': `_jwt=${readSecret(SECRET_KEYS.SCALE_COOKIE)}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
json: {
|
||||
variant: {
|
||||
name: 'New Variant',
|
||||
appId: '',
|
||||
taxonomy: null
|
||||
},
|
||||
prompt: {
|
||||
id: '',
|
||||
template: '{{input}}\n',
|
||||
exampleVariables: {},
|
||||
variablesSourceDataId: null,
|
||||
systemMessage: request.body.sysprompt
|
||||
},
|
||||
modelParameters: {
|
||||
id: '',
|
||||
modelId: 'GPT4',
|
||||
modelType: 'OpenAi',
|
||||
maxTokens: request.body.max_tokens,
|
||||
temperature: request.body.temp,
|
||||
stop: "user:",
|
||||
suffix: null,
|
||||
topP: request.body.top_p,
|
||||
logprobs: null,
|
||||
logitBias: request.body.logit_bias
|
||||
},
|
||||
inputs: [
|
||||
{
|
||||
index: '-1',
|
||||
valueByName: {
|
||||
input: request.body.prompt
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
meta: {
|
||||
values: {
|
||||
'variant.taxonomy': ['undefined'],
|
||||
'prompt.variablesSourceDataId': ['undefined'],
|
||||
'modelParameters.suffix': ['undefined'],
|
||||
'modelParameters.logprobs': ['undefined'],
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
.then(response => response.json())
|
||||
.then(data => {
|
||||
console.log(data.result.data.json.outputs[0])
|
||||
return response_generate_scale.send({output: data.result.data.json.outputs[0]});
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error('Error:', error)
|
||||
return response_generate_scale.send({error: true})
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
async function sendClaudeRequest(request, response) {
|
||||
const fetch = require('node-fetch').default;
|
||||
|
||||
|
|
@ -3988,7 +4054,8 @@ const SECRET_KEYS = {
|
|||
DEEPL: 'deepl',
|
||||
OPENROUTER: 'api_key_openrouter',
|
||||
SCALE: 'api_key_scale',
|
||||
AI21: 'api_key_ai21'
|
||||
AI21: 'api_key_ai21',
|
||||
SCALE_COOKIE: 'scale_cookie',
|
||||
}
|
||||
|
||||
function migrateSecrets() {
|
||||
|
|
|
|||
Loading…
Reference in New Issue