diff --git a/public/index.html b/public/index.html index 63ab8f9ce..14327cb34 100644 --- a/public/index.html +++ b/public/index.html @@ -635,7 +635,7 @@ Max prompt cost: Unknown
-
+
Temperature
@@ -710,7 +710,7 @@
-
+
Top P
@@ -1578,7 +1578,7 @@
-
+
Logit Bias
@@ -1976,18 +1976,34 @@
-

Scale API Key

-
- -
+
+

Scale API Key

+
+ +
+
+
+ For privacy reasons, your API key will be hidden after you reload the page. +
+

Scale API URL

+
-
- For privacy reasons, your API key will be hidden after you reload the page. +
+

Scale Cookie (_jwt)

+
+ +
+
+
+ For privacy reasons, your cookie will be hidden after you reload the page. +
-

Scale API URL

- +
diff --git a/public/scripts/RossAscends-mods.js b/public/scripts/RossAscends-mods.js index f17335e59..ac8f4e6b1 100644 --- a/public/scripts/RossAscends-mods.js +++ b/public/scripts/RossAscends-mods.js @@ -365,7 +365,7 @@ function RA_autoconnect(PrevApi) { case 'openai': if (((secret_state[SECRET_KEYS.OPENAI] || oai_settings.reverse_proxy) && oai_settings.chat_completion_source == chat_completion_sources.OPENAI) || ((secret_state[SECRET_KEYS.CLAUDE] || oai_settings.reverse_proxy) && oai_settings.chat_completion_source == chat_completion_sources.CLAUDE) - || (secret_state[SECRET_KEYS.SCALE] && oai_settings.chat_completion_source == chat_completion_sources.SCALE) + || ((secret_state[SECRET_KEYS.SCALE] || secret_state[SECRET_KEYS.SCALE_COOKIE]) && oai_settings.chat_completion_source == chat_completion_sources.SCALE) || (oai_settings.chat_completion_source == chat_completion_sources.WINDOWAI) || (secret_state[SECRET_KEYS.OPENROUTER] && oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER) || (secret_state[SECRET_KEYS.AI21] && oai_settings.chat_completion_source == chat_completion_sources.AI21) diff --git a/public/scripts/group-chats.js b/public/scripts/group-chats.js index b62249e89..ab6a1e0fd 100644 --- a/public/scripts/group-chats.js +++ b/public/scripts/group-chats.js @@ -1386,7 +1386,7 @@ export async function createNewGroupChat(groupId) { group.chat_metadata = {}; updateChatMetadata(group.chat_metadata, true); - await editGroup(group.id, true); + await editGroup(group.id, true, false); await getGroupChat(group.id); } diff --git a/public/scripts/openai.js b/public/scripts/openai.js index b720a684c..fdc411216 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -109,8 +109,8 @@ const max_4k = 4095; const max_8k = 8191; const max_16k = 16383; const max_32k = 32767; -const scale_max = 7900; // Probably more. Save some for the system prompt defined on Scale site. -const claude_max = 8000; // We have a proper tokenizer, so theoretically could be larger (up to 9k) +const scale_max = 8191; +const claude_max = 9000; // We have a proper tokenizer, so theoretically could be larger (up to 9k) const palm2_max = 7500; // The real context window is 8192, spare some for padding due to using turbo tokenizer const claude_100k_max = 99000; let ai21_max = 9200; //can easily fit 9k gpt tokens because j2's tokenizer is efficient af @@ -219,6 +219,7 @@ const default_settings = { assistant_prefill: '', use_ai21_tokenizer: false, exclude_assistant: false, + use_alt_scale: false, }; const oai_settings = { @@ -261,6 +262,7 @@ const oai_settings = { assistant_prefill: '', use_ai21_tokenizer: false, exclude_assistant: false, + use_alt_scale: false, }; let openai_setting_names; @@ -1082,6 +1084,47 @@ function saveModelList(data) { } } +async function sendAltScaleRequest(openai_msgs_tosend, logit_bias, signal) { + const generate_url = '/generate_altscale'; + + let firstSysMsgs = [] + for(let msg of openai_msgs_tosend){ + if(msg.role === 'system') { + firstSysMsgs.push(substituteParams(msg.name ? msg.name + ": " + msg.content : msg.content)); + } else { + break; + } + } + + let subsequentMsgs = openai_msgs_tosend.slice(firstSysMsgs.length); + + const joinedSysMsgs = substituteParams(firstSysMsgs.join("\n")); + const joinedSubsequentMsgs = subsequentMsgs.reduce((acc, obj) => { + return acc + obj.role + ": " + obj.content + "\n"; + }, ""); + + openai_msgs_tosend = substituteParams(joinedSubsequentMsgs); + + const generate_data = { + sysprompt: joinedSysMsgs, + prompt: openai_msgs_tosend, + temp: parseFloat(oai_settings.temp_openai), + top_p: parseFloat(oai_settings.top_p_openai), + max_tokens: parseFloat(oai_settings.openai_max_tokens), + logit_bias: logit_bias, + } + + const response = await fetch(generate_url, { + method: 'POST', + body: JSON.stringify(generate_data), + headers: getRequestHeaders(), + signal: signal + }); + + const data = await response.json(); + return data.output; +} + async function sendOpenAIRequest(type, openai_msgs_tosend, signal) { // Provide default abort signal if (!signal) { @@ -1118,7 +1161,7 @@ async function sendOpenAIRequest(type, openai_msgs_tosend, signal) { return sendWindowAIRequest(openai_msgs_tosend, signal, stream); } - const logitBiasSources = [chat_completion_sources.OPENAI, chat_completion_sources.OPENROUTER]; + const logitBiasSources = [chat_completion_sources.OPENAI, chat_completion_sources.OPENROUTER, chat_completion_sources.SCALE]; if (oai_settings.bias_preset_selected && logitBiasSources.includes(oai_settings.chat_completion_source) && Array.isArray(oai_settings.bias_presets[oai_settings.bias_preset_selected]) @@ -1127,6 +1170,10 @@ async function sendOpenAIRequest(type, openai_msgs_tosend, signal) { biasCache = logit_bias; } + if (isScale && oai_settings.use_alt_scale) { + return sendAltScaleRequest(openai_msgs_tosend, logit_bias, signal) + } + const model = getChatCompletionModel(); const generate_data = { "messages": openai_msgs_tosend, @@ -1971,6 +2018,7 @@ function loadOpenAISettings(data, settings) { if (settings.openai_model !== undefined) oai_settings.openai_model = settings.openai_model; if (settings.use_ai21_tokenizer !== undefined) { oai_settings.use_ai21_tokenizer = !!settings.use_ai21_tokenizer; oai_settings.use_ai21_tokenizer ? ai21_max = 8191 : ai21_max = 9200; } if (settings.exclude_assistant !== undefined) oai_settings.exclude_assistant = !!settings.exclude_assistant; + if (settings.use_alt_scale !== undefined) { oai_settings.use_alt_scale = !!settings.use_alt_scale; updateScaleForm(); } $('#stream_toggle').prop('checked', oai_settings.stream_openai); $('#api_url_scale').val(oai_settings.api_url_scale); $('#openai_proxy_password').val(oai_settings.proxy_password); @@ -2001,6 +2049,7 @@ function loadOpenAISettings(data, settings) { $('#openai_external_category').toggle(oai_settings.show_external_models); $('#use_ai21_tokenizer').prop('checked', oai_settings.use_ai21_tokenizer); $('#exclude_assistant').prop('checked', oai_settings.exclude_assistant); + $('#scale-alt').prop('checked', oai_settings.use_alt_scale); if (settings.impersonation_prompt !== undefined) oai_settings.impersonation_prompt = settings.impersonation_prompt; $('#impersonation_prompt_textarea').val(oai_settings.impersonation_prompt); @@ -2199,6 +2248,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) { assistant_prefill: settings.assistant_prefill, use_ai21_tokenizer: settings.use_ai21_tokenizer, exclude_assistant: settings.exclude_assistant, + use_alt_scale: settings.use_alt_scale, }; const savePresetSettings = await fetch(`/savepreset_openai?name=${name}`, { @@ -2536,7 +2586,8 @@ function onSettingsPresetChange() { assistant_prefill: ['#claude_assistant_prefill', 'assistant_prefill', false], use_ai21_tokenizer: ['#use_ai21_tokenizer', 'use_ai21_tokenizer', false], exclude_assistant: ['#exclude_assistant', 'exclude_assistant', false], - }; + use_alt_scale: ['#use_alt_scale', 'use_alt_scale', false], + }; const presetName = $('#settings_perset_openai').find(":selected").text(); oai_settings.preset_settings_openai = presetName; @@ -2831,20 +2882,31 @@ async function onConnectButtonClick(e) { if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) { const api_key_scale = $('#api_key_scale').val().trim(); + const scale_cookie = $('#scale_cookie').val().trim(); if (api_key_scale.length) { await writeSecret(SECRET_KEYS.SCALE, api_key_scale); } - if (!oai_settings.api_url_scale) { + if (scale_cookie.length) { + await writeSecret(SECRET_KEYS.SCALE_COOKIE, scale_cookie); + } + + if (!oai_settings.api_url_scale && !oai_settings.use_alt_scale) { console.log('No API URL saved for Scale'); return; } - if (!secret_state[SECRET_KEYS.SCALE]) { + if (!secret_state[SECRET_KEYS.SCALE] && !oai_settings.use_alt_scale) { console.log('No secret key saved for Scale'); return; } + + if (!secret_state[SECRET_KEYS.SCALE_COOKIE] && oai_settings.use_alt_scale) { + console.log("No cookie set for Scale"); + return; + } + } if (oai_settings.chat_completion_source == chat_completion_sources.CLAUDE) { @@ -2958,11 +3020,27 @@ function onProxyPasswordShowClick() { $(this).toggleClass('fa-eye-slash fa-eye'); } +function updateScaleForm() { + if (oai_settings.use_alt_scale) { + $('#normal_scale_form').css('display', 'none'); + $('#alt_scale_form').css('display', ''); + } else { + $('#normal_scale_form').css('display', ''); + $('#alt_scale_form').css('display', 'none'); + } +} + $(document).ready(async function () { await loadTokenCache(); $('#test_api_button').on('click', testApiConnection); + $('#scale-alt').on('change', function () { + oai_settings.use_alt_scale = !!$('#scale-alt').prop('checked'); + saveSettingsDebounced(); + updateScaleForm(); + }); + $(document).on('input', '#temp_openai', function () { oai_settings.temp_openai = Number($(this).val()); $('#temp_counter_openai').text(Number($(this).val()).toFixed(2)); diff --git a/public/scripts/secrets.js b/public/scripts/secrets.js index 202870c50..c695ca88c 100644 --- a/public/scripts/secrets.js +++ b/public/scripts/secrets.js @@ -9,6 +9,7 @@ export const SECRET_KEYS = { OPENROUTER: 'api_key_openrouter', SCALE: 'api_key_scale', AI21: 'api_key_ai21', + SCALE_COOKIE: 'scale_cookie', } const INPUT_MAP = { @@ -20,6 +21,7 @@ const INPUT_MAP = { [SECRET_KEYS.OPENROUTER]: '#api_key_openrouter', [SECRET_KEYS.SCALE]: '#api_key_scale', [SECRET_KEYS.AI21]: '#api_key_ai21', + [SECRET_KEYS.SCALE_COOKIE]: '#scale_cookie', } async function clearSecret() { diff --git a/server.js b/server.js index 7b74cf7aa..f990e049a 100644 --- a/server.js +++ b/server.js @@ -3300,6 +3300,72 @@ async function sendScaleRequest(request, response) { } } +app.post("/generate_altscale", jsonParser, function (request, response_generate_scale) { + if(!request.body) return response_generate_scale.sendStatus(400); + + fetch('https://dashboard.scale.com/spellbook/api/trpc/v2.variant.run', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'cookie': `_jwt=${readSecret(SECRET_KEYS.SCALE_COOKIE)}`, + }, + body: JSON.stringify({ + json: { + variant: { + name: 'New Variant', + appId: '', + taxonomy: null + }, + prompt: { + id: '', + template: '{{input}}\n', + exampleVariables: {}, + variablesSourceDataId: null, + systemMessage: request.body.sysprompt + }, + modelParameters: { + id: '', + modelId: 'GPT4', + modelType: 'OpenAi', + maxTokens: request.body.max_tokens, + temperature: request.body.temp, + stop: "user:", + suffix: null, + topP: request.body.top_p, + logprobs: null, + logitBias: request.body.logit_bias + }, + inputs: [ + { + index: '-1', + valueByName: { + input: request.body.prompt + } + } + ] + }, + meta: { + values: { + 'variant.taxonomy': ['undefined'], + 'prompt.variablesSourceDataId': ['undefined'], + 'modelParameters.suffix': ['undefined'], + 'modelParameters.logprobs': ['undefined'], + } + } + }) + }) + .then(response => response.json()) + .then(data => { + console.log(data.result.data.json.outputs[0]) + return response_generate_scale.send({output: data.result.data.json.outputs[0]}); + }) + .catch((error) => { + console.error('Error:', error) + return response_generate_scale.send({error: true}) + }); + +}); + async function sendClaudeRequest(request, response) { const fetch = require('node-fetch').default; @@ -3988,7 +4054,8 @@ const SECRET_KEYS = { DEEPL: 'deepl', OPENROUTER: 'api_key_openrouter', SCALE: 'api_key_scale', - AI21: 'api_key_ai21' + AI21: 'api_key_ai21', + SCALE_COOKIE: 'scale_cookie', } function migrateSecrets() {