mirror of
https://github.com/SillyTavern/SillyTavern.git
synced 2025-06-05 21:59:27 +02:00
Add Groq as chat completion source
This commit is contained in:
Cohee
48
public/img/groq.svg
Normal file
48
public/img/groq.svg
Normal file
@ -0,0 +1,48 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<svg
|
||||
width="107.644"
|
||||
height="156.436"
|
||||
viewBox="0 0 107.644 156.436"
|
||||
fill="none"
|
||||
version="1.1"
|
||||
id="svg9"
|
||||
sodipodi:docname="groqcloud_dark_v2.svg"
|
||||
inkscape:version="1.3 (0e150ed, 2023-07-21)"
|
||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
xmlns:svg="http://www.w3.org/2000/svg">
|
||||
<sodipodi:namedview
|
||||
id="namedview9"
|
||||
pagecolor="#ffffff"
|
||||
bordercolor="#000000"
|
||||
borderopacity="0.25"
|
||||
inkscape:showpageshadow="2"
|
||||
inkscape:pageopacity="0.0"
|
||||
inkscape:pagecheckerboard="0"
|
||||
inkscape:deskcolor="#d1d1d1"
|
||||
inkscape:zoom="0.667"
|
||||
inkscape:cx="499.25037"
|
||||
inkscape:cy="56.971514"
|
||||
inkscape:window-width="1312"
|
||||
inkscape:window-height="449"
|
||||
inkscape:window-x="0"
|
||||
inkscape:window-y="38"
|
||||
inkscape:window-maximized="0"
|
||||
inkscape:current-layer="svg9" />
|
||||
<defs
|
||||
id="defs9">
|
||||
<clipPath
|
||||
id="clip0_872_2594">
|
||||
<rect
|
||||
width="1000"
|
||||
height="200.345"
|
||||
id="rect9"
|
||||
x="0"
|
||||
y="0" />
|
||||
</clipPath>
|
||||
</defs>
|
||||
<path
|
||||
d="M 54.0487,0.00281139 C 24.4736,-0.29748861 0.303066,23.497811 0.00281057,53.072911 -0.297445,82.648011 23.4978,106.89401 53.0729,107.11901 c 0.3003,0 0.6756,0 0.9758,0 H 71.6888 V 87.077011 H 54.0487 c -18.4656,0.225 -33.6285,-14.563 -33.8537,-33.1033 -0.2252,-18.4657 14.5624,-33.6286 33.1031,-33.8538 0.2252,0 0.5255,0 0.7506,0 18.4657,0 33.5536,15.0128 33.5536,33.4784 v 49.316699 c 0,18.316 -14.9377,33.254 -33.2533,33.479 -8.7825,0 -17.1145,-3.603 -23.2698,-9.834 l -14.187,14.187 c 9.8333,9.909 23.1947,15.539 37.1565,15.689 h 0.7507 c 29.1998,-0.451 52.6946,-24.096 52.8446,-53.296 V 52.247211 C 106.894,23.197511 83.1735,0.00281139 54.1238,0.00281139 Z"
|
||||
id="path7" />
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.9 KiB |
@ -423,7 +423,7 @@
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="range-block" data-source="openai,claude,windowai,openrouter,ai21,scale,makersuite,mistralai,custom,cohere,perplexity">
|
||||
<div class="range-block" data-source="openai,claude,windowai,openrouter,ai21,scale,makersuite,mistralai,custom,cohere,perplexity,groq">
|
||||
<div class="range-block-title" data-i18n="Temperature">
|
||||
Temperature
|
||||
</div>
|
||||
@ -436,7 +436,7 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div data-newbie-hidden class="range-block" data-source="openai,openrouter,ai21,custom,cohere,perplexity">
|
||||
<div data-newbie-hidden class="range-block" data-source="openai,openrouter,ai21,custom,cohere,perplexity,groq">
|
||||
<div class="range-block-title" data-i18n="Frequency Penalty">
|
||||
Frequency Penalty
|
||||
</div>
|
||||
@ -449,7 +449,7 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div data-newbie-hidden class="range-block" data-source="openai,openrouter,ai21,custom,cohere,perplexity">
|
||||
<div data-newbie-hidden class="range-block" data-source="openai,openrouter,ai21,custom,cohere,perplexity,groq">
|
||||
<div class="range-block-title" data-i18n="Presence Penalty">
|
||||
Presence Penalty
|
||||
</div>
|
||||
@ -488,7 +488,7 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div data-newbie-hidden class="range-block" data-source="openai,claude,openrouter,ai21,scale,makersuite,mistralai,custom,cohere,perplexity">
|
||||
<div data-newbie-hidden class="range-block" data-source="openai,claude,openrouter,ai21,scale,makersuite,mistralai,custom,cohere,perplexity,groq">
|
||||
<div class="range-block-title" data-i18n="Top-p">
|
||||
Top P
|
||||
</div>
|
||||
@ -724,7 +724,7 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div data-newbie-hidden class="range-block" data-source="openai,openrouter,mistralai,custom,cohere">
|
||||
<div data-newbie-hidden class="range-block" data-source="openai,openrouter,mistralai,custom,cohere,groq">
|
||||
<div class="range-block-title justifyLeft" data-i18n="Seed">
|
||||
Seed
|
||||
</div>
|
||||
@ -2266,6 +2266,7 @@
|
||||
<option value="ai21">AI21</option>
|
||||
<option value="claude">Claude</option>
|
||||
<option value="cohere">Cohere</option>
|
||||
<option value="groq">Groq</option>
|
||||
<option value="makersuite">Google MakerSuite</option>
|
||||
<option value="mistralai">MistralAI</option>
|
||||
<option value="openrouter">OpenRouter</option>
|
||||
@ -2670,6 +2671,23 @@
|
||||
</select>
|
||||
</div>
|
||||
</form>
|
||||
<div id="groq_form" data-source="groq">
|
||||
<h4 data-i18n="Groq API Key">Groq API Key</h4>
|
||||
<div class="flex-container">
|
||||
<input id="api_key_groq" name="api_key_groq" class="text_pole flex1" maxlength="500" value="" type="text" autocomplete="off">
|
||||
<div title="Clear your API key" data-i18n="[title]Clear your API key" class="menu_button fa-solid fa-circle-xmark clear-api-key" data-key="api_key_groq"></div>
|
||||
</div>
|
||||
<div data-for="api_key_groq" class="neutral_warning">
|
||||
For privacy reasons, your API key will be hidden after you reload the page.
|
||||
</div>
|
||||
<h4 data-i18n="Groq Model">Groq Model</h4>
|
||||
<select id="model_groq_select">
|
||||
<option value="llama3-8b-8192">llama3-8b-8192</option>
|
||||
<option value="llama3-70b-8192">llama3-70b-8192</option>
|
||||
<option value="mixtral-8x7b-32768">mixtral-8x7b-32768</option>
|
||||
<option value="gemma-7b-it">gemma-7b-it</option>
|
||||
</select>
|
||||
</div>
|
||||
<div id="perplexity_form" data-source="perplexity">
|
||||
<h4 data-i18n="Perplexity API Key">Perplexity API Key</h4>
|
||||
<div class="flex-container">
|
||||
|
||||
@ -8002,6 +8002,11 @@ const CONNECT_API_MAP = {
|
||||
button: '#api_button_openai',
|
||||
source: chat_completion_sources.PERPLEXITY,
|
||||
},
|
||||
'groq': {
|
||||
selected: 'openai',
|
||||
button: '#api_button_openai',
|
||||
source: chat_completion_sources.GROQ,
|
||||
},
|
||||
'infermaticai': {
|
||||
selected: 'textgenerationwebui',
|
||||
button: '#api_button_textgenerationwebui',
|
||||
|
||||
@ -378,6 +378,7 @@ function RA_autoconnect(PrevApi) {
|
||||
|| (secret_state[SECRET_KEYS.MISTRALAI] && oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI)
|
||||
|| (secret_state[SECRET_KEYS.COHERE] && oai_settings.chat_completion_source == chat_completion_sources.COHERE)
|
||||
|| (secret_state[SECRET_KEYS.PERPLEXITY] && oai_settings.chat_completion_source == chat_completion_sources.PERPLEXITY)
|
||||
|| (secret_state[SECRET_KEYS.GROQ] && oai_settings.chat_completion_source == chat_completion_sources.GROQ)
|
||||
|| (isValidUrl(oai_settings.custom_url) && oai_settings.chat_completion_source == chat_completion_sources.CUSTOM)
|
||||
) {
|
||||
$('#api_button_openai').trigger('click');
|
||||
|
||||
@ -174,6 +174,7 @@ export const chat_completion_sources = {
|
||||
CUSTOM: 'custom',
|
||||
COHERE: 'cohere',
|
||||
PERPLEXITY: 'perplexity',
|
||||
GROQ: 'groq',
|
||||
};
|
||||
|
||||
const character_names_behavior = {
|
||||
@ -241,6 +242,7 @@ const default_settings = {
|
||||
mistralai_model: 'mistral-medium-latest',
|
||||
cohere_model: 'command-r',
|
||||
perplexity_model: 'llama-3-70b-instruct',
|
||||
groq_model: 'llama3-70b-8192',
|
||||
custom_model: '',
|
||||
custom_url: '',
|
||||
custom_include_body: '',
|
||||
@ -314,6 +316,7 @@ const oai_settings = {
|
||||
mistralai_model: 'mistral-medium-latest',
|
||||
cohere_model: 'command-r',
|
||||
perplexity_model: 'llama-3-70b-instruct',
|
||||
groq_model: 'llama3-70b-8192',
|
||||
custom_model: '',
|
||||
custom_url: '',
|
||||
custom_include_body: '',
|
||||
@ -1438,6 +1441,8 @@ function getChatCompletionModel() {
|
||||
return oai_settings.cohere_model;
|
||||
case chat_completion_sources.PERPLEXITY:
|
||||
return oai_settings.perplexity_model;
|
||||
case chat_completion_sources.GROQ:
|
||||
return oai_settings.groq_model;
|
||||
default:
|
||||
throw new Error(`Unknown chat completion source: ${oai_settings.chat_completion_source}`);
|
||||
}
|
||||
@ -1664,6 +1669,7 @@ async function sendOpenAIRequest(type, messages, signal) {
|
||||
const isCustom = oai_settings.chat_completion_source == chat_completion_sources.CUSTOM;
|
||||
const isCohere = oai_settings.chat_completion_source == chat_completion_sources.COHERE;
|
||||
const isPerplexity = oai_settings.chat_completion_source == chat_completion_sources.PERPLEXITY;
|
||||
const isGroq = oai_settings.chat_completion_source == chat_completion_sources.GROQ;
|
||||
const isTextCompletion = (isOAI && textCompletionModels.includes(oai_settings.openai_model)) || (isOpenRouter && oai_settings.openrouter_force_instruct && power_user.instruct.enabled);
|
||||
const isQuiet = type === 'quiet';
|
||||
const isImpersonate = type === 'impersonate';
|
||||
@ -1821,6 +1827,14 @@ async function sendOpenAIRequest(type, messages, signal) {
|
||||
delete generate_data['stop'];
|
||||
}
|
||||
|
||||
// https://console.groq.com/docs/openai
|
||||
if (isGroq) {
|
||||
delete generate_data.logprobs;
|
||||
delete generate_data.logit_bias;
|
||||
delete generate_data.top_logprobs;
|
||||
delete generate_data.n;
|
||||
}
|
||||
|
||||
if ((isOAI || isOpenRouter || isMistral || isCustom || isCohere) && oai_settings.seed >= 0) {
|
||||
generate_data['seed'] = oai_settings.seed;
|
||||
}
|
||||
@ -2688,6 +2702,7 @@ function loadOpenAISettings(data, settings) {
|
||||
oai_settings.mistralai_model = settings.mistralai_model ?? default_settings.mistralai_model;
|
||||
oai_settings.cohere_model = settings.cohere_model ?? default_settings.cohere_model;
|
||||
oai_settings.perplexity_model = settings.perplexity_model ?? default_settings.perplexity_model;
|
||||
oai_settings.groq_model = settings.groq_model ?? default_settings.groq_model;
|
||||
oai_settings.custom_model = settings.custom_model ?? default_settings.custom_model;
|
||||
oai_settings.custom_url = settings.custom_url ?? default_settings.custom_url;
|
||||
oai_settings.custom_include_body = settings.custom_include_body ?? default_settings.custom_include_body;
|
||||
@ -2755,6 +2770,8 @@ function loadOpenAISettings(data, settings) {
|
||||
$(`#model_cohere_select option[value="${oai_settings.cohere_model}"`).attr('selected', true);
|
||||
$('#model_perplexity_select').val(oai_settings.perplexity_model);
|
||||
$(`#model_perplexity_select option[value="${oai_settings.perplexity_model}"`).attr('selected', true);
|
||||
$('#model_groq_select').val(oai_settings.groq_model);
|
||||
$(`#model_groq_select option[value="${oai_settings.groq_model}"`).attr('selected', true);
|
||||
$('#custom_model_id').val(oai_settings.custom_model);
|
||||
$('#custom_api_url_text').val(oai_settings.custom_url);
|
||||
$('#openai_max_context').val(oai_settings.openai_max_context);
|
||||
@ -2904,7 +2921,14 @@ async function getStatusOpen() {
|
||||
return resultCheckStatus();
|
||||
}
|
||||
|
||||
const noValidateSources = [chat_completion_sources.SCALE, chat_completion_sources.CLAUDE, chat_completion_sources.AI21, chat_completion_sources.MAKERSUITE, chat_completion_sources.PERPLEXITY];
|
||||
const noValidateSources = [
|
||||
chat_completion_sources.SCALE,
|
||||
chat_completion_sources.CLAUDE,
|
||||
chat_completion_sources.AI21,
|
||||
chat_completion_sources.MAKERSUITE,
|
||||
chat_completion_sources.PERPLEXITY,
|
||||
chat_completion_sources.GROQ,
|
||||
];
|
||||
if (noValidateSources.includes(oai_settings.chat_completion_source)) {
|
||||
let status = 'Unable to verify key; press "Test Message" to validate.';
|
||||
setOnlineStatus(status);
|
||||
@ -2996,6 +3020,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
|
||||
mistralai_model: settings.mistralai_model,
|
||||
cohere_model: settings.cohere_model,
|
||||
perplexity_model: settings.perplexity_model,
|
||||
groq_model: settings.groq_model,
|
||||
custom_model: settings.custom_model,
|
||||
custom_url: settings.custom_url,
|
||||
custom_include_body: settings.custom_include_body,
|
||||
@ -3391,6 +3416,7 @@ function onSettingsPresetChange() {
|
||||
mistralai_model: ['#model_mistralai_select', 'mistralai_model', false],
|
||||
cohere_model: ['#model_cohere_select', 'cohere_model', false],
|
||||
perplexity_model: ['#model_perplexity_select', 'perplexity_model', false],
|
||||
groq_model: ['#model_groq_select', 'groq_model', false],
|
||||
custom_model: ['#custom_model_id', 'custom_model', false],
|
||||
custom_url: ['#custom_api_url_text', 'custom_url', false],
|
||||
custom_include_body: ['#custom_include_body', 'custom_include_body', false],
|
||||
@ -3619,6 +3645,11 @@ async function onModelChange() {
|
||||
oai_settings.perplexity_model = value;
|
||||
}
|
||||
|
||||
if ($(this).is('#model_groq_select')) {
|
||||
console.log('Groq model changed to', value);
|
||||
oai_settings.groq_model = value;
|
||||
}
|
||||
|
||||
if (value && $(this).is('#model_custom_select')) {
|
||||
console.log('Custom model changed to', value);
|
||||
oai_settings.custom_model = value;
|
||||
@ -3796,6 +3827,25 @@ async function onModelChange() {
|
||||
$('#temp_openai').attr('max', oai_max_temp).val(oai_settings.temp_openai).trigger('input');
|
||||
}
|
||||
|
||||
if (oai_settings.chat_completion_source == chat_completion_sources.GROQ) {
|
||||
if (oai_settings.max_context_unlocked) {
|
||||
$('#openai_max_context').attr('max', unlocked_max);
|
||||
}
|
||||
else if (['llama3-8b-8192', 'llama3-70b-8192', 'gemma-7b-it'].includes(oai_settings.groq_model)) {
|
||||
$('#openai_max_context').attr('max', max_8k);
|
||||
}
|
||||
else if (['mixtral-8x7b-32768'].includes(oai_settings.groq_model)) {
|
||||
$('#openai_max_context').attr('max', max_32k);
|
||||
}
|
||||
else {
|
||||
$('#openai_max_context').attr('max', max_4k);
|
||||
}
|
||||
oai_settings.openai_max_context = Math.min(Number($('#openai_max_context').attr('max')), oai_settings.openai_max_context);
|
||||
$('#openai_max_context').val(oai_settings.openai_max_context).trigger('input');
|
||||
oai_settings.temp_openai = Math.min(oai_max_temp, oai_settings.temp_openai);
|
||||
$('#temp_openai').attr('max', oai_max_temp).val(oai_settings.temp_openai).trigger('input');
|
||||
}
|
||||
|
||||
if (oai_settings.chat_completion_source == chat_completion_sources.AI21) {
|
||||
if (oai_settings.max_context_unlocked) {
|
||||
$('#openai_max_context').attr('max', unlocked_max);
|
||||
@ -4016,6 +4066,19 @@ async function onConnectButtonClick(e) {
|
||||
}
|
||||
}
|
||||
|
||||
if (oai_settings.chat_completion_source == chat_completion_sources.GROQ) {
|
||||
const api_key_groq = String($('#api_key_groq').val()).trim();
|
||||
|
||||
if (api_key_groq.length) {
|
||||
await writeSecret(SECRET_KEYS.GROQ, api_key_groq);
|
||||
}
|
||||
|
||||
if (!secret_state[SECRET_KEYS.GROQ]) {
|
||||
console.log('No secret key saved for Groq');
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
startStatusLoading();
|
||||
saveSettingsDebounced();
|
||||
await getStatusOpen();
|
||||
@ -4057,6 +4120,9 @@ function toggleChatCompletionForms() {
|
||||
else if (oai_settings.chat_completion_source == chat_completion_sources.PERPLEXITY) {
|
||||
$('#model_perplexity_select').trigger('change');
|
||||
}
|
||||
else if (oai_settings.chat_completion_source == chat_completion_sources.GROQ) {
|
||||
$('#model_groq_select').trigger('change');
|
||||
}
|
||||
else if (oai_settings.chat_completion_source == chat_completion_sources.CUSTOM) {
|
||||
$('#model_custom_select').trigger('change');
|
||||
}
|
||||
@ -4727,6 +4793,7 @@ $(document).ready(async function () {
|
||||
$('#model_mistralai_select').on('change', onModelChange);
|
||||
$('#model_cohere_select').on('change', onModelChange);
|
||||
$('#model_perplexity_select').on('change', onModelChange);
|
||||
$('#model_groq_select').on('change', onModelChange);
|
||||
$('#model_custom_select').on('change', onModelChange);
|
||||
$('#settings_preset_openai').on('change', onSettingsPresetChange);
|
||||
$('#new_oai_preset').on('click', onNewPresetClick);
|
||||
|
||||
@ -26,6 +26,7 @@ export const SECRET_KEYS = {
|
||||
LLAMACPP: 'api_key_llamacpp',
|
||||
COHERE: 'api_key_cohere',
|
||||
PERPLEXITY: 'api_key_perplexity',
|
||||
GROQ: 'api_key_groq',
|
||||
};
|
||||
|
||||
const INPUT_MAP = {
|
||||
@ -53,6 +54,7 @@ const INPUT_MAP = {
|
||||
[SECRET_KEYS.LLAMACPP]: '#api_key_llamacpp',
|
||||
[SECRET_KEYS.COHERE]: '#api_key_cohere',
|
||||
[SECRET_KEYS.PERPLEXITY]: '#api_key_perplexity',
|
||||
[SECRET_KEYS.GROQ]: '#api_key_groq',
|
||||
};
|
||||
|
||||
async function clearSecret() {
|
||||
|
||||
@ -1678,6 +1678,7 @@ function modelCallback(_, model) {
|
||||
{ id: 'model_custom_select', api: 'openai', type: chat_completion_sources.CUSTOM },
|
||||
{ id: 'model_cohere_select', api: 'openai', type: chat_completion_sources.COHERE },
|
||||
{ id: 'model_perplexity_select', api: 'openai', type: chat_completion_sources.PERPLEXITY },
|
||||
{ id: 'model_groq_select', api: 'openai', type: chat_completion_sources.GROQ },
|
||||
{ id: 'model_novel_select', api: 'novel', type: null },
|
||||
{ id: 'horde_model', api: 'koboldhorde', type: null },
|
||||
];
|
||||
|
||||
@ -536,6 +536,15 @@ export function getTokenizerModel() {
|
||||
}
|
||||
}
|
||||
|
||||
if (oai_settings.chat_completion_source === chat_completion_sources.GROQ) {
|
||||
if (oai_settings.groq_model.includes('llama-3') || oai_settings.groq_model.includes('llama3')) {
|
||||
return llama3Tokenizer;
|
||||
}
|
||||
if (oai_settings.groq_model.includes('mistral') || oai_settings.groq_model.includes('mixtral')) {
|
||||
return mistralTokenizer;
|
||||
}
|
||||
}
|
||||
|
||||
// Default to Turbo 3.5
|
||||
return turboTokenizer;
|
||||
}
|
||||
|
||||
@ -192,6 +192,7 @@ const CHAT_COMPLETION_SOURCES = {
|
||||
CUSTOM: 'custom',
|
||||
COHERE: 'cohere',
|
||||
PERPLEXITY: 'perplexity',
|
||||
GROQ: 'groq',
|
||||
};
|
||||
|
||||
const UPLOADS_PATH = './uploads';
|
||||
|
||||
@ -15,6 +15,7 @@ const API_CLAUDE = 'https://api.anthropic.com/v1';
|
||||
const API_MISTRAL = 'https://api.mistral.ai/v1';
|
||||
const API_COHERE = 'https://api.cohere.ai/v1';
|
||||
const API_PERPLEXITY = 'https://api.perplexity.ai';
|
||||
const API_GROQ = 'https://api.groq.com/openai/v1';
|
||||
|
||||
/**
|
||||
* Applies a post-processing step to the generated messages.
|
||||
@ -879,6 +880,11 @@ router.post('/generate', jsonParser, function (request, response) {
|
||||
headers = {};
|
||||
bodyParams = {};
|
||||
request.body.messages = postProcessPrompt(request.body.messages, 'claude', request.body.char_name, request.body.user_name);
|
||||
} else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.GROQ) {
|
||||
apiUrl = API_GROQ;
|
||||
apiKey = readSecret(request.user.directories, SECRET_KEYS.GROQ);
|
||||
headers = {};
|
||||
bodyParams = {};
|
||||
} else {
|
||||
console.log('This chat completion source is not supported yet.');
|
||||
return response.status(400).send({ error: true });
|
||||
|
||||
@ -38,6 +38,7 @@ const SECRET_KEYS = {
|
||||
LLAMACPP: 'api_key_llamacpp',
|
||||
COHERE: 'api_key_cohere',
|
||||
PERPLEXITY: 'api_key_perplexity',
|
||||
GROQ: 'api_key_groq',
|
||||
};
|
||||
|
||||
// These are the keys that are safe to expose, even if allowKeysExposure is false
|
||||
|
||||
Reference in New Issue
Block a user