Merge pull request #1565 from SillyTavern/togetherai

Add TogetherAI as a text completion source

default/settings.json

@@ -3,7 +3,6 @@
     "username": "User",
     "api_server": "http://127.0.0.1:5000/api",
     "api_server_textgenerationwebui": "http://127.0.0.1:5000/api",
-    "api_use_mancer_webui": false,
     "preset_settings": "RecoveredRuins",
     "user_avatar": "user-default.png",
     "amount_gen": 250,

public/img/togetherai.svg

@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   width="32"
+   height="32"
+   viewBox="0 0 32 32"
+   version="1.1"
+   id="svg4"
+   sodipodi:docname="togetherai.svg"
+   inkscape:version="1.3 (0e150ed, 2023-07-21)"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <sodipodi:namedview
+     id="namedview4"
+     pagecolor="#ffffff"
+     bordercolor="#000000"
+     borderopacity="0.25"
+     inkscape:showpageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1"
+     inkscape:zoom="7.375"
+     inkscape:cx="15.932203"
+     inkscape:cy="15.932203"
+     inkscape:window-width="1280"
+     inkscape:window-height="688"
+     inkscape:window-x="0"
+     inkscape:window-y="25"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="g4" />
+  <g
+     clip-path="url(#clip0_542_18748)"
+     id="g4">
+    <path
+       id="rect1"
+       d="M 5.6464844 0 C 2.5180256 -1.1842331e-15 3.5527101e-15 2.5180256 0 5.6464844 L 0 26.353516 C -1.1842331e-15 29.481971 2.5180256 32 5.6464844 32 L 26.353516 32 C 29.481971 32 32 29.481971 32 26.353516 L 32 5.6464844 C 32 2.5180256 29.481971 3.5527101e-15 26.353516 0 L 5.6464844 0 z M 9.6464844 4 A 5.6470599 5.6470599 0 0 1 15.294922 9.6464844 A 5.6470599 5.6470599 0 0 1 9.6464844 15.294922 A 5.6470599 5.6470599 0 0 1 4 9.6464844 A 5.6470599 5.6470599 0 0 1 9.6464844 4 z M 22.824219 4 A 5.6470599 5.6470599 0 0 1 28.470703 9.6464844 A 5.6470599 5.6470599 0 0 1 22.824219 15.294922 A 5.6470599 5.6470599 0 0 1 17.175781 9.6464844 A 5.6470599 5.6470599 0 0 1 22.824219 4 z M 9.6464844 17.175781 A 5.6470599 5.6470599 0 0 1 15.294922 22.824219 A 5.6470599 5.6470599 0 0 1 9.6464844 28.470703 A 5.6470599 5.6470599 0 0 1 4 22.824219 A 5.6470599 5.6470599 0 0 1 9.6464844 17.175781 z M 22.824219 17.175781 A 5.6470599 5.6470599 0 0 1 28.470703 22.824219 A 5.6470599 5.6470599 0 0 1 22.824219 28.470703 A 5.6470599 5.6470599 0 0 1 17.175781 22.824219 A 5.6470599 5.6470599 0 0 1 22.824219 17.175781 z " />
+    <circle
+       cx="9.64706"
+       cy="9.64706"
+       r="5.64706"
+       opacity="0.45"
+       id="circle9" />
+  </g>
+  <defs
+     id="defs4">
+    <clipPath
+       id="clip0_542_18748">
+      <rect
+         width="32"
+         height="32"
+         id="rect4" />
+    </clipPath>
+  </defs>
+</svg>

public/index.html

@@ -69,7 +69,7 @@
     <script type="module" src="scripts/group-chats.js"></script>
     <script type="module" src="scripts/kai-settings.js"></script>
     <script type="module" src="scripts/textgen-settings.js"></script>
-    <script type="module" src="scripts/mancer-settings.js"></script>
+    <script type="module" src="scripts/textgen-models.js"></script>
     <script type="module" src="scripts/bookmarks.js"></script>
     <script type="module" src="scripts/horde.js"></script>
     <script type="module" src="scripts/RossAscends-mods.js"></script>
@@ -1592,8 +1592,8 @@
                             <option value="kobold"><span data-i18n="KoboldAI">KoboldAI Classic</span></option>
                             <option value="koboldhorde"><span data-i18n="KoboldAI Horde">KoboldAI Horde</span></option>
                             <option value="novel"><span data-i18n="NovelAI">NovelAI</span></option>
-                            <option value="textgenerationwebui"><span data-i18n="Text Completion">Text Completion (ooba, Mancer, Aphrodite, TabbyAPI, KoboldCpp)</span></option>
+                            <option value="textgenerationwebui"><span data-i18n="Text Completion">Text Completion</span></option>
-                            <option value="openai"><span data-i18n="Chat Completion (OpenAI, Claude, Window/OpenRouter, Scale, AI21, Google MakerSuite, MistralAI)">Chat Completion (OpenAI, Claude, Window, OpenRouter, Scale, AI21, Google MakerSuite, MistralAI)</span></option>
+                            <option value="openai"><span data-i18n="Chat Completion">Chat Completion</span></option>
                         </select>
                     </div>
                     <div id="kobold_horde" style="position: relative;"> <!-- shows the kobold settings -->
@@ -1723,8 +1723,27 @@
                                     <option value="aphrodite">Aphrodite</option>
                                     <option value="tabby">TabbyAPI</option>
                                     <option value="koboldcpp">KoboldCpp</option>
+                                    <option value="togetherai">TogetherAI</option>
                                 </select>
                             </div>
+                            <div data-tg-type="togetherai" class="flex-container flexFlowColumn">
+                                <h4 data-i18n="TogetherAI API Key">TogetherAI API Key</h4>
+                                <div class="flex-container">
+                                    <input id="api_key_togetherai" name="api_key_togetherai" class="text_pole flex1" maxlength="500" value="" type="text" autocomplete="off">
+                                    <div title="Clear your API key" data-i18n="[title]Clear your API key" class="menu_button fa-solid fa-circle-xmark clear-api-key" data-key="api_key_togetherai"></div>
+                                </div>
+                                <div data-for="api_key_togetherai" class="neutral_warning">
+                                    For privacy reasons, your API key will be hidden after you reload the page.
+                                </div>
+                                <div>
+                                    <h4 data-i18n="TogetherAI Model">TogetherAI Model</h4>
+                                    <select id="model_togetherai_select">
+                                        <option>
+                                            -- Connect to the API --
+                                        </option>
+                                    </select>
+                                </div>
+                            </div>
                             <div data-tg-type="mancer" class="flex-container flexFlowColumn">
                                 <div class="flex-container flexFlowColumn">
                                 </div>

public/script.js

@@ -19,10 +19,10 @@ import {
     getTextGenUrlSourceId,
     textgen_types,
     textgenerationwebui_banned_in_macros,
-    MANCER_SERVER,
+    getTextGenServer,
 } from './scripts/textgen-settings.js';
 
-const { MANCER } = textgen_types;
+const { MANCER, TOGETHERAI } = textgen_types;
 
 import {
     world_info,
@@ -189,7 +189,7 @@ import { createPersona, initPersonas, selectCurrentPersona, setPersonaDescriptio
 import { getBackgrounds, initBackgrounds, loadBackgroundSettings, background_settings } from './scripts/backgrounds.js';
 import { hideLoader, showLoader } from './scripts/loader.js';
 import { BulkEditOverlay, CharacterContextMenu } from './scripts/BulkEditOverlay.js';
-import { loadMancerModels } from './scripts/mancer-settings.js';
+import { loadMancerModels, loadTogetherAIModels } from './scripts/textgen-models.js';
 import { appendFileContent, hasPendingFileAttachment, populateFileAttachment } from './scripts/chats.js';
 import { replaceVariableMacros } from './scripts/variables.js';
 import { initPresetManager } from './scripts/preset-manager.js';
@@ -931,9 +931,7 @@ async function getStatusKobold() {
 async function getStatusTextgen() {
     const url = '/api/backends/text-completions/status';
 
-    let endpoint = textgen_settings.type === MANCER ?
+    let endpoint = getTextGenServer();
-        MANCER_SERVER :
-        api_server_textgenerationwebui;
 
     if (!endpoint) {
         console.warn('No endpoint for status check');
@@ -949,7 +947,8 @@ async function getStatusTextgen() {
                 api_type: textgen_settings.type,
                 legacy_api:
                     textgen_settings.legacy_api &&
-                    textgen_settings.type !== MANCER,
+                    textgen_settings.type !== MANCER &&
+                    textgen_settings.type !== TOGETHERAI,
             }),
             signal: abortStatusCheck.signal,
         });
@@ -959,6 +958,9 @@ async function getStatusTextgen() {
         if (textgen_settings.type === MANCER) {
             online_status = textgen_settings.mancer_model;
             loadMancerModels(data?.data);
+        } else if (textgen_settings.type === TOGETHERAI) {
+            online_status = textgen_settings.togetherai_model;
+            loadTogetherAIModels(data?.data);
         } else {
             online_status = data?.result;
         }
@@ -2958,7 +2960,8 @@ async function Generate(type, { automatic_trigger, force_name2, quiet_prompt, qu
     if (main_api === 'textgenerationwebui' &&
         textgen_settings.streaming &&
         textgen_settings.legacy_api &&
-        textgen_settings.type !== MANCER) {
+        textgen_settings.type !== MANCER &&
+        textgen_settings.type !== TOGETHERAI) {
         toastr.error('Streaming is not supported for the Legacy API. Update Ooba and use --extensions openai to enable streaming.', undefined, { timeOut: 10000, preventDuplicates: true });
         unblockGeneration();
         return Promise.resolve();
@@ -7514,6 +7517,11 @@ async function connectAPISlash(_, text) {
             button: '#api_button_textgenerationwebui',
             type: textgen_types.KOBOLDCPP,
         },
+        'togetherai': {
+            selected: 'textgenerationwebui',
+            button: '#api_button_textgenerationwebui',
+            type: textgen_types.TOGETHERAI,
+        },
         'oai': {
             selected: 'openai',
             source: 'openai',
@@ -7840,7 +7848,7 @@ jQuery(async function () {
     }
 
     registerSlashCommand('dupe', DupeChar, [], '– duplicates the currently selected character', true, true);
-    registerSlashCommand('api', connectAPISlash, [], '<span class="monospace">(kobold, horde, novel, ooba, tabby, mancer, aphrodite, kcpp, oai, claude, windowai, openrouter, scale, ai21, makersuite, mistralai)</span> – connect to an API', true, true);
+    registerSlashCommand('api', connectAPISlash, [], '<span class="monospace">(kobold, horde, novel, ooba, tabby, mancer, aphrodite, kcpp, oai, claude, windowai, openrouter, scale, ai21, makersuite, mistralai, togetherai)</span> – connect to an API', true, true);
     registerSlashCommand('impersonate', doImpersonate, ['imp'], '– calls an impersonation response', true, true);
     registerSlashCommand('delchat', doDeleteChat, [], '– deletes the current chat', true, true);
     registerSlashCommand('closechat', doCloseChat, [], '– closes the current chat', true, true);
@@ -8394,6 +8402,11 @@ jQuery(async function () {
             await writeSecret(SECRET_KEYS.TABBY, tabbyKey);
         }
 
+        const togetherKey = String($('#api_key_togetherai').val()).trim();
+        if (togetherKey.length) {
+            await writeSecret(SECRET_KEYS.TOGETHERAI, togetherKey);
+        }
+
         const urlSourceId = getTextGenUrlSourceId();
 
         if (urlSourceId && $(urlSourceId).val() !== '') {

public/scripts/RossAscends-mods.js

@@ -382,7 +382,9 @@ function RA_autoconnect(PrevApi) {
                 }
                 break;
             case 'textgenerationwebui':
-                if (textgen_settings.type === textgen_types.MANCER && secret_state[SECRET_KEYS.MANCER]) {
+                if ((textgen_settings.type === textgen_types.MANCER && secret_state[SECRET_KEYS.MANCER]) ||
+                    (textgen_settings.type === textgen_types.TOGETHERAI && secret_state[SECRET_KEYS.TOGETHERAI])
+                ) {
                     $('#api_button_textgenerationwebui').trigger('click');
                 }
                 else if (api_server_textgenerationwebui && isValidUrl(api_server_textgenerationwebui)) {

public/scripts/extensions/stable-diffusion/index.js

@@ -46,6 +46,7 @@ const sources = {
     vlad: 'vlad',
     openai: 'openai',
     comfy: 'comfy',
+    togetherai: 'togetherai',
 };
 
 const generationMode = {
@@ -917,7 +918,7 @@ async function onModelChange() {
     extension_settings.sd.model = $('#sd_model').find(':selected').val();
     saveSettingsDebounced();
 
-    const cloudSources = [sources.horde, sources.novel, sources.openai];
+    const cloudSources = [sources.horde, sources.novel, sources.openai, sources.togetherai];
 
     if (cloudSources.includes(extension_settings.sd.source)) {
         return;
@@ -1050,11 +1051,14 @@ async function loadSamplers() {
             samplers = await loadVladSamplers();
             break;
         case sources.openai:
-            samplers = await loadOpenAiSamplers();
+            samplers = ['N/A'];
             break;
         case sources.comfy:
             samplers = await loadComfySamplers();
             break;
+        case sources.togetherai:
+            samplers = ['N/A'];
+            break;
     }
 
     for (const sampler of samplers) {
@@ -1064,6 +1068,11 @@ async function loadSamplers() {
         option.selected = sampler === extension_settings.sd.sampler;
         $('#sd_sampler').append(option);
     }
+
+    if (!extension_settings.sd.sampler && samplers.length > 0) {
+        extension_settings.sd.sampler = samplers[0];
+        $('#sd_sampler').val(extension_settings.sd.sampler).trigger('change');
+    }
 }
 
 async function loadHordeSamplers() {
@@ -1120,10 +1129,6 @@ async function loadAutoSamplers() {
     }
 }
 
-async function loadOpenAiSamplers() {
-    return ['N/A'];
-}
-
 async function loadVladSamplers() {
     if (!extension_settings.sd.vlad_url) {
         return [];
@@ -1212,6 +1217,9 @@ async function loadModels() {
         case sources.comfy:
             models = await loadComfyModels();
             break;
+        case sources.togetherai:
+            models = await loadTogetherAIModels();
+            break;
     }
 
     for (const model of models) {
@@ -1221,6 +1229,30 @@ async function loadModels() {
         option.selected = model.value === extension_settings.sd.model;
         $('#sd_model').append(option);
     }
+
+    if (!extension_settings.sd.model && models.length > 0) {
+        extension_settings.sd.model = models[0].value;
+        $('#sd_model').val(extension_settings.sd.model).trigger('change');
+    }
+}
+
+async function loadTogetherAIModels() {
+    if (!secret_state[SECRET_KEYS.TOGETHERAI]) {
+        console.debug('TogetherAI API key is not set.');
+        return [];
+    }
+
+    const result = await fetch('/api/sd/together/models', {
+        method: 'POST',
+        headers: getRequestHeaders(),
+    });
+
+    if (result.ok) {
+        const data = await result.json();
+        return data;
+    }
+
+    return [];
 }
 
 async function loadHordeModels() {
@@ -1434,6 +1466,9 @@ async function loadSchedulers() {
         case sources.openai:
             schedulers = ['N/A'];
             break;
+        case sources.togetherai:
+            schedulers = ['N/A'];
+            break;
         case sources.comfy:
             schedulers = await loadComfySchedulers();
             break;
@@ -1493,6 +1528,9 @@ async function loadVaes() {
         case sources.openai:
             vaes = ['N/A'];
             break;
+        case sources.togetherai:
+            vaes = ['N/A'];
+            break;
         case sources.comfy:
             vaes = await loadComfyVaes();
             break;
@@ -1873,6 +1911,9 @@ async function sendGenerationRequest(generationType, prompt, characterName = nul
             case sources.comfy:
                 result = await generateComfyImage(prefixedPrompt);
                 break;
+            case sources.togetherai:
+                result = await generateTogetherAIImage(prefixedPrompt);
+                break;
         }
 
         if (!result.data) {
@@ -1895,6 +1936,29 @@ async function sendGenerationRequest(generationType, prompt, characterName = nul
     callback ? callback(prompt, base64Image, generationType) : sendMessage(prompt, base64Image, generationType);
 }
 
+async function generateTogetherAIImage(prompt) {
+    const result = await fetch('/api/sd/together/generate', {
+        method: 'POST',
+        headers: getRequestHeaders(),
+        body: JSON.stringify({
+            prompt: prompt,
+            negative_prompt: extension_settings.sd.negative_prompt,
+            model: extension_settings.sd.model,
+            steps: extension_settings.sd.steps,
+            width: extension_settings.sd.width,
+            height: extension_settings.sd.height,
+        }),
+    });
+
+    if (result.ok) {
+        const data = await result.json();
+        return { format: 'jpg', data: data?.output?.choices?.[0]?.image_base64 };
+    } else {
+        const text = await result.text();
+        throw new Error(text);
+    }
+}
+
 /**
  * Generates an "extras" image using a provided prompt and other settings.
  *
@@ -2435,6 +2499,8 @@ function isValidState() {
             return secret_state[SECRET_KEYS.OPENAI];
         case sources.comfy:
             return true;
+        case sources.togetherai:
+            return secret_state[SECRET_KEYS.TOGETHERAI];
     }
 }
 

public/scripts/extensions/stable-diffusion/settings.html

@@ -35,6 +35,7 @@
                 <option value="novel">NovelAI Diffusion</option>
                 <option value="openai">OpenAI (DALL-E)</option>
                 <option value="comfy">ComfyUI</option>
+                <option value="togetherai">TogetherAI</option>
             </select>
             <div data-sd-source="auto">
                 <label for="sd_auto_url">SD Web UI URL</label>

public/scripts/mancer-settings.js

@@ -1,64 +0,0 @@
-import { setGenerationParamsFromPreset } from '../script.js';
-import { isMobile } from './RossAscends-mods.js';
-import { textgenerationwebui_settings as textgen_settings } from './textgen-settings.js';
-
-let models = [];
-
-export async function loadMancerModels(data) {
-    if (!Array.isArray(data)) {
-        console.error('Invalid Mancer models data', data);
-        return;
-    }
-
-    models = data;
-
-    $('#mancer_model').empty();
-    for (const model of data) {
-        const option = document.createElement('option');
-        option.value = model.id;
-        option.text = model.name;
-        option.selected = model.id === textgen_settings.mancer_model;
-        $('#mancer_model').append(option);
-    }
-}
-
-function onMancerModelSelect() {
-    const modelId = String($('#mancer_model').val());
-    textgen_settings.mancer_model = modelId;
-    $('#api_button_textgenerationwebui').trigger('click');
-
-    const limits = models.find(x => x.id === modelId)?.limits;
-    setGenerationParamsFromPreset({ max_length: limits.context, genamt: limits.completion });
-}
-
-function getMancerModelTemplate(option) {
-    const model = models.find(x => x.id === option?.element?.value);
-
-    if (!option.id || !model) {
-        return option.text;
-    }
-
-    const creditsPerPrompt = (model.limits?.context - model.limits?.completion) * model.pricing?.prompt;
-    const creditsPerCompletion = model.limits?.completion * model.pricing?.completion;
-    const creditsTotal = Math.round(creditsPerPrompt + creditsPerCompletion).toFixed(0);
-
-    return $((`
-        <div class="flex-container flexFlowColumn">
-            <div><strong>${DOMPurify.sanitize(model.name)}</strong> | <span>${model.limits?.context} ctx</span> / <span>${model.limits?.completion} res</span> | <small>Credits per request (max): ${creditsTotal}</small></div>
-        </div>
-    `));
-}
-
-jQuery(function () {
-    $('#mancer_model').on('change', onMancerModelSelect);
-
-    if (!isMobile()) {
-        $('#mancer_model').select2({
-            placeholder: 'Select a model',
-            searchInputPlaceholder: 'Search models...',
-            searchInputCssClass: 'text_pole',
-            width: '100%',
-            templateResult: getMancerModelTemplate,
-        });
-    }
-});

public/scripts/secrets.js

@@ -15,6 +15,7 @@ export const SECRET_KEYS = {
     MAKERSUITE: 'api_key_makersuite',
     SERPAPI: 'api_key_serpapi',
     MISTRALAI: 'api_key_mistralai',
+    TOGETHERAI: 'api_key_togetherai',
 };
 
 const INPUT_MAP = {
@@ -31,6 +32,7 @@ const INPUT_MAP = {
     [SECRET_KEYS.APHRODITE]: '#api_key_aphrodite',
     [SECRET_KEYS.TABBY]: '#api_key_tabby',
     [SECRET_KEYS.MISTRALAI]: '#api_key_mistralai',
+    [SECRET_KEYS.TOGETHERAI]: '#api_key_togetherai',
 };
 
 async function clearSecret() {

public/scripts/textgen-models.js

@@ -0,0 +1,120 @@
+import { setGenerationParamsFromPreset } from '../script.js';
+import { isMobile } from './RossAscends-mods.js';
+import { textgenerationwebui_settings as textgen_settings } from './textgen-settings.js';
+
+let mancerModels = [];
+let togetherModels = [];
+
+export async function loadTogetherAIModels(data) {
+    if (!Array.isArray(data)) {
+        console.error('Invalid Together AI models data', data);
+        return;
+    }
+
+    togetherModels = data;
+
+    $('#model_togetherai_select').empty();
+    for (const model of data) {
+        // Hey buddy, I think you've got the wrong door.
+        if (model.display_type === 'image') {
+            continue;
+        }
+
+        const option = document.createElement('option');
+        option.value = model.name;
+        option.text = model.display_name;
+        option.selected = model.name === textgen_settings.togetherai_model;
+        $('#model_togetherai_select').append(option);
+    }
+}
+
+export async function loadMancerModels(data) {
+    if (!Array.isArray(data)) {
+        console.error('Invalid Mancer models data', data);
+        return;
+    }
+
+    mancerModels = data;
+
+    $('#mancer_model').empty();
+    for (const model of data) {
+        const option = document.createElement('option');
+        option.value = model.id;
+        option.text = model.name;
+        option.selected = model.id === textgen_settings.mancer_model;
+        $('#mancer_model').append(option);
+    }
+}
+
+function onMancerModelSelect() {
+    const modelId = String($('#mancer_model').val());
+    textgen_settings.mancer_model = modelId;
+    $('#api_button_textgenerationwebui').trigger('click');
+
+    const limits = mancerModels.find(x => x.id === modelId)?.limits;
+    setGenerationParamsFromPreset({ max_length: limits.context, genamt: limits.completion });
+}
+
+
+function onTogetherModelSelect() {
+    const modelName = String($('#model_togetherai_select').val());
+    textgen_settings.togetherai_model = modelName;
+    $('#api_button_textgenerationwebui').trigger('click');
+    const model = togetherModels.find(x => x.name === modelName);
+    setGenerationParamsFromPreset({ max_length: model.context_length });
+}
+
+function getMancerModelTemplate(option) {
+    const model = mancerModels.find(x => x.id === option?.element?.value);
+
+    if (!option.id || !model) {
+        return option.text;
+    }
+
+    const creditsPerPrompt = (model.limits?.context - model.limits?.completion) * model.pricing?.prompt;
+    const creditsPerCompletion = model.limits?.completion * model.pricing?.completion;
+    const creditsTotal = Math.round(creditsPerPrompt + creditsPerCompletion).toFixed(0);
+
+    return $((`
+        <div class="flex-container flexFlowColumn">
+            <div><strong>${DOMPurify.sanitize(model.name)}</strong> | <span>${model.limits?.context} ctx</span> / <span>${model.limits?.completion} res</span> | <small>Credits per request (max): ${creditsTotal}</small></div>
+        </div>
+    `));
+}
+
+function getTogetherModelTemplate(option) {
+    const model = togetherModels.find(x => x.name === option?.element?.value);
+
+    if (!option.id || !model) {
+        return option.text;
+    }
+
+    return $((`
+        <div class="flex-container flexFlowColumn">
+            <div><strong>${DOMPurify.sanitize(model.name)}</strong> | <span>${model.context_length || '???'} tokens</span></div>
+            <div><small>${DOMPurify.sanitize(model.description)}</small></div>
+        </div>
+    `));
+}
+
+jQuery(function () {
+    $('#mancer_model').on('change', onMancerModelSelect);
+    $('#model_togetherai_select').on('change', onTogetherModelSelect);
+
+    if (!isMobile()) {
+        $('#mancer_model').select2({
+            placeholder: 'Select a model',
+            searchInputPlaceholder: 'Search models...',
+            searchInputCssClass: 'text_pole',
+            width: '100%',
+            templateResult: getMancerModelTemplate,
+        });
+        $('#model_togetherai_select').select2({
+            placeholder: 'Select a model',
+            searchInputPlaceholder: 'Search models...',
+            searchInputCssClass: 'text_pole',
+            width: '100%',
+            templateResult: getTogetherModelTemplate,
+        });
+    }
+});

public/scripts/textgen-settings.js

@@ -31,15 +31,17 @@ export const textgen_types = {
     APHRODITE: 'aphrodite',
     TABBY: 'tabby',
     KOBOLDCPP: 'koboldcpp',
+    TOGETHERAI: 'togetherai',
 };
 
-const { MANCER, APHRODITE } = textgen_types;
+const { MANCER, APHRODITE, TOGETHERAI } = textgen_types;
 
 // Maybe let it be configurable in the future?
 // (7 days later) The future has come.
 const MANCER_SERVER_KEY = 'mancer_server';
 const MANCER_SERVER_DEFAULT = 'https://neuro.mancer.tech';
-export let MANCER_SERVER = localStorage.getItem(MANCER_SERVER_KEY) ?? MANCER_SERVER_DEFAULT;
+let MANCER_SERVER = localStorage.getItem(MANCER_SERVER_KEY) ?? MANCER_SERVER_DEFAULT;
+let TOGETHERAI_SERVER = 'https://api.together.xyz';
 
 const KOBOLDCPP_ORDER = [6, 0, 1, 3, 4, 2, 5];
 const settings = {
@@ -89,6 +91,7 @@ const settings = {
     //prompt_log_probs_aphrodite: 0,
     type: textgen_types.OOBA,
     mancer_model: 'mytholite',
+    togetherai_model: 'Gryphe/MythoMax-L2-13b',
     legacy_api: false,
     sampler_order: KOBOLDCPP_ORDER,
     n: 1,
@@ -164,8 +167,8 @@ async function selectPreset(name) {
 
 function formatTextGenURL(value) {
     try {
-        // Mancer doesn't need any formatting (it's hardcoded)
+        // Mancer/Together doesn't need any formatting (it's hardcoded)
-        if (settings.type === MANCER) {
+        if (settings.type === MANCER || settings.type === TOGETHERAI) {
             return value;
         }
 
@@ -546,6 +549,10 @@ function getModel() {
         return settings.mancer_model;
     }
 
+    if (settings.type === TOGETHERAI) {
+        return settings.togetherai_model;
+    }
+
     if (settings.type === APHRODITE) {
         return online_status;
     }
@@ -553,6 +560,18 @@ function getModel() {
     return undefined;
 }
 
+export function getTextGenServer() {
+    if (settings.type === MANCER) {
+        return MANCER_SERVER;
+    }
+
+    if (settings.type === TOGETHERAI) {
+        return TOGETHERAI_SERVER;
+    }
+
+    return api_server_textgenerationwebui;
+}
+
 export function getTextGenGenerationData(finalPrompt, maxTokens, isImpersonate, isContinue, cfgValues, type) {
     const canMultiSwipe = !isContinue && !isImpersonate && type !== 'quiet';
     let APIflags = {
@@ -590,10 +609,8 @@ export function getTextGenGenerationData(finalPrompt, maxTokens, isImpersonate,
             toIntArray(getCustomTokenBans()) :
             getCustomTokenBans(),
         'api_type': settings.type,
-        'api_server': settings.type === MANCER ?
+        'api_server': getTextGenServer(),
-            MANCER_SERVER :
+        'legacy_api': settings.legacy_api && settings.type !== MANCER && settings.type !== TOGETHERAI,
-            api_server_textgenerationwebui,
-        'legacy_api': settings.legacy_api && settings.type !== MANCER,
         'sampler_order': settings.type === textgen_types.KOBOLDCPP ?
             settings.sampler_order :
             undefined,

public/scripts/tokenizers.js

@@ -6,7 +6,7 @@ import { getStringHash } from './utils.js';
 import { kai_flags } from './kai-settings.js';
 import { textgen_types, textgenerationwebui_settings as textgen_settings } from './textgen-settings.js';
 
-const { OOBA, TABBY, KOBOLDCPP, MANCER } = textgen_types;
+const { OOBA, TABBY, KOBOLDCPP, MANCER, TOGETHERAI } = textgen_types;
 
 export const CHARACTERS_PER_TOKEN_RATIO = 3.35;
 const TOKENIZER_WARNING_KEY = 'tokenizationWarningShown';
@@ -540,7 +540,8 @@ function getTextgenAPITokenizationParams(str) {
         url: api_server_textgenerationwebui,
         legacy_api:
             textgen_settings.legacy_api &&
-            textgen_settings.type !== MANCER,
+            textgen_settings.type !== MANCER &&
+            textgen_settings.type !== TOGETHERAI,
     };
 }
 

src/additional-headers.js

@@ -11,6 +11,14 @@ function getMancerHeaders() {
     }) : {};
 }
 
+function getTogetherAIHeaders() {
+    const apiKey = readSecret(SECRET_KEYS.TOGETHERAI);
+
+    return apiKey ? ({
+        'Authorization': `Bearer ${apiKey}`,
+    }) : {};
+}
+
 function getAphroditeHeaders() {
     const apiKey = readSecret(SECRET_KEYS.APHRODITE);
 
@@ -58,6 +66,9 @@ function setAdditionalHeaders(request, args, server) {
         case TEXTGEN_TYPES.TABBY:
             headers = getTabbyHeaders();
             break;
+        case TEXTGEN_TYPES.TOGETHERAI:
+            headers = getTogetherAIHeaders();
+            break;
         default:
             headers = server ? getOverrideHeaders((new URL(server))?.host) : {};
             break;

src/constants.js

@@ -171,8 +171,21 @@ const TEXTGEN_TYPES = {
     APHRODITE: 'aphrodite',
     TABBY: 'tabby',
     KOBOLDCPP: 'koboldcpp',
+    TOGETHERAI: 'togetherai',
 };
 
+// https://docs.together.ai/reference/completions
+const TOGETHERAI_KEYS = [
+    'model',
+    'prompt',
+    'max_tokens',
+    'temperature',
+    'top_p',
+    'top_k',
+    'repetition_penalty',
+    'stream',
+];
+
 const AVATAR_WIDTH = 400;
 const AVATAR_HEIGHT = 600;
 
@@ -186,4 +199,5 @@ module.exports = {
     CHAT_COMPLETION_SOURCES,
     AVATAR_WIDTH,
     AVATAR_HEIGHT,
+    TOGETHERAI_KEYS,
 };

src/endpoints/backends/text-completions.js

@@ -1,8 +1,9 @@
 const express = require('express');
 const fetch = require('node-fetch').default;
+const _ = require('lodash');
 
 const { jsonParser } = require('../../express-common');
-const { TEXTGEN_TYPES } = require('../../constants');
+const { TEXTGEN_TYPES, TOGETHERAI_KEYS } = require('../../constants');
 const { forwardFetchResponse } = require('../../util');
 const { setAdditionalHeaders } = require('../../additional-headers');
 
@@ -46,6 +47,9 @@ router.post('/status', jsonParser, async function (request, response) {
                 case TEXTGEN_TYPES.TABBY:
                     url += '/v1/model/list';
                     break;
+                case TEXTGEN_TYPES.TOGETHERAI:
+                    url += '/api/models?&info';
+                    break;
             }
         }
 
@@ -56,13 +60,18 @@ router.post('/status', jsonParser, async function (request, response) {
             return response.status(400);
         }
 
-        const data = await modelsReply.json();
+        let data = await modelsReply.json();
 
         if (request.body.legacy_api) {
             console.log('Legacy API response:', data);
             return response.send({ result: data?.result });
         }
 
+        // Rewrap to OAI-like response
+        if (request.body.api_type === TEXTGEN_TYPES.TOGETHERAI && Array.isArray(data)) {
+            data = { data: data.map(x => ({ id: x.name, ...x })) };
+        }
+
         if (!Array.isArray(data.data)) {
             console.log('Models response is not an array.');
             return response.status(400);
@@ -145,6 +154,7 @@ router.post('/generate', jsonParser, async function (request, response_generate)
                 case TEXTGEN_TYPES.OOBA:
                 case TEXTGEN_TYPES.TABBY:
                 case TEXTGEN_TYPES.KOBOLDCPP:
+                case TEXTGEN_TYPES.TOGETHERAI:
                     url += '/v1/completions';
                     break;
                 case TEXTGEN_TYPES.MANCER:
@@ -163,6 +173,15 @@ router.post('/generate', jsonParser, async function (request, response_generate)
 
         setAdditionalHeaders(request, args, baseUrl);
 
+        if (request.body.api_type === TEXTGEN_TYPES.TOGETHERAI) {
+            const stop = Array.isArray(request.body.stop) ? request.body.stop[0] : '';
+            request.body = _.pickBy(request.body, (_, key) => TOGETHERAI_KEYS.includes(key));
+            if (typeof stop === 'string' && stop.length > 0) {
+                request.body.stop = stop;
+            }
+            args.body = JSON.stringify(request.body);
+        }
+
         if (request.body.stream) {
             const completionsStream = await fetch(url, args);
             // Pipe remote SSE stream to Express response

src/endpoints/secrets.js

@@ -25,6 +25,7 @@ const SECRET_KEYS = {
     DEEPLX_URL: 'deeplx_url',
     MAKERSUITE: 'api_key_makersuite',
     SERPAPI: 'api_key_serpapi',
+    TOGETHERAI: 'api_key_togetherai',
     MISTRALAI: 'api_key_mistralai',
 };
 

src/endpoints/stable-diffusion.js

@@ -1,11 +1,12 @@
 const express = require('express');
 const fetch = require('node-fetch').default;
 const sanitize = require('sanitize-filename');
-const { getBasicAuthHeader, delay } = require('../util.js');
+const { getBasicAuthHeader, delay, getHexString } = require('../util.js');
 const fs = require('fs');
 const { DIRECTORIES } = require('../constants.js');
 const writeFileAtomicSync = require('write-file-atomic').sync;
 const { jsonParser } = require('../express-common');
+const { readSecret, SECRET_KEYS } = require('./secrets.js');
 
 /**
  * Sanitizes a string.
@@ -545,6 +546,99 @@ comfy.post('/generate', jsonParser, async (request, response) => {
     }
 });
 
+const together = express.Router();
+
+together.post('/models', jsonParser, async (_, response) => {
+    try {
+        const key = readSecret(SECRET_KEYS.TOGETHERAI);
+
+        if (!key) {
+            console.log('TogetherAI key not found.');
+            return response.sendStatus(400);
+        }
+
+        const modelsResponse = await fetch('https://api.together.xyz/api/models', {
+            method: 'GET',
+            headers: {
+                'Authorization': `Bearer ${key}`,
+            },
+        });
+
+        if (!modelsResponse.ok) {
+            console.log('TogetherAI returned an error.');
+            return response.sendStatus(500);
+        }
+
+        const data = await modelsResponse.json();
+
+        if (!Array.isArray(data)) {
+            console.log('TogetherAI returned invalid data.');
+            return response.sendStatus(500);
+        }
+
+        const models = data
+            .filter(x => x.display_type === 'image')
+            .map(x => ({ value: x.name, text: x.display_name }));
+
+        return response.send(models);
+    } catch (error) {
+        console.log(error);
+        return response.sendStatus(500);
+    }
+});
+
+together.post('/generate', jsonParser, async (request, response) => {
+    try {
+        const key = readSecret(SECRET_KEYS.TOGETHERAI);
+
+        if (!key) {
+            console.log('TogetherAI key not found.');
+            return response.sendStatus(400);
+        }
+
+        console.log('TogetherAI request:', request.body);
+
+        const result = await fetch('https://api.together.xyz/api/inference', {
+            method: 'POST',
+            body: JSON.stringify({
+                request_type: 'image-model-inference',
+                prompt: request.body.prompt,
+                negative_prompt: request.body.negative_prompt,
+                height: request.body.height,
+                width: request.body.width,
+                model: request.body.model,
+                steps: request.body.steps,
+                n: 1,
+                seed: Math.floor(Math.random() * 10_000_000), // Limited to 10000 on playground, works fine with more.
+                sessionKey: getHexString(40), // Don't know if that's supposed to be random or not. It works either way.
+            }),
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${key}`,
+            },
+        });
+
+        if (!result.ok) {
+            console.log('TogetherAI returned an error.');
+            return response.sendStatus(500);
+        }
+
+        const data = await result.json();
+        console.log('TogetherAI response:', data);
+
+        if (data.status !== 'finished') {
+            console.log('TogetherAI job failed.');
+            return response.sendStatus(500);
+        }
+
+        return response.send(data);
+    } catch (error) {
+        console.log(error);
+        return response.sendStatus(500);
+    }
+});
+
 router.use('/comfy', comfy);
+router.use('/together', together);
 
 module.exports = { router };

src/util.js

@@ -105,6 +105,21 @@ function delay(ms) {
     return new Promise(resolve => setTimeout(resolve, ms));
 }
 
+/**
+ * Generates a random hex string of the given length.
+ * @param {number} length String length
+ * @returns {string} Random hex string
+ * @example getHexString(8) // 'a1b2c3d4'
+ */
+function getHexString(length) {
+    const chars = '0123456789abcdef';
+    let result = '';
+    for (let i = 0; i < length; i++) {
+        result += chars[Math.floor(Math.random() * chars.length)];
+    }
+    return result;
+}
+
 /**
  * Extracts a file with given extension from an ArrayBuffer containing a ZIP archive.
  * @param {ArrayBuffer} archiveBuffer Buffer containing a ZIP archive
@@ -404,4 +419,5 @@ module.exports = {
     removeOldBackups,
     getImages,
     forwardFetchResponse,
+    getHexString,
 };