Merge pull request #1565 from SillyTavern/togetherai

Add TogetherAI as a text completion source
2025-03-05 20:29:22 +01:00 · 2023-12-18 14:52:36 +02:00 · 2023-12-18 14:52:36 +02:00 · be5d428706
commit be5d428706
parent 2c501891b5 ac70a0a592
18 changed files with 483 additions and 97 deletions
--- a/default/settings.json
+++ b/default/settings.json
@ -3,7 +3,6 @@
    "username": "User",
    "api_server": "http://127.0.0.1:5000/api",
    "api_server_textgenerationwebui": "http://127.0.0.1:5000/api",
-    "api_use_mancer_webui": false,
    "preset_settings": "RecoveredRuins",
    "user_avatar": "user-default.png",
    "amount_gen": 250,
--- a/public/img/togetherai.svg
+++ b/public/img/togetherai.svg
@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   width="32"
+   height="32"
+   viewBox="0 0 32 32"
+   version="1.1"
+   id="svg4"
+   sodipodi:docname="togetherai.svg"
+   inkscape:version="1.3 (0e150ed, 2023-07-21)"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <sodipodi:namedview
+     id="namedview4"
+     pagecolor="#ffffff"
+     bordercolor="#000000"
+     borderopacity="0.25"
+     inkscape:showpageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1"
+     inkscape:zoom="7.375"
+     inkscape:cx="15.932203"
+     inkscape:cy="15.932203"
+     inkscape:window-width="1280"
+     inkscape:window-height="688"
+     inkscape:window-x="0"
+     inkscape:window-y="25"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="g4" />
+  <g
+     clip-path="url(#clip0_542_18748)"
+     id="g4">
+    <path
+       id="rect1"
+       d="M 5.6464844 0 C 2.5180256 -1.1842331e-15 3.5527101e-15 2.5180256 0 5.6464844 L 0 26.353516 C -1.1842331e-15 29.481971 2.5180256 32 5.6464844 32 L 26.353516 32 C 29.481971 32 32 29.481971 32 26.353516 L 32 5.6464844 C 32 2.5180256 29.481971 3.5527101e-15 26.353516 0 L 5.6464844 0 z M 9.6464844 4 A 5.6470599 5.6470599 0 0 1 15.294922 9.6464844 A 5.6470599 5.6470599 0 0 1 9.6464844 15.294922 A 5.6470599 5.6470599 0 0 1 4 9.6464844 A 5.6470599 5.6470599 0 0 1 9.6464844 4 z M 22.824219 4 A 5.6470599 5.6470599 0 0 1 28.470703 9.6464844 A 5.6470599 5.6470599 0 0 1 22.824219 15.294922 A 5.6470599 5.6470599 0 0 1 17.175781 9.6464844 A 5.6470599 5.6470599 0 0 1 22.824219 4 z M 9.6464844 17.175781 A 5.6470599 5.6470599 0 0 1 15.294922 22.824219 A 5.6470599 5.6470599 0 0 1 9.6464844 28.470703 A 5.6470599 5.6470599 0 0 1 4 22.824219 A 5.6470599 5.6470599 0 0 1 9.6464844 17.175781 z M 22.824219 17.175781 A 5.6470599 5.6470599 0 0 1 28.470703 22.824219 A 5.6470599 5.6470599 0 0 1 22.824219 28.470703 A 5.6470599 5.6470599 0 0 1 17.175781 22.824219 A 5.6470599 5.6470599 0 0 1 22.824219 17.175781 z " />
+    <circle
+       cx="9.64706"
+       cy="9.64706"
+       r="5.64706"
+       opacity="0.45"
+       id="circle9" />
+  </g>
+  <defs
+     id="defs4">
+    <clipPath
+       id="clip0_542_18748">
+      <rect
+         width="32"
+         height="32"
+         id="rect4" />
+    </clipPath>
+  </defs>
+</svg>
--- a/public/index.html
+++ b/public/index.html
@ -69,7 +69,7 @@
    <script type="module" src="scripts/group-chats.js"></script>
    <script type="module" src="scripts/kai-settings.js"></script>
    <script type="module" src="scripts/textgen-settings.js"></script>
-    <script type="module" src="scripts/mancer-settings.js"></script>
+    <script type="module" src="scripts/textgen-models.js"></script>
    <script type="module" src="scripts/bookmarks.js"></script>
    <script type="module" src="scripts/horde.js"></script>
    <script type="module" src="scripts/RossAscends-mods.js"></script>
@ -1592,8 +1592,8 @@
                            <option value="kobold"><span data-i18n="KoboldAI">KoboldAI Classic</span></option>
                            <option value="koboldhorde"><span data-i18n="KoboldAI Horde">KoboldAI Horde</span></option>
                            <option value="novel"><span data-i18n="NovelAI">NovelAI</span></option>
-                            <option value="textgenerationwebui"><span data-i18n="Text Completion">Text Completion (ooba, Mancer, Aphrodite, TabbyAPI, KoboldCpp)</span></option>
-                            <option value="openai"><span data-i18n="Chat Completion (OpenAI, Claude, Window/OpenRouter, Scale, AI21, Google MakerSuite, MistralAI)">Chat Completion (OpenAI, Claude, Window, OpenRouter, Scale, AI21, Google MakerSuite, MistralAI)</span></option>
+                            <option value="textgenerationwebui"><span data-i18n="Text Completion">Text Completion</span></option>
+                            <option value="openai"><span data-i18n="Chat Completion">Chat Completion</span></option>
                        </select>
                    </div>
                    <div id="kobold_horde" style="position: relative;"> <!-- shows the kobold settings -->
@ -1723,8 +1723,27 @@
                                    <option value="aphrodite">Aphrodite</option>
                                    <option value="tabby">TabbyAPI</option>
                                    <option value="koboldcpp">KoboldCpp</option>
+                                    <option value="togetherai">TogetherAI</option>
                                </select>
                            </div>
+                            <div data-tg-type="togetherai" class="flex-container flexFlowColumn">
+                                <h4 data-i18n="TogetherAI API Key">TogetherAI API Key</h4>
+                                <div class="flex-container">
+                                    <input id="api_key_togetherai" name="api_key_togetherai" class="text_pole flex1" maxlength="500" value="" type="text" autocomplete="off">
+                                    <div title="Clear your API key" data-i18n="[title]Clear your API key" class="menu_button fa-solid fa-circle-xmark clear-api-key" data-key="api_key_togetherai"></div>
+                                </div>
+                                <div data-for="api_key_togetherai" class="neutral_warning">
+                                    For privacy reasons, your API key will be hidden after you reload the page.
+                                </div>
+                                <div>
+                                    <h4 data-i18n="TogetherAI Model">TogetherAI Model</h4>
+                                    <select id="model_togetherai_select">
+                                        <option>
+                                            -- Connect to the API --
+                                        </option>
+                                    </select>
+                                </div>
+                            </div>
                            <div data-tg-type="mancer" class="flex-container flexFlowColumn">
                                <div class="flex-container flexFlowColumn">
                                </div>
--- a/public/script.js
+++ b/public/script.js
@ -19,10 +19,10 @@ import {
    getTextGenUrlSourceId,
    textgen_types,
    textgenerationwebui_banned_in_macros,
-    MANCER_SERVER,
+    getTextGenServer,
 } from './scripts/textgen-settings.js';

-const { MANCER } = textgen_types;
+const { MANCER, TOGETHERAI } = textgen_types;

 import {
    world_info,
@ -189,7 +189,7 @@ import { createPersona, initPersonas, selectCurrentPersona, setPersonaDescriptio
 import { getBackgrounds, initBackgrounds, loadBackgroundSettings, background_settings } from './scripts/backgrounds.js';
 import { hideLoader, showLoader } from './scripts/loader.js';
 import { BulkEditOverlay, CharacterContextMenu } from './scripts/BulkEditOverlay.js';
-import { loadMancerModels } from './scripts/mancer-settings.js';
+import { loadMancerModels, loadTogetherAIModels } from './scripts/textgen-models.js';
 import { appendFileContent, hasPendingFileAttachment, populateFileAttachment } from './scripts/chats.js';
 import { replaceVariableMacros } from './scripts/variables.js';
 import { initPresetManager } from './scripts/preset-manager.js';
@ -931,9 +931,7 @@ async function getStatusKobold() {
 async function getStatusTextgen() {
    const url = '/api/backends/text-completions/status';

-    let endpoint = textgen_settings.type === MANCER ?
-        MANCER_SERVER :
-        api_server_textgenerationwebui;
+    let endpoint = getTextGenServer();

    if (!endpoint) {
        console.warn('No endpoint for status check');
@ -949,7 +947,8 @@ async function getStatusTextgen() {
                api_type: textgen_settings.type,
                legacy_api:
                    textgen_settings.legacy_api &&
-                    textgen_settings.type !== MANCER,
+                    textgen_settings.type !== MANCER &&
+                    textgen_settings.type !== TOGETHERAI,
            }),
            signal: abortStatusCheck.signal,
        });
@ -959,6 +958,9 @@ async function getStatusTextgen() {
        if (textgen_settings.type === MANCER) {
            online_status = textgen_settings.mancer_model;
            loadMancerModels(data?.data);
+        } else if (textgen_settings.type === TOGETHERAI) {
+            online_status = textgen_settings.togetherai_model;
+            loadTogetherAIModels(data?.data);
        } else {
            online_status = data?.result;
        }
@ -2958,7 +2960,8 @@ async function Generate(type, { automatic_trigger, force_name2, quiet_prompt, qu
    if (main_api === 'textgenerationwebui' &&
        textgen_settings.streaming &&
        textgen_settings.legacy_api &&
-        textgen_settings.type !== MANCER) {
+        textgen_settings.type !== MANCER &&
+        textgen_settings.type !== TOGETHERAI) {
        toastr.error('Streaming is not supported for the Legacy API. Update Ooba and use --extensions openai to enable streaming.', undefined, { timeOut: 10000, preventDuplicates: true });
        unblockGeneration();
        return Promise.resolve();
@ -7514,6 +7517,11 @@ async function connectAPISlash(_, text) {
            button: '#api_button_textgenerationwebui',
            type: textgen_types.KOBOLDCPP,
        },
+        'togetherai': {
+            selected: 'textgenerationwebui',
+            button: '#api_button_textgenerationwebui',
+            type: textgen_types.TOGETHERAI,
+        },
        'oai': {
            selected: 'openai',
            source: 'openai',
@ -7840,7 +7848,7 @@ jQuery(async function () {
    }

    registerSlashCommand('dupe', DupeChar, [], '– duplicates the currently selected character', true, true);
-    registerSlashCommand('api', connectAPISlash, [], '<span class="monospace">(kobold, horde, novel, ooba, tabby, mancer, aphrodite, kcpp, oai, claude, windowai, openrouter, scale, ai21, makersuite, mistralai)</span> – connect to an API', true, true);
+    registerSlashCommand('api', connectAPISlash, [], '<span class="monospace">(kobold, horde, novel, ooba, tabby, mancer, aphrodite, kcpp, oai, claude, windowai, openrouter, scale, ai21, makersuite, mistralai, togetherai)</span> – connect to an API', true, true);
    registerSlashCommand('impersonate', doImpersonate, ['imp'], '– calls an impersonation response', true, true);
    registerSlashCommand('delchat', doDeleteChat, [], '– deletes the current chat', true, true);
    registerSlashCommand('closechat', doCloseChat, [], '– closes the current chat', true, true);
@ -8394,6 +8402,11 @@ jQuery(async function () {
            await writeSecret(SECRET_KEYS.TABBY, tabbyKey);
        }

+        const togetherKey = String($('#api_key_togetherai').val()).trim();
+        if (togetherKey.length) {
+            await writeSecret(SECRET_KEYS.TOGETHERAI, togetherKey);
+        }
+
        const urlSourceId = getTextGenUrlSourceId();

        if (urlSourceId && $(urlSourceId).val() !== '') {
--- a/public/scripts/RossAscends-mods.js
+++ b/public/scripts/RossAscends-mods.js
@ -382,7 +382,9 @@ function RA_autoconnect(PrevApi) {
                }
                break;
            case 'textgenerationwebui':
-                if (textgen_settings.type === textgen_types.MANCER && secret_state[SECRET_KEYS.MANCER]) {
+                if ((textgen_settings.type === textgen_types.MANCER && secret_state[SECRET_KEYS.MANCER]) ||
+                    (textgen_settings.type === textgen_types.TOGETHERAI && secret_state[SECRET_KEYS.TOGETHERAI])
+                ) {
                    $('#api_button_textgenerationwebui').trigger('click');
                }
                else if (api_server_textgenerationwebui && isValidUrl(api_server_textgenerationwebui)) {
--- a/public/scripts/extensions/stable-diffusion/index.js
+++ b/public/scripts/extensions/stable-diffusion/index.js
@ -46,6 +46,7 @@ const sources = {
    vlad: 'vlad',
    openai: 'openai',
    comfy: 'comfy',
+    togetherai: 'togetherai',
 };

 const generationMode = {
@ -917,7 +918,7 @@ async function onModelChange() {
    extension_settings.sd.model = $('#sd_model').find(':selected').val();
    saveSettingsDebounced();

-    const cloudSources = [sources.horde, sources.novel, sources.openai];
+    const cloudSources = [sources.horde, sources.novel, sources.openai, sources.togetherai];

    if (cloudSources.includes(extension_settings.sd.source)) {
        return;
@ -1050,11 +1051,14 @@ async function loadSamplers() {
            samplers = await loadVladSamplers();
            break;
        case sources.openai:
-            samplers = await loadOpenAiSamplers();
+            samplers = ['N/A'];
            break;
        case sources.comfy:
            samplers = await loadComfySamplers();
            break;
+        case sources.togetherai:
+            samplers = ['N/A'];
+            break;
    }

    for (const sampler of samplers) {
@ -1064,6 +1068,11 @@ async function loadSamplers() {
        option.selected = sampler === extension_settings.sd.sampler;
        $('#sd_sampler').append(option);
    }
+
+    if (!extension_settings.sd.sampler && samplers.length > 0) {
+        extension_settings.sd.sampler = samplers[0];
+        $('#sd_sampler').val(extension_settings.sd.sampler).trigger('change');
+    }
 }

 async function loadHordeSamplers() {
@ -1120,10 +1129,6 @@ async function loadAutoSamplers() {
    }
 }

-async function loadOpenAiSamplers() {
-    return ['N/A'];
-}
-
 async function loadVladSamplers() {
    if (!extension_settings.sd.vlad_url) {
        return [];
@ -1212,6 +1217,9 @@ async function loadModels() {
        case sources.comfy:
            models = await loadComfyModels();
            break;
+        case sources.togetherai:
+            models = await loadTogetherAIModels();
+            break;
    }

    for (const model of models) {
@ -1221,6 +1229,30 @@ async function loadModels() {
        option.selected = model.value === extension_settings.sd.model;
        $('#sd_model').append(option);
    }
+
+    if (!extension_settings.sd.model && models.length > 0) {
+        extension_settings.sd.model = models[0].value;
+        $('#sd_model').val(extension_settings.sd.model).trigger('change');
+    }
+}
+
+async function loadTogetherAIModels() {
+    if (!secret_state[SECRET_KEYS.TOGETHERAI]) {
+        console.debug('TogetherAI API key is not set.');
+        return [];
+    }
+
+    const result = await fetch('/api/sd/together/models', {
+        method: 'POST',
+        headers: getRequestHeaders(),
+    });
+
+    if (result.ok) {
+        const data = await result.json();
+        return data;
+    }
+
+    return [];
 }

 async function loadHordeModels() {
@ -1434,6 +1466,9 @@ async function loadSchedulers() {
        case sources.openai:
            schedulers = ['N/A'];
            break;
+        case sources.togetherai:
+            schedulers = ['N/A'];
+            break;
        case sources.comfy:
            schedulers = await loadComfySchedulers();
            break;
@ -1493,6 +1528,9 @@ async function loadVaes() {
        case sources.openai:
            vaes = ['N/A'];
            break;
+        case sources.togetherai:
+            vaes = ['N/A'];
+            break;
        case sources.comfy:
            vaes = await loadComfyVaes();
            break;
@ -1873,6 +1911,9 @@ async function sendGenerationRequest(generationType, prompt, characterName = nul
            case sources.comfy:
                result = await generateComfyImage(prefixedPrompt);
                break;
+            case sources.togetherai:
+                result = await generateTogetherAIImage(prefixedPrompt);
+                break;
        }

        if (!result.data) {
@ -1895,6 +1936,29 @@ async function sendGenerationRequest(generationType, prompt, characterName = nul
    callback ? callback(prompt, base64Image, generationType) : sendMessage(prompt, base64Image, generationType);
 }

+async function generateTogetherAIImage(prompt) {
+    const result = await fetch('/api/sd/together/generate', {
+        method: 'POST',
+        headers: getRequestHeaders(),
+        body: JSON.stringify({
+            prompt: prompt,
+            negative_prompt: extension_settings.sd.negative_prompt,
+            model: extension_settings.sd.model,
+            steps: extension_settings.sd.steps,
+            width: extension_settings.sd.width,
+            height: extension_settings.sd.height,
+        }),
+    });
+
+    if (result.ok) {
+        const data = await result.json();
+        return { format: 'jpg', data: data?.output?.choices?.[0]?.image_base64 };
+    } else {
+        const text = await result.text();
+        throw new Error(text);
+    }
+}
+
 /**
 * Generates an "extras" image using a provided prompt and other settings.
 *
@ -2435,6 +2499,8 @@ function isValidState() {
            return secret_state[SECRET_KEYS.OPENAI];
        case sources.comfy:
            return true;
+        case sources.togetherai:
+            return secret_state[SECRET_KEYS.TOGETHERAI];
    }
 }

--- a/public/scripts/extensions/stable-diffusion/settings.html
+++ b/public/scripts/extensions/stable-diffusion/settings.html
@ -35,6 +35,7 @@
                <option value="novel">NovelAI Diffusion</option>
                <option value="openai">OpenAI (DALL-E)</option>
                <option value="comfy">ComfyUI</option>
+                <option value="togetherai">TogetherAI</option>
            </select>
            <div data-sd-source="auto">
                <label for="sd_auto_url">SD Web UI URL</label>
--- a/public/scripts/mancer-settings.js
+++ b/public/scripts/mancer-settings.js
@ -1,64 +0,0 @@
-import { setGenerationParamsFromPreset } from '../script.js';
-import { isMobile } from './RossAscends-mods.js';
-import { textgenerationwebui_settings as textgen_settings } from './textgen-settings.js';
-
-let models = [];
-
-export async function loadMancerModels(data) {
-    if (!Array.isArray(data)) {
-        console.error('Invalid Mancer models data', data);
-        return;
-    }
-
-    models = data;
-
-    $('#mancer_model').empty();
-    for (const model of data) {
-        const option = document.createElement('option');
-        option.value = model.id;
-        option.text = model.name;
-        option.selected = model.id === textgen_settings.mancer_model;
-        $('#mancer_model').append(option);
-    }
-}
-
-function onMancerModelSelect() {
-    const modelId = String($('#mancer_model').val());
-    textgen_settings.mancer_model = modelId;
-    $('#api_button_textgenerationwebui').trigger('click');
-
-    const limits = models.find(x => x.id === modelId)?.limits;
-    setGenerationParamsFromPreset({ max_length: limits.context, genamt: limits.completion });
-}
-
-function getMancerModelTemplate(option) {
-    const model = models.find(x => x.id === option?.element?.value);
-
-    if (!option.id || !model) {
-        return option.text;
-    }
-
-    const creditsPerPrompt = (model.limits?.context - model.limits?.completion) * model.pricing?.prompt;
-    const creditsPerCompletion = model.limits?.completion * model.pricing?.completion;
-    const creditsTotal = Math.round(creditsPerPrompt + creditsPerCompletion).toFixed(0);
-
-    return $((`
-        <div class="flex-container flexFlowColumn">
-            <div><strong>${DOMPurify.sanitize(model.name)}</strong> | <span>${model.limits?.context} ctx</span> / <span>${model.limits?.completion} res</span> | <small>Credits per request (max): ${creditsTotal}</small></div>
-        </div>
-    `));
-}
-
-jQuery(function () {
-    $('#mancer_model').on('change', onMancerModelSelect);
-
-    if (!isMobile()) {
-        $('#mancer_model').select2({
-            placeholder: 'Select a model',
-            searchInputPlaceholder: 'Search models...',
-            searchInputCssClass: 'text_pole',
-            width: '100%',
-            templateResult: getMancerModelTemplate,
-        });
-    }
-});
--- a/public/scripts/secrets.js
+++ b/public/scripts/secrets.js
@ -15,6 +15,7 @@ export const SECRET_KEYS = {
    MAKERSUITE: 'api_key_makersuite',
    SERPAPI: 'api_key_serpapi',
    MISTRALAI: 'api_key_mistralai',
+    TOGETHERAI: 'api_key_togetherai',
 };

 const INPUT_MAP = {
@ -31,6 +32,7 @@ const INPUT_MAP = {
    [SECRET_KEYS.APHRODITE]: '#api_key_aphrodite',
    [SECRET_KEYS.TABBY]: '#api_key_tabby',
    [SECRET_KEYS.MISTRALAI]: '#api_key_mistralai',
+    [SECRET_KEYS.TOGETHERAI]: '#api_key_togetherai',
 };

 async function clearSecret() {
--- a/public/scripts/textgen-models.js
+++ b/public/scripts/textgen-models.js
@ -0,0 +1,120 @@
+import { setGenerationParamsFromPreset } from '../script.js';
+import { isMobile } from './RossAscends-mods.js';
+import { textgenerationwebui_settings as textgen_settings } from './textgen-settings.js';
+
+let mancerModels = [];
+let togetherModels = [];
+
+export async function loadTogetherAIModels(data) {
+    if (!Array.isArray(data)) {
+        console.error('Invalid Together AI models data', data);
+        return;
+    }
+
+    togetherModels = data;
+
+    $('#model_togetherai_select').empty();
+    for (const model of data) {
+        // Hey buddy, I think you've got the wrong door.
+        if (model.display_type === 'image') {
+            continue;
+        }
+
+        const option = document.createElement('option');
+        option.value = model.name;
+        option.text = model.display_name;
+        option.selected = model.name === textgen_settings.togetherai_model;
+        $('#model_togetherai_select').append(option);
+    }
+}
+
+export async function loadMancerModels(data) {
+    if (!Array.isArray(data)) {
+        console.error('Invalid Mancer models data', data);
+        return;
+    }
+
+    mancerModels = data;
+
+    $('#mancer_model').empty();
+    for (const model of data) {
+        const option = document.createElement('option');
+        option.value = model.id;
+        option.text = model.name;
+        option.selected = model.id === textgen_settings.mancer_model;
+        $('#mancer_model').append(option);
+    }
+}
+
+function onMancerModelSelect() {
+    const modelId = String($('#mancer_model').val());
+    textgen_settings.mancer_model = modelId;
+    $('#api_button_textgenerationwebui').trigger('click');
+
+    const limits = mancerModels.find(x => x.id === modelId)?.limits;
+    setGenerationParamsFromPreset({ max_length: limits.context, genamt: limits.completion });
+}
+
+
+function onTogetherModelSelect() {
+    const modelName = String($('#model_togetherai_select').val());
+    textgen_settings.togetherai_model = modelName;
+    $('#api_button_textgenerationwebui').trigger('click');
+    const model = togetherModels.find(x => x.name === modelName);
+    setGenerationParamsFromPreset({ max_length: model.context_length });
+}
+
+function getMancerModelTemplate(option) {
+    const model = mancerModels.find(x => x.id === option?.element?.value);
+
+    if (!option.id || !model) {
+        return option.text;
+    }
+
+    const creditsPerPrompt = (model.limits?.context - model.limits?.completion) * model.pricing?.prompt;
+    const creditsPerCompletion = model.limits?.completion * model.pricing?.completion;
+    const creditsTotal = Math.round(creditsPerPrompt + creditsPerCompletion).toFixed(0);
+
+    return $((`
+        <div class="flex-container flexFlowColumn">
+            <div><strong>${DOMPurify.sanitize(model.name)}</strong> | <span>${model.limits?.context} ctx</span> / <span>${model.limits?.completion} res</span> | <small>Credits per request (max): ${creditsTotal}</small></div>
+        </div>
+    `));
+}
+
+function getTogetherModelTemplate(option) {
+    const model = togetherModels.find(x => x.name === option?.element?.value);
+
+    if (!option.id || !model) {
+        return option.text;
+    }
+
+    return $((`
+        <div class="flex-container flexFlowColumn">
+            <div><strong>${DOMPurify.sanitize(model.name)}</strong> | <span>${model.context_length || '???'} tokens</span></div>
+            <div><small>${DOMPurify.sanitize(model.description)}</small></div>
+        </div>
+    `));
+}
+
+jQuery(function () {
+    $('#mancer_model').on('change', onMancerModelSelect);
+    $('#model_togetherai_select').on('change', onTogetherModelSelect);
+
+    if (!isMobile()) {
+        $('#mancer_model').select2({
+            placeholder: 'Select a model',
+            searchInputPlaceholder: 'Search models...',
+            searchInputCssClass: 'text_pole',
+            width: '100%',
+            templateResult: getMancerModelTemplate,
+        });
+        $('#model_togetherai_select').select2({
+            placeholder: 'Select a model',
+            searchInputPlaceholder: 'Search models...',
+            searchInputCssClass: 'text_pole',
+            width: '100%',
+            templateResult: getTogetherModelTemplate,
+        });
+    }
+});
--- a/public/scripts/textgen-settings.js
+++ b/public/scripts/textgen-settings.js
@ -31,15 +31,17 @@ export const textgen_types = {
    APHRODITE: 'aphrodite',
    TABBY: 'tabby',
    KOBOLDCPP: 'koboldcpp',
+    TOGETHERAI: 'togetherai',
 };

-const { MANCER, APHRODITE } = textgen_types;
+const { MANCER, APHRODITE, TOGETHERAI } = textgen_types;

 // Maybe let it be configurable in the future?
 // (7 days later) The future has come.
 const MANCER_SERVER_KEY = 'mancer_server';
 const MANCER_SERVER_DEFAULT = 'https://neuro.mancer.tech';
-export let MANCER_SERVER = localStorage.getItem(MANCER_SERVER_KEY) ?? MANCER_SERVER_DEFAULT;
+let MANCER_SERVER = localStorage.getItem(MANCER_SERVER_KEY) ?? MANCER_SERVER_DEFAULT;
+let TOGETHERAI_SERVER = 'https://api.together.xyz';

 const KOBOLDCPP_ORDER = [6, 0, 1, 3, 4, 2, 5];
 const settings = {
@ -89,6 +91,7 @@ const settings = {
    //prompt_log_probs_aphrodite: 0,
    type: textgen_types.OOBA,
    mancer_model: 'mytholite',
+    togetherai_model: 'Gryphe/MythoMax-L2-13b',
    legacy_api: false,
    sampler_order: KOBOLDCPP_ORDER,
    n: 1,
@ -164,8 +167,8 @@ async function selectPreset(name) {

 function formatTextGenURL(value) {
    try {
-        // Mancer doesn't need any formatting (it's hardcoded)
-        if (settings.type === MANCER) {
+        // Mancer/Together doesn't need any formatting (it's hardcoded)
+        if (settings.type === MANCER || settings.type === TOGETHERAI) {
            return value;
        }

@ -546,6 +549,10 @@ function getModel() {
        return settings.mancer_model;
    }

+    if (settings.type === TOGETHERAI) {
+        return settings.togetherai_model;
+    }
+
    if (settings.type === APHRODITE) {
        return online_status;
    }
@ -553,6 +560,18 @@ function getModel() {
    return undefined;
 }

+export function getTextGenServer() {
+    if (settings.type === MANCER) {
+        return MANCER_SERVER;
+    }
+
+    if (settings.type === TOGETHERAI) {
+        return TOGETHERAI_SERVER;
+    }
+
+    return api_server_textgenerationwebui;
+}
+
 export function getTextGenGenerationData(finalPrompt, maxTokens, isImpersonate, isContinue, cfgValues, type) {
    const canMultiSwipe = !isContinue && !isImpersonate && type !== 'quiet';
    let APIflags = {
@ -590,10 +609,8 @@ export function getTextGenGenerationData(finalPrompt, maxTokens, isImpersonate,
            toIntArray(getCustomTokenBans()) :
            getCustomTokenBans(),
        'api_type': settings.type,
-        'api_server': settings.type === MANCER ?
-            MANCER_SERVER :
-            api_server_textgenerationwebui,
-        'legacy_api': settings.legacy_api && settings.type !== MANCER,
+        'api_server': getTextGenServer(),
+        'legacy_api': settings.legacy_api && settings.type !== MANCER && settings.type !== TOGETHERAI,
        'sampler_order': settings.type === textgen_types.KOBOLDCPP ?
            settings.sampler_order :
            undefined,
--- a/public/scripts/tokenizers.js
+++ b/public/scripts/tokenizers.js
@ -6,7 +6,7 @@ import { getStringHash } from './utils.js';
 import { kai_flags } from './kai-settings.js';
 import { textgen_types, textgenerationwebui_settings as textgen_settings } from './textgen-settings.js';

-const { OOBA, TABBY, KOBOLDCPP, MANCER } = textgen_types;
+const { OOBA, TABBY, KOBOLDCPP, MANCER, TOGETHERAI } = textgen_types;

 export const CHARACTERS_PER_TOKEN_RATIO = 3.35;
 const TOKENIZER_WARNING_KEY = 'tokenizationWarningShown';
@ -540,7 +540,8 @@ function getTextgenAPITokenizationParams(str) {
        url: api_server_textgenerationwebui,
        legacy_api:
            textgen_settings.legacy_api &&
-            textgen_settings.type !== MANCER,
+            textgen_settings.type !== MANCER &&
+            textgen_settings.type !== TOGETHERAI,
    };
 }

--- a/src/additional-headers.js
+++ b/src/additional-headers.js
@ -11,6 +11,14 @@ function getMancerHeaders() {
    }) : {};
 }

+function getTogetherAIHeaders() {
+    const apiKey = readSecret(SECRET_KEYS.TOGETHERAI);
+
+    return apiKey ? ({
+        'Authorization': `Bearer ${apiKey}`,
+    }) : {};
+}
+
 function getAphroditeHeaders() {
    const apiKey = readSecret(SECRET_KEYS.APHRODITE);

@ -58,6 +66,9 @@ function setAdditionalHeaders(request, args, server) {
        case TEXTGEN_TYPES.TABBY:
            headers = getTabbyHeaders();
            break;
+        case TEXTGEN_TYPES.TOGETHERAI:
+            headers = getTogetherAIHeaders();
+            break;
        default:
            headers = server ? getOverrideHeaders((new URL(server))?.host) : {};
            break;
--- a/src/constants.js
+++ b/src/constants.js
@ -171,8 +171,21 @@ const TEXTGEN_TYPES = {
    APHRODITE: 'aphrodite',
    TABBY: 'tabby',
    KOBOLDCPP: 'koboldcpp',
+    TOGETHERAI: 'togetherai',
 };

+// https://docs.together.ai/reference/completions
+const TOGETHERAI_KEYS = [
+    'model',
+    'prompt',
+    'max_tokens',
+    'temperature',
+    'top_p',
+    'top_k',
+    'repetition_penalty',
+    'stream',
+];
+
 const AVATAR_WIDTH = 400;
 const AVATAR_HEIGHT = 600;

@ -186,4 +199,5 @@ module.exports = {
    CHAT_COMPLETION_SOURCES,
    AVATAR_WIDTH,
    AVATAR_HEIGHT,
+    TOGETHERAI_KEYS,
 };
--- a/src/endpoints/backends/text-completions.js
+++ b/src/endpoints/backends/text-completions.js
@ -1,8 +1,9 @@
 const express = require('express');
 const fetch = require('node-fetch').default;
+const _ = require('lodash');

 const { jsonParser } = require('../../express-common');
-const { TEXTGEN_TYPES } = require('../../constants');
+const { TEXTGEN_TYPES, TOGETHERAI_KEYS } = require('../../constants');
 const { forwardFetchResponse } = require('../../util');
 const { setAdditionalHeaders } = require('../../additional-headers');

@ -46,6 +47,9 @@ router.post('/status', jsonParser, async function (request, response) {
                case TEXTGEN_TYPES.TABBY:
                    url += '/v1/model/list';
                    break;
+                case TEXTGEN_TYPES.TOGETHERAI:
+                    url += '/api/models?&info';
+                    break;
            }
        }

@ -56,13 +60,18 @@ router.post('/status', jsonParser, async function (request, response) {
            return response.status(400);
        }

-        const data = await modelsReply.json();
+        let data = await modelsReply.json();

        if (request.body.legacy_api) {
            console.log('Legacy API response:', data);
            return response.send({ result: data?.result });
        }

+        // Rewrap to OAI-like response
+        if (request.body.api_type === TEXTGEN_TYPES.TOGETHERAI && Array.isArray(data)) {
+            data = { data: data.map(x => ({ id: x.name, ...x })) };
+        }
+
        if (!Array.isArray(data.data)) {
            console.log('Models response is not an array.');
            return response.status(400);
@ -145,6 +154,7 @@ router.post('/generate', jsonParser, async function (request, response_generate)
                case TEXTGEN_TYPES.OOBA:
                case TEXTGEN_TYPES.TABBY:
                case TEXTGEN_TYPES.KOBOLDCPP:
+                case TEXTGEN_TYPES.TOGETHERAI:
                    url += '/v1/completions';
                    break;
                case TEXTGEN_TYPES.MANCER:
@ -163,6 +173,15 @@ router.post('/generate', jsonParser, async function (request, response_generate)

        setAdditionalHeaders(request, args, baseUrl);

+        if (request.body.api_type === TEXTGEN_TYPES.TOGETHERAI) {
+            const stop = Array.isArray(request.body.stop) ? request.body.stop[0] : '';
+            request.body = _.pickBy(request.body, (_, key) => TOGETHERAI_KEYS.includes(key));
+            if (typeof stop === 'string' && stop.length > 0) {
+                request.body.stop = stop;
+            }
+            args.body = JSON.stringify(request.body);
+        }
+
        if (request.body.stream) {
            const completionsStream = await fetch(url, args);
            // Pipe remote SSE stream to Express response
--- a/src/endpoints/secrets.js
+++ b/src/endpoints/secrets.js
@ -25,6 +25,7 @@ const SECRET_KEYS = {
    DEEPLX_URL: 'deeplx_url',
    MAKERSUITE: 'api_key_makersuite',
    SERPAPI: 'api_key_serpapi',
+    TOGETHERAI: 'api_key_togetherai',
    MISTRALAI: 'api_key_mistralai',
 };

--- a/src/endpoints/stable-diffusion.js
+++ b/src/endpoints/stable-diffusion.js
@ -1,11 +1,12 @@
 const express = require('express');
 const fetch = require('node-fetch').default;
 const sanitize = require('sanitize-filename');
-const { getBasicAuthHeader, delay } = require('../util.js');
+const { getBasicAuthHeader, delay, getHexString } = require('../util.js');
 const fs = require('fs');
 const { DIRECTORIES } = require('../constants.js');
 const writeFileAtomicSync = require('write-file-atomic').sync;
 const { jsonParser } = require('../express-common');
+const { readSecret, SECRET_KEYS } = require('./secrets.js');

 /**
 * Sanitizes a string.
@ -545,6 +546,99 @@ comfy.post('/generate', jsonParser, async (request, response) => {
    }
 });

+const together = express.Router();
+
+together.post('/models', jsonParser, async (_, response) => {
+    try {
+        const key = readSecret(SECRET_KEYS.TOGETHERAI);
+
+        if (!key) {
+            console.log('TogetherAI key not found.');
+            return response.sendStatus(400);
+        }
+
+        const modelsResponse = await fetch('https://api.together.xyz/api/models', {
+            method: 'GET',
+            headers: {
+                'Authorization': `Bearer ${key}`,
+            },
+        });
+
+        if (!modelsResponse.ok) {
+            console.log('TogetherAI returned an error.');
+            return response.sendStatus(500);
+        }
+
+        const data = await modelsResponse.json();
+
+        if (!Array.isArray(data)) {
+            console.log('TogetherAI returned invalid data.');
+            return response.sendStatus(500);
+        }
+
+        const models = data
+            .filter(x => x.display_type === 'image')
+            .map(x => ({ value: x.name, text: x.display_name }));
+
+        return response.send(models);
+    } catch (error) {
+        console.log(error);
+        return response.sendStatus(500);
+    }
+});
+
+together.post('/generate', jsonParser, async (request, response) => {
+    try {
+        const key = readSecret(SECRET_KEYS.TOGETHERAI);
+
+        if (!key) {
+            console.log('TogetherAI key not found.');
+            return response.sendStatus(400);
+        }
+
+        console.log('TogetherAI request:', request.body);
+
+        const result = await fetch('https://api.together.xyz/api/inference', {
+            method: 'POST',
+            body: JSON.stringify({
+                request_type: 'image-model-inference',
+                prompt: request.body.prompt,
+                negative_prompt: request.body.negative_prompt,
+                height: request.body.height,
+                width: request.body.width,
+                model: request.body.model,
+                steps: request.body.steps,
+                n: 1,
+                seed: Math.floor(Math.random() * 10_000_000), // Limited to 10000 on playground, works fine with more.
+                sessionKey: getHexString(40), // Don't know if that's supposed to be random or not. It works either way.
+            }),
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${key}`,
+            },
+        });
+
+        if (!result.ok) {
+            console.log('TogetherAI returned an error.');
+            return response.sendStatus(500);
+        }
+
+        const data = await result.json();
+        console.log('TogetherAI response:', data);
+
+        if (data.status !== 'finished') {
+            console.log('TogetherAI job failed.');
+            return response.sendStatus(500);
+        }
+
+        return response.send(data);
+    } catch (error) {
+        console.log(error);
+        return response.sendStatus(500);
+    }
+});
+
 router.use('/comfy', comfy);
+router.use('/together', together);

 module.exports = { router };
--- a/src/util.js
+++ b/src/util.js
@ -105,6 +105,21 @@ function delay(ms) {
    return new Promise(resolve => setTimeout(resolve, ms));
 }

+/**
+ * Generates a random hex string of the given length.
+ * @param {number} length String length
+ * @returns {string} Random hex string
+ * @example getHexString(8) // 'a1b2c3d4'
+ */
+function getHexString(length) {
+    const chars = '0123456789abcdef';
+    let result = '';
+    for (let i = 0; i < length; i++) {
+        result += chars[Math.floor(Math.random() * chars.length)];
+    }
+    return result;
+}
+
 /**
 * Extracts a file with given extension from an ArrayBuffer containing a ZIP archive.
 * @param {ArrayBuffer} archiveBuffer Buffer containing a ZIP archive
@ -404,4 +419,5 @@ module.exports = {
    removeOldBackups,
    getImages,
    forwardFetchResponse,
+    getHexString,
 };