Merge pull request #3112 from kallewoof/202411-backend-maxctx

feature: allow auto-use of max context size given by backend

public/index.html

@@ -2566,15 +2566,21 @@
                                     <input id="koboldcpp_api_url_text" class="text_pole wide100p" value="" autocomplete="off" data-server-history="koboldcpp">
                                 </div>
                             </div>
+                            <div class="flex-container flexFlowColumn marginTopBot5">
+                                <label data-tg-type="ooba" class="checkbox_label" for="bypass_status_check_textgenerationwebui">
+                                    <input type="checkbox" id="bypass_status_check_textgenerationwebui" />
+                                    <span data-i18n="Bypass status check">Bypass status check</span>
+                                </label>
+                                <label data-tg-type="koboldcpp, llamacpp" class="checkbox_label" for="context_size_derived">
+                                    <input type="checkbox" id="context_size_derived" />
+                                    <span data-i18n="Derive context size from backend">Derive context size from backend</span>
+                                </label>
+                            </div>
                             <div class="flex-container">
                                 <div id="api_button_textgenerationwebui" class="api_button menu_button menu_button_icon" type="submit" data-i18n="Connect" data-server-connect="ooba_blocking,vllm,aphrodite,tabby,koboldcpp,ollama,llamacpp,huggingface">Connect</div>
                                 <div data-tg-type="openrouter" class="menu_button menu_button_icon openrouter_authorize" title="Get your OpenRouter API token using OAuth flow. You will be redirected to openrouter.ai" data-i18n="Authorize;[title]Get your OpenRouter API token using OAuth flow. You will be redirected to openrouter.ai">Authorize</div>
                                 <div class="api_loading menu_button menu_button_icon" data-i18n="Cancel">Cancel</div>
                             </div>
-                            <label data-tg-type="ooba" class="checkbox_label margin-bot-10px" for="bypass_status_check_textgenerationwebui">
-                                <input type="checkbox" id="bypass_status_check_textgenerationwebui" />
-                                <span data-i18n="Bypass status check">Bypass status check</span>
-                            </label>
                         </form>
                         <div class="online_status">
                             <div class="online_status_indicator"></div>

public/script.js

@@ -1238,8 +1238,9 @@ async function getStatusTextgen() {
 
         const wantsInstructDerivation = (power_user.instruct.enabled && power_user.instruct.derived);
         const wantsContextDerivation = power_user.context_derived;
+        const wantsContextSize = power_user.context_size_derived;
         const supportsChatTemplate = [textgen_types.KOBOLDCPP, textgen_types.LLAMACPP].includes(textgen_settings.type);
-        if (supportsChatTemplate && (wantsInstructDerivation || wantsContextDerivation)) {
+        if (supportsChatTemplate && (wantsInstructDerivation || wantsContextDerivation || wantsContextSize)) {
             const response = await fetch('/api/backends/text-completions/props', {
                 method: 'POST',
                 headers: getRequestHeaders(),
@@ -1253,6 +1254,17 @@ async function getStatusTextgen() {
                 const data = await response.json();
                 if (data) {
                     const { chat_template, chat_template_hash } = data;
+                    if (wantsContextSize && 'default_generation_settings' in data) {
+                        const backend_max_context = data['default_generation_settings']['n_ctx'];
+                        const old_value = max_context;
+                        if (max_context !== backend_max_context) {
+                            setGenerationParamsFromPreset({ max_length: backend_max_context });
+                        }
+                        if (old_value !== max_context) {
+                            console.log(`Auto-switched max context from ${old_value} to ${max_context}`);
+                            toastr.info(`${old_value} ⇒ ${max_context}`, 'Context Size Changed');
+                        }
+                    }
                     console.log(`We have chat template ${chat_template.split('\n')[0]}...`);
                     const templates = await deriveTemplatesFromChatTemplate(chat_template, chat_template_hash);
                     if (templates) {
@@ -6822,6 +6834,10 @@ export async function saveSettings(type) {
     });
 }
 
+/**
+ * Sets the generation parameters from a preset object.
+ * @param {{ genamt?: number, max_length?: number }} preset Preset object
+ */
 export function setGenerationParamsFromPreset(preset) {
     const needsUnlock = (preset.max_length ?? max_context) > MAX_CONTEXT_DEFAULT || (preset.genamt ?? amount_gen) > MAX_RESPONSE_DEFAULT;
     $('#max_context_unlocked').prop('checked', needsUnlock).trigger('change');

public/scripts/power-user.js

@@ -246,6 +246,7 @@ let power_user = {
     },
 
     context_derived: false,
+    context_size_derived: false,
 
     sysprompt: {
         enabled: true,
@@ -1482,6 +1483,7 @@ async function loadPowerUserSettings(settings, data) {
     $('#example_messages_behavior').val(getExampleMessagesBehavior());
     $(`#example_messages_behavior option[value="${getExampleMessagesBehavior()}"]`).prop('selected', true);
     $('#context_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.context_derived);
+    $('#context_size_derived').prop('checked', !!power_user.context_size_derived);
 
     $('#console_log_prompts').prop('checked', power_user.console_log_prompts);
     $('#request_token_probabilities').prop('checked', power_user.request_token_probabilities);
@@ -3079,6 +3081,16 @@ $(document).ready(() => {
         $('#context_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.context_derived);
     });
 
+    $('#context_size_derived').on('input', function () {
+        const value = !!$(this).prop('checked');
+        power_user.context_size_derived = value;
+        saveSettingsDebounced();
+    });
+
+    $('#context_size_derived').on('change', function () {
+        $('#context_size_derived').prop('checked', !!power_user.context_size_derived);
+    });
+
     $('#always-force-name2-checkbox').change(function () {
         power_user.always_force_name2 = !!$(this).prop('checked');
         saveSettingsDebounced();

src/endpoints/backends/text-completions.js

@@ -152,7 +152,7 @@ router.post('/status', jsonParser, async function (request, response) {
 
         if (!modelsReply.ok) {
             console.log('Models endpoint is offline.');
-            return response.status(400);
+            return response.sendStatus(400);
         }
 
         /** @type {any} */
@@ -173,7 +173,7 @@ router.post('/status', jsonParser, async function (request, response) {
 
         if (!Array.isArray(data.data)) {
             console.log('Models response is not an array.');
-            return response.status(400);
+            return response.sendStatus(400);
         }
 
         const modelIds = data.data.map(x => x.id);
@@ -224,7 +224,7 @@ router.post('/status', jsonParser, async function (request, response) {
         return response.send({ result, data: data.data });
     } catch (error) {
         console.error(error);
-        return response.status(500);
+        return response.sendStatus(500);
     }
 });
 
@@ -244,7 +244,7 @@ router.post('/props', jsonParser, async function (request, response) {
         const propsReply = await fetch(propsUrl, args);
 
         if (!propsReply.ok) {
-            return response.status(400);
+            return response.sendStatus(400);
         }
 
         /** @type {any} */
@@ -258,7 +258,7 @@ router.post('/props', jsonParser, async function (request, response) {
         return response.send(props);
     } catch (error) {
         console.error(error);
-        return response.status(500);
+        return response.sendStatus(500);
     }
 });
 
@@ -450,7 +450,7 @@ ollama.post('/download', jsonParser, async function (request, response) {
         return response.send({ ok: true });
     } catch (error) {
         console.error(error);
-        return response.status(500);
+        return response.sendStatus(500);
     }
 });
 
@@ -493,7 +493,7 @@ ollama.post('/caption-image', jsonParser, async function (request, response) {
         return response.send({ caption });
     } catch (error) {
         console.error(error);
-        return response.status(500);
+        return response.sendStatus(500);
     }
 });
 
@@ -540,7 +540,7 @@ llamacpp.post('/caption-image', jsonParser, async function (request, response) {
 
     } catch (error) {
         console.error(error);
-        return response.status(500);
+        return response.sendStatus(500);
     }
 });
 
@@ -569,7 +569,7 @@ llamacpp.post('/props', jsonParser, async function (request, response) {
 
     } catch (error) {
         console.error(error);
-        return response.status(500);
+        return response.sendStatus(500);
     }
 });
 
@@ -619,7 +619,7 @@ llamacpp.post('/slots', jsonParser, async function (request, response) {
 
     } catch (error) {
         console.error(error);
-        return response.status(500);
+        return response.sendStatus(500);
     }
 });
 
@@ -665,7 +665,7 @@ tabby.post('/download', jsonParser, async function (request, response) {
         return response.send({ ok: true });
     } catch (error) {
         console.error(error);
-        return response.status(500);
+        return response.sendStatus(500);
     }
 });