rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3112 from kallewoof/202411-backend-maxctx

Browse Source 
feature: allow auto-use of max context size given by backend
This commit is contained in:
Cohee
2024-12-01 15:15:15 +02:00
committed by GitHub
parent 66a862f797 8e94589f83
commit 39c3924b3f
4 changed files with 50 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
public/index.html
View File

@@ -2566,15 +2566,21 @@
<input id="koboldcpp_api_url_text" class="text_pole wide100p" value="" autocomplete="off" data-server-history="koboldcpp">
</div>
</div>
<div class="flex-container flexFlowColumn marginTopBot5">
<label data-tg-type="ooba" class="checkbox_label" for="bypass_status_check_textgenerationwebui">
<input type="checkbox" id="bypass_status_check_textgenerationwebui" />
<span data-i18n="Bypass status check">Bypass status check</span>
</label>
<label data-tg-type="koboldcpp, llamacpp" class="checkbox_label" for="context_size_derived">
<input type="checkbox" id="context_size_derived" />
<span data-i18n="Derive context size from backend">Derive context size from backend</span>
</label>
</div>
<div class="flex-container">
<div id="api_button_textgenerationwebui" class="api_button menu_button menu_button_icon" type="submit" data-i18n="Connect" data-server-connect="ooba_blocking,vllm,aphrodite,tabby,koboldcpp,ollama,llamacpp,huggingface">Connect</div>
<div data-tg-type="openrouter" class="menu_button menu_button_icon openrouter_authorize" title="Get your OpenRouter API token using OAuth flow. You will be redirected to openrouter.ai" data-i18n="Authorize;[title]Get your OpenRouter API token using OAuth flow. You will be redirected to openrouter.ai">Authorize</div>
<div class="api_loading menu_button menu_button_icon" data-i18n="Cancel">Cancel</div>
</div>
<label data-tg-type="ooba" class="checkbox_label margin-bot-10px" for="bypass_status_check_textgenerationwebui">
<input type="checkbox" id="bypass_status_check_textgenerationwebui" />
<span data-i18n="Bypass status check">Bypass status check</span>
</label>
</form>
<div class="online_status">
<div class="online_status_indicator"></div>

18
public/script.js
View File

@@ -1238,8 +1238,9 @@ async function getStatusTextgen() {
const wantsInstructDerivation = (power_user.instruct.enabled && power_user.instruct.derived);
const wantsContextDerivation = power_user.context_derived;
const wantsContextSize = power_user.context_size_derived;
const supportsChatTemplate = [textgen_types.KOBOLDCPP, textgen_types.LLAMACPP].includes(textgen_settings.type);
if (supportsChatTemplate && (wantsInstructDerivation || wantsContextDerivation)) {
if (supportsChatTemplate && (wantsInstructDerivation || wantsContextDerivation || wantsContextSize)) {
const response = await fetch('/api/backends/text-completions/props', {
method: 'POST',
headers: getRequestHeaders(),
@@ -1253,6 +1254,17 @@ async function getStatusTextgen() {
const data = await response.json();
if (data) {
const { chat_template, chat_template_hash } = data;
if (wantsContextSize && 'default_generation_settings' in data) {
const backend_max_context = data['default_generation_settings']['n_ctx'];
const old_value = max_context;
if (max_context !== backend_max_context) {
setGenerationParamsFromPreset({ max_length: backend_max_context });
}
if (old_value !== max_context) {
console.log(`Auto-switched max context from ${old_value} to ${max_context}`);
toastr.info(`${old_value} ⇒ ${max_context}`, 'Context Size Changed');
}
}
console.log(`We have chat template ${chat_template.split('\n')[0]}...`);
const templates = await deriveTemplatesFromChatTemplate(chat_template, chat_template_hash);
if (templates) {
@@ -6822,6 +6834,10 @@ export async function saveSettings(type) {
});
}
/**
* Sets the generation parameters from a preset object.
* @param {{ genamt?: number, max_length?: number }} preset Preset object
*/
export function setGenerationParamsFromPreset(preset) {
const needsUnlock = (preset.max_length ?? max_context) > MAX_CONTEXT_DEFAULT || (preset.genamt ?? amount_gen) > MAX_RESPONSE_DEFAULT;
$('#max_context_unlocked').prop('checked', needsUnlock).trigger('change');

12
public/scripts/power-user.js
View File

@@ -246,6 +246,7 @@ let power_user = {
},
context_derived: false,
context_size_derived: false,
sysprompt: {
enabled: true,
@@ -1482,6 +1483,7 @@ async function loadPowerUserSettings(settings, data) {
$('#example_messages_behavior').val(getExampleMessagesBehavior());
$(`#example_messages_behavior option[value="${getExampleMessagesBehavior()}"]`).prop('selected', true);
$('#context_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.context_derived);
$('#context_size_derived').prop('checked', !!power_user.context_size_derived);
$('#console_log_prompts').prop('checked', power_user.console_log_prompts);
$('#request_token_probabilities').prop('checked', power_user.request_token_probabilities);
@@ -3079,6 +3081,16 @@ $(document).ready(() => {
$('#context_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.context_derived);
});
$('#context_size_derived').on('input', function () {
const value = !!$(this).prop('checked');
power_user.context_size_derived = value;
saveSettingsDebounced();
});
$('#context_size_derived').on('change', function () {
$('#context_size_derived').prop('checked', !!power_user.context_size_derived);
});
$('#always-force-name2-checkbox').change(function () {
power_user.always_force_name2 = !!$(this).prop('checked');
saveSettingsDebounced();

22
src/endpoints/backends/text-completions.js
View File

@@ -152,7 +152,7 @@ router.post('/status', jsonParser, async function (request, response) {
if (!modelsReply.ok) {
console.log('Models endpoint is offline.');
return response.status(400);
return response.sendStatus(400);
}
/** @type {any} */
@@ -173,7 +173,7 @@ router.post('/status', jsonParser, async function (request, response) {
if (!Array.isArray(data.data)) {
console.log('Models response is not an array.');
return response.status(400);
return response.sendStatus(400);
}
const modelIds = data.data.map(x => x.id);
@@ -224,7 +224,7 @@ router.post('/status', jsonParser, async function (request, response) {
return response.send({ result, data: data.data });
} catch (error) {
console.error(error);
return response.status(500);
return response.sendStatus(500);
}
});
@@ -244,7 +244,7 @@ router.post('/props', jsonParser, async function (request, response) {
const propsReply = await fetch(propsUrl, args);
if (!propsReply.ok) {
return response.status(400);
return response.sendStatus(400);
}
/** @type {any} */
@@ -258,7 +258,7 @@ router.post('/props', jsonParser, async function (request, response) {
return response.send(props);
} catch (error) {
console.error(error);
return response.status(500);
return response.sendStatus(500);
}
});
@@ -450,7 +450,7 @@ ollama.post('/download', jsonParser, async function (request, response) {
return response.send({ ok: true });
} catch (error) {
console.error(error);
return response.status(500);
return response.sendStatus(500);
}
});
@@ -493,7 +493,7 @@ ollama.post('/caption-image', jsonParser, async function (request, response) {
return response.send({ caption });
} catch (error) {
console.error(error);
return response.status(500);
return response.sendStatus(500);
}
});
@@ -540,7 +540,7 @@ llamacpp.post('/caption-image', jsonParser, async function (request, response) {
} catch (error) {
console.error(error);
return response.status(500);
return response.sendStatus(500);
}
});
@@ -569,7 +569,7 @@ llamacpp.post('/props', jsonParser, async function (request, response) {
} catch (error) {
console.error(error);
return response.status(500);
return response.sendStatus(500);
}
});
@@ -619,7 +619,7 @@ llamacpp.post('/slots', jsonParser, async function (request, response) {
} catch (error) {
console.error(error);
return response.status(500);
return response.sendStatus(500);
}
});
@@ -665,7 +665,7 @@ tabby.post('/download', jsonParser, async function (request, response) {
return response.send({ ok: true });
} catch (error) {
console.error(error);
return response.status(500);
return response.sendStatus(500);
}
});