2023-07-20 19:32:15 +02:00
import {
getRequestHeaders ,
getStoppingStrings ,
max _context ,
2023-11-07 23:17:13 +01:00
online _status ,
2023-07-20 19:32:15 +02:00
saveSettingsDebounced ,
2023-07-23 22:52:31 +02:00
setGenerationParamsFromPreset ,
2023-11-08 01:52:03 +01:00
setOnlineStatus ,
2023-11-16 15:58:49 +01:00
substituteParams ,
2023-12-02 19:04:51 +01:00
} from '../script.js' ;
2023-12-18 17:57:10 +01:00
import { BIAS _CACHE , createNewLogitBiasEntry , displayLogitBias , getLogitBiasListResult } from './logit-bias.js' ;
2023-07-20 19:32:15 +02:00
2023-08-07 22:46:32 +02:00
import {
power _user ,
2023-11-13 10:13:39 +01:00
registerDebugFunction ,
2023-12-02 19:04:51 +01:00
} from './power-user.js' ;
2023-12-07 04:55:17 +01:00
import EventSourceStream from './sse-stream.js' ;
2023-12-02 19:04:51 +01:00
import { SENTENCEPIECE _TOKENIZERS , getTextTokens , tokenizers } from './tokenizers.js' ;
import { getSortableDelay , onlyUnique } from './utils.js' ;
2023-08-07 22:46:32 +02:00
2023-07-20 19:32:15 +02:00
export {
2023-12-03 16:03:19 +01:00
settings as textgenerationwebui _settings ,
2023-07-20 19:32:15 +02:00
loadTextGenSettings ,
generateTextGenWithStreaming ,
2023-08-03 12:07:54 +02:00
formatTextGenURL ,
2023-12-02 20:11:06 +01:00
} ;
2023-07-20 19:32:15 +02:00
2023-09-28 18:10:00 +02:00
export const textgen _types = {
OOBA : 'ooba' ,
MANCER : 'mancer' ,
APHRODITE : 'aphrodite' ,
2023-11-19 16:14:53 +01:00
TABBY : 'tabby' ,
KOBOLDCPP : 'koboldcpp' ,
2023-12-17 22:38:03 +01:00
TOGETHERAI : 'togetherai' ,
2023-12-18 21:38:28 +01:00
LLAMACPP : 'llamacpp' ,
2023-12-19 15:38:11 +01:00
OLLAMA : 'ollama' ,
2023-09-28 18:10:00 +02:00
} ;
2023-12-19 15:38:11 +01:00
const { MANCER , APHRODITE , TOGETHERAI , OOBA , OLLAMA , LLAMACPP } = textgen _types ;
2023-12-18 17:57:10 +01:00
const BIAS _KEY = '#textgenerationwebui_api-settings' ;
2023-12-03 16:03:19 +01:00
2023-11-07 23:17:13 +01:00
// Maybe let it be configurable in the future?
2023-11-13 10:13:39 +01:00
// (7 days later) The future has come.
const MANCER _SERVER _KEY = 'mancer_server' ;
const MANCER _SERVER _DEFAULT = 'https://neuro.mancer.tech' ;
2023-12-17 22:38:03 +01:00
let MANCER _SERVER = localStorage . getItem ( MANCER _SERVER _KEY ) ? ? MANCER _SERVER _DEFAULT ;
let TOGETHERAI _SERVER = 'https://api.together.xyz' ;
2023-11-07 23:17:13 +01:00
2023-12-19 15:38:11 +01:00
const SERVER _INPUTS = {
[ textgen _types . OOBA ] : '#textgenerationwebui_api_url_text' ,
[ textgen _types . APHRODITE ] : '#aphrodite_api_url_text' ,
[ textgen _types . TABBY ] : '#tabby_api_url_text' ,
[ textgen _types . KOBOLDCPP ] : '#koboldcpp_api_url_text' ,
[ textgen _types . LLAMACPP ] : '#llamacpp_api_url_text' ,
[ textgen _types . OLLAMA ] : '#ollama_api_url_text' ,
} ;
2023-11-21 00:47:57 +01:00
const KOBOLDCPP _ORDER = [ 6 , 0 , 1 , 3 , 4 , 2 , 5 ] ;
2023-12-03 16:03:19 +01:00
const settings = {
2023-07-20 19:32:15 +02:00
temp : 0.7 ,
2023-11-05 15:50:07 +01:00
temperature _last : true ,
2023-07-20 19:32:15 +02:00
top _p : 0.5 ,
top _k : 40 ,
top _a : 0 ,
tfs : 1 ,
epsilon _cutoff : 0 ,
eta _cutoff : 0 ,
typical _p : 1 ,
2023-11-05 15:08:06 +01:00
min _p : 0 ,
2023-07-20 19:32:15 +02:00
rep _pen : 1.2 ,
2023-07-22 00:38:35 +02:00
rep _pen _range : 0 ,
2023-07-20 19:32:15 +02:00
no _repeat _ngram _size : 0 ,
penalty _alpha : 0 ,
num _beams : 1 ,
length _penalty : 1 ,
min _length : 0 ,
encoder _rep _pen : 1 ,
2023-10-25 21:21:49 +02:00
freq _pen : 0 ,
presence _pen : 0 ,
2023-07-20 19:32:15 +02:00
do _sample : true ,
early _stopping : false ,
seed : - 1 ,
preset : 'Default' ,
add _bos _token : true ,
stopping _strings : [ ] ,
truncation _length : 2048 ,
ban _eos _token : false ,
skip _special _tokens : true ,
streaming : false ,
mirostat _mode : 0 ,
mirostat _tau : 5 ,
mirostat _eta : 0.1 ,
2023-08-14 12:06:20 +02:00
guidance _scale : 1 ,
negative _prompt : '' ,
2023-09-27 21:09:09 +02:00
grammar _string : '' ,
banned _tokens : '' ,
2023-11-04 03:16:12 +01:00
//n_aphrodite: 1,
//best_of_aphrodite: 1,
2023-11-13 08:36:01 +01:00
ignore _eos _token _aphrodite : false ,
spaces _between _special _tokens _aphrodite : true ,
2023-11-04 03:16:12 +01:00
//logits_processors_aphrodite: [],
//log_probs_aphrodite: 0,
//prompt_log_probs_aphrodite: 0,
2023-09-28 18:10:00 +02:00
type : textgen _types . OOBA ,
2023-11-07 23:17:13 +01:00
mancer _model : 'mytholite' ,
2023-12-17 22:38:03 +01:00
togetherai _model : 'Gryphe/MythoMax-L2-13b' ,
2023-12-19 15:38:11 +01:00
ollama _model : '' ,
2023-11-08 09:13:28 +01:00
legacy _api : false ,
2023-11-21 00:47:57 +01:00
sampler _order : KOBOLDCPP _ORDER ,
2023-12-18 17:57:10 +01:00
logit _bias : [ ] ,
2023-12-03 19:40:09 +01:00
n : 1 ,
2023-12-19 15:38:11 +01:00
server _urls : { } ,
2023-07-20 19:32:15 +02:00
} ;
2023-10-05 12:10:41 +02:00
export let textgenerationwebui _banned _in _macros = [ ] ;
2023-07-23 22:52:31 +02:00
export let textgenerationwebui _presets = [ ] ;
export let textgenerationwebui _preset _names = [ ] ;
2023-07-20 19:32:15 +02:00
const setting _names = [
2023-12-02 19:04:51 +01:00
'temp' ,
'temperature_last' ,
'rep_pen' ,
'rep_pen_range' ,
'no_repeat_ngram_size' ,
'top_k' ,
'top_p' ,
'top_a' ,
'tfs' ,
'epsilon_cutoff' ,
'eta_cutoff' ,
'typical_p' ,
'min_p' ,
'penalty_alpha' ,
'num_beams' ,
'length_penalty' ,
'min_length' ,
'encoder_rep_pen' ,
'freq_pen' ,
'presence_pen' ,
'do_sample' ,
'early_stopping' ,
'seed' ,
'add_bos_token' ,
'ban_eos_token' ,
'skip_special_tokens' ,
'streaming' ,
'mirostat_mode' ,
'mirostat_tau' ,
'mirostat_eta' ,
'guidance_scale' ,
'negative_prompt' ,
'grammar_string' ,
'banned_tokens' ,
'legacy_api' ,
2023-11-04 03:16:12 +01:00
//'n_aphrodite',
//'best_of_aphrodite',
2023-11-13 08:36:01 +01:00
'ignore_eos_token_aphrodite' ,
'spaces_between_special_tokens_aphrodite' ,
2023-11-04 03:16:12 +01:00
//'logits_processors_aphrodite',
//'log_probs_aphrodite',
//'prompt_log_probs_aphrodite'
2023-12-02 19:04:51 +01:00
'sampler_order' ,
2023-12-03 19:40:09 +01:00
'n' ,
2023-12-18 17:57:10 +01:00
'logit_bias' ,
2023-07-20 19:32:15 +02:00
] ;
2023-12-19 15:38:11 +01:00
export function validateTextGenUrl ( ) {
const selector = SERVER _INPUTS [ settings . type ] ;
if ( ! selector ) {
return ;
}
const control = $ ( selector ) ;
const url = String ( control . val ( ) ) . trim ( ) ;
const formattedUrl = formatTextGenURL ( url ) ;
if ( ! formattedUrl ) {
toastr . error ( 'Enter a valid API URL' , 'Text Completion API' ) ;
return ;
}
control . val ( formattedUrl ) ;
}
export function getTextGenServer ( ) {
if ( settings . type === MANCER ) {
return MANCER _SERVER ;
}
if ( settings . type === TOGETHERAI ) {
return TOGETHERAI _SERVER ;
}
return settings . server _urls [ settings . type ] ? ? '' ;
}
2023-11-05 15:08:06 +01:00
async function selectPreset ( name ) {
2023-07-20 19:32:15 +02:00
const preset = textgenerationwebui _presets [ textgenerationwebui _preset _names . indexOf ( name ) ] ;
if ( ! preset ) {
return ;
}
2023-12-03 16:03:19 +01:00
settings . preset = name ;
2023-07-20 19:32:15 +02:00
for ( const name of setting _names ) {
const value = preset [ name ] ;
setSettingByName ( name , value , true ) ;
}
2023-07-23 22:52:31 +02:00
setGenerationParamsFromPreset ( preset ) ;
2023-12-18 21:38:28 +01:00
BIAS _CACHE . delete ( BIAS _KEY ) ;
2023-12-18 17:57:10 +01:00
displayLogitBias ( preset . logit _bias , BIAS _KEY ) ;
2023-07-20 19:32:15 +02:00
saveSettingsDebounced ( ) ;
}
2023-11-07 23:17:13 +01:00
function formatTextGenURL ( value ) {
2023-08-03 12:07:54 +02:00
try {
2023-12-17 22:38:03 +01:00
// Mancer/Together doesn't need any formatting (it's hardcoded)
if ( settings . type === MANCER || settings . type === TOGETHERAI ) {
2023-11-08 17:16:47 +01:00
return value ;
}
2023-08-03 12:07:54 +02:00
const url = new URL ( value ) ;
2023-12-03 16:03:19 +01:00
if ( url . pathname === '/api' && ! settings . legacy _api ) {
2023-12-02 19:04:51 +01:00
toastr . info ( 'Enable Legacy API or start Ooba with the OpenAI extension enabled.' , 'Legacy API URL detected. Generation may fail.' , { preventDuplicates : true , timeOut : 10000 , extendedTimeOut : 20000 } ) ;
2023-11-08 16:54:08 +01:00
url . pathname = '' ;
2023-08-03 12:07:54 +02:00
}
2023-11-08 09:25:49 +01:00
2023-12-03 16:03:19 +01:00
if ( ! power _user . relaxed _api _urls && settings . legacy _api ) {
2023-11-08 09:25:49 +01:00
url . pathname = '/api' ;
}
2023-08-07 22:46:32 +02:00
return url . toString ( ) ;
2023-12-02 15:14:48 +01:00
} catch {
// Just using URL as a validation check
}
2023-08-03 12:07:54 +02:00
return null ;
}
2023-07-20 19:32:15 +02:00
function convertPresets ( presets ) {
2023-10-05 12:10:41 +02:00
return Array . isArray ( presets ) ? presets . map ( ( p ) => JSON . parse ( p ) ) : [ ] ;
2023-07-20 19:32:15 +02:00
}
2023-09-27 21:09:09 +02:00
/ * *
* @ returns { string } String with comma - separated banned token IDs
* /
function getCustomTokenBans ( ) {
2023-12-03 16:03:19 +01:00
if ( ! settings . banned _tokens && ! textgenerationwebui _banned _in _macros . length ) {
2023-09-27 21:09:09 +02:00
return '' ;
}
2023-11-21 00:04:27 +01:00
const tokenizer = SENTENCEPIECE _TOKENIZERS . includes ( power _user . tokenizer ) ? power _user . tokenizer : tokenizers . LLAMA ;
2023-09-27 21:09:09 +02:00
const result = [ ] ;
2023-12-03 16:03:19 +01:00
const sequences = settings . banned _tokens
2023-10-05 12:10:41 +02:00
. split ( '\n' )
. concat ( textgenerationwebui _banned _in _macros )
. filter ( x => x . length > 0 )
. filter ( onlyUnique ) ;
//debug
if ( textgenerationwebui _banned _in _macros . length ) {
2023-12-02 19:04:51 +01:00
console . log ( '=== Found banned word sequences in the macros:' , textgenerationwebui _banned _in _macros , 'Resulting array of banned sequences (will be used this generation turn):' , sequences ) ;
2023-10-05 12:10:41 +02:00
}
//clean old temporary bans found in macros before, for the next generation turn.
textgenerationwebui _banned _in _macros = [ ] ;
2023-09-27 21:09:09 +02:00
for ( const line of sequences ) {
// Raw token ids, JSON serialized
if ( line . startsWith ( '[' ) && line . endsWith ( ']' ) ) {
try {
const tokens = JSON . parse ( line ) ;
if ( Array . isArray ( tokens ) && tokens . every ( t => Number . isInteger ( t ) ) ) {
result . push ( ... tokens ) ;
} else {
throw new Error ( 'Not an array of integers' ) ;
}
} catch ( err ) {
console . log ( ` Failed to parse bad word token list: ${ line } ` , err ) ;
}
} else {
try {
2023-11-21 00:04:27 +01:00
const tokens = getTextTokens ( tokenizer , line ) ;
2023-09-27 21:09:09 +02:00
result . push ( ... tokens ) ;
} catch {
console . log ( ` Could not tokenize raw text: ${ line } ` ) ;
}
}
}
return result . filter ( onlyUnique ) . map ( x => String ( x ) ) . join ( ',' ) ;
}
2023-12-18 17:57:10 +01:00
/ * *
* Calculates logit bias object from the logit bias list .
* @ returns { object } Logit bias object
* /
function calculateLogitBias ( ) {
if ( ! Array . isArray ( settings . logit _bias ) || settings . logit _bias . length === 0 ) {
return { } ;
}
const tokenizer = SENTENCEPIECE _TOKENIZERS . includes ( power _user . tokenizer ) ? power _user . tokenizer : tokenizers . LLAMA ;
const result = { } ;
/ * *
* Adds bias to the logit bias object .
* @ param { number } bias
* @ param { number [ ] } sequence
* @ returns { object } Accumulated logit bias object
* /
function addBias ( bias , sequence ) {
if ( sequence . length === 0 ) {
return ;
}
for ( const logit of sequence ) {
const key = String ( logit ) ;
result [ key ] = bias ;
}
return result ;
}
getLogitBiasListResult ( settings . logit _bias , tokenizer , addBias ) ;
return result ;
}
2023-12-03 21:44:36 +01:00
function loadTextGenSettings ( data , loadedSettings ) {
2023-07-20 19:32:15 +02:00
textgenerationwebui _presets = convertPresets ( data . textgenerationwebui _presets ) ;
textgenerationwebui _preset _names = data . textgenerationwebui _preset _names ? ? [ ] ;
2023-12-03 21:44:36 +01:00
Object . assign ( settings , loadedSettings . textgenerationwebui _settings ? ? { } ) ;
2023-07-20 19:32:15 +02:00
2023-12-19 15:38:11 +01:00
if ( loadedSettings . api _server _textgenerationwebui ) {
for ( const type of Object . keys ( SERVER _INPUTS ) ) {
settings . server _urls [ type ] = loadedSettings . api _server _textgenerationwebui ;
}
delete loadedSettings . api _server _textgenerationwebui ;
}
for ( const [ type , selector ] of Object . entries ( SERVER _INPUTS ) ) {
const control = $ ( selector ) ;
control . val ( settings . server _urls [ type ] ? ? '' ) . on ( 'input' , function ( ) {
settings . server _urls [ type ] = String ( $ ( this ) . val ( ) ) ;
saveSettingsDebounced ( ) ;
} ) ;
}
2023-12-03 21:44:36 +01:00
if ( loadedSettings . api _use _mancer _webui ) {
2023-12-03 16:03:19 +01:00
settings . type = MANCER ;
2023-09-28 18:10:00 +02:00
}
2023-07-20 19:32:15 +02:00
for ( const name of textgenerationwebui _preset _names ) {
const option = document . createElement ( 'option' ) ;
option . value = name ;
option . innerText = name ;
$ ( '#settings_preset_textgenerationwebui' ) . append ( option ) ;
}
2023-12-03 16:03:19 +01:00
if ( settings . preset ) {
$ ( '#settings_preset_textgenerationwebui' ) . val ( settings . preset ) ;
2023-07-20 19:32:15 +02:00
}
for ( const i of setting _names ) {
2023-12-03 16:03:19 +01:00
const value = settings [ i ] ;
2023-07-20 19:32:15 +02:00
setSettingByName ( i , value ) ;
}
2023-09-28 18:10:00 +02:00
2023-12-03 16:03:19 +01:00
$ ( '#textgen_type' ) . val ( settings . type ) ;
showTypeSpecificControls ( settings . type ) ;
2023-12-18 21:38:28 +01:00
BIAS _CACHE . delete ( BIAS _KEY ) ;
2023-12-18 17:57:10 +01:00
displayLogitBias ( settings . logit _bias , BIAS _KEY ) ;
2023-11-13 08:36:01 +01:00
//this is needed because showTypeSpecificControls() does not handle NOT declarations
2023-12-03 16:03:19 +01:00
if ( settings . type === textgen _types . APHRODITE ) {
2023-11-13 08:36:01 +01:00
$ ( '[data-forAphro=False]' ) . each ( function ( ) {
2023-12-02 20:11:06 +01:00
$ ( this ) . hide ( ) ;
} ) ;
2023-11-13 08:36:01 +01:00
} else {
$ ( '[data-forAphro=False]' ) . each ( function ( ) {
2023-12-02 20:11:06 +01:00
$ ( this ) . show ( ) ;
} ) ;
2023-11-13 08:36:01 +01:00
}
2023-11-13 10:13:39 +01:00
registerDebugFunction ( 'change-mancer-url' , 'Change Mancer base URL' , 'Change Mancer API server base URL' , ( ) => {
const result = prompt ( ` Enter Mancer base URL \n Default: ${ MANCER _SERVER _DEFAULT } ` , MANCER _SERVER ) ;
if ( result ) {
localStorage . setItem ( MANCER _SERVER _KEY , result ) ;
MANCER _SERVER = result ;
}
} ) ;
2023-09-28 18:10:00 +02:00
}
2023-11-21 00:47:57 +01:00
/ * *
* Sorts the sampler items by the given order .
* @ param { any [ ] } orderArray Sampler order array .
* /
function sortItemsByOrder ( orderArray ) {
console . debug ( 'Preset samplers order: ' + orderArray ) ;
2023-12-02 19:04:51 +01:00
const $draggableItems = $ ( '#koboldcpp_order' ) ;
2023-11-21 00:47:57 +01:00
for ( let i = 0 ; i < orderArray . length ; i ++ ) {
const index = orderArray [ i ] ;
const $item = $draggableItems . find ( ` [data-id=" ${ index } "] ` ) . detach ( ) ;
$draggableItems . append ( $item ) ;
}
}
2023-09-28 18:10:00 +02:00
jQuery ( function ( ) {
2023-11-21 00:47:57 +01:00
$ ( '#koboldcpp_order' ) . sortable ( {
delay : getSortableDelay ( ) ,
stop : function ( ) {
const order = [ ] ;
$ ( '#koboldcpp_order' ) . children ( ) . each ( function ( ) {
order . push ( $ ( this ) . data ( 'id' ) ) ;
} ) ;
2023-12-03 16:03:19 +01:00
settings . sampler _order = order ;
console . log ( 'Samplers reordered:' , settings . sampler _order ) ;
2023-11-21 00:47:57 +01:00
saveSettingsDebounced ( ) ;
} ,
} ) ;
$ ( '#koboldcpp_default_order' ) . on ( 'click' , function ( ) {
2023-12-03 16:03:19 +01:00
settings . sampler _order = KOBOLDCPP _ORDER ;
sortItemsByOrder ( settings . sampler _order ) ;
2023-11-21 00:47:57 +01:00
saveSettingsDebounced ( ) ;
} ) ;
2023-09-28 18:10:00 +02:00
$ ( '#textgen_type' ) . on ( 'change' , function ( ) {
const type = String ( $ ( this ) . val ( ) ) ;
2023-12-03 16:03:19 +01:00
settings . type = type ;
2023-09-28 18:10:00 +02:00
2023-12-03 16:03:19 +01:00
if ( settings . type === textgen _types . APHRODITE ) {
2023-11-13 08:36:01 +01:00
//this is needed because showTypeSpecificControls() does not handle NOT declarations
$ ( '[data-forAphro=False]' ) . each ( function ( ) {
2023-12-02 20:11:06 +01:00
$ ( this ) . hide ( ) ;
} ) ;
$ ( '#mirostat_mode_textgenerationwebui' ) . attr ( 'step' , 2 ) ; //Aphro disallows mode 1
$ ( '#do_sample_textgenerationwebui' ) . prop ( 'checked' , true ) ; //Aphro should always do sample; 'otherwise set temp to 0 to mimic no sample'
$ ( '#ban_eos_token_textgenerationwebui' ) . prop ( 'checked' , false ) ; //Aphro should not ban EOS, just ignore it; 'add token '2' to ban list do to this'
2023-11-13 08:36:01 +01:00
//special handling for Aphrodite topK -1 disable state
2023-12-02 20:11:06 +01:00
$ ( '#top_k_textgenerationwebui' ) . attr ( 'min' , - 1 ) ;
2023-12-03 16:03:19 +01:00
if ( $ ( '#top_k_textgenerationwebui' ) . val ( ) === '0' || settings [ 'top_k' ] === 0 ) {
settings [ 'top_k' ] = - 1 ;
2023-12-02 20:11:06 +01:00
$ ( '#top_k_textgenerationwebui' ) . val ( '-1' ) . trigger ( 'input' ) ;
2023-11-13 08:36:01 +01:00
}
} else {
//this is needed because showTypeSpecificControls() does not handle NOT declarations
$ ( '[data-forAphro=False]' ) . each ( function ( ) {
2023-12-02 20:11:06 +01:00
$ ( this ) . show ( ) ;
} ) ;
$ ( '#mirostat_mode_textgenerationwebui' ) . attr ( 'step' , 1 ) ;
2023-11-13 08:36:01 +01:00
//undo special Aphrodite setup for topK
2023-12-02 20:11:06 +01:00
$ ( '#top_k_textgenerationwebui' ) . attr ( 'min' , 0 ) ;
2023-12-03 16:03:19 +01:00
if ( $ ( '#top_k_textgenerationwebui' ) . val ( ) === '-1' || settings [ 'top_k' ] === - 1 ) {
settings [ 'top_k' ] = 0 ;
2023-12-02 20:11:06 +01:00
$ ( '#top_k_textgenerationwebui' ) . val ( '0' ) . trigger ( 'input' ) ;
2023-11-13 08:36:01 +01:00
}
}
2023-11-04 03:16:12 +01:00
2023-11-08 01:52:03 +01:00
showTypeSpecificControls ( type ) ;
setOnlineStatus ( 'no_connection' ) ;
2023-12-18 21:38:28 +01:00
BIAS _CACHE . delete ( BIAS _KEY ) ;
2023-09-28 18:10:00 +02:00
2023-11-08 01:52:03 +01:00
$ ( '#main_api' ) . trigger ( 'change' ) ;
2023-12-19 15:38:11 +01:00
if ( ! SERVER _INPUTS [ type ] || settings . server _urls [ type ] ) {
$ ( '#api_button_textgenerationwebui' ) . trigger ( 'click' ) ;
}
2023-09-28 18:10:00 +02:00
saveSettingsDebounced ( ) ;
} ) ;
2023-09-27 21:09:09 +02:00
$ ( '#settings_preset_textgenerationwebui' ) . on ( 'change' , function ( ) {
2023-07-20 19:32:15 +02:00
const presetName = $ ( this ) . val ( ) ;
selectPreset ( presetName ) ;
} ) ;
for ( const i of setting _names ) {
2023-12-02 19:04:51 +01:00
$ ( ` # ${ i } _textgenerationwebui ` ) . attr ( 'x-setting-id' , i ) ;
$ ( document ) . on ( 'input' , ` # ${ i } _textgenerationwebui ` , function ( ) {
2023-07-20 19:32:15 +02:00
const isCheckbox = $ ( this ) . attr ( 'type' ) == 'checkbox' ;
2023-08-14 12:06:20 +02:00
const isText = $ ( this ) . attr ( 'type' ) == 'text' || $ ( this ) . is ( 'textarea' ) ;
2023-12-02 19:04:51 +01:00
const id = $ ( this ) . attr ( 'x-setting-id' ) ;
2023-07-20 19:32:15 +02:00
if ( isCheckbox ) {
const value = $ ( this ) . prop ( 'checked' ) ;
2023-12-03 16:03:19 +01:00
settings [ id ] = value ;
2023-07-20 19:32:15 +02:00
}
else if ( isText ) {
const value = $ ( this ) . val ( ) ;
2023-12-03 16:03:19 +01:00
settings [ id ] = value ;
2023-07-20 19:32:15 +02:00
}
else {
2023-08-22 18:35:56 +02:00
const value = Number ( $ ( this ) . val ( ) ) ;
2023-10-26 13:41:12 +02:00
$ ( ` # ${ id } _counter_textgenerationwebui ` ) . val ( value ) ;
2023-12-03 16:03:19 +01:00
settings [ id ] = value ;
2023-11-13 08:36:01 +01:00
//special handling for aphrodite using -1 as disabled instead of 0
2023-12-03 08:55:38 +01:00
if ( $ ( this ) . attr ( 'id' ) === 'top_k_textgenerationwebui' &&
2023-12-03 16:03:19 +01:00
settings . type === textgen _types . APHRODITE &&
2023-12-03 08:55:38 +01:00
value === 0 ) {
2023-12-03 16:03:19 +01:00
settings [ id ] = - 1 ;
2023-12-02 20:11:06 +01:00
$ ( this ) . val ( - 1 ) ;
2023-11-13 08:36:01 +01:00
}
2023-07-20 19:32:15 +02:00
}
saveSettingsDebounced ( ) ;
} ) ;
}
2023-12-18 17:57:10 +01:00
$ ( '#textgen_logit_bias_new_entry' ) . on ( 'click' , ( ) => createNewLogitBiasEntry ( settings . logit _bias , BIAS _KEY ) ) ;
2023-12-02 20:11:06 +01:00
} ) ;
2023-07-20 19:32:15 +02:00
2023-11-08 01:52:03 +01:00
function showTypeSpecificControls ( type ) {
$ ( '[data-tg-type]' ) . each ( function ( ) {
2023-12-18 21:38:28 +01:00
const tgTypes = $ ( this ) . attr ( 'data-tg-type' ) . split ( ',' ) ;
for ( const tgType of tgTypes ) {
if ( tgType === type || tgType == 'all' ) {
$ ( this ) . show ( ) ;
return ;
} else {
$ ( this ) . hide ( ) ;
}
2023-11-08 01:52:03 +01:00
}
} ) ;
}
2023-11-21 00:47:57 +01:00
function setSettingByName ( setting , value , trigger ) {
2023-07-20 19:32:15 +02:00
if ( value === null || value === undefined ) {
return ;
}
2023-11-21 00:47:57 +01:00
if ( 'sampler_order' === setting ) {
value = Array . isArray ( value ) ? value : KOBOLDCPP _ORDER ;
sortItemsByOrder ( value ) ;
2023-12-03 16:03:19 +01:00
settings . sampler _order = value ;
2023-11-21 00:47:57 +01:00
return ;
}
2023-12-18 17:57:10 +01:00
if ( 'logit_bias' === setting ) {
settings . logit _bias = Array . isArray ( value ) ? value : [ ] ;
return ;
}
2023-11-21 00:47:57 +01:00
const isCheckbox = $ ( ` # ${ setting } _textgenerationwebui ` ) . attr ( 'type' ) == 'checkbox' ;
const isText = $ ( ` # ${ setting } _textgenerationwebui ` ) . attr ( 'type' ) == 'text' || $ ( ` # ${ setting } _textgenerationwebui ` ) . is ( 'textarea' ) ;
2023-07-20 19:32:15 +02:00
if ( isCheckbox ) {
const val = Boolean ( value ) ;
2023-11-21 00:47:57 +01:00
$ ( ` # ${ setting } _textgenerationwebui ` ) . prop ( 'checked' , val ) ;
2023-07-20 19:32:15 +02:00
}
else if ( isText ) {
2023-11-21 00:47:57 +01:00
$ ( ` # ${ setting } _textgenerationwebui ` ) . val ( value ) ;
2023-07-20 19:32:15 +02:00
}
else {
const val = parseFloat ( value ) ;
2023-11-21 00:47:57 +01:00
$ ( ` # ${ setting } _textgenerationwebui ` ) . val ( val ) ;
$ ( ` # ${ setting } _counter_textgenerationwebui ` ) . val ( val ) ;
2023-11-05 15:08:06 +01:00
if ( power _user . enableZenSliders ) {
2023-12-02 20:11:06 +01:00
let zenSlider = $ ( ` # ${ setting } _textgenerationwebui_zenslider ` ) . slider ( ) ;
zenSlider . slider ( 'option' , 'value' , val ) ;
2023-11-05 15:08:06 +01:00
zenSlider . slider ( 'option' , 'slide' )
. call ( zenSlider , null , {
2023-12-02 21:06:57 +01:00
handle : $ ( '.ui-slider-handle' , zenSlider ) , value : val ,
2023-11-05 15:08:06 +01:00
} ) ;
}
2023-07-20 19:32:15 +02:00
}
if ( trigger ) {
2023-11-21 00:47:57 +01:00
$ ( ` # ${ setting } _textgenerationwebui ` ) . trigger ( 'input' ) ;
2023-07-20 19:32:15 +02:00
}
}
async function generateTextGenWithStreaming ( generate _data , signal ) {
2023-11-07 23:17:13 +01:00
generate _data . stream = true ;
2023-10-08 22:42:28 +02:00
2023-12-12 05:13:26 +01:00
const response = await fetch ( '/api/backends/text-completions/generate' , {
2023-07-20 19:32:15 +02:00
headers : {
... getRequestHeaders ( ) ,
} ,
body : JSON . stringify ( generate _data ) ,
method : 'POST' ,
signal : signal ,
} ) ;
2023-12-07 17:02:39 +01:00
if ( ! response . ok ) {
2023-12-08 01:01:08 +01:00
tryParseStreamingError ( response , await response . text ( ) ) ;
2023-12-07 17:02:39 +01:00
throw new Error ( ` Got response status ${ response . status } ` ) ;
}
2023-12-07 04:55:17 +01:00
const eventStream = new EventSourceStream ( ) ;
response . body . pipeThrough ( eventStream ) ;
const reader = eventStream . readable . getReader ( ) ;
2023-07-20 19:32:15 +02:00
return async function * streamData ( ) {
2023-12-07 04:55:17 +01:00
let text = '' ;
2023-12-03 19:40:09 +01:00
const swipes = [ ] ;
2023-07-20 19:32:15 +02:00
while ( true ) {
const { done , value } = await reader . read ( ) ;
2023-12-07 04:55:17 +01:00
if ( done ) return ;
if ( value . data === '[DONE]' ) return ;
2023-07-20 19:32:15 +02:00
2023-12-07 04:55:17 +01:00
tryParseStreamingError ( response , value . data ) ;
2023-07-20 19:32:15 +02:00
2023-12-07 04:55:17 +01:00
let data = JSON . parse ( value . data ) ;
2023-09-28 18:10:00 +02:00
2023-12-18 21:38:28 +01:00
if ( data ? . choices ? . [ 0 ] ? . index > 0 ) {
2023-12-07 04:55:17 +01:00
const swipeIndex = data . choices [ 0 ] . index - 1 ;
swipes [ swipeIndex ] = ( swipes [ swipeIndex ] || '' ) + data . choices [ 0 ] . text ;
} else {
2023-12-18 21:38:28 +01:00
text += data ? . choices ? . [ 0 ] ? . text || data ? . content || '' ;
2023-09-28 18:10:00 +02:00
}
2023-11-07 23:17:13 +01:00
2023-12-07 04:55:17 +01:00
yield { text , swipes } ;
2023-07-20 19:32:15 +02:00
}
2023-12-02 20:11:06 +01:00
} ;
2023-07-20 19:32:15 +02:00
}
2023-11-07 23:17:13 +01:00
/ * *
* Parses errors in streaming responses and displays them in toastr .
2023-12-08 01:01:08 +01:00
* @ param { Response } response - Response from the server .
* @ param { string } decoded - Decoded response body .
2023-11-07 23:17:13 +01:00
* @ returns { void } Nothing .
* /
2023-12-08 01:01:08 +01:00
function tryParseStreamingError ( response , decoded ) {
2023-11-07 23:17:13 +01:00
let data = { } ;
try {
2023-12-08 01:01:08 +01:00
data = JSON . parse ( decoded ) ;
2023-11-07 23:17:13 +01:00
} catch {
// No JSON. Do nothing.
}
2023-11-07 23:25:06 +01:00
const message = data ? . error ? . message || data ? . message ;
if ( message ) {
2023-12-08 01:01:08 +01:00
toastr . error ( message , 'Text Completion API' ) ;
2023-11-07 23:25:06 +01:00
throw new Error ( message ) ;
2023-11-07 23:17:13 +01:00
}
}
2023-12-18 21:38:28 +01:00
/ * *
* Converts a string of comma - separated integers to an array of integers .
* @ param { string } string Input string
* @ returns { number [ ] } Array of integers
* /
2023-11-07 23:17:13 +01:00
function toIntArray ( string ) {
if ( ! string ) {
return [ ] ;
}
return string . split ( ',' ) . map ( x => parseInt ( x ) ) . filter ( x => ! isNaN ( x ) ) ;
}
function getModel ( ) {
2023-12-03 16:03:19 +01:00
if ( settings . type === MANCER ) {
return settings . mancer _model ;
2023-11-07 23:17:13 +01:00
}
2023-12-17 22:38:03 +01:00
if ( settings . type === TOGETHERAI ) {
return settings . togetherai _model ;
}
2023-12-03 16:03:19 +01:00
if ( settings . type === APHRODITE ) {
2023-11-07 23:17:13 +01:00
return online _status ;
}
2023-12-19 15:38:11 +01:00
if ( settings . type === OLLAMA ) {
if ( ! settings . ollama _model ) {
toastr . error ( 'No Ollama model selected.' , 'Text Completion API' ) ;
throw new Error ( 'No Ollama model selected' ) ;
}
2023-12-17 22:38:03 +01:00
2023-12-19 15:38:11 +01:00
return settings . ollama _model ;
2023-12-17 22:38:03 +01:00
}
2023-12-19 15:38:11 +01:00
return undefined ;
2023-12-17 22:38:03 +01:00
}
2023-12-03 19:56:25 +01:00
export function getTextGenGenerationData ( finalPrompt , maxTokens , isImpersonate , isContinue , cfgValues , type ) {
const canMultiSwipe = ! isContinue && ! isImpersonate && type !== 'quiet' ;
2023-12-18 21:38:28 +01:00
let params = {
2023-08-23 16:44:38 +02:00
'prompt' : finalPrompt ,
2023-11-07 23:17:13 +01:00
'model' : getModel ( ) ,
2023-11-22 15:16:48 +01:00
'max_new_tokens' : maxTokens ,
'max_tokens' : maxTokens ,
2023-12-03 16:03:19 +01:00
'temperature' : settings . temp ,
'top_p' : settings . top _p ,
'typical_p' : settings . typical _p ,
'min_p' : settings . min _p ,
'repetition_penalty' : settings . rep _pen ,
'frequency_penalty' : settings . freq _pen ,
'presence_penalty' : settings . presence _pen ,
'top_k' : settings . top _k ,
'min_length' : settings . min _length ,
'min_tokens' : settings . min _length ,
'num_beams' : settings . num _beams ,
'length_penalty' : settings . length _penalty ,
'early_stopping' : settings . early _stopping ,
'add_bos_token' : settings . add _bos _token ,
2023-11-22 15:16:48 +01:00
'stopping_strings' : getStoppingStrings ( isImpersonate , isContinue ) ,
'stop' : getStoppingStrings ( isImpersonate , isContinue ) ,
2023-07-20 19:32:15 +02:00
'truncation_length' : max _context ,
2023-12-03 16:03:19 +01:00
'ban_eos_token' : settings . ban _eos _token ,
'skip_special_tokens' : settings . skip _special _tokens ,
'top_a' : settings . top _a ,
'tfs' : settings . tfs ,
'epsilon_cutoff' : settings . epsilon _cutoff ,
'eta_cutoff' : settings . eta _cutoff ,
'mirostat_mode' : settings . mirostat _mode ,
'mirostat_tau' : settings . mirostat _tau ,
'mirostat_eta' : settings . mirostat _eta ,
'custom_token_bans' : settings . type === textgen _types . APHRODITE ?
2023-12-03 08:55:38 +01:00
toIntArray ( getCustomTokenBans ( ) ) :
getCustomTokenBans ( ) ,
2023-12-03 16:03:19 +01:00
'api_type' : settings . type ,
2023-12-17 22:38:03 +01:00
'api_server' : getTextGenServer ( ) ,
2023-12-18 21:38:28 +01:00
'legacy_api' : settings . legacy _api && ( settings . type === OOBA || settings . type === APHRODITE ) ,
'sampler_order' : settings . type === textgen _types . KOBOLDCPP ? settings . sampler _order : undefined ,
2023-11-13 08:36:01 +01:00
} ;
2023-12-18 21:38:28 +01:00
const nonAphroditeParams = {
2023-12-03 16:03:19 +01:00
'repetition_penalty_range' : settings . rep _pen _range ,
'encoder_repetition_penalty' : settings . encoder _rep _pen ,
'no_repeat_ngram_size' : settings . no _repeat _ngram _size ,
'penalty_alpha' : settings . penalty _alpha ,
'temperature_last' : settings . temperature _last ,
'do_sample' : settings . do _sample ,
'seed' : settings . seed ,
'guidance_scale' : cfgValues ? . guidanceScale ? . value ? ? settings . guidance _scale ? ? 1 ,
'negative_prompt' : cfgValues ? . negativePrompt ? ? substituteParams ( settings . negative _prompt ) ? ? '' ,
'grammar_string' : settings . grammar _string ,
2023-12-19 15:38:11 +01:00
// llama.cpp aliases. In case someone wants to use LM Studio as Text Completion API
'repeat_penalty' : settings . rep _pen ,
'tfs_z' : settings . tfs ,
'repeat_last_n' : settings . rep _pen _range ,
'n_predict' : settings . maxTokens ,
'mirostat' : settings . mirostat _mode ,
'ignore_eos' : settings . ban _eos _token ,
2023-12-02 20:11:06 +01:00
} ;
2023-12-18 21:38:28 +01:00
const aphroditeParams = {
2023-12-03 22:24:39 +01:00
'n' : canMultiSwipe ? settings . n : 1 ,
'best_of' : canMultiSwipe ? settings . n : 1 ,
2023-12-03 16:03:19 +01:00
'ignore_eos' : settings . ignore _eos _token _aphrodite ,
'spaces_between_special_tokens' : settings . spaces _between _special _tokens _aphrodite ,
2023-12-03 22:24:39 +01:00
//'logits_processors': settings.logits_processors_aphrodite,
//'logprobs': settings.log_probs_aphrodite,
//'prompt_logprobs': settings.prompt_log_probs_aphrodite,
2023-12-02 20:11:06 +01:00
} ;
2023-12-19 15:38:11 +01:00
if ( settings . type === APHRODITE ) {
2023-12-18 21:38:28 +01:00
params = Object . assign ( params , aphroditeParams ) ;
2023-11-13 08:36:01 +01:00
} else {
2023-12-18 21:38:28 +01:00
params = Object . assign ( params , nonAphroditeParams ) ;
2023-11-13 08:36:01 +01:00
}
2023-12-18 17:57:10 +01:00
if ( Array . isArray ( settings . logit _bias ) && settings . logit _bias . length ) {
const logitBias = BIAS _CACHE . get ( BIAS _KEY ) || calculateLogitBias ( ) ;
BIAS _CACHE . set ( BIAS _KEY , logitBias ) ;
2023-12-18 21:38:28 +01:00
params . logit _bias = logitBias ;
}
2023-12-19 15:38:11 +01:00
if ( settings . type === LLAMACPP || settings . type === OLLAMA ) {
2023-12-18 21:38:28 +01:00
// Convert bias and token bans to array of arrays
const logitBiasArray = ( params . logit _bias && typeof params . logit _bias === 'object' && Object . keys ( params . logit _bias ) . length > 0 )
? Object . entries ( params . logit _bias ) . map ( ( [ key , value ] ) => [ Number ( key ) , value ] )
: [ ] ;
const tokenBans = toIntArray ( getCustomTokenBans ( ) ) ;
logitBiasArray . push ( ... tokenBans . map ( x => [ Number ( x ) , false ] ) ) ;
const llamaCppParams = {
'logit_bias' : logitBiasArray ,
2023-12-19 15:38:11 +01:00
// Conflicts with ooba's grammar_string
'grammar' : settings . grammar _string ,
2023-12-25 01:42:03 +01:00
'cache_prompt' : true ,
2023-12-18 21:38:28 +01:00
} ;
params = Object . assign ( params , llamaCppParams ) ;
}
return params ;
2023-07-20 19:32:15 +02:00
}
2023-11-07 23:17:13 +01:00