Merge pull request #3475 from SillyTavern/ollama-batch

Ollama: Add num_batch config value

src/constants.js

@@ -304,6 +304,7 @@ export const TOGETHERAI_KEYS = [
 export const OLLAMA_KEYS = [
     'num_predict',
     'num_ctx',
+    'num_batch',
     'stop',
     'temperature',
     'repeat_penalty',

src/endpoints/backends/text-completions.js

@@ -373,6 +373,10 @@ router.post('/generate', jsonParser, async function (request, response) {
 
         if (request.body.api_type === TEXTGEN_TYPES.OLLAMA) {
             const keepAlive = getConfigValue('ollama.keepAlive', -1);
+            const numBatch = getConfigValue('ollama.batchSize', -1);
+            if (numBatch > 0) {
+                request.body['num_batch'] = numBatch;
+            }
             args.body = JSON.stringify({
                 model: request.body.model,
                 prompt: request.body.prompt,