diff --git a/package-lock.json b/package-lock.json index 4e8300d82..38862ee3b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,6 @@ "@agnai/sentencepiece-js": "^1.1.1", "@agnai/web-tokenizers": "^0.1.3", "@dqbd/tiktoken": "^1.0.2", - "axios": "^1.4.0", "command-exists": "^1.2.9", "compression": "^1", "cookie-parser": "^1.4.6", @@ -768,11 +767,6 @@ "node": ">=8" } }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" - }, "node_modules/at-least-node": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", @@ -782,16 +776,6 @@ "node": ">= 4.0.0" } }, - "node_modules/axios": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz", - "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==", - "dependencies": { - "follow-redirects": "^1.15.0", - "form-data": "^4.0.0", - "proxy-from-env": "^1.1.0" - } - }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -1010,17 +994,6 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/command-exists": { "version": "1.2.9", "resolved": "https://registry.npmjs.org/command-exists/-/command-exists-1.2.9.tgz", @@ -1204,14 +1177,6 @@ "node": ">=8" } }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -1480,38 +1445,6 @@ "node": ">= 0.8" } }, - "node_modules/follow-redirects": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", - "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, - "node_modules/form-data": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", - "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -2717,11 +2650,6 @@ "node": ">= 0.10" } }, - "node_modules/proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" - }, "node_modules/pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", diff --git a/package.json b/package.json index f3a1019e9..86b5a33d7 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,6 @@ "@agnai/sentencepiece-js": "^1.1.1", "@agnai/web-tokenizers": "^0.1.3", "@dqbd/tiktoken": "^1.0.2", - "axios": "^1.4.0", "command-exists": "^1.2.9", "compression": "^1", "cookie-parser": "^1.4.6", diff --git a/public/index.html b/public/index.html index 2e2fc6671..e2a8437d6 100644 --- a/public/index.html +++ b/public/index.html @@ -2120,9 +2120,9 @@
- - - + + +
@@ -2303,7 +2303,7 @@ Input Sequence
- +
@@ -2311,7 +2311,7 @@ Output Sequence
- +
@@ -2321,7 +2321,7 @@ First Output Sequence
- +
@@ -2329,7 +2329,7 @@ Last Output Sequence
- +
@@ -2339,7 +2339,7 @@ System Sequence Prefix
- +
@@ -2347,7 +2347,7 @@ System Sequence Suffix
- +
@@ -2357,7 +2357,7 @@ Stop Sequence
- +
@@ -2365,7 +2365,7 @@ Separator
- +
diff --git a/server.js b/server.js index ca72f999f..fee94db2c 100644 --- a/server.js +++ b/server.js @@ -32,7 +32,6 @@ const multer = require("multer"); const responseTime = require('response-time'); // net related library imports -const axios = require('axios'); const DeviceDetector = require("device-detector-js"); const fetch = require('node-fetch').default; const ipaddr = require('ipaddr.js'); @@ -3580,15 +3579,15 @@ app.post("/generate_openai", jsonParser, function (request, response_generate_op controller.abort(); }); + /** @type {import('node-fetch').RequestInit} */ const config = { method: 'post', - url: endpointUrl, headers: { 'Content-Type': 'application/json', 'Authorization': 'Bearer ' + api_key_openai, ...headers, }, - data: { + body: JSON.stringify({ "messages": isTextCompletion === false ? request.body.messages : undefined, "prompt": isTextCompletion === true ? textPrompt : undefined, "model": request.body.model, @@ -3602,75 +3601,62 @@ app.post("/generate_openai", jsonParser, function (request, response_generate_op "stop": request.body.stop, "logit_bias": request.body.logit_bias, ...bodyParams, - }, + }), signal: controller.signal, + timeout: 0, }; - console.log(config.data); + console.log(JSON.parse(String(config.body))); - if (request.body.stream) { - config.responseType = 'stream'; - } + makeRequest(config, response_generate_openai, request); + /** + * + * @param {*} config + * @param {express.Response} response_generate_openai + * @param {express.Request} request + * @param {Number} retries + * @param {Number} timeout + */ async function makeRequest(config, response_generate_openai, request, retries = 5, timeout = 5000) { try { - // @ts-ignore - axios typings are wrong, this is actually callable https://github.com/axios/axios/issues/5213 - const response = await axios(config); + const fetchResponse = await fetch(endpointUrl, config) - if (response.status <= 299) { + if (fetchResponse.ok) { if (request.body.stream) { console.log('Streaming request in progress'); - response.data.pipe(response_generate_openai); - response.data.on('end', () => { + fetchResponse.body.pipe(response_generate_openai); + fetchResponse.body.on('end', () => { console.log('Streaming request finished'); response_generate_openai.end(); }); } else { - response_generate_openai.send(response.data); - console.log(response.data); - console.log(response.data?.choices[0]?.message); + let json = await fetchResponse.json() + response_generate_openai.send(json); + console.log(json); + console.log(json?.choices[0]?.message); } - } else { - handleErrorResponse(response, response_generate_openai, request); - } - } catch (error) { - if (error.response && error.response.status === 429 && retries > 0) { + } else if (fetchResponse.status === 429 && retries > 0) { console.log(`Out of quota, retrying in ${Math.round(timeout / 1000)}s`); setTimeout(() => { makeRequest(config, response_generate_openai, request, retries - 1); }, timeout); } else { - let errorData = error?.response?.data; - - if (request.body.stream) { - try { - const chunks = await readAllChunks(errorData); - const blob = new Blob(chunks, { type: 'application/json' }); - const text = await blob.text(); - errorData = JSON.parse(text); - } catch { - console.warn('Error parsing streaming response'); - } - } else { - errorData = typeof errorData === 'string' ? tryParse(errorData) : errorData; - } - - handleError(error, response_generate_openai, errorData); + await handleErrorResponse(fetchResponse); + } + } catch (error) { + console.log('Generation failed', error); + if (!response_generate_openai.headersSent) { + response_generate_openai.send({ error: true }); + } else { + response_generate_openai.end(); } } } - function handleErrorResponse(response, response_generate_openai, request) { - if (response.status >= 400 && response.status <= 504) { - console.log('Error occurred:', response.status, response.data); - response_generate_openai.send({ error: true }); - } - } - - function handleError(error, response_generate_openai, errorData) { - console.error('Error:', error?.message); - - let message = error?.response?.statusText; + async function handleErrorResponse(response) { + const responseText = await response.text(); + const errorData = tryParse(responseText); const statusMessages = { 400: 'Bad request', @@ -3680,24 +3666,21 @@ app.post("/generate_openai", jsonParser, function (request, response_generate_op 404: 'Not found', 429: 'Too many requests', 451: 'Unavailable for legal reasons', + 502: 'Bad gateway', }; - const status = error?.response?.status; - if (statusMessages.hasOwnProperty(status)) { - message = errorData?.error?.message || statusMessages[status]; - console.log(message); - } + const message = errorData?.error?.message || statusMessages[response.status] || 'Unknown error occurred'; + const quota_error = response.status === 429 && errorData?.error?.type === 'insufficient_quota'; + console.log(message); - const quota_error = error?.response?.status === 429 && errorData?.error?.type === 'insufficient_quota'; - const response = { error: { message }, quota_error: quota_error } if (!response_generate_openai.headersSent) { - response_generate_openai.send(response); + response_generate_openai.send({ error: { message }, quota_error: quota_error }); } else if (!response_generate_openai.writableEnded) { response_generate_openai.write(response); + } else { + response_generate_openai.end(); } } - - makeRequest(config, response_generate_openai, request); }); app.post("/tokenize_openai", jsonParser, function (request, response_tokenize_openai) {