Change endpoint from persons to people

This commit is contained in:
xfarrow
2025-03-23 21:00:08 +01:00
parent 4ae263662c
commit d005193f63
7158 changed files with 700476 additions and 735 deletions

125
backend/apis/nodejs/node_modules/knex/scripts/build.js generated vendored Normal file
View File

@ -0,0 +1,125 @@
#!/usr/bin/env node
const fs = require('fs');
const path = require('path');
const child_process = require('child_process');
const _ = require('lodash');
const exec = function (cmd, args) {
return new Promise(function (resolve, reject) {
// Execute command
const child = child_process.exec(cmd, {
cwd: process.cwd(),
env: process.env,
});
// Pass stdout and stderr
child.stdout.on('data', function (data) {
process.stdout.write(data.toString());
});
child.stderr.on('data', function (data) {
process.stderr.write(data.toString());
});
// Handle result
child.on('exit', function (code) {
if (code) reject(code);
else resolve();
});
child.on('error', reject);
});
};
const CWD = process.cwd();
const POSTINSTALL_BUILD_CWD = process.env.POSTINSTALL_BUILD_CWD;
// If we didn't have this check, then we'd be stuck in an infinite `postinstall`
// loop, since we run `npm install --only=dev` below, triggering another
// `postinstall`. We can't use `--ignore-scripts` because that ignores scripts
// on all the modules that get installed, too, which would break stuff. So
// instead, we set an environment variable, `POSTINSTALL_BUILD_CWD`, that keeps
// track of what we're installing. It's more than just a yes/no flag because
// the dev dependencies we're installing might use `postinstall-build` too, and
// we don't want the flag to prevent them from running.
if (POSTINSTALL_BUILD_CWD !== CWD) {
const BUILD_ARTIFACT = process.argv[2];
const BUILD_COMMAND = process.argv[3];
fs.stat(BUILD_ARTIFACT, function (err, stats) {
if (err || !(stats.isFile() || stats.isDirectory())) {
// This script will run again after we run `npm install` below. Set an
// environment variable to tell it to skip the check. Really we just want
// the execSync's `env` to be modified, but it's easier just modify and
// pass along the entire `process.env`.
process.env.POSTINSTALL_BUILD_CWD = CWD;
// We already have prod dependencies, that's what triggered `postinstall`
// in the first place. So only install dev.
// Fetch package.json
const pkgJson = require(path.join(CWD, 'package.json'));
const devDeps = pkgJson.devDependencies;
// Values listed under `buildDependencies` contain the dependency names
// that are required for `lib` building.
const buildDependencies = _.pick(devDeps, pkgJson.buildDependencies);
// Proceed only if there is something to install
if (!_.isEmpty(buildDependencies)) {
const opts = { env: process.env, stdio: 'inherit' };
console.log('Building Knex.js');
// Map all key (dependency) value (semver) pairs to
// "dependency@semver dependency@semver ..." string that can be used
// for `npm install` command
const installArgs = _(buildDependencies)
.pickBy(function (semver, dep) {
// Check if the dependency is already installed
try {
require(dep);
return false;
} catch (err) {
return true;
}
})
.map(function (semver, dep) {
// Format installable dependencies
return dep + '@' + semver;
})
.value()
.join(' ');
const needsDepInstallation = !_.isEmpty(installArgs);
const dependenciesInstalledQ = needsDepInstallation
? exec('npm install ' + installArgs, opts)
: Promise.resolve();
dependenciesInstalledQ
.then(function () {
console.log('✓');
// Don't need the flag anymore as `postinstall` was already run.
// Change it back so the environment is minimally changed for the
// remaining commands.
process.env.POSTINSTALL_BUILD_CWD = POSTINSTALL_BUILD_CWD;
console.log('Building compiled files (' + BUILD_COMMAND + ')');
return exec(BUILD_COMMAND, opts);
})
.catch(function (err) {
console.error(err);
process.exit(1);
})
.then(function () {
if (process.env.NODE_ENV === 'production') {
console.log('✓');
console.log('Pruning dev dependencies for production build');
return exec('npm prune --production', opts);
} else {
console.log('Skipping npm prune');
}
})
.then(function () {
console.log('✓');
})
.catch(function (err) {
console.error(err);
process.exit(1);
});
}
}
});
}

31
backend/apis/nodejs/node_modules/knex/scripts/clean.js generated vendored Normal file
View File

@ -0,0 +1,31 @@
#!/usr/bin/env node
const fs = require('fs');
const path = require('path');
const { execSync } = require('child_process');
function main() {
const repoDir = path.dirname(__dirname);
const gitDir = path.join(repoDir, '.git');
const gitDirExists = doesDirectoryExist(gitDir);
if (!gitDirExists) {
console.log("No .git directory detected so can not clean 'lib/'. Exiting.");
process.exit(0);
}
console.log(
"Cleaning 'lib/' of outputted files from Typescript compilation ..."
);
const cmd = 'git clean -f -X lib/';
const output = execSync(cmd, { cwd: repoDir });
console.log(output.toString('utf8'));
console.log('Done');
}
function doesDirectoryExist(p) {
if (fs.existsSync(p)) {
return fs.lstatSync(p).isDirectory();
}
return false;
}
main();

View File

@ -0,0 +1,152 @@
version: '3'
services:
mssql:
image: mcr.microsoft.com/mssql/server:2019-latest
ports:
- '21433:1433'
environment:
- ACCEPT_EULA=Y
- SA_PASSWORD=S0meVeryHardPassword
healthcheck:
test: /opt/mssql-tools/bin/sqlcmd -S mssql -U sa -P 'S0meVeryHardPassword' -Q 'select 1'
waitmssql:
image: mcr.microsoft.com/mssql/server:2017-latest
links:
- mssql
depends_on:
- mssql
entrypoint:
- bash
- -c
# https://docs.microsoft.com/en-us/sql/relational-databases/logs/control-transaction-durability?view=sql-server-ver15#bkmk_DbControl
- 'until /opt/mssql-tools/bin/sqlcmd -S mssql -U sa -P S0meVeryHardPassword -d master -Q "CREATE DATABASE knex_test; ALTER DATABASE knex_test SET ALLOW_SNAPSHOT_ISOLATION ON; ALTER DATABASE knex_test SET DELAYED_DURABILITY = FORCED"; do sleep 5; done'
mysql:
image: mysql
# https://dev.mysql.com/doc/refman/8.0/en/mysql-acid.html
# https://dev.mysql.com/doc/refman/8.0/en/innodb-parameters.html#sysvar_innodb_flush_method
# nosync only for unix, edit for local start on other systems
command: --default-authentication-plugin=mysql_native_password --sync_binlog=0 --innodb_doublewrite=OFF --innodb-flush-log-at-trx-commit=0 --innodb-flush-method=nosync
ports:
- '23306:3306'
environment:
- MYSQL_ROOT_PASSWORD=testrootpassword
- MYSQL_DATABASE=knex_test
- MYSQL_USER=testuser
- MYSQL_PASSWORD=testpassword
healthcheck:
test:
[
'CMD',
'/usr/bin/mysql',
'-hlocalhost',
'-utestuser',
'-ptestpassword',
'-e',
'SELECT 1',
]
interval: 30s
timeout: 5s
retries: 3
restart: always
volumes:
- mysql_data:/var/lib/mysql
waitmysql:
image: mysql
links:
- mysql
depends_on:
- mysql
entrypoint:
- bash
- -c
- 'until /usr/bin/mysql -hmysql -utestuser -ptestpassword -e "SELECT 1"; do sleep 5; done'
postgres:
image: postgres:13-alpine
# see https://www.postgresql.org/docs/current/non-durability.html
command: '-c full_page_writes=off -c fsync=off -c synchronous_commit=off'
ports:
- '25432:5432'
environment:
- POSTGRES_USER=testuser
- POSTGRES_PASSWORD=knextest
- POSTGRES_DB=knex_test
waitpostgres:
image: postgres:13-alpine
links:
- postgres
depends_on:
- postgres
entrypoint:
- bash
- -c
- 'until /usr/local/bin/psql postgres://testuser:knextest@postgres/knex_test -c "SELECT 1"; do sleep 5; done'
cockroachdb:
image: cockroachdb/cockroach:latest-v21.2
container_name: crdb
hostname: crdb
command: start-single-node --cluster-name=example-single-node --insecure
ports:
- '26257:26257'
- '8080:8080'
waitcockroachdb:
container_name: crdb-init
hostname: crdb-init
image: timveil/cockroachdb-remote-client:latest
environment:
- COCKROACH_HOST=crdb:26257
- COCKROACH_INSECURE=true
- DATABASE_NAME=test
depends_on:
- cockroachdb
pgnative:
image: postgres:13-alpine
# see https://www.postgresql.org/docs/current/non-durability.html
command: '-c full_page_writes=off -c fsync=off -c synchronous_commit=off'
ports:
- '25433:5432'
environment:
- POSTGRES_USER=testuser
- POSTGRES_PASSWORD=knextest
- POSTGRES_DB=knex_test
waitpgnative:
image: postgres:13-alpine
links:
- pgnative
depends_on:
- pgnative
entrypoint:
- bash
- -c
- 'until /usr/local/bin/psql postgres://testuser:knextest@pgnative/knex_test -c "SELECT 1"; do sleep 5; done'
oracledb:
image: quillbuilduser/oracle-18-xe
container_name: oracledb_container
ports:
- '21521:1521'
environment:
- ORACLE_ALLOW_REMOTE=true
waitoracledb:
image: quillbuilduser/oracle-18-xe
links:
- oracledb
depends_on:
- oracledb
environment:
- ORACLE_HOME=/opt/oracle/product/18c/dbhomeXE
entrypoint:
- bash
- -c
- 'until /opt/oracle/product/18c/dbhomeXE/bin/sqlplus -s sys/Oracle18@oracledb/XE as sysdba <<< "SELECT 13376411 FROM DUAL; exit;" | grep "13376411"; do echo "Could not connect to oracle... sleep for a while"; sleep 5; done'
volumes:
mysql_data:
driver_opts:
type: tmpfs
device: tmpfs

View File

@ -0,0 +1,24 @@
# Checklist for crating knex @next releases
1. Go through all commits since the last release and add them to CHANGELOG.md under unreleased changes section.
2. Commit changes to CHANGELOG
3. Check that master compiles and tests are running fine (check also that CI tests are passing)
```sh
npm run build
# run bunch of tests, but skipping coverage which doesn't really work locally at least
npm plaintest
npm bin_test
npm oracledb:test
npm mssql:init
npm mssql:test
npm mssql:destroy
```
4. Update package.json version to be e.g. 0.16.0-next1 or 0.16.0-next2 and commit yo master
5. Publish it under @next tag
```sh
npm publish --tag next
```

View File

@ -0,0 +1,82 @@
#!/usr/bin/env bash
# Exit on error
set -e
# Directory constants
repo_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." >/dev/null 2>&1 && pwd )"
exec_dir="$( pwd )"
script_dir="$repo_dir/scripts/"
docker_compose_file="$repo_dir/scripts/docker-compose.yml"
help_text="
Helper script to install oracle drivers on local linux machine from Oracle
database container.
oracledb-install-driver-libs.sh COMMAND
COMMAND:
run: Do the driver install.
dry-run: Do the driver install but do not save any files.
help: Print this menu.
NOTES FOR USAGE:
1. This script is tested to work on Ubuntu 18.04 LTS.
2. This script requires you to have sudo capabilities so to use ldconfig.
"
# Main script logic
cmd="$1"
function main () {
case "$1" in
"run")
printf "Starting run ...\n"
do_install true
exit 0
;;
"dry-run")
printf "Starting dry-run ...\n"
do_install false
exit 0
;;
"help"|"--help"|"-h"|"")
printf "$help_text"
exit 0
;;
*)
printf "Unsupported command: $cmd\n"
printf "Try running with 'help' to see supported commands.\n"
exit 1
;;
esac
}
function do_install () {
do_changes="$1"
printf "\nEnsuring oracle containers from docker-compose are up ...\n"
docker-compose -f "$docker_compose_file" up --build -d oracledb
docker-compose -f "$docker_compose_file" up waitoracledb
printf "\nSleeping an extra 15 seconds to ensure oracle has fully started ...\n"
sleep 15
printf "\nInstalling oracle client libs to db container ...\n"
set -x
docker-compose -f "$docker_compose_file" exec -T oracledb curl http://yum.oracle.com/public-yum-ol7.repo -o /etc/yum.repos.d/public-yum-ol7.repo
docker-compose -f "$docker_compose_file" exec -T oracledb yum install -y yum-utils
docker-compose -f "$docker_compose_file" exec -T oracledb yum-config-manager --enable ol7_oracle_instantclient
docker-compose -f "$docker_compose_file" exec -T oracledb yum install -y oracle-instantclient18.3-basiclite
set +x
printf "\nCopying to host's ~/lib directory and adding to ldconfig ...\n"
if [ "$do_changes" = "true" ]; then
set -x
docker cp oracledb_container:/usr/lib/oracle/18.3/client64/lib/ ~/
sudo sh -c "echo $HOME/lib > /etc/ld.so.conf.d/oracle-instantclient.conf"
sudo ldconfig
set +x
else
printf "(skipping because dry-run)\n"
fi
}
# Start the bash app's main function
main "$cmd"

View File

@ -0,0 +1,36 @@
#!/bin/bash -e
changelog=node_modules/.bin/changelog
update_version() {
echo "$(node -p "p=require('./${1}');p.version='${2}';JSON.stringify(p,null,2)")" > $1
echo "Updated ${1} version to ${2}"
}
current_version=$(node -p "require('./package').version")
printf "Next version (current is $current_version)? "
read next_version
if ! [[ $next_version =~ ^[0-9]\.[0-9]+\.[0-9](-.+)? ]]; then
echo "Version must be a valid semver string, e.g. 1.0.2 or 2.3.0-beta.1"
exit 1
fi
next_ref="v$next_version"
git add -u
npm run build
npm run format:check
npm run lint:everything
npm test
update_version 'package.json' $next_version
git commit -am "release $next_version"
git tag $next_version
git push --tags
npm publish

View File

@ -0,0 +1,35 @@
require('sqlite3');
const Knex = require('knex');
const knexSqlite = Knex({
client: 'sqlite',
connection: ':memory:',
});
// eslint-disable-next-line no-unused-vars
const knexMysql = Knex({
client: 'mysql2',
});
const knexPg = Knex({
client: 'pg',
});
(async function run() {
await knexSqlite.schema.createTable('test', (t) => {
t.increments('id').primary();
t.string('data');
});
await knexSqlite('test').insert([{ data: 'foo' }, { data: 'bar' }]);
console.log('test table data:', await knexSqlite('test'));
console.log(
knexPg({ f: 'foo', b: 'bar' })
.select('foo.*')
.where('f.name', knexPg.raw('??', ['b.name']))
.whereIn('something', knexPg('bar').select('id'))
.toSQL().sql
);
})();

View File

@ -0,0 +1,18 @@
# Test scripts to evaluate stability of drivers / pool etc.
# To run this test you need to be in this directory + have node >= 8
# and startup docker containers with proxy and sql servers
docker-compose up --no-start
docker-compose start
# Select different test script to run:
node mysql2-random-hanging-every-now-and-then.js 2> /dev/null | grep -B500 -A2 -- "- STATS"
node mysql2-sudden-exit-without-error
node knex-stress-test.js | grep -A 3 -- "- STATS "
node reconnect-test-mysql-based-drivers.js 2> /dev/null | grep -A 3 -- "- STATS "
# Shut down docker instances when done:
docker-compose down

View File

@ -0,0 +1,57 @@
version: '3'
services:
toxiproxy:
image: shopify/toxiproxy
ports:
- '8474:8474'
- '23306:23306'
- '25432:25432'
- '25433:25433'
- '21521:21521'
- '21433:21433'
links:
- 'mysql'
- 'postgresql'
- 'pgnative'
- 'oracledb'
- 'mssql'
mysql:
image: mysql:5.7
ports:
- '33306:3306'
environment:
- TZ=UTC
- MYSQL_ROOT_PASSWORD=mysqlrootpassword
postgresql:
image: mdillon/postgis
ports:
- '35432:5432'
environment:
- POSTGRES_PASSWORD=postgresrootpassword
- POSTGRES_USER=postgres
pgnative:
image: mdillon/postgis
ports:
- '35433:5432'
environment:
- POSTGRES_PASSWORD=postgresrootpassword
- POSTGRES_USER=postgres
oracledb:
image: quillbuilduser/oracle-18-xe
ports:
- '31521:1521'
environment:
- ORACLE_ALLOW_REMOTE=true
mssql:
image: microsoft/mssql-server-linux:2017-latest
ports:
- '31433:1433'
environment:
- ACCEPT_EULA=Y
- SA_PASSWORD=S0meVeryHardPassword

View File

@ -0,0 +1,212 @@
const Knex = require('../../lib');
const toxiproxy = require('toxiproxy-node-client');
const toxicli = new toxiproxy.Toxiproxy('http://localhost:8474');
const rp = require('request-promise-native');
const delay = require('../../lib/execution/internal/delay');
// init instances
const pg = Knex({
client: 'pg',
connection:
'postgres://postgres:postgresrootpassword@localhost:25432/postgres',
pool: { max: 50 },
});
const pgnative = Knex({
client: 'pgnative',
connection:
'postgres://postgres:postgresrootpassword@localhost:25433/postgres',
pool: { max: 50 },
});
const mysql2 = Knex({
client: 'mysql2',
connection:
'mysql://root:mysqlrootpassword@localhost:23306/?charset=utf8&connectTimeout=500',
pool: { max: 50 },
});
const mysql = Knex({
client: 'mysql',
connection:
'mysql://root:mysqlrootpassword@localhost:23306/?charset=utf8&connectTimeout=500',
pool: { max: 50 },
});
const mssql = Knex({
client: 'mssql',
connection: {
port: 21433,
user: 'sa',
password: 'S0meVeryHardPassword',
server: 'localhost',
requestTimeout: 500,
},
pool: { max: 50 },
});
/* TODO: figure out how to nicely install oracledb node driver on osx
const oracledb = Knex({
client: 'oracledb',
connection: {
user : "travis",
password : "travis",
connectString : "localhost/XE",
// https://github.com/oracle/node-oracledb/issues/525
stmtCacheSize : 0
},
pool: { max: 50 }
});
*/
const counters = {};
function setQueryCounters(instance, name) {
const counts = (counters[name] = { queries: 0, results: 0, errors: 0 });
instance.on('query', () => (counts.queries += 1));
instance.on('query-response', () => (counts.results += 1));
instance.on('query-error', () => (counts.errors += 1));
}
setQueryCounters(pg, 'pg');
setQueryCounters(mysql, 'mysql');
setQueryCounters(mysql2, 'mysql2');
setQueryCounters(mssql, 'mssql');
const _ = require('lodash');
// start printing out counters
let lastCounters = _.cloneDeep(counters);
setInterval(() => {
const reqsPerSec = {};
for (const key of Object.keys(counters)) {
reqsPerSec[key] = {
queries: (counters[key].queries - lastCounters[key].queries) / 2,
results: (counters[key].results - lastCounters[key].results) / 2,
errors: (counters[key].errors - lastCounters[key].errors) / 2,
};
}
console.log(
'------------------------ STATS PER SECOND ------------------------'
);
console.dir(reqsPerSec, { colors: true });
console.log(
'------------------------------- EOS ------------------------------'
);
lastCounters = _.cloneDeep(counters);
}, 2000);
async function killConnectionsPg(client) {
return client.raw(`SELECT pg_terminate_backend(pg_stat_activity.pid)
FROM pg_stat_activity
WHERE pg_stat_activity.datname = 'postgres'
AND pid <> pg_backend_pid()`);
}
async function killConnectionsMyslq(client) {
const [rows] = await client.raw(`SHOW FULL PROCESSLIST`);
await Promise.all(rows.map((row) => client.raw(`KILL ${row.Id}`)));
}
async function killConnectionsMssql() {
const rows = await mssql('sys.dm_exec_sessions').select('session_id');
await Promise.all(rows.map((row) => mssql.raw(`KILL ${row.session_id}`)));
}
async function main() {
async function loopQueries(prefix, query) {
const queries = () => [...Array(50).fill(query)];
// eslint-disable-next-line no-constant-condition
while (true) {
try {
await Promise.all(queries());
} catch (err) {
console.log(prefix, err);
}
}
}
async function recreateProxy(serviceName, listenPort, proxyToPort) {
try {
await rp.delete({
url: `${toxicli.host}/proxies/${serviceName}`,
});
} catch (err) {
/* empty */
}
const proxy = await toxicli.createProxy({
name: serviceName,
listen: `0.0.0.0:${listenPort}`,
upstream: `${serviceName}:${proxyToPort}`,
});
// add some latency
await proxy.addToxic(
new toxiproxy.Toxic(proxy, {
type: 'latency',
attributes: { latency: 1, jitter: 1 },
})
);
// cause connections to be closed every 500 bytes
await proxy.addToxic(
new toxiproxy.Toxic(proxy, {
type: 'limit_data',
attributes: { bytes: 5000 },
})
);
}
// create TCP proxies for simulating bad connections etc.
async function recreateProxies() {
await recreateProxy('postgresql', 25432, 5432);
await recreateProxy('postgresql', 25433, 5433);
await recreateProxy('mysql', 23306, 3306);
await recreateProxy('oracledb', 21521, 1521);
await recreateProxy('mssql', 21433, 1433);
}
await recreateProxies();
loopQueries('PSQL:', pg.raw('select 1'));
loopQueries('PSQL TO:', pg.raw('select 1').timeout(20));
loopQueries('PGNATIVE:', pgnative.raw('select 1'));
loopQueries('PGNATIVE TO:', pgnative.raw('select 1').timeout(20));
loopQueries('MYSQL:', mysql.raw('select 1'));
loopQueries('MYSQL TO:', mysql.raw('select 1').timeout(20));
// mysql2 still crashes app (without connection killer nor timeouts)
// https://github.com/sidorares/node-mysql2/issues/731
// loopQueries('MYSQL2:', mysql2.raw('select 1'));
// loopQueries('MYSQL2 TO:', mysql2.raw('select 1').timeout(20));
loopQueries('MSSQL:', mssql.raw('select 1'));
loopQueries('MSSQL TO:', mssql.raw('select 1').timeout(20));
setInterval(recreateProxies, 2000);
// eslint-disable-next-line no-constant-condition
while (true) {
await delay(20); // kill everything every quite often from server side
try {
await Promise.all([
killConnectionsPg(pg),
killConnectionsPg(pgnative),
killConnectionsMyslq(mysql),
// killConnectionsMyslq(mysql2),
killConnectionsMssql(),
]);
} catch (err) {
console.log('KILLER ERROR:', err);
}
}
}
process.on('exit', () => console.log('- STATS PRINT NEAR END LOGS ')); // marker for grep...
main();

View File

@ -0,0 +1,149 @@
/**
* Test case for figuring out robust way to recognize if connection is dead
* for mysql based drivers.
*/
const delay = require('../../lib/execution/internal/delay');
const toxiproxy = require('toxiproxy-node-client');
const toxicli = new toxiproxy.Toxiproxy('http://localhost:8474');
const rp = require('request-promise-native');
const _ = require('lodash');
async function stdMysqlQuery(con, sql) {
return new Promise((resolve, reject) => {
try {
con.query(
{
sql,
timeout: 500,
},
function (error, results, fields) {
if (error) {
reject(error);
} else {
resolve(results);
}
}
);
} catch (err) {
reject(err); // double sure...
}
});
}
const mysql2 = require('mysql2');
let mysql2Con = { _fatalError: 'initmefirst' };
async function mysql2Query(sql) {
// recreate connection on fatal error
if (mysql2Con._fatalError) {
console.log('========== Reconnecting mysql2');
mysql2Con = mysql2.createConnection({
host: 'localhost',
user: 'root',
password: 'mysqlrootpassword',
port: 23306,
connectTimeout: 500,
debug: true,
});
mysql2Con.on('error', (err) => {
console.log('- STATS Mysql2 connection died:', err);
});
}
console.log('================ MYSQL2 Running query ======');
const res = await stdMysqlQuery(mysql2Con, sql);
console.log('====================== done ================');
return res;
}
const counters = {};
function setMysqlQueryCounters(name) {
counters[name] = { queries: 0, results: 0, errors: 0 };
}
setMysqlQueryCounters('mysql2');
// start printing out counters
let lastCounters = _.cloneDeep(counters);
setInterval(() => {
const reqsPerSec = {};
for (const key of Object.keys(counters)) {
reqsPerSec[key] = {
queries: counters[key].queries - lastCounters[key].queries,
results: counters[key].results - lastCounters[key].results,
errors: counters[key].errors - lastCounters[key].errors,
};
}
console.log(
'------------------------ STATS PER SECOND ------------------------'
);
console.dir(reqsPerSec, { colors: true });
console.log(
'------------------------------- EOS ------------------------------'
);
lastCounters = _.cloneDeep(counters);
}, 1000);
async function recreateProxy(serviceName, listenPort, proxyToPort) {
try {
await rp.delete({
url: `${toxicli.host}/proxies/${serviceName}`,
});
} catch (err) {
/* empty */
}
const proxy = await toxicli.createProxy({
name: serviceName,
listen: `0.0.0.0:${listenPort}`,
upstream: `${serviceName}:${proxyToPort}`,
});
// add some latency
await proxy.addToxic(
new toxiproxy.Toxic(proxy, {
type: 'latency',
attributes: { latency: 1, jitter: 1 },
})
);
// cause connections to be closed every some transferred bytes
await proxy.addToxic(
new toxiproxy.Toxic(proxy, {
type: 'limit_data',
attributes: { bytes: 1000 },
})
);
}
async function main() {
await recreateProxy('mysql', 23306, 3306);
setInterval(() => recreateProxy('mysql', 23306, 3306), 2000);
async function loopQueries(prefix, query) {
const counts = counters[prefix];
// eslint-disable-next-line no-constant-condition
while (true) {
try {
counts.queries += 1;
// without this delay we might endup to busy failure loop
await delay(0);
await query();
counts.results += 1;
} catch (err) {
counts.errors += 1;
console.log(prefix, err);
}
}
}
loopQueries('mysql2', () => mysql2Query('select 1'));
// wait forever
// eslint-disable-next-line no-constant-condition
while (true) {
await delay(1000);
}
}
main()
.then(() => console.log('DONE'))
.catch((err) => console.log(err));

View File

@ -0,0 +1,101 @@
/**
* Test case when mysql2 driver strangely exits when one tries to send query
* to dead connection.
*/
const delay = require('../../lib/execution/internal/delay');
const toxiproxy = require('toxiproxy-node-client');
const toxicli = new toxiproxy.Toxiproxy('http://localhost:8474');
const rp = require('request-promise-native');
// drops old toxicproxy and creates new
async function recreateProxy(serviceName, listenPort, proxyToPort) {
try {
await rp.delete({
url: `${toxicli.host}/proxies/${serviceName}`,
});
} catch (err) {
// there was no proxy by that name... its ok
}
await toxicli.createProxy({
name: serviceName,
listen: `0.0.0.0:${listenPort}`,
upstream: `${serviceName}:${proxyToPort}`,
});
}
async function insanelyParanoidQuery(con) {
console.log('sending query');
const res = await new Promise((resolve, reject) => {
try {
con.query('select 1', [], function (err, rows, fields) {
if (err) {
reject(err);
} else {
resolve(rows);
}
});
} catch (err) {
console.log('Huh synchronous exception?! (shouldnt be possible)');
reject(err);
}
});
console.log(res);
}
async function main() {
// create proxy from localhost:23306 -> mysqldocker:3306
await recreateProxy('mysql', 23306, 3306);
// ------------- setup mysql2 db driver connection
const mysql2 = require('mysql2'); // with mysql this works...
const mysql2Con = await mysql2.createConnection({
host: 'localhost',
user: 'root',
password: 'mysqlrootpassword',
port: 23306,
});
mysql2Con.on('error', function (err) {
console.log("I'm dead", err);
});
console.log('Going to cut connections');
// cut connection during recreate
await recreateProxy('mysql', 23306, 3306);
console.log('Proxy recreated... start waiting');
// wait forever
// eslint-disable-next-line no-constant-condition
while (true) {
await delay(1000);
try {
await insanelyParanoidQuery(mysql2Con);
} catch (err) {
console.log('query failed:', err);
}
await recreateProxy('mysql', 23306, 3306);
}
}
main()
.then(() => console.log('Process stopped normally'))
.catch((err) => {
console.log('Process stopped to failure', err);
});
console.log('Waiting for eventloop to stop...');
process.on('uncaughtException', function (err) {
console.log('uncaughtException', err);
});
process.on('exit', function () {
console.log(
'Did someone call process.exit() or wat? exitCode:',
process.exitCode
);
});

View File

@ -0,0 +1,188 @@
/**
* Test case for figuring out robust way to recognize if connection is dead
* for mysql based drivers.
*/
const delay = require('../../lib/execution/internal/delay');
const toxiproxy = require('toxiproxy-node-client');
const toxicli = new toxiproxy.Toxiproxy('http://localhost:8474');
const rp = require('request-promise-native');
async function stdMysqlQuery(con, sql) {
return new Promise((resolve, reject) => {
try {
con.query(
{
sql,
timeout: 4000,
},
function (error, results, fields) {
if (error) {
reject(error);
} else {
resolve(results);
}
}
);
} catch (err) {
reject(err); // double sure...
}
});
}
// ALL THE DRIVERS HAS DIFFERENT BAG OF TRICKS TO RECOVER AND
// RECOGNIZE WHEN CONNECTION HAS BEEN CLOSED
// ------------- setup mysql db driver connection
const mysql = require('mysql');
let mysqlCon = { state: 'disconnected' };
async function mysqlQuery(sql) {
// best way to check if connection is still alive
if (mysqlCon.state === 'disconnected') {
console.log('reconnecting mysql');
mysqlCon = mysql.createConnection({
host: 'localhost',
user: 'root',
password: 'mysqlrootpassword',
port: 23306,
connectTimeout: 500,
});
// not always triggered, if this happens during query
mysqlCon.on('error', (err) => {
console.log('- STATS Mysql connection died:', err);
});
}
return stdMysqlQuery(mysqlCon, sql);
}
// ------------- setup mysql2 db driver connection
const mysql2 = require('mysql2');
let mysql2Con = { _fatalError: 'initmefirst' };
async function mysql2Query(sql) {
if (mysql2Con._fatalError) {
console.log('reconnecting mysql2');
mysql2Con = mysql2.createConnection({
host: 'localhost',
user: 'root',
password: 'mysqlrootpassword',
port: 23306,
connectTimeout: 500,
});
mysql2Con.on('error', (err) => {
console.log('- STATS Mysql2 connection died:', err);
});
}
console.log('================ MYSQL2 Running query....');
const res = await stdMysqlQuery(mysql2Con, sql);
console.log('=========== done');
return res;
}
const counters = {};
function setMysqlQueryCounters(name) {
counters[name] = { queries: 0, results: 0, errors: 0 };
}
setMysqlQueryCounters('mysql');
setMysqlQueryCounters('mysql2');
const _ = require('lodash');
// start printing out counters
let lastCounters = _.cloneDeep(counters);
setInterval(() => {
const reqsPerSec = {};
for (const key of Object.keys(counters)) {
reqsPerSec[key] = {
queries: counters[key].queries - lastCounters[key].queries,
results: counters[key].results - lastCounters[key].results,
errors: counters[key].errors - lastCounters[key].errors,
};
}
console.log(
'------------------------ STATS PER SECOND ------------------------'
);
console.dir(reqsPerSec, { colors: true });
console.log(
'------------------------------- EOS ------------------------------'
);
lastCounters = _.cloneDeep(counters);
// if hang
///if (reqsPerSec.mysql2.queries === 0) process.exit(0);
}, 1000);
async function main() {
async function recreateProxy(serviceName, listenPort, proxyToPort) {
try {
await rp.delete({
url: `${toxicli.host}/proxies/${serviceName}`,
});
} catch (err) {
/* empty */
}
const proxy = await toxicli.createProxy({
name: serviceName,
listen: `0.0.0.0:${listenPort}`,
upstream: `${serviceName}:${proxyToPort}`,
});
// add some latency
await proxy.addToxic(
new toxiproxy.Toxic(proxy, {
type: 'latency',
attributes: { latency: 1, jitter: 1 },
})
);
// cause connections to be closed every some transferred bytes
await proxy.addToxic(
new toxiproxy.Toxic(proxy, {
type: 'limit_data',
attributes: { bytes: 1000 },
})
);
}
// create TCP proxies for simulating bad connections etc.
async function recreateProxies() {
console.log('----- Recreating proxies -> cutting connections completely');
await recreateProxy('postgresql', 25432, 5432);
await recreateProxy('mysql', 23306, 3306);
await recreateProxy('oracledb', 21521, 1521);
}
setInterval(() => recreateProxies(), 2000);
async function loopQueries(prefix, query) {
const counts = counters[prefix];
// eslint-disable-next-line no-constant-condition
while (true) {
try {
counts.queries += 1;
// without this delay we endup to busy failure loop
await delay(0);
await query();
counts.results += 1;
} catch (err) {
counts.errors += 1;
console.log(prefix, err);
}
}
}
await recreateProxies();
loopQueries('mysql', () => mysqlQuery('select 1'));
loopQueries('mysql2', () => mysql2Query('select 1'));
// wait forever
// eslint-disable-next-line no-constant-condition
while (true) {
await delay(1000);
}
}
main()
.then(() => console.log('DONE'))
.catch((err) => console.log(err));

View File

@ -0,0 +1,90 @@
#!/usr/bin/env node
const path = require('path');
const fs = require('fs');
// Directory constants
const scriptDirectory = __dirname;
const repoDirectory = path.join(scriptDirectory, '..');
const libDirectory = path.join(repoDirectory, 'lib');
const helpText = `
Helper script to update lib/.gitignore for all .js files from .ts files.
update_gitignore_for_tsc_output.js COMMAND
COMMAND:
run: Update lib/.gitignore file.
help: Print this menu.
NOTES FOR USAGE:
1. This script is tested to work on Ubuntu 18.04 LTS.
`;
const gitignoreHeader = `# DO NOT EDIT, GENERATED BY: scripts/update_gitignore_for_tsc_output.js
# Do not include tsc generated type definitions
**/*.d.ts
# Do not include tsc source maps
**/*.js.map
# Do not include .js files from .ts files
`;
function main(cliCommand) {
if (cliCommand === 'run') {
console.log('Generating lib/.gitignore ...');
// Find all .ts files in lib/
const directoriesToProcess = [libDirectory];
const tsFiles = [];
while (directoriesToProcess.length > 0) {
const directory = directoriesToProcess.pop();
if (!fs.existsSync(directory)) {
throw new Error("Directory doesn't exist:", directory);
}
const files = fs.readdirSync(directory);
files.forEach((file) => {
const filename = path.join(directory, file);
const stat = fs.lstatSync(filename);
if (stat.isDirectory()) {
directoriesToProcess.push(filename);
} else if (filename.endsWith('.ts') && !filename.endsWith('.d.ts')) {
tsFiles.push(filename);
console.log('Found .ts file:', filename);
}
});
}
// Get paths of .js files to ignore
const jsFilesToIgnore = tsFiles.map((filepath) => {
// Cuts off `${libDirectory}/`
const relativeTsPath = filepath.slice(libDirectory.length + 1);
// Swaps .ts for .js file ending
const relativeJsPath =
relativeTsPath.slice(0, relativeTsPath.length - 3) + '.js';
// Always use POSIX-style path separators - .gitignore requires it
return relativeJsPath.split(path.sep).join(path.posix.sep);
});
const jsFilesToIgnoreString = jsFilesToIgnore.join('\n');
const libGitignorePath = path.join(libDirectory, '.gitignore');
fs.writeFileSync(
libGitignorePath,
gitignoreHeader + jsFilesToIgnoreString + '\n'
);
console.log('DONE');
} else if (['help', '--help', '-h', undefined].includes(cliCommand)) {
console.log(helpText);
} else {
console.log(`Unsupported command: ${cliCommand}`);
console.log("Try running with 'help' to see supported commands.");
process.exit(1);
}
}
// Main script logic
const cliCommand = process.argv[2];
// Start the bash app's main function
main(cliCommand);