fabio286
/
antares
mirror of https://github.com/Fabio286/antares.git
1
1
Fork 0
Code Issues Releases Wiki Activity

perf: split the export select query to avoid running out of memory

This commit is contained in:
Fabio Di Stasio 2021-11-01 16:12:20 +01:00
parent d9d3bf2bc9
commit 409ed54608
1 changed files with 49 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
src/main/libs/exporters/sql/MysqlExporter.js
View File

@ -52,6 +52,7 @@ ${footer}
async getTableInsert (tableName) {
let rowCount = 0;
let sqlStr = '';
const pageSize = 1000;
const countResults = await this._client.raw(
`SELECT COUNT(1) as count FROM \`${this.schemaName}\`.\`${tableName}\``
@ -59,6 +60,7 @@ ${footer}
if (countResults.rows.length === 1) rowCount = countResults.rows[0].count;
if (rowCount > 0) {
const totalPages = Math.ceil(rowCount / pageSize);
let queryLength = 0;
let rowsWritten = 0;
const { sqlInsertDivider, sqlInsertAfter } = this._options;
@ -71,65 +73,67 @@ ${footer}
', '
)}) VALUES`;
const tableResult = await this._client.raw(
`SELECT ${columnNames.join(', ')} FROM \`${
this.schemaName
}\`.\`${tableName}\``
);
sqlStr += `LOCK TABLES \`${tableName}\` WRITE;\n`;
sqlStr += `/*!40000 ALTER TABLE \`${tableName}\` DISABLE KEYS */;`;
sqlStr += '\n\n';
sqlStr += insertStmt;
for (let pageNumber = 0; pageNumber < totalPages; pageNumber++) {
const tableResult = await this._client.raw(
`SELECT ${columnNames.join(', ')} FROM \`${
this.schemaName
}\`.\`${tableName}\`
LIMIT ${pageSize} OFFSET ${pageSize * pageNumber}`
);
for (const rowIndex in tableResult.rows) {
const row = tableResult.rows[rowIndex];
let sqlInsertString = '';
sqlStr += insertStmt;
if (
(sqlInsertDivider === 'bytes' &&
queryLength >= sqlInsertAfter * 1024) ||
(sqlInsertDivider === 'rows' && rowsWritten === sqlInsertAfter)
) {
sqlInsertString += `;\n${insertStmt}\n\t(`;
for (const rowIndex in tableResult.rows) {
const row = tableResult.rows[rowIndex];
let sqlInsertString = '';
queryLength = 0;
rowsWritten = 0;
}
else if (parseInt(rowIndex) === 0) sqlInsertString += '\n\t(';
else sqlInsertString += ',\n\t(';
if (
(sqlInsertDivider === 'bytes' &&
queryLength >= sqlInsertAfter * 1024) ||
(sqlInsertDivider === 'rows' && rowsWritten === sqlInsertAfter)
) {
sqlInsertString += `;\n${insertStmt}\n\t(`;
for (const i in columns) {
const column = columns[i];
const val = row[column.name];
if (val === null) sqlInsertString += 'NULL';
else if (BIT.includes(column.type)) {
sqlInsertString += `b'${hexToBinary(
Buffer.from(val).toString('hex')
)}'`;
queryLength = 0;
rowsWritten = 0;
}
else if (BLOB.includes(column.type))
sqlInsertString += `X'${val.toString('hex').toUpperCase()}'`;
else if (val === '') sqlInsertString += '\'\'';
else {
sqlInsertString +=
typeof val === 'string' ? this.escapeAndQuote(val) : val;
else if (parseInt(rowIndex) === 0) sqlInsertString += '\n\t(';
else sqlInsertString += ',\n\t(';
for (const i in columns) {
const column = columns[i];
const val = row[column.name];
if (val === null) sqlInsertString += 'NULL';
else if (BIT.includes(column.type)) {
sqlInsertString += `b'${hexToBinary(
Buffer.from(val).toString('hex')
)}'`;
}
else if (BLOB.includes(column.type))
sqlInsertString += `X'${val.toString('hex').toUpperCase()}'`;
else if (val === '') sqlInsertString += '\'\'';
else {
sqlInsertString +=
typeof val === 'string' ? this.escapeAndQuote(val) : val;
}
if (parseInt(i) !== columns.length - 1) sqlInsertString += ', ';
}
if (parseInt(i) !== columns.length - 1) sqlInsertString += ', ';
sqlInsertString += ')';
sqlStr += sqlInsertString;
queryLength += sqlInsertString.length;
rowsWritten++;
}
sqlInsertString += ')';
sqlStr += sqlInsertString;
queryLength += sqlInsertString.length;
rowsWritten++;
sqlStr += ';\n\n';
}
sqlStr += ';\n\n';
sqlStr += `/*!40000 ALTER TABLE \`${tableName}\` ENABLE KEYS */;\n`;
sqlStr += 'UNLOCK TABLES;';
}