fabio286
/
antares
mirror of https://github.com/Fabio286/antares.git
1
1
Fork 0
Code Issues Releases Wiki Activity

perf: split the export select query to avoid running out of memory

This commit is contained in:
Fabio Di Stasio 2021-11-01 16:12:20 +01:00
parent d9d3bf2bc9
commit 409ed54608
1 changed files with 49 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
src/main/libs/exporters/sql/MysqlExporter.js
View File

@ -52,6 +52,7 @@ ${footer}
async getTableInsert (tableName) { async getTableInsert (tableName) {
let rowCount = 0; let rowCount = 0;
let sqlStr = ''; let sqlStr = '';
const pageSize = 1000;
const countResults = await this._client.raw( const countResults = await this._client.raw(
`SELECT COUNT(1) as count FROM \`${this.schemaName}\`.\`${tableName}\`` `SELECT COUNT(1) as count FROM \`${this.schemaName}\`.\`${tableName}\``
@ -59,6 +60,7 @@ ${footer}
if (countResults.rows.length === 1) rowCount = countResults.rows[0].count; if (countResults.rows.length === 1) rowCount = countResults.rows[0].count;
if (rowCount > 0) { if (rowCount > 0) {
const totalPages = Math.ceil(rowCount / pageSize);
let queryLength = 0; let queryLength = 0;
let rowsWritten = 0; let rowsWritten = 0;
const { sqlInsertDivider, sqlInsertAfter } = this._options; const { sqlInsertDivider, sqlInsertAfter } = this._options;
@ -71,65 +73,67 @@ ${footer}
', ' ', '
)}) VALUES`; )}) VALUES`;
const tableResult = await this._client.raw(
`SELECT ${columnNames.join(', ')} FROM \`${
this.schemaName
}\`.\`${tableName}\``
);
sqlStr += `LOCK TABLES \`${tableName}\` WRITE;\n`; sqlStr += `LOCK TABLES \`${tableName}\` WRITE;\n`;
sqlStr += `/*!40000 ALTER TABLE \`${tableName}\` DISABLE KEYS */;`; sqlStr += `/*!40000 ALTER TABLE \`${tableName}\` DISABLE KEYS */;`;
sqlStr += '\n\n'; sqlStr += '\n\n';
sqlStr += insertStmt; for (let pageNumber = 0; pageNumber < totalPages; pageNumber++) {
const tableResult = await this._client.raw(
`SELECT ${columnNames.join(', ')} FROM \`${
this.schemaName
}\`.\`${tableName}\`
LIMIT ${pageSize} OFFSET ${pageSize * pageNumber}`
);
for (const rowIndex in tableResult.rows) { sqlStr += insertStmt;
const row = tableResult.rows[rowIndex];
let sqlInsertString = '';
if ( for (const rowIndex in tableResult.rows) {
(sqlInsertDivider === 'bytes' && const row = tableResult.rows[rowIndex];
queryLength >= sqlInsertAfter * 1024) || let sqlInsertString = '';
(sqlInsertDivider === 'rows' && rowsWritten === sqlInsertAfter)
) {
sqlInsertString += `;\n${insertStmt}\n\t(`;
queryLength = 0; if (
rowsWritten = 0; (sqlInsertDivider === 'bytes' &&
} queryLength >= sqlInsertAfter * 1024) ||
else if (parseInt(rowIndex) === 0) sqlInsertString += '\n\t('; (sqlInsertDivider === 'rows' && rowsWritten === sqlInsertAfter)
else sqlInsertString += ',\n\t('; ) {
sqlInsertString += `;\n${insertStmt}\n\t(`;
for (const i in columns) { queryLength = 0;
const column = columns[i]; rowsWritten = 0;
const val = row[column.name];
if (val === null) sqlInsertString += 'NULL';
else if (BIT.includes(column.type)) {
sqlInsertString += `b'${hexToBinary(
Buffer.from(val).toString('hex')
)}'`;
} }
else if (BLOB.includes(column.type)) else if (parseInt(rowIndex) === 0) sqlInsertString += '\n\t(';
sqlInsertString += `X'${val.toString('hex').toUpperCase()}'`; else sqlInsertString += ',\n\t(';
else if (val === '') sqlInsertString += '\'\'';
else { for (const i in columns) {
sqlInsertString += const column = columns[i];
typeof val === 'string' ? this.escapeAndQuote(val) : val; const val = row[column.name];
if (val === null) sqlInsertString += 'NULL';
else if (BIT.includes(column.type)) {
sqlInsertString += `b'${hexToBinary(
Buffer.from(val).toString('hex')
)}'`;
}
else if (BLOB.includes(column.type))
sqlInsertString += `X'${val.toString('hex').toUpperCase()}'`;
else if (val === '') sqlInsertString += '\'\'';
else {
sqlInsertString +=
typeof val === 'string' ? this.escapeAndQuote(val) : val;
}
if (parseInt(i) !== columns.length - 1) sqlInsertString += ', ';
} }
if (parseInt(i) !== columns.length - 1) sqlInsertString += ', '; sqlInsertString += ')';
sqlStr += sqlInsertString;
queryLength += sqlInsertString.length;
rowsWritten++;
} }
sqlStr += ';\n\n';
sqlInsertString += ')';
sqlStr += sqlInsertString;
queryLength += sqlInsertString.length;
rowsWritten++;
} }
sqlStr += ';\n\n';
sqlStr += `/*!40000 ALTER TABLE \`${tableName}\` ENABLE KEYS */;\n`; sqlStr += `/*!40000 ALTER TABLE \`${tableName}\` ENABLE KEYS */;\n`;
sqlStr += 'UNLOCK TABLES;'; sqlStr += 'UNLOCK TABLES;';
} }