fabio286
/
antares
mirror of https://github.com/Fabio286/antares.git
1
1
Fork 0
Code Issues Releases Wiki Activity

perf: split the export select query to avoid running out of memory

This commit is contained in:
Fabio Di Stasio 2021-11-01 16:12:20 +01:00
parent d9d3bf2bc9
commit 409ed54608
1 changed files with 49 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/main/libs/exporters/sql/MysqlExporter.js
View File

@ -52,6 +52,7 @@ ${footer}
async getTableInsert (tableName) { async getTableInsert (tableName) {
let rowCount = 0; let rowCount = 0;
let sqlStr = ''; let sqlStr = '';
const pageSize = 1000;
const countResults = await this._client.raw( const countResults = await this._client.raw(
`SELECT COUNT(1) as count FROM \`${this.schemaName}\`.\`${tableName}\`` `SELECT COUNT(1) as count FROM \`${this.schemaName}\`.\`${tableName}\``
@ -59,6 +60,7 @@ ${footer}
if (countResults.rows.length === 1) rowCount = countResults.rows[0].count; if (countResults.rows.length === 1) rowCount = countResults.rows[0].count;
if (rowCount > 0) { if (rowCount > 0) {
const totalPages = Math.ceil(rowCount / pageSize);
let queryLength = 0; let queryLength = 0;
let rowsWritten = 0; let rowsWritten = 0;
const { sqlInsertDivider, sqlInsertAfter } = this._options; const { sqlInsertDivider, sqlInsertAfter } = this._options;
@ -71,16 +73,18 @@ ${footer}
', ' ', '
)}) VALUES`; )}) VALUES`;
const tableResult = await this._client.raw(
`SELECT ${columnNames.join(', ')} FROM \`${
this.schemaName
}\`.\`${tableName}\``
);
sqlStr += `LOCK TABLES \`${tableName}\` WRITE;\n`; sqlStr += `LOCK TABLES \`${tableName}\` WRITE;\n`;
sqlStr += `/*!40000 ALTER TABLE \`${tableName}\` DISABLE KEYS */;`; sqlStr += `/*!40000 ALTER TABLE \`${tableName}\` DISABLE KEYS */;`;
sqlStr += '\n\n'; sqlStr += '\n\n';
for (let pageNumber = 0; pageNumber < totalPages; pageNumber++) {
const tableResult = await this._client.raw(
`SELECT ${columnNames.join(', ')} FROM \`${
this.schemaName
}\`.\`${tableName}\`
LIMIT ${pageSize} OFFSET ${pageSize * pageNumber}`
);
sqlStr += insertStmt; sqlStr += insertStmt;
for (const rowIndex in tableResult.rows) { for (const rowIndex in tableResult.rows) {
@ -127,8 +131,8 @@ ${footer}
queryLength += sqlInsertString.length; queryLength += sqlInsertString.length;
rowsWritten++; rowsWritten++;
} }
sqlStr += ';\n\n'; sqlStr += ';\n\n';
}
sqlStr += `/*!40000 ALTER TABLE \`${tableName}\` ENABLE KEYS */;\n`; sqlStr += `/*!40000 ALTER TABLE \`${tableName}\` ENABLE KEYS */;\n`;
sqlStr += 'UNLOCK TABLES;'; sqlStr += 'UNLOCK TABLES;';