diff --git a/.gitignore b/.gitignore index 60bac061..6294b57c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ node_modules thumbs.db NOTES.md *.txt -package-lock.json \ No newline at end of file +package-lock.json +*.heapsnapshot \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index a6027bca..3de1b3cf 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -2,8 +2,8 @@ "version": "0.2.0", "configurations": [ { - "cwd": "${workspaceFolder}", "name": "Electron: Main", + "cwd": "${workspaceFolder}", "port": 9222, "protocol": "inspector", "request": "attach", @@ -18,7 +18,17 @@ "sourceMaps": true, "type": "chrome", "webRoot": "${workspaceFolder}" - } + }, + { + "name": "Electron: Worker", + "cwd": "${workspaceFolder}", + "port": 9224, + "protocol": "inspector", + "request": "attach", + "sourceMaps": true, + "type": "node", + "timeout": 1000000 + }, ], "compounds": [ { diff --git a/package.json b/package.json index fd1b89b7..09578655 100644 --- a/package.json +++ b/package.json @@ -8,13 +8,14 @@ "scripts": { "debug": "npm run rebuild:electron && npm run debug-runner", "debug-runner": "node scripts/devRunner.js --remote-debug", - "compile": "npm run compile:main && npm run compile:renderer", + "compile": "npm run compile:main && npm run compile:workers && npm run compile:renderer", "compile:main": "webpack --mode=production --config webpack.main.config.js", + "compile:workers": "webpack --mode=production --config webpack.workers.config.js", "compile:renderer": "webpack --mode=production --config webpack.renderer.config.js", "build": "cross-env NODE_ENV=production npm run compile", "build:local": "npm run build && electron-builder", "build:appx": "npm run build:local -- --win appx", - "rebuild:electron": "npm run postinstall", + "rebuild:electron": "rimraf ./dist && npm run postinstall", "release": "standard-version", "release:pre": "npm run release -- --prerelease alpha", "postinstall": "electron-builder install-app-deps", @@ -136,7 +137,6 @@ "all-contributors-cli": "^6.20.0", "babel-loader": "^8.2.3", "chalk": "^4.1.2", - "clean-webpack-plugin": "^4.0.0", "cross-env": "^7.0.2", "css-loader": "^6.5.0", "electron": "^17.0.1", @@ -154,6 +154,7 @@ "node-loader": "^2.0.0", "playwright": "^1.18.1", "progress-webpack-plugin": "^1.0.12", + "rimraf": "^3.0.2", "sass": "^1.42.1", "sass-loader": "^12.3.0", "standard-version": "^9.3.1", diff --git a/scripts/devRunner.js b/scripts/devRunner.js index ed3eff40..f50f8d6a 100644 --- a/scripts/devRunner.js +++ b/scripts/devRunner.js @@ -12,7 +12,7 @@ const { spawn } = require('child_process'); const mainConfig = require('../webpack.main.config'); const rendererConfig = require('../webpack.renderer.config'); -// const workersConfig = require('../webpack.workers.config'); +const workersConfig = require('../webpack.workers.config'); let electronProcess = null; let manualRestart = null; @@ -64,7 +64,7 @@ async function restartElectron () { } function startMain () { - const webpackSetup = webpack(mainConfig); + const webpackSetup = webpack([mainConfig, workersConfig]); webpackSetup.compilers.forEach((compiler) => { const { name } = compiler; diff --git a/src/common/FakerMethods.js b/src/common/FakerMethods.js index fe7c7ec4..e401fa11 100644 --- a/src/common/FakerMethods.js +++ b/src/common/FakerMethods.js @@ -134,7 +134,7 @@ export default class { { name: 'phoneNumberFormat', group: 'phone', types: ['string'] }, { name: 'phoneFormats', group: 'phone', types: ['string'] }, - { name: 'number', group: 'random', types: ['string', 'number'], params: ['min', 'max'] }, + { name: 'number', group: 'datatype', types: ['string', 'number'], params: ['min', 'max'] }, { name: 'float', group: 'random', types: ['string', 'float'], params: ['min', 'max'] }, { name: 'arrayElement', group: 'random', types: ['string'] }, { name: 'arrayElements', group: 'random', types: ['string'] }, diff --git a/src/common/customizations/defaults.js b/src/common/customizations/defaults.js index 4a417792..423c31ae 100644 --- a/src/common/customizations/defaults.js +++ b/src/common/customizations/defaults.js @@ -38,6 +38,8 @@ module.exports = { databaseEdit: false, schemaEdit: false, schemaDrop: false, + schemaExport: false, + schemaImport: false, tableSettings: false, tableOptions: false, tableArray: false, diff --git a/src/common/customizations/mysql.js b/src/common/customizations/mysql.js index 624edbe8..075c498c 100644 --- a/src/common/customizations/mysql.js +++ b/src/common/customizations/mysql.js @@ -34,6 +34,8 @@ module.exports = { schedulerAdd: true, schemaEdit: true, schemaDrop: true, + schemaExport: true, + schemaImport: true, tableSettings: true, viewSettings: true, triggerSettings: true, diff --git a/src/common/customizations/postgresql.js b/src/common/customizations/postgresql.js index 4d5cda98..178eee93 100644 --- a/src/common/customizations/postgresql.js +++ b/src/common/customizations/postgresql.js @@ -32,6 +32,8 @@ module.exports = { functionAdd: true, schemaDrop: true, databaseEdit: false, + schemaExport: false, + schemaImport: false, tableSettings: true, viewSettings: true, triggerSettings: true, diff --git a/src/common/fieldTypes.js b/src/common/fieldTypes.js index a7d3399f..cbd8cbe6 100644 --- a/src/common/fieldTypes.js +++ b/src/common/fieldTypes.js @@ -41,6 +41,7 @@ export const NUMBER = [ export const FLOAT = [ 'FLOAT', + 'DECIMAL', 'DOUBLE', 'REAL', 'DOUBLE PRECISION', diff --git a/src/common/libs/hexToBinary.js b/src/common/libs/hexToBinary.js index 38c59e5d..305bcf7e 100644 --- a/src/common/libs/hexToBinary.js +++ b/src/common/libs/hexToBinary.js @@ -33,7 +33,7 @@ const lookup = { */ export default function hexToBinary (hex) { let binary = ''; - for (let i = 0, len = hex.length; i < len; i++) + for (let i = 0; i < hex.length; i++) binary += lookup[hex[i]]; return binary; diff --git a/src/common/libs/mimeFromHex.js b/src/common/libs/mimeFromHex.js index 6d292de6..837b127e 100644 --- a/src/common/libs/mimeFromHex.js +++ b/src/common/libs/mimeFromHex.js @@ -23,7 +23,7 @@ export function mimeFromHex (hex) { case '425A68': return { ext: 'bz2', mime: 'application/x-bzip2' }; default: - switch (hex) { // 4 bites + switch (hex) { // 4 bytes case '89504E47': return { ext: 'png', mime: 'image/png' }; case '47494638': diff --git a/src/common/libs/sqlParser.js b/src/common/libs/sqlParser.js new file mode 100644 index 00000000..933277b5 --- /dev/null +++ b/src/common/libs/sqlParser.js @@ -0,0 +1,93 @@ +import { Transform } from 'stream'; + +export default class SqlParser extends Transform { + constructor (opts) { + opts = { + delimiter: ';', + encoding: 'utf8', + writableObjectMode: true, + readableObjectMode: true, + ...opts + }; + super(opts); + this._buffer = ''; + this._lastChar = ''; + this._last9Chars = ''; + this.encoding = opts.encoding; + this.delimiter = opts.delimiter; + + this.isEscape = false; + this.currentQuote = null; + this.isDelimiter = false; + } + + _transform (chunk, encoding, next) { + for (const char of chunk.toString(this.encoding)) { + this.checkEscape(); + this._buffer += char; + this._lastChar = char; + this._last9Chars += char.trim().toLocaleLowerCase(); + + if (this._last9Chars.length > 9) + this._last9Chars = this._last9Chars.slice(-9); + + this.checkNewDelimiter(char); + this.checkQuote(char); + const query = this.getQuery(); + + if (query) + this.push(query); + } + next(); + } + + checkEscape () { + if (this._buffer.length > 0) { + this.isEscape = this._lastChar === '\\' + ? !this.isEscape + : false; + } + } + + checkNewDelimiter (char) { + if (this.currentQuote === null && this._last9Chars === 'delimiter') { + this.isDelimiter = true; + this._buffer = ''; + } + else { + const isNewLine = char === '\n' || char === '\r'; + if (isNewLine && this.isDelimiter) { + this.isDelimiter = false; + this.delimiter = this._buffer.trim(); + this._buffer = ''; + } + } + } + + checkQuote (char) { + const isQuote = !this.isEscape && (char === '\'' || char === '"'); + if (isQuote && this.currentQuote === char) + this.currentQuote = null; + + else if (isQuote && this.currentQuote === null) + this.currentQuote = char; + } + + getQuery () { + if (this.isDelimiter) + return false; + + let query = false; + let demiliterFound = false; + if (this.currentQuote === null && this._buffer.length >= this.delimiter.length) + demiliterFound = this._last9Chars.slice(-this.delimiter.length) === this.delimiter; + + if (demiliterFound) { + const parsedStr = this._buffer.trim(); + query = parsedStr.slice(0, parsedStr.length - this.delimiter.length); + this._buffer = ''; + } + + return query; + } +} diff --git a/src/main/ipc-handlers/application.js b/src/main/ipc-handlers/application.js index 31a3fe0d..a8033a2d 100644 --- a/src/main/ipc-handlers/application.js +++ b/src/main/ipc-handlers/application.js @@ -1,4 +1,4 @@ -import { app, ipcMain } from 'electron'; +import { app, ipcMain, dialog } from 'electron'; export default () => { ipcMain.on('close-app', () => { @@ -9,4 +9,12 @@ export default () => { const key = false; event.returnValue = key; }); + + ipcMain.handle('showOpenDialog', (event, options) => { + return dialog.showOpenDialog(options); + }); + + ipcMain.handle('get-download-dir-path', () => { + return app.getPath('downloads'); + }); }; diff --git a/src/main/ipc-handlers/connection.js b/src/main/ipc-handlers/connection.js index c539f75a..696aeb99 100644 --- a/src/main/ipc-handlers/connection.js +++ b/src/main/ipc-handlers/connection.js @@ -40,7 +40,7 @@ export default connections => { } try { - const connection = await ClientsFactory.getConnection({ + const connection = await ClientsFactory.getClient({ client: conn.client, params }); @@ -100,7 +100,7 @@ export default connections => { } try { - const connection = ClientsFactory.getConnection({ + const connection = ClientsFactory.getClient({ client: conn.client, params, poolSize: 5 diff --git a/src/main/ipc-handlers/schema.js b/src/main/ipc-handlers/schema.js index 203e7d80..b8149be2 100644 --- a/src/main/ipc-handlers/schema.js +++ b/src/main/ipc-handlers/schema.js @@ -1,7 +1,15 @@ +import fs from 'fs'; +import path from 'path'; +import { fork } from 'child_process'; +import { ipcMain, dialog } from 'electron'; -import { ipcMain } from 'electron'; +// @TODO: need some factories +const isDevelopment = process.env.NODE_ENV !== 'production'; export default connections => { + let exporter = null; + let importer = null; + ipcMain.handle('create-schema', async (event, params) => { try { await connections[params.uid].createSchema(params); @@ -37,9 +45,16 @@ export default connections => { ipcMain.handle('get-schema-collation', async (event, params) => { try { - const collation = await connections[params.uid].getDatabaseCollation(params); + const collation = await connections[params.uid].getDatabaseCollation( + params + ); - return { status: 'success', response: collation.rows.length ? collation.rows[0].DEFAULT_COLLATION_NAME : '' }; + return { + status: 'success', + response: collation.rows.length + ? collation.rows[0].DEFAULT_COLLATION_NAME + : '' + }; } catch (err) { return { status: 'error', response: err.toString() }; @@ -48,7 +63,9 @@ export default connections => { ipcMain.handle('get-structure', async (event, params) => { try { - const structure = await connections[params.uid].getStructure(params.schemas); + const structure = await connections[params.uid].getStructure( + params.schemas + ); return { status: 'success', response: structure }; } @@ -155,6 +172,169 @@ export default connections => { } }); + ipcMain.handle('export', (event, { uid, type, tables, ...rest }) => { + if (exporter !== null) return; + + return new Promise((resolve, reject) => { + (async () => { + if (fs.existsSync(rest.outputFile)) { // If file exists ask for replace + const result = await dialog.showMessageBox({ + type: 'warning', + message: `File ${rest.outputFile} already exists. Do you want to replace it?`, + detail: 'A file with the same name already exists in the target folder. Replacing it will overwrite its current contents.', + buttons: ['Cancel', 'Replace'], + defaultId: 0, + cancelId: 0 + }); + + if (result.response !== 1) { + resolve({ + type: 'error', + response: 'Operation aborted' + }); + return; + } + } + + // Init exporter process + exporter = fork(isDevelopment ? './dist/exporter.js' : path.resolve(__dirname, './exporter.js'), [], { + execArgv: isDevelopment ? ['--inspect=9224'] : undefined + }); + exporter.send({ + type: 'init', + client: { + name: type, + config: await connections[uid].getDbConfig() + }, + tables, + options: rest + }); + + // Exporter message listener + exporter.on('message', ({ type, payload }) => { + switch (type) { + case 'export-progress': + event.sender.send('export-progress', payload); + break; + case 'end': + setTimeout(() => { // Ensures that writing process has finished + exporter.kill(); + exporter = null; + }, 2000); + resolve({ status: 'success', response: payload }); + break; + case 'cancel': + exporter.kill(); + exporter = null; + resolve({ status: 'error', response: 'Operation cancelled' }); + break; + case 'error': + exporter.kill(); + exporter = null; + resolve({ status: 'error', response: payload }); + break; + } + }); + + exporter.on('exit', code => { + exporter = null; + resolve({ status: 'error', response: `Operation ended with code: ${code}` }); + }); + })(); + }); + }); + + ipcMain.handle('abort-export', async event => { + let willAbort = false; + + if (exporter) { + const result = await dialog.showMessageBox({ + type: 'warning', + message: 'Are you sure you want to abort the export', + buttons: ['Cancel', 'Abort'], + defaultId: 0, + cancelId: 0 + }); + + if (result.response === 1) { + willAbort = true; + exporter.send({ type: 'cancel' }); + } + } + + return { status: 'success', response: { willAbort } }; + }); + + ipcMain.handle('import-sql', async (event, options) => { + if (importer !== null) return; + + return new Promise((resolve, reject) => { + (async () => { + const dbConfig = await connections[options.uid].getDbConfig(); + + // Init importer process + importer = fork(isDevelopment ? './dist/importer.js' : path.resolve(__dirname, './importer.js'), [], { + execArgv: isDevelopment ? ['--inspect=9224'] : undefined + }); + importer.send({ + type: 'init', + dbConfig, + options + }); + + // Importer message listener + importer.on('message', ({ type, payload }) => { + switch (type) { + case 'import-progress': + event.sender.send('import-progress', payload); + break; + case 'query-error': + event.sender.send('query-error', payload); + break; + case 'end': + setTimeout(() => { // Ensures that writing process has finished + importer?.kill(); + importer = null; + }, 2000); + resolve({ status: 'success', response: payload }); + break; + case 'cancel': + importer.kill(); + importer = null; + resolve({ status: 'error', response: 'Operation cancelled' }); + break; + case 'error': + importer.kill(); + importer = null; + resolve({ status: 'error', response: payload }); + break; + } + }); + })(); + }); + }); + + ipcMain.handle('abort-import-sql', async event => { + let willAbort = false; + + if (importer) { + const result = await dialog.showMessageBox({ + type: 'warning', + message: 'Are you sure you want to abort the import', + buttons: ['Cancel', 'Abort'], + defaultId: 0, + cancelId: 0 + }); + + if (result.response === 1) { + willAbort = true; + importer.send({ type: 'cancel' }); + } + } + + return { status: 'success', response: { willAbort } }; + }); + ipcMain.handle('kill-tab-query', async (event, { uid, tabUid }) => { if (!tabUid) return; diff --git a/src/main/ipc-handlers/tables.js b/src/main/ipc-handlers/tables.js index 7791aa5b..79e0034f 100644 --- a/src/main/ipc-handlers/tables.js +++ b/src/main/ipc-handlers/tables.js @@ -149,7 +149,7 @@ export default (connections) => { } } } - else if ([...BIT].includes(params.type)) { + else if (BIT.includes(params.type)) { escapedParam = `b'${sqlEscaper(params.content)}'`; reload = true; } diff --git a/src/main/libs/AntaresCore.js b/src/main/libs/AntaresCore.js index d618a8b1..7f5cd0ae 100644 --- a/src/main/libs/AntaresCore.js +++ b/src/main/libs/AntaresCore.js @@ -1,4 +1,10 @@ 'use strict'; +const queryLogger = sql => { + // Remove comments, newlines and multiple spaces + const escapedSql = sql.replace(/(\/\*(.|[\r\n])*?\*\/)|(--(.*|[\r\n]))/gm, '').replace(/\s\s+/g, ' '); + console.log(escapedSql); +}; + /** * As Simple As Possible Query Builder Core * @@ -17,7 +23,7 @@ export class AntaresCore { this._poolSize = args.poolSize || false; this._connection = null; this._ssh = null; - this._logger = args.logger || console.log; + this._logger = args.logger || queryLogger; this._queryDefaults = { schema: '', diff --git a/src/main/libs/ClientsFactory.js b/src/main/libs/ClientsFactory.js index 6543eaff..0ab6d5ba 100644 --- a/src/main/libs/ClientsFactory.js +++ b/src/main/libs/ClientsFactory.js @@ -2,13 +2,6 @@ import { MySQLClient } from './clients/MySQLClient'; import { PostgreSQLClient } from './clients/PostgreSQLClient'; import { SQLiteClient } from './clients/SQLiteClient'; - -const queryLogger = sql => { - // Remove comments, newlines and multiple spaces - const escapedSql = sql.replace(/(\/\*(.|[\r\n])*?\*\/)|(--(.*|[\r\n]))/gm, '').replace(/\s\s+/g, ' '); - console.log(escapedSql); -}; - export class ClientsFactory { /** * Returns a database connection based on received args. @@ -29,9 +22,7 @@ export class ClientsFactory { * @returns Database Connection * @memberof ClientsFactory */ - static getConnection (args) { - args.logger = queryLogger; - + static getClient (args) { switch (args.client) { case 'mysql': case 'maria': diff --git a/src/main/libs/clients/MySQLClient.js b/src/main/libs/clients/MySQLClient.js index 43cdc5d7..e29f7907 100644 --- a/src/main/libs/clients/MySQLClient.js +++ b/src/main/libs/clients/MySQLClient.js @@ -101,6 +101,26 @@ export class MySQLClient extends AntaresCore { .filter(_type => _type.name === type.toUpperCase())[0]; } + _reducer (acc, curr) { + const type = typeof curr; + + switch (type) { + case 'number': + case 'string': + return [...acc, curr]; + case 'object': + if (Array.isArray(curr)) + return [...acc, ...curr]; + else { + const clausoles = []; + for (const key in curr) + clausoles.push(`\`${key}\` ${curr[key]}`); + + return clausoles; + } + } + } + /** * * @returns dbConfig @@ -498,6 +518,7 @@ export class MySQLClient extends AntaresCore { charset: field.CHARACTER_SET_NAME, collation: field.COLLATION_NAME, autoIncrement: field.EXTRA.includes('auto_increment'), + generated: field.EXTRA.toLowerCase().includes('generated'), onUpdate: field.EXTRA.toLowerCase().includes('on update') ? field.EXTRA.substr(field.EXTRA.indexOf('on update') + 9, field.EXTRA.length).trim() : '', @@ -1602,7 +1623,7 @@ export class MySQLClient extends AntaresCore { let insertRaw = ''; if (this._query.insert.length) { - const fieldsList = Object.keys(this._query.insert[0]); + const fieldsList = Object.keys(this._query.insert[0]).map(col => '`' + col + '`'); const rowsList = this._query.insert.map(el => `(${Object.values(el).join(', ')})`); insertRaw = `(${fieldsList.join(', ')}) VALUES ${rowsList.join(', ')} `; diff --git a/src/main/libs/exporters/BaseExporter.js b/src/main/libs/exporters/BaseExporter.js new file mode 100644 index 00000000..50d35a07 --- /dev/null +++ b/src/main/libs/exporters/BaseExporter.js @@ -0,0 +1,77 @@ +import fs from 'fs'; +import path from 'path'; +import EventEmitter from 'events'; + +export class BaseExporter extends EventEmitter { + constructor (tables, options) { + super(); + this._tables = tables; + this._options = options; + this._isCancelled = false; + this._outputStream = fs.createWriteStream(this._options.outputFile, { + flags: 'w' + }); + this._state = {}; + + this._outputStream.once('error', err => { + this._isCancelled = true; + this.emit('error', err); + }); + } + + async run () { + try { + this.emit('start', this); + await this.dump(); + } + catch (err) { + this.emit('error', err); + throw err; + } + finally { + this._outputStream.end(); + this.emit('end'); + } + } + + get isCancelled () { + return this._isCancelled; + } + + get outputFile () { + return this._options.outputFile; + } + + outputFileExists () { + return fs.existsSync(this._options.outputFile); + } + + cancel () { + this._isCancelled = true; + this.emit('cancel'); + this.emitUpdate({ op: 'cancelling' }); + } + + emitUpdate (state) { + this.emit('progress', { ...this._state, ...state }); + } + + writeString (data) { + if (this._isCancelled) return; + + try { + fs.accessSync(this._options.outputFile); + } + catch (err) { + this._isCancelled = true; + + const fileName = path.basename(this._options.outputFile); + this.emit('error', `The file ${fileName} is not accessible`); + } + this._outputStream.write(data); + } + + dump () { + throw new Error('Exporter must implement the "dump" method'); + } +} diff --git a/src/main/libs/exporters/sql/MysqlExporter.js b/src/main/libs/exporters/sql/MysqlExporter.js new file mode 100644 index 00000000..f571ae4d --- /dev/null +++ b/src/main/libs/exporters/sql/MysqlExporter.js @@ -0,0 +1,412 @@ +import { SqlExporter } from './SqlExporter'; +import { BLOB, BIT, DATE, DATETIME, FLOAT, SPATIAL, IS_MULTI_SPATIAL, NUMBER } from 'common/fieldTypes'; +import hexToBinary from 'common/libs/hexToBinary'; +import { getArrayDepth } from 'common/libs/getArrayDepth'; +import moment from 'moment'; +import { lineString, point, polygon } from '@turf/helpers'; + +export default class MysqlExporter extends SqlExporter { + async getSqlHeader () { + let dump = await super.getSqlHeader(); + dump += ` + + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +SET NAMES utf8mb4; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;`; + + return dump; + } + + async getFooter () { + const footer = await super.getFooter(); + + return `/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; + +${footer} +`; + } + + async getCreateTable (tableName) { + const { rows } = await this._client.raw( + `SHOW CREATE TABLE \`${this.schemaName}\`.\`${tableName}\`` + ); + + if (rows.length !== 1) return ''; + + const col = 'Create View' in rows[0] ? 'Create View' : 'Create Table'; + + return rows[0][col] + ';'; + } + + getDropTable (tableName) { + return `DROP TABLE IF EXISTS \`${tableName}\`;`; + } + + async * getTableInsert (tableName) { + let rowCount = 0; + let sqlStr = ''; + + const countResults = await this._client.raw(`SELECT COUNT(1) as count FROM \`${this.schemaName}\`.\`${tableName}\``); + if (countResults.rows.length === 1) rowCount = countResults.rows[0].count; + + if (rowCount > 0) { + let queryLength = 0; + let rowsWritten = 0; + let rowIndex = 0; + const { sqlInsertDivider, sqlInsertAfter } = this._options; + const columns = await this._client.getTableColumns({ + table: tableName, + schema: this.schemaName + }); + + const notGeneratedColumns = columns.filter(col => !col.generated); + const columnNames = notGeneratedColumns.map(col => '`' + col.name + '`'); + const insertStmt = `INSERT INTO \`${tableName}\` (${columnNames.join( + ', ' + )}) VALUES`; + + sqlStr += `LOCK TABLES \`${tableName}\` WRITE;\n`; + sqlStr += `/*!40000 ALTER TABLE \`${tableName}\` DISABLE KEYS */;`; + sqlStr += '\n\n'; + yield sqlStr; + + yield insertStmt; + + const stream = await this._queryStream( + `SELECT ${columnNames.join(', ')} FROM \`${this.schemaName}\`.\`${tableName}\`` + ); + + for await (const row of stream) { + if (this.isCancelled) { + stream.destroy(); + yield null; + return; + } + + let sqlInsertString = ''; + + if ( + (sqlInsertDivider === 'bytes' && queryLength >= sqlInsertAfter * 1024) || + (sqlInsertDivider === 'rows' && rowsWritten === sqlInsertAfter) + ) { + sqlInsertString += `;\n${insertStmt}\n\t(`; + queryLength = 0; + rowsWritten = 0; + } + else if (parseInt(rowIndex) === 0) sqlInsertString += '\n\t('; + else sqlInsertString += ',\n\t('; + + for (const i in notGeneratedColumns) { + const column = notGeneratedColumns[i]; + const val = row[column.name]; + + if (val === null) sqlInsertString += 'NULL'; + else if (DATE.includes(column.type)) { + sqlInsertString += moment(val).isValid() + ? this.escapeAndQuote(moment(val).format('YYYY-MM-DD')) + : val; + } + else if (DATETIME.includes(column.type)) { + let datePrecision = ''; + for (let i = 0; i < column.precision; i++) + datePrecision += i === 0 ? '.S' : 'S'; + + sqlInsertString += moment(val).isValid() + ? this.escapeAndQuote(moment(val).format(`YYYY-MM-DD HH:mm:ss${datePrecision}`)) + : this.escapeAndQuote(val); + } + else if (BIT.includes(column.type)) + sqlInsertString += `b'${hexToBinary(Buffer.from(val).toString('hex'))}'`; + else if (BLOB.includes(column.type)) + sqlInsertString += `X'${val.toString('hex').toUpperCase()}'`; + else if (NUMBER.includes(column.type)) + sqlInsertString += val; + else if (FLOAT.includes(column.type)) + sqlInsertString += parseFloat(val); + else if (SPATIAL.includes(column.type)) { + let geoJson; + if (IS_MULTI_SPATIAL.includes(column.type)) { + const features = []; + for (const element of val) + features.push(this.getMarkers(element)); + + geoJson = { + type: 'FeatureCollection', + features + }; + } + else + geoJson = this._getGeoJSON(val); + + sqlInsertString += `ST_GeomFromGeoJSON('${JSON.stringify(geoJson)}')`; + } + else if (val === '') sqlInsertString += '\'\''; + else { + sqlInsertString += typeof val === 'string' + ? this.escapeAndQuote(val) + : typeof val === 'object' + ? this.escapeAndQuote(JSON.stringify(val)) + : val; + } + + if (parseInt(i) !== notGeneratedColumns.length - 1) + sqlInsertString += ', '; + } + + sqlInsertString += ')'; + + queryLength += sqlInsertString.length; + rowsWritten++; + rowIndex++; + yield sqlInsertString; + } + + sqlStr = ';\n\n'; + sqlStr += `/*!40000 ALTER TABLE \`${tableName}\` ENABLE KEYS */;\n`; + sqlStr += 'UNLOCK TABLES;'; + + yield sqlStr; + } + } + + async getViews () { + const { rows: views } = await this._client.raw( + `SHOW TABLE STATUS FROM \`${this.schemaName}\` WHERE Comment = 'VIEW'` + ); + let sqlString = ''; + + for (const view of views) { + sqlString += `DROP VIEW IF EXISTS \`${view.Name}\`;\n`; + const viewSyntax = await this.getCreateTable(view.Name); + sqlString += viewSyntax.replaceAll('`' + this.schemaName + '`.', ''); + sqlString += '\n'; + } + + return sqlString; + } + + async getTriggers () { + const { rows: triggers } = await this._client.raw( + `SHOW TRIGGERS FROM \`${this.schemaName}\`` + ); + const generatedTables = this._tables + .filter(t => t.includeStructure) + .map(t => t.table); + + let sqlString = ''; + + for (const trigger of triggers) { + const { + Trigger: name, + Timing: timing, + Event: event, + Table: table, + Statement: statement, + sql_mode: sqlMode + } = trigger; + + if (!generatedTables.includes(table)) continue; + + const definer = this.getEscapedDefiner(trigger.Definer); + sqlString += '/*!50003 SET @OLD_SQL_MODE=@@SQL_MODE*/;;\n'; + sqlString += `/*!50003 SET SQL_MODE="${sqlMode}" */;\n`; + sqlString += 'DELIMITER ;;\n'; + sqlString += '/*!50003 CREATE*/ '; + sqlString += `/*!50017 DEFINER=${definer}*/ `; + sqlString += `/*!50003 TRIGGER \`${name}\` ${timing} ${event} ON \`${table}\` FOR EACH ROW ${statement}*/;;\n`; + sqlString += 'DELIMITER ;\n'; + sqlString += '/*!50003 SET SQL_MODE=@OLD_SQL_MODE */;\n\n'; + } + + return sqlString; + } + + async getSchedulers () { + const { rows: schedulers } = await this._client.raw( + `SELECT *, EVENT_SCHEMA AS \`Db\`, EVENT_NAME AS \`Name\` FROM information_schema.\`EVENTS\` WHERE EVENT_SCHEMA = '${this.schemaName}'` + ); + let sqlString = ''; + + for (const scheduler of schedulers) { + const { + EVENT_NAME: name, + SQL_MODE: sqlMode, + EVENT_TYPE: type, + INTERVAL_VALUE: intervalValue, + INTERVAL_FIELD: intervalField, + STARTS: starts, + ENDS: ends, + EXECUTE_AT: at, + ON_COMPLETION: onCompletion, + STATUS: status, + EVENT_DEFINITION: definition + } = scheduler; + + const definer = this.getEscapedDefiner(scheduler.DEFINER); + const comment = this.escapeAndQuote(scheduler.EVENT_COMMENT); + + sqlString += `/*!50106 DROP EVENT IF EXISTS \`${name}\` */;\n`; + sqlString += '/*!50003 SET @OLD_SQL_MODE=@@SQL_MODE*/;;\n'; + sqlString += `/*!50003 SET SQL_MODE='${sqlMode}' */;\n`; + sqlString += 'DELIMITER ;;\n'; + sqlString += '/*!50106 CREATE*/ '; + sqlString += `/*!50117 DEFINER=${definer}*/ `; + sqlString += `/*!50106 EVENT \`${name}\` ON SCHEDULE `; + if (type === 'RECURRING') { + sqlString += `EVERY ${intervalValue} ${intervalField} STARTS '${starts}' `; + + if (ends) sqlString += `ENDS '${ends}' `; + } + else sqlString += `AT '${at}' `; + sqlString += `ON COMPLETION ${onCompletion} ${ + status === 'disabled' ? 'DISABLE' : 'ENABLE' + } COMMENT ${comment || '\'\''} DO ${definition}*/;;\n`; + sqlString += 'DELIMITER ;\n'; + sqlString += '/*!50003 SET SQL_MODE=@OLD_SQL_MODE*/;;\n'; + } + + return sqlString; + } + + async getFunctions () { + const { rows: functions } = await this._client.raw( + `SHOW FUNCTION STATUS WHERE \`Db\` = '${this.schemaName}';` + ); + + let sqlString = ''; + + for (const func of functions) { + const definer = this.getEscapedDefiner(func.Definer); + sqlString += await this.getRoutineSyntax( + func.Name, + func.Type, + definer + ); + } + + return sqlString; + } + + async getRoutines () { + const { rows: routines } = await this._client.raw( + `SHOW PROCEDURE STATUS WHERE \`Db\` = '${this.schemaName}';` + ); + + let sqlString = ''; + + for (const routine of routines) { + const definer = this.getEscapedDefiner(routine.Definer); + + sqlString += await this.getRoutineSyntax( + routine.Name, + routine.Type, + definer + ); + } + + return sqlString; + } + + async getRoutineSyntax (name, type, definer) { + const { rows: routines } = await this._client.raw( + `SHOW CREATE ${type} \`${this.schemaName}\`.\`${name}\`` + ); + + if (routines.length === 0) return ''; + + const routine = routines[0]; + + const fieldName = `Create ${type === 'PROCEDURE' ? 'Procedure' : 'Function'}`; + const sqlMode = routine.sql_mode; + const createProcedure = routine[fieldName]; + let sqlString = ''; + + if (createProcedure) { // If procedure body not empty + const startOffset = createProcedure.indexOf(type); + const procedureBody = createProcedure.substring(startOffset); + + sqlString += `/*!50003 DROP ${type} IF EXISTS ${name}*/;;\n`; + sqlString += '/*!50003 SET @OLD_SQL_MODE=@@SQL_MODE*/;;\n'; + sqlString += `/*!50003 SET SQL_MODE="${sqlMode}"*/;;\n`; + sqlString += 'DELIMITER ;;\n'; + sqlString += `/*!50003 CREATE*/ /*!50020 DEFINER=${definer}*/ /*!50003 ${procedureBody}*/;;\n`; + sqlString += 'DELIMITER ;\n'; + sqlString += '/*!50003 SET SQL_MODE=@OLD_SQL_MODE*/;\n'; + } + + return sqlString; + } + + async _queryStream (sql) { + if (process.env.NODE_ENV === 'development') console.log('EXPORTER:', sql); + const isPool = typeof this._client._connection.getConnection === 'function'; + const connection = isPool ? await this._client._connection.getConnection() : this._client._connection; + const stream = connection.connection.query(sql).stream(); + const dispose = () => connection.destroy(); + + stream.on('end', dispose); + stream.on('error', dispose); + stream.on('close', dispose); + return stream; + } + + getEscapedDefiner (definer) { + return definer + .split('@') + .map(part => '`' + part + '`') + .join('@'); + } + + escapeAndQuote (val) { + // eslint-disable-next-line no-control-regex + const CHARS_TO_ESCAPE = /[\0\b\t\n\r\x1a"'\\]/g; + const CHARS_ESCAPE_MAP = { + '\0': '\\0', + '\b': '\\b', + '\t': '\\t', + '\n': '\\n', + '\r': '\\r', + '\x1a': '\\Z', + '"': '\\"', + '\'': '\\\'', + '\\': '\\\\' + }; + let chunkIndex = CHARS_TO_ESCAPE.lastIndex = 0; + let escapedVal = ''; + let match; + + while ((match = CHARS_TO_ESCAPE.exec(val))) { + escapedVal += val.slice(chunkIndex, match.index) + CHARS_ESCAPE_MAP[match[0]]; + chunkIndex = CHARS_TO_ESCAPE.lastIndex; + } + + if (chunkIndex === 0) + return `'${val}'`; + + if (chunkIndex < val.length) + return `'${escapedVal + val.slice(chunkIndex)}'`; + + return `'${escapedVal}'`; + } + + _getGeoJSON (val) { + if (Array.isArray(val)) { + if (getArrayDepth(val) === 1) + return lineString(val.reduce((acc, curr) => [...acc, [curr.x, curr.y]], [])); + else + return polygon(val.map(arr => arr.reduce((acc, curr) => [...acc, [curr.x, curr.y]], []))); + } + else + return point([val.x, val.y]); + } +} diff --git a/src/main/libs/exporters/sql/SqlExporter.js b/src/main/libs/exporters/sql/SqlExporter.js new file mode 100644 index 00000000..21164201 --- /dev/null +++ b/src/main/libs/exporters/sql/SqlExporter.js @@ -0,0 +1,162 @@ +import moment from 'moment'; +import { BaseExporter } from '../BaseExporter'; + +export class SqlExporter extends BaseExporter { + constructor (client, tables, options) { + super(tables, options); + this._client = client; + this._commentChar = '#'; + } + + get schemaName () { + return this._options.schema; + } + + get host () { + return this._client._params.host; + } + + async getServerVersion () { + const version = await this._client.getVersion(); + return `${version.name} ${version.number}`; + } + + async dump () { + const { includes } = this._options; + const extraItems = Object.keys(includes).filter(key => includes[key]); + const totalTableToProcess = this._tables.filter( + t => t.includeStructure || t.includeContent || t.includeDropStatement + ).length; + const processingItemCount = totalTableToProcess + extraItems.length; + + const exportState = { + totalItems: processingItemCount, + currentItemIndex: 0, + currentItem: '', + op: '' + }; + + const header = await this.getSqlHeader(); + this.writeString(header); + this.writeString('\n\n\n'); + + for (const item of this._tables) { + // user abort operation + if (this.isCancelled) return; + + // skip item if not set to output any detail for them + if ( + !item.includeStructure && + !item.includeContent && + !item.includeDropStatement + ) + continue; + + exportState.currentItemIndex++; + exportState.currentItem = item.table; + exportState.op = 'FETCH'; + + this.emitUpdate(exportState); + + const tableHeader = this.buildComment( + `Dump of table ${item.table}\n------------------------------------------------------------` + ); + this.writeString(tableHeader); + this.writeString('\n\n'); + + if (item.includeDropStatement) { + const dropTableSyntax = this.getDropTable(item.table); + this.writeString(dropTableSyntax); + this.writeString('\n\n'); + } + + if (item.includeStructure) { + const createTableSyntax = await this.getCreateTable(item.table); + this.writeString(createTableSyntax); + this.writeString('\n\n'); + } + + if (item.includeContent) { + exportState.op = 'WRITE'; + this.emitUpdate(exportState); + for await (const sqlStr of this.getTableInsert(item.table)) { + if (this.isCancelled) return; + this.writeString(sqlStr); + } + + this.writeString('\n\n'); + } + + this.writeString('\n\n'); + } + + for (const item of extraItems) { + const processingMethod = `get${item.charAt(0).toUpperCase() + item.slice(1)}`; + exportState.currentItemIndex++; + exportState.currentItem = item; + exportState.op = 'PROCESSING'; + this.emitUpdate(exportState); + + if (this[processingMethod]) { + const data = await this[processingMethod](); + if (data !== '') { + const header = + this.buildComment( + `Dump of ${item}\n------------------------------------------------------------` + ) + '\n\n'; + + this.writeString(header); + this.writeString(data); + this.writeString('\n\n'); + } + } + } + + const footer = await this.getFooter(); + this.writeString(footer); + } + + buildComment (text) { + return text + .split('\n') + .map(txt => `${this._commentChar} ${txt}`) + .join('\n'); + } + + async getSqlHeader () { + const serverVersion = await this.getServerVersion(); + const header = `************************************************************ +Antares - SQL Client +Version ${process.env.PACKAGE_VERSION} + +https://antares-sql.app/ +https://github.com/Fabio286/antares + +Host: ${this.host} (${serverVersion}) +Database: ${this.schemaName} +Generation time: ${moment().format()} +************************************************************`; + + return this.buildComment(header); + } + + async getFooter () { + return this.buildComment(`Dump completed on ${moment().format()}`); + } + + getCreateTable (tableName) { + throw new Error( + 'Sql Exporter must implement the "getCreateTable" method' + ); + } + + getDropTable (tableName) { + throw new Error('Sql Exporter must implement the "getDropTable" method'); + } + + getTableInsert (tableName) { + throw new Error( + 'Sql Exporter must implement the "getTableInsert" method' + ); + } +} diff --git a/src/main/libs/importers/BaseImporter.js b/src/main/libs/importers/BaseImporter.js new file mode 100644 index 00000000..901a5d98 --- /dev/null +++ b/src/main/libs/importers/BaseImporter.js @@ -0,0 +1,53 @@ +import fs from 'fs'; +import EventEmitter from 'events'; + +export class BaseImporter extends EventEmitter { + constructor (options) { + super(); + this._options = options; + this._isCancelled = false; + this._fileHandler = fs.createReadStream(this._options.file, { + flags: 'r', + highWaterMark: 4 * 1024 + }); + this._state = {}; + + this._fileHandler.once('error', err => { + this._isCancelled = true; + this.emit('error', err); + }); + } + + async run () { + try { + this.emit('start', this); + await this.import(); + } + catch (err) { + this.emit('error', err); + throw err; + } + finally { + this._fileHandler.close(); + this.emit('end'); + } + } + + get isCancelled () { + return this._isCancelled; + } + + cancel () { + this._isCancelled = true; + this.emit('cancel'); + this.emitUpdate({ op: 'cancelling' }); + } + + emitUpdate (state) { + this.emit('progress', { ...this._state, ...state }); + } + + import () { + throw new Error('Exporter must implement the "import" method'); + } +} diff --git a/src/main/libs/importers/sql/MysqlImporter.js b/src/main/libs/importers/sql/MysqlImporter.js new file mode 100644 index 00000000..05c39941 --- /dev/null +++ b/src/main/libs/importers/sql/MysqlImporter.js @@ -0,0 +1,85 @@ +import fs from 'fs/promises'; +import SqlParser from '../../../../common/libs/sqlParser'; +import { BaseImporter } from '../BaseImporter'; + +export default class MysqlImporter extends BaseImporter { + constructor (client, options) { + super(options); + this._client = client; + } + + async import () { + try { + const { size: totalFileSize } = await fs.stat(this._options.file); + const parser = new SqlParser(); + let readPosition = 0; + let queryCount = 0; + + this.emitUpdate({ + fileSize: totalFileSize, + readPosition: 0, + percentage: 0, + queryCount: 0 + }); + + // 1. detect file encoding + // 2. set fh encoding + // 3. detect sql mode + // 4. restore sql mode in case of exception + + return new Promise((resolve, reject) => { + this._fileHandler.pipe(parser); + + parser.on('error', reject); + + parser.on('close', async () => { + console.log('TOTAL QUERIES', queryCount); + console.log('import end'); + resolve(); + }); + + parser.on('data', async (query) => { + queryCount++; + parser.pause(); + + try { + await this._client.query(query); + } + catch (error) { + this.emit('query-error', { + sql: query, + message: error.sqlMessage, + sqlSnippet: error.sql, + time: new Date().getTime() + }); + } + + this.emitUpdate({ + queryCount, + readPosition, + percentage: readPosition / totalFileSize * 100 + }); + this._fileHandler.pipe(parser); + parser.resume(); + }); + + parser.on('pause', () => { + this._fileHandler.unpipe(parser); + this._fileHandler.readableFlowing = false; + }); + + this._fileHandler.on('data', (chunk) => { + readPosition += chunk.length; + }); + + this._fileHandler.on('error', (err) => { + console.log(err); + reject(err); + }); + }); + } + catch (err) { + console.log(err); + } + } +} diff --git a/src/main/workers/exporter.js b/src/main/workers/exporter.js new file mode 100644 index 00000000..43a0aff0 --- /dev/null +++ b/src/main/workers/exporter.js @@ -0,0 +1,60 @@ +import { ClientsFactory } from '../libs/ClientsFactory'; +import MysqlExporter from '../libs/exporters/sql/MysqlExporter.js'; +import fs from 'fs'; +let exporter; + +process.on('message', async ({ type, client, tables, options }) => { + if (type === 'init') { + const connection = await ClientsFactory.getClient({ + client: client.name, + params: client.config, + poolSize: 5 + }); + await connection.connect(); + + switch (client.name) { + case 'mysql': + case 'maria': + exporter = new MysqlExporter(connection, tables, options); + break; + default: + process.send({ + type: 'error', + payload: `"${client.name}" exporter not aviable` + }); + return; + } + + exporter.once('error', err => { + console.error(err); + process.send({ + type: 'error', + payload: err.toString() + }); + }); + + exporter.once('end', () => { + process.send({ + type: 'end', + payload: { cancelled: exporter.isCancelled } + }); + connection.destroy(); + }); + + exporter.once('cancel', () => { + fs.unlinkSync(exporter.outputFile); + process.send({ type: 'cancel' }); + }); + + exporter.on('progress', state => { + process.send({ + type: 'export-progress', + payload: state + }); + }); + + exporter.run(); + } + else if (type === 'cancel') + exporter.cancel(); +}); diff --git a/src/main/workers/importer.js b/src/main/workers/importer.js new file mode 100644 index 00000000..672c8a11 --- /dev/null +++ b/src/main/workers/importer.js @@ -0,0 +1,68 @@ +import { ClientsFactory } from '../libs/ClientsFactory'; +import MysqlImporter from '../libs/importers/sql/MysqlImporter'; +let importer; + +process.on('message', async ({ type, dbConfig, options }) => { + if (type === 'init') { + const connection = await ClientsFactory.getClient({ + client: options.type, + params: { + ...dbConfig, + schema: options.schema + }, + poolSize: 1 + }); + + const pool = await connection.getConnectionPool(); + + switch (options.type) { + case 'mysql': + case 'maria': + importer = new MysqlImporter(pool, options); + break; + default: + process.send({ + type: 'error', + payload: `"${options.type}" importer not aviable` + }); + return; + } + + importer.once('error', err => { + console.error(err); + process.send({ + type: 'error', + payload: err.toString() + }); + }); + + importer.once('end', () => { + process.send({ + type: 'end', + payload: { cancelled: importer.isCancelled } + }); + }); + + importer.once('cancel', () => { + process.send({ type: 'cancel' }); + }); + + importer.on('progress', state => { + process.send({ + type: 'import-progress', + payload: state + }); + }); + + importer.on('query-error', state => { + process.send({ + type: 'query-error', + payload: state + }); + }); + + importer.run(); + } + else if (type === 'cancel') + importer.cancel(); +}); diff --git a/src/renderer/components/BaseConfirmModal.vue b/src/renderer/components/BaseConfirmModal.vue index e717a7ea..e562d9fa 100644 --- a/src/renderer/components/BaseConfirmModal.vue +++ b/src/renderer/components/BaseConfirmModal.vue @@ -110,5 +110,4 @@ export default { .modal.modal-sm .modal-container { padding: 0; } - diff --git a/src/renderer/components/ModalExportSchema.vue b/src/renderer/components/ModalExportSchema.vue new file mode 100644 index 00000000..c7a5d02a --- /dev/null +++ b/src/renderer/components/ModalExportSchema.vue @@ -0,0 +1,491 @@ + + + + + diff --git a/src/renderer/components/ModalImportSchema.vue b/src/renderer/components/ModalImportSchema.vue new file mode 100644 index 00000000..e6886f30 --- /dev/null +++ b/src/renderer/components/ModalImportSchema.vue @@ -0,0 +1,181 @@ +