mirror of
				https://github.com/Fabio286/antares.git
				synced 2025-06-05 21:59:22 +02:00 
			
		
		
		
	refactor: db exporter ts refactor
This commit is contained in:
		| @@ -1,5 +1,9 @@ | |||||||
| import * as mysql from 'mysql2/promise'; | import * as mysql from 'mysql2/promise'; | ||||||
| import * as pg from 'pg'; | import * as pg from 'pg'; | ||||||
|  | import MysqlExporter from 'src/main/libs/exporters/sql/MysqlExporter'; | ||||||
|  | import PostgreSQLExporter from 'src/main/libs/exporters/sql/PostgreSQLExporter'; | ||||||
|  | import MySQLImporter from 'src/main/libs/importers/sql/MysqlImporter'; | ||||||
|  | import PostgreSQLImporter from 'src/main/libs/importers/sql/PostgreSQLImporter'; | ||||||
| import SSHConfig from 'ssh2-promise/lib/sshConfig'; | import SSHConfig from 'ssh2-promise/lib/sshConfig'; | ||||||
| import { MySQLClient } from '../../main/libs/clients/MySQLClient'; | import { MySQLClient } from '../../main/libs/clients/MySQLClient'; | ||||||
| import { PostgreSQLClient } from '../../main/libs/clients/PostgreSQLClient'; | import { PostgreSQLClient } from '../../main/libs/clients/PostgreSQLClient'; | ||||||
| @@ -7,6 +11,8 @@ import { SQLiteClient } from '../../main/libs/clients/SQLiteClient'; | |||||||
|  |  | ||||||
| export type Client = MySQLClient | PostgreSQLClient | SQLiteClient | export type Client = MySQLClient | PostgreSQLClient | SQLiteClient | ||||||
| export type ClientCode = 'mysql' | 'maria' | 'pg' | 'sqlite' | export type ClientCode = 'mysql' | 'maria' | 'pg' | 'sqlite' | ||||||
|  | export type Exporter = MysqlExporter | PostgreSQLExporter | ||||||
|  | export type Importer = MySQLImporter | PostgreSQLImporter | ||||||
|  |  | ||||||
| /** | /** | ||||||
|  * Pasameters needed to create a new Antares connection to a database |  * Pasameters needed to create a new Antares connection to a database | ||||||
|   | |||||||
							
								
								
									
										28
									
								
								src/common/interfaces/exporter.ts
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										28
									
								
								src/common/interfaces/exporter.ts
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,28 @@ | |||||||
|  | export interface TableParams { | ||||||
|  |    table: string; | ||||||
|  |    includeStructure: boolean; | ||||||
|  |    includeContent: boolean; | ||||||
|  |    includeDropStatement: boolean; | ||||||
|  | } | ||||||
|  |  | ||||||
|  | export interface ExportOptions { | ||||||
|  |    schema: string; | ||||||
|  |    includes: { | ||||||
|  |       functions: boolean; | ||||||
|  |       views: boolean; | ||||||
|  |       triggers: boolean; | ||||||
|  |       routines: boolean; | ||||||
|  |       schedulers: boolean; | ||||||
|  |    }; | ||||||
|  |    outputFormat: 'sql' | 'sql.zip'; | ||||||
|  |    outputFile: string; | ||||||
|  |    sqlInsertAfter: number; | ||||||
|  |    sqlInsertDivider: 'bytes' | 'rows'; | ||||||
|  | } | ||||||
|  |  | ||||||
|  | export interface ExportState { | ||||||
|  |    totalItems?: number; | ||||||
|  |    currentItemIndex?: number; | ||||||
|  |    currentItem?: string; | ||||||
|  |    op?: string; | ||||||
|  | } | ||||||
| @@ -1,6 +1,6 @@ | |||||||
| import * as antares from 'common/interfaces/antares'; | import * as antares from 'common/interfaces/antares'; | ||||||
| import * as workers from 'common/interfaces/workers'; | import * as workers from 'common/interfaces/workers'; | ||||||
| import fs from 'fs'; | import * as fs from 'fs'; | ||||||
| import path from 'path'; | import path from 'path'; | ||||||
| import { ChildProcess, fork } from 'child_process'; | import { ChildProcess, fork } from 'child_process'; | ||||||
| import { ipcMain, dialog } from 'electron'; | import { ipcMain, dialog } from 'electron'; | ||||||
|   | |||||||
| @@ -9,8 +9,8 @@ export class MySQLClient extends AntaresCore { | |||||||
|    private _schema?: string; |    private _schema?: string; | ||||||
|    private _runningConnections: Map<string, number>; |    private _runningConnections: Map<string, number>; | ||||||
|    private _connectionsToCommit: Map<string, mysql.Connection | mysql.PoolConnection>; |    private _connectionsToCommit: Map<string, mysql.Connection | mysql.PoolConnection>; | ||||||
|    protected _connection?: mysql.Connection | mysql.Pool; |    _connection?: mysql.Connection | mysql.Pool; | ||||||
|    protected _params: mysql.ConnectionOptions & {schema: string; ssl?: mysql.SslOptions; ssh?: SSHConfig; readonly: boolean}; |    _params: mysql.ConnectionOptions & {schema: string; ssl?: mysql.SslOptions; ssh?: SSHConfig; readonly: boolean}; | ||||||
|  |  | ||||||
|    private types: {[key: number]: string} = { |    private types: {[key: number]: string} = { | ||||||
|       0: 'DECIMAL', |       0: 'DECIMAL', | ||||||
| @@ -445,18 +445,18 @@ export class MySQLClient extends AntaresCore { | |||||||
|    async getTableColumns ({ schema, table }: { schema: string; table: string }) { |    async getTableColumns ({ schema, table }: { schema: string; table: string }) { | ||||||
|       interface TableColumnsResult { |       interface TableColumnsResult { | ||||||
|          COLUMN_TYPE: string; |          COLUMN_TYPE: string; | ||||||
|          NUMERIC_PRECISION: string; |          NUMERIC_PRECISION: number; | ||||||
|          COLUMN_NAME: string; |          COLUMN_NAME: string; | ||||||
|          COLUMN_DEFAULT: string; |          COLUMN_DEFAULT: string; | ||||||
|          COLUMN_KEY: string; |          COLUMN_KEY: string; | ||||||
|          DATA_TYPE: string; |          DATA_TYPE: string; | ||||||
|          TABLE_SCHEMA: string; |          TABLE_SCHEMA: string; | ||||||
|          TABLE_NAME: string; |          TABLE_NAME: string; | ||||||
|          NUMERIC_SCALE: string; |          NUMERIC_SCALE: number; | ||||||
|          DATETIME_PRECISION: string; |          DATETIME_PRECISION: number; | ||||||
|          CHARACTER_MAXIMUM_LENGTH: string; |          CHARACTER_MAXIMUM_LENGTH: number; | ||||||
|          IS_NULLABLE: string; |          IS_NULLABLE: string; | ||||||
|          ORDINAL_POSITION: string; |          ORDINAL_POSITION: number; | ||||||
|          CHARACTER_SET_NAME: string; |          CHARACTER_SET_NAME: string; | ||||||
|          COLLATION_NAME: string; |          COLLATION_NAME: string; | ||||||
|          EXTRA: string; |          EXTRA: string; | ||||||
|   | |||||||
| @@ -24,7 +24,6 @@ export class PostgreSQLClient extends AntaresCore { | |||||||
|    private _runningConnections: Map<string, number>; |    private _runningConnections: Map<string, number>; | ||||||
|    private _connectionsToCommit: Map<string, pg.Client | pg.PoolClient>; |    private _connectionsToCommit: Map<string, pg.Client | pg.PoolClient>; | ||||||
|    protected _connection?: pg.Client | pg.Pool; |    protected _connection?: pg.Client | pg.Pool; | ||||||
|    protected _params: pg.ClientConfig & {schema: string; ssl?: mysql.SslOptions; ssh?: SSHConfig; readonly: boolean}; |  | ||||||
|    private types: {[key: string]: string} = {}; |    private types: {[key: string]: string} = {}; | ||||||
|    private _arrayTypes: {[key: string]: string} = { |    private _arrayTypes: {[key: string]: string} = { | ||||||
|       _int2: 'SMALLINT', |       _int2: 'SMALLINT', | ||||||
| @@ -36,6 +35,8 @@ export class PostgreSQLClient extends AntaresCore { | |||||||
|       _varchar: 'CHARACTER VARYING' |       _varchar: 'CHARACTER VARYING' | ||||||
|    } |    } | ||||||
|  |  | ||||||
|  |    _params: pg.ClientConfig & {schema: string; ssl?: mysql.SslOptions; ssh?: SSHConfig; readonly: boolean}; | ||||||
|  |  | ||||||
|    constructor (args: antares.ClientParams) { |    constructor (args: antares.ClientParams) { | ||||||
|       super(args); |       super(args); | ||||||
|  |  | ||||||
| @@ -173,6 +174,10 @@ export class PostgreSQLClient extends AntaresCore { | |||||||
|       } |       } | ||||||
|    } |    } | ||||||
|  |  | ||||||
|  |    getCollations (): null[] { | ||||||
|  |       return []; | ||||||
|  |    } | ||||||
|  |  | ||||||
|    async getStructure (schemas: Set<string>) { |    async getStructure (schemas: Set<string>) { | ||||||
|       /* eslint-disable camelcase */ |       /* eslint-disable camelcase */ | ||||||
|       interface ShowTableResult { |       interface ShowTableResult { | ||||||
|   | |||||||
| @@ -8,7 +8,7 @@ export class SQLiteClient extends AntaresCore { | |||||||
|    private _schema?: string; |    private _schema?: string; | ||||||
|    private _connectionsToCommit: Map<string, sqlite.Database>; |    private _connectionsToCommit: Map<string, sqlite.Database>; | ||||||
|    protected _connection?: sqlite.Database; |    protected _connection?: sqlite.Database; | ||||||
|    protected _params: { databasePath: string; readonly: boolean}; |    _params: { databasePath: string; readonly: boolean}; | ||||||
|  |  | ||||||
|    constructor (args: antares.ClientParams) { |    constructor (args: antares.ClientParams) { | ||||||
|       super(args); |       super(args); | ||||||
|   | |||||||
| @@ -1,10 +1,18 @@ | |||||||
| import fs from 'fs'; | import * as exporter from 'common/interfaces/exporter'; | ||||||
| import { createGzip } from 'zlib'; | import * as fs from 'fs'; | ||||||
| import path from 'path'; | import { createGzip, Gzip } from 'zlib'; | ||||||
| import EventEmitter from 'events'; | import * as path from 'path'; | ||||||
|  | import * as EventEmitter from 'events'; | ||||||
| 
 | 
 | ||||||
| export class BaseExporter extends EventEmitter { | export class BaseExporter extends EventEmitter { | ||||||
|    constructor (tables, options) { |    protected _tables; | ||||||
|  |    protected _options; | ||||||
|  |    protected _isCancelled; | ||||||
|  |    protected _outputFileStream: fs.WriteStream; | ||||||
|  |    protected _processedStream: fs.WriteStream | Gzip; | ||||||
|  |    protected _state; | ||||||
|  | 
 | ||||||
|  |    constructor (tables: exporter.TableParams[], options: exporter.ExportOptions) { | ||||||
|       super(); |       super(); | ||||||
|       this._tables = tables; |       this._tables = tables; | ||||||
|       this._options = options; |       this._options = options; | ||||||
| @@ -60,11 +68,11 @@ export class BaseExporter extends EventEmitter { | |||||||
|       this.emitUpdate({ op: 'cancelling' }); |       this.emitUpdate({ op: 'cancelling' }); | ||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
|    emitUpdate (state) { |    emitUpdate (state: exporter.ExportState) { | ||||||
|       this.emit('progress', { ...this._state, ...state }); |       this.emit('progress', { ...this._state, ...state }); | ||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
|    writeString (data) { |    writeString (data: string) { | ||||||
|       if (this._isCancelled) return; |       if (this._isCancelled) return; | ||||||
| 
 | 
 | ||||||
|       try { |       try { | ||||||
| @@ -1,14 +1,20 @@ | |||||||
|  | import * as exporter from 'common/interfaces/exporter'; | ||||||
|  | import * as mysql from 'mysql2/promise'; | ||||||
| import { SqlExporter } from './SqlExporter'; | import { SqlExporter } from './SqlExporter'; | ||||||
| import { BLOB, BIT, DATE, DATETIME, FLOAT, SPATIAL, IS_MULTI_SPATIAL, NUMBER } from 'common/fieldTypes'; | import { BLOB, BIT, DATE, DATETIME, FLOAT, SPATIAL, IS_MULTI_SPATIAL, NUMBER } from 'common/fieldTypes'; | ||||||
| import hexToBinary from 'common/libs/hexToBinary'; | import hexToBinary from 'common/libs/hexToBinary'; | ||||||
| import { getArrayDepth } from 'common/libs/getArrayDepth'; | import { getArrayDepth } from 'common/libs/getArrayDepth'; | ||||||
| import moment from 'moment'; | import * as moment from 'moment'; | ||||||
| import { lineString, point, polygon } from '@turf/helpers'; | import { lineString, point, polygon } from '@turf/helpers'; | ||||||
|  | import { MySQLClient } from '../../clients/MySQLClient'; | ||||||
| 
 | 
 | ||||||
| export default class MysqlExporter extends SqlExporter { | export default class MysqlExporter extends SqlExporter { | ||||||
|    constructor (...args) { |    protected _client: MySQLClient; | ||||||
|       super(...args); |  | ||||||
| 
 | 
 | ||||||
|  |    constructor (client: MySQLClient, tables: exporter.TableParams[], options: exporter.ExportOptions) { | ||||||
|  |       super(tables, options); | ||||||
|  | 
 | ||||||
|  |       this._client = client; | ||||||
|       this._commentChar = '#'; |       this._commentChar = '#'; | ||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
| @@ -42,7 +48,7 @@ ${footer} | |||||||
| `;
 | `;
 | ||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
|    async getCreateTable (tableName) { |    async getCreateTable (tableName: string) { | ||||||
|       const { rows } = await this._client.raw( |       const { rows } = await this._client.raw( | ||||||
|          `SHOW CREATE TABLE \`${this.schemaName}\`.\`${tableName}\`` |          `SHOW CREATE TABLE \`${this.schemaName}\`.\`${tableName}\`` | ||||||
|       ); |       ); | ||||||
| @@ -54,11 +60,11 @@ ${footer} | |||||||
|       return rows[0][col] + ';'; |       return rows[0][col] + ';'; | ||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
|    getDropTable (tableName) { |    getDropTable (tableName: string) { | ||||||
|       return `DROP TABLE IF EXISTS \`${tableName}\`;`; |       return `DROP TABLE IF EXISTS \`${tableName}\`;`; | ||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
|    async * getTableInsert (tableName) { |    async * getTableInsert (tableName: string) { | ||||||
|       let rowCount = 0; |       let rowCount = 0; | ||||||
|       let sqlStr = ''; |       let sqlStr = ''; | ||||||
| 
 | 
 | ||||||
| @@ -109,7 +115,7 @@ ${footer} | |||||||
|                queryLength = 0; |                queryLength = 0; | ||||||
|                rowsWritten = 0; |                rowsWritten = 0; | ||||||
|             } |             } | ||||||
|             else if (parseInt(rowIndex) === 0) sqlInsertString += '\n\t('; |             else if (rowIndex === 0) sqlInsertString += '\n\t('; | ||||||
|             else sqlInsertString += ',\n\t('; |             else sqlInsertString += ',\n\t('; | ||||||
| 
 | 
 | ||||||
|             for (const i in notGeneratedColumns) { |             for (const i in notGeneratedColumns) { | ||||||
| @@ -124,7 +130,7 @@ ${footer} | |||||||
|                } |                } | ||||||
|                else if (DATETIME.includes(column.type)) { |                else if (DATETIME.includes(column.type)) { | ||||||
|                   let datePrecision = ''; |                   let datePrecision = ''; | ||||||
|                   for (let i = 0; i < column.precision; i++) |                   for (let i = 0; i < column.datePrecision; i++) | ||||||
|                      datePrecision += i === 0 ? '.S' : 'S'; |                      datePrecision += i === 0 ? '.S' : 'S'; | ||||||
| 
 | 
 | ||||||
|                   sqlInsertString += moment(val).isValid() |                   sqlInsertString += moment(val).isValid() | ||||||
| @@ -144,7 +150,7 @@ ${footer} | |||||||
|                   if (IS_MULTI_SPATIAL.includes(column.type)) { |                   if (IS_MULTI_SPATIAL.includes(column.type)) { | ||||||
|                      const features = []; |                      const features = []; | ||||||
|                      for (const element of val) |                      for (const element of val) | ||||||
|                         features.push(this.getMarkers(element)); |                         features.push(this._getGeoJSON(element)); | ||||||
| 
 | 
 | ||||||
|                      geoJson = { |                      geoJson = { | ||||||
|                         type: 'FeatureCollection', |                         type: 'FeatureCollection', | ||||||
| @@ -323,7 +329,7 @@ ${footer} | |||||||
|       return sqlString; |       return sqlString; | ||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
|    async getRoutineSyntax (name, type, definer) { |    async getRoutineSyntax (name: string, type: string, definer: string) { | ||||||
|       const { rows: routines } = await this._client.raw( |       const { rows: routines } = await this._client.raw( | ||||||
|          `SHOW CREATE ${type} \`${this.schemaName}\`.\`${name}\`` |          `SHOW CREATE ${type} \`${this.schemaName}\`.\`${name}\`` | ||||||
|       ); |       ); | ||||||
| @@ -353,12 +359,13 @@ ${footer} | |||||||
|       return sqlString; |       return sqlString; | ||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
|    async _queryStream (sql) { |    async _queryStream (sql: string) { | ||||||
|       if (process.env.NODE_ENV === 'development') console.log('EXPORTER:', sql); |       if (process.env.NODE_ENV === 'development') console.log('EXPORTER:', sql); | ||||||
|       const isPool = typeof this._client._connection.getConnection === 'function'; |       const isPool = 'getConnection' in this._client._connection; | ||||||
|       const connection = isPool ? await this._client._connection.getConnection() : this._client._connection; |       const connection = isPool ? await (this._client._connection as mysql.Pool).getConnection() : this._client._connection; | ||||||
|       const stream = connection.connection.query(sql).stream(); |       // eslint-disable-next-line @typescript-eslint/no-explicit-any
 | ||||||
|       const dispose = () => connection.destroy(); |       const stream = (connection as any).connection.query(sql).stream(); | ||||||
|  |       const dispose = () => (connection as mysql.PoolConnection).release(); | ||||||
| 
 | 
 | ||||||
|       stream.on('end', dispose); |       stream.on('end', dispose); | ||||||
|       stream.on('error', dispose); |       stream.on('error', dispose); | ||||||
| @@ -366,17 +373,17 @@ ${footer} | |||||||
|       return stream; |       return stream; | ||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
|    getEscapedDefiner (definer) { |    getEscapedDefiner (definer: string) { | ||||||
|       return definer |       return definer | ||||||
|          .split('@') |          .split('@') | ||||||
|          .map(part => '`' + part + '`') |          .map(part => '`' + part + '`') | ||||||
|          .join('@'); |          .join('@'); | ||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
|    escapeAndQuote (val) { |    escapeAndQuote (val: string) { | ||||||
|       // eslint-disable-next-line no-control-regex
 |       // eslint-disable-next-line no-control-regex
 | ||||||
|       const CHARS_TO_ESCAPE = /[\0\b\t\n\r\x1a"'\\]/g; |       const CHARS_TO_ESCAPE = /[\0\b\t\n\r\x1a"'\\]/g; | ||||||
|       const CHARS_ESCAPE_MAP = { |       const CHARS_ESCAPE_MAP: {[key: string]: string} = { | ||||||
|          '\0': '\\0', |          '\0': '\\0', | ||||||
|          '\b': '\\b', |          '\b': '\\b', | ||||||
|          '\t': '\\t', |          '\t': '\\t', | ||||||
| @@ -405,14 +412,16 @@ ${footer} | |||||||
|       return `'${escapedVal}'`; |       return `'${escapedVal}'`; | ||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
|    _getGeoJSON (val) { |    /* eslint-disable @typescript-eslint/no-explicit-any */ | ||||||
|  |    _getGeoJSON (val: any) { | ||||||
|       if (Array.isArray(val)) { |       if (Array.isArray(val)) { | ||||||
|          if (getArrayDepth(val) === 1) |          if (getArrayDepth(val) === 1) | ||||||
|             return lineString(val.reduce((acc, curr) => [...acc, [curr.x, curr.y]], [])); |             return lineString(val.reduce((acc, curr) => [...acc, [curr.x, curr.y]], [])); | ||||||
|          else |          else | ||||||
|             return polygon(val.map(arr => arr.reduce((acc, curr) => [...acc, [curr.x, curr.y]], []))); |             return polygon(val.map(arr => arr.reduce((acc: any, curr: any) => [...acc, [curr.x, curr.y]], []))); | ||||||
|       } |       } | ||||||
|       else |       else | ||||||
|          return point([val.x, val.y]); |          return point([val.x, val.y]); | ||||||
|    } |    } | ||||||
|  |    /* eslint-enable @typescript-eslint/no-explicit-any */ | ||||||
| } | } | ||||||
| @@ -1,12 +1,21 @@ | |||||||
|  | import * as antares from 'common/interfaces/antares'; | ||||||
|  | import * as exporter from 'common/interfaces/exporter'; | ||||||
| import { SqlExporter } from './SqlExporter'; | import { SqlExporter } from './SqlExporter'; | ||||||
| import { BLOB, BIT, DATE, DATETIME, FLOAT, NUMBER, TEXT_SEARCH } from 'common/fieldTypes'; | import { BLOB, BIT, DATE, DATETIME, FLOAT, NUMBER, TEXT_SEARCH } from 'common/fieldTypes'; | ||||||
| import hexToBinary from 'common/libs/hexToBinary'; | import hexToBinary from 'common/libs/hexToBinary'; | ||||||
| import { getArrayDepth } from 'common/libs/getArrayDepth'; | import * as moment from 'moment'; | ||||||
| import moment from 'moment'; | // eslint-disable-next-line @typescript-eslint/ban-ts-comment
 | ||||||
| import { lineString, point, polygon } from '@turf/helpers'; | // @ts-ignore
 | ||||||
| import QueryStream from 'pg-query-stream'; | import * as QueryStream from 'pg-query-stream'; | ||||||
|  | import { PostgreSQLClient } from '../../clients/PostgreSQLClient'; | ||||||
| 
 | 
 | ||||||
| export default class PostgreSQLExporter extends SqlExporter { | export default class PostgreSQLExporter extends SqlExporter { | ||||||
|  |    constructor (client: PostgreSQLClient, tables: exporter.TableParams[], options: exporter.ExportOptions) { | ||||||
|  |       super(tables, options); | ||||||
|  | 
 | ||||||
|  |       this._client = client; | ||||||
|  |    } | ||||||
|  | 
 | ||||||
|    async getSqlHeader () { |    async getSqlHeader () { | ||||||
|       let dump = await super.getSqlHeader(); |       let dump = await super.getSqlHeader(); | ||||||
|       dump += ` |       dump += ` | ||||||
| @@ -30,11 +39,28 @@ SET row_security = off;\n\n\n`; | |||||||
|       return dump; |       return dump; | ||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
|    async getCreateTable (tableName) { |    async getCreateTable (tableName: string) { | ||||||
|  |       /* eslint-disable camelcase */ | ||||||
|  |       interface SequenceRecord { | ||||||
|  |          sequence_catalog: string; | ||||||
|  |          sequence_schema: string; | ||||||
|  |          sequence_name: string; | ||||||
|  |          data_type: string; | ||||||
|  |          numeric_precision: number; | ||||||
|  |          numeric_precision_radix: number; | ||||||
|  |          numeric_scale: number; | ||||||
|  |          start_value: string; | ||||||
|  |          minimum_value: string; | ||||||
|  |          maximum_value: string; | ||||||
|  |          increment: string; | ||||||
|  |          cycle_option: string; | ||||||
|  |       } | ||||||
|  |       /* eslint-enable camelcase */ | ||||||
|  | 
 | ||||||
|       let createSql = ''; |       let createSql = ''; | ||||||
|       const sequences = []; |       const sequences = []; | ||||||
|       const columnsSql = []; |       const columnsSql = []; | ||||||
|       const arrayTypes = { |       const arrayTypes: {[key: string]: string} = { | ||||||
|          _int2: 'smallint', |          _int2: 'smallint', | ||||||
|          _int4: 'integer', |          _int4: 'integer', | ||||||
|          _int8: 'bigint', |          _int8: 'bigint', | ||||||
| @@ -60,7 +86,7 @@ SET row_security = off;\n\n\n`; | |||||||
|          if (fieldType === 'USER-DEFINED') fieldType = `"${this.schemaName}".${column.udt_name}`; |          if (fieldType === 'USER-DEFINED') fieldType = `"${this.schemaName}".${column.udt_name}`; | ||||||
|          else if (fieldType === 'ARRAY') { |          else if (fieldType === 'ARRAY') { | ||||||
|             if (Object.keys(arrayTypes).includes(fieldType)) |             if (Object.keys(arrayTypes).includes(fieldType)) | ||||||
|                fieldType = arrayTypes[type] + '[]'; |                fieldType = arrayTypes[column.udt_name] + '[]'; | ||||||
|             else |             else | ||||||
|                fieldType = column.udt_name.replaceAll('_', '') + '[]'; |                fieldType = column.udt_name.replaceAll('_', '') + '[]'; | ||||||
|          } |          } | ||||||
| @@ -91,7 +117,7 @@ SET row_security = off;\n\n\n`; | |||||||
|             .schema('information_schema') |             .schema('information_schema') | ||||||
|             .from('sequences') |             .from('sequences') | ||||||
|             .where({ sequence_schema: `= '${this.schemaName}'`, sequence_name: `= '${sequence}'` }) |             .where({ sequence_schema: `= '${this.schemaName}'`, sequence_name: `= '${sequence}'` }) | ||||||
|             .run(); |             .run<SequenceRecord>(); | ||||||
| 
 | 
 | ||||||
|          if (rows.length) { |          if (rows.length) { | ||||||
|             createSql += `CREATE SEQUENCE "${this.schemaName}"."${sequence}"
 |             createSql += `CREATE SEQUENCE "${this.schemaName}"."${sequence}"
 | ||||||
| @@ -119,7 +145,7 @@ SET row_security = off;\n\n\n`; | |||||||
|          .schema('pg_catalog') |          .schema('pg_catalog') | ||||||
|          .from('pg_indexes') |          .from('pg_indexes') | ||||||
|          .where({ schemaname: `= '${this.schemaName}'`, tablename: `= '${tableName}'` }) |          .where({ schemaname: `= '${this.schemaName}'`, tablename: `= '${tableName}'` }) | ||||||
|          .run(); |          .run<{indexdef: string}>(); | ||||||
| 
 | 
 | ||||||
|       for (const index of indexes) |       for (const index of indexes) | ||||||
|          createSql += `${index.indexdef};\n`; |          createSql += `${index.indexdef};\n`; | ||||||
| @@ -157,11 +183,11 @@ SET row_security = off;\n\n\n`; | |||||||
|       return createSql; |       return createSql; | ||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
|    getDropTable (tableName) { |    getDropTable (tableName: string) { | ||||||
|       return `DROP TABLE IF EXISTS "${this.schemaName}"."${tableName}";`; |       return `DROP TABLE IF EXISTS "${this.schemaName}"."${tableName}";`; | ||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
|    async * getTableInsert (tableName) { |    async * getTableInsert (tableName: string) { | ||||||
|       let rowCount = 0; |       let rowCount = 0; | ||||||
|       const sqlStr = ''; |       const sqlStr = ''; | ||||||
| 
 | 
 | ||||||
| @@ -205,14 +231,14 @@ SET row_security = off;\n\n\n`; | |||||||
|                } |                } | ||||||
|                else if (DATETIME.includes(column.type)) { |                else if (DATETIME.includes(column.type)) { | ||||||
|                   let datePrecision = ''; |                   let datePrecision = ''; | ||||||
|                   for (let i = 0; i < column.precision; i++) |                   for (let i = 0; i < column.datePrecision; i++) | ||||||
|                      datePrecision += i === 0 ? '.S' : 'S'; |                      datePrecision += i === 0 ? '.S' : 'S'; | ||||||
| 
 | 
 | ||||||
|                   sqlInsertString += moment(val).isValid() |                   sqlInsertString += moment(val).isValid() | ||||||
|                      ? this.escapeAndQuote(moment(val).format(`YYYY-MM-DD HH:mm:ss${datePrecision}`)) |                      ? this.escapeAndQuote(moment(val).format(`YYYY-MM-DD HH:mm:ss${datePrecision}`)) | ||||||
|                      : this.escapeAndQuote(val); |                      : this.escapeAndQuote(val); | ||||||
|                } |                } | ||||||
|                else if (column.isArray) { |                else if ('isArray' in column) { | ||||||
|                   let parsedVal; |                   let parsedVal; | ||||||
|                   if (Array.isArray(val)) |                   if (Array.isArray(val)) | ||||||
|                      parsedVal = JSON.stringify(val).replaceAll('[', '{').replaceAll(']', '}'); |                      parsedVal = JSON.stringify(val).replaceAll('[', '{').replaceAll(']', '}'); | ||||||
| @@ -254,7 +280,7 @@ SET row_security = off;\n\n\n`; | |||||||
| 
 | 
 | ||||||
|    async getCreateTypes () { |    async getCreateTypes () { | ||||||
|       let sqlString = ''; |       let sqlString = ''; | ||||||
|       const { rows: types } = await this._client.raw(` |       const { rows: types } = await this._client.raw<antares.QueryResult<{typname: string; enumlabel: string}>>(` | ||||||
|          SELECT pg_type.typname, pg_enum.enumlabel  |          SELECT pg_type.typname, pg_enum.enumlabel  | ||||||
|          FROM pg_type  |          FROM pg_type  | ||||||
|          JOIN pg_enum ON pg_enum.enumtypid = pg_type.oid; |          JOIN pg_enum ON pg_enum.enumtypid = pg_type.oid; | ||||||
| @@ -360,6 +386,15 @@ SET row_security = off;\n\n\n`; | |||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
|    async getTriggers () { |    async getTriggers () { | ||||||
|  |       /* eslint-disable camelcase */ | ||||||
|  |       interface TriggersResult { | ||||||
|  |          event_object_table: string; | ||||||
|  |          table_name: string; | ||||||
|  |          trigger_name: string; | ||||||
|  |          events: string[]; | ||||||
|  |          event_manipulation: string; | ||||||
|  |       } | ||||||
|  |       /* eslint-enable camelcase */ | ||||||
|       let sqlString = ''; |       let sqlString = ''; | ||||||
| 
 | 
 | ||||||
|       // Trigger functions
 |       // Trigger functions
 | ||||||
| @@ -374,7 +409,7 @@ SET row_security = off;\n\n\n`; | |||||||
|          sqlString += `\n${functionDef[0].definition};\n`; |          sqlString += `\n${functionDef[0].definition};\n`; | ||||||
|       } |       } | ||||||
| 
 | 
 | ||||||
|       const { rows: triggers } = await this._client.raw( |       const { rows: triggers } = await this._client.raw<antares.QueryResult<TriggersResult>>( | ||||||
|          `SELECT * FROM "information_schema"."triggers" WHERE "trigger_schema"='${this.schemaName}'` |          `SELECT * FROM "information_schema"."triggers" WHERE "trigger_schema"='${this.schemaName}'` | ||||||
|       ); |       ); | ||||||
| 
 | 
 | ||||||
| @@ -430,11 +465,12 @@ SET row_security = off;\n\n\n`; | |||||||
|       return sqlString; |       return sqlString; | ||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
|    async _queryStream (sql) { |    async _queryStream (sql: string) { | ||||||
|       if (process.env.NODE_ENV === 'development') console.log('EXPORTER:', sql); |       if (process.env.NODE_ENV === 'development') console.log('EXPORTER:', sql); | ||||||
|       const connection = await this._client.getConnection(); |       const connection = await this._client.getConnection(); | ||||||
|       const query = new QueryStream(sql, null); |       const query = new QueryStream(sql, null); | ||||||
|       const stream = connection.query(query); |       // eslint-disable-next-line @typescript-eslint/no-explicit-any
 | ||||||
|  |       const stream = (connection as any).query(query); | ||||||
|       const dispose = () => connection.end(); |       const dispose = () => connection.end(); | ||||||
| 
 | 
 | ||||||
|       stream.on('end', dispose); |       stream.on('end', dispose); | ||||||
| @@ -443,17 +479,10 @@ SET row_security = off;\n\n\n`; | |||||||
|       return stream; |       return stream; | ||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
|    getEscapedDefiner (definer) { |    escapeAndQuote (val: string) { | ||||||
|       return definer |  | ||||||
|          .split('@') |  | ||||||
|          .map(part => '`' + part + '`') |  | ||||||
|          .join('@'); |  | ||||||
|    } |  | ||||||
| 
 |  | ||||||
|    escapeAndQuote (val) { |  | ||||||
|       // eslint-disable-next-line no-control-regex
 |       // eslint-disable-next-line no-control-regex
 | ||||||
|       const CHARS_TO_ESCAPE = /[\0\b\t\n\r\x1a"'\\]/g; |       const CHARS_TO_ESCAPE = /[\0\b\t\n\r\x1a"'\\]/g; | ||||||
|       const CHARS_ESCAPE_MAP = { |       const CHARS_ESCAPE_MAP: {[key: string]: string} = { | ||||||
|          '\0': '\\0', |          '\0': '\\0', | ||||||
|          '\b': '\\b', |          '\b': '\\b', | ||||||
|          '\t': '\\t', |          '\t': '\\t', | ||||||
| @@ -481,15 +510,4 @@ SET row_security = off;\n\n\n`; | |||||||
| 
 | 
 | ||||||
|       return `'${escapedVal}'`; |       return `'${escapedVal}'`; | ||||||
|    } |    } | ||||||
| 
 |  | ||||||
|    _getGeoJSON (val) { |  | ||||||
|       if (Array.isArray(val)) { |  | ||||||
|          if (getArrayDepth(val) === 1) |  | ||||||
|             return lineString(val.reduce((acc, curr) => [...acc, [curr.x, curr.y]], [])); |  | ||||||
|          else |  | ||||||
|             return polygon(val.map(arr => arr.reduce((acc, curr) => [...acc, [curr.x, curr.y]], []))); |  | ||||||
|       } |  | ||||||
|       else |  | ||||||
|          return point([val.x, val.y]); |  | ||||||
|    } |  | ||||||
| } | } | ||||||
| @@ -1,13 +1,12 @@ | |||||||
| import moment from 'moment'; | import * as moment from 'moment'; | ||||||
|  | import { MySQLClient } from '../../clients/MySQLClient'; | ||||||
|  | import { PostgreSQLClient } from '../../clients/PostgreSQLClient'; | ||||||
| import { BaseExporter } from '../BaseExporter'; | import { BaseExporter } from '../BaseExporter'; | ||||||
| 
 | 
 | ||||||
| export class SqlExporter extends BaseExporter { | export class SqlExporter extends BaseExporter { | ||||||
|    constructor (client, tables, options) { |    protected _client: MySQLClient | PostgreSQLClient; | ||||||
|       super(tables, options); |    protected _commentChar = '--' | ||||||
|       this._client = client; |    protected _postTablesSql = '' | ||||||
|       this._commentChar = '--'; |  | ||||||
|       this._postTablesSql = ''; |  | ||||||
|    } |  | ||||||
| 
 | 
 | ||||||
|    get schemaName () { |    get schemaName () { | ||||||
|       return this._options.schema; |       return this._options.schema; | ||||||
| @@ -24,7 +23,7 @@ export class SqlExporter extends BaseExporter { | |||||||
| 
 | 
 | ||||||
|    async dump () { |    async dump () { | ||||||
|       const { includes } = this._options; |       const { includes } = this._options; | ||||||
|       const extraItems = Object.keys(includes).filter(key => includes[key]); |       const extraItems = Object.keys(includes).filter((key: 'functions' | 'views' | 'triggers' | 'routines' | 'schedulers') => includes[key]); | ||||||
|       const totalTableToProcess = this._tables.filter( |       const totalTableToProcess = this._tables.filter( | ||||||
|          t => t.includeStructure || t.includeContent || t.includeDropStatement |          t => t.includeStructure || t.includeContent || t.includeDropStatement | ||||||
|       ).length; |       ).length; | ||||||
| @@ -98,14 +97,15 @@ export class SqlExporter extends BaseExporter { | |||||||
|       } |       } | ||||||
| 
 | 
 | ||||||
|       for (const item of extraItems) { |       for (const item of extraItems) { | ||||||
|          const processingMethod = `get${item.charAt(0).toUpperCase() + item.slice(1)}`; |          type exporterMethods = 'getViews' | 'getTriggers' | 'getSchedulers' | 'getFunctions' | 'getRoutines' | ||||||
|  |          const processingMethod = `get${item.charAt(0).toUpperCase() + item.slice(1)}` as exporterMethods; | ||||||
|          exportState.currentItemIndex++; |          exportState.currentItemIndex++; | ||||||
|          exportState.currentItem = item; |          exportState.currentItem = item; | ||||||
|          exportState.op = 'PROCESSING'; |          exportState.op = 'PROCESSING'; | ||||||
|          this.emitUpdate(exportState); |          this.emitUpdate(exportState); | ||||||
| 
 | 
 | ||||||
|          if (this[processingMethod]) { |          if (this[processingMethod]) { | ||||||
|             const data = await this[processingMethod](); |             const data = await this[processingMethod]() as unknown as string; | ||||||
|             if (data !== '') { |             if (data !== '') { | ||||||
|                const header = |                const header = | ||||||
|                   this.buildComment( |                   this.buildComment( | ||||||
| @@ -123,7 +123,7 @@ export class SqlExporter extends BaseExporter { | |||||||
|       this.writeString(footer); |       this.writeString(footer); | ||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
|    buildComment (text) { |    buildComment (text: string) { | ||||||
|       return text |       return text | ||||||
|          .split('\n') |          .split('\n') | ||||||
|          .map(txt => `${this._commentChar} ${txt}`) |          .map(txt => `${this._commentChar} ${txt}`) | ||||||
| @@ -151,22 +151,37 @@ Generation time: ${moment().format()} | |||||||
|       return this.buildComment(`Dump completed on ${moment().format()}`); |       return this.buildComment(`Dump completed on ${moment().format()}`); | ||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
|    // eslint-disable-next-line @typescript-eslint/no-unused-vars
 |    /* eslint-disable @typescript-eslint/no-unused-vars */ | ||||||
|    getCreateTable (_tableName) { |    getCreateTable (_tableName: string): Promise<string> { | ||||||
|       throw new Error( |       throw new Error('Sql Exporter must implement the "getCreateTable" method'); | ||||||
|          'Sql Exporter must implement the "getCreateTable" method' |  | ||||||
|       ); |  | ||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
|    // eslint-disable-next-line @typescript-eslint/no-unused-vars
 |    getDropTable (_tableName: string): string { | ||||||
|    getDropTable (_tableName) { |  | ||||||
|       throw new Error('Sql Exporter must implement the "getDropTable" method'); |       throw new Error('Sql Exporter must implement the "getDropTable" method'); | ||||||
|    } |    } | ||||||
| 
 | 
 | ||||||
|    // eslint-disable-next-line @typescript-eslint/no-unused-vars
 |    getTableInsert (_tableName: string): AsyncGenerator<string> { | ||||||
|    getTableInsert (_tableName) { |       throw new Error('Sql Exporter must implement the "getTableInsert" method'); | ||||||
|       throw new Error( |  | ||||||
|          'Sql Exporter must implement the "getTableInsert" method' |  | ||||||
|       ); |  | ||||||
|    } |    } | ||||||
|  | 
 | ||||||
|  |    getViews () { | ||||||
|  |       throw new Error('Method "getViews" not implemented'); | ||||||
|  |    } | ||||||
|  | 
 | ||||||
|  |    getTriggers () { | ||||||
|  |       throw new Error('Method "getTriggers" not implemented'); | ||||||
|  |    } | ||||||
|  | 
 | ||||||
|  |    getSchedulers () { | ||||||
|  |       throw new Error('Method "getSchedulers" not implemented'); | ||||||
|  |    } | ||||||
|  | 
 | ||||||
|  |    getFunctions () { | ||||||
|  |       throw new Error('Method "getFunctions" not implemented'); | ||||||
|  |    } | ||||||
|  | 
 | ||||||
|  |    getRoutines () { | ||||||
|  |       throw new Error('Method "getRoutines" not implemented'); | ||||||
|  |    } | ||||||
|  |    /* eslint-enable @typescript-eslint/no-unused-vars */ | ||||||
| } | } | ||||||
| @@ -1,8 +1,11 @@ | |||||||
| import fs from 'fs'; | import * as antares from 'common/interfaces/antares'; | ||||||
|  | import * as fs from 'fs'; | ||||||
|  | import { MySQLClient } from '../libs/clients/MySQLClient'; | ||||||
|  | import { PostgreSQLClient } from '../libs/clients/PostgreSQLClient'; | ||||||
| import { ClientsFactory } from '../libs/ClientsFactory'; | import { ClientsFactory } from '../libs/ClientsFactory'; | ||||||
| import MysqlExporter from '../libs/exporters/sql/MysqlExporter.js'; | import MysqlExporter from '../libs/exporters/sql/MysqlExporter'; | ||||||
| import PostgreSQLExporter from '../libs/exporters/sql/PostgreSQLExporter'; | import PostgreSQLExporter from '../libs/exporters/sql/PostgreSQLExporter'; | ||||||
| let exporter; | let exporter: antares.Exporter; | ||||||
| 
 | 
 | ||||||
| process.on('message', async ({ type, client, tables, options }) => { | process.on('message', async ({ type, client, tables, options }) => { | ||||||
|    if (type === 'init') { |    if (type === 'init') { | ||||||
| @@ -10,16 +13,16 @@ process.on('message', async ({ type, client, tables, options }) => { | |||||||
|          client: client.name, |          client: client.name, | ||||||
|          params: client.config, |          params: client.config, | ||||||
|          poolSize: 5 |          poolSize: 5 | ||||||
|       }); |       }) as MySQLClient | PostgreSQLClient; | ||||||
|       await connection.connect(); |       await connection.connect(); | ||||||
| 
 | 
 | ||||||
|       switch (client.name) { |       switch (client.name) { | ||||||
|          case 'mysql': |          case 'mysql': | ||||||
|          case 'maria': |          case 'maria': | ||||||
|             exporter = new MysqlExporter(connection, tables, options); |             exporter = new MysqlExporter(connection as MySQLClient, tables, options); | ||||||
|             break; |             break; | ||||||
|          case 'pg': |          case 'pg': | ||||||
|             exporter = new PostgreSQLExporter(connection, tables, options); |             exporter = new PostgreSQLExporter(connection as PostgreSQLClient, tables, options); | ||||||
|             break; |             break; | ||||||
|          default: |          default: | ||||||
|             process.send({ |             process.send({ | ||||||
| @@ -62,3 +65,5 @@ process.on('message', async ({ type, client, tables, options }) => { | |||||||
|    else if (type === 'cancel') |    else if (type === 'cancel') | ||||||
|       exporter.cancel(); |       exporter.cancel(); | ||||||
| }); | }); | ||||||
|  | 
 | ||||||
|  | process.on('beforeExit', console.log); | ||||||
| @@ -13,7 +13,7 @@ const config = { | |||||||
|    mode: process.env.NODE_ENV, |    mode: process.env.NODE_ENV, | ||||||
|    devtool: isDevMode ? 'eval-source-map' : false, |    devtool: isDevMode ? 'eval-source-map' : false, | ||||||
|    entry: { |    entry: { | ||||||
|       exporter: path.join(__dirname, './src/main/workers/exporter.js'), |       exporter: path.join(__dirname, './src/main/workers/exporter.ts'), | ||||||
|       importer: path.join(__dirname, './src/main/workers/importer.js') |       importer: path.join(__dirname, './src/main/workers/importer.js') | ||||||
|    }, |    }, | ||||||
|    target: 'node', |    target: 'node', | ||||||
|   | |||||||
		Reference in New Issue
	
	Block a user