fabio286
/
antares
mirror of https://github.com/Fabio286/antares.git
1
1
Fork 0
Code Issues Releases Wiki Activity

refactor: db exporter ts refactor

This commit is contained in:
Fabio Di Stasio 2022-04-15 23:13:23 +02:00
parent a315eeae6c
commit 5dbc127b51
12 changed files with 198 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/common/interfaces/antares.ts
View File

@ -1,5 +1,9 @@
import * as mysql from 'mysql2/promise';
import * as pg from 'pg';
import MysqlExporter from 'src/main/libs/exporters/sql/MysqlExporter';
import PostgreSQLExporter from 'src/main/libs/exporters/sql/PostgreSQLExporter';
import MySQLImporter from 'src/main/libs/importers/sql/MysqlImporter';
import PostgreSQLImporter from 'src/main/libs/importers/sql/PostgreSQLImporter';
import SSHConfig from 'ssh2-promise/lib/sshConfig';
import { MySQLClient } from '../../main/libs/clients/MySQLClient';
import { PostgreSQLClient } from '../../main/libs/clients/PostgreSQLClient';
@ -7,6 +11,8 @@ import { SQLiteClient } from '../../main/libs/clients/SQLiteClient';
export type Client = MySQLClient | PostgreSQLClient | SQLiteClient
export type ClientCode = 'mysql' | 'maria' | 'pg' | 'sqlite'
export type Exporter = MysqlExporter | PostgreSQLExporter
export type Importer = MySQLImporter | PostgreSQLImporter
/**
* Pasameters needed to create a new Antares connection to a database

28
src/common/interfaces/exporter.ts Normal file
View File

@ -0,0 +1,28 @@
export interface TableParams {
table: string;
includeStructure: boolean;
includeContent: boolean;
includeDropStatement: boolean;
}
export interface ExportOptions {
schema: string;
includes: {
functions: boolean;
views: boolean;
triggers: boolean;
routines: boolean;
schedulers: boolean;
};
outputFormat: 'sql' | 'sql.zip';
outputFile: string;
sqlInsertAfter: number;
sqlInsertDivider: 'bytes' | 'rows';
}
export interface ExportState {
totalItems?: number;
currentItemIndex?: number;
currentItem?: string;
op?: string;
}

2
src/main/ipc-handlers/schema.ts
View File

@ -1,6 +1,6 @@
import * as antares from 'common/interfaces/antares';
import * as workers from 'common/interfaces/workers';
import fs from 'fs';
import * as fs from 'fs';
import path from 'path';
import { ChildProcess, fork } from 'child_process';
import { ipcMain, dialog } from 'electron';

14
src/main/libs/clients/MySQLClient.ts
View File

@ -9,8 +9,8 @@ export class MySQLClient extends AntaresCore {
private _schema?: string;
private _runningConnections: Map<string, number>;
private _connectionsToCommit: Map<string, mysql.Connection | mysql.PoolConnection>;
protected _connection?: mysql.Connection | mysql.Pool;
protected _params: mysql.ConnectionOptions & {schema: string; ssl?: mysql.SslOptions; ssh?: SSHConfig; readonly: boolean};
_connection?: mysql.Connection | mysql.Pool;
_params: mysql.ConnectionOptions & {schema: string; ssl?: mysql.SslOptions; ssh?: SSHConfig; readonly: boolean};
private types: {[key: number]: string} = {
0: 'DECIMAL',
@ -445,18 +445,18 @@ export class MySQLClient extends AntaresCore {
async getTableColumns ({ schema, table }: { schema: string; table: string }) {
interface TableColumnsResult {
COLUMN_TYPE: string;
NUMERIC_PRECISION: string;
NUMERIC_PRECISION: number;
COLUMN_NAME: string;
COLUMN_DEFAULT: string;
COLUMN_KEY: string;
DATA_TYPE: string;
TABLE_SCHEMA: string;
TABLE_NAME: string;
NUMERIC_SCALE: string;
DATETIME_PRECISION: string;
CHARACTER_MAXIMUM_LENGTH: string;
NUMERIC_SCALE: number;
DATETIME_PRECISION: number;
CHARACTER_MAXIMUM_LENGTH: number;
IS_NULLABLE: string;
ORDINAL_POSITION: string;
ORDINAL_POSITION: number;
CHARACTER_SET_NAME: string;
COLLATION_NAME: string;
EXTRA: string;

7
src/main/libs/clients/PostgreSQLClient.ts
View File

@ -24,7 +24,6 @@ export class PostgreSQLClient extends AntaresCore {
private _runningConnections: Map<string, number>;
private _connectionsToCommit: Map<string, pg.Client | pg.PoolClient>;
protected _connection?: pg.Client | pg.Pool;
protected _params: pg.ClientConfig & {schema: string; ssl?: mysql.SslOptions; ssh?: SSHConfig; readonly: boolean};
private types: {[key: string]: string} = {};
private _arrayTypes: {[key: string]: string} = {
_int2: 'SMALLINT',
@ -36,6 +35,8 @@ export class PostgreSQLClient extends AntaresCore {
_varchar: 'CHARACTER VARYING'
}
_params: pg.ClientConfig & {schema: string; ssl?: mysql.SslOptions; ssh?: SSHConfig; readonly: boolean};
constructor (args: antares.ClientParams) {
super(args);
@ -173,6 +174,10 @@ export class PostgreSQLClient extends AntaresCore {
}
}
getCollations (): null[] {
return [];
}
async getStructure (schemas: Set<string>) {
/* eslint-disable camelcase */
interface ShowTableResult {

2
src/main/libs/clients/SQLiteClient.ts
View File

@ -8,7 +8,7 @@ export class SQLiteClient extends AntaresCore {
private _schema?: string;
private _connectionsToCommit: Map<string, sqlite.Database>;
protected _connection?: sqlite.Database;
protected _params: { databasePath: string; readonly: boolean};
_params: { databasePath: string; readonly: boolean};
constructor (args: antares.ClientParams) {
super(args);

22
src/main/libs/exporters/BaseExporter.js → src/main/libs/exporters/BaseExporter.ts
View File

@ -1,10 +1,18 @@
import fs from 'fs';
import { createGzip } from 'zlib';
import path from 'path';
import EventEmitter from 'events';
import * as exporter from 'common/interfaces/exporter';
import * as fs from 'fs';
import { createGzip, Gzip } from 'zlib';
import * as path from 'path';
import * as EventEmitter from 'events';
export class BaseExporter extends EventEmitter {
constructor (tables, options) {
protected _tables;
protected _options;
protected _isCancelled;
protected _outputFileStream: fs.WriteStream;
protected _processedStream: fs.WriteStream | Gzip;
protected _state;
constructor (tables: exporter.TableParams[], options: exporter.ExportOptions) {
super();
this._tables = tables;
this._options = options;
@ -60,11 +68,11 @@ export class BaseExporter extends EventEmitter {
this.emitUpdate({ op: 'cancelling' });
}
emitUpdate (state) {
emitUpdate (state: exporter.ExportState) {
this.emit('progress', { ...this._state, ...state });
}
writeString (data) {
writeString (data: string) {
if (this._isCancelled) return;
try {

49
src/main/libs/exporters/sql/MysqlExporter.js → src/main/libs/exporters/sql/MysqlExporter.ts
View File

@ -1,14 +1,20 @@
import * as exporter from 'common/interfaces/exporter';
import * as mysql from 'mysql2/promise';
import { SqlExporter } from './SqlExporter';
import { BLOB, BIT, DATE, DATETIME, FLOAT, SPATIAL, IS_MULTI_SPATIAL, NUMBER } from 'common/fieldTypes';
import hexToBinary from 'common/libs/hexToBinary';
import { getArrayDepth } from 'common/libs/getArrayDepth';
import moment from 'moment';
import * as moment from 'moment';
import { lineString, point, polygon } from '@turf/helpers';
import { MySQLClient } from '../../clients/MySQLClient';
export default class MysqlExporter extends SqlExporter {
constructor (...args) {
super(...args);
protected _client: MySQLClient;
constructor (client: MySQLClient, tables: exporter.TableParams[], options: exporter.ExportOptions) {
super(tables, options);
this._client = client;
this._commentChar = '#';
}
@ -42,7 +48,7 @@ ${footer}
`;
}
async getCreateTable (tableName) {
async getCreateTable (tableName: string) {
const { rows } = await this._client.raw(
`SHOW CREATE TABLE \`${this.schemaName}\`.\`${tableName}\``
);
@ -54,11 +60,11 @@ ${footer}
return rows[0][col] + ';';
}
getDropTable (tableName) {
getDropTable (tableName: string) {
return `DROP TABLE IF EXISTS \`${tableName}\`;`;
}
async * getTableInsert (tableName) {
async * getTableInsert (tableName: string) {
let rowCount = 0;
let sqlStr = '';
@ -109,7 +115,7 @@ ${footer}
queryLength = 0;
rowsWritten = 0;
}
else if (parseInt(rowIndex) === 0) sqlInsertString += '\n\t(';
else if (rowIndex === 0) sqlInsertString += '\n\t(';
else sqlInsertString += ',\n\t(';
for (const i in notGeneratedColumns) {
@ -124,7 +130,7 @@ ${footer}
}
else if (DATETIME.includes(column.type)) {
let datePrecision = '';
for (let i = 0; i < column.precision; i++)
for (let i = 0; i < column.datePrecision; i++)
datePrecision += i === 0 ? '.S' : 'S';
sqlInsertString += moment(val).isValid()
@ -144,7 +150,7 @@ ${footer}
if (IS_MULTI_SPATIAL.includes(column.type)) {
const features = [];
for (const element of val)
features.push(this.getMarkers(element));
features.push(this._getGeoJSON(element));
geoJson = {
type: 'FeatureCollection',
@ -323,7 +329,7 @@ ${footer}
return sqlString;
}
async getRoutineSyntax (name, type, definer) {
async getRoutineSyntax (name: string, type: string, definer: string) {
const { rows: routines } = await this._client.raw(
`SHOW CREATE ${type} \`${this.schemaName}\`.\`${name}\``
);
@ -353,12 +359,13 @@ ${footer}
return sqlString;
}
async _queryStream (sql) {
async _queryStream (sql: string) {
if (process.env.NODE_ENV === 'development') console.log('EXPORTER:', sql);
const isPool = typeof this._client._connection.getConnection === 'function';
const connection = isPool ? await this._client._connection.getConnection() : this._client._connection;
const stream = connection.connection.query(sql).stream();
const dispose = () => connection.destroy();
const isPool = 'getConnection' in this._client._connection;
const connection = isPool ? await (this._client._connection as mysql.Pool).getConnection() : this._client._connection;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const stream = (connection as any).connection.query(sql).stream();
const dispose = () => (connection as mysql.PoolConnection).release();
stream.on('end', dispose);
stream.on('error', dispose);
@ -366,17 +373,17 @@ ${footer}
return stream;
}
getEscapedDefiner (definer) {
getEscapedDefiner (definer: string) {
return definer
.split('@')
.map(part => '`' + part + '`')
.join('@');
}
escapeAndQuote (val) {
escapeAndQuote (val: string) {
// eslint-disable-next-line no-control-regex
const CHARS_TO_ESCAPE = /[\0\b\t\n\r\x1a"'\\]/g;
const CHARS_ESCAPE_MAP = {
const CHARS_ESCAPE_MAP: {[key: string]: string} = {
'\0': '\\0',
'\b': '\\b',
'\t': '\\t',
@ -405,14 +412,16 @@ ${footer}
return `'${escapedVal}'`;
}
_getGeoJSON (val) {
/* eslint-disable @typescript-eslint/no-explicit-any */
_getGeoJSON (val: any) {
if (Array.isArray(val)) {
if (getArrayDepth(val) === 1)
return lineString(val.reduce((acc, curr) => [...acc, [curr.x, curr.y]], []));
else
return polygon(val.map(arr => arr.reduce((acc, curr) => [...acc, [curr.x, curr.y]], [])));
return polygon(val.map(arr => arr.reduce((acc: any, curr: any) => [...acc, [curr.x, curr.y]], [])));
}
else
return point([val.x, val.y]);
}
/* eslint-enable @typescript-eslint/no-explicit-any */
}

92
src/main/libs/exporters/sql/PostgreSQLExporter.js → src/main/libs/exporters/sql/PostgreSQLExporter.ts
View File

@ -1,12 +1,21 @@
import * as antares from 'common/interfaces/antares';
import * as exporter from 'common/interfaces/exporter';
import { SqlExporter } from './SqlExporter';
import { BLOB, BIT, DATE, DATETIME, FLOAT, NUMBER, TEXT_SEARCH } from 'common/fieldTypes';
import hexToBinary from 'common/libs/hexToBinary';
import { getArrayDepth } from 'common/libs/getArrayDepth';
import moment from 'moment';
import { lineString, point, polygon } from '@turf/helpers';
import QueryStream from 'pg-query-stream';
import * as moment from 'moment';
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
import * as QueryStream from 'pg-query-stream';
import { PostgreSQLClient } from '../../clients/PostgreSQLClient';
export default class PostgreSQLExporter extends SqlExporter {
constructor (client: PostgreSQLClient, tables: exporter.TableParams[], options: exporter.ExportOptions) {
super(tables, options);
this._client = client;
}
async getSqlHeader () {
let dump = await super.getSqlHeader();
dump += `
@ -30,11 +39,28 @@ SET row_security = off;\n\n\n`;
return dump;
}
async getCreateTable (tableName) {
async getCreateTable (tableName: string) {
/* eslint-disable camelcase */
interface SequenceRecord {
sequence_catalog: string;
sequence_schema: string;
sequence_name: string;
data_type: string;
numeric_precision: number;
numeric_precision_radix: number;
numeric_scale: number;
start_value: string;
minimum_value: string;
maximum_value: string;
increment: string;
cycle_option: string;
}
/* eslint-enable camelcase */
let createSql = '';
const sequences = [];
const columnsSql = [];
const arrayTypes = {
const arrayTypes: {[key: string]: string} = {
_int2: 'smallint',
_int4: 'integer',
_int8: 'bigint',
@ -60,7 +86,7 @@ SET row_security = off;\n\n\n`;
if (fieldType === 'USER-DEFINED') fieldType = `"${this.schemaName}".${column.udt_name}`;
else if (fieldType === 'ARRAY') {
if (Object.keys(arrayTypes).includes(fieldType))
fieldType = arrayTypes[type] + '[]';
fieldType = arrayTypes[column.udt_name] + '[]';
else
fieldType = column.udt_name.replaceAll('_', '') + '[]';
}
@ -91,7 +117,7 @@ SET row_security = off;\n\n\n`;
.schema('information_schema')
.from('sequences')
.where({ sequence_schema: `= '${this.schemaName}'`, sequence_name: `= '${sequence}'` })
.run();
.run<SequenceRecord>();
if (rows.length) {
createSql += `CREATE SEQUENCE "${this.schemaName}"."${sequence}"
@ -119,7 +145,7 @@ SET row_security = off;\n\n\n`;
.schema('pg_catalog')
.from('pg_indexes')
.where({ schemaname: `= '${this.schemaName}'`, tablename: `= '${tableName}'` })
.run();
.run<{indexdef: string}>();
for (const index of indexes)
createSql += `${index.indexdef};\n`;
@ -157,11 +183,11 @@ SET row_security = off;\n\n\n`;
return createSql;
}
getDropTable (tableName) {
getDropTable (tableName: string) {
return `DROP TABLE IF EXISTS "${this.schemaName}"."${tableName}";`;
}
async * getTableInsert (tableName) {
async * getTableInsert (tableName: string) {
let rowCount = 0;
const sqlStr = '';
@ -205,14 +231,14 @@ SET row_security = off;\n\n\n`;
}
else if (DATETIME.includes(column.type)) {
let datePrecision = '';
for (let i = 0; i < column.precision; i++)
for (let i = 0; i < column.datePrecision; i++)
datePrecision += i === 0 ? '.S' : 'S';
sqlInsertString += moment(val).isValid()
? this.escapeAndQuote(moment(val).format(`YYYY-MM-DD HH:mm:ss${datePrecision}`))
: this.escapeAndQuote(val);
}
else if (column.isArray) {
else if ('isArray' in column) {
let parsedVal;
if (Array.isArray(val))
parsedVal = JSON.stringify(val).replaceAll('[', '{').replaceAll(']', '}');
@ -254,7 +280,7 @@ SET row_security = off;\n\n\n`;
async getCreateTypes () {
let sqlString = '';
const { rows: types } = await this._client.raw(`
const { rows: types } = await this._client.raw<antares.QueryResult<{typname: string; enumlabel: string}>>(`
SELECT pg_type.typname, pg_enum.enumlabel
FROM pg_type
JOIN pg_enum ON pg_enum.enumtypid = pg_type.oid;
@ -360,6 +386,15 @@ SET row_security = off;\n\n\n`;
}
async getTriggers () {
/* eslint-disable camelcase */
interface TriggersResult {
event_object_table: string;
table_name: string;
trigger_name: string;
events: string[];
event_manipulation: string;
}
/* eslint-enable camelcase */
let sqlString = '';
// Trigger functions
@ -374,7 +409,7 @@ SET row_security = off;\n\n\n`;
sqlString += `\n${functionDef[0].definition};\n`;
}
const { rows: triggers } = await this._client.raw(
const { rows: triggers } = await this._client.raw<antares.QueryResult<TriggersResult>>(
`SELECT * FROM "information_schema"."triggers" WHERE "trigger_schema"='${this.schemaName}'`
);
@ -430,11 +465,12 @@ SET row_security = off;\n\n\n`;
return sqlString;
}
async _queryStream (sql) {
async _queryStream (sql: string) {
if (process.env.NODE_ENV === 'development') console.log('EXPORTER:', sql);
const connection = await this._client.getConnection();
const query = new QueryStream(sql, null);
const stream = connection.query(query);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const stream = (connection as any).query(query);
const dispose = () => connection.end();
stream.on('end', dispose);
@ -443,17 +479,10 @@ SET row_security = off;\n\n\n`;
return stream;
}
getEscapedDefiner (definer) {
return definer
.split('@')
.map(part => '`' + part + '`')
.join('@');
}
escapeAndQuote (val) {
escapeAndQuote (val: string) {
// eslint-disable-next-line no-control-regex
const CHARS_TO_ESCAPE = /[\0\b\t\n\r\x1a"'\\]/g;
const CHARS_ESCAPE_MAP = {
const CHARS_ESCAPE_MAP: {[key: string]: string} = {
'\0': '\\0',
'\b': '\\b',
'\t': '\\t',
@ -481,15 +510,4 @@ SET row_security = off;\n\n\n`;
return `'${escapedVal}'`;
}
_getGeoJSON (val) {
if (Array.isArray(val)) {
if (getArrayDepth(val) === 1)
return lineString(val.reduce((acc, curr) => [...acc, [curr.x, curr.y]], []));
else
return polygon(val.map(arr => arr.reduce((acc, curr) => [...acc, [curr.x, curr.y]], [])));
}
else
return point([val.x, val.y]);
}
}

61
src/main/libs/exporters/sql/SqlExporter.js → src/main/libs/exporters/sql/SqlExporter.ts
View File

@ -1,13 +1,12 @@
import moment from 'moment';
import * as moment from 'moment';
import { MySQLClient } from '../../clients/MySQLClient';
import { PostgreSQLClient } from '../../clients/PostgreSQLClient';
import { BaseExporter } from '../BaseExporter';
export class SqlExporter extends BaseExporter {
constructor (client, tables, options) {
super(tables, options);
this._client = client;
this._commentChar = '--';
this._postTablesSql = '';
}
protected _client: MySQLClient | PostgreSQLClient;
protected _commentChar = '--'
protected _postTablesSql = ''
get schemaName () {
return this._options.schema;
@ -24,7 +23,7 @@ export class SqlExporter extends BaseExporter {
async dump () {
const { includes } = this._options;
const extraItems = Object.keys(includes).filter(key => includes[key]);
const extraItems = Object.keys(includes).filter((key: 'functions' | 'views' | 'triggers' | 'routines' | 'schedulers') => includes[key]);
const totalTableToProcess = this._tables.filter(
t => t.includeStructure || t.includeContent || t.includeDropStatement
).length;
@ -98,14 +97,15 @@ export class SqlExporter extends BaseExporter {
}
for (const item of extraItems) {
const processingMethod = `get${item.charAt(0).toUpperCase() + item.slice(1)}`;
type exporterMethods = 'getViews' | 'getTriggers' | 'getSchedulers' | 'getFunctions' | 'getRoutines'
const processingMethod = `get${item.charAt(0).toUpperCase() + item.slice(1)}` as exporterMethods;
exportState.currentItemIndex++;
exportState.currentItem = item;
exportState.op = 'PROCESSING';
this.emitUpdate(exportState);
if (this[processingMethod]) {
const data = await this[processingMethod]();
const data = await this[processingMethod]() as unknown as string;
if (data !== '') {
const header =
this.buildComment(
@ -123,7 +123,7 @@ export class SqlExporter extends BaseExporter {
this.writeString(footer);
}
buildComment (text) {
buildComment (text: string) {
return text
.split('\n')
.map(txt => `${this._commentChar} ${txt}`)
@ -151,22 +151,37 @@ Generation time: ${moment().format()}
return this.buildComment(`Dump completed on ${moment().format()}`);
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
getCreateTable (_tableName) {
throw new Error(
'Sql Exporter must implement the "getCreateTable" method'
);
/* eslint-disable @typescript-eslint/no-unused-vars */
getCreateTable (_tableName: string): Promise<string> {
throw new Error('Sql Exporter must implement the "getCreateTable" method');
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
getDropTable (_tableName) {
getDropTable (_tableName: string): string {
throw new Error('Sql Exporter must implement the "getDropTable" method');
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
getTableInsert (_tableName) {
throw new Error(
'Sql Exporter must implement the "getTableInsert" method'
);
getTableInsert (_tableName: string): AsyncGenerator<string> {
throw new Error('Sql Exporter must implement the "getTableInsert" method');
}
getViews () {
throw new Error('Method "getViews" not implemented');
}
getTriggers () {
throw new Error('Method "getTriggers" not implemented');
}
getSchedulers () {
throw new Error('Method "getSchedulers" not implemented');
}
getFunctions () {
throw new Error('Method "getFunctions" not implemented');
}
getRoutines () {
throw new Error('Method "getRoutines" not implemented');
}
/* eslint-enable @typescript-eslint/no-unused-vars */
}

17
src/main/workers/exporter.js → src/main/workers/exporter.ts
View File

@ -1,8 +1,11 @@
import fs from 'fs';
import * as antares from 'common/interfaces/antares';
import * as fs from 'fs';
import { MySQLClient } from '../libs/clients/MySQLClient';
import { PostgreSQLClient } from '../libs/clients/PostgreSQLClient';
import { ClientsFactory } from '../libs/ClientsFactory';
import MysqlExporter from '../libs/exporters/sql/MysqlExporter.js';
import MysqlExporter from '../libs/exporters/sql/MysqlExporter';
import PostgreSQLExporter from '../libs/exporters/sql/PostgreSQLExporter';
let exporter;
let exporter: antares.Exporter;
process.on('message', async ({ type, client, tables, options }) => {
if (type === 'init') {
@ -10,16 +13,16 @@ process.on('message', async ({ type, client, tables, options }) => {
client: client.name,
params: client.config,
poolSize: 5
});
}) as MySQLClient | PostgreSQLClient;
await connection.connect();
switch (client.name) {
case 'mysql':
case 'maria':
exporter = new MysqlExporter(connection, tables, options);
exporter = new MysqlExporter(connection as MySQLClient, tables, options);
break;
case 'pg':
exporter = new PostgreSQLExporter(connection, tables, options);
exporter = new PostgreSQLExporter(connection as PostgreSQLClient, tables, options);
break;
default:
process.send({
@ -62,3 +65,5 @@ process.on('message', async ({ type, client, tables, options }) => {
else if (type === 'cancel')
exporter.cancel();
});
process.on('beforeExit', console.log);

2
webpack.workers.config.js
View File

@ -13,7 +13,7 @@ const config = {
mode: process.env.NODE_ENV,
devtool: isDevMode ? 'eval-source-map' : false,
entry: {
exporter: path.join(__dirname, './src/main/workers/exporter.js'),
exporter: path.join(__dirname, './src/main/workers/exporter.ts'),
importer: path.join(__dirname, './src/main/workers/importer.js')
},
target: 'node',