From 79f32ca442b0f6597caf52391b2db80926ce5781 Mon Sep 17 00:00:00 2001 From: Fabio Di Stasio Date: Sun, 17 Apr 2022 12:01:07 +0200 Subject: [PATCH] refactor: db import ts refactor --- src/common/interfaces/antares.ts | 2 +- src/common/interfaces/importer.ts | 16 +++++++++++ src/common/interfaces/parser.ts | 0 .../{BaseImporter.js => BaseImporter.ts} | 14 +++++++--- .../{MysqlImporter.js => MySQLlImporter.ts} | 13 ++++++--- ...reSQLImporter.js => PostgreSQLImporter.ts} | 11 ++++++-- .../{MySQLParser.js => MySQLParser.ts} | 22 +++++++++++---- ...ostgreSQLParser.js => PostgreSQLParser.ts} | 28 +++++++++++++++---- src/main/workers/{importer.js => importer.ts} | 15 ++++++---- 9 files changed, 92 insertions(+), 29 deletions(-) create mode 100644 src/common/interfaces/importer.ts create mode 100644 src/common/interfaces/parser.ts rename src/main/libs/importers/{BaseImporter.js => BaseImporter.ts} (76%) rename src/main/libs/importers/sql/{MysqlImporter.js => MySQLlImporter.ts} (83%) rename src/main/libs/importers/sql/{PostgreSQLImporter.js => PostgreSQLImporter.ts} (85%) rename src/main/libs/parsers/{MySQLParser.js => MySQLParser.ts} (80%) rename src/main/libs/parsers/{PostgreSQLParser.js => PostgreSQLParser.ts} (83%) rename src/main/workers/{importer.js => importer.ts} (74%) diff --git a/src/common/interfaces/antares.ts b/src/common/interfaces/antares.ts index 20e11a61..9e75a34d 100644 --- a/src/common/interfaces/antares.ts +++ b/src/common/interfaces/antares.ts @@ -2,7 +2,7 @@ import * as mysql from 'mysql2/promise'; import * as pg from 'pg'; import MysqlExporter from 'src/main/libs/exporters/sql/MysqlExporter'; import PostgreSQLExporter from 'src/main/libs/exporters/sql/PostgreSQLExporter'; -import MySQLImporter from 'src/main/libs/importers/sql/MysqlImporter'; +import MySQLImporter from 'src/main/libs/importers/sql/MySQLlImporter'; import PostgreSQLImporter from 'src/main/libs/importers/sql/PostgreSQLImporter'; import SSHConfig from 'ssh2-promise/lib/sshConfig'; import { MySQLClient } from '../../main/libs/clients/MySQLClient'; diff --git a/src/common/interfaces/importer.ts b/src/common/interfaces/importer.ts new file mode 100644 index 00000000..d0aa985a --- /dev/null +++ b/src/common/interfaces/importer.ts @@ -0,0 +1,16 @@ +import * as antares from './antares'; + +export interface ImportOptions { + uid: string; + schema: string; + type: antares.ClientCode; + file: string; +} + +export interface ImportState { + fileSize?: number; + readPosition?: number; + percentage?: number; + queryCount?: number; + op?: string; +} diff --git a/src/common/interfaces/parser.ts b/src/common/interfaces/parser.ts new file mode 100644 index 00000000..e69de29b diff --git a/src/main/libs/importers/BaseImporter.js b/src/main/libs/importers/BaseImporter.ts similarity index 76% rename from src/main/libs/importers/BaseImporter.js rename to src/main/libs/importers/BaseImporter.ts index 901a5d98..807aedea 100644 --- a/src/main/libs/importers/BaseImporter.js +++ b/src/main/libs/importers/BaseImporter.ts @@ -1,8 +1,14 @@ -import fs from 'fs'; -import EventEmitter from 'events'; +import * as importer from 'common/interfaces/importer'; +import * as fs from 'fs'; +import * as EventEmitter from 'events'; export class BaseImporter extends EventEmitter { - constructor (options) { + protected _options; + protected _isCancelled; + protected _fileHandler; + protected _state; + + constructor (options: importer.ImportOptions) { super(); this._options = options; this._isCancelled = false; @@ -43,7 +49,7 @@ export class BaseImporter extends EventEmitter { this.emitUpdate({ op: 'cancelling' }); } - emitUpdate (state) { + emitUpdate (state: importer.ImportState) { this.emit('progress', { ...this._state, ...state }); } diff --git a/src/main/libs/importers/sql/MysqlImporter.js b/src/main/libs/importers/sql/MySQLlImporter.ts similarity index 83% rename from src/main/libs/importers/sql/MysqlImporter.js rename to src/main/libs/importers/sql/MySQLlImporter.ts index dc9349da..51c8e418 100644 --- a/src/main/libs/importers/sql/MysqlImporter.js +++ b/src/main/libs/importers/sql/MySQLlImporter.ts @@ -1,14 +1,18 @@ -import fs from 'fs/promises'; +import * as mysql from 'mysql2'; +import * as importer from 'common/interfaces/importer'; +import * as fs from 'fs/promises'; import MySQLParser from '../../parsers/MySQLParser'; import { BaseImporter } from '../BaseImporter'; export default class MySQLImporter extends BaseImporter { - constructor (client, options) { + protected _client: mysql.Pool + + constructor (client: mysql.Pool, options: importer.ImportOptions) { super(options); this._client = client; } - async import () { + async import (): Promise { try { const { size: totalFileSize } = await fs.stat(this._options.file); const parser = new MySQLParser(); @@ -60,7 +64,8 @@ export default class MySQLImporter extends BaseImporter { parser.on('pause', () => { this._fileHandler.unpipe(parser); - this._fileHandler.readableFlowing = false; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (this._fileHandler as any).readableFlowing = false; }); this._fileHandler.on('data', (chunk) => { diff --git a/src/main/libs/importers/sql/PostgreSQLImporter.js b/src/main/libs/importers/sql/PostgreSQLImporter.ts similarity index 85% rename from src/main/libs/importers/sql/PostgreSQLImporter.js rename to src/main/libs/importers/sql/PostgreSQLImporter.ts index a18f22c9..eb6a79fd 100644 --- a/src/main/libs/importers/sql/PostgreSQLImporter.js +++ b/src/main/libs/importers/sql/PostgreSQLImporter.ts @@ -1,14 +1,18 @@ +import * as pg from 'pg'; +import * as importer from 'common/interfaces/importer'; import fs from 'fs/promises'; import PostgreSQLParser from '../../parsers/PostgreSQLParser'; import { BaseImporter } from '../BaseImporter'; export default class PostgreSQLImporter extends BaseImporter { - constructor (client, options) { + protected _client: pg.PoolClient; + + constructor (client: pg.PoolClient, options: importer.ImportOptions) { super(options); this._client = client; } - async import () { + async import (): Promise { try { const { size: totalFileSize } = await fs.stat(this._options.file); const parser = new PostgreSQLParser(); @@ -60,7 +64,8 @@ export default class PostgreSQLImporter extends BaseImporter { parser.on('pause', () => { this._fileHandler.unpipe(parser); - this._fileHandler.readableFlowing = false; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (this._fileHandler as any).readableFlowing = false; }); this._fileHandler.on('data', (chunk) => { diff --git a/src/main/libs/parsers/MySQLParser.js b/src/main/libs/parsers/MySQLParser.ts similarity index 80% rename from src/main/libs/parsers/MySQLParser.js rename to src/main/libs/parsers/MySQLParser.ts index 3157a5e3..66bdd997 100644 --- a/src/main/libs/parsers/MySQLParser.js +++ b/src/main/libs/parsers/MySQLParser.ts @@ -1,7 +1,17 @@ -import { Transform } from 'stream'; +import { Transform, TransformCallback, TransformOptions } from 'stream'; export default class MySQLParser extends Transform { - constructor (opts) { + private _buffer: string; + private _lastChar: string; + private _last9Chars: string; + + encoding: BufferEncoding; + delimiter: string; + isEscape: boolean; + currentQuote: string; + isDelimiter: boolean; + + constructor (opts?: TransformOptions & { delimiter: string }) { opts = { delimiter: ';', encoding: 'utf8', @@ -21,7 +31,7 @@ export default class MySQLParser extends Transform { this.isDelimiter = false; } - _transform (chunk, encoding, next) { + _transform (chunk: Buffer, encoding: BufferEncoding, next: TransformCallback) { for (const char of chunk.toString(this.encoding)) { this.checkEscape(); this._buffer += char; @@ -49,7 +59,7 @@ export default class MySQLParser extends Transform { } } - checkNewDelimiter (char) { + checkNewDelimiter (char: string) { if (this.currentQuote === null && this._last9Chars === 'delimiter') { this.isDelimiter = true; this._buffer = ''; @@ -64,7 +74,7 @@ export default class MySQLParser extends Transform { } } - checkQuote (char) { + checkQuote (char: string) { const isQuote = !this.isEscape && (char === '\'' || char === '"'); if (isQuote && this.currentQuote === char) this.currentQuote = null; @@ -77,7 +87,7 @@ export default class MySQLParser extends Transform { if (this.isDelimiter) return false; - let query = false; + let query: false | string = false; let demiliterFound = false; if (this.currentQuote === null && this._buffer.length >= this.delimiter.length) demiliterFound = this._last9Chars.slice(-this.delimiter.length) === this.delimiter; diff --git a/src/main/libs/parsers/PostgreSQLParser.js b/src/main/libs/parsers/PostgreSQLParser.ts similarity index 83% rename from src/main/libs/parsers/PostgreSQLParser.js rename to src/main/libs/parsers/PostgreSQLParser.ts index 883a07ac..a4a35cfa 100644 --- a/src/main/libs/parsers/PostgreSQLParser.js +++ b/src/main/libs/parsers/PostgreSQLParser.ts @@ -1,7 +1,23 @@ -import { Transform } from 'stream'; +import { Transform, TransformCallback, TransformOptions } from 'stream'; export default class PostgreSQLParser extends Transform { - constructor (opts) { + private _buffer: string; + private _lastChar: string; + private _lastChars: string; + private _bodyWrapper: string; + private _bodyWrapperBuffer: string; + private _firstDollarFound: boolean; + private _isBody: boolean; + private _isSingleLineComment: boolean; + private _isMultiLineComment: boolean; + + encoding: BufferEncoding; + delimiter: string; + isEscape: boolean; + currentQuote: string; + isDelimiter: boolean; + + constructor (opts?: TransformOptions & { delimiter: string }) { opts = { delimiter: ';', encoding: 'utf8', @@ -30,7 +46,7 @@ export default class PostgreSQLParser extends Transform { return this._isSingleLineComment || this._isMultiLineComment; } - _transform (chunk, encoding, next) { + _transform (chunk: Buffer, encoding: BufferEncoding, next: TransformCallback) { for (const char of chunk.toString(this.encoding)) { this.checkEscape(); this._buffer += char; @@ -82,7 +98,7 @@ export default class PostgreSQLParser extends Transform { } } - checkBodyWrapper (char) { + checkBodyWrapper (char: string) { if (this._isBody) this._isBody = this._lastChars !== this._bodyWrapper; @@ -111,7 +127,7 @@ export default class PostgreSQLParser extends Transform { } } - checkQuote (char) { + checkQuote (char: string) { const isQuote = !this.isEscape && (char === '\'' || char === '"'); if (isQuote && this.currentQuote === char) this.currentQuote = null; @@ -124,7 +140,7 @@ export default class PostgreSQLParser extends Transform { if (this._isBody || this._isComment) return false; - let query = false; + let query: false | string = false; let demiliterFound = false; if (this.currentQuote === null && this._buffer.length >= this.delimiter.length) diff --git a/src/main/workers/importer.js b/src/main/workers/importer.ts similarity index 74% rename from src/main/workers/importer.js rename to src/main/workers/importer.ts index f1c27636..67d56031 100644 --- a/src/main/workers/importer.js +++ b/src/main/workers/importer.ts @@ -1,7 +1,12 @@ +import * as antares from 'common/interfaces/antares'; +import * as pg from 'pg'; +import * as mysql from 'mysql2'; +import { MySQLClient } from '../libs/clients/MySQLClient'; +import { PostgreSQLClient } from '../libs/clients/PostgreSQLClient'; import { ClientsFactory } from '../libs/ClientsFactory'; -import MySQLImporter from '../libs/importers/sql/MysqlImporter'; +import MySQLImporter from '../libs/importers/sql/MySQLlImporter'; import PostgreSQLImporter from '../libs/importers/sql/PostgreSQLImporter'; -let importer; +let importer: antares.Importer; process.on('message', async ({ type, dbConfig, options }) => { if (type === 'init') { @@ -12,17 +17,17 @@ process.on('message', async ({ type, dbConfig, options }) => { schema: options.schema }, poolSize: 1 - }); + }) as MySQLClient | PostgreSQLClient; const pool = await connection.getConnectionPool(); switch (options.type) { case 'mysql': case 'maria': - importer = new MySQLImporter(pool, options); + importer = new MySQLImporter(pool as unknown as mysql.Pool, options); break; case 'pg': - importer = new PostgreSQLImporter(pool, options); + importer = new PostgreSQLImporter(pool as unknown as pg.PoolClient, options); break; default: process.send({