mirror of https://github.com/Fabio286/antares.git
refactor: db import ts refactor
This commit is contained in:
parent
ce0f278caf
commit
79f32ca442
|
@ -2,7 +2,7 @@ import * as mysql from 'mysql2/promise';
|
|||
import * as pg from 'pg';
|
||||
import MysqlExporter from 'src/main/libs/exporters/sql/MysqlExporter';
|
||||
import PostgreSQLExporter from 'src/main/libs/exporters/sql/PostgreSQLExporter';
|
||||
import MySQLImporter from 'src/main/libs/importers/sql/MysqlImporter';
|
||||
import MySQLImporter from 'src/main/libs/importers/sql/MySQLlImporter';
|
||||
import PostgreSQLImporter from 'src/main/libs/importers/sql/PostgreSQLImporter';
|
||||
import SSHConfig from 'ssh2-promise/lib/sshConfig';
|
||||
import { MySQLClient } from '../../main/libs/clients/MySQLClient';
|
||||
|
|
|
@ -0,0 +1,16 @@
|
|||
import * as antares from './antares';
|
||||
|
||||
export interface ImportOptions {
|
||||
uid: string;
|
||||
schema: string;
|
||||
type: antares.ClientCode;
|
||||
file: string;
|
||||
}
|
||||
|
||||
export interface ImportState {
|
||||
fileSize?: number;
|
||||
readPosition?: number;
|
||||
percentage?: number;
|
||||
queryCount?: number;
|
||||
op?: string;
|
||||
}
|
|
@ -1,8 +1,14 @@
|
|||
import fs from 'fs';
|
||||
import EventEmitter from 'events';
|
||||
import * as importer from 'common/interfaces/importer';
|
||||
import * as fs from 'fs';
|
||||
import * as EventEmitter from 'events';
|
||||
|
||||
export class BaseImporter extends EventEmitter {
|
||||
constructor (options) {
|
||||
protected _options;
|
||||
protected _isCancelled;
|
||||
protected _fileHandler;
|
||||
protected _state;
|
||||
|
||||
constructor (options: importer.ImportOptions) {
|
||||
super();
|
||||
this._options = options;
|
||||
this._isCancelled = false;
|
||||
|
@ -43,7 +49,7 @@ export class BaseImporter extends EventEmitter {
|
|||
this.emitUpdate({ op: 'cancelling' });
|
||||
}
|
||||
|
||||
emitUpdate (state) {
|
||||
emitUpdate (state: importer.ImportState) {
|
||||
this.emit('progress', { ...this._state, ...state });
|
||||
}
|
||||
|
|
@ -1,14 +1,18 @@
|
|||
import fs from 'fs/promises';
|
||||
import * as mysql from 'mysql2';
|
||||
import * as importer from 'common/interfaces/importer';
|
||||
import * as fs from 'fs/promises';
|
||||
import MySQLParser from '../../parsers/MySQLParser';
|
||||
import { BaseImporter } from '../BaseImporter';
|
||||
|
||||
export default class MySQLImporter extends BaseImporter {
|
||||
constructor (client, options) {
|
||||
protected _client: mysql.Pool
|
||||
|
||||
constructor (client: mysql.Pool, options: importer.ImportOptions) {
|
||||
super(options);
|
||||
this._client = client;
|
||||
}
|
||||
|
||||
async import () {
|
||||
async import (): Promise<void> {
|
||||
try {
|
||||
const { size: totalFileSize } = await fs.stat(this._options.file);
|
||||
const parser = new MySQLParser();
|
||||
|
@ -60,7 +64,8 @@ export default class MySQLImporter extends BaseImporter {
|
|||
|
||||
parser.on('pause', () => {
|
||||
this._fileHandler.unpipe(parser);
|
||||
this._fileHandler.readableFlowing = false;
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(this._fileHandler as any).readableFlowing = false;
|
||||
});
|
||||
|
||||
this._fileHandler.on('data', (chunk) => {
|
|
@ -1,14 +1,18 @@
|
|||
import * as pg from 'pg';
|
||||
import * as importer from 'common/interfaces/importer';
|
||||
import fs from 'fs/promises';
|
||||
import PostgreSQLParser from '../../parsers/PostgreSQLParser';
|
||||
import { BaseImporter } from '../BaseImporter';
|
||||
|
||||
export default class PostgreSQLImporter extends BaseImporter {
|
||||
constructor (client, options) {
|
||||
protected _client: pg.PoolClient;
|
||||
|
||||
constructor (client: pg.PoolClient, options: importer.ImportOptions) {
|
||||
super(options);
|
||||
this._client = client;
|
||||
}
|
||||
|
||||
async import () {
|
||||
async import (): Promise<void> {
|
||||
try {
|
||||
const { size: totalFileSize } = await fs.stat(this._options.file);
|
||||
const parser = new PostgreSQLParser();
|
||||
|
@ -60,7 +64,8 @@ export default class PostgreSQLImporter extends BaseImporter {
|
|||
|
||||
parser.on('pause', () => {
|
||||
this._fileHandler.unpipe(parser);
|
||||
this._fileHandler.readableFlowing = false;
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(this._fileHandler as any).readableFlowing = false;
|
||||
});
|
||||
|
||||
this._fileHandler.on('data', (chunk) => {
|
|
@ -1,7 +1,17 @@
|
|||
import { Transform } from 'stream';
|
||||
import { Transform, TransformCallback, TransformOptions } from 'stream';
|
||||
|
||||
export default class MySQLParser extends Transform {
|
||||
constructor (opts) {
|
||||
private _buffer: string;
|
||||
private _lastChar: string;
|
||||
private _last9Chars: string;
|
||||
|
||||
encoding: BufferEncoding;
|
||||
delimiter: string;
|
||||
isEscape: boolean;
|
||||
currentQuote: string;
|
||||
isDelimiter: boolean;
|
||||
|
||||
constructor (opts?: TransformOptions & { delimiter: string }) {
|
||||
opts = {
|
||||
delimiter: ';',
|
||||
encoding: 'utf8',
|
||||
|
@ -21,7 +31,7 @@ export default class MySQLParser extends Transform {
|
|||
this.isDelimiter = false;
|
||||
}
|
||||
|
||||
_transform (chunk, encoding, next) {
|
||||
_transform (chunk: Buffer, encoding: BufferEncoding, next: TransformCallback) {
|
||||
for (const char of chunk.toString(this.encoding)) {
|
||||
this.checkEscape();
|
||||
this._buffer += char;
|
||||
|
@ -49,7 +59,7 @@ export default class MySQLParser extends Transform {
|
|||
}
|
||||
}
|
||||
|
||||
checkNewDelimiter (char) {
|
||||
checkNewDelimiter (char: string) {
|
||||
if (this.currentQuote === null && this._last9Chars === 'delimiter') {
|
||||
this.isDelimiter = true;
|
||||
this._buffer = '';
|
||||
|
@ -64,7 +74,7 @@ export default class MySQLParser extends Transform {
|
|||
}
|
||||
}
|
||||
|
||||
checkQuote (char) {
|
||||
checkQuote (char: string) {
|
||||
const isQuote = !this.isEscape && (char === '\'' || char === '"');
|
||||
if (isQuote && this.currentQuote === char)
|
||||
this.currentQuote = null;
|
||||
|
@ -77,7 +87,7 @@ export default class MySQLParser extends Transform {
|
|||
if (this.isDelimiter)
|
||||
return false;
|
||||
|
||||
let query = false;
|
||||
let query: false | string = false;
|
||||
let demiliterFound = false;
|
||||
if (this.currentQuote === null && this._buffer.length >= this.delimiter.length)
|
||||
demiliterFound = this._last9Chars.slice(-this.delimiter.length) === this.delimiter;
|
|
@ -1,7 +1,23 @@
|
|||
import { Transform } from 'stream';
|
||||
import { Transform, TransformCallback, TransformOptions } from 'stream';
|
||||
|
||||
export default class PostgreSQLParser extends Transform {
|
||||
constructor (opts) {
|
||||
private _buffer: string;
|
||||
private _lastChar: string;
|
||||
private _lastChars: string;
|
||||
private _bodyWrapper: string;
|
||||
private _bodyWrapperBuffer: string;
|
||||
private _firstDollarFound: boolean;
|
||||
private _isBody: boolean;
|
||||
private _isSingleLineComment: boolean;
|
||||
private _isMultiLineComment: boolean;
|
||||
|
||||
encoding: BufferEncoding;
|
||||
delimiter: string;
|
||||
isEscape: boolean;
|
||||
currentQuote: string;
|
||||
isDelimiter: boolean;
|
||||
|
||||
constructor (opts?: TransformOptions & { delimiter: string }) {
|
||||
opts = {
|
||||
delimiter: ';',
|
||||
encoding: 'utf8',
|
||||
|
@ -30,7 +46,7 @@ export default class PostgreSQLParser extends Transform {
|
|||
return this._isSingleLineComment || this._isMultiLineComment;
|
||||
}
|
||||
|
||||
_transform (chunk, encoding, next) {
|
||||
_transform (chunk: Buffer, encoding: BufferEncoding, next: TransformCallback) {
|
||||
for (const char of chunk.toString(this.encoding)) {
|
||||
this.checkEscape();
|
||||
this._buffer += char;
|
||||
|
@ -82,7 +98,7 @@ export default class PostgreSQLParser extends Transform {
|
|||
}
|
||||
}
|
||||
|
||||
checkBodyWrapper (char) {
|
||||
checkBodyWrapper (char: string) {
|
||||
if (this._isBody)
|
||||
this._isBody = this._lastChars !== this._bodyWrapper;
|
||||
|
||||
|
@ -111,7 +127,7 @@ export default class PostgreSQLParser extends Transform {
|
|||
}
|
||||
}
|
||||
|
||||
checkQuote (char) {
|
||||
checkQuote (char: string) {
|
||||
const isQuote = !this.isEscape && (char === '\'' || char === '"');
|
||||
if (isQuote && this.currentQuote === char)
|
||||
this.currentQuote = null;
|
||||
|
@ -124,7 +140,7 @@ export default class PostgreSQLParser extends Transform {
|
|||
if (this._isBody || this._isComment)
|
||||
return false;
|
||||
|
||||
let query = false;
|
||||
let query: false | string = false;
|
||||
let demiliterFound = false;
|
||||
|
||||
if (this.currentQuote === null && this._buffer.length >= this.delimiter.length)
|
|
@ -1,7 +1,12 @@
|
|||
import * as antares from 'common/interfaces/antares';
|
||||
import * as pg from 'pg';
|
||||
import * as mysql from 'mysql2';
|
||||
import { MySQLClient } from '../libs/clients/MySQLClient';
|
||||
import { PostgreSQLClient } from '../libs/clients/PostgreSQLClient';
|
||||
import { ClientsFactory } from '../libs/ClientsFactory';
|
||||
import MySQLImporter from '../libs/importers/sql/MysqlImporter';
|
||||
import MySQLImporter from '../libs/importers/sql/MySQLlImporter';
|
||||
import PostgreSQLImporter from '../libs/importers/sql/PostgreSQLImporter';
|
||||
let importer;
|
||||
let importer: antares.Importer;
|
||||
|
||||
process.on('message', async ({ type, dbConfig, options }) => {
|
||||
if (type === 'init') {
|
||||
|
@ -12,17 +17,17 @@ process.on('message', async ({ type, dbConfig, options }) => {
|
|||
schema: options.schema
|
||||
},
|
||||
poolSize: 1
|
||||
});
|
||||
}) as MySQLClient | PostgreSQLClient;
|
||||
|
||||
const pool = await connection.getConnectionPool();
|
||||
|
||||
switch (options.type) {
|
||||
case 'mysql':
|
||||
case 'maria':
|
||||
importer = new MySQLImporter(pool, options);
|
||||
importer = new MySQLImporter(pool as unknown as mysql.Pool, options);
|
||||
break;
|
||||
case 'pg':
|
||||
importer = new PostgreSQLImporter(pool, options);
|
||||
importer = new PostgreSQLImporter(pool as unknown as pg.PoolClient, options);
|
||||
break;
|
||||
default:
|
||||
process.send({
|
Loading…
Reference in New Issue