2022-04-15 23:13:23 +02:00
import * as antares from 'common/interfaces/antares' ;
import * as exporter from 'common/interfaces/exporter' ;
2023-08-18 15:57:31 +02:00
import { valueToSqlString } from 'common/libs/sqlUtils' ;
2022-04-15 23:13:23 +02:00
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
import * as QueryStream from 'pg-query-stream' ;
2023-08-18 15:57:31 +02:00
2022-04-15 23:13:23 +02:00
import { PostgreSQLClient } from '../../clients/PostgreSQLClient' ;
2023-08-18 15:57:31 +02:00
import { SqlExporter } from './SqlExporter' ;
2022-03-21 18:32:45 +01:00
export default class PostgreSQLExporter extends SqlExporter {
2022-04-15 23:13:23 +02:00
constructor ( client : PostgreSQLClient , tables : exporter.TableParams [ ] , options : exporter.ExportOptions ) {
super ( tables , options ) ;
this . _client = client ;
}
2022-03-21 18:32:45 +01:00
async getSqlHeader ( ) {
let dump = await super . getSqlHeader ( ) ;
dump += `
SET statement_timeout = 0 ;
SET lock_timeout = 0 ;
SET idle_in_transaction_session_timeout = 0 ;
SET client_encoding = 'UTF8' ;
SET standard_conforming_strings = on ;
2022-04-02 11:44:55 +02:00
SELECT pg_catalog . set_config ( 'search_path' , '' , false ) ;
2022-03-21 18:32:45 +01:00
SET check_function_bodies = false ;
SET xmloption = content ;
SET client_min_messages = warning ;
2022-03-22 12:40:14 +01:00
SET row_security = off ; \ n \ n \ n ` ;
2022-04-02 11:44:55 +02:00
if ( this . schemaName !== 'public' ) dump += ` CREATE SCHEMA " ${ this . schemaName } "; \ n \ n ` ;
2022-03-23 13:26:46 +01:00
dump += await this . getCreateTypes ( ) ;
2022-03-21 18:32:45 +01:00
return dump ;
}
2022-04-15 23:13:23 +02:00
async getCreateTable ( tableName : string ) {
/* eslint-disable camelcase */
interface SequenceRecord {
sequence_catalog : string ;
sequence_schema : string ;
sequence_name : string ;
data_type : string ;
numeric_precision : number ;
numeric_precision_radix : number ;
numeric_scale : number ;
start_value : string ;
minimum_value : string ;
maximum_value : string ;
increment : string ;
cycle_option : string ;
}
/* eslint-enable camelcase */
2022-03-21 18:32:45 +01:00
let createSql = '' ;
const sequences = [ ] ;
const columnsSql = [ ] ;
2024-01-19 18:03:20 +01:00
const arrayTypes : Record < string , string > = {
2022-03-22 12:40:14 +01:00
_int2 : 'smallint' ,
_int4 : 'integer' ,
_int8 : 'bigint' ,
_float4 : 'real' ,
_float8 : 'double precision' ,
_char : '"char"' ,
_varchar : 'character varying'
} ;
2022-03-21 18:32:45 +01:00
// Table columns
2022-04-02 11:44:55 +02:00
const { rows } = await this . _client . raw ( `
SELECT *
FROM "information_schema" . "columns"
WHERE "table_schema" = '${this.schemaName}'
AND "table_name" = '${tableName}'
ORDER BY "ordinal_position" ASC
` , { schema: 'information_schema' });
2022-03-21 18:32:45 +01:00
if ( ! rows . length ) return '' ;
for ( const column of rows ) {
2022-03-22 12:40:14 +01:00
let fieldType = column . data_type ;
2022-04-02 11:44:55 +02:00
if ( fieldType === 'USER-DEFINED' ) fieldType = ` " ${ this . schemaName } ". ${ column . udt_name } ` ;
2022-03-22 12:40:14 +01:00
else if ( fieldType === 'ARRAY' ) {
if ( Object . keys ( arrayTypes ) . includes ( fieldType ) )
2022-04-15 23:13:23 +02:00
fieldType = arrayTypes [ column . udt_name ] + '[]' ;
2022-03-22 12:40:14 +01:00
else
fieldType = column . udt_name . replaceAll ( '_' , '' ) + '[]' ;
}
2022-03-21 18:32:45 +01:00
const columnArr = [
` " ${ column . column_name } " ` ,
2022-03-22 12:40:14 +01:00
` ${ fieldType } ${ column . character_maximum_length ? ` ( ${ column . character_maximum_length } ) ` : '' } `
2022-03-21 18:32:45 +01:00
] ;
if ( column . column_default ) {
columnArr . push ( ` DEFAULT ${ column . column_default } ` ) ;
if ( column . column_default . includes ( 'nextval' ) ) {
2022-04-02 11:44:55 +02:00
const sequenceName = column . column_default . split ( '\'' ) [ 1 ] ;
2022-03-21 18:32:45 +01:00
sequences . push ( sequenceName ) ;
}
}
if ( column . is_nullable === 'NO' ) columnArr . push ( 'NOT NULL' ) ;
columnsSql . push ( columnArr . join ( ' ' ) ) ;
}
// Table sequences
2022-04-02 11:44:55 +02:00
for ( let sequence of sequences ) {
if ( sequence . includes ( '.' ) ) sequence = sequence . split ( '.' ) [ 1 ] ;
2022-03-21 18:32:45 +01:00
const { rows } = await this . _client
. select ( '*' )
. schema ( 'information_schema' )
. from ( 'sequences' )
. where ( { sequence_schema : ` = ' ${ this . schemaName } ' ` , sequence_name : ` = ' ${ sequence } ' ` } )
2022-04-15 23:13:23 +02:00
. run < SequenceRecord > ( ) ;
2022-03-21 18:32:45 +01:00
if ( rows . length ) {
2022-04-02 11:44:55 +02:00
createSql += ` CREATE SEQUENCE " ${ this . schemaName } "." ${ sequence } "
2022-03-21 18:32:45 +01:00
START WITH $ { rows [ 0 ] . start_value }
INCREMENT BY $ { rows [ 0 ] . increment }
MINVALUE $ { rows [ 0 ] . minimum_value }
MAXVALUE $ { rows [ 0 ] . maximum_value }
CACHE 1 ; \ n ` ;
2022-03-22 12:40:14 +01:00
// createSql += `\nALTER TABLE "${sequence}" OWNER TO ${this._client._params.user};\n\n`;
2022-03-21 18:32:45 +01:00
}
}
// Table create
2022-04-02 11:44:55 +02:00
createSql += ` \ nCREATE TABLE " ${ this . schemaName } "." ${ tableName } "(
2022-03-21 18:32:45 +01:00
$ { columnsSql . join ( ',\n ' ) }
) ; \ n ` ;
2022-03-22 12:40:14 +01:00
// createSql += `\nALTER TABLE "${tableName}" OWNER TO ${this._client._params.user};\n\n`;
2022-03-21 18:32:45 +01:00
// Table indexes
2022-03-22 12:40:14 +01:00
createSql += '\n' ;
2022-03-21 18:32:45 +01:00
const { rows : indexes } = await this . _client
. select ( '*' )
. schema ( 'pg_catalog' )
. from ( 'pg_indexes' )
. where ( { schemaname : ` = ' ${ this . schemaName } ' ` , tablename : ` = ' ${ tableName } ' ` } )
2022-04-15 23:13:23 +02:00
. run < { indexdef : string } > ( ) ;
2022-03-21 18:32:45 +01:00
for ( const index of indexes )
2022-04-02 11:44:55 +02:00
createSql += ` ${ index . indexdef } ; \ n ` ;
2022-03-21 18:32:45 +01:00
// Table foreigns
const { rows : foreigns } = await this . _client . raw ( `
SELECT
tc . table_schema ,
tc . constraint_name ,
tc . table_name ,
kcu . column_name ,
ccu . table_schema AS foreign_table_schema ,
ccu . table_name AS foreign_table_name ,
ccu . column_name AS foreign_column_name ,
rc . update_rule ,
rc . delete_rule
FROM information_schema . table_constraints AS tc
JOIN information_schema . key_column_usage AS kcu
ON tc . constraint_name = kcu . constraint_name
AND tc . table_schema = kcu . table_schema
JOIN information_schema . constraint_column_usage AS ccu
ON ccu . constraint_name = tc . constraint_name
AND ccu . table_schema = tc . table_schema
JOIN information_schema . referential_constraints AS rc
ON rc . constraint_name = kcu . constraint_name
WHERE tc . constraint_type = 'FOREIGN KEY' AND tc . table_schema = '${this.schemaName}'
AND tc . table_name = '${tableName}'
` );
for ( const foreign of foreigns ) {
2022-04-02 11:44:55 +02:00
this . _postTablesSql += ` \ nALTER TABLE ONLY " ${ this . schemaName } "." ${ tableName } "
ADD CONSTRAINT "${foreign.constraint_name}" FOREIGN KEY ( "${foreign.column_name}" ) REFERENCES "${this.schemaName}" . "${foreign.foreign_table_name}" ( "${foreign.foreign_column_name}" ) ON UPDATE $ { foreign . update_rule } ON DELETE $ { foreign . delete_rule } ; \ n ` ;
2022-03-21 18:32:45 +01:00
}
return createSql ;
}
2022-04-15 23:13:23 +02:00
getDropTable ( tableName : string ) {
2022-04-02 11:44:55 +02:00
return ` DROP TABLE IF EXISTS " ${ this . schemaName } "." ${ tableName } "; ` ;
2022-03-21 18:32:45 +01:00
}
2022-04-15 23:13:23 +02:00
async * getTableInsert ( tableName : string ) {
2022-03-21 18:32:45 +01:00
let rowCount = 0 ;
2022-03-22 12:59:13 +01:00
const sqlStr = '' ;
2022-03-21 18:32:45 +01:00
2022-03-23 13:26:46 +01:00
const countResults = await this . _client . raw ( ` SELECT COUNT(1) as count FROM " ${ this . schemaName } "." ${ tableName } " ` ) ;
2022-03-21 18:32:45 +01:00
if ( countResults . rows . length === 1 ) rowCount = countResults . rows [ 0 ] . count ;
if ( rowCount > 0 ) {
const columns = await this . _client . getTableColumns ( {
table : tableName ,
schema : this.schemaName
} ) ;
2022-04-02 11:44:55 +02:00
const columnNames = columns . map ( col = > '"' + col . name + '"' ) . join ( ', ' ) ;
2022-03-21 18:32:45 +01:00
yield sqlStr ;
const stream = await this . _queryStream (
2022-03-23 13:26:46 +01:00
` SELECT ${ columnNames } FROM " ${ this . schemaName } "." ${ tableName } " `
2022-03-21 18:32:45 +01:00
) ;
for await ( const row of stream ) {
if ( this . isCancelled ) {
stream . destroy ( ) ;
yield null ;
return ;
}
2022-04-02 11:44:55 +02:00
let sqlInsertString = ` INSERT INTO " ${ this . schemaName } "." ${ tableName } " ( ${ columnNames } ) VALUES ` ;
2022-03-21 18:32:45 +01:00
sqlInsertString += ' (' ;
2022-04-02 11:44:55 +02:00
for ( const i in columns ) {
const column = columns [ i ] ;
2022-03-21 18:32:45 +01:00
const val = row [ column . name ] ;
2022-07-25 12:19:58 +02:00
sqlInsertString += valueToSqlString ( { val , client : 'pg' , field : column } ) ;
2022-03-21 18:32:45 +01:00
2022-04-02 11:44:55 +02:00
if ( parseInt ( i ) !== columns . length - 1 )
2022-03-21 18:32:45 +01:00
sqlInsertString += ', ' ;
}
2022-03-22 12:40:14 +01:00
sqlInsertString += ');\n' ;
2022-03-21 18:32:45 +01:00
yield sqlInsertString ;
}
yield sqlStr ;
}
}
2022-03-23 13:26:46 +01:00
async getCreateTypes ( ) {
2022-03-22 12:40:14 +01:00
let sqlString = '' ;
2022-04-15 23:13:23 +02:00
const { rows : types } = await this . _client . raw < antares.QueryResult < { typname : string ; enumlabel : string } > > ( `
2022-03-22 12:40:14 +01:00
SELECT pg_type . typname , pg_enum . enumlabel
FROM pg_type
JOIN pg_enum ON pg_enum . enumtypid = pg_type . oid ;
` );
if ( types . length ) { // TODO: refactor
sqlString += this . buildComment ( 'Dump of types\n------------------------------------------------------------' ) + '\n\n' ;
const typesArr = types . reduce ( ( arr , type ) = > {
if ( arr . every ( el = > el . name !== type . typname ) )
arr . push ( { name : type.typname , enums : [ this . escapeAndQuote ( type . enumlabel ) ] } ) ;
else {
const i = arr . findIndex ( el = > el . name === type . typname ) ;
arr [ i ] . enums . push ( this . escapeAndQuote ( type . enumlabel ) ) ;
}
return arr ;
} , [ ] ) ;
for ( const type of typesArr ) {
2022-04-02 11:44:55 +02:00
sqlString += ` CREATE TYPE " ${ this . schemaName } "." ${ type . name } " AS ENUM (
2022-03-22 12:40:14 +01:00
$ { type . enums . join ( ',\n\t' ) }
) ; ` ;
}
// sqlString += `\nALTER TYPE "${tableName}" OWNER TO ${this._client._params.user};\n`
}
return sqlString ;
}
2022-03-27 11:41:35 +02:00
async getCreateAggregates ( ) {
let sqlString = '' ;
const { rows : aggregates } = await this . _client . raw ( `
SELECT proname
FROM pg_proc
WHERE prokind = 'a'
AND pronamespace : : regnamespace : : text = '${this.schemaName}'
ORDER BY 1 ;
` );
if ( aggregates . length ) {
for ( const aggregate of aggregates ) {
const { rows : aggregateDef } = await this . _client . raw (
` SELECT
format (
E 'CREATE AGGREGATE %s (\n%s\n);'
, ( pg_identify_object ( 'pg_proc' : : regclass , aggfnoid , 0 ) ) . identity
, array_to_string (
ARRAY [
format ( E '\tSFUNC = %s' , aggtransfn : : regproc )
, format ( E '\tSTYPE = %s' , format_type ( aggtranstype , NULL ) )
, CASE aggfinalfn WHEN '-' : : regproc THEN NULL ELSE format ( E '\tFINALFUNC = %s' , aggfinalfn : : text ) END
, CASE aggsortop WHEN 0 THEN NULL ELSE format ( E '\tSORTOP = %s' , oprname ) END
, CASE WHEN agginitval IS NULL THEN NULL ELSE format ( E '\tINITCOND = %s' , agginitval ) END
]
, E ',\n'
)
)
FROM pg_aggregate
LEFT JOIN pg_operator ON pg_operator . oid = aggsortop
WHERE aggfnoid = '${this.schemaName}.${aggregate.proname}' : : regproc ; `
) ;
if ( aggregateDef . length )
2022-04-02 11:44:55 +02:00
sqlString += '\n\n' + aggregateDef [ 0 ] . format ;
2022-03-27 11:41:35 +02:00
}
}
return sqlString + '\n\n\n' ;
}
2022-03-21 18:32:45 +01:00
async getViews ( ) {
2022-03-22 17:25:34 +01:00
const { rows : views } = await this . _client . raw ( ` SELECT * FROM "pg_views" WHERE "schemaname"=' ${ this . schemaName } ' ` ) ;
2022-03-21 18:32:45 +01:00
let sqlString = '' ;
for ( const view of views ) {
2022-03-27 11:41:35 +02:00
sqlString += ` \ nDROP VIEW IF EXISTS " ${ view . viewname } "; \ n ` ;
2022-04-01 18:36:02 +02:00
2022-04-02 11:44:55 +02:00
// const { rows: columns } = await this._client
// .select('*')
// .schema('information_schema')
// .from('columns')
// .where({ table_schema: `= '${this.schemaName}'`, table_name: `= '${view.viewname}'` })
// .orderBy({ ordinal_position: 'ASC' })
// .run();
// sqlString += `
// CREATE VIEW "${this.schemaName}"."${view.viewname}" AS
// SELECT
// ${columns.reduce((acc, curr) => {
// const fieldType = curr.data_type === 'USER-DEFINED' ? curr.udt_name : curr.data_type;
// acc.push(`NULL::${fieldType}${curr.character_maximum_length ? `(${curr.character_maximum_length})` : ''} AS "${curr.column_name}"`);
// return acc;
// }, []).join(',\n ')};
// `;
sqlString += ` \ nCREATE OR REPLACE VIEW " ${ this . schemaName } "." ${ view . viewname } " AS \ n ${ view . definition } \ n ` ;
2022-03-21 18:32:45 +01:00
}
return sqlString ;
}
async getTriggers ( ) {
2022-04-15 23:13:23 +02:00
/* eslint-disable camelcase */
interface TriggersResult {
event_object_table : string ;
table_name : string ;
trigger_name : string ;
events : string [ ] ;
event_manipulation : string ;
}
/* eslint-enable camelcase */
2022-03-27 11:41:35 +02:00
let sqlString = '' ;
// Trigger functions
const { rows : triggerFunctions } = await this . _client . raw (
` SELECT DISTINCT routine_name AS name FROM information_schema.routines WHERE routine_type = 'FUNCTION' AND routine_schema = ' ${ this . schemaName } ' AND data_type = 'trigger' `
) ;
for ( const func of triggerFunctions ) {
const { rows : functionDef } = await this . _client . raw (
` SELECT pg_get_functiondef((SELECT oid FROM pg_proc WHERE proname = ' ${ func . name } ')) AS definition `
) ;
2022-04-02 11:44:55 +02:00
sqlString += ` \ n ${ functionDef [ 0 ] . definition } ; \ n ` ;
2022-03-27 11:41:35 +02:00
}
2022-04-15 23:13:23 +02:00
const { rows : triggers } = await this . _client . raw < antares.QueryResult < TriggersResult > > (
2022-03-22 17:25:34 +01:00
` SELECT * FROM "information_schema"."triggers" WHERE "trigger_schema"=' ${ this . schemaName } ' `
2022-03-21 18:32:45 +01:00
) ;
2022-03-22 17:25:34 +01:00
const remappedTriggers = triggers . reduce ( ( acc , trigger ) = > {
const i = acc . findIndex ( t = > t . trigger_name === trigger . trigger_name && t . event_object_table === trigger . event_object_table ) ;
if ( i === - 1 ) {
trigger . events = [ trigger . event_manipulation ] ;
acc . push ( trigger ) ;
}
else
acc [ i ] . events . push ( trigger . event_manipulation ) ;
2022-03-21 18:32:45 +01:00
2022-03-22 17:25:34 +01:00
return acc ;
} , [ ] ) ;
2022-03-21 18:32:45 +01:00
2022-03-22 17:25:34 +01:00
for ( const trigger of remappedTriggers )
2022-04-02 11:44:55 +02:00
sqlString += ` \ nCREATE TRIGGER " ${ trigger . trigger_name } " ${ trigger . action_timing } ${ trigger . events . join ( ' OR ' ) } ON " ${ this . schemaName } "." ${ trigger . event_object_table } " FOR EACH ${ trigger . action_orientation } ${ trigger . action_statement } ; \ n ` ;
2022-03-21 18:32:45 +01:00
return sqlString ;
}
2022-03-23 13:26:46 +01:00
async getFunctions ( ) {
2022-03-21 18:32:45 +01:00
let sqlString = '' ;
2022-03-23 13:26:46 +01:00
const { rows : functions } = await this . _client . raw (
` SELECT DISTINCT routine_name AS name FROM information_schema.routines WHERE routine_type = 'FUNCTION' AND routine_schema = ' ${ this . schemaName } ' AND data_type != 'trigger' `
) ;
2022-03-21 18:32:45 +01:00
2022-03-23 13:26:46 +01:00
for ( const func of functions ) {
const { rows : functionDef } = await this . _client . raw (
` SELECT pg_get_functiondef((SELECT oid FROM pg_proc WHERE proname = ' ${ func . name } ')) AS definition `
2022-03-21 18:32:45 +01:00
) ;
2022-04-02 11:44:55 +02:00
sqlString += ` \ n ${ functionDef [ 0 ] . definition } ; \ n ` ;
2022-03-21 18:32:45 +01:00
}
2022-03-27 11:41:35 +02:00
sqlString += await this . getCreateAggregates ( ) ;
2022-03-21 18:32:45 +01:00
return sqlString ;
}
2022-03-23 13:26:46 +01:00
async getRoutines ( ) {
2022-03-21 18:32:45 +01:00
let sqlString = '' ;
2022-03-23 13:26:46 +01:00
const { rows : functions } = await this . _client . raw (
` SELECT DISTINCT routine_name AS name FROM information_schema.routines WHERE routine_type = 'PROCEDURE' AND routine_schema = ' ${ this . schemaName } ' `
) ;
2022-03-21 18:32:45 +01:00
2022-03-23 13:26:46 +01:00
for ( const func of functions ) {
const { rows : functionDef } = await this . _client . raw (
` SELECT pg_get_functiondef((SELECT oid FROM pg_proc WHERE proname = ' ${ func . name } ')) AS definition `
) ;
2022-04-02 11:44:55 +02:00
sqlString += ` \ n ${ functionDef [ 0 ] . definition } ; \ n ` ;
2022-03-21 18:32:45 +01:00
}
return sqlString ;
}
2022-04-15 23:13:23 +02:00
async _queryStream ( sql : string ) {
2022-03-21 18:32:45 +01:00
const connection = await this . _client . getConnection ( ) ;
const query = new QueryStream ( sql , null ) ;
2022-04-15 23:13:23 +02:00
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const stream = ( connection as any ) . query ( query ) ;
2022-03-21 18:32:45 +01:00
const dispose = ( ) = > connection . end ( ) ;
stream . on ( 'end' , dispose ) ;
stream . on ( 'error' , dispose ) ;
stream . on ( 'close' , dispose ) ;
return stream ;
}
2022-04-15 23:13:23 +02:00
escapeAndQuote ( val : string ) {
2022-03-21 18:32:45 +01:00
// eslint-disable-next-line no-control-regex
const CHARS_TO_ESCAPE = /[\0\b\t\n\r\x1a"'\\]/g ;
2024-01-19 18:03:20 +01:00
const CHARS_ESCAPE_MAP : Record < string , string > = {
2022-03-21 18:32:45 +01:00
'\0' : '\\0' ,
'\b' : '\\b' ,
'\t' : '\\t' ,
'\n' : '\\n' ,
'\r' : '\\r' ,
'\x1a' : '\\Z' ,
'"' : '\\"' ,
'\'' : '\\\'' ,
'\\' : '\\\\'
} ;
let chunkIndex = CHARS_TO_ESCAPE . lastIndex = 0 ;
let escapedVal = '' ;
let match ;
while ( ( match = CHARS_TO_ESCAPE . exec ( val ) ) ) {
escapedVal += val . slice ( chunkIndex , match . index ) + CHARS_ESCAPE_MAP [ match [ 0 ] ] ;
chunkIndex = CHARS_TO_ESCAPE . lastIndex ;
}
if ( chunkIndex === 0 )
return ` ' ${ val } ' ` ;
if ( chunkIndex < val . length )
return ` ' ${ escapedVal + val . slice ( chunkIndex ) } ' ` ;
return ` ' ${ escapedVal } ' ` ;
}
}