Skip to content

fix: PostgreSQL dollar-tag detection inside quoted strings #1028

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Kovania wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fix: PostgreSQL dollar-tag detection inside quoted strings #1028

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
195 changes: 87 additions & 108 deletions src/common/libs/sqlUtils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import hexToBinary, { HexChar } from './hexToBinary';
* @param {ClientCode} dbType - The database type (e.g., 'pg', 'mysql').
* @returns {string[]} - An array of separated SQL queries.
*/
export const querySplitter =(sql: string, dbType: ClientCode): string[] => {
export const querySplitter = (sql: string, dbType: ClientCode): string[] => {
const queries: string[] = [];
let currentQuery = '';
let insideBlock = false;
Expand All @@ -42,50 +42,63 @@ export const querySplitter =(sql: string, dbType: ClientCode): string[] => {

for (let i = 0; i < line.length; i++) {
const char = line[i];
const nextChar = line[i + 1] || '';

// Handle dollar-quoted blocks in PostgreSQL BEFORE handling regular strings
if (dbType === 'pg' && !insideString && char === '$') {
const remainingText = line.slice(i);
const match = remainingText.match(dollarTagRegex);

if (match && match.index === 0) {
const tag = match[0];

if (!insideDollarTag) {
// Starting a dollar-quoted block
insideDollarTag = true;
dollarTagDelimiter = tag;
currentQuery += tag;
i += tag.length - 1;
continue;
}
else if (dollarTagDelimiter === tag) {
// Ending a dollar-quoted block
insideDollarTag = false;
dollarTagDelimiter = null;
currentQuery += tag;
i += tag.length - 1;
continue;
}
}
}

// Handle string boundaries
if ((char === '\'' || char === '"') && (!insideString || char === stringDelimiter)) {
// Handle string boundaries with proper escaping (only single quotes for PostgreSQL string literals)
if (!insideDollarTag && char === '\'') {
if (!insideString) {
// Starting a string
insideString = true;
stringDelimiter = char;
}
else {
else if (char === stringDelimiter) {
// Check for escaped quotes (e.g., '' in PostgreSQL)
if (dbType === 'pg' && nextChar === '\'') {
// Escaped quote, add both and skip next
currentQuery += char + nextChar;
i++;
continue;
}
// Ending a string
insideString = false;
stringDelimiter = null;
}
}

currentQuery += char;

if (dbType === 'pg') {
// Handle dollar-quoted blocks in PostgreSQL
if (!insideString && line.slice(i).match(dollarTagRegex)) {
const match = line.slice(i).match(dollarTagRegex);
if (match) {
const tag = match[0];
if (!insideDollarTag) {
insideDollarTag = true;
dollarTagDelimiter = tag;
currentQuery += tag;
i += tag.length - 1;
}
else if (dollarTagDelimiter === tag) {
insideDollarTag = false;
dollarTagDelimiter = null;
currentQuery += tag;
i += tag.length - 1;
}
}
}
}

// Check BEGIN-END blocks
if (!insideString && !insideDollarTag) {
if (beginRegex.test(line))
insideBlock = true;
if (beginRegex.test(line)) insideBlock = true;

if (insideBlock && endRegex.test(line))
insideBlock = false;
if (insideBlock && endRegex.test(line)) insideBlock = false;
}
}

Expand All @@ -97,8 +110,7 @@ export const querySplitter =(sql: string, dbType: ClientCode): string[] => {
}

// Add any remaining query
if (currentQuery.trim())
queries.push(currentQuery.trim());
if (currentQuery.trim()) queries.push(currentQuery.trim());

return queries;
};
Expand All @@ -119,8 +131,7 @@ export const removeComments = (sql: string): string => {
const nextChar = sql[i + 1] || '';

// Handle single-line comments (--)
if (!insideMultiLineComment && char === '-' && nextChar === '-')
insideSingleLineComment = true;
if (!insideMultiLineComment && char === '-' && nextChar === '-') insideSingleLineComment = true;

// Handle multi-line comments (/* */)
if (!insideSingleLineComment && char === '/' && nextChar === '*') {
Expand All @@ -137,14 +148,12 @@ export const removeComments = (sql: string): string => {

// Skip characters inside comments
if (insideSingleLineComment) {
if (char === '\n')
insideSingleLineComment = false;
if (char === '\n') insideSingleLineComment = false;

continue;
}

if (insideMultiLineComment)
continue;
if (insideMultiLineComment) continue;

// Append non-comment characters to the result
result += char;
Expand All @@ -163,7 +172,7 @@ export const sqlEscaper = (string: string): string => {
// eslint-disable-next-line no-control-regex
const pattern = /[\0\x08\x09\x1a\n\r"'\\\%]/gm;
const regex = new RegExp(pattern);
return string.replace(regex, char => {
return string.replace(regex, (char) => {
const m = ['\\0', '\\x08', '\\x09', '\\x1a', '\\n', '\\r', '\'', '\"', '\\', '\\\\', '%'];
const r = ['\\\\0', '\\\\b', '\\\\t', '\\\\z', '\\\\n', '\\\\r', '\\\'', '\\\"', '\\\\', '\\\\\\\\', '\%'];
return r[m.indexOf(char)] || char;
Expand All @@ -178,13 +187,10 @@ export const sqlEscaper = (string: string): string => {
*/
export const objectToGeoJSON = (val: any) => {
if (Array.isArray(val)) {
if (getArrayDepth(val) === 1)
return lineString(val.reduce((acc, curr) => [...acc, [curr.x, curr.y]], []));
else
return polygon(val.map(arr => arr.reduce((acc: any, curr: any) => [...acc, [curr.x, curr.y]], [])));
if (getArrayDepth(val) === 1) return lineString(val.reduce((acc, curr) => [...acc, [curr.x, curr.y]], []));
else return polygon(val.map((arr) => arr.reduce((acc: any, curr: any) => [...acc, [curr.x, curr.y]], [])));
}
else
return point([val.x, val.y]);
else return point([val.x, val.y]);
};

/**
Expand All @@ -209,10 +215,9 @@ export const escapeAndQuote = (val: string, client: ClientCode) => {
'\\': '\\\\'
};

if (sw === '"')
CHARS_ESCAPE_MAP['"'] = '\\"';
if (sw === '"') CHARS_ESCAPE_MAP['"'] = '\\"';

let chunkIndex = CHARS_TO_ESCAPE.lastIndex = 0;
let chunkIndex = (CHARS_TO_ESCAPE.lastIndex = 0);
let escapedVal = '';
let match;

Expand All @@ -221,11 +226,9 @@ export const escapeAndQuote = (val: string, client: ClientCode) => {
chunkIndex = CHARS_TO_ESCAPE.lastIndex;
}

if (chunkIndex === 0)
return `${sw}${val}${sw}`;
if (chunkIndex === 0) return `${sw}${val}${sw}`;

if (chunkIndex < val.length)
return `${sw}${escapedVal + val.slice(chunkIndex)}${sw}`;
if (chunkIndex < val.length) return `${sw}${escapedVal + val.slice(chunkIndex)}${sw}`;

return `${sw}${escapedVal}${sw}`;
};
Expand All @@ -237,87 +240,66 @@ export const escapeAndQuote = (val: string, client: ClientCode) => {
* @returns {string} - The generated SQL string.
*/
export const valueToSqlString = (args: {
val: any;
client: ClientCode;
field: { type: string; datePrecision?: number; precision?: number | false; isArray?: boolean };
val: any;
client: ClientCode;
field: { type: string; datePrecision?: number; precision?: number | false; isArray?: boolean };
}): string => {
let parsedValue;
const { val, client, field } = args;
const { stringsWrapper: sw } = customizations[client];

if (val === null)
parsedValue = 'NULL';
else if (DATE.includes(field.type)) {
parsedValue = moment(val).isValid()
? escapeAndQuote(moment(val).format('YYYY-MM-DD'), client)
: val;
}
if (val === null) parsedValue = 'NULL';
else if (DATE.includes(field.type))
parsedValue = moment(val).isValid() ? escapeAndQuote(moment(val).format('YYYY-MM-DD'), client) : val;
else if (DATETIME.includes(field.type)) {
let datePrecision = '';
for (let i = 0; i < field.datePrecision; i++)
datePrecision += i === 0 ? '.S' : 'S';
for (let i = 0; i < field.datePrecision; i++) datePrecision += i === 0 ? '.S' : 'S';

parsedValue = moment(val).isValid()
? escapeAndQuote(moment(val).format(`YYYY-MM-DD HH:mm:ss${datePrecision}`), client)
: escapeAndQuote(val, client);
}
else if ('isArray' in field && field.isArray) {
let localVal;
if (Array.isArray(val)) {
localVal = JSON
.stringify(val)
.replaceAll('[', '{')
.replaceAll(']', '}');
}
else {
localVal = typeof val === 'string'
? val
.replaceAll('[', '{')
.replaceAll(']', '}')
: '';
}
if (Array.isArray(val))
localVal = JSON.stringify(val).replaceAll('[', '{').replaceAll(']', '}');
else
localVal = typeof val === 'string' ? val.replaceAll('[', '{').replaceAll(']', '}') : '';

parsedValue = `'${localVal}'`;
}
else if (TEXT_SEARCH.includes(field.type))
parsedValue = `'${val.replaceAll('\'', '\'\'')}'`;
else if (TEXT_SEARCH.includes(field.type)) parsedValue = `'${val.replaceAll('\'', '\'\'')}'`;
else if (BIT.includes(field.type))
parsedValue = `b'${hexToBinary(Buffer.from(new Uint8Array(Object.values(val))).toString('hex') as undefined as HexChar[])}'`;
else if (BLOB.includes(field.type)) {
let buffer: Buffer;
if (val instanceof Uint8Array)
buffer = Buffer.from(val);
else
buffer = val;
if (val instanceof Uint8Array) buffer = Buffer.from(val);
else buffer = val;

if (['mysql', 'maria'].includes(client))
parsedValue = `X'${buffer.toString('hex').toUpperCase()}'`;
else if (client === 'pg')
parsedValue = `decode('${buffer.toString('hex').toUpperCase()}', 'hex')`;
if (['mysql', 'maria'].includes(client)) parsedValue = `X'${buffer.toString('hex').toUpperCase()}'`;
else if (client === 'pg') parsedValue = `decode('${buffer.toString('hex').toUpperCase()}', 'hex')`;
}
else if (NUMBER.includes(field.type))
parsedValue = val;
else if (FLOAT.includes(field.type))
parsedValue = parseFloat(val);
else if (NUMBER.includes(field.type)) parsedValue = val;
else if (FLOAT.includes(field.type)) parsedValue = parseFloat(val);
else if (SPATIAL.includes(field.type)) {
let geoJson;
if (IS_MULTI_SPATIAL.includes(field.type)) {
const features = [];
for (const element of val)
features.push(objectToGeoJSON(element));
for (const element of val) features.push(objectToGeoJSON(element));

geoJson = {
type: 'FeatureCollection',
features
};
}
else
geoJson = objectToGeoJSON(val);
else geoJson = objectToGeoJSON(val);

parsedValue = `ST_GeomFromGeoJSON('${JSON.stringify(geoJson)}')`;
}
else if (val === '') parsedValue = `${sw}${sw}`;
else {
parsedValue = typeof val === 'string'
parsedValue =
typeof val === 'string'
? escapeAndQuote(val, client)
: typeof val === 'object'
? escapeAndQuote(JSON.stringify(val), client)
Expand All @@ -334,17 +316,17 @@ export const valueToSqlString = (args: {
* @returns {string} - The generated SQL INSERT query.
*/
export const jsonToSqlInsert = (args: {
json: Record<string, any>[];
client: ClientCode;
fields: Record<string, { type: string; datePrecision: number }>;
table: string;
options?: { sqlInsertAfter: number; sqlInsertDivider: 'bytes' | 'rows' };
json: Record<string, any>[];
client: ClientCode;
fields: Record<string, { type: string; datePrecision: number }>;
table: string;
options?: { sqlInsertAfter: number; sqlInsertDivider: 'bytes' | 'rows' };
}) => {
const { client, json, fields, table, options } = args;
const sqlInsertAfter = options && options.sqlInsertAfter ? options.sqlInsertAfter : 1;
const sqlInsertDivider = options && options.sqlInsertDivider ? options.sqlInsertDivider : 'rows';
const { elementsWrapper: ew } = customizations[client];
const fieldNames = Object.keys(json[0]).map(key => `${ew}${key.split('.').pop()}${ew}`);
const fieldNames = Object.keys(json[0]).map((key) => `${ew}${key.split('.').pop()}${ew}`);
let insertStmt = `INSERT INTO ${ew}${table}${ew} (${fieldNames.join(', ')}) VALUES `;
let insertsString = '';
let queryLength = 0;
Expand All @@ -353,13 +335,11 @@ export const jsonToSqlInsert = (args: {
for (const row of json) {
const values = [];

values.push(Object.keys(row).map(key => (
valueToSqlString({ val: row[key], client, field: fields[key] })
)));
values.push(Object.keys(row).map((key) => valueToSqlString({ val: row[key], client, field: fields[key] })));

if (
(sqlInsertDivider === 'bytes' && queryLength >= sqlInsertAfter * 1024) ||
(sqlInsertDivider === 'rows' && rowsWritten === sqlInsertAfter)
(sqlInsertDivider === 'rows' && rowsWritten === sqlInsertAfter)
) {
insertsString += insertStmt + ';';
insertStmt = `\nINSERT INTO ${ew}${table}${ew} (${fieldNames.join(', ')}) VALUES `;
Expand All @@ -373,8 +353,7 @@ export const jsonToSqlInsert = (args: {
queryLength = insertStmt.length;
}

if (rowsWritten > 0)
insertsString += insertStmt + ';';
if (rowsWritten > 0) insertsString += insertStmt + ';';

return insertsString;
};
Expand Down