Moduletrace/bun-sqlite
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add queue tables, quote SQL identifiers (v1.3.0)

Browse Source 
- isQueue table option adds queue columns and a (status, available_at)
  index; BunSQLite.queue exposes add, claim, complete, fail and
  release_stale with atomic claims
- Set busy_timeout on connections so workers in separate processes wait
  instead of failing with SQLITE_BUSY
- Quote table and column names with backticks in generated SQL, so
  reserved words (to, order, group) work and unknown columns error
  instead of matching as string literals
- Tests for queue ops, multi-process claims and reserved-word names

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
tben
2026-10-18 16:29:14 +01:00
co-authored by Claude Opus 5.5
parent 07807cb19d
commit 938a1da71d
45 changed files with 1524 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dist/functions/grab-sqlite-client.js
Vendored
+3
View File
@@ -13,6 +13,9 @@ export default function grabSQLiteClient({ config }) {
create: true,
});
sqliteVec.load(SQLiteClient);
// Wait for other connections' write locks instead of failing with
// SQLITE_BUSY (queue workers in separate processes)
SQLiteClient.run("PRAGMA busy_timeout = 5000;");
if (config.wal_mode) {
SQLiteClient.run("PRAGMA journal_mode = WAL;");
}
dist/index.d.ts
Vendored
+9 -1
View File
@@ -3,6 +3,7 @@ import DbInsert from "./lib/sqlite/db-insert";
import DbSelect from "./lib/sqlite/db-select";
import DbSQL from "./lib/sqlite/db-sql";
import DbUpdate from "./lib/sqlite/db-update";
import { QueueAdd, QueueClaim, QueueComplete, QueueFail, QueueReleaseStale } from "./lib/sqlite/db-queue";
import grabDbSchema from "./utils/grab-db-schema";
import grabJoinFieldsFromQueryObject from "./utils/grab-join-fields-from-query-object";
declare const BunSQLite: {
@@ -11,6 +12,13 @@ declare const BunSQLite: {
readonly update: typeof DbUpdate;
readonly delete: typeof DbDelete;
readonly sql: typeof DbSQL;
readonly queue: {
readonly add: typeof QueueAdd;
readonly claim: typeof QueueClaim;
readonly complete: typeof QueueComplete;
readonly fail: typeof QueueFail;
readonly release_stale: typeof QueueReleaseStale;
};
readonly utils: {
readonly grab_db_schema: typeof grabDbSchema;
readonly grab_join_fields_from_query_object: typeof grabJoinFieldsFromQueryObject;
@@ -18,4 +26,4 @@ declare const BunSQLite: {
};
export default BunSQLite;
export type * from "./types";
export { UsersOmitedFields, MariaDBCollations, TextFieldTypesArray, BUN_SQLITE_DATATYPES, ServerQueryOperators, ServerQueryEqualities, SQlComparisons, DataCrudRequestMethods, DataCrudRequestMethodsLowerCase, DsqlCrudActions, QueryFields, DockerComposeServices, IndexTypes, DefaultFields, } from "./types";
export { UsersOmitedFields, MariaDBCollations, TextFieldTypesArray, BUN_SQLITE_DATATYPES, ServerQueryOperators, ServerQueryEqualities, SQlComparisons, DataCrudRequestMethods, DataCrudRequestMethodsLowerCase, DsqlCrudActions, QueryFields, DockerComposeServices, IndexTypes, DefaultFields, QueueFields, QueueStatuses, } from "./types";
dist/index.js
Vendored
+9 -1
View File
@@ -4,6 +4,7 @@ import DbInsert from "./lib/sqlite/db-insert";
import DbSelect from "./lib/sqlite/db-select";
import DbSQL from "./lib/sqlite/db-sql";
import DbUpdate from "./lib/sqlite/db-update";
import { QueueAdd, QueueClaim, QueueComplete, QueueFail, QueueReleaseStale, } from "./lib/sqlite/db-queue";
import grabDbSchema from "./utils/grab-db-schema";
import grabJoinFieldsFromQueryObject from "./utils/grab-join-fields-from-query-object";
/**
@@ -22,10 +23,17 @@ const BunSQLite = {
update: withInit(DbUpdate),
delete: withInit(DbDelete),
sql: withInit(DbSQL),
queue: {
add: withInit(QueueAdd),
claim: withInit(QueueClaim),
complete: withInit(QueueComplete),
fail: withInit(QueueFail),
release_stale: withInit(QueueReleaseStale),
},
utils: {
grab_db_schema: withInit(grabDbSchema),
grab_join_fields_from_query_object: grabJoinFieldsFromQueryObject,
},
};
export default BunSQLite;
export { UsersOmitedFields, MariaDBCollations, TextFieldTypesArray, BUN_SQLITE_DATATYPES, ServerQueryOperators, ServerQueryEqualities, SQlComparisons, DataCrudRequestMethods, DataCrudRequestMethodsLowerCase, DsqlCrudActions, QueryFields, DockerComposeServices, IndexTypes, DefaultFields, } from "./types";
export { UsersOmitedFields, MariaDBCollations, TextFieldTypesArray, BUN_SQLITE_DATATYPES, ServerQueryOperators, ServerQueryEqualities, SQlComparisons, DataCrudRequestMethods, DataCrudRequestMethodsLowerCase, DsqlCrudActions, QueryFields, DockerComposeServices, IndexTypes, DefaultFields, QueueFields, QueueStatuses, } from "./types";
dist/lib/grab-duplicate-safe-insert-sql.js
Vendored
+4 -4
View File
@@ -1,5 +1,5 @@
import init from "../functions/init";
import { sqlIdent, sqlQuoteIdent } from "../utils/sql-safe";
import { sqlName } from "../utils/sql-safe";
export default async function ({ sql: passed_sql, table, data }) {
let sql = passed_sql;
const { dbSchema } = init();
@@ -9,7 +9,7 @@ export default async function ({ sql: passed_sql, table, data }) {
const set_sql_arr = Object.keys(Array.isArray(data) ? data[0] : data)
.filter((field) => field !== "updated_at")
.map((field) => {
const column = sqlIdent(field, "column name");
const column = sqlName(field, "column name");
return `${column} = excluded.${column}`;
});
set_sql_arr.push(`updated_at = ${now}`);
@@ -19,13 +19,13 @@ export default async function ({ sql: passed_sql, table, data }) {
const field = unique_fields[i];
if (!field?.fieldName)
continue;
sql += ` ON CONFLICT(${sqlQuoteIdent(field.fieldName)}) DO UPDATE SET ${set_sql}`;
sql += ` ON CONFLICT(${sqlName(field.fieldName, "column name")}) DO UPDATE SET ${set_sql}`;
}
if (table_schema.uniqueConstraints?.[0]) {
for (let i = 0; i < table_schema.uniqueConstraints.length; i++) {
const constraint = table_schema.uniqueConstraints[i];
const columns = constraint?.constraintTableFields
?.map((c) => sqlQuoteIdent(String(c.value)))
?.map((c) => sqlName(c.value, "column name"))
?.join(", ");
if (!columns)
continue;
dist/lib/schema/create-db-schema.js
Vendored
+5
View File
@@ -19,6 +19,11 @@ export default function createDBSchema(params) {
* Reorder Tables (parents before children with FKs)
*/
const db_schema = orderDBSchema(params);
for (const table of db_schema.tables) {
if (table.isQueue && table.isVector) {
throw new Error(`Table "${table.tableName}" can't be both a queue and a vector table`);
}
}
db.run("PRAGMA foreign_keys = OFF;");
db.run("BEGIN");
try {
dist/lib/sqlite/db-delete.js
Vendored
+2 -2
View File
@@ -1,7 +1,7 @@
import dbHandler from "../db-handler";
import _ from "lodash";
import sqlGenerator from "../../utils/sql-generator";
import { sqlIdent } from "../../utils/sql-safe";
import { sqlName } from "../../utils/sql-safe";
export default async function DbDelete({ table, query, targetId, config, }) {
let sqlObj = null;
try {
@@ -13,7 +13,7 @@ export default async function DbDelete({ table, query, targetId, config, }) {
});
const whereClause = sqlObj.where.string;
if (whereClause) {
let sql = `DELETE FROM ${sqlIdent(table, "table name")} ${whereClause}`;
let sql = `DELETE FROM ${sqlName(table, "table name")} ${whereClause}`;
sqlObj.string = sql;
sqlObj.values = sqlObj.where.values;
const res = dbHandler({
dist/lib/sqlite/db-queue.d.ts
Vendored
+64
View File
@@ -0,0 +1,64 @@
import type { APIResponseObject, BunSQLiteConfig } from "../../types";
type BaseParams<Table extends string = string> = {
table: Table;
config?: BunSQLiteConfig;
};
/**
* # Add jobs to a queue table
* @description `delay` (ms) holds the jobs back before they can be claimed
*/
export declare function QueueAdd<Schema extends {
[k: string]: any;
} = {
[k: string]: any;
}, Table extends string = string>({ table, data, delay, config, }: BaseParams<Table> & {
data: Schema[];
delay?: number;
}): Promise<APIResponseObject<Schema>>;
/**
* # Claim jobs from a queue table
* @description Claimed jobs are set to `processing` in one statement, so
* two workers can never claim the same job. Stale locks are released first.
*/
export declare function QueueClaim<Schema extends {
[k: string]: any;
} = {
[k: string]: any;
}, Table extends string = string>({ table, worker, limit, config, }: BaseParams<Table> & {
worker?: string;
limit?: number;
}): Promise<APIResponseObject<Schema>>;
/**
* # Mark a claimed job as done
* @description Deletes the job instead when `deleteOnComplete` is set
*/
export declare function QueueComplete<Schema extends {
[k: string]: any;
} = {
[k: string]: any;
}, Table extends string = string>({ table, id, config, }: BaseParams<Table> & {
id: number | string;
}): Promise<APIResponseObject<Schema>>;
/**
* # Mark a claimed job as failed
* @description The job goes back to `pending` with an exponential
* `backoff` delay, or to `failed` once it has used `maxAttempts`
*/
export declare function QueueFail<Schema extends {
[k: string]: any;
} = {
[k: string]: any;
}, Table extends string = string>({ table, id, error, config, }: BaseParams<Table> & {
id: number | string;
error?: string;
}): Promise<APIResponseObject<Schema>>;
/**
* # Release stale locks
* @description For jobs whose worker died. `claim` also does this.
*/
export declare function QueueReleaseStale<Schema extends {
[k: string]: any;
} = {
[k: string]: any;
}, Table extends string = string>({ table, config }: BaseParams<Table>): Promise<APIResponseObject<Schema>>;
export {};
dist/lib/sqlite/db-queue.js
Vendored
+186
View File
@@ -0,0 +1,186 @@
import dbHandler from "../db-handler";
import init from "../../functions/init";
import DbInsert from "./db-insert";
import { sqlInt, sqlName } from "../../utils/sql-safe";
const DefaultQueueOptions = {
maxAttempts: 3,
lockTimeout: 300_000,
backoff: 1000,
deleteOnComplete: false,
};
/**
* Validate the table name and resolve its queue options from the schema
*/
function resolveQueue(table) {
const { dbSchema } = init();
const tableSchema = dbSchema.tables.find((t) => t.tableName === table);
if (!tableSchema?.isQueue) {
throw new Error(`Table "${table}" is not a queue table (isQueue)`);
}
const options = {
...DefaultQueueOptions,
...(typeof tableSchema.isQueue === "object" ? tableSchema.isQueue : {}),
};
return { tableName: sqlName(table, "table name"), options };
}
/**
* Move jobs that have been `processing` past `lockTimeout` back to
* `pending`, or to `failed` once they are out of attempts
*/
function releaseStaleSql(tableName) {
return `UPDATE ${tableName} SET status = CASE WHEN attempts >= ? THEN 'failed' ELSE 'pending' END, last_error = CASE WHEN attempts >= ? THEN 'Lock timed out' ELSE last_error END, locked_at = NULL, locked_by = NULL, updated_at = ? WHERE status = 'processing' AND locked_at < ? RETURNING *`;
}
function rowsResponse(res, sql) {
if (!res.success) {
return { ...res, debug: { sql } };
}
const rows = res.payload || [];
return {
success: true,
payload: rows,
singleRes: rows[0] ?? null,
debug: { sql },
};
}
/**
* # Add jobs to a queue table
* @description `delay` (ms) holds the jobs back before they can be claimed
*/
export async function QueueAdd({ table, data, delay = 0, config, }) {
try {
resolveQueue(table);
const available_at = Date.now() + sqlInt(delay, "delay");
return await DbInsert({
table,
data: data.map((d) => ({
...d,
status: "pending",
attempts: 0,
available_at,
})),
config,
});
}
catch (error) {
return { success: false, error: error.message };
}
}
/**
* # Claim jobs from a queue table
* @description Claimed jobs are set to `processing` in one statement, so
* two workers can never claim the same job. Stale locks are released first.
*/
export async function QueueClaim({ table, worker, limit = 1, config, }) {
let sql = "";
try {
const { tableName, options } = resolveQueue(table);
const now = Date.now();
const released = dbHandler({
query: releaseStaleSql(tableName),
values: [
options.maxAttempts,
options.maxAttempts,
now,
now - options.lockTimeout,
],
config,
});
if (!released.success) {
return released;
}
sql = `UPDATE ${tableName} SET status = 'processing', attempts = attempts + 1, locked_at = ?, locked_by = ?, updated_at = ? WHERE id IN (SELECT id FROM ${tableName} WHERE status = 'pending' AND available_at <= ? ORDER BY id LIMIT ?) RETURNING *`;
const res = dbHandler({
query: sql,
values: [now, worker ?? null, now, now, sqlInt(limit, "limit")],
config,
});
return rowsResponse(res, sql);
}
catch (error) {
return { success: false, error: error.message, debug: { sql } };
}
}
/**
* # Mark a claimed job as done
* @description Deletes the job instead when `deleteOnComplete` is set
*/
export async function QueueComplete({ table, id, config, }) {
let sql = "";
try {
const { tableName, options } = resolveQueue(table);
sql = options.deleteOnComplete
? `DELETE FROM ${tableName} WHERE id = ? AND status = 'processing' RETURNING *`
: `UPDATE ${tableName} SET status = 'completed', locked_at = NULL, locked_by = NULL, updated_at = ? WHERE id = ? AND status = 'processing' RETURNING *`;
const res = dbHandler({
query: sql,
values: options.deleteOnComplete ? [id] : [Date.now(), id],
config,
});
const out = rowsResponse(res, sql);
return out.success && !out.payload?.length
? { ...out, success: false, msg: "No processing job with this id" }
: out;
}
catch (error) {
return { success: false, error: error.message, debug: { sql } };
}
}
/**
* # Mark a claimed job as failed
* @description The job goes back to `pending` with an exponential
* `backoff` delay, or to `failed` once it has used `maxAttempts`
*/
export async function QueueFail({ table, id, error, config, }) {
let sql = "";
try {
const { tableName, options } = resolveQueue(table);
const now = Date.now();
sql = `UPDATE ${tableName} SET status = CASE WHEN attempts >= ? THEN 'failed' ELSE 'pending' END, available_at = CASE WHEN attempts >= ? THEN available_at ELSE ? + ? * (1 << MIN(MAX(attempts - 1, 0), 30)) END, last_error = ?, locked_at = NULL, locked_by = NULL, updated_at = ? WHERE id = ? AND status = 'processing' RETURNING *`;
const res = dbHandler({
query: sql,
values: [
options.maxAttempts,
options.maxAttempts,
now,
options.backoff,
error ?? null,
now,
id,
],
config,
});
const out = rowsResponse(res, sql);
return out.success && !out.payload?.length
? { ...out, success: false, msg: "No processing job with this id" }
: out;
}
catch (error) {
return { success: false, error: error.message, debug: { sql } };
}
}
/**
* # Release stale locks
* @description For jobs whose worker died. `claim` also does this.
*/
export async function QueueReleaseStale({ table, config }) {
let sql = "";
try {
const { tableName, options } = resolveQueue(table);
const now = Date.now();
sql = releaseStaleSql(tableName);
const res = dbHandler({
query: sql,
values: [
options.maxAttempts,
options.maxAttempts,
now,
now - options.lockTimeout,
],
config,
});
return rowsResponse(res, sql);
}
catch (error) {
return { success: false, error: error.message, debug: { sql } };
}
}
dist/lib/sqlite/db-update.js
Vendored
+3 -3
View File
@@ -3,7 +3,7 @@ import _ from "lodash";
import init from "../../functions/init";
import sqlGenerator from "../../utils/sql-generator";
import { sqlValue } from "../../utils/sql-insert-generator";
import { sqlIdent } from "../../utils/sql-safe";
import { sqlName } from "../../utils/sql-safe";
export default async function DbUpdate({ table, data, query, targetId, returning = true, config, }) {
let sqlObj = {
string: "",
@@ -23,7 +23,7 @@ export default async function DbUpdate({ table, data, query, targetId, returning
let values = [];
const whereClause = sqlQueryObj.where.string;
if (whereClause) {
let sql = `UPDATE ${sqlIdent(table, "table name")} SET`;
let sql = `UPDATE ${sqlName(table, "table name")} SET`;
const finalData = {
updated_at: Date.now(),
...data,
@@ -34,7 +34,7 @@ export default async function DbUpdate({ table, data, query, targetId, returning
if (!key)
continue;
const isLast = i == keys.length - 1;
sql += ` ${sqlIdent(key, "column name")}=?`;
sql += ` ${sqlName(key, "column name")}=?`;
values.push(sqlValue(finalData[key]));
if (!isLast) {
sql += `,`;
dist/types/index.d.ts
Vendored
+34
View File
@@ -89,6 +89,35 @@ export interface BUN_SQLITE_TableSchemaType {
* Type of vector. Defaults to `vec0`
*/
vectorType?: string;
/**
* Make this a queue table. Adds the `QueueFields` columns and an index
* on `(status, available_at)`. Use `BunSQLite.queue` to work the queue.
*/
isQueue?: boolean | BUN_SQLITE_QueueOptions;
}
/**
* Options for a queue table (`isQueue`).
*/
export interface BUN_SQLITE_QueueOptions {
/**
* Claims allowed before a failing job is marked `failed`. Defaults to `3`
*/
maxAttempts?: number;
/**
* Milliseconds a claimed job can stay `processing` before it is
* released back to the queue. Defaults to `300000` (5 minutes)
*/
lockTimeout?: number;
/**
* Base retry delay in milliseconds, doubled on every attempt. Defaults
* to `1000`
*/
backoff?: number;
/**
* Delete jobs on completion instead of marking them `completed`.
* Defaults to `false`
*/
deleteOnComplete?: boolean;
}
/**
* Reference object used to link a table to one of its child tables.
@@ -1420,6 +1449,11 @@ export type SchemaChange = {
* Default fields automatically suggested for new tables.
*/
export declare const DefaultFields: BUN_SQLITE_FieldSchemaType[];
export declare const QueueStatuses: readonly ["pending", "processing", "completed", "failed"];
/**
* Fields added to queue tables (`isQueue`)
*/
export declare const QueueFields: BUN_SQLITE_FieldSchemaType[];
export type BunSQLiteQueryFieldValues<F extends string = string, T extends string = string> = {
field: F;
table?: T;
dist/types/index.js
Vendored
+47
View File
@@ -185,3 +185,50 @@ export const DefaultFields = [
fieldDescription: "The time when the record was updated. (Unix Timestamp)",
},
];
export const QueueStatuses = [
"pending",
"processing",
"completed",
"failed",
];
/**
* Fields added to queue tables (`isQueue`)
*/
export const QueueFields = [
{
fieldName: "status",
dataType: "TEXT",
notNullValue: true,
defaultValue: "pending",
fieldDescription: `The state of the job. (${QueueStatuses.join(" | ")})`,
},
{
fieldName: "attempts",
dataType: "INTEGER",
notNullValue: true,
defaultValue: 0,
fieldDescription: "The number of times the job has been claimed.",
},
{
fieldName: "available_at",
dataType: "INTEGER",
notNullValue: true,
defaultValue: 0,
fieldDescription: "The time from which the job can be claimed. (Unix Timestamp)",
},
{
fieldName: "locked_at",
dataType: "INTEGER",
fieldDescription: "The time when the job was claimed. (Unix Timestamp)",
},
{
fieldName: "locked_by",
dataType: "TEXT",
fieldDescription: "The worker that claimed the job.",
},
{
fieldName: "last_error",
dataType: "TEXT",
fieldDescription: "The error from the last failed attempt.",
},
];
dist/utils/append-default-fields-to-db-schema.js
Vendored
+12 -1
View File
@@ -1,11 +1,22 @@
import _ from "lodash";
import { DefaultFields } from "../types";
import { DefaultFields, QueueFields, } from "../types";
const QueueFieldNames = QueueFields.map((f) => f.fieldName);
export default function ({ dbSchema }) {
const finaldbSchema = _.cloneDeep(dbSchema);
finaldbSchema.tables = finaldbSchema.tables.map((t) => {
const newTable = _.cloneDeep(t);
newTable.fields = newTable.fields.filter((f) => !f.fieldName?.match(/^(id|created_at|updated_at)$/));
newTable.fields.unshift(...DefaultFields);
if (newTable.isQueue) {
newTable.fields = newTable.fields.filter((f) => !QueueFieldNames.includes(f.fieldName));
newTable.fields.push(...QueueFields);
const indexName = `idx_${newTable.tableName}_queue`;
newTable.indexes = (newTable.indexes || []).filter((idx) => idx.indexName !== indexName);
newTable.indexes.push({
indexName,
indexTableFields: ["status", "available_at"],
});
}
return newTable;
});
return finaldbSchema;
dist/utils/sql-generator-gen-join-str.js
Vendored
+6 -3
View File
@@ -1,4 +1,4 @@
import { sqlIdent, sqlOperator, sqlRawEquality } from "./sql-safe";
import { sqlName, sqlOperator, sqlRawEquality } from "./sql-safe";
export default function sqlGenGenJoinStr({ join, mtch, table_name }) {
let values = [];
if (mtch.__batch) {
@@ -24,7 +24,10 @@ export default function sqlGenGenJoinStr({ join, mtch, table_name }) {
};
}
const equality = mtch.raw_equality ? sqlRawEquality(mtch.raw_equality) : "=";
const lhs = `${sqlIdent(typeof mtch.source == "object" ? mtch.source.tableName : table_name, "table name")}.${sqlIdent(typeof mtch.source == "object" ? mtch.source.fieldName : mtch.source, "field name")}`;
// `table_name` is already validated and quoted by `sqlGenerator`
const lhs = `${typeof mtch.source == "object"
? sqlName(mtch.source.tableName, "table name")
: table_name}.${sqlName(typeof mtch.source == "object" ? mtch.source.fieldName : mtch.source, "field name")}`;
if (mtch.between) {
values.push(mtch.between.min, mtch.between.max);
return {
@@ -50,7 +53,7 @@ export default function sqlGenGenJoinStr({ join, mtch, table_name }) {
const targetField = typeof mtch.target == "object"
? mtch.target.fieldName
: mtch.target;
return `${sqlIdent(targetTable, "table name")}.${sqlIdent(targetField, "field name")}`;
return `${sqlName(targetTable, "table name")}.${sqlName(targetField, "field name")}`;
})();
return {
str: `${lhs} ${equality} ${rhs}`,
dist/utils/sql-generator-gen-query-str.js
Vendored
+13 -13
View File
@@ -2,7 +2,7 @@ import { isUndefined } from "lodash";
import sqlGenGrabConcatStr from "./sql-generator-grab-concat-str";
import sqlGenGenJoinStr from "./sql-generator-gen-join-str";
import sqlGenGrabSelectFieldSQL from "./sql-generator-grab-select-field-sql";
import { sqlIdent, sqlJoinType, sqlOperator } from "./sql-safe";
import { sqlName, sqlJoinType, sqlOperator } from "./sql-safe";
export default function sqlGenGenQueryStr(params) {
let str = "SELECT";
const genObject = params.genObject;
@@ -40,9 +40,9 @@ export default function sqlGenGenQueryStr(params) {
const countSubQuery = genObject.countSubQueries[i];
if (!countSubQuery)
continue;
const countTable = sqlIdent(countSubQuery.table, "table name");
const countTable = sqlName(countSubQuery.table, "table name");
const tableAlias = countSubQuery.table_alias
? sqlIdent(countSubQuery.table_alias, "alias")
? sqlName(countSubQuery.table_alias, "alias")
: undefined;
let subQStr = `(SELECT COUNT(*)`;
subQStr += ` FROM ${countTable}${tableAlias ? ` ${tableAlias}` : ""}`;
@@ -51,19 +51,19 @@ export default function sqlGenGenQueryStr(params) {
const csqSrc = countSubQuery.srcTrgMap[j];
if (!csqSrc)
continue;
subQStr += ` ${tableAlias || countTable}.${sqlIdent(csqSrc.src, "field name")}`;
subQStr += ` ${tableAlias || countTable}.${sqlName(csqSrc.src, "field name")}`;
if (typeof csqSrc.trg == "string") {
subQStr += ` = ?`;
sqlSearhValues.push(csqSrc.trg);
}
else if (typeof csqSrc.trg == "object") {
subQStr += ` = ${sqlIdent(csqSrc.trg.table, "table name")}.${sqlIdent(csqSrc.trg.field, "field name")}`;
subQStr += ` = ${sqlName(csqSrc.trg.table, "table name")}.${sqlName(csqSrc.trg.field, "field name")}`;
}
if (j < countSubQuery.srcTrgMap.length - 1) {
subQStr += ` AND `;
}
}
subQStr += ` )) AS ${sqlIdent(countSubQuery.alias, "alias")}`;
subQStr += ` )) AS ${sqlName(countSubQuery.alias, "alias")}`;
countSqls.push(subQStr);
}
str += `, ${countSqls.join(",")}`;
@@ -74,13 +74,13 @@ export default function sqlGenGenQueryStr(params) {
.flat()
.filter((j) => !isUndefined(j))
.map((joinObj) => {
const joinTableName = sqlIdent(joinObj.alias ? joinObj.alias : joinObj.tableName, "join table name");
const joinTableName = sqlName(joinObj.alias ? joinObj.alias : joinObj.tableName, "join table name");
if (existingJoinTableNames.includes(joinTableName))
return null;
existingJoinTableNames.push(joinTableName);
if (joinObj.group_concat) {
return sqlGenGrabConcatStr({
field: `${joinTableName}.${sqlIdent(joinObj.group_concat.field, "field name")}`,
field: `${joinTableName}.${sqlName(joinObj.group_concat.field, "field name")}`,
alias: joinObj.group_concat.alias,
separator: joinObj.group_concat.separator,
distinct: joinObj.group_concat.distinct,
@@ -90,9 +90,9 @@ export default function sqlGenGenQueryStr(params) {
return joinObj.selectFields
.map((selectField) => {
if (typeof selectField != "object") {
return `${joinTableName}.${sqlIdent(selectField, "field name")}`;
return `${joinTableName}.${sqlName(selectField, "field name")}`;
}
const field = `${joinTableName}.${sqlIdent(selectField.field, "field name")}`;
const field = `${joinTableName}.${sqlName(selectField.field, "field name")}`;
if (selectField.group_concat &&
selectField.alias) {
return sqlGenGrabConcatStr({
@@ -122,7 +122,7 @@ export default function sqlGenGenQueryStr(params) {
aliasSelectField = `DISTINCT ${field}`;
}
if (selectField.alias)
aliasSelectField += ` AS ${sqlIdent(selectField.alias, "alias")}`;
aliasSelectField += ` AS ${sqlName(selectField.alias, "alias")}`;
return aliasSelectField;
})
.join(",");
@@ -145,11 +145,11 @@ export default function sqlGenGenQueryStr(params) {
.flat()
.filter((j) => !isUndefined(j))
.map((join) => {
const joinTable = sqlIdent(join.tableName, "table name");
const joinTable = sqlName(join.tableName, "table name");
return (sqlJoinType(join.joinType) +
" " +
(join.alias
? `${joinTable} ${sqlIdent(join.alias, "alias")}`
? `${joinTable} ${sqlName(join.alias, "alias")}`
: joinTable) +
" ON " +
(() => {
dist/utils/sql-generator-gen-search-str.js
Vendored
+4 -4
View File
@@ -1,5 +1,5 @@
import sqlGenOperatorGen from "./sql-gen-operator-gen";
import { sqlField, sqlIdent, sqlOperator, sqlRawEquality } from "./sql-safe";
import { sqlColumn, sqlName, sqlOperator, sqlRawEquality } from "./sql-safe";
function isPresent(val) {
return val !== undefined && val !== null;
}
@@ -8,10 +8,10 @@ function isValuesObject(val) {
}
export default function sqlGenGenSearchStr({ queryObj, join, field, table_name, }) {
let sqlSearhValues = [];
const safeField = sqlField(field);
const safeField = sqlColumn(field);
const finalFieldName = (() => {
if (queryObj?.tableName) {
return `${sqlIdent(queryObj.tableName, "table name")}.${safeField}`;
return `${sqlName(queryObj.tableName, "table name")}.${safeField}`;
}
if (join) {
return `${table_name}.${safeField}`;
@@ -28,7 +28,7 @@ export default function sqlGenGenSearchStr({ queryObj, join, field, table_name,
equality: val.equality || queryObj.equality,
fieldName: finalFieldName,
value: isFieldValue
? `${sqlIdent(val.tableName, "table name")}.${sqlIdent(val.fieldName, "field name")}`
? `${sqlName(val.tableName, "table name")}.${sqlName(val.fieldName, "field name")}`
: val.value,
isValueFieldValue: isFieldValue,
});
dist/utils/sql-generator-grab-concat-str.d.ts
Vendored
+3
View File
@@ -1,4 +1,7 @@
type Param = {
/**
* An already validated and quoted column reference
*/
field: string;
alias: string;
separator?: string;
dist/utils/sql-generator-grab-concat-str.js
Vendored
+3 -3
View File
@@ -1,14 +1,14 @@
import { sqlField, sqlIdent, sqlStringLiteral } from "./sql-safe";
import { sqlName, sqlStringLiteral } from "./sql-safe";
export default function sqlGenGrabConcatStr({ alias, field, separator = ",", distinct, }) {
let gc = `GROUP_CONCAT(`;
if (distinct) {
gc += `DISTINCT `;
}
gc += sqlField(field);
gc += field;
if (!distinct) {
gc += `, ${sqlStringLiteral(String(separator))}`;
}
gc += `)`;
gc += ` AS ${sqlIdent(alias, "alias")}`;
gc += ` AS ${sqlName(alias, "alias")}`;
return gc;
}
dist/utils/sql-generator-grab-select-field-sql.js
Vendored
+5 -5
View File
@@ -1,20 +1,20 @@
import sqlGenGrabConcatStr from "./sql-generator-grab-concat-str";
import { sqlField, sqlIdent } from "./sql-safe";
import { sqlColumn, sqlName } from "./sql-safe";
export default function sqlGenGrabSelectFieldSQL({ selectFields, append_table_names, table_name }) {
let str = "";
str += ` ${selectFields
?.map((fld) => {
if (typeof fld != "object") {
return sqlField(fld);
return sqlColumn(fld);
}
const fld_name = sqlField(fld.fieldName);
const fld_name = sqlColumn(fld.fieldName);
const final_fld_name = append_table_names && !fld_name.includes(".")
? `${table_name}.${fld_name}`
: fld_name;
if (fld.group_concat) {
return sqlGenGrabConcatStr({
field: final_fld_name,
alias: fld.group_concat.alias || fld.alias || fld_name,
alias: fld.group_concat.alias || fld.alias || String(fld.fieldName),
separator: fld.group_concat.separator,
distinct: fld.group_concat.distinct,
});
@@ -46,7 +46,7 @@ export default function sqlGenGrabSelectFieldSQL({ selectFields, append_table_na
fld_str += final_fld_name;
}
if (alias) {
fld_str += ` AS ${sqlIdent(alias, "alias")}`;
fld_str += ` AS ${sqlName(alias, "alias")}`;
}
return fld_str;
})
dist/utils/sql-generator.js
Vendored
+7 -7
View File
@@ -1,13 +1,13 @@
import sqlGenGenSearchStr from "./sql-generator-gen-search-str";
import sqlGenGenQueryStr from "./sql-generator-gen-query-str";
import { sqlField, sqlIdent, sqlInt, sqlOperator, sqlOrderStrategy, } from "./sql-safe";
import { sqlColumn, sqlInt, sqlName, sqlOperator, sqlOrderStrategy, } from "./sql-safe";
/**
* # SQL Query Generator
* @description Generates a parameterized SQLite SELECT query. All values are
* returned in `values` to be bound; identifiers are validated.
*/
export default function sqlGenerator({ tableName: passedTableName, genObject, count, targetId, }) {
const tableName = sqlIdent(passedTableName, "table name");
const tableName = sqlName(passedTableName, "table name");
const finalQuery = genObject?.query ? genObject.query : undefined;
const queryKeys = finalQuery ? Object.keys(finalQuery) : undefined;
const sqlSearhValues = [];
@@ -83,10 +83,10 @@ export default function sqlGenerator({ tableName: passedTableName, genObject, co
const grabGroupStr = (group) => {
if (typeof group == "object") {
return group.table
? `${sqlIdent(group.table, "table name")}.${sqlIdent(group.field, "field name")}`
: sqlField(group.field);
? `${sqlName(group.table, "table name")}.${sqlName(group.field, "field name")}`
: sqlColumn(group.field);
}
return sqlField(group);
return sqlColumn(group);
};
const group_by_txt = Array.isArray(genObject.group)
? genObject.group.map(grabGroupStr).join(",")
@@ -95,8 +95,8 @@ export default function sqlGenerator({ tableName: passedTableName, genObject, co
}
function grabOrderString(order) {
const field = genObject?.join && !String(order.field).includes(".")
? `${tableName}.${sqlIdent(order.field, "order field")}`
: sqlField(order.field, "order field");
? `${tableName}.${sqlName(order.field, "order field")}`
: sqlColumn(order.field, "order field");
return ` ${field} ${sqlOrderStrategy(order.strategy)}`;
}
if (genObject?.order) {
dist/utils/sql-insert-generator.js
Vendored
+3 -3
View File
@@ -1,10 +1,10 @@
import { sqlIdent } from "./sql-safe";
import { sqlIdent, sqlName } from "./sql-safe";
/**
* # SQL Insert Generator
*/
export default function sqlInsertGenerator({ tableName, data, dbFullName, }) {
const finalDbName = dbFullName
? `${sqlIdent(dbFullName, "database name")}.`
? `${sqlName(dbFullName, "database name")}.`
: "";
if (Array.isArray(data) && data?.[0]) {
let insertKeys = [];
@@ -32,7 +32,7 @@ export default function sqlInsertGenerator({ tableName, data, dbFullName, }) {
})
.join(",")})`);
});
let query = `INSERT INTO ${finalDbName}${sqlIdent(tableName, "table name")} (${insertKeys.join(",")}) VALUES ${queryBatches.join(",")}`;
let query = `INSERT INTO ${finalDbName}${sqlName(tableName, "table name")} (${insertKeys.map((ky) => sqlName(ky)).join(",")}) VALUES ${queryBatches.join(",")}`;
return {
query: query,
values: queryValues,
dist/utils/sql-safe.d.ts
Vendored
+13
View File
@@ -11,6 +11,19 @@ export declare function sqlIdent(name: unknown, label?: string): string;
* @description Accepts `field`, `table.field` or `table.*`
*/
export declare function sqlField(name: unknown, label?: string): string;
/**
* # Validate and quote an identifier for queries
* @description Quoting lets reserved words (`to`, `order`, ...) be used as
* names. Backticks, not double quotes: SQLite reads an unknown `"name"` as
* a string literal, so a typo would silently match nothing instead of
* failing with "no such column".
*/
export declare function sqlName(name: unknown, label?: string): string;
/**
* # Validate and quote a column reference
* @description Accepts `field`, `table.field` or `table.*`
*/
export declare function sqlColumn(name: unknown, label?: string): string;
/**
* # Quote an identifier for DDL statements
*/
dist/utils/sql-safe.js
Vendored
+20
View File
@@ -25,6 +25,26 @@ export function sqlField(name, label = "field name") {
}
return str;
}
/**
* # Validate and quote an identifier for queries
* @description Quoting lets reserved words (`to`, `order`, ...) be used as
* names. Backticks, not double quotes: SQLite reads an unknown `"name"` as
* a string literal, so a typo would silently match nothing instead of
* failing with "no such column".
*/
export function sqlName(name, label = "identifier") {
return `\`${sqlIdent(name, label)}\``;
}
/**
* # Validate and quote a column reference
* @description Accepts `field`, `table.field` or `table.*`
*/
export function sqlColumn(name, label = "field name") {
return sqlField(name, label)
.split(".")
.map((part) => (part === "*" ? part : `\`${part}\``))
.join(".");
}
/**
* # Quote an identifier for DDL statements
*/