Moduletrace/bun-sqlite
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add queue tables, quote SQL identifiers (v1.3.0)

Browse Source 
- isQueue table option adds queue columns and a (status, available_at)
  index; BunSQLite.queue exposes add, claim, complete, fail and
  release_stale with atomic claims
- Set busy_timeout on connections so workers in separate processes wait
  instead of failing with SQLITE_BUSY
- Quote table and column names with backticks in generated SQL, so
  reserved words (to, order, group) work and unknown columns error
  instead of matching as string literals
- Tests for queue ops, multi-process claims and reserved-word names

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
tben
2026-10-18 16:29:14 +01:00
co-authored by Claude Opus 5.5
parent 07807cb19d
commit 938a1da71d
45 changed files with 1524 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dist/utils/append-default-fields-to-db-schema.js
Vendored
+12 -1
View File
@@ -1,11 +1,22 @@
import _ from "lodash";
import { DefaultFields } from "../types";
import { DefaultFields, QueueFields, } from "../types";
const QueueFieldNames = QueueFields.map((f) => f.fieldName);
export default function ({ dbSchema }) {
const finaldbSchema = _.cloneDeep(dbSchema);
finaldbSchema.tables = finaldbSchema.tables.map((t) => {
const newTable = _.cloneDeep(t);
newTable.fields = newTable.fields.filter((f) => !f.fieldName?.match(/^(id|created_at|updated_at)$/));
newTable.fields.unshift(...DefaultFields);
if (newTable.isQueue) {
newTable.fields = newTable.fields.filter((f) => !QueueFieldNames.includes(f.fieldName));
newTable.fields.push(...QueueFields);
const indexName = `idx_${newTable.tableName}_queue`;
newTable.indexes = (newTable.indexes || []).filter((idx) => idx.indexName !== indexName);
newTable.indexes.push({
indexName,
indexTableFields: ["status", "available_at"],
});
}
return newTable;
});
return finaldbSchema;
dist/utils/sql-generator-gen-join-str.js
Vendored
+6 -3
View File
@@ -1,4 +1,4 @@
import { sqlIdent, sqlOperator, sqlRawEquality } from "./sql-safe";
import { sqlName, sqlOperator, sqlRawEquality } from "./sql-safe";
export default function sqlGenGenJoinStr({ join, mtch, table_name }) {
let values = [];
if (mtch.__batch) {
@@ -24,7 +24,10 @@ export default function sqlGenGenJoinStr({ join, mtch, table_name }) {
};
}
const equality = mtch.raw_equality ? sqlRawEquality(mtch.raw_equality) : "=";
const lhs = `${sqlIdent(typeof mtch.source == "object" ? mtch.source.tableName : table_name, "table name")}.${sqlIdent(typeof mtch.source == "object" ? mtch.source.fieldName : mtch.source, "field name")}`;
// `table_name` is already validated and quoted by `sqlGenerator`
const lhs = `${typeof mtch.source == "object"
? sqlName(mtch.source.tableName, "table name")
: table_name}.${sqlName(typeof mtch.source == "object" ? mtch.source.fieldName : mtch.source, "field name")}`;
if (mtch.between) {
values.push(mtch.between.min, mtch.between.max);
return {
@@ -50,7 +53,7 @@ export default function sqlGenGenJoinStr({ join, mtch, table_name }) {
const targetField = typeof mtch.target == "object"
? mtch.target.fieldName
: mtch.target;
return `${sqlIdent(targetTable, "table name")}.${sqlIdent(targetField, "field name")}`;
return `${sqlName(targetTable, "table name")}.${sqlName(targetField, "field name")}`;
})();
return {
str: `${lhs} ${equality} ${rhs}`,
dist/utils/sql-generator-gen-query-str.js
Vendored
+13 -13
View File
@@ -2,7 +2,7 @@ import { isUndefined } from "lodash";
import sqlGenGrabConcatStr from "./sql-generator-grab-concat-str";
import sqlGenGenJoinStr from "./sql-generator-gen-join-str";
import sqlGenGrabSelectFieldSQL from "./sql-generator-grab-select-field-sql";
import { sqlIdent, sqlJoinType, sqlOperator } from "./sql-safe";
import { sqlName, sqlJoinType, sqlOperator } from "./sql-safe";
export default function sqlGenGenQueryStr(params) {
let str = "SELECT";
const genObject = params.genObject;
@@ -40,9 +40,9 @@ export default function sqlGenGenQueryStr(params) {
const countSubQuery = genObject.countSubQueries[i];
if (!countSubQuery)
continue;
const countTable = sqlIdent(countSubQuery.table, "table name");
const countTable = sqlName(countSubQuery.table, "table name");
const tableAlias = countSubQuery.table_alias
? sqlIdent(countSubQuery.table_alias, "alias")
? sqlName(countSubQuery.table_alias, "alias")
: undefined;
let subQStr = `(SELECT COUNT(*)`;
subQStr += ` FROM ${countTable}${tableAlias ? ` ${tableAlias}` : ""}`;
@@ -51,19 +51,19 @@ export default function sqlGenGenQueryStr(params) {
const csqSrc = countSubQuery.srcTrgMap[j];
if (!csqSrc)
continue;
subQStr += ` ${tableAlias || countTable}.${sqlIdent(csqSrc.src, "field name")}`;
subQStr += ` ${tableAlias || countTable}.${sqlName(csqSrc.src, "field name")}`;
if (typeof csqSrc.trg == "string") {
subQStr += ` = ?`;
sqlSearhValues.push(csqSrc.trg);
}
else if (typeof csqSrc.trg == "object") {
subQStr += ` = ${sqlIdent(csqSrc.trg.table, "table name")}.${sqlIdent(csqSrc.trg.field, "field name")}`;
subQStr += ` = ${sqlName(csqSrc.trg.table, "table name")}.${sqlName(csqSrc.trg.field, "field name")}`;
}
if (j < countSubQuery.srcTrgMap.length - 1) {
subQStr += ` AND `;
}
}
subQStr += ` )) AS ${sqlIdent(countSubQuery.alias, "alias")}`;
subQStr += ` )) AS ${sqlName(countSubQuery.alias, "alias")}`;
countSqls.push(subQStr);
}
str += `, ${countSqls.join(",")}`;
@@ -74,13 +74,13 @@ export default function sqlGenGenQueryStr(params) {
.flat()
.filter((j) => !isUndefined(j))
.map((joinObj) => {
const joinTableName = sqlIdent(joinObj.alias ? joinObj.alias : joinObj.tableName, "join table name");
const joinTableName = sqlName(joinObj.alias ? joinObj.alias : joinObj.tableName, "join table name");
if (existingJoinTableNames.includes(joinTableName))
return null;
existingJoinTableNames.push(joinTableName);
if (joinObj.group_concat) {
return sqlGenGrabConcatStr({
field: `${joinTableName}.${sqlIdent(joinObj.group_concat.field, "field name")}`,
field: `${joinTableName}.${sqlName(joinObj.group_concat.field, "field name")}`,
alias: joinObj.group_concat.alias,
separator: joinObj.group_concat.separator,
distinct: joinObj.group_concat.distinct,
@@ -90,9 +90,9 @@ export default function sqlGenGenQueryStr(params) {
return joinObj.selectFields
.map((selectField) => {
if (typeof selectField != "object") {
return `${joinTableName}.${sqlIdent(selectField, "field name")}`;
return `${joinTableName}.${sqlName(selectField, "field name")}`;
}
const field = `${joinTableName}.${sqlIdent(selectField.field, "field name")}`;
const field = `${joinTableName}.${sqlName(selectField.field, "field name")}`;
if (selectField.group_concat &&
selectField.alias) {
return sqlGenGrabConcatStr({
@@ -122,7 +122,7 @@ export default function sqlGenGenQueryStr(params) {
aliasSelectField = `DISTINCT ${field}`;
}
if (selectField.alias)
aliasSelectField += ` AS ${sqlIdent(selectField.alias, "alias")}`;
aliasSelectField += ` AS ${sqlName(selectField.alias, "alias")}`;
return aliasSelectField;
})
.join(",");
@@ -145,11 +145,11 @@ export default function sqlGenGenQueryStr(params) {
.flat()
.filter((j) => !isUndefined(j))
.map((join) => {
const joinTable = sqlIdent(join.tableName, "table name");
const joinTable = sqlName(join.tableName, "table name");
return (sqlJoinType(join.joinType) +
" " +
(join.alias
? `${joinTable} ${sqlIdent(join.alias, "alias")}`
? `${joinTable} ${sqlName(join.alias, "alias")}`
: joinTable) +
" ON " +
(() => {
dist/utils/sql-generator-gen-search-str.js
Vendored
+4 -4
View File
@@ -1,5 +1,5 @@
import sqlGenOperatorGen from "./sql-gen-operator-gen";
import { sqlField, sqlIdent, sqlOperator, sqlRawEquality } from "./sql-safe";
import { sqlColumn, sqlName, sqlOperator, sqlRawEquality } from "./sql-safe";
function isPresent(val) {
return val !== undefined && val !== null;
}
@@ -8,10 +8,10 @@ function isValuesObject(val) {
}
export default function sqlGenGenSearchStr({ queryObj, join, field, table_name, }) {
let sqlSearhValues = [];
const safeField = sqlField(field);
const safeField = sqlColumn(field);
const finalFieldName = (() => {
if (queryObj?.tableName) {
return `${sqlIdent(queryObj.tableName, "table name")}.${safeField}`;
return `${sqlName(queryObj.tableName, "table name")}.${safeField}`;
}
if (join) {
return `${table_name}.${safeField}`;
@@ -28,7 +28,7 @@ export default function sqlGenGenSearchStr({ queryObj, join, field, table_name,
equality: val.equality || queryObj.equality,
fieldName: finalFieldName,
value: isFieldValue
? `${sqlIdent(val.tableName, "table name")}.${sqlIdent(val.fieldName, "field name")}`
? `${sqlName(val.tableName, "table name")}.${sqlName(val.fieldName, "field name")}`
: val.value,
isValueFieldValue: isFieldValue,
});
dist/utils/sql-generator-grab-concat-str.d.ts
Vendored
+3
View File
@@ -1,4 +1,7 @@
type Param = {
/**
* An already validated and quoted column reference
*/
field: string;
alias: string;
separator?: string;
dist/utils/sql-generator-grab-concat-str.js
Vendored
+3 -3
View File
@@ -1,14 +1,14 @@
import { sqlField, sqlIdent, sqlStringLiteral } from "./sql-safe";
import { sqlName, sqlStringLiteral } from "./sql-safe";
export default function sqlGenGrabConcatStr({ alias, field, separator = ",", distinct, }) {
let gc = `GROUP_CONCAT(`;
if (distinct) {
gc += `DISTINCT `;
}
gc += sqlField(field);
gc += field;
if (!distinct) {
gc += `, ${sqlStringLiteral(String(separator))}`;
}
gc += `)`;
gc += ` AS ${sqlIdent(alias, "alias")}`;
gc += ` AS ${sqlName(alias, "alias")}`;
return gc;
}
dist/utils/sql-generator-grab-select-field-sql.js
Vendored
+5 -5
View File
@@ -1,20 +1,20 @@
import sqlGenGrabConcatStr from "./sql-generator-grab-concat-str";
import { sqlField, sqlIdent } from "./sql-safe";
import { sqlColumn, sqlName } from "./sql-safe";
export default function sqlGenGrabSelectFieldSQL({ selectFields, append_table_names, table_name }) {
let str = "";
str += ` ${selectFields
?.map((fld) => {
if (typeof fld != "object") {
return sqlField(fld);
return sqlColumn(fld);
}
const fld_name = sqlField(fld.fieldName);
const fld_name = sqlColumn(fld.fieldName);
const final_fld_name = append_table_names && !fld_name.includes(".")
? `${table_name}.${fld_name}`
: fld_name;
if (fld.group_concat) {
return sqlGenGrabConcatStr({
field: final_fld_name,
alias: fld.group_concat.alias || fld.alias || fld_name,
alias: fld.group_concat.alias || fld.alias || String(fld.fieldName),
separator: fld.group_concat.separator,
distinct: fld.group_concat.distinct,
});
@@ -46,7 +46,7 @@ export default function sqlGenGrabSelectFieldSQL({ selectFields, append_table_na
fld_str += final_fld_name;
}
if (alias) {
fld_str += ` AS ${sqlIdent(alias, "alias")}`;
fld_str += ` AS ${sqlName(alias, "alias")}`;
}
return fld_str;
})
dist/utils/sql-generator.js
Vendored
+7 -7
View File
@@ -1,13 +1,13 @@
import sqlGenGenSearchStr from "./sql-generator-gen-search-str";
import sqlGenGenQueryStr from "./sql-generator-gen-query-str";
import { sqlField, sqlIdent, sqlInt, sqlOperator, sqlOrderStrategy, } from "./sql-safe";
import { sqlColumn, sqlInt, sqlName, sqlOperator, sqlOrderStrategy, } from "./sql-safe";
/**
* # SQL Query Generator
* @description Generates a parameterized SQLite SELECT query. All values are
* returned in `values` to be bound; identifiers are validated.
*/
export default function sqlGenerator({ tableName: passedTableName, genObject, count, targetId, }) {
const tableName = sqlIdent(passedTableName, "table name");
const tableName = sqlName(passedTableName, "table name");
const finalQuery = genObject?.query ? genObject.query : undefined;
const queryKeys = finalQuery ? Object.keys(finalQuery) : undefined;
const sqlSearhValues = [];
@@ -83,10 +83,10 @@ export default function sqlGenerator({ tableName: passedTableName, genObject, co
const grabGroupStr = (group) => {
if (typeof group == "object") {
return group.table
? `${sqlIdent(group.table, "table name")}.${sqlIdent(group.field, "field name")}`
: sqlField(group.field);
? `${sqlName(group.table, "table name")}.${sqlName(group.field, "field name")}`
: sqlColumn(group.field);
}
return sqlField(group);
return sqlColumn(group);
};
const group_by_txt = Array.isArray(genObject.group)
? genObject.group.map(grabGroupStr).join(",")
@@ -95,8 +95,8 @@ export default function sqlGenerator({ tableName: passedTableName, genObject, co
}
function grabOrderString(order) {
const field = genObject?.join && !String(order.field).includes(".")
? `${tableName}.${sqlIdent(order.field, "order field")}`
: sqlField(order.field, "order field");
? `${tableName}.${sqlName(order.field, "order field")}`
: sqlColumn(order.field, "order field");
return ` ${field} ${sqlOrderStrategy(order.strategy)}`;
}
if (genObject?.order) {
dist/utils/sql-insert-generator.js
Vendored
+3 -3
View File
@@ -1,10 +1,10 @@
import { sqlIdent } from "./sql-safe";
import { sqlIdent, sqlName } from "./sql-safe";
/**
* # SQL Insert Generator
*/
export default function sqlInsertGenerator({ tableName, data, dbFullName, }) {
const finalDbName = dbFullName
? `${sqlIdent(dbFullName, "database name")}.`
? `${sqlName(dbFullName, "database name")}.`
: "";
if (Array.isArray(data) && data?.[0]) {
let insertKeys = [];
@@ -32,7 +32,7 @@ export default function sqlInsertGenerator({ tableName, data, dbFullName, }) {
})
.join(",")})`);
});
let query = `INSERT INTO ${finalDbName}${sqlIdent(tableName, "table name")} (${insertKeys.join(",")}) VALUES ${queryBatches.join(",")}`;
let query = `INSERT INTO ${finalDbName}${sqlName(tableName, "table name")} (${insertKeys.map((ky) => sqlName(ky)).join(",")}) VALUES ${queryBatches.join(",")}`;
return {
query: query,
values: queryValues,
dist/utils/sql-safe.d.ts
Vendored
+13
View File
@@ -11,6 +11,19 @@ export declare function sqlIdent(name: unknown, label?: string): string;
* @description Accepts `field`, `table.field` or `table.*`
*/
export declare function sqlField(name: unknown, label?: string): string;
/**
* # Validate and quote an identifier for queries
* @description Quoting lets reserved words (`to`, `order`, ...) be used as
* names. Backticks, not double quotes: SQLite reads an unknown `"name"` as
* a string literal, so a typo would silently match nothing instead of
* failing with "no such column".
*/
export declare function sqlName(name: unknown, label?: string): string;
/**
* # Validate and quote a column reference
* @description Accepts `field`, `table.field` or `table.*`
*/
export declare function sqlColumn(name: unknown, label?: string): string;
/**
* # Quote an identifier for DDL statements
*/
dist/utils/sql-safe.js
Vendored
+20
View File
@@ -25,6 +25,26 @@ export function sqlField(name, label = "field name") {
}
return str;
}
/**
* # Validate and quote an identifier for queries
* @description Quoting lets reserved words (`to`, `order`, ...) be used as
* names. Backticks, not double quotes: SQLite reads an unknown `"name"` as
* a string literal, so a typo would silently match nothing instead of
* failing with "no such column".
*/
export function sqlName(name, label = "identifier") {
return `\`${sqlIdent(name, label)}\``;
}
/**
* # Validate and quote a column reference
* @description Accepts `field`, `table.field` or `table.*`
*/
export function sqlColumn(name, label = "field name") {
return sqlField(name, label)
.split(".")
.map((part) => (part === "*" ? part : `\`${part}\``))
.join(".");
}
/**
* # Quote an identifier for DDL statements
*/