Add queue tables, quote SQL identifiers (v1.3.0)

- isQueue table option adds queue columns and a (status, available_at)
  index; BunSQLite.queue exposes add, claim, complete, fail and
  release_stale with atomic claims
- Set busy_timeout on connections so workers in separate processes wait
  instead of failing with SQLITE_BUSY
- Quote table and column names with backticks in generated SQL, so
  reserved words (to, order, group) work and unknown columns error
  instead of matching as string literals
- Tests for queue ops, multi-process claims and reserved-word names

Co-Authored-By: Claude Opus 5.5 <[email protected]>

dist/lib/grab-duplicate-safe-insert-sql.js

@@ -1,5 +1,5 @@
 import init from "../functions/init";
-import { sqlIdent, sqlQuoteIdent } from "../utils/sql-safe";
+import { sqlName } from "../utils/sql-safe";
 export default async function ({ sql: passed_sql, table, data }) {
     let sql = passed_sql;
     const { dbSchema } = init();
@@ -9,7 +9,7 @@ export default async function ({ sql: passed_sql, table, data }) {
         const set_sql_arr = Object.keys(Array.isArray(data) ? data[0] : data)
             .filter((field) => field !== "updated_at")
             .map((field) => {
-            const column = sqlIdent(field, "column name");
+            const column = sqlName(field, "column name");
             return `${column} = excluded.${column}`;
         });
         set_sql_arr.push(`updated_at = ${now}`);
@@ -19,13 +19,13 @@ export default async function ({ sql: passed_sql, table, data }) {
             const field = unique_fields[i];
             if (!field?.fieldName)
                 continue;
-            sql += ` ON CONFLICT(${sqlQuoteIdent(field.fieldName)}) DO UPDATE SET ${set_sql}`;
+            sql += ` ON CONFLICT(${sqlName(field.fieldName, "column name")}) DO UPDATE SET ${set_sql}`;
         }
         if (table_schema.uniqueConstraints?.[0]) {
             for (let i = 0; i < table_schema.uniqueConstraints.length; i++) {
                 const constraint = table_schema.uniqueConstraints[i];
                 const columns = constraint?.constraintTableFields
-                    ?.map((c) => sqlQuoteIdent(String(c.value)))
+                    ?.map((c) => sqlName(c.value, "column name"))
                     ?.join(", ");
                 if (!columns)
                     continue;

dist/lib/schema/create-db-schema.js

@@ -19,6 +19,11 @@ export default function createDBSchema(params) {
      * Reorder Tables (parents before children with FKs)
      */
     const db_schema = orderDBSchema(params);
+    for (const table of db_schema.tables) {
+        if (table.isQueue && table.isVector) {
+            throw new Error(`Table "${table.tableName}" can't be both a queue and a vector table`);
+        }
+    }
     db.run("PRAGMA foreign_keys = OFF;");
     db.run("BEGIN");
     try {

dist/lib/sqlite/db-delete.js

@@ -1,7 +1,7 @@
 import dbHandler from "../db-handler";
 import _ from "lodash";
 import sqlGenerator from "../../utils/sql-generator";
-import { sqlIdent } from "../../utils/sql-safe";
+import { sqlName } from "../../utils/sql-safe";
 export default async function DbDelete({ table, query, targetId, config, }) {
     let sqlObj = null;
     try {
@@ -13,7 +13,7 @@ export default async function DbDelete({ table, query, targetId, config, }) {
         });
         const whereClause = sqlObj.where.string;
         if (whereClause) {
-            let sql = `DELETE FROM ${sqlIdent(table, "table name")} ${whereClause}`;
+            let sql = `DELETE FROM ${sqlName(table, "table name")} ${whereClause}`;
             sqlObj.string = sql;
             sqlObj.values = sqlObj.where.values;
             const res = dbHandler({

dist/lib/sqlite/db-queue.d.ts

@@ -0,0 +1,64 @@
+import type { APIResponseObject, BunSQLiteConfig } from "../../types";
+type BaseParams<Table extends string = string> = {
+    table: Table;
+    config?: BunSQLiteConfig;
+};
+/**
+ * # Add jobs to a queue table
+ * @description `delay` (ms) holds the jobs back before they can be claimed
+ */
+export declare function QueueAdd<Schema extends {
+    [k: string]: any;
+} = {
+    [k: string]: any;
+}, Table extends string = string>({ table, data, delay, config, }: BaseParams<Table> & {
+    data: Schema[];
+    delay?: number;
+}): Promise<APIResponseObject<Schema>>;
+/**
+ * # Claim jobs from a queue table
+ * @description Claimed jobs are set to `processing` in one statement, so
+ * two workers can never claim the same job. Stale locks are released first.
+ */
+export declare function QueueClaim<Schema extends {
+    [k: string]: any;
+} = {
+    [k: string]: any;
+}, Table extends string = string>({ table, worker, limit, config, }: BaseParams<Table> & {
+    worker?: string;
+    limit?: number;
+}): Promise<APIResponseObject<Schema>>;
+/**
+ * # Mark a claimed job as done
+ * @description Deletes the job instead when `deleteOnComplete` is set
+ */
+export declare function QueueComplete<Schema extends {
+    [k: string]: any;
+} = {
+    [k: string]: any;
+}, Table extends string = string>({ table, id, config, }: BaseParams<Table> & {
+    id: number | string;
+}): Promise<APIResponseObject<Schema>>;
+/**
+ * # Mark a claimed job as failed
+ * @description The job goes back to `pending` with an exponential
+ * `backoff` delay, or to `failed` once it has used `maxAttempts`
+ */
+export declare function QueueFail<Schema extends {
+    [k: string]: any;
+} = {
+    [k: string]: any;
+}, Table extends string = string>({ table, id, error, config, }: BaseParams<Table> & {
+    id: number | string;
+    error?: string;
+}): Promise<APIResponseObject<Schema>>;
+/**
+ * # Release stale locks
+ * @description For jobs whose worker died. `claim` also does this.
+ */
+export declare function QueueReleaseStale<Schema extends {
+    [k: string]: any;
+} = {
+    [k: string]: any;
+}, Table extends string = string>({ table, config }: BaseParams<Table>): Promise<APIResponseObject<Schema>>;
+export {};

dist/lib/sqlite/db-queue.js

@@ -0,0 +1,186 @@
+import dbHandler from "../db-handler";
+import init from "../../functions/init";
+import DbInsert from "./db-insert";
+import { sqlInt, sqlName } from "../../utils/sql-safe";
+const DefaultQueueOptions = {
+    maxAttempts: 3,
+    lockTimeout: 300_000,
+    backoff: 1000,
+    deleteOnComplete: false,
+};
+/**
+ * Validate the table name and resolve its queue options from the schema
+ */
+function resolveQueue(table) {
+    const { dbSchema } = init();
+    const tableSchema = dbSchema.tables.find((t) => t.tableName === table);
+    if (!tableSchema?.isQueue) {
+        throw new Error(`Table "${table}" is not a queue table (isQueue)`);
+    }
+    const options = {
+        ...DefaultQueueOptions,
+        ...(typeof tableSchema.isQueue === "object" ? tableSchema.isQueue : {}),
+    };
+    return { tableName: sqlName(table, "table name"), options };
+}
+/**
+ * Move jobs that have been `processing` past `lockTimeout` back to
+ * `pending`, or to `failed` once they are out of attempts
+ */
+function releaseStaleSql(tableName) {
+    return `UPDATE ${tableName} SET status = CASE WHEN attempts >= ? THEN 'failed' ELSE 'pending' END, last_error = CASE WHEN attempts >= ? THEN 'Lock timed out' ELSE last_error END, locked_at = NULL, locked_by = NULL, updated_at = ? WHERE status = 'processing' AND locked_at < ? RETURNING *`;
+}
+function rowsResponse(res, sql) {
+    if (!res.success) {
+        return { ...res, debug: { sql } };
+    }
+    const rows = res.payload || [];
+    return {
+        success: true,
+        payload: rows,
+        singleRes: rows[0] ?? null,
+        debug: { sql },
+    };
+}
+/**
+ * # Add jobs to a queue table
+ * @description `delay` (ms) holds the jobs back before they can be claimed
+ */
+export async function QueueAdd({ table, data, delay = 0, config, }) {
+    try {
+        resolveQueue(table);
+        const available_at = Date.now() + sqlInt(delay, "delay");
+        return await DbInsert({
+            table,
+            data: data.map((d) => ({
+                ...d,
+                status: "pending",
+                attempts: 0,
+                available_at,
+            })),
+            config,
+        });
+    }
+    catch (error) {
+        return { success: false, error: error.message };
+    }
+}
+/**
+ * # Claim jobs from a queue table
+ * @description Claimed jobs are set to `processing` in one statement, so
+ * two workers can never claim the same job. Stale locks are released first.
+ */
+export async function QueueClaim({ table, worker, limit = 1, config, }) {
+    let sql = "";
+    try {
+        const { tableName, options } = resolveQueue(table);
+        const now = Date.now();
+        const released = dbHandler({
+            query: releaseStaleSql(tableName),
+            values: [
+                options.maxAttempts,
+                options.maxAttempts,
+                now,
+                now - options.lockTimeout,
+            ],
+            config,
+        });
+        if (!released.success) {
+            return released;
+        }
+        sql = `UPDATE ${tableName} SET status = 'processing', attempts = attempts + 1, locked_at = ?, locked_by = ?, updated_at = ? WHERE id IN (SELECT id FROM ${tableName} WHERE status = 'pending' AND available_at <= ? ORDER BY id LIMIT ?) RETURNING *`;
+        const res = dbHandler({
+            query: sql,
+            values: [now, worker ?? null, now, now, sqlInt(limit, "limit")],
+            config,
+        });
+        return rowsResponse(res, sql);
+    }
+    catch (error) {
+        return { success: false, error: error.message, debug: { sql } };
+    }
+}
+/**
+ * # Mark a claimed job as done
+ * @description Deletes the job instead when `deleteOnComplete` is set
+ */
+export async function QueueComplete({ table, id, config, }) {
+    let sql = "";
+    try {
+        const { tableName, options } = resolveQueue(table);
+        sql = options.deleteOnComplete
+            ? `DELETE FROM ${tableName} WHERE id = ? AND status = 'processing' RETURNING *`
+            : `UPDATE ${tableName} SET status = 'completed', locked_at = NULL, locked_by = NULL, updated_at = ? WHERE id = ? AND status = 'processing' RETURNING *`;
+        const res = dbHandler({
+            query: sql,
+            values: options.deleteOnComplete ? [id] : [Date.now(), id],
+            config,
+        });
+        const out = rowsResponse(res, sql);
+        return out.success && !out.payload?.length
+            ? { ...out, success: false, msg: "No processing job with this id" }
+            : out;
+    }
+    catch (error) {
+        return { success: false, error: error.message, debug: { sql } };
+    }
+}
+/**
+ * # Mark a claimed job as failed
+ * @description The job goes back to `pending` with an exponential
+ * `backoff` delay, or to `failed` once it has used `maxAttempts`
+ */
+export async function QueueFail({ table, id, error, config, }) {
+    let sql = "";
+    try {
+        const { tableName, options } = resolveQueue(table);
+        const now = Date.now();
+        sql = `UPDATE ${tableName} SET status = CASE WHEN attempts >= ? THEN 'failed' ELSE 'pending' END, available_at = CASE WHEN attempts >= ? THEN available_at ELSE ? + ? * (1 << MIN(MAX(attempts - 1, 0), 30)) END, last_error = ?, locked_at = NULL, locked_by = NULL, updated_at = ? WHERE id = ? AND status = 'processing' RETURNING *`;
+        const res = dbHandler({
+            query: sql,
+            values: [
+                options.maxAttempts,
+                options.maxAttempts,
+                now,
+                options.backoff,
+                error ?? null,
+                now,
+                id,
+            ],
+            config,
+        });
+        const out = rowsResponse(res, sql);
+        return out.success && !out.payload?.length
+            ? { ...out, success: false, msg: "No processing job with this id" }
+            : out;
+    }
+    catch (error) {
+        return { success: false, error: error.message, debug: { sql } };
+    }
+}
+/**
+ * # Release stale locks
+ * @description For jobs whose worker died. `claim` also does this.
+ */
+export async function QueueReleaseStale({ table, config }) {
+    let sql = "";
+    try {
+        const { tableName, options } = resolveQueue(table);
+        const now = Date.now();
+        sql = releaseStaleSql(tableName);
+        const res = dbHandler({
+            query: sql,
+            values: [
+                options.maxAttempts,
+                options.maxAttempts,
+                now,
+                now - options.lockTimeout,
+            ],
+            config,
+        });
+        return rowsResponse(res, sql);
+    }
+    catch (error) {
+        return { success: false, error: error.message, debug: { sql } };
+    }
+}

dist/lib/sqlite/db-update.js

@@ -3,7 +3,7 @@ import _ from "lodash";
 import init from "../../functions/init";
 import sqlGenerator from "../../utils/sql-generator";
 import { sqlValue } from "../../utils/sql-insert-generator";
-import { sqlIdent } from "../../utils/sql-safe";
+import { sqlName } from "../../utils/sql-safe";
 export default async function DbUpdate({ table, data, query, targetId, returning = true, config, }) {
     let sqlObj = {
         string: "",
@@ -23,7 +23,7 @@ export default async function DbUpdate({ table, data, query, targetId, returning
         let values = [];
         const whereClause = sqlQueryObj.where.string;
         if (whereClause) {
-            let sql = `UPDATE ${sqlIdent(table, "table name")} SET`;
+            let sql = `UPDATE ${sqlName(table, "table name")} SET`;
             const finalData = {
                 updated_at: Date.now(),
                 ...data,
@@ -34,7 +34,7 @@ export default async function DbUpdate({ table, data, query, targetId, returning
                 if (!key)
                     continue;
                 const isLast = i == keys.length - 1;
-                sql += ` ${sqlIdent(key, "column name")}=?`;
+                sql += ` ${sqlName(key, "column name")}=?`;
                 values.push(sqlValue(finalData[key]));
                 if (!isLast) {
                     sql += `,`;