Add queue tables

A table with `queue_table` set works as a job queue that many workers can
pull from at once. Claims use SELECT ... FOR UPDATE SKIP LOCKED over the
idx_queue_claim index order, so each job goes to one worker and only the
claimed rows are locked.

- Schema: adds the queue_* columns and the idx_queue_claim /
  idx_queue_lease indexes. Declaring these names yourself is an error.
- `queue_table` accepts `true` or { max_attempts, lease_ms, keep_completed }
- BunMariaDB.queue: push, claim, ack, fail, extend, cancel
- Statuses: pending, processing, done, failed, cancelled
- Leases use the database clock. Expired leases are reclaimed (or
  dead-lettered on the last attempt) and old lock tokens stop working.
- insert/update can't set queue state fields. update/delete skip claimed
  jobs.

Tests: allow the test container's self-signed certificate, and give the
queue test its own client so it doesn't reuse one closed by db.test.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
2026-10-18 15:43:22 +01:00
co-authored by Claude Opus 5.5
parent 688f4bcb58
commit 7c470e0225
16 changed files with 1091 additions and 4 deletions
+71
View File
@@ -0,0 +1,71 @@
import dbHandler from "../db-handler";
import type { BunMariaDBConfig, DBResponseObject } from "../../types";
import { quoteIdentifier } from "../../utils/sql-identifier";
import { QUEUE_NOW_SQL, requireQueueTable } from "./queue-utils";
type Params<Table extends string = string> = {
table: Table;
id: number | string;
/** `queue_lock_token` from the claimed row */
lock_token: string;
error?: string;
/** Wait this long before the job can be retried. Defaults to 0 */
retry_delay_ms?: number;
/** Mark the job `failed` instead of retrying once it reaches this many attempts. Defaults to the table's `max_attempts` */
max_attempts?: number;
config?: BunMariaDBConfig;
};
/**
* Release a claimed job after a failed attempt. It goes back to `pending`
* (after `retry_delay_ms`) or to `failed` once `max_attempts` is reached.
*/
export default async function QueueFail<Table extends string = string>({
table,
id,
lock_token,
error,
retry_delay_ms = 0,
max_attempts,
config,
}: Params<Table>): Promise<DBResponseObject> {
try {
const options = requireQueueTable(table);
const tableSql = quoteIdentifier(table);
const retryDelayMs = Math.max(0, Math.floor(Number(retry_delay_ms) || 0));
const maxAttempts = Math.max(
1,
Math.floor(Number(max_attempts ?? options.max_attempts) || 1),
);
const res = await dbHandler({
query: `UPDATE ${tableSql} SET queue_status = IF(queue_attempts >= ?, 'failed', 'pending'), queue_available_at = ${QUEUE_NOW_SQL} + ?, queue_lease_expires_at = NULL, queue_lock_token = NULL, queue_last_error = ?, updated_at = ${QUEUE_NOW_SQL} WHERE id = ? AND queue_status = 'processing' AND queue_lock_token = ?`,
values: [maxAttempts, retryDelayMs, error ?? null, id, lock_token],
config,
});
if (!res.success) {
return { ...res, success: false };
}
if (!res.db_res?.affectedRows) {
return {
...res,
success: false,
msg: "Job is not claimed with this lock token",
};
}
return await dbHandler({
query: `SELECT * FROM ${quoteIdentifier(table)} WHERE id = ?`,
values: [id],
config,
});
} catch (error: any) {
return {
success: false,
error: error.message,
};
}
}