Keep bot jobs and harness calls from hanging

src/utils/task-timeout-error.ts

@@ -0,0 +1,10 @@
+/**
+ * A task was stopped because it ran past its deadline (`withTimeout`).
+ * Its message is safe to show: it says what ran too long, not internals.
+ */
+export default class TaskTimeoutError extends Error {
+    constructor({ message }: { message: string }) {
+        super(message);
+        this.name = "TaskTimeoutError";
+    }
+}

src/utils/with-timeout.test.ts

@@ -0,0 +1,100 @@
+import { describe, expect, test } from "bun:test";
+import TaskTimeoutError from "./task-timeout-error";
+import withTimeout from "./with-timeout";
+
+const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+describe("withTimeout", () => {
+    test("returns the task's result", async () => {
+        const result = await withTimeout({
+            timeout_ms: 100,
+            task: async () => "done",
+        });
+
+        expect(result).toBe("done");
+    });
+
+    test("stops a task that runs past its deadline", async () => {
+        const error = await withTimeout({
+            timeout_ms: 10,
+            message: "Too slow.",
+            task: () => wait(200),
+        }).catch((e) => e);
+
+        expect(error).toBeInstanceOf(TaskTimeoutError);
+        expect(error instanceof TaskTimeoutError && error.message).toBe("Too slow.");
+    });
+
+    test("ends the wait even when the task ignores its signal", async () => {
+        const error = await withTimeout({
+            timeout_ms: 10,
+            task: () => wait(200),
+        }).catch((e) => e);
+
+        expect(error).toBeInstanceOf(TaskTimeoutError);
+    });
+
+    test("aborts the task's signal on the deadline", async () => {
+        let aborted = false;
+
+        await withTimeout({
+            timeout_ms: 10,
+            task: async (task_signal) => {
+                task_signal.addEventListener("abort", () => {
+                    aborted = true;
+                });
+
+                await wait(50);
+            },
+        }).catch(() => {});
+
+        expect(aborted).toBe(true);
+    });
+
+    test("passes the caller's cancellation through", async () => {
+        const controller = new AbortController();
+        let aborted = false;
+        let started: () => void = () => {};
+        const running_task = new Promise<void>((resolve) => {
+            started = resolve;
+        });
+
+        const running = withTimeout({
+            timeout_ms: 1000,
+            signal: controller.signal,
+            task: async (task_signal) => {
+                task_signal.addEventListener("abort", () => {
+                    aborted = true;
+                });
+
+                started();
+                await wait(200);
+            },
+        });
+
+        await running_task;
+        controller.abort();
+        const error = await running.catch((e) => e);
+
+        expect(error).toBeInstanceOf(Error);
+        expect(aborted).toBe(true);
+    });
+
+    test("doesn't start a task when already cancelled", async () => {
+        const controller = new AbortController();
+        let started = false;
+
+        controller.abort();
+
+        const error = await withTimeout({
+            timeout_ms: 100,
+            signal: controller.signal,
+            task: async () => {
+                started = true;
+            },
+        }).catch((e) => e);
+
+        expect(error).toBeInstanceOf(Error);
+        expect(started).toBe(false);
+    });
+});

src/utils/with-timeout.ts

@@ -0,0 +1,85 @@
+import TaskTimeoutError from "./task-timeout-error";
+
+type Params<T> = {
+    /**
+     * The work. It gets a signal that aborts when the deadline passes or
+     * the caller cancels, so its requests stop too.
+     */
+    task: (signal: AbortSignal) => Promise<T>;
+    timeout_ms: number;
+    /**
+     * The caller's own cancellation, e.g. a job that lost its lock.
+     */
+    signal?: AbortSignal;
+    /**
+     * Shown when the deadline passes.
+     */
+    message?: string;
+    /**
+     * Called once the deadline passes, before the task is stopped. Use it
+     * to tell a deadline apart from the caller's own cancellation.
+     */
+    on_timeout?: () => void;
+};
+
+/**
+ * Runs a task with a deadline, resolving or rejecting with whatever the
+ * task does. When the deadline passes (or the caller's signal aborts) the
+ * task's signal is aborted and the wait ends at once, even if the task
+ * never settles, so a hung call can't hold a queue slot or a lock forever.
+ * Late results and errors from the task are dropped.
+ */
+export default function withTimeout<T>({
+    task,
+    timeout_ms,
+    signal,
+    message = "The task took too long.",
+    on_timeout,
+}: Params<T>): Promise<T> {
+    if (signal?.aborted) {
+        return Promise.reject(reasonOf({ signal }));
+    }
+
+    const controller = new AbortController();
+    const parent = signal;
+    const on_abort = () => controller.abort(reasonOf({ signal: parent! }));
+
+    parent?.addEventListener("abort", on_abort, { once: true });
+
+    const timer = setTimeout(() => {
+        on_timeout?.();
+        controller.abort(new TaskTimeoutError({ message }));
+    }, timeout_ms);
+
+    const done = new Promise<T>((resolve, reject) => {
+        controller.signal.addEventListener(
+            "abort",
+            () => reject(controller.signal.reason),
+            { once: true },
+        );
+
+        /**
+         * `Promise.resolve().then` turns a synchronous throw from the task
+         * into a rejection instead of escaping the executor. A task
+         * cancelled before it starts is never started.
+         */
+        Promise.resolve()
+            .then(() => {
+                if (controller.signal.aborted) {
+                    throw reasonOf({ signal: controller.signal });
+                }
+
+                return task(controller.signal);
+            })
+            .then(resolve, reject);
+    });
+
+    return done.finally(() => {
+        clearTimeout(timer);
+        parent?.removeEventListener("abort", on_abort);
+    });
+}
+
+function reasonOf({ signal }: { signal: AbortSignal }) {
+    return signal.reason instanceof Error ? signal.reason : new Error("Cancelled.");
+}