Let a super admin read the database from the app

A new super admin settings page, /admin/settings/database, lists every
table with its record count and size, and one page of the open table's
records newest key first. Values are cut inside SQL (400 characters, with
a flag saying so) and a record opens whole in a modal, so a table of long
texts reads without shipping it all. A search matches every column's
text, word by word or as a phrase, which is the one thing that also works
on a virtual table. Nothing in it writes: the catalog, the records and
the single record are all reads, GET only.
This commit is contained in:
2026-10-23 06:03:42 +01:00
parent 9b26c169d6
commit 41bb4aaac2
48 changed files with 2573 additions and 0 deletions
@@ -0,0 +1,59 @@
import BunSQLite from "@moduletrace/bun-sqlite";
import quoteSqlIdentifier from "./quote-sql-identifier";
import resolveTableKey from "./resolve-table-key";
import type { DatabaseTableSummary } from "@/src/types";
import type { TableCatalogEntry } from "./table-catalog-types";
type Params = {
tables: TableCatalogEntry[];
/**
* How many bytes each table takes on disk, by name; empty when SQLite
* can't measure it, which leaves the size column blank.
*/
sizes: Map<string, number>;
};
type CountRow = { rows: number };
/**
* The catalog as the browser lists it: every table with its record count,
* its size on disk, its columns and what identifies its records.
*
* Each count is its own tiny statement rather than one aggregate, because
* a virtual table has no rowid to build one from, and the whole read stays
* in the low milliseconds for the install's fifty-odd tables.
*/
export default async function buildDatabaseTables({ tables, sizes }: Params): Promise<DatabaseTableSummary[]> {
const summaries: DatabaseTableSummary[] = [];
for (const table of tables) {
const size = sizes.get(table.name);
summaries.push({
name: table.name,
type: table.type,
rows: await countRows({ table: table.name }),
size_bytes: size === undefined ? null : size,
columns: table.columns,
key: resolveTableKey({ columns: table.columns, has_rowid: table.has_rowid }),
});
}
return summaries;
}
async function countRows({ table }: { table: string }): Promise<number | null> {
const res = await BunSQLite.sql<CountRow>({
sql: `SELECT COUNT(*) AS rows FROM ${quoteSqlIdentifier({ name: table })}`,
});
if (!res.success || !res.payload?.[0]) {
/**
* A table the module won't count (an index over a table whose
* module is missing) is still a table; it just has no count.
*/
return null;
}
return Number(res.payload[0].rows) || 0;
}
@@ -0,0 +1,43 @@
import { metaColumnAlias, RecordKeyAlias } from "./build-records-sql";
import toDatabaseCell from "./to-database-cell";
import type { DatabaseRecord, DatabaseTableColumn } from "@/src/types";
type Params = {
/**
* The row as SQLite returned it, its meta columns included.
*/
row: Record<string, unknown>;
columns: DatabaseTableColumn[];
max_chars: number;
};
/**
* One row of a records query as a record the browser can draw: its key,
* and every column's value as a cell. The meta columns the query added
* (`typeof` and `length` per column) are read and dropped here, so they
* never reach the client.
*/
export default function buildRecordRow({ row, columns, max_chars }: Params): DatabaseRecord {
const cells: Record<string, DatabaseRecord["cells"][string]> = {};
for (const column of columns) {
cells[column.name] = toDatabaseCell({
value: row[column.name],
type: String(row[metaColumnAlias({ name: column.name, kind: "type" })] ?? "null"),
length: Number(row[metaColumnAlias({ name: column.name, kind: "length" })] ?? 0),
max_chars,
});
}
const key = row[RecordKeyAlias];
return {
key:
typeof key == "string" || typeof key == "number"
? key
: key === null || key === undefined
? null
: String(key),
cells,
};
}
@@ -0,0 +1,337 @@
import { Database } from "bun:sqlite";
import { beforeAll, describe, expect, test } from "bun:test";
import buildRecordsSql, {
buildCountSql,
buildRecordSql,
buildRecordSelect,
metaColumnAlias,
parseRecordKey,
RecordKeyAlias,
} from "./build-records-sql";
import buildSearchClause, { searchWords } from "./build-search-clause";
import buildRecordRow from "./build-record-row";
import buildTableCatalogSql from "./build-table-catalog-sql";
import groupTableCatalog from "./group-table-catalog";
import resolveTableKey from "./resolve-table-key";
import quoteSqlIdentifier from "./quote-sql-identifier";
import type { TableCatalogRow } from "./table-catalog-types";
import type { DatabaseTableColumn } from "@/src/types";
const db = new Database(":memory:");
const messages_columns: DatabaseTableColumn[] = [
{ name: "id", type: "INTEGER", pk: true },
{ name: "role", type: "TEXT", pk: false },
{ name: "text", type: "TEXT", pk: false },
];
beforeAll(() => {
db.run(`CREATE TABLE "messages" ("id" INTEGER PRIMARY KEY, "role" TEXT, "text" TEXT)`);
db.run(`INSERT INTO "messages" VALUES
(1, 'user', 'hello there'),
(2, 'bot', 'Hello there, how can I help?'),
(3, 'user', 'a "quoted" table name and a long body'),
(4, 'bot', NULL)`);
db.run(`CREATE TABLE "no_key" ("a" TEXT, "b" TEXT)`);
db.run(`INSERT INTO "no_key" VALUES ('one', 'two')`);
});
function rows({ sql, values, max_chars = 400 }: { sql: string; values: string[]; max_chars?: number }) {
return db
.query(sql)
.all(...values)
.map((row) =>
buildRecordRow({ row: row as Record<string, unknown>, columns: messages_columns, max_chars }),
);
}
describe("quoteSqlIdentifier", () => {
test("wraps a plain name", () => {
expect(quoteSqlIdentifier({ name: "bots" })).toBe(`"bots"`);
});
test("doubles a quote instead of letting it out", () => {
expect(quoteSqlIdentifier({ name: `a"b` })).toBe(`"a""b"`);
});
});
describe("resolveTableKey", () => {
test("a single primary key column names the records", () => {
expect(resolveTableKey({ columns: messages_columns, has_rowid: true })).toEqual({ kind: "column", name: "id" });
});
test("no primary key falls back to the rowid", () => {
expect(resolveTableKey({ columns: [{ name: "a", type: "TEXT", pk: false }], has_rowid: true })).toEqual({
kind: "rowid",
});
});
test("a virtual table with neither has no key", () => {
expect(resolveTableKey({ columns: [{ name: "a", type: "TEXT", pk: false }], has_rowid: false })).toEqual({
kind: "none",
});
});
test("a composite primary key falls back to the rowid", () => {
expect(
resolveTableKey({
columns: [
{ name: "a", type: "TEXT", pk: true },
{ name: "b", type: "TEXT", pk: true },
],
has_rowid: true,
}),
).toEqual({ kind: "rowid" });
});
});
describe("buildRecordsSql", () => {
test("returns a page of records newest key first", () => {
const query = buildRecordsSql({
table: "messages",
columns: messages_columns.map((column) => column.name),
key: { kind: "column", name: "id" },
max_chars: 400,
limit: 2,
offset: 0,
search: null,
});
const records = rows(query);
expect(records.map((record) => record.key)).toEqual([4, 3]);
expect(records[0]?.cells["text"]?.kind).toBe("null");
expect(records[1]?.cells["role"]?.text).toBe("user");
});
test("pages with the offset it is given", () => {
const query = buildRecordsSql({
table: "messages",
columns: messages_columns.map((column) => column.name),
key: { kind: "column", name: "id" },
max_chars: 400,
limit: 2,
offset: 2,
search: null,
});
expect(rows(query).map((record) => record.key)).toEqual([2, 1]);
});
test("cuts a long value and says so", () => {
const query = buildRecordsSql({
table: "messages",
columns: messages_columns.map((column) => column.name),
key: { kind: "column", name: "id" },
max_chars: 4,
limit: 1,
offset: 1,
search: null,
});
const [record] = rows({ ...query, max_chars: 4 });
expect(record?.cells["text"]?.text).toBe(`a "q`); // four characters, cut
expect(record?.cells["text"]?.truncated).toBe(true);
});
test("orders by the rowid when the table has no primary key", () => {
const query = buildRecordsSql({
table: "no_key",
columns: ["a", "b"],
key: { kind: "rowid" },
max_chars: 400,
limit: 10,
offset: 0,
search: null,
});
const record = db.query(query.sql).get(...query.values) as Record<string, unknown>;
expect(Number(record[RecordKeyAlias])).toBe(1);
});
test("a table with no key comes back without one", () => {
const query = buildRecordsSql({
table: "no_key",
columns: ["a", "b"],
key: { kind: "none" },
max_chars: 400,
limit: 10,
offset: 0,
search: null,
});
const record = db.query(query.sql).get(...query.values) as Record<string, unknown>;
expect(query.sql).not.toContain(`ORDER BY`);
expect(record[RecordKeyAlias]).toBeUndefined();
});
test("selects a meta column per column, so a cut value can be known", () => {
const select = buildRecordSelect({
table: "messages",
columns: messages_columns.map((column) => column.name),
key: { kind: "column", name: "id" },
max_chars: 400,
});
expect(select.sql).toContain(quoteSqlIdentifier({ name: metaColumnAlias({ name: "text", kind: "length" }) }));
expect(select.sql).toContain(quoteSqlIdentifier({ name: metaColumnAlias({ name: "text", kind: "type" }) }));
});
});
describe("search", () => {
const columns = messages_columns.map((column) => column.name);
const every_word = { text: "hello there", mode: "every_word" as const, max_words: 8 };
test("every word matches anywhere in the record, in any order", () => {
const query = buildRecordsSql({
table: "messages",
columns,
key: { kind: "column", name: "id" },
max_chars: 400,
limit: 10,
offset: 0,
search: every_word,
});
expect(rows(query).map((record) => record.key).sort()).toEqual([1, 2]);
});
test("a phrase only matches where the words sit together", () => {
const query = buildRecordsSql({
table: "messages",
columns,
key: { kind: "column", name: "id" },
max_chars: 400,
limit: 10,
offset: 0,
search: { text: "hello there", mode: "phrase", max_words: 8 },
});
expect(rows(query).map((record) => record.key).sort()).toEqual([1, 2]);
});
test("a phrase of one word that sits apart doesn't match", () => {
const query = buildRecordsSql({
table: "messages",
columns,
key: { kind: "column", name: "id" },
max_chars: 400,
limit: 10,
offset: 0,
search: { text: "help hello", mode: "phrase", max_words: 8 },
});
expect(rows(query)).toEqual([]);
});
test("prefixes find a value, wherever it is in a word", () => {
const query = buildRecordsSql({
table: "messages",
columns,
key: { kind: "column", name: "id" },
max_chars: 400,
limit: 10,
offset: 0,
search: { text: "quoted", mode: "every_word", max_words: 8 },
});
expect(rows(query).map((record) => record.key)).toEqual([3]);
});
test("matches an id as text", () => {
const query = buildRecordsSql({
table: "messages",
columns,
key: { kind: "column", name: "id" },
max_chars: 400,
limit: 10,
offset: 0,
search: { text: "3", mode: "every_word", max_words: 8 },
});
expect(rows(query).map((record) => record.key)).toEqual([3]);
});
test("counts only what matched", () => {
const count = buildCountSql({ table: "messages", columns, search: every_word });
const result = db.query(count.sql).get(...count.values) as { total: number };
expect(result.total).toBe(2);
});
test("counts the whole table with no search", () => {
const count = buildCountSql({ table: "messages", columns, search: null });
const result = db.query(count.sql).get(...count.values) as { total: number };
expect(result.total).toBe(4);
});
test("an empty search matches everything rather than failing", () => {
const clause = buildSearchClause({ columns, search: " ", mode: "every_word", max_words: 8 });
expect(clause).toEqual({ sql: "1", values: [] });
});
test("keeps a search to its words", () => {
expect(searchWords({ search: " a b c ", max_words: 2 })).toEqual(["a", "b"]);
});
test("a table with no columns matches nothing", () => {
const clause = buildSearchClause({ columns: [], search: "x", mode: "every_word", max_words: 8 });
expect(clause.values).toEqual(["x"]);
expect(clause.sql).toContain(`''`);
});
});
describe("buildRecordSql", () => {
test("reads one record by its key", () => {
const query = buildRecordSql({
table: "messages",
columns: messages_columns.map((column) => column.name),
key: { kind: "column", name: "id" },
max_chars: 400,
value: 2,
});
const [record] = rows({ sql: query.sql, values: query.values as string[] });
expect(record?.key).toBe(2);
expect(record?.cells["text"]?.text).toBe("Hello there, how can I help?");
});
test("finds nothing for a key that isn't there", () => {
const query = buildRecordSql({
table: "messages",
columns: messages_columns.map((column) => column.name),
key: { kind: "column", name: "id" },
max_chars: 400,
value: 99,
});
expect(db.query(query.sql).all(...(query.values as string[]))).toEqual([]);
});
});
describe("parseRecordKey", () => {
test("an integer key comes back as a number", () => {
expect(parseRecordKey({ value: "42" })).toBe(42);
});
test("a text key stays text", () => {
expect(parseRecordKey({ value: "abc" })).toBe("abc");
});
test("a number too big to be safe stays text", () => {
expect(parseRecordKey({ value: "99999999999999999999" })).toBe("99999999999999999999");
});
});
describe("buildTableCatalogSql", () => {
test("reads every table and its columns, without SQLite's own tables", () => {
const res = db.query(buildTableCatalogSql()).all() as TableCatalogRow[];
const tables = groupTableCatalog({ rows: res });
expect(tables.map((table) => table.name)).toEqual(["messages", "no_key"]);
expect(tables[0]?.columns.map((column) => column.name)).toEqual(["id", "role", "text"]);
expect(tables[0]?.has_rowid).toBe(true);
});
});
@@ -0,0 +1,189 @@
import buildSearchClause from "./build-search-clause";
import quoteSqlIdentifier from "./quote-sql-identifier";
import type { DatabaseSearchMode, DatabaseTableKey } from "@/src/types";
/**
* The column every record's key comes back in.
*/
export const RecordKeyAlias = "__cobble_key";
type MetaKind = "type" | "length";
/**
* The column SQLite's own type of a value comes back in, beside the value.
*/
export function metaColumnAlias({ name, kind }: { name: string; kind: MetaKind }): string {
return `__cobble_${kind}__${name}`;
}
type ColumnListParams = {
/**
* The table's columns, in their declared order.
*/
columns: string[];
/**
* The most characters of a value the query takes out of SQLite.
*/
max_chars: number;
};
/**
* What a query selects: every column, cut to `max_chars` so no huge value
* (a record's long text, a job's checkpoint) crosses out of SQLite, and
* beside each one its stored type and length, the two things that say how
* to read it and whether it was cut.
*/
export function buildRecordColumns({ columns, max_chars }: ColumnListParams): string[] {
const selected: string[] = [];
for (const name of columns) {
const column = quoteSqlIdentifier({ name });
selected.push(`substr(${column}, 1, ${max_chars}) AS ${column}`);
selected.push(
`typeof(${column}) AS ${quoteSqlIdentifier({ name: metaColumnAlias({ name, kind: "type" }) })}`,
);
selected.push(
`length(${column}) AS ${quoteSqlIdentifier({ name: metaColumnAlias({ name, kind: "length" }) })}`,
);
}
return selected;
}
/**
* The key column, which is what identifies a record: a primary key, the
* rowid, or nothing at all for a virtual table that has neither.
*/
export function buildKeyColumn({ key }: { key: DatabaseTableKey }): string[] {
if (key.kind == "column" && key.name) {
return [`${quoteSqlIdentifier({ name: key.name })} AS "${RecordKeyAlias}"`];
}
if (key.kind == "rowid") {
/**
* Quoted: `rowid` is read as the ref's rowid that way, and as a
* column of the table's own named `rowid` otherwise.
*/
return [`"rowid" AS "${RecordKeyAlias}"`];
}
return [];
}
type SelectParams = ColumnListParams & {
table: string;
key: DatabaseTableKey;
/**
* What to look for, when the browser is narrowing the table down.
*/
search?: DatabaseSearch | null;
};
/**
* A search a records query narrows its table by.
*/
export type DatabaseSearch = {
text: string;
mode: DatabaseSearchMode;
/**
* How many words at most are matched, so a long line can't build a
* query of a hundred conditions.
*/
max_words: number;
};
type Query = { sql: string; values: string[] };
/**
* The `SELECT` a page of records and one record both start from: the key,
* every column with its meta columns, and, when something is searched, a
* `WHERE` over all of the table's text.
*/
export function buildRecordSelect({ table, columns, key, max_chars, search }: SelectParams): Query {
const selected = [...buildKeyColumn({ key }), ...buildRecordColumns({ columns, max_chars })];
const from = `SELECT ${selected.join(", ")} FROM ${quoteSqlIdentifier({ name: table })}`;
if (!search) return { sql: from, values: [] };
const where = buildSearchClause({
columns,
search: search.text,
mode: search.mode,
max_words: search.max_words,
});
return { sql: `${from} WHERE ${where.sql}`, values: where.values };
}
type PageParams = SelectParams & {
limit: number;
offset: number;
};
/**
* One page of a table's records, newest key first, so a freshly written
* record shows at the top. A table with no key to order by comes back in
* whatever order SQLite reads it.
*/
export default function buildRecordsSql({ limit, offset, ...params }: PageParams): Query {
const select = buildRecordSelect(params);
const order = params.key.kind == "none" ? "" : ` ORDER BY "${RecordKeyAlias}" DESC`;
return {
sql: `${select.sql}${order} LIMIT ${limit} OFFSET ${offset}`,
values: select.values,
};
}
type RecordParams = SelectParams & {
/**
* The record's key, as it came in the query string.
*/
value: string | number;
};
/**
* One record, by its key. Only asked for a table that has one, so a row
* always comes back.
*/
export function buildRecordSql({ value, ...params }: RecordParams): { sql: string; values: (string | number)[] } {
const select = buildRecordSelect(params);
return {
sql: `${select.sql} WHERE "${RecordKeyAlias}" = ? LIMIT 1`,
values: [...select.values, value],
};
}
type CountParams = Omit<SelectParams, "key" | "max_chars">;
/**
* How many records the table holds, or how many the search matched, for
* the pager under it.
*/
export function buildCountSql({ table, columns, search }: CountParams): Query {
const from = `SELECT COUNT(*) AS total FROM ${quoteSqlIdentifier({ name: table })}`;
if (!search) return { sql: from, values: [] };
const where = buildSearchClause({
columns,
search: search.text,
mode: search.mode,
max_words: search.max_words,
});
return { sql: `${from} WHERE ${where.sql}`, values: where.values };
}
/**
* The record SQL's parameter (a primary key or a rowid), read from a
* query string: digits are a number, so an integer key compares as one,
* and anything else stays a string.
*/
export function parseRecordKey({ value }: { value: string }): string | number {
const text = value.trim();
return /^-?\d+$/.test(text) && Number.isSafeInteger(Number(text)) ? Number(text) : text;
}
@@ -0,0 +1,85 @@
import quoteSqlIdentifier from "./quote-sql-identifier";
import type { DatabaseSearchMode } from "@/src/types";
type Params = {
/**
* The table's columns, in their declared order.
*/
columns: string[];
search: string;
mode: DatabaseSearchMode;
/**
* How many words at most are matched, so a long line can't build a
* query of a hundred conditions.
*/
max_words: number;
};
/**
* Splits a search into its words, in the case they were typed (SQLite's
* `lower` folds ASCII case).
*/
export function searchWords({ search, max_words }: { search: string; max_words: number }): string[] {
return search
.trim()
.split(/\s+/)
.filter(Boolean)
.slice(0, max_words);
}
type WordParams = {
/**
* The SQL that reads every column of a row as one text blob.
*/
text: string;
search: string;
mode: DatabaseSearchMode;
max_words: number;
};
/**
* A search as a `WHERE` condition, with the parameters its placeholders
* take, in order.
*/
export function searchCondition({ text, search, mode, max_words }: WordParams): { sql: string; values: string[] } {
if (mode == "phrase") {
return {
sql: `instr(lower(${text}), lower(?)) > 0`,
values: [search.trim()],
};
}
const words = searchWords({ search, max_words });
if (!words.length) {
return { sql: `1`, values: [] };
}
return {
sql: words.map(() => `instr(lower(${text}), lower(?)) > 0`).join(` AND `),
values: words,
};
}
/**
* A search over every column of a table, as a condition the records query
* puts in its `WHERE` (and the count query puts in its own), with the
* parameters its placeholders take, in order.
*
* The columns are joined into one text blob per row, so a value is found
* wherever it lives: an id, a name, a message's words. Reading every row
* is the only honest way to find a value in a table with no index for it;
* the page sizes keep what comes back small, not the cost of the search.
*/
export default function buildSearchClause({ columns, search, mode, max_words }: Params): {
sql: string;
values: string[];
} {
const text = columns.length
? columns
.map((name) => `COALESCE(CAST(${quoteSqlIdentifier({ name })} AS TEXT), '')`)
.join(` || ' ' || `)
: `''`;
return searchCondition({ text, search, mode, max_words });
}
@@ -0,0 +1,33 @@
/**
* The tables the database browser lists, each with its columns in their
* declared order, in one statement: `pragma_table_list` finds the tables
* and `pragma_table_info` reads each one's columns, the pragma taking the
* table name as an argument the way a table-valued function does.
*
* Left out: SQLite's own `sqlite_*` tables, and the shadow tables a
* virtual table keeps its internals in (the `bot_file_vectors_*` tables
* behind the sqlite-vec index), which are storage, not rows anyone
* browses.
*/
export default function buildTableCatalogSql(): string {
return `SELECT
l.name AS table_name,
l.type AS table_type,
l.wr AS without_rowid,
p.cid AS column_index,
p.name AS column_name,
p.type AS column_type,
p.pk AS column_pk
FROM pragma_table_list AS l
JOIN pragma_table_info(l.name) AS p
WHERE l.schema = 'main'
AND l.type IN ('table', 'virtual')
AND l.name NOT LIKE 'sqlite\\_%' ESCAPE '\\'
AND NOT EXISTS (
SELECT 1
FROM pragma_table_list AS v
WHERE v.type = 'virtual'
AND l.name LIKE v.name || '\\_%' ESCAPE '\\'
)
ORDER BY l.name ASC, p.cid ASC`;
}
@@ -0,0 +1,19 @@
/**
* How many bytes each table takes on disk. `dbstat` reports pages per
* b-tree; an index's pages are named after the index (`autoindex` or the
* schema's own), so the join puts them back under the table they belong
* to. A table with no pages yet has no row here.
*/
export default function buildTableSizesSql(): string {
return `SELECT
COALESCE(i.tbl_name, d.name) AS table_name,
SUM(d.pgsize) AS bytes
FROM dbstat AS d
LEFT JOIN sqlite_master AS i ON i.type = 'index' AND i.name = d.name
GROUP BY COALESCE(i.tbl_name, d.name)`;
}
export type TableSizeRow = {
table_name: string;
bytes: number;
};
@@ -0,0 +1,30 @@
import { describe, expect, test } from "bun:test";
import clampPage from "./clamp-page";
describe("clampPage", () => {
test("keeps a page that is inside the table", () => {
expect(clampPage({ page: 2, total: 120, page_size: 50 })).toBe(2);
});
test("a page past the end lands on the last one", () => {
expect(clampPage({ page: 99, total: 120, page_size: 50 })).toBe(3);
});
test("an empty table still has a first page", () => {
expect(clampPage({ page: 4, total: 0, page_size: 50 })).toBe(1);
});
test("a page below one is the first", () => {
expect(clampPage({ page: 0, total: 120, page_size: 50 })).toBe(1);
expect(clampPage({ page: -3, total: 120, page_size: 50 })).toBe(1);
});
test("a table that fits exactly ends on the page asked for", () => {
expect(clampPage({ page: 2, total: 100, page_size: 50 })).toBe(2);
expect(clampPage({ page: 3, total: 100, page_size: 50 })).toBe(2);
});
test("a page size of nothing doesn't divide by zero", () => {
expect(clampPage({ page: 1, total: 10, page_size: 0 })).toBe(1);
});
});
@@ -0,0 +1,23 @@
type Params = {
/**
* The page asked for, from 1.
*/
page: number;
/**
* How many records the table holds (or how many matched a search).
*/
total: number;
page_size: number;
};
/**
* The page that actually gets read: a page past the end of the table lands
* on its last one (so a stale link or a deleted record never shows an empty
* page), and a table with nothing in it still has a first page.
*/
export default function clampPage({ page, total, page_size }: Params): number {
const size = Math.max(1, Math.trunc(page_size) || 1);
const last = Math.max(1, Math.ceil(total / size));
return Math.min(Math.max(1, Math.trunc(page) || 1), last);
}
@@ -0,0 +1,25 @@
import BunSQLite from "@moduletrace/bun-sqlite";
import buildDatabaseTables from "./build-database-tables";
import buildTableCatalogSql from "./build-table-catalog-sql";
import grabTableSizes from "./grab-table-sizes";
import groupTableCatalog from "./group-table-catalog";
import type { TableCatalogRow } from "./table-catalog-types";
import type { DatabaseTableSummary } from "@/src/types";
/**
* Every table the database browser lists: the catalog read in one
* statement (`buildTableCatalogSql`), the sizes on disk, then a count each.
* Empty when the catalog can't be read, which leaves the page saying so
* rather than failing.
*/
export default async function grabDatabaseTables(): Promise<DatabaseTableSummary[]> {
const res = await BunSQLite.sql<TableCatalogRow>({ sql: buildTableCatalogSql() });
if (!res.success || !res.payload) return [];
const tables = groupTableCatalog({ rows: res.payload });
if (!tables.length) return [];
return await buildDatabaseTables({ tables, sizes: await grabTableSizes() });
}
@@ -0,0 +1,36 @@
import AppError from "@/src/utils/app-error";
import grabDatabaseTables from "./grab-database-tables";
import type { DatabaseTableSummary } from "@/src/types";
type Params = {
/**
* The table asked for, as it came in the query string. The first table
* is used when nothing is asked for.
*/
table?: string | null;
};
/**
* One table, or the first one when nothing is asked for. Anything else is
* refused with a message safe to show: the browser only ever reads tables
* that are in the database, never a name it built itself.
*/
export default async function grabTableCatalogEntry({ table }: Params): Promise<DatabaseTableSummary> {
const tables = await grabDatabaseTables();
if (!tables.length) {
throw new AppError(`Couldn't read the database's tables.`);
}
const first = tables[0];
if (!table && first) return first;
const entry = tables.find((candidate) => candidate.name == table);
if (!entry) {
throw new AppError(`There's no table called "${table}".`);
}
return entry;
}
@@ -0,0 +1,58 @@
import BunSQLite from "@moduletrace/bun-sqlite";
import AppError from "@/src/utils/app-error";
import buildRecordRow from "./build-record-row";
import { buildRecordSql, parseRecordKey } from "./build-records-sql";
import grabTableCatalogEntry from "./grab-table-catalog-entry";
import type { DatabaseRecordPayload } from "@/src/types";
import { AppData } from "@/src/data/app-data";
type Params = {
table?: string | null;
/**
* The record's key, as it came in the query string.
*/
key: string;
};
/**
* One record of one table, in full for its modal: every value read whole
* (up to `AppData.DatabaseBrowserValueMaxChars`, since a megabyte of blob
* or checkpoint is not something to render), so a cell cut short in the
* table can still be read to its end.
*/
export default async function grabTableRecord({ table, key }: Params): Promise<DatabaseRecordPayload> {
const entry = await grabTableCatalogEntry({ table });
if (entry.key.kind == "none") {
throw new AppError(`"${entry.name}" has no key to open a single record by.`);
}
const columns = entry.columns.map((column) => column.name);
const query = buildRecordSql({
table: entry.name,
columns,
key: entry.key,
max_chars: AppData.DatabaseBrowserValueMaxChars,
value: parseRecordKey({ value: key }),
});
const res = await BunSQLite.sql<Record<string, unknown>>(query);
if (!res.success) {
throw new AppError(`Couldn't read that record of "${entry.name}".`);
}
const row = res.payload?.[0];
if (!row) {
throw new AppError(`That record isn't in "${entry.name}" any more.`);
}
return {
table: entry,
record: buildRecordRow({
row,
columns: entry.columns,
max_chars: AppData.DatabaseBrowserValueMaxChars,
}),
};
}
@@ -0,0 +1,89 @@
import BunSQLite from "@moduletrace/bun-sqlite";
import AppError from "@/src/utils/app-error";
import buildRecordRow from "./build-record-row";
import buildRecordsSql, { buildCountSql } from "./build-records-sql";
import clampPage from "./clamp-page";
import grabTableCatalogEntry from "./grab-table-catalog-entry";
import type { DatabaseRecord, DatabaseRecordsPayload, DatabaseSearchMode } from "@/src/types";
import { AppData } from "@/src/data/app-data";
type Params = {
table?: string | null;
/**
* Which page of records, from 1.
*/
page?: number;
/**
* What to look for in the table's records, across every column's text.
*/
search?: string;
mode?: DatabaseSearchMode;
};
type TotalRow = { total: number };
/**
* One page of one table's records, read-only: the table asked for with all
* of its columns, its records newest key first, and how many matched so the
* pager knows where the end is.
*
* Every value crosses out of SQLite already cut to
* `AppData.DatabaseBrowserCellMaxChars`, so a table of long texts can't
* turn one page into a megabyte of JSON.
*/
export default async function grabTableRecords({
table,
page = 1,
search = "",
mode = "every_word",
}: Params): Promise<DatabaseRecordsPayload> {
const entry = await grabTableCatalogEntry({ table });
const page_size = AppData.DatabaseBrowserPageSize;
const current_page = Math.max(1, Math.trunc(page || 1));
const columns = entry.columns.map((column) => column.name);
const trimmed = search.trim();
const search_params = trimmed
? { text: trimmed, mode, max_words: AppData.DatabaseBrowserMaxSearchWords }
: null;
const count_res = await BunSQLite.sql<TotalRow>(
buildCountSql({
table: entry.name,
columns,
search: search_params,
}),
);
const total = Number(count_res.payload?.[0]?.total) || 0;
const final_page = clampPage({ page: current_page, total, page_size });
const page_query = buildRecordsSql({
table: entry.name,
columns,
key: entry.key,
max_chars: AppData.DatabaseBrowserCellMaxChars,
limit: page_size,
offset: (final_page - 1) * page_size,
search: search_params,
});
const res = await BunSQLite.sql<Record<string, unknown>>(page_query);
if (!res.success || !res.payload) {
throw new AppError(`Couldn't read the records of "${entry.name}".`);
}
const records: DatabaseRecord[] = res.payload.map((row) =>
buildRecordRow({
row,
columns: entry.columns,
max_chars: AppData.DatabaseBrowserCellMaxChars,
}),
);
return {
table: entry,
records,
total,
page: final_page,
page_size,
search: trimmed,
mode,
};
}
@@ -0,0 +1,20 @@
import BunSQLite from "@moduletrace/bun-sqlite";
import buildTableSizesSql, { type TableSizeRow } from "./build-table-sizes-sql";
/**
* How many bytes each table takes on disk, by table name. Empty when
* SQLite was built without the `dbstat` vtab, which only costs the browser
* its size column: the records still read.
*/
export default async function grabTableSizes(): Promise<Map<string, number>> {
const sizes = new Map<string, number>();
const res = await BunSQLite.sql<TableSizeRow>({ sql: buildTableSizesSql() });
if (!res.success || !res.payload) return sizes;
for (const row of res.payload) {
sizes.set(row.table_name, Number(row.bytes) || 0);
}
return sizes;
}
@@ -0,0 +1,35 @@
import type { TableCatalogEntry, TableCatalogRow } from "./table-catalog-types";
type Params = {
rows: TableCatalogRow[];
};
/**
* Groups the catalog's rows per table, keeping the statement's order, so
* the list of tables and every table's columns come back in one read.
*/
export default function groupTableCatalog({ rows }: Params): TableCatalogEntry[] {
const entries: TableCatalogEntry[] = [];
for (const row of rows) {
let entry = entries.at(-1);
if (!entry || entry.name != row.table_name) {
entry = {
name: row.table_name,
type: row.table_type,
has_rowid: row.table_type == "table" && !row.without_rowid,
columns: [],
};
entries.push(entry);
}
entry.columns.push({
name: row.column_name,
type: row.column_type || "",
pk: Boolean(row.column_pk),
});
}
return entries;
}
@@ -0,0 +1,9 @@
/**
* Quotes a table or column name for SQL, wrapping it in double quotes and
* doubling any it holds. The database browser builds its statements from
* the database's own names, and every one of them still goes through here
* so a name with a quote or a space in it can't break out of the query.
*/
export default function quoteSqlIdentifier({ name }: { name: string }): string {
return `"${name.replace(/"/g, `""`)}"`;
}
@@ -0,0 +1,38 @@
import type { DatabaseTableColumn, DatabaseTableKey } from "@/src/types";
type Params = {
/**
* The table's columns in their declared order.
*/
columns: DatabaseTableColumn[];
/**
* Whether the table is one SQLite gives a rowid to: every ordinary
* table has one, a virtual table and a `WITHOUT ROWID` table do not.
*/
has_rowid: boolean;
};
/**
* What a table's records are identified by: its primary key when that is
* one column (a sqlite-vec index keeps its own `id`), else the rowid every
* ordinary SQLite table has, else nothing, which leaves the browser able
* to read a page but not to open one record.
*
* A composite primary key falls back to the rowid, which is the record's
* own and shows in the key column either way.
*/
export default function resolveTableKey({ columns, has_rowid }: Params): DatabaseTableKey {
const pk_columns = columns.filter((column) => column.pk);
const [pk_column] = pk_columns;
if (pk_column && pk_columns.length == 1) {
return { kind: "column", name: pk_column.name };
}
if (has_rowid) {
return { kind: "rowid" };
}
return { kind: "none" };
}
@@ -0,0 +1,33 @@
/**
* One row of the table catalog: one column of one table.
*/
export type TableCatalogRow = {
table_name: string;
table_type: "table" | "virtual";
/**
* 1 when the table was created `WITHOUT ROWID`.
*/
without_rowid: number;
column_index: number;
column_name: string;
column_type: string | null;
/**
* Where the column sits in the table's primary key; 0 when it isn't
* one.
*/
column_pk: number;
};
/**
* One table of the catalog.
*/
export type TableCatalogEntry = {
name: string;
type: "table" | "virtual";
/**
* Whether SQLite gives the table a rowid: an ordinary table does, a
* `WITHOUT ROWID` table and a virtual table do not.
*/
has_rowid: boolean;
columns: { name: string; type: string; pk: boolean }[];
};
@@ -0,0 +1,48 @@
import formatBytes from "@/src/utils/format-bytes";
import type { DatabaseCell } from "@/src/types";
type Params = {
/**
* The value read out of SQLite, already cut to the length asked for.
*/
value: unknown;
/**
* SQLite's own type for the stored value: `integer`, `real`, `text`,
* `blob` or `null`.
*/
type: string;
/**
* How long the stored value is (characters for text, bytes for a
* blob), which says whether `value` is the whole of it.
*/
length: number;
max_chars: number;
};
/**
* One stored value as a cell shows it. Text is cut at `max_chars` and
* marked truncated, a blob is only its size (its bytes would fill the page
* with noise), a number is printed as it is, and NULL is said, never left
* blank.
*/
export default function toDatabaseCell({ value, type, length, max_chars }: Params): DatabaseCell {
if (type == "null" || value === null || value === undefined) {
return { text: "", kind: "null", truncated: false };
}
if (type == "blob") {
return {
text: `<${formatBytes({ bytes: length })} of binary data>`,
kind: "blob",
truncated: false,
};
}
const text = String(value);
if (type == "integer" || type == "real") {
return { text, kind: "number", truncated: false };
}
return { text, kind: "text", truncated: length > max_chars };
}