diff --git a/README.md b/README.md index aa04a1e..0edc5d3 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,8 @@ A schema-driven SQLite manager for [Bun](https://bun.sh), featuring automatic sc - [`typedef`](#typedef--generate-typescript-types-only) - [`backup`](#backup--back-up-the-database) - [`restore`](#restore--restore-the-database-from-a-backup) + - [`export`](#export--portable-database--schema-archive) + - [`import`](#import--database-file-sql-script-or-export-archive) - [CRUD API](#crud-api) - [Select](#select) - [Insert](#insert) @@ -47,7 +49,7 @@ A schema-driven SQLite manager for [Bun](https://bun.sh), featuring automatic sc ## Prerequisites -This works for just `bun`, so `bun` should be installed. +This works for just `bun`, so `bun` (>= 1.3) should be installed. --- @@ -92,7 +94,6 @@ import type { BunSQLiteConfig } from "@moduletrace/bun-sqlite"; const config: BunSQLiteConfig = { db_name: "my-app.db", - db_schema_file_name: "schema.ts", db_dir: "./db", // optional: where to store the db file db_backup_dir: ".backups", // optional: name of backups directory. Relative to the db dir. typedef_file_path: "./db/types/db.ts", // optional: where to write generated types @@ -103,7 +104,7 @@ export default config; ### 2. Define your schema -Create `./db/schema.ts` (matching `db_schema_file_name` above): +Create `./db/schema.ts` in `db_dir` (or `schema.js`, `schema.json`, `schema.yaml`, `schema.yml`): ```ts import type { BUN_SQLITE_DatabaseSchemaType } from "@moduletrace/bun-sqlite"; @@ -163,16 +164,18 @@ await BunSQLite.delete({ table: "users", targetId: 1 }); ## Configuration -The config file must be named `bun-sqlite.config.ts` and placed at the root of your project. +The config file is `bun-sqlite.config.ts` at the root of your project. `bun-sqlite.config.js`, `.json`, `.yaml` and `.yml` also work; keep only one. `.ts`/`.js` files must `export default` the config. | Field | Type | Required | Description | | --------------------- | -------- | -------- | ------------------------------------------------------------------------------------------ | | `db_name` | `string` | Yes | Filename for the SQLite database (e.g. `"app.db"`) | -| `db_schema_file_name` | `string` | Yes | Filename of the schema file relative to `db_dir` (or root if `db_dir` is not set) | +| `db_schema_file_name` | `string` | No | Schema filename relative to `db_dir`. Defaults to `schema.ts\|js\|json\|yaml\|yml` in `db_dir` | | `db_backup_dir` | `string` | No | Directory for database backups, relative to `db_dir` | | `db_dir` | `string` | No | Root directory for the database file and schema. Defaults to project root | | `typedef_file_path` | `string` | No | Output path for generated TypeScript types, relative to project root | | `max_backups` | `number` | No | Maximum number of backup files to keep. Oldest are deleted automatically. Defaults to `10` | +| `max_exports` | `number` | No | Maximum number of archives kept in `{db_dir}/.exports` by `export`. Defaults to `10` | +| `wal_mode` | `boolean`| No | Enable SQLite `WAL` journal mode | --- @@ -334,6 +337,43 @@ bunx bun-sqlite restore --- +### `export` — Portable database + schema archive + +```bash +# Write {db_name}-{timestamp}.tar.gz into {db_dir}/.exports +bunx bun-sqlite export + +# Explicit path / format +bunx bun-sqlite export -o ./my-app.tar.gz +bunx bun-sqlite export -f zip +``` + +The archive holds `database.sqlite` (a consistent `VACUUM INTO` snapshot) and your schema file as `schema.`. Archives in `.exports` are trimmed to `max_exports`. `.zip` needs the `zip` binary. + +--- + +### `import` — Database file, SQL script, or export archive + +```bash +# Interactive picker (from {db_dir}/.exports) +bunx bun-sqlite import + +# Full export (database + schema) +bunx bun-sqlite import ./my-app.tar.gz + +# Archive: database only / schema only +bunx bun-sqlite import ./my-app.tar.gz --sql-only +bunx bun-sqlite import ./my-app.tar.gz --schema-only + +# A SQLite database file, or an SQL script (run into a fresh database) +bunx bun-sqlite import ./other.db +bunx bun-sqlite import ./dump.sql +``` + +The current database is backed up to `db_backup_dir` before it is replaced. A `.ts`/`.js` schema from an archive runs as code on the next load, so `import` asks before writing it (pass `--yes` to skip the prompt). + +--- + ## CRUD API Import the default export: @@ -757,12 +797,16 @@ bun-sqlite/ │ │ ├── schema.ts # `schema` command │ │ ├── typedef.ts # `typedef` command │ │ ├── backup.ts # `backup` command -│ │ └── restore.ts # `restore` command +│ │ ├── restore.ts # `restore` command +│ │ ├── export.ts # `export` command +│ │ └── import.ts # `import` command │ ├── functions/ -│ │ └── init.ts # Config + schema loader +│ │ ├── init.ts # Loads config, schema and client into globals +│ │ └── grab-sqlite-client.ts # Database client (bun:sqlite + sqlite-vec) +│ ├── lib/ +│ │ ├── db-handler.ts # Runs every CRUD query +│ │ └── schema/ # Schema synchronization (one step per file, + tests) │ ├── lib/sqlite/ -│ │ ├── index.ts # Database client (bun:sqlite + sqlite-vec) -│ │ ├── db-schema-manager.ts # Schema synchronization engine │ │ ├── db-select.ts # Select implementation │ │ ├── db-insert.ts # Insert implementation │ │ ├── db-update.ts # Update implementation @@ -778,7 +822,9 @@ bun-sqlite/ │ ├── sql-gen-operator-gen.ts # Equality operator mapper │ ├── sql-equality-parser.ts # Equality string parser │ ├── append-default-fields-to-db-schema.ts -│ ├── grab-db-dir.ts # Resolve db/backup directory paths +│ ├── resolve-and-load-data-file.ts # Load .ts/.js/.json/.yaml config and schema files +│ ├── export-archive.ts # Read/write export archives +│ ├── grab-db-dir.ts # Resolve db/backup/export directory paths │ ├── grab-db-backup-file-name.ts # Generate timestamped backup filename │ ├── grab-sorted-backups.ts # List backups sorted newest-first │ ├── grab-backup-data.ts # Parse metadata from a backup filename diff --git a/package.json b/package.json index 8925273..9206470 100644 --- a/package.json +++ b/package.json @@ -1,15 +1,35 @@ { "name": "@moduletrace/bun-sqlite", "version": "1.1.13", - "description": "SQLite manager for Bun", + "description": "Schema-driven SQLite manager for Bun", "author": "Benjamin Toby", - "main": "dist/index.js", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + }, + "./types": { + "types": "./dist/types/index.d.ts", + "import": "./dist/types/index.js", + "default": "./dist/types/index.js" + } + }, "bin": { - "bun-sqlite": "dist/commands/index.js" + "bun-sqlite": "./dist/commands/index.js" + }, + "engines": { + "bun": ">=1.3.0" }, "scripts": { "dev": "tsc --watch", - "compile": "rm -rf dist && tsc" + "compile": "rm -rf dist tsconfig.tsbuildinfo && tsc -p tsconfig.json", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "bun test src", + "prepublishOnly": "bun run compile" }, "devDependencies": { "@types/bun": "latest", @@ -29,6 +49,14 @@ "type": "git", "url": "git+https://git.tben.me/Moduletrace/bun-sqlite.git" }, + "keywords": [ + "bun", + "sqlite", + "schema", + "orm", + "sql", + "vector" + ], "dependencies": { "@inquirer/prompts": "^8.3.0", "chalk": "^5.6.2", diff --git a/publish.sh b/publish.sh index c52bbc5..1365913 100755 --- a/publish.sh +++ b/publish.sh @@ -1,17 +1,52 @@ #!/bin/bash -set -e +set -euo pipefail -if [ -z "$1" ]; then +cd "$(dirname "$0")" + +if [ -z "${1:-}" ]; then msg="Updates" else msg="$1" fi -tsc --noEmit -rm -rf dist -tsc -git add . -git commit -m "$msg" -git push +# Always use the project's TypeScript (global `tsc` may not emit) +export PATH="$(pwd)/node_modules/.bin:$PATH" + +if [ ! -x "./node_modules/.bin/tsc" ]; then + echo "Installing dependencies..." + bun install +fi + +echo "Typechecking..." +tsc -p tsconfig.json --noEmit + +echo "Compiling..." +# Clear emit output AND incremental cache. Otherwise a prior --noEmit run leaves +# tsbuildinfo "up to date" and tsc emits nothing after `rm -rf dist`. +rm -rf dist tsconfig.tsbuildinfo +tsc -p tsconfig.json + +if [ ! -f dist/commands/index.js ]; then + echo "ERROR: compile failed — dist/commands/index.js was not produced" + exit 1 +fi + +# Ensure CLI shebang survived compile +if ! head -1 dist/commands/index.js | grep -q '^#!/usr/bin/env bun'; then + echo "Restoring bun shebang on dist/commands/index.js" + printf '%s\n' '#!/usr/bin/env bun' | cat - dist/commands/index.js > dist/commands/index.js.tmp + mv dist/commands/index.js.tmp dist/commands/index.js +fi + +chmod +x dist/commands/index.js + +echo "Build OK: $(head -1 dist/commands/index.js)" + +echo "Publishing package..." +git add -A +git commit -m "$msg" || true +git push || true bun publish + +echo "Done." diff --git a/src/commands/admin/index.ts b/src/commands/admin/index.ts index 10578f2..986f132 100644 --- a/src/commands/admin/index.ts +++ b/src/commands/admin/index.ts @@ -1,22 +1,14 @@ import { Command } from "commander"; -import init from "../../functions/init"; -import grabDBDir from "../../utils/grab-db-dir"; import chalk from "chalk"; import { select } from "@inquirer/prompts"; -import { Database } from "bun:sqlite"; import listTables from "./list-tables"; import runSQL from "./run-sql"; -import * as sqliteVec from "sqlite-vec"; export default function () { return new Command("admin") .description("View Tables and Data, Run SQL Queries, Etc.") .action(async () => { - const { config } = await init(); - const { db_file_path } = grabDBDir({ config }); - const db = new Database(db_file_path); - - sqliteVec.load(db); + const db = global.SQLITE_CLIENT; console.log(chalk.bold(chalk.blue("\nBun SQLite Admin\n"))); diff --git a/src/commands/backup.ts b/src/commands/backup.ts index a3b1692..43d5d0f 100644 --- a/src/commands/backup.ts +++ b/src/commands/backup.ts @@ -1,11 +1,9 @@ import { Command } from "commander"; -import init from "../functions/init"; import path from "path"; import grabDBDir from "../utils/grab-db-dir"; import grabDBBackupFileName from "../utils/grab-db-backup-file-name"; import chalk from "chalk"; import trimBackups from "../utils/trim-backups"; -import DbClient from "../lib/sqlite"; import snapshotDb from "../utils/snapshot-db"; export default function () { @@ -14,14 +12,14 @@ export default function () { .action(async (opts) => { console.log(`Backing up database ...`); - const { config } = await init(); + const config = global.CONFIG; const { backup_dir } = grabDBDir({ config }); const new_db_file_name = grabDBBackupFileName({ config }); snapshotDb({ - db: DbClient, + db: global.SQLITE_CLIENT, dst: path.join(backup_dir, new_db_file_name), }); diff --git a/src/commands/export.ts b/src/commands/export.ts new file mode 100644 index 0000000..7d10685 --- /dev/null +++ b/src/commands/export.ts @@ -0,0 +1,105 @@ +import { Command } from "commander"; +import path from "path"; +import fs from "fs"; +import chalk from "chalk"; +import grabDBDir from "../utils/grab-db-dir"; +import grabDirNames from "../data/grab-dir-names"; +import snapshotDb from "../utils/snapshot-db"; +import { writeExportArchive } from "../utils/export-archive"; +import trimExports from "../utils/trim-exports"; +import { AppData } from "../data/app-data"; + +function defaultExportFileName( + dbName: string, + format: "tar.gz" | "zip", +): string { + return `${dbName}-${Date.now()}.${format}`; +} + +export default function () { + return new Command("export") + .description("Export database file + schema into a portable archive") + .option( + "-o, --output ", + "Output archive path (.tar.gz or .zip). Defaults to db exports dir", + ) + .option( + "-f, --format ", + "Archive format when --output is omitted: tar.gz | zip", + "tar.gz", + ) + .action(async (opts) => { + console.log(`Exporting database ...`); + + const config = global.CONFIG; + const { export_dir } = grabDBDir({ config }); + const { BUN_SQLITE_TEMP_DIR } = grabDirNames(); + + if (!fs.existsSync(export_dir)) { + fs.mkdirSync(export_dir, { recursive: true }); + } + + const formatRaw = String(opts.format || "tar.gz").toLowerCase(); + const format: "tar.gz" | "zip" = + formatRaw === "zip" ? "zip" : "tar.gz"; + + let outPath: string; + if (opts.output) { + outPath = path.resolve(opts.output); + fs.mkdirSync(path.dirname(outPath), { recursive: true }); + } else { + outPath = path.join( + export_dir, + defaultExportFileName(config.db_name, format), + ); + } + + const snapshot_path = path.join( + BUN_SQLITE_TEMP_DIR, + `export-${Date.now()}.db`, + ); + + /** Remove the temp database file before exiting */ + const exit = (code: number): never => { + fs.rmSync(snapshot_path, { force: true }); + process.exit(code); + }; + + try { + snapshotDb({ db: global.SQLITE_CLIENT, dst: snapshot_path }); + + /** + * The schema is always stored as `schema.` so `import` + * can find it, whatever `db_schema_file_name` is. + */ + const schemaFileName = `${AppData.DbSchemaFileName}${path.extname(global.SCHEMA_FILE_PATH)}`; + + await writeExportArchive({ + contents: { + db: new Uint8Array(fs.readFileSync(snapshot_path)), + schema: fs.readFileSync( + global.SCHEMA_FILE_PATH, + "utf-8", + ), + schemaFileName, + }, + outPath, + }); + + if (path.dirname(outPath) === export_dir) { + trimExports({ config }); + } + + console.log( + `${chalk.bold(chalk.green(`DB Export Success!`))} → ${outPath}`, + ); + console.log( + chalk.dim(`Contains: database.sqlite + ${schemaFileName}`), + ); + exit(0); + } catch (error: any) { + console.error(chalk.red(`Export ERROR => ${error.message}`)); + exit(1); + } + }); +} diff --git a/src/commands/import.ts b/src/commands/import.ts new file mode 100644 index 0000000..05635e5 --- /dev/null +++ b/src/commands/import.ts @@ -0,0 +1,283 @@ +import { Command } from "commander"; +import path from "path"; +import fs from "fs"; +import chalk from "chalk"; +import { Database } from "bun:sqlite"; +import * as sqliteVec from "sqlite-vec"; +import { confirm, select } from "@inquirer/prompts"; +import grabDBDir from "../utils/grab-db-dir"; +import grabDirNames from "../data/grab-dir-names"; +import grabSortedExports from "../utils/grab-sorted-exports"; +import grabBackupData from "../utils/grab-backup-data"; +import grabDBBackupFileName from "../utils/grab-db-backup-file-name"; +import snapshotDb from "../utils/snapshot-db"; +import trimBackups from "../utils/trim-backups"; +import replaceDbFile from "../utils/replace-db-file"; +import { + isArchivePath, + isSQLiteFile, + isSqlPath, + readExportArchive, +} from "../utils/export-archive"; +import { resolveDataFile } from "../utils/resolve-and-load-data-file"; +import { AppData } from "../data/app-data"; + +function formatChoice(name: string, index: number): string { + const { backup_date } = grabBackupData({ + backup_name: name.replace(/\.(tar\.gz|tgz|tar|zip)$/i, ""), + }); + const time = Number.isNaN(backup_date.getTime()) + ? "unknown date" + : `${backup_date.toDateString()} ${backup_date.getHours()}:${backup_date.getMinutes()}:${backup_date.getSeconds().toString().padStart(2, "0")}`; + return `#${index + 1}: ${name} (${time})`; +} + +/** + * # Replace the live database with the database file at `src` + * @description The current database is backed up first. + */ +function swapDatabase(src: string) { + const config = global.CONFIG; + const { backup_dir, db_file_path } = grabDBDir({ config }); + + const backup_path = path.join(backup_dir, grabDBBackupFileName({ config })); + snapshotDb({ db: global.SQLITE_CLIENT, dst: backup_path }); + trimBackups({ config }); + console.log(chalk.dim(`Backed up current database → ${backup_path}`)); + + global.SQLITE_CLIENT.close(); + replaceDbFile({ src, db_file_path }); +} + +/** + * # Build a database file from an SQL script + */ +function buildDbFromSql({ sql, dst }: { sql: string; dst: string }) { + fs.mkdirSync(path.dirname(dst), { recursive: true }); + fs.rmSync(dst, { force: true }); + + const db = new Database(dst, { create: true }); + + try { + sqliteVec.load(db); + db.run(sql); + } finally { + db.close(); + } +} + +/** + * # Write the schema file from an archive into the project + */ +function writeSchema({ + schema, + schemaFileName, +}: { + schema: string; + schemaFileName: string; +}) { + const config = global.CONFIG; + const { db_dir } = grabDBDir({ config }); + + if (config.db_schema_file_name) { + const schemaPath = path.join(db_dir, config.db_schema_file_name); + + if (path.extname(schemaPath) !== path.extname(schemaFileName)) { + throw new Error( + `Archive schema is \`${schemaFileName}\` but \`db_schema_file_name\` is \`${config.db_schema_file_name}\`. Use --sql-only, or change \`db_schema_file_name\`.`, + ); + } + + fs.writeFileSync(schemaPath, schema, "utf-8"); + console.log(chalk.green(`Schema written → ${schemaPath}`)); + return; + } + + const existing = resolveDataFile(db_dir, AppData.DbSchemaFileName); + const schemaPath = path.join(db_dir, schemaFileName); + + // Remove a differently-named schema so only one format remains + if (existing && path.resolve(existing.path) !== path.resolve(schemaPath)) { + fs.unlinkSync(existing.path); + console.log( + chalk.dim(`Removed previous schema file: ${existing.basename}`), + ); + } + + fs.writeFileSync(schemaPath, schema, "utf-8"); + console.log(chalk.green(`Schema written → ${schemaPath}`)); +} + +export default function () { + return new Command("import") + .description( + "Import a SQLite database file, an SQL script, or a full export archive (database + schema)", + ) + .argument( + "[file]", + "Path to a SQLite file, .sql file, or export archive (.tar.gz / .tar / .zip)", + ) + .option( + "--sql-only", + "When importing an archive, restore the database only (skip writing schema)", + ) + .option( + "--schema-only", + "When importing an archive, write schema only (skip database restore)", + ) + .option( + "-y, --yes", + "Do not ask for confirmation before writing an executable (.ts/.js) schema from the archive", + ) + .action(async (fileArg: string | undefined, opts) => { + console.log(`Importing database ...`); + + const config = global.CONFIG; + const { db_dir, export_dir } = grabDBDir({ config }); + const { BUN_SQLITE_TEMP_DIR } = grabDirNames(); + + const temp_db_path = path.join( + BUN_SQLITE_TEMP_DIR, + `import-${Date.now()}.db`, + ); + + /** Remove the temp database file before exiting */ + const exit = (code: number): never => { + fs.rmSync(temp_db_path, { force: true }); + process.exit(code); + }; + + try { + let filePath = fileArg ? path.resolve(fileArg) : ""; + + if (!filePath) { + const candidates = grabSortedExports({ config }).filter( + (b) => isArchivePath(b), + ); + + if (!candidates[0]) { + console.error( + `No export archives found in \`${export_dir}\`. Use \`export\`, or pass a file path.`, + ); + exit(1); + } + + const selected = await select({ + message: "Select an export to import:", + choices: candidates.map((b, i) => ({ + name: formatChoice(b, i), + value: b, + })), + }); + + filePath = path.join(export_dir, selected); + } + + if (!fs.existsSync(filePath)) { + console.error(chalk.red(`File not found: ${filePath}`)); + exit(1); + } + + if (opts.sqlOnly && opts.schemaOnly) { + console.error( + chalk.red( + `Cannot combine --sql-only and --schema-only.`, + ), + ); + exit(1); + } + + if (!isArchivePath(filePath)) { + if (opts.schemaOnly) { + console.error( + chalk.red( + `--schema-only requires an export archive.`, + ), + ); + exit(1); + } + + if (isSqlPath(filePath)) { + buildDbFromSql({ + sql: fs.readFileSync(filePath, "utf-8"), + dst: temp_db_path, + }); + swapDatabase(temp_db_path); + } else if ( + isSQLiteFile(new Uint8Array(fs.readFileSync(filePath))) + ) { + swapDatabase(filePath); + } else { + console.error( + chalk.red( + `Unsupported file: ${filePath}. Use a SQLite database file, .sql, .tar.gz, .tar, or .zip.`, + ), + ); + exit(1); + } + + console.log( + `${chalk.bold(chalk.green(`DB Import Success!`))} ← ${filePath}`, + ); + exit(0); + } + + const { db, schema, schemaFileName } = + await readExportArchive(filePath); + + /** + * A .ts/.js schema is executed (require) on the next run. + * Confirm before writing code from an archive into the project. + */ + const isExecutableSchema = /\.(ts|js)$/i.test(schemaFileName); + + if (!opts.sqlOnly && isExecutableSchema && !opts.yes) { + if (!process.stdin.isTTY) { + console.error( + chalk.red( + `Archive contains an executable schema (${schemaFileName}). Re-run with --yes to accept it, or --sql-only to skip it.`, + ), + ); + exit(1); + } + + console.log( + chalk.yellow( + `\nThis archive contains an executable schema file (${schemaFileName}).\nIt will run as code the next time bun-sqlite or your app loads it.\nOnly continue if you trust the source of this archive.\n`, + ), + ); + + const proceed = await confirm({ + message: `Write ${schemaFileName} into ${db_dir}?`, + default: false, + }); + + if (!proceed) { + console.log(chalk.dim(`Import cancelled.`)); + exit(1); + } + } + + if (!opts.sqlOnly) { + writeSchema({ schema, schemaFileName }); + } + + if (!opts.schemaOnly) { + fs.mkdirSync(path.dirname(temp_db_path), { + recursive: true, + }); + fs.writeFileSync(temp_db_path, db); + swapDatabase(temp_db_path); + console.log(chalk.green(`Database restored from archive`)); + } + + console.log( + `${chalk.bold(chalk.green(`DB Import Success!`))} ← ${filePath}`, + ); + exit(0); + } catch (error: any) { + console.error(chalk.red(`Import ERROR => ${error.message}`)); + exit(1); + } + }); +} diff --git a/src/commands/index.ts b/src/commands/index.ts index d03757a..a3f3d2b 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -5,12 +5,17 @@ import schema from "./schema"; import typedef from "./typedef"; import backup from "./backup"; import restore from "./restore"; +import exportCmd from "./export"; +import importCmd from "./import"; import admin from "./admin"; +import init from "../functions/init"; -/** - * # Declare Global Variables - */ -declare global {} +try { + init(); +} catch (error: any) { + console.error(`Initialization ERROR => ${error.message}`); + process.exit(1); +} /** * # Describe Program @@ -27,6 +32,8 @@ program.addCommand(schema()); program.addCommand(typedef()); program.addCommand(backup()); program.addCommand(restore()); +program.addCommand(exportCmd()); +program.addCommand(importCmd()); program.addCommand(admin()); /** diff --git a/src/commands/restore.ts b/src/commands/restore.ts index 4bc4c88..da4bcbe 100644 --- a/src/commands/restore.ts +++ b/src/commands/restore.ts @@ -1,7 +1,5 @@ import { Command } from "commander"; -import init from "../functions/init"; import grabDBDir from "../utils/grab-db-dir"; -import DbClient from "../lib/sqlite"; import replaceDbFile from "../utils/replace-db-file"; import chalk from "chalk"; import grabSortedBackups from "../utils/grab-sorted-backups"; @@ -15,7 +13,7 @@ export default function () { .action(async (opts) => { console.log(`Restoring up database ...`); - const { config } = await init(); + const config = global.CONFIG; const { backup_dir, db_file_path } = grabDBDir({ config }); @@ -42,7 +40,7 @@ export default function () { }), }); - DbClient.close(); + global.SQLITE_CLIENT.close(); replaceDbFile({ src: path.join(backup_dir, selected_backup), diff --git a/src/commands/schema.ts b/src/commands/schema.ts index 5ece199..b761be8 100644 --- a/src/commands/schema.ts +++ b/src/commands/schema.ts @@ -1,15 +1,12 @@ import { Command } from "commander"; -import { SQLiteSchemaManager } from "../lib/sqlite/db-schema-manager"; -import init from "../functions/init"; +import createDBSchema from "../lib/schema/create-db-schema"; import grabDirNames from "../data/grab-dir-names"; import path from "path"; import dbSchemaToTypeDef from "../lib/sqlite/schema-to-typedef"; -import _ from "lodash"; import appendDefaultFieldsToDbSchema from "../utils/append-default-fields-to-db-schema"; import chalk from "chalk"; import { writeLiveSchema } from "../functions/live-schema"; import grabDBDir from "../utils/grab-db-dir"; -import DbClient from "../lib/sqlite"; import snapshotDb from "../utils/snapshot-db"; import replaceDbFile from "../utils/replace-db-file"; @@ -24,7 +21,9 @@ export default function () { .action(async (opts) => { console.log(`Starting process ...`); - const { config, dbSchema } = await init(); + const config = global.CONFIG; + const dbSchema = global.DB_SCHEMA; + const DbClient = global.SQLITE_CLIENT; const { ROOT_DIR, BUN_SQLITE_TEMP_DB_FILE_PATH } = grabDirNames(); const { db_file_path } = grabDBDir({ config }); @@ -38,13 +37,13 @@ export default function () { dbSchema, }); - const manager = new SQLiteSchemaManager({ - schema: finaldbSchema, + createDBSchema({ + db: DbClient, + db_schema: finaldbSchema, recreate_vector_table: isVector, }); - await manager.syncSchema(); - manager.close(); + DbClient.close(); if (isTypeDef && config.typedef_file_path) { const out_file = path.resolve( diff --git a/src/commands/typedef.ts b/src/commands/typedef.ts index 0422e48..4ef6350 100644 --- a/src/commands/typedef.ts +++ b/src/commands/typedef.ts @@ -1,5 +1,4 @@ import { Command } from "commander"; -import init from "../functions/init"; import dbSchemaToTypeDef from "../lib/sqlite/schema-to-typedef"; import path from "path"; import grabDirNames from "../data/grab-dir-names"; @@ -12,7 +11,8 @@ export default function () { .action(async (opts) => { console.log(`Creating Type Definition From DB Schema ...`); - const { config, dbSchema } = await init(); + const config = global.CONFIG; + const dbSchema = global.DB_SCHEMA; const { ROOT_DIR } = grabDirNames(); const finaldbSchema = appendDefaultFieldsToDbSchema({ dbSchema }); @@ -28,7 +28,9 @@ export default function () { config, }); } else { - console.error(``); + console.error( + `\`typedef_file_path\` is required in your config`, + ); process.exit(1); } diff --git a/src/data/app-data.ts b/src/data/app-data.ts index c406a89..f73ea03 100644 --- a/src/data/app-data.ts +++ b/src/data/app-data.ts @@ -1,6 +1,11 @@ export const AppData = { - ConfigFileName: "bun-sqlite.config.ts", + ConfigFileName: "bun-sqlite.config", MaxBackups: 10, + MaxExports: 10, DefaultBackupDirName: ".backups", + DefaultExportDirName: ".exports", DbSchemaManagerTableName: "__db_schema_manager__", + DbSchemaFileName: "schema", + /** Priority order when resolving config/schema files */ + SupportedDataFileExtensions: [".ts", ".js", ".json", ".yaml", ".yml"] as const, } as const; diff --git a/src/functions/grab-sqlite-client.ts b/src/functions/grab-sqlite-client.ts new file mode 100644 index 0000000..95acb7c --- /dev/null +++ b/src/functions/grab-sqlite-client.ts @@ -0,0 +1,33 @@ +import { Database } from "bun:sqlite"; +import * as sqliteVec from "sqlite-vec"; +import type { BunSQLiteConfig } from "../types"; +import grabDBDir from "../utils/grab-db-dir"; + +type Params = { + config: BunSQLiteConfig; +}; + +/** Reuse one connection per database file. */ +const clientCache = new Map(); + +export default function grabSQLiteClient({ config }: Params): Database { + const { db_file_path } = grabDBDir({ config }); + + const cached = clientCache.get(db_file_path); + if (cached) { + return cached; + } + + const SQLiteClient = new Database(db_file_path, { + create: true, + }); + + sqliteVec.load(SQLiteClient); + + if (config.wal_mode) { + SQLiteClient.run("PRAGMA journal_mode = WAL;"); + } + + clientCache.set(db_file_path, SQLiteClient); + return SQLiteClient; +} diff --git a/src/functions/init.ts b/src/functions/init.ts index f5ee5ea..180642b 100644 --- a/src/functions/init.ts +++ b/src/functions/init.ts @@ -1,5 +1,6 @@ import path from "path"; import fs from "fs"; +import type { Database } from "bun:sqlite"; import { AppData } from "../data/app-data"; import grabDirNames from "../data/grab-dir-names"; import type { @@ -7,50 +8,153 @@ import type { BunSQLiteConfigReturn, BUN_SQLITE_DatabaseSchemaType, } from "../types"; +import setSQLiteClient from "./set-sqlite-client"; +import grabDBDir from "../utils/grab-db-dir"; +import { + loadDataFile, + resolveAndLoadDataFile, + supportedDataFileNames, + type LoadedDataFile, + type ResolvedDataFile, +} from "../utils/resolve-and-load-data-file"; + +/** + * # Declare Global Variables + */ +declare global { + var CONFIG: BunSQLiteConfig; + var DB_SCHEMA: BUN_SQLITE_DatabaseSchemaType; + var SQLITE_CLIENT: Database; + var CONFIG_FILE_PATH: string; + var SCHEMA_FILE_PATH: string; +} + +/** + * # Load config, schema and the SQLite client into globals + * @description Runs once per process. Later calls return the loaded + * globals. + */ +export default function init(): BunSQLiteConfigReturn { + if (global.CONFIG && global.DB_SCHEMA && global.SQLITE_CLIENT) { + return { config: global.CONFIG, dbSchema: global.DB_SCHEMA }; + } -export default async function init(): Promise { const { ROOT_DIR } = grabDirNames(); const { ConfigFileName } = AppData; - const ConfigFilePath = path.join(ROOT_DIR, ConfigFileName); + const loadedConfig = resolveAndLoadDataFile( + ROOT_DIR, + ConfigFileName, + ); - if (!fs.existsSync(ConfigFilePath)) { + if (!loadedConfig) { throw new Error( - `Please create a \`${ConfigFileName}\` file at the root of your project. (Looked for ${ConfigFilePath})`, + `Please create a ${supportedDataFileNames(ConfigFileName)} file at the root of your project. (Looked in ${ROOT_DIR})`, ); } - const ConfigImport = await import(ConfigFilePath); - const Config = ConfigImport["default"] as BunSQLiteConfig; + const Config = loadedConfig.data; + + if (!Config || typeof Config !== "object") { + throw new Error( + `Invalid config in \`${loadedConfig.path}\`. Expected a config object${isCodeFile(loadedConfig) ? " (export default)" : ""}.`, + ); + } if (!Config.db_name) { throw new Error(`\`db_name\` is required in your config`); } - if (!Config.db_schema_file_name) { - throw new Error(`\`db_schema_file_name\` is required in your config`); + const { db_dir, backup_dir } = grabDBDir({ config: Config }); + + if (!fs.existsSync(db_dir)) { + fs.mkdirSync(db_dir, { recursive: true }); } - let db_dir = ROOT_DIR; + const loadedSchema = loadSchema({ config: Config, db_dir }); - if (Config.db_dir) { - db_dir = path.resolve(ROOT_DIR, Config.db_dir); + const DbSchema = loadedSchema.data; - if (!fs.existsSync(db_dir)) { - fs.mkdirSync(db_dir, { recursive: true }); - } + if (!DbSchema || typeof DbSchema !== "object") { + throw new Error( + `Invalid schema in \`${loadedSchema.path}\`. Expected a schema object${isCodeFile(loadedSchema) ? " (export default)" : ""}.`, + ); } - const DBSchemaFilePath = path.join(db_dir, Config.db_schema_file_name); - const DbSchemaImport = await import(DBSchemaFilePath); - const DbSchema = DbSchemaImport["default"] as BUN_SQLITE_DatabaseSchemaType; - - const backup_dir = Config.db_backup_dir || AppData["DefaultBackupDirName"]; - - const BackupDir = path.resolve(db_dir, backup_dir); - if (!fs.existsSync(BackupDir)) { - fs.mkdirSync(BackupDir, { recursive: true }); + if (!fs.existsSync(backup_dir)) { + fs.mkdirSync(backup_dir, { recursive: true }); } + global.CONFIG = Config; + global.DB_SCHEMA = DbSchema; + global.CONFIG_FILE_PATH = loadedConfig.path; + global.SCHEMA_FILE_PATH = loadedSchema.path; + + setSQLiteClient({ config: Config }); + return { config: Config, dbSchema: DbSchema }; } + +/** + * # Load the schema file + * @description Uses `db_schema_file_name` when set, otherwise + * `schema.ts|js|json|yaml|yml` in `db_dir`. + */ +function loadSchema({ + config, + db_dir, +}: { + config: BunSQLiteConfig; + db_dir: string; +}): LoadedDataFile { + if (config.db_schema_file_name) { + const schema_file_path = path.join(db_dir, config.db_schema_file_name); + const extension = path.extname(schema_file_path); + + const supported = AppData.SupportedDataFileExtensions.find( + (ext) => ext === extension, + ); + + if (!supported) { + throw new Error( + `Unsupported schema file \`${config.db_schema_file_name}\`. Use one of: ${AppData.SupportedDataFileExtensions.join(", ")}`, + ); + } + + if (!fs.existsSync(schema_file_path)) { + throw new Error(`Schema file not found: ${schema_file_path}`); + } + + const resolved: ResolvedDataFile = { + path: schema_file_path, + basename: path.basename(schema_file_path), + extension: supported, + format: + supported === ".yml" + ? "yaml" + : (supported.slice(1) as ResolvedDataFile["format"]), + }; + + return { + ...resolved, + data: loadDataFile(resolved), + }; + } + + const loaded = resolveAndLoadDataFile( + db_dir, + AppData.DbSchemaFileName, + ); + + if (!loaded) { + throw new Error( + `Please create a schema file (${supportedDataFileNames(AppData.DbSchemaFileName)}) in \`${db_dir}\`, or set \`db_schema_file_name\` in your config.`, + ); + } + + return loaded; +} + +function isCodeFile(file: ResolvedDataFile) { + return file.format === "ts" || file.format === "js"; +} diff --git a/src/functions/set-sqlite-client.ts b/src/functions/set-sqlite-client.ts new file mode 100644 index 0000000..0c2bb07 --- /dev/null +++ b/src/functions/set-sqlite-client.ts @@ -0,0 +1,10 @@ +import type { BunSQLiteConfig } from "../types"; +import grabSQLiteClient from "./grab-sqlite-client"; + +type Params = { + config: BunSQLiteConfig; +}; + +export default function setSQLiteClient({ config }: Params): void { + global.SQLITE_CLIENT = grabSQLiteClient({ config }); +} diff --git a/src/index.ts b/src/index.ts index f3006aa..f6ad547 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,4 @@ +import init from "./functions/init"; import DbDelete from "./lib/sqlite/db-delete"; import DbInsert from "./lib/sqlite/db-insert"; import DbSelect from "./lib/sqlite/db-select"; @@ -6,6 +7,8 @@ import DbUpdate from "./lib/sqlite/db-update"; import grabDbSchema from "./utils/grab-db-schema"; import grabJoinFieldsFromQueryObject from "./utils/grab-join-fields-from-query-object"; +init(); + const BunSQLite = { select: DbSelect, insert: DbInsert, @@ -19,3 +22,21 @@ const BunSQLite = { } as const; export default BunSQLite; + +export type * from "./types"; +export { + UsersOmitedFields, + MariaDBCollations, + TextFieldTypesArray, + BUN_SQLITE_DATATYPES, + ServerQueryOperators, + ServerQueryEqualities, + SQlComparisons, + DataCrudRequestMethods, + DataCrudRequestMethodsLowerCase, + DsqlCrudActions, + QueryFields, + DockerComposeServices, + IndexTypes, + DefaultFields, +} from "./types"; diff --git a/src/lib/db-handler.ts b/src/lib/db-handler.ts new file mode 100644 index 0000000..b2d7c9d --- /dev/null +++ b/src/lib/db-handler.ts @@ -0,0 +1,73 @@ +import type { Database } from "bun:sqlite"; +import init from "../functions/init"; +import grabSQLiteClient from "../functions/grab-sqlite-client"; +import type { + APIResponseObject, + BunSQLiteConfig, + SQLInsertGenValueType, +} from "../types"; + +type Param = { + query: string; + values?: SQLInsertGenValueType[]; + config?: BunSQLiteConfig; +}; + +/** + * Resolve the SQLite client. The global client is used unless a `config` + * for another database is passed. + */ +function resolveClient(config?: BunSQLiteConfig): Database { + if (config) { + return grabSQLiteClient({ config }); + } + + init(); + return global.SQLITE_CLIENT; +} + +/** + * # Main DB Handler Function + * @description Any statement that produces columns (SELECT, WITH, PRAGMA, + * RETURNING, ...) returns rows. Everything else goes through `run`, which + * also handles multiple statements. + */ +export default function dbHandler< + T extends { [k: string]: any } = { [k: string]: any }, +>({ query, values, config }: Param): APIResponseObject { + try { + const CLIENT = resolveClient(config); + + const stmt = CLIENT.prepare(query); + const returnsRows = stmt.columnNames.length > 0; + + try { + if (returnsRows) { + const rows = stmt.all(...((values || []) as any[])) as T[]; + + return { + success: true, + payload: rows, + singleRes: rows[0], + }; + } + } finally { + stmt.finalize(); + } + + const res = CLIENT.run(query, (values || []) as any[]); + + return { + success: true, + postInsertReturn: { + affectedRows: res.changes, + insertId: Number(res.lastInsertRowid), + }, + }; + } catch (error: any) { + return { + success: false, + error: error.message, + }; + } +} diff --git a/src/lib/grab-duplicate-safe-insert-sql.ts b/src/lib/grab-duplicate-safe-insert-sql.ts index e638cc9..0e3fff7 100644 --- a/src/lib/grab-duplicate-safe-insert-sql.ts +++ b/src/lib/grab-duplicate-safe-insert-sql.ts @@ -9,7 +9,7 @@ type Params = { export default async function ({ sql: passed_sql, table, data }: Params) { let sql = passed_sql; - const { dbSchema } = await init(); + const { dbSchema } = init(); const table_schema = dbSchema.tables.find((t) => t.tableName == table); const now = Date.now(); diff --git a/src/lib/schema/build-column-definition.ts b/src/lib/schema/build-column-definition.ts new file mode 100644 index 0000000..ba5f5cd --- /dev/null +++ b/src/lib/schema/build-column-definition.ts @@ -0,0 +1,65 @@ +import type { BUN_SQLITE_FieldSchemaType } from "../../types"; +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; +import mapDataType from "./map-data-types"; + +/** + * Build column definition SQL + */ +export default function buildColumnDefinition( + field: BUN_SQLITE_FieldSchemaType, + is_vector?: boolean, +): string { + if (!field.fieldName) { + throw new Error("Field name is required"); + } + + // vec0 can't parse quoted column names + const fieldName = is_vector + ? field.sideCar + ? `+${field.fieldName}` + : `${field.fieldName}` + : q(field.fieldName); + + const parts: string[] = [fieldName]; + + // Data type mapping + parts.push(mapDataType(field)); + + // Primary key + if (field.primaryKey) { + parts.push("PRIMARY KEY"); + if (field.autoIncrement) { + parts.push("AUTOINCREMENT"); + } + } + + // Not null + if (field.notNullValue && !field.primaryKey) { + parts.push("NOT NULL"); + } + + // Unique + if (field.unique && !field.primaryKey) { + parts.push("UNIQUE"); + } + + if (field.vectorDistanceMetric) { + parts.push(`distance_metric=${field.vectorDistanceMetric}`); + } + + // Default value + if (field.defaultValue !== undefined) { + if (typeof field.defaultValue === "string") { + parts.push( + // Escape single quotes by doubling them to prevent SQL injection and wrap in single quotes + `DEFAULT '${field.defaultValue.replace(/'/g, "''")}'`, + ); + } else { + parts.push(`DEFAULT ${field.defaultValue}`); + } + } else if (field.defaultValueLiteral) { + parts.push(`DEFAULT ${field.defaultValueLiteral}`); + } + + return parts.join(" "); +} diff --git a/src/lib/schema/build-foreign-key-constraint.test.ts b/src/lib/schema/build-foreign-key-constraint.test.ts new file mode 100644 index 0000000..021d94c --- /dev/null +++ b/src/lib/schema/build-foreign-key-constraint.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, test } from "bun:test"; +import type { BUN_SQLITE_FieldSchemaType } from "../../types"; +import buildForeignKeyConstraint from "./build-foreign-key-constraint"; + +function field( + fk: Partial> = {}, +): BUN_SQLITE_FieldSchemaType { + return { + fieldName: "user_id", + dataType: "INTEGER", + foreignKey: { + destinationTableName: "users", + destinationTableColumnName: "id", + ...fk, + }, + }; +} + +describe("buildForeignKeyConstraint", () => { + test("references the destination column", () => { + expect(buildForeignKeyConstraint(field())).toBe( + `FOREIGN KEY ("user_id") REFERENCES "users"("id")`, + ); + }); + + test("adds cascade rules", () => { + expect( + buildForeignKeyConstraint( + field({ cascadeDelete: true, cascadeUpdate: true }), + ), + ).toBe( + `FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE`, + ); + }); + + test("quotes identifiers", () => { + expect( + buildForeignKeyConstraint( + field({ destinationTableName: `we"ird` }), + ), + ).toBe(`FOREIGN KEY ("user_id") REFERENCES "we""ird"("id")`); + }); +}); diff --git a/src/lib/schema/build-foreign-key-constraint.ts b/src/lib/schema/build-foreign-key-constraint.ts new file mode 100644 index 0000000..715d0d6 --- /dev/null +++ b/src/lib/schema/build-foreign-key-constraint.ts @@ -0,0 +1,22 @@ +import type { BUN_SQLITE_FieldSchemaType } from "../../types"; +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; + +/** + * Build foreign key constraint + */ +export default function buildForeignKeyConstraint( + field: BUN_SQLITE_FieldSchemaType, +): string { + const fk = field.foreignKey!; + let constraint = `FOREIGN KEY (${q(field.fieldName!)}) REFERENCES ${q(String(fk.destinationTableName))}(${q(String(fk.destinationTableColumnName))})`; + + if (fk.cascadeDelete) { + constraint += " ON DELETE CASCADE"; + } + + if (fk.cascadeUpdate) { + constraint += " ON UPDATE CASCADE"; + } + + return constraint; +} diff --git a/src/lib/schema/check-if-table-exists.ts b/src/lib/schema/check-if-table-exists.ts new file mode 100644 index 0000000..7f2b811 --- /dev/null +++ b/src/lib/schema/check-if-table-exists.ts @@ -0,0 +1,14 @@ +import type { Database } from "bun:sqlite"; + +type Params = { + db: Database; + table_name: string; +}; + +export default function checkIfTableExists({ db, table_name }: Params) { + const tableExists = db + .query(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`) + .get(table_name) as any; + + return Boolean(tableExists?.name); +} diff --git a/src/lib/schema/create-db-manager-table.ts b/src/lib/schema/create-db-manager-table.ts new file mode 100644 index 0000000..446cf74 --- /dev/null +++ b/src/lib/schema/create-db-manager-table.ts @@ -0,0 +1,13 @@ +import type { Database } from "bun:sqlite"; +import { AppData } from "../../data/app-data"; +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; + +export default function createDBManagerTable({ db }: { db: Database }) { + db.run(` + CREATE TABLE IF NOT EXISTS ${q(AppData["DbSchemaManagerTableName"])} ( + table_name TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) + `); +} diff --git a/src/lib/schema/create-db-schema.test.ts b/src/lib/schema/create-db-schema.test.ts new file mode 100644 index 0000000..e1fb65d --- /dev/null +++ b/src/lib/schema/create-db-schema.test.ts @@ -0,0 +1,146 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import type { + BUN_SQLITE_FieldSchemaType, + BUN_SQLITE_TableSchemaType, +} from "../../types"; +import createDBSchema from "./create-db-schema"; + +const id: BUN_SQLITE_FieldSchemaType = { + fieldName: "id", + dataType: "INTEGER", + primaryKey: true, + autoIncrement: true, +}; + +const users: BUN_SQLITE_TableSchemaType = { + tableName: "users", + fields: [id, { fieldName: "name", dataType: "TEXT" }], +}; + +const posts: BUN_SQLITE_TableSchemaType = { + tableName: "posts", + fields: [ + id, + { + fieldName: "user_id", + dataType: "INTEGER", + foreignKey: { + destinationTableName: "users", + destinationTableColumnName: "id", + cascadeDelete: true, + }, + }, + { fieldName: "title", dataType: "TEXT" }, + ], + indexes: [{ indexName: "posts_title_idx", indexTableFields: ["title"] }], +}; + +let db: Database; + +function sync(tables: BUN_SQLITE_TableSchemaType[]) { + createDBSchema({ db, db_schema: { tables } }); +} + +function tableNames() { + return ( + db + .query( + `SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE '\\_\\_%' ESCAPE '\\' AND name NOT LIKE 'sqlite_%' ORDER BY name`, + ) + .all() as { name: string }[] + ).map((r) => r.name); +} + +beforeEach(() => { + spyOn(console, "log").mockImplementation(() => {}); + db = new Database(":memory:"); +}); + +afterEach(() => { + db.close(); +}); + +describe("createDBSchema", () => { + test("creates tables, foreign keys and indexes", () => { + sync([posts, users]); + + expect(tableNames()).toEqual(["posts", "users"]); + expect( + db.query(`PRAGMA foreign_key_list("posts")`).all(), + ).toMatchObject([{ table: "users", from: "user_id", to: "id" }]); + expect( + db + .query( + `SELECT name FROM sqlite_master WHERE type='index' AND name = 'posts_title_idx'`, + ) + .get(), + ).toBeTruthy(); + }); + + test("keeps data (and child rows) when a table is rebuilt", () => { + sync([users, posts]); + db.run(`INSERT INTO users (name) VALUES ('ada')`); + db.run(`INSERT INTO posts (user_id, title) VALUES (1, 'hello')`); + + sync([ + { + ...users, + fields: [...users.fields, { fieldName: "email", dataType: "TEXT" }], + }, + posts, + ]); + + expect(db.query(`SELECT name, email FROM users`).all()).toEqual([ + { name: "ada", email: null }, + ]); + expect(db.query(`SELECT title FROM posts`).all()).toEqual([ + { title: "hello" }, + ]); + }); + + test("drops tables removed from the schema", () => { + sync([users, posts]); + sync([users]); + + expect(tableNames()).toEqual(["users"]); + }); + + test("renames tables with tableNameOld", () => { + sync([users]); + db.run(`INSERT INTO users (name) VALUES ('ada')`); + + sync([{ ...users, tableName: "people", tableNameOld: "users" }]); + + expect(tableNames()).toEqual(["people"]); + expect(db.query(`SELECT name FROM people`).all()).toEqual([ + { name: "ada" }, + ]); + }); + + test("rolls back on foreign key violations", () => { + sync([users, posts]); + db.run(`PRAGMA foreign_keys = OFF`); + db.run(`INSERT INTO posts (user_id, title) VALUES (99, 'orphan')`); + db.run(`PRAGMA foreign_keys = ON`); + + expect(() => + sync([ + { + ...users, + fields: [ + ...users.fields, + { fieldName: "email", dataType: "TEXT" }, + ], + }, + posts, + ]), + ).toThrow(/Foreign key violations/); + + expect( + (db.query(`PRAGMA table_info("users")`).all() as any[]).map( + (c) => c.name, + ), + ).toEqual(["id", "name"]); + }); +}); diff --git a/src/lib/schema/create-db-schema.ts b/src/lib/schema/create-db-schema.ts new file mode 100644 index 0000000..fefc7f6 --- /dev/null +++ b/src/lib/schema/create-db-schema.ts @@ -0,0 +1,84 @@ +import _ from "lodash"; +import type { CreateDBSchemaParams } from "../../types"; +import createDBManagerTable from "./create-db-manager-table"; +import dropRemovedTables from "./drop-removed-tables"; +import getExistingTablesFromTablesManagerTable from "./get-existing-tables-from-tables-manager-table"; +import handleDBSchemaTable from "./handle-db-schema-table"; +import orderDBSchema from "./order-db-schema"; + +/** + * # Sync the database to the schema + * @description Tables are rebuilt by copying into a new table and dropping + * the old one. With foreign keys ON, `DROP TABLE` runs an implicit + * `DELETE FROM` that fires `ON DELETE CASCADE` on child tables, so keys are + * disabled for the sync and checked before committing. (`PRAGMA + * foreign_keys` is a no-op inside a transaction.) + */ +export default function createDBSchema(params: CreateDBSchemaParams) { + const { db } = params; + + console.log("Starting schema synchronization..."); + + /** + * Create Schema Manager Table + */ + createDBManagerTable({ db }); + + /** + * Reorder Tables (parents before children with FKs) + */ + const db_schema = orderDBSchema(params); + + db.run("PRAGMA foreign_keys = OFF;"); + db.run("BEGIN"); + + try { + const existing_tables = getExistingTablesFromTablesManagerTable({ + db, + }); + + /** + * Create or update tables + */ + for (const table of db_schema.tables) { + handleDBSchemaTable({ + ...params, + db_schema, + table, + existing_tables, + }); + } + + /** + * Drop tables that no longer exist in schema + */ + dropRemovedTables({ + db, + existing_tables, + schema_tables: db_schema.tables.map((t) => t.tableName), + }); + + const fkViolations = db.query(`PRAGMA foreign_key_check`).all() as { + table: string; + parent: string; + }[]; + + if (fkViolations.length) { + const tables = _.uniq( + fkViolations.map((v) => `${v.table} -> ${v.parent}`), + ); + throw new Error( + `Foreign key violations after schema sync: ${tables.join(", ")}`, + ); + } + + db.run("COMMIT"); + } catch (error) { + if (db.inTransaction) db.run("ROLLBACK"); + throw error; + } finally { + db.run("PRAGMA foreign_keys = ON;"); + } + + console.log("Schema synchronization complete!"); +} diff --git a/src/lib/schema/create-table.ts b/src/lib/schema/create-table.ts new file mode 100644 index 0000000..723144e --- /dev/null +++ b/src/lib/schema/create-table.ts @@ -0,0 +1,59 @@ +import type { + BUN_SQLITE_TableSchemaType, + CreateDBSchemaParams, +} from "../../types"; +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; +import buildColumnDefinition from "./build-column-definition"; +import buildForeignKeyConstraint from "./build-foreign-key-constraint"; +import resolveTable from "./resolve-table"; + +type Params = CreateDBSchemaParams & { + table: BUN_SQLITE_TableSchemaType; +}; + +/** + * Create a new table + */ +export default function createTable({ db, db_schema, table }: Params) { + if (!table.tableName.match(/_temp_\d+$/)) { + console.log(`Creating table: ${table.tableName}`); + } + + const new_table = resolveTable({ table, db_schema }); + + const columns: string[] = []; + const foreignKeys: string[] = []; + + for (const field of new_table.fields) { + columns.push(buildColumnDefinition(field, new_table.isVector)); + + if (field.foreignKey && !new_table.isVector) { + foreignKeys.push(buildForeignKeyConstraint(field)); + } + } + + // Add unique constraints + for (const constraint of new_table.uniqueConstraints || []) { + if (!constraint.constraintTableFields?.length) continue; + + const fieldNames = constraint.constraintTableFields.map((f) => + String(f.value), + ); + const constraintName = + constraint.constraintName || `unique_${fieldNames.join(", ")}`; + + columns.push( + `CONSTRAINT ${q(constraintName)} UNIQUE (${fieldNames + .map(q) + .join(", ")})`, + ); + } + + const allConstraints = [...columns, ...foreignKeys]; + + const sql = new_table.isVector + ? `CREATE VIRTUAL TABLE ${q(new_table.tableName)} USING ${new_table.vectorType || "vec0"}(${allConstraints.join(", ")})` + : `CREATE TABLE ${q(new_table.tableName)} (${allConstraints.join(", ")})`; + + db.run(sql); +} diff --git a/src/lib/schema/drop-removed-tables.ts b/src/lib/schema/drop-removed-tables.ts new file mode 100644 index 0000000..aa30092 --- /dev/null +++ b/src/lib/schema/drop-removed-tables.ts @@ -0,0 +1,43 @@ +import type { Database } from "bun:sqlite"; +import { readLiveSchema } from "../../functions/live-schema"; +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; +import removeDBManagerTable from "./remove-db-manager-table"; + +type Params = { + db: Database; + existing_tables: string[]; + schema_tables: string[]; +}; + +/** + * Drop tables that are no longer in the schema + */ +export default function dropRemovedTables({ + db, + existing_tables, + schema_tables, +}: Params) { + console.log(`Cleaning up tables ...`); + + const tablesToDrop = existing_tables.filter( + (t) => + !schema_tables.includes(t) && + !schema_tables.find((scT) => t.startsWith(scT + "_")), + ); + + const currentSchema = readLiveSchema(); + + for (const table of currentSchema?.tables || []) { + if (!table?.tableName) continue; + + if (!schema_tables.includes(table.tableName)) { + tablesToDrop.push(table.tableName); + } + } + + for (const tableName of tablesToDrop) { + console.log(`Dropping table: ${tableName}`); + db.run(`DROP TABLE IF EXISTS ${q(tableName)}`); + removeDBManagerTable({ db, table_name: tableName }); + } +} diff --git a/src/lib/schema/get-existing-tables-from-tables-manager-table.ts b/src/lib/schema/get-existing-tables-from-tables-manager-table.ts new file mode 100644 index 0000000..ae6db42 --- /dev/null +++ b/src/lib/schema/get-existing-tables-from-tables-manager-table.ts @@ -0,0 +1,15 @@ +import type { Database } from "bun:sqlite"; +import { AppData } from "../../data/app-data"; +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; + +export default function getExistingTablesFromTablesManagerTable({ + db, +}: { + db: Database; +}): string[] { + const results = db + .query(`SELECT table_name FROM ${q(AppData["DbSchemaManagerTableName"])}`) + .all() as { table_name: string }[]; + + return results.map((r) => r.table_name); +} diff --git a/src/lib/schema/get-table-columns.ts b/src/lib/schema/get-table-columns.ts new file mode 100644 index 0000000..7439ec2 --- /dev/null +++ b/src/lib/schema/get-table-columns.ts @@ -0,0 +1,17 @@ +import type { Database } from "bun:sqlite"; +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; + +type Params = { + db: Database; + table_name: string; +}; + +export default function getTableColumns({ + db, + table_name, +}: Params): { name: string; type: string }[] { + return db.query(`PRAGMA table_info(${q(table_name)})`).all() as { + name: string; + type: string; + }[]; +} diff --git a/src/lib/schema/handle-db-schema-table.ts b/src/lib/schema/handle-db-schema-table.ts new file mode 100644 index 0000000..636c1ff --- /dev/null +++ b/src/lib/schema/handle-db-schema-table.ts @@ -0,0 +1,50 @@ +import type { + BUN_SQLITE_TableSchemaType, + CreateDBSchemaParams, +} from "../../types"; +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; +import createTable from "./create-table"; +import insertDBManagerTable from "./insert-db-manager-table"; +import removeDBManagerTable from "./remove-db-manager-table"; +import syncIndexes from "./sync-indexes"; +import updateTable from "./update-table"; + +type Params = CreateDBSchemaParams & { + table: BUN_SQLITE_TableSchemaType; + existing_tables: string[]; +}; + +/** + * Sync a single table (rename, create or update, then indexes) + */ +export default function handleDBSchemaTable(params: Params) { + const { db, table, existing_tables } = params; + + let tableExists = existing_tables.includes(table.tableName); + + // Handle table rename + if ( + table.tableNameOld && + table.tableNameOld !== table.tableName && + existing_tables.includes(table.tableNameOld) + ) { + console.log( + `Renaming table: ${table.tableNameOld} -> ${table.tableName}`, + ); + db.run( + `ALTER TABLE ${q(table.tableNameOld)} RENAME TO ${q(table.tableName)}`, + ); + insertDBManagerTable({ db, table_name: table.tableName }); + removeDBManagerTable({ db, table_name: table.tableNameOld }); + tableExists = true; + } + + if (!tableExists) { + createTable(params); + insertDBManagerTable({ db, table_name: table.tableName }); + } else { + updateTable(params); + } + + syncIndexes({ db, table }); +} diff --git a/src/lib/schema/insert-db-manager-table.ts b/src/lib/schema/insert-db-manager-table.ts new file mode 100644 index 0000000..e6a4e40 --- /dev/null +++ b/src/lib/schema/insert-db-manager-table.ts @@ -0,0 +1,15 @@ +import type { Database } from "bun:sqlite"; +import { AppData } from "../../data/app-data"; +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; + +type Params = { + db: Database; + table_name: string; +}; + +export default function insertDBManagerTable({ db, table_name }: Params) { + db.run( + `INSERT INTO ${q(AppData["DbSchemaManagerTableName"])} (table_name,created_at,updated_at) VALUES (?, ?, ?)`, + [table_name, Date.now(), Date.now()], + ); +} diff --git a/src/lib/schema/map-data-types.ts b/src/lib/schema/map-data-types.ts new file mode 100644 index 0000000..a8bfd14 --- /dev/null +++ b/src/lib/schema/map-data-types.ts @@ -0,0 +1,58 @@ +import type { + BUN_SQLITE_DATATYPES, + BUN_SQLITE_FieldSchemaType, +} from "../../types"; + +/** + * Map DSQL data types to SQLite types + */ +export default function mapDataType( + field: BUN_SQLITE_FieldSchemaType, +): (typeof BUN_SQLITE_DATATYPES)[number]["value"] { + const dataType = field.dataType?.toLowerCase() || "text"; + const vectorSize = field.vectorSize || 1536; + + // Vector Embeddings + if (field.isVector) { + return `FLOAT[${vectorSize}]` as any; + } + + // Integer types + if ( + dataType.includes("int") || + dataType === "bigint" || + dataType === "smallint" || + dataType === "tinyint" + ) { + return "INTEGER"; + } + + // Real/Float types + if ( + dataType.includes("real") || + dataType.includes("float") || + dataType.includes("double") || + dataType === "decimal" || + dataType === "numeric" + ) { + return "REAL"; + } + + // Blob types + if (dataType.includes("blob") || dataType.includes("binary")) { + return "BLOB"; + } + + // Boolean + if (dataType === "boolean" || dataType === "bool") { + return "INTEGER"; // SQLite uses INTEGER for boolean (0/1) + } + + // Date/Time types + if (dataType.includes("date") || dataType.includes("time")) { + return "TEXT"; // SQLite stores dates as TEXT or INTEGER + } + + // Default to TEXT for all text-based types + return "TEXT"; +} diff --git a/src/lib/schema/order-db-schema.test.ts b/src/lib/schema/order-db-schema.test.ts new file mode 100644 index 0000000..5d392f3 --- /dev/null +++ b/src/lib/schema/order-db-schema.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, test } from "bun:test"; +import type { BUN_SQLITE_TableSchemaType } from "../../types"; +import orderDBSchema from "./order-db-schema"; + +function table({ + name, + refs = [], +}: { + name: string; + refs?: string[]; +}): BUN_SQLITE_TableSchemaType { + return { + tableName: name, + fields: refs.map((ref) => ({ + fieldName: `${ref}_id`, + dataType: "INTEGER", + foreignKey: { + destinationTableName: ref, + destinationTableColumnName: "id", + }, + })), + }; +} + +function order(tables: BUN_SQLITE_TableSchemaType[]) { + const res = orderDBSchema({ db_schema: { tables } }); + return res.tables.map((t) => t.tableName); +} + +describe("orderDBSchema", () => { + test("places parents before children", () => { + expect( + order([ + table({ name: "comments", refs: ["posts", "users"] }), + table({ name: "posts", refs: ["users"] }), + table({ name: "users" }), + ]), + ).toEqual(["users", "posts", "comments"]); + }); + + test("keeps declaration order when there are no dependencies", () => { + expect( + order([table({ name: "b" }), table({ name: "a" })]), + ).toEqual(["b", "a"]); + }); + + test("handles self references without recursing forever", () => { + expect( + order([ + table({ name: "categories", refs: ["categories"] }), + table({ name: "comments", refs: ["comments", "users"] }), + table({ name: "users" }), + ]), + ).toEqual(["categories", "users", "comments"]); + }); + + test("handles cycles and emits each table once", () => { + const res = order([ + table({ name: "a", refs: ["b"] }), + table({ name: "b", refs: ["a"] }), + table({ name: "c", refs: ["a"] }), + ]); + expect(res).toHaveLength(3); + expect(new Set(res).size).toBe(3); + expect(res.indexOf("c")).toBeGreaterThan(res.indexOf("a")); + }); + + test("ignores references to tables outside the schema", () => { + expect( + order([table({ name: "posts", refs: ["external_users"] })]), + ).toEqual(["posts"]); + }); +}); diff --git a/src/lib/schema/order-db-schema.ts b/src/lib/schema/order-db-schema.ts new file mode 100644 index 0000000..ae3bae5 --- /dev/null +++ b/src/lib/schema/order-db-schema.ts @@ -0,0 +1,67 @@ +import _ from "lodash"; +import type { + BUN_SQLITE_DatabaseSchemaType, + BUN_SQLITE_TableSchemaType, + CreateDBSchemaParams, +} from "../../types"; + +/** + * Topologically order tables so referenced (parent) tables come before the + * tables that reference them. Declaration order is kept wherever the FK + * graph allows it. + * + * - Self references (e.g. `comments.parent_comment_id`) are ignored. + * - References to tables outside the schema are ignored. + * - Cycles do not throw: a table already on the current path is skipped, + * so every table is still emitted exactly once. Foreign keys are not + * enforced during the sync, so cycles are safe. + */ +export default function orderDBSchema({ + db_schema, +}: Pick): BUN_SQLITE_DatabaseSchemaType { + const new_db_schema = _.cloneDeep(db_schema); + const tables = new_db_schema.tables; + + const tables_by_name = new Map(); + for (const table of tables) { + if (table) tables_by_name.set(table.tableName, table); + } + + const ordered: BUN_SQLITE_TableSchemaType[] = []; + const visited = new Set(); + const visiting = new Set(); + + function visit(table: BUN_SQLITE_TableSchemaType) { + if (visited.has(table.tableName) || visiting.has(table.tableName)) { + return; + } + + visiting.add(table.tableName); + + for (const field of table.fields || []) { + const dst_table_name = field?.foreignKey?.destinationTableName; + + if (!dst_table_name || dst_table_name === table.tableName) { + continue; + } + + const fk_table = tables_by_name.get(dst_table_name); + + if (fk_table) { + visit(fk_table); + } + } + + visiting.delete(table.tableName); + visited.add(table.tableName); + ordered.push(table); + } + + for (const table of tables) { + if (table) visit(table); + } + + new_db_schema.tables = ordered; + + return new_db_schema; +} diff --git a/src/lib/schema/remove-db-manager-table.ts b/src/lib/schema/remove-db-manager-table.ts new file mode 100644 index 0000000..56e3bea --- /dev/null +++ b/src/lib/schema/remove-db-manager-table.ts @@ -0,0 +1,15 @@ +import type { Database } from "bun:sqlite"; +import { AppData } from "../../data/app-data"; +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; + +type Params = { + db: Database; + table_name: string; +}; + +export default function removeDBManagerTable({ db, table_name }: Params) { + db.run( + `DELETE FROM ${q(AppData["DbSchemaManagerTableName"])} WHERE table_name = ?`, + [table_name], + ); +} diff --git a/src/lib/schema/resolve-table.ts b/src/lib/schema/resolve-table.ts new file mode 100644 index 0000000..049ed13 --- /dev/null +++ b/src/lib/schema/resolve-table.ts @@ -0,0 +1,39 @@ +import _ from "lodash"; +import type { + BUN_SQLITE_DatabaseSchemaType, + BUN_SQLITE_TableSchemaType, +} from "../../types"; + +type Params = { + table: BUN_SQLITE_TableSchemaType; + db_schema: BUN_SQLITE_DatabaseSchemaType; +}; + +/** + * Resolve a table's effective definition. Tables with `parentTableName` + * take their structure from the parent. Always returns a copy: the schema + * objects are never mutated. + */ +export default function resolveTable({ + table, + db_schema, +}: Params): BUN_SQLITE_TableSchemaType { + if (!table.parentTableName) { + return _.cloneDeep(table); + } + + const parent_table = db_schema.tables.find( + (t) => t.tableName === table.parentTableName, + ); + + if (!parent_table) { + throw new Error( + `Parent table \`${table.parentTableName}\` not found for \`${table.tableName}\``, + ); + } + + return _.merge(_.cloneDeep(parent_table), { + tableName: table.tableName, + tableDescription: table.tableDescription, + }); +} diff --git a/src/lib/schema/sync-indexes.ts b/src/lib/schema/sync-indexes.ts new file mode 100644 index 0000000..05b43a1 --- /dev/null +++ b/src/lib/schema/sync-indexes.ts @@ -0,0 +1,53 @@ +import type { Database } from "bun:sqlite"; +import type { BUN_SQLITE_TableSchemaType } from "../../types"; +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; + +type Params = { + db: Database; + table: BUN_SQLITE_TableSchemaType; +}; + +/** + * Sync indexes for a table + */ +export default function syncIndexes({ db, table }: Params) { + if (!table.indexes || table.indexes.length === 0) { + return; + } + + const existingIndexes = ( + db + .query( + `SELECT name FROM sqlite_master WHERE type='index' AND tbl_name=? AND name NOT LIKE 'sqlite_%'`, + ) + .all(table.tableName) as { name: string }[] + ).map((r) => r.name); + + // Drop indexes not in schema + for (const indexName of existingIndexes) { + const stillExists = table.indexes.some( + (idx) => idx.indexName === indexName, + ); + if (!stillExists) { + console.log(`Dropping index: ${indexName}`); + db.run(`DROP INDEX IF EXISTS ${q(indexName)}`); + } + } + + // Create new indexes + for (const index of table.indexes) { + if (!index.indexName || !index.indexTableFields?.length) { + continue; + } + + if (!existingIndexes.includes(index.indexName)) { + console.log(`Creating index: ${index.indexName}`); + const fields = index.indexTableFields + .map((f) => q(String(f))) + .join(", "); + db.run( + `CREATE INDEX ${q(index.indexName)} ON ${q(table.tableName)} (${fields})`, + ); + } + } +} diff --git a/src/lib/schema/update-table.ts b/src/lib/schema/update-table.ts new file mode 100644 index 0000000..1c31893 --- /dev/null +++ b/src/lib/schema/update-table.ts @@ -0,0 +1,91 @@ +import type { + BUN_SQLITE_TableSchemaType, + CreateDBSchemaParams, +} from "../../types"; +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; +import checkIfTableExists from "./check-if-table-exists"; +import createTable from "./create-table"; +import getTableColumns from "./get-table-columns"; +import resolveTable from "./resolve-table"; + +type Params = CreateDBSchemaParams & { + table: BUN_SQLITE_TableSchemaType; +}; + +/** + * # Update an existing table + * @description SQLite has limited ALTER TABLE support, so the table is + * rebuilt: create a temp table with the new schema, copy the data over, + * drop the old table and rename the temp table. + */ +export default function updateTable(params: Params) { + const { db, db_schema, table } = params; + + console.log(`Updating table: ${table.tableName}`); + + if (table.isVector) { + recreateVectorTable(params); + return; + } + + const tempTableName = `${table.tableName}_temp_${Date.now()}`; + + // Columns are matched by name only: SQLite converts values to the new + // column affinity on copy, so a type change keeps the data instead of + // silently dropping the column. + const existingColumns = getTableColumns({ db, table_name: table.tableName }); + const columnsToKeep = resolveTable({ table, db_schema }) + .fields.filter( + (f) => + f.fieldName && + existingColumns.find((c) => c.name == f.fieldName), + ) + .map((f) => f.fieldName as string); + + // Create temp table with new schema + createTable({ ...params, table: { ...table, tableName: tempTableName } }); + + // Copy data if there are common columns + if (columnsToKeep.length > 0) { + const columnList = columnsToKeep.map(q).join(", "); + db.run( + `INSERT INTO ${q(tempTableName)} (${columnList}) SELECT ${columnList} FROM ${q(table.tableName)}`, + ); + } + + db.run(`DROP TABLE ${q(table.tableName)}`); + db.run(`ALTER TABLE ${q(tempTableName)} RENAME TO ${q(table.tableName)}`); +} + +/** + * Vector tables are only rebuilt when `recreate_vector_table` is set. + */ +function recreateVectorTable(params: Params) { + const { db, table, recreate_vector_table } = params; + + if (!recreate_vector_table) { + return; + } + + let existingRows: { [k: string]: any }[] = []; + + if (checkIfTableExists({ db, table_name: table.tableName })) { + existingRows = db + .query(`SELECT * FROM ${q(table.tableName)}`) + .all() as { [k: string]: any }[]; + + db.run(`DROP TABLE ${q(table.tableName)}`); + } + + createTable(params); + + for (const row of existingRows) { + const columns = Object.keys(row); + const placeholders = columns.map(() => "?").join(", "); + + db.run( + `INSERT INTO ${q(table.tableName)} (${columns.map(q).join(", ")}) VALUES (${placeholders})`, + Object.values(row), + ); + } +} diff --git a/src/lib/sqlite/db-delete.ts b/src/lib/sqlite/db-delete.ts index 1cf31f4..a8fffd1 100644 --- a/src/lib/sqlite/db-delete.ts +++ b/src/lib/sqlite/db-delete.ts @@ -1,7 +1,12 @@ -import DbClient from "."; +import dbHandler from "../db-handler"; import _ from "lodash"; -import type { APIResponseObject, ServerQueryParam } from "../../types"; +import type { + APIResponseObject, + BunSQLiteConfig, + ServerQueryParam, +} from "../../types"; import sqlGenerator from "../../utils/sql-generator"; +import { sqlIdent } from "../../utils/sql-safe"; type Params< Schema extends { [k: string]: any } = { [k: string]: any }, @@ -10,6 +15,7 @@ type Params< table: Table; query?: ServerQueryParam; targetId?: number | string; + config?: BunSQLiteConfig; }; export default async function DbDelete< @@ -19,6 +25,7 @@ export default async function DbDelete< table, query, targetId, + config, }: Params): Promise { let sqlObj: ReturnType | null = null; @@ -47,19 +54,24 @@ export default async function DbDelete< const whereClause = sqlObj.where.string; if (whereClause) { - let sql = `DELETE FROM ${table} ${whereClause}`; + let sql = `DELETE FROM ${sqlIdent(table, "table name")} ${whereClause}`; sqlObj.string = sql; sqlObj.values = sqlObj.where.values; - const res = DbClient.run(sql, sqlObj.values); + const res = dbHandler({ + query: sql, + values: sqlObj.values, + config, + }); + + if (!res.success) { + return { ...res, debug: { sqlObj } }; + } return { - success: Boolean(res.changes), - postInsertReturn: { - affectedRows: res.changes, - insertId: Number(res.lastInsertRowid), - }, + success: Boolean(res.postInsertReturn?.affectedRows), + postInsertReturn: res.postInsertReturn, debug: { sqlObj, }, diff --git a/src/lib/sqlite/db-insert.ts b/src/lib/sqlite/db-insert.ts index 5dce989..64924aa 100644 --- a/src/lib/sqlite/db-insert.ts +++ b/src/lib/sqlite/db-insert.ts @@ -1,5 +1,9 @@ -import DbClient from "."; -import type { APIResponseObject, SQLInsertGenReturn } from "../../types"; +import dbHandler from "../db-handler"; +import type { + APIResponseObject, + BunSQLiteConfig, + SQLInsertGenReturn, +} from "../../types"; import sqlInsertGenerator from "../../utils/sql-insert-generator"; import grabDuplicateSafeInsertSql from "../grab-duplicate-safe-insert-sql"; @@ -10,6 +14,7 @@ type Params< table: Table; data: Schema[]; update_on_duplicate?: boolean; + config?: BunSQLiteConfig; }; export default async function DbInsert< @@ -19,6 +24,7 @@ export default async function DbInsert< table, data, update_on_duplicate, + config, }: Params): Promise { let sqlObj: SQLInsertGenReturn | null = null; @@ -43,14 +49,19 @@ export default async function DbInsert< (sqlObj || ({} as any)).query = sql; - const res = DbClient.run(sql, sqlObj?.values || []); + const res = dbHandler({ + query: sql, + values: sqlObj?.values || [], + config, + }); + + if (!res.success) { + return { ...res, debug: { sqlObj } }; + } return { - success: Boolean(Number(res.lastInsertRowid)), - postInsertReturn: { - affectedRows: res.changes, - insertId: Number(res.lastInsertRowid), - }, + success: Boolean(res.postInsertReturn?.insertId), + postInsertReturn: res.postInsertReturn, debug: { sqlObj, }, diff --git a/src/lib/sqlite/db-schema-manager.ts b/src/lib/sqlite/db-schema-manager.ts deleted file mode 100644 index 103c497..0000000 --- a/src/lib/sqlite/db-schema-manager.ts +++ /dev/null @@ -1,661 +0,0 @@ -#!/usr/bin/env bun - -import { Database } from "bun:sqlite"; -import _ from "lodash"; -import DbClient from "."; -import type { - BUN_SQLITE_DatabaseSchemaType, - BUN_SQLITE_DATATYPES, - BUN_SQLITE_FieldSchemaType, - BUN_SQLITE_TableSchemaType, -} from "../../types"; -import { AppData } from "../../data/app-data"; -import { readLiveSchema } from "../../functions/live-schema"; -import { sqlQuoteIdent as q } from "../../utils/sql-safe"; - -// Schema Manager Class -class SQLiteSchemaManager { - private db: Database; - private db_manager_table_name: string; - private recreate_vector_table: boolean; - private db_schema: BUN_SQLITE_DatabaseSchemaType; - - constructor({ - schema, - recreate_vector_table = false, - }: { - schema: BUN_SQLITE_DatabaseSchemaType; - recreate_vector_table?: boolean; - }) { - this.db = DbClient; - this.db_manager_table_name = AppData["DbSchemaManagerTableName"]; - this.db.run("PRAGMA foreign_keys = ON;"); - this.recreate_vector_table = recreate_vector_table; - this.createDbManagerTable(); - this.db_schema = schema; - } - - private createDbManagerTable() { - this.db.run(` - CREATE TABLE IF NOT EXISTS ${this.db_manager_table_name} ( - table_name TEXT NOT NULL, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL - ) - `); - } - - private insertDbManagerTable(tableName: string) { - this.db.run( - `INSERT INTO ${this.db_manager_table_name} (table_name,created_at,updated_at) VALUES (?, ?, ?)`, - [tableName, Date.now(), Date.now()], - ); - } - - private removeDbManagerTable(tableName: string) { - this.db.run( - `DELETE FROM ${this.db_manager_table_name} WHERE table_name = ?`, - [tableName], - ); - } - - /** - * Main synchronization method - */ - async syncSchema(): Promise { - console.log("Starting schema synchronization..."); - - /** - * Tables are rebuilt by copying into a new table and dropping the - * old one. With foreign keys ON, `DROP TABLE` runs an implicit - * `DELETE FROM` that fires `ON DELETE CASCADE` on child tables, so - * keys are disabled for the sync and checked before committing. - * (`PRAGMA foreign_keys` is a no-op inside a transaction.) - */ - this.db.run("PRAGMA foreign_keys = OFF;"); - this.db.run("BEGIN"); - - try { - const existingTables = this.getExistingTables(); - const schemaTables = this.db_schema.tables.map((t) => t.tableName); - - // 1. Create or update tables - for (const table of this.db_schema.tables) { - await this.syncTable(table, existingTables); - } - - // 2. Drop tables that no longer exist in schema - await this.dropRemovedTables(existingTables, schemaTables); - - const fkViolations = this.db - .query(`PRAGMA foreign_key_check`) - .all() as { table: string; parent: string }[]; - - if (fkViolations.length) { - const tables = _.uniq( - fkViolations.map((v) => `${v.table} -> ${v.parent}`), - ); - throw new Error( - `Foreign key violations after schema sync: ${tables.join(", ")}`, - ); - } - - this.db.run("COMMIT"); - } catch (error) { - if (this.db.inTransaction) this.db.run("ROLLBACK"); - throw error; - } finally { - this.db.run("PRAGMA foreign_keys = ON;"); - } - - console.log("Schema synchronization complete!"); - } - - /** - * Get list of existing tables in the database - */ - private getExistingTables(): string[] { - let sql = `SELECT table_name FROM ${this.db_manager_table_name}`; - - const query = this.db.query(sql); - const results = query.all() as { table_name: string }[]; - - return results.map((r) => r.table_name); - } - - /** - * Drop tables that are no longer in the schema - */ - private async dropRemovedTables( - existingTables: string[], - schemaTables: string[], - ): Promise { - console.log(`Cleaning up tables ...`); - - const tablesToDrop = existingTables.filter( - (t) => - !schemaTables.includes(t) && - !schemaTables.find((scT) => t.startsWith(scT + "_")), - ); - - const currentSchema = readLiveSchema(); - - if (currentSchema?.tables?.[0]) { - for (let i = 0; i < currentSchema.tables.length; i++) { - const table = currentSchema.tables[i]; - - if (!table?.tableName) continue; - - const does_table_exist = schemaTables.find( - (t) => t == table.tableName, - ); - - if (!does_table_exist) { - tablesToDrop.push(table.tableName); - } - } - } - - for (const tableName of tablesToDrop) { - console.log(`Dropping table: ${tableName}`); - this.db.run(`DROP TABLE IF EXISTS ${q(tableName)}`); - this.removeDbManagerTable(tableName); - } - } - - /** - * Sync a single table (create or update) - */ - private async syncTable( - table: BUN_SQLITE_TableSchemaType, - existingTables: string[], - ): Promise { - let tableExists = existingTables.includes(table.tableName); - - // Handle table rename - if (table.tableNameOld && table.tableNameOld !== table.tableName) { - if (existingTables.includes(table.tableNameOld)) { - console.log( - `Renaming table: ${table.tableNameOld} -> ${table.tableName}`, - ); - this.db.run( - `ALTER TABLE ${q(table.tableNameOld)} RENAME TO ${q(table.tableName)}`, - ); - this.insertDbManagerTable(table.tableName); - this.removeDbManagerTable(table.tableNameOld); - tableExists = true; - } - } - - if (!tableExists) { - // Create new table - await this.createTable(table); - this.insertDbManagerTable(table.tableName); - } else { - // Update existing table - await this.updateTable(table); - } - - // Sync indexes - await this.syncIndexes(table); - } - - /** - * Create a new table - */ - private async createTable( - table: BUN_SQLITE_TableSchemaType, - ): Promise { - if (!table.tableName.match(/_temp_\d+$/)) { - console.log(`Creating table: ${table.tableName}`); - } - - const new_table = this.resolveTable(table); - - const columns: string[] = []; - const foreignKeys: string[] = []; - - for (const field of new_table.fields) { - const columnDef = this.buildColumnDefinition( - field, - new_table.isVector, - ); - columns.push(columnDef); - - if (field.foreignKey && !new_table.isVector) { - foreignKeys.push(this.buildForeignKeyConstraint(field)); - } - } - - // Add unique constraints - if (new_table.uniqueConstraints) { - for (const constraint of new_table.uniqueConstraints) { - if ( - constraint.constraintTableFields && - constraint.constraintTableFields.length > 0 - ) { - const fieldNames = constraint.constraintTableFields.map( - (f) => String(f.value), - ); - const constraintName = - constraint.constraintName || - `unique_${fieldNames.join(", ")}`; - columns.push( - `CONSTRAINT ${q(constraintName)} UNIQUE (${fieldNames - .map(q) - .join(", ")})`, - ); - } - } - } - - const allConstraints = [...columns, ...foreignKeys]; - - const sql = new_table.isVector - ? `CREATE VIRTUAL TABLE ${q(new_table.tableName)} USING ${new_table.vectorType || "vec0"}(${allConstraints.join(", ")})` - : `CREATE TABLE ${q(new_table.tableName)} (${allConstraints.join(", ")})`; - - this.db.run(sql); - } - - /** - * Resolve a table's effective definition. Tables with - * `parentTableName` take their structure from the parent. Always - * returns a copy: the schema objects are never mutated. - */ - private resolveTable( - table: BUN_SQLITE_TableSchemaType, - ): BUN_SQLITE_TableSchemaType { - if (!table.parentTableName) { - return _.cloneDeep(table); - } - - const parent_table = this.db_schema.tables.find( - (t) => t.tableName === table.parentTableName, - ); - - if (!parent_table) { - throw new Error( - `Parent table \`${table.parentTableName}\` not found for \`${table.tableName}\``, - ); - } - - return _.merge(_.cloneDeep(parent_table), { - tableName: table.tableName, - tableDescription: table.tableDescription, - }); - } - - /** - * Update an existing table - */ - private async updateTable( - table: BUN_SQLITE_TableSchemaType, - ): Promise { - console.log(`Updating table: ${table.tableName}`); - - // const existingColumns = this.getTableColumns(table.tableName); - // const schemaColumns = table.fields.map((f) => f.fieldName || ""); - - // // SQLite has limited ALTER TABLE support - // // We need to use the recreation strategy for complex changes - - // const columnsToAdd = table.fields.filter( - // (f) => - // f.fieldName && - // !existingColumns.find( - // (c) => - // c.name == f.fieldName && c.type == this.mapDataType(f), - // ), - // ); - - // // const columnsToRemove = existingColumns.filter( - // // (c) => !schemaColumns.includes(c.name), - // // ); - - // // const columnsToUpdate = table.fields.filter( - // // (f) => - // // f.fieldName && - // // f.updatedField && - // // existingColumns.find( - // // (c) => - // // c.name == f.fieldName && c.type == this.mapDataType(f), - // // ), - // // ); - - // for (const field of columnsToAdd) { - // await this.addColumn(table.tableName, field); - // } - - // // // Simple case: only adding columns - // // if (columnsToRemove.length === 0 && columnsToUpdate.length === 0) { - // // for (const field of columnsToAdd) { - // // await this.addColumn(table.tableName, field); - // // } - // // } else { - // // // Complex case: need to recreate table - // // } - - await this.recreateTable(table); - } - - /** - * Get existing columns for a table - */ - private getTableColumns( - tableName: string, - ): { name: string; type: string }[] { - const query = this.db.query(`PRAGMA table_info(${q(tableName)})`); - const results = query.all() as { name: string; type: string }[]; - return results; - } - - /** - * Add a new column to existing table - */ - private async addColumn( - tableName: string, - field: BUN_SQLITE_FieldSchemaType, - ): Promise { - console.log(`Adding column: ${tableName}.${field.fieldName}`); - - const columnDef = this.buildColumnDefinition(field); - // Remove PRIMARY KEY and UNIQUE constraints for ALTER TABLE ADD COLUMN - const cleanDef = columnDef - .replace(/PRIMARY KEY/gi, "") - .replace(/AUTOINCREMENT/gi, "") - .replace(/UNIQUE/gi, "") - .trim(); - - const sql = `ALTER TABLE ${q(tableName)} ADD COLUMN ${cleanDef}`; - - this.db.run(sql); - } - - private checkIfTableExists(table: string) { - const tableExists = this.db - .query( - `SELECT name FROM sqlite_master WHERE type='table' AND name=?`, - ) - .get(table) as any; - - return Boolean(tableExists?.name); - } - - /** - * Recreate table (for complex schema changes) - */ - private async recreateTable( - table: BUN_SQLITE_TableSchemaType, - ): Promise { - if (table.isVector) { - if (!this.recreate_vector_table) { - return; - } - - const does_table_exist = this.checkIfTableExists(table.tableName); - - let existingRows: { [k: string]: any }[] = []; - - if (does_table_exist) { - existingRows = this.db - .query(`SELECT * FROM ${q(table.tableName)}`) - .all() as { [k: string]: any }[]; - - this.db.run(`DROP TABLE ${q(table.tableName)}`); - } - - await this.createTable(table); - - if (existingRows.length > 0) { - for (let i = 0; i < existingRows.length; i++) { - const row = existingRows[i]; - if (!row) continue; - - const columns = Object.keys(row); - const placeholders = columns.map(() => "?").join(", "); - - this.db.run( - `INSERT INTO ${q(table.tableName)} (${columns.map(q).join(", ")}) VALUES (${placeholders})`, - Object.values(row), - ); - } - } - - return; - } - - const tempTableName = `${table.tableName}_temp_${Date.now()}`; - - // Get existing data. Columns are matched by name only: SQLite - // converts values to the new column affinity on copy, so a type - // change keeps the data instead of silently dropping the column. - const existingColumns = this.getTableColumns(table.tableName); - const columnsToKeep = this.resolveTable(table) - .fields.filter( - (f) => - f.fieldName && - existingColumns.find((c) => c.name == f.fieldName), - ) - .map((f) => f.fieldName as string); - - // Create temp table with new schema - const tempTable = { ...table, tableName: tempTableName }; - await this.createTable(tempTable); - - // Copy data if there are common columns - if (columnsToKeep.length > 0) { - const columnList = columnsToKeep.map(q).join(", "); - this.db.run( - `INSERT INTO ${q(tempTableName)} (${columnList}) SELECT ${columnList} FROM ${q(table.tableName)}`, - ); - } - - // Drop old table - this.db.run(`DROP TABLE ${q(table.tableName)}`); - - // Rename temp table - this.db.run( - `ALTER TABLE ${q(tempTableName)} RENAME TO ${q(table.tableName)}`, - ); - } - - /** - * Build column definition SQL - */ - private buildColumnDefinition( - field: BUN_SQLITE_FieldSchemaType, - is_vector?: boolean, - ): string { - if (!field.fieldName) { - throw new Error("Field name is required"); - } - - // vec0 can't parse quoted column names - const fieldName = is_vector - ? field.sideCar - ? `+${field.fieldName}` - : `${field.fieldName}` - : q(field.fieldName); - - const parts: string[] = [fieldName]; - - // Data type mapping - const dataType = this.mapDataType(field); - parts.push(dataType); - - // Primary key - if (field.primaryKey) { - parts.push("PRIMARY KEY"); - if (field.autoIncrement) { - parts.push("AUTOINCREMENT"); - } - } - - // Not null - if (field.notNullValue || field.primaryKey) { - if (!field.primaryKey) { - parts.push("NOT NULL"); - } - } - - // Unique - if (field.unique && !field.primaryKey) { - parts.push("UNIQUE"); - } - - if (field.vectorDistanceMetric) { - parts.push(`distance_metric=${field.vectorDistanceMetric}`); - } - - // Default value - if (field.defaultValue !== undefined) { - if (typeof field.defaultValue === "string") { - parts.push( - // Escape single quotes by doubling them to prevent SQL injection and wrap in single quotes - `DEFAULT '${field.defaultValue.replace(/'/g, "''")}'`, - ); - } else { - parts.push(`DEFAULT ${field.defaultValue}`); - } - } else if (field.defaultValueLiteral) { - parts.push(`DEFAULT ${field.defaultValueLiteral}`); - } - - return parts.join(" "); - } - - /** - * Map DSQL data types to SQLite types - */ - private mapDataType( - field: BUN_SQLITE_FieldSchemaType, - ): (typeof BUN_SQLITE_DATATYPES)[number]["value"] { - const dataType = field.dataType?.toLowerCase() || "text"; - const vectorSize = field.vectorSize || 1536; - - // Vector Embeddings - if (field.isVector) { - return `FLOAT[${vectorSize}]` as any; - } - - // Integer types - if ( - dataType.includes("int") || - dataType === "bigint" || - dataType === "smallint" || - dataType === "tinyint" - ) { - return "INTEGER"; - } - - // Real/Float types - if ( - dataType.includes("real") || - dataType.includes("float") || - dataType.includes("double") || - dataType === "decimal" || - dataType === "numeric" - ) { - return "REAL"; - } - - // Blob types - if (dataType.includes("blob") || dataType.includes("binary")) { - return "BLOB"; - } - - // Boolean - if (dataType === "boolean" || dataType === "bool") { - return "INTEGER"; // SQLite uses INTEGER for boolean (0/1) - } - - // Date/Time types - if (dataType.includes("date") || dataType.includes("time")) { - return "TEXT"; // SQLite stores dates as TEXT or INTEGER - } - - // Default to TEXT for all text-based types - return "TEXT"; - } - - /** - * Build foreign key constraint - */ - private buildForeignKeyConstraint( - field: BUN_SQLITE_FieldSchemaType, - ): string { - const fk = field.foreignKey!; - let constraint = `FOREIGN KEY (${q(field.fieldName!)}) REFERENCES ${q(String(fk.destinationTableName))}(${q(String(fk.destinationTableColumnName))})`; - - if (fk.cascadeDelete) { - constraint += " ON DELETE CASCADE"; - } - - if (fk.cascadeUpdate) { - constraint += " ON UPDATE CASCADE"; - } - - return constraint; - } - - /** - * Sync indexes for a table - */ - private async syncIndexes( - table: BUN_SQLITE_TableSchemaType, - ): Promise { - if (!table.indexes || table.indexes.length === 0) { - return; - } - - // Get existing indexes - const query = this.db.query( - `SELECT name FROM sqlite_master WHERE type='index' AND tbl_name=? AND name NOT LIKE 'sqlite_%'`, - ); - const existingIndexes = ( - query.all(table.tableName) as { name: string }[] - ).map((r) => r.name); - - // Drop indexes not in schema - for (const indexName of existingIndexes) { - const stillExists = table.indexes.some( - (idx) => idx.indexName === indexName, - ); - if (!stillExists) { - console.log(`Dropping index: ${indexName}`); - this.db.run(`DROP INDEX IF EXISTS ${q(indexName)}`); - } - } - - // Create new indexes - for (const index of table.indexes) { - if ( - !index.indexName || - !index.indexTableFields || - index.indexTableFields.length === 0 - ) { - continue; - } - - if (!existingIndexes.includes(index.indexName)) { - console.log(`Creating index: ${index.indexName}`); - const fields = index.indexTableFields - .map((f) => q(String(f))) - .join(", "); - this.db.run( - `CREATE INDEX ${q(index.indexName)} ON ${q(table.tableName)} (${fields})`, - ); - } - } - } - - /** - * Close database connection - */ - close(): void { - this.db.close(); - } -} - -export { SQLiteSchemaManager }; diff --git a/src/lib/sqlite/db-select.ts b/src/lib/sqlite/db-select.ts index cd2b6e1..0f44f3d 100644 --- a/src/lib/sqlite/db-select.ts +++ b/src/lib/sqlite/db-select.ts @@ -1,6 +1,10 @@ -import DbClient from "."; +import dbHandler from "../db-handler"; import _ from "lodash"; -import type { APIResponseObject, ServerQueryParam } from "../../types"; +import type { + APIResponseObject, + BunSQLiteConfig, + ServerQueryParam, +} from "../../types"; import sqlGenerator from "../../utils/sql-generator"; type Params< @@ -11,6 +15,7 @@ type Params< table: Table; count?: boolean; targetId?: number | string; + config?: BunSQLiteConfig; }; export default async function DbSelect< @@ -21,6 +26,7 @@ export default async function DbSelect< query, count, targetId, + config, }: Params): Promise> { let sqlObj: ReturnType | null = null; @@ -48,9 +54,17 @@ export default async function DbSelect< const sql = sqlObj.string; - const batchRes = DbClient.query(sql).all( - ...sqlObj.values, - ); + const res = dbHandler({ + query: sql, + values: sqlObj.values, + config, + }); + + if (!res.success) { + return { ...res, debug: { sqlObj, sql } }; + } + + const batchRes = res.payload || []; let resp: APIResponseObject = { success: Boolean(batchRes[0]), @@ -71,11 +85,23 @@ export default async function DbSelect< const count_sql = `SELECT COUNT(*) AS count FROM (${count_sql_object.string}) AS c`; - const count_res = DbClient.query<{ count: number }, any[]>( - count_sql, - ).get(...count_sql_object.values); + const count_res = dbHandler<{ count: number }>({ + query: count_sql, + values: count_sql_object.values, + config, + }); - resp["count"] = Number(count_res?.count); + if (!count_res.success) { + return { + ...resp, + success: false, + error: count_res.error, + msg: "Database count failed", + debug: { ...resp.debug, count_sql }, + }; + } + + resp["count"] = Number(count_res.singleRes?.count); resp["debug"]["count_sql"] = count_sql; } diff --git a/src/lib/sqlite/db-sql.ts b/src/lib/sqlite/db-sql.ts index e082d68..d1ea074 100644 --- a/src/lib/sqlite/db-sql.ts +++ b/src/lib/sqlite/db-sql.ts @@ -1,54 +1,35 @@ -import DbClient from "."; -import _ from "lodash"; -import type { APIResponseObject, SQLInsertGenValueType } from "../../types"; +import dbHandler from "../db-handler"; +import type { + APIResponseObject, + BunSQLiteConfig, + SQLInsertGenValueType, +} from "../../types"; type Params = { sql: string; values?: SQLInsertGenValueType[]; + config?: BunSQLiteConfig; }; export default async function DbSQL< T extends { [k: string]: any } = { [k: string]: any }, ->({ sql, values }: Params): Promise> { - try { - const trimmed_sql = sql.trim(); +>({ sql, values, config }: Params): Promise> { + const trimmed_sql = sql.trim(); - /** - * Any statement that produces columns (SELECT, WITH, PRAGMA, - * RETURNING, ...) returns rows. Everything else goes through `run`, - * which also handles multiple statements. - */ - const stmt = DbClient.prepare(trimmed_sql); - const returnsRows = stmt.columnNames.length > 0; + const res = dbHandler({ query: trimmed_sql, values, config }); - const res = returnsRows - ? stmt.all(...(values || [])) - : DbClient.run(trimmed_sql, values || []); - - stmt.finalize(); - - return { - success: true, - payload: Array.isArray(res) ? (res as T[]) : undefined, - singleRes: Array.isArray(res) ? (res as T[])?.[0] : undefined, - postInsertReturn: Array.isArray(res) - ? undefined - : { - affectedRows: res.changes, - insertId: Number(res.lastInsertRowid), - }, - debug: { - sqlObj: { - sql: trimmed_sql, - values, - }, - sql, - }, - }; - } catch (error: any) { - return { - success: false, - error: error.message, - }; + if (!res.success) { + return res; } + + return { + ...res, + debug: { + sqlObj: { + sql: trimmed_sql, + values, + }, + sql, + }, + }; } diff --git a/src/lib/sqlite/db-update.ts b/src/lib/sqlite/db-update.ts index 92866d9..ff61b10 100644 --- a/src/lib/sqlite/db-update.ts +++ b/src/lib/sqlite/db-update.ts @@ -1,7 +1,8 @@ -import DbClient from "."; +import dbHandler from "../db-handler"; import _ from "lodash"; import type { APIResponseObject, + BunSQLiteConfig, SQLInsertGenValueType, ServerQueryParam, } from "../../types"; @@ -17,6 +18,7 @@ type Params< data: Schema; query?: ServerQueryParam; targetId?: number | string; + config?: BunSQLiteConfig; }; export default async function DbUpdate< @@ -27,6 +29,7 @@ export default async function DbUpdate< data, query, targetId, + config, }: Params): Promise { let sqlObj: Pick, "string" | "values"> = { string: "", @@ -89,14 +92,15 @@ export default async function DbUpdate< sqlObj.string = sql; sqlObj.values = values as any[]; - const res = DbClient.run(sql, values); + const res = dbHandler({ query: sql, values, config }); + + if (!res.success) { + return { ...res, debug: { sqlObj } }; + } return { - success: Boolean(res.changes), - postInsertReturn: { - affectedRows: res.changes, - insertId: Number(res.lastInsertRowid), - }, + success: Boolean(res.postInsertReturn?.affectedRows), + postInsertReturn: res.postInsertReturn, debug: { sqlObj, }, diff --git a/src/lib/sqlite/index.ts b/src/lib/sqlite/index.ts deleted file mode 100644 index 7500549..0000000 --- a/src/lib/sqlite/index.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { Database } from "bun:sqlite"; -import * as sqliteVec from "sqlite-vec"; -import grabDirNames from "../../data/grab-dir-names"; -import init from "../../functions/init"; -import grabDBDir from "../../utils/grab-db-dir"; - -const { ROOT_DIR } = grabDirNames(); -const { config } = await init(); - -let db_dir = ROOT_DIR; - -if (config.db_dir) { - db_dir = config.db_dir; -} - -const { db_file_path } = grabDBDir({ config }); - -const DbClient = new Database(db_file_path, { - create: true, -}); - -sqliteVec.load(DbClient); - -if (config.wal_mode) { - DbClient.run("PRAGMA journal_mode = WAL;"); -} - -export default DbClient; diff --git a/src/lib/sqlite/schema.ts b/src/lib/sqlite/schema.ts deleted file mode 100644 index 7a12a02..0000000 --- a/src/lib/sqlite/schema.ts +++ /dev/null @@ -1,7 +0,0 @@ -import _ from "lodash"; -import type { BUN_SQLITE_DatabaseSchemaType } from "../../types"; - -export const DbSchema: BUN_SQLITE_DatabaseSchemaType = { - dbName: "travis-ai", - tables: [], -}; diff --git a/src/types/index.ts b/src/types/index.ts index c4f18c2..eecbdc9 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1537,14 +1537,19 @@ export type BunSQLiteConfig = { db_name: string; /** * The Name of the Database Schema File. Eg `db_schema.ts`. This is - * relative to `db_dir`, or root dir if `db_dir` is not provided + * relative to `db_dir`, or root dir if `db_dir` is not provided. When + * omitted, `schema.ts|js|json|yaml|yml` in `db_dir` is used. */ - db_schema_file_name: string; + db_schema_file_name?: string; /** * The Directory for backups. Relative to db_dir. */ db_backup_dir?: string; max_backups?: number; + /** + * Max number of archives kept in `{db_dir}/.exports` by `export` + */ + max_exports?: number; /** * The Root Directory for the DB file and schema */ @@ -1568,6 +1573,15 @@ export type BunSQLiteConfigReturn = { dbSchema: BUN_SQLITE_DatabaseSchemaType; }; +/** + * Params shared by the schema sync functions in `lib/schema`. + */ +export type CreateDBSchemaParams = { + db: import("bun:sqlite").Database; + db_schema: BUN_SQLITE_DatabaseSchemaType; + recreate_vector_table?: boolean; +}; + /** * Default fields automatically suggested for new tables. */ diff --git a/src/utils/export-archive.ts b/src/utils/export-archive.ts new file mode 100644 index 0000000..09083bb --- /dev/null +++ b/src/utils/export-archive.ts @@ -0,0 +1,253 @@ +import fs from "fs"; +import path from "path"; +import { AppData } from "../data/app-data"; +import grabDirNames from "../data/grab-dir-names"; +import { isSchemaFileName } from "./resolve-and-load-data-file"; + +export const ExportArchiveMembers = { + DbFileName: "database.sqlite", + SchemaFileName: AppData.DbSchemaFileName, +} as const; + +export type ExportArchiveContents = { + db: Uint8Array; + schema: string; + /** Archive member basename, e.g. schema.ts / schema.json / schema.yaml */ + schemaFileName: string; +}; + +const ARCHIVE_EXTENSIONS = [".tar.gz", ".tgz", ".tar", ".zip"] as const; + +/** Every SQLite database file starts with this header */ +const SQLITE_HEADER = "SQLite format 3\0"; + +export function isArchivePath(filePath: string): boolean { + const lower = filePath.toLowerCase(); + return ARCHIVE_EXTENSIONS.some((ext) => lower.endsWith(ext)); +} + +export function isSqlPath(filePath: string): boolean { + return filePath.toLowerCase().endsWith(".sql"); +} + +export function isSQLiteFile(bytes: Uint8Array): boolean { + return ( + new TextDecoder().decode(bytes.subarray(0, SQLITE_HEADER.length)) === + SQLITE_HEADER + ); +} + +/** + * Create a portable export archive (tar / tar.gz / zip) with the database + * file + schema file. + */ +export async function writeExportArchive({ + contents, + outPath, +}: { + contents: ExportArchiveContents; + outPath: string; +}): Promise { + const lower = outPath.toLowerCase(); + if (lower.endsWith(".zip")) { + await writeZipArchive({ contents, outPath }); + return; + } + + const members = { + [ExportArchiveMembers.DbFileName]: contents.db, + [contents.schemaFileName]: contents.schema, + }; + + const gzip = lower.endsWith(".gz") || lower.endsWith(".tgz"); + if (gzip) { + await Bun.Archive.write(outPath, members, { compress: "gzip" }); + } else { + await Bun.Archive.write(outPath, members); + } +} + +/** + * Read a portable export archive (tar / tar.gz / zip). + */ +export async function readExportArchive( + archivePath: string, +): Promise { + if (archivePath.toLowerCase().endsWith(".zip")) { + return readZipArchive(archivePath); + } + + const bytes = await Bun.file(archivePath).bytes(); + const archive = new Bun.Archive(bytes); + const files = await archive.files(); + + let db: Uint8Array | null = null; + let schemaEntry: { content: string; fileName: string } | null = null; + + for (const [entry, file] of files) { + const base = path.basename(entry); + + if (!db && base === ExportArchiveMembers.DbFileName) { + db = await file.bytes(); + } else if (!schemaEntry && isSchemaFileName(base)) { + schemaEntry = { + content: await file.text(), + fileName: schemaMemberName(base), + }; + } + } + + return validateContents({ db, schemaEntry }); +} + +function schemaMemberName(base: string) { + return base === AppData.DbSchemaFileName ? "schema.ts" : base; +} + +function validateContents({ + db, + schemaEntry, +}: { + db: Uint8Array | null; + schemaEntry: { content: string; fileName: string } | null; +}): ExportArchiveContents { + if (!db) { + throw new Error( + `Archive is missing the database file (expected \`${ExportArchiveMembers.DbFileName}\`)`, + ); + } + + if (!isSQLiteFile(db)) { + throw new Error( + `\`${ExportArchiveMembers.DbFileName}\` in the archive is not a SQLite database`, + ); + } + + if (!schemaEntry) { + throw new Error( + `Archive is missing schema file (expected \`${AppData.DbSchemaFileName}.ts|js|json|yaml|yml\`)`, + ); + } + + return { + db, + schema: schemaEntry.content, + schemaFileName: schemaEntry.fileName, + }; +} + +async function runCommand(cmd: string[], cwd?: string) { + const proc = Bun.spawn(cmd, { + cwd, + stdout: "pipe", + stderr: "pipe", + }); + + const [stderr, exitCode] = await Promise.all([ + new Response(proc.stderr).text(), + proc.exited, + ]); + + if (exitCode !== 0) { + throw new Error( + `${cmd[0]} failed (exit ${exitCode}): ${stderr || "unknown error"}. Ensure \`${cmd[0]}\` is installed, or use .tar.gz.`, + ); + } +} + +function makeTempDir(prefix: string) { + const { BUN_SQLITE_TEMP_DIR } = grabDirNames(); + const tempDir = path.join( + BUN_SQLITE_TEMP_DIR, + `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2)}`, + ); + + fs.mkdirSync(tempDir, { recursive: true }); + return tempDir; +} + +async function writeZipArchive({ + contents, + outPath, +}: { + contents: ExportArchiveContents; + outPath: string; +}): Promise { + const tempDir = makeTempDir("export"); + + try { + fs.writeFileSync( + path.join(tempDir, ExportArchiveMembers.DbFileName), + contents.db, + ); + fs.writeFileSync( + path.join(tempDir, contents.schemaFileName), + contents.schema, + "utf-8", + ); + + await runCommand( + [ + "zip", + "-q", + "-j", + path.resolve(outPath), + ExportArchiveMembers.DbFileName, + contents.schemaFileName, + ], + tempDir, + ); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} + +async function readZipArchive( + archivePath: string, +): Promise { + const tempDir = makeTempDir("import"); + + try { + await runCommand([ + "unzip", + "-q", + "-o", + path.resolve(archivePath), + "-d", + tempDir, + ]); + + let db: Uint8Array | null = null; + let schemaEntry: { content: string; fileName: string } | null = null; + + const stack = [tempDir]; + while (stack.length) { + const current = stack.pop()!; + for (const entry of fs.readdirSync(current, { + withFileTypes: true, + })) { + const full = path.join(current, entry.name); + // Never follow symlinks from an extracted (untrusted) archive + if (entry.isDirectory()) { + stack.push(full); + } else if (!entry.isFile()) { + continue; + } else if ( + !db && + entry.name === ExportArchiveMembers.DbFileName + ) { + db = new Uint8Array(fs.readFileSync(full)); + } else if (!schemaEntry && isSchemaFileName(entry.name)) { + schemaEntry = { + content: fs.readFileSync(full, "utf-8"), + fileName: schemaMemberName(entry.name), + }; + } + } + } + + return validateContents({ db, schemaEntry }); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} diff --git a/src/utils/grab-db-dir.ts b/src/utils/grab-db-dir.ts index b668157..3836a16 100644 --- a/src/utils/grab-db-dir.ts +++ b/src/utils/grab-db-dir.ts @@ -13,14 +13,15 @@ export default function grabDBDir({ config }: Params) { let db_dir = ROOT_DIR; if (config.db_dir) { - db_dir = config.db_dir; + db_dir = path.resolve(ROOT_DIR, config.db_dir); } const backup_dir_name = config.db_backup_dir || AppData["DefaultBackupDirName"]; const backup_dir = path.resolve(db_dir, backup_dir_name); + const export_dir = path.resolve(db_dir, AppData["DefaultExportDirName"]); const db_file_path = path.resolve(db_dir, config.db_name); - return { db_dir, backup_dir, db_file_path }; + return { db_dir, backup_dir, export_dir, db_file_path }; } diff --git a/src/utils/grab-db-schema.ts b/src/utils/grab-db-schema.ts index edb2b08..0cc8712 100644 --- a/src/utils/grab-db-schema.ts +++ b/src/utils/grab-db-schema.ts @@ -1,6 +1,6 @@ import init from "../functions/init"; export default async function grabDbSchema() { - const { dbSchema } = await init(); + const { dbSchema } = init(); return dbSchema; } diff --git a/src/utils/grab-sorted-exports.ts b/src/utils/grab-sorted-exports.ts new file mode 100644 index 0000000..1b69572 --- /dev/null +++ b/src/utils/grab-sorted-exports.ts @@ -0,0 +1,34 @@ +import fs from "fs"; +import type { BunSQLiteConfig } from "../types"; +import grabDBDir from "./grab-db-dir"; + +type Params = { + config: BunSQLiteConfig; +}; + +function stripExportExtension(name: string): string { + const lower = name.toLowerCase(); + if (lower.endsWith(".tar.gz")) return name.slice(0, -7); + if (lower.endsWith(".tgz")) return name.slice(0, -4); + if (lower.endsWith(".tar")) return name.slice(0, -4); + if (lower.endsWith(".zip")) return name.slice(0, -4); + return name; +} + +function exportTimestamp(name: string): number { + const base = stripExportExtension(name); + const ts = Number(base.split("-").pop()); + return Number.isFinite(ts) ? ts : 0; +} + +export default function grabSortedExports({ config }: Params) { + const { export_dir } = grabDBDir({ config }); + + if (!fs.existsSync(export_dir)) { + return [] as string[]; + } + + const exports = fs.readdirSync(export_dir); + + return exports.sort((a, b) => exportTimestamp(b) - exportTimestamp(a)); +} diff --git a/src/utils/resolve-and-load-data-file.ts b/src/utils/resolve-and-load-data-file.ts new file mode 100644 index 0000000..6c558bf --- /dev/null +++ b/src/utils/resolve-and-load-data-file.ts @@ -0,0 +1,132 @@ +import fs from "fs"; +import path from "path"; +import { AppData } from "../data/app-data"; + +export type DataFileFormat = "ts" | "js" | "json" | "yaml"; + +export type ResolvedDataFile = { + path: string; + basename: string; + extension: (typeof AppData.SupportedDataFileExtensions)[number]; + format: DataFileFormat; +}; + +export type LoadedDataFile = ResolvedDataFile & { + data: T; +}; + +function extensionToFormat( + extension: (typeof AppData.SupportedDataFileExtensions)[number], +): DataFileFormat { + switch (extension) { + case ".ts": + return "ts"; + case ".js": + return "js"; + case ".json": + return "json"; + case ".yaml": + case ".yml": + return "yaml"; + } +} + +/** + * Resolve a basename (no extension) to an existing file among supported formats. + * Priority: .ts > .js > .json > .yaml > .yml + * Errors if multiple matches exist. + */ +export function resolveDataFile( + dir: string, + baseName: string, +): ResolvedDataFile | null { + const matches: ResolvedDataFile[] = []; + + for (const extension of AppData.SupportedDataFileExtensions) { + const filePath = path.join(dir, `${baseName}${extension}`); + if (fs.existsSync(filePath) && fs.statSync(filePath).isFile()) { + matches.push({ + path: filePath, + basename: `${baseName}${extension}`, + extension, + format: extensionToFormat(extension), + }); + } + } + + if (matches.length === 0) { + return null; + } + + if (matches.length > 1) { + const found = matches.map((m) => m.basename).join(", "); + throw new Error( + `Multiple \`${baseName}\` files found (${found}). Keep only one of: ${AppData.SupportedDataFileExtensions.join(", ")}`, + ); + } + + return matches[0]!; +} + +export function supportedDataFileNames(baseName: string): string { + return AppData.SupportedDataFileExtensions.map( + (ext) => `\`${baseName}${ext}\``, + ).join(", "); +} + +export function isSchemaFileName(name: string): boolean { + const base = path.basename(name); + if (base === AppData.DbSchemaFileName) { + return true; + } + return AppData.SupportedDataFileExtensions.some( + (ext) => base === `${AppData.DbSchemaFileName}${ext}`, + ); +} + +/** + * Load a data file as a plain object. + * - .ts / .js: require() and use default export (or module itself) + * - .json: JSON.parse + * - .yaml / .yml: Bun.YAML.parse + */ +export function loadDataFile(resolved: ResolvedDataFile): T { + if (resolved.format === "ts" || resolved.format === "js") { + const imported = require(resolved.path); + const data = + imported && typeof imported === "object" && "default" in imported + ? imported.default + : imported; + + if (data == null) { + throw new Error( + `No default export from \`${resolved.path}\`. Please export a default module.`, + ); + } + + return data as T; + } + + const text = fs.readFileSync(resolved.path, "utf-8"); + + if (resolved.format === "json") { + return JSON.parse(text) as T; + } + + return Bun.YAML.parse(text) as T; +} + +export function resolveAndLoadDataFile( + dir: string, + baseName: string, +): LoadedDataFile | null { + const resolved = resolveDataFile(dir, baseName); + if (!resolved) { + return null; + } + + return { + ...resolved, + data: loadDataFile(resolved), + }; +} diff --git a/src/utils/trim-exports.ts b/src/utils/trim-exports.ts new file mode 100644 index 0000000..b38dc99 --- /dev/null +++ b/src/utils/trim-exports.ts @@ -0,0 +1,34 @@ +import fs from "fs"; +import path from "path"; +import _ from "lodash"; +import type { BunSQLiteConfig } from "../types"; +import { AppData } from "../data/app-data"; +import grabDBDir from "./grab-db-dir"; +import grabSortedExports from "./grab-sorted-exports"; + +type Params = { + config: BunSQLiteConfig; +}; + +export default function trimExports({ config }: Params) { + const { export_dir } = grabDBDir({ config }); + // Only consider archives written by `export` (`{db_name}-{timestamp}.tar.gz|zip`) + const exportNamePattern = new RegExp( + `^${_.escapeRegExp(config.db_name)}-\\d+\\.(tar\\.gz|zip)$`, + ); + + const exports = grabSortedExports({ config }).filter( + (name) => + exportNamePattern.test(name) && + fs.statSync(path.join(export_dir, name)).isFile(), + ); + const max_exports = config.max_exports || AppData["MaxExports"]; + + for (let i = 0; i < exports.length; i++) { + const export_name = exports[i]; + if (!export_name) continue; + if (i > max_exports - 1) { + fs.unlinkSync(path.join(export_dir, export_name)); + } + } +} diff --git a/tsconfig.json b/tsconfig.json index 2b0f895..2c745f4 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -21,8 +21,9 @@ "maxNodeModuleJsDepth": 10, "forceConsistentCasingInFileNames": true, "incremental": true, + "rootDir": "src", "outDir": "dist" }, "include": ["src"], - "exclude": ["node_modules", "dist"] + "exclude": ["node_modules", "dist", "src/**/*.test.ts"] }