diff --git a/dist/commands/admin/index.js b/dist/commands/admin/index.js index e3eac2e..24bfa76 100644 --- a/dist/commands/admin/index.js +++ b/dist/commands/admin/index.js @@ -1,20 +1,13 @@ import { Command } from "commander"; -import init from "../../functions/init"; -import grabDBDir from "../../utils/grab-db-dir"; import chalk from "chalk"; import { select } from "@inquirer/prompts"; -import { Database } from "bun:sqlite"; import listTables from "./list-tables"; import runSQL from "./run-sql"; -import * as sqliteVec from "sqlite-vec"; export default function () { return new Command("admin") .description("View Tables and Data, Run SQL Queries, Etc.") .action(async () => { - const { config } = await init(); - const { db_file_path } = grabDBDir({ config }); - const db = new Database(db_file_path); - sqliteVec.load(db); + const db = global.SQLITE_CLIENT; console.log(chalk.bold(chalk.blue("\nBun SQLite Admin\n"))); try { while (true) { diff --git a/dist/commands/backup.js b/dist/commands/backup.js index 03c7d3c..f612566 100644 --- a/dist/commands/backup.js +++ b/dist/commands/backup.js @@ -1,20 +1,22 @@ import { Command } from "commander"; -import init from "../functions/init"; import path from "path"; import grabDBDir from "../utils/grab-db-dir"; -import fs from "fs"; import grabDBBackupFileName from "../utils/grab-db-backup-file-name"; import chalk from "chalk"; import trimBackups from "../utils/trim-backups"; +import snapshotDb from "../utils/snapshot-db"; export default function () { return new Command("backup") .description("Backup Database") .action(async (opts) => { console.log(`Backing up database ...`); - const { config } = await init(); - const { backup_dir, db_file_path } = grabDBDir({ config }); + const config = global.CONFIG; + const { backup_dir } = grabDBDir({ config }); const new_db_file_name = grabDBBackupFileName({ config }); - fs.cpSync(db_file_path, path.join(backup_dir, new_db_file_name)); + snapshotDb({ + db: global.SQLITE_CLIENT, + dst: path.join(backup_dir, new_db_file_name), + }); trimBackups({ config }); console.log(`${chalk.bold(chalk.green(`DB Backup Success!`))}`); process.exit(); diff --git a/dist/commands/export.d.ts b/dist/commands/export.d.ts new file mode 100644 index 0000000..f202e7d --- /dev/null +++ b/dist/commands/export.d.ts @@ -0,0 +1,2 @@ +import { Command } from "commander"; +export default function (): Command; diff --git a/dist/commands/export.js b/dist/commands/export.js new file mode 100644 index 0000000..ff5ffae --- /dev/null +++ b/dist/commands/export.js @@ -0,0 +1,70 @@ +import { Command } from "commander"; +import path from "path"; +import fs from "fs"; +import chalk from "chalk"; +import grabDBDir from "../utils/grab-db-dir"; +import grabDirNames from "../data/grab-dir-names"; +import snapshotDb from "../utils/snapshot-db"; +import { writeExportArchive } from "../utils/export-archive"; +import trimExports from "../utils/trim-exports"; +import { AppData } from "../data/app-data"; +function defaultExportFileName(dbName, format) { + return `${dbName}-${Date.now()}.${format}`; +} +export default function () { + return new Command("export") + .description("Export database file + schema into a portable archive") + .option("-o, --output ", "Output archive path (.tar.gz or .zip). Defaults to db exports dir") + .option("-f, --format ", "Archive format when --output is omitted: tar.gz | zip", "tar.gz") + .action(async (opts) => { + console.log(`Exporting database ...`); + const config = global.CONFIG; + const { export_dir } = grabDBDir({ config }); + const { BUN_SQLITE_TEMP_DIR } = grabDirNames(); + if (!fs.existsSync(export_dir)) { + fs.mkdirSync(export_dir, { recursive: true }); + } + const formatRaw = String(opts.format || "tar.gz").toLowerCase(); + const format = formatRaw === "zip" ? "zip" : "tar.gz"; + let outPath; + if (opts.output) { + outPath = path.resolve(opts.output); + fs.mkdirSync(path.dirname(outPath), { recursive: true }); + } + else { + outPath = path.join(export_dir, defaultExportFileName(config.db_name, format)); + } + const snapshot_path = path.join(BUN_SQLITE_TEMP_DIR, `export-${Date.now()}.db`); + /** Remove the temp database file before exiting */ + const exit = (code) => { + fs.rmSync(snapshot_path, { force: true }); + process.exit(code); + }; + try { + snapshotDb({ db: global.SQLITE_CLIENT, dst: snapshot_path }); + /** + * The schema is always stored as `schema.` so `import` + * can find it, whatever `db_schema_file_name` is. + */ + const schemaFileName = `${AppData.DbSchemaFileName}${path.extname(global.SCHEMA_FILE_PATH)}`; + await writeExportArchive({ + contents: { + db: new Uint8Array(fs.readFileSync(snapshot_path)), + schema: fs.readFileSync(global.SCHEMA_FILE_PATH, "utf-8"), + schemaFileName, + }, + outPath, + }); + if (path.dirname(outPath) === export_dir) { + trimExports({ config }); + } + console.log(`${chalk.bold(chalk.green(`DB Export Success!`))} → ${outPath}`); + console.log(chalk.dim(`Contains: database.sqlite + ${schemaFileName}`)); + exit(0); + } + catch (error) { + console.error(chalk.red(`Export ERROR => ${error.message}`)); + exit(1); + } + }); +} diff --git a/dist/commands/import.d.ts b/dist/commands/import.d.ts new file mode 100644 index 0000000..f202e7d --- /dev/null +++ b/dist/commands/import.d.ts @@ -0,0 +1,2 @@ +import { Command } from "commander"; +export default function (): Command; diff --git a/dist/commands/import.js b/dist/commands/import.js new file mode 100644 index 0000000..11f90c5 --- /dev/null +++ b/dist/commands/import.js @@ -0,0 +1,187 @@ +import { Command } from "commander"; +import path from "path"; +import fs from "fs"; +import chalk from "chalk"; +import { Database } from "bun:sqlite"; +import * as sqliteVec from "sqlite-vec"; +import { confirm, select } from "@inquirer/prompts"; +import grabDBDir from "../utils/grab-db-dir"; +import grabDirNames from "../data/grab-dir-names"; +import grabSortedExports from "../utils/grab-sorted-exports"; +import grabBackupData from "../utils/grab-backup-data"; +import grabDBBackupFileName from "../utils/grab-db-backup-file-name"; +import snapshotDb from "../utils/snapshot-db"; +import trimBackups from "../utils/trim-backups"; +import replaceDbFile from "../utils/replace-db-file"; +import { isArchivePath, isSQLiteFile, isSqlPath, readExportArchive, } from "../utils/export-archive"; +import { resolveDataFile } from "../utils/resolve-and-load-data-file"; +import { AppData } from "../data/app-data"; +function formatChoice(name, index) { + const { backup_date } = grabBackupData({ + backup_name: name.replace(/\.(tar\.gz|tgz|tar|zip)$/i, ""), + }); + const time = Number.isNaN(backup_date.getTime()) + ? "unknown date" + : `${backup_date.toDateString()} ${backup_date.getHours()}:${backup_date.getMinutes()}:${backup_date.getSeconds().toString().padStart(2, "0")}`; + return `#${index + 1}: ${name} (${time})`; +} +/** + * # Replace the live database with the database file at `src` + * @description The current database is backed up first. + */ +function swapDatabase(src) { + const config = global.CONFIG; + const { backup_dir, db_file_path } = grabDBDir({ config }); + const backup_path = path.join(backup_dir, grabDBBackupFileName({ config })); + snapshotDb({ db: global.SQLITE_CLIENT, dst: backup_path }); + trimBackups({ config }); + console.log(chalk.dim(`Backed up current database → ${backup_path}`)); + global.SQLITE_CLIENT.close(); + replaceDbFile({ src, db_file_path }); +} +/** + * # Build a database file from an SQL script + */ +function buildDbFromSql({ sql, dst }) { + fs.mkdirSync(path.dirname(dst), { recursive: true }); + fs.rmSync(dst, { force: true }); + const db = new Database(dst, { create: true }); + try { + sqliteVec.load(db); + db.run(sql); + } + finally { + db.close(); + } +} +/** + * # Write the schema file from an archive into the project + */ +function writeSchema({ schema, schemaFileName, }) { + const config = global.CONFIG; + const { db_dir } = grabDBDir({ config }); + if (config.db_schema_file_name) { + const schemaPath = path.join(db_dir, config.db_schema_file_name); + if (path.extname(schemaPath) !== path.extname(schemaFileName)) { + throw new Error(`Archive schema is \`${schemaFileName}\` but \`db_schema_file_name\` is \`${config.db_schema_file_name}\`. Use --sql-only, or change \`db_schema_file_name\`.`); + } + fs.writeFileSync(schemaPath, schema, "utf-8"); + console.log(chalk.green(`Schema written → ${schemaPath}`)); + return; + } + const existing = resolveDataFile(db_dir, AppData.DbSchemaFileName); + const schemaPath = path.join(db_dir, schemaFileName); + // Remove a differently-named schema so only one format remains + if (existing && path.resolve(existing.path) !== path.resolve(schemaPath)) { + fs.unlinkSync(existing.path); + console.log(chalk.dim(`Removed previous schema file: ${existing.basename}`)); + } + fs.writeFileSync(schemaPath, schema, "utf-8"); + console.log(chalk.green(`Schema written → ${schemaPath}`)); +} +export default function () { + return new Command("import") + .description("Import a SQLite database file, an SQL script, or a full export archive (database + schema)") + .argument("[file]", "Path to a SQLite file, .sql file, or export archive (.tar.gz / .tar / .zip)") + .option("--sql-only", "When importing an archive, restore the database only (skip writing schema)") + .option("--schema-only", "When importing an archive, write schema only (skip database restore)") + .option("-y, --yes", "Do not ask for confirmation before writing an executable (.ts/.js) schema from the archive") + .action(async (fileArg, opts) => { + console.log(`Importing database ...`); + const config = global.CONFIG; + const { db_dir, export_dir } = grabDBDir({ config }); + const { BUN_SQLITE_TEMP_DIR } = grabDirNames(); + const temp_db_path = path.join(BUN_SQLITE_TEMP_DIR, `import-${Date.now()}.db`); + /** Remove the temp database file before exiting */ + const exit = (code) => { + fs.rmSync(temp_db_path, { force: true }); + process.exit(code); + }; + try { + let filePath = fileArg ? path.resolve(fileArg) : ""; + if (!filePath) { + const candidates = grabSortedExports({ config }).filter((b) => isArchivePath(b)); + if (!candidates[0]) { + console.error(`No export archives found in \`${export_dir}\`. Use \`export\`, or pass a file path.`); + exit(1); + } + const selected = await select({ + message: "Select an export to import:", + choices: candidates.map((b, i) => ({ + name: formatChoice(b, i), + value: b, + })), + }); + filePath = path.join(export_dir, selected); + } + if (!fs.existsSync(filePath)) { + console.error(chalk.red(`File not found: ${filePath}`)); + exit(1); + } + if (opts.sqlOnly && opts.schemaOnly) { + console.error(chalk.red(`Cannot combine --sql-only and --schema-only.`)); + exit(1); + } + if (!isArchivePath(filePath)) { + if (opts.schemaOnly) { + console.error(chalk.red(`--schema-only requires an export archive.`)); + exit(1); + } + if (isSqlPath(filePath)) { + buildDbFromSql({ + sql: fs.readFileSync(filePath, "utf-8"), + dst: temp_db_path, + }); + swapDatabase(temp_db_path); + } + else if (isSQLiteFile(new Uint8Array(fs.readFileSync(filePath)))) { + swapDatabase(filePath); + } + else { + console.error(chalk.red(`Unsupported file: ${filePath}. Use a SQLite database file, .sql, .tar.gz, .tar, or .zip.`)); + exit(1); + } + console.log(`${chalk.bold(chalk.green(`DB Import Success!`))} ← ${filePath}`); + exit(0); + } + const { db, schema, schemaFileName } = await readExportArchive(filePath); + /** + * A .ts/.js schema is executed (require) on the next run. + * Confirm before writing code from an archive into the project. + */ + const isExecutableSchema = /\.(ts|js)$/i.test(schemaFileName); + if (!opts.sqlOnly && isExecutableSchema && !opts.yes) { + if (!process.stdin.isTTY) { + console.error(chalk.red(`Archive contains an executable schema (${schemaFileName}). Re-run with --yes to accept it, or --sql-only to skip it.`)); + exit(1); + } + console.log(chalk.yellow(`\nThis archive contains an executable schema file (${schemaFileName}).\nIt will run as code the next time bun-sqlite or your app loads it.\nOnly continue if you trust the source of this archive.\n`)); + const proceed = await confirm({ + message: `Write ${schemaFileName} into ${db_dir}?`, + default: false, + }); + if (!proceed) { + console.log(chalk.dim(`Import cancelled.`)); + exit(1); + } + } + if (!opts.sqlOnly) { + writeSchema({ schema, schemaFileName }); + } + if (!opts.schemaOnly) { + fs.mkdirSync(path.dirname(temp_db_path), { + recursive: true, + }); + fs.writeFileSync(temp_db_path, db); + swapDatabase(temp_db_path); + console.log(chalk.green(`Database restored from archive`)); + } + console.log(`${chalk.bold(chalk.green(`DB Import Success!`))} ← ${filePath}`); + exit(0); + } + catch (error) { + console.error(chalk.red(`Import ERROR => ${error.message}`)); + exit(1); + } + }); +} diff --git a/dist/commands/index.d.ts b/dist/commands/index.d.ts index 4cb206d..c1cf237 100644 --- a/dist/commands/index.d.ts +++ b/dist/commands/index.d.ts @@ -1,6 +1,2 @@ #!/usr/bin/env bun -/** - * # Declare Global Variables - */ -declare global { } export {}; diff --git a/dist/commands/index.js b/dist/commands/index.js old mode 100644 new mode 100755 index a00c16a..de75d22 --- a/dist/commands/index.js +++ b/dist/commands/index.js @@ -4,14 +4,27 @@ import schema from "./schema"; import typedef from "./typedef"; import backup from "./backup"; import restore from "./restore"; +import exportCmd from "./export"; +import importCmd from "./import"; import admin from "./admin"; +import init from "../functions/init"; +import { readFileSync } from "fs"; +try { + init(); +} +catch (error) { + console.error(`Initialization ERROR => ${error.message}`); + process.exit(1); +} +/** Resolves to the package root from both src/commands and dist/commands */ +const { version } = JSON.parse(readFileSync(new URL("../../package.json", import.meta.url), "utf-8")); /** * # Describe Program */ program .name(`bun-sqlite`) .description(`SQLite manager for Bun`) - .version(`1.0.0`); + .version(version); /** * # Declare Commands */ @@ -19,6 +32,8 @@ program.addCommand(schema()); program.addCommand(typedef()); program.addCommand(backup()); program.addCommand(restore()); +program.addCommand(exportCmd()); +program.addCommand(importCmd()); program.addCommand(admin()); /** * # Handle Unavailable Commands @@ -30,4 +45,7 @@ program.on("command:*", () => { /** * # Parse Arguments */ -program.parse(Bun.argv); +program.parseAsync(Bun.argv).catch((error) => { + console.error(`ERROR => ${error?.message || error}`); + process.exit(1); +}); diff --git a/dist/commands/restore.js b/dist/commands/restore.js index 99c4bfd..b511cb2 100644 --- a/dist/commands/restore.js +++ b/dist/commands/restore.js @@ -1,7 +1,6 @@ import { Command } from "commander"; -import init from "../functions/init"; import grabDBDir from "../utils/grab-db-dir"; -import fs from "fs"; +import replaceDbFile from "../utils/replace-db-file"; import chalk from "chalk"; import grabSortedBackups from "../utils/grab-sorted-backups"; import { select } from "@inquirer/prompts"; @@ -12,7 +11,7 @@ export default function () { .description("Restore Database") .action(async (opts) => { console.log(`Restoring up database ...`); - const { config } = await init(); + const config = global.CONFIG; const { backup_dir, db_file_path } = grabDBDir({ config }); const backups = grabSortedBackups({ config }); if (!backups?.[0]) { @@ -32,13 +31,17 @@ export default function () { }; }), }); - fs.cpSync(path.join(backup_dir, selected_backup), db_file_path); + global.SQLITE_CLIENT.close(); + replaceDbFile({ + src: path.join(backup_dir, selected_backup), + db_file_path, + }); console.log(`${chalk.bold(chalk.green(`DB Restore Success!`))}`); process.exit(); } catch (error) { console.error(`Backup Restore ERROR => ${error.message}`); - process.exit(); + process.exit(1); } }); } diff --git a/dist/commands/schema.js b/dist/commands/schema.js index 04a9236..0c29d00 100644 --- a/dist/commands/schema.js +++ b/dist/commands/schema.js @@ -1,38 +1,96 @@ import { Command } from "commander"; -import { SQLiteSchemaManager } from "../lib/sqlite/db-schema-manager"; -import init from "../functions/init"; +import createDBSchema from "../lib/schema/create-db-schema"; import grabDirNames from "../data/grab-dir-names"; import path from "path"; import dbSchemaToTypeDef from "../lib/sqlite/schema-to-typedef"; -import _ from "lodash"; import appendDefaultFieldsToDbSchema from "../utils/append-default-fields-to-db-schema"; import chalk from "chalk"; import { writeLiveSchema } from "../functions/live-schema"; import grabDBDir from "../utils/grab-db-dir"; -import { cpSync } from "fs"; +import snapshotDb from "../utils/snapshot-db"; +import replaceDbFile from "../utils/replace-db-file"; +import { confirm } from "@inquirer/prompts"; export default function () { return new Command("schema") .description("Build DB From Schema") .option("-v, --vector", "Recreate Vector Tables. This will drop and rebuild all vector tables") .option("-t, --typedef", "Generate typescript type definitions") + .option("--dry-run", "Print the schema changes that would be applied, without applying them") + .option("-y, --yes", "Apply destructive changes (dropped tables / columns) without confirmation") .action(async (opts) => { console.log(`Starting process ...`); - const { config, dbSchema } = await init(); + const config = global.CONFIG; + const dbSchema = global.DB_SCHEMA; + const DbClient = global.SQLITE_CLIENT; const { ROOT_DIR, BUN_SQLITE_TEMP_DB_FILE_PATH } = grabDirNames(); const { db_file_path } = grabDBDir({ config }); - cpSync(db_file_path, BUN_SQLITE_TEMP_DB_FILE_PATH); + const isVector = Boolean(opts.vector || opts.v); + const isTypeDef = Boolean(opts.typedef || opts.t); + const finaldbSchema = appendDefaultFieldsToDbSchema({ + dbSchema, + }); + const params = { + db: DbClient, + db_schema: finaldbSchema, + recreate_vector_table: isVector, + }; + /** + * Plan: run the sync in a transaction that is rolled back + */ + const plan = []; try { - const isVector = Boolean(opts.vector || opts.v); - const isTypeDef = Boolean(opts.typedef || opts.t); - const finaldbSchema = appendDefaultFieldsToDbSchema({ - dbSchema, + createDBSchema({ ...params, dry_run: true, changes: plan }); + } + catch (error) { + console.error(chalk.red(`Schema sync ERROR => ${error.message}`)); + process.exit(1); + } + const destructive = plan.filter((change) => change.destructive); + if (opts.dryRun) { + if (!plan.length) { + console.log(chalk.green(`Schema is up to date.`)); + } + else { + console.log(chalk.bold(`Planned schema changes:\n`)); + for (const change of plan) { + console.log(change.destructive + ? chalk.red(` ${change.description}`) + : ` ${change.description}`); + } + if (destructive.length) { + console.log(chalk.red(`\n${destructive.length} destructive change(s) (in red) would discard data.`)); + } + } + process.exit(0); + } + if (destructive.length && !opts.yes) { + console.log(chalk.yellow(`\nThese changes will permanently discard data:\n`)); + for (const change of destructive) { + console.log(chalk.red(` ${change.description}`)); + } + console.log(); + if (!process.stdin.isTTY) { + console.error(chalk.red(`Refusing to apply destructive changes non-interactively. Re-run with --yes to apply them.`)); + process.exit(1); + } + const proceed = await confirm({ + message: `Apply ${destructive.length} destructive change(s)?`, + default: false, }); - const manager = new SQLiteSchemaManager({ - schema: finaldbSchema, - recreate_vector_table: isVector, + if (!proceed) { + console.log(chalk.dim(`Schema sync cancelled.`)); + process.exit(1); + } + } + snapshotDb({ db: DbClient, dst: BUN_SQLITE_TEMP_DB_FILE_PATH }); + try { + createDBSchema({ + ...params, + allow_destructive: opts.yes + ? true + : new Set(destructive.map((c) => c.description)), }); - await manager.syncSchema(); - manager.close(); + DbClient.close(); if (isTypeDef && config.typedef_file_path) { const out_file = path.resolve(ROOT_DIR, config.typedef_file_path); dbSchemaToTypeDef({ @@ -47,7 +105,11 @@ export default function () { } catch (error) { console.log(error); - cpSync(BUN_SQLITE_TEMP_DB_FILE_PATH, db_file_path); + DbClient.close(); + replaceDbFile({ + src: BUN_SQLITE_TEMP_DB_FILE_PATH, + db_file_path, + }); process.exit(1); } }); diff --git a/dist/commands/typedef.js b/dist/commands/typedef.js index d167d4e..35aa89f 100644 --- a/dist/commands/typedef.js +++ b/dist/commands/typedef.js @@ -1,5 +1,4 @@ import { Command } from "commander"; -import init from "../functions/init"; import dbSchemaToTypeDef from "../lib/sqlite/schema-to-typedef"; import path from "path"; import grabDirNames from "../data/grab-dir-names"; @@ -10,7 +9,8 @@ export default function () { .description("Build DB From Schema") .action(async (opts) => { console.log(`Creating Type Definition From DB Schema ...`); - const { config, dbSchema } = await init(); + const config = global.CONFIG; + const dbSchema = global.DB_SCHEMA; const { ROOT_DIR } = grabDirNames(); const finaldbSchema = appendDefaultFieldsToDbSchema({ dbSchema }); if (config.typedef_file_path) { @@ -22,7 +22,7 @@ export default function () { }); } else { - console.error(``); + console.error(`\`typedef_file_path\` is required in your config`); process.exit(1); } console.log(`${chalk.bold(chalk.green(`Typedef gen success!`))}`); diff --git a/dist/data/app-data.d.ts b/dist/data/app-data.d.ts index f3eb051..317c88f 100644 --- a/dist/data/app-data.d.ts +++ b/dist/data/app-data.d.ts @@ -1,6 +1,11 @@ export declare const AppData: { - readonly ConfigFileName: "bun-sqlite.config.ts"; + readonly ConfigFileName: "bun-sqlite.config"; readonly MaxBackups: 10; + readonly MaxExports: 10; readonly DefaultBackupDirName: ".backups"; + readonly DefaultExportDirName: ".exports"; readonly DbSchemaManagerTableName: "__db_schema_manager__"; + readonly DbSchemaFileName: "schema"; + /** Priority order when resolving config/schema files */ + readonly SupportedDataFileExtensions: readonly [".ts", ".js", ".json", ".yaml", ".yml"]; }; diff --git a/dist/data/app-data.js b/dist/data/app-data.js index 439bcb7..7b4b174 100644 --- a/dist/data/app-data.js +++ b/dist/data/app-data.js @@ -1,6 +1,11 @@ export const AppData = { - ConfigFileName: "bun-sqlite.config.ts", + ConfigFileName: "bun-sqlite.config", MaxBackups: 10, + MaxExports: 10, DefaultBackupDirName: ".backups", + DefaultExportDirName: ".exports", DbSchemaManagerTableName: "__db_schema_manager__", + DbSchemaFileName: "schema", + /** Priority order when resolving config/schema files */ + SupportedDataFileExtensions: [".ts", ".js", ".json", ".yaml", ".yml"], }; diff --git a/dist/functions/grab-sqlite-client.d.ts b/dist/functions/grab-sqlite-client.d.ts new file mode 100644 index 0000000..8a6b6dc --- /dev/null +++ b/dist/functions/grab-sqlite-client.d.ts @@ -0,0 +1,7 @@ +import { Database } from "bun:sqlite"; +import type { BunSQLiteConfig } from "../types"; +type Params = { + config: BunSQLiteConfig; +}; +export default function grabSQLiteClient({ config }: Params): Database; +export {}; diff --git a/dist/functions/grab-sqlite-client.js b/dist/functions/grab-sqlite-client.js new file mode 100644 index 0000000..c9a00ef --- /dev/null +++ b/dist/functions/grab-sqlite-client.js @@ -0,0 +1,21 @@ +import { Database } from "bun:sqlite"; +import * as sqliteVec from "sqlite-vec"; +import grabDBDir from "../utils/grab-db-dir"; +/** Reuse one connection per database file. */ +const clientCache = new Map(); +export default function grabSQLiteClient({ config }) { + const { db_file_path } = grabDBDir({ config }); + const cached = clientCache.get(db_file_path); + if (cached) { + return cached; + } + const SQLiteClient = new Database(db_file_path, { + create: true, + }); + sqliteVec.load(SQLiteClient); + if (config.wal_mode) { + SQLiteClient.run("PRAGMA journal_mode = WAL;"); + } + clientCache.set(db_file_path, SQLiteClient); + return SQLiteClient; +} diff --git a/dist/functions/init.d.ts b/dist/functions/init.d.ts index c7c6d0b..438451a 100644 --- a/dist/functions/init.d.ts +++ b/dist/functions/init.d.ts @@ -1,2 +1,18 @@ -import type { BunSQLiteConfigReturn } from "../types"; -export default function init(): Promise; +import type { Database } from "bun:sqlite"; +import type { BunSQLiteConfig, BunSQLiteConfigReturn, BUN_SQLITE_DatabaseSchemaType } from "../types"; +/** + * # Declare Global Variables + */ +declare global { + var CONFIG: BunSQLiteConfig; + var DB_SCHEMA: BUN_SQLITE_DatabaseSchemaType; + var SQLITE_CLIENT: Database; + var CONFIG_FILE_PATH: string; + var SCHEMA_FILE_PATH: string; +} +/** + * # Load config, schema and the SQLite client into globals + * @description Runs once per process. Later calls return the loaded + * globals. + */ +export default function init(): BunSQLiteConfigReturn; diff --git a/dist/functions/init.js b/dist/functions/init.js index 4635e0b..1b58e57 100644 --- a/dist/functions/init.js +++ b/dist/functions/init.js @@ -2,45 +2,85 @@ import path from "path"; import fs from "fs"; import { AppData } from "../data/app-data"; import grabDirNames from "../data/grab-dir-names"; -export default async function init() { - try { - const { ROOT_DIR } = grabDirNames(); - const { ConfigFileName } = AppData; - const ConfigFilePath = path.join(ROOT_DIR, ConfigFileName); - if (!fs.existsSync(ConfigFilePath)) { - console.log("ConfigFilePath", ConfigFilePath); - console.error(`Please create a \`${ConfigFileName}\` file at the root of your project.`); - process.exit(1); - } - const ConfigImport = await import(ConfigFilePath); - const Config = ConfigImport["default"]; - if (!Config.db_name) { - console.error(`\`db_name\` is required in your config`); - process.exit(1); - } - if (!Config.db_schema_file_name) { - console.error(`\`db_schema_file_name\` is required in your config`); - process.exit(1); - } - let db_dir = ROOT_DIR; - if (Config.db_dir) { - db_dir = path.resolve(ROOT_DIR, Config.db_dir); - if (!fs.existsSync(Config.db_dir)) { - fs.mkdirSync(Config.db_dir, { recursive: true }); - } - } - const DBSchemaFilePath = path.join(db_dir, Config.db_schema_file_name); - const DbSchemaImport = await import(DBSchemaFilePath); - const DbSchema = DbSchemaImport["default"]; - const backup_dir = Config.db_backup_dir || AppData["DefaultBackupDirName"]; - const BackupDir = path.resolve(db_dir, backup_dir); - if (!fs.existsSync(BackupDir)) { - fs.mkdirSync(BackupDir, { recursive: true }); - } - return { config: Config, dbSchema: DbSchema }; +import setSQLiteClient from "./set-sqlite-client"; +import grabDBDir from "../utils/grab-db-dir"; +import { loadDataFile, resolveAndLoadDataFile, supportedDataFileNames, } from "../utils/resolve-and-load-data-file"; +/** + * # Load config, schema and the SQLite client into globals + * @description Runs once per process. Later calls return the loaded + * globals. + */ +export default function init() { + if (global.CONFIG && global.DB_SCHEMA && global.SQLITE_CLIENT) { + return { config: global.CONFIG, dbSchema: global.DB_SCHEMA }; } - catch (error) { - console.error(`Initialization ERROR => ` + error.message); - process.exit(1); + const { ROOT_DIR } = grabDirNames(); + const { ConfigFileName } = AppData; + const loadedConfig = resolveAndLoadDataFile(ROOT_DIR, ConfigFileName); + if (!loadedConfig) { + throw new Error(`Please create a ${supportedDataFileNames(ConfigFileName)} file at the root of your project. (Looked in ${ROOT_DIR})`); } + const Config = loadedConfig.data; + if (!Config || typeof Config !== "object") { + throw new Error(`Invalid config in \`${loadedConfig.path}\`. Expected a config object${isCodeFile(loadedConfig) ? " (export default)" : ""}.`); + } + if (!Config.db_name) { + throw new Error(`\`db_name\` is required in your config`); + } + const { db_dir, backup_dir } = grabDBDir({ config: Config }); + if (!fs.existsSync(db_dir)) { + fs.mkdirSync(db_dir, { recursive: true }); + } + const loadedSchema = loadSchema({ config: Config, db_dir }); + const DbSchema = loadedSchema.data; + if (!DbSchema || typeof DbSchema !== "object") { + throw new Error(`Invalid schema in \`${loadedSchema.path}\`. Expected a schema object${isCodeFile(loadedSchema) ? " (export default)" : ""}.`); + } + if (!fs.existsSync(backup_dir)) { + fs.mkdirSync(backup_dir, { recursive: true }); + } + global.CONFIG = Config; + global.DB_SCHEMA = DbSchema; + global.CONFIG_FILE_PATH = loadedConfig.path; + global.SCHEMA_FILE_PATH = loadedSchema.path; + setSQLiteClient({ config: Config }); + return { config: Config, dbSchema: DbSchema }; +} +/** + * # Load the schema file + * @description Uses `db_schema_file_name` when set, otherwise + * `schema.ts|js|json|yaml|yml` in `db_dir`. + */ +function loadSchema({ config, db_dir, }) { + if (config.db_schema_file_name) { + const schema_file_path = path.join(db_dir, config.db_schema_file_name); + const extension = path.extname(schema_file_path); + const supported = AppData.SupportedDataFileExtensions.find((ext) => ext === extension); + if (!supported) { + throw new Error(`Unsupported schema file \`${config.db_schema_file_name}\`. Use one of: ${AppData.SupportedDataFileExtensions.join(", ")}`); + } + if (!fs.existsSync(schema_file_path)) { + throw new Error(`Schema file not found: ${schema_file_path}`); + } + const resolved = { + path: schema_file_path, + basename: path.basename(schema_file_path), + extension: supported, + format: supported === ".yml" + ? "yaml" + : supported.slice(1), + }; + return { + ...resolved, + data: loadDataFile(resolved), + }; + } + const loaded = resolveAndLoadDataFile(db_dir, AppData.DbSchemaFileName); + if (!loaded) { + throw new Error(`Please create a schema file (${supportedDataFileNames(AppData.DbSchemaFileName)}) in \`${db_dir}\`, or set \`db_schema_file_name\` in your config.`); + } + return loaded; +} +function isCodeFile(file) { + return file.format === "ts" || file.format === "js"; } diff --git a/dist/functions/set-sqlite-client.d.ts b/dist/functions/set-sqlite-client.d.ts new file mode 100644 index 0000000..c408684 --- /dev/null +++ b/dist/functions/set-sqlite-client.d.ts @@ -0,0 +1,6 @@ +import type { BunSQLiteConfig } from "../types"; +type Params = { + config: BunSQLiteConfig; +}; +export default function setSQLiteClient({ config }: Params): void; +export {}; diff --git a/dist/functions/set-sqlite-client.js b/dist/functions/set-sqlite-client.js new file mode 100644 index 0000000..094f562 --- /dev/null +++ b/dist/functions/set-sqlite-client.js @@ -0,0 +1,4 @@ +import grabSQLiteClient from "./grab-sqlite-client"; +export default function setSQLiteClient({ config }) { + global.SQLITE_CLIENT = grabSQLiteClient({ config }); +} diff --git a/dist/index.d.ts b/dist/index.d.ts index 653e62b..6228ef6 100644 --- a/dist/index.d.ts +++ b/dist/index.d.ts @@ -17,3 +17,5 @@ declare const BunSQLite: { }; }; export default BunSQLite; +export type * from "./types"; +export { UsersOmitedFields, MariaDBCollations, TextFieldTypesArray, BUN_SQLITE_DATATYPES, ServerQueryOperators, ServerQueryEqualities, SQlComparisons, DataCrudRequestMethods, DataCrudRequestMethodsLowerCase, DsqlCrudActions, QueryFields, DockerComposeServices, IndexTypes, DefaultFields, } from "./types"; diff --git a/dist/index.js b/dist/index.js index 32a414a..65bc39d 100644 --- a/dist/index.js +++ b/dist/index.js @@ -1,3 +1,4 @@ +import init from "./functions/init"; import DbDelete from "./lib/sqlite/db-delete"; import DbInsert from "./lib/sqlite/db-insert"; import DbSelect from "./lib/sqlite/db-select"; @@ -5,15 +6,26 @@ import DbSQL from "./lib/sqlite/db-sql"; import DbUpdate from "./lib/sqlite/db-update"; import grabDbSchema from "./utils/grab-db-schema"; import grabJoinFieldsFromQueryObject from "./utils/grab-join-fields-from-query-object"; +/** + * Initialize lazily on first call, so importing the library never throws. + * Setup errors (missing config / schema) reject the call instead. + */ +function withInit(fn) { + return (async (...args) => { + init(); + return fn(...args); + }); +} const BunSQLite = { - select: DbSelect, - insert: DbInsert, - update: DbUpdate, - delete: DbDelete, - sql: DbSQL, + select: withInit(DbSelect), + insert: withInit(DbInsert), + update: withInit(DbUpdate), + delete: withInit(DbDelete), + sql: withInit(DbSQL), utils: { - grab_db_schema: grabDbSchema, + grab_db_schema: withInit(grabDbSchema), grab_join_fields_from_query_object: grabJoinFieldsFromQueryObject, }, }; export default BunSQLite; +export { UsersOmitedFields, MariaDBCollations, TextFieldTypesArray, BUN_SQLITE_DATATYPES, ServerQueryOperators, ServerQueryEqualities, SQlComparisons, DataCrudRequestMethods, DataCrudRequestMethodsLowerCase, DsqlCrudActions, QueryFields, DockerComposeServices, IndexTypes, DefaultFields, } from "./types"; diff --git a/dist/lib/db-handler.d.ts b/dist/lib/db-handler.d.ts new file mode 100644 index 0000000..a6ae69f --- /dev/null +++ b/dist/lib/db-handler.d.ts @@ -0,0 +1,18 @@ +import type { APIResponseObject, BunSQLiteConfig, SQLInsertGenValueType } from "../types"; +type Param = { + query: string; + values?: SQLInsertGenValueType[]; + config?: BunSQLiteConfig; +}; +/** + * # Main DB Handler Function + * @description Any statement that produces columns (SELECT, WITH, PRAGMA, + * RETURNING, ...) returns rows. Everything else goes through `run`, which + * also handles multiple statements. + */ +export default function dbHandler({ query, values, config }: Param): APIResponseObject; +export {}; diff --git a/dist/lib/db-handler.js b/dist/lib/db-handler.js new file mode 100644 index 0000000..e5375eb --- /dev/null +++ b/dist/lib/db-handler.js @@ -0,0 +1,53 @@ +import init from "../functions/init"; +import grabSQLiteClient from "../functions/grab-sqlite-client"; +/** + * Resolve the SQLite client. The global client is used unless a `config` + * for another database is passed. + */ +function resolveClient(config) { + if (config) { + return grabSQLiteClient({ config }); + } + init(); + return global.SQLITE_CLIENT; +} +/** + * # Main DB Handler Function + * @description Any statement that produces columns (SELECT, WITH, PRAGMA, + * RETURNING, ...) returns rows. Everything else goes through `run`, which + * also handles multiple statements. + */ +export default function dbHandler({ query, values, config }) { + try { + const CLIENT = resolveClient(config); + const stmt = CLIENT.prepare(query); + const returnsRows = stmt.columnNames.length > 0; + try { + if (returnsRows) { + const rows = stmt.all(...(values || [])); + return { + success: true, + payload: rows, + singleRes: rows[0], + }; + } + } + finally { + stmt.finalize(); + } + const res = CLIENT.run(query, (values || [])); + return { + success: true, + postInsertReturn: { + affectedRows: res.changes, + insertId: Number(res.lastInsertRowid), + }, + }; + } + catch (error) { + return { + success: false, + error: error.message, + }; + } +} diff --git a/dist/lib/grab-duplicate-safe-insert-sql.js b/dist/lib/grab-duplicate-safe-insert-sql.js index c13fd87..b4c5bc9 100644 --- a/dist/lib/grab-duplicate-safe-insert-sql.js +++ b/dist/lib/grab-duplicate-safe-insert-sql.js @@ -1,22 +1,35 @@ import init from "../functions/init"; +import { sqlIdent, sqlQuoteIdent } from "../utils/sql-safe"; export default async function ({ sql: passed_sql, table, data }) { let sql = passed_sql; - const { dbSchema } = await init(); + const { dbSchema } = init(); const table_schema = dbSchema.tables.find((t) => t.tableName == table); const now = Date.now(); if (table_schema?.tableName) { - const set_sql_arr = Object.keys(Array.isArray(data) ? data[0] : data).map((field) => `${field} = excluded.${field}`); + const set_sql_arr = Object.keys(Array.isArray(data) ? data[0] : data) + .filter((field) => field !== "updated_at") + .map((field) => { + const column = sqlIdent(field, "column name"); + return `${column} = excluded.${column}`; + }); set_sql_arr.push(`updated_at = ${now}`); const set_sql = set_sql_arr.join(", "); const unique_fields = table_schema.fields.filter((f) => f.unique); for (let i = 0; i < unique_fields.length; i++) { const field = unique_fields[i]; - sql += ` ON CONFLICT(${field?.fieldName}) DO UPDATE SET ${set_sql}`; + if (!field?.fieldName) + continue; + sql += ` ON CONFLICT(${sqlQuoteIdent(field.fieldName)}) DO UPDATE SET ${set_sql}`; } if (table_schema.uniqueConstraints?.[0]) { for (let i = 0; i < table_schema.uniqueConstraints.length; i++) { const constraint = table_schema.uniqueConstraints[i]; - sql += ` ON CONFLICT(${constraint?.constraintTableFields?.map((c) => c.value)?.join(", ")}) DO UPDATE SET ${set_sql}`; + const columns = constraint?.constraintTableFields + ?.map((c) => sqlQuoteIdent(String(c.value))) + ?.join(", "); + if (!columns) + continue; + sql += ` ON CONFLICT(${columns}) DO UPDATE SET ${set_sql}`; } } } diff --git a/dist/lib/schema/build-column-definition.d.ts b/dist/lib/schema/build-column-definition.d.ts new file mode 100644 index 0000000..15ca5b5 --- /dev/null +++ b/dist/lib/schema/build-column-definition.d.ts @@ -0,0 +1,5 @@ +import type { BUN_SQLITE_FieldSchemaType } from "../../types"; +/** + * Build column definition SQL + */ +export default function buildColumnDefinition(field: BUN_SQLITE_FieldSchemaType, is_vector?: boolean): string; diff --git a/dist/lib/schema/build-column-definition.js b/dist/lib/schema/build-column-definition.js new file mode 100644 index 0000000..c0af290 --- /dev/null +++ b/dist/lib/schema/build-column-definition.js @@ -0,0 +1,52 @@ +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; +import mapDataType from "./map-data-types"; +/** + * Build column definition SQL + */ +export default function buildColumnDefinition(field, is_vector) { + if (!field.fieldName) { + throw new Error("Field name is required"); + } + // vec0 can't parse quoted column names + const fieldName = is_vector + ? field.sideCar + ? `+${field.fieldName}` + : `${field.fieldName}` + : q(field.fieldName); + const parts = [fieldName]; + // Data type mapping + parts.push(mapDataType(field)); + // Primary key + if (field.primaryKey) { + parts.push("PRIMARY KEY"); + if (field.autoIncrement) { + parts.push("AUTOINCREMENT"); + } + } + // Not null + if (field.notNullValue && !field.primaryKey) { + parts.push("NOT NULL"); + } + // Unique + if (field.unique && !field.primaryKey) { + parts.push("UNIQUE"); + } + if (field.vectorDistanceMetric) { + parts.push(`distance_metric=${field.vectorDistanceMetric}`); + } + // Default value + if (field.defaultValue !== undefined) { + if (typeof field.defaultValue === "string") { + parts.push( + // Escape single quotes by doubling them to prevent SQL injection and wrap in single quotes + `DEFAULT '${field.defaultValue.replace(/'/g, "''")}'`); + } + else { + parts.push(`DEFAULT ${field.defaultValue}`); + } + } + else if (field.defaultValueLiteral) { + parts.push(`DEFAULT ${field.defaultValueLiteral}`); + } + return parts.join(" "); +} diff --git a/dist/lib/schema/build-create-table-sql.d.ts b/dist/lib/schema/build-create-table-sql.d.ts new file mode 100644 index 0000000..78ac1b0 --- /dev/null +++ b/dist/lib/schema/build-create-table-sql.d.ts @@ -0,0 +1,10 @@ +import type { BUN_SQLITE_DatabaseSchemaType, BUN_SQLITE_TableSchemaType } from "../../types"; +type Params = { + db_schema: BUN_SQLITE_DatabaseSchemaType; + table: BUN_SQLITE_TableSchemaType; +}; +/** + * Build the `CREATE TABLE` (or `CREATE VIRTUAL TABLE`) statement for a table + */ +export default function buildCreateTableSql({ db_schema, table }: Params): string; +export {}; diff --git a/dist/lib/schema/build-create-table-sql.js b/dist/lib/schema/build-create-table-sql.js new file mode 100644 index 0000000..77dc295 --- /dev/null +++ b/dist/lib/schema/build-create-table-sql.js @@ -0,0 +1,32 @@ +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; +import buildColumnDefinition from "./build-column-definition"; +import buildForeignKeyConstraint from "./build-foreign-key-constraint"; +import resolveTable from "./resolve-table"; +/** + * Build the `CREATE TABLE` (or `CREATE VIRTUAL TABLE`) statement for a table + */ +export default function buildCreateTableSql({ db_schema, table }) { + const new_table = resolveTable({ table, db_schema }); + const columns = []; + const foreignKeys = []; + for (const field of new_table.fields) { + columns.push(buildColumnDefinition(field, new_table.isVector)); + if (field.foreignKey && !new_table.isVector) { + foreignKeys.push(buildForeignKeyConstraint(field)); + } + } + // Add unique constraints + for (const constraint of new_table.uniqueConstraints || []) { + if (!constraint.constraintTableFields?.length) + continue; + const fieldNames = constraint.constraintTableFields.map((f) => String(f.value)); + const constraintName = constraint.constraintName || `unique_${fieldNames.join(", ")}`; + columns.push(`CONSTRAINT ${q(constraintName)} UNIQUE (${fieldNames + .map(q) + .join(", ")})`); + } + const allConstraints = [...columns, ...foreignKeys]; + return new_table.isVector + ? `CREATE VIRTUAL TABLE ${q(new_table.tableName)} USING ${new_table.vectorType || "vec0"}(${allConstraints.join(", ")})` + : `CREATE TABLE ${q(new_table.tableName)} (${allConstraints.join(", ")})`; +} diff --git a/dist/lib/schema/build-foreign-key-constraint.d.ts b/dist/lib/schema/build-foreign-key-constraint.d.ts new file mode 100644 index 0000000..b29797a --- /dev/null +++ b/dist/lib/schema/build-foreign-key-constraint.d.ts @@ -0,0 +1,5 @@ +import type { BUN_SQLITE_FieldSchemaType } from "../../types"; +/** + * Build foreign key constraint + */ +export default function buildForeignKeyConstraint(field: BUN_SQLITE_FieldSchemaType): string; diff --git a/dist/lib/schema/build-foreign-key-constraint.js b/dist/lib/schema/build-foreign-key-constraint.js new file mode 100644 index 0000000..5a803de --- /dev/null +++ b/dist/lib/schema/build-foreign-key-constraint.js @@ -0,0 +1,15 @@ +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; +/** + * Build foreign key constraint + */ +export default function buildForeignKeyConstraint(field) { + const fk = field.foreignKey; + let constraint = `FOREIGN KEY (${q(field.fieldName)}) REFERENCES ${q(String(fk.destinationTableName))}(${q(String(fk.destinationTableColumnName))})`; + if (fk.cascadeDelete) { + constraint += " ON DELETE CASCADE"; + } + if (fk.cascadeUpdate) { + constraint += " ON UPDATE CASCADE"; + } + return constraint; +} diff --git a/dist/lib/schema/check-if-table-exists.d.ts b/dist/lib/schema/check-if-table-exists.d.ts new file mode 100644 index 0000000..fea7f21 --- /dev/null +++ b/dist/lib/schema/check-if-table-exists.d.ts @@ -0,0 +1,7 @@ +import type { Database } from "bun:sqlite"; +type Params = { + db: Database; + table_name: string; +}; +export default function checkIfTableExists({ db, table_name }: Params): boolean; +export {}; diff --git a/dist/lib/schema/check-if-table-exists.js b/dist/lib/schema/check-if-table-exists.js new file mode 100644 index 0000000..0b09afd --- /dev/null +++ b/dist/lib/schema/check-if-table-exists.js @@ -0,0 +1,6 @@ +export default function checkIfTableExists({ db, table_name }) { + const tableExists = db + .query(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`) + .get(table_name); + return Boolean(tableExists?.name); +} diff --git a/dist/lib/schema/create-db-manager-table.d.ts b/dist/lib/schema/create-db-manager-table.d.ts new file mode 100644 index 0000000..e83486a --- /dev/null +++ b/dist/lib/schema/create-db-manager-table.d.ts @@ -0,0 +1,4 @@ +import type { Database } from "bun:sqlite"; +export default function createDBManagerTable({ db }: { + db: Database; +}): void; diff --git a/dist/lib/schema/create-db-manager-table.js b/dist/lib/schema/create-db-manager-table.js new file mode 100644 index 0000000..a43c761 --- /dev/null +++ b/dist/lib/schema/create-db-manager-table.js @@ -0,0 +1,11 @@ +import { AppData } from "../../data/app-data"; +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; +export default function createDBManagerTable({ db }) { + db.run(` + CREATE TABLE IF NOT EXISTS ${q(AppData["DbSchemaManagerTableName"])} ( + table_name TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) + `); +} diff --git a/dist/lib/schema/create-db-schema.d.ts b/dist/lib/schema/create-db-schema.d.ts new file mode 100644 index 0000000..0714009 --- /dev/null +++ b/dist/lib/schema/create-db-schema.d.ts @@ -0,0 +1,11 @@ +import type { CreateDBSchemaParams } from "../../types"; +/** + * # Sync the database to the schema + * @description Changed tables are rebuilt by copying into a new table and + * dropping the old one. With foreign keys ON, `DROP TABLE` runs an implicit + * `DELETE FROM` that fires `ON DELETE CASCADE` on child tables, so keys are + * disabled for the sync and checked before committing. (`PRAGMA + * foreign_keys` is a no-op inside a transaction.) With `dry_run` the + * transaction is rolled back, so `changes` is the exact plan. + */ +export default function createDBSchema(params: CreateDBSchemaParams): void; diff --git a/dist/lib/schema/create-db-schema.js b/dist/lib/schema/create-db-schema.js new file mode 100644 index 0000000..7194bf6 --- /dev/null +++ b/dist/lib/schema/create-db-schema.js @@ -0,0 +1,66 @@ +import _ from "lodash"; +import createDBManagerTable from "./create-db-manager-table"; +import dropRemovedTables from "./drop-removed-tables"; +import getExistingTablesFromTablesManagerTable from "./get-existing-tables-from-tables-manager-table"; +import handleDBSchemaTable from "./handle-db-schema-table"; +import orderDBSchema from "./order-db-schema"; +/** + * # Sync the database to the schema + * @description Changed tables are rebuilt by copying into a new table and + * dropping the old one. With foreign keys ON, `DROP TABLE` runs an implicit + * `DELETE FROM` that fires `ON DELETE CASCADE` on child tables, so keys are + * disabled for the sync and checked before committing. (`PRAGMA + * foreign_keys` is a no-op inside a transaction.) With `dry_run` the + * transaction is rolled back, so `changes` is the exact plan. + */ +export default function createDBSchema(params) { + const { db, dry_run } = params; + /** + * Reorder Tables (parents before children with FKs) + */ + const db_schema = orderDBSchema(params); + db.run("PRAGMA foreign_keys = OFF;"); + db.run("BEGIN"); + try { + /** + * Create Schema Manager Table + */ + createDBManagerTable({ db }); + const existing_tables = getExistingTablesFromTablesManagerTable({ + db, + }); + /** + * Create or update tables + */ + for (const table of db_schema.tables) { + handleDBSchemaTable({ + ...params, + db_schema, + table, + existing_tables, + }); + } + /** + * Drop tables that no longer exist in schema + */ + dropRemovedTables({ + ...params, + existing_tables, + schema_tables: db_schema.tables.map((t) => t.tableName), + }); + const fkViolations = db.query(`PRAGMA foreign_key_check`).all(); + if (fkViolations.length) { + const tables = _.uniq(fkViolations.map((v) => `${v.table} -> ${v.parent}`)); + throw new Error(`Foreign key violations after schema sync: ${tables.join(", ")}`); + } + db.run(dry_run ? "ROLLBACK" : "COMMIT"); + } + catch (error) { + if (db.inTransaction) + db.run("ROLLBACK"); + throw error; + } + finally { + db.run("PRAGMA foreign_keys = ON;"); + } +} diff --git a/dist/lib/schema/create-table.d.ts b/dist/lib/schema/create-table.d.ts new file mode 100644 index 0000000..dcb7ce4 --- /dev/null +++ b/dist/lib/schema/create-table.d.ts @@ -0,0 +1,9 @@ +import type { BUN_SQLITE_TableSchemaType, CreateDBSchemaParams } from "../../types"; +type Params = CreateDBSchemaParams & { + table: BUN_SQLITE_TableSchemaType; +}; +/** + * Create a new table + */ +export default function createTable(params: Params): void; +export {}; diff --git a/dist/lib/schema/create-table.js b/dist/lib/schema/create-table.js new file mode 100644 index 0000000..745d2d1 --- /dev/null +++ b/dist/lib/schema/create-table.js @@ -0,0 +1,12 @@ +import buildCreateTableSql from "./build-create-table-sql"; +import recordSchemaChange from "./record-schema-change"; +/** + * Create a new table + */ +export default function createTable(params) { + const { db, db_schema, table } = params; + recordSchemaChange(params, { + description: `Create table "${table.tableName}"`, + }); + db.run(buildCreateTableSql({ db_schema, table })); +} diff --git a/dist/lib/schema/drop-removed-tables.d.ts b/dist/lib/schema/drop-removed-tables.d.ts new file mode 100644 index 0000000..5beed69 --- /dev/null +++ b/dist/lib/schema/drop-removed-tables.d.ts @@ -0,0 +1,10 @@ +import type { CreateDBSchemaParams } from "../../types"; +type Params = CreateDBSchemaParams & { + existing_tables: string[]; + schema_tables: string[]; +}; +/** + * Drop tables that are no longer in the schema + */ +export default function dropRemovedTables(params: Params): void; +export {}; diff --git a/dist/lib/schema/drop-removed-tables.js b/dist/lib/schema/drop-removed-tables.js new file mode 100644 index 0000000..9fbe93e --- /dev/null +++ b/dist/lib/schema/drop-removed-tables.js @@ -0,0 +1,34 @@ +import _ from "lodash"; +import { readLiveSchema } from "../../functions/live-schema"; +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; +import removeDBManagerTable from "./remove-db-manager-table"; +import recordSchemaChange from "./record-schema-change"; +import checkIfTableExists from "./check-if-table-exists"; +/** + * Drop tables that are no longer in the schema + */ +export default function dropRemovedTables(params) { + const { db, existing_tables, schema_tables } = params; + const tablesToDrop = existing_tables.filter((t) => !schema_tables.includes(t) && + !schema_tables.find((scT) => t.startsWith(scT + "_"))); + const currentSchema = readLiveSchema(); + for (const table of currentSchema?.tables || []) { + if (!table?.tableName) + continue; + if (!schema_tables.includes(table.tableName)) { + tablesToDrop.push(table.tableName); + } + } + for (const tableName of _.uniq(tablesToDrop)) { + if (!checkIfTableExists({ db, table_name: tableName })) { + removeDBManagerTable({ db, table_name: tableName }); + continue; + } + recordSchemaChange(params, { + description: `Drop table "${tableName}"`, + destructive: true, + }); + db.run(`DROP TABLE IF EXISTS ${q(tableName)}`); + removeDBManagerTable({ db, table_name: tableName }); + } +} diff --git a/dist/lib/schema/get-existing-tables-from-tables-manager-table.d.ts b/dist/lib/schema/get-existing-tables-from-tables-manager-table.d.ts new file mode 100644 index 0000000..ad9786f --- /dev/null +++ b/dist/lib/schema/get-existing-tables-from-tables-manager-table.d.ts @@ -0,0 +1,4 @@ +import type { Database } from "bun:sqlite"; +export default function getExistingTablesFromTablesManagerTable({ db, }: { + db: Database; +}): string[]; diff --git a/dist/lib/schema/get-existing-tables-from-tables-manager-table.js b/dist/lib/schema/get-existing-tables-from-tables-manager-table.js new file mode 100644 index 0000000..27c949f --- /dev/null +++ b/dist/lib/schema/get-existing-tables-from-tables-manager-table.js @@ -0,0 +1,8 @@ +import { AppData } from "../../data/app-data"; +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; +export default function getExistingTablesFromTablesManagerTable({ db, }) { + const results = db + .query(`SELECT table_name FROM ${q(AppData["DbSchemaManagerTableName"])}`) + .all(); + return results.map((r) => r.table_name); +} diff --git a/dist/lib/schema/get-table-columns.d.ts b/dist/lib/schema/get-table-columns.d.ts new file mode 100644 index 0000000..85a8d17 --- /dev/null +++ b/dist/lib/schema/get-table-columns.d.ts @@ -0,0 +1,10 @@ +import type { Database } from "bun:sqlite"; +type Params = { + db: Database; + table_name: string; +}; +export default function getTableColumns({ db, table_name, }: Params): { + name: string; + type: string; +}[]; +export {}; diff --git a/dist/lib/schema/get-table-columns.js b/dist/lib/schema/get-table-columns.js new file mode 100644 index 0000000..e6e422d --- /dev/null +++ b/dist/lib/schema/get-table-columns.js @@ -0,0 +1,4 @@ +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; +export default function getTableColumns({ db, table_name, }) { + return db.query(`PRAGMA table_info(${q(table_name)})`).all(); +} diff --git a/dist/lib/schema/handle-db-schema-table.d.ts b/dist/lib/schema/handle-db-schema-table.d.ts new file mode 100644 index 0000000..f9d4226 --- /dev/null +++ b/dist/lib/schema/handle-db-schema-table.d.ts @@ -0,0 +1,10 @@ +import type { BUN_SQLITE_TableSchemaType, CreateDBSchemaParams } from "../../types"; +type Params = CreateDBSchemaParams & { + table: BUN_SQLITE_TableSchemaType; + existing_tables: string[]; +}; +/** + * Sync a single table (rename, create or update, then indexes) + */ +export default function handleDBSchemaTable(params: Params): void; +export {}; diff --git a/dist/lib/schema/handle-db-schema-table.js b/dist/lib/schema/handle-db-schema-table.js new file mode 100644 index 0000000..dff8f84 --- /dev/null +++ b/dist/lib/schema/handle-db-schema-table.js @@ -0,0 +1,34 @@ +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; +import createTable from "./create-table"; +import insertDBManagerTable from "./insert-db-manager-table"; +import recordSchemaChange from "./record-schema-change"; +import removeDBManagerTable from "./remove-db-manager-table"; +import syncIndexes from "./sync-indexes"; +import updateTable from "./update-table"; +/** + * Sync a single table (rename, create or update, then indexes) + */ +export default function handleDBSchemaTable(params) { + const { db, table, existing_tables } = params; + let tableExists = existing_tables.includes(table.tableName); + // Handle table rename + if (table.tableNameOld && + table.tableNameOld !== table.tableName && + existing_tables.includes(table.tableNameOld)) { + recordSchemaChange(params, { + description: `Rename table "${table.tableNameOld}" -> "${table.tableName}"`, + }); + db.run(`ALTER TABLE ${q(table.tableNameOld)} RENAME TO ${q(table.tableName)}`); + insertDBManagerTable({ db, table_name: table.tableName }); + removeDBManagerTable({ db, table_name: table.tableNameOld }); + tableExists = true; + } + if (!tableExists) { + createTable(params); + insertDBManagerTable({ db, table_name: table.tableName }); + } + else { + updateTable(params); + } + syncIndexes(params); +} diff --git a/dist/lib/schema/insert-db-manager-table.d.ts b/dist/lib/schema/insert-db-manager-table.d.ts new file mode 100644 index 0000000..50e0736 --- /dev/null +++ b/dist/lib/schema/insert-db-manager-table.d.ts @@ -0,0 +1,7 @@ +import type { Database } from "bun:sqlite"; +type Params = { + db: Database; + table_name: string; +}; +export default function insertDBManagerTable({ db, table_name }: Params): void; +export {}; diff --git a/dist/lib/schema/insert-db-manager-table.js b/dist/lib/schema/insert-db-manager-table.js new file mode 100644 index 0000000..b459b4e --- /dev/null +++ b/dist/lib/schema/insert-db-manager-table.js @@ -0,0 +1,5 @@ +import { AppData } from "../../data/app-data"; +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; +export default function insertDBManagerTable({ db, table_name }) { + db.run(`INSERT INTO ${q(AppData["DbSchemaManagerTableName"])} (table_name,created_at,updated_at) VALUES (?, ?, ?)`, [table_name, Date.now(), Date.now()]); +} diff --git a/dist/lib/schema/map-data-types.d.ts b/dist/lib/schema/map-data-types.d.ts new file mode 100644 index 0000000..37484ce --- /dev/null +++ b/dist/lib/schema/map-data-types.d.ts @@ -0,0 +1,5 @@ +import type { BUN_SQLITE_DATATYPES, BUN_SQLITE_FieldSchemaType } from "../../types"; +/** + * Map DSQL data types to SQLite types + */ +export default function mapDataType(field: BUN_SQLITE_FieldSchemaType): (typeof BUN_SQLITE_DATATYPES)[number]["value"]; diff --git a/dist/lib/schema/map-data-types.js b/dist/lib/schema/map-data-types.js new file mode 100644 index 0000000..af0b12d --- /dev/null +++ b/dist/lib/schema/map-data-types.js @@ -0,0 +1,40 @@ +/** + * Map DSQL data types to SQLite types + */ +export default function mapDataType(field) { + const dataType = field.dataType?.toLowerCase() || "text"; + const vectorSize = field.vectorSize || 1536; + // Vector Embeddings + if (field.isVector) { + return `FLOAT[${vectorSize}]`; + } + // Integer types + if (dataType.includes("int") || + dataType === "bigint" || + dataType === "smallint" || + dataType === "tinyint") { + return "INTEGER"; + } + // Real/Float types + if (dataType.includes("real") || + dataType.includes("float") || + dataType.includes("double") || + dataType === "decimal" || + dataType === "numeric") { + return "REAL"; + } + // Blob types + if (dataType.includes("blob") || dataType.includes("binary")) { + return "BLOB"; + } + // Boolean + if (dataType === "boolean" || dataType === "bool") { + return "INTEGER"; // SQLite uses INTEGER for boolean (0/1) + } + // Date/Time types + if (dataType.includes("date") || dataType.includes("time")) { + return "TEXT"; // SQLite stores dates as TEXT or INTEGER + } + // Default to TEXT for all text-based types + return "TEXT"; +} diff --git a/dist/lib/schema/order-db-schema.d.ts b/dist/lib/schema/order-db-schema.d.ts new file mode 100644 index 0000000..57e8bdb --- /dev/null +++ b/dist/lib/schema/order-db-schema.d.ts @@ -0,0 +1,13 @@ +import type { BUN_SQLITE_DatabaseSchemaType, CreateDBSchemaParams } from "../../types"; +/** + * Topologically order tables so referenced (parent) tables come before the + * tables that reference them. Declaration order is kept wherever the FK + * graph allows it. + * + * - Self references (e.g. `comments.parent_comment_id`) are ignored. + * - References to tables outside the schema are ignored. + * - Cycles do not throw: a table already on the current path is skipped, + * so every table is still emitted exactly once. Foreign keys are not + * enforced during the sync, so cycles are safe. + */ +export default function orderDBSchema({ db_schema, }: Pick): BUN_SQLITE_DatabaseSchemaType; diff --git a/dist/lib/schema/order-db-schema.js b/dist/lib/schema/order-db-schema.js new file mode 100644 index 0000000..c812350 --- /dev/null +++ b/dist/lib/schema/order-db-schema.js @@ -0,0 +1,49 @@ +import _ from "lodash"; +/** + * Topologically order tables so referenced (parent) tables come before the + * tables that reference them. Declaration order is kept wherever the FK + * graph allows it. + * + * - Self references (e.g. `comments.parent_comment_id`) are ignored. + * - References to tables outside the schema are ignored. + * - Cycles do not throw: a table already on the current path is skipped, + * so every table is still emitted exactly once. Foreign keys are not + * enforced during the sync, so cycles are safe. + */ +export default function orderDBSchema({ db_schema, }) { + const new_db_schema = _.cloneDeep(db_schema); + const tables = new_db_schema.tables; + const tables_by_name = new Map(); + for (const table of tables) { + if (table) + tables_by_name.set(table.tableName, table); + } + const ordered = []; + const visited = new Set(); + const visiting = new Set(); + function visit(table) { + if (visited.has(table.tableName) || visiting.has(table.tableName)) { + return; + } + visiting.add(table.tableName); + for (const field of table.fields || []) { + const dst_table_name = field?.foreignKey?.destinationTableName; + if (!dst_table_name || dst_table_name === table.tableName) { + continue; + } + const fk_table = tables_by_name.get(dst_table_name); + if (fk_table) { + visit(fk_table); + } + } + visiting.delete(table.tableName); + visited.add(table.tableName); + ordered.push(table); + } + for (const table of tables) { + if (table) + visit(table); + } + new_db_schema.tables = ordered; + return new_db_schema; +} diff --git a/dist/lib/schema/record-schema-change.d.ts b/dist/lib/schema/record-schema-change.d.ts new file mode 100644 index 0000000..f5feb49 --- /dev/null +++ b/dist/lib/schema/record-schema-change.d.ts @@ -0,0 +1,7 @@ +import type { CreateDBSchemaParams, SchemaChange } from "../../types"; +/** + * # Record a schema change before it runs + * @description Logs the change (unless this is a dry run) and refuses + * destructive changes that were not approved. + */ +export default function recordSchemaChange(params: Pick, change: SchemaChange): void; diff --git a/dist/lib/schema/record-schema-change.js b/dist/lib/schema/record-schema-change.js new file mode 100644 index 0000000..7eabed6 --- /dev/null +++ b/dist/lib/schema/record-schema-change.js @@ -0,0 +1,19 @@ +/** + * # Record a schema change before it runs + * @description Logs the change (unless this is a dry run) and refuses + * destructive changes that were not approved. + */ +export default function recordSchemaChange(params, change) { + params.changes?.push(change); + if (params.dry_run) { + return; + } + const allowed = params.allow_destructive; + if (change.destructive && + allowed !== undefined && + allowed !== true && + !allowed.has(change.description)) { + throw new Error(`Refusing unapproved destructive change: ${change.description}. Re-run with --dry-run to review, or --yes to allow it.`); + } + console.log(change.description); +} diff --git a/dist/lib/schema/remove-db-manager-table.d.ts b/dist/lib/schema/remove-db-manager-table.d.ts new file mode 100644 index 0000000..79dbce4 --- /dev/null +++ b/dist/lib/schema/remove-db-manager-table.d.ts @@ -0,0 +1,7 @@ +import type { Database } from "bun:sqlite"; +type Params = { + db: Database; + table_name: string; +}; +export default function removeDBManagerTable({ db, table_name }: Params): void; +export {}; diff --git a/dist/lib/schema/remove-db-manager-table.js b/dist/lib/schema/remove-db-manager-table.js new file mode 100644 index 0000000..94ac742 --- /dev/null +++ b/dist/lib/schema/remove-db-manager-table.js @@ -0,0 +1,5 @@ +import { AppData } from "../../data/app-data"; +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; +export default function removeDBManagerTable({ db, table_name }) { + db.run(`DELETE FROM ${q(AppData["DbSchemaManagerTableName"])} WHERE table_name = ?`, [table_name]); +} diff --git a/dist/lib/schema/resolve-table.d.ts b/dist/lib/schema/resolve-table.d.ts new file mode 100644 index 0000000..a8da530 --- /dev/null +++ b/dist/lib/schema/resolve-table.d.ts @@ -0,0 +1,12 @@ +import type { BUN_SQLITE_DatabaseSchemaType, BUN_SQLITE_TableSchemaType } from "../../types"; +type Params = { + table: BUN_SQLITE_TableSchemaType; + db_schema: BUN_SQLITE_DatabaseSchemaType; +}; +/** + * Resolve a table's effective definition. Tables with `parentTableName` + * take their structure from the parent. Always returns a copy: the schema + * objects are never mutated. + */ +export default function resolveTable({ table, db_schema, }: Params): BUN_SQLITE_TableSchemaType; +export {}; diff --git a/dist/lib/schema/resolve-table.js b/dist/lib/schema/resolve-table.js new file mode 100644 index 0000000..78c3c07 --- /dev/null +++ b/dist/lib/schema/resolve-table.js @@ -0,0 +1,19 @@ +import _ from "lodash"; +/** + * Resolve a table's effective definition. Tables with `parentTableName` + * take their structure from the parent. Always returns a copy: the schema + * objects are never mutated. + */ +export default function resolveTable({ table, db_schema, }) { + if (!table.parentTableName) { + return _.cloneDeep(table); + } + const parent_table = db_schema.tables.find((t) => t.tableName === table.parentTableName); + if (!parent_table) { + throw new Error(`Parent table \`${table.parentTableName}\` not found for \`${table.tableName}\``); + } + return _.merge(_.cloneDeep(parent_table), { + tableName: table.tableName, + tableDescription: table.tableDescription, + }); +} diff --git a/dist/lib/schema/sync-indexes.d.ts b/dist/lib/schema/sync-indexes.d.ts new file mode 100644 index 0000000..c91c740 --- /dev/null +++ b/dist/lib/schema/sync-indexes.d.ts @@ -0,0 +1,9 @@ +import type { BUN_SQLITE_TableSchemaType, CreateDBSchemaParams } from "../../types"; +type Params = CreateDBSchemaParams & { + table: BUN_SQLITE_TableSchemaType; +}; +/** + * Sync indexes for a table + */ +export default function syncIndexes(params: Params): void; +export {}; diff --git a/dist/lib/schema/sync-indexes.js b/dist/lib/schema/sync-indexes.js new file mode 100644 index 0000000..f0827bd --- /dev/null +++ b/dist/lib/schema/sync-indexes.js @@ -0,0 +1,39 @@ +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; +import recordSchemaChange from "./record-schema-change"; +/** + * Sync indexes for a table + */ +export default function syncIndexes(params) { + const { db, table } = params; + if (!table.indexes || table.indexes.length === 0) { + return; + } + const existingIndexes = db + .query(`SELECT name FROM sqlite_master WHERE type='index' AND tbl_name=? AND name NOT LIKE 'sqlite_%'`) + .all(table.tableName).map((r) => r.name); + // Drop indexes not in schema + for (const indexName of existingIndexes) { + const stillExists = table.indexes.some((idx) => idx.indexName === indexName); + if (!stillExists) { + recordSchemaChange(params, { + description: `Drop index "${indexName}"`, + }); + db.run(`DROP INDEX IF EXISTS ${q(indexName)}`); + } + } + // Create new indexes + for (const index of table.indexes) { + if (!index.indexName || !index.indexTableFields?.length) { + continue; + } + if (!existingIndexes.includes(index.indexName)) { + recordSchemaChange(params, { + description: `Create index "${index.indexName}" on "${table.tableName}"`, + }); + const fields = index.indexTableFields + .map((f) => q(String(f))) + .join(", "); + db.run(`CREATE INDEX ${q(index.indexName)} ON ${q(table.tableName)} (${fields})`); + } + } +} diff --git a/dist/lib/schema/update-table.d.ts b/dist/lib/schema/update-table.d.ts new file mode 100644 index 0000000..1a5f4e2 --- /dev/null +++ b/dist/lib/schema/update-table.d.ts @@ -0,0 +1,13 @@ +import type { BUN_SQLITE_TableSchemaType, CreateDBSchemaParams } from "../../types"; +type Params = CreateDBSchemaParams & { + table: BUN_SQLITE_TableSchemaType; +}; +/** + * # Update an existing table + * @description SQLite has limited ALTER TABLE support, so a changed table is + * rebuilt: create a temp table with the new schema, copy the data over, + * drop the old table and rename the temp table. Unchanged tables are left + * alone. + */ +export default function updateTable(params: Params): void; +export {}; diff --git a/dist/lib/schema/update-table.js b/dist/lib/schema/update-table.js new file mode 100644 index 0000000..d53f430 --- /dev/null +++ b/dist/lib/schema/update-table.js @@ -0,0 +1,88 @@ +import { sqlQuoteIdent as q } from "../../utils/sql-safe"; +import buildCreateTableSql from "./build-create-table-sql"; +import checkIfTableExists from "./check-if-table-exists"; +import getTableColumns from "./get-table-columns"; +import recordSchemaChange from "./record-schema-change"; +import resolveTable from "./resolve-table"; +/** + * # Update an existing table + * @description SQLite has limited ALTER TABLE support, so a changed table is + * rebuilt: create a temp table with the new schema, copy the data over, + * drop the old table and rename the temp table. Unchanged tables are left + * alone. + */ +export default function updateTable(params) { + const { db, db_schema, table } = params; + if (table.isVector) { + recreateVectorTable(params); + return; + } + /** + * SQLite keeps the `CREATE TABLE` text (with the renamed table name), so + * an identical statement means the table already matches the schema. + */ + const current = db + .query(`SELECT sql FROM sqlite_master WHERE type='table' AND name=?`) + .get(table.tableName); + if (current?.sql === buildCreateTableSql({ db_schema, table })) { + return; + } + const tempTableName = `${table.tableName}_temp_${Date.now()}`; + // Columns are matched by name only: SQLite converts values to the new + // column affinity on copy, so a type change keeps the data instead of + // silently dropping the column. + const existingColumns = getTableColumns({ db, table_name: table.tableName }); + const schemaColumns = resolveTable({ table, db_schema }).fields.map((f) => f.fieldName); + const columnsToKeep = existingColumns + .map((c) => c.name) + .filter((name) => schemaColumns.includes(name)); + const columnsToDrop = existingColumns + .map((c) => c.name) + .filter((name) => !schemaColumns.includes(name)); + recordSchemaChange(params, { + description: `Rebuild table "${table.tableName}"`, + }); + for (const column of columnsToDrop) { + recordSchemaChange(params, { + description: `Drop column "${table.tableName}"."${column}"`, + destructive: true, + }); + } + // Create temp table with new schema + db.run(buildCreateTableSql({ + db_schema, + table: { ...table, tableName: tempTableName }, + })); + // Copy data if there are common columns + if (columnsToKeep.length > 0) { + const columnList = columnsToKeep.map(q).join(", "); + db.run(`INSERT INTO ${q(tempTableName)} (${columnList}) SELECT ${columnList} FROM ${q(table.tableName)}`); + } + db.run(`DROP TABLE ${q(table.tableName)}`); + db.run(`ALTER TABLE ${q(tempTableName)} RENAME TO ${q(table.tableName)}`); +} +/** + * Vector tables are only rebuilt when `recreate_vector_table` is set. + */ +function recreateVectorTable(params) { + const { db, db_schema, table, recreate_vector_table } = params; + if (!recreate_vector_table) { + return; + } + recordSchemaChange(params, { + description: `Recreate vector table "${table.tableName}"`, + }); + let existingRows = []; + if (checkIfTableExists({ db, table_name: table.tableName })) { + existingRows = db + .query(`SELECT * FROM ${q(table.tableName)}`) + .all(); + db.run(`DROP TABLE ${q(table.tableName)}`); + } + db.run(buildCreateTableSql({ db_schema, table })); + for (const row of existingRows) { + const columns = Object.keys(row); + const placeholders = columns.map(() => "?").join(", "); + db.run(`INSERT INTO ${q(table.tableName)} (${columns.map(q).join(", ")}) VALUES (${placeholders})`, Object.values(row)); + } +} diff --git a/dist/lib/sqlite/db-delete.d.ts b/dist/lib/sqlite/db-delete.d.ts index c1c63ba..1eb8bbc 100644 --- a/dist/lib/sqlite/db-delete.d.ts +++ b/dist/lib/sqlite/db-delete.d.ts @@ -1,4 +1,4 @@ -import type { APIResponseObject, ServerQueryParam } from "../../types"; +import type { APIResponseObject, BunSQLiteConfig, ServerQueryParam } from "../../types"; type Params; targetId?: number | string; + config?: BunSQLiteConfig; }; export default function DbDelete({ table, query, targetId, }: Params): Promise; +}, Table extends string = string>({ table, query, targetId, config, }: Params): Promise; export {}; diff --git a/dist/lib/sqlite/db-delete.js b/dist/lib/sqlite/db-delete.js index a23a327..4695486 100644 --- a/dist/lib/sqlite/db-delete.js +++ b/dist/lib/sqlite/db-delete.js @@ -1,34 +1,32 @@ -import DbClient from "."; +import dbHandler from "../db-handler"; import _ from "lodash"; import sqlGenerator from "../../utils/sql-generator"; -export default async function DbDelete({ table, query, targetId, }) { +import { sqlIdent } from "../../utils/sql-safe"; +export default async function DbDelete({ table, query, targetId, config, }) { let sqlObj = null; try { - let finalQuery = query || {}; - if (targetId) { - finalQuery = _.merge(finalQuery, { - query: { - id: { - value: String(targetId), - }, - }, - }); - } + const finalQuery = query || {}; sqlObj = sqlGenerator({ tableName: table, - genObject: finalQuery, + genObject: _.pick(finalQuery, ["query", "searchOperator"]), + targetId, }); - const whereClause = sqlObj.string.match(/WHERE .*/)?.[0]; + const whereClause = sqlObj.where.string; if (whereClause) { - let sql = `DELETE FROM ${table} ${whereClause}`; + let sql = `DELETE FROM ${sqlIdent(table, "table name")} ${whereClause}`; sqlObj.string = sql; - const res = DbClient.run(sql, sqlObj.values); + sqlObj.values = sqlObj.where.values; + const res = dbHandler({ + query: sql, + values: sqlObj.values, + config, + }); + if (!res.success) { + return { ...res, debug: { sqlObj } }; + } return { - success: Boolean(res.changes), - postInsertReturn: { - affectedRows: res.changes, - insertId: Number(res.lastInsertRowid), - }, + success: Boolean(res.postInsertReturn?.affectedRows), + postInsertReturn: res.postInsertReturn, debug: { sqlObj, }, diff --git a/dist/lib/sqlite/db-insert.d.ts b/dist/lib/sqlite/db-insert.d.ts index d5101ec..2a74b11 100644 --- a/dist/lib/sqlite/db-insert.d.ts +++ b/dist/lib/sqlite/db-insert.d.ts @@ -1,4 +1,4 @@ -import type { APIResponseObject } from "../../types"; +import type { APIResponseObject, BunSQLiteConfig } from "../../types"; type Params({ table, data, update_on_duplicate, }: Params): Promise; +}, Table extends string = string>({ table, data, update_on_duplicate, config, }: Params): Promise; export {}; diff --git a/dist/lib/sqlite/db-insert.js b/dist/lib/sqlite/db-insert.js index f5d37e8..8237dcc 100644 --- a/dist/lib/sqlite/db-insert.js +++ b/dist/lib/sqlite/db-insert.js @@ -1,7 +1,7 @@ -import DbClient from "."; +import dbHandler from "../db-handler"; import sqlInsertGenerator from "../../utils/sql-insert-generator"; import grabDuplicateSafeInsertSql from "../grab-duplicate-safe-insert-sql"; -export default async function DbInsert({ table, data, update_on_duplicate, }) { +export default async function DbInsert({ table, data, update_on_duplicate, config, }) { let sqlObj = null; try { const finalData = data.map((d) => ({ @@ -19,13 +19,17 @@ export default async function DbInsert({ table, data, update_on_duplicate, }) { sql = await grabDuplicateSafeInsertSql({ data, table, sql }); } (sqlObj || {}).query = sql; - const res = DbClient.run(sql, sqlObj?.values || []); + const res = dbHandler({ + query: sql, + values: sqlObj?.values || [], + config, + }); + if (!res.success) { + return { ...res, debug: { sqlObj } }; + } return { - success: Boolean(Number(res.lastInsertRowid)), - postInsertReturn: { - affectedRows: res.changes, - insertId: Number(res.lastInsertRowid), - }, + success: Boolean(res.postInsertReturn?.insertId), + postInsertReturn: res.postInsertReturn, debug: { sqlObj, }, diff --git a/dist/lib/sqlite/db-schema-manager.d.ts b/dist/lib/sqlite/db-schema-manager.d.ts deleted file mode 100644 index dbd446c..0000000 --- a/dist/lib/sqlite/db-schema-manager.d.ts +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bun -import type { BUN_SQLITE_DatabaseSchemaType } from "../../types"; -declare class SQLiteSchemaManager { - private db; - private db_manager_table_name; - private recreate_vector_table; - private db_schema; - constructor({ schema, recreate_vector_table, }: { - schema: BUN_SQLITE_DatabaseSchemaType; - recreate_vector_table?: boolean; - }); - private createDbManagerTable; - private insertDbManagerTable; - private removeDbManagerTable; - /** - * Main synchronization method - */ - syncSchema(): Promise; - /** - * Get list of existing tables in the database - */ - private getExistingTables; - /** - * Drop tables that are no longer in the schema - */ - private dropRemovedTables; - /** - * Sync a single table (create or update) - */ - private syncTable; - /** - * Create a new table - */ - private createTable; - /** - * Update an existing table - */ - private updateTable; - /** - * Get existing columns for a table - */ - private getTableColumns; - /** - * Add a new column to existing table - */ - private addColumn; - private checkIfTableExists; - /** - * Recreate table (for complex schema changes) - */ - private recreateTable; - /** - * Build column definition SQL - */ - private buildColumnDefinition; - /** - * Map DSQL data types to SQLite types - */ - private mapDataType; - /** - * Build foreign key constraint - */ - private buildForeignKeyConstraint; - /** - * Sync indexes for a table - */ - private syncIndexes; - /** - * Close database connection - */ - close(): void; -} -export { SQLiteSchemaManager }; diff --git a/dist/lib/sqlite/db-schema-manager.js b/dist/lib/sqlite/db-schema-manager.js deleted file mode 100644 index b306fc7..0000000 --- a/dist/lib/sqlite/db-schema-manager.js +++ /dev/null @@ -1,425 +0,0 @@ -#!/usr/bin/env bun -import { Database } from "bun:sqlite"; -import _ from "lodash"; -import DbClient from "."; -import { AppData } from "../../data/app-data"; -import { readLiveSchema } from "../../functions/live-schema"; -// Schema Manager Class -class SQLiteSchemaManager { - db; - db_manager_table_name; - recreate_vector_table; - db_schema; - constructor({ schema, recreate_vector_table = false, }) { - this.db = DbClient; - this.db_manager_table_name = AppData["DbSchemaManagerTableName"]; - this.db.run("PRAGMA foreign_keys = ON;"); - this.recreate_vector_table = recreate_vector_table; - this.createDbManagerTable(); - this.db_schema = schema; - } - createDbManagerTable() { - this.db.run(` - CREATE TABLE IF NOT EXISTS ${this.db_manager_table_name} ( - table_name TEXT NOT NULL, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL - ) - `); - } - insertDbManagerTable(tableName) { - this.db.run(`INSERT INTO ${this.db_manager_table_name} (table_name,created_at,updated_at) VALUES (?, ?, ?)`, [tableName, Date.now(), Date.now()]); - } - removeDbManagerTable(tableName) { - this.db.run(`DELETE FROM ${this.db_manager_table_name} WHERE table_name = ?`, [tableName]); - } - /** - * Main synchronization method - */ - async syncSchema() { - console.log("Starting schema synchronization..."); - const existingTables = this.getExistingTables(); - const schemaTables = this.db_schema.tables.map((t) => t.tableName); - // 2. Create or update tables - for (const table of this.db_schema.tables) { - await this.syncTable(table, existingTables); - } - // 1. Drop tables that no longer exist in schema - await this.dropRemovedTables(existingTables, schemaTables); - console.log("Schema synchronization complete!"); - } - /** - * Get list of existing tables in the database - */ - getExistingTables() { - let sql = `SELECT table_name FROM ${this.db_manager_table_name}`; - const query = this.db.query(sql); - const results = query.all(); - return results.map((r) => r.table_name); - } - /** - * Drop tables that are no longer in the schema - */ - async dropRemovedTables(existingTables, schemaTables) { - console.log(`Cleaning up tables ...`); - const tablesToDrop = existingTables.filter((t) => !schemaTables.includes(t) && - !schemaTables.find((scT) => t.startsWith(scT + "_"))); - const currentSchema = readLiveSchema(); - if (currentSchema?.tables?.[0]) { - for (let i = 0; i < currentSchema.tables.length; i++) { - const table = currentSchema.tables[i]; - if (!table?.tableName) - continue; - const does_table_exist = schemaTables.find((t) => t == table.tableName); - if (!does_table_exist) { - tablesToDrop.push(table.tableName); - } - } - } - for (const tableName of tablesToDrop) { - console.log(`Dropping table: ${tableName}`); - this.db.run(`DROP TABLE IF EXISTS "${tableName}"`); - this.db.run(`DELETE FROM ${this.db_manager_table_name} WHERE table_name = "${tableName}"`); - } - } - /** - * Sync a single table (create or update) - */ - async syncTable(table, existingTables) { - let tableExists = existingTables.includes(table.tableName); - // Handle table rename - if (table.tableNameOld && table.tableNameOld !== table.tableName) { - if (existingTables.includes(table.tableNameOld)) { - console.log(`Renaming table: ${table.tableNameOld} -> ${table.tableName}`); - this.db.run(`ALTER TABLE "${table.tableNameOld}" RENAME TO "${table.tableName}"`); - this.insertDbManagerTable(table.tableName); - this.removeDbManagerTable(table.tableNameOld); - tableExists = true; - } - } - if (!tableExists) { - // Create new table - await this.createTable(table); - this.insertDbManagerTable(table.tableName); - } - else { - // Update existing table - await this.updateTable(table); - } - // Sync indexes - await this.syncIndexes(table); - } - /** - * Create a new table - */ - async createTable(table) { - if (!table.tableName.match(/_temp_\d+$/)) { - console.log(`Creating table: ${table.tableName}`); - } - let new_table = _.cloneDeep(table); - if (new_table.parentTableName) { - const parent_table = this.db_schema.tables.find((t) => t.tableName === new_table.parentTableName); - if (!parent_table) { - throw new Error(`Parent table \`${new_table.parentTableName}\` not found for \`${new_table.tableName}\``); - } - new_table = _.merge(parent_table, { - tableName: new_table.tableName, - tableDescription: new_table.tableDescription, - }); - } - const columns = []; - const foreignKeys = []; - for (const field of new_table.fields) { - const columnDef = this.buildColumnDefinition(field, table.isVector); - columns.push(columnDef); - if (field.foreignKey && !table.isVector) { - foreignKeys.push(this.buildForeignKeyConstraint(field)); - } - } - // Add unique constraints - if (new_table.uniqueConstraints) { - for (const constraint of new_table.uniqueConstraints) { - if (constraint.constraintTableFields && - constraint.constraintTableFields.length > 0) { - const fields = constraint.constraintTableFields - .map((f) => `"${f.value}"`) - .join(", "); - const constraintName = constraint.constraintName || - `unique_${fields.replace(/"/g, "")}`; - columns.push(`CONSTRAINT "${constraintName}" UNIQUE (${fields})`); - } - } - } - const allConstraints = [...columns, ...foreignKeys]; - const sql = new_table.isVector - ? `CREATE VIRTUAL TABLE "${new_table.tableName}" USING ${new_table.vectorType || "vec0"}(${allConstraints.join(", ")})` - : `CREATE TABLE "${new_table.tableName}" (${allConstraints.join(", ")})`; - this.db.run(sql); - } - /** - * Update an existing table - */ - async updateTable(table) { - console.log(`Updating table: ${table.tableName}`); - // const existingColumns = this.getTableColumns(table.tableName); - // const schemaColumns = table.fields.map((f) => f.fieldName || ""); - // // SQLite has limited ALTER TABLE support - // // We need to use the recreation strategy for complex changes - // const columnsToAdd = table.fields.filter( - // (f) => - // f.fieldName && - // !existingColumns.find( - // (c) => - // c.name == f.fieldName && c.type == this.mapDataType(f), - // ), - // ); - // // const columnsToRemove = existingColumns.filter( - // // (c) => !schemaColumns.includes(c.name), - // // ); - // // const columnsToUpdate = table.fields.filter( - // // (f) => - // // f.fieldName && - // // f.updatedField && - // // existingColumns.find( - // // (c) => - // // c.name == f.fieldName && c.type == this.mapDataType(f), - // // ), - // // ); - // for (const field of columnsToAdd) { - // await this.addColumn(table.tableName, field); - // } - // // // Simple case: only adding columns - // // if (columnsToRemove.length === 0 && columnsToUpdate.length === 0) { - // // for (const field of columnsToAdd) { - // // await this.addColumn(table.tableName, field); - // // } - // // } else { - // // // Complex case: need to recreate table - // // } - await this.recreateTable(table); - } - /** - * Get existing columns for a table - */ - getTableColumns(tableName) { - const query = this.db.query(`PRAGMA table_info("${tableName}")`); - const results = query.all(); - return results; - } - /** - * Add a new column to existing table - */ - async addColumn(tableName, field) { - console.log(`Adding column: ${tableName}.${field.fieldName}`); - const columnDef = this.buildColumnDefinition(field); - // Remove PRIMARY KEY and UNIQUE constraints for ALTER TABLE ADD COLUMN - const cleanDef = columnDef - .replace(/PRIMARY KEY/gi, "") - .replace(/AUTOINCREMENT/gi, "") - .replace(/UNIQUE/gi, "") - .trim(); - const sql = `ALTER TABLE "${tableName}" ADD COLUMN ${cleanDef}`; - this.db.run(sql); - } - checkIfTableExists(table) { - const tableExists = this.db - .query(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`) - .get(table); - return Boolean(tableExists?.name); - } - /** - * Recreate table (for complex schema changes) - */ - async recreateTable(table) { - if (table.isVector) { - if (!this.recreate_vector_table) { - return; - } - const does_table_exist = this.checkIfTableExists(table.tableName); - let existingRows = []; - if (does_table_exist) { - existingRows = this.db - .query(`SELECT * FROM "${table.tableName}"`) - .all(); - this.db.run(`DROP TABLE "${table.tableName}"`); - } - await this.createTable(table); - if (existingRows.length > 0) { - for (let i = 0; i < existingRows.length; i++) { - const row = existingRows[i]; - if (!row) - continue; - const columns = Object.keys(row); - const placeholders = columns.map(() => "?").join(", "); - this.db.run(`INSERT INTO "${table.tableName}" (${columns.join(", ")}) VALUES (${placeholders})`, Object.values(row)); - } - } - return; - } - const tempTableName = `${table.tableName}_temp_${Date.now()}`; - // Get existing data - const existingColumns = this.getTableColumns(table.tableName); - const columnsToKeep = table.fields - .filter((f) => f.fieldName && - existingColumns.find((c) => c.name == f.fieldName && - c.type == this.mapDataType(f))) - .map((f) => f.fieldName); - // Create temp table with new schema - const tempTable = { ...table, tableName: tempTableName }; - await this.createTable(tempTable); - // Copy data if there are common columns - if (columnsToKeep.length > 0) { - const columnList = columnsToKeep.map((c) => `"${c}"`).join(", "); - this.db.run(`INSERT INTO "${tempTableName}" (${columnList}) SELECT ${columnList} FROM "${table.tableName}"`); - } - // Drop old table - this.db.run(`DROP TABLE "${table.tableName}"`); - // Rename temp table - this.db.run(`ALTER TABLE "${tempTableName}" RENAME TO "${table.tableName}"`); - } - /** - * Build column definition SQL - */ - buildColumnDefinition(field, is_vector) { - if (!field.fieldName) { - throw new Error("Field name is required"); - } - const fieldName = field.sideCar - ? `+${field.fieldName}` - : `${field.fieldName}`; - const parts = [fieldName]; - // Data type mapping - const dataType = this.mapDataType(field); - parts.push(dataType); - // Primary key - if (field.primaryKey) { - parts.push("PRIMARY KEY"); - if (field.autoIncrement) { - parts.push("AUTOINCREMENT"); - } - } - // Not null - if (field.notNullValue || field.primaryKey) { - if (!field.primaryKey) { - parts.push("NOT NULL"); - } - } - // Unique - if (field.unique && !field.primaryKey) { - parts.push("UNIQUE"); - } - if (field.vectorDistanceMetric) { - parts.push(`distance_metric=${field.vectorDistanceMetric}`); - } - // Default value - if (field.defaultValue !== undefined) { - if (typeof field.defaultValue === "string") { - parts.push( - // Escape single quotes by doubling them to prevent SQL injection and wrap in single quotes - `DEFAULT '${field.defaultValue.replace(/'/g, "''")}'`); - } - else { - parts.push(`DEFAULT ${field.defaultValue}`); - } - } - else if (field.defaultValueLiteral) { - parts.push(`DEFAULT ${field.defaultValueLiteral}`); - } - return parts.join(" "); - } - /** - * Map DSQL data types to SQLite types - */ - mapDataType(field) { - const dataType = field.dataType?.toLowerCase() || "text"; - const vectorSize = field.vectorSize || 1536; - // Vector Embeddings - if (field.isVector) { - return `FLOAT[${vectorSize}]`; - } - // Integer types - if (dataType.includes("int") || - dataType === "bigint" || - dataType === "smallint" || - dataType === "tinyint") { - return "INTEGER"; - } - // Real/Float types - if (dataType.includes("real") || - dataType.includes("float") || - dataType.includes("double") || - dataType === "decimal" || - dataType === "numeric") { - return "REAL"; - } - // Blob types - if (dataType.includes("blob") || dataType.includes("binary")) { - return "BLOB"; - } - // Boolean - if (dataType === "boolean" || dataType === "bool") { - return "INTEGER"; // SQLite uses INTEGER for boolean (0/1) - } - // Date/Time types - if (dataType.includes("date") || dataType.includes("time")) { - return "TEXT"; // SQLite stores dates as TEXT or INTEGER - } - // Default to TEXT for all text-based types - return "TEXT"; - } - /** - * Build foreign key constraint - */ - buildForeignKeyConstraint(field) { - const fk = field.foreignKey; - let constraint = `FOREIGN KEY ("${field.fieldName}") REFERENCES "${fk.destinationTableName}"("${fk.destinationTableColumnName}")`; - if (fk.cascadeDelete) { - constraint += " ON DELETE CASCADE"; - } - if (fk.cascadeUpdate) { - constraint += " ON UPDATE CASCADE"; - } - return constraint; - } - /** - * Sync indexes for a table - */ - async syncIndexes(table) { - if (!table.indexes || table.indexes.length === 0) { - return; - } - // Get existing indexes - const query = this.db.query(`SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='${table.tableName}' AND name NOT LIKE 'sqlite_%'`); - const existingIndexes = query.all().map((r) => r.name); - // Drop indexes not in schema - for (const indexName of existingIndexes) { - const stillExists = table.indexes.some((idx) => idx.indexName === indexName); - if (!stillExists) { - console.log(`Dropping index: ${indexName}`); - this.db.run(`DROP INDEX IF EXISTS "${indexName}"`); - } - } - // Create new indexes - for (const index of table.indexes) { - if (!index.indexName || - !index.indexTableFields || - index.indexTableFields.length === 0) { - continue; - } - if (!existingIndexes.includes(index.indexName)) { - console.log(`Creating index: ${index.indexName}`); - const fields = index.indexTableFields - .map((f) => `"${f}"`) - .join(", "); - this.db.run(`CREATE INDEX "${index.indexName}" ON "${table.tableName}" (${fields})`); - } - } - } - /** - * Close database connection - */ - close() { - this.db.close(); - } -} -export { SQLiteSchemaManager }; diff --git a/dist/lib/sqlite/db-schema-to-typedef.js b/dist/lib/sqlite/db-schema-to-typedef.js index bb2eabf..ba338b2 100644 --- a/dist/lib/sqlite/db-schema-to-typedef.js +++ b/dist/lib/sqlite/db-schema-to-typedef.js @@ -18,7 +18,7 @@ export default function dbSchemaToType({ config, dbSchema, }) { if (final_table.parentTableName) { const parent_table = dbTablesSchemas.find((t) => t.tableName === final_table.parentTableName); if (parent_table) { - final_table = _.merge(parent_table, { + final_table = _.merge(_.cloneDeep(parent_table), { tableName: final_table.tableName, tableDescription: final_table.tableDescription, }); diff --git a/dist/lib/sqlite/db-select.d.ts b/dist/lib/sqlite/db-select.d.ts index ed27786..8274a47 100644 --- a/dist/lib/sqlite/db-select.d.ts +++ b/dist/lib/sqlite/db-select.d.ts @@ -1,4 +1,4 @@ -import type { APIResponseObject, ServerQueryParam } from "../../types"; +import type { APIResponseObject, BunSQLiteConfig, ServerQueryParam } from "../../types"; type Params({ table, query, count, targetId, }: Params): Promise>; +}, Table extends string = string>({ table, query, count, targetId, config, }: Params): Promise>; export {}; diff --git a/dist/lib/sqlite/db-select.js b/dist/lib/sqlite/db-select.js index 8b1674f..7de22e8 100644 --- a/dist/lib/sqlite/db-select.js +++ b/dist/lib/sqlite/db-select.js @@ -1,27 +1,24 @@ -import mysql from "mysql"; -import DbClient from "."; -import _ from "lodash"; +import dbHandler from "../db-handler"; import sqlGenerator from "../../utils/sql-generator"; -export default async function DbSelect({ table, query, count, targetId, }) { +export default async function DbSelect({ table, query, count, targetId, config, }) { let sqlObj = null; try { - let finalQuery = query || {}; - if (targetId) { - finalQuery = _.merge(finalQuery, { - query: { - id: { - value: String(targetId), - }, - }, - }); - } + const finalQuery = query || {}; sqlObj = sqlGenerator({ tableName: table, genObject: finalQuery, + targetId, }); - let sql = mysql.format(sqlObj.string, sqlObj.values); - const res = DbClient.query(sql); - const batchRes = res.all(); + const sql = sqlObj.string; + const res = dbHandler({ + query: sql, + values: sqlObj.values, + config, + }); + if (!res.success) { + return { ...res, debug: { sqlObj, sql } }; + } + const batchRes = res.payload || []; let resp = { success: Boolean(batchRes[0]), payload: batchRes, @@ -32,16 +29,28 @@ export default async function DbSelect({ table, query, count, targetId, }) { }, }; if (count) { - let count_sql_object = sqlGenerator({ + const count_sql_object = sqlGenerator({ tableName: table, genObject: finalQuery, + targetId, count, }); - let count_sql = mysql.format(count_sql_object.string, count_sql_object.values); - count_sql = `SELECT COUNT(*) FROM (${count_sql}) as c`; - const count_res = DbClient.query(count_sql).all(); - const count_val = count_res[0]?.["COUNT(*)"]; - resp["count"] = Number(count_val); + const count_sql = `SELECT COUNT(*) AS count FROM (${count_sql_object.string}) AS c`; + const count_res = dbHandler({ + query: count_sql, + values: count_sql_object.values, + config, + }); + if (!count_res.success) { + return { + ...resp, + success: false, + error: count_res.error, + msg: "Database count failed", + debug: { ...resp.debug, count_sql }, + }; + } + resp["count"] = Number(count_res.singleRes?.count); resp["debug"]["count_sql"] = count_sql; } return resp; diff --git a/dist/lib/sqlite/db-sql.d.ts b/dist/lib/sqlite/db-sql.d.ts index e528035..c40b68b 100644 --- a/dist/lib/sqlite/db-sql.d.ts +++ b/dist/lib/sqlite/db-sql.d.ts @@ -1,11 +1,12 @@ -import type { APIResponseObject, SQLInsertGenValueType } from "../../types"; +import type { APIResponseObject, BunSQLiteConfig, SQLInsertGenValueType } from "../../types"; type Params = { sql: string; values?: SQLInsertGenValueType[]; + config?: BunSQLiteConfig; }; export default function DbSQL({ sql, values }: Params): Promise>; +}>({ sql, values, config }: Params): Promise>; export {}; diff --git a/dist/lib/sqlite/db-sql.js b/dist/lib/sqlite/db-sql.js index 33fd427..8cefedc 100644 --- a/dist/lib/sqlite/db-sql.js +++ b/dist/lib/sqlite/db-sql.js @@ -1,34 +1,18 @@ -import DbClient from "."; -import _ from "lodash"; -export default async function DbSQL({ sql, values }) { - try { - const trimmed_sql = sql.trim(); - const res = trimmed_sql.match(/^select/i) - ? DbClient.query(trimmed_sql).all(...(values || [])) - : DbClient.run(trimmed_sql, values || []); - return { - success: true, - payload: Array.isArray(res) ? res : undefined, - singleRes: Array.isArray(res) ? res?.[0] : undefined, - postInsertReturn: Array.isArray(res) - ? undefined - : { - affectedRows: res.changes, - insertId: Number(res.lastInsertRowid), - }, - debug: { - sqlObj: { - sql: trimmed_sql, - values, - }, - sql, +import dbHandler from "../db-handler"; +export default async function DbSQL({ sql, values, config }) { + const trimmed_sql = sql.trim(); + const res = dbHandler({ query: trimmed_sql, values, config }); + if (!res.success) { + return res; + } + return { + ...res, + debug: { + sqlObj: { + sql: trimmed_sql, + values, }, - }; - } - catch (error) { - return { - success: false, - error: error.message, - }; - } + sql, + }, + }; } diff --git a/dist/lib/sqlite/db-update.d.ts b/dist/lib/sqlite/db-update.d.ts index 68bff0c..a68c2ab 100644 --- a/dist/lib/sqlite/db-update.d.ts +++ b/dist/lib/sqlite/db-update.d.ts @@ -1,4 +1,4 @@ -import type { APIResponseObject, ServerQueryParam } from "../../types"; +import type { APIResponseObject, BunSQLiteConfig, ServerQueryParam } from "../../types"; type Params; targetId?: number | string; + config?: BunSQLiteConfig; }; export default function DbUpdate({ table, data, query, targetId, }: Params): Promise; +}, Table extends string = string>({ table, data, query, targetId, config, }: Params): Promise; export {}; diff --git a/dist/lib/sqlite/db-update.js b/dist/lib/sqlite/db-update.js index 1f45e86..c4b5e29 100644 --- a/dist/lib/sqlite/db-update.js +++ b/dist/lib/sqlite/db-update.js @@ -1,27 +1,24 @@ -import DbClient from "."; +import dbHandler from "../db-handler"; import _ from "lodash"; import sqlGenerator from "../../utils/sql-generator"; -export default async function DbUpdate({ table, data, query, targetId, }) { - let sqlObj = { string: "", values: [] }; +import { sqlValue } from "../../utils/sql-insert-generator"; +import { sqlIdent } from "../../utils/sql-safe"; +export default async function DbUpdate({ table, data, query, targetId, config, }) { + let sqlObj = { + string: "", + values: [], + }; try { - let finalQuery = query || {}; - if (targetId) { - finalQuery = _.merge(finalQuery, { - query: { - id: { - value: String(targetId), - }, - }, - }); - } + const finalQuery = query || {}; const sqlQueryObj = sqlGenerator({ tableName: table, - genObject: finalQuery, + genObject: _.pick(finalQuery, ["query", "searchOperator"]), + targetId, }); let values = []; - const whereClause = sqlQueryObj.string.match(/WHERE .*/)?.[0]; + const whereClause = sqlQueryObj.where.string; if (whereClause) { - let sql = `UPDATE ${table} SET`; + let sql = `UPDATE ${sqlIdent(table, "table name")} SET`; const finalData = { updated_at: Date.now(), ...data, @@ -32,26 +29,23 @@ export default async function DbUpdate({ table, data, query, targetId, }) { if (!key) continue; const isLast = i == keys.length - 1; - sql += ` ${key}=?`; - const value = finalData[key]; - values.push(typeof value == "number" || typeof value == "string" - ? value - : value || null); + sql += ` ${sqlIdent(key, "column name")}=?`; + values.push(sqlValue(finalData[key])); if (!isLast) { sql += `,`; } } sql += ` ${whereClause}`; - values = [...values, ...sqlQueryObj.values]; + values = [...values, ...sqlQueryObj.where.values]; sqlObj.string = sql; sqlObj.values = values; - const res = DbClient.run(sql, values); + const res = dbHandler({ query: sql, values, config }); + if (!res.success) { + return { ...res, debug: { sqlObj } }; + } return { - success: Boolean(res.changes), - postInsertReturn: { - affectedRows: res.changes, - insertId: Number(res.lastInsertRowid), - }, + success: Boolean(res.postInsertReturn?.affectedRows), + postInsertReturn: res.postInsertReturn, debug: { sqlObj, }, diff --git a/dist/lib/sqlite/index.d.ts b/dist/lib/sqlite/index.d.ts deleted file mode 100644 index 438b68a..0000000 --- a/dist/lib/sqlite/index.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { Database } from "bun:sqlite"; -declare const DbClient: Database; -export default DbClient; diff --git a/dist/lib/sqlite/index.js b/dist/lib/sqlite/index.js deleted file mode 100644 index 8f48803..0000000 --- a/dist/lib/sqlite/index.js +++ /dev/null @@ -1,20 +0,0 @@ -import { Database } from "bun:sqlite"; -import * as sqliteVec from "sqlite-vec"; -import grabDirNames from "../../data/grab-dir-names"; -import init from "../../functions/init"; -import grabDBDir from "../../utils/grab-db-dir"; -const { ROOT_DIR } = grabDirNames(); -const { config } = await init(); -let db_dir = ROOT_DIR; -if (config.db_dir) { - db_dir = config.db_dir; -} -const { db_file_path } = grabDBDir({ config }); -const DbClient = new Database(db_file_path, { - create: true, -}); -sqliteVec.load(DbClient); -if (config.wal_mode) { - DbClient.run("PRAGMA journal_mode = WAL;"); -} -export default DbClient; diff --git a/dist/lib/sqlite/schema.d.ts b/dist/lib/sqlite/schema.d.ts deleted file mode 100644 index 355768e..0000000 --- a/dist/lib/sqlite/schema.d.ts +++ /dev/null @@ -1,2 +0,0 @@ -import type { BUN_SQLITE_DatabaseSchemaType } from "../../types"; -export declare const DbSchema: BUN_SQLITE_DatabaseSchemaType; diff --git a/dist/lib/sqlite/schema.js b/dist/lib/sqlite/schema.js deleted file mode 100644 index 5778e50..0000000 --- a/dist/lib/sqlite/schema.js +++ /dev/null @@ -1,5 +0,0 @@ -import _ from "lodash"; -export const DbSchema = { - dbName: "travis-ai", - tables: [], -}; diff --git a/dist/types/index.d.ts b/dist/types/index.d.ts index 33bf1e8..12cc939 100644 --- a/dist/types/index.d.ts +++ b/dist/types/index.d.ts @@ -1353,14 +1353,19 @@ export type BunSQLiteConfig = { db_name: string; /** * The Name of the Database Schema File. Eg `db_schema.ts`. This is - * relative to `db_dir`, or root dir if `db_dir` is not provided + * relative to `db_dir`, or root dir if `db_dir` is not provided. When + * omitted, `schema.ts|js|json|yaml|yml` in `db_dir` is used. */ - db_schema_file_name: string; + db_schema_file_name?: string; /** * The Directory for backups. Relative to db_dir. */ db_backup_dir?: string; max_backups?: number; + /** + * Max number of archives kept in `{db_dir}/.exports` by `export` + */ + max_exports?: number; /** * The Root Directory for the DB file and schema */ @@ -1382,6 +1387,35 @@ export type BunSQLiteConfigReturn = { config: BunSQLiteConfig; dbSchema: BUN_SQLITE_DatabaseSchemaType; }; +/** + * Params shared by the schema sync functions in `lib/schema`. + */ +export type CreateDBSchemaParams = { + db: import("bun:sqlite").Database; + db_schema: BUN_SQLITE_DatabaseSchemaType; + recreate_vector_table?: boolean; + /** + * Run the sync inside a transaction and roll it back. The planned + * changes are recorded in `changes`. + */ + dry_run?: boolean; + /** + * Collects every change made (or planned) by the sync + */ + changes?: SchemaChange[]; + /** + * Destructive changes that may run. `true` (or unset) allows all of + * them; otherwise only changes whose `description` is in the set. + */ + allow_destructive?: true | Set; +}; +/** + * A change made by the schema sync. Destructive changes discard data. + */ +export type SchemaChange = { + description: string; + destructive?: boolean; +}; /** * Default fields automatically suggested for new tables. */ diff --git a/dist/utils/export-archive.d.ts b/dist/utils/export-archive.d.ts new file mode 100644 index 0000000..30a69c9 --- /dev/null +++ b/dist/utils/export-archive.d.ts @@ -0,0 +1,25 @@ +export declare const ExportArchiveMembers: { + readonly DbFileName: "database.sqlite"; + readonly SchemaFileName: "schema"; +}; +export type ExportArchiveContents = { + db: Uint8Array; + schema: string; + /** Archive member basename, e.g. schema.ts / schema.json / schema.yaml */ + schemaFileName: string; +}; +export declare function isArchivePath(filePath: string): boolean; +export declare function isSqlPath(filePath: string): boolean; +export declare function isSQLiteFile(bytes: Uint8Array): boolean; +/** + * Create a portable export archive (tar / tar.gz / zip) with the database + * file + schema file. + */ +export declare function writeExportArchive({ contents, outPath, }: { + contents: ExportArchiveContents; + outPath: string; +}): Promise; +/** + * Read a portable export archive (tar / tar.gz / zip). + */ +export declare function readExportArchive(archivePath: string): Promise; diff --git a/dist/utils/export-archive.js b/dist/utils/export-archive.js new file mode 100644 index 0000000..a56f245 --- /dev/null +++ b/dist/utils/export-archive.js @@ -0,0 +1,173 @@ +import fs from "fs"; +import path from "path"; +import { AppData } from "../data/app-data"; +import grabDirNames from "../data/grab-dir-names"; +import { isSchemaFileName } from "./resolve-and-load-data-file"; +export const ExportArchiveMembers = { + DbFileName: "database.sqlite", + SchemaFileName: AppData.DbSchemaFileName, +}; +const ARCHIVE_EXTENSIONS = [".tar.gz", ".tgz", ".tar", ".zip"]; +/** Every SQLite database file starts with this header */ +const SQLITE_HEADER = "SQLite format 3\0"; +export function isArchivePath(filePath) { + const lower = filePath.toLowerCase(); + return ARCHIVE_EXTENSIONS.some((ext) => lower.endsWith(ext)); +} +export function isSqlPath(filePath) { + return filePath.toLowerCase().endsWith(".sql"); +} +export function isSQLiteFile(bytes) { + return (new TextDecoder().decode(bytes.subarray(0, SQLITE_HEADER.length)) === + SQLITE_HEADER); +} +/** + * Create a portable export archive (tar / tar.gz / zip) with the database + * file + schema file. + */ +export async function writeExportArchive({ contents, outPath, }) { + const lower = outPath.toLowerCase(); + if (lower.endsWith(".zip")) { + await writeZipArchive({ contents, outPath }); + return; + } + const members = { + [ExportArchiveMembers.DbFileName]: contents.db, + [contents.schemaFileName]: contents.schema, + }; + const gzip = lower.endsWith(".gz") || lower.endsWith(".tgz"); + if (gzip) { + await Bun.Archive.write(outPath, members, { compress: "gzip" }); + } + else { + await Bun.Archive.write(outPath, members); + } +} +/** + * Read a portable export archive (tar / tar.gz / zip). + */ +export async function readExportArchive(archivePath) { + if (archivePath.toLowerCase().endsWith(".zip")) { + return readZipArchive(archivePath); + } + const bytes = await Bun.file(archivePath).bytes(); + const archive = new Bun.Archive(bytes); + const files = await archive.files(); + let db = null; + let schemaEntry = null; + for (const [entry, file] of files) { + const base = path.basename(entry); + if (!db && base === ExportArchiveMembers.DbFileName) { + db = await file.bytes(); + } + else if (!schemaEntry && isSchemaFileName(base)) { + schemaEntry = { + content: await file.text(), + fileName: schemaMemberName(base), + }; + } + } + return validateContents({ db, schemaEntry }); +} +function schemaMemberName(base) { + return base === AppData.DbSchemaFileName ? "schema.ts" : base; +} +function validateContents({ db, schemaEntry, }) { + if (!db) { + throw new Error(`Archive is missing the database file (expected \`${ExportArchiveMembers.DbFileName}\`)`); + } + if (!isSQLiteFile(db)) { + throw new Error(`\`${ExportArchiveMembers.DbFileName}\` in the archive is not a SQLite database`); + } + if (!schemaEntry) { + throw new Error(`Archive is missing schema file (expected \`${AppData.DbSchemaFileName}.ts|js|json|yaml|yml\`)`); + } + return { + db, + schema: schemaEntry.content, + schemaFileName: schemaEntry.fileName, + }; +} +async function runCommand(cmd, cwd) { + const proc = Bun.spawn(cmd, { + cwd, + stdout: "pipe", + stderr: "pipe", + }); + const [stderr, exitCode] = await Promise.all([ + new Response(proc.stderr).text(), + proc.exited, + ]); + if (exitCode !== 0) { + throw new Error(`${cmd[0]} failed (exit ${exitCode}): ${stderr || "unknown error"}. Ensure \`${cmd[0]}\` is installed, or use .tar.gz.`); + } +} +function makeTempDir(prefix) { + const { BUN_SQLITE_TEMP_DIR } = grabDirNames(); + const tempDir = path.join(BUN_SQLITE_TEMP_DIR, `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2)}`); + fs.mkdirSync(tempDir, { recursive: true }); + return tempDir; +} +async function writeZipArchive({ contents, outPath, }) { + const tempDir = makeTempDir("export"); + try { + fs.writeFileSync(path.join(tempDir, ExportArchiveMembers.DbFileName), contents.db); + fs.writeFileSync(path.join(tempDir, contents.schemaFileName), contents.schema, "utf-8"); + await runCommand([ + "zip", + "-q", + "-j", + path.resolve(outPath), + ExportArchiveMembers.DbFileName, + contents.schemaFileName, + ], tempDir); + } + finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} +async function readZipArchive(archivePath) { + const tempDir = makeTempDir("import"); + try { + await runCommand([ + "unzip", + "-q", + "-o", + path.resolve(archivePath), + "-d", + tempDir, + ]); + let db = null; + let schemaEntry = null; + const stack = [tempDir]; + while (stack.length) { + const current = stack.pop(); + for (const entry of fs.readdirSync(current, { + withFileTypes: true, + })) { + const full = path.join(current, entry.name); + // Never follow symlinks from an extracted (untrusted) archive + if (entry.isDirectory()) { + stack.push(full); + } + else if (!entry.isFile()) { + continue; + } + else if (!db && + entry.name === ExportArchiveMembers.DbFileName) { + db = new Uint8Array(fs.readFileSync(full)); + } + else if (!schemaEntry && isSchemaFileName(entry.name)) { + schemaEntry = { + content: fs.readFileSync(full, "utf-8"), + fileName: schemaMemberName(entry.name), + }; + } + } + } + return validateContents({ db, schemaEntry }); + } + finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} diff --git a/dist/utils/grab-db-dir.d.ts b/dist/utils/grab-db-dir.d.ts index b01b221..edada94 100644 --- a/dist/utils/grab-db-dir.d.ts +++ b/dist/utils/grab-db-dir.d.ts @@ -5,6 +5,7 @@ type Params = { export default function grabDBDir({ config }: Params): { db_dir: string; backup_dir: string; + export_dir: string; db_file_path: string; }; export {}; diff --git a/dist/utils/grab-db-dir.js b/dist/utils/grab-db-dir.js index 89f5a77..9b379d4 100644 --- a/dist/utils/grab-db-dir.js +++ b/dist/utils/grab-db-dir.js @@ -5,10 +5,11 @@ export default function grabDBDir({ config }) { const { ROOT_DIR } = grabDirNames(); let db_dir = ROOT_DIR; if (config.db_dir) { - db_dir = config.db_dir; + db_dir = path.resolve(ROOT_DIR, config.db_dir); } const backup_dir_name = config.db_backup_dir || AppData["DefaultBackupDirName"]; const backup_dir = path.resolve(db_dir, backup_dir_name); + const export_dir = path.resolve(db_dir, AppData["DefaultExportDirName"]); const db_file_path = path.resolve(db_dir, config.db_name); - return { db_dir, backup_dir, db_file_path }; + return { db_dir, backup_dir, export_dir, db_file_path }; } diff --git a/dist/utils/grab-db-schema.d.ts b/dist/utils/grab-db-schema.d.ts index 8f9df70..74359b2 100644 --- a/dist/utils/grab-db-schema.d.ts +++ b/dist/utils/grab-db-schema.d.ts @@ -1 +1 @@ -export default function grabDbSchema(): Promise; +export default function grabDbSchema(): Promise; diff --git a/dist/utils/grab-db-schema.js b/dist/utils/grab-db-schema.js index 4780edb..1ca8870 100644 --- a/dist/utils/grab-db-schema.js +++ b/dist/utils/grab-db-schema.js @@ -1,5 +1,5 @@ import init from "../functions/init"; export default async function grabDbSchema() { - const { dbSchema } = await init(); + const { dbSchema } = init(); return dbSchema; } diff --git a/dist/utils/grab-sorted-backups.js b/dist/utils/grab-sorted-backups.js index 0318bb7..2d5d5e7 100644 --- a/dist/utils/grab-sorted-backups.js +++ b/dist/utils/grab-sorted-backups.js @@ -1,18 +1,25 @@ import grabDBDir from "../utils/grab-db-dir"; import fs from "fs"; +import _ from "lodash"; export default function grabSortedBackups({ config }) { const { backup_dir } = grabDBDir({ config }); - const backups = fs.readdirSync(backup_dir); + if (!fs.existsSync(backup_dir)) + return []; + /** + * Only files named by `grabDBBackupFileName` for this database. Anything + * else in the directory is never listed (or trimmed). + */ + const backup_name_regex = new RegExp(`^${_.escapeRegExp(config.db_name)}-\\d+$`); + const backups = fs + .readdirSync(backup_dir) + .filter((b) => backup_name_regex.test(b)); /** * Order Backups. Most recent first. */ const ordered_backups = backups.sort((a, b) => { const a_date = Number(a.split("-").pop()); const b_date = Number(b.split("-").pop()); - if (a_date > b_date) { - return -1; - } - return 1; + return b_date - a_date; }); return ordered_backups; } diff --git a/dist/utils/grab-sorted-exports.d.ts b/dist/utils/grab-sorted-exports.d.ts new file mode 100644 index 0000000..3470e14 --- /dev/null +++ b/dist/utils/grab-sorted-exports.d.ts @@ -0,0 +1,6 @@ +import type { BunSQLiteConfig } from "../types"; +type Params = { + config: BunSQLiteConfig; +}; +export default function grabSortedExports({ config }: Params): string[]; +export {}; diff --git a/dist/utils/grab-sorted-exports.js b/dist/utils/grab-sorted-exports.js new file mode 100644 index 0000000..b30700c --- /dev/null +++ b/dist/utils/grab-sorted-exports.js @@ -0,0 +1,27 @@ +import fs from "fs"; +import grabDBDir from "./grab-db-dir"; +function stripExportExtension(name) { + const lower = name.toLowerCase(); + if (lower.endsWith(".tar.gz")) + return name.slice(0, -7); + if (lower.endsWith(".tgz")) + return name.slice(0, -4); + if (lower.endsWith(".tar")) + return name.slice(0, -4); + if (lower.endsWith(".zip")) + return name.slice(0, -4); + return name; +} +function exportTimestamp(name) { + const base = stripExportExtension(name); + const ts = Number(base.split("-").pop()); + return Number.isFinite(ts) ? ts : 0; +} +export default function grabSortedExports({ config }) { + const { export_dir } = grabDBDir({ config }); + if (!fs.existsSync(export_dir)) { + return []; + } + const exports = fs.readdirSync(export_dir); + return exports.sort((a, b) => exportTimestamp(b) - exportTimestamp(a)); +} diff --git a/dist/utils/replace-db-file.d.ts b/dist/utils/replace-db-file.d.ts new file mode 100644 index 0000000..f61c6fd --- /dev/null +++ b/dist/utils/replace-db-file.d.ts @@ -0,0 +1,13 @@ +type Params = { + src: string; + db_file_path: string; +}; +/** + * # Replace the database file with a copy of `src` + * @description All connections to `db_file_path` must be closed first. + * Leftover `-wal`/`-shm`/`-journal` files are removed so SQLite doesn't + * replay stale pages onto the new file, and the copy is swapped in with an + * atomic rename. + */ +export default function replaceDbFile({ src, db_file_path }: Params): void; +export {}; diff --git a/dist/utils/replace-db-file.js b/dist/utils/replace-db-file.js new file mode 100644 index 0000000..ecefeee --- /dev/null +++ b/dist/utils/replace-db-file.js @@ -0,0 +1,16 @@ +import fs from "fs"; +/** + * # Replace the database file with a copy of `src` + * @description All connections to `db_file_path` must be closed first. + * Leftover `-wal`/`-shm`/`-journal` files are removed so SQLite doesn't + * replay stale pages onto the new file, and the copy is swapped in with an + * atomic rename. + */ +export default function replaceDbFile({ src, db_file_path }) { + const tmp_file_path = `${db_file_path}.replace-${Date.now()}`; + fs.copyFileSync(src, tmp_file_path); + for (const suffix of ["-wal", "-shm", "-journal"]) { + fs.rmSync(`${db_file_path}${suffix}`, { force: true }); + } + fs.renameSync(tmp_file_path, db_file_path); +} diff --git a/dist/utils/resolve-and-load-data-file.d.ts b/dist/utils/resolve-and-load-data-file.d.ts new file mode 100644 index 0000000..959795d --- /dev/null +++ b/dist/utils/resolve-and-load-data-file.d.ts @@ -0,0 +1,27 @@ +import { AppData } from "../data/app-data"; +export type DataFileFormat = "ts" | "js" | "json" | "yaml"; +export type ResolvedDataFile = { + path: string; + basename: string; + extension: (typeof AppData.SupportedDataFileExtensions)[number]; + format: DataFileFormat; +}; +export type LoadedDataFile = ResolvedDataFile & { + data: T; +}; +/** + * Resolve a basename (no extension) to an existing file among supported formats. + * Priority: .ts > .js > .json > .yaml > .yml + * Errors if multiple matches exist. + */ +export declare function resolveDataFile(dir: string, baseName: string): ResolvedDataFile | null; +export declare function supportedDataFileNames(baseName: string): string; +export declare function isSchemaFileName(name: string): boolean; +/** + * Load a data file as a plain object. + * - .ts / .js: require() and use default export (or module itself) + * - .json: JSON.parse + * - .yaml / .yml: Bun.YAML.parse + */ +export declare function loadDataFile(resolved: ResolvedDataFile): T; +export declare function resolveAndLoadDataFile(dir: string, baseName: string): LoadedDataFile | null; diff --git a/dist/utils/resolve-and-load-data-file.js b/dist/utils/resolve-and-load-data-file.js new file mode 100644 index 0000000..e806a61 --- /dev/null +++ b/dist/utils/resolve-and-load-data-file.js @@ -0,0 +1,86 @@ +import fs from "fs"; +import path from "path"; +import { AppData } from "../data/app-data"; +function extensionToFormat(extension) { + switch (extension) { + case ".ts": + return "ts"; + case ".js": + return "js"; + case ".json": + return "json"; + case ".yaml": + case ".yml": + return "yaml"; + } +} +/** + * Resolve a basename (no extension) to an existing file among supported formats. + * Priority: .ts > .js > .json > .yaml > .yml + * Errors if multiple matches exist. + */ +export function resolveDataFile(dir, baseName) { + const matches = []; + for (const extension of AppData.SupportedDataFileExtensions) { + const filePath = path.join(dir, `${baseName}${extension}`); + if (fs.existsSync(filePath) && fs.statSync(filePath).isFile()) { + matches.push({ + path: filePath, + basename: `${baseName}${extension}`, + extension, + format: extensionToFormat(extension), + }); + } + } + if (matches.length === 0) { + return null; + } + if (matches.length > 1) { + const found = matches.map((m) => m.basename).join(", "); + throw new Error(`Multiple \`${baseName}\` files found (${found}). Keep only one of: ${AppData.SupportedDataFileExtensions.join(", ")}`); + } + return matches[0]; +} +export function supportedDataFileNames(baseName) { + return AppData.SupportedDataFileExtensions.map((ext) => `\`${baseName}${ext}\``).join(", "); +} +export function isSchemaFileName(name) { + const base = path.basename(name); + if (base === AppData.DbSchemaFileName) { + return true; + } + return AppData.SupportedDataFileExtensions.some((ext) => base === `${AppData.DbSchemaFileName}${ext}`); +} +/** + * Load a data file as a plain object. + * - .ts / .js: require() and use default export (or module itself) + * - .json: JSON.parse + * - .yaml / .yml: Bun.YAML.parse + */ +export function loadDataFile(resolved) { + if (resolved.format === "ts" || resolved.format === "js") { + const imported = require(resolved.path); + const data = imported && typeof imported === "object" && "default" in imported + ? imported.default + : imported; + if (data == null) { + throw new Error(`No default export from \`${resolved.path}\`. Please export a default module.`); + } + return data; + } + const text = fs.readFileSync(resolved.path, "utf-8"); + if (resolved.format === "json") { + return JSON.parse(text); + } + return Bun.YAML.parse(text); +} +export function resolveAndLoadDataFile(dir, baseName) { + const resolved = resolveDataFile(dir, baseName); + if (!resolved) { + return null; + } + return { + ...resolved, + data: loadDataFile(resolved), + }; +} diff --git a/dist/utils/snapshot-db.d.ts b/dist/utils/snapshot-db.d.ts new file mode 100644 index 0000000..0609718 --- /dev/null +++ b/dist/utils/snapshot-db.d.ts @@ -0,0 +1,12 @@ +import type { Database } from "bun:sqlite"; +type Params = { + db: Database; + dst: string; +}; +/** + * # Write a consistent copy of the database to `dst` + * @description Uses `VACUUM INTO`, which includes changes still in the WAL + * file (a plain file copy misses them). + */ +export default function snapshotDb({ db, dst }: Params): void; +export {}; diff --git a/dist/utils/snapshot-db.js b/dist/utils/snapshot-db.js new file mode 100644 index 0000000..f35476b --- /dev/null +++ b/dist/utils/snapshot-db.js @@ -0,0 +1,12 @@ +import fs from "fs"; +import path from "path"; +/** + * # Write a consistent copy of the database to `dst` + * @description Uses `VACUUM INTO`, which includes changes still in the WAL + * file (a plain file copy misses them). + */ +export default function snapshotDb({ db, dst }) { + fs.mkdirSync(path.dirname(dst), { recursive: true }); + fs.rmSync(dst, { force: true }); + db.run("VACUUM INTO ?", [dst]); +} diff --git a/dist/utils/sql-gen-operator-gen.d.ts b/dist/utils/sql-gen-operator-gen.d.ts index d6adb66..1d7e2b9 100644 --- a/dist/utils/sql-gen-operator-gen.d.ts +++ b/dist/utils/sql-gen-operator-gen.d.ts @@ -6,15 +6,20 @@ type Params = { queryObj: ServerQueryObject<{ [key: string]: any; }, string>; + /** + * `value` is an already-validated `table.field` reference rather than a + * literal, so it is inlined instead of bound + */ isValueFieldValue?: boolean; }; type Return = { - str?: string; - param?: SQLInsertGenValueType; + str: string; + params: SQLInsertGenValueType[]; }; /** * # SQL Gen Operator Gen - * @description Generates an SQL operator for node module `mysql` or `serverless-mysql` + * @description Generates an SQL condition for a single field. Literal values + * are always returned in `params` to be bound, never inlined. */ export default function sqlGenOperatorGen({ fieldName, value, equality, queryObj, isValueFieldValue, }: Params): Return; export {}; diff --git a/dist/utils/sql-gen-operator-gen.js b/dist/utils/sql-gen-operator-gen.js index 9e23fc4..83d9328 100644 --- a/dist/utils/sql-gen-operator-gen.js +++ b/dist/utils/sql-gen-operator-gen.js @@ -1,133 +1,93 @@ import sqlEqualityParser from "./sql-equality-parser"; +import { sqlIdent } from "./sql-safe"; /** * # SQL Gen Operator Gen - * @description Generates an SQL operator for node module `mysql` or `serverless-mysql` + * @description Generates an SQL condition for a single field. Literal values + * are always returned in `params` to be bound, never inlined. */ export default function sqlGenOperatorGen({ fieldName, value, equality, queryObj, isValueFieldValue, }) { - if (queryObj.nullValue) { - return { str: `${fieldName} IS NULL` }; + if (queryObj.nullValue || equality == "IS NULL") { + return { str: `${fieldName} IS NULL`, params: [] }; } - if (queryObj.notNullValue) { - return { str: `${fieldName} IS NOT NULL` }; + if (queryObj.notNullValue || equality == "IS NOT NULL") { + return { str: `${fieldName} IS NOT NULL`, params: [] }; } - if (value) { - const finalValue = isValueFieldValue ? value : "?"; - const finalParams = isValueFieldValue ? undefined : value; - if (equality == "MATCH") { - return { - str: `MATCH(${fieldName}) AGAINST(${finalValue} IN NATURAL LANGUAGE MODE)`, - param: finalParams, - }; + const finalValue = isValueFieldValue ? String(value) : "?"; + const params = isValueFieldValue + ? [] + : [value ?? ""]; + const wrappedValue = isValueFieldValue ? `'%' || ${value} || '%'` : "?"; + const wrappedParams = isValueFieldValue + ? [] + : [`%${value ?? ""}%`]; + switch (equality) { + case "MATCH": + case "MATCH_BOOLEAN": + case "FULLTEXT": { + const matchValue = queryObj.vectorFunction && !isValueFieldValue + ? `${sqlIdent(queryObj.vectorFunction, "vector function")}(?)` + : finalValue; + return { str: `${fieldName} MATCH ${matchValue}`, params }; } - else if (equality == "MATCH_BOOLEAN") { + case "LIKE_LOWER": return { - str: `MATCH(${fieldName}) AGAINST(${finalValue} IN BOOLEAN MODE)`, - param: finalParams, + str: `LOWER(${fieldName}) LIKE LOWER(${wrappedValue})`, + params: wrappedParams, }; - } - else if (equality == "LIKE_LOWER") { + case "LIKE_LOWER_RAW": return { str: `LOWER(${fieldName}) LIKE LOWER(${finalValue})`, - param: `%${finalParams}%`, + params, }; - } - else if (equality == "LIKE_LOWER_RAW") { + case "LIKE": return { - str: `LOWER(${fieldName}) LIKE LOWER(${finalValue})`, - param: finalParams, + str: `${fieldName} LIKE ${wrappedValue}`, + params: wrappedParams, }; - } - else if (equality == "LIKE") { + case "LIKE_RAW": + return { str: `${fieldName} LIKE ${finalValue}`, params }; + case "NOT_LIKE_LOWER": return { - str: `${fieldName} LIKE ${finalValue}`, - param: `%${finalParams}%`, + str: `LOWER(${fieldName}) NOT LIKE LOWER(${wrappedValue})`, + params: wrappedParams, }; - } - else if (equality == "LIKE_RAW") { - return { - str: `${fieldName} LIKE ${finalValue}`, - param: finalParams, - }; - } - else if (equality == "NOT_LIKE_LOWER") { + case "NOT_LIKE_LOWER_RAW": return { str: `LOWER(${fieldName}) NOT LIKE LOWER(${finalValue})`, - param: `%${finalParams}%`, + params, }; - } - else if (equality == "NOT_LIKE_LOWER_RAW") { + case "NOT LIKE": return { - str: `LOWER(${fieldName}) NOT LIKE LOWER(${finalValue})`, - param: finalParams, + str: `${fieldName} NOT LIKE ${wrappedValue}`, + params: wrappedParams, }; - } - else if (equality == "NOT LIKE") { - return { - str: `${fieldName} NOT LIKE ${finalValue}`, - param: finalParams, - }; - } - else if (equality == "NOT LIKE_RAW") { - return { - str: `${fieldName} NOT LIKE ${finalValue}`, - param: finalParams, - }; - } - else if (equality == "REGEXP") { + case "NOT LIKE_RAW": + return { str: `${fieldName} NOT LIKE ${finalValue}`, params }; + case "REGEXP": + /** + * SQLite has no built-in REGEXP function. This only works when + * an extension providing `regexp()` is loaded. + */ return { str: `LOWER(${fieldName}) REGEXP LOWER(${finalValue})`, - param: finalParams, + params, }; - } - else if (equality == "FULLTEXT") { - return { - str: `MATCH(${fieldName}) AGAINST(${finalValue} IN BOOLEAN MODE)`, - param: finalParams, - }; - } - else if (equality == "NOT EQUAL") { - return { - str: `${fieldName} != ${finalValue}`, - param: finalParams, - }; - } - else if (equality == "IS NOT") { - return { - str: `${fieldName} IS NOT ${finalValue}`, - param: finalParams, - }; - } - else if (equality) { + case "NOT EQUAL": + return { str: `${fieldName} != ${finalValue}`, params }; + case "IS NOT": + return { str: `${fieldName} IS NOT ${finalValue}`, params }; + case "IN": + case "NOT IN": + return { str: `${fieldName} ${equality} (${finalValue})`, params }; + case "BETWEEN": + case "NOT BETWEEN": + throw new Error(`\`${equality}\` on \`${fieldName}\` requires an array of two values`); + case undefined: + return { str: `${fieldName} = ${finalValue}`, params }; + default: return { str: `${fieldName} ${sqlEqualityParser(equality)} ${finalValue}`, - param: finalParams, + params, }; - } - else { - return { - str: `${fieldName} = ${finalValue}`, - param: finalParams, - }; - } - } - else { - if (equality == "IS NULL") { - return { str: `${fieldName} IS NULL` }; - } - else if (equality == "IS NOT NULL") { - return { str: `${fieldName} IS NOT NULL` }; - } - else if (equality) { - return { - str: `${fieldName} ${sqlEqualityParser(equality)} ?`, - param: value, - }; - } - else { - return { - str: `${fieldName} = ?`, - param: value, - }; - } } } diff --git a/dist/utils/sql-generator-gen-join-str.js b/dist/utils/sql-generator-gen-join-str.js index 0120afa..bfae69e 100644 --- a/dist/utils/sql-generator-gen-join-str.js +++ b/dist/utils/sql-generator-gen-join-str.js @@ -1,3 +1,4 @@ +import { sqlIdent, sqlOperator, sqlRawEquality } from "./sql-safe"; export default function sqlGenGenJoinStr({ join, mtch, table_name }) { let values = []; if (mtch.__batch) { @@ -13,7 +14,7 @@ export default function sqlGenGenJoinStr({ join, mtch, table_name }) { btch_mtch += str; values.push(...batch_values); if (i < mtch.__batch.matches.length - 1) { - btch_mtch += ` ${mtch.__batch.operator || "OR"} `; + btch_mtch += ` ${sqlOperator(mtch.__batch.operator, "OR")} `; } } btch_mtch += `)`; @@ -22,28 +23,8 @@ export default function sqlGenGenJoinStr({ join, mtch, table_name }) { values, }; } - const equality = mtch.raw_equality || "="; - const lhs = `${typeof mtch.source == "object" ? mtch.source.tableName : table_name}.${typeof mtch.source == "object" ? mtch.source.fieldName : mtch.source}`; - const rhs = `${(() => { - if (mtch.targetLiteral) { - values.push(mtch.targetLiteral); - // if (typeof mtch.targetLiteral == "number") { - // return `${mtch.targetLiteral}`; - // } - // return `'${mtch.targetLiteral}'`; - return `?`; - } - if (join.alias) { - return `${typeof mtch.target == "object" - ? mtch.target.tableName - : join.alias}.${typeof mtch.target == "object" - ? mtch.target.fieldName - : mtch.target}`; - } - return `${typeof mtch.target == "object" - ? mtch.target.tableName - : join.tableName}.${typeof mtch.target == "object" ? mtch.target.fieldName : mtch.target}`; - })()}`; + const equality = mtch.raw_equality ? sqlRawEquality(mtch.raw_equality) : "="; + const lhs = `${sqlIdent(typeof mtch.source == "object" ? mtch.source.tableName : table_name, "table name")}.${sqlIdent(typeof mtch.source == "object" ? mtch.source.fieldName : mtch.source, "field name")}`; if (mtch.between) { values.push(mtch.between.min, mtch.between.max); return { @@ -58,6 +39,19 @@ export default function sqlGenGenJoinStr({ join, mtch, table_name }) { values, }; } + const rhs = (() => { + if (mtch.targetLiteral !== undefined && mtch.targetLiteral !== null) { + values.push(mtch.targetLiteral); + return `?`; + } + const targetTable = typeof mtch.target == "object" + ? mtch.target.tableName + : join.alias || join.tableName; + const targetField = typeof mtch.target == "object" + ? mtch.target.fieldName + : mtch.target; + return `${sqlIdent(targetTable, "table name")}.${sqlIdent(targetField, "field name")}`; + })(); return { str: `${lhs} ${equality} ${rhs}`, values, diff --git a/dist/utils/sql-generator-gen-query-str.d.ts b/dist/utils/sql-generator-gen-query-str.d.ts index 886d00a..89021f1 100644 --- a/dist/utils/sql-generator-gen-query-str.d.ts +++ b/dist/utils/sql-generator-gen-query-str.d.ts @@ -8,8 +8,6 @@ type Param)[]; append_table_names?: boolean; table_name: string; - full_text_match_str?: string; - full_text_search_str?: string; }; export default function sqlGenGenQueryStr !isUndefined(j)) - .map((joinObj) => { - const joinTableName = joinObj.alias - ? joinObj.alias - : joinObj.tableName; - if (existingJoinTableNames.includes(joinTableName)) - return null; - existingJoinTableNames.push(joinTableName); - if (joinObj.group_concat) { + const joinSelectStr = genObject.join + .flat() + .filter((j) => !isUndefined(j)) + .map((joinObj) => { + const joinTableName = sqlIdent(joinObj.alias ? joinObj.alias : joinObj.tableName, "join table name"); + if (existingJoinTableNames.includes(joinTableName)) + return null; + existingJoinTableNames.push(joinTableName); + if (joinObj.group_concat) { + return sqlGenGrabConcatStr({ + field: `${joinTableName}.${sqlIdent(joinObj.group_concat.field, "field name")}`, + alias: joinObj.group_concat.alias, + separator: joinObj.group_concat.separator, + distinct: joinObj.group_concat.distinct, + }); + } + else if (joinObj.selectFields) { + return joinObj.selectFields + .map((selectField) => { + if (typeof selectField != "object") { + return `${joinTableName}.${sqlIdent(selectField, "field name")}`; + } + const field = `${joinTableName}.${sqlIdent(selectField.field, "field name")}`; + if (selectField.group_concat && + selectField.alias) { return sqlGenGrabConcatStr({ - field: `${joinTableName}.${joinObj.group_concat.field}`, - alias: joinObj.group_concat.alias, - separator: joinObj.group_concat.separator, + field, + alias: selectField.alias, + separator: selectField.group_concat.separator, + distinct: selectField.group_concat.distinct, }); } - else if (joinObj.selectFields) { - return joinObj.selectFields - .map((selectField) => { - if (typeof selectField == "string") { - return `${joinTableName}.${selectField}`; - } - else if (typeof selectField == "object") { - let aliasSelectField = `${joinTableName}.${selectField.field}`; - if (selectField.count) { - aliasSelectField = `COUNT(${joinTableName}.${selectField.field})`; - } - else if (selectField.sum) { - aliasSelectField = `SUM(${selectField.distinct ? "DISTINCT " : ""}${joinTableName}.${selectField.field})`; - } - else if (selectField.average) { - aliasSelectField = `AVERAGE(${joinTableName}.${selectField.field})`; - } - else if (selectField.max) { - aliasSelectField = `MAX(${joinTableName}.${selectField.field})`; - } - else if (selectField.min) { - aliasSelectField = `MIN(${joinTableName}.${selectField.field})`; - } - else if (selectField.group_concat && - selectField.alias) { - return sqlGenGrabConcatStr({ - field: `${joinTableName}.${selectField.field}`, - alias: selectField.alias, - separator: selectField.group_concat - .separator, - distinct: selectField.group_concat - .distinct, - }); - } - else if (selectField.distinct) { - aliasSelectField = `DISTINCT ${joinTableName}.${selectField.field}`; - } - if (selectField.alias) - aliasSelectField += ` AS ${selectField.alias}`; - return aliasSelectField; - } - }) - .join(","); + let aliasSelectField = field; + if (selectField.count) { + aliasSelectField = `COUNT(${field})`; } - else { - return `${joinTableName}.*`; + else if (selectField.sum) { + aliasSelectField = `SUM(${selectField.distinct ? "DISTINCT " : ""}${field})`; } + else if (selectField.average) { + aliasSelectField = `AVG(${field})`; + } + else if (selectField.max) { + aliasSelectField = `MAX(${field})`; + } + else if (selectField.min) { + aliasSelectField = `MIN(${field})`; + } + else if (selectField.distinct) { + aliasSelectField = `DISTINCT ${field}`; + } + if (selectField.alias) + aliasSelectField += ` AS ${sqlIdent(selectField.alias, "alias")}`; + return aliasSelectField; }) - .filter((_) => Boolean(_)) .join(","); - } - if (genObject?.fullTextSearch && - full_text_match_str && - full_text_search_str) { - str += `, ${full_text_match_str} AS ${genObject.fullTextSearch.scoreAlias}`; - sqlSearhValues.push(full_text_search_str); + } + else { + return `${joinTableName}.*`; + } + }) + .filter((_) => Boolean(_)) + .join(","); + if (joinSelectStr) { + str += "," + joinSelectStr; + } } str += ` FROM ${table_name}`; if (genObject?.join) { @@ -152,11 +145,12 @@ export default function sqlGenGenQueryStr(params) { .flat() .filter((j) => !isUndefined(j)) .map((join) => { - return (join.joinType + + const joinTable = sqlIdent(join.tableName, "table name"); + return (sqlJoinType(join.joinType) + " " + (join.alias - ? `${join.tableName}` + " " + join.alias - : `${join.tableName}`) + + ? `${joinTable} ${sqlIdent(join.alias, "alias")}` + : joinTable) + " ON " + (() => { if (Array.isArray(join.match)) { @@ -171,9 +165,7 @@ export default function sqlGenGenQueryStr(params) { sqlSearhValues.push(...values); return str; }) - .join(join.operator - ? ` ${join.operator} ` - : " AND ") + + .join(` ${sqlOperator(join.operator)} `) + ")"); } else if (typeof join.match == "object") { diff --git a/dist/utils/sql-generator-gen-search-str.js b/dist/utils/sql-generator-gen-search-str.js index e7cf269..b5119db 100644 --- a/dist/utils/sql-generator-gen-search-str.js +++ b/dist/utils/sql-generator-gen-search-str.js @@ -1,92 +1,112 @@ import sqlGenOperatorGen from "./sql-gen-operator-gen"; +import { sqlField, sqlIdent, sqlOperator, sqlRawEquality } from "./sql-safe"; +function isPresent(val) { + return val !== undefined && val !== null; +} +function isValuesObject(val) { + return typeof val == "object" && val !== null && !ArrayBuffer.isView(val); +} export default function sqlGenGenSearchStr({ queryObj, join, field, table_name, }) { let sqlSearhValues = []; + const safeField = sqlField(field); const finalFieldName = (() => { if (queryObj?.tableName) { - return `${queryObj.tableName}.${field}`; + return `${sqlIdent(queryObj.tableName, "table name")}.${safeField}`; } if (join) { - return `${table_name}.${field}`; + return `${table_name}.${safeField}`; } - return field; + return safeField; })(); - let str = `${finalFieldName}=?`; function grabValue(val) { - const valueParsed = val; - if (!valueParsed) + if (!isPresent(val)) return; - const valueString = typeof valueParsed == "string" || typeof valueParsed == "number" - ? valueParsed - : valueParsed - ? valueParsed.fieldName && valueParsed.tableName - ? `${valueParsed.tableName}.${valueParsed.fieldName}` - : valueParsed.value - : undefined; - const valueEquality = typeof valueParsed == "object" - ? valueParsed.equality || queryObj.equality - : queryObj.equality; - const operatorStrParam = sqlGenOperatorGen({ + if (isValuesObject(val)) { + const isFieldValue = Boolean(val.fieldName && val.tableName); + return sqlGenOperatorGen({ + queryObj, + equality: val.equality || queryObj.equality, + fieldName: finalFieldName, + value: isFieldValue + ? `${sqlIdent(val.tableName, "table name")}.${sqlIdent(val.fieldName, "field name")}` + : val.value, + isValueFieldValue: isFieldValue, + }); + } + return sqlGenOperatorGen({ queryObj, - equality: valueEquality, - fieldName: finalFieldName || "", - value: valueString || "", - isValueFieldValue: Boolean(typeof valueParsed == "object" && - valueParsed.fieldName && - valueParsed.tableName), + equality: queryObj.equality, + fieldName: finalFieldName, + value: val, }); - return operatorStrParam; } + let str; if (Array.isArray(queryObj.value)) { - const strArray = []; - queryObj.value.forEach((val) => { - const operatorStrParam = grabValue(val); - if (!operatorStrParam) - return; - if (operatorStrParam.str && operatorStrParam.param) { + const items = queryObj.value.filter(isPresent); + const rawValues = items + .map((v) => (isValuesObject(v) ? v.value : v)) + .filter(isPresent); + const equality = queryObj.equality; + if ((equality == "IN" || equality == "NOT IN") && !rawValues.length) { + // Empty IN () is invalid SQL: IN matches nothing, NOT IN matches everything + str = equality == "IN" ? "1 = 0" : "1 = 1"; + } + else if (equality == "IN" || equality == "NOT IN") { + str = `${finalFieldName} ${equality} (${rawValues + .map(() => "?") + .join(", ")})`; + sqlSearhValues.push(...rawValues); + } + else if (equality == "BETWEEN" || equality == "NOT BETWEEN") { + if (rawValues.length !== 2) { + throw new Error(`\`${equality}\` on \`${finalFieldName}\` requires exactly two values`); + } + str = `${finalFieldName} ${equality} ? AND ?`; + sqlSearhValues.push(...rawValues); + } + else { + if (!items.length) { + throw new Error(`Empty value array for \`${finalFieldName}\``); + } + const strArray = []; + for (const val of items) { + const operatorStrParam = grabValue(val); + if (!operatorStrParam) + continue; strArray.push(operatorStrParam.str); - sqlSearhValues.push(operatorStrParam.param); - } - else if (operatorStrParam.str) { - strArray.push(operatorStrParam.str); - } - }); - str = "(" + strArray.join(` ${queryObj.operator || "AND"} `) + ")"; - } - else if (typeof queryObj.value == "object") { - const operatorStrParam = grabValue(queryObj.value); - if (operatorStrParam?.str) { - str = operatorStrParam.str; - if (operatorStrParam.param) { - sqlSearhValues.push(operatorStrParam.param); + sqlSearhValues.push(...operatorStrParam.params); } + str = + "(" + + strArray.join(` ${sqlOperator(queryObj.operator)} `) + + ")"; } } - else if (queryObj.raw_equality && queryObj.value) { - str = `${finalFieldName} ${queryObj.raw_equality} ?`; - sqlSearhValues.push(queryObj.value); + else if (queryObj.raw_equality && isPresent(queryObj.value)) { + const value = isValuesObject(queryObj.value) + ? queryObj.value.value + : queryObj.value; + str = `${finalFieldName} ${sqlRawEquality(queryObj.raw_equality)} ?`; + sqlSearhValues.push(value ?? null); } else if (queryObj.between) { str = `${finalFieldName} BETWEEN ? AND ?`; sqlSearhValues.push(queryObj.between.min, queryObj.between.max); } + else if (queryObj.not_between) { + str = `${finalFieldName} NOT BETWEEN ? AND ?`; + sqlSearhValues.push(queryObj.not_between.min, queryObj.not_between.max); + } else { - const valueParsed = queryObj.value ? queryObj.value : undefined; - const operatorStrParam = sqlGenOperatorGen({ - equality: queryObj.equality, - fieldName: finalFieldName || "", - value: valueParsed, - queryObj, - }); - if (operatorStrParam.str && operatorStrParam.param) { - str = operatorStrParam.str; - sqlSearhValues.push(operatorStrParam.param); - } - else if (operatorStrParam.str && !operatorStrParam.str.match(/\?/)) { - str = operatorStrParam.str; - } - else { - sqlSearhValues.push(valueParsed || ""); - } + const operatorStrParam = grabValue(queryObj.value) || + sqlGenOperatorGen({ + equality: queryObj.equality, + fieldName: finalFieldName, + value: undefined, + queryObj, + }); + str = operatorStrParam.str; + sqlSearhValues.push(...operatorStrParam.params); } return { str, values: sqlSearhValues }; } diff --git a/dist/utils/sql-generator-grab-concat-str.js b/dist/utils/sql-generator-grab-concat-str.js index 8993f00..cafc548 100644 --- a/dist/utils/sql-generator-grab-concat-str.js +++ b/dist/utils/sql-generator-grab-concat-str.js @@ -1,13 +1,14 @@ +import { sqlField, sqlIdent, sqlStringLiteral } from "./sql-safe"; export default function sqlGenGrabConcatStr({ alias, field, separator = ",", distinct, }) { let gc = `GROUP_CONCAT(`; if (distinct) { gc += `DISTINCT `; } - gc += `${field}`; + gc += sqlField(field); if (!distinct) { - gc += `, '${separator}'`; + gc += `, ${sqlStringLiteral(String(separator))}`; } gc += `)`; - gc += ` AS ${alias}`; + gc += ` AS ${sqlIdent(alias, "alias")}`; return gc; } diff --git a/dist/utils/sql-generator-grab-select-field-sql.js b/dist/utils/sql-generator-grab-select-field-sql.js index 4056788..e164b38 100644 --- a/dist/utils/sql-generator-grab-select-field-sql.js +++ b/dist/utils/sql-generator-grab-select-field-sql.js @@ -1,53 +1,53 @@ import sqlGenGrabConcatStr from "./sql-generator-grab-concat-str"; +import { sqlField, sqlIdent } from "./sql-safe"; export default function sqlGenGrabSelectFieldSQL({ selectFields, append_table_names, table_name }) { let str = ""; str += ` ${selectFields ?.map((fld) => { + if (typeof fld != "object") { + return sqlField(fld); + } + const fld_name = sqlField(fld.fieldName); + const final_fld_name = append_table_names && !fld_name.includes(".") + ? `${table_name}.${fld_name}` + : fld_name; + if (fld.group_concat) { + return sqlGenGrabConcatStr({ + field: final_fld_name, + alias: fld.group_concat.alias || fld.alias || fld_name, + separator: fld.group_concat.separator, + distinct: fld.group_concat.distinct, + }); + } let fld_str = ``; - const final_fld_name = typeof fld == "object" - ? append_table_names - ? `${table_name}.${String(fld)}` - : `${String(fld.fieldName)}` - : `${String(fld)}`; - if (typeof fld == "object") { - const fld_name = `${String(fld.fieldName)}`; - if (fld.count) { - fld_str += `COUNT(${fld_name})`; - } - else if (fld.sum) { - fld_str += `SUM(${fld_name})`; - } - else if (fld.average) { - fld_str += `AVERAGE(${fld_name})`; - } - else if (fld.max) { - fld_str += `MAX(${fld_name})`; - } - else if (fld.min) { - fld_str += `MIN(${fld_name})`; - } - else if (fld.distinct) { - fld_str += `DISTINCT ${fld_name}`; - } - else if (fld.group_concat) { - fld_str += sqlGenGrabConcatStr({ - field: fld_name, - alias: fld.group_concat.alias, - separator: fld.group_concat.separator, - distinct: fld.group_concat.distinct, - }); - } - else { - fld_str += - final_fld_name + (fld.alias ? ` as ${fld.alias}` : ``); - } - if (fld.alias) { - fld_str += ` AS ${fld.alias}`; + let alias = fld.alias; + if (fld.count) { + fld_str += `COUNT(${final_fld_name})`; + if (!alias && typeof fld.count == "object") { + alias = fld.count.alias; } } + else if (fld.sum) { + fld_str += `SUM(${final_fld_name})`; + } + else if (fld.average) { + fld_str += `AVG(${final_fld_name})`; + } + else if (fld.max) { + fld_str += `MAX(${final_fld_name})`; + } + else if (fld.min) { + fld_str += `MIN(${final_fld_name})`; + } + else if (fld.distinct) { + fld_str += `DISTINCT ${final_fld_name}`; + } else { fld_str += final_fld_name; } + if (alias) { + fld_str += ` AS ${sqlIdent(alias, "alias")}`; + } return fld_str; }) .join(",")}`; diff --git a/dist/utils/sql-generator.d.ts b/dist/utils/sql-generator.d.ts index b6eea8f..b10361e 100644 --- a/dist/utils/sql-generator.d.ts +++ b/dist/utils/sql-generator.d.ts @@ -8,18 +8,32 @@ type Param({ tableName, genObject, dbFullName, count }: Param): Return; +}>({ tableName: passedTableName, genObject, count, targetId, }: Param): Return; export {}; diff --git a/dist/utils/sql-generator.js b/dist/utils/sql-generator.js index 16aa373..a589705 100644 --- a/dist/utils/sql-generator.js +++ b/dist/utils/sql-generator.js @@ -1,29 +1,23 @@ import sqlGenGenSearchStr from "./sql-generator-gen-search-str"; import sqlGenGenQueryStr from "./sql-generator-gen-query-str"; +import { sqlField, sqlIdent, sqlInt, sqlOperator, sqlOrderStrategy, } from "./sql-safe"; /** * # SQL Query Generator - * @description Generates an SQL Query for node module `mysql` or `serverless-mysql` + * @description Generates a parameterized SQLite SELECT query. All values are + * returned in `values` to be bound; identifiers are validated. */ -export default function sqlGenerator({ tableName, genObject, dbFullName, count }) { +export default function sqlGenerator({ tableName: passedTableName, genObject, count, targetId, }) { + const tableName = sqlIdent(passedTableName, "table name"); const finalQuery = genObject?.query ? genObject.query : undefined; const queryKeys = finalQuery ? Object.keys(finalQuery) : undefined; const sqlSearhValues = []; - let fullTextMatchStr = genObject?.fullTextSearch - ? ` MATCH(${genObject.fullTextSearch.fields - .map((f) => genObject.join ? `${tableName}.${String(f)}` : `${String(f)}`) - .join(",")}) AGAINST (? IN BOOLEAN MODE)` - : undefined; - const fullTextSearchStr = genObject?.fullTextSearch - ? genObject.fullTextSearch.searchTerm - .split(` `) - .map((t) => `${t}`) - .join(" ") - : undefined; + const whereValues = []; + if (genObject?.fullTextSearch) { + throw new Error("`fullTextSearch` is not supported by SQLite. Use an FTS5 table and the `MATCH` equality instead."); + } let { str: queryString, values } = sqlGenGenQueryStr({ table_name: tableName, append_table_names: true, - full_text_match_str: fullTextMatchStr, - full_text_search_str: fullTextSearchStr, genObject, }); sqlSearhValues.push(...values); @@ -34,7 +28,8 @@ export default function sqlGenerator({ tableName, genObject, dbFullName, count } if (queryObj.__query) { const subQueryGroup = queryObj.__query; const subSearchKeys = Object.keys(subQueryGroup); - const subSearchString = subSearchKeys.map((_field) => { + const subSearchString = subSearchKeys + .map((_field) => { const newSubQueryObj = subQueryGroup?.[_field]; if (newSubQueryObj) { const { str, values } = sqlGenGenSearchStr({ @@ -43,12 +38,15 @@ export default function sqlGenerator({ tableName, genObject, dbFullName, count } join: genObject?.join, table_name: tableName, }); - sqlSearhValues.push(...values); + whereValues.push(...values); return str; } - }); + }) + .filter((str) => typeof str == "string"); + if (!subSearchString.length) + return; return ("(" + - subSearchString.join(` ${queryObj.operator || "AND"} `) + + subSearchString.join(` ${sqlOperator(queryObj.operator)} `) + ")"); } const { str, values } = sqlGenGenSearchStr({ @@ -57,96 +55,96 @@ export default function sqlGenerator({ tableName, genObject, dbFullName, count } join: genObject?.join, table_name: tableName, }); - sqlSearhValues.push(...values); + whereValues.push(...values); return str; }); const cleanedUpSearchStr = sqlSearhString?.filter((str) => typeof str == "string"); - const isSearchStr = cleanedUpSearchStr?.[0] && cleanedUpSearchStr.find((str) => str); - if (isSearchStr) { - const stringOperator = genObject?.searchOperator || "AND"; - queryString += ` WHERE ${cleanedUpSearchStr.join(` ${stringOperator} `)}`; + const whereConditions = []; + if (cleanedUpSearchStr?.[0]) { + const stringOperator = sqlOperator(genObject?.searchOperator); + whereConditions.push(`(${cleanedUpSearchStr.join(` ${stringOperator} `)})`); } - if (genObject?.fullTextSearch && fullTextSearchStr && fullTextMatchStr) { - queryString += `${isSearchStr ? " AND" : " WHERE"} ${fullTextMatchStr}`; - sqlSearhValues.push(fullTextSearchStr); + if (targetId !== undefined && targetId !== null && targetId !== "") { + const id = Number(targetId); + if (!Number.isSafeInteger(id)) { + throw new Error(`Invalid targetId: ${JSON.stringify(targetId)}`); + } + whereConditions.push(`${genObject?.join ? `${tableName}.` : ""}id = ?`); + whereValues.push(id); + } + const whereString = whereConditions.length + ? `WHERE ${whereConditions.join(" AND ")}` + : ""; + if (whereString) { + queryString += ` ${whereString}`; + sqlSearhValues.push(...whereValues); } if (genObject?.group) { - let group_by_txt = ``; - if (typeof genObject.group == "string") { - group_by_txt = genObject.group; - } - else if (Array.isArray(genObject.group)) { - for (let i = 0; i < genObject.group.length; i++) { - const group = genObject.group[i]; - if (typeof group == "string") { - group_by_txt += `\`${group.toString()}\``; - } - else if (typeof group == "object" && group.table) { - group_by_txt += `${group.table}.${String(group.field)}`; - } - else if (typeof group == "object") { - group_by_txt += `${String(group.field)}`; - } - if (i < genObject.group.length - 1) { - group_by_txt += ","; - } + const grabGroupStr = (group) => { + if (typeof group == "object") { + return group.table + ? `${sqlIdent(group.table, "table name")}.${sqlIdent(group.field, "field name")}` + : sqlField(group.field); } - } - else if (typeof genObject.group == "object") { - if (genObject.group.table) { - group_by_txt = `${genObject.group.table}.${String(genObject.group.field)}`; - } - else { - group_by_txt = `${String(genObject.group.field)}`; - } - } + return sqlField(group); + }; + const group_by_txt = Array.isArray(genObject.group) + ? genObject.group.map(grabGroupStr).join(",") + : grabGroupStr(genObject.group); queryString += ` GROUP BY ${group_by_txt}`; } function grabOrderString(order) { - let orderFields = []; - let orderSrt = ``; - if (genObject?.fullTextSearch && genObject.fullTextSearch.scoreAlias) { - orderFields.push(genObject.fullTextSearch.scoreAlias); - } - else if (genObject?.join) { - orderFields.push(`${tableName}.${String(order.field)}`); - } - else { - orderFields.push(order.field); - } - orderSrt += ` ${orderFields.join(", ")} ${order.strategy}`; - return orderSrt; + const field = genObject?.join && !String(order.field).includes(".") + ? `${tableName}.${sqlIdent(order.field, "order field")}` + : sqlField(order.field, "order field"); + return ` ${field} ${sqlOrderStrategy(order.strategy)}`; } if (genObject?.order) { let orderSrt = ` ORDER BY`; - if (Array.isArray(genObject.order)) { - for (let i = 0; i < genObject.order.length; i++) { - const order = genObject.order[i]; - if (order) { - orderSrt += - grabOrderString(order) + - (i < genObject.order.length - 1 ? `,` : ""); - } - } - } - else { - orderSrt += grabOrderString(genObject.order); - } + const orders = (Array.isArray(genObject.order) ? genObject.order : [genObject.order]).filter(Boolean); + orderSrt += orders.map(grabOrderString).join(","); queryString += ` ${orderSrt}`; } - if (genObject?.limit && !count) - queryString += ` LIMIT ${genObject.limit}`; - if (genObject?.offset) { - queryString += ` OFFSET ${genObject.offset}`; - } - else if (genObject?.page && genObject.limit && !count) { - queryString += ` OFFSET ${(genObject.page - 1) * genObject.limit}`; + /** + * Pagination is skipped for count queries so the count covers every + * matching row + */ + if (!count) { + const { limit: rawLimit, offset: rawOffset, page } = genObject || {}; + /** + * `limit: 0` has always meant "no limit" + */ + const limit = rawLimit ? sqlInt(rawLimit, "limit") : undefined; + const offset = isSet(rawOffset) + ? sqlInt(rawOffset, "offset") + : isSet(page) && limit !== undefined + ? Math.max(sqlInt(page, "page") - 1, 0) * limit + : undefined; + if (limit !== undefined) { + queryString += ` LIMIT ${limit}`; + } + else if (offset) { + /** + * SQLite doesn't allow OFFSET without LIMIT + */ + queryString += ` LIMIT -1`; + } + if (offset) { + queryString += ` OFFSET ${offset}`; + } } return { string: queryString, values: sqlSearhValues, + where: { + string: whereString, + values: whereString ? whereValues : [], + }, }; } +function isSet(val) { + return val !== undefined && val !== null; +} // let queryString = (() => { // let str = "SELECT"; // if (genObject?.select_sql) { diff --git a/dist/utils/sql-insert-generator.d.ts b/dist/utils/sql-insert-generator.d.ts index 1b43778..8fcc01c 100644 --- a/dist/utils/sql-insert-generator.d.ts +++ b/dist/utils/sql-insert-generator.d.ts @@ -1,5 +1,11 @@ -import type { SQLInsertGenParams, SQLInsertGenReturn } from "../types"; +import type { SQLInsertGenParams, SQLInsertGenReturn, SQLInsertGenValueType } from "../types"; /** * # SQL Insert Generator */ export default function sqlInsertGenerator({ tableName, data, dbFullName, }: SQLInsertGenParams): SQLInsertGenReturn | undefined; +/** + * # Normalize a JS value for binding + * @description Keeps `0`, `""` and `false` (as `0`) instead of turning them + * into `NULL` + */ +export declare function sqlValue(value: unknown): SQLInsertGenValueType; diff --git a/dist/utils/sql-insert-generator.js b/dist/utils/sql-insert-generator.js index 40b0a50..b898307 100644 --- a/dist/utils/sql-insert-generator.js +++ b/dist/utils/sql-insert-generator.js @@ -1,60 +1,56 @@ +import { sqlIdent } from "./sql-safe"; /** * # SQL Insert Generator */ export default function sqlInsertGenerator({ tableName, data, dbFullName, }) { - const finalDbName = dbFullName ? `${dbFullName}.` : ""; - try { - if (Array.isArray(data) && data?.[0]) { - let insertKeys = []; - data.forEach((dt) => { - const kys = Object.keys(dt); - kys.forEach((ky) => { - if (!insertKeys.includes(ky)) { - insertKeys.push(ky); - } - }); + const finalDbName = dbFullName + ? `${sqlIdent(dbFullName, "database name")}.` + : ""; + if (Array.isArray(data) && data?.[0]) { + let insertKeys = []; + data.forEach((dt) => { + const kys = Object.keys(dt); + kys.forEach((ky) => { + if (!insertKeys.includes(ky)) { + insertKeys.push(sqlIdent(ky, "column name")); + } }); - let queryBatches = []; - let queryValues = []; - data.forEach((item) => { - queryBatches.push(`(${insertKeys - .map((ky) => { - const value = item[ky]; - const finalValue = typeof value == "string" || - typeof value == "number" - ? value - : typeof value == "function" - ? value().value - : value - ? value - : null; - if (!finalValue && - typeof finalValue !== "number" && - typeof finalValue !== "string") { - queryValues.push(null); - return "?"; - } - queryValues.push(finalValue); - const placeholder = typeof value == "function" - ? value().placeholder - : "?"; - return placeholder; - }) - .filter((k) => Boolean(k)) - .join(",")})`); - }); - let query = `INSERT INTO ${finalDbName}${tableName} (${insertKeys.join(",")}) VALUES ${queryBatches.join(",")}`; - return { - query: query, - values: queryValues, - }; - } - else { - return undefined; - } + }); + let queryBatches = []; + let queryValues = []; + data.forEach((item) => { + queryBatches.push(`(${insertKeys + .map((ky) => { + const value = item[ky]; + if (typeof value == "function") { + const fnValue = value(); + queryValues.push(fnValue.value ?? null); + return fnValue.placeholder; + } + queryValues.push(sqlValue(value)); + return "?"; + }) + .join(",")})`); + }); + let query = `INSERT INTO ${finalDbName}${sqlIdent(tableName, "table name")} (${insertKeys.join(",")}) VALUES ${queryBatches.join(",")}`; + return { + query: query, + values: queryValues, + }; } - catch ( /** @type {any} */error) { - console.log(`SQL insert gen ERROR: ${error.message}`); + else { return undefined; } } +/** + * # Normalize a JS value for binding + * @description Keeps `0`, `""` and `false` (as `0`) instead of turning them + * into `NULL` + */ +export function sqlValue(value) { + if (value === undefined || value === null) + return null; + if (typeof value == "boolean") + return value ? 1 : 0; + return value; +} diff --git a/dist/utils/sql-safe.d.ts b/dist/utils/sql-safe.d.ts new file mode 100644 index 0000000..bb678df --- /dev/null +++ b/dist/utils/sql-safe.d.ts @@ -0,0 +1,30 @@ +import { SQlComparisons, ServerQueryOperators } from "../types"; +declare const JOIN_TYPES: readonly ["INNER JOIN", "JOIN", "LEFT JOIN", "RIGHT JOIN"]; +/** + * # Validate a bare SQL identifier (table, column, alias) + * @description Identifiers can't be bound as parameters, so anything that + * gets interpolated into SQL must pass this check first. + */ +export declare function sqlIdent(name: unknown, label?: string): string; +/** + * # Validate a column reference + * @description Accepts `field`, `table.field` or `table.*` + */ +export declare function sqlField(name: unknown, label?: string): string; +/** + * # Quote an identifier for DDL statements + */ +export declare function sqlQuoteIdent(name: string): string; +/** + * # Quote a string literal + */ +export declare function sqlStringLiteral(value: string): string; +/** + * # Validate a non-negative integer (LIMIT, OFFSET, page) + */ +export declare function sqlInt(value: unknown, label?: string): number; +export declare function sqlOrderStrategy(strategy: unknown): "ASC" | "DESC"; +export declare function sqlOperator(operator: unknown, fallback?: (typeof ServerQueryOperators)[number]): (typeof ServerQueryOperators)[number]; +export declare function sqlRawEquality(equality: unknown): (typeof SQlComparisons)[number]; +export declare function sqlJoinType(joinType: unknown): (typeof JOIN_TYPES)[number]; +export {}; diff --git a/dist/utils/sql-safe.js b/dist/utils/sql-safe.js new file mode 100644 index 0000000..244d125 --- /dev/null +++ b/dist/utils/sql-safe.js @@ -0,0 +1,79 @@ +import { SQlComparisons, ServerQueryOperators } from "../types"; +const IDENTIFIER_REGEX = /^[A-Za-z_][A-Za-z0-9_]*$/; +const FIELD_REGEX = /^([A-Za-z_][A-Za-z0-9_]*\.)?([A-Za-z_][A-Za-z0-9_]*|\*)$/; +const JOIN_TYPES = ["INNER JOIN", "JOIN", "LEFT JOIN", "RIGHT JOIN"]; +/** + * # Validate a bare SQL identifier (table, column, alias) + * @description Identifiers can't be bound as parameters, so anything that + * gets interpolated into SQL must pass this check first. + */ +export function sqlIdent(name, label = "identifier") { + const str = String(name); + if (typeof name == "symbol" || !IDENTIFIER_REGEX.test(str)) { + throw new Error(`Invalid SQL ${label}: ${JSON.stringify(str)}`); + } + return str; +} +/** + * # Validate a column reference + * @description Accepts `field`, `table.field` or `table.*` + */ +export function sqlField(name, label = "field name") { + const str = String(name); + if (typeof name == "symbol" || !FIELD_REGEX.test(str)) { + throw new Error(`Invalid SQL ${label}: ${JSON.stringify(str)}`); + } + return str; +} +/** + * # Quote an identifier for DDL statements + */ +export function sqlQuoteIdent(name) { + return `"${name.replace(/"/g, '""')}"`; +} +/** + * # Quote a string literal + */ +export function sqlStringLiteral(value) { + return `'${value.replace(/'/g, "''")}'`; +} +/** + * # Validate a non-negative integer (LIMIT, OFFSET, page) + */ +export function sqlInt(value, label = "integer") { + const num = Number(value); + if (!Number.isSafeInteger(num) || num < 0) { + throw new Error(`Invalid SQL ${label}: ${JSON.stringify(value)}`); + } + return num; +} +export function sqlOrderStrategy(strategy) { + if (strategy === undefined || strategy === null) + return "ASC"; + const str = String(strategy).toUpperCase(); + if (str !== "ASC" && str !== "DESC") { + throw new Error(`Invalid SQL order strategy: ${JSON.stringify(strategy)}`); + } + return str; +} +export function sqlOperator(operator, fallback = "AND") { + if (operator === undefined || operator === null) + return fallback; + const str = String(operator).toUpperCase(); + if (!ServerQueryOperators.includes(str)) { + throw new Error(`Invalid SQL operator: ${JSON.stringify(operator)}`); + } + return str; +} +export function sqlRawEquality(equality) { + if (!SQlComparisons.includes(equality)) { + throw new Error(`Invalid SQL comparison: ${JSON.stringify(equality)}`); + } + return equality; +} +export function sqlJoinType(joinType) { + if (!JOIN_TYPES.includes(joinType)) { + throw new Error(`Invalid SQL join type: ${JSON.stringify(joinType)}`); + } + return joinType; +} diff --git a/dist/utils/trim-exports.d.ts b/dist/utils/trim-exports.d.ts new file mode 100644 index 0000000..74a497f --- /dev/null +++ b/dist/utils/trim-exports.d.ts @@ -0,0 +1,6 @@ +import type { BunSQLiteConfig } from "../types"; +type Params = { + config: BunSQLiteConfig; +}; +export default function trimExports({ config }: Params): void; +export {}; diff --git a/dist/utils/trim-exports.js b/dist/utils/trim-exports.js new file mode 100644 index 0000000..1285ff7 --- /dev/null +++ b/dist/utils/trim-exports.js @@ -0,0 +1,22 @@ +import fs from "fs"; +import path from "path"; +import _ from "lodash"; +import { AppData } from "../data/app-data"; +import grabDBDir from "./grab-db-dir"; +import grabSortedExports from "./grab-sorted-exports"; +export default function trimExports({ config }) { + const { export_dir } = grabDBDir({ config }); + // Only consider archives written by `export` (`{db_name}-{timestamp}.tar.gz|zip`) + const exportNamePattern = new RegExp(`^${_.escapeRegExp(config.db_name)}-\\d+\\.(tar\\.gz|zip)$`); + const exports = grabSortedExports({ config }).filter((name) => exportNamePattern.test(name) && + fs.statSync(path.join(export_dir, name)).isFile()); + const max_exports = config.max_exports || AppData["MaxExports"]; + for (let i = 0; i < exports.length; i++) { + const export_name = exports[i]; + if (!export_name) + continue; + if (i > max_exports - 1) { + fs.unlinkSync(path.join(export_dir, export_name)); + } + } +} diff --git a/package.json b/package.json index 2ec20b8..846b800 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@moduletrace/bun-sqlite", - "version": "1.1.13", + "version": "1.2.0", "description": "Schema-driven SQLite manager for Bun", "author": "Benjamin Toby", "type": "module",