diff --git a/.gitignore b/.gitignore index 2bd9555..65b3bcc 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ node_modules # output out +dist *.tgz # code coverage @@ -32,4 +33,5 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json # Finder (MacOS) folder config .DS_Store /test -.vscode \ No newline at end of file +.vscode +.dump \ No newline at end of file diff --git a/README.md b/README.md index 83c48ce..5484b66 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# Bun Mariadb +# Bun MariaDB -@moduletrace/bun-mariadb +`@moduletrace/bun-mariadb` -A schema-driven MariaDB manager for [Bun](https://bun.sh), featuring automatic schema synchronization, type-safe CRUD operations, and TypeScript type definition generation. +A schema-driven MariaDB manager for [Bun](https://bun.sh), featuring automatic schema synchronization, type-safe CRUD, native `VECTOR` support, HTML sanitization, and TypeScript type definition generation. --- @@ -11,14 +11,16 @@ A schema-driven MariaDB manager for [Bun](https://bun.sh), featuring automatic s - [Features](#features) - [Prerequisites](#prerequisites) - [Installation](#installation) +- [Environment Variables](#environment-variables) - [Quick Start](#quick-start) - [Configuration](#configuration) - [Schema Definition](#schema-definition) - [CLI Commands](#cli-commands) - [`schema`](#schema--sync-database-to-schema) - [`typedef`](#typedef--generate-typescript-types-only) - - [`backup`](#backup--back-up-the-database) - - [`restore`](#restore--restore-the-database-from-a-backup) + - [`backup`](#backup--sql-dump-backup) + - [`restore`](#restore--restore-from-an-sql-dump) + - [`admin`](#admin--interactive-admin-tools) - [CRUD API](#crud-api) - [Select](#select) - [Insert](#insert) @@ -26,7 +28,8 @@ A schema-driven MariaDB manager for [Bun](https://bun.sh), featuring automatic s - [Delete](#delete) - [Raw SQL](#raw-sql) - [Query API Reference](#query-api-reference) -- [Vector Table Support](#vector-table-support) +- [Vector Support](#vector-support) +- [HTML Sanitization](#html-sanitization) - [TypeScript Type Generation](#typescript-type-generation) - [Default Fields](#default-fields) - [Project Structure](#project-structure) @@ -35,47 +38,66 @@ A schema-driven MariaDB manager for [Bun](https://bun.sh), featuring automatic s ## Features -- **Schema-first design** — define your database in TypeScript; the library syncs your SQLite file to match -- **Automatic migrations** — adds new columns, recreates tables for complex changes, drops removed tables -- **Type-safe CRUD** — fully generic `select`, `insert`, `update`, `delete` functions with TypeScript generics -- **Rich query DSL** — filtering, ordering, pagination, joins, grouping, full-text search, sub-query counts -- **TypeScript codegen** — generate `.ts` type definitions from your schema automatically -- **Zero-config defaults** — `id`, `created_at`, and `updated_at` fields are added to every table automatically +- **Schema-first design** — define your database in TypeScript; the library syncs MariaDB to match +- **Automatic migrations** — adds/modifies/drops columns, manages indexes & foreign keys, drops removed tables +- **Type-safe CRUD** — generic `select`, `insert`, `update`, `delete`, and raw `sql` +- **Rich query DSL** — filtering, ordering, pagination, joins, grouping, full-text search +- **Native VECTOR columns** — MariaDB `VECTOR(n)` types and `VECTOR INDEX` support +- **HTML sanitization** — fields with `html: true` are sanitized on insert/update +- **TypeScript codegen** — generate `.ts` type definitions from your schema +- **Zero-config defaults** — `id`, `created_at`, and `updated_at` are added to every table automatically --- ## Prerequisites -This works for just `bun`, so `bun` should be installed. +- [Bun](https://bun.sh) installed +- A running MariaDB server (11.7+ recommended for native `VECTOR` support) --- ## Installation -There are two ways to install BunMariaDB: +### Via the private npm registry -### Via the official npm registry - -`@moduletrace/bun-mariadb` is published to a private Gitea npm registry. You must configure your package manager to resolve the `@moduletrace` scope from that registry before installing. - -Add the following to your project's `.npmrc` file (create it at the root of your project if it doesn't exist): +`@moduletrace/bun-mariadb` is published to a private Gitea npm registry. Configure the `@moduletrace` scope first: ```ini +# .npmrc @moduletrace:registry=https://git.tben.me/api/packages/moduletrace/npm/ ``` -After this you can run - ```bash -bun add @moduletrace/bun-sqlite +bun add @moduletrace/bun-mariadb ``` -### Via Github - -To install directly from github simply run +### Via Git ```bash -bun add github:moduletrace/bun-sqlite +bun add github:moduletrace/bun-mariadb +``` + +--- + +## Environment Variables + +Connection settings are read from the environment (not the config file): + +| Variable | Required | Description | +| --------------------------------- | -------- | ------------------------------------ | +| `BUN_MARIADB_SERVER_HOST` | Yes | MariaDB host | +| `BUN_MARIADB_SERVER_USERNAME` | Yes | Database user | +| `BUN_MARIADB_SERVER_PASSWORD` | Yes | Database password | +| `BUN_MARIADB_SERVER_PORT` | No | Port (server default if omitted) | +| `BUN_MARIADB_SERVER_SSL_KEY_PATH` | No | Optional SSL key path (legacy/env) | + +Example `.env`: + +```env +BUN_MARIADB_SERVER_HOST=127.0.0.1 +BUN_MARIADB_SERVER_PORT=3306 +BUN_MARIADB_SERVER_USERNAME=root +BUN_MARIADB_SERVER_PASSWORD=secret ``` --- @@ -84,17 +106,16 @@ bun add github:moduletrace/bun-sqlite ### 1. Create the config file -Create `bun-sqlite.config.ts` at your project root: +Create `bun-mariadb.config.ts` at your project root: ```ts -import type { BunSQLiteConfig } from "@moduletrace/bun-sqlite"; +import type { BunMariaDBConfig } from "@moduletrace/bun-mariadb"; -const config: BunSQLiteConfig = { - db_name: "my-app.db", - db_schema_file_name: "schema.ts", - db_dir: "./db", // optional: where to store the db file - db_backup_dir: ".backups", // optional: name of backups directory. Relative to the db dir. - typedef_file_path: "./db/types/db.ts", // optional: where to write generated types +const config: BunMariaDBConfig = { + db_name: "my_app", + db_dir: "./db", + typedef_file_path: "./db/types/db.ts", + // ssl_ca: "./db/ca-cert.pem", }; export default config; @@ -102,20 +123,20 @@ export default config; ### 2. Define your schema -Create `./db/schema.ts` (matching `db_schema_file_name` above): +Create `./db/schema.ts` (filename is fixed as `schema.ts` inside `db_dir`): ```ts -import type { BUN_MARIADB_DatabaseSchemaType } from "@moduletrace/bun-sqlite"; +import type { BUN_MARIADB_DatabaseSchemaType } from "@moduletrace/bun-mariadb"; const schema: BUN_MARIADB_DatabaseSchemaType = { - dbName: "my-app", tables: [ { tableName: "users", fields: [ - { fieldName: "first_name", dataType: "TEXT" }, - { fieldName: "last_name", dataType: "TEXT" }, - { fieldName: "email", dataType: "TEXT", unique: true }, + { fieldName: "first_name", dataType: "VARCHAR", integerLength: 255 }, + { fieldName: "last_name", dataType: "VARCHAR", integerLength: 255 }, + { fieldName: "email", dataType: "VARCHAR", integerLength: 255, unique: true }, + { fieldName: "bio", dataType: "LONGTEXT", html: true }, ], }, ], @@ -124,54 +145,60 @@ const schema: BUN_MARIADB_DatabaseSchemaType = { export default schema; ``` -### 3. Sync the schema to SQLite +### 3. Sync the schema to MariaDB ```bash -bunx bun-sqlite schema +bunx bun-mariadb schema ``` -This creates the SQLite database file and creates/updates all tables to match your schema. +This creates/updates all tables, indexes, and foreign keys to match your schema. ### 4. Use the CRUD API ```ts -import BunSQLite from "@moduletrace/bun-sqlite"; +import BunMariaDB from "@moduletrace/bun-mariadb"; // Insert -await BunSQLite.insert({ +await BunMariaDB.insert({ table: "users", data: [{ first_name: "Alice", email: "alice@example.com" }], }); // Select -const result = await BunSQLite.select({ table: "users" }); -console.log(result.payload); // Alice's row +const result = await BunMariaDB.select({ table: "users" }); +console.log(result.payload); // Update -await BunSQLite.update({ +await BunMariaDB.update({ table: "users", targetId: 1, data: { first_name: "Alicia" }, }); // Delete -await BunSQLite.delete({ table: "users", targetId: 1 }); +await BunMariaDB.delete({ table: "users", targetId: 1 }); ``` --- ## Configuration -The config file must be named `bun-sqlite.config.ts` and placed at the root of your project. +The config file must be named `bun-mariadb.config.ts` and placed at the project root. -| Field | Type | Required | Description | -| --------------------- | -------- | -------- | ------------------------------------------------------------------------------------------ | -| `db_name` | `string` | Yes | Filename for the SQLite database (e.g. `"app.db"`) | -| `db_schema_file_name` | `string` | Yes | Filename of the schema file relative to `db_dir` (or root if `db_dir` is not set) | -| `db_backup_dir` | `string` | No | Directory for database backups, relative to `db_dir` | -| `db_dir` | `string` | No | Root directory for the database file and schema. Defaults to project root | -| `typedef_file_path` | `string` | No | Output path for generated TypeScript types, relative to project root | -| `max_backups` | `number` | No | Maximum number of backup files to keep. Oldest are deleted automatically. Defaults to `10` | +| Field | Type | Required | Description | +| ------------------- | -------- | -------- | --------------------------------------------------------------------------- | +| `db_name` | `string` | Yes | MariaDB database name | +| `db_dir` | `string` | Yes | Directory for schema, types, and local artifacts (relative to project root) | +| `db_backup_dir` | `string` | No | Backup directory name, relative to `db_dir` (default: `.backups`) | +| `max_backups` | `number` | No | Max backup files to keep (default: `10`) | +| `typedef_file_path` | `string` | No | Output path for generated TypeScript types (relative to project root) | +| `db_config` | `object` | No | Extra options passed to Bun's `SQL` MariaDB adapter | +| `charset` | `string` | No | Database charset (default: `utf8mb4`) | +| `connection_timeout`| `number` | No | Connection timeout in ms (default: `10000`) | +| `ssl_ca` | `string` | No | Path to SSL CA certificate (relative to project root) | +| `html_sanitize` | `object` | No | Extra HTML sanitizer allowlists (see [HTML Sanitization](#html-sanitization)) | + +Schema file path is always: `{db_dir}/schema.ts`. --- @@ -181,8 +208,8 @@ The config file must be named `bun-sqlite.config.ts` and placed at the root of y ```ts interface BUN_MARIADB_DatabaseSchemaType { - dbName?: string; tables: BUN_MARIADB_TableSchemaType[]; + collation?: "utf8mb4_bin" | "utf8mb4_unicode_520_ci"; } ``` @@ -195,10 +222,10 @@ interface BUN_MARIADB_TableSchemaType { fields: BUN_MARIADB_FieldSchemaType[]; indexes?: BUN_MARIADB_IndexSchemaType[]; uniqueConstraints?: BUN_MARIADB_UniqueConstraintSchemaType[]; - parentTableName?: string; // inherit fields from another table in the schema - tableNameOld?: string; // rename: set this to the old name to trigger ALTER TABLE RENAME - isVector?: boolean; // mark this as a sqlite-vec virtual table - vectorType?: string; // virtual table type, defaults to "vec0" + parentTableName?: string; // inherit / merge fields from another table + tableNameOld?: string; // rename: old name triggers ALTER TABLE RENAME + collation?: "utf8mb4_bin" | "utf8mb4_unicode_520_ci"; + isVector?: boolean; // mark as vector-oriented table } ``` @@ -207,18 +234,32 @@ interface BUN_MARIADB_TableSchemaType { ```ts type BUN_MARIADB_FieldSchemaType = { fieldName?: string; - dataType: "TEXT" | "INTEGER"; + dataType: + | "CHAR" | "VARCHAR" | "TEXT" | "TINYTEXT" | "MEDIUMTEXT" | "LONGTEXT" + | "TINYINT" | "SMALLINT" | "MEDIUMINT" | "INT" | "BIGINT" + | "FLOAT" | "DOUBLE" | "DECIMAL" + | "BINARY" | "VARBINARY" | "BLOB" | "TINYBLOB" | "MEDIUMBLOB" | "LONGBLOB" + | "DATE" | "TIME" | "DATETIME" | "TIMESTAMP" | "YEAR" + | "BOOLEAN" | "UUID" | "JSON" | "INET6" | "ENUM" | "SET" | "VECTOR"; primaryKey?: boolean; autoIncrement?: boolean; notNullValue?: boolean; unique?: boolean; defaultValue?: string | number; - defaultValueLiteral?: string; // raw SQL literal, e.g. "CURRENT_TIMESTAMP" + defaultValueLiteral?: string; // raw SQL, e.g. "CURRENT_TIMESTAMP" + onUpdate?: string; + onUpdateLiteral?: string; foreignKey?: BUN_MARIADB_ForeignKeyType; - isVector?: boolean; // vector embedding column - vectorSize?: number; // embedding dimensions (default: 1536) - sideCar?: boolean; // sqlite-vec "+" prefix for side-car columns - updatedField?: boolean; // flag that this field definition has changed + integerLength?: string | number; // e.g. VARCHAR length + decimals?: string | number; // DECIMAL scale + options?: (string | number)[]; // ENUM / SET values + isVector?: boolean; // native VECTOR column + vectorSize?: number; // dimensions (default: 1536) + // Content mode flags (editor metadata); only `html: true` affects runtime: + html?: boolean; + markdown?: boolean; + plain?: boolean; + // ... }; ``` @@ -226,8 +267,9 @@ type BUN_MARIADB_FieldSchemaType = { ```ts interface BUN_MARIADB_ForeignKeyType { - destinationTableName: string; - destinationTableColumnName: string; + foreignKeyName?: string; + destinationTableName?: string; + destinationTableColumnName?: string; cascadeDelete?: boolean; cascadeUpdate?: boolean; } @@ -238,8 +280,10 @@ interface BUN_MARIADB_ForeignKeyType { ```ts interface BUN_MARIADB_IndexSchemaType { indexName?: string; - indexType?: "regular" | "full_text" | "vector"; - indexTableFields?: { value: string; dataType: string }[]; + indexTableFields?: string[]; + indexType?: "BTREE" | "HASH" | "FULLTEXT" | "SPATIAL" | "VECTOR"; + vectorDistanceMetric?: "euclidean" | "cosine"; // VECTOR indexes only + comment?: string; } ``` @@ -256,104 +300,95 @@ interface BUN_MARIADB_UniqueConstraintSchemaType { ## CLI Commands -The package provides a `bun-sqlite` CLI binary. +CLI binary: `bun-mariadb`. ### `schema` — Sync database to schema ```bash -bunx bun-sqlite schema [options] +bunx bun-mariadb schema [options] ``` -| Option | Description | -| ----------------- | ---------------------------------------------------------- | -| `-v`, `--vector` | Drop and recreate all vector (`sqlite-vec`) virtual tables | -| `-t`, `--typedef` | Also generate TypeScript type definitions after syncing | +| Option | Description | +| ----------------- | ------------------------------------------------------- | +| `-t`, `--typedef` | Also generate TypeScript type definitions after syncing | **Examples:** ```bash # Sync schema only -bunx bun-sqlite schema +bunx bun-mariadb schema # Sync schema and regenerate types -bunx bun-sqlite schema --typedef - -# Sync schema, recreate vector tables, and regenerate types -bunx bun-sqlite schema --vector --typedef +bunx bun-mariadb schema --typedef ``` +What it does: + +1. Ensures the internal `__db_schema_manager__` tracking table exists +2. Orders tables by foreign-key dependencies +3. Creates missing tables / surgically updates existing ones +4. Syncs indexes (including `VECTOR INDEX`) +5. Drops tables removed from the schema (skips tables still referenced by FKs) +6. Optionally writes TypeScript types and a live schema snapshot + ### `typedef` — Generate TypeScript types only ```bash -bunx bun-sqlite typedef +bunx bun-mariadb typedef ``` -Reads the schema and writes TypeScript type definitions to the path configured in `typedef_file_path`. +Writes types to `typedef_file_path` from config. ---- - -### `backup` — Back up the database +### `backup` — SQL dump backup ```bash -bunx bun-sqlite backup +bunx bun-mariadb backup ``` -Copies the current database file into `db_backup_dir` with a timestamped filename. After copying, the oldest backups are automatically pruned so the number of stored backups never exceeds `max_backups` (default: 10). +Runs `mariadb-dump` (or `mysqldump`) against the configured database and writes a timestamped `.sql` file into `db_backup_dir`. Old dumps are pruned to `max_backups`. -**Example:** +Requires `mariadb-dump` or `mysqldump` on `PATH`. + +### `restore` — Restore from an SQL dump ```bash -bunx bun-sqlite backup -# Backing up database ... -# DB Backup Success! +bunx bun-mariadb restore ``` ---- +Interactive picker of available `.sql` backups (newest first). Imports the selected dump via `mariadb` / `mysql` into the configured database. -### `restore` — Restore the database from a backup +Requires `mariadb` or `mysql` on `PATH`. + +### `admin` — Interactive admin tools ```bash -bunx bun-sqlite restore +bunx bun-mariadb admin ``` -Presents an interactive list of available backups sorted by date (newest first). Select a backup to overwrite the current database file with it. - -**Example:** - -```bash -bunx bun-sqlite restore -# Restoring up database ... -# ? Select a backup: (Use arrow keys) -# ❯ Backup #1: Mon Mar 02 2026 14:30:00 -# Backup #2: Sun Mar 01 2026 09:15:42 -# DB Restore Success! -``` - -> If no backups exist, the command exits with an error and a reminder to run `backup` first. +Interactive menu to list tables / inspect data and run SQL. --- ## CRUD API -Import the default export: - ```ts -import BunSQLite from "@moduletrace/bun-sqlite"; +import BunMariaDB from "@moduletrace/bun-mariadb"; ``` -All methods return an `APIResponseObject`: +All methods return a `DBResponseObject`: ```ts { success: boolean; - payload?: T[]; // array of rows (select) - singleRes?: T; // first row (select) - count?: number; // total count (when count: true) - postInsertReturn?: { - affectedRows?: number; - insertId?: number; + payload?: T[]; // rows (select / returning) + single_res?: T; // first row + count?: number; + insert_return?: { + count?: number; + last_insert_id?: number; + affected_rows?: number; }; - error?: string; + error?: any; msg?: string; debug?: any; } @@ -364,7 +399,7 @@ All methods return an `APIResponseObject`: ### Select ```ts -BunSQLite.select({ table, query?, count?, targetId? }) +BunMariaDB.select({ table, query?, count?, targetId? }) ``` | Parameter | Type | Description | @@ -377,14 +412,11 @@ BunSQLite.select({ table, query?, count?, targetId? }) **Examples:** ```ts -// Get all users -const res = await BunSQLite.select({ table: "users" }); +const res = await BunMariaDB.select({ table: "users" }); -// Get by ID -const res = await BunSQLite.select({ table: "users", targetId: 42 }); +const res = await BunMariaDB.select({ table: "users", targetId: 42 }); -// Filter with LIKE -const res = await BunSQLite.select({ +const res = await BunMariaDB.select({ table: "users", query: { query: { @@ -393,12 +425,10 @@ const res = await BunSQLite.select({ }, }); -// Count rows -const res = await BunSQLite.select({ table: "users", count: true }); +const res = await BunMariaDB.select({ table: "users", count: true }); console.log(res.count); -// Pagination -const res = await BunSQLite.select({ +const res = await BunMariaDB.select({ table: "users", query: { limit: 10, page: 2 }, }); @@ -409,20 +439,21 @@ const res = await BunSQLite.select({ ### Insert ```ts -BunSQLite.insert({ table, data }); +BunMariaDB.insert({ table, data, update_on_duplicate? }) ``` -| Parameter | Type | Description | -| --------- | -------- | ------------------------------ | -| `table` | `string` | Table name | -| `data` | `T[]` | Array of row objects to insert | +| Parameter | Type | Description | +| --------------------- | --------- | ------------------------------------------------ | +| `table` | `string` | Table name | +| `data` | `T[]` | Array of row objects to insert | +| `update_on_duplicate` | `boolean` | Use `ON DUPLICATE KEY UPDATE` when unique exists | `created_at` and `updated_at` are set automatically to `Date.now()`. -**Example:** +Fields with `html: true` are sanitized before insert. ```ts -const res = await BunSQLite.insert({ +const res = await BunMariaDB.insert({ table: "users", data: [ { first_name: "Alice", last_name: "Smith", email: "alice@example.com" }, @@ -430,7 +461,7 @@ const res = await BunSQLite.insert({ ], }); -console.log(res.postInsertReturn?.insertId); // last inserted row ID +console.log(res.insert_return?.last_insert_id); ``` --- @@ -438,7 +469,7 @@ console.log(res.postInsertReturn?.insertId); // last inserted row ID ### Update ```ts -BunSQLite.update({ table, data, query?, targetId? }) +BunMariaDB.update({ table, data, query?, targetId? }) ``` | Parameter | Type | Description | @@ -448,22 +479,16 @@ BunSQLite.update({ table, data, query?, targetId? }) | `query` | `ServerQueryParam` | WHERE clause conditions | | `targetId` | `string \| number` | Shorthand to filter by `id` | -A WHERE clause is required. If no condition matches, `success` is `false`. - -`updated_at` is set automatically to `Date.now()`. - -**Examples:** +A WHERE clause is required. `updated_at` is set automatically. HTML fields are sanitized. ```ts -// Update by ID -await BunSQLite.update({ +await BunMariaDB.update({ table: "users", targetId: 1, data: { first_name: "Alicia" }, }); -// Update with custom query -await BunSQLite.update({ +await BunMariaDB.update({ table: "users", data: { last_name: "Doe" }, query: { @@ -479,25 +504,15 @@ await BunSQLite.update({ ### Delete ```ts -BunSQLite.delete({ table, query?, targetId? }) +BunMariaDB.delete({ table, query?, targetId? }) ``` -| Parameter | Type | Description | -| ---------- | --------------------- | --------------------------- | -| `table` | `string` | Table name | -| `query` | `ServerQueryParam` | WHERE clause conditions | -| `targetId` | `string \| number` | Shorthand to filter by `id` | - -A WHERE clause is required. If no condition is provided, `success` is `false`. - -**Examples:** +A WHERE clause is required. ```ts -// Delete by ID -await BunSQLite.delete({ table: "users", targetId: 1 }); +await BunMariaDB.delete({ table: "users", targetId: 1 }); -// Delete with condition -await BunSQLite.delete({ +await BunMariaDB.delete({ table: "users", query: { query: { @@ -512,39 +527,26 @@ await BunSQLite.delete({ ### Raw SQL ```ts -BunSQLite.sql({ sql, values? }) +BunMariaDB.sql({ sql, values? }) ``` -| Parameter | Type | Description | -| --------- | ---------------------- | -------------------- | -| `sql` | `string` | Raw SQL statement | -| `values` | `(string \| number)[]` | Parameterized values | - -SELECT statements return rows; all other statements return `postInsertReturn`. - -**Examples:** - ```ts -// SELECT -const res = await BunSQLite.sql({ sql: "SELECT * FROM users" }); -console.log(res.payload); - -// INSERT with params -await BunSQLite.sql({ - sql: "INSERT INTO users (first_name, email) VALUES (?, ?)", - values: ["Charlie", "charlie@example.com"], +const res = await BunMariaDB.sql({ + sql: "SELECT * FROM users WHERE id = ?", + values: [1], }); +console.log(res.payload); ``` --- ## Query API Reference -The `query` parameter on `select`, `update`, and `delete` accepts a `ServerQueryParam` object: +`query` on `select` / `update` / `delete` accepts `ServerQueryParam`: ```ts type ServerQueryParam = { - query?: { [key in keyof T]: ServerQueryObject }; + query?: { [key in keyof T]?: ServerQueryObject }; selectFields?: (keyof T | { fieldName: keyof T; alias?: string })[]; omitFields?: (keyof T)[]; limit?: number; @@ -553,12 +555,9 @@ type ServerQueryParam = { order?: | { field: keyof T; strategy: "ASC" | "DESC" } | { field: keyof T; strategy: "ASC" | "DESC" }[]; - searchOperator?: "AND" | "OR"; // how multiple query fields are joined (default: AND) + searchOperator?: "AND" | "OR"; join?: ServerQueryParamsJoin[]; - group?: - | keyof T - | { field: keyof T; table?: string } - | (keyof T | { field: keyof T; table?: string })[]; + group?: keyof T | { field: keyof T; table?: string } | Array<...>; countSubQueries?: ServerQueryParamsCount[]; fullTextSearch?: { fields: (keyof T)[]; @@ -570,32 +569,29 @@ type ServerQueryParam = { ### Equality Operators -Set `equality` on any query field to control the comparison: - | Equality | SQL Equivalent | | ----------------------- | ------------------------------------------------------ | | `EQUAL` (default) | `=` | | `NOT EQUAL` | `!=` | | `LIKE` | `LIKE '%value%'` | -| `LIKE_RAW` | `LIKE 'value'` (no auto-wrapping) | +| `LIKE_RAW` | `LIKE 'value'` | | `LIKE_LOWER` | `LOWER(field) LIKE '%value%'` | | `NOT LIKE` | `NOT LIKE '%value%'` | | `GREATER THAN` | `>` | | `GREATER THAN OR EQUAL` | `>=` | | `LESS THAN` | `<` | | `LESS THAN OR EQUAL` | `<=` | -| `IN` | `IN (val1, val2, ...)` — pass array as value | +| `IN` | `IN (...)` | | `NOT IN` | `NOT IN (...)` | -| `BETWEEN` | `BETWEEN val1 AND val2` — pass `[val1, val2]` as value | +| `BETWEEN` | `BETWEEN a AND b` | | `IS NULL` | `IS NULL` | | `IS NOT NULL` | `IS NOT NULL` | -| `MATCH` | sqlite-vec vector nearest-neighbor search | - -**Example:** +| `REGEXP` | `REGEXP` | +| `FULLTEXT` | full-text match | +| `MATCH` | vector / specialized match | ```ts -// Find users with email NOT NULL, ordered by created_at DESC, limit 20 -const res = await BunSQLite.select({ +const res = await BunMariaDB.select({ table: "users", query: { query: { @@ -610,7 +606,7 @@ const res = await BunSQLite.select({ ### JOIN ```ts -const res = await BunSQLite.select({ +const res = await BunMariaDB.select({ table: "posts", query: { join: [ @@ -627,83 +623,105 @@ const res = await BunSQLite.select({ --- -## Vector Table Support +## Vector Support -`@moduletrace/bun-sqlite` integrates with [`sqlite-vec`](https://github.com/asg017/sqlite-vec) for storing and querying vector embeddings. +MariaDB native `VECTOR(n)` columns and `VECTOR INDEX` are supported (MariaDB 11.7+). -### Define a vector table in the schema +### Schema ```ts { tableName: "documents", isVector: true, - vectorType: "vec0", // defaults to "vec0" fields: [ { fieldName: "embedding", - dataType: "TEXT", + dataType: "VECTOR", isVector: true, - vectorSize: 1536, // embedding dimensions + vectorSize: 1536, }, { fieldName: "title", - dataType: "TEXT", - sideCar: true, // stored as a side-car column (+title) for efficiency + dataType: "VARCHAR", + integerLength: 255, }, + ], + indexes: [ { - fieldName: "body", - dataType: "TEXT", - sideCar: true, + indexName: "idx_documents_embedding", + indexType: "VECTOR", + indexTableFields: ["embedding"], + vectorDistanceMetric: "cosine", // or "euclidean" (default) }, ], } ``` -> **Side-car columns** (`sideCar: true`) use sqlite-vec's `+column` syntax. They are stored separately from the vector index, keeping the index lean and fast while still being queryable alongside vector results. +Notes: -### Sync vector tables +- `isVector: true` or `dataType: "VECTOR"` maps to `VECTOR(n)` (`n` defaults to `1536`) +- Vector columns are created as `NOT NULL` (required for vector indexes) +- Dimension / storage changes trigger an automatic table recreate and data copy +- No special CLI flag is required + +### Sync ```bash -# Initial sync -bunx bun-sqlite schema - -# Recreate vector tables (e.g. after changing vectorSize) -bunx bun-sqlite schema --vector +bunx bun-mariadb schema ``` -### Query vectors +--- + +## HTML Sanitization + +Fields with **explicit** `html: true` are sanitized on **insert** and **update** only. ```ts -const res = await BunSQLite.select({ - table: "documents", - query: { - query: { - embedding: { - equality: "MATCH", - value: "", - vector: true, - vectorFunction: "vec_f32", - }, - }, - limit: 5, - }, -}); +{ fieldName: "body", dataType: "LONGTEXT", html: true } ``` +Other fields (including text that happens to contain HTML tags) are **not** sanitized. + +### Extend the allowlist + +```ts +// bun-mariadb.config.ts +const config: BunMariaDBConfig = { + db_name: "my_app", + db_dir: "./db", + html_sanitize: { + allowed_tags: ["iframe", "video"], + allowed_attributes: { + iframe: ["src", "width", "height", "allow", "allowfullscreen"], + video: ["src", "controls", "width", "height"], + "*": ["data-id"], + }, + }, +}; +``` + +Config values are **appended** to the built-in defaults (safe rich-text tags/attributes). + --- ## TypeScript Type Generation -Run the `typedef` command (or pass `--typedef` to `schema`) to generate a `.ts` file containing: +```bash +bunx bun-mariadb typedef +# or +bunx bun-mariadb schema --typedef +``` -- A `const` array of all table names (`BunSQLiteTables`) -- A `type` for each table (named `BUN_MARIADB__`) -- A union type `BUN_MARIADB__ALL_TYPEDEFS` +Generates: -**Example output** (`db/types/db.ts`): +- A `const` array of table names +- A type per table (`BUN_MARIADB__`) +- A union of all table types + +Example: ```ts -export const BunSQLiteTables = ["users"] as const; +export const BunMariaDBTables = ["users"] as const; export type BUN_MARIADB_MY_APP_USERS = { /** The unique identifier of the record. */ @@ -720,14 +738,12 @@ export type BUN_MARIADB_MY_APP_USERS = { export type BUN_MARIADB_MY_APP_ALL_TYPEDEFS = BUN_MARIADB_MY_APP_USERS; ``` -Use the generated types with the CRUD API for full type safety: - ```ts -import BunSQLite from "@moduletrace/bun-sqlite"; -import { BUN_MARIADB_MY_APP_USERS, BunSQLiteTables } from "./db/types/db"; +import BunMariaDB from "@moduletrace/bun-mariadb"; +import type { BUN_MARIADB_MY_APP_USERS } from "./db/types/db"; -const res = await BunSQLite.select({ - table: "users" as (typeof BunSQLiteTables)[number], +const res = await BunMariaDB.select({ + table: "users", }); ``` @@ -735,64 +751,60 @@ const res = await BunSQLite.select({ ## Default Fields -Every table automatically receives the following fields — you do not need to declare them in your schema: +Every table automatically receives: -| Field | Type | Description | -| ------------ | -------------------------------------------- | -------------------------------------- | -| `id` | `INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL` | Unique row identifier | -| `created_at` | `INTEGER` | Unix timestamp set on insert | -| `updated_at` | `INTEGER` | Unix timestamp updated on every update | +| Field | Type | Description | +| ------------ | ------------------------------- | -------------------------------------- | +| `id` | `BIGINT PRIMARY KEY AUTO_INCREMENT NOT NULL` | Unique row identifier | +| `created_at` | `BIGINT` | Unix timestamp set on insert | +| `updated_at` | `BIGINT` | Unix timestamp updated on every update | + +You do not need to declare these in your schema. --- ## Project Structure ``` -bun-sqlite/ +bun-mariadb/ ├── src/ -│ ├── index.ts # Main export (BunSQLite object) +│ ├── index.ts # Main export (BunMariaDB object) │ ├── commands/ │ │ ├── index.ts # CLI entry point │ │ ├── schema.ts # `schema` command │ │ ├── typedef.ts # `typedef` command │ │ ├── backup.ts # `backup` command -│ │ └── restore.ts # `restore` command +│ │ ├── restore.ts # `restore` command +│ │ └── admin/ # Interactive admin tools │ ├── functions/ -│ │ └── init.ts # Config + schema loader -│ ├── lib/sqlite/ -│ │ ├── index.ts # Database client (bun:sqlite + sqlite-vec) -│ │ ├── db-schema-manager.ts # Schema synchronization engine -│ │ ├── db-select.ts # Select implementation -│ │ ├── db-insert.ts # Insert implementation -│ │ ├── db-update.ts # Update implementation -│ │ ├── db-delete.ts # Delete implementation -│ │ ├── db-sql.ts # Raw SQL implementation -│ │ ├── db-generate-type-defs.ts # Type def generator -│ │ └── schema-to-typedef.ts # Schema-to-TypeScript converter +│ │ ├── init.ts # Config + schema + client loader +│ │ ├── grab-mariadb-client.ts # Bun SQL MariaDB client +│ │ └── live-schema.ts # Live schema snapshot I/O +│ ├── lib/ +│ │ ├── db-handler.ts # Shared query runner +│ │ ├── mariadb/ +│ │ │ ├── db-select.ts +│ │ │ ├── db-insert.ts +│ │ │ ├── db-update.ts +│ │ │ ├── db-delete.ts +│ │ │ ├── db-sql.ts +│ │ │ └── schema-to-typedef.ts +│ │ └── schema/ # Schema sync engine (modular) +│ │ ├── create-db-schema.ts +│ │ ├── create-table.ts +│ │ ├── update-table.ts +│ │ ├── recreate-table.ts +│ │ ├── sync-indexes.ts +│ │ └── ... │ ├── types/ -│ │ └── index.ts # All TypeScript types and interfaces +│ │ └── index.ts │ └── utils/ -│ ├── sql-generator.ts # SELECT query builder -│ ├── sql-insert-generator.ts # INSERT query builder -│ ├── sql-gen-operator-gen.ts # Equality operator mapper -│ ├── sql-equality-parser.ts # Equality string parser -│ ├── append-default-fields-to-db-schema.ts -│ ├── grab-db-dir.ts # Resolve db/backup directory paths -│ ├── grab-db-backup-file-name.ts # Generate timestamped backup filename -│ ├── grab-sorted-backups.ts # List backups sorted newest-first -│ ├── grab-backup-data.ts # Parse metadata from a backup filename -│ └── trim-backups.ts # Prune oldest backups over max_backups +│ ├── sql-generator.ts +│ ├── sql-insert-generator.ts +│ ├── sanitize-html-fields.ts +│ └── ... └── test/ - └── test-01/ # Example project using the library - ├── bun-sqlite.config.ts - ├── db/ - │ ├── bun-sqlite-schema.ts - │ └── types/bun-sqlite.ts # Generated types - └── src/ - ├── insert.ts - ├── select.ts - ├── delete.ts - └── sql.ts + └── coderank/ # Example project ``` --- diff --git a/bun.lock b/bun.lock index 2f2abf5..add1c9a 100644 --- a/bun.lock +++ b/bun.lock @@ -8,23 +8,22 @@ "@inquirer/prompts": "^8.3.0", "chalk": "^5.6.2", "commander": "^14.0.3", - "inquirer": "^13.3.2", "lodash": "^4.17.23", - "mariadb": "^3.5.3", - "mysql": "^2.18.1", "sanitize-html": "^2.17.6", }, "devDependencies": { "@types/bun": "latest", - "@types/inquirer": "^9.0.9", "@types/lodash": "^4.17.24", - "@types/mysql": "^2.15.27", "@types/node": "^25.3.3", "@types/sanitize-html": "^2.16.1", + "typescript": "~5.8.0", }, "peerDependencies": { - "typescript": "^5", + "typescript": ">=5", }, + "optionalPeers": [ + "typescript", + ], }, }, "packages": { @@ -62,22 +61,12 @@ "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], - "@types/geojson": ["@types/geojson@7946.0.16", "", {}, "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg=="], - - "@types/inquirer": ["@types/inquirer@9.0.10", "", { "dependencies": { "@types/through": "*", "rxjs": "^7.2.0" } }, "sha512-vFW2WbXwO9eZpRT5GJGFJ/shgyMNnYozmnjakt9jCQSS1lvqX8pZEQMjJ9RdDPct/YxwciQ8+V8OYn9euIrZDA=="], - "@types/lodash": ["@types/lodash@4.17.24", "", {}, "sha512-gIW7lQLZbue7lRSWEFql49QJJWThrTFFeIMJdp3eH4tKoxm1OvEPg02rm4wCCSHS0cL3/Fizimb35b7k8atwsQ=="], - "@types/mysql": ["@types/mysql@2.15.27", "", { "dependencies": { "@types/node": "*" } }, "sha512-YfWiV16IY0OeBfBCk8+hXKmdTKrKlwKN1MNKAPBu5JYxLwBEZl7QzeEpGnlZb3VMGJrrGmB84gXiH+ofs/TezA=="], - "@types/node": ["@types/node@25.9.4", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-dszCsrKb5U7ZsVZBWiHFklTloVl0mSEnWH/iZXfZUlI4rzCUnsvGmgqfuVRHL54ugE7/wRuxEIXRa2iMZ+BG6g=="], "@types/sanitize-html": ["@types/sanitize-html@2.16.1", "", { "dependencies": { "htmlparser2": "^10.1" } }, "sha512-n9wjs8bCOTyN/ynwD8s/nTcTreIHB1vf31vhLMGqUPNHaweKC4/fAl4Dj+hUlCTKYgm4P3k83fmiFfzkZ6sgMA=="], - "@types/through": ["@types/through@0.0.33", "", { "dependencies": { "@types/node": "*" } }, "sha512-HsJ+z3QuETzP3cswwtzt2vEIiHBk/dCcHGhbmG5X3ecnwFD/lPrMpliGXxSCg03L9AhrdwA4Oz/qfspkDW+xGQ=="], - - "bignumber.js": ["bignumber.js@9.0.0", "", {}, "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A=="], - "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], "chalk": ["chalk@5.6.2", "", {}, "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA=="], @@ -88,14 +77,10 @@ "commander": ["commander@14.0.3", "", {}, "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw=="], - "core-util-is": ["core-util-is@1.0.3", "", {}, "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="], - "dayjs": ["dayjs@1.11.21", "", {}, "sha512-98IT+HOahAisibz/yjKbzuOBwYcjJ7BCLPzARyHiyEBmRz4fatF+KPJszEHXsGYjUG234aH/cOjW1wwTbKUZlA=="], "deepmerge": ["deepmerge@4.3.1", "", {}, "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A=="], - "denque": ["denque@2.1.0", "", {}, "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="], - "dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="], "domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="], @@ -118,26 +103,14 @@ "iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], - "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], - - "inquirer": ["inquirer@13.4.3", "", { "dependencies": { "@inquirer/ansi": "^2.0.5", "@inquirer/core": "^11.1.10", "@inquirer/prompts": "^8.4.3", "@inquirer/type": "^4.0.5", "mute-stream": "^3.0.0", "run-async": "^4.0.6", "rxjs": "^7.8.2" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-EPd3IqieHSavSOXh+LZhrIkdQcOELWeRblLT6kslQr+cF9XTh/HxZdSt1YkHH1iq4dvqBnV42uwg2YlorgOy6g=="], - "is-plain-object": ["is-plain-object@5.0.0", "", {}, "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q=="], - "isarray": ["isarray@1.0.0", "", {}, "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="], - "launder": ["launder@1.7.1", "", { "dependencies": { "dayjs": "^1.11.7" } }, "sha512-mU6WRz5EusL9ZZuiZ5SO4Y6C0P9PAUR9iwdb6bzj4KDihm28DiHFw+/yk9DBH4f+Pv1wuzQ4e2jV3oQ7mkIqvw=="], "lodash": ["lodash@4.18.1", "", {}, "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q=="], - "lru-cache": ["lru-cache@11.5.1", "", {}, "sha512-RPimw/7aMdv2oqRrxKwvZXcPfwBrn/JZ2xYcY9Hus/6LaS3VOAKVWKWgNLCFSiOm1ESXinjsDlidVU7JlnCN2A=="], - - "mariadb": ["mariadb@3.5.3", "", { "dependencies": { "@types/geojson": "^7946.0.16", "@types/node": ">=20", "denque": "^2.1.0", "iconv-lite": "^0.7.2", "lru-cache": "^11.5.0" } }, "sha512-i053Kc0MgdUv/hu9mCyq67TYfPXFj3/MV8I7ZW5wvJNixIyXC0VztMPUjIVj/449nQo+BsxFD4Fdk/sA/uqKPQ=="], - "mute-stream": ["mute-stream@3.0.0", "", {}, "sha512-dkEJPVvun4FryqBmZ5KhDo0K9iDXAwn08tMLDinNdRBNPcYEDiWYysLcc6k3mjTMlbP9KyylvRpd4wFtwrT9rw=="], - "mysql": ["mysql@2.18.1", "", { "dependencies": { "bignumber.js": "9.0.0", "readable-stream": "2.3.7", "safe-buffer": "5.1.2", "sqlstring": "2.3.1" } }, "sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig=="], - "nanoid": ["nanoid@3.3.16", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q=="], "parse-srcset": ["parse-srcset@1.0.2", "", {}, "sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q=="], @@ -146,16 +119,6 @@ "postcss": ["postcss@8.5.19", "", { "dependencies": { "nanoid": "^3.3.12", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-Mz8SaolMd8nB+G13WkORcxQKHZ/NE4xXevtkJHVuG+guo9/wYKlIMTKAqGdEmYOXR2ijPjTYNHssizdaVSUNdQ=="], - "process-nextick-args": ["process-nextick-args@2.0.1", "", {}, "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="], - - "readable-stream": ["readable-stream@2.3.7", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw=="], - - "run-async": ["run-async@4.0.6", "", {}, "sha512-IoDlSLTs3Yq593mb3ZoKWKXMNu3UpObxhgA/Xuid5p4bbfi2jdY1Hj0m1K+0/tEuQTxIGMhQDqGjKb7RuxGpAQ=="], - - "rxjs": ["rxjs@7.8.2", "", { "dependencies": { "tslib": "^2.1.0" } }, "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA=="], - - "safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="], - "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], "sanitize-html": ["sanitize-html@2.17.6", "", { "dependencies": { "deepmerge": "^4.2.2", "escape-string-regexp": "^4.0.0", "htmlparser2": "^12.0.0", "is-plain-object": "^5.0.0", "launder": "^1.7.1", "parse-srcset": "^1.0.2", "postcss": "^8.3.11" } }, "sha512-M4bo9tfv1yfhQZZKkc6dL07ALrGJtfvNOuhX3hU9AVPR/uPQ+nKOJBqTYc7LfMQblTW04mtSWDJWEyLvygJsLA=="], @@ -164,18 +127,10 @@ "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], - "sqlstring": ["sqlstring@2.3.1", "", {}, "sha512-ooAzh/7dxIG5+uDik1z/Rd1vli0+38izZhGzSa34FwR7IbelPWCCKSNIl8jlL/F7ERvy8CB2jNeM1E9i9mXMAQ=="], - - "string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], - - "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], - - "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + "typescript": ["typescript@5.8.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ=="], "undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], - "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], - "dom-serializer/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="], "sanitize-html/htmlparser2": ["htmlparser2@12.0.0", "", { "dependencies": { "domelementtype": "^3.0.0", "domhandler": "^6.0.0", "domutils": "^4.0.2", "entities": "^8.0.0" } }, "sha512-Tz7u1i95/g2x2jz81+x0FBVhBhY5aRTvD3tXXdFaljuNdzDLJ8UGNRrTcj2cgQvAg3iW/h77Fz15nLW0L0CrZw=="], diff --git a/package.json b/package.json index 739e023..732960d 100644 --- a/package.json +++ b/package.json @@ -1,26 +1,42 @@ { "name": "@moduletrace/bun-mariadb", - "version": "1.0.0", - "description": "Mariadb manager for Bun", + "version": "1.0.1", + "description": "Schema-driven MariaDB manager for Bun", "author": "Benjamin Toby", - "main": "dist/index.js", + "license": "MIT", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + } + }, "bin": { - "bun-mariadb": "dist/commands/index.js" + "bun-mariadb": "./dist/commands/index.js" + }, + "engines": { + "bun": ">=1.0.0" }, "scripts": { - "dev": "tsc --watch", - "compile": "rm -rf dist && tsc" + "dev": "tsc --watch" }, "devDependencies": { "@types/bun": "latest", - "@types/inquirer": "^9.0.9", "@types/lodash": "^4.17.24", - "@types/mysql": "^2.15.27", "@types/node": "^25.3.3", - "@types/sanitize-html": "^2.16.1" + "@types/sanitize-html": "^2.16.1", + "typescript": "~5.8.0" }, "peerDependencies": { - "typescript": "^5" + "typescript": ">=5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } }, "files": [ "dist", @@ -31,14 +47,19 @@ "type": "git", "url": "git+https://git.tben.me/Moduletrace/bun-mariadb.git" }, + "keywords": [ + "bun", + "mariadb", + "schema", + "orm", + "sql", + "vector" + ], "dependencies": { "@inquirer/prompts": "^8.3.0", "chalk": "^5.6.2", "commander": "^14.0.3", - "inquirer": "^13.3.2", "lodash": "^4.17.23", - "mariadb": "^3.5.3", - "mysql": "^2.18.1", "sanitize-html": "^2.17.6" } } diff --git a/publish.sh b/publish.sh index c52bbc5..04ba013 100755 --- a/publish.sh +++ b/publish.sh @@ -1,8 +1,8 @@ #!/bin/bash -set -e +set -euo pipefail -if [ -z "$1" ]; then +if [ -z "${1:-}" ]; then msg="Updates" else msg="$1" @@ -11,7 +11,14 @@ fi tsc --noEmit rm -rf dist tsc + +# Ensure CLI shebang survived compile +if ! head -1 dist/commands/index.js | grep -q '#!/usr/bin/env bun'; then + echo "ERROR: dist/commands/index.js is missing the bun shebang" + exit 1 +fi + git add . -git commit -m "$msg" +git commit -m "$msg" || true git push bun publish diff --git a/src/commands/admin/index.ts b/src/commands/admin/index.ts index 8690d10..a9b38d4 100644 --- a/src/commands/admin/index.ts +++ b/src/commands/admin/index.ts @@ -1,5 +1,4 @@ import { Command } from "commander"; -import grabDBDir from "../../utils/grab-db-dir"; import chalk from "chalk"; import { select } from "@inquirer/prompts"; import listTables from "./list-tables"; @@ -9,9 +8,6 @@ export default function () { return new Command("admin") .description("View Tables and Data, Run SQL Queries, Etc.") .action(async () => { - const config = global.CONFIG; - const { db_file_path } = grabDBDir({ config }); - console.log(chalk.bold(chalk.blue("\nBun MariaDB Admin\n"))); try { diff --git a/src/commands/admin/list-tables.ts b/src/commands/admin/list-tables.ts index 0d8711e..6146001 100644 --- a/src/commands/admin/list-tables.ts +++ b/src/commands/admin/list-tables.ts @@ -4,24 +4,16 @@ import { AppData } from "../../data/app-data"; import showEntries from "./show-entries"; import showFields from "./show-fields"; import dbHandler from "../../lib/db-handler"; +import MariaDBQuoteGen from "../../lib/schema/mariadb-quote-gen"; type Params = {}; -type ColumnInfo = { - cid: number; - name: string; - type: string; - notnull: number; - dflt_value: string | null; - pk: number; -}; - export default async function listTables( params?: Params, ): Promise<"__exit__" | void> { const tables = ( - await dbHandler({ - query: `SELECT table_name FROM ${AppData["DbSchemaManagerTableName"]}`, + await dbHandler<{ table_name: string }>({ + query: `SELECT table_name FROM ${MariaDBQuoteGen(AppData["DbSchemaManagerTableName"])}`, }) ).payload; @@ -30,7 +22,6 @@ export default async function listTables( return; } - // Level 1: table selection loop while (true) { const tableName = await select({ message: "Select a table:", @@ -47,7 +38,6 @@ export default async function listTables( if (tableName === "__back__") break; if (tableName === "__exit__") return "__exit__"; - // Level 2: action loop — stays here until "Go Back" while (true) { const action = await select({ message: `"${tableName}" — choose an action:`, @@ -73,24 +63,40 @@ export default async function listTables( if (result === "__exit__") return "__exit__"; } - // if (action === "schema") { - // const columns = db - // .query(`PRAGMA table_info("${tableName}")`) - // .all(); + if (action === "schema") { + const columns = ( + await dbHandler<{ + ORDINAL_POSITION: number; + COLUMN_NAME: string; + COLUMN_TYPE: string; + IS_NULLABLE: string; + COLUMN_DEFAULT: string | null; + COLUMN_KEY: string; + EXTRA: string; + }>({ + query: `SELECT ORDINAL_POSITION, COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, COLUMN_KEY, EXTRA FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION`, + values: [tableName], + }) + ).payload; - // console.log(`\n${chalk.bold(`Schema for "${tableName}":`)} \n`); - // console.table( - // columns.map((c) => ({ - // "#": c.cid, - // Name: c.name, - // Type: c.type, - // "Not Null": c.notnull ? "YES" : "NO", - // Default: c.dflt_value ?? "(none)", - // "Primary Key": c.pk ? "YES" : "NO", - // })), - // ); - // console.log(); - // } + console.log(`\n${chalk.bold(`Schema for "${tableName}":`)} \n`); + if (columns?.length) { + console.table( + columns.map((c) => ({ + "#": c.ORDINAL_POSITION, + Name: c.COLUMN_NAME, + Type: c.COLUMN_TYPE, + Nullable: c.IS_NULLABLE, + Default: c.COLUMN_DEFAULT ?? "(none)", + Key: c.COLUMN_KEY || "", + Extra: c.EXTRA || "", + })), + ); + } else { + console.log(chalk.yellow("No columns found.")); + } + console.log(); + } } } } diff --git a/src/commands/admin/run-sql.ts b/src/commands/admin/run-sql.ts index b5511d4..823d0d4 100644 --- a/src/commands/admin/run-sql.ts +++ b/src/commands/admin/run-sql.ts @@ -13,18 +13,19 @@ export default async function runSQL(params?: Params) { try { const res = await dbHandler({ query: sql }); - if (res.payload) { + if (res.payload?.length) { console.log( `\n${chalk.bold(`Result (${res.payload.length} row${res.payload.length !== 1 ? "s" : ""}):`)} \n`, ); - if (res.payload.length) console.table(res.payload); - else console.log(chalk.yellow("No res returned.\n")); - } else if (res.single_res) { + console.table(res.payload); + } else if (res.success) { console.log( chalk.green( - `\nSuccess! Affected rows: ${res.single_res.changes}, Last insert ID: ${res.single_res.lastInsertRowid}\n`, + `\nSuccess! Affected rows: ${res.insert_return?.affected_rows ?? res.count ?? 0}, Last insert ID: ${res.insert_return?.last_insert_id ?? "—"}\n`, ), ); + } else { + console.error(chalk.red(`\nSQL Error: ${res.error || res.msg}\n`)); } } catch (error: any) { console.error(chalk.red(`\nSQL Error: ${error.message}\n`)); diff --git a/src/commands/admin/show-entries.ts b/src/commands/admin/show-entries.ts index 8dd1c6b..2ed9970 100644 --- a/src/commands/admin/show-entries.ts +++ b/src/commands/admin/show-entries.ts @@ -1,11 +1,11 @@ import chalk from "chalk"; import { select, input } from "@inquirer/prompts"; import dbHandler from "../../lib/db-handler"; +import MariaDBQuoteGen from "../../lib/schema/mariadb-quote-gen"; type Params = { tableName: string; }; -type ColumnInfo = { cid: number; name: string }; const LIMIT = 50; @@ -14,36 +14,38 @@ export default async function showEntries({ tableName }: Params) { let searchField: string | null = null; let searchTerm: string | null = null; + const quotedTable = MariaDBQuoteGen(tableName); + while (true) { const offset = page * LIMIT; const rows = searchTerm ? ( await dbHandler({ - query: `SELECT * FROM "${tableName}" WHERE "${searchField}" LIKE ? LIMIT ${LIMIT} OFFSET ${offset}`, + query: `SELECT * FROM ${quotedTable} WHERE ${MariaDBQuoteGen(searchField!)} LIKE ? LIMIT ${LIMIT} OFFSET ${offset}`, values: [`%${searchTerm}%`], }) ).payload : ( await dbHandler({ - query: `SELECT * FROM "${tableName}" LIMIT ${LIMIT} OFFSET ${offset}`, + query: `SELECT * FROM ${quotedTable} LIMIT ${LIMIT} OFFSET ${offset}`, }) ).payload; const countRow = searchTerm ? ( - await dbHandler({ - query: `SELECT COUNT(*) as count FROM "${tableName}" WHERE "${searchField}" LIKE ?`, + await dbHandler<{ count: number }>({ + query: `SELECT COUNT(*) as count FROM ${quotedTable} WHERE ${MariaDBQuoteGen(searchField!)} LIKE ?`, values: [`%${searchTerm}%`], }) ).payload : ( - await dbHandler({ - query: `SELECT COUNT(*) as count FROM "${tableName}"`, + await dbHandler<{ count: number }>({ + query: `SELECT COUNT(*) as count FROM ${quotedTable}`, }) ).payload; - const total = countRow?.[0]?.count; + const total = Number(countRow?.[0]?.count || 0); const searchInfo = searchTerm ? chalk.dim(` · searching "${searchField}" = "${searchTerm}"`) : ""; @@ -57,8 +59,7 @@ export default async function showEntries({ tableName }: Params) { ); if (rows.length) { - console.log(rows); - // if (rows.length) console.table(rows); + console.table(rows); } else { console.log(chalk.yellow("No rows found.")); console.log(); @@ -85,19 +86,31 @@ export default async function showEntries({ tableName }: Params) { searchTerm = null; page = 0; } - // if (action === "search") { - // const columns = db - // .query(`PRAGMA table_info("${tableName}")`) - // .all(); - // searchField = await select({ - // message: "Search by field:", - // choices: columns.map((c) => ({ name: c.name, value: c.name })), - // }); - // searchTerm = await input({ - // message: `Search term for "${searchField}":`, - // validate: (v) => v.trim().length > 0 || "Cannot be empty", - // }); - // page = 0; - // } + if (action === "search") { + const columns = ( + await dbHandler<{ COLUMN_NAME: string }>({ + query: `SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION`, + values: [tableName], + }) + ).payload; + + if (!columns?.length) { + console.log(chalk.yellow("No columns found for search.")); + continue; + } + + searchField = await select({ + message: "Search by field:", + choices: columns.map((c) => ({ + name: c.COLUMN_NAME, + value: c.COLUMN_NAME, + })), + }); + searchTerm = await input({ + message: `Search term for "${searchField}":`, + validate: (v) => v.trim().length > 0 || "Cannot be empty", + }); + page = 0; + } } } diff --git a/src/commands/admin/show-fields.ts b/src/commands/admin/show-fields.ts index 57f003e..7a58432 100644 --- a/src/commands/admin/show-fields.ts +++ b/src/commands/admin/show-fields.ts @@ -1,92 +1,109 @@ +import chalk from "chalk"; +import { select } from "@inquirer/prompts"; +import dbHandler from "../../lib/db-handler"; + type Params = { tableName: string; }; -type ColumnInfo = { - cid: number; - name: string; - type: string; - notnull: number; - dflt_value: string | null; - pk: number; -}; - -type IndexInfo = { name: string; unique: number; origin: string }; -type IndexColumn = { name: string }; -type ForeignKey = { - id: number; - from: string; - table: string; - to: string; - on_update: string; - on_delete: string; -}; - export default async function showFields({ tableName, }: Params): Promise<"__exit__" | void> { - // const columns = db - // .query(`PRAGMA table_info("${tableName}")`) - // .all(); - // const indexes = db - // .query(`PRAGMA index_list("${tableName}")`) - // .all(); - // const foreignKeys = db - // .query(`PRAGMA foreign_key_list("${tableName}")`) - // .all(); - // const indexedFields = new Map(); - // for (const idx of indexes) { - // const cols = db - // .query(`PRAGMA index_info("${idx.name}")`) - // .all(); - // for (const col of cols) { - // indexedFields.set(col.name, { unique: idx.unique === 1 }); - // } - // } - // while (true) { - // const fieldName = await select({ - // message: `"${tableName}" — select a field:`, - // choices: [ - // ...columns.map((c) => ({ name: c.name, value: c.name })), - // { name: chalk.dim("← Go Back"), value: "__back__" }, - // { name: chalk.dim("✕ Exit"), value: "__exit__" }, - // ], - // }); - // if (fieldName === "__back__") break; - // if (fieldName === "__exit__") return "__exit__"; - // const col = columns.find((c) => c.name === fieldName)!; - // const idx = indexedFields.get(fieldName); - // const fk = foreignKeys.find((f) => f.from === fieldName); - // console.log(`\n${chalk.bold(`Field: "${fieldName}"`)}\n`); - // console.log(` ${chalk.dim("Table")} ${tableName}`); - // console.log(` ${chalk.dim("Column #")} ${col.cid}`); - // console.log( - // ` ${chalk.dim("Type")} ${col.type || chalk.italic("(none)")}`, - // ); - // console.log( - // ` ${chalk.dim("Primary Key")} ${col.pk ? chalk.green("YES") : "NO"}`, - // ); - // console.log( - // ` ${chalk.dim("Not Null")} ${col.notnull ? chalk.yellow("YES") : "NO"}`, - // ); - // console.log( - // ` ${chalk.dim("Default")} ${col.dflt_value ?? chalk.italic("(none)")}`, - // ); - // console.log( - // ` ${chalk.dim("Indexed")} ${idx ? chalk.cyan("YES") : "NO"}`, - // ); - // console.log( - // ` ${chalk.dim("Unique")} ${idx?.unique ? chalk.cyan("YES") : "NO"}`, - // ); - // if (fk) { - // console.log( - // ` ${chalk.dim("Foreign Key")} ${chalk.magenta(`${fk.table}(${fk.to})`)}`, - // ); - // console.log(` ${chalk.dim("On Update")} ${fk.on_update}`); - // console.log(` ${chalk.dim("On Delete")} ${fk.on_delete}`); - // } else { - // console.log(` ${chalk.dim("Foreign Key")} NO`); - // } - // console.log(); - // } + const columns = ( + await dbHandler<{ + ORDINAL_POSITION: number; + COLUMN_NAME: string; + COLUMN_TYPE: string; + IS_NULLABLE: string; + COLUMN_DEFAULT: string | null; + COLUMN_KEY: string; + EXTRA: string; + COLUMN_COMMENT: string; + }>({ + query: `SELECT ORDINAL_POSITION, COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, COLUMN_KEY, EXTRA, COLUMN_COMMENT FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION`, + values: [tableName], + }) + ).payload; + + if (!columns?.length) { + console.log(chalk.yellow(`\nNo columns found for "${tableName}".\n`)); + return; + } + + const indexes = ( + await dbHandler<{ + INDEX_NAME: string; + COLUMN_NAME: string; + NON_UNIQUE: number; + INDEX_TYPE: string; + }>({ + query: `SELECT INDEX_NAME, COLUMN_NAME, NON_UNIQUE, INDEX_TYPE FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY INDEX_NAME, SEQ_IN_INDEX`, + values: [tableName], + }) + ).payload; + + const foreignKeys = ( + await dbHandler<{ + CONSTRAINT_NAME: string; + COLUMN_NAME: string; + REFERENCED_TABLE_NAME: string; + REFERENCED_COLUMN_NAME: string; + }>({ + query: `SELECT CONSTRAINT_NAME, COLUMN_NAME, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME FROM information_schema.KEY_COLUMN_USAGE WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND REFERENCED_TABLE_NAME IS NOT NULL`, + values: [tableName], + }) + ).payload; + + while (true) { + const fieldName = await select({ + message: `"${tableName}" — select a field:`, + choices: [ + ...columns.map((c) => ({ + name: c.COLUMN_NAME, + value: c.COLUMN_NAME, + })), + { name: chalk.dim("← Go Back"), value: "__back__" }, + { name: chalk.dim("✕ Exit"), value: "__exit__" }, + ], + }); + + if (fieldName === "__back__") break; + if (fieldName === "__exit__") return "__exit__"; + + const col = columns.find((c) => c.COLUMN_NAME === fieldName)!; + const colIndexes = + indexes?.filter((i) => i.COLUMN_NAME === fieldName) || []; + const fk = foreignKeys?.find((f) => f.COLUMN_NAME === fieldName); + + console.log(`\n${chalk.bold(`Field: "${fieldName}"`)}\n`); + console.log(` ${chalk.dim("Table")} ${tableName}`); + console.log(` ${chalk.dim("Column #")} ${col.ORDINAL_POSITION}`); + console.log(` ${chalk.dim("Type")} ${col.COLUMN_TYPE}`); + console.log( + ` ${chalk.dim("Primary Key")} ${col.COLUMN_KEY === "PRI" ? chalk.green("YES") : "NO"}`, + ); + console.log( + ` ${chalk.dim("Not Null")} ${col.IS_NULLABLE === "NO" ? chalk.yellow("YES") : "NO"}`, + ); + console.log( + ` ${chalk.dim("Default")} ${col.COLUMN_DEFAULT ?? chalk.italic("(none)")}`, + ); + console.log( + ` ${chalk.dim("Extra")} ${col.EXTRA || chalk.italic("(none)")}`, + ); + console.log( + ` ${chalk.dim("Indexed")} ${colIndexes.length ? chalk.cyan(colIndexes.map((i) => i.INDEX_NAME).join(", ")) : "NO"}`, + ); + if (fk) { + console.log( + ` ${chalk.dim("Foreign Key")} ${chalk.magenta(`${fk.REFERENCED_TABLE_NAME}(${fk.REFERENCED_COLUMN_NAME})`)}`, + ); + } else { + console.log(` ${chalk.dim("Foreign Key")} NO`); + } + if (col.COLUMN_COMMENT) { + console.log(` ${chalk.dim("Comment")} ${col.COLUMN_COMMENT}`); + } + console.log(); + } } diff --git a/src/commands/backup.ts b/src/commands/backup.ts index 5545f4f..243e6af 100644 --- a/src/commands/backup.ts +++ b/src/commands/backup.ts @@ -1,28 +1,98 @@ import { Command } from "commander"; import path from "path"; -import grabDBDir from "../utils/grab-db-dir"; import fs from "fs"; +import grabDBDir from "../utils/grab-db-dir"; import grabDBBackupFileName from "../utils/grab-db-backup-file-name"; import chalk from "chalk"; import trimBackups from "../utils/trim-backups"; +import mariadbCliEnv, { + mariadbCliConnectionArgs, +} from "../utils/mariadb-cli-env"; + +/** + * Prefer mariadb-dump, fall back to mysqldump. + */ +function resolveDumpBinary(): string { + const candidates = ["mariadb-dump", "mysqldump"]; + for (const bin of candidates) { + try { + const result = Bun.spawnSync(["which", bin], { + stdout: "pipe", + stderr: "pipe", + }); + if (result.exitCode === 0) { + return new TextDecoder().decode(result.stdout).trim() || bin; + } + } catch { + // try next + } + } + return "mariadb-dump"; +} export default function () { return new Command("backup") - .description("Backup Database") - .action(async (opts) => { + .description("Backup MariaDB database (SQL dump)") + .action(async () => { console.log(`Backing up database ...`); const config = global.CONFIG; + const { backup_dir } = grabDBDir({ config }); - const { backup_dir, db_file_path } = grabDBDir({ config }); + if (!fs.existsSync(backup_dir)) { + fs.mkdirSync(backup_dir, { recursive: true }); + } - const new_db_file_name = grabDBBackupFileName({ config }); + const backup_file_name = grabDBBackupFileName({ config }); + const backup_path = path.join(backup_dir, backup_file_name); - fs.cpSync(db_file_path, path.join(backup_dir, new_db_file_name)); + const dumpBin = resolveDumpBinary(); + const args = [ + dumpBin, + ...mariadbCliConnectionArgs(), + "--single-transaction", + "--routines", + "--triggers", + "--events", + config.db_name, + ]; - trimBackups({ config }); + try { + const proc = Bun.spawn(args, { + stdout: "pipe", + stderr: "pipe", + env: mariadbCliEnv(), + }); - console.log(`${chalk.bold(chalk.green(`DB Backup Success!`))}`); - process.exit(); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + + if (exitCode !== 0) { + console.error( + chalk.red( + `Backup failed (exit ${exitCode}): ${stderr || "unknown error"}`, + ), + ); + process.exit(1); + } + + fs.writeFileSync(backup_path, stdout, "utf-8"); + trimBackups({ config }); + + console.log( + `${chalk.bold(chalk.green(`DB Backup Success!`))} → ${backup_path}`, + ); + process.exit(0); + } catch (error: any) { + console.error( + chalk.red( + `Backup ERROR => ${error.message}. Ensure \`mariadb-dump\` or \`mysqldump\` is installed.`, + ), + ); + process.exit(1); + } }); } diff --git a/src/commands/restore.ts b/src/commands/restore.ts index 7ff67b3..6f2005c 100644 --- a/src/commands/restore.ts +++ b/src/commands/restore.ts @@ -6,22 +6,47 @@ import grabSortedBackups from "../utils/grab-sorted-backups"; import { select } from "@inquirer/prompts"; import grabBackupData from "../utils/grab-backup-data"; import path from "path"; +import mariadbCliEnv, { + mariadbCliConnectionArgs, +} from "../utils/mariadb-cli-env"; + +/** + * Prefer mariadb client, fall back to mysql. + */ +function resolveClientBinary(): string { + const candidates = ["mariadb", "mysql"]; + for (const bin of candidates) { + try { + const result = Bun.spawnSync(["which", bin], { + stdout: "pipe", + stderr: "pipe", + }); + if (result.exitCode === 0) { + return new TextDecoder().decode(result.stdout).trim() || bin; + } + } catch { + // try next + } + } + return "mariadb"; +} export default function () { return new Command("restore") - .description("Restore Database") - .action(async (opts) => { - console.log(`Restoring up database ...`); + .description("Restore MariaDB database from an SQL dump") + .action(async () => { + console.log(`Restoring database ...`); const config = global.CONFIG; + const { backup_dir } = grabDBDir({ config }); - const { backup_dir, db_file_path } = grabDBDir({ config }); - - const backups = grabSortedBackups({ config }); + const backups = grabSortedBackups({ config }).filter((b) => + b.endsWith(".sql"), + ); if (!backups?.[0]) { console.error( - `No Backups to restore. Use the \`backup\` command to create a backup`, + `No SQL backups to restore. Use the \`backup\` command to create a backup.`, ); process.exit(1); } @@ -40,16 +65,49 @@ export default function () { }), }); - fs.cpSync(path.join(backup_dir, selected_backup), db_file_path); + const backup_path = path.join(backup_dir, selected_backup); + if (!fs.existsSync(backup_path)) { + console.error(`Backup file not found: ${backup_path}`); + process.exit(1); + } + + const clientBin = resolveClientBinary(); + const sql = fs.readFileSync(backup_path, "utf-8"); + + const args = [ + clientBin, + ...mariadbCliConnectionArgs(), + config.db_name, + ]; + + const proc = Bun.spawn(args, { + stdin: new Blob([sql]), + stdout: "pipe", + stderr: "pipe", + env: mariadbCliEnv(), + }); + + const [stderr, exitCode] = await Promise.all([ + new Response(proc.stderr).text(), + proc.exited, + ]); + + if (exitCode !== 0) { + console.error( + chalk.red( + `Restore failed (exit ${exitCode}): ${stderr || "unknown error"}`, + ), + ); + process.exit(1); + } console.log( - `${chalk.bold(chalk.green(`DB Restore Success!`))}`, + `${chalk.bold(chalk.green(`DB Restore Success!`))} ← ${selected_backup}`, ); - - process.exit(); + process.exit(0); } catch (error: any) { console.error(`Backup Restore ERROR => ${error.message}`); - process.exit(); + process.exit(1); } }); } diff --git a/src/commands/schema.ts b/src/commands/schema.ts index ed00d10..1ddc5b7 100644 --- a/src/commands/schema.ts +++ b/src/commands/schema.ts @@ -1,9 +1,7 @@ import { Command } from "commander"; -import { MariaDBSchemaManager } from "../lib/mariadb/db-schema-manager"; import grabDirNames from "../data/grab-dir-names"; import path from "path"; import dbSchemaToTypeDef from "../lib/mariadb/schema-to-typedef"; -import _ from "lodash"; import appendDefaultFieldsToDbSchema from "../utils/append-default-fields-to-db-schema"; import chalk from "chalk"; import { writeLiveSchema } from "../functions/live-schema"; @@ -12,10 +10,6 @@ import createDBSchema from "../lib/schema/create-db-schema"; export default function () { return new Command("schema") .description("Build DB From Schema") - .option( - "-v, --vector", - "Recreate Vector Tables. This will drop and rebuild all vector tables", - ) .option("-t, --typedef", "Generate typescript type definitions") .action(async (opts) => { console.log(`Starting process ...`); @@ -32,14 +26,10 @@ export default function () { dbSchema, }); - // const manager = new MariaDBSchemaManager({ - // schema: finaldbSchema, - // }); - - // await manager.syncSchema(); - // manager.close(); - - await createDBSchema({ db_schema: finaldbSchema }); + await createDBSchema({ + db_schema: finaldbSchema, + config, + }); if (isTypeDef && config.typedef_file_path) { const out_file = path.resolve( @@ -59,9 +49,9 @@ export default function () { console.log( `${chalk.bold(chalk.green(`DB Schema setup success!`))}`, ); - process.exit(); + process.exit(0); } catch (error) { - console.log(error); + console.error(error); process.exit(1); } }); diff --git a/src/commands/typedef.ts b/src/commands/typedef.ts index 4edec32..dc09c98 100644 --- a/src/commands/typedef.ts +++ b/src/commands/typedef.ts @@ -7,8 +7,8 @@ import chalk from "chalk"; export default function () { return new Command("typedef") - .description("Build DB From Schema") - .action(async (opts) => { + .description("Generate TypeScript type definitions from the schema") + .action(async () => { console.log(`Creating Type Definition From DB Schema ...`); const config = global.CONFIG; @@ -18,23 +18,21 @@ export default function () { const finaldbSchema = appendDefaultFieldsToDbSchema({ dbSchema }); - if (config.typedef_file_path) { - const out_file = path.resolve( - ROOT_DIR, - config.typedef_file_path, + if (!config.typedef_file_path) { + console.error( + `\`typedef_file_path\` is required in bun-mariadb.config.ts to generate types.`, ); - dbSchemaToTypeDef({ - dbSchema: finaldbSchema, - dst_file: out_file, - config, - }); - } else { - console.error(``); process.exit(1); } - console.log(`${chalk.bold(chalk.green(`Typedef gen success!`))}`); + const out_file = path.resolve(ROOT_DIR, config.typedef_file_path); + dbSchemaToTypeDef({ + dbSchema: finaldbSchema, + dst_file: out_file, + config, + }); - process.exit(); + console.log(`${chalk.bold(chalk.green(`Typedef gen success!`))}`); + process.exit(0); }); } diff --git a/src/data/grab-dir-names.ts b/src/data/grab-dir-names.ts index 5b85663..b52bb47 100644 --- a/src/data/grab-dir-names.ts +++ b/src/data/grab-dir-names.ts @@ -9,10 +9,6 @@ export default function grabDirNames(params?: Params) { const ROOT_DIR = process.cwd(); const BUN_MARIADB_DIR = path.join(ROOT_DIR, ".bun-mariadb"); const BUN_MARIADB_TEMP_DIR = path.join(BUN_MARIADB_DIR, ".tmp"); - const BUN_MARIADB_TEMP_DB_FILE_PATH = path.join( - BUN_MARIADB_TEMP_DIR, - "temp.db", - ); const BUN_MARIADB_LIVE_SCHEMA = path.join( BUN_MARIADB_DIR, "live-schema.json", @@ -23,6 +19,5 @@ export default function grabDirNames(params?: Params) { BUN_MARIADB_DIR, BUN_MARIADB_TEMP_DIR, BUN_MARIADB_LIVE_SCHEMA, - BUN_MARIADB_TEMP_DB_FILE_PATH, }; } diff --git a/src/functions/grab-mariadb-client.ts b/src/functions/grab-mariadb-client.ts index acad097..2b8760c 100644 --- a/src/functions/grab-mariadb-client.ts +++ b/src/functions/grab-mariadb-client.ts @@ -7,12 +7,40 @@ type Params = { config: BunMariaDBConfig; }; +/** Reuse one client per database name to avoid exhausting MariaDB connections. */ +const clientCache = new Map(); + export default function grabMariaDBClient({ config, }: Params): Bun.SQL | undefined { + const cacheKey = config.db_name || "__default__"; + + const cached = clientCache.get(cacheKey); + if (cached) { + return cached; + } + + // Prefer the process-wide client when it matches the requested database + if ( + global.MARIADB_CLIENT && + (!global.CONFIG?.db_name || global.CONFIG.db_name === config.db_name) + ) { + clientCache.set(cacheKey, global.MARIADB_CLIENT); + return global.MARIADB_CLIENT; + } + const { ROOT_DIR } = grabDirNames(); try { + const tls = config.ssl_ca + ? { + ca: Bun.file(path.resolve(ROOT_DIR, config.ssl_ca)), + rejectUnauthorized: false, + } + : { + rejectUnauthorized: false, + }; + const MariaDBClient = new SQL({ hostname: process.env.BUN_MARIADB_SERVER_HOST, username: process.env.BUN_MARIADB_SERVER_USERNAME, @@ -22,17 +50,11 @@ export default function grabMariaDBClient({ ? Number(process.env.BUN_MARIADB_SERVER_PORT) : undefined, ...config.db_config, - tls: config.ssl_ca - ? { - ca: Bun.file(path.resolve(ROOT_DIR, config.ssl_ca)), - rejectUnauthorized: false, - } - : { - rejectUnauthorized: false, - }, + tls, adapter: "mariadb", - }); + } as Bun.SQL.Options); + clientCache.set(cacheKey, MariaDBClient); return MariaDBClient; } catch (error: any) { console.error(`Couldn't grab MariaDB Client => ` + error.message); diff --git a/src/functions/init.ts b/src/functions/init.ts index 8abd767..9c5a47b 100644 --- a/src/functions/init.ts +++ b/src/functions/init.ts @@ -7,7 +7,6 @@ import { type BUN_MARIADB_DatabaseSchemaType, RequiredENVs, } from "../types"; -import { SQL } from "bun"; import setMariaDBClient from "./set-mariadb-client"; /** @@ -70,8 +69,8 @@ export default function init(): void { } const db_dir = path.resolve(ROOT_DIR, Config.db_dir); - if (!fs.existsSync(Config.db_dir)) { - fs.mkdirSync(Config.db_dir, { recursive: true }); + if (!fs.existsSync(db_dir)) { + fs.mkdirSync(db_dir, { recursive: true }); } const DBSchemaFilePath = path.join(db_dir, AppData["DbSchemaFileName"]); diff --git a/src/index.ts b/src/index.ts index 64f7c37..f15e03b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -22,3 +22,15 @@ const BunMariaDB = { } as const; export default BunMariaDB; + +export type { + BunMariaDBConfig, + BUN_MARIADB_DatabaseSchemaType, + BUN_MARIADB_TableSchemaType, + BUN_MARIADB_FieldSchemaType, + BUN_MARIADB_IndexSchemaType, + BUN_MARIADB_UniqueConstraintSchemaType, + BUN_MARIADB_ForeignKeyType, + DBResponseObject, + ServerQueryParam, +} from "./types"; diff --git a/src/lib/db-handler.ts b/src/lib/db-handler.ts index 1823736..ef89078 100644 --- a/src/lib/db-handler.ts +++ b/src/lib/db-handler.ts @@ -1,66 +1,86 @@ -import grabMariaDBClient from "../functions/grab-mariadb-client"; -import type { - BunMariaDBConfig, - DBInsertReturn, - DBResponseObject, -} from "../types"; - -type Param = { - query: string; - values?: any[]; - config?: BunMariaDBConfig; -}; - -/** - * # Main DB Handler Function - * @requires DSQL_DB_CONN - Gobal Variable for Datasquirel Database - */ -export default async function dbHandler< - T extends { [k: string]: any } = { [k: string]: any }, ->({ query, values, config }: Param): Promise> { - try { - const CLIENT = config - ? grabMariaDBClient({ config }) - : global.MARIADB_CLIENT; - - if (!CLIENT) { - throw new Error(`Couldn't grab MariaDB Client.`); - } - - const res = await CLIENT.unsafe(query, values); - - const count = res.count; - - const res_array = (() => { - try { - return JSON.parse(JSON.stringify(res)) as T[]; - } catch (error) { - return undefined; - } - })(); - - const last_insert_id = res_array?.[0] ? res_array[0]?.id : undefined; - const affected_rows = res_array?.[0] ? res_array.length : undefined; - - const insert_return: DBInsertReturn = { - count, - last_insert_id, - affected_rows, - }; - - return { - success: true, - payload: res_array, - single_res: res_array?.[0], - insert_return, - count, - }; - } catch (error: any) { - return { - success: false, - error: "DB Handler Error => " + error.message, - msg: "DB Handler Error => " + error.message, - }; - } finally { - } -} +import grabMariaDBClient from "../functions/grab-mariadb-client"; +import type { + BunMariaDBConfig, + DBInsertReturn, + DBResponseObject, +} from "../types"; + +type Param = { + query: string; + values?: any[]; + config?: BunMariaDBConfig; +}; + +/** + * Resolve a shared MariaDB client. Prefer the global client so schema sync + * and CRUD do not open a new connection per query. + */ +function resolveClient(config?: BunMariaDBConfig): Bun.SQL | undefined { + const resolvedConfig = config || global.CONFIG; + + // Same database as the global client → reuse it + if ( + global.MARIADB_CLIENT && + (!resolvedConfig?.db_name || + !global.CONFIG?.db_name || + resolvedConfig.db_name === global.CONFIG.db_name) + ) { + return global.MARIADB_CLIENT; + } + + if (resolvedConfig) { + return grabMariaDBClient({ config: resolvedConfig }); + } + + return global.MARIADB_CLIENT; +} + +/** + * # Main DB Handler Function + */ +export default async function dbHandler< + T extends { [k: string]: any } = { [k: string]: any }, +>({ query, values, config }: Param): Promise> { + try { + const CLIENT = resolveClient(config); + + if (!CLIENT) { + throw new Error(`Couldn't grab MariaDB Client.`); + } + + const res = await CLIENT.unsafe(query, values); + + const count = res.count; + + const res_array = (() => { + try { + return JSON.parse(JSON.stringify(res)) as T[]; + } catch (error) { + return undefined; + } + })(); + + const last_insert_id = res_array?.[0] ? res_array[0]?.id : undefined; + const affected_rows = res_array?.[0] ? res_array.length : undefined; + + const insert_return: DBInsertReturn = { + count, + last_insert_id, + affected_rows, + }; + + return { + success: true, + payload: res_array, + single_res: res_array?.[0], + insert_return, + count, + }; + } catch (error: any) { + return { + success: false, + error: "DB Handler Error => " + error.message, + msg: "DB Handler Error => " + error.message, + }; + } +} diff --git a/src/lib/grab-db-ssl.ts b/src/lib/grab-db-ssl.ts deleted file mode 100644 index 057046d..0000000 --- a/src/lib/grab-db-ssl.ts +++ /dev/null @@ -1,29 +0,0 @@ -import fs from "fs"; -import type { ConnectionConfig } from "mariadb"; -import path from "path"; - -/** - * # Grab SSL - */ -export default function grabDbSSL(): ConnectionConfig["ssl"] { - const caProivdedPath = - global.CONFIG.ssl_ca || process.env.BUN_MARIADB_SERVER_SSL_KEY_PATH; - - if (!caProivdedPath?.match(/./)) { - return { - rejectUnauthorized: false, - }; - } - - const ca_file_path = path.resolve(process.cwd(), caProivdedPath); - - if (!fs.existsSync(ca_file_path)) { - console.log(`${ca_file_path} does not exist`); - return undefined; - } - - return { - ca: fs.readFileSync(ca_file_path), - rejectUnauthorized: false, - }; -} diff --git a/src/lib/mariadb/db-schema-manager.ts b/src/lib/mariadb/db-schema-manager.ts deleted file mode 100644 index 960b52f..0000000 --- a/src/lib/mariadb/db-schema-manager.ts +++ /dev/null @@ -1,932 +0,0 @@ -#!/usr/bin/env bun - -import _ from "lodash"; -import dbHandler from "../db-handler"; -import type { - BUN_MARIADB_DatabaseSchemaType, - BUN_MARIADB_FieldSchemaType, - BUN_MARIADB_TableSchemaType, -} from "../../types"; -import { AppData } from "../../data/app-data"; - -type QueryValues = any[]; - -type TableInfoRow = { - table_name: string; -}; - -type ColumnInfoRow = { - name: string; - type: string; - comment?: string; -}; - -type IndexInfoRow = { - name: string; -}; - -class MariaDBSchemaManager { - private db_manager_table_name: string; - private db_schema: BUN_MARIADB_DatabaseSchemaType; - - constructor({ schema }: { schema: BUN_MARIADB_DatabaseSchemaType }) { - this.db_manager_table_name = AppData["DbSchemaManagerTableName"]; - this.db_schema = schema; - } - - public async syncSchema(): Promise { - console.log("Starting schema synchronization..."); - - await this.createDbManagerTable(); - const orderedTables = this.sortTablesByDependencies( - this.db_schema.tables, - ); - - const existingTables = await this.getExistingTables(); - const schemaTables = this.db_schema.tables.map((t) => t.tableName); - - for (const table of orderedTables) { - const resolvedTargetTable = this.resolveTable(table); - await this.syncTable(resolvedTargetTable, existingTables); - } - - await this.dropRemovedTables(existingTables, schemaTables); - console.log("Schema synchronization complete!"); - } - - private quoteIdentifier(identifier: string): string { - return `\`${identifier.replace(/`/g, "``")}\``; - } - - private schemaCondition(): { where: string; values: string[] } { - const config = global.CONFIG; - const databaseName = config?.db_name; - - if (databaseName) { - return { - where: "TABLE_SCHEMA = ?", - values: [databaseName], - }; - } - - return { - where: "TABLE_SCHEMA = DATABASE()", - values: [], - }; - } - - private async run(query: string, values?: QueryValues): Promise { - const res = await dbHandler({ - query, - values: values as any, - }); - - if (!res.success) { - throw new Error( - `Database query failed: ${query} ... ERROR: ${res.error || res.msg}`, - ); - } - } - - private async query>( - query: string, - values?: QueryValues, - ): Promise { - const res = await dbHandler({ - query, - values: values as any, - }); - - if (!res.success) { - throw new Error( - `Database query failed: ${query} ... ERROR: ${res.error || res.msg}`, - ); - } - - return (res.payload || []) as T[]; - } - - private async createDbManagerTable(): Promise { - await this.run(` - CREATE TABLE IF NOT EXISTS ${this.quoteIdentifier(this.db_manager_table_name)} ( - table_name VARCHAR(255) NOT NULL PRIMARY KEY, - created_at BIGINT NOT NULL, - updated_at BIGINT NOT NULL - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci - `); - } - - private sortTablesByDependencies(tables: any[]): any[] { - const sorted: any[] = []; - const visited = new Set(); - const tempVisited = new Set(); - - const tableMap = new Map(); - for (const table of tables) { - if (table.tableName) { - tableMap.set(table.tableName, table); - } - } - - const visit = (tableName: string) => { - if (tempVisited.has(tableName)) { - console.warn( - `Circular dependency detected around table: ${tableName}. Proceeding carefully.`, - ); - return; - } - - if (!visited.has(tableName)) { - tempVisited.add(tableName); - - const tableDef = tableMap.get(tableName); - if (tableDef && tableDef.fields) { - for (const field of tableDef.fields) { - if ( - field.foreignKey && - field.foreignKey.destinationTableName - ) { - const parentTable = - field.foreignKey.destinationTableName; - if ( - tableMap.has(parentTable) && - parentTable !== tableName - ) { - visit(parentTable); - } - } - } - } - - tempVisited.delete(tableName); - visited.add(tableName); - if (tableDef) { - sorted.push(tableDef); - } - } - }; - - for (const table of tables) { - if (table.tableName && !visited.has(table.tableName)) { - visit(table.tableName); - } - } - - return sorted; - } - - private async insertDbManagerTable(tableName: string): Promise { - const now = Date.now(); - - await this.run( - `INSERT INTO ${this.quoteIdentifier(this.db_manager_table_name)} (table_name, created_at, updated_at) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = VALUES(updated_at)`, - [tableName, now, now], - ); - } - - private async removeDbManagerTable(tableName: string): Promise { - await this.run( - `DELETE FROM ${this.quoteIdentifier(this.db_manager_table_name)} WHERE table_name = ?`, - [tableName], - ); - } - - private async getExistingTables(): Promise { - const rows = await this.query( - `SELECT table_name FROM ${this.quoteIdentifier(this.db_manager_table_name)}`, - ); - - return rows.map((row) => row.table_name); - } - - private async getLiveTableNames(): Promise { - const schemaCond = this.schemaCondition(); - const rows = await this.query<{ TABLE_NAME: string }>( - `SELECT TABLE_NAME FROM information_schema.TABLES WHERE ${schemaCond.where} AND TABLE_TYPE = 'BASE TABLE'`, - schemaCond.values, - ); - - return rows.map((row) => row.TABLE_NAME); - } - - private async dropRemovedTables( - existingTables: string[], - schemaTables: string[], - ): Promise { - console.log(`Cleaning up tables ...`); - - // Safely drop only tables that are tracked by the manager but no longer in the schema - const tablesToDrop: string[] = existingTables.filter( - (tableName) => !schemaTables.includes(tableName), - ); - - const uniqueTablesToDrop = _.uniq(tablesToDrop); - - for (const tableName of uniqueTablesToDrop) { - console.log(`Dropping table: ${tableName}`); - await this.run( - `DROP TABLE IF EXISTS ${this.quoteIdentifier(tableName)}`, - ); - await this.removeDbManagerTable(tableName); - } - } - - private async syncTable( - resolvedTable: BUN_MARIADB_TableSchemaType, - existingTables: string[], - ): Promise { - const liveTables = await this.getLiveTableNames(); - let tableExistsTracked = existingTables.includes( - resolvedTable.tableName, - ); - let tableExistsLive = liveTables.includes(resolvedTable.tableName); - let wasRenamed = false; - - if ( - resolvedTable.tableNameOld && - resolvedTable.tableNameOld !== resolvedTable.tableName - ) { - if (liveTables.includes(resolvedTable.tableNameOld)) { - console.log( - `Renaming table: ${resolvedTable.tableNameOld} -> ${resolvedTable.tableName}`, - ); - await this.run( - `RENAME TABLE ${this.quoteIdentifier(resolvedTable.tableNameOld)} TO ${this.quoteIdentifier(resolvedTable.tableName)}`, - ); - await this.insertDbManagerTable(resolvedTable.tableName); - await this.removeDbManagerTable(resolvedTable.tableNameOld); - tableExistsTracked = true; - tableExistsLive = true; - wasRenamed = true; - } - } - - if (!tableExistsTracked && !tableExistsLive) { - await this.createTable(resolvedTable); - await this.insertDbManagerTable(resolvedTable.tableName); - } else { - if (!wasRenamed) { - await this.updateTable(resolvedTable); - } - await this.insertDbManagerTable(resolvedTable.tableName); - } - - await this.syncIndexes(resolvedTable); - } - - private resolveTable( - table: BUN_MARIADB_TableSchemaType, - ): BUN_MARIADB_TableSchemaType { - if (!table.parentTableName) { - return _.cloneDeep(table); - } - - const parentTable = this.db_schema.tables.find( - (schemaTable) => schemaTable.tableName === table.parentTableName, - ); - - if (!parentTable) { - throw new Error( - `Parent table \`${table.parentTableName}\` not found for \`${table.tableName}\``, - ); - } - - const mergedFieldsMap = new Map(); - - (parentTable.fields || []).forEach((f) => { - if (f.fieldName) mergedFieldsMap.set(f.fieldName, _.cloneDeep(f)); - }); - - (table.fields || []).forEach((f) => { - if (f.fieldName) { - const existing = mergedFieldsMap.get(f.fieldName) || {}; - // Deep merge to properly handle nested objects like foreignKey - mergedFieldsMap.set(f.fieldName, _.merge({}, existing, f)); - } - }); - - return { - ..._.cloneDeep(parentTable), - tableName: table.tableName, - tableDescription: - table.tableDescription || parentTable.tableDescription, - collation: table.collation || parentTable.collation, - isVector: - table.isVector !== undefined - ? table.isVector - : parentTable.isVector, - fields: Array.from(mergedFieldsMap.values()), - indexes: _.uniqBy( - [...(parentTable.indexes || []), ...(table.indexes || [])], - "indexName", - ), - uniqueConstraints: [ - ...(parentTable.uniqueConstraints || []), - ...(table.uniqueConstraints || []), - ], - }; - } - - private async createTable( - table: BUN_MARIADB_TableSchemaType, - ): Promise { - if (!table.tableName.match(/_temp_\d+$/)) { - console.log(`Creating table: ${table.tableName}`); - } - - const newTable = this.resolveTable(table); - const columnDefinitions: string[] = []; - const foreignKeys: string[] = []; - const primaryKeys: string[] = []; - - for (const field of newTable.fields || []) { - columnDefinitions.push(this.buildColumnDefinition(field)); - - if (field.primaryKey && field.fieldName) { - primaryKeys.push(field.fieldName); - } - - if (field.foreignKey && !newTable.isVector) { - foreignKeys.push(this.buildForeignKeyConstraint(field)); - } - } - - // Add composite or single primary key at the table level - if (primaryKeys.length > 0) { - const pkCols = primaryKeys - .map((k) => this.quoteIdentifier(k)) - .join(", "); - columnDefinitions.push(`PRIMARY KEY (${pkCols})`); - } - - if (newTable.uniqueConstraints) { - for (const constraint of newTable.uniqueConstraints) { - if ( - constraint.constraintTableFields && - constraint.constraintTableFields.length > 0 - ) { - const fields = constraint.constraintTableFields - .map((field) => this.quoteIdentifier(field.value)) - .join(", "); - const constraintName = - constraint.constraintName || - `unique_${fields.replace(/`/g, "")}`; - - columnDefinitions.push( - `CONSTRAINT ${this.quoteIdentifier(constraintName)} UNIQUE (${fields})`, - ); - } - } - } - - const sql = `CREATE TABLE IF NOT EXISTS ${this.quoteIdentifier(newTable.tableName)} (${[...columnDefinitions, ...foreignKeys].join(", ")})${this.buildTableOptions(newTable)}`; - - await this.run(sql); - } - - private buildTableOptions(table: BUN_MARIADB_TableSchemaType): string { - const options = ["ENGINE=InnoDB"]; - - if (table.collation) { - options.push( - "DEFAULT CHARSET=utf8mb4", - `COLLATE ${table.collation}`, - ); - } - - return ` ${options.join(" ")}`; - } - - private async updateTable( - resolvedTable: BUN_MARIADB_TableSchemaType, - ): Promise { - const existingColumns = await this.getTableColumns( - resolvedTable.tableName, - ); - - if (existingColumns.length === 0) { - await this.createTable(resolvedTable); - return; - } - - const liveFieldsMap = new Map< - string, - { type: string; comment: string } - >( - existingColumns.map((col) => [ - col.name, - { type: col.type.toLowerCase(), comment: col.comment || "" }, - ]), - ); - const codeFieldsMap = new Map( - (resolvedTable.fields || []).map((f) => [f.fieldName, f]), - ); - - const fieldsToAdd: BUN_MARIADB_FieldSchemaType[] = []; - const fieldsToModify: BUN_MARIADB_FieldSchemaType[] = []; - const fieldsToDrop: string[] = []; - - for (const field of resolvedTable.fields || []) { - if (!field.fieldName) continue; - - const liveField = liveFieldsMap.get(field.fieldName); - - if (!liveField) { - fieldsToAdd.push(field); - } else { - const resolvedType = this.mapDataType(field).toLowerCase(); - - let typeDiverged = !resolvedType.startsWith(liveField.type); - - if (field.isVector || field.dataType === "VECTOR") { - const dimensions = field.vectorSize || 1536; - const expectedNativeToken = `vector(${dimensions})`; - typeDiverged = liveField.type !== expectedNativeToken; - } - - if (typeDiverged) { - fieldsToModify.push(field); - } - } - } - - for (const col of existingColumns) { - if (!codeFieldsMap.has(col.name)) { - fieldsToDrop.push(col.name); - } - } - - if ( - fieldsToAdd.length === 0 && - fieldsToModify.length === 0 && - fieldsToDrop.length === 0 - ) { - return; - } - - console.log( - `Surgically updating table structure from database layout: ${resolvedTable.tableName}`, - ); - - // Handle dependencies before dropping columns - if (fieldsToDrop.length > 0) { - const schemaCond = this.schemaCondition(); - - // Check if any dropped column is part of the primary key - const pkRows = await this.query<{ COLUMN_NAME: string }>( - `SELECT COLUMN_NAME FROM information_schema.KEY_COLUMN_USAGE WHERE ${schemaCond.where} AND TABLE_NAME = ? AND CONSTRAINT_NAME = 'PRIMARY'`, - [...schemaCond.values, resolvedTable.tableName], - ); - const pkColumnNames = pkRows.map((r) => r.COLUMN_NAME); - - if (fieldsToDrop.some((f) => pkColumnNames.includes(f))) { - console.log( - `Dropping primary key because a PK column is being dropped`, - ); - await this.run( - `ALTER TABLE ${this.quoteIdentifier(resolvedTable.tableName)} DROP PRIMARY KEY`, - ); - } - - // Drop secondary indexes that include dropped columns - const indexRows = await this.query<{ - INDEX_NAME: string; - COLUMN_NAME: string; - }>( - `SELECT INDEX_NAME, COLUMN_NAME FROM information_schema.STATISTICS WHERE ${schemaCond.where} AND TABLE_NAME = ? AND INDEX_NAME <> 'PRIMARY'`, - [...schemaCond.values, resolvedTable.tableName], - ); - - const indexesToDrop = new Set(); - for (const row of indexRows) { - if (fieldsToDrop.includes(row.COLUMN_NAME)) { - indexesToDrop.add(row.INDEX_NAME); - } - } - - for (const indexName of indexesToDrop) { - console.log( - `Dropping index ${indexName} because it contains a dropped column`, - ); - await this.run( - `DROP INDEX ${this.quoteIdentifier(indexName)} ON ${this.quoteIdentifier(resolvedTable.tableName)}`, - ); - } - } - - for (const field of fieldsToAdd) { - await this.addColumn(resolvedTable.tableName, field); - } - - for (const field of fieldsToModify) { - try { - await this.modifyColumn(resolvedTable.tableName, field); - } catch (err: any) { - if (field.isVector || field.dataType === "VECTOR") { - console.warn( - `[Vector Resize] Re-aligning dimension spaces natively via safe migration schema rebuild.`, - ); - await this.recreateTable(resolvedTable); - return; - } - throw err; - } - } - - for (const fieldName of fieldsToDrop) { - await this.dropColumn(resolvedTable.tableName, fieldName); - } - } - - private async dropColumn( - tableName: string, - fieldName: string, - ): Promise { - console.log(`Dropping column: ${tableName}.${fieldName}`); - await this.run( - `ALTER TABLE ${this.quoteIdentifier(tableName)} DROP COLUMN ${this.quoteIdentifier(fieldName)}`, - ); - } - - private async modifyColumn( - tableName: string, - field: BUN_MARIADB_FieldSchemaType, - ): Promise { - console.log(`Modifying column: ${tableName}.${field.fieldName}`); - const columnDef = this.buildColumnDefinition(field).trim(); - await this.run( - `ALTER TABLE ${this.quoteIdentifier(tableName)} MODIFY COLUMN ${columnDef}`, - ); - } - - private async getTableColumns(tableName: string): Promise { - const schemaCond = this.schemaCondition(); - const rows = await this.query<{ - COLUMN_NAME: string; - COLUMN_TYPE: string; - COLUMN_COMMENT: string; - }>( - `SELECT COLUMN_NAME, COLUMN_TYPE, COLUMN_COMMENT FROM information_schema.COLUMNS WHERE ${schemaCond.where} AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION`, - [...schemaCond.values, tableName], - ); - - return rows.map((row) => ({ - name: row.COLUMN_NAME, - type: row.COLUMN_TYPE, - comment: row.COLUMN_COMMENT, - })); - } - - private async addColumn( - tableName: string, - field: BUN_MARIADB_FieldSchemaType, - ): Promise { - console.log(`Adding column: ${tableName}.${field.fieldName}`); - const columnDef = this.buildColumnDefinition(field).trim(); - await this.run( - `ALTER TABLE ${this.quoteIdentifier(tableName)} ADD COLUMN IF NOT EXISTS ${columnDef}`, - ); - } - - private async checkIfTableExists(table: string): Promise { - const schemaCond = this.schemaCondition(); - const row = await this.query<{ table_exists: number }>( - `SELECT 1 AS \`table_exists\` FROM information_schema.TABLES WHERE ${schemaCond.where} AND TABLE_NAME = ? LIMIT 1`, - [...schemaCond.values, table], - ); - - return Boolean(row[0]?.table_exists); - } - - private async recreateTable( - table: BUN_MARIADB_TableSchemaType, - ): Promise { - const doesTableExist = await this.checkIfTableExists(table.tableName); - if (!doesTableExist) { - await this.createTable(table); - return; - } - - const tempTableName = `${table.tableName}_temp_${Date.now()}`; - const backupOldTableName = `${table.tableName}_old_${Date.now()}`; - const existingColumns = await this.getTableColumns(table.tableName); - - const columnsToKeep = (table.fields || []) - .filter((field) => - existingColumns.some( - (column) => column.name === field.fieldName, - ), - ) - .map((field) => field.fieldName) - .filter((fieldName): fieldName is string => Boolean(fieldName)); - - await this.createTable({ ...table, tableName: tempTableName }); - - if (columnsToKeep.length > 0) { - const columnList = columnsToKeep - .map((column) => this.quoteIdentifier(column)) - .join(", "); - - await this.run( - `INSERT INTO ${this.quoteIdentifier(tempTableName)} (${columnList}) SELECT ${columnList} FROM ${this.quoteIdentifier(table.tableName)}`, - ); - } - - // Disable foreign key checks to allow dropping/renaming tables with external references - await this.run(`SET FOREIGN_KEY_CHECKS = 0`); - try { - await this.run( - `RENAME TABLE ${this.quoteIdentifier(table.tableName)} TO ${this.quoteIdentifier(backupOldTableName)}`, - ); - await this.run( - `RENAME TABLE ${this.quoteIdentifier(tempTableName)} TO ${this.quoteIdentifier(table.tableName)}`, - ); - await this.run( - `DROP TABLE ${this.quoteIdentifier(backupOldTableName)}`, - ); - } finally { - await this.run(`SET FOREIGN_KEY_CHECKS = 1`); - } - } - - private buildColumnDefinition(field: BUN_MARIADB_FieldSchemaType): string { - if (!field.fieldName) { - throw new Error("Field name is required"); - } - - const parts: string[] = [this.quoteIdentifier(field.fieldName)]; - parts.push(this.mapDataType(field)); - - if (field.autoIncrement) { - parts.push("AUTO_INCREMENT"); - } - - if (field.notNullValue || field.primaryKey || field.isVector) { - if (!field.primaryKey) { - parts.push("NOT NULL"); - } - } - - if (field.unique && !field.primaryKey) { - parts.push("UNIQUE"); - } - - if (field.defaultValue !== undefined) { - if (typeof field.defaultValue === "string") { - parts.push( - `DEFAULT '${field.defaultValue.replace(/'/g, "''")}'`, - ); - } else { - parts.push(`DEFAULT ${field.defaultValue}`); - } - } else if (field.defaultValueLiteral) { - parts.push(`DEFAULT ${field.defaultValueLiteral}`); - } - - if (field.onUpdate) { - parts.push(`ON UPDATE ${field.onUpdate}`); - } else if (field.onUpdateLiteral) { - parts.push(`ON UPDATE ${field.onUpdateLiteral}`); - } - - return parts.join(" "); - } - - private mapDataType(field: BUN_MARIADB_FieldSchemaType): string { - const dataType = field.dataType?.toUpperCase() || "TEXT"; - const vectorSize = field.vectorSize || 1536; - - if (field.isVector) { - return `LONGTEXT COMMENT 'vector_size=${vectorSize}'`; - } - - switch (dataType) { - case "CHAR": - return `CHAR(${field.integerLength || 255})`; - case "VARCHAR": - return `VARCHAR(${field.integerLength || 255})`; - case "TEXT": - return "TEXT"; - case "TINYTEXT": - return "TINYTEXT"; - case "MEDIUMTEXT": - return "MEDIUMTEXT"; - case "LONGTEXT": - return "LONGTEXT"; - case "TINYINT": - return field.integerLength - ? `TINYINT(${field.integerLength})` - : "TINYINT"; - case "SMALLINT": - return field.integerLength - ? `SMALLINT(${field.integerLength})` - : "SMALLINT"; - case "MEDIUMINT": - return field.integerLength - ? `MEDIUMINT(${field.integerLength})` - : "MEDIUMINT"; - case "INT": - return field.integerLength - ? `INT(${field.integerLength})` - : "INT"; - case "BIGINT": - return field.integerLength - ? `BIGINT(${field.integerLength})` - : "BIGINT"; - case "FLOAT": - return "FLOAT"; - case "DOUBLE": - return "DOUBLE"; - case "DECIMAL": - if (field.integerLength && field.decimals) { - return `DECIMAL(${field.integerLength}, ${field.decimals})`; - } - return "DECIMAL(10,2)"; - case "BINARY": - return `BINARY(${field.integerLength || 1})`; - case "VARBINARY": - return `VARBINARY(${field.integerLength || 255})`; - case "BLOB": - return "BLOB"; - case "TINYBLOB": - return "TINYBLOB"; - case "MEDIUMBLOB": - return "MEDIUMBLOB"; - case "LONGBLOB": - return "LONGBLOB"; - case "DATE": - return "DATE"; - case "TIME": - return "TIME"; - case "DATETIME": - return "DATETIME"; - case "TIMESTAMP": - return "TIMESTAMP"; - case "YEAR": - return "YEAR"; - case "UUID": - return "CHAR(36)"; // MariaDB does not have a native UUID type - case "JSON": - return "JSON"; - case "INET6": - return "INET6"; - case "VECTOR": { - const dimensions = field.vectorSize || 1536; - return `VECTOR(${dimensions})`; - } - case "BOOLEAN": - return "TINYINT(1)"; - case "ENUM": { - const enumVals = (field.options || []) - .map((v) => `'${String(v).replace(/'/g, "''")}'`) - .join(", "); - return `ENUM(${enumVals || "''"})`; - } - case "SET": { - const setVals = (field.options || []) - .map((v) => `'${String(v).replace(/'/g, "''")}'`) - .join(", "); - return `SET(${setVals || "''"})`; - } - default: - return "TEXT"; - } - } - - private buildForeignKeyConstraint( - field: BUN_MARIADB_FieldSchemaType, - ): string { - const fk = field.foreignKey!; - const constraintName = fk.foreignKeyName - ? `CONSTRAINT ${this.quoteIdentifier(fk.foreignKeyName)} ` - : ""; - - let constraint = `${constraintName}FOREIGN KEY (${this.quoteIdentifier(field.fieldName!)}) REFERENCES ${this.quoteIdentifier(fk.destinationTableName!)}(${this.quoteIdentifier(fk.destinationTableColumnName!)})`; - - if (fk.cascadeDelete) { - constraint += " ON DELETE CASCADE"; - } - - if (fk.cascadeUpdate) { - constraint += " ON UPDATE CASCADE"; - } - - return constraint; - } - - private async syncIndexes( - table: BUN_MARIADB_TableSchemaType, - ): Promise { - const schemaCond = this.schemaCondition(); - const rows = await this.query<{ - INDEX_NAME: string; - COLUMN_NAME: string; - INDEX_TYPE: string; - }>( - `SELECT INDEX_NAME, COLUMN_NAME, INDEX_TYPE FROM information_schema.STATISTICS WHERE ${schemaCond.where} AND TABLE_NAME = ? AND INDEX_NAME <> 'PRIMARY' ORDER BY INDEX_NAME, SEQ_IN_INDEX`, - [...schemaCond.values, table.tableName], - ); - - const existingIndexesMap = new Map< - string, - { columns: string[]; type: string } - >(); - for (const row of rows) { - if (!existingIndexesMap.has(row.INDEX_NAME)) { - existingIndexesMap.set(row.INDEX_NAME, { - columns: [], - type: row.INDEX_TYPE, - }); - } - existingIndexesMap - .get(row.INDEX_NAME)! - .columns.push(row.COLUMN_NAME); - } - - // Drop indexes that are no longer in schema OR have changed columns - for (const [indexName, details] of existingIndexesMap.entries()) { - const schemaIndex = table.indexes?.find( - (i) => i.indexName === indexName, - ); - - if (!schemaIndex) { - console.log(`Dropping index: ${indexName}`); - await this.run( - `DROP INDEX ${this.quoteIdentifier(indexName)} ON ${this.quoteIdentifier(table.tableName)}`, - ); - } else { - const schemaColumns = schemaIndex.indexTableFields || []; - const columnsMatch = - details.columns.length === schemaColumns.length && - details.columns.every( - (col, idx) => col === schemaColumns[idx], - ); - - if (!columnsMatch) { - console.log(`Recreating changed index: ${indexName}`); - await this.run( - `DROP INDEX ${this.quoteIdentifier(indexName)} ON ${this.quoteIdentifier(table.tableName)}`, - ); - existingIndexesMap.delete(indexName); - } - } - } - - // Create missing indexes - for (const index of table.indexes || []) { - if ( - !index.indexName || - !index.indexTableFields || - index.indexTableFields.length === 0 - ) { - continue; - } - - if (!existingIndexesMap.has(index.indexName)) { - const isVectorIndex = - table.isVector || - table.fields?.some( - (f) => - f.fieldName === index.indexTableFields![0] && - f.isVector, - ); - - if (isVectorIndex) { - console.log(`Creating Vector index: ${index.indexName}`); - const targetField = this.quoteIdentifier( - index.indexTableFields[0]!, - ); - - const distanceMetric = - index.indexType?.toLowerCase() === "cosine" - ? "cosine" - : "euclidean"; - - const sql = `ALTER TABLE ${this.quoteIdentifier(table.tableName)} ADD VECTOR INDEX ${this.quoteIdentifier(index.indexName)} (${targetField}) M=8 DISTANCE=${distanceMetric}`; - await this.run(sql); - } else { - console.log(`Creating standard index: ${index.indexName}`); - const fields = index.indexTableFields - .map((field) => this.quoteIdentifier(field)) - .join(", "); - const typeUpper = index.indexType?.toUpperCase(); - const isSpecialType = - typeUpper === "FULLTEXT" || typeUpper === "SPATIAL"; - const indexPrefix = isSpecialType ? `${typeUpper} ` : ""; - const indexSuffix = - !isSpecialType && - (typeUpper === "BTREE" || typeUpper === "HASH") - ? ` USING ${typeUpper}` - : ""; - - const sql = `CREATE ${indexPrefix}INDEX ${this.quoteIdentifier(index.indexName)} ON ${this.quoteIdentifier(table.tableName)} (${fields})${indexSuffix}`; - await this.run(sql); - } - } - } - } - - close(): void {} -} - -export { MariaDBSchemaManager }; diff --git a/src/lib/mariadb/index.ts b/src/lib/mariadb/index.ts deleted file mode 100644 index 1dc5bce..0000000 --- a/src/lib/mariadb/index.ts +++ /dev/null @@ -1,49 +0,0 @@ -// import { SQL } from "bun"; -// import grabDirNames from "../../data/grab-dir-names"; -// import path from "path"; - -// if (!global.CONFIG) { -// console.error(`Couldn't grab global Config.`); -// process.exit(1); -// } - -// if (!global.DB_SCHEMA) { -// console.error(`Couldn't grab Database Schema.`); -// process.exit(1); -// } - -// const config = global.CONFIG; - -// const { ROOT_DIR } = grabDirNames(); - -// const MariaDBClient = new SQL({ -// hostname: process.env.BUN_MARIADB_SERVER_HOST, -// username: process.env.BUN_MARIADB_SERVER_USERNAME, -// password: process.env.BUN_MARIADB_SERVER_PASSWORD, -// database: config.db_name, -// port: process.env.BUN_MARIADB_SERVER_PORT -// ? Number(process.env.BUN_MARIADB_SERVER_PORT) -// : undefined, -// ...config.db_config, -// tls: config.ssl_ca -// ? { -// ca: Bun.file(path.resolve(ROOT_DIR, config.ssl_ca)), -// rejectUnauthorized: false, -// } -// : { -// rejectUnauthorized: false, -// }, -// adapter: "mariadb", -// }); - -// global.MARIADB_CLIENT = MariaDBClient; - -// const test = await MariaDBClient.unsafe(`SHOW DATABASES`); - -// if (!test.count) { -// console.error(`MariaDBClient Error: Database not ready.`); -// console.log(test); -// process.exit(1); -// } - -// export default MariaDBClient; diff --git a/src/lib/schema/build-column-definition.ts b/src/lib/schema/build-column-definition.ts index 5308142..3cb75b9 100644 --- a/src/lib/schema/build-column-definition.ts +++ b/src/lib/schema/build-column-definition.ts @@ -1,4 +1,5 @@ import type { BUN_MARIADB_FieldSchemaType } from "../../types"; +import isVectorField from "./is-vector-field"; import mapDataType from "./map-data-types"; import MariaDBQuoteGen from "./mariadb-quote-gen"; @@ -16,13 +17,19 @@ export default function buildColumnDefinition( parts.push("AUTO_INCREMENT"); } - if (field.notNullValue || field.primaryKey || field.isVector) { + // Vector columns used in VECTOR INDEX must be NOT NULL + if ( + field.notNullValue || + field.primaryKey || + isVectorField(field) + ) { if (!field.primaryKey) { parts.push("NOT NULL"); } } - if (field.unique && !field.primaryKey) { + // VECTOR columns cannot be UNIQUE in the usual sense + if (field.unique && !field.primaryKey && !isVectorField(field)) { parts.push("UNIQUE"); } diff --git a/src/lib/schema/handle-db-schema-table.ts b/src/lib/schema/handle-db-schema-table.ts index f9a5011..b80298c 100644 --- a/src/lib/schema/handle-db-schema-table.ts +++ b/src/lib/schema/handle-db-schema-table.ts @@ -19,26 +19,23 @@ export default async function handleDBSchemaTable({ }: CreateDBSchemaTableHandlerParams) { const resolvedTable = resolveTable(table, db_schema); - const schemaCond = schemaCondition(config); - const liveTables = await querySchemaRows<{ TABLE_NAME: string }>({ - query: `SELECT TABLE_NAME FROM information_schema.TABLES WHERE ${schemaCond.where} AND TABLE_TYPE = 'BASE TABLE'`, - values: schemaCond.values, - config, - }); - const liveTableNames = liveTables.map((t) => t.TABLE_NAME); - let tableExistsTracked = Boolean(db_manager_table_name); - let tableExistsLive = Boolean( - existing_live_table?.TABLE_NAME || - liveTableNames.includes(resolvedTable.tableName), - ); + let tableExistsLive = Boolean(existing_live_table?.TABLE_NAME); let wasRenamed = false; if ( resolvedTable.tableNameOld && resolvedTable.tableNameOld !== resolvedTable.tableName ) { - if (liveTableNames.includes(resolvedTable.tableNameOld)) { + // Only hit information_schema when a rename is declared + const schemaCond = schemaCondition(config); + const liveTables = await querySchemaRows<{ TABLE_NAME: string }>({ + query: `SELECT TABLE_NAME FROM information_schema.TABLES WHERE ${schemaCond.where} AND TABLE_TYPE = 'BASE TABLE' AND TABLE_NAME = ?`, + values: [...schemaCond.values, resolvedTable.tableNameOld], + config, + }); + + if (liveTables.length > 0) { console.log( `Renaming table: ${resolvedTable.tableNameOld} -> ${resolvedTable.tableName}`, ); @@ -68,7 +65,10 @@ export default async function handleDBSchemaTable({ }); } else { if (!wasRenamed) { - await updateTable({ table: resolvedTable, config }); + await updateTable({ + table: resolvedTable, + config, + }); } await upsertDbManagerTable({ tableName: resolvedTable.tableName, diff --git a/src/lib/schema/is-vector-field.ts b/src/lib/schema/is-vector-field.ts new file mode 100644 index 0000000..dd55485 --- /dev/null +++ b/src/lib/schema/is-vector-field.ts @@ -0,0 +1,11 @@ +import type { BUN_MARIADB_FieldSchemaType } from "../../types"; + +/** + * True when a field is a MariaDB native vector column. + */ +export default function isVectorField( + field?: BUN_MARIADB_FieldSchemaType, +): boolean { + if (!field) return false; + return field.isVector === true || field.dataType === "VECTOR"; +} diff --git a/src/lib/schema/map-data-types.ts b/src/lib/schema/map-data-types.ts index 11f32d4..6f2265c 100644 --- a/src/lib/schema/map-data-types.ts +++ b/src/lib/schema/map-data-types.ts @@ -6,8 +6,9 @@ export default function mapDataType( const dataType = field.dataType?.toUpperCase() || "TEXT"; const vectorSize = field.vectorSize || 1536; - if (field.isVector) { - return `LONGTEXT COMMENT 'vector_size=${vectorSize}'`; + // Native MariaDB VECTOR type (11.7+). Prefer this over LONGTEXT storage. + if (field.isVector || dataType === "VECTOR") { + return `VECTOR(${vectorSize})`; } switch (dataType) { @@ -78,10 +79,6 @@ export default function mapDataType( return "JSON"; case "INET6": return "INET6"; - case "VECTOR": { - const dimensions = field.vectorSize || 1536; - return `VECTOR(${dimensions})`; - } case "BOOLEAN": return "TINYINT(1)"; case "ENUM": { diff --git a/src/lib/schema/recreate-table.ts b/src/lib/schema/recreate-table.ts index abc9a66..0b120d6 100644 --- a/src/lib/schema/recreate-table.ts +++ b/src/lib/schema/recreate-table.ts @@ -25,6 +25,10 @@ async function checkIfTableExists({ return Boolean(rows[0]?.table_exists); } +/** + * Full table rebuild. For `isVector` tables this drops and recreates in place + * (preserving rows when possible). For regular tables it uses a temp-table swap. + */ export default async function recreateTable({ table, config, @@ -42,6 +46,58 @@ export default async function recreateTable({ return; } + /** + * Vector tables: drop + recreate + reinsert (MariaDB VECTOR INDEX / dim + * changes are not reliably alterable in place). + */ + if (table.isVector) { + console.log(`Recreating vector table: ${table.tableName}`); + + const existingRows = await querySchemaRows>({ + query: `SELECT * FROM ${MariaDBQuoteGen(table.tableName)}`, + config, + }); + + await runSchemaQuery({ query: `SET FOREIGN_KEY_CHECKS = 0`, config }); + try { + await runSchemaQuery({ + query: `DROP TABLE IF EXISTS ${MariaDBQuoteGen(table.tableName)}`, + config, + }); + await createTable({ table, config }); + } finally { + await runSchemaQuery({ query: `SET FOREIGN_KEY_CHECKS = 1`, config }); + } + + if (existingRows.length > 0) { + const schemaFieldNames = new Set( + (table.fields || []) + .map((f) => f.fieldName) + .filter((n): n is string => Boolean(n)), + ); + + for (const row of existingRows) { + const columns = Object.keys(row).filter((c) => + schemaFieldNames.has(c), + ); + if (columns.length === 0) continue; + + const placeholders = columns.map(() => "?").join(", "); + const columnList = columns + .map((c) => MariaDBQuoteGen(c)) + .join(", "); + + await runSchemaQuery({ + query: `INSERT INTO ${MariaDBQuoteGen(table.tableName)} (${columnList}) VALUES (${placeholders})`, + values: columns.map((c) => row[c] ?? null), + config, + }); + } + } + + return; + } + const tempTableName = `${table.tableName}_temp_${Date.now()}`; const backupOldTableName = `${table.tableName}_old_${Date.now()}`; const existingColumns = await getTableColumns({ diff --git a/src/lib/schema/sync-indexes.ts b/src/lib/schema/sync-indexes.ts index 2adae15..be1b350 100644 --- a/src/lib/schema/sync-indexes.ts +++ b/src/lib/schema/sync-indexes.ts @@ -1,11 +1,33 @@ import type { + BUN_MARIADB_IndexSchemaType, BUN_MARIADB_TableSchemaType, BunMariaDBConfig, } from "../../types"; +import isVectorField from "./is-vector-field"; import MariaDBQuoteGen from "./mariadb-quote-gen"; import runSchemaQuery, { querySchemaRows } from "./run-schema-query"; import schemaCondition from "./schema-condition"; +function isVectorIndexDef( + index: BUN_MARIADB_IndexSchemaType, + table: BUN_MARIADB_TableSchemaType, +): boolean { + if (index.indexType === "VECTOR") return true; + if (table.isVector) return true; + + const firstFieldName = index.indexTableFields?.[0]; + if (!firstFieldName) return false; + + const field = table.fields?.find((f) => f.fieldName === firstFieldName); + return isVectorField(field); +} + +function vectorDistanceMetric(index: BUN_MARIADB_IndexSchemaType): string { + if (index.vectorDistanceMetric === "cosine") return "cosine"; + if (index.vectorDistanceMetric === "euclidean") return "euclidean"; + return "euclidean"; +} + export default async function syncIndexes({ table, config, @@ -121,20 +143,10 @@ export default async function syncIndexes({ } if (!existingIndexesMap.has(index.indexName)) { - const isVectorIndex = - table.isVector || - table.fields?.some( - (f) => - f.fieldName === index.indexTableFields![0] && f.isVector, - ); - - if (isVectorIndex) { + if (isVectorIndexDef(index, table)) { console.log(`Creating Vector index: ${index.indexName}`); const targetField = MariaDBQuoteGen(index.indexTableFields[0]!); - const distanceMetric = - (index.indexType as string)?.toLowerCase() === "cosine" - ? "cosine" - : "euclidean"; + const distanceMetric = vectorDistanceMetric(index); await runSchemaQuery({ query: `ALTER TABLE ${MariaDBQuoteGen(table.tableName)} ADD VECTOR INDEX ${MariaDBQuoteGen(index.indexName)} (${targetField}) M=8 DISTANCE=${distanceMetric}`, diff --git a/src/lib/schema/update-table.ts b/src/lib/schema/update-table.ts index c17c389..34af85a 100644 --- a/src/lib/schema/update-table.ts +++ b/src/lib/schema/update-table.ts @@ -6,6 +6,7 @@ import type { import buildColumnDefinition from "./build-column-definition"; import createTable from "./create-table"; import getTableColumns from "./get-table-columns"; +import isVectorField from "./is-vector-field"; import mapDataType from "./map-data-types"; import MariaDBQuoteGen from "./mariadb-quote-gen"; import recreateTable from "./recreate-table"; @@ -29,6 +30,30 @@ function columnTypesMatch(liveType: string, expectedType: string): boolean { return false; } +function vectorTypeDiverged( + liveType: string, + liveComment: string, + field: BUN_MARIADB_FieldSchemaType, +): boolean { + const dimensions = field.vectorSize || 1536; + const expectedNative = `vector(${dimensions})`; + const live = liveType.toLowerCase().replace(/\s+/g, ""); + + if (live === expectedNative) return false; + + // Legacy LONGTEXT storage with vector_size comment + if (live.startsWith("longtext") || live.startsWith("text")) { + const match = liveComment.match(/vector_size\s*=\s*(\d+)/i); + if (match && Number(match[1]) === dimensions) { + // Still legacy storage — treat as diverged so we can migrate to VECTOR + return true; + } + return true; + } + + return true; +} + async function addColumn({ tableName, field, @@ -109,6 +134,7 @@ export default async function updateTable({ const fieldsToAdd: BUN_MARIADB_FieldSchemaType[] = []; const fieldsToModify: BUN_MARIADB_FieldSchemaType[] = []; const fieldsToDrop: string[] = []; + let needsVectorRecreate = false; for (const field of table.fields || []) { if (!field.fieldName) continue; @@ -116,6 +142,8 @@ export default async function updateTable({ const liveField = liveFieldsMap.get(field.fieldName); if (!liveField) { + // Adding a new vector column can require rebuild if VECTOR INDEX + // constraints conflict; still try surgical add first. fieldsToAdd.push(field); } else { let typeDiverged = !columnTypesMatch( @@ -123,10 +151,15 @@ export default async function updateTable({ mapDataType(field), ); - if (field.isVector || field.dataType === "VECTOR") { - const dimensions = field.vectorSize || 1536; - const expectedNativeToken = `vector(${dimensions})`; - typeDiverged = liveField.type.toLowerCase() !== expectedNativeToken; + if (isVectorField(field)) { + typeDiverged = vectorTypeDiverged( + liveField.type, + liveField.comment, + field, + ); + if (typeDiverged) { + needsVectorRecreate = true; + } } if (typeDiverged) { @@ -135,6 +168,15 @@ export default async function updateTable({ } } + // Vector dimension / storage type changes → full rebuild automatically + if (needsVectorRecreate) { + console.log( + `Vector column change detected on \`${table.tableName}\`; recreating table`, + ); + await recreateTable({ table, config }); + return; + } + for (const col of existingColumns) { if (!codeFieldsMap.has(col.name)) { fieldsToDrop.push(col.name); @@ -208,7 +250,7 @@ export default async function updateTable({ try { await modifyColumn({ tableName: table.tableName, field, config }); } catch (err: any) { - if (field.isVector || field.dataType === "VECTOR") { + if (isVectorField(field)) { console.warn( `[Vector Resize] Re-aligning dimension spaces natively via safe migration schema rebuild.`, ); diff --git a/src/types/index.ts b/src/types/index.ts index 48403ea..6f84ba3 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,5 +1,4 @@ import type { RequestOptions } from "https"; -import type { ConnectionConfig } from "mariadb"; /** * Fully-qualified database name used when a database needs to be referenced @@ -98,13 +97,9 @@ export interface BUN_MARIADB_TableSchemaType { childTableDbId?: string | number; collation?: (typeof MariaDBCollations)[number]; /** - * If this is a vector table + * If this is a vector-oriented table (native MariaDB VECTOR columns/indexes) */ isVector?: boolean; - /** - * Type of vector. Defaults to `vec0` - */ - vectorType?: string; } /** @@ -234,6 +229,7 @@ export const MariaDBIndexTypes = [ "HASH", "FULLTEXT", "SPATIAL", + "VECTOR", ] as const; export type MariaDBIndexType = (typeof MariaDBIndexTypes)[number]; @@ -252,9 +248,13 @@ export interface BUN_MARIADB_IndexSchemaType { */ indexTableFields?: string[]; /** - * Under the hood index type (BTREE, HASH) or modifier (FULLTEXT, SPATIAL) + * Under the hood index type (BTREE, HASH) or modifier (FULLTEXT, SPATIAL, VECTOR) */ indexType?: MariaDBIndexType; + /** + * Distance metric for VECTOR indexes (`euclidean` | `cosine`). Defaults to euclidean. + */ + vectorDistanceMetric?: "euclidean" | "cosine"; /** * Optional documentation or tuning note inside the DB metadata @@ -1519,7 +1519,7 @@ export type BunMariaDBConfig = { db_backup_dir?: string; max_backups?: number; /** - * The Root Directory for the DB file and schema + * Root directory for schema, types, and local artifacts (relative to project root) */ db_dir?: string; /** diff --git a/src/utils/grab-backup-data.ts b/src/utils/grab-backup-data.ts index a44109e..ba00f00 100644 --- a/src/utils/grab-backup-data.ts +++ b/src/utils/grab-backup-data.ts @@ -2,8 +2,12 @@ type Params = { backup_name: string; }; +/** + * Parse timestamped backup names: `{db_name}-{timestamp}[.sql]` + */ export default function grabBackupData({ backup_name }: Params) { - const backup_parts = backup_name.split("-"); + const normalized = backup_name.replace(/\.sql$/, ""); + const backup_parts = normalized.split("-"); const backup_date_timestamp = Number(backup_parts.pop()); const origin_backup_name = backup_parts.join("-"); diff --git a/src/utils/grab-db-backup-file-name.ts b/src/utils/grab-db-backup-file-name.ts index f632173..80f6d04 100644 --- a/src/utils/grab-db-backup-file-name.ts +++ b/src/utils/grab-db-backup-file-name.ts @@ -5,7 +5,5 @@ type Params = { }; export default function grabDBBackupFileName({ config }: Params) { - const new_db_file_name = `${config.db_name}-${Date.now()}`; - - return new_db_file_name; + return `${config.db_name}-${Date.now()}.sql`; } diff --git a/src/utils/grab-db-dir.ts b/src/utils/grab-db-dir.ts index 8e9dcd7..2a6cae7 100644 --- a/src/utils/grab-db-dir.ts +++ b/src/utils/grab-db-dir.ts @@ -10,17 +10,14 @@ type Params = { export default function grabDBDir({ config }: Params) { const { ROOT_DIR } = grabDirNames(); - let db_dir = ROOT_DIR; - - if (config.db_dir) { - db_dir = config.db_dir; - } + const db_dir = config.db_dir + ? path.resolve(ROOT_DIR, config.db_dir) + : ROOT_DIR; const backup_dir_name = config.db_backup_dir || AppData["DefaultBackupDirName"]; const backup_dir = path.resolve(db_dir, backup_dir_name); - const db_file_path = path.resolve(db_dir, config.db_name); - return { db_dir, backup_dir, db_file_path }; + return { db_dir, backup_dir }; } diff --git a/src/utils/grab-sorted-backups.ts b/src/utils/grab-sorted-backups.ts index 96c4c19..6db916f 100644 --- a/src/utils/grab-sorted-backups.ts +++ b/src/utils/grab-sorted-backups.ts @@ -6,24 +6,23 @@ type Params = { config: BunMariaDBConfig; }; +function backupTimestamp(name: string): number { + const base = name.replace(/\.sql$/, ""); + const ts = Number(base.split("-").pop()); + return Number.isFinite(ts) ? ts : 0; +} + export default function grabSortedBackups({ config }: Params) { const { backup_dir } = grabDBDir({ config }); + if (!fs.existsSync(backup_dir)) { + return [] as string[]; + } + const backups = fs.readdirSync(backup_dir); /** * Order Backups. Most recent first. */ - const ordered_backups = backups.sort((a, b) => { - const a_date = Number(a.split("-").pop()); - const b_date = Number(b.split("-").pop()); - - if (a_date > b_date) { - return -1; - } - - return 1; - }); - - return ordered_backups; + return backups.sort((a, b) => backupTimestamp(b) - backupTimestamp(a)); } diff --git a/src/utils/mariadb-cli-env.ts b/src/utils/mariadb-cli-env.ts new file mode 100644 index 0000000..ab74bc3 --- /dev/null +++ b/src/utils/mariadb-cli-env.ts @@ -0,0 +1,28 @@ +/** + * Build env for mariadb / mariadb-dump child processes without putting + * the password on the process argv (visible via `ps`). + */ +export default function mariadbCliEnv(): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = { ...process.env }; + + const password = process.env.BUN_MARIADB_SERVER_PASSWORD; + if (password) { + // Standard MySQL/MariaDB client env vars (prefer not using -p on argv) + env.MYSQL_PWD = password; + env.MARIADB_PWD = password; + } + + return env; +} + +export function mariadbCliConnectionArgs(): string[] { + const host = process.env.BUN_MARIADB_SERVER_HOST || "127.0.0.1"; + const user = process.env.BUN_MARIADB_SERVER_USERNAME || "root"; + const port = process.env.BUN_MARIADB_SERVER_PORT; + + return [ + `-h${host}`, + `-u${user}`, + ...(port ? [`-P${port}`] : []), + ]; +} diff --git a/src/utils/sql-gen-operator-gen.ts b/src/utils/sql-gen-operator-gen.ts index 3ffd445..94f0479 100644 --- a/src/utils/sql-gen-operator-gen.ts +++ b/src/utils/sql-gen-operator-gen.ts @@ -25,7 +25,7 @@ type Return = { /** * # SQL Gen Operator Gen - * @description Generates an SQL operator for node module `mysql` or `serverless-mysql` + * @description Maps query equality operators to MariaDB SQL fragments */ export default function sqlGenOperatorGen({ fieldName, diff --git a/src/utils/sql-generator.ts b/src/utils/sql-generator.ts index 9b6511f..20fc350 100644 --- a/src/utils/sql-generator.ts +++ b/src/utils/sql-generator.ts @@ -20,7 +20,7 @@ type Return = { /** * # SQL Query Generator - * @description Generates an SQL Query for node module `mysql` or `serverless-mysql` + * @description Builds parameterized SELECT SQL for MariaDB */ export default function sqlGenerator< T extends { [key: string]: any } = { [key: string]: any }, diff --git a/tsconfig.json b/tsconfig.json index 2b0f895..34ebb79 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -21,6 +21,7 @@ "maxNodeModuleJsDepth": 10, "forceConsistentCasingInFileNames": true, "incremental": true, + "rootDir": "src", "outDir": "dist" }, "include": ["src"],