diff --git a/dist/commands/admin/index.js b/dist/commands/admin/index.js index 169d369..e3eac2e 100644 --- a/dist/commands/admin/index.js +++ b/dist/commands/admin/index.js @@ -6,6 +6,7 @@ import { select } from "@inquirer/prompts"; import { Database } from "bun:sqlite"; import listTables from "./list-tables"; import runSQL from "./run-sql"; +import * as sqliteVec from "sqlite-vec"; export default function () { return new Command("admin") .description("View Tables and Data, Run SQL Queries, Etc.") @@ -13,6 +14,7 @@ export default function () { const { config } = await init(); const { db_file_path } = grabDBDir({ config }); const db = new Database(db_file_path); + sqliteVec.load(db); console.log(chalk.bold(chalk.blue("\nBun SQLite Admin\n"))); try { while (true) { diff --git a/dist/lib/sqlite/db-schema-manager.d.ts b/dist/lib/sqlite/db-schema-manager.d.ts index 7a6883a..dbd446c 100644 --- a/dist/lib/sqlite/db-schema-manager.d.ts +++ b/dist/lib/sqlite/db-schema-manager.d.ts @@ -44,6 +44,7 @@ declare class SQLiteSchemaManager { * Add a new column to existing table */ private addColumn; + private checkIfTableExists; /** * Recreate table (for complex schema changes) */ diff --git a/dist/lib/sqlite/db-schema-manager.js b/dist/lib/sqlite/db-schema-manager.js index 719ebe6..7add6e5 100644 --- a/dist/lib/sqlite/db-schema-manager.js +++ b/dist/lib/sqlite/db-schema-manager.js @@ -102,13 +102,10 @@ class SQLiteSchemaManager { await this.createTable(table); this.insertDbManagerTable(table.tableName); } - else if (!table.isVector) { + else { // Update existing table await this.updateTable(table); } - else { - return; - } // Sync indexes await this.syncIndexes(table); } @@ -133,9 +130,9 @@ class SQLiteSchemaManager { const columns = []; const foreignKeys = []; for (const field of new_table.fields) { - const columnDef = this.buildColumnDefinition(field); + const columnDef = this.buildColumnDefinition(field, table.isVector); columns.push(columnDef); - if (field.foreignKey) { + if (field.foreignKey && !table.isVector) { foreignKeys.push(this.buildForeignKeyConstraint(field)); } } @@ -224,6 +221,12 @@ class SQLiteSchemaManager { const sql = `ALTER TABLE "${tableName}" ADD COLUMN ${cleanDef}`; this.db.run(sql); } + checkIfTableExists(table) { + const tableExists = this.db + .query(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`) + .get(table); + return Boolean(tableExists?.name); + } /** * Recreate table (for complex schema changes) */ @@ -232,11 +235,14 @@ class SQLiteSchemaManager { if (!this.recreate_vector_table) { return; } - console.log(`Recreating vector table: ${table.tableName}`); - const existingRows = this.db - .query(`SELECT * FROM "${table.tableName}"`) - .all(); - this.db.run(`DROP TABLE "${table.tableName}"`); + const does_table_exist = this.checkIfTableExists(table.tableName); + let existingRows = []; + if (does_table_exist) { + existingRows = this.db + .query(`SELECT * FROM "${table.tableName}"`) + .all(); + this.db.run(`DROP TABLE "${table.tableName}"`); + } await this.createTable(table); if (existingRows.length > 0) { for (let i = 0; i < existingRows.length; i++) { @@ -274,7 +280,7 @@ class SQLiteSchemaManager { /** * Build column definition SQL */ - buildColumnDefinition(field) { + buildColumnDefinition(field, is_vector) { if (!field.fieldName) { throw new Error("Field name is required"); } @@ -284,7 +290,12 @@ class SQLiteSchemaManager { const parts = [fieldName]; // Data type mapping const dataType = this.mapDataType(field); - parts.push(dataType); + if (dataType == "BLOB") { + parts.push("FLOAT[128]"); + } + else { + parts.push(dataType); + } // Primary key if (field.primaryKey) { parts.push("PRIMARY KEY"); diff --git a/dist/types/index.d.ts b/dist/types/index.d.ts index ee3ca37..580ccf5 100644 --- a/dist/types/index.d.ts +++ b/dist/types/index.d.ts @@ -85,6 +85,9 @@ export interface BUN_SQLITE_TableSchemaType { * If this is a vector table */ isVector?: boolean; + /** + * Type of vector. Defaults to `vec0` + */ vectorType?: string; } /** diff --git a/package.json b/package.json index bf2d3f5..73a74cb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@moduletrace/bun-sqlite", - "version": "1.0.24", + "version": "1.0.25", "description": "SQLite manager for Bun", "author": "Benjamin Toby", "main": "dist/index.js", diff --git a/src/commands/admin/index.ts b/src/commands/admin/index.ts index 2d946e0..10578f2 100644 --- a/src/commands/admin/index.ts +++ b/src/commands/admin/index.ts @@ -6,6 +6,7 @@ import { select } from "@inquirer/prompts"; import { Database } from "bun:sqlite"; import listTables from "./list-tables"; import runSQL from "./run-sql"; +import * as sqliteVec from "sqlite-vec"; export default function () { return new Command("admin") @@ -15,6 +16,8 @@ export default function () { const { db_file_path } = grabDBDir({ config }); const db = new Database(db_file_path); + sqliteVec.load(db); + console.log(chalk.bold(chalk.blue("\nBun SQLite Admin\n"))); try { diff --git a/src/lib/sqlite/db-schema-manager.ts b/src/lib/sqlite/db-schema-manager.ts index 16a1db8..bf53eb3 100644 --- a/src/lib/sqlite/db-schema-manager.ts +++ b/src/lib/sqlite/db-schema-manager.ts @@ -5,6 +5,7 @@ import _ from "lodash"; import DbClient from "."; import type { BUN_SQLITE_DatabaseSchemaType, + BUN_SQLITE_DATATYPES, BUN_SQLITE_FieldSchemaType, BUN_SQLITE_TableSchemaType, } from "../../types"; @@ -159,11 +160,9 @@ class SQLiteSchemaManager { // Create new table await this.createTable(table); this.insertDbManagerTable(table.tableName); - } else if (!table.isVector) { + } else { // Update existing table await this.updateTable(table); - } else { - return; } // Sync indexes @@ -203,10 +202,10 @@ class SQLiteSchemaManager { const foreignKeys: string[] = []; for (const field of new_table.fields) { - const columnDef = this.buildColumnDefinition(field); + const columnDef = this.buildColumnDefinition(field, table.isVector); columns.push(columnDef); - if (field.foreignKey) { + if (field.foreignKey && !table.isVector) { foreignKeys.push(this.buildForeignKeyConstraint(field)); } } @@ -326,6 +325,16 @@ class SQLiteSchemaManager { this.db.run(sql); } + private checkIfTableExists(table: string) { + const tableExists = this.db + .query( + `SELECT name FROM sqlite_master WHERE type='table' AND name=?`, + ) + .get(table) as any; + + return Boolean(tableExists?.name); + } + /** * Recreate table (for complex schema changes) */ @@ -337,13 +346,18 @@ class SQLiteSchemaManager { return; } - console.log(`Recreating vector table: ${table.tableName}`); + const does_table_exist = this.checkIfTableExists(table.tableName); - const existingRows = this.db - .query(`SELECT * FROM "${table.tableName}"`) - .all() as { [k: string]: any }[]; + let existingRows: { [k: string]: any }[] = []; + + if (does_table_exist) { + existingRows = this.db + .query(`SELECT * FROM "${table.tableName}"`) + .all() as { [k: string]: any }[]; + + this.db.run(`DROP TABLE "${table.tableName}"`); + } - this.db.run(`DROP TABLE "${table.tableName}"`); await this.createTable(table); if (existingRows.length > 0) { @@ -404,7 +418,10 @@ class SQLiteSchemaManager { /** * Build column definition SQL */ - private buildColumnDefinition(field: BUN_SQLITE_FieldSchemaType): string { + private buildColumnDefinition( + field: BUN_SQLITE_FieldSchemaType, + is_vector?: boolean, + ): string { if (!field.fieldName) { throw new Error("Field name is required"); } @@ -417,7 +434,11 @@ class SQLiteSchemaManager { // Data type mapping const dataType = this.mapDataType(field); - parts.push(dataType); + if (dataType == "BLOB") { + parts.push("FLOAT[128]"); + } else { + parts.push(dataType); + } // Primary key if (field.primaryKey) { @@ -459,13 +480,15 @@ class SQLiteSchemaManager { /** * Map DSQL data types to SQLite types */ - private mapDataType(field: BUN_SQLITE_FieldSchemaType): string { + private mapDataType( + field: BUN_SQLITE_FieldSchemaType, + ): (typeof BUN_SQLITE_DATATYPES)[number]["value"] { const dataType = field.dataType?.toLowerCase() || "text"; const vectorSize = field.vectorSize || 1536; // Vector Embeddings if (field.isVector) { - return `FLOAT[${vectorSize}]`; + return `FLOAT[${vectorSize}]` as any; } // Integer types diff --git a/src/types/index.ts b/src/types/index.ts index a8840e5..4a393a7 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -104,6 +104,9 @@ export interface BUN_SQLITE_TableSchemaType { * If this is a vector table */ isVector?: boolean; + /** + * Type of vector. Defaults to `vec0` + */ vectorType?: string; }