From 3b5fba40634d8212647290abcfa3c23e96383fb0 Mon Sep 17 00:00:00 2001 From: Benjamin Toby Date: Tue, 14 Jul 2026 08:12:13 +0100 Subject: [PATCH] Updates --- package.json | 2 +- src/commands/index.ts | 24 +- src/commands/schema.ts | 13 +- src/data/app-data.ts | 2 + src/functions/grab-mariadb-client.ts | 41 ++ src/functions/init.ts | 38 +- src/functions/set-mariadb-client.ts | 29 + src/index.ts | 6 +- src/lib/db-handler.ts | 21 +- src/lib/grab-db-connection.ts | 42 -- src/lib/mariadb/db-delete.ts | 9 +- src/lib/mariadb/db-insert.ts | 9 +- src/lib/mariadb/db-schema-manager.ts | 607 +++++++++++++----- src/lib/mariadb/db-select.ts | 12 +- src/lib/mariadb/db-update.ts | 5 + src/lib/mariadb/index.ts | 81 ++- src/lib/schema/build-column-definition.ts | 46 ++ src/lib/schema/create-db-manager-table.ts | 22 + src/lib/schema/create-db-schema.ts | 23 + ...isting-tables-from-tables-manager-table.ts | 18 + src/lib/schema/handle-db-schema-table.ts | 46 ++ src/lib/schema/handle-db-schema-tables.ts | 57 ++ src/lib/schema/map-data-types.ts | 102 +++ src/lib/schema/mariadb-quote-gen.ts | 3 + src/lib/schema/order-db-schema.ts | 86 +++ src/types/index.ts | 74 ++- 26 files changed, 1097 insertions(+), 321 deletions(-) create mode 100644 src/functions/grab-mariadb-client.ts create mode 100644 src/functions/set-mariadb-client.ts delete mode 100644 src/lib/grab-db-connection.ts create mode 100644 src/lib/schema/build-column-definition.ts create mode 100644 src/lib/schema/create-db-manager-table.ts create mode 100644 src/lib/schema/create-db-schema.ts create mode 100644 src/lib/schema/get-existing-tables-from-tables-manager-table.ts create mode 100644 src/lib/schema/handle-db-schema-table.ts create mode 100644 src/lib/schema/handle-db-schema-tables.ts create mode 100644 src/lib/schema/map-data-types.ts create mode 100644 src/lib/schema/mariadb-quote-gen.ts create mode 100644 src/lib/schema/order-db-schema.ts diff --git a/package.json b/package.json index 2d14448..a250dfd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@moduletrace/bun-mariadb", - "version": "1.1.11", + "version": "1.0.0", "description": "Mariadb manager for Bun", "author": "Benjamin Toby", "main": "dist/index.js", diff --git a/src/commands/index.ts b/src/commands/index.ts index 699c757..219afeb 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -6,31 +6,9 @@ import typedef from "./typedef"; import backup from "./backup"; import restore from "./restore"; import admin from "./admin"; -import type { - BUN_MARIADB_DatabaseSchemaType, - BunMariaDBConfig, -} from "../types"; import init from "../functions/init"; -/** - * # Declare Global Variables - */ -declare global { - var CONFIG: BunMariaDBConfig; - var DB_SCHEMA: BUN_MARIADB_DatabaseSchemaType; -} - -await init(); - -if (!global.CONFIG) { - console.error(`Couldn't grab global Config.`); - process.exit(1); -} - -if (!global.DB_SCHEMA) { - console.error(`Couldn't grab Database Schema.`); - process.exit(1); -} +init(); /** * # Describe Program diff --git a/src/commands/schema.ts b/src/commands/schema.ts index 74cf507..ed00d10 100644 --- a/src/commands/schema.ts +++ b/src/commands/schema.ts @@ -7,6 +7,7 @@ import _ from "lodash"; import appendDefaultFieldsToDbSchema from "../utils/append-default-fields-to-db-schema"; import chalk from "chalk"; import { writeLiveSchema } from "../functions/live-schema"; +import createDBSchema from "../lib/schema/create-db-schema"; export default function () { return new Command("schema") @@ -31,12 +32,14 @@ export default function () { dbSchema, }); - const manager = new MariaDBSchemaManager({ - schema: finaldbSchema, - }); + // const manager = new MariaDBSchemaManager({ + // schema: finaldbSchema, + // }); - await manager.syncSchema(); - manager.close(); + // await manager.syncSchema(); + // manager.close(); + + await createDBSchema({ db_schema: finaldbSchema }); if (isTypeDef && config.typedef_file_path) { const out_file = path.resolve( diff --git a/src/data/app-data.ts b/src/data/app-data.ts index 64f7838..2452228 100644 --- a/src/data/app-data.ts +++ b/src/data/app-data.ts @@ -4,4 +4,6 @@ export const AppData = { DefaultBackupDirName: ".backups", DbSchemaManagerTableName: "__db_schema_manager__", DbSchemaFileName: "schema.ts", + MaxInitRetries: 50, + InitRetryIntervalMilliseconds: 5000, } as const; diff --git a/src/functions/grab-mariadb-client.ts b/src/functions/grab-mariadb-client.ts new file mode 100644 index 0000000..acad097 --- /dev/null +++ b/src/functions/grab-mariadb-client.ts @@ -0,0 +1,41 @@ +import path from "path"; +import grabDirNames from "../data/grab-dir-names"; +import { type BunMariaDBConfig } from "../types"; +import { SQL } from "bun"; + +type Params = { + config: BunMariaDBConfig; +}; + +export default function grabMariaDBClient({ + config, +}: Params): Bun.SQL | undefined { + const { ROOT_DIR } = grabDirNames(); + + try { + const MariaDBClient = new SQL({ + hostname: process.env.BUN_MARIADB_SERVER_HOST, + username: process.env.BUN_MARIADB_SERVER_USERNAME, + password: process.env.BUN_MARIADB_SERVER_PASSWORD, + database: config.db_name, + port: process.env.BUN_MARIADB_SERVER_PORT + ? Number(process.env.BUN_MARIADB_SERVER_PORT) + : undefined, + ...config.db_config, + tls: config.ssl_ca + ? { + ca: Bun.file(path.resolve(ROOT_DIR, config.ssl_ca)), + rejectUnauthorized: false, + } + : { + rejectUnauthorized: false, + }, + adapter: "mariadb", + }); + + return MariaDBClient; + } catch (error: any) { + console.error(`Couldn't grab MariaDB Client => ` + error.message); + return; + } +} diff --git a/src/functions/init.ts b/src/functions/init.ts index 272ca30..8abd767 100644 --- a/src/functions/init.ts +++ b/src/functions/init.ts @@ -4,12 +4,22 @@ import { AppData } from "../data/app-data"; import grabDirNames from "../data/grab-dir-names"; import { type BunMariaDBConfig, - type BunMariaDBConfigReturn, type BUN_MARIADB_DatabaseSchemaType, RequiredENVs, } from "../types"; +import { SQL } from "bun"; +import setMariaDBClient from "./set-mariadb-client"; -export default async function init(): Promise { +/** + * # Declare Global Variables + */ +declare global { + var CONFIG: BunMariaDBConfig; + var DB_SCHEMA: BUN_MARIADB_DatabaseSchemaType; + var MARIADB_CLIENT: Bun.SQL; +} + +export default function init(): void { try { const { ROOT_DIR } = grabDirNames(); const { ConfigFileName } = AppData; @@ -23,7 +33,7 @@ export default async function init(): Promise { process.exit(1); } - const ConfigImport = await import(ConfigFilePath); + const ConfigImport = require(ConfigFilePath); const Config = ConfigImport["default"] as BunMariaDBConfig; if (!Config) { @@ -73,7 +83,7 @@ export default async function init(): Promise { process.exit(1); } - const DbSchemaImport = await import(DBSchemaFilePath); + const DbSchemaImport = require(DBSchemaFilePath); const DbSchema = DbSchemaImport[ "default" ] as BUN_MARIADB_DatabaseSchemaType; @@ -89,10 +99,22 @@ export default async function init(): Promise { global.CONFIG = Config; global.DB_SCHEMA = DbSchema; - return { - config: Config, - dbSchema: DbSchema, - }; + if (!global.CONFIG) { + console.error(`Couldn't grab global Config.`); + process.exit(1); + } + + if (!global.DB_SCHEMA) { + console.error(`Couldn't grab Database Schema.`); + process.exit(1); + } + + setMariaDBClient({ config: Config }); + + if (!global.MARIADB_CLIENT) { + console.error(`Couldn't set MariaDB Client.`); + process.exit(1); + } } catch (error: any) { console.error(`Initialization ERROR => ` + error.message); process.exit(1); diff --git a/src/functions/set-mariadb-client.ts b/src/functions/set-mariadb-client.ts new file mode 100644 index 0000000..b8e4801 --- /dev/null +++ b/src/functions/set-mariadb-client.ts @@ -0,0 +1,29 @@ +import { type BunMariaDBConfig } from "../types"; +import grabMariaDBClient from "./grab-mariadb-client"; + +type Params = { + config: BunMariaDBConfig; +}; + +export default function setMariaDBClient({ config }: Params): void { + try { + const MariaDBClient = grabMariaDBClient({ config }); + + if (!MariaDBClient) { + throw new Error(`Couldn't grab MariaDB Client`); + } + + global.MARIADB_CLIENT = MariaDBClient; + + // const test = await MariaDBClient.unsafe(`SHOW DATABASES`); + + // if (!test.count) { + // console.error(`MariaDBClient Error: Database not ready.`); + // console.log(test); + // process.exit(1); + // } + } catch (error: any) { + console.error(`ERROR setting MariaDB Client => ` + error.message); + process.exit(1); + } +} diff --git a/src/index.ts b/src/index.ts index 8417ce8..64f7c37 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,14 +4,10 @@ import DbInsert from "./lib/mariadb/db-insert"; import DbSelect from "./lib/mariadb/db-select"; import DbSQL from "./lib/mariadb/db-sql"; import DbUpdate from "./lib/mariadb/db-update"; -import type { BUN_MARIADB_DatabaseSchemaType, BunMariaDBConfig } from "./types"; import grabDbSchema from "./utils/grab-db-schema"; import grabJoinFieldsFromQueryObject from "./utils/grab-join-fields-from-query-object"; -declare global { - var CONFIG: BunMariaDBConfig; - var DB_SCHEMA: BUN_MARIADB_DatabaseSchemaType; -} +init(); const BunMariaDB = { select: DbSelect, diff --git a/src/lib/db-handler.ts b/src/lib/db-handler.ts index ccbe9a0..1823736 100644 --- a/src/lib/db-handler.ts +++ b/src/lib/db-handler.ts @@ -1,9 +1,14 @@ -import type { DBInsertReturn, DBResponseObject } from "../types"; -import MariaDBClient from "./mariadb"; +import grabMariaDBClient from "../functions/grab-mariadb-client"; +import type { + BunMariaDBConfig, + DBInsertReturn, + DBResponseObject, +} from "../types"; type Param = { query: string; values?: any[]; + config?: BunMariaDBConfig; }; /** @@ -12,9 +17,17 @@ type Param = { */ export default async function dbHandler< T extends { [k: string]: any } = { [k: string]: any }, ->({ query, values }: Param): Promise { +>({ query, values, config }: Param): Promise> { try { - const res = await MariaDBClient.unsafe(query, values); + const CLIENT = config + ? grabMariaDBClient({ config }) + : global.MARIADB_CLIENT; + + if (!CLIENT) { + throw new Error(`Couldn't grab MariaDB Client.`); + } + + const res = await CLIENT.unsafe(query, values); const count = res.count; diff --git a/src/lib/grab-db-connection.ts b/src/lib/grab-db-connection.ts deleted file mode 100644 index 2511d55..0000000 --- a/src/lib/grab-db-connection.ts +++ /dev/null @@ -1,42 +0,0 @@ -import * as mariadb from "mariadb"; -import type { Connection } from "mariadb"; -import type { DsqlConnectionParam } from "../types"; -import grabDbSSL from "./grab-db-ssl"; - -/** - * # Grab General CONNECTION for DSQL - */ -export default async function grabDBConnection( - param?: DsqlConnectionParam, -): Promise { - const configData = global.CONFIG; - const CONN_TIMEOUT = configData?.connection_timeout || 10000; - - const config: mariadb.ConnectionConfig = { - host: process.env.BUN_MARIADB_SERVER_HOST, - user: process.env.BUN_MARIADB_SERVER_USERNAME, - password: process.env.BUN_MARIADB_SERVER_PASSWORD, - database: configData?.db_name, - port: process.env.BUN_MARIADB_SERVER_PORT - ? Number(process.env.BUN_MARIADB_SERVER_PORT) - : undefined, - charset: configData?.charset || "utf8mb4", - ssl: grabDbSSL(), - bigIntAsNumber: true, - supportBigNumbers: true, - bigNumberStrings: false, - dateStrings: true, - metaAsArray: true, - socketTimeout: CONN_TIMEOUT, - connectTimeout: CONN_TIMEOUT, - compress: true, - ...param?.config, - }; - - try { - return await mariadb.createConnection(config); - } catch (error) { - console.log(`Error Grabbing DSQL Connection =>`, config); - throw error; - } -} diff --git a/src/lib/mariadb/db-delete.ts b/src/lib/mariadb/db-delete.ts index c6fc0b4..1f99f7b 100644 --- a/src/lib/mariadb/db-delete.ts +++ b/src/lib/mariadb/db-delete.ts @@ -1,6 +1,10 @@ import dbHandler from "../db-handler"; import _ from "lodash"; -import type { DBResponseObject, ServerQueryParam } from "../../types"; +import type { + BunMariaDBConfig, + DBResponseObject, + ServerQueryParam, +} from "../../types"; import sqlGenerator from "../../utils/sql-generator"; type Params< @@ -10,6 +14,7 @@ type Params< table: Table; query?: ServerQueryParam; targetId?: number | string; + config?: BunMariaDBConfig; }; function quoteIdentifier(identifier: string): string { @@ -23,6 +28,7 @@ export default async function DbDelete< table, query, targetId, + config, }: Params): Promise { let sqlObj: ReturnType | null = null; @@ -66,6 +72,7 @@ export default async function DbDelete< const res = await dbHandler({ query: sql, values: sqlObj.values as any, + config, }); if (!res.success) { diff --git a/src/lib/mariadb/db-insert.ts b/src/lib/mariadb/db-insert.ts index 64fcb99..5e74090 100644 --- a/src/lib/mariadb/db-insert.ts +++ b/src/lib/mariadb/db-insert.ts @@ -1,5 +1,9 @@ import dbHandler from "../db-handler"; -import type { DBResponseObject, SQLInsertGenReturn } from "../../types"; +import type { + BunMariaDBConfig, + DBResponseObject, + SQLInsertGenReturn, +} from "../../types"; import sqlInsertGenerator from "../../utils/sql-insert-generator"; import grabDuplicateSafeInsertSql from "../grab-duplicate-safe-insert-sql"; @@ -10,6 +14,7 @@ type Params< table: Table; data: Schema[]; update_on_duplicate?: boolean; + config?: BunMariaDBConfig; }; export default async function DbInsert< @@ -19,6 +24,7 @@ export default async function DbInsert< table, data, update_on_duplicate, + config, }: Params): Promise { let sqlObj: SQLInsertGenReturn | null = null; @@ -53,6 +59,7 @@ export default async function DbInsert< const res = await dbHandler({ query: sql, values: sqlObj.values, + config, }); if (!res.success) { diff --git a/src/lib/mariadb/db-schema-manager.ts b/src/lib/mariadb/db-schema-manager.ts index e672c04..960b52f 100644 --- a/src/lib/mariadb/db-schema-manager.ts +++ b/src/lib/mariadb/db-schema-manager.ts @@ -8,7 +8,6 @@ import type { BUN_MARIADB_TableSchemaType, } from "../../types"; import { AppData } from "../../data/app-data"; -import { readLiveSchema } from "../../functions/live-schema"; type QueryValues = any[]; @@ -35,20 +34,23 @@ class MariaDBSchemaManager { this.db_schema = schema; } - async syncSchema(): Promise { + public async syncSchema(): Promise { console.log("Starting schema synchronization..."); await this.createDbManagerTable(); + const orderedTables = this.sortTablesByDependencies( + this.db_schema.tables, + ); const existingTables = await this.getExistingTables(); const schemaTables = this.db_schema.tables.map((t) => t.tableName); - for (const table of this.db_schema.tables) { - await this.syncTable(table, existingTables); + for (const table of orderedTables) { + const resolvedTargetTable = this.resolveTable(table); + await this.syncTable(resolvedTargetTable, existingTables); } await this.dropRemovedTables(existingTables, schemaTables); - console.log("Schema synchronization complete!"); } @@ -80,7 +82,9 @@ class MariaDBSchemaManager { }); if (!res.success) { - throw new Error(`Database query failed: ${query}`); + throw new Error( + `Database query failed: ${query} ... ERROR: ${res.error || res.msg}`, + ); } } @@ -94,7 +98,9 @@ class MariaDBSchemaManager { }); if (!res.success) { - throw new Error(`Database query failed: ${query}`); + throw new Error( + `Database query failed: ${query} ... ERROR: ${res.error || res.msg}`, + ); } return (res.payload || []) as T[]; @@ -110,6 +116,65 @@ class MariaDBSchemaManager { `); } + private sortTablesByDependencies(tables: any[]): any[] { + const sorted: any[] = []; + const visited = new Set(); + const tempVisited = new Set(); + + const tableMap = new Map(); + for (const table of tables) { + if (table.tableName) { + tableMap.set(table.tableName, table); + } + } + + const visit = (tableName: string) => { + if (tempVisited.has(tableName)) { + console.warn( + `Circular dependency detected around table: ${tableName}. Proceeding carefully.`, + ); + return; + } + + if (!visited.has(tableName)) { + tempVisited.add(tableName); + + const tableDef = tableMap.get(tableName); + if (tableDef && tableDef.fields) { + for (const field of tableDef.fields) { + if ( + field.foreignKey && + field.foreignKey.destinationTableName + ) { + const parentTable = + field.foreignKey.destinationTableName; + if ( + tableMap.has(parentTable) && + parentTable !== tableName + ) { + visit(parentTable); + } + } + } + } + + tempVisited.delete(tableName); + visited.add(tableName); + if (tableDef) { + sorted.push(tableDef); + } + } + }; + + for (const table of tables) { + if (table.tableName && !visited.has(table.tableName)) { + visit(table.tableName); + } + } + + return sorted; + } + private async insertDbManagerTable(tableName: string): Promise { const now = Date.now(); @@ -150,30 +215,11 @@ class MariaDBSchemaManager { ): Promise { console.log(`Cleaning up tables ...`); + // Safely drop only tables that are tracked by the manager but no longer in the schema const tablesToDrop: string[] = existingTables.filter( - (tableName) => - !schemaTables.includes(tableName) && - !schemaTables.some((schemaTable) => - tableName.startsWith(`${schemaTable}_`), - ), + (tableName) => !schemaTables.includes(tableName), ); - const currentSchema = readLiveSchema(); - - if (currentSchema?.tables?.[0]) { - for (const table of currentSchema.tables) { - if (!table?.tableName) continue; - - const doesTableExist = schemaTables.find( - (tableName) => tableName === table.tableName, - ); - - if (!doesTableExist) { - tablesToDrop.push(table.tableName); - } - } - } - const uniqueTablesToDrop = _.uniq(tablesToDrop); for (const tableName of uniqueTablesToDrop) { @@ -186,37 +232,46 @@ class MariaDBSchemaManager { } private async syncTable( - table: BUN_MARIADB_TableSchemaType, + resolvedTable: BUN_MARIADB_TableSchemaType, existingTables: string[], ): Promise { - let tableExists = existingTables.includes(table.tableName); const liveTables = await this.getLiveTableNames(); + let tableExistsTracked = existingTables.includes( + resolvedTable.tableName, + ); + let tableExistsLive = liveTables.includes(resolvedTable.tableName); let wasRenamed = false; - if (table.tableNameOld && table.tableNameOld !== table.tableName) { - if (liveTables.includes(table.tableNameOld)) { + if ( + resolvedTable.tableNameOld && + resolvedTable.tableNameOld !== resolvedTable.tableName + ) { + if (liveTables.includes(resolvedTable.tableNameOld)) { console.log( - `Renaming table: ${table.tableNameOld} -> ${table.tableName}`, + `Renaming table: ${resolvedTable.tableNameOld} -> ${resolvedTable.tableName}`, ); await this.run( - `RENAME TABLE ${this.quoteIdentifier(table.tableNameOld)} TO ${this.quoteIdentifier(table.tableName)}`, + `RENAME TABLE ${this.quoteIdentifier(resolvedTable.tableNameOld)} TO ${this.quoteIdentifier(resolvedTable.tableName)}`, ); - await this.insertDbManagerTable(table.tableName); - await this.removeDbManagerTable(table.tableNameOld); - tableExists = true; + await this.insertDbManagerTable(resolvedTable.tableName); + await this.removeDbManagerTable(resolvedTable.tableNameOld); + tableExistsTracked = true; + tableExistsLive = true; wasRenamed = true; } } - if (!tableExists) { - await this.createTable(table); - await this.insertDbManagerTable(table.tableName); - } else if (!wasRenamed) { - await this.updateTable(table); - await this.insertDbManagerTable(table.tableName); + if (!tableExistsTracked && !tableExistsLive) { + await this.createTable(resolvedTable); + await this.insertDbManagerTable(resolvedTable.tableName); + } else { + if (!wasRenamed) { + await this.updateTable(resolvedTable); + } + await this.insertDbManagerTable(resolvedTable.tableName); } - await this.syncIndexes(table); + await this.syncIndexes(resolvedTable); } private resolveTable( @@ -239,27 +294,37 @@ class MariaDBSchemaManager { const mergedFieldsMap = new Map(); (parentTable.fields || []).forEach((f) => { - if (f.fieldName) mergedFieldsMap.set(f.fieldName, f); + if (f.fieldName) mergedFieldsMap.set(f.fieldName, _.cloneDeep(f)); }); (table.fields || []).forEach((f) => { if (f.fieldName) { const existing = mergedFieldsMap.get(f.fieldName) || {}; - mergedFieldsMap.set(f.fieldName, { ...existing, ...f }); + // Deep merge to properly handle nested objects like foreignKey + mergedFieldsMap.set(f.fieldName, _.merge({}, existing, f)); } }); - return _.merge({}, parentTable, { + return { + ..._.cloneDeep(parentTable), tableName: table.tableName, - tableDescription: table.tableDescription, - collation: table.collation, + tableDescription: + table.tableDescription || parentTable.tableDescription, + collation: table.collation || parentTable.collation, + isVector: + table.isVector !== undefined + ? table.isVector + : parentTable.isVector, fields: Array.from(mergedFieldsMap.values()), - indexes: [...(parentTable.indexes || []), ...(table.indexes || [])], + indexes: _.uniqBy( + [...(parentTable.indexes || []), ...(table.indexes || [])], + "indexName", + ), uniqueConstraints: [ ...(parentTable.uniqueConstraints || []), ...(table.uniqueConstraints || []), ], - }); + }; } private async createTable( @@ -272,15 +337,28 @@ class MariaDBSchemaManager { const newTable = this.resolveTable(table); const columnDefinitions: string[] = []; const foreignKeys: string[] = []; + const primaryKeys: string[] = []; for (const field of newTable.fields || []) { columnDefinitions.push(this.buildColumnDefinition(field)); + if (field.primaryKey && field.fieldName) { + primaryKeys.push(field.fieldName); + } + if (field.foreignKey && !newTable.isVector) { foreignKeys.push(this.buildForeignKeyConstraint(field)); } } + // Add composite or single primary key at the table level + if (primaryKeys.length > 0) { + const pkCols = primaryKeys + .map((k) => this.quoteIdentifier(k)) + .join(", "); + columnDefinitions.push(`PRIMARY KEY (${pkCols})`); + } + if (newTable.uniqueConstraints) { for (const constraint of newTable.uniqueConstraints) { if ( @@ -320,58 +398,165 @@ class MariaDBSchemaManager { } private async updateTable( - table: BUN_MARIADB_TableSchemaType, + resolvedTable: BUN_MARIADB_TableSchemaType, ): Promise { - console.log(`Updating table: ${table.tableName}`); - - const resolvedTable = this.resolveTable(table); const existingColumns = await this.getTableColumns( resolvedTable.tableName, ); - const missingFields = (resolvedTable.fields || []).filter( - (field) => - !existingColumns.some((col) => col.name === field.fieldName), - ); - - const hasModifiedFields = (resolvedTable.fields || []).some((field) => { - const current = existingColumns.find( - (col) => col.name === field.fieldName, - ); - if (!current) return false; - - const resolvedType = this.mapDataType(field).toLowerCase(); - const currentType = current.type.toLowerCase(); - - // If it's a vector column, compare the explicit metadata size inside the column comment fields - if (field.isVector || resolvedTable.isVector) { - const targetComment = `vector_size=${field.vectorSize || 1536}`; - return current.comment !== targetComment; - } - - return !resolvedType.startsWith(currentType); - }); - - // Fast path: Only adding new columns without metadata alterations or vector shifts - if ( - missingFields.length > 0 && - !hasModifiedFields && - !resolvedTable.isVector - ) { - for (const field of missingFields) { - await this.addColumn(resolvedTable.tableName, field); - } + if (existingColumns.length === 0) { + await this.createTable(resolvedTable); return; } - // If changes were explicitly made or vector sizing changed, execute safe zero-data-loss rebuild - if ( - hasModifiedFields || - missingFields.length > 0 || - resolvedTable.isVector - ) { - await this.recreateTable(resolvedTable); + const liveFieldsMap = new Map< + string, + { type: string; comment: string } + >( + existingColumns.map((col) => [ + col.name, + { type: col.type.toLowerCase(), comment: col.comment || "" }, + ]), + ); + const codeFieldsMap = new Map( + (resolvedTable.fields || []).map((f) => [f.fieldName, f]), + ); + + const fieldsToAdd: BUN_MARIADB_FieldSchemaType[] = []; + const fieldsToModify: BUN_MARIADB_FieldSchemaType[] = []; + const fieldsToDrop: string[] = []; + + for (const field of resolvedTable.fields || []) { + if (!field.fieldName) continue; + + const liveField = liveFieldsMap.get(field.fieldName); + + if (!liveField) { + fieldsToAdd.push(field); + } else { + const resolvedType = this.mapDataType(field).toLowerCase(); + + let typeDiverged = !resolvedType.startsWith(liveField.type); + + if (field.isVector || field.dataType === "VECTOR") { + const dimensions = field.vectorSize || 1536; + const expectedNativeToken = `vector(${dimensions})`; + typeDiverged = liveField.type !== expectedNativeToken; + } + + if (typeDiverged) { + fieldsToModify.push(field); + } + } } + + for (const col of existingColumns) { + if (!codeFieldsMap.has(col.name)) { + fieldsToDrop.push(col.name); + } + } + + if ( + fieldsToAdd.length === 0 && + fieldsToModify.length === 0 && + fieldsToDrop.length === 0 + ) { + return; + } + + console.log( + `Surgically updating table structure from database layout: ${resolvedTable.tableName}`, + ); + + // Handle dependencies before dropping columns + if (fieldsToDrop.length > 0) { + const schemaCond = this.schemaCondition(); + + // Check if any dropped column is part of the primary key + const pkRows = await this.query<{ COLUMN_NAME: string }>( + `SELECT COLUMN_NAME FROM information_schema.KEY_COLUMN_USAGE WHERE ${schemaCond.where} AND TABLE_NAME = ? AND CONSTRAINT_NAME = 'PRIMARY'`, + [...schemaCond.values, resolvedTable.tableName], + ); + const pkColumnNames = pkRows.map((r) => r.COLUMN_NAME); + + if (fieldsToDrop.some((f) => pkColumnNames.includes(f))) { + console.log( + `Dropping primary key because a PK column is being dropped`, + ); + await this.run( + `ALTER TABLE ${this.quoteIdentifier(resolvedTable.tableName)} DROP PRIMARY KEY`, + ); + } + + // Drop secondary indexes that include dropped columns + const indexRows = await this.query<{ + INDEX_NAME: string; + COLUMN_NAME: string; + }>( + `SELECT INDEX_NAME, COLUMN_NAME FROM information_schema.STATISTICS WHERE ${schemaCond.where} AND TABLE_NAME = ? AND INDEX_NAME <> 'PRIMARY'`, + [...schemaCond.values, resolvedTable.tableName], + ); + + const indexesToDrop = new Set(); + for (const row of indexRows) { + if (fieldsToDrop.includes(row.COLUMN_NAME)) { + indexesToDrop.add(row.INDEX_NAME); + } + } + + for (const indexName of indexesToDrop) { + console.log( + `Dropping index ${indexName} because it contains a dropped column`, + ); + await this.run( + `DROP INDEX ${this.quoteIdentifier(indexName)} ON ${this.quoteIdentifier(resolvedTable.tableName)}`, + ); + } + } + + for (const field of fieldsToAdd) { + await this.addColumn(resolvedTable.tableName, field); + } + + for (const field of fieldsToModify) { + try { + await this.modifyColumn(resolvedTable.tableName, field); + } catch (err: any) { + if (field.isVector || field.dataType === "VECTOR") { + console.warn( + `[Vector Resize] Re-aligning dimension spaces natively via safe migration schema rebuild.`, + ); + await this.recreateTable(resolvedTable); + return; + } + throw err; + } + } + + for (const fieldName of fieldsToDrop) { + await this.dropColumn(resolvedTable.tableName, fieldName); + } + } + + private async dropColumn( + tableName: string, + fieldName: string, + ): Promise { + console.log(`Dropping column: ${tableName}.${fieldName}`); + await this.run( + `ALTER TABLE ${this.quoteIdentifier(tableName)} DROP COLUMN ${this.quoteIdentifier(fieldName)}`, + ); + } + + private async modifyColumn( + tableName: string, + field: BUN_MARIADB_FieldSchemaType, + ): Promise { + console.log(`Modifying column: ${tableName}.${field.fieldName}`); + const columnDef = this.buildColumnDefinition(field).trim(); + await this.run( + `ALTER TABLE ${this.quoteIdentifier(tableName)} MODIFY COLUMN ${columnDef}`, + ); } private async getTableColumns(tableName: string): Promise { @@ -397,15 +582,9 @@ class MariaDBSchemaManager { field: BUN_MARIADB_FieldSchemaType, ): Promise { console.log(`Adding column: ${tableName}.${field.fieldName}`); - - const columnDef = this.buildColumnDefinition(field) - .replace(/PRIMARY KEY/gi, "") - .replace(/AUTO_INCREMENT/gi, "") - .replace(/UNIQUE/gi, "") - .trim(); - + const columnDef = this.buildColumnDefinition(field).trim(); await this.run( - `ALTER TABLE ${this.quoteIdentifier(tableName)} ADD COLUMN ${columnDef}`, + `ALTER TABLE ${this.quoteIdentifier(tableName)} ADD COLUMN IF NOT EXISTS ${columnDef}`, ); } @@ -453,15 +632,21 @@ class MariaDBSchemaManager { ); } - await this.run( - `RENAME TABLE ${this.quoteIdentifier(table.tableName)} TO ${this.quoteIdentifier(backupOldTableName)}`, - ); - await this.run( - `RENAME TABLE ${this.quoteIdentifier(tempTableName)} TO ${this.quoteIdentifier(table.tableName)}`, - ); - await this.run( - `DROP TABLE ${this.quoteIdentifier(backupOldTableName)}`, - ); + // Disable foreign key checks to allow dropping/renaming tables with external references + await this.run(`SET FOREIGN_KEY_CHECKS = 0`); + try { + await this.run( + `RENAME TABLE ${this.quoteIdentifier(table.tableName)} TO ${this.quoteIdentifier(backupOldTableName)}`, + ); + await this.run( + `RENAME TABLE ${this.quoteIdentifier(tempTableName)} TO ${this.quoteIdentifier(table.tableName)}`, + ); + await this.run( + `DROP TABLE ${this.quoteIdentifier(backupOldTableName)}`, + ); + } finally { + await this.run(`SET FOREIGN_KEY_CHECKS = 1`); + } } private buildColumnDefinition(field: BUN_MARIADB_FieldSchemaType): string { @@ -470,18 +655,13 @@ class MariaDBSchemaManager { } const parts: string[] = [this.quoteIdentifier(field.fieldName)]; - parts.push(this.mapDataType(field)); - if (field.primaryKey) { - parts.push("PRIMARY KEY"); - - if (field.autoIncrement) { - parts.push("AUTO_INCREMENT"); - } + if (field.autoIncrement) { + parts.push("AUTO_INCREMENT"); } - if (field.notNullValue || field.primaryKey) { + if (field.notNullValue || field.primaryKey || field.isVector) { if (!field.primaryKey) { parts.push("NOT NULL"); } @@ -521,57 +701,93 @@ class MariaDBSchemaManager { } switch (dataType) { + case "CHAR": + return `CHAR(${field.integerLength || 255})`; case "VARCHAR": return `VARCHAR(${field.integerLength || 255})`; - case "TEXT": return "TEXT"; - + case "TINYTEXT": + return "TINYTEXT"; + case "MEDIUMTEXT": + return "MEDIUMTEXT"; case "LONGTEXT": return "LONGTEXT"; - case "TINYINT": return field.integerLength ? `TINYINT(${field.integerLength})` : "TINYINT"; - + case "SMALLINT": + return field.integerLength + ? `SMALLINT(${field.integerLength})` + : "SMALLINT"; + case "MEDIUMINT": + return field.integerLength + ? `MEDIUMINT(${field.integerLength})` + : "MEDIUMINT"; case "INT": return field.integerLength ? `INT(${field.integerLength})` : "INT"; - case "BIGINT": return field.integerLength ? `BIGINT(${field.integerLength})` : "BIGINT"; - + case "FLOAT": + return "FLOAT"; + case "DOUBLE": + return "DOUBLE"; case "DECIMAL": if (field.integerLength && field.decimals) { return `DECIMAL(${field.integerLength}, ${field.decimals})`; } return "DECIMAL(10,2)"; - - case "DOUBLE": - return "DOUBLE"; - + case "BINARY": + return `BINARY(${field.integerLength || 1})`; + case "VARBINARY": + return `VARBINARY(${field.integerLength || 255})`; case "BLOB": return "BLOB"; - + case "TINYBLOB": + return "TINYBLOB"; + case "MEDIUMBLOB": + return "MEDIUMBLOB"; case "LONGBLOB": return "LONGBLOB"; - - case "BOOLEAN": - return "TINYINT(1)"; - - case "DATETIME": - return "DATETIME"; - - case "TIMESTAMP": - return "TIMESTAMP"; - case "DATE": return "DATE"; - + case "TIME": + return "TIME"; + case "DATETIME": + return "DATETIME"; + case "TIMESTAMP": + return "TIMESTAMP"; + case "YEAR": + return "YEAR"; + case "UUID": + return "CHAR(36)"; // MariaDB does not have a native UUID type + case "JSON": + return "JSON"; + case "INET6": + return "INET6"; + case "VECTOR": { + const dimensions = field.vectorSize || 1536; + return `VECTOR(${dimensions})`; + } + case "BOOLEAN": + return "TINYINT(1)"; + case "ENUM": { + const enumVals = (field.options || []) + .map((v) => `'${String(v).replace(/'/g, "''")}'`) + .join(", "); + return `ENUM(${enumVals || "''"})`; + } + case "SET": { + const setVals = (field.options || []) + .map((v) => `'${String(v).replace(/'/g, "''")}'`) + .join(", "); + return `SET(${setVals || "''"})`; + } default: return "TEXT"; } @@ -601,31 +817,63 @@ class MariaDBSchemaManager { private async syncIndexes( table: BUN_MARIADB_TableSchemaType, ): Promise { - if (!table.indexes || table.indexes.length === 0) { - return; - } - const schemaCond = this.schemaCondition(); - const rows = await this.query( - `SELECT INDEX_NAME AS name FROM information_schema.STATISTICS WHERE ${schemaCond.where} AND TABLE_NAME = ? AND INDEX_NAME <> 'PRIMARY' GROUP BY INDEX_NAME ORDER BY INDEX_NAME`, + const rows = await this.query<{ + INDEX_NAME: string; + COLUMN_NAME: string; + INDEX_TYPE: string; + }>( + `SELECT INDEX_NAME, COLUMN_NAME, INDEX_TYPE FROM information_schema.STATISTICS WHERE ${schemaCond.where} AND TABLE_NAME = ? AND INDEX_NAME <> 'PRIMARY' ORDER BY INDEX_NAME, SEQ_IN_INDEX`, [...schemaCond.values, table.tableName], ); - const existingIndexes = rows.map((row) => row.name); - for (const indexName of existingIndexes) { - const stillExists = table.indexes.some( - (index) => index.indexName === indexName, + const existingIndexesMap = new Map< + string, + { columns: string[]; type: string } + >(); + for (const row of rows) { + if (!existingIndexesMap.has(row.INDEX_NAME)) { + existingIndexesMap.set(row.INDEX_NAME, { + columns: [], + type: row.INDEX_TYPE, + }); + } + existingIndexesMap + .get(row.INDEX_NAME)! + .columns.push(row.COLUMN_NAME); + } + + // Drop indexes that are no longer in schema OR have changed columns + for (const [indexName, details] of existingIndexesMap.entries()) { + const schemaIndex = table.indexes?.find( + (i) => i.indexName === indexName, ); - if (!stillExists) { + if (!schemaIndex) { console.log(`Dropping index: ${indexName}`); await this.run( `DROP INDEX ${this.quoteIdentifier(indexName)} ON ${this.quoteIdentifier(table.tableName)}`, ); + } else { + const schemaColumns = schemaIndex.indexTableFields || []; + const columnsMatch = + details.columns.length === schemaColumns.length && + details.columns.every( + (col, idx) => col === schemaColumns[idx], + ); + + if (!columnsMatch) { + console.log(`Recreating changed index: ${indexName}`); + await this.run( + `DROP INDEX ${this.quoteIdentifier(indexName)} ON ${this.quoteIdentifier(table.tableName)}`, + ); + existingIndexesMap.delete(indexName); + } } } - for (const index of table.indexes) { + // Create missing indexes + for (const index of table.indexes || []) { if ( !index.indexName || !index.indexTableFields || @@ -634,27 +882,46 @@ class MariaDBSchemaManager { continue; } - if (!existingIndexes.includes(index.indexName)) { - console.log(`Creating index: ${index.indexName}`); + if (!existingIndexesMap.has(index.indexName)) { + const isVectorIndex = + table.isVector || + table.fields?.some( + (f) => + f.fieldName === index.indexTableFields![0] && + f.isVector, + ); - const fields = index.indexTableFields - .map((field) => this.quoteIdentifier(field)) - .join(", "); + if (isVectorIndex) { + console.log(`Creating Vector index: ${index.indexName}`); + const targetField = this.quoteIdentifier( + index.indexTableFields[0]!, + ); - const typeUpper = index.indexType?.toUpperCase(); - const isSpecialType = - typeUpper === "FULLTEXT" || typeUpper === "SPATIAL"; - const indexPrefix = isSpecialType ? `${typeUpper} ` : ""; + const distanceMetric = + index.indexType?.toLowerCase() === "cosine" + ? "cosine" + : "euclidean"; - const indexSuffix = - !isSpecialType && - (typeUpper === "BTREE" || typeUpper === "HASH") - ? ` USING ${typeUpper}` - : ""; + const sql = `ALTER TABLE ${this.quoteIdentifier(table.tableName)} ADD VECTOR INDEX ${this.quoteIdentifier(index.indexName)} (${targetField}) M=8 DISTANCE=${distanceMetric}`; + await this.run(sql); + } else { + console.log(`Creating standard index: ${index.indexName}`); + const fields = index.indexTableFields + .map((field) => this.quoteIdentifier(field)) + .join(", "); + const typeUpper = index.indexType?.toUpperCase(); + const isSpecialType = + typeUpper === "FULLTEXT" || typeUpper === "SPATIAL"; + const indexPrefix = isSpecialType ? `${typeUpper} ` : ""; + const indexSuffix = + !isSpecialType && + (typeUpper === "BTREE" || typeUpper === "HASH") + ? ` USING ${typeUpper}` + : ""; - const sql = `CREATE ${indexPrefix}INDEX ${this.quoteIdentifier(index.indexName)} ON ${this.quoteIdentifier(table.tableName)} (${fields})${indexSuffix}`; - - await this.run(sql); + const sql = `CREATE ${indexPrefix}INDEX ${this.quoteIdentifier(index.indexName)} ON ${this.quoteIdentifier(table.tableName)} (${fields})${indexSuffix}`; + await this.run(sql); + } } } } diff --git a/src/lib/mariadb/db-select.ts b/src/lib/mariadb/db-select.ts index 8b65e6a..11f2ea3 100644 --- a/src/lib/mariadb/db-select.ts +++ b/src/lib/mariadb/db-select.ts @@ -1,6 +1,10 @@ import dbHandler from "../db-handler"; import _ from "lodash"; -import type { DBResponseObject, ServerQueryParam } from "../../types"; +import type { + BunMariaDBConfig, + DBResponseObject, + ServerQueryParam, +} from "../../types"; import sqlGenerator from "../../utils/sql-generator"; type Params< @@ -11,6 +15,7 @@ type Params< table: Table; count?: boolean; targetId?: number | string; + config?: BunMariaDBConfig; }; function quoteIdentifier(identifier: string): string { @@ -25,6 +30,7 @@ export default async function DbSelect< query, count, targetId, + config, }: Params): Promise> { let sqlObj: ReturnType | null = null; @@ -52,6 +58,7 @@ export default async function DbSelect< const res = await dbHandler({ query: sqlObj.string, values: sqlObj.values as any, + config, }); if (!res.success) { @@ -89,6 +96,7 @@ export default async function DbSelect< const countRes = await dbHandler<{ count: number }>({ query: countSql, values: countSqlObject.values as any, + config, }); if (!countRes.success) { @@ -104,7 +112,7 @@ export default async function DbSelect< } const countRows = countRes.payload || []; - const countVal = countRows[0]?.count ?? countRows[0]?.["COUNT(*)"]; + const countVal = countRows[0]?.count ?? countRows[0]?.["count"]; resp = { ...resp, diff --git a/src/lib/mariadb/db-update.ts b/src/lib/mariadb/db-update.ts index b4b540d..790f055 100644 --- a/src/lib/mariadb/db-update.ts +++ b/src/lib/mariadb/db-update.ts @@ -1,6 +1,7 @@ import dbHandler from "../db-handler"; import _ from "lodash"; import type { + BunMariaDBConfig, DBResponseObject, SQLInsertGenValueType, ServerQueryParam, @@ -15,6 +16,7 @@ type Params< data: Schema; query?: ServerQueryParam; targetId?: number | string; + config?: BunMariaDBConfig; }; function quoteIdentifier(identifier: string): string { @@ -29,6 +31,7 @@ export default async function DbUpdate< data, query, targetId, + config, }: Params): Promise { let sqlObj: ReturnType = { string: "", values: [] }; @@ -93,6 +96,7 @@ export default async function DbUpdate< const res = await dbHandler({ query: sql, values: values as any, + config, }); sqlObj.string = sql; @@ -122,6 +126,7 @@ export default async function DbUpdate< const updated_res = await dbHandler({ query: updated_sql, values: updated_sql_values, + config, }); const affected_rows = updated_res.payload?.length; diff --git a/src/lib/mariadb/index.ts b/src/lib/mariadb/index.ts index 4e105b1..1dc5bce 100644 --- a/src/lib/mariadb/index.ts +++ b/src/lib/mariadb/index.ts @@ -1,50 +1,49 @@ -import { SQL } from "bun"; -import init from "../../functions/init"; -import grabDirNames from "../../data/grab-dir-names"; -import path from "path"; +// import { SQL } from "bun"; +// import grabDirNames from "../../data/grab-dir-names"; +// import path from "path"; -await init(); +// if (!global.CONFIG) { +// console.error(`Couldn't grab global Config.`); +// process.exit(1); +// } -if (!global.CONFIG) { - console.error(`Couldn't grab global Config.`); - process.exit(1); -} +// if (!global.DB_SCHEMA) { +// console.error(`Couldn't grab Database Schema.`); +// process.exit(1); +// } -if (!global.DB_SCHEMA) { - console.error(`Couldn't grab Database Schema.`); - process.exit(1); -} +// const config = global.CONFIG; -const config = global.CONFIG; +// const { ROOT_DIR } = grabDirNames(); -const { ROOT_DIR } = grabDirNames(); +// const MariaDBClient = new SQL({ +// hostname: process.env.BUN_MARIADB_SERVER_HOST, +// username: process.env.BUN_MARIADB_SERVER_USERNAME, +// password: process.env.BUN_MARIADB_SERVER_PASSWORD, +// database: config.db_name, +// port: process.env.BUN_MARIADB_SERVER_PORT +// ? Number(process.env.BUN_MARIADB_SERVER_PORT) +// : undefined, +// ...config.db_config, +// tls: config.ssl_ca +// ? { +// ca: Bun.file(path.resolve(ROOT_DIR, config.ssl_ca)), +// rejectUnauthorized: false, +// } +// : { +// rejectUnauthorized: false, +// }, +// adapter: "mariadb", +// }); -const MariaDBClient = new SQL({ - hostname: process.env.BUN_MARIADB_SERVER_HOST, - username: process.env.BUN_MARIADB_SERVER_USERNAME, - password: process.env.BUN_MARIADB_SERVER_PASSWORD, - database: config.db_name, - port: process.env.BUN_MARIADB_SERVER_PORT - ? Number(process.env.BUN_MARIADB_SERVER_PORT) - : undefined, - ...config.db_config, - tls: config.ssl_ca - ? { - ca: Bun.file(path.resolve(ROOT_DIR, config.ssl_ca)), - rejectUnauthorized: false, - } - : { - rejectUnauthorized: false, - }, - adapter: "mariadb", -}); +// global.MARIADB_CLIENT = MariaDBClient; -const test = await MariaDBClient.unsafe(`SHOW DATABASES`); +// const test = await MariaDBClient.unsafe(`SHOW DATABASES`); -if (!test.count) { - console.error(`MariaDBClient Error: Database not ready.`); - console.log(test); - process.exit(1); -} +// if (!test.count) { +// console.error(`MariaDBClient Error: Database not ready.`); +// console.log(test); +// process.exit(1); +// } -export default MariaDBClient; +// export default MariaDBClient; diff --git a/src/lib/schema/build-column-definition.ts b/src/lib/schema/build-column-definition.ts new file mode 100644 index 0000000..5308142 --- /dev/null +++ b/src/lib/schema/build-column-definition.ts @@ -0,0 +1,46 @@ +import type { BUN_MARIADB_FieldSchemaType } from "../../types"; +import mapDataType from "./map-data-types"; +import MariaDBQuoteGen from "./mariadb-quote-gen"; + +export default function buildColumnDefinition( + field: BUN_MARIADB_FieldSchemaType, +): string { + if (!field.fieldName) { + throw new Error("Field name is required"); + } + + const parts: string[] = [MariaDBQuoteGen(field.fieldName)]; + parts.push(mapDataType(field)); + + if (field.autoIncrement) { + parts.push("AUTO_INCREMENT"); + } + + if (field.notNullValue || field.primaryKey || field.isVector) { + if (!field.primaryKey) { + parts.push("NOT NULL"); + } + } + + if (field.unique && !field.primaryKey) { + parts.push("UNIQUE"); + } + + if (field.defaultValue !== undefined) { + if (typeof field.defaultValue === "string") { + parts.push(`DEFAULT '${field.defaultValue.replace(/'/g, "''")}'`); + } else { + parts.push(`DEFAULT ${field.defaultValue}`); + } + } else if (field.defaultValueLiteral) { + parts.push(`DEFAULT ${field.defaultValueLiteral}`); + } + + if (field.onUpdate) { + parts.push(`ON UPDATE ${field.onUpdate}`); + } else if (field.onUpdateLiteral) { + parts.push(`ON UPDATE ${field.onUpdateLiteral}`); + } + + return parts.join(" "); +} diff --git a/src/lib/schema/create-db-manager-table.ts b/src/lib/schema/create-db-manager-table.ts new file mode 100644 index 0000000..6d9b2ab --- /dev/null +++ b/src/lib/schema/create-db-manager-table.ts @@ -0,0 +1,22 @@ +import { AppData } from "../../data/app-data"; +import type { CreateDBSchemaParams } from "../../types"; +import dbHandler from "../db-handler"; +import MariaDBQuoteGen from "./mariadb-quote-gen"; + +export default async function createDBManagerTable({ + db_schema, + config, +}: CreateDBSchemaParams) { + let sql = ``; + + sql += `CREATE TABLE IF NOT EXISTS ${MariaDBQuoteGen(AppData["DbSchemaManagerTableName"])} (\n`; + sql += ` table_name VARCHAR(255) NOT NULL PRIMARY KEY,\n`; + sql += ` created_at BIGINT NOT NULL,\n`; + sql += ` updated_at BIGINT NOT NULL\n`; + sql += `) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci\n`; + + await dbHandler({ + query: sql, + config, + }); +} diff --git a/src/lib/schema/create-db-schema.ts b/src/lib/schema/create-db-schema.ts new file mode 100644 index 0000000..d6a87c8 --- /dev/null +++ b/src/lib/schema/create-db-schema.ts @@ -0,0 +1,23 @@ +import type { CreateDBSchemaParams } from "../../types"; +import createDBManagerTable from "./create-db-manager-table"; +import handleDBSchemaTables from "./handle-db-schema-tables"; +import orderDBSchema from "./order-db-schema"; + +export default async function createDBSchema(params: CreateDBSchemaParams) { + /** + * Create Schema Manager Table + */ + await createDBManagerTable(params); + + /** + * Reorder Tables + */ + const ordered_db_schema = await orderDBSchema(params); + + /** + * Handle Tables + */ + await handleDBSchemaTables({ ...params, db_schema: ordered_db_schema }); + + process.exit(); +} diff --git a/src/lib/schema/get-existing-tables-from-tables-manager-table.ts b/src/lib/schema/get-existing-tables-from-tables-manager-table.ts new file mode 100644 index 0000000..ee9741c --- /dev/null +++ b/src/lib/schema/get-existing-tables-from-tables-manager-table.ts @@ -0,0 +1,18 @@ +import { AppData } from "../../data/app-data"; +import type { + BUN_MARIADB_DB_TABLE_MANAGER_TABLE, + CreateDBSchemaParams, +} from "../../types"; +import dbHandler from "../db-handler"; +import MariaDBQuoteGen from "./mariadb-quote-gen"; + +export default async function getExistingTablesFromTablesManagerTable({ + config, +}: CreateDBSchemaParams) { + const rows = await dbHandler({ + query: `SELECT table_name FROM ${MariaDBQuoteGen(AppData["DbSchemaManagerTableName"])}`, + config, + }); + + return rows.payload?.map((row) => row.table_name) || []; +} diff --git a/src/lib/schema/handle-db-schema-table.ts b/src/lib/schema/handle-db-schema-table.ts new file mode 100644 index 0000000..53c7026 --- /dev/null +++ b/src/lib/schema/handle-db-schema-table.ts @@ -0,0 +1,46 @@ +import { AppData } from "../../data/app-data"; +import type { + BUN_MARIADB_DB_TABLE_MANAGER_TABLE, + BUN_MARIADB_INFORMATION_SCHEMA_TABLES, + CreateDBSchemaTableHandlerParams, +} from "../../types"; +import dbHandler from "../db-handler"; +import DbInsert from "../mariadb/db-insert"; +import buildColumnDefinition from "./build-column-definition"; +import MariaDBQuoteGen from "./mariadb-quote-gen"; + +export default async function handleDBSchemaTable({ + db_schema, + config, + table, + db_manager_table_name, + existing_live_table, +}: CreateDBSchemaTableHandlerParams) { + if (!db_manager_table_name || !existing_live_table?.TABLE_NAME) { + const insert_table = await DbInsert( + { + data: [ + { + table_name: table.tableName, + }, + ], + table: AppData["DbSchemaManagerTableName"], + config, + }, + ); + + let sql = ``; + + sql += `CREATE TABLE IF NOT EXISTS ${MariaDBQuoteGen(table.tableName)}`; + sql += ` (`; + + for (let i = 0; i < table.fields.length; i++) { + const field = table.fields[i]; + if (!field) continue; + const col_def = buildColumnDefinition(field); + sql += ` ${col_def}`; + } + + sql += ` )`; + } +} diff --git a/src/lib/schema/handle-db-schema-tables.ts b/src/lib/schema/handle-db-schema-tables.ts new file mode 100644 index 0000000..31eebdb --- /dev/null +++ b/src/lib/schema/handle-db-schema-tables.ts @@ -0,0 +1,57 @@ +import { AppData } from "../../data/app-data"; +import type { + BUN_MARIADB_INFORMATION_SCHEMA_TABLES, + CreateDBSchemaParams, +} from "../../types"; +import dbHandler from "../db-handler"; +import getExistingTablesFromTablesManagerTable from "./get-existing-tables-from-tables-manager-table"; +import handleDBSchemaTable from "./handle-db-schema-table"; + +export default async function handleDBSchemaTables( + params: CreateDBSchemaParams, +) { + const { db_schema, config } = params; + + /** + * Grab Tables that have been recorded in the Schema + * Manager Table + */ + const existing_schema_tables = + await getExistingTablesFromTablesManagerTable(params); + + const existing_live_tables = + await dbHandler({ + query: `SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_TYPE = 'BASE TABLE' AND TABLE_NAME != ?`, + config: params.config, + values: [AppData["DbSchemaManagerTableName"]], + }); + + for (let i = 0; i < db_schema.tables.length; i++) { + const table = db_schema.tables[i]; + + const existing_table = existing_schema_tables.find( + (t) => t == table?.tableName, + ); + + const existing_live_table = existing_live_tables.payload?.find( + (t) => t.TABLE_NAME == table?.tableName, + ); + + if (!table) { + if (existing_live_table?.TABLE_NAME) { + await dbHandler({ + query: `DROP TABLE ${existing_live_table.TABLE_NAME}`, + config: params.config, + }); + } + continue; + } + + await handleDBSchemaTable({ + ...params, + table, + db_manager_table_name: existing_table, + existing_live_table, + }); + } +} diff --git a/src/lib/schema/map-data-types.ts b/src/lib/schema/map-data-types.ts new file mode 100644 index 0000000..11f32d4 --- /dev/null +++ b/src/lib/schema/map-data-types.ts @@ -0,0 +1,102 @@ +import type { BUN_MARIADB_FieldSchemaType } from "../../types"; + +export default function mapDataType( + field: BUN_MARIADB_FieldSchemaType, +): string { + const dataType = field.dataType?.toUpperCase() || "TEXT"; + const vectorSize = field.vectorSize || 1536; + + if (field.isVector) { + return `LONGTEXT COMMENT 'vector_size=${vectorSize}'`; + } + + switch (dataType) { + case "CHAR": + return `CHAR(${field.integerLength || 255})`; + case "VARCHAR": + return `VARCHAR(${field.integerLength || 255})`; + case "TEXT": + return "TEXT"; + case "TINYTEXT": + return "TINYTEXT"; + case "MEDIUMTEXT": + return "MEDIUMTEXT"; + case "LONGTEXT": + return "LONGTEXT"; + case "TINYINT": + return field.integerLength + ? `TINYINT(${field.integerLength})` + : "TINYINT"; + case "SMALLINT": + return field.integerLength + ? `SMALLINT(${field.integerLength})` + : "SMALLINT"; + case "MEDIUMINT": + return field.integerLength + ? `MEDIUMINT(${field.integerLength})` + : "MEDIUMINT"; + case "INT": + return field.integerLength ? `INT(${field.integerLength})` : "INT"; + case "BIGINT": + return field.integerLength + ? `BIGINT(${field.integerLength})` + : "BIGINT"; + case "FLOAT": + return "FLOAT"; + case "DOUBLE": + return "DOUBLE"; + case "DECIMAL": + if (field.integerLength && field.decimals) { + return `DECIMAL(${field.integerLength}, ${field.decimals})`; + } + return "DECIMAL(10,2)"; + case "BINARY": + return `BINARY(${field.integerLength || 1})`; + case "VARBINARY": + return `VARBINARY(${field.integerLength || 255})`; + case "BLOB": + return "BLOB"; + case "TINYBLOB": + return "TINYBLOB"; + case "MEDIUMBLOB": + return "MEDIUMBLOB"; + case "LONGBLOB": + return "LONGBLOB"; + case "DATE": + return "DATE"; + case "TIME": + return "TIME"; + case "DATETIME": + return "DATETIME"; + case "TIMESTAMP": + return "TIMESTAMP"; + case "YEAR": + return "YEAR"; + case "UUID": + return "CHAR(36)"; // MariaDB does not have a native UUID type + case "JSON": + return "JSON"; + case "INET6": + return "INET6"; + case "VECTOR": { + const dimensions = field.vectorSize || 1536; + return `VECTOR(${dimensions})`; + } + case "BOOLEAN": + return "TINYINT(1)"; + case "ENUM": { + const enumVals = (field.options || []) + .map((v) => `'${String(v).replace(/'/g, "''")}'`) + .join(", "); + return `ENUM(${enumVals || "''"})`; + } + case "SET": { + const setVals = (field.options || []) + .map((v) => `'${String(v).replace(/'/g, "''")}'`) + .join(", "); + return `SET(${setVals || "''"})`; + } + default: + return "TEXT"; + } +} diff --git a/src/lib/schema/mariadb-quote-gen.ts b/src/lib/schema/mariadb-quote-gen.ts new file mode 100644 index 0000000..bd7d104 --- /dev/null +++ b/src/lib/schema/mariadb-quote-gen.ts @@ -0,0 +1,3 @@ +export default function MariaDBQuoteGen(str: string) { + return `\`${str.replace(/`/g, "``")}\``; +} diff --git a/src/lib/schema/order-db-schema.ts b/src/lib/schema/order-db-schema.ts new file mode 100644 index 0000000..86b0c10 --- /dev/null +++ b/src/lib/schema/order-db-schema.ts @@ -0,0 +1,86 @@ +import _ from "lodash"; +import type { + BUN_MARIADB_DatabaseSchemaType, + BUN_MARIADB_TableSchemaType, + CreateDBSchemaParams, +} from "../../types"; + +export default async function orderDBSchema({ + db_schema, +}: CreateDBSchemaParams): Promise { + let new_db_schema = _.cloneDeep(db_schema); + + const tables = new_db_schema.tables; + + let new_tables_set = new Set(); + let new_tables_start_set = new Set(); + let new_tables_end_set = new Set(); + + function setParentTable(table: BUN_MARIADB_TableSchemaType) { + const fields = table.fields; + let does_table_have_foreign_keys = false; + + for (let f = 0; f < fields.length; f++) { + const field = fields[f]; + if (!field) continue; + + const dst_table_name = field.foreignKey?.destinationTableName; + + if (dst_table_name) { + const fk_table = tables.find( + (tb) => tb.tableName == dst_table_name, + ); + + if ( + fk_table && + !new_tables_start_set.has(fk_table) && + !new_tables_end_set.has(fk_table) + ) { + setParentTable(fk_table); + } + + new_tables_end_set.add(table); + + if (fk_table) { + new_tables_start_set.add(fk_table); + } + + does_table_have_foreign_keys = true; + } + } + + return { does_table_have_foreign_keys }; + } + + for (let i = 0; i < tables.length; i++) { + const table = tables[i]; + + if (!table) { + continue; + } + + let { does_table_have_foreign_keys } = setParentTable(table); + + if (does_table_have_foreign_keys) { + new_tables_end_set.add(table); + } else { + new_tables_start_set.add(table); + } + } + + const parsed_tables = [ + ...Array.from(new_tables_start_set), + ...Array.from(new_tables_end_set), + ]; + + for (let nt = 0; nt < parsed_tables.length; nt++) { + const new_table = parsed_tables[nt]; + if (new_table) { + new_tables_set.add(new_table); + } + } + + new_db_schema.tables = Array.from(new_tables_set); + + return new_db_schema; +} diff --git a/src/types/index.ts b/src/types/index.ts index 8f78b0e..b961da8 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -135,20 +135,49 @@ export const TextFieldTypesArray = [ * Native MariaDB column types supported by the schema builder. */ export const BUN_MARIADB_DATATYPES = [ + // Strings & Text + { value: "CHAR" }, { value: "VARCHAR" }, { value: "TEXT" }, + { value: "TINYTEXT" }, + { value: "MEDIUMTEXT" }, { value: "LONGTEXT" }, + + // Numeric Integers { value: "TINYINT" }, + { value: "SMALLINT" }, + { value: "MEDIUMINT" }, { value: "INT" }, { value: "BIGINT" }, - { value: "DECIMAL" }, + + // Numeric Floats & Exact + { value: "FLOAT" }, { value: "DOUBLE" }, + { value: "DECIMAL" }, + + // Binary Layouts + { value: "BINARY" }, + { value: "VARBINARY" }, { value: "BLOB" }, + { value: "TINYBLOB" }, + { value: "MEDIUMBLOB" }, { value: "LONGBLOB" }, - { value: "BOOLEAN" }, + + // Temporal (Dates / Times) + { value: "DATE" }, + { value: "TIME" }, { value: "DATETIME" }, { value: "TIMESTAMP" }, - { value: "DATE" }, + { value: "YEAR" }, + + // Core Schema Primitives + { value: "BOOLEAN" }, + { value: "UUID" }, + { value: "JSON" }, + { value: "INET6" }, + { value: "ENUM" }, + { value: "SET" }, + { value: "VECTOR" }, ] as const; /** @@ -1561,21 +1590,6 @@ export type BunMariaDBQueryFieldValues< export type QueryRawValueType = string | number | null | undefined; -export type DsqlConnectionParam = { - /** - * No Database Connection - */ - noDb?: boolean; - /** - * Database Name - */ - database?: string; - /** - * Debug - */ - config?: ConnectionConfig; -}; - export type DBResponseObject< T extends { [k: string]: any } = { [k: string]: any }, > = { @@ -1602,3 +1616,27 @@ export const RequiredENVs = [ "BUN_MARIADB_SERVER_PASSWORD", "BUN_MARIADB_SERVER_SSL_KEY_PATH", ] as const; + +export type CreateDBSchemaParams = { + db_schema: BUN_MARIADB_DatabaseSchemaType; + config?: BunMariaDBConfig; +}; + +export type CreateDBSchemaTableHandlerParams = CreateDBSchemaParams & { + table: BUN_MARIADB_TableSchemaType; + db_manager_table_name?: string; + existing_live_table?: BUN_MARIADB_INFORMATION_SCHEMA_TABLES; +}; + +export type BUN_MARIADB_DB_TABLE_MANAGER_TABLE = { + table_name?: string; + created_at?: number; + updated_at?: number; +}; + +export type BUN_MARIADB_INFORMATION_SCHEMA_TABLES = { + TABLE_NAME?: string; + TABLE_TYPE?: "BASE TABLE"; + ENGINE?: "InnoDB"; + VERSION?: number; +};