#!/usr/bin/env bun import _ from "lodash"; import dbHandler from "../db-handler"; import { AppData } from "../../data/app-data"; import { readLiveSchema } from "../../functions/live-schema"; class MariaDBSchemaManager { db_manager_table_name; recreate_vector_table; db_schema; constructor({ schema, recreate_vector_table = false, }) { this.db_manager_table_name = AppData["DbSchemaManagerTableName"]; this.recreate_vector_table = recreate_vector_table; this.db_schema = schema; } async syncSchema() { console.log("Starting schema synchronization..."); await this.createDbManagerTable(); const existingTables = await this.getExistingTables(); const schemaTables = this.db_schema.tables.map((t) => t.tableName); for (const table of this.db_schema.tables) { await this.syncTable(table, existingTables); } await this.dropRemovedTables(existingTables, schemaTables); console.log("Schema synchronization complete!"); } getDatabaseName() { return this.db_schema.dbName || this.db_schema.dbSlug; } quoteIdentifier(identifier) { return `\`${identifier.replace(/`/g, "``")}\``; } tableSchemaWhere(tableName) { const databaseName = this.getDatabaseName(); if (databaseName) { return { where: "TABLE_SCHEMA = ?", values: [databaseName, tableName], }; } return { where: "TABLE_SCHEMA = DATABASE()", values: [tableName], }; } async run(query, values) { const res = await dbHandler({ query, values: values, config: this.getDatabaseName() ? { database: this.getDatabaseName() } : undefined, }); if (!res.success) { throw new Error(`Database query failed: ${query}`); } } async query(query, values) { const res = await dbHandler({ query, values: values, config: this.getDatabaseName() ? { database: this.getDatabaseName() } : undefined, }); if (!res.success) { throw new Error(`Database query failed: ${query}`); } return (res.payload || []); } async createDbManagerTable() { await this.run(` CREATE TABLE IF NOT EXISTS ${this.quoteIdentifier(this.db_manager_table_name)} ( table_name VARCHAR(255) NOT NULL PRIMARY KEY, created_at BIGINT NOT NULL, updated_at BIGINT NOT NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci `); } async insertDbManagerTable(tableName) { const now = Date.now(); await this.run(`INSERT INTO ${this.quoteIdentifier(this.db_manager_table_name)} (table_name, created_at, updated_at) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = VALUES(updated_at)`, [tableName, now, now]); } async removeDbManagerTable(tableName) { await this.run(`DELETE FROM ${this.quoteIdentifier(this.db_manager_table_name)} WHERE table_name = ?`, [tableName]); } async getExistingTables() { const rows = await this.query(`SELECT table_name FROM ${this.quoteIdentifier(this.db_manager_table_name)}`); return rows.map((row) => row.table_name); } async getLiveTableNames() { const tableSchemaWhere = this.tableSchemaWhere(""); const rows = await this.query(`SELECT TABLE_NAME FROM information_schema.TABLES WHERE ${tableSchemaWhere.where} AND TABLE_TYPE = 'BASE TABLE'`, tableSchemaWhere.values); return rows.map((row) => row.TABLE_NAME); } async dropRemovedTables(existingTables, schemaTables) { console.log(`Cleaning up tables ...`); const tablesToDrop = existingTables.filter((tableName) => !schemaTables.includes(tableName) && !schemaTables.some((schemaTable) => tableName.startsWith(`${schemaTable}_`))); const currentSchema = readLiveSchema(); if (currentSchema?.tables?.[0]) { for (const table of currentSchema.tables) { if (!table?.tableName) continue; const doesTableExist = schemaTables.find((tableName) => tableName === table.tableName); if (!doesTableExist) { tablesToDrop.push(table.tableName); } } } for (const tableName of tablesToDrop) { console.log(`Dropping table: ${tableName}`); await this.run(`DROP TABLE IF EXISTS ${this.quoteIdentifier(tableName)}`); await this.removeDbManagerTable(tableName); } } async syncTable(table, existingTables) { let tableExists = existingTables.includes(table.tableName); const liveTables = await this.getLiveTableNames(); if (table.tableNameOld && table.tableNameOld !== table.tableName) { if (liveTables.includes(table.tableNameOld)) { console.log(`Renaming table: ${table.tableNameOld} -> ${table.tableName}`); await this.run(`RENAME TABLE ${this.quoteIdentifier(table.tableNameOld)} TO ${this.quoteIdentifier(table.tableName)}`); await this.insertDbManagerTable(table.tableName); await this.removeDbManagerTable(table.tableNameOld); tableExists = true; } } if (!tableExists) { await this.createTable(table); await this.insertDbManagerTable(table.tableName); } else { await this.updateTable(table); await this.insertDbManagerTable(table.tableName); } await this.syncIndexes(table); } resolveTable(table) { if (!table.parentTableName) { return _.cloneDeep(table); } const parentTable = this.db_schema.tables.find((schemaTable) => schemaTable.tableName === table.parentTableName); if (!parentTable) { throw new Error(`Parent table \`${table.parentTableName}\` not found for \`${table.tableName}\``); } return _.merge({}, parentTable, { tableName: table.tableName, tableDescription: table.tableDescription, collation: table.collation, fields: [...(parentTable.fields || []), ...(table.fields || [])], indexes: [...(parentTable.indexes || []), ...(table.indexes || [])], uniqueConstraints: [ ...(parentTable.uniqueConstraints || []), ...(table.uniqueConstraints || []), ], }); } async createTable(table) { if (!table.tableName.match(/_temp_\d+$/)) { console.log(`Creating table: ${table.tableName}`); } const newTable = this.resolveTable(table); const columnDefinitions = []; const foreignKeys = []; for (const field of newTable.fields || []) { columnDefinitions.push(this.buildColumnDefinition(field)); if (field.foreignKey && !newTable.isVector) { foreignKeys.push(this.buildForeignKeyConstraint(field)); } } if (newTable.uniqueConstraints) { for (const constraint of newTable.uniqueConstraints) { if (constraint.constraintTableFields && constraint.constraintTableFields.length > 0) { const fields = constraint.constraintTableFields .map((field) => this.quoteIdentifier(field.value)) .join(", "); const constraintName = constraint.constraintName || `unique_${fields.replace(/`/g, "")}`; columnDefinitions.push(`CONSTRAINT ${this.quoteIdentifier(constraintName)} UNIQUE (${fields})`); } } } const sql = `CREATE TABLE IF NOT EXISTS ${this.quoteIdentifier(newTable.tableName)} (${[...columnDefinitions, ...foreignKeys].join(", ")})${this.buildTableOptions(newTable)}`; await this.run(sql); } buildTableOptions(table) { const options = ["ENGINE=InnoDB"]; if (table.collation) { options.push("DEFAULT CHARSET=utf8mb4", `COLLATE ${table.collation}`); } return ` ${options.join(" ")}`; } async updateTable(table) { console.log(`Updating table: ${table.tableName}`); await this.recreateTable(table); } async getTableColumns(tableName) { const tableSchemaWhere = this.tableSchemaWhere(tableName); const rows = await this.query(`SELECT COLUMN_NAME AS name, COLUMN_TYPE AS type FROM information_schema.COLUMNS WHERE ${tableSchemaWhere.where} AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION`, tableSchemaWhere.values); return rows.map((row) => ({ name: row.COLUMN_NAME, type: row.COLUMN_TYPE, })); } async addColumn(tableName, field) { console.log(`Adding column: ${tableName}.${field.fieldName}`); const columnDef = this.buildColumnDefinition(field) .replace(/PRIMARY KEY/gi, "") .replace(/AUTO_INCREMENT/gi, "") .replace(/UNIQUE/gi, "") .trim(); await this.run(`ALTER TABLE ${this.quoteIdentifier(tableName)} ADD COLUMN ${columnDef}`); } async checkIfTableExists(table) { const tableSchemaWhere = this.tableSchemaWhere(table); const row = await this.query(`SELECT 1 AS exists FROM information_schema.TABLES WHERE ${tableSchemaWhere.where} AND TABLE_NAME = ? LIMIT 1`, tableSchemaWhere.values); return Boolean(row[0]?.exists); } async recreateTable(table) { if (table.isVector && !this.recreate_vector_table) { return; } const doesTableExist = await this.checkIfTableExists(table.tableName); if (table.isVector) { let existingRows = []; if (doesTableExist) { existingRows = await this.query(`SELECT * FROM ${this.quoteIdentifier(table.tableName)}`); await this.run(`DROP TABLE ${this.quoteIdentifier(table.tableName)}`); } await this.createTable(table); if (existingRows.length > 0) { await this.insertRows(table.tableName, existingRows); } return; } const tempTableName = `${table.tableName}_temp_${Date.now()}`; const existingColumns = await this.getTableColumns(table.tableName); const columnsToKeep = table.fields .filter((field) => existingColumns.some((column) => column.name === field.fieldName)) .map((field) => field.fieldName) .filter((fieldName) => Boolean(fieldName)); await this.createTable({ ...table, tableName: tempTableName }); if (columnsToKeep.length > 0) { const columnList = columnsToKeep .map((column) => this.quoteIdentifier(column)) .join(", "); await this.run(`INSERT INTO ${this.quoteIdentifier(tempTableName)} (${columnList}) SELECT ${columnList} FROM ${this.quoteIdentifier(table.tableName)}`); } await this.run(`DROP TABLE ${this.quoteIdentifier(table.tableName)}`); await this.run(`RENAME TABLE ${this.quoteIdentifier(tempTableName)} TO ${this.quoteIdentifier(table.tableName)}`); } async insertRows(tableName, rows) { for (const row of rows) { const columns = Object.keys(row); if (columns.length === 0) { continue; } const values = columns.map((column) => row[column]); const columnList = columns .map((column) => this.quoteIdentifier(column)) .join(", "); const placeholders = columns.map(() => "?").join(", "); await this.run(`INSERT INTO ${this.quoteIdentifier(tableName)} (${columnList}) VALUES (${placeholders})`, values); } } buildColumnDefinition(field) { if (!field.fieldName) { throw new Error("Field name is required"); } const parts = [this.quoteIdentifier(field.fieldName)]; parts.push(this.mapDataType(field)); if (field.primaryKey) { parts.push("PRIMARY KEY"); if (field.autoIncrement) { parts.push("AUTO_INCREMENT"); } } if (field.notNullValue || field.primaryKey) { if (!field.primaryKey) { parts.push("NOT NULL"); } } if (field.unique && !field.primaryKey) { parts.push("UNIQUE"); } if (field.defaultValue !== undefined) { if (typeof field.defaultValue === "string") { parts.push(`DEFAULT '${field.defaultValue.replace(/'/g, "''")}'`); } else { parts.push(`DEFAULT ${field.defaultValue}`); } } else if (field.defaultValueLiteral) { parts.push(`DEFAULT ${field.defaultValueLiteral}`); } if (field.onUpdate) { parts.push(`ON UPDATE ${field.onUpdate}`); } else if (field.onUpdateLiteral) { parts.push(`ON UPDATE ${field.onUpdateLiteral}`); } return parts.join(" "); } mapDataType(field) { const dataType = field.dataType?.toLowerCase() || "text"; const vectorSize = field.vectorSize || 1536; if (field.isVector) { return `LONGTEXT COMMENT 'vector_size=${vectorSize}'`; } if (dataType.includes("int") || dataType === "bigint" || dataType === "smallint" || dataType === "tinyint") { if (field.integerLength) { return `INT(${field.integerLength})`; } return "INT"; } if (dataType === "bigint") { if (field.integerLength) { return `BIGINT(${field.integerLength})`; } return "BIGINT"; } if (dataType === "smallint") { if (field.integerLength) { return `SMALLINT(${field.integerLength})`; } return "SMALLINT"; } if (dataType === "tinyint") { if (field.integerLength) { return `TINYINT(${field.integerLength})`; } return "TINYINT"; } if (dataType.includes("double")) { return "DOUBLE"; } if (dataType.includes("float")) { return "FLOAT"; } if (dataType.includes("decimal") || dataType.includes("numeric")) { if (field.integerLength && field.decimals) { return `DECIMAL(${field.integerLength}, ${field.decimals})`; } return "DECIMAL"; } if (dataType.includes("blob") || dataType.includes("binary")) { return "BLOB"; } if (dataType === "boolean" || dataType === "bool") { return "TINYINT(1)"; } if (dataType.includes("timestamp")) { return "TIMESTAMP"; } if (dataType.includes("datetime")) { return "DATETIME"; } if (dataType.includes("date") || dataType.includes("time")) { return "DATETIME"; } if (dataType.includes("varchar")) { if (field.integerLength) { return `VARCHAR(${field.integerLength})`; } return "VARCHAR(255)"; } return "TEXT"; } buildForeignKeyConstraint(field) { const fk = field.foreignKey; const constraintName = fk.foreignKeyName ? `CONSTRAINT ${this.quoteIdentifier(fk.foreignKeyName)} ` : ""; let constraint = `${constraintName}FOREIGN KEY (${this.quoteIdentifier(field.fieldName)}) REFERENCES ${this.quoteIdentifier(fk.destinationTableName)}(${this.quoteIdentifier(fk.destinationTableColumnName)})`; if (fk.cascadeDelete) { constraint += " ON DELETE CASCADE"; } if (fk.cascadeUpdate) { constraint += " ON UPDATE CASCADE"; } return constraint; } async syncIndexes(table) { if (!table.indexes || table.indexes.length === 0) { return; } const tableSchemaWhere = this.tableSchemaWhere(table.tableName); const rows = await this.query(`SELECT INDEX_NAME AS name FROM information_schema.STATISTICS WHERE ${tableSchemaWhere.where} AND TABLE_NAME = ? AND INDEX_NAME <> 'PRIMARY' GROUP BY INDEX_NAME ORDER BY INDEX_NAME`, tableSchemaWhere.values); const existingIndexes = rows.map((row) => row.name); for (const indexName of existingIndexes) { const stillExists = table.indexes.some((index) => index.indexName === indexName); if (!stillExists) { console.log(`Dropping index: ${indexName}`); await this.run(`DROP INDEX ${this.quoteIdentifier(indexName)} ON ${this.quoteIdentifier(table.tableName)}`); } } for (const index of table.indexes) { if (!index.indexName || !index.indexTableFields || index.indexTableFields.length === 0) { continue; } if (!existingIndexes.includes(index.indexName)) { console.log(`Creating index: ${index.indexName}`); const fields = index.indexTableFields .map((field) => this.quoteIdentifier(field)) .join(", "); await this.run(`CREATE INDEX ${this.quoteIdentifier(index.indexName)} ON ${this.quoteIdentifier(table.tableName)} (${fields})`); } } } close() { } } export { MariaDBSchemaManager };