420 lines
18 KiB
JavaScript
420 lines
18 KiB
JavaScript
#!/usr/bin/env bun
|
|
import _ from "lodash";
|
|
import dbHandler from "../db-handler";
|
|
import { AppData } from "../../data/app-data";
|
|
import { readLiveSchema } from "../../functions/live-schema";
|
|
class MariaDBSchemaManager {
|
|
db_manager_table_name;
|
|
recreate_vector_table;
|
|
db_schema;
|
|
constructor({ schema, recreate_vector_table = false, }) {
|
|
this.db_manager_table_name = AppData["DbSchemaManagerTableName"];
|
|
this.recreate_vector_table = recreate_vector_table;
|
|
this.db_schema = schema;
|
|
}
|
|
async syncSchema() {
|
|
console.log("Starting schema synchronization...");
|
|
await this.createDbManagerTable();
|
|
const existingTables = await this.getExistingTables();
|
|
const schemaTables = this.db_schema.tables.map((t) => t.tableName);
|
|
for (const table of this.db_schema.tables) {
|
|
await this.syncTable(table, existingTables);
|
|
}
|
|
await this.dropRemovedTables(existingTables, schemaTables);
|
|
console.log("Schema synchronization complete!");
|
|
}
|
|
getDatabaseName() {
|
|
return this.db_schema.dbName || this.db_schema.dbSlug;
|
|
}
|
|
quoteIdentifier(identifier) {
|
|
return `\`${identifier.replace(/`/g, "``")}\``;
|
|
}
|
|
tableSchemaWhere(tableName) {
|
|
const databaseName = this.getDatabaseName();
|
|
if (databaseName) {
|
|
return {
|
|
where: "TABLE_SCHEMA = ?",
|
|
values: [databaseName, tableName],
|
|
};
|
|
}
|
|
return {
|
|
where: "TABLE_SCHEMA = DATABASE()",
|
|
values: [tableName],
|
|
};
|
|
}
|
|
async run(query, values) {
|
|
const res = await dbHandler({
|
|
query,
|
|
values: values,
|
|
config: this.getDatabaseName()
|
|
? { database: this.getDatabaseName() }
|
|
: undefined,
|
|
});
|
|
if (!res.success) {
|
|
throw new Error(`Database query failed: ${query}`);
|
|
}
|
|
}
|
|
async query(query, values) {
|
|
const res = await dbHandler({
|
|
query,
|
|
values: values,
|
|
config: this.getDatabaseName()
|
|
? { database: this.getDatabaseName() }
|
|
: undefined,
|
|
});
|
|
if (!res.success) {
|
|
throw new Error(`Database query failed: ${query}`);
|
|
}
|
|
return (res.payload || []);
|
|
}
|
|
async createDbManagerTable() {
|
|
await this.run(`
|
|
CREATE TABLE IF NOT EXISTS ${this.quoteIdentifier(this.db_manager_table_name)} (
|
|
table_name VARCHAR(255) NOT NULL PRIMARY KEY,
|
|
created_at BIGINT NOT NULL,
|
|
updated_at BIGINT NOT NULL
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci
|
|
`);
|
|
}
|
|
async insertDbManagerTable(tableName) {
|
|
const now = Date.now();
|
|
await this.run(`INSERT INTO ${this.quoteIdentifier(this.db_manager_table_name)} (table_name, created_at, updated_at) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = VALUES(updated_at)`, [tableName, now, now]);
|
|
}
|
|
async removeDbManagerTable(tableName) {
|
|
await this.run(`DELETE FROM ${this.quoteIdentifier(this.db_manager_table_name)} WHERE table_name = ?`, [tableName]);
|
|
}
|
|
async getExistingTables() {
|
|
const rows = await this.query(`SELECT table_name FROM ${this.quoteIdentifier(this.db_manager_table_name)}`);
|
|
return rows.map((row) => row.table_name);
|
|
}
|
|
async getLiveTableNames() {
|
|
const tableSchemaWhere = this.tableSchemaWhere("");
|
|
const rows = await this.query(`SELECT TABLE_NAME FROM information_schema.TABLES WHERE ${tableSchemaWhere.where} AND TABLE_TYPE = 'BASE TABLE'`, tableSchemaWhere.values);
|
|
return rows.map((row) => row.TABLE_NAME);
|
|
}
|
|
async dropRemovedTables(existingTables, schemaTables) {
|
|
console.log(`Cleaning up tables ...`);
|
|
const tablesToDrop = existingTables.filter((tableName) => !schemaTables.includes(tableName) &&
|
|
!schemaTables.some((schemaTable) => tableName.startsWith(`${schemaTable}_`)));
|
|
const currentSchema = readLiveSchema();
|
|
if (currentSchema?.tables?.[0]) {
|
|
for (const table of currentSchema.tables) {
|
|
if (!table?.tableName)
|
|
continue;
|
|
const doesTableExist = schemaTables.find((tableName) => tableName === table.tableName);
|
|
if (!doesTableExist) {
|
|
tablesToDrop.push(table.tableName);
|
|
}
|
|
}
|
|
}
|
|
for (const tableName of tablesToDrop) {
|
|
console.log(`Dropping table: ${tableName}`);
|
|
await this.run(`DROP TABLE IF EXISTS ${this.quoteIdentifier(tableName)}`);
|
|
await this.removeDbManagerTable(tableName);
|
|
}
|
|
}
|
|
async syncTable(table, existingTables) {
|
|
let tableExists = existingTables.includes(table.tableName);
|
|
const liveTables = await this.getLiveTableNames();
|
|
if (table.tableNameOld && table.tableNameOld !== table.tableName) {
|
|
if (liveTables.includes(table.tableNameOld)) {
|
|
console.log(`Renaming table: ${table.tableNameOld} -> ${table.tableName}`);
|
|
await this.run(`RENAME TABLE ${this.quoteIdentifier(table.tableNameOld)} TO ${this.quoteIdentifier(table.tableName)}`);
|
|
await this.insertDbManagerTable(table.tableName);
|
|
await this.removeDbManagerTable(table.tableNameOld);
|
|
tableExists = true;
|
|
}
|
|
}
|
|
if (!tableExists) {
|
|
await this.createTable(table);
|
|
await this.insertDbManagerTable(table.tableName);
|
|
}
|
|
else {
|
|
await this.updateTable(table);
|
|
await this.insertDbManagerTable(table.tableName);
|
|
}
|
|
await this.syncIndexes(table);
|
|
}
|
|
resolveTable(table) {
|
|
if (!table.parentTableName) {
|
|
return _.cloneDeep(table);
|
|
}
|
|
const parentTable = this.db_schema.tables.find((schemaTable) => schemaTable.tableName === table.parentTableName);
|
|
if (!parentTable) {
|
|
throw new Error(`Parent table \`${table.parentTableName}\` not found for \`${table.tableName}\``);
|
|
}
|
|
return _.merge({}, parentTable, {
|
|
tableName: table.tableName,
|
|
tableDescription: table.tableDescription,
|
|
collation: table.collation,
|
|
fields: [...(parentTable.fields || []), ...(table.fields || [])],
|
|
indexes: [...(parentTable.indexes || []), ...(table.indexes || [])],
|
|
uniqueConstraints: [
|
|
...(parentTable.uniqueConstraints || []),
|
|
...(table.uniqueConstraints || []),
|
|
],
|
|
});
|
|
}
|
|
async createTable(table) {
|
|
if (!table.tableName.match(/_temp_\d+$/)) {
|
|
console.log(`Creating table: ${table.tableName}`);
|
|
}
|
|
const newTable = this.resolveTable(table);
|
|
const columnDefinitions = [];
|
|
const foreignKeys = [];
|
|
for (const field of newTable.fields || []) {
|
|
columnDefinitions.push(this.buildColumnDefinition(field));
|
|
if (field.foreignKey && !newTable.isVector) {
|
|
foreignKeys.push(this.buildForeignKeyConstraint(field));
|
|
}
|
|
}
|
|
if (newTable.uniqueConstraints) {
|
|
for (const constraint of newTable.uniqueConstraints) {
|
|
if (constraint.constraintTableFields &&
|
|
constraint.constraintTableFields.length > 0) {
|
|
const fields = constraint.constraintTableFields
|
|
.map((field) => this.quoteIdentifier(field.value))
|
|
.join(", ");
|
|
const constraintName = constraint.constraintName ||
|
|
`unique_${fields.replace(/`/g, "")}`;
|
|
columnDefinitions.push(`CONSTRAINT ${this.quoteIdentifier(constraintName)} UNIQUE (${fields})`);
|
|
}
|
|
}
|
|
}
|
|
const sql = `CREATE TABLE IF NOT EXISTS ${this.quoteIdentifier(newTable.tableName)} (${[...columnDefinitions, ...foreignKeys].join(", ")})${this.buildTableOptions(newTable)}`;
|
|
await this.run(sql);
|
|
}
|
|
buildTableOptions(table) {
|
|
const options = ["ENGINE=InnoDB"];
|
|
if (table.collation) {
|
|
options.push("DEFAULT CHARSET=utf8mb4", `COLLATE ${table.collation}`);
|
|
}
|
|
return ` ${options.join(" ")}`;
|
|
}
|
|
async updateTable(table) {
|
|
console.log(`Updating table: ${table.tableName}`);
|
|
await this.recreateTable(table);
|
|
}
|
|
async getTableColumns(tableName) {
|
|
const tableSchemaWhere = this.tableSchemaWhere(tableName);
|
|
const rows = await this.query(`SELECT COLUMN_NAME AS name, COLUMN_TYPE AS type FROM information_schema.COLUMNS WHERE ${tableSchemaWhere.where} AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION`, tableSchemaWhere.values);
|
|
return rows.map((row) => ({
|
|
name: row.COLUMN_NAME,
|
|
type: row.COLUMN_TYPE,
|
|
}));
|
|
}
|
|
async addColumn(tableName, field) {
|
|
console.log(`Adding column: ${tableName}.${field.fieldName}`);
|
|
const columnDef = this.buildColumnDefinition(field)
|
|
.replace(/PRIMARY KEY/gi, "")
|
|
.replace(/AUTO_INCREMENT/gi, "")
|
|
.replace(/UNIQUE/gi, "")
|
|
.trim();
|
|
await this.run(`ALTER TABLE ${this.quoteIdentifier(tableName)} ADD COLUMN ${columnDef}`);
|
|
}
|
|
async checkIfTableExists(table) {
|
|
const tableSchemaWhere = this.tableSchemaWhere(table);
|
|
const row = await this.query(`SELECT 1 AS exists FROM information_schema.TABLES WHERE ${tableSchemaWhere.where} AND TABLE_NAME = ? LIMIT 1`, tableSchemaWhere.values);
|
|
return Boolean(row[0]?.exists);
|
|
}
|
|
async recreateTable(table) {
|
|
if (table.isVector && !this.recreate_vector_table) {
|
|
return;
|
|
}
|
|
const doesTableExist = await this.checkIfTableExists(table.tableName);
|
|
if (table.isVector) {
|
|
let existingRows = [];
|
|
if (doesTableExist) {
|
|
existingRows = await this.query(`SELECT * FROM ${this.quoteIdentifier(table.tableName)}`);
|
|
await this.run(`DROP TABLE ${this.quoteIdentifier(table.tableName)}`);
|
|
}
|
|
await this.createTable(table);
|
|
if (existingRows.length > 0) {
|
|
await this.insertRows(table.tableName, existingRows);
|
|
}
|
|
return;
|
|
}
|
|
const tempTableName = `${table.tableName}_temp_${Date.now()}`;
|
|
const existingColumns = await this.getTableColumns(table.tableName);
|
|
const columnsToKeep = table.fields
|
|
.filter((field) => existingColumns.some((column) => column.name === field.fieldName))
|
|
.map((field) => field.fieldName)
|
|
.filter((fieldName) => Boolean(fieldName));
|
|
await this.createTable({ ...table, tableName: tempTableName });
|
|
if (columnsToKeep.length > 0) {
|
|
const columnList = columnsToKeep
|
|
.map((column) => this.quoteIdentifier(column))
|
|
.join(", ");
|
|
await this.run(`INSERT INTO ${this.quoteIdentifier(tempTableName)} (${columnList}) SELECT ${columnList} FROM ${this.quoteIdentifier(table.tableName)}`);
|
|
}
|
|
await this.run(`DROP TABLE ${this.quoteIdentifier(table.tableName)}`);
|
|
await this.run(`RENAME TABLE ${this.quoteIdentifier(tempTableName)} TO ${this.quoteIdentifier(table.tableName)}`);
|
|
}
|
|
async insertRows(tableName, rows) {
|
|
for (const row of rows) {
|
|
const columns = Object.keys(row);
|
|
if (columns.length === 0) {
|
|
continue;
|
|
}
|
|
const values = columns.map((column) => row[column]);
|
|
const columnList = columns
|
|
.map((column) => this.quoteIdentifier(column))
|
|
.join(", ");
|
|
const placeholders = columns.map(() => "?").join(", ");
|
|
await this.run(`INSERT INTO ${this.quoteIdentifier(tableName)} (${columnList}) VALUES (${placeholders})`, values);
|
|
}
|
|
}
|
|
buildColumnDefinition(field) {
|
|
if (!field.fieldName) {
|
|
throw new Error("Field name is required");
|
|
}
|
|
const parts = [this.quoteIdentifier(field.fieldName)];
|
|
parts.push(this.mapDataType(field));
|
|
if (field.primaryKey) {
|
|
parts.push("PRIMARY KEY");
|
|
if (field.autoIncrement) {
|
|
parts.push("AUTO_INCREMENT");
|
|
}
|
|
}
|
|
if (field.notNullValue || field.primaryKey) {
|
|
if (!field.primaryKey) {
|
|
parts.push("NOT NULL");
|
|
}
|
|
}
|
|
if (field.unique && !field.primaryKey) {
|
|
parts.push("UNIQUE");
|
|
}
|
|
if (field.defaultValue !== undefined) {
|
|
if (typeof field.defaultValue === "string") {
|
|
parts.push(`DEFAULT '${field.defaultValue.replace(/'/g, "''")}'`);
|
|
}
|
|
else {
|
|
parts.push(`DEFAULT ${field.defaultValue}`);
|
|
}
|
|
}
|
|
else if (field.defaultValueLiteral) {
|
|
parts.push(`DEFAULT ${field.defaultValueLiteral}`);
|
|
}
|
|
if (field.onUpdate) {
|
|
parts.push(`ON UPDATE ${field.onUpdate}`);
|
|
}
|
|
else if (field.onUpdateLiteral) {
|
|
parts.push(`ON UPDATE ${field.onUpdateLiteral}`);
|
|
}
|
|
return parts.join(" ");
|
|
}
|
|
mapDataType(field) {
|
|
const dataType = field.dataType?.toLowerCase() || "text";
|
|
const vectorSize = field.vectorSize || 1536;
|
|
if (field.isVector) {
|
|
return `LONGTEXT COMMENT 'vector_size=${vectorSize}'`;
|
|
}
|
|
if (dataType.includes("int") ||
|
|
dataType === "bigint" ||
|
|
dataType === "smallint" ||
|
|
dataType === "tinyint") {
|
|
if (field.integerLength) {
|
|
return `INT(${field.integerLength})`;
|
|
}
|
|
return "INT";
|
|
}
|
|
if (dataType === "bigint") {
|
|
if (field.integerLength) {
|
|
return `BIGINT(${field.integerLength})`;
|
|
}
|
|
return "BIGINT";
|
|
}
|
|
if (dataType === "smallint") {
|
|
if (field.integerLength) {
|
|
return `SMALLINT(${field.integerLength})`;
|
|
}
|
|
return "SMALLINT";
|
|
}
|
|
if (dataType === "tinyint") {
|
|
if (field.integerLength) {
|
|
return `TINYINT(${field.integerLength})`;
|
|
}
|
|
return "TINYINT";
|
|
}
|
|
if (dataType.includes("double")) {
|
|
return "DOUBLE";
|
|
}
|
|
if (dataType.includes("float")) {
|
|
return "FLOAT";
|
|
}
|
|
if (dataType.includes("decimal") || dataType.includes("numeric")) {
|
|
if (field.integerLength && field.decimals) {
|
|
return `DECIMAL(${field.integerLength}, ${field.decimals})`;
|
|
}
|
|
return "DECIMAL";
|
|
}
|
|
if (dataType.includes("blob") || dataType.includes("binary")) {
|
|
return "BLOB";
|
|
}
|
|
if (dataType === "boolean" || dataType === "bool") {
|
|
return "TINYINT(1)";
|
|
}
|
|
if (dataType.includes("timestamp")) {
|
|
return "TIMESTAMP";
|
|
}
|
|
if (dataType.includes("datetime")) {
|
|
return "DATETIME";
|
|
}
|
|
if (dataType.includes("date") || dataType.includes("time")) {
|
|
return "DATETIME";
|
|
}
|
|
if (dataType.includes("varchar")) {
|
|
if (field.integerLength) {
|
|
return `VARCHAR(${field.integerLength})`;
|
|
}
|
|
return "VARCHAR(255)";
|
|
}
|
|
return "TEXT";
|
|
}
|
|
buildForeignKeyConstraint(field) {
|
|
const fk = field.foreignKey;
|
|
const constraintName = fk.foreignKeyName
|
|
? `CONSTRAINT ${this.quoteIdentifier(fk.foreignKeyName)} `
|
|
: "";
|
|
let constraint = `${constraintName}FOREIGN KEY (${this.quoteIdentifier(field.fieldName)}) REFERENCES ${this.quoteIdentifier(fk.destinationTableName)}(${this.quoteIdentifier(fk.destinationTableColumnName)})`;
|
|
if (fk.cascadeDelete) {
|
|
constraint += " ON DELETE CASCADE";
|
|
}
|
|
if (fk.cascadeUpdate) {
|
|
constraint += " ON UPDATE CASCADE";
|
|
}
|
|
return constraint;
|
|
}
|
|
async syncIndexes(table) {
|
|
if (!table.indexes || table.indexes.length === 0) {
|
|
return;
|
|
}
|
|
const tableSchemaWhere = this.tableSchemaWhere(table.tableName);
|
|
const rows = await this.query(`SELECT INDEX_NAME AS name FROM information_schema.STATISTICS WHERE ${tableSchemaWhere.where} AND TABLE_NAME = ? AND INDEX_NAME <> 'PRIMARY' GROUP BY INDEX_NAME ORDER BY INDEX_NAME`, tableSchemaWhere.values);
|
|
const existingIndexes = rows.map((row) => row.name);
|
|
for (const indexName of existingIndexes) {
|
|
const stillExists = table.indexes.some((index) => index.indexName === indexName);
|
|
if (!stillExists) {
|
|
console.log(`Dropping index: ${indexName}`);
|
|
await this.run(`DROP INDEX ${this.quoteIdentifier(indexName)} ON ${this.quoteIdentifier(table.tableName)}`);
|
|
}
|
|
}
|
|
for (const index of table.indexes) {
|
|
if (!index.indexName ||
|
|
!index.indexTableFields ||
|
|
index.indexTableFields.length === 0) {
|
|
continue;
|
|
}
|
|
if (!existingIndexes.includes(index.indexName)) {
|
|
console.log(`Creating index: ${index.indexName}`);
|
|
const fields = index.indexTableFields
|
|
.map((field) => this.quoteIdentifier(field))
|
|
.join(", ");
|
|
await this.run(`CREATE INDEX ${this.quoteIdentifier(index.indexName)} ON ${this.quoteIdentifier(table.tableName)} (${fields})`);
|
|
}
|
|
}
|
|
}
|
|
close() { }
|
|
}
|
|
export { MariaDBSchemaManager };
|