From b5e4724e03726189630db1bc101dc80aa332e986 Mon Sep 17 00:00:00 2001 From: Benjamin Toby Date: Wed, 24 Jun 2026 17:00:14 +0100 Subject: [PATCH] First working version. --- src/commands/schema.ts | 6 +-- src/lib/db-handler.ts | 6 ++- src/lib/mariadb/db-delete.ts | 17 +++---- src/lib/mariadb/db-insert.ts | 20 ++++---- src/lib/mariadb/db-schema-manager.ts | 75 ++++++++++------------------ src/lib/mariadb/db-select.ts | 8 +-- src/lib/mariadb/db-sql.ts | 14 ++---- src/lib/mariadb/db-update.ts | 55 +++++++++++++------- src/types/index.ts | 42 +--------------- 9 files changed, 92 insertions(+), 151 deletions(-) diff --git a/src/commands/schema.ts b/src/commands/schema.ts index e543a59..74cf507 100644 --- a/src/commands/schema.ts +++ b/src/commands/schema.ts @@ -7,8 +7,6 @@ import _ from "lodash"; import appendDefaultFieldsToDbSchema from "../utils/append-default-fields-to-db-schema"; import chalk from "chalk"; import { writeLiveSchema } from "../functions/live-schema"; -import grabDBDir from "../utils/grab-db-dir"; -import { cpSync } from "fs"; export default function () { return new Command("schema") @@ -24,10 +22,9 @@ export default function () { const config = global.CONFIG; const dbSchema = global.DB_SCHEMA; - const { ROOT_DIR, BUN_MARIADB_TEMP_DB_FILE_PATH } = grabDirNames(); + const { ROOT_DIR } = grabDirNames(); try { - const isVector = Boolean(opts.vector || opts.v); const isTypeDef = Boolean(opts.typedef || opts.t); const finaldbSchema = appendDefaultFieldsToDbSchema({ @@ -36,7 +33,6 @@ export default function () { const manager = new MariaDBSchemaManager({ schema: finaldbSchema, - recreate_vector_table: isVector, }); await manager.syncSchema(); diff --git a/src/lib/db-handler.ts b/src/lib/db-handler.ts index 84fdf30..ccbe9a0 100644 --- a/src/lib/db-handler.ts +++ b/src/lib/db-handler.ts @@ -17,8 +17,6 @@ export default async function dbHandler< const res = await MariaDBClient.unsafe(query, values); const count = res.count; - const last_insert_id = res.lastInsertRowid; - const affected_rows = res.affectedRows; const res_array = (() => { try { @@ -28,6 +26,9 @@ export default async function dbHandler< } })(); + const last_insert_id = res_array?.[0] ? res_array[0]?.id : undefined; + const affected_rows = res_array?.[0] ? res_array.length : undefined; + const insert_return: DBInsertReturn = { count, last_insert_id, @@ -39,6 +40,7 @@ export default async function dbHandler< payload: res_array, single_res: res_array?.[0], insert_return, + count, }; } catch (error: any) { return { diff --git a/src/lib/mariadb/db-delete.ts b/src/lib/mariadb/db-delete.ts index 0c143f5..c6fc0b4 100644 --- a/src/lib/mariadb/db-delete.ts +++ b/src/lib/mariadb/db-delete.ts @@ -1,6 +1,6 @@ import dbHandler from "../db-handler"; import _ from "lodash"; -import type { APIResponseObject, ServerQueryParam } from "../../types"; +import type { DBResponseObject, ServerQueryParam } from "../../types"; import sqlGenerator from "../../utils/sql-generator"; type Params< @@ -23,7 +23,7 @@ export default async function DbDelete< table, query, targetId, -}: Params): Promise { +}: Params): Promise { let sqlObj: ReturnType | null = null; try { @@ -59,8 +59,8 @@ export default async function DbDelete< }; } - const sql = `DELETE FROM ${quoteIdentifier(table)} ${whereClause}`; - + let sql = `DELETE FROM ${quoteIdentifier(table)} ${whereClause}`; + sql += ` RETURNING *`; sqlObj.string = sql; const res = await dbHandler({ @@ -78,14 +78,9 @@ export default async function DbDelete< }; } - const singleRes = res.single_res as any; - return { - success: Boolean(singleRes?.affectedRows), - postInsertReturn: { - affectedRows: Number(singleRes?.affectedRows), - insertId: Number(singleRes?.insertId), - }, + ..._.omit(res, ["payload"]), + success: Boolean(res.insert_return?.last_insert_id), debug: { sqlObj, }, diff --git a/src/lib/mariadb/db-insert.ts b/src/lib/mariadb/db-insert.ts index 94c9eb1..64fcb99 100644 --- a/src/lib/mariadb/db-insert.ts +++ b/src/lib/mariadb/db-insert.ts @@ -1,5 +1,5 @@ import dbHandler from "../db-handler"; -import type { APIResponseObject, SQLInsertGenReturn } from "../../types"; +import type { DBResponseObject, SQLInsertGenReturn } from "../../types"; import sqlInsertGenerator from "../../utils/sql-insert-generator"; import grabDuplicateSafeInsertSql from "../grab-duplicate-safe-insert-sql"; @@ -19,7 +19,7 @@ export default async function DbInsert< table, data, update_on_duplicate, -}: Params): Promise { +}: Params): Promise { let sqlObj: SQLInsertGenReturn | null = null; try { @@ -48,9 +48,11 @@ export default async function DbInsert< sql = await grabDuplicateSafeInsertSql({ data, table, sql }); } + sql += ` RETURNING *`; + const res = await dbHandler({ query: sql, - values: sqlObj.values as any, + values: sqlObj.values, }); if (!res.success) { @@ -63,14 +65,12 @@ export default async function DbInsert< }; } - const singleRes = res.single_res as any; - return { - success: Boolean(singleRes?.affectedRows || singleRes?.insertId), - postInsertReturn: { - affectedRows: Number(singleRes?.affectedRows), - insertId: Number(singleRes?.insertId), - }, + ...res, + success: Boolean( + res?.insert_return?.affected_rows || + res.insert_return?.last_insert_id, + ), debug: { sqlObj, }, diff --git a/src/lib/mariadb/db-schema-manager.ts b/src/lib/mariadb/db-schema-manager.ts index 4670bde..e672c04 100644 --- a/src/lib/mariadb/db-schema-manager.ts +++ b/src/lib/mariadb/db-schema-manager.ts @@ -19,6 +19,7 @@ type TableInfoRow = { type ColumnInfoRow = { name: string; type: string; + comment?: string; }; type IndexInfoRow = { @@ -93,9 +94,6 @@ class MariaDBSchemaManager { }); if (!res.success) { - console.log("res", res); - console.log("query", query); - console.log("values", values); throw new Error(`Database query failed: ${query}`); } @@ -343,9 +341,18 @@ class MariaDBSchemaManager { if (!current) return false; const resolvedType = this.mapDataType(field).toLowerCase(); - return !resolvedType.startsWith(current.type.toLowerCase()); + const currentType = current.type.toLowerCase(); + + // If it's a vector column, compare the explicit metadata size inside the column comment fields + if (field.isVector || resolvedTable.isVector) { + const targetComment = `vector_size=${field.vectorSize || 1536}`; + return current.comment !== targetComment; + } + + return !resolvedType.startsWith(currentType); }); + // Fast path: Only adding new columns without metadata alterations or vector shifts if ( missingFields.length > 0 && !hasModifiedFields && @@ -357,7 +364,14 @@ class MariaDBSchemaManager { return; } - await this.recreateTable(resolvedTable); + // If changes were explicitly made or vector sizing changed, execute safe zero-data-loss rebuild + if ( + hasModifiedFields || + missingFields.length > 0 || + resolvedTable.isVector + ) { + await this.recreateTable(resolvedTable); + } } private async getTableColumns(tableName: string): Promise { @@ -365,14 +379,16 @@ class MariaDBSchemaManager { const rows = await this.query<{ COLUMN_NAME: string; COLUMN_TYPE: string; + COLUMN_COMMENT: string; }>( - `SELECT COLUMN_NAME, COLUMN_TYPE FROM information_schema.COLUMNS WHERE ${schemaCond.where} AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION`, + `SELECT COLUMN_NAME, COLUMN_TYPE, COLUMN_COMMENT FROM information_schema.COLUMNS WHERE ${schemaCond.where} AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION`, [...schemaCond.values, tableName], ); return rows.map((row) => ({ name: row.COLUMN_NAME, type: row.COLUMN_TYPE, + comment: row.COLUMN_COMMENT, })); } @@ -392,6 +408,7 @@ class MariaDBSchemaManager { `ALTER TABLE ${this.quoteIdentifier(tableName)} ADD COLUMN ${columnDef}`, ); } + private async checkIfTableExists(table: string): Promise { const schemaCond = this.schemaCondition(); const row = await this.query<{ table_exists: number }>( @@ -401,29 +418,13 @@ class MariaDBSchemaManager { return Boolean(row[0]?.table_exists); } + private async recreateTable( table: BUN_MARIADB_TableSchemaType, ): Promise { const doesTableExist = await this.checkIfTableExists(table.tableName); - - if (table.isVector) { - let existingRows: Record[] = []; - - if (doesTableExist) { - existingRows = await this.query>( - `SELECT * FROM ${this.quoteIdentifier(table.tableName)}`, - ); - await this.run( - `DROP TABLE ${this.quoteIdentifier(table.tableName)}`, - ); - } - + if (!doesTableExist) { await this.createTable(table); - - if (existingRows.length > 0) { - await this.insertRows(table.tableName, existingRows); - } - return; } @@ -463,30 +464,6 @@ class MariaDBSchemaManager { ); } - private async insertRows( - tableName: string, - rows: Record[], - ): Promise { - for (const row of rows) { - const columns = Object.keys(row); - - if (columns.length === 0) { - continue; - } - - const values = columns.map((column) => row[column]); - const columnList = columns - .map((column) => this.quoteIdentifier(column)) - .join(", "); - const placeholders = columns.map(() => "?").join(", "); - - await this.run( - `INSERT INTO ${this.quoteIdentifier(tableName)} (${columnList}) VALUES (${placeholders})`, - values, - ); - } - } - private buildColumnDefinition(field: BUN_MARIADB_FieldSchemaType): string { if (!field.fieldName) { throw new Error("Field name is required"); @@ -664,13 +641,11 @@ class MariaDBSchemaManager { .map((field) => this.quoteIdentifier(field)) .join(", "); - // Determine if we need a specialized modifier prefix like FULLTEXT or SPATIAL const typeUpper = index.indexType?.toUpperCase(); const isSpecialType = typeUpper === "FULLTEXT" || typeUpper === "SPATIAL"; const indexPrefix = isSpecialType ? `${typeUpper} ` : ""; - // Append USING BTREE/HASH if it's a normal index type option const indexSuffix = !isSpecialType && (typeUpper === "BTREE" || typeUpper === "HASH") diff --git a/src/lib/mariadb/db-select.ts b/src/lib/mariadb/db-select.ts index 4040d28..8b65e6a 100644 --- a/src/lib/mariadb/db-select.ts +++ b/src/lib/mariadb/db-select.ts @@ -1,6 +1,6 @@ import dbHandler from "../db-handler"; import _ from "lodash"; -import type { APIResponseObject, ServerQueryParam } from "../../types"; +import type { DBResponseObject, ServerQueryParam } from "../../types"; import sqlGenerator from "../../utils/sql-generator"; type Params< @@ -25,7 +25,7 @@ export default async function DbSelect< query, count, targetId, -}: Params): Promise> { +}: Params): Promise> { let sqlObj: ReturnType | null = null; try { @@ -67,10 +67,10 @@ export default async function DbSelect< const batchRes = (res.payload || []) as Schema[]; - let resp: APIResponseObject = { + let resp: DBResponseObject = { success: true, payload: batchRes, - singleRes: batchRes[0], + single_res: batchRes[0], debug: { sqlObj, sql: sqlObj.string, diff --git a/src/lib/mariadb/db-sql.ts b/src/lib/mariadb/db-sql.ts index ef327bf..2faf55b 100644 --- a/src/lib/mariadb/db-sql.ts +++ b/src/lib/mariadb/db-sql.ts @@ -1,5 +1,5 @@ import dbHandler from "../db-handler"; -import type { APIResponseObject, SQLInsertGenValueType } from "../../types"; +import type { DBResponseObject, SQLInsertGenValueType } from "../../types"; type Params = { sql: string; @@ -8,7 +8,7 @@ type Params = { export default async function DbSQL< T extends { [k: string]: any } = { [k: string]: any }, ->({ sql, values }: Params): Promise> { +>({ sql, values }: Params): Promise> { try { const trimmedSql = sql.trim(); const isSelect = trimmedSql.match(/^select/i); @@ -33,19 +33,13 @@ export default async function DbSQL< } const payload = isSelect ? ((res.payload || []) as T[]) : undefined; - const singleRes = isSelect ? payload?.[0] : (res.single_res as T); + const single_res = isSelect ? payload?.[0] : (res.single_res as T); const singleRaw = res.single_res as any; return { success: true, payload, - singleRes, - postInsertReturn: isSelect - ? undefined - : { - affectedRows: Number(singleRaw?.affectedRows), - insertId: Number(singleRaw?.insertId), - }, + single_res, debug: { sqlObj: { sql: trimmedSql, diff --git a/src/lib/mariadb/db-update.ts b/src/lib/mariadb/db-update.ts index e0da450..b4b540d 100644 --- a/src/lib/mariadb/db-update.ts +++ b/src/lib/mariadb/db-update.ts @@ -1,7 +1,7 @@ import dbHandler from "../db-handler"; import _ from "lodash"; import type { - APIResponseObject, + DBResponseObject, SQLInsertGenValueType, ServerQueryParam, } from "../../types"; @@ -29,7 +29,7 @@ export default async function DbUpdate< data, query, targetId, -}: Params): Promise { +}: Params): Promise { let sqlObj: ReturnType = { string: "", values: [] }; try { @@ -63,7 +63,8 @@ export default async function DbUpdate< } let values: SQLInsertGenValueType[] = []; - let sql = `UPDATE ${quoteIdentifier(table)} SET`; + let sql = ``; + sql += `UPDATE ${quoteIdentifier(table)} SET`; const finalData: { [k: string]: SQLInsertGenValueType } = { updated_at: Date.now(), @@ -89,31 +90,47 @@ export default async function DbUpdate< sql += ` ${whereClause}`; values = [...values, ...sqlQueryObj.values]; - sqlObj.string = sql; - sqlObj.values = values as any[]; - const res = await dbHandler({ query: sql, values: values as any, }); - if (!res.success) { - return { - success: false, - msg: "Database update failed", - debug: { - sqlObj, - }, - }; + sqlObj.string = sql; + sqlObj.values = values as any[]; + + let updated_sql = ``; + let updated_sql_values: any[] = []; + + updated_sql += `SELECT * FROM ${quoteIdentifier(table)} ${whereClause}`; + updated_sql_values = [...updated_sql_values, ...sqlQueryObj.values]; + updated_sql += ` AND `; + for (let i = 0; i < keys.length; i++) { + const key = keys[i]; + if (!key) continue; + if (key == "updated_at") continue; + + const isLast = i == keys.length - 1; + + updated_sql += ` ${quoteIdentifier(key)}=?`; + updated_sql_values.push(finalData[key] ?? null); + + if (!isLast) { + updated_sql += ` AND `; + } } - const singleRes = res.single_res as any; + const updated_res = await dbHandler({ + query: updated_sql, + values: updated_sql_values, + }); + + const affected_rows = updated_res.payload?.length; return { - success: Boolean(singleRes?.affectedRows || singleRes?.changedRows), - postInsertReturn: { - affectedRows: Number(singleRes?.affectedRows), - insertId: Number(singleRes?.insertId), + ...res, + success: Boolean(affected_rows), + insert_return: { + affected_rows, }, debug: { sqlObj, diff --git a/src/types/index.ts b/src/types/index.ts index 1fdb84e..8f78b0e 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1290,46 +1290,6 @@ export type ResponseQueryObject = { params?: (string | number)[]; }; -/** - * Common API response wrapper used by data, auth, media, and utility - * endpoints. - */ -export type APIResponseObject< - T extends { [k: string]: any } = { [k: string]: any }, -> = { - success: boolean; - payload?: T[] | null; - singleRes?: T | null; - stringRes?: string | null; - numberRes?: number | null; - postInsertReturn?: PostInsertReturn | null; - payloadBase64?: string; - payloadThumbnailBase64?: string; - payloadURL?: string; - payloadThumbnailURL?: string; - error?: any; - msg?: string; - queryObject?: ResponseQueryObject; - countQueryObject?: ResponseQueryObject; - status?: number; - count?: number; - errors?: BunMariaDBErrorObject[]; - debug?: any; - batchPayload?: any[][] | null; - errorData?: any; - token?: string; - csrf?: string; - cookieNames?: any; - key?: string; - userId?: string | number; - code?: string; - createdAt?: number; - email?: string; - requestOptions?: RequestOptions; - logoutUser?: boolean; - redirect?: string; -}; - /** * # Docker Compose Types */ @@ -1625,6 +1585,8 @@ export type DBResponseObject< insert_return?: DBInsertReturn; error?: any; msg?: string; + debug?: any; + count?: number; }; export type DBInsertReturn = {