First working version.

This commit is contained in:
Benjamin Toby 2026-06-24 17:00:14 +01:00
parent 44f6c18a84
commit b5e4724e03
9 changed files with 92 additions and 151 deletions

View File

@ -7,8 +7,6 @@ import _ from "lodash";
import appendDefaultFieldsToDbSchema from "../utils/append-default-fields-to-db-schema"; import appendDefaultFieldsToDbSchema from "../utils/append-default-fields-to-db-schema";
import chalk from "chalk"; import chalk from "chalk";
import { writeLiveSchema } from "../functions/live-schema"; import { writeLiveSchema } from "../functions/live-schema";
import grabDBDir from "../utils/grab-db-dir";
import { cpSync } from "fs";
export default function () { export default function () {
return new Command("schema") return new Command("schema")
@ -24,10 +22,9 @@ export default function () {
const config = global.CONFIG; const config = global.CONFIG;
const dbSchema = global.DB_SCHEMA; const dbSchema = global.DB_SCHEMA;
const { ROOT_DIR, BUN_MARIADB_TEMP_DB_FILE_PATH } = grabDirNames(); const { ROOT_DIR } = grabDirNames();
try { try {
const isVector = Boolean(opts.vector || opts.v);
const isTypeDef = Boolean(opts.typedef || opts.t); const isTypeDef = Boolean(opts.typedef || opts.t);
const finaldbSchema = appendDefaultFieldsToDbSchema({ const finaldbSchema = appendDefaultFieldsToDbSchema({
@ -36,7 +33,6 @@ export default function () {
const manager = new MariaDBSchemaManager({ const manager = new MariaDBSchemaManager({
schema: finaldbSchema, schema: finaldbSchema,
recreate_vector_table: isVector,
}); });
await manager.syncSchema(); await manager.syncSchema();

View File

@ -17,8 +17,6 @@ export default async function dbHandler<
const res = await MariaDBClient.unsafe(query, values); const res = await MariaDBClient.unsafe(query, values);
const count = res.count; const count = res.count;
const last_insert_id = res.lastInsertRowid;
const affected_rows = res.affectedRows;
const res_array = (() => { const res_array = (() => {
try { try {
@ -28,6 +26,9 @@ export default async function dbHandler<
} }
})(); })();
const last_insert_id = res_array?.[0] ? res_array[0]?.id : undefined;
const affected_rows = res_array?.[0] ? res_array.length : undefined;
const insert_return: DBInsertReturn = { const insert_return: DBInsertReturn = {
count, count,
last_insert_id, last_insert_id,
@ -39,6 +40,7 @@ export default async function dbHandler<
payload: res_array, payload: res_array,
single_res: res_array?.[0], single_res: res_array?.[0],
insert_return, insert_return,
count,
}; };
} catch (error: any) { } catch (error: any) {
return { return {

View File

@ -1,6 +1,6 @@
import dbHandler from "../db-handler"; import dbHandler from "../db-handler";
import _ from "lodash"; import _ from "lodash";
import type { APIResponseObject, ServerQueryParam } from "../../types"; import type { DBResponseObject, ServerQueryParam } from "../../types";
import sqlGenerator from "../../utils/sql-generator"; import sqlGenerator from "../../utils/sql-generator";
type Params< type Params<
@ -23,7 +23,7 @@ export default async function DbDelete<
table, table,
query, query,
targetId, targetId,
}: Params<Schema, Table>): Promise<APIResponseObject> { }: Params<Schema, Table>): Promise<DBResponseObject> {
let sqlObj: ReturnType<typeof sqlGenerator> | null = null; let sqlObj: ReturnType<typeof sqlGenerator> | null = null;
try { try {
@ -59,8 +59,8 @@ export default async function DbDelete<
}; };
} }
const sql = `DELETE FROM ${quoteIdentifier(table)} ${whereClause}`; let sql = `DELETE FROM ${quoteIdentifier(table)} ${whereClause}`;
sql += ` RETURNING *`;
sqlObj.string = sql; sqlObj.string = sql;
const res = await dbHandler({ const res = await dbHandler({
@ -78,14 +78,9 @@ export default async function DbDelete<
}; };
} }
const singleRes = res.single_res as any;
return { return {
success: Boolean(singleRes?.affectedRows), ..._.omit(res, ["payload"]),
postInsertReturn: { success: Boolean(res.insert_return?.last_insert_id),
affectedRows: Number(singleRes?.affectedRows),
insertId: Number(singleRes?.insertId),
},
debug: { debug: {
sqlObj, sqlObj,
}, },

View File

@ -1,5 +1,5 @@
import dbHandler from "../db-handler"; import dbHandler from "../db-handler";
import type { APIResponseObject, SQLInsertGenReturn } from "../../types"; import type { DBResponseObject, SQLInsertGenReturn } from "../../types";
import sqlInsertGenerator from "../../utils/sql-insert-generator"; import sqlInsertGenerator from "../../utils/sql-insert-generator";
import grabDuplicateSafeInsertSql from "../grab-duplicate-safe-insert-sql"; import grabDuplicateSafeInsertSql from "../grab-duplicate-safe-insert-sql";
@ -19,7 +19,7 @@ export default async function DbInsert<
table, table,
data, data,
update_on_duplicate, update_on_duplicate,
}: Params<Schema, Table>): Promise<APIResponseObject> { }: Params<Schema, Table>): Promise<DBResponseObject> {
let sqlObj: SQLInsertGenReturn | null = null; let sqlObj: SQLInsertGenReturn | null = null;
try { try {
@ -48,9 +48,11 @@ export default async function DbInsert<
sql = await grabDuplicateSafeInsertSql({ data, table, sql }); sql = await grabDuplicateSafeInsertSql({ data, table, sql });
} }
sql += ` RETURNING *`;
const res = await dbHandler({ const res = await dbHandler({
query: sql, query: sql,
values: sqlObj.values as any, values: sqlObj.values,
}); });
if (!res.success) { if (!res.success) {
@ -63,14 +65,12 @@ export default async function DbInsert<
}; };
} }
const singleRes = res.single_res as any;
return { return {
success: Boolean(singleRes?.affectedRows || singleRes?.insertId), ...res,
postInsertReturn: { success: Boolean(
affectedRows: Number(singleRes?.affectedRows), res?.insert_return?.affected_rows ||
insertId: Number(singleRes?.insertId), res.insert_return?.last_insert_id,
}, ),
debug: { debug: {
sqlObj, sqlObj,
}, },

View File

@ -19,6 +19,7 @@ type TableInfoRow = {
type ColumnInfoRow = { type ColumnInfoRow = {
name: string; name: string;
type: string; type: string;
comment?: string;
}; };
type IndexInfoRow = { type IndexInfoRow = {
@ -93,9 +94,6 @@ class MariaDBSchemaManager {
}); });
if (!res.success) { if (!res.success) {
console.log("res", res);
console.log("query", query);
console.log("values", values);
throw new Error(`Database query failed: ${query}`); throw new Error(`Database query failed: ${query}`);
} }
@ -343,9 +341,18 @@ class MariaDBSchemaManager {
if (!current) return false; if (!current) return false;
const resolvedType = this.mapDataType(field).toLowerCase(); const resolvedType = this.mapDataType(field).toLowerCase();
return !resolvedType.startsWith(current.type.toLowerCase()); const currentType = current.type.toLowerCase();
// If it's a vector column, compare the explicit metadata size inside the column comment fields
if (field.isVector || resolvedTable.isVector) {
const targetComment = `vector_size=${field.vectorSize || 1536}`;
return current.comment !== targetComment;
}
return !resolvedType.startsWith(currentType);
}); });
// Fast path: Only adding new columns without metadata alterations or vector shifts
if ( if (
missingFields.length > 0 && missingFields.length > 0 &&
!hasModifiedFields && !hasModifiedFields &&
@ -357,22 +364,31 @@ class MariaDBSchemaManager {
return; return;
} }
// If changes were explicitly made or vector sizing changed, execute safe zero-data-loss rebuild
if (
hasModifiedFields ||
missingFields.length > 0 ||
resolvedTable.isVector
) {
await this.recreateTable(resolvedTable); await this.recreateTable(resolvedTable);
} }
}
private async getTableColumns(tableName: string): Promise<ColumnInfoRow[]> { private async getTableColumns(tableName: string): Promise<ColumnInfoRow[]> {
const schemaCond = this.schemaCondition(); const schemaCond = this.schemaCondition();
const rows = await this.query<{ const rows = await this.query<{
COLUMN_NAME: string; COLUMN_NAME: string;
COLUMN_TYPE: string; COLUMN_TYPE: string;
COLUMN_COMMENT: string;
}>( }>(
`SELECT COLUMN_NAME, COLUMN_TYPE FROM information_schema.COLUMNS WHERE ${schemaCond.where} AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION`, `SELECT COLUMN_NAME, COLUMN_TYPE, COLUMN_COMMENT FROM information_schema.COLUMNS WHERE ${schemaCond.where} AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION`,
[...schemaCond.values, tableName], [...schemaCond.values, tableName],
); );
return rows.map((row) => ({ return rows.map((row) => ({
name: row.COLUMN_NAME, name: row.COLUMN_NAME,
type: row.COLUMN_TYPE, type: row.COLUMN_TYPE,
comment: row.COLUMN_COMMENT,
})); }));
} }
@ -392,6 +408,7 @@ class MariaDBSchemaManager {
`ALTER TABLE ${this.quoteIdentifier(tableName)} ADD COLUMN ${columnDef}`, `ALTER TABLE ${this.quoteIdentifier(tableName)} ADD COLUMN ${columnDef}`,
); );
} }
private async checkIfTableExists(table: string): Promise<boolean> { private async checkIfTableExists(table: string): Promise<boolean> {
const schemaCond = this.schemaCondition(); const schemaCond = this.schemaCondition();
const row = await this.query<{ table_exists: number }>( const row = await this.query<{ table_exists: number }>(
@ -401,29 +418,13 @@ class MariaDBSchemaManager {
return Boolean(row[0]?.table_exists); return Boolean(row[0]?.table_exists);
} }
private async recreateTable( private async recreateTable(
table: BUN_MARIADB_TableSchemaType, table: BUN_MARIADB_TableSchemaType,
): Promise<void> { ): Promise<void> {
const doesTableExist = await this.checkIfTableExists(table.tableName); const doesTableExist = await this.checkIfTableExists(table.tableName);
if (!doesTableExist) {
if (table.isVector) {
let existingRows: Record<string, any>[] = [];
if (doesTableExist) {
existingRows = await this.query<Record<string, any>>(
`SELECT * FROM ${this.quoteIdentifier(table.tableName)}`,
);
await this.run(
`DROP TABLE ${this.quoteIdentifier(table.tableName)}`,
);
}
await this.createTable(table); await this.createTable(table);
if (existingRows.length > 0) {
await this.insertRows(table.tableName, existingRows);
}
return; return;
} }
@ -463,30 +464,6 @@ class MariaDBSchemaManager {
); );
} }
private async insertRows(
tableName: string,
rows: Record<string, any>[],
): Promise<void> {
for (const row of rows) {
const columns = Object.keys(row);
if (columns.length === 0) {
continue;
}
const values = columns.map((column) => row[column]);
const columnList = columns
.map((column) => this.quoteIdentifier(column))
.join(", ");
const placeholders = columns.map(() => "?").join(", ");
await this.run(
`INSERT INTO ${this.quoteIdentifier(tableName)} (${columnList}) VALUES (${placeholders})`,
values,
);
}
}
private buildColumnDefinition(field: BUN_MARIADB_FieldSchemaType): string { private buildColumnDefinition(field: BUN_MARIADB_FieldSchemaType): string {
if (!field.fieldName) { if (!field.fieldName) {
throw new Error("Field name is required"); throw new Error("Field name is required");
@ -664,13 +641,11 @@ class MariaDBSchemaManager {
.map((field) => this.quoteIdentifier(field)) .map((field) => this.quoteIdentifier(field))
.join(", "); .join(", ");
// Determine if we need a specialized modifier prefix like FULLTEXT or SPATIAL
const typeUpper = index.indexType?.toUpperCase(); const typeUpper = index.indexType?.toUpperCase();
const isSpecialType = const isSpecialType =
typeUpper === "FULLTEXT" || typeUpper === "SPATIAL"; typeUpper === "FULLTEXT" || typeUpper === "SPATIAL";
const indexPrefix = isSpecialType ? `${typeUpper} ` : ""; const indexPrefix = isSpecialType ? `${typeUpper} ` : "";
// Append USING BTREE/HASH if it's a normal index type option
const indexSuffix = const indexSuffix =
!isSpecialType && !isSpecialType &&
(typeUpper === "BTREE" || typeUpper === "HASH") (typeUpper === "BTREE" || typeUpper === "HASH")

View File

@ -1,6 +1,6 @@
import dbHandler from "../db-handler"; import dbHandler from "../db-handler";
import _ from "lodash"; import _ from "lodash";
import type { APIResponseObject, ServerQueryParam } from "../../types"; import type { DBResponseObject, ServerQueryParam } from "../../types";
import sqlGenerator from "../../utils/sql-generator"; import sqlGenerator from "../../utils/sql-generator";
type Params< type Params<
@ -25,7 +25,7 @@ export default async function DbSelect<
query, query,
count, count,
targetId, targetId,
}: Params<Schema, Table>): Promise<APIResponseObject<Schema>> { }: Params<Schema, Table>): Promise<DBResponseObject<Schema>> {
let sqlObj: ReturnType<typeof sqlGenerator> | null = null; let sqlObj: ReturnType<typeof sqlGenerator> | null = null;
try { try {
@ -67,10 +67,10 @@ export default async function DbSelect<
const batchRes = (res.payload || []) as Schema[]; const batchRes = (res.payload || []) as Schema[];
let resp: APIResponseObject<Schema> = { let resp: DBResponseObject<Schema> = {
success: true, success: true,
payload: batchRes, payload: batchRes,
singleRes: batchRes[0], single_res: batchRes[0],
debug: { debug: {
sqlObj, sqlObj,
sql: sqlObj.string, sql: sqlObj.string,

View File

@ -1,5 +1,5 @@
import dbHandler from "../db-handler"; import dbHandler from "../db-handler";
import type { APIResponseObject, SQLInsertGenValueType } from "../../types"; import type { DBResponseObject, SQLInsertGenValueType } from "../../types";
type Params = { type Params = {
sql: string; sql: string;
@ -8,7 +8,7 @@ type Params = {
export default async function DbSQL< export default async function DbSQL<
T extends { [k: string]: any } = { [k: string]: any }, T extends { [k: string]: any } = { [k: string]: any },
>({ sql, values }: Params): Promise<APIResponseObject<T>> { >({ sql, values }: Params): Promise<DBResponseObject<T>> {
try { try {
const trimmedSql = sql.trim(); const trimmedSql = sql.trim();
const isSelect = trimmedSql.match(/^select/i); const isSelect = trimmedSql.match(/^select/i);
@ -33,19 +33,13 @@ export default async function DbSQL<
} }
const payload = isSelect ? ((res.payload || []) as T[]) : undefined; const payload = isSelect ? ((res.payload || []) as T[]) : undefined;
const singleRes = isSelect ? payload?.[0] : (res.single_res as T); const single_res = isSelect ? payload?.[0] : (res.single_res as T);
const singleRaw = res.single_res as any; const singleRaw = res.single_res as any;
return { return {
success: true, success: true,
payload, payload,
singleRes, single_res,
postInsertReturn: isSelect
? undefined
: {
affectedRows: Number(singleRaw?.affectedRows),
insertId: Number(singleRaw?.insertId),
},
debug: { debug: {
sqlObj: { sqlObj: {
sql: trimmedSql, sql: trimmedSql,

View File

@ -1,7 +1,7 @@
import dbHandler from "../db-handler"; import dbHandler from "../db-handler";
import _ from "lodash"; import _ from "lodash";
import type { import type {
APIResponseObject, DBResponseObject,
SQLInsertGenValueType, SQLInsertGenValueType,
ServerQueryParam, ServerQueryParam,
} from "../../types"; } from "../../types";
@ -29,7 +29,7 @@ export default async function DbUpdate<
data, data,
query, query,
targetId, targetId,
}: Params<Schema, Table>): Promise<APIResponseObject> { }: Params<Schema, Table>): Promise<DBResponseObject> {
let sqlObj: ReturnType<typeof sqlGenerator> = { string: "", values: [] }; let sqlObj: ReturnType<typeof sqlGenerator> = { string: "", values: [] };
try { try {
@ -63,7 +63,8 @@ export default async function DbUpdate<
} }
let values: SQLInsertGenValueType[] = []; let values: SQLInsertGenValueType[] = [];
let sql = `UPDATE ${quoteIdentifier(table)} SET`; let sql = ``;
sql += `UPDATE ${quoteIdentifier(table)} SET`;
const finalData: { [k: string]: SQLInsertGenValueType } = { const finalData: { [k: string]: SQLInsertGenValueType } = {
updated_at: Date.now(), updated_at: Date.now(),
@ -89,31 +90,47 @@ export default async function DbUpdate<
sql += ` ${whereClause}`; sql += ` ${whereClause}`;
values = [...values, ...sqlQueryObj.values]; values = [...values, ...sqlQueryObj.values];
sqlObj.string = sql;
sqlObj.values = values as any[];
const res = await dbHandler({ const res = await dbHandler({
query: sql, query: sql,
values: values as any, values: values as any,
}); });
if (!res.success) { sqlObj.string = sql;
return { sqlObj.values = values as any[];
success: false,
msg: "Database update failed", let updated_sql = ``;
debug: { let updated_sql_values: any[] = [];
sqlObj,
}, updated_sql += `SELECT * FROM ${quoteIdentifier(table)} ${whereClause}`;
}; updated_sql_values = [...updated_sql_values, ...sqlQueryObj.values];
updated_sql += ` AND `;
for (let i = 0; i < keys.length; i++) {
const key = keys[i];
if (!key) continue;
if (key == "updated_at") continue;
const isLast = i == keys.length - 1;
updated_sql += ` ${quoteIdentifier(key)}=?`;
updated_sql_values.push(finalData[key] ?? null);
if (!isLast) {
updated_sql += ` AND `;
}
} }
const singleRes = res.single_res as any; const updated_res = await dbHandler({
query: updated_sql,
values: updated_sql_values,
});
const affected_rows = updated_res.payload?.length;
return { return {
success: Boolean(singleRes?.affectedRows || singleRes?.changedRows), ...res,
postInsertReturn: { success: Boolean(affected_rows),
affectedRows: Number(singleRes?.affectedRows), insert_return: {
insertId: Number(singleRes?.insertId), affected_rows,
}, },
debug: { debug: {
sqlObj, sqlObj,

View File

@ -1290,46 +1290,6 @@ export type ResponseQueryObject = {
params?: (string | number)[]; params?: (string | number)[];
}; };
/**
* Common API response wrapper used by data, auth, media, and utility
* endpoints.
*/
export type APIResponseObject<
T extends { [k: string]: any } = { [k: string]: any },
> = {
success: boolean;
payload?: T[] | null;
singleRes?: T | null;
stringRes?: string | null;
numberRes?: number | null;
postInsertReturn?: PostInsertReturn | null;
payloadBase64?: string;
payloadThumbnailBase64?: string;
payloadURL?: string;
payloadThumbnailURL?: string;
error?: any;
msg?: string;
queryObject?: ResponseQueryObject;
countQueryObject?: ResponseQueryObject;
status?: number;
count?: number;
errors?: BunMariaDBErrorObject[];
debug?: any;
batchPayload?: any[][] | null;
errorData?: any;
token?: string;
csrf?: string;
cookieNames?: any;
key?: string;
userId?: string | number;
code?: string;
createdAt?: number;
email?: string;
requestOptions?: RequestOptions;
logoutUser?: boolean;
redirect?: string;
};
/** /**
* # Docker Compose Types * # Docker Compose Types
*/ */
@ -1625,6 +1585,8 @@ export type DBResponseObject<
insert_return?: DBInsertReturn; insert_return?: DBInsertReturn;
error?: any; error?: any;
msg?: string; msg?: string;
debug?: any;
count?: number;
}; };
export type DBInsertReturn = { export type DBInsertReturn = {