Moduletrace/bun-mariadb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updates

Browse Source
This commit is contained in:
tben
2026-07-14 09:10:30 +01:00
parent 79c5b896fe
commit 56462e6cc2
38 changed files with 1056 additions and 1720 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/db-handler.ts
+86 -66
View File
@@ -1,66 +1,86 @@
import grabMariaDBClient from "../functions/grab-mariadb-client";
import type {
BunMariaDBConfig,
DBInsertReturn,
DBResponseObject,
} from "../types";
type Param = {
query: string;
values?: any[];
config?: BunMariaDBConfig;
};
/**
* # Main DB Handler Function
* @requires DSQL_DB_CONN - Gobal Variable for Datasquirel Database
*/
export default async function dbHandler<
T extends { [k: string]: any } = { [k: string]: any },
>({ query, values, config }: Param): Promise<DBResponseObject<T>> {
try {
const CLIENT = config
? grabMariaDBClient({ config })
: global.MARIADB_CLIENT;
if (!CLIENT) {
throw new Error(`Couldn't grab MariaDB Client.`);
}
const res = await CLIENT.unsafe(query, values);
const count = res.count;
const res_array = (() => {
try {
return JSON.parse(JSON.stringify(res)) as T[];
} catch (error) {
return undefined;
}
})();
const last_insert_id = res_array?.[0] ? res_array[0]?.id : undefined;
const affected_rows = res_array?.[0] ? res_array.length : undefined;
const insert_return: DBInsertReturn = {
count,
last_insert_id,
affected_rows,
};
return {
success: true,
payload: res_array,
single_res: res_array?.[0],
insert_return,
count,
};
} catch (error: any) {
return {
success: false,
error: "DB Handler Error => " + error.message,
msg: "DB Handler Error => " + error.message,
};
} finally {
}
}
import grabMariaDBClient from "../functions/grab-mariadb-client";
import type {
BunMariaDBConfig,
DBInsertReturn,
DBResponseObject,
} from "../types";
type Param = {
query: string;
values?: any[];
config?: BunMariaDBConfig;
};
/**
* Resolve a shared MariaDB client. Prefer the global client so schema sync
* and CRUD do not open a new connection per query.
*/
function resolveClient(config?: BunMariaDBConfig): Bun.SQL | undefined {
const resolvedConfig = config || global.CONFIG;
// Same database as the global client → reuse it
if (
global.MARIADB_CLIENT &&
(!resolvedConfig?.db_name ||
!global.CONFIG?.db_name ||
resolvedConfig.db_name === global.CONFIG.db_name)
) {
return global.MARIADB_CLIENT;
}
if (resolvedConfig) {
return grabMariaDBClient({ config: resolvedConfig });
}
return global.MARIADB_CLIENT;
}
/**
* # Main DB Handler Function
*/
export default async function dbHandler<
T extends { [k: string]: any } = { [k: string]: any },
>({ query, values, config }: Param): Promise<DBResponseObject<T>> {
try {
const CLIENT = resolveClient(config);
if (!CLIENT) {
throw new Error(`Couldn't grab MariaDB Client.`);
}
const res = await CLIENT.unsafe(query, values);
const count = res.count;
const res_array = (() => {
try {
return JSON.parse(JSON.stringify(res)) as T[];
} catch (error) {
return undefined;
}
})();
const last_insert_id = res_array?.[0] ? res_array[0]?.id : undefined;
const affected_rows = res_array?.[0] ? res_array.length : undefined;
const insert_return: DBInsertReturn = {
count,
last_insert_id,
affected_rows,
};
return {
success: true,
payload: res_array,
single_res: res_array?.[0],
insert_return,
count,
};
} catch (error: any) {
return {
success: false,
error: "DB Handler Error => " + error.message,
msg: "DB Handler Error => " + error.message,
};
}
}
src/lib/grab-db-ssl.ts
-29
View File
@@ -1,29 +0,0 @@
import fs from "fs";
import type { ConnectionConfig } from "mariadb";
import path from "path";
/**
* # Grab SSL
*/
export default function grabDbSSL(): ConnectionConfig["ssl"] {
const caProivdedPath =
global.CONFIG.ssl_ca || process.env.BUN_MARIADB_SERVER_SSL_KEY_PATH;
if (!caProivdedPath?.match(/./)) {
return {
rejectUnauthorized: false,
};
}
const ca_file_path = path.resolve(process.cwd(), caProivdedPath);
if (!fs.existsSync(ca_file_path)) {
console.log(`${ca_file_path} does not exist`);
return undefined;
}
return {
ca: fs.readFileSync(ca_file_path),
rejectUnauthorized: false,
};
}
src/lib/mariadb/db-schema-manager.ts
-932
View File
File diff suppressed because it is too large Load Diff
src/lib/mariadb/index.ts
-49
View File
@@ -1,49 +0,0 @@
// import { SQL } from "bun";
// import grabDirNames from "../../data/grab-dir-names";
// import path from "path";
// if (!global.CONFIG) {
// console.error(`Couldn't grab global Config.`);
// process.exit(1);
// }
// if (!global.DB_SCHEMA) {
// console.error(`Couldn't grab Database Schema.`);
// process.exit(1);
// }
// const config = global.CONFIG;
// const { ROOT_DIR } = grabDirNames();
// const MariaDBClient = new SQL({
// hostname: process.env.BUN_MARIADB_SERVER_HOST,
// username: process.env.BUN_MARIADB_SERVER_USERNAME,
// password: process.env.BUN_MARIADB_SERVER_PASSWORD,
// database: config.db_name,
// port: process.env.BUN_MARIADB_SERVER_PORT
// ? Number(process.env.BUN_MARIADB_SERVER_PORT)
// : undefined,
// ...config.db_config,
// tls: config.ssl_ca
// ? {
// ca: Bun.file(path.resolve(ROOT_DIR, config.ssl_ca)),
// rejectUnauthorized: false,
// }
// : {
// rejectUnauthorized: false,
// },
// adapter: "mariadb",
// });
// global.MARIADB_CLIENT = MariaDBClient;
// const test = await MariaDBClient.unsafe(`SHOW DATABASES`);
// if (!test.count) {
// console.error(`MariaDBClient Error: Database not ready.`);
// console.log(test);
// process.exit(1);
// }
// export default MariaDBClient;
src/lib/schema/build-column-definition.ts
+9 -2
View File
@@ -1,4 +1,5 @@
import type { BUN_MARIADB_FieldSchemaType } from "../../types";
import isVectorField from "./is-vector-field";
import mapDataType from "./map-data-types";
import MariaDBQuoteGen from "./mariadb-quote-gen";
@@ -16,13 +17,19 @@ export default function buildColumnDefinition(
parts.push("AUTO_INCREMENT");
}
if (field.notNullValue || field.primaryKey || field.isVector) {
// Vector columns used in VECTOR INDEX must be NOT NULL
if (
field.notNullValue ||
field.primaryKey ||
isVectorField(field)
) {
if (!field.primaryKey) {
parts.push("NOT NULL");
}
}
if (field.unique && !field.primaryKey) {
// VECTOR columns cannot be UNIQUE in the usual sense
if (field.unique && !field.primaryKey && !isVectorField(field)) {
parts.push("UNIQUE");
}
src/lib/schema/handle-db-schema-table.ts
+14 -14
View File
@@ -19,26 +19,23 @@ export default async function handleDBSchemaTable({
}: CreateDBSchemaTableHandlerParams) {
const resolvedTable = resolveTable(table, db_schema);
const schemaCond = schemaCondition(config);
const liveTables = await querySchemaRows<{ TABLE_NAME: string }>({
query: `SELECT TABLE_NAME FROM information_schema.TABLES WHERE ${schemaCond.where} AND TABLE_TYPE = 'BASE TABLE'`,
values: schemaCond.values,
config,
});
const liveTableNames = liveTables.map((t) => t.TABLE_NAME);
let tableExistsTracked = Boolean(db_manager_table_name);
let tableExistsLive = Boolean(
existing_live_table?.TABLE_NAME ||
liveTableNames.includes(resolvedTable.tableName),
);
let tableExistsLive = Boolean(existing_live_table?.TABLE_NAME);
let wasRenamed = false;
if (
resolvedTable.tableNameOld &&
resolvedTable.tableNameOld !== resolvedTable.tableName
) {
if (liveTableNames.includes(resolvedTable.tableNameOld)) {
// Only hit information_schema when a rename is declared
const schemaCond = schemaCondition(config);
const liveTables = await querySchemaRows<{ TABLE_NAME: string }>({
query: `SELECT TABLE_NAME FROM information_schema.TABLES WHERE ${schemaCond.where} AND TABLE_TYPE = 'BASE TABLE' AND TABLE_NAME = ?`,
values: [...schemaCond.values, resolvedTable.tableNameOld],
config,
});
if (liveTables.length > 0) {
console.log(
`Renaming table: ${resolvedTable.tableNameOld} -> ${resolvedTable.tableName}`,
);
@@ -68,7 +65,10 @@ export default async function handleDBSchemaTable({
});
} else {
if (!wasRenamed) {
await updateTable({ table: resolvedTable, config });
await updateTable({
table: resolvedTable,
config,
});
}
await upsertDbManagerTable({
tableName: resolvedTable.tableName,
src/lib/schema/is-vector-field.ts
+11
View File
@@ -0,0 +1,11 @@
import type { BUN_MARIADB_FieldSchemaType } from "../../types";
/**
* True when a field is a MariaDB native vector column.
*/
export default function isVectorField(
field?: BUN_MARIADB_FieldSchemaType,
): boolean {
if (!field) return false;
return field.isVector === true || field.dataType === "VECTOR";
}
src/lib/schema/map-data-types.ts
+3 -6
View File
@@ -6,8 +6,9 @@ export default function mapDataType(
const dataType = field.dataType?.toUpperCase() || "TEXT";
const vectorSize = field.vectorSize || 1536;
if (field.isVector) {
return `LONGTEXT COMMENT 'vector_size=${vectorSize}'`;
// Native MariaDB VECTOR type (11.7+). Prefer this over LONGTEXT storage.
if (field.isVector || dataType === "VECTOR") {
return `VECTOR(${vectorSize})`;
}
switch (dataType) {
@@ -78,10 +79,6 @@ export default function mapDataType(
return "JSON";
case "INET6":
return "INET6";
case "VECTOR": {
const dimensions = field.vectorSize || 1536;
return `VECTOR(${dimensions})`;
}
case "BOOLEAN":
return "TINYINT(1)";
case "ENUM": {
src/lib/schema/recreate-table.ts
+56
View File
@@ -25,6 +25,10 @@ async function checkIfTableExists({
return Boolean(rows[0]?.table_exists);
}
/**
* Full table rebuild. For `isVector` tables this drops and recreates in place
* (preserving rows when possible). For regular tables it uses a temp-table swap.
*/
export default async function recreateTable({
table,
config,
@@ -42,6 +46,58 @@ export default async function recreateTable({
return;
}
/**
* Vector tables: drop + recreate + reinsert (MariaDB VECTOR INDEX / dim
* changes are not reliably alterable in place).
*/
if (table.isVector) {
console.log(`Recreating vector table: ${table.tableName}`);
const existingRows = await querySchemaRows<Record<string, any>>({
query: `SELECT * FROM ${MariaDBQuoteGen(table.tableName)}`,
config,
});
await runSchemaQuery({ query: `SET FOREIGN_KEY_CHECKS = 0`, config });
try {
await runSchemaQuery({
query: `DROP TABLE IF EXISTS ${MariaDBQuoteGen(table.tableName)}`,
config,
});
await createTable({ table, config });
} finally {
await runSchemaQuery({ query: `SET FOREIGN_KEY_CHECKS = 1`, config });
}
if (existingRows.length > 0) {
const schemaFieldNames = new Set(
(table.fields || [])
.map((f) => f.fieldName)
.filter((n): n is string => Boolean(n)),
);
for (const row of existingRows) {
const columns = Object.keys(row).filter((c) =>
schemaFieldNames.has(c),
);
if (columns.length === 0) continue;
const placeholders = columns.map(() => "?").join(", ");
const columnList = columns
.map((c) => MariaDBQuoteGen(c))
.join(", ");
await runSchemaQuery({
query: `INSERT INTO ${MariaDBQuoteGen(table.tableName)} (${columnList}) VALUES (${placeholders})`,
values: columns.map((c) => row[c] ?? null),
config,
});
}
}
return;
}
const tempTableName = `${table.tableName}_temp_${Date.now()}`;
const backupOldTableName = `${table.tableName}_old_${Date.now()}`;
const existingColumns = await getTableColumns({
src/lib/schema/sync-indexes.ts
+24 -12
View File
@@ -1,11 +1,33 @@
import type {
BUN_MARIADB_IndexSchemaType,
BUN_MARIADB_TableSchemaType,
BunMariaDBConfig,
} from "../../types";
import isVectorField from "./is-vector-field";
import MariaDBQuoteGen from "./mariadb-quote-gen";
import runSchemaQuery, { querySchemaRows } from "./run-schema-query";
import schemaCondition from "./schema-condition";
function isVectorIndexDef(
index: BUN_MARIADB_IndexSchemaType,
table: BUN_MARIADB_TableSchemaType,
): boolean {
if (index.indexType === "VECTOR") return true;
if (table.isVector) return true;
const firstFieldName = index.indexTableFields?.[0];
if (!firstFieldName) return false;
const field = table.fields?.find((f) => f.fieldName === firstFieldName);
return isVectorField(field);
}
function vectorDistanceMetric(index: BUN_MARIADB_IndexSchemaType): string {
if (index.vectorDistanceMetric === "cosine") return "cosine";
if (index.vectorDistanceMetric === "euclidean") return "euclidean";
return "euclidean";
}
export default async function syncIndexes({
table,
config,
@@ -121,20 +143,10 @@ export default async function syncIndexes({
}
if (!existingIndexesMap.has(index.indexName)) {
const isVectorIndex =
table.isVector ||
table.fields?.some(
(f) =>
f.fieldName === index.indexTableFields![0] && f.isVector,
);
if (isVectorIndex) {
if (isVectorIndexDef(index, table)) {
console.log(`Creating Vector index: ${index.indexName}`);
const targetField = MariaDBQuoteGen(index.indexTableFields[0]!);
const distanceMetric =
(index.indexType as string)?.toLowerCase() === "cosine"
? "cosine"
: "euclidean";
const distanceMetric = vectorDistanceMetric(index);
await runSchemaQuery({
query: `ALTER TABLE ${MariaDBQuoteGen(table.tableName)} ADD VECTOR INDEX ${MariaDBQuoteGen(index.indexName)} (${targetField}) M=8 DISTANCE=${distanceMetric}`,
src/lib/schema/update-table.ts
+47 -5
View File
@@ -6,6 +6,7 @@ import type {
import buildColumnDefinition from "./build-column-definition";
import createTable from "./create-table";
import getTableColumns from "./get-table-columns";
import isVectorField from "./is-vector-field";
import mapDataType from "./map-data-types";
import MariaDBQuoteGen from "./mariadb-quote-gen";
import recreateTable from "./recreate-table";
@@ -29,6 +30,30 @@ function columnTypesMatch(liveType: string, expectedType: string): boolean {
return false;
}
function vectorTypeDiverged(
liveType: string,
liveComment: string,
field: BUN_MARIADB_FieldSchemaType,
): boolean {
const dimensions = field.vectorSize || 1536;
const expectedNative = `vector(${dimensions})`;
const live = liveType.toLowerCase().replace(/\s+/g, "");
if (live === expectedNative) return false;
// Legacy LONGTEXT storage with vector_size comment
if (live.startsWith("longtext") || live.startsWith("text")) {
const match = liveComment.match(/vector_size\s*=\s*(\d+)/i);
if (match && Number(match[1]) === dimensions) {
// Still legacy storage — treat as diverged so we can migrate to VECTOR
return true;
}
return true;
}
return true;
}
async function addColumn({
tableName,
field,
@@ -109,6 +134,7 @@ export default async function updateTable({
const fieldsToAdd: BUN_MARIADB_FieldSchemaType[] = [];
const fieldsToModify: BUN_MARIADB_FieldSchemaType[] = [];
const fieldsToDrop: string[] = [];
let needsVectorRecreate = false;
for (const field of table.fields || []) {
if (!field.fieldName) continue;
@@ -116,6 +142,8 @@ export default async function updateTable({
const liveField = liveFieldsMap.get(field.fieldName);
if (!liveField) {
// Adding a new vector column can require rebuild if VECTOR INDEX
// constraints conflict; still try surgical add first.
fieldsToAdd.push(field);
} else {
let typeDiverged = !columnTypesMatch(
@@ -123,10 +151,15 @@ export default async function updateTable({
mapDataType(field),
);
if (field.isVector || field.dataType === "VECTOR") {
const dimensions = field.vectorSize || 1536;
const expectedNativeToken = `vector(${dimensions})`;
typeDiverged = liveField.type.toLowerCase() !== expectedNativeToken;
if (isVectorField(field)) {
typeDiverged = vectorTypeDiverged(
liveField.type,
liveField.comment,
field,
);
if (typeDiverged) {
needsVectorRecreate = true;
}
}
if (typeDiverged) {
@@ -135,6 +168,15 @@ export default async function updateTable({
}
}
// Vector dimension / storage type changes → full rebuild automatically
if (needsVectorRecreate) {
console.log(
`Vector column change detected on \`${table.tableName}\`; recreating table`,
);
await recreateTable({ table, config });
return;
}
for (const col of existingColumns) {
if (!codeFieldsMap.has(col.name)) {
fieldsToDrop.push(col.name);
@@ -208,7 +250,7 @@ export default async function updateTable({
try {
await modifyColumn({ tableName: table.tableName, field, config });
} catch (err: any) {
if (field.isVector || field.dataType === "VECTOR") {
if (isVectorField(field)) {
console.warn(
`[Vector Resize] Re-aligning dimension spaces natively via safe migration schema rebuild.`,
);