Moduletrace/bun-mariadb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updates

Browse Source
This commit is contained in:
tben
2026-07-14 09:10:30 +01:00
parent 79c5b896fe
commit 56462e6cc2
38 changed files with 1056 additions and 1720 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/schema/build-column-definition.ts
+9 -2
View File
@@ -1,4 +1,5 @@
import type { BUN_MARIADB_FieldSchemaType } from "../../types";
import isVectorField from "./is-vector-field";
import mapDataType from "./map-data-types";
import MariaDBQuoteGen from "./mariadb-quote-gen";
@@ -16,13 +17,19 @@ export default function buildColumnDefinition(
parts.push("AUTO_INCREMENT");
}
if (field.notNullValue || field.primaryKey || field.isVector) {
// Vector columns used in VECTOR INDEX must be NOT NULL
if (
field.notNullValue ||
field.primaryKey ||
isVectorField(field)
) {
if (!field.primaryKey) {
parts.push("NOT NULL");
}
}
if (field.unique && !field.primaryKey) {
// VECTOR columns cannot be UNIQUE in the usual sense
if (field.unique && !field.primaryKey && !isVectorField(field)) {
parts.push("UNIQUE");
}
src/lib/schema/handle-db-schema-table.ts
+14 -14
View File
@@ -19,26 +19,23 @@ export default async function handleDBSchemaTable({
}: CreateDBSchemaTableHandlerParams) {
const resolvedTable = resolveTable(table, db_schema);
const schemaCond = schemaCondition(config);
const liveTables = await querySchemaRows<{ TABLE_NAME: string }>({
query: `SELECT TABLE_NAME FROM information_schema.TABLES WHERE ${schemaCond.where} AND TABLE_TYPE = 'BASE TABLE'`,
values: schemaCond.values,
config,
});
const liveTableNames = liveTables.map((t) => t.TABLE_NAME);
let tableExistsTracked = Boolean(db_manager_table_name);
let tableExistsLive = Boolean(
existing_live_table?.TABLE_NAME ||
liveTableNames.includes(resolvedTable.tableName),
);
let tableExistsLive = Boolean(existing_live_table?.TABLE_NAME);
let wasRenamed = false;
if (
resolvedTable.tableNameOld &&
resolvedTable.tableNameOld !== resolvedTable.tableName
) {
if (liveTableNames.includes(resolvedTable.tableNameOld)) {
// Only hit information_schema when a rename is declared
const schemaCond = schemaCondition(config);
const liveTables = await querySchemaRows<{ TABLE_NAME: string }>({
query: `SELECT TABLE_NAME FROM information_schema.TABLES WHERE ${schemaCond.where} AND TABLE_TYPE = 'BASE TABLE' AND TABLE_NAME = ?`,
values: [...schemaCond.values, resolvedTable.tableNameOld],
config,
});
if (liveTables.length > 0) {
console.log(
`Renaming table: ${resolvedTable.tableNameOld} -> ${resolvedTable.tableName}`,
);
@@ -68,7 +65,10 @@ export default async function handleDBSchemaTable({
});
} else {
if (!wasRenamed) {
await updateTable({ table: resolvedTable, config });
await updateTable({
table: resolvedTable,
config,
});
}
await upsertDbManagerTable({
tableName: resolvedTable.tableName,
src/lib/schema/is-vector-field.ts
+11
View File
@@ -0,0 +1,11 @@
import type { BUN_MARIADB_FieldSchemaType } from "../../types";
/**
* True when a field is a MariaDB native vector column.
*/
export default function isVectorField(
field?: BUN_MARIADB_FieldSchemaType,
): boolean {
if (!field) return false;
return field.isVector === true || field.dataType === "VECTOR";
}
src/lib/schema/map-data-types.ts
+3 -6
View File
@@ -6,8 +6,9 @@ export default function mapDataType(
const dataType = field.dataType?.toUpperCase() || "TEXT";
const vectorSize = field.vectorSize || 1536;
if (field.isVector) {
return `LONGTEXT COMMENT 'vector_size=${vectorSize}'`;
// Native MariaDB VECTOR type (11.7+). Prefer this over LONGTEXT storage.
if (field.isVector || dataType === "VECTOR") {
return `VECTOR(${vectorSize})`;
}
switch (dataType) {
@@ -78,10 +79,6 @@ export default function mapDataType(
return "JSON";
case "INET6":
return "INET6";
case "VECTOR": {
const dimensions = field.vectorSize || 1536;
return `VECTOR(${dimensions})`;
}
case "BOOLEAN":
return "TINYINT(1)";
case "ENUM": {
src/lib/schema/recreate-table.ts
+56
View File
@@ -25,6 +25,10 @@ async function checkIfTableExists({
return Boolean(rows[0]?.table_exists);
}
/**
* Full table rebuild. For `isVector` tables this drops and recreates in place
* (preserving rows when possible). For regular tables it uses a temp-table swap.
*/
export default async function recreateTable({
table,
config,
@@ -42,6 +46,58 @@ export default async function recreateTable({
return;
}
/**
* Vector tables: drop + recreate + reinsert (MariaDB VECTOR INDEX / dim
* changes are not reliably alterable in place).
*/
if (table.isVector) {
console.log(`Recreating vector table: ${table.tableName}`);
const existingRows = await querySchemaRows<Record<string, any>>({
query: `SELECT * FROM ${MariaDBQuoteGen(table.tableName)}`,
config,
});
await runSchemaQuery({ query: `SET FOREIGN_KEY_CHECKS = 0`, config });
try {
await runSchemaQuery({
query: `DROP TABLE IF EXISTS ${MariaDBQuoteGen(table.tableName)}`,
config,
});
await createTable({ table, config });
} finally {
await runSchemaQuery({ query: `SET FOREIGN_KEY_CHECKS = 1`, config });
}
if (existingRows.length > 0) {
const schemaFieldNames = new Set(
(table.fields || [])
.map((f) => f.fieldName)
.filter((n): n is string => Boolean(n)),
);
for (const row of existingRows) {
const columns = Object.keys(row).filter((c) =>
schemaFieldNames.has(c),
);
if (columns.length === 0) continue;
const placeholders = columns.map(() => "?").join(", ");
const columnList = columns
.map((c) => MariaDBQuoteGen(c))
.join(", ");
await runSchemaQuery({
query: `INSERT INTO ${MariaDBQuoteGen(table.tableName)} (${columnList}) VALUES (${placeholders})`,
values: columns.map((c) => row[c] ?? null),
config,
});
}
}
return;
}
const tempTableName = `${table.tableName}_temp_${Date.now()}`;
const backupOldTableName = `${table.tableName}_old_${Date.now()}`;
const existingColumns = await getTableColumns({
src/lib/schema/sync-indexes.ts
+24 -12
View File
@@ -1,11 +1,33 @@
import type {
BUN_MARIADB_IndexSchemaType,
BUN_MARIADB_TableSchemaType,
BunMariaDBConfig,
} from "../../types";
import isVectorField from "./is-vector-field";
import MariaDBQuoteGen from "./mariadb-quote-gen";
import runSchemaQuery, { querySchemaRows } from "./run-schema-query";
import schemaCondition from "./schema-condition";
function isVectorIndexDef(
index: BUN_MARIADB_IndexSchemaType,
table: BUN_MARIADB_TableSchemaType,
): boolean {
if (index.indexType === "VECTOR") return true;
if (table.isVector) return true;
const firstFieldName = index.indexTableFields?.[0];
if (!firstFieldName) return false;
const field = table.fields?.find((f) => f.fieldName === firstFieldName);
return isVectorField(field);
}
function vectorDistanceMetric(index: BUN_MARIADB_IndexSchemaType): string {
if (index.vectorDistanceMetric === "cosine") return "cosine";
if (index.vectorDistanceMetric === "euclidean") return "euclidean";
return "euclidean";
}
export default async function syncIndexes({
table,
config,
@@ -121,20 +143,10 @@ export default async function syncIndexes({
}
if (!existingIndexesMap.has(index.indexName)) {
const isVectorIndex =
table.isVector ||
table.fields?.some(
(f) =>
f.fieldName === index.indexTableFields![0] && f.isVector,
);
if (isVectorIndex) {
if (isVectorIndexDef(index, table)) {
console.log(`Creating Vector index: ${index.indexName}`);
const targetField = MariaDBQuoteGen(index.indexTableFields[0]!);
const distanceMetric =
(index.indexType as string)?.toLowerCase() === "cosine"
? "cosine"
: "euclidean";
const distanceMetric = vectorDistanceMetric(index);
await runSchemaQuery({
query: `ALTER TABLE ${MariaDBQuoteGen(table.tableName)} ADD VECTOR INDEX ${MariaDBQuoteGen(index.indexName)} (${targetField}) M=8 DISTANCE=${distanceMetric}`,
src/lib/schema/update-table.ts
+47 -5
View File
@@ -6,6 +6,7 @@ import type {
import buildColumnDefinition from "./build-column-definition";
import createTable from "./create-table";
import getTableColumns from "./get-table-columns";
import isVectorField from "./is-vector-field";
import mapDataType from "./map-data-types";
import MariaDBQuoteGen from "./mariadb-quote-gen";
import recreateTable from "./recreate-table";
@@ -29,6 +30,30 @@ function columnTypesMatch(liveType: string, expectedType: string): boolean {
return false;
}
function vectorTypeDiverged(
liveType: string,
liveComment: string,
field: BUN_MARIADB_FieldSchemaType,
): boolean {
const dimensions = field.vectorSize || 1536;
const expectedNative = `vector(${dimensions})`;
const live = liveType.toLowerCase().replace(/\s+/g, "");
if (live === expectedNative) return false;
// Legacy LONGTEXT storage with vector_size comment
if (live.startsWith("longtext") || live.startsWith("text")) {
const match = liveComment.match(/vector_size\s*=\s*(\d+)/i);
if (match && Number(match[1]) === dimensions) {
// Still legacy storage — treat as diverged so we can migrate to VECTOR
return true;
}
return true;
}
return true;
}
async function addColumn({
tableName,
field,
@@ -109,6 +134,7 @@ export default async function updateTable({
const fieldsToAdd: BUN_MARIADB_FieldSchemaType[] = [];
const fieldsToModify: BUN_MARIADB_FieldSchemaType[] = [];
const fieldsToDrop: string[] = [];
let needsVectorRecreate = false;
for (const field of table.fields || []) {
if (!field.fieldName) continue;
@@ -116,6 +142,8 @@ export default async function updateTable({
const liveField = liveFieldsMap.get(field.fieldName);
if (!liveField) {
// Adding a new vector column can require rebuild if VECTOR INDEX
// constraints conflict; still try surgical add first.
fieldsToAdd.push(field);
} else {
let typeDiverged = !columnTypesMatch(
@@ -123,10 +151,15 @@ export default async function updateTable({
mapDataType(field),
);
if (field.isVector || field.dataType === "VECTOR") {
const dimensions = field.vectorSize || 1536;
const expectedNativeToken = `vector(${dimensions})`;
typeDiverged = liveField.type.toLowerCase() !== expectedNativeToken;
if (isVectorField(field)) {
typeDiverged = vectorTypeDiverged(
liveField.type,
liveField.comment,
field,
);
if (typeDiverged) {
needsVectorRecreate = true;
}
}
if (typeDiverged) {
@@ -135,6 +168,15 @@ export default async function updateTable({
}
}
// Vector dimension / storage type changes → full rebuild automatically
if (needsVectorRecreate) {
console.log(
`Vector column change detected on \`${table.tableName}\`; recreating table`,
);
await recreateTable({ table, config });
return;
}
for (const col of existingColumns) {
if (!codeFieldsMap.has(col.name)) {
fieldsToDrop.push(col.name);
@@ -208,7 +250,7 @@ export default async function updateTable({
try {
await modifyColumn({ tableName: table.tableName, field, config });
} catch (err: any) {
if (field.isVector || field.dataType === "VECTOR") {
if (isVectorField(field)) {
console.warn(
`[Vector Resize] Re-aligning dimension spaces natively via safe migration schema rebuild.`,
);