Moduletrace/bun-mariadb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update vector columns logic

Browse Source
This commit is contained in:
tben
2026-08-05 13:28:57 +01:00
parent 4f203ce4e7
commit c97bedc271
7 changed files with 142 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
-1
View File
@@ -902,7 +902,6 @@ bun-mariadb/
│ │ ├── create-db-schema.ts
│ │ ├── create-table.ts
│ │ ├── update-table.ts
│ │ ├── recreate-table.ts
│ │ ├── sync-indexes.ts
│ │ └── ...
│ ├── types/
dist/lib/schema/recreate-table.d.ts
Vendored
-8
View File
@@ -1,8 +0,0 @@
import type { BUN_MARIADB_TableSchemaType, BunMariaDBConfig } from "../../types";
/**
* Full table rebuild using a temp-table swap (preserving rows when possible).
*/
export default function recreateTable({ table, config, }: {
table: BUN_MARIADB_TableSchemaType;
config?: BunMariaDBConfig;
}): Promise<void>;
dist/lib/schema/recreate-table.js
Vendored
-68
View File
@@ -1,68 +0,0 @@
import createTable from "./create-table";
import getTableColumns from "./get-table-columns";
import MariaDBQuoteGen from "./mariadb-quote-gen";
import runSchemaQuery, { querySchemaRows } from "./run-schema-query";
import schemaCondition from "./schema-condition";
async function checkIfTableExists({ tableName, config, }) {
const schemaCond = schemaCondition(config);
const rows = await querySchemaRows({
query: `SELECT 1 AS \`table_exists\` FROM information_schema.TABLES WHERE ${schemaCond.where} AND TABLE_NAME = ? LIMIT 1`,
values: [...schemaCond.values, tableName],
config,
});
return Boolean(rows[0]?.table_exists);
}
/**
* Full table rebuild using a temp-table swap (preserving rows when possible).
*/
export default async function recreateTable({ table, config, }) {
const doesTableExist = await checkIfTableExists({
tableName: table.tableName,
config,
});
if (!doesTableExist) {
await createTable({ table, config });
return;
}
const tempTableName = `${table.tableName}_temp_${Date.now()}`;
const backupOldTableName = `${table.tableName}_old_${Date.now()}`;
const existingColumns = await getTableColumns({
tableName: table.tableName,
config,
});
const columnsToKeep = (table.fields || [])
.filter((field) => existingColumns.some((column) => column.name === field.fieldName))
.map((field) => field.fieldName)
.filter((fieldName) => Boolean(fieldName));
await createTable({
table: { ...table, tableName: tempTableName },
config,
});
if (columnsToKeep.length > 0) {
const columnList = columnsToKeep
.map((column) => MariaDBQuoteGen(column))
.join(", ");
await runSchemaQuery({
query: `INSERT INTO ${MariaDBQuoteGen(tempTableName)} (${columnList}) SELECT ${columnList} FROM ${MariaDBQuoteGen(table.tableName)}`,
config,
});
}
await runSchemaQuery({ query: `SET FOREIGN_KEY_CHECKS = 0`, config });
try {
await runSchemaQuery({
query: `RENAME TABLE ${MariaDBQuoteGen(table.tableName)} TO ${MariaDBQuoteGen(backupOldTableName)}`,
config,
});
await runSchemaQuery({
query: `RENAME TABLE ${MariaDBQuoteGen(tempTableName)} TO ${MariaDBQuoteGen(table.tableName)}`,
config,
});
await runSchemaQuery({
query: `DROP TABLE ${MariaDBQuoteGen(backupOldTableName)}`,
config,
});
}
finally {
await runSchemaQuery({ query: `SET FOREIGN_KEY_CHECKS = 1`, config });
}
}
dist/lib/schema/update-table.js
Vendored
+57 -20
View File
@@ -4,7 +4,6 @@ import getTableColumns, {} from "./get-table-columns";
import isVectorField from "./is-vector-field";
import mapDataType from "./map-data-types";
import MariaDBQuoteGen from "./mariadb-quote-gen";
import recreateTable from "./recreate-table";
import runSchemaQuery, { querySchemaRows } from "./run-schema-query";
import schemaCondition from "./schema-condition";
import { dropForeignKeysOnColumns } from "./sync-foreign-keys";
@@ -141,6 +140,50 @@ async function dropColumn({ tableName, fieldName, config, }) {
config,
});
}
/**
* Drop + re-add a column (values discarded). Used when VECTOR dimensions change —
* MODIFY cannot resize VECTOR, and a full table rebuild is unnecessary.
*/
async function recreateColumn({ tableName, field, config, }) {
if (!field.fieldName)
return;
console.log(`Recreating column: ${tableName}.${field.fieldName} (values will be cleared)`);
await dropForeignKeysOnColumns({
tableName,
columns: [field.fieldName],
config,
});
const schemaCond = schemaCondition(config);
const indexRows = await querySchemaRows({
query: `SELECT INDEX_NAME, COLUMN_NAME FROM information_schema.STATISTICS WHERE ${schemaCond.where} AND TABLE_NAME = ? AND INDEX_NAME <> 'PRIMARY'`,
values: [...schemaCond.values, tableName],
config,
});
const indexesToDrop = new Set();
for (const row of indexRows) {
if (row.COLUMN_NAME === field.fieldName) {
indexesToDrop.add(row.INDEX_NAME);
}
}
for (const indexName of indexesToDrop) {
console.log(`Dropping index ${indexName} because column ${field.fieldName} is being recreated`);
try {
await runSchemaQuery({
query: `DROP INDEX ${MariaDBQuoteGen(indexName)} ON ${MariaDBQuoteGen(tableName)}`,
config,
});
}
catch (err) {
if (String(err?.message || "").includes("needed in a foreign key constraint")) {
console.warn(`Skipping drop of index ${indexName}: required by a foreign key constraint`);
continue;
}
throw err;
}
}
await dropColumn({ tableName, fieldName: field.fieldName, config });
await addColumn({ tableName, field, config });
}
export default async function updateTable({ table, config, }) {
const existingColumns = await getTableColumns({
tableName: table.tableName,
@@ -154,8 +197,8 @@ export default async function updateTable({ table, config, }) {
const codeFieldsMap = new Map((table.fields || []).map((f) => [f.fieldName, f]));
const fieldsToAdd = [];
const fieldsToModify = [];
const fieldsToRecreate = [];
const fieldsToDrop = [];
let needsVectorRecreate = false;
for (const field of table.fields || []) {
if (!field.fieldName)
continue;
@@ -168,7 +211,9 @@ export default async function updateTable({ table, config, }) {
if (isVectorField(field)) {
typeDiverged = vectorTypeDiverged(liveField.type, liveField.comment || "", field);
if (typeDiverged) {
needsVectorRecreate = true;
// VECTOR dimensions / storage cannot be MODIFYed — drop + re-add column
fieldsToRecreate.push(field);
continue;
}
}
const attrsDiverged = !typeDiverged && columnAttributesDiverged(liveField, field);
@@ -177,12 +222,6 @@ export default async function updateTable({ table, config, }) {
}
}
}
// Vector dimension / storage type changes → full rebuild automatically
if (needsVectorRecreate) {
console.log(`Vector column change detected on \`${table.tableName}\`; recreating table`);
await recreateTable({ table, config });
return;
}
for (const col of existingColumns) {
if (!codeFieldsMap.has(col.name)) {
fieldsToDrop.push(col.name);
@@ -190,6 +229,7 @@ export default async function updateTable({ table, config, }) {
}
if (fieldsToAdd.length === 0 &&
fieldsToModify.length === 0 &&
fieldsToRecreate.length === 0 &&
fieldsToDrop.length === 0) {
return;
}
@@ -257,17 +297,14 @@ export default async function updateTable({ table, config, }) {
await addColumn({ tableName: table.tableName, field, config });
}
for (const field of fieldsToModify) {
try {
await modifyColumn({ tableName: table.tableName, field, config });
}
catch (err) {
if (isVectorField(field)) {
console.warn(`[Vector Resize] Re-aligning dimension spaces natively via safe migration schema rebuild.`);
await recreateTable({ table, config });
return;
}
throw err;
}
await modifyColumn({ tableName: table.tableName, field, config });
}
for (const field of fieldsToRecreate) {
await recreateColumn({
tableName: table.tableName,
field,
config,
});
}
for (const fieldName of fieldsToDrop) {
await dropColumn({ tableName: table.tableName, fieldName, config });
package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@moduletrace/bun-mariadb",
"version": "1.0.11",
"version": "1.0.12",
"description": "Schema-driven MariaDB manager for Bun",
"author": "Benjamin Toby",
"license": "MIT",
src/lib/schema/recreate-table.ts
-95
View File
@@ -1,95 +0,0 @@
import type {
BUN_MARIADB_TableSchemaType,
BunMariaDBConfig,
} from "../../types";
import createTable from "./create-table";
import getTableColumns from "./get-table-columns";
import MariaDBQuoteGen from "./mariadb-quote-gen";
import runSchemaQuery, { querySchemaRows } from "./run-schema-query";
import schemaCondition from "./schema-condition";
async function checkIfTableExists({
tableName,
config,
}: {
tableName: string;
config?: BunMariaDBConfig;
}): Promise<boolean> {
const schemaCond = schemaCondition(config);
const rows = await querySchemaRows<{ table_exists: number }>({
query: `SELECT 1 AS \`table_exists\` FROM information_schema.TABLES WHERE ${schemaCond.where} AND TABLE_NAME = ? LIMIT 1`,
values: [...schemaCond.values, tableName],
config,
});
return Boolean(rows[0]?.table_exists);
}
/**
* Full table rebuild using a temp-table swap (preserving rows when possible).
*/
export default async function recreateTable({
table,
config,
}: {
table: BUN_MARIADB_TableSchemaType;
config?: BunMariaDBConfig;
}): Promise<void> {
const doesTableExist = await checkIfTableExists({
tableName: table.tableName,
config,
});
if (!doesTableExist) {
await createTable({ table, config });
return;
}
const tempTableName = `${table.tableName}_temp_${Date.now()}`;
const backupOldTableName = `${table.tableName}_old_${Date.now()}`;
const existingColumns = await getTableColumns({
tableName: table.tableName,
config,
});
const columnsToKeep = (table.fields || [])
.filter((field) =>
existingColumns.some((column) => column.name === field.fieldName),
)
.map((field) => field.fieldName)
.filter((fieldName): fieldName is string => Boolean(fieldName));
await createTable({
table: { ...table, tableName: tempTableName },
config,
});
if (columnsToKeep.length > 0) {
const columnList = columnsToKeep
.map((column) => MariaDBQuoteGen(column))
.join(", ");
await runSchemaQuery({
query: `INSERT INTO ${MariaDBQuoteGen(tempTableName)} (${columnList}) SELECT ${columnList} FROM ${MariaDBQuoteGen(table.tableName)}`,
config,
});
}
await runSchemaQuery({ query: `SET FOREIGN_KEY_CHECKS = 0`, config });
try {
await runSchemaQuery({
query: `RENAME TABLE ${MariaDBQuoteGen(table.tableName)} TO ${MariaDBQuoteGen(backupOldTableName)}`,
config,
});
await runSchemaQuery({
query: `RENAME TABLE ${MariaDBQuoteGen(tempTableName)} TO ${MariaDBQuoteGen(table.tableName)}`,
config,
});
await runSchemaQuery({
query: `DROP TABLE ${MariaDBQuoteGen(backupOldTableName)}`,
config,
});
} finally {
await runSchemaQuery({ query: `SET FOREIGN_KEY_CHECKS = 1`, config });
}
}
src/lib/schema/update-table.ts
+84 -24
View File
@@ -11,7 +11,6 @@ import getTableColumns, { type ColumnInfoRow } from "./get-table-columns";
import isVectorField from "./is-vector-field";
import mapDataType from "./map-data-types";
import MariaDBQuoteGen from "./mariadb-quote-gen";
import recreateTable from "./recreate-table";
import runSchemaQuery, { querySchemaRows } from "./run-schema-query";
import schemaCondition from "./schema-condition";
import { dropForeignKeysOnColumns } from "./sync-foreign-keys";
@@ -197,6 +196,76 @@ async function dropColumn({
});
}
/**
* Drop + re-add a column (values discarded). Used when VECTOR dimensions change —
* MODIFY cannot resize VECTOR, and a full table rebuild is unnecessary.
*/
async function recreateColumn({
tableName,
field,
config,
}: {
tableName: string;
field: BUN_MARIADB_FieldSchemaType;
config?: BunMariaDBConfig;
}): Promise<void> {
if (!field.fieldName) return;
console.log(
`Recreating column: ${tableName}.${field.fieldName} (values will be cleared)`,
);
await dropForeignKeysOnColumns({
tableName,
columns: [field.fieldName],
config,
});
const schemaCond = schemaCondition(config);
const indexRows = await querySchemaRows<{
INDEX_NAME: string;
COLUMN_NAME: string;
}>({
query: `SELECT INDEX_NAME, COLUMN_NAME FROM information_schema.STATISTICS WHERE ${schemaCond.where} AND TABLE_NAME = ? AND INDEX_NAME <> 'PRIMARY'`,
values: [...schemaCond.values, tableName],
config,
});
const indexesToDrop = new Set<string>();
for (const row of indexRows) {
if (row.COLUMN_NAME === field.fieldName) {
indexesToDrop.add(row.INDEX_NAME);
}
}
for (const indexName of indexesToDrop) {
console.log(
`Dropping index ${indexName} because column ${field.fieldName} is being recreated`,
);
try {
await runSchemaQuery({
query: `DROP INDEX ${MariaDBQuoteGen(indexName)} ON ${MariaDBQuoteGen(tableName)}`,
config,
});
} catch (err: any) {
if (
String(err?.message || "").includes(
"needed in a foreign key constraint",
)
) {
console.warn(
`Skipping drop of index ${indexName}: required by a foreign key constraint`,
);
continue;
}
throw err;
}
}
await dropColumn({ tableName, fieldName: field.fieldName, config });
await addColumn({ tableName, field, config });
}
export default async function updateTable({
table,
config,
@@ -223,8 +292,8 @@ export default async function updateTable({
const fieldsToAdd: BUN_MARIADB_FieldSchemaType[] = [];
const fieldsToModify: BUN_MARIADB_FieldSchemaType[] = [];
const fieldsToRecreate: BUN_MARIADB_FieldSchemaType[] = [];
const fieldsToDrop: string[] = [];
let needsVectorRecreate = false;
for (const field of table.fields || []) {
if (!field.fieldName) continue;
@@ -246,7 +315,9 @@ export default async function updateTable({
field,
);
if (typeDiverged) {
needsVectorRecreate = true;
// VECTOR dimensions / storage cannot be MODIFYed — drop + re-add column
fieldsToRecreate.push(field);
continue;
}
}
@@ -259,15 +330,6 @@ export default async function updateTable({
}
}
// Vector dimension / storage type changes → full rebuild automatically
if (needsVectorRecreate) {
console.log(
`Vector column change detected on \`${table.tableName}\`; recreating table`,
);
await recreateTable({ table, config });
return;
}
for (const col of existingColumns) {
if (!codeFieldsMap.has(col.name)) {
fieldsToDrop.push(col.name);
@@ -277,6 +339,7 @@ export default async function updateTable({
if (
fieldsToAdd.length === 0 &&
fieldsToModify.length === 0 &&
fieldsToRecreate.length === 0 &&
fieldsToDrop.length === 0
) {
return;
@@ -370,18 +433,15 @@ export default async function updateTable({
}
for (const field of fieldsToModify) {
try {
await modifyColumn({ tableName: table.tableName, field, config });
} catch (err: any) {
if (isVectorField(field)) {
console.warn(
`[Vector Resize] Re-aligning dimension spaces natively via safe migration schema rebuild.`,
);
await recreateTable({ table, config });
return;
}
throw err;
}
await modifyColumn({ tableName: table.tableName, field, config });
}
for (const field of fieldsToRecreate) {
await recreateColumn({
tableName: table.tableName,
field,
config,
});
}
for (const fieldName of fieldsToDrop) {