diff --git a/bun.lock b/bun.lock index 2533766..2f2abf5 100644 --- a/bun.lock +++ b/bun.lock @@ -12,6 +12,7 @@ "lodash": "^4.17.23", "mariadb": "^3.5.3", "mysql": "^2.18.1", + "sanitize-html": "^2.17.6", }, "devDependencies": { "@types/bun": "latest", @@ -19,6 +20,7 @@ "@types/lodash": "^4.17.24", "@types/mysql": "^2.15.27", "@types/node": "^25.3.3", + "@types/sanitize-html": "^2.16.1", }, "peerDependencies": { "typescript": "^5", @@ -70,6 +72,8 @@ "@types/node": ["@types/node@25.9.4", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-dszCsrKb5U7ZsVZBWiHFklTloVl0mSEnWH/iZXfZUlI4rzCUnsvGmgqfuVRHL54ugE7/wRuxEIXRa2iMZ+BG6g=="], + "@types/sanitize-html": ["@types/sanitize-html@2.16.1", "", { "dependencies": { "htmlparser2": "^10.1" } }, "sha512-n9wjs8bCOTyN/ynwD8s/nTcTreIHB1vf31vhLMGqUPNHaweKC4/fAl4Dj+hUlCTKYgm4P3k83fmiFfzkZ6sgMA=="], + "@types/through": ["@types/through@0.0.33", "", { "dependencies": { "@types/node": "*" } }, "sha512-HsJ+z3QuETzP3cswwtzt2vEIiHBk/dCcHGhbmG5X3ecnwFD/lPrMpliGXxSCg03L9AhrdwA4Oz/qfspkDW+xGQ=="], "bignumber.js": ["bignumber.js@9.0.0", "", {}, "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A=="], @@ -86,22 +90,44 @@ "core-util-is": ["core-util-is@1.0.3", "", {}, "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="], + "dayjs": ["dayjs@1.11.21", "", {}, "sha512-98IT+HOahAisibz/yjKbzuOBwYcjJ7BCLPzARyHiyEBmRz4fatF+KPJszEHXsGYjUG234aH/cOjW1wwTbKUZlA=="], + + "deepmerge": ["deepmerge@4.3.1", "", {}, "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A=="], + "denque": ["denque@2.1.0", "", {}, "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="], + "dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="], + + "domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="], + + "domhandler": ["domhandler@5.0.3", "", { "dependencies": { "domelementtype": "^2.3.0" } }, "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w=="], + + "domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="], + + "entities": ["entities@7.0.1", "", {}, "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA=="], + + "escape-string-regexp": ["escape-string-regexp@4.0.0", "", {}, "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="], + "fast-string-truncated-width": ["fast-string-truncated-width@3.0.3", "", {}, "sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g=="], "fast-string-width": ["fast-string-width@3.0.2", "", { "dependencies": { "fast-string-truncated-width": "^3.0.2" } }, "sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg=="], "fast-wrap-ansi": ["fast-wrap-ansi@0.2.2", "", { "dependencies": { "fast-string-width": "^3.0.2" } }, "sha512-7F2Fl+TjRSenLqlU3UjSH0iyqopqoZIu7eZVpEirP2g1GtWa2G/ecEmBdgz31+Mxr+ELclgg6sokpSFIQiZ02Q=="], + "htmlparser2": ["htmlparser2@10.1.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "entities": "^7.0.1" } }, "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ=="], + "iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], "inquirer": ["inquirer@13.4.3", "", { "dependencies": { "@inquirer/ansi": "^2.0.5", "@inquirer/core": "^11.1.10", "@inquirer/prompts": "^8.4.3", "@inquirer/type": "^4.0.5", "mute-stream": "^3.0.0", "run-async": "^4.0.6", "rxjs": "^7.8.2" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-EPd3IqieHSavSOXh+LZhrIkdQcOELWeRblLT6kslQr+cF9XTh/HxZdSt1YkHH1iq4dvqBnV42uwg2YlorgOy6g=="], + "is-plain-object": ["is-plain-object@5.0.0", "", {}, "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q=="], + "isarray": ["isarray@1.0.0", "", {}, "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="], + "launder": ["launder@1.7.1", "", { "dependencies": { "dayjs": "^1.11.7" } }, "sha512-mU6WRz5EusL9ZZuiZ5SO4Y6C0P9PAUR9iwdb6bzj4KDihm28DiHFw+/yk9DBH4f+Pv1wuzQ4e2jV3oQ7mkIqvw=="], + "lodash": ["lodash@4.18.1", "", {}, "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q=="], "lru-cache": ["lru-cache@11.5.1", "", {}, "sha512-RPimw/7aMdv2oqRrxKwvZXcPfwBrn/JZ2xYcY9Hus/6LaS3VOAKVWKWgNLCFSiOm1ESXinjsDlidVU7JlnCN2A=="], @@ -112,6 +138,14 @@ "mysql": ["mysql@2.18.1", "", { "dependencies": { "bignumber.js": "9.0.0", "readable-stream": "2.3.7", "safe-buffer": "5.1.2", "sqlstring": "2.3.1" } }, "sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig=="], + "nanoid": ["nanoid@3.3.16", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q=="], + + "parse-srcset": ["parse-srcset@1.0.2", "", {}, "sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q=="], + + "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], + + "postcss": ["postcss@8.5.19", "", { "dependencies": { "nanoid": "^3.3.12", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-Mz8SaolMd8nB+G13WkORcxQKHZ/NE4xXevtkJHVuG+guo9/wYKlIMTKAqGdEmYOXR2ijPjTYNHssizdaVSUNdQ=="], + "process-nextick-args": ["process-nextick-args@2.0.1", "", {}, "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="], "readable-stream": ["readable-stream@2.3.7", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw=="], @@ -124,8 +158,12 @@ "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], + "sanitize-html": ["sanitize-html@2.17.6", "", { "dependencies": { "deepmerge": "^4.2.2", "escape-string-regexp": "^4.0.0", "htmlparser2": "^12.0.0", "is-plain-object": "^5.0.0", "launder": "^1.7.1", "parse-srcset": "^1.0.2", "postcss": "^8.3.11" } }, "sha512-M4bo9tfv1yfhQZZKkc6dL07ALrGJtfvNOuhX3hU9AVPR/uPQ+nKOJBqTYc7LfMQblTW04mtSWDJWEyLvygJsLA=="], + "signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="], + "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], + "sqlstring": ["sqlstring@2.3.1", "", {}, "sha512-ooAzh/7dxIG5+uDik1z/Rd1vli0+38izZhGzSa34FwR7IbelPWCCKSNIl8jlL/F7ERvy8CB2jNeM1E9i9mXMAQ=="], "string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], @@ -137,5 +175,19 @@ "undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], + + "dom-serializer/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="], + + "sanitize-html/htmlparser2": ["htmlparser2@12.0.0", "", { "dependencies": { "domelementtype": "^3.0.0", "domhandler": "^6.0.0", "domutils": "^4.0.2", "entities": "^8.0.0" } }, "sha512-Tz7u1i95/g2x2jz81+x0FBVhBhY5aRTvD3tXXdFaljuNdzDLJ8UGNRrTcj2cgQvAg3iW/h77Fz15nLW0L0CrZw=="], + + "sanitize-html/htmlparser2/domelementtype": ["domelementtype@3.0.0", "", {}, "sha512-umCQid3jKbDmVjx8jGaW7uUykm4DEUeyV21hPxNMo2nV955DhUThwqyOIDtreepP31hl84X7G5U9ZfsWvIB3Pg=="], + + "sanitize-html/htmlparser2/domhandler": ["domhandler@6.0.1", "", { "dependencies": { "domelementtype": "^3.0.0" } }, "sha512-gYzvtM72ZtxQO0T048kd6HWSbbGCNOUwcnfQ01cqIJ4X2IYKFFHZ5mKvrQETcFXxsRObZulDaKmy//R7TPtsBg=="], + + "sanitize-html/htmlparser2/domutils": ["domutils@4.0.2", "", { "dependencies": { "dom-serializer": "^3.0.0", "domelementtype": "^3.0.0", "domhandler": "^6.0.0" } }, "sha512-qI4JLRKnSzqFqr7hAlS5xQDusBCjKSEG4t4+7aNrIQMHBcsC2TGEhuyABJdYkgSewL57PNLYEiibY2iPKhKpaA=="], + + "sanitize-html/htmlparser2/entities": ["entities@8.0.0", "", {}, "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA=="], + + "sanitize-html/htmlparser2/domutils/dom-serializer": ["dom-serializer@3.1.1", "", { "dependencies": { "domelementtype": "^3.0.0", "domhandler": "^6.0.0", "entities": "^8.0.0" } }, "sha512-4MEa38/QexBob6gFNwu+EGdWvhJ1OKuNwdYY3Y3NyeWDQfnGeDYQUDfIRzWu5B5gsv03so2Uxd28YC6zrsx3Lw=="], } } diff --git a/package.json b/package.json index a250dfd..739e023 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "@types/inquirer": "^9.0.9", "@types/lodash": "^4.17.24", "@types/mysql": "^2.15.27", - "@types/node": "^25.3.3" + "@types/node": "^25.3.3", + "@types/sanitize-html": "^2.16.1" }, "peerDependencies": { "typescript": "^5" @@ -37,6 +38,7 @@ "inquirer": "^13.3.2", "lodash": "^4.17.23", "mariadb": "^3.5.3", - "mysql": "^2.18.1" + "mysql": "^2.18.1", + "sanitize-html": "^2.17.6" } } diff --git a/src/lib/mariadb/db-insert.ts b/src/lib/mariadb/db-insert.ts index 5e74090..89f9fd6 100644 --- a/src/lib/mariadb/db-insert.ts +++ b/src/lib/mariadb/db-insert.ts @@ -5,6 +5,7 @@ import type { SQLInsertGenReturn, } from "../../types"; import sqlInsertGenerator from "../../utils/sql-insert-generator"; +import { sanitizeHtmlFieldsBatch } from "../../utils/sanitize-html-fields"; import grabDuplicateSafeInsertSql from "../grab-duplicate-safe-insert-sql"; type Params< @@ -29,7 +30,13 @@ export default async function DbInsert< let sqlObj: SQLInsertGenReturn | null = null; try { - const finalData: { [k: string]: any }[] = data.map((d) => ({ + const sanitizedData = sanitizeHtmlFieldsBatch({ + table, + data, + config, + }); + + const finalData: { [k: string]: any }[] = sanitizedData.map((d) => ({ created_at: Date.now(), updated_at: Date.now(), ...d, diff --git a/src/lib/mariadb/db-update.ts b/src/lib/mariadb/db-update.ts index 790f055..a4ff941 100644 --- a/src/lib/mariadb/db-update.ts +++ b/src/lib/mariadb/db-update.ts @@ -7,6 +7,7 @@ import type { ServerQueryParam, } from "../../types"; import sqlGenerator from "../../utils/sql-generator"; +import sanitizeHtmlFields from "../../utils/sanitize-html-fields"; type Params< Schema extends { [k: string]: any } = { [k: string]: any }, @@ -69,9 +70,15 @@ export default async function DbUpdate< let sql = ``; sql += `UPDATE ${quoteIdentifier(table)} SET`; + const sanitizedData = sanitizeHtmlFields({ + table, + data, + config, + }); + const finalData: { [k: string]: SQLInsertGenValueType } = { updated_at: Date.now(), - ...data, + ...sanitizedData, }; const keys = Object.keys(finalData); diff --git a/src/lib/schema/build-foreign-key-constraint.ts b/src/lib/schema/build-foreign-key-constraint.ts new file mode 100644 index 0000000..a6d854b --- /dev/null +++ b/src/lib/schema/build-foreign-key-constraint.ts @@ -0,0 +1,23 @@ +import type { BUN_MARIADB_FieldSchemaType } from "../../types"; +import MariaDBQuoteGen from "./mariadb-quote-gen"; + +export default function buildForeignKeyConstraint( + field: BUN_MARIADB_FieldSchemaType, +): string { + const fk = field.foreignKey!; + const constraintName = fk.foreignKeyName + ? `CONSTRAINT ${MariaDBQuoteGen(fk.foreignKeyName)} ` + : ""; + + let constraint = `${constraintName}FOREIGN KEY (${MariaDBQuoteGen(field.fieldName!)}) REFERENCES ${MariaDBQuoteGen(fk.destinationTableName!)}(${MariaDBQuoteGen(fk.destinationTableColumnName!)})`; + + if (fk.cascadeDelete) { + constraint += " ON DELETE CASCADE"; + } + + if (fk.cascadeUpdate) { + constraint += " ON UPDATE CASCADE"; + } + + return constraint; +} diff --git a/src/lib/schema/build-table-options.ts b/src/lib/schema/build-table-options.ts new file mode 100644 index 0000000..9267845 --- /dev/null +++ b/src/lib/schema/build-table-options.ts @@ -0,0 +1,13 @@ +import type { BUN_MARIADB_TableSchemaType } from "../../types"; + +export default function buildTableOptions( + table: BUN_MARIADB_TableSchemaType, +): string { + const options = ["ENGINE=InnoDB"]; + + if (table.collation) { + options.push("DEFAULT CHARSET=utf8mb4", `COLLATE ${table.collation}`); + } + + return ` ${options.join(" ")}`; +} diff --git a/src/lib/schema/create-db-schema.ts b/src/lib/schema/create-db-schema.ts index d6a87c8..809ffe9 100644 --- a/src/lib/schema/create-db-schema.ts +++ b/src/lib/schema/create-db-schema.ts @@ -10,14 +10,12 @@ export default async function createDBSchema(params: CreateDBSchemaParams) { await createDBManagerTable(params); /** - * Reorder Tables + * Reorder Tables (parents before children with FKs) */ const ordered_db_schema = await orderDBSchema(params); /** - * Handle Tables + * Handle Tables (create / update / drop) */ await handleDBSchemaTables({ ...params, db_schema: ordered_db_schema }); - - process.exit(); } diff --git a/src/lib/schema/create-table.ts b/src/lib/schema/create-table.ts new file mode 100644 index 0000000..4344a74 --- /dev/null +++ b/src/lib/schema/create-table.ts @@ -0,0 +1,66 @@ +import type { + BUN_MARIADB_TableSchemaType, + BunMariaDBConfig, +} from "../../types"; +import buildColumnDefinition from "./build-column-definition"; +import buildForeignKeyConstraint from "./build-foreign-key-constraint"; +import buildTableOptions from "./build-table-options"; +import MariaDBQuoteGen from "./mariadb-quote-gen"; +import runSchemaQuery from "./run-schema-query"; + +export default async function createTable({ + table, + config, +}: { + table: BUN_MARIADB_TableSchemaType; + config?: BunMariaDBConfig; +}): Promise { + if (!table.tableName.match(/_temp_\d+$/)) { + console.log(`Creating table: ${table.tableName}`); + } + + const columnDefinitions: string[] = []; + const foreignKeys: string[] = []; + const primaryKeys: string[] = []; + + for (const field of table.fields || []) { + columnDefinitions.push(buildColumnDefinition(field)); + + if (field.primaryKey && field.fieldName) { + primaryKeys.push(field.fieldName); + } + + if (field.foreignKey && !table.isVector) { + foreignKeys.push(buildForeignKeyConstraint(field)); + } + } + + if (primaryKeys.length > 0) { + const pkCols = primaryKeys.map((k) => MariaDBQuoteGen(k)).join(", "); + columnDefinitions.push(`PRIMARY KEY (${pkCols})`); + } + + if (table.uniqueConstraints) { + for (const constraint of table.uniqueConstraints) { + if ( + constraint.constraintTableFields && + constraint.constraintTableFields.length > 0 + ) { + const fields = constraint.constraintTableFields + .map((field) => MariaDBQuoteGen(field.value)) + .join(", "); + const constraintName = + constraint.constraintName || + `unique_${fields.replace(/`/g, "")}`; + + columnDefinitions.push( + `CONSTRAINT ${MariaDBQuoteGen(constraintName)} UNIQUE (${fields})`, + ); + } + } + } + + const sql = `CREATE TABLE IF NOT EXISTS ${MariaDBQuoteGen(table.tableName)} (${[...columnDefinitions, ...foreignKeys].join(", ")})${buildTableOptions(table)}`; + + await runSchemaQuery({ query: sql, config }); +} diff --git a/src/lib/schema/get-table-columns.ts b/src/lib/schema/get-table-columns.ts new file mode 100644 index 0000000..5bc9b83 --- /dev/null +++ b/src/lib/schema/get-table-columns.ts @@ -0,0 +1,34 @@ +import type { BunMariaDBConfig } from "../../types"; +import { querySchemaRows } from "./run-schema-query"; +import schemaCondition from "./schema-condition"; + +export type ColumnInfoRow = { + name: string; + type: string; + comment?: string; +}; + +export default async function getTableColumns({ + tableName, + config, +}: { + tableName: string; + config?: BunMariaDBConfig; +}): Promise { + const schemaCond = schemaCondition(config); + const rows = await querySchemaRows<{ + COLUMN_NAME: string; + COLUMN_TYPE: string; + COLUMN_COMMENT: string; + }>({ + query: `SELECT COLUMN_NAME, COLUMN_TYPE, COLUMN_COMMENT FROM information_schema.COLUMNS WHERE ${schemaCond.where} AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION`, + values: [...schemaCond.values, tableName], + config, + }); + + return rows.map((row) => ({ + name: row.COLUMN_NAME, + type: row.COLUMN_TYPE, + comment: row.COLUMN_COMMENT, + })); +} diff --git a/src/lib/schema/handle-db-schema-table.ts b/src/lib/schema/handle-db-schema-table.ts index 53c7026..f9a5011 100644 --- a/src/lib/schema/handle-db-schema-table.ts +++ b/src/lib/schema/handle-db-schema-table.ts @@ -1,13 +1,14 @@ -import { AppData } from "../../data/app-data"; -import type { - BUN_MARIADB_DB_TABLE_MANAGER_TABLE, - BUN_MARIADB_INFORMATION_SCHEMA_TABLES, - CreateDBSchemaTableHandlerParams, -} from "../../types"; -import dbHandler from "../db-handler"; -import DbInsert from "../mariadb/db-insert"; -import buildColumnDefinition from "./build-column-definition"; +import type { CreateDBSchemaTableHandlerParams } from "../../types"; +import createTable from "./create-table"; import MariaDBQuoteGen from "./mariadb-quote-gen"; +import resolveTable from "./resolve-table"; +import runSchemaQuery, { querySchemaRows } from "./run-schema-query"; +import schemaCondition from "./schema-condition"; +import syncIndexes from "./sync-indexes"; +import updateTable from "./update-table"; +import upsertDbManagerTable, { + removeDbManagerTable, +} from "./upsert-db-manager-table"; export default async function handleDBSchemaTable({ db_schema, @@ -16,31 +17,64 @@ export default async function handleDBSchemaTable({ db_manager_table_name, existing_live_table, }: CreateDBSchemaTableHandlerParams) { - if (!db_manager_table_name || !existing_live_table?.TABLE_NAME) { - const insert_table = await DbInsert( - { - data: [ - { - table_name: table.tableName, - }, - ], - table: AppData["DbSchemaManagerTableName"], + const resolvedTable = resolveTable(table, db_schema); + + const schemaCond = schemaCondition(config); + const liveTables = await querySchemaRows<{ TABLE_NAME: string }>({ + query: `SELECT TABLE_NAME FROM information_schema.TABLES WHERE ${schemaCond.where} AND TABLE_TYPE = 'BASE TABLE'`, + values: schemaCond.values, + config, + }); + const liveTableNames = liveTables.map((t) => t.TABLE_NAME); + + let tableExistsTracked = Boolean(db_manager_table_name); + let tableExistsLive = Boolean( + existing_live_table?.TABLE_NAME || + liveTableNames.includes(resolvedTable.tableName), + ); + let wasRenamed = false; + + if ( + resolvedTable.tableNameOld && + resolvedTable.tableNameOld !== resolvedTable.tableName + ) { + if (liveTableNames.includes(resolvedTable.tableNameOld)) { + console.log( + `Renaming table: ${resolvedTable.tableNameOld} -> ${resolvedTable.tableName}`, + ); + await runSchemaQuery({ + query: `RENAME TABLE ${MariaDBQuoteGen(resolvedTable.tableNameOld)} TO ${MariaDBQuoteGen(resolvedTable.tableName)}`, config, - }, - ); - - let sql = ``; - - sql += `CREATE TABLE IF NOT EXISTS ${MariaDBQuoteGen(table.tableName)}`; - sql += ` (`; - - for (let i = 0; i < table.fields.length; i++) { - const field = table.fields[i]; - if (!field) continue; - const col_def = buildColumnDefinition(field); - sql += ` ${col_def}`; + }); + await upsertDbManagerTable({ + tableName: resolvedTable.tableName, + config, + }); + await removeDbManagerTable({ + tableName: resolvedTable.tableNameOld, + config, + }); + tableExistsTracked = true; + tableExistsLive = true; + wasRenamed = true; } - - sql += ` )`; } + + if (!tableExistsTracked && !tableExistsLive) { + await createTable({ table: resolvedTable, config }); + await upsertDbManagerTable({ + tableName: resolvedTable.tableName, + config, + }); + } else { + if (!wasRenamed) { + await updateTable({ table: resolvedTable, config }); + } + await upsertDbManagerTable({ + tableName: resolvedTable.tableName, + config, + }); + } + + await syncIndexes({ table: resolvedTable, config }); } diff --git a/src/lib/schema/handle-db-schema-tables.ts b/src/lib/schema/handle-db-schema-tables.ts index 31eebdb..0ca2b40 100644 --- a/src/lib/schema/handle-db-schema-tables.ts +++ b/src/lib/schema/handle-db-schema-tables.ts @@ -1,11 +1,15 @@ +import _ from "lodash"; import { AppData } from "../../data/app-data"; import type { BUN_MARIADB_INFORMATION_SCHEMA_TABLES, CreateDBSchemaParams, } from "../../types"; -import dbHandler from "../db-handler"; -import getExistingTablesFromTablesManagerTable from "./get-existing-tables-from-tables-manager-table"; import handleDBSchemaTable from "./handle-db-schema-table"; +import getExistingTablesFromTablesManagerTable from "./get-existing-tables-from-tables-manager-table"; +import MariaDBQuoteGen from "./mariadb-quote-gen"; +import runSchemaQuery, { querySchemaRows } from "./run-schema-query"; +import schemaCondition from "./schema-condition"; +import { removeDbManagerTable } from "./upsert-db-manager-table"; export default async function handleDBSchemaTables( params: CreateDBSchemaParams, @@ -19,34 +23,26 @@ export default async function handleDBSchemaTables( const existing_schema_tables = await getExistingTablesFromTablesManagerTable(params); + const schemaCond = schemaCondition(config); const existing_live_tables = - await dbHandler({ - query: `SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_TYPE = 'BASE TABLE' AND TABLE_NAME != ?`, - config: params.config, - values: [AppData["DbSchemaManagerTableName"]], + await querySchemaRows({ + query: `SELECT TABLE_NAME FROM information_schema.TABLES WHERE ${schemaCond.where} AND TABLE_TYPE = 'BASE TABLE' AND TABLE_NAME != ?`, + values: [...schemaCond.values, AppData["DbSchemaManagerTableName"]], + config, }); for (let i = 0; i < db_schema.tables.length; i++) { const table = db_schema.tables[i]; + if (!table) continue; const existing_table = existing_schema_tables.find( - (t) => t == table?.tableName, + (t) => t == table.tableName, ); - const existing_live_table = existing_live_tables.payload?.find( - (t) => t.TABLE_NAME == table?.tableName, + const existing_live_table = existing_live_tables.find( + (t) => t.TABLE_NAME == table.tableName, ); - if (!table) { - if (existing_live_table?.TABLE_NAME) { - await dbHandler({ - query: `DROP TABLE ${existing_live_table.TABLE_NAME}`, - config: params.config, - }); - } - continue; - } - await handleDBSchemaTable({ ...params, table, @@ -54,4 +50,84 @@ export default async function handleDBSchemaTables( existing_live_table, }); } + + /** + * Drop tables tracked by the manager but no longer in the schema. + * Skip drops when remaining schema tables still reference the table via FK + * (e.g. external tables like `users` that are referenced but not managed). + */ + const schemaTableNames = db_schema.tables.map((t) => t.tableName); + const tablesToDrop = _.uniq( + existing_schema_tables.filter( + (tableName): tableName is string => + Boolean(tableName) && !schemaTableNames.includes(tableName!), + ), + ); + + if (tablesToDrop.length === 0) { + return; + } + + const fkRows = await querySchemaRows<{ + TABLE_NAME: string; + REFERENCED_TABLE_NAME: string; + }>({ + query: `SELECT TABLE_NAME, REFERENCED_TABLE_NAME FROM information_schema.KEY_COLUMN_USAGE WHERE ${schemaCond.where} AND REFERENCED_TABLE_NAME IS NOT NULL`, + values: schemaCond.values, + config, + }); + + const referencedByRemaining = new Map(); + for (const row of fkRows) { + if ( + !row.REFERENCED_TABLE_NAME || + !tablesToDrop.includes(row.REFERENCED_TABLE_NAME) + ) { + continue; + } + + // Referenced by a table we are keeping + if ( + schemaTableNames.includes(row.TABLE_NAME) || + !tablesToDrop.includes(row.TABLE_NAME) + ) { + const list = referencedByRemaining.get(row.REFERENCED_TABLE_NAME) || []; + if (!list.includes(row.TABLE_NAME)) { + list.push(row.TABLE_NAME); + } + referencedByRemaining.set(row.REFERENCED_TABLE_NAME, list); + } + } + + const safeToDrop: string[] = []; + + for (const tableName of tablesToDrop) { + const dependents = referencedByRemaining.get(tableName); + if (dependents && dependents.length > 0) { + console.warn( + `Skipping drop of table \`${tableName}\`: still referenced by ${dependents.join(", ")}. Removing from schema manager tracking only.`, + ); + await removeDbManagerTable({ tableName, config }); + continue; + } + safeToDrop.push(tableName); + } + + if (safeToDrop.length === 0) { + return; + } + + await runSchemaQuery({ query: `SET FOREIGN_KEY_CHECKS = 0`, config }); + try { + for (const tableName of safeToDrop) { + console.log(`Dropping table: ${tableName}`); + await runSchemaQuery({ + query: `DROP TABLE IF EXISTS ${MariaDBQuoteGen(tableName)}`, + config, + }); + await removeDbManagerTable({ tableName, config }); + } + } finally { + await runSchemaQuery({ query: `SET FOREIGN_KEY_CHECKS = 1`, config }); + } } diff --git a/src/lib/schema/recreate-table.ts b/src/lib/schema/recreate-table.ts new file mode 100644 index 0000000..abc9a66 --- /dev/null +++ b/src/lib/schema/recreate-table.ts @@ -0,0 +1,92 @@ +import type { + BUN_MARIADB_TableSchemaType, + BunMariaDBConfig, +} from "../../types"; +import createTable from "./create-table"; +import getTableColumns from "./get-table-columns"; +import MariaDBQuoteGen from "./mariadb-quote-gen"; +import runSchemaQuery, { querySchemaRows } from "./run-schema-query"; +import schemaCondition from "./schema-condition"; + +async function checkIfTableExists({ + tableName, + config, +}: { + tableName: string; + config?: BunMariaDBConfig; +}): Promise { + const schemaCond = schemaCondition(config); + const rows = await querySchemaRows<{ table_exists: number }>({ + query: `SELECT 1 AS \`table_exists\` FROM information_schema.TABLES WHERE ${schemaCond.where} AND TABLE_NAME = ? LIMIT 1`, + values: [...schemaCond.values, tableName], + config, + }); + + return Boolean(rows[0]?.table_exists); +} + +export default async function recreateTable({ + table, + config, +}: { + table: BUN_MARIADB_TableSchemaType; + config?: BunMariaDBConfig; +}): Promise { + const doesTableExist = await checkIfTableExists({ + tableName: table.tableName, + config, + }); + + if (!doesTableExist) { + await createTable({ table, config }); + return; + } + + const tempTableName = `${table.tableName}_temp_${Date.now()}`; + const backupOldTableName = `${table.tableName}_old_${Date.now()}`; + const existingColumns = await getTableColumns({ + tableName: table.tableName, + config, + }); + + const columnsToKeep = (table.fields || []) + .filter((field) => + existingColumns.some((column) => column.name === field.fieldName), + ) + .map((field) => field.fieldName) + .filter((fieldName): fieldName is string => Boolean(fieldName)); + + await createTable({ + table: { ...table, tableName: tempTableName }, + config, + }); + + if (columnsToKeep.length > 0) { + const columnList = columnsToKeep + .map((column) => MariaDBQuoteGen(column)) + .join(", "); + + await runSchemaQuery({ + query: `INSERT INTO ${MariaDBQuoteGen(tempTableName)} (${columnList}) SELECT ${columnList} FROM ${MariaDBQuoteGen(table.tableName)}`, + config, + }); + } + + await runSchemaQuery({ query: `SET FOREIGN_KEY_CHECKS = 0`, config }); + try { + await runSchemaQuery({ + query: `RENAME TABLE ${MariaDBQuoteGen(table.tableName)} TO ${MariaDBQuoteGen(backupOldTableName)}`, + config, + }); + await runSchemaQuery({ + query: `RENAME TABLE ${MariaDBQuoteGen(tempTableName)} TO ${MariaDBQuoteGen(table.tableName)}`, + config, + }); + await runSchemaQuery({ + query: `DROP TABLE ${MariaDBQuoteGen(backupOldTableName)}`, + config, + }); + } finally { + await runSchemaQuery({ query: `SET FOREIGN_KEY_CHECKS = 1`, config }); + } +} diff --git a/src/lib/schema/resolve-table.ts b/src/lib/schema/resolve-table.ts new file mode 100644 index 0000000..3acc4b9 --- /dev/null +++ b/src/lib/schema/resolve-table.ts @@ -0,0 +1,56 @@ +import _ from "lodash"; +import type { + BUN_MARIADB_DatabaseSchemaType, + BUN_MARIADB_FieldSchemaType, + BUN_MARIADB_TableSchemaType, +} from "../../types"; + +export default function resolveTable( + table: BUN_MARIADB_TableSchemaType, + db_schema: BUN_MARIADB_DatabaseSchemaType, +): BUN_MARIADB_TableSchemaType { + if (!table.parentTableName) { + return _.cloneDeep(table); + } + + const parentTable = db_schema.tables.find( + (schemaTable) => schemaTable.tableName === table.parentTableName, + ); + + if (!parentTable) { + throw new Error( + `Parent table \`${table.parentTableName}\` not found for \`${table.tableName}\``, + ); + } + + const mergedFieldsMap = new Map(); + + (parentTable.fields || []).forEach((f) => { + if (f.fieldName) mergedFieldsMap.set(f.fieldName, _.cloneDeep(f)); + }); + + (table.fields || []).forEach((f) => { + if (f.fieldName) { + const existing = mergedFieldsMap.get(f.fieldName) || {}; + mergedFieldsMap.set(f.fieldName, _.merge({}, existing, f)); + } + }); + + return { + ..._.cloneDeep(parentTable), + tableName: table.tableName, + tableDescription: table.tableDescription || parentTable.tableDescription, + collation: table.collation || parentTable.collation, + isVector: + table.isVector !== undefined ? table.isVector : parentTable.isVector, + fields: Array.from(mergedFieldsMap.values()), + indexes: _.uniqBy( + [...(parentTable.indexes || []), ...(table.indexes || [])], + "indexName", + ), + uniqueConstraints: [ + ...(parentTable.uniqueConstraints || []), + ...(table.uniqueConstraints || []), + ], + }; +} diff --git a/src/lib/schema/run-schema-query.ts b/src/lib/schema/run-schema-query.ts new file mode 100644 index 0000000..018fe22 --- /dev/null +++ b/src/lib/schema/run-schema-query.ts @@ -0,0 +1,48 @@ +import type { BunMariaDBConfig } from "../../types"; +import dbHandler from "../db-handler"; + +export default async function runSchemaQuery({ + query, + values, + config, +}: { + query: string; + values?: any[]; + config?: BunMariaDBConfig; +}): Promise { + const res = await dbHandler({ + query, + values, + config, + }); + + if (!res.success) { + throw new Error( + `Database query failed: ${query} ... ERROR: ${res.error || res.msg}`, + ); + } +} + +export async function querySchemaRows>({ + query, + values, + config, +}: { + query: string; + values?: any[]; + config?: BunMariaDBConfig; +}): Promise { + const res = await dbHandler({ + query, + values, + config, + }); + + if (!res.success) { + throw new Error( + `Database query failed: ${query} ... ERROR: ${res.error || res.msg}`, + ); + } + + return (res.payload || []) as T[]; +} diff --git a/src/lib/schema/schema-condition.ts b/src/lib/schema/schema-condition.ts new file mode 100644 index 0000000..f3f5ad2 --- /dev/null +++ b/src/lib/schema/schema-condition.ts @@ -0,0 +1,20 @@ +import type { BunMariaDBConfig } from "../../types"; + +export default function schemaCondition(config?: BunMariaDBConfig): { + where: string; + values: string[]; +} { + const databaseName = config?.db_name || global.CONFIG?.db_name; + + if (databaseName) { + return { + where: "TABLE_SCHEMA = ?", + values: [databaseName], + }; + } + + return { + where: "TABLE_SCHEMA = DATABASE()", + values: [], + }; +} diff --git a/src/lib/schema/sync-indexes.ts b/src/lib/schema/sync-indexes.ts new file mode 100644 index 0000000..2adae15 --- /dev/null +++ b/src/lib/schema/sync-indexes.ts @@ -0,0 +1,165 @@ +import type { + BUN_MARIADB_TableSchemaType, + BunMariaDBConfig, +} from "../../types"; +import MariaDBQuoteGen from "./mariadb-quote-gen"; +import runSchemaQuery, { querySchemaRows } from "./run-schema-query"; +import schemaCondition from "./schema-condition"; + +export default async function syncIndexes({ + table, + config, +}: { + table: BUN_MARIADB_TableSchemaType; + config?: BunMariaDBConfig; +}): Promise { + const schemaCond = schemaCondition(config); + const rows = await querySchemaRows<{ + INDEX_NAME: string; + COLUMN_NAME: string; + INDEX_TYPE: string; + }>({ + query: `SELECT INDEX_NAME, COLUMN_NAME, INDEX_TYPE FROM information_schema.STATISTICS WHERE ${schemaCond.where} AND TABLE_NAME = ? AND INDEX_NAME <> 'PRIMARY' ORDER BY INDEX_NAME, SEQ_IN_INDEX`, + values: [...schemaCond.values, table.tableName], + config, + }); + + /** + * Indexes required by foreign keys / unique constraints cannot be dropped + * freely. Skip those when cleaning up schema indexes. + */ + const protectedConstraintRows = await querySchemaRows<{ + CONSTRAINT_NAME: string; + CONSTRAINT_TYPE: string; + }>({ + query: `SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE FROM information_schema.TABLE_CONSTRAINTS WHERE ${schemaCond.where} AND TABLE_NAME = ? AND CONSTRAINT_TYPE IN ('FOREIGN KEY', 'UNIQUE')`, + values: [...schemaCond.values, table.tableName], + config, + }); + const protectedIndexNames = new Set( + protectedConstraintRows.map((r) => r.CONSTRAINT_NAME), + ); + + // Column-level UNIQUE creates an index often named after the column + for (const field of table.fields || []) { + if (field.unique && field.fieldName) { + protectedIndexNames.add(field.fieldName); + } + } + for (const constraint of table.uniqueConstraints || []) { + if (constraint.constraintName) { + protectedIndexNames.add(constraint.constraintName); + } + } + + const existingIndexesMap = new Map< + string, + { columns: string[]; type: string } + >(); + + for (const row of rows) { + if (!existingIndexesMap.has(row.INDEX_NAME)) { + existingIndexesMap.set(row.INDEX_NAME, { + columns: [], + type: row.INDEX_TYPE, + }); + } + existingIndexesMap.get(row.INDEX_NAME)!.columns.push(row.COLUMN_NAME); + } + + for (const [indexName, details] of existingIndexesMap.entries()) { + if (protectedIndexNames.has(indexName)) { + continue; + } + + const schemaIndex = table.indexes?.find((i) => i.indexName === indexName); + + if (!schemaIndex) { + console.log(`Dropping index: ${indexName}`); + try { + await runSchemaQuery({ + query: `DROP INDEX ${MariaDBQuoteGen(indexName)} ON ${MariaDBQuoteGen(table.tableName)}`, + config, + }); + } catch (err: any) { + if ( + String(err?.message || "").includes( + "needed in a foreign key constraint", + ) + ) { + console.warn( + `Skipping drop of index ${indexName}: required by a foreign key constraint`, + ); + continue; + } + throw err; + } + } else { + const schemaColumns = schemaIndex.indexTableFields || []; + const columnsMatch = + details.columns.length === schemaColumns.length && + details.columns.every((col, idx) => col === schemaColumns[idx]); + + if (!columnsMatch) { + console.log(`Recreating changed index: ${indexName}`); + await runSchemaQuery({ + query: `DROP INDEX ${MariaDBQuoteGen(indexName)} ON ${MariaDBQuoteGen(table.tableName)}`, + config, + }); + existingIndexesMap.delete(indexName); + } + } + } + + for (const index of table.indexes || []) { + if ( + !index.indexName || + !index.indexTableFields || + index.indexTableFields.length === 0 + ) { + continue; + } + + if (!existingIndexesMap.has(index.indexName)) { + const isVectorIndex = + table.isVector || + table.fields?.some( + (f) => + f.fieldName === index.indexTableFields![0] && f.isVector, + ); + + if (isVectorIndex) { + console.log(`Creating Vector index: ${index.indexName}`); + const targetField = MariaDBQuoteGen(index.indexTableFields[0]!); + const distanceMetric = + (index.indexType as string)?.toLowerCase() === "cosine" + ? "cosine" + : "euclidean"; + + await runSchemaQuery({ + query: `ALTER TABLE ${MariaDBQuoteGen(table.tableName)} ADD VECTOR INDEX ${MariaDBQuoteGen(index.indexName)} (${targetField}) M=8 DISTANCE=${distanceMetric}`, + config, + }); + } else { + console.log(`Creating standard index: ${index.indexName}`); + const fields = index.indexTableFields + .map((field) => MariaDBQuoteGen(field)) + .join(", "); + const typeUpper = index.indexType?.toUpperCase(); + const isSpecialType = + typeUpper === "FULLTEXT" || typeUpper === "SPATIAL"; + const indexPrefix = isSpecialType ? `${typeUpper} ` : ""; + const indexSuffix = + !isSpecialType && + (typeUpper === "BTREE" || typeUpper === "HASH") + ? ` USING ${typeUpper}` + : ""; + + await runSchemaQuery({ + query: `CREATE ${indexPrefix}INDEX ${MariaDBQuoteGen(index.indexName)} ON ${MariaDBQuoteGen(table.tableName)} (${fields})${indexSuffix}`, + config, + }); + } + } + } +} diff --git a/src/lib/schema/update-table.ts b/src/lib/schema/update-table.ts new file mode 100644 index 0000000..c17c389 --- /dev/null +++ b/src/lib/schema/update-table.ts @@ -0,0 +1,225 @@ +import type { + BUN_MARIADB_FieldSchemaType, + BUN_MARIADB_TableSchemaType, + BunMariaDBConfig, +} from "../../types"; +import buildColumnDefinition from "./build-column-definition"; +import createTable from "./create-table"; +import getTableColumns from "./get-table-columns"; +import mapDataType from "./map-data-types"; +import MariaDBQuoteGen from "./mariadb-quote-gen"; +import recreateTable from "./recreate-table"; +import runSchemaQuery, { querySchemaRows } from "./run-schema-query"; +import schemaCondition from "./schema-condition"; + +/** + * Compare live COLUMN_TYPE with schema-mapped type. + * Live types often include display widths (e.g. bigint(20) vs BIGINT). + */ +function columnTypesMatch(liveType: string, expectedType: string): boolean { + const live = liveType.toLowerCase().replace(/\s+/g, ""); + const expected = expectedType.toLowerCase().replace(/\s+/g, ""); + + if (live === expected) return true; + // live may include display width: bigint(20) vs bigint + if (live.startsWith(`${expected}(`)) return true; + // expected may include length live omits in some versions + if (expected.startsWith(`${live}(`)) return true; + + return false; +} + +async function addColumn({ + tableName, + field, + config, +}: { + tableName: string; + field: BUN_MARIADB_FieldSchemaType; + config?: BunMariaDBConfig; +}): Promise { + console.log(`Adding column: ${tableName}.${field.fieldName}`); + const columnDef = buildColumnDefinition(field).trim(); + await runSchemaQuery({ + query: `ALTER TABLE ${MariaDBQuoteGen(tableName)} ADD COLUMN IF NOT EXISTS ${columnDef}`, + config, + }); +} + +async function modifyColumn({ + tableName, + field, + config, +}: { + tableName: string; + field: BUN_MARIADB_FieldSchemaType; + config?: BunMariaDBConfig; +}): Promise { + console.log(`Modifying column: ${tableName}.${field.fieldName}`); + const columnDef = buildColumnDefinition(field).trim(); + await runSchemaQuery({ + query: `ALTER TABLE ${MariaDBQuoteGen(tableName)} MODIFY COLUMN ${columnDef}`, + config, + }); +} + +async function dropColumn({ + tableName, + fieldName, + config, +}: { + tableName: string; + fieldName: string; + config?: BunMariaDBConfig; +}): Promise { + console.log(`Dropping column: ${tableName}.${fieldName}`); + await runSchemaQuery({ + query: `ALTER TABLE ${MariaDBQuoteGen(tableName)} DROP COLUMN ${MariaDBQuoteGen(fieldName)}`, + config, + }); +} + +export default async function updateTable({ + table, + config, +}: { + table: BUN_MARIADB_TableSchemaType; + config?: BunMariaDBConfig; +}): Promise { + const existingColumns = await getTableColumns({ + tableName: table.tableName, + config, + }); + + if (existingColumns.length === 0) { + await createTable({ table, config }); + return; + } + + const liveFieldsMap = new Map( + existingColumns.map((col) => [ + col.name, + { type: col.type.toLowerCase(), comment: col.comment || "" }, + ]), + ); + const codeFieldsMap = new Map( + (table.fields || []).map((f) => [f.fieldName, f]), + ); + + const fieldsToAdd: BUN_MARIADB_FieldSchemaType[] = []; + const fieldsToModify: BUN_MARIADB_FieldSchemaType[] = []; + const fieldsToDrop: string[] = []; + + for (const field of table.fields || []) { + if (!field.fieldName) continue; + + const liveField = liveFieldsMap.get(field.fieldName); + + if (!liveField) { + fieldsToAdd.push(field); + } else { + let typeDiverged = !columnTypesMatch( + liveField.type, + mapDataType(field), + ); + + if (field.isVector || field.dataType === "VECTOR") { + const dimensions = field.vectorSize || 1536; + const expectedNativeToken = `vector(${dimensions})`; + typeDiverged = liveField.type.toLowerCase() !== expectedNativeToken; + } + + if (typeDiverged) { + fieldsToModify.push(field); + } + } + } + + for (const col of existingColumns) { + if (!codeFieldsMap.has(col.name)) { + fieldsToDrop.push(col.name); + } + } + + if ( + fieldsToAdd.length === 0 && + fieldsToModify.length === 0 && + fieldsToDrop.length === 0 + ) { + return; + } + + console.log( + `Surgically updating table structure from database layout: ${table.tableName}`, + ); + + if (fieldsToDrop.length > 0) { + const schemaCond = schemaCondition(config); + + const pkRows = await querySchemaRows<{ COLUMN_NAME: string }>({ + query: `SELECT COLUMN_NAME FROM information_schema.KEY_COLUMN_USAGE WHERE ${schemaCond.where} AND TABLE_NAME = ? AND CONSTRAINT_NAME = 'PRIMARY'`, + values: [...schemaCond.values, table.tableName], + config, + }); + const pkColumnNames = pkRows.map((r) => r.COLUMN_NAME); + + if (fieldsToDrop.some((f) => pkColumnNames.includes(f))) { + console.log( + `Dropping primary key because a PK column is being dropped`, + ); + await runSchemaQuery({ + query: `ALTER TABLE ${MariaDBQuoteGen(table.tableName)} DROP PRIMARY KEY`, + config, + }); + } + + const indexRows = await querySchemaRows<{ + INDEX_NAME: string; + COLUMN_NAME: string; + }>({ + query: `SELECT INDEX_NAME, COLUMN_NAME FROM information_schema.STATISTICS WHERE ${schemaCond.where} AND TABLE_NAME = ? AND INDEX_NAME <> 'PRIMARY'`, + values: [...schemaCond.values, table.tableName], + config, + }); + + const indexesToDrop = new Set(); + for (const row of indexRows) { + if (fieldsToDrop.includes(row.COLUMN_NAME)) { + indexesToDrop.add(row.INDEX_NAME); + } + } + + for (const indexName of indexesToDrop) { + console.log( + `Dropping index ${indexName} because it contains a dropped column`, + ); + await runSchemaQuery({ + query: `DROP INDEX ${MariaDBQuoteGen(indexName)} ON ${MariaDBQuoteGen(table.tableName)}`, + config, + }); + } + } + + for (const field of fieldsToAdd) { + await addColumn({ tableName: table.tableName, field, config }); + } + + for (const field of fieldsToModify) { + try { + await modifyColumn({ tableName: table.tableName, field, config }); + } catch (err: any) { + if (field.isVector || field.dataType === "VECTOR") { + console.warn( + `[Vector Resize] Re-aligning dimension spaces natively via safe migration schema rebuild.`, + ); + await recreateTable({ table, config }); + return; + } + throw err; + } + } + + for (const fieldName of fieldsToDrop) { + await dropColumn({ tableName: table.tableName, fieldName, config }); + } +} diff --git a/src/lib/schema/upsert-db-manager-table.ts b/src/lib/schema/upsert-db-manager-table.ts new file mode 100644 index 0000000..7a45a9b --- /dev/null +++ b/src/lib/schema/upsert-db-manager-table.ts @@ -0,0 +1,33 @@ +import { AppData } from "../../data/app-data"; +import type { BunMariaDBConfig } from "../../types"; +import MariaDBQuoteGen from "./mariadb-quote-gen"; +import runSchemaQuery from "./run-schema-query"; + +export default async function upsertDbManagerTable({ + tableName, + config, +}: { + tableName: string; + config?: BunMariaDBConfig; +}): Promise { + const now = Date.now(); + await runSchemaQuery({ + query: `INSERT INTO ${MariaDBQuoteGen(AppData["DbSchemaManagerTableName"])} (table_name, created_at, updated_at) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = VALUES(updated_at)`, + values: [tableName, now, now], + config, + }); +} + +export async function removeDbManagerTable({ + tableName, + config, +}: { + tableName: string; + config?: BunMariaDBConfig; +}): Promise { + await runSchemaQuery({ + query: `DELETE FROM ${MariaDBQuoteGen(AppData["DbSchemaManagerTableName"])} WHERE table_name = ?`, + values: [tableName], + config, + }); +} diff --git a/src/types/index.ts b/src/types/index.ts index b961da8..48403ea 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -120,7 +120,6 @@ export interface BUN_MARIADB_ChildrenTablesType { */ export const TextFieldTypesArray = [ { title: "Plain Text", value: "plain" }, - { title: "Rich Text", value: "richText" }, { title: "Markdown", value: "markdown" }, { title: "JSON", value: "json" }, { title: "YAML", value: "yaml" }, @@ -1544,6 +1543,21 @@ export type BunMariaDBConfig = { * File path to the SSL certificate */ ssl_ca?: string; + /** + * Extra HTML sanitization allowlists appended to the defaults + * when inserting/updating fields marked `html: true`. + */ + html_sanitize?: { + /** + * Additional tags to allow (merged with built-in defaults). + */ + allowed_tags?: string[]; + /** + * Additional attributes to allow per tag (merged with built-in defaults). + * Values for each tag are appended to any existing allowed attributes. + */ + allowed_attributes?: Record; + }; }; /** diff --git a/src/utils/sanitize-html-fields.ts b/src/utils/sanitize-html-fields.ts new file mode 100644 index 0000000..e83c34c --- /dev/null +++ b/src/utils/sanitize-html-fields.ts @@ -0,0 +1,99 @@ +import sanitizeHtml from "sanitize-html"; +import type { + BUN_MARIADB_DatabaseSchemaType, + BUN_MARIADB_FieldSchemaType, + BunMariaDBConfig, +} from "../types"; +import { readLiveSchema } from "../functions/live-schema"; +import getSanitizeHtmlOptions from "./sanitize-html-options"; + +function grabTableFields(tableName: string): BUN_MARIADB_FieldSchemaType[] { + const dbSchema: BUN_MARIADB_DatabaseSchemaType | undefined = + global.DB_SCHEMA || readLiveSchema(); + + const tableSchema = dbSchema?.tables?.find((t) => t.tableName === tableName); + + return tableSchema?.fields || []; +} + +/** + * Only fields with an explicit `html: true` flag are sanitized. + * Missing / falsy / non-true values are never sanitized. + */ +function isExplicitHtmlField(field?: BUN_MARIADB_FieldSchemaType): boolean { + return field?.html === true; +} + +function sanitizeValue( + value: unknown, + config?: BunMariaDBConfig, +): unknown { + if (typeof value !== "string") { + return value; + } + + return sanitizeHtml(value, getSanitizeHtmlOptions(config)); +} + +/** + * Sanitize string values ONLY for schema fields with explicit `html: true`. + * Other fields (including plain text that happens to contain HTML) are left untouched. + */ +export default function sanitizeHtmlFields< + T extends Record = Record, +>({ + table, + data, + config, +}: { + table: string; + data: T; + config?: BunMariaDBConfig; +}): T { + const fields = grabTableFields(table); + if (fields.length === 0) { + return data; + } + + const htmlFieldNames = new Set( + fields + .filter(isExplicitHtmlField) + .map((f) => f.fieldName) + .filter((name): name is string => Boolean(name)), + ); + + // No explicitly marked html fields on this table — skip entirely + if (htmlFieldNames.size === 0) { + return data; + } + + const sanitized = { ...data } as Record; + const resolvedConfig = config || global.CONFIG; + + for (const key of Object.keys(sanitized)) { + // Only sanitize keys that map to fields with html: true + if (!htmlFieldNames.has(key)) continue; + sanitized[key] = sanitizeValue(sanitized[key], resolvedConfig); + } + + return sanitized as T; +} + +/** + * Sanitize an array of row objects for insert. + */ +export function sanitizeHtmlFieldsBatch< + T extends Record = Record, +>({ + table, + data, + config, +}: { + table: string; + data: T[]; + config?: BunMariaDBConfig; +}): T[] { + return data.map((row) => + sanitizeHtmlFields({ table, data: row, config }), + ); +} diff --git a/src/utils/sanitize-html-options.ts b/src/utils/sanitize-html-options.ts new file mode 100644 index 0000000..f673b96 --- /dev/null +++ b/src/utils/sanitize-html-options.ts @@ -0,0 +1,86 @@ +import type { IOptions } from "sanitize-html"; +import type { BunMariaDBConfig } from "../types"; + +export const defaultSanitizeHtmlOptions: IOptions = { + allowedTags: [ + "b", + "i", + "em", + "strong", + "a", + "p", + "span", + "ul", + "ol", + "li", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "img", + "div", + "button", + "pre", + "code", + "br", + "hr", + "blockquote", + "table", + "tr", + "td", + "th", + "thead", + "tbody", + "tfoot", + "caption", + "colgroup", + "col", + ], + allowedAttributes: { + a: ["href", "title", "class", "style", "target", "rel"], + img: ["src", "alt", "width", "height", "class", "style"], + "*": ["style", "class", "title", "id"], + }, +}; + +function uniqueStrings(values: string[]): string[] { + return Array.from(new Set(values)); +} + +/** + * Build sanitize-html options, appending any tags/attributes from config. + */ +export default function getSanitizeHtmlOptions( + config?: BunMariaDBConfig, +): IOptions { + const cfg = config || global.CONFIG; + const extra = cfg?.html_sanitize; + + const baseTags = defaultSanitizeHtmlOptions.allowedTags || []; + const baseAttrs = { + ...(defaultSanitizeHtmlOptions.allowedAttributes || {}), + } as Record; + + const allowedTags = uniqueStrings([ + ...(Array.isArray(baseTags) ? baseTags : []), + ...(extra?.allowed_tags || []), + ]); + + const allowedAttributes: Record = { ...baseAttrs }; + + for (const [tag, attrs] of Object.entries( + extra?.allowed_attributes || {}, + )) { + allowedAttributes[tag] = uniqueStrings([ + ...(allowedAttributes[tag] || []), + ...attrs, + ]); + } + + return { + allowedTags, + allowedAttributes, + }; +}