Moduletrace/datasquirel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updates

Browse Source
This commit is contained in:
Benjamin Toby
2025-08-02 17:14:46 +01:00
parent 9ca64cf25e
commit 71a8431de5
24 changed files with 164 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-shared/data/data-types.ts
+7 -1
View File
@@ -6,7 +6,7 @@ const DataTypes = [
argument: true,
description:
"Varchar is simply letters and numbers within the range 0 - 255",
maxValue: 255,
maxValue: 2000,
},
{
title: "TINYINT",
@@ -110,6 +110,12 @@ const DataTypes = [
name: "TIMESTAMP",
description: "Time Stamp",
},
{
title: "VECTOR",
name: "VECTOR",
description: "Vector Field for vector-based applications",
maxValue: 2147483647,
},
] as const;
export default DataTypes;
package-shared/functions/backend/db/addDbEntry.ts
+39 -11
View File
@@ -118,13 +118,19 @@ export default async function addDbEntry<
const dataKeys = Object.keys(data);
let insertKeysArray = [];
let insertValuesArray = [];
let insertValuesArray: (string | number)[] = [];
for (let i = 0; i < dataKeys.length; i++) {
try {
const dataKey = dataKeys[i];
let value = data[dataKey];
const targetFieldSchema = tableSchema
? tableSchema?.fields?.find(
(field) => field.fieldName === dataKey
)
: null;
const parsedValue = grabParsedValue({
dataKey,
encryptionKey,
@@ -137,7 +143,9 @@ export default async function addDbEntry<
insertKeysArray.push("`" + dataKey + "`");
if (typeof parsedValue == "number") {
if (targetFieldSchema?.dataType?.match(/vector/i)) {
insertValuesArray.push(`VEC_FromText('${parsedValue}')`);
} else if (typeof parsedValue == "number") {
insertValuesArray.push(String(parsedValue));
} else {
insertValuesArray.push(parsedValue);
@@ -163,11 +171,28 @@ export default async function addDbEntry<
insertKeysArray.push("`date_updated_code`");
insertValuesArray.push(Date.now());
const queryValuesArray = insertValuesArray;
const queryValuesArray = insertValuesArray as (string | number)[];
return { queryValuesArray, insertValuesArray, insertKeysArray };
}
function grabQueryValuesString(arr: (string | number)[]) {
return arr
.map((v, i) => {
if (v.toString().match(/VEC_FromText/i)) {
return v;
}
return "?";
})
.join(",");
}
function grabFinalQueryValuesArr(arr: (string | number)[]) {
return arr
.filter((v) => !v.toString().match(/VEC_FromText/i))
.map((v) => String(v));
}
if (newData) {
const { insertKeysArray, insertValuesArray, queryValuesArray } =
generateQuery(newData);
@@ -176,12 +201,14 @@ export default async function addDbEntry<
isMaster && !dbFullName ? "" : `\`${dbFullName}\`.`
}\`${tableName}\` (${insertKeysArray.join(
","
)}) VALUES (${insertValuesArray.map(() => "?").join(",")})`;
)}) VALUES (${grabQueryValuesString(insertValuesArray)})`;
const finalQueryValues = grabFinalQueryValuesArr(queryValuesArray);
const newInsert = await connDbHandler(
null,
query,
queryValuesArray,
finalQueryValues,
debug
);
@@ -190,7 +217,7 @@ export default async function addDbEntry<
payload: newInsert,
queryObject: {
sql: query,
params: queryValuesArray,
params: finalQueryValues,
},
};
} else if (newBatchData) {
@@ -216,16 +243,17 @@ export default async function addDbEntry<
}\`${tableName}\` (${batchInsertKeysArray?.join(
","
)}) VALUES ${batchInsertValuesArray
.map((vl) => `(${vl.map(() => "?").join(",")})`)
.map((vl) => `(${grabQueryValuesString(vl)})`)
.join(",")}`;
console.log("query", query);
console.log("batchQueryValuesArray", batchQueryValuesArray);
const finalQueryValues = grabFinalQueryValuesArr(
batchQueryValuesArray.flat()
);
const newInsert = await connDbHandler(
null,
query,
batchQueryValuesArray.flat(),
finalQueryValues,
debug
);
@@ -242,7 +270,7 @@ export default async function addDbEntry<
payload: newInsert,
queryObject: {
sql: query,
params: batchQueryValuesArray.flat(),
params: finalQueryValues,
},
};
} else {
package-shared/functions/backend/db/grab-parsed-value.ts
+2 -6
View File
@@ -25,13 +25,9 @@ export default function grabParsedValue({
}: Param): any {
let newValue = value;
const targetFieldSchemaArray = tableSchema
? tableSchema?.fields?.filter((field) => field.fieldName === dataKey)
const targetFieldSchema = tableSchema
? tableSchema?.fields?.find((field) => field.fieldName === dataKey)
: null;
const targetFieldSchema =
targetFieldSchemaArray && targetFieldSchemaArray[0]
? targetFieldSchemaArray[0]
: null;
if (typeof newValue == "undefined") return;
if (typeof newValue == "object" && !newValue) newValue = null;
package-shared/functions/backend/db/updateDbEntry.ts
+11 -1
View File
@@ -80,6 +80,12 @@ export default async function updateDbEntry<
const dataKey = dataKeys[i];
let value = newData[dataKey];
const targetFieldSchema = tableSchema
? tableSchema?.fields?.find(
(field) => field.fieldName === dataKey
)
: null;
const parsedValue = grabParsedValue({
dataKey,
encryptionKey,
@@ -90,7 +96,11 @@ export default async function updateDbEntry<
if (typeof parsedValue == "undefined") continue;
updateKeyValueArray.push(`\`${dataKey}\`=?`);
if (targetFieldSchema?.dataType?.match(/vector/i)) {
updateKeyValueArray.push(`\`${dataKey}\`=VEC_FromText(?)`);
} else {
updateKeyValueArray.push(`\`${dataKey}\`=?`);
}
if (typeof parsedValue == "number") {
updateValues.push(String(parsedValue));
package-shared/functions/backend/parseDbResults.ts
+4
View File
@@ -49,6 +49,10 @@ export default function parseDbResults({
});
}
}
if (value && typeof value == "object") {
result[resultFieldName] = "";
}
}
parsedResults.push(result);
package-shared/shell/createDbFromSchema/handle-indexes.ts
+5 -1
View File
@@ -47,7 +47,11 @@ export default async function handleIndexescreateDbFromSchema({
* doesn't exist in MYSQL db
*/
const queryString = `CREATE${
indexType == "full_text" ? " FULLTEXT" : ""
indexType == "full_text"
? " FULLTEXT"
: indexType == "vector"
? " VECTOR"
: ""
} INDEX \`${alias}\` ON \`${dbFullName}\`.\`${tableName}\`(${indexTableFields
?.map((nm) => nm.value)
.map((nm) => `\`${nm}\``)
package-shared/types/index.ts
+7 -1
View File
@@ -1813,6 +1813,7 @@ export type MediaUploadDataType = ImageObjectType &
privateFolder?: boolean;
overwrite?: boolean;
updatedMediaRecord?: DSQL_DATASQUIREL_USER_MEDIA;
existingMediaRecord?: DSQL_DATASQUIREL_USER_MEDIA;
existingMediaRecordId?: number;
};
@@ -1921,7 +1922,7 @@ export type DefaultEntryType = {
[k: string]: string | number | null;
};
export const IndexTypes = ["regular", "full_text"] as const;
export const IndexTypes = ["regular", "full_text", "vector"] as const;
export type LoginUserParam = {
apiKey?: string;
@@ -2344,6 +2345,11 @@ export type AddMediaAPIBody = {
update?: boolean;
};
export type ReplaceMediaAPIBody = {
mediaId: number;
media: MediaUploadDataType;
};
export const TargetMediaParadigms = ["info", "preview"] as const;
export type TargetMediaDataType = {
package-shared/utils/db/schema/data-type-parser.ts
+4 -1
View File
@@ -6,6 +6,7 @@ export const DataTypesWithNumbers: (typeof DataTypes)[number]["name"][] = [
"DOUBLE",
"FLOAT",
"VARCHAR",
"VECTOR",
];
export const DataTypesWithTwoNumbers: (typeof DataTypes)[number]["name"][] = [
@@ -17,6 +18,7 @@ export const DataTypesWithTwoNumbers: (typeof DataTypes)[number]["name"][] = [
type Return = {
type: (typeof DataTypes)[number]["name"];
limit?: number;
defaultNumber?: number;
decimal?: number;
};
@@ -24,7 +26,7 @@ export default function dataTypeParser(dataType?: string): Return {
if (!dataType) {
return {
type: "VARCHAR",
limit: 250,
defaultNumber: 250,
};
}
@@ -50,5 +52,6 @@ export default function dataTypeParser(dataType?: string): Return {
return {
type,
limit: number ? numberfy(number) : undefined,
defaultNumber: type == "VECTOR" ? 120 : 10,
};
}
package-shared/utils/grab-api-base-path.ts
+1 -1
View File
@@ -1,4 +1,4 @@
const APIParadigms = ["crud", "media", "schema"] as const;
import { APIParadigms } from "../types";
type Params = {
version?: string;