Moduletrace/datasquirel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updates

Browse Source
This commit is contained in:
Benjamin Toby
2025-06-04 08:43:16 +01:00
parent 1364074c2c
commit 761348de08
37 changed files with 577 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-shared/functions/api/query/get.ts
+1 -1
View File
@@ -66,7 +66,7 @@ export default async function apiGet<
let tableSchema: DSQL_TableSchemaType | undefined;
if (dbSchema) {
const targetTable = dbSchema.tables.find(
const targetTable = dbSchema.tables?.find(
(table) => table.tableName === tableName
);
package-shared/functions/backend/createDbSchemaFromDb.ts
+163
View File
@@ -0,0 +1,163 @@
import varDatabaseDbHandler from "../../functions/backend/varDatabaseDbHandler";
import { default as grabUserSchemaData } from "../../functions/backend/grabUserSchemaData";
import { default as setUserSchemaData } from "../../functions/backend/setUserSchemaData";
import addDbEntry from "../../functions/backend/db/addDbEntry";
import slugToCamelTitle from "../../shell/utils/slugToCamelTitle";
import { DSQL_DATASQUIREL_USER_DATABASES } from "@/package-shared/types/dsql";
import {
DSQL_FieldSchemaType,
DSQL_MYSQL_SHOW_COLUMNS_Type,
DSQL_TableSchemaType,
} from "@/package-shared/types";
type Params = {
userId: number | string;
database: DSQL_DATASQUIREL_USER_DATABASES;
};
export default async function createDbSchemaFromDb({
userId,
database,
}: Params) {
try {
if (!userId) {
console.log("No user Id provided");
return;
}
const userSchemaData = grabUserSchemaData({ userId });
if (!userSchemaData) throw new Error("User schema data not found!");
const targetDb: { tables: object[] } = userSchemaData.filter(
(dbObject) => dbObject.dbFullName === database.db_full_name
)[0];
const existingTables = await varDatabaseDbHandler({
database: database.db_full_name,
queryString: `SHOW TABLES FROM ${database.db_full_name}`,
});
if (!existingTables) throw new Error("No Existing Tables");
for (let i = 0; i < existingTables.length; i++) {
const table = existingTables[i];
const tableName = Object.values(table)[0] as string;
const tableInsert = await addDbEntry({
dbFullName: "datasquirel",
tableName: "user_database_tables",
data: {
user_id: userId,
db_id: database.id,
db_slug: database.db_slug,
table_name: slugToCamelTitle(tableName),
table_slug: tableName,
},
});
const tableObject: DSQL_TableSchemaType = {
tableName: tableName,
tableFullName: slugToCamelTitle(tableName) || "",
fields: [],
indexes: [],
};
const tableColumns: DSQL_MYSQL_SHOW_COLUMNS_Type[] =
await varDatabaseDbHandler({
database: database.db_full_name,
queryString: `SHOW COLUMNS FROM ${database.db_full_name}.${tableName}`,
});
if (tableColumns) {
for (let k = 0; k < tableColumns.length; k++) {
const tableColumn = tableColumns[k];
const { Field, Type, Null, Key, Default, Extra } =
tableColumn;
const fieldObject: DSQL_FieldSchemaType = {
fieldName: Field,
dataType: Type.toUpperCase(),
};
if (Null?.match(/^no$/i)) fieldObject.notNullValue = true;
if (Key?.match(/^pri$/i)) fieldObject.primaryKey = true;
if (Default?.toString()?.match(/./))
fieldObject.defaultValue = Default;
if (Default?.toString()?.match(/timestamp/i)) {
delete fieldObject.defaultValue;
fieldObject.defaultValueLiteral = Default;
}
if (Extra?.toString()?.match(/auto_increment/i))
fieldObject.autoIncrement = true;
tableObject.fields.push(fieldObject);
}
}
const tableIndexes = await varDatabaseDbHandler({
database: database.db_full_name,
queryString: `SHOW INDEXES FROM ${database.db_full_name}.${tableName}`,
});
if (tableIndexes) {
for (let m = 0; m < tableIndexes.length; m++) {
const indexObject = tableIndexes[m];
const {
Table,
Key_name,
Column_name,
Null,
Index_type,
Index_comment,
} = indexObject;
if (!Index_comment?.match(/^schema_index$/)) continue;
const indexNewObject: import("@/package-shared/types").DSQL_IndexSchemaType =
{
indexType: Index_type?.match(/fulltext/i)
? "fullText"
: "regular",
indexName: Key_name,
indexTableFields: [],
};
const targetTableFieldObject = tableColumns?.filter(
(col) => col.Field === Column_name
)[0];
const existingIndexField = tableObject.indexes?.filter(
(indx) => indx.indexName == Key_name
);
if (existingIndexField && existingIndexField[0]) {
existingIndexField[0].indexTableFields?.push({
value: Column_name,
dataType: targetTableFieldObject.Type.toUpperCase(),
});
} else {
indexNewObject.indexTableFields = [
{
value: Column_name,
dataType:
targetTableFieldObject.Type.toUpperCase(),
},
];
tableObject.indexes?.push(indexNewObject);
}
}
}
targetDb.tables.push(tableObject);
}
setUserSchemaData({ schemaData: userSchemaData, userId });
return true;
} catch (error) {
console.log(error);
return false;
}
}
package-shared/functions/backend/db/addDbEntry.ts
+5 -30
View File
@@ -7,13 +7,14 @@ import connDbHandler from "../../../utils/db/conn-db-handler";
import checkIfIsMaster from "../../../utils/check-if-is-master";
import { DbContextsArray } from "./runQuery";
import debugLog from "../../../utils/logging/debug-log";
import { PostInsertReturn } from "../../../types";
type Param = {
type Param<T extends { [k: string]: any } = any> = {
dbContext?: (typeof DbContextsArray)[number];
paradigm?: "Read Only" | "Full Access";
dbFullName?: string;
tableName: string;
data: any;
data: T;
tableSchema?: import("../../../types").DSQL_TableSchemaType;
duplicateColumnName?: string;
duplicateColumnValue?: string;
@@ -27,7 +28,7 @@ type Param = {
/**
* Add a db Entry Function
*/
export default async function addDbEntry({
export default async function addDbEntry<T extends { [k: string]: any } = any>({
dbContext,
paradigm,
dbFullName,
@@ -41,10 +42,7 @@ export default async function addDbEntry({
encryptionSalt,
forceLocal,
debug,
}: Param): Promise<any> {
/**
* Initialize variables
*/
}: Param<T>): Promise<PostInsertReturn | null> {
const isMaster = forceLocal
? true
: checkIfIsMaster({ dbContext, dbFullName });
@@ -64,10 +62,6 @@ export default async function addDbEntry({
? global.DSQL_DB_CONN
: global.DSQL_READ_ONLY_DB_CONN || global.DSQL_DB_CONN;
////////////////////////////////////////
////////////////////////////////////////
////////////////////////////////////////
if (data?.["date_created_timestamp"]) delete data["date_created_timestamp"];
if (data?.["date_updated_timestamp"]) delete data["date_updated_timestamp"];
if (data?.["date_updated"]) delete data["date_updated"];
@@ -75,10 +69,6 @@ export default async function addDbEntry({
if (data?.["date_created"]) delete data["date_created"];
if (data?.["date_created_code"]) delete data["date_created_code"];
////////////////////////////////////////
////////////////////////////////////////
////////////////////////////////////////
if (duplicateColumnName && typeof duplicateColumnName === "string") {
const checkDuplicateQuery = `SELECT * FROM ${
isMaster ? "" : `\`${dbFullName}\`.`
@@ -107,11 +97,6 @@ export default async function addDbEntry({
}
}
/**
* Declare variables
*
* @description Declare "results" variable
*/
const dataKeys = Object.keys(data);
let insertKeysArray = [];
@@ -120,7 +105,6 @@ export default async function addDbEntry({
for (let i = 0; i < dataKeys.length; i++) {
try {
const dataKey = dataKeys[i];
// @ts-ignore
let value = data?.[dataKey];
const targetFieldSchemaArray = tableSchema
@@ -186,8 +170,6 @@ export default async function addDbEntry({
}
}
////////////////////////////////////////
if (!data?.["date_created"]) {
insertKeysArray.push("`date_created`");
insertValuesArray.push(Date());
@@ -198,8 +180,6 @@ export default async function addDbEntry({
insertValuesArray.push(Date.now());
}
////////////////////////////////////////
if (!data?.["date_updated"]) {
insertKeysArray.push("`date_updated`");
insertValuesArray.push(Date());
@@ -210,8 +190,6 @@ export default async function addDbEntry({
insertValuesArray.push(Date.now());
}
////////////////////////////////////////
const query = `INSERT INTO ${
isMaster ? "" : `\`${dbFullName}\`.`
}\`${tableName}\` (${insertKeysArray.join(",")}) VALUES (${insertValuesArray
@@ -254,8 +232,5 @@ export default async function addDbEntry({
});
}
/**
* Return statement
*/
return newInsert;
}
package-shared/functions/backend/db/updateDbEntry.ts
+10 -7
View File
@@ -4,8 +4,9 @@ import encrypt from "../../dsql/encrypt";
import checkIfIsMaster from "../../../utils/check-if-is-master";
import connDbHandler from "../../../utils/db/conn-db-handler";
import { DbContextsArray } from "./runQuery";
import { PostInsertReturn } from "../../../types";
type Param = {
type Param<T extends { [k: string]: any } = any> = {
dbContext?: (typeof DbContextsArray)[number];
dbFullName?: string;
tableName: string;
@@ -13,7 +14,7 @@ type Param = {
encryptionSalt?: string;
data: any;
tableSchema?: import("../../../types").DSQL_TableSchemaType;
identifierColumnName: string;
identifierColumnName: keyof T;
identifierValue: string | number;
forceLocal?: boolean;
};
@@ -22,7 +23,9 @@ type Param = {
* # Update DB Function
* @description
*/
export default async function updateDbEntry({
export default async function updateDbEntry<
T extends { [k: string]: any } = any
>({
dbContext,
dbFullName,
tableName,
@@ -33,7 +36,7 @@ export default async function updateDbEntry({
encryptionKey,
encryptionSalt,
forceLocal,
}: Param): Promise<object | null> {
}: Param<T>): Promise<PostInsertReturn | null> {
/**
* Check if data is valid
*/
@@ -157,9 +160,9 @@ export default async function updateDbEntry({
const query = `UPDATE ${
isMaster ? "" : `\`${dbFullName}\`.`
}\`${tableName}\` SET ${updateKeyValueArray.join(
","
)} WHERE \`${identifierColumnName}\`=?`;
}\`${tableName}\` SET ${updateKeyValueArray.join(",")} WHERE \`${
identifierColumnName as string
}\`=?`;
updateValues.push(identifierValue);
package-shared/functions/backend/grabUserSchemaData.ts
+14 -12
View File
@@ -1,25 +1,27 @@
import { DSQL_DatabaseSchemaType, UserType } from "@/package-shared/types";
import serverError from "./serverError";
import fs from "fs";
import path from "path";
import grabDirNames from "@/package-shared/utils/backend/names/grab-dir-names";
import { EJSON } from "@/client-exports";
type Params = {
userId?: string | number;
};
/**
* # Grab User Schema Data
*/
export default function grabUserSchemaData({
userId,
}: {
userId: string | number;
}): import("../../types").DSQL_DatabaseSchemaType[] | null {
}: Params): DSQL_DatabaseSchemaType[] | null {
try {
const userSchemaFilePath = path.resolve(
process.cwd(),
`${process.env.DSQL_USER_DB_SCHEMA_PATH}/user-${userId}/main.json`
const { userSchemaMainJSONFilePath } = grabDirNames({ userId });
const schemaJSON = fs.readFileSync(
userSchemaMainJSONFilePath || "",
"utf-8"
);
const userSchemaData = JSON.parse(
fs.readFileSync(userSchemaFilePath, "utf-8")
);
return userSchemaData;
const schemaObj = EJSON.parse(schemaJSON) as DSQL_DatabaseSchemaType[];
return schemaObj;
} catch (error: any) {
serverError({
component: "grabUserSchemaData",
package-shared/functions/backend/setUserSchemaData.ts
+8 -5
View File
@@ -2,6 +2,7 @@ import serverError from "./serverError";
import fs from "fs";
import path from "path";
import { DSQL_DatabaseSchemaType } from "../../types";
import grabDirNames from "@/package-shared/utils/backend/names/grab-dir-names";
type Param = {
userId: string | number;
@@ -16,12 +17,14 @@ export default function setUserSchemaData({
schemaData,
}: Param): boolean {
try {
const userSchemaFilePath = path.resolve(
process.cwd(),
`${process.env.DSQL_USER_DB_SCHEMA_PATH}/user-${userId}/main.json`
);
const { userSchemaMainJSONFilePath } = grabDirNames({ userId });
if (!userSchemaMainJSONFilePath) {
throw new Error(`No User Schema JSON found!`);
}
fs.writeFileSync(
userSchemaFilePath,
userSchemaMainJSONFilePath,
JSON.stringify(schemaData),
"utf8"
);
package-shared/functions/dsql/db-schema-to-type.ts
+1 -1
View File
@@ -64,7 +64,7 @@ export default function dbSchemaToType(params?: Params): string[] | undefined {
datasquirelSchema.dbName ||
datasquirelSchema.dbFullName?.replace(/datasquirel_user_\d+_/, "")
)
.toUpperCase()
?.toUpperCase()
.replace(/ /g, "_");
const schemas = dbTablesSchemas
package-shared/functions/dsql/sql/sql-generator.ts
+3 -1
View File
@@ -55,7 +55,9 @@ export default function sqlGenerator<
let str = `${finalFieldName}=?`;
if (
if (queryObj.nullValue) {
str = `${finalFieldName} IS NULL`;
} else if (
typeof queryObj.value == "string" ||
typeof queryObj.value == "number"
) {
package-shared/shell/createDbFromSchema/check-db-record.ts
+2 -2
View File
@@ -31,7 +31,7 @@ export default async function checkDbRecordCreateDbSchema({
let recordedDbEntryArray = userId
? await varDatabaseDbHandler({
queryString: `SELECT * FROM datasquirel.user_databases WHERE db_full_name = ?`,
queryValuesArray: [dbFullName],
queryValuesArray: [dbFullName || "NULL"],
})
: undefined;
@@ -59,7 +59,7 @@ export default async function checkDbRecordCreateDbSchema({
if (newDbEntry.insertId) {
recordedDbEntryArray = await varDatabaseDbHandler({
queryString: `SELECT * FROM datasquirel.user_databases WHERE db_full_name = ?`,
queryValuesArray: [dbFullName],
queryValuesArray: [dbFullName || "NULL"],
});
recordedDbEntry = recordedDbEntryArray?.[0];
}
package-shared/shell/createDbFromSchema/index.ts
+8 -2
View File
@@ -49,6 +49,8 @@ export default async function createDbFromSchema({
const { dbFullName, tables, dbSlug, childrenDatabases } = database;
if (!dbFullName) continue;
if (targetDatabase && dbFullName != targetDatabase) {
continue;
}
@@ -221,11 +223,15 @@ export default async function createDbFromSchema({
if (childrenDatabases?.[0]) {
for (let ch = 0; ch < childrenDatabases.length; ch++) {
const childDb = childrenDatabases[ch];
const { dbFullName } = childDb;
const { dbId } = childDb;
const targetDatabase = dbSchema.find(
(dbSch) => dbSch.id == dbId
);
await createDbFromSchema({
userId,
targetDatabase: dbFullName,
targetDatabase: targetDatabase?.dbFullName,
});
}
}
package-shared/types/index.ts
+39 -7
View File
@@ -1,16 +1,20 @@
import type { RequestOptions } from "https";
import {
DSQL_DATASQUIREL_PROCESS_QUEUE,
DSQL_DATASQUIREL_USER_DATABASE_TABLES,
DSQL_DATASQUIREL_USER_DATABASES,
DSQL_DATASQUIREL_USER_MEDIA,
} from "./dsql";
import { Editor } from "tinymce";
import sharp from "sharp";
export type DSQL_DatabaseFullName = string;
export interface DSQL_DatabaseSchemaType {
dbName: string;
dbSlug: string;
dbFullName: string;
id?: number | string;
dbName?: string;
dbSlug?: string;
dbFullName?: string;
dbDescription?: string;
dbImage?: string;
tables: DSQL_TableSchemaType[];
@@ -21,10 +25,12 @@ export interface DSQL_DatabaseSchemaType {
}
export interface DSQL_ChildrenDatabaseObject {
dbFullName: string;
dbId?: string | number;
dbFullName?: string;
}
export interface DSQL_TableSchemaType {
id?: number | string;
tableName: string;
tableFullName: string;
tableDescription?: string;
@@ -540,11 +546,11 @@ export interface LoginFormContextType {
export interface CreateAccountContextType {
user?: UserType | null;
query: CreateAccountQueryType;
query: InviteObjectType;
invitingUser: any;
}
export interface CreateAccountQueryType {
export interface InviteObjectType {
invite?: number;
database_access?: string;
priviledge?: string;
@@ -824,8 +830,9 @@ export interface DbConnectContextType {
export interface ImageObjectType {
imageName?: string;
mimeType?: string;
mimeType?: keyof sharp.FormatEnum | sharp.AvailableFormatInfo;
imageSize?: number;
thumbnailSize?: number;
private?: boolean;
imageBase64?: string;
imageBase64Full?: string;
@@ -1127,6 +1134,7 @@ export type ServerQueryParam<
export type ServerQueryObject<T extends object = { [key: string]: any }> = {
value?: string | string[];
nullValue?: boolean;
operator?: (typeof ServerQueryOperators)[number];
equality?: (typeof ServerQueryEqualities)[number];
tableName?: string;
@@ -1606,10 +1614,20 @@ export type PagePropsType = {
pageUrl?: string | null;
query?: any;
databases?: DSQL_DATASQUIREL_USER_DATABASES[] | null;
database?: DSQL_DATASQUIREL_USER_DATABASES | null;
databaseTables?: DSQL_DATASQUIREL_USER_DATABASE_TABLES[] | null;
databaseTable?: DSQL_DATASQUIREL_USER_DATABASE_TABLES | null;
dbCount?: number | null;
tableCount?: number | null;
mediaCount?: number | null;
apiKeysCount?: number | null;
databaseSchema?: DSQL_DatabaseSchemaType | null;
tableSchema?: DSQL_TableSchemaType | null;
userMedia?: DSQL_DATASQUIREL_USER_MEDIA[] | null;
mediaCurrentFolder?: string | null;
appData?: DsqlAppData | null;
staticHost?: string | null;
folders?: string[] | null;
};
export type APIResponseObject<T extends any = any> = {
@@ -1782,3 +1800,17 @@ export type DsqlAppData = {
DSQL_FACEBOOK_APP_ID?: string;
DSQL_GITHUB_ID?: string;
};
export const MediaTypes = ["image", "file"] as const;
export type MediaUploadDataType = ImageObjectType &
FileObjectType & { private?: boolean };
export const ImageMimeTypes: (keyof sharp.FormatEnum)[] = [
"webp",
"gif",
"svg",
"png",
"jpeg",
"jpg",
] as const;
package-shared/utils/backend/names/grab-db-full-name.ts
+22
View File
@@ -0,0 +1,22 @@
import { DATASQUIREL_LoggedInUser, UserType } from "../../../types";
type Param = {
user?: DATASQUIREL_LoggedInUser | UserType;
userId?: string | number | null;
dbSlug?: string;
};
export default function grabUserDbFullName({ dbSlug, user, userId }: Param) {
const finalUserId = user?.id || userId;
if (!finalUserId || !dbSlug)
throw new Error(
`Couldn't grab full DB name. Missing parameters finalUserId || dbSlug`
);
if (dbSlug.match(/[^a-zA-Z0-9-_]/)) {
throw new Error(`Invalid Database slug`);
}
return `datasquirel_user_${finalUserId}_${dbSlug}`;
}
package-shared/utils/backend/names/grab-dir-names.ts
+20 -4
View File
@@ -8,6 +8,11 @@ type Param = {
};
export default function grabDirNames(param?: Param) {
const appDir = param?.appDir || process.env.DSQL_APP_DIR;
const STATIC_ROOT = process.env.DSQL_STATIC_SERVER_DIR || "/static";
const finalUserId = param?.user?.id || param?.userId;
const publicImagesDir = path.join(STATIC_ROOT, `images`);
if (!appDir)
throw new Error("Please provide the `DSQL_APP_DIR` env variable.");
@@ -32,11 +37,15 @@ export default function grabDirNames(param?: Param) {
);
const usersSchemaDir = path.join(schemasDir, `users`);
const targetUserSchemaDir = finalUserId
? path.join(usersSchemaDir, `user-${finalUserId}`)
: undefined;
const userTempSQLFilePath = targetUserSchemaDir
? path.join(targetUserSchemaDir, `tmp.sql`)
: undefined;
const userDirPath = param?.user?.id
? path.join(usersSchemaDir, `user-${param.user.id}`)
: param?.userId
? path.join(usersSchemaDir, `user-${param.userId}`)
const userDirPath = finalUserId
? path.join(usersSchemaDir, `user-${finalUserId}`)
: undefined;
const userSchemaMainJSONFilePath = userDirPath
? path.join(userDirPath, `main.json`)
@@ -61,6 +70,10 @@ export default function grabDirNames(param?: Param) {
? path.join(userPrivateSQLExportsDir, userPrivateDbExportZipFileName)
: undefined;
const userPublicMediaDir = finalUserId
? path.join(publicImagesDir, `user-images/user-${finalUserId}`)
: undefined;
const userPrivateDbImportZipFileName = `db-export.zip`;
const userPrivateDbImportZipFilePath = userPrivateSQLExportsDir
? path.join(userPrivateSQLExportsDir, userPrivateDbImportZipFileName)
@@ -101,6 +114,7 @@ export default function grabDirNames(param?: Param) {
tempDirName,
defaultTableFieldsJSONFilePath,
usersSchemaDir,
targetUserSchemaDir,
userSchemaMainJSONFilePath,
userPrivateMediaDir,
userPrivateExportsDir,
@@ -121,5 +135,7 @@ export default function grabDirNames(param?: Param) {
siteSetupFile,
envFile,
testEnvFile,
userPublicMediaDir,
userTempSQLFilePath,
};
}
package-shared/utils/grab-db-full-name.ts
+6 -1
View File
@@ -1,5 +1,5 @@
type Param = {
dbName: string;
dbName?: string;
userId?: string | number;
};
@@ -7,6 +7,11 @@ type Param = {
* # Grab Database Full Name
*/
export default function grabDbFullName({ dbName, userId }: Param): string {
if (!dbName)
throw new Error(
`Database name not provided to db name parser funciton`
);
const sanitizedName = dbName.replace(/[^a-z0-9\_]/g, "");
const cleanedDbName = sanitizedName.replace(/datasquirel_user_\d+_/, "");