Moduletrace/datasquirel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updates

Browse Source
This commit is contained in:
Benjamin Toby
2025-07-18 18:34:04 +01:00
parent a53b6e6974
commit 20a390e4a8
73 changed files with 1261 additions and 751 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-shared/functions/api/query-dsql-api.ts
+11 -2
View File
@@ -9,6 +9,7 @@ import {
import grabHostNames from "../../utils/grab-host-names";
import serializeQuery from "../../utils/serialize-query";
import { RequestOptions } from "https";
import _ from "lodash";
type Param<T = { [k: string]: any }> = {
body?: T;
@@ -115,7 +116,12 @@ export default async function queryDSQLAPI<
payload: undefined,
msg: `An error occurred while parsing the response`,
error: error.message,
errorData: { requestOptions, grabedHostNames },
errorData: {
requestOptions,
grabedHostNames: _.omit(grabedHostNames, [
"scheme",
]),
},
} as APIResponseObject);
}
});
@@ -138,7 +144,10 @@ export default async function queryDSQLAPI<
payload: undefined,
msg: `An error occurred while making the request`,
error: err.message,
errorData: { requestOptions, grabedHostNames },
errorData: {
requestOptions,
grabedHostNames: _.omit(grabedHostNames, ["scheme"]),
},
} as APIResponseObject);
});
package-shared/functions/api/social-login/handleSocialDb.ts
+9 -27
View File
@@ -7,7 +7,7 @@ import encrypt from "../../dsql/encrypt";
import addDbEntry from "../../backend/db/addDbEntry";
import loginSocialUser from "./loginSocialUser";
import {
APILoginFunctionReturn,
APIResponseObject,
HandleSocialDbFunctionParams,
} from "../../../types";
import grabDirNames from "../../../utils/backend/names/grab-dir-names";
@@ -27,7 +27,7 @@ export default async function handleSocialDb({
debug,
loginOnly,
apiUserId,
}: HandleSocialDbFunctionParams): Promise<APILoginFunctionReturn> {
}: HandleSocialDbFunctionParams): Promise<APIResponseObject> {
try {
const finalDbName = grabDbFullName({
dbName: database,
@@ -200,16 +200,9 @@ export default async function handleSocialDb({
}).then(() => {});
}
const { STATIC_ROOT } = grabDirNames();
if (!STATIC_ROOT) {
console.log("Static File ENV not Found!");
return {
success: false,
payload: null,
msg: "Static File ENV not Found!",
};
}
const { userPrivateMediaDir, userPublicMediaDir } = grabDirNames({
userId: newUser.payload.insertId,
});
/**
* Create new user folder and file
@@ -217,21 +210,10 @@ export default async function handleSocialDb({
* @description Create new user folder and file
*/
if (!database || database?.match(/^datasquirel$/)) {
let newUserSchemaFolderPath = `${process.env.DSQL_USER_DB_SCHEMA_PATH}/user-${newUser.payload.insertId}`;
let newUserMediaFolderPath = path.join(
STATIC_ROOT,
`images/user-images/user-${newUser.payload.insertId}`
);
fs.mkdirSync(newUserSchemaFolderPath);
fs.mkdirSync(newUserMediaFolderPath);
fs.writeFileSync(
`${newUserSchemaFolderPath}/main.json`,
JSON.stringify([]),
"utf8"
);
userPublicMediaDir &&
fs.mkdirSync(userPublicMediaDir, { recursive: true });
userPrivateMediaDir &&
fs.mkdirSync(userPrivateMediaDir, { recursive: true });
}
return await loginSocialUser({
package-shared/functions/api/social-login/loginSocialUser.ts
+12 -69
View File
@@ -1,16 +1,9 @@
import addAdminUserOnLogin from "../../backend/addAdminUserOnLogin";
import dbHandler from "../../backend/dbHandler";
import {
APILoginFunctionReturn,
DATASQUIREL_LoggedInUser,
} from "../../../types";
import { APIResponseObject } from "../../../types";
import loginUser from "../../../actions/users/login-user";
type Param = {
user: {
first_name: string;
last_name: string;
email: string;
social_id: string | number;
};
social_platform: string;
invitation?: any;
@@ -32,68 +25,18 @@ export default async function loginSocialUser({
database,
additionalFields,
debug,
}: Param): Promise<APILoginFunctionReturn> {
}: Param): Promise<APIResponseObject> {
const finalDbName = database ? database : "datasquirel";
const dbAppend = database ? `\`${finalDbName}\`.` : "";
const foundUserQuery = `SELECT * FROM ${dbAppend}\`users\` WHERE email=?`;
const foundUserValues = [user.email];
const foundUser = (await dbHandler({
query: foundUserQuery,
values: foundUserValues,
let userPayload = await loginUser({
database: finalDbName,
})) as any[];
payload: { email: user.email },
skipPassword: true,
skipWriteAuthFile: true,
additionalFields,
debug,
useLocal: true,
});
if (!foundUser?.[0])
return {
success: false,
payload: null,
msg: "Couldn't find Social User.",
};
let csrfKey =
Math.random().toString(36).substring(2) +
"-" +
Math.random().toString(36).substring(2);
let userPayload: DATASQUIREL_LoggedInUser = {
id: foundUser[0].id,
uuid: foundUser[0].uuid,
first_name: foundUser[0].first_name,
last_name: foundUser[0].last_name,
username: foundUser[0].username,
user_type: foundUser[0].user_type,
email: foundUser[0].email,
social_id: foundUser[0].social_id,
image: foundUser[0].image,
image_thumbnail: foundUser[0].image_thumbnail,
verification_status: foundUser[0].verification_status,
social_login: foundUser[0].social_login,
social_platform: foundUser[0].social_platform,
csrf_k: csrfKey,
logged_in_status: true,
date: Date.now(),
};
if (additionalFields?.[0]) {
additionalFields.forEach((key) => {
userPayload[key] = foundUser[0][key];
});
}
if (invitation && (!database || database?.match(/^datasquirel$/))) {
addAdminUserOnLogin({
query: invitation,
user: userPayload,
});
}
let result: APILoginFunctionReturn = {
success: true,
payload: userPayload,
csrf: csrfKey,
};
return result;
return userPayload;
}
package-shared/functions/api/users/social/api-google-login.ts
+2 -2
View File
@@ -3,9 +3,9 @@ import handleSocialDb from "../../social-login/handleSocialDb";
import EJSON from "../../../../utils/ejson";
import {
APIGoogleLoginFunctionParams,
APILoginFunctionReturn,
GoogleOauth2User,
} from "../../../../types";
import { APIResponseObject } from "../../../../types";
/**
* # API google login
@@ -18,7 +18,7 @@ export default async function apiGoogleLogin({
debug,
loginOnly,
apiUserId,
}: APIGoogleLoginFunctionParams): Promise<APILoginFunctionReturn> {
}: APIGoogleLoginFunctionParams): Promise<APIResponseObject> {
try {
const gUser: GoogleOauth2User | undefined = await new Promise(
(resolve, reject) => {
package-shared/functions/backend/auth/post-login-response-handler.ts
+97
View File
@@ -0,0 +1,97 @@
import { ServerResponse } from "http";
import { APIResponseObject } from "../../../types";
import encrypt from "../../dsql/encrypt";
import debugLog from "../../../utils/logging/debug-log";
import getAuthCookieNames from "../cookies/get-auth-cookie-names";
import { writeAuthFile } from "./write-auth-files";
import grabCookieExpiryDate from "../../../utils/grab-cookie-expirt-date";
function debugFn(log: any, label?: string) {
debugLog({ log, addTime: true, title: "loginUser", label });
}
type Params = {
database: string;
httpResponse: APIResponseObject;
response?: ServerResponse & { [s: string]: any };
encryptionKey?: string;
encryptionSalt?: string;
debug?: boolean;
skipWriteAuthFile?: boolean;
token?: boolean;
cleanupTokens?: boolean;
secureCookie?: boolean;
};
/**
* # Login A user
*/
export default function postLoginResponseHandler({
database,
httpResponse,
response,
encryptionKey,
encryptionSalt,
debug,
token,
skipWriteAuthFile,
cleanupTokens,
secureCookie,
}: Params): boolean {
const COOKIE_EXPIRY_DATE = grabCookieExpiryDate();
if (httpResponse?.success) {
let encryptedPayload = encrypt({
data: JSON.stringify(httpResponse.payload),
encryptionKey,
encryptionSalt,
});
try {
if (token && encryptedPayload)
httpResponse["token"] = encryptedPayload;
} catch (error: any) {
console.log("Login User HTTP Response Error:", error.message);
}
const cookieNames = getAuthCookieNames({
database,
});
if (httpResponse.csrf && !skipWriteAuthFile) {
writeAuthFile(
httpResponse.csrf,
JSON.stringify(httpResponse.payload),
cleanupTokens && httpResponse.payload?.id
? { userId: httpResponse.payload.id }
: undefined
);
}
httpResponse["cookieNames"] = cookieNames;
httpResponse["key"] = String(encryptedPayload);
const authKeyName = cookieNames.keyCookieName;
const csrfName = cookieNames.csrfCookieName;
if (debug) {
debugFn(authKeyName, "authKeyName");
debugFn(csrfName, "csrfName");
debugFn(encryptedPayload, "encryptedPayload");
}
response?.setHeader("Set-Cookie", [
`${authKeyName}=${encryptedPayload};samesite=strict;path=/;HttpOnly=true;Expires=${COOKIE_EXPIRY_DATE}${
secureCookie ? ";Secure=true" : ""
}`,
`${csrfName}=${httpResponse.payload?.csrf_k};samesite=strict;path=/;HttpOnly=true;Expires=${COOKIE_EXPIRY_DATE}`,
]);
if (debug) {
debugFn("Response Sent!");
}
return true;
} else {
return false;
}
}
package-shared/functions/backend/backups/su/add-backup/index.ts
+6 -5
View File
@@ -1,6 +1,6 @@
import _ from "lodash";
import path from "path";
import writeBacupFiles from "./write-backup-files";
import writeBackupFiles from "./write-backup-files";
import { APIResponseObject } from "../../../../../types";
import grabDirNames from "../../../../../utils/backend/names/grab-dir-names";
import {
@@ -19,9 +19,10 @@ export default async function suAddBackup({
targetUserId,
}: Params): Promise<APIResponseObject> {
try {
const { mainBackupDir, userBackupDir } = grabDirNames({
userId: targetUserId,
});
const { mainBackupDir, userBackupDir, STATIC_ROOT, privateDataDir } =
grabDirNames({
userId: targetUserId,
});
if (targetUserId && !userBackupDir) {
return {
@@ -63,7 +64,7 @@ export default async function suAddBackup({
};
}
const writeBackup = await writeBacupFiles({
const writeBackup = await writeBackupFiles({
backup: newlyAddedBackup,
});
package-shared/functions/backend/backups/su/add-backup/write-backup-files.ts
+2
View File
@@ -22,6 +22,8 @@ export default async function writeBackupFiles({
schemasBackupDirName,
targetUserPrivateDir,
oldSchemasDir,
STATIC_ROOT,
privateDataDir,
} = grabDirNames({
userId: backup.user_id,
});
package-shared/functions/backend/backups/su/download-backup.ts
+73
View File
@@ -0,0 +1,73 @@
import fs from "fs";
import path from "path";
import _ from "lodash";
import { DSQL_DATASQUIREL_BACKUPS } from "../../../../types/dsql";
import { APIResponseObject } from "../../../../types";
import grabDirNames from "../../../../utils/backend/names/grab-dir-names";
import { NextApiResponse } from "next";
import { execSync } from "child_process";
type Params = {
backup: DSQL_DATASQUIREL_BACKUPS;
res: NextApiResponse;
};
export default async function downloadBackup({
backup,
res,
}: Params): Promise<APIResponseObject> {
try {
const { mainBackupDir, userBackupDir, tempBackupExportName } =
grabDirNames({
userId: backup.user_id,
});
if (backup.user_id && !userBackupDir) {
return {
success: false,
msg: `Error grabbing user backup directory`,
};
}
if (!backup.uuid) {
return {
success: false,
msg: `No UUID found for backup`,
};
}
const allBackupsDir =
backup.user_id && userBackupDir ? userBackupDir : mainBackupDir;
const targetBackupDir = path.join(allBackupsDir, backup.uuid);
const zipFilesCmd = execSync(
`tar -cJf ${tempBackupExportName} ${backup.uuid}`,
{
cwd: allBackupsDir,
}
);
const exportFilePath = path.join(allBackupsDir, tempBackupExportName);
const readStream = fs.createReadStream(exportFilePath);
readStream.pipe(res);
readStream.on("end", () => {
console.log("Pipe Complete!");
setTimeout(() => {
execSync(`rm -f ${tempBackupExportName}`, {
cwd: allBackupsDir,
});
}, 1000);
});
return { success: true };
} catch (error: any) {
return {
success: false,
msg: `Failed to write backup files`,
error: error.message,
};
}
}
package-shared/functions/backend/db/addDbEntry.ts
+12 -56
View File
@@ -1,8 +1,5 @@
import sanitizeHtml from "sanitize-html";
import sanitizeHtmlOptions from "../html/sanitizeHtmlOptions";
import updateDbEntry from "./updateDbEntry";
import _ from "lodash";
import encrypt from "../../dsql/encrypt";
import connDbHandler from "../../../utils/db/conn-db-handler";
import checkIfIsMaster from "../../../utils/check-if-is-master";
import { DbContextsArray } from "./runQuery";
@@ -13,6 +10,7 @@ import {
PostInsertReturn,
} from "../../../types";
import purgeDefaultFields from "../../../utils/purge-default-fields";
import grabParsedValue from "./grab-parsed-value";
export type AddDbEntryParam<
T extends { [k: string]: any } = any,
@@ -127,64 +125,22 @@ export default async function addDbEntry<
const dataKey = dataKeys[i];
let value = data[dataKey];
const targetFieldSchemaArray = tableSchema
? tableSchema?.fields?.filter(
(field) => field.fieldName == dataKey
)
: null;
const targetFieldSchema =
targetFieldSchemaArray && targetFieldSchemaArray[0]
? targetFieldSchemaArray[0]
: null;
const parsedValue = grabParsedValue({
dataKey,
encryptionKey,
encryptionSalt,
tableSchema,
value,
});
if (value == null || value == undefined) continue;
if (
targetFieldSchema?.dataType?.match(/int$/i) &&
typeof value == "string" &&
!value?.match(/./)
)
continue;
if (targetFieldSchema?.encrypted) {
value = encrypt({
data: value,
encryptionKey,
encryptionSalt,
});
console.log("DSQL: Encrypted value =>", value);
}
const htmlRegex = /<[^>]+>/g;
if (
targetFieldSchema?.richText ||
String(value).match(htmlRegex)
) {
value = sanitizeHtml(value, sanitizeHtmlOptions);
}
if (targetFieldSchema?.pattern) {
const pattern = new RegExp(
targetFieldSchema.pattern,
targetFieldSchema.patternFlags || ""
);
if (!pattern.test(value)) {
console.log("DSQL: Pattern not matched =>", value);
value = "";
}
}
if (typeof parsedValue == "undefined") continue;
insertKeysArray.push("`" + dataKey + "`");
if (typeof value === "object") {
value = JSON.stringify(value);
}
if (typeof value == "number") {
insertValuesArray.push(String(value));
if (typeof parsedValue == "number") {
insertValuesArray.push(String(parsedValue));
} else {
insertValuesArray.push(value);
insertValuesArray.push(parsedValue);
}
} catch (error: any) {
console.log(
package-shared/functions/backend/db/grab-parsed-value.ts
+93
View File
@@ -0,0 +1,93 @@
import sanitizeHtml from "sanitize-html";
import sanitizeHtmlOptions from "../html/sanitizeHtmlOptions";
import encrypt from "../../dsql/encrypt";
import { DSQL_TableSchemaType } from "../../../types";
import _ from "lodash";
type Param = {
value?: any;
tableSchema?: DSQL_TableSchemaType;
encryptionKey?: string;
encryptionSalt?: string;
dataKey: string;
};
/**
* # Update DB Function
* @description
*/
export default function grabParsedValue({
value,
tableSchema,
encryptionKey,
encryptionSalt,
dataKey,
}: Param): any {
let newValue = value;
const targetFieldSchemaArray = tableSchema
? tableSchema?.fields?.filter((field) => field.fieldName === dataKey)
: null;
const targetFieldSchema =
targetFieldSchemaArray && targetFieldSchemaArray[0]
? targetFieldSchemaArray[0]
: null;
if (typeof newValue == "undefined") return;
if (typeof newValue == "object" && !newValue) newValue = "";
const htmlRegex = /<[^>]+>/g;
if (targetFieldSchema?.richText || String(newValue).match(htmlRegex)) {
newValue = sanitizeHtml(newValue, sanitizeHtmlOptions);
}
if (
targetFieldSchema?.dataType?.match(/int$/i) &&
typeof value == "string" &&
!value?.match(/./)
) {
value = "";
}
if (targetFieldSchema?.encrypted) {
newValue = encrypt({
data: newValue,
encryptionKey,
encryptionSalt,
});
}
if (typeof newValue === "object") {
newValue = JSON.stringify(newValue);
}
if (targetFieldSchema?.pattern) {
const pattern = new RegExp(
targetFieldSchema.pattern,
targetFieldSchema.patternFlags || ""
);
if (!pattern.test(newValue)) {
console.log("DSQL: Pattern not matched =>", newValue);
newValue = "";
}
}
if (typeof newValue === "string" && newValue.match(/^null$/i)) {
newValue = {
toSqlString: function () {
return "NULL";
},
};
}
if (typeof newValue === "string" && !newValue.match(/./i)) {
newValue = {
toSqlString: function () {
return "NULL";
},
};
}
return newValue;
}
package-shared/functions/backend/db/updateDbEntry.ts
+12 -61
View File
@@ -1,6 +1,3 @@
import sanitizeHtml from "sanitize-html";
import sanitizeHtmlOptions from "../html/sanitizeHtmlOptions";
import encrypt from "../../dsql/encrypt";
import checkIfIsMaster from "../../../utils/check-if-is-master";
import connDbHandler from "../../../utils/db/conn-db-handler";
import { DbContextsArray } from "./runQuery";
@@ -11,6 +8,7 @@ import {
} from "../../../types";
import _ from "lodash";
import purgeDefaultFields from "../../../utils/purge-default-fields";
import grabParsedValue from "./grab-parsed-value";
type Param<T extends { [k: string]: any } = any> = {
dbContext?: (typeof DbContextsArray)[number];
@@ -82,69 +80,22 @@ export default async function updateDbEntry<
const dataKey = dataKeys[i];
let value = newData[dataKey];
const targetFieldSchemaArray = tableSchema
? tableSchema?.fields?.filter(
(field) => field.fieldName === dataKey
)
: null;
const targetFieldSchema =
targetFieldSchemaArray && targetFieldSchemaArray[0]
? targetFieldSchemaArray[0]
: null;
const parsedValue = grabParsedValue({
dataKey,
encryptionKey,
encryptionSalt,
tableSchema,
value,
});
if (value == null || value == undefined) continue;
const htmlRegex = /<[^>]+>/g;
if (targetFieldSchema?.richText || String(value).match(htmlRegex)) {
value = sanitizeHtml(value, sanitizeHtmlOptions);
}
if (targetFieldSchema?.encrypted) {
value = encrypt({
data: value,
encryptionKey,
encryptionSalt,
});
}
if (typeof value === "object") {
value = JSON.stringify(value);
}
if (targetFieldSchema?.pattern) {
const pattern = new RegExp(
targetFieldSchema.pattern,
targetFieldSchema.patternFlags || ""
);
if (!pattern.test(value)) {
console.log("DSQL: Pattern not matched =>", value);
value = "";
}
}
if (typeof value === "string" && value.match(/^null$/i)) {
value = {
toSqlString: function () {
return "NULL";
},
};
}
if (typeof value === "string" && !value.match(/./i)) {
value = {
toSqlString: function () {
return "NULL";
},
};
}
if (typeof parsedValue == "undefined") continue;
updateKeyValueArray.push(`\`${dataKey}\`=?`);
if (typeof value == "number") {
updateValues.push(String(value));
if (typeof parsedValue == "number") {
updateValues.push(String(parsedValue));
} else {
updateValues.push(value);
updateValues.push(parsedValue);
}
////////////////////////////////////////
package-shared/functions/backend/handle-backup.ts
+13 -6
View File
@@ -15,7 +15,7 @@ export default async function handleBackup({
}: HandleBackupParams) {
const { appConfig } = grabConfig();
const maxBackups = appConfig.main.max_backups?.value || 20;
const maxBackups = appConfig.main.max_backups?.value || 4;
const { count: existingAppBackupsCount } =
await dbGrabUserResource<DSQL_DATASQUIREL_BACKUPS>({
@@ -33,7 +33,9 @@ export default async function handleBackup({
});
if (existingAppBackupsCount && existingAppBackupsCount >= maxBackups) {
const { single: oldestAppBackup } =
console.log(`Backups exceed Limit ...`);
const { batch: oldestAppBackups } =
await dbGrabUserResource<DSQL_DATASQUIREL_BACKUPS>({
tableName: "backups",
isSuperUser: true,
@@ -46,14 +48,19 @@ export default async function handleBackup({
},
order: {
field: "id",
strategy: "ASC",
strategy: "DESC",
},
limit: 1,
},
});
if (oldestAppBackup?.id) {
await deleteBackup({ backup: oldestAppBackup });
if (oldestAppBackups) {
for (let i = 0; i < oldestAppBackups.length; i++) {
const backup = oldestAppBackups[i];
console.log(`Handling Backup ${backup.uuid} ...`);
if (i < maxBackups - 1) continue;
console.log(`Deleting Backup ${backup.uuid} ...`);
await deleteBackup({ backup: backup });
}
}
}
package-shared/functions/backend/httpsRequest.ts
+10 -22
View File
@@ -7,6 +7,7 @@ type Param = {
url?: string;
method?: string;
hostname?: string;
host?: string;
path?: string;
port?: number | string;
headers?: object;
@@ -16,28 +17,24 @@ type Param = {
/**
* # Make Https Request
*/
export default function httpsRequest({
export default function httpsRequest<Res extends any = any>({
url,
method,
hostname,
host,
path,
headers,
body,
port,
scheme,
}: Param) {
}: Param): Promise<Res> {
const reqPayloadString = body ? JSON.stringify(body) : null;
const PARSED_URL = url ? new URL(url) : null;
////////////////////////////////////////////////
////////////////////////////////////////////////
////////////////////////////////////////////////
/** @type {any} */
let requestOptions: any = {
method: method || "GET",
hostname: PARSED_URL ? PARSED_URL.hostname : hostname,
hostname: PARSED_URL ? PARSED_URL.hostname : host || hostname,
port: scheme?.match(/https/i)
? 443
: PARSED_URL
@@ -51,7 +48,6 @@ export default function httpsRequest({
};
if (path) requestOptions.path = path;
// if (href) requestOptions.href = href;
if (headers) requestOptions.headers = headers;
if (body) {
@@ -61,10 +57,6 @@ export default function httpsRequest({
: undefined;
}
////////////////////////////////////////////////
////////////////////////////////////////////////
////////////////////////////////////////////////
return new Promise((res, rej) => {
const httpsRequest = (
scheme?.match(/https/i)
@@ -73,25 +65,21 @@ export default function httpsRequest({
? https
: http
).request(
/* ====== Request Options object ====== */
requestOptions,
////////////////////////////////////////////////
////////////////////////////////////////////////
////////////////////////////////////////////////
/* ====== Callback function ====== */
(response) => {
var str = "";
// ## another chunk of data has been received, so append it to `str`
response.on("data", function (chunk) {
str += chunk;
});
// ## the whole response has been received, so we just print it out here
response.on("end", function () {
res(str);
try {
res(JSON.parse(str));
} catch (error) {
res(str as any);
}
});
response.on("error", (error) => {