Moduletrace/datasquirel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updates

Browse Source
This commit is contained in:
Benjamin Toby
2025-07-18 18:34:04 +01:00
parent a53b6e6974
commit 20a390e4a8
73 changed files with 1261 additions and 751 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-shared/README.md
+12
View File
@@ -4,4 +4,16 @@ This directory contains data (mostly type definitions) shared by both the datasq
## Functions
### Actions
These are functions that are used by both the datasquirel NPM package and the datasquirel web app
### Utils
These are utility functions that are used by both the datasquirel NPM package and the datasquirel web app
### API
These are API functions that are used by both the datasquirel NPM package and the datasquirel web app
## Types
package-shared/actions/users/login-user.ts
+11 -47
View File
@@ -12,6 +12,7 @@ import debugLog from "../../utils/logging/debug-log";
import grabCookieExpiryDate from "../../utils/grab-cookie-expirt-date";
import grabUserDSQLAPIPath from "../../utils/backend/users/grab-api-path";
import queryDSQLAPI from "../../functions/api/query-dsql-api";
import postLoginResponseHandler from "../../functions/backend/auth/post-login-response-handler";
function debugFn(log: any, label?: string) {
debugLog({ log, addTime: true, title: "loginUser", label });
@@ -130,55 +131,18 @@ export default async function loginUser<
*/
if (httpResponse?.success) {
let encryptedPayload = encrypt({
data: JSON.stringify(httpResponse.payload),
encryptionKey: finalEncryptionKey,
encryptionSalt: finalEncryptionSalt,
});
try {
if (token && encryptedPayload)
httpResponse["token"] = encryptedPayload;
} catch (error: any) {
console.log("Login User HTTP Response Error:", error.message);
}
const cookieNames = getAuthCookieNames({
postLoginResponseHandler({
database,
httpResponse,
cleanupTokens,
debug,
encryptionKey,
encryptionSalt,
response,
secureCookie,
skipWriteAuthFile,
token,
});
if (httpResponse.csrf && !skipWriteAuthFile) {
writeAuthFile(
httpResponse.csrf,
JSON.stringify(httpResponse.payload),
cleanupTokens && httpResponse.payload?.id
? { userId: httpResponse.payload.id }
: undefined
);
}
httpResponse["cookieNames"] = cookieNames;
httpResponse["key"] = String(encryptedPayload);
const authKeyName = cookieNames.keyCookieName;
const csrfName = cookieNames.csrfCookieName;
if (debug) {
debugFn(authKeyName, "authKeyName");
debugFn(csrfName, "csrfName");
debugFn(encryptedPayload, "encryptedPayload");
}
response?.setHeader("Set-Cookie", [
`${authKeyName}=${encryptedPayload};samesite=strict;path=/;HttpOnly=true;Expires=${COOKIE_EXPIRY_DATE}${
secureCookie ? ";Secure=true" : ""
}`,
`${csrfName}=${httpResponse.payload?.csrf_k};samesite=strict;path=/;HttpOnly=true;Expires=${COOKIE_EXPIRY_DATE}`,
]);
if (debug) {
debugFn("Response Sent!");
}
}
return httpResponse;
package-shared/actions/users/send-email-code.ts
+4 -1
View File
@@ -28,6 +28,7 @@ export default async function sendEmailCode(
useLocal,
apiVersion,
dbUserId,
html,
} = params;
const defaultTempLoginFieldName = "temp_login_code";
@@ -35,7 +36,9 @@ export default async function sendEmailCode(
? temp_code_field_name
: defaultTempLoginFieldName;
const emailHtml = `<p>Please use this code to login</p>\n<h2>{{code}}</h2>\n<p>Please note that this code expires after 15 minutes</p>`;
const emailHtml =
html ||
`<p>Please use this code to login</p>\n<h2>{{code}}</h2>\n<p>Please note that this code expires after 15 minutes</p>`;
const apiSendEmailCodeParams: APISendEmailCodeFunctionParams = {
database,
package-shared/actions/users/social/google-auth.ts
+13 -34
View File
@@ -1,15 +1,12 @@
import encrypt from "../../../functions/dsql/encrypt";
import apiGoogleLogin from "../../../functions/api/users/social/api-google-login";
import getAuthCookieNames from "../../../functions/backend/cookies/get-auth-cookie-names";
import { writeAuthFile } from "../../../functions/backend/auth/write-auth-files";
import {
APIGoogleLoginFunctionParams,
APIResponseObject,
GoogleAuthParams,
} from "../../../types";
import grabCookieExpiryDate from "../../../utils/grab-cookie-expirt-date";
import queryDSQLAPI from "../../../functions/api/query-dsql-api";
import grabUserDSQLAPIPath from "../../../utils/backend/users/grab-api-path";
import postLoginResponseHandler from "../../../functions/backend/auth/post-login-response-handler";
/**
* # SERVER FUNCTION: Login with google Function
@@ -29,9 +26,9 @@ export default async function googleAuth({
loginOnly,
useLocal,
apiVersion,
skipWriteAuthFile,
cleanupTokens,
}: GoogleAuthParams): Promise<APIResponseObject> {
const COOKIE_EXPIRY_DATE = grabCookieExpiryDate();
const finalEncryptionKey =
encryptionKey || process.env.DSQL_ENCRYPTION_PASSWORD;
const finalEncryptionSalt =
@@ -111,36 +108,18 @@ export default async function googleAuth({
*
* @description make a request to datasquirel.com
*/
if (httpResponse?.success && httpResponse?.payload) {
let encryptedPayload = encrypt({
data: JSON.stringify(httpResponse.payload),
encryptionKey: finalEncryptionKey,
encryptionSalt: finalEncryptionSalt,
});
const cookieNames = getAuthCookieNames({
if (httpResponse?.success && httpResponse?.payload && database) {
postLoginResponseHandler({
database,
httpResponse,
cleanupTokens,
debug,
encryptionKey,
encryptionSalt,
response,
secureCookie,
skipWriteAuthFile,
});
if (httpResponse.csrf) {
writeAuthFile(
httpResponse.csrf,
JSON.stringify(httpResponse.payload)
);
}
httpResponse["cookieNames"] = cookieNames;
httpResponse["key"] = String(encryptedPayload);
const authKeyName = cookieNames.keyCookieName;
const csrfName = cookieNames.csrfCookieName;
response?.setHeader("Set-Cookie", [
`${authKeyName}=${encryptedPayload};samesite=strict;path=/;HttpOnly=true;;Expires=${COOKIE_EXPIRY_DATE}${
secureCookie ? ";Secure=true" : ""
}`,
`${csrfName}=${httpResponse.payload?.csrf_k};samesite=strict;path=/;HttpOnly=true;;Expires=${COOKIE_EXPIRY_DATE}`,
]);
}
return httpResponse;
package-shared/dict/app-names.ts
+2
View File
@@ -6,4 +6,6 @@ export const AppNames = {
PrivateMediaInsertTriggerName: "dsql_trg_user_private_folders_insert",
PrivateMediaDeleteTriggerName: "dsql_trg_user_private_folders_delete",
WebsocketPathname: "dsql-websocket",
ReverseProxyForwardURLHeaderName: "x-original-uri",
PrivateAPIAuthHeaderName: "x-api-auth-key",
} as const;
package-shared/functions/api/query-dsql-api.ts
+11 -2
View File
@@ -9,6 +9,7 @@ import {
import grabHostNames from "../../utils/grab-host-names";
import serializeQuery from "../../utils/serialize-query";
import { RequestOptions } from "https";
import _ from "lodash";
type Param<T = { [k: string]: any }> = {
body?: T;
@@ -115,7 +116,12 @@ export default async function queryDSQLAPI<
payload: undefined,
msg: `An error occurred while parsing the response`,
error: error.message,
errorData: { requestOptions, grabedHostNames },
errorData: {
requestOptions,
grabedHostNames: _.omit(grabedHostNames, [
"scheme",
]),
},
} as APIResponseObject);
}
});
@@ -138,7 +144,10 @@ export default async function queryDSQLAPI<
payload: undefined,
msg: `An error occurred while making the request`,
error: err.message,
errorData: { requestOptions, grabedHostNames },
errorData: {
requestOptions,
grabedHostNames: _.omit(grabedHostNames, ["scheme"]),
},
} as APIResponseObject);
});
package-shared/functions/api/social-login/handleSocialDb.ts
+9 -27
View File
@@ -7,7 +7,7 @@ import encrypt from "../../dsql/encrypt";
import addDbEntry from "../../backend/db/addDbEntry";
import loginSocialUser from "./loginSocialUser";
import {
APILoginFunctionReturn,
APIResponseObject,
HandleSocialDbFunctionParams,
} from "../../../types";
import grabDirNames from "../../../utils/backend/names/grab-dir-names";
@@ -27,7 +27,7 @@ export default async function handleSocialDb({
debug,
loginOnly,
apiUserId,
}: HandleSocialDbFunctionParams): Promise<APILoginFunctionReturn> {
}: HandleSocialDbFunctionParams): Promise<APIResponseObject> {
try {
const finalDbName = grabDbFullName({
dbName: database,
@@ -200,16 +200,9 @@ export default async function handleSocialDb({
}).then(() => {});
}
const { STATIC_ROOT } = grabDirNames();
if (!STATIC_ROOT) {
console.log("Static File ENV not Found!");
return {
success: false,
payload: null,
msg: "Static File ENV not Found!",
};
}
const { userPrivateMediaDir, userPublicMediaDir } = grabDirNames({
userId: newUser.payload.insertId,
});
/**
* Create new user folder and file
@@ -217,21 +210,10 @@ export default async function handleSocialDb({
* @description Create new user folder and file
*/
if (!database || database?.match(/^datasquirel$/)) {
let newUserSchemaFolderPath = `${process.env.DSQL_USER_DB_SCHEMA_PATH}/user-${newUser.payload.insertId}`;
let newUserMediaFolderPath = path.join(
STATIC_ROOT,
`images/user-images/user-${newUser.payload.insertId}`
);
fs.mkdirSync(newUserSchemaFolderPath);
fs.mkdirSync(newUserMediaFolderPath);
fs.writeFileSync(
`${newUserSchemaFolderPath}/main.json`,
JSON.stringify([]),
"utf8"
);
userPublicMediaDir &&
fs.mkdirSync(userPublicMediaDir, { recursive: true });
userPrivateMediaDir &&
fs.mkdirSync(userPrivateMediaDir, { recursive: true });
}
return await loginSocialUser({
package-shared/functions/api/social-login/loginSocialUser.ts
+12 -69
View File
@@ -1,16 +1,9 @@
import addAdminUserOnLogin from "../../backend/addAdminUserOnLogin";
import dbHandler from "../../backend/dbHandler";
import {
APILoginFunctionReturn,
DATASQUIREL_LoggedInUser,
} from "../../../types";
import { APIResponseObject } from "../../../types";
import loginUser from "../../../actions/users/login-user";
type Param = {
user: {
first_name: string;
last_name: string;
email: string;
social_id: string | number;
};
social_platform: string;
invitation?: any;
@@ -32,68 +25,18 @@ export default async function loginSocialUser({
database,
additionalFields,
debug,
}: Param): Promise<APILoginFunctionReturn> {
}: Param): Promise<APIResponseObject> {
const finalDbName = database ? database : "datasquirel";
const dbAppend = database ? `\`${finalDbName}\`.` : "";
const foundUserQuery = `SELECT * FROM ${dbAppend}\`users\` WHERE email=?`;
const foundUserValues = [user.email];
const foundUser = (await dbHandler({
query: foundUserQuery,
values: foundUserValues,
let userPayload = await loginUser({
database: finalDbName,
})) as any[];
payload: { email: user.email },
skipPassword: true,
skipWriteAuthFile: true,
additionalFields,
debug,
useLocal: true,
});
if (!foundUser?.[0])
return {
success: false,
payload: null,
msg: "Couldn't find Social User.",
};
let csrfKey =
Math.random().toString(36).substring(2) +
"-" +
Math.random().toString(36).substring(2);
let userPayload: DATASQUIREL_LoggedInUser = {
id: foundUser[0].id,
uuid: foundUser[0].uuid,
first_name: foundUser[0].first_name,
last_name: foundUser[0].last_name,
username: foundUser[0].username,
user_type: foundUser[0].user_type,
email: foundUser[0].email,
social_id: foundUser[0].social_id,
image: foundUser[0].image,
image_thumbnail: foundUser[0].image_thumbnail,
verification_status: foundUser[0].verification_status,
social_login: foundUser[0].social_login,
social_platform: foundUser[0].social_platform,
csrf_k: csrfKey,
logged_in_status: true,
date: Date.now(),
};
if (additionalFields?.[0]) {
additionalFields.forEach((key) => {
userPayload[key] = foundUser[0][key];
});
}
if (invitation && (!database || database?.match(/^datasquirel$/))) {
addAdminUserOnLogin({
query: invitation,
user: userPayload,
});
}
let result: APILoginFunctionReturn = {
success: true,
payload: userPayload,
csrf: csrfKey,
};
return result;
return userPayload;
}
package-shared/functions/api/users/social/api-google-login.ts
+2 -2
View File
@@ -3,9 +3,9 @@ import handleSocialDb from "../../social-login/handleSocialDb";
import EJSON from "../../../../utils/ejson";
import {
APIGoogleLoginFunctionParams,
APILoginFunctionReturn,
GoogleOauth2User,
} from "../../../../types";
import { APIResponseObject } from "../../../../types";
/**
* # API google login
@@ -18,7 +18,7 @@ export default async function apiGoogleLogin({
debug,
loginOnly,
apiUserId,
}: APIGoogleLoginFunctionParams): Promise<APILoginFunctionReturn> {
}: APIGoogleLoginFunctionParams): Promise<APIResponseObject> {
try {
const gUser: GoogleOauth2User | undefined = await new Promise(
(resolve, reject) => {
package-shared/functions/backend/auth/post-login-response-handler.ts
+97
View File
@@ -0,0 +1,97 @@
import { ServerResponse } from "http";
import { APIResponseObject } from "../../../types";
import encrypt from "../../dsql/encrypt";
import debugLog from "../../../utils/logging/debug-log";
import getAuthCookieNames from "../cookies/get-auth-cookie-names";
import { writeAuthFile } from "./write-auth-files";
import grabCookieExpiryDate from "../../../utils/grab-cookie-expirt-date";
function debugFn(log: any, label?: string) {
debugLog({ log, addTime: true, title: "loginUser", label });
}
type Params = {
database: string;
httpResponse: APIResponseObject;
response?: ServerResponse & { [s: string]: any };
encryptionKey?: string;
encryptionSalt?: string;
debug?: boolean;
skipWriteAuthFile?: boolean;
token?: boolean;
cleanupTokens?: boolean;
secureCookie?: boolean;
};
/**
* # Login A user
*/
export default function postLoginResponseHandler({
database,
httpResponse,
response,
encryptionKey,
encryptionSalt,
debug,
token,
skipWriteAuthFile,
cleanupTokens,
secureCookie,
}: Params): boolean {
const COOKIE_EXPIRY_DATE = grabCookieExpiryDate();
if (httpResponse?.success) {
let encryptedPayload = encrypt({
data: JSON.stringify(httpResponse.payload),
encryptionKey,
encryptionSalt,
});
try {
if (token && encryptedPayload)
httpResponse["token"] = encryptedPayload;
} catch (error: any) {
console.log("Login User HTTP Response Error:", error.message);
}
const cookieNames = getAuthCookieNames({
database,
});
if (httpResponse.csrf && !skipWriteAuthFile) {
writeAuthFile(
httpResponse.csrf,
JSON.stringify(httpResponse.payload),
cleanupTokens && httpResponse.payload?.id
? { userId: httpResponse.payload.id }
: undefined
);
}
httpResponse["cookieNames"] = cookieNames;
httpResponse["key"] = String(encryptedPayload);
const authKeyName = cookieNames.keyCookieName;
const csrfName = cookieNames.csrfCookieName;
if (debug) {
debugFn(authKeyName, "authKeyName");
debugFn(csrfName, "csrfName");
debugFn(encryptedPayload, "encryptedPayload");
}
response?.setHeader("Set-Cookie", [
`${authKeyName}=${encryptedPayload};samesite=strict;path=/;HttpOnly=true;Expires=${COOKIE_EXPIRY_DATE}${
secureCookie ? ";Secure=true" : ""
}`,
`${csrfName}=${httpResponse.payload?.csrf_k};samesite=strict;path=/;HttpOnly=true;Expires=${COOKIE_EXPIRY_DATE}`,
]);
if (debug) {
debugFn("Response Sent!");
}
return true;
} else {
return false;
}
}
package-shared/functions/backend/backups/su/add-backup/index.ts
+6 -5
View File
@@ -1,6 +1,6 @@
import _ from "lodash";
import path from "path";
import writeBacupFiles from "./write-backup-files";
import writeBackupFiles from "./write-backup-files";
import { APIResponseObject } from "../../../../../types";
import grabDirNames from "../../../../../utils/backend/names/grab-dir-names";
import {
@@ -19,9 +19,10 @@ export default async function suAddBackup({
targetUserId,
}: Params): Promise<APIResponseObject> {
try {
const { mainBackupDir, userBackupDir } = grabDirNames({
userId: targetUserId,
});
const { mainBackupDir, userBackupDir, STATIC_ROOT, privateDataDir } =
grabDirNames({
userId: targetUserId,
});
if (targetUserId && !userBackupDir) {
return {
@@ -63,7 +64,7 @@ export default async function suAddBackup({
};
}
const writeBackup = await writeBacupFiles({
const writeBackup = await writeBackupFiles({
backup: newlyAddedBackup,
});
package-shared/functions/backend/backups/su/add-backup/write-backup-files.ts
+2
View File
@@ -22,6 +22,8 @@ export default async function writeBackupFiles({
schemasBackupDirName,
targetUserPrivateDir,
oldSchemasDir,
STATIC_ROOT,
privateDataDir,
} = grabDirNames({
userId: backup.user_id,
});
package-shared/functions/backend/backups/su/download-backup.ts
+73
View File
@@ -0,0 +1,73 @@
import fs from "fs";
import path from "path";
import _ from "lodash";
import { DSQL_DATASQUIREL_BACKUPS } from "../../../../types/dsql";
import { APIResponseObject } from "../../../../types";
import grabDirNames from "../../../../utils/backend/names/grab-dir-names";
import { NextApiResponse } from "next";
import { execSync } from "child_process";
type Params = {
backup: DSQL_DATASQUIREL_BACKUPS;
res: NextApiResponse;
};
export default async function downloadBackup({
backup,
res,
}: Params): Promise<APIResponseObject> {
try {
const { mainBackupDir, userBackupDir, tempBackupExportName } =
grabDirNames({
userId: backup.user_id,
});
if (backup.user_id && !userBackupDir) {
return {
success: false,
msg: `Error grabbing user backup directory`,
};
}
if (!backup.uuid) {
return {
success: false,
msg: `No UUID found for backup`,
};
}
const allBackupsDir =
backup.user_id && userBackupDir ? userBackupDir : mainBackupDir;
const targetBackupDir = path.join(allBackupsDir, backup.uuid);
const zipFilesCmd = execSync(
`tar -cJf ${tempBackupExportName} ${backup.uuid}`,
{
cwd: allBackupsDir,
}
);
const exportFilePath = path.join(allBackupsDir, tempBackupExportName);
const readStream = fs.createReadStream(exportFilePath);
readStream.pipe(res);
readStream.on("end", () => {
console.log("Pipe Complete!");
setTimeout(() => {
execSync(`rm -f ${tempBackupExportName}`, {
cwd: allBackupsDir,
});
}, 1000);
});
return { success: true };
} catch (error: any) {
return {
success: false,
msg: `Failed to write backup files`,
error: error.message,
};
}
}
package-shared/functions/backend/db/addDbEntry.ts
+12 -56
View File
@@ -1,8 +1,5 @@
import sanitizeHtml from "sanitize-html";
import sanitizeHtmlOptions from "../html/sanitizeHtmlOptions";
import updateDbEntry from "./updateDbEntry";
import _ from "lodash";
import encrypt from "../../dsql/encrypt";
import connDbHandler from "../../../utils/db/conn-db-handler";
import checkIfIsMaster from "../../../utils/check-if-is-master";
import { DbContextsArray } from "./runQuery";
@@ -13,6 +10,7 @@ import {
PostInsertReturn,
} from "../../../types";
import purgeDefaultFields from "../../../utils/purge-default-fields";
import grabParsedValue from "./grab-parsed-value";
export type AddDbEntryParam<
T extends { [k: string]: any } = any,
@@ -127,64 +125,22 @@ export default async function addDbEntry<
const dataKey = dataKeys[i];
let value = data[dataKey];
const targetFieldSchemaArray = tableSchema
? tableSchema?.fields?.filter(
(field) => field.fieldName == dataKey
)
: null;
const targetFieldSchema =
targetFieldSchemaArray && targetFieldSchemaArray[0]
? targetFieldSchemaArray[0]
: null;
const parsedValue = grabParsedValue({
dataKey,
encryptionKey,
encryptionSalt,
tableSchema,
value,
});
if (value == null || value == undefined) continue;
if (
targetFieldSchema?.dataType?.match(/int$/i) &&
typeof value == "string" &&
!value?.match(/./)
)
continue;
if (targetFieldSchema?.encrypted) {
value = encrypt({
data: value,
encryptionKey,
encryptionSalt,
});
console.log("DSQL: Encrypted value =>", value);
}
const htmlRegex = /<[^>]+>/g;
if (
targetFieldSchema?.richText ||
String(value).match(htmlRegex)
) {
value = sanitizeHtml(value, sanitizeHtmlOptions);
}
if (targetFieldSchema?.pattern) {
const pattern = new RegExp(
targetFieldSchema.pattern,
targetFieldSchema.patternFlags || ""
);
if (!pattern.test(value)) {
console.log("DSQL: Pattern not matched =>", value);
value = "";
}
}
if (typeof parsedValue == "undefined") continue;
insertKeysArray.push("`" + dataKey + "`");
if (typeof value === "object") {
value = JSON.stringify(value);
}
if (typeof value == "number") {
insertValuesArray.push(String(value));
if (typeof parsedValue == "number") {
insertValuesArray.push(String(parsedValue));
} else {
insertValuesArray.push(value);
insertValuesArray.push(parsedValue);
}
} catch (error: any) {
console.log(
package-shared/functions/backend/db/grab-parsed-value.ts
+93
View File
@@ -0,0 +1,93 @@
import sanitizeHtml from "sanitize-html";
import sanitizeHtmlOptions from "../html/sanitizeHtmlOptions";
import encrypt from "../../dsql/encrypt";
import { DSQL_TableSchemaType } from "../../../types";
import _ from "lodash";
type Param = {
value?: any;
tableSchema?: DSQL_TableSchemaType;
encryptionKey?: string;
encryptionSalt?: string;
dataKey: string;
};
/**
* # Update DB Function
* @description
*/
export default function grabParsedValue({
value,
tableSchema,
encryptionKey,
encryptionSalt,
dataKey,
}: Param): any {
let newValue = value;
const targetFieldSchemaArray = tableSchema
? tableSchema?.fields?.filter((field) => field.fieldName === dataKey)
: null;
const targetFieldSchema =
targetFieldSchemaArray && targetFieldSchemaArray[0]
? targetFieldSchemaArray[0]
: null;
if (typeof newValue == "undefined") return;
if (typeof newValue == "object" && !newValue) newValue = "";
const htmlRegex = /<[^>]+>/g;
if (targetFieldSchema?.richText || String(newValue).match(htmlRegex)) {
newValue = sanitizeHtml(newValue, sanitizeHtmlOptions);
}
if (
targetFieldSchema?.dataType?.match(/int$/i) &&
typeof value == "string" &&
!value?.match(/./)
) {
value = "";
}
if (targetFieldSchema?.encrypted) {
newValue = encrypt({
data: newValue,
encryptionKey,
encryptionSalt,
});
}
if (typeof newValue === "object") {
newValue = JSON.stringify(newValue);
}
if (targetFieldSchema?.pattern) {
const pattern = new RegExp(
targetFieldSchema.pattern,
targetFieldSchema.patternFlags || ""
);
if (!pattern.test(newValue)) {
console.log("DSQL: Pattern not matched =>", newValue);
newValue = "";
}
}
if (typeof newValue === "string" && newValue.match(/^null$/i)) {
newValue = {
toSqlString: function () {
return "NULL";
},
};
}
if (typeof newValue === "string" && !newValue.match(/./i)) {
newValue = {
toSqlString: function () {
return "NULL";
},
};
}
return newValue;
}
package-shared/functions/backend/db/updateDbEntry.ts
+12 -61
View File
@@ -1,6 +1,3 @@
import sanitizeHtml from "sanitize-html";
import sanitizeHtmlOptions from "../html/sanitizeHtmlOptions";
import encrypt from "../../dsql/encrypt";
import checkIfIsMaster from "../../../utils/check-if-is-master";
import connDbHandler from "../../../utils/db/conn-db-handler";
import { DbContextsArray } from "./runQuery";
@@ -11,6 +8,7 @@ import {
} from "../../../types";
import _ from "lodash";
import purgeDefaultFields from "../../../utils/purge-default-fields";
import grabParsedValue from "./grab-parsed-value";
type Param<T extends { [k: string]: any } = any> = {
dbContext?: (typeof DbContextsArray)[number];
@@ -82,69 +80,22 @@ export default async function updateDbEntry<
const dataKey = dataKeys[i];
let value = newData[dataKey];
const targetFieldSchemaArray = tableSchema
? tableSchema?.fields?.filter(
(field) => field.fieldName === dataKey
)
: null;
const targetFieldSchema =
targetFieldSchemaArray && targetFieldSchemaArray[0]
? targetFieldSchemaArray[0]
: null;
const parsedValue = grabParsedValue({
dataKey,
encryptionKey,
encryptionSalt,
tableSchema,
value,
});
if (value == null || value == undefined) continue;
const htmlRegex = /<[^>]+>/g;
if (targetFieldSchema?.richText || String(value).match(htmlRegex)) {
value = sanitizeHtml(value, sanitizeHtmlOptions);
}
if (targetFieldSchema?.encrypted) {
value = encrypt({
data: value,
encryptionKey,
encryptionSalt,
});
}
if (typeof value === "object") {
value = JSON.stringify(value);
}
if (targetFieldSchema?.pattern) {
const pattern = new RegExp(
targetFieldSchema.pattern,
targetFieldSchema.patternFlags || ""
);
if (!pattern.test(value)) {
console.log("DSQL: Pattern not matched =>", value);
value = "";
}
}
if (typeof value === "string" && value.match(/^null$/i)) {
value = {
toSqlString: function () {
return "NULL";
},
};
}
if (typeof value === "string" && !value.match(/./i)) {
value = {
toSqlString: function () {
return "NULL";
},
};
}
if (typeof parsedValue == "undefined") continue;
updateKeyValueArray.push(`\`${dataKey}\`=?`);
if (typeof value == "number") {
updateValues.push(String(value));
if (typeof parsedValue == "number") {
updateValues.push(String(parsedValue));
} else {
updateValues.push(value);
updateValues.push(parsedValue);
}
////////////////////////////////////////
package-shared/functions/backend/handle-backup.ts
+13 -6
View File
@@ -15,7 +15,7 @@ export default async function handleBackup({
}: HandleBackupParams) {
const { appConfig } = grabConfig();
const maxBackups = appConfig.main.max_backups?.value || 20;
const maxBackups = appConfig.main.max_backups?.value || 4;
const { count: existingAppBackupsCount } =
await dbGrabUserResource<DSQL_DATASQUIREL_BACKUPS>({
@@ -33,7 +33,9 @@ export default async function handleBackup({
});
if (existingAppBackupsCount && existingAppBackupsCount >= maxBackups) {
const { single: oldestAppBackup } =
console.log(`Backups exceed Limit ...`);
const { batch: oldestAppBackups } =
await dbGrabUserResource<DSQL_DATASQUIREL_BACKUPS>({
tableName: "backups",
isSuperUser: true,
@@ -46,14 +48,19 @@ export default async function handleBackup({
},
order: {
field: "id",
strategy: "ASC",
strategy: "DESC",
},
limit: 1,
},
});
if (oldestAppBackup?.id) {
await deleteBackup({ backup: oldestAppBackup });
if (oldestAppBackups) {
for (let i = 0; i < oldestAppBackups.length; i++) {
const backup = oldestAppBackups[i];
console.log(`Handling Backup ${backup.uuid} ...`);
if (i < maxBackups - 1) continue;
console.log(`Deleting Backup ${backup.uuid} ...`);
await deleteBackup({ backup: backup });
}
}
}
package-shared/functions/backend/httpsRequest.ts
+10 -22
View File
@@ -7,6 +7,7 @@ type Param = {
url?: string;
method?: string;
hostname?: string;
host?: string;
path?: string;
port?: number | string;
headers?: object;
@@ -16,28 +17,24 @@ type Param = {
/**
* # Make Https Request
*/
export default function httpsRequest({
export default function httpsRequest<Res extends any = any>({
url,
method,
hostname,
host,
path,
headers,
body,
port,
scheme,
}: Param) {
}: Param): Promise<Res> {
const reqPayloadString = body ? JSON.stringify(body) : null;
const PARSED_URL = url ? new URL(url) : null;
////////////////////////////////////////////////
////////////////////////////////////////////////
////////////////////////////////////////////////
/** @type {any} */
let requestOptions: any = {
method: method || "GET",
hostname: PARSED_URL ? PARSED_URL.hostname : hostname,
hostname: PARSED_URL ? PARSED_URL.hostname : host || hostname,
port: scheme?.match(/https/i)
? 443
: PARSED_URL
@@ -51,7 +48,6 @@ export default function httpsRequest({
};
if (path) requestOptions.path = path;
// if (href) requestOptions.href = href;
if (headers) requestOptions.headers = headers;
if (body) {
@@ -61,10 +57,6 @@ export default function httpsRequest({
: undefined;
}
////////////////////////////////////////////////
////////////////////////////////////////////////
////////////////////////////////////////////////
return new Promise((res, rej) => {
const httpsRequest = (
scheme?.match(/https/i)
@@ -73,25 +65,21 @@ export default function httpsRequest({
? https
: http
).request(
/* ====== Request Options object ====== */
requestOptions,
////////////////////////////////////////////////
////////////////////////////////////////////////
////////////////////////////////////////////////
/* ====== Callback function ====== */
(response) => {
var str = "";
// ## another chunk of data has been received, so append it to `str`
response.on("data", function (chunk) {
str += chunk;
});
// ## the whole response has been received, so we just print it out here
response.on("end", function () {
res(str);
try {
res(JSON.parse(str));
} catch (error) {
res(str as any);
}
});
response.on("error", (error) => {
package-shared/types/dsql.ts
+19
View File
@@ -19,6 +19,7 @@ export const DsqlTables = [
"mariadb_users",
"mariadb_user_databases",
"mariadb_user_tables",
"user_private_media_keys",
] as const
export type DSQL_DATASQUIREL_USERS = {
@@ -392,4 +393,22 @@ export type DSQL_DATASQUIREL_MARIADB_USER_TABLES = {
date_updated?: string;
date_updated_code?: number;
date_updated_timestamp?: string;
}
export type DSQL_DATASQUIREL_USER_PRIVATE_MEDIA_KEYS = {
id?: number;
uuid?: string;
user_id?: number;
media_id?: number;
key?: string;
description?: string;
expiration?: number;
expiration_paradigm?: "seconds" | "minutes" | "hours" | "days" | "weeks" | "months" | "years";
expiration_milliseconds?: number;
date_created?: string;
date_created_code?: number;
date_created_timestamp?: string;
date_updated?: string;
date_updated_code?: number;
date_updated_timestamp?: string;
}
package-shared/types/index.ts
+171 -9
View File
@@ -429,6 +429,8 @@ export interface PostInsertReturn {
export type UserType = DATASQUIREL_LoggedInUser & {
isSuperUser?: boolean;
staticHost?: string;
appHost?: string;
appName?: string;
};
export interface ApiKeyDef {
@@ -1596,6 +1598,9 @@ export type PagePropsType = {
appVersion?: (typeof AppVersions)[number];
isMailAvailable?: boolean;
websocketURL?: string;
docsObject?: DocsServerProps | null;
docsPages?: DocsLinkType[] | null;
docsPageEditURL?: string | null;
};
export type APIResponseObject<T extends any = any> = {
@@ -1649,6 +1654,8 @@ export const WebSocketEvents = [
"client:dev:queue",
"client:delete-queue",
"client:pty-shell",
"client:su-logs",
"client:su-kill-logs",
/**
* # Server Events
@@ -1663,12 +1670,16 @@ export const WebSocketEvents = [
"server:dev:queue",
"server:queue-deleted",
"server:pty-shell",
"server:su-logs",
"server:su-kill-logs",
] as const;
export type WebSocketDataType = {
event: (typeof WebSocketEvents)[number];
data?: {
queue?: DSQL_DATASQUIREL_PROCESS_QUEUE;
containerName?: string;
killLogs?: boolean;
};
error?: string;
message?: string;
@@ -1680,15 +1691,6 @@ export const DatasquirelWindowEvents = [
"queue-running",
] as const;
export type DatasquirelWindowEventPayloadType = {
event: (typeof DatasquirelWindowEvents)[number];
data?: {
queue?: DSQL_DATASQUIREL_PROCESS_QUEUE;
};
error?: string;
message?: string;
};
/**
* # Docker Compose Types
*/
@@ -1798,6 +1800,7 @@ export type DsqlAppData = {
DSQL_FACEBOOK_APP_ID?: string | null;
DSQL_GITHUB_ID?: string | null;
DSQL_HOST_MACHINE_IP?: string | null;
DSQL_DEPLOYMENT_NAME?: string | null;
};
export const MediaTypes = ["image", "file", "video"] as const;
@@ -2008,6 +2011,7 @@ export type DefaultLocalResourcesHookParams<
> = {
refresh?: number;
setLoading?: React.Dispatch<React.SetStateAction<boolean>>;
setReady?: React.Dispatch<React.SetStateAction<boolean>>;
loadingEndTimeout?: number;
user?: UserType | null;
ready?: boolean;
@@ -2422,6 +2426,10 @@ export type SendEmailCodeParams = {
useLocal?: boolean;
apiVersion?: string;
dbUserId?: string | number;
/**
* HTML string with {{code}} placeholder for the code
*/
html?: string;
};
export type UpdateUserParams<
@@ -2479,4 +2487,158 @@ export type GoogleAuthParams = {
*/
useLocal?: boolean;
apiVersion?: string;
skipWriteAuthFile?: boolean;
cleanupTokens?: boolean;
};
export type ContactFormType = {
first_name?: string;
last_name?: string;
email?: string;
phone?: string;
message?: string;
};
export const TimeParadigms = [
{
value: "seconds",
label: "Seconds",
},
{
value: "minutes",
label: "Minutes",
},
{
value: "hours",
label: "Hours",
},
{
value: "days",
label: "Days",
},
{
value: "weeks",
label: "Weeks",
},
{
value: "months",
label: "Months",
},
{
value: "years",
label: "Years",
},
] as const;
export type GithubPublicAPIResJSON = {
name: string;
path: string;
sha: string;
size: number;
url: string;
html_url: string;
git_url: string;
download_url: string;
type: string;
content: string;
encoding: string;
_links: {
self: string;
git: string;
html: string;
};
};
export type DocsServerProps = {
md?: string | null;
mdx_source?: any | null;
meta_title?: string | null;
meta_description?: string | null;
page_title?: string | null;
page_description?: string | null;
page_path?: string | null;
};
export type DocsLinkType = {
title: string;
href: string;
strict?: boolean;
children?: DocsLinkType[];
editPage?: string;
};
export interface GiteaBranchRes {
name: string;
commit: GiteaBranchResCommit;
protected: boolean;
required_approvals: number;
enable_status_check: boolean;
status_check_contexts: any[];
user_can_push: boolean;
user_can_merge: boolean;
effective_branch_protection_name: string;
}
export interface GiteaBranchResCommit {
id: string;
message: string;
url: string;
author: GiteaBranchResCommitAuthor;
committer: GiteaBranchResCommitCommitter;
verification: GiteaBranchResCommitVerification;
timestamp: string;
added: any;
removed: any;
modified: any;
}
export interface GiteaBranchResCommitAuthor {
name: string;
email: string;
username: string;
}
export interface GiteaBranchResCommitCommitter {
name: string;
email: string;
username: string;
}
export interface GiteaBranchResCommitVerification {
verified: boolean;
reason: string;
signature: string;
signer: any;
payload: string;
}
export interface GiteaTreeRes {
sha: string;
url: string;
tree: GiteaTreeResTree[];
truncated: boolean;
page: number;
total_count: number;
}
export interface GiteaTreeResTree {
path: string;
mode: string;
type: "tree" | "blob";
size: number;
sha: string;
url: string;
}
export const OpsActions = [
"exit",
"test",
"restart-web-app",
"restart-db",
"restart-all",
"clear",
] as const;
export type OpsObject = {
action: (typeof OpsActions)[number];
};
package-shared/utils/backend/names/grab-dir-names.ts
+6
View File
@@ -269,6 +269,10 @@ export default function grabDirNames(param?: Param) {
distroDirName
);
const tempBackupExportName = "tmp-export-backup.tar.xz";
const opsJSONFileName = "ops.json";
return {
appDir,
privateDataDir,
@@ -362,5 +366,7 @@ export default function grabDirNames(param?: Param) {
dsqlDbDockerComposeFileAlt,
dsqlDbDockerComposeFileName,
dsqlDbDockerComposeFileNameAlt,
tempBackupExportName,
opsJSONFileName,
};
}
package-shared/utils/backend/names/grab-docker-stack-services-names.ts
+7 -2
View File
@@ -1,5 +1,10 @@
export default function grabDockerStackServicesNames() {
const deploymentName = process.env.DSQL_DEPLOYMENT_NAME || "dsql";
type Params = {
deploymentName?: string | null;
};
export default function grabDockerStackServicesNames(params?: Params) {
const deploymentName =
params?.deploymentName || process.env.DSQL_DEPLOYMENT_NAME || "dsql";
const maxScaleServiceName = `${deploymentName}-dsql-maxscale`;
const dbServiceName = `${deploymentName}-dsql-db`;
package-shared/utils/backend/names/grab-ip-addresses.ts
+2
View File
@@ -21,6 +21,7 @@ export default function grabIPAddresses() {
const mainDBIP = `${globalIPPrefix}.${db}`;
const webSocketIP = `${globalIPPrefix}.${websocket}`;
const dbCronIP = `${globalIPPrefix}.${db_cron}`;
const reverseProxyIP = `${globalIPPrefix}.${reverse_proxy}`;
const localHostIP = `${globalIPPrefix}.1`;
return {
@@ -32,5 +33,6 @@ export default function grabIPAddresses() {
globalIPPrefix,
webSocketIP,
dbCronIP,
reverseProxyIP,
};
}
package-shared/utils/generate-random-string.ts
+16
View File
@@ -0,0 +1,16 @@
export default function genRndStr(length?: number, symbols?: boolean) {
let characters =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
if (symbols) characters += "-_[]()@";
let result = "";
const finalLength = length || 12;
for (let i = 0; i < finalLength; i++) {
const randomIndex = Math.floor(Math.random() * characters.length);
result += characters.charAt(randomIndex);
}
return result;
}
package-shared/utils/grab-machine-ip.ts
+21
View File
@@ -0,0 +1,21 @@
import { networkInterfaces } from "os";
export default function getMachineIPAddress() {
try {
const interfaces = networkInterfaces();
for (const ifaceName in interfaces) {
const iface = interfaces[ifaceName];
if (Array.isArray(iface)) {
for (const address of iface) {
if (address.family === "IPv4" && !address.internal) {
return address.address;
}
}
}
}
return null;
} catch (error: any) {
console.error(`Error accessing network interfaces: ${error.message}`);
return null;
}
}
package-shared/utils/grab-user-main-sql-user-name.ts
+1 -1
View File
@@ -18,7 +18,7 @@ export default function grabUserMainSqlUserName({
const finalUsername = username || sqlUsername;
const finalHost = HOST || maxScaleIP || "127.0.0.1";
const fullName = `${finalUsername}@${webAppIP}`;
const fullName = `${finalUsername}@${finalHost}`;
return {
username: finalUsername,
package-shared/utils/setup-db.ts
-62
View File
@@ -1,62 +0,0 @@
import grabDbSSL from "./backend/grabDbSSL";
import mariadb, { ConnectionConfig } from "mariadb";
type Params = {
useLocal?: boolean;
dbConfig?: ConnectionConfig;
ssl?: boolean;
connectionLimit?: number;
};
export default async function setupDSQLDb({ dbConfig, ssl }: Params) {
const conn = await mariadb.createConnection({
host: process.env.DSQL_DB_HOST,
user: process.env.DSQL_DB_USERNAME,
password: process.env.DSQL_DB_PASSWORD,
database: process.env.DSQL_DB_NAME,
charset: "utf8mb4",
...dbConfig,
ssl: ssl ? grabDbSSL() : undefined,
supportBigNumbers: true,
bigNumberStrings: false,
dateStrings: true,
bigIntAsNumber: true,
metaAsArray: true,
});
// const conn = mariadb.createPool({
// host: process.env.DSQL_DB_HOST,
// user: process.env.DSQL_DB_USERNAME,
// password: process.env.DSQL_DB_PASSWORD,
// database: process.env.DSQL_DB_NAME,
// charset: "utf8mb4",
// ...dbConfig,
// ssl: ssl ? grabDbSSL() : undefined,
// connectionLimit,
// supportBigNumbers: true,
// bigNumberStrings: false,
// dateStrings: true,
// });
// let readOnlyConnection;
// if (addReadOnlyConn) {
// readOnlyConnection = mariadb.createPool({
// host: process.env.DSQL_DB_HOST,
// user: process.env.DSQL_DB_READ_ONLY_USERNAME,
// password: process.env.DSQL_DB_READ_ONLY_PASSWORD,
// database: process.env.DSQL_DB_NAME,
// charset: "utf8mb4",
// ...readOnlyDbConfig,
// ssl: ssl ? grabDbSSL() : undefined,
// connectionLimit,
// });
// global.DSQL_READ_ONLY_DB_CONN = readOnlyConnection;
// }
return {
conn,
// readOnlyConnection,
};
}