Moduletrace/datasquirel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updates

Browse Source
This commit is contained in:
Benjamin Toby
2025-07-06 15:32:28 +01:00
parent 3597b11342
commit b38ddc9f21
50 changed files with 453 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-shared/functions/backend/dbHandler.ts
-1
View File
@@ -39,7 +39,6 @@ export default async function dbHandler({
if (!noErrorLogs) {
console.log("ERROR in dbHandler =>", error.message);
console.log(error);
console.log(CONNECTION.config());
const tmpFolder = path.resolve(process.cwd(), "./.tmp");
package-shared/functions/backend/su-db-handler.ts
+9 -9
View File
@@ -1,6 +1,7 @@
import mysql from "serverless-mysql";
import { UserType } from "../../types";
import grabDbSSL from "../../utils/backend/grabDbSSL";
import connDbHandler from "../../utils/db/conn-db-handler";
import mariadb from "mariadb";
type Params = {
query?: string;
@@ -15,14 +16,13 @@ export default async function suDbHandler({
user,
values,
}: Params) {
const connection = mysql({
config: {
host: process.env.DSQL_DB_HOST,
user: process.env.DSQL_DB_USERNAME,
password: process.env.DSQL_DB_PASSWORD,
database: database,
charset: "utf8mb4",
},
const connection = mariadb.createPool({
host: process.env.DSQL_DB_HOST,
user: process.env.DSQL_DB_USERNAME,
password: process.env.DSQL_DB_PASSWORD,
database: database,
charset: "utf8mb4",
ssl: grabDbSSL(),
});
const results = await connDbHandler(connection, query);
package-shared/functions/backend/user-db-handler.ts
+9 -9
View File
@@ -1,7 +1,8 @@
import mysql from "serverless-mysql";
import mariadb from "mariadb";
import { DSQL_TableSchemaType, UserType } from "../../types";
import grabMariadbMainUserForUser from "./grab-mariadb-main-user-for-user";
import connDbHandler from "../../utils/db/conn-db-handler";
import grabDbSSL from "../../utils/backend/grabDbSSL";
type Params = {
query?: string;
@@ -23,14 +24,13 @@ export default async function userDbHandler({
const { fullName, host, username, password } =
await grabMariadbMainUserForUser({ user });
const connection = mysql({
config: {
host,
user: username,
password: password,
database: database,
charset: "utf8mb4",
},
const connection = mariadb.createPool({
host,
user: username,
password: password,
database: database,
charset: "utf8mb4",
ssl: grabDbSSL(),
});
const results = await connDbHandler(connection, query);
package-shared/functions/backend/varDatabaseDbHandler.ts
-1
View File
@@ -29,7 +29,6 @@ export default async function varDatabaseDbHandler({
if (debug) {
console.log(`varDatabaseDbHandler:query:`, queryString);
console.log(`varDatabaseDbHandler:values:`, queryValuesArray);
console.log(`varDatabaseDbHandler:CONNECTION:`, CONNECTION.getConfig());
}
let results = await connDbHandler(
package-shared/shell/checkDb.ts
-1
View File
@@ -1,5 +1,4 @@
require("dotenv").config({ path: "./../.env" });
import mysql from "serverless-mysql";
import grabDSQLConnection from "../utils/grab-dsql-connection";
/**
package-shared/shell/updateSSLUsers.ts
-1
View File
@@ -1,5 +1,4 @@
require("dotenv").config({ path: "./../.env" });
import mysql from "serverless-mysql";
import grabDSQLConnection from "../utils/grab-dsql-connection";
import grabSQLKeyName from "../utils/grab-sql-key-name";
package-shared/types/index.ts
+1
View File
@@ -2135,6 +2135,7 @@ export type SiteConfigMariadbServer = {
root: SiteConfigMariadbServerUser;
replication: SiteConfigMariadbServerUser;
};
ssl_dir?: string;
};
export type SiteConfigMariadbServerUser = {
package-shared/utils/backend/global-db/DB_HANDLER.ts
+2 -2
View File
@@ -4,14 +4,14 @@ import grabDSQLConnection from "../../grab-dsql-connection";
* # DSQL user read-only DB handler
* @requires DSQL_DB_CONN - Gobal Variable for Datasquirel Database
*/
export default async function DB_HANDLER(...args: any[]) {
export default async function DB_HANDLER(query: string, values?: any[]) {
const CONNECTION = grabDSQLConnection();
try {
if (!CONNECTION)
throw new Error("No Connection provided to DB_HANDLER function!");
const results = await CONNECTION.query(...args);
const results = await CONNECTION.query(query, values);
return JSON.parse(JSON.stringify(results));
} catch (error: any) {
package-shared/utils/backend/global-db/LOCAL_DB_HANDLER.ts
+2 -2
View File
@@ -3,11 +3,11 @@ import grabDSQLConnection from "../../grab-dsql-connection";
/**
* # DSQL user read-only DB handler
*/
export default async function LOCAL_DB_HANDLER(...args: any[]) {
export default async function LOCAL_DB_HANDLER(query: string, values?: any[]) {
const MASTER = grabDSQLConnection();
try {
const results = await MASTER.query(...args);
const results = await MASTER.query(query, values);
return JSON.parse(JSON.stringify(results));
} catch (error: any) {
package-shared/utils/backend/global-db/NO_DB_HANDLER.ts
+2 -4
View File
@@ -1,16 +1,14 @@
import mysql from "serverless-mysql";
import grabDbSSL from "../grabDbSSL";
import grabDSQLConnection from "../../grab-dsql-connection";
/**
* # DSQL user read-only DB handler
*/
export default function NO_DB_HANDLER(...args: any[]) {
export default function NO_DB_HANDLER(query: string, values?: any[]) {
const CONNECTION = grabDSQLConnection();
try {
return new Promise((resolve, reject) => {
CONNECTION.query(...args)
CONNECTION.query(query, values)
.then((results) => {
CONNECTION.end();
resolve(JSON.parse(JSON.stringify(results)));
package-shared/utils/backend/global-db/ROOT_DB_HANDLER.ts
+2 -2
View File
@@ -3,12 +3,12 @@ import grabDSQLConnection from "../../grab-dsql-connection";
/**
* # Root DB handler
*/
export default function ROOT_DB_HANDLER(...args: any[]) {
export default function ROOT_DB_HANDLER(query: string, values?: any[]) {
const CONNECTION = grabDSQLConnection();
try {
return new Promise((resolve, reject) => {
CONNECTION.query(...args)
CONNECTION.query(query, values)
.then((results) => {
CONNECTION.end();
resolve(JSON.parse(JSON.stringify(results)));
package-shared/utils/backend/grabDbSSL.ts
+8 -13
View File
@@ -1,22 +1,19 @@
import fs from "fs";
import grabDirNames from "./names/grab-dir-names";
import type { ConnectionConfig } from "mariadb";
type Return =
| string
| (import("tls").SecureContextOptions & {
rejectUnauthorized?: boolean | undefined;
})
| undefined;
type Return = ConnectionConfig["ssl"] | undefined;
/**
* # Grall SSL
* # Grab SSL
*/
export default function grabDbSSL(): Return {
const SSL_DIR = process.env.DSQL_SSL_DIR;
if (!SSL_DIR?.match(/./)) {
const { maxscaleSSLDir } = grabDirNames();
if (!maxscaleSSLDir?.match(/./)) {
return undefined;
}
const caFilePath = `${SSL_DIR}/ca-cert.pem`;
const caFilePath = `${maxscaleSSLDir}/ca-cert.pem`;
if (!fs.existsSync(caFilePath)) {
console.log(`${caFilePath} does not exist`);
@@ -24,9 +21,7 @@ export default function grabDbSSL(): Return {
}
return {
ca: fs.readFileSync(`${SSL_DIR}/ca-cert.pem`),
// key: fs.readFileSync(`${SSL_DIR}/client-key.pem`),
// cert: fs.readFileSync(`${SSL_DIR}/client-cert.pem`),
ca: fs.readFileSync(`${maxscaleSSLDir}/ca-cert.pem`),
rejectUnauthorized: false,
};
}
package-shared/utils/backend/names/grab-dir-names.ts
+9
View File
@@ -27,6 +27,11 @@ export default function grabDirNames(param?: Param) {
const appSSLDir = path.join(appDir, "ssl");
const mainSSLDir = path.join(DATA_DIR, "ssl");
const maxscaleSSLDir = path.join(mainSSLDir, "maxscale");
const mainDBSSLDir = path.join(mainSSLDir, "main");
const replica1DBSSLDir = path.join(mainSSLDir, "replica-1");
const replica2DBSSLDir = path.join(mainSSLDir, "replica-2");
const privateDataDir = path.join(DATA_DIR, "private");
/**
@@ -228,5 +233,9 @@ export default function grabDirNames(param?: Param) {
sqlBackupDirName,
schemasBackupDirName,
userMainShemaJSONFilePath,
maxscaleSSLDir,
mainDBSSLDir,
replica1DBSSLDir,
replica2DBSSLDir,
};
}
package-shared/utils/backend/names/grab-docker-stack-services-names.ts
+26
View File
@@ -0,0 +1,26 @@
export default function grabDockerStackServicesNames() {
const deploymentName = process.env.DSQL_DEPLOYMENT_NAME || "dsql";
const maxScaleServiceName = `${deploymentName}-dsql-maxscale`;
const dbServiceName = `${deploymentName}-dsql-db`;
const dbCronServiceName = `${deploymentName}-dsql-db-cron`;
const postDbSetupServiceName = `${deploymentName}-dsql-post-db-setup`;
const webAppServiceName = `${deploymentName}-dsql-web-app`;
const webAppCronServiceName = `${deploymentName}-dsql-web-app-cron`;
const webAppPostDbSetupServiceName = `${deploymentName}-dsql-web-app-post-db-setup`;
const dbReplica1ServiceName = `${deploymentName}-dsql-db-replica-1`;
const dbReplica2ServiceName = `${deploymentName}-dsql-db-replica-2`;
return {
deploymentName,
maxScaleServiceName,
dbServiceName,
dbCronServiceName,
postDbSetupServiceName,
webAppServiceName,
webAppCronServiceName,
webAppPostDbSetupServiceName,
dbReplica1ServiceName,
dbReplica2ServiceName,
};
}
package-shared/utils/backend/names/grab-ip-addresses.ts
+10 -1
View File
@@ -8,6 +8,15 @@ export default function grabIPAddresses() {
const webAppIP = `${globalIPPrefix}.${web}`;
const appCronIP = `${globalIPPrefix}.${cron}`;
const maxScaleIP = `${globalIPPrefix}.${maxscale}`;
const mainDBIP = `${globalIPPrefix}.${db}`;
const localHostIP = `${globalIPPrefix}.1`;
return { webAppIP, appCronIP, maxScaleIP, globalIPPrefix };
return {
webAppIP,
appCronIP,
maxScaleIP,
mainDBIP,
localHostIP,
globalIPPrefix,
};
}
package-shared/utils/db/conn-db-handler.ts
+22 -12
View File
@@ -1,6 +1,6 @@
import { ServerlessMysql } from "serverless-mysql";
import debugLog from "../logging/debug-log";
import { DSQLErrorObject } from "../../types";
import type { ConnectionConfig, Pool } from "mariadb";
export type ConnDBHandlerQueryObject = {
query: string;
@@ -9,8 +9,9 @@ export type ConnDBHandlerQueryObject = {
type Return<ReturnType = any> =
| ReturnType
| ReturnType[]
| null
| { error?: string; errors?: DSQLErrorObject[] };
| { error?: string; errors?: DSQLErrorObject[]; config?: ConnectionConfig };
/**
* # Run Query From MySQL Connection
@@ -19,9 +20,9 @@ type Return<ReturnType = any> =
*/
export default async function connDbHandler<ReturnType = any>(
/**
* ServerlessMySQL Connection Object
* MariaDB Connection Pool Object
*/
conn?: ServerlessMysql,
connPool?: Pool,
/**
* String Or `ConnDBHandlerQueryObject` Array
*/
@@ -33,13 +34,13 @@ export default async function connDbHandler<ReturnType = any>(
debug?: boolean
): Promise<Return<ReturnType>> {
try {
if (!conn) throw new Error("No Connection Found!");
if (!connPool) throw new Error("No Connection Found!");
if (!query) throw new Error("Query String Required!");
let queryErrorArray: DSQLErrorObject[] = [];
if (typeof query == "string") {
const res = await conn.query(trimQuery(query), values);
const res = await connPool.query(trimQuery(query), values);
if (debug) {
debugLog({
@@ -49,7 +50,11 @@ export default async function connDbHandler<ReturnType = any>(
});
}
return JSON.parse(JSON.stringify(res));
if (Array.isArray(res)) {
return Array.from(res);
}
return res;
} else if (typeof query == "object") {
const resArray = [];
@@ -62,7 +67,7 @@ export default async function connDbHandler<ReturnType = any>(
currentQueryError.sql = queryObj.query;
currentQueryError.sqlValues = queryObj.values;
const queryObjRes = await conn.query(
const queryObjRes = await connPool.query(
trimQuery(queryObj.query),
queryObj.values
);
@@ -75,7 +80,11 @@ export default async function connDbHandler<ReturnType = any>(
});
}
resArray.push(JSON.parse(JSON.stringify(queryObjRes)));
if (Array.isArray(queryObjRes)) {
resArray.push(Array.from(queryObjRes));
} else {
resArray.push(queryObjRes);
}
} catch (error: any) {
global.ERROR_CALLBACK?.(
`Connection DB Handler Query Error`,
@@ -101,7 +110,7 @@ export default async function connDbHandler<ReturnType = any>(
};
}
return resArray as any;
return resArray;
} else {
return null;
}
@@ -118,12 +127,13 @@ export default async function connDbHandler<ReturnType = any>(
return {
error: `Connection DB Handler Error: ${error.message}`,
// config: conn,
};
} finally {
conn?.end();
connPool?.end();
}
}
function trimQuery(query: string) {
return query.replace(/\n/gm, "").replace(/ {2,}/g, "").trim();
return query.replace(/\n/gm, " ").replace(/ {2,}/g, " ").trim();
}
package-shared/utils/grab-dsql-connection.ts
+53 -58
View File
@@ -1,4 +1,5 @@
import mysql, { ServerlessMysql } from "serverless-mysql";
import mariadb, { Pool } from "mariadb";
import grabDbSSL from "./backend/grabDbSSL";
type Param = {
/**
@@ -22,65 +23,11 @@ type Param = {
/**
* # Grab General CONNECTION for DSQL
*/
export default function grabDSQLConnection(param?: Param): ServerlessMysql {
export default function grabDSQLConnection(param?: Param): Pool {
if (global.DSQL_USE_LOCAL || param?.local) {
return (
global.DSQL_DB_CONN ||
mysql({
config: {
host: process.env.DSQL_DB_HOST,
user: process.env.DSQL_DB_USERNAME,
password: process.env.DSQL_DB_PASSWORD,
database: param?.noDb
? undefined
: process.env.DSQL_DB_NAME,
port: process.env.DSQL_DB_PORT
? Number(process.env.DSQL_DB_PORT)
: undefined,
charset: "utf8mb4",
},
})
);
}
if (param?.ro) {
return (
global.DSQL_READ_ONLY_DB_CONN ||
mysql({
config: {
host: process.env.DSQL_DB_HOST,
user: process.env.DSQL_DB_READ_ONLY_USERNAME,
password: process.env.DSQL_DB_READ_ONLY_PASSWORD,
port: process.env.DSQL_DB_PORT
? Number(process.env.DSQL_DB_PORT)
: undefined,
charset: "utf8mb4",
},
})
);
}
if (param?.fa) {
return (
global.DSQL_FULL_ACCESS_DB_CONN ||
mysql({
config: {
host: process.env.DSQL_DB_HOST,
user: process.env.DSQL_DB_FULL_ACCESS_USERNAME,
password: process.env.DSQL_DB_FULL_ACCESS_PASSWORD,
port: process.env.DSQL_DB_PORT
? Number(process.env.DSQL_DB_PORT)
: undefined,
charset: "utf8mb4",
},
})
);
}
return (
global.DSQL_DB_CONN ||
mysql({
config: {
mariadb.createPool({
host: process.env.DSQL_DB_HOST,
user: process.env.DSQL_DB_USERNAME,
password: process.env.DSQL_DB_PASSWORD,
@@ -89,7 +36,55 @@ export default function grabDSQLConnection(param?: Param): ServerlessMysql {
? Number(process.env.DSQL_DB_PORT)
: undefined,
charset: "utf8mb4",
},
ssl: grabDbSSL(),
})
);
}
if (param?.ro) {
return (
global.DSQL_READ_ONLY_DB_CONN ||
mariadb.createPool({
host: process.env.DSQL_DB_HOST,
user: process.env.DSQL_DB_READ_ONLY_USERNAME,
password: process.env.DSQL_DB_READ_ONLY_PASSWORD,
port: process.env.DSQL_DB_PORT
? Number(process.env.DSQL_DB_PORT)
: undefined,
charset: "utf8mb4",
ssl: grabDbSSL(),
})
);
}
if (param?.fa) {
return (
global.DSQL_FULL_ACCESS_DB_CONN ||
mariadb.createPool({
host: process.env.DSQL_DB_HOST,
user: process.env.DSQL_DB_FULL_ACCESS_USERNAME,
password: process.env.DSQL_DB_FULL_ACCESS_PASSWORD,
port: process.env.DSQL_DB_PORT
? Number(process.env.DSQL_DB_PORT)
: undefined,
charset: "utf8mb4",
ssl: grabDbSSL(),
})
);
}
return (
global.DSQL_DB_CONN ||
mariadb.createPool({
host: process.env.DSQL_DB_HOST,
user: process.env.DSQL_DB_USERNAME,
password: process.env.DSQL_DB_PASSWORD,
database: param?.noDb ? undefined : process.env.DSQL_DB_NAME,
port: process.env.DSQL_DB_PORT
? Number(process.env.DSQL_DB_PORT)
: undefined,
charset: "utf8mb4",
ssl: grabDbSSL(),
})
);
}
package-shared/utils/setup-db.ts
+53
View File
@@ -0,0 +1,53 @@
import grabDbSSL from "./backend/grabDbSSL";
import mariadb, { ConnectionConfig } from "mariadb";
type Params = {
useLocal?: boolean;
dbConfig?: ConnectionConfig;
ssl?: boolean;
connectionLimit?: number;
};
export default function setupDSQLDb({
useLocal,
dbConfig,
ssl,
connectionLimit = 100,
}: Params) {
global.DSQL_USE_LOCAL = useLocal || true;
const pool = mariadb.createPool({
host: process.env.DSQL_DB_HOST,
user: process.env.DSQL_DB_USERNAME,
password: process.env.DSQL_DB_PASSWORD,
database: process.env.DSQL_DB_NAME,
charset: "utf8mb4",
...dbConfig,
ssl: ssl ? grabDbSSL() : undefined,
connectionLimit,
});
global.DSQL_DB_CONN = pool;
// let readOnlyConnection;
// if (addReadOnlyConn) {
// readOnlyConnection = mariadb.createPool({
// host: process.env.DSQL_DB_HOST,
// user: process.env.DSQL_DB_READ_ONLY_USERNAME,
// password: process.env.DSQL_DB_READ_ONLY_PASSWORD,
// database: process.env.DSQL_DB_NAME,
// charset: "utf8mb4",
// ...readOnlyDbConfig,
// ssl: ssl ? grabDbSSL() : undefined,
// connectionLimit,
// });
// global.DSQL_READ_ONLY_DB_CONN = readOnlyConnection;
// }
return {
pool,
// readOnlyConnection,
};
}