Moduletrace/datasquirel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updates

Browse Source
This commit is contained in:
Benjamin Toby
2025-07-06 17:00:35 +01:00
parent b38ddc9f21
commit 6acf0f2cd6
48 changed files with 353 additions and 622 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-shared/functions/backend/dbHandler.ts
+6 -2
View File
@@ -17,7 +17,7 @@ export default async function dbHandler({
values,
noErrorLogs,
}: Param): Promise<any[] | object | null> {
const CONNECTION = grabDSQLConnection();
const CONNECTION = await grabDSQLConnection();
let results;
@@ -58,7 +58,11 @@ export default async function dbHandler({
}
if (results) {
return JSON.parse(JSON.stringify(results));
if (Array.isArray(results)) {
return Array.from(results);
} else {
return results;
}
} else {
return null;
}
package-shared/functions/backend/fullAccessDbHandler.ts
-1
View File
@@ -53,7 +53,6 @@ export default async function fullAccessDbHandler({
*/
return error.message;
} finally {
DB_CONN?.end();
}
/**
package-shared/functions/backend/noDatabaseDbHandler.ts
-56
View File
@@ -1,56 +0,0 @@
import fs from "fs";
import serverError from "./serverError";
import NO_DB_HANDLER from "../../utils/backend/global-db/NO_DB_HANDLER";
/**
* # No Database DB Handler
*/
export default async function noDatabaseDbHandler(
queryString: string
): Promise<any> {
process.env.NODE_ENV?.match(/dev/) &&
fs.appendFileSync(
"./.tmp/sqlQuery.sql",
queryString + "\n" + Date() + "\n\n\n",
"utf8"
);
/**
* Declare variables
*
* @description Declare "results" variable
*/
let results;
/**
* Fetch from db
*
* @description Fetch data from db if no cache
*/
try {
/** ********************* Run Query */
results = await NO_DB_HANDLER(queryString);
////////////////////////////////////////
////////////////////////////////////////
////////////////////////////////////////
} catch (/** @type {any} */ error: any) {
serverError({
component: "noDatabaseDbHandler",
message: error.message,
});
console.log("ERROR in noDatabaseDbHandler =>", error.message);
}
/**
* Return results
*
* @description Return results add to cache if "req" param is passed
*/
if (results) {
return results;
} else {
return null;
}
}
package-shared/functions/backend/su-db-handler.ts
+16 -1
View File
@@ -16,15 +16,30 @@ export default async function suDbHandler({
user,
values,
}: Params) {
const connection = mariadb.createPool({
const connection = await mariadb.createConnection({
host: process.env.DSQL_DB_HOST,
user: process.env.DSQL_DB_USERNAME,
password: process.env.DSQL_DB_PASSWORD,
database: database,
charset: "utf8mb4",
ssl: grabDbSSL(),
supportBigNumbers: true,
bigNumberStrings: false,
dateStrings: true,
});
// const connection = mariadb.createPool({
// host: process.env.DSQL_DB_HOST,
// user: process.env.DSQL_DB_USERNAME,
// password: process.env.DSQL_DB_PASSWORD,
// database: database,
// charset: "utf8mb4",
// ssl: grabDbSSL(),
// supportBigNumbers: true,
// bigNumberStrings: false,
// dateStrings: true,
// });
const results = await connDbHandler(connection, query);
return results;
package-shared/functions/backend/user-db-handler.ts
+4 -1
View File
@@ -24,13 +24,16 @@ export default async function userDbHandler({
const { fullName, host, username, password } =
await grabMariadbMainUserForUser({ user });
const connection = mariadb.createPool({
const connection = await mariadb.createConnection({
host,
user: username,
password: password,
database: database,
charset: "utf8mb4",
ssl: grabDbSSL(),
supportBigNumbers: true,
bigNumberStrings: false,
dateStrings: true,
});
const results = await connDbHandler(connection, query);
package-shared/functions/backend/varDatabaseDbHandler.ts
+5 -3
View File
@@ -22,9 +22,11 @@ export default async function varDatabaseDbHandler({
tableSchema,
debug,
}: Param): Promise<any> {
let CONNECTION = grabDSQLConnection({ fa: true });
if (global.DSQL_USE_LOCAL) CONNECTION = grabDSQLConnection({ local: true });
if (database?.match(/^datasquirel$/)) CONNECTION = grabDSQLConnection();
let CONNECTION = await grabDSQLConnection({ fa: true });
if (global.DSQL_USE_LOCAL)
CONNECTION = await grabDSQLConnection({ local: true });
if (database?.match(/^datasquirel$/))
CONNECTION = await grabDSQLConnection();
if (debug) {
console.log(`varDatabaseDbHandler:query:`, queryString);
package-shared/functions/backend/varReadOnlyDatabaseDbHandler.ts
-1
View File
@@ -38,7 +38,6 @@ export default async function varReadOnlyDatabaseDbHandler({
return error.message;
} finally {
DB_CONN?.end();
}
if (results) {
package-shared/functions/dsql/sync-databases/index.ts
-117
View File
@@ -1,117 +0,0 @@
import mysql, { Connection } from "mysql";
import { exec } from "child_process";
import { promisify } from "util";
// Configuration interface
interface DatabaseConfig {
host: string;
user: string;
password: string;
database?: string; // Optional for global connection
}
// Master status interface
interface MasterStatus {
File: string;
Position: number;
Binlog_Do_DB?: string;
Binlog_Ignore_DB?: string;
}
function getConnection(config: DatabaseConfig): Connection {
return mysql.createConnection(config);
}
function getMasterStatus(config: DatabaseConfig): Promise<MasterStatus> {
return new Promise((resolve, reject) => {
const connection = getConnection(config);
connection.query("SHOW MASTER STATUS", (error, results) => {
connection.end();
if (error) reject(error);
else resolve(results[0] as MasterStatus);
});
});
}
async function syncDatabases() {
const config: DatabaseConfig = {
host: "localhost",
user: "root",
password: "your_password",
};
let lastPosition: number | null = null; // Track last synced position
while (true) {
try {
// Get current master status
const { File, Position } = await getMasterStatus(config);
// Determine start position (use lastPosition or 4 if first run)
const startPosition = lastPosition !== null ? lastPosition + 1 : 4;
if (startPosition >= Position) {
await new Promise((resolve) => setTimeout(resolve, 5000)); // Wait 5 seconds if no new changes
continue;
}
// Execute mysqlbinlog to get changes
const execPromise = promisify(exec);
const { stdout } = await execPromise(
`mysqlbinlog --database=db_master ${File} --start-position=${startPosition} --stop-position=${Position}`
);
if (stdout) {
const connection = getConnection({
...config,
database: "db_slave",
});
return new Promise((resolve, reject) => {
connection.query(stdout, (error) => {
connection.end();
if (error) reject(error);
else {
lastPosition = Position;
console.log(
`Synced up to position ${Position} at ${new Date().toISOString()}`
);
resolve(null);
}
});
});
}
} catch (error) {
console.error("Sync error:", error);
}
await new Promise((resolve) => setTimeout(resolve, 5000)); // Check every 5 seconds
}
}
// Initialize db_slave with db_master data
async function initializeSlave() {
const config: DatabaseConfig = {
host: "localhost",
user: "root",
password: "your_password",
};
try {
await promisify(exec)(
`mysqldump -u ${config.user} -p${config.password} db_master > db_master_backup.sql`
);
await promisify(exec)(
`mysql -u ${config.user} -p${config.password} db_slave < db_master_backup.sql`
);
console.log("Slave initialized with master data");
} catch (error) {
console.error("Initialization error:", error);
}
}
// Run the sync process
async function main() {
await initializeSlave();
await syncDatabases();
}
main().catch(console.error);
package-shared/shell/checkDb.ts
+2 -2
View File
@@ -13,7 +13,7 @@ import grabDSQLConnection from "../utils/grab-dsql-connection";
* @returns {Promise<object|null>}
*/
(async () => {
const CONNECTION = grabDSQLConnection();
const CONNECTION = await grabDSQLConnection();
try {
const result = await CONNECTION.query(
@@ -24,7 +24,7 @@ import grabDSQLConnection from "../utils/grab-dsql-connection";
console.log("Connection query ERROR =>", error.message);
global.ERROR_CALLBACK?.(`Error Checking DB`, error as Error);
} finally {
CONNECTION?.end();
await CONNECTION?.end();
process.exit();
}
})();
package-shared/shell/test-external-server.ts
+2 -2
View File
@@ -13,7 +13,7 @@ import grabDSQLConnection from "../utils/grab-dsql-connection";
* @returns {Promise<object|null>}
*/
(async () => {
const CONNECTION = grabDSQLConnection({ noDb: true });
const CONNECTION = await grabDSQLConnection({ noDb: true });
/**
* Switch Database
@@ -32,7 +32,7 @@ import grabDSQLConnection from "../utils/grab-dsql-connection";
error as Error
);
} finally {
CONNECTION?.end();
await CONNECTION?.end();
process.exit();
}
})();
package-shared/shell/updateSSLUsers.ts
+2 -2
View File
@@ -14,7 +14,7 @@ import grabSQLKeyName from "../utils/grab-sql-key-name";
* @returns {Promise<object|null>}
*/
(async () => {
const CONNECTION = grabDSQLConnection();
const CONNECTION = await grabDSQLConnection();
try {
const result = await CONNECTION.query(
@@ -48,7 +48,7 @@ import grabSQLKeyName from "../utils/grab-sql-key-name";
} catch (error: any) {
global.ERROR_CALLBACK?.(`Error Updating SSL Users`, error as Error);
} finally {
CONNECTION.end();
await CONNECTION?.end();
process.exit();
}
})();
package-shared/utils/backend/global-db/DB_HANDLER.ts
+6 -2
View File
@@ -5,7 +5,7 @@ import grabDSQLConnection from "../../grab-dsql-connection";
* @requires DSQL_DB_CONN - Gobal Variable for Datasquirel Database
*/
export default async function DB_HANDLER(query: string, values?: any[]) {
const CONNECTION = grabDSQLConnection();
const CONNECTION = await grabDSQLConnection();
try {
if (!CONNECTION)
@@ -13,7 +13,11 @@ export default async function DB_HANDLER(query: string, values?: any[]) {
const results = await CONNECTION.query(query, values);
return JSON.parse(JSON.stringify(results));
if (Array.isArray(results)) {
return Array.from(results);
} else {
return results;
}
} catch (error: any) {
global.ERROR_CALLBACK?.(`DB_HANDLER Error`, error as Error);
return {
package-shared/utils/backend/global-db/DSQL_USER_DB_HANDLER.ts
+3 -3
View File
@@ -17,8 +17,8 @@ export default async function DSQL_USER_DB_HANDLER({
}: Param) {
const CONNECTION =
paradigm == "Read Only"
? grabDSQLConnection({ ro: true })
: grabDSQLConnection({ fa: true });
? await grabDSQLConnection({ ro: true })
: await grabDSQLConnection({ fa: true });
try {
return await connDbHandler(CONNECTION, queryString, queryValues);
@@ -26,6 +26,6 @@ export default async function DSQL_USER_DB_HANDLER({
global.ERROR_CALLBACK?.(`DSQL_USER_DB_HANDLER Error`, error as Error);
return null;
} finally {
CONNECTION?.end();
await CONNECTION?.end();
}
}
package-shared/utils/backend/global-db/LOCAL_DB_HANDLER.ts
+8 -4
View File
@@ -4,12 +4,16 @@ import grabDSQLConnection from "../../grab-dsql-connection";
* # DSQL user read-only DB handler
*/
export default async function LOCAL_DB_HANDLER(query: string, values?: any[]) {
const MASTER = grabDSQLConnection();
const CONNECTION = await grabDSQLConnection();
try {
const results = await MASTER.query(query, values);
const results = await CONNECTION.query(query, values);
return JSON.parse(JSON.stringify(results));
if (Array.isArray(results)) {
return Array.from(results);
} else {
return results;
}
} catch (error: any) {
global.ERROR_CALLBACK?.(`LOCAL_DB_HANDLER Error`, error as Error);
return {
@@ -17,6 +21,6 @@ export default async function LOCAL_DB_HANDLER(query: string, values?: any[]) {
error: error.message,
};
} finally {
await MASTER?.end();
await CONNECTION?.end();
}
}
package-shared/utils/backend/global-db/NO_DB_HANDLER.ts
+12 -9
View File
@@ -3,22 +3,27 @@ import grabDSQLConnection from "../../grab-dsql-connection";
/**
* # DSQL user read-only DB handler
*/
export default function NO_DB_HANDLER(query: string, values?: any[]) {
const CONNECTION = grabDSQLConnection();
export default async function NO_DB_HANDLER(query: string, values?: any[]) {
const CONNECTION = await grabDSQLConnection();
try {
return new Promise((resolve, reject) => {
CONNECTION.query(query, values)
.then((results) => {
CONNECTION.end();
resolve(JSON.parse(JSON.stringify(results)));
.then(async (results) => {
if (Array.isArray(results)) {
resolve(Array.from(results));
} else {
resolve(results);
}
})
.catch((err) => {
CONNECTION.end();
.catch(async (err) => {
resolve({
error: err.message,
sql: err.sql,
});
})
.finally(async () => {
await CONNECTION?.end();
});
});
} catch (error: any) {
@@ -27,7 +32,5 @@ export default function NO_DB_HANDLER(query: string, values?: any[]) {
success: false,
error: error.message,
};
} finally {
CONNECTION?.end();
}
}
package-shared/utils/backend/global-db/ROOT_DB_HANDLER.ts
-33
View File
@@ -1,33 +0,0 @@
import grabDSQLConnection from "../../grab-dsql-connection";
/**
* # Root DB handler
*/
export default function ROOT_DB_HANDLER(query: string, values?: any[]) {
const CONNECTION = grabDSQLConnection();
try {
return new Promise((resolve, reject) => {
CONNECTION.query(query, values)
.then((results) => {
CONNECTION.end();
resolve(JSON.parse(JSON.stringify(results)));
})
.catch((err) => {
CONNECTION.end();
resolve({
error: err.message,
sql: err.sql,
});
});
});
} catch (error: any) {
global.ERROR_CALLBACK?.(`ROOT_DB_HANDLER Error`, error as Error);
return {
success: false,
error: error.message,
};
} finally {
CONNECTION?.end();
}
}
package-shared/utils/db/conn-db-handler.ts
+10 -7
View File
@@ -1,6 +1,6 @@
import debugLog from "../logging/debug-log";
import { DSQLErrorObject } from "../../types";
import type { ConnectionConfig, Pool } from "mariadb";
import mariadb, { Connection, ConnectionConfig, Pool } from "mariadb";
export type ConnDBHandlerQueryObject = {
query: string;
@@ -20,9 +20,9 @@ type Return<ReturnType = any> =
*/
export default async function connDbHandler<ReturnType = any>(
/**
* MariaDB Connection Pool Object
* MariaDB Connection
*/
connPool?: Pool,
conn?: mariadb.Connection,
/**
* String Or `ConnDBHandlerQueryObject` Array
*/
@@ -34,13 +34,13 @@ export default async function connDbHandler<ReturnType = any>(
debug?: boolean
): Promise<Return<ReturnType>> {
try {
if (!connPool) throw new Error("No Connection Found!");
if (!conn) throw new Error("No Connection Found!");
if (!query) throw new Error("Query String Required!");
let queryErrorArray: DSQLErrorObject[] = [];
if (typeof query == "string") {
const res = await connPool.query(trimQuery(query), values);
const res = await conn.query(trimQuery(query), values);
if (debug) {
debugLog({
@@ -67,7 +67,7 @@ export default async function connDbHandler<ReturnType = any>(
currentQueryError.sql = queryObj.query;
currentQueryError.sqlValues = queryObj.values;
const queryObjRes = await connPool.query(
const queryObjRes = await conn.query(
trimQuery(queryObj.query),
queryObj.values
);
@@ -90,6 +90,9 @@ export default async function connDbHandler<ReturnType = any>(
`Connection DB Handler Query Error`,
error as Error
);
console.log("query", query);
resArray.push(null);
currentQueryError["error"] = error.message;
queryErrorArray.push(currentQueryError);
@@ -130,7 +133,7 @@ export default async function connDbHandler<ReturnType = any>(
// config: conn,
};
} finally {
connPool?.end();
await conn?.end();
}
}
package-shared/utils/grab-dsql-connection.ts
+24 -10
View File
@@ -1,4 +1,4 @@
import mariadb, { Pool } from "mariadb";
import mariadb, { Connection, Pool } from "mariadb";
import grabDbSSL from "./backend/grabDbSSL";
type Param = {
@@ -23,11 +23,13 @@ type Param = {
/**
* # Grab General CONNECTION for DSQL
*/
export default function grabDSQLConnection(param?: Param): Pool {
export default async function grabDSQLConnection(
param?: Param
): Promise<Connection> {
if (global.DSQL_USE_LOCAL || param?.local) {
return (
global.DSQL_DB_CONN ||
mariadb.createPool({
(await mariadb.createConnection({
host: process.env.DSQL_DB_HOST,
user: process.env.DSQL_DB_USERNAME,
password: process.env.DSQL_DB_PASSWORD,
@@ -37,14 +39,17 @@ export default function grabDSQLConnection(param?: Param): Pool {
: undefined,
charset: "utf8mb4",
ssl: grabDbSSL(),
})
supportBigNumbers: true,
bigNumberStrings: false,
dateStrings: true,
}))
);
}
if (param?.ro) {
return (
global.DSQL_READ_ONLY_DB_CONN ||
mariadb.createPool({
(await mariadb.createConnection({
host: process.env.DSQL_DB_HOST,
user: process.env.DSQL_DB_READ_ONLY_USERNAME,
password: process.env.DSQL_DB_READ_ONLY_PASSWORD,
@@ -53,14 +58,17 @@ export default function grabDSQLConnection(param?: Param): Pool {
: undefined,
charset: "utf8mb4",
ssl: grabDbSSL(),
})
supportBigNumbers: true,
bigNumberStrings: false,
dateStrings: true,
}))
);
}
if (param?.fa) {
return (
global.DSQL_FULL_ACCESS_DB_CONN ||
mariadb.createPool({
(await mariadb.createConnection({
host: process.env.DSQL_DB_HOST,
user: process.env.DSQL_DB_FULL_ACCESS_USERNAME,
password: process.env.DSQL_DB_FULL_ACCESS_PASSWORD,
@@ -69,13 +77,16 @@ export default function grabDSQLConnection(param?: Param): Pool {
: undefined,
charset: "utf8mb4",
ssl: grabDbSSL(),
})
supportBigNumbers: true,
bigNumberStrings: false,
dateStrings: true,
}))
);
}
return (
global.DSQL_DB_CONN ||
mariadb.createPool({
(await mariadb.createConnection({
host: process.env.DSQL_DB_HOST,
user: process.env.DSQL_DB_USERNAME,
password: process.env.DSQL_DB_PASSWORD,
@@ -85,6 +96,9 @@ export default function grabDSQLConnection(param?: Param): Pool {
: undefined,
charset: "utf8mb4",
ssl: grabDbSSL(),
})
supportBigNumbers: true,
bigNumberStrings: false,
dateStrings: true,
}))
);
}
package-shared/utils/setup-db.ts
+22 -6
View File
@@ -8,15 +8,15 @@ type Params = {
connectionLimit?: number;
};
export default function setupDSQLDb({
export default async function setupDSQLDb({
useLocal,
dbConfig,
ssl,
connectionLimit = 100,
connectionLimit = 20,
}: Params) {
global.DSQL_USE_LOCAL = useLocal || true;
const pool = mariadb.createPool({
const conn = await mariadb.createConnection({
host: process.env.DSQL_DB_HOST,
user: process.env.DSQL_DB_USERNAME,
password: process.env.DSQL_DB_PASSWORD,
@@ -24,10 +24,26 @@ export default function setupDSQLDb({
charset: "utf8mb4",
...dbConfig,
ssl: ssl ? grabDbSSL() : undefined,
connectionLimit,
supportBigNumbers: true,
bigNumberStrings: false,
dateStrings: true,
});
global.DSQL_DB_CONN = pool;
// const conn = mariadb.createPool({
// host: process.env.DSQL_DB_HOST,
// user: process.env.DSQL_DB_USERNAME,
// password: process.env.DSQL_DB_PASSWORD,
// database: process.env.DSQL_DB_NAME,
// charset: "utf8mb4",
// ...dbConfig,
// ssl: ssl ? grabDbSSL() : undefined,
// connectionLimit,
// supportBigNumbers: true,
// bigNumberStrings: false,
// dateStrings: true,
// });
global.DSQL_DB_CONN = conn;
// let readOnlyConnection;
@@ -47,7 +63,7 @@ export default function setupDSQLDb({
// }
return {
pool,
conn,
// readOnlyConnection,
};
}