Updates

.gitignore

@@ -40,4 +40,5 @@ yarn-error.log*
 *.tsbuildinfo
 next-env.d.ts
 /src/db/turboci-admin
-.backups
+.backups
+/secrets

src/cron/functions/check-services.ts

@@ -1,3 +1,9 @@
+import {
+    NormalizedServerObject,
+    ParsedDeploymentServiceConfig,
+} from "@/src/types";
+import execSSH from "@/src/utils/exec-ssh";
+import serviceFlight from "@/src/utils/flight";
 import grabTurboCiConfig from "@/src/utils/grab-turboci-config";
 
 export default async function cronCheckServices() {
@@ -13,12 +19,53 @@ export default async function cronCheckServices() {
         for (let srv = 0; srv < service.servers.length; srv++) {
             const server = service.servers[srv];
 
-            console.log("service", service.service_name);
-            console.log(server.private_ip);
-
             if (service.healthcheck) {
-                let cmd = ``;
+                const test = await healthcheck({ server, service });
+
+                if (!test) {
+                    const MAX_RETRIES = 5;
+                    let retries = 0;
+
+                    while (retries < MAX_RETRIES) {
+                        await serviceFlight({
+                            deployment: config,
+                            servers: [server],
+                            service,
+                        });
+
+                        await Bun.sleep(4000);
+
+                        const retest = await healthcheck({ server, service });
+
+                        if (retest) {
+                            break;
+                        } else {
+                            retries++;
+                        }
+                    }
+                }
             }
         }
     }
 }
+
+async function healthcheck({
+    server,
+    service,
+}: {
+    service: ParsedDeploymentServiceConfig;
+    server: NormalizedServerObject;
+}) {
+    if (!service.healthcheck?.cmd || !server.private_ip) {
+        return false;
+    }
+
+    const res = await execSSH({
+        cmd: service.healthcheck.cmd,
+        ip: server.private_ip,
+    });
+
+    const test = Boolean(res?.match(service.healthcheck.test));
+
+    return test;
+}

src/data/app-data.ts

@@ -1,4 +1,18 @@
 export const AppData = {
     TerminalBinName: "ttyd",
     CronInterval: 30000,
+    max_instances: 200,
+    max_clusters: 1000,
+    max_servers_batch: 50,
+    load_balancer_fail_timeout_secs: 5,
+    load_balancer_max_fails: 1,
+    load_balancer_next_upstream_tries: 3,
+    load_balancer_next_upstream_timeout: 10,
+    load_balancer_connect_timeout: 3,
+    load_balancer_send_timeout: 60,
+    load_balancer_read_timeout: 60,
+    certbot_http_challenge_port: 8888,
+    private_server_batch_exec_size: 50,
+    ssh_max_tries: 50,
+    ssh_try_timeout_milliseconds: 5000,
 } as const;

src/types/index.ts

@@ -157,3 +157,10 @@ export type TCIConfigServiceConfigDirMApping = {
     use_gitignore?: boolean;
     relay_ignore?: string[];
 };
+
+export type ServiceScriptObject = {
+    sh: string;
+    service_name: string;
+    deployment_name: string;
+    work_dir?: string;
+};

src/utils/app-names.ts

@@ -0,0 +1,27 @@
+export const AppNames = {
+    TurboCIDefaultDir: ".turboci",
+    TurboCISSHKeyName: "turboci",
+    TurboCILabelNameKey: "turboci_deployment_name",
+    TurboCILabelServiceNameKey: "turboci_service_name",
+    DefaultConfigFile: "config.yaml",
+    DefaultConfigTSFile: "config.ts",
+    FileFlag: "-f, --file <path>",
+    SkipServiceFlag: "-s, --skip <service-name>",
+    TargetServicesFlag: "-t, --target <service-name>",
+    HetznerAPIKeyEnvName: "TURBOCI_HETZNER_API_KEY",
+    AWSAccessKeyEnvName: "TURBOCI_AWS_ACCESS_KEY",
+    AWSSecretAccessKeyEnvName: "TURBOCI_AWS_SECRET_ACCESS_KEY",
+    AzureAPIKeyEnvName: "TURBOCI_AZURE_API_KEY",
+    GCPServiceAccountEmail: "TURBOCI_GCP_SERVICE_ACCOUNT_EMAIL",
+    GCPProjectID: "TURBOCI_GCP_PROJECT_ID",
+    GCPServiceAccountPrivateKey: "TURBOCI_GCP_SERVICE_ACCOUNT_PRIVATE_KEY",
+    RsyncDefaultIgnoreFile: "turboci.ignore",
+    TurbosyncPreflightDefaultFile: "turboci.preflight.sh",
+    TurbosyncPostflightDefaultFile: "turboci.postflight.sh",
+    TurbosyncStartDefaultFile: "turboci.start.sh",
+    LoadBalancerUpstreamName: "turboci_load_balancer_upstream",
+    LoadBalancerBakcupUpstreamName: "turboci_load_balancer_backup_upstream",
+    LoadBalancerServerName: "turboci_lb.local",
+    CertbotSSLCertName: "turboci",
+    HealthcheckErrorMsg: "TurboCI Healthcheck Error",
+} as const;

src/utils/bun-grab-private-ips-bulk-scripts.ts

@@ -0,0 +1,103 @@
+import _ from "lodash";
+import turboCIPkgrabDirNames from "./turboci-pkg-grab-dir-names";
+import { AppData } from "../data/app-data";
+
+type Params = {
+    private_server_ips: string[];
+    script: string;
+    work_dir?: string;
+    parrallel?: boolean;
+    no_process_logs?: boolean;
+};
+
+export default function bunGrabPrivateIPsBulkScripts({
+    private_server_ips,
+    script,
+    work_dir,
+    parrallel,
+    no_process_logs,
+}: Params) {
+    const { relayServerSshPrivateKeyFile } = turboCIPkgrabDirNames();
+
+    let bunCmd = "";
+
+    bunCmd += `import _ from "lodash";\n`;
+    bunCmd += `import { execSync } from "child_process";\n`;
+    bunCmd += `\n`;
+    bunCmd += `const SSH_KEY = "${relayServerSshPrivateKeyFile}";\n`;
+    bunCmd += `const SSH_OPTS = \`-i \${SSH_KEY} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -C -c aes128-ctr\`;\n`;
+    bunCmd += `const TIMEOUT = ${AppData["ssh_try_timeout_milliseconds"]};\n`;
+    bunCmd += `const MAX_ATTEMPTS = ${AppData["ssh_max_tries"]};\n`;
+
+    bunCmd += `const REMOTE_HOSTS = [${private_server_ips
+        .map((h) => `"${h.replace(/\"/g, "")}"`)
+        .join(", ")}];\n`;
+
+    bunCmd += `const DEFAULT_SSH_USER = "root";\n`;
+    bunCmd += `const BATCH_SIZE = ${AppData["private_server_batch_exec_size"]};\n`;
+
+    bunCmd += `async function run(host: string) {\n`;
+    bunCmd += `    let attempt = 0;\n`;
+    if (!no_process_logs) {
+        bunCmd += `    console.log(\`Setting up \${host} ...\`);\n`;
+    }
+    bunCmd += `    while (attempt < MAX_ATTEMPTS) {\n`;
+    bunCmd += `        attempt += 1;\n`;
+    bunCmd += `        if (attempt > MAX_ATTEMPTS) {\n`;
+    if (!no_process_logs) {
+        bunCmd += `            console.log(\`Error: \${host} not ready after \${MAX_ATTEMPTS} attempts. Exiting.\`);\n`;
+    }
+    bunCmd += `            process.exit(1);\n`;
+    bunCmd += `        }\n`;
+    bunCmd += `        try {\n`;
+    if (!no_process_logs) {
+        bunCmd += `            console.log(\`Waiting for \${host} to be ready... (attempt \${attempt}/\${MAX_ATTEMPTS})\`);\n`;
+    }
+    bunCmd += `            let testCmd = \`ssh \${SSH_OPTS} \${DEFAULT_SSH_USER}@\${host} echo "Running ..."\`;\n`;
+    bunCmd += `            let testRes = execSync(testCmd, { encoding: "utf-8" });\n`;
+    bunCmd += `            if (testRes?.match(/Running/)) break;\n`;
+    bunCmd += `        } catch (error) {\n`;
+    if (!no_process_logs) {
+        bunCmd += `            console.log(\`Attempt \${attempt} failed!\`);\n`;
+    }
+    bunCmd += `        }\n`;
+    bunCmd += `        await Bun.sleep(TIMEOUT);\n`;
+    bunCmd += `    }\n`;
+    bunCmd += `    attempt = 0;\n`;
+    bunCmd += `\n`;
+
+    bunCmd += `    let execCmd = \`ssh \${SSH_OPTS} \${DEFAULT_SSH_USER}@\${host} << 'TURBOCIEXEC'\\n\`;\n`;
+
+    if (work_dir) {
+        bunCmd += `    execCmd += \`cd ${work_dir}\\n\`;\n`;
+    }
+
+    bunCmd += `    execCmd += \`${script}\\n\`;\n`;
+    bunCmd += `    execCmd += \`TURBOCIEXEC\\n\`;\n`;
+    bunCmd += `    try {\n`;
+    bunCmd += `        const res = execSync(execCmd, { encoding: "utf-8" });\n`;
+    bunCmd += `        console.log(res);\n`;
+    bunCmd += `    } catch (error) {\n`;
+    bunCmd += `        process.exit(1);\n`;
+    bunCmd += `    }\n`;
+    bunCmd += `}\n`;
+    bunCmd += `\n`;
+
+    if (parrallel) {
+        bunCmd += `const first_host = REMOTE_HOSTS.splice(0,1)[0];\n`;
+        bunCmd += `await run(first_host)\n`;
+        bunCmd += `\n`;
+        bunCmd += `const chunks = _.chunk(REMOTE_HOSTS, BATCH_SIZE);\n`;
+        bunCmd += `for (let i = 0; i < chunks.length; i++) {\n`;
+        bunCmd += `    const chunk = chunks[i];\n`;
+        bunCmd += `    const runChunk = await Promise.all(chunk.map(h => run(h)));\n`;
+        bunCmd += `}\n`;
+    } else {
+        bunCmd += `for (let i = 0; i < REMOTE_HOSTS.length; i++) {\n`;
+        bunCmd += `    const host = REMOTE_HOSTS[i];\n`;
+        bunCmd += `    const runHost = await run(host);\n`;
+        bunCmd += `}\n`;
+    }
+
+    return bunCmd;
+}

src/utils/flight.ts

@@ -0,0 +1,200 @@
+import chalk from "chalk";
+import { existsSync, readFileSync, statSync } from "fs";
+import path from "path";
+import {
+    NormalizedServerObject,
+    ParsedDeploymentServiceConfig,
+    ServiceScriptObject,
+    TCIGlobalConfig,
+} from "../types";
+import { AppNames } from "./app-names";
+import grabSHEnvs from "./grab-sh-env";
+import bunGrabPrivateIPsBulkScripts from "./bun-grab-private-ips-bulk-scripts";
+import relayExecSSH from "./relay-exec-ssh";
+
+const Paradigms = ["preflight", "start", "postflight"] as const;
+
+type Params = {
+    deployment: TCIGlobalConfig;
+    service: ParsedDeploymentServiceConfig;
+    servers: NormalizedServerObject[];
+};
+
+export default async function serviceFlight({
+    deployment,
+    service,
+    servers,
+}: Params) {
+    const allCommands: {
+        [k in (typeof Paradigms)[number]]?: ServiceScriptObject;
+    } = {};
+
+    for (let i = 0; i < Paradigms.length; i++) {
+        const paradigm = Paradigms[i];
+        if (!paradigm) continue;
+
+        let serviceType = service.type || "default";
+
+        let sh = "";
+        let work_dir = service.dir_mappings?.[0]?.dst;
+
+        switch (serviceType) {
+            case "default":
+                const defaultCmds =
+                    paradigm == "preflight"
+                        ? service.run?.preflight?.cmds
+                        : paradigm == "postflight"
+                          ? service.run?.postflight?.cmds
+                          : paradigm == "start"
+                            ? service.run?.start?.cmds
+                            : undefined;
+
+                if (defaultCmds) {
+                    for (let i = 0; i < defaultCmds.length; i++) {
+                        const cmd = defaultCmds[i];
+                        sh += cmd + "\n";
+                    }
+                }
+
+                const new_work_dir =
+                    paradigm == "preflight"
+                        ? service.run?.preflight?.work_dir
+                        : paradigm == "postflight"
+                          ? service.run?.postflight?.work_dir
+                          : paradigm == "start"
+                            ? service.run?.start?.work_dir
+                            : work_dir;
+
+                work_dir = new_work_dir;
+
+                const target_file = service.run?.preflight?.file
+                    ? path.resolve(process.cwd(), service.run?.preflight?.file)
+                    : undefined;
+
+                if (
+                    target_file &&
+                    existsSync(target_file) &&
+                    statSync(target_file).isFile()
+                ) {
+                    const targetFileSHText = readFileSync(
+                        target_file,
+                        "utf-8",
+                    ).replace(/!#\/bin\/.*/, "");
+
+                    sh += `\n${targetFileSHText}\n`;
+                    break;
+                }
+
+                const defaultFile = path.join(
+                    process.cwd(),
+                    paradigm == "preflight"
+                        ? AppNames["TurbosyncPreflightDefaultFile"]
+                        : paradigm == "postflight"
+                          ? AppNames["TurbosyncPostflightDefaultFile"]
+                          : paradigm == "start"
+                            ? AppNames["TurbosyncStartDefaultFile"]
+                            : "",
+                );
+
+                if (existsSync(defaultFile) && statSync(defaultFile).isFile()) {
+                    const shText = readFileSync(defaultFile, "utf-8").replace(
+                        /!#\/bin\/.*/,
+                        "",
+                    );
+
+                    sh += `\n${shText}\n`;
+                }
+
+                break;
+
+            case "load_balancer":
+                break;
+
+            default:
+                break;
+        }
+
+        allCommands[paradigm] = {
+            sh,
+            work_dir,
+            deployment_name: deployment.deployment_name,
+            service_name: service.service_name,
+        };
+    }
+
+    const targetServicesShArr = [
+        allCommands.preflight,
+        allCommands.start,
+        allCommands.postflight,
+    ];
+
+    let finalCmds = `set -e\n\n`;
+
+    const envsStr = grabSHEnvs({ deployment, service });
+
+    finalCmds += envsStr;
+
+    for (let i = 0; i < targetServicesShArr.length; i++) {
+        const serviceSh = targetServicesShArr[i];
+
+        if (!serviceSh || !serviceSh.sh.match(/./)) {
+            continue;
+        }
+
+        if (serviceSh.work_dir) {
+            finalCmds += `cd ${serviceSh.work_dir}\n`;
+        }
+
+        finalCmds += `${serviceSh.sh}\n`;
+    }
+
+    if (!finalCmds.match(/\w/)) {
+        return true;
+    }
+
+    if (service.healthcheck?.cmd && service.healthcheck?.test) {
+        let healthcheckCmd = ``;
+        healthcheckCmd += `healthcheck_max_attempts=5\n`;
+        healthcheckCmd += `healthcheck_attempt=1\n`;
+        healthcheckCmd += `while [ $healthcheck_attempt -le $healthcheck_max_attempts ]; do\n`;
+        healthcheckCmd += `    if ${service.healthcheck.cmd} | grep "${service.healthcheck.test}"; then\n`;
+        healthcheckCmd += `        echo "Healthcheck succeeded."\n`;
+        healthcheckCmd += `        exit 0\n`;
+        healthcheckCmd += `    else\n`;
+        healthcheckCmd += `        ((healthcheck_attempt++))\n`;
+        healthcheckCmd += `        sleep 5\n`;
+        healthcheckCmd += `    fi\n`;
+        healthcheckCmd += `done\n`;
+        healthcheckCmd += `echo "${AppNames["HealthcheckErrorMsg"]}" >&2\n`;
+        healthcheckCmd += `exit 1\n`;
+        finalCmds += healthcheckCmd;
+    }
+
+    const private_server_ips = servers
+        .map((srv) => srv.private_ip)
+        .filter((srv) => Boolean(srv)) as string[];
+
+    const finalCmdBun = bunGrabPrivateIPsBulkScripts({
+        private_server_ips,
+        script: finalCmds,
+        parrallel: true,
+    });
+
+    const run = await relayExecSSH({
+        cmd: finalCmdBun,
+        exit_on_error: true,
+        log_error: true,
+        bun: true,
+    });
+
+    if (!run || run.match(new RegExp(`${AppNames["HealthcheckErrorMsg"]}`))) {
+        console.error(
+            `\nERROR running applications: ${chalk.white(
+                chalk.italic(service.service_name),
+            )} flight failed!\n`,
+        );
+        process.exit(1);
+    }
+
+    return true;
+}

src/utils/grab-sh-env.ts

@@ -0,0 +1,71 @@
+import _ from "lodash";
+import path from "path";
+import { ParsedDeploymentServiceConfig, TCIGlobalConfig } from "../types";
+import parseEnv from "./parse-env";
+
+type Params = {
+    deployment: Omit<TCIGlobalConfig, "services">;
+    service?: ParsedDeploymentServiceConfig;
+};
+
+export default function grabSHEnvs({ deployment, service }: Params) {
+    let env: { [k: string]: string } = {};
+
+    const deploymentEnvs = deployment.env;
+    const deploymentEnvFile = deployment.env_file;
+
+    if (deploymentEnvs) {
+        env = _.merge(env, deploymentEnvs);
+    }
+
+    if (deploymentEnvFile) {
+        const envFileVars = readEnvFile({ filePath: deploymentEnvFile });
+        env = _.merge(env, envFileVars);
+    }
+
+    if (service) {
+        const serviceEnvs = service.env;
+        const serviceEnvFile = service.env_file;
+
+        if (serviceEnvs) {
+            env = _.merge(env, serviceEnvs);
+        }
+
+        if (serviceEnvFile) {
+            const envFileVars = readEnvFile({ filePath: serviceEnvFile });
+            env = _.merge(env, envFileVars);
+        }
+    }
+
+    let envSh = ``;
+
+    const ENV_KEYS = Object.keys(env);
+
+    for (let i = 0; i < ENV_KEYS.length; i++) {
+        const env_key = ENV_KEYS[i];
+        if (!env_key) continue;
+        const env_value = env[env_key];
+        if (!env_value) continue;
+
+        if (env_value.match(/\"/)) {
+            console.error(`Please omit \`\"\` from all env variables`);
+            process.exit(1);
+        }
+
+        envSh += `export ${env_key}="${env_value}"\n`;
+    }
+
+    envSh += `\n`;
+
+    return envSh;
+}
+
+function readEnvFile({ filePath }: { filePath: string }):
+    | {
+          [k: string]: string;
+      }
+    | undefined {
+    const finalFilePath = path.resolve(process.cwd(), filePath);
+
+    return parseEnv(finalFilePath);
+}

src/utils/parse-env.ts

@@ -0,0 +1,40 @@
+import fs from "fs";
+
+export default function parseEnv(
+    /** The file path to the env. Eg. /app/.env */ envFile: string
+) {
+    if (!fs.existsSync(envFile)) return undefined;
+
+    const envTextContent = fs.readFileSync(envFile, "utf-8");
+    const envLines = envTextContent
+        .split("\n")
+        .map((ln) => ln.trim())
+        .filter((ln) => {
+            const commentLine = ln.match(/^\#/);
+            const validEnv = ln.match(/.*\=/);
+
+            if (commentLine) return false;
+            if (validEnv) return true;
+            return false;
+        });
+
+    const newEnvObj: { [k: string]: string } = {};
+
+    for (let i = 0; i < envLines.length; i++) {
+        const emvLine = envLines[i];
+        if (!emvLine) continue;
+        const envLineArr = emvLine.split("=");
+        const envTitle = envLineArr[0];
+        const envValue = envLineArr[1] as string | undefined;
+
+        if (!envTitle?.match(/./)) continue;
+
+        if (envValue?.match(/./)) {
+            newEnvObj[envTitle] = envValue;
+        } else {
+            newEnvObj[envTitle] = "";
+        }
+    }
+
+    return newEnvObj as { [k: string]: string };
+}

src/utils/relay-exec-ssh.ts

@@ -0,0 +1,111 @@
+import { exec, execSync, type ExecSyncOptions } from "child_process";
+import grabSSHPrefix from "./grab-ssh-prefix";
+import { writeFileSync } from "fs";
+import { TCIConfig } from "../types";
+import turboCIPkgrabDirNames from "./turboci-pkg-grab-dir-names";
+
+type Param = {
+    cmd: string | string[];
+    debug?: boolean;
+    user?: string;
+    options?: ExecSyncOptions;
+    detached?: boolean;
+    return_cmd_only?: boolean;
+    exit_on_error?: boolean;
+    log_error?: boolean;
+    bun?: boolean;
+};
+
+export default async function relayExecSSH({
+    cmd,
+    debug,
+    user = "root",
+    options,
+    detached,
+    return_cmd_only,
+    exit_on_error,
+    log_error,
+    bun,
+}: Param): Promise<string | undefined> {
+    try {
+        const {
+            relayServerBunScriptsDir,
+            relayServerBunScriptFile,
+            relayShExecFile,
+        } = turboCIPkgrabDirNames();
+
+        let relaySh = `#!/bin/bash\n`;
+
+        const parsedCmd =
+            typeof cmd == "string"
+                ? cmd
+                : Array.isArray(cmd)
+                  ? cmd.join("\n")
+                  : undefined;
+
+        if (bun) {
+            if (exit_on_error) {
+                relaySh += `set -e\n`;
+                relaySh += `\n`;
+            }
+            relaySh += `mkdir -p ${relayServerBunScriptsDir}\n`;
+            relaySh += `cat << 'RELAYHEREDOC' > ${relayServerBunScriptFile}\n`;
+            relaySh += `${parsedCmd}\n`;
+            relaySh += `RELAYHEREDOC\n`;
+            relaySh += `bun ${relayServerBunScriptFile}\n`;
+        } else {
+            const finalSumCmd = exit_on_error
+                ? `set -e\n\n${parsedCmd}\n`
+                : parsedCmd;
+
+            relaySh += `${finalSumCmd}\n`;
+        }
+
+        if (return_cmd_only) {
+            return relaySh;
+        }
+
+        if (debug) {
+            console.log("====================================================");
+            console.log("====================================================");
+            console.log("====================================================");
+            console.log(`SSH Command =>`, relaySh);
+            console.log("====================================================");
+            console.log("====================================================");
+            console.log("====================================================");
+        }
+
+        writeFileSync(relayShExecFile, relaySh);
+
+        let relayCmd = ``;
+
+        relayCmd += ` 'chmod +x ${relayShExecFile} && /bin/bash ${relayShExecFile}'\n`;
+
+        if (detached) {
+            exec(relayCmd);
+        } else {
+            const str = execSync(relayCmd, {
+                stdio: ["pipe", "pipe", "pipe"],
+                ...options,
+                encoding: "utf-8",
+            });
+
+            if (debug) {
+                console.log("============================================");
+                console.log("============================================");
+                console.log("============================================");
+                console.log(`SSH Result =>`, str);
+                console.log("============================================");
+                console.log("============================================");
+                console.log("============================================");
+            }
+
+            return str.trim();
+        }
+    } catch (error: any) {
+        if (debug || log_error) {
+            console.error(`Relay SSH Error: ${error.message}`);
+        }
+        return undefined;
+    }
+}

src/utils/turboci-pkg-grab-dir-names.ts

@@ -0,0 +1,43 @@
+import path from "path";
+import { AppNames } from "./app-names";
+
+type Params = {
+    name?: string;
+};
+
+export default function turboCIPkgrabDirNames(params?: Params) {
+    const relayTurboCIDir = "/root/.turboci";
+    const relayConfigDir = path.join(relayTurboCIDir, ".config");
+    const relayConfigJSON = path.join(relayConfigDir, "turboci.json");
+    const relayServerSSHDir = path.join(relayTurboCIDir, ".ssh");
+    const relayServerBunScriptsDir = path.join(relayTurboCIDir, ".bun");
+    const relayServerBunScriptFile = path.join(
+        relayServerBunScriptsDir,
+        "run.ts",
+    );
+    const relayServerSshPublicKeyFile = path.join(
+        relayServerSSHDir,
+        `${AppNames["TurboCISSHKeyName"]}.pub`,
+    );
+    const relayServerSshPrivateKeyFile = path.join(
+        relayServerSSHDir,
+        AppNames["TurboCISSHKeyName"],
+    );
+
+    const relayServerRsyncDir = "/root/.turboci/.rsync";
+    const relayShDir = "/root/.turboci/.sh";
+    const relayShExecFile = path.join(relayShDir, "relay.sh");
+
+    return {
+        relayServerSSHDir,
+        relayServerSshPublicKeyFile,
+        relayServerSshPrivateKeyFile,
+        relayServerRsyncDir,
+        relayServerBunScriptsDir,
+        relayServerBunScriptFile,
+        relayShDir,
+        relayShExecFile,
+        relayConfigDir,
+        relayConfigJSON,
+    };
+}