Updates

src/cron/functions/check-services.ts

@@ -1,3 +1,9 @@
+import {
+    NormalizedServerObject,
+    ParsedDeploymentServiceConfig,
+} from "@/src/types";
+import execSSH from "@/src/utils/exec-ssh";
+import serviceFlight from "@/src/utils/flight";
 import grabTurboCiConfig from "@/src/utils/grab-turboci-config";
 
 export default async function cronCheckServices() {
@@ -13,12 +19,53 @@ export default async function cronCheckServices() {
         for (let srv = 0; srv < service.servers.length; srv++) {
             const server = service.servers[srv];
 
-            console.log("service", service.service_name);
-            console.log(server.private_ip);
-
             if (service.healthcheck) {
-                let cmd = ``;
+                const test = await healthcheck({ server, service });
+
+                if (!test) {
+                    const MAX_RETRIES = 5;
+                    let retries = 0;
+
+                    while (retries < MAX_RETRIES) {
+                        await serviceFlight({
+                            deployment: config,
+                            servers: [server],
+                            service,
+                        });
+
+                        await Bun.sleep(4000);
+
+                        const retest = await healthcheck({ server, service });
+
+                        if (retest) {
+                            break;
+                        } else {
+                            retries++;
+                        }
+                    }
+                }
             }
         }
     }
 }
+
+async function healthcheck({
+    server,
+    service,
+}: {
+    service: ParsedDeploymentServiceConfig;
+    server: NormalizedServerObject;
+}) {
+    if (!service.healthcheck?.cmd || !server.private_ip) {
+        return false;
+    }
+
+    const res = await execSSH({
+        cmd: service.healthcheck.cmd,
+        ip: server.private_ip,
+    });
+
+    const test = Boolean(res?.match(service.healthcheck.test));
+
+    return test;
+}