Serve preview URLs over Traefik+TLS, stabilize them, and speed up builds.

Ingress / preview URLs: - Default the app Ingress class and ACME HTTP-01 solver to Traefik (k3s default) via a new INGRESS_CLASS env, instead of hardcoding nginx — fixes 404s on clusters without ingress-nginx. - Only put public, real-TLD hosts (custom domain + preview) in the TLS block; the internal *.apps.cloudhost.local host no longer poisons the Let's Encrypt order, so certs actually issue. - Make the per-app preview number stable across redeploys so URLs stop breaking, and let PREVIEW_BASE_DOMAIN configure the base domain. Registry pulls: - Point the k3s registries.yaml mirror endpoint at the registry NodePort on loopback so node containerd never depends on cluster DNS (image pulls survive node restarts). Builds: - Pin the Kaniko image, use IfNotPresent pull policy, drop the dead build queue/processor, and retry transient Kubernetes API errors while polling build jobs. Logs & apps list: - fluent-bit reads log files from head so startup output reaches Elasticsearch. - Order joined deployments newest-first so the apps list shows the latest deployment status. Allocation: - Reserve in-flight (pending/building) capacity and stop globally degrading the cluster on a single allocation failure, so concurrent deploys don't starve or wrongly report "no healthy cluster". Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-06-02 15:44:08 +03:30
parent 786689e0fd
commit 4301277b48
13 changed files with 244 additions and 109 deletions
@@ -300,12 +300,15 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
    }

    const candidates = await this.getCachedHealthyClusters(pool || undefined, options.excludeClusterIds || []);
-    const appCounts = await this.getAppCounts(candidates.map((cluster) => cluster.id));
+    const clusterIds = candidates.map((cluster) => cluster.id);
+    const appCounts = await this.getAppCounts(clusterIds);
+    const reservations = await this.getInFlightReservations(clusterIds, options.applicationId);
    const candidateScores: Record<string, any>[] = [];
    const rejectionReasons: Record<string, any>[] = [];

    for (const cluster of candidates) {
-      const rejection = this.getClusterRejectionReason(cluster, estimatedRequest);
+      const reserved = reservations.get(cluster.id) || { cpuMillicores: 0, memoryMi: 0, pods: 0 };
+      const rejection = this.getClusterRejectionReason(cluster, estimatedRequest, reserved);
      if (rejection) {
        rejectionReasons.push({ clusterId: cluster.id, clusterName: cluster.name, reason: rejection });
        continue;
@@ -313,8 +316,8 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {

      const strategy = this.resolveStrategy(pool?.strategy);
      const appCount = appCounts.get(cluster.id) || 0;
-      const score = this.scoreCluster(cluster, estimatedRequest, appCount, strategy, (dto as any).region);
-      const resourceMetrics = this.getResourceMetrics(cluster, estimatedRequest, appCount);
+      const score = this.scoreCluster(cluster, estimatedRequest, appCount, strategy, (dto as any).region, reserved);
+      const resourceMetrics = this.getResourceMetrics(cluster, estimatedRequest, appCount, reserved);
      candidateScores.push({
        clusterId: cluster.id,
        clusterName: cluster.name,
@@ -382,11 +385,13 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
    message: string,
  ): Promise<void> {
    if (clusterId) {
-      await this.clustersRepository.update(clusterId, {
-        healthStatus: 'degraded',
-        healthMessage: message,
-        lastHealthCheckedAt: new Date(),
-      });
+      // A single deployment failure (quota, image pull, app bug, transient
+      // scheduling pressure) does NOT mean the cluster is unhealthy — flipping
+      // its healthStatus to 'degraded' would poison it for every other app and,
+      // when there's only one cluster, cause "no healthy cluster has capacity"
+      // for the next deployment. The fallback loop already excludes this cluster
+      // for *this* app via excludeClusterIds, so here we only invalidate the
+      // cached capacity snapshot to force a fresh read on the next allocation.
      this.clusterHealthCache.delete(clusterId);
    }

@@ -1023,6 +1028,13 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {

    const { username, password } = this.registryService.getRegistryCredentials();
    const dsName = 'cloudhost-k3s-registry-mirrors';
+    // The mirror endpoint must be reachable by the node's containerd, which does
+    // NOT use cluster DNS — so we point it at the registry NodePort on loopback
+    // (http://127.0.0.1:<nodePort>) instead of the in-cluster service DNS name.
+    // Otherwise image pulls break whenever node-level resolution of
+    // *.svc.cluster.local is unavailable (e.g. right after a node restart).
+    const registryNodePort = 30500;
+    const nodePortHost = `127.0.0.1:${registryNodePort}`;
    const configureScript = [
      'set -e',
      'REG=/host/etc/rancher/k3s/registries.yaml',
@@ -1031,12 +1043,16 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
      'mirrors:',
      `  "${registryUrl}":`,
      '    endpoint:',
-      `      - "http://${registryUrl}"`,
+      `      - "http://${nodePortHost}"`,
      'configs:',
      `  "${registryUrl}":`,
      '    auth:',
      `      username: ${JSON.stringify(username)}`,
      `      password: ${JSON.stringify(password)}`,
+      `  "${nodePortHost}":`,
+      '    auth:',
+      `      username: ${JSON.stringify(username)}`,
+      `      password: ${JSON.stringify(password)}`,
      'EOFREG',
      'if [ ! -f "$REG" ] || ! cmp -s /tmp/cloudhost-registries.yaml "$REG"; then',
      '  cp /tmp/cloudhost-registries.yaml "$REG"',
@@ -1212,7 +1228,11 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
    }
  }

-  private getClusterRejectionReason(cluster: Cluster, estimatedRequest: Record<string, any>): string | null {
+  private getClusterRejectionReason(
+    cluster: Cluster,
+    estimatedRequest: Record<string, any>,
+    reserved: { cpuMillicores: number; memoryMi: number; pods: number } = { cpuMillicores: 0, memoryMi: 0, pods: 0 },
+  ): string | null {
    if (cluster.status !== ClusterStatus.ACTIVE) {
      return `status=${cluster.status}`;
    }
@@ -1221,7 +1241,7 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
      return `health=${cluster.healthStatus}`;
    }

-    const metrics = this.getResourceMetrics(cluster, estimatedRequest, 0);
+    const metrics = this.getResourceMetrics(cluster, estimatedRequest, 0, reserved);
    const { available, utilization } = metrics;

    if (available.cpuMillicores > 0 && available.cpuMillicores < estimatedRequest.cpuMillicores) {
@@ -1252,8 +1272,9 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
    appCount: number,
    strategy: PoolStrategy,
    desiredRegion?: string,
+    reserved: { cpuMillicores: number; memoryMi: number; pods: number } = { cpuMillicores: 0, memoryMi: 0, pods: 0 },
  ): number {
-    const metrics = this.getResourceMetrics(cluster, estimatedRequest, appCount);
+    const metrics = this.getResourceMetrics(cluster, estimatedRequest, appCount, reserved);
    const capacityScore = metrics.capacityScore;
    const appPenalty = Math.min(appCount, 100) * 0.75;

@@ -1281,6 +1302,7 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
    cluster: Cluster,
    estimatedRequest: Record<string, any>,
    appCount: number,
+    reserved: { cpuMillicores: number; memoryMi: number; pods: number } = { cpuMillicores: 0, memoryMi: 0, pods: 0 },
  ): {
    available: { cpuMillicores: number; memoryMi: number; storageMi: number; pods: number };
    utilization: { cpu: number; memory: number; storage: number; pods: number; appPressure: number; average: number };
@@ -1297,17 +1319,23 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
    const nodeCount = Number(resources.nodeCount || 0);
    const podCapacity = Number(resources.podCapacity || nodeCount * 110 || 0);

+    // Include capacity reserved by in-flight deployments whose pods don't exist
+    // on the cluster yet, so concurrent allocations don't over-commit the node.
+    const cpuUsed = cpuRequested + (reserved.cpuMillicores || 0);
+    const memoryUsed = memoryRequested + (reserved.memoryMi || 0);
+    const podsUsed = podCount + (reserved.pods || 0);
+
    const available = {
-      cpuMillicores: Math.max(cpuCapacity - cpuRequested, 0),
-      memoryMi: Math.max(memoryCapacity - memoryRequested, 0),
+      cpuMillicores: Math.max(cpuCapacity - cpuUsed, 0),
+      memoryMi: Math.max(memoryCapacity - memoryUsed, 0),
      storageMi: storageCapacity > 0 ? Math.max(storageCapacity - storageUsed, 0) : 0,
-      pods: podCapacity > 0 ? Math.max(podCapacity - podCount, 0) : 0,
+      pods: podCapacity > 0 ? Math.max(podCapacity - podsUsed, 0) : 0,
    };
    const utilization = {
-      cpu: this.utilizationRatio(cpuRequested, cpuCapacity),
-      memory: this.utilizationRatio(memoryRequested, memoryCapacity),
+      cpu: this.utilizationRatio(cpuUsed, cpuCapacity),
+      memory: this.utilizationRatio(memoryUsed, memoryCapacity),
      storage: this.utilizationRatio(storageUsed, storageCapacity),
-      pods: this.utilizationRatio(podCount, podCapacity),
+      pods: this.utilizationRatio(podsUsed, podCapacity),
      appPressure: Math.min(appCount / 100, 1),
      average: 0,
    };
@@ -1377,6 +1405,60 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
    return new Map(rows.map((row) => [row.clusterId, parseInt(row.count, 10)]));
  }

+  /**
+   * Capacity that has been allocated to deployments which are still building but
+   * whose pods don't exist on the cluster yet (status pending/building). The
+   * cluster's `availableResources` snapshot only reflects pods that already exist
+   * (and is cached up to cacheTtlMs), so without this, two deployments started
+   * within the same window both read the same stale capacity and can both be
+   * placed on a cluster that only has room for one — the second then can't be
+   * scheduled and hangs. We subtract these in-flight requests during allocation.
+   *
+   * @param excludeApplicationId skip an app's own in-flight deployment (used on
+   *        fallback re-allocation so the app doesn't reserve capacity against itself).
+   */
+  private async getInFlightReservations(
+    clusterIds: string[],
+    excludeApplicationId?: string,
+  ): Promise<Map<string, { cpuMillicores: number; memoryMi: number; pods: number }>> {
+    const result = new Map<string, { cpuMillicores: number; memoryMi: number; pods: number }>();
+    if (clusterIds.length === 0) {
+      return result;
+    }
+
+    const rows: any[] = await this.dataSource.query(`
+      SELECT a.id as "applicationId",
+             a."clusterId" as "clusterId",
+             a."cpuRequest" as "cpuRequest",
+             a."memoryRequest" as "memoryRequest",
+             a.replicas as "replicas",
+             a."databaseType" as "databaseType",
+             a."enableRedis" as "enableRedis",
+             a."enableRabbitmq" as "enableRabbitmq",
+             a."appStorageSize" as "appStorageSize",
+             a."dbStorageSize" as "dbStorageSize",
+             a."optionalServiceResources" as "optionalServiceResources"
+      FROM deployments d
+      JOIN applications a ON a.id = d."applicationId"
+      WHERE a."clusterId" = ANY($1)
+        AND d.status IN ('pending', 'building')
+    `, [clusterIds]);
+
+    for (const row of rows) {
+      if (excludeApplicationId && row.applicationId === excludeApplicationId) {
+        continue;
+      }
+      const est = this.estimateApplicationRequest(row as CreateApplicationDto);
+      const current = result.get(row.clusterId) || { cpuMillicores: 0, memoryMi: 0, pods: 0 };
+      current.cpuMillicores += est.cpuMillicores;
+      current.memoryMi += est.memoryMi;
+      current.pods += est.podEstimate;
+      result.set(row.clusterId, current);
+    }
+
+    return result;
+  }
+
  private parseStorageToMi(storage: string): number {
    if (!storage) return 0;
    if (storage.endsWith('Ki')) return parseFloat(storage) / 1024;