Improve cluster allocation strategy.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/src/deployments/deployments.service.ts

@@ -7,6 +7,7 @@ import { ApplicationsService } from '../applications/applications.service';
 import { KubernetesService } from '../kubernetes/kubernetes.service';
 import { BuildService, BuildProgress, BuildCancelledError } from '../build/build.service';
 import { AppLifecycleStatus, DeploymentStatus } from '../common/enums';
+import { ClustersService } from '../clusters/clusters.service';
 
 @Injectable()
 export class DeploymentsService {
@@ -19,6 +20,7 @@ export class DeploymentsService {
     private applicationsService: ApplicationsService,
     private kubernetesService: KubernetesService,
     private buildService: BuildService,
+    private clustersService: ClustersService,
   ) {}
 
   async triggerDeployment(applicationId: string, userId: string): Promise<Deployment> {
@@ -62,11 +64,9 @@ export class DeploymentsService {
         message: 'Deploying to Kubernetes...',
       });
 
-      // If a DB dump will be restored, deploy with 0 replicas first so WordPress
-      // does not initialize empty tables before the dump is imported.
       const hasDbDump = app.dbDumpPath && fs.existsSync(app.dbDumpPath);
-      const deployApp = hasDbDump ? { ...app, replicas: 0 } : app;
-      const k8sResources = await this.kubernetesService.deployApplication(deployApp, imageUri);
+      const { app: deployedApp, k8sResources } = await this.deployWithClusterFallback(deploymentId, app, imageUri, hasDbDump);
+      app = deployedApp;
 
       // Step 3.5: Restore DB dump if one was uploaded (must happen after deploy creates the namespace + DB)
       if (hasDbDump) {
@@ -149,6 +149,70 @@ export class DeploymentsService {
     }
   }
 
+  private async deployWithClusterFallback(
+    deploymentId: string,
+    app: any,
+    imageUri: string,
+    hasDbDump: boolean,
+  ): Promise<{ app: any; k8sResources: Record<string, any> }> {
+    const failedClusterIds: string[] = [];
+    let currentApp = app;
+    let lastError: any;
+    const maxAttempts = Number(process.env.CLUSTER_DEPLOY_FALLBACK_ATTEMPTS || 3);
+
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+      if (await this.isDeploymentCancelled(deploymentId)) {
+        throw new Error('Deployment cancelled by user');
+      }
+
+      try {
+        this.buildService.setProgress(deploymentId, {
+          phase: 'deploying',
+          percent: Math.min(92 + attempt, 95),
+          message: attempt === 1
+            ? 'Deploying to selected cluster...'
+            : `Retrying deployment on fallback cluster (${attempt}/${maxAttempts})...`,
+        });
+
+        const deployApp = hasDbDump ? { ...currentApp, replicas: 0 } : currentApp;
+        const k8sResources = await this.kubernetesService.deployApplication(deployApp, imageUri);
+        return { app: currentApp, k8sResources };
+      } catch (error: any) {
+        lastError = error;
+        failedClusterIds.push(currentApp.clusterId);
+        const failureMessage = error?.message || 'Deployment failed on selected cluster';
+        await this.clustersService.markAllocationFailure(currentApp.id, currentApp.clusterId, failureMessage);
+
+        if (attempt >= maxAttempts) {
+          break;
+        }
+
+        try {
+          const fallback = await this.clustersService.chooseFallbackClusterForApplication(
+            currentApp,
+            failedClusterIds,
+            failureMessage,
+          );
+          const updatedApp = await this.applicationsService.updateClusterAssignment(
+            currentApp.id,
+            fallback.cluster.id,
+            fallback.pool?.id,
+          );
+          await this.clustersService.attachAllocationToApplication(fallback.allocationLogId, currentApp.id);
+          this.logger.warn(
+            `Deployment ${deploymentId} falling back from cluster ${currentApp.clusterId || 'none'} to ${fallback.cluster.id}`,
+          );
+          currentApp = { ...currentApp, ...updatedApp, clusterId: fallback.cluster.id, poolId: fallback.pool?.id || currentApp.poolId };
+        } catch (fallbackError: any) {
+          this.logger.warn(`No fallback cluster available for deployment ${deploymentId}: ${fallbackError.message}`);
+          break;
+        }
+      }
+    }
+
+    throw lastError;
+  }
+
   async updateStatus(id: string, status: DeploymentStatus): Promise<void> {
     if (await this.isDeploymentCancelled(id)) {
       return;