Harden platform security, reliability, and CI after full audit.

Close deployment IDOR and gate stub payment endpoints, add production
secret validation, health probes, Redis-backed build progress, GitHub
Actions CI, expanded tests, billing/k8s refactors, and ops runbooks.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/src/kubernetes/kubernetes.service.ts

@@ -15,6 +15,8 @@ import { ensureAppUrlEnv } from '../applications/app-url.util';
 import { AppRuntime, DatabaseType, CustomDomainStatus, ServiceAccessTarget, ProductType, isManagedProductType } from '../common/enums';
 import { HelmService } from './helm.service';
 import { RegistryService } from './registry.service';
+import { K8sClientService } from './k8s-client.service';
+import { K8sLifecycleService } from './k8s-lifecycle.service';
 import { registerKubeconfigNoProxy } from '../common/kubernetes-proxy.util';
 
 const execFileAsync = promisify(execFile);
@@ -74,6 +76,8 @@ export class KubernetesService implements OnModuleInit {
     private clustersService: ClustersService,
     private helmService: HelmService,
     private registryService: RegistryService,
+    private k8sClientService: K8sClientService,
+    private k8sLifecycleService: K8sLifecycleService,
     @InjectRepository(Deployment)
     private deploymentsRepository: Repository<Deployment>,
   ) {}
@@ -99,34 +103,6 @@ export class KubernetesService implements OnModuleInit {
     // Helm chart is used for deployments — no local template loading needed
   }
 
-  private async getK8sClient(clusterId?: string): Promise<{
-    coreApi: k8s.CoreV1Api;
-    appsApi: k8s.AppsV1Api;
-    networkingApi: k8s.NetworkingV1Api;
-    kc: k8s.KubeConfig;
-  }> {
-    const cluster = clusterId ? await this.clustersService.findOne(clusterId) : await this.clustersService.getDefault();
-
-    const kc = new k8s.KubeConfig();
-    registerKubeconfigNoProxy(cluster.kubeconfig);
-    kc.loadFromString(cluster.kubeconfig);
-
-    return {
-      coreApi: kc.makeApiClient(k8s.CoreV1Api),
-      appsApi: kc.makeApiClient(k8s.AppsV1Api),
-      networkingApi: kc.makeApiClient(k8s.NetworkingV1Api),
-      kc,
-    };
-  }
-
-  /**
-   * Get the raw kubeconfig string for a cluster.
-   */
-  private async getKubeconfig(clusterId?: string): Promise<string> {
-    const cluster = clusterId ? await this.clustersService.findOne(clusterId) : await this.clustersService.getDefault();
-    return cluster.kubeconfig;
-  }
-
   /**
    * Build Helm values object from an Application entity and image URI.
    */
@@ -412,7 +388,7 @@ export class KubernetesService implements OnModuleInit {
   }
 
   async waitForApplicationReady(app: Application, timeoutMs = 600_000, shouldAbort?: () => Promise<boolean>): Promise<void> {
-    const { coreApi, appsApi } = await this.getK8sClient(app.clusterId);
+    const { coreApi, appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
     const namespace = `user-${app.userId.split('-')[0]}`;
     const managed = isManagedProductType(app.productType);
     const workloads = [
@@ -463,14 +439,14 @@ export class KubernetesService implements OnModuleInit {
     const previewNumber = app.customDomain ? null : await this.resolvePreviewNumber(app.id);
 
     try {
-      const kubeconfig = await this.getKubeconfig(app.clusterId);
+      const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
       const imageUri = app.latestImageTag ? this.registryService.normalizeImageReference(app.latestImageTag) : '';
       const values = this.buildHelmValues(app, imageUri, previewNumber);
       await this.helmService.installOrUpgrade(app.name, namespace, values, kubeconfig);
       this.logger.log(`Updated ingress for ${app.name} via Helm (customDomain: ${customDomain || 'none'})`);
     } catch (helmError: any) {
       this.logger.warn(`Helm ingress update failed for ${app.name}, using direct K8s API: ${helmError.message}`);
-      const { networkingApi } = await this.getK8sClient(app.clusterId);
+      const { networkingApi } = await this.k8sClientService.getK8sClient(app.clusterId);
       const ctx: ManifestContext = {
         appName: app.name,
         namespace,
@@ -513,9 +489,9 @@ export class KubernetesService implements OnModuleInit {
   // ── Helm-based deployment ─────────────────────────────────────────
 
   private async deployViaHelm(app: Application, imageUri: string, previewNumber?: string | null): Promise<Record<string, any>> {
-    const kubeconfig = await this.getKubeconfig(app.clusterId);
+    const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
     await this.ensurePlatformStorageClass(kubeconfig);
-    const { coreApi } = await this.getK8sClient(app.clusterId);
+    const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
     const values = this.buildHelmValues(app, imageUri, previewNumber);
     const namespace = values.app.namespace as string;
     await this.registryService.ensureRegistryPullSecret(coreApi, namespace);
@@ -531,7 +507,7 @@ export class KubernetesService implements OnModuleInit {
   }
 
   private async deployManagedViaHelm(app: Application): Promise<Record<string, any>> {
-    const kubeconfig = await this.getKubeconfig(app.clusterId);
+    const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
     await this.ensurePlatformStorageClass(kubeconfig);
     const values = this.buildManagedHelmValues(app);
     const namespace = values.app.namespace;
@@ -549,8 +525,8 @@ export class KubernetesService implements OnModuleInit {
   // ── Direct K8s API deployment (fallback) ──────────────────────────
 
   private async deployManagedViaK8sApi(app: Application): Promise<Record<string, any>> {
-    const { coreApi, appsApi } = await this.getK8sClient(app.clusterId);
-    const kubeconfig = await this.getKubeconfig(app.clusterId);
+    const { coreApi, appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
+    const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
     await this.ensurePlatformStorageClass(kubeconfig);
     const namespace = `user-${app.userId.split('-')[0]}`;
     const context: ManifestContext = {
@@ -619,8 +595,8 @@ export class KubernetesService implements OnModuleInit {
     if (isManagedProductType(app.productType)) {
       return this.deployManagedViaK8sApi(app);
     }
-    const { coreApi, appsApi, networkingApi } = await this.getK8sClient(app.clusterId);
-    const kubeconfig = await this.getKubeconfig(app.clusterId);
+    const { coreApi, appsApi, networkingApi } = await this.k8sClientService.getK8sClient(app.clusterId);
+    const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
     await this.ensurePlatformStorageClass(kubeconfig);
     const domain = this.configService.get('platform.domain');
 
@@ -2247,33 +2223,11 @@ export class KubernetesService implements OnModuleInit {
   }
 
   async getPodLogs(app: Application): Promise<string> {
-    const { coreApi } = await this.getK8sClient(app.clusterId);
-    const namespace = `user-${app.userId.split('-')[0]}`;
-    const podLabel = this.primaryWorkloadLabel(app);
-
-    const pods = await coreApi.listNamespacedPod({
-      namespace,
-      labelSelector: `app=${podLabel}`,
-    });
-
-    if (pods.items.length === 0) {
-      return 'No pods found for this application.';
-    }
-
-    const podName = pods.items[0].metadata?.name;
-    if (!podName) return 'Pod name not found.';
-
-    const logResponse = await coreApi.readNamespacedPodLog({
-      name: podName,
-      namespace,
-      tailLines: 200,
-    });
-
-    return logResponse;
+    return this.k8sLifecycleService.getPodLogs(app);
   }
 
   async scaleDeployment(app: Application, replicas: number): Promise<void> {
-    const { appsApi } = await this.getK8sClient(app.clusterId);
+    const { appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
     const namespace = `user-${app.userId.split('-')[0]}`;
 
     await appsApi.patchNamespacedDeployment({ name: app.name, namespace, body: { spec: { replicas } } }, k8s.setHeaderOptions('Content-Type', 'application/merge-patch+json'));
@@ -2313,7 +2267,7 @@ export class KubernetesService implements OnModuleInit {
   }
 
   async captureWorkloadReplicaSnapshot(app: Application): Promise<Record<string, number>> {
-    const { appsApi } = await this.getK8sClient(app.clusterId);
+    const { appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
     const namespace = `user-${app.userId.split('-')[0]}`;
     const snapshot: Record<string, number> = {};
 
@@ -2342,7 +2296,7 @@ export class KubernetesService implements OnModuleInit {
    * Returns the replica snapshot captured before scaling.
    */
   async suspendApplication(app: Application): Promise<Record<string, number>> {
-    const { appsApi } = await this.getK8sClient(app.clusterId);
+    const { appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
     const namespace = `user-${app.userId.split('-')[0]}`;
 
     this.logger.log(`Suspending application ${app.name} in namespace ${namespace}`);
@@ -2368,7 +2322,7 @@ export class KubernetesService implements OnModuleInit {
    * Resume a suspended application using saved replica counts when available.
    */
   async resumeApplication(app: Application): Promise<void> {
-    const { appsApi } = await this.getK8sClient(app.clusterId);
+    const { appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
     const namespace = `user-${app.userId.split('-')[0]}`;
 
     this.logger.log(`Resuming application ${app.name} in namespace ${namespace}`);
@@ -2400,7 +2354,7 @@ export class KubernetesService implements OnModuleInit {
   }
 
   async restartDeployment(app: Application): Promise<void> {
-    const { appsApi } = await this.getK8sClient(app.clusterId);
+    const { appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
     const namespace = `user-${app.userId.split('-')[0]}`;
     const deploymentName = isManagedProductType(app.productType) ? this.primaryWorkloadLabel(app) : app.name;
 
@@ -2595,7 +2549,7 @@ export class KubernetesService implements OnModuleInit {
    * Includes application workload, database, and optional Redis / RabbitMQ when enabled.
    */
   async getResourceUsage(app: Application): Promise<any> {
-    const { coreApi, appsApi, kc } = await this.getK8sClient(app.clusterId);
+    const { coreApi, appsApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
     const namespace = `user-${app.userId.split('-')[0]}`;
 
     const workloads: any[] = [];
@@ -2685,7 +2639,7 @@ export class KubernetesService implements OnModuleInit {
     },
     workload: 'app' | 'database' | 'redis' | 'rabbitmq' = 'app',
   ): Promise<void> {
-    const { appsApi } = await this.getK8sClient(app.clusterId);
+    const { appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
     const namespace = `user-${app.userId.split('-')[0]}`;
 
     const target = this.workloadDeploymentTarget(app, workload);
@@ -2807,7 +2761,7 @@ export class KubernetesService implements OnModuleInit {
       throw new BadRequestException('Application is not assigned to a cluster');
     }
 
-    const { coreApi, kc } = await this.getK8sClient(app.clusterId);
+    const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
     const namespace = this.getUserNamespace(app.userId);
     const { selector, targetPort, portName } = this.resolveAccessTarget(app, target);
     const shortId = grantId.split('-')[0];
@@ -2874,7 +2828,7 @@ export class KubernetesService implements OnModuleInit {
     const manifestPath = path.join(tmpDir, 'service.json');
 
     try {
-      fs.writeFileSync(kubeconfigPath, await this.getKubeconfig(clusterId), {
+      fs.writeFileSync(kubeconfigPath, await this.k8sClientService.getKubeconfig(clusterId), {
         mode: 0o600,
       });
       fs.writeFileSync(manifestPath, JSON.stringify(service), { mode: 0o600 });
@@ -2895,7 +2849,7 @@ export class KubernetesService implements OnModuleInit {
   }
 
   async revokeTemporaryAccess(clusterId: string, namespace: string, k8sServiceName: string): Promise<void> {
-    const { coreApi } = await this.getK8sClient(clusterId);
+    const { coreApi } = await this.k8sClientService.getK8sClient(clusterId);
     try {
       await coreApi.deleteNamespacedService({
         name: k8sServiceName,
@@ -2912,7 +2866,7 @@ export class KubernetesService implements OnModuleInit {
   async deleteTemporaryAccessServicesForApp(app: Application): Promise<void> {
     if (!app.clusterId) return;
     const namespace = this.getUserNamespace(app.userId);
-    const { coreApi } = await this.getK8sClient(app.clusterId);
+    const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
 
     try {
       const services = await coreApi.listNamespacedService({
@@ -2942,7 +2896,7 @@ export class KubernetesService implements OnModuleInit {
         };
       case ServiceAccessTarget.REDIS: {
         if (!app.clusterId) return {};
-        const { coreApi } = await this.getK8sClient(app.clusterId);
+        const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
         const secret = await coreApi.readNamespacedSecret({
           name: `${app.name}-redis-secret`,
           namespace,
@@ -2953,7 +2907,7 @@ export class KubernetesService implements OnModuleInit {
       case ServiceAccessTarget.RABBITMQ_AMQP:
       case ServiceAccessTarget.RABBITMQ_MANAGEMENT: {
         if (!app.clusterId) return {};
-        const { coreApi } = await this.getK8sClient(app.clusterId);
+        const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
         const secret = await coreApi.readNamespacedSecret({
           name: `${app.name}-rabbitmq-secret`,
           namespace,
@@ -2980,7 +2934,7 @@ export class KubernetesService implements OnModuleInit {
     host: string;
     ingressUrl?: string;
   }> {
-    const { coreApi, networkingApi, kc } = await this.getK8sClient(app.clusterId);
+    const { coreApi, networkingApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
     const namespace = this.getUserNamespace(app.userId);
     const domain = this.configService.get('platform.domain');
     const hostIp = this.getClusterHostIp(kc);
@@ -3044,13 +2998,13 @@ export class KubernetesService implements OnModuleInit {
 
   async deleteApplication(app: Application): Promise<void> {
     const namespace = this.getUserNamespace(app.userId);
-    const { coreApi, appsApi, networkingApi } = await this.getK8sClient(app.clusterId);
+    const { coreApi, appsApi, networkingApi } = await this.k8sClientService.getK8sClient(app.clusterId);
 
     await this.deleteTemporaryAccessServicesForApp(app);
 
     // Step 1: Try Helm uninstall (handles most resources)
     try {
-      const kubeconfig = await this.getKubeconfig(app.clusterId);
+      const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
       await this.helmService.uninstall(app.name, namespace, kubeconfig);
       this.logger.log(`Helm release ${app.name} uninstalled from ${namespace}`);
     } catch (error: any) {
@@ -3171,8 +3125,8 @@ export class KubernetesService implements OnModuleInit {
     } = {},
   ): Promise<void> {
     const namespace = this.getUserNamespace(app.userId);
-    const source = await this.getK8sClient(app.clusterId);
-    const target = await this.getK8sClient(targetClusterId);
+    const source = await this.k8sClientService.getK8sClient(app.clusterId);
+    const target = await this.k8sClientService.getK8sClient(targetClusterId);
 
     await this.ensureNamespaceOnCluster(target.coreApi, namespace);
     await options.log?.('transfer-secrets-configs', 'Target namespace ensured', { namespace });
@@ -3250,8 +3204,8 @@ export class KubernetesService implements OnModuleInit {
     const targetKubeconfig = path.join(tempDir, 'target.kubeconfig');
 
     fs.mkdirSync(tempDir, { recursive: true });
-    fs.writeFileSync(sourceKubeconfig, await this.getKubeconfig(sourceClusterId), { mode: 0o600 });
-    fs.writeFileSync(targetKubeconfig, await this.getKubeconfig(targetClusterId), { mode: 0o600 });
+    fs.writeFileSync(sourceKubeconfig, await this.k8sClientService.getKubeconfig(sourceClusterId), { mode: 0o600 });
+    fs.writeFileSync(targetKubeconfig, await this.k8sClientService.getKubeconfig(targetClusterId), { mode: 0o600 });
 
     try {
       await this.createPvcCopyPod(sourceKubeconfig, namespace, sourcePod, pvcName);
@@ -3407,7 +3361,7 @@ export class KubernetesService implements OnModuleInit {
    * Polls pod status with label selector `app=<appName>-db`.
    */
   async waitForDatabaseReady(app: Application, timeoutMs = 120_000): Promise<void> {
-    const { coreApi } = await this.getK8sClient(app.clusterId);
+    const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
     const namespace = `user-${app.userId.split('-')[0]}`;
     const dbLabel = `${app.name}-db`;
     const start = Date.now();
@@ -3513,7 +3467,7 @@ export class KubernetesService implements OnModuleInit {
    * then runs a restore Job that mounts the PVC and imports the dump.
    */
   async restoreDatabaseDump(app: Application, dumpFilePath: string): Promise<{ success: boolean; logs: string }> {
-    const { coreApi, kc } = await this.getK8sClient(app.clusterId);
+    const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
     const batchApi = kc.makeApiClient(k8s.BatchV1Api);
     const namespace = `user-${app.userId.split('-')[0]}`;
     const dbName = `${app.name}-db`;
@@ -3821,7 +3775,7 @@ export class KubernetesService implements OnModuleInit {
    * Used when legacy PVCs were created without storageClassName.
    */
   private async migrateDatabasePvcToResizableStorage(app: Application, newSize: string, storageClassName: string): Promise<{ success: boolean; message: string }> {
-    const { coreApi, appsApi, kc } = await this.getK8sClient(app.clusterId);
+    const { coreApi, appsApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
     const batchApi = kc.makeApiClient(k8s.BatchV1Api);
     const namespace = `user-${app.userId.split('-')[0]}`;
     const oldPvcName = `${app.name}-db`;
@@ -3997,7 +3951,7 @@ export class KubernetesService implements OnModuleInit {
    * K8s only supports PVC expansion, not shrinking.
    */
   async resizeDatabasePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
-    const { coreApi, kc } = await this.getK8sClient(app.clusterId);
+    const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
     const namespace = `user-${app.userId.split('-')[0]}`;
     const pvcName = `${app.name}-db`;
 
@@ -4070,7 +4024,7 @@ export class KubernetesService implements OnModuleInit {
    */
   async getDatabasePvcSize(app: Application): Promise<string> {
     try {
-      const { coreApi } = await this.getK8sClient(app.clusterId);
+      const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
       const namespace = `user-${app.userId.split('-')[0]}`;
       const pvcName = `${app.name}-db`;
 
@@ -4096,7 +4050,7 @@ export class KubernetesService implements OnModuleInit {
     totalAllocatedGb: number;
     totalUsedGb: number;
   }> {
-    const { coreApi } = await this.getK8sClient(app.clusterId);
+    const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
     const namespace = `user-${app.userId.split('-')[0]}`;
 
     const result = {
@@ -4271,7 +4225,7 @@ export class KubernetesService implements OnModuleInit {
    * Returns usage in GB.
    */
   private async getPvcUsageFromPod(app: Application, deploymentName: string, mountPath: string, namespace: string, containerName: string): Promise<number> {
-    const { coreApi, kc } = await this.getK8sClient(app.clusterId);
+    const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
 
     const pods = await coreApi.listNamespacedPod({
       namespace,
@@ -4305,7 +4259,7 @@ export class KubernetesService implements OnModuleInit {
    * Expand a named PVC (Redis, RabbitMQ, or other optional service volumes).
    */
   async resizeNamedPvc(app: Application, pvcName: string, newSize: string, label: string): Promise<{ success: boolean; message: string }> {
-    const { coreApi } = await this.getK8sClient(app.clusterId);
+    const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
     const namespace = `user-${app.userId.split('-')[0]}`;
 
     try {
@@ -4362,7 +4316,7 @@ export class KubernetesService implements OnModuleInit {
    * Resize app storage PVC (all app types).
    */
   async resizeAppStoragePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
-    const { coreApi } = await this.getK8sClient(app.clusterId);
+    const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
     const namespace = `user-${app.userId.split('-')[0]}`;
 
     // Try new unified name first, then legacy wp-content name
@@ -4427,7 +4381,7 @@ export class KubernetesService implements OnModuleInit {
    * Strategy: Run dump command, then sleep for 60s to allow exec retrieval.
    */
   async exportDatabaseDump(app: Application, onProgress?: (percent: number) => void): Promise<{ data: Buffer | null; logs: string }> {
-    const { coreApi, kc } = await this.getK8sClient(app.clusterId);
+    const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
     const batchApi = kc.makeApiClient(k8s.BatchV1Api);
     const namespace = `user-${app.userId.split('-')[0]}`;
     const dbName = `${app.name}-db`;
@@ -4608,7 +4562,7 @@ export class KubernetesService implements OnModuleInit {
    * Strategy: Create archive, then sleep to allow exec retrieval.
    */
   async archiveWpContent(app: Application): Promise<{ data: Buffer | null; logs: string }> {
-    const { coreApi, kc } = await this.getK8sClient(app.clusterId);
+    const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
     const batchApi = kc.makeApiClient(k8s.BatchV1Api);
     const namespace = `user-${app.userId.split('-')[0]}`;
     const pvcName = `${app.name}-storage`;
@@ -4760,7 +4714,7 @@ export class KubernetesService implements OnModuleInit {
    * Restore wp-content from a tar.gz archive into the WordPress PVC.
    */
   async restoreWpContent(app: Application, archiveBuffer: Buffer): Promise<{ success: boolean; logs: string }> {
-    const { coreApi, kc } = await this.getK8sClient(app.clusterId);
+    const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
     const batchApi = kc.makeApiClient(k8s.BatchV1Api);
     const namespace = `user-${app.userId.split('-')[0]}`;
     const pvcName = `${app.name}-storage`;
@@ -4902,7 +4856,7 @@ export class KubernetesService implements OnModuleInit {
     const releaseName = app.name;
 
     try {
-      const kubeconfig = await this.getKubeconfig(app.clusterId);
+      const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
       const helmRevisions = await this.helmService.history(releaseName, namespace, kubeconfig);
 
       if (!helmRevisions || helmRevisions.length === 0) {
@@ -4940,7 +4894,7 @@ export class KubernetesService implements OnModuleInit {
     const releaseName = app.name;
 
     try {
-      const kubeconfig = await this.getKubeconfig(app.clusterId);
+      const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
       await this.helmService.rollback(releaseName, targetRevision, namespace, kubeconfig);
       this.logger.log(`Rolled back ${releaseName} to Helm revision ${targetRevision}`);
       return {