import { Injectable, Logger, OnModuleInit, BadRequestException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import * as k8s from '@kubernetes/client-node';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { execFile } from 'child_process';
import { promisify } from 'util';
import { PassThrough } from 'stream';
import { ClustersService } from '../clusters/clusters.service';
import { Application } from '../applications/entities/application.entity';
import { Deployment } from '../deployments/entities/deployment.entity';
import { ensureAppUrlEnv } from '../applications/app-url.util';
import { AppRuntime, DatabaseType, CustomDomainStatus, ServiceAccessTarget, ProductType, isManagedProductType } from '../common/enums';
import { HelmService } from './helm.service';
import { RegistryService } from './registry.service';
import { K8sClientService } from './k8s-client.service';
import { K8sLifecycleService } from './k8s-lifecycle.service';
import { userNamespace, userIdSlug } from './k8s-workload.util';
import { registerKubeconfigNoProxy } from '../common/kubernetes-proxy.util';

const execFileAsync = promisify(execFile);

interface ManifestContext {
  appName: string;
  namespace: string;
  image: string;
  port: number;
  replicas: number;
  cpuRequest: string;
  cpuLimit: string;
  memoryRequest: string;
  memoryLimit: string;
  envVars: Record<string, string>;
  runtime: string;
  databaseType: string;
  domain: string;
  subdomain: string;
  dbUsername: string;
  dbPassword: string;
  dbVersion: string;
  dbStorageSize: string;
  dbCpuRequest: string;
  dbCpuLimit: string;
  dbMemoryRequest: string;
  dbMemoryLimit: string;
  appStorageSize: string;
  enableRedis: boolean;
  redisVersion: string;
  enableRabbitmq: boolean;
  rabbitmqVersion: string;
  enableElasticsearch: boolean;
  elasticsearchVersion: string;
  logPaths: string[];
  ownerId: string;
  applicationId: string;
}

type StorageUsageSlice = {
  allocatedRaw: string;
  allocatedGi: number;
  usedGi: number;
  availableGi: number;
  usedPercent: number;
  allocated: string;
  used: string;
  available: string;
};

@Injectable()
export class KubernetesService implements OnModuleInit {
  private readonly logger = new Logger(KubernetesService.name);

  constructor(
    private configService: ConfigService,
    private clustersService: ClustersService,
    private helmService: HelmService,
    private registryService: RegistryService,
    private k8sClientService: K8sClientService,
    private k8sLifecycleService: K8sLifecycleService,
    @InjectRepository(Deployment)
    private deploymentsRepository: Repository<Deployment>,
  ) {}

  /**
   * Stable per-app preview number, reused from the latest deployment that has
   * one. Used to restore the preview host on the Ingress (e.g. after a custom
   * domain is removed) without triggering a fresh deploy.
   */
  private async resolvePreviewNumber(applicationId: string): Promise<string | null> {
    const existing = await this.deploymentsRepository
      .createQueryBuilder('d')
      .select('d.previewSubdomain', 'previewSubdomain')
      .where('d.applicationId = :applicationId', { applicationId })
      .andWhere('d.previewSubdomain IS NOT NULL')
      .orderBy('d.createdAt', 'DESC')
      .limit(1)
      .getRawOne<{ previewSubdomain: string }>();
    return existing?.previewSubdomain || null;
  }

  onModuleInit() {
    // Helm chart is used for deployments — no local template loading needed
  }

  /**
   * Build Helm values object from an Application entity and image URI.
   */
  /**
   * Resolve the database workload's CPU/RAM. Prefers the user-selected
   * per-database resources (optionalServiceResources.database); falls back to
   * the app's main resources for managed databases / legacy apps.
   */
  private resolveDatabaseResources(app: Application) {
    const res = app.optionalServiceResources?.database;
    return {
      cpuRequest: res?.cpuRequest || app.cpuRequest || '100m',
      cpuLimit: res?.cpuLimit || app.cpuLimit || '500m',
      memoryRequest: res?.memoryRequest || app.memoryRequest || '256Mi',
      memoryLimit: res?.memoryLimit || app.memoryLimit || '512Mi',
    };
  }

  private buildRedisHelmBlock(app: Application) {
    const res = app.optionalServiceResources?.redis;
    const storageGi = res?.storageGi ?? 1;
    return {
      enabled: app.enableRedis || false,
      version: app.redisVersion || '7.2',
      storageSize: `${storageGi}Gi`,
      resources: {
        cpuRequest: res?.cpuRequest || '50m',
        cpuLimit: res?.cpuLimit || '200m',
        memoryRequest: res?.memoryRequest || '64Mi',
        memoryLimit: res?.memoryLimit || '256Mi',
      },
    };
  }

  private buildRabbitmqHelmBlock(app: Application) {
    const res = app.optionalServiceResources?.rabbitmq;
    const storageGi = res?.storageGi ?? 2;
    return {
      enabled: app.enableRabbitmq || false,
      version: app.rabbitmqVersion || '3.13',
      storageSize: `${storageGi}Gi`,
      resources: {
        cpuRequest: res?.cpuRequest || '100m',
        cpuLimit: res?.cpuLimit || '500m',
        memoryRequest: res?.memoryRequest || '256Mi',
        memoryLimit: res?.memoryLimit || '512Mi',
      },
    };
  }

  private resolveEnvVars(app: Application): Record<string, string> {
    const platformDomain = this.configService.get('platform.domain') || 'apps.cloudhost.ir';
    return ensureAppUrlEnv(app, platformDomain);
  }

  private helmGlobalStorageValues(): Record<string, unknown> {
    const storageClass = this.configService.get<string>('platform.storageClass') || '';
    const createStorageClass = this.configService.get<boolean>('platform.createStorageClass') === true;
    const storageProvisioner = this.configService.get<string>('platform.storageProvisioner') || 'rancher.io/local-path';
    if (!storageClass) {
      return {
        storageClass: '',
        createStorageClass: false,
        storageProvisioner,
      };
    }
    return { storageClass, createStorageClass, storageProvisioner };
  }

  /** Create platform StorageClass on the target cluster when configured (Helm chart + K8s API fallback). */
  private async ensurePlatformStorageClass(kubeconfig: string): Promise<void> {
    const storageClass = this.configService.get<string>('platform.storageClass')?.trim();
    const createStorageClass = this.configService.get<boolean>('platform.createStorageClass') === true;
    if (!storageClass || !createStorageClass) {
      return;
    }

    const kc = new k8s.KubeConfig();
    kc.loadFromString(kubeconfig);
    const storageApi = kc.makeApiClient(k8s.StorageV1Api);
    const provisioner = this.configService.get<string>('platform.storageProvisioner') || 'rancher.io/local-path';

    try {
      await storageApi.readStorageClass({ name: storageClass });
      return;
    } catch (err: any) {
      if (err.code !== 404 && err.body?.code !== 404) {
        throw err;
      }
    }

    await storageApi.createStorageClass({
      body: {
        apiVersion: 'storage.k8s.io/v1',
        kind: 'StorageClass',
        metadata: { name: storageClass },
        provisioner,
        allowVolumeExpansion: true,
        reclaimPolicy: 'Delete',
        volumeBindingMode: 'WaitForFirstConsumer',
      },
    });
    this.logger.log(`Created StorageClass "${storageClass}" (provisioner: ${provisioner})`);
  }

  /** Helm values for managed_database / managed_redis / managed_rabbitmq (no app workload). */
  /**
   * Return the app's database password, generating and PERSISTING one if it is
   * missing. Without persistence a fresh password would be generated on every
   * helm upgrade, breaking auth against the database's persisted volume.
   */
  private ensureDbPassword(app: Application): string {
    if (!app.dbPassword) {
      app.dbPassword = this.generatePassword();
      this.deploymentsRepository.manager
        .getRepository(Application)
        .update(app.id, { dbPassword: app.dbPassword })
        .catch((e: any) =>
          this.logger.warn(`Failed to persist generated dbPassword for ${app.name}: ${e.message}`),
        );
      this.logger.warn(`App ${app.name} had no dbPassword — generated and persisted one`);
    }
    return app.dbPassword;
  }

  private buildManagedHelmValues(app: Application): Record<string, any> {
    const namespace = this.getUserNamespace(app.userId);
    const pullRegistryUrl = this.registryService.getRegistryUrl();
    const isPostgres = app.databaseType === DatabaseType.POSTGRESQL;
    const productType = app.productType;

    const values: Record<string, any> = {
      global: this.helmGlobalStorageValues(),
      app: {
        enabled: false,
        name: app.name,
        namespace,
        runtime: app.runtime,
        image: '',
        port: app.port || 3000,
        replicas: 0,
        storageSize: app.appStorageSize || '2Gi',
      },
      resources: {
        cpuRequest: app.cpuRequest,
        cpuLimit: app.cpuLimit,
        memoryRequest: app.memoryRequest,
        memoryLimit: app.memoryLimit,
      },
      envVars: {},
      ingress: {
        enabled: false,
        subdomain: app.subdomain || app.name,
        domain: this.configService.get('platform.domain'),
        clusterIssuer: 'letsencrypt-prod',
        customDomain: '',
      },
      registry: { url: pullRegistryUrl },
      database: {
        enabled: false,
        type: app.databaseType,
        version: app.dbVersion || (isPostgres ? '16' : '8.0'),
        username: app.dbUsername || 'appuser',
        password:
          app.databaseType && app.databaseType !== DatabaseType.NONE
            ? this.ensureDbPassword(app)
            : '',
        storageSize: app.dbStorageSize || '1Gi',
        resources: this.resolveDatabaseResources(app),
      },
      redis: { enabled: false, storageSize: '1Gi', resources: {} },
      rabbitmq: { enabled: false, storageSize: '2Gi', resources: {} },
      wordpress: { enabled: false },
      elasticsearch: {
        enabled: false,
        logPaths: [],
        ownerId: app.userId,
        applicationId: app.id,
      },
      images: { baseRegistry: this.configService.get<string>('build.baseImageRegistry') || '' },
      changeCause: `Helm provision ${app.name} (${productType}) at ${new Date().toISOString()}`,
    };

    switch (productType) {
      case ProductType.MANAGED_DATABASE:
        values.database.enabled = true;
        values.redis.enabled = false;
        values.rabbitmq.enabled = false;
        break;
      case ProductType.MANAGED_REDIS:
        values.redis = this.buildRedisHelmBlock(app);
        values.redis.enabled = true;
        break;
      case ProductType.MANAGED_RABBITMQ:
        values.rabbitmq = this.buildRabbitmqHelmBlock(app);
        values.rabbitmq.enabled = true;
        break;
      default:
        break;
    }

    return values;
  }

  private buildHelmValues(app: Application, imageUri: string, previewNumber?: string | null): Record<string, any> {
    const domain = this.configService.get('platform.domain');
    const previewRootDomain = this.configService.get<string>('platform.previewRootDomain') || domain;
    const namespacePrefix = userIdSlug(app.userId);
    const previewHost = previewNumber && !this.hasVerifiedCustomDomain(app)
      ? `${namespacePrefix}-${previewNumber}.${previewRootDomain}`
      : '';
    const pullRegistryUrl = this.registryService.getRegistryUrl();
    const isWordPress = app.runtime === AppRuntime.WORDPRESS;
    const hasDb = app.databaseType !== DatabaseType.NONE;
    const isPostgres = app.databaseType === DatabaseType.POSTGRESQL;

    const values: Record<string, any> = {
      global: this.helmGlobalStorageValues(),
      app: {
        enabled: true,
        name: app.name,
        namespace: this.getUserNamespace(app.userId),
        runtime: app.runtime,
        image: imageUri,
        port: app.port,
        replicas: app.replicas || 1,
        storageSize: app.appStorageSize || '2Gi',
      },
      resources: {
        cpuRequest: app.cpuRequest,
        cpuLimit: app.cpuLimit,
        memoryRequest: app.memoryRequest,
        memoryLimit: app.memoryLimit,
      },
      envVars: this.resolveEnvVars(app),
      ingress: {
        enabled: true,
        className: this.configService.get<string>('platform.ingressClass') || 'traefik',
        subdomain: app.subdomain || app.name,
        domain: domain,
        clusterIssuer: 'letsencrypt-prod',
        customDomain: app.customDomain && app.customDomainStatus === CustomDomainStatus.VERIFIED ? app.customDomain : '',
        previewHost,
      },
      registry: {
        url: pullRegistryUrl,
      },
      database: {
        enabled: hasDb,
        type: app.databaseType,
        version: app.dbVersion || (isPostgres ? '16' : '8.0'),
        username: app.dbUsername || 'appuser',
        password: hasDb ? this.ensureDbPassword(app) : '',
        storageSize: app.dbStorageSize || '1Gi',
        resources: this.resolveDatabaseResources(app),
      },
      wordpress: {
        enabled: isWordPress,
      },
      redis: this.buildRedisHelmBlock(app),
      rabbitmq: this.buildRabbitmqHelmBlock(app),
      elasticsearch: {
        enabled: app.enableElasticsearch || false,
        logPaths: app.logPaths || [],
        ownerId: app.userId,
        applicationId: app.id,
        elasticPassword: this.configService.get<string>('elasticsearch.password'),
        fluentbitPassword: this.configService.get<string>('elasticsearch.fluentbitPassword'),
        kibanaPassword: this.configService.get<string>('elasticsearch.kibanaPassword'),
      },
      images: { baseRegistry: this.configService.get<string>('build.baseImageRegistry') || '' },
      changeCause: `Deploy ${imageUri} at ${new Date().toISOString()}`,
    };

    return values;
  }

  /** Install or upgrade only the workload for a managed service (database, Redis, or RabbitMQ). */
  async deployManagedService(app: Application): Promise<Record<string, any>> {
    if (!isManagedProductType(app.productType)) {
      throw new BadRequestException('deployManagedService requires a managed product type');
    }
    try {
      return await this.deployManagedViaHelm(app);
    } catch (helmError: any) {
      this.logger.warn(`Helm provision failed for managed ${app.name}, falling back to direct K8s API: ${helmError.message}`);
      return await this.deployManagedViaK8sApi(app);
    }
  }

  async deployApplication(app: Application, imageUri: string, opts?: { previewNumber?: string | null }): Promise<Record<string, any>> {
    if (isManagedProductType(app.productType)) {
      return this.deployManagedService(app);
    }

    const previewNumber = opts?.previewNumber ?? null;
    const workloadImage = this.registryService.normalizeImageReference(imageUri);
    if (workloadImage !== imageUri) {
      this.logger.log(`Using in-cluster registry image for ${app.name}: ${workloadImage}`);
    }

    // Try Helm first, fall back to direct K8s API if Helm is unavailable
    try {
      return await this.deployViaHelm(app, workloadImage, previewNumber);
    } catch (helmError: any) {
      this.logger.warn(`Helm deploy failed for ${app.name}, falling back to direct K8s API: ${helmError.message}`);
      return await this.deployViaK8sApi(app, workloadImage, previewNumber);
    }
  }

  async waitForApplicationReady(app: Application, timeoutMs = 600_000, shouldAbort?: () => Promise<boolean>): Promise<void> {
    const { coreApi, appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
    const namespace = this.getUserNamespace(app.userId);
    const managed = isManagedProductType(app.productType);
    const workloads = [
      ...(!managed ? [{ name: app.name, replicas: app.replicas || 1 }] : []),
      ...(app.databaseType !== DatabaseType.NONE ? [{ name: `${app.name}-db`, replicas: 1 }] : []),
      ...(app.enableRedis ? [{ name: `${app.name}-redis`, replicas: 1 }] : []),
      ...(app.enableRabbitmq ? [{ name: `${app.name}-rabbitmq`, replicas: 1 }] : []),
    ];
    const start = Date.now();
    let lastSummary = '';

    this.logger.log(`Waiting for ${app.name} workloads to become Ready in ${namespace}: ${workloads.map((w) => w.name).join(', ')}`);

    while (Date.now() - start < timeoutMs) {
      if (await shouldAbort?.()) {
        throw new Error('Deployment cancelled by user');
      }

      const statuses = await Promise.all(workloads.map((workload) => this.getDeploymentReadiness(appsApi, namespace, workload.name, workload.replicas)));
      lastSummary = statuses.map((s) => `${s.name} ${s.readyReplicas}/${s.desiredReplicas}`).join(', ');

      if (statuses.every((s) => s.ready)) {
        this.logger.log(`All workloads for ${app.name} are Ready (${lastSummary})`);
        return;
      }

      await new Promise((resolve) => setTimeout(resolve, 5000));
    }

    const podSummary = await this.describeWorkloadPods(
      coreApi,
      namespace,
      workloads.map((w) => w.name),
    );
    throw new Error(`Application workloads did not become ready within ${Math.round(timeoutMs / 1000)}s. ` + `Readiness: ${lastSummary || 'no deployment status available'}. ${podSummary}`);
  }

  async updateIngress(app: Application): Promise<void> {
    const domain = this.configService.get('platform.domain');
    const subdomain = app.subdomain || app.name;
    const namespace = this.getUserNamespace(app.userId);
    const customDomain = app.customDomain && app.customDomainStatus === CustomDomainStatus.VERIFIED ? app.customDomain : undefined;

    // When there's no verified custom domain, restore the stable preview host so
    // the app stays reachable (and keeps its TLS) — e.g. after a custom domain is
    // removed. buildHelmValues only emits a preview host when there's no
    // app.customDomain at all, so we only resolve it when none is set.
    const previewNumber = app.customDomain ? null : await this.resolvePreviewNumber(app.id);

    try {
      const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
      const imageUri = app.latestImageTag ? this.registryService.normalizeImageReference(app.latestImageTag) : '';
      const values = this.buildHelmValues(app, imageUri, previewNumber);
      await this.helmService.installOrUpgrade(app.name, namespace, values, kubeconfig);
      this.logger.log(`Updated ingress for ${app.name} via Helm (customDomain: ${customDomain || 'none'})`);
    } catch (helmError: any) {
      this.logger.warn(`Helm ingress update failed for ${app.name}, using direct K8s API: ${helmError.message}`);
      const { networkingApi } = await this.k8sClientService.getK8sClient(app.clusterId);
      const ctx: ManifestContext = {
        appName: app.name,
        namespace,
        image: '',
        port: app.port,
        replicas: app.replicas,
        cpuRequest: app.cpuRequest,
        cpuLimit: app.cpuLimit,
        memoryRequest: app.memoryRequest,
        memoryLimit: app.memoryLimit,
        envVars: this.resolveEnvVars(app),
        runtime: app.runtime,
        databaseType: app.databaseType,
        domain,
        subdomain,
        dbUsername: app.dbUsername || '',
        dbPassword: app.dbPassword || '',
        dbVersion: app.dbVersion || '',
        dbStorageSize: app.dbStorageSize || '1Gi',
        dbCpuRequest: this.resolveDatabaseResources(app).cpuRequest,
        dbCpuLimit: this.resolveDatabaseResources(app).cpuLimit,
        dbMemoryRequest: this.resolveDatabaseResources(app).memoryRequest,
        dbMemoryLimit: this.resolveDatabaseResources(app).memoryLimit,
        appStorageSize: app.appStorageSize || '2Gi',
        enableRedis: app.enableRedis || false,
        redisVersion: app.redisVersion || '7.2',
        enableRabbitmq: app.enableRabbitmq || false,
        rabbitmqVersion: app.rabbitmqVersion || '3.13',
        enableElasticsearch: app.enableElasticsearch || false,
        elasticsearchVersion: app.elasticsearchVersion || '8.12',
        logPaths: app.logPaths || [],
        ownerId: app.userId,
        applicationId: app.id,
      };
      await this.applyIngress(networkingApi, ctx, customDomain, previewNumber);
      this.logger.log(`Updated ingress for ${app.name} via K8s API (customDomain: ${customDomain || 'none'})`);
    }
  }

  // ── Helm-based deployment ─────────────────────────────────────────

  private async deployViaHelm(app: Application, imageUri: string, previewNumber?: string | null): Promise<Record<string, any>> {
    const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
    await this.ensurePlatformStorageClass(kubeconfig);
    const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
    const values = this.buildHelmValues(app, imageUri, previewNumber);
    const namespace = values.app.namespace as string;
    await this.registryService.ensureRegistryPullSecret(coreApi, namespace);
    const releaseName = app.name;

    const result = await this.helmService.installOrUpgrade(releaseName, namespace, values, kubeconfig);

    this.logger.log(`Successfully deployed ${app.name} to namespace ${namespace} via Helm`);
    return {
      helm: { release: releaseName, namespace, stdout: result.stdout },
      values,
    };
  }

  private async deployManagedViaHelm(app: Application): Promise<Record<string, any>> {
    const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
    await this.ensurePlatformStorageClass(kubeconfig);
    const values = this.buildManagedHelmValues(app);
    const namespace = values.app.namespace;
    const releaseName = app.name;

    const result = await this.helmService.installOrUpgrade(releaseName, namespace, values, kubeconfig);

    this.logger.log(`Successfully provisioned managed service ${app.name} in ${namespace} via Helm`);
    return {
      helm: { release: releaseName, namespace, stdout: result.stdout },
      values,
    };
  }

  // ── Direct K8s API deployment (fallback) ──────────────────────────

  private async deployManagedViaK8sApi(app: Application): Promise<Record<string, any>> {
    const { coreApi, appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
    const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
    await this.ensurePlatformStorageClass(kubeconfig);
    const namespace = this.getUserNamespace(app.userId);
    const context: ManifestContext = {
      appName: app.name,
      namespace,
      image: '',
      port: app.port || 3000,
      replicas: 0,
      cpuRequest: app.cpuRequest,
      cpuLimit: app.cpuLimit,
      memoryRequest: app.memoryRequest,
      memoryLimit: app.memoryLimit,
      envVars: {},
      runtime: app.runtime,
      databaseType: app.databaseType,
      domain: this.configService.get('platform.domain') || 'apps.cloudhost.ir',
      subdomain: app.subdomain || app.name,
      dbUsername: app.dbUsername || 'appuser',
      dbPassword:
        app.databaseType && app.databaseType !== DatabaseType.NONE
          ? this.ensureDbPassword(app)
          : '',
      dbVersion: app.dbVersion || '',
      dbStorageSize: app.dbStorageSize || '1Gi',
      dbCpuRequest: this.resolveDatabaseResources(app).cpuRequest,
      dbCpuLimit: this.resolveDatabaseResources(app).cpuLimit,
      dbMemoryRequest: this.resolveDatabaseResources(app).memoryRequest,
      dbMemoryLimit: this.resolveDatabaseResources(app).memoryLimit,
      appStorageSize: app.appStorageSize || '2Gi',
      enableRedis: false,
      redisVersion: app.redisVersion || '7.2',
      enableRabbitmq: false,
      rabbitmqVersion: app.rabbitmqVersion || '3.13',
      enableElasticsearch: false,
      elasticsearchVersion: app.elasticsearchVersion || '8.12',
      logPaths: [],
      ownerId: app.userId,
      applicationId: app.id,
    };

    const manifests: Record<string, any> = {};

    await this.ensureNamespace(coreApi, namespace);

    switch (app.productType) {
      case ProductType.MANAGED_DATABASE:
        context.databaseType = app.databaseType;
        manifests.database = await this.deployDatabase(coreApi, appsApi, context);
        break;
      case ProductType.MANAGED_REDIS:
        context.enableRedis = true;
        await this.deployRedis(coreApi, appsApi, context);
        manifests.redis = true;
        break;
      case ProductType.MANAGED_RABBITMQ:
        context.enableRabbitmq = true;
        await this.deployRabbitmq(coreApi, appsApi, context);
        manifests.rabbitmq = true;
        break;
      default:
        throw new BadRequestException(`Unsupported managed product type: ${app.productType}`);
    }

    this.logger.log(`Provisioned managed service ${app.name} in ${namespace} via K8s API`);
    return manifests;
  }

  private async deployViaK8sApi(app: Application, imageUri: string, previewNumber?: string | null): Promise<Record<string, any>> {
    if (isManagedProductType(app.productType)) {
      return this.deployManagedViaK8sApi(app);
    }
    const { coreApi, appsApi, networkingApi } = await this.k8sClientService.getK8sClient(app.clusterId);
    const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
    await this.ensurePlatformStorageClass(kubeconfig);
    const domain = this.configService.get('platform.domain');

    const context: ManifestContext = {
      appName: app.name,
      namespace: this.getUserNamespace(app.userId),
      image: imageUri,
      port: app.port,
      replicas: app.replicas,
      cpuRequest: app.cpuRequest,
      cpuLimit: app.cpuLimit,
      memoryRequest: app.memoryRequest,
      memoryLimit: app.memoryLimit,
      envVars: this.resolveEnvVars(app),
      runtime: app.runtime,
      databaseType: app.databaseType,
      domain: domain,
      subdomain: app.subdomain || app.name,
      dbUsername: app.dbUsername || 'appuser',
      dbPassword:
        app.databaseType && app.databaseType !== DatabaseType.NONE
          ? this.ensureDbPassword(app)
          : '',
      dbVersion: app.dbVersion || '',
      dbStorageSize: app.dbStorageSize || '1Gi',
      dbCpuRequest: this.resolveDatabaseResources(app).cpuRequest,
      dbCpuLimit: this.resolveDatabaseResources(app).cpuLimit,
      dbMemoryRequest: this.resolveDatabaseResources(app).memoryRequest,
      dbMemoryLimit: this.resolveDatabaseResources(app).memoryLimit,
      appStorageSize: app.appStorageSize || '2Gi',
      enableRedis: app.enableRedis || false,
      redisVersion: app.redisVersion || '7.2',
      enableRabbitmq: app.enableRabbitmq || false,
      rabbitmqVersion: app.rabbitmqVersion || '3.13',
      enableElasticsearch: app.enableElasticsearch || false,
      elasticsearchVersion: app.elasticsearchVersion || '8.12',
      logPaths: app.logPaths || [],
      ownerId: app.userId,
      applicationId: app.id,
    };

    const manifests: Record<string, any> = {};

    try {
      // 1. Ensure namespace exists
      await this.ensureNamespace(coreApi, context.namespace);

      // 1b. Image pull secret for in-cluster registry
      await this.registryService.ensureRegistryPullSecret(coreApi, context.namespace);

      // 2. Create/Update secrets for env vars
      if (Object.keys(context.envVars).length > 0) {
        manifests.secret = await this.applySecret(coreApi, context);
      }

      // 3. Deploy database if needed
      if (context.databaseType !== DatabaseType.NONE) {
        manifests.database = await this.deployDatabase(coreApi, appsApi, context);
      }

      // 3.5 Deploy optional services
      if (context.enableRedis) {
        await this.deployRedis(coreApi, appsApi, context);
        manifests.redis = true;
      }
      if (context.enableRabbitmq) {
        await this.deployRabbitmq(coreApi, appsApi, context);
        manifests.rabbitmq = true;
      }

      // 3.7 Logging: credentials secret + Fluent Bit config
      if (context.enableElasticsearch) {
        await this.ensureElasticsearchCredentialsSecret(coreApi, context.namespace);
        await this.createFluentBitConfigMap(coreApi, context);
        manifests.fluentBitConfig = true;
      }

      // 3.8 Create app storage PVC (for all app types)
      manifests.appStoragePvc = await this.applyAppStoragePvc(coreApi, context);

      // 4. Create Deployment
      manifests.deployment = await this.applyDeployment(appsApi, context);

      // 5. Create Service
      manifests.service = await this.applyService(coreApi, context);

      // 6. Create Ingress
      const customDomain = app.customDomain && app.customDomainStatus === CustomDomainStatus.VERIFIED ? app.customDomain : undefined;
      manifests.ingress = await this.applyIngress(networkingApi, context, customDomain, previewNumber);

      this.logger.log(`Successfully deployed ${app.name} to namespace ${context.namespace} via K8s API`);
    } catch (error: any) {
      this.logger.error(`Failed to deploy ${app.name} via K8s API:`, error.body || error.message);
      throw error;
    }

    return manifests;
  }

  // ── Private K8s resource methods ──────────────────────────────────

  private async ensureNamespace(coreApi: k8s.CoreV1Api, namespace: string): Promise<void> {
    try {
      await coreApi.readNamespace({ name: namespace });
    } catch {
      await coreApi.createNamespace({
        body: { metadata: { name: namespace } },
      });
      this.logger.log(`Created namespace: ${namespace}`);
    }
  }

  private async applySecret(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise<any> {
    const secretData: Record<string, string> = {};
    for (const [key, value] of Object.entries(ctx.envVars)) {
      secretData[key] = Buffer.from(String(value)).toString('base64');
    }

    const secret = {
      apiVersion: 'v1',
      kind: 'Secret',
      metadata: {
        name: `${ctx.appName}-env`,
        namespace: ctx.namespace,
      },
      data: secretData,
    };

    try {
      await coreApi.replaceNamespacedSecret({
        name: `${ctx.appName}-env`,
        namespace: ctx.namespace,
        body: secret,
      });
    } catch {
      await coreApi.createNamespacedSecret({
        namespace: ctx.namespace,
        body: secret,
      });
    }
    return secret;
  }

  private async applyDeployment(appsApi: k8s.AppsV1Api, ctx: ManifestContext): Promise<any> {
    const envFrom: any[] = [];
    if (Object.keys(ctx.envVars).length > 0) {
      envFrom.push({ secretRef: { name: `${ctx.appName}-env` } });
    }

    // Database connection env vars
    const extraEnv: any[] = [];
    if (ctx.databaseType === DatabaseType.POSTGRESQL) {
      extraEnv.push(
        { name: 'DB_HOST', value: `${ctx.appName}-db` },
        { name: 'DB_PORT', value: '5432' },
        { name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') },
        {
          name: 'DB_USER',
          valueFrom: {
            secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' },
          },
        },
        {
          name: 'DB_PASSWORD',
          valueFrom: {
            secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' },
          },
        },
        {
          name: 'DATABASE_URL',
          value: `postgresql://$(DB_USER):$(DB_PASSWORD)@${ctx.appName}-db:5432/${ctx.appName.replace(/-/g, '_')}`,
        },
      );
    } else if (ctx.databaseType === DatabaseType.MYSQL || ctx.databaseType === DatabaseType.MARIADB) {
      extraEnv.push(
        { name: 'DB_HOST', value: `${ctx.appName}-db` },
        { name: 'DB_PORT', value: '3306' },
        { name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') },
        {
          name: 'DB_USER',
          valueFrom: {
            secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' },
          },
        },
        {
          name: 'DB_PASSWORD',
          valueFrom: {
            secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' },
          },
        },
        {
          name: 'DATABASE_URL',
          value: `mysql://$(DB_USER):$(DB_PASSWORD)@${ctx.appName}-db:3306/${ctx.appName.replace(/-/g, '_')}`,
        },
      );
    } else if (ctx.databaseType === DatabaseType.MONGODB) {
      extraEnv.push(
        { name: 'DB_HOST', value: `${ctx.appName}-db` },
        { name: 'DB_PORT', value: '27017' },
        { name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') },
        {
          name: 'DB_USER',
          valueFrom: {
            secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' },
          },
        },
        {
          name: 'DB_PASSWORD',
          valueFrom: {
            secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' },
          },
        },
        {
          name: 'MONGODB_URI',
          value: `mongodb://$(DB_USER):$(DB_PASSWORD)@${ctx.appName}-db:27017/${ctx.appName.replace(/-/g, '_')}?authSource=admin`,
        },
        {
          name: 'DATABASE_URL',
          value: `mongodb://$(DB_USER):$(DB_PASSWORD)@${ctx.appName}-db:27017/${ctx.appName.replace(/-/g, '_')}?authSource=admin`,
        },
      );
    }

    if ((ctx.runtime === AppRuntime.LARAVEL || ctx.runtime === AppRuntime.PHP) && ctx.databaseType !== DatabaseType.NONE) {
      const dbConnection = ctx.databaseType === DatabaseType.POSTGRESQL ? 'pgsql' : ctx.databaseType === DatabaseType.MONGODB ? 'mongodb' : 'mysql';
      extraEnv.push(
        { name: 'DB_CONNECTION', value: dbConnection },
        { name: 'DB_DATABASE', value: ctx.appName.replace(/-/g, '_') },
        {
          name: 'DB_USERNAME',
          valueFrom: {
            secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' },
          },
        },
      );
    }

    // Redis connection env vars
    if (ctx.enableRedis) {
      const redisName = `${ctx.appName}-redis`;
      extraEnv.push(
        { name: 'REDIS_HOST', value: redisName },
        { name: 'REDIS_PORT', value: '6379' },
        {
          name: 'REDIS_PASSWORD',
          valueFrom: {
            secretKeyRef: { name: `${redisName}-secret`, key: 'password' },
          },
        },
        {
          name: 'REDIS_URL',
          value: `redis://:$(REDIS_PASSWORD)@${redisName}:6379`,
        },
      );
    }

    // RabbitMQ connection env vars
    if (ctx.enableRabbitmq) {
      const rabbitName = `${ctx.appName}-rabbitmq`;
      extraEnv.push(
        { name: 'RABBITMQ_HOST', value: rabbitName },
        { name: 'RABBITMQ_PORT', value: '5672' },
        { name: 'RABBITMQ_MANAGEMENT_PORT', value: '15672' },
        {
          name: 'RABBITMQ_USER',
          valueFrom: {
            secretKeyRef: { name: `${rabbitName}-secret`, key: 'username' },
          },
        },
        {
          name: 'RABBITMQ_PASSWORD',
          valueFrom: {
            secretKeyRef: { name: `${rabbitName}-secret`, key: 'password' },
          },
        },
        {
          name: 'AMQP_URL',
          value: `amqp://$(RABBITMQ_USER):$(RABBITMQ_PASSWORD)@${rabbitName}:5672`,
        },
      );
    }

    // WordPress-specific env vars
    if (ctx.runtime === AppRuntime.WORDPRESS && ctx.databaseType !== DatabaseType.NONE) {
      const dbPort = ctx.databaseType === DatabaseType.POSTGRESQL ? '5432' : '3306';
      extraEnv.push(
        { name: 'WORDPRESS_DB_HOST', value: `${ctx.appName}-db:${dbPort}` },
        { name: 'WORDPRESS_DB_NAME', value: ctx.appName.replace(/-/g, '_') },
        {
          name: 'WORDPRESS_DB_USER',
          valueFrom: {
            secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' },
          },
        },
        {
          name: 'WORDPRESS_DB_PASSWORD',
          valueFrom: {
            secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' },
          },
        },
        { name: 'WORDPRESS_TABLE_PREFIX', value: 'wp_' },
      );
    }

    const deployment: k8s.V1Deployment = {
      apiVersion: 'apps/v1',
      kind: 'Deployment',
      metadata: {
        name: ctx.appName,
        namespace: ctx.namespace,
        labels: { app: ctx.appName, runtime: ctx.runtime },
        annotations: {
          'kubernetes.io/change-cause': `Deploy ${ctx.image} at ${new Date().toISOString()}`,
        },
      },
      spec: {
        revisionHistoryLimit: 10,
        replicas: ctx.replicas,
        selector: { matchLabels: { app: ctx.appName } },
        template: {
          metadata: { labels: { app: ctx.appName, runtime: ctx.runtime } },
          spec: {
            imagePullSecrets: [{ name: 'registry-pull-secret' }],
            containers: this.buildContainersSpec(ctx, envFrom, extraEnv),
            volumes: this.buildVolumesSpec(ctx),
          },
        },
      },
    };

    try {
      await appsApi.replaceNamespacedDeployment({
        name: ctx.appName,
        namespace: ctx.namespace,
        body: deployment,
      });
    } catch {
      await appsApi.createNamespacedDeployment({
        namespace: ctx.namespace,
        body: deployment,
      });
    }
    return deployment;
  }

  private async applyAppStoragePvc(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise<any> {
    const pvcName = `${ctx.appName}-storage`;
    const pvc = {
      apiVersion: 'v1',
      kind: 'PersistentVolumeClaim',
      metadata: {
        name: pvcName,
        namespace: ctx.namespace,
        labels: { app: ctx.appName, runtime: ctx.runtime },
      },
      spec: {
        accessModes: ['ReadWriteOnce'],
        resources: { requests: { storage: ctx.appStorageSize || '2Gi' } },
        ...(this.configService.get<string>('platform.storageClass')
          ? {
              storageClassName: this.configService.get<string>('platform.storageClass'),
            }
          : {}),
      },
    };

    try {
      await coreApi.readNamespacedPersistentVolumeClaim({
        name: pvcName,
        namespace: ctx.namespace,
      });
      this.logger.log(`PVC ${pvcName} already exists, skipping`);
    } catch {
      await coreApi.createNamespacedPersistentVolumeClaim({
        namespace: ctx.namespace,
        body: pvc,
      });
      this.logger.log(`Created app storage PVC: ${pvcName}`);
    }
    return pvc;
  }

  /**
   * Get the storage mount path based on runtime type
   */
  private getStorageMountPath(runtime: string): string {
    switch (runtime) {
      case AppRuntime.WORDPRESS:
        return '/var/www/html/wp-content';
      case AppRuntime.LARAVEL:
      case AppRuntime.PHP:
        return '/var/www/html/storage';
      case AppRuntime.DJANGO:
        return '/app/media';
      case AppRuntime.PYTHON:
      case AppRuntime.GO:
      case AppRuntime.DOTNET:
      case AppRuntime.NODEJS:
      default:
        return '/app/data';
    }
  }

  /**
   * Build containers spec for the app deployment, including optional Fluent Bit sidecar
   */
  private buildContainersSpec(ctx: ManifestContext, envFrom: any[], extraEnv: any[]): any[] {
    const containers: any[] = [];

    // Main application container
    const appContainer: any = {
      name: ctx.appName,
      // Image tags are unique per build (name:timestamp) and immutable, so
      // IfNotPresent is correct and avoids re-pulling on every restart/scale-up.
      image: ctx.image,
      imagePullPolicy: 'IfNotPresent',
      ports: [{ containerPort: ctx.port }],
      envFrom,
      env: extraEnv,
      resources: {
        requests: { cpu: ctx.cpuRequest, memory: ctx.memoryRequest },
        limits: { cpu: ctx.cpuLimit, memory: ctx.memoryLimit },
      },
      readinessProbe: {
        tcpSocket: { port: ctx.port as any },
        initialDelaySeconds: 5,
        periodSeconds: 5,
      },
      livenessProbe: {
        tcpSocket: { port: ctx.port as any },
        initialDelaySeconds: 15,
        periodSeconds: 10,
        failureThreshold: 5,
      },
      volumeMounts: [
        {
          name: 'app-storage',
          mountPath: this.getStorageMountPath(ctx.runtime),
        },
      ],
    };

    // Add log volume mount if Elasticsearch is enabled
    if (ctx.enableElasticsearch) {
      appContainer.volumeMounts.push({
        name: 'app-logs',
        mountPath: '/var/log/app',
      });
      this.applyLoggingCommandWrapper(appContainer, ctx.runtime);
    }

    containers.push(appContainer);

    // Add Fluent Bit sidecar for log collection if Elasticsearch is enabled
    if (ctx.enableElasticsearch) {
      containers.push({
        name: 'fluent-bit',
        image: 'fluent/fluent-bit:2.2',
        resources: {
          requests: { cpu: '10m', memory: '32Mi' },
          limits: { cpu: '50m', memory: '64Mi' },
        },
        volumeMounts: [
          { name: 'app-logs', mountPath: '/var/log/app', readOnly: true },
          { name: 'fluent-bit-config', mountPath: '/fluent-bit/etc' },
        ],
        env: [
          { name: 'APP_NAME', value: ctx.appName },
          { name: 'APP_NAMESPACE', value: ctx.namespace },
          { name: 'ES_HOST', value: 'elasticsearch.logging.svc.cluster.local' },
          { name: 'ES_PORT', value: '9200' },
          {
            name: 'ES_PASSWORD',
            valueFrom: {
              secretKeyRef: {
                name: 'elasticsearch-credentials',
                key: 'ELASTIC_PASSWORD',
                optional: true,
              },
            },
          },
        ],
      });
    }

    return containers;
  }

  /**
   * Build volumes spec for the app deployment
   */
  private buildVolumesSpec(ctx: ManifestContext): any[] {
    const volumes: any[] = [
      {
        name: 'app-storage',
        persistentVolumeClaim: { claimName: `${ctx.appName}-storage` },
      },
    ];

    if (ctx.enableElasticsearch) {
      // Shared log volume between app and fluent-bit
      volumes.push({
        name: 'app-logs',
        emptyDir: {},
      });

      // Fluent Bit config as ConfigMap
      volumes.push({
        name: 'fluent-bit-config',
        configMap: { name: `${ctx.appName}-fluent-bit-config` },
      });
    }

    return volumes;
  }

  /**
   * Get default log paths based on runtime type
   */
  private getDefaultLogPaths(runtime: string): string[] {
    switch (runtime) {
      case AppRuntime.WORDPRESS:
        return ['/var/www/html/wp-content/debug.log', '/var/log/app/*.log'];
      case AppRuntime.LARAVEL:
        return ['/var/www/html/storage/logs/*.log', '/var/log/app/*.log'];
      case AppRuntime.PHP:
        return ['/var/www/html/storage/logs/*.log', '/var/log/php/*.log', '/var/log/app/*.log'];
      default:
        // Node/Go/Python/.NET log to stdout — captured into /var/log/app/app.log at runtime
        return ['/var/log/app/*.log'];
    }
  }

  /**
   * Redirect stdout/stderr into the shared log volume so Fluent Bit can tail them.
   */
  private applyLoggingCommandWrapper(container: any, runtime: string): void {
    const startCmd = this.getRuntimeStartCommand(runtime);
    if (!startCmd) return;
    container.command = ['sh', '-c'];
    container.args = [`mkdir -p /var/log/app && (${startCmd}) >> /var/log/app/app.log 2>&1`];
  }

  /** Shell command that mirrors CloudHost-generated image ENTRYPOINT/CMD per runtime. */
  private getRuntimeStartCommand(runtime: string): string | null {
    switch (runtime) {
      case AppRuntime.NODEJS:
        return 'if [ -f /app/.mode ] && [ "$(cat /app/.mode)" = "standalone" ] && [ -f server.js ]; ' + 'then node server.js; else npm start; fi';
      case AppRuntime.GO:
        return './main';
      case AppRuntime.PYTHON:
        return (
          'if [ -f main.py ]; then ' +
          'if grep -qi fastapi main.py; then exec uvicorn main:app --host 0.0.0.0 --port ${PORT:-3000}; ' +
          'elif grep -qi flask main.py; then exec gunicorn -w 4 -b 0.0.0.0:${PORT:-3000} main:app; ' +
          'else exec python main.py; fi; ' +
          'elif [ -f app.py ]; then ' +
          'if grep -qi fastapi app.py; then exec uvicorn app:app --host 0.0.0.0 --port ${PORT:-3000}; ' +
          'elif grep -qi flask app.py; then exec gunicorn -w 4 -b 0.0.0.0:${PORT:-3000} app:app; ' +
          'else exec python app.py; fi; ' +
          'else exec gunicorn -w 4 -b 0.0.0.0:${PORT:-3000} app:app; fi'
        );
      case AppRuntime.DJANGO:
        return 'python manage.py runserver 0.0.0.0:${PORT:-8000}';
      case AppRuntime.DOTNET:
        return 'DLL=$(find . -maxdepth 1 -name "*.dll" ! -name "*.deps.dll" ! -name "*.runtimeconfig.dll" | head -1) ' + '&& dotnet "$DLL"';
      case AppRuntime.WORDPRESS:
      case AppRuntime.LARAVEL:
      case AppRuntime.PHP:
        return null;
      default:
        return null;
    }
  }

  /** Replicate logging credentials into the app namespace for Fluent Bit sidecars. */
  private async ensureElasticsearchCredentialsSecret(coreApi: k8s.CoreV1Api, namespace: string): Promise<void> {
    const name = 'elasticsearch-credentials';
    const stringData: { [key: string]: string } = {
      ELASTIC_PASSWORD: this.configService.get<string>('elasticsearch.password') || '',
      FLUENTBIT_PASSWORD: this.configService.get<string>('elasticsearch.fluentbitPassword') || '',
      KIBANA_SYSTEM_PASSWORD: this.configService.get<string>('elasticsearch.kibanaPassword') || '',
    };

    try {
      await coreApi.readNamespacedSecret({ name, namespace });
      await coreApi.replaceNamespacedSecret({
        name,
        namespace,
        body: {
          apiVersion: 'v1',
          kind: 'Secret',
          metadata: { name, namespace },
          type: 'Opaque',
          stringData,
        },
      });
    } catch (err: any) {
      if (err.code === 404 || err.body?.code === 404) {
        await coreApi.createNamespacedSecret({
          namespace,
          body: {
            apiVersion: 'v1',
            kind: 'Secret',
            metadata: { name, namespace },
            type: 'Opaque',
            stringData,
          },
        });
        this.logger.log(`Created ${name} secret in ${namespace}`);
      } else {
        throw err;
      }
    }
  }

  /**
   * Build Fluent Bit configuration for log collection
   */
  private buildFluentBitConfig(appName: string, namespace: string, runtime: string, ownerId: string, applicationId: string, workload: string, customLogPaths?: string[]): string {
    const logPaths = customLogPaths && customLogPaths.length > 0 ? customLogPaths : this.getDefaultLogPaths(runtime);
    const pathsStr = logPaths.join(',');

    return `
[SERVICE]
    Flush         5
    Daemon        Off
    Log_Level     info
    Parsers_File  /fluent-bit/etc/parsers.conf

[INPUT]
    Name          tail
    Path          ${pathsStr}
    Tag           app.${appName}
    Refresh_Interval 5
    Mem_Buf_Limit 5MB
    Skip_Long_Lines On

[FILTER]
    Name          record_modifier
    Match         *
    Record        app ${appName}
    Record        applicationName ${appName}
    Record        namespace ${namespace}
    Record        runtime ${runtime}
    Record        ownerId ${ownerId}
    Record        applicationId ${applicationId}
    Record        workload ${workload}

[FILTER]
    Name          parser
    Match         *
    Key_Name      log
    Parser        json
    Reserve_Data  On
    Preserve_Key  On

[OUTPUT]
    Name          es
    Match         *
    Host          \${ES_HOST}
    Port          \${ES_PORT}
    HTTP_User     elastic
    HTTP_Passwd   \${ES_PASSWORD}
    Index         logs-${namespace}-${appName}
    Logstash_Format On
    Logstash_Prefix logs-${namespace}
    Suppress_Type_Name On
    tls           Off
    Retry_Limit   3
`;
  }

  /**
   * Create Fluent Bit ConfigMap for an app
   */
  private async createFluentBitConfigMap(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise<void> {
    if (!ctx.enableElasticsearch) return;

    const customLogPaths = ctx.logPaths && ctx.logPaths.length > 0 ? ctx.logPaths : undefined;

    const configMap = {
      apiVersion: 'v1',
      kind: 'ConfigMap',
      metadata: {
        name: `${ctx.appName}-fluent-bit-config`,
        namespace: ctx.namespace,
        labels: { app: ctx.appName },
      },
      data: {
        'fluent-bit.conf': this.buildFluentBitConfig(ctx.appName, ctx.namespace, ctx.runtime, ctx.ownerId, ctx.applicationId, 'app', customLogPaths),
        'parsers.conf': `
[PARSER]
    Name        json
    Format      json
    Time_Key    time
    Time_Format %Y-%m-%dT%H:%M:%S.%L

[PARSER]
    Name        docker
    Format      json
    Time_Key    time
    Time_Format %Y-%m-%dT%H:%M:%S.%L
`,
      },
    };

    try {
      await coreApi.replaceNamespacedConfigMap({
        name: `${ctx.appName}-fluent-bit-config`,
        namespace: ctx.namespace,
        body: configMap,
      });
    } catch {
      await coreApi.createNamespacedConfigMap({
        namespace: ctx.namespace,
        body: configMap,
      });
    }
    this.logger.log(`Created Fluent Bit ConfigMap for ${ctx.appName}`);
  }

  private buildWorkloadFluentBitConfig(ctx: ManifestContext, workload: 'redis' | 'rabbitmq' | 'database', resourceName: string): string {
    const logGlob = `/var/log/pods/*${resourceName}*/*/*.log`;
    return `
[SERVICE]
    Flush         5
    Daemon        Off
    Log_Level     info
    Parsers_File  /fluent-bit/etc/parsers.conf

[INPUT]
    Name              tail
    Path              ${logGlob}
    Tag               ${workload}.${resourceName}
    Refresh_Interval  5
    Mem_Buf_Limit     5MB
    Skip_Long_Lines   On
    Parser            docker

[FILTER]
    Name          record_modifier
    Match         *
    Record        app ${ctx.appName}
    Record        applicationName ${ctx.appName}
    Record        namespace ${ctx.namespace}
    Record        ownerId ${ctx.ownerId}
    Record        applicationId ${ctx.applicationId}
    Record        workload ${workload}

[OUTPUT]
    Name          es
    Match         *
    Host          \${ES_HOST}
    Port          \${ES_PORT}
    HTTP_User     elastic
    HTTP_Passwd   \${ES_PASSWORD}
    Index         logs-${ctx.namespace}-${ctx.appName}
    Logstash_Format On
    Logstash_Prefix logs-${ctx.namespace}
    Suppress_Type_Name On
    tls           Off
    Retry_Limit   3
`;
  }

  private async attachWorkloadLogShipper(
    coreApi: k8s.CoreV1Api,
    ctx: ManifestContext,
    workload: 'redis' | 'rabbitmq' | 'database',
    resourceName: string,
  ): Promise<{ containers: k8s.V1Container[]; volumes: k8s.V1Volume[] }> {
    if (!ctx.enableElasticsearch) {
      return { containers: [], volumes: [] };
    }

    const configMapName = `${resourceName}-log-shipper-config`;
    const configMap = {
      apiVersion: 'v1',
      kind: 'ConfigMap',
      metadata: {
        name: configMapName,
        namespace: ctx.namespace,
        labels: { app: resourceName, 'cloudhost.io/log-shipper': 'true' },
      },
      data: {
        'fluent-bit.conf': this.buildWorkloadFluentBitConfig(ctx, workload, resourceName),
        'parsers.conf': `
[PARSER]
    Name        docker
    Format      json
    Time_Key    time
    Time_Format %Y-%m-%dT%H:%M:%S.%L
`,
      },
    };

    try {
      await coreApi.replaceNamespacedConfigMap({
        name: configMapName,
        namespace: ctx.namespace,
        body: configMap,
      });
    } catch {
      await coreApi.createNamespacedConfigMap({
        namespace: ctx.namespace,
        body: configMap,
      });
    }

    return {
      containers: [
        {
          name: 'log-shipper',
          image: 'fluent/fluent-bit:2.2',
          resources: {
            requests: { cpu: '10m', memory: '32Mi' },
            limits: { cpu: '50m', memory: '64Mi' },
          },
          volumeMounts: [
            { name: 'varlogpods', mountPath: '/var/log/pods', readOnly: true },
            { name: 'log-shipper-config', mountPath: '/fluent-bit/etc' },
          ],
          env: [
            {
              name: 'ES_HOST',
              value: 'elasticsearch.logging.svc.cluster.local',
            },
            { name: 'ES_PORT', value: '9200' },
            {
              name: 'ES_PASSWORD',
              valueFrom: {
                secretKeyRef: {
                  name: 'elasticsearch-credentials',
                  key: 'ELASTIC_PASSWORD',
                  optional: true,
                },
              },
            },
          ],
        },
      ],
      volumes: [
        {
          name: 'varlogpods',
          hostPath: { path: '/var/log/pods', type: 'Directory' },
        },
        { name: 'log-shipper-config', configMap: { name: configMapName } },
      ],
    };
  }

  private async applyService(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise<any> {
    const service: k8s.V1Service = {
      apiVersion: 'v1',
      kind: 'Service',
      metadata: { name: ctx.appName, namespace: ctx.namespace },
      spec: {
        selector: { app: ctx.appName },
        ports: [{ port: 80, targetPort: ctx.port as any, protocol: 'TCP' }],
        type: 'ClusterIP',
      },
    };

    try {
      await coreApi.replaceNamespacedService({
        name: ctx.appName,
        namespace: ctx.namespace,
        body: service,
      });
    } catch {
      await coreApi.createNamespacedService({
        namespace: ctx.namespace,
        body: service,
      });
    }
    return service;
  }

  private async applyIngress(networkingApi: k8s.NetworkingV1Api, ctx: ManifestContext, customDomain?: string, previewNumber?: string | null): Promise<any> {
    const host = `${ctx.subdomain}.${ctx.domain}`;
    const rules: k8s.V1IngressRule[] = [
      {
        host,
        http: {
          paths: [
            {
              path: '/',
              pathType: 'Prefix',
              backend: { service: { name: ctx.appName, port: { number: 80 } } },
            },
          ],
        },
      },
    ];
    // Only PUBLIC, real-TLD hosts may go into the TLS block. The internal
    // `${subdomain}.${domain}` host (e.g. *.apps.cloudhost.local) is not a valid
    // public suffix — including it makes Let's Encrypt reject the whole order,
    // which would also block the cert for the legitimate preview/custom domains.
    const tlsHosts: string[] = [];

    if (customDomain) {
      rules.push({
        host: customDomain,
        http: {
          paths: [
            {
              path: '/',
              pathType: 'Prefix',
              backend: { service: { name: ctx.appName, port: { number: 80 } } },
            },
          ],
        },
      });
      tlsHosts.push(customDomain);
    }

    const previewRootDomain = this.configService.get<string>('platform.previewRootDomain') || ctx.domain;
    const namespacePrefix = userIdSlug(ctx.ownerId);
    const previewHost = previewNumber && !customDomain ? `${namespacePrefix}-${previewNumber}.${previewRootDomain}` : '';
    if (previewHost) {
      rules.push({
        host: previewHost,
        http: {
          paths: [
            {
              path: '/',
              pathType: 'Prefix',
              backend: { service: { name: ctx.appName, port: { number: 80 } } },
            },
          ],
        },
      });
      tlsHosts.push(previewHost);
    }

    const ingressClass = this.configService.get<string>('platform.ingressClass') || 'traefik';
    // Request a managed cert only when we actually have a public host to issue for.
    const annotations: Record<string, string> = {};
    const tls: k8s.V1IngressTLS[] = [];
    if (tlsHosts.length > 0) {
      annotations['cert-manager.io/cluster-issuer'] = 'letsencrypt-prod';
      tls.push({ hosts: tlsHosts, secretName: `${ctx.appName}-tls` });
    }

    const ingress: k8s.V1Ingress = {
      apiVersion: 'networking.k8s.io/v1',
      kind: 'Ingress',
      metadata: {
        name: ctx.appName,
        namespace: ctx.namespace,
        annotations,
      },
      spec: {
        ingressClassName: ingressClass,
        rules,
        ...(tls.length > 0 ? { tls } : {}),
      },
    };

    try {
      await networkingApi.replaceNamespacedIngress({
        name: ctx.appName,
        namespace: ctx.namespace,
        body: ingress,
      });
    } catch {
      await networkingApi.createNamespacedIngress({
        namespace: ctx.namespace,
        body: ingress,
      });
    }
    return ingress;
  }

  private async deployDatabase(coreApi: k8s.CoreV1Api, appsApi: k8s.AppsV1Api, ctx: ManifestContext): Promise<any> {
    const dbName = `${ctx.appName}-db`;

    // Create DB secret
    await this.createDbSecret(coreApi, ctx.namespace, ctx.appName, ctx.dbPassword, ctx.dbUsername);

    // Create PVC for DB
    await this.createPVC(coreApi, ctx.namespace, dbName, ctx.dbStorageSize);

    // Deploy database based on type
    const dbType = ctx.databaseType;
    let image: string;
    let port: number;
    let dataPath: string;
    let envVars: any[];
    let readinessProbe: any;
    let livenessProbe: any;

    switch (dbType) {
      case DatabaseType.POSTGRESQL: {
        const pgVer = ctx.dbVersion || '16';
        const pgDatabase = ctx.appName.replace(/-/g, '_');
        image = `postgres:${pgVer}-alpine`;
        port = 5432;
        dataPath = '/var/lib/postgresql/data';
        envVars = [
          { name: 'PGDATA', value: '/var/lib/postgresql/data/pgdata' },
          { name: 'POSTGRES_DB', value: pgDatabase },
          {
            name: 'POSTGRES_USER',
            valueFrom: {
              secretKeyRef: {
                name: `${ctx.appName}-db-secret`,
                key: 'username',
              },
            },
          },
          {
            name: 'POSTGRES_PASSWORD',
            valueFrom: {
              secretKeyRef: {
                name: `${ctx.appName}-db-secret`,
                key: 'password',
              },
            },
          },
        ];
        const pgReady = ['pg_isready', '-U', ctx.dbUsername, '-d', pgDatabase];
        readinessProbe = {
          exec: { command: pgReady },
          initialDelaySeconds: 10,
          periodSeconds: 5,
          failureThreshold: 6,
        };
        livenessProbe = {
          exec: { command: pgReady },
          initialDelaySeconds: 30,
          periodSeconds: 10,
          failureThreshold: 5,
        };
        break;
      }

      case DatabaseType.MYSQL:
        const mysqlVer = ctx.dbVersion || '8.0';
        image = `mysql:${mysqlVer}`;
        port = 3306;
        dataPath = '/var/lib/mysql';
        envVars = [
          { name: 'MYSQL_DATABASE', value: ctx.appName.replace(/-/g, '_') },
          {
            name: 'MYSQL_USER',
            valueFrom: {
              secretKeyRef: {
                name: `${ctx.appName}-db-secret`,
                key: 'username',
              },
            },
          },
          {
            name: 'MYSQL_PASSWORD',
            valueFrom: {
              secretKeyRef: {
                name: `${ctx.appName}-db-secret`,
                key: 'password',
              },
            },
          },
          {
            name: 'MYSQL_ROOT_PASSWORD',
            valueFrom: {
              secretKeyRef: {
                name: `${ctx.appName}-db-secret`,
                key: 'password',
              },
            },
          },
        ];
        readinessProbe = {
          exec: { command: ['mysqladmin', 'ping', '-h', '127.0.0.1'] },
          initialDelaySeconds: 10,
          periodSeconds: 5,
          failureThreshold: 6,
        };
        livenessProbe = {
          exec: { command: ['mysqladmin', 'ping', '-h', '127.0.0.1'] },
          initialDelaySeconds: 30,
          periodSeconds: 10,
          failureThreshold: 5,
        };
        break;

      case DatabaseType.MARIADB:
        const mariaVer = ctx.dbVersion || '11.4';
        image = `mariadb:${mariaVer}`;
        port = 3306;
        dataPath = '/var/lib/mysql';
        envVars = [
          { name: 'MARIADB_DATABASE', value: ctx.appName.replace(/-/g, '_') },
          {
            name: 'MARIADB_USER',
            valueFrom: {
              secretKeyRef: {
                name: `${ctx.appName}-db-secret`,
                key: 'username',
              },
            },
          },
          {
            name: 'MARIADB_PASSWORD',
            valueFrom: {
              secretKeyRef: {
                name: `${ctx.appName}-db-secret`,
                key: 'password',
              },
            },
          },
          {
            name: 'MARIADB_ROOT_PASSWORD',
            valueFrom: {
              secretKeyRef: {
                name: `${ctx.appName}-db-secret`,
                key: 'password',
              },
            },
          },
        ];
        readinessProbe = {
          exec: {
            command: ['healthcheck.sh', '--connect', '--innodb_initialized'],
          },
          initialDelaySeconds: 10,
          periodSeconds: 5,
          failureThreshold: 6,
        };
        livenessProbe = {
          exec: {
            command: ['healthcheck.sh', '--connect', '--innodb_initialized'],
          },
          initialDelaySeconds: 30,
          periodSeconds: 10,
          failureThreshold: 5,
        };
        break;

      case DatabaseType.MONGODB:
        const mongoVer = ctx.dbVersion || '7.0';
        image = `mongo:${mongoVer}`;
        port = 27017;
        dataPath = '/data/db';
        envVars = [
          {
            name: 'MONGO_INITDB_DATABASE',
            value: ctx.appName.replace(/-/g, '_'),
          },
          {
            name: 'MONGO_INITDB_ROOT_USERNAME',
            valueFrom: {
              secretKeyRef: {
                name: `${ctx.appName}-db-secret`,
                key: 'username',
              },
            },
          },
          {
            name: 'MONGO_INITDB_ROOT_PASSWORD',
            valueFrom: {
              secretKeyRef: {
                name: `${ctx.appName}-db-secret`,
                key: 'password',
              },
            },
          },
        ];
        readinessProbe = {
          exec: { command: ['mongosh', '--eval', 'db.adminCommand("ping")'] },
          initialDelaySeconds: 10,
          periodSeconds: 5,
          failureThreshold: 6,
        };
        livenessProbe = {
          exec: { command: ['mongosh', '--eval', 'db.adminCommand("ping")'] },
          initialDelaySeconds: 30,
          periodSeconds: 10,
          failureThreshold: 5,
        };
        break;

      default:
        throw new Error(`Unsupported database type: ${dbType}`);
    }

    const dbLogShipper = await this.attachWorkloadLogShipper(coreApi, ctx, 'database', dbName);

    const dbDeployment: k8s.V1Deployment = {
      apiVersion: 'apps/v1',
      kind: 'Deployment',
      metadata: {
        name: dbName,
        namespace: ctx.namespace,
        labels: { app: dbName },
      },
      spec: {
        replicas: 1,
        selector: { matchLabels: { app: dbName } },
        template: {
          metadata: { labels: { app: dbName } },
          spec: {
            containers: [
              {
                name: dbName,
                image,
                ports: [{ containerPort: port }],
                env: envVars,
                volumeMounts: [{ name: 'db-storage', mountPath: dataPath }],
                resources: {
                  requests: { cpu: ctx.dbCpuRequest, memory: ctx.dbMemoryRequest },
                  limits: { cpu: ctx.dbCpuLimit, memory: ctx.dbMemoryLimit },
                },
                readinessProbe,
                livenessProbe,
              },
              ...dbLogShipper.containers,
            ],
            volumes: [
              {
                name: 'db-storage',
                persistentVolumeClaim: { claimName: dbName },
              },
              ...dbLogShipper.volumes,
            ],
          },
        },
      },
    };

    try {
      await appsApi.replaceNamespacedDeployment({
        name: dbName,
        namespace: ctx.namespace,
        body: dbDeployment,
      });
    } catch {
      await appsApi.createNamespacedDeployment({
        namespace: ctx.namespace,
        body: dbDeployment,
      });
    }

    // Create DB Service
    const dbService: k8s.V1Service = {
      apiVersion: 'v1',
      kind: 'Service',
      metadata: { name: dbName, namespace: ctx.namespace },
      spec: {
        selector: { app: dbName },
        ports: [{ port, targetPort: port as any, protocol: 'TCP' }],
        type: 'ClusterIP',
      },
    };

    try {
      await coreApi.replaceNamespacedService({
        name: dbName,
        namespace: ctx.namespace,
        body: dbService,
      });
    } catch {
      await coreApi.createNamespacedService({
        namespace: ctx.namespace,
        body: dbService,
      });
    }

    return { deployment: dbDeployment, service: dbService };
  }

  private async createDbSecret(coreApi: k8s.CoreV1Api, namespace: string, appName: string, password: string, username: string = 'appuser'): Promise<void> {
    const secret = {
      apiVersion: 'v1',
      kind: 'Secret',
      metadata: { name: `${appName}-db-secret`, namespace },
      data: {
        username: Buffer.from(username).toString('base64'),
        password: Buffer.from(password).toString('base64'),
      },
    };

    try {
      await coreApi.replaceNamespacedSecret({
        name: `${appName}-db-secret`,
        namespace,
        body: secret,
      });
    } catch {
      await coreApi.createNamespacedSecret({ namespace, body: secret });
    }
  }

  private async createPVC(coreApi: k8s.CoreV1Api, namespace: string, name: string, size: string): Promise<void> {
    const storageClass = this.configService.get<string>('platform.storageClass');
    const pvc: k8s.V1PersistentVolumeClaim = {
      apiVersion: 'v1',
      kind: 'PersistentVolumeClaim',
      metadata: { name, namespace },
      spec: {
        accessModes: ['ReadWriteOnce'],
        resources: { requests: { storage: size } },
        ...(storageClass ? { storageClassName: storageClass } : {}),
      },
    };

    try {
      await coreApi.readNamespacedPersistentVolumeClaim({ name, namespace });
      // PVC exists, don't recreate
    } catch {
      await coreApi.createNamespacedPersistentVolumeClaim({
        namespace,
        body: pvc,
      });
    }
  }

  /**
   * Deploy Redis for an application
   */
  private async deployRedis(coreApi: k8s.CoreV1Api, appsApi: k8s.AppsV1Api, ctx: ManifestContext): Promise<void> {
    const redisName = `${ctx.appName}-redis`;

    // Create PVC for Redis persistence
    await this.createPVC(coreApi, ctx.namespace, `${redisName}-data`, '1Gi');

    // Only create Redis password secret if it doesn't already exist
    try {
      await coreApi.readNamespacedSecret({
        name: `${redisName}-secret`,
        namespace: ctx.namespace,
      });
      this.logger.log(`Redis secret ${redisName}-secret already exists, skipping`);
    } catch {
      const redisPassword = this.generatePassword(16);
      const redisSecret = {
        apiVersion: 'v1',
        kind: 'Secret',
        metadata: { name: `${redisName}-secret`, namespace: ctx.namespace },
        data: {
          password: Buffer.from(redisPassword).toString('base64'),
        },
      };
      await coreApi.createNamespacedSecret({
        namespace: ctx.namespace,
        body: redisSecret,
      });
    }

    const logShipper = await this.attachWorkloadLogShipper(coreApi, ctx, 'redis', redisName);

    // Create Redis Deployment
    const redisDeployment: k8s.V1Deployment = {
      apiVersion: 'apps/v1',
      kind: 'Deployment',
      metadata: { name: redisName, namespace: ctx.namespace },
      spec: {
        replicas: 1,
        selector: { matchLabels: { app: redisName } },
        template: {
          metadata: { labels: { app: redisName } },
          spec: {
            containers: [
              {
                name: 'redis',
                image: `redis:${ctx.redisVersion}-alpine`,
                args: ['--requirepass', '$(REDIS_PASSWORD)'],
                ports: [{ containerPort: 6379 }],
                env: [
                  {
                    name: 'REDIS_PASSWORD',
                    valueFrom: {
                      secretKeyRef: {
                        name: `${redisName}-secret`,
                        key: 'password',
                      },
                    },
                  },
                ],
                volumeMounts: [{ name: 'redis-data', mountPath: '/data' }],
                resources: {
                  requests: { cpu: '50m', memory: '64Mi' },
                  limits: { cpu: '200m', memory: '256Mi' },
                },
                readinessProbe: {
                  exec: { command: ['redis-cli', 'ping'] },
                  initialDelaySeconds: 5,
                  periodSeconds: 10,
                },
                livenessProbe: {
                  exec: { command: ['redis-cli', 'ping'] },
                  initialDelaySeconds: 15,
                  periodSeconds: 20,
                },
              },
              ...logShipper.containers,
            ],
            volumes: [
              {
                name: 'redis-data',
                persistentVolumeClaim: { claimName: `${redisName}-data` },
              },
              ...logShipper.volumes,
            ],
          },
        },
      },
    };

    try {
      await appsApi.replaceNamespacedDeployment({
        name: redisName,
        namespace: ctx.namespace,
        body: redisDeployment,
      });
    } catch {
      await appsApi.createNamespacedDeployment({
        namespace: ctx.namespace,
        body: redisDeployment,
      });
    }

    // Create Redis Service
    const redisService = {
      apiVersion: 'v1',
      kind: 'Service',
      metadata: { name: redisName, namespace: ctx.namespace },
      spec: {
        selector: { app: redisName },
        ports: [{ port: 6379, targetPort: 6379 as any, protocol: 'TCP' }],
        type: 'ClusterIP',
      },
    };

    try {
      await coreApi.replaceNamespacedService({
        name: redisName,
        namespace: ctx.namespace,
        body: redisService,
      });
    } catch {
      await coreApi.createNamespacedService({
        namespace: ctx.namespace,
        body: redisService,
      });
    }

    this.logger.log(`Redis deployed for ${ctx.appName}`);
  }

  /**
   * Deploy RabbitMQ for an application
   */
  private async deployRabbitmq(coreApi: k8s.CoreV1Api, appsApi: k8s.AppsV1Api, ctx: ManifestContext): Promise<void> {
    const rabbitName = `${ctx.appName}-rabbitmq`;

    // Create PVC for RabbitMQ persistence
    await this.createPVC(coreApi, ctx.namespace, `${rabbitName}-data`, '2Gi');

    // Only create RabbitMQ credentials secret if it doesn't already exist
    try {
      await coreApi.readNamespacedSecret({
        name: `${rabbitName}-secret`,
        namespace: ctx.namespace,
      });
      this.logger.log(`RabbitMQ secret ${rabbitName}-secret already exists, skipping`);
    } catch {
      const rabbitUser = 'appuser';
      const rabbitPassword = this.generatePassword(16);
      const rabbitSecret = {
        apiVersion: 'v1',
        kind: 'Secret',
        metadata: { name: `${rabbitName}-secret`, namespace: ctx.namespace },
        data: {
          username: Buffer.from(rabbitUser).toString('base64'),
          password: Buffer.from(rabbitPassword).toString('base64'),
        },
      };
      await coreApi.createNamespacedSecret({
        namespace: ctx.namespace,
        body: rabbitSecret,
      });
    }

    const rabbitLogShipper = await this.attachWorkloadLogShipper(coreApi, ctx, 'rabbitmq', rabbitName);

    // Create RabbitMQ Deployment
    const rabbitDeployment: k8s.V1Deployment = {
      apiVersion: 'apps/v1',
      kind: 'Deployment',
      metadata: { name: rabbitName, namespace: ctx.namespace },
      spec: {
        replicas: 1,
        selector: { matchLabels: { app: rabbitName } },
        template: {
          metadata: { labels: { app: rabbitName } },
          spec: {
            containers: [
              {
                name: 'rabbitmq',
                image: `rabbitmq:${ctx.rabbitmqVersion}-management-alpine`,
                ports: [
                  { containerPort: 5672, name: 'amqp' },
                  { containerPort: 15672, name: 'management' },
                ],
                env: [
                  {
                    name: 'RABBITMQ_DEFAULT_USER',
                    valueFrom: {
                      secretKeyRef: {
                        name: `${rabbitName}-secret`,
                        key: 'username',
                      },
                    },
                  },
                  {
                    name: 'RABBITMQ_DEFAULT_PASS',
                    valueFrom: {
                      secretKeyRef: {
                        name: `${rabbitName}-secret`,
                        key: 'password',
                      },
                    },
                  },
                ],
                volumeMounts: [{ name: 'rabbitmq-data', mountPath: '/var/lib/rabbitmq' }],
                resources: {
                  requests: { cpu: '100m', memory: '256Mi' },
                  limits: { cpu: '500m', memory: '512Mi' },
                },
                readinessProbe: {
                  exec: { command: ['rabbitmq-diagnostics', '-q', 'ping'] },
                  initialDelaySeconds: 20,
                  periodSeconds: 10,
                  timeoutSeconds: 5,
                },
                livenessProbe: {
                  exec: { command: ['rabbitmq-diagnostics', '-q', 'status'] },
                  initialDelaySeconds: 60,
                  periodSeconds: 30,
                  timeoutSeconds: 10,
                },
              },
              ...rabbitLogShipper.containers,
            ],
            volumes: [
              {
                name: 'rabbitmq-data',
                persistentVolumeClaim: { claimName: `${rabbitName}-data` },
              },
              ...rabbitLogShipper.volumes,
            ],
          },
        },
      },
    };

    try {
      await appsApi.replaceNamespacedDeployment({
        name: rabbitName,
        namespace: ctx.namespace,
        body: rabbitDeployment,
      });
    } catch {
      await appsApi.createNamespacedDeployment({
        namespace: ctx.namespace,
        body: rabbitDeployment,
      });
    }

    // Create RabbitMQ Services (AMQP and Management)
    const rabbitService = {
      apiVersion: 'v1',
      kind: 'Service',
      metadata: { name: rabbitName, namespace: ctx.namespace },
      spec: {
        selector: { app: rabbitName },
        ports: [
          {
            port: 5672,
            targetPort: 5672 as any,
            protocol: 'TCP',
            name: 'amqp',
          },
          {
            port: 15672,
            targetPort: 15672 as any,
            protocol: 'TCP',
            name: 'management',
          },
        ],
        type: 'ClusterIP',
      },
    };

    try {
      await coreApi.replaceNamespacedService({
        name: rabbitName,
        namespace: ctx.namespace,
        body: rabbitService,
      });
    } catch {
      await coreApi.createNamespacedService({
        namespace: ctx.namespace,
        body: rabbitService,
      });
    }

    this.logger.log(`RabbitMQ deployed for ${ctx.appName}`);
  }

  private primaryWorkloadLabel(app: Application): string {
    if (isManagedProductType(app.productType)) {
      switch (app.productType) {
        case ProductType.MANAGED_DATABASE:
          return `${app.name}-db`;
        case ProductType.MANAGED_REDIS:
          return `${app.name}-redis`;
        case ProductType.MANAGED_RABBITMQ:
          return `${app.name}-rabbitmq`;
        default:
          break;
      }
    }
    return app.name;
  }

  async getPodLogs(app: Application): Promise<string> {
    return this.k8sLifecycleService.getPodLogs(app);
  }

  async scaleDeployment(app: Application, replicas: number): Promise<void> {
    const { appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
    const namespace = this.getUserNamespace(app.userId);

    await appsApi.patchNamespacedDeployment({ name: app.name, namespace, body: { spec: { replicas } } }, k8s.setHeaderOptions('Content-Type', 'application/merge-patch+json'));
  }

  /** All K8s Deployments that belong to an application stack (default replica targets). */
  private getApplicationWorkloadDeployments(app: Application): { name: string; runningReplicas: number }[] {
    const managed = isManagedProductType(app.productType);
    const workloads: { name: string; runningReplicas: number }[] = [];

    if (!managed) {
      workloads.push({ name: app.name, runningReplicas: app.replicas || 1 });
    }

    if (app.databaseType && app.databaseType !== DatabaseType.NONE) {
      workloads.push({ name: `${app.name}-db`, runningReplicas: 1 });
    }
    if (app.enableRedis) {
      workloads.push({ name: `${app.name}-redis`, runningReplicas: 1 });
    }
    if (app.enableRabbitmq) {
      workloads.push({ name: `${app.name}-rabbitmq`, runningReplicas: 1 });
    }

    return workloads;
  }

  private resolveWorkloadReplicas(app: Application): { name: string; runningReplicas: number }[] {
    const workloads = this.getApplicationWorkloadDeployments(app);
    const saved = app.suspendedReplicas;
    if (!saved) return workloads;

    return workloads.map((workload) => ({
      name: workload.name,
      runningReplicas: saved[workload.name] ?? workload.runningReplicas,
    }));
  }

  async captureWorkloadReplicaSnapshot(app: Application): Promise<Record<string, number>> {
    const { appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
    const namespace = this.getUserNamespace(app.userId);
    const snapshot: Record<string, number> = {};

    for (const workload of this.getApplicationWorkloadDeployments(app)) {
      try {
        const deployment = await appsApi.readNamespacedDeployment({
          name: workload.name,
          namespace,
        });
        snapshot[workload.name] = deployment.spec?.replicas ?? workload.runningReplicas;
      } catch (e: any) {
        if (e?.code === 404 || e?.response?.statusCode === 404) continue;
        snapshot[workload.name] = workload.runningReplicas;
      }
    }

    return snapshot;
  }

  private async patchDeploymentReplicas(appsApi: k8s.AppsV1Api, namespace: string, deploymentName: string, replicas: number): Promise<void> {
    await appsApi.patchNamespacedDeployment({ name: deploymentName, namespace, body: { spec: { replicas } } }, k8s.setHeaderOptions('Content-Type', 'application/merge-patch+json'));
  }

  /**
   * Suspend an application by scaling all stack deployments to 0 replicas.
   * Returns the replica snapshot captured before scaling.
   */
  async suspendApplication(app: Application): Promise<Record<string, number>> {
    const { appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
    const namespace = this.getUserNamespace(app.userId);

    this.logger.log(`Suspending application ${app.name} in namespace ${namespace}`);

    const snapshot = await this.captureWorkloadReplicaSnapshot(app);
    const workloads = this.getApplicationWorkloadDeployments(app);

    for (const workload of workloads) {
      try {
        await this.patchDeploymentReplicas(appsApi, namespace, workload.name, 0);
        this.logger.log(`Scaled ${workload.name} to 0 replicas`);
      } catch (e: any) {
        if (e?.response?.statusCode !== 404) {
          this.logger.warn(`Failed to scale ${workload.name}: ${e.message}`);
        }
      }
    }

    await this.deleteTemporaryAccessServicesForApp(app);

    return snapshot;
  }

  /**
   * Resume a suspended application using saved replica counts when available.
   */
  async resumeApplication(app: Application): Promise<void> {
    const { appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
    const namespace = this.getUserNamespace(app.userId);

    this.logger.log(`Resuming application ${app.name} in namespace ${namespace}`);

    const workloads = this.resolveWorkloadReplicas(app);
    const dependencies = workloads.filter((w) => w.name !== app.name);
    const main = workloads.find((w) => w.name === app.name);

    for (const workload of dependencies) {
      try {
        await this.patchDeploymentReplicas(appsApi, namespace, workload.name, workload.runningReplicas);
        this.logger.log(`Scaled ${workload.name} to ${workload.runningReplicas} replica(s)`);
      } catch (e: any) {
        if (e?.response?.statusCode !== 404) {
          this.logger.warn(`Failed to scale ${workload.name}: ${e.message}`);
        }
      }
    }

    if (!main) return;

    try {
      await this.patchDeploymentReplicas(appsApi, namespace, main.name, main.runningReplicas);
      this.logger.log(`Scaled ${main.name} to ${main.runningReplicas} replica(s)`);
    } catch (e: any) {
      this.logger.warn(`Failed to scale ${main.name}: ${e.message}`);
      throw e;
    }
  }

  async restartDeployment(app: Application): Promise<void> {
    const { appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
    const namespace = this.getUserNamespace(app.userId);
    const deploymentName = isManagedProductType(app.productType) ? this.primaryWorkloadLabel(app) : app.name;

    await appsApi.patchNamespacedDeployment(
      {
        name: deploymentName,
        namespace,
        body: {
          spec: {
            template: {
              metadata: {
                annotations: {
                  'kubectl.kubernetes.io/restartedAt': new Date().toISOString(),
                },
              },
            },
          },
        },
      },
      k8s.setHeaderOptions('Content-Type', 'application/merge-patch+json'),
    );
  }

  /** Map optional workload keys to Deployment + primary container name */
  workloadDeploymentTarget(app: Application, workload: 'app' | 'database' | 'redis' | 'rabbitmq'): { deploymentName: string; containerName: string } | null {
    switch (workload) {
      case 'app':
        return { deploymentName: app.name, containerName: app.name };
      case 'database':
        if (!app.databaseType || app.databaseType === DatabaseType.NONE) return null;
        return {
          deploymentName: `${app.name}-db`,
          containerName: `${app.name}-db`,
        };
      case 'redis':
        if (!app.enableRedis) return null;
        return { deploymentName: `${app.name}-redis`, containerName: 'redis' };
      case 'rabbitmq':
        if (!app.enableRabbitmq) return null;
        return {
          deploymentName: `${app.name}-rabbitmq`,
          containerName: 'rabbitmq',
        };
      default:
        return null;
    }
  }

  private async fetchNamespacePodMetrics(kc: k8s.KubeConfig, namespace: string, appLabelValue: string): Promise<any[]> {
    try {
      const opts: any = {};
      await kc.applyToHTTPSOptions(opts);
      const metricsUrl = `${kc.getCurrentCluster()?.server}/apis/metrics.k8s.io/v1beta1/namespaces/${namespace}/pods`;
      const https = require('https');
      const http = require('http');
      const url = new URL(metricsUrl);
      const labelQ = `labelSelector=${encodeURIComponent(`app=${appLabelValue}`)}`;

      return await new Promise((resolve) => {
        const client = url.protocol === 'https:' ? https : http;
        const reqOpts: any = {
          hostname: url.hostname,
          port: url.port,
          path: `${url.pathname}?${labelQ}`,
          method: 'GET',
          headers: opts.headers || {},
          rejectUnauthorized: false,
        };
        if (opts.ca) reqOpts.ca = opts.ca;
        if (opts.cert) reqOpts.cert = opts.cert;
        if (opts.key) reqOpts.key = opts.key;

        const req = client.request(reqOpts, (res: any) => {
          let data = '';
          res.on('data', (chunk: string) => (data += chunk));
          res.on('end', () => {
            try {
              const parsed = JSON.parse(data);
              const items = (parsed.items || []).map((item: any) => ({
                name: item.metadata?.name,
                containers: (item.containers || []).map((c: any) => ({
                  name: c.name,
                  cpu: c.usage?.cpu || '0',
                  memory: c.usage?.memory || '0',
                })),
                cpu: item.containers?.[0]?.usage?.cpu || '0',
                memory: item.containers?.[0]?.usage?.memory || '0',
              }));
              resolve(items);
            } catch {
              resolve([]);
            }
          });
        });
        req.on('error', () => resolve([]));
        req.end();
      });
    } catch {
      return [];
    }
  }

  private async getSingleWorkloadUsage(
    app: Application,
    namespace: string,
    appsApi: k8s.AppsV1Api,
    coreApi: k8s.CoreV1Api,
    kc: k8s.KubeConfig,
    opts: {
      key: string;
      title: string;
      deploymentName: string;
      primaryContainerName: string;
      fallbackCpuRequest?: string;
      fallbackCpuLimit?: string;
      fallbackMemoryRequest?: string;
      fallbackMemoryLimit?: string;
      fallbackReplicas?: number;
    },
  ): Promise<any | null> {
    let deployment: k8s.V1Deployment | null = null;
    try {
      const depResponse = await appsApi.readNamespacedDeployment({
        name: opts.deploymentName,
        namespace,
      });
      deployment = depResponse;
    } catch {
      return null;
    }

    const podsResponse = await coreApi.listNamespacedPod({
      namespace,
      labelSelector: `app=${opts.deploymentName}`,
    });

    const pods = podsResponse.items.map((pod) => ({
      name: pod.metadata?.name,
      status: pod.status?.phase,
      ready: pod.status?.conditions?.find((c) => c.type === 'Ready')?.status === 'True',
      restarts: pod.status?.containerStatuses?.reduce((s, c) => s + (c.restartCount || 0), 0) || 0,
      startedAt: pod.status?.startTime,
    }));

    const podMetrics = await this.fetchNamespacePodMetrics(kc, namespace, opts.deploymentName);

    const container = deployment.spec?.template?.spec?.containers?.find((c) => c.name === opts.primaryContainerName) || deployment.spec?.template?.spec?.containers?.[0];

    const sidecars = (deployment.spec?.template?.spec?.containers || [])
      .filter((c) => c.name !== opts.primaryContainerName)
      .map((c) => ({
        name: c.name,
        cpuRequest: c.resources?.requests?.cpu,
        cpuLimit: c.resources?.limits?.cpu,
        memoryRequest: c.resources?.requests?.memory,
        memoryLimit: c.resources?.limits?.memory,
      }));

    const configured = {
      cpuRequest: container?.resources?.requests?.cpu || opts.fallbackCpuRequest,
      cpuLimit: container?.resources?.limits?.cpu || opts.fallbackCpuLimit,
      memoryRequest: container?.resources?.requests?.memory || opts.fallbackMemoryRequest,
      memoryLimit: container?.resources?.limits?.memory || opts.fallbackMemoryLimit,
      replicas: deployment.spec?.replicas ?? opts.fallbackReplicas ?? 1,
      readyReplicas: deployment.status?.readyReplicas || 0,
      availableReplicas: deployment.status?.availableReplicas || 0,
    };

    const metrics = podMetrics.map((m: any) => {
      const matchC = m.containers?.find((c: any) => c.name === opts.primaryContainerName) || m.containers?.[0];
      return {
        name: m.name,
        cpu: matchC?.cpu || m.cpu || '0',
        memory: matchC?.memory || m.memory || '0',
        containers: m.containers,
      };
    });

    return {
      key: opts.key,
      title: opts.title,
      deploymentName: opts.deploymentName,
      configured,
      pods,
      metrics,
      sidecars: sidecars.length ? sidecars : undefined,
    };
  }

  /**
   * Get real-time resource usage (CPU/Memory) for an app's pods via metrics-server.
   * Includes application workload, database, and optional Redis / RabbitMQ when enabled.
   */
  async getResourceUsage(app: Application): Promise<any> {
    const { coreApi, appsApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
    const namespace = this.getUserNamespace(app.userId);

    const workloads: any[] = [];

    const appW = await this.getSingleWorkloadUsage(app, namespace, appsApi, coreApi, kc, {
      key: 'app',
      title: 'Application',
      deploymentName: app.name,
      primaryContainerName: app.name,
      fallbackCpuRequest: app.cpuRequest,
      fallbackCpuLimit: app.cpuLimit,
      fallbackMemoryRequest: app.memoryRequest,
      fallbackMemoryLimit: app.memoryLimit,
      fallbackReplicas: app.replicas,
    });
    if (appW) workloads.push(appW);

    if (app.databaseType && app.databaseType !== DatabaseType.NONE) {
      const dbW = await this.getSingleWorkloadUsage(app, namespace, appsApi, coreApi, kc, {
        key: 'database',
        title: 'Database',
        deploymentName: `${app.name}-db`,
        primaryContainerName: `${app.name}-db`,
        fallbackCpuRequest: '100m',
        fallbackCpuLimit: '500m',
        fallbackMemoryRequest: '256Mi',
        fallbackMemoryLimit: '512Mi',
        fallbackReplicas: 1,
      });
      if (dbW) workloads.push(dbW);
    }

    if (app.enableRedis) {
      const r = await this.getSingleWorkloadUsage(app, namespace, appsApi, coreApi, kc, {
        key: 'redis',
        title: 'Redis',
        deploymentName: `${app.name}-redis`,
        primaryContainerName: 'redis',
        fallbackCpuRequest: '50m',
        fallbackCpuLimit: '200m',
        fallbackMemoryRequest: '64Mi',
        fallbackMemoryLimit: '256Mi',
        fallbackReplicas: 1,
      });
      if (r) workloads.push(r);
    }

    if (app.enableRabbitmq) {
      const mq = await this.getSingleWorkloadUsage(app, namespace, appsApi, coreApi, kc, {
        key: 'rabbitmq',
        title: 'RabbitMQ',
        deploymentName: `${app.name}-rabbitmq`,
        primaryContainerName: 'rabbitmq',
        fallbackCpuRequest: '100m',
        fallbackCpuLimit: '500m',
        fallbackMemoryRequest: '256Mi',
        fallbackMemoryLimit: '512Mi',
        fallbackReplicas: 1,
      });
      if (mq) workloads.push(mq);
    }

    const primary = workloads.find((w) => w.key === 'app') || workloads[0];
    const loggingNote = app.enableElasticsearch ? 'Elasticsearch log forwarding is enabled via Fluent Bit sidecar on application pods.' : undefined;

    return {
      workloads,
      loggingNote,
      configured: primary?.configured,
      pods: primary?.pods || [],
      metrics: primary?.metrics || [],
    };
  }

  /**
   * Update resource limits/requests and replicas on a live K8s deployment.
   * @param workload Which deployment to patch (default: main application). Replicas only apply to `app`.
   */
  async updateResources(
    app: Application,
    resources: {
      cpuRequest?: string;
      cpuLimit?: string;
      memoryRequest?: string;
      memoryLimit?: string;
      replicas?: number;
    },
    workload: 'app' | 'database' | 'redis' | 'rabbitmq' = 'app',
  ): Promise<void> {
    const { appsApi } = await this.k8sClientService.getK8sClient(app.clusterId);
    const namespace = this.getUserNamespace(app.userId);

    const target = this.workloadDeploymentTarget(app, workload);
    if (!target) {
      throw new BadRequestException(workload === 'database' ? 'This application does not have a database deployment.' : `Workload "${workload}" is not enabled for this application.`);
    }

    const patch: any = { spec: {} };

    if (resources.replicas !== undefined && workload === 'app') {
      patch.spec.replicas = resources.replicas;
    }

    if (resources.cpuRequest || resources.cpuLimit || resources.memoryRequest || resources.memoryLimit) {
      patch.spec.template = {
        spec: {
          containers: [
            {
              name: target.containerName,
              resources: {
                requests: {
                  ...(resources.cpuRequest && { cpu: resources.cpuRequest }),
                  ...(resources.memoryRequest && {
                    memory: resources.memoryRequest,
                  }),
                },
                limits: {
                  ...(resources.cpuLimit && { cpu: resources.cpuLimit }),
                  ...(resources.memoryLimit && {
                    memory: resources.memoryLimit,
                  }),
                },
              },
            },
          ],
        },
      };
    }

    await appsApi.patchNamespacedDeployment({ name: target.deploymentName, namespace, body: patch }, k8s.setHeaderOptions('Content-Type', 'application/strategic-merge-patch+json'));

    this.logger.log(`Updated resources for ${target.deploymentName} (${workload}): ${JSON.stringify(resources)}`);
  }

  getUserNamespace(userId: string): string {
    return userNamespace(userId);
  }

  /** Preview URL stays available until the custom domain is verified (not merely requested). */
  private hasVerifiedCustomDomain(app: Application): boolean {
    return !!(app.customDomain && app.customDomainStatus === CustomDomainStatus.VERIFIED);
  }

  private getClusterHostIp(kc: k8s.KubeConfig): string {
    const clusterServer = kc.getCurrentCluster()?.server || '';
    try {
      return new URL(clusterServer).hostname;
    } catch {
      return '127.0.0.1';
    }
  }

  private toDnsLabel(value: string): string {
    return value
      .toLowerCase()
      .replace(/[^a-z0-9-]/g, '-')
      .replace(/-+/g, '-')
      .replace(/^-|-$/g, '')
      .slice(0, 63)
      .replace(/-$/g, '');
  }

  resolveAccessTarget(
    app: Application,
    target: ServiceAccessTarget,
  ): {
    selector: Record<string, string>;
    targetPort: number;
    portName?: string;
  } {
    switch (target) {
      case ServiceAccessTarget.DATABASE: {
        if (!app.databaseType || app.databaseType === DatabaseType.NONE) {
          throw new BadRequestException('Application has no database');
        }
        let targetPort = 3306;
        if (app.databaseType === DatabaseType.POSTGRESQL) targetPort = 5432;
        else if (app.databaseType === DatabaseType.MONGODB) targetPort = 27017;
        return { selector: { app: `${app.name}-db` }, targetPort };
      }
      case ServiceAccessTarget.REDIS:
        if (!app.enableRedis) throw new BadRequestException('Redis is not enabled for this application');
        return { selector: { app: `${app.name}-redis` }, targetPort: 6379 };
      case ServiceAccessTarget.RABBITMQ_AMQP:
        if (!app.enableRabbitmq) throw new BadRequestException('RabbitMQ is not enabled for this application');
        return {
          selector: { app: `${app.name}-rabbitmq` },
          targetPort: 5672,
          portName: 'amqp',
        };
      case ServiceAccessTarget.RABBITMQ_MANAGEMENT:
        if (!app.enableRabbitmq) throw new BadRequestException('RabbitMQ is not enabled for this application');
        return {
          selector: { app: `${app.name}-rabbitmq` },
          targetPort: 15672,
          portName: 'management',
        };
      default:
        throw new BadRequestException(`Unknown access target: ${target}`);
    }
  }

  async createTemporaryAccess(
    app: Application,
    target: ServiceAccessTarget,
    grantId: string,
  ): Promise<{
    host: string;
    nodePort: number;
    k8sServiceName: string;
    targetPort: number;
  }> {
    if (!app.clusterId) {
      throw new BadRequestException('Application is not assigned to a cluster');
    }

    const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
    const namespace = this.getUserNamespace(app.userId);
    const { selector, targetPort, portName } = this.resolveAccessTarget(app, target);
    const shortId = grantId.split('-')[0];
    const k8sServiceName = this.toDnsLabel(`${app.name}-${target}-access-${shortId}`);

    const portSpec: k8s.V1ServicePort = {
      port: targetPort,
      targetPort: targetPort,
      protocol: 'TCP',
    };
    if (portName) portSpec.name = portName;

    const service: k8s.V1Service = {
      apiVersion: 'v1',
      kind: 'Service',
      metadata: {
        name: k8sServiceName,
        namespace,
        labels: {
          app: selector.app,
          'cloudhost.io/access-grant': 'true',
          'cloudhost.io/grant-id': grantId,
          'cloudhost.io/application-id': app.id,
        },
      },
      spec: {
        type: 'NodePort',
        selector,
        ports: [portSpec],
      },
    };

    let created: k8s.V1Service;
    try {
      created = await coreApi.createNamespacedService({
        namespace,
        body: service,
      });
    } catch (error: any) {
      const message = error.body?.message || error.response?.body?.message || error.message || 'HTTP request failed';
      this.logger.warn(`K8s client failed to create temporary access service ${k8sServiceName}; trying kubectl fallback: ${message}`);
      created = await this.applyServiceWithKubectl(app.clusterId, namespace, service);
    }
    const nodePort = created.spec?.ports?.[0]?.nodePort;
    if (!nodePort) {
      try {
        await coreApi.deleteNamespacedService({
          name: k8sServiceName,
          namespace,
        });
      } catch {}
      throw new Error('Failed to allocate NodePort for temporary access');
    }

    const host = this.getClusterHostIp(kc);
    this.logger.log(`Temporary access for ${app.name} target=${target}: ${host}:${nodePort} (service ${k8sServiceName})`);

    return { host, nodePort, k8sServiceName, targetPort };
  }

  private async applyServiceWithKubectl(clusterId: string | undefined, namespace: string, service: k8s.V1Service): Promise<k8s.V1Service> {
    const tmpDir = fs.mkdtempSync(path.join('/tmp', 'cloudhost-access-'));
    const kubeconfigPath = path.join(tmpDir, 'kubeconfig.yaml');
    const manifestPath = path.join(tmpDir, 'service.json');

    try {
      fs.writeFileSync(kubeconfigPath, await this.k8sClientService.getKubeconfig(clusterId), {
        mode: 0o600,
      });
      fs.writeFileSync(manifestPath, JSON.stringify(service), { mode: 0o600 });

      await execFileAsync('kubectl', ['--kubeconfig', kubeconfigPath, 'apply', '-n', namespace, '-f', manifestPath]);

      const { stdout } = await execFileAsync('kubectl', ['--kubeconfig', kubeconfigPath, 'get', 'service', service.metadata!.name!, '-n', namespace, '-o', 'json']);

      return JSON.parse(stdout) as k8s.V1Service;
    } catch (error: any) {
      const message = error.stderr || error.message || 'kubectl failed';
      throw new Error(`Failed to create temporary access service: ${message}`);
    } finally {
      try {
        fs.rmSync(tmpDir, { recursive: true, force: true });
      } catch {}
    }
  }

  async revokeTemporaryAccess(clusterId: string, namespace: string, k8sServiceName: string): Promise<void> {
    const { coreApi } = await this.k8sClientService.getK8sClient(clusterId);
    try {
      await coreApi.deleteNamespacedService({
        name: k8sServiceName,
        namespace,
      });
      this.logger.log(`Revoked temporary access service ${k8sServiceName} in ${namespace}`);
    } catch (e: any) {
      if (e.code !== 404 && e.response?.statusCode !== 404) {
        this.logger.warn(`Failed to delete access service ${k8sServiceName}: ${e.message}`);
      }
    }
  }

  async deleteTemporaryAccessServicesForApp(app: Application): Promise<void> {
    if (!app.clusterId) return;
    const namespace = this.getUserNamespace(app.userId);
    const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);

    try {
      const services = await coreApi.listNamespacedService({
        namespace,
        labelSelector: 'cloudhost.io/access-grant=true',
      });
      for (const svc of services.items) {
        const appId = svc.metadata?.labels?.['cloudhost.io/application-id'];
        if (appId === app.id && svc.metadata?.name) {
          await this.revokeTemporaryAccess(app.clusterId, namespace, svc.metadata.name);
        }
      }
    } catch (e: any) {
      this.logger.warn(`Failed to list temporary access services for ${app.name}: ${e.message}`);
    }
  }

  async readAccessCredentials(app: Application, target: ServiceAccessTarget): Promise<Record<string, string | number | undefined>> {
    const namespace = this.getUserNamespace(app.userId);

    switch (target) {
      case ServiceAccessTarget.DATABASE:
        return {
          username: app.dbUsername || 'appuser',
          password: app.dbPassword || undefined,
          database: app.name.replace(/-/g, '_'),
        };
      case ServiceAccessTarget.REDIS: {
        if (!app.clusterId) return {};
        const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
        const secret = await coreApi.readNamespacedSecret({
          name: `${app.name}-redis-secret`,
          namespace,
        });
        const password = secret.data?.password ? Buffer.from(secret.data.password, 'base64').toString('utf8') : undefined;
        return { password };
      }
      case ServiceAccessTarget.RABBITMQ_AMQP:
      case ServiceAccessTarget.RABBITMQ_MANAGEMENT: {
        if (!app.clusterId) return {};
        const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
        const secret = await coreApi.readNamespacedSecret({
          name: `${app.name}-rabbitmq-secret`,
          namespace,
        });
        const username = secret.data?.username ? Buffer.from(secret.data.username, 'base64').toString('utf8') : 'appuser';
        const password = secret.data?.password ? Buffer.from(secret.data.password, 'base64').toString('utf8') : undefined;
        return { username, password };
      }
      default:
        return {};
    }
  }

  /**
   * Get preview info for a deployed application.
   * Returns ingress URL when available; only reads an existing NodePort (never patches ClusterIP).
   */
  async getPreviewInfo(
    app: Application,
    previewNumber?: string | null,
  ): Promise<{
    url: string;
    nodePort: number;
    host: string;
    ingressUrl?: string;
  }> {
    const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
    const namespace = this.getUserNamespace(app.userId);
    const domain = this.configService.get('platform.domain');
    const hostIp = this.getClusterHostIp(kc);

    const subdomain = app.subdomain || app.name;
    const verifiedCustomDomain = app.customDomain && app.customDomainStatus === CustomDomainStatus.VERIFIED ? app.customDomain : null;
    const previewRootDomain = this.configService.get<string>('platform.previewRootDomain') || domain;
    const namespacePrefix = userIdSlug(app.userId);

    let ingressUrl = `https://${subdomain}.${domain}`;
    if (verifiedCustomDomain) {
      ingressUrl = `https://${verifiedCustomDomain}`;
    } else if (previewNumber) {
      ingressUrl = `https://${namespacePrefix}-${previewNumber}.${previewRootDomain}`;
    }

    let nodePort = 0;
    try {
      const svcResponse = await coreApi.readNamespacedService({
        name: app.name,
        namespace,
      });
      if (svcResponse.spec?.type === 'NodePort') {
        nodePort = svcResponse.spec.ports?.[0]?.nodePort || 0;
      }
    } catch (e: any) {
      this.logger.warn(`Failed to read service for ${app.name}: ${e.message}`);
    }

    const url = ingressUrl || (nodePort > 0 ? `http://${hostIp}:${nodePort}` : '');

    return {
      url,
      nodePort,
      host: hostIp,
      ingressUrl,
    };
  }

  async deleteApplication(app: Application): Promise<void> {
    const namespace = this.getUserNamespace(app.userId);
    const { coreApi, appsApi, networkingApi } = await this.k8sClientService.getK8sClient(app.clusterId);

    await this.deleteTemporaryAccessServicesForApp(app);

    // Step 1: Try Helm uninstall (handles most resources)
    try {
      const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
      await this.helmService.uninstall(app.name, namespace, kubeconfig);
      this.logger.log(`Helm release ${app.name} uninstalled from ${namespace}`);
    } catch (error: any) {
      this.logger.warn(`Helm uninstall failed for ${app.name}: ${error.message} — proceeding with manual cleanup`);
      // Manual cleanup of core resources if helm wasn't managing them
      try {
        await appsApi.deleteNamespacedDeployment({ name: app.name, namespace });
      } catch {}
      try {
        await coreApi.deleteNamespacedService({ name: app.name, namespace });
      } catch {}
      try {
        await networkingApi.deleteNamespacedIngress({
          name: app.name,
          namespace,
        });
      } catch {}
    }

    // Step 2: Delete resources with "helm.sh/resource-policy: keep" (PVCs, db-secret)
    // These survive helm uninstall by design, so we must delete them explicitly
    const dbName = `${app.name}-db`;
    const resourcesToDelete = [
      // Database resources
      () => appsApi.deleteNamespacedDeployment({ name: dbName, namespace }),
      () => coreApi.deleteNamespacedService({ name: dbName, namespace }),
      () =>
        coreApi.deleteNamespacedPersistentVolumeClaim({
          name: dbName,
          namespace,
        }),
      () =>
        coreApi.deleteNamespacedSecret({
          name: `${app.name}-db-secret`,
          namespace,
        }),
      // App storage PVC (all app types)
      () =>
        coreApi.deleteNamespacedPersistentVolumeClaim({
          name: `${app.name}-storage`,
          namespace,
        }),
      // Legacy: also try deleting old wp-content PVC name for backward compatibility
      () =>
        coreApi.deleteNamespacedPersistentVolumeClaim({
          name: `${app.name}-wp-content`,
          namespace,
        }),
      // App env secret
      () => coreApi.deleteNamespacedSecret({ name: `${app.name}-env`, namespace }),
      // Fluent Bit config (if elasticsearch was enabled)
      () =>
        coreApi.deleteNamespacedConfigMap({
          name: `${app.name}-fluent-bit-config`,
          namespace,
        }),
      // Redis resources
      () =>
        appsApi.deleteNamespacedDeployment({
          name: `${app.name}-redis`,
          namespace,
        }),
      () =>
        coreApi.deleteNamespacedService({
          name: `${app.name}-redis`,
          namespace,
        }),
      () =>
        coreApi.deleteNamespacedPersistentVolumeClaim({
          name: `${app.name}-redis-data`,
          namespace,
        }),
      () =>
        coreApi.deleteNamespacedSecret({
          name: `${app.name}-redis-secret`,
          namespace,
        }),
      // RabbitMQ resources
      () =>
        appsApi.deleteNamespacedDeployment({
          name: `${app.name}-rabbitmq`,
          namespace,
        }),
      () =>
        coreApi.deleteNamespacedService({
          name: `${app.name}-rabbitmq`,
          namespace,
        }),
      () =>
        coreApi.deleteNamespacedPersistentVolumeClaim({
          name: `${app.name}-rabbitmq-data`,
          namespace,
        }),
      () =>
        coreApi.deleteNamespacedSecret({
          name: `${app.name}-rabbitmq-secret`,
          namespace,
        }),
      // TLS secret created by cert-manager
      () => coreApi.deleteNamespacedSecret({ name: `${app.name}-tls`, namespace }),
    ];

    for (const deleteFn of resourcesToDelete) {
      try {
        await deleteFn();
      } catch {}
    }

    this.logger.log(`All K8s resources cleaned up for ${app.name} in ${namespace}`);
  }

  async prepareApplicationMigration(
    app: Application,
    targetClusterId: string,
    options: {
      migrateStorage?: boolean;
      log?: (step: string, message: string, metadata?: Record<string, any>) => Promise<void>;
    } = {},
  ): Promise<void> {
    const namespace = this.getUserNamespace(app.userId);
    const source = await this.k8sClientService.getK8sClient(app.clusterId);
    const target = await this.k8sClientService.getK8sClient(targetClusterId);

    await this.ensureNamespaceOnCluster(target.coreApi, namespace);
    await options.log?.('transfer-secrets-configs', 'Target namespace ensured', { namespace });

    const secretNames = [`${app.name}-env`, `${app.name}-db-secret`, `${app.name}-redis-secret`, `${app.name}-rabbitmq-secret`, `${app.name}-tls`];
    for (const secretName of secretNames) {
      try {
        const secret = await source.coreApi.readNamespacedSecret({
          name: secretName,
          namespace,
        });
        await this.upsertSecret(target.coreApi, namespace, this.cleanK8sObject(secret));
        await options.log?.('transfer-secrets-configs', `Secret ${secretName} copied`);
      } catch (error: any) {
        if (!this.isK8sNotFound(error)) {
          throw error;
        }
      }
    }

    const configMapNames = [`${app.name}-fluent-bit-config`];
    for (const configMapName of configMapNames) {
      try {
        const configMap = await source.coreApi.readNamespacedConfigMap({
          name: configMapName,
          namespace,
        });
        await this.upsertConfigMap(target.coreApi, namespace, this.cleanK8sObject(configMap));
        await options.log?.('transfer-secrets-configs', `ConfigMap ${configMapName} copied`);
      } catch (error: any) {
        if (!this.isK8sNotFound(error)) {
          throw error;
        }
      }
    }

    if (options.migrateStorage !== false) {
      const pvcNames = [`${app.name}-storage`, `${app.name}-wp-content`, `${app.name}-db`, `${app.name}-redis-data`, `${app.name}-rabbitmq-data`];
      for (const pvcName of pvcNames) {
        try {
          const pvc = await source.coreApi.readNamespacedPersistentVolumeClaim({
            name: pvcName,
            namespace,
          });
          await this.upsertPvcDefinition(target.coreApi, namespace, this.cleanPvcForMigration(pvc));
          await options.log?.('transfer-volumes', `PVC definition ${pvcName} prepared on target cluster`, {
            note: 'Starting best-effort PVC data copy through migration helper pods.',
          });
          await this.copyPvcDataBetweenClusters(app.clusterId!, targetClusterId, namespace, pvcName, options.log);
        } catch (error: any) {
          if (!this.isK8sNotFound(error)) {
            throw error;
          }
        }
      }
    }
  }

  private async copyPvcDataBetweenClusters(
    sourceClusterId: string,
    targetClusterId: string,
    namespace: string,
    pvcName: string,
    log?: (step: string, message: string, metadata?: Record<string, any>) => Promise<void>,
  ): Promise<void> {
    const safeName = pvcName
      .replace(/[^a-z0-9-]/gi, '-')
      .toLowerCase()
      .slice(0, 32);
    const suffix = `${Date.now()}`.slice(-6);
    const sourcePod = `migrate-src-${safeName}-${suffix}`;
    const targetPod = `migrate-dst-${safeName}-${suffix}`;
    const tempDir = path.join('/tmp', `app-migration-${safeName}-${suffix}`);
    const sourceKubeconfig = path.join(tempDir, 'source.kubeconfig');
    const targetKubeconfig = path.join(tempDir, 'target.kubeconfig');

    fs.mkdirSync(tempDir, { recursive: true });
    fs.writeFileSync(sourceKubeconfig, await this.k8sClientService.getKubeconfig(sourceClusterId), { mode: 0o600 });
    fs.writeFileSync(targetKubeconfig, await this.k8sClientService.getKubeconfig(targetClusterId), { mode: 0o600 });

    try {
      await this.createPvcCopyPod(sourceKubeconfig, namespace, sourcePod, pvcName);
      await this.createPvcCopyPod(targetKubeconfig, namespace, targetPod, pvcName);
      await log?.('transfer-volumes', `Copy helper pods ready for PVC ${pvcName}`);

      const localDataPath = path.join(tempDir, 'data');
      await execFileAsync('kubectl', ['--kubeconfig', sourceKubeconfig, '-n', namespace, 'cp', `${sourcePod}:/data`, localDataPath], {
        timeout: 30 * 60 * 1000,
      });
      await execFileAsync('kubectl', ['--kubeconfig', targetKubeconfig, '-n', namespace, 'cp', `${localDataPath}/.`, `${targetPod}:/data`], {
        timeout: 30 * 60 * 1000,
      });
      await log?.('transfer-volumes', `PVC data copied for ${pvcName}`);
    } finally {
      await execFileAsync('kubectl', ['--kubeconfig', sourceKubeconfig, '-n', namespace, 'delete', 'pod', sourcePod, '--ignore-not-found=true']).catch(() => undefined);
      await execFileAsync('kubectl', ['--kubeconfig', targetKubeconfig, '-n', namespace, 'delete', 'pod', targetPod, '--ignore-not-found=true']).catch(() => undefined);
      fs.rmSync(tempDir, { recursive: true, force: true });
    }
  }

  private async createPvcCopyPod(kubeconfigPath: string, namespace: string, podName: string, pvcName: string): Promise<void> {
    const manifestPath = path.join('/tmp', `${podName}.json`);
    const manifest = {
      apiVersion: 'v1',
      kind: 'Pod',
      metadata: { name: podName, namespace },
      spec: {
        restartPolicy: 'Never',
        containers: [
          {
            name: 'copy',
            image: 'busybox:1.36',
            command: ['sh', '-c', 'mkdir -p /data && sleep 3600'],
            volumeMounts: [{ name: 'data', mountPath: '/data' }],
          },
        ],
        volumes: [{ name: 'data', persistentVolumeClaim: { claimName: pvcName } }],
      },
    };

    fs.writeFileSync(manifestPath, JSON.stringify(manifest));
    try {
      await execFileAsync('kubectl', ['--kubeconfig', kubeconfigPath, 'apply', '-f', manifestPath], { timeout: 120_000 });
      await execFileAsync('kubectl', ['--kubeconfig', kubeconfigPath, '-n', namespace, 'wait', '--for=condition=Ready', `pod/${podName}`, '--timeout=180s'], { timeout: 210_000 });
    } finally {
      fs.rmSync(manifestPath, { force: true });
    }
  }

  private async ensureNamespaceOnCluster(coreApi: k8s.CoreV1Api, namespace: string): Promise<void> {
    try {
      await coreApi.readNamespace({ name: namespace });
    } catch (error: any) {
      if (this.isK8sNotFound(error)) {
        await coreApi.createNamespace({
          body: { metadata: { name: namespace } },
        });
        return;
      }
      throw error;
    }
  }

  private async upsertSecret(coreApi: k8s.CoreV1Api, namespace: string, secret: k8s.V1Secret): Promise<void> {
    secret.metadata = { ...(secret.metadata || {}), namespace };
    try {
      await coreApi.replaceNamespacedSecret({
        name: secret.metadata.name!,
        namespace,
        body: secret,
      });
    } catch (error: any) {
      if (this.isK8sNotFound(error)) {
        await coreApi.createNamespacedSecret({ namespace, body: secret });
        return;
      }
      throw error;
    }
  }

  private async upsertConfigMap(coreApi: k8s.CoreV1Api, namespace: string, configMap: k8s.V1ConfigMap): Promise<void> {
    configMap.metadata = { ...(configMap.metadata || {}), namespace };
    try {
      await coreApi.replaceNamespacedConfigMap({
        name: configMap.metadata.name!,
        namespace,
        body: configMap,
      });
    } catch (error: any) {
      if (this.isK8sNotFound(error)) {
        await coreApi.createNamespacedConfigMap({ namespace, body: configMap });
        return;
      }
      throw error;
    }
  }

  private async upsertPvcDefinition(coreApi: k8s.CoreV1Api, namespace: string, pvc: k8s.V1PersistentVolumeClaim): Promise<void> {
    pvc.metadata = { ...(pvc.metadata || {}), namespace };
    try {
      await coreApi.readNamespacedPersistentVolumeClaim({
        name: pvc.metadata.name!,
        namespace,
      });
    } catch (error: any) {
      if (this.isK8sNotFound(error)) {
        await coreApi.createNamespacedPersistentVolumeClaim({
          namespace,
          body: pvc,
        });
        return;
      }
      throw error;
    }
  }

  private cleanK8sObject<T extends { metadata?: k8s.V1ObjectMeta }>(obj: T): T {
    const metadata = { ...(obj.metadata || {}) };
    delete metadata.uid;
    delete metadata.resourceVersion;
    delete metadata.generation;
    delete metadata.creationTimestamp;
    delete metadata.managedFields;
    delete metadata.selfLink;
    return {
      ...obj,
      metadata,
    };
  }

  private cleanPvcForMigration(pvc: k8s.V1PersistentVolumeClaim): k8s.V1PersistentVolumeClaim {
    const cleaned = this.cleanK8sObject(pvc);
    return {
      apiVersion: cleaned.apiVersion,
      kind: cleaned.kind,
      metadata: cleaned.metadata,
      spec: {
        accessModes: cleaned.spec?.accessModes,
        resources: cleaned.spec?.resources,
        storageClassName: cleaned.spec?.storageClassName,
        volumeMode: cleaned.spec?.volumeMode,
      },
    };
  }

  private isK8sNotFound(error: any): boolean {
    return error?.statusCode === 404 || error?.body?.code === 404;
  }

  /**
   * Wait for the database pod to become Ready.
   * Polls pod status with label selector `app=<appName>-db`.
   */
  async waitForDatabaseReady(app: Application, timeoutMs = 120_000): Promise<void> {
    const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
    const namespace = this.getUserNamespace(app.userId);
    const dbLabel = `${app.name}-db`;
    const start = Date.now();

    this.logger.log(`Waiting for DB pod (app=${dbLabel}) to become Ready in ${namespace}...`);

    while (Date.now() - start < timeoutMs) {
      try {
        const pods = await coreApi.listNamespacedPod({
          namespace,
          labelSelector: `app=${dbLabel}`,
        });
        for (const pod of pods.items) {
          const readyCond = (pod.status?.conditions || []).find((c) => c.type === 'Ready');
          if (readyCond?.status === 'True') {
            this.logger.log(`DB pod ${pod.metadata?.name} is Ready (${Date.now() - start}ms)`);
            return;
          }
        }
      } catch {
        // Pods may not exist yet
      }
      await new Promise((r) => setTimeout(r, 3000));
    }
    this.logger.warn(`DB pod did not become Ready within ${timeoutMs / 1000}s — proceeding anyway`);
  }

  private async getDeploymentReadiness(
    appsApi: k8s.AppsV1Api,
    namespace: string,
    name: string,
    expectedReplicas: number,
  ): Promise<{
    name: string;
    desiredReplicas: number;
    readyReplicas: number;
    availableReplicas: number;
    ready: boolean;
  }> {
    try {
      const response = await appsApi.readNamespacedDeployment({
        name,
        namespace,
      });
      const deployment = response;
      const desiredReplicas = deployment.spec?.replicas ?? expectedReplicas;
      const readyReplicas = deployment.status?.readyReplicas ?? 0;
      const availableReplicas = deployment.status?.availableReplicas ?? 0;
      const observedGeneration = deployment.status?.observedGeneration ?? 0;
      const generation = deployment.metadata?.generation ?? 0;

      return {
        name,
        desiredReplicas,
        readyReplicas,
        availableReplicas,
        ready: desiredReplicas === 0 || (readyReplicas >= desiredReplicas && availableReplicas >= desiredReplicas && observedGeneration >= generation),
      };
    } catch {
      return {
        name,
        desiredReplicas: expectedReplicas,
        readyReplicas: 0,
        availableReplicas: 0,
        ready: false,
      };
    }
  }

  private async describeWorkloadPods(coreApi: k8s.CoreV1Api, namespace: string, workloadNames: string[]): Promise<string> {
    const podLines: string[] = [];

    for (const workloadName of workloadNames) {
      try {
        const pods = await coreApi.listNamespacedPod({
          namespace,
          labelSelector: `app=${workloadName}`,
        });

        for (const pod of pods.items) {
          const ready = pod.status?.conditions?.find((c) => c.type === 'Ready')?.status === 'True';
          const scheduled = pod.status?.conditions?.find((c) => c.type === 'PodScheduled');
          const waitingReasons = (pod.status?.containerStatuses || [])
            .map((status) => status.state?.waiting?.reason)
            .filter(Boolean)
            .join('/');
          const reason = waitingReasons || scheduled?.reason || pod.status?.reason || pod.status?.phase || 'Unknown';
          const message = scheduled?.message || pod.status?.message || '';

          podLines.push(`${pod.metadata?.name || workloadName}: phase=${pod.status?.phase || 'Unknown'}, ready=${ready}, reason=${reason}${message ? ` (${message})` : ''}`);
        }
      } catch {
        podLines.push(`${workloadName}: pods unavailable`);
      }
    }

    return podLines.length ? `Pods: ${podLines.join('; ')}` : 'No pods found for expected workloads.';
  }

  /**
   * Restore a SQL dump file into the application's database.
   * Uses a PVC + helper pod + kubectl cp to transfer the dump (supports large files),
   * then runs a restore Job that mounts the PVC and imports the dump.
   */
  async restoreDatabaseDump(app: Application, dumpFilePath: string): Promise<{ success: boolean; logs: string }> {
    const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
    const batchApi = kc.makeApiClient(k8s.BatchV1Api);
    const namespace = this.getUserNamespace(app.userId);
    const dbName = `${app.name}-db`;
    const ts = Date.now();
    const pvcName = `${app.name}-db-dump-${ts}`;
    const helperPodName = `${pvcName}-helper`;
    const jobName = `${app.name}-db-restore-${ts}`;

    const dumpSize = fs.statSync(dumpFilePath).size;
    const pvcSizeGi = Math.max(1, Math.ceil((dumpSize * 2) / (1024 * 1024 * 1024)));

    this.logger.log(`Restoring DB dump for ${app.name}: ${(dumpSize / 1024 / 1024).toFixed(1)} MB via PVC`);

    // ── 1. Create PVC for the dump file ──
    await coreApi.createNamespacedPersistentVolumeClaim({
      namespace,
      body: {
        apiVersion: 'v1',
        kind: 'PersistentVolumeClaim',
        metadata: { name: pvcName, namespace },
        spec: {
          accessModes: ['ReadWriteOnce'],
          resources: { requests: { storage: `${pvcSizeGi}Gi` } },
        },
      },
    });

    // ── 2. Helper pod to receive the dump via kubectl cp ──
    const helperPod: k8s.V1Pod = {
      apiVersion: 'v1',
      kind: 'Pod',
      metadata: { name: helperPodName, namespace },
      spec: {
        containers: [
          {
            name: 'helper',
            image: 'busybox:1.36',
            command: ['sh', '-c', 'sleep 3600'],
            volumeMounts: [{ name: 'dump', mountPath: '/data' }],
            resources: {
              requests: { cpu: '100m', memory: '128Mi' },
              limits: { cpu: '500m', memory: '256Mi' },
            },
          },
        ],
        volumes: [{ name: 'dump', persistentVolumeClaim: { claimName: pvcName } }],
        restartPolicy: 'Never',
      },
    };

    await coreApi.createNamespacedPod({ namespace, body: helperPod });

    // Wait for helper pod Running
    const podTimeout = 120_000;
    const podStart = Date.now();
    while (Date.now() - podStart < podTimeout) {
      const pod = await coreApi.readNamespacedPod({
        name: helperPodName,
        namespace,
      });
      if (pod.status?.phase === 'Running') break;
      if (pod.status?.phase === 'Failed') throw new Error('DB dump helper pod failed to start');
      await new Promise((r) => setTimeout(r, 2000));
    }

    // ── 3. kubectl cp the dump file into the helper pod ──
    const tmpKubeconfig = path.join('/tmp', `kubeconfig-dbdump-${ts}.yaml`);
    const kcYaml = kc.exportConfig();
    fs.writeFileSync(tmpKubeconfig, kcYaml);

    // Verify tar is available inside the helper pod
    try {
      await execFileAsync('kubectl', ['--kubeconfig', tmpKubeconfig, 'exec', '-n', namespace, helperPodName, '--', 'tar', '--version'], { timeout: 30_000 });
    } catch (e: any) {
      this.logger.warn(`tar check in DB dump helper pod failed: ${e.message}`);
    }

    try {
      await execFileAsync('kubectl', ['--kubeconfig', tmpKubeconfig, 'cp', dumpFilePath, `${namespace}/${helperPodName}:/data/dump.sql`, '--retries', '3'], {
        maxBuffer: 50 * 1024 * 1024,
        timeout: 600_000,
      });
      this.logger.log(`kubectl cp dump completed (${(dumpSize / 1024 / 1024).toFixed(1)} MB)`);
    } finally {
      try {
        fs.unlinkSync(tmpKubeconfig);
      } catch {}
      try {
        await coreApi.deleteNamespacedPod({ name: helperPodName, namespace });
      } catch {}
    }

    // ── 4. Build restore command (per database type) ──
    const { image, restoreCommand: command } = this.databaseDumpSpec(app, dbName);

    // ── 5. Create the restore Job ──
    const job: k8s.V1Job = {
      apiVersion: 'batch/v1',
      kind: 'Job',
      metadata: { name: jobName, namespace },
      spec: {
        ttlSecondsAfterFinished: 300,
        backoffLimit: 1,
        template: {
          spec: {
            restartPolicy: 'Never',
            containers: [
              {
                name: 'restore',
                image,
                command,
                env: [
                  {
                    name: 'DB_USER',
                    valueFrom: {
                      secretKeyRef: {
                        name: `${app.name}-db-secret`,
                        key: 'username',
                      },
                    },
                  },
                  {
                    name: 'DB_PASSWORD',
                    valueFrom: {
                      secretKeyRef: {
                        name: `${app.name}-db-secret`,
                        key: 'password',
                      },
                    },
                  },
                ],
                volumeMounts: [{ name: 'dump-volume', mountPath: '/dump', readOnly: true }],
                resources: {
                  requests: { cpu: '200m', memory: '256Mi' },
                  limits: { cpu: '1', memory: '1Gi' },
                },
              },
            ],
            volumes: [
              {
                name: 'dump-volume',
                persistentVolumeClaim: { claimName: pvcName },
              },
            ],
          },
        },
      },
    };

    try {
      await batchApi.createNamespacedJob({ namespace, body: job });
      this.logger.log(`Created DB restore job ${jobName} for ${app.name}`);
    } catch (e: any) {
      try {
        await coreApi.deleteNamespacedPersistentVolumeClaim({
          name: pvcName,
          namespace,
        });
      } catch {}
      this.logger.error(`Failed to create restore job: ${e.message}`);
      throw new Error('Failed to create database restore job');
    }

    // ── 6. Wait for the job to complete (max 10 minutes for large dumps) ──
    const timeout = 600_000;
    const start = Date.now();
    let completed = false;
    let failed = false;

    while (Date.now() - start < timeout) {
      await new Promise((r) => setTimeout(r, 5000));
      try {
        const jobStatus = await batchApi.readNamespacedJob({
          name: jobName,
          namespace,
        });
        const status = jobStatus.status;
        if (status?.succeeded && status.succeeded > 0) {
          completed = true;
          break;
        }
        if (status?.failed && status.failed > 0) {
          failed = true;
          break;
        }
      } catch {
        // Job may not be ready yet
      }
    }

    // ── 7. Get logs from the restore job pod ──
    let logs = '';
    try {
      const pods = await coreApi.listNamespacedPod({
        namespace,
        labelSelector: `job-name=${jobName}`,
      });
      if (pods.items.length > 0) {
        const podName = pods.items[0].metadata?.name;
        if (podName) {
          const logResponse = await coreApi.readNamespacedPodLog({
            name: podName,
            namespace,
          });
          logs = logResponse || '';
        }
      }
    } catch (e: any) {
      this.logger.warn(`Could not get restore job logs: ${e.message}`);
    }

    // ── 8. Clean up the dump PVC ──
    try {
      await coreApi.deleteNamespacedPersistentVolumeClaim({
        name: pvcName,
        namespace,
      });
      this.logger.log(`Cleaned up dump PVC: ${pvcName}`);
    } catch {}

    if (!completed && !failed) {
      this.logger.warn(`DB restore job ${jobName} timed out`);
      return {
        success: false,
        logs: logs || 'Restore job timed out after 10 minutes',
      };
    }

    if (failed) {
      this.logger.warn(`DB restore job ${jobName} failed`);
      return { success: false, logs: logs || 'Restore job failed' };
    }

    this.logger.log(`DB restore for ${app.name} completed successfully`);
    return { success: true, logs };
  }

  private isPvcResizeForbiddenError(err: unknown): boolean {
    const msg = (err as { body?: { message?: string }; message?: string })?.body?.message || (err as Error)?.message || '';
    return /forbidden|resize|storageclass|dynamically provisioned/i.test(msg);
  }

  private async resolvePvcStorageClassName(coreApi: k8s.CoreV1Api, pvc: k8s.V1PersistentVolumeClaim): Promise<string | undefined> {
    let scName = pvc.spec?.storageClassName;
    if (scName) return scName;
    const volumeName = pvc.spec?.volumeName;
    if (!volumeName) return undefined;
    try {
      const pv = await coreApi.readPersistentVolume({ name: volumeName });
      scName = pv.spec?.storageClassName;
      if (pv.spec?.hostPath || pv.spec?.nfs || pv.spec?.local) {
        return undefined;
      }
      return scName;
    } catch {
      return undefined;
    }
  }

  private async ensureStorageClassAllowsExpansion(kc: k8s.KubeConfig, storageClassName: string): Promise<{ ok: boolean; message?: string }> {
    const storageApi = kc.makeApiClient(k8s.StorageV1Api);
    try {
      const sc = await storageApi.readStorageClass({ name: storageClassName });
      if (sc.allowVolumeExpansion) {
        return { ok: true };
      }
      await storageApi.patchStorageClass({ name: storageClassName, body: { allowVolumeExpansion: true } }, k8s.setHeaderOptions('Content-Type', 'application/strategic-merge-patch+json'));
      this.logger.log(`Enabled allowVolumeExpansion on StorageClass ${storageClassName}`);
      return { ok: true };
    } catch (e: any) {
      return {
        ok: false,
        message: `StorageClass "${storageClassName}" does not support expansion: ${e.message}`,
      };
    }
  }

  private async patchPvcStorageSize(coreApi: k8s.CoreV1Api, pvcName: string, namespace: string, newSize: string): Promise<void> {
    await coreApi.patchNamespacedPersistentVolumeClaim(
      {
        name: pvcName,
        namespace,
        body: [
          {
            op: 'replace',
            path: '/spec/resources/requests/storage',
            value: newSize,
          },
        ],
      },
      k8s.setHeaderOptions('Content-Type', 'application/json-patch+json'),
    );
  }

  /**
   * Migrate DB PVC to a resize-capable StorageClass (one-time copy).
   * Used when legacy PVCs were created without storageClassName.
   */
  private async migrateDatabasePvcToResizableStorage(app: Application, newSize: string, storageClassName: string): Promise<{ success: boolean; message: string }> {
    const { coreApi, appsApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
    const batchApi = kc.makeApiClient(k8s.BatchV1Api);
    const namespace = this.getUserNamespace(app.userId);
    const oldPvcName = `${app.name}-db`;
    const newPvcName = `${app.name}-db-resizable`;
    const deploymentName = `${app.name}-db`;

    try {
      try {
        await coreApi.readNamespacedPersistentVolumeClaim({
          name: newPvcName,
          namespace,
        });
      } catch {
        await coreApi.createNamespacedPersistentVolumeClaim({
          namespace,
          body: {
            apiVersion: 'v1',
            kind: 'PersistentVolumeClaim',
            metadata: { name: newPvcName, namespace },
            spec: {
              accessModes: ['ReadWriteOnce'],
              storageClassName,
              resources: { requests: { storage: newSize } },
            },
          },
        });
      }

      await appsApi.patchNamespacedDeployment({ name: deploymentName, namespace, body: { spec: { replicas: 0 } } }, k8s.setHeaderOptions('Content-Type', 'application/strategic-merge-patch+json'));
      await this.waitForDeploymentReplicas(appsApi, namespace, deploymentName, 0, 120_000);

      const jobName = `${app.name}-pvc-migrate-${Date.now()}`;
      await batchApi.createNamespacedJob({
        namespace,
        body: {
          apiVersion: 'batch/v1',
          kind: 'Job',
          metadata: { name: jobName, namespace },
          spec: {
            ttlSecondsAfterFinished: 300,
            backoffLimit: 1,
            template: {
              spec: {
                restartPolicy: 'Never',
                containers: [
                  {
                    name: 'copy',
                    image: 'busybox:1.36',
                    command: ['sh', '-c', 'set -e; mkdir -p /dest; if [ -d /src ] && [ "$(ls -A /src 2>/dev/null)" ]; then cp -a /src/. /dest/; fi; touch /dest/.cloudhost-migrated'],
                    volumeMounts: [
                      { name: 'src', mountPath: '/src', readOnly: true },
                      { name: 'dest', mountPath: '/dest' },
                    ],
                  },
                ],
                volumes: [
                  {
                    name: 'src',
                    persistentVolumeClaim: { claimName: oldPvcName },
                  },
                  {
                    name: 'dest',
                    persistentVolumeClaim: { claimName: newPvcName },
                  },
                ],
              },
            },
          },
        },
      });

      const jobOk = await this.waitForJobComplete(batchApi, coreApi, namespace, jobName, 600_000);
      if (!jobOk) {
        return {
          success: false,
          message: 'Storage migration job failed or timed out. Database was scaled down; check cluster jobs.',
        };
      }

      const depRes = await appsApi.readNamespacedDeployment({
        name: deploymentName,
        namespace,
      });
      const existingVolumes = depRes.spec?.template?.spec?.volumes || [];
      const updatedVolumes = existingVolumes.map((vol) => {
        if (vol.name === 'db-storage' && vol.persistentVolumeClaim) {
          return { ...vol, persistentVolumeClaim: { claimName: newPvcName } };
        }
        return vol;
      });
      if (!updatedVolumes.some((v) => v.name === 'db-storage')) {
        updatedVolumes.push({
          name: 'db-storage',
          persistentVolumeClaim: { claimName: newPvcName },
        });
      }

      await appsApi.patchNamespacedDeployment(
        {
          name: deploymentName,
          namespace,
          body: {
            spec: {
              replicas: 1,
              template: { spec: { volumes: updatedVolumes } },
            },
          },
        },
        k8s.setHeaderOptions('Content-Type', 'application/strategic-merge-patch+json'),
      );

      try {
        await coreApi.deleteNamespacedPersistentVolumeClaim({
          name: oldPvcName,
          namespace,
        });
      } catch {
        this.logger.warn(`Could not delete old PVC ${oldPvcName} after migration`);
      }

      this.logger.log(`Migrated database PVC ${oldPvcName} → ${newPvcName} (${newSize})`);
      return {
        success: true,
        message: `Database storage migrated to expandable disk and set to ${newSize}. A brief restart was required.`,
      };
    } catch (e: any) {
      this.logger.error(`PVC migration failed for ${app.name}: ${e.message}`);
      try {
        await appsApi.patchNamespacedDeployment({ name: deploymentName, namespace, body: { spec: { replicas: 1 } } }, k8s.setHeaderOptions('Content-Type', 'application/strategic-merge-patch+json'));
      } catch {}
      return {
        success: false,
        message: e.body?.message || e.message || 'Failed to migrate database storage',
      };
    }
  }

  private async waitForDeploymentReplicas(appsApi: k8s.AppsV1Api, namespace: string, name: string, target: number, timeoutMs: number): Promise<boolean> {
    const start = Date.now();
    while (Date.now() - start < timeoutMs) {
      try {
        const dep = await appsApi.readNamespacedDeployment({ name, namespace });
        const ready = dep.status?.readyReplicas ?? 0;
        const replicas = dep.spec?.replicas ?? 0;
        if (target === 0 && replicas === 0) return true;
        if (target > 0 && ready >= target && replicas >= target) return true;
      } catch {}
      await new Promise((r) => setTimeout(r, 3000));
    }
    return false;
  }

  private async waitForJobComplete(batchApi: k8s.BatchV1Api, coreApi: k8s.CoreV1Api, namespace: string, jobName: string, timeoutMs: number): Promise<boolean> {
    const start = Date.now();
    while (Date.now() - start < timeoutMs) {
      try {
        const job = await batchApi.readNamespacedJob({
          name: jobName,
          namespace,
        });
        const succeeded = job.status?.succeeded ?? 0;
        const failed = job.status?.failed ?? 0;
        if (succeeded > 0) return true;
        if (failed > 0) return false;
      } catch {}
      await new Promise((r) => setTimeout(r, 3000));
    }
    return false;
  }

  /**
   * Resize (expand) the database PVC for an application.
   * K8s only supports PVC expansion, not shrinking.
   */
  async resizeDatabasePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
    const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
    const namespace = this.getUserNamespace(app.userId);
    const pvcName = `${app.name}-db`;

    try {
      const currentPvc = await coreApi.readNamespacedPersistentVolumeClaim({
        name: pvcName,
        namespace,
      });
      const currentSize = currentPvc.spec?.resources?.requests?.storage || '1Gi';

      const currentGi = parseInt(String(currentSize).replace(/Gi/i, ''), 10) || 1;
      const newGi = parseInt(String(newSize).replace(/Gi/i, ''), 10) || 1;

      if (newGi <= currentGi) {
        return {
          success: false,
          message: `New size (${newSize}) must be larger than current size (${currentSize})`,
        };
      }

      const scName = await this.resolvePvcStorageClassName(coreApi, currentPvc);
      if (scName) {
        const scCheck = await this.ensureStorageClassAllowsExpansion(kc, scName);
        if (!scCheck.ok) {
          return {
            success: false,
            message: scCheck.message || 'StorageClass does not allow expansion',
          };
        }
      }

      try {
        await this.patchPvcStorageSize(coreApi, pvcName, namespace, newSize);
        this.logger.log(`Resized PVC ${pvcName} from ${currentSize} to ${newSize}`);
        return {
          success: true,
          message: `Database storage expanded from ${currentSize} to ${newSize}`,
        };
      } catch (patchErr: any) {
        if (!this.isPvcResizeForbiddenError(patchErr)) {
          throw patchErr;
        }
        const targetSc = this.configService.get<string>('platform.storageClass');
        if (!targetSc) {
          return {
            success: false,
            message: 'This disk cannot be expanded in place. Set PLATFORM_STORAGE_CLASS (e.g. cloudhost-expandable) and redeploy, or contact support.',
          };
        }
        if (pvcName.endsWith('-resizable')) {
          return {
            success: false,
            message: patchErr.body?.message || patchErr.message || 'Failed to resize database storage',
          };
        }
        this.logger.warn(`In-place resize failed for ${pvcName}, migrating to StorageClass ${targetSc}`);
        return this.migrateDatabasePvcToResizableStorage(app, newSize, targetSc);
      }
    } catch (e: any) {
      this.logger.error(`Failed to resize PVC ${pvcName}: ${e.message}`);
      return {
        success: false,
        message: e.body?.message || e.message || 'Failed to resize database storage',
      };
    }
  }

  /**
   * Get current PVC size for an application's database.
   */
  async getDatabasePvcSize(app: Application): Promise<string> {
    try {
      const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
      const namespace = this.getUserNamespace(app.userId);
      const pvcName = `${app.name}-db`;

      const pvc = await coreApi.readNamespacedPersistentVolumeClaim({
        name: pvcName,
        namespace,
      });
      return pvc.spec?.resources?.requests?.storage || app.dbStorageSize || '1Gi';
    } catch {
      return app.dbStorageSize || '1Gi';
    }
  }

  /**
   * Get comprehensive storage usage for an application.
   * Includes app + DB PVCs and optional Redis/RabbitMQ data volumes when enabled.
   */
  async getStorageUsage(app: Application): Promise<{
    database: StorageUsageSlice | null;
    appStorage: StorageUsageSlice | null;
    redisStorage: StorageUsageSlice | null;
    rabbitmqStorage: StorageUsageSlice | null;
    totalAllocatedGb: number;
    totalUsedGb: number;
  }> {
    const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
    const namespace = this.getUserNamespace(app.userId);

    const result = {
      database: null as StorageUsageSlice | null,
      appStorage: null as StorageUsageSlice | null,
      redisStorage: null as StorageUsageSlice | null,
      rabbitmqStorage: null as StorageUsageSlice | null,
      totalAllocatedGb: 0,
      totalUsedGb: 0,
    };

    const parseToGb = (size: string): number => {
      if (!size) return 0;
      const match = size.match(/^(\d+(?:\.\d+)?)(Ki|Mi|Gi|Ti)?$/i);
      if (!match) return 0;
      const value = parseFloat(match[1]);
      const unit = (match[2] || 'Gi').toLowerCase();
      switch (unit) {
        case 'ki':
          return value / (1024 * 1024);
        case 'mi':
          return value / 1024;
        case 'gi':
          return value;
        case 'ti':
          return value * 1024;
        default:
          return value;
      }
    };

    const formatSize = (gb: number): string => {
      if (gb >= 1) return `${gb.toFixed(1)}Gi`;
      const mb = gb * 1024;
      if (mb >= 1) return `${mb.toFixed(0)}Mi`;
      return `${(mb * 1024).toFixed(0)}Ki`;
    };

    const makeSlice = (allocatedStr: string, allocatedGb: number, usedGb: number): StorageUsageSlice => {
      const availableGb = Math.max(0, allocatedGb - usedGb);
      const usedPercent = allocatedGb > 0 ? Math.round((usedGb / allocatedGb) * 100) : 0;
      return {
        allocatedRaw: allocatedStr,
        allocatedGi: allocatedGb,
        usedGi: usedGb,
        availableGi: availableGb,
        usedPercent,
        allocated: allocatedStr,
        used: formatSize(usedGb),
        available: formatSize(availableGb),
      };
    };

    if (app.databaseType && app.databaseType !== DatabaseType.NONE) {
      try {
        const pvcName = `${app.name}-db`;
        const pvc = await coreApi.readNamespacedPersistentVolumeClaim({
          name: pvcName,
          namespace,
        });
        const allocatedStr = pvc.spec?.resources?.requests?.storage || app.dbStorageSize || '1Gi';
        const allocatedGb = parseToGb(allocatedStr);

        let usedGb = 0;
        try {
          const dbDep = `${app.name}-db`;
          const dbContainer = `${app.name}-db`;
          if (app.databaseType === DatabaseType.POSTGRESQL) {
            usedGb = await this.getPvcUsageFromPod(app, dbDep, '/var/lib/postgresql/data', namespace, dbContainer);
          } else if (app.databaseType === DatabaseType.MYSQL || app.databaseType === DatabaseType.MARIADB) {
            usedGb = await this.getPvcUsageFromPod(app, dbDep, '/var/lib/mysql', namespace, dbContainer);
          } else if (app.databaseType === DatabaseType.MONGODB) {
            usedGb = await this.getPvcUsageFromPod(app, dbDep, '/data/db', namespace, dbContainer);
          }
        } catch {
          usedGb = allocatedGb * 0.1;
        }

        result.database = makeSlice(allocatedStr, allocatedGb, usedGb);
        result.totalAllocatedGb += allocatedGb;
        result.totalUsedGb += usedGb;
      } catch (e: any) {
        this.logger.warn(`Failed to get DB storage usage for ${app.name}: ${e.message}`);
      }
    }

    try {
      let pvc;
      let pvcName = `${app.name}-storage`;
      try {
        pvc = await coreApi.readNamespacedPersistentVolumeClaim({
          name: pvcName,
          namespace,
        });
      } catch {
        pvcName = `${app.name}-wp-content`;
        pvc = await coreApi.readNamespacedPersistentVolumeClaim({
          name: pvcName,
          namespace,
        });
      }

      const allocatedStr = pvc.spec?.resources?.requests?.storage || app.appStorageSize || '2Gi';
      const allocatedGb = parseToGb(allocatedStr);
      const mountPath = this.getStorageMountPath(app.runtime);

      let usedGb = 0;
      try {
        usedGb = await this.getPvcUsageFromPod(app, app.name, mountPath, namespace, app.name);
      } catch {
        usedGb = allocatedGb * 0.1;
      }

      result.appStorage = makeSlice(allocatedStr, allocatedGb, usedGb);
      result.totalAllocatedGb += allocatedGb;
      result.totalUsedGb += usedGb;
    } catch (e: any) {
      this.logger.warn(`Failed to get app storage usage for ${app.name}: ${e.message}`);
    }

    if (app.enableRedis) {
      try {
        const redisPvc = `${app.name}-redis-data`;
        const pvc = await coreApi.readNamespacedPersistentVolumeClaim({
          name: redisPvc,
          namespace,
        });
        const allocatedStr = pvc.spec?.resources?.requests?.storage || '1Gi';
        const allocatedGb = parseToGb(allocatedStr);
        let usedGb = 0;
        try {
          usedGb = await this.getPvcUsageFromPod(app, `${app.name}-redis`, '/data', namespace, 'redis');
        } catch {
          usedGb = allocatedGb * 0.05;
        }
        result.redisStorage = makeSlice(allocatedStr, allocatedGb, usedGb);
        result.totalAllocatedGb += allocatedGb;
        result.totalUsedGb += usedGb;
      } catch (e: any) {
        this.logger.warn(`Failed to get Redis storage usage for ${app.name}: ${e.message}`);
      }
    }

    if (app.enableRabbitmq) {
      try {
        const mqPvc = `${app.name}-rabbitmq-data`;
        const pvc = await coreApi.readNamespacedPersistentVolumeClaim({
          name: mqPvc,
          namespace,
        });
        const allocatedStr = pvc.spec?.resources?.requests?.storage || '1Gi';
        const allocatedGb = parseToGb(allocatedStr);
        let usedGb = 0;
        try {
          usedGb = await this.getPvcUsageFromPod(app, `${app.name}-rabbitmq`, '/var/lib/rabbitmq', namespace, 'rabbitmq');
        } catch {
          usedGb = allocatedGb * 0.05;
        }
        result.rabbitmqStorage = makeSlice(allocatedStr, allocatedGb, usedGb);
        result.totalAllocatedGb += allocatedGb;
        result.totalUsedGb += usedGb;
      } catch (e: any) {
        this.logger.warn(`Failed to get RabbitMQ storage usage for ${app.name}: ${e.message}`);
      }
    }

    return result;
  }

  /**
   * Get PVC usage by executing du command in a pod.
   * Returns usage in GB.
   */
  private async getPvcUsageFromPod(app: Application, deploymentName: string, mountPath: string, namespace: string, containerName: string): Promise<number> {
    const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);

    const pods = await coreApi.listNamespacedPod({
      namespace,
      labelSelector: `app=${deploymentName}`,
    });
    const runningPod = pods.items.find((p) => p.status?.phase === 'Running');

    if (!runningPod?.metadata?.name) {
      throw new Error('No running pod found');
    }

    const exec = new k8s.Exec(kc);
    const chunks: Buffer[] = [];

    const stdout = new PassThrough();
    stdout.on('data', (chunk: Buffer) => chunks.push(chunk));

    await new Promise<void>((resolve, reject) => {
      exec.exec(namespace, runningPod.metadata!.name!, containerName, ['du', '-sb', mountPath], stdout, null, null, false, (status: k8s.V1Status) => {
        if (status.status === 'Success') resolve();
        else reject(new Error(status.message || 'du command failed'));
      });
    });

    const output = Buffer.concat(chunks).toString().trim();
    const bytes = parseInt(output.split(/\s+/)[0], 10) || 0;
    return bytes / (1024 * 1024 * 1024);
  }

  /**
   * Expand a named PVC (Redis, RabbitMQ, or other optional service volumes).
   */
  async resizeNamedPvc(app: Application, pvcName: string, newSize: string, label: string): Promise<{ success: boolean; message: string }> {
    const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
    const namespace = this.getUserNamespace(app.userId);

    try {
      const pvc = await coreApi.readNamespacedPersistentVolumeClaim({
        name: pvcName,
        namespace,
      });
      const currentSize = pvc.spec?.resources?.requests?.storage || '1Gi';
      const parseGi = (s: string) => parseInt(String(s).replace(/Gi/i, ''), 10) || 0;

      if (parseGi(newSize) <= parseGi(currentSize)) {
        return {
          success: false,
          message: `New size (${newSize}) must be larger than current size (${currentSize})`,
        };
      }

      await this.patchPvcStorageSize(coreApi, pvcName, namespace, newSize);
      this.logger.log(`Expanded ${pvcName} from ${currentSize} to ${newSize}`);
      return {
        success: true,
        message: `${label} storage expanded from ${currentSize} to ${newSize}`,
      };
    } catch (e: any) {
      this.logger.error(`Failed to resize ${pvcName}: ${e.message}`);
      return {
        success: false,
        message: e.body?.message || e.message || `Failed to resize ${label} storage`,
      };
    }
  }

  async resizeRedisStoragePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
    if (!app.enableRedis) {
      return {
        success: false,
        message: 'Redis is not enabled for this application',
      };
    }
    return this.resizeNamedPvc(app, `${app.name}-redis-data`, newSize, 'Redis');
  }

  async resizeRabbitmqStoragePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
    if (!app.enableRabbitmq) {
      return {
        success: false,
        message: 'RabbitMQ is not enabled for this application',
      };
    }
    return this.resizeNamedPvc(app, `${app.name}-rabbitmq-data`, newSize, 'RabbitMQ');
  }

  /**
   * Resize app storage PVC (all app types).
   */
  async resizeAppStoragePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
    const { coreApi } = await this.k8sClientService.getK8sClient(app.clusterId);
    const namespace = this.getUserNamespace(app.userId);

    // Try new unified name first, then legacy wp-content name
    let pvcName = `${app.name}-storage`;
    let pvcResponse;
    try {
      pvcResponse = await coreApi.readNamespacedPersistentVolumeClaim({
        name: pvcName,
        namespace,
      });
    } catch {
      // Fallback to legacy WordPress PVC name
      pvcName = `${app.name}-wp-content`;
      pvcResponse = await coreApi.readNamespacedPersistentVolumeClaim({
        name: pvcName,
        namespace,
      });
    }

    try {
      const currentSize = pvcResponse.spec?.resources?.requests?.storage || '2Gi';

      // Parse sizes for comparison
      const parseGi = (s: string) => parseInt(s.replace(/Gi$/i, ''), 10) || 0;
      if (parseGi(newSize) <= parseGi(currentSize)) {
        return {
          success: false,
          message: `Cannot shrink storage. Current: ${currentSize}, Requested: ${newSize}`,
        };
      }

      // Patch PVC to expand
      await coreApi.patchNamespacedPersistentVolumeClaim(
        {
          name: pvcName,
          namespace,
          body: { spec: { resources: { requests: { storage: newSize } } } },
        },
        k8s.setHeaderOptions('Content-Type', 'application/merge-patch+json'),
      );

      this.logger.log(`Expanded ${pvcName} from ${currentSize} to ${newSize}`);
      return {
        success: true,
        message: `App storage expanded from ${currentSize} to ${newSize}`,
      };
    } catch (e: any) {
      this.logger.error(`Failed to resize ${pvcName}: ${e.message}`);
      return {
        success: false,
        message: e.body?.message || e.message || 'Failed to resize app storage',
      };
    }
  }

  // ─── Snapshot helpers ───────────────────────────────

  /**
   * Per-database tooling for dump/restore jobs. `dumpCommand` writes to
   * `outputPath`; `restoreCommand` reads from `/dump/dump.sql` (the copied
   * dump file keeps that name regardless of format — mongodump archives are
   * binary but mongorestore does not care about the extension).
   */
  private databaseDumpSpec(app: Application, dbHost: string): {
    image: string;
    outputPath: string;
    dumpCommand: string[];
    restoreCommand: string[];
  } {
    const dbDatabase = app.name.replace(/-/g, '_');
    switch (app.databaseType) {
      case DatabaseType.POSTGRESQL: {
        const image = `postgres:${app.dbVersion || '16'}-alpine`;
        return {
          image,
          outputPath: '/dump/output.sql',
          dumpCommand: ['sh', '-c', `PGPASSWORD="$DB_PASSWORD" pg_dump -h ${dbHost} -U "$DB_USER" -d ${dbDatabase} --no-owner --no-acl > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"; sleep 900`],
          restoreCommand: ['sh', '-c', `PGPASSWORD="$DB_PASSWORD" psql -h ${dbHost} -U "$DB_USER" -d ${dbDatabase} -f /dump/dump.sql 2>&1`],
        };
      }
      case DatabaseType.MONGODB: {
        const image = `mongo:${app.dbVersion || '7.0'}`;
        const auth = `-u "$DB_USER" -p "$DB_PASSWORD" --authenticationDatabase admin`;
        return {
          image,
          outputPath: '/dump/output.archive',
          dumpCommand: ['sh', '-c', `mongodump --host ${dbHost} ${auth} --db ${dbDatabase} --archive=/dump/output.archive --gzip 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"; sleep 900`],
          restoreCommand: ['sh', '-c', `mongorestore --host ${dbHost} ${auth} --nsInclude '${dbDatabase}.*' --archive=/dump/dump.sql --gzip --drop 2>&1`],
        };
      }
      case DatabaseType.MARIADB: {
        const image = `mariadb:${app.dbVersion || '11'}`;
        return {
          image,
          outputPath: '/dump/output.sql',
          dumpCommand: ['sh', '-c', `mariadb-dump -h ${dbHost} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"; sleep 900`],
          restoreCommand: ['sh', '-c', `mariadb -h ${dbHost} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} < /dump/dump.sql 2>&1`],
        };
      }
      default: {
        const image = `mysql:${app.dbVersion || '8.0'}`;
        return {
          image,
          outputPath: '/dump/output.sql',
          dumpCommand: ['sh', '-c', `mysqldump -h ${dbHost} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"; sleep 900`],
          restoreCommand: ['sh', '-c', `mysql -h ${dbHost} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} < /dump/dump.sql 2>&1`],
        };
      }
    }
  }

  /**
   * Export (dump) the application database to a local file via a K8s Job.
   * Returns the dump as a Buffer, or null on failure.
   *
   * Strategy: Run dump command, then sleep for 60s to allow exec retrieval.
   */
  async exportDatabaseDump(app: Application, onProgress?: (percent: number) => void): Promise<{ data: Buffer | null; logs: string }> {
    const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
    const batchApi = kc.makeApiClient(k8s.BatchV1Api);
    const namespace = this.getUserNamespace(app.userId);
    const dbName = `${app.name}-db`;
    const jobName = `${app.name}-db-dump-${Date.now()}`;

    // Dump command writes to spec.outputPath, then sleeps to allow exec retrieval
    const { image, outputPath, dumpCommand: command } = this.databaseDumpSpec(app, dbName);

    const job: k8s.V1Job = {
      apiVersion: 'batch/v1',
      kind: 'Job',
      metadata: { name: jobName, namespace },
      spec: {
        ttlSecondsAfterFinished: 180,
        activeDeadlineSeconds: 1200,
        backoffLimit: 0,
        template: {
          spec: {
            restartPolicy: 'Never',
            containers: [
              {
                name: 'dump',
                image,
                command,
                env: [
                  {
                    name: 'DB_USER',
                    valueFrom: {
                      secretKeyRef: {
                        name: `${app.name}-db-secret`,
                        key: 'username',
                      },
                    },
                  },
                  {
                    name: 'DB_PASSWORD',
                    valueFrom: {
                      secretKeyRef: {
                        name: `${app.name}-db-secret`,
                        key: 'password',
                      },
                    },
                  },
                ],
                volumeMounts: [{ name: 'dump-vol', mountPath: '/dump' }],
                resources: {
                  requests: { cpu: '100m', memory: '128Mi' },
                  limits: { cpu: '500m', memory: '512Mi' },
                },
              },
            ],
            volumes: [{ name: 'dump-vol', emptyDir: {} }],
          },
        },
      },
    };

    try {
      await batchApi.createNamespacedJob({ namespace, body: job });
    } catch (e: any) {
      this.logger.error(`Failed to create DB dump job: ${e.message}`);
      return { data: null, logs: `Failed to create dump job: ${e.message}` };
    }

    // Wait for dump to complete (check logs for DUMP_DONE marker)
    const timeout = 900_000;
    const start = Date.now();
    let dumpDone = false;
    let podName: string | undefined;

    while (Date.now() - start < timeout) {
      await new Promise((r) => setTimeout(r, 3000));
      const elapsed = Date.now() - start;
      const waitPct = Math.min(75, Math.round((elapsed / timeout) * 75));
      onProgress?.(10 + waitPct);
      try {
        const pods = await coreApi.listNamespacedPod({
          namespace,
          labelSelector: `job-name=${jobName}`,
        });
        if (pods.items.length > 0) {
          podName = pods.items[0].metadata?.name;
          const phase = pods.items[0].status?.phase;

          // Check if pod is Running (container is in sleep phase after dump)
          if (podName && phase === 'Running') {
            try {
              const logRes = await coreApi.readNamespacedPodLog({
                name: podName,
                namespace,
                container: 'dump',
                follow: false,
                tailLines: 50,
              });
              if (logRes?.includes('DUMP_DONE')) {
                dumpDone = true;
                onProgress?.(88);
                break;
              }
            } catch {}
          }

          // If Failed, exit early
          if (phase === 'Failed') break;
        }
      } catch {}
    }

    let dumpBuffer: Buffer | null = null;
    let logs = '';

    if (dumpDone && podName) {
      try {
        // Use kubectl cp equivalent via Exec with proper streams
        const exec = new k8s.Exec(kc);
        const chunks: Buffer[] = [];

        const stdoutStream = new PassThrough();
        const stderrStream = new PassThrough();

        stdoutStream.on('data', (chunk: Buffer) => {
          chunks.push(chunk);
        });

        stderrStream.on('data', (chunk: Buffer) => {
          logs += chunk.toString();
        });

        await new Promise<void>((resolve, reject) => {
          exec.exec(namespace, podName!, 'dump', ['cat', outputPath], stdoutStream, stderrStream, null, false, (status: k8s.V1Status) => {
            if (status.status === 'Success') resolve();
            else reject(new Error(status.message || 'exec failed'));
          });
        });

        if (chunks.length > 0) {
          dumpBuffer = Buffer.concat(chunks);
          onProgress?.(95);
          this.logger.log(`DB dump retrieved: ${dumpBuffer.length} bytes`);
        }
      } catch (e: any) {
        this.logger.warn(`Exec failed for DB dump: ${e.message}`);
        logs = e.message;
      }
    }

    // Cleanup: delete the job early to free resources
    try {
      await batchApi.deleteNamespacedJob({
        name: jobName,
        namespace,
        propagationPolicy: 'Background',
      });
    } catch {}

    if (!dumpBuffer) {
      return {
        data: null,
        logs: logs || 'Dump failed or could not be retrieved',
      };
    }

    return { data: dumpBuffer, logs: 'OK' };
  }

  /**
   * Archive the wp-content directory from a WordPress app's PVC via a K8s Job.
   * The job creates a tar.gz of /var/www/html/wp-content and we retrieve it via exec.
   *
   * Strategy: Create archive, then sleep to allow exec retrieval.
   */
  async archiveWpContent(app: Application): Promise<{ data: Buffer | null; logs: string }> {
    const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
    const batchApi = kc.makeApiClient(k8s.BatchV1Api);
    const namespace = this.getUserNamespace(app.userId);
    const pvcName = `${app.name}-storage`;
    const jobName = `${app.name}-wp-archive-${Date.now()}`;

    const job: k8s.V1Job = {
      apiVersion: 'batch/v1',
      kind: 'Job',
      metadata: { name: jobName, namespace },
      spec: {
        ttlSecondsAfterFinished: 180,
        activeDeadlineSeconds: 1200,
        backoffLimit: 0,
        template: {
          spec: {
            restartPolicy: 'Never',
            containers: [
              {
                name: 'archiver',
                image: 'alpine:3.19',
                command: ['sh', '-c', 'apk add --no-cache tar gzip > /dev/null 2>&1 && cd /wp-content && tar czf /output/wp-content.tar.gz . && echo "ARCHIVE_DONE" && sleep 900'],
                volumeMounts: [
                  {
                    name: 'wp-content',
                    mountPath: '/wp-content',
                    readOnly: true,
                  },
                  { name: 'output', mountPath: '/output' },
                ],
                resources: {
                  requests: { cpu: '100m', memory: '64Mi' },
                  limits: { cpu: '500m', memory: '512Mi' },
                },
              },
            ],
            volumes: [
              {
                name: 'wp-content',
                persistentVolumeClaim: { claimName: pvcName },
              },
              { name: 'output', emptyDir: {} },
            ],
          },
        },
      },
    };

    try {
      await batchApi.createNamespacedJob({ namespace, body: job });
    } catch (e: any) {
      this.logger.error(`Failed to create wp-content archive job: ${e.message}`);
      return { data: null, logs: `Failed to create archive job: ${e.message}` };
    }

    // Wait for archive to complete (check logs for ARCHIVE_DONE marker)
    const timeout = 900_000;
    const start = Date.now();
    let archiveDone = false;
    let podName: string | undefined;

    while (Date.now() - start < timeout) {
      await new Promise((r) => setTimeout(r, 3000));
      try {
        const pods = await coreApi.listNamespacedPod({
          namespace,
          labelSelector: `job-name=${jobName}`,
        });
        if (pods.items.length > 0) {
          podName = pods.items[0].metadata?.name;
          const phase = pods.items[0].status?.phase;

          if (podName && phase === 'Running') {
            try {
              const logRes = await coreApi.readNamespacedPodLog({
                name: podName,
                namespace,
                container: 'archiver',
                follow: false,
                tailLines: 50,
              });
              if (logRes?.includes('ARCHIVE_DONE')) {
                archiveDone = true;
                break;
              }
            } catch {}
          }

          if (phase === 'Failed') break;
        }
      } catch {}
    }

    let archiveBuffer: Buffer | null = null;
    let logs = '';

    if (archiveDone && podName) {
      try {
        const exec = new k8s.Exec(kc);
        const chunks: Buffer[] = [];

        const stdoutStream = new PassThrough();
        const stderrStream = new PassThrough();

        stdoutStream.on('data', (chunk: Buffer) => {
          chunks.push(chunk);
        });

        stderrStream.on('data', (chunk: Buffer) => {
          logs += chunk.toString();
        });

        await new Promise<void>((resolve, reject) => {
          exec.exec(namespace, podName!, 'archiver', ['cat', '/output/wp-content.tar.gz'], stdoutStream, stderrStream, null, false, (status: k8s.V1Status) => {
            if (status.status === 'Success') resolve();
            else reject(new Error(status.message || 'exec failed'));
          });
        });

        if (chunks.length > 0) {
          archiveBuffer = Buffer.concat(chunks);
          this.logger.log(`wp-content archive retrieved: ${archiveBuffer.length} bytes`);
        }
      } catch (e: any) {
        this.logger.warn(`Exec failed for wp-content archive: ${e.message}`);
        logs = e.message;
      }
    }

    // Cleanup
    try {
      await batchApi.deleteNamespacedJob({
        name: jobName,
        namespace,
        propagationPolicy: 'Background',
      });
    } catch {}

    if (!archiveBuffer) {
      return {
        data: null,
        logs: logs || 'Archive failed or could not be retrieved',
      };
    }

    return { data: archiveBuffer, logs: 'OK' };
  }

  /**
   * Restore wp-content from a tar.gz archive into the WordPress PVC.
   *
   * The archive is streamed into a helper pod with `kubectl cp` (a Secret
   * would be capped at ~1MiB — far too small for real wp-content) and
   * extracted in place onto the mounted PVC.
   */
  async restoreWpContent(app: Application, archiveBuffer: Buffer): Promise<{ success: boolean; logs: string }> {
    const { coreApi, kc } = await this.k8sClientService.getK8sClient(app.clusterId);
    const namespace = this.getUserNamespace(app.userId);
    const pvcName = `${app.name}-storage`;
    const ts = Date.now();
    const helperPodName = `${app.name}-wp-restore-${ts}`;

    const helperPod: k8s.V1Pod = {
      apiVersion: 'v1',
      kind: 'Pod',
      metadata: { name: helperPodName, namespace },
      spec: {
        containers: [
          {
            name: 'restore',
            image: 'alpine:3.19',
            command: ['sh', '-c', 'sleep 3600'],
            volumeMounts: [{ name: 'wp-content', mountPath: '/wp-content' }],
            resources: {
              requests: { cpu: '100m', memory: '128Mi' },
              limits: { cpu: '500m', memory: '512Mi' },
            },
          },
        ],
        volumes: [{ name: 'wp-content', persistentVolumeClaim: { claimName: pvcName } }],
        restartPolicy: 'Never',
      },
    };

    const tmpArchive = path.join(os.tmpdir(), `wp-content-restore-${ts}.tar.gz`);
    const tmpKubeconfig = path.join(os.tmpdir(), `kubeconfig-wprestore-${ts}.yaml`);

    try {
      fs.writeFileSync(tmpArchive, archiveBuffer);
      fs.writeFileSync(tmpKubeconfig, kc.exportConfig());

      await coreApi.createNamespacedPod({ namespace, body: helperPod });

      // Wait for helper pod Running
      const podTimeout = 120_000;
      const podStart = Date.now();
      while (Date.now() - podStart < podTimeout) {
        const pod = await coreApi.readNamespacedPod({ name: helperPodName, namespace });
        if (pod.status?.phase === 'Running') break;
        if (pod.status?.phase === 'Failed') throw new Error('wp-content restore helper pod failed to start');
        await new Promise((r) => setTimeout(r, 2000));
      }

      await execFileAsync(
        'kubectl',
        ['--kubeconfig', tmpKubeconfig, 'cp', tmpArchive, `${namespace}/${helperPodName}:/tmp/wp-content.tar.gz`, '--retries', '3'],
        { maxBuffer: 50 * 1024 * 1024, timeout: 600_000 },
      );

      const { stdout, stderr } = await execFileAsync(
        'kubectl',
        [
          '--kubeconfig', tmpKubeconfig, 'exec', '-n', namespace, helperPodName, '--',
          'sh', '-c',
          'rm -rf /wp-content/* /wp-content/.[!.]* 2>/dev/null; tar xzf /tmp/wp-content.tar.gz -C /wp-content && echo RESTORE_DONE',
        ],
        { maxBuffer: 10 * 1024 * 1024, timeout: 600_000 },
      );

      const logs = `${stdout || ''}${stderr || ''}`;
      const success = logs.includes('RESTORE_DONE');
      return { success, logs: logs || (success ? 'Restore completed' : 'Restore failed') };
    } catch (e: any) {
      this.logger.error(`wp-content restore failed for ${app.name}: ${e.message}`);
      return { success: false, logs: e.message || 'wp-content restore failed' };
    } finally {
      try {
        fs.unlinkSync(tmpArchive);
      } catch {}
      try {
        fs.unlinkSync(tmpKubeconfig);
      } catch {}
      try {
        await coreApi.deleteNamespacedPod({ name: helperPodName, namespace });
      } catch {}
    }
  }

  // ─── K8s Revision-based Rollback ─────────────────────

  /**
   * Get the list of Helm release revisions for an application.
   * Returns up to 10 revisions sorted newest-first.
   */
  async getDeploymentRevisions(app: Application): Promise<{
    revisions: Array<{
      revision: number;
      image: string;
      changeCause: string;
      createdAt: string;
      replicas: number;
      isCurrent: boolean;
    }>;
    currentRevision: number;
  }> {
    const namespace = this.getUserNamespace(app.userId);
    const releaseName = app.name;

    try {
      const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
      const helmRevisions = await this.helmService.history(releaseName, namespace, kubeconfig);

      if (!helmRevisions || helmRevisions.length === 0) {
        return { revisions: [], currentRevision: 0 };
      }

      // The last "deployed" revision is the current one
      const currentRev = helmRevisions.filter((r) => r.status === 'deployed').sort((a, b) => b.revision - a.revision)[0];
      const currentRevision = currentRev ? currentRev.revision : 0;

      const revisions = helmRevisions
        .sort((a, b) => b.revision - a.revision)
        .slice(0, 10)
        .map((r) => ({
          revision: r.revision,
          image: '', // Helm history doesn't expose image; frontend will use description
          changeCause: r.description || `${r.status} — ${r.chart}`,
          createdAt: r.updated,
          replicas: 0,
          isCurrent: r.revision === currentRevision,
        }));

      return { revisions, currentRevision };
    } catch (e: any) {
      this.logger.warn(`Could not get Helm history for ${releaseName}: ${e.message}`);
      return { revisions: [], currentRevision: 0 };
    }
  }

  /**
   * Rollback a Helm release to a specific revision.
   */
  async rollbackDeploymentRevision(app: Application, targetRevision: number): Promise<{ success: boolean; message: string }> {
    const namespace = this.getUserNamespace(app.userId);
    const releaseName = app.name;

    try {
      const kubeconfig = await this.k8sClientService.getKubeconfig(app.clusterId);
      await this.helmService.rollback(releaseName, targetRevision, namespace, kubeconfig);
      this.logger.log(`Rolled back ${releaseName} to Helm revision ${targetRevision}`);
      return {
        success: true,
        message: `Rolled back to Helm revision ${targetRevision}`,
      };
    } catch (e: any) {
      this.logger.error(`Helm rollback failed for ${releaseName}: ${e.message}`);
      return { success: false, message: e.message };
    }
  }

  private generatePassword(length = 24): string {
    const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
    let password = '';
    for (let i = 0; i < length; i++) {
      password += chars.charAt(Math.floor(Math.random() * chars.length));
    }
    return password;
  }
}