import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import * as k8s from '@kubernetes/client-node';
import * as fs from 'fs';
import * as path from 'path';
import { execFile } from 'child_process';
import { promisify } from 'util';
import { PassThrough } from 'stream';
import { ClustersService } from '../clusters/clusters.service';
import { Application } from '../applications/entities/application.entity';
import { AppRuntime, DatabaseType } from '../common/enums';
import { HelmService } from './helm.service';

const execFileAsync = promisify(execFile);

interface ManifestContext {
  appName: string;
  namespace: string;
  image: string;
  port: number;
  replicas: number;
  cpuRequest: string;
  cpuLimit: string;
  memoryRequest: string;
  memoryLimit: string;
  envVars: Record<string, string>;
  runtime: string;
  databaseType: string;
  domain: string;
  subdomain: string;
  dbUsername: string;
  dbPassword: string;
  dbVersion: string;
  dbStorageSize: string;
  appStorageSize: string;
  enableRedis: boolean;
  enableRabbitmq: boolean;
  enableElasticsearch: boolean;
  logPaths: string[];
}

@Injectable()
export class KubernetesService implements OnModuleInit {
  private readonly logger = new Logger(KubernetesService.name);

  constructor(
    private configService: ConfigService,
    private clustersService: ClustersService,
    private helmService: HelmService,
  ) {}

  onModuleInit() {
    // Helm chart is used for deployments — no local template loading needed
  }

  private async getK8sClient(clusterId?: string): Promise<{
    coreApi: k8s.CoreV1Api;
    appsApi: k8s.AppsV1Api;
    networkingApi: k8s.NetworkingV1Api;
    kc: k8s.KubeConfig;
  }> {
    const cluster = clusterId
      ? await this.clustersService.findOne(clusterId)
      : await this.clustersService.getDefault();

    const kc = new k8s.KubeConfig();
    kc.loadFromString(cluster.kubeconfig);

    return {
      coreApi: kc.makeApiClient(k8s.CoreV1Api),
      appsApi: kc.makeApiClient(k8s.AppsV1Api),
      networkingApi: kc.makeApiClient(k8s.NetworkingV1Api),
      kc,
    };
  }

  /**
   * Get the raw kubeconfig string for a cluster.
   */
  private async getKubeconfig(clusterId?: string): Promise<string> {
    const cluster = clusterId
      ? await this.clustersService.findOne(clusterId)
      : await this.clustersService.getDefault();
    return cluster.kubeconfig;
  }

  /**
   * Build Helm values object from an Application entity and image URI.
   */
  private buildHelmValues(app: Application, imageUri: string): Record<string, any> {
    const domain = this.configService.get('platform.domain');
    const pullRegistryUrl = this.configService.get<string>('registry.pullUrl') || 'localhost:30500';
    const isWordPress = app.runtime === AppRuntime.WORDPRESS;
    const hasDb = app.databaseType !== DatabaseType.NONE;
    const isPostgres = app.databaseType === DatabaseType.POSTGRESQL;

    const values: Record<string, any> = {
      app: {
        name: app.name,
        namespace: `user-${app.userId.split('-')[0]}`,
        runtime: app.runtime,
        image: imageUri,
        port: app.port,
        replicas: app.replicas,
      },
      resources: {
        cpuRequest: app.cpuRequest,
        cpuLimit: app.cpuLimit,
        memoryRequest: app.memoryRequest,
        memoryLimit: app.memoryLimit,
      },
      envVars: app.envVars || {},
      ingress: {
        enabled: true,
        subdomain: app.subdomain || app.name,
        domain: domain,
        clusterIssuer: 'letsencrypt-prod',
      },
      registry: {
        url: pullRegistryUrl,
      },
      database: {
        enabled: hasDb,
        type: app.databaseType,
        version: app.dbVersion || (isPostgres ? '16' : '8.0'),
        username: app.dbUsername || 'appuser',
        password: app.dbPassword || this.generatePassword(),
        storageSize: app.dbStorageSize || '1Gi',
        resources: {
          cpuRequest: '100m',
          cpuLimit: '500m',
          memoryRequest: '256Mi',
          memoryLimit: '512Mi',
        },
      },
      wordpress: {
        enabled: isWordPress,
        wpContentStorageSize: '2Gi',
      },
      redis: {
        enabled: app.enableRedis || false,
        storageSize: '1Gi',
        resources: {
          cpuRequest: '50m',
          cpuLimit: '200m',
          memoryRequest: '64Mi',
          memoryLimit: '256Mi',
        },
      },
      rabbitmq: {
        enabled: app.enableRabbitmq || false,
        storageSize: '2Gi',
        resources: {
          cpuRequest: '100m',
          cpuLimit: '500m',
          memoryRequest: '256Mi',
          memoryLimit: '512Mi',
        },
      },
      elasticsearch: {
        enabled: app.enableElasticsearch || false,
        logPaths: app.logPaths || [],
      },
      changeCause: `Deploy ${imageUri} at ${new Date().toISOString()}`,
    };

    return values;
  }

  async deployApplication(app: Application, imageUri: string): Promise<Record<string, any>> {
    // Try Helm first, fall back to direct K8s API if Helm is unavailable
    try {
      return await this.deployViaHelm(app, imageUri);
    } catch (helmError: any) {
      this.logger.warn(
        `Helm deploy failed for ${app.name}, falling back to direct K8s API: ${helmError.message}`,
      );
      return await this.deployViaK8sApi(app, imageUri);
    }
  }

  // ── Helm-based deployment ─────────────────────────────────────────

  private async deployViaHelm(app: Application, imageUri: string): Promise<Record<string, any>> {
    const kubeconfig = await this.getKubeconfig(app.clusterId);
    const values = this.buildHelmValues(app, imageUri);
    const namespace = values.app.namespace;
    const releaseName = app.name;

    const result = await this.helmService.installOrUpgrade(
      releaseName,
      namespace,
      values,
      kubeconfig,
    );

    this.logger.log(`Successfully deployed ${app.name} to namespace ${namespace} via Helm`);
    return { helm: { release: releaseName, namespace, stdout: result.stdout }, values };
  }

  // ── Direct K8s API deployment (fallback) ──────────────────────────

  private async deployViaK8sApi(app: Application, imageUri: string): Promise<Record<string, any>> {
    const { coreApi, appsApi, networkingApi } = await this.getK8sClient(app.clusterId);
    const domain = this.configService.get('platform.domain');

    const context: ManifestContext = {
      appName: app.name,
      namespace: `user-${app.userId.split('-')[0]}`,
      image: imageUri,
      port: app.port,
      replicas: app.replicas,
      cpuRequest: app.cpuRequest,
      cpuLimit: app.cpuLimit,
      memoryRequest: app.memoryRequest,
      memoryLimit: app.memoryLimit,
      envVars: app.envVars || {},
      runtime: app.runtime,
      databaseType: app.databaseType,
      domain: domain,
      subdomain: app.subdomain || app.name,
      dbUsername: app.dbUsername || 'appuser',
      dbPassword: app.dbPassword || this.generatePassword(),
      dbVersion: app.dbVersion || '',
      dbStorageSize: app.dbStorageSize || '1Gi',
      appStorageSize: app.appStorageSize || '2Gi',
      enableRedis: app.enableRedis || false,
      enableRabbitmq: app.enableRabbitmq || false,
      enableElasticsearch: app.enableElasticsearch || false,
      logPaths: app.logPaths || [],
    };

    const manifests: Record<string, any> = {};

    try {
      // 1. Ensure namespace exists
      await this.ensureNamespace(coreApi, context.namespace);

      // 2. Create/Update secrets for env vars
      if (Object.keys(context.envVars).length > 0) {
        manifests.secret = await this.applySecret(coreApi, context);
      }

      // 3. Deploy database if needed
      if (context.databaseType !== DatabaseType.NONE) {
        manifests.database = await this.deployDatabase(coreApi, appsApi, context);
      }

      // 3.5 Deploy optional services
      if (context.enableRedis) {
        await this.deployRedis(coreApi, appsApi, context);
        manifests.redis = true;
      }
      if (context.enableRabbitmq) {
        await this.deployRabbitmq(coreApi, appsApi, context);
        manifests.rabbitmq = true;
      }

      // 3.7 Create Fluent Bit ConfigMap if Elasticsearch is enabled
      if (context.enableElasticsearch) {
        await this.createFluentBitConfigMap(coreApi, context);
        manifests.fluentBitConfig = true;
      }

      // 3.8 Create app storage PVC (for all app types)
      manifests.appStoragePvc = await this.applyAppStoragePvc(coreApi, context);

      // 4. Create Deployment
      manifests.deployment = await this.applyDeployment(appsApi, context);

      // 5. Create Service
      manifests.service = await this.applyService(coreApi, context);

      // 6. Create Ingress
      manifests.ingress = await this.applyIngress(networkingApi, context);

      this.logger.log(`Successfully deployed ${app.name} to namespace ${context.namespace} via K8s API`);
    } catch (error: any) {
      this.logger.error(`Failed to deploy ${app.name} via K8s API:`, error.body || error.message);
      throw error;
    }

    return manifests;
  }

  // ── Private K8s resource methods ──────────────────────────────────

  private async ensureNamespace(coreApi: k8s.CoreV1Api, namespace: string): Promise<void> {
    try {
      await coreApi.readNamespace(namespace);
    } catch {
      await coreApi.createNamespace({
        metadata: { name: namespace },
      });
      this.logger.log(`Created namespace: ${namespace}`);
    }
  }

  private async applySecret(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise<any> {
    const secretData: Record<string, string> = {};
    for (const [key, value] of Object.entries(ctx.envVars)) {
      secretData[key] = Buffer.from(String(value)).toString('base64');
    }

    const secret = {
      apiVersion: 'v1',
      kind: 'Secret',
      metadata: {
        name: `${ctx.appName}-env`,
        namespace: ctx.namespace,
      },
      data: secretData,
    };

    try {
      await coreApi.replaceNamespacedSecret(`${ctx.appName}-env`, ctx.namespace, secret);
    } catch {
      await coreApi.createNamespacedSecret(ctx.namespace, secret);
    }
    return secret;
  }

  private async applyDeployment(appsApi: k8s.AppsV1Api, ctx: ManifestContext): Promise<any> {
    const envFrom: any[] = [];
    if (Object.keys(ctx.envVars).length > 0) {
      envFrom.push({ secretRef: { name: `${ctx.appName}-env` } });
    }

    // Database connection env vars
    const extraEnv: any[] = [];
    if (ctx.databaseType === DatabaseType.POSTGRESQL) {
      extraEnv.push(
        { name: 'DB_HOST', value: `${ctx.appName}-db` },
        { name: 'DB_PORT', value: '5432' },
        { name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') },
        { name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
        { name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
        { name: 'DATABASE_URL', value: `postgresql://$(DB_USER):$(DB_PASSWORD)@${ctx.appName}-db:5432/${ctx.appName.replace(/-/g, '_')}` },
      );
    } else if (ctx.databaseType === DatabaseType.MYSQL || ctx.databaseType === DatabaseType.MARIADB) {
      extraEnv.push(
        { name: 'DB_HOST', value: `${ctx.appName}-db` },
        { name: 'DB_PORT', value: '3306' },
        { name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') },
        { name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
        { name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
        { name: 'DATABASE_URL', value: `mysql://$(DB_USER):$(DB_PASSWORD)@${ctx.appName}-db:3306/${ctx.appName.replace(/-/g, '_')}` },
      );
    } else if (ctx.databaseType === DatabaseType.MONGODB) {
      extraEnv.push(
        { name: 'DB_HOST', value: `${ctx.appName}-db` },
        { name: 'DB_PORT', value: '27017' },
        { name: 'DB_NAME', value: ctx.appName.replace(/-/g, '_') },
        { name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
        { name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
        { name: 'MONGODB_URI', value: `mongodb://$(DB_USER):$(DB_PASSWORD)@${ctx.appName}-db:27017/${ctx.appName.replace(/-/g, '_')}?authSource=admin` },
        { name: 'DATABASE_URL', value: `mongodb://$(DB_USER):$(DB_PASSWORD)@${ctx.appName}-db:27017/${ctx.appName.replace(/-/g, '_')}?authSource=admin` },
      );
    }

    // Redis connection env vars
    if (ctx.enableRedis) {
      const redisName = `${ctx.appName}-redis`;
      extraEnv.push(
        { name: 'REDIS_HOST', value: redisName },
        { name: 'REDIS_PORT', value: '6379' },
        { name: 'REDIS_PASSWORD', valueFrom: { secretKeyRef: { name: `${redisName}-secret`, key: 'password' } } },
        { name: 'REDIS_URL', value: `redis://:$(REDIS_PASSWORD)@${redisName}:6379` },
      );
    }

    // RabbitMQ connection env vars
    if (ctx.enableRabbitmq) {
      const rabbitName = `${ctx.appName}-rabbitmq`;
      extraEnv.push(
        { name: 'RABBITMQ_HOST', value: rabbitName },
        { name: 'RABBITMQ_PORT', value: '5672' },
        { name: 'RABBITMQ_MANAGEMENT_PORT', value: '15672' },
        { name: 'RABBITMQ_USER', valueFrom: { secretKeyRef: { name: `${rabbitName}-secret`, key: 'username' } } },
        { name: 'RABBITMQ_PASSWORD', valueFrom: { secretKeyRef: { name: `${rabbitName}-secret`, key: 'password' } } },
        { name: 'AMQP_URL', value: `amqp://$(RABBITMQ_USER):$(RABBITMQ_PASSWORD)@${rabbitName}:5672` },
      );
    }

    // WordPress-specific env vars
    if (ctx.runtime === AppRuntime.WORDPRESS && ctx.databaseType !== DatabaseType.NONE) {
      const dbPort = ctx.databaseType === DatabaseType.POSTGRESQL ? '5432' : '3306';
      extraEnv.push(
        { name: 'WORDPRESS_DB_HOST', value: `${ctx.appName}-db:${dbPort}` },
        { name: 'WORDPRESS_DB_NAME', value: ctx.appName.replace(/-/g, '_') },
        { name: 'WORDPRESS_DB_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
        { name: 'WORDPRESS_DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
        { name: 'WORDPRESS_TABLE_PREFIX', value: 'wp_' },
      );
    }

    const deployment: k8s.V1Deployment = {
      apiVersion: 'apps/v1',
      kind: 'Deployment',
      metadata: {
        name: ctx.appName,
        namespace: ctx.namespace,
        labels: { app: ctx.appName, runtime: ctx.runtime },
        annotations: {
          'kubernetes.io/change-cause': `Deploy ${ctx.image} at ${new Date().toISOString()}`,
        },
      },
      spec: {
        revisionHistoryLimit: 10,
        replicas: ctx.replicas,
        selector: { matchLabels: { app: ctx.appName } },
        template: {
          metadata: { labels: { app: ctx.appName, runtime: ctx.runtime } },
          spec: {
            containers: this.buildContainersSpec(ctx, envFrom, extraEnv),
            volumes: this.buildVolumesSpec(ctx),
          },
        },
      },
    };

    try {
      await appsApi.replaceNamespacedDeployment(ctx.appName, ctx.namespace, deployment);
    } catch {
      await appsApi.createNamespacedDeployment(ctx.namespace, deployment);
    }
    return deployment;
  }

  private async applyAppStoragePvc(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise<any> {
    const pvcName = `${ctx.appName}-storage`;
    const pvc = {
      apiVersion: 'v1',
      kind: 'PersistentVolumeClaim',
      metadata: { name: pvcName, namespace: ctx.namespace, labels: { app: ctx.appName, runtime: ctx.runtime } },
      spec: {
        accessModes: ['ReadWriteOnce'],
        resources: { requests: { storage: ctx.appStorageSize || '2Gi' } },
      },
    };

    try {
      await coreApi.readNamespacedPersistentVolumeClaim(pvcName, ctx.namespace);
      this.logger.log(`PVC ${pvcName} already exists, skipping`);
    } catch {
      await coreApi.createNamespacedPersistentVolumeClaim(ctx.namespace, pvc);
      this.logger.log(`Created app storage PVC: ${pvcName}`);
    }
    return pvc;
  }

  /**
   * Get the storage mount path based on runtime type
   */
  private getStorageMountPath(runtime: string): string {
    switch (runtime) {
      case AppRuntime.WORDPRESS:
        return '/var/www/html/wp-content';
      case AppRuntime.LARAVEL:
      case AppRuntime.PHP:
        return '/var/www/html/storage';
      case AppRuntime.DJANGO:
        return '/app/media';
      case AppRuntime.PYTHON:
      case AppRuntime.GO:
      case AppRuntime.DOTNET:
      case AppRuntime.NODEJS:
      default:
        return '/app/data';
    }
  }

  /**
   * Build containers spec for the app deployment, including optional Fluent Bit sidecar
   */
  private buildContainersSpec(ctx: ManifestContext, envFrom: any[], extraEnv: any[]): any[] {
    const containers: any[] = [];

    // Main application container
    const appContainer: any = {
      name: ctx.appName,
      image: ctx.image,
      imagePullPolicy: 'Always',
      ports: [{ containerPort: ctx.port }],
      envFrom,
      env: extraEnv,
      resources: {
        requests: { cpu: ctx.cpuRequest, memory: ctx.memoryRequest },
        limits: { cpu: ctx.cpuLimit, memory: ctx.memoryLimit },
      },
      readinessProbe: {
        tcpSocket: { port: ctx.port as any },
        initialDelaySeconds: 5,
        periodSeconds: 5,
      },
      livenessProbe: {
        tcpSocket: { port: ctx.port as any },
        initialDelaySeconds: 15,
        periodSeconds: 10,
        failureThreshold: 5,
      },
      volumeMounts: [{ name: 'app-storage', mountPath: this.getStorageMountPath(ctx.runtime) }],
    };

    // Add log volume mount if Elasticsearch is enabled
    if (ctx.enableElasticsearch) {
      appContainer.volumeMounts.push({ name: 'app-logs', mountPath: '/var/log/app' });
    }

    containers.push(appContainer);

    // Add Fluent Bit sidecar for log collection if Elasticsearch is enabled
    if (ctx.enableElasticsearch) {
      const logPaths = ctx.logPaths && ctx.logPaths.length > 0 
        ? ctx.logPaths 
        : ['/var/log/app/*.log'];

      const fluentbitConfig = this.buildFluentBitConfig(ctx.appName, ctx.namespace, logPaths);

      containers.push({
        name: 'fluent-bit',
        image: 'fluent/fluent-bit:2.2',
        resources: {
          requests: { cpu: '10m', memory: '32Mi' },
          limits: { cpu: '50m', memory: '64Mi' },
        },
        volumeMounts: [
          { name: 'app-logs', mountPath: '/var/log/app', readOnly: true },
          { name: 'fluent-bit-config', mountPath: '/fluent-bit/etc' },
        ],
        env: [
          { name: 'APP_NAME', value: ctx.appName },
          { name: 'APP_NAMESPACE', value: ctx.namespace },
          // Elasticsearch host - assumes cluster-level ES at elasticsearch.logging namespace
          { name: 'ES_HOST', value: 'elasticsearch.logging.svc.cluster.local' },
          { name: 'ES_PORT', value: '9200' },
        ],
      });
    }

    return containers;
  }

  /**
   * Build volumes spec for the app deployment
   */
  private buildVolumesSpec(ctx: ManifestContext): any[] {
    const volumes: any[] = [
      {
        name: 'app-storage',
        persistentVolumeClaim: { claimName: `${ctx.appName}-storage` },
      },
    ];

    if (ctx.enableElasticsearch) {
      // Shared log volume between app and fluent-bit
      volumes.push({
        name: 'app-logs',
        emptyDir: {},
      });

      // Fluent Bit config as ConfigMap
      volumes.push({
        name: 'fluent-bit-config',
        configMap: { name: `${ctx.appName}-fluent-bit-config` },
      });
    }

    return volumes;
  }

  /**
   * Build Fluent Bit configuration for log collection
   */
  private buildFluentBitConfig(appName: string, namespace: string, logPaths: string[]): string {
    const pathsStr = logPaths.join(',');
    return `
[SERVICE]
    Flush         5
    Daemon        Off
    Log_Level     info

[INPUT]
    Name          tail
    Path          ${pathsStr}
    Tag           app.${appName}
    Parser        json
    Refresh_Interval 5

[FILTER]
    Name          record_modifier
    Match         *
    Record        app ${appName}
    Record        namespace ${namespace}

[OUTPUT]
    Name          es
    Match         *
    Host          \${ES_HOST}
    Port          \${ES_PORT}
    Index         logs-${namespace}-${appName}
    Type          _doc
    Logstash_Format On
    Logstash_Prefix logs-${namespace}
    Suppress_Type_Name On
`;
  }

  /**
   * Create Fluent Bit ConfigMap for an app
   */
  private async createFluentBitConfigMap(
    coreApi: k8s.CoreV1Api,
    ctx: ManifestContext,
  ): Promise<void> {
    if (!ctx.enableElasticsearch) return;

    const logPaths = ctx.logPaths && ctx.logPaths.length > 0 
      ? ctx.logPaths 
      : ['/var/log/app/*.log'];

    const configMap = {
      apiVersion: 'v1',
      kind: 'ConfigMap',
      metadata: { 
        name: `${ctx.appName}-fluent-bit-config`, 
        namespace: ctx.namespace,
        labels: { app: ctx.appName },
      },
      data: {
        'fluent-bit.conf': this.buildFluentBitConfig(ctx.appName, ctx.namespace, logPaths),
        'parsers.conf': `
[PARSER]
    Name        json
    Format      json
    Time_Key    time
    Time_Format %Y-%m-%dT%H:%M:%S.%L

[PARSER]
    Name        docker
    Format      json
    Time_Key    time
    Time_Format %Y-%m-%dT%H:%M:%S.%L
`,
      },
    };

    try {
      await coreApi.replaceNamespacedConfigMap(`${ctx.appName}-fluent-bit-config`, ctx.namespace, configMap);
    } catch {
      await coreApi.createNamespacedConfigMap(ctx.namespace, configMap);
    }
    this.logger.log(`Created Fluent Bit ConfigMap for ${ctx.appName}`);
  }

  private async applyService(coreApi: k8s.CoreV1Api, ctx: ManifestContext): Promise<any> {
    const service: k8s.V1Service = {
      apiVersion: 'v1',
      kind: 'Service',
      metadata: { name: ctx.appName, namespace: ctx.namespace },
      spec: {
        selector: { app: ctx.appName },
        ports: [{ port: 80, targetPort: ctx.port as any, protocol: 'TCP' }],
        type: 'ClusterIP',
      },
    };

    try {
      await coreApi.replaceNamespacedService(ctx.appName, ctx.namespace, service);
    } catch {
      await coreApi.createNamespacedService(ctx.namespace, service);
    }
    return service;
  }

  private async applyIngress(networkingApi: k8s.NetworkingV1Api, ctx: ManifestContext): Promise<any> {
    const host = `${ctx.subdomain}.${ctx.domain}`;
    const ingress: k8s.V1Ingress = {
      apiVersion: 'networking.k8s.io/v1',
      kind: 'Ingress',
      metadata: {
        name: ctx.appName,
        namespace: ctx.namespace,
        annotations: {
          'cert-manager.io/cluster-issuer': 'letsencrypt-prod',
        },
      },
      spec: {
        ingressClassName: 'nginx',
        rules: [
          {
            host,
            http: {
              paths: [
                {
                  path: '/',
                  pathType: 'Prefix',
                  backend: { service: { name: ctx.appName, port: { number: 80 } } },
                },
              ],
            },
          },
        ],
        tls: [{ hosts: [host], secretName: `${ctx.appName}-tls` }],
      },
    };

    try {
      await networkingApi.replaceNamespacedIngress(ctx.appName, ctx.namespace, ingress);
    } catch {
      await networkingApi.createNamespacedIngress(ctx.namespace, ingress);
    }
    return ingress;
  }

  private async deployDatabase(
    coreApi: k8s.CoreV1Api,
    appsApi: k8s.AppsV1Api,
    ctx: ManifestContext,
  ): Promise<any> {
    const dbName = `${ctx.appName}-db`;

    // Create DB secret
    await this.createDbSecret(coreApi, ctx.namespace, ctx.appName, ctx.dbPassword, ctx.dbUsername);

    // Create PVC for DB
    await this.createPVC(coreApi, ctx.namespace, dbName, ctx.dbStorageSize);

    // Deploy database based on type
    const dbType = ctx.databaseType;
    let image: string;
    let port: number;
    let dataPath: string;
    let envVars: any[];
    let readinessProbe: any;
    let livenessProbe: any;

    switch (dbType) {
      case DatabaseType.POSTGRESQL:
        const pgVer = ctx.dbVersion || '16';
        image = `postgres:${pgVer}-alpine`;
        port = 5432;
        dataPath = '/var/lib/postgresql/data';
        envVars = [
          { name: 'PGDATA', value: '/var/lib/postgresql/data/pgdata' },
          { name: 'POSTGRES_DB', value: ctx.appName.replace(/-/g, '_') },
          { name: 'POSTGRES_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
          { name: 'POSTGRES_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
        ];
        readinessProbe = { exec: { command: ['pg_isready', '-U', ctx.dbUsername] }, initialDelaySeconds: 10, periodSeconds: 5, failureThreshold: 6 };
        livenessProbe = { exec: { command: ['pg_isready', '-U', ctx.dbUsername] }, initialDelaySeconds: 30, periodSeconds: 10, failureThreshold: 5 };
        break;

      case DatabaseType.MYSQL:
        const mysqlVer = ctx.dbVersion || '8.0';
        image = `mysql:${mysqlVer}`;
        port = 3306;
        dataPath = '/var/lib/mysql';
        envVars = [
          { name: 'MYSQL_DATABASE', value: ctx.appName.replace(/-/g, '_') },
          { name: 'MYSQL_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
          { name: 'MYSQL_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
          { name: 'MYSQL_ROOT_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
        ];
        readinessProbe = { exec: { command: ['mysqladmin', 'ping', '-h', '127.0.0.1'] }, initialDelaySeconds: 10, periodSeconds: 5, failureThreshold: 6 };
        livenessProbe = { exec: { command: ['mysqladmin', 'ping', '-h', '127.0.0.1'] }, initialDelaySeconds: 30, periodSeconds: 10, failureThreshold: 5 };
        break;

      case DatabaseType.MARIADB:
        const mariaVer = ctx.dbVersion || '11.4';
        image = `mariadb:${mariaVer}`;
        port = 3306;
        dataPath = '/var/lib/mysql';
        envVars = [
          { name: 'MARIADB_DATABASE', value: ctx.appName.replace(/-/g, '_') },
          { name: 'MARIADB_USER', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
          { name: 'MARIADB_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
          { name: 'MARIADB_ROOT_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
        ];
        readinessProbe = { exec: { command: ['healthcheck.sh', '--connect', '--innodb_initialized'] }, initialDelaySeconds: 10, periodSeconds: 5, failureThreshold: 6 };
        livenessProbe = { exec: { command: ['healthcheck.sh', '--connect', '--innodb_initialized'] }, initialDelaySeconds: 30, periodSeconds: 10, failureThreshold: 5 };
        break;

      case DatabaseType.MONGODB:
        const mongoVer = ctx.dbVersion || '7.0';
        image = `mongo:${mongoVer}`;
        port = 27017;
        dataPath = '/data/db';
        envVars = [
          { name: 'MONGO_INITDB_DATABASE', value: ctx.appName.replace(/-/g, '_') },
          { name: 'MONGO_INITDB_ROOT_USERNAME', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'username' } } },
          { name: 'MONGO_INITDB_ROOT_PASSWORD', valueFrom: { secretKeyRef: { name: `${ctx.appName}-db-secret`, key: 'password' } } },
        ];
        readinessProbe = { exec: { command: ['mongosh', '--eval', 'db.adminCommand("ping")'] }, initialDelaySeconds: 10, periodSeconds: 5, failureThreshold: 6 };
        livenessProbe = { exec: { command: ['mongosh', '--eval', 'db.adminCommand("ping")'] }, initialDelaySeconds: 30, periodSeconds: 10, failureThreshold: 5 };
        break;

      default:
        throw new Error(`Unsupported database type: ${dbType}`);
    }

    const dbDeployment: k8s.V1Deployment = {
      apiVersion: 'apps/v1',
      kind: 'Deployment',
      metadata: { name: dbName, namespace: ctx.namespace, labels: { app: dbName } },
      spec: {
        replicas: 1,
        selector: { matchLabels: { app: dbName } },
        template: {
          metadata: { labels: { app: dbName } },
          spec: {
            containers: [
              {
                name: dbName,
                image,
                ports: [{ containerPort: port }],
                env: envVars,
                volumeMounts: [{ name: 'db-storage', mountPath: dataPath }],
                resources: {
                  requests: { cpu: '100m', memory: '256Mi' },
                  limits: { cpu: '500m', memory: '512Mi' },
                },
                readinessProbe,
                livenessProbe,
              },
            ],
            volumes: [
              { name: 'db-storage', persistentVolumeClaim: { claimName: dbName } },
            ],
          },
        },
      },
    };

    try {
      await appsApi.replaceNamespacedDeployment(dbName, ctx.namespace, dbDeployment);
    } catch {
      await appsApi.createNamespacedDeployment(ctx.namespace, dbDeployment);
    }

    // Create DB Service
    const dbService: k8s.V1Service = {
      apiVersion: 'v1',
      kind: 'Service',
      metadata: { name: dbName, namespace: ctx.namespace },
      spec: {
        selector: { app: dbName },
        ports: [{ port, targetPort: port as any, protocol: 'TCP' }],
        type: 'ClusterIP',
      },
    };

    try {
      await coreApi.replaceNamespacedService(dbName, ctx.namespace, dbService);
    } catch {
      await coreApi.createNamespacedService(ctx.namespace, dbService);
    }

    return { deployment: dbDeployment, service: dbService };
  }

  private async createDbSecret(
    coreApi: k8s.CoreV1Api,
    namespace: string,
    appName: string,
    password: string,
    username: string = 'appuser',
  ): Promise<void> {
    const secret = {
      apiVersion: 'v1',
      kind: 'Secret',
      metadata: { name: `${appName}-db-secret`, namespace },
      data: {
        username: Buffer.from(username).toString('base64'),
        password: Buffer.from(password).toString('base64'),
      },
    };

    try {
      await coreApi.replaceNamespacedSecret(`${appName}-db-secret`, namespace, secret);
    } catch {
      await coreApi.createNamespacedSecret(namespace, secret);
    }
  }

  private async createPVC(
    coreApi: k8s.CoreV1Api,
    namespace: string,
    name: string,
    size: string,
  ): Promise<void> {
    const pvc: k8s.V1PersistentVolumeClaim = {
      apiVersion: 'v1',
      kind: 'PersistentVolumeClaim',
      metadata: { name, namespace },
      spec: {
        accessModes: ['ReadWriteOnce'],
        resources: { requests: { storage: size } },
      },
    };

    try {
      await coreApi.readNamespacedPersistentVolumeClaim(name, namespace);
      // PVC exists, don't recreate
    } catch {
      await coreApi.createNamespacedPersistentVolumeClaim(namespace, pvc);
    }
  }

  /**
   * Deploy Redis for an application
   */
  private async deployRedis(
    coreApi: k8s.CoreV1Api,
    appsApi: k8s.AppsV1Api,
    ctx: ManifestContext,
  ): Promise<void> {
    const redisName = `${ctx.appName}-redis`;

    // Create PVC for Redis persistence
    await this.createPVC(coreApi, ctx.namespace, `${redisName}-data`, '1Gi');

    // Create Redis password secret
    const redisPassword = this.generatePassword(16);
    const redisSecret = {
      apiVersion: 'v1',
      kind: 'Secret',
      metadata: { name: `${redisName}-secret`, namespace: ctx.namespace },
      data: {
        password: Buffer.from(redisPassword).toString('base64'),
      },
    };

    try {
      await coreApi.replaceNamespacedSecret(`${redisName}-secret`, ctx.namespace, redisSecret);
    } catch {
      await coreApi.createNamespacedSecret(ctx.namespace, redisSecret);
    }

    // Create Redis Deployment
    const redisDeployment: k8s.V1Deployment = {
      apiVersion: 'apps/v1',
      kind: 'Deployment',
      metadata: { name: redisName, namespace: ctx.namespace },
      spec: {
        replicas: 1,
        selector: { matchLabels: { app: redisName } },
        template: {
          metadata: { labels: { app: redisName } },
          spec: {
            containers: [
              {
                name: 'redis',
                image: 'redis:7.2-alpine',
                args: ['--requirepass', '$(REDIS_PASSWORD)'],
                ports: [{ containerPort: 6379 }],
                env: [
                  {
                    name: 'REDIS_PASSWORD',
                    valueFrom: {
                      secretKeyRef: { name: `${redisName}-secret`, key: 'password' },
                    },
                  },
                ],
                volumeMounts: [
                  { name: 'redis-data', mountPath: '/data' },
                ],
                resources: {
                  requests: { cpu: '50m', memory: '64Mi' },
                  limits: { cpu: '200m', memory: '256Mi' },
                },
                readinessProbe: {
                  exec: { command: ['redis-cli', 'ping'] },
                  initialDelaySeconds: 5,
                  periodSeconds: 10,
                },
                livenessProbe: {
                  exec: { command: ['redis-cli', 'ping'] },
                  initialDelaySeconds: 15,
                  periodSeconds: 20,
                },
              },
            ],
            volumes: [
              {
                name: 'redis-data',
                persistentVolumeClaim: { claimName: `${redisName}-data` },
              },
            ],
          },
        },
      },
    };

    try {
      await appsApi.replaceNamespacedDeployment(redisName, ctx.namespace, redisDeployment);
    } catch {
      await appsApi.createNamespacedDeployment(ctx.namespace, redisDeployment);
    }

    // Create Redis Service
    const redisService = {
      apiVersion: 'v1',
      kind: 'Service',
      metadata: { name: redisName, namespace: ctx.namespace },
      spec: {
        selector: { app: redisName },
        ports: [{ port: 6379, targetPort: 6379 as any, protocol: 'TCP' }],
        type: 'ClusterIP',
      },
    };

    try {
      await coreApi.replaceNamespacedService(redisName, ctx.namespace, redisService);
    } catch {
      await coreApi.createNamespacedService(ctx.namespace, redisService);
    }

    this.logger.log(`Redis deployed for ${ctx.appName}`);
  }

  /**
   * Deploy RabbitMQ for an application
   */
  private async deployRabbitmq(
    coreApi: k8s.CoreV1Api,
    appsApi: k8s.AppsV1Api,
    ctx: ManifestContext,
  ): Promise<void> {
    const rabbitName = `${ctx.appName}-rabbitmq`;

    // Create PVC for RabbitMQ persistence
    await this.createPVC(coreApi, ctx.namespace, `${rabbitName}-data`, '2Gi');

    // Create RabbitMQ credentials secret
    const rabbitUser = 'appuser';
    const rabbitPassword = this.generatePassword(16);
    const rabbitSecret = {
      apiVersion: 'v1',
      kind: 'Secret',
      metadata: { name: `${rabbitName}-secret`, namespace: ctx.namespace },
      data: {
        username: Buffer.from(rabbitUser).toString('base64'),
        password: Buffer.from(rabbitPassword).toString('base64'),
      },
    };

    try {
      await coreApi.replaceNamespacedSecret(`${rabbitName}-secret`, ctx.namespace, rabbitSecret);
    } catch {
      await coreApi.createNamespacedSecret(ctx.namespace, rabbitSecret);
    }

    // Create RabbitMQ Deployment
    const rabbitDeployment: k8s.V1Deployment = {
      apiVersion: 'apps/v1',
      kind: 'Deployment',
      metadata: { name: rabbitName, namespace: ctx.namespace },
      spec: {
        replicas: 1,
        selector: { matchLabels: { app: rabbitName } },
        template: {
          metadata: { labels: { app: rabbitName } },
          spec: {
            containers: [
              {
                name: 'rabbitmq',
                image: 'rabbitmq:3.13-management-alpine',
                ports: [
                  { containerPort: 5672, name: 'amqp' },
                  { containerPort: 15672, name: 'management' },
                ],
                env: [
                  {
                    name: 'RABBITMQ_DEFAULT_USER',
                    valueFrom: {
                      secretKeyRef: { name: `${rabbitName}-secret`, key: 'username' },
                    },
                  },
                  {
                    name: 'RABBITMQ_DEFAULT_PASS',
                    valueFrom: {
                      secretKeyRef: { name: `${rabbitName}-secret`, key: 'password' },
                    },
                  },
                ],
                volumeMounts: [
                  { name: 'rabbitmq-data', mountPath: '/var/lib/rabbitmq' },
                ],
                resources: {
                  requests: { cpu: '100m', memory: '256Mi' },
                  limits: { cpu: '500m', memory: '512Mi' },
                },
                readinessProbe: {
                  exec: { command: ['rabbitmq-diagnostics', '-q', 'ping'] },
                  initialDelaySeconds: 20,
                  periodSeconds: 10,
                  timeoutSeconds: 5,
                },
                livenessProbe: {
                  exec: { command: ['rabbitmq-diagnostics', '-q', 'status'] },
                  initialDelaySeconds: 60,
                  periodSeconds: 30,
                  timeoutSeconds: 10,
                },
              },
            ],
            volumes: [
              {
                name: 'rabbitmq-data',
                persistentVolumeClaim: { claimName: `${rabbitName}-data` },
              },
            ],
          },
        },
      },
    };

    try {
      await appsApi.replaceNamespacedDeployment(rabbitName, ctx.namespace, rabbitDeployment);
    } catch {
      await appsApi.createNamespacedDeployment(ctx.namespace, rabbitDeployment);
    }

    // Create RabbitMQ Services (AMQP and Management)
    const rabbitService = {
      apiVersion: 'v1',
      kind: 'Service',
      metadata: { name: rabbitName, namespace: ctx.namespace },
      spec: {
        selector: { app: rabbitName },
        ports: [
          { port: 5672, targetPort: 5672 as any, protocol: 'TCP', name: 'amqp' },
          { port: 15672, targetPort: 15672 as any, protocol: 'TCP', name: 'management' },
        ],
        type: 'ClusterIP',
      },
    };

    try {
      await coreApi.replaceNamespacedService(rabbitName, ctx.namespace, rabbitService);
    } catch {
      await coreApi.createNamespacedService(ctx.namespace, rabbitService);
    }

    this.logger.log(`RabbitMQ deployed for ${ctx.appName}`);
  }

  async getPodLogs(app: Application): Promise<string> {
    const { coreApi } = await this.getK8sClient(app.clusterId);
    const namespace = `user-${app.userId.split('-')[0]}`;

    const pods = await coreApi.listNamespacedPod(
      namespace,
      undefined,
      undefined,
      undefined,
      undefined,
      `app=${app.name}`,
    );

    if (pods.body.items.length === 0) {
      return 'No pods found for this application.';
    }

    const podName = pods.body.items[0].metadata?.name;
    if (!podName) return 'Pod name not found.';

    const logResponse = await coreApi.readNamespacedPodLog(
      podName,
      namespace,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      200,
    );

    return logResponse.body;
  }

  async scaleDeployment(app: Application, replicas: number): Promise<void> {
    const { appsApi } = await this.getK8sClient(app.clusterId);
    const namespace = `user-${app.userId.split('-')[0]}`;

    await appsApi.patchNamespacedDeployment(
      app.name,
      namespace,
      { spec: { replicas } },
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      { headers: { 'Content-Type': 'application/merge-patch+json' } },
    );
  }

  /**
   * Suspend an application by scaling deployment to 0 replicas.
   * This keeps all resources (PVC, Service, Ingress) but stops the pods.
   * Also scales database deployment to 0 if exists.
   */
  async suspendApplication(app: Application): Promise<void> {
    const { appsApi } = await this.getK8sClient(app.clusterId);
    const namespace = `user-${app.userId.split('-')[0]}`;

    this.logger.log(`Suspending application ${app.name} in namespace ${namespace}`);

    // Scale main deployment to 0
    try {
      await appsApi.patchNamespacedDeployment(
        app.name,
        namespace,
        { spec: { replicas: 0 } },
        undefined,
        undefined,
        undefined,
        undefined,
        undefined,
        { headers: { 'Content-Type': 'application/merge-patch+json' } },
      );
      this.logger.log(`Scaled ${app.name} to 0 replicas`);
    } catch (e: any) {
      this.logger.warn(`Failed to scale ${app.name}: ${e.message}`);
    }

    // Scale database deployment to 0 if exists
    if (app.databaseType && app.databaseType !== DatabaseType.NONE) {
      try {
        await appsApi.patchNamespacedDeployment(
          `${app.name}-db`,
          namespace,
          { spec: { replicas: 0 } },
          undefined,
          undefined,
          undefined,
          undefined,
          undefined,
          { headers: { 'Content-Type': 'application/merge-patch+json' } },
        );
        this.logger.log(`Scaled ${app.name}-db to 0 replicas`);
      } catch (e: any) {
        // Database might not exist, that's ok
        if (e?.response?.statusCode !== 404) {
          this.logger.warn(`Failed to scale ${app.name}-db: ${e.message}`);
        }
      }
    }
  }

  /**
   * Resume a suspended application by scaling deployment back to original replicas.
   * Also scales database deployment back to 1 if exists.
   */
  async resumeApplication(app: Application): Promise<void> {
    const { appsApi } = await this.getK8sClient(app.clusterId);
    const namespace = `user-${app.userId.split('-')[0]}`;
    const replicas = app.replicas || 1;

    this.logger.log(`Resuming application ${app.name} in namespace ${namespace}`);

    // Scale database deployment back first (so it's ready when app starts)
    if (app.databaseType && app.databaseType !== DatabaseType.NONE) {
      try {
        await appsApi.patchNamespacedDeployment(
          `${app.name}-db`,
          namespace,
          { spec: { replicas: 1 } },
          undefined,
          undefined,
          undefined,
          undefined,
          undefined,
          { headers: { 'Content-Type': 'application/merge-patch+json' } },
        );
        this.logger.log(`Scaled ${app.name}-db to 1 replica`);
      } catch (e: any) {
        if (e?.response?.statusCode !== 404) {
          this.logger.warn(`Failed to scale ${app.name}-db: ${e.message}`);
        }
      }
    }

    // Scale main deployment back
    try {
      await appsApi.patchNamespacedDeployment(
        app.name,
        namespace,
        { spec: { replicas } },
        undefined,
        undefined,
        undefined,
        undefined,
        undefined,
        { headers: { 'Content-Type': 'application/merge-patch+json' } },
      );
      this.logger.log(`Scaled ${app.name} to ${replicas} replicas`);
    } catch (e: any) {
      this.logger.warn(`Failed to scale ${app.name}: ${e.message}`);
      throw e;
    }
  }

  async restartDeployment(app: Application): Promise<void> {
    const { appsApi } = await this.getK8sClient(app.clusterId);
    const namespace = `user-${app.userId.split('-')[0]}`;

    await appsApi.patchNamespacedDeployment(
      app.name,
      namespace,
      {
        spec: {
          template: {
            metadata: {
              annotations: {
                'kubectl.kubernetes.io/restartedAt': new Date().toISOString(),
              },
            },
          },
        },
      },
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      { headers: { 'Content-Type': 'application/merge-patch+json' } },
    );
  }

  /**
   * Get real-time resource usage (CPU/Memory) for an app's pods via metrics-server.
   * Also returns the configured requests/limits and pod status.
   */
  async getResourceUsage(app: Application): Promise<any> {
    const { coreApi, appsApi, kc } = await this.getK8sClient(app.clusterId);
    const namespace = `user-${app.userId.split('-')[0]}`;

    // Get deployment info for configured resources
    let deployment: k8s.V1Deployment | null = null;
    try {
      const depResponse = await appsApi.readNamespacedDeployment(app.name, namespace);
      deployment = depResponse.body;
    } catch {
      // Deployment may not exist yet
    }

    // Get pods
    const podsResponse = await coreApi.listNamespacedPod(
      namespace,
      undefined,
      undefined,
      undefined,
      undefined,
      `app=${app.name}`,
    );

    const pods = podsResponse.body.items.map((pod) => ({
      name: pod.metadata?.name,
      status: pod.status?.phase,
      ready: pod.status?.conditions?.find((c) => c.type === 'Ready')?.status === 'True',
      restarts: pod.status?.containerStatuses?.[0]?.restartCount || 0,
      startedAt: pod.status?.startTime,
    }));

    // Try to get metrics from metrics-server via custom API
    let podMetrics: any[] = [];
    try {
      const metricsClient = new k8s.CustomObjectsApi(kc.getCurrentCluster()?.server);
      // Use the kc to make a raw request to metrics API
      const opts: any = {};
      await kc.applyToRequest(opts);

      const metricsUrl = `${kc.getCurrentCluster()?.server}/apis/metrics.k8s.io/v1beta1/namespaces/${namespace}/pods`;

      const https = require('https');
      const http = require('http');
      const url = new URL(metricsUrl);

      podMetrics = await new Promise((resolve) => {
        const client = url.protocol === 'https:' ? https : http;
        const reqOpts: any = {
          hostname: url.hostname,
          port: url.port,
          path: url.pathname + `?labelSelector=app%3D${app.name}`,
          method: 'GET',
          headers: opts.headers || {},
          rejectUnauthorized: false,
        };

        // Apply TLS from kubeconfig
        if (opts.ca) reqOpts.ca = opts.ca;
        if (opts.cert) reqOpts.cert = opts.cert;
        if (opts.key) reqOpts.key = opts.key;

        const req = client.request(reqOpts, (res: any) => {
          let data = '';
          res.on('data', (chunk: string) => (data += chunk));
          res.on('end', () => {
            try {
              const parsed = JSON.parse(data);
              const items = (parsed.items || []).map((item: any) => ({
                name: item.metadata?.name,
                cpu: item.containers?.[0]?.usage?.cpu || '0',
                memory: item.containers?.[0]?.usage?.memory || '0',
              }));
              resolve(items);
            } catch {
              resolve([]);
            }
          });
        });
        req.on('error', () => resolve([]));
        req.end();
      });
    } catch {
      this.logger.warn(`Metrics not available for ${app.name}`);
    }

    // Get configured resources from deployment
    const container = deployment?.spec?.template?.spec?.containers?.[0];
    const configured = {
      cpuRequest: container?.resources?.requests?.cpu || app.cpuRequest,
      cpuLimit: container?.resources?.limits?.cpu || app.cpuLimit,
      memoryRequest: container?.resources?.requests?.memory || app.memoryRequest,
      memoryLimit: container?.resources?.limits?.memory || app.memoryLimit,
      replicas: deployment?.spec?.replicas ?? app.replicas,
      readyReplicas: deployment?.status?.readyReplicas || 0,
      availableReplicas: deployment?.status?.availableReplicas || 0,
    };

    return {
      configured,
      pods,
      metrics: podMetrics,
    };
  }

  /**
   * Update resource limits/requests and replicas on a live K8s deployment.
   */
  async updateResources(
    app: Application,
    resources: { cpuRequest?: string; cpuLimit?: string; memoryRequest?: string; memoryLimit?: string; replicas?: number },
  ): Promise<void> {
    const { appsApi } = await this.getK8sClient(app.clusterId);
    const namespace = `user-${app.userId.split('-')[0]}`;

    const patch: any = { spec: {} };

    if (resources.replicas !== undefined) {
      patch.spec.replicas = resources.replicas;
    }

    if (resources.cpuRequest || resources.cpuLimit || resources.memoryRequest || resources.memoryLimit) {
      patch.spec.template = {
        spec: {
          containers: [
            {
              name: app.name,
              resources: {
                requests: {
                  ...(resources.cpuRequest && { cpu: resources.cpuRequest }),
                  ...(resources.memoryRequest && { memory: resources.memoryRequest }),
                },
                limits: {
                  ...(resources.cpuLimit && { cpu: resources.cpuLimit }),
                  ...(resources.memoryLimit && { memory: resources.memoryLimit }),
                },
              },
            },
          ],
        },
      };
    }

    await appsApi.patchNamespacedDeployment(
      app.name,
      namespace,
      patch,
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      { headers: { 'Content-Type': 'application/strategic-merge-patch+json' } },
    );

    this.logger.log(`Updated resources for ${app.name}: ${JSON.stringify(resources)}`);
  }

  /**
   * Get preview info for a deployed application.
   * Patches the service to NodePort if needed, and returns the access URL.
   */
  async getPreviewInfo(app: Application): Promise<{
    url: string;
    nodePort: number;
    host: string;
    ingressUrl?: string;
  }> {
    const { coreApi, networkingApi, kc } = await this.getK8sClient(app.clusterId);
    const namespace = `user-${app.userId.split('-')[0]}`;
    const domain = this.configService.get('platform.domain');
    const clusterServer = kc.getCurrentCluster()?.server || '';
    // Extract host IP from cluster API server URL (e.g., https://217.197.107.252:6443 → 217.197.107.252)
    let hostIp = '127.0.0.1';
    try {
      const serverUrl = new URL(clusterServer);
      hostIp = serverUrl.hostname;
    } catch {}

    // Read current service
    let nodePort = 0;
    try {
      const svcResponse = await coreApi.readNamespacedService(app.name, namespace);
      const svc = svcResponse.body;

      if (svc.spec?.type === 'NodePort') {
        // Already NodePort, read the assigned port
        nodePort = svc.spec.ports?.[0]?.nodePort || 0;
      } else {
        // Patch ClusterIP → NodePort so we can access from outside
        const patchBody = {
          spec: {
            type: 'NodePort',
            ports: [
              {
                port: 80,
                targetPort: app.port,
                protocol: 'TCP',
              },
            ],
          },
        };

        const patchedResponse = await coreApi.patchNamespacedService(
          app.name,
          namespace,
          patchBody,
          undefined,
          undefined,
          undefined,
          undefined,
          undefined,
          { headers: { 'Content-Type': 'application/strategic-merge-patch+json' } },
        );
        nodePort = patchedResponse.body.spec?.ports?.[0]?.nodePort || 0;
        this.logger.log(`Patched service ${app.name} to NodePort: ${nodePort}`);
      }
    } catch (e: any) {
      this.logger.warn(`Failed to get/patch service for ${app.name}: ${e.message}`);
      throw new Error(`Service not found for "${app.name}". Make sure the app is deployed.`);
    }

    // Build ingress URL
    const subdomain = app.subdomain || app.name;
    const ingressUrl = `https://${subdomain}.${domain}`;

    return {
      url: `http://${hostIp}:${nodePort}`,
      nodePort,
      host: hostIp,
      ingressUrl,
    };
  }

  async deleteApplication(app: Application): Promise<void> {
    const namespace = `user-${app.userId.split('-')[0]}`;
    const { coreApi, appsApi, networkingApi } = await this.getK8sClient(app.clusterId);

    // Step 1: Try Helm uninstall (handles most resources)
    try {
      const kubeconfig = await this.getKubeconfig(app.clusterId);
      await this.helmService.uninstall(app.name, namespace, kubeconfig);
      this.logger.log(`Helm release ${app.name} uninstalled from ${namespace}`);
    } catch (error: any) {
      this.logger.warn(`Helm uninstall failed for ${app.name}: ${error.message} — proceeding with manual cleanup`);
      // Manual cleanup of core resources if helm wasn't managing them
      try { await appsApi.deleteNamespacedDeployment(app.name, namespace); } catch {}
      try { await coreApi.deleteNamespacedService(app.name, namespace); } catch {}
      try { await networkingApi.deleteNamespacedIngress(app.name, namespace); } catch {}
    }

    // Step 2: Delete resources with "helm.sh/resource-policy: keep" (PVCs, db-secret)
    // These survive helm uninstall by design, so we must delete them explicitly
    const dbName = `${app.name}-db`;
    const resourcesToDelete = [
      // Database resources
      () => appsApi.deleteNamespacedDeployment(dbName, namespace),
      () => coreApi.deleteNamespacedService(dbName, namespace),
      () => coreApi.deleteNamespacedPersistentVolumeClaim(dbName, namespace),
      () => coreApi.deleteNamespacedSecret(`${app.name}-db-secret`, namespace),
      // App storage PVC (all app types)
      () => coreApi.deleteNamespacedPersistentVolumeClaim(`${app.name}-storage`, namespace),
      // Legacy: also try deleting old wp-content PVC name for backward compatibility
      () => coreApi.deleteNamespacedPersistentVolumeClaim(`${app.name}-wp-content`, namespace),
      // App env secret
      () => coreApi.deleteNamespacedSecret(`${app.name}-env`, namespace),
      // Registry pull secret (shared, but labeled per-app — safe to delete)
      () => coreApi.deleteNamespacedSecret('registry-pull-secret', namespace),
      // TLS secret created by cert-manager
      () => coreApi.deleteNamespacedSecret(`${app.name}-tls`, namespace),
    ];

    for (const deleteFn of resourcesToDelete) {
      try { await deleteFn(); } catch {}
    }

    this.logger.log(`All K8s resources cleaned up for ${app.name} in ${namespace}`);
  }

  /**
   * Wait for the database pod to become Ready.
   * Polls pod status with label selector `app=<appName>-db`.
   */
  async waitForDatabaseReady(app: Application, timeoutMs = 120_000): Promise<void> {
    const { coreApi } = await this.getK8sClient(app.clusterId);
    const namespace = `user-${app.userId.split('-')[0]}`;
    const dbLabel = `${app.name}-db`;
    const start = Date.now();

    this.logger.log(`Waiting for DB pod (app=${dbLabel}) to become Ready in ${namespace}...`);

    while (Date.now() - start < timeoutMs) {
      try {
        const pods = await coreApi.listNamespacedPod(
          namespace, undefined, undefined, undefined, undefined, `app=${dbLabel}`,
        );
        for (const pod of pods.body.items) {
          const readyCond = (pod.status?.conditions || []).find(c => c.type === 'Ready');
          if (readyCond?.status === 'True') {
            this.logger.log(`DB pod ${pod.metadata?.name} is Ready (${Date.now() - start}ms)`);
            return;
          }
        }
      } catch {
        // Pods may not exist yet
      }
      await new Promise(r => setTimeout(r, 3000));
    }
    this.logger.warn(`DB pod did not become Ready within ${timeoutMs / 1000}s — proceeding anyway`);
  }

  /**
   * Restore a SQL dump file into the application's database.
   * Uses a PVC + helper pod + kubectl cp to transfer the dump (supports large files),
   * then runs a restore Job that mounts the PVC and imports the dump.
   */
  async restoreDatabaseDump(app: Application, dumpFilePath: string): Promise<{ success: boolean; logs: string }> {
    const { coreApi, kc } = await this.getK8sClient(app.clusterId);
    const batchApi = kc.makeApiClient(k8s.BatchV1Api);
    const namespace = `user-${app.userId.split('-')[0]}`;
    const dbName = `${app.name}-db`;
    const ts = Date.now();
    const pvcName = `${app.name}-db-dump-${ts}`;
    const helperPodName = `${pvcName}-helper`;
    const jobName = `${app.name}-db-restore-${ts}`;
    const isPostgres = app.databaseType === DatabaseType.POSTGRESQL;
    const dbDatabase = app.name.replace(/-/g, '_');

    const dumpSize = fs.statSync(dumpFilePath).size;
    const pvcSizeGi = Math.max(1, Math.ceil((dumpSize * 2) / (1024 * 1024 * 1024)));

    this.logger.log(`Restoring DB dump for ${app.name}: ${(dumpSize / 1024 / 1024).toFixed(1)} MB via PVC`);

    // ── 1. Create PVC for the dump file ──
    await coreApi.createNamespacedPersistentVolumeClaim(namespace, {
      apiVersion: 'v1',
      kind: 'PersistentVolumeClaim',
      metadata: { name: pvcName, namespace },
      spec: {
        accessModes: ['ReadWriteOnce'],
        resources: { requests: { storage: `${pvcSizeGi}Gi` } },
      },
    });

    // ── 2. Helper pod to receive the dump via kubectl cp ──
    const helperPod: k8s.V1Pod = {
      apiVersion: 'v1',
      kind: 'Pod',
      metadata: { name: helperPodName, namespace },
      spec: {
        containers: [{
          name: 'helper',
          image: 'busybox:1.36',
          command: ['sh', '-c', 'sleep 3600'],
          volumeMounts: [{ name: 'dump', mountPath: '/data' }],
          resources: { requests: { cpu: '100m', memory: '128Mi' }, limits: { cpu: '500m', memory: '256Mi' } },
        }],
        volumes: [{ name: 'dump', persistentVolumeClaim: { claimName: pvcName } }],
        restartPolicy: 'Never',
      },
    };

    await coreApi.createNamespacedPod(namespace, helperPod);

    // Wait for helper pod Running
    const podTimeout = 120_000;
    const podStart = Date.now();
    while (Date.now() - podStart < podTimeout) {
      const pod = await coreApi.readNamespacedPod(helperPodName, namespace);
      if (pod.body.status?.phase === 'Running') break;
      if (pod.body.status?.phase === 'Failed') throw new Error('DB dump helper pod failed to start');
      await new Promise(r => setTimeout(r, 2000));
    }

    // ── 3. kubectl cp the dump file into the helper pod ──
    const tmpKubeconfig = path.join('/tmp', `kubeconfig-dbdump-${ts}.yaml`);
    const kcYaml = kc.exportConfig();
    fs.writeFileSync(tmpKubeconfig, kcYaml);

    // Verify tar is available inside the helper pod
    try {
      await execFileAsync('kubectl', [
        '--kubeconfig', tmpKubeconfig,
        'exec', '-n', namespace, helperPodName, '--', 'tar', '--version',
      ], { timeout: 30_000 });
    } catch (e: any) {
      this.logger.warn(`tar check in DB dump helper pod failed: ${e.message}`);
    }

    try {
      await execFileAsync('kubectl', [
        '--kubeconfig', tmpKubeconfig,
        'cp', dumpFilePath, `${namespace}/${helperPodName}:/data/dump.sql`,
        '--retries', '3',
      ], { maxBuffer: 50 * 1024 * 1024, timeout: 600_000 });
      this.logger.log(`kubectl cp dump completed (${(dumpSize / 1024 / 1024).toFixed(1)} MB)`);
    } finally {
      try { fs.unlinkSync(tmpKubeconfig); } catch {}
      try { await coreApi.deleteNamespacedPod(helperPodName, namespace); } catch {}
    }

    // ── 4. Build restore command ──
    const command = isPostgres
      ? [
          'sh', '-c',
          `PGPASSWORD="$DB_PASSWORD" psql -h ${dbName} -U "$DB_USER" -d ${dbDatabase} -f /dump/dump.sql 2>&1`,
        ]
      : [
          'sh', '-c',
          `mysql -h ${dbName} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} < /dump/dump.sql 2>&1`,
        ];

    const defaultDbVer = isPostgres ? '16' : '8.0';
    const restoreDbVer = app.dbVersion || defaultDbVer;
    const image = isPostgres ? `postgres:${restoreDbVer}-alpine` : `mysql:${restoreDbVer}`;

    // ── 5. Create the restore Job ──
    const job: k8s.V1Job = {
      apiVersion: 'batch/v1',
      kind: 'Job',
      metadata: { name: jobName, namespace },
      spec: {
        ttlSecondsAfterFinished: 300,
        backoffLimit: 1,
        template: {
          spec: {
            restartPolicy: 'Never',
            containers: [
              {
                name: 'restore',
                image,
                command,
                env: [
                  { name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${app.name}-db-secret`, key: 'username' } } },
                  { name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${app.name}-db-secret`, key: 'password' } } },
                ],
                volumeMounts: [
                  { name: 'dump-volume', mountPath: '/dump', readOnly: true },
                ],
                resources: {
                  requests: { cpu: '200m', memory: '256Mi' },
                  limits: { cpu: '1', memory: '1Gi' },
                },
              },
            ],
            volumes: [
              {
                name: 'dump-volume',
                persistentVolumeClaim: { claimName: pvcName },
              },
            ],
          },
        },
      },
    };

    try {
      await batchApi.createNamespacedJob(namespace, job);
      this.logger.log(`Created DB restore job ${jobName} for ${app.name}`);
    } catch (e: any) {
      try { await coreApi.deleteNamespacedPersistentVolumeClaim(pvcName, namespace); } catch {}
      this.logger.error(`Failed to create restore job: ${e.message}`);
      throw new Error('Failed to create database restore job');
    }

    // ── 6. Wait for the job to complete (max 10 minutes for large dumps) ──
    const timeout = 600_000;
    const start = Date.now();
    let completed = false;
    let failed = false;

    while (Date.now() - start < timeout) {
      await new Promise((r) => setTimeout(r, 5000));
      try {
        const jobStatus = await batchApi.readNamespacedJob(jobName, namespace);
        const status = jobStatus.body.status;
        if (status?.succeeded && status.succeeded > 0) { completed = true; break; }
        if (status?.failed && status.failed > 0) { failed = true; break; }
      } catch {
        // Job may not be ready yet
      }
    }

    // ── 7. Get logs from the restore job pod ──
    let logs = '';
    try {
      const pods = await coreApi.listNamespacedPod(
        namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`,
      );
      if (pods.body.items.length > 0) {
        const podName = pods.body.items[0].metadata?.name;
        if (podName) {
          const logResponse = await coreApi.readNamespacedPodLog(podName, namespace);
          logs = logResponse.body || '';
        }
      }
    } catch (e: any) {
      this.logger.warn(`Could not get restore job logs: ${e.message}`);
    }

    // ── 8. Clean up the dump PVC ──
    try {
      await coreApi.deleteNamespacedPersistentVolumeClaim(pvcName, namespace);
      this.logger.log(`Cleaned up dump PVC: ${pvcName}`);
    } catch {}

    if (!completed && !failed) {
      this.logger.warn(`DB restore job ${jobName} timed out`);
      return { success: false, logs: logs || 'Restore job timed out after 10 minutes' };
    }

    if (failed) {
      this.logger.warn(`DB restore job ${jobName} failed`);
      return { success: false, logs: logs || 'Restore job failed' };
    }

    this.logger.log(`DB restore for ${app.name} completed successfully`);
    return { success: true, logs };
  }

  /**
   * Resize (expand) the database PVC for an application.
   * K8s only supports PVC expansion, not shrinking.
   */
  async resizeDatabasePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
    const { coreApi } = await this.getK8sClient(app.clusterId);
    const namespace = `user-${app.userId.split('-')[0]}`;
    const pvcName = `${app.name}-db`;

    try {
      // Read current PVC to check current size
      const currentPvc = await coreApi.readNamespacedPersistentVolumeClaim(pvcName, namespace);
      const currentSize = currentPvc.body.spec?.resources?.requests?.storage || '1Gi';

      const currentGi = parseInt(currentSize.replace('Gi', ''), 10) || 1;
      const newGi = parseInt(newSize.replace('Gi', ''), 10) || 1;

      if (newGi <= currentGi) {
        return { success: false, message: `New size (${newSize}) must be larger than current size (${currentSize})` };
      }

      // Patch PVC to expand
      const patch = [
        {
          op: 'replace',
          path: '/spec/resources/requests/storage',
          value: newSize,
        },
      ];

      await coreApi.patchNamespacedPersistentVolumeClaim(
        pvcName,
        namespace,
        patch,
        undefined,
        undefined,
        undefined,
        undefined,
        undefined,
        { headers: { 'Content-Type': 'application/json-patch+json' } },
      );

      this.logger.log(`Resized PVC ${pvcName} from ${currentSize} to ${newSize}`);
      return { success: true, message: `Database storage expanded from ${currentSize} to ${newSize}` };
    } catch (e: any) {
      this.logger.error(`Failed to resize PVC ${pvcName}: ${e.message}`);
      return { success: false, message: e.body?.message || e.message || 'Failed to resize database storage' };
    }
  }

  /**
   * Get current PVC size for an application's database.
   */
  async getDatabasePvcSize(app: Application): Promise<string> {
    try {
      const { coreApi } = await this.getK8sClient(app.clusterId);
      const namespace = `user-${app.userId.split('-')[0]}`;
      const pvcName = `${app.name}-db`;

      const pvc = await coreApi.readNamespacedPersistentVolumeClaim(pvcName, namespace);
      return pvc.body.spec?.resources?.requests?.storage || app.dbStorageSize || '1Gi';
    } catch {
      return app.dbStorageSize || '1Gi';
    }
  }

  /**
   * Get comprehensive storage usage for an application.
   * Returns allocated, used, and available storage for database and app (wp-content) PVCs.
   */
  async getStorageUsage(app: Application): Promise<{
    database: { allocated: string; used: string; available: string; usedPercent: number } | null;
    appStorage: { allocated: string; used: string; available: string; usedPercent: number } | null;
    totalAllocatedGb: number;
    totalUsedGb: number;
  }> {
    const { coreApi } = await this.getK8sClient(app.clusterId);
    const namespace = `user-${app.userId.split('-')[0]}`;

    const result: {
      database: { allocated: string; used: string; available: string; usedPercent: number } | null;
      appStorage: { allocated: string; used: string; available: string; usedPercent: number } | null;
      totalAllocatedGb: number;
      totalUsedGb: number;
    } = {
      database: null,
      appStorage: null,
      totalAllocatedGb: 0,
      totalUsedGb: 0,
    };

    // Helper to parse size strings to GB
    const parseToGb = (size: string): number => {
      if (!size) return 0;
      const match = size.match(/^(\d+(?:\.\d+)?)(Ki|Mi|Gi|Ti)?$/i);
      if (!match) return 0;
      const value = parseFloat(match[1]);
      const unit = (match[2] || 'Gi').toLowerCase();
      switch (unit) {
        case 'ki': return value / (1024 * 1024);
        case 'mi': return value / 1024;
        case 'gi': return value;
        case 'ti': return value * 1024;
        default: return value;
      }
    };

    // Helper to format GB to human readable
    const formatSize = (gb: number): string => {
      if (gb >= 1) return `${gb.toFixed(1)}Gi`;
      const mb = gb * 1024;
      if (mb >= 1) return `${mb.toFixed(0)}Mi`;
      return `${(mb * 1024).toFixed(0)}Ki`;
    };

    // Get database PVC usage
    if (app.databaseType && app.databaseType !== DatabaseType.NONE) {
      try {
        const pvcName = `${app.name}-db`;
        const pvc = await coreApi.readNamespacedPersistentVolumeClaim(pvcName, namespace);
        const allocatedStr = pvc.body.spec?.resources?.requests?.storage || app.dbStorageSize || '1Gi';
        const allocatedGb = parseToGb(allocatedStr);

        // Try to get actual usage from pod exec (du command)
        let usedGb = 0;
        try {
          usedGb = await this.getPvcUsageFromPod(app, `${app.name}-db`, '/var/lib/postgresql/data', namespace);
          if (usedGb === 0 && app.databaseType === DatabaseType.MYSQL) {
            usedGb = await this.getPvcUsageFromPod(app, `${app.name}-db`, '/var/lib/mysql', namespace);
          }
        } catch {
          // Estimate ~10% usage if we can't get actual
          usedGb = allocatedGb * 0.1;
        }

        const availableGb = Math.max(0, allocatedGb - usedGb);
        const usedPercent = allocatedGb > 0 ? Math.round((usedGb / allocatedGb) * 100) : 0;

        result.database = {
          allocated: allocatedStr,
          used: formatSize(usedGb),
          available: formatSize(availableGb),
          usedPercent,
        };
        result.totalAllocatedGb += allocatedGb;
        result.totalUsedGb += usedGb;
      } catch (e: any) {
        this.logger.warn(`Failed to get DB storage usage for ${app.name}: ${e.message}`);
      }
    }

    // Get app storage PVC usage (all app types have storage now)
    try {
      // Try new unified name first, then legacy wp-content name for backward compatibility
      let pvc;
      let pvcName = `${app.name}-storage`;
      try {
        pvc = await coreApi.readNamespacedPersistentVolumeClaim(pvcName, namespace);
      } catch {
        // Fallback to legacy WordPress PVC name
        pvcName = `${app.name}-wp-content`;
        pvc = await coreApi.readNamespacedPersistentVolumeClaim(pvcName, namespace);
      }

      const allocatedStr = pvc.body.spec?.resources?.requests?.storage || app.appStorageSize || '2Gi';
      const allocatedGb = parseToGb(allocatedStr);

      // Determine mount path based on runtime
      const mountPath = this.getStorageMountPath(app.runtime);

      // Try to get actual usage from pod exec
      let usedGb = 0;
      try {
        usedGb = await this.getPvcUsageFromPod(app, app.name, mountPath, namespace);
      } catch {
        usedGb = allocatedGb * 0.1;
      }

      const availableGb = Math.max(0, allocatedGb - usedGb);
      const usedPercent = allocatedGb > 0 ? Math.round((usedGb / allocatedGb) * 100) : 0;

      result.appStorage = {
        allocated: allocatedStr,
        used: formatSize(usedGb),
        available: formatSize(availableGb),
        usedPercent,
      };
      result.totalAllocatedGb += allocatedGb;
      result.totalUsedGb += usedGb;
    } catch (e: any) {
      this.logger.warn(`Failed to get app storage usage for ${app.name}: ${e.message}`);
    }

    return result;
  }

  /**
   * Get PVC usage by executing du command in a pod.
   * Returns usage in GB.
   */
  private async getPvcUsageFromPod(app: Application, deploymentName: string, mountPath: string, namespace: string): Promise<number> {
    const { coreApi, kc } = await this.getK8sClient(app.clusterId);

    // Find a running pod for this deployment
    const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `app=${deploymentName}`);
    const runningPod = pods.body.items.find(p => p.status?.phase === 'Running');

    if (!runningPod?.metadata?.name) {
      throw new Error('No running pod found');
    }

    const exec = new k8s.Exec(kc);
    const chunks: Buffer[] = [];

    const stdout = new PassThrough();
    stdout.on('data', (chunk: Buffer) => chunks.push(chunk));

    await new Promise<void>((resolve, reject) => {
      exec.exec(
        namespace,
        runningPod.metadata!.name!,
        deploymentName === `${app.name}-db` ? 'db' : app.name,
        ['du', '-sb', mountPath],
        stdout,
        null,
        null,
        false,
        (status: k8s.V1Status) => {
          if (status.status === 'Success') resolve();
          else reject(new Error(status.message || 'du command failed'));
        },
      );
    });

    const output = Buffer.concat(chunks).toString().trim();
    const bytes = parseInt(output.split(/\s+/)[0], 10) || 0;
    return bytes / (1024 * 1024 * 1024); // Convert to GB
  }

  /**
   * Resize app storage PVC (all app types).
   */
  async resizeAppStoragePvc(app: Application, newSize: string): Promise<{ success: boolean; message: string }> {
    const { coreApi } = await this.getK8sClient(app.clusterId);
    const namespace = `user-${app.userId.split('-')[0]}`;

    // Try new unified name first, then legacy wp-content name
    let pvcName = `${app.name}-storage`;
    let pvcResponse;
    try {
      pvcResponse = await coreApi.readNamespacedPersistentVolumeClaim(pvcName, namespace);
    } catch {
      // Fallback to legacy WordPress PVC name
      pvcName = `${app.name}-wp-content`;
      pvcResponse = await coreApi.readNamespacedPersistentVolumeClaim(pvcName, namespace);
    }

    try {
      const currentSize = pvcResponse.body.spec?.resources?.requests?.storage || '2Gi';

      // Parse sizes for comparison
      const parseGi = (s: string) => parseInt(s.replace(/Gi$/i, ''), 10) || 0;
      if (parseGi(newSize) <= parseGi(currentSize)) {
        return { success: false, message: `Cannot shrink storage. Current: ${currentSize}, Requested: ${newSize}` };
      }

      // Patch PVC to expand
      await coreApi.patchNamespacedPersistentVolumeClaim(
        pvcName,
        namespace,
        { spec: { resources: { requests: { storage: newSize } } } },
        undefined,
        undefined,
        undefined,
        undefined,
        undefined,
        { headers: { 'Content-Type': 'application/merge-patch+json' } },
      );

      this.logger.log(`Expanded ${pvcName} from ${currentSize} to ${newSize}`);
      return { success: true, message: `App storage expanded from ${currentSize} to ${newSize}` };
    } catch (e: any) {
      this.logger.error(`Failed to resize ${pvcName}: ${e.message}`);
      return { success: false, message: e.body?.message || e.message || 'Failed to resize app storage' };
    }
  }

  // ─── Snapshot helpers ───────────────────────────────

  /**
   * Export (dump) the application database to a local file via a K8s Job.
   * Returns the dump as a Buffer, or null on failure.
   * 
   * Strategy: Run dump command, then sleep for 60s to allow exec retrieval.
   */
  async exportDatabaseDump(app: Application): Promise<{ data: Buffer | null; logs: string }> {
    const { coreApi, kc } = await this.getK8sClient(app.clusterId);
    const batchApi = kc.makeApiClient(k8s.BatchV1Api);
    const namespace = `user-${app.userId.split('-')[0]}`;
    const dbName = `${app.name}-db`;
    const jobName = `${app.name}-db-dump-${Date.now()}`;
    const isPostgres = app.databaseType === DatabaseType.POSTGRESQL;
    const dbDatabase = app.name.replace(/-/g, '_');

    const defaultDbVer = isPostgres ? '16' : '8.0';
    const dbVer = app.dbVersion || defaultDbVer;
    const image = isPostgres ? `postgres:${dbVer}-alpine` : `mysql:${dbVer}`;

    // Dump command writes to /dump/output.sql, then sleeps to allow exec retrieval
    const command = isPostgres
      ? ['sh', '-c', `PGPASSWORD="$DB_PASSWORD" pg_dump -h ${dbName} -U "$DB_USER" -d ${dbDatabase} --no-owner --no-acl > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"; sleep 900`]
      : ['sh', '-c', `mysqldump -h ${dbName} -u "$DB_USER" -p"$DB_PASSWORD" ${dbDatabase} > /dump/output.sql 2>/tmp/err; cat /tmp/err >&2; echo "DUMP_DONE"; sleep 900`];

    const job: k8s.V1Job = {
      apiVersion: 'batch/v1',
      kind: 'Job',
      metadata: { name: jobName, namespace },
      spec: {
        ttlSecondsAfterFinished: 180,
        activeDeadlineSeconds: 1200,
        backoffLimit: 0,
        template: {
          spec: {
            restartPolicy: 'Never',
            containers: [{
              name: 'dump',
              image,
              command,
              env: [
                { name: 'DB_USER', valueFrom: { secretKeyRef: { name: `${app.name}-db-secret`, key: 'username' } } },
                { name: 'DB_PASSWORD', valueFrom: { secretKeyRef: { name: `${app.name}-db-secret`, key: 'password' } } },
              ],
              volumeMounts: [{ name: 'dump-vol', mountPath: '/dump' }],
              resources: { requests: { cpu: '100m', memory: '128Mi' }, limits: { cpu: '500m', memory: '512Mi' } },
            }],
            volumes: [{ name: 'dump-vol', emptyDir: {} }],
          },
        },
      },
    };

    try {
      await batchApi.createNamespacedJob(namespace, job);
    } catch (e: any) {
      this.logger.error(`Failed to create DB dump job: ${e.message}`);
      return { data: null, logs: `Failed to create dump job: ${e.message}` };
    }

    // Wait for dump to complete (check logs for DUMP_DONE marker)
    const timeout = 900_000;
    const start = Date.now();
    let dumpDone = false;
    let podName: string | undefined;

    while (Date.now() - start < timeout) {
      await new Promise((r) => setTimeout(r, 3000));
      try {
        const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`);
        if (pods.body.items.length > 0) {
          podName = pods.body.items[0].metadata?.name;
          const phase = pods.body.items[0].status?.phase;
          
          // Check if pod is Running (container is in sleep phase after dump)
          if (podName && phase === 'Running') {
            try {
              const logRes = await coreApi.readNamespacedPodLog(podName, namespace, 'dump', false, undefined, undefined, undefined, undefined, undefined, 50);
              if (logRes.body?.includes('DUMP_DONE')) {
                dumpDone = true;
                break;
              }
            } catch {}
          }
          
          // If Failed, exit early
          if (phase === 'Failed') break;
        }
      } catch {}
    }

    let dumpBuffer: Buffer | null = null;
    let logs = '';

    if (dumpDone && podName) {
      try {
        // Use kubectl cp equivalent via Exec with proper streams
        const exec = new k8s.Exec(kc);
        const chunks: Buffer[] = [];
        
        const stdoutStream = new PassThrough();
        const stderrStream = new PassThrough();
        
        stdoutStream.on('data', (chunk: Buffer) => {
          chunks.push(chunk);
        });
        
        stderrStream.on('data', (chunk: Buffer) => {
          logs += chunk.toString();
        });
        
        await new Promise<void>((resolve, reject) => {
          exec.exec(
            namespace, podName!, 'dump',
            ['cat', '/dump/output.sql'],
            stdoutStream,
            stderrStream,
            null,
            false,
            (status: k8s.V1Status) => {
              if (status.status === 'Success') resolve();
              else reject(new Error(status.message || 'exec failed'));
            },
          );
        });

        if (chunks.length > 0) {
          dumpBuffer = Buffer.concat(chunks);
          this.logger.log(`DB dump retrieved: ${dumpBuffer.length} bytes`);
        }
      } catch (e: any) {
        this.logger.warn(`Exec failed for DB dump: ${e.message}`);
        logs = e.message;
      }
    }

    // Cleanup: delete the job early to free resources
    try {
      await batchApi.deleteNamespacedJob(jobName, namespace, undefined, undefined, undefined, undefined, 'Background');
    } catch {}

    if (!dumpBuffer) {
      return { data: null, logs: logs || 'Dump failed or could not be retrieved' };
    }

    return { data: dumpBuffer, logs: 'OK' };
  }

  /**
   * Archive the wp-content directory from a WordPress app's PVC via a K8s Job.
   * The job creates a tar.gz of /var/www/html/wp-content and we retrieve it via exec.
   * 
   * Strategy: Create archive, then sleep to allow exec retrieval.
   */
  async archiveWpContent(app: Application): Promise<{ data: Buffer | null; logs: string }> {
    const { coreApi, kc } = await this.getK8sClient(app.clusterId);
    const batchApi = kc.makeApiClient(k8s.BatchV1Api);
    const namespace = `user-${app.userId.split('-')[0]}`;
    const pvcName = `${app.name}-wp-content`;
    const jobName = `${app.name}-wp-archive-${Date.now()}`;

    const job: k8s.V1Job = {
      apiVersion: 'batch/v1',
      kind: 'Job',
      metadata: { name: jobName, namespace },
      spec: {
        ttlSecondsAfterFinished: 180,
        activeDeadlineSeconds: 1200,
        backoffLimit: 0,
        template: {
          spec: {
            restartPolicy: 'Never',
            containers: [{
              name: 'archiver',
              image: 'alpine:3.19',
              command: ['sh', '-c', 'apk add --no-cache tar gzip > /dev/null 2>&1 && cd /wp-content && tar czf /output/wp-content.tar.gz . && echo "ARCHIVE_DONE" && sleep 900'],
              volumeMounts: [
                { name: 'wp-content', mountPath: '/wp-content', readOnly: true },
                { name: 'output', mountPath: '/output' },
              ],
              resources: { requests: { cpu: '100m', memory: '64Mi' }, limits: { cpu: '500m', memory: '512Mi' } },
            }],
            volumes: [
              { name: 'wp-content', persistentVolumeClaim: { claimName: pvcName } },
              { name: 'output', emptyDir: {} },
            ],
          },
        },
      },
    };

    try {
      await batchApi.createNamespacedJob(namespace, job);
    } catch (e: any) {
      this.logger.error(`Failed to create wp-content archive job: ${e.message}`);
      return { data: null, logs: `Failed to create archive job: ${e.message}` };
    }

    // Wait for archive to complete (check logs for ARCHIVE_DONE marker)
    const timeout = 900_000;
    const start = Date.now();
    let archiveDone = false;
    let podName: string | undefined;

    while (Date.now() - start < timeout) {
      await new Promise((r) => setTimeout(r, 3000));
      try {
        const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`);
        if (pods.body.items.length > 0) {
          podName = pods.body.items[0].metadata?.name;
          const phase = pods.body.items[0].status?.phase;
          
          if (podName && phase === 'Running') {
            try {
              const logRes = await coreApi.readNamespacedPodLog(podName, namespace, 'archiver', false, undefined, undefined, undefined, undefined, undefined, 50);
              if (logRes.body?.includes('ARCHIVE_DONE')) {
                archiveDone = true;
                break;
              }
            } catch {}
          }
          
          if (phase === 'Failed') break;
        }
      } catch {}
    }

    let archiveBuffer: Buffer | null = null;
    let logs = '';

    if (archiveDone && podName) {
      try {
        const exec = new k8s.Exec(kc);
        const chunks: Buffer[] = [];

        const stdoutStream = new PassThrough();
        const stderrStream = new PassThrough();
        
        stdoutStream.on('data', (chunk: Buffer) => {
          chunks.push(chunk);
        });
        
        stderrStream.on('data', (chunk: Buffer) => {
          logs += chunk.toString();
        });

        await new Promise<void>((resolve, reject) => {
          exec.exec(
            namespace, podName!, 'archiver',
            ['cat', '/output/wp-content.tar.gz'],
            stdoutStream,
            stderrStream,
            null,
            false,
            (status: k8s.V1Status) => {
              if (status.status === 'Success') resolve();
              else reject(new Error(status.message || 'exec failed'));
            },
          );
        });

        if (chunks.length > 0) {
          archiveBuffer = Buffer.concat(chunks);
          this.logger.log(`wp-content archive retrieved: ${archiveBuffer.length} bytes`);
        }
      } catch (e: any) {
        this.logger.warn(`Exec failed for wp-content archive: ${e.message}`);
        logs = e.message;
      }
    }

    // Cleanup
    try {
      await batchApi.deleteNamespacedJob(jobName, namespace, undefined, undefined, undefined, undefined, 'Background');
    } catch {}

    if (!archiveBuffer) {
      return { data: null, logs: logs || 'Archive failed or could not be retrieved' };
    }

    return { data: archiveBuffer, logs: 'OK' };
  }

  /**
   * Restore wp-content from a tar.gz archive into the WordPress PVC.
   */
  async restoreWpContent(app: Application, archiveBuffer: Buffer): Promise<{ success: boolean; logs: string }> {
    const { coreApi, kc } = await this.getK8sClient(app.clusterId);
    const batchApi = kc.makeApiClient(k8s.BatchV1Api);
    const namespace = `user-${app.userId.split('-')[0]}`;
    const pvcName = `${app.name}-wp-content`;
    const jobName = `${app.name}-wp-restore-${Date.now()}`;
    const secretName = `${jobName}-archive`;

    // Store archive in a secret
    const archiveSecret = {
      apiVersion: 'v1',
      kind: 'Secret',
      metadata: { name: secretName, namespace },
      data: { 'wp-content.tar.gz': archiveBuffer.toString('base64') },
    };

    try {
      await coreApi.createNamespacedSecret(namespace, archiveSecret);
    } catch (e: any) {
      return { success: false, logs: `Failed to create archive secret: ${e.message}` };
    }

    const job: k8s.V1Job = {
      apiVersion: 'batch/v1',
      kind: 'Job',
      metadata: { name: jobName, namespace },
      spec: {
        ttlSecondsAfterFinished: 120,
        backoffLimit: 0,
        template: {
          spec: {
            restartPolicy: 'Never',
            containers: [{
              name: 'restore',
              image: 'alpine:3.19',
              command: ['sh', '-c', 'apk add --no-cache tar gzip > /dev/null 2>&1 && rm -rf /wp-content/* && cd /wp-content && tar xzf /archive/wp-content.tar.gz && echo "RESTORE_DONE"'],
              volumeMounts: [
                { name: 'wp-content', mountPath: '/wp-content' },
                { name: 'archive', mountPath: '/archive', readOnly: true },
              ],
              resources: { requests: { cpu: '100m', memory: '64Mi' }, limits: { cpu: '500m', memory: '256Mi' } },
            }],
            volumes: [
              { name: 'wp-content', persistentVolumeClaim: { claimName: pvcName } },
              { name: 'archive', secret: { secretName } },
            ],
          },
        },
      },
    };

    try {
      await batchApi.createNamespacedJob(namespace, job);
    } catch (e: any) {
      try { await coreApi.deleteNamespacedSecret(secretName, namespace); } catch {}
      return { success: false, logs: `Failed to create restore job: ${e.message}` };
    }

    // Wait
    const timeout = 300_000;
    const start = Date.now();
    let succeeded = false;
    let failed = false;
    while (Date.now() - start < timeout) {
      await new Promise((r) => setTimeout(r, 3000));
      try {
        const st = await batchApi.readNamespacedJob(jobName, namespace);
        if (st.body.status?.succeeded && st.body.status.succeeded > 0) { succeeded = true; break; }
        if (st.body.status?.failed && st.body.status.failed > 0) { failed = true; break; }
      } catch {}
    }

    let logs = '';
    try {
      const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, `job-name=${jobName}`);
      if (pods.body.items.length > 0 && pods.body.items[0].metadata?.name) {
        const logRes = await coreApi.readNamespacedPodLog(pods.body.items[0].metadata.name, namespace);
        logs = logRes.body || '';
      }
    } catch {}

    try { await coreApi.deleteNamespacedSecret(secretName, namespace); } catch {}

    return { success: succeeded && !failed, logs: logs || (succeeded ? 'Restore completed' : 'Restore failed or timed out') };
  }

  // ─── K8s Revision-based Rollback ─────────────────────

  /**
   * Get the list of Helm release revisions for an application.
   * Returns up to 10 revisions sorted newest-first.
   */
  async getDeploymentRevisions(app: Application): Promise<{
    revisions: Array<{
      revision: number;
      image: string;
      changeCause: string;
      createdAt: string;
      replicas: number;
      isCurrent: boolean;
    }>;
    currentRevision: number;
  }> {
    const namespace = `user-${app.userId.split('-')[0]}`;
    const releaseName = app.name;

    try {
      const kubeconfig = await this.getKubeconfig(app.clusterId);
      const helmRevisions = await this.helmService.history(releaseName, namespace, kubeconfig);

      if (!helmRevisions || helmRevisions.length === 0) {
        return { revisions: [], currentRevision: 0 };
      }

      // The last "deployed" revision is the current one
      const currentRev = helmRevisions
        .filter((r) => r.status === 'deployed')
        .sort((a, b) => b.revision - a.revision)[0];
      const currentRevision = currentRev ? currentRev.revision : 0;

      const revisions = helmRevisions
        .sort((a, b) => b.revision - a.revision)
        .slice(0, 10)
        .map((r) => ({
          revision: r.revision,
          image: '', // Helm history doesn't expose image; frontend will use description
          changeCause: r.description || `${r.status} — ${r.chart}`,
          createdAt: r.updated,
          replicas: 0,
          isCurrent: r.revision === currentRevision,
        }));

      return { revisions, currentRevision };
    } catch (e: any) {
      this.logger.warn(`Could not get Helm history for ${releaseName}: ${e.message}`);
      return { revisions: [], currentRevision: 0 };
    }
  }

  /**
   * Rollback a Helm release to a specific revision.
   */
  async rollbackDeploymentRevision(
    app: Application,
    targetRevision: number,
  ): Promise<{ success: boolean; message: string }> {
    const namespace = `user-${app.userId.split('-')[0]}`;
    const releaseName = app.name;

    try {
      const kubeconfig = await this.getKubeconfig(app.clusterId);
      await this.helmService.rollback(releaseName, targetRevision, namespace, kubeconfig);
      this.logger.log(`Rolled back ${releaseName} to Helm revision ${targetRevision}`);
      return { success: true, message: `Rolled back to Helm revision ${targetRevision}` };
    } catch (e: any) {
      this.logger.error(`Helm rollback failed for ${releaseName}: ${e.message}`);
      return { success: false, message: e.message };
    }
  }

  private generatePassword(length = 24): string {
    const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
    let password = '';
    for (let i = 0; i < length; i++) {
      password += chars.charAt(Math.floor(Math.random() * chars.length));
    }
    return password;
  }
}