import { Injectable, Logger, Inject } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { Redis } from 'ioredis';
import { REDIS_CLIENT } from '../common/redis/redis.module';
import * as k8s from '@kubernetes/client-node';
import { ChildProcess } from 'child_process';
import * as net from 'net';
import { Application } from '../applications/entities/application.entity';
import { AppRuntime } from '../common/enums';
import { ClustersService } from '../clusters/clusters.service';
import { RegistryService } from '../kubernetes/registry.service';
import { StorageService } from '../common/storage/storage.service';

export class BuildCancelledError extends Error {
  constructor() {
    super('Build cancelled by user');
    this.name = 'BuildCancelledError';
  }
}

interface ActiveBuildSession {
  cancelled: boolean;
  coreApi?: k8s.CoreV1Api;
  batchApi?: k8s.BatchV1Api;
  namespace?: string;
  buildPodName?: string;
  sourcePvcName?: string;
  helperPodName?: string;
  clusterId?: string;
  processes: ChildProcess[];
  socket?: net.Socket;
}

/**
 * Serializable subset of a build session persisted to Redis so a build can be
 * cancelled (or have its live logs read) from a backend replica other than the
 * one running the build. `clusterId` lets that replica rebuild a K8s client.
 */
interface PersistedBuildSession {
  namespace?: string;
  buildPodName?: string;
  sourcePvcName?: string;
  helperPodName?: string;
  clusterId?: string;
}

export interface BuildProgress {
  phase: 'uploading' | 'building' | 'deploying' | 'done' | 'failed' | 'cancelled';
  percent: number;
  bytesUploaded?: number;
  totalBytes?: number;
  message?: string;
}

@Injectable()
export class BuildService {
  private readonly logger = new Logger(BuildService.name);
  /** Local, non-serializable session state (child processes, sockets, API clients). */
  private readonly activeBuilds = new Map<string, ActiveBuildSession>();
  /** Build state TTL in Redis (1h) — long enough for the slowest build + final read. */
  private static readonly STATE_TTL_SECONDS = 3600;
  /**
   * Kaniko executor image. Pinned (not `:latest`) so it can be cached on the node
   * with imagePullPolicy=IfNotPresent — avoids re-pulling the ~250MB image on every build.
   */
  private readonly kanikoImage = process.env.KANIKO_IMAGE || 'gcr.io/kaniko-project/executor:v1.23.2';

  constructor(
    private configService: ConfigService,
    private clustersService: ClustersService,
    private registryService: RegistryService,
    private storageService: StorageService,
    @Inject(REDIS_CLIENT) private readonly redis: Redis,
  ) {}

  // ─── Redis keys for cross-replica build state ──────────────────────
  private progressKey(id: string): string {
    return `build:progress:${id}`;
  }
  private sessionKey(id: string): string {
    return `build:session:${id}`;
  }
  private cancelKey(id: string): string {
    return `build:cancelled:${id}`;
  }

  private beginBuildSession(deploymentId: string): void {
    this.activeBuilds.set(deploymentId, { cancelled: false, processes: [] });
  }

  private getSession(deploymentId?: string): ActiveBuildSession | undefined {
    if (!deploymentId) return undefined;
    return this.activeBuilds.get(deploymentId);
  }

  private updateBuildSession(deploymentId: string, update: Partial<ActiveBuildSession>): void {
    const session = this.activeBuilds.get(deploymentId);
    if (session) Object.assign(session, update);
    // Mirror the serializable subset to Redis so another replica can cancel /
    // read logs for this build. Fire-and-forget — never block the build on it.
    const persisted: PersistedBuildSession = {
      namespace: session?.namespace,
      buildPodName: session?.buildPodName,
      sourcePvcName: session?.sourcePvcName,
      helperPodName: session?.helperPodName,
      clusterId: session?.clusterId,
    };
    void this.redis
      .set(this.sessionKey(deploymentId), JSON.stringify(persisted), 'EX', BuildService.STATE_TTL_SECONDS)
      .catch(() => undefined);
  }

  /** Read the persisted (cross-replica) session metadata for a build. */
  private async readPersistedSession(deploymentId: string): Promise<PersistedBuildSession | null> {
    try {
      const raw = await this.redis.get(this.sessionKey(deploymentId));
      return raw ? (JSON.parse(raw) as PersistedBuildSession) : null;
    } catch {
      return null;
    }
  }

  /** Build a CoreV1Api/BatchV1Api pair for a cluster (used for cross-replica cleanup). */
  private async makeClusterApis(clusterId?: string): Promise<{ coreApi: k8s.CoreV1Api; batchApi: k8s.BatchV1Api } | null> {
    try {
      const cluster = clusterId ? await this.clustersService.findOne(clusterId) : await this.clustersService.getDefault();
      const kc = new k8s.KubeConfig();
      kc.loadFromString(cluster.kubeconfig);
      return { coreApi: kc.makeApiClient(k8s.CoreV1Api), batchApi: kc.makeApiClient(k8s.BatchV1Api) };
    } catch {
      return null;
    }
  }

  /**
   * Whether this build has been cancelled — checks both the local session flag
   * and the shared Redis flag, so a cancel issued on any replica is observed by
   * the replica actually running the build.
   */
  private async isCancelledShared(deploymentId?: string): Promise<boolean> {
    if (!deploymentId) return false;
    if (this.activeBuilds.get(deploymentId)?.cancelled) return true;
    try {
      return (await this.redis.exists(this.cancelKey(deploymentId))) === 1;
    } catch {
      return false;
    }
  }

  private async throwIfCancelledShared(deploymentId?: string): Promise<void> {
    if (await this.isCancelledShared(deploymentId)) {
      throw new BuildCancelledError();
    }
  }

  private registerProcess(deploymentId: string | undefined, proc: ChildProcess): void {
    const session = this.getSession(deploymentId);
    if (!session) return;
    session.processes.push(proc);
    if (session.cancelled) {
      try {
        proc.kill('SIGKILL');
      } catch {
        /* ignore */
      }
    }
  }

  private registerSocket(deploymentId: string | undefined, socket: net.Socket): void {
    const session = this.getSession(deploymentId);
    if (!session) return;
    if (session.socket) {
      try {
        session.socket.destroy();
      } catch {
        /* ignore */
      }
    }
    session.socket = socket;
    if (session.cancelled) {
      try {
        socket.destroy();
      } catch {
        /* ignore */
      }
    }
  }

  private throwIfCancelled(deploymentId?: string): void {
    if (deploymentId && this.activeBuilds.get(deploymentId)?.cancelled) {
      throw new BuildCancelledError();
    }
  }

  private endBuildSession(deploymentId?: string): void {
    if (!deploymentId) return;
    this.activeBuilds.delete(deploymentId);
    void this.redis.del(this.sessionKey(deploymentId), this.cancelKey(deploymentId)).catch(() => undefined);
  }

  /** Delete the K8s artifacts (helper pod, build job, source PVC, dockerfile configmap) of one build. */
  private async deleteBuildArtifacts(
    coreApi: k8s.CoreV1Api,
    batchApi: k8s.BatchV1Api,
    namespace: string,
    names: { buildPodName?: string; sourcePvcName?: string; helperPodName?: string },
  ): Promise<void> {
    const { buildPodName, sourcePvcName, helperPodName } = names;
    const cleanup: Promise<unknown>[] = [];
    if (helperPodName) {
      cleanup.push(coreApi.deleteNamespacedPod({ name: helperPodName, namespace, gracePeriodSeconds: 0 }).catch(() => undefined));
    }
    if (buildPodName) {
      cleanup.push(
        batchApi
          .deleteNamespacedJob({ name: buildPodName, namespace, gracePeriodSeconds: 0, propagationPolicy: 'Foreground' })
          .catch(() => undefined),
      );
    }
    if (sourcePvcName) {
      cleanup.push(coreApi.deleteNamespacedPersistentVolumeClaim({ name: sourcePvcName, namespace }).catch(() => undefined));
    }
    if (buildPodName) {
      cleanup.push(coreApi.deleteNamespacedConfigMap({ name: `${buildPodName}-dockerfile`, namespace }).catch(() => undefined));
    }
    await Promise.all(cleanup);
  }

  async cancelBuild(deploymentId: string): Promise<void> {
    this.logger.log(`Cancelling build for deployment ${deploymentId}`);
    // Shared flag so the (possibly different) replica running the build observes
    // the cancellation via isCancelledShared and aborts its wait loop.
    void this.redis.set(this.cancelKey(deploymentId), '1', 'EX', BuildService.STATE_TTL_SECONDS).catch(() => undefined);

    const session = this.activeBuilds.get(deploymentId);
    if (session) {
      // Local build — stop in-process work and clean up via the live API clients.
      session.cancelled = true;
      if (session.socket) {
        try {
          session.socket.destroy();
        } catch {
          /* ignore */
        }
      }
      for (const proc of session.processes) {
        try {
          proc.kill('SIGKILL');
        } catch {
          /* ignore */
        }
      }
      if (session.coreApi && session.batchApi && session.namespace) {
        await this.deleteBuildArtifacts(session.coreApi, session.batchApi, session.namespace, session);
        this.logger.log(`Cleaned up K8s build resources for deployment ${deploymentId}`);
      }
    } else {
      // Build is running on another replica (or already finished) — reconstruct
      // a client from the persisted session metadata and clean up its artifacts.
      const persisted = await this.readPersistedSession(deploymentId);
      if (persisted?.namespace) {
        const apis = await this.makeClusterApis(persisted.clusterId);
        if (apis) {
          await this.deleteBuildArtifacts(apis.coreApi, apis.batchApi, persisted.namespace, persisted);
          this.logger.log(`Cleaned up cross-replica K8s build resources for deployment ${deploymentId}`);
        }
      }
    }

    this.setProgress(deploymentId, { phase: 'cancelled', percent: 0, message: 'Cancelled by user' });
    this.endBuildSession(deploymentId);
  }

  /** Delete all in-flight build artifacts for an app (helper pods, jobs, PVCs, configmaps). */
  async cleanupBuildResourcesForApp(app: Application): Promise<void> {
    const buildNamespace = this.configService.get<string>('build.namespace') || 'cloudhost-builds';
    const prefix = `build-${app.name}-`;

    const cluster = app.clusterId ? await this.clustersService.findOne(app.clusterId) : await this.clustersService.getDefault();

    const kc = new k8s.KubeConfig();
    kc.loadFromString(cluster.kubeconfig);
    const coreApi = kc.makeApiClient(k8s.CoreV1Api);
    const batchApi = kc.makeApiClient(k8s.BatchV1Api);

    const cleanup: Promise<unknown>[] = [];

    const [pods, pvcs, jobs, configMaps] = await Promise.all([
      coreApi.listNamespacedPod({ namespace: buildNamespace }),
      coreApi.listNamespacedPersistentVolumeClaim({
        namespace: buildNamespace,
      }),
      batchApi.listNamespacedJob({ namespace: buildNamespace }),
      coreApi.listNamespacedConfigMap({ namespace: buildNamespace }),
    ]);

    for (const pod of pods.items) {
      const name = pod.metadata?.name || '';
      if (name.startsWith(prefix)) {
        cleanup.push(
          coreApi
            .deleteNamespacedPod({
              name,
              namespace: buildNamespace,
              gracePeriodSeconds: 0,
            })
            .catch(() => undefined),
        );
      }
    }
    for (const pvc of pvcs.items) {
      const name = pvc.metadata?.name || '';
      if (name.startsWith(prefix)) {
        cleanup.push(
          coreApi
            .deleteNamespacedPersistentVolumeClaim({
              name,
              namespace: buildNamespace,
            })
            .catch(() => undefined),
        );
      }
    }
    for (const job of jobs.items) {
      const name = job.metadata?.name || '';
      if (name.startsWith(prefix)) {
        cleanup.push(
          batchApi
            .deleteNamespacedJob({
              name,
              namespace: buildNamespace,
              gracePeriodSeconds: 0,
              propagationPolicy: 'Foreground',
            })
            .catch(() => undefined),
        );
      }
    }
    for (const cm of configMaps.items) {
      const name = cm.metadata?.name || '';
      if (name.startsWith(prefix)) {
        cleanup.push(coreApi.deleteNamespacedConfigMap({ name, namespace: buildNamespace }).catch(() => undefined));
      }
    }

    await Promise.all(cleanup);
    this.logger.log(`Cleaned up all build resources matching "${prefix}*" in ${buildNamespace}`);
  }

  async getProgress(deploymentId: string): Promise<BuildProgress | null> {
    try {
      const raw = await this.redis.get(this.progressKey(deploymentId));
      return raw ? (JSON.parse(raw) as BuildProgress) : null;
    } catch {
      return null;
    }
  }

  /**
   * Best-effort progress write. Kept synchronous (fire-and-forget) so the many
   * call sites in the deploy pipeline don't need to await telemetry; a Redis
   * blip must never fail a build.
   */
  setProgress(deploymentId: string | undefined, progress: BuildProgress): void {
    if (!deploymentId) return;
    void this.redis
      .set(this.progressKey(deploymentId), JSON.stringify(progress), 'EX', BuildService.STATE_TTL_SECONDS)
      .catch(() => undefined);
  }

  async clearProgress(deploymentId: string): Promise<void> {
    await this.redis.del(this.progressKey(deploymentId)).catch(() => undefined);
  }

  /**
   * Builds a Docker image for the application using Kaniko inside K8s.
   * Returns { imageUri, buildLog } — the full image URI and the build logs.
   */
  async buildImage(app: Application, deploymentId?: string): Promise<{ imageUri: string; buildLog: string }> {
    const registryUrl = this.registryService.getRegistryUrl();
    const buildNamespace = this.registryService.getBuildNamespace();
    const tag = `${Date.now()}`;
    const imageUri = this.registryService.buildImageReference(app.userId, app.name, tag);

    this.logger.log(`Starting image build for ${app.name} → ${imageUri}`);

    if (deploymentId) {
      this.beginBuildSession(deploymentId);
    }

    // Build mode:
    //  • templated  → WordPress (and fresh installs): a generated Dockerfile is
    //    injected via ConfigMap (Nixpacks can't build a WordPress upload).
    //  • nixpacks    → every code runtime: an init container picks the user's own
    //    Dockerfile if present (BYO), otherwise generates one with Nixpacks.
    const useTemplated = app.runtime === AppRuntime.WORDPRESS;
    const dockerfileContent = useTemplated ? this.wordpressDockerfile(app) : null;

    // Create Kaniko build pod
    const buildPodName = `build-${app.name}-${tag}`.substring(0, 63).replace(/[^a-z0-9-]/g, '');

    if (deploymentId) {
      this.updateBuildSession(deploymentId, { buildPodName });
    }

    // Use the cluster's kubeconfig instead of default
    const cluster = app.clusterId ? await this.clustersService.findOne(app.clusterId) : await this.clustersService.getDefault();

    const kc = new k8s.KubeConfig();
    kc.loadFromString(cluster.kubeconfig);
    const coreApi = kc.makeApiClient(k8s.CoreV1Api);
    const batchApi = kc.makeApiClient(k8s.BatchV1Api);

    if (deploymentId) {
      this.updateBuildSession(deploymentId, {
        coreApi,
        batchApi,
        namespace: buildNamespace,
        clusterId: cluster.id,
      });
    }

    // Ensure the build namespace exists
    await this.ensureNamespace(coreApi, buildNamespace);
    await this.throwIfCancelledShared(deploymentId);

    // Determine source: uploaded code (MinIO object key in app.codePath) or git URL.
    const hasUploadedCode = !!app.codePath;
    const hasGitUrl = !!app.gitUrl;

    // For private git repos, the token is delivered via a per-build Secret (env)
    // and used through git's credential store inside the pod — never embedded in
    // the clone URL/args or the Job manifest (which would leak it into etcd/logs).
    let gitHost = '';
    if (hasGitUrl) {
      try {
        gitHost = new URL(app.gitUrl!).host;
      } catch {
        /* malformed URL — fall back to inline injection below */
      }
    }
    const useGitTokenSecret = hasGitUrl && !!app.gitToken && !!gitHost;
    const gitSecretName = `${buildPodName}-git`;

    // Create ConfigMap with Dockerfile
    const dockerfileConfigMap = {
      apiVersion: 'v1',
      kind: 'ConfigMap',
      metadata: {
        name: `${buildPodName}-dockerfile`,
        namespace: buildNamespace,
      },
      data: {
        Dockerfile: dockerfileContent ?? '',
      },
    };

    // For uploaded code, mint a short-lived presigned URL the build pod downloads
    // from MinIO (replaces the PVC + helper pod + kubectl cp upload path).
    let sourceDownloadUrl: string | undefined;
    if (hasUploadedCode) {
      sourceDownloadUrl = await this.storageService.presignSourceGet(app.codePath!);
    }

    // Build the Kaniko Job spec
    // Always use dir context — init containers prepare /workspace/source
    const kanikoArgs = [
      '--dockerfile=/workspace/Dockerfile',
      '--context=dir:///workspace/source',
      `--destination=${imageUri}`,
      '--cache=true',
      `--cache-repo=${registryUrl}/${app.userId}/cache`,
      '--insecure',
      '--skip-tls-verify',
      '--single-snapshot',
      '--snapshot-mode=redo',
    ];

    const volumes: any[] = [
      {
        name: 'docker-config',
        secret: { secretName: 'registry-credentials' },
      },
      {
        name: 'workspace',
        emptyDir: {},
      },
    ];

    // The generated Dockerfile is only mounted (via ConfigMap) in templated mode.
    if (useTemplated) {
      volumes.push({ name: 'dockerfile', configMap: { name: `${buildPodName}-dockerfile` } });
    }

    // In templated mode each staging container copies the ConfigMap Dockerfile to
    // /workspace/Dockerfile; in nixpacks mode the nixpacks-prepare container writes
    // it instead, so staging just lays down the source.
    const copyTemplatedDockerfile = useTemplated ? 'cp /workspace/Dockerfile /workspace-out/Dockerfile &&' : '';
    const stagingDockerfileMounts = useTemplated
      ? [{ name: 'dockerfile', mountPath: '/workspace/Dockerfile', subPath: 'Dockerfile' }]
      : [];

    const initContainers: any[] = [];

    if (hasUploadedCode && sourceDownloadUrl) {
      // Download the source archive from MinIO via the presigned URL, then unzip
      // it into /workspace/source (no PVC, no helper pod, no credentials in-pod).
      initContainers.push({
        name: 'fetch-source',
        image: 'alpine:3.19',
        imagePullPolicy: 'IfNotPresent',
        env: [{ name: 'SOURCE_URL', value: sourceDownloadUrl }],
        command: [
          'sh',
          '-c',
          `
          apk add --no-cache unzip tar gzip wget &&
          ${copyTemplatedDockerfile}
          echo ">>> Downloading source archive from object storage..." &&
          wget -q -O /tmp/source.zip "$SOURCE_URL" &&
          mkdir -p /tmp/extract &&
          cd /tmp/extract &&
          if tar tzf /tmp/source.zip >/dev/null 2>&1; then
            echo ">>> Detected gzip tarball" &&
            tar xzf /tmp/source.zip
          elif unzip -t /tmp/source.zip >/dev/null 2>&1; then
            echo ">>> Detected zip archive" &&
            unzip -q /tmp/source.zip
          else
            echo "ERROR: source archive is not a valid zip or tar.gz" && exit 1
          fi &&
          echo "--- Extracted contents ---" &&
          ls -la /tmp/extract/ &&
          mkdir -p /workspace-out/source &&
          ITEMS=$(ls -1 /tmp/extract/ | head -5) &&
          COUNT=$(ls -1 /tmp/extract/ | wc -l) &&
          if [ "$COUNT" -eq 1 ] && [ -d "/tmp/extract/$ITEMS" ]; then
            echo ">>> Single subfolder detected: $ITEMS — flattening to root" &&
            cp -a /tmp/extract/$ITEMS/. /workspace-out/source/
          else
            echo ">>> Multiple items or files — copying as-is" &&
            cp -a /tmp/extract/. /workspace-out/source/
          fi &&
          rm -rf /tmp/extract /tmp/source.zip &&
          echo "--- Final workspace contents ---" &&
          ls -la /workspace-out/source/
        `,
        ],
        volumeMounts: [
          { name: 'workspace', mountPath: '/workspace-out' },
          ...stagingDockerfileMounts,
        ],
      });
    } else if (hasGitUrl) {
      const branch = app.gitBranch || 'main';
      const gitCopyDockerfile = useTemplated ? 'cp /dockerfile/Dockerfile /workspace-out/Dockerfile &&' : '';

      // Clone command. Three cases:
      //  • token + parseable host → token comes from $GIT_TOKEN (Secret env) via
      //    git's credential store; the clone URL stays token-free.
      //  • token + unparseable host (rare) → fall back to inline token injection.
      //  • no token (public repo) → plain clone.
      let cloneCmd: string;
      let gitEnv: any[] | undefined;
      if (useGitTokenSecret) {
        gitEnv = [{ name: 'GIT_TOKEN', valueFrom: { secretKeyRef: { name: gitSecretName, key: 'token' } } }];
        cloneCmd =
          `git config --global credential.helper store && ` +
          `printf 'https://%s@%s\\n' "$GIT_TOKEN" '${gitHost}' > "$HOME/.git-credentials" && ` +
          `chmod 600 "$HOME/.git-credentials" && ` +
          `git clone --depth 1 --branch ${branch} '${app.gitUrl}' /workspace-out/source && ` +
          `rm -f "$HOME/.git-credentials" &&`;
      } else if (app.gitToken) {
        const cloneUrl = app.gitUrl!.replace('https://', `https://${app.gitToken}@`);
        cloneCmd = `git clone --depth 1 --branch ${branch} ${cloneUrl} /workspace-out/source &&`;
      } else {
        cloneCmd = `git clone --depth 1 --branch ${branch} '${app.gitUrl}' /workspace-out/source &&`;
      }

      // Clone git repo into /workspace/source
      initContainers.push({
        name: 'git-clone',
        image: 'alpine/git:2.43.0',
        imagePullPolicy: 'IfNotPresent',
        env: gitEnv,
        command: [
          'sh',
          '-c',
          `
          echo ">>> Cloning branch '${branch}' from ${app.gitUrl}" &&
          ${cloneCmd}
          ${gitCopyDockerfile}
          echo ">>> Workspace contents:" &&
          ls -la /workspace-out/source/
        `,
        ],
        volumeMounts: [
          { name: 'workspace', mountPath: '/workspace-out' },
          ...(useTemplated ? [{ name: 'dockerfile', mountPath: '/dockerfile' }] : []),
        ],
      });
    } else if (useTemplated) {
      // No uploaded code and no git — only valid for templated fresh installs
      // (e.g. fresh WordPress). Create empty source dir + copy Dockerfile.
      initContainers.push({
        name: 'prepare-workspace',
        image: 'alpine:3.19',
        imagePullPolicy: 'IfNotPresent',
        command: [
          'sh',
          '-c',
          `
          mkdir -p /workspace-out/source &&
          cp /dockerfile/Dockerfile /workspace-out/Dockerfile &&
          echo ">>> Prepared empty workspace for fresh install" &&
          ls -la /workspace-out/
        `,
        ],
        volumeMounts: [
          { name: 'workspace', mountPath: '/workspace-out' },
          { name: 'dockerfile', mountPath: '/dockerfile' },
        ],
      });
    }

    // Nixpacks mode: after the source is staged, pick the user's Dockerfile (BYO)
    // or generate one with Nixpacks, writing the result to /workspace/Dockerfile.
    if (!useTemplated) {
      initContainers.push(this.nixpacksPrepareInitContainer(app));
    }

    // Kaniko container volume mounts
    const kanikoVolumeMounts: any[] = [
      { name: 'docker-config', mountPath: '/kaniko/.docker' },
      { name: 'workspace', mountPath: '/workspace' },
    ];

    const buildJob: k8s.V1Job = {
      apiVersion: 'batch/v1',
      kind: 'Job',
      metadata: {
        name: buildPodName,
        namespace: buildNamespace,
      },
      spec: {
        backoffLimit: 1,
        ttlSecondsAfterFinished: 300,
        template: {
          spec: {
            serviceAccountName: this.configService.get<string>('build.serviceAccount'),
            initContainers: initContainers.length > 0 ? initContainers : undefined,
            containers: [
              {
                name: 'kaniko',
                image: this.kanikoImage,
                imagePullPolicy: 'IfNotPresent',
                args: kanikoArgs,
                volumeMounts: kanikoVolumeMounts,
                resources: {
                  requests: { cpu: '500m', memory: '1Gi' },
                  limits: { cpu: '2', memory: '4Gi' },
                },
              },
            ],
            restartPolicy: 'Never',
            volumes,
          },
        },
      },
    };

    try {
      if (useTemplated) {
        const t0 = Date.now();
        await coreApi.createNamespacedConfigMap({
          namespace: buildNamespace!,
          body: dockerfileConfigMap,
        });
        this.logger.log(`[timing] ConfigMap created in ${Date.now() - t0}ms`);
      }

      // Per-build Secret holding the git token (mounted as $GIT_TOKEN env).
      if (useGitTokenSecret) {
        await coreApi.createNamespacedSecret({
          namespace: buildNamespace!,
          body: {
            metadata: { name: gitSecretName, namespace: buildNamespace },
            type: 'Opaque',
            data: { token: Buffer.from(app.gitToken!).toString('base64') },
          },
        });
      }

      const t1 = Date.now();
      await batchApi.createNamespacedJob({
        namespace: buildNamespace!,
        body: buildJob,
      });
      this.logger.log(`[timing] Job created in ${Date.now() - t1}ms`);

      // Wait for build to complete
      this.setProgress(deploymentId, {
        phase: 'building',
        percent: 15,
        message: 'Building Docker image...',
      });
      const buildTimeout = this.configService.get<number>('build.timeoutSeconds') || 600;
      await this.waitForJobCompletion(batchApi, coreApi, buildPodName, buildNamespace!, buildTimeout, deploymentId);

      // Capture build logs on success
      let buildLog = '';
      try {
        buildLog = await this.getBuildLogs(coreApi, buildPodName, buildNamespace!);
      } catch {}

      this.logger.log(`Build completed successfully: ${imageUri}`);
      return { imageUri, buildLog };
    } catch (error: any) {
      if (error instanceof BuildCancelledError || error?.name === 'BuildCancelledError') {
        throw error;
      }
      // Try to get build logs for debugging
      let buildLog = '';
      try {
        buildLog = await this.getBuildLogs(coreApi, buildPodName, buildNamespace!);
        this.logger.error(`Build logs for ${buildPodName}:\n${buildLog}`);
      } catch {}

      this.logger.error(`Build failed for ${app.name}:`, error.body || error.message);
      const err = new Error(`Image build failed: ${error.body?.message || error.message}`);
      (err as any).buildLog = buildLog;
      throw err;
    } finally {
      // Clean up Dockerfile ConfigMap (templated mode only)
      if (useTemplated) {
        try {
          await coreApi.deleteNamespacedConfigMap({
            name: `${buildPodName}-dockerfile`,
            namespace: buildNamespace!,
          });
        } catch (e: any) {
          this.logger.warn(`Failed to clean up ConfigMap: ${e.message}`);
        }
      }
      // Clean up the per-build git-token Secret.
      if (useGitTokenSecret) {
        try {
          await coreApi.deleteNamespacedSecret({ name: gitSecretName, namespace: buildNamespace! });
        } catch (e: any) {
          this.logger.warn(`Failed to clean up git-token Secret: ${e.message}`);
        }
      }
      this.endBuildSession(deploymentId);
    }
  }

  /**
   * Ensure the build namespace exists with all required resources
   * (namespace, service account, registry-credentials secret).
   */
  private async ensureNamespace(coreApi: k8s.CoreV1Api, namespace: string): Promise<void> {
    // 1. Ensure namespace
    try {
      await coreApi.readNamespace({ name: namespace });
    } catch (err: any) {
      if (err.code === 404 || err.body?.code === 404) {
        this.logger.log(`Namespace "${namespace}" not found — creating it`);
        await coreApi.createNamespace({
          body: { metadata: { name: namespace } },
        });
      } else {
        throw err;
      }
    }

    // 2. Ensure service account for Kaniko
    const saName = this.configService.get<string>('build.serviceAccount') || 'kaniko-builder';
    try {
      await coreApi.readNamespacedServiceAccount({ name: saName, namespace });
    } catch (err: any) {
      if (err.code === 404 || err.body?.code === 404) {
        this.logger.log(`ServiceAccount "${saName}" not found in "${namespace}" — creating it`);
        await coreApi.createNamespacedServiceAccount({
          namespace,
          body: { metadata: { name: saName, namespace } },
        });
      } else {
        throw err;
      }
    }

    // 3. Ensure registry-credentials secret (docker config for Kaniko to push)
    const registrySecretName = 'registry-credentials';
    try {
      await coreApi.readNamespacedSecret({
        name: registrySecretName,
        namespace,
      });
    } catch (err: any) {
      if (err.code === 404 || err.body?.code === 404) {
        this.logger.log(`Secret "${registrySecretName}" not found in "${namespace}" — creating it`);
        await coreApi.createNamespacedSecret({
          namespace,
          body: {
            metadata: { name: registrySecretName, namespace },
            type: 'kubernetes.io/dockerconfigjson',
            data: {
              '.dockerconfigjson': Buffer.from(this.registryService.buildDockerConfigJson()).toString('base64'),
            },
          },
        });
      } else {
        throw err;
      }
    }
  }

  /** Single-quote a string for safe inclusion in a `sh -c` command. */
  private shellQuote(value: string): string {
    return `'${value.replace(/'/g, `'\\''`)}'`;
  }

  /**
   * NIXPACKS_* planning env (read by the nixpacks process itself) derived from the
   * app's selected runtime version. Other versions are inferred by Nixpacks from
   * project files (go.mod, .python-version, …), the Nixpacks-idiomatic way.
   */
  private nixpacksPlanEnv(app: Application): { name: string; value: string }[] {
    const env: { name: string; value: string }[] = [];
    if (app.runtime === AppRuntime.NODEJS && app.runtimeVersion) {
      env.push({ name: 'NIXPACKS_NODE_VERSION', value: String(app.runtimeVersion) });
    }
    if ((app.runtime === AppRuntime.PYTHON || app.runtime === AppRuntime.DJANGO) && app.runtimeVersion) {
      env.push({ name: 'NIXPACKS_PYTHON_VERSION', value: String(app.runtimeVersion) });
    }
    return env;
  }

  /**
   * Init container for nixpacks (non-WordPress) builds. Turns staged source at
   * /workspace/source into a Dockerfile at /workspace/Dockerfile:
   *   • if the source ships its own Dockerfile → use it (BYO, full user control),
   *   • otherwise generate one with Nixpacks (`nixpacks build --out`).
   * Mirror/proxy env from `build.nixpacksBuildEnv` is baked into the generated
   * image so package installs in the Kaniko stage work behind the Iran network.
   */
  private nixpacksPrepareInitContainer(app: Application): any {
    const image = this.configService.get<string>('build.nixpacksImage') || 'ghcr.io/railwayapp/nixpacks:latest';
    const buildEnv = this.configService.get<string[]>('build.nixpacksBuildEnv') || [];
    const envFlags = buildEnv.map((kv) => `--env ${this.shellQuote(kv)}`).join(' ');
    const planEnv = this.nixpacksPlanEnv(app);

    return {
      name: 'nixpacks-prepare',
      image,
      imagePullPolicy: 'IfNotPresent',
      env: planEnv.length ? planEnv : undefined,
      command: [
        'sh',
        '-c',
        `
        set -e
        cd /workspace
        if [ -f source/Dockerfile ]; then
          echo ">>> Using user-provided Dockerfile (BYO)" &&
          cp source/Dockerfile /workspace/Dockerfile
        else
          echo ">>> No Dockerfile found — generating with Nixpacks" &&
          nixpacks build source --out source ${envFlags} &&
          cp source/.nixpacks/Dockerfile /workspace/Dockerfile &&
          echo "--- Generated Dockerfile ---" &&
          cat /workspace/Dockerfile
        fi
        `,
      ],
      volumeMounts: [{ name: 'workspace', mountPath: '/workspace' }],
    };
  }

  private wordpressDockerfile(app: Application): string {
    const wpVersion = app.runtimeVersion || '6.7';
    const phpVersion = app.phpVersion || '8.3';
    const hasUploadedCode = !!app.codePath;

    return `FROM wordpress:${wpVersion}-php${phpVersion}-apache

# Install additional PHP extensions commonly needed by WordPress
RUN docker-php-ext-install opcache

# Enable Apache mod_rewrite for pretty permalinks
RUN a2enmod rewrite

# Increase PHP upload limits for WordPress media
RUN echo "upload_max_filesize = 64M\\npost_max_size = 64M\\nmax_execution_time = 300\\nmemory_limit = 256M" > /usr/local/etc/php/conf.d/uploads.ini

${
  hasUploadedCode
    ? `# Copy user's custom WordPress files
COPY . /tmp/user-content

# Auto-detect: full public_html root (has wp-admin) vs wp-content only
# ── public_html mode ──
#   wp-admin/ and wp-includes/ replace the base-image core so the user's
#   exact WordPress version & patches are preserved.
#   wp-content/ is staged in /usr/src/wordpress-user/ and merged into the
#   PVC on first boot (same as migrate mode).
#   wp-config.php is saved separately so docker-entrypoint.sh can still
#   inject WORDPRESS_DB_* env-vars when no config exists yet.
# ── migrate mode (no wp-admin) ──
#   Only wp-content + config files are processed.
RUN mkdir -p /usr/src/wordpress-user && \\
    if [ -d /tmp/user-content/wp-admin ]; then \\
      echo ">>> Full WordPress root (public_html) detected" && \\
      echo ">>> Copying wp-admin/ to /var/www/html/" && \\
      rm -rf /var/www/html/wp-admin && \\
      cp -a /tmp/user-content/wp-admin /var/www/html/wp-admin && \\
      echo ">>> Copying wp-includes/ to /var/www/html/" && \\
      rm -rf /var/www/html/wp-includes && \\
      cp -a /tmp/user-content/wp-includes /var/www/html/wp-includes && \\
      echo ">>> Copying root PHP files (index.php, wp-login.php, ...)" && \\
      find /tmp/user-content -maxdepth 1 -name "*.php" ! -name "wp-config.php" \\
        -exec cp {} /var/www/html/ \\; 2>/dev/null || true && \\
      echo ">>> Copying other root files/dirs (fonts, assets, etc.)" && \\
      for item in /tmp/user-content/*; do \\
        name=$(basename "$item"); \\
        case "$name" in \\
          wp-admin|wp-includes|wp-content|wp-config.php|.htaccess) ;; \\
          *.php) ;; \\
          *) \\
            if [ -f "$item" ]; then \\
              echo "    root file: $name" && \\
              cp "$item" /var/www/html/; \\
            elif [ -d "$item" ]; then \\
              echo "    root dir:  $name/" && \\
              cp -a "$item" /var/www/html/; \\
            fi ;; \\
        esac; \\
      done; \\
    else \\
      echo ">>> wp-content / config files only (migrate mode)"; \\
    fi && \\
    if [ -d /tmp/user-content/wp-content ]; then \\
      echo ">>> Staging user wp-content (themes, plugins, uploads)..." && \\
      cp -a /tmp/user-content/wp-content /usr/src/wordpress-user/wp-content; \\
    elif [ -d /tmp/user-content/themes ] || [ -d /tmp/user-content/plugins ] || [ -d /tmp/user-content/uploads ]; then \\
      echo ">>> Staging loose themes/plugins/uploads into wp-content..." && \\
      mkdir -p /usr/src/wordpress-user/wp-content && \\
      [ -d /tmp/user-content/themes ] && cp -a /tmp/user-content/themes /usr/src/wordpress-user/wp-content/ || true && \\
      [ -d /tmp/user-content/plugins ] && cp -a /tmp/user-content/plugins /usr/src/wordpress-user/wp-content/ || true && \\
      [ -d /tmp/user-content/uploads ] && cp -a /tmp/user-content/uploads /usr/src/wordpress-user/wp-content/ || true; \\
    fi && \\
    # wp-config.php is intentionally NOT copied — docker-entrypoint.sh generates it
    # from WORDPRESS_DB_* env vars so credentials always match the deployed database.
    if [ -f /tmp/user-content/.htaccess ]; then \\
      echo ">>> Copying .htaccess" && \\
      cp /tmp/user-content/.htaccess /var/www/html/.htaccess; \\
    fi && \\
    rm -rf /tmp/user-content && \\
    echo ">>> WordPress user content staged"

# Custom entrypoint:
#  1. Merge staged wp-content into the PVC mount (every start — idempotent)
#  2. Hand off to official docker-entrypoint.sh which creates wp-config.php
#     from WORDPRESS_DB_* env vars (never use uploaded wp-config with old credentials)
RUN { \\
  echo '#!/bin/bash'; \\
  echo 'set -e'; \\
  echo ''; \\
  echo '# ── Merge user wp-content into PVC ──'; \\
  echo 'if [ -d /usr/src/wordpress-user/wp-content ]; then'; \\
  echo '  echo ">>> Merging user wp-content into PVC..."'; \\
  echo '  mkdir -p /var/www/html/wp-content'; \\
  echo '  cp -a /usr/src/wordpress-user/wp-content/. /var/www/html/wp-content/'; \\
  echo '  chown -R www-data:www-data /var/www/html/wp-content'; \\
  echo '  echo ">>> User wp-content merged successfully"'; \\
  echo 'fi'; \\
  echo ''; \\
  echo 'exec docker-entrypoint.sh apache2-foreground'; \\
} > /usr/local/bin/cloudhost-entrypoint.sh && chmod +x /usr/local/bin/cloudhost-entrypoint.sh
`
    : `# Fresh install — no user content to merge
`
}
# Set proper ownership
RUN chown -R www-data:www-data /var/www/html

EXPOSE 80
${
  hasUploadedCode
    ? `ENTRYPOINT ["cloudhost-entrypoint.sh"]
CMD []`
    : `CMD ["apache2-foreground"]`
}
`;
  }

  /**
   * Whether a K8s API error is a transient connectivity/availability blip that
   * should be retried rather than failing the operation. Covers socket-level
   * errors, request timeouts (incl. the apiserver "request did not complete
   * within the allotted timeout" message), DNS hiccups and 5xx/429 responses.
   */
  private isTransientK8sError(err: any): boolean {
    const statusCode = err?.statusCode ?? err?.response?.statusCode ?? err?.body?.code;
    if (typeof statusCode === 'number' && (statusCode >= 500 || statusCode === 429)) {
      return true;
    }
    const haystack = [err?.code, err?.message, err?.body?.message, err?.cause?.code].filter(Boolean).join(' ');
    return /ECONNRESET|ECONNREFUSED|ETIMEDOUT|ESOCKETTIMEDOUT|EPIPE|EAI_AGAIN|ENOTFOUND|ENETUNREACH|socket hang up|timed? ?out|allotted timeout|did not complete|Client network socket disconnected/i.test(
      haystack,
    );
  }

  private async waitForJobCompletion(batchApi: k8s.BatchV1Api, coreApi: k8s.CoreV1Api, jobName: string, namespace: string, timeoutSeconds: number, deploymentId?: string): Promise<void> {
    const startTime = Date.now();
    const timeoutMs = timeoutSeconds * 1000;
    let lastLoggedStatus = '';

    while (Date.now() - startTime < timeoutMs) {
      await this.throwIfCancelledShared(deploymentId);
      const elapsed = Date.now() - startTime;
      const buildPercent = Math.min(90, 15 + Math.round((elapsed / timeoutMs) * 75));
      this.setProgress(deploymentId, {
        phase: 'building',
        percent: buildPercent,
        message: 'Building Docker image...',
      });
      // ── Check Job status (with retry for transient connection errors) ──
      let job: k8s.V1Job;
      try {
        job = await batchApi.readNamespacedJob({ name: jobName, namespace });
      } catch (pollErr: any) {
        // The Kaniko job keeps running independently of these status polls.
        // A single API blip (timeout, reset, 5xx, DNS) must NOT abort a build
        // that is still progressing — just retry on the next poll tick.
        if (this.isTransientK8sError(pollErr)) {
          const detail = pollErr?.code || pollErr?.message || pollErr?.statusCode || 'unknown';
          this.logger.warn(`Transient K8s API error polling job ${jobName}: ${detail} — retrying in 5s`);
          await new Promise((r) => setTimeout(r, 5000));
          continue;
        }
        throw pollErr;
      }
      const status = job.status;

      if (status?.succeeded && status.succeeded > 0) {
        this.logger.log(`Build job ${jobName} succeeded`);
        return;
      }

      // Check if the Job has permanently failed (all retries exhausted)
      const failedCondition = (status?.conditions || []).find((c) => c.type === 'Failed' && c.status === 'True');
      if (failedCondition) {
        const logs = await this.getBuildLogs(coreApi, jobName, namespace);
        throw new Error(`Build job ${jobName} failed.\nLogs:\n${logs}`);
      }

      // Safety net: if failures exceed backoffLimit and no pod is still running
      const backoffLimit = job.spec?.backoffLimit ?? 0;
      const failedCount = status?.failed ?? 0;
      if (failedCount > backoffLimit) {
        // Double-check: are there still active pods?
        const activePods = (status as any)?.active ?? 0;
        if (activePods === 0) {
          const logs = await this.getBuildLogs(coreApi, jobName, namespace);
          throw new Error(`Build job ${jobName} failed: ${failedCount} failures exceeded backoffLimit=${backoffLimit}.\nLogs:\n${logs}`);
        }
      }

      // Log intermediate pod failures (retries still available)
      if (failedCount > 0) {
        this.logger.warn(`Build job ${jobName}: ${failedCount} pod failure(s), backoffLimit=${backoffLimit} — retrying...`);
      }

      // ── Check Pod status for early failure detection ──
      try {
        const pods = await coreApi.listNamespacedPod({
          namespace,
          labelSelector: `job-name=${jobName}`,
        });

        for (const pod of pods.items) {
          const podName = pod.metadata?.name || 'unknown';
          const phase = pod.status?.phase;

          // Check all container statuses (init + regular) for stuck states
          const allStatuses = [...(pod.status?.initContainerStatuses || []), ...(pod.status?.containerStatuses || [])];

          for (const cs of allStatuses) {
            const waiting = cs.state?.waiting;
            if (waiting?.reason) {
              const reason = waiting.reason;
              const msg = waiting.message || '';

              // These are unrecoverable — fail fast instead of waiting 10 minutes
              const fatalReasons = ['ErrImagePull', 'ImagePullBackOff', 'CreateContainerConfigError', 'InvalidImageName', 'CrashLoopBackOff'];

              if (fatalReasons.includes(reason)) {
                const logs = await this.getBuildLogs(coreApi, jobName, namespace);
                throw new Error(`Build pod ${podName} stuck: ${reason} — ${msg}\nLogs:\n${logs}`);
              }

              // Log non-fatal waiting states periodically
              const statusKey = `${podName}:${cs.name}:${reason}`;
              if (statusKey !== lastLoggedStatus) {
                this.logger.warn(`Pod ${podName} container "${cs.name}": ${reason} — ${msg}`);
                lastLoggedStatus = statusKey;
              }
            }
          }

          // Log phase changes
          const phaseKey = `${podName}:${phase}`;
          if (phaseKey !== lastLoggedStatus && phase !== 'Succeeded') {
            this.logger.log(`Build pod ${podName}: phase=${phase}`);
            lastLoggedStatus = phaseKey;
          }
        }
      } catch (podErr: any) {
        // Don't fail the whole build just because pod status check failed
        if (podErr.message?.includes('stuck:') || podErr.message?.includes('Build pod')) {
          throw podErr; // Re-throw our own fatal errors
        }
        this.logger.warn(`Could not check pod status: ${podErr.message}`);
      }

      // Wait 5 seconds before polling again
      await new Promise((resolve) => setTimeout(resolve, 5000));
    }

    // Timeout — get logs for debugging
    let logs = '';
    try {
      logs = await this.getBuildLogs(coreApi, jobName, namespace);
    } catch {}
    throw new Error(`Build job ${jobName} timed out after ${timeoutSeconds}s\nLogs:\n${logs}`);
  }

  /**
   * Live build logs for an in-progress build, read straight from the running
   * build pod (init + kaniko containers). Returns null when there is no active
   * build session for this deployment (e.g. build already finished/cleaned up),
   * so callers can fall back to the persisted build log.
   */
  async getLiveBuildLog(deploymentId: string): Promise<string | null> {
    const session = this.activeBuilds.get(deploymentId);
    if (session?.coreApi && session.namespace && session.buildPodName) {
      try {
        return await this.getBuildLogs(session.coreApi, session.buildPodName, session.namespace);
      } catch {
        return null;
      }
    }
    // Build is running on another replica — read logs via the persisted session.
    const persisted = await this.readPersistedSession(deploymentId);
    if (persisted?.namespace && persisted.buildPodName) {
      const apis = await this.makeClusterApis(persisted.clusterId);
      if (apis) {
        try {
          return await this.getBuildLogs(apis.coreApi, persisted.buildPodName, persisted.namespace);
        } catch {
          return null;
        }
      }
    }
    return null;
  }

  private async getBuildLogs(coreApi: k8s.CoreV1Api, jobName: string, namespace: string): Promise<string> {
    try {
      const pods = await coreApi.listNamespacedPod({
        namespace,
        labelSelector: `job-name=${jobName}`,
      });

      if (pods.items.length === 0) {
        return 'No pods found for build job.';
      }

      const podName = pods.items[0].metadata?.name;
      if (!podName) return 'Pod name not found.';

      // Get logs from all containers (init + kaniko)
      let allLogs = '';
      const containers = [...(pods.items[0].spec?.initContainers || []), ...(pods.items[0].spec?.containers || [])];

      for (const container of containers) {
        try {
          const logResponse = await coreApi.readNamespacedPodLog({
            name: podName,
            namespace,
            container: container.name,
            tailLines: 500,
          });
          allLogs += `\n--- ${container.name} ---\n${logResponse}`;
        } catch {
          allLogs += `\n--- ${container.name} --- (no logs available)`;
        }
      }
      return allLogs;
    } catch (e: any) {
      return `Failed to retrieve logs: ${e.message}`;
    }
  }
}