Add application migration workflow.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/src/app.module.ts

@@ -13,6 +13,7 @@ import { TicketsModule } from './tickets/tickets.module';
 import { BillingModule } from './billing/billing.module';
 import { SnapshotsModule } from './snapshots/snapshots.module';
 import { LifecycleModule } from './lifecycle/lifecycle.module';
+import { ApplicationMigrationsModule } from './application-migrations/application-migrations.module';
 import configuration from './config/configuration';
 
 @Module({
@@ -64,6 +65,7 @@ import configuration from './config/configuration';
     BillingModule,
     SnapshotsModule,
     LifecycleModule,
+    ApplicationMigrationsModule,
   ],
 })
 export class AppModule {}

backend/src/application-migrations/application-migrations.controller.ts

@@ -0,0 +1,51 @@
+import { Body, Controller, Get, Param, Post, Query, Request, UseGuards } from '@nestjs/common';
+import { AuthGuard } from '@nestjs/passport';
+import { ApiBearerAuth, ApiOperation, ApiTags } from '@nestjs/swagger';
+import { ApplicationMigrationsService } from './application-migrations.service';
+import { CreateApplicationMigrationDto } from './dto/application-migration.dto';
+import { RolesGuard } from '../common/guards/roles.guard';
+import { Roles } from '../common/decorators/roles.decorator';
+import { UserRole } from '../common/enums';
+
+@ApiTags('Application Migrations')
+@ApiBearerAuth()
+@Controller('application-migrations')
+@UseGuards(AuthGuard('jwt'), RolesGuard)
+@Roles(UserRole.ADMIN)
+export class ApplicationMigrationsController {
+  constructor(private readonly migrationsService: ApplicationMigrationsService) {}
+
+  @Post('applications/:applicationId')
+  @ApiOperation({ summary: 'Queue application migration to another cluster (Super Admin)' })
+  async create(
+    @Param('applicationId') applicationId: string,
+    @Request() req: any,
+    @Body() dto: CreateApplicationMigrationDto,
+  ) {
+    return this.migrationsService.create(applicationId, req.user.id, dto);
+  }
+
+  @Get()
+  @ApiOperation({ summary: 'List application migration jobs (Super Admin)' })
+  async list(@Query('applicationId') applicationId?: string) {
+    return this.migrationsService.list(applicationId);
+  }
+
+  @Get(':id')
+  @ApiOperation({ summary: 'Get application migration details (Super Admin)' })
+  async findOne(@Param('id') id: string) {
+    return this.migrationsService.findOne(id);
+  }
+
+  @Get(':id/events')
+  @ApiOperation({ summary: 'Get application migration event log (Super Admin)' })
+  async events(@Param('id') id: string) {
+    return this.migrationsService.events(id);
+  }
+
+  @Post(':id/retry')
+  @ApiOperation({ summary: 'Retry failed application migration (Super Admin)' })
+  async retry(@Param('id') id: string, @Request() req: any) {
+    return this.migrationsService.retry(id, req.user.id);
+  }
+}

backend/src/application-migrations/application-migrations.module.ts

@@ -0,0 +1,25 @@
+import { Module, forwardRef } from '@nestjs/common';
+import { BullModule } from '@nestjs/bull';
+import { TypeOrmModule } from '@nestjs/typeorm';
+import { ApplicationMigrationsController } from './application-migrations.controller';
+import { ApplicationMigrationsProcessor } from './application-migrations.processor';
+import { ApplicationMigrationsService } from './application-migrations.service';
+import { ApplicationMigrationJob } from './entities/application-migration-job.entity';
+import { ApplicationMigrationEvent } from './entities/application-migration-event.entity';
+import { ApplicationsModule } from '../applications/applications.module';
+import { ClustersModule } from '../clusters/clusters.module';
+import { KubernetesModule } from '../kubernetes/kubernetes.module';
+
+@Module({
+  imports: [
+    TypeOrmModule.forFeature([ApplicationMigrationJob, ApplicationMigrationEvent]),
+    BullModule.registerQueue({ name: 'application-migrations' }),
+    forwardRef(() => ApplicationsModule),
+    ClustersModule,
+    KubernetesModule,
+  ],
+  controllers: [ApplicationMigrationsController],
+  providers: [ApplicationMigrationsService, ApplicationMigrationsProcessor],
+  exports: [ApplicationMigrationsService],
+})
+export class ApplicationMigrationsModule {}

backend/src/application-migrations/application-migrations.processor.ts

@@ -0,0 +1,13 @@
+import { Process, Processor } from '@nestjs/bull';
+import { Job } from 'bull';
+import { ApplicationMigrationsService } from './application-migrations.service';
+
+@Processor('application-migrations')
+export class ApplicationMigrationsProcessor {
+  constructor(private readonly migrationsService: ApplicationMigrationsService) {}
+
+  @Process('run')
+  async run(job: Job<{ migrationId: string }>): Promise<void> {
+    await this.migrationsService.process(job.data.migrationId);
+  }
+}

backend/src/application-migrations/application-migrations.service.ts

@@ -0,0 +1,263 @@
+import { BadRequestException, Injectable, NotFoundException } from '@nestjs/common';
+import { InjectQueue } from '@nestjs/bull';
+import { InjectRepository } from '@nestjs/typeorm';
+import { Queue } from 'bull';
+import { Repository } from 'typeorm';
+import { ApplicationMigrationJob } from './entities/application-migration-job.entity';
+import { ApplicationMigrationEvent } from './entities/application-migration-event.entity';
+import { CreateApplicationMigrationDto } from './dto/application-migration.dto';
+import { ApplicationsService } from '../applications/applications.service';
+import { ClustersService } from '../clusters/clusters.service';
+import { KubernetesService } from '../kubernetes/kubernetes.service';
+import { ClusterStatus } from '../common/enums';
+
+@Injectable()
+export class ApplicationMigrationsService {
+  constructor(
+    @InjectRepository(ApplicationMigrationJob)
+    private readonly jobsRepository: Repository<ApplicationMigrationJob>,
+    @InjectRepository(ApplicationMigrationEvent)
+    private readonly eventsRepository: Repository<ApplicationMigrationEvent>,
+    @InjectQueue('application-migrations')
+    private readonly queue: Queue,
+    private readonly applicationsService: ApplicationsService,
+    private readonly clustersService: ClustersService,
+    private readonly kubernetesService: KubernetesService,
+  ) {}
+
+  async create(
+    applicationId: string,
+    requestedBy: string,
+    dto: CreateApplicationMigrationDto,
+  ): Promise<ApplicationMigrationJob> {
+    const app = await this.applicationsService.findOne(applicationId);
+    if (!app.clusterId) {
+      throw new BadRequestException('Application does not have a source cluster assignment');
+    }
+    if (app.clusterId === dto.targetClusterId) {
+      throw new BadRequestException('Target cluster must be different from the source cluster');
+    }
+
+    const target = await this.clustersService.findOne(dto.targetClusterId);
+    if (target.status !== ClusterStatus.ACTIVE || target.healthStatus !== 'healthy') {
+      throw new BadRequestException('Target cluster must be active and healthy before migration');
+    }
+
+    const activeJob = await this.jobsRepository.findOne({
+      where: [
+        { applicationId, status: 'queued' },
+        { applicationId, status: 'running' },
+        { applicationId, status: 'rolling_back' },
+      ],
+      order: { createdAt: 'DESC' },
+    });
+    if (activeJob) {
+      throw new BadRequestException('A migration is already queued or running for this application');
+    }
+
+    const job = await this.jobsRepository.save(this.jobsRepository.create({
+      applicationId,
+      requestedBy,
+      sourceClusterId: app.clusterId,
+      targetClusterId: dto.targetClusterId,
+      status: 'queued',
+      maxAttempts: 3,
+      metadata: {
+        note: dto.note,
+        migrateStorage: dto.migrateStorage !== false,
+      },
+    }));
+    await this.log(job.id, 'queued', 'Migration job queued', { targetClusterId: dto.targetClusterId });
+    await this.queue.add('run', { migrationId: job.id }, {
+      attempts: 1,
+      removeOnComplete: true,
+      removeOnFail: false,
+    });
+    return this.findOne(job.id);
+  }
+
+  async list(applicationId?: string): Promise<ApplicationMigrationJob[]> {
+    return this.jobsRepository.find({
+      where: applicationId ? { applicationId } : {},
+      relations: ['application', 'sourceCluster', 'targetCluster'],
+      order: { createdAt: 'DESC' },
+      take: 100,
+    });
+  }
+
+  async findOne(id: string): Promise<ApplicationMigrationJob> {
+    const job = await this.jobsRepository.findOne({
+      where: { id },
+      relations: ['application', 'sourceCluster', 'targetCluster'],
+    });
+    if (!job) {
+      throw new NotFoundException('Migration job not found');
+    }
+    return job;
+  }
+
+  async events(id: string): Promise<ApplicationMigrationEvent[]> {
+    await this.findOne(id);
+    return this.eventsRepository.find({
+      where: { migrationId: id },
+      order: { createdAt: 'ASC' },
+    });
+  }
+
+  async retry(id: string, requestedBy: string): Promise<ApplicationMigrationJob> {
+    const job = await this.findOne(id);
+    if (!['failed', 'rolled_back'].includes(job.status)) {
+      throw new BadRequestException('Only failed or rolled back migrations can be retried');
+    }
+    if (job.attempts >= job.maxAttempts) {
+      throw new BadRequestException('Migration retry limit reached');
+    }
+
+    await this.jobsRepository.update(id, {
+      status: 'queued',
+      errorMessage: null,
+      currentStep: 'queued',
+      completedAt: null,
+      metadata: {
+        ...(job.metadata || {}),
+        retriedBy: requestedBy,
+        retriedAt: new Date().toISOString(),
+      } as any,
+    });
+    await this.log(id, 'queued', 'Migration retry queued', { requestedBy });
+    await this.queue.add('run', { migrationId: id }, {
+      attempts: 1,
+      removeOnComplete: true,
+      removeOnFail: false,
+    });
+    return this.findOne(id);
+  }
+
+  async process(migrationId: string): Promise<void> {
+    const job = await this.findOne(migrationId);
+    const app = await this.applicationsService.findOne(job.applicationId);
+    const sourceClusterId = job.sourceClusterId;
+    const targetClusterId = job.targetClusterId;
+    const sourceApp = { ...app, clusterId: sourceClusterId };
+    const targetApp = { ...app, clusterId: targetClusterId };
+
+    await this.updateJob(job.id, 'running', 'validate-target', { attempts: job.attempts + 1, startedAt: new Date() });
+
+    try {
+      await this.validateTargetCluster(targetClusterId);
+      await this.log(job.id, 'validate-target', 'Target cluster is active and healthy', { targetClusterId });
+
+      await this.updateJob(job.id, 'running', 'transfer-secrets-configs');
+      await this.kubernetesService.prepareApplicationMigration(sourceApp as any, targetClusterId, {
+        migrateStorage: job.metadata?.migrateStorage !== false,
+        log: (step, message, metadata) => this.log(job.id, step, message, metadata),
+      });
+
+      if (!app.latestImageTag) {
+        throw new BadRequestException('Application must have a deployed image before migration');
+      }
+
+      await this.updateJob(job.id, 'running', 'deploy-target');
+      await this.log(job.id, 'deploy-target', 'Deploying application on target cluster before cutover');
+      const targetResources = await this.kubernetesService.deployApplication(targetApp as any, app.latestImageTag);
+
+      await this.updateJob(job.id, 'running', 'health-check');
+      await this.kubernetesService.waitForApplicationReady(targetApp as any, 600_000);
+      await this.log(job.id, 'health-check', 'Target deployment is healthy');
+
+      await this.updateJob(job.id, 'running', 'update-ingress');
+      await this.kubernetesService.updateIngress(targetApp as any);
+      await this.log(job.id, 'update-ingress', 'Ingress/load balancer updated on target cluster');
+
+      await this.updateJob(job.id, 'running', 'cutover');
+      await this.applicationsService.updateClusterAssignment(app.id, targetClusterId, app.poolId);
+      await this.log(job.id, 'cutover', 'Application traffic cut over to target cluster', {
+        sourceClusterId,
+        targetClusterId,
+      });
+
+      await this.updateJob(job.id, 'running', 'cleanup-source');
+      await this.kubernetesService.deleteApplication(sourceApp as any);
+      await this.log(job.id, 'cleanup-source', 'Source cluster deployment removed after successful migration');
+
+      await this.jobsRepository.update(job.id, {
+        status: 'completed',
+        currentStep: 'completed',
+        completedAt: new Date(),
+        metadata: {
+          ...(job.metadata || {}),
+          targetResources,
+        },
+      });
+      await this.log(job.id, 'completed', 'Migration completed successfully');
+    } catch (error: any) {
+      await this.handleFailure(job.id, sourceApp, targetApp, error);
+      throw error;
+    }
+  }
+
+  private async validateTargetCluster(targetClusterId: string): Promise<void> {
+    const target = await this.clustersService.findOne(targetClusterId);
+    if (target.status !== ClusterStatus.ACTIVE || target.healthStatus !== 'healthy') {
+      throw new BadRequestException('Target cluster is not active and healthy');
+    }
+  }
+
+  private async handleFailure(jobId: string, sourceApp: any, targetApp: any, error: any): Promise<void> {
+    const message = error?.message || 'Migration failed';
+    await this.jobsRepository.update(jobId, {
+      status: 'rolling_back',
+      currentStep: 'rollback',
+      errorMessage: message,
+    });
+    await this.log(jobId, 'rollback', 'Migration failed; rolling back target resources', { error: message }, 'error');
+
+    try {
+      await this.kubernetesService.deleteApplication(targetApp);
+      await this.applicationsService.updateClusterAssignment(sourceApp.id, sourceApp.clusterId, sourceApp.poolId);
+      await this.jobsRepository.update(jobId, {
+        status: 'rolled_back',
+        currentStep: 'rolled_back',
+        completedAt: new Date(),
+      });
+      await this.log(jobId, 'rolled_back', 'Rollback completed; source cluster assignment preserved');
+    } catch (rollbackError: any) {
+      await this.jobsRepository.update(jobId, {
+        status: 'failed',
+        currentStep: 'failed',
+        completedAt: new Date(),
+        errorMessage: `${message}; rollback failed: ${rollbackError.message}`,
+      });
+      await this.log(jobId, 'failed', `Rollback failed: ${rollbackError.message}`, undefined, 'error');
+    }
+  }
+
+  private async updateJob(
+    id: string,
+    status: ApplicationMigrationJob['status'],
+    currentStep: string,
+    extra: Partial<ApplicationMigrationJob> = {},
+  ): Promise<void> {
+    await this.jobsRepository.update(id, {
+      status,
+      currentStep,
+      ...extra,
+    });
+  }
+
+  private async log(
+    migrationId: string,
+    step: string,
+    message: string,
+    metadata?: Record<string, any>,
+    level: 'info' | 'warn' | 'error' = 'info',
+  ): Promise<void> {
+    await this.eventsRepository.save(this.eventsRepository.create({
+      migrationId,
+      step,
+      message,
+      metadata,
+      level,
+    }));
+  }
+}

backend/src/application-migrations/dto/application-migration.dto.ts

@@ -0,0 +1,18 @@
+import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
+import { IsBoolean, IsOptional, IsString, IsUUID } from 'class-validator';
+
+export class CreateApplicationMigrationDto {
+  @ApiProperty({ description: 'Destination Kubernetes cluster ID' })
+  @IsUUID()
+  targetClusterId: string;
+
+  @ApiPropertyOptional({ description: 'Optional operator note for audit logs' })
+  @IsOptional()
+  @IsString()
+  note?: string;
+
+  @ApiPropertyOptional({ description: 'Copy PVC definitions and attempt provider-neutral storage preparation' })
+  @IsOptional()
+  @IsBoolean()
+  migrateStorage?: boolean;
+}

backend/src/application-migrations/entities/application-migration-event.entity.ts

@@ -0,0 +1,37 @@
+import {
+  Column,
+  CreateDateColumn,
+  Entity,
+  JoinColumn,
+  ManyToOne,
+  PrimaryGeneratedColumn,
+} from 'typeorm';
+import { ApplicationMigrationJob } from './application-migration-job.entity';
+
+@Entity('application_migration_events')
+export class ApplicationMigrationEvent {
+  @PrimaryGeneratedColumn('uuid')
+  id: string;
+
+  @Column()
+  migrationId: string;
+
+  @ManyToOne(() => ApplicationMigrationJob, { onDelete: 'CASCADE' })
+  @JoinColumn({ name: 'migrationId' })
+  migration: ApplicationMigrationJob;
+
+  @Column()
+  step: string;
+
+  @Column({ default: 'info' })
+  level: 'info' | 'warn' | 'error';
+
+  @Column()
+  message: string;
+
+  @Column({ type: 'jsonb', nullable: true })
+  metadata: Record<string, any>;
+
+  @CreateDateColumn()
+  createdAt: Date;
+}

backend/src/application-migrations/entities/application-migration-job.entity.ts

@@ -0,0 +1,79 @@
+import {
+  Column,
+  CreateDateColumn,
+  Entity,
+  JoinColumn,
+  ManyToOne,
+  PrimaryGeneratedColumn,
+  UpdateDateColumn,
+} from 'typeorm';
+import { Application } from '../../applications/entities/application.entity';
+import { Cluster } from '../../clusters/entities/cluster.entity';
+
+export type ApplicationMigrationStatus =
+  | 'queued'
+  | 'running'
+  | 'completed'
+  | 'failed'
+  | 'rolling_back'
+  | 'rolled_back';
+
+@Entity('application_migration_jobs')
+export class ApplicationMigrationJob {
+  @PrimaryGeneratedColumn('uuid')
+  id: string;
+
+  @Column()
+  applicationId: string;
+
+  @ManyToOne(() => Application, { onDelete: 'CASCADE' })
+  @JoinColumn({ name: 'applicationId' })
+  application: Application;
+
+  @Column()
+  requestedBy: string;
+
+  @Column({ type: 'uuid' })
+  sourceClusterId: string;
+
+  @ManyToOne(() => Cluster, { onDelete: 'RESTRICT' })
+  @JoinColumn({ name: 'sourceClusterId' })
+  sourceCluster: Cluster;
+
+  @Column({ type: 'uuid' })
+  targetClusterId: string;
+
+  @ManyToOne(() => Cluster, { onDelete: 'RESTRICT' })
+  @JoinColumn({ name: 'targetClusterId' })
+  targetCluster: Cluster;
+
+  @Column({ default: 'queued' })
+  status: ApplicationMigrationStatus;
+
+  @Column({ default: 0 })
+  attempts: number;
+
+  @Column({ default: 3 })
+  maxAttempts: number;
+
+  @Column({ type: 'varchar', nullable: true })
+  currentStep: string | null;
+
+  @Column({ type: 'varchar', nullable: true })
+  errorMessage: string | null;
+
+  @Column({ type: 'jsonb', nullable: true })
+  metadata: Record<string, any> | null;
+
+  @Column({ type: 'timestamptz', nullable: true })
+  startedAt: Date | null;
+
+  @Column({ type: 'timestamptz', nullable: true })
+  completedAt: Date | null;
+
+  @CreateDateColumn()
+  createdAt: Date;
+
+  @UpdateDateColumn()
+  updatedAt: Date;
+}

backend/src/clusters/clusters.service.ts

@@ -22,6 +22,7 @@ import { CreateClusterPoolDto, UpdateClusterPoolDto } from './dto/cluster-pool.d
 import { ClusterStatus } from '../common/enums';
 import { ElasticsearchService } from '../kubernetes/elasticsearch.service';
 import { CreateApplicationDto } from '../applications/dto/application.dto';
+import { registerKubeconfigNoProxy } from '../common/kubernetes-proxy.util';
 
 @Injectable()
 export class ClustersService implements OnModuleInit, OnModuleDestroy {
@@ -77,8 +78,11 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
    */
   async testConnection(kubeconfig: string): Promise<{ connected: boolean; version?: string; error?: string }> {
     try {
+      const decryptedKubeconfig = this.decryptKubeconfig(kubeconfig);
+      registerKubeconfigNoProxy(decryptedKubeconfig);
+
       const kc = new k8s.KubeConfig();
-      kc.loadFromString(this.decryptKubeconfig(kubeconfig));
+      kc.loadFromString(decryptedKubeconfig);
 
       const versionApi = kc.makeApiClient(k8s.VersionApi);
       const result = await versionApi.getCode();
@@ -702,6 +706,7 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
     }
 
     const kc = new k8s.KubeConfig();
+    registerKubeconfigNoProxy(cluster.kubeconfig);
     kc.loadFromString(cluster.kubeconfig);
     const coreApi = kc.makeApiClient(k8s.CoreV1Api);
 
@@ -825,6 +830,7 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
    */
   async bootstrapCluster(kubeconfig: string): Promise<void> {
     const kc = new k8s.KubeConfig();
+    registerKubeconfigNoProxy(kubeconfig);
     kc.loadFromString(kubeconfig);
     const coreApi = kc.makeApiClient(k8s.CoreV1Api);
     const appsApi = kc.makeApiClient(k8s.AppsV1Api);

backend/src/common/kubernetes-proxy.util.ts

@@ -0,0 +1,42 @@
+import * as k8s from '@kubernetes/client-node';
+
+function normalizeNoProxyValue(value?: string): string[] {
+  return (value || '')
+    .split(',')
+    .map((entry) => entry.trim().replace(/^['"]|['"]$/g, ''))
+    .filter(Boolean);
+}
+
+function appendNoProxyEntries(envKey: 'NO_PROXY' | 'no_proxy', entries: string[]): void {
+  const existing = normalizeNoProxyValue(process.env[envKey]);
+  if (existing.includes('*')) {
+    return;
+  }
+
+  const merged = new Set(existing);
+  for (const entry of entries) {
+    merged.add(entry);
+  }
+
+  process.env[envKey] = Array.from(merged).join(',');
+}
+
+export function registerKubeconfigNoProxy(kubeconfig: string): string[] {
+  const kc = new k8s.KubeConfig();
+  kc.loadFromString(kubeconfig);
+
+  const cluster = kc.getCurrentCluster() || kc.getClusters()[0];
+  if (!cluster?.server) {
+    return [];
+  }
+
+  const url = new URL(cluster.server);
+  const entries = new Set<string>();
+  entries.add(url.hostname);
+  entries.add(url.host);
+
+  appendNoProxyEntries('NO_PROXY', Array.from(entries));
+  appendNoProxyEntries('no_proxy', Array.from(entries));
+
+  return Array.from(entries);
+}

backend/src/kubernetes/kubernetes.service.ts

@@ -11,6 +11,7 @@ import { Application } from '../applications/entities/application.entity';
 import { ensureAppUrlEnv } from '../applications/app-url.util';
 import { AppRuntime, DatabaseType, CustomDomainStatus, ServiceAccessTarget } from '../common/enums';
 import { HelmService } from './helm.service';
+import { registerKubeconfigNoProxy } from '../common/kubernetes-proxy.util';
 
 const execFileAsync = promisify(execFile);
 
@@ -81,6 +82,7 @@ export class KubernetesService implements OnModuleInit {
       : await this.clustersService.getDefault();
 
     const kc = new k8s.KubeConfig();
+    registerKubeconfigNoProxy(cluster.kubeconfig);
     kc.loadFromString(cluster.kubeconfig);
 
     return {
@@ -218,7 +220,11 @@ export class KubernetesService implements OnModuleInit {
     }
   }
 
-  async waitForApplicationReady(app: Application, timeoutMs = 600_000): Promise<void> {
+  async waitForApplicationReady(
+    app: Application,
+    timeoutMs = 600_000,
+    shouldAbort?: () => Promise<boolean>,
+  ): Promise<void> {
     const { coreApi, appsApi } = await this.getK8sClient(app.clusterId);
     const namespace = `user-${app.userId.split('-')[0]}`;
     const workloads = [
@@ -235,6 +241,10 @@ export class KubernetesService implements OnModuleInit {
     );
 
     while (Date.now() - start < timeoutMs) {
+      if (await shouldAbort?.()) {
+        throw new Error('Deployment cancelled by user');
+      }
+
       const statuses = await Promise.all(
         workloads.map((workload) => this.getDeploymentReadiness(appsApi, namespace, workload.name, workload.replicas)),
       );
@@ -2046,6 +2056,16 @@ export class KubernetesService implements OnModuleInit {
     }
   }
 
+  private toDnsLabel(value: string): string {
+    return value
+      .toLowerCase()
+      .replace(/[^a-z0-9-]/g, '-')
+      .replace(/-+/g, '-')
+      .replace(/^-|-$/g, '')
+      .slice(0, 63)
+      .replace(/-$/g, '');
+  }
+
   resolveAccessTarget(
     app: Application,
     target: ServiceAccessTarget,
@@ -2087,7 +2107,7 @@ export class KubernetesService implements OnModuleInit {
     const namespace = this.getUserNamespace(app.userId);
     const { selector, targetPort, portName } = this.resolveAccessTarget(app, target);
     const shortId = grantId.split('-')[0];
-    const k8sServiceName = `${app.name}-${target}-access-${shortId}`.slice(0, 63);
+    const k8sServiceName = this.toDnsLabel(`${app.name}-${target}-access-${shortId}`);
 
     const portSpec: k8s.V1ServicePort = {
       port: targetPort,
@@ -2116,7 +2136,16 @@ export class KubernetesService implements OnModuleInit {
       },
     };
 
-    const created = await coreApi.createNamespacedService(namespace, service);
+    let created: { body: k8s.V1Service };
+    try {
+      created = await coreApi.createNamespacedService(namespace, service);
+    } catch (error: any) {
+      const message = error.body?.message || error.response?.body?.message || error.message || 'HTTP request failed';
+      this.logger.warn(
+        `K8s client failed to create temporary access service ${k8sServiceName}; trying kubectl fallback: ${message}`,
+      );
+      created = { body: await this.applyServiceWithKubectl(app.clusterId, namespace, service) };
+    }
     const nodePort = created.body.spec?.ports?.[0]?.nodePort;
     if (!nodePort) {
       try {
@@ -2133,6 +2162,50 @@ export class KubernetesService implements OnModuleInit {
     return { host, nodePort, k8sServiceName, targetPort };
   }
 
+  private async applyServiceWithKubectl(
+    clusterId: string | undefined,
+    namespace: string,
+    service: k8s.V1Service,
+  ): Promise<k8s.V1Service> {
+    const tmpDir = fs.mkdtempSync(path.join('/tmp', 'cloudhost-access-'));
+    const kubeconfigPath = path.join(tmpDir, 'kubeconfig.yaml');
+    const manifestPath = path.join(tmpDir, 'service.json');
+
+    try {
+      fs.writeFileSync(kubeconfigPath, await this.getKubeconfig(clusterId), { mode: 0o600 });
+      fs.writeFileSync(manifestPath, JSON.stringify(service), { mode: 0o600 });
+
+      await execFileAsync('kubectl', [
+        '--kubeconfig',
+        kubeconfigPath,
+        'apply',
+        '-n',
+        namespace,
+        '-f',
+        manifestPath,
+      ]);
+
+      const { stdout } = await execFileAsync('kubectl', [
+        '--kubeconfig',
+        kubeconfigPath,
+        'get',
+        'service',
+        service.metadata!.name!,
+        '-n',
+        namespace,
+        '-o',
+        'json',
+      ]);
+
+      return JSON.parse(stdout) as k8s.V1Service;
+    } catch (error: any) {
+      const message = error.stderr || error.message || 'kubectl failed';
+      throw new Error(`Failed to create temporary access service: ${message}`);
+    } finally {
+      try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
+    }
+  }
+
   async revokeTemporaryAccess(
     clusterId: string,
     namespace: string,
@@ -2341,6 +2414,239 @@ export class KubernetesService implements OnModuleInit {
     this.logger.log(`All K8s resources cleaned up for ${app.name} in ${namespace}`);
   }
 
+  async prepareApplicationMigration(
+    app: Application,
+    targetClusterId: string,
+    options: {
+      migrateStorage?: boolean;
+      log?: (step: string, message: string, metadata?: Record<string, any>) => Promise<void>;
+    } = {},
+  ): Promise<void> {
+    const namespace = this.getUserNamespace(app.userId);
+    const source = await this.getK8sClient(app.clusterId);
+    const target = await this.getK8sClient(targetClusterId);
+
+    await this.ensureNamespaceOnCluster(target.coreApi, namespace);
+    await options.log?.('transfer-secrets-configs', 'Target namespace ensured', { namespace });
+
+    const secretNames = [
+      `${app.name}-env`,
+      `${app.name}-db-secret`,
+      `${app.name}-redis-secret`,
+      `${app.name}-rabbitmq-secret`,
+      `${app.name}-tls`,
+    ];
+    for (const secretName of secretNames) {
+      try {
+        const secret = await source.coreApi.readNamespacedSecret(secretName, namespace);
+        await this.upsertSecret(target.coreApi, namespace, this.cleanK8sObject(secret.body));
+        await options.log?.('transfer-secrets-configs', `Secret ${secretName} copied`);
+      } catch (error: any) {
+        if (!this.isK8sNotFound(error)) {
+          throw error;
+        }
+      }
+    }
+
+    const configMapNames = [
+      `${app.name}-fluent-bit-config`,
+    ];
+    for (const configMapName of configMapNames) {
+      try {
+        const configMap = await source.coreApi.readNamespacedConfigMap(configMapName, namespace);
+        await this.upsertConfigMap(target.coreApi, namespace, this.cleanK8sObject(configMap.body));
+        await options.log?.('transfer-secrets-configs', `ConfigMap ${configMapName} copied`);
+      } catch (error: any) {
+        if (!this.isK8sNotFound(error)) {
+          throw error;
+        }
+      }
+    }
+
+    if (options.migrateStorage !== false) {
+      const pvcNames = [
+        `${app.name}-storage`,
+        `${app.name}-wp-content`,
+        `${app.name}-db`,
+        `${app.name}-redis-data`,
+        `${app.name}-rabbitmq-data`,
+      ];
+      for (const pvcName of pvcNames) {
+        try {
+          const pvc = await source.coreApi.readNamespacedPersistentVolumeClaim(pvcName, namespace);
+          await this.upsertPvcDefinition(target.coreApi, namespace, this.cleanPvcForMigration(pvc.body));
+          await options.log?.('transfer-volumes', `PVC definition ${pvcName} prepared on target cluster`, {
+            note: 'Starting best-effort PVC data copy through migration helper pods.',
+          });
+          await this.copyPvcDataBetweenClusters(app.clusterId!, targetClusterId, namespace, pvcName, options.log);
+        } catch (error: any) {
+          if (!this.isK8sNotFound(error)) {
+            throw error;
+          }
+        }
+      }
+    }
+  }
+
+  private async copyPvcDataBetweenClusters(
+    sourceClusterId: string,
+    targetClusterId: string,
+    namespace: string,
+    pvcName: string,
+    log?: (step: string, message: string, metadata?: Record<string, any>) => Promise<void>,
+  ): Promise<void> {
+    const safeName = pvcName.replace(/[^a-z0-9-]/gi, '-').toLowerCase().slice(0, 32);
+    const suffix = `${Date.now()}`.slice(-6);
+    const sourcePod = `migrate-src-${safeName}-${suffix}`;
+    const targetPod = `migrate-dst-${safeName}-${suffix}`;
+    const tempDir = path.join('/tmp', `app-migration-${safeName}-${suffix}`);
+    const sourceKubeconfig = path.join(tempDir, 'source.kubeconfig');
+    const targetKubeconfig = path.join(tempDir, 'target.kubeconfig');
+
+    fs.mkdirSync(tempDir, { recursive: true });
+    fs.writeFileSync(sourceKubeconfig, await this.getKubeconfig(sourceClusterId), { mode: 0o600 });
+    fs.writeFileSync(targetKubeconfig, await this.getKubeconfig(targetClusterId), { mode: 0o600 });
+
+    try {
+      await this.createPvcCopyPod(sourceKubeconfig, namespace, sourcePod, pvcName);
+      await this.createPvcCopyPod(targetKubeconfig, namespace, targetPod, pvcName);
+      await log?.('transfer-volumes', `Copy helper pods ready for PVC ${pvcName}`);
+
+      const localDataPath = path.join(tempDir, 'data');
+      await execFileAsync('kubectl', ['--kubeconfig', sourceKubeconfig, '-n', namespace, 'cp', `${sourcePod}:/data`, localDataPath], {
+        timeout: 30 * 60 * 1000,
+      });
+      await execFileAsync('kubectl', ['--kubeconfig', targetKubeconfig, '-n', namespace, 'cp', `${localDataPath}/.`, `${targetPod}:/data`], {
+        timeout: 30 * 60 * 1000,
+      });
+      await log?.('transfer-volumes', `PVC data copied for ${pvcName}`);
+    } finally {
+      await execFileAsync('kubectl', ['--kubeconfig', sourceKubeconfig, '-n', namespace, 'delete', 'pod', sourcePod, '--ignore-not-found=true']).catch(() => undefined);
+      await execFileAsync('kubectl', ['--kubeconfig', targetKubeconfig, '-n', namespace, 'delete', 'pod', targetPod, '--ignore-not-found=true']).catch(() => undefined);
+      fs.rmSync(tempDir, { recursive: true, force: true });
+    }
+  }
+
+  private async createPvcCopyPod(
+    kubeconfigPath: string,
+    namespace: string,
+    podName: string,
+    pvcName: string,
+  ): Promise<void> {
+    const manifestPath = path.join('/tmp', `${podName}.json`);
+    const manifest = {
+      apiVersion: 'v1',
+      kind: 'Pod',
+      metadata: { name: podName, namespace },
+      spec: {
+        restartPolicy: 'Never',
+        containers: [{
+          name: 'copy',
+          image: 'busybox:1.36',
+          command: ['sh', '-c', 'mkdir -p /data && sleep 3600'],
+          volumeMounts: [{ name: 'data', mountPath: '/data' }],
+        }],
+        volumes: [{ name: 'data', persistentVolumeClaim: { claimName: pvcName } }],
+      },
+    };
+
+    fs.writeFileSync(manifestPath, JSON.stringify(manifest));
+    try {
+      await execFileAsync('kubectl', ['--kubeconfig', kubeconfigPath, 'apply', '-f', manifestPath], { timeout: 120_000 });
+      await execFileAsync('kubectl', ['--kubeconfig', kubeconfigPath, '-n', namespace, 'wait', '--for=condition=Ready', `pod/${podName}`, '--timeout=180s'], { timeout: 210_000 });
+    } finally {
+      fs.rmSync(manifestPath, { force: true });
+    }
+  }
+
+  private async ensureNamespaceOnCluster(coreApi: k8s.CoreV1Api, namespace: string): Promise<void> {
+    try {
+      await coreApi.readNamespace(namespace);
+    } catch (error: any) {
+      if (this.isK8sNotFound(error)) {
+        await coreApi.createNamespace({ metadata: { name: namespace } });
+        return;
+      }
+      throw error;
+    }
+  }
+
+  private async upsertSecret(coreApi: k8s.CoreV1Api, namespace: string, secret: k8s.V1Secret): Promise<void> {
+    secret.metadata = { ...(secret.metadata || {}), namespace };
+    try {
+      await coreApi.replaceNamespacedSecret(secret.metadata.name!, namespace, secret);
+    } catch (error: any) {
+      if (this.isK8sNotFound(error)) {
+        await coreApi.createNamespacedSecret(namespace, secret);
+        return;
+      }
+      throw error;
+    }
+  }
+
+  private async upsertConfigMap(coreApi: k8s.CoreV1Api, namespace: string, configMap: k8s.V1ConfigMap): Promise<void> {
+    configMap.metadata = { ...(configMap.metadata || {}), namespace };
+    try {
+      await coreApi.replaceNamespacedConfigMap(configMap.metadata.name!, namespace, configMap);
+    } catch (error: any) {
+      if (this.isK8sNotFound(error)) {
+        await coreApi.createNamespacedConfigMap(namespace, configMap);
+        return;
+      }
+      throw error;
+    }
+  }
+
+  private async upsertPvcDefinition(
+    coreApi: k8s.CoreV1Api,
+    namespace: string,
+    pvc: k8s.V1PersistentVolumeClaim,
+  ): Promise<void> {
+    pvc.metadata = { ...(pvc.metadata || {}), namespace };
+    try {
+      await coreApi.readNamespacedPersistentVolumeClaim(pvc.metadata.name!, namespace);
+    } catch (error: any) {
+      if (this.isK8sNotFound(error)) {
+        await coreApi.createNamespacedPersistentVolumeClaim(namespace, pvc);
+        return;
+      }
+      throw error;
+    }
+  }
+
+  private cleanK8sObject<T extends { metadata?: k8s.V1ObjectMeta }>(obj: T): T {
+    const metadata = { ...(obj.metadata || {}) };
+    delete metadata.uid;
+    delete metadata.resourceVersion;
+    delete metadata.generation;
+    delete metadata.creationTimestamp;
+    delete metadata.managedFields;
+    delete metadata.selfLink;
+    return {
+      ...obj,
+      metadata,
+    };
+  }
+
+  private cleanPvcForMigration(pvc: k8s.V1PersistentVolumeClaim): k8s.V1PersistentVolumeClaim {
+    const cleaned = this.cleanK8sObject(pvc);
+    return {
+      apiVersion: cleaned.apiVersion,
+      kind: cleaned.kind,
+      metadata: cleaned.metadata,
+      spec: {
+        accessModes: cleaned.spec?.accessModes,
+        resources: cleaned.spec?.resources,
+        storageClassName: cleaned.spec?.storageClassName,
+        volumeMode: cleaned.spec?.volumeMode,
+      },
+    };
+  }
+
+  private isK8sNotFound(error: any): boolean {
+    return error?.statusCode === 404 || error?.body?.code === 404;
+  }
+
   /**
    * Wait for the database pod to become Ready.
    * Polls pod status with label selector `app=<appName>-db`.