diff --git a/backend/migrations/014_application_migrations.sql b/backend/migrations/014_application_migrations.sql new file mode 100644 index 0000000..9dce530 --- /dev/null +++ b/backend/migrations/014_application_migrations.sql @@ -0,0 +1,36 @@ +CREATE TABLE IF NOT EXISTS application_migration_jobs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + "applicationId" UUID NOT NULL REFERENCES applications(id) ON DELETE CASCADE, + "requestedBy" UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + "sourceClusterId" UUID NOT NULL REFERENCES clusters(id) ON DELETE RESTRICT, + "targetClusterId" UUID NOT NULL REFERENCES clusters(id) ON DELETE RESTRICT, + status VARCHAR NOT NULL DEFAULT 'queued', + attempts INTEGER NOT NULL DEFAULT 0, + "maxAttempts" INTEGER NOT NULL DEFAULT 3, + "currentStep" VARCHAR, + "errorMessage" VARCHAR, + metadata JSONB, + "startedAt" TIMESTAMPTZ, + "completedAt" TIMESTAMPTZ, + "createdAt" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + "updatedAt" TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_application_migration_jobs_app_created + ON application_migration_jobs("applicationId", "createdAt" DESC); + +CREATE INDEX IF NOT EXISTS idx_application_migration_jobs_status + ON application_migration_jobs(status); + +CREATE TABLE IF NOT EXISTS application_migration_events ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + "migrationId" UUID NOT NULL REFERENCES application_migration_jobs(id) ON DELETE CASCADE, + step VARCHAR NOT NULL, + level VARCHAR NOT NULL DEFAULT 'info', + message VARCHAR NOT NULL, + metadata JSONB, + "createdAt" TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_application_migration_events_migration_created + ON application_migration_events("migrationId", "createdAt" ASC); diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index 93749fd..c1a400b 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -13,6 +13,7 @@ import { TicketsModule } from './tickets/tickets.module'; import { BillingModule } from './billing/billing.module'; import { SnapshotsModule } from './snapshots/snapshots.module'; import { LifecycleModule } from './lifecycle/lifecycle.module'; +import { ApplicationMigrationsModule } from './application-migrations/application-migrations.module'; import configuration from './config/configuration'; @Module({ @@ -64,6 +65,7 @@ import configuration from './config/configuration'; BillingModule, SnapshotsModule, LifecycleModule, + ApplicationMigrationsModule, ], }) export class AppModule {} diff --git a/backend/src/application-migrations/application-migrations.controller.ts b/backend/src/application-migrations/application-migrations.controller.ts new file mode 100644 index 0000000..676d31b --- /dev/null +++ b/backend/src/application-migrations/application-migrations.controller.ts @@ -0,0 +1,51 @@ +import { Body, Controller, Get, Param, Post, Query, Request, UseGuards } from '@nestjs/common'; +import { AuthGuard } from '@nestjs/passport'; +import { ApiBearerAuth, ApiOperation, ApiTags } from '@nestjs/swagger'; +import { ApplicationMigrationsService } from './application-migrations.service'; +import { CreateApplicationMigrationDto } from './dto/application-migration.dto'; +import { RolesGuard } from '../common/guards/roles.guard'; +import { Roles } from '../common/decorators/roles.decorator'; +import { UserRole } from '../common/enums'; + +@ApiTags('Application Migrations') +@ApiBearerAuth() +@Controller('application-migrations') +@UseGuards(AuthGuard('jwt'), RolesGuard) +@Roles(UserRole.ADMIN) +export class ApplicationMigrationsController { + constructor(private readonly migrationsService: ApplicationMigrationsService) {} + + @Post('applications/:applicationId') + @ApiOperation({ summary: 'Queue application migration to another cluster (Super Admin)' }) + async create( + @Param('applicationId') applicationId: string, + @Request() req: any, + @Body() dto: CreateApplicationMigrationDto, + ) { + return this.migrationsService.create(applicationId, req.user.id, dto); + } + + @Get() + @ApiOperation({ summary: 'List application migration jobs (Super Admin)' }) + async list(@Query('applicationId') applicationId?: string) { + return this.migrationsService.list(applicationId); + } + + @Get(':id') + @ApiOperation({ summary: 'Get application migration details (Super Admin)' }) + async findOne(@Param('id') id: string) { + return this.migrationsService.findOne(id); + } + + @Get(':id/events') + @ApiOperation({ summary: 'Get application migration event log (Super Admin)' }) + async events(@Param('id') id: string) { + return this.migrationsService.events(id); + } + + @Post(':id/retry') + @ApiOperation({ summary: 'Retry failed application migration (Super Admin)' }) + async retry(@Param('id') id: string, @Request() req: any) { + return this.migrationsService.retry(id, req.user.id); + } +} diff --git a/backend/src/application-migrations/application-migrations.module.ts b/backend/src/application-migrations/application-migrations.module.ts new file mode 100644 index 0000000..7c97513 --- /dev/null +++ b/backend/src/application-migrations/application-migrations.module.ts @@ -0,0 +1,25 @@ +import { Module, forwardRef } from '@nestjs/common'; +import { BullModule } from '@nestjs/bull'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { ApplicationMigrationsController } from './application-migrations.controller'; +import { ApplicationMigrationsProcessor } from './application-migrations.processor'; +import { ApplicationMigrationsService } from './application-migrations.service'; +import { ApplicationMigrationJob } from './entities/application-migration-job.entity'; +import { ApplicationMigrationEvent } from './entities/application-migration-event.entity'; +import { ApplicationsModule } from '../applications/applications.module'; +import { ClustersModule } from '../clusters/clusters.module'; +import { KubernetesModule } from '../kubernetes/kubernetes.module'; + +@Module({ + imports: [ + TypeOrmModule.forFeature([ApplicationMigrationJob, ApplicationMigrationEvent]), + BullModule.registerQueue({ name: 'application-migrations' }), + forwardRef(() => ApplicationsModule), + ClustersModule, + KubernetesModule, + ], + controllers: [ApplicationMigrationsController], + providers: [ApplicationMigrationsService, ApplicationMigrationsProcessor], + exports: [ApplicationMigrationsService], +}) +export class ApplicationMigrationsModule {} diff --git a/backend/src/application-migrations/application-migrations.processor.ts b/backend/src/application-migrations/application-migrations.processor.ts new file mode 100644 index 0000000..89c4884 --- /dev/null +++ b/backend/src/application-migrations/application-migrations.processor.ts @@ -0,0 +1,13 @@ +import { Process, Processor } from '@nestjs/bull'; +import { Job } from 'bull'; +import { ApplicationMigrationsService } from './application-migrations.service'; + +@Processor('application-migrations') +export class ApplicationMigrationsProcessor { + constructor(private readonly migrationsService: ApplicationMigrationsService) {} + + @Process('run') + async run(job: Job<{ migrationId: string }>): Promise { + await this.migrationsService.process(job.data.migrationId); + } +} diff --git a/backend/src/application-migrations/application-migrations.service.ts b/backend/src/application-migrations/application-migrations.service.ts new file mode 100644 index 0000000..5586b6b --- /dev/null +++ b/backend/src/application-migrations/application-migrations.service.ts @@ -0,0 +1,263 @@ +import { BadRequestException, Injectable, NotFoundException } from '@nestjs/common'; +import { InjectQueue } from '@nestjs/bull'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Queue } from 'bull'; +import { Repository } from 'typeorm'; +import { ApplicationMigrationJob } from './entities/application-migration-job.entity'; +import { ApplicationMigrationEvent } from './entities/application-migration-event.entity'; +import { CreateApplicationMigrationDto } from './dto/application-migration.dto'; +import { ApplicationsService } from '../applications/applications.service'; +import { ClustersService } from '../clusters/clusters.service'; +import { KubernetesService } from '../kubernetes/kubernetes.service'; +import { ClusterStatus } from '../common/enums'; + +@Injectable() +export class ApplicationMigrationsService { + constructor( + @InjectRepository(ApplicationMigrationJob) + private readonly jobsRepository: Repository, + @InjectRepository(ApplicationMigrationEvent) + private readonly eventsRepository: Repository, + @InjectQueue('application-migrations') + private readonly queue: Queue, + private readonly applicationsService: ApplicationsService, + private readonly clustersService: ClustersService, + private readonly kubernetesService: KubernetesService, + ) {} + + async create( + applicationId: string, + requestedBy: string, + dto: CreateApplicationMigrationDto, + ): Promise { + const app = await this.applicationsService.findOne(applicationId); + if (!app.clusterId) { + throw new BadRequestException('Application does not have a source cluster assignment'); + } + if (app.clusterId === dto.targetClusterId) { + throw new BadRequestException('Target cluster must be different from the source cluster'); + } + + const target = await this.clustersService.findOne(dto.targetClusterId); + if (target.status !== ClusterStatus.ACTIVE || target.healthStatus !== 'healthy') { + throw new BadRequestException('Target cluster must be active and healthy before migration'); + } + + const activeJob = await this.jobsRepository.findOne({ + where: [ + { applicationId, status: 'queued' }, + { applicationId, status: 'running' }, + { applicationId, status: 'rolling_back' }, + ], + order: { createdAt: 'DESC' }, + }); + if (activeJob) { + throw new BadRequestException('A migration is already queued or running for this application'); + } + + const job = await this.jobsRepository.save(this.jobsRepository.create({ + applicationId, + requestedBy, + sourceClusterId: app.clusterId, + targetClusterId: dto.targetClusterId, + status: 'queued', + maxAttempts: 3, + metadata: { + note: dto.note, + migrateStorage: dto.migrateStorage !== false, + }, + })); + await this.log(job.id, 'queued', 'Migration job queued', { targetClusterId: dto.targetClusterId }); + await this.queue.add('run', { migrationId: job.id }, { + attempts: 1, + removeOnComplete: true, + removeOnFail: false, + }); + return this.findOne(job.id); + } + + async list(applicationId?: string): Promise { + return this.jobsRepository.find({ + where: applicationId ? { applicationId } : {}, + relations: ['application', 'sourceCluster', 'targetCluster'], + order: { createdAt: 'DESC' }, + take: 100, + }); + } + + async findOne(id: string): Promise { + const job = await this.jobsRepository.findOne({ + where: { id }, + relations: ['application', 'sourceCluster', 'targetCluster'], + }); + if (!job) { + throw new NotFoundException('Migration job not found'); + } + return job; + } + + async events(id: string): Promise { + await this.findOne(id); + return this.eventsRepository.find({ + where: { migrationId: id }, + order: { createdAt: 'ASC' }, + }); + } + + async retry(id: string, requestedBy: string): Promise { + const job = await this.findOne(id); + if (!['failed', 'rolled_back'].includes(job.status)) { + throw new BadRequestException('Only failed or rolled back migrations can be retried'); + } + if (job.attempts >= job.maxAttempts) { + throw new BadRequestException('Migration retry limit reached'); + } + + await this.jobsRepository.update(id, { + status: 'queued', + errorMessage: null, + currentStep: 'queued', + completedAt: null, + metadata: { + ...(job.metadata || {}), + retriedBy: requestedBy, + retriedAt: new Date().toISOString(), + } as any, + }); + await this.log(id, 'queued', 'Migration retry queued', { requestedBy }); + await this.queue.add('run', { migrationId: id }, { + attempts: 1, + removeOnComplete: true, + removeOnFail: false, + }); + return this.findOne(id); + } + + async process(migrationId: string): Promise { + const job = await this.findOne(migrationId); + const app = await this.applicationsService.findOne(job.applicationId); + const sourceClusterId = job.sourceClusterId; + const targetClusterId = job.targetClusterId; + const sourceApp = { ...app, clusterId: sourceClusterId }; + const targetApp = { ...app, clusterId: targetClusterId }; + + await this.updateJob(job.id, 'running', 'validate-target', { attempts: job.attempts + 1, startedAt: new Date() }); + + try { + await this.validateTargetCluster(targetClusterId); + await this.log(job.id, 'validate-target', 'Target cluster is active and healthy', { targetClusterId }); + + await this.updateJob(job.id, 'running', 'transfer-secrets-configs'); + await this.kubernetesService.prepareApplicationMigration(sourceApp as any, targetClusterId, { + migrateStorage: job.metadata?.migrateStorage !== false, + log: (step, message, metadata) => this.log(job.id, step, message, metadata), + }); + + if (!app.latestImageTag) { + throw new BadRequestException('Application must have a deployed image before migration'); + } + + await this.updateJob(job.id, 'running', 'deploy-target'); + await this.log(job.id, 'deploy-target', 'Deploying application on target cluster before cutover'); + const targetResources = await this.kubernetesService.deployApplication(targetApp as any, app.latestImageTag); + + await this.updateJob(job.id, 'running', 'health-check'); + await this.kubernetesService.waitForApplicationReady(targetApp as any, 600_000); + await this.log(job.id, 'health-check', 'Target deployment is healthy'); + + await this.updateJob(job.id, 'running', 'update-ingress'); + await this.kubernetesService.updateIngress(targetApp as any); + await this.log(job.id, 'update-ingress', 'Ingress/load balancer updated on target cluster'); + + await this.updateJob(job.id, 'running', 'cutover'); + await this.applicationsService.updateClusterAssignment(app.id, targetClusterId, app.poolId); + await this.log(job.id, 'cutover', 'Application traffic cut over to target cluster', { + sourceClusterId, + targetClusterId, + }); + + await this.updateJob(job.id, 'running', 'cleanup-source'); + await this.kubernetesService.deleteApplication(sourceApp as any); + await this.log(job.id, 'cleanup-source', 'Source cluster deployment removed after successful migration'); + + await this.jobsRepository.update(job.id, { + status: 'completed', + currentStep: 'completed', + completedAt: new Date(), + metadata: { + ...(job.metadata || {}), + targetResources, + }, + }); + await this.log(job.id, 'completed', 'Migration completed successfully'); + } catch (error: any) { + await this.handleFailure(job.id, sourceApp, targetApp, error); + throw error; + } + } + + private async validateTargetCluster(targetClusterId: string): Promise { + const target = await this.clustersService.findOne(targetClusterId); + if (target.status !== ClusterStatus.ACTIVE || target.healthStatus !== 'healthy') { + throw new BadRequestException('Target cluster is not active and healthy'); + } + } + + private async handleFailure(jobId: string, sourceApp: any, targetApp: any, error: any): Promise { + const message = error?.message || 'Migration failed'; + await this.jobsRepository.update(jobId, { + status: 'rolling_back', + currentStep: 'rollback', + errorMessage: message, + }); + await this.log(jobId, 'rollback', 'Migration failed; rolling back target resources', { error: message }, 'error'); + + try { + await this.kubernetesService.deleteApplication(targetApp); + await this.applicationsService.updateClusterAssignment(sourceApp.id, sourceApp.clusterId, sourceApp.poolId); + await this.jobsRepository.update(jobId, { + status: 'rolled_back', + currentStep: 'rolled_back', + completedAt: new Date(), + }); + await this.log(jobId, 'rolled_back', 'Rollback completed; source cluster assignment preserved'); + } catch (rollbackError: any) { + await this.jobsRepository.update(jobId, { + status: 'failed', + currentStep: 'failed', + completedAt: new Date(), + errorMessage: `${message}; rollback failed: ${rollbackError.message}`, + }); + await this.log(jobId, 'failed', `Rollback failed: ${rollbackError.message}`, undefined, 'error'); + } + } + + private async updateJob( + id: string, + status: ApplicationMigrationJob['status'], + currentStep: string, + extra: Partial = {}, + ): Promise { + await this.jobsRepository.update(id, { + status, + currentStep, + ...extra, + }); + } + + private async log( + migrationId: string, + step: string, + message: string, + metadata?: Record, + level: 'info' | 'warn' | 'error' = 'info', + ): Promise { + await this.eventsRepository.save(this.eventsRepository.create({ + migrationId, + step, + message, + metadata, + level, + })); + } +} diff --git a/backend/src/application-migrations/dto/application-migration.dto.ts b/backend/src/application-migrations/dto/application-migration.dto.ts new file mode 100644 index 0000000..50ad8f2 --- /dev/null +++ b/backend/src/application-migrations/dto/application-migration.dto.ts @@ -0,0 +1,18 @@ +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { IsBoolean, IsOptional, IsString, IsUUID } from 'class-validator'; + +export class CreateApplicationMigrationDto { + @ApiProperty({ description: 'Destination Kubernetes cluster ID' }) + @IsUUID() + targetClusterId: string; + + @ApiPropertyOptional({ description: 'Optional operator note for audit logs' }) + @IsOptional() + @IsString() + note?: string; + + @ApiPropertyOptional({ description: 'Copy PVC definitions and attempt provider-neutral storage preparation' }) + @IsOptional() + @IsBoolean() + migrateStorage?: boolean; +} diff --git a/backend/src/application-migrations/entities/application-migration-event.entity.ts b/backend/src/application-migrations/entities/application-migration-event.entity.ts new file mode 100644 index 0000000..1379467 --- /dev/null +++ b/backend/src/application-migrations/entities/application-migration-event.entity.ts @@ -0,0 +1,37 @@ +import { + Column, + CreateDateColumn, + Entity, + JoinColumn, + ManyToOne, + PrimaryGeneratedColumn, +} from 'typeorm'; +import { ApplicationMigrationJob } from './application-migration-job.entity'; + +@Entity('application_migration_events') +export class ApplicationMigrationEvent { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column() + migrationId: string; + + @ManyToOne(() => ApplicationMigrationJob, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'migrationId' }) + migration: ApplicationMigrationJob; + + @Column() + step: string; + + @Column({ default: 'info' }) + level: 'info' | 'warn' | 'error'; + + @Column() + message: string; + + @Column({ type: 'jsonb', nullable: true }) + metadata: Record; + + @CreateDateColumn() + createdAt: Date; +} diff --git a/backend/src/application-migrations/entities/application-migration-job.entity.ts b/backend/src/application-migrations/entities/application-migration-job.entity.ts new file mode 100644 index 0000000..265cd1e --- /dev/null +++ b/backend/src/application-migrations/entities/application-migration-job.entity.ts @@ -0,0 +1,79 @@ +import { + Column, + CreateDateColumn, + Entity, + JoinColumn, + ManyToOne, + PrimaryGeneratedColumn, + UpdateDateColumn, +} from 'typeorm'; +import { Application } from '../../applications/entities/application.entity'; +import { Cluster } from '../../clusters/entities/cluster.entity'; + +export type ApplicationMigrationStatus = + | 'queued' + | 'running' + | 'completed' + | 'failed' + | 'rolling_back' + | 'rolled_back'; + +@Entity('application_migration_jobs') +export class ApplicationMigrationJob { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column() + applicationId: string; + + @ManyToOne(() => Application, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'applicationId' }) + application: Application; + + @Column() + requestedBy: string; + + @Column({ type: 'uuid' }) + sourceClusterId: string; + + @ManyToOne(() => Cluster, { onDelete: 'RESTRICT' }) + @JoinColumn({ name: 'sourceClusterId' }) + sourceCluster: Cluster; + + @Column({ type: 'uuid' }) + targetClusterId: string; + + @ManyToOne(() => Cluster, { onDelete: 'RESTRICT' }) + @JoinColumn({ name: 'targetClusterId' }) + targetCluster: Cluster; + + @Column({ default: 'queued' }) + status: ApplicationMigrationStatus; + + @Column({ default: 0 }) + attempts: number; + + @Column({ default: 3 }) + maxAttempts: number; + + @Column({ type: 'varchar', nullable: true }) + currentStep: string | null; + + @Column({ type: 'varchar', nullable: true }) + errorMessage: string | null; + + @Column({ type: 'jsonb', nullable: true }) + metadata: Record | null; + + @Column({ type: 'timestamptz', nullable: true }) + startedAt: Date | null; + + @Column({ type: 'timestamptz', nullable: true }) + completedAt: Date | null; + + @CreateDateColumn() + createdAt: Date; + + @UpdateDateColumn() + updatedAt: Date; +} diff --git a/backend/src/clusters/clusters.service.ts b/backend/src/clusters/clusters.service.ts index f5905bd..1253056 100644 --- a/backend/src/clusters/clusters.service.ts +++ b/backend/src/clusters/clusters.service.ts @@ -22,6 +22,7 @@ import { CreateClusterPoolDto, UpdateClusterPoolDto } from './dto/cluster-pool.d import { ClusterStatus } from '../common/enums'; import { ElasticsearchService } from '../kubernetes/elasticsearch.service'; import { CreateApplicationDto } from '../applications/dto/application.dto'; +import { registerKubeconfigNoProxy } from '../common/kubernetes-proxy.util'; @Injectable() export class ClustersService implements OnModuleInit, OnModuleDestroy { @@ -77,8 +78,11 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy { */ async testConnection(kubeconfig: string): Promise<{ connected: boolean; version?: string; error?: string }> { try { + const decryptedKubeconfig = this.decryptKubeconfig(kubeconfig); + registerKubeconfigNoProxy(decryptedKubeconfig); + const kc = new k8s.KubeConfig(); - kc.loadFromString(this.decryptKubeconfig(kubeconfig)); + kc.loadFromString(decryptedKubeconfig); const versionApi = kc.makeApiClient(k8s.VersionApi); const result = await versionApi.getCode(); @@ -702,6 +706,7 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy { } const kc = new k8s.KubeConfig(); + registerKubeconfigNoProxy(cluster.kubeconfig); kc.loadFromString(cluster.kubeconfig); const coreApi = kc.makeApiClient(k8s.CoreV1Api); @@ -825,6 +830,7 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy { */ async bootstrapCluster(kubeconfig: string): Promise { const kc = new k8s.KubeConfig(); + registerKubeconfigNoProxy(kubeconfig); kc.loadFromString(kubeconfig); const coreApi = kc.makeApiClient(k8s.CoreV1Api); const appsApi = kc.makeApiClient(k8s.AppsV1Api); diff --git a/backend/src/common/kubernetes-proxy.util.ts b/backend/src/common/kubernetes-proxy.util.ts new file mode 100644 index 0000000..3355314 --- /dev/null +++ b/backend/src/common/kubernetes-proxy.util.ts @@ -0,0 +1,42 @@ +import * as k8s from '@kubernetes/client-node'; + +function normalizeNoProxyValue(value?: string): string[] { + return (value || '') + .split(',') + .map((entry) => entry.trim().replace(/^['"]|['"]$/g, '')) + .filter(Boolean); +} + +function appendNoProxyEntries(envKey: 'NO_PROXY' | 'no_proxy', entries: string[]): void { + const existing = normalizeNoProxyValue(process.env[envKey]); + if (existing.includes('*')) { + return; + } + + const merged = new Set(existing); + for (const entry of entries) { + merged.add(entry); + } + + process.env[envKey] = Array.from(merged).join(','); +} + +export function registerKubeconfigNoProxy(kubeconfig: string): string[] { + const kc = new k8s.KubeConfig(); + kc.loadFromString(kubeconfig); + + const cluster = kc.getCurrentCluster() || kc.getClusters()[0]; + if (!cluster?.server) { + return []; + } + + const url = new URL(cluster.server); + const entries = new Set(); + entries.add(url.hostname); + entries.add(url.host); + + appendNoProxyEntries('NO_PROXY', Array.from(entries)); + appendNoProxyEntries('no_proxy', Array.from(entries)); + + return Array.from(entries); +} diff --git a/backend/src/kubernetes/kubernetes.service.ts b/backend/src/kubernetes/kubernetes.service.ts index c1894de..2a399a7 100644 --- a/backend/src/kubernetes/kubernetes.service.ts +++ b/backend/src/kubernetes/kubernetes.service.ts @@ -11,6 +11,7 @@ import { Application } from '../applications/entities/application.entity'; import { ensureAppUrlEnv } from '../applications/app-url.util'; import { AppRuntime, DatabaseType, CustomDomainStatus, ServiceAccessTarget } from '../common/enums'; import { HelmService } from './helm.service'; +import { registerKubeconfigNoProxy } from '../common/kubernetes-proxy.util'; const execFileAsync = promisify(execFile); @@ -81,6 +82,7 @@ export class KubernetesService implements OnModuleInit { : await this.clustersService.getDefault(); const kc = new k8s.KubeConfig(); + registerKubeconfigNoProxy(cluster.kubeconfig); kc.loadFromString(cluster.kubeconfig); return { @@ -218,7 +220,11 @@ export class KubernetesService implements OnModuleInit { } } - async waitForApplicationReady(app: Application, timeoutMs = 600_000): Promise { + async waitForApplicationReady( + app: Application, + timeoutMs = 600_000, + shouldAbort?: () => Promise, + ): Promise { const { coreApi, appsApi } = await this.getK8sClient(app.clusterId); const namespace = `user-${app.userId.split('-')[0]}`; const workloads = [ @@ -235,6 +241,10 @@ export class KubernetesService implements OnModuleInit { ); while (Date.now() - start < timeoutMs) { + if (await shouldAbort?.()) { + throw new Error('Deployment cancelled by user'); + } + const statuses = await Promise.all( workloads.map((workload) => this.getDeploymentReadiness(appsApi, namespace, workload.name, workload.replicas)), ); @@ -2046,6 +2056,16 @@ export class KubernetesService implements OnModuleInit { } } + private toDnsLabel(value: string): string { + return value + .toLowerCase() + .replace(/[^a-z0-9-]/g, '-') + .replace(/-+/g, '-') + .replace(/^-|-$/g, '') + .slice(0, 63) + .replace(/-$/g, ''); + } + resolveAccessTarget( app: Application, target: ServiceAccessTarget, @@ -2087,7 +2107,7 @@ export class KubernetesService implements OnModuleInit { const namespace = this.getUserNamespace(app.userId); const { selector, targetPort, portName } = this.resolveAccessTarget(app, target); const shortId = grantId.split('-')[0]; - const k8sServiceName = `${app.name}-${target}-access-${shortId}`.slice(0, 63); + const k8sServiceName = this.toDnsLabel(`${app.name}-${target}-access-${shortId}`); const portSpec: k8s.V1ServicePort = { port: targetPort, @@ -2116,7 +2136,16 @@ export class KubernetesService implements OnModuleInit { }, }; - const created = await coreApi.createNamespacedService(namespace, service); + let created: { body: k8s.V1Service }; + try { + created = await coreApi.createNamespacedService(namespace, service); + } catch (error: any) { + const message = error.body?.message || error.response?.body?.message || error.message || 'HTTP request failed'; + this.logger.warn( + `K8s client failed to create temporary access service ${k8sServiceName}; trying kubectl fallback: ${message}`, + ); + created = { body: await this.applyServiceWithKubectl(app.clusterId, namespace, service) }; + } const nodePort = created.body.spec?.ports?.[0]?.nodePort; if (!nodePort) { try { @@ -2133,6 +2162,50 @@ export class KubernetesService implements OnModuleInit { return { host, nodePort, k8sServiceName, targetPort }; } + private async applyServiceWithKubectl( + clusterId: string | undefined, + namespace: string, + service: k8s.V1Service, + ): Promise { + const tmpDir = fs.mkdtempSync(path.join('/tmp', 'cloudhost-access-')); + const kubeconfigPath = path.join(tmpDir, 'kubeconfig.yaml'); + const manifestPath = path.join(tmpDir, 'service.json'); + + try { + fs.writeFileSync(kubeconfigPath, await this.getKubeconfig(clusterId), { mode: 0o600 }); + fs.writeFileSync(manifestPath, JSON.stringify(service), { mode: 0o600 }); + + await execFileAsync('kubectl', [ + '--kubeconfig', + kubeconfigPath, + 'apply', + '-n', + namespace, + '-f', + manifestPath, + ]); + + const { stdout } = await execFileAsync('kubectl', [ + '--kubeconfig', + kubeconfigPath, + 'get', + 'service', + service.metadata!.name!, + '-n', + namespace, + '-o', + 'json', + ]); + + return JSON.parse(stdout) as k8s.V1Service; + } catch (error: any) { + const message = error.stderr || error.message || 'kubectl failed'; + throw new Error(`Failed to create temporary access service: ${message}`); + } finally { + try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {} + } + } + async revokeTemporaryAccess( clusterId: string, namespace: string, @@ -2341,6 +2414,239 @@ export class KubernetesService implements OnModuleInit { this.logger.log(`All K8s resources cleaned up for ${app.name} in ${namespace}`); } + async prepareApplicationMigration( + app: Application, + targetClusterId: string, + options: { + migrateStorage?: boolean; + log?: (step: string, message: string, metadata?: Record) => Promise; + } = {}, + ): Promise { + const namespace = this.getUserNamespace(app.userId); + const source = await this.getK8sClient(app.clusterId); + const target = await this.getK8sClient(targetClusterId); + + await this.ensureNamespaceOnCluster(target.coreApi, namespace); + await options.log?.('transfer-secrets-configs', 'Target namespace ensured', { namespace }); + + const secretNames = [ + `${app.name}-env`, + `${app.name}-db-secret`, + `${app.name}-redis-secret`, + `${app.name}-rabbitmq-secret`, + `${app.name}-tls`, + ]; + for (const secretName of secretNames) { + try { + const secret = await source.coreApi.readNamespacedSecret(secretName, namespace); + await this.upsertSecret(target.coreApi, namespace, this.cleanK8sObject(secret.body)); + await options.log?.('transfer-secrets-configs', `Secret ${secretName} copied`); + } catch (error: any) { + if (!this.isK8sNotFound(error)) { + throw error; + } + } + } + + const configMapNames = [ + `${app.name}-fluent-bit-config`, + ]; + for (const configMapName of configMapNames) { + try { + const configMap = await source.coreApi.readNamespacedConfigMap(configMapName, namespace); + await this.upsertConfigMap(target.coreApi, namespace, this.cleanK8sObject(configMap.body)); + await options.log?.('transfer-secrets-configs', `ConfigMap ${configMapName} copied`); + } catch (error: any) { + if (!this.isK8sNotFound(error)) { + throw error; + } + } + } + + if (options.migrateStorage !== false) { + const pvcNames = [ + `${app.name}-storage`, + `${app.name}-wp-content`, + `${app.name}-db`, + `${app.name}-redis-data`, + `${app.name}-rabbitmq-data`, + ]; + for (const pvcName of pvcNames) { + try { + const pvc = await source.coreApi.readNamespacedPersistentVolumeClaim(pvcName, namespace); + await this.upsertPvcDefinition(target.coreApi, namespace, this.cleanPvcForMigration(pvc.body)); + await options.log?.('transfer-volumes', `PVC definition ${pvcName} prepared on target cluster`, { + note: 'Starting best-effort PVC data copy through migration helper pods.', + }); + await this.copyPvcDataBetweenClusters(app.clusterId!, targetClusterId, namespace, pvcName, options.log); + } catch (error: any) { + if (!this.isK8sNotFound(error)) { + throw error; + } + } + } + } + } + + private async copyPvcDataBetweenClusters( + sourceClusterId: string, + targetClusterId: string, + namespace: string, + pvcName: string, + log?: (step: string, message: string, metadata?: Record) => Promise, + ): Promise { + const safeName = pvcName.replace(/[^a-z0-9-]/gi, '-').toLowerCase().slice(0, 32); + const suffix = `${Date.now()}`.slice(-6); + const sourcePod = `migrate-src-${safeName}-${suffix}`; + const targetPod = `migrate-dst-${safeName}-${suffix}`; + const tempDir = path.join('/tmp', `app-migration-${safeName}-${suffix}`); + const sourceKubeconfig = path.join(tempDir, 'source.kubeconfig'); + const targetKubeconfig = path.join(tempDir, 'target.kubeconfig'); + + fs.mkdirSync(tempDir, { recursive: true }); + fs.writeFileSync(sourceKubeconfig, await this.getKubeconfig(sourceClusterId), { mode: 0o600 }); + fs.writeFileSync(targetKubeconfig, await this.getKubeconfig(targetClusterId), { mode: 0o600 }); + + try { + await this.createPvcCopyPod(sourceKubeconfig, namespace, sourcePod, pvcName); + await this.createPvcCopyPod(targetKubeconfig, namespace, targetPod, pvcName); + await log?.('transfer-volumes', `Copy helper pods ready for PVC ${pvcName}`); + + const localDataPath = path.join(tempDir, 'data'); + await execFileAsync('kubectl', ['--kubeconfig', sourceKubeconfig, '-n', namespace, 'cp', `${sourcePod}:/data`, localDataPath], { + timeout: 30 * 60 * 1000, + }); + await execFileAsync('kubectl', ['--kubeconfig', targetKubeconfig, '-n', namespace, 'cp', `${localDataPath}/.`, `${targetPod}:/data`], { + timeout: 30 * 60 * 1000, + }); + await log?.('transfer-volumes', `PVC data copied for ${pvcName}`); + } finally { + await execFileAsync('kubectl', ['--kubeconfig', sourceKubeconfig, '-n', namespace, 'delete', 'pod', sourcePod, '--ignore-not-found=true']).catch(() => undefined); + await execFileAsync('kubectl', ['--kubeconfig', targetKubeconfig, '-n', namespace, 'delete', 'pod', targetPod, '--ignore-not-found=true']).catch(() => undefined); + fs.rmSync(tempDir, { recursive: true, force: true }); + } + } + + private async createPvcCopyPod( + kubeconfigPath: string, + namespace: string, + podName: string, + pvcName: string, + ): Promise { + const manifestPath = path.join('/tmp', `${podName}.json`); + const manifest = { + apiVersion: 'v1', + kind: 'Pod', + metadata: { name: podName, namespace }, + spec: { + restartPolicy: 'Never', + containers: [{ + name: 'copy', + image: 'busybox:1.36', + command: ['sh', '-c', 'mkdir -p /data && sleep 3600'], + volumeMounts: [{ name: 'data', mountPath: '/data' }], + }], + volumes: [{ name: 'data', persistentVolumeClaim: { claimName: pvcName } }], + }, + }; + + fs.writeFileSync(manifestPath, JSON.stringify(manifest)); + try { + await execFileAsync('kubectl', ['--kubeconfig', kubeconfigPath, 'apply', '-f', manifestPath], { timeout: 120_000 }); + await execFileAsync('kubectl', ['--kubeconfig', kubeconfigPath, '-n', namespace, 'wait', '--for=condition=Ready', `pod/${podName}`, '--timeout=180s'], { timeout: 210_000 }); + } finally { + fs.rmSync(manifestPath, { force: true }); + } + } + + private async ensureNamespaceOnCluster(coreApi: k8s.CoreV1Api, namespace: string): Promise { + try { + await coreApi.readNamespace(namespace); + } catch (error: any) { + if (this.isK8sNotFound(error)) { + await coreApi.createNamespace({ metadata: { name: namespace } }); + return; + } + throw error; + } + } + + private async upsertSecret(coreApi: k8s.CoreV1Api, namespace: string, secret: k8s.V1Secret): Promise { + secret.metadata = { ...(secret.metadata || {}), namespace }; + try { + await coreApi.replaceNamespacedSecret(secret.metadata.name!, namespace, secret); + } catch (error: any) { + if (this.isK8sNotFound(error)) { + await coreApi.createNamespacedSecret(namespace, secret); + return; + } + throw error; + } + } + + private async upsertConfigMap(coreApi: k8s.CoreV1Api, namespace: string, configMap: k8s.V1ConfigMap): Promise { + configMap.metadata = { ...(configMap.metadata || {}), namespace }; + try { + await coreApi.replaceNamespacedConfigMap(configMap.metadata.name!, namespace, configMap); + } catch (error: any) { + if (this.isK8sNotFound(error)) { + await coreApi.createNamespacedConfigMap(namespace, configMap); + return; + } + throw error; + } + } + + private async upsertPvcDefinition( + coreApi: k8s.CoreV1Api, + namespace: string, + pvc: k8s.V1PersistentVolumeClaim, + ): Promise { + pvc.metadata = { ...(pvc.metadata || {}), namespace }; + try { + await coreApi.readNamespacedPersistentVolumeClaim(pvc.metadata.name!, namespace); + } catch (error: any) { + if (this.isK8sNotFound(error)) { + await coreApi.createNamespacedPersistentVolumeClaim(namespace, pvc); + return; + } + throw error; + } + } + + private cleanK8sObject(obj: T): T { + const metadata = { ...(obj.metadata || {}) }; + delete metadata.uid; + delete metadata.resourceVersion; + delete metadata.generation; + delete metadata.creationTimestamp; + delete metadata.managedFields; + delete metadata.selfLink; + return { + ...obj, + metadata, + }; + } + + private cleanPvcForMigration(pvc: k8s.V1PersistentVolumeClaim): k8s.V1PersistentVolumeClaim { + const cleaned = this.cleanK8sObject(pvc); + return { + apiVersion: cleaned.apiVersion, + kind: cleaned.kind, + metadata: cleaned.metadata, + spec: { + accessModes: cleaned.spec?.accessModes, + resources: cleaned.spec?.resources, + storageClassName: cleaned.spec?.storageClassName, + volumeMode: cleaned.spec?.volumeMode, + }, + }; + } + + private isK8sNotFound(error: any): boolean { + return error?.statusCode === 404 || error?.body?.code === 404; + } + /** * Wait for the database pod to become Ready. * Polls pod status with label selector `app=-db`. diff --git a/frontend/src/app/dashboard/admin/apps/page.tsx b/frontend/src/app/dashboard/admin/apps/page.tsx index cf867d5..d017aa6 100644 --- a/frontend/src/app/dashboard/admin/apps/page.tsx +++ b/frontend/src/app/dashboard/admin/apps/page.tsx @@ -5,8 +5,8 @@ import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'; import Link from 'next/link'; import api from '@/lib/api'; import { toast } from 'react-toastify'; -import type { Application, AppLifecycleStatus, BillingCycle } from '@/types'; -import { Search, X, Package, Hexagon, User, Database, Box, AlertTriangle, Clock } from 'lucide-react'; +import type { Application, AppLifecycleStatus, ApplicationMigrationEvent, ApplicationMigrationJob, BillingCycle, Cluster } from '@/types'; +import { Search, X, Package, Hexagon, User, Database, Box, AlertTriangle, Clock, ArrowRightLeft, RefreshCw } from 'lucide-react'; import { useConfirm } from '@/components/confirm-modal'; import { useDebounce } from '@/hooks/useDebounce'; @@ -65,6 +65,8 @@ export default function AdminAppsPage() { const confirm = useConfirm(); const [search, setSearch] = useState(''); const debouncedSearch = useDebounce(search, 400); + const [migrateApp, setMigrateApp] = useState(null); + const [targetClusterId, setTargetClusterId] = useState(''); const { data: apps = [], isLoading } = useQuery({ queryKey: ['admin-applications', debouncedSearch], @@ -83,6 +85,55 @@ export default function AdminAppsPage() { onError: () => toast.error('Failed to delete application'), }); + const { data: clusters = [] } = useQuery({ + queryKey: ['admin-clusters'], + queryFn: () => api.get('/clusters').then((r) => r.data), + }); + + const { data: migrations = [] } = useQuery({ + queryKey: ['application-migrations'], + queryFn: () => api.get('/application-migrations').then((r) => r.data), + refetchInterval: 5000, + }); + + const selectedMigration = migrateApp + ? migrations.find((migration) => migration.applicationId === migrateApp.id) + : undefined; + const clusterById = new Map(clusters.map((cluster) => [cluster.id, cluster])); + const currentCluster = migrateApp?.clusterId ? clusterById.get(migrateApp.clusterId) : undefined; + const targetClusters = migrateApp + ? clusters.filter((cluster) => cluster.id !== migrateApp.clusterId) + : []; + + const { data: selectedMigrationEvents = [] } = useQuery({ + queryKey: ['application-migration-events', selectedMigration?.id], + queryFn: () => api.get(`/application-migrations/${selectedMigration!.id}/events`).then((r) => r.data), + enabled: !!selectedMigration?.id, + refetchInterval: selectedMigration && ['queued', 'running', 'rolling_back'].includes(selectedMigration.status) ? 3000 : false, + }); + + const migrateMutation = useMutation({ + mutationFn: ({ appId, clusterId }: { appId: string; clusterId: string }) => + api.post(`/application-migrations/applications/${appId}`, { + targetClusterId: clusterId, + migrateStorage: true, + }), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['application-migrations'] }); + toast.success('Migration job queued'); + }, + onError: (err: any) => toast.error(err?.response?.data?.message || 'Failed to queue migration'), + }); + + const retryMigration = useMutation({ + mutationFn: (id: string) => api.post(`/application-migrations/${id}/retry`), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['application-migrations'] }); + toast.success('Migration retry queued'); + }, + onError: (err: any) => toast.error(err?.response?.data?.message || 'Failed to retry migration'), + }); + // Compute status counts from apps const statusCounts = apps.reduce( (acc, app) => { @@ -224,6 +275,7 @@ export default function AdminAppsPage() { Owner Status Service + Cluster Plan / Expiry Actions @@ -233,6 +285,8 @@ export default function AdminAppsPage() { const latestStatus = app.deployments?.[0]?.status || 'pending'; const lifecycle = app.lifecycleStatus || 'active'; const expiry = formatExpiry(app.planExpiresAt); + const latestMigration = migrations.find((migration) => migration.applicationId === app.id); + const assignedCluster = app.clusterId ? clusterById.get(app.clusterId) : undefined; return ( @@ -273,6 +327,23 @@ export default function AdminAppsPage() {

Delete: {formatDeletionDate(app.scheduledDeletionAt)}

)} + + {assignedCluster ? ( +
+

{assignedCluster.name}

+

+ {assignedCluster.region || 'N/A'} · {assignedCluster.status}/{assignedCluster.healthStatus || 'unknown'} +

+
+ ) : app.clusterId ? ( +
+

Unknown cluster

+

{app.clusterId.slice(0, 8)}

+
+ ) : ( + Not assigned + )} + {app.billingCycle && ( {cycleLabels[app.billingCycle] || app.billingCycle} @@ -290,6 +361,28 @@ export default function AdminAppsPage() { View + + {latestMigration && ( + + {latestMigration.status} + + )} + + +
+ +
+

Current cluster

+

+ {currentCluster?.name || (migrateApp.clusterId ? `Unknown cluster (${migrateApp.clusterId.slice(0, 8)})` : 'Not assigned')} +

+ {currentCluster && ( +

+ {currentCluster.region || 'N/A'} · {currentCluster.status}/{currentCluster.healthStatus || 'unknown'} +

+ )} +
+ +

+ The current cluster is excluded. Migration is blocked for unhealthy, inactive, or maintenance clusters. +

+ {targetClusters.length === 0 && ( +

+ No other cluster is available as a migration target. +

+ )} +
+ + {selectedMigration && ( +
+
+
+

Latest migration

+

+ {selectedMigration.currentStep || 'queued'} · attempts {selectedMigration.attempts}/{selectedMigration.maxAttempts} +

+
+ + {selectedMigration.status} + +
+ {selectedMigration.errorMessage && ( +

{selectedMigration.errorMessage}

+ )} +
+ {selectedMigrationEvents.length === 0 ? ( +

No events yet.

+ ) : selectedMigrationEvents.map((event) => ( +
+ {new Date(event.createdAt).toLocaleTimeString()} + + {event.step} + + {event.message} +
+ ))} +
+ {['failed', 'rolled_back'].includes(selectedMigration.status) && ( + + )} +
+ )} + +
+ + +
+ + + )} ); } diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 35d5c0a..2b0a7b1 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -304,6 +304,44 @@ export interface ClusterAllocationLog { createdAt: string; } +export type ApplicationMigrationStatus = + | 'queued' + | 'running' + | 'completed' + | 'failed' + | 'rolling_back' + | 'rolled_back'; + +export interface ApplicationMigrationEvent { + id: string; + migrationId: string; + step: string; + level: 'info' | 'warn' | 'error'; + message: string; + metadata?: Record; + createdAt: string; +} + +export interface ApplicationMigrationJob { + id: string; + applicationId: string; + requestedBy: string; + sourceClusterId: string; + targetClusterId: string; + sourceCluster?: Cluster; + targetCluster?: Cluster; + status: ApplicationMigrationStatus; + attempts: number; + maxAttempts: number; + currentStep?: string; + errorMessage?: string; + metadata?: Record; + startedAt?: string; + completedAt?: string; + createdAt: string; + updatedAt: string; +} + export interface PodInfo { name: string; status: string;