Add application migration workflow.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
keyhan
@@ -11,6 +11,7 @@ import { Application } from '../applications/entities/application.entity';
|
||||
import { ensureAppUrlEnv } from '../applications/app-url.util';
|
||||
import { AppRuntime, DatabaseType, CustomDomainStatus, ServiceAccessTarget } from '../common/enums';
|
||||
import { HelmService } from './helm.service';
|
||||
import { registerKubeconfigNoProxy } from '../common/kubernetes-proxy.util';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
@@ -81,6 +82,7 @@ export class KubernetesService implements OnModuleInit {
|
||||
: await this.clustersService.getDefault();
|
||||
|
||||
const kc = new k8s.KubeConfig();
|
||||
registerKubeconfigNoProxy(cluster.kubeconfig);
|
||||
kc.loadFromString(cluster.kubeconfig);
|
||||
|
||||
return {
|
||||
@@ -218,7 +220,11 @@ export class KubernetesService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
async waitForApplicationReady(app: Application, timeoutMs = 600_000): Promise<void> {
|
||||
async waitForApplicationReady(
|
||||
app: Application,
|
||||
timeoutMs = 600_000,
|
||||
shouldAbort?: () => Promise<boolean>,
|
||||
): Promise<void> {
|
||||
const { coreApi, appsApi } = await this.getK8sClient(app.clusterId);
|
||||
const namespace = `user-${app.userId.split('-')[0]}`;
|
||||
const workloads = [
|
||||
@@ -235,6 +241,10 @@ export class KubernetesService implements OnModuleInit {
|
||||
);
|
||||
|
||||
while (Date.now() - start < timeoutMs) {
|
||||
if (await shouldAbort?.()) {
|
||||
throw new Error('Deployment cancelled by user');
|
||||
}
|
||||
|
||||
const statuses = await Promise.all(
|
||||
workloads.map((workload) => this.getDeploymentReadiness(appsApi, namespace, workload.name, workload.replicas)),
|
||||
);
|
||||
@@ -2046,6 +2056,16 @@ export class KubernetesService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
private toDnsLabel(value: string): string {
|
||||
return value
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9-]/g, '-')
|
||||
.replace(/-+/g, '-')
|
||||
.replace(/^-|-$/g, '')
|
||||
.slice(0, 63)
|
||||
.replace(/-$/g, '');
|
||||
}
|
||||
|
||||
resolveAccessTarget(
|
||||
app: Application,
|
||||
target: ServiceAccessTarget,
|
||||
@@ -2087,7 +2107,7 @@ export class KubernetesService implements OnModuleInit {
|
||||
const namespace = this.getUserNamespace(app.userId);
|
||||
const { selector, targetPort, portName } = this.resolveAccessTarget(app, target);
|
||||
const shortId = grantId.split('-')[0];
|
||||
const k8sServiceName = `${app.name}-${target}-access-${shortId}`.slice(0, 63);
|
||||
const k8sServiceName = this.toDnsLabel(`${app.name}-${target}-access-${shortId}`);
|
||||
|
||||
const portSpec: k8s.V1ServicePort = {
|
||||
port: targetPort,
|
||||
@@ -2116,7 +2136,16 @@ export class KubernetesService implements OnModuleInit {
|
||||
},
|
||||
};
|
||||
|
||||
const created = await coreApi.createNamespacedService(namespace, service);
|
||||
let created: { body: k8s.V1Service };
|
||||
try {
|
||||
created = await coreApi.createNamespacedService(namespace, service);
|
||||
} catch (error: any) {
|
||||
const message = error.body?.message || error.response?.body?.message || error.message || 'HTTP request failed';
|
||||
this.logger.warn(
|
||||
`K8s client failed to create temporary access service ${k8sServiceName}; trying kubectl fallback: ${message}`,
|
||||
);
|
||||
created = { body: await this.applyServiceWithKubectl(app.clusterId, namespace, service) };
|
||||
}
|
||||
const nodePort = created.body.spec?.ports?.[0]?.nodePort;
|
||||
if (!nodePort) {
|
||||
try {
|
||||
@@ -2133,6 +2162,50 @@ export class KubernetesService implements OnModuleInit {
|
||||
return { host, nodePort, k8sServiceName, targetPort };
|
||||
}
|
||||
|
||||
private async applyServiceWithKubectl(
|
||||
clusterId: string | undefined,
|
||||
namespace: string,
|
||||
service: k8s.V1Service,
|
||||
): Promise<k8s.V1Service> {
|
||||
const tmpDir = fs.mkdtempSync(path.join('/tmp', 'cloudhost-access-'));
|
||||
const kubeconfigPath = path.join(tmpDir, 'kubeconfig.yaml');
|
||||
const manifestPath = path.join(tmpDir, 'service.json');
|
||||
|
||||
try {
|
||||
fs.writeFileSync(kubeconfigPath, await this.getKubeconfig(clusterId), { mode: 0o600 });
|
||||
fs.writeFileSync(manifestPath, JSON.stringify(service), { mode: 0o600 });
|
||||
|
||||
await execFileAsync('kubectl', [
|
||||
'--kubeconfig',
|
||||
kubeconfigPath,
|
||||
'apply',
|
||||
'-n',
|
||||
namespace,
|
||||
'-f',
|
||||
manifestPath,
|
||||
]);
|
||||
|
||||
const { stdout } = await execFileAsync('kubectl', [
|
||||
'--kubeconfig',
|
||||
kubeconfigPath,
|
||||
'get',
|
||||
'service',
|
||||
service.metadata!.name!,
|
||||
'-n',
|
||||
namespace,
|
||||
'-o',
|
||||
'json',
|
||||
]);
|
||||
|
||||
return JSON.parse(stdout) as k8s.V1Service;
|
||||
} catch (error: any) {
|
||||
const message = error.stderr || error.message || 'kubectl failed';
|
||||
throw new Error(`Failed to create temporary access service: ${message}`);
|
||||
} finally {
|
||||
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
async revokeTemporaryAccess(
|
||||
clusterId: string,
|
||||
namespace: string,
|
||||
@@ -2341,6 +2414,239 @@ export class KubernetesService implements OnModuleInit {
|
||||
this.logger.log(`All K8s resources cleaned up for ${app.name} in ${namespace}`);
|
||||
}
|
||||
|
||||
async prepareApplicationMigration(
|
||||
app: Application,
|
||||
targetClusterId: string,
|
||||
options: {
|
||||
migrateStorage?: boolean;
|
||||
log?: (step: string, message: string, metadata?: Record<string, any>) => Promise<void>;
|
||||
} = {},
|
||||
): Promise<void> {
|
||||
const namespace = this.getUserNamespace(app.userId);
|
||||
const source = await this.getK8sClient(app.clusterId);
|
||||
const target = await this.getK8sClient(targetClusterId);
|
||||
|
||||
await this.ensureNamespaceOnCluster(target.coreApi, namespace);
|
||||
await options.log?.('transfer-secrets-configs', 'Target namespace ensured', { namespace });
|
||||
|
||||
const secretNames = [
|
||||
`${app.name}-env`,
|
||||
`${app.name}-db-secret`,
|
||||
`${app.name}-redis-secret`,
|
||||
`${app.name}-rabbitmq-secret`,
|
||||
`${app.name}-tls`,
|
||||
];
|
||||
for (const secretName of secretNames) {
|
||||
try {
|
||||
const secret = await source.coreApi.readNamespacedSecret(secretName, namespace);
|
||||
await this.upsertSecret(target.coreApi, namespace, this.cleanK8sObject(secret.body));
|
||||
await options.log?.('transfer-secrets-configs', `Secret ${secretName} copied`);
|
||||
} catch (error: any) {
|
||||
if (!this.isK8sNotFound(error)) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const configMapNames = [
|
||||
`${app.name}-fluent-bit-config`,
|
||||
];
|
||||
for (const configMapName of configMapNames) {
|
||||
try {
|
||||
const configMap = await source.coreApi.readNamespacedConfigMap(configMapName, namespace);
|
||||
await this.upsertConfigMap(target.coreApi, namespace, this.cleanK8sObject(configMap.body));
|
||||
await options.log?.('transfer-secrets-configs', `ConfigMap ${configMapName} copied`);
|
||||
} catch (error: any) {
|
||||
if (!this.isK8sNotFound(error)) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (options.migrateStorage !== false) {
|
||||
const pvcNames = [
|
||||
`${app.name}-storage`,
|
||||
`${app.name}-wp-content`,
|
||||
`${app.name}-db`,
|
||||
`${app.name}-redis-data`,
|
||||
`${app.name}-rabbitmq-data`,
|
||||
];
|
||||
for (const pvcName of pvcNames) {
|
||||
try {
|
||||
const pvc = await source.coreApi.readNamespacedPersistentVolumeClaim(pvcName, namespace);
|
||||
await this.upsertPvcDefinition(target.coreApi, namespace, this.cleanPvcForMigration(pvc.body));
|
||||
await options.log?.('transfer-volumes', `PVC definition ${pvcName} prepared on target cluster`, {
|
||||
note: 'Starting best-effort PVC data copy through migration helper pods.',
|
||||
});
|
||||
await this.copyPvcDataBetweenClusters(app.clusterId!, targetClusterId, namespace, pvcName, options.log);
|
||||
} catch (error: any) {
|
||||
if (!this.isK8sNotFound(error)) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async copyPvcDataBetweenClusters(
|
||||
sourceClusterId: string,
|
||||
targetClusterId: string,
|
||||
namespace: string,
|
||||
pvcName: string,
|
||||
log?: (step: string, message: string, metadata?: Record<string, any>) => Promise<void>,
|
||||
): Promise<void> {
|
||||
const safeName = pvcName.replace(/[^a-z0-9-]/gi, '-').toLowerCase().slice(0, 32);
|
||||
const suffix = `${Date.now()}`.slice(-6);
|
||||
const sourcePod = `migrate-src-${safeName}-${suffix}`;
|
||||
const targetPod = `migrate-dst-${safeName}-${suffix}`;
|
||||
const tempDir = path.join('/tmp', `app-migration-${safeName}-${suffix}`);
|
||||
const sourceKubeconfig = path.join(tempDir, 'source.kubeconfig');
|
||||
const targetKubeconfig = path.join(tempDir, 'target.kubeconfig');
|
||||
|
||||
fs.mkdirSync(tempDir, { recursive: true });
|
||||
fs.writeFileSync(sourceKubeconfig, await this.getKubeconfig(sourceClusterId), { mode: 0o600 });
|
||||
fs.writeFileSync(targetKubeconfig, await this.getKubeconfig(targetClusterId), { mode: 0o600 });
|
||||
|
||||
try {
|
||||
await this.createPvcCopyPod(sourceKubeconfig, namespace, sourcePod, pvcName);
|
||||
await this.createPvcCopyPod(targetKubeconfig, namespace, targetPod, pvcName);
|
||||
await log?.('transfer-volumes', `Copy helper pods ready for PVC ${pvcName}`);
|
||||
|
||||
const localDataPath = path.join(tempDir, 'data');
|
||||
await execFileAsync('kubectl', ['--kubeconfig', sourceKubeconfig, '-n', namespace, 'cp', `${sourcePod}:/data`, localDataPath], {
|
||||
timeout: 30 * 60 * 1000,
|
||||
});
|
||||
await execFileAsync('kubectl', ['--kubeconfig', targetKubeconfig, '-n', namespace, 'cp', `${localDataPath}/.`, `${targetPod}:/data`], {
|
||||
timeout: 30 * 60 * 1000,
|
||||
});
|
||||
await log?.('transfer-volumes', `PVC data copied for ${pvcName}`);
|
||||
} finally {
|
||||
await execFileAsync('kubectl', ['--kubeconfig', sourceKubeconfig, '-n', namespace, 'delete', 'pod', sourcePod, '--ignore-not-found=true']).catch(() => undefined);
|
||||
await execFileAsync('kubectl', ['--kubeconfig', targetKubeconfig, '-n', namespace, 'delete', 'pod', targetPod, '--ignore-not-found=true']).catch(() => undefined);
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
private async createPvcCopyPod(
|
||||
kubeconfigPath: string,
|
||||
namespace: string,
|
||||
podName: string,
|
||||
pvcName: string,
|
||||
): Promise<void> {
|
||||
const manifestPath = path.join('/tmp', `${podName}.json`);
|
||||
const manifest = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Pod',
|
||||
metadata: { name: podName, namespace },
|
||||
spec: {
|
||||
restartPolicy: 'Never',
|
||||
containers: [{
|
||||
name: 'copy',
|
||||
image: 'busybox:1.36',
|
||||
command: ['sh', '-c', 'mkdir -p /data && sleep 3600'],
|
||||
volumeMounts: [{ name: 'data', mountPath: '/data' }],
|
||||
}],
|
||||
volumes: [{ name: 'data', persistentVolumeClaim: { claimName: pvcName } }],
|
||||
},
|
||||
};
|
||||
|
||||
fs.writeFileSync(manifestPath, JSON.stringify(manifest));
|
||||
try {
|
||||
await execFileAsync('kubectl', ['--kubeconfig', kubeconfigPath, 'apply', '-f', manifestPath], { timeout: 120_000 });
|
||||
await execFileAsync('kubectl', ['--kubeconfig', kubeconfigPath, '-n', namespace, 'wait', '--for=condition=Ready', `pod/${podName}`, '--timeout=180s'], { timeout: 210_000 });
|
||||
} finally {
|
||||
fs.rmSync(manifestPath, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureNamespaceOnCluster(coreApi: k8s.CoreV1Api, namespace: string): Promise<void> {
|
||||
try {
|
||||
await coreApi.readNamespace(namespace);
|
||||
} catch (error: any) {
|
||||
if (this.isK8sNotFound(error)) {
|
||||
await coreApi.createNamespace({ metadata: { name: namespace } });
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async upsertSecret(coreApi: k8s.CoreV1Api, namespace: string, secret: k8s.V1Secret): Promise<void> {
|
||||
secret.metadata = { ...(secret.metadata || {}), namespace };
|
||||
try {
|
||||
await coreApi.replaceNamespacedSecret(secret.metadata.name!, namespace, secret);
|
||||
} catch (error: any) {
|
||||
if (this.isK8sNotFound(error)) {
|
||||
await coreApi.createNamespacedSecret(namespace, secret);
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async upsertConfigMap(coreApi: k8s.CoreV1Api, namespace: string, configMap: k8s.V1ConfigMap): Promise<void> {
|
||||
configMap.metadata = { ...(configMap.metadata || {}), namespace };
|
||||
try {
|
||||
await coreApi.replaceNamespacedConfigMap(configMap.metadata.name!, namespace, configMap);
|
||||
} catch (error: any) {
|
||||
if (this.isK8sNotFound(error)) {
|
||||
await coreApi.createNamespacedConfigMap(namespace, configMap);
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async upsertPvcDefinition(
|
||||
coreApi: k8s.CoreV1Api,
|
||||
namespace: string,
|
||||
pvc: k8s.V1PersistentVolumeClaim,
|
||||
): Promise<void> {
|
||||
pvc.metadata = { ...(pvc.metadata || {}), namespace };
|
||||
try {
|
||||
await coreApi.readNamespacedPersistentVolumeClaim(pvc.metadata.name!, namespace);
|
||||
} catch (error: any) {
|
||||
if (this.isK8sNotFound(error)) {
|
||||
await coreApi.createNamespacedPersistentVolumeClaim(namespace, pvc);
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private cleanK8sObject<T extends { metadata?: k8s.V1ObjectMeta }>(obj: T): T {
|
||||
const metadata = { ...(obj.metadata || {}) };
|
||||
delete metadata.uid;
|
||||
delete metadata.resourceVersion;
|
||||
delete metadata.generation;
|
||||
delete metadata.creationTimestamp;
|
||||
delete metadata.managedFields;
|
||||
delete metadata.selfLink;
|
||||
return {
|
||||
...obj,
|
||||
metadata,
|
||||
};
|
||||
}
|
||||
|
||||
private cleanPvcForMigration(pvc: k8s.V1PersistentVolumeClaim): k8s.V1PersistentVolumeClaim {
|
||||
const cleaned = this.cleanK8sObject(pvc);
|
||||
return {
|
||||
apiVersion: cleaned.apiVersion,
|
||||
kind: cleaned.kind,
|
||||
metadata: cleaned.metadata,
|
||||
spec: {
|
||||
accessModes: cleaned.spec?.accessModes,
|
||||
resources: cleaned.spec?.resources,
|
||||
storageClassName: cleaned.spec?.storageClassName,
|
||||
volumeMode: cleaned.spec?.volumeMode,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private isK8sNotFound(error: any): boolean {
|
||||
return error?.statusCode === 404 || error?.body?.code === 404;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for the database pod to become Ready.
|
||||
* Polls pod status with label selector `app=<appName>-db`.
|
||||
|
||||
Reference in New Issue
Block a user