feat(build): revamp app build pipeline (queue, Nixpacks, MinIO, Trivy, registry GC)
Rework the application build/deploy pipeline for scalability, reproducibility, and security: - Build queue: deploys run through a bounded-concurrency Bull queue (BUILD_CONCURRENCY, default 3) so concurrent user deploys can't flood the cluster with Kaniko jobs. Build state (progress / cancel / session) moves from in-memory Maps to Redis, so cancel + live logs work across backend replicas. - Nixpacks + BYO Dockerfile: code runtimes build via Nixpacks (or the user's own Dockerfile when present); the hand-written per-runtime Dockerfile generators and runtime auto-detection are removed. WordPress keeps its templated path. Build-time mirror env (NIXPACKS_BUILD_ENV) supports the Iran network. - Source upload to MinIO: archives stream to in-cluster MinIO; build pods pull via a presigned URL. Removes the PVC + helper pod + kubectl cp upload path. - Report-only Trivy scan after build; per-severity summary stored on the deployment and shown as a badge in the dashboard. Never gates a deploy. - Registry GC: a Redis-locked daily job keeps the newest N image tags per app (REGISTRY_KEEP_VERSIONS, default 3) and reclaims disk via garbage-collect. - Hardening: git tokens are delivered via a per-build Secret + git credential store instead of being embedded in the clone URL / Job manifest; build timeout is configurable. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
keyhan
@@ -1102,9 +1102,133 @@ export class ClustersService implements OnModuleInit, OnModuleDestroy {
|
||||
|
||||
await this.ensureK3sRegistryMirrors(appsApi, registryUrl);
|
||||
|
||||
// ── 8. MinIO (S3-compatible) for application source archives ──
|
||||
await this.ensureMinioInfrastructure(coreApi, appsApi, buildNs);
|
||||
|
||||
this.logger.log(`✅ Cluster bootstrap complete — registry: ${registryUrl}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Provision in-cluster MinIO (Deployment + PVC + Service + credentials Secret)
|
||||
* in the build namespace. Application source archives are uploaded here by the
|
||||
* API and pulled by build pods via presigned URLs.
|
||||
*/
|
||||
private async ensureMinioInfrastructure(coreApi: k8s.CoreV1Api, appsApi: k8s.AppsV1Api, buildNs: string): Promise<void> {
|
||||
const accessKey = this.configService.get<string>('minio.accessKey') || 'cloudhost';
|
||||
const secretKey = this.configService.get<string>('minio.secretKey') || '';
|
||||
const pvcName = 'minio-data';
|
||||
const deployName = 'minio';
|
||||
const svcName = 'minio';
|
||||
const secretName = 'minio-credentials';
|
||||
|
||||
// 1. Credentials Secret (shared by the MinIO server env and the API client)
|
||||
const minioSecret: k8s.V1Secret = {
|
||||
metadata: { name: secretName, namespace: buildNs },
|
||||
type: 'Opaque',
|
||||
data: {
|
||||
accesskey: Buffer.from(accessKey).toString('base64'),
|
||||
secretkey: Buffer.from(secretKey).toString('base64'),
|
||||
},
|
||||
};
|
||||
try {
|
||||
await coreApi.readNamespacedSecret({ name: secretName, namespace: buildNs });
|
||||
await coreApi.replaceNamespacedSecret({ name: secretName, namespace: buildNs, body: minioSecret });
|
||||
} catch (err: any) {
|
||||
if (err.code === 404 || err.body?.code === 404) {
|
||||
await coreApi.createNamespacedSecret({ namespace: buildNs, body: minioSecret });
|
||||
this.logger.log(`Created ${secretName} Secret`);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Data PVC
|
||||
try {
|
||||
await coreApi.readNamespacedPersistentVolumeClaim({ name: pvcName, namespace: buildNs });
|
||||
} catch (err: any) {
|
||||
if (err.code === 404 || err.body?.code === 404) {
|
||||
await coreApi.createNamespacedPersistentVolumeClaim({
|
||||
namespace: buildNs,
|
||||
body: {
|
||||
metadata: { name: pvcName, namespace: buildNs },
|
||||
spec: { accessModes: ['ReadWriteOnce'], resources: { requests: { storage: '20Gi' } } },
|
||||
},
|
||||
});
|
||||
this.logger.log(`Created PVC "${pvcName}" (20Gi)`);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Deployment
|
||||
try {
|
||||
await appsApi.readNamespacedDeployment({ name: deployName, namespace: buildNs });
|
||||
} catch (err: any) {
|
||||
if (err.code === 404 || err.body?.code === 404) {
|
||||
await appsApi.createNamespacedDeployment({
|
||||
namespace: buildNs,
|
||||
body: {
|
||||
metadata: { name: deployName, namespace: buildNs, labels: { app: 'minio' } },
|
||||
spec: {
|
||||
replicas: 1,
|
||||
selector: { matchLabels: { app: 'minio' } },
|
||||
template: {
|
||||
metadata: { labels: { app: 'minio' } },
|
||||
spec: {
|
||||
containers: [
|
||||
{
|
||||
name: 'minio',
|
||||
image: process.env.MINIO_IMAGE || 'minio/minio:latest',
|
||||
args: ['server', '/data', '--console-address', ':9001'],
|
||||
env: [
|
||||
{ name: 'MINIO_ROOT_USER', valueFrom: { secretKeyRef: { name: secretName, key: 'accesskey' } } },
|
||||
{ name: 'MINIO_ROOT_PASSWORD', valueFrom: { secretKeyRef: { name: secretName, key: 'secretkey' } } },
|
||||
],
|
||||
ports: [{ containerPort: 9000 }, { containerPort: 9001 }],
|
||||
volumeMounts: [{ name: 'data', mountPath: '/data' }],
|
||||
resources: {
|
||||
requests: { cpu: '100m', memory: '256Mi' },
|
||||
limits: { cpu: '1', memory: '1Gi' },
|
||||
},
|
||||
},
|
||||
],
|
||||
volumes: [{ name: 'data', persistentVolumeClaim: { claimName: pvcName } }],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
this.logger.log(`Created MinIO Deployment`);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// 4. ClusterIP Service (API :9000, console :9001)
|
||||
try {
|
||||
await coreApi.readNamespacedService({ name: svcName, namespace: buildNs });
|
||||
} catch (err: any) {
|
||||
if (err.code === 404 || err.body?.code === 404) {
|
||||
await coreApi.createNamespacedService({
|
||||
namespace: buildNs,
|
||||
body: {
|
||||
metadata: { name: svcName, namespace: buildNs, labels: { app: 'minio' } },
|
||||
spec: {
|
||||
selector: { app: 'minio' },
|
||||
ports: [
|
||||
{ name: 'api', port: 9000, targetPort: 9000 as any, protocol: 'TCP' },
|
||||
{ name: 'console', port: 9001, targetPort: 9001 as any, protocol: 'TCP' },
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
this.logger.log(`Created MinIO Service`);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** In-cluster registry mirror for k3s/containerd (HTTP). Removes legacy external-registry DaemonSet if present. */
|
||||
private async ensureK3sRegistryMirrors(appsApi: k8s.AppsV1Api, registryUrl: string): Promise<void> {
|
||||
const namespace = 'kube-system';
|
||||
|
||||
Reference in New Issue
Block a user