Serve preview URLs over Traefik+TLS, stabilize them, and speed up builds.

Ingress / preview URLs:
- Default the app Ingress class and ACME HTTP-01 solver to Traefik
  (k3s default) via a new INGRESS_CLASS env, instead of hardcoding nginx —
  fixes 404s on clusters without ingress-nginx.
- Only put public, real-TLD hosts (custom domain + preview) in the TLS
  block; the internal *.apps.cloudhost.local host no longer poisons the
  Let's Encrypt order, so certs actually issue.
- Make the per-app preview number stable across redeploys so URLs stop
  breaking, and let PREVIEW_BASE_DOMAIN configure the base domain.

Registry pulls:
- Point the k3s registries.yaml mirror endpoint at the registry NodePort on
  loopback so node containerd never depends on cluster DNS (image pulls
  survive node restarts).

Builds:
- Pin the Kaniko image, use IfNotPresent pull policy, drop the dead build
  queue/processor, and retry transient Kubernetes API errors while polling
  build jobs.

Logs & apps list:
- fluent-bit reads log files from head so startup output reaches
  Elasticsearch.
- Order joined deployments newest-first so the apps list shows the latest
  deployment status.

Allocation:
- Reserve in-flight (pending/building) capacity and stop globally degrading
  the cluster on a single allocation failure, so concurrent deploys don't
  starve or wrongly report "no healthy cluster".

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
keyhan
2026-06-02 15:44:08 +03:30
parent 786689e0fd
commit 4301277b48
13 changed files with 244 additions and 109 deletions
+25 -10
View File
@@ -289,7 +289,7 @@ export class KubernetesService implements OnModuleInit {
const namespacePrefix = app.userId.split('-')[0];
const previewHost =
previewNumber && !app.customDomain
? `${namespacePrefix}-${previewNumber}-preview.${previewRootDomain}`
? `${namespacePrefix}-${previewNumber}.${previewRootDomain}`
: '';
const pullRegistryUrl = this.registryService.getRegistryUrl();
const isWordPress = app.runtime === AppRuntime.WORDPRESS;
@@ -317,6 +317,7 @@ export class KubernetesService implements OnModuleInit {
envVars: this.resolveEnvVars(app),
ingress: {
enabled: true,
className: this.configService.get<string>('platform.ingressClass') || 'traefik',
subdomain: app.subdomain || app.name,
domain: domain,
clusterIssuer: 'letsencrypt-prod',
@@ -931,8 +932,10 @@ export class KubernetesService implements OnModuleInit {
// Main application container
const appContainer: any = {
name: ctx.appName,
// Image tags are unique per build (name:timestamp) and immutable, so
// IfNotPresent is correct and avoids re-pulling on every restart/scale-up.
image: ctx.image,
imagePullPolicy: 'Always',
imagePullPolicy: 'IfNotPresent',
ports: [{ containerPort: ctx.port }],
envFrom,
env: extraEnv,
@@ -1402,7 +1405,11 @@ export class KubernetesService implements OnModuleInit {
},
},
];
const tlsHosts = [host];
// Only PUBLIC, real-TLD hosts may go into the TLS block. The internal
// `${subdomain}.${domain}` host (e.g. *.apps.cloudhost.local) is not a valid
// public suffix — including it makes Let's Encrypt reject the whole order,
// which would also block the cert for the legitimate preview/custom domains.
const tlsHosts: string[] = [];
if (customDomain) {
rules.push({
@@ -1424,7 +1431,7 @@ export class KubernetesService implements OnModuleInit {
const namespacePrefix = ctx.ownerId.split('-')[0];
const previewHost =
previewNumber && !customDomain
? `${namespacePrefix}-${previewNumber}-preview.${previewRootDomain}`
? `${namespacePrefix}-${previewNumber}.${previewRootDomain}`
: '';
if (previewHost) {
rules.push({
@@ -1442,20 +1449,28 @@ export class KubernetesService implements OnModuleInit {
tlsHosts.push(previewHost);
}
const ingressClass =
this.configService.get<string>('platform.ingressClass') || 'traefik';
// Request a managed cert only when we actually have a public host to issue for.
const annotations: Record<string, string> = {};
const tls: k8s.V1IngressTLS[] = [];
if (tlsHosts.length > 0) {
annotations['cert-manager.io/cluster-issuer'] = 'letsencrypt-prod';
tls.push({ hosts: tlsHosts, secretName: `${ctx.appName}-tls` });
}
const ingress: k8s.V1Ingress = {
apiVersion: 'networking.k8s.io/v1',
kind: 'Ingress',
metadata: {
name: ctx.appName,
namespace: ctx.namespace,
annotations: {
'cert-manager.io/cluster-issuer': 'letsencrypt-prod',
},
annotations,
},
spec: {
ingressClassName: 'nginx',
ingressClassName: ingressClass,
rules,
tls: [{ hosts: tlsHosts, secretName: `${ctx.appName}-tls` }],
...(tls.length > 0 ? { tls } : {}),
},
};
@@ -2780,7 +2795,7 @@ export class KubernetesService implements OnModuleInit {
if (verifiedCustomDomain) {
ingressUrl = `https://${verifiedCustomDomain}`;
} else if (previewNumber) {
ingressUrl = `https://${namespacePrefix}-${previewNumber}-preview.${previewRootDomain}`;
ingressUrl = `https://${namespacePrefix}-${previewNumber}.${previewRootDomain}`;
}
return {