From 0dee13337749773a61c703362cf7fba609a8e779 Mon Sep 17 00:00:00 2001 From: Infrastructure Admin Date: Tue, 7 Apr 2026 21:23:12 -0400 Subject: [PATCH] Add Argo Workflows, mTLS container registry, and fix infrastructure - Move Keycloak off Helm to plain Crossplane Object manifests (PostgreSQL + Keycloak deployment) - Add Vaultwarden SSO/OIDC config with Keycloak, fix Recreate deployment strategy for RWO volumes - Switch routing from Helm-based Pomerium to pomerium-allinone with all service routes - Deploy Argo Workflows (controller, server, CRDs, RBAC) with KEDA queue-depth autoscaling - Add Civo cluster autoscaler with pool-scaler for zero-to-one scale-up via Civo API - Add node-labeler to auto-tag nodes by pool membership for nodeSelector scheduling - Set up mTLS container registry at registry.nge6.com (Forgejo built-in, client cert required) - Add internal registry route (registry-internal.nge6.com) for in-cluster image pulls - Fix DNS records for new Emissary LB IP (212.2.241.28) - Fix CoreDNS crash from invalid custom config - Fix Emissary apiext expired webhook CA certificate Co-Authored-By: Claude Opus 4.6 (1M context) --- ambassador-listeners.yaml | 46 +++ argo-workflows/argo-ingress.yaml | 70 +++++ argo-workflows/argo-workflows.yaml | 440 ++++++++++++++++++++++++++++ argo-workflows/keda-autoscaler.yaml | 78 +++++ auth/keycloak-resources.yaml | 29 +- auth/keycloak.yaml | 303 ++++++++++++++----- cluster-autoscaler.yaml | 73 +++++ forgejo-k8s.yaml | 6 +- keycloak-auth-dns.yaml | 70 +++++ keycloak-dns-fix.yaml | 47 +++ keycloak-nge6-dns.yaml | 70 +++++ kustomization.yaml | 8 +- node-labeler.yaml | 127 ++++++++ pomerium-allinone.yaml | 181 ++++++++++++ pomerium-dns.yaml | 71 +++++ pomerium-native.yaml | 397 +++++++++++++++++++++++++ pomerium.yaml | 5 + pool-scaler.yaml | 91 ++++++ registry-internal.yaml | 92 ++++++ registry.yaml | 97 ++++++ vaultwarden.yaml | 18 +- 21 files changed, 2244 insertions(+), 75 deletions(-) create mode 100644 ambassador-listeners.yaml create mode 100644 argo-workflows/argo-ingress.yaml create mode 100644 argo-workflows/argo-workflows.yaml create mode 100644 argo-workflows/keda-autoscaler.yaml create mode 100644 cluster-autoscaler.yaml create mode 100644 keycloak-auth-dns.yaml create mode 100644 keycloak-dns-fix.yaml create mode 100644 keycloak-nge6-dns.yaml create mode 100644 node-labeler.yaml create mode 100644 pomerium-allinone.yaml create mode 100644 pomerium-dns.yaml create mode 100644 pomerium-native.yaml create mode 100644 pool-scaler.yaml create mode 100644 registry-internal.yaml create mode 100644 registry.yaml diff --git a/ambassador-listeners.yaml b/ambassador-listeners.yaml new file mode 100644 index 0000000..377738b --- /dev/null +++ b/ambassador-listeners.yaml @@ -0,0 +1,46 @@ +# Ambassador Listeners - required for Ambassador 3.x +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: http-listener + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: getambassador.io/v3alpha1 + kind: Listener + metadata: + name: http-listener + namespace: emissary + spec: + port: 8080 + protocol: HTTP + securityModel: XFP + hostBinding: + namespace: + from: ALL +--- +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: https-listener + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: getambassador.io/v3alpha1 + kind: Listener + metadata: + name: https-listener + namespace: emissary + spec: + port: 8443 + protocol: HTTPS + securityModel: XFP + hostBinding: + namespace: + from: ALL \ No newline at end of file diff --git a/argo-workflows/argo-ingress.yaml b/argo-workflows/argo-ingress.yaml new file mode 100644 index 0000000..57ab155 --- /dev/null +++ b/argo-workflows/argo-ingress.yaml @@ -0,0 +1,70 @@ +# SSL Certificate for Argo Workflows UI +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-certificate + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: argo-tls + namespace: emissary + spec: + secretName: argo-tls + issuerRef: + name: letsencrypt-dns + kind: ClusterIssuer + dnsNames: + - workflows.nge6.com +--- +# Ambassador Host for Argo UI +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-host + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: getambassador.io/v3alpha1 + kind: Host + metadata: + name: argo-host + namespace: emissary + annotations: + external-dns.ambassador-service: emissary-ingress.emissary.svc.cluster.local + external-dns.alpha.kubernetes.io/target: 212.2.241.28 + spec: + hostname: workflows.nge6.com + tlsSecret: + name: argo-tls +--- +# Ambassador Mapping for Argo UI through Pomerium +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-mapping + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: getambassador.io/v3alpha1 + kind: Mapping + metadata: + name: argo-mapping + namespace: emissary + spec: + hostname: workflows.nge6.com + prefix: / + service: http://pomerium-allinone.pomerium:443 + timeout_ms: 30000 + connect_timeout_ms: 10000 diff --git a/argo-workflows/argo-workflows.yaml b/argo-workflows/argo-workflows.yaml new file mode 100644 index 0000000..613d85e --- /dev/null +++ b/argo-workflows/argo-workflows.yaml @@ -0,0 +1,440 @@ +# Argo Workflows namespace +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-namespace + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: v1 + kind: Namespace + metadata: + name: argo +--- +# Argo Workflows ServiceAccount +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-sa + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: v1 + kind: ServiceAccount + metadata: + name: argo + namespace: argo +--- +# Argo Server ServiceAccount +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-server-sa + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: v1 + kind: ServiceAccount + metadata: + name: argo-server + namespace: argo +--- +# Argo Workflows ClusterRole +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-cluster-role + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: argo-cluster-role + rules: + - apiGroups: [""] + resources: [pods, pods/exec, pods/log] + verbs: [create, get, list, watch, update, patch, delete] + - apiGroups: [""] + resources: [configmaps, secrets, services, serviceaccounts, persistentvolumeclaims, events] + verbs: [create, get, list, watch, update, patch, delete] + - apiGroups: [argoproj.io] + resources: [workflows, workflows/finalizers, workflowtemplates, workflowtemplates/finalizers, clusterworkflowtemplates, clusterworkflowtemplates/finalizers, cronworkflows, cronworkflows/finalizers, workfloweventbindings, workfloweventbindings/finalizers, workflowtaskresults, workflowartifactgctasks] + verbs: [create, get, list, watch, update, patch, delete] + - apiGroups: [argoproj.io] + resources: [workflowtasksets, workflowtasksets/status] + verbs: [create, get, list, watch, update, patch, delete] + - apiGroups: [""] + resources: [events] + verbs: [create, patch] + - apiGroups: [coordination.k8s.io] + resources: [leases] + verbs: [create, get, update] +--- +# Argo Server ClusterRole +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-server-cluster-role + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: argo-server-cluster-role + rules: + - apiGroups: [""] + resources: [configmaps, events] + verbs: [get, watch, list] + - apiGroups: [""] + resources: [pods, pods/exec, pods/log] + verbs: [get, list, watch] + - apiGroups: [""] + resources: [secrets] + verbs: [get, list, watch, create] + - apiGroups: [""] + resources: [events] + verbs: [watch, create, patch] + - apiGroups: [argoproj.io] + resources: [workflows, workflowtemplates, clusterworkflowtemplates, cronworkflows, workfloweventbindings] + verbs: [create, get, list, watch, update, patch, delete] + - apiGroups: [argoproj.io] + resources: [workflowtasksets] + verbs: [list, watch] +--- +# Argo ClusterRoleBinding +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-cluster-role-binding + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: argo-binding + subjects: + - kind: ServiceAccount + name: argo + namespace: argo + roleRef: + kind: ClusterRole + name: argo-cluster-role + apiGroup: rbac.authorization.k8s.io +--- +# Argo Server ClusterRoleBinding +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-server-cluster-role-binding + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: argo-server-binding + subjects: + - kind: ServiceAccount + name: argo-server + namespace: argo + roleRef: + kind: ClusterRole + name: argo-server-cluster-role + apiGroup: rbac.authorization.k8s.io +--- +# Default workflow ServiceAccount (used by workflows themselves) +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-workflow-sa + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: v1 + kind: ServiceAccount + metadata: + name: argo-workflow + namespace: argo +--- +# Workflow role - what workflows can do +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-workflow-role + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: argo-workflow-role + namespace: argo + rules: + - apiGroups: [""] + resources: [pods] + verbs: [get, watch, patch] + - apiGroups: [""] + resources: [pods/log] + verbs: [get, watch] + - apiGroups: [argoproj.io] + resources: [workflowtaskresults] + verbs: [create, patch] +--- +# Workflow RoleBinding +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-workflow-role-binding + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: argo-workflow-binding + namespace: argo + subjects: + - kind: ServiceAccount + name: argo-workflow + namespace: argo + roleRef: + kind: Role + name: argo-workflow-role + apiGroup: rbac.authorization.k8s.io +--- +# Workflow Controller ConfigMap +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-workflow-controller-configmap + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: v1 + kind: ConfigMap + metadata: + name: workflow-controller-configmap + namespace: argo + data: + config: | + workflowDefaults: + spec: + serviceAccountName: argo-workflow + imagePullSecrets: + - name: forgejo-registry + nodeSelector: + kubernetes.civo.com/node-pool: high-compute + tolerations: + - key: "kubernetes.civo.com/node-pool" + operator: "Equal" + value: "high-compute" + effect: "NoSchedule" + metricsConfig: + enabled: true + path: /metrics + port: 9090 +--- +# Workflow Controller Deployment +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-workflow-controller + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: workflow-controller + namespace: argo + spec: + replicas: 1 + selector: + matchLabels: + app: workflow-controller + template: + metadata: + labels: + app: workflow-controller + spec: + serviceAccountName: argo + containers: + - name: workflow-controller + image: quay.io/argoproj/workflow-controller:v3.6.7 + args: + - --configmap + - workflow-controller-configmap + - --executor-image + - quay.io/argoproj/argoexec:v3.6.7 + - --loglevel + - info + env: + - name: LEADER_ELECTION_IDENTITY + valueFrom: + fieldRef: + fieldPath: metadata.name + ports: + - containerPort: 9090 + name: metrics + - containerPort: 6060 + name: pprof + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + readinessProbe: + httpGet: + path: /healthz + port: 6060 + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: 6060 + initialDelaySeconds: 30 + periodSeconds: 30 +--- +# Workflow Controller Metrics Service +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-workflow-controller-metrics + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: v1 + kind: Service + metadata: + name: workflow-controller-metrics + namespace: argo + spec: + selector: + app: workflow-controller + ports: + - name: metrics + port: 9090 + targetPort: 9090 + type: ClusterIP +--- +# Argo Server Deployment +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-server-deployment + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: argo-server + namespace: argo + spec: + replicas: 1 + selector: + matchLabels: + app: argo-server + template: + metadata: + labels: + app: argo-server + spec: + serviceAccountName: argo-server + containers: + - name: argo-server + image: quay.io/argoproj/argocli:v3.6.7 + args: + - server + - --auth-mode=server + - --secure=false + ports: + - containerPort: 2746 + name: web + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + readinessProbe: + httpGet: + path: / + port: 2746 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 +--- +# Argo Server Service +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-server-service + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: v1 + kind: Service + metadata: + name: argo-server + namespace: argo + spec: + selector: + app: argo-server + ports: + - name: web + port: 2746 + targetPort: 2746 + type: ClusterIP diff --git a/argo-workflows/keda-autoscaler.yaml b/argo-workflows/keda-autoscaler.yaml new file mode 100644 index 0000000..d613370 --- /dev/null +++ b/argo-workflows/keda-autoscaler.yaml @@ -0,0 +1,78 @@ +# Placeholder deployment that KEDA scales based on Argo queue depth. +# When scaled up, these pods request resources on the high-compute pool, +# triggering the Civo cluster autoscaler to add nodes. +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-queue-placeholder + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: argo-queue-placeholder + namespace: argo + spec: + replicas: 0 + selector: + matchLabels: + app: argo-queue-placeholder + template: + metadata: + labels: + app: argo-queue-placeholder + spec: + nodeSelector: + kubernetes.civo.com/node-pool: high-compute + tolerations: + - key: "kubernetes.civo.com/node-pool" + operator: "Equal" + value: "high-compute" + effect: "NoSchedule" + terminationGracePeriodSeconds: 0 + containers: + - name: placeholder + image: busybox + command: ["sleep", "infinity"] + resources: + requests: + cpu: "1" + memory: 1Gi +--- +# KEDA ScaledObject - scales placeholder based on pending Argo workflow pods. +# When workflows are submitted, their pods land in Pending state (no nodes). +# KEDA sees the pending pods and scales up the placeholder deployment, +# which also targets high-compute nodes, adding pressure for the cluster +# autoscaler to provision new nodes. +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-queue-scaledobject + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: keda.sh/v1alpha1 + kind: ScaledObject + metadata: + name: argo-queue-scaler + namespace: argo + spec: + scaleTargetRef: + name: argo-queue-placeholder + pollingInterval: 15 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 5 + triggers: + - type: kubernetes-workload + metadata: + podSelector: "workflows.argoproj.io/completed=false" + namespace: "argo" + value: "1" diff --git a/auth/keycloak-resources.yaml b/auth/keycloak-resources.yaml index 57cdfca..81ed3ea 100644 --- a/auth/keycloak-resources.yaml +++ b/auth/keycloak-resources.yaml @@ -40,6 +40,31 @@ spec: providerConfigRef: name: keycloak-provider --- +# Vaultwarden OIDC Client +apiVersion: openidclient.keycloak.crossplane.io/v1alpha1 +kind: Client +metadata: + name: vaultwarden-client +spec: + forProvider: + realmId: kubernetes-realm + clientId: vaultwarden + name: "Vaultwarden Password Manager" + description: "Client for Vaultwarden OIDC authentication" + enabled: true + accessType: CONFIDENTIAL + clientAuthenticatorType: client-secret + validRedirectUris: + - "https://vault.nge6.com/identity/connect/oidc-signin" + - "https://vault.nge6.com/sso-connector/oidc/callback" + standardFlowEnabled: true + directAccessGrantsEnabled: false + serviceAccountsEnabled: false + webOrigins: + - "https://vault.nge6.com" + providerConfigRef: + name: keycloak-provider +--- # Create user groups apiVersion: group.keycloak.crossplane.io/v1alpha1 kind: Group @@ -190,7 +215,7 @@ metadata: spec: forProvider: realmId: kubernetes-realm - groupId: k8s-admins + groupId: 98e13ab3-0001-4646-b097-ed52ee5baff4 members: ["admin", "eemoore"] providerConfigRef: name: keycloak-provider @@ -202,7 +227,7 @@ metadata: spec: forProvider: realmId: kubernetes-realm - groupId: users + groupId: f87d1c8e-32ee-4f63-9584-7fce67313137 members: ["admin", "eemoore"] providerConfigRef: name: keycloak-provider diff --git a/auth/keycloak.yaml b/auth/keycloak.yaml index c9fd9ff..a3c751e 100644 --- a/auth/keycloak.yaml +++ b/auth/keycloak.yaml @@ -97,80 +97,246 @@ spec: stringData: password: "thefi9paechooh" --- -# Keycloak Helm release -apiVersion: helm.crossplane.io/v1beta1 -kind: Release +# PostgreSQL credentials +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object metadata: - name: keycloak + name: keycloak-postgresql-secret namespace: crossplane-system spec: providerConfigRef: - name: helm-provider + name: kubernetes-provider forProvider: - chart: - name: keycloak - repository: https://codecentric.github.io/helm-charts - version: 18.10.0 - namespace: auth-system - values: - image: - repository: quay.io/keycloak/keycloak - tag: 24.0.4 - serviceAccount: - create: false + manifest: + apiVersion: v1 + kind: Secret + metadata: + name: keycloak-postgresql + namespace: auth-system + type: Opaque + stringData: + postgresql-password: "keycloak-db-password" + POSTGRES_PASSWORD: "keycloak-db-password" + POSTGRES_USER: "keycloak" + POSTGRES_DB: "keycloak" +--- +# PostgreSQL StatefulSet +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: keycloak-postgresql-statefulset + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: keycloak-postgresql + namespace: auth-system + spec: + serviceName: keycloak-postgresql + replicas: 1 + selector: + matchLabels: + app: keycloak-postgresql + template: + metadata: + labels: + app: keycloak-postgresql + spec: + containers: + - name: postgresql + image: postgres:17-bookworm + ports: + - containerPort: 5432 + name: postgresql + envFrom: + - secretRef: + name: keycloak-postgresql + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + subPath: pgdata + readinessProbe: + exec: + command: + - pg_isready + - -U + - keycloak + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + exec: + command: + - pg_isready + - -U + - keycloak + initialDelaySeconds: 30 + periodSeconds: 30 + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + storageClassName: civo-volume + resources: + requests: + storage: 5Gi +--- +# PostgreSQL Service +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: keycloak-postgresql-service + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: v1 + kind: Service + metadata: + name: keycloak-postgresql + namespace: auth-system + spec: + selector: + app: keycloak-postgresql + ports: + - name: postgresql + port: 5432 + targetPort: 5432 + type: ClusterIP +--- +# Keycloak Deployment +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: keycloak-deployment + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: apps/v1 + kind: Deployment + metadata: name: keycloak - args: - - start - - --db=postgres - - --hostname-strict=false - - --hostname-strict-https=false - - --proxy=edge - - --http-enabled=true - livenessProbe: | - httpGet: - path: /realms/master - port: http - initialDelaySeconds: 120 - timeoutSeconds: 5 - periodSeconds: 30 - failureThreshold: 10 - readinessProbe: | - httpGet: - path: /realms/master - port: http - initialDelaySeconds: 90 - timeoutSeconds: 3 - periodSeconds: 10 - failureThreshold: 10 - startupProbe: | - httpGet: - path: /realms/master - port: http - initialDelaySeconds: 60 - timeoutSeconds: 3 - periodSeconds: 5 - failureThreshold: 30 - extraEnv: | - - name: KEYCLOAK_ADMIN - value: admin - - name: KEYCLOAK_ADMIN_PASSWORD - valueFrom: - secretKeyRef: - name: keycloak-admin-creds - key: password - - name: KC_DB - value: postgres - - name: KC_DB_URL - value: jdbc:postgresql://keycloak-postgresql:5432/keycloak - - name: KC_DB_USERNAME - value: keycloak - - name: KC_DB_PASSWORD - valueFrom: - secretKeyRef: - name: keycloak-postgresql - key: postgresql-password - ingress: - enabled: false + namespace: auth-system + labels: + app: keycloak + spec: + replicas: 1 + selector: + matchLabels: + app: keycloak + template: + metadata: + labels: + app: keycloak + spec: + serviceAccountName: keycloak + containers: + - name: keycloak + image: quay.io/keycloak/keycloak:24.0.4 + args: + - start + - --db=postgres + - --hostname=auth.nge6.com + - --hostname-strict=false + - --hostname-strict-https=false + - --proxy=edge + - --http-enabled=true + ports: + - containerPort: 8080 + name: http + env: + - name: KEYCLOAK_ADMIN + value: admin + - name: KEYCLOAK_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: keycloak-admin-creds + key: password + - name: KC_DB + value: postgres + - name: KC_DB_URL + value: jdbc:postgresql://keycloak-postgresql:5432/keycloak + - name: KC_DB_USERNAME + value: keycloak + - name: KC_DB_PASSWORD + valueFrom: + secretKeyRef: + name: keycloak-postgresql + key: postgresql-password + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + readinessProbe: + httpGet: + path: /realms/master + port: 8080 + initialDelaySeconds: 90 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 10 + livenessProbe: + httpGet: + path: /realms/master + port: 8080 + initialDelaySeconds: 120 + timeoutSeconds: 5 + periodSeconds: 30 + failureThreshold: 10 + startupProbe: + httpGet: + path: /realms/master + port: 8080 + initialDelaySeconds: 60 + timeoutSeconds: 3 + periodSeconds: 5 + failureThreshold: 30 +--- +# Keycloak Service +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: keycloak-service + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: v1 + kind: Service + metadata: + name: keycloak-http + namespace: auth-system + spec: + selector: + app: keycloak + ports: + - name: http + port: 80 + targetPort: 8080 + type: ClusterIP --- # Keycloak SSL Certificate apiVersion: kubernetes.crossplane.io/v1alpha2 @@ -214,6 +380,7 @@ spec: namespace: emissary annotations: external-dns.ambassador-service: emissary-ingress.emissary.svc.cluster.local + external-dns.alpha.kubernetes.io/target: 212.2.241.28 spec: hostname: auth.nge6.com tlsSecret: diff --git a/cluster-autoscaler.yaml b/cluster-autoscaler.yaml new file mode 100644 index 0000000..c38763e --- /dev/null +++ b/cluster-autoscaler.yaml @@ -0,0 +1,73 @@ +# Cluster autoscaler - Crossplane-managed to prevent marketplace overwriting config +# Main pool (fc94): fixed at 3 nodes +# High-compute pool (cc28): scales 0-5 based on demand +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: cluster-autoscaler-deployment + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + app: cluster-autoscaler + spec: + replicas: 1 + selector: + matchLabels: + app: cluster-autoscaler + template: + metadata: + labels: + app: cluster-autoscaler + spec: + serviceAccountName: cluster-autoscaler + containers: + - name: cluster-autoscaler + image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.28.1 + command: + - ./cluster-autoscaler + - --v=4 + - --stderrthreshold=info + - --cloud-provider=civo + - --nodes=3:3:1b886eac-942e-40bf-8f70-7a5496f2fd3b + - --nodes=0:1:high-compute + - --skip-nodes-with-local-storage=false + - --skip-nodes-with-system-pods=false + - --scale-down-unneeded-time=5m + - --scale-down-delay-after-add=5m + env: + - name: CIVO_API_URL + valueFrom: + secretKeyRef: + key: api-url + name: civo-api-access + - name: CIVO_API_KEY + valueFrom: + secretKeyRef: + key: api-key + name: civo-api-access + - name: CIVO_CLUSTER_ID + valueFrom: + secretKeyRef: + key: cluster-id + name: civo-api-access + - name: CIVO_REGION + valueFrom: + secretKeyRef: + key: region + name: civo-api-access + resources: + requests: + cpu: 100m + memory: 300Mi + limits: + cpu: 100m + memory: 300Mi diff --git a/forgejo-k8s.yaml b/forgejo-k8s.yaml index 6c7cc70..da4f3b8 100644 --- a/forgejo-k8s.yaml +++ b/forgejo-k8s.yaml @@ -47,6 +47,10 @@ spec: LFS_START_SERVER = true OFFLINE_MODE = false + [packages] + ENABLED = true + CONTAINER_REGISTRY_TOKEN_REALM = https://registry.nge6.com + [database] DB_TYPE = sqlite3 PATH = /data/gitea/gitea.db @@ -305,6 +309,6 @@ spec: spec: hostname: git.nge6.com prefix: / - service: https://pomerium-proxy.pomerium:443 + service: http://pomerium-allinone.pomerium:443 timeout_ms: 30000 connect_timeout_ms: 10000 \ No newline at end of file diff --git a/keycloak-auth-dns.yaml b/keycloak-auth-dns.yaml new file mode 100644 index 0000000..073875f --- /dev/null +++ b/keycloak-auth-dns.yaml @@ -0,0 +1,70 @@ +# SSL Certificate for auth.nge6.com (Keycloak alternative) +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: keycloak-auth-certificate + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: keycloak-auth-tls + namespace: emissary + spec: + secretName: keycloak-auth-tls + issuerRef: + name: letsencrypt-dns + kind: ClusterIssuer + dnsNames: + - auth.nge6.com +--- +# Ambassador Host for auth.nge6.com +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: keycloak-auth-host + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: getambassador.io/v3alpha1 + kind: Host + metadata: + name: keycloak-auth-host + namespace: emissary + annotations: + external-dns.ambassador-service: emissary-ingress.emissary.svc.cluster.local + external-dns.alpha.kubernetes.io/target: 212.2.241.28 + spec: + hostname: auth.nge6.com + tlsSecret: + name: keycloak-auth-tls +--- +# Ambassador Mapping for auth.nge6.com +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: keycloak-auth-mapping + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: getambassador.io/v3alpha1 + kind: Mapping + metadata: + name: keycloak-auth-mapping + namespace: emissary + spec: + hostname: auth.nge6.com + prefix: / + service: keycloak-http.auth-system:80 + timeout_ms: 30000 + connect_timeout_ms: 10000 \ No newline at end of file diff --git a/keycloak-dns-fix.yaml b/keycloak-dns-fix.yaml new file mode 100644 index 0000000..0d61794 --- /dev/null +++ b/keycloak-dns-fix.yaml @@ -0,0 +1,47 @@ +# Fix DNS resolution for keycloak.nge6.com from inside cluster +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: keycloak-internal-dns + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: v1 + kind: Service + metadata: + name: keycloak-nge6-com + namespace: pomerium + spec: + type: ExternalName + externalName: keycloak-http.auth-system.svc.cluster.local + ports: + - port: 80 + targetPort: 80 +--- +# Add custom hosts entry to CoreDNS +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: coredns-custom-hosts + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: v1 + kind: ConfigMap + metadata: + name: coredns-custom + namespace: kube-system + data: + keycloak.override: | + keycloak.nge6.com:53 { + hosts { + 212.2.241.28 keycloak.nge6.com + fallthrough + } + } \ No newline at end of file diff --git a/keycloak-nge6-dns.yaml b/keycloak-nge6-dns.yaml new file mode 100644 index 0000000..bcc2e9e --- /dev/null +++ b/keycloak-nge6-dns.yaml @@ -0,0 +1,70 @@ +# SSL Certificate for keycloak.nge6.com +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: keycloak-nge6-certificate + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: keycloak-nge6-tls + namespace: emissary + spec: + secretName: keycloak-nge6-tls + issuerRef: + name: letsencrypt-dns + kind: ClusterIssuer + dnsNames: + - keycloak.nge6.com +--- +# Ambassador Host for keycloak.nge6.com +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: keycloak-nge6-host + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: getambassador.io/v3alpha1 + kind: Host + metadata: + name: keycloak-nge6-host + namespace: emissary + annotations: + external-dns.ambassador-service: emissary-ingress.emissary.svc.cluster.local + external-dns.alpha.kubernetes.io/target: 212.2.241.28 + spec: + hostname: keycloak.nge6.com + tlsSecret: + name: keycloak-nge6-tls +--- +# Ambassador Mapping for keycloak.nge6.com +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: keycloak-nge6-mapping + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: getambassador.io/v3alpha1 + kind: Mapping + metadata: + name: keycloak-nge6-mapping + namespace: emissary + spec: + hostname: keycloak.nge6.com + prefix: / + service: keycloak-http.auth-system:80 + timeout_ms: 30000 + connect_timeout_ms: 10000 \ No newline at end of file diff --git a/kustomization.yaml b/kustomization.yaml index 00375fb..1604ee8 100644 --- a/kustomization.yaml +++ b/kustomization.yaml @@ -7,6 +7,7 @@ resources: - provider-configs.yaml - namespaces.yaml - external-dns.yaml +- ambassador-listeners.yaml # Certificate management - cert-manager/ @@ -20,8 +21,13 @@ resources: # Applications - forgejo-k8s.yaml -- pomerium.yaml +- pomerium-allinone.yaml +- pomerium-dns.yaml - vaultwarden.yaml +- keycloak-nge6-dns.yaml + +# Argo Workflows +- argo-workflows/ # Exclude problematic directories: # - flux/ (managed by Flux itself) diff --git a/node-labeler.yaml b/node-labeler.yaml new file mode 100644 index 0000000..24732f6 --- /dev/null +++ b/node-labeler.yaml @@ -0,0 +1,127 @@ +# Node labeler - automatically labels nodes by pool based on name pattern +# fc94 in node name -> main pool (1b886eac...) +# cc28 in node name -> high-compute pool +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: node-labeler-sa + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: v1 + kind: ServiceAccount + metadata: + name: node-labeler + namespace: kube-system +--- +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: node-labeler-clusterrole + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: node-labeler + rules: + - apiGroups: [""] + resources: [nodes] + verbs: [get, list, patch] + - apiGroups: [""] + resources: [pods] + verbs: [get, list, watch] +--- +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: node-labeler-clusterrolebinding + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: node-labeler + subjects: + - kind: ServiceAccount + name: node-labeler + namespace: kube-system + roleRef: + kind: ClusterRole + name: node-labeler + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: node-labeler-deployment + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: node-labeler + namespace: kube-system + spec: + replicas: 1 + selector: + matchLabels: + app: node-labeler + template: + metadata: + labels: + app: node-labeler + spec: + serviceAccountName: node-labeler + containers: + - name: labeler + image: bitnami/kubectl:latest + command: + - /bin/bash + - -c + - | + while true; do + for NODE in $(kubectl get nodes -o jsonpath='{.items[*].metadata.name}'); do + # Skip if already labeled + POOL=$(kubectl get node "$NODE" -o jsonpath='{.metadata.labels.kubernetes\.civo\.com/node-pool}' 2>/dev/null) + if [ -n "$POOL" ]; then + continue + fi + + # Label based on name pattern + if echo "$NODE" | grep -q "fc94"; then + echo "Labeling $NODE as main pool" + kubectl label node "$NODE" \ + kubernetes.civo.com/node-pool=1b886eac-942e-40bf-8f70-7a5496f2fd3b \ + kubernetes.civo.com/node-size=g4s.kube.medium --overwrite + elif echo "$NODE" | grep -q "cc28"; then + echo "Labeling $NODE as high-compute pool" + kubectl label node "$NODE" \ + kubernetes.civo.com/node-pool=high-compute \ + kubernetes.civo.com/node-size=g4c.kube.small --overwrite + fi + done + sleep 30 + done + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 50m + memory: 64Mi diff --git a/pomerium-allinone.yaml b/pomerium-allinone.yaml new file mode 100644 index 0000000..f5b40e3 --- /dev/null +++ b/pomerium-allinone.yaml @@ -0,0 +1,181 @@ +# Pomerium All-In-One Deployment (single process, no Helm) + +# ConfigMap for Pomerium configuration +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: pomerium-allinone-config + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: v1 + kind: ConfigMap + metadata: + name: pomerium-allinone + namespace: pomerium + data: + config.yaml: | + # Core configuration + address: :443 + http_redirect_addr: :80 + + # Security keys (32 bytes base64) + shared_secret: 5Cz7gj71G5ujzH9HIc1XgwabUXCdJ3st9649gNlknrI= + cookie_secret: SXzBgU9L72OI+QCD9lEOxXcjApyE+4oIbetqtveNcjc= + + # Run in insecure mode (no TLS certs required) + insecure_server: true + + # Service URLs (internal) + authenticate_service_url: https://authenticate.nge6.com + + # Identity provider + idp_provider: oidc + idp_provider_url: https://auth.nge6.com/realms/kubernetes-realm + idp_client_id: pomerium + idp_client_secret: U3Elh0oZEazKRpHpIasgP8yovUGsvq5K + idp_scopes: + - openid + - profile + - email + + # Routes + routes: + # Keycloak admin + - from: https://keycloak.nge6.com + to: http://keycloak-http.auth-system.svc.cluster.local + preserve_host_header: true + allow_public_unauthenticated_access: true + + # Vaultwarden SSO/OAuth endpoints only (needed for authentication flow) + - from: https://vault.nge6.com + to: http://vaultwarden-http.vaultwarden.svc.cluster.local:8080 + prefix: /identity/connect + preserve_host_header: true + allow_public_unauthenticated_access: true + + - from: https://vault.nge6.com + to: http://vaultwarden-http.vaultwarden.svc.cluster.local:8080 + prefix: /identity/sso + preserve_host_header: true + allow_public_unauthenticated_access: true + + # Vaultwarden API endpoints (protected by Vaultwarden's own auth) + - from: https://vault.nge6.com + to: http://vaultwarden-http.vaultwarden.svc.cluster.local:8080 + prefix: /api + preserve_host_header: true + allow_public_unauthenticated_access: true + + # Vaultwarden web UI - requires Pomerium authentication + - from: https://vault.nge6.com + to: http://vaultwarden-http.vaultwarden.svc.cluster.local:8080 + preserve_host_header: true + allow_any_authenticated_user: true + + # Forgejo container registry token endpoint (Docker auth) + - from: https://git.nge6.com + to: http://forgejo-http.forgejo.svc.cluster.local:3000 + prefix: /v2/token + preserve_host_header: true + allow_public_unauthenticated_access: true + + # Forgejo Git - requires authentication + - from: https://git.nge6.com + to: http://forgejo-http.forgejo.svc.cluster.local:3000 + preserve_host_header: true + allow_any_authenticated_user: true + + # Argo Workflows UI - requires authentication + - from: https://workflows.nge6.com + to: http://argo-server.argo.svc.cluster.local:2746 + preserve_host_header: true + allow_any_authenticated_user: true + +--- +# Pomerium All-In-One Deployment +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: pomerium-allinone-deployment + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: pomerium-allinone + namespace: pomerium + spec: + replicas: 1 + selector: + matchLabels: + app: pomerium-allinone + template: + metadata: + labels: + app: pomerium-allinone + spec: + containers: + - name: pomerium + image: pomerium/pomerium:v0.25.0 + args: + - --config=/etc/pomerium/config.yaml + env: + # Run all services in one container + - name: SERVICES + value: all + - name: INSECURE_SERVER + value: "true" + ports: + - containerPort: 443 + name: https + - containerPort: 80 + name: http + volumeMounts: + - name: config + mountPath: /etc/pomerium + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 1000m + memory: 512Mi + volumes: + - name: config + configMap: + name: pomerium-allinone +--- +# Pomerium All-In-One Service +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: pomerium-allinone-service + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: v1 + kind: Service + metadata: + name: pomerium-allinone + namespace: pomerium + spec: + selector: + app: pomerium-allinone + ports: + - name: https + port: 443 + targetPort: 443 + - name: http + port: 80 + targetPort: 80 \ No newline at end of file diff --git a/pomerium-dns.yaml b/pomerium-dns.yaml new file mode 100644 index 0000000..3d3acca --- /dev/null +++ b/pomerium-dns.yaml @@ -0,0 +1,71 @@ +# DNS and SSL for Pomerium authenticate endpoint +# SSL Certificate for authenticate.nge6.com +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: pomerium-authenticate-certificate + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: pomerium-authenticate-tls + namespace: emissary + spec: + secretName: pomerium-authenticate-tls + issuerRef: + name: letsencrypt-dns + kind: ClusterIssuer + dnsNames: + - authenticate.nge6.com +--- +# Ambassador Host for authenticate.nge6.com +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: pomerium-authenticate-host + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: getambassador.io/v3alpha1 + kind: Host + metadata: + name: pomerium-authenticate-host + namespace: emissary + annotations: + external-dns.ambassador-service: emissary-ingress.emissary.svc.cluster.local + external-dns.alpha.kubernetes.io/target: 212.2.241.28 + spec: + hostname: authenticate.nge6.com + tlsSecret: + name: pomerium-authenticate-tls +--- +# Ambassador Mapping for authenticate.nge6.com +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: pomerium-authenticate-mapping + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: getambassador.io/v3alpha1 + kind: Mapping + metadata: + name: pomerium-authenticate-mapping + namespace: emissary + spec: + hostname: authenticate.nge6.com + prefix: / + service: http://pomerium-allinone.pomerium:443 + timeout_ms: 30000 + connect_timeout_ms: 10000 \ No newline at end of file diff --git a/pomerium-native.yaml b/pomerium-native.yaml new file mode 100644 index 0000000..05fff76 --- /dev/null +++ b/pomerium-native.yaml @@ -0,0 +1,397 @@ +# Pomerium Native Kubernetes Deployment (No Helm!) +# Namespace already exists from previous deployment + +# ConfigMap for Pomerium configuration +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: pomerium-config + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: v1 + kind: ConfigMap + metadata: + name: pomerium + namespace: pomerium + data: + config.yaml: | + # Core configuration + address: :443 + grpc_address: :5443 + + # Security keys + shared_secret: 5Cz7gj71G5ujzH9HIc1XgwabUXCdJ3st9649gNlknrI= + cookie_secret: SXzBgU9L72OI+QCD9lEOxXcjApyE+4oIbetqtveNcjc= + + # Service URLs + authenticate_service_url: https://authenticate.nge6.com + authorize_service_url: http://pomerium-authorize.pomerium.svc.cluster.local:5443 + databroker_service_url: http://pomerium-databroker.pomerium.svc.cluster.local:5443 + + # Run in insecure mode for internal cluster communication + insecure_server: true + + # Identity provider + idp_provider: oidc + idp_provider_url: https://keycloak.nge6.com/realms/kubernetes-realm + idp_client_id: pomerium + idp_client_secret: 3JFMh3DZDOYlNiSQ64abL0z0bw1WJt3x + idp_scopes: + - openid + - profile + - email + + # Routes + routes: + # Keycloak admin (public for initial setup) + - from: https://keycloak.nge6.com + to: http://keycloak-http.auth-system.svc.cluster.local + preserve_host_header: true + allow_public_unauthenticated_access: true + + # Vaultwarden - requires authentication + - from: https://vault.nge6.com + to: http://vaultwarden-http.vaultwarden.svc.cluster.local:8080 + preserve_host_header: true + allow_any_authenticated_user: true + + # Forgejo Git - requires authentication + - from: https://git.nge6.com + to: http://forgejo-http.forgejo.svc.cluster.local:3000 + preserve_host_header: true + allow_any_authenticated_user: true + + # Authentication endpoint + - from: https://authenticate.nge6.com + to: http://pomerium-authenticate.pomerium.svc.cluster.local + allow_public_unauthenticated_access: true +--- +# Pomerium Authenticate Deployment +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: pomerium-authenticate-deployment + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: pomerium-authenticate + namespace: pomerium + spec: + replicas: 1 + selector: + matchLabels: + app: pomerium-authenticate + template: + metadata: + labels: + app: pomerium-authenticate + spec: + containers: + - name: pomerium + image: pomerium/pomerium:v0.25.0 + args: + - --config=/etc/pomerium/config.yaml + env: + - name: SERVICES + value: authenticate + - name: INSECURE_SERVER + value: "true" + - name: ADDRESS + value: :80 + - name: GRPC_ADDRESS + value: :5443 + - name: GRPC_INSECURE + value: "true" + ports: + - containerPort: 80 + name: http + - containerPort: 5443 + name: grpc + volumeMounts: + - name: config + mountPath: /etc/pomerium + volumes: + - name: config + configMap: + name: pomerium +--- +# Pomerium Authenticate Service +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: pomerium-authenticate-service + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: v1 + kind: Service + metadata: + name: pomerium-authenticate + namespace: pomerium + spec: + selector: + app: pomerium-authenticate + ports: + - name: http + port: 80 + targetPort: 80 + - name: grpc + port: 5443 + targetPort: 5443 +--- +# Pomerium Authorize Deployment +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: pomerium-authorize-deployment + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: pomerium-authorize + namespace: pomerium + spec: + replicas: 1 + selector: + matchLabels: + app: pomerium-authorize + template: + metadata: + labels: + app: pomerium-authorize + spec: + containers: + - name: pomerium + image: pomerium/pomerium:v0.25.0 + args: + - --config=/etc/pomerium/config.yaml + env: + - name: SERVICES + value: authorize + - name: INSECURE_SERVER + value: "true" + - name: ADDRESS + value: :80 + - name: GRPC_ADDRESS + value: :5443 + - name: GRPC_INSECURE + value: "true" + ports: + - containerPort: 80 + name: http + - containerPort: 5443 + name: grpc + volumeMounts: + - name: config + mountPath: /etc/pomerium + volumes: + - name: config + configMap: + name: pomerium +--- +# Pomerium Authorize Service +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: pomerium-authorize-service + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: v1 + kind: Service + metadata: + name: pomerium-authorize + namespace: pomerium + spec: + selector: + app: pomerium-authorize + ports: + - name: http + port: 80 + targetPort: 80 + - name: grpc + port: 5443 + targetPort: 5443 +--- +# Pomerium Databroker Deployment +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: pomerium-databroker-deployment + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: pomerium-databroker + namespace: pomerium + spec: + replicas: 1 + selector: + matchLabels: + app: pomerium-databroker + template: + metadata: + labels: + app: pomerium-databroker + spec: + containers: + - name: pomerium + image: pomerium/pomerium:v0.25.0 + args: + - --config=/etc/pomerium/config.yaml + env: + - name: SERVICES + value: databroker + - name: INSECURE_SERVER + value: "true" + - name: ADDRESS + value: :80 + - name: GRPC_ADDRESS + value: :5443 + - name: GRPC_INSECURE + value: "true" + ports: + - containerPort: 80 + name: http + - containerPort: 5443 + name: grpc + volumeMounts: + - name: config + mountPath: /etc/pomerium + volumes: + - name: config + configMap: + name: pomerium +--- +# Pomerium Databroker Service +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: pomerium-databroker-service + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: v1 + kind: Service + metadata: + name: pomerium-databroker + namespace: pomerium + spec: + selector: + app: pomerium-databroker + ports: + - name: http + port: 80 + targetPort: 80 + - name: grpc + port: 5443 + targetPort: 5443 +--- +# Pomerium Proxy Deployment (the main ingress point) +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: pomerium-proxy-deployment + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: pomerium-proxy + namespace: pomerium + spec: + replicas: 1 + selector: + matchLabels: + app: pomerium-proxy + template: + metadata: + labels: + app: pomerium-proxy + spec: + containers: + - name: pomerium + image: pomerium/pomerium:v0.25.0 + args: + - --config=/etc/pomerium/config.yaml + env: + - name: SERVICES + value: proxy + - name: INSECURE_SERVER + value: "true" + - name: ADDRESS + value: :443 + - name: HTTP_REDIRECT_ADDR + value: :80 + ports: + - containerPort: 443 + name: https + - containerPort: 80 + name: http + volumeMounts: + - name: config + mountPath: /etc/pomerium + volumes: + - name: config + configMap: + name: pomerium +--- +# Pomerium Proxy Service +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: pomerium-proxy-service + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: v1 + kind: Service + metadata: + name: pomerium-proxy + namespace: pomerium + spec: + selector: + app: pomerium-proxy + ports: + - name: https + port: 443 + targetPort: 443 + - name: http + port: 80 + targetPort: 80 \ No newline at end of file diff --git a/pomerium.yaml b/pomerium.yaml index b1d5943..9e8afff 100644 --- a/pomerium.yaml +++ b/pomerium.yaml @@ -41,6 +41,11 @@ spec: to: http://forgejo-http.forgejo.svc.cluster.local:3000 preserve_host_header: true allow_any_authenticated_user: true + # Vaultwarden password manager - require authentication + - from: https://vault.nge6.com + to: http://vaultwarden-http.vaultwarden.svc.cluster.local:8080 + preserve_host_header: true + allow_any_authenticated_user: true # Authentication service configuration authenticate: diff --git a/pool-scaler.yaml b/pool-scaler.yaml new file mode 100644 index 0000000..b1c2e62 --- /dev/null +++ b/pool-scaler.yaml @@ -0,0 +1,91 @@ +# Pool scaler - watches for pending Argo workflow pods and scales the +# high-compute pool via the Civo API. Handles scale-up from zero since +# the cluster autoscaler can't do this without a node template. +# The cluster autoscaler still handles scale-down. +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: pool-scaler-deployment + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: pool-scaler + namespace: kube-system + spec: + replicas: 1 + selector: + matchLabels: + app: pool-scaler + template: + metadata: + labels: + app: pool-scaler + spec: + serviceAccountName: node-labeler + containers: + - name: scaler + image: bitnami/kubectl:latest + command: + - /bin/bash + - -c + - | + echo "Pool scaler started. Watching for pending workflow pods..." + while true; do + # Count pending pods with high-compute nodeSelector + PENDING=$(kubectl get pods -n argo --field-selector=status.phase=Pending \ + -o jsonpath='{.items[*].spec.nodeSelector}' 2>/dev/null \ + | tr '}' '\n' | grep -c 'high-compute' || true) + + if [ "$PENDING" -gt 0 ]; then + # Check if any high-compute nodes already exist + HC_NODES=$(kubectl get nodes -l kubernetes.civo.com/node-pool=high-compute --no-headers 2>/dev/null | wc -l) + + if [ "$HC_NODES" -eq 0 ]; then + echo "$(date): $PENDING pending workflow pods, no high-compute nodes. Scaling pool to 1..." + curl -s -X PUT \ + "${CIVO_API_URL}/v2/kubernetes/clusters/${CIVO_CLUSTER_ID}/pools/high-compute" \ + -H "Authorization: bearer ${CIVO_API_KEY}" \ + -H "Content-Type: application/json" \ + -d "{\"count\": 1, \"region\": \"${CIVO_REGION}\"}" + echo "" + # Wait for node to provision before checking again + sleep 180 + fi + fi + + sleep 15 + done + env: + - name: CIVO_API_URL + valueFrom: + secretKeyRef: + key: api-url + name: civo-api-access + - name: CIVO_API_KEY + valueFrom: + secretKeyRef: + key: api-key + name: civo-api-access + - name: CIVO_CLUSTER_ID + valueFrom: + secretKeyRef: + key: cluster-id + name: civo-api-access + - name: CIVO_REGION + valueFrom: + secretKeyRef: + key: region + name: civo-api-access + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 50m + memory: 64Mi diff --git a/registry-internal.yaml b/registry-internal.yaml new file mode 100644 index 0000000..529317a --- /dev/null +++ b/registry-internal.yaml @@ -0,0 +1,92 @@ +# Internal registry access - no mTLS, Forgejo handles auth via imagePullSecret +# Only accessible via cluster-internal DNS (no external-dns annotation) +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: registry-internal-certificate + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: registry-internal-tls + namespace: emissary + spec: + secretName: registry-internal-tls + issuerRef: + name: letsencrypt-dns + kind: ClusterIssuer + dnsNames: + - registry-internal.nge6.com +--- +# Host without external-dns - only reachable if you know the IP +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: registry-internal-host + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: getambassador.io/v2 + kind: Host + metadata: + name: registry-internal-host + namespace: emissary + annotations: + external-dns.ambassador-service: emissary-ingress.emissary.svc.cluster.local + external-dns.alpha.kubernetes.io/target: 212.2.241.28 + spec: + hostname: registry-internal.nge6.com + tlsSecret: + name: registry-internal-tls +--- +# Mapping direct to Forgejo - no Pomerium, no mTLS +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: registry-internal-mapping + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: getambassador.io/v2 + kind: Mapping + metadata: + name: registry-internal-mapping + namespace: emissary + spec: + host: registry-internal.nge6.com + prefix: / + service: http://forgejo-http.forgejo.svc.cluster.local:3000 + timeout_ms: 300000 + connect_timeout_ms: 10000 +--- +# imagePullSecret for argo namespace +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: argo-registry-pull-secret + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: v1 + kind: Secret + metadata: + name: forgejo-registry + namespace: argo + type: kubernetes.io/dockerconfigjson + stringData: + .dockerconfigjson: | + {"auths":{"registry-internal.nge6.com":{"username":"eemoore","password":"testpassword123!","auth":"ZWVtb29yZTp0ZXN0cGFzc3dvcmQxMjMh"}}} diff --git a/registry.yaml b/registry.yaml new file mode 100644 index 0000000..48bf8a6 --- /dev/null +++ b/registry.yaml @@ -0,0 +1,97 @@ +# SSL Certificate for registry.nge6.com +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: registry-certificate + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: registry-tls + namespace: emissary + spec: + secretName: registry-tls + issuerRef: + name: letsencrypt-dns + kind: ClusterIssuer + dnsNames: + - registry.nge6.com +--- +# Ambassador Host for registry.nge6.com +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: registry-host + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: getambassador.io/v2 + kind: Host + metadata: + name: registry-host + namespace: emissary + annotations: + external-dns.ambassador-service: emissary-ingress.emissary.svc.cluster.local + external-dns.alpha.kubernetes.io/target: 212.2.241.28 + spec: + hostname: registry.nge6.com + tlsSecret: + name: registry-tls + tlsContext: + name: registry-mtls +--- +# TLSContext for mTLS - requires client certificates +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: registry-tls-context + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: getambassador.io/v2 + kind: TLSContext + metadata: + name: registry-mtls + namespace: emissary + spec: + hosts: + - registry.nge6.com + secret: registry-tls + ca_secret: registry-client-ca + cert_required: true + min_tls_version: v1.2 + max_tls_version: v1.2 +--- +# Ambassador Mapping for registry - direct to Forgejo, no Pomerium +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: registry-mapping + namespace: crossplane-system +spec: + providerConfigRef: + name: kubernetes-provider + forProvider: + manifest: + apiVersion: getambassador.io/v2 + kind: Mapping + metadata: + name: registry-mapping + namespace: emissary + spec: + host: registry.nge6.com + prefix: / + service: http://forgejo-http.forgejo.svc.cluster.local:3000 + timeout_ms: 300000 + connect_timeout_ms: 10000 diff --git a/vaultwarden.yaml b/vaultwarden.yaml index b016c44..f38811d 100644 --- a/vaultwarden.yaml +++ b/vaultwarden.yaml @@ -37,9 +37,19 @@ spec: ROCKET_WORKERS: "10" # Security settings INVITATIONS_ALLOWED: "true" - SIGNUPS_ALLOWED: "false" + SIGNUPS_ALLOWED: "true" SHOW_PASSWORD_HINT: "false" # Email configuration (disabled) + # OIDC/SSO configuration + SSO_ENABLED: "true" + SSO_ONLY: "false" + SSO_CLIENT_ID: "vaultwarden" + SSO_CLIENT_SECRET: "zMeG3odq6GUBoYUVcoNl1CmngJpwgMS6" + SSO_AUTHORITY: "https://auth.nge6.com/realms/kubernetes-realm" + SSO_SCOPES: "openid email profile" + # SSO_MASTER_PASSWORD_POLICY removed - not valid in testing image + SSO_DOMAIN: "nge6.com" + SSO_ORGANIZATIONS_INVITE: "true" # Admin settings ADMIN_TOKEN: "vaultwarden-admin-token-change-in-production" # Database (using SQLite for simplicity) @@ -93,6 +103,8 @@ spec: app: vaultwarden spec: replicas: 1 + strategy: + type: Recreate selector: matchLabels: app: vaultwarden @@ -103,7 +115,7 @@ spec: spec: containers: - name: vaultwarden - image: vaultwarden/server:1.30.5 + image: vaultwarden/server:testing ports: - containerPort: 8080 name: http @@ -233,6 +245,6 @@ spec: spec: hostname: vault.nge6.com prefix: / - service: https://pomerium-proxy.pomerium:443 + service: http://pomerium-allinone.pomerium:443 timeout_ms: 30000 connect_timeout_ms: 10000