DevOps Automation Specialist

      sse_algorithm = "AES256"
    }
  }
}

resource "aws_s3_bucket_lifecycle_configuration" "audit" {
  bucket = aws_s3_bucket.audit.id
  rule {
    id     = "tiering"
    status = "Enabled"
    filter {}
    transition {
      days          = 90
      storage_class = "GLACIER_IR"
    }
  }
}

data "aws_iam_policy_document" "audit" {
  statement {
    sid = "CloudTrailAclCheck"
    principals {
      type        = "Service"
      identifiers = ["cloudtrail.amazonaws.com"]
    }
    actions   = ["s3:GetBucketAcl"]
    resources = [aws_s3_bucket.audit.arn]
    condition {
      test     = "StringEquals"
      variable = "aws:SourceArn"
      values   = [local.trail_arn]
    }
  }
  statement {
    sid = "CloudTrailWrite"
    principals {
      type        = "Service"
      identifiers = ["cloudtrail.amazonaws.com"]
    }
    actions   = ["s3:PutObject"]
    resources = ["${aws_s3_bucket.audit.arn}/cloudtrail/AWSLogs/${local.account_id}/*"]
    condition {
      test     = "StringEquals"
      variable = "s3:x-amz-acl"
      values   = ["bucket-owner-full-control"]
    }
    condition {
      test     = "StringEquals"
      variable = "aws:SourceArn"
      values   = [local.trail_arn]
    }
  }
  statement {
    sid    = "DenyInsecureTransport"
    effect = "Deny"
    principals {
      type        = "AWS"
      identifiers = ["*"]
    }
    actions   = ["s3:*"]
    resources = [aws_s3_bucket.audit.arn, "${aws_s3_bucket.audit.arn}/*"]
    condition {
      test     = "Bool"
      variable = "aws:SecureTransport"
      values   = ["false"]
    }
  }
}

resource "aws_s3_bucket_policy" "audit" {
  bucket = aws_s3_bucket.audit.id
  policy = data.aws_iam_policy_document.audit.json
}

resource "aws_cloudtrail" "main" {
  name                          = local.trail_name
  s3_bucket_name                = aws_s3_bucket.audit.id
  s3_key_prefix                 = "cloudtrail"
  is_multi_region_trail         = true
  include_global_service_events = true
  enable_log_file_validation    = true # tamper-evidence digests
  depends_on                    = [aws_s3_bucket_policy.audit]
}

output "audit_bucket" { value = aws_s3_bucket.audit.id }

4.3 Kubernetes manifests (Kustomize + Argo Rollouts)

${VARS} placeholders are rendered by scripts/deploy.sh using envsubst with an explicit variable allow-list. Their values come from the SSM parameters that Terraform writes. The {{ }} templates used by Argo and ESO are left untouched.

deploy/k8s/base/kustomization.yaml

apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: acme
labels:
  - pairs:
      app.kubernetes.io/part-of: acme
    includeSelectors: false
resources:
  - secrets.yaml
  - analysis-templates.yaml
  - api.yaml
  - web.yaml
  - network-policy.yaml
generatorOptions:
  disableNameSuffixHash: true
configMapGenerator:
  - name: web-runtime-config
    literals:
      - 'config.js=window.__APP_CONFIG__ = { "apiUrl": "${API_URL}", "release": "${GIT_SHA}" };'

deploy/k8s/base/secrets.yaml

# Phase "pre" is applied (and synced) before migrations and rollouts.
apiVersion: external-secrets.io/v1
kind: SecretStore
metadata:
  name: aws-secrets-manager
  labels: { deploy.acme.io/phase: pre }
spec:
  provider:
    aws:
      service: SecretsManager
      region: "${AWS_REGION}"      # auth: ESO controller's IRSA role
---
# Human-managed app secrets, passed through as-is.
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
  name: api-app
  labels: { deploy.acme.io/phase: pre }
spec:
  refreshInterval: 1h
  secretStoreRef: { name: aws-secrets-manager, kind: SecretStore }
  target: { name: api-app, creationPolicy: Owner }
  dataFrom:
    - extract:
        key: "${APP_SECRET_ID}"
---
# Runtime connection strings. Uses the least-privileged app DB role, NOT the rotating master user.
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
  name: api-connection
  labels: { deploy.acme.io/phase: pre }
spec:
  refreshInterval: 1h
  secretStoreRef: { name: aws-secrets-manager, kind: SecretStore }
  target:
    name: api-connection
    creationPolicy: Owner
    template:
      engineVersion: v2
      data:
        DATABASE_URL: 'postgresql://{{ .db_user | urlquery }}:{{ .db_password | urlquery }}@{{ .db_host }}:{{ .db_port }}/{{ .db_name }}?sslmode=verify-full'
        REDIS_URL: 'rediss://:{{ .redis_auth | urlquery }}@{{ .redis_host }}:6379'
  data:
    - secretKey: db_user
      remoteRef: { key: "${APP_SECRET_ID}", property: DB_APP_USER }
    - secretKey: db_password
      remoteRef: { key: "${APP_SECRET_ID}", property: DB_APP_PASSWORD }
    - secretKey: db_host
      remoteRef: { key: "${APP_INFRA_SECRET_ID}", property: DB_HOST }
    - secretKey: db_port
      remoteRef: { key: "${APP_INFRA_SECRET_ID}", property: DB_PORT }
    - secretKey: db_name
      remoteRef: { key: "${APP_INFRA_SECRET_ID}", property: DB_NAME }
    - secretKey: redis_host
      remoteRef: { key: "${APP_INFRA_SECRET_ID}", property: REDIS_HOST }
    - secretKey: redis_auth
      remoteRef: { key: "${APP_INFRA_SECRET_ID}", property: REDIS_AUTH_TOKEN }
---
# Master credentials, used ONLY by the migration Job. Force-synced before each migration.
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
  name: api-migrator
  labels: { deploy.acme.io/phase: pre }
spec:
  refreshInterval: 5m
  secretStoreRef: { name: aws-secrets-manager, kind: SecretStore }
  target:
    name: api-migrator
    creationPolicy: Owner
    template:
      engineVersion: v2
      data:
        MIGRATION_DATABASE_URL: 'postgresql://{{ .user | urlquery }}:{{ .password | urlquery }}@{{ .db_host }}:{{ .db_port }}/{{ .db_name }}?sslmode=verify-full'
  data:
    - secretKey: user
      remoteRef: { key: "${DB_ADMIN_SECRET_ID}", property: username }
    - secretKey: password
      remoteRef: { key: "${DB_ADMIN_SECRET_ID}", property: password }
    - secretKey: db_host
      remoteRef: { key: "${APP_INFRA_SECRET_ID}", property: DB_HOST }
    - secretKey: db_port
      remoteRef: { key: "${APP_INFRA_SECRET_ID}", property: DB_PORT }
    - secretKey: db_name
      remoteRef: { key: "${APP_INFRA_SECRET_ID}", property: DB_NAME }

deploy/k8s/base/analysis-templates.yaml

# Pre-promotion: smoke-test the green (preview) pods before any traffic reaches them.
apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: smoke-test
spec:
  args:
    - name: service-url
    - name: paths
  metrics:
    - name: http-smoke
      count: 1
      provider:
        job:
          spec:
            backoffLimit: 0
            activeDeadlineSeconds: 300
            template:
              spec:
                restartPolicy: Never
                automountServiceAccountToken: false
                securityContext:
                  runAsNonRoot: true
                  runAsUser: 100
                  seccompProfile: { type: RuntimeDefault }
                containers:
                  - name: smoke
                    image: curlimages/curl:8.11.1
                    securityContext:
                      allowPrivilegeEscalation: false
                      readOnlyRootFilesystem: true
                      capabilities: { drop: [ALL] }
                    command: [sh, -c]
                    args:
                      - |
                        set -eu
                        for p in {{args.paths}}; do
                          echo "GET {{args.service-url}}$p"
                          curl -fsS --max-time 5 --retry 10 --retry-delay 3 --retry-all-errors \
                               -o /dev/null -w '%{http_code} %{time_total}s\n' "{{args.service-url}}$p"
                        done
---
# Post-promotion: real traffic through the ALB target group. Failure => auto-abort => traffic back to blue.
apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: alb-health
spec:
  args:
    - name: target-group     # CloudWatch dimension, e.g. targetgroup/acme-production-api/abc123
    - name: load-balancer    # e.g. app/acme-production/def456
  metrics:
    - name: target-5xx-rate
      initialDelay: 60s
      interval: 60s
      count: 5
      failureLimit: 1
      successCondition: "all(result[0].Values, {# <= 0.01})"
      provider:
        cloudWatch:
          interval: 5m
          metricDataQueries:
            - id: rate
              expression: "IF(req > 0, FILL(err, 0) / req, 0)"
            - id: err
              returnData: false
              metricStat:
                period: 60
                stat: Sum
                metric:
                  namespace: AWS/ApplicationELB
                  metricName: HTTPCode_Target_5XX_Count
                  dimensions:
                    - { name: LoadBalancer, value: "{{args.load-balancer}}" }
                    - { name: TargetGroup, value: "{{args.target-group}}" }
            - id: req
              returnData: false
              metricStat:
                period: 60
                stat: Sum
                metric:
                  namespace: AWS/ApplicationELB
                  metricName: RequestCount
                  dimensions:
                    - { name: LoadBalancer, value: "{{args.load-balancer}}" }
                    - { name: TargetGroup, value: "{{args.target-group}}" }
    - name: target-p95-latency
      initialDelay: 60s
      interval: 60s
      count: 5
      failureLimit: 1
      successCondition: "all(result[0].Values, {# <= 1.0})"
      provider:
        cloudWatch:
          interval: 5m
          metricDataQueries:
            - id: p95
              metricStat:
                period: 60
                stat: p95
                metric:
                  namespace: AWS/ApplicationELB
                  metricName: TargetResponseTime
                  dimensions:
                    - { name: LoadBalancer, value: "{{args.load-balancer}}" }
                    - { name: TargetGroup, value: "{{args.target-group}}" }
    - name: healthy-hosts
      initialDelay: 30s
      interval: 60s
      count: 5
      failureLimit: 1
      successCondition: "all(result[0].Values, {# >= 1})"
      provider:
        cloudWatch:
          interval: 3m
          metricDataQueries:
            - id: healthy
              metricStat:
                period: 60
                stat: Minimum
                metric:
                  namespace: AWS/ApplicationELB
                  metricName: HealthyHostCount
                  dimensions:
                    - { name: LoadBalancer, value: "{{args.load-balancer}}" }
                    - { name: TargetGroup, value: "{{args.target-group}}" }

deploy/k8s/base/api.yaml

apiVersion: v1
kind: ServiceAccount
metadata: { name: api }
automountServiceAccountToken: false
---
apiVersion: v1
kind: Service
metadata: { name: api-active }
spec:
  type: ClusterIP
  selector: { app.kubernetes.io/name: api }     # Argo adds rollouts-pod-template-hash
  ports: [{ name: http, port: 80, targetPort: http }]
---
apiVersion: v1
kind: Service
metadata: { name: api-preview }
spec:
  type: ClusterIP
  selector: { app.kubernetes.io/name: api }
  ports: [{ name: http, port: 80, targetPort: http }]
---
# Binds the Terraform-owned ALB target group to whatever pods api-active selects.
apiVersion: elbv2.k8s.aws/v1beta1
kind: TargetGroupBinding
metadata: { name: api }
spec:
  serviceRef: { name: api-active, port: 80 }
  targetGroupARN: "${API_TG_ARN}"
  targetType: ip
---
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata: { name: api }
spec:
  # replicas intentionally omitted: owned by the HPA
  revisionHistoryLimit: 5
  progressDeadlineSeconds: 900
  progressDeadlineAbort: true
  selector:
    matchLabels: { app.kubernetes.io/name: api }
  strategy:
    blueGreen:
      activeService: api-active
      previewService: api-preview
      autoPromotionEnabled: true              # the human gate is the GitHub Environment
      prePromotionAnalysis:
        templates: [{ templateName: smoke-test }]
        args:
          - { name: service-url, value: "http://api-preview" }
          - { name: paths, value: "/healthz /readyz" }
      postPromotionAnalysis:
        templates: [{ templateName: alb-health }]
        args:
          - { name: target-group, value: "${API_TG_SUFFIX}" }
          - { name: load-balancer, value: "${ALB_SUFFIX}" }
      scaleDownDelaySeconds: 600              # blue stays warm => instant rollback
      abortScaleDownDelaySeconds: 600
  template:
    metadata:
      labels: { app.kubernetes.io/name: api }
    spec:
      serviceAccountName: api
      automountServiceAccountToken: false
      terminationGracePeriodSeconds: 60
      securityContext:
        runAsNonRoot: true
        runAsUser: 1000
        runAsGroup: 1000
        fsGroup: 1000
        seccompProfile: { type: RuntimeDefault }
      topologySpreadConstraints:
        - maxSkew: 1
          topologyKey: topology.kubernetes.io/zone
          whenUnsatisfiable: ScheduleAnyway
          labelSelector: { matchLabels: { app.kubernetes.io/name: api } }
          matchLabelKeys: [rollouts-pod-template-hash]
      containers:
        - name: api
          image: "${API_IMAGE}"
          imagePullPolicy: IfNotPresent
          ports: [{ name: http, containerPort: 3000 }]
          env:
            - { name: NODE_ENV, value: production }
            - { name: PORT, value: "3000" }
            - { name: RELEASE, value: "${GIT_SHA}" }
          envFrom:
            - secretRef: { name: api-connection }
            - secretRef: { name: api-app }
          resources:
            requests: { cpu: 250m, memory: 384Mi }
            limits: { memory: 512Mi }
          startupProbe:
            httpGet: { path: /healthz, port: http }
            periodSeconds: 5
            failureThreshold: 24
          readinessProbe:
            httpGet: { path: /readyz, port: http }
            periodSeconds: 5
            failureThreshold: 3
          livenessProbe:
            httpGet: { path: /healthz, port: http }
            periodSeconds: 10
            failureThreshold: 3
          lifecycle:
            preStop:
              sleep: { seconds: 20 }          # let the ALB deregister before SIGTERM
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities: { drop: [ALL] }
          volumeMounts: [{ name: tmp, mountPath: /tmp }]
      volumes:
        - name: tmp
          emptyDir: { sizeLimit: 64Mi }
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata: { name: api }
spec:
  scaleTargetRef: { apiVersion: argoproj.io/v1alpha1, kind: Rollout, name: api }
  minReplicas: 2
  maxReplicas: 6
  metrics:
    - type: Resource
      resource: { name: cpu, target: { type: Utilization, averageUtilization: 70 } }
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata: { name: api }
spec:
  maxUnavailable: 1
  selector: { matchLabels: { app.kubernetes.io/name: api } }

deploy/k8s/base/web.yaml

apiVersion: v1
kind: ServiceAccount
metadata: { name: web }
automountServiceAccountToken: false
---
apiVersion: v1
kind: Service
metadata: { name: web-active }
spec:
  selector: { app.kubernetes.io/name: web }
  ports: [{ name: http, port: 80, targetPort: http }]
---
apiVersion: v1
kind: Service
metadata: { name: web-preview }
spec:
  selector: { app.kubernetes.io/name: web }
  ports: [{ name: http, port: 80, targetPort: http }]
---
apiVersion: elbv2.k8s.aws/v1beta1
kind: TargetGroupBinding
metadata: { name: web }
spec:
  serviceRef: { name: web-active, port: 80 }
  targetGroupARN: "${WEB_TG_ARN}"
  targetType: ip
---
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata: { name: web }
spec:
  revisionHistoryLimit: 5
  progressDeadlineSeconds: 600
  progressDeadlineAbort: true
  selector:
    matchLabels: { app.kubernetes.io/name: web }
  strategy:
    blueGreen:
      activeService: web-active
      previewService: web-preview
      autoPromotionEnabled: true
      prePromotionAnalysis:
        templates: [{ templateName: smoke-test }]
        args:
          - { name: service-url, value: "http://web-preview" }
          - { name: paths, value: "/healthz / /config.js" }
      postPromotionAnalysis:
        templates: [{ templateName: alb-health }]
        args:
          - { name: target-group, value: "${WEB_TG_SUFFIX}" }
          - { name: load-balancer, value: "${ALB_SUFFIX}" }
      scaleDownDelaySeconds: 600
      abortScaleDownDelaySeconds: 600
  template:
    metadata:
      labels: { app.kubernetes.io/name: web }
    spec:
      serviceAccountName: web
      automountServiceAccountToken: false
      terminationGracePeriodSeconds: 45
      securityContext:
        runAsNonRoot: true
        runAsUser: 101
        runAsGroup: 101
        seccompProfile: { type: RuntimeDefault }
      topologySpreadConstraints:
        - maxSkew: 1
          topologyKey: topology.kubernetes.io/zone
          whenUnsatisfiable: ScheduleAnyway
          labelSelector: { matchLabels: { app.kubernetes.io/name: web } }
          matchLabelKeys: [rollouts-pod-template-hash]
      containers:
        - name: web
          image: "${WEB_IMAGE}"
          ports: [{ name: http, containerPort: 8080 }]
          resources:
            requests: { cpu: 50m, memory: 64Mi }
            limits: { memory: 128Mi }
          readinessProbe:
            httpGet: { path: /healthz, port: http }
            periodSeconds: 5
          livenessProbe:
            httpGet: { path: /healthz, port: http }
            periodSeconds: 10
          lifecycle:
            preStop:
              sleep: { seconds: 20 }
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities: { drop: [ALL] }
          volumeMounts:
            - { name: tmp, mountPath: /tmp }
            - { name: runtime-config, mountPath: /usr/share/nginx/html/config.js, subPath: config.js, readOnly: true }
      volumes:
        - name: tmp
          emptyDir: { sizeLimit: 32Mi }
        - name: runtime-config
          configMap: { name: web-runtime-config }
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata: { name: web }
spec:
  scaleTargetRef: { apiVersion: argoproj.io/v1alpha1, kind: Rollout, name: web }
  minReplicas: 2
  maxReplicas: 4
  metrics:
    - type: Resource
      resource: { name: cpu, target: { type: Utilization, averageUtilization: 70 } }
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata: { name: web }
spec:
  maxUnavailable: 1
  selector: { matchLabels: { app.kubernetes.io/name: web } }

deploy/k8s/base/network-policy.yaml

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: { name: default-deny-ingress }
spec:
  podSelector: {}
  policyTypes: [Ingress]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: { name: allow-alb-and-namespace }
spec:
  podSelector:
    matchExpressions:
      - { key: app.kubernetes.io/name, operator: In, values: [api, web] }
  policyTypes: [Ingress]
  ingress:
    - from:
        - ipBlock: { cidr: "${ALB_SOURCE_CIDR}" }   # ALB ENIs live only in the public subnets
      ports: [{ port: 3000 }, { port: 8080 }]
    - from:
        - podSelector: {}                         # same-namespace (analysis Jobs, web->api)

deploy/k8s/overlays/staging/kustomization.yaml

apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources: [../../base]
patches:
  - target: { kind: Rollout }
    patch: |-
      - op: replace
        path: /spec/strategy/blueGreen/scaleDownDelaySeconds
        value: 120
  - target: { kind: HorizontalPodAutoscaler }
    patch: |-
      - op: replace
        path: /spec/minReplicas
        value: 1
      - op: replace
        path: /spec/maxReplicas
        value: 2

deploy/k8s/overlays/production/kustomization.yaml

apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources: [../../base]
patches:
  - target: { kind: Rollout }
    patch: |-
      - op: replace
        path: /spec/strategy/blueGreen/scaleDownDelaySeconds
        value: 900
  - target: { kind: HorizontalPodAutoscaler, name: api }
    patch: |-
      - op: replace
        path: /spec/minReplicas
        value: 3
      - op: replace
        path: /spec/maxReplicas
        value: 8

deploy/k8s/jobs/migrate.yaml

apiVersion: batch/v1
kind: Job
metadata:
  name: "api-migrate-${SHORT_SHA}"
  namespace: "${APP_NAMESPACE}"
  labels: { app.kubernetes.io/name: api-migrate, app.kubernetes.io/part-of: acme }
spec:
  backoffLimit: 0
  activeDeadlineSeconds: 900
  ttlSecondsAfterFinished: 604800
  template:
    metadata:
      labels: { app.kubernetes.io/name: api-migrate }
    spec:
      restartPolicy: Never
      serviceAccountName: api
      automountServiceAccountToken: false
      securityContext:
        runAsNonRoot: true
        runAsUser: 1000
        runAsGroup: 1000
        seccompProfile: { type: RuntimeDefault }
      containers:
        - name: migrate
          image: "${API_IMAGE}"
          command: ["node", "dist/migrate.js"]      # adjust to your migration tool
          envFrom:
            - secretRef: { name: api-migrator }     # MIGRATION_DATABASE_URL (master)
            - secretRef: { name: api-app }          # DB_APP_USER/DB_APP_PASSWORD to ensure the app role
          resources:
            requests: { cpu: 100m, memory: 256Mi }
            limits: { memory: 512Mi }
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities: { drop: [ALL] }
          volumeMounts: [{ name: tmp, mountPath: /tmp }]
      volumes:
        - name: tmp
          emptyDir: {}

4.4 Container images and app contract

apps/api/Dockerfile

# syntax=docker/dockerfile:1.7
# Node 20 is EOL (Apr 2026). Pin by digest via Renovate: node:22-alpine3.22@sha256:...
ARG NODE_IMAGE=node:22-alpine3.22

FROM ${NODE_IMAGE} AS deps
WORKDIR /app
COPY package.json package-lock.json ./
RUN --mount=type=cache,target=/root/.npm npm ci --omit=dev

FROM ${NODE_IMAGE} AS build
WORKDIR /app
COPY package.json package-lock.json ./
RUN --mount=type=cache,target=/root/.npm npm ci
COPY . .
RUN npm run build --if-present

FROM ${NODE_IMAGE} AS runtime
RUN apk add --no-cache tini && apk upgrade --no-cache
ENV NODE_ENV=production \
    NODE_EXTRA_CA_CERTS=/app/certs/rds-global-bundle.pem
WORKDIR /app
# RDS CA bundle so sslmode=verify-full works with node-postgres
ADD --chown=node:node https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem /app/certs/rds-global-bundle.pem
COPY --from=deps  --chown=node:node /app/node_modules ./node_modules
COPY --from=build --chown=node:node /app/package.json ./
COPY --from=build --chown=node:node /app/dist ./dist
USER node
EXPOSE 3000
ENTRYPOINT ["/sbin/tini", "--"]
CMD ["node", "dist/server.js"]

apps/api/src/lifecycle.js: the zero-downtime contract. Call start() from your server bootstrap.

// Register BEFORE auth middleware: app.use(healthRouter) first, or call start() on a fresh app and mount.
export function start(app, { port = Number(process.env.PORT) || 3000, checks = [] } = {}) {
  let draining = false;

  app.get('/healthz', (_req, res) => res.status(200).send('ok')); // liveness + ALB TG check: no deps
  app.get('/readyz', async (_req, res) => {                      // k8s readiness: deps with timeouts
    if (draining) return res.status(503).send('draining');
    try {
      await Promise.all(checks.map((c) => Promise.race([
        c(), new Promise((_, rej) => setTimeout(() => rej(new Error('timeout')), 1500)),
      ])));
      res.status(200).send('ready');
    } catch {
      res.status(503).send('not ready');
    }
  });

  const server = app.listen(port);
  server.keepAliveTimeout = 65_000; // MUST exceed ALB idle_timeout (60s) or you get sporadic 502s
  server.headersTimeout = 66_000;

  const shutdown = (signal) => {
    draining = true;
    console.log(JSON.stringify({ level: 'info', msg: 'shutdown', signal, release: process.env.RELEASE }));
    server.close(() => process.exit(0));        // finish in-flight, reject new
    setTimeout(() => process.exit(1), 30_000).unref();
  };
  process.on('SIGTERM', shutdown);
  process.on('SIGINT', shutdown);
  return server;
}
// usage: start(app, { checks: [() => pool.query('SELECT 1'), () => redis.ping()] })

apps/web/Dockerfile

# syntax=docker/dockerfile:1.7
FROM node:22-alpine3.22 AS build
WORKDIR /app
COPY package.json package-lock.json ./
RUN --mount=type=cache,target=/root/.npm npm ci
COPY . .
RUN npm run build

FROM nginxinc/nginx-unprivileged:1.28-alpine
COPY security-headers.conf /etc/nginx/snippets/security-headers.conf
COPY nginx.conf /etc/nginx/conf.d/default.conf
COPY --from=build /app/dist /usr/share/nginx/html
EXPOSE 8080

apps/web/security-headers.conf

add_header X-Content-Type-Options "nosniff" always;
add_header X-Frame-Options "DENY" always;
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
# Tighten connect-src to your API host(s) once known.
add_header Content-Security-Policy "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; connect-src 'self' https:; frame-ancestors 'none'; base-uri 'self'; form-action 'self'" always;

apps/web/nginx.conf

server {
  listen 8080;
  server_name _;
  root /usr/share/nginx/html;
  server_tokens off;

  # add_header in a location drops server-level headers, so every location includes the snippet.
  location = /healthz {
    access_log off;
    default_type text/plain;
    return 200 "ok";
  }

  location = /config.js {                 # runtime config: same image across envs
    include /etc/nginx/snippets/security-headers.conf;
    add_header Cache-Control "no-store" always;
  }

  location /assets/ {                     # Vite hashed assets
    include /etc/nginx/snippets/security-headers.conf;
    add_header Cache-Control "public, max-age=31536000, immutable" always;
    try_files $uri =404;
  }

  location / {
    include /etc/nginx/snippets/security-headers.conf;
    add_header Cache-Control "no-cache" always;
    try_files $uri /index.html;
  }
}

In index.html, add <script src="/config.js"></script> before the bundle, and read window.__APP_CONFIG__.apiUrl instead of import.meta.env. This lets you build once and promote the same image. A per-environment Vite build would break digest promotion.

4.5 Scripts

scripts/deploy.sh

#!/usr/bin/env bash
# Usage: ENVIRONMENT=staging API_IMAGE=... WEB_IMAGE=... GIT_SHA=... AWS_REGION=us-east-1 scripts/deploy.sh
set -euo pipefail
: "${ENVIRONMENT:?}" "${API_IMAGE:?}" "${WEB_IMAGE:?}" "${GIT_SHA:?}" "${AWS_REGION:?}"

PROJECT="${PROJECT:-acme}"
export APP_NAMESPACE="${APP_NAMESPACE:-$PROJECT}"
export SHORT_SHA="${GIT_SHA:0:12}" API_IMAGE WEB_IMAGE GIT_SHA AWS_REGION
NS="$APP_NAMESPACE"
CLUSTER="${PROJECT}-${ENVIRONMENT}"
ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-25m}"
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
OUT="$(mktemp -d)"

aws eks update-kubeconfig --name "$CLUSTER" --region "$AWS_REGION" --alias "$CLUSTER" >/dev/null

# 1) Platform outputs written by Terraform: /acme/<env>/deploy/api_tg_arn -> API_TG_ARN
while IFS=$'\t' read -r name value; do
  key="$(basename "$name" | tr '[:lower:]' '[:upper:]')"
  export "$key=$value"
done < <(aws ssm get-parameters-by-path --path "/${PROJECT}/${ENVIRONMENT}/deploy/" \
          --query 'Parameters[].[Name,Value]' --output text)

VARS='${API_IMAGE} ${WEB_IMAGE} ${GIT_SHA} ${SHORT_SHA} ${AWS_REGION} ${APP_NAMESPACE}
      ${API_TG_ARN} ${WEB_TG_ARN} ${API_TG_SUFFIX} ${WEB_TG_SUFFIX} ${ALB_SUFFIX} ${ALB_SOURCE_CIDR}
      ${DB_ADMIN_SECRET_ID} ${APP_INFRA_SECRET_ID} ${APP_SECRET_ID} ${API_URL}'
for v in $(echo "$VARS" | tr -d '${}'); do
  : "${!v:?missing $v, check SSM /${PROJECT}/${ENVIRONMENT}/deploy/}"
done

# 2) Render + validate before touching anything
kubectl kustomize "$ROOT/deploy/k8s/overlays/$ENVIRONMENT" | envsubst "$VARS" > "$OUT/app.yaml"
envsubst "$VARS" < "$ROOT/deploy/k8s/jobs/migrate.yaml" > "$OUT/migrate.yaml"
if grep -n '\${' "$OUT/app.yaml" "$OUT/migrate.yaml"; then echo "::error::unrendered placeholders"; exit 1; fi
APPLY=(kubectl apply --server-side --field-manager=github-actions)
"${APPLY[@]}" --dry-run=server -f "$OUT/app.yaml" >/dev/null

# 3) Secrets first (force-sync so migrations never use a pre-rotation master password)
"${APPLY[@]}" -l deploy.acme.io/phase=pre -f "$OUT/app.yaml"
for es in api-app api-connection api-migrator; do
  kubectl -n "$NS" annotate externalsecret "$es" force-sync="$(date +%s)" --overwrite >/dev/null
done
sleep 5
kubectl -n "$NS" wait --for=condition=Ready externalsecret --all --timeout=180s

# 4) Migrations (idempotent per commit; must be backward compatible: expand/contract)
JOB="api-migrate-${SHORT_SHA}"
if [[ "$(kubectl -n "$NS" get job "$JOB" -o jsonpath='{.status.succeeded}' 2>/dev/null || true)" != "1" ]]; then
  kubectl -n "$NS" delete job "$JOB" --ignore-not-found --wait=true
  "${APPLY[@]}" -f "$OUT/migrate.yaml"
  for _ in $(seq 1 90); do
    s="$(kubectl -n "$NS" get job "$JOB" -o jsonpath='{.status.succeeded}/{.status.failed}')"
    case "$s" in
      1/*) echo "migration ok"; break ;;
      */1) kubectl -n "$NS" logs "job/$JOB" --tail=200 || true; echo "::error::migration failed"; exit 1 ;;
    esac
    sleep 10
  done
  [[ "$(kubectl -n "$NS" get job "$JOB" -o jsonpath='{.status.succeeded}')" == "1" ]] \
    || { echo "::error::migration timed out"; exit 1; }
fi

# 5) Blue/green rollout. Argo auto-aborts (traffic back to blue) on any analysis failure.
"${APPLY[@]}" -f "$OUT/app.yaml"
rc=0
for r in api web; do
  if ! kubectl argo rollouts status "$r" -n "$NS" --timeout "$ROLLOUT_TIMEOUT"; then
    rc=1
    echo "::error::rollout $r failed; ensuring abort (active stays on previous version)"
    kubectl argo rollouts abort "$r" -n "$NS" || true
    kubectl argo rollouts get rollout "$r" -n "$NS" || true
  fi
done
exit "$rc"

scripts/install-tools.sh

#!/usr/bin/env bash
set -euo pipefail
: "${KUBECTL_VERSION:?}" "${ARGO_ROLLOUTS_VERSION:?}"
: "${ARGO_ROLLOUTS_SHA256:?set repo var ARGO_ROLLOUTS_SHA256 (sha256 of kubectl-argo-rollouts-linux-amd64)}"
BIN="${RUNNER_TEMP:-/tmp}/bin"; mkdir -p "$BIN"

curl -fsSLo "$BIN/kubectl" "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
echo "$(curl -fsSL "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl.sha256")  $BIN/kubectl" | sha256sum -c -

curl -fsSLo "$BIN/kubectl-argo-rollouts" \
  "https://github.com/argoproj/argo-rollouts/releases/download/${ARGO_ROLLOUTS_VERSION}/kubectl-argo-rollouts-linux-amd64"
echo "${ARGO_ROLLOUTS_SHA256}  $BIN/kubectl-argo-rollouts" | sha256sum -c -

chmod +x "$BIN"/*
echo "$BIN" >> "${GITHUB_PATH:-/dev/null}"
export PATH="$BIN:$PATH"; kubectl version --client

scripts/install-terragrunt.sh

#!/usr/bin/env bash
set -euo pipefail
: "${TG_VERSION:?}"
BIN="${RUNNER_TEMP:-/tmp}/bin"; mkdir -p "$BIN"
base="https://github.com/gruntwork-io/terragrunt/releases/download/v${TG_VERSION}"
curl -fsSLo "$BIN/terragrunt_linux_amd64" "$base/terragrunt_linux_amd64"
curl -fsSL "$base/SHA256SUMS" | grep ' terragrunt_linux_amd64$' | (cd "$BIN" && sha256sum -c -)
mv "$BIN/terragrunt_linux_amd64" "$BIN/terragrunt" && chmod +x "$BIN/terragrunt"
echo "$BIN" >> "${GITHUB_PATH:-/dev/null}"

scripts/tg.sh

#!/usr/bin/env bash
# Usage: scripts/tg.sh <plan|apply> <env> [units...]
# Default units exclude ci-access and audit: those are bootstrap-only and applied by a human admin.
set -euo pipefail
mode="$1"; env="$2"; shift 2
units=("$@"); [[ ${#units[@]} -gt 0 ]] || units=(network eks platform data edge observability)
root="$(git rev-parse --show-toplevel)/infra/live/${env}"

for u in "${units[@]}"; do
  echo "::group::${mode} ${env}/${u}"
  pushd "$root/$u" >/dev/null
  case "$mode" in
    plan)  terragrunt plan -input=false -lock=false ;;
    apply) terragrunt plan -input=false -out=tfplan && terragrunt apply -input=false tfplan ;;
    *)     echo "unknown mode: $mode" >&2; exit 2 ;;
  esac
  popd >/dev/null
  echo "::endgroup::"
done

.trivyignore

# One CVE per line with justification + expiry, e.g.:
# CVE-2024-XXXXX  # not reachable: dev-only transitive dep; review by 2026-12-31

.zap/rules.tsv (tab-separated; every IGNORE needs a justification in the PR)

10096	IGNORE	(Timestamp Disclosure - Unix)
10027	IGNORE	(Information Disclosure - Suspicious Comments)
10055	WARN	(CSP)

4.6 GitHub Actions

Replace action tags with full commit SHAs (e.g. pinact run or Renovate helpers:pinGitHubActionDigests). Tags are shown here so the file stays readable.

.github/workflows/ci.yml

name: ci
on:
  pull_request:
  workflow_call:

permissions:
  contents: read

jobs:
  api:
    runs-on: ubuntu-24.04
    defaults: { run: { working-directory: apps/api } }
    services:
      postgres:
        image: postgres:15-alpine
        env: { POSTGRES_PASSWORD: postgres, POSTGRES_DB: app_test }
        ports: ["5432:5432"]
        options: >-
          --health-cmd "pg_isready -U postgres" --health-interval 5s --health-retries 10
      redis:
        image: redis:7-alpine
        ports: ["6379:6379"]
        options: >-
          --health-cmd "redis-cli ping" --health-interval 5s --health-retries 10
    env:
      MIGRATION_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/app_test
      DATABASE_URL: postgresql://postgres:postgres@localhost:5432/app_test
      REDIS_URL: redis://localhost:6379
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 22
          cache: npm
          cache-dependency-path: apps/api/package-lock.json
      - run: npm ci
      - run: npm run lint
      - run: npm run build --if-present
      - run: npm run migrate --if-present
      - run: npm test -- --coverage
      - run: npm audit --omit=dev --audit-level=high

  web:
    runs-on: ubuntu-24.04
    defaults: { run: { working-directory: apps/web } }
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 22
          cache: npm
          cache-dependency-path: apps/web/package-lock.json
      - run: npm ci
      - run: npm run lint
      - run: npm run typecheck --if-present
      - run: npm test --if-present
      - run: npm run build
      - run: npm audit --omit=dev --audit-level=high

  repo-security:
    runs-on: ubuntu-24.04
    env: { TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db }
    steps:
      - uses: actions/checkout@v4
      - name: Trivy fs (vulns, secrets, IaC/Dockerfile misconfig)
        uses: aquasecurity/trivy-action@0.28.0
        with:
          scan-type: fs
          scan-ref: .
          scanners: vuln,secret,misconfig
          severity: CRITICAL,HIGH
          ignore-unfixed: true
          exit-code: "1"
          trivyignores: .trivyignore

  images:
    runs-on: ubuntu-24.04
    strategy: { matrix: { app: [api, web] } }
    env: { TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db }
    steps:
      - uses: actions/checkout@v4
      - uses: docker/setup-buildx-action@v3
      - uses: docker/build-push-action@v6
        with:
          context: apps/${{ matrix.app }}
          load: true
          push: false
          tags: local/${{ matrix.app }}:ci
          cache-from: type=gha,scope=${{ matrix.app }}
      - uses: aquasecurity/trivy-action@0.28.0
        with:
          image-ref: local/${{ matrix.app }}:ci
          severity: CRITICAL,HIGH
          ignore-unfixed: true
          exit-code: "1"
          trivyignores: .trivyignore

  manifests:
    runs-on: ubuntu-24.04
    env: { KUBECONFORM_VERSION: v0.6.7 }
    steps:
      - uses: actions/checkout@v4
      - name: Install kubeconform
        run: |
          curl -fsSL "https://github.com/yannh/kubeconform/releases/download/${KUBECONFORM_VERSION}/kubeconform-linux-amd64.tar.gz" | tar xz -C /usr/local/bin kubeconform
      - name: Render + validate overlays
        run: |
          for env in staging production; do
            kubectl kustomize "deploy/k8s/overlays/$env" | \
              kubeconform -strict -summary -ignore-missing-schemas \
                -schema-location default \
                -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json'
          done

.github/workflows/deploy.yml

name: deploy
on:
  push:
    branches: [main]
    paths-ignore: ["infra/**", "**/*.md", "docs/**"]
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: deploy-main
  cancel-in-progress: false   # never cancel a half-finished deploy

env:
  AWS_REGION: us-east-1
  PROJECT: acme
  BUILD_ACCOUNT_ID: "111122223333"
  TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db

jobs:
  ci:
    uses: ./.github/workflows/ci.yml
    permissions: { contents: read }

  build:
    needs: ci
    runs-on: ubuntu-24.04
    permissions: { contents: read, id-token: write, attestations: write }
    strategy:
      fail-fast: true
      matrix: { app: [api, web] }
    env:
      REPO: acme/${{ matrix.app }}
    steps:
      - uses: actions/checkout@v4
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::${{ env.BUILD_ACCOUNT_ID }}:role/${{ env.PROJECT }}-gha-publisher
          role-session-name: gha-build-${{ github.run_id }}-${{ github.run_attempt }}
          aws-region: ${{ env.AWS_REGION }}
      - id: ecr
        uses: aws-actions/amazon-ecr-login@v2

      - id: exists   # idempotent re-runs: tags are immutable
        run: |
          if aws ecr describe-images --repository-name "$REPO" --image-ids imageTag="$GITHUB_SHA" >/dev/null 2>&1; then
            echo "skip=true" >> "$GITHUB_OUTPUT"
          else
            echo "skip=false" >> "$GITHUB_OUTPUT"
          fi

      - if: steps.exists.outputs.skip == 'false'
        uses: docker/setup-buildx-action@v3
      - if: steps.exists.outputs.skip == 'false'
        uses: docker/build-push-action@v6
        with:
          context: apps/${{ matrix.app }}
          platforms: linux/amd64
          push: true
          provenance: false      # attestations go to GitHub; keeps a single-manifest digest in ECR
          sbom: false
          tags: ${{ steps.ecr.outputs.registry }}/${{ env.REPO }}:${{ github.sha }}
          labels: |
            org.opencontainers.image.revision=${{ github.sha }}
            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
          cache-from: type=gha,scope=${{ matrix.app }}
          cache-to: type=gha,mode=max,scope=${{ matrix.app }}

      - id: digest
        run: |
          d=$(aws ecr describe-images --repository-name "$REPO" --image-ids imageTag="$GITHUB_SHA" \
                --query 'imageDetails[0].imageDigest' --output text)
          echo "digest=$d" >> "$GITHUB_OUTPUT"
          echo "ref=${{ steps.ecr.outputs.registry }}/${REPO}@${d}" >> "$GITHUB_OUTPUT"

      - name: Trivy image gate (always runs, even on re-runs)
        uses: aquasecurity/trivy-action@0.28.0
        with:
          image-ref: ${{ steps.digest.outputs.ref }}
          severity: CRITICAL,HIGH
          ignore-unfixed: true
          exit-code: "1"
          trivyignores: .trivyignore

      - if: steps.exists.outputs.skip == 'false' && vars.ENABLE_ATTESTATIONS == 'true'
        uses: actions/attest-build-provenance@v2
        with:
          subject-name: ${{ steps.ecr.outputs.registry }}/${{ env.REPO }}
          subject-digest: ${{ steps.digest.outputs.digest }}

  deploy-staging:
    needs: build
    uses: ./.github/workflows/_deploy.yml
    with:
      environment: staging
      source-account-id: "111122223333"
    permissions: { contents: read, id-token: write, actions: read }

  zap-staging:
    needs: deploy-staging
    runs-on: ubuntu-24.04
    permissions: { contents: read }
    strategy:
      fail-fast: false
      matrix: { target: [app, api] }
    steps:
      - uses: actions/checkout@v4
      - uses: zaproxy/action-baseline@v0.14.0
        with:
          target: ${{ matrix.target == 'app' && vars.STAGING_APP_URL || vars.STAGING_API_URL }}
          rules_file_name: .zap/rules.tsv
          cmd_options: ${{ matrix.target == 'app' && '-a -j' || '-a' }}
          fail_action: true
          allow_issue_writing: false
          artifact_name: zap-${{ matrix.target }}

  deploy-production:
    needs: [build, zap-staging]   # Trivy + ZAP gates; manual approval via the "production" environment
    uses: ./.github/workflows/_deploy.yml
    with:
      environment: production
      source-account-id: "111122223333"
    permissions: { contents: read, id-token: write, actions: read }

.github/workflows/_deploy.yml

name: _deploy
on:
  workflow_call:
    inputs:
      environment:
        type: string
        required: true
      source-account-id:
        type: string
        required: true
        description: Account whose ECR holds the scanned build images

env:
  AWS_REGION: us-east-1
  PROJECT: acme
  KUBECTL_VERSION: v1.34.1          # keep within ±1 minor of the cluster
  ARGO_ROLLOUTS_VERSION: v1.8.0
  TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db

jobs:
  deploy:
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    environment:
      name: ${{ inputs.environment }}
      url: ${{ vars.APP_URL }}
    concurrency:
      group: deploy-${{ inputs.environment }}
      cancel-in-progress: false
    permissions: { contents: read, id-token: write, actions: read }
    steps:
      - uses: actions/checkout@v4

      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ env.PROJECT }}-gha-deployer
          role-session-name: gha-deploy-${{ inputs.environment }}-${{ github.run_id }}-${{ github.run_attempt }}
          aws-region: ${{ env.AWS_REGION }}

      - id: ecr
        uses: aws-actions/amazon-ecr-login@v2
        with:
          registries: ${{ inputs.source-account-id == vars.AWS_ACCOUNT_ID && vars.AWS_ACCOUNT_ID || format('{0},{1}', inputs.source-account-id, vars.AWS_ACCOUNT_ID) }}

      - uses: imjasonh/setup-crane@v0.4

      - name: Resolve digests & promote (byte-identical copy)
        id: images
        env:
          SRC_ACCOUNT: ${{ inputs.source-account-id }}
          DST_ACCOUNT: ${{ vars.AWS_ACCOUNT_ID }}
        run: |
          set -euo pipefail
          SRC_REG="${SRC_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com"
          DST_REG="${DST_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com"
          for app in api web; do
            repo="${PROJECT}/${app}"
            digest=$(aws ecr describe-images --registry-id "$SRC_ACCOUNT" --repository-name "$repo" \
                      --image-ids imageTag="$GITHUB_SHA" --query 'imageDetails[0].imageDigest' --output text)
            [[ "$digest" == sha256:* ]] || { echo "::error::$repo:$GITHUB_SHA not found in $SRC_ACCOUNT"; exit 1; }
            if [[ "$SRC_ACCOUNT" != "$DST_ACCOUNT" ]]; then
              existing=$(aws ecr describe-images --repository-name "$repo" --image-ids imageTag="$GITHUB_SHA" \
                          --query 'imageDetails[0].imageDigest' --output text 2>/dev/null || true)
              if [[ -z "$existing" || "$existing" == "None" ]]; then
                crane copy "${SRC_REG}/${repo}@${digest}" "${DST_REG}/${repo}:${GITHUB_SHA}"
              elif [[ "$existing" != "$digest" ]]; then
                echo "::error::digest mismatch for ${repo}:${GITHUB_SHA}"; exit 1
              fi
            fi
            echo "${app}_image=${DST_REG}/${repo}@${digest}" >> "$GITHUB_OUTPUT"
          done

      - name: Trivy re-scan api (prod gate, catches CVEs published since build)
        if: inputs.environment == 'production'
        uses: aquasecurity/trivy-action@0.28.0
        with:
          image-ref: ${{ steps.images.outputs.api_image }}
          severity: CRITICAL,HIGH
          ignore-unfixed: true
          exit-code: "1"
          trivyignores: .trivyignore
      - name: Trivy re-scan web
        if: inputs.environment == 'production'
        uses: aquasecurity/trivy-action@0.28.0
        with:
          image-ref: ${{ steps.images.outputs.web_image }}
          severity: CRITICAL,HIGH
          ignore-unfixed: true
          exit-code: "1"
          trivyignores: .trivyignore

      - name: Install kubectl + argo rollouts plugin
        env: { ARGO_ROLLOUTS_SHA256: "${{ vars.ARGO_ROLLOUTS_SHA256 }}" }
        run: scripts/install-tools.sh

      - name: Deploy (secrets → migrate → blue/green → CloudWatch analysis)
        id: deploy
        env:
          ENVIRONMENT: ${{ inputs.environment }}
          API_IMAGE: ${{ steps.images.outputs.api_image }}
          WEB_IMAGE: ${{ steps.images.outputs.web_image }}
          GIT_SHA: ${{ github.sha }}
        run: scripts/deploy.sh

      - name: Write immutable deployment record (Object Lock)
        if: always() && steps.images.outcome == 'success'
        env:
          GH_TOKEN: ${{ github.token }}
          ENVIRONMENT: ${{ inputs.environment }}
          OUTCOME: ${{ steps.deploy.outcome }}
          API_IMAGE: ${{ steps.images.outputs.api_image }}
          WEB_IMAGE: ${{ steps.images.outputs.web_image }}
          BUCKET: ${{ env.PROJECT }}-audit-${{ vars.AWS_ACCOUNT_ID }}
        run: |
          set -euo pipefail
          approvals=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/approvals" 2>/dev/null || echo '[]')
          jq -n \
            --arg env "$ENVIRONMENT" --arg outcome "$OUTCOME" --arg sha "$GITHUB_SHA" \
            --arg actor "$GITHUB_ACTOR" --arg api "$API_IMAGE" --arg web "$WEB_IMAGE" \
            --arg run "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}" \
            --argjson approvals "$approvals" \
            '{schema:1, timestamp:(now|todate), environment:$env, outcome:$outcome, commit:$sha,
              triggered_by:$actor, run:$run, images:{api:$api, web:$web},
              approvals:[$approvals[] | {user:.user.login, state, comment, environments:[.environments[].name]}]}' \
            > record.json
          key="deployments/${ENVIRONMENT}/$(date -u +%Y/%m/%d)/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
          aws s3api put-object --bucket "$BUCKET" --key "$key" --body record.json --checksum-algorithm SHA256
          { echo "### Deployment record"; echo '```json'; cat record.json; echo '```'; } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/infra.yml

name: infra
on:
  pull_request:
    paths: ["infra/**", ".github/workflows/infra.yml", "scripts/tg.sh"]
  push:
    branches: [main]
    paths: ["infra/**", ".github/workflows/infra.yml", "scripts/tg.sh"]
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: infra-${{ github.event_name }}-${{ github.ref }}
  cancel-in-progress: false

env:
  AWS_REGION: us-east-1
  PROJECT: acme
  TF_VERSION: 1.7.5
  TG_VERSION: 0.80.0        # pin to a verified release; bump deliberately
  TF_IN_AUTOMATION: "true"
  TG_NON_INTERACTIVE: "true"

jobs:
  fmt:
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v4
      - uses: hashicorp/setup-terraform@v3
        with: { terraform_version: "${{ env.TF_VERSION }}", terraform_wrapper: false }
      - run: terraform fmt -check -recursive infra/modules

  plan:
    if: github.event_name == 'pull_request'
    needs: fmt
    runs-on: ubuntu-24.04
    permissions: { contents: read, id-token: write }
    strategy:
      fail-fast: false
      matrix:
        include:
          - { env: staging, account: "111122223333" }
          - { env: production, account: "444455556666" }
    steps:
      - uses: actions/checkout@v4
      - uses: hashicorp/setup-terraform@v3
        with: { terraform_version: "${{ env.TF_VERSION }}", terraform_wrapper: false }
      - run: scripts/install-terragrunt.sh
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::${{ matrix.account }}:role/${{ env.PROJECT }}-gha-tf-plan
          role-session-name: gha-tfplan-${{ github.run_id }}
          aws-region: ${{ env.AWS_REGION }}
      # platform is excluded: the plan role has no cluster access; it is planned in the apply job.
      - run: scripts/tg.sh plan ${{ matrix.env }} network eks data edge observability 2>&1 | tee "$RUNNER_TEMP/plan.txt"
      - if: always()
        run: |
          { echo "### Plan: ${{ matrix.env }}"; echo '```';
            grep -E '^(Plan:|No changes|Error)' "$RUNNER_TEMP/plan.txt" || true; echo '```'; } >> "$GITHUB_STEP_SUMMARY"

  apply-staging:
    if: github.event_name != 'pull_request'
    needs: fmt
    runs-on: ubuntu-24.04
    environment: infra-staging
    permissions: { contents: read, id-token: write }
    steps:
      - uses: actions/checkout@v4
      - uses: hashicorp/setup-terraform@v3
        with: { terraform_version: "${{ env.TF_VERSION }}", terraform_wrapper: false }
      - run: scripts/install-terragrunt.sh
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ env.PROJECT }}-gha-tf-apply
          role-session-name: gha-tfapply-staging-${{ github.run_id }}
          aws-region: ${{ env.AWS_REGION }}
      - run: scripts/tg.sh apply staging

  apply-production:
    needs: apply-staging
    runs-on: ubuntu-24.04
    environment: infra-production     # required reviewers
    permissions: { contents: read, id-token: write }
    steps:
      - uses: actions/checkout@v4
      - uses: hashicorp/setup-terraform@v3
        with: { terraform_version: "${{ env.TF_VERSION }}", terraform_wrapper: false }
      - run: scripts/install-terragrunt.sh
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ env.PROJECT }}-gha-tf-apply
          role-session-name: gha-tfapply-production-${{ github.run_id }}
          aws-region: ${{ env.AWS_REGION }}
      - run: scripts/tg.sh apply production

5. Observability

Useful additions:

• Deploy impact query. This Logs Insights query compares error counts per release:

  stats count(*) as n, sum(status>=500) as errors by release | sort n desc
  

• CloudWatch dashboard. Put the TG 5xx rate and p95 alarms on one dashboard with vertical annotations at deploy times. You can source those from the S3 deployment records.
• Tracing (optional). Add ADOT with X-Ray (~$5–15/month at this scale) once you need latency breakdowns between the API, Postgres and Redis.
• Alert routing. Point the SNS topic at Slack (AWS Chatbot) or PagerDuty. Page on "endpoint down", "no healthy hosts" and "5xx rate". Send the rest to a ticket queue.

6. Setup & Run

1. Prerequisites. Install AWS CLI v2, Terraform 1.7.5, Terragrunt, kubectl and gh. Create SSO admin profiles acme-staging and acme-prod. Make sure the Route 53 zones exist, and that staging.example.com is delegated to the staging account. Update the REPLACE values and github_repo in both env.hcl files.

2. Bootstrap the CI identity and audit trail (human admin, once per account). Do prod ci-access first so its deployer role exists for the staging ECR pull policy. The state bucket and lock table are created on first run. Newer Terragrunt versions need --backend-bootstrap; older ones prompt instead.

   export AWS_PROFILE=acme-prod
   (cd infra/live/production/ci-access && terragrunt apply --backend-bootstrap)
   (cd infra/live/production/audit     && terragrunt apply)
   export AWS_PROFILE=acme-staging
   (cd infra/live/staging/ci-access && terragrunt apply --backend-bootstrap)
   (cd infra/live/staging/audit     && terragrunt apply)
   

3. First infrastructure apply (human, so you can watch it). Later changes go through infra.yml.

   AWS_PROFILE=acme-staging scripts/tg.sh apply staging
   AWS_PROFILE=acme-prod    scripts/tg.sh apply production
   

4. Populate the human-managed app secret in each account. Generate the password; never reuse it between environments.

   aws secretsmanager put-secret-value --profile acme-staging \
     --secret-id acme/staging/api-app \
     --secret-string "$(jq -n --arg p "$(openssl rand -base64 36 | tr -d '/+=')" \
        '{DB_APP_USER:"app_user", DB_APP_PASSWORD:$p, JWT_SECRET:"<generate>"}')"
   

   Your first migration must ensure the app role exists. Make it idempotent and read the values from DB_APP_USER / DB_APP_PASSWORD:

   DO $$ BEGIN
     IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'app_user') THEN CREATE ROLE app_user LOGIN; END IF;
   END $$;
   ALTER ROLE app_user PASSWORD '<DB_APP_PASSWORD, parameterized by your migration code>';
   GRANT CONNECT ON DATABASE app TO app_user;
   GRANT USAGE ON SCHEMA public TO app_user;
   GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO app_user;
   GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO app_user;
   ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO app_user;
   ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO app_user;
   

5. Configure GitHub environments, variables and protections.

   REPO=your-org/your-repo
   TEAM_ID=$(gh api orgs/your-org/teams/platform --jq .id)
   for e in staging infra-staging; do
     gh api -X PUT "repos/$REPO/environments/$e" --input - <<< \
       '{"deployment_branch_policy":{"protected_branches":true,"custom_branch_policies":false}}'
   done
   for e in production infra-production; do
     gh api -X PUT "repos/$REPO/environments/$e" --input - <<EOF
   {"prevent_self_review":true,"reviewers":[{"type":"Team","id":$TEAM_ID}],
    "deployment_branch_policy":{"protected_branches":true,"custom_branch_policies":false}}
   EOF
   done
   gh variable set AWS_ACCOUNT_ID --env staging          --body 111122223333
   gh variable set AWS_ACCOUNT_ID --env infra-staging    --body 111122223333
   gh variable set AWS_ACCOUNT_ID --env production       --body 444455556666
   gh variable set AWS_ACCOUNT_ID --env infra-production --body 444455556666
   gh variable set APP_URL --env staging    --body https://app.staging.example.com
   gh variable set APP_URL --env production --body https://app.example.com
   gh variable set STAGING_APP_URL --body https://app.staging.example.com
   gh variable set STAGING_API_URL --body https://api.staging.example.com
   gh variable set ARGO_ROLLOUTS_SHA256 --body "$(curl -fsSL \
     https://github.com/argoproj/argo-rollouts/releases/download/v1.8.0/kubectl-argo-rollouts-linux-amd64 | sha256sum | cut -d' ' -f1)"
   

   • Verify that ARGO_ROLLOUTS_SHA256 matches the checksum published on the release page.
   • Protect main: require PR review, require the ci checks and require linear history.
   • Add .github/CODEOWNERS entries for infra/, deploy/ and .github/.

6. First app deploy. Merge to main. On the very first rollout there is no stable version, so Argo skips analysis and promotes immediately. Do this before sending real traffic.

7. Game-day rollback test in staging. Deploy a build that returns 500s on 20% of requests. Confirm that the post-promotion analysis fails, Argo aborts, traffic returns to blue within about a minute, and the deploy job goes red with a deployment record showing outcome: failure.

7. Notes & Gotchas

Deploy safety

• Schema changes must be expand/contract. Migrations run before the swap, and blue keeps running on the new schema for 10–15 minutes and during any rollback. Never drop or rename a column in the same release that stops using it.
• API/web version skew. If one Rollout aborts and the other succeeds, the web and API versions diverge. Keep the API backward compatible for at least one release. Also keep Redis key formats and session shapes compatible, since both colours share the cache.
• Low-traffic analysis. With no requests there is no 5xx data, and the analysis passes. The Route 53 checks provide a baseline of traffic. In prod, consider a CloudWatch Synthetics canary (~$10/month) that exercises real API paths.
• ALB keep-alive. keepAliveTimeout must be greater than the ALB's 60-second idle timeout, otherwise you get random 502s. This is a very common cause of deploy-time 502s.
• Secret rotation.
  • RDS rotates the managed master password every 7 days. That is why the app uses app_user and only migrations use master credentials, force-synced before each run.
  • Pods don't pick up updated Secrets until they restart. Add Stakater Reloader if you rotate api-app secrets.
• TLS to Postgres. node-postgres treats sslmode=require like verify-full. Without the RDS CA bundle (NODE_EXTRA_CA_CERTS), connections fail.
• Capacity for blue-green. The node group is static and there is no autoscaler. Blue-green needs about 2× the peak app pods at promotion time. The prod sizing handles HPA max 8 for the API. For more headroom, add Karpenter; otherwise preview pods go Pending and the rollout stalls (with no downtime, but it doesn't proceed).
• NetworkPolicy. Verify in staging that kubelet probes aren't blocked by the VPC CNI policy agent before relying on this in prod. Egress is left open.

Security

• Public EKS endpoint. It is protected by IAM and access entries, and CI can only act inside the acme namespace. For stricter control, use self-hosted runners in the VPC (e.g. Actions Runner Controller) and set cluster_endpoint_public_access = false.
• tf-apply is admin, with guardrails. Explicit denies protect the CI identity, CloudTrail and the audit bucket. Approval is gated by environment. Next steps:
  • add a permissions boundary or SCPs if you adopt AWS Organizations;
  • customise the GitHub OIDC sub claim to include job_workflow_ref, so only deploy.yml and infra.yml can assume the roles.
• Terraform state holds the Redis auth token. Anyone who can open a PR can assume the plan role and read state. Restrict repo write access, or move Redis to ElastiCache IAM auth.
• Object Lock mode. GOVERNANCE can be bypassed by principals with s3:BypassGovernanceRetention, which tf-apply is explicitly denied. COMPLIANCE cannot be shortened even by root; switch to it only after you are sure of the retention period.
• Pin every action by SHA. Tags are mutable, and supply-chain attacks on Actions have happened. Also pin base images by digest and let Renovate bump them.
• WAF CommonRuleSet blocks request bodies over 8 KB (SizeRestrictions_BODY). Add a scoped exclusion if the API accepts uploads.
• Artifact attestations on private repos need GitHub Enterprise Cloud. Hence the ENABLE_ATTESTATIONS toggle.

Versions (verify before first apply)

• EKS version: pick the newest version in standard support.
• Helm chart pins: check the Kubernetes compatibility of each chart. ESO must be 0.16 or later for external-secrets.io/v1.
• Terragrunt release: TG_VERSION.
• kubectl: stay within ±1 minor of the cluster.
• EKS add-ons use most_recent = true. For strict pinning, resolve versions with aws eks describe-addon-versions and hardcode them.

SOC2 evidence (what an auditor will ask for)

• PR review and CODEOWNERS on main.
• GitHub environment approvals with prevent_self_review (separation of duties).
• Deployment records in Object Lock S3 (who, what digest, approver, outcome).
• CloudTrail with log-file validation.
• EKS audit logs.
• The kubectl exec alarm.
• Scan results from Trivy and ZAP artifacts.
• If you're on GitHub Enterprise, also stream the GitHub audit log.

Estimated monthly cost (us-east-1, on-demand list prices)

• Combined: about $900–1,000 per month, leaving roughly 60% of the budget as headroom.
• Biggest risks to the budget:
  • staying on an EKS version in extended support (+$365 per cluster);
  • unbounded Container Insights log volume (retention is already set);
  • NAT data charges from chatty egress.
• Further savings, if needed:
  • a single NAT gateway in prod (trades away AZ resilience);
  • Graceful Graviton nodes (m7g/t4g), which needs multi-arch builds;
  • serving the SPA from S3 + CloudFront instead of pods.