Production Patterns — Probes, Affinity & Pod Disruption

spec:

  containers:

    - name: app

      image: my-app:v1.2.3



      # startupProbe: gives slow-starting containers time to init

      # Checked every periodSeconds, up to failureThreshold times

      # Total grace: 30s * 10 = 5 minutes max startup time

      # ONLY checked during startup; disables liveness until it succeeds

      startupProbe:

        httpGet:

          path: /health/startup   # a fast endpoint that returns 200 when ready to start

          port: 8080

        failureThreshold: 30      # 30 * 10s = 5 minutes max

        periodSeconds: 10



      # readinessProbe: controls TRAFFIC routing (not restart)

      # If failing: removed from Service endpoints (no traffic)

      # If passing: added to Service endpoints (gets traffic)

      # ⚠️ Use for: dependency checks (DB connected?), circuit breaker open?

      readinessProbe:

        httpGet:

          path: /health/ready

          port: 8080

        initialDelaySeconds: 0     # startupProbe covers startup

        periodSeconds: 10

        failureThreshold: 3        # 3 failures = removed from endpoints

        successThreshold: 1        # 1 success = back in rotation



      # livenessProbe: triggers RESTART if failing

      # ⚠️ Be conservative — aggressive liveness = restart loops under load

      # Should ONLY fail if the process is genuinely stuck

      livenessProbe:

        httpGet:

          path: /health/live

          port: 8080

        periodSeconds: 15

        failureThreshold: 3        # 45 seconds to trigger restart

        timeoutSeconds: 5



# Common mistake: liveness probe checking DB connectivity

# If DB is down, liveness restarts ALL pods → more load on recovering DB → cascade failure

# Readiness: check DB (remove from traffic)

# Liveness:  only check if process is stuck (not external deps)

spec:

  template:

    spec:

      # Prefer spreading pods across zones (soft rule — best effort)

      affinity:

        podAntiAffinity:

          preferredDuringSchedulingIgnoredDuringExecution:

            - weight: 100

              podAffinityTerm:

                labelSelector:

                  matchLabels:

                    app: my-app

                topologyKey: topology.kubernetes.io/zone   # spread across AZs

          # Hard rule (never schedule 2 pods on same node):

          requiredDuringSchedulingIgnoredDuringExecution:

            - labelSelector:

                matchLabels:

                  app: my-app

              topologyKey: kubernetes.io/hostname           # unique per node



      # topologySpreadConstraints (modern, more powerful alternative)

      topologySpreadConstraints:

        - maxSkew: 1                         # max 1 pod difference between zones

          topologyKey: topology.kubernetes.io/zone

          whenUnsatisfiable: DoNotSchedule   # hard (ScheduleAnyway = soft)

          labelSelector:

            matchLabels:

              app: my-app

        - maxSkew: 1

          topologyKey: kubernetes.io/hostname

          whenUnsatisfiable: ScheduleAnyway  # prefer, but don't block

          labelSelector:

            matchLabels:

              app: my-app



# Result: 6 pods across 3 zones = 2 pods per zone

# Zone failure: only 2/6 pods lost (traffic served by remaining 4)

# PodDisruptionBudget — protect SLA during maintenance

apiVersion: policy/v1

kind: PodDisruptionBudget

metadata:

  name: my-app-pdb

spec:

  # minAvailable: minimum pods that MUST be available

  minAvailable: 2          # always keep at least 2 pods running



  # OR: maxUnavailable: maximum pods that CAN be unavailable

  # maxUnavailable: 1      # at most 1 pod can be down at a time



  selector:

    matchLabels:

      app: my-app



# Effect: kubectl drain node-1 will wait/fail if draining would

# violate the PDB (i.e., would reduce available pods below minAvailable)



# Node drain respects PDBs:

kubectl drain node-1 --ignore-daemonsets --delete-emptydir-data

# Evicting pod my-app-7f9c8d-abc12...

# eviction.policy/v1 PodDisruptionBudget: Cannot evict pod as it would

# violate the pod's disruption budget. ← PDB protected the pod



# For 3 replicas:

# minAvailable: 2 → at most 1 can be evicted at a time

# minAvailable: 1 → 2 can be evicted (risky)

# maxUnavailable: 0 → zero downtime maintenance (slowest drain)

# maxUnavailable: 1 → one pod at a time (most common)



# Check PDB status

kubectl get pdb

# NAME         MIN AVAILABLE  MAX UNAVAILABLE  ALLOWED DISRUPTIONS  AGE

# my-app-pdb   2              N/A              1                    5d