Istio Observability Series (2/2): Golden Signals for the Control Plane — Monitoring istiod

# P99 xDS push time
histogram_quantile(0.99,
  sum(rate(pilot_xds_push_time_bucket[5m])) by (le)
)

# P99 total convergence time (config change → proxy updated)
histogram_quantile(0.99,
  sum(rate(pilot_proxy_convergence_time_bucket[5m])) by (le)
)

# Push queue wait time (indicates istiod is backlogged)
histogram_quantile(0.99,
  sum(rate(pilot_proxy_queue_time_bucket[5m])) by (le)
)

# Debounce time (how long istiod waits to batch changes)
histogram_quantile(0.50,
  sum(rate(pilot_debounce_time_bucket[5m])) by (le)
)

# Alert: xDS push time exceeds 10 seconds
- alert: SlowXDSPush
  expr: |
    histogram_quantile(0.99,
      sum(rate(pilot_xds_push_time_bucket[5m])) by (le)
    ) > 10
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: "istiod xDS push P99 >10s — sidecars are receiving stale config"

# Alert: Total convergence time exceeds 30 seconds
- alert: SlowConfigConvergence
  expr: |
    histogram_quantile(0.99,
      sum(rate(pilot_proxy_convergence_time_bucket[5m])) by (le)
    ) > 30
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: "Config convergence P99 >30s — config changes are severely delayed"

# Alert: Push queue backlog
- alert: XDSPushQueueBacklog
  expr: |
    histogram_quantile(0.99,
      sum(rate(pilot_proxy_queue_time_bucket[5m])) by (le)
    ) > 5
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: "istiod push queue backlogged — config updates are delayed"

# xDS pushes per second, by type
sum(rate(pilot_xds_pushes[5m])) by (type)

# Number of connected sidecars
pilot_xds

# Total services in the mesh
pilot_services

# Config size being pushed (helps identify bloated configs)
histogram_quantile(0.99,
  sum(rate(pilot_xds_config_size_bytes_bucket[5m])) by (le)
)

# Push rate trending (should be steady, spikes = config churn)
sum(rate(pilot_xds_pushes[5m]))

# Alert: Number of connected proxies dropped
- alert: ConnectedProxiesDrop
  expr: |
    pilot_xds < (pilot_xds offset 10m) * 0.8
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: "Connected proxy count dropped >20% — sidecars may be disconnecting"

# Alert: Excessive push rate (config churn)
- alert: ExcessiveXDSPushRate
  expr: |
    sum(rate(pilot_xds_pushes[5m])) > 100
  for: 10m
  labels:
    severity: warning
  annotations:
    summary: "istiod pushing >100 xDS updates/sec — check for config flapping"

# Total xDS rejections per second
sum(rate(pilot_total_xds_rejects[5m]))

# Rejections by type (which xDS resource is being rejected)
sum(rate(pilot_xds_cds_reject[5m]))  # Cluster config rejected
sum(rate(pilot_xds_lds_reject[5m]))  # Listener config rejected
sum(rate(pilot_xds_eds_reject[5m]))  # Endpoint config rejected
sum(rate(pilot_xds_rds_reject[5m]))  # Route config rejected

# Internal errors in istiod (bugs or resource exhaustion)
sum(rate(pilot_total_xds_internal_errors[5m]))

# Listener conflicts (common cause of subtle routing bugs)
pilot_conflict_inbound_listener
pilot_conflict_outbound_listener_http_over_current_tcp

# Config validation failures at admission
sum(rate(galley_validation_failed[5m]))

# Alert: Any xDS rejections (proxy is refusing config)
- alert: XDSConfigRejected
  expr: |
    sum(rate(pilot_total_xds_rejects[5m])) > 0
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: "Proxies are rejecting xDS config — check istiod logs for invalid configuration"

# Alert: istiod internal errors
- alert: IstiodInternalErrors
  expr: |
    sum(rate(pilot_total_xds_internal_errors[5m])) > 0
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: "istiod is experiencing internal errors"

# Alert: Listener port conflicts
- alert: ListenerConflicts
  expr: pilot_conflict_inbound_listener > 0
  for: 1m
  labels:
    severity: warning
  annotations:
    summary: "Port conflicts detected — services may have overlapping ports"

# Find which proxies are rejecting config
kubectl logs deploy/istiod -n istio-system | grep -i "nack"

# Check a specific proxy's sync status
istioctl proxy-status

# Output shows SYNCED vs NOT SENT vs STALE
# NAME             CDS    LDS    EDS    RDS    ECDS   ISTIOD
# pod-abc.default  SYNCED SYNCED SYNCED SYNCED        istiod-xyz
# pod-def.default  SYNCED STALE  SYNCED SYNCED        istiod-xyz
#                         ^^^^^
#                         This proxy has stale listener config

# istiod CPU usage (cores)
rate(process_cpu_seconds_total{app="istiod"}[5m])

# istiod memory (MB)
go_memstats_alloc_bytes{app="istiod"} / 1024 / 1024

# Goroutine count (scales with connected proxies)
go_goroutines{app="istiod"}

# Certificate issuance rate
sum(rate(citadel_server_success_cert_issuance_count[5m]))

# CSR failures (certificate rotation problems)
sum(rate(citadel_server_csr_parsing_err_count[5m]))

# Ratio: proxies per istiod replica (sizing metric)
pilot_xds / count(up{app="istiod"})

# Alert: istiod CPU is saturated
- alert: IstiodHighCPU
  expr: |
    rate(process_cpu_seconds_total{app="istiod"}[5m]) > 0.8
  for: 10m
  labels:
    severity: warning
  annotations:
    summary: "istiod CPU utilization >80% — consider scaling istiod replicas"

# Alert: istiod memory pressure
- alert: IstiodHighMemory
  expr: |
    go_memstats_alloc_bytes{app="istiod"} > 2 * 1024 * 1024 * 1024
  for: 10m
  labels:
    severity: warning
  annotations:
    summary: "istiod memory usage >2GB — check for config bloat or proxy count"

# Alert: Certificate signing failures
- alert: CertSigningFailures
  expr: |
    sum(rate(citadel_server_csr_parsing_err_count[5m])) > 0
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: "Certificate signing failures — mTLS certificate rotation may be broken"

# Alert: Too many proxies per istiod (scaling threshold)
- alert: IstiodOverloaded
  expr: |
    pilot_xds / count(up{app="istiod"}) > 1000
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: "istiod managing >1000 proxies per replica — consider horizontal scaling"

# Certificate issuance success rate
sum(rate(citadel_server_success_cert_issuance_count[5m]))
/
sum(rate(citadel_server_csr_count[5m]))

# Time until root CA expires (days)
(citadel_server_root_cert_expiry_timestamp - time()) / 86400

# CSR failures
sum(rate(citadel_server_csr_parsing_err_count[5m]))
+ sum(rate(citadel_server_authentication_failure_count[5m]))

# Alert: Root CA expires within 30 days
- alert: RootCAExpiringSoon
  expr: |
    (citadel_server_root_cert_expiry_timestamp - time()) / 86400 < 30
  for: 1h
  labels:
    severity: critical
  annotations:
    summary: "Istio root CA expires in {{ $value | humanizeDuration }} — rotate immediately"

# Alert: Certificate issuance failures
- alert: CertIssuanceFailures
  expr: |
    sum(rate(citadel_server_csr_parsing_err_count[5m]))
    + sum(rate(citadel_server_authentication_failure_count[5m]))
    > 0
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: "Certificate issuance failures — pods may lose mTLS connectivity"

# Count config objects
kubectl get virtualservices --all-namespaces --no-headers | wc -l
kubectl get destinationrules --all-namespaces --no-headers | wc -l
kubectl get serviceentries --all-namespaces --no-headers | wc -l
kubectl get envoyfilters --all-namespaces --no-headers | wc -l