Metrics & Service Monitoring

# PromQL examples for RED metrics (Spring Boot / Micrometer)

# Rate: requests per second (5-min average)
rate(http_server_requests_seconds_count{job="order-service"}[5m])

# Error rate: 5xx errors as % of total requests
sum(rate(http_server_requests_seconds_count{status=~"5.."}[5m]))
  / sum(rate(http_server_requests_seconds_count[5m])) * 100

# Duration: P99 latency for POST /orders
histogram_quantile(0.99,
  sum(rate(http_server_requests_seconds_bucket{
    uri="/orders", method="POST"}[5m])) by (le))

# USE metrics (infrastructure)
# Utilisation: JVM thread pool active vs max
hikaricp_connections_active / hikaricp_connections_max

# Saturation: pending (queued) HikariCP connection requests
hikaricp_connections_pending

# JVM memory utilisation
jvm_memory_used_bytes{area="heap"} / jvm_memory_max_bytes{area="heap"}

<!-- pom.xml — add Prometheus registry -->
<dependency>
    <groupId>io.micrometer</groupId>
    <artifactId>micrometer-registry-prometheus</artifactId>
</dependency>

# application.yml — expose Prometheus endpoint
management:
  endpoints:
    web:
      exposure:
        include: health,info,prometheus,metrics
  metrics:
    distribution:
      percentiles-histogram:
        http.server.requests: true   # enable histogram for quantile queries
      percentiles:
        http.server.requests: 0.5,0.95,0.99
    tags:
      application: ${spring.application.name}   # tag all metrics with app name
      environment: ${spring.profiles.active}

# Custom business metric example
@Service
public class OrderService {
    private final Counter ordersPlaced;

    public OrderService(MeterRegistry registry) {
        ordersPlaced = Counter.builder("orders.placed")
            .description("Total orders placed")
            .tag("channel", "web")
            .register(registry);
    }

    public Order placeOrder(OrderRequest req) {
        Order order = processOrder(req);
        ordersPlaced.increment();
        return order;
    }
}

# prometheus-rules.yml — alerting rules
groups:
  - name: order-service
    rules:
      - alert: HighErrorRate
        expr: |
          sum(rate(http_server_requests_seconds_count{
            job="order-service", status=~"5.."}[5m]))
          / sum(rate(http_server_requests_seconds_count{job="order-service"}[5m]))
          > 0.05
        for: 2m    # must hold for 2 min to fire (avoids transient spikes)
        labels:
          severity: critical
          team: platform
        annotations:
          summary: "High error rate on order-service"
          description: "Error rate is {{ humanizePercentage $value }} (threshold: 5%)"
          runbook: "https://wiki.example.com/runbooks/order-service-errors"

      - alert: P99LatencyHigh
        expr: |
          histogram_quantile(0.99,
            sum(rate(http_server_requests_seconds_bucket{
              job="order-service"}[5m])) by (le))
          > 2.0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "P99 latency above 2s on order-service"