Prometheus 监控&报警

groups:
  - name: automq_customer_alerts
    rules:
      - alert: AutoMQBrokerCPUSafetyWatermarkHigh
        expr: max by (job) (automq_cpu_ratio) > 0.8
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Cluster {{ $labels.job }} reached 80% of AutoMQ's CPU safety watermark on its busiest broker"

      - alert: AutoMQBrokerNetworkUsageHigh
        expr: max by (job) (automq_network_ratio * 0.8) > 0.8
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Cluster {{ $labels.job }} network usage exceeded 80% on its busiest broker"

      - alert: AutoMQTopicWriteRateDrop
        expr: |
          (
            sum by (job, topic) (max by (job, topic, partition) (
              rate(kafka_network_io_bytes_total{direction="in", topic=~".+"}[5m])
            ))
            < 0.5 *
            sum by (job, topic) (max by (job, topic, partition) (
              rate(kafka_network_io_bytes_total{direction="in", topic=~".+"}[5m] offset 10m)
            ))
          )
          and on (job, topic)
          sum by (job, topic) (max by (job, topic, partition) (
            rate(kafka_network_io_bytes_total{direction="in", topic=~".+"}[5m] offset 10m)
          )) >= 5 * 1024 * 1024
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Topic {{ $labels.topic }} total write rate dropped in cluster {{ $labels.job }}"

      - alert: AutoMQConsumerTimeLagHigh
        expr: |
          max by (job, group, topic) (
            consumer_estimated_time_lag_milliseconds{group=~".+", topic=~".+"} >= 0
          ) > 120000
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Consumer group {{ $labels.group }} in cluster {{ $labels.job }} is more than 2 minutes behind on topic {{ $labels.topic }}"

      - alert: AutoMQConsumerTimeLagAboveYesterday
        expr: |
          (
            max by (job, group, topic) (
              consumer_estimated_time_lag_milliseconds{group=~".+", topic=~".+"} >= 0
            )
            -
            max by (job, group, topic) (
              consumer_estimated_time_lag_milliseconds{group=~".+", topic=~".+"} offset 1d >= 0
            )
          ) > 600000
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Consumer group {{ $labels.group }} in cluster {{ $labels.job }} exceeds yesterday's lag by 10 minutes on topic {{ $labels.topic }}"

groups:
  - name: connect_alerts
    rules:
      - alert: TasksFailed
        expr: sum(kafka_connect_worker_metrics_connector_failed_task_count{connect_instance_id=~"<your-connect-instance-id>", connector=~"<your-connector-name>"}) > 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Connector tasks failed for {{ $labels.connector }} in connect instance {{ $labels.connect_instance_id }}"
          description: "The number of failed tasks for connector {{ $labels.connector }} is greater than 0."

      - alert: CPUUsage
        expr: rate(jvm_cpu_time_seconds_total{connect_instance_id=~"<your-connect-instance-id>"}[5m]) > 0.8
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "High CPU usage {{ printf \"%0.2f\" $value }} for connect instance {{ $labels.connect_instance_id }}"
          description: "The CPU usage of connect instance {{ $labels.connect_instance_id }} is exceeding 80%."

      - alert: TotalRecordFailures
        expr: kafka_task_error_metrics_total_record_failures_total{connect_instance_id=~"<your-connect-instance-id>", connector=~"<your-connector-name>"} > 0
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "Record failures detected for connector {{ $labels.connector }} task {{ $labels.task }}"
          description: "The total record failures for connector {{ $labels.connector }} task {{ $labels.task }} is greater than 0."

      - alert: FailedAuthenticationConnections
        expr: kafka_connect_metrics_failed_authentication_total{connect_instance_id=~"<your-connect-instance-id>", client_id!=""} > 0
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "Failed authentication connections for client {{ $labels.client_id }} in connect instance {{ $labels.connect_instance_id }}"
          description: "The number of failed authentication connections for client {{ $labels.client_id }} is greater than 0."