Prometheus监控ElasticSearch

下载exporter

下载地址：https://github.com/justwatchcom/elasticsearch_exporter/releases

tar zxvf elasticsearch_exporter-1.1.0.linux-amd64.tar.gz
cd elasticsearch_exporter-1.1.0.linux-amd64
cp elasticsearch_exporter /usr/local/bin
# 启动
nohup elasticsearch_exporter --es.uri=http://localhost:9200 --es.all --es.cluster_settings --es.indices --es.indices_settings --es.shards --es.snapshots >> logs/elasticsearch_exporter.log &

查看监控的指标

curl -s http://localhost:9114/metrics |grep elasticsearch_cluster_health_status

配置prometheus

增加es指标搜集配置

- job_name: elasticsearch
  static_configs:
    - targets: ['192.168.1.112:9114']

配置grafana监控模板

导入grafana模板id：6483

配置rules

groups:
- name: mysqld_rules
  rules:
  - alert: ElasticsearchHeapUsageTooHigh
    expr: 100 * elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"} > 90
    for: 2m
    labels:
      severity: critical
    annotations:
      summary: Elasticsearch Heap Usage Too High (instance {{ $labels.instance }})
      description: The heap usage is over 90%\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}
  - alert: ElasticsearchHeapUsageWarning
    expr: 100 * elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"} > 80
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: Elasticsearch Heap Usage warning (instance {{ $labels.instance }})
      description: The heap usage is over 80%\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}
  - alert: ElasticsearchDiskOutOfSpace
    expr: 100 * elasticsearch_filesystem_data_available_bytes / elasticsearch_filesystem_data_size_bytes  < 10
    for: 2m
    labels:
      severity: critical
    annotations:
      summary: Elasticsearch disk out of space (instance {{ $labels.instance }})
      description: The disk usage is over 90%\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}
  - alert: ElasticsearchDiskSpaceLow
    expr: 100 * elasticsearch_filesystem_data_available_bytes / elasticsearch_filesystem_data_size_bytes  < 20
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: Elasticsearch disk space low (instance {{ $labels.instance }})
      description: The disk usage is over 80%\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}
  - alert: ElasticsearchClusterRed
    expr: elasticsearch_cluster_health_status{color="red"} == 1
    for: 0m
    labels:
      severity: critical
    annotations:
      summary: Elasticsearch Cluster Red (instance {{ $labels.instance }})
      description: Elastic Cluster Red status\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}
  - alert: ElasticsearchClusterYellow
    expr: elasticsearch_cluster_health_status{color="yellow"} == 1
    for: 3m
    labels:
      severity: warning
    annotations:
      summary: Elasticsearch Cluster Yellow (instance {{ $labels.instance }})
      description: Elastic Cluster Yellow status\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}
  - alert: ElasticsearchHealthyNodes
    expr: elasticsearch_cluster_health_number_of_nodes < 3
    for: 0m
    labels:
      severity: critical
    annotations:
      summary: Elasticsearch Healthy Nodes (instance {{ $labels.instance }})
      description: Missing node in Elasticsearch cluster\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}
  - alert: ElasticsearchHealthyDataNodes
    expr: elasticsearch_cluster_health_number_of_data_nodes < 3
    for: 0m
    labels:
      severity: critical
    annotations:
      summary: Elasticsearch Healthy Data Nodes (instance {{ $labels.instance }})
      description: Missing data node in Elasticsearch cluster\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}
  - alert: ElasticsearchRelocatingShards
    expr: elasticsearch_cluster_health_relocating_shards > 0
    for: 0m
    labels:
      severity: info
    annotations:
      summary: Elasticsearch relocating shards (instance {{ $labels.instance }})
      description: Elasticsearch is relocating shards\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}
  - alert: ElasticsearchRelocatingShardsTooLong
    expr: elasticsearch_cluster_health_relocating_shards > 0
    for: 15m
    labels:
      severity: warning
    annotations:
      summary: Elasticsearch relocating shards too long (instance {{ $labels.instance }})
      description: Elasticsearch has been relocating shards for 15min\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}
  - alert: ElasticsearchInitializingShards
    expr: elasticsearch_cluster_health_initializing_shards > 0
    for: 0m
    labels:
      severity: info
    annotations:
      summary: Elasticsearch initializing shards (instance {{ $labels.instance }})
      description: Elasticsearch is initializing shards\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}
  - alert: ElasticsearchInitializingShardsTooLong
    expr: elasticsearch_cluster_health_initializing_shards > 0
    for: 15m
    labels:
      severity: warning
    annotations:
      summary: Elasticsearch initializing shards too long (instance {{ $labels.instance }})
      description: Elasticsearch has been initializing shards for 15 min\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}
  - alert: ElasticsearchUnassignedShards
    expr: elasticsearch_cluster_health_unassigned_shards > 0
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: Elasticsearch unassigned shards (instance {{ $labels.instance }})
      description: Elasticsearch has unassigned shards\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}
  - alert: ElasticsearchPendingTasks
    expr: elasticsearch_cluster_health_number_of_pending_tasks > 0
    for: 15m
    labels:
      severity: warning
    annotations:
      summary: Elasticsearch pending tasks (instance {{ $labels.instance }})
      description: Elasticsearch has pending tasks. Cluster works slowly.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}
  - alert: ElasticsearchNoNewDocuments
    expr: increase(elasticsearch_indices_docs{es_data_node="true"}[10m]) < 1
    for: 0m
    labels:
      severity: warning
    annotations:
      summary: Elasticsearch no new documents (instance {{ $labels.instance }})
      description: No new documents for 10 min!\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}