基于 Prometheus 构建黑盒监控体系（已用于线上环境）

[root@cinder1 src]# wget https://github.com/prometheus/blackbox_exporter/releases/download/v0.16.0/blackbox_exporter-0.16.0.linux-amd64.tar.gz[root@cinder1 src]#tar -zxvf blackbox_exporter-0.16.0.linux-amd64.tar.gz -C /usr/local[root@cinder1 src]#mv /usr/local/blackbox_exporter-0.16.0.linux-amd64 /usr/local/blackbox_exporter

[root@cinder1 src]# cat /etc/systemd/system/blackbox_exporter.service [Unit]Description=blackbox_exporterAfter=network.target 
[Service]WorkingDirectory=/usr/local/blackboxExecStart=/usr/local/blackbox/blackbox_exporter \         --config.file=/usr/local/blackbox/blackbox.yml[Install]WantedBy=multi-user.target

[root@cinder1 src]# ss -tunlp|grep 9115tcp    LISTEN     0      128      :::9115                 :::*                   users:(("blackbox_export",pid=2517722,fd=3))

- job_name: "icmp_ping"    metrics_path: /probe    params:      module: [icmp]  # 使用icmp模块    file_sd_configs:    - refresh_interval: 10s      files:      - "/home/prometheus/conf/ping_status*.yml"  #具体的配置文件    relabel_configs:    - source_labels: [__address__]      regex: (.*)(:80)?      target_label: __param_target      replacement: ${1}    - source_labels: [__param_target]      target_label: instance    - source_labels: [__param_target]      regex: (.*)      target_label: ping      replacement: ${1}    - source_labels: []      regex: .*      target_label: __address__      replacement: 192.168.1.14:9115

[root@cinder1 conf]# cat ping_status.yml - targets: ['220.181.38.150','14.215.177.39','180.101.49.12','14.215.177.39','180.101.49.11','14.215.177.38','14.215.177.38']  labels:    group: '一线城市-电信网络监控'- targets: ['112.80.248.75','163.177.151.109','61.135.169.125','163.177.151.110','180.101.49.11','61.135.169.121','180.101.49.11']  labels:    group: '一线城市-联通网络监控'- targets: ['183.232.231.172','36.152.44.95','182.61.200.6','36.152.44.96','220.181.38.149']  labels:    group: '一线城市-移动网络监控' 

  - job_name: "blackbox"    metrics_path: /probe    params:      module: [http_2xx]  #使用http模块    file_sd_configs:     - refresh_interval: 1m      files:       - "/home/prometheus/conf/blackbox*.yml"    relabel_configs:    - source_labels: [__address__]      target_label: __param_target    - source_labels: [__param_target]      target_label: instance    - target_label: __address__      replacement: 192.168.1.14:9115

[root@cinder1 conf]# cat /home/prometheus/conf/blackbox-dis.yml - targets:  - https://www.zhibo8.cc  - https://www.baidu.com#配置相关URL

  - job_name: "check_get"    metrics_path: /probe    params:      module: [http_2xx]  # Look for a HTTP 200 response.    file_sd_configs:    - refresh_interval: 1m      files:      - "/home/prometheus/conf/service_get.yml"    relabel_configs:    - source_labels: [__address__]      target_label: __param_target    - source_labels: [__param_target]      target_label: instance    - target_label: __address__      replacement: 192.168.1.14:9115

[root@cinder1 conf]# cat service_get.yml - targets:  - http://10.10.1.123:10000/pmkb/atc_tcbi  - http://10.10.1.123:10000/pmkb/get_ship_lock_count  - http://10.10.1.123:10000/pmkb/get_terminal_count_by_city  - http://10.10.1.123:10000/pmkb/get_terminal_monitor?industry=1  - http://10.10.1.123:10000/pmkb/get_terminal_comparison?industry=1  - http://10.10.1.123:10000/pmkb/get_terminal_city_count_industry?industry=1  - http://10.10.1.123:10000/pmkb/industry_stat?industry=1  - http://10.10.1.123:10000/pmkb/get_company_car_count?industry=1  - http://10.10.1.123:10000/pmkb/get_terminal_month_countbyi?industry=1  labels:    group: 'service'

[root@cinder1 blackbox]# cat blackbox.yml modules:  http_2xx:    prober: http  http_post_2xx:   #这个模块名称可以自己定义    prober: http    http:      method: POST      headers:        Content-Type: application/json   #添加头部      body: '{"username":"admin","password":"123456"}'  #发送的相关数据，这里我们以登录接口为例

  - job_name: "check_service"    metrics_path: /probe    params:      module: [http_post_2xx]  # 这里要对应配置文件里，定义的模块    file_sd_configs:     - refresh_interval: 1m      files:       - "/home/prometheus/conf/service_post.yml"    relabel_configs:    - source_labels: [__address__]      target_label: __param_target    - source_labels: [__param_target]      target_label: instance    - target_label: __address__      replacement: 192.168.1.14:9115

[root@cinder1 conf]# cat service_post.yml - targets:  - http://10.2.4.103:5000/devops/api/v1.0/login  labels:    group: 'service'

  - job_name: 'port_status'    metrics_path: /probe    params:      module: [tcp_connect]  #使用tcp模块    static_configs:      - targets: ['10.10.1.35:8068','10.10.1.35:8069']  #对应主机接口        labels:          instance: 'port_status'          group: 'tcp'    relabel_configs:    - source_labels: [__address__]      target_label: __param_target     - target_label: __address__      replacement: 192.168.1.14:9115 

[root@cinder1 rules]# cat blackbox.yml groups:- name: blackbox_network_stats  rules:  - alert: blackbox_network_stats    expr: probe_success == 0    for: 1m    labels:      severity: critical    annotations:      summary: "接口/主机/端口 {{ $labels.instance }}  无法联通"      description: "请尽快检测"

[root@cinder1 rules]# cat ssl.yml groups:- name: check_ssl_status  rules:  - alert: "ssl证书过期警告"    expr: (probe_ssl_earliest_cert_expiry - time())/86400 <30    for: 1h    labels:      severity: warn    annotations:      description: '域名{{$labels.instance}}的证书还有{{ printf "%.1f" $value }}天就过期了,请尽快更新证书'      summary: "ssl证书过期警告"