loki, promtail & prometheus
requirements:
- running grafana instance (see: Grafana & influxDB on Almalinux)
get latest release from Grafana realease page
# get loki
> cd /tmp \
&& curl -OL \
"https://github.com/grafana/loki/releases/download/v2.4.1/loki-linux-amd64.zip" \
&& curl -OL \
"https://github.com/grafana/loki/releases/download/v2.4.1/promtail-linux-amd64.zip"
> unzip "loki-linux-amd64.zip"
> chmod a+x "loki-linux-amd64"
> unzip "promtail-linux-amd64.zip"
> chmod a+x "promtail-linux-amd64.zip"
> cp /tmp/loki-linux-amd64 /usr/local/bin/loki
> cp /tmp/promtail-linux-amd64 /usr/local/bin/promtail
get configfiles
# loki
> mkdir -p /etc/grafana/loki \
&& cd /etc/grafana/loki \
&& wget \
https://raw.githubusercontent.com/grafana/loki/master/cmd/loki/loki-local-config.yaml \
&& wget \
https://raw.githubusercontent.com/grafana/loki/main/clients/cmd/promtail/promtail-local-config.yaml
test start
# loki
> loki -config.file=loki-local-config.yaml
# promtail
> promtail -config.file=promtail-local-config.yaml
# add grafana source "loki" with URL http://localhost:3100
systemd unit - loki - /etc/systemd/system/loki.service
[Unit]
Description=Loki service
After=network.target
[Service]
Type=simple
ExecStart=/usr/local/bin/loki --config.file /etc/grafana/loki/loki-local-config.yaml
[Install]
WantedBy=multi-user.target
systemd unit - promtail - /etc/systemd/system/promtail.service
[Unit]
Description=Promtail service
After=network.target
[Service]
Type=simple
ExecStart=/usr/local/bin/promtail --config.file /etc/grafana/loki/promtail-local-config.yaml
[Install]
WantedBy=multi-user.target
start and enable loki services
> systemctl start loki \
&& systemctl start promtail
> systemctl enable loki \
&& systemctl enable promtail
install node_exporter
> dnf install golang-github-prometheus-node-exporter
> edit /etc/sysconfig/node_exporter
OPTIONS="--collector.textfile.directory /var/lib/node_exporter/textfile_collector --web.config=/var/lib/node_exporter/web.yml
> openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 \
-keyout /var/lib/node_exporter/node-exporter.key \
-out /var/lib/node_exporter/node-exporter.crt \
-subj "/C=DE/ST=SA/L=Magdeburg/O=x33u.org/CN=ho-monitor-02" \
-addext "subjectAltName = DNS:ho-monitor-02"
create password using bcrypt
> dnf install python3-bcrypt
> edit gen-pass.py
import getpass
import bcrypt
password = getpass.getpass("password: ")
hashed_password = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt())
print(hashed_password.decode())
> python gen-pass.py
> edit /var/lib/node_exporter/web.yml
tls_server_config:
cert_file: /var/lib/node_exporter/node-exporter.crt
key_file: /var/lib/node_exporter/node-exporter.key
basic_auth_users:
nodeexporter-login: $2b$12xxxxxxxx
> chown -R node_exporter:node_exporter /var/lib/node_exporter
> systemctl start node_exporter \
&& systemctl enable node_exporter
> firewall-cmd --permanent \
--add-rich-rule='rule family="ipv4" source address="192.168.2.8" port protocol="tcp" port="9100" accept'
> firewall-cmd --reload
install prometheus
> dnf install golang-github-prometheus
> edit /etc/sysconfig/prometheus
CONFIG_FILE=/etc/prometheus/prometheus.yml
STORAGE_TSDB_PATH=/var/lib/prometheus
WEB_CONSOLE_LIBRARIES_PATH=/etc/prometheus/console_libraries
WEB_CONSOLE_TEMPLATES_PATH=/etc/prometheus/consoles
WEB_LISTEN_ADDRESS=127.0.0.1:9090 #ip address
WEB_CONFIG=/etc/prometheus/web.yml
> edit /etc/prometheus/web.yml
tls_server_config:
cert_file: /etc/prometheus/node-exporter.crt
key_file: /etc/prometheus/node-exporter.key
basic_auth_users:
operator-login: $2b$12$xxxxxxx
> edit /usr/lib/systemd/system/prometheus.service
...
ExecStart=/usr/bin/prometheus \
--config.file=${CONFIG_FILE} \
--storage.tsdb.path=${STORAGE_TSDB_PATH} \
--web.console.libraries=${WEB_CONSOLE_LIBRARIES_PATH} \
--web.console.templates=${WEB_CONSOLE_TEMPLATES_PATH} \
--web.listen-address=${WEB_LISTEN_ADDRESS} \
--web.config.file=${WEB_CONFIG}
> systemctl daemon-reload
> mkdir -p /etc/prometheus/ssl
> openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 \
-keyout /etc/prometheus/ssl/node-exporter.key \
-out /etc/prometheus/ssl/node-exporter.crt \
-subj "/C=DE/ST=SA/L=Magdeburg/O=x33u.org/CN=ho-monitor-01" \
-addext "subjectAltName = DNS:ho-streambox-01"
> chown -R prometheus:prometheus /etc/prometheus/ssl
> edit /etc/prometheus/prometheus.yml
...
scrape_configs:
- job_name: 'prometheus'
scheme: https
tls_config:
ca_file: /etc/prometheus/node-exporter.crt
insecure_skip_verify: true
basic_auth:
username: prometheus-login
password: <PASSWORD--NO-HASH>
static_configs:
- targets: ['localhost:9090']
- job_name: 'node1'
scheme: https
tls_config:
ca_file: /etc/prometheus/node-exporter.crt
insecure_skip_verify: true
basic_auth:
username: nodeexporter-login
password: <PASSWORD--NO-HASH>
static_configs:
- targets: ['127.0.0.1:9100']
> firewall-cmd --zone=public --add-port=9090/tcp --permanent
> firewall-cmd --reload
start and enable services
> systemctl start prometheus \
&& systemctl enable prometheus
promtail example
server:
# http_listen_address: 192.168.2.10
http_listen_port: 9080
grpc_listen_port: 0
positions:
filename: /tmp/positions.yaml
clients:
- url: http://192.168.2.8:3100/loki/api/v1/push
scrape_configs:
- job_name: system
pipeline_stages:
- match:
selector: '{filename=~".*fail2ban.log"}'
stages:
- regex:
expression:
"(?P<time>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{0,3})?\\D?
(?P<message>fail2ban.*(?P<pid>\\[\\d*\\]: )(?P<level>[A-Z]{4,7}) .*
(?:(?:\\[|Jail ')(?P<jail>\\D*)(?:\\]|'))?.*)"
- template:
source: message
template: '{{ Replace .Value .pid "" -1 }}'
- regex:
expression: '(?P<message>.*)(?: - \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})'
source: message
- template:
source: time
template: '{{ Replace .Value "," "." -1 }}'
- timestamp:
source: time
format: "2006-01-02 15:04:05.000"
- output:
source: message
static_configs:
- targets:
- localhost
labels:
job: ho-mainframe-03-varlogs
__path__: /var/log/**/**log
- job_name: journal
journal:
max_age: 12h
labels:
job: ho-mainframe-03/systemd-journal
relabel_configs:
- source_labels: ['__journal__systemd_unit']
target_label: 'unit
26-09-2021