loki, promtail & prometheus :: x33u

requirements:

running grafana instance (see: Grafana & influxDB on Almalinux)

get latest release from Grafana realease page

# get loki
> cd /tmp \
  &&  curl -OL \
  "https://github.com/grafana/loki/releases/download/v2.4.1/loki-linux-amd64.zip" \
  && curl -OL \
  "https://github.com/grafana/loki/releases/download/v2.4.1/promtail-linux-amd64.zip"

> unzip "loki-linux-amd64.zip"
> chmod a+x "loki-linux-amd64"

> unzip "promtail-linux-amd64.zip"
> chmod a+x "promtail-linux-amd64.zip"

> cp /tmp/loki-linux-amd64 /usr/local/bin/loki
> cp /tmp/promtail-linux-amd64 /usr/local/bin/promtail

get configfiles

# loki
> mkdir -p /etc/grafana/loki \
  && cd /etc/grafana/loki \
  && wget \
  https://raw.githubusercontent.com/grafana/loki/master/cmd/loki/loki-local-config.yaml \
  && wget \
  https://raw.githubusercontent.com/grafana/loki/main/clients/cmd/promtail/promtail-local-config.yaml

test start

# loki
> loki -config.file=loki-local-config.yaml

# promtail
> promtail -config.file=promtail-local-config.yaml

# add grafana source "loki" with URL http://localhost:3100

systemd unit - loki - /etc/systemd/system/loki.service

[Unit]
Description=Loki service
After=network.target

[Service]
Type=simple
ExecStart=/usr/local/bin/loki --config.file  /etc/grafana/loki/loki-local-config.yaml

[Install]
WantedBy=multi-user.target

systemd unit - promtail - /etc/systemd/system/promtail.service

[Unit]
Description=Promtail service
After=network.target

[Service]
Type=simple
ExecStart=/usr/local/bin/promtail --config.file  /etc/grafana/loki/promtail-local-config.yaml

[Install]
WantedBy=multi-user.target

start and enable loki services

> systemctl start loki \
  && systemctl start promtail
> systemctl enable loki \
  && systemctl enable promtail

install node_exporter

> dnf install golang-github-prometheus-node-exporter

> edit /etc/sysconfig/node_exporter
OPTIONS="--collector.textfile.directory /var/lib/node_exporter/textfile_collector --web.config=/var/lib/node_exporter/web.yml

> openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 \
  -keyout /var/lib/node_exporter/node-exporter.key \
  -out /var/lib/node_exporter/node-exporter.crt \
  -subj "/C=DE/ST=SA/L=Magdeburg/O=x33u.org/CN=ho-monitor-02" \
  -addext "subjectAltName = DNS:ho-monitor-02"

create password using bcrypt

> dnf install python3-bcrypt
> edit gen-pass.py

import getpass
import bcrypt

password = getpass.getpass("password: ")
hashed_password = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt())
print(hashed_password.decode())

> python gen-pass.py

> edit /var/lib/node_exporter/web.yml
tls_server_config:
  cert_file: /var/lib/node_exporter/node-exporter.crt
  key_file: /var/lib/node_exporter/node-exporter.key

basic_auth_users:
    nodeexporter-login: $2b$12xxxxxxxx

> chown -R node_exporter:node_exporter /var/lib/node_exporter

> systemctl start node_exporter \
  &&  systemctl enable node_exporter

> firewall-cmd --permanent \
  --add-rich-rule='rule family="ipv4" source address="192.168.2.8" port protocol="tcp" port="9100" accept'
> firewall-cmd --reload

install prometheus

> dnf install golang-github-prometheus

> edit /etc/sysconfig/prometheus
CONFIG_FILE=/etc/prometheus/prometheus.yml
STORAGE_TSDB_PATH=/var/lib/prometheus
WEB_CONSOLE_LIBRARIES_PATH=/etc/prometheus/console_libraries
WEB_CONSOLE_TEMPLATES_PATH=/etc/prometheus/consoles
WEB_LISTEN_ADDRESS=127.0.0.1:9090 #ip address
WEB_CONFIG=/etc/prometheus/web.yml

> edit /etc/prometheus/web.yml
tls_server_config:
  cert_file: /etc/prometheus/node-exporter.crt
  key_file: /etc/prometheus/node-exporter.key

basic_auth_users:
  operator-login: $2b$12$xxxxxxx

> edit /usr/lib/systemd/system/prometheus.service
...
ExecStart=/usr/bin/prometheus \
          --config.file=${CONFIG_FILE} \
          --storage.tsdb.path=${STORAGE_TSDB_PATH} \
          --web.console.libraries=${WEB_CONSOLE_LIBRARIES_PATH} \
          --web.console.templates=${WEB_CONSOLE_TEMPLATES_PATH} \
          --web.listen-address=${WEB_LISTEN_ADDRESS} \
          --web.config.file=${WEB_CONFIG}

> systemctl daemon-reload

> mkdir -p /etc/prometheus/ssl

> openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 \
  -keyout /etc/prometheus/ssl/node-exporter.key \
  -out /etc/prometheus/ssl/node-exporter.crt \
  -subj "/C=DE/ST=SA/L=Magdeburg/O=x33u.org/CN=ho-monitor-01" \
  -addext "subjectAltName = DNS:ho-streambox-01"

> chown -R prometheus:prometheus /etc/prometheus/ssl

> edit /etc/prometheus/prometheus.yml
...
scrape_configs:
  - job_name: 'prometheus'
    scheme: https
    tls_config:
      ca_file: /etc/prometheus/node-exporter.crt
      insecure_skip_verify: true
    basic_auth:
      username: prometheus-login
      password: <PASSWORD--NO-HASH>
    static_configs:
    - targets: ['localhost:9090']
  - job_name: 'node1'
    scheme: https
    tls_config:
      ca_file: /etc/prometheus/node-exporter.crt
      insecure_skip_verify: true
    basic_auth:
      username: nodeexporter-login
      password: <PASSWORD--NO-HASH>
    static_configs:
    - targets: ['127.0.0.1:9100']

> firewall-cmd --zone=public --add-port=9090/tcp  --permanent
> firewall-cmd --reload

start and enable services

> systemctl start prometheus \
  &&  systemctl enable prometheus

promtail example

server:
#  http_listen_address: 192.168.2.10

  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://192.168.2.8:3100/loki/api/v1/push

scrape_configs:
  - job_name: system
    pipeline_stages:
      - match:
          selector: '{filename=~".*fail2ban.log"}'
          stages:
            - regex:
                expression:
                  "(?P<time>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{0,3})?\\D?
                  (?P<message>fail2ban.*(?P<pid>\\[\\d*\\]: )(?P<level>[A-Z]{4,7}) .*
                  (?:(?:\\[|Jail ')(?P<jail>\\D*)(?:\\]|'))?.*)"
            - template:
                source: message
                template: '{{ Replace .Value .pid "" -1 }}'
            - regex:
                expression: '(?P<message>.*)(?: - \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})'
                source: message
            - template:
                source: time
                template: '{{ Replace .Value "," "." -1 }}'
            - timestamp:
                source: time
                format: "2006-01-02 15:04:05.000"
            - output:
                source: message


    static_configs:
      - targets:
          - localhost
        labels:
          job: ho-mainframe-03-varlogs
          __path__: /var/log/**/**log

  - job_name: journal
    journal:
      max_age: 12h
      labels:
        job: ho-mainframe-03/systemd-journal
    relabel_configs:
      - source_labels: ['__journal__systemd_unit']
        target_label: 'unit

security | server | software | stuff