Grafana & influxDB on Almalinux :: x33u

goals:

secure connected grafana dashboard
reverse proxy served on port 443 for grafana
secure influxdb installation with collectd support
privilege seperation for db management

get startet

install httpd

## install httpd
>  dnf install httpd mod_ssl

install grafana

## paste grafana repo in /etc/yum.repos.d/grafana.repo
[grafana]
name=grafana
baseurl=https://packages.grafana.com/oss/rpm
repo_gpgcheck=1
enabled=1
gpgcheck=1
gpgkey=https://packages.grafana.com/gpg.key
sslverify=1
sslcacert=/etc/pki/tls/certs/ca-bundle.crt

## refresh repo and install grafana
> dnf install grafana

## start and enable service
> systemctl enable grafana-server
> systemctl start grafana-server

install influxdb

## paste influxdb repo in /etc/yum.repos.d/influxdb.repo
[influxdb]
name = InfluxDB Repository - RHEL 8
baseurl = https://repos.influxdata.com/rhel/8/x86_64/stable
enabled = 1
gpgcheck = 1
gpgkey = https://repos.influxdata.com/influxdb.key

## refresh repo and install influxdb
> dnf install influxdb

## generate self signed certificates
> openssl req -x509 -nodes -subj "/C=XX/ST=XX/L=XX/O=XX/OU=XX/CN=XX" \
  -newkey rsa:4096 -keyout /etc/pki/tls/private/influxdb.key \
  -out /etc/pki/tls/private/influxdb.crt -days 365
> chown -R influxdb:influxdb /etc/pki/tls/private/influxdb.*

## start and enable service
> systemctl enable influxdb
> systemctl start influxdb

reverse proxy

## generate self signed certificates
> openssl req -x509 -nodes -subj "/C=XX/ST=XX/L=XX/O=XX/OU=XX/CN=XX" \
  -newkey rsa:4096 -keyout /etc/pki/tls/private/httpd.key \
  -out /etc/pki/tls/private/httpd.crt -days 365

## edit /etc/httpd/conf.d/grafana_proxy.conf
<VirtualHost *:443>
  ServerName localhost

  SSLEngine on
  SSLCipherSuite EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH
  SSLProtocol All -SSLv2 -SSLv3 -TLSv1 -TLSv1.1
  SSLHonorCipherOrder On
  SSLCertificateFile /etc/pki/tls/private/httpd.crt
  SSLCertificateKeyFile /etc/pki/tls/private/httpd.key

  Header always set Strict-Transport-Security "max-age=63072000; includeSubdomains;"
  Header always set X-Frame-Options DENY
  Header set X-Content-Type-Options "nosniff"

<IfModule mod_proxy.c>
    ProxyRequests Off
    <Proxy *>
        Require all granted
    </Proxy>
    ProxyPass / http://localhost:3000/
    ProxyPassReverse / http://localhost:3000/
</IfModule>
</VirtualHost>


## restart service and enable service
> systemctl enable httpd
> systemctl restart httpd

## selinux policy
> setsebool -P httpd_can_network_connect on

## open firewall on 443
> firewall-cmd --zone=public --add-service=https --permanent
> firewall-cmd --reload

setup influxdb

## edit /etc/influxdb/influxdb.conf
[meta]
  dir = "/var/lib/influxdb/meta"
[data]
  dir = "/var/lib/influxdb/data"
  wal-dir = "/var/lib/influxdb/wal"
  series-id-set-cache-size = 100
[coordinator]
[retention]
[shard-precreation]
[monitor]
[http]
  https-enabled = true
  https-certificate = "/etc/pki/tls/private/influxdb.crt"
  https-private-key = "/etc/pki/tls/private/influxdb.key"
[logging]
[subscriber]
[[graphite]]
[[collectd]]
[[opentsdb]]
[[udp]]
[continuous_queries]
[tls]
  ciphers = [
    "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
    "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305",
  ]
  min-version = "tls1.3"


## restart service
> systemctl restart influxdb

## check tls connection
> influx -ssl -unsafeSsl -host localhost

influxdb config

## connect to db
> influx -ssl -unsafeSsl -host localhost
## create db admin
CREATE USER <username> WITH PASSWORD '<password>' WITH ALL PRIVILEGES
## create user only for read access
CREATE USER user WITH PASSWORD '<password>'
## create user telegraf
CREATE USER telegraf WITH PASSWORD '<password>'
## create user collectd
CREATE USER collectd WITH PASSWORD '<password>'
## exit db
quit

## add authentication to /etc/influxdb/influxdb.conf
[http]
  auth-enabled = true
...

## restart service
> systemctl restart influxdb

## login with
> influx -ssl -unsafeSsl -host example.com -username user -password <password>

## create telegraf database
CREATE DATABASE telegraf
## add read only user to telegraf database
GRANT READ ON telegraf TO user
## add all allowed user to telegraf database
GRANT ALL ON telegraf TO telegraf

## create collectd database
CREATE DATABASE collectd
## add read only user to collectd database
GRANT READ ON collectd TO user
## add all allowed user to telegraf database
GRANT ALL ON collectd TO collectd
## exit db
quit

## open port for external
> firewall-cmd --zone=public --add-port=8086/tcp --permanent
> firewall-cmd --reload

connect grafana to influxdb

## now we can login to grafana dashbord and connect to influxdb
## setup influxdb data source like:
> URL:
* https://localhost:8086
> Auth:
* Skip TLS Verify = true
> Database
* telegraf
> User
* user # the read only user
> save & test

install telegraf

## refresh repo and install influxdb
> dnf install telegraf

## generate certificates
> openssl req -x509 -nodes -subj "/C=XX/ST=XX/L=XX/O=XX/OU=XX/CN=XX" \
  -newkey rsa:4096 -keyout /etc/pki/tls/private/telegraf.key \
  -out /etc/pki/tls/private/telegraf.crt -days 365
> chown -R telegraf:telegraf /etc/pki/tls/private/telegraf.*

## edit /etc/telegraf/telegraf.conf
[global_tags]
[agent]
  interval = "10s"
  round_interval = true
  metric_batch_size = 1000
  metric_buffer_limit = 10000
  collection_jitter = "0s"
  flush_interval = "10s"
  flush_jitter = "0s"
  precision = ""
  hostname = ""
  omit_hostname = false
[[outputs.influxdb]]
  urls = ["https://127.0.0.1:8086"]
  insecure_skip_verify = true
  tls_cert = "/etc/pki/tls/private/telegraf.crt"
  tls_key = "/etc/pki/tls/private/telegraf.key"
  database = "telegraf"
  username = "telegraf"
  password = "<password>"
  skip_database_creation = true
[[inputs.cpu]]
  percpu = true
  totalcpu = true
  collect_cpu_time = false
  report_active = false
[[inputs.disk]]
  ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"]
[[inputs.diskio]]
[[inputs.kernel]]
[[inputs.mem]]
[[inputs.processes]]
[[inputs.swap]]
[[inputs.system]]

## start and enable service
> systemctl enable telegraf
> systemctl start telegraf

configure collectd

## for collectd support edit /etc/influxdb/influxdb.conf
[[collectd]]
  enabled = true
  bind-address = ":25826"
  database = "collectd"
  retention-policy = ""
  batch-size = 5000
  batch-pending = 10
  batch-timeout = "10s"
  read-buffer = 0
  typesdb = "/usr/share/collectd/types.db"
  security-level = "encrypt"
  auth-file = "/etc/collectd/auth_file"
  parse-multivalue-plugin = "split"

## make directorys and download types.db
> mkdir -p /usr/share/collectd/collectd
> wget -P /usr/share/collectd/ https://raw.githubusercontent.com/collectd/collectd/master/src/types.db
> chown -R influxdb:influxdb /usr/share/collectd/
> mkdir -p /etc/collectd
## setup auth_file on /etc/collectd/auth_file
collectd: <password>
> chown influxdb:influxdb /etc/collectd/auth_file

## setup auth_file on /etc/collectd/auth_file
collectd: <password>

## restart service
> systemctl restart influxdb

## wait a second and check if the service still run
> systemctl status influxdb | grep -i collectd

setup collectd client

## collectd openbsd example file /etc/collectd.conf
Hostname    "example.com"
FQDNLookup   true
BaseDir     "/var/collectd"
PIDFile     "/var/collectd/collectd.pid"
TypesDB     "/usr/local/share/collectd/types.db"
Interval 10.0
# Client
LoadPlugin "network"
LoadPlugin "interface"
LoadPlugin "logfile"
LoadPlugin "cpu"
LoadPlugin "memory"
LoadPlugin "swap"
<Plugin interface>
    Interface "vr0"
    Interface "vr1"
    Interface "vr2"
    IgnoreSelected false
</Plugin>
# Client
<Plugin network>
    <Server "example.com" "25826">
        SecurityLevel Encrypt
        Username "collectd"
        Password "<password>"
        Interface "vio0"
    </Server>
    TimeToLive 128
    MaxPacketSize 1452
</Plugin>

See: TypesDB

telegraf client installation

## paste influxdb repo in /etc/yum.repos.d/influxdb.repo
[influxdb]
name = InfluxDB Repository - RHEL 8
baseurl = https://repos.influxdata.com/rhel/8/x86_64/stable
enabled = 1
gpgcheck = 1
gpgkey = https://repos.influxdata.com/influxdb.key

## refresh repo and install influxdb
> dnf install telegraf

## generate certificates
> openssl req -x509 -nodes -subj "/C=XX/ST=XX/L=XX/O=XX/OU=XX/CN=XX" \
  -newkey rsa:4096 -keyout /etc/pki/tls/private/telegraf.key \
  -out /etc/pki/tls/private/telegraf.crt -days 365
> chown -R telegraf:telegraf /etc/pki/tls/private/telegraf.*

## edit /etc/telegraf/telegraf.conf
[global_tags]
[agent]
  interval = "10s"
  round_interval = true
  metric_batch_size = 1000
  metric_buffer_limit = 10000
  collection_jitter = "0s"
  flush_interval = "10s"
  flush_jitter = "0s"
  precision = ""
  hostname = ""
  omit_hostname = false
[[outputs.influxdb]]
  urls = ["https://<server>:8086"]
  insecure_skip_verify = true
  tls_cert = "/etc/pki/tls/private/telegraf.crt"
  tls_key = "/etc/pki/tls/private/telegraf.key"
  database = "telegraf"
  username = "telegraf"
  password = "<password>"
  skip_database_creation = true
[[inputs.cpu]]
  percpu = true
  totalcpu = true
  collect_cpu_time = false
  report_active = false
[[inputs.disk]]
  ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"]
[[inputs.diskio]]
[[inputs.kernel]]
[[inputs.mem]]
[[inputs.io]]
[[inputs.netstat]]
[[inputs.net]]
[[inputs.processes]]
[[inputs.swap]]
[[inputs.system]]

## start and enable service
> systemctl enable telegraf
> systemctl start telegraf

tested with grafana dashboard 5955 & 10578

security | server | software | stuff

goals:

get startet