Grafana & influxDB on Almalinux
goals:
- secure connected grafana dashboard
- reverse proxy served on port 443 for grafana
- secure influxdb installation with collectd support
- privilege seperation for db management
get startet
install httpd
## install httpd
> dnf install httpd mod_ssl
install grafana
## paste grafana repo in /etc/yum.repos.d/grafana.repo
[grafana]
name=grafana
baseurl=https://packages.grafana.com/oss/rpm
repo_gpgcheck=1
enabled=1
gpgcheck=1
gpgkey=https://packages.grafana.com/gpg.key
sslverify=1
sslcacert=/etc/pki/tls/certs/ca-bundle.crt
## refresh repo and install grafana
> dnf install grafana
## start and enable service
> systemctl enable grafana-server
> systemctl start grafana-server
install influxdb
## paste influxdb repo in /etc/yum.repos.d/influxdb.repo
[influxdb]
name = InfluxDB Repository - RHEL 8
baseurl = https://repos.influxdata.com/rhel/8/x86_64/stable
enabled = 1
gpgcheck = 1
gpgkey = https://repos.influxdata.com/influxdb.key
## refresh repo and install influxdb
> dnf install influxdb
## generate self signed certificates
> openssl req -x509 -nodes -subj "/C=XX/ST=XX/L=XX/O=XX/OU=XX/CN=XX" \
-newkey rsa:4096 -keyout /etc/pki/tls/private/influxdb.key \
-out /etc/pki/tls/private/influxdb.crt -days 365
> chown -R influxdb:influxdb /etc/pki/tls/private/influxdb.*
## start and enable service
> systemctl enable influxdb
> systemctl start influxdb
reverse proxy
## generate self signed certificates
> openssl req -x509 -nodes -subj "/C=XX/ST=XX/L=XX/O=XX/OU=XX/CN=XX" \
-newkey rsa:4096 -keyout /etc/pki/tls/private/httpd.key \
-out /etc/pki/tls/private/httpd.crt -days 365
## edit /etc/httpd/conf.d/grafana_proxy.conf
<VirtualHost *:443>
ServerName localhost
SSLEngine on
SSLCipherSuite EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH
SSLProtocol All -SSLv2 -SSLv3 -TLSv1 -TLSv1.1
SSLHonorCipherOrder On
SSLCertificateFile /etc/pki/tls/private/httpd.crt
SSLCertificateKeyFile /etc/pki/tls/private/httpd.key
Header always set Strict-Transport-Security "max-age=63072000; includeSubdomains;"
Header always set X-Frame-Options DENY
Header set X-Content-Type-Options "nosniff"
<IfModule mod_proxy.c>
ProxyRequests Off
<Proxy *>
Require all granted
</Proxy>
ProxyPass / http://localhost:3000/
ProxyPassReverse / http://localhost:3000/
</IfModule>
</VirtualHost>
## restart service and enable service
> systemctl enable httpd
> systemctl restart httpd
## selinux policy
> setsebool -P httpd_can_network_connect on
## open firewall on 443
> firewall-cmd --zone=public --add-service=https --permanent
> firewall-cmd --reload
setup influxdb
## edit /etc/influxdb/influxdb.conf
[meta]
dir = "/var/lib/influxdb/meta"
[data]
dir = "/var/lib/influxdb/data"
wal-dir = "/var/lib/influxdb/wal"
series-id-set-cache-size = 100
[coordinator]
[retention]
[shard-precreation]
[monitor]
[http]
https-enabled = true
https-certificate = "/etc/pki/tls/private/influxdb.crt"
https-private-key = "/etc/pki/tls/private/influxdb.key"
[logging]
[subscriber]
[[graphite]]
[[collectd]]
[[opentsdb]]
[[udp]]
[continuous_queries]
[tls]
ciphers = [
"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305",
]
min-version = "tls1.3"
## restart service
> systemctl restart influxdb
## check tls connection
> influx -ssl -unsafeSsl -host localhost
influxdb config
## connect to db
> influx -ssl -unsafeSsl -host localhost
## create db admin
CREATE USER <username> WITH PASSWORD '<password>' WITH ALL PRIVILEGES
## create user only for read access
CREATE USER user WITH PASSWORD '<password>'
## create user telegraf
CREATE USER telegraf WITH PASSWORD '<password>'
## create user collectd
CREATE USER collectd WITH PASSWORD '<password>'
## exit db
quit
## add authentication to /etc/influxdb/influxdb.conf
[http]
auth-enabled = true
...
## restart service
> systemctl restart influxdb
## login with
> influx -ssl -unsafeSsl -host example.com -username user -password <password>
## create telegraf database
CREATE DATABASE telegraf
## add read only user to telegraf database
GRANT READ ON telegraf TO user
## add all allowed user to telegraf database
GRANT ALL ON telegraf TO telegraf
## create collectd database
CREATE DATABASE collectd
## add read only user to collectd database
GRANT READ ON collectd TO user
## add all allowed user to telegraf database
GRANT ALL ON collectd TO collectd
## exit db
quit
## open port for external
> firewall-cmd --zone=public --add-port=8086/tcp --permanent
> firewall-cmd --reload
connect grafana to influxdb
## now we can login to grafana dashbord and connect to influxdb
## setup influxdb data source like:
> URL:
* https://localhost:8086
> Auth:
* Skip TLS Verify = true
> Database
* telegraf
> User
* user # the read only user
> save & test
install telegraf
## refresh repo and install influxdb
> dnf install telegraf
## generate certificates
> openssl req -x509 -nodes -subj "/C=XX/ST=XX/L=XX/O=XX/OU=XX/CN=XX" \
-newkey rsa:4096 -keyout /etc/pki/tls/private/telegraf.key \
-out /etc/pki/tls/private/telegraf.crt -days 365
> chown -R telegraf:telegraf /etc/pki/tls/private/telegraf.*
## edit /etc/telegraf/telegraf.conf
[global_tags]
[agent]
interval = "10s"
round_interval = true
metric_batch_size = 1000
metric_buffer_limit = 10000
collection_jitter = "0s"
flush_interval = "10s"
flush_jitter = "0s"
precision = ""
hostname = ""
omit_hostname = false
[[outputs.influxdb]]
urls = ["https://127.0.0.1:8086"]
insecure_skip_verify = true
tls_cert = "/etc/pki/tls/private/telegraf.crt"
tls_key = "/etc/pki/tls/private/telegraf.key"
database = "telegraf"
username = "telegraf"
password = "<password>"
skip_database_creation = true
[[inputs.cpu]]
percpu = true
totalcpu = true
collect_cpu_time = false
report_active = false
[[inputs.disk]]
ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"]
[[inputs.diskio]]
[[inputs.kernel]]
[[inputs.mem]]
[[inputs.processes]]
[[inputs.swap]]
[[inputs.system]]
## start and enable service
> systemctl enable telegraf
> systemctl start telegraf
configure collectd
## for collectd support edit /etc/influxdb/influxdb.conf
[[collectd]]
enabled = true
bind-address = ":25826"
database = "collectd"
retention-policy = ""
batch-size = 5000
batch-pending = 10
batch-timeout = "10s"
read-buffer = 0
typesdb = "/usr/share/collectd/types.db"
security-level = "encrypt"
auth-file = "/etc/collectd/auth_file"
parse-multivalue-plugin = "split"
## make directorys and download types.db
> mkdir -p /usr/share/collectd/collectd
> wget -P /usr/share/collectd/ https://raw.githubusercontent.com/collectd/collectd/master/src/types.db
> chown -R influxdb:influxdb /usr/share/collectd/
> mkdir -p /etc/collectd
## setup auth_file on /etc/collectd/auth_file
collectd: <password>
> chown influxdb:influxdb /etc/collectd/auth_file
## setup auth_file on /etc/collectd/auth_file
collectd: <password>
## restart service
> systemctl restart influxdb
## wait a second and check if the service still run
> systemctl status influxdb | grep -i collectd
setup collectd client
## collectd openbsd example file /etc/collectd.conf
Hostname "example.com"
FQDNLookup true
BaseDir "/var/collectd"
PIDFile "/var/collectd/collectd.pid"
TypesDB "/usr/local/share/collectd/types.db"
Interval 10.0
# Client
LoadPlugin "network"
LoadPlugin "interface"
LoadPlugin "logfile"
LoadPlugin "cpu"
LoadPlugin "memory"
LoadPlugin "swap"
<Plugin interface>
Interface "vr0"
Interface "vr1"
Interface "vr2"
IgnoreSelected false
</Plugin>
# Client
<Plugin network>
<Server "example.com" "25826">
SecurityLevel Encrypt
Username "collectd"
Password "<password>"
Interface "vio0"
</Server>
TimeToLive 128
MaxPacketSize 1452
</Plugin>
See: TypesDB
telegraf client installation
## paste influxdb repo in /etc/yum.repos.d/influxdb.repo
[influxdb]
name = InfluxDB Repository - RHEL 8
baseurl = https://repos.influxdata.com/rhel/8/x86_64/stable
enabled = 1
gpgcheck = 1
gpgkey = https://repos.influxdata.com/influxdb.key
## refresh repo and install influxdb
> dnf install telegraf
## generate certificates
> openssl req -x509 -nodes -subj "/C=XX/ST=XX/L=XX/O=XX/OU=XX/CN=XX" \
-newkey rsa:4096 -keyout /etc/pki/tls/private/telegraf.key \
-out /etc/pki/tls/private/telegraf.crt -days 365
> chown -R telegraf:telegraf /etc/pki/tls/private/telegraf.*
## edit /etc/telegraf/telegraf.conf
[global_tags]
[agent]
interval = "10s"
round_interval = true
metric_batch_size = 1000
metric_buffer_limit = 10000
collection_jitter = "0s"
flush_interval = "10s"
flush_jitter = "0s"
precision = ""
hostname = ""
omit_hostname = false
[[outputs.influxdb]]
urls = ["https://<server>:8086"]
insecure_skip_verify = true
tls_cert = "/etc/pki/tls/private/telegraf.crt"
tls_key = "/etc/pki/tls/private/telegraf.key"
database = "telegraf"
username = "telegraf"
password = "<password>"
skip_database_creation = true
[[inputs.cpu]]
percpu = true
totalcpu = true
collect_cpu_time = false
report_active = false
[[inputs.disk]]
ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"]
[[inputs.diskio]]
[[inputs.kernel]]
[[inputs.mem]]
[[inputs.io]]
[[inputs.netstat]]
[[inputs.net]]
[[inputs.processes]]
[[inputs.swap]]
[[inputs.system]]
## start and enable service
> systemctl enable telegraf
> systemctl start telegraf
24-10-2020