aim:

  • run a ha cluster using hetzner’s cloud offer

requirements:

  • 2 haproxy server incl. keepalived
  • 3 master nodes containing etcd
  • 2 or more worker nodes
  • 1 additional cluster network
  • 1 floating ip

setup hcloud

# install pkg
> pacman -S hcloud

# add (rw) api token
> hcloud context create k8s-ha-setup

do not forget shell completion

create network

# add network
> hcloud network create \
  --ip-range 10.58.0.0/16 \
  --name k8s-network
# ...
# Network 000000 created

# add subnet
> hcloud network add-subnet k8s-network \
  --network-zone eu-central \
  --type server \
  --ip-range 10.58.0.0/16
# ...
# Subnet added to network 000000

create haproxy server

> hcloud server create \
  --name haproxy-1 \
  --image debian-11 \
  --location hel1 \
  --type cx11 \
  --ssh-key my-shh-key \
  --network 000000 \
  --start-after-create
# cx11=1vcpu,2gb-ram,20gb-ssd

> hcloud server create \
  --name haproxy-2 \
  --image debian-11 \
  --location hel1 \
  --type cx11 \
  --ssh-key my-shh-key \
  --network 000000 \
  --start-after-create
# cx11=1vcpu,2gb-ram,20gb-ssd

create master nodes

> hcloud server create \
  --name master-1 \
  --image debian-11 \
  --location hel1 \
  --type cpx11 \
  --ssh-key my-shh-key \
  --network 000000 \
  --start-after-create
# cpx11=2vcpu,2gb-ram,40gb-ssd

> hcloud server create \
  --name master-2 \
  --image debian-11 \
  --location hel1 \
  --type cpx11 \
  --ssh-key my-shh-key \
  --network 000000 \
  --start-after-create
# cpx11=2vcpu,2gb-ram,40gb-ssd

> hcloud server create \
  --name master-3 \
  --image debian-11 \
  --location hel1 \
  --type cpx11 \
  --ssh-key my-shh-key \
  --network 000000 \
  --start-after-create
# cpx11=2vcpu,2gb-ram,40gb-ssd

create worker nodes

> hcloud server create \
  --name worker-1 \
  --image debian-11 \
  --location hel1 \
  --type cx11 \
  --ssh-key my-shh-key \
  --network 000000 \
  --start-after-create
# cx11=1vcpu,2gb-ram,20gb-ssd

> hcloud server create \
  --name worker-2 \
  --image debian-11 \
  --location hel1 \
  --type cx11 \
  --ssh-key my-shh-key \
  --network 000000 \
  --start-after-create
# cx11=1vcpu,2gb-ram,20gb-ssd

on haproxy-1|2 do

> apt update

> apt install -y \
  haproxy

> mv /etc/haproxy/haproxy.cfg \
  /etc/haproxy/haproxy.cfg.BK

> append /etc/haproxy/haproxy.cfg
...
frontend kube-apiserver
  bind *:6443
  mode tcp
  option tcplog
  default_backend kube-apiserver

backend kube-apiserver
    mode tcp
    option tcplog
    option tcp-check
    balance roundrobin
#    default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
    server kube-apiserver-1 10.58.0.4:6443 check # master-1
    server kube-apiserver-2 10.58.0.5:6443 check # master-2
    server kube-apiserver-3 10.58.0.6:6443 check # master-3


> systemctl restart haproxy \
  && systemctl enable haproxy

> systemctl status haproxy
  ...
  backend 'kube-apiserver' has no server available!

on all master nodes run

# swap off
> swapoff -a; sed -i '/swap/d' /etc/fstab

# set timezone
> ln -sf \
  /usr/share/zoneinfo/Europe/Berlin \
  /etc/localtime

# create sysconfig file
> cat >>/etc/sysctl.d/kubernetes.conf<<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

# apply sysconfig
> sysctl --system

# install requirements
> apt update && \
  apt install -y \
  ca-certificates \
  curl \
  gnupg \
  lsb-release

# install docker
> curl -fsSL \
  https://download.docker.com/linux/debian/gpg \
  | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg

> echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \
  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

> apt update && \
  apt install -y \
  docker-ce \
  docker-ce-cli \
  containerd.io

# install kubeadm etc
> curl -fsSLo \
  /usr/share/keyrings/kubernetes-archive-keyring.gpg \
  https://packages.cloud.google.com/apt/doc/apt-key.gpg


> echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" \
  | sudo tee /etc/apt/sources.list.d/kubernetes.list

> apt update && \
  apt install -y \
  kubelet \
  kubeadm \
  kubectl && \
  apt-mark hold \
  kubelet \
  kubeadm \
  kubectl

on first master

# bootstrap master-1
> kubeadm init \
  --control-plane-endpoint="10.58.0.2:6443" \
  --upload-certs \
  --pod-network-cidr=10.58.0.0/16

## == CNI
# please install only one provider
# a) calico
> kubectl \
  --kubeconfig=/etc/kubernetes/admin.conf \
  create -f https://docs.projectcalico.org/v3.20/manifests/calico.yaml
# b) weave - not encrypted by default
> kubectl \
  --kubeconfig=/etc/kubernetes/admin.conf \
  apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')"

> mkdir ~/.kube
> cp /etc/kubernetes/admin.conf ~/.kube/config

on master 2 & 3 run the “join master” output from master-1

# master-2
> kubeadm join 10.58.0.2:6443 --token XXXX.xxxxxxxxxxx \
	--discovery-token-ca-cert-hash sha256:xxxxxxxxxxxxxxxxxxxx \
  --control-plane --certificate-key xxxxxxxxxxxxxxxxxx
> mkdir ~/.kube
> cp /etc/kubernetes/admin.conf ~/.kube/config

# master-3
> kubeadm join 10.58.0.2:6443 --token XXXX.xxxxxxxxxxx \
	--discovery-token-ca-cert-hash sha256:xxxxxxxxxxxxxxxxxxxx \
  --control-plane --certificate-key xxxxxxxxxxxxxxxxxx
> mkdir ~/.kube
> cp /etc/kubernetes/admin.conf ~/.kube/config

# verify
> kubectl get nodes
  ...
  NAME       STATUS   ROLES                  AGE    VERSION
  master-1   Ready    control-plane,master   102s   v1.22.4
  master-2   Ready    control-plane,master   79s    v1.22.4
  master-3   Ready    control-plane,master   29s    v1.22.4

on all worker nodes run

# swap off
> swapoff -a; sed -i '/swap/d' /etc/fstab

# set timezone
> ln -sf \
  /usr/share/zoneinfo/Europe/Berlin \
  /etc/localtime

# create sysconfig file
> cat >>/etc/sysctl.d/kubernetes.conf<<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

# apply sysconfig
> sysctl --system

# install requirements
> apt update && \
  apt install -y \
  ca-certificates \
  curl \
  gnupg \
  lsb-release

# install docker
> curl -fsSL \
  https://download.docker.com/linux/debian/gpg \
  | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg

> echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \
  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

> apt update && \
  apt install -y \
  docker-ce \
  docker-ce-cli \
  containerd.io

# install kubeadm etc
> curl -fsSLo \
  /usr/share/keyrings/kubernetes-archive-keyring.gpg \
  https://packages.cloud.google.com/apt/doc/apt-key.gpg


> echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" \
  | sudo tee /etc/apt/sources.list.d/kubernetes.list

> apt update && \
  apt install -y \
  kubelet \
  kubeadm \
  kubectl && \
  apt-mark hold \
  kubelet \
  kubeadm \
  kubectl

> kubeadm join 10.58.0.2:6443 --token XXXX.xxxxxxxxxxx \
	--discovery-token-ca-cert-hash sha256:xxxxxxxxxxxxxxxxxxxx

check cluster

> kubectl get nodes
  ...
  NAME       STATUS   ROLES                  AGE     VERSION
  master-1   Ready    control-plane,master   14m     v1.22.4
  master-2   Ready    control-plane,master   13m     v1.22.4
  master-3   Ready    control-plane,master   12m     v1.22.4
  worker-1   Ready    <none>                 4m43s   v1.22.4
  worker-2   Ready    <none> 

create floating ip

> hcloud floating-ip create \
  --type ipv4 \
  --home-location hel1 \
  --name public-ip

on all worker nodes run

> cat <<EOF > /etc/network/interfaces.d/60-my-floating-ip.cfg
auto eth0:1
iface eth0:1 inet static
 address <floating_ip>
 netmask 32
EOF

> systemctl restart networking

om master run

> kubectl apply -f \
  https://raw.githubusercontent.com/google/metallb/v0.11.0/manifests/metallb.yaml

edit metallb-floatingip.yml
---
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: metallb-system
  name: config
data:
  config: |
    address-pools:
    - name: default
      protocol: layer2
      addresses:
      - 95.216.180.60/32

> kubectl apply -f metallb-floatingip.yml

> kubectl create namespace fip-controller

> kubectl apply -f \
  https://raw.githubusercontent.com/cbeneke/hcloud-fip-controller/master/deploy/rbac.yaml

> kubectl apply -f \
  https://raw.githubusercontent.com/cbeneke/hcloud-fip-controller/master/deploy/deployment.yaml

> cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: ConfigMap
metadata:
  name: fip-controller-config
  namespace: fip-controller
data:
  config.json: |
    {
      "hcloud_floating_ips": [ "<floating_ip>" ]
    }
---
apiVersion: v1
kind: Secret
metadata:
  name: fip-controller-secrets
  namespace: fip-controller
stringData:
  HCLOUD_API_TOKEN: "<api_token>"
EOF

#> kubectl apply -f \
#  https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/mandatory.yaml
> kubectl apply -f \
  https://raw.githubusercontent.com/kubernetes/ingress-nginx/130af33510882ae62c89277f2ad0baca50e0fafe/deploy/static/mandatory.yaml

> cat <<EOF | kubectl apply -f -
kind: Service
apiVersion: v1
metadata:
  name: ingress-nginx
  namespace: ingress-nginx
  annotations:
    metallb.universe.tf/allow-shared-ip: "floating-ip"
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
spec:
  type: LoadBalancer
  selector:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
  ports:
    - name: http
      port: 80
      targetPort: http
    - name: https
      port: 443
      targetPort: https
EOF