tls bootstrapping worker nodes

on master-1

> sudo nano  /etc/systemd/system/kube-apiserver.service

ensure bootstrap token based authentication is enabled on the kube-apiserver: --enable-bootstrap-token-auth=true

> sudo nano  /etc/systemd/system/kube-controller-manager.service

the certificate requests are signed by the kube-controller-manager ultimately. the kube-controller-manager requires the ca certificate and key to perform these operations:
--cluster-signing-cert-file=/var/lib/kubernetes/ca.crt
--cluster-signing-key-file=/var/lib/kubernetes/ca.key

on worker-2

> wget -q --show-progress --https-only --timestamping \
  https://storage.googleapis.com/kubernetes-release/release/v1.13.0/bin/linux/amd64/kubectl \
  https://storage.googleapis.com/kubernetes-release/release/v1.13.0/bin/linux/amd64/kube-proxy \
  https://storage.googleapis.com/kubernetes-release/release/v1.13.0/bin/linux/amd64/kubelet
> sudo mkdir -p \
  /etc/cni/net.d \
  /opt/cni/bin \
  /var/lib/kubelet \
  /var/lib/kube-proxy \
  /var/lib/kubernetes \
  /var/run/kubernetes
{
  chmod +x kubectl kube-proxy kubelet
  sudo mv kubectl kube-proxy kubelet /usr/local/bin/
}

copy /var/lib/kubernetes/ca.crt from master-1 to worker-2

on worker-2

> sudo mv ca.crt /var/lib/kubernetes/
> sudo chmod 644 /var/lib/kubernetes/ca.crt
> sudo chown root:root /var/lib/kubernetes/ca.crt

step 1 - create the boostrap token to be used by nodes (kubelets) to invoke certificate api

on master-1

> cat > bootstrap-token-07401b.yaml <<EOF
apiVersion: v1
kind: Secret
metadata:
  # Name MUST be of form "bootstrap-token-<token id>"
  name: bootstrap-token-07401b
  namespace: kube-system

# Type MUST be 'bootstrap.kubernetes.io/token'
type: bootstrap.kubernetes.io/token
stringData:
  # Human readable description. Optional.
  description: "The default bootstrap token generated by 'kubeadm init'."

  # Token ID and secret. Required.
  token-id: 07401b
  token-secret: f395accd246ae52d

  # Expiration. Optional.
  expiration: 2021-03-10T03:22:11Z

  # Allowed usages.
  usage-bootstrap-authentication: "true"
  usage-bootstrap-signing: "true"

  # Extra groups to authenticate the token as. Must start with "system:bootstrappers:"
  auth-extra-groups: system:bootstrappers:worker
EOF
> kubectl create -f bootstrap-token-07401b.yaml

things to note:
expiration - make sure its set to a date in the future.
auth-extra-groups - this is the group the worker nodes are part of. it must start with system:bootstrappers: this group does not exist already. this group is associated with this token.


step 2 - authorize workers (kubelets) to create csr

> kubectl create clusterrolebinding \
  create-csrs-for-bootstrapping \  
  --clusterrole=system:node-bootstrapper \
  --group=system:bootstrappers

— OR —

cat > csrs-for-bootstrapping.yaml <<EOF
# enable bootstrapping nodes to create CSR
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: create-csrs-for-bootstrapping
subjects:
- kind: Group
  name: system:bootstrappers
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: ClusterRole
  name: system:node-bootstrapper
  apiGroup: rbac.authorization.k8s.io
EOF
> kubectl create -f csrs-for-bootstrapping.yaml

step 3 - authorize workers (kubelets) to approve csr

> kubectl create clusterrolebinding \
  auto-approve-csrs-for-group \
  --clusterrole=system:certificates.k8s. \
  io:certificatesigningrequests:nodeclient \
  --group=system:bootstrappers

— OR —

cat > auto-approve-csrs-for-group.yaml <<EOF
# Approve all CSRs for the group "system:bootstrappers"
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: auto-approve-csrs-for-group
subjects:
- kind: Group
  name: system:bootstrappers
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: ClusterRole
  name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
  apiGroup: rbac.authorization.k8s.io
EOF
> kubectl create -f auto-approve-csrs-for-group.yaml

step 4 - authorize workers (kubelets) to auto renew certificates on expiration

> kubectl create clusterrolebinding \
  auto-approve-renewals-for-nodes \
  --clusterrole=system:certificates.k8s \
  io:certificatesigningrequests:selfnodeclient \
  --group=system:nodes

— OR —

cat > auto-approve-renewals-for-nodes.yaml <<EOF
# Approve renewal CSRs for the group "system:nodes"
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: auto-approve-renewals-for-nodes
subjects:
- kind: Group
  name: system:nodes
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: ClusterRole
  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
  apiGroup: rbac.authorization.k8s.io
EOF
> kubectl create -f auto-approve-renewals-for-nodes.yaml

step 5 - configure kubelet to tls bootstrap - it is now time to configure the second worker to tls bootstrap using the token we generated previously.

for worker-1 we started by creating a kubeconfig file with the TLS certificates that we manually generated. Here, we don’t have the certificates yet. So we cannot create a kubeconfig file. Instead we create a bootstrap-kubeconfig file with information about the token we created. This is to be done on the worker-2 node.

on worker-2

> sudo kubectl config --kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig \
  set-cluster bootstrap --server='https://192.168.5.30:6443' \
  --certificate-authority=/var/lib/kubernetes/ca.crt

> sudo kubectl config --kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig \
  set-credentials kubelet-bootstrap \
  --token=07401b.f395accd246ae52d

> sudo kubectl config --kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig \
  set-context bootstrap --user=kubelet-bootstrap --cluster=bootstrap

> sudo kubectl config --kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig \
  use-context bootstrap

— Or —

worker-2$ cat <<EOF | sudo tee /var/lib/kubelet/bootstrap-kubeconfig
apiVersion: v1
clusters:
- cluster:
    certificate-authority: /var/lib/kubernetes/ca.crt
    server: https://192.168.5.30:6443
  name: bootstrap
contexts:
- context:
    cluster: bootstrap
    user: kubelet-bootstrap
  name: bootstrap
current-context: bootstrap
kind: Config
preferences: {}
users:
- name: kubelet-bootstrap
  user:
    token: 07401b.f395accd246ae52d
EOF

step 6 - create kubelet config File

create the kubelet-config.yaml configuration file

cat <<EOF | sudo tee /var/lib/kubelet/kubelet-config.yaml
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
  anonymous:
    enabled: false
  webhook:
    enabled: true
  x509:
    clientCAFile: "/var/lib/kubernetes/ca.crt"
authorization:
  mode: Webhook
clusterDomain: "cluster.local"
clusterDNS:
  - "10.96.0.10"
resolvConf: "/run/systemd/resolve/resolv.conf"
runtimeRequestTimeout: "15m"
EOF

Note: We are not specifying the certificate details - tlsCertFile and tlsPrivateKeyFile in this file

step 7 - configure kubelet service

create the kubelet.service systemd unit file

cat <<EOF | sudo tee /etc/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/kubernetes/kubernetes
After=docker.service
Requires=docker.service

[Service]
ExecStart=/usr/local/bin/kubelet \\
  --bootstrap-kubeconfig="/var/lib/kubelet/bootstrap-kubeconfig" \\
  --config=/var/lib/kubelet/kubelet-config.yaml \\
  --image-pull-progress-deadline=2m \\
  --kubeconfig=/var/lib/kubelet/kubeconfig \\
  --cert-dir=/var/lib/kubelet/pki/ \\
  --rotate-certificates=true \\
  --network-plugin=cni \\
  --register-node=true \\
  --v=2
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

Things to note here:
bootstrap-kubeconfig: Location of the bootstrap-kubeconfig file
cert-dir: The directory where the generated certificates are stored
rotate-certificates: Rotates client certificates when they expire

step 8 - configure the kubernetes proxy

in one of the previous steps we created the kube-proxy.kubeconfig file.
check here if you missed it

on worker-2

> sudo mv kube-proxy.kubeconfig /var/lib/kube-proxy/kubeconfig

create the kube-proxy-config.yaml configuration file

cat <<EOF | sudo tee /var/lib/kube-proxy/kube-proxy-config.yaml
kind: KubeProxyConfiguration
apiVersion: kubeproxy.config.k8s.io/v1alpha1
clientConnection:
  kubeconfig: "/var/lib/kube-proxy/kubeconfig"
mode: "iptables"
clusterCIDR: "192.168.5.0/24"
EOF

create the kube-proxy.service systemd unit file

cat <<EOF | sudo tee /etc/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Kube Proxy
Documentation=https://github.com/kubernetes/kubernetes

[Service]
ExecStart=/usr/local/bin/kube-proxy \\
  --config=/var/lib/kube-proxy/kube-proxy-config.yaml
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

step 9 - start the worker services

on worker-2

# reload daemon, enable and start kube-proxy
{
  sudo systemctl daemon-reload
  sudo systemctl enable kubelet kube-proxy
  sudo systemctl start kubelet kube-proxy
}

step 10 - approve server csr

on master-1

> kubectl get csr
  ---
  NAME                                                   AGE   REQUESTOR                 CONDITION
  node-csr-_4WyF-dRPZQlZw-kmsbaebyfj5MIRxklqY621Lv0cfo   30s   system:bootstrap:07401b   Pending

note: you have to change the expiration date in the config above

on master-1

> kubectl certificate approve node-csr-_4WyF-dRPZQlZw-kmsbaebyfj5MIRxklqY621Lv0cfo
  --
  certificatesigningrequest.certificates.k8s.io/node-csr-_4WyF ... approve

on master-1

> kubectl get nodes --kubeconfig admin.kubeconfig
  ---
  NAME       STATUS     ROLES    AGE     VERSION
  worker-1   NotReady   <none>   96m     v1.13.0
  worker-2   NotReady   <none>   2m56s   v1.13.0

— copyleft —

all commands shown on this page are from mmumshad’s fork of “kubernetes-the-hard-way” by kelseyhightower on github