data encryption

create a generic secret on master

> kubectl create secret generic kubernetes-the-hard-way \
  --from-literal="mykey=mydata"

print a hexdump of the kubernetes-the-hard-way secret stored in etcd

> sudo ETCDCTL_API=3 etcdctl get \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/etcd/ca.crt \
  --cert=/etc/etcd/etcd-server.crt \
  --key=/etc/etcd/etcd-server.key\
  /registry/secrets/default/kubernetes-the-hard-way | hexdump -C

# output
  ---
  00000000  2f 72 65 67 69 73 74 72  79 2f 73 65 63 72 65 74  |/registry/secret|                                               
  00000010  73 2f 64 65 66 61 75 6c  74 2f 6b 75 62 65 72 6e  |s/default/kubern|                                               
  00000020  65 74 65 73 2d 74 68 65  2d 68 61 72 64 2d 77 61  |etes-the-hard-wa|                                               
  00000030  79 0a 6b 38 73 3a 65 6e  63 3a 61 65 73 63 62 63  |y.k8s:enc:aescbc|                                               
  00000040  3a 76 31 3a 6b 65 79 31  3a 89 99 96 d7 8b 49 b2  |:v1:key1:.....I.|                                               
  00000050  9e a6 44 01 fe 75 d4 06  cf c5 23 8c 27 09 5e 99  |..D..u....#.'.^.|                                               
  00000060  61 10 4a 86 65 93 ec de  0a e5 bf 11 46 c9 59 06  |a.J.e.......F.Y.|                                               
  00000070  de 92 30 23 a8 aa d2 73  cb 11 e4 bc 9f a7 ad 72  |..0#...s.......r|                                               
  00000080  1c 0c e9 f5 52 06 13 55  aa c7 2f 33 cf e3 a3 cc  |....R..U../3....|                                               
  00000090  34 4e c2 c0 1a 27 0b 46  59 f8 bb 6a 86 c8 12 d5  |4N...'.FY..j....|                                               
  000000a0  78 c2 a5 ff 4b 0b f9 d0  9e 18 bc 65 e9 0f f0 e9  |x...K......e....|                                               
  000000b0  e4 e2 37 df f9 33 4f 7f  f4 c4 9c 85 8f 07 60 b4  |..7..3O.......`.|                                               
  000000c0  3a a9 44 4e 83 12 15 70  7b 52 cb 71 18 8d 8b f2  |:.DN...p{R.q....|                                               
  000000d0  04 8f b9 97 cc 7b 92 35  5a cb 8d 99 54 94 d7 7f  |.....{.5Z...T...|                                               
  000000e0  25 7b 9a bf dd d9 f3 f1  11 0a                    |%{........|

the etcd key should be prefixed with k8s:enc:aescbc:v1:key1 - which indicates the aescbc provider was used to encrypt the data with the key1 encryption key.

# delete secret
> kubectl delete secret kubernetes-the-hard-way

deployment

# create a nginx web server deployment
> kubectl create deployment nginx --image=nginx

# list pods in nginx deployment
> kubectl get pods -l app=nginx
  ---
  NAME                   READY   STATUS    RESTARTS   AGE
  nginx-5c7588df-6rdpj   1/1     Running   0          86s

services

# create a service to expose deployment nginx on node ports
> kubectl expose deploy nginx --type=NodePort --port 80

# setup port number env
> PORT_NUMBER=$(kubectl get svc -l app=nginx -o jsonpath="{.items[0].spec.ports[0].nodePort}")

# show nginx test pages on booth workers
> curl http://worker-1:$PORT_NUMBER
> curl http://worker-2:$PORT_NUMBER
  ---
  <!DOCTYPE html>
  <html>
  <head>
  <title>Welcome to nginx!</title>
  ...

logs

# retrieve full name of the nginx pod
# set pod name env
> POD_NAME=$(kubectl get pods -l app=nginx -o jsonpath="{.items[0].metadata.name}")

# print nginx pod logs
> kubectl logs $POD_NAME
  ---
  Error from server: Get https://worker-2:10250/containerLogs/default/nginx-5c7588df-6rdpj/nginx: x509: certificate signed by unknown authority

troubleshoot - logs:
adding --rotate-server-certificates=true to /etc/systemd/system/kubelet.service file on each worker node and restarting the service

cat <<EOF | sudo tee /etc/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/kubernetes/kubernetes
After=docker.service
Requires=docker.service

[Service]
ExecStart=/usr/local/bin/kubelet \
--bootstrap-kubeconfig="/var/lib/kubelet/bootstrap-kubeconfig" \
--config=/var/lib/kubelet/kubelet-config.yaml \
--image-pull-progress-deadline=2m \
--kubeconfig=/var/lib/kubelet/kubeconfig \
--cert-dir=/var/lib/kubelet/pki/ \
--rotate-certificates=true \
--rotate-server-certificates=true \
--network-plugin=cni \
--register-node=true \
--v=2
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF
# reload daemon
> sudo systemctl daemon-reload

# restart kubelet
> sudo systemctl restart kubelet.service

# try again - error
> kubectl logs $POD_NAME
  ---
  Error from server: Get https://worker-2:10250/containerLogs/default/nginx-5c7588df-6rdpj/nginx: remote error: tls: internal error

# check csr
> kubectl get csr
  ---
  NAME        AGE   REQUESTOR              CONDITION
  csr-2b5dn   56s   system:node:worker-2   Pending
  csr-9jqs9   73s   system:node:worker-1   Pending
  csr-rqtlf   74s   system:node:worker-1   Approved,Issued

# approve pending requests
> kubectl certificate approve csr-2b5dn csr-9jqs9
  ---
  certificatesigningrequest.certificates.k8s.io/csr-2b5dn approved
  certificatesigningrequest.certificates.k8s.io/csr-9jqs9 approved

# try again - and it worked
> kubectl logs $POD_NAME
  ---
  /docker-entrypoint.sh: Configuration complete; ready for start up                                                            
  2021/09/19 12:14:19 [notice] 1#1: using the "epoll" event method                                                             
  2021/09/19 12:14:19 [notice] 1#1: nginx/1.21.3
  2021/09/19 12:14:19 [notice] 1#1: built by gcc 8.3.0 (Debian 8.3.0-6)                                                        
  2021/09/19 12:14:19 [notice] 1#1: OS: Linux 4.15.0-156-generic
  2021/09/19 12:14:19 [notice] 1#1: getrlimit(RLIMIT_NOFILE): 1048576:1048576                                                  
  2021/09/19 12:14:19 [notice] 1#1: start worker processes
  2021/09/19 12:14:19 [notice] 1#1: start worker process 31
  10.32.0.1 - - [19/Sep/2021:12:17:15 +0000] "GET / HTTP/1.1" 200 615 "-" "curl/7.58.0" "-"                                    
  10.44.0.0 - - [19/Sep/2021:12:17:22 +0000] "GET / HTTP/1.1" 200 615 "-" "curl/7.58.0" "-"

print nginx version by executing the nginx -v command inside the nginx container

> kubectl exec -ti $POD_NAME -- nginx -v
  ---
  nginx version: nginx/1.21.3

— copyleft —

all commands shown on this page are from mmumshad’s fork of “kubernetes-the-hard-way” by kelseyhightower on github