k8s thw 15 - smoke test
data encryption
create a generic secret on master
> kubectl create secret generic kubernetes-the-hard-way \
--from-literal="mykey=mydata"
print a hexdump of the kubernetes-the-hard-way secret stored in etcd
> sudo ETCDCTL_API=3 etcdctl get \
--endpoints=https://127.0.0.1:2379 \
--cacert=/etc/etcd/ca.crt \
--cert=/etc/etcd/etcd-server.crt \
--key=/etc/etcd/etcd-server.key\
/registry/secrets/default/kubernetes-the-hard-way | hexdump -C
# output
---
00000000 2f 72 65 67 69 73 74 72 79 2f 73 65 63 72 65 74 |/registry/secret|
00000010 73 2f 64 65 66 61 75 6c 74 2f 6b 75 62 65 72 6e |s/default/kubern|
00000020 65 74 65 73 2d 74 68 65 2d 68 61 72 64 2d 77 61 |etes-the-hard-wa|
00000030 79 0a 6b 38 73 3a 65 6e 63 3a 61 65 73 63 62 63 |y.k8s:enc:aescbc|
00000040 3a 76 31 3a 6b 65 79 31 3a 89 99 96 d7 8b 49 b2 |:v1:key1:.....I.|
00000050 9e a6 44 01 fe 75 d4 06 cf c5 23 8c 27 09 5e 99 |..D..u....#.'.^.|
00000060 61 10 4a 86 65 93 ec de 0a e5 bf 11 46 c9 59 06 |a.J.e.......F.Y.|
00000070 de 92 30 23 a8 aa d2 73 cb 11 e4 bc 9f a7 ad 72 |..0#...s.......r|
00000080 1c 0c e9 f5 52 06 13 55 aa c7 2f 33 cf e3 a3 cc |....R..U../3....|
00000090 34 4e c2 c0 1a 27 0b 46 59 f8 bb 6a 86 c8 12 d5 |4N...'.FY..j....|
000000a0 78 c2 a5 ff 4b 0b f9 d0 9e 18 bc 65 e9 0f f0 e9 |x...K......e....|
000000b0 e4 e2 37 df f9 33 4f 7f f4 c4 9c 85 8f 07 60 b4 |..7..3O.......`.|
000000c0 3a a9 44 4e 83 12 15 70 7b 52 cb 71 18 8d 8b f2 |:.DN...p{R.q....|
000000d0 04 8f b9 97 cc 7b 92 35 5a cb 8d 99 54 94 d7 7f |.....{.5Z...T...|
000000e0 25 7b 9a bf dd d9 f3 f1 11 0a |%{........|
the etcd key should be prefixed with k8s:enc:aescbc:v1:key1 - which indicates the aescbc provider was used to encrypt the data with the key1 encryption key.
# delete secret
> kubectl delete secret kubernetes-the-hard-way
deployment
# create a nginx web server deployment
> kubectl create deployment nginx --image=nginx
# list pods in nginx deployment
> kubectl get pods -l app=nginx
---
NAME READY STATUS RESTARTS AGE
nginx-5c7588df-6rdpj 1/1 Running 0 86s
services
# create a service to expose deployment nginx on node ports
> kubectl expose deploy nginx --type=NodePort --port 80
# setup port number env
> PORT_NUMBER=$(kubectl get svc -l app=nginx -o jsonpath="{.items[0].spec.ports[0].nodePort}")
# show nginx test pages on booth workers
> curl http://worker-1:$PORT_NUMBER
> curl http://worker-2:$PORT_NUMBER
---
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
...
logs
# retrieve full name of the nginx pod
# set pod name env
> POD_NAME=$(kubectl get pods -l app=nginx -o jsonpath="{.items[0].metadata.name}")
# print nginx pod logs
> kubectl logs $POD_NAME
---
Error from server: Get https://worker-2:10250/containerLogs/default/nginx-5c7588df-6rdpj/nginx: x509: certificate signed by unknown authority
troubleshoot - logs:
adding --rotate-server-certificates=true to /etc/systemd/system/kubelet.service file on each worker node and restarting the service
cat <<EOF | sudo tee /etc/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/kubernetes/kubernetes
After=docker.service
Requires=docker.service
[Service]
ExecStart=/usr/local/bin/kubelet \
--bootstrap-kubeconfig="/var/lib/kubelet/bootstrap-kubeconfig" \
--config=/var/lib/kubelet/kubelet-config.yaml \
--image-pull-progress-deadline=2m \
--kubeconfig=/var/lib/kubelet/kubeconfig \
--cert-dir=/var/lib/kubelet/pki/ \
--rotate-certificates=true \
--rotate-server-certificates=true \
--network-plugin=cni \
--register-node=true \
--v=2
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
# reload daemon
> sudo systemctl daemon-reload
# restart kubelet
> sudo systemctl restart kubelet.service
# try again - error
> kubectl logs $POD_NAME
---
Error from server: Get https://worker-2:10250/containerLogs/default/nginx-5c7588df-6rdpj/nginx: remote error: tls: internal error
# check csr
> kubectl get csr
---
NAME AGE REQUESTOR CONDITION
csr-2b5dn 56s system:node:worker-2 Pending
csr-9jqs9 73s system:node:worker-1 Pending
csr-rqtlf 74s system:node:worker-1 Approved,Issued
# approve pending requests
> kubectl certificate approve csr-2b5dn csr-9jqs9
---
certificatesigningrequest.certificates.k8s.io/csr-2b5dn approved
certificatesigningrequest.certificates.k8s.io/csr-9jqs9 approved
# try again - and it worked
> kubectl logs $POD_NAME
---
/docker-entrypoint.sh: Configuration complete; ready for start up
2021/09/19 12:14:19 [notice] 1#1: using the "epoll" event method
2021/09/19 12:14:19 [notice] 1#1: nginx/1.21.3
2021/09/19 12:14:19 [notice] 1#1: built by gcc 8.3.0 (Debian 8.3.0-6)
2021/09/19 12:14:19 [notice] 1#1: OS: Linux 4.15.0-156-generic
2021/09/19 12:14:19 [notice] 1#1: getrlimit(RLIMIT_NOFILE): 1048576:1048576
2021/09/19 12:14:19 [notice] 1#1: start worker processes
2021/09/19 12:14:19 [notice] 1#1: start worker process 31
10.32.0.1 - - [19/Sep/2021:12:17:15 +0000] "GET / HTTP/1.1" 200 615 "-" "curl/7.58.0" "-"
10.44.0.0 - - [19/Sep/2021:12:17:22 +0000] "GET / HTTP/1.1" 200 615 "-" "curl/7.58.0" "-"
print nginx version by executing the nginx -v command inside the nginx container
> kubectl exec -ti $POD_NAME -- nginx -v
---
nginx version: nginx/1.21.3
— copyleft —
all commands shown on this page are from mmumshad’s fork of “kubernetes-the-hard-way” by kelseyhightower on github
19-09-2021
