[CKA][실습] 6. Security

Practice Test - View certificate details

## 인증서 확인
$ ps -ef | grep api
root        3281    2773  0 06:20 ?        00:00:14 kube-apiserver --advertise-address=192.18.243.3 --allow-privileged=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/pki/ca.crt --enable-admission-plugins=NodeRestriction --enable-bootstrap-token-auth=true --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key --etcd-servers=https://127.0.0.1:2379 --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key --requestheader-allowed-names=front-proxy-client --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/etc/kubernetes/pki/sa.pub --service-account-signing-key-file=/etc/kubernetes/pki/sa.key --service-cluster-ip-range=10.96.0.0/12 --tls-cert-file=/etc/kubernetes/pki/apiserver.crt --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
root        7281    7137  0 06:23 pts/0    00:00:00 grep --color=auto api

$ ps -ef | grep etcd
root        3281    2773  0 06:20 ?        00:00:32 kube-apiserver --advertise-address=192.18.243.3 --allow-privileged=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/pki/ca.crt --enable-admission-plugins=NodeRestriction --enable-bootstrap-token-auth=true --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key --etcd-servers=https://127.0.0.1:2379 --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key --requestheader-allowed-names=front-proxy-client --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/etc/kubernetes/pki/sa.pub --service-account-signing-key-file=/etc/kubernetes/pki/sa.key --service-cluster-ip-range=10.96.0.0/12 --tls-cert-file=/etc/kubernetes/pki/apiserver.crt --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
root        3301    2749  0 06:20 ?        00:00:16 etcd --advertise-client-urls=https://192.18.243.3:2379 --cert-file=/etc/kubernetes/pki/etcd/server.crt --client-cert-auth=true --data-dir=/var/lib/etcd --experimental-initial-corrupt-check=true --experimental-watch-progress-notify-interval=5s --initial-advertise-peer-urls=https://192.18.243.3:2380 --initial-cluster=controlplane=https://192.18.243.3:2380 --key-file=/etc/kubernetes/pki/etcd/server.key --listen-client-urls=https://127.0.0.1:2379,https://192.18.243.3:2379 --listen-metrics-urls=http://127.0.0.1:2381 --listen-peer-urls=https://192.18.243.3:2380 --name=controlplane --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt --peer-client-cert-auth=true --peer-key-file=/etc/kubernetes/pki/etcd/peer.key --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt --snapshot-count=10000 --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
root        8070    7137  0 06:28 pts/0    00:00:00 grep --color=auto etcd

$ openssl x509 -in /etc/kubernetes/pki/apiserver.crt -text -noout
Certificate:
    Data:
        Version: 3 (0x2)
        Serial Number: 7728013006652639726 (0x6b3f6ae2812109ee)
        Signature Algorithm: sha256WithRSAEncryption
        Issuer: CN = kubernetes
        Validity
            Not Before: Mar  6 11:20:50 2023 GMT
            Not After : Mar  5 11:20:51 2024 GMT
        Subject: CN = kube-apiserver
        Subject Public Key Info:
            Public Key Algorithm: rsaEncryption
                RSA Public-Key: (2048 bit)
                Modulus:
                    00:ac:a1:da:29:f1:84:84:6a:29:85:c2:b4:a7:57:
                    19:b8:17:15:ca:b9:f7:1d:46:ee:b7:35:6b:57:b7:
                    ae:a5:51:ed:bb:a2:eb:be:f1:18:1a:b5:e5:3a:c3:
                    fd:9e:42:2b:2a:44:2f:68:5d:53:a8:0d:5e:1e:03:
                    51:73:7b:ce:d8:59:5e:12:94:32:33:01:99:c2:ad:
                    e4:ce:ee
...
$ openssl x509 -in /etc/kubernetes/pki/etcd/server.crt -text -noout
Certificate:
    Data:
        Version: 3 (0x2)
        Serial Number: 339386737420626119 (0x4b5be70856f68c7)
        Signature Algorithm: sha256WithRSAEncryption
        Issuer: CN = etcd-ca
        Validity
            Not Before: Mar  6 11:20:51 2023 GMT
            Not After : Mar  5 11:20:51 2024 GMT
        Subject: CN = controlplane
...
$ openssl x509 -in /etc/kubernetes/pki/ca.crt -text -noout

• etcd 오류

## etcd 에러 로그 확인
$ cd /var/log/containers
$  ls -al | grep etcd
lrwxrwxrwx 1 root root   87 Mar  6 06:40 etcd-controlplane_kube-system_etcd-7b77e08c5d12d6aa6f33b9f3fc7b422e15757e41c11ffde943746e5dce94ca10.log -> /var/log/pods/kube-system_etcd-controlplane_42be5d217b25713f36ed515e78b23b3a/etcd/5.log
$ tail etcd-controlplane_kube-system_etcd-7b77e08c5d12d6aa6f33b9f3fc7b422e15757e41c11ffde943746e5dce94ca10.log
2023-03-06T06:40:26.017827949-05:00 stderr F {"level":"fatal","ts":"2023-03-06T11:40:26.017Z","caller":"etcdmain/etcd.go:219","msg":"listener failed","error":"open /etc/kubernetes/pki/etcd/server-certificate.crt: no such file or directory","stacktrace":"go.etcd.io/etcd/server/v3/etcdmain.startEtcdOrProxyV2\n\tgo.etcd.io/etcd/server/v3/etcdmain/etcd.go:219\ngo.etcd.io/etcd/server/v3/etcdmain.Main\n\tgo.etcd.io/etcd/server/v3/etcdmain/main.go:40\nmain.main\n\tgo.etcd.io/etcd/server/v3/main.go:32\nruntime.main\n\truntime/proc.go:225"}
$ ls -al /etc/kubernetes/pki/etcd/total 40
drwxr-xr-x 2 root root 4096 Mar  6 06:20 .
drwxr-xr-x 3 root root 4096 Mar  6 06:20 ..
-rw-r--r-- 1 root root 1086 Mar  6 06:20 ca.crt
-rw------- 1 root root 1679 Mar  6 06:20 ca.key
-rw-r--r-- 1 root root 1159 Mar  6 06:20 healthcheck-client.crt
-rw------- 1 root root 1679 Mar  6 06:20 healthcheck-client.key
-rw-r--r-- 1 root root 1208 Mar  6 06:20 peer.crt
-rw------- 1 root root 1675 Mar  6 06:20 peer.key
-rw-r--r-- 1 root root 1208 Mar  6 06:20 server.crt
-rw------- 1 root root 1679 Mar  6 06:20 server.key
$ vi /etc/kubernetes/manifests/etcd.yaml
apiVersion: v1
kind: Pod
metadata:
  annotations:
    kubeadm.kubernetes.io/etcd.advertise-client-urls: https://192.18.243.3:2379
  creationTimestamp: null
  labels:
    component: etcd
    tier: control-plane
  name: etcd
  namespace: kube-system
spec:
  containers:
  - command:
    - etcd
    - --advertise-client-urls=https://192.18.243.3:2379
    - --cert-file=/etc/kubernetes/pki/etcd/server-certificate.crt ## 수정 (server-certificate -> server)
    - --client-cert-auth=true
    - --data-dir=/var/lib/etcd
    - --experimental-initial-corrupt-check=true
    - --experimental-watch-progress-notify-interval=5s
    - --initial-advertise-peer-urls=https://192.18.243.3:2380
    - --initial-cluster=controlplane=https://192.18.243.3:2380
    - --key-file=/etc/kubernetes/pki/etcd/server.key
    - --listen-client-urls=https://127.0.0.1:2379,https://192.18.243.3:2379
    - --listen-metrics-urls=http://127.0.0.1:2381
    - --listen-peer-urls=https://192.18.243.3:2380
    - --name=controlplane
    - --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt
    - --peer-client-cert-auth=true
    - --peer-key-file=/etc/kubernetes/pki/etcd/peer.key
    - --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
    - --snapshot-count=10000
    - --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt

• kubeapi-server 오류

## kubeapi-server 에러 확인
$ cd /var/log/containers
$ tail kube-apiserver-controlplane_kube-system_kube-apiserver-b5b30d4378d56eb088d6d35208591945f042e42a1e382886cff04c34ed7bbb13.log
2023-03-06T06:50:56.811390239-05:00 stderr F I0306 11:50:56.811211       1 dynamic_cafile_content.go:171] "Shutting down controller" name="request-header::/etc/kubernetes/pki/front-proxy-ca.crt"
## 로그 확인 방법 2
$ crictl ps -a | grep api
11f57136ad9fe       a31e1d84401e6       29 seconds ago       Running              kube-apiserver            2                   1e2c9f93613f3       kube-apiserver-controlplane

$ crictl logs 1e2c9f93613f3
W0306 11:54:40.637582       1 logging.go:59] [core] [Channel #4 SubChannel #5] grpc: addrConn.createTransport failed to connect to {
  "Addr": "127.0.0.1:2379",
  "ServerName": "127.0.0.1",
  "Attributes": null,
  "BalancerAttributes": null,
  "Type": 0,
  "Metadata": null
}. Err: connection error: desc = "transport: authentication handshake failed: x509: certificate signed by unknown authority"
E0306 11:54:43.004649       1 run.go:74] "command failed" err="context deadline exceeded"

$ cat /etc/kubernetes/manifests/kube-apiserver.yaml | grep -i ca
    - --client-ca-file=/etc/kubernetes/pki/ca.crt
    - --etcd-cafile=/etc/kubernetes/pki/ca.crt
    - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt
    - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
    - --service-account-issuer=https://kubernetes.default.svc.cluster.local
      name: ca-certs
    - mountPath: /etc/ca-certificates
      name: etc-ca-certificates
    - mountPath: /usr/local/share/ca-certificates
      name: usr-local-share-ca-certificates
    - mountPath: /usr/share/ca-certificates
      name: usr-share-ca-certificates
  priorityClassName: system-node-critical
    name: ca-certs
      path: /etc/ca-certificates
    name: etc-ca-certificates
      path: /usr/local/share/ca-certificates
    name: usr-local-share-ca-certificates
      path: /usr/share/ca-certificates
    name: usr-share-ca-certificates
    
$ vi /etc/kubernetes/manifests/kube-apiserver.yaml
apiVersion: v1
kind: Pod
metadata:
  annotations:
    kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint: 192.18.243.3:6443
  creationTimestamp: null
  labels:
    component: kube-apiserver
    tier: control-plane
  name: kube-apiserver
  namespace: kube-system
spec:
  containers:
  - command:
    - kube-apiserver
    - --advertise-address=192.18.243.3
    - --allow-privileged=true
    - --authorization-mode=Node,RBAC
    - --client-ca-file=/etc/kubernetes/pki/ca.crt
    - --enable-admission-plugins=NodeRestriction
    - --enable-bootstrap-token-auth=true
    - --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt  ## 수정
    - --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt
    - --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key
    - --etcd-servers=https://127.0.0.1:2379
    - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt