Certificated Kubernetes Administrator - Secret

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

# kubectl describe pods -n kube-system kube-apiserver-controlplane
Command:
      kube-apiserver
      --advertise-address=192.8.216.9
      --allow-privileged=true
      --authorization-mode=Node,RBAC
      --client-ca-file=/etc/kubernetes/pki/ca.crt
      --enable-admission-plugins=NodeRestriction
      --enable-bootstrap-token-auth=true
      --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt
      --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt
      --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key
      --etcd-servers=https://127.0.0.1:2379
      --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt
      --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
      --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
      --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
      --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
      --requestheader-allowed-names=front-proxy-client
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
      --requestheader-extra-headers-prefix=X-Remote-Extra-
      --requestheader-group-headers=X-Remote-Group
      --requestheader-username-headers=X-Remote-User
      --secure-port=6443
      --service-account-issuer=https://kubernetes.default.svc.cluster.local
      --service-account-key-file=/etc/kubernetes/pki/sa.pub
      --service-account-signing-key-file=/etc/kubernetes/pki/sa.key
      --service-cluster-ip-range=10.96.0.0/12
      --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
      --tls-private-key-file=/etc/kubernetes/pki/apiserver.key

#kubectl describe pods -n kube-system etcd-controlplane
Command:
      etcd
      --advertise-client-urls=https://192.8.216.9:2379
      --cert-file=/etc/kubernetes/pki/etcd/server.crt
      --client-cert-auth=true
      --data-dir=/var/lib/etcd
      --experimental-initial-corrupt-check=true
      --experimental-watch-progress-notify-interval=5s
      --initial-advertise-peer-urls=https://192.8.216.9:2380
      --initial-cluster=controlplane=https://192.8.216.9:2380
      --key-file=/etc/kubernetes/pki/etcd/server.key
      --listen-client-urls=https://127.0.0.1:2379,https://192.8.216.9:2379
      --listen-metrics-urls=http://127.0.0.1:2381
      --listen-peer-urls=https://192.8.216.9:2380
      --name=controlplane
      --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt
      --peer-client-cert-auth=true
      --peer-key-file=/etc/kubernetes/pki/etcd/peer.key
      --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
      --snapshot-count=10000
      --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

openssl x509 -in /etc/kubernetes/pki/apiserver.crt -text -noout
Certificate:
    Data:
        Version: 3 (0x2)
        Serial Number: 4169429065381004709 (0x39dcc9a4f63945a5)
        Signature Algorithm: sha256WithRSAEncryption
        Issuer: CN = kubernetes
        Validity
            Not Before: May 26 06:08:28 2024 GMT
            Not After : May 26 06:13:28 2025 GMT
        Subject: CN = kube-apiserver
        Subject Public Key Info:
            Public Key Algorithm: rsaEncryption
                Public-Key: (2048 bit)
                Modulus:
                    00:a9:e4:03:72:0f:63:ee:57:c2:25:b7:ff:57:6a:
                    9f:78:37:07:52:11:48:98:d8:b4:a3:ca:4a:05:36:
                    52:af:0d:50:69:60:60:1d:b9:e7:1a:98:6f:e4:2b:
                    73:66:aa:9d:77:ce:f3:18:e7:b1:09:83:ec:bb:ce:
                    5f:fd:88:a1:48:7f:50:94:26:a1:4b:ef:12:bf:92:
                    25:ce:a5:8d:6c:bb:7a:3e:6f:f5:78:2f:47:8a:e3:
                    18:b9:14:11:69:f2:a4:0d:4c:f9:c7:1b:e2:b5:ab:
                    09:7b:7b:3a:cf:37:bf:e6:8b:8e:2f:b1:89:0c:2a:
                    bc:2a:a1:14:b2:04:61:6c:6e:01:42:52:bb:c9:bf:
                    f0:a2:cf:53:fd:0a:be:64:34:1a:f0:93:84:57:4f:
                    f3:9a:b9:02:41:f9:21:3e:4a:a8:a4:4b:ca:41:7b:
                    8f:ed:98:b1:ef:91:58:20:ba:dd:74:45:a4:13:81:
                    ba:6f:7b:07:8d:d6:6d:bf:03:15:01:40:4a:e1:52:
                    1d:ba:a4:5e:80:1a:42:9f:0e:4a:b4:ef:0f:d0:5c:
                    51:c0:90:e6:44:e5:4e:03:7f:1c:85:6f:b6:26:23:
                    52:f7:e2:f8:b8:77:6f:81:36:93:ea:42:55:90:7a:
                    c0:7c:e4:f4:66:5b:a0:45:c2:e0:f3:99:53:1e:aa:
                    2b:95
                Exponent: 65537 (0x10001)
        X509v3 extensions:
            X509v3 Key Usage: critical
                Digital Signature, Key Encipherment
            X509v3 Extended Key Usage:
                TLS Web Server Authentication
            X509v3 Basic Constraints: critical
                CA:FALSE
            X509v3 Authority Key Identifier:
                F3:54:72:08:EE:F6:FC:9A:27:C9:61:F6:37:6D:32:B3:8B:86:A7:9B
            X509v3 Subject Alternative Name:
                DNS:controlplane, DNS:kubernetes, DNS:kubernetes.default, DNS:kubernetes.default.svc, DNS:kubernetes.default.svc.cluster.local, IP Address:10.96.0.1, IP Address:192.8.216.9
    Signature Algorithm: sha256WithRSAEncryption
    Signature Value:
        62:9d:43:f9:a8:db:73:5f:36:fe:8e:14:b4:00:9a:27:f5:7f:
        76:6a:ab:64:a4:65:7c:2a:ca:43:c7:6a:f3:fd:10:0c:a3:d5:
        f8:68:31:23:3a:0a:7e:10:51:1c:1d:9c:36:d0:05:2a:12:97:
        6e:8d:29:d4:9b:42:95:54:42:77:b6:cc:72:82:7e:98:f7:8b:
        8d:77:f9:cf:ae:4e:1a:d4:3c:a4:67:e5:94:1e:a7:d9:dd:8a:
        73:e4:d6:58:32:0e:4d:0e:8c:1b:51:a7:85:06:23:1c:ae:cf:
        30:2f:62:24:1c:1c:1b:a8:45:c9:69:59:08:17:c9:0d:b8:08:
        99:f4:01:05:76:2a:e7:4c:a2:f1:7e:71:a0:b6:6e:95:c5:55:
        3b:8c:67:a6:e1:e5:31:ca:2a:9d:b0:45:df:89:d0:77:2c:d9:
        9a:89:ee:35:99🆎34:8e:b3:41:a7:f7:3a:6e:83:4f:a1:05:
        65:42:7b:db:91:9f:0f:55:36:ec:82:12:78:cf:bf:57:1a:95:
        31:18:f2:6e:5a:7b:61:11:cc:06:2b:f0:2c:32:dc:b9:b0:af:
        e6:55:44:f8:98:0b:76:02:72:40:a4:7a:1a:84:2a:40:e1:a8:
        e2:c2:f4:3a:8c:da:9b:83:ca:05:7b:81:b9:48:ac:6e:58:13:

1
2

127.0.0.1:2379", }. Err: connection error: desc = "transport: authentication handshake failed: tls: failed to verify certificate: x509: certificate signed by unknown authority"
W0526 06:48:05.594458       1 logging.go:59] [core] [Channel #1 SubChannel #3] grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:2379", ServerName: "127.0.0.1:2379", }. Err: connection error: desc = "transport: authentication handshake failed: tls: failed to verify certificate: x509: certificate signed by unknown authority"

1
2
3
4
5

cat /etc/kubernetes/manifests/kube-apiserver.yaml | grep "etcd"
    - --etcd-cafile=/etc/kubernetes/pki/ca.crt
    - --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt
    - --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key
    - --etcd-servers=https://127.0.0.1:2379

1
2
3
4
5

# generate private key
openssl genrsa -out mykey.key 2048

# generate csr
openssl req -new -key mykey.key -out mycsr.csr -sujb "/CN=example.com/O=My organization/C=US"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

# base64 로 인코딩
cat mycsr.csr | base64 -w 0

vi create-csr-object.yaml
# apiVersion: certificates.k8s.io/v1
# kind: CertificateSigningRequest
# metadata:
#   name: temp
# spec:
#   groups:
#   - system:authenticated
#   request: <위에서 인코딩한 값>
#   signerName: kubernetes.io/kube-apiserver-client
#   usages:
#   - client auth

1
2

kubectl certificate approve temp
kubectl certificate deny temp

1
2
3
4
5
6
7
8

kubectl describe role -n kube-system kube-proxy
Name:         kube-proxy
Labels:       <none>
Annotations:  <none>
PolicyRule:
  Resources   Non-Resource URLs  Resource Names  Verbs
  ---------   -----------------  --------------  -----
  configmaps  []                 [kube-proxy]    [get]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

kubectl describe rolebindings.rbac.authorization.k8s.io -n kube-system kube-proxy
Name:         kube-proxy
Labels:       <none>
Annotations:  <none>
Role:
  Kind:  Role
  Name:  kube-proxy
Subjects:
  Kind   Name                                             Namespace
  ----   ----                                             ---------
  Group  system:bootstrappers:kubeadm:default-node-token

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

kubectl create role developer --verb=list,create,delete --resource=pods
kubectl create rolebinding  dev-user-binding --role=developer --user=dev-user

# 추가로 여러개의 리소스에 대한 규칙을 하나의 role에 부여하는 것도 가능하다.
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  creationTimestamp: "2024-05-26T14:36:47Z"
  name: developer
  namespace: blue
  resourceVersion: "2165"
  uid: bd5331df-f342-4d58-88b5-fff760d5b765
rules:
- apiGroups:
  - ""
  resourceNames:
  - blue-app
  - dark-blue-app
  resources:
  - pods
  verbs:
  - get
  - watch
  - create
  - delete
- apiGroups:
  - apps
  resources:
  - deployments
  verbs:
  - create

1
2
3
4
5

kubectl create deployment test --image busybox --dry-run=client --as dev-user
deployment.apps/test created (dry run)

kubectl get deployments.apps --as dev-user
Error from server (Forbidden): deployments.apps is forbidden: User "dev-user" cannot list resource "deployments" in API group "apps" in the namespace "default"

1
2
3
4
5

kubectl create secret docker-registry  <name> \
  --docker-server=DOCKER_REGISTRY_SERVER \
  --docker-username=DOCKER_USER \
  --docker-password=DOCKER_PASSWORD \
  --docker-email=DOCKER_EMAIL

1
2
3
4
5

spec:
  imagePullSecrets:
   - name : <name>
  containers:
   ...

1

kubectl exec [pod-name] -- whoami

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

 apiVersion: v1
kind: Pod
metadata:
  name: multi-pod
spec:
  securityContext:
    runAsUser: 1001
  containers:
  -  image: ubuntu
     name: web
     command: ["sleep", "5000"]
     securityContext:
      runAsUser: 1002

  -  image: ubuntu
     name: sidecar
     command: ["sleep", "5000"]

1
2
3

securityContext:
  capabilities:
    add: ["NET_ADMIN", "SYS_TIME"]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: internal-policy
  namespace: default
spec:
  podSelector:
    matchLabels:
      name: internal
  policyTypes:
    - Egress
    - Ingress
  ingress:
    - {}
  egress:
    - to:
        - podSelector:
            matchLabels:
              name: mysql
      ports:
        - protocol: TCP
          port: 3306

    - to:
        - podSelector:
            matchLabels:
              name: payroll
      ports:
        - protocol: TCP
          port: 8080

    - ports:
        - port: 53
          protocol: UDP
        - port: 53
          protocol: TCP