[!TIP]

certificates
1.6.1.4.1 -- certificates

generic

  • certificates for CA

    • ca-key.pem
    • ca.pem
    • kubernetes-key.pem
    • kubernetes.pem
    • kube-proxy.pem
    • kube-proxy-key.pem
    • admin.pem
    • admin-key.pem
  • certificate component

SERVICES CERTIFICATES
etcd ca.pem, kubernetes-key.pem, kubernetes.pem
kube-apiserver ca.pem, kubernetes-key.pem, kubernetes.pem
kubelet ca.pem
kube-proxy ca.pem, kube-proxy-key.pem, kube-proxy.pem
kubectl ca.pem, admin-key.pem, admin.pem
kube-controller-manager ca-key.pem, ca.pem

samples

etcd

/usr/local/bin/etcd \\
  --cert-file=/etc/etcd/kube-etcd.pem \\                   # 对外提供服务的服务器证书
  --key-file=/etc/etcd/kube-etcd-key.pem \\                # 服务器证书对应的私钥
  --peer-cert-file=/etc/etcd/kube-etcd-peer.pem \\         # peer 证书,用于 etcd 节点之间的相互访问
  --peer-key-file=/etc/etcd/kube-etcd-peer-key.pem \\      # peer 证书对应的私钥
  --trusted-ca-file=/etc/etcd/cluster-root-ca.pem \\       # 用于验证访问 etcd 服务器的客户端证书的 CA 根证书
  --peer-trusted-ca-file=/etc/etcd/cluster-root-ca.pem\\   # 用于验证 peer 证书的 CA 根证书
  ...

kube-apiserver

/usr/local/bin/kube-apiserver \\
  --tls-cert-file=/var/lib/kubernetes/kube-apiserver.pem \\                             # 用于对外提供服务的服务器证书
  --tls-private-key-file=/var/lib/kubernetes/kube-apiserver-key.pem \\                  # 服务器证书对应的私钥
  --etcd-certfile=/var/lib/kubernetes/kube-apiserver-etcd-client.pem \\                 # 用于访问 etcd 的客户端证书
  --etcd-keyfile=/var/lib/kubernetes/kube-apiserver-etcd-client-key.pem \\              # 用于访问 etcd 的客户端证书的私钥
  --kubelet-client-certificate=/var/lib/kubernetes/kube-apiserver-kubelet-client.pem \\ # 用于访问 kubelet 的客户端证书
  --kubelet-client-key=/var/lib/kubernetes/kube-apiserver-kubelet-client-key.pem \\     # 用于访问 kubelet 的客户端证书的私钥
  --client-ca-file=/var/lib/kubernetes/cluster-root-ca.pem \\                           # 用于验证访问 kube-apiserver 的客户端的证书的 CA 根证书
  --etcd-cafile=/var/lib/kubernetes/cluster-root-ca.pem \\                              # 用于验证 etcd 服务器证书的 CA 根证书
  --kubelet-certificate-authority=/var/lib/kubernetes/cluster-root-ca.pem \\            # 用于验证 kubelet 服务器证书的 CA 根证书
  --service-account-key-file=/var/lib/kubernetes/service-account.pem \\                 # 用于验证 service account token 的公钥
  ...

show secrets tls.crt

[!TIP|label:references:]

create secrets

  • by command

    $ kubectl create secret tls my-certs \
              --key .devops/certs/server.key \
              --cert .devops/certs/server.crt \
              -n ingress-nginx
    
  • by yaml

    $ echo "apiVersion: v1
    kind: Secret
    type: kubernetes.io/tls
    metadata:
      name: mytest-cert
      namespace: ingress-nginx
    data:
      tls.crt: $(cat $HOME/.devops/certs/server.csr | base64 -w0)
      tls.key: $(cat $HOME/.devops/certs/server.key | base64 -w0)" |
    kubectl apply -f -
    

duplicate secrets to the other ns

$ kubectl -n ingress-nginx get secrets my-certs -o yaml --export | kubectl apply -n devops -f -

show server.crt

$ kubectl -n kube-system \
          get secrets sample-tls \
          -o yaml \
          -o "jsonpath={.data['tls\.crt']}" |
    base64 -d -w0 |
    sed '/-----END CERTIFICATE-----/q' |
    openssl x509 -text -noout |
    grep 'Not'
            Not Before: Sep 14 00:00:00 2021 GMT
            Not After : Aug 18 23:59:59 2022 GM

show all tls

$ kubectl get ingress --all-namespaces --no-headers |
          awk '{print $1}' |
          sort -u |
          while read -r ns; do
            echo "-- ${ns} --";
              kubectl -n ${ns} get secret sample-tls -o yaml -o "jsonpath={.data['tls\.crt']}" |
                      base64 -d -w0 |
                      sed '/-----END CERTIFICATE-----/q' |
                      openssl x509 -text -noout | grep 'Not'
          done

show ca.crt

$ kubectl -n kube-system \
          get secrets sample-tls \
          -o yaml \
          -o "jsonpath={.data['tls\.crt']}" |
    base64 -d -w0 |
    awk '/-BEGIN CERTIFICATE-/ && c++, /-END CERTIFICATE-/' |
    openssl x509 -text -noout |
    grep 'Not'
            Not Before: Apr 14 00:00:00 2021 GMT
            Not After : Apr 13 23:59:59 2031 GMT

renew both certificates and kubeconfig

check info

crt

$ find /etc/kubernetes/pki/ -type f -name "*.crt" -print |
       egrep -v 'ca.crt$' |
       xargs -L 1 -t -i bash -c 'openssl x509 -noout -text -in {} | grep After'
bash -c openssl x509 -noout -text -in /etc/kubernetes/pki/apiserver-kubelet-client.crt | grep After
            Not After : Sep 16 07:51:58 2020 GMT
bash -c openssl x509 -noout -text -in /etc/kubernetes/pki/apiserver.crt | grep After
            Not After : Sep 16 07:51:59 2020 GMT
bash -c openssl x509 -noout -text -in /etc/kubernetes/pki/front-proxy-client.crt | grep After
            Not After : Sep 16 07:52:00 2020 GMT
  • or

    $ find /etc/kubernetes/pki/ -type f -name "*.crt" -print |
           egrep -v 'ca.crt$' |
           xargs -L 1 -t  -i bash -c 'openssl x509 -enddate -noout -in {}'
    
  • or

    $ ls -1 /etc/kubernetes/pki/*.crt |
           grep -Ev 'ca.crt$' |
           xargs -L 1 -t  -i bash -c 'openssl x509 -enddate -noout -in {}'
    

pem for external etcd

  $ for i in ca client server peer; do
      echo /etc/etcd/ssl/$i.pem
      openssl x509 -enddate -noout -in /etc/etcd/ssl/$i.pem
    done
  /etc/etcd/ssl/ca.pem
  notAfter=Sep  8 10:44:00 2024 GMT
  /etc/etcd/ssl/client.pem
  notAfter=Sep  8 10:49:00 2024 GMT
  /etc/etcd/ssl/server.pem
  notAfter=Sep  8 11:03:00 2024 GMT
  /etc/etcd/ssl/peer.pem
  notAfter=Sep  8 11:03:00 2024 GMT
  • or

    $ find /etc/etcd/ssl/ -type f -name '*.pem' |
           egrep -v '*-key.pem$' |
           xargs -L 1 -t -i bash -c 'openssl x509 -enddate -noout -in {}'
    bash -c openssl x509 -enddate -noout -in /etc/etcd/ssl/ca.pem
    notAfter=Sep  8 10:44:00 2024 GMT
    bash -c openssl x509 -enddate -noout -in /etc/etcd/ssl/client.pem
    notAfter=Sep  8 10:49:00 2024 GMT
    bash -c openssl x509 -enddate -noout -in /etc/etcd/ssl/server.pem
    notAfter=Sep  8 11:03:00 2024 GMT
    bash -c openssl x509 -enddate -noout -in /etc/etcd/ssl/peer.pem
    notAfter=Sep  8 11:03:00 2024 GMT
    
  • or

    $ ls -1 /etc/etcd/ssl/*.pem |
            grep -Ev '\-key.pem$' |
            xargs -L 1 -t  -i bash -c 'openssl x509 -enddate -noout -in {}'
    

backup

# timestampe=$(date +"%Y%m%d%H%M%S")
$ timestampe=$(date +"%Y%m%d")
$ backupFolder="$HOME/k8s-cert-expired-${timestampe}"

$ mkdir "${backupFolder}"
$ sudo cp -rp --parents /etc/kubernetes/pki "${backupFolder}"

# for external etcd
# sudo cp -rp --parents /etc/etcd/ssl "${backupFolder}"

# for kubelet
$ sudo cp -rp /var/lib/kubelet/config.yaml{,.backup.${timestampe}}
$ sudo cp -rp --parents /var/lib/kubelet/pki "${backupFolder}"
$ sudo cp -r /var/lib/kubelet/pki{,.backup.${timestampe}}
$ sudo cp -rp --parents /var/lib/kubelet/config.yaml "${backupFolder}"

# for kubeconfig
$ sudo cp -rp --parents /etc/kubernetes/*.conf "${backupFolder}"
$ sudo cp -rp ~/.kube/config{,.backup.${timestampe}}

clean environment

# for `/etc/kubernetes/pki`

# or
$ echo {apiserver,apiserver-kubelet-client,apiserver-etcd-client,front-proxy-client} |
       fmt -1 |
       xargs -I{} bash -c "sudo cp -rp /etc/kubernetes/pki/{}.crt{,.backup.${timestampe}};
                           sudo mv /etc/kubernetes/pki/{}.key{,.backup.${timestampe}}"

# for kubeconfig
$ echo {admin,kubelet,controller-manager,scheduler} |
       fmt -1 |
       xargs -I{} bash -c "sudo mv /etc/kubernetes/{}.conf{,.backup.${timestampe}}"

$ echo {peer,healthcheck-client,server}.{crt,key} |
       fmt -1 |
       xargs -I{} bash -c "sudo mv /etc/kubernets/pki/etcd/${}{,.backup.${timestampe}}"

restore backup

TBD

v1.12.3

[!TIP]

$ kubectl version --short
Client Version: v1.12.3
Server Version: v1.12.3

references:

renew certificates

# get target cluster kubeadm-cfg.yml
$ kubectl get cm kubeadm-config -n kube-system -o=jsonpath="{.data.ClusterConfiguration}"
$ sudo kubeadm [--config kubeadm.yml] alpha phase certs renew [commands]

Available Commands:

commands comments
all renew all available certificates
apiserver Generates the certificate for serving the kubernetes API
apiserver-etcd-client Generates the client apiserver uses to access etcd
apiserver-kubelet-client Generates the Client certificate for the API server to connect to kubelet
front-proxy-client Generates the client for the front proxy
etcd-healthcheck-client Generates the client certificate for liveness probes to healtcheck etcd
etcd-peer Generates the credentials for etcd nodes to communicate with each other
etcd-server Generates the certificate for serving etcd
  • i.e.

    # get target cluster kubeadm-cfg.yml
    $ kubectl get cm kubeadm-config -n kube-system -o=jsonpath="{.data.ClusterConfiguration}"
    $ sudo kubeadm --config ~/kubeadm.yml alpha phase certs renew all
    
    # or
    $ sudo kubeadm --config ~/kubeadm.yml alpha phase certs renew etcd-server
    $ sudo kubeadm --config ~/kubeadm.yml alpha phase certs renew apiserver-kubelet-client
    $ sudo kubeadm --config ~/kubeadm.yml alpha phase certs renew front-proxy-client
    
    # for /etc/kubernetes/pki/*.crt
    $ echo {apiserver,apiserver-kubelet-client,front-proxy-client} |
           fmt -1 |
           xargs -I{} bash -c "sudo kubeadm --config ~/kubeadm.yml alpha phase certs renew {}"
    
    # for /etc/kubernetes/pki/etcd/*.crt
    $ echo {etcd-server,etcd-peer,etcd-healthcheck-client} |
           fmt -1 |
           xargs -I{} bash -c "sudo kubeadm --config ~/kubeadm.yml alpha phase certs renew {}"
    

generate new certificates

$ sudo kubeadm [--config kubeadm.yml] alpha phase certs [commands]
commands comments
all Generates all PKI assets necessary to establish the control plane
apiserver Generates the certificate for serving the kubernetes API
apiserver-etcd-client Generates the client apiserver uses to access etcd
apiserver-kubelet-client Generates the Client certificate for the API server to connect to kubelet
ca Generates the self-signed kubernetes CA to provision identities for other kuberenets components
etcd-ca Generates the self-signed CA to provision identities for etcd
etcd-healthcheck-client Generates the client certificate for liveness probes to healtcheck etcd
etcd-peer Generates the credentials for etcd nodes to communicate with each other
etcd-server Generates the certificate for serving etcd
front-proxy-ca Generates the self-signed CA to provision identities for front proxy
front-proxy-client Generates the client for the front proxy
sa Generates a private key for signing service account tokens along with its public key
renew Renews certificates for a Kubernetes cluster
  • re-generate /etc/kubernetes/pki/etcd/*.crt for modify X509 Subject Alternative Name:

    $ sudo kubeadm --config ~/kubeadm.yml alpha phase certs etcd-server
    $ sudo kubeadm --config ~/kubeadm.yml alpha phase certs etcd-peer
    $ sudo kubeadm --config ~/kubeadm.yml alpha phase certs etcd-healthcheck-client
    
    # or
    $ echo {etcd-server,etcd-peer,etcd-healthcheck-client} |
           fmt -1 |
           xargs -I{} bash -c "sudo kubeadm --config ~/kubeadm.yml alpha phase certs {}"
    
    • check X509 Subject Alternative Name

      $ openssl x509 -noout -text -in /path/to/NAME.crt
      
    • check expire date

      $ openssl x509 -noout -enddate -in /path/to/NAME.crt
      

renew kubeconfig

# clean all config in /etc/kubenernets/*.conf, i.e.:
# echo {admin,controller-manager,kubelet,scheduler} | fmt -1 | xargs -I{} bash -c "sudo rm -rf {}.conf"
$ sudo kubeadm [--config ~/kubeadm.yml] alpha phase kubeconfig [commands]

renew all kubeconfig

$ sudo kubeadm --config ~/kubeadm.yml alpha phase kubeconfig all
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/admin.conf"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/kubelet.conf"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/controller-manager.conf"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/scheduler.conf"

# or
$ echo {admin,controller-manager,kubelet,scheduler} |
       fmt -1 |
       xargs -I{} bash -c "sudo kubeadm --config ~/kubeadm.yml alpha phase kubeconfig {}"

Available Commands:

commands comments
all Generates all kubeconfig files necessary to establish the control plane and the admin kubeconfig file
admin Generates a kubeconfig file for the admin to use and for kubeadm itself
controller_manager Generates a kubeconfig file for the controller manager to use
kubelet Generates a kubeconfig file for the kubelet to use. Please note that this should be used *only* for bootstrapping purposes
scheduler Generates a kubeconfig file for the scheduler to use
user Outputs a kubeconfig file for an additional user

update ~/.kube/config

$ sudo rm -rf ~/.kube/config
$ sudo cp /etc/kubernetes/admin.conf ~/.kube/config
$ sudo chown devops:devops ~/.kube/config
$ sudo chmod 644 ~/.kube/config

sync to peer controllers

[!NOTE|label:login to peer controller first]

$ ssh devops@<peerController>
# for k8s certs
$ find /etc/kubernetes/pki -type f -regextype posix-extended -regex '^.+/pki/[^/]+\.(key|crt|pub)$' -print
       xargs -L1 -t -i bash -c 'sudo rsync -avzrlpgoDP -e "ssh -q -i $HOME/.ssh/id_ed25519" --rsync-path='sudo rsync' devops@<majorController>:{} {}'
# or
$ find /etc/kubernetes/pki/ -type f -regex '^.*\.\(key\|crt\|pub\)$' -print |
       xargs -L1 -t -i bash -c 'sudo rsync -avzrlpgoDP -e "ssh -q -i $HOME/.ssh/id_ed25519" --rsync-path='sudo rsync' devops@<majorController>:{} {}'
# or
$ for pkg in '*.key' '*.crt' '*.pub'; do
    sudo rsync -avzrlpgoDP \
               -e "ssh -i $HOME/.ssh/id_ed25519" \
               --rsync-path='sudo rsync' \
               devops@<majorController>:"/etc/kubernetes/pki/${pkg}" /etc/kubernetes/pki/
  done

# for stacked etcd
$ find /etc/kubernetes/pki/etcd -type f -regextype posix-extended -regex '^.*(server|healthcheck-client|peer)\.(crt|key)$' |
       xargs -L1 -t -i bash -c 'sudo rsync -avzrlpgoDP -e "ssh -q -i $HOME/.ssh/id_ed25519" --rsync-path='sudo rsync' devops@<majorController>:{} {}'
# or
$ find /etc/kubernetes/pki/etcd -type f -regex '^.*\(server\|healthcheck-client\|peer\)\.\(crt\|key\)$' |
       xargs -L1 -t -i bash -c 'sudo rsync -avzrlpgoDP -e "ssh -q -i $HOME/.ssh/id_ed25519" --rsync-path='sudo rsync' devops@<majorController>:{} {}'
# or
$ for _i in server healthcheck-client peer; do
    sudo rsync -avzrlpgoDP  \
               -e "ssh -i $HOME/.ssh/id_ed25519" \
               --rsync-path='sudo rsync' \
               devops@<majorController>"/etc/kubernetes/pki/etcd/${_i}.{crt,key}" /etc/kubernetes/pki/etcd/
 done

# for kubeconfig
$ find /etc/kubernetes -type f -regextype posix-extended -regex '^/etc/kubernetes/(admin|kubelet|controller-manager|scheduler)\.conf$' -print |
       xargs -L1 -t -i bash -c 'sudo rsync -avzrlpgoDP -e "ssh -q -i $HOME/.ssh/id_ed25519" --rsync-path='sudo rsync' devops@<majorController>:{} {}'
# or
$ find /etc/kubernetes -type f -regex '^\/etc\/kubernetes\/\(admin\|kubelet\|controller-manager\|scheduler\)\.conf$' -print |
       xargs -L1 -t -i bash -c 'sudo rsync -avzrlpgoDP -e "ssh -q -i $HOME/.ssh/id_ed25519" --rsync-path='sudo rsync' devops@<majorController>:{} {}'
# or
$ for _i in admin kubelet controller-manager scheduler; do
    sudo rsync -avzrlpgoDP \
               -e "ssh -i $HOME/.ssh/id_ed25519" \
               --rsync-path='sudo rsync' \
               devops@<majorController>:"/etc/kubernetes/${_i}.conf" /etc/kubernetes/
 done

restart kubelet

kill all services

$ sudo kill -s SIGHUP $(pidof kube-apiserver)
$ sudo kill -s SIGHUP $(pidof kube-controller-manager)
$ sudo kill -s SIGHUP $(pidof kube-scheduler)

restart service

$ sudo rm -rf /var/lib/kubelet/pki/*
$ sudo systemctl status kubelet
$ sudo systemctl restart kubelet
$ sudo systemctl --no-pager -l status kubelet

v1.15.3

reference:

[!TIP] for external etcd topology

$ kubectl version --short
Client Version: v1.15.3
Server Version: v1.15.3

renew certificates

[!NOTE|label:in major controller] NOTE: major controller is the controller node bind with load balance ip.
the key controller node picked by keepalived. check it by using:

$ ip a s "${interface}" | sed -rn 's|\W*inet[^6]\W*([0-9\.]{7,15}).*$|\1|p'


references:

CERTIFICATE FILES PATH
  • apiserver.crt
  • apiserver.key
  • apiserver-kubelet-client.crt
  • apiserver-kubelet-client.key
  • front-proxy-client.crt
  • front-proxy-client.key
/etc/kubernetes/pki
$ echo 'apiserver apiserver-kubelet-client front-proxy-client' |
       xargs -t -n1 sudo kubeadm alpha certs renew

# or
$ for i in apiserver apiserver-kubelet-client front-proxy-client; do
   sudo kubeadm alpha certs renew ${i}
  done
certificate for serving the Kubernetes API renewed
certificate for the API server to connect to kubelet renewed
certificate for the front proxy client renewed
  • or

    $ echo 'apiserver apiserver-kubelet-client front-proxy-client' |
         xargs -t -n1 sudo kubeadm --config kubeadm.yml alpha certs renew
    
    # or
    $ for i in apiserver apiserver-kubelet-client front-proxy-client; do
        sudo kubeadm --config kubeadm-conf.yaml alpha certs renew ${i}
      done
    certificate for serving the Kubernetes API renewed
    certificate for the API server to connect to kubelet renewed
    certificate for the front proxy client renewed
    

sync to peer controllers

[!NOTE] sync renewed certificates to peer controllers

$ leadIP=<majorController>

$ find /etc/kubernetes/pki -type f -regextype posix-extended -regex '^.+/pki/[^/]+\.(key|crt|pub)$' -print
       xargs -L1 -t -i bash -c 'sudo rsync -avzrlpgoDP -e "ssh -q -i $HOME/.ssh/id_ed25519" --rsync-path='sudo rsync' devops@<majorController>:{} {}'
# or
$ find /etc/kubernetes/pki/ -type f -regex '^.*\.\(key\|crt\|pub\)$' -print |
       xargs -L1 -t -i bash -c 'sudo rsync -avzrlpgoDP -e "ssh -q -i $HOME/.ssh/id_ed25519" --rsync-path='sudo rsync' devops@<majorController>:{} {}'
# or
$ for pkg in '*.key' '*.crt' '*.pub'; do
    sudo rsync -avzrlpgoDP \
               --rsync-path='sudo rsync' \
               root@${leadIP}:"/etc/kubernetes/pki/${pkg}" \
               /etc/kubernetes/pki/
  done
  • verify

    verify
    $ find /etc/kubernetes/pki/ -type f -name "*.crt" -print |
               egrep -v 'ca.crt$' |
               xargs -L 1 -t  -i bash -c 'openssl x509 -enddate -noout -in {}'
    bash -c openssl x509 -enddate -noout -in /etc/kubernetes/pki/apiserver.crt
    notAfter=Sep 18 12:10:31 2021 GMT
    bash -c openssl x509 -enddate -noout -in /etc/kubernetes/pki/apiserver-kubelet-client.crt
    notAfter=Sep 18 12:10:31 2021 GMT
    bash -c openssl x509 -enddate -noout -in /etc/kubernetes/pki/front-proxy-client.crt
    notAfter=Sep 18 12:10:31 2021 GMT
    
    • or
      $ find /etc/kubernetes/pki/ -type f -name "*.crt" -print |
             xargs -L 1 -t -i bash -c 'openssl x509 -in {} -noout -text |grep "Not "'
      bash -c openssl x509 -in /etc/kubernetes/pki/ca.crt -noout -text |grep "Not "
                  Not Before: Sep 17 07:51:58 2019 GMT
                  Not After : Sep 14 07:51:58 2029 GMT
      bash -c openssl x509 -in /etc/kubernetes/pki/front-proxy-ca.crt -noout -text |grep "Not "
                  Not Before: Sep 17 07:52:00 2019 GMT
                  Not After : Sep 14 07:52:00 2029 GMT
      bash -c openssl x509 -in /etc/kubernetes/pki/apiserver.crt -noout -text |grep "Not "
                  Not Before: Sep 17 07:51:58 2019 GMT
                  Not After : Sep 18 12:10:31 2021 GMT
      bash -c openssl x509 -in /etc/kubernetes/pki/apiserver-kubelet-client.crt -noout -text |grep "Not "
                  Not Before: Sep 17 07:51:58 2019 GMT
                  Not After : Sep 18 12:10:31 2021 GMT
      bash -c openssl x509 -in /etc/kubernetes/pki/front-proxy-client.crt -noout -text |grep "Not "
                  Not Before: Sep 17 07:52:00 2019 GMT
                  Not After : Sep 18 12:10:31 2021 GMT
      

renew kubeconfig

CONFIG FILES PATH
  • admin.conf
  • kubelet.conf
  • controller-manager.conf
  • scheduler.conf
/etc/kubernetes/
# to get kubeadm-cfg.yml
$ kubectl get cm kubeadm-config -n kube-system -o=jsonpath="{.data.ClusterConfiguration}"
$ sudo kubeadm --config kubeadm-cfg.yml init phase kubeconfig all
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file

setup ~/.kube/config

$ sudo cp /etc/kubernetes/admin.conf ~/.kube/config
$ sudo chown $(id -u):$(id -g) $HOME/.kube/config
$ sudo chmod 644 $HOME/.kube/config
configuration file path
config ~/.kube

restart the controller components

$ sudo kill -s SIGHUP $(pidof kube-apiserver)
$ sudo kill -s SIGHUP $(pidof kube-controller-manager)
$ sudo kill -s SIGHUP $(pidof kube-scheduler)
  • or
    $ echo {kube-apiserver,kube-controller-manager,kube-scheduler} |
           fmt -1 |
           xargs -I{} bash -c "sudo kill -s SIGHUP $(pidof {}) "
    

restart kubelet service

$ sudo rm -rf /var/lib/kubelet/pki/*
$ sudo systemctl restart kubelet
  • verify
    verify
    $ sudo systemctl status kubelet
    ● kubelet.service - kubelet: The Kubernetes Node Agent
       Loaded: loaded (/usr/lib/systemd/system/kubelet.service; enabled; vendor preset: disabled)
      Drop-In: /usr/lib/systemd/system/kubelet.service.d
               └─10-kubeadm.conf
       Active: active (running) since Mon 2020-09-21 04:25:33 PDT; 55s ago
         Docs: https://kubernetes.io/docs/
     Main PID: 11891 (kubelet)
        Tasks: 19
       Memory: 126.5M
       CGroup: /system.slice/kubelet.service
               └─11891 /usr/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/...
    ...
    

verify

via kubernetes api (load.balance.ip.address:6443)

$ echo -n |
       openssl s_client -connect x.x.x.:6443 2>&1 |
       sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' |
       openssl x509 -text -noout |
       grep Not
            Not Before: Sep 17 07:51:58 2019 GMT
            Not After : Sep 21 09:09:00 2021 GMT

extend X509v3 Subject Alternative Name in apiserver.crt

$ ssh controller02

# 生成2048位的密钥对
$ openssl genrsa -out apiserver-controller02.key 2048

# 生成证书签署请求文件
$ sudo openssl req -new \
                   -key apiserver-controller02.key \
                   -subj "/CN=kube-apiserver," \
                   -out apiserver-controller02.csr

# 编辑apiserver-controller02.ext文件,内容如下:
$ cat apiserver-controller02.ext
subjectAltName = DNS:controller02,DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc, DNS:kubernetes.default.svc.cluster.local, IP:10.96.0.1, IP:10.24.138.208

# 使用ca.key和ca.crt签署上述请求
$ sudo openssl x509 -req \
                    -in apiserver-controller02.csr \
                    -CA /etc/kubernetes/pki/ca.crt \
                    -CAkey /etc/kubernetes/pki/ca.key \
                    -CAcreateserial \
                    -out apiserver-controller02.crt \
                    -days 365 \
                    -extfile apiserver-controller02.ext
Signature ok
subject=/CN=10.24.138.208
Getting CA Private Key

# 查看新生成的证书:
$ sudo openssl x509 -noout -text -in apiserver-controller02.crt
Certificate:
    Data:
        Version: 3 (0x2)
        Serial Number: 16019625340257831745 (0xde51245f10ea0b41)
    Signature Algorithm: sha256WithRSAEncryption
        Issuer: CN=kubernetes
        Validity
            Not Before: May 12 08:40:40 2017 GMT
            Not After : May 12 08:40:40 2018 GMT
        Subject: CN=kube-apiserver,
        Subject Public Key Info:
            ... ...
        X509v3 extensions:
            X509v3 Subject Alternative Name:
                DNS:controller02, DNS:kubernetes, DNS:kubernetes.default, DNS:kubernetes.default.svc, DNS:kubernetes.default.svc.cluster.local, IP Address:10.96.0.1, IP Address:10.24.138.208

using new certificate for apiserver

$ sudo cat /etc/kubernetes/manifests/kube-apiserver.yaml
..
- --tls-cert-file=/etc/kubernetes/pki/apiserver-controller02.crt
- --tls-private-key-file=/etc/kubernetes/pki/apiserver-controller02.key
..

renew work node

backup

$ mkdir k8s-cert-expired
$ sudo cp -rp /var/lib/kubelet/pki k8s-cert-expired/
$ sudo cp -r /var/lib/kubelet/pki{,.orig}

restart kubelet

$ sudo rm -rf /var/lib/kubelet/pki/*
$ sudo systemctl restart kubelet
$ sudo systemctl status kubelet.service

certificates generation

pfx

$ grep certificate-authority-data ~/.kube/config |
       awk '{print $2}' |
       base64 -d > ca.crt
$ grep client-certificate-data ~/.kube/config |
       awk '{print $2}' |
       base64 -d > client.crt
$ grep client-key-data ~/.kube/config |
       awk '{print $2}' |
       base64 -d > client.key

$ openssl pkcs12 -export -out cert.pfx -inkey client.key -in client.crt -certfile ca.crt
Enter Export Password: marslo
Verifying - Enter Export Password: marslo

$ ls
ca.crt  cert.pfx  client.crt  client.key

renew kubeconfig only

basic environment

[!TIP]

$ kubectl version --short
Client Version: v1.12.3
Server Version: v1.12.3
  • certs

    $ find /etc/kubernetes/pki/ -type f -name "*.crt" -print |
           egrep -v 'ca.crt$' |
           xargs -L 1 -t  -i bash -c 'openssl x509 -noout -text -in {} | grep After'
    bash -c openssl x509 -noout -text -in /etc/kubernetes/pki/front-proxy-client.crt | grep Not
                Not After : May 28 11:48:39 2022 GMT
    bash -c openssl x509 -noout -text -in /etc/kubernetes/pki/etcd/server.crt | grep Not
                Not After : May 28 11:48:40 2022 GMT
    bash -c openssl x509 -noout -text -in /etc/kubernetes/pki/etcd/peer.crt | grep Not
                Not After : May 28 11:48:41 2022 GMT
    bash -c openssl x509 -noout -text -in /etc/kubernetes/pki/etcd/healthcheck-client.crt | grep Not
                Not After : May 28 11:48:40 2022 GMT
    bash -c openssl x509 -noout -text -in /etc/kubernetes/pki/apiserver.crt | grep Not
                Not After : May 28 11:48:37 2022 GMT
    bash -c openssl x509 -noout -text -in /etc/kubernetes/pki/apiserver-kubelet-client.crt | grep Not
                Not After : May 28 11:48:38 2022 GMT
    bash -c openssl x509 -noout -text -in /etc/kubernetes/pki/apiserver-etcd-client.crt | grep Not
                Not After : May 28 11:48:42 2022 GMT
    
  • kubectl config view

    $ kubectl config get-contexts
    CURRENT   NAME                          CLUSTER      AUTHINFO           NAMESPACE
    *         kubernetes-admin@kubernetes   kubernetes   kubernetes-admin
    
    $ kubectl config current-context
    kubernetes-admin@kubernetes
    
    $ kubectl config view -o jsonpath={.contexts}; echo
    [map[name:kubernetes-admin@kubernetes context:map[user:kubernetes-admin cluster:kubernetes]]]
    
    $ kubectl config view -o jsonpath={.users}; echo
    [map[name:kubernetes-admin user:map[client-certificate-data:REDACTED client-key-data:REDACTED]]]
    
    $ kubectl config view
    apiVersion: v1
    clusters:
    - cluster:
        certificate-authority-data: DATA+OMITTED
        server: https://1.2.3.4:1234
      name: kubernetes
    contexts:
    - context:
        cluster: kubernetes
        user: kubernetes-admin
      name: kubernetes-admin@kubernetes
    current-context: kubernetes-admin@kubernetes
    kind: Config
    preferences: {}
    users:
    - name: kubernetes-admin
      user:
        client-certificate-data: REDACTED
        client-key-data: REDACTED
    
  • kubeconfig

    $ kubectl --kubeconfig=config get namespace
    error: the server doesn't have a resource type "namespace"
    
    $ sudo grep 'client-certificate-data' $HOME/.kube/config |
           awk '{print $2}' |
           base64 -d |
           openssl x509 -text -noout |
           grep -E 'Not|Subject:'
    
                Not Before: Dec  8 05:43:01 2020 GMT
                Not After : Dec  8 05:43:01 2021 GMT
            Subject: O=system:masters, CN=kubernetes-admin
    

renew kubeconfig

references:

Subjects:

configuration files
config subject
controller-manager.conf Subject: CN=system:kube-contdoller-manager
admin.conf Subject: O=system:masters, CN=kubernetes-admin
scheduler.conf Subject: O=system:masters, CN=system:kube-scheduler
kubelet.conf Subject: O=system:nodes, CN=system:node:kubernetes-master01 ( CN=system:node: )

configuration files
certs subject
front-proxy-client.crt Subject: CN=front-proxy-client
server.crt Subject: CN=kubernetes-master01 ( CN=HOSTNAME )
peer.crt Subject: CN=kubernetes-master01 ( CN=HOSTNAME )
healthcheck-client.crt Subject: O=system:masters, CN=kube-etcd-healthcheck-client
apiserver.crt Subject: CN=kube-apiserver
apiserver-kubelet-client.crt Subject: O=system:masters, CN=kube-apiserver-kubelet-client
apiserver-etcd-client.crt Subject: O=system:masters, CN=kube-apiserver-etcd-client

generate new certificate (csr)

$ openssl req -subj "/O=system:masters/CN=kubernetes-admin" \
              -new \
              -newkey rsa:2048 \
              -nodes \
              -out marslo.csr \
              -keyout marslo.key  \
              -out marslo.csr
  • or
    $ openssl genrsa -out marslo.key 2048
    $ openssl req -new -key marslo.key -out marslo.csr -subj "/O=system:masters/CN=kubernetes-admin"
    

signing the certificate via ca.crt

$ sudo openssl x509 -req \
                    -in marslo.csr \
                    -CA /etc/kubernetes/pki/ca.crt \
                    -CAkey /etc/kubernetes/pki/ca.key \
                    -CAcreateserial \
                    -out marslo.crt \
                    -days 365 \
                    -sha256
  • result

    $ ls -Altrh . /etc/kubernetes/pki/ca*
    -rw------- 1 root root 1.7K Dec  6  2018 ca.key
    -rw-r--r-- 1 root root 1.1K Dec  6  2018 ca.crt
    -rw-r--r-- 1 root root   17 Dec 15 01:31 ca.srl
    
    $ ls -Altrh ./
    -rw-rw-r-- 1 devops devops 1.7K Dec 15 01:31 marslo.key
    -rw-rw-r-- 1 devops devops  936 Dec 15 01:31 marslo.csr
    -rw-r--r-- 1 root   root   1021 Dec 15 01:31 marslo.crt
    
    $ sudo openssl x509 -in marslo.crt -text -noout | grep -E 'Subject:|Not'
                Not Before: Dec 15 09:31:55 2021 GMT
                Not After : Dec 15 09:31:55 2022 GMT
            Subject: O=system:masters, CN=kubernetes-admin
    

[!TIP]

$ cp ~/.kube/config config

$ kubectl --kubeconfig=config get ns
error: You must be logged in to the server (Unauthorized)

renew via kubeadm alpha

  • 1.15-

    $ sudo kubeadm [--config ~/kubeadm.yml] alpha phase kubeconfig all
    
    • renew all certs
      $ sudo kubeadm [--config ~/kubeadm.yml] alpha phase certs renew all
      
    • re-generate all certs
      $ sudo kubeadm [--config ~/kubeadm.yml] alpha phase certs all
      
  • v1.15+

    $ sudo kubeadm [--config ~/kubeadm.yml] alpha certs renew all
    
    • renew all certs
      $ sudo kubeadm [--config ~/kubeadm.yml] alpha certs renew all
      

renew via kubectl config

reference:

$ kubectl config set-credentials kubernetes-admin \
                 --embed-certs=true \
                 --certificate-authority=/etc/kubernetes/pki/ca.crt \
                 --client-certificate=./marslo.crt \
                 --client-key=./marslo.key \
                 --kubeconfig=config

$ kubectl config set-context kubernetes-admin@kubernetes \
                 --cluster=kubernetes \
                 --user=kubernetes-admin

renew via base64 manually

$ sed -re "s/(.*client-certificate-data:)(.*)$/\1 $(cat marslo.crt | base64 -w0)/g" -i config
$ sed -re "s/(.*client-key-data:)(.*)$/\1 $(cat marslo.key| base64 -w0)/g" -i config

vaildate

$ kubectl --kubeconfig=config get ns | grep kube
kube-public            Active   3y10d
kube-system            Active   3y10d

more details

reference:

  • conf:

    # current kubeconfig context
    $ kubectl config view --raw -o json |
              jq ".users[] | select(.name==\"$(kubectl config view -o jsonpath='{.users[].name}')\")" |
              jq -r '.user["client-certificate-data"]' |
              base64 -d |
              openssl x509 -text |
              grep "Subject:"
            Subject: O=system:masters, CN=kubernetes-admin
    
    # for all confs
    $ find /etc/kubernetes/ -type f -name "*.conf" -print |
           grep -Ev 'kubelet.conf$' |
           xargs -L1 -t -i bash -c "sudo grep 'client-certificate-data' {} \
                                         | awk '{print \$2}' \
                                         | base64 -d \
                                         | openssl x509 -noout -text \
                                         | grep --color=always Subject\: \
                                   "
    bash -c sudo grep 'client-certificate-data' /etc/kubernetes/controller-manager.conf | awk '{print $2}' | base64 -d | openssl x509 -noout -text | grep --color=always Subject\:
            Subject: CN=system:kube-controller-manager
    bash -c sudo grep 'client-certificate-data' /etc/kubernetes/admin.conf | awk '{print $2}' | base64 -d | openssl x509 -noout -text | grep --color=always Subject\:
            Subject: O=system:masters, CN=kubernetes-admin
    bash -c sudo grep 'client-certificate-data' /etc/kubernetes/scheduler.conf | awk '{print $2}' | base64 -d | openssl x509 -noout -text | grep --color=always Subject\:
            Subject: O=system:masters, CN=system:kube-scheduler
    
    $ sudo openssl x509 -in $(sudo grep 'client-certificate' /etc/kubernetes/kubelet.conf |
           awk '{print $2}') -text -noout | grep --color=always Subject\:
            Subject: O=system:nodes, CN=system:node:kubernetes-master01
    
    • certs

      $ find /etc/kubernetes/pki/ -type f -name "*.crt" -print |
            grep -Ev 'ca.crt$' |
            xargs -L1 -t -i bash -c 'openssl x509 -noout -text -in {} | grep --color=always Subject\:'
      bash -c openssl x509 -noout -text -in /etc/kubernetes/pki/front-proxy-client.crt | grep --color=always Subject\:
              Subject: CN=front-proxy-client
      bash -c openssl x509 -noout -text -in /etc/kubernetes/pki/etcd/server.crt | grep --color=always Subject\:
              Subject: CN=kubernetes-master01
      bash -c openssl x509 -noout -text -in /etc/kubernetes/pki/etcd/peer.crt | grep --color=always Subject\:
              Subject: CN=kubernetes-master01
      bash -c openssl x509 -noout -text -in /etc/kubernetes/pki/etcd/healthcheck-client.crt | grep --color=always Subject\:
      ubject: O=system:masters, CN=kube-apiserver-etcd-client
      bash -c openssl x509 -noout -text -in /etc/kubernetes/pki/apiserver.crt | grep --color=always Subject\:
              Subject: CN=kube-apiserver
      bash -c openssl x509 -noout -text -in /etc/kubernetes/pki/apiserver-kubelet-client.crt | grep --color=always Subject\:
              Subject: O=system:masters, CN=kube-apiserver-kubelet-client
      bash -c openssl x509 -noout -text -in /etc/kubernetes/pki/apiserver-etcd-client.crt | grep --color=always Subject\:
              Subject: O=system:masters, CN=kube-apiserver-etcd-client
      
    • about system:masters

      $ kubectl get clusterrolebinding cluster-admin -o yaml
      $ kubectl get clusterrolebinding cluster-admin -o json | jq -r .subjects[0].name
      system:masters
      
      $ kubectl get clusterrolebindings -o json |
                jq -r '.items[] | select(.subjects[0].kind=="Group") | select(.subjects[0].name=="system:masters") | .metadata.name'
      cluster-admin
      

tricky

modify default certificate to 10 years

[!NOTE|label:references:]

  • CertificateValidity

    $ git clone git@github.com:kubernetes/kubernetes.git && cd Kubernetes
    $ grep CertificateValidity cmd/kubeadm/app/constants/constants.go
      // CertificateValidity defines the validity for all the signed certificates generated by kubeadm
      CertificateValidity = time.Hour * 24 * 365 * 10
    
    $ make cross
    
  • manifests/kube-controller-manager.yaml

    $ sudo cat /etc/kubernetes/manifests/kube-controller-manager.yaml
    controllerManager:
      extraArgs:
        v: "4"
        node-cidr-mask-size: "19"
        deployment-controller-sync-period: "10s"
        # 在 kubeadm 配置文件中设置证书有效期为 10 年
        experimental-cluster-signing-duration: "86700h"
        node-monitor-grace-period: "20s"
        pod-eviction-timeout: "2m"
        terminated-pod-gc-threshold: "30"
    
    # renew
    $ kubeadm alpha certs renew all --use-api
    
    # approve
    $ kubectl -n kube-system get csr
    NAME                                 AGE  REQUESTORE        CONDITION
    kubeadm-cert-kubernetes-admin-648w4  47s  kubernetes-admin  pending
    
    $ kubectl certificate approve kubeadm-cert-kubernetes-admin-648w4
    certificatesigningrequest.certificates.k8s.io/kubeadm-cert-kubernetes-admin-648w4 approved
    $ kubectl -n kube-system get csr
    NAME                                 AGE  REQUESTORE        CONDITION
    kubeadm-cert-kube-apiserver-bgmcs    2s   kubernetes-admin  pending
    kubeadm-cert-kubernetes-admin-648w4  47s  kubernetes-admin  Approved,Issued
    
    $ kubectl certificate approve kubeadm-cert-kube-apiserver-bgmcs
    certificatesigningrequest.certificates.k8s.io/kubeadm-cert-kube-apiserver-bgmcs approved
    $ kubectl -n kube-system get csr
    NAME                                 AGE  REQUESTORE        CONDITION
    kubeadm-cert-kube-apiserver-bgmcs    2s   kubernetes-admin  Approved,Issued
    kubeadm-cert-kubernetes-admin-648w4  47s  kubernetes-admin  Approved,Issued
    
    $ kubectl certificate approve kubeadm-cert-kube-apiserver-kubelet-client-r9lmh
    $ kubectl certificate approve kubeadm-cert-system:kube-contrller-manager-kzx49
    $ kubectl certificate approve kubeadm-cert-font-proxy-client-9kxgj
    $ kubectl certificate approve kubeadm-cert-system:kube-scheduler-8jbb9
    
    $ kubectl -n kube-system get csr
    NAME                                              AGE    REQUESTORE        CONDITION
    kubeadm-cert-font-proxy-client-9kxgj              57s    kubernetes-admin  Approved,Issued
    kubeadm-cert-kube-apiserver-bgmcs                 3m9s   kubernetes-admin  Approved,Issued
    kubeadm-cert-kube-apiserver-kubelet-client-r9lmh  2m57s  kubernetes-admin  Approved,Issued
    kubeadm-cert-kubernetes-admin-648w4               4m19s  kubernetes-admin  Approved,Issued
    kubeadm-cert-system:kube-contrller-manager-kzx49  70s    kubernetes-admin  Approved,Issued
    kubeadm-cert-system:kube-scheduler-8jbb9          49s    kubernetes-admin  Approved,Issued
    
    • older version : v1.15

      $ cat /etc/kubernetes/manifests/kube-controller-manager.yaml
      apiVersion: v1
      kind: Pod
      metadata:
        creationTimestamp: null
        labels:
          component: kube-controller-manager
          tier: control-plane
        name: kube-controller-manager
        namespace: kube-system
      spec:
        containers:
        - command:
          - kube-controller-manager
          ...
          - --experimental-cluster-signing-duration=87600h
          ...
      ...
      
      $ kubeadm alpha certs renew all --config /etc/kubernetes/kubeadm-config.yaml --use-api
      $ kubectl certificate approve ...
      
      # upgrade kubeconfg
      $ kubeadm init phase kubeconfig all --config /etc/kubernetes/kubeadm-config.yaml
      
      $ cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
      $ chown $(id -u):$(id -g) $HOME/.kube/config
      
      # restart components
      $ docker restart $(docker ps | grep etcd  | awk '{ print $1 }')
      $ docker restart $(docker ps | grep kube-apiserver  | awk '{ print $1 }')
      $ docker restart $(docker ps | grep kube-scheduler  | awk '{ print $1 })
      $ docker restart $(docker ps | grep kube-controller  | awk '{ print $1 }')
      $ systemctl daemon-reload && systemctl restart kubelet
      
      # check
      $ echo | openssl s_client -showcerts -connect 127.0.0.1:6443 -servername api 2>/dev/null | openssl x509 -noout -enddate
      

reference

[!TIP] reference:

required certificates

DEFAULT CN PARENT CA O (IN SUBJECT) KIND HOSTS (SAN)
kube-etcd etcd-ca - server, client hostname
Host_IP
localhost
127.0.0.1
kube-etcd-peer etcd-ca - server, client hostname
Host_IP
localhost
127.0.0.1
kube-etcd-healthcheck-client etcd-ca - client -
kube-apiserver-etcd-client etcd-ca system:masters client -
kube-apiserver kubernetes-ca - server hostname, Host_IP, advertise_IP, [1]
kube-apiserver-kubelet-client kubernetes-ca system:masters client -
front-proxy-client kubernetes-front-proxy-ca - client -

Certificate paths

DEFAULT CN RECOMMENDED KEY PATH RECOMMENDED CERT PATH COMMAND KEY ARGUMENT CERT ARGUMENT
etcd-ca etcd/ca.key etcd/ca.crt kube-apiserver - --etcd-cafile
kube-apiserver-etcd-client apiserver-etcd-client.key apiserver-etcd-client.crt kube-apiserver --etcd-keyfile --etcd-certfile
kubernetes-ca ca.key ca.crt kube-apiserver - --client-ca-file
kubernetes-ca ca.key ca.crt kube-controller-manager --cluster-signing-key-file --client-ca-file
--root-ca-file
--cluster-signing-cert-file
kube-apiserver apiserver.key apiserver.crt kube-apiserver --tls-private-key-file --tls-cert-file
kube-apiserver-kubelet-client apiserver-kubelet-client.key apiserver-kubelet-client.crt kube-apiserver --kubelet-client-key --kubelet-client-certificate
front-proxy-ca front-proxy-ca.key front-proxy-ca.crt kube-apiserver - --requestheader-client-ca-file
front-proxy-ca front-proxy-ca.key front-proxy-ca.crt kube-controller-manager - --requestheader-client-ca-file
front-proxy-client front-proxy-client.key front-proxy-client.crt kube-apiserver --proxy-client-key-file --proxy-client-cert-file
etcd-ca etcd/ca.key etcd/ca.crt etcd - --trusted-ca-file
--peer-trusted-ca-file
kube-etcd etcd/server.key etcd/server.crt etcd --key-file --cert-file
kube-etcd-peer etcd/peer.key etcd/peer.crt etcd --peer-key-file --peer-cert-file
etcd-ca - etcd/ca.crt etcdctl - --cacert
kube-etcd-healthcheck-client etcd/healthcheck-client.key etcd/healthcheck-client.crt etcdctl --key --cert

Configure certificates for user accounts

FILENAME CREDENTIAL NAME DEFAULT CN O (IN SUBJECT)
admin.conf default-admin kubernetes-admin system:masters
kubelet.conf default-auth system:node:<nodeName> (see note) system:nodes
controller-manager.conf default-controller-manager system:kube-controller-manager -
scheduler.conf default-scheduler system:kube-scheduler -

files are used as follows

FILENAME COMMAND COMMENT
admin.conf kubectl Configures administrator user for the cluster
kubelet.conf kubelet One required for each node in the cluster.
controller-manager.conf kube-controller-manager Must be added to manifest in manifests/kube-controller-manager.yaml
scheduler.conf kube-scheduler Must be added to manifest in manifests/kube-scheduler.yaml
Copyright © marslo 2020-2023 all right reserved,powered by GitbookLast Modified: 2024-03-12 15:01:30

results matching ""

    No results matching ""