[기타] CentOS 7 에 Kubernetes 기본 Install

# root 로 초기 로그인. sudo 를 사용하기 위해 일반유저 생성 및 패스워드 지정
useradd -d /home/username username
echo "password" | passwd username --stdin 

# 일반유저는 su 사용하지 못하도록 권한 조정
chmod 700 /usr/bin/su

# sudoer 의 기본 wheel 그룹에 계정 추가
sed -ie '/wheel/s/$/\:username/' /etc/group

# Timezone 설정
sudo timedatectl set-timezone Asia/Seoul

# SWAP OFF
sudo swapoff -a
sudo sed -i -e '/swap/d' /etc/fstab

# firewalld off
sudo systemctl stop firewalld && sudo systemctl disable firewalld

# Selinux 
setenforce 0 
sudo sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config

# Hostname 
sudo hostnamectl set-hostname controller 
sudo hostnamectl set-hostname worker-01
sudo hostnamectl set-hostname worker-02

## Controller / Worker 
#curl -s https://get.docker.com | sudo sh
curl -fsSL https://get.docker.com -o get-docker.sh
sudo sh get-docker.sh

## Check 
sudo docker -v 
sudo docker ps -a  

## Controller / Worker 
sudo mkdir /etc/docker
cat <<EOF | sudo tee /etc/docker/daemon.json
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2"
}
EOF

## Docker enable && restart 
sudo systemctl enable docker
sudo systemctl daemon-reload
sudo systemctl restart docker  

## Packages Repo
sudo cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
EOF

## Install
sudo yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes

# Controller. 일반적으로 컨트롤러 자신의 공인IP 를 통해 API 서버임을 알린다 (Advertise) 
sudo kubeadm init --ignore-preflight-errors=all --pod-network-cidr=192.168.0.0/16 --apiserver-advertise-address=10.10.10.237

# Regular User Privileges 
  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

# Network Plugin Setting ( Calico )
curl https://projectcalico.docs.tigera.io/manifests/calico.yaml -O
kubectl apply -f calico.yaml

# System Namespace ( kube-system ) check. CoreDNS 가 올라오면 정상
kubectl get pods -o wide -A

NAMESPACE     NAME                                      READY   STATUS    RESTARTS   AGE     IP             NODE         NOMINATED NODE   READINESS GATES
kube-system   calico-kube-controllers-7c845d499-p85pm   1/1     Running   0          3m6s    192.168.49.3   controller   <none>           <none>
kube-system   calico-node-fnm2q                         1/1     Running   0          3m6s    10.10.10.237   controller   <none>           <none>
kube-system   coredns-64897985d-cgvml                   1/1     Running   0          5m41s   192.168.49.2   controller   <none>           <none>
kube-system   coredns-64897985d-vdckf                   1/1     Running   0          5m42s   192.168.49.1   controller   <none>           <none>
kube-system   etcd-controller                           1/1     Running   0          5m54s   10.10.10.237   controller   <none>           <none>
kube-system   kube-apiserver-controller                 1/1     Running   0          5m54s   10.10.10.237   controller   <none>           <none>
kube-system   kube-controller-manager-controller        1/1     Running   0          6m      10.10.10.237   controller   <none>           <none>
kube-system   kube-proxy-nn5zn                          1/1     Running   0          5m42s   10.10.10.237   controller   <none>           <none>
kube-system   kube-scheduler-controller                 1/1     Running   0          5m54s   10.10.10.237   controller   <none>           <none>


# (공통) Multi NIC 를 가지는 환경에서 INTERNAL-IP 설정
다수의 네트워크가 존재하는 서버의 경우 K8S 에서 첫 번째 NIC 의 IP 가 INTERNAL-IP 로 자동 설정된다.  
해당 INTERNAL-IP 를 Init 시 설정한 
kubeadm --apiserver-advertise-address 와 동일한 IP 대역을 사용하도록 수동 설정한다.  

cat << EOF | sudo tee /etc/default/kubelet
KUBELET_EXTRA_ARGS='--node-ip $(hostname -I | cut -d ' ' -f2)'
EOF
sudo systemctl daemon-reload
sudo systemctl restart kubelet
kubectl cluster-info

# Worker-01과 Woker-02 Node 에도 User Privileges 설정 복사 및 적용한다
sudo scp /etc/kubernetes//admin.conf username@10.10.10.204:/home/username/admin.conf
sudo scp /etc/kubernetes//admin.conf username@10.10.10.190:/home/username/admin.conf

# Worker 
mkdir -p $HOME/.kube
sudo cp -i ./admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

# 각 Worker 에서 kubeadm 를 사용해서 Join 한다. 
sudo kubeadm join 10.10.10.237:6443 --token jgocer.fu65ql39kdod5qi0 \
        --discovery-token-ca-cert-hash sha256:3cb85267e89913d7865d219922daaa8fc6e788dd2be0e2f80fae27176e2dfe3b

# 토큰값을 나중에 재 확인해야 할 경우
kubeadm token create --print-join-command

# Check
kubectl get nodes -o wide
NAME         STATUS     ROLES                  AGE   VERSION   INTERNAL-IP    EXTERNAL-IP   OS-IMAGE                KERNEL-VERSION           CONTAINER-RUNTIME
controller   Ready      control-plane,master   16m   v1.23.5   10.10.10.237   <none>        CentOS Linux 7 (Core)   3.10.0-1062.el7.x86_64   docker://20.10.14
worker-01    Ready      <none>                 55s   v1.23.5   10.10.10.204   <none>        CentOS Linux 7 (Core)   3.10.0-1062.el7.x86_64   docker://20.10.14
worker-02    NotReady   <none>                 38s   v1.23.5   10.10.10.190   <none>        CentOS Linux 7 (Core)   3.10.0-1062.el7.x86_64   docker://20.10.14

# Check Pod Create
kubectl run hello --image=nginx --dry-run=client -o yaml | kubectl apply -f-
pod/hello created
[myungin.baek@controller ~]$ kubectl get pods -o wide
NAME    READY   STATUS    RESTARTS   AGE   IP              NODE        NOMINATED NODE   READINESS GATES
hello   1/1     Running   0          42s   192.168.171.1   worker-01   <none>           <none>