@zhangyy 2021-12-01T02:13:40.000000Z 字数 9045 阅读 704

k8s 1.22.4 二进制部署

kubernetes系列

k8s 1.22.4 二进制部署

一：系统环境初始化

1.1 系统环境

cat /etc/hosts
----
172.16.10.11 flyfishsrvs01
172.16.10.12 flyfishsrvs02
172.16.10.13 flyfishsrvs03
172.16.10.14 flyfishsrvs04
172.16.10.15 flyfishsrvs05
172.16.10.16 flyfishsrvs06
172.16.10.17 flyfishsrvs07
-----
先安装单master版本后续扩容成多master 
系统关闭firewalld/selinux /清空iptables防火墙规则

1.2 升级系统内核

所有机器都要升级内核
#查看当前内核版本
uname -r
uname -a
cat /etc/redhat-release 
#添加yum源仓库
mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup 
curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo 
curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
yum install -y https://www.elrepo.org/elrepo-release-7.0-4.el7.elrepo.noarch.rpm
#更新yum源仓库
yum -y update
#查看可用的系统内核包
yum --disablerepo="*" --enablerepo="elrepo-kernel" list available
#安装内核，注意先要查看可用内核，我安装的是5.4版本的内核
yum --enablerepo=elrepo-kernel install kernel-lt
#查看目前可用内核
awk -F\' '$1=="menuentry " {print i++ " : " $2}' /etc/grub2.cfg
#使用序号为0的内核，序号0是前面查出来的可用内核编号
grub2-set-default 0
#生成 grub 配置文件并重启
grub2-mkconfig -o /boot/grub2/grub.cfg
reboot

image_1fldfh8jn86b1851ml18quhe13e.png-64.5kB

1.2 环境配置

#修改时区，同步时间
yum install chrond -y
vim /etc/chrony.conf
-----
ntpdate ntp1.aliyun.com iburst
-----
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
echo 'Asia/Shanghai' > /etc/timezone
#关闭防火墙，selinux
systemctl stop firewalld
systemctl disable firewalld
sed -i 's/enforcing/disabled/' /etc/selinux/config 
setenforce 0
## 关闭swap
swapoff -a  
sed -ri 's/.*swap.*/#&/' /etc/fstab
#系统优化
cat > /etc/sysctl.d/k8s_better.conf << EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
vm.swappiness=0
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF
sysctl -p /etc/sysctl.d/k8s_better.conf
#确保每台机器的uuid不一致，如果是克隆机器，修改网卡配置文件删除uuid那一行
cat /sys/class/dmi/id/product_uuid

image_1fldfod3v1np1fp8k4j1ok9abm6q.png-124.4kB

1.3 安装docker

全部节点安装：
安装docker
这里介绍yum源安装
1. 卸载旧版本
yum remove docker \
docker-client \
docker-client-latest \
docker-common \
docker-latest \
docker-latest-logrotate \
docker-logrotate \
docker-engine \
docker-ce
rm -rf /var/lib/docker
2.安装必备软件包
yum install -y yum-utils device-mapper-persistent-data lvm2
3.设置yum源
#建议使用阿里源
yum-config-manager \
    --add-repo \
    http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
4.安装docker
这里介绍两种安装方式，我是用的是第一种
1）使用yum命令安装
#查看可用版本
yum list docker-ce --showduplicates | sort -r
#安装18.09.1的版本，安装其他版本照套格式就行
yum install docker-ce-18.09.1 docker-ce-cli-18.09.1 containerd.io
#安装最新版本
yum install -y docker-ce docker-ce-cli containerd.io    
mkdir -p /etc/docker 
vim /etc/docker/daemon.json
-----
{
  "exec-opts": [
    "native.cgroupdriver=systemd"
  ],
  "log-driver": "json-file",
  "log-level": "warn",
  "log-opts": {
    "max-size": "1000m",
    "max-file": "3"
  },
  "registry-mirrors": [
    "https://b9pmyelo.mirror.aliyuncs.com"
  ],
  "insecure-registries": [],
  "selinux-enabled": false
}

image_1flpqmffkm0m1sn198ta7q96p1p.png-93.7kB

image_1flpqrbl45141g471rvp1bj2gkf26.png-150.5kB

二：部署etcd集群

2.1 关于签名证书

wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
chmod +x cfssl_linux-amd64 cfssljson_linux-amd64 cfssl-certinfo_linux-amd64
mv cfssl_linux-amd64 /usr/bin/cfssl
mv cfssljson_linux-amd64 /usr/bin/cfssljson
mv cfssl-certinfo_linux-amd64 /usr/bin/cfssl-certinfo

image_1fldg056r178g1nua1lje1eq3u87k.png-112.3kB

mkdir -p ~/TLS/{etcd,k8s}
cd ~/TLS/etcd
自签CA：
cat > ca-config.json << EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "www": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF
cat > ca-csr.json << EOF
{
    "CN": "etcd CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing"
        }
    ]
}
EOF
生成证书：
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
会生成ca.pem和ca-key.pem文件

image_1flpr2h9p1jtfuus1o0ogmg6bk2j.png-182kB

2. 使用自签CA签发Etcd HTTPS证书
创建证书申请文件：
cat > server-csr.json << EOF
{
    "CN": "etcd",
    "hosts": [
    "172.16.10.11",
    "172.16.10.12",
    "172.16.10.13",
    "172.16.10.14",
    "172.16.10.15",
    "172.16.10.16",
    "172.16.10.17",
    "172.16.10.18",
    "172.16.10.19",
    "172.16.10.200"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing"
        }
    ]
}
EOF
注：上述文件hosts字段中IP为所有etcd节点的集群内部通信IP，一个都不能少！为了方便后期扩容可以多写几个预留的IP。
生成证书：
  cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
会生成server.pem和server-key.pem文件。

image_1flpr7vq2te6hs5vvq1cs6c9n3g.png-172.5kB

2.2 部署etcd

1. Etcd 的概念：
Etcd 是一个分布式键值存储系统，Kubernetes使用Etcd进行数据存储，所以先准备一个Etcd数据库，为解决Etcd单点故障，应采用集群方式部署，这里使用3台组建集群，可容忍1台机器故障，当然，你也可以使用5台组建集群，可容忍2台机器故障。
下载地址： https://github.com/etcd-io/etcd/releases
以下在节点1上操作，为简化操作，待会将节点1生成的所有文件拷贝到节点2和节点3.

2. 安装配置etcd
mkdir /opt/etcd/{bin,cfg,ssl} -p
tar zxvf etcd-v3.5.1-linux-amd64.tar.gz
mv etcd-v3.5.1-linux-amd64/{etcd,etcdctl} /opt/etcd/bin/

image_1fldipv98102t1lp176f1v6aj2ten.png-220.6kB

flyfishsrvs01 etcd 配置文件
cat > /opt/etcd/cfg/etcd.conf << EOF
#[Member]
ETCD_NAME="etcd-1"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://172.16.10.11:2380"
ETCD_LISTEN_CLIENT_URLS="https://172.16.10.11:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://172.16.10.11:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://172.16.10.11:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://172.16.10.11:2380,etcd-2=https://172.16.10.12:2380,etcd-3=https://172.16.10.13:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF
---
ETCD_NAME：节点名称，集群中唯一
ETCD_DATA_DIR：数据目录
ETCD_LISTEN_PEER_URLS：集群通信监听地址
ETCD_LISTEN_CLIENT_URLS：客户端访问监听地址
ETCD_INITIAL_ADVERTISE_PEER_URLS：集群通告地址
ETCD_ADVERTISE_CLIENT_URLS：客户端通告地址
ETCD_INITIAL_CLUSTER：集群节点地址
ETCD_INITIAL_CLUSTER_TOKEN：集群Token
ETCD_INITIAL_CLUSTER_STATE：加入集群的当前状态，new是新集群，existing表示加入已有集群

3. systemd管理etcd
cat > /usr/lib/systemd/system/etcd.service << EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=/opt/etcd/cfg/etcd.conf
ExecStart=/opt/etcd/bin/etcd \
--cert-file=/opt/etcd/ssl/server.pem \
--key-file=/opt/etcd/ssl/server-key.pem \
--peer-cert-file=/opt/etcd/ssl/server.pem \
--peer-key-file=/opt/etcd/ssl/server-key.pem \
--trusted-ca-file=/opt/etcd/ssl/ca.pem \
--peer-trusted-ca-file=/opt/etcd/ssl/ca.pem \
--logger=zap
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF

image_1fldjihdi1ijec1de87u233jq9.png-167.6kB

4. 拷贝刚才生成的证书
把刚才生成的证书拷贝到配置文件中的路径：
cp ~/TLS/etcd/ca*pem ~/TLS/etcd/server*pem /opt/etcd/ssl/

5. 同步所有主机
scp -r /opt/etcd/ root@172.16.10.12:/opt/
scp -r /opt/etcd/ root@172.16.10.13:/opt/
scp /usr/lib/systemd/system/etcd.service root@172.16.10.12:/usr/lib/systemd/system/
scp /usr/lib/systemd/system/etcd.service root@172.16.10.13:/usr/lib/systemd/system/

image_1flprfab11gs8e1m87v1to8197g4a.png-89.2kB

image_1flprfnld1qll1h071b011am91jgt4n.png-95.2kB

flyfishsrvs02 etcd 
vim /opt/etcd/cfg/etcd.conf 
-----
#[Member]
ETCD_NAME="etcd-2"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://172.16.10.12:2380"
ETCD_LISTEN_CLIENT_URLS="https://172.16.10.12:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://172.16.10.12:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://172.16.10.12:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://172.16.10.11:2380,etcd-2=https://172.16.10.12:2380,etcd-3=https://172.16.10.13:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
----

flyfishsrvs03 etcd 
vim /opt/etcd/cfg/etcd.conf 
----
#[Member]
ETCD_NAME="etcd-3"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://172.16.10.13:2380"
ETCD_LISTEN_CLIENT_URLS="https://172.16.10.13:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://172.16.10.13:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://172.16.10.13:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://172.16.10.11:2380,etcd-2=https://172.16.10.12:2380,etcd-3=https://172.16.10.13:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
-----

6. 启动etcd:
systemctl daemon-reload
systemctl start etcd
systemctl enable etcd

image_1flprk868fu619c76sd14h014ak5k.png-87.3kB

验证：
     ETCDCTL_API=3 /opt/etcd/bin/etcdctl --cacert=/opt/etcd/ssl/ca.pem --cert=/opt/etcd/ssl/server.pem --key=/opt/etcd/ssl/server-key.pem --endpoints="https://172.16.10.11:2379,https://172.16.10.12:2379,https://172.16.10.13:2379" endpoint health --write-out=table

image_1flprmmgadrl5db85svlusl561.png-120.7kB

2.3 部署k8s1.22.4

1. 从Github下载二进制文件
下载地址： https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.22.md
注：打开链接你会发现里面有很多包，下载一个server包就够了，包含了Master和Worker Node二进制文件。

image_1flprsia81p5k1mpc1ei02hqt1a6e.png-325kB

image_1flpruda4a6218p4gos1an9vo18h.png-356.3kB

2. 创建k8s 的kube-apiserver证书
cd ~/TLS/k8s
cat > ca-config.json << EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF
cat > ca-csr.json << EOF
{
    "CN": "kubernetes",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF
生成证书：
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
会生成ca.pem和ca-key.pem文件。

image_1flps2ea0dq9cp41o5db6l1fql9e.png-181.1kB

2. 使用自签CA签发kube-apiserver HTTPS证书
创建证书申请文件：
cat > server-csr.json << EOF
{
    "CN": "kubernetes",
    "hosts": [
      "10.0.0.1",
      "127.0.0.1",
      "172.16.10.11",
      "172.16.10.12",
      "172.16.10.13",
      "172.16.10.14",
      "172.16.10.15",
      "172.16.10.16",
      "172.16.10.17",
      "172.16.10.18",
      "172.16.10.19",
      "172.16.10.200",
      "kubernetes",
      "kubernetes.default",
      "kubernetes.default.svc",
      "kubernetes.default.svc.cluster",
      "kubernetes.default.svc.cluster.local"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF
注：上述文件hosts字段中IP为所有Master/LB/VIP IP，一个都不能少！为了方便后期扩容可以多写几个预留的IP。
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server