@zhangyy
2020-05-09T10:01:17.000000Z
字数 12039
阅读 147
kubernetes系列
- 一: 环境部署规划
- 二:系统环境初始化
- 三:配置ETCD集群
cat /etc/hosts
----
192.168.100.11 node01.flyfish
192.168.100.12 node02.flyfish
192.168.100.13 node03.flyfish
192.168.100.14 node04.flyfish
192.168.100.15 node05.flyfish
----
1.etcd集群规划
---
192.168.100.11
192.168.100.12
192.168.100.13
---
2.Kubernetes master节点集群规划
---
192.168.100.11
192.168.100.12
192.168.100.13
---
3.Kubernetes node节点集群规划
---
192.168.100.14
192.168.100.15
---
4. Kubernetes pod 网络规划
---
10.80.0.0/12
---
5.Kubernetes service 网络规划
---
10.66.0.0/16
---
6.Kubernetes 集群域名
---
cluster.local
---
7.Kubernetes 集群dns
---
10.66.0.2
---
8.Kubernetes 高可用方案
---
nginx+confd
---
# centos (所有主机)
sed -i 's/SELINUX=.*/SELINUX=disabled/g' /etc/selinux/config
systemctl stop firewalld && systemctl disable firewalld
setenforce 0
#go 环境部署
yum install go
go get github.com/cloudflare/cfssl/cmd/cfssl
go get github.com/cloudflare/cfssl/cmd/cfssljson
mv /go/ /usr/local/
vim /etc/profile
---
GOBIN=/root/go/bin/
PATH=$PATH:$GOBIN:$HOME/bin
export PATH
---
mkdir etcd
cd etcd
vim etcd.sh
---
# 设置证书环境变量
# 设置证书使用时间87600h 10年
export EXPIRY_TIME="87600h"
# 签发证书IP
export ETCD_MEMBER_1_IP="192.168.100.11"
export ETCD_MEMBER_2_IP="192.168.100.12"
export ETCD_MEMBER_3_IP="192.168.100.13"
# 机器名
export ETCD_MEMBER_1_HOSTNAMES="node01.flyfish"
export ETCD_MEMBER_2_HOSTNAMES="node02.flyfish"
export ETCD_MEMBER_3_HOSTNAMES="node03.flyfish"
# etcd 集群通讯证书
export ETCD_SERVER_HOSTNAMES="\"${ETCD_MEMBER_1_HOSTNAMES}\",\"${ETCD_MEMBER_2_HOSTNAMES}\",\"${ETCD_MEMBER_3_HOSTNAMES}\""
export ETCD_SERVER_IPS="\"${ETCD_MEMBER_1_IP}\",\"${ETCD_MEMBER_2_IP}\",\"${ETCD_MEMBER_3_IP}\""
#证书所需要的配置参数
export CERT_ST="ShangHai"
export CERT_L="ShangHai"
export CERT_O="k8s"
export CERT_OU="Qist"
export CERT_PROFILE="kubernetes"
# 设置工作目录
export HOST_PATH=`pwd`
# 创建etcd K8S 证书json 存放目录
mkdir -p ${HOST_PATH}/cfssl/{k8s,etcd}
# 创建签发证书存放目录
mkdir -p ${HOST_PATH}/cfssl/pki/{k8s,etcd}
# CA 配置文件用于配置根证书的使用场景 (profile) 和具体参数 (usage,过期时间、服务端认证、客户端认证、加密等),后续在签名其它证书时需要指定特定场景。
cat << EOF | tee ${HOST_PATH}/cfssl/ca-config.json
{
"signing": {
"default": {
"expiry": "${EXPIRY_TIME}"
},
"profiles": {
"${CERT_PROFILE}": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "${EXPIRY_TIME}"
}
}
}
}
EOF
# 创建 ETCD CA 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/etcd/etcd-ca-csr.json
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "$CERT_ST",
"L": "$CERT_L",
"O": "$CERT_O",
"OU": "$CERT_OU"
}
],
"ca": {
"expiry": "${EXPIRY_TIME}"
}
}
EOF
# etcd ca 证书签发
cfssl gencert -initca ${HOST_PATH}/cfssl/etcd/etcd-ca-csr.json | \
cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-ca
# 创建 ETCD Server 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/etcd/etcd-server.json
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
${ETCD_SERVER_IPS},
${ETCD_SERVER_HOSTNAMES}
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "$CERT_ST",
"L": "$CERT_L",
"O": "$CERT_O",
"OU": "$CERT_OU"
}
]
}
EOF
# 生成 ETCD Server 证书和私钥
cfssl gencert \
-ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem \
-ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem \
-config=${HOST_PATH}/cfssl/ca-config.json \
-profile=${CERT_PROFILE} \
${HOST_PATH}/cfssl/etcd/etcd-server.json | \
cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-server
# 创建 ETCD Member 1 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_1_HOSTNAMES}.json
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"${ETCD_MEMBER_1_IP}",
"${ETCD_MEMBER_1_HOSTNAMES}"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "$CERT_ST",
"L": "$CERT_L",
"O": "$CERT_O",
"OU": "$CERT_OU"
}
]
}
EOF
# 生成 ETCD Member 1 证书和私钥
cfssl gencert \
-ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem \
-ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem \
-config=${HOST_PATH}/cfssl/ca-config.json \
-profile=${CERT_PROFILE} \
${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_1_HOSTNAMES}.json | \
cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-member-${ETCD_MEMBER_1_HOSTNAMES}
# 创建 ETCD Member 2 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_2_HOSTNAMES}.json
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"${ETCD_MEMBER_2_IP}",
"${ETCD_MEMBER_2_HOSTNAMES}"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "$CERT_ST",
"L": "$CERT_L",
"O": "$CERT_O",
"OU": "$CERT_OU"
}
]
}
EOF
# 生成 ETCD Member 2 证书和私钥
cfssl gencert \
-ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem \
-ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem \
-config=${HOST_PATH}/cfssl/ca-config.json \
-profile=${CERT_PROFILE} \
${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_2_HOSTNAMES}.json | \
cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-member-${ETCD_MEMBER_2_HOSTNAMES}
# 创建 ETCD Member 3 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_3_HOSTNAMES}.json
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"${ETCD_MEMBER_3_IP}",
"${ETCD_MEMBER_3_HOSTNAMES}"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "$CERT_ST",
"L": "$CERT_L",
"O": "$CERT_O",
"OU": "$CERT_OU"
}
]
}
EOF
# 生成 ETCD Member 3 证书和私钥
cfssl gencert \
-ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem \
-ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem \
-config=${HOST_PATH}/cfssl/ca-config.json \
-profile=${CERT_PROFILE} \
${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_3_HOSTNAMES}.json | \
cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-member-${ETCD_MEMBER_3_HOSTNAMES}
# 创建 ETCD Client 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/etcd/etcd-client.json
{
"CN": "client",
"hosts": [""],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "$CERT_ST",
"L": "$CERT_L",
"O": "$CERT_O",
"OU": "$CERT_OU"
}
]
}
EOF
# 生成 ETCD Client 证书和私钥
cfssl gencert \
-ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem \
-ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem \
-config=${HOST_PATH}/cfssl/ca-config.json \
-profile=${CERT_PROFILE} \
${HOST_PATH}/cfssl/etcd/etcd-client.json | \
cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-client
---
chmod +x etcd.sh
./etcd.sh
# 分发生成的证书到所有需要部署etcd 节点
ssh 192.168.100.11 mkdir -p /opt/etcd/ssl
ssh 192.168.100.12 mkdir -p /opt/etcd/ssl
ssh 192.168.100.13 mkdir -p /opt/etcd/ssl
# 分发文件
scp -r ./cfssl/pki/etcd/* 192.168.100.11:/opt/etcd/ssl/
scp -r ./cfssl/pki/etcd/* 192.168.100.12:/opt/etcd/ssl/
scp -r ./cfssl/pki/etcd/* 192.168.100.13:/opt/etcd/ssl/
mv etcd.sh /root
vim etcd-soft.sh
----
wget https://github.com/etcd-io/etcd/releases/download/v3.4.7/etcd-v3.4.7-linux-amd64.tar.gz
# 解压下载好文件
tar -zxvf etcd-v3.4.7-linux-amd64.tar.gz
# 创建二进制远程存放目录
ssh 192.168.100.11 mkdir -p /opt/etcd/bin
ssh 192.168.100.12 mkdir -p /opt/etcd/bin
ssh 192.168.100.13 mkdir -p /opt/etcd/bin
# 分发解压好二进制文件
cd etcd-v3.4.7-linux-amd64/
scp -r etcd* 192.168.100.11:/opt/etcd/bin
scp -r etcd* 192.168.100.12:/opt/etcd/bin
scp -r etcd* 192.168.100.13:/opt/etcd/bin
---
chmod +x etcd-soft.sh
./etcd-soft.sh
# 创建配置文件存放目录
ssh 192.168.100.11 mkdir -p /opt/etcd/conf
ssh 192.168.100.12 mkdir -p /opt/etcd/conf
ssh 192.168.100.13 mkdir -p /opt/etcd/conf
# 192.168.100.11 配置
ssh 192.168.100.11
cat << EOF | tee /opt/etcd/conf/etcd
ETCD_OPTS="--name=node01.flyfish \\
--data-dir=/opt/etcd/data/default.etcd \\
--wal-dir=/opt/etcd/data/default.etcd/wal \\
--listen-peer-urls=https://192.168.100.11:2380 \\
--listen-client-urls=https://192.168.100.11:2379,https://127.0.0.1:2379 \\
--advertise-client-urls=https://192.168.100.11:2379 \\
--initial-advertise-peer-urls=https://192.168.100.11:2380 \\
--initial-cluster=node01.flyfish=https://192.168.100.11:2380,node02.flyfish=https://192.168.100.12:2380,node03.flyfish=https://192.168.100.13:2380 \\
--initial-cluster-token=node01.flyfish=https://192.168.100.11:2380,node02.flyfish=https://192.168.100.12:2380,node03.flyfish=https://192.168.100.13:2380 \\
--initial-cluster-state=new \\
--heartbeat-interval=6000 \\
--election-timeout=30000 \\
--snapshot-count=5000 \\
--auto-compaction-retention=1 \\
--max-request-bytes=33554432 \\
--quota-backend-bytes=17179869184 \\
--trusted-ca-file=/opt/etcd/ssl/etcd-ca.pem \\
--cert-file=/opt/etcd/ssl/etcd-server.pem \\
--key-file=/opt/etcd/ssl/etcd-server-key.pem \\
--peer-cert-file=/opt/etcd/ssl/etcd-member-node01.flyfish.pem \\
--peer-key-file=/opt/etcd/ssl/etcd-member-node01.flyfish-key.pem \\
--peer-client-cert-auth \\
--enable-v2=true \\
--peer-trusted-ca-file=/opt/etcd/ssl/etcd-ca.pem"
EOF
# 192.168.100.12 配置
ssh 192.168.100.12
cat << EOF | tee /opt/etcd/conf/etcd
ETCD_OPTS="--name=node02.flyfish \\
--data-dir=/opt/etcd/data/default.etcd \\
--wal-dir=/opt/etcd/data/default.etcd/wal \\
--listen-peer-urls=https://192.168.100.12:2380 \\
--listen-client-urls=https://192.168.100.12:2379,https://127.0.0.1:2379 \\
--advertise-client-urls=https://192.168.100.12:2379 \\
--initial-advertise-peer-urls=https://192.168.100.12:2380 \\
--initial-cluster=node01.flyfish=https://192.168.100.11:2380,node02.flyfish=https://192.168.100.12:2380,node03.flyfish=https://192.168.100.13:2380 \\
--initial-cluster-token=node01=https://192.168.100.11:2380,node02.flyfish=https://192.168.100.12:2380,node03.flyfish=https://192.168.100.13:2380 \\
--initial-cluster-state=new \\
--heartbeat-interval=6000 \\
--election-timeout=30000 \\
--snapshot-count=5000 \\
--auto-compaction-retention=1 \\
--max-request-bytes=33554432 \\
--quota-backend-bytes=17179869184 \\
--trusted-ca-file=/opt/etcd/ssl/etcd-ca.pem \\
--cert-file=/opt/etcd/ssl/etcd-server.pem \\
--key-file=/opt/etcd/ssl/etcd-server-key.pem \\
--peer-cert-file=/opt/etcd/ssl/etcd-member-node02.flyfish.pem \\
--peer-key-file=/opt/etcd/ssl/etcd-member-node02.flyfish-key.pem \\
--peer-client-cert-auth \\
--enable-v2=true \\
--peer-trusted-ca-file=/opt/etcd/ssl/etcd-ca.pem"
EOF
# 192.168.100.13 配置
ssh 192.168.100.13
cat << EOF | tee /opt/etcd/conf/etcd
ETCD_OPTS="--name=node03.flyfish \\
--data-dir=/opt/etcd/data/default.etcd \\
--wal-dir=/opt/etcd/data/default.etcd/wal \\
--listen-peer-urls=https://192.168.100.13:2380 \\
--listen-client-urls=https://192.168.100.13:2379,https://127.0.0.1:2379 \\
--advertise-client-urls=https://192.168.100.13:2379 \\
--initial-advertise-peer-urls=https://192.168.100.13:2380 \\
--initial-cluster=node01.flyfish=https://192.168.100.11:2380,node02.flyfish=https://192.168.100.12:2380,node03.flyfish=https://192.168.100.13:2380 \\
--initial-cluster-token=node01.flyfish=https://192.168.100.11:2380,node02.flyfish=https://192.168.100.12:2380,node03.flyfish=https://192.168.100.13:2380 \\
--initial-cluster-state=new \\
--heartbeat-interval=6000 \\
--election-timeout=30000 \\
--snapshot-count=5000 \\
--auto-compaction-retention=1 \\
--max-request-bytes=33554432 \\
--quota-backend-bytes=17179869184 \\
--trusted-ca-file=/opt/etcd/ssl/etcd-ca.pem \\
--cert-file=/opt/etcd/ssl/etcd-server.pem \\
--key-file=/opt/etcd/ssl/etcd-server-key.pem \\
--peer-cert-file=/opt/etcd/ssl/etcd-member-node03.flyfish.pem \\
--peer-key-file=/opt/etcd/ssl/etcd-member-node03.flyfish-key.pem \\
--peer-client-cert-auth \\
--enable-v2=true \\
--peer-trusted-ca-file=/opt/etcd/ssl/etcd-ca.pem"
EOF
cat << EOF | tee etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/etcd-io/etcd
[Service]
Type=notify
LimitNOFILE=65535
LimitNPROC=65535
LimitCORE=infinity
LimitMEMLOCK=infinity
User=etcd
Group=etcd
WorkingDirectory=/opt/etcd/data/default.etcd
EnvironmentFile=-/opt/etcd/conf/etcd
ExecStart=/opt/etcd/bin/etcd \$ETCD_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
# 上传启动文件到服务器
scp etcd.service 192.168.100.11:/usr/lib/systemd/system
scp etcd.service 192.168.100.12:/usr/lib/systemd/system
scp etcd.service 192.168.100.12:/usr/lib/systemd/system
# 创建etcd 用户
ssh 192.168.100.11 useradd etcd -s /sbin/nologin -M
ssh 192.168.100.12 useradd etcd -s /sbin/nologin -M
ssh 192.168.100.13 useradd etcd -s /sbin/nologin -M
# 创建etcd 存储文件目录
ssh 192.168.100.11 mkdir -p /opt/etcd/data/default.etcd/wal
ssh 192.168.100.12 mkdir -p /opt/etcd/data/default.etcd/wal
ssh 192.168.100.13 mkdir -p /opt/etcd/data/default.etcd/wal
# 给/apps/etcd etcd 用户权限
ssh 192.168.100.11 chown -R etcd:etcd /opt/etcd/
ssh 192.168.100.12 chown -R etcd:etcd /opt/etcd/
ssh 192.168.100.13 chown -R etcd:etcd /opt/etcd/
# 刷新service
systemctl daemon-reload
# 设置开机启动
systemctl enable etcd.service
# 启动etcd
systemctl start etcd.service
# 查看启动状态
systemctl status etcd.service
# 验证etcd 集群是否正常 任意节点
vi ~/.bashrc
export ETCDCTL_API=3
export ENDPOINTS=https://192.168.100.11:2379,https://192.168.100.12:2379,https://192.168.100.13:2379
alias ctl='/opt/etcd/bin/etcdctl --endpoints=${ENDPOINTS} --cacert=/opt/etcd/ssl/etcd-ca.pem --cert=/opt/etcd/ssl/etcd-client.pem --key=/opt/etcd/ssl/etcd-client-key.pem'
# 保存
source ~/.bashrc
# 验证集群是否正常
ctl endpoint status
ctl endpoint hashkv
ctl endpoint health
ctl member list
如果有request sent was ignored (cluster ID mismatch: peer[2a7d2f116f343f6]=9f3896b564270b78, local=85959a7a5b22596f)
这样的报错 启动不了 则要 删除 缓存
rm -rf /opt/etcd/data/default.etcd/wal/*
然后在重新启动
service etcd restart