@zhangyy
2020-05-09T02:01:17.000000Z
字数 12039
阅读 513
kubernetes系列
- 一: 环境部署规划
- 二:系统环境初始化
- 三:配置ETCD集群
cat /etc/hosts----192.168.100.11 node01.flyfish192.168.100.12 node02.flyfish192.168.100.13 node03.flyfish192.168.100.14 node04.flyfish192.168.100.15 node05.flyfish----1.etcd集群规划---192.168.100.11192.168.100.12192.168.100.13---2.Kubernetes master节点集群规划---192.168.100.11192.168.100.12192.168.100.13---3.Kubernetes node节点集群规划---192.168.100.14192.168.100.15---4. Kubernetes pod 网络规划---10.80.0.0/12---5.Kubernetes service 网络规划---10.66.0.0/16---6.Kubernetes 集群域名---cluster.local---7.Kubernetes 集群dns---10.66.0.2---8.Kubernetes 高可用方案---nginx+confd---
# centos (所有主机)sed -i 's/SELINUX=.*/SELINUX=disabled/g' /etc/selinux/configsystemctl stop firewalld && systemctl disable firewalldsetenforce 0

#go 环境部署yum install gogo get github.com/cloudflare/cfssl/cmd/cfsslgo get github.com/cloudflare/cfssl/cmd/cfssljsonmv /go/ /usr/local/vim /etc/profile---GOBIN=/root/go/bin/PATH=$PATH:$GOBIN:$HOME/binexport PATH---




mkdir etcdcd etcdvim etcd.sh---# 设置证书环境变量# 设置证书使用时间87600h 10年export EXPIRY_TIME="87600h"# 签发证书IPexport ETCD_MEMBER_1_IP="192.168.100.11"export ETCD_MEMBER_2_IP="192.168.100.12"export ETCD_MEMBER_3_IP="192.168.100.13"# 机器名export ETCD_MEMBER_1_HOSTNAMES="node01.flyfish"export ETCD_MEMBER_2_HOSTNAMES="node02.flyfish"export ETCD_MEMBER_3_HOSTNAMES="node03.flyfish"# etcd 集群通讯证书export ETCD_SERVER_HOSTNAMES="\"${ETCD_MEMBER_1_HOSTNAMES}\",\"${ETCD_MEMBER_2_HOSTNAMES}\",\"${ETCD_MEMBER_3_HOSTNAMES}\""export ETCD_SERVER_IPS="\"${ETCD_MEMBER_1_IP}\",\"${ETCD_MEMBER_2_IP}\",\"${ETCD_MEMBER_3_IP}\""#证书所需要的配置参数export CERT_ST="ShangHai"export CERT_L="ShangHai"export CERT_O="k8s"export CERT_OU="Qist"export CERT_PROFILE="kubernetes"# 设置工作目录export HOST_PATH=`pwd`# 创建etcd K8S 证书json 存放目录mkdir -p ${HOST_PATH}/cfssl/{k8s,etcd}# 创建签发证书存放目录mkdir -p ${HOST_PATH}/cfssl/pki/{k8s,etcd}# CA 配置文件用于配置根证书的使用场景 (profile) 和具体参数 (usage,过期时间、服务端认证、客户端认证、加密等),后续在签名其它证书时需要指定特定场景。cat << EOF | tee ${HOST_PATH}/cfssl/ca-config.json{"signing": {"default": {"expiry": "${EXPIRY_TIME}"},"profiles": {"${CERT_PROFILE}": {"usages": ["signing","key encipherment","server auth","client auth"],"expiry": "${EXPIRY_TIME}"}}}}EOF# 创建 ETCD CA 配置文件cat << EOF | tee ${HOST_PATH}/cfssl/etcd/etcd-ca-csr.json{"CN": "etcd","key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","ST": "$CERT_ST","L": "$CERT_L","O": "$CERT_O","OU": "$CERT_OU"}],"ca": {"expiry": "${EXPIRY_TIME}"}}EOF# etcd ca 证书签发cfssl gencert -initca ${HOST_PATH}/cfssl/etcd/etcd-ca-csr.json | \cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-ca# 创建 ETCD Server 配置文件cat << EOF | tee ${HOST_PATH}/cfssl/etcd/etcd-server.json{"CN": "etcd","hosts": ["127.0.0.1",${ETCD_SERVER_IPS},${ETCD_SERVER_HOSTNAMES}],"key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","ST": "$CERT_ST","L": "$CERT_L","O": "$CERT_O","OU": "$CERT_OU"}]}EOF# 生成 ETCD Server 证书和私钥cfssl gencert \-ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem \-ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem \-config=${HOST_PATH}/cfssl/ca-config.json \-profile=${CERT_PROFILE} \${HOST_PATH}/cfssl/etcd/etcd-server.json | \cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-server# 创建 ETCD Member 1 配置文件cat << EOF | tee ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_1_HOSTNAMES}.json{"CN": "etcd","hosts": ["127.0.0.1","${ETCD_MEMBER_1_IP}","${ETCD_MEMBER_1_HOSTNAMES}"],"key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","ST": "$CERT_ST","L": "$CERT_L","O": "$CERT_O","OU": "$CERT_OU"}]}EOF# 生成 ETCD Member 1 证书和私钥cfssl gencert \-ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem \-ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem \-config=${HOST_PATH}/cfssl/ca-config.json \-profile=${CERT_PROFILE} \${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_1_HOSTNAMES}.json | \cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-member-${ETCD_MEMBER_1_HOSTNAMES}# 创建 ETCD Member 2 配置文件cat << EOF | tee ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_2_HOSTNAMES}.json{"CN": "etcd","hosts": ["127.0.0.1","${ETCD_MEMBER_2_IP}","${ETCD_MEMBER_2_HOSTNAMES}"],"key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","ST": "$CERT_ST","L": "$CERT_L","O": "$CERT_O","OU": "$CERT_OU"}]}EOF# 生成 ETCD Member 2 证书和私钥cfssl gencert \-ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem \-ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem \-config=${HOST_PATH}/cfssl/ca-config.json \-profile=${CERT_PROFILE} \${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_2_HOSTNAMES}.json | \cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-member-${ETCD_MEMBER_2_HOSTNAMES}# 创建 ETCD Member 3 配置文件cat << EOF | tee ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_3_HOSTNAMES}.json{"CN": "etcd","hosts": ["127.0.0.1","${ETCD_MEMBER_3_IP}","${ETCD_MEMBER_3_HOSTNAMES}"],"key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","ST": "$CERT_ST","L": "$CERT_L","O": "$CERT_O","OU": "$CERT_OU"}]}EOF# 生成 ETCD Member 3 证书和私钥cfssl gencert \-ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem \-ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem \-config=${HOST_PATH}/cfssl/ca-config.json \-profile=${CERT_PROFILE} \${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_3_HOSTNAMES}.json | \cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-member-${ETCD_MEMBER_3_HOSTNAMES}# 创建 ETCD Client 配置文件cat << EOF | tee ${HOST_PATH}/cfssl/etcd/etcd-client.json{"CN": "client","hosts": [""],"key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","ST": "$CERT_ST","L": "$CERT_L","O": "$CERT_O","OU": "$CERT_OU"}]}EOF# 生成 ETCD Client 证书和私钥cfssl gencert \-ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem \-ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem \-config=${HOST_PATH}/cfssl/ca-config.json \-profile=${CERT_PROFILE} \${HOST_PATH}/cfssl/etcd/etcd-client.json | \cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-client---chmod +x etcd.sh./etcd.sh



# 分发生成的证书到所有需要部署etcd 节点ssh 192.168.100.11 mkdir -p /opt/etcd/sslssh 192.168.100.12 mkdir -p /opt/etcd/sslssh 192.168.100.13 mkdir -p /opt/etcd/ssl# 分发文件scp -r ./cfssl/pki/etcd/* 192.168.100.11:/opt/etcd/ssl/scp -r ./cfssl/pki/etcd/* 192.168.100.12:/opt/etcd/ssl/scp -r ./cfssl/pki/etcd/* 192.168.100.13:/opt/etcd/ssl/


mv etcd.sh /rootvim etcd-soft.sh----wget https://github.com/etcd-io/etcd/releases/download/v3.4.7/etcd-v3.4.7-linux-amd64.tar.gz# 解压下载好文件tar -zxvf etcd-v3.4.7-linux-amd64.tar.gz# 创建二进制远程存放目录ssh 192.168.100.11 mkdir -p /opt/etcd/binssh 192.168.100.12 mkdir -p /opt/etcd/binssh 192.168.100.13 mkdir -p /opt/etcd/bin# 分发解压好二进制文件cd etcd-v3.4.7-linux-amd64/scp -r etcd* 192.168.100.11:/opt/etcd/binscp -r etcd* 192.168.100.12:/opt/etcd/binscp -r etcd* 192.168.100.13:/opt/etcd/bin---chmod +x etcd-soft.sh./etcd-soft.sh


# 创建配置文件存放目录ssh 192.168.100.11 mkdir -p /opt/etcd/confssh 192.168.100.12 mkdir -p /opt/etcd/confssh 192.168.100.13 mkdir -p /opt/etcd/conf# 192.168.100.11 配置ssh 192.168.100.11cat << EOF | tee /opt/etcd/conf/etcdETCD_OPTS="--name=node01.flyfish \\--data-dir=/opt/etcd/data/default.etcd \\--wal-dir=/opt/etcd/data/default.etcd/wal \\--listen-peer-urls=https://192.168.100.11:2380 \\--listen-client-urls=https://192.168.100.11:2379,https://127.0.0.1:2379 \\--advertise-client-urls=https://192.168.100.11:2379 \\--initial-advertise-peer-urls=https://192.168.100.11:2380 \\--initial-cluster=node01.flyfish=https://192.168.100.11:2380,node02.flyfish=https://192.168.100.12:2380,node03.flyfish=https://192.168.100.13:2380 \\--initial-cluster-token=node01.flyfish=https://192.168.100.11:2380,node02.flyfish=https://192.168.100.12:2380,node03.flyfish=https://192.168.100.13:2380 \\--initial-cluster-state=new \\--heartbeat-interval=6000 \\--election-timeout=30000 \\--snapshot-count=5000 \\--auto-compaction-retention=1 \\--max-request-bytes=33554432 \\--quota-backend-bytes=17179869184 \\--trusted-ca-file=/opt/etcd/ssl/etcd-ca.pem \\--cert-file=/opt/etcd/ssl/etcd-server.pem \\--key-file=/opt/etcd/ssl/etcd-server-key.pem \\--peer-cert-file=/opt/etcd/ssl/etcd-member-node01.flyfish.pem \\--peer-key-file=/opt/etcd/ssl/etcd-member-node01.flyfish-key.pem \\--peer-client-cert-auth \\--enable-v2=true \\--peer-trusted-ca-file=/opt/etcd/ssl/etcd-ca.pem"EOF# 192.168.100.12 配置ssh 192.168.100.12cat << EOF | tee /opt/etcd/conf/etcdETCD_OPTS="--name=node02.flyfish \\--data-dir=/opt/etcd/data/default.etcd \\--wal-dir=/opt/etcd/data/default.etcd/wal \\--listen-peer-urls=https://192.168.100.12:2380 \\--listen-client-urls=https://192.168.100.12:2379,https://127.0.0.1:2379 \\--advertise-client-urls=https://192.168.100.12:2379 \\--initial-advertise-peer-urls=https://192.168.100.12:2380 \\--initial-cluster=node01.flyfish=https://192.168.100.11:2380,node02.flyfish=https://192.168.100.12:2380,node03.flyfish=https://192.168.100.13:2380 \\--initial-cluster-token=node01=https://192.168.100.11:2380,node02.flyfish=https://192.168.100.12:2380,node03.flyfish=https://192.168.100.13:2380 \\--initial-cluster-state=new \\--heartbeat-interval=6000 \\--election-timeout=30000 \\--snapshot-count=5000 \\--auto-compaction-retention=1 \\--max-request-bytes=33554432 \\--quota-backend-bytes=17179869184 \\--trusted-ca-file=/opt/etcd/ssl/etcd-ca.pem \\--cert-file=/opt/etcd/ssl/etcd-server.pem \\--key-file=/opt/etcd/ssl/etcd-server-key.pem \\--peer-cert-file=/opt/etcd/ssl/etcd-member-node02.flyfish.pem \\--peer-key-file=/opt/etcd/ssl/etcd-member-node02.flyfish-key.pem \\--peer-client-cert-auth \\--enable-v2=true \\--peer-trusted-ca-file=/opt/etcd/ssl/etcd-ca.pem"EOF# 192.168.100.13 配置ssh 192.168.100.13cat << EOF | tee /opt/etcd/conf/etcdETCD_OPTS="--name=node03.flyfish \\--data-dir=/opt/etcd/data/default.etcd \\--wal-dir=/opt/etcd/data/default.etcd/wal \\--listen-peer-urls=https://192.168.100.13:2380 \\--listen-client-urls=https://192.168.100.13:2379,https://127.0.0.1:2379 \\--advertise-client-urls=https://192.168.100.13:2379 \\--initial-advertise-peer-urls=https://192.168.100.13:2380 \\--initial-cluster=node01.flyfish=https://192.168.100.11:2380,node02.flyfish=https://192.168.100.12:2380,node03.flyfish=https://192.168.100.13:2380 \\--initial-cluster-token=node01.flyfish=https://192.168.100.11:2380,node02.flyfish=https://192.168.100.12:2380,node03.flyfish=https://192.168.100.13:2380 \\--initial-cluster-state=new \\--heartbeat-interval=6000 \\--election-timeout=30000 \\--snapshot-count=5000 \\--auto-compaction-retention=1 \\--max-request-bytes=33554432 \\--quota-backend-bytes=17179869184 \\--trusted-ca-file=/opt/etcd/ssl/etcd-ca.pem \\--cert-file=/opt/etcd/ssl/etcd-server.pem \\--key-file=/opt/etcd/ssl/etcd-server-key.pem \\--peer-cert-file=/opt/etcd/ssl/etcd-member-node03.flyfish.pem \\--peer-key-file=/opt/etcd/ssl/etcd-member-node03.flyfish-key.pem \\--peer-client-cert-auth \\--enable-v2=true \\--peer-trusted-ca-file=/opt/etcd/ssl/etcd-ca.pem"EOF



cat << EOF | tee etcd.service[Unit]Description=Etcd ServerAfter=network.targetAfter=network-online.targetWants=network-online.targetDocumentation=https://github.com/etcd-io/etcd[Service]Type=notifyLimitNOFILE=65535LimitNPROC=65535LimitCORE=infinityLimitMEMLOCK=infinityUser=etcdGroup=etcdWorkingDirectory=/opt/etcd/data/default.etcdEnvironmentFile=-/opt/etcd/conf/etcdExecStart=/opt/etcd/bin/etcd \$ETCD_OPTSRestart=on-failure[Install]WantedBy=multi-user.targetEOF
# 上传启动文件到服务器scp etcd.service 192.168.100.11:/usr/lib/systemd/systemscp etcd.service 192.168.100.12:/usr/lib/systemd/systemscp etcd.service 192.168.100.12:/usr/lib/systemd/system
# 创建etcd 用户ssh 192.168.100.11 useradd etcd -s /sbin/nologin -Mssh 192.168.100.12 useradd etcd -s /sbin/nologin -Mssh 192.168.100.13 useradd etcd -s /sbin/nologin -M# 创建etcd 存储文件目录ssh 192.168.100.11 mkdir -p /opt/etcd/data/default.etcd/walssh 192.168.100.12 mkdir -p /opt/etcd/data/default.etcd/walssh 192.168.100.13 mkdir -p /opt/etcd/data/default.etcd/wal# 给/apps/etcd etcd 用户权限ssh 192.168.100.11 chown -R etcd:etcd /opt/etcd/ssh 192.168.100.12 chown -R etcd:etcd /opt/etcd/ssh 192.168.100.13 chown -R etcd:etcd /opt/etcd/

# 刷新servicesystemctl daemon-reload# 设置开机启动systemctl enable etcd.service# 启动etcdsystemctl start etcd.service# 查看启动状态systemctl status etcd.service# 验证etcd 集群是否正常 任意节点vi ~/.bashrcexport ETCDCTL_API=3export ENDPOINTS=https://192.168.100.11:2379,https://192.168.100.12:2379,https://192.168.100.13:2379alias ctl='/opt/etcd/bin/etcdctl --endpoints=${ENDPOINTS} --cacert=/opt/etcd/ssl/etcd-ca.pem --cert=/opt/etcd/ssl/etcd-client.pem --key=/opt/etcd/ssl/etcd-client-key.pem'# 保存source ~/.bashrc# 验证集群是否正常ctl endpoint statusctl endpoint hashkvctl endpoint healthctl member list

如果有request sent was ignored (cluster ID mismatch: peer[2a7d2f116f343f6]=9f3896b564270b78, local=85959a7a5b22596f)这样的报错 启动不了 则要 删除 缓存rm -rf /opt/etcd/data/default.etcd/wal/*然后在重新启动service etcd restart
