|
0. 环境介绍
三台主机，主机名和IP地址如下
VIP: 172.18.0.200
主机名：k8s-master01 IP: 172.18.0.142 做主 做证书服务器
主机名：k8s-master02 IP: 172.18.0.143 做从
主机名：k8s-master03 IP: 172.18.0.146 做从 网络规划：
服务器网段：172.18.0.0/16
容器网段：172.30.0.0/16
service网段：169.169.0.0/16 1. 关闭selinux和firewalld（三台都需要操作）
[root@k8s-master01 ~]# setenforce 0 > /dev/null 2>&1 && sed -i -e 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
[root@k8s-master01 ~]# systemctl stop firewalld
[root@k8s-master01 ~]# systemctl disable firewalld 2. 安装ntp（三台都需要操作）
[root@k8s-master01 ~]# yum -y install ntp
[root@k8s-master01 ~]# systemctl enable ntpd
[root@k8s-master01 ~]# systemctl start ntpd 3. docker安装（三台都需要操作）
[root@k8s-master01 ~]# yum -y install docker
[root@k8s-master01 ~]# systemctl start docker
[root@k8s-master01 ~]# systemctl enable docker 4. 生成证书（k8s-master01上操作即可，然后把证书复制到其他两台的/etc/kubernetes/ssl下）
4.1 生成CA证书和私钥
[root@k8s-master01 ~]# mkdir /root/ssl
[root@k8s-master01 ~]# cd /root/ssl/
[root@k8s-master01 ssl]# openssl genrsa -out ca.key 2048
[root@k8s-master01 ssl]# openssl req -x509 -new -nodes -key ca.key -subj "/CN=" -days 3650 -out ca.crt
4.2 生成apiserver用的证书和私钥
[root@k8s-master01 ssl]# openssl genrsa -out kube-apiserver.key 2048
[root@k8s-master01 ssl]# vim kube-apiserver.cnf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[v3_req]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[alt_names]
DNS.1 = kubernetes
DNS.2 = kubernetes.default
DNS.3 = kubernetes.default.svc
DNS.4 = kubernetes.default.svc.cluster.local
DNS.5 = k8s-master
DNS.6 = k8s-master01
DNS.7 = k8s-master02
DNS.8 = k8s-master03
IP.1 = 169.169.0.1
IP.2 = 172.18.0.142
IP.3 = 172.18.0.143
IP.4 = 172.18.0.146
IP.5 = 172.18.0.200
[root@k8s-master01 ssl]# openssl req -new -key kube-apiserver.key -subj "/CN=k8s-master" -config kube-apiserver.cnf -out kube-apiserver.csr
[root@k8s-master01 ssl]# openssl x509 -req -in kube-apiserver.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 3650 -extensions v3_req -extfile kube-apiserver.cnf -out kube-apiserver.crt
4.3 生成kube-controller-manager的客户端证书和私钥
[root@k8s-master01 ssl]# openssl genrsa -out kube-controller-manager.key 2048
[root@k8s-master01 ssl]# openssl req -new -key kube-controller-manager.key -subj "/CN=k8s-controller-manager" -out kube-controller-manager.csr
[root@k8s-master01 ssl]# openssl x509 -req -in kube-controller-manager.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out kube-controller-manager.crt -days 3650
4.4 生成kube-scheduler的客户端证书和私钥
这里可以参照4.3生成，也可以和controller-manager使用同一个证书。由于我们会选择controller-manager和scheduler部署在同一台机器，所以没必要再生成一个正式，完全可以使用同一个，这里我们选择使用同一个
4.5 生成kubelet的客户端证书和私钥
[root@k8s-master01 ssl]# openssl genrsa -out kubelet.key 2048
[root@k8s-master01 ssl]# openssl req -new -key kubelet.key -subj "/CN=k8s-kubelet" -out kubelet.csr
[root@k8s-master01 ssl]# openssl x509 -req -in kubelet.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out kubelet.crt -days 3650
4.6 生成kube-proxy的客户端证书和私钥
这里可以参照4.5生成，也可以和kubelet使用同一个证书。由于我们会选择kube-proxy和kubelet部署在同一台机器，所以没必要再生成一个正式，完全可以使用同一个，这里我们选择使用同一个
4.7 拷贝所有证书到/etc/kuberntes/ssl/下
[root@k8s-master01 bin]# mkdir /etc/kubernetes/ssl -pv
[root@k8s-master01 ssl]# cp ./*.crt ./*.key /etc/kubernetes/ssl/
[root@k8s-master01 ssl]# ls /etc/kubernetes/ssl/
ca.crt ca.key kube-apiserver.crt kube-apiserver.key kube-controller-manager.crt kube-controller-manager.key kubelet.crt kubelet.key 5. 部署kube-apiserver服务（三台都需要操作）
5.1 将kube-apiserver的二进制文件拷贝到/usr/bin/下
[root@k8s-master01 bin]# ls /usr/bin/kube-apiserver
/usr/bin/kube-apiserver
[root@k8s-master01 bin]# chmod +x /usr/bin/kube-apiserver
5.2 生成kube-apiserver的service unit文件
[root@k8s-master01 bin]# vim /usr/lib/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target
After=etcd.service [Service]
Type=notify
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/apiserver
ExecStart=/usr/bin/kube-apiserver $KUBE_API_ARGS
Restart=on-failure
LimitNOFILE=65535 [Install]
WantedBy=multi-user.target
5.3 编辑配置文件
[root@k8s-master01 bin]# vim /etc/kubernetes/apiserver
KUBE_API_ARGS="--etcd-servers=http://172.18.0.142:2379,172.18.0.143:2379,172.18.0.146:2379 --bind-address=0.0.0.0 --secure-port=6443 --service-cluster-ip-range=169.169.0.0/16 --service-node-port-range=1-65535 --admission-control=Namesp
aceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota --logtostderr=false --log-dir=/opt/logs/kubernetes --v=2 --client-ca-file=/etc/kubernetes/ssl/ca.crt --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver.key --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.crt --tls-ca-file=/etc/kubernetes/ssl/ca.crt --allow-privileged=true"
5.4 创建日志目录
[root@k8s-master01 ssl]# mkdir /opt/logs/kubernetes -pv
5.4 启动服务
[root@k8s-master01 ssl]# systemctl daemon-reload
[root@k8s-master01 ssl]# systemctl enable kube-apiserver
[root@k8s-master01 ssl]# systemctl start kube-apiserver 6. 部署keeperalive做高可用
6.1 安装keepalived
[root@k8s-master01 ssl]# yum -y install keepalived
[root@k8s-master01 ssl]# cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak 6.2 配置主节点(k8s-master01)的keepalived
[root@k8s-master01 ssl]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived global_defs {
notification_email {
hdb@
}
notification_email_from admin@
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id KUBE_APISERVER_HA
} vrrp_script chk_kube_apiserver {
script "curl -k https://127.0.0.1:6443"
interval 3
timeout 9
fall 2
rise 2
} vrrp_instance VI_1 {
state BACKUP
interface eno16777728
virtual_router_id 111
priority 100
advert_int 1
nopreempt
authentication {
auth_type PASS
auth_pass heyjava
}
virtual_ipaddress {
172.18.0.200
}
track_script {
chk_kube_apiserver
}
notify_master "/etc/keepalived/notify.py -n master -a 172.18.0.200"
notify_backup "/etc/keepalived/notify.py -n backup -a 172.18.0.200"
notify_fault "/etc/keepalived/notify.py -n fault -a 172.18.0.200"
}
6.3 配置从节点(k8s-master02)的keepalived
! Configuration File for keepalived global_defs {
notification_email {
hdb@
}
notification_email_from admin@
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id KUBE_APISERVER_HA
} vrrp_script chk_kube_apiserver {
script "curl -k https://127.0.0.1:6443"
interval 3
timeout 9
fall 2
rise 2
} vrrp_instance VI_1 {
state BACKUP
interface eno16777728
virtual_router_id 111
priority 99
advert_int 1
nopreempt
authentication {
auth_type PASS
auth_pass heyjava
}
virtual_ipaddress {
172.18.0.200
}
track_script {
chk_kube_apiserver
}
notify_master "/etc/keepalived/notify.py -n master -a 172.18.0.200"
notify_backup "/etc/keepalived/notify.py -n backup -a 172.18.0.200"
notify_fault "/etc/keepalived/notify.py -n fault -a 172.18.0.200"
}
6.4 配置从节点(k8s-master03)的keepalived
! Configuration File for keepalived global_defs {
notification_email {
hdb@
}
notification_email_from admin@
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id KUBE_APISERVER_HA
} vrrp_script chk_kube_apiserver {
script "curl -k https://127.0.0.1:6443"
interval 3
timeout 9
fall 2
rise 2
} vrrp_instance VI_1 {
state BACKUP
interface eno16777728
virtual_router_id 111
priority 98
advert_int 1
nopreempt
authentication {
auth_type PASS
auth_pass heyjava
}
virtual_ipaddress {
172.18.0.200
}
track_script {
chk_kube_apiserver
}
notify_master "/etc/keepalived/notify.py -n master -a 172.18.0.200"
notify_backup "/etc/keepalived/notify.py -n backup -a 172.18.0.200"
notify_fault "/etc/keepalived/notify.py -n fault -a 172.18.0.200"
}
6.5 编写通知脚本（三台机器都需要）
[root@k8s-master01 ssl]# vi /etc/keepalived/notify.py
#/usr/bin/python
#-*- coding:utf-8 -*- '''
@file: notify.py
@author: Hu Dongbiao
@date: 2016/12/15 11:24
@version: 1.0
@email: hdb@
''' import argparse
import sys
import smtplib
from email.mime.text import MIMEText #解析传进来的参数
parser = argparse.ArgumentParser(description=u"vrrp状态切换通知脚本")
parser.add_argument("-n", "--notify", choices=["master", "backup", "fault"], help=u"指定通知的类型，即vrrp角色切换的目标角色")
parser.add_argument("-a", "--address", help=u"指定相关虚拟路由器的VIP地址")
args = parser.parse_args()
# notify是当前角色，为master，backup，fault中的一个
notify = args.notify
# address是vrrp虚拟地址
address = args.address # 发送告警邮件
smtp_host = 'smtp.exmail.qq.com'
smtp_user = 'admin@'
smtp_password = 'Tzg2014'
mail_from = 'admin@'
mail_to = 'hdb@,hxf@,clb@'
mail_subject = u'[监控]VRRP角色切换'
mail_body = '''
<p>管理员，你好:</p>
<p style="text-indent:2em;"><strong>您的HA地址{vrrp_address}已切换角色为{vrrp_role}，请及时处理</strong></p>
'''.format(vrrp_address=address, vrrp_role=notify)
msg = MIMEText(mail_body, 'html', 'utf-8')
msg['From'] = mail_from
msg['To'] = mail_to
msg['Subject'] = mail_subject
smtp = smtplib.SMTP()
smtp.connect(smtp_host)
smtp.login(smtp_user,smtp_password)
smtp.sendmail(mail_from, mail_to, msg.as_string())
smtp.quit()
[root@k8s-master01 ssl]# chmod +x /etc/keepalived/notify.py
6.6 启动keepalived服务（三台都需要）
[root@k8s-master01 ssl]# systemctl enable keepalived
[root@k8s-master01 ssl]# systemctl start keepalived
[root@k8s-master01 ssl]# systemctl status keepalived
6.7 验证keepalived是否工作正常
1）验证主节点是否接管VIP 172.18.0.200
2）停止主节点的kube-apiserver服务，验证从节点1是否接管VIP
3）再停止从节点1，验证从节点2是否接管VIP
注：这里通知邮件没发出来，以后再来排查 故障1：
Sep 8 11:26:08 k8s-master01 systemd: Starting Kubernetes API Server...
Sep 8 11:26:09 k8s-master01 kube-apiserver: Unable to find suitable network address.error='Unable to select an IP.'. Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.
Sep 8 11:26:09 k8s-master01 systemd: kube-apiserver.service: main process exited, code=exited, status=1/FAILURE
Sep 8 11:26:09 k8s-master01 systemd: Failed to start Kubernetes API Server.
Sep 8 11:26:09 k8s-master01 systemd: Unit kube-apiserver.service entered failed state.
Sep 8 11:26:09 k8s-master01 systemd: kube-apiserver.service failed.
Sep 8 11:26:10 k8s-master01 systemd: kube-apiserver.service holdoff time over, scheduling restart.
解决：设置默认网关
|
