kubernetes v1.14.0高可用master集群部署（使用kubeadm，离线安装）

echo """
vm.swappiness = 0
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
""" > /etc/sysctl.conf
sysctl -p

centos7添加bridge-nf-call-ip6tables出现No such file or directory,简单来说就是执行一下 modprobe br_netfilter

ntpdate -u ntp.api.bz

参考文章

grub2-set-default 0 && grub2-mkconfig -o /etc/grub2.cfg
grubby --default-kernel
grubby --args="user_namespace.enable=1" --update-kernel="$(grubby --default-kernel)"

uname -a
cat > /etc/sysconfig/modules/ipvs.modules &1
    if [ $? -eq 0 ]; then
        /sbin/modprobe \${kernel_module}
    fi
done
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep ip_vs

执行sysctl -p报错可执行modprobe br_netfilter，请参考centos7添加bridge-nf-call-ip6tables出现No such file or directory

# https://github.com/moby/moby/issues/31208 
# ipvsadm -l --timout
# 修复ipvs模式下长连接timeout问题 小于900即可
cat  ./cluster-info
bash -c "$(curl -fsSL https://raw.githubusercontent.com/hnbcao/kubeadm-ha-master/v1.14.0/keepalived-haproxy.sh)"

4、部署HA Master

HA Master的部署过程已经自动化，请在master-1上执行如下命令，并注意修改IP;

脚本主要执行三步：

1)、重置kubelet设置

kubeadm reset -f
rm -rf /etc/kubernetes/pki/

2)、编写节点配置文件并初始化master1的kubelet

echo """
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
kubernetesVersion: v1.13.0
controlPlaneEndpoint: "${VIP}:8443"
maxPods: 100
networkPlugin: cni
imageRepository: registry.aliyuncs.com/google_containers
apiServer:
  certSANs:
  - ${CP0_IP}
  - ${CP1_IP}
  - ${CP2_IP}
  - ${VIP}
networking:
  # This CIDR is a Calico default. Substitute or remove for your CNI provider.
  podSubnet: ${CIDR}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
""" > /etc/kubernetes/kubeadm-config.yaml
kubeadm init --config /etc/kubernetes/kubeadm-config.yaml
mkdir -p $HOME/.kube
cp -f /etc/kubernetes/admin.conf ${HOME}/.kube/config

• 关于默认网关问题，如果有多张网卡，需要先将默认网关切换到集群使用的那张网卡上，否则可能会出现etcd无法连接等问题。（应用我用的虚拟机，有一张网卡无法做到各个节点胡同；route查看当前网关信息，route del default删除默认网关，route add default enth0设置默认网关enth0为网卡名）

3)、拷贝相关证书到master2、master3

for index in 1 2; do
  ip=${IPS[${index}]}
  ssh $ip "mkdir -p /etc/kubernetes/pki/etcd; mkdir -p ~/.kube/"
  scp /etc/kubernetes/pki/ca.crt $ip:/etc/kubernetes/pki/ca.crt
  scp /etc/kubernetes/pki/ca.key $ip:/etc/kubernetes/pki/ca.key
  scp /etc/kubernetes/pki/sa.key $ip:/etc/kubernetes/pki/sa.key
  scp /etc/kubernetes/pki/sa.pub $ip:/etc/kubernetes/pki/sa.pub
  scp /etc/kubernetes/pki/front-proxy-ca.crt $ip:/etc/kubernetes/pki/front-proxy-ca.crt
  scp /etc/kubernetes/pki/front-proxy-ca.key $ip:/etc/kubernetes/pki/front-proxy-ca.key
  scp /etc/kubernetes/pki/etcd/ca.crt $ip:/etc/kubernetes/pki/etcd/ca.crt
  scp /etc/kubernetes/pki/etcd/ca.key $ip:/etc/kubernetes/pki/etcd/ca.key
  scp /etc/kubernetes/admin.conf $ip:/etc/kubernetes/admin.conf
  scp /etc/kubernetes/admin.conf $ip:~/.kube/config

  ssh ${ip} "${JOIN_CMD} --experimental-control-plane"
done

4)、master2、master3加入节点

JOIN_CMD=`kubeadm token create --print-join-command`
ssh ${ip} "${JOIN_CMD} --experimental-control-plane"

完整脚本：

# 部署HA master
 
bash -c "$(curl -fsSL https://raw.githubusercontent.com/hnbcao/kubeadm-ha-master/v1.14.0/kube-ha.sh)"

5、加入节点

• 各个节点需要配置keepalived 和 haproxy

  #/etc/haproxy/haproxy.cfg
  global
      log         127.0.0.1 local2
      chroot      /var/lib/haproxy
      pidfile     /var/run/haproxy.pid
      maxconn     4000
      user        haproxy
      group       haproxy
      daemon
      stats socket /var/lib/haproxy/stats
  
  defaults
      mode                    tcp
      log                     global
      option                  tcplog
      option                  dontlognull
      option                  redispatch
      retries                 3
      timeout queue           1m
      timeout connect         10s
      timeout client          1m
      timeout server          1m
      timeout check           10s
      maxconn                 3000
  
  listen stats
      mode   http
      bind :10086
      stats   enable
      stats   uri     /admin?stats
      stats   auth    admin:admin
      stats   admin   if TRUE
      
  frontend  k8s_https *:8443
      mode      tcp
      maxconn      2000
      default_backend     https_sri
      
  backend https_sri
      balance      roundrobin
      server master1-api ${MASTER1_IP}:6443  check inter 10000 fall 2 rise 2 weight 1
      server master2-api ${MASTER2_IP}:6443  check inter 10000 fall 2 rise 2 weight 1
      server master3-api ${MASTER3_IP}:6443  check inter 10000 fall 2 rise 2 weight 1
  

  #/etc/keepalived/keepalived.conf 
  global_defs {
     router_id LVS_DEVEL
  }
  
  vrrp_script check_haproxy {
      script /etc/keepalived/check_haproxy.sh
      interval 3
  }
  
  vrrp_instance VI_1 {
      state MASTER
      interface eth0
      virtual_router_id 80
      priority 100
      advert_int 1
      authentication {
          auth_type PASS
          auth_pass just0kk
      }
      virtual_ipaddress {
          ${VIP}/24
      }
      track_script {   
          check_haproxy
      }
  }
  
  }
  

注意两个配置中的{MASTER2 _ IP}, {VIP}需要替换为自己集群相应的IP地址

• 重启keepalived和haproxy

  systemctl stop keepalived
  systemctl enable keepalived
  systemctl start keepalived
  systemctl stop haproxy
  systemctl enable haproxy
  systemctl start haproxy
  

• 节点加入命令获取

  #master节点执行该命令，再在节点执行获取到的命令
  kubeadm token create --print-join-command
  

6、结束安装

当前集群安装完毕，还需要安装cni插件，推荐使用calico，性能优于其他。