基于kubernetes的etcd 3.3.10外部集群

etcd是一个分布式键值存储，它提供了一种在一组机器上存储数据的可靠方法。它是开源的，可在GitHub上获得。etcd在网络分区期间优雅地处理leader选举，并且可以容忍机器故障，包括leader。

应用程序可以将数据读写到etcd中。一个简单的用例是将etcd中的数据库连接详细信息或功能标记存储为键值对。可以监视这些值，允许您的应用在更改时重新配置。

高级用法利用一致性保证来实现数据库leader选举或跨工作集群进行分布式锁定etcd是用Go编写的，它具有出色的跨平台支持，较小二进制文件和活跃的社区。etcd机器之间的通信通过Raft一致性算法处理。

此处的ETCD主要用来部署kubernetes高可用集群，此后的使用都是基于kubernetes。

参考kubernetes官网：https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/

etcd配置：https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/hardware.md#hardware-recommendations集群文档：https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/clustering.md示例参考：https://github.com/etcd-io/etcd/tree/master/hack/tls-setup

参考：https://github.com/etcd-io/etcd/tree/master/hack/tls-setup/config参考：https://k8smeetup.github.io/docs/setup/independent/high-availability/我们首先配置etcd证书，etcd我们将会用在kubernetes上。这一步是必须的相比较github上etcd的示例，我们简单修改下

在这之前 ，我们有必要修改一下主机名，并且配通所有的ssh

hostnamectl set-hostname etcd1
hostnamectl set-hostname etcd2
hostnamectl set-hostname etcd3

cat >>  /etc/hosts << EOF
172.25.50.16 etcd1
172.25.50.17 etcd2
172.25.50.18 etcd3
EOF

[root@linuxea.com-16 /etc/etcda]# ssh-keygen -t rsa
[root@linuxea.com-16 /etc/etcda]# for i in 172.25.50.{17,18};do ssh-copy-id $i; done

安装cfssl和cfssljson

[root@linuxea.com-16 ~]#  curl -so /usr/local/bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
[root@linuxea.com-16 ~]#  curl -o /usr/local/bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
[root@linuxea.com-16 ~]#  chmod +x /usr/local/bin/cfssl*

生成证书

[root@linuxea.com-16 ~]# mkdir -p /etc/kubernetes/pki/etcd
[root@linuxea.com-16 ~]# cd /etc/kubernetes/pki/etcd

• ca-config.json

  cat > cat /etc/kubernetes/pki/etcd/ca-config.json  << EOF
  {
     "signing": {
         "default": {
             "expiry": "876000h"
         },
         "profiles": {
             "server": {
                 "expiry": "876000h",
                 "usages": [
                     "signing",
                     "key encipherment",
                     "server auth",
                     "client auth"
                 ]
             },
             "client": {
                 "expiry": "876000h",
                 "usages": [
                     "signing",
                     "key encipherment",
                     "client auth"
                 ]
             },
             "peer": {
                 "expiry": "876000h",
                 "usages": [
                     "signing",
                     "key encipherment",
                     "server auth",
                     "client auth"
                 ]
             }
         }
     }
   }
  EOF

• ca-csr.json

  cat > ca-csr.json << EOL
  {
  "CN": "etcd",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Shanghai",
      "L": "Shanghai",
      "O": "etcd",
      "OU": "Etcd Security"
    }
  ]
  }
  EOL

  生成CA证书

  [root@linuxea.com-16 /etc/kubernetes/pki/etcd]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
  2018/12/25 16:29:14 [INFO] generating a new CA key and certificate from CSR
  2018/12/25 16:29:14 [INFO] generate received request
  2018/12/25 16:29:14 [INFO] received CSR
  2018/12/25 16:29:14 [INFO] generating key: rsa-2048
  2018/12/25 16:29:14 [INFO] encoded CSR
  2018/12/25 16:29:14 [INFO] signed certificate with serial number 472142876620060394898834048533122419461412171471

  [root@linuxea.com-16 /etc/kubernetes/pki/etcd]# ll
  total 20
  -rw-r--r-- 1 root root  905 Dec 25 16:28 ca-config.json
  -rw-r--r-- 1 root root 1005 Dec 25 16:29 ca.csr
  -rw-r--r-- 1 root root  212 Dec 25 16:29 ca-csr.json
  -rw------- 1 root root 1679 Dec 25 16:29 ca-key.pem
  -rw-r--r-- 1 root root 1371 Dec 25 16:29 ca.pem

生成 etcd 客户端证书

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# cat client.json 
 {
     "CN": "client",
     "key": {
         "algo": "ecdsa",
         "size": 256
     }
 }

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client.json | cfssljson -bare client
2018/12/25 16:29:56 [INFO] generate received request
2018/12/25 16:29:56 [INFO] received CSR
2018/12/25 16:29:56 [INFO] generating key: ecdsa-256
2018/12/25 16:29:56 [INFO] encoded CSR
2018/12/25 16:29:56 [INFO] signed certificate with serial number 644510971695673396838569226835778482472560755733
2018/12/25 16:29:56 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

如下

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# ll
total 36
-rw-r--r-- 1 root root  905 Dec 25 16:28 ca-config.json
-rw-r--r-- 1 root root 1005 Dec 25 16:29 ca.csr
-rw-r--r-- 1 root root  212 Dec 25 16:29 ca-csr.json
-rw------- 1 root root 1679 Dec 25 16:29 ca-key.pem
-rw-r--r-- 1 root root 1371 Dec 25 16:29 ca.pem
-rw-r--r-- 1 root root  351 Dec 25 16:29 client.csr
-rw-r--r-- 1 root root   95 Dec 25 16:29 client.json
-rw------- 1 root root  227 Dec 25 16:29 client-key.pem
-rw-r--r-- 1 root root  997 Dec 25 16:29 client.pem

config.json

对于config.json有两种方式,第一种,使用官网的，如下

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# cfssl print-defaults csr > config.json

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# sed -i '0,/CN/{s/example.net/'"$PEER_NAME"'/}' config.json

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# sed -i 's/www.example.net/'"$PRIVATE_IP"'/' config.json

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# sed -i 's/example.net/'"$PUBLIC_IP"'/' config.json

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# cat config.json
{
    "CN": "etcd1",
    "hosts": [
        "",
        "172.25.50.16"
    ],
    "key": {
        "algo": "ecdsa",
        "size": 256
    },
    "names": [
        {
            "C": "US",
            "L": "CA",
            "ST": "San Francisco"
        }
    ]
}

第二种方式，直接在这里编辑， 填写参与集群的ip

cat > /etc/kubernetes/pki/etcd/config.json << EOF
{
    "CN": "etcd1",
    "hosts": [
      "127.0.0.1",
      "172.25.50.16",
      "172.25.50.17",
      "172.25.50.18"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "Shanghai",
            "L": "Shanghai",
            "O": "etcd",
            "OU": "Etcd Security"
        }
    ]
}
EOF

运行 cfssl 命令，将会生成peer.pem、peer-key.pem、server.pem、server-key.pem。

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server config.json | cfssljson -bare server
2018/12/25 16:37:53 [INFO] generate received request
2018/12/25 16:37:53 [INFO] received CSR
2018/12/25 16:37:53 [INFO] generating key: rsa-2048
2018/12/25 16:37:54 [INFO] encoded CSR
2018/12/25 16:37:54 [INFO] signed certificate with serial number 397776469717117599117003178668354588092528739871
2018/12/25 16:37:54 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]#  cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer config.json | cfssljson -bare peer
2018/12/25 16:37:59 [INFO] generate received request
2018/12/25 16:37:59 [INFO] received CSR
2018/12/25 16:37:59 [INFO] generating key: rsa-2048
2018/12/25 16:37:59 [INFO] encoded CSR
2018/12/25 16:37:59 [INFO] signed certificate with serial number 453856739993256449551996181659627954567417235192
2018/12/25 16:37:59 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

如下

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# ll
total 64
-rw-r--r-- 1 root root  905 Dec 25 16:28 ca-config.json
-rw-r--r-- 1 root root 1005 Dec 25 16:29 ca.csr
-rw-r--r-- 1 root root  212 Dec 25 16:29 ca-csr.json
-rw------- 1 root root 1679 Dec 25 16:29 ca-key.pem
-rw-r--r-- 1 root root 1371 Dec 25 16:29 ca.pem
-rw-r--r-- 1 root root  351 Dec 25 16:29 client.csr
-rw-r--r-- 1 root root   95 Dec 25 16:29 client.json
-rw------- 1 root root  227 Dec 25 16:29 client-key.pem
-rw-r--r-- 1 root root  997 Dec 25 16:29 client.pem
-rw-r--r-- 1 root root  375 Dec 25 16:37 config.json
-rw-r--r-- 1 root root 1078 Dec 25 16:37 peer.csr
-rw------- 1 root root 1679 Dec 25 16:37 peer-key.pem
-rw-r--r-- 1 root root 1456 Dec 25 16:37 peer.pem
-rw-r--r-- 1 root root 1078 Dec 25 16:37 server.csr
-rw------- 1 root root 1679 Dec 25 16:37 server-key.pem
-rw-r--r-- 1 root root 1456 Dec 25 16:37 server.pem

证书传递

将这些生成的证书复制到etcd2和etcd3上

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# for i in 172.25.50.{17,18} ;do scp -r /etc/kubernetes $i:/etc ;done
ca-config.json                   100%  905   635.1KB/s   00:00    
ca-csr.json                      100%  212   174.5KB/s   00:00    
ca.pem                           100% 1371     1.0MB/s   00:00    
ca-key.pem                       100% 1679     1.3MB/s   00:00    
ca.csr                           100% 1005   773.7KB/s   00:00    
client.json                      100%   95    76.1KB/s   00:00    
client.pem                       100%  997   751.7KB/s   00:00    
client-key.pem                   100%  227   171.9KB/s   00:00    
client.csr                       100%  351   256.1KB/s   00:00    
config.json                      100%  375    96.5KB/s   00:00    
server.pem                       100% 1456   365.4KB/s   00:00    
server-key.pem                   100% 1679   425.8KB/s   00:00    
server.csr                       100% 1078   276.8KB/s   00:00    
peer.pem                         100% 1456   366.5KB/s   00:00    
peer-key.pem                     100% 1679   439.1KB/s   00:00    
peer.csr                         100% 1078   289.2KB/s   00:00    
ca-config.json                   100%  905   569.4KB/s   00:00    
ca-csr.json                      100%  212   134.9KB/s   00:00    
ca.pem                           100% 1371   944.5KB/s   00:00    
ca-key.pem                       100% 1679     1.1MB/s   00:00    
ca.csr                           100% 1005   605.7KB/s   00:00    
client.json                      100%   95    63.3KB/s   00:00    
client.pem                       100%  997   748.6KB/s   00:00    
client-key.pem                   100%  227   151.1KB/s   00:00    
client.csr                       100%  351   244.9KB/s   00:00    
config.json                      100%  375    90.4KB/s   00:00    
server.pem                       100% 1456   322.2KB/s   00:00    
server-key.pem                   100% 1679   372.1KB/s   00:00    
server.csr                       100% 1078   253.3KB/s   00:00    
peer.pem                         100% 1456   325.0KB/s   00:00    
peer-key.pem                     100% 1679   394.1KB/s   00:00    
peer.csr                         100% 1078   259.3KB/s   00:00  

安装etcd

这里的环境变量在三台参与集群的机器分别运行

版本3.3.10

export ETCD_VERSION=v3.3.10
curl -sSL https://github.com/coreos/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz | tar -xzv --strip-components=1 -C /usr/local/bin/
rm -rf etcd-$ETCD_VERSION-linux-amd64*

先决变量

主机名和ip地址，这里的eth0应该与机器的网卡相符

export PEER_NAME=$(hostname)
export PRIVATE_IP=$(ip addr show eth0 | grep -Po 'inet K[d.]+')

环境变量写入到/etc/etcd.env

touch /etc/etcd.env
echo "PEER_NAME=$PEER_NAME" >> /etc/etcd.env
echo "PRIVATE_IP=$PRIVATE_IP" >> /etc/etcd.env

启动脚本即将用到的变量，这里标记的是参与etcd集群的三个ip地址

export etcd0_ip_address=172.25.50.16
export etcd1_ip_address=172.25.50.17
export etcd2_ip_address=172.25.50.18

启动脚本

这里面的变量就是上面设置的

cat > /etc/systemd/system/etcd.service << EOL
[Unit]
Description=etcd
Documentation=https://github.com/coreos/etcd
Conflicts=etcd.service
Conflicts=etcd2.service

[Service]
EnvironmentFile=/etc/etcd.env
Type=notify
Restart=always
RestartSec=5s
LimitNOFILE=40000
TimeoutStartSec=0

ExecStart=/usr/local/bin/etcd --name ${PEER_NAME} 
    --data-dir /var/lib/etcd 
    --listen-client-urls https://${PRIVATE_IP}:2379 
    --advertise-client-urls https://${PRIVATE_IP}:2379 
    --listen-peer-urls https://${PRIVATE_IP}:2380 
    --initial-advertise-peer-urls https://${PRIVATE_IP}:2380 
    --cert-file=/etc/kubernetes/pki/etcd/server.pem 
    --key-file=/etc/kubernetes/pki/etcd/server-key.pem 
    --client-cert-auth 
    --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem 
    --peer-cert-file=/etc/kubernetes/pki/etcd/peer.pem 
    --peer-key-file=/etc/kubernetes/pki/etcd/peer-key.pem 
    --peer-client-cert-auth 
    --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem 
    --initial-cluster etcd1=https://${etcd0_ip_address}:2380,etcd2=https://${etcd1_ip_address}:2380,etcd3=https://${etcd2_ip_address}:2380 
    --initial-cluster-state new

[Install]
WantedBy=multi-user.target

EOL

systemctl daemon-reload
systemctl enable etcd.service
systemctl start etcd

集群状态

查看集群状态是需要证书的，我们配置一个环境变量

CMD='--cacert=/etc/kubernetes/pki/etcd/ca.pem --cert=/etc/kubernetes/pki/etcd/server.pem --key=/etc/kubernetes/pki/etcd/server-key.pem'
CMD1='https://172.25.50.16:2379,https://172.25.50.17:2379,https://172.25.50.18:2379'
CMD2='--ca-file=/etc/kubernetes/pki/etcd/ca.pem --cert-file=/etc/kubernetes/pki/etcd/server.pem --key-file=/etc/kubernetes/pki/etcd/server-key.pem'

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# for i in 172.25.50.{16,17,18}; do ETCDCTL_API=3 etcdctl --endpoints=https://${i}:2379 $CMD endpoint health; done
https://172.25.50.16:2379 is healthy: successfully committed proposal: took = 1.984026ms
https://172.25.50.17:2379 is healthy: successfully committed proposal: took = 3.357136ms
https://172.25.50.18:2379 is healthy: successfully committed proposal: took = 3.55185ms

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# ETCDCTL_API=3 etcdctl --endpoints=https://172.25.50.16:2379 $CMD member list
2e70a124f01a4a5, started, etcd3, https://172.25.50.18:2380, https://172.25.50.18:2379
5fba4c5d1e214899, started, etcd2, https://172.25.50.17:2380, https://172.25.50.17:2379
b55bca6849256d2d, started, etcd1, https://172.25.50.16:2380, https://172.25.50.16:2379
[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# 

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# etcdctl -C $CMD1 $CMD2 cluster-health
member 2e70a124f01a4a5 is healthy: got healthy result from https://172.25.50.18:2379
member 5fba4c5d1e214899 is healthy: got healthy result from https://172.25.50.17:2379
member b55bca6849256d2d is healthy: got healthy result from https://172.25.50.16:2379
cluster is healthy

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# curl -Lk --cert ./server.pem --key ./server-key.pem  https://172.25.50.16:2379/metrics|grep -v debugging

延伸阅读：https://coreos.com/etcd/docs/latest/metrics.html

docker安装

如果安装此前的配置，docker的配置应该如下

• 172.25.50.16

  docker run --net=host -d -v /etc/kubernetes/pki/etcd/:/etc/kubernetes/pki/etcd/ -p 4001:4001 -p 2380:2380 -p 2379:2379 
   --name etcd quay.io/coreos/etcd:v3.3.10 
   etcd -name etcd1 
   --data-dir /var/lib/etcd  
   -advertise-client-urls https://172.25.50.16:2379,https://172.25.50.16:4001 
   -listen-client-urls https://0.0.0.0:2379,https://0.0.0.0:4001 
   -initial-advertise-peer-urls https://172.25.50.16:2380 
   -listen-peer-urls https://0.0.0.0:2380 
   --cert-file=/etc/kubernetes/pki/etcd/server.pem 
   --key-file=/etc/kubernetes/pki/etcd/server-key.pem 
   --client-cert-auth 
   --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem 
   --peer-cert-file=/etc/kubernetes/pki/etcd/peer.pem 
   --peer-key-file=/etc/kubernetes/pki/etcd/peer-key.pem 
   --peer-client-cert-auth 
   --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem 
   -initial-cluster-token etcd-cluster 
   -initial-cluster etcd1=https://172.25.50.16:2380,etcd2=https://172.25.50.17:2380,etcd3=https://172.25.50.18:2380 
   -initial-cluster-state new

• 172.25.50.17

  docker run --net=host -d -v /etc/kubernetes/pki/etcd/:/etc/kubernetes/pki/etcd/ -p 4001:4001 -p 2380:2380 -p 2379:2379 
   --name etcd quay.io/coreos/etcd:v3.3.10 
   etcd -name etcd2 
   --data-dir /var/lib/etcd 
   -advertise-client-urls https://172.25.50.17:2379,https://172.25.50.17:4001 
   -listen-client-urls https://0.0.0.0:2379,https://0.0.0.0:4001 
   -initial-advertise-peer-urls https://172.25.50.17:2380 
   -listen-peer-urls https://0.0.0.0:2380 
   --cert-file=/etc/kubernetes/pki/etcd/server.pem 
   --key-file=/etc/kubernetes/pki/etcd/server-key.pem 
   --client-cert-auth 
   --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem 
   --peer-cert-file=/etc/kubernetes/pki/etcd/peer.pem 
   --peer-key-file=/etc/kubernetes/pki/etcd/peer-key.pem 
   --peer-client-cert-auth 
   --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem 
   -initial-cluster-token etcd-cluster 
   -initial-cluster etcd1=https://172.25.50.16:2380,etcd2=https://172.25.50.17:2380,etcd3=https://172.25.50.18:2380 
   -initial-cluster-state new 

• 172.25.50.18

  docker run --net=host -d -v /etc/kubernetes/pki/etcd/:/etc/kubernetes/pki/etcd/ -p 4001:4001 -p 2380:2380 -p 2379:2379 
   --name etcd quay.io/coreos/etcd:v3.3.10 
   etcd -name etcd3 
   --data-dir /var/lib/etcd 
   -advertise-client-urls https://172.25.50.18:2379,https://172.25.50.18:4001 
   -listen-client-urls https://0.0.0.0:2379,https://0.0.0.0:4001 
   -initial-advertise-peer-urls https://172.25.50.18:2380 
   -listen-peer-urls https://0.0.0.0:2380 
   --cert-file=/etc/kubernetes/pki/etcd/server.pem 
   --key-file=/etc/kubernetes/pki/etcd/server-key.pem 
   --client-cert-auth 
   --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem 
   --peer-cert-file=/etc/kubernetes/pki/etcd/peer.pem 
   --peer-key-file=/etc/kubernetes/pki/etcd/peer-key.pem 
   --peer-client-cert-auth 
   --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem 
   -initial-cluster-token etcd-cluster 
   -initial-cluster etcd1=https://172.25.50.16:2380,etcd2=https://172.25.50.17:2380,etcd3=https://172.25.50.18:2380 
   -initial-cluster-state new  

  这样比较麻烦，我们简化一下

• 先决条件：

配置各个主机的hostname

[root@DT_Node-172_25_50_16 /etc/kubernetes/pki/etcd]# hostnamectl set-hostname etcd1
[root@DT_Node-172_25_50_17 /etc/kubernetes/pki/etcd]# hostnamectl set-hostname etcd2
[root@DT_Node-172_25_50_18 /etc/kubernetes/pki/etcd]# hostnamectl set-hostname etcd3

• 环境变量

export PEER_NAME=$(hostname)
export PRIVATE_IP=$(ip addr show eth0 | grep -Po 'inet K[d.]+')
export etcd0_ip_address=172.25.50.16
export etcd1_ip_address=172.25.50.17
export etcd2_ip_address=172.25.50.18

现在docker的启动命令就如下所示了：

docker run --net=host -d -v /etc/kubernetes/pki/etcd/:/etc/kubernetes/pki/etcd/ -p 4001:4001 -p 2380:2380 -p 2379:2379 
 -v /data/etcd:/data/etcd 
 --name etcd quay.io/coreos/etcd:v3.3.10 
 etcd -name ${PEER_NAME} 
 --data-dir /data/etcd 
 -advertise-client-urls https://${PRIVATE_IP}:2379,https://${PRIVATE_IP}:4001 
 -listen-client-urls https://0.0.0.0:2379,https://0.0.0.0:4001 
 -initial-advertise-peer-urls https://${PRIVATE_IP}:2380 
 -listen-peer-urls https://0.0.0.0:2380 
 --cert-file=/etc/kubernetes/pki/etcd/server.pem 
 --key-file=/etc/kubernetes/pki/etcd/server-key.pem 
 --client-cert-auth 
 --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem 
 --peer-cert-file=/etc/kubernetes/pki/etcd/peer.pem 
 --peer-key-file=/etc/kubernetes/pki/etcd/peer-key.pem 
 --peer-client-cert-auth 
 --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem 
 -initial-cluster-token etcd-cluster 
 -initial-cluster etcd1=https://${etcd0_ip_address}:2380,etcd2=https://${etcd1_ip_address}:2380,etcd3=https://${etcd2_ip_address}:2380 
 -initial-cluster-state new

但是这样还是不太方便，我们写成docker-compose即可

docker-compose

version: '2.2'
services:
  etcd:
    image: marksugar/coreos-etcd:v3.3.10
    container_name: etcd3
    restart: always
    privileged: true
    network_mode: "host"
    volumes:
      - /data/etcd:/data/etcd
      - /etc/kubernetes/pki/etcd/:/etc/kubernetes/pki/etcd/
    command: "etcd -name ${PEER_NAME} --data-dir /data/etcd -advertise-client-urls https://${PRIVATE_IP}:2379,https://${PRIVATE_IP}:4001 -listen-client-urls https://0.0.0.0:2379,https://0.0.0.0:4001 -initial-advertise-peer-urls https://${PRIVATE_IP}:2380 -listen-peer-urls https://0.0.0.0:2380 --cert-file=/etc/kubernetes/pki/etcd/server.pem --key-file=/etc/kubernetes/pki/etcd/server-key.pem --client-cert-auth --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem --peer-cert-file=/etc/kubernetes/pki/etcd/peer.pem --peer-key-file=/etc/kubernetes/pki/etcd/peer-key.pem --peer-client-cert-auth --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem -initial-cluster-token etcd-cluster -initial-cluster etcd1=https://${etcd0_ip_address}:2380,etcd2=https://${etcd1_ip_address}:2380,etcd3=https://${etcd2_ip_address}:2380 -initial-cluster-state new "
    cpu_shares: 90
    mem_limit: 2048m
    logging:
      driver: "json-file"
      options:
        max-size: "200M"
    labels:
      SERVICE_TAGS: etcd

• 必须：-v /data/etcd:/data/etcd

如果丢失数据-dir ==永远丢失成员。

附上三个快速重置的脚本：脚本运行后会自动删除集群内的数据存储目录，而后重启当前节点的etcd

#!/bin/bash
#########################################################################
# File Name: start.sh
# Author: www.linuxea.com
# Email: usertzc@163.com
# Version:
# Created Time: Wed 02 Jan 2019 11:14:26 AM CST
#########################################################################
for i in 172.25.50.{16,17,18};do ssh $i "docker rm -f etcd && rm -rf /data/etcd && ls /data";done
hostnamectl set-hostname etcd1
export PEER_NAME=$(hostname)
export PRIVATE_IP=$(ip addr show eth0 | grep -Po 'inet K[d.]+')
export etcd0_ip_address=172.25.50.16
export etcd1_ip_address=172.25.50.17
export etcd2_ip_address=172.25.50.18
hostname
docker-compose -f /opt/docker-compose.yaml up -d

#CMD='--cacert=/etc/kubernetes/pki/etcd/ca.pem --cert=/etc/kubernetes/pki/etcd/server.pem --key=/etc/kubernetes/pki/etcd/server-key.pem'
#CMD1='https://172.25.50.16:2379,https://172.25.50.17:2379,https://172.25.50.18:2379'
#CMD2='--ca-file=/etc/kubernetes/pki/etcd/ca.pem --cert-file=/etc/kubernetes/pki/etcd/server.pem --key-file=/etc/kubernetes/pki/etcd/server-key.pem'
#for i in 172.25.50.{16,17,18}; do ETCDCTL_API=3 etcdctl --endpoints=https://${i}:2379 $CMD endpoint health; done
#cd /etc/kubernetes/pki/etcd/ && scp -P22992 ca.pem client.pem client-key.pem 172.25.50.13:/etc/kubernetes/pki/etcd/

脚本2中仅仅设置了环境变量和启动的docker-compose.yaml

#!/bin/bash
#########################################################################
# File Name: start.sh
# Author: www.linuxea.com
# Email: usertzc@163.com
# Version:
# Created Time: Wed 02 Jan 2019 11:15:02 AM CST
#########################################################################
#docker rm -f etcd && rm -rf /data/etcd
hostnamectl set-hostname etcd2
export PEER_NAME=$(hostname)
export PRIVATE_IP=$(ip addr show eth0 | grep -Po 'inet K[d.]+')
export etcd0_ip_address=172.25.50.16
export etcd1_ip_address=172.25.50.17
export etcd2_ip_address=172.25.50.18
hostname
docker-compose -f /opt/docker-compose.yaml up -d

脚本3和2几乎 一样，除了名称外

#!/bin/bash
#########################################################################
# File Name: start.sh
# Author: www.linuxea.com
# Email: usertzc@163.com
# Version:
# Created Time: Wed 02 Jan 2019 11:15:26 AM CST
#########################################################################
#docker rm -f etcd && rm -rf /data/etcd

hostnamectl set-hostname etcd3
export PEER_NAME=$(hostname)
export PRIVATE_IP=$(ip addr show eth0 | grep -Po 'inet K[d.]+')
export etcd0_ip_address=172.25.50.16
export etcd1_ip_address=172.25.50.17
export etcd2_ip_address=172.25.50.18
hostname
docker-compose -f /opt/docker-compose.yaml up -d

延伸阅读：https://coreos.com/etcd/docs/latest/v2/docker_guide.html https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/container.md

监控

https://coreos.com/etcd/docs/latest/metrics.html

https://etcd.readthedocs.io/en/latest/operate.html#v3-3

如果要在kubernetes中监控外部etcd，可参考：https://github.com/marksugar/k8s-pgmon