Experimental environment planning

Summary

Use VMwork virtual machine to deploy small cluster with single master and double nodes, and install etcd on both master and node to implement etcd cluster.

Software adopted version

Server role assignment

Previous attention

All experimental virtual machines have turned off selinux core protection function

Build working directory for binary package

[root@master ~]# cd /
[root@master /]# mkdir k8s

Install ETCD database (three node backup)

master side

Download cfssl, a ca Certificate creation and management tool

[root@master k8s]# curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /usr/local/bin/cfssl
[root@master k8s]# curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /usr/local/bin/cfssljson
[root@master k8s]# curl -L https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o /usr/local/bin/cfssl-certinfo
[root@master k8s]# chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/local/bin/cfssl-certinfo

Generate ca certificate

//Define ca certificate and generate ca certificate configuration file
[root@master k8s]# cat > ca-config.json <<EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "www": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF

//Generate certificate signature file
[root@master k8s]# cat > ca-csr.json <<EOF
{
    "CN": "etcd CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing"
        }
    ]
}
EOF

//Generate CA certificate, ca.pem, CA key.pem
[root@master k8s]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

[root@master k8s]# ls
ca-config.json  ca-csr.json  ca.pem   ca.csr     ca-key.pem   
"ca.pem"         //ca certificate file
"ca-key.pem"         //ca key certificate file

Generate communication certificate between nodes of etcd

(note the change of IP address)

//Generate communication verification signature between etcd nodes
//Node address must be changed
[root@master k8s]# cat > server-csr.json <<EOF
{
    "CN": "etcd",
    "hosts": [
    "192.168.142.220",
    "192.168.142.136",
    "192.168.142.132"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing"
        }
    ]
}
EOF

//Generate etcd Communication Certificate for etcd communication verification
[root@master k8s]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server

[root@master k8s]# ls
ca-config.json  ca-csr.json  ca.pem        server.csr       server-key.pem
ca.csr          ca-key.pem   etcd-cert.sh  server-csr.json  server.pem
"server.pem"&"server-key.pem"      //Verification certificate of communication between etcd s

Configure etcd binaries

//Unzip and install etcd
[root@master k8s]# tar zxf etcd-v3.3.10-linux-amd64.tar.gz

//Establish a directory to store etcd configuration files, commands and certificates
[root@master k8s]# mkdir -p /opt/etcd/{cfg,bin,ssl}
[root@master k8s]# ls /opt/etcd/
bin  cfg  ssl

//Storing files in etcd by category
[root@master k8s]# mv etcd-v3.3.10-linux-amd64/etcd /opt/etcd/bin/
[root@master k8s]# mv etcd-v3.3.10-linux-amd64/etcdctl /opt/etcd/bin/
[root@master k8s]# cp -p *.pem /opt/etcd/ssl/

//etcd command file
[root@master k8s]# ls /opt/etcd/bin/
etcd  etcdctl

//etcd certificate
[root@master k8s]# ls /opt/etcd/ssl/
ca-key.pem  ca.pem  server-key.pem  server.pem

Create etcd profile

[root@master k8s]# cat <<EOF >/opt/etcd/cfg/etcd
//[Member]
ETCD_NAME="etcd01"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.142.220:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.142.220:2379"

//[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.142.220:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.142.220:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.142.220:2380,etcd02=https://192.168.142.136:2380,etcd03=https://192.168.142.132:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF

Create etcd startup script

[root@master k8s]# cat <<EOF >/usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/opt/etcd/cfg/etcd
ExecStart=/opt/etcd/bin/etcd \
--name=etcd01 \
--data-dir=\${ETCD_DATA_DIR} \
--listen-peer-urls=\${ETCD_LISTEN_PEER_URLS} \
--listen-client-urls=\${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
--advertise-client-urls=\${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-advertise-peer-urls=\${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster=\${ETCD_INITIAL_CLUSTER} \
--initial-cluster-token=\${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-cluster-state=new \
--cert-file=/opt/etcd/ssl/server.pem \
--key-file=/opt/etcd/ssl/server-key.pem \
--peer-cert-file=/opt/etcd/ssl/server.pem \
--peer-key-file=/opt/etcd/ssl/server-key.pem \
--trusted-ca-file=/opt/etcd/ssl/ca.pem \
--peer-trusted-ca-file=/opt/etcd/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

Push certificate, configuration file, startup script to node

//Push certificates and configuration files to other nodes
[root@master ~]# scp -r /opt/etcd/ root@192.168.142.136:/opt/
[root@master ~]# scp -r /opt/etcd/ root@192.168.142.132:/opt/

//Push start script
[root@master ~]# scp /usr/lib/systemd/system/etcd.service root@192.168.142.136:/usr/lib/systemd/system/
[root@master ~]# scp /usr/lib/systemd/system/etcd.service root@192.168.142.132:/usr/lib/systemd/system/

node side

Change the profile received by each node node

node1

[root@node1 etcd]# vim /opt/etcd/cfg/etcd
//[Member]
ETCD_NAME="etcd02"                      //Node name
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.142.136:2380 "/ / node address
ETCD_LISTEN_CLIENT_URLS="https://192.168.142.136:2379 "/ / node address

//[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.142.136:2380 "/ / node address
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.142.136:2379 "/ / node address

node 2

[root@node2 ~]# vim /opt/etcd/cfg/etcd
//[Member]
ETCD_NAME="etcd03"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.142.132:2380 "/ / node name
ETCD_LISTEN_CLIENT_URLS="https://192.168.142.132:2379 "/ / node name

//[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.142.132:2380 "/ / node name
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.142.132:2379 "/ / node name

All nodes!!!!

Opening service

//Turn off the firewall at each node
[root@node1 etcd]# systemctl stop firewalld.service
[root@node1 etcd]# setenforce 0
[root@node1 etcd]# systemctl disable firewalld.service

//Start etcd service
[root@node1 etcd]# systemctl start etcd
[root@node1 etcd]# systemctl enable etcd

Cluster health check, view cluster members

//Health check of cluster nodes
[root@master ~]# /opt/etcd/bin/etcdctl --ca-file=/opt/etcd/ssl/ca.pem \
--cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem \
--endpoints="https://192.168.142.220:2379,https://192.168.142.136:2379,https://192.168.142.132:2379" \
cluster-health

//View etcd cluster members
[root@master ~]# /opt/etcd/bin/etcdctl --ca-file=/opt/etcd/ssl/ca.pem \
--cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem \
--endpoints="https://192.168.142.220:2379,https://192.168.142.136:2379,https://192.168.142.132:2379" member list

To be continued~~~