189 8069 5689
# 影响k3s 高可用的阻力就是所有master 节点的证书统一，解决方案是先成功部署一个master 节点然后把节点生成
证书复制到其它master节点包括token，同时使用etcd 作为数据库# 操作系统：centos8 1905
#k3s版本：v0.9.1
#etcd 版本: v3.4.1
# etcd 服务器IP：192.168.30.50，192.168.30.51，192.168.30.52
# 安装目录：/apps/业务
# 服务器节点IP：192.168.30.50，192.168.30.51，192.168.30.52 node节点192.168.30.53，vip 节点：192.168.30.59
# k3s 集群域名： cluster.local
# k3s api 接口域名：api.k3s.tyong.com
# k3s cluster-cidr：10.48.0.0/12
# k3s service-cidr：10.64.0.0/16
#k3s cluster-DNS：10.64.0.2# 所有节点
# 下载etcd 二进制
wget https://github.com/etcd-io/etcd/releases/download/v3.4.1/etcd-v3.4.1-linux-amd64.tar.gz
# 解压二进制文件
tar -xvf etcd-v3.4.1-linux-amd64.tar.gz
# 创建etcd 运行目录
mkdir -p /apps/etcd/{bin,conf,ssl,data}
# 复制二进制到运行目录
cd etcd-v3.4.1-linux-amd64
mv mv etcd* /apps/etcd/bin
# 下载K3S
wget https://github.com/rancher/k3s/releases/download/v0.9.1/k3s
# 可执行权限
chmod +x k3s
# 复制k3s 到运行目录
mv k3s /usr/local/bin/
# 创建软链方便使用
cd /usr/local/bin/
ln -sf k3s kubectl
ln -sf k3s crictl
ln -sf k3s ctr
# 习惯修改
vi ~/.bashrc
alias docker='k3s crictl'
. ~/.bashrc# 设置 system.conf
cat >> /etc/systemd/system.conf << EOF
DefaultLimitMEMLOCK=infinity
DefaultLimitCORE=infinity
DefaultCPUAccounting=yes
DefaultMemoryAccounting=yes
DefaultLimitNOFILE=1024000
DefaultLimitNPROC=1024000
EOF
# 设置关闭防火墙及SELINUX
sed -i 's/SELINUX=.*/SELINUX=disabled/g' /etc/selinux/config
systemctl stop firewalld && systemctl disable firewalld
setenforce 0
# 关闭Swap
swapoff -a && sysctl -w vm.swappiness=0
vi /etc/fstab
#/dev/mapper/centos-swap swap swap defaults 0 0
# 设置 sysctl.conf 内核配置
true > /etc/sysctl.conf
cat >> /etc/sysctl.conf << EOF
net.ipv4.ip_forward = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
kernel.sysrq = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 1
fs.file-max = 1024000
fs.nr_open = 1024000
vm.swappiness = 0
vm.max_map_count = 2048000
vm.overcommit_memory = 1
kernel.sem =5010 641280 5010 128
kernel.pid_max = 4194303
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 68719476736
kernel.shmall = 4294967296
net.ipv4.tcp_max_tw_buckets = 6000
net.ipv4.tcp_sack = 1
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_mem = 786432 1697152 1945728
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 65536 16777216
net.core.wmem_default = 8388608
net.core.rmem_default = 8388608
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.core.netdev_max_backlog = 2048000
net.core.somaxconn = 65535
net.ipv4.tcp_max_orphans = 3276800
net.ipv4.tcp_max_syn_backlog = 2048000
net.ipv4.tcp_mem = 94500000 915000000 927000000
net.ipv4.tcp_fin_timeout = 1
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 10
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.ip_local_port_range = 1024 65535
net.ipv4.neigh.default.gc_stale_time=120
net.ipv4.conf.default.rp_filter=0
net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.all.arp_announce=2
net.ipv4.conf.lo.arp_announce=2
sunrpc.tcp_slot_table_entries=256
EOF
/sbin/sysctl -p
# 设置limits.conf
cat >> /etc/security/limits.conf << EOF
* soft nproc 1024000
* hard nproc 1024000
* soft nofile 1024000
* hard nofile 1024000
* soft core 1024000
* hard core 1024000
######big mem ########
#* hard memlock unlimited
#* soft memlock unlimited
EOF
# centos8 已经取消20-nproc.conf 文件
# 设置NetworkManager 配置静态IP
vi /etc/sysconfig/network-scripts/ifcfg-eth0
TYPE=Ethernet
BOOTPROTO=static
DEFROUTE=yes
NAME=eth0
DEVICE=eth0
ONBOOT=yes
IPADDR="192.168.30.50"
PREFIX="24"
GATEWAY="192.168.30.1"
DNS1="192.168.30.10"
# 生效配置
nmcli c reload
# centos8 已经取消network 访问管理网络 其它节点参考# 操作节点：192.168.30.50
# 部署go 环境变量，当然也可以在工作机器部署
#安装及配置CFSSL
yum install go
vi ~/.bash_profile
GOBIN=/root/go/bin/
PATH=$PATH:$GOBIN:$HOME/bin
export PATH
go get github.com/cloudflare/cfssl/cmd/cfssl
go get github.com/cloudflare/cfssl/cmd/cfssljson
# 创建etcdCA 证书配置
mkdir -p /apps/work/k8s/cfssl/ && \
cat << EOF | tee /apps/work/k8s/cfssl/ca-config.json
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "87600h"
}
}
}
}
EOF
---------------------------------------------------------------------
mkdir -p /apps/work/k8s/cfssl/etcd
cat << EOF | tee /apps/work/k8s/cfssl/etcd/etcd-ca-csr.json
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "GuangDong",
"L": "GuangZhou",
"O": "cluster",
"OU": "ETCD"
}
]
}
EOF
# 生成ca证书
----------------------------------------------------------------------
cfssl gencert -initca \
/apps/work/k8s/cfssl/etcd/etcd-ca-csr.json | \
cfssljson -bare ./etcd-ca
# 创建etcd server 证书配置
-----------------------------------------------------------------------------
export ETCD_SERVER_IPS=" \
\"192.168.30.50\", \
\"192.168.30.51\", \
\"192.168.30.52\" \
" && \
export ETCD_SERVER_HOSTNAMES=" \
\"k3s-001\", \
\"k3s-002\", \
\"k3s-003\" \
" && \
cat << EOF | tee /apps/work/k8s/cfssl/etcd/etcd_server.json
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
${ETCD_SERVER_IPS},
${ETCD_SERVER_HOSTNAMES}
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "GuangDong",
"L": "GuangZhou",
"O": "cluster",
"OU": "ETCD"
}
]
}
EOF
# 生成etcd server 证书
-----------------------------------------------------------------------------
cfssl gencert \
-ca=./etcd-ca.pem \
-ca-key=./etcd-ca-key.pem \
-config=/apps/work/k8s/cfssl/ca-config.json \
-profile=kubernetes \
/apps/work/k8s/cfssl/etcd/etcd_server.json | \
cfssljson -bare ./etcd_server
# 创建member证书 k3s-01 节点
--------------------------------------------------------------------------------------
export ETCD_MEMBER_1_IP=" \
\"192.168.30.50\" \
" && \
export ETCD_MEMBER_1_HOSTNAMES="k3s-001\
" && \
cat << EOF | tee /apps/work/k8s/cfssl/etcd/${ETCD_MEMBER_1_HOSTNAMES}.json
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
${ETCD_MEMBER_1_IP},
"${ETCD_MEMBER_1_HOSTNAMES}"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "GuangDong",
"L": "GuangZhou",
"O": "cluster",
"OU": "ETCD"
}
]
}
EOF
# 生成k3s-01 节点证书
-----------------------------------------------------------------------------
cfssl gencert \
-ca=./etcd-ca.pem \
-ca-key=./etcd-ca-key.pem \
-config=/apps/work/k8s/cfssl/ca-config.json \
-profile=kubernetes \
/apps/work/k8s/cfssl/etcd/${ETCD_MEMBER_1_HOSTNAMES}.json | \
cfssljson -bare ./etcd_member_${ETCD_MEMBER_1_HOSTNAMES}
# 创建生成k3s-02 节点配置
-----------------------------------------------------------------------------
export ETCD_MEMBER_2_IP=" \
\"192.168.30.51\" \
" && \
export ETCD_MEMBER_2_HOSTNAMES="k3s-002\
" && \
cat << EOF | tee /apps/work/k8s/cfssl/etcd/${ETCD_MEMBER_2_HOSTNAMES}.json
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
${ETCD_MEMBER_2_IP},
"${ETCD_MEMBER_2_HOSTNAMES}"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "GuangDong",
"L": "GuangZhou",
"O": "cluster",
"OU": "ETCD"
}
]
}
EOF
# 生成k3s-02 节点证书
--------------------------------------------------------------------------
cfssl gencert \
-ca=./etcd-ca.pem \
-ca-key=./etcd-ca-key.pem \
-config=/apps/work/k8s/cfssl/ca-config.json \
-profile=kubernetes \
/apps/work/k8s/cfssl/etcd/${ETCD_MEMBER_2_HOSTNAMES}.json | \
cfssljson -bare ./etcd_member_${ETCD_MEMBER_2_HOSTNAMES}
# 创建k3s-03 节点配置
--------------------------------------------------------------------------
export ETCD_MEMBER_3_IP=" \
\"192.168.30.52\" \
" && \
export ETCD_MEMBER_3_HOSTNAMES="k3s-003\
" && \
cat << EOF | tee /apps/work/k8s/cfssl/etcd/${ETCD_MEMBER_3_HOSTNAMES}.json
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
${ETCD_MEMBER_3_IP},
"${ETCD_MEMBER_3_HOSTNAMES}"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "GuangDong",
"L": "GuangZhou",
"O": "cluster",
"OU": "ETCD"
}
]
}
EOF
# 生成k3s-03 节点证书
------------------------------------------------------------------------------
cfssl gencert \
-ca=./etcd-ca.pem \
-ca-key=./etcd-ca-key.pem \
-config=/apps/work/k8s/cfssl/ca-config.json \
-profile=kubernetes \
/apps/work/k8s/cfssl/etcd/${ETCD_MEMBER_3_HOSTNAMES}.json | \
cfssljson -bare ./etcd_member_${ETCD_MEMBER_3_HOSTNAMES}
# 创建etcd client 证书
-----------------------------------------------------------------------------------
cat << EOF | tee /apps/work/k8s/cfssl/etcd/etcd_client.json
{
"CN": "client",
"hosts": [""],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "GuangDong",
"L": "GuangZhou",
"O": "cluster",
"OU": "ETCD"
}
]
}
EOF
# 生成etcd client 证书
--------------------------------------------------------------
cfssl gencert \
-ca=./etcd-ca.pem \
-ca-key=./etcd-ca-key.pem \
-config=/apps/work/k8s/cfssl/ca-config.json \
-profile=kubernetes \
/apps/work/k8s/cfssl/etcd/etcd_client.json | \
cfssljson -bare ./etcd_client
# 配置etcd01 启动文件
vi /apps/etcd/conf/etcd
------------------------------------------------------------------------------------------------------------------------------------------------
ETCD_OPTS="--name=k3s-001 \
--data-dir=/apps/etcd/data/default.etcd \
--listen-peer-urls=https://192.168.30.50:2380 \
--listen-client-urls=https://192.168.30.50:2379,https://127.0.0.1:2379 \
--advertise-client-urls=https://192.168.30.50:2379 \
--initial-advertise-peer-urls=https://192.168.30.50:2380 \
--initial-cluster=k3s-001=https://192.168.30.50:2380,k3s-002=https://192.168.30.51:2380,k3s-003=https://192.168.30.52:2380 \
--initial-cluster-token=k3s-001=https://192.168.30.50:2380,k3s-002=https://192.168.30.51:2380,k3s-003=https://192.168.30.52:2380 \
--initial-cluster-state=new \
--heartbeat-interval=6000 \
--election-timeout=30000 \
--snapshot-count=5000 \
--auto-compaction-retention=1 \
--max-request-bytes=33554432 \
--quota-backend-bytes=17179869184 \
--trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem \
--cert-file=/apps/etcd/ssl/etcd_server.pem \
--key-file=/apps/etcd/ssl/etcd_server-key.pem \
--peer-cert-file=/apps/etcd/ssl/etcd_member_k3s-001.pem \
--peer-key-file=/apps/etcd/ssl/etcd_member_k3s-001-key.pem \
--peer-client-cert-auth \
--peer-trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem"
# 创建etcd 启动文件
------------------------------------------------------------------------
vi /usr/lib/systemd/system/etcd.service
---------------------------------------------------------------------
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
LimitNOFILE=1024000
LimitNPROC=1024000
LimitCORE=infinity
LimitMEMLOCK=infinity
User=etcd
Group=etcd
EnvironmentFile=-/apps/etcd/conf/etcd
ExecStart=/apps/etcd/bin/etcd $ETCD_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
-----------------------------------------------------------------
# 创建etcd 用户
useradd etcd -s /sbin/nologin -M
-----------------------------------------------------------------
# 给予/apps/etcd 目录etcd用户权限
chown -R etcd:etcd /apps/etcd
----------------------------------------------------------------
# k3s-02 节点创建etcd 用户
useradd etcd -s /sbin/nologin -M
-----------------------------------------------------------------
# k3s-03 节点创建etcd 用户
useradd etcd -s /sbin/nologin -M
-----------------------------------------------------------------
# 分发文件到 k3s-02 k3s-03 节点
scp -r /apps/etcd 192.168.30.51:/apps/
scp -r /apps/etcd 192.168.30.52:/apps/
# 分发启动文件到 k3s-02 k3s-03 节点
scp /usr/lib/systemd/system/etcd.service 192.168.30.51: /usr/lib/systemd/system/etcd.service
scp /usr/lib/systemd/system/etcd.service 192.168.30.52: /usr/lib/systemd/system/etcd.service
# 修改k3s-02 /apps/etcd/conf/etcd 文件
vi /apps/etcd/conf/etcd
--------------------------------------------------------------------------
ETCD_OPTS="--name=k3s-002 \
--data-dir=/apps/etcd/data/default.etcd \
--listen-peer-urls=https://192.168.30.51:2380 \
--listen-client-urls=https://192.168.30.51:2379,https://127.0.0.1:2379 \
--advertise-client-urls=https://192.168.30.51:2379 \
--initial-advertise-peer-urls=https://192.168.30.51:2380 \
--initial-cluster=k3s-001=https://192.168.30.50:2380,k3s-002=https://192.168.30.51:2380,k3s-003=https://192.168.30.52:2380 \
--initial-cluster-token=k3s-001=https://192.168.30.50:2380,k3s-002=https://192.168.30.51:2380,k3s-003=https://192.168.30.52:2380 \
--initial-cluster-state=new \
--heartbeat-interval=6000 \
--election-timeout=30000 \
--snapshot-count=5000 \
--auto-compaction-retention=1 \
--max-request-bytes=33554432 \
--quota-backend-bytes=17179869184 \
--trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem \
--cert-file=/apps/etcd/ssl/etcd_server.pem \
--key-file=/apps/etcd/ssl/etcd_server-key.pem \
--peer-cert-file=/apps/etcd/ssl/etcd_member_k3s-002.pem \
--peer-key-file=/apps/etcd/ssl/etcd_member_k3s-002-key.pem \
--peer-client-cert-auth \
--peer-trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem"
# 修改 修改k3s-03 /apps/etcd/conf/etcd 文件
----------------------------------------------------------------------------
vi /apps/etcd/conf/etcd
----------------------------------------------------------------------------
ETCD_OPTS="--name=k3s-003 \
--data-dir=/apps/etcd/data/default.etcd \
--listen-peer-urls=https://192.168.30.52:2380 \
--listen-client-urls=https://192.168.30.52:2379,https://127.0.0.1:2379 \
--advertise-client-urls=https://192.168.30.52:2379 \
--initial-advertise-peer-urls=https://192.168.30.52:2380 \
--initial-cluster=k3s-001=https://192.168.30.50:2380,k3s-002=https://192.168.30.51:2380,k3s-003=https://192.168.30.52:2380 \
--initial-cluster-token=k3s-001=https://192.168.30.50:2380,k3s-002=https://192.168.30.51:2380,k3s-003=https://192.168.30.52:2380 \
--initial-cluster-state=new \
--heartbeat-interval=6000 \
--election-timeout=30000 \
--snapshot-count=5000 \
--auto-compaction-retention=1 \
--max-request-bytes=33554432 \
--quota-backend-bytes=17179869184 \
--trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem \
--cert-file=/apps/etcd/ssl/etcd_server.pem \
--key-file=/apps/etcd/ssl/etcd_server-key.pem \
--peer-cert-file=/apps/etcd/ssl/etcd_member_k3s-003.pem \
--peer-key-file=/apps/etcd/ssl/etcd_member_k3s-003-key.pem \
--peer-client-cert-auth \
--peer-trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem"
-----------------------------------------------------------------------------
# 启动etcd 集群 k3s-01 k3s-02 k3s-03
systemctl start etcd
# 设置开机启动 k3s-01 k3s-02 k3s-03
systemctl enable etcd
----------------------------------------------------------------------------------
# 验证K3S 是否正常
vi /etc/profile
export ETCDCTL_API=3
export ENDPOINTS=https://192.168.30.50:2379,https://192.168.30.51:2379,https://192.168.30.52:2379
# 生效环境变量
. /etc/profile
# 配置命令别名 alias
vi /root/.bashrc
alias etcdctl='/apps/etcd/bin/etcdctl --endpoints=${ENDPOINTS} --cacert=/apps/etcd/ssl/etcd-ca.pem'
# 生效
. /root/.bashrc
# 验证集群
etcdctl member list
etcdctl endpoint status
https://192.168.30.50:2379, 7b98f2ed4d780753, 3.3.12, 290 MB, true, 37886, 82704406
https://192.168.30.51:2379, 47fa5d2eb78a7751, 3.3.12, 289 MB, false, 37886, 82704408
https://192.168.30.52:2379, 76c6cd81499cf7ba, 3.3.12, 289 MB, false, 37886, 82704433
# etcd 集群正常# 添加一个虚拟IP
ip addr add 192.168.30.59/24 dev eth0
# 安装依赖
--------------------------------------------
dnf install epel-release
---------------------------
dnf install dnf-utils ipvsadm telnet wget net-tools conntrack ipset jq iptables curl sysstat libseccomp socat nfs-utils fuse fuse-devel
------------------------------------------------
# centos 8 不能自动加载ipvs 创建开机加载
cat << EOF > /etc/sysconfig/modules/ipvs.modules
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
# /etc/sysconfig/modules/ipvs.modules 可执行权限
chmod +x /etc/sysconfig/modules/ipvs.modules
# 执行 /etc/sysconfig/modules/ipvs.modules
/etc/sysconfig/modules/ipvs.modules
-----------------------------------
mkdir -p /apps/k3s
# 操作节点k3s-01 节点
# 创建k3s env
vi /etc/sysconfig/k3s.env
----------------------------------------------
K3S_SERVER_OPTS='--data-dir=/apps/k3s \
--no-deploy=traefik \
--no-deploy=coredns \
--no-deploy=servicelb \
--no-deploy=helm-install-traefik \
--kube-proxy-arg="proxy-mode=ipvs" \
--kube-proxy-arg="masquerade-all=true" \
--cluster-cidr="10.48.0.0/12" \
--service-cidr="10.64.0.0/16" \
--cluster-dns="10.64.0.2" \
--cluster-domain="cluster.local" \
--tls-san="192.168.30.51" \
--tls-san="192.168.30.52" \
--tls-san="192.168.30.59" \
--tls-san="192.168.30.50" \
--tls-san="api.k3s.tyong.com" \
--tls-san="kubernetes" \
--tls-san="kubernetes.default" \
--tls-san="kubernetes.default.svc" \
--tls-san="kubernetes.default.svc.cluster.local" \
--storage-endpoint=etcd \
--kube-apiserver-arg="etcd-cafile=/apps/etcd/ssl/etcd-ca.pem" \
--kube-apiserver-arg="etcd-certfile=/apps/etcd/ssl/etcd_client.pem" \
--kube-apiserver-arg="etcd-keyfile=/apps/etcd/ssl/etcd_client-key.pem" \
--kube-apiserver-arg="etcd-prefix=/registry" \
--kube-apiserver-arg="etcd-servers=https://192.168.30.50:2379,https://192.168.30.51:2379,https://192.168.30.52:2379" \
--kube-apiserver-arg="runtime-config=api/all=true" \
--kube-apiserver-arg="enable-admission-plugins=DefaultStorageClass,DefaultTolerationSeconds,LimitRanger,NamespaceExists,NamespaceLifecycle,NodeRestriction,OwnerReferencesPermissionEnforcement,PodNodeSelector,PersistentVolumeClaimResize,PodPreset,PodTolerationRestriction,ResourceQuota,ServiceAccount,StorageObjectInUseProtection,MutatingAdmissionWebhook,ValidatingAdmissionWebhook" \
--kube-apiserver-arg="disable-admission-plugins=DenyEscalatingExec,ExtendedResourceToleration,ImagePolicyWebhook,LimitPodHardAntiAffinityTopology,NamespaceAutoProvision,Priority,EventRateLimit,PodSecurityPolicy" \
--kube-controller-arg="horizontal-pod-autoscaler-use-rest-clients=true" \
--pause-image=docker.io/juestnow/pause-amd64:3.1 \
--resolv-conf="/etc/resolv.conf"'
----------------------------------------------------------------------------------------------------------------------------------
# 创建k3s 启动文件
vi /etc/systemd/system/k3s.service
-----------------------------------------------------------------------------------------------------------------------------
[Unit]
Description=Lightweight Kubernetes
Documentation=https://k3s.io
After=network-online.target
[Service]
Type=notify
EnvironmentFile=/etc/sysconfig/k3s.env
ExecStartPre=-/sbin/modprobe br_netfilter
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/k3s \
server $K3S_SERVER_OPTS \
KillMode=process
Delegate=yes
LimitNOFILE=1024000
LimitNPROC=1024000
LimitCORE=infinity
TasksMax=infinity
TimeoutStartSec=0
Restart=always
RestartSec=5s
[Install]
WantedBy=multi-user.target
# 启动k3s
systemctl enable k3s && systemctl start k3s
# 等待K3S 启动正常 然后关闭k3s
-----------------------------------------------
systemctl stop k3s
--------------------------------
# 复制K3S 生成的配置文件到其它master节点
scp -r /etc/rancher 192.168.30.51:/etc/rancher
scp -r /etc/rancher 192.168.30.52:/etc/rancher
scp -r /apps/k3s 192.168.30.51:/apps/
scp -r /apps/k3s 192.168.30.52:/apps/
-------------------------------------------
#启动 k3s-01 节点
-------------------------------------------
systemctl start k3s
-------------------------------------------
ssh 192.168.30.51,52
systemctl enable k3s && systemctl start k3s
---------------------------------------------
# 验证k3s 节点是否启动正常
k3s kubectl get node
# kubeconfig 文件生成账号密码 操作K3S 集群用到admin 权限
cat /etc/rancher/k3s/k3s.yaml
# 远程操作
scp /etc/rancher/k3s/k3s.yaml /root/.kube/config
vim /root/.kube/config
#127.0.0.1 改成远程服务器IP 192.168.30.50，51，52 测试所有节点是否能正常返回如果都返回正常证明集群部署成功# 依赖coredns 可以先部署coredns 也可以把--no-deploy=coredns 删除这边使用自建coredns
--------------------------------------------------------------------------------------------------------------------------
vi coredns.yaml
-------------------------------------------------------------------------------------------------------------------------
# __MACHINE_GENERATED_WARNING__
apiVersion: v1
kind: ServiceAccount
metadata:
name: coredns
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
kubernetes.io/bootstrapping: rbac-defaults
addonmanager.kubernetes.io/mode: Reconcile
name: system:coredns
rules:
- apiGroups:
- ""
resources:
- endpoints
- services
- pods
- namespaces
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
labels:
kubernetes.io/bootstrapping: rbac-defaults
addonmanager.kubernetes.io/mode: EnsureExists
name: system:coredns
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:coredns
subjects:
- kind: ServiceAccount
name: coredns
namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
name: coredns
namespace: kube-system
labels:
addonmanager.kubernetes.io/mode: EnsureExists
data:
Corefile: |
.:53 {
errors
health
kubernetes mddgame.local in-addr.arpa ip6.arpa {
pods insecure
upstream /etc/resolv.conf
fallthrough in-addr.arpa ip6.arpa
}
prometheus :9153
forward . /etc/resolv.conf
cache 30
reload
loadbalance
}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: coredns
namespace: kube-system
labels:
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
kubernetes.io/name: "CoreDNS"
spec:
# replicas: not specified here:
# 1. In order to make Addon Manager do not reconcile this replicas parameter.
# 2. Default is 1.
# 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
selector:
matchLabels:
k8s-app: kube-dns
template:
metadata:
labels:
k8s-app: kube-dns
annotations:
seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
spec:
priorityClassName: system-cluster-critical
serviceAccountName: coredns
tolerations:
- key: "CriticalAddonsOnly"
operator: "Exists"
nodeSelector:
beta.kubernetes.io/os: linux
containers:
- name: coredns
image: coredns/coredns
imagePullPolicy: Always
resources:
limits:
memory: 170Mi
requests:
cpu: 100m
memory: 70Mi
args: [ "-conf", "/etc/coredns/Corefile" ]
volumeMounts:
- name: config-volume
mountPath: /etc/coredns
readOnly: true
ports:
- containerPort: 53
name: dns
protocol: UDP
- containerPort: 53
name: dns-tcp
protocol: TCP
- containerPort: 9153
name: metrics
protocol: TCP
livenessProbe:
httpGet:
path: /health
port: 8080
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
readinessProbe:
httpGet:
path: /health
port: 8080
scheme: HTTP
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- NET_BIND_SERVICE
drop:
- all
readOnlyRootFilesystem: true
dnsPolicy: Default
volumes:
- name: config-volume
configMap:
name: coredns
items:
- key: Corefile
path: Corefile
---
apiVersion: v1
kind: Service
metadata:
name: kube-dns
namespace: kube-system
annotations:
prometheus.io/port: "9153"
prometheus.io/scrape: "true"
labels:
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
kubernetes.io/name: "CoreDNS"
spec:
selector:
k8s-app: kube-dns
clusterIP: 10.64.0.2
ports:
- name: dns
port: 53
protocol: UDP
- name: dns-tcp
port: 53
protocol: TCP
- name: metrics
port: 9153
protocol: TCP
# 创建coredns dns
k3s kubectl apply -f coredns.yaml
k3s kubectl get pod -A
# 等待部署完成
# 安装依赖
--------------------------------------------
dnf install epel-release
---------------------------
dnf install dnf-utils ipvsadm telnet wget net-tools conntrack ipset jq iptables curl sysstat libseccomp socat nfs-utils fuse fuse-devel
------------------------------------------------
# centos 8 不能自动加载ipvs 创建开机加载
cat << EOF > /etc/sysconfig/modules/ipvs.modules
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
# /etc/sysconfig/modules/ipvs.modules 可执行权限
chmod +x /etc/sysconfig/modules/ipvs.modules
# 执行 /etc/sysconfig/modules/ipvs.modules
/etc/sysconfig/modules/ipvs.modules
-----------------------------------
# 获取token 任意master 节点 所有master 节点token 一点要一致
---------------------------------------------
cat /apps/k3s/server/node-token
K1000966fac151ec94a53040dadd727a4ef1ccac022aa8747f0b601ca33665417ea::node:0aa3ce3afaf275fd33ae6a2a9580d3a0
-----------------------------------------------------------------------------------------------------------------
# 创建k3s agent env
vi /etc/sysconfig/k3a.env
----------------------------------------------------------
K3S_AGENT_OPTS='--data-dir=/apps/k3s \
--kube-proxy-arg="proxy-mode=ipvs" \
--kube-proxy-arg="masquerade-all=true" \
--pause-image=docker.io/juestnow/pause-amd64:3.1 \
--resolv-conf="/etc/resolv.conf" \
--server=https://192.168.30.59:6443 \
--token=K1000966fac151ec94a53040dadd727a4ef1ccac022aa8747f0b601ca33665417ea::node:0aa3ce3afaf275fd33ae6a2a9580d3a0'
--------------------------------------------------------------------------------------------------------------------------------
# 创建 启动脚本
vi /etc/systemd/system/k3a.service
----------------------------------------------------------------------------------------------------
[Unit]
Description=Lightweight Kubernetes
Documentation=https://k3s.io
After=network-online.target
[Service]
Type=notify
EnvironmentFile=/etc/sysconfig/k3a.env
ExecStartPre=-/sbin/modprobe br_netfilter
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/k3s \
agent $K3S_AGENT_OPTS \
KillMode=process
Delegate=yes
LimitNOFILE=1024000
LimitNPROC=1024000
LimitCORE=infinity
TasksMax=infinity
TimeoutStartSec=0
Restart=always
RestartSec=5s
[Install]
WantedBy=multi-user.target
-----------------------------------------------------------------------------
# 启动 k3s agent
systemctl enable k3a && systemctl start k3a
# 验证 agent 是否加入集群
k3s kubectl get node
# 应该有work 名字的节点
# 这个与K8S 集群几乎没任何区别可以部署监控及kubernetes-dashboard及所有应用
# k3s 默认使用containerd kubelet 还是不能监控pod 网络当然切换成docker 就可以
# 单mater 部署这里就不展开讨论，网络上很多这样的示例。
# agent 会t同时连接3台master 节点任意节点关闭都不会对agent 节点有影响可以不用考虑 haproxy做代理