负载均衡
nginx负载均衡
两台nginx负载均衡
vim /etc/nginx/nginx.conf
stream {upstream kube-apiserver {server 192.168.0.11:6443 max_fails=3 fail_timeout=30s;#server 192.168.0.12:6443 max_fails=3 fail_timeout=30s;#server 192.168.0.13:6443 max_fails=3 fail_timeout=30s;}server {listen 6443;proxy_connect_timeout 2s;proxy_timeout 900s;proxy_pass kube-apiserver;}
}
keepalived
nginx检测脚本
vim /data/shell/check_nginx_status.sh
#!/bin/bash
nginx_status=$(ps -ef | grep nginx | grep -v grep | grep -v check | wc -l)if [ $nginx_status -eq 0 ];thensystemctl stop keepalived.service
fi
master节点keepalived配置（不抢占机制）
vim /etc/keepalived/keepalived.conf
global_defs {router_id real-server1}vrrp_script chk_nginx {script "/data/shell/check_nginx_status.sh"interval 2
}vrrp_instance VI_1 {state BACKUPinterface ens32virtual_router_id 151priority 100nopreemptadvert_int 5authentication {auth_type PASSauth_pass 1111}virtual_ipaddress {192.168.0.10/24}track_script { chk_nginx}}
backup节点keepalived配置
vim /etc/keepalived/keepalived.conf
global_defs {router_id real-server2}vrrp_script chk_nginx {script "/data/shell/check_nginx_status.sh"interval 2
}vrrp_instance VI_1 {state BACKUPinterface ens32virtual_router_id 151priority 50nopreemptadvert_int 5authentication {auth_type PASSauth_pass 1111}virtual_ipaddress {192.168.0.10/24}track_script { chk_nginx}}
k8s节点系统设置
vim /etc/modules-load.d/containerd.conf
overlay
br_netfilter
modprobe overlay
modprobe br_netfilter
vim /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
vm.swappiness=0
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
sysctl --system
ipvsadm
yum install ipset ipvsadm
modprobe br_netfilter
modprobe overlay
modprobe ip_conntrack
modprobe ip_vs
modprobe ip_vs_rr
modprobe ip_vs_wrr
modprobe ip_vs_sh
modprobe nf_conntrack
lsmod | grep conntrack
lsmod | grep br_netfilt
lsmod | grep overlay
lsmod |egrep "ip_vs|nf_conntrack"
cat > /etc/modules-load.d/kubernetes.conf << EOF
# /etc/modules-load.d/kubernetes.conf
br_netfilter
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack_ipv4
ip_tables
EOF
chmod a+x /etc/modules-load.d/kubernetes.conf
containerd
wget https://github.com/containerd/containerd/releases/download/v1.7.23/cri-containerd-cni-1.7.23-linux-amd64.tar.gztar xvf cri-containerd-cni-1.7.23-linux-amd64.tar.gz -C /wget https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64
cp runc.amd64 /usr/local/sbin/runc
mkdir -p /etc/containerd
containerd config default > /etc/containerd/config.toml
vim /etc/containerd/config.toml
...
SystemdCgroup = true
...
sandbox_image = "registry.cn-beijing.aliyuncs.com/wuxingge123/pause:3.9"
...[plugins."io.containerd.grpc.v1.cri".registry.mirrors][plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]endpoint = ["https://p4oudlho.mirror.aliyuncs.com"]配置私有仓库
[plugins."io.containerd.grpc.v1.cri".registry][plugins."io.containerd.grpc.v1.cri".registry.mirrors][plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]endpoint = ["https://bqr1dr1n.mirror.aliyuncs.com"][plugins."io.containerd.grpc.v1.cri".registry.configs][plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.k8s.local".tls]insecure_skip_verify = true[plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.k8s.local".auth]username = "admin"password = "Harbor12345"
启动containerd
systemctl start containerd.service
systemctl enable containerd.service
k8s
安装
https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.30/rpm/x86_64/
vim /etc/yum.repos.d/k8s.repo
[kubernetes]
name=kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.30/rpm/
enabled=1
gpgcheck=0
在线安装
yum install kubelet-1.30.0 kubectl-1.30.0 kubeadm-1.30.0
离线安装
yum localinstall kubernetes-cni-1.4.0-150500.1.1.x86_64.rpm
yum localinstall cri-tools-1.30.0-150500.1.1.x86_64.rpm
yum localinstall kubeadm-1.30.0-150500.1.1.x86_64.rpm kubectl-1.30.0-150500.1.1.x86_64.rpm kubelet-1.30.0-150500.1.1.x86_64.rpm
配置kubelet
vim /etc/sysconfig/kubelet
KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"
systemctl enable kubelet
下载镜像
kubeadm config images list --kubernetes-version=v1.30.0 --image-repository registry.cn-beijing.aliyuncs.com/wuxingge123kubeadm config images pull --kubernetes-version=v1.30.0 --image-repository registry.cn-beijing.aliyuncs.com/wuxingge123
calico准备
wget https://docs.projectcalico.org/v3.25/manifests/calico.yaml --no-check-certificate
vim calico.yaml
- name: CALICO_IPV4POOL_CIDRvalue: "10.224.0.0/16"
下载calico镜像
ctr -n k8s.io image pull --all-platforms registry.cn-beijing.aliyuncs.com/wuxingge123/cni:v3.25.0ctr -n k8s.io image pull --all-platforms registry.cn-beijing.aliyuncs.com/wuxingge123/node:v3.25.0ctr -n k8s.io image pull --all-platforms registry.cn-beijing.aliyuncs.com/wuxingge123/kube-controllers:v3.25.0
k8s init
生成初始化配置文件
kubeadm config print init-defaults > kubeadm-config.yaml
vim kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:- system:bootstrappers:kubeadm:default-node-tokentoken: abcdef.0123456789abcdefttl: 24h0m0susages:- signing- authentication
kind: InitConfiguration
localAPIEndpoint:advertiseAddress: 192.168.0.11bindPort: 6443
nodeRegistration:criSocket: unix:///var/run/containerd/containerd.sockimagePullPolicy: IfNotPresenttaints: null
---
apiServer:timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: 192.168.0.10:6443
controllerManager: {}
dns: {}
etcd:local:dataDir: /var/lib/etcd
imageRepository: swr.cn-jl-1.manageone.cloud.cnpc/gsms-project
kind: ClusterConfiguration
kubernetesVersion: 1.30.0
networking:dnsDomain: cluster.localpodSubnet: 10.224.0.0/16serviceSubnet: 10.96.0.0/12
scheduler: {}
初始化集群
kubeadm init --config kubeadm-config.yaml --upload-certs
初始化成功显示以下内容
Your Kubernetes control-plane has initialized successfully!To start using your cluster, you need to run the following as a regular user:mkdir -p $HOME/.kubesudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/configsudo chown $(id -u):$(id -g) $HOME/.kube/configAlternatively, if you are the root user, you can run:export KUBECONFIG=/etc/kubernetes/admin.confYou should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:https://kubernetes.io/docs/concepts/cluster-administration/addons/You can now join any number of the control-plane node running the following command on each as root:kubeadm join 192.168.0.10:6443 --token abcdef.0123456789abcdef \--discovery-token-ca-cert-hash sha256:aac70b668d8010aca7af9a27ad9451468fc985dfd8f52a52025d14b180e16464 \--control-plane --certificate-key 8ea5c7b87d52438496fce053b1e9788217ffe74ed397d89f6a114e65d06c8826Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.Then you can join any number of worker nodes by running the following on each as root:kubeadm join 192.168.0.10:6443 --token abcdef.0123456789abcdef \--discovery-token-ca-cert-hash sha256:aac70b668d8010aca7af9a27ad9451468fc985dfd8f52a52025d14b180e16464
配置kubectl客户端
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
设置kubectl命令补全
echo "source <(kubectl completion bash)" >> /etc/profile
部署calico
kubectl apply -f calico.yaml
添加master节点
kubeadm join 192.168.0.10:6443 --token abcdef.0123456789abcdef \--discovery-token-ca-cert-hash sha256:aac70b668d8010aca7af9a27ad9451468fc985dfd8f52a52025d14b180e16464 \--control-plane --certificate-key 8ea5c7b87d52438496fce053b1e9788217ffe74ed397d89f6a114e65d06c8826
添加node节点
kubeadm join 192.168.0.10:6443 --token abcdef.0123456789abcdef \--discovery-token-ca-cert-hash sha256:aac70b668d8010aca7af9a27ad9451468fc985dfd8f52a52025d14b180e16464
ingress-nginx v1.11.2
kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.11.2/deploy/static/provider/cloud/deploy.yaml
metrics-server
wget https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml