2. cka考试环境k8s搭建（Ubuntu非高可用版）¶
1. 安装前准备工作¶
0. 切换到root用户¶
sudo -i
1. 设置hosts解析¶
操作节点：所有节点（k8s-master）均需执行
- 修改hostname
hostname必须只能包含小写字母、数字、","、"-"，且开头结尾必须是小写字母或数字
# 在master节点 hostnamectl set-hostname k8s-master #设置master节点的hostname # slave1节点 hostnamectl set-hostname k8s-worker-node1
2. 调整系统配置¶
操作节点： 所有的master和slave节点（k8s-master,k8s-slave）需要执行
本章下述操作均以k8s-master为例，其他节点均是相同的操作（ip和hostname的值换成对应机器的真实值）
设置iptables
iptables -P FORWARD ACCEPT
/etc/init.d/ufw stop
ufw disable
- 关闭swap
swapoff -a # 防止开机自动挂载 swap 分区 sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab - 修改内核参数
cat <<EOF > /etc/sysctl.d/k8s.conf net.bridge.bridge-nf-call-ip6tables = 1 net.bridge.bridge-nf-call-iptables = 1 net.ipv4.ip_forward=1 vm.max_map_count=262144 EOF modprobe br_netfilter sysctl -p /etc/sysctl.d/k8s.conf
3. 设置apt源¶
apt-get update && apt-get install -y apt-transport-https ca-certificates software-properties-common
curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add -
curl -fsSL https://mirrors.ustc.edu.cn/docker-ce/linux/ubuntu/gpg | sudo apt-key add
add-apt-repository "deb [arch=amd64] https://mirrors.ustc.edu.cn/docker-ce/linux/ubuntu $(lsb_release -cs) stable"
add-apt-repository "deb [arch=amd64] https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial main"
apt-get update
#若上步出现NO_PUBLICKEY问题，参考https://www.cnblogs.com/jiangzuo/p/13667011.html
4. 安装docker¶
操作节点： 所有节点
#查看所有可安装的docker-ce版本
apt list docker-ce -a
#如果看不到可安装版本，再执行上一步操作即可！
apt-get install docker-ce=5:20.10.8~3-0~ubuntu-bionic
#安装失败，尝试下面的命令（focal貌似是长期支持版本的意思，自己百度）安装比较慢，看网速
apt-get install docker-ce=5:20.10.8~3-0~ubuntu-focal -y
## 启动docker
systemctl enable docker && systemctl start docker
root@k8s-master:~# apt list docker-ce -a
Listing... Done
docker-ce/focal 5:20.10.10~3-0~ubuntu-focal amd64
docker-ce/focal 5:20.10.9~3-0~ubuntu-focal amd64
docker-ce/focal 5:20.10.8~3-0~ubuntu-focal amd64
docker-ce/focal 5:20.10.7~3-0~ubuntu-focal amd64
docker-ce/focal 5:20.10.6~3-0~ubuntu-focal amd64
docker-ce/focal 5:20.10.5~3-0~ubuntu-focal amd64
docker-ce/focal 5:20.10.4~3-0~ubuntu-focal amd64
docker-ce/focal 5:20.10.3~3-0~ubuntu-focal amd64
docker-ce/focal 5:20.10.2~3-0~ubuntu-focal amd64
docker-ce/focal 5:20.10.1~3-0~ubuntu-focal amd64
docker-ce/focal 5:20.10.0~3-0~ubuntu-focal amd64
docker-ce/focal 5:19.03.15~3-0~ubuntu-focal amd64
docker-ce/focal 5:19.03.14~3-0~ubuntu-focal amd64
docker-ce/focal 5:19.03.13~3-0~ubuntu-focal amd64
docker-ce/focal 5:19.03.12~3-0~ubuntu-focal amd64
docker-ce/focal 5:19.03.11~3-0~ubuntu-focal amd64
docker-ce/focal 5:19.03.10~3-0~ubuntu-focal amd64
docker-ce/focal 5:19.03.9~3-0~ubuntu-focal amd64
2. 部署kubernetes¶
1. 安装 kubeadm, kubelet 和 kubectl¶
操作节点： 所有的master和slave节点(k8s-master,k8s-slave) 需要执行
apt-get install kubelet=1.21.0-00 kubectl=1.21.0-00 kubeadm=1.21.0-00
## 查看kubeadm 版本
kubeadm version
kubectl version
kubelet --version
## 设置kubelet开机启动
systemctl enable kubelet
2. 初始化配置文件¶
操作节点： 只在master节点（k8s-master）执行
#执行此操作得到yaml
kubeadm config print init-defaults > kubeadm.yaml
ls
#以下vim操作
apt install -y vim
#
vim kubeadm.yaml
#
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 10.0.24.4 # 修改为master节点ip
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: node # 删掉此行，删掉此行，删掉此行
taints: null
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers # 修改此处镜像repo
kind: ClusterConfiguration
kubernetesVersion: 1.21.0
networking:
dnsDomain: cluster.local
podSubnet: 10.244.0.0/16 # 添加此行
serviceSubnet: 10.96.0.0/12
scheduler: {}
3. 提前下载镜像¶
操作节点：只在master节点（k8s-master）执行
# 提前下载镜像到本地
kubeadm config images pull --config kubeadm.yaml
#以下
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-apiserver:v1.21.0
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-controller-manager:v1.21.0
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-scheduler:v1.21.0
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-proxy:v1.21.0
[config/images] Pulled registry.aliyuncs.com/google_containers/pause:3.4.1
[config/images] Pulled registry.aliyuncs.com/google_containers/etcd:3.4.13-0
failed to pull image "registry.aliyuncs.com/google_containers/coredns/coredns:v1.8.0": output: Error response from daemon: pull access denied for registry.aliyuncs.com/google_containers/coredns/coredns, repository does not exist or may require 'docker login': denied: requested access to the resource is denied
, error: exit status 1
To see the stack trace of this error execute with --v=5 or higher
提示找不到coredns的镜像，我们可以通过如下方式解决：
# 这个最好所有节点都执行！！！因为coredns-pod启动后不一定在哪个节点上！
docker pull coredns/coredns:1.8.0
docker tag coredns/coredns:1.8.0 registry.aliyuncs.com/google_containers/coredns/coredns:v1.8.0
4. 初始化master节点¶
操作节点：只在master节点（k8s-master）执行
kubeadm init --config kubeadm.yaml
root@k8s-master:~# kubeadm init --config kubeadm.yaml
[init] Using Kubernetes version: v1.21.0
[preflight] Running pre-flight checks
[WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/
error execution phase preflight: [preflight] Some fatal errors occurred:
[ERROR NumCPU]: the number of available CPUs 1 is less than the required 2
[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...`
To see the stack trace of this error execute with --v=5 or higher
root@k8s-master:~# kubeadm init --config kubeadm.yaml --ignore-preflight-errors=NumCPU
若初始化成功后，最后会提示如下信息：
...
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
Alternatively, if you are the root user, you can run:
export KUBECONFIG=/etc/kubernetes/admin.conf
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 192.168.136.138:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:3a7987c9f5007ebac7980e6614281ee0e064c760c8db012471f9f662289cc9ce
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
**⚠️注意：**此时使用 kubectl get nodes查看节点应该处于notReady状态，因为还未配置网络插件
若执行初始化过程中出错，根据错误信息调整后，执行kubeadm reset后再次执行init操作即可
5. 添加slave节点到集群中¶
操作节点：所有的slave节点（k8s-slave）需要执行
在每台slave节点，执行如下命令，该命令是在kubeadm init成功后提示信息中打印出来的，需要替换成实际init后打印出的命令。
kubeadm join 192.168.136.138:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:1c4305f032f4bf534f628c32f5039084f4b103c922ff71b12a5f0f98d1ca9a4f
6. 安装calico插件¶
操作节点：只在master节点（k8s-master）执行
1. 下载资源文件¶
wget https://docs.projectcalico.org/manifests/calico-etcd.yaml
2. 查看calico配置文件¶
root@k8s-master:~# cat calico-etcd.yaml -n
1 ---
2 # Source: calico/templates/calico-etcd-secrets.yaml
3 # The following contains k8s Secrets for use with a TLS enabled etcd cluster.
4 # For information on populating Secrets, see http://kubernetes.io/docs/user-guide/secrets/
5 #apiVersion: v1
6 #kind: Secret
7 #type: Opaque
8 #metadata:
9 # name: calico-etcd-secrets
10 # namespace: kube-system
11 #data:
12 # Populate the following with etcd TLS configuration if desired, but leave blank if
13 # not using TLS for etcd.
14 # The keys below should be uncommented and the values populated with the base64
15 # encoded contents of each file that would be associated with the TLS data.
16 # Example command for encoding a file contents: cat <file> | base64 -w 0
17 # etcd-key: null
18 # etcd-cert: null
19 # etcd-ca: null
20 ---
21 # Source: calico/templates/calico-config.yaml
22 # This ConfigMap is used to configure a self-hosted Calico installation.
23 kind: ConfigMap
24 apiVersion: v1
25 metadata:
26 name: calico-config
27 namespace: kube-system
28 data:
29 # Configure this with the location of your etcd cluster.
30 etcd_endpoints: "http://192.168.178.100:2379"
31 # If you're using TLS enabled etcd uncomment the following.
32 # You must also populate the Secret below with these files.
33 etcd_ca: "/calico-secrets/etcd-ca" # "/calico-secrets/etcd-ca"
34 etcd_cert: "/calico-secrets/etcd-cert" # "/calico-secrets/etcd-cert"
35 etcd_key: "/calico-secrets/etcd-key" # "/calico-secrets/etcd-key"
36 # Typha is disabled.
37 typha_service_name: "none"
38 # Configure the backend to use.
39 calico_backend: "bird"
40
41 # Configure the MTU to use for workload interfaces and tunnels.
42 # By default, MTU is auto-detected, and explicitly setting this field should not be required.
43 # You can override auto-detection by providing a non-zero value.
44 veth_mtu: "0"
45
46 # The CNI network configuration to install on each node. The special
47 # values in this config will be automatically populated.
48 cni_network_config: |-
49 {
50 "name": "k8s-pod-network",
51 "cniVersion": "0.3.1",
52 "plugins": [
53 {
54 "type": "calico",
55 "log_level": "info",
56 "log_file_path": "/var/log/calico/cni/cni.log",
57 "etcd_endpoints": "__ETCD_ENDPOINTS__",
58 "etcd_key_file": "__ETCD_KEY_FILE__",
59 "etcd_cert_file": "__ETCD_CERT_FILE__",
60 "etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__",
61 "mtu": __CNI_MTU__,
62 "ipam": {
63 "type": "calico-ipam"
64 },
65 "policy": {
66 "type": "k8s"
67 },
68 "kubernetes": {
69 "kubeconfig": "__KUBECONFIG_FILEPATH__"
70 }
71 },
72 {
73 "type": "portmap",
74 "snat": true,
75 "capabilities": {"portMappings": true}
76 },
77 {
78 "type": "bandwidth",
79 "capabilities": {"bandwidth": true}
80 }
81 ]
82 }
83
84 ---
85 # Source: calico/templates/calico-kube-controllers-rbac.yaml
86
87 # Include a clusterrole for the kube-controllers component,
88 # and bind it to the calico-kube-controllers serviceaccount.
89 kind: ClusterRole
90 apiVersion: rbac.authorization.k8s.io/v1
91 metadata:
92 name: calico-kube-controllers
93 rules:
94 # Pods are monitored for changing labels.
95 # The node controller monitors Kubernetes nodes.
96 # Namespace and serviceaccount labels are used for policy.
97 - apiGroups: [""]
98 resources:
99 - pods
100 - nodes
101 - namespaces
102 - serviceaccounts
103 verbs:
104 - watch
105 - list
106 - get
107 # Watch for changes to Kubernetes NetworkPolicies.
108 - apiGroups: ["networking.k8s.io"]
109 resources:
110 - networkpolicies
111 verbs:
112 - watch
113 - list
114 ---
115 kind: ClusterRoleBinding
116 apiVersion: rbac.authorization.k8s.io/v1
117 metadata:
118 name: calico-kube-controllers
119 roleRef:
120 apiGroup: rbac.authorization.k8s.io
121 kind: ClusterRole
122 name: calico-kube-controllers
123 subjects:
124 - kind: ServiceAccount
125 name: calico-kube-controllers
126 namespace: kube-system
127 ---
128
129 ---
130 # Source: calico/templates/calico-node-rbac.yaml
131 # Include a clusterrole for the calico-node DaemonSet,
132 # and bind it to the calico-node serviceaccount.
133 kind: ClusterRole
134 apiVersion: rbac.authorization.k8s.io/v1
135 metadata:
136 name: calico-node
137 rules:
138 # The CNI plugin needs to get pods, nodes, and namespaces.
139 - apiGroups: [""]
140 resources:
141 - pods
142 - nodes
143 - namespaces
144 verbs:
145 - get
146 # EndpointSlices are used for Service-based network policy rule
147 # enforcement.
148 - apiGroups: ["discovery.k8s.io"]
149 resources:
150 - endpointslices
151 verbs:
152 - watch
153 - list
154 - apiGroups: [""]
155 resources:
156 - endpoints
157 - services
158 verbs:
159 # Used to discover service IPs for advertisement.
160 - watch
161 - list
162 # Pod CIDR auto-detection on kubeadm needs access to config maps.
163 - apiGroups: [""]
164 resources:
165 - configmaps
166 verbs:
167 - get
168 - apiGroups: [""]
169 resources:
170 - nodes/status
171 verbs:
172 # Needed for clearing NodeNetworkUnavailable flag.
173 - patch
174
175 ---
176 apiVersion: rbac.authorization.k8s.io/v1
177 kind: ClusterRoleBinding
178 metadata:
179 name: calico-node
180 roleRef:
181 apiGroup: rbac.authorization.k8s.io
182 kind: ClusterRole
183 name: calico-node
184 subjects:
185 - kind: ServiceAccount
186 name: calico-node
187 namespace: kube-system
188
189 ---
190 # Source: calico/templates/calico-node.yaml
191 # This manifest installs the calico-node container, as well
192 # as the CNI plugins and network config on
193 # each master and worker node in a Kubernetes cluster.
194 kind: DaemonSet
195 apiVersion: apps/v1
196 metadata:
197 name: calico-node
198 namespace: kube-system
199 labels:
200 k8s-app: calico-node
201 spec:
202 selector:
203 matchLabels:
204 k8s-app: calico-node
205 updateStrategy:
206 type: RollingUpdate
207 rollingUpdate:
208 maxUnavailable: 1
209 template:
210 metadata:
211 labels:
212 k8s-app: calico-node
213 spec:
214 nodeSelector:
215 kubernetes.io/os: linux
216 hostNetwork: true
217 tolerations:
218 # Make sure calico-node gets scheduled on all nodes.
219 - effect: NoSchedule
220 operator: Exists
221 # Mark the pod as a critical add-on for rescheduling.
222 - key: CriticalAddonsOnly
223 operator: Exists
224 - effect: NoExecute
225 operator: Exists
226 serviceAccountName: calico-node
227 # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
228 # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
229 terminationGracePeriodSeconds: 0
230 priorityClassName: system-node-critical
231 initContainers:
232 # This container installs the CNI binaries
233 # and CNI network config file on each node.
234 - name: install-cni
235 image: docker.io/calico/cni:v3.21.0
236 command: ["/opt/cni/bin/install"]
237 envFrom:
238 - configMapRef:
239 # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
240 name: kubernetes-services-endpoint
241 optional: true
242 env:
243 # Name of the CNI config file to create.
244 - name: CNI_CONF_NAME
245 value: "10-calico.conflist"
246 # The CNI network config to install on each node.
247 - name: CNI_NETWORK_CONFIG
248 valueFrom:
249 configMapKeyRef:
250 name: calico-config
251 key: cni_network_config
252 # The location of the etcd cluster.
253 - name: ETCD_ENDPOINTS
254 valueFrom:
255 configMapKeyRef:
256 name: calico-config
257 key: etcd_endpoints
258 # CNI MTU Config variable
259 - name: CNI_MTU
260 valueFrom:
261 configMapKeyRef:
262 name: calico-config
263 key: veth_mtu
264 # Prevents the container from sleeping forever.
265 - name: SLEEP
266 value: "false"
267 volumeMounts:
268 - mountPath: /host/opt/cni/bin
269 name: cni-bin-dir
270 - mountPath: /host/etc/cni/net.d
271 name: cni-net-dir
272 - mountPath: /calico-secrets
273 name: etcd-certs
274 securityContext:
275 privileged: true
276 # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes
277 # to communicate with Felix over the Policy Sync API.
278 - name: flexvol-driver
279 image: docker.io/calico/pod2daemon-flexvol:v3.21.0
280 volumeMounts:
281 - name: flexvol-driver-host
282 mountPath: /host/driver
283 securityContext:
284 privileged: true
285 containers:
286 # Runs calico-node container on each Kubernetes node. This
287 # container programs network policy and routes on each
288 # host.
289 - name: calico-node
290 image: docker.io/calico/node:v3.21.0
291 envFrom:
292 - configMapRef:
293 # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
294 name: kubernetes-services-endpoint
295 optional: true
296 env:
297 - name: KUBERNETES_SERVICE_HOST
298 value: "192.168.178.100"
299 - name: KUBERNETES_SERVICE_PORT
300 value: "6443"
301 - name: KUBERNETES_SERVICE_PORT_HTTPS
302 value: "6443"
303 # The location of the etcd cluster.
304 - name: ETCD_ENDPOINTS
305 valueFrom:
306 configMapKeyRef:
307 name: calico-config
308 key: etcd_endpoints
309 # Location of the CA certificate for etcd.
310 - name: ETCD_CA_CERT_FILE
311 valueFrom:
312 configMapKeyRef:
313 name: calico-config
314 key: etcd_ca
315 # Location of the client key for etcd.
316 - name: ETCD_KEY_FILE
317 valueFrom:
318 configMapKeyRef:
319 name: calico-config
320 key: etcd_key
321 # Location of the client certificate for etcd.
322 - name: ETCD_CERT_FILE
323 valueFrom:
324 configMapKeyRef:
325 name: calico-config
326 key: etcd_cert
327 # Set noderef for node controller.
328 - name: CALICO_K8S_NODE_REF
329 valueFrom:
330 fieldRef:
331 fieldPath: spec.nodeName
332 # Choose the backend to use.
333 - name: CALICO_NETWORKING_BACKEND
334 valueFrom:
335 configMapKeyRef:
336 name: calico-config
337 key: calico_backend
338 # Cluster type to identify the deployment type
339 - name: CLUSTER_TYPE
340 value: "k8s,bgp"
341 # Auto-detect the BGP IP address.
342 - name: IP
343 value: "autodetect"
344 # Enable IPIP
345 - name: CALICO_IPV4POOL_IPIP
346 value: "Always"
347 # Enable or Disable VXLAN on the default IP pool.
348 - name: CALICO_IPV4POOL_VXLAN
349 value: "Never"
350 # Set MTU for tunnel device used if ipip is enabled
351 - name: FELIX_IPINIPMTU
352 valueFrom:
353 configMapKeyRef:
354 name: calico-config
355 key: veth_mtu
356 # Set MTU for the VXLAN tunnel device.
357 - name: FELIX_VXLANMTU
358 valueFrom:
359 configMapKeyRef:
360 name: calico-config
361 key: veth_mtu
362 # Set MTU for the Wireguard tunnel device.
363 - name: FELIX_WIREGUARDMTU
364 valueFrom:
365 configMapKeyRef:
366 name: calico-config
367 key: veth_mtu
368 # The default IPv4 pool to create on startup if none exists. Pod IPs will be
369 # chosen from this range. Changing this value after installation will have
370 # no effect. This should fall within `--cluster-cidr`.
371 - name: CALICO_IPV4POOL_CIDR
372 value: "10.244.0.0/16"
373 # Disable file logging so `kubectl logs` works.
374 - name: CALICO_DISABLE_FILE_LOGGING
375 value: "true"
376 # Set Felix endpoint to host default action to ACCEPT.
377 - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
378 value: "ACCEPT"
379 # Disable IPv6 on Kubernetes.
380 - name: FELIX_IPV6SUPPORT
381 value: "false"
382 - name: FELIX_HEALTHENABLED
383 value: "true"
384 securityContext:
385 privileged: true
386 resources:
387 requests:
388 cpu: 250m
389 lifecycle:
390 preStop:
391 exec:
392 command:
393 - /bin/calico-node
394 - -shutdown
395 livenessProbe:
396 exec:
397 command:
398 - /bin/calico-node
399 - -felix-live
400 - -bird-live
401 periodSeconds: 10
402 initialDelaySeconds: 10
403 failureThreshold: 6
404 timeoutSeconds: 10
405 readinessProbe:
406 exec:
407 command:
408 - /bin/calico-node
409 - -felix-ready
410 - -bird-ready
411 periodSeconds: 10
412 timeoutSeconds: 10
413 volumeMounts:
414 # For maintaining CNI plugin API credentials.
415 - mountPath: /host/etc/cni/net.d
416 name: cni-net-dir
417 readOnly: false
418 - mountPath: /lib/modules
419 name: lib-modules
420 readOnly: true
421 - mountPath: /run/xtables.lock
422 name: xtables-lock
423 readOnly: false
424 - mountPath: /var/run/calico
425 name: var-run-calico
426 readOnly: false
427 - mountPath: /var/lib/calico
428 name: var-lib-calico
429 readOnly: false
430 - mountPath: /calico-secrets
431 name: etcd-certs
432 - name: policysync
433 mountPath: /var/run/nodeagent
434 # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the
435 # parent directory.
436 - name: sysfs
437 mountPath: /sys/fs/
438 # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host.
439 # If the host is known to mount that filesystem already then Bidirectional can be omitted.
440 mountPropagation: Bidirectional
441 - name: cni-log-dir
442 mountPath: /var/log/calico/cni
443 readOnly: true
444 volumes:
445 # Used by calico-node.
446 - name: lib-modules
447 hostPath:
448 path: /lib/modules
449 - name: var-run-calico
450 hostPath:
451 path: /var/run/calico
452 - name: var-lib-calico
453 hostPath:
454 path: /var/lib/calico
455 - name: xtables-lock
456 hostPath:
457 path: /run/xtables.lock
458 type: FileOrCreate
459 - name: sysfs
460 hostPath:
461 path: /sys/fs/
462 type: DirectoryOrCreate
463 # Used to install CNI.
464 - name: cni-bin-dir
465 hostPath:
466 path: /opt/cni/bin
467 - name: cni-net-dir
468 hostPath:
469 path: /etc/cni/net.d
470 # Used to access CNI logs.
471 - name: cni-log-dir
472 hostPath:
473 path: /var/log/calico/cni
474 # Mount in the etcd TLS secrets with mode 400.
475 # See https://kubernetes.io/docs/concepts/configuration/secret/
476 - name: etcd-certs
477 secret:
478 secretName: calico-etcd-secrets
479 defaultMode: 0400
480 # Used to create per-pod Unix Domain Sockets
481 - name: policysync
482 hostPath:
483 type: DirectoryOrCreate
484 path: /var/run/nodeagent
485 # Used to install Flex Volume Driver
486 - name: flexvol-driver-host
487 hostPath:
488 type: DirectoryOrCreate
489 path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds
490 ---
491
492 apiVersion: v1
493 kind: ServiceAccount
494 metadata:
495 name: calico-node
496 namespace: kube-system
497
498 ---
499 # Source: calico/templates/calico-kube-controllers.yaml
500 # See https://github.com/projectcalico/kube-controllers
501 apiVersion: apps/v1
502 kind: Deployment
503 metadata:
504 name: calico-kube-controllers
505 namespace: kube-system
506 labels:
507 k8s-app: calico-kube-controllers
508 spec:
509 # The controllers can only have a single active instance.
510 replicas: 1
511 selector:
512 matchLabels:
513 k8s-app: calico-kube-controllers
514 strategy:
515 type: Recreate
516 template:
517 metadata:
518 name: calico-kube-controllers
519 namespace: kube-system
520 labels:
521 k8s-app: calico-kube-controllers
522 spec:
523 nodeSelector:
524 kubernetes.io/os: linux
525 tolerations:
526 # Mark the pod as a critical add-on for rescheduling.
527 - key: CriticalAddonsOnly
528 operator: Exists
529 - key: node-role.kubernetes.io/master
530 effect: NoSchedule
531 serviceAccountName: calico-kube-controllers
532 priorityClassName: system-cluster-critical
533 # The controllers must run in the host network namespace so that
534 # it isn't governed by policy that would prevent it from working.
535 hostNetwork: true
536 containers:
537 - name: calico-kube-controllers
538 image: docker.io/calico/kube-controllers:v3.21.0
539 env:
540 # The location of the etcd cluster.
541 - name: ETCD_ENDPOINTS
542 valueFrom:
543 configMapKeyRef:
544 name: calico-config
545 key: etcd_endpoints
546 # Location of the CA certificate for etcd.
547 - name: ETCD_CA_CERT_FILE
548 valueFrom:
549 configMapKeyRef:
550 name: calico-config
551 key: etcd_ca
552 # Location of the client key for etcd.
553 - name: ETCD_KEY_FILE
554 valueFrom:
555 configMapKeyRef:
556 name: calico-config
557 key: etcd_key
558 # Location of the client certificate for etcd.
559 - name: ETCD_CERT_FILE
560 valueFrom:
561 configMapKeyRef:
562 name: calico-config
563 key: etcd_cert
564 # Choose which controllers to run.
565 - name: ENABLED_CONTROLLERS
566 value: policy,namespace,serviceaccount,workloadendpoint,node
567 volumeMounts:
568 # Mount in the etcd TLS secrets.
569 - mountPath: /calico-secrets
570 name: etcd-certs
571 livenessProbe:
572 exec:
573 command:
574 - /usr/bin/check-status
575 - -l
576 periodSeconds: 10
577 initialDelaySeconds: 10
578 failureThreshold: 6
579 timeoutSeconds: 10
580 readinessProbe:
581 exec:
582 command:
583 - /usr/bin/check-status
584 - -r
585 periodSeconds: 10
586 volumes:
587 # Mount in the etcd TLS secrets with mode 400.
588 # See https://kubernetes.io/docs/concepts/configuration/secret/
589 - name: etcd-certs
590 secret:
591 secretName: calico-etcd-secrets
592 defaultMode: 0440
593
594 ---
595
596 apiVersion: v1
597 kind: ServiceAccount
598 metadata:
599 name: calico-kube-controllers
600 namespace: kube-system
601
602 ---
603
604 # This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict
605
606 apiVersion: policy/v1beta1
607 kind: PodDisruptionBudget
608 metadata:
609 name: calico-kube-controllers
610 namespace: kube-system
611 labels:
612 k8s-app: calico-kube-controllers
613 spec:
614 maxUnavailable: 1
615 selector:
616 matchLabels:
617 k8s-app: calico-kube-controllers
618
619 ---
620 # Source: calico/templates/calico-typha.yaml
621
622 ---
623 # Source: calico/templates/configure-canal.yaml
624
625 ---
626 # Source: calico/templates/kdd-crds.yaml
627
628
root@k8s-master:~#
3. 修改配置¶
1. 注释掉文件的前22行¶
1 ---
2 # Source: calico/templates/calico-etcd-secrets.yaml
3 # The following contains k8s Secrets for use with a TLS enabled etcd cluster.
4 # For information on populating Secrets, see http://kubernetes.io/docs/user-guide/secrets/
5 #apiVersion: v1
6 #kind: Secret
7 #type: Opaque
8 #metadata:
9 # name: calico-etcd-secrets
10 # namespace: kube-system
11 #data:
12 # Populate the following with etcd TLS configuration if desired, but leave blank if
13 # not using TLS for etcd.
14 # The keys below should be uncommented and the values populated with the base64
15 # encoded contents of each file that would be associated with the TLS data.
16 # Example command for encoding a file contents: cat <file> | base64 -w 0
17 # etcd-key: null
18 # etcd-cert: null
19 # etcd-ca: null
20 ---
21 # Source: calico/templates/calico-config.yaml
22 # This ConfigMap is used to configure a self-hosted Calico installation.
23 kind: ConfigMap
24 apiVersion: v1
25 metadata:
26 name: calico-config
27 namespace: kube-system
...
2. 修改configmap¶
注意30-35行，其中etcd_endpoints换成环境的etcd地址
23 kind: ConfigMap
24 apiVersion: v1
25 metadata:
26 name: calico-config
27 namespace: kube-system
28 data:
29 # Configure this with the location of your etcd cluster.
30 etcd_endpoints: "https://10.0.24.4:2379"
31 # If you're using TLS enabled etcd uncomment the following.
32 # You must also populate the Secret below with these files.
33 etcd_ca: "/calico-secrets/etcd-ca" # "/calico-secrets/etcd-ca"
34 etcd_cert: "/calico-secrets/etcd-cert" # "/calico-secrets/etcd-cert"
35 etcd_key: "/calico-secrets/etcd-key" # "/calico-secrets/etcd-key"
36 # Typha is disabled.
37 typha_service_name: "none"
38 # Configure the backend to use.
39 calico_backend: "bird"
3. 添加calico-node环境变量¶
注意297-302行为新添加
285 containers:
286 # Runs calico-node container on each Kubernetes node. This
287 # container programs network policy and routes on each
288 # host.
289 - name: calico-node
290 image: docker.io/calico/node:v3.20.0
291 envFrom:
292 - configMapRef:
293 # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
294 name: kubernetes-services-endpoint
295 optional: true
296 env:
297 - name: KUBERNETES_SERVICE_HOST
298 value: "10.0.24.4"
299 - name: KUBERNETES_SERVICE_PORT
300 value: "6443"
301 - name: KUBERNETES_SERVICE_PORT_HTTPS
302 value: "6443"
303 # The location of the etcd cluster.
304 - name: ETCD_ENDPOINTS
305 valueFrom:
306 configMapKeyRef:
307 name: calico-config
308 key: etcd_endpoints
309 # Location of the CA certificate for etcd.
310 - name: ETCD_CA_CERT_FILE
4. 修改CIDR¶
注意371-372行，value值为k8s集群初始化的pod-network-cidr
370 # no effect. This should fall within `--cluster-cidr`.
371 - name: CALICO_IPV4POOL_CIDR
372 value: "10.244.0.0/16"
373 # Disable file logging so `kubectl logs` works.
374 - name: CALICO_DISABLE_FILE_LOGGING
375 value: "true"
5. 创建secret¶
kubectl -n kube-system create secret generic calico-etcd-secrets --from-file=etcd-ca=/etc/kubernetes/pki/etcd/ca.crt --from-file=etcd-cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt --from-file=etcd-key=/etc/kubernetes/pki/etcd/healthcheck-client.key
6. 创建calico资源清单¶
kubectl apply -f calico-etcd.yaml
7. 等待pod启动完成¶
kubectl -n kube-system get po
# kubectl -n kube-system get po -owide -w
NAME READY STATUS RESTARTS AGE
calico-kube-controllers-59db5cf8fd-fpzdq 1/1 Running 1 32m
calico-node-d2xq4 1/1 Running 1 32m
calico-node-ppzjk 1/1 Running 1 32m
7. 验证集群¶
操作节点： 在master节点（k8s-master）执行
kubectl get nodes #观察集群节点是否全部Ready
创建测试nginx服务
kubectl run test-nginx --image=nginx:alpine
如果查看master节点有污点，不让建pod，如下结尾处
root@k8s-master:~# kubectl get pod
NAME READY STATUS RESTARTS AGE
test-nginx 0/1 Pending 0 8s
root@k8s-master:~#
root@k8s-master:~# kubectl describe pod test-nginx
Name: test-nginx
Namespace: default
Priority: 0
Node: <none>
Labels: run=test-nginx
Annotations: <none>
Status: Pending
IP:
IPs: <none>
Containers:
test-nginx:
Image: nginx:alpine
Port: <none>
Host Port: <none>
Environment: <none>
Mounts:
/var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-l2srq (ro)
Conditions:
Type Status
PodScheduled False
Volumes:
kube-api-access-l2srq:
Type: Projected (a volume that contains injected data from multiple sources)
TokenExpirationSeconds: 3607
ConfigMapName: kube-root-ca.crt
ConfigMapOptional: <nil>
DownwardAPI: true
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Warning FailedScheduling 23s default-scheduler 0/1 nodes are available: 1 node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate.
Warning FailedScheduling 22s default-scheduler 0/1 nodes are available: 1 node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate.
则执行如下命令，去掉master节点的污点
#master
kubectl taint node k8s-master node-role.kubernetes.io/master:NoSchedule-
#疑问：如何查看节点是否有污点来着？？？
kubectl describe node |grep -i taint
root@k8s-master:~# kubectl describe node |grep -i taint
Taints: node-role.kubernetes.io/master:NoSchedule
root@k8s-master:~#
root@k8s-master:~# kubectl taint node k8s-master node-role.kubernetes.io/master:NoSchedule-
node/k8s-master untainted
root@k8s-master:~#
root@k8s-master:~# kubectl describe node |grep -i taint
Taints: <none>
root@k8s-master:~#
查看pod是否创建成功，并访问pod ip测试是否可用
$ kubectl get po -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
test-nginx-5bd8859b98-5nnnw 1/1 Running 0 9s 10.244.1.2 k8s-slave1 <none> <none>
$ curl 10.244.1.2
...
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
8. 清理环境¶
如果你的集群安装过程中遇到了其他问题，我们可以使用下面的命令来进行重置：
# 在全部集群节点执行
kubeadm reset
# 先执行上一步，然后执行下面的，最好一步一步来！
ifconfig cni0 down && ip link delete cni0
ifconfig flannel.1 down && ip link delete flannel.1
# 删除相关文件
rm -rf /run/calico
rm -rf /var/lib/cni/
mv /etc/kubernetes/* /tmp
mv /var/lib/etcd/* /tmp
mv ~/.kube/* /tmp
# 防火墙相关
iptables -F
iptables -t nat -F
ipvsadm -C
# 删除IP关联
ip link del kube-ipvs0
ip link del dummy0
最后更新:
2022-02-22 04:55:01