{"id":36022,"date":"2022-09-12T03:31:19","date_gmt":"2023-06-15T19:31:19","guid":{"rendered":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8kubeadm%e5%9c%a8gce%e4%b8%8a%e6%9e%84%e5%bb%ba%e4%b8%80%e4%b8%aa%e5%8c%85%e5%90%ab1%e4%b8%aa%e6%8e%a7%e5%88%b6%e5%b9%b3%e9%9d%a2%e5%92%8c3%e4%b8%aa%e5%b7%a5%e4%bd%9c%e8%8a%82\/"},"modified":"2024-01-15T08:45:20","modified_gmt":"2024-01-15T00:45:20","slug":"%e4%bd%bf%e7%94%a8kubeadm%e5%9c%a8gce%e4%b8%8a%e6%9e%84%e5%bb%ba%e4%b8%80%e4%b8%aa%e5%8c%85%e5%90%ab1%e4%b8%aa%e6%8e%a7%e5%88%b6%e5%b9%b3%e9%9d%a2%e5%92%8c3%e4%b8%aa%e5%b7%a5%e4%bd%9c%e8%8a%82","status":"publish","type":"post","link":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8kubeadm%e5%9c%a8gce%e4%b8%8a%e6%9e%84%e5%bb%ba%e4%b8%80%e4%b8%aa%e5%8c%85%e5%90%ab1%e4%b8%aa%e6%8e%a7%e5%88%b6%e5%b9%b3%e9%9d%a2%e5%92%8c3%e4%b8%aa%e5%b7%a5%e4%bd%9c%e8%8a%82\/","title":{"rendered":"\u4f7f\u7528kubeadm\u5728GCE\u4e0a\u6784\u5efa\u4e00\u4e2a\u5305\u542b1\u4e2a\u63a7\u5236\u5e73\u9762\u548c3\u4e2a\u5de5\u4f5c\u8282\u70b9\u7684Kubernetes\u96c6\u7fa4"},"content":{"rendered":"

\u51c6\u5907\u4e00\u4e2aGCE\u5b9e\u4f8b\u4f5c\u4e3a\u63a7\u5236\u5e73\u9762\uff0c\u4e09\u4e2aGCE\u5b9e\u4f8b\u4f5c\u4e3a\u5de5\u4f5c\u8282\u70b9\uff0c\u7136\u540e\u4f7f\u7528kubeadm\u6784\u5efaKubernetes\u96c6\u7fa4\u3002
\n\u540c\u65f6\uff0c\u9009\u62e9containerd\u4f5c\u4e3a\u5bb9\u5668\u8fd0\u884c\u65f6\u3002<\/p>\n

\u5728\u672c\u5730\u673a\u5668\u4e0a\u4f7f\u7528\u4e86 fish shell\uff0c\u5728 SSH \u5230 GCE \u5b9e\u4f8b\u4e0a\u4f7f\u7528\u4e86 bash shell\u3002\u4e3a\u4e86\u907f\u514d\u6df7\u6dc6\uff0c\u5728\u4ee3\u7801\u5757\u7684\u6807\u9898\u4e2d\u6807\u660e\u4e86\u4f7f\u7528\u7684shell\u540d\u79f0\u3002<\/p>\n

\u6240\u6709\u7248\u672c<\/p>\n

$<\/span> cat<\/span> \/etc\/os-release\r\nNAME=\"Ubuntu\"\r\nVERSION=\"20.04.2 LTS (Focal Fossa)\"\r\nID=ubuntu\r\nID_LIKE=debian\r\nPRETTY_NAME=\"Ubuntu 20.04.2 LTS\"\r\nVERSION_ID=\"20.04\"\r\nHOME_URL=\"https:\/\/www.ubuntu.com\/\"\r\nSUPPORT_URL=\"https:\/\/help.ubuntu.com\/\"\r\nBUG_REPORT_URL=\"https:\/\/bugs.launchpad.net\/ubuntu\/\"\r\nPRIVACY_POLICY_URL=\"https:\/\/www.ubuntu.com\/legal\/terms-and-policies\/privacy-policy\"\r\nVERSION_CODENAME=focal\r\nUBUNTU_CODENAME=focal\r\n<\/span>$<\/span> kubeadm version\r\nkubeadm version: &version.Info{Major:\"1\", Minor:\"20\", GitVersion:\"v1.20.4\", GitCommit:\"e87da0bd6e03ec3fea7933c4b5263d151aafd07c\", GitTreeState:\"clean\", BuildDate:\"2021-02-18T16:09:38Z\", GoVersion:\"go1.15.8\", Compiler:\"gc\", Platform:\"linux\/amd64\"}\r\n<\/span>$<\/span> containerd -v<\/span>\r\ncontainerd github.com\/containerd\/containerd 1.3.3-0ubuntu2.2\r\n<\/span>$<\/span> kubelet --version<\/span>\r\nKubernetes v1.20.4\r\n<\/span>$<\/span> kubectl version --client<\/span>\r\nClient Version: version.Info{Major:\"1\", Minor:\"20\", GitVersion:\"v1.20.4\", GitCommit:\"e87da0bd6e03ec3fea7933c4b5263d151aafd07c\", GitTreeState:\"clean\", BuildDate:\"2021-02-18T16:12:00Z\", GoVersion:\"go1.15.8\", Compiler:\"gc\", Platform:\"linux\/amd64\"}\r\n<\/span><\/code><\/pre>\n
$<\/span> k get po -A<\/span> -o<\/span> jsonpath<\/span>=<\/span>\"{..image}\"<\/span> --kubeconfig<\/span> .\/admin.conf |\\<\/span>\r\n  tr<\/span> -s<\/span> '[[:space:]]'<\/span> '\\n'<\/span> |\\<\/span>\r\n  sort<\/span> |\\<\/span>\r\n  uniq<\/span> -c<\/span>\r\n  16 docker.io\/calico\/cni:v3.18.0\r\n   2 docker.io\/calico\/kube-controllers:v3.18.0\r\n   8 docker.io\/calico\/node:v3.18.0\r\n   8 docker.io\/calico\/pod2daemon-flexvol:v3.18.0\r\n   3 docker.io\/library\/nginx:latest\r\n   4 k8s.gcr.io\/coredns:1.7.0\r\n   2 k8s.gcr.io\/etcd:3.4.13-0\r\n   2 k8s.gcr.io\/kube-apiserver:v1.20.4\r\n   2 k8s.gcr.io\/kube-controller-manager:v1.20.4\r\n   8 k8s.gcr.io\/kube-proxy:v1.20.4\r\n   2 k8s.gcr.io\/kube-scheduler:v1.20.4\r\n<\/span><\/code><\/pre>\n
\u521b\u5efa\u8ba1\u7b97\u8d44\u6e90<\/h1>\n
\u6211\u4eec\u5c06\u53c2\u8003\u300aKubernetes the Hard Way\u300b\u4e2d\u7684\u201cProvisioning Compute Resources\u201d\u548cCalico\u5728Google Compute Engine\uff08GCE\uff09\u4e0a\u7684\u201cSelf-managed Kubernetes\u201d\u6765\u8fdb\u884c\u64cd\u4f5c\u3002<\/p>\n
\u521b\u5efaVPC<\/h2>\n
\u521b\u5efa\u4e00\u4e2aVPC\u6765\u521b\u5efa\u96c6\u7fa4\u3002<\/p>\n
$<\/span> gcloud compute networks create kubernetes-by-kubeadm --subnet-mode<\/span> custom\r\nCreated [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/global\/networks\/kubernetes-by-kubeadm].\r\nNAME                   SUBNET_MODE  BGP_ROUTING_MODE  IPV4_RANGE  GATEWAY_IPV4\r\nkubernetes-by-kubeadm  CUSTOM       REGIONAL\r\n<\/span><\/code><\/pre>\n
$<\/span> gcloud compute networks subnets create kubernetes \\<\/span>\r\n  --network<\/span> kubernetes-by-kubeadm \\<\/span>\r\n  --range<\/span> 10.240.0.0\/24\r\nCreated [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/regions\/asia-northeast1\/subnetworks\/kubernetes].\r\nNAME        REGION           NETWORK                RANGE\r\nkubernetes  asia-northeast1  kubernetes-by-kubeadm  10.240.0.0\/24\r\n<\/span><\/code><\/pre>\n
\u521b\u5efa\u9632\u706b\u5899\u89c4\u5219<\/h2>\n
\u96c6\u7fa4\u5185\u90e8\u5141\u8bb8 tcp\u3001udp\u3001icmp\u3001ipip \u7684\u901a\u4fe1\uff0c\u800c\u4ece\u96c6\u7fa4\u5916\u90e8\u5141\u8bb8 tcp:22\u3001tcp:6443\u3001icmp \u7684\u901a\u4fe1\u3002<\/p>\n
$<\/span> gcloud compute firewall-rules create kubernetes-by-kubeadm-allow-internal \\<\/span>\r\n  --allow<\/span> tcp,udp,icmp,ipip \\<\/span>\r\n  --network<\/span> kubernetes-by-kubeadm \\<\/span>\r\n  --source-ranges<\/span> 10.240.0.0\/24,10.200.0.0\/16\r\nCreating firewall...\u2839Created [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/global\/firewalls\/kubernetes-by-kubeadm-allow-internal].\r\nCreating firewall...done.\r\nNAME                                  NETWORK                DIRECTION  PRIORITY  ALLOW              DENY  DISABLED\r\nkubernetes-by-kubeadm-allow-internal  kubernetes-by-kubeadm  INGRESS    1000      tcp,udp,icmp,ipip        False\r\n<\/span>$<\/span> gcloud compute firewall-rules create kubernetes-by-kubeadm-allow-external \\<\/span>\r\n  --allow<\/span> tcp:22,tcp:6443,icmp \\<\/span>\r\n  --network<\/span> kubernetes-by-kubeadm \\<\/span>\r\n  --source-ranges<\/span> 0.0.0.0\/0\r\nCreating firewall...\u2839Created [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/global\/firewalls\/kubernetes-by-kubeadm-allow-external].\r\nCreating firewall...done.\r\nNAME                                  NETWORK                DIRECTION  PRIORITY  ALLOW                         DENY  DISABLED\r\nkubernetes-by-kubeadm-allow-external  kubernetes-by-kubeadm  INGRESS    1000      tcp:22,tcp:443,tcp:6443,icmp        False\r\n<\/span><\/code><\/pre>\n
\u5206\u914d\u516c\u5171 IP<\/h2>\n
\u7531\u65bc\u9019\u6b21\u53ea\u6709\u4e00\u500b\u63a7\u5236\u5e73\u9762\uff0c\u6211\u5011\u53ef\u4ee5\u5c07\u70ba\u63a7\u5236\u5e73\u9762\u6e96\u5099\u7684\u5be6\u4f8b\u7684\u5916\u90e8IP\u76f4\u63a5\u7528\u4f5cAPI\u7d42\u7aef\u9ede\uff0c\u4f46\u8003\u616e\u5230\u5be6\u73fe\u9ad8\u53ef\u7528\u67b6\u69cb\u4e26\u9032\u884c\u8ca0\u8f09\u5e73\u8861\uff0c\u6211\u5011\u61c9\u8a72\u9810\u5148\u5206\u914d\u4e00\u500b\u516c\u6709IP\u5730\u5740\u3002<\/p>\n
$<\/span> gcloud compute addresses create kubernetes-by-kubeadm \\<\/span>\r\n  --region<\/span> (<\/span>gcloud config get-value compute\/region)<\/span>\r\nCreated [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/regions\/asia-northeast1\/addresses\/kubernetes-by-kubeadm].\r\n<\/span>$<\/span> gcloud compute addresses list\r\nNAME                   ADDRESS\/RANGE  TYPE      PURPOSE  NETWORK  REGION           SUBNET  STATUS\r\nkubernetes-by-kubeadm  34.85.15.20    EXTERNAL                    asia-northeast1          RESERVED\r\n<\/span><\/code><\/pre>\n
\u521b\u5efa\u8ba1\u7b97\u5b9e\u4f8b<\/h2>\n
\u9996\u5148\u662f\u7528\u4e8e\u63a7\u5236\u5e73\u9762\u7684\u5b9e\u4f8b\u3002<\/p>\n
$<\/span> gcloud compute instances create controller \\<\/span>\r\n  --async<\/span> \\<\/span>\r\n  --boot-disk-size<\/span> 200GB \\<\/span>\r\n  --can-ip-forward<\/span> \\<\/span>\r\n  --image-family<\/span> ubuntu-2004-lts \\<\/span>\r\n  --image-project<\/span> ubuntu-os-cloud \\<\/span>\r\n  --machine-type<\/span> n1-standard-4 \\<\/span>\r\n  --private-network-ip<\/span> 10.240.0.10 \\<\/span>\r\n  --scopes<\/span> compute-rw,storage-ro,service-management,service-control,logging-write,monitoring \\<\/span>\r\n  --subnet<\/span> kubernetes \\<\/span>\r\n  --tags<\/span> kubernetes-by-kubeadm,controller\r\nNOTE: The users will be charged for public IPs when VMs are created.\r\nInstance creation in progress for [controller]: https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/zones\/asia-northeast1-a\/operations\/operation-1613193837534-5bb30f5a3dc98-ba859467-b3c39485\r\nUse [gcloud compute operations describe URI] command to check the status of the operation(s).\r\n<\/span><\/code><\/pre>\n
\u63a5\u4e0b\u6765\uff0c\u521b\u5efa\u4e09\u4e2a\u7528\u4e8e\u5de5\u4f5c\u8282\u70b9\u7684\u5b9e\u4f8b\u3002<\/p>\n
$<\/span> for <\/span>i in <\/span>0 1 2\r\n    gcloud compute instances create worker-{$<\/span>i}<\/span> \\<\/span>\r\n      --async<\/span> \\<\/span>\r\n      --boot-disk-size<\/span> 200GB \\<\/span>\r\n      --can-ip-forward<\/span> \\<\/span>\r\n      --image-family<\/span> ubuntu-2004-lts \\<\/span>\r\n      --image-project<\/span> ubuntu-os-cloud \\<\/span>\r\n      --machine-type<\/span> n1-standard-4 \\<\/span>\r\n      --private-network-ip<\/span> 10.240.0.2{<\/span>$i<\/span>}<\/span> \\<\/span>\r\n      --scopes<\/span> compute-rw,storage-ro,service-management,service-control,logging-write,monitoring \\<\/span>\r\n      --subnet<\/span> kubernetes \\<\/span>\r\n      --tags<\/span> kubernetes-by-kubeadm,worker\r\n  end\r\nNOTE: The users will be charged for public IPs when VMs are created.\r\nInstance creation in progress for [worker-0]: https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/zones\/asia-northeast1-a\/operations\/operation-1613193959993-5bb30fcf070d9-aa954478-17a67d14\r\nUse [gcloud compute operations describe URI] command to check the status of the operation(s).\r\nNOTE: The users will be charged for public IPs when VMs are created.\r\nInstance creation in progress for [worker-1]: https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/zones\/asia-northeast1-a\/operations\/operation-1613193964271-5bb30fd31b636-5bc6d372-19d8209d\r\nUse [gcloud compute operations describe URI] command to check the status of the operation(s).\r\nNOTE: The users will be charged for public IPs when VMs are created.\r\nInstance creation in progress for [worker-2]: https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/zones\/asia-northeast1-a\/operations\/operation-1613193968163-5bb30fd6d1b22-133c16bc-8135db2a\r\nUse [gcloud compute operations describe URI] command to check the status of the operation(s).\r\n<\/span><\/code><\/pre>\n
\u5916\u90e8\u8d1f\u8f7d\u5747\u8861\u5668\u7684\u914d\u7f6e<\/h2>\n
\u5728\u8fd9\u4e2a\u65f6\u5019\uff0c\u6211\u4eec\u4f1a\u5bf9\u521a\u624d\u5206\u914d\u7684\u516c\u5171IP\u5730\u5740\u8fdb\u884c\u8d1f\u8f7d\u5747\u8861\uff08LB\uff09\u7684\u8bbe\u7f6e\u3002
\n\u6211\u4eec\u4f1a\u8fdb\u884c\u5065\u5eb7\u68c0\u67e5\u7684\u8bbe\u7f6e\uff0c\u5e76\u4e3a\u76ee\u6807\u6c60\u8bbe\u7f6e\u7528\u4e8e\u63a7\u5236\u5e73\u9762\u7684\u5b9e\u4f8b\u3002<\/p>\n
$<\/span> set <\/span>KUBERNETES_PUBLIC_ADDRESS (<\/span>gcloud compute addresses describe kubernetes-by-kubeadm \\<\/span>\r\n    --region<\/span> (<\/span>gcloud config get-value compute\/region)<\/span> \\<\/span>\r\n    --format<\/span> 'value(address)'<\/span>)<\/span>\r\n\r\n$<\/span> gcloud compute http-health-checks create kubernetes \\<\/span>\r\n    --description<\/span> \"Kubernetes Health Check\"<\/span> \\<\/span>\r\n    --host<\/span> \"kubernetes.default.svc.cluster.local\"<\/span> \\<\/span>\r\n    --request-path<\/span> \"\/healthz\"<\/span>\r\nCreated [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/global\/httpHealthChecks\/kubernetes].\r\nNAME        HOST                                  PORT  REQUEST_PATH\r\nkubernetes  kubernetes.default.svc.cluster.local  80    \/healthz\r\n\r\n<\/span>$<\/span> gcloud compute firewall-rules create kubernetes-by-kubeadm-allow-health-check \\<\/span>\r\n    --network<\/span> kubernetes-by-kubeadm \\<\/span>\r\n    --source-ranges<\/span> 209.85.152.0\/22,209.85.204.0\/22,35.191.0.0\/16 \\<\/span>\r\n    --allow<\/span> tcp\r\nCreating firewall...\u2839Created [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/global\/firewalls\/kubernetes-by-kubeadm-allow-health-check].\r\nCreating firewall...done.\r\nNAME                                      NETWORK                DIRECTION  PRIORITY  ALLOW  DENY  DISABLED\r\nkubernetes-by-kubeadm-allow-health-check  kubernetes-by-kubeadm  INGRESS    1000      tcp          False\r\n\r\n<\/span>$<\/span> gcloud compute target-pools create kubernetes-target-pool \\<\/span>\r\n    --http-health-check<\/span> kubernetes\r\nCreated [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/regions\/asia-northeast1\/targetPools\/kubernetes-target-pool].\r\nNAME                    REGION           SESSION_AFFINITY  BACKUP  HEALTH_CHECKS\r\nkubernetes-target-pool  asia-northeast1  NONE                      kubernetes\r\n\r\n<\/span>$<\/span> gcloud compute target-pools add-instances kubernetes-target-pool \\<\/span>\r\n    --instances<\/span> controller\r\nUpdated [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/regions\/asia-northeast1\/targetPools\/kubernetes-target-pool].\r\n\r\n<\/span>$<\/span> gcloud compute forwarding-rules create kubernetes-forwarding-rule \\<\/span>\r\n    --address<\/span> $KUBERNETES_PUBLIC_ADDRESS<\/span> \\<\/span>\r\n    --ports<\/span> 6443 \\<\/span>\r\n    --region<\/span> (<\/span>gcloud config get-value compute\/region)<\/span> \\<\/span>\r\n    --target-pool<\/span> kubernetes-target-pool\r\nCreated [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/regions\/asia-northeast1\/forwardingRules\/kubernetes-forwarding-rule].\r\n<\/span><\/code><\/pre>\n
\u6bcf\u4e2a\u5b9e\u4f8b\u7684\u51c6\u5907\u5de5\u4f5c<\/h1>\n
\u6211\u4eec\u5c06\u6309\u7167\u300a\u516c\u5f0f\u6587\u6863\u7684Installing kubeadm\u300b\u8fdb\u884c\u53c2\u8003\u548c\u64cd\u4f5c\u3002<\/p>\n
\u786e\u8ba4\u76ee\u6807\u5b9e\u4f8b<\/h2>\n
$<\/span> gcloud compute instances list --filter<\/span>=<\/span>\"tags.items=kubernetes-by-kubeadm\"<\/span>\r\nNAME        ZONE               MACHINE_TYPE   PREEMPTIBLE  INTERNAL_IP  EXTERNAL_IP     STATUS\r\ncontroller  asia-northeast1-a  n1-standard-4               10.240.0.10  35.221.119.152  RUNNING\r\nworker-0    asia-northeast1-a  n1-standard-4               10.240.0.20  35.221.99.135   RUNNING\r\nworker-1    asia-northeast1-a  n1-standard-4               10.240.0.21  34.84.119.161   RUNNING\r\nworker-2    asia-northeast1-a  n1-standard-4               10.240.0.22  34.85.61.122    RUNNING\r\n<\/span><\/code><\/pre>\n
\u8bf7\u8fde\u63a5\u5230\u6bcf\u4e2a\u5b9e\u4f8b\u5e76\u6267\u884c gcloud compute ssh $INSTANCE_NAME \u547d\u4ee4\uff0c\u7136\u540e\u8fdb\u884c\u5404\u79cd\u51c6\u5907\u5de5\u4f5c\u3002
\n\u518d\u6b21\u63d0\u9192\uff0c\u5b9e\u4f8b\u5185\u7684shell\u662fbash\u3002<\/p>\n
\u4f7fiptables\u80fd\u591f\u5904\u7406\u901a\u8fc7\u6865\u63a5\u7684\u6d41\u91cf<\/h2>\n
hitsumabushi845@controller:~$<\/span> sudo <\/span>modprobe br_netfilter\r\nhitsumabushi845@controller:~$<\/span> lsmod | grep <\/span>br_netfilter\r\nbr_netfilter           28672  0\r\nbridge                176128  1 br_netfilter\r\n<\/span>hitsumabushi845@controller:~$<\/span> cat<\/span> <<<\/span>EOF<\/span> | sudo tee \/etc\/sysctl.d\/k8s.conf\r\n<\/span>><\/span> net.bridge.bridge-nf-call-ip6tables = 1\r\n<\/span>><\/span> net.bridge.bridge-nf-call-iptables = 1\r\n<\/span>><\/span> EOF\r\n<\/span>net.bridge.bridge-nf-call-ip6tables = 1\r\nnet.bridge.bridge-nf-call-iptables = 1\r\n<\/span>hitsumabushi845@controller:~$<\/span> sudo <\/span>sysctl --system<\/span>\r\n* Applying \/etc\/sysctl.d\/10-console-messages.conf ...\r\nkernel.printk = 4 4 1 7\r\n* Applying \/etc\/sysctl.d\/10-ipv6-privacy.conf ...\r\nnet.ipv6.conf.all.use_tempaddr = 2\r\nnet.ipv6.conf.default.use_tempaddr = 2\r\n* Applying \/etc\/sysctl.d\/10-kernel-hardening.conf ...\r\nkernel.kptr_restrict = 1\r\n* Applying \/etc\/sysctl.d\/10-link-restrictions.conf ...\r\nfs.protected_hardlinks = 1\r\nfs.protected_symlinks = 1\r\n* Applying \/etc\/sysctl.d\/10-magic-sysrq.conf ...\r\nkernel.sysrq = 176\r\n* Applying \/etc\/sysctl.d\/10-network-security.conf ...\r\nnet.ipv4.conf.default.rp_filter = 2\r\nnet.ipv4.conf.all.rp_filter = 2\r\n* Applying \/etc\/sysctl.d\/10-ptrace.conf ...\r\nkernel.yama.ptrace_scope = 1\r\n* Applying \/etc\/sysctl.d\/10-zeropage.conf ...\r\nvm.mmap_min_addr = 65536\r\n* Applying \/usr\/lib\/sysctl.d\/50-default.conf ...\r\nnet.ipv4.conf.default.promote_secondaries = 1\r\nsysctl: setting key \"net.ipv4.conf.all.promote_secondaries\": Invalid argument\r\nnet.ipv4.ping_group_range = 0 2147483647\r\nnet.core.default_qdisc = fq_codel\r\nfs.protected_regular = 1\r\nfs.protected_fifos = 1\r\n* Applying \/usr\/lib\/sysctl.d\/50-pid-max.conf ...\r\nkernel.pid_max = 4194304\r\n* Applying \/etc\/sysctl.d\/60-gce-network-security.conf ...\r\nnet.ipv4.tcp_syncookies = 1\r\nnet.ipv4.conf.all.accept_source_route = 0\r\nnet.ipv4.conf.default.accept_source_route = 0\r\nnet.ipv4.conf.all.accept_redirects = 0\r\nnet.ipv4.conf.default.accept_redirects = 0\r\nnet.ipv4.conf.all.secure_redirects = 1\r\nnet.ipv4.conf.default.secure_redirects = 1\r\nnet.ipv4.ip_forward = 0\r\nnet.ipv4.conf.all.send_redirects = 0\r\nnet.ipv4.conf.default.send_redirects = 0\r\nnet.ipv4.conf.all.rp_filter = 1\r\nnet.ipv4.conf.default.rp_filter = 1\r\nnet.ipv4.icmp_echo_ignore_broadcasts = 1\r\nnet.ipv4.icmp_ignore_bogus_error_responses = 1\r\nnet.ipv4.conf.all.log_martians = 1\r\nnet.ipv4.conf.default.log_martians = 1\r\nkernel.randomize_va_space = 2\r\nkernel.panic = 10\r\n* Applying \/etc\/sysctl.d\/99-cloudimg-ipv6.conf ...\r\nnet.ipv6.conf.all.use_tempaddr = 0\r\nnet.ipv6.conf.default.use_tempaddr = 0\r\n* Applying \/etc\/sysctl.d\/99-sysctl.conf ...\r\n* Applying \/etc\/sysctl.d\/k8s.conf ...\r\nnet.bridge.bridge-nf-call-ip6tables = 1\r\nnet.bridge.bridge-nf-call-iptables = 1\r\n* Applying \/usr\/lib\/sysctl.d\/protect-links.conf ...\r\nfs.protected_fifos = 1\r\nfs.protected_hardlinks = 1\r\nfs.protected_regular = 2\r\nfs.protected_symlinks = 1\r\n* Applying \/etc\/sysctl.conf ...\r\n<\/span><\/code><\/pre>\n
\u5b89\u88c5 containerd<\/h2>\n
\u6211\u4eec\u5c06\u6309\u7167Container\u8fd0\u884c\u65f6\u4e2dcontainerd\u76f8\u5173\u90e8\u5206\u7684\u5185\u5bb9\u6765\u8fdb\u884c\u3002<\/p>\n
hitsumabushi845@controller:~$<\/span> cat<\/span> <<<\/span>EOF<\/span> | sudo tee \/etc\/modules-load.d\/containerd.conf\r\n<\/span>><\/span> overlay\r\n<\/span>><\/span> br_netfilter\r\n<\/span>><\/span> EOF\r\n<\/span>overlay\r\nbr_netfilter\r\n<\/span>hitsumabushi845@controller:~$<\/span> sudo <\/span>modprobe overlay\r\nhitsumabushi845@controller:~$<\/span> sudo <\/span>modprobe br_netfilter\r\nhitsumabushi845@controller:~$<\/span> cat<\/span> <<<\/span>EOF<\/span> | sudo tee \/etc\/sysctl.d\/99-kubernetes-cri.conf\r\n<\/span>><\/span> net.bridge.bridge-nf-call-iptables  = 1\r\n<\/span>><\/span> net.ipv4.ip_forward                 = 1\r\n<\/span>><\/span> net.bridge.bridge-nf-call-ip6tables = 1\r\n<\/span>><\/span> EOF\r\n<\/span>net.bridge.bridge-nf-call-iptables  = 1\r\nnet.ipv4.ip_forward                 = 1\r\nnet.bridge.bridge-nf-call-ip6tables = 1\r\n<\/span>hitsumabushi845@controller:~$<\/span> sudo <\/span>sysctl --system<\/span>\r\n(\u7565)\r\n<\/span>hitsumabushi845@controller:~$<\/span> sudo <\/span>apt-get update &&<\/span> sudo <\/span>apt-get install<\/span> -y<\/span> containerd\r\n(\u7565)\r\n<\/span>hitsumabushi845@controller:~$<\/span> sudo mkdir<\/span> -p<\/span> \/etc\/containerd\r\nhitsumabushi845@controller:~$<\/span> containerd config default | sudo tee<\/span> \/etc\/containerd\/config.toml\r\n(\u7565)\r\n<\/span>hitsumabushi845@controller:~$<\/span> sudo <\/span>systemctl restart containerd\r\nhitsumabushi845@controller:~$<\/span> sudo <\/span>systemctl status containerd\r\n\u25cf containerd.service - containerd container runtime\r\n<\/span>     Loaded: loaded (\/lib\/systemd\/system\/containerd.service;<\/span> enabled;<\/span> vendor preset: enabled)<\/span>\r\n     Active: active (running) since Sun 2021-02-21 10:51:28 UTC;<\/span> 10s ago\r\n       Docs: https:\/\/containerd.io\r\n    Process: 14555 ExecStartPre=\/sbin\/modprobe overlay (code=exited, status=0\/SUCCESS)\r\n   Main PID: 14562 (containerd)\r\n      Tasks: 14\r\n     Memory: 21.3M\r\n     CGroup: \/system.slice\/containerd.service\r\n             \u2514\u250014562 \/usr\/bin\/containerd\r\n\r\nFeb 21 10:51:28 controller containerd[14562]: time=\"2021-02-21T10:51:28.302856785Z\" level=error msg=\"Failed to load cni during init, please check CRI plugin status before setting up network for pods\" error=\"cni config load failed: no network config found in \/etc\/cni\/net.d: cni plugin not initialized: failed to load cni config\"\r\nFeb 21 10:51:28 controller containerd[14562]: time=\"2021-02-21T10:51:28.303184344Z\" level=info msg=\"loading plugin \\\"io.containerd.grpc.v1.introspection\\\"...\" type=io.containerd.grpc.v1\r\nFeb 21 10:51:28 controller containerd[14562]: time=\"2021-02-21T10:51:28.303712626Z\" level=info msg=\"Start subscribing containerd event\"\r\nFeb 21 10:51:28 controller containerd[14562]: time=\"2021-02-21T10:51:28.303792746Z\" level=info msg=\"Start recovering state\"\r\nFeb 21 10:51:28 controller containerd[14562]: time=\"2021-02-21T10:51:28.303902734Z\" level=info msg=serving... address=\/run\/containerd\/containerd.sock.ttrpc\r\nFeb 21 10:51:28 controller containerd[14562]: time=\"2021-02-21T10:51:28.303999145Z\" level=info msg=\"Start event monitor\"\r\nFeb 21 10:51:28 controller containerd[14562]: time=\"2021-02-21T10:51:28.304029295Z\" level=info msg=serving... address=\/run\/containerd\/containerd.sock\r\nFeb 21 10:51:28 controller containerd[14562]: time=\"2021-02-21T10:51:28.304054635Z\" level=info msg=\"containerd successfully booted in 0.042356s\"\r\nFeb 21 10:51:28 controller containerd[14562]: time=\"2021-02-21T10:51:28.304030429Z\" level=info msg=\"Start snapshots syncer\"\r\nFeb 21 10:51:28 controller containerd[14562]: time=\"2021-02-21T10:51:28.304557561Z\" level=info msg=\"Start streaming server\"\r\n<\/span><\/code><\/pre>\n
kubeadm\u3001kubelet\u3001kubectl \u7684\u5b89\u88c5<\/h2>\n
\u8fd4\u56de\u5230\u516c\u5f0f\u6587\u6863\u7684\u201c\u5b89\u88c5 kubeadm\u201d\u90e8\u5206\u3002<\/p>\n
hitsumabushi845@controller:~$<\/span> sudo <\/span>apt-get update &&<\/span> sudo <\/span>apt-get install<\/span> -y<\/span> apt-transport-https curl\r\n(\u7565)\r\n<\/span>hitsumabushi845@controller:~$<\/span> curl -s<\/span> https:\/\/packages.cloud.google.com\/apt\/doc\/apt-key.gpg | sudo <\/span>apt-key add -\r\nOK\r\n<\/span>hitsumabushi845@controller:~$<\/span> cat<\/span> <<<\/span>EOF<\/span> | sudo tee \/etc\/apt\/sources.list.d\/kubernetes.list\r\n<\/span>><\/span> deb https:\/\/apt.kubernetes.io\/ kubernetes-xenial main\r\n<\/span>><\/span> EOF\r\n<\/span>deb https:\/\/apt.kubernetes.io\/ kubernetes-xenial main\r\n<\/span>hitsumabushi845@controller:~$<\/span> sudo <\/span>apt-get update\r\n(\u7565)\r\n<\/span>hitsumabushi845@controller:~$<\/span> sudo <\/span>apt-get install<\/span> -y<\/span> kubelet kubeadm kubectl\r\n(\u7565)\r\n<\/span>hitsumabushi845@controller:~$<\/span> sudo <\/span>apt-mark hold kubelet kubeadm kubectl\r\nkubelet set on hold.\r\nkubeadm set on hold.\r\nkubectl set on hold.\r\n<\/span><\/code><\/pre>\n
\u8fd9\u4e9b\u4efb\u52a1\u5c06\u5728\u63a7\u5236\u5668\u3001\u5de5\u4f5c\u673a-0\u3001\u5de5\u4f5c\u673a-1\u548c\u5de5\u4f5c\u673a-2\u7684\u5404\u4e2a\u5b9e\u4f8b\u4e0a\u6267\u884c\u3002<\/p>\n
\u521b\u5efa\u63a7\u5236\u5e73\u9762<\/h1>\n
\u4ece\u8fd9\u91cc\u5f00\u59cb\uff0c\u6309\u7167\u4f7f\u7528kubeadm\u521b\u5efa\u96c6\u7fa4\u7684\u6b65\u9aa4\u8fdb\u884c\u3002<\/p>\n
\u6267\u884c kubeadm init\u3002<\/h2>\n
\u8fd9\u6b21\u6211\u4eec\u5c06\u5206\u914d\u5916\u90e8IP\u4f5c\u4e3aAPI\u670d\u52a1\u5668\u7684\u7ec8\u7ed3\u70b9\uff0c\u56e0\u6b64\u5728–control-plane-endpoint\u4e2d\u6307\u5b9a\u5916\u90e8IP\u3002\u6b64\u5916\uff0cPod CIDR\u88ab\u8bbe\u7f6e\u4e3a10.200.0.0\/16\u3002<\/p>\n
hitsumabushi845@controller:~$<\/span> sudo <\/span>kubeadm init --control-plane-endpoint<\/span>=<\/span>34.85.15.20 --pod-network-cidr<\/span>=<\/span>10.200.0.0\/16\r\n[init] Using Kubernetes version: v1.20.4\r\n[preflight] Running pre-flight checks\r\n[preflight] Pulling images required for setting up a Kubernetes cluster\r\n[preflight] This might take a minute or two, depending on the speed of your internet connection\r\n[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'\r\n[certs] Using certificateDir folder \"\/etc\/kubernetes\/pki\"\r\n[certs] Generating \"ca\" certificate and key\r\n[certs] Generating \"apiserver\" certificate and key\r\n[certs] apiserver serving cert is signed for DNS names [controller kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 10.240.0.10 34.85.15.20]\r\n[certs] Generating \"apiserver-kubelet-client\" certificate and key\r\n[certs] Generating \"front-proxy-ca\" certificate and key\r\n[certs] Generating \"front-proxy-client\" certificate and key\r\n[certs] Generating \"etcd\/ca\" certificate and key\r\n[certs] Generating \"etcd\/server\" certificate and key\r\n[certs] etcd\/server serving cert is signed for DNS names [controller localhost] and IPs [10.240.0.10 127.0.0.1 ::1]\r\n[certs] Generating \"etcd\/peer\" certificate and key\r\n[certs] etcd\/peer serving cert is signed for DNS names [controller localhost] and IPs [10.240.0.10 127.0.0.1 ::1]\r\n[certs] Generating \"etcd\/healthcheck-client\" certificate and key\r\n[certs] Generating \"apiserver-etcd-client\" certificate and key\r\n[certs] Generating \"sa\" key and public key\r\n[kubeconfig] Using kubeconfig folder \"\/etc\/kubernetes\"\r\n[kubeconfig] Writing \"admin.conf\" kubeconfig file\r\n[kubeconfig] Writing \"kubelet.conf\" kubeconfig file\r\n[kubeconfig] Writing \"controller-manager.conf\" kubeconfig file\r\n[kubeconfig] Writing \"scheduler.conf\" kubeconfig file\r\n[kubelet-start] Writing kubelet environment file with flags to file \"\/var\/lib\/kubelet\/kubeadm-flags.env\"\r\n[kubelet-start] Writing kubelet configuration to file \"\/var\/lib\/kubelet\/config.yaml\"\r\n[kubelet-start] Starting the kubelet\r\n[control-plane] Using manifest folder \"\/etc\/kubernetes\/manifests\"\r\n[control-plane] Creating static Pod manifest for \"kube-apiserver\"\r\n[control-plane] Creating static Pod manifest for \"kube-controller-manager\"\r\n[control-plane] Creating static Pod manifest for \"kube-scheduler\"\r\n[etcd] Creating static Pod manifest for local etcd in \"\/etc\/kubernetes\/manifests\"\r\n[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory \"\/etc\/kubernetes\/manifests\". This can take up to 4m0s\r\n[kubelet-check] Initial timeout of 40s passed.\r\n[apiclient] All control plane components are healthy after 104.006101 seconds\r\n[upload-config] Storing the configuration used in ConfigMap \"kubeadm-config\" in the \"kube-system\" Namespace\r\n[kubelet] Creating a ConfigMap \"kubelet-config-1.20\" in namespace kube-system with the configuration for the kubelets in the cluster\r\n[upload-certs] Skipping phase. Please see --upload-certs\r\n[mark-control-plane] Marking the node controller as control-plane by adding the labels \"node-role.kubernetes.io\/master=''\" and \"node-role.kubernetes.io\/control-plane='' (deprecated)\"\r\n[mark-control-plane] Marking the node controller as control-plane by adding the taints [node-role.kubernetes.io\/master:NoSchedule]\r\n[bootstrap-token] Using token: 3cznxo.v1ax148y0hjdzail\r\n[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles\r\n[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to get nodes\r\n[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials\r\n[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token\r\n[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster\r\n[bootstrap-token] Creating the \"cluster-info\" ConfigMap in the \"kube-public\" namespace\r\n[kubelet-finalize] Updating \"\/etc\/kubernetes\/kubelet.conf\" to point to a rotatable kubelet client certificate and key\r\n[addons] Applied essential addon: CoreDNS\r\n[addons] Applied essential addon: kube-proxy\r\n\r\nYour Kubernetes control-plane has initialized successfully!\r\n\r\nTo start using your cluster, you need to run the following as a regular user:\r\n\r\n<\/span>  mkdir -p $<\/span>HOME\/.kube\r\n  sudo cp -i \/etc\/kubernetes\/admin.conf $<\/span>HOME\/.kube\/config\r\n  sudo chown $<\/span>(<\/span>id<\/span> -u<\/span>)<\/span>:$(<\/span>id<\/span> -g<\/span>)<\/span> $HOME<\/span>\/.kube\/config\r\n\r\nAlternatively, if you are the root user, you can run:\r\n\r\n  export KUBECONFIG=\/etc\/kubernetes\/admin.conf\r\n\r\nYou should now deploy a pod network to the cluster.\r\nRun \"kubectl apply -f [podnetwork].yaml\" with one of the options listed at:\r\n  https:\/\/kubernetes.io\/docs\/concepts\/cluster-administration\/addons\/\r\n\r\nYou can now join any number of control-plane nodes by copying certificate authorities\r\nand service account keys on each node and then running the following as root:\r\n\r\n  kubeadm join 34.85.15.20:6443 --token 3cznxo.v1ax148y0hjdzail \\\r\n    --discovery-token-ca-cert-hash sha256:d778f85f07c092a196b77e1669dfceed74b9092587293274fcc8652a9936511f \\\r\n    --control-plane\r\n\r\nThen you can join any number of worker nodes by running the following on each as root:\r\n\r\nkubeadm join 34.85.15.20:6443 --token 3cznxo.v1ax148y0hjdzail \\\r\n    --discovery-token-ca-cert-hash sha256:d778f85f07c092a196b77e1669dfceed74b9092587293274fcc8652a9936511f\r\n<\/span><\/code><\/pre>\n
\u5f53\u6210\u529f\u521d\u59cb\u5316\u540e\uff0c\u5728\u5e95\u90e8\u4f1a\u663e\u793a\u51fa\u5404\u79cd\u547d\u4ee4\u3002
\n\u5982\u679c\u4e0d\u662f\u4ee5\u8d85\u7ea7\u7528\u6237\u8eab\u4efd\u4f7f\u7528kubectl\u547d\u4ee4\uff0c\u5219\u5728\u4e0b\u65b9\u6572\u51fb\u3002<\/p>\n
  mkdir<\/span> -p<\/span> $HOME<\/span>\/.kube\r\n  sudo cp<\/span> -i<\/span> \/etc\/kubernetes\/admin.conf $HOME<\/span>\/.kube\/config\r\n  sudo chown<\/span> $(<\/span>id<\/span> -u<\/span>)<\/span>:$(<\/span>id<\/span> -g<\/span>)<\/span> $HOME<\/span>\/.kube\/config\r\n<\/code><\/pre>\n
\u6267\u884c\u3002<\/p>\n
\u5982\u679c\u8981\u6dfb\u52a0\u63a7\u5236\u5e73\u9762\u8282\u70b9\uff0c\u53ef\u4ee5\u8fd9\u6837\u505a<\/p>\n
  kubeadm join <\/span>34.85.15.20:6443 --token<\/span> 3cznxo.v1ax148y0hjdzail \\<\/span>\r\n    --discovery-token-ca-cert-hash<\/span> sha256:d778f85f07c092a196b77e1669dfceed74b9092587293274fcc8652a9936511f \\<\/span>\r\n    --control-plane<\/span>\r\n<\/code><\/pre>\n
\u6267\u884c\u3002<\/p>\n
\u5982\u679c\u8981\u6dfb\u52a0\u5de5\u4f5c\u8282\u70b9\uff0c<\/p>\n
kubeadm join <\/span>34.85.15.20:6443 --token<\/span> 3cznxo.v1ax148y0hjdzail \\<\/span>\r\n    --discovery-token-ca-cert-hash<\/span> sha256:d778f85f07c092a196b77e1669dfceed74b9092587293274fcc8652a9936511f\r\n<\/code><\/pre>\n
\u4fdd\u7559\u6267\u884c\u7279\u5b9a\u547d\u4ee4\uff0c\u5c24\u5176\u662f\u5728\u540e\u7eed\u5de5\u4f5c\u4e2d\u9700\u8981\u6dfb\u52a0\u5de5\u4f5c\u8282\u70b9\u7684\u547d\u4ee4\u3002<\/p>\n
\u8bf7\u786e\u8ba4 kubectl \u7684\u6267\u884c\u60c5\u51b5\u3002<\/h2>\n
\u4f7f\u7528\u5148\u524d\u63d0\u5230\u7684\u547d\u4ee4\uff0c\u786e\u4fdd\u63a7\u5236\u5668\u5b9e\u4f8b\u53ef\u4ee5\u4ee5\u666e\u901a\u7528\u6237\u8eab\u4efd\u6267\u884c kubectl\u3002<\/p>\n
hitsumabushi845@controller:~$<\/span> mkdir<\/span> -p<\/span> $HOME<\/span>\/.kube\r\nhitsumabushi845@controller:~$<\/span> sudo cp<\/span> -i<\/span> \/etc\/kubernetes\/admin.conf $HOME<\/span>\/.kube\/config\r\nhitsumabushi845@controller:~$<\/span> sudo chown<\/span> $(<\/span>id<\/span> -u<\/span>)<\/span>:$(<\/span>id<\/span> -g<\/span>)<\/span> $HOME<\/span>\/.kube\/config\r\nhitsumabushi845@controller:~$<\/span> kubectl version\r\nClient Version: version.Info{Major:\"1\", Minor:\"20\", GitVersion:\"v1.20.4\", GitCommit:\"e87da0bd6e03ec3fea7933c4b5263d151aafd07c\", GitTreeState:\"clean\", BuildDate:\"2021-02-18T16:12:00Z\", GoVersion:\"go1.15.8\", Compiler:\"gc\", Platform:\"linux\/amd64\"}\r\nServer Version: version.Info{Major:\"1\", Minor:\"20\", GitVersion:\"v1.20.4\", GitCommit:\"e87da0bd6e03ec3fea7933c4b5263d151aafd07c\", GitTreeState:\"clean\", BuildDate:\"2021-02-18T16:03:00Z\", GoVersion:\"go1.15.8\", Compiler:\"gc\", Platform:\"linux\/amd64\"}\r\n<\/span><\/code><\/pre>\n
\u786e\u8ba4\u662f\u5426\u901a\u8fc7\u4f7f\u7528 “kubectl get nodes” \u547d\u4ee4\u83b7\u5f97\u8fd4\u56de\u7ed3\u679c\u3002<\/p>\n
hitsumabushi845@controller:~$<\/span> kubectl get nodes\r\nNAME         STATUS     ROLES                  AGE   VERSION\r\ncontroller   NotReady   control-plane,master   47m   v1.20.4\r\n<\/span><\/code><\/pre>\n
\u63a7\u5236\u5668\u663e\u793a\u4e3a\u8282\u70b9\uff0c\u4f46\u7531\u4e8e\u88ab\u6807\u8bb0\u4e3a NoSchedule \u7684 Taint\uff0c\u901a\u5e38\u4e0d\u4f1a\u5c06 Pod \u8c03\u5ea6\u5230\u8fd9\u4e2a\u8282\u70b9\u3002<\/p>\n
hitsumabushi845@controller:~$<\/span> kubectl describe node controller | grep <\/span>Taint\r\nTaints:             node-role.kubernetes.io\/master:NoSchedule\r\n<\/span><\/code><\/pre>\n
\u5b89\u88c5Calico\u3002<\/h2>\n
\u4e0b\u8f7d calico.yaml \u6587\u4ef6\u3002<\/p>\n
hitsumabushi845@controller:~$<\/span> sudo <\/span>curl -OL<\/span> https:\/\/docs.projectcalico.org\/manifests\/calico.yaml\r\n  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\r\n                                 Dload  Upload   Total   Spent    Left  Speed\r\n100 20847  100 20847    0     0  50112      0 --:--:-- --:--:-- --:--:-- 50233\r\n<\/span><\/code><\/pre>\n
\u7531\u4e8e\u5728\u6267\u884c kubeadm init \u65f6\u6307\u5b9a\u4e86 –pod-network-cidr\uff0c\u6240\u4ee5\u9700\u8981\u4fee\u6539 calico.yaml \u4e2d CALICO_IPV4POOL_CIDR \u7684\u503c\u3002
\n\u53e6\u5916\uff0c\u4e3a\u4e86\u4fdd\u9669\u8d77\u89c1\uff0c\u660e\u786e\u5c06 FELIX_IPTABLESBACKEND \u7684\u503c\u8bbe\u7f6e\u4e3a NFT\uff08\u9ed8\u8ba4\u4e3a Auto\uff09\u3002<\/p>\n
-            # - name: CALICO_IPV4POOL_CIDR\r\n-            #   value: \"192.168.0.0\/16\"\r\n<\/span>+            - name: CALICO_IPV4POOL_CIDR\r\n+              value: \"10.200.0.0\/16\"\r\n+            - name: FELIX_IPTABLESBACKEND\r\n+              value: NFT\r\n<\/span><\/code><\/pre>\n
hitsumabushi845@controller:~$<\/span> kubectl apply -f<\/span> calico.yaml\r\nconfigmap\/calico-config created\r\n<\/span>Warning: apiextensions.k8s.io\/v1beta1 CustomResourceDefinition is deprecated in v1.16+, unavailable in v1.22+;<\/span> use apiextensions.k8s.io\/v1 CustomResourceDefinition\r\ncustomresourcedefinition.apiextensions.k8s.io\/felixconfigurations.crd.projectcalico.org created\r\ncustomresourcedefinition.apiextensions.k8s.io\/ipamblocks.crd.projectcalico.org created\r\ncustomresourcedefinition.apiextensions.k8s.io\/blockaffinities.crd.projectcalico.org created\r\ncustomresourcedefinition.apiextensions.k8s.io\/ipamhandles.crd.projectcalico.org created\r\ncustomresourcedefinition.apiextensions.k8s.io\/ipamconfigs.crd.projectcalico.org created\r\ncustomresourcedefinition.apiextensions.k8s.io\/bgppeers.crd.projectcalico.org created\r\ncustomresourcedefinition.apiextensions.k8s.io\/bgpconfigurations.crd.projectcalico.org created\r\ncustomresourcedefinition.apiextensions.k8s.io\/ippools.crd.projectcalico.org created\r\ncustomresourcedefinition.apiextensions.k8s.io\/hostendpoints.crd.projectcalico.org created\r\ncustomresourcedefinition.apiextensions.k8s.io\/clusterinformations.crd.projectcalico.org created\r\ncustomresourcedefinition.apiextensions.k8s.io\/globalnetworkpolicies.crd.projectcalico.org created\r\ncustomresourcedefinition.apiextensions.k8s.io\/globalnetworksets.crd.projectcalico.org created\r\ncustomresourcedefinition.apiextensions.k8s.io\/networkpolicies.crd.projectcalico.org created\r\ncustomresourcedefinition.apiextensions.k8s.io\/networksets.crd.projectcalico.org created\r\nclusterrole.rbac.authorization.k8s.io\/calico-kube-controllers created\r\nclusterrolebinding.rbac.authorization.k8s.io\/calico-kube-controllers created\r\nclusterrole.rbac.authorization.k8s.io\/calico-node created\r\nclusterrolebinding.rbac.authorization.k8s.io\/calico-node created\r\ndaemonset.apps\/calico-node created\r\nserviceaccount\/calico-node created\r\ndeployment.apps\/calico-kube-controllers created\r\nserviceaccount\/calico-kube-controllers created\r\n<\/span>hitsumabushi845@controller:~$<\/span> kubectl get pods -A<\/span>\r\nNAMESPACE     NAME                                       READY   STATUS    RESTARTS   AGE\r\nkube-system   calico-kube-controllers-6b8f6f78dc-k84wt   1\/1     Running   0          5m\r\nkube-system   calico-node-4jrkc                          1\/1     Running   0          5m\r\nkube-system   coredns-74ff55c5b-5ctxr                    1\/1     Running   0          52m\r\nkube-system   coredns-74ff55c5b-95lxc                    1\/1     Running   0          52m\r\nkube-system   etcd-controller                            1\/1     Running   0          52m\r\nkube-system   kube-apiserver-controller                  1\/1     Running   1          52m\r\nkube-system   kube-controller-manager-controller         1\/1     Running   0          52m\r\nkube-system   kube-proxy-2sgv7                           1\/1     Running   0          52m\r\nkube-system   kube-scheduler-controller                  1\/1     Running   0          52m\r\n<\/span>hitsumabushi845@controller:~$<\/span> kubectl get nodes\r\nNAME         STATUS   ROLES                  AGE   VERSION\r\ncontroller   Ready    control-plane,master   53m   v1.20.4\r\n<\/span><\/code><\/pre>\n
\u521b\u5efaWorker\u8282\u70b9<\/h1>\n
\u7531\u4e8e\u76ee\u524d\u4e3a\u6b62\u7684\u5de5\u4f5c\u5df2\u7ecf\u5b8c\u6210\u4e86\u63a7\u5236\u5e73\u9762\u7684\u521b\u5efa\uff0c\u6240\u4ee5\u63a5\u4e0b\u6765\u6211\u4eec\u5c06\u6dfb\u52a0\u5de5\u4f5c\u8282\u70b9\u3002<\/p>\n
\u6267\u884c kubeadm join<\/h2>\n
\u5728\u5de5\u4f5c\u8282\u70b9\u7684\u5b9e\u4f8b\u4e0a\u6267\u884c\u4e0a\u8ff0\u7684kubeadm join\u547d\u4ee4\u3002<\/p>\n
hitsumabushi845@worker-0:~$<\/span> sudo <\/span>kubeadm join <\/span>34.85.15.20:6443 --token<\/span> 3cznxo.v1ax148y0hjdzail     --discovery-token-ca-cert-hash<\/span> sha256:d778f85f07c092a196b77e1669dfceed74b9092587293274fcc8652a9936511f\r\n[preflight] Running pre-flight checks\r\n[preflight] Reading configuration from the cluster...\r\n[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'\r\n[kubelet-start] Writing kubelet configuration to file \"\/var\/lib\/kubelet\/config.yaml\"\r\n[kubelet-start] Writing kubelet environment file with flags to file \"\/var\/lib\/kubelet\/kubeadm-flags.env\"\r\n[kubelet-start] Starting the kubelet\r\n[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...\r\n\r\nThis node has joined the cluster:\r\n* Certificate signing request was sent to apiserver and a response was received.\r\n* The Kubelet was informed of the new secure connection details.\r\n\r\nRun 'kubectl get nodes' on the control-plane to see this node join the cluster.\r\n<\/span><\/code><\/pre>\n
\u786e\u5b9a<\/h2>\n
\u5728\u63a7\u5236\u5e73\u9762\u5b9e\u4f8b\u4e0a\u6267\u884c kubectl get nodes \u547d\u4ee4\u3002<\/p>\n
hitsumabushi845@controller:~$<\/span> kubectl get nodes\r\nNAME         STATUS   ROLES                  AGE   VERSION\r\ncontroller   Ready    control-plane,master   57m   v1.20.4\r\n<\/span>worker-0     Ready    <none><\/span>                 73s   v1.20.4\r\n<\/code><\/pre>\n
\u6211\u5df2\u7ecf\u786e\u8ba4\u6dfb\u52a0\u4e86\u5de5\u4f5c\u8282\u70b9\u3002
\n\u540c\u6837\u5730\uff0c\u6211\u4eec\u8fd8\u9700\u8981\u5728worker-1\u548cworker-2\u7684\u5b9e\u4f8b\u4e0a\u8fdb\u884c\u64cd\u4f5c\uff0c\u4ee5\u786e\u4fdd\u67093\u4e2a\u5de5\u4f5c\u8282\u70b9\u5b58\u5728\u3002<\/p>\n
hitsumabushi845@controller:~$<\/span> kubectl get nodes\r\nNAME         STATUS   ROLES                  AGE     VERSION\r\ncontroller   Ready    control-plane,master   72m     v1.20.4\r\n<\/span>worker-0     Ready    <none><\/span>                 16m     v1.20.4\r\nworker-1     Ready    <none><\/span>                 4m49s   v1.20.4\r\nworker-2     Ready    <none><\/span>                 26s     v1.20.4\r\n<\/code><\/pre>\n
\u8ba9\u672c\u5730\u80fd\u591f\u4f7f\u7528kubectl\u547d\u4ee4\u884c\u5de5\u5177<\/h1>\n
\u4ece\u63a7\u5236\u5e73\u9762\u5b9e\u4f8b\u4e2d\u590d\u5236kubeconfig\u6587\u4ef6\u3002<\/p>\n
$<\/span> gcloud compute scp root@controller:\/etc\/kubernetes\/admin.conf .<\/span>\r\nadmin.conf\r\n<\/span><\/code><\/pre>\n
\u786e\u8ba4\u4f7f\u7528 –kubeconfig \u9009\u9879\u6307\u5b9a admin.conf \u7684\u526f\u672c\u540e\uff0ckubectl \u80fd\u591f\u6267\u884c\u547d\u4ee4\u3002
\n\u5982\u679c\u6bcf\u6b21\u6307\u5b9a –kubeconfig \u5f88\u9ebb\u70e6\uff0c\u53ef\u4ee5\u8bbe\u7f6e $KUBECONFIG \u73af\u5883\u53d8\u91cf\u6765\u7b80\u5316\u64cd\u4f5c\u3002<\/p>\n
$<\/span> k get nodes --kubeconfig<\/span> .\/admin.conf\r\nNAME         STATUS   ROLES                  AGE     VERSION\r\ncontroller   Ready    control-plane,master   74m     v1.20.4\r\n<\/span>worker-0     Ready    <none><\/span>                 18m     v1.20.4\r\nworker-1     Ready    <none><\/span>                 6m56s   v1.20.4\r\nworker-2     Ready    <none><\/span>                 2m33s   v1.20.4\r\n<\/code><\/pre>\n
\u786e\u8ba4\u64cd\u4f5c<\/h1>\n
\u5728\u8fd9\u4e2a\u9636\u6bb5\uff0c\u6211\u4eec\u5df2\u7ecf\u6210\u529f\u521b\u5efa\u4e86\u4e00\u4e2a\u62e5\u67091\u4e2a\u63a7\u5236\u5e73\u9762\u8282\u70b9\u548c3\u4e2a\u5de5\u4f5c\u8282\u70b9\u7684\u96c6\u7fa4\u3002\u6211\u4eec\u5c06\u53c2\u8003kubernetes-the-hard-way\u4e2d\u7684Smoke Test\u6765\u9a8c\u8bc1\u96c6\u7fa4\u7684\u8fd0\u884c\u60c5\u51b5\u3002<\/p>\n
\u521b\u5efa\u90e8\u7f72<\/h2>\n
\u786e\u8ba4\u80fd\u591f\u521b\u5efa\u90e8\u7f72\u8d44\u6e90\u3002<\/p>\n
$<\/span> k create deploy nginx --image<\/span>=<\/span>nginx --replicas<\/span>=<\/span>3 --kubeconfig<\/span> .\/admin.conf\r\ndeployment.apps\/nginx created\r\n<\/span>$<\/span> k get deploy --kubeconfig<\/span> .\/admin.conf\r\nNAME    READY   UP-TO-DATE   AVAILABLE   AGE\r\nnginx   3\/3     3            3           47s\r\n<\/span>$<\/span> k get po -owide<\/span> --kubeconfig<\/span> .\/admin.conf\r\nNAME                     READY   STATUS    RESTARTS   AGE   IP               NODE       NOMINATED NODE   READINESS GATES\r\n<\/span>nginx-6799fc88d8-69bjt   1\/1     Running   0          29m   10.200.43.2      worker-0   <none><\/span>           <none>\r\nnginx-6799fc88d8-8gdqj   1\/1     Running   0          29m   10.200.133.194   worker-2   <none><\/span>           <none>\r\nnginx-6799fc88d8-d92bc   1\/1     Running   0          29m   10.200.226.66    worker-1   <none><\/span>           <none>\r\n<\/code><\/pre>\n
\u786e\u8ba4\u7aef\u53e3\u8f6c\u53d1<\/h2>\n
\u6211\u8981\u5c1d\u8bd5\u5bf9\u5df2\u521b\u5efa\u7684nginx Pod\u8fdb\u884c\u7aef\u53e3\u8f6c\u53d1\u3002<\/p>\n
$<\/span> set <\/span>POD_NAME (<\/span>k get po -l<\/span> app<\/span>=<\/span>nginx -o<\/span> jsonpath<\/span>=<\/span>\"{.items[0].metadata.name}\"<\/span> --kubeconfig<\/span> .\/admin.conf)<\/span>\r\n$<\/span> echo<\/span> $POD_NAME<\/span>\r\nnginx-6799fc88d8-4ttf9\r\n<\/span>$<\/span> k port-forward $POD_NAME<\/span> 8080:80 --kubeconfig<\/span> .\/admin.conf\r\nForwarding from 127.0.0.1:8080 -><\/span> 80\r\nForwarding from [::1]:8080 -><\/span> 80\r\n<\/code><\/pre>\n
\u4ece\u53e6\u4e00\u4e2a\u7ec8\u7aef\u7a97\u53e3\u4e2d\uff0c\u786e\u8ba4\u53ef\u4ee5\u5bf9\u8fdb\u884c\u7aef\u53e3\u8f6c\u53d1\u7684 Pod \u4f7f\u7528 curl \u547d\u4ee4\u3002<\/p>\n
$<\/span> curl --head<\/span> http:\/\/127.0.0.1:8080\r\nHTTP\/1.1 200 OK\r\nServer: nginx\/1.19.7\r\nDate: Sun, 21 Feb 2021 13:46:42 GMT\r\nContent-Type: text\/html\r\nContent-Length: 612\r\nLast-Modified: Tue, 16 Feb 2021 15:57:18 GMT\r\nConnection: keep-alive\r\nETag: \"602beb5e-264\"\r\nAccept-Ranges: bytes\r\n<\/span><\/code><\/pre>\n
\u4e3a\u4e86\u786e\u8ba4\uff0c\u6211\u4f1a\u67e5\u770b\u76ee\u6807 Pod \u7684\u65e5\u5fd7\u3002
\n\u5728\u65e5\u5fd7\u7684\u6700\u5e95\u90e8\u53ef\u4ee5\u770b\u5230\u521a\u624d\u7684 curl \u8bf7\u6c42\u7684\u65e5\u5fd7\u3002<\/p>\n
$<\/span> k logs $POD_NAME<\/span> --kubeconfig<\/span> .\/admin.conf\r\n\/docker-entrypoint.sh: \/docker-entrypoint.d\/ is not empty, will attempt to perform configuration\r\n\/docker-entrypoint.sh: Looking for shell scripts in \/docker-entrypoint.d\/\r\n\/docker-entrypoint.sh: Launching \/docker-entrypoint.d\/10-listen-on-ipv6-by-default.sh\r\n10-listen-on-ipv6-by-default.sh: info: Getting the checksum of \/etc\/nginx\/conf.d\/default.conf\r\n10-listen-on-ipv6-by-default.sh: info: Enabled listen on IPv6 in \/etc\/nginx\/conf.d\/default.conf\r\n\/docker-entrypoint.sh: Launching \/docker-entrypoint.d\/20-envsubst-on-templates.sh\r\n\/docker-entrypoint.sh: Launching \/docker-entrypoint.d\/30-tune-worker-processes.sh\r\n<\/span>\/docker-entrypoint.sh: Configuration complete;<\/span> ready for <\/span>start up\r\n127.0.0.1 - - [21\/Feb\/2021:13:46:42 +0000] \"HEAD \/ HTTP\/1.1\" 200 0 \"-\" \"curl\/7.64.1\" \"-\"\r\n<\/span><\/code><\/pre>\n
\u4f7f\u7528NodePort\u670d\u52a1\u5c06\u90e8\u7f72\u516c\u5f00\u5230\u5916\u90e8<\/h2>\n
\u6211\u5011\u5c07\u4f7f\u7528NodePort\u670d\u52d9\u4f86\u6e2c\u8a66\u5148\u524d\u5275\u5efa\u7684\u90e8\u7f72\u7684\u5916\u90e8\u516c\u958b\u3002<\/p>\n
$<\/span> k expose deploy nginx --port<\/span> 80 --type<\/span> NodePort --kubeconfig<\/span>  .\/admin.conf\r\nservice\/nginx exposed\r\n<\/span>$<\/span> set <\/span>NODE_PORT (<\/span>k get svc nginx --output<\/span>=<\/span>jsonpath<\/span>=<\/span>'{range .spec.ports[0]}{.nodePort}'<\/span> --kubeconfig<\/span> .\/admin.conf)<\/span>\r\n$<\/span> echo<\/span> $NODE_PORT<\/span>\r\n31120\r\n<\/span><\/code><\/pre>\n
\u5728\u7b2c31120\u53f7\u7aef\u53e3\u4e0a\u90e8\u7f72\u5df2\u7ecf\u53d1\u5e03\uff0c\u4f46\u662f\u7531\u4e8eVPC\u7684\u9632\u706b\u5899\u89c4\u5219\u5c1a\u672a\u5141\u8bb8\u5bf9\u8be5\u7aef\u53e3\u7684\u5916\u90e8\u901a\u4fe1\uff0c\u56e0\u6b64\u9700\u8981\u6dfb\u52a0\u89c4\u5219\u3002<\/p>\n
$<\/span> gcloud compute firewall-rules create kubernetes-by-kubeadm-allow-nginx-service \\<\/span>\r\n  --allow<\/span>=<\/span>tcp:{<\/span>$NODE_PORT<\/span>}<\/span> \\<\/span>\r\n  --network<\/span> kubernetes-by-kubeadm\r\nCreating firewall...\u2839Created [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/global\/firewalls\/kubernetes-by-kubeadm-allow-nginx-service].\r\nCreating firewall...done.\r\nNAME                                       NETWORK                DIRECTION  PRIORITY  ALLOW      DENY  DISABLED\r\nkubernetes-by-kubeadm-allow-nginx-service  kubernetes-by-kubeadm  INGRESS    1000      tcp:31120        False\r\n<\/span><\/code><\/pre>\n
\u6211\u5011\u65b0\u589e\u4e86\u4e00\u689d\u898f\u5247\uff0c\u4ee5\u78ba\u4fdd\u820731120\u865f\u7aef\u53e3\u7684\u5916\u90e8\u9023\u901a\u6027\uff0c\u63a5\u4e0b\u4f86\u8981\u78ba\u8a8dcurl\u662f\u5426\u53ef\u4f7f\u7528\u3002<\/p>\n
$<\/span> set <\/span>EXTERNAL_IP (<\/span>gcloud compute instances describe worker-0 --format<\/span> 'value(networkInterfaces[0].accessConfigs[0].natIP)'<\/span>)<\/span>\r\n$<\/span> curl -I<\/span> http:\/\/{<\/span>$EXTERNAL_IP<\/span>}<\/span>:{<\/span>$NODE_PORT<\/span>}<\/span>\r\nHTTP\/1.1 200 OK\r\nServer: nginx\/1.19.7\r\nDate: Sun, 21 Feb 2021 13:54:45 GMT\r\nContent-Type: text\/html\r\nContent-Length: 612\r\nLast-Modified: Tue, 16 Feb 2021 15:57:18 GMT\r\nConnection: keep-alive\r\nETag: \"602beb5e-264\"\r\nAccept-Ranges: bytes\r\n<\/span><\/code><\/pre>\n
\u6211\u786e\u8ba4\u901a\u8fc7\u4f7f\u7528NodePort\u670d\u52a1\u53ef\u4ee5\u5c06Deployment\u5916\u90e8\u516c\u5f00\u3002<\/p>\n
\u6574\u7406\u6536\u62fe<\/h1>\n
\u56e0\u6b64\uff0c\u6211\u4eec\u80fd\u5728GCE\u4e0a\u6784\u5efa\u4e86\u4e00\u4e2ak8s\u96c6\u7fa4\u3002
\n\u6700\u540e\u6211\u4eec\u5c06\u8fdb\u884c\u6e05\u7406\u5de5\u4f5c\uff0c\u5220\u9664GCP\u4e0a\u7684\u5404\u79cd\u8d44\u6e90\u3002<\/p>\n
\u5220\u9664 GCE \u5b9e\u4f8b<\/h2>\n
$<\/span> gcloud -q<\/span> compute instances delete \\<\/span>\r\n  controller worker-0 worker-1 worker-2 \\<\/span>\r\n  --zone<\/span> (<\/span>gcloud config get-value compute\/zone)<\/span>\r\nDeleted [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/zones\/asia-northeast1-a\/instances\/controller].\r\nDeleted [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/zones\/asia-northeast1-a\/instances\/worker-0].\r\nDeleted [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/zones\/asia-northeast1-a\/instances\/worker-1].\r\nDeleted [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/zones\/asia-northeast1-a\/instances\/worker-2].\r\n<\/span><\/code><\/pre>\n
\u5220\u9664\u7f51\u7edc\u8d44\u6e90<\/h2>\n
\u5220\u9664\u5916\u90e8\u8d1f\u8f7d\u5747\u8861\u5668<\/h3>\n
$<\/span> gcloud -q<\/span> compute forwarding-rules list\r\nNAME                        REGION           IP_ADDRESS   IP_PROTOCOL  TARGET\r\nkubernetes-forwarding-rule  asia-northeast1  34.85.15.20  TCP          asia-northeast1\/targetPools\/kubernetes-target-pool\r\n<\/span>$<\/span> gcloud -q<\/span> compute forwarding-rules delete kubernetes-forwarding-rule \\<\/span>\r\n  --region<\/span> (<\/span>gcloud config get-value compute\/region)<\/span>\r\nDeleted [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/regions\/asia-northeast1\/forwardingRules\/kubernetes-forwarding-rule].\r\n<\/span>$<\/span> gcloud -q<\/span> compute forwarding-rules list\r\nListed 0 items.\r\n\r\n<\/span>$<\/span> gcloud -q<\/span> compute target-pools list\r\nNAME                    REGION           SESSION_AFFINITY  BACKUP  HEALTH_CHECKS\r\nkubernetes-target-pool  asia-northeast1  NONE                      kubernetes\r\n<\/span>$<\/span> gcloud -q<\/span> compute target-pools delete kubernetes-target-pool\r\nDeleted [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/regions\/asia-northeast1\/targetPools\/kubernetes-target-pool].\r\n<\/span>$<\/span> gcloud -q<\/span> compute target-pools list\r\nListed 0 items.\r\n\r\n<\/span>$<\/span> gcloud -q<\/span> compute http-health-checks list\r\nNAME        HOST                                  PORT  REQUEST_PATH\r\nkubernetes  kubernetes.default.svc.cluster.local  80    \/healthz\r\n<\/span>$<\/span> gcloud -q<\/span> compute http-health-checks delete kubernetes\r\nDeleted [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/global\/httpHealthChecks\/kubernetes].\r\n<\/span>$<\/span> gcloud -q<\/span> compute http-health-checks list\r\nListed 0 items.\r\n\r\n<\/span>$<\/span> gcloud -q<\/span> compute addresses list\r\nNAME                   ADDRESS\/RANGE  TYPE      PURPOSE  NETWORK  REGION           SUBNET  STATUS\r\nkubernetes-by-kubeadm  34.85.15.20    EXTERNAL                    asia-northeast1          RESERVED\r\n<\/span>$<\/span> gcloud -q<\/span> compute addresses delete kubernetes-by-kubeadm\r\nDeleted [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/regions\/asia-northeast1\/addresses\/kubernetes-by-kubeadm].\r\n<\/span>$<\/span> gcloud -q<\/span> compute addresses list\r\nListed 0 items.\r\n<\/span><\/code><\/pre>\n
\u5220\u9664\u9632\u706b\u5899\u89c4\u5219<\/h3>\n
$<\/span> gcloud -q<\/span> compute firewall-rules list\r\nNAME                                       NETWORK                DIRECTION  PRIORITY  ALLOW                         DENY  DISABLED\r\ndefault-allow-icmp                         default                INGRESS    65534     icmp                                False\r\ndefault-allow-internal                     default                INGRESS    65534     tcp:0-65535,udp:0-65535,icmp        False\r\ndefault-allow-rdp                          default                INGRESS    65534     tcp:3389                            False\r\ndefault-allow-ssh                          default                INGRESS    65534     tcp:22                              False\r\nkubernetes-by-kubeadm-allow-external       kubernetes-by-kubeadm  INGRESS    1000      tcp:22,tcp:6443,icmp                False\r\nkubernetes-by-kubeadm-allow-health-check   kubernetes-by-kubeadm  INGRESS    1000      tcp                                 False\r\nkubernetes-by-kubeadm-allow-internal       kubernetes-by-kubeadm  INGRESS    1000      tcp,udp,icmp                        False\r\nkubernetes-by-kubeadm-allow-nginx-service  kubernetes-by-kubeadm  INGRESS    1000      tcp:31120                           False\r\n\r\nTo show all fields of the firewall, please show in JSON format: --format=json\r\nTo show all fields in table format, please see the examples in --help.\r\n\r\n<\/span>$<\/span> gcloud -q<\/span> compute firewall-rules delete \\<\/span>\r\n  kubernetes-by-kubeadm-allow-external \\<\/span>\r\n  kubernetes-by-kubeadm-allow-internal \\<\/span>\r\n  kubernetes-by-kubeadm-allow-health-check \\<\/span>\r\n  kubernetes-by-kubeadm-allow-nginx-service\r\nDeleted [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/global\/firewalls\/kubernetes-by-kubeadm-allow-external].\r\nDeleted [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/global\/firewalls\/kubernetes-by-kubeadm-allow-internal].\r\nDeleted [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/global\/firewalls\/kubernetes-by-kubeadm-allow-health-check].\r\nDeleted [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/global\/firewalls\/kubernetes-by-kubeadm-allow-nginx-service].\r\n<\/span>$<\/span> gcloud -q<\/span> compute firewall-rules list\r\nNAME                    NETWORK  DIRECTION  PRIORITY  ALLOW                         DENY  DISABLED\r\ndefault-allow-icmp      default  INGRESS    65534     icmp                                False\r\ndefault-allow-internal  default  INGRESS    65534     tcp:0-65535,udp:0-65535,icmp        False\r\ndefault-allow-rdp       default  INGRESS    65534     tcp:3389                            False\r\ndefault-allow-ssh       default  INGRESS    65534     tcp:22                              False\r\n\r\nTo show all fields of the firewall, please show in JSON format: --format=json\r\nTo show all fields in table format, please see the examples in --help.\r\n<\/span><\/code><\/pre>\n
\u5220\u9664VPC\u5b50\u7f51<\/h3>\n
$<\/span> gcloud -q<\/span> compute networks subnets list --filter<\/span>=<\/span>\"network:kubernetes-by-kubeadm\"<\/span>\r\nNAME        REGION           NETWORK                RANGE\r\nkubernetes  asia-northeast1  kubernetes-by-kubeadm  10.240.0.0\/24\r\n<\/span>$<\/span> gcloud -q<\/span> compute networks subnets delete kubernetes\r\nDeleted [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/regions\/asia-northeast1\/subnetworks\/kubernetes].\r\n<\/span>$<\/span> gcloud -q<\/span> compute networks subnets list --filter<\/span>=<\/span>\"network:kubernetes-by-kubeadm\"<\/span>\r\nListed 0 items.\r\n<\/span><\/code><\/pre>\n
\u5220\u9664 VPC \u7f51\u7edc<\/h3>\n
$<\/span> gcloud -q<\/span> compute networks list\r\nNAME                   SUBNET_MODE  BGP_ROUTING_MODE  IPV4_RANGE  GATEWAY_IPV4\r\ndefault                AUTO         REGIONAL\r\nkubernetes-by-kubeadm  CUSTOM       REGIONAL\r\n<\/span>$<\/span> gcloud -q<\/span> compute networks delete kubernetes-by-kubeadm\r\nDeleted [https:\/\/www.googleapis.com\/compute\/v1\/projects\/sandbox-project\/global\/networks\/kubernetes-by-kubeadm].\r\n<\/span>$<\/span> gcloud -q<\/span> compute networks list\r\nNAME     SUBNET_MODE  BGP_ROUTING_MODE  IPV4_RANGE  GATEWAY_IPV4\r\ndefault  AUTO         REGIONAL\r\n<\/span><\/code><\/pre>\n
\u53ea\u8981\u6309\u7167 Hard Way \u7684\u8981\u6c42\u505a\u5c31\u597d\uff0c\u4e0d\u9700\u8981\u989d\u5916\u6307\u5b9a\u3002<\/div>\n","protected":false},"excerpt":{"rendered":"
\u51c6\u5907\u4e00\u4e2aGCE\u5b9e\u4f8b\u4f5c\u4e3a\u63a7\u5236\u5e73\u9762\uff0c\u4e09\u4e2aGCE\u5b9e\u4f8b\u4f5c\u4e3a\u5de5\u4f5c\u8282\u70b9\uff0c\u7136\u540e\u4f7f\u7528kubeadm\u6784\u5efaKubernetes\u96c6\u7fa4 […]<\/p>\n","protected":false},"author":7,"featured_media":0,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-36022","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"yoast_head":"\n\u4f7f\u7528kubeadm\u5728GCE\u4e0a\u6784\u5efa\u4e00\u4e2a\u5305\u542b1\u4e2a\u63a7\u5236\u5e73\u9762\u548c3\u4e2a\u5de5\u4f5c\u8282\u70b9\u7684Kubernetes\u96c6\u7fa4 - Blog - Silicon Cloud<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.silicloud.com\/zh\/blog\/\u4f7f\u7528kubeadm\u5728gce\u4e0a\u6784\u5efa\u4e00\u4e2a\u5305\u542b1\u4e2a\u63a7\u5236\u5e73\u9762\u548c3\u4e2a\u5de5\u4f5c\u8282\/\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"\u4f7f\u7528kubeadm\u5728GCE\u4e0a\u6784\u5efa\u4e00\u4e2a\u5305\u542b1\u4e2a\u63a7\u5236\u5e73\u9762\u548c3\u4e2a\u5de5\u4f5c\u8282\u70b9\u7684Kubernetes\u96c6\u7fa4\" \/>\n<meta property=\"og:description\" content=\"\u51c6\u5907\u4e00\u4e2aGCE\u5b9e\u4f8b\u4f5c\u4e3a\u63a7\u5236\u5e73\u9762\uff0c\u4e09\u4e2aGCE\u5b9e\u4f8b\u4f5c\u4e3a\u5de5\u4f5c\u8282\u70b9\uff0c\u7136\u540e\u4f7f\u7528kubeadm\u6784\u5efaKubernetes\u96c6\u7fa4 […]\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.silicloud.com\/zh\/blog\/\u4f7f\u7528kubeadm\u5728gce\u4e0a\u6784\u5efa\u4e00\u4e2a\u5305\u542b1\u4e2a\u63a7\u5236\u5e73\u9762\u548c3\u4e2a\u5de5\u4f5c\u8282\/\" \/>\n<meta property=\"og:site_name\" content=\"Blog - Silicon Cloud\" \/>\n<meta property=\"article:published_time\" content=\"2023-06-15T19:31:19+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2024-01-15T00:45:20+00:00\" \/>\n<meta name=\"author\" content=\"\u79d1, \u9896\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"\u79d1, \u9896\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"22 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8kubeadm%e5%9c%a8gce%e4%b8%8a%e6%9e%84%e5%bb%ba%e4%b8%80%e4%b8%aa%e5%8c%85%e5%90%ab1%e4%b8%aa%e6%8e%a7%e5%88%b6%e5%b9%b3%e9%9d%a2%e5%92%8c3%e4%b8%aa%e5%b7%a5%e4%bd%9c%e8%8a%82\/\",\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8kubeadm%e5%9c%a8gce%e4%b8%8a%e6%9e%84%e5%bb%ba%e4%b8%80%e4%b8%aa%e5%8c%85%e5%90%ab1%e4%b8%aa%e6%8e%a7%e5%88%b6%e5%b9%b3%e9%9d%a2%e5%92%8c3%e4%b8%aa%e5%b7%a5%e4%bd%9c%e8%8a%82\/\",\"name\":\"\u4f7f\u7528kubeadm\u5728GCE\u4e0a\u6784\u5efa\u4e00\u4e2a\u5305\u542b1\u4e2a\u63a7\u5236\u5e73\u9762\u548c3\u4e2a\u5de5\u4f5c\u8282\u70b9\u7684Kubernetes\u96c6\u7fa4 - Blog - Silicon Cloud\",\"isPartOf\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#website\"},\"datePublished\":\"2023-06-15T19:31:19+00:00\",\"dateModified\":\"2024-01-15T00:45:20+00:00\",\"author\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/8ca01ba7f7362ad4edb7da206a12f29e\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8kubeadm%e5%9c%a8gce%e4%b8%8a%e6%9e%84%e5%bb%ba%e4%b8%80%e4%b8%aa%e5%8c%85%e5%90%ab1%e4%b8%aa%e6%8e%a7%e5%88%b6%e5%b9%b3%e9%9d%a2%e5%92%8c3%e4%b8%aa%e5%b7%a5%e4%bd%9c%e8%8a%82\/#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8kubeadm%e5%9c%a8gce%e4%b8%8a%e6%9e%84%e5%bb%ba%e4%b8%80%e4%b8%aa%e5%8c%85%e5%90%ab1%e4%b8%aa%e6%8e%a7%e5%88%b6%e5%b9%b3%e9%9d%a2%e5%92%8c3%e4%b8%aa%e5%b7%a5%e4%bd%9c%e8%8a%82\/\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8kubeadm%e5%9c%a8gce%e4%b8%8a%e6%9e%84%e5%bb%ba%e4%b8%80%e4%b8%aa%e5%8c%85%e5%90%ab1%e4%b8%aa%e6%8e%a7%e5%88%b6%e5%b9%b3%e9%9d%a2%e5%92%8c3%e4%b8%aa%e5%b7%a5%e4%bd%9c%e8%8a%82\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.silicloud.com\/zh\/blog\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u4f7f\u7528kubeadm\u5728GCE\u4e0a\u6784\u5efa\u4e00\u4e2a\u5305\u542b1\u4e2a\u63a7\u5236\u5e73\u9762\u548c3\u4e2a\u5de5\u4f5c\u8282\u70b9\u7684Kubernetes\u96c6\u7fa4\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#website\",\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/\",\"name\":\"Blog - Silicon Cloud\",\"description\":\"\",\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/8ca01ba7f7362ad4edb7da206a12f29e\",\"name\":\"\u79d1, \u9896\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/secure.gravatar.com\/avatar\/8a6fb3cc7ba2f69d2189ba532aec4633ea7ed75ac0af162ec367cb3abc0fb2af?s=96&d=mm&r=g\",\"contentUrl\":\"https:\/\/secure.gravatar.com\/avatar\/8a6fb3cc7ba2f69d2189ba532aec4633ea7ed75ac0af162ec367cb3abc0fb2af?s=96&d=mm&r=g\",\"caption\":\"\u79d1, \u9896\"},\"url\":\"https:\/\/www.silicloud.com\/zh\/blog\/author\/keying\/\"},{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8kubeadm%e5%9c%a8gce%e4%b8%8a%e6%9e%84%e5%bb%ba%e4%b8%80%e4%b8%aa%e5%8c%85%e5%90%ab1%e4%b8%aa%e6%8e%a7%e5%88%b6%e5%b9%b3%e9%9d%a2%e5%92%8c3%e4%b8%aa%e5%b7%a5%e4%bd%9c%e8%8a%82\/#local-main-organization-logo\",\"url\":\"\",\"contentUrl\":\"\",\"caption\":\"Blog - Silicon Cloud\"}]}<\/script>\n<!-- \/ Yoast SEO Premium plugin. -->","yoast_head_json":{"title":"\u4f7f\u7528kubeadm\u5728GCE\u4e0a\u6784\u5efa\u4e00\u4e2a\u5305\u542b1\u4e2a\u63a7\u5236\u5e73\u9762\u548c3\u4e2a\u5de5\u4f5c\u8282\u70b9\u7684Kubernetes\u96c6\u7fa4 - Blog - Silicon Cloud","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.silicloud.com\/zh\/blog\/\u4f7f\u7528kubeadm\u5728gce\u4e0a\u6784\u5efa\u4e00\u4e2a\u5305\u542b1\u4e2a\u63a7\u5236\u5e73\u9762\u548c3\u4e2a\u5de5\u4f5c\u8282\/","og_locale":"zh_CN","og_type":"article","og_title":"\u4f7f\u7528kubeadm\u5728GCE\u4e0a\u6784\u5efa\u4e00\u4e2a\u5305\u542b1\u4e2a\u63a7\u5236\u5e73\u9762\u548c3\u4e2a\u5de5\u4f5c\u8282\u70b9\u7684Kubernetes\u96c6\u7fa4","og_description":"\u51c6\u5907\u4e00\u4e2aGCE\u5b9e\u4f8b\u4f5c\u4e3a\u63a7\u5236\u5e73\u9762\uff0c\u4e09\u4e2aGCE\u5b9e\u4f8b\u4f5c\u4e3a\u5de5\u4f5c\u8282\u70b9\uff0c\u7136\u540e\u4f7f\u7528kubeadm\u6784\u5efaKubernetes\u96c6\u7fa4 […]","og_url":"https:\/\/www.silicloud.com\/zh\/blog\/\u4f7f\u7528kubeadm\u5728gce\u4e0a\u6784\u5efa\u4e00\u4e2a\u5305\u542b1\u4e2a\u63a7\u5236\u5e73\u9762\u548c3\u4e2a\u5de5\u4f5c\u8282\/","og_site_name":"Blog - Silicon Cloud","article_published_time":"2023-06-15T19:31:19+00:00","article_modified_time":"2024-01-15T00:45:20+00:00","author":"\u79d1, \u9896","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"\u79d1, \u9896","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"22 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8kubeadm%e5%9c%a8gce%e4%b8%8a%e6%9e%84%e5%bb%ba%e4%b8%80%e4%b8%aa%e5%8c%85%e5%90%ab1%e4%b8%aa%e6%8e%a7%e5%88%b6%e5%b9%b3%e9%9d%a2%e5%92%8c3%e4%b8%aa%e5%b7%a5%e4%bd%9c%e8%8a%82\/","url":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8kubeadm%e5%9c%a8gce%e4%b8%8a%e6%9e%84%e5%bb%ba%e4%b8%80%e4%b8%aa%e5%8c%85%e5%90%ab1%e4%b8%aa%e6%8e%a7%e5%88%b6%e5%b9%b3%e9%9d%a2%e5%92%8c3%e4%b8%aa%e5%b7%a5%e4%bd%9c%e8%8a%82\/","name":"\u4f7f\u7528kubeadm\u5728GCE\u4e0a\u6784\u5efa\u4e00\u4e2a\u5305\u542b1\u4e2a\u63a7\u5236\u5e73\u9762\u548c3\u4e2a\u5de5\u4f5c\u8282\u70b9\u7684Kubernetes\u96c6\u7fa4 - Blog - Silicon Cloud","isPartOf":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/#website"},"datePublished":"2023-06-15T19:31:19+00:00","dateModified":"2024-01-15T00:45:20+00:00","author":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/8ca01ba7f7362ad4edb7da206a12f29e"},"breadcrumb":{"@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8kubeadm%e5%9c%a8gce%e4%b8%8a%e6%9e%84%e5%bb%ba%e4%b8%80%e4%b8%aa%e5%8c%85%e5%90%ab1%e4%b8%aa%e6%8e%a7%e5%88%b6%e5%b9%b3%e9%9d%a2%e5%92%8c3%e4%b8%aa%e5%b7%a5%e4%bd%9c%e8%8a%82\/#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8kubeadm%e5%9c%a8gce%e4%b8%8a%e6%9e%84%e5%bb%ba%e4%b8%80%e4%b8%aa%e5%8c%85%e5%90%ab1%e4%b8%aa%e6%8e%a7%e5%88%b6%e5%b9%b3%e9%9d%a2%e5%92%8c3%e4%b8%aa%e5%b7%a5%e4%bd%9c%e8%8a%82\/"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8kubeadm%e5%9c%a8gce%e4%b8%8a%e6%9e%84%e5%bb%ba%e4%b8%80%e4%b8%aa%e5%8c%85%e5%90%ab1%e4%b8%aa%e6%8e%a7%e5%88%b6%e5%b9%b3%e9%9d%a2%e5%92%8c3%e4%b8%aa%e5%b7%a5%e4%bd%9c%e8%8a%82\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.silicloud.com\/zh\/blog\/"},{"@type":"ListItem","position":2,"name":"\u4f7f\u7528kubeadm\u5728GCE\u4e0a\u6784\u5efa\u4e00\u4e2a\u5305\u542b1\u4e2a\u63a7\u5236\u5e73\u9762\u548c3\u4e2a\u5de5\u4f5c\u8282\u70b9\u7684Kubernetes\u96c6\u7fa4"}]},{"@type":"WebSite","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#website","url":"https:\/\/www.silicloud.com\/zh\/blog\/","name":"Blog - Silicon Cloud","description":"","inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/8ca01ba7f7362ad4edb7da206a12f29e","name":"\u79d1, \u9896","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.silicloud.com\/zh\/blog\/#\/schema\/person\/image\/","url":"https:\/\/secure.gravatar.com\/avatar\/8a6fb3cc7ba2f69d2189ba532aec4633ea7ed75ac0af162ec367cb3abc0fb2af?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/8a6fb3cc7ba2f69d2189ba532aec4633ea7ed75ac0af162ec367cb3abc0fb2af?s=96&d=mm&r=g","caption":"\u79d1, \u9896"},"url":"https:\/\/www.silicloud.com\/zh\/blog\/author\/keying\/"},{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.silicloud.com\/zh\/blog\/%e4%bd%bf%e7%94%a8kubeadm%e5%9c%a8gce%e4%b8%8a%e6%9e%84%e5%bb%ba%e4%b8%80%e4%b8%aa%e5%8c%85%e5%90%ab1%e4%b8%aa%e6%8e%a7%e5%88%b6%e5%b9%b3%e9%9d%a2%e5%92%8c3%e4%b8%aa%e5%b7%a5%e4%bd%9c%e8%8a%82\/#local-main-organization-logo","url":"","contentUrl":"","caption":"Blog - Silicon Cloud"}]}},"_links":{"self":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/36022","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/users\/7"}],"replies":[{"embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/comments?post=36022"}],"version-history":[{"count":2,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/36022\/revisions"}],"predecessor-version":[{"id":57897,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/posts\/36022\/revisions\/57897"}],"wp:attachment":[{"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/media?parent=36022"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/categories?post=36022"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.silicloud.com\/zh\/blog\/wp-json\/wp\/v2\/tags?post=36022"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}