root@deploy1002:~# kubectl describe node kubernetes1014.eqiad.wmnet Name: kubernetes1014.eqiad.wmnet Roles: Labels: beta.kubernetes.io/arch=amd64 beta.kubernetes.io/os=linux failure-domain.beta.kubernetes.io/region=eqiad failure-domain.beta.kubernetes.io/zone=row-d kubernetes.io/arch=amd64 kubernetes.io/hostname=kubernetes1014.eqiad.wmnet kubernetes.io/os=linux node.kubernetes.io/disk-type=ssd Annotations: node.alpha.kubernetes.io/ttl: 0 projectcalico.org/IPv4Address: 10.64.48.230/22 projectcalico.org/IPv6Address: 2620:0:861:107:10:64:48:230/64 volumes.kubernetes.io/controller-managed-attach-detach: true CreationTimestamp: Tue, 23 Mar 2021 09:53:07 +0000 Taints: Unschedulable: false Conditions: Type Status LastHeartbeatTime LastTransitionTime Reason Message ---- ------ ----------------- ------------------ ------ ------- NetworkUnavailable False Thu, 09 Dec 2021 14:34:47 +0000 Thu, 09 Dec 2021 14:34:47 +0000 CalicoIsUp Calico is running on this node MemoryPressure False Mon, 07 Feb 2022 06:07:23 +0000 Tue, 23 Mar 2021 09:53:07 +0000 KubeletHasSufficientMemory kubelet has sufficient mem ory available DiskPressure False Mon, 07 Feb 2022 06:07:23 +0000 Wed, 29 Sep 2021 07:28:27 +0000 KubeletHasNoDiskPressure kubelet has no disk pressu re PIDPressure False Mon, 07 Feb 2022 06:07:23 +0000 Tue, 23 Mar 2021 09:53:07 +0000 KubeletHasSufficientPID kubelet has sufficient PID available Ready True Mon, 07 Feb 2022 06:07:23 +0000 Wed, 29 Sep 2021 09:46:08 +0000 KubeletReady kubelet is posting ready s tatus. AppArmor enabled Addresses: InternalIP: 10.64.48.230 Hostname: kubernetes1014.eqiad.wmnet Capacity: cpu: 48 ephemeral-storage: 28688292Ki hugepages-1Gi: 0 hugepages-2Mi: 0 memory: 97409220Ki pods: 110 Allocatable: cpu: 48 ephemeral-storage: 26439129864 hugepages-1Gi: 0 hugepages-2Mi: 0 memory: 97306820Ki pods: 110 System Info: Machine ID: 73679900aa984a45bf760f538524aba6 System UUID: 4c4c4544-004b-4310-8051-b1c04f473133 Boot ID: 7d518087-e900-48d4-86bd-3ce1104ad812 Kernel Version: 4.19.0-0.bpo.14-amd64 OS Image: Debian GNU/Linux 9 (stretch) Operating System: linux Architecture: amd64 Container Runtime Version: docker://18.6.3 Kubelet Version: v1.16.15 Kube-Proxy Version: v1.16.15 Non-terminated Pods: (35 in total) Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits AGE --------- ---- ------------ ---------- --------------- ------------- --- apertium apertium-production-6b99ff7694-8d6pw 300m (0%) 1600m (3%) 300Mi (0%) 1300Mi (1%) 20d api-gateway api-gateway-production-5889589fc-dz2p8 900m (1%) 1400m (2%) 900Mi (0%) 1200Mi (1%) 12d blubberoid blubberoid-production-66947dddf9-4df7f 300m (0%) 1500m (3%) 200Mi (0%) 600Mi (0%) 5d17 h changeprop-jobqueue changeprop-production-b5fbcffbb-x9q7n 700m (1%) 1200m (2%) 500Mi (0%) 1700Mi (1%) 18d changeprop changeprop-production-78fb8b7cd4-hfpwf 700m (1%) 3200m (6%) 500Mi (0%) 1700Mi (1%) 18d cxserver cxserver-production-694c7db9c-6zkjh 300m (0%) 1500m (3%) 200Mi (0%) 900Mi (0%) 3d18 h eventgate-analytics-external eventgate-analytics-external-production-7577b69644-lzzhx 400m (0%) 2500m (5%) 250Mi (0%) 800Mi (0%) 11d eventgate-analytics eventgate-analytics-production-7b8ccf799b-gwhzj 400m (0%) 2500m (5%) 250Mi (0%) 800Mi (0%) 10d eventgate-analytics eventgate-analytics-production-7b8ccf799b-twbq8 400m (0%) 2500m (5%) 250Mi (0%) 800Mi (0%) 10d eventgate-logging-external eventgate-logging-external-production-99fd8cc75-m2l47 700m (1%) 3 (6%) 250Mi (0%) 800Mi (0%) 11d eventgate-main eventgate-main-production-fb5d896b9-rdxsn 400m (0%) 2500m (5%) 250Mi (0%) 1100Mi (1%) 4d16 h eventstreams-internal eventstreams-internal-main-5bffd6d7c6-t2xkt 400m (0%) 2 (4%) 600Mi (0%) 1500Mi (1%) 10d eventstreams eventstreams-production-7b64d578d8-nzhzq 400m (0%) 2 (4%) 600Mi (0%) 1500Mi (1%) 10d kube-system calico-node-86v2g 350m (0%) 0 (0%) 400Mi (0%) 400Mi (0%) 59d kube-system coredns-584d8549b5-hr6tg 100m (0%) 0 (0%) 70Mi (0%) 170Mi (0%) 130d linkrecommendation linkrecommendation-internal-58fd7dd4d8-vvdh5 2050m (4%) 4600m (9%) 700Mi (0%) 1250Mi (1%) 3d17 h mathoid mathoid-production-596cb7c944-v6brp 300m (0%) 1500m (3%) 300Mi (0%) 1524288k (1%) 73d mathoid mathoid-production-596cb7c944-wmhrs 300m (0%) 1500m (3%) 300Mi (0%) 1524288k (1%) 73d mobileapps mobileapps-production-746f7579c9-58b7q 1300m (2%) 1800m (3%) 550Mi (0%) 1200Mi (1%) 73d mobileapps mobileapps-production-746f7579c9-79hdk 1300m (2%) 1800m (3%) 550Mi (0%) 1200Mi (1%) 73d mobileapps mobileapps-production-746f7579c9-7wx27 1300m (2%) 1800m (3%) 550Mi (0%) 1200Mi (1%) 73d mobileapps mobileapps-production-746f7579c9-8ncx6 1300m (2%) 1800m (3%) 550Mi (0%) 1200Mi (1%) 73d mobileapps mobileapps-production-746f7579c9-96xk7 1300m (2%) 1800m (3%) 550Mi (0%) 1200Mi (1%) 73d mobileapps mobileapps-production-746f7579c9-ck7bl 1300m (2%) 1800m (3%) 550Mi (0%) 1200Mi (1%) 73d mobileapps mobileapps-production-746f7579c9-kjdng 1300m (2%) 1800m (3%) 550Mi (0%) 1200Mi (1%) 73d rdf-streaming-updater flink-session-cluster-main-taskmanager-6cdd5c87b7-kd5fp 1200m (2%) 4500m (9%) 2152857600 (2%) 2624288k (2%) 32d recommendation-api recommendation-api-production-64d668997b-nmlsh 700m (1%) 2100m (4%) 350Mi (0%) 900Mi (0%) 73d shellbox-constraints shellbox-main-64999bc485-djmbd 700m (1%) 2700m (5%) 1524Mi (1%) 3148Mi (3%) 25d shellbox-media shellbox-main-5c56fb99cc-nblgf 3600m (7%) 4700m (9%) 1524Mi (1%) 3148Mi (3%) 25d shellbox-syntaxhighlight shellbox-main-7795bf4db7-b5cbs 2100m (4%) 3700m (7%) 1524Mi (1%) 3148Mi (3%) 25d shellbox-timeline shellbox-main-6796665947-n5cr9 2100m (4%) 3700m (7%) 1524Mi (1%) 3148Mi (3%) 25d shellbox shellbox-main-7765d7c9db-l4wt5 700m (1%) 2700m (5%) 1524Mi (1%) 3148Mi (3%) 25d termbox termbox-production-6f5b9d8cf-tlmjf 400m (0%) 1600m (3%) 340Mi (0%) 900Mi (0%) 6d15 h wikifeeds wikifeeds-production-84d464cb98-74dzk 1600m (3%) 3600m (7%) 1200Mi (1%) 1900Mi (1%) 69d zotero zotero-production-685c4f4cb6-cmckq 400m (0%) 1500m (3%) 300Mi (0%) 2548Mi (2%) 73d Allocated resources: (Total limits may be over 100 percent, i.e., overcommitted.) Resource Requests Limits -------- -------- ------ cpu 32 (66%) 78400m (163%) memory 23073920Ki (23%) 54859467008 (55%) ephemeral-storage 0 (0%) 0 (0%) Events: root@deploy1002:~# ping kubernetes1014 PING kubernetes1014(kubernetes1014.eqiad.wmnet (2620:0:861:107:10:64:48:230)) 56 data bytes 64 bytes from kubernetes1014.eqiad.wmnet (2620:0:861:107:10:64:48:230): icmp_seq=1 ttl=63 time=0.230 ms 64 bytes from kubernetes1014.eqiad.wmnet (2620:0:861:107:10:64:48:230): icmp_seq=2 ttl=63 time=0.252 ms 64 bytes from kubernetes1014.eqiad.wmnet (2620:0:861:107:10:64:48:230): icmp_seq=3 ttl=63 time=0.253 ms ^C --- kubernetes1014 ping statistics --- 3 packets transmitted, 3 received, 0% packet loss, time 10ms rtt min/avg/max/mdev = 0.230/0.245/0.253/0.010 ms root@deploy1002:~# kubectl drain kubernetes1014.eqiad.wmnet node/kubernetes1014.eqiad.wmnet cordoned error: unable to drain node "kubernetes1014.eqiad.wmnet", aborting command... There are pending nodes to be drained: kubernetes1014.eqiad.wmnet cannot delete Pods with local storage (use --delete-local-data to override): api-gateway/api-gateway-production-5889589fc-dz2p8, shellbox-constraints/shel lbox-main-64999bc485-djmbd, shellbox-media/shellbox-main-5c56fb99cc-nblgf, shellbox-syntaxhighlight/shellbox-main-7795bf4db7-b5cbs, shellbox-timeline/shel lbox-main-6796665947-n5cr9, shellbox/shellbox-main-7765d7c9db-l4wt5 cannot delete DaemonSet-managed Pods (use --ignore-daemonsets to ignore): kube-system/calico-node-86v2g root@deploy1002:~# kubectl drain kubernetes1014.eqiad.wmnet --delete-local-data --ignore-daemonsets node/kubernetes1014.eqiad.wmnet already cordoned WARNING: ignoring DaemonSet-managed Pods: kube-system/calico-node-86v2g evicting pod "blubberoid-production-66947dddf9-4df7f" evicting pod "eventgate-analytics-production-7b8ccf799b-gwhzj" evicting pod "zotero-production-685c4f4cb6-cmckq" evicting pod "apertium-production-6b99ff7694-8d6pw" evicting pod "changeprop-production-b5fbcffbb-x9q7n" evicting pod "eventgate-analytics-production-7b8ccf799b-twbq8" evicting pod "api-gateway-production-5889589fc-dz2p8" evicting pod "changeprop-production-78fb8b7cd4-hfpwf" evicting pod "shellbox-main-64999bc485-djmbd" evicting pod "eventgate-logging-external-production-99fd8cc75-m2l47" evicting pod "mobileapps-production-746f7579c9-8ncx6" evicting pod "shellbox-main-7765d7c9db-l4wt5" evicting pod "eventgate-main-production-fb5d896b9-rdxsn" evicting pod "mathoid-production-596cb7c944-wmhrs" evicting pod "termbox-production-6f5b9d8cf-tlmjf" evicting pod "coredns-584d8549b5-hr6tg" evicting pod "eventgate-analytics-external-production-7577b69644-lzzhx" evicting pod "mobileapps-production-746f7579c9-ck7bl" evicting pod "shellbox-main-5c56fb99cc-nblgf" evicting pod "eventstreams-internal-main-5bffd6d7c6-t2xkt" evicting pod "cxserver-production-694c7db9c-6zkjh" evicting pod "shellbox-main-7795bf4db7-b5cbs" evicting pod "mobileapps-production-746f7579c9-kjdng" evicting pod "mobileapps-production-746f7579c9-7wx27" evicting pod "mobileapps-production-746f7579c9-79hdk" evicting pod "eventstreams-production-7b64d578d8-nzhzq" evicting pod "shellbox-main-6796665947-n5cr9" evicting pod "wikifeeds-production-84d464cb98-74dzk" evicting pod "mobileapps-production-746f7579c9-58b7q" evicting pod "mobileapps-production-746f7579c9-96xk7" evicting pod "recommendation-api-production-64d668997b-nmlsh" evicting pod "flink-session-cluster-main-taskmanager-6cdd5c87b7-kd5fp" evicting pod "linkrecommendation-internal-58fd7dd4d8-vvdh5" evicting pod "mathoid-production-596cb7c944-v6brp" pod/api-gateway-production-5889589fc-dz2p8 evicted pod/mobileapps-production-746f7579c9-7wx27 evicted pod/changeprop-production-b5fbcffbb-x9q7n evicted pod/cxserver-production-694c7db9c-6zkjh evicted pod/eventgate-analytics-production-7b8ccf799b-gwhzj evicted pod/flink-session-cluster-main-taskmanager-6cdd5c87b7-kd5fp evicted pod/eventgate-analytics-production-7b8ccf799b-twbq8 evicted pod/mobileapps-production-746f7579c9-kjdng evicted pod/mobileapps-production-746f7579c9-79hdk evicted pod/mobileapps-production-746f7579c9-58b7q evicted pod/eventgate-main-production-fb5d896b9-rdxsn evicted pod/mathoid-production-596cb7c944-v6brp evicted pod/eventstreams-production-7b64d578d8-nzhzq evicted pod/mobileapps-production-746f7579c9-8ncx6 evicted pod/apertium-production-6b99ff7694-8d6pw evicted pod/shellbox-main-7795bf4db7-b5cbs evicted pod/eventgate-analytics-external-production-7577b69644-lzzhx evicted pod/zotero-production-685c4f4cb6-cmckq evicted pod/eventgate-logging-external-production-99fd8cc75-m2l47 evicted pod/blubberoid-production-66947dddf9-4df7f evicted pod/mathoid-production-596cb7c944-wmhrs evicted pod/shellbox-main-7765d7c9db-l4wt5 evicted pod/shellbox-main-6796665947-n5cr9 evicted pod/coredns-584d8549b5-hr6tg evicted pod/shellbox-main-64999bc485-djmbd evicted pod/mobileapps-production-746f7579c9-96xk7 evicted pod/changeprop-production-78fb8b7cd4-hfpwf evicted pod/wikifeeds-production-84d464cb98-74dzk evicted pod/shellbox-main-5c56fb99cc-nblgf evicted pod/mobileapps-production-746f7579c9-ck7bl evicted pod/linkrecommendation-internal-58fd7dd4d8-vvdh5 evicted pod/eventstreams-internal-main-5bffd6d7c6-t2xkt evicted pod/termbox-production-6f5b9d8cf-tlmjf evicted pod/recommendation-api-production-64d668997b-nmlsh evicted node/kubernetes1014.eqiad.wmnet evicted root@deploy1002:~#