Readiness probe failed: zookeeper nc check error ,this is basic images has network problems, how to fix? #63
Description
k8s version is 1.7 network is flanneld overlay
[root@master1 nginx]# kubectl get nodes -o wide
NAME STATUS AGE VERSION EXTERNAL-IP OS-IMAGE KERNEL-VERSION
db.k8s.novalocal Ready 14d v1.7.1 CentOS Linux 7 (Core) 4.4.77-1.el7.elrepo.x86_64
elk-k8sdata.novalocal Ready 18d v1.7.1 CentOS Linux 7 (Core) 4.4.77-1.el7.elrepo.x86_64
jenkins-master.k8s.novalocal Ready 12d v1.7.1 CentOS Linux 7 (Core) 4.4.77-1.el7.elrepo.x86_64
k8s-monitor.novalocal Ready 18d v1.7.1 CentOS Linux 7 (Core) 4.4.77-1.el7.elrepo.x86_64
lb1.k8s.novalocal Ready 14d v1.7.1 CentOS Linux 7 (Core) 4.4.77-1.el7.elrepo.x86_64
master1.k8s.novalocal Ready 18d v1.7.1 CentOS Linux 7 (Core) 4.4.77-1.el7.elrepo.x86_64
node1.k8s.novalocal Ready 19d v1.7.1 CentOS Linux 7 (Core) 4.4.77-1.el7.elrepo.x86_64
node2.k8s.novalocal Ready 19d v1.7.1 CentOS Linux 7 (Core) 4.4.77-1.el7.elrepo.x86_64
node3.k8s.novalocal Ready 19d v1.7.1 CentOS Linux 7 (Core) 4.4.77-1.el7.elrepo.x86_64
[root@master1 nginx]#
zookeeper nc check error ,this is images network bug!
readinessProbe:
exec:
command:
- /bin/sh
- -c
- '[ "imok" = "$(echo ruok | nc -w 1 127.0.0.1 2181)" ]'
[root@master1 ~]# kubectl get pod,svc -n kafka -o wide
NAME READY STATUS RESTARTS AGE IP NODE
po/kafka-0 1/1 Running 0 5d 172.16.47.7 elk-k8sdata.novalocal
po/kafka-1 1/1 Running 0 5d 172.16.1.3 node3.k8s.novalocal
po/kafka-2 1/1 Running 0 5d 172.16.38.6 node1.k8s.novalocal
po/pzoo-0 1/1 Running 0 16h 172.16.71.5 k8s-monitor.novalocal
po/pzoo-1 1/1 Running 0 16h 172.16.47.6 elk-k8sdata.novalocal
po/pzoo-2 1/1 Running 0 16h 172.16.5.6 db.k8s.novalocal
po/zoo-0 1/1 Running 0 16h 172.16.18.3 node2.k8s.novalocal
po/zoo-1 1/1 Running 0 16h 172.16.19.5 jenkins-master.k8s.novalocal
NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
svc/broker None 9092/TCP 5d app=kafka
svc/pzoo None 2888/TCP,3888/TCP 5d app=zookeeper,storage=persistent
svc/zoo None 2888/TCP,3888/TCP 5d app=zookeeper,storage=ephemeral
svc/zookeeper 172.17.200.109 2181/TCP 5d app=zookeeper
[root@master1 ~]#
[root@master1 ~]# kubectl exec -it pzoo-0 /bin/bash -n kafka
root@pzoo-0:/opt/kafka# sh
while true ;do sleep 1 ;imok=$(echo ruok | nc -w 1 pzoo-2.pzoo.kafka.svc.cluster.local 2181 );echo $imok ;done
imok
imok
imok
^C
while true ;do sleep 1 ;imok=$(echo ruok | nc -w 1 pzoo-0.pzoo.kafka.svc.cluster.local 2181 );echo $imok ;done
imok
imok
imok
imok
imok
imok
imok
while true ;do sleep 1 ;imok=$(echo ruok | nc -w 1 127.0.0.1 2181 );echo $imok ;done
imok
imok
imok
imok
imok
imok
imok
imok
imok
Then nginx NC, but no problem, is not images network has problems, basic images has problems, how to fix?
root@nginx-0:/# while true ;do sleep 1 ;imok=$(echo ruok | nc -w 1 nginx-1.web 80 );echo $imok ;done
nter>nginx/1.13.3/center>d> nter>nginx/1.13.3/center>d> nter>nginx/1.13.3/center>d> nter>nginx/1.13.3/center>d> nter>nginx/1.13.3/center>d> nter>nginx/1.13.3/center>d> nter>nginx/1.13.3/center>d> nter>nginx/1.13.3/center>d> ^C root@nginx-0:/# while true ;do sleep 1 ;imok=$(echo ruok | nc -w 1 nginx-0.web 80 );echo $imok ;done nter>nginx/1.13.3/center>d> nter>nginx/1.13.3/center>d> nter>nginx/1.13.3/center>d> nter>nginx/1.13.3/center>d> nter>nginx/1.13.3/center>d> nter>nginx/1.13.3/center>d> nter>nginx/1.13.3/center>d> ^C root@nginx-0:/# while true ;do sleep 1 ;imok=$(echo ruok | nc -w 1 zookeeper.kafka.svc.cluster.local 2181 );echo $imok ;done imok imok imok imok imok imok imok imok imok imok imok imok imok imok imok imok imok imok imok imok ^C root@nginx-0:/# while true ;do sleep 1 ;imok=$(echo ruok | nc -w 1 pzoo-0.pzoo.kafka.svc.cluster.local 2181 );echo $imok ;done imok imok imok imok imok imok imok imok imok imok imok imok imok imok imok imok ^C root@nginx-0:/# while true ;do sleep 1 ;imok=$(echo ruok | nc -w 1 pzoo-1.pzoo.kafka.svc.cluster.local 2181 );echo $imok ;done imok imok imok imok imok imok imok imok imok imok imok imok imok imok imok imok imok imok imok ^C root@nginx-0:/# while true ;do sleep 1 ;imok=$(echo ruok | nc -w 1 pzoo-2.pzoo.kafka.svc.cluster.local 2181 );echo $imok ;done imok imok imok imok imok imok imok imok imok imok imok imok imok imok