Duplicated secret volume in pod spec #594
Description
When specifying secret(e.g. --conf spark.kubernetes.driver.secrets.test=pass) in spark-submit command line, there will be an error "Duplicate value "XXX-volume". Because the volume is added twice to pod spec: mountSecret for main container, mountSecret for init container.
Command line:
$ bin/spark-submit --deploy-mode cluster --class org.apache.spark.examples.SparkPi
--master k8s://http://localhost:8080 --kubernetes-namespace default
--conf spark.executor.instances=5
--conf spark.app.name=spark-pi
--conf spark.kubernetes.driver.docker.image=jungong/spark-driver:hdfs
--conf spark.kubernetes.executor.docker.image=jungong/spark-executor:hdfs
--conf spark.kubernetes.initcontainer.docker.image=jungong/spark-init:hdfs
--conf spark.kubernetes.resourceStagingServer.uri=http://10.178.106.222:31000
--conf spark.kubernetes.initcontainer.inannotation=true
--conf spark.kubernetes.driver.secrets.test=pass
examples/jars/spark-examples_2.11-2.2.0-k8s-0.5.0.jar
Error message:
Exception in thread "main" io.fabric8.kubernetes.client.KubernetesClientException:
Failure executing: POST at: http://localhost:8080/api/v1/namespaces/default/pods.
Message: Pod "spark-pi-1514974049945-driver" is invalid: spec.volumes[7].name:
Duplicate value: "test-volume". Received status: Status(apiVersion=v1, code=422,
details=StatusDetails(causes=[StatusCause(field=spec.volumes[7].name, message=Duplicate
value: "test-volume", reason=FieldValueDuplicate, additionalProperties={})], group=null, kind=Pod,
name=spark-pi-1514974049945-driver, retryAfterSeconds=null, uid=null, additionalProperties={}),
kind=Status, message=Pod "spark-pi-1514974049945-driver" is invalid: spec.volumes[7].name:
Duplicate value: "test-volume", metadata=ListMeta(resourceVersion=null, selfLink=null,
additionalProperties={}), reason=Invalid, status=Failure, additionalProperties={}).
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.requestFailure(OperationSupport.java:470)
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.assertResponseCode(OperationSupport.java:409)
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:379)
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:343)
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleCreate(OperationSupport.java:226)
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.handleCreate(BaseOperation.java:769)
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.create(BaseOperation.java:356)
at org.apache.spark.deploy.k8s.submit.Client$$anonfun$run$3.apply(Client.scala:132)
at org.apache.spark.deploy.k8s.submit.Client$$anonfun$run$3.apply(Client.scala:131)
at org.apache.spark.util.Utils$.tryWithResource(Utils.scala:2551)
at org.apache.spark.deploy.k8s.submit.Client.run(Client.scala:131)
at org.apache.spark.deploy.k8s.submit.Client$$anonfun$run$5.apply(Client.scala:200)
at org.apache.spark.deploy.k8s.submit.Client$$anonfun$run$5.apply(Client.scala:193)
at org.apache.spark.util.Utils$.tryWithResource(Utils.scala:2551)
at org.apache.spark.deploy.k8s.submit.Client$.run(Client.scala:193)
at org.apache.spark.deploy.k8s.submit.Client$.main(Client.scala:213)
at org.apache.spark.deploy.k8s.submit.Client.main(Client.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:786)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:120)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)