From ba528b911718675becf1623c803489b4cc3e68ec Mon Sep 17 00:00:00 2001 From: "Allan Douglas R. de Oliveira" Date: Sun, 13 Apr 2014 21:09:47 -0300 Subject: [PATCH 1/3] Added SPARK_WORKER_INSTANCES parameter --- ec2/deploy.generic/root/spark-ec2/ec2-variables.sh | 2 ++ ec2/spark_ec2.py | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ec2/deploy.generic/root/spark-ec2/ec2-variables.sh b/ec2/deploy.generic/root/spark-ec2/ec2-variables.sh index 42e8faa26ed0..7e10e09ba584 100644 --- a/ec2/deploy.generic/root/spark-ec2/ec2-variables.sh +++ b/ec2/deploy.generic/root/spark-ec2/ec2-variables.sh @@ -28,3 +28,5 @@ export SPARK_VERSION="{{spark_version}}" export SHARK_VERSION="{{shark_version}}" export HADOOP_MAJOR_VERSION="{{hadoop_major_version}}" export SWAP_MB="{{swap}}" +export SPARK_WORKER_INSTANCES="{{spark_worker_instances}}" + diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py index 31209a662bbe..a5861756d7fb 100755 --- a/ec2/spark_ec2.py +++ b/ec2/spark_ec2.py @@ -103,6 +103,9 @@ def parse_args(): help="When destroying a cluster, delete the security groups that were created") parser.add_option("--use-existing-master", action="store_true", default=False, help="Launch fresh slaves, but use an existing stopped master if possible") + parser.add_option("--worker-instances", type="int", default=1, + help="Number of instances per worker: variable SPARK_WORKER_INSTANCES (default: 1)") + (opts, args) = parser.parse_args() if len(args) != 2: @@ -223,7 +226,7 @@ def launch_cluster(conn, opts, cluster_name): sys.exit(1) if opts.key_pair is None: print >> stderr, "ERROR: Must provide a key pair name (-k) to use on instances." - sys.exit(1) + sys.exit(1) print "Setting up security groups..." master_group = get_or_make_group(conn, cluster_name + "-master") slave_group = get_or_make_group(conn, cluster_name + "-slaves") @@ -551,7 +554,8 @@ def deploy_files(conn, root_dir, opts, master_nodes, slave_nodes, modules): "modules": '\n'.join(modules), "spark_version": spark_v, "shark_version": shark_v, - "hadoop_major_version": opts.hadoop_major_version + "hadoop_major_version": opts.hadoop_major_version, + "spark_worker_instances": opts.worker_instances } # Create a temp directory in which we will place all the files to be From 6c346716e5dd85bb4806ea2a71fa3a2fd00bbfcc Mon Sep 17 00:00:00 2001 From: "Allan Douglas R. de Oliveira" Date: Mon, 14 Apr 2014 10:35:03 -0300 Subject: [PATCH 2/3] Use number of worker instances as string on template --- ec2/spark_ec2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py index a5861756d7fb..a84d6601fbb2 100755 --- a/ec2/spark_ec2.py +++ b/ec2/spark_ec2.py @@ -555,7 +555,7 @@ def deploy_files(conn, root_dir, opts, master_nodes, slave_nodes, modules): "spark_version": spark_v, "shark_version": shark_v, "hadoop_major_version": opts.hadoop_major_version, - "spark_worker_instances": opts.worker_instances + "spark_worker_instances": "%d" % opts.worker_instances } # Create a temp directory in which we will place all the files to be From d6c5d6564b3025eef88d499b253e850dd07f397a Mon Sep 17 00:00:00 2001 From: "Allan Douglas R. de Oliveira" Date: Thu, 1 May 2014 11:50:14 -0300 Subject: [PATCH 3/3] Added master opts parameter --- ec2/deploy.generic/root/spark-ec2/ec2-variables.sh | 2 +- ec2/spark_ec2.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ec2/deploy.generic/root/spark-ec2/ec2-variables.sh b/ec2/deploy.generic/root/spark-ec2/ec2-variables.sh index 7e10e09ba584..3570891be804 100644 --- a/ec2/deploy.generic/root/spark-ec2/ec2-variables.sh +++ b/ec2/deploy.generic/root/spark-ec2/ec2-variables.sh @@ -29,4 +29,4 @@ export SHARK_VERSION="{{shark_version}}" export HADOOP_MAJOR_VERSION="{{hadoop_major_version}}" export SWAP_MB="{{swap}}" export SPARK_WORKER_INSTANCES="{{spark_worker_instances}}" - +export SPARK_MASTER_OPTS="{{spark_master_opts}}" diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py index a84d6601fbb2..db393748a33b 100755 --- a/ec2/spark_ec2.py +++ b/ec2/spark_ec2.py @@ -105,6 +105,9 @@ def parse_args(): help="Launch fresh slaves, but use an existing stopped master if possible") parser.add_option("--worker-instances", type="int", default=1, help="Number of instances per worker: variable SPARK_WORKER_INSTANCES (default: 1)") + parser.add_option("--master-opts", type="string", default="", + help="Extra options to give to master through SPARK_MASTER_OPTS variable (e.g -Dspark.worker.timeout=180)") + (opts, args) = parser.parse_args() @@ -555,7 +558,8 @@ def deploy_files(conn, root_dir, opts, master_nodes, slave_nodes, modules): "spark_version": spark_v, "shark_version": shark_v, "hadoop_major_version": opts.hadoop_major_version, - "spark_worker_instances": "%d" % opts.worker_instances + "spark_worker_instances": "%d" % opts.worker_instances, + "spark_master_opts": opts.master_opts } # Create a temp directory in which we will place all the files to be