Skip to content

[SPARK-2140] Updating heap memory calculation for YARN stable and alpha. #2253

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

[SPARK-2140] Updating heap memory calculation for YARN stable and alpha. #2253

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
52b4e45
[SPARK-2140] Updating heap memory calculation for YARN stable and alpha.
Sep 3, 2014
5ad89da
[SPARK-2140] Removing calculateAMMemory functions since they are no l…
Sep 4, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 0 additions & 8 deletions yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,14 +103,6 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa
appContext
}

def calculateAMMemory(newApp: GetNewApplicationResponse): Int = {
val minResMemory = newApp.getMinimumResourceCapability().getMemory()
val amMemory = ((args.amMemory / minResMemory) * minResMemory) +
((if ((args.amMemory % minResMemory) == 0) 0 else minResMemory) -
memoryOverhead)
amMemory
}

def setupSecurityToken(amContainer: ContainerLaunchContext) = {
// Setup security tokens.
val dob = new DataOutputBuffer()
Expand Down
4 changes: 1 addition & 3 deletions yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -300,8 +300,6 @@ trait ClientBase extends Logging {
retval.toString
}

def calculateAMMemory(newApp: GetNewApplicationResponse): Int

def setupSecurityToken(amContainer: ContainerLaunchContext)

def createContainerLaunchContext(
Expand Down Expand Up @@ -346,7 +344,7 @@ trait ClientBase extends Logging {
}
amContainer.setEnvironment(env)

val amMemory = calculateAMMemory(newApp)
val amMemory = args.amMemory

val javaOpts = ListBuffer[String]()

Expand Down
3 changes: 0 additions & 3 deletions yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,9 +238,6 @@ class ClientBaseSuite extends FunSuite with Matchers {
val sparkConf: SparkConf,
val yarnConf: YarnConfiguration) extends ClientBase {

override def calculateAMMemory(newApp: GetNewApplicationResponse): Int =
throw new UnsupportedOperationException()

override def setupSecurityToken(amContainer: ContainerLaunchContext): Unit =
throw new UnsupportedOperationException()

Expand Down
9 changes: 0 additions & 9 deletions yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,15 +103,6 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa
clusterMetrics.getNumNodeManagers)
}

def calculateAMMemory(newApp: GetNewApplicationResponse) :Int = {
// TODO: Need a replacement for the following code to fix -Xmx?
// val minResMemory: Int = newApp.getMinimumResourceCapability().getMemory()
// var amMemory = ((args.amMemory / minResMemory) * minResMemory) +
// ((if ((args.amMemory % minResMemory) == 0) 0 else minResMemory) -
// memoryOverhead )
args.amMemory
}

def setupSecurityToken(amContainer: ContainerLaunchContext) = {
// Setup security tokens.
val dob = new DataOutputBuffer()
Expand Down