Skip to content

[SPARK-19033][Core] Add admin acls for history server #16470

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

[SPARK-19033][Core] Add admin acls for history server #16470

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c0c1933
Add admin acls for history server
jerryshao Jan 4, 2017
f4357e8
Address the comments
jerryshao Jan 5, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 14 additions & 6 deletions core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,13 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
.map { d => Utils.resolveURI(d).toString }
.getOrElse(DEFAULT_LOG_DIR)

private val HISTORY_UI_ACLS_ENABLE = conf.getBoolean("spark.history.ui.acls.enable", false)
private val HISTORY_UI_ADMIN_ACLS = conf.get("spark.history.ui.admin.acls", "")
private val HISTORY_UI_ADMIN_ACLS_GROUPS = conf.get("spark.history.ui.admin.acls.groups", "")
logInfo(s"History server ui acls " + (if (HISTORY_UI_ACLS_ENABLE) "enabled" else "disabled") +
"; users with admin permissions: " + HISTORY_UI_ADMIN_ACLS.toString +
"; groups with admin permissions" + HISTORY_UI_ADMIN_ACLS_GROUPS.toString)

private val hadoopConf = SparkHadoopUtil.get.newConfiguration(conf)
private val fs = Utils.getHadoopFileSystem(logDir, hadoopConf)

Expand Down Expand Up @@ -250,13 +257,14 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
val appListener = replay(fileStatus, isApplicationCompleted(fileStatus), replayBus)

if (appListener.appId.isDefined) {
val uiAclsEnabled = conf.getBoolean("spark.history.ui.acls.enable", false)
ui.getSecurityManager.setAcls(uiAclsEnabled)
ui.getSecurityManager.setAcls(HISTORY_UI_ACLS_ENABLE)
// make sure to set admin acls before view acls so they are properly picked up
ui.getSecurityManager.setAdminAcls(appListener.adminAcls.getOrElse(""))
ui.getSecurityManager.setViewAcls(attempt.sparkUser,
appListener.viewAcls.getOrElse(""))
ui.getSecurityManager.setAdminAclsGroups(appListener.adminAclsGroups.getOrElse(""))
val adminAcls = HISTORY_UI_ADMIN_ACLS + "," + appListener.adminAcls.getOrElse("")
ui.getSecurityManager.setAdminAcls(adminAcls)
ui.getSecurityManager.setViewAcls(attempt.sparkUser, appListener.viewAcls.getOrElse(""))
val adminAclsGroups = HISTORY_UI_ADMIN_ACLS_GROUPS + "," +
appListener.adminAclsGroups.getOrElse("")
ui.getSecurityManager.setAdminAclsGroups(adminAclsGroups)
ui.getSecurityManager.setViewAclsGroups(appListener.viewAclsGroups.getOrElse(""))
Some(LoadedAppUI(ui, updateProbe(appId, attemptId, attempt.fileSize)))
} else {
Expand Down
111 changes: 110 additions & 1 deletion core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,11 @@ import org.scalatest.BeforeAndAfter
import org.scalatest.Matchers
import org.scalatest.concurrent.Eventually._

import org.apache.spark.{SparkConf, SparkFunSuite}
import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
import org.apache.spark.internal.Logging
import org.apache.spark.io._
import org.apache.spark.scheduler._
import org.apache.spark.security.GroupMappingServiceProvider
import org.apache.spark.util.{Clock, JsonProtocol, ManualClock, Utils}

class FsHistoryProviderSuite extends SparkFunSuite with BeforeAndAfter with Matchers with Logging {
Expand Down Expand Up @@ -474,6 +475,102 @@ class FsHistoryProviderSuite extends SparkFunSuite with BeforeAndAfter with Matc
}
}

test("support history server ui admin acls") {
Copy link
Contributor

@tgravescs tgravescs Jan 4, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it would be nice to have tests for empty history admin acls to make sure parsing/concatenation is working properly.

def createAndCheck(conf: SparkConf, properties: (String, String)*)
(checkFn: SecurityManager => Unit): Unit = {
// Empty the testDir for each test.
if (testDir.exists() && testDir.isDirectory) {
testDir.listFiles().foreach { f => if (f.isFile) f.delete() }
}

var provider: FsHistoryProvider = null
try {
provider = new FsHistoryProvider(conf)
val log = newLogFile("app1", Some("attempt1"), inProgress = false)
writeFile(log, true, None,
SparkListenerApplicationStart("app1", Some("app1"), System.currentTimeMillis(),
"test", Some("attempt1")),
SparkListenerEnvironmentUpdate(Map(
"Spark Properties" -> properties.toSeq,
"JVM Information" -> Seq.empty,
"System Properties" -> Seq.empty,
"Classpath Entries" -> Seq.empty
)),
SparkListenerApplicationEnd(System.currentTimeMillis()))

provider.checkForLogs()
val appUi = provider.getAppUI("app1", Some("attempt1"))

assert(appUi.nonEmpty)
val securityManager = appUi.get.ui.securityManager
checkFn(securityManager)
} finally {
if (provider != null) {
provider.stop()
}
}
}

// Test both history ui admin acls and application acls are configured.
val conf1 = createTestConf()
.set("spark.history.ui.acls.enable", "true")
.set("spark.history.ui.admin.acls", "user1,user2")
.set("spark.history.ui.admin.acls.groups", "group1")
.set("spark.user.groups.mapping", classOf[TestGroupsMappingProvider].getName)

createAndCheck(conf1, ("spark.admin.acls", "user"), ("spark.admin.acls.groups", "group")) {
securityManager =>
// Test whether user has permission to access UI.
securityManager.checkUIViewPermissions("user1") should be (true)
securityManager.checkUIViewPermissions("user2") should be (true)
securityManager.checkUIViewPermissions("user") should be (true)
securityManager.checkUIViewPermissions("abc") should be (false)

// Test whether user with admin group has permission to access UI.
securityManager.checkUIViewPermissions("user3") should be (true)
securityManager.checkUIViewPermissions("user4") should be (true)
securityManager.checkUIViewPermissions("user5") should be (true)
securityManager.checkUIViewPermissions("user6") should be (false)
}

// Test only history ui admin acls are configured.
val conf2 = createTestConf()
.set("spark.history.ui.acls.enable", "true")
.set("spark.history.ui.admin.acls", "user1,user2")
.set("spark.history.ui.admin.acls.groups", "group1")
.set("spark.user.groups.mapping", classOf[TestGroupsMappingProvider].getName)
createAndCheck(conf2) { securityManager =>
// Test whether user has permission to access UI.
securityManager.checkUIViewPermissions("user1") should be (true)
securityManager.checkUIViewPermissions("user2") should be (true)
// Check the unknown "user" should return false
securityManager.checkUIViewPermissions("user") should be (false)

// Test whether user with admin group has permission to access UI.
securityManager.checkUIViewPermissions("user3") should be (true)
securityManager.checkUIViewPermissions("user4") should be (true)
// Check the "user5" without mapping relation should return false
securityManager.checkUIViewPermissions("user5") should be (false)
}

// Test neither history ui admin acls nor application acls are configured.
val conf3 = createTestConf()
.set("spark.history.ui.acls.enable", "true")
.set("spark.user.groups.mapping", classOf[TestGroupsMappingProvider].getName)
createAndCheck(conf3) { securityManager =>
// Test whether user has permission to access UI.
securityManager.checkUIViewPermissions("user1") should be (false)
securityManager.checkUIViewPermissions("user2") should be (false)
securityManager.checkUIViewPermissions("user") should be (false)

// Test whether user with admin group has permission to access UI.
// Check should be failed since we don't have acl group settings.
securityManager.checkUIViewPermissions("user3") should be (false)
securityManager.checkUIViewPermissions("user4") should be (false)
securityManager.checkUIViewPermissions("user5") should be (false)
}
}

/**
* Asks the provider to check for logs and calls a function to perform checks on the updated
* app list. Example:
Expand Down Expand Up @@ -532,3 +629,15 @@ class FsHistoryProviderSuite extends SparkFunSuite with BeforeAndAfter with Matc
}

}

class TestGroupsMappingProvider extends GroupMappingServiceProvider {
private val mappings = Map(
"user3" -> "group1",
"user4" -> "group1",
"user5" -> "group")

override def getGroups(username: String): Set[String] = {
mappings.get(username).map(Set(_)).getOrElse(Set.empty)
}
}

22 changes: 22 additions & 0 deletions docs/monitoring.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,28 @@ The history server can be configured as follows:
If disabled, no access control checks are made.
</td>
</tr>
<tr>
<td>spark.history.ui.admin.acls</td>
<td>empty</td>
<td>
Comma separated list of users/administrators that have view access to all the Spark applications in
history server. By default only the users permitted to view the application at run-time could
access the related application history, with this, configured users/administrators could also
have the permission to access it.
Putting a "*" in the list means any user can have the privilege of admin.
</td>
</tr>
<tr>
<td>spark.history.ui.admin.acls.groups</td>
<td>empty</td>
<td>
Comma separated list of groups that have view access to all the Spark applications in
history server. By default only the groups permitted to view the application at run-time could
access the related application history, with this, configured groups could also
have the permission to access it.
Putting a "*" in the list means any group can have the privilege of admin.
</td>
</tr>
<tr>
<td>spark.history.fs.cleaner.enabled</td>
<td>false</td>
Expand Down