From bb5928f9bc5335e66e0b2157b7b0ff89d20c2b15 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 14 Sep 2018 15:25:53 +0300 Subject: [PATCH 01/10] Structured audit logging (#31931) Changes the format of log events in the audit logfile. It also changes the filename suffix from `_access` to `_audit`. The new entry format is consistent with Elastic Common Schema. Entries are formatted as JSON with no nested objects and field names have a dotted syntax. Moreover, log entries themselves are not spaced by commas and there is exactly one entry per line. In addition, entry fields are ordered, unlike a typical JSON doc, such that a human would not strain his eyes over jumbled fields from one line to the other; the order is defined in the log4j2 properties file. The implementation utilizes the log4j2's `StringMapMessage`. This means that the application builds the log event as a map and the log4j logic (the appender's layout) handle the format internally. The layout, such as the set of printed fields and their order, can be changed at runtime without restarting the node. --- .../core/src/main/config/log4j2.properties | 61 +- .../audit/logfile/CapturingLogger.java | 68 +- x-pack/plugin/security/build.gradle | 1 + .../audit/logfile/LoggingAuditTrail.java | 825 +++++++------ .../test/SecuritySettingsSource.java | 7 +- .../AuditTrailSettingsUpdateTests.java | 57 +- .../logfile/LoggingAuditTrailFilterTests.java | 24 +- .../audit/logfile/LoggingAuditTrailTests.java | 1053 ++++++++++------- .../authc/file/FileUserPasswdStoreTests.java | 8 +- .../authc/file/FileUserRolesStoreTests.java | 8 +- .../authc/support/DnRoleMapperTests.java | 12 +- .../authz/store/FileRolesStoreTests.java | 27 +- x-pack/qa/sql/security/build.gradle | 2 +- .../qa/sql/security/RestSqlSecurityIT.java | 12 +- .../qa/sql/security/SqlSecurityTestCase.java | 103 +- 15 files changed, 1386 insertions(+), 882 deletions(-) diff --git a/x-pack/plugin/core/src/main/config/log4j2.properties b/x-pack/plugin/core/src/main/config/log4j2.properties index c4cdbc0640c85..1c9358f3cc490 100644 --- a/x-pack/plugin/core/src/main/config/log4j2.properties +++ b/x-pack/plugin/core/src/main/config/log4j2.properties @@ -1,9 +1,64 @@ appender.audit_rolling.type = RollingFile appender.audit_rolling.name = audit_rolling -appender.audit_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_access.log +appender.audit_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_audit.log appender.audit_rolling.layout.type = PatternLayout -appender.audit_rolling.layout.pattern = [%d{ISO8601}] %m%n -appender.audit_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_access-%d{yyyy-MM-dd}.log +appender.audit_rolling.layout.pattern = {\ + "@timestamp":"%d{ISO8601}"\ + %varsNotEmpty{, "node.name":"%enc{%map{node.name}}{JSON}"}\ + %varsNotEmpty{, "node.id":"%enc{%map{node.id}}{JSON}"}\ + %varsNotEmpty{, "host.name":"%enc{%map{host.name}}{JSON}"}\ + %varsNotEmpty{, "host.ip":"%enc{%map{host.ip}}{JSON}"}\ + %varsNotEmpty{, "event.type":"%enc{%map{event.type}}{JSON}"}\ + %varsNotEmpty{, "event.action":"%enc{%map{event.action}}{JSON}"}\ + %varsNotEmpty{, "user.name":"%enc{%map{user.name}}{JSON}"}\ + %varsNotEmpty{, "user.run_by.name":"%enc{%map{user.run_by.name}}{JSON}"}\ + %varsNotEmpty{, "user.run_as.name":"%enc{%map{user.run_as.name}}{JSON}"}\ + %varsNotEmpty{, "user.realm":"%enc{%map{user.realm}}{JSON}"}\ + %varsNotEmpty{, "user.run_by.realm":"%enc{%map{user.run_by.realm}}{JSON}"}\ + %varsNotEmpty{, "user.run_as.realm":"%enc{%map{user.run_as.realm}}{JSON}"}\ + %varsNotEmpty{, "user.roles":%map{user.roles}}\ + %varsNotEmpty{, "origin.type":"%enc{%map{origin.type}}{JSON}"}\ + %varsNotEmpty{, "origin.address":"%enc{%map{origin.address}}{JSON}"}\ + %varsNotEmpty{, "realm":"%enc{%map{realm}}{JSON}"}\ + %varsNotEmpty{, "url.path":"%enc{%map{url.path}}{JSON}"}\ + %varsNotEmpty{, "url.query":"%enc{%map{url.query}}{JSON}"}\ + %varsNotEmpty{, "request.body":"%enc{%map{request.body}}{JSON}"}\ + %varsNotEmpty{, "action":"%enc{%map{action}}{JSON}"}\ + %varsNotEmpty{, "request.name":"%enc{%map{request.name}}{JSON}"}\ + %varsNotEmpty{, "indices":%map{indices}}\ + %varsNotEmpty{, "opaque_id":"%enc{%map{opaque_id}}{JSON}"}\ + %varsNotEmpty{, "transport.profile":"%enc{%map{transport.profile}}{JSON}"}\ + %varsNotEmpty{, "rule":"%enc{%map{rule}}{JSON}"}\ + %varsNotEmpty{, "event.category":"%enc{%map{event.category}}{JSON}"}\ + }%n +# "node.name" node name from the `elasticsearch.yml` settings +# "node.id" node id which should not change between cluster restarts +# "host.name" unresolved hostname of the local node +# "host.ip" the local bound ip (i.e. the ip listening for connections) +# "event.type" a received REST request is translated into one or more transport requests. This indicates which processing layer generated the event "rest" or "transport" (internal) +# "event.action" the name of the audited event, eg. "authentication_failed", "access_granted", "run_as_granted", etc. +# "user.name" the subject name as authenticated by a realm +# "user.run_by.name" the original authenticated subject name that is impersonating another one. +# "user.run_as.name" if this "event.action" is of a run_as type, this is the subject name to be impersonated as. +# "user.realm" the name of the realm that authenticated "user.name" +# "user.run_by.realm" the realm name of the impersonating subject ("user.run_by.name") +# "user.run_as.realm" if this "event.action" is of a run_as type, this is the realm name the impersonated user is looked up from +# "user.roles" the roles array of the user; these are the roles that are granting privileges +# "origin.type" it is "rest" if the event is originating (is in relation to) a REST request; possible other values are "transport" and "ip_filter" +# "origin.address" the remote address and port of the first network hop, i.e. a REST proxy or another cluster node +# "realm" name of a realm that has generated an "authentication_failed" or an "authentication_successful"; the subject is not yet authenticated +# "url.path" the URI component between the port and the query string; it is percent (URL) encoded +# "url.query" the URI component after the path and before the fragment; it is percent (URL) encoded +# "request.body" the content of the request body entity, JSON escaped +# "action" an action is the most granular operation that is authorized and this identifies it in a namespaced way (internal) +# "request.name" if the event is in connection to a transport message this is the name of the request class, similar to how rest requests are identified by the url path (internal) +# "indices" the array of indices that the "action" is acting upon +# "opaque_id" opaque value conveyed by the "X-Opaque-Id" request header +# "transport.profile" name of the transport profile in case this is a "connection_granted" or "connection_denied" event +# "rule" name of the applied rulee if the "origin.type" is "ip_filter" +# "event.category" fixed value "elasticsearch-audit" + +appender.audit_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_audit-%d{yyyy-MM-dd}.log appender.audit_rolling.policies.type = Policies appender.audit_rolling.policies.time.type = TimeBasedTriggeringPolicy appender.audit_rolling.policies.time.interval = 1 diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/audit/logfile/CapturingLogger.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/audit/logfile/CapturingLogger.java index 2091f8fb75fde..ede18c8241b34 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/audit/logfile/CapturingLogger.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/audit/logfile/CapturingLogger.java @@ -8,26 +8,51 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.Layout; import org.apache.logging.log4j.core.LogEvent; import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.StringLayout; import org.apache.logging.log4j.core.appender.AbstractAppender; import org.apache.logging.log4j.core.config.Configuration; import org.apache.logging.log4j.core.config.LoggerConfig; import org.apache.logging.log4j.core.filter.RegexFilter; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.logging.Loggers; import java.util.ArrayList; import java.util.List; +/** + * Logger that captures events and appends them to in memory lists, with one + * list for each log level. This works with the global log manager context, + * meaning that there could only be a single logger with the same name. + */ public class CapturingLogger { - public static Logger newCapturingLogger(final Level level) throws IllegalAccessException { + /** + * Constructs a new {@link CapturingLogger} named as the fully qualified name of + * the invoking method. One name can be assigned to a single logger globally, so + * don't call this method multiple times in the same method. + * + * @param level + * The minimum priority level of events that will be captured. + * @param layout + * Optional parameter allowing to set the layout format of events. + * This is useful because events are captured to be inspected (and + * parsed) later. When parsing, it is useful to be in control of the + * printing format as well. If not specified, + * {@code event.getMessage().getFormattedMessage()} is called to + * format the event. + * @return The new logger. + */ + public static Logger newCapturingLogger(final Level level, @Nullable StringLayout layout) throws IllegalAccessException { + // careful, don't "bury" this on the call stack, unless you know what you're doing final StackTraceElement caller = Thread.currentThread().getStackTrace()[2]; final String name = caller.getClassName() + "." + caller.getMethodName() + "." + level.toString(); final Logger logger = ESLoggerFactory.getLogger(name); Loggers.setLevel(logger, level); - final MockAppender appender = new MockAppender(name); + final MockAppender appender = new MockAppender(name, layout); appender.start(); Loggers.addAppender(logger, appender); return logger; @@ -40,11 +65,27 @@ private static MockAppender getMockAppender(final String name) { return (MockAppender) loggerConfig.getAppenders().get(name); } + /** + * Checks if the logger's appender has captured any events. + * + * @param name + * The unique global name of the logger. + * @return {@code true} if no event has been captured, {@code false} otherwise. + */ public static boolean isEmpty(final String name) { final MockAppender appender = getMockAppender(name); return appender.isEmpty(); } + /** + * Gets the captured events for a logger by its name. + * + * @param name + * The unique global name of the logger. + * @param level + * The priority level of the captured events to be returned. + * @return A list of captured events formated to {@code String}. + */ public static List output(final String name, final Level level) { final MockAppender appender = getMockAppender(name); return appender.output(level); @@ -58,8 +99,8 @@ private static class MockAppender extends AbstractAppender { public final List debug = new ArrayList<>(); public final List trace = new ArrayList<>(); - private MockAppender(final String name) throws IllegalAccessException { - super(name, RegexFilter.createFilter(".*(\n.*)*", new String[0], false, null, null), null); + private MockAppender(final String name, StringLayout layout) throws IllegalAccessException { + super(name, RegexFilter.createFilter(".*(\n.*)*", new String[0], false, null, null), layout); } @Override @@ -68,25 +109,34 @@ public void append(LogEvent event) { // we can not keep a reference to the event here because Log4j is using a thread // local instance under the hood case "ERROR": - error.add(event.getMessage().getFormattedMessage()); + error.add(formatMessage(event)); break; case "WARN": - warn.add(event.getMessage().getFormattedMessage()); + warn.add(formatMessage(event)); break; case "INFO": - info.add(event.getMessage().getFormattedMessage()); + info.add(formatMessage(event)); break; case "DEBUG": - debug.add(event.getMessage().getFormattedMessage()); + debug.add(formatMessage(event)); break; case "TRACE": - trace.add(event.getMessage().getFormattedMessage()); + trace.add(formatMessage(event)); break; default: throw invalidLevelException(event.getLevel()); } } + private String formatMessage(LogEvent event) { + final Layout layout = getLayout(); + if (layout instanceof StringLayout) { + return ((StringLayout) layout).toSerializable(event); + } else { + return event.getMessage().getFormattedMessage(); + } + } + private IllegalArgumentException invalidLevelException(Level level) { return new IllegalArgumentException("invalid level, expected [ERROR|WARN|INFO|DEBUG|TRACE] but was [" + level + "]"); } diff --git a/x-pack/plugin/security/build.gradle b/x-pack/plugin/security/build.gradle index 710f28722b620..9f88733404b4a 100644 --- a/x-pack/plugin/security/build.gradle +++ b/x-pack/plugin/security/build.gradle @@ -139,6 +139,7 @@ artifacts { } sourceSets.test.resources { srcDir '../core/src/test/resources' + srcDir '../core/src/main/config' } dependencyLicenses { mapping from: /java-support|opensaml-.*/, to: 'shibboleth' diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index 54e10925a985a..359fe7cd464a4 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -6,15 +6,16 @@ package org.elasticsearch.xpack.security.audit.logfile; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.StringMapMessage; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; @@ -37,12 +38,17 @@ import org.elasticsearch.xpack.security.rest.RemoteHostHeader; import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule; +import com.fasterxml.jackson.core.io.JsonStringEncoder; + +import org.apache.logging.log4j.LogManager; + import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -53,7 +59,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.elasticsearch.common.Strings.arrayToCommaDelimitedString; import static org.elasticsearch.xpack.core.security.SecurityField.setting; import static org.elasticsearch.xpack.security.audit.AuditLevel.ACCESS_DENIED; import static org.elasticsearch.xpack.security.audit.AuditLevel.ACCESS_GRANTED; @@ -72,55 +77,81 @@ public class LoggingAuditTrail extends AbstractComponent implements AuditTrail, ClusterStateListener { + public static final String REST_ORIGIN_FIELD_VALUE = "rest"; + public static final String LOCAL_ORIGIN_FIELD_VALUE = "local_node"; + public static final String TRANSPORT_ORIGIN_FIELD_VALUE = "transport"; + public static final String IP_FILTER_ORIGIN_FIELD_VALUE = "ip_filter"; + + // changing any of this names requires changing the log4j2.properties file too + public static final String ORIGIN_TYPE_FIELD_NAME = "origin.type"; + public static final String ORIGIN_ADDRESS_FIELD_NAME = "origin.address"; + public static final String NODE_NAME_FIELD_NAME = "node.name"; + public static final String NODE_ID_FIELD_NAME = "node.id"; + public static final String HOST_ADDRESS_FIELD_NAME = "host.ip"; + public static final String HOST_NAME_FIELD_NAME = "host.name"; + public static final String EVENT_TYPE_FIELD_NAME = "event.type"; + public static final String EVENT_ACTION_FIELD_NAME = "event.action"; + public static final String PRINCIPAL_FIELD_NAME = "user.name"; + public static final String PRINCIPAL_RUN_BY_FIELD_NAME = "user.run_by.name"; + public static final String PRINCIPAL_RUN_AS_FIELD_NAME = "user.run_as.name"; + public static final String PRINCIPAL_REALM_FIELD_NAME = "user.realm"; + public static final String PRINCIPAL_RUN_BY_REALM_FIELD_NAME = "user.run_by.realm"; + public static final String PRINCIPAL_RUN_AS_REALM_FIELD_NAME = "user.run_as.realm"; + public static final String PRINCIPAL_ROLES_FIELD_NAME = "user.roles"; + public static final String REALM_FIELD_NAME = "realm"; + public static final String URL_PATH_FIELD_NAME = "url.path"; + public static final String URL_QUERY_FIELD_NAME = "url.query"; + public static final String REQUEST_BODY_FIELD_NAME = "request.body"; + public static final String ACTION_FIELD_NAME = "action"; + public static final String INDICES_FIELD_NAME = "indices"; + public static final String REQUEST_NAME_FIELD_NAME = "request.name"; + public static final String TRANSPORT_PROFILE_FIELD_NAME = "transport.profile"; + public static final String RULE_FIELD_NAME = "rule"; + public static final String OPAQUE_ID_FIELD_NAME = "opaque_id"; + public static final String NAME = "logfile"; - public static final Setting HOST_ADDRESS_SETTING = - Setting.boolSetting(setting("audit.logfile.prefix.emit_node_host_address"), false, Property.NodeScope, Property.Dynamic); - public static final Setting HOST_NAME_SETTING = - Setting.boolSetting(setting("audit.logfile.prefix.emit_node_host_name"), false, Property.NodeScope, Property.Dynamic); - public static final Setting NODE_NAME_SETTING = - Setting.boolSetting(setting("audit.logfile.prefix.emit_node_name"), true, Property.NodeScope, Property.Dynamic); - private static final List DEFAULT_EVENT_INCLUDES = Arrays.asList( - ACCESS_DENIED.toString(), - ACCESS_GRANTED.toString(), - ANONYMOUS_ACCESS_DENIED.toString(), - AUTHENTICATION_FAILED.toString(), - CONNECTION_DENIED.toString(), - TAMPERED_REQUEST.toString(), - RUN_AS_DENIED.toString(), - RUN_AS_GRANTED.toString() - ); - public static final Setting> INCLUDE_EVENT_SETTINGS = - Setting.listSetting(setting("audit.logfile.events.include"), DEFAULT_EVENT_INCLUDES, Function.identity(), Property.NodeScope, - Property.Dynamic); - public static final Setting> EXCLUDE_EVENT_SETTINGS = - Setting.listSetting(setting("audit.logfile.events.exclude"), Collections.emptyList(), Function.identity(), Property.NodeScope, - Property.Dynamic); - public static final Setting INCLUDE_REQUEST_BODY = - Setting.boolSetting(setting("audit.logfile.events.emit_request_body"), false, Property.NodeScope, Property.Dynamic); + public static final Setting EMIT_HOST_ADDRESS_SETTING = Setting + .boolSetting(setting("audit.logfile.prefix.emit_node_host_address"), false, Property.NodeScope, Property.Dynamic); + public static final Setting EMIT_HOST_NAME_SETTING = Setting.boolSetting(setting("audit.logfile.prefix.emit_node_host_name"), + false, Property.NodeScope, Property.Dynamic); + public static final Setting EMIT_NODE_NAME_SETTING = Setting.boolSetting(setting("audit.logfile.prefix.emit_node_name"), false, + Property.NodeScope, Property.Dynamic); + public static final Setting EMIT_NODE_ID_SETTING = Setting.boolSetting(setting("audit.logfile.prefix.emit_node_id"), true, + Property.NodeScope, Property.Dynamic); + private static final List DEFAULT_EVENT_INCLUDES = Arrays.asList(ACCESS_DENIED.toString(), ACCESS_GRANTED.toString(), + ANONYMOUS_ACCESS_DENIED.toString(), AUTHENTICATION_FAILED.toString(), CONNECTION_DENIED.toString(), TAMPERED_REQUEST.toString(), + RUN_AS_DENIED.toString(), RUN_AS_GRANTED.toString()); + public static final Setting> INCLUDE_EVENT_SETTINGS = Setting.listSetting(setting("audit.logfile.events.include"), + DEFAULT_EVENT_INCLUDES, Function.identity(), Property.NodeScope, Property.Dynamic); + public static final Setting> EXCLUDE_EVENT_SETTINGS = Setting.listSetting(setting("audit.logfile.events.exclude"), + Collections.emptyList(), Function.identity(), Property.NodeScope, Property.Dynamic); + public static final Setting INCLUDE_REQUEST_BODY = Setting.boolSetting(setting("audit.logfile.events.emit_request_body"), + false, Property.NodeScope, Property.Dynamic); private static final String FILTER_POLICY_PREFIX = setting("audit.logfile.events.ignore_filters."); // because of the default wildcard value (*) for the field filter, a policy with // an unspecified filter field will match events that have any value for that // particular field, as well as events with that particular field missing - private static final Setting.AffixSetting> FILTER_POLICY_IGNORE_PRINCIPALS = - Setting.affixKeySetting(FILTER_POLICY_PREFIX, "users", (key) -> Setting.listSetting(key, Collections.singletonList("*"), - Function.identity(), Property.NodeScope, Property.Dynamic)); - private static final Setting.AffixSetting> FILTER_POLICY_IGNORE_REALMS = - Setting.affixKeySetting(FILTER_POLICY_PREFIX, "realms", (key) -> Setting.listSetting(key, Collections.singletonList("*"), - Function.identity(), Property.NodeScope, Property.Dynamic)); - private static final Setting.AffixSetting> FILTER_POLICY_IGNORE_ROLES = - Setting.affixKeySetting(FILTER_POLICY_PREFIX, "roles", (key) -> Setting.listSetting(key, Collections.singletonList("*"), - Function.identity(), Property.NodeScope, Property.Dynamic)); - private static final Setting.AffixSetting> FILTER_POLICY_IGNORE_INDICES = - Setting.affixKeySetting(FILTER_POLICY_PREFIX, "indices", (key) -> Setting.listSetting(key, Collections.singletonList("*"), - Function.identity(), Property.NodeScope, Property.Dynamic)); + private static final Setting.AffixSetting> FILTER_POLICY_IGNORE_PRINCIPALS = Setting.affixKeySetting(FILTER_POLICY_PREFIX, + "users", + (key) -> Setting.listSetting(key, Collections.singletonList("*"), Function.identity(), Property.NodeScope, Property.Dynamic)); + private static final Setting.AffixSetting> FILTER_POLICY_IGNORE_REALMS = Setting.affixKeySetting(FILTER_POLICY_PREFIX, + "realms", + (key) -> Setting.listSetting(key, Collections.singletonList("*"), Function.identity(), Property.NodeScope, Property.Dynamic)); + private static final Setting.AffixSetting> FILTER_POLICY_IGNORE_ROLES = Setting.affixKeySetting(FILTER_POLICY_PREFIX, + "roles", + (key) -> Setting.listSetting(key, Collections.singletonList("*"), Function.identity(), Property.NodeScope, Property.Dynamic)); + private static final Setting.AffixSetting> FILTER_POLICY_IGNORE_INDICES = Setting.affixKeySetting(FILTER_POLICY_PREFIX, + "indices", + (key) -> Setting.listSetting(key, Collections.singletonList("*"), Function.identity(), Property.NodeScope, Property.Dynamic)); private final Logger logger; - final EventFilterPolicyRegistry eventFilterPolicyRegistry; private final ThreadContext threadContext; + final EventFilterPolicyRegistry eventFilterPolicyRegistry; // package for testing volatile EnumSet events; boolean includeRequestBody; - LocalNodeInfo localNodeInfo; + // fields that all entries have in common + EntryCommonFields entryCommonFields; @Override public String name() { @@ -128,7 +159,7 @@ public String name() { } public LoggingAuditTrail(Settings settings, ClusterService clusterService, ThreadPool threadPool) { - this(settings, clusterService, Loggers.getLogger(LoggingAuditTrail.class), threadPool.getThreadContext()); + this(settings, clusterService, LogManager.getLogger(), threadPool.getThreadContext()); } LoggingAuditTrail(Settings settings, ClusterService clusterService, Logger logger, ThreadContext threadContext) { @@ -137,20 +168,18 @@ public LoggingAuditTrail(Settings settings, ClusterService clusterService, Threa this.events = parse(INCLUDE_EVENT_SETTINGS.get(settings), EXCLUDE_EVENT_SETTINGS.get(settings)); this.includeRequestBody = INCLUDE_REQUEST_BODY.get(settings); this.threadContext = threadContext; - this.localNodeInfo = new LocalNodeInfo(settings, null); + this.entryCommonFields = new EntryCommonFields(settings, null); this.eventFilterPolicyRegistry = new EventFilterPolicyRegistry(settings); clusterService.addListener(this); clusterService.getClusterSettings().addSettingsUpdateConsumer(newSettings -> { - final LocalNodeInfo localNodeInfo = this.localNodeInfo; - final Settings.Builder builder = Settings.builder().put(localNodeInfo.settings).put(newSettings, false); - this.localNodeInfo = new LocalNodeInfo(builder.build(), localNodeInfo.localNode); + this.entryCommonFields = this.entryCommonFields.withNewSettings(newSettings); this.includeRequestBody = INCLUDE_REQUEST_BODY.get(newSettings); // `events` is a volatile field! Keep `events` write last so that - // `localNodeInfo` and `includeRequestBody` writes happen-before! `events` is - // always read before `localNodeInfo` and `includeRequestBody`. + // `entryCommonFields` and `includeRequestBody` writes happen-before! `events` is + // always read before `entryCommonFields` and `includeRequestBody`. this.events = parse(INCLUDE_EVENT_SETTINGS.get(newSettings), EXCLUDE_EVENT_SETTINGS.get(newSettings)); - }, Arrays.asList(HOST_ADDRESS_SETTING, HOST_NAME_SETTING, NODE_NAME_SETTING, INCLUDE_EVENT_SETTINGS, EXCLUDE_EVENT_SETTINGS, - INCLUDE_REQUEST_BODY)); + }, Arrays.asList(EMIT_HOST_ADDRESS_SETTING, EMIT_HOST_NAME_SETTING, EMIT_NODE_NAME_SETTING, EMIT_NODE_ID_SETTING, + INCLUDE_EVENT_SETTINGS, EXCLUDE_EVENT_SETTINGS, INCLUDE_REQUEST_BODY)); clusterService.getClusterSettings().addAffixUpdateConsumer(FILTER_POLICY_IGNORE_PRINCIPALS, (policyName, filtersList) -> { final Optional policy = eventFilterPolicyRegistry.get(policyName); final EventFilterPolicy newPolicy = policy.orElse(new EventFilterPolicy(policyName, settings)) @@ -159,35 +188,36 @@ public LoggingAuditTrail(Settings settings, ClusterService clusterService, Threa }, (policyName, filtersList) -> EventFilterPolicy.parsePredicate(filtersList)); clusterService.getClusterSettings().addAffixUpdateConsumer(FILTER_POLICY_IGNORE_REALMS, (policyName, filtersList) -> { final Optional policy = eventFilterPolicyRegistry.get(policyName); - final EventFilterPolicy newPolicy = policy.orElse(new EventFilterPolicy(policyName, settings)) - .changeRealmsFilter(filtersList); + final EventFilterPolicy newPolicy = policy.orElse(new EventFilterPolicy(policyName, settings)).changeRealmsFilter(filtersList); this.eventFilterPolicyRegistry.set(policyName, newPolicy); }, (policyName, filtersList) -> EventFilterPolicy.parsePredicate(filtersList)); clusterService.getClusterSettings().addAffixUpdateConsumer(FILTER_POLICY_IGNORE_ROLES, (policyName, filtersList) -> { final Optional policy = eventFilterPolicyRegistry.get(policyName); - final EventFilterPolicy newPolicy = policy.orElse(new EventFilterPolicy(policyName, settings)) - .changeRolesFilter(filtersList); + final EventFilterPolicy newPolicy = policy.orElse(new EventFilterPolicy(policyName, settings)).changeRolesFilter(filtersList); this.eventFilterPolicyRegistry.set(policyName, newPolicy); }, (policyName, filtersList) -> EventFilterPolicy.parsePredicate(filtersList)); clusterService.getClusterSettings().addAffixUpdateConsumer(FILTER_POLICY_IGNORE_INDICES, (policyName, filtersList) -> { final Optional policy = eventFilterPolicyRegistry.get(policyName); - final EventFilterPolicy newPolicy = policy.orElse(new EventFilterPolicy(policyName, settings)) - .changeIndicesFilter(filtersList); + final EventFilterPolicy newPolicy = policy.orElse(new EventFilterPolicy(policyName, settings)).changeIndicesFilter(filtersList); this.eventFilterPolicyRegistry.set(policyName, newPolicy); }, (policyName, filtersList) -> EventFilterPolicy.parsePredicate(filtersList)); } @Override public void authenticationSuccess(String realm, User user, RestRequest request) { - if (events.contains(AUTHENTICATION_SUCCESS) && (eventFilterPolicyRegistry.ignorePredicate() - .test(new AuditEventMetaInfo(Optional.of(user), Optional.of(realm), Optional.empty(), Optional.empty())) == false)) { - if (includeRequestBody) { - logger.info("{}[rest] [authentication_success]\t{}, realm=[{}], uri=[{}], params=[{}]{}, request_body=[{}]", - localNodeInfo.prefix, principal(user), realm, request.uri(), request.params(), opaqueId(), restRequestContent(request)); - } else { - logger.info("{}[rest] [authentication_success]\t{}, realm=[{}], uri=[{}], params=[{}]{}", - localNodeInfo.prefix, principal(user), realm, request.uri(), request.params(), opaqueId()); - } + if (events.contains(AUTHENTICATION_SUCCESS) && eventFilterPolicyRegistry.ignorePredicate() + .test(new AuditEventMetaInfo(Optional.of(user), Optional.of(realm), Optional.empty(), Optional.empty())) == false) { + final StringMapMessage logEntry = new LogEntryBuilder() + .with(EVENT_TYPE_FIELD_NAME, REST_ORIGIN_FIELD_VALUE) + .with(EVENT_ACTION_FIELD_NAME, "authentication_success") + .with(REALM_FIELD_NAME, realm) + .withRestUri(request) + .withPrincipal(user) + .withRestOrigin(request) + .withRequestBody(request) + .withOpaqueId(threadContext) + .build(); + logger.info(logEntry); } } @@ -197,16 +227,18 @@ public void authenticationSuccess(String realm, User user, String action, Transp final Optional indices = indices(message); if (eventFilterPolicyRegistry.ignorePredicate() .test(new AuditEventMetaInfo(Optional.of(user), Optional.of(realm), Optional.empty(), indices)) == false) { - final LocalNodeInfo localNodeInfo = this.localNodeInfo; - if (indices.isPresent()) { - logger.info("{}[transport] [authentication_success]\t{}, {}, realm=[{}], action=[{}], indices=[{}], request=[{}]{}", - localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), principal(user), realm, action, - arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName(), opaqueId()); - } else { - logger.info("{}[transport] [authentication_success]\t{}, {}, realm=[{}], action=[{}], request=[{}]{}", - localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), principal(user), realm, action, - message.getClass().getSimpleName(), opaqueId()); - } + final StringMapMessage logEntry = new LogEntryBuilder() + .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE) + .with(EVENT_ACTION_FIELD_NAME, "authentication_success") + .with(REALM_FIELD_NAME, realm) + .with(ACTION_FIELD_NAME, action) + .with(REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()) + .withPrincipal(user) + .withRestOrTransportOrigin(message, threadContext) + .with(INDICES_FIELD_NAME, indices.orElse(null)) + .withOpaqueId(threadContext) + .build(); + logger.info(logEntry); } } } @@ -217,16 +249,16 @@ public void anonymousAccessDenied(String action, TransportMessage message) { final Optional indices = indices(message); if (eventFilterPolicyRegistry.ignorePredicate() .test(new AuditEventMetaInfo(Optional.empty(), Optional.empty(), indices)) == false) { - final LocalNodeInfo localNodeInfo = this.localNodeInfo; - if (indices.isPresent()) { - logger.info("{}[transport] [anonymous_access_denied]\t{}, action=[{}], indices=[{}], request=[{}]{}", - localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), action, - arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName(), opaqueId()); - } else { - logger.info("{}[transport] [anonymous_access_denied]\t{}, action=[{}], request=[{}]{}", - localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), action, - message.getClass().getSimpleName(), opaqueId()); - } + final StringMapMessage logEntry = new LogEntryBuilder() + .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE) + .with(EVENT_ACTION_FIELD_NAME, "anonymous_access_denied") + .with(ACTION_FIELD_NAME, action) + .with(REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()) + .withRestOrTransportOrigin(message, threadContext) + .with(INDICES_FIELD_NAME, indices.orElse(null)) + .withOpaqueId(threadContext) + .build(); + logger.info(logEntry); } } } @@ -234,14 +266,16 @@ localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), a @Override public void anonymousAccessDenied(RestRequest request) { if (events.contains(ANONYMOUS_ACCESS_DENIED) - && (eventFilterPolicyRegistry.ignorePredicate().test(AuditEventMetaInfo.EMPTY) == false)) { - if (includeRequestBody) { - logger.info("{}[rest] [anonymous_access_denied]\t{}, uri=[{}]{}, request_body=[{}]", localNodeInfo.prefix, - hostAttributes(request), request.uri(), opaqueId(), restRequestContent(request)); - } else { - logger.info("{}[rest] [anonymous_access_denied]\t{}, uri=[{}]{}", localNodeInfo.prefix, - hostAttributes(request), request.uri(), opaqueId()); - } + && eventFilterPolicyRegistry.ignorePredicate().test(AuditEventMetaInfo.EMPTY) == false) { + final StringMapMessage logEntry = new LogEntryBuilder() + .with(EVENT_TYPE_FIELD_NAME, REST_ORIGIN_FIELD_VALUE) + .with(EVENT_ACTION_FIELD_NAME, "anonymous_access_denied") + .withRestUri(request) + .withRestOrigin(request) + .withRequestBody(request) + .withOpaqueId(threadContext) + .build(); + logger.info(logEntry); } } @@ -251,31 +285,33 @@ public void authenticationFailed(AuthenticationToken token, String action, Trans final Optional indices = indices(message); if (eventFilterPolicyRegistry.ignorePredicate() .test(new AuditEventMetaInfo(Optional.of(token), Optional.empty(), indices)) == false) { - final LocalNodeInfo localNodeInfo = this.localNodeInfo; - if (indices.isPresent()) { - logger.info("{}[transport] [authentication_failed]\t{}, principal=[{}], action=[{}], indices=[{}], request=[{}]{}", - localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), token.principal(), action, - arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName(), opaqueId()); - } else { - logger.info("{}[transport] [authentication_failed]\t{}, principal=[{}], action=[{}], request=[{}]{}", - localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), token.principal(), action, - message.getClass().getSimpleName(), opaqueId()); - } + final StringMapMessage logEntry = new LogEntryBuilder() + .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE) + .with(EVENT_ACTION_FIELD_NAME, "authentication_failed") + .with(ACTION_FIELD_NAME, action) + .with(PRINCIPAL_FIELD_NAME, token.principal()) + .with(REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()) + .withRestOrTransportOrigin(message, threadContext) + .with(INDICES_FIELD_NAME, indices.orElse(null)) + .withOpaqueId(threadContext) + .build(); + logger.info(logEntry); } } } @Override public void authenticationFailed(RestRequest request) { - if (events.contains(AUTHENTICATION_FAILED) - && (eventFilterPolicyRegistry.ignorePredicate().test(AuditEventMetaInfo.EMPTY) == false)) { - if (includeRequestBody) { - logger.info("{}[rest] [authentication_failed]\t{}, uri=[{}]{}, request_body=[{}]", localNodeInfo.prefix, - hostAttributes(request), request.uri(), opaqueId(), restRequestContent(request)); - } else { - logger.info("{}[rest] [authentication_failed]\t{}, uri=[{}]{}", localNodeInfo.prefix, - hostAttributes(request), request.uri(), opaqueId()); - } + if (events.contains(AUTHENTICATION_FAILED) && eventFilterPolicyRegistry.ignorePredicate().test(AuditEventMetaInfo.EMPTY) == false) { + final StringMapMessage logEntry = new LogEntryBuilder() + .with(EVENT_TYPE_FIELD_NAME, REST_ORIGIN_FIELD_VALUE) + .with(EVENT_ACTION_FIELD_NAME, "authentication_failed") + .withRestUri(request) + .withRestOrigin(request) + .withRequestBody(request) + .withOpaqueId(threadContext) + .build(); + logger.info(logEntry); } } @@ -285,33 +321,34 @@ public void authenticationFailed(String action, TransportMessage message) { final Optional indices = indices(message); if (eventFilterPolicyRegistry.ignorePredicate() .test(new AuditEventMetaInfo(Optional.empty(), Optional.empty(), indices)) == false) { - final LocalNodeInfo localNodeInfo = this.localNodeInfo; - if (indices.isPresent()) { - logger.info("{}[transport] [authentication_failed]\t{}, action=[{}], indices=[{}], request=[{}]{}", - localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), action, - arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName(), opaqueId()); - } else { - logger.info("{}[transport] [authentication_failed]\t{}, action=[{}], request=[{}]{}", - localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), action, - message.getClass().getSimpleName(), opaqueId()); - } + final StringMapMessage logEntry = new LogEntryBuilder() + .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE) + .with(EVENT_ACTION_FIELD_NAME, "authentication_failed") + .with(ACTION_FIELD_NAME, action) + .with(REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()) + .withRestOrTransportOrigin(message, threadContext) + .with(INDICES_FIELD_NAME, indices.orElse(null)) + .withOpaqueId(threadContext) + .build(); + logger.info(logEntry); } } } @Override public void authenticationFailed(AuthenticationToken token, RestRequest request) { - if (events.contains(AUTHENTICATION_FAILED) - && (eventFilterPolicyRegistry.ignorePredicate() - .test(new AuditEventMetaInfo(Optional.of(token), Optional.empty(), Optional.empty())) == false)) { - if (includeRequestBody) { - logger.info("{}[rest] [authentication_failed]\t{}, principal=[{}], uri=[{}]{}, request_body=[{}]", - localNodeInfo.prefix, hostAttributes(request), token.principal(), request.uri(), opaqueId(), - restRequestContent(request)); - } else { - logger.info("{}[rest] [authentication_failed]\t{}, principal=[{}], uri=[{}]{}", - localNodeInfo.prefix, hostAttributes(request), token.principal(), request.uri(), opaqueId()); - } + if (events.contains(AUTHENTICATION_FAILED) && eventFilterPolicyRegistry.ignorePredicate() + .test(new AuditEventMetaInfo(Optional.of(token), Optional.empty(), Optional.empty())) == false) { + final StringMapMessage logEntry = new LogEntryBuilder() + .with(EVENT_TYPE_FIELD_NAME, REST_ORIGIN_FIELD_VALUE) + .with(EVENT_ACTION_FIELD_NAME, "authentication_failed") + .with(PRINCIPAL_FIELD_NAME, token.principal()) + .withRestUri(request) + .withRestOrigin(request) + .withRequestBody(request) + .withOpaqueId(threadContext) + .build(); + logger.info(logEntry); } } @@ -321,36 +358,37 @@ public void authenticationFailed(String realm, AuthenticationToken token, String final Optional indices = indices(message); if (eventFilterPolicyRegistry.ignorePredicate() .test(new AuditEventMetaInfo(Optional.of(token), Optional.of(realm), indices)) == false) { - final LocalNodeInfo localNodeInfo = this.localNodeInfo; - if (indices.isPresent()) { - logger.info( - "{}[transport] [realm_authentication_failed]\trealm=[{}], {}, principal=[{}], action=[{}], indices=[{}], " - + "request=[{}]{}", - localNodeInfo.prefix, realm, originAttributes(threadContext, message, localNodeInfo), token.principal(), action, - arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName(), opaqueId()); - } else { - logger.info("{}[transport] [realm_authentication_failed]\trealm=[{}], {}, principal=[{}], action=[{}], request=[{}]{}", - localNodeInfo.prefix, realm, originAttributes(threadContext, message, localNodeInfo), token.principal(), action, - message.getClass().getSimpleName(), opaqueId()); - } + final StringMapMessage logEntry = new LogEntryBuilder() + .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE) + .with(EVENT_ACTION_FIELD_NAME, "realm_authentication_failed") + .with(REALM_FIELD_NAME, realm) + .with(PRINCIPAL_FIELD_NAME, token.principal()) + .with(ACTION_FIELD_NAME, action) + .with(REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()) + .withRestOrTransportOrigin(message, threadContext) + .with(INDICES_FIELD_NAME, indices.orElse(null)) + .withOpaqueId(threadContext) + .build(); + logger.info(logEntry); } } } @Override public void authenticationFailed(String realm, AuthenticationToken token, RestRequest request) { - if (events.contains(REALM_AUTHENTICATION_FAILED) - && (eventFilterPolicyRegistry.ignorePredicate() - .test(new AuditEventMetaInfo(Optional.of(token), Optional.of(realm), Optional.empty())) == false)) { - if (includeRequestBody) { - logger.info("{}[rest] [realm_authentication_failed]\trealm=[{}], {}, principal=[{}], uri=[{}]{}, " - + "request_body=[{}]", - localNodeInfo.prefix, realm, hostAttributes(request), token.principal(), request.uri(), opaqueId(), - restRequestContent(request)); - } else { - logger.info("{}[rest] [realm_authentication_failed]\trealm=[{}], {}, principal=[{}], uri=[{}]{}", - localNodeInfo.prefix, realm, hostAttributes(request), token.principal(), request.uri(), opaqueId()); - } + if (events.contains(REALM_AUTHENTICATION_FAILED) && eventFilterPolicyRegistry.ignorePredicate() + .test(new AuditEventMetaInfo(Optional.of(token), Optional.of(realm), Optional.empty())) == false) { + final StringMapMessage logEntry = new LogEntryBuilder() + .with(EVENT_TYPE_FIELD_NAME, REST_ORIGIN_FIELD_VALUE) + .with(EVENT_ACTION_FIELD_NAME, "realm_authentication_failed") + .with(REALM_FIELD_NAME, realm) + .with(PRINCIPAL_FIELD_NAME, token.principal()) + .withRestUri(request) + .withRestOrigin(request) + .withRequestBody(request) + .withOpaqueId(threadContext) + .build(); + logger.info(logEntry); } } @@ -362,17 +400,18 @@ public void accessGranted(Authentication authentication, String action, Transpor final Optional indices = indices(message); if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(user), Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), indices)) == false) { - final LocalNodeInfo localNodeInfo = this.localNodeInfo; - if (indices.isPresent()) { - logger.info("{}[transport] [access_granted]\t{}, {}, roles=[{}], action=[{}], indices=[{}], request=[{}]{}", - localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), subject(authentication), - arrayToCommaDelimitedString(roleNames), action, arrayToCommaDelimitedString(indices.get()), - message.getClass().getSimpleName(), opaqueId()); - } else { - logger.info("{}[transport] [access_granted]\t{}, {}, roles=[{}], action=[{}], request=[{}]{}", - localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), subject(authentication), - arrayToCommaDelimitedString(roleNames), action, message.getClass().getSimpleName(), opaqueId()); - } + final StringMapMessage logEntry = new LogEntryBuilder() + .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE) + .with(EVENT_ACTION_FIELD_NAME, "access_granted") + .with(ACTION_FIELD_NAME, action) + .with(REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()) + .withSubject(authentication) + .withRestOrTransportOrigin(message, threadContext) + .with(INDICES_FIELD_NAME, indices.orElse(null)) + .with(PRINCIPAL_ROLES_FIELD_NAME, roleNames) + .withOpaqueId(threadContext) + .build(); + logger.info(logEntry); } } } @@ -383,31 +422,34 @@ public void accessDenied(Authentication authentication, String action, Transport final Optional indices = indices(message); if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()), Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), indices)) == false) { - final LocalNodeInfo localNodeInfo = this.localNodeInfo; - if (indices.isPresent()) { - logger.info("{}[transport] [access_denied]\t{}, {}, roles=[{}], action=[{}], indices=[{}], request=[{}]{}", - localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), subject(authentication), - arrayToCommaDelimitedString(roleNames), action, arrayToCommaDelimitedString(indices.get()), - message.getClass().getSimpleName(), opaqueId()); - } else { - logger.info("{}[transport] [access_denied]\t{}, {}, roles=[{}], action=[{}], request=[{}]{}", - localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), subject(authentication), - arrayToCommaDelimitedString(roleNames), action, message.getClass().getSimpleName(), opaqueId()); - } + final StringMapMessage logEntry = new LogEntryBuilder() + .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE) + .with(EVENT_ACTION_FIELD_NAME, "access_denied") + .with(ACTION_FIELD_NAME, action) + .with(REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()) + .withSubject(authentication) + .withRestOrTransportOrigin(message, threadContext) + .with(INDICES_FIELD_NAME, indices.orElse(null)) + .with(PRINCIPAL_ROLES_FIELD_NAME, roleNames) + .withOpaqueId(threadContext) + .build(); + logger.info(logEntry); } } } @Override public void tamperedRequest(RestRequest request) { - if (events.contains(TAMPERED_REQUEST) && (eventFilterPolicyRegistry.ignorePredicate().test(AuditEventMetaInfo.EMPTY) == false)) { - if (includeRequestBody) { - logger.info("{}[rest] [tampered_request]\t{}, uri=[{}]{}, request_body=[{}]", localNodeInfo.prefix, - hostAttributes(request), request.uri(), opaqueId(), restRequestContent(request)); - } else { - logger.info("{}[rest] [tampered_request]\t{}, uri=[{}]{}", localNodeInfo.prefix, hostAttributes(request), - request.uri(), opaqueId()); - } + if (events.contains(TAMPERED_REQUEST) && eventFilterPolicyRegistry.ignorePredicate().test(AuditEventMetaInfo.EMPTY) == false) { + final StringMapMessage logEntry = new LogEntryBuilder() + .with(EVENT_TYPE_FIELD_NAME, REST_ORIGIN_FIELD_VALUE) + .with(EVENT_ACTION_FIELD_NAME, "tampered_request") + .withRestUri(request) + .withRestOrigin(request) + .withRequestBody(request) + .withOpaqueId(threadContext) + .build(); + logger.info(logEntry); } } @@ -417,53 +459,70 @@ public void tamperedRequest(String action, TransportMessage message) { final Optional indices = indices(message); if (eventFilterPolicyRegistry.ignorePredicate() .test(new AuditEventMetaInfo(Optional.empty(), Optional.empty(), indices)) == false) { - final LocalNodeInfo localNodeInfo = this.localNodeInfo; - if (indices.isPresent()) { - logger.info("{}[transport] [tampered_request]\t{}, action=[{}], indices=[{}], request=[{}]{}", - localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), action, - arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName(), opaqueId()); - } else { - logger.info("{}[transport] [tampered_request]\t{}, action=[{}], request=[{}]{}", localNodeInfo.prefix, - originAttributes(threadContext, message, localNodeInfo), action, message.getClass().getSimpleName(), - opaqueId()); - } + final StringMapMessage logEntry = new LogEntryBuilder() + .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE) + .with(EVENT_ACTION_FIELD_NAME, "tampered_request") + .with(ACTION_FIELD_NAME, action) + .with(REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()) + .withRestOrTransportOrigin(message, threadContext) + .with(INDICES_FIELD_NAME, indices.orElse(null)) + .withOpaqueId(threadContext) + .build(); + logger.info(logEntry); } } } @Override - public void tamperedRequest(User user, String action, TransportMessage request) { + public void tamperedRequest(User user, String action, TransportMessage message) { if (events.contains(TAMPERED_REQUEST)) { - final Optional indices = indices(request); + final Optional indices = indices(message); if (eventFilterPolicyRegistry.ignorePredicate() .test(new AuditEventMetaInfo(Optional.of(user), Optional.empty(), Optional.empty(), indices)) == false) { - final LocalNodeInfo localNodeInfo = this.localNodeInfo; - if (indices.isPresent()) { - logger.info("{}[transport] [tampered_request]\t{}, {}, action=[{}], indices=[{}], request=[{}]{}", - localNodeInfo.prefix, originAttributes(threadContext, request, localNodeInfo), principal(user), action, - arrayToCommaDelimitedString(indices.get()), request.getClass().getSimpleName(), opaqueId()); - } else { - logger.info("{}[transport] [tampered_request]\t{}, {}, action=[{}], request=[{}]{}", localNodeInfo.prefix, - originAttributes(threadContext, request, localNodeInfo), principal(user), action, - request.getClass().getSimpleName(), opaqueId()); - } + final StringMapMessage logEntry = new LogEntryBuilder() + .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE) + .with(EVENT_ACTION_FIELD_NAME, "tampered_request") + .with(ACTION_FIELD_NAME, action) + .with(REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()) + .withRestOrTransportOrigin(message, threadContext) + .withPrincipal(user) + .with(INDICES_FIELD_NAME, indices.orElse(null)) + .withOpaqueId(threadContext) + .build(); + logger.info(logEntry); } } } @Override public void connectionGranted(InetAddress inetAddress, String profile, SecurityIpFilterRule rule) { - if (events.contains(CONNECTION_GRANTED) && (eventFilterPolicyRegistry.ignorePredicate().test(AuditEventMetaInfo.EMPTY) == false)) { - logger.info("{}[ip_filter] [connection_granted]\torigin_address=[{}], transport_profile=[{}], rule=[{}]{}", - localNodeInfo.prefix, NetworkAddress.format(inetAddress), profile, rule, opaqueId()); + if (events.contains(CONNECTION_GRANTED) && eventFilterPolicyRegistry.ignorePredicate().test(AuditEventMetaInfo.EMPTY) == false) { + final StringMapMessage logEntry = new LogEntryBuilder() + .with(EVENT_TYPE_FIELD_NAME, IP_FILTER_ORIGIN_FIELD_VALUE) + .with(EVENT_ACTION_FIELD_NAME, "connection_granted") + .with(ORIGIN_TYPE_FIELD_NAME, IP_FILTER_ORIGIN_FIELD_VALUE) + .with(ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(inetAddress)) + .with(TRANSPORT_PROFILE_FIELD_NAME, profile) + .with(RULE_FIELD_NAME, rule.toString()) + .withOpaqueId(threadContext) + .build(); + logger.info(logEntry); } } @Override public void connectionDenied(InetAddress inetAddress, String profile, SecurityIpFilterRule rule) { - if (events.contains(CONNECTION_DENIED) && (eventFilterPolicyRegistry.ignorePredicate().test(AuditEventMetaInfo.EMPTY) == false)) { - logger.info("{}[ip_filter] [connection_denied]\torigin_address=[{}], transport_profile=[{}], rule=[{}]{}", - localNodeInfo.prefix, NetworkAddress.format(inetAddress), profile, rule, opaqueId()); + if (events.contains(CONNECTION_DENIED) && eventFilterPolicyRegistry.ignorePredicate().test(AuditEventMetaInfo.EMPTY) == false) { + final StringMapMessage logEntry = new LogEntryBuilder() + .with(EVENT_TYPE_FIELD_NAME, IP_FILTER_ORIGIN_FIELD_VALUE) + .with(EVENT_ACTION_FIELD_NAME, "connection_denied") + .with(ORIGIN_TYPE_FIELD_NAME, IP_FILTER_ORIGIN_FIELD_VALUE) + .with(ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(inetAddress)) + .with(TRANSPORT_PROFILE_FIELD_NAME, profile) + .with(RULE_FIELD_NAME, rule.toString()) + .withOpaqueId(threadContext) + .build(); + logger.info(logEntry); } } @@ -473,17 +532,18 @@ public void runAsGranted(Authentication authentication, String action, Transport final Optional indices = indices(message); if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()), Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), indices)) == false) { - final LocalNodeInfo localNodeInfo = this.localNodeInfo; - if (indices.isPresent()) { - logger.info("{}[transport] [run_as_granted]\t{}, {}, roles=[{}], action=[{}], indices=[{}], request=[{}]{}", - localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), runAsSubject(authentication), - arrayToCommaDelimitedString(roleNames), action, arrayToCommaDelimitedString(indices.get()), - message.getClass().getSimpleName(), opaqueId()); - } else { - logger.info("{}[transport] [run_as_granted]\t{}, {}, roles=[{}], action=[{}], request=[{}]{}", - localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), runAsSubject(authentication), - arrayToCommaDelimitedString(roleNames), action, message.getClass().getSimpleName(), opaqueId()); - } + final StringMapMessage logEntry = new LogEntryBuilder() + .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE) + .with(EVENT_ACTION_FIELD_NAME, "run_as_granted") + .with(ACTION_FIELD_NAME, action) + .with(REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()) + .withRunAsSubject(authentication) + .withRestOrTransportOrigin(message, threadContext) + .with(INDICES_FIELD_NAME, indices.orElse(null)) + .with(PRINCIPAL_ROLES_FIELD_NAME, roleNames) + .withOpaqueId(threadContext) + .build(); + logger.info(logEntry); } } } @@ -494,17 +554,18 @@ public void runAsDenied(Authentication authentication, String action, TransportM final Optional indices = indices(message); if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()), Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), indices)) == false) { - final LocalNodeInfo localNodeInfo = this.localNodeInfo; - if (indices.isPresent()) { - logger.info("{}[transport] [run_as_denied]\t{}, {}, roles=[{}], action=[{}], indices=[{}], request=[{}]{}", - localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), runAsSubject(authentication), - arrayToCommaDelimitedString(roleNames), action, arrayToCommaDelimitedString(indices.get()), - message.getClass().getSimpleName(), opaqueId()); - } else { - logger.info("{}[transport] [run_as_denied]\t{}, {}, roles=[{}], action=[{}], request=[{}]{}", - localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), runAsSubject(authentication), - arrayToCommaDelimitedString(roleNames), action, message.getClass().getSimpleName(), opaqueId()); - } + final StringMapMessage logEntry = new LogEntryBuilder() + .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE) + .with(EVENT_ACTION_FIELD_NAME, "run_as_denied") + .with(ACTION_FIELD_NAME, action) + .with(REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()) + .withRunAsSubject(authentication) + .withRestOrTransportOrigin(message, threadContext) + .with(INDICES_FIELD_NAME, indices.orElse(null)) + .with(PRINCIPAL_ROLES_FIELD_NAME, roleNames) + .withOpaqueId(threadContext) + .build(); + logger.info(logEntry); } } } @@ -512,118 +573,183 @@ localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), r @Override public void runAsDenied(Authentication authentication, RestRequest request, String[] roleNames) { if (events.contains(RUN_AS_DENIED) - && (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()), - Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), Optional.empty())) == false)) { - if (includeRequestBody) { - logger.info("{}[rest] [run_as_denied]\t{}, {}, roles=[{}], uri=[{}], request_body=[{}]{}", - localNodeInfo.prefix, hostAttributes(request), runAsSubject(authentication), - arrayToCommaDelimitedString(roleNames), request.uri(), restRequestContent(request), opaqueId()); + && eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()), + Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), Optional.empty())) == false) { + final StringMapMessage logEntry = new LogEntryBuilder() + .with(EVENT_TYPE_FIELD_NAME, REST_ORIGIN_FIELD_VALUE) + .with(EVENT_ACTION_FIELD_NAME, "run_as_denied") + .with(PRINCIPAL_ROLES_FIELD_NAME, roleNames) + .withRestUri(request) + .withRunAsSubject(authentication) + .withRestOrigin(request) + .withRequestBody(request) + .withOpaqueId(threadContext) + .build(); + logger.info(logEntry); + } + } + + private class LogEntryBuilder { + + private final StringMapMessage logEntry; + + LogEntryBuilder() { + logEntry = new StringMapMessage(LoggingAuditTrail.this.entryCommonFields.commonFields); + } + + LogEntryBuilder withRestUri(RestRequest request) { + final int queryStringIndex = request.uri().indexOf('?'); + int queryStringLength = request.uri().indexOf('#'); + if (queryStringLength < 0) { + queryStringLength = request.uri().length(); + } + if (queryStringIndex < 0) { + logEntry.with(URL_PATH_FIELD_NAME, request.uri().substring(0, queryStringLength)); } else { - logger.info("{}[rest] [run_as_denied]\t{}, {}, roles=[{}], uri=[{}]{}", localNodeInfo.prefix, - hostAttributes(request), runAsSubject(authentication), arrayToCommaDelimitedString(roleNames), request.uri(), - opaqueId()); + logEntry.with(URL_PATH_FIELD_NAME, request.uri().substring(0, queryStringIndex)); + } + if (queryStringIndex > -1) { + logEntry.with(URL_QUERY_FIELD_NAME, request.uri().substring(queryStringIndex + 1, queryStringLength)); } + return this; } - } - static String runAsSubject(Authentication authentication) { - final StringBuilder sb = new StringBuilder("principal=["); - sb.append(authentication.getUser().authenticatedUser().principal()); - sb.append("], realm=["); - sb.append(authentication.getAuthenticatedBy().getName()); - sb.append("], run_as_principal=["); - sb.append(authentication.getUser().principal()); - if (authentication.getLookedUpBy() != null) { - sb.append("], run_as_realm=[").append(authentication.getLookedUpBy().getName()); + LogEntryBuilder withRunAsSubject(Authentication authentication) { + logEntry.with(PRINCIPAL_FIELD_NAME, authentication.getUser().authenticatedUser().principal()) + .with(PRINCIPAL_REALM_FIELD_NAME, authentication.getAuthenticatedBy().getName()) + .with(PRINCIPAL_RUN_AS_FIELD_NAME, authentication.getUser().principal()); + if (authentication.getLookedUpBy() != null) { + logEntry.with(PRINCIPAL_RUN_AS_REALM_FIELD_NAME, authentication.getLookedUpBy().getName()); + } + return this; } - sb.append("]"); - return sb.toString(); - } - static String subject(Authentication authentication) { - final StringBuilder sb = new StringBuilder("principal=["); - sb.append(authentication.getUser().principal()).append("], realm=["); - if (authentication.getUser().isRunAs()) { - sb.append(authentication.getLookedUpBy().getName()).append("], run_by_principal=["); - sb.append(authentication.getUser().authenticatedUser().principal()).append("], run_by_realm=["); + LogEntryBuilder withRestOrigin(RestRequest request) { + assert LOCAL_ORIGIN_FIELD_VALUE.equals(logEntry.get(ORIGIN_TYPE_FIELD_NAME)); // this is the default + final InetSocketAddress socketAddress = request.getHttpChannel().getRemoteAddress(); + if (socketAddress != null) { + logEntry.with(ORIGIN_TYPE_FIELD_NAME, REST_ORIGIN_FIELD_VALUE) + .with(ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(socketAddress)); + } + // fall through to local_node default + return this; } - sb.append(authentication.getAuthenticatedBy().getName()).append("]"); - return sb.toString(); - } - private static String hostAttributes(RestRequest request) { - String formattedAddress; - final SocketAddress socketAddress = request.getRemoteAddress(); - if (socketAddress instanceof InetSocketAddress) { - formattedAddress = NetworkAddress.format(((InetSocketAddress) socketAddress).getAddress()); - } else { - formattedAddress = socketAddress.toString(); + LogEntryBuilder withRestOrTransportOrigin(TransportMessage message, ThreadContext threadContext) { + assert LOCAL_ORIGIN_FIELD_VALUE.equals(logEntry.get(ORIGIN_TYPE_FIELD_NAME)); // this is the default + final InetSocketAddress restAddress = RemoteHostHeader.restRemoteAddress(threadContext); + if (restAddress != null) { + logEntry.with(ORIGIN_TYPE_FIELD_NAME, REST_ORIGIN_FIELD_VALUE) + .with(ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(restAddress)); + } else { + final TransportAddress address = message.remoteAddress(); + if (address != null) { + logEntry.with(ORIGIN_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE) + .with(ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(address.address())); + } + } + // fall through to local_node default + return this; } - return "origin_address=[" + formattedAddress + "]"; - } - protected static String originAttributes(ThreadContext threadContext, TransportMessage message, LocalNodeInfo localNodeInfo) { - return restOriginTag(threadContext).orElse(transportOriginTag(message).orElse(localNodeInfo.localOriginTag)); - } + LogEntryBuilder withRequestBody(RestRequest request) { + if (includeRequestBody) { + final String requestContent = restRequestContent(request); + if (Strings.hasLength(requestContent)) { + logEntry.with(REQUEST_BODY_FIELD_NAME, requestContent); + } + } + return this; + } - private String opaqueId() { - String opaqueId = threadContext.getHeader(Task.X_OPAQUE_ID); - if (opaqueId != null) { - return ", opaque_id=[" + opaqueId + "]"; - } else { - return ""; + LogEntryBuilder withOpaqueId(ThreadContext threadContext) { + final String opaqueId = threadContext.getHeader(Task.X_OPAQUE_ID); + if (opaqueId != null) { + logEntry.with(OPAQUE_ID_FIELD_NAME, opaqueId); + } + return this; } - } - private static Optional restOriginTag(ThreadContext threadContext) { - final InetSocketAddress restAddress = RemoteHostHeader.restRemoteAddress(threadContext); - if (restAddress == null) { - return Optional.empty(); + LogEntryBuilder withPrincipal(User user) { + logEntry.with(PRINCIPAL_FIELD_NAME, user.principal()); + if (user.isRunAs()) { + logEntry.with(PRINCIPAL_RUN_BY_FIELD_NAME, user.authenticatedUser().principal()); + } + return this; } - return Optional.of(new StringBuilder("origin_type=[rest], origin_address=[").append(NetworkAddress.format(restAddress.getAddress())) - .append("]") - .toString()); - } - private static Optional transportOriginTag(TransportMessage message) { - final TransportAddress address = message.remoteAddress(); - if (address == null) { - return Optional.empty(); + LogEntryBuilder withSubject(Authentication authentication) { + logEntry.with(PRINCIPAL_FIELD_NAME, authentication.getUser().principal()); + if (authentication.getUser().isRunAs()) { + logEntry.with(PRINCIPAL_REALM_FIELD_NAME, authentication.getLookedUpBy().getName()) + .with(PRINCIPAL_RUN_BY_FIELD_NAME, authentication.getUser().authenticatedUser().principal()) + .with(PRINCIPAL_RUN_BY_REALM_FIELD_NAME, authentication.getAuthenticatedBy().getName()); + } else { + logEntry.with(PRINCIPAL_REALM_FIELD_NAME, authentication.getAuthenticatedBy().getName()); + } + return this; + } + + LogEntryBuilder with(String key, String value) { + if (value != null) { + logEntry.with(key, value); + } + return this; } - return Optional.of( - new StringBuilder("origin_type=[transport], origin_address=[").append(NetworkAddress.format(address.address().getAddress())) - .append("]") - .toString()); + + LogEntryBuilder with(String key, String[] values) { + if (values != null) { + logEntry.with(key, toQuotedJsonArray(values)); + } + return this; + } + + StringMapMessage build() { + return logEntry; + } + + String toQuotedJsonArray(String[] values) { + assert values != null; + final StringBuilder stringBuilder = new StringBuilder(); + final JsonStringEncoder jsonStringEncoder = JsonStringEncoder.getInstance(); + stringBuilder.append("["); + for (final String value : values) { + if (value != null) { + if (stringBuilder.length() > 1) { + stringBuilder.append(","); + } + stringBuilder.append("\""); + jsonStringEncoder.quoteAsString(value, stringBuilder); + stringBuilder.append("\""); + } + } + stringBuilder.append("]"); + return stringBuilder.toString(); + } + } - static Optional indices(TransportMessage message) { + + private static Optional indices(TransportMessage message) { if (message instanceof IndicesRequest) { final String[] indices = ((IndicesRequest) message).indices(); - if ((indices != null) && (indices.length != 0)) { + if (indices != null) { return Optional.of(((IndicesRequest) message).indices()); } } return Optional.empty(); } - static String effectiveRealmName(Authentication authentication) { + private static String effectiveRealmName(Authentication authentication) { return authentication.getLookedUpBy() != null ? authentication.getLookedUpBy().getName() : authentication.getAuthenticatedBy().getName(); } - static String principal(User user) { - final StringBuilder builder = new StringBuilder("principal=["); - builder.append(user.principal()); - if (user.isRunAs()) { - builder.append("], run_by_principal=[").append(user.authenticatedUser().principal()); - } - return builder.append("]").toString(); - } - public static void registerSettings(List> settings) { - settings.add(HOST_ADDRESS_SETTING); - settings.add(HOST_NAME_SETTING); - settings.add(NODE_NAME_SETTING); + settings.add(EMIT_HOST_ADDRESS_SETTING); + settings.add(EMIT_HOST_NAME_SETTING); + settings.add(EMIT_NODE_NAME_SETTING); + settings.add(EMIT_NODE_ID_SETTING); settings.add(INCLUDE_EVENT_SETTINGS); settings.add(EXCLUDE_EVENT_SETTINGS); settings.add(INCLUDE_REQUEST_BODY); @@ -819,56 +945,53 @@ public void clusterChanged(ClusterChangedEvent event) { void updateLocalNodeInfo(DiscoveryNode newLocalNode) { // check if local node changed - final LocalNodeInfo localNodeInfo = this.localNodeInfo; - if ((localNodeInfo.localNode == null) || (localNodeInfo.localNode.equals(newLocalNode) == false)) { + final EntryCommonFields localNodeInfo = this.entryCommonFields; + if (localNodeInfo.localNode == null || localNodeInfo.localNode.equals(newLocalNode) == false) { // no need to synchronize, called only from the cluster state applier thread - this.localNodeInfo = new LocalNodeInfo(localNodeInfo.settings, newLocalNode); + this.entryCommonFields = this.entryCommonFields.withNewLocalNode(newLocalNode); } } - static class LocalNodeInfo { + static class EntryCommonFields { private final Settings settings; private final DiscoveryNode localNode; - final String prefix; - private final String localOriginTag; + final Map commonFields; - LocalNodeInfo(Settings settings, @Nullable DiscoveryNode newLocalNode) { + EntryCommonFields(Settings settings, @Nullable DiscoveryNode newLocalNode) { this.settings = settings; this.localNode = newLocalNode; - this.prefix = resolvePrefix(settings, newLocalNode); - this.localOriginTag = localOriginTag(newLocalNode); - } - - static String resolvePrefix(Settings settings, @Nullable DiscoveryNode localNode) { - final StringBuilder builder = new StringBuilder(); - if (HOST_ADDRESS_SETTING.get(settings)) { - final String address = localNode != null ? localNode.getHostAddress() : null; - if (address != null) { - builder.append("[").append(address).append("] "); + final Map commonFields = new HashMap<>(); + if (EMIT_NODE_NAME_SETTING.get(settings)) { + final String nodeName = Node.NODE_NAME_SETTING.get(settings); + if (Strings.hasLength(nodeName)) { + commonFields.put(NODE_NAME_FIELD_NAME, nodeName); } } - if (HOST_NAME_SETTING.get(settings)) { - final String hostName = localNode != null ? localNode.getHostName() : null; - if (hostName != null) { - builder.append("[").append(hostName).append("] "); + if (newLocalNode != null && newLocalNode.getAddress() != null) { + if (EMIT_HOST_ADDRESS_SETTING.get(settings)) { + commonFields.put(HOST_ADDRESS_FIELD_NAME, newLocalNode.getAddress().getAddress()); } - } - if (NODE_NAME_SETTING.get(settings)) { - final String name = Node.NODE_NAME_SETTING.get(settings); - if (name != null) { - builder.append("[").append(name).append("] "); + if (EMIT_HOST_NAME_SETTING.get(settings)) { + commonFields.put(HOST_NAME_FIELD_NAME, newLocalNode.getAddress().address().getHostString()); + } + if (EMIT_NODE_ID_SETTING.get(settings)) { + commonFields.put(NODE_ID_FIELD_NAME, newLocalNode.getId()); } + // the default origin is local + commonFields.put(ORIGIN_ADDRESS_FIELD_NAME, newLocalNode.getAddress().toString()); } - return builder.toString(); + // the default origin is local + commonFields.put(ORIGIN_TYPE_FIELD_NAME, LOCAL_ORIGIN_FIELD_VALUE); + this.commonFields = Collections.unmodifiableMap(commonFields); } - private static String localOriginTag(@Nullable DiscoveryNode localNode) { - if (localNode == null) { - return "origin_type=[local_node]"; - } - return new StringBuilder("origin_type=[local_node], origin_address=[").append(localNode.getHostAddress()) - .append("]") - .toString(); + EntryCommonFields withNewSettings(Settings newSettings) { + final Settings mergedSettings = Settings.builder().put(this.settings).put(newSettings, false).build(); + return new EntryCommonFields(mergedSettings, this.localNode); + } + + EntryCommonFields withNewLocalNode(DiscoveryNode newLocalNode) { + return new EntryCommonFields(this.settings, newLocalNode); } } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java index 210d2f748bbae..ef3d290f7980e 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java @@ -135,9 +135,10 @@ public Settings nodeSettings(int nodeOrdinal) { .put(XPackSettings.WATCHER_ENABLED.getKey(), false) .put(XPackSettings.MONITORING_ENABLED.getKey(), false) .put(XPackSettings.AUDIT_ENABLED.getKey(), randomBoolean()) - .put(LoggingAuditTrail.HOST_ADDRESS_SETTING.getKey(), randomBoolean()) - .put(LoggingAuditTrail.HOST_NAME_SETTING.getKey(), randomBoolean()) - .put(LoggingAuditTrail.NODE_NAME_SETTING.getKey(), randomBoolean()) + .put(LoggingAuditTrail.EMIT_HOST_ADDRESS_SETTING.getKey(), randomBoolean()) + .put(LoggingAuditTrail.EMIT_HOST_NAME_SETTING.getKey(), randomBoolean()) + .put(LoggingAuditTrail.EMIT_NODE_NAME_SETTING.getKey(), randomBoolean()) + .put(LoggingAuditTrail.EMIT_NODE_ID_SETTING.getKey(), randomBoolean()) .put("xpack.security.authc.realms.file.type", FileRealmSettings.TYPE) .put("xpack.security.authc.realms.file.order", 0) .put("xpack.security.authc.realms.index.type", NativeRealmSettings.TYPE) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/AuditTrailSettingsUpdateTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/AuditTrailSettingsUpdateTests.java index 2f32ba9c537f1..e05f4620ccca2 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/AuditTrailSettingsUpdateTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/AuditTrailSettingsUpdateTests.java @@ -21,11 +21,11 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; -import java.util.regex.Pattern; - import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.startsWith; +import static org.hamcrest.Matchers.is; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -106,28 +106,45 @@ public void testInvalidFilterSettings() throws Exception { public void testDynamicHostSettings() { final boolean persistent = randomBoolean(); final Settings.Builder settingsBuilder = Settings.builder(); - settingsBuilder.put(LoggingAuditTrail.HOST_ADDRESS_SETTING.getKey(), true); - settingsBuilder.put(LoggingAuditTrail.HOST_NAME_SETTING.getKey(), true); - settingsBuilder.put(LoggingAuditTrail.NODE_NAME_SETTING.getKey(), true); + settingsBuilder.put(LoggingAuditTrail.EMIT_HOST_ADDRESS_SETTING.getKey(), true); + settingsBuilder.put(LoggingAuditTrail.EMIT_HOST_NAME_SETTING.getKey(), true); + settingsBuilder.put(LoggingAuditTrail.EMIT_NODE_NAME_SETTING.getKey(), true); + settingsBuilder.put(LoggingAuditTrail.EMIT_NODE_ID_SETTING.getKey(), true); updateSettings(settingsBuilder.build(), persistent); - final LoggingAuditTrail loggingAuditTrail = ((LoggingAuditTrail) internalCluster().getInstances(AuditTrailService.class) + final LoggingAuditTrail loggingAuditTrail = (LoggingAuditTrail) internalCluster().getInstances(AuditTrailService.class) .iterator() .next() .getAuditTrails() .iterator() - .next()); - assertTrue(Pattern.matches("\\[127\\.0\\.0\\.1\\] \\[127\\.0\\.0\\.1\\] \\[node_.*\\] ", loggingAuditTrail.localNodeInfo.prefix)); - settingsBuilder.put(LoggingAuditTrail.HOST_ADDRESS_SETTING.getKey(), false); + .next(); + assertThat(loggingAuditTrail.entryCommonFields.commonFields.get(LoggingAuditTrail.NODE_NAME_FIELD_NAME), startsWith("node_")); + assertThat(loggingAuditTrail.entryCommonFields.commonFields.containsKey(LoggingAuditTrail.NODE_ID_FIELD_NAME), is(true)); + assertThat(loggingAuditTrail.entryCommonFields.commonFields.get(LoggingAuditTrail.HOST_ADDRESS_FIELD_NAME), is("127.0.0.1")); + assertThat(loggingAuditTrail.entryCommonFields.commonFields.get(LoggingAuditTrail.HOST_NAME_FIELD_NAME), is("127.0.0.1")); + settingsBuilder.put(LoggingAuditTrail.EMIT_HOST_ADDRESS_SETTING.getKey(), false); + updateSettings(settingsBuilder.build(), persistent); + assertThat(loggingAuditTrail.entryCommonFields.commonFields.get(LoggingAuditTrail.NODE_NAME_FIELD_NAME), startsWith("node_")); + assertThat(loggingAuditTrail.entryCommonFields.commonFields.containsKey(LoggingAuditTrail.NODE_ID_FIELD_NAME), is(true)); + assertThat(loggingAuditTrail.entryCommonFields.commonFields.containsKey(LoggingAuditTrail.HOST_ADDRESS_FIELD_NAME), is(false)); + assertThat(loggingAuditTrail.entryCommonFields.commonFields.get(LoggingAuditTrail.HOST_NAME_FIELD_NAME), is("127.0.0.1")); + settingsBuilder.put(LoggingAuditTrail.EMIT_HOST_NAME_SETTING.getKey(), false); updateSettings(settingsBuilder.build(), persistent); - assertTrue(Pattern.matches("\\[127\\.0\\.0\\.1\\] \\[node_.*\\] ", loggingAuditTrail.localNodeInfo.prefix)); - settingsBuilder.put(LoggingAuditTrail.HOST_ADDRESS_SETTING.getKey(), true); - settingsBuilder.put(LoggingAuditTrail.HOST_NAME_SETTING.getKey(), false); + assertThat(loggingAuditTrail.entryCommonFields.commonFields.get(LoggingAuditTrail.NODE_NAME_FIELD_NAME), startsWith("node_")); + assertThat(loggingAuditTrail.entryCommonFields.commonFields.containsKey(LoggingAuditTrail.NODE_ID_FIELD_NAME), is(true)); + assertThat(loggingAuditTrail.entryCommonFields.commonFields.containsKey(LoggingAuditTrail.HOST_ADDRESS_FIELD_NAME), is(false)); + assertThat(loggingAuditTrail.entryCommonFields.commonFields.containsKey(LoggingAuditTrail.HOST_NAME_FIELD_NAME), is(false)); + settingsBuilder.put(LoggingAuditTrail.EMIT_NODE_NAME_SETTING.getKey(), false); updateSettings(settingsBuilder.build(), persistent); - assertTrue(Pattern.matches("\\[127\\.0\\.0\\.1\\] \\[node_.*\\] ", loggingAuditTrail.localNodeInfo.prefix)); - settingsBuilder.put(LoggingAuditTrail.HOST_NAME_SETTING.getKey(), true); - settingsBuilder.put(LoggingAuditTrail.NODE_NAME_SETTING.getKey(), false); + assertThat(loggingAuditTrail.entryCommonFields.commonFields.containsKey(LoggingAuditTrail.NODE_NAME_FIELD_NAME), is(false)); + assertThat(loggingAuditTrail.entryCommonFields.commonFields.containsKey(LoggingAuditTrail.NODE_ID_FIELD_NAME), is(true)); + assertThat(loggingAuditTrail.entryCommonFields.commonFields.containsKey(LoggingAuditTrail.HOST_ADDRESS_FIELD_NAME), is(false)); + assertThat(loggingAuditTrail.entryCommonFields.commonFields.containsKey(LoggingAuditTrail.HOST_NAME_FIELD_NAME), is(false)); + settingsBuilder.put(LoggingAuditTrail.EMIT_NODE_ID_SETTING.getKey(), false); updateSettings(settingsBuilder.build(), persistent); - assertTrue(Pattern.matches("\\[127\\.0\\.0\\.1\\] \\[127\\.0\\.0\\.1\\] ", loggingAuditTrail.localNodeInfo.prefix)); + assertThat(loggingAuditTrail.entryCommonFields.commonFields.containsKey(LoggingAuditTrail.NODE_NAME_FIELD_NAME), is(false)); + assertThat(loggingAuditTrail.entryCommonFields.commonFields.containsKey(LoggingAuditTrail.NODE_ID_FIELD_NAME), is(false)); + assertThat(loggingAuditTrail.entryCommonFields.commonFields.containsKey(LoggingAuditTrail.HOST_ADDRESS_FIELD_NAME), is(false)); + assertThat(loggingAuditTrail.entryCommonFields.commonFields.containsKey(LoggingAuditTrail.HOST_NAME_FIELD_NAME), is(false)); } public void testDynamicRequestBodySettings() { @@ -136,12 +153,12 @@ public void testDynamicRequestBodySettings() { final Settings.Builder settingsBuilder = Settings.builder(); settingsBuilder.put(LoggingAuditTrail.INCLUDE_REQUEST_BODY.getKey(), enableRequestBody); updateSettings(settingsBuilder.build(), persistent); - final LoggingAuditTrail loggingAuditTrail = ((LoggingAuditTrail) internalCluster().getInstances(AuditTrailService.class) + final LoggingAuditTrail loggingAuditTrail = (LoggingAuditTrail) internalCluster().getInstances(AuditTrailService.class) .iterator() .next() .getAuditTrails() .iterator() - .next()); + .next(); assertEquals(enableRequestBody, loggingAuditTrail.includeRequestBody); settingsBuilder.put(LoggingAuditTrail.INCLUDE_REQUEST_BODY.getKey(), !enableRequestBody); updateSettings(settingsBuilder.build(), persistent); @@ -158,12 +175,12 @@ public void testDynamicEventsSettings() { settingsBuilder.putList(LoggingAuditTrail.INCLUDE_EVENT_SETTINGS.getKey(), includedEvents); settingsBuilder.putList(LoggingAuditTrail.EXCLUDE_EVENT_SETTINGS.getKey(), excludedEvents); updateSettings(settingsBuilder.build(), randomBoolean()); - final LoggingAuditTrail loggingAuditTrail = ((LoggingAuditTrail) internalCluster().getInstances(AuditTrailService.class) + final LoggingAuditTrail loggingAuditTrail = (LoggingAuditTrail) internalCluster().getInstances(AuditTrailService.class) .iterator() .next() .getAuditTrails() .iterator() - .next()); + .next(); assertEquals(AuditLevel.parse(includedEvents, excludedEvents), loggingAuditTrail.events); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java index 4c9df8fd9d382..67f38c0e58159 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java @@ -59,9 +59,6 @@ public class LoggingAuditTrailFilterTests extends ESTestCase { private Settings settings; private DiscoveryNode localNode; private ClusterService clusterService; - private ThreadContext threadContext; - private Logger logger; - List logOutput; @Before public void init() throws Exception { @@ -83,12 +80,11 @@ public void init() throws Exception { arg0.updateLocalNodeInfo(localNode); return null; }).when(clusterService).addListener(Mockito.isA(LoggingAuditTrail.class)); - threadContext = new ThreadContext(Settings.EMPTY); - logger = CapturingLogger.newCapturingLogger(Level.INFO); - logOutput = CapturingLogger.output(logger.getName(), Level.INFO); } public void testSingleCompletePolicyPredicate() throws Exception { + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); // create complete filter policy final Settings.Builder settingsBuilder = Settings.builder().put(settings); // filter by username @@ -179,6 +175,8 @@ public void testSingleCompletePolicyPredicate() throws Exception { } public void testSingleCompleteWithEmptyFieldPolicyPredicate() throws Exception { + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); // create complete filter policy final Settings.Builder settingsBuilder = Settings.builder().put(settings); // filter by username @@ -275,6 +273,8 @@ public void testSingleCompleteWithEmptyFieldPolicyPredicate() throws Exception { } public void testTwoPolicyPredicatesWithMissingFields() throws Exception { + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); final Settings.Builder settingsBuilder = Settings.builder().put(settings); // first policy: realms and roles filters final List filteredRealms = randomNonEmptyListOfFilteredNames(); @@ -341,6 +341,8 @@ public void testTwoPolicyPredicatesWithMissingFields() throws Exception { } public void testUsersFilter() throws Exception { + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); final List allFilteredUsers = new ArrayList<>(); final Settings.Builder settingsBuilder = Settings.builder().put(settings); for (int i = 0; i < randomIntBetween(1, 4); i++) { @@ -387,6 +389,7 @@ public void testUsersFilter() throws Exception { final MockToken unfilteredToken = new MockToken(UNFILTER_MARKER + randomAlphaOfLengthBetween(1, 4)); final LoggingAuditTrail auditTrail = new LoggingAuditTrail(settingsBuilder.build(), clusterService, logger, threadContext); + final List logOutput = CapturingLogger.output(logger.getName(), Level.INFO); // anonymous accessDenied auditTrail.anonymousAccessDenied("_action", message); if (filterMissingUser) { @@ -623,6 +626,8 @@ public void testUsersFilter() throws Exception { } public void testRealmsFilter() throws Exception { + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); final List allFilteredRealms = new ArrayList<>(); final Settings.Builder settingsBuilder = Settings.builder().put(settings); for (int i = 0; i < randomIntBetween(1, 4); i++) { @@ -659,6 +664,7 @@ public void testRealmsFilter() throws Exception { final MockToken authToken = new MockToken("token1"); final LoggingAuditTrail auditTrail = new LoggingAuditTrail(settingsBuilder.build(), clusterService, logger, threadContext); + final List logOutput = CapturingLogger.output(logger.getName(), Level.INFO); // anonymous accessDenied auditTrail.anonymousAccessDenied("_action", message); if (filterMissingRealm) { @@ -908,6 +914,8 @@ public void testRealmsFilter() throws Exception { } public void testRolesFilter() throws Exception { + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); final List> allFilteredRoles = new ArrayList<>(); final Settings.Builder settingsBuilder = Settings.builder().put(settings); for (int i = 0; i < randomIntBetween(1, 4); i++) { @@ -966,6 +974,7 @@ public void testRolesFilter() throws Exception { final MockToken authToken = new MockToken("token1"); final LoggingAuditTrail auditTrail = new LoggingAuditTrail(settingsBuilder.build(), clusterService, logger, threadContext); + final List logOutput = CapturingLogger.output(logger.getName(), Level.INFO); // anonymous accessDenied auditTrail.anonymousAccessDenied("_action", message); if (filterMissingRoles) { @@ -1179,6 +1188,8 @@ public void testRolesFilter() throws Exception { } public void testIndicesFilter() throws Exception { + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); final List> allFilteredIndices = new ArrayList<>(); final Settings.Builder settingsBuilder = Settings.builder().put(settings); for (int i = 0; i < randomIntBetween(1, 3); i++) { @@ -1236,6 +1247,7 @@ public void testIndicesFilter() throws Exception { final TransportMessage noIndexMessage = new MockMessage(threadContext); final LoggingAuditTrail auditTrail = new LoggingAuditTrail(settingsBuilder.build(), clusterService, logger, threadContext); + final List logOutput = CapturingLogger.output(logger.getName(), Level.INFO); // anonymous accessDenied auditTrail.anonymousAccessDenied("_action", noIndexMessage); if (filterMissingIndices) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java index 1059e22abd663..6c302c9311f3d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java @@ -7,6 +7,7 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.layout.PatternLayout; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -14,6 +15,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.settings.ClusterSettings; @@ -39,23 +41,29 @@ import org.elasticsearch.xpack.security.rest.RemoteHostHeader; import org.elasticsearch.xpack.security.transport.filter.IPFilter; import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule; +import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; +import org.junit.BeforeClass; import org.mockito.stubbing.Answer; import java.io.IOException; +import java.io.InputStream; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; +import java.util.Properties; +import java.util.regex.Pattern; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.containsString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -113,17 +121,46 @@ protected String expectedMessage() { }; protected abstract boolean hasContent(); + protected abstract BytesReference content(); + protected abstract String expectedMessage(); } - private String prefix; + private static PatternLayout patternLayout; private Settings settings; private DiscoveryNode localNode; private ClusterService clusterService; private ThreadContext threadContext; private boolean includeRequestBody; - private String opaqueId; + private Map commonFields; + private Logger logger; + private LoggingAuditTrail auditTrail; + + @BeforeClass + public static void lookupPatternLayout() throws Exception { + final Properties properties = new Properties(); + try (InputStream configStream = LoggingAuditTrail.class.getClassLoader().getResourceAsStream("log4j2.properties")) { + properties.load(configStream); + } + // This is a minimal and brittle parsing of the security log4j2 config + // properties. If any of these fails, then surely the config file changed. In + // this case adjust the assertions! The goal of this assertion chain is to + // validate that the layout pattern we are testing with is indeed the one + // attached to the LoggingAuditTrail.class logger. + assertThat(properties.getProperty("logger.xpack_security_audit_logfile.name"), is(LoggingAuditTrail.class.getName())); + assertThat(properties.getProperty("logger.xpack_security_audit_logfile.appenderRef.audit_rolling.ref"), is("audit_rolling")); + assertThat(properties.getProperty("appender.audit_rolling.name"), is("audit_rolling")); + assertThat(properties.getProperty("appender.audit_rolling.layout.type"), is("PatternLayout")); + final String patternLayoutFormat = properties.getProperty("appender.audit_rolling.layout.pattern"); + assertThat(patternLayoutFormat, is(notNullValue())); + patternLayout = PatternLayout.newBuilder().withPattern(patternLayoutFormat).withCharset(StandardCharsets.UTF_8).build(); + } + + @AfterClass + public static void releasePatternLayout() { + patternLayout = null; + } @Before public void init() throws Exception { @@ -135,7 +172,7 @@ public void init() throws Exception { .put("xpack.security.audit.logfile.events.emit_request_body", includeRequestBody) .build(); localNode = mock(DiscoveryNode.class); - when(localNode.getHostAddress()).thenReturn(buildNewFakeTransportAddress().toString()); + when(localNode.getAddress()).thenReturn(buildNewFakeTransportAddress()); clusterService = mock(ClusterService.class); when(clusterService.localNode()).thenReturn(localNode); Mockito.doAnswer((Answer) invocation -> { @@ -145,597 +182,708 @@ public void init() throws Exception { }).when(clusterService).addListener(Mockito.isA(LoggingAuditTrail.class)); final ClusterSettings clusterSettings = mockClusterSettings(); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); - prefix = LoggingAuditTrail.LocalNodeInfo.resolvePrefix(settings, localNode); + commonFields = new LoggingAuditTrail.EntryCommonFields(settings, localNode).commonFields; threadContext = new ThreadContext(Settings.EMPTY); if (randomBoolean()) { - String id = randomAlphaOfLength(10); - threadContext.putHeader(Task.X_OPAQUE_ID, id); - opaqueId = ", opaque_id=[" + id + "]"; - } else { - opaqueId = ""; + threadContext.putHeader(Task.X_OPAQUE_ID, randomAlphaOfLengthBetween(1, 4)); } + logger = CapturingLogger.newCapturingLogger(Level.INFO, patternLayout); + auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); + } + + @After + public void clearLog() throws Exception { + CapturingLogger.output(logger.getName(), Level.INFO).clear(); } public void testAnonymousAccessDeniedTransport() throws Exception { - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); - final String origins = LoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); + auditTrail.anonymousAccessDenied("_action", message); - if (message instanceof IndicesRequest) { - assertMsg(logger, Level.INFO, prefix + "[transport] [anonymous_access_denied]\t" + origins + - ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); - } else { - assertMsg(logger, Level.INFO, prefix + "[transport] [anonymous_access_denied]\t" + origins + - ", action=[_action], request=[MockMessage]" + opaqueId); - } + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + final MapBuilder checkedArrayFields = new MapBuilder<>(); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "anonymous_access_denied") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action"); + indicesRequest(message, checkedFields, checkedArrayFields); + restOrTransportOrigin(message, threadContext, checkedFields); + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled CapturingLogger.output(logger.getName(), Level.INFO).clear(); - settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "anonymous_access_denied").build(); + settings = Settings.builder() + .put(settings) + .put("xpack.security.audit.logfile.events.exclude", "anonymous_access_denied") + .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.anonymousAccessDenied("_action", message); assertEmptyLog(logger); } public void testAnonymousAccessDeniedRest() throws Exception { - final InetAddress address = forge("_hostname", randomBoolean() ? "127.0.0.1" : "::1"); - final Tuple tuple = prepareRestContent("_uri", new InetSocketAddress(address, 9200)); + final InetSocketAddress address = new InetSocketAddress(forge("_hostname", randomBoolean() ? "127.0.0.1" : "::1"), + randomIntBetween(9200, 9300)); + final Tuple tuple = prepareRestContent("_uri", address); final String expectedMessage = tuple.v1().expectedMessage(); final RestRequest request = tuple.v2(); - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.anonymousAccessDenied(request); - if (includeRequestBody) { - assertMsg(logger, Level.INFO, prefix + "[rest] [anonymous_access_denied]\torigin_address=[" + - NetworkAddress.format(address) + "], uri=[_uri]" + opaqueId + ", request_body=[" + expectedMessage + "]"); - } else { - assertMsg(logger, Level.INFO, prefix + "[rest] [anonymous_access_denied]\torigin_address=[" + - NetworkAddress.format(address) + "], uri=[_uri]" + opaqueId); - } + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.REST_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "anonymous_access_denied") + .put(LoggingAuditTrail.ORIGIN_TYPE_FIELD_NAME, LoggingAuditTrail.REST_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(address)) + .put(LoggingAuditTrail.REQUEST_BODY_FIELD_NAME, + includeRequestBody && Strings.hasLength(expectedMessage) ? expectedMessage : null) + .put(LoggingAuditTrail.URL_PATH_FIELD_NAME, "_uri") + .put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, null); + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap()); // test disabled CapturingLogger.output(logger.getName(), Level.INFO).clear(); - settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "anonymous_access_denied").build(); + settings = Settings.builder() + .put(settings) + .put("xpack.security.audit.logfile.events.exclude", "anonymous_access_denied") + .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.anonymousAccessDenied(request); assertEmptyLog(logger); } public void testAuthenticationFailed() throws Exception { - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); + final AuthenticationToken mockToken = new MockToken(); final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); - final String origins = LoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); - auditTrail.authenticationFailed(new MockToken(), "_action", message); - if (message instanceof IndicesRequest) { - assertMsg(logger, Level.INFO, prefix + "[transport] [authentication_failed]\t" + origins + - ", principal=[_principal], action=[_action], indices=[" + indices(message) + - "], request=[MockIndicesRequest]" + opaqueId); - } else { - assertMsg(logger, Level.INFO, prefix + "[transport] [authentication_failed]\t" + origins + - ", principal=[_principal], action=[_action], request=[MockMessage]" + opaqueId); - } + + auditTrail.authenticationFailed(mockToken, "_action", message); + final MapBuilder checkedArrayFields = new MapBuilder<>(); + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "authentication_failed") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action") + .put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, mockToken.principal()) + .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()); + restOrTransportOrigin(message, threadContext, checkedFields); + indicesRequest(message, checkedFields, checkedArrayFields); + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled CapturingLogger.output(logger.getName(), Level.INFO).clear(); - settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "authentication_failed").build(); + settings = Settings.builder() + .put(settings) + .put("xpack.security.audit.logfile.events.exclude", "authentication_failed") + .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.authenticationFailed(new MockToken(), "_action", message); assertEmptyLog(logger); } public void testAuthenticationFailedNoToken() throws Exception { - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); - final String origins = LoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); + auditTrail.authenticationFailed("_action", message); - if (message instanceof IndicesRequest) { - assertMsg(logger, Level.INFO, prefix + "[transport] [authentication_failed]\t" + origins + - ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); - } else { - assertMsg(logger, Level.INFO, prefix + "[transport] [authentication_failed]\t" + origins + - ", action=[_action], request=[MockMessage]" + opaqueId); - } + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + final MapBuilder checkedArrayFields = new MapBuilder<>(); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "authentication_failed") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action") + .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()); + restOrTransportOrigin(message, threadContext, checkedFields); + indicesRequest(message, checkedFields, checkedArrayFields); + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled CapturingLogger.output(logger.getName(), Level.INFO).clear(); - settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "authentication_failed").build(); + settings = Settings.builder() + .put(settings) + .put("xpack.security.audit.logfile.events.exclude", "authentication_failed") + .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.authenticationFailed("_action", message); assertEmptyLog(logger); } public void testAuthenticationFailedRest() throws Exception { - final InetAddress address = forge("_hostname", randomBoolean() ? "127.0.0.1" : "::1"); - final Tuple tuple = prepareRestContent("_uri", new InetSocketAddress(address, 9200)); + final Map params = new HashMap<>(); + if (randomBoolean()) { + params.put("foo", "bar"); + } + final InetSocketAddress address = new InetSocketAddress(forge("_hostname", randomBoolean() ? "127.0.0.1" : "::1"), + randomIntBetween(9200, 9300)); + final Tuple tuple = prepareRestContent("_uri", address, params); final String expectedMessage = tuple.v1().expectedMessage(); final RestRequest request = tuple.v2(); - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); - auditTrail.authenticationFailed(new MockToken(), request); - if (includeRequestBody) { - assertMsg(logger, Level.INFO, prefix + "[rest] [authentication_failed]\torigin_address=[" + - NetworkAddress.format(address) + "], principal=[_principal], uri=[_uri]" + opaqueId + ", request_body=[" + - expectedMessage + "]"); - } else { - assertMsg(logger, Level.INFO, prefix + "[rest] [authentication_failed]\torigin_address=[" + - NetworkAddress.format(address) + "], principal=[_principal], uri=[_uri]" + opaqueId); - } + final AuthenticationToken mockToken = new MockToken(); + + auditTrail.authenticationFailed(mockToken, request); + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.REST_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "authentication_failed") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, null) + .put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, mockToken.principal()) + .put(LoggingAuditTrail.ORIGIN_TYPE_FIELD_NAME, LoggingAuditTrail.REST_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(address)) + .put(LoggingAuditTrail.REQUEST_BODY_FIELD_NAME, + includeRequestBody && Strings.hasLength(expectedMessage) ? expectedMessage : null) + .put(LoggingAuditTrail.URL_PATH_FIELD_NAME, "_uri") + .put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, params.isEmpty() ? null : "foo=bar"); + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap()); // test disabled CapturingLogger.output(logger.getName(), Level.INFO).clear(); - settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "authentication_failed").build(); + settings = Settings.builder() + .put(settings) + .put("xpack.security.audit.logfile.events.exclude", "authentication_failed") + .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.authenticationFailed(new MockToken(), request); assertEmptyLog(logger); } public void testAuthenticationFailedRestNoToken() throws Exception { - final InetAddress address = forge("_hostname", randomBoolean() ? "127.0.0.1" : "::1"); - final Tuple tuple = prepareRestContent("_uri", new InetSocketAddress(address, 9200)); + final Map params = new HashMap<>(); + if (randomBoolean()) { + params.put("bar", "baz"); + } + final InetSocketAddress address = new InetSocketAddress(forge("_hostname", randomBoolean() ? "127.0.0.1" : "::1"), + randomIntBetween(9200, 9300)); + final Tuple tuple = prepareRestContent("_uri", address, params); final String expectedMessage = tuple.v1().expectedMessage(); final RestRequest request = tuple.v2(); - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.authenticationFailed(request); - if (includeRequestBody) { - assertMsg(logger, Level.INFO, prefix + "[rest] [authentication_failed]\torigin_address=[" + - NetworkAddress.format(address) + "], uri=[_uri]" + opaqueId + ", request_body=[" + expectedMessage + "]"); - } else { - assertMsg(logger, Level.INFO, prefix + "[rest] [authentication_failed]\torigin_address=[" + - NetworkAddress.format(address) + "], uri=[_uri]" + opaqueId); - } + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.REST_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "authentication_failed") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, null) + .put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, null) + .put(LoggingAuditTrail.ORIGIN_TYPE_FIELD_NAME, LoggingAuditTrail.REST_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(address)) + .put(LoggingAuditTrail.REQUEST_BODY_FIELD_NAME, + includeRequestBody && Strings.hasLength(expectedMessage) ? expectedMessage : null) + .put(LoggingAuditTrail.URL_PATH_FIELD_NAME, "_uri") + .put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, params.isEmpty() ? null : "bar=baz"); + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap()); // test disabled CapturingLogger.output(logger.getName(), Level.INFO).clear(); - settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "authentication_failed").build(); + settings = Settings.builder() + .put(settings) + .put("xpack.security.audit.logfile.events.exclude", "authentication_failed") + .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.authenticationFailed(request); assertEmptyLog(logger); } public void testAuthenticationFailedRealm() throws Exception { - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); + final AuthenticationToken mockToken = new MockToken(); final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); - auditTrail.authenticationFailed("_realm", new MockToken(), "_action", message); + final String realm = randomAlphaOfLengthBetween(1, 6); + auditTrail.authenticationFailed(realm, mockToken, "_action", message); assertEmptyLog(logger); // test enabled - settings = - Settings.builder().put(settings).put("xpack.security.audit.logfile.events.include", "realm_authentication_failed").build(); + settings = Settings.builder() + .put(settings) + .put("xpack.security.audit.logfile.events.include", "realm_authentication_failed") + .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); - final String origins = LoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); - auditTrail.authenticationFailed("_realm", new MockToken(), "_action", message); - if (message instanceof IndicesRequest) { - assertMsg(logger, Level.INFO, prefix + "[transport] [realm_authentication_failed]\trealm=[_realm], " + origins + - ", principal=[_principal], action=[_action], indices=[" + indices(message) + "], " + - "request=[MockIndicesRequest]" + opaqueId); - } else { - assertMsg(logger, Level.INFO, prefix + "[transport] [realm_authentication_failed]\trealm=[_realm], " + origins + - ", principal=[_principal], action=[_action], request=[MockMessage]" + opaqueId); - } + auditTrail.authenticationFailed(realm, mockToken, "_action", message); + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + final MapBuilder checkedArrayFields = new MapBuilder<>(); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "realm_authentication_failed") + .put(LoggingAuditTrail.REALM_FIELD_NAME, realm) + .put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, mockToken.principal()) + .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action") + .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()); + restOrTransportOrigin(message, threadContext, checkedFields); + indicesRequest(message, checkedFields, checkedArrayFields); + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); } public void testAuthenticationFailedRealmRest() throws Exception { - final InetAddress address = forge("_hostname", randomBoolean() ? "127.0.0.1" : "::1"); - final Tuple tuple = prepareRestContent("_uri", new InetSocketAddress(address, 9200)); + final Map params = new HashMap<>(); + if (randomBoolean()) { + params.put("_param", "baz"); + } + final InetSocketAddress address = new InetSocketAddress(forge("_hostname", randomBoolean() ? "127.0.0.1" : "::1"), + randomIntBetween(9200, 9300)); + final Tuple tuple = prepareRestContent("_uri", address, params); final String expectedMessage = tuple.v1().expectedMessage(); final RestRequest request = tuple.v2(); - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); - auditTrail.authenticationFailed("_realm", new MockToken(), request); + final AuthenticationToken mockToken = new MockToken(); + final String realm = randomAlphaOfLengthBetween(1, 6); + auditTrail.authenticationFailed(realm, mockToken, request); assertEmptyLog(logger); // test enabled - settings = - Settings.builder().put(settings).put("xpack.security.audit.logfile.events.include", "realm_authentication_failed").build(); + settings = Settings.builder() + .put(settings) + .put("xpack.security.audit.logfile.events.include", "realm_authentication_failed") + .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); - auditTrail.authenticationFailed("_realm", new MockToken(), request); - if (includeRequestBody) { - assertMsg(logger, Level.INFO, prefix + "[rest] [realm_authentication_failed]\trealm=[_realm], origin_address=[" + - NetworkAddress.format(address) + "], principal=[_principal], uri=[_uri]" + opaqueId + ", request_body=[" + - expectedMessage + "]"); - } else { - assertMsg(logger, Level.INFO, prefix + "[rest] [realm_authentication_failed]\trealm=[_realm], origin_address=[" + - NetworkAddress.format(address) + "], principal=[_principal], uri=[_uri]" + opaqueId); - } + auditTrail.authenticationFailed(realm, mockToken, request); + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.REST_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "realm_authentication_failed") + .put(LoggingAuditTrail.REALM_FIELD_NAME, realm) + .put(LoggingAuditTrail.ORIGIN_TYPE_FIELD_NAME, LoggingAuditTrail.REST_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(address)) + .put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, mockToken.principal()) + .put(LoggingAuditTrail.ACTION_FIELD_NAME, null) + .put(LoggingAuditTrail.REQUEST_BODY_FIELD_NAME, + includeRequestBody && Strings.hasLength(expectedMessage) ? expectedMessage : null) + .put(LoggingAuditTrail.URL_PATH_FIELD_NAME, "_uri") + .put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, params.isEmpty() ? null : "_param=baz"); + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap()); } public void testAccessGranted() throws Exception { - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); - final String origins = LoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); - final boolean runAs = randomBoolean(); - User user; - if (runAs) { - user = new User("running as", new String[]{"r2"}, new User("_username", new String[] {"r1"})); - } else { - user = new User("_username", new String[]{"r1"}); - } - final String role = randomAlphaOfLengthBetween(1, 6); - auditTrail.accessGranted(createAuthentication(user), "_action", message, new String[] { role }); - final String userInfo = (runAs ? "principal=[running as], realm=[lookRealm], run_by_principal=[_username], run_by_realm=[authRealm]" - : "principal=[_username], realm=[authRealm]") + ", roles=[" + role + "]"; - if (message instanceof IndicesRequest) { - assertMsg(logger, Level.INFO, prefix + "[transport] [access_granted]\t" + origins + ", " + userInfo + - ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); - } else { - assertMsg(logger, Level.INFO, prefix + "[transport] [access_granted]\t" + origins + ", " + userInfo + - ", action=[_action], request=[MockMessage]" + opaqueId); - } + final String[] roles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4)); + final Authentication authentication = createAuthentication(); + + auditTrail.accessGranted(authentication, "_action", message, roles); + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + final MapBuilder checkedArrayFields = new MapBuilder<>(); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "access_granted") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action") + .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()); + checkedArrayFields.put(LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME, roles); + subject(authentication, checkedFields); + restOrTransportOrigin(message, threadContext, checkedFields); + indicesRequest(message, checkedFields, checkedArrayFields); + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled CapturingLogger.output(logger.getName(), Level.INFO).clear(); - settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "access_granted").build(); + settings = Settings.builder() + .put(settings) + .put("xpack.security.audit.logfile.events.exclude", "access_granted") + .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); - auditTrail.accessGranted(createAuthentication(user), "_action", message, new String[] { role }); + auditTrail.accessGranted(authentication, "_action", message, roles); assertEmptyLog(logger); } public void testAccessGrantedInternalSystemAction() throws Exception { - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); - final String role = randomAlphaOfLengthBetween(1, 6); - auditTrail.accessGranted(createAuthentication(SystemUser.INSTANCE), "internal:_action", message, new String[] { role }); + final String[] roles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4)); + final Authentication authentication = new Authentication(SystemUser.INSTANCE, new RealmRef("_reserved", "test", "foo"), null); + auditTrail.accessGranted(authentication, "internal:_action", message, roles); assertEmptyLog(logger); // test enabled - settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.include", "system_access_granted").build(); + settings = Settings.builder() + .put(settings) + .put("xpack.security.audit.logfile.events.include", "system_access_granted") + .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); - final String origins = LoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); - auditTrail.accessGranted(createAuthentication(SystemUser.INSTANCE), "internal:_action", message, new String[] { role }); - if (message instanceof IndicesRequest) { - assertMsg(logger, Level.INFO, prefix + "[transport] [access_granted]\t" + origins + ", principal=[" + - SystemUser.INSTANCE.principal() - + "], realm=[authRealm], roles=[" + role + "], action=[internal:_action], indices=[" + indices(message) - + "], request=[MockIndicesRequest]" + opaqueId); - } else { - assertMsg(logger, Level.INFO, prefix + "[transport] [access_granted]\t" + origins + ", principal=[" + - SystemUser.INSTANCE.principal() + "], realm=[authRealm], roles=[" + role - + "], action=[internal:_action], request=[MockMessage]" + opaqueId); - } + auditTrail.accessGranted(authentication, "internal:_action", message, roles); + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + final MapBuilder checkedArrayFields = new MapBuilder<>(); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "access_granted") + .put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, SystemUser.INSTANCE.principal()) + .put(LoggingAuditTrail.PRINCIPAL_REALM_FIELD_NAME, "_reserved") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, "internal:_action") + .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()); + checkedArrayFields.put(LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME, roles); + restOrTransportOrigin(message, threadContext, checkedFields); + indicesRequest(message, checkedFields, checkedArrayFields); + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); } public void testAccessGrantedInternalSystemActionNonSystemUser() throws Exception { - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); - final String origins = LoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); - final boolean runAs = randomBoolean(); - User user; - if (runAs) { - user = new User("running as", new String[]{"r2"}, new User("_username", new String[] {"r1"})); - } else { - user = new User("_username", new String[]{"r1"}); - } - final String role = randomAlphaOfLengthBetween(1, 6); - auditTrail.accessGranted(createAuthentication(user), "internal:_action", message, new String[] { role }); - final String userInfo = (runAs ? "principal=[running as], realm=[lookRealm], run_by_principal=[_username], run_by_realm=[authRealm]" - : "principal=[_username], realm=[authRealm]") + ", roles=[" + role + "]"; - if (message instanceof IndicesRequest) { - assertMsg(logger, Level.INFO, prefix + "[transport] [access_granted]\t" + origins + ", " + userInfo + - ", action=[internal:_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); - } else { - assertMsg(logger, Level.INFO, prefix + "[transport] [access_granted]\t" + origins + ", " + userInfo + - ", action=[internal:_action], request=[MockMessage]" + opaqueId); - } + final String[] roles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4)); + final Authentication authentication = createAuthentication(); + + auditTrail.accessGranted(authentication, "internal:_action", message, roles); + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + final MapBuilder checkedArrayFields = new MapBuilder<>(); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "access_granted") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, "internal:_action") + .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()); + checkedArrayFields.put(LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME, roles); + subject(authentication, checkedFields); + restOrTransportOrigin(message, threadContext, checkedFields); + indicesRequest(message, checkedFields, checkedArrayFields); + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled CapturingLogger.output(logger.getName(), Level.INFO).clear(); - settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "access_granted").build(); + settings = Settings.builder() + .put(settings) + .put("xpack.security.audit.logfile.events.exclude", "access_granted") + .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); - auditTrail.accessGranted(createAuthentication(user), "internal:_action", message, new String[] { role }); + auditTrail.accessGranted(authentication, "internal:_action", message, roles); assertEmptyLog(logger); } public void testAccessDenied() throws Exception { - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); - final String origins = LoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); - final boolean runAs = randomBoolean(); - User user; - if (runAs) { - user = new User("running as", new String[]{"r2"}, new User("_username", new String[] {"r1"})); - } else { - user = new User("_username", new String[]{"r1"}); - } - final String role = randomAlphaOfLengthBetween(1, 6); - auditTrail.accessDenied(createAuthentication(user), "_action", message, new String[] { role }); - final String userInfo = (runAs ? "principal=[running as], realm=[lookRealm], run_by_principal=[_username], run_by_realm=[authRealm]" - : "principal=[_username], realm=[authRealm]") + ", roles=[" + role + "]"; - if (message instanceof IndicesRequest) { - assertMsg(logger, Level.INFO, prefix + "[transport] [access_denied]\t" + origins + ", " + userInfo + - ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); - } else { - assertMsg(logger, Level.INFO, prefix + "[transport] [access_denied]\t" + origins + ", " + userInfo + - ", action=[_action], request=[MockMessage]" + opaqueId); - } + final String[] roles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4)); + final Authentication authentication = createAuthentication(); + + auditTrail.accessDenied(authentication, "_action/bar", message, roles); + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + final MapBuilder checkedArrayFields = new MapBuilder<>(); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "access_denied") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action/bar") + .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()); + checkedArrayFields.put(LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME, roles); + subject(authentication, checkedFields); + restOrTransportOrigin(message, threadContext, checkedFields); + indicesRequest(message, checkedFields, checkedArrayFields); + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled CapturingLogger.output(logger.getName(), Level.INFO).clear(); - settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "access_denied").build(); + settings = Settings.builder() + .put(settings) + .put("xpack.security.audit.logfile.events.exclude", "access_denied") + .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); - auditTrail.accessDenied(createAuthentication(user), "_action", message, new String[] { role }); + auditTrail.accessDenied(authentication, "_action", message, roles); assertEmptyLog(logger); } public void testTamperedRequestRest() throws Exception { - final InetAddress address = forge("_hostname", randomBoolean() ? "127.0.0.1" : "::1"); - final Tuple tuple = prepareRestContent("_uri", new InetSocketAddress(address, 9200)); + final Map params = new HashMap<>(); + if (randomBoolean()) { + params.put("_param", "baz"); + } + final InetSocketAddress address = new InetSocketAddress(forge("_hostname", randomBoolean() ? "127.0.0.1" : "::1"), + randomIntBetween(9200, 9300)); + final Tuple tuple = prepareRestContent("_uri", address, params); final String expectedMessage = tuple.v1().expectedMessage(); final RestRequest request = tuple.v2(); - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.tamperedRequest(request); - if (includeRequestBody) { - assertMsg(logger, Level.INFO, prefix + "[rest] [tampered_request]\torigin_address=[" + - NetworkAddress.format(address) + "], uri=[_uri]" + opaqueId + ", request_body=[" + expectedMessage + "]"); - } else { - assertMsg(logger, Level.INFO, prefix + "[rest] [tampered_request]\torigin_address=[" + - NetworkAddress.format(address) + "], uri=[_uri]" + opaqueId); - } + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.REST_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "tampered_request") + .put(LoggingAuditTrail.ORIGIN_TYPE_FIELD_NAME, LoggingAuditTrail.REST_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(address)) + .put(LoggingAuditTrail.REQUEST_BODY_FIELD_NAME, + includeRequestBody && Strings.hasLength(expectedMessage) ? expectedMessage : null) + .put(LoggingAuditTrail.URL_PATH_FIELD_NAME, "_uri") + .put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, params.isEmpty() ? null : "_param=baz"); + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap()); // test disabled CapturingLogger.output(logger.getName(), Level.INFO).clear(); - settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "tampered_request").build(); + settings = Settings.builder() + .put(settings) + .put("xpack.security.audit.logfile.events.exclude", "tampered_request") + .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.tamperedRequest(request); assertEmptyLog(logger); } public void testTamperedRequest() throws Exception { - final String action = "_action"; final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - final LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); - final String origins = LoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); - auditTrail.tamperedRequest(action, message); - if (message instanceof IndicesRequest) { - assertMsg(logger, Level.INFO, prefix + "[transport] [tampered_request]\t" + origins + - ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); - } else { - assertMsg(logger, Level.INFO, prefix + "[transport] [tampered_request]\t" + origins + - ", action=[_action], request=[MockMessage]" + opaqueId); - } - // test disabled + auditTrail.tamperedRequest("_action", message); + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + final MapBuilder checkedArrayFields = new MapBuilder<>(); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "tampered_request") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action") + .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()); + restOrTransportOrigin(message, threadContext, checkedFields); + indicesRequest(message, checkedFields, checkedArrayFields); + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); + // test disabled + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + settings = Settings.builder() + .put(settings) + .put("xpack.security.audit.logfile.events.exclude", "tampered_request") + .build(); + auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.tamperedRequest("_action", message); + assertEmptyLog(logger); } public void testTamperedRequestWithUser() throws Exception { - final String action = "_action"; + final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); final boolean runAs = randomBoolean(); - User user; + final User user; if (runAs) { - user = new User("running as", new String[]{"r2"}, new User("_username", new String[] {"r1"})); + user = new User("running_as", new String[] { "r2" }, new User("_username", new String[] { "r1" })); } else { - user = new User("_username", new String[]{"r1"}); + user = new User("_username", new String[] { "r1" }); } - final String userInfo = runAs ? "principal=[running as], run_by_principal=[_username]" : "principal=[_username]"; - final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); - final String origins = LoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); - auditTrail.tamperedRequest(user, action, message); - if (message instanceof IndicesRequest) { - assertMsg(logger, Level.INFO, prefix + "[transport] [tampered_request]\t" + origins + ", " + userInfo + - ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); + + auditTrail.tamperedRequest(user, "_action", message); + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + final MapBuilder checkedArrayFields = new MapBuilder<>(); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "tampered_request") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action") + .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()); + if (runAs) { + checkedFields.put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, "running_as"); + checkedFields.put(LoggingAuditTrail.PRINCIPAL_RUN_BY_FIELD_NAME, "_username"); } else { - assertMsg(logger, Level.INFO, prefix + "[transport] [tampered_request]\t" + origins + ", " + userInfo + - ", action=[_action], request=[MockMessage]" + opaqueId); + checkedFields.put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, "_username"); } + restOrTransportOrigin(message, threadContext, checkedFields); + indicesRequest(message, checkedFields, checkedArrayFields); + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled CapturingLogger.output(logger.getName(), Level.INFO).clear(); - settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "tampered_request").build(); + settings = Settings.builder() + .put(settings) + .put("xpack.security.audit.logfile.events.exclude", "tampered_request") + .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); - auditTrail.tamperedRequest(user, action, message); + auditTrail.tamperedRequest(user, "_action", message); assertEmptyLog(logger); } public void testConnectionDenied() throws Exception { - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); final InetAddress inetAddress = InetAddress.getLoopbackAddress(); final SecurityIpFilterRule rule = new SecurityIpFilterRule(false, "_all"); - auditTrail.connectionDenied(inetAddress, "default", rule); - assertMsg(logger, Level.INFO, String.format(Locale.ROOT, prefix + - "[ip_filter] [connection_denied]\torigin_address=[%s], transport_profile=[%s], rule=[deny %s]" + opaqueId, - NetworkAddress.format(inetAddress), "default", "_all")); + final String profile = randomAlphaOfLengthBetween(1, 6); + + auditTrail.connectionDenied(inetAddress, profile, rule); + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.IP_FILTER_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "connection_denied") + .put(LoggingAuditTrail.ORIGIN_TYPE_FIELD_NAME, LoggingAuditTrail.IP_FILTER_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(inetAddress)) + .put(LoggingAuditTrail.TRANSPORT_PROFILE_FIELD_NAME, profile) + .put(LoggingAuditTrail.RULE_FIELD_NAME, "deny _all"); + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap()); // test disabled CapturingLogger.output(logger.getName(), Level.INFO).clear(); - settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "connection_denied").build(); + settings = Settings.builder() + .put(settings) + .put("xpack.security.audit.logfile.events.exclude", "connection_denied") + .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); - auditTrail.connectionDenied(inetAddress, "default", rule); + auditTrail.connectionDenied(inetAddress, profile, rule); assertEmptyLog(logger); } public void testConnectionGranted() throws Exception { - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); final InetAddress inetAddress = InetAddress.getLoopbackAddress(); final SecurityIpFilterRule rule = IPFilter.DEFAULT_PROFILE_ACCEPT_ALL; - auditTrail.connectionGranted(inetAddress, "default", rule); + final String profile = randomAlphaOfLengthBetween(1, 6); + + auditTrail.connectionGranted(inetAddress, profile, rule); assertEmptyLog(logger); // test enabled - CapturingLogger.output(logger.getName(), Level.INFO).clear(); - settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.include", "connection_granted").build(); + settings = Settings.builder() + .put(settings) + .put("xpack.security.audit.logfile.events.include", "connection_granted") + .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); - auditTrail.connectionGranted(inetAddress, "default", rule); - assertMsg(logger, Level.INFO, String.format(Locale.ROOT, prefix + "[ip_filter] [connection_granted]\torigin_address=[%s], " + - "transport_profile=[default], rule=[allow default:accept_all]" + opaqueId, - NetworkAddress.format(inetAddress))); + auditTrail.connectionGranted(inetAddress, profile, rule); + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.IP_FILTER_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "connection_granted") + .put(LoggingAuditTrail.ORIGIN_TYPE_FIELD_NAME, LoggingAuditTrail.IP_FILTER_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(inetAddress)) + .put(LoggingAuditTrail.TRANSPORT_PROFILE_FIELD_NAME, profile) + .put(LoggingAuditTrail.RULE_FIELD_NAME, "allow default:accept_all"); + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap()); } public void testRunAsGranted() throws Exception { - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); - final String origins = LoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); - final User user = new User("running as", new String[]{"r2"}, new User("_username", new String[] {"r1"})); - final String role = randomAlphaOfLengthBetween(1, 6); - auditTrail.runAsGranted(createAuthentication(user), "_action", message, new String[] { role }); - if (message instanceof IndicesRequest) { - assertMsg(logger, Level.INFO, - prefix + "[transport] [run_as_granted]\t" + origins - + ", principal=[_username], realm=[authRealm], run_as_principal=[running as], run_as_realm=[lookRealm], roles=[" - + role + "], action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); - } else { - assertMsg(logger, Level.INFO, - prefix + "[transport] [run_as_granted]\t" + origins - + ", principal=[_username], realm=[authRealm], run_as_principal=[running as], run_as_realm=[lookRealm], roles=[" - + role + "], action=[_action], request=[MockMessage]" + opaqueId); - } + final String[] roles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4)); + final Authentication authentication = new Authentication( + new User("running as", new String[] { "r2" }, new User("_username", new String[] { "r1" })), + new RealmRef("authRealm", "test", "foo"), + new RealmRef("lookRealm", "up", "by")); + + auditTrail.runAsGranted(authentication, "_action", message, roles); + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + final MapBuilder checkedArrayFields = new MapBuilder<>(); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "run_as_granted") + .put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, "_username") + .put(LoggingAuditTrail.PRINCIPAL_REALM_FIELD_NAME, "authRealm") + .put(LoggingAuditTrail.PRINCIPAL_RUN_AS_FIELD_NAME, "running as") + .put(LoggingAuditTrail.PRINCIPAL_RUN_AS_REALM_FIELD_NAME, "lookRealm") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action") + .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()); + checkedArrayFields.put(LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME, roles); + restOrTransportOrigin(message, threadContext, checkedFields); + indicesRequest(message, checkedFields, checkedArrayFields); + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled CapturingLogger.output(logger.getName(), Level.INFO).clear(); - settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "run_as_granted").build(); + settings = Settings.builder() + .put(settings) + .put("xpack.security.audit.logfile.events.exclude", "run_as_granted") + .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); - auditTrail.runAsGranted(createAuthentication(user), "_action", message, new String[] { role }); + auditTrail.runAsGranted(authentication, "_action", message, roles); assertEmptyLog(logger); } public void testRunAsDenied() throws Exception { - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); - final String origins = LoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); - final User user = new User("running as", new String[]{"r2"}, new User("_username", new String[] {"r1"})); - final String role = randomAlphaOfLengthBetween(1, 6); - auditTrail.runAsDenied(createAuthentication(user), "_action", message, new String[] { role }); - if (message instanceof IndicesRequest) { - assertMsg(logger, Level.INFO, - prefix + "[transport] [run_as_denied]\t" + origins - + ", principal=[_username], realm=[authRealm], run_as_principal=[running as], run_as_realm=[lookRealm], roles=[" - + role + "], action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); - } else { - assertMsg(logger, Level.INFO, - prefix + "[transport] [run_as_denied]\t" + origins - + ", principal=[_username], realm=[authRealm], run_as_principal=[running as], run_as_realm=[lookRealm], roles=[" - + role + "], action=[_action], request=[MockMessage]" + opaqueId); - } + final String[] roles = randomArray(0, 4, String[]::new, () -> randomAlphaOfLengthBetween(1, 4)); + final Authentication authentication = new Authentication( + new User("running as", new String[] { "r2" }, new User("_username", new String[] { "r1" })), + new RealmRef("authRealm", "test", "foo"), + new RealmRef("lookRealm", "up", "by")); + + auditTrail.runAsDenied(authentication, "_action", message, roles); + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + final MapBuilder checkedArrayFields = new MapBuilder<>(); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "run_as_denied") + .put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, "_username") + .put(LoggingAuditTrail.PRINCIPAL_REALM_FIELD_NAME, "authRealm") + .put(LoggingAuditTrail.PRINCIPAL_RUN_AS_FIELD_NAME, "running as") + .put(LoggingAuditTrail.PRINCIPAL_RUN_AS_REALM_FIELD_NAME, "lookRealm") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action") + .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()); + checkedArrayFields.put(LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME, roles); + restOrTransportOrigin(message, threadContext, checkedFields); + indicesRequest(message, checkedFields, checkedArrayFields); + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled CapturingLogger.output(logger.getName(), Level.INFO).clear(); - settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "run_as_denied").build(); + settings = Settings.builder() + .put(settings) + .put("xpack.security.audit.logfile.events.exclude", "run_as_denied") + .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); - auditTrail.runAsDenied(createAuthentication(user), "_action", message, new String[] { role }); + auditTrail.runAsDenied(authentication, "_action", message, roles); assertEmptyLog(logger); } - public void testOriginAttributes() throws Exception { - final MockMessage message = new MockMessage(threadContext); - final LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); - final String text = LoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); - final InetSocketAddress restAddress = RemoteHostHeader.restRemoteAddress(threadContext); - if (restAddress != null) { - assertThat(text, equalTo("origin_type=[rest], origin_address=[" + - NetworkAddress.format(restAddress.getAddress()) + "]")); - return; - } - final TransportAddress address = message.remoteAddress(); - if (address == null) { - assertThat(text, equalTo("origin_type=[local_node], origin_address=[" + localNode.getHostAddress() + "]")); - return; - } - - assertThat(text, equalTo("origin_type=[transport], origin_address=[" + - NetworkAddress.format(address.address().getAddress()) + "]")); - } - public void testAuthenticationSuccessRest() throws Exception { final Map params = new HashMap<>(); - params.put("foo", "bar"); - final InetAddress address = forge("_hostname", randomBoolean() ? "127.0.0.1" : "::1"); - final Tuple tuple = prepareRestContent("_uri", new InetSocketAddress(address, 9200), params); + if (randomBoolean()) { + params.put("foo", "bar"); + params.put("evac", "true"); + } + final InetSocketAddress address = new InetSocketAddress(forge("_hostname", randomBoolean() ? "127.0.0.1" : "::1"), + randomIntBetween(9200, 9300)); + final Tuple tuple = prepareRestContent("_uri", address, params); final String expectedMessage = tuple.v1().expectedMessage(); final RestRequest request = tuple.v2(); - final boolean runAs = randomBoolean(); - User user; - if (runAs) { - user = new User("running as", new String[]{"r2"}, new User("_username", new String[] {"r1"})); + final String realm = randomAlphaOfLengthBetween(1, 6); + final User user; + if (randomBoolean()) { + user = new User("running as", new String[] { "r2" }, new User("_username", new String[] { "r1" })); } else { user = new User("_username", new String[] { "r1" }); } - final String userInfo = runAs ? "principal=[running as], run_by_principal=[_username]" : "principal=[_username]"; - final String realm = "_realm"; - Settings settings = Settings.builder().put(this.settings) - .put("xpack.security.audit.logfile.events.include", "authentication_success") - .build(); - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); + // event by default disabled auditTrail.authenticationSuccess(realm, user, request); - if (includeRequestBody) { - assertMsg(logger, Level.INFO, - prefix + "[rest] [authentication_success]\t" + userInfo + ", realm=[_realm], uri=[_uri], params=[" + params - + "]" + opaqueId + ", request_body=[" + expectedMessage + "]"); - } else { - assertMsg(logger, Level.INFO, - prefix + "[rest] [authentication_success]\t" + userInfo + ", realm=[_realm], uri=[_uri], params=[" + params - + "]" + opaqueId); - } + assertEmptyLog(logger); - // test disabled - CapturingLogger.output(logger.getName(), Level.INFO).clear(); - settings = Settings.builder().put(this.settings).put("xpack.security.audit.logfile.events.exclude", "authentication_success") + settings = Settings.builder() + .put(this.settings) + .put("xpack.security.audit.logfile.events.include", "authentication_success") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.authenticationSuccess(realm, user, request); - assertEmptyLog(logger); + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.REST_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "authentication_success") + .put(LoggingAuditTrail.REALM_FIELD_NAME, realm) + .put(LoggingAuditTrail.ORIGIN_TYPE_FIELD_NAME, LoggingAuditTrail.REST_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(address)) + .put(LoggingAuditTrail.REQUEST_BODY_FIELD_NAME, + includeRequestBody && Strings.hasLength(expectedMessage) ? expectedMessage : null) + .put(LoggingAuditTrail.URL_PATH_FIELD_NAME, "_uri") + .put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, params.isEmpty() ? null : "foo=bar&evac=true"); + if (user.isRunAs()) { + checkedFields.put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, "running as"); + checkedFields.put(LoggingAuditTrail.PRINCIPAL_RUN_BY_FIELD_NAME, "_username"); + } else { + checkedFields.put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, "_username"); + } + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap()); } public void testAuthenticationSuccessTransport() throws Exception { - Settings settings = Settings.builder().put(this.settings) - .put("xpack.security.audit.logfile.events.include", "authentication_success").build(); - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); - final String origins = LoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); - final boolean runAs = randomBoolean(); - User user; - if (runAs) { - user = new User("running as", new String[]{"r2"}, new User("_username", new String[] {"r1"})); + final User user; + if (randomBoolean()) { + user = new User("running as", new String[] { "r2" }, new User("_username", new String[] { "r1" })); } else { user = new User("_username", new String[] { "r1" }); } - final String userInfo = runAs ? "principal=[running as], run_by_principal=[_username]" : "principal=[_username]"; - final String realm = "_realm"; + final String realm = randomAlphaOfLengthBetween(1, 6); + + // event by default disabled auditTrail.authenticationSuccess(realm, user, "_action", message); - if (message instanceof IndicesRequest) { - assertMsg(logger, Level.INFO, prefix + "[transport] [authentication_success]\t" + origins + ", " + userInfo - + ", realm=[_realm], action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); - } else { - assertMsg(logger, Level.INFO, prefix + "[transport] [authentication_success]\t" + origins + ", " + userInfo - + ", realm=[_realm], action=[_action], request=[MockMessage]" + opaqueId); - } + assertEmptyLog(logger); - // test disabled - CapturingLogger.output(logger.getName(), Level.INFO).clear(); settings = Settings.builder() .put(this.settings) - .put("xpack.security.audit.logfile.events.exclude", "authentication_success") + .put("xpack.security.audit.logfile.events.include", "authentication_success") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.authenticationSuccess(realm, user, "_action", message); - assertEmptyLog(logger); + final MapBuilder checkedFields = new MapBuilder<>(commonFields); + final MapBuilder checkedArrayFields = new MapBuilder<>(); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "authentication_success") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action") + .put(LoggingAuditTrail.REALM_FIELD_NAME, realm) + .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()); + if (user.isRunAs()) { + checkedFields.put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, "running as"); + checkedFields.put(LoggingAuditTrail.PRINCIPAL_RUN_BY_FIELD_NAME, "_username"); + } else { + checkedFields.put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, "_username"); + } + restOrTransportOrigin(message, threadContext, checkedFields); + indicesRequest(message, checkedFields, checkedArrayFields); + opaqueId(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); } public void testRequestsWithoutIndices() throws Exception { - final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - final Settings allEventsSettings = Settings.builder() + settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.include", "_all") .build(); - final LoggingAuditTrail auditTrail = new LoggingAuditTrail(allEventsSettings, clusterService, logger, threadContext); + auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); final User user = new User("_username", new String[] { "r1" }); final String role = randomAlphaOfLengthBetween(1, 6); final String realm = randomAlphaOfLengthBetween(1, 6); @@ -748,48 +896,93 @@ public void testRequestsWithoutIndices() throws Exception { for (final TransportMessage message : messages) { auditTrail.anonymousAccessDenied("_action", message); assertThat(output.size(), is(logEntriesCount++)); - assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices="))); auditTrail.authenticationFailed(new MockToken(), "_action", message); assertThat(output.size(), is(logEntriesCount++)); - assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices="))); auditTrail.authenticationFailed("_action", message); assertThat(output.size(), is(logEntriesCount++)); - assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices="))); auditTrail.authenticationFailed(realm, new MockToken(), "_action", message); assertThat(output.size(), is(logEntriesCount++)); - assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); - auditTrail.accessGranted(createAuthentication(user), "_action", message, new String[] { role }); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices="))); + auditTrail.accessGranted(createAuthentication(), "_action", message, new String[] { role }); assertThat(output.size(), is(logEntriesCount++)); - assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); - auditTrail.accessDenied(createAuthentication(user), "_action", message, new String[] { role }); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices="))); + auditTrail.accessDenied(createAuthentication(), "_action", message, new String[] { role }); assertThat(output.size(), is(logEntriesCount++)); - assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices="))); auditTrail.tamperedRequest("_action", message); assertThat(output.size(), is(logEntriesCount++)); - assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices="))); auditTrail.tamperedRequest(user, "_action", message); assertThat(output.size(), is(logEntriesCount++)); - assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); - auditTrail.runAsGranted(createAuthentication(user), "_action", message, new String[] { role }); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices="))); + auditTrail.runAsGranted(createAuthentication(), "_action", message, new String[] { role }); assertThat(output.size(), is(logEntriesCount++)); - assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); - auditTrail.runAsDenied(createAuthentication(user), "_action", message, new String[] { role }); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices="))); + auditTrail.runAsDenied(createAuthentication(), "_action", message, new String[] { role }); assertThat(output.size(), is(logEntriesCount++)); - assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices="))); auditTrail.authenticationSuccess(realm, user, "_action", message); assertThat(output.size(), is(logEntriesCount++)); - assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices="))); } } - private void assertMsg(Logger logger, Level level, String message) { - final List output = CapturingLogger.output(logger.getName(), level); - assertThat(output.size(), is(1)); - assertThat(output.get(0), equalTo(message)); + private void assertMsg(Logger logger, Map checkFields) { + assertMsg(logger, checkFields, Collections.emptyMap()); + } + + private void assertMsg(Logger logger, Map checkFields, Map checkArrayFields) { + final List output = CapturingLogger.output(logger.getName(), Level.INFO); + assertThat("Exactly one logEntry expected. Found: " + output.size(), output.size(), is(1)); + if (checkFields == null) { + // only check msg existence + return; + } + String logLine = output.get(0); + // check each field + for (final Map.Entry checkField : checkFields.entrySet()) { + if (null == checkField.getValue()) { + // null checkField means that the field does not exist + assertThat("Field: " + checkField.getKey() + " should be missing.", + logLine.contains(Pattern.quote("\"" + checkField.getKey() + "\":")), is(false)); + } else { + final String quotedValue = "\"" + checkField.getValue().replaceAll("\"", "\\\\\"") + "\""; + final Pattern logEntryFieldPattern = Pattern.compile(Pattern.quote("\"" + checkField.getKey() + "\":" + quotedValue)); + assertThat("Field " + checkField.getKey() + " value mismatch. Expected " + quotedValue, + logEntryFieldPattern.matcher(logLine).find(), is(true)); + // remove checked field + logLine = logEntryFieldPattern.matcher(logLine).replaceFirst(""); + } + } + for (final Map.Entry checkArrayField : checkArrayFields.entrySet()) { + if (null == checkArrayField.getValue()) { + // null checkField means that the field does not exist + assertThat("Field: " + checkArrayField.getKey() + " should be missing.", + logLine.contains(Pattern.quote("\"" + checkArrayField.getKey() + "\":")), is(false)); + } else { + final String quotedValue = "[" + Arrays.asList(checkArrayField.getValue()) + .stream() + .filter(s -> s != null) + .map(s -> "\"" + s.replaceAll("\"", "\\\\\"") + "\"") + .reduce((x, y) -> x + "," + y) + .orElse("") + "]"; + final Pattern logEntryFieldPattern = Pattern.compile(Pattern.quote("\"" + checkArrayField.getKey() + "\":" + quotedValue)); + assertThat("Field " + checkArrayField.getKey() + " value mismatch. Expected " + quotedValue, + logEntryFieldPattern.matcher(logLine).find(), is(true)); + // remove checked field + logLine = logEntryFieldPattern.matcher(logLine).replaceFirst(""); + } + } + logLine = logLine.replaceFirst("\"@timestamp\":\"[^\"]*\"", "").replaceAll("[{},]", ""); + // check no extra fields + assertThat("Log event has extra unexpected content: " + logLine, Strings.hasText(logLine), is(false)); } private void assertEmptyLog(Logger logger) { - assertThat(CapturingLogger.isEmpty(logger.getName()), is(true)); + assertThat("Logger is not empty", CapturingLogger.isEmpty(logger.getName()), is(true)); } protected Tuple prepareRestContent(String uri, InetSocketAddress remoteAddress) { @@ -802,7 +995,18 @@ private Tuple prepareRestContent(String uri, InetSocke if (content.hasContent()) { builder.withContent(content.content(), XContentType.JSON); } - builder.withPath(uri); + if (params.isEmpty()) { + builder.withPath(uri); + } else { + final StringBuilder queryString = new StringBuilder("?"); + for (final Map.Entry entry : params.entrySet()) { + if (queryString.length() > 1) { + queryString.append('&'); + } + queryString.append(entry.getKey() + "=" + entry.getValue()); + } + builder.withPath(uri + queryString.toString()); + } builder.withRemoteAddress(remoteAddress); builder.withParams(params); return new Tuple<>(content, builder.build()); @@ -814,12 +1018,16 @@ protected static InetAddress forge(String hostname, String address) throws IOExc return InetAddress.getByAddress(hostname, bytes); } - private static String indices(TransportMessage message) { - return Strings.arrayToCommaDelimitedString(((IndicesRequest) message).indices()); - } - - private static Authentication createAuthentication(User user) { - final RealmRef lookedUpBy = user.authenticatedUser() == user ? null : new RealmRef("lookRealm", "up", "by"); + private static Authentication createAuthentication() { + final RealmRef lookedUpBy; + final User user; + if (randomBoolean()) { + user = new User("running_as", new String[] { "r2" }, new User("_username", new String[] { "r1" })); + lookedUpBy = new RealmRef("lookRealm", "up", "by"); + } else { + user = new User("_username", new String[] { "r1" }); + lookedUpBy = null; + } return new Authentication(user, new RealmRef("authRealm", "test", "foo"), lookedUpBy); } @@ -849,7 +1057,8 @@ static class MockMessage extends TransportMessage { static class MockIndicesRequest extends org.elasticsearch.action.MockIndicesRequest { MockIndicesRequest(ThreadContext threadContext) throws IOException { - super(IndicesOptions.strictExpandOpenAndForbidClosed(), "idx1", "idx2"); + super(IndicesOptions.strictExpandOpenAndForbidClosed(), + randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4))); if (randomBoolean()) { remoteAddress(buildNewFakeTransportAddress()); } @@ -882,4 +1091,50 @@ public void clearCredentials() { } } + private static void restOrTransportOrigin(TransportMessage message, ThreadContext threadContext, + MapBuilder checkedFields) { + final InetSocketAddress restAddress = RemoteHostHeader.restRemoteAddress(threadContext); + if (restAddress != null) { + checkedFields.put(LoggingAuditTrail.ORIGIN_TYPE_FIELD_NAME, LoggingAuditTrail.REST_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(restAddress)); + } else { + final TransportAddress address = message.remoteAddress(); + if (address != null) { + checkedFields.put(LoggingAuditTrail.ORIGIN_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(address.address())); + } + } + } + + private static void subject(Authentication authentication, MapBuilder checkedFields) { + checkedFields.put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, authentication.getUser().principal()); + if (authentication.getUser().isRunAs()) { + checkedFields.put(LoggingAuditTrail.PRINCIPAL_REALM_FIELD_NAME, authentication.getLookedUpBy().getName()) + .put(LoggingAuditTrail.PRINCIPAL_RUN_BY_FIELD_NAME, authentication.getUser().authenticatedUser().principal()) + .put(LoggingAuditTrail.PRINCIPAL_RUN_BY_REALM_FIELD_NAME, authentication.getAuthenticatedBy().getName()); + } else { + checkedFields.put(LoggingAuditTrail.PRINCIPAL_REALM_FIELD_NAME, authentication.getAuthenticatedBy().getName()); + } + } + + private static void opaqueId(ThreadContext threadContext, MapBuilder checkedFields) { + final String opaqueId = threadContext.getHeader(Task.X_OPAQUE_ID); + if (opaqueId != null) { + checkedFields.put(LoggingAuditTrail.OPAQUE_ID_FIELD_NAME, opaqueId); + } else { + checkedFields.put(LoggingAuditTrail.OPAQUE_ID_FIELD_NAME, null); + } + } + + private static void indicesRequest(TransportMessage message, MapBuilder checkedFields, + MapBuilder checkedArrayFields) { + if (message instanceof IndicesRequest) { + checkedFields.put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, MockIndicesRequest.class.getSimpleName()); + checkedArrayFields.put(LoggingAuditTrail.INDICES_FIELD_NAME, ((IndicesRequest) message).indices()); + } else { + checkedFields.put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, MockMessage.class.getSimpleName()); + checkedArrayFields.put(LoggingAuditTrail.INDICES_FIELD_NAME, null); + } + } + } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStoreTests.java index 739952af63ee7..3acb4888d7834 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStoreTests.java @@ -185,7 +185,7 @@ public void testParseFile() throws Exception { public void testParseFile_Empty() throws Exception { Path empty = createTempFile(); - Logger logger = CapturingLogger.newCapturingLogger(Level.DEBUG); + Logger logger = CapturingLogger.newCapturingLogger(Level.DEBUG, null); Map users = FileUserPasswdStore.parseFile(empty, logger, Settings.EMPTY); assertThat(users.isEmpty(), is(true)); List events = CapturingLogger.output(logger.getName(), Level.DEBUG); @@ -195,7 +195,7 @@ public void testParseFile_Empty() throws Exception { public void testParseFile_WhenFileDoesNotExist() throws Exception { Path file = createTempDir().resolve(randomAlphaOfLength(10)); - Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); + Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); Map users = FileUserPasswdStore.parseFile(file, logger, Settings.EMPTY); assertThat(users, nullValue()); users = FileUserPasswdStore.parseFileLenient(file, logger, Settings.EMPTY); @@ -207,7 +207,7 @@ public void testParseFile_WhenCannotReadFile() throws Exception { Path file = createTempFile(); // writing in utf_16 should cause a parsing error as we try to read the file in utf_8 Files.write(file, Collections.singletonList("aldlfkjldjdflkjd"), StandardCharsets.UTF_16); - Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); + Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); try { FileUserPasswdStore.parseFile(file, logger, Settings.EMPTY); fail("expected a parse failure"); @@ -228,7 +228,7 @@ public void testParseFileLenient_WhenCannotReadFile() throws Exception { Path file = createTempFile(); // writing in utf_16 should cause a parsing error as we try to read the file in utf_8 Files.write(file, Collections.singletonList("aldlfkjldjdflkjd"), StandardCharsets.UTF_16); - Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); + Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); Map users = FileUserPasswdStore.parseFileLenient(file, logger, Settings.EMPTY); assertThat(users, notNullValue()); assertThat(users.isEmpty(), is(true)); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserRolesStoreTests.java index f987f4df5724f..8e4011b1159d5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserRolesStoreTests.java @@ -175,7 +175,7 @@ public void testParseFile() throws Exception { public void testParseFileEmpty() throws Exception { Path empty = createTempFile(); - Logger log = CapturingLogger.newCapturingLogger(Level.DEBUG); + Logger log = CapturingLogger.newCapturingLogger(Level.DEBUG, null); FileUserRolesStore.parseFile(empty, log); List events = CapturingLogger.output(log.getName(), Level.DEBUG); assertThat(events.size(), is(1)); @@ -184,7 +184,7 @@ public void testParseFileEmpty() throws Exception { public void testParseFileWhenFileDoesNotExist() throws Exception { Path file = createTempDir().resolve(randomAlphaOfLength(10)); - Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); + Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); Map usersRoles = FileUserRolesStore.parseFile(file, logger); assertThat(usersRoles, nullValue()); usersRoles = FileUserRolesStore.parseFileLenient(file, logger); @@ -199,7 +199,7 @@ public void testParseFileWhenCannotReadFile() throws Exception { // writing in utf_16 should cause a parsing error as we try to read the file in utf_8 Files.write(file, lines, StandardCharsets.UTF_16); - Logger logger = CapturingLogger.newCapturingLogger(Level.DEBUG); + Logger logger = CapturingLogger.newCapturingLogger(Level.DEBUG, null); try { FileUserRolesStore.parseFile(file, logger); fail("expected a parse failure"); @@ -256,7 +256,7 @@ public void testParseFileLenientWhenCannotReadFile() throws Exception { // writing in utf_16 should cause a parsing error as we try to read the file in utf_8 Files.write(file, lines, StandardCharsets.UTF_16); - Logger logger = CapturingLogger.newCapturingLogger(Level.DEBUG); + Logger logger = CapturingLogger.newCapturingLogger(Level.DEBUG, null); Map usersRoles = FileUserRolesStore.parseFileLenient(file, logger); assertThat(usersRoles, notNullValue()); assertThat(usersRoles.isEmpty(), is(true)); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/DnRoleMapperTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/DnRoleMapperTests.java index f6d18b7cfc106..263c5ee4929b4 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/DnRoleMapperTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/DnRoleMapperTests.java @@ -199,7 +199,7 @@ public void testAddNullListener() throws Exception { public void testParseFile() throws Exception { Path file = getDataPath("role_mapping.yml"); - Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); + Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); Map> mappings = DnRoleMapper.parseFile(file, logger, "_type", "_name", false); assertThat(mappings, notNullValue()); assertThat(mappings.size(), is(3)); @@ -229,18 +229,19 @@ public void testParseFile() throws Exception { public void testParseFile_Empty() throws Exception { Path file = createTempDir().resolve("foo.yaml"); Files.createFile(file); - Logger logger = CapturingLogger.newCapturingLogger(Level.DEBUG); + Logger logger = CapturingLogger.newCapturingLogger(Level.DEBUG, null); Map> mappings = DnRoleMapper.parseFile(file, logger, "_type", "_name", false); assertThat(mappings, notNullValue()); assertThat(mappings.isEmpty(), is(true)); List events = CapturingLogger.output(logger.getName(), Level.DEBUG); assertThat(events.size(), is(1)); assertThat(events.get(0), containsString("[0] role mappings found")); + events.clear(); } public void testParseFile_WhenFileDoesNotExist() throws Exception { Path file = createTempDir().resolve(randomAlphaOfLength(10)); - Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); + Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); Map> mappings = DnRoleMapper.parseFile(file, logger, "_type", "_name", false); assertThat(mappings, notNullValue()); assertThat(mappings.isEmpty(), is(true)); @@ -257,7 +258,7 @@ public void testParseFile_WhenCannotReadFile() throws Exception { Path file = createTempFile("", ".yml"); // writing in utf_16 should cause a parsing error as we try to read the file in utf_8 Files.write(file, Collections.singletonList("aldlfkjldjdflkjd"), StandardCharsets.UTF_16); - Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); + Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); try { DnRoleMapper.parseFile(file, logger, "_type", "_name", false); fail("expected a parse failure"); @@ -270,13 +271,14 @@ public void testParseFileLenient_WhenCannotReadFile() throws Exception { Path file = createTempFile("", ".yml"); // writing in utf_16 should cause a parsing error as we try to read the file in utf_8 Files.write(file, Collections.singletonList("aldlfkjldjdflkjd"), StandardCharsets.UTF_16); - Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); + Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); Map> mappings = DnRoleMapper.parseFileLenient(file, logger, "_type", "_name"); assertThat(mappings, notNullValue()); assertThat(mappings.isEmpty(), is(true)); List events = CapturingLogger.output(logger.getName(), Level.ERROR); assertThat(events.size(), is(1)); assertThat(events.get(0), containsString("failed to parse role mappings file")); + events.clear(); } public void testYaml() throws Exception { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/FileRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/FileRolesStoreTests.java index 1e2428e77791b..5cb93b898ba52 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/FileRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/FileRolesStoreTests.java @@ -237,7 +237,9 @@ public void testParseFile() throws Exception { public void testParseFileWithFLSAndDLSDisabled() throws Exception { Path path = getDataPath("roles.yml"); - Logger logger = CapturingLogger.newCapturingLogger(Level.ERROR); + Logger logger = CapturingLogger.newCapturingLogger(Level.ERROR, null); + List events = CapturingLogger.output(logger.getName(), Level.ERROR); + events.clear(); Map roles = FileRolesStore.parseFile(path, logger, Settings.builder() .put(XPackSettings.DLS_FLS_ENABLED.getKey(), false) .build(), new XPackLicenseState(Settings.EMPTY)); @@ -247,7 +249,6 @@ public void testParseFileWithFLSAndDLSDisabled() throws Exception { assertThat(roles.get("role_query"), nullValue()); assertThat(roles.get("role_query_fields"), nullValue()); - List events = CapturingLogger.output(logger.getName(), Level.ERROR); assertThat(events, hasSize(3)); assertThat( events.get(0), @@ -263,7 +264,9 @@ public void testParseFileWithFLSAndDLSDisabled() throws Exception { public void testParseFileWithFLSAndDLSUnlicensed() throws Exception { Path path = getDataPath("roles.yml"); - Logger logger = CapturingLogger.newCapturingLogger(Level.WARN); + Logger logger = CapturingLogger.newCapturingLogger(Level.WARN, null); + List events = CapturingLogger.output(logger.getName(), Level.WARN); + events.clear(); XPackLicenseState licenseState = mock(XPackLicenseState.class); when(licenseState.isDocumentAndFieldLevelSecurityAllowed()).thenReturn(false); Map roles = FileRolesStore.parseFile(path, logger, Settings.EMPTY, licenseState); @@ -273,7 +276,6 @@ public void testParseFileWithFLSAndDLSUnlicensed() throws Exception { assertNotNull(roles.get("role_query")); assertNotNull(roles.get("role_query_fields")); - List events = CapturingLogger.output(logger.getName(), Level.WARN); assertThat(events, hasSize(3)); assertThat( events.get(0), @@ -369,7 +371,9 @@ public void testThatEmptyFileDoesNotResultInLoop() throws Exception { public void testThatInvalidRoleDefinitions() throws Exception { Path path = getDataPath("invalid_roles.yml"); - Logger logger = CapturingLogger.newCapturingLogger(Level.ERROR); + Logger logger = CapturingLogger.newCapturingLogger(Level.ERROR, null); + List entries = CapturingLogger.output(logger.getName(), Level.ERROR); + entries.clear(); Map roles = FileRolesStore.parseFile(path, logger, Settings.EMPTY, new XPackLicenseState(Settings.EMPTY)); assertThat(roles.size(), is(1)); assertThat(roles, hasKey("valid_role")); @@ -379,7 +383,6 @@ public void testThatInvalidRoleDefinitions() throws Exception { assertThat(role, notNullValue()); assertThat(role.names(), equalTo(new String[] { "valid_role" })); - List entries = CapturingLogger.output(logger.getName(), Level.ERROR); assertThat(entries, hasSize(6)); assertThat( entries.get(0), @@ -395,12 +398,13 @@ public void testThatInvalidRoleDefinitions() throws Exception { public void testThatRoleNamesDoesNotResolvePermissions() throws Exception { Path path = getDataPath("invalid_roles.yml"); - Logger logger = CapturingLogger.newCapturingLogger(Level.ERROR); + Logger logger = CapturingLogger.newCapturingLogger(Level.ERROR, null); + List events = CapturingLogger.output(logger.getName(), Level.ERROR); + events.clear(); Set roleNames = FileRolesStore.parseFileForRoleNames(path, logger); assertThat(roleNames.size(), is(6)); assertThat(roleNames, containsInAnyOrder("valid_role", "role1", "role2", "role3", "role4", "role5")); - List events = CapturingLogger.output(logger.getName(), Level.ERROR); assertThat(events, hasSize(1)); assertThat( events.get(0), @@ -408,9 +412,9 @@ public void testThatRoleNamesDoesNotResolvePermissions() throws Exception { } public void testReservedRoles() throws Exception { - - Logger logger = CapturingLogger.newCapturingLogger(Level.INFO); - + Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + List events = CapturingLogger.output(logger.getName(), Level.ERROR); + events.clear(); Path path = getDataPath("reserved_roles.yml"); Map roles = FileRolesStore.parseFile(path, logger, Settings.EMPTY, new XPackLicenseState(Settings.EMPTY)); assertThat(roles, notNullValue()); @@ -418,7 +422,6 @@ public void testReservedRoles() throws Exception { assertThat(roles, hasKey("admin")); - List events = CapturingLogger.output(logger.getName(), Level.ERROR); assertThat(events, notNullValue()); assertThat(events, hasSize(4)); // the system role will always be checked first diff --git a/x-pack/qa/sql/security/build.gradle b/x-pack/qa/sql/security/build.gradle index 5c3169d9d20dc..d1acbcc125adc 100644 --- a/x-pack/qa/sql/security/build.gradle +++ b/x-pack/qa/sql/security/build.gradle @@ -38,7 +38,7 @@ subprojects { integTestRunner { systemProperty 'tests.audit.logfile', - "${ -> integTest.nodes[0].homeDir}/logs/${ -> integTest.nodes[0].clusterName }_access.log" + "${ -> integTest.nodes[0].homeDir}/logs/${ -> integTest.nodes[0].clusterName }_audit.log" } runqa { diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/RestSqlSecurityIT.java b/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/RestSqlSecurityIT.java index 28644d3509d80..3c89d87d38d05 100644 --- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/RestSqlSecurityIT.java +++ b/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/RestSqlSecurityIT.java @@ -252,14 +252,14 @@ public AuditLogAsserter expect(String eventType, String action, String principal final Matcher runByRealmMatcher = realm.equals("default_file") ? Matchers.nullValue(String.class) : Matchers.is("default_file"); logCheckers.add( - m -> eventType.equals(m.get("event_type")) + m -> eventType.equals(m.get("event.action")) && action.equals(m.get("action")) - && principal.equals(m.get("principal")) - && realm.equals(m.get("realm")) - && runByPrincipalMatcher.matches(m.get("run_by_principal")) - && runByRealmMatcher.matches(m.get("run_by_realm")) + && principal.equals(m.get("user.name")) + && realm.equals(m.get("user.realm")) + && runByPrincipalMatcher.matches(m.get("user.run_by.name")) + && runByRealmMatcher.matches(m.get("user.run_by.realm")) && indicesMatcher.matches(m.get("indices")) - && request.equals(m.get("request"))); + && request.equals(m.get("request.name"))); return this; } diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/SqlSecurityTestCase.java b/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/SqlSecurityTestCase.java index 83047c93da327..1bea73b56a2c7 100644 --- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/SqlSecurityTestCase.java +++ b/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/SqlSecurityTestCase.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.qa.sql.security; import org.apache.lucene.util.SuppressForbidden; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.SpecialPermission; import org.elasticsearch.action.admin.indices.get.GetIndexAction; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; @@ -14,6 +15,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.rest.ESRestTestCase; import org.hamcrest.Matcher; @@ -32,17 +34,16 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.function.Function; -import java.util.regex.Pattern; import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.hasItems; public abstract class SqlSecurityTestCase extends ESRestTestCase { @@ -515,21 +516,22 @@ public AuditLogAsserter expect(boolean granted, String action, String principal, default: throw new IllegalArgumentException("Unknown action [" + action + "]"); } - final String eventType = granted ? "access_granted" : "access_denied"; + final String eventAction = granted ? "access_granted" : "access_denied"; final String realm = principal.equals("test_admin") ? "default_file" : "default_native"; - return expect(eventType, action, principal, realm, indicesMatcher, request); + return expect(eventAction, action, principal, realm, indicesMatcher, request); } - public AuditLogAsserter expect(String eventType, String action, String principal, String realm, + public AuditLogAsserter expect(String eventAction, String action, String principal, String realm, Matcher> indicesMatcher, String request) { - logCheckers.add(m -> eventType.equals(m.get("event_type")) + logCheckers.add(m -> + eventAction.equals(m.get("event.action")) && action.equals(m.get("action")) - && principal.equals(m.get("principal")) - && realm.equals(m.get("realm")) - && Matchers.nullValue(String.class).matches(m.get("run_by_principal")) - && Matchers.nullValue(String.class).matches(m.get("run_by_realm")) + && principal.equals(m.get("user.name")) + && realm.equals(m.get("user.realm")) + && Matchers.nullValue(String.class).matches(m.get("user.run_by.name")) + && Matchers.nullValue(String.class).matches(m.get("user.run_by.realm")) && indicesMatcher.matches(m.get("indices")) - && request.equals(m.get("request")) + && request.equals(m.get("request.name")) ); return this; } @@ -554,56 +556,39 @@ public void assertLogs() throws Exception { List> logs = new ArrayList<>(); String line; - Pattern logPattern = Pattern.compile( - ("PART PART PART PART origin_type=PART, origin_address=PART, principal=PART, realm=PART, " - + "(?:run_as_principal=IGN, )?(?:run_as_realm=IGN, )?(?:run_by_principal=PART, )?(?:run_by_realm=PART, )?" - + "roles=PART, action=\\[(.*?)\\], (?:indices=PART, )?request=PART") - .replace(" ", "\\s+").replace("PART", "\\[([^\\]]*)\\]").replace("IGN", "\\[[^\\]]*\\]")); - // fail(logPattern.toString()); while ((line = logReader.readLine()) != null) { - java.util.regex.Matcher m = logPattern.matcher(line); - if (false == m.matches()) { - throw new IllegalArgumentException("Unrecognized log: " + line); - } - int i = 1; - Map log = new HashMap<>(); - /* We *could* parse the date but leaving it in the original format makes it - * easier to find the lines in the file that this log comes from. */ - log.put("time", m.group(i++)); - log.put("node", m.group(i++)); - log.put("origin", m.group(i++)); - String eventType = m.group(i++); - if (false == ("access_denied".equals(eventType) || "access_granted".equals(eventType))) { - continue; - } - log.put("event_type", eventType); - log.put("origin_type", m.group(i++)); - log.put("origin_address", m.group(i++)); - String principal = m.group(i++); - log.put("principal", principal); - log.put("realm", m.group(i++)); - log.put("run_by_principal", m.group(i++)); - log.put("run_by_realm", m.group(i++)); - log.put("roles", m.group(i++)); - String action = m.group(i++); - if (false == (SQL_ACTION_NAME.equals(action) || GetIndexAction.NAME.equals(action))) { - //TODO we may want to extend this and the assertions to SearchAction.NAME as well - continue; - } - log.put("action", action); - // Use a sorted list for indices for consistent error reporting - List indices = new ArrayList<>(Strings.tokenizeByCommaToSet(m.group(i++))); - Collections.sort(indices); - if ("test_admin".equals(principal)) { - /* Sometimes we accidentally sneak access to the security tables. This is fine, SQL - * drops them from the interface. So we might have access to them, but we don't show - * them. */ - indices.remove(".security"); - indices.remove(".security-6"); + try { + final Map log = XContentHelper.convertToMap(JsonXContent.jsonXContent, line, false); + if (false == ("access_denied".equals(log.get("event.action")) + || "access_granted".equals(log.get("event.action")))) { + continue; + } + assertThat(log.containsKey("action"), is(true)); + if (false == (SQL_ACTION_NAME.equals(log.get("action")) || GetIndexAction.NAME.equals(log.get("action")))) { + // TODO we may want to extend this and the assertions to SearchAction.NAME as well + continue; + } + assertThat(log.containsKey("user.name"), is(true)); + List indices = new ArrayList<>(); + if (log.containsKey("indices")) { + indices = (ArrayList) log.get("indices"); + if ("test_admin".equals(log.get("user.name"))) { + /* + * Sometimes we accidentally sneak access to the security tables. This is fine, + * SQL drops them from the interface. So we might have access to them, but we + * don't show them. + */ + indices.remove(".security"); + indices.remove(".security-6"); + } + } + // Use a sorted list for indices for consistent error reporting + Collections.sort(indices); + log.put("indices", indices); + logs.add(log); + } catch (final ElasticsearchParseException e) { + throw new IllegalArgumentException("Unrecognized log: " + line, e); } - log.put("indices", indices); - log.put("request", m.group(i)); - logs.add(log); } List> allLogs = new ArrayList<>(logs); List notMatching = new ArrayList<>(); From 1ca9d19c1e7ebd8b25ae1f535c299a429336bdd9 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 20 Sep 2018 13:53:02 +0300 Subject: [PATCH 02/10] Resurrect the previous format alongside the new one --- .../core/src/main/config/log4j2.properties | 38 +++ .../audit/logfile/CapturingLogger.java | 76 ++++-- .../audit/logfile/LoggingAuditTrail.java | 14 +- .../audit/logfile/LoggingAuditTrailTests.java | 218 ++++++++++++++---- 4 files changed, 290 insertions(+), 56 deletions(-) diff --git a/x-pack/plugin/core/src/main/config/log4j2.properties b/x-pack/plugin/core/src/main/config/log4j2.properties index 1c9358f3cc490..afb1e9c02b032 100644 --- a/x-pack/plugin/core/src/main/config/log4j2.properties +++ b/x-pack/plugin/core/src/main/config/log4j2.properties @@ -1,3 +1,40 @@ +appender.deprecated_audit_rolling.type = RollingFile +appender.deprecated_audit_rolling.name = deprecated_audit_rolling +appender.deprecated_audit_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_access.log +appender.deprecated_audit_rolling.layout.type = PatternLayout +appender.deprecated_audit_rolling.layout.pattern = \ + [%d{ISO8601}]\ + %varsNotEmpty{ [%map{host.ip}]}\ + %varsNotEmpty{ [%map{host.name}]}\ + %varsNotEmpty{ [%map{node.name}]}\ + \ [%map{event.type}]\ + \ [%map{event.action}]\ + \ \ \ + %varsNotEmpty{realm=[%map{realm}], }\ + origin_type=[%map{origin.type}]\ + %varsNotEmpty{, origin_address=[%map{origin.address}]}\ + %varsNotEmpty{, principal=[%map{user.name}]}\ + %varsNotEmpty{, realm=[%map{user.realm}]}\ + %varsNotEmpty{, run_by_principal=[%map{user.run_by.name}]}\ + %varsNotEmpty{, run_as_principal=[%map{user.run_as.name}]}\ + %varsNotEmpty{, run_by_realm=[%map{user.run_by.realm}]}\ + %varsNotEmpty{, run_as_realm=[%map{user.run_as.realm}]}\ + %varsNotEmpty{, roles=[%map{deprecated.user.roles}]}\ + %varsNotEmpty{, action=[%map{action}]}\ + %varsNotEmpty{, indices=[%map{deprecated.indices}]}\ + %varsNotEmpty{, request=[%map{request.name}]}\ + %varsNotEmpty{, transport_profile=[%map{transport.profile}]}\ + %varsNotEmpty{, rule=[%map{rule}]}\ + %varsNotEmpty{, uri=[%map{deprecated.uri}]}\ + %varsNotEmpty{, opaque_id=[%map{opaque_id}]}\ + %varsNotEmpty{, request_body=[%map{request.body}]}\ + %n +appender.deprecated_audit_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_access-%d{yyyy-MM-dd}.log +appender.deprecated_audit_rolling.policies.type = Policies +appender.deprecated_audit_rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.deprecated_audit_rolling.policies.time.interval = 1 +appender.deprecated_audit_rolling.policies.time.modulate = true + appender.audit_rolling.type = RollingFile appender.audit_rolling.name = audit_rolling appender.audit_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_audit.log @@ -67,6 +104,7 @@ appender.audit_rolling.policies.time.modulate = true logger.xpack_security_audit_logfile.name = org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail logger.xpack_security_audit_logfile.level = info logger.xpack_security_audit_logfile.appenderRef.audit_rolling.ref = audit_rolling +logger.xpack_security_audit_logfile.appenderRef.deprecated_audit_rolling.ref = deprecated_audit_rolling logger.xpack_security_audit_logfile.additivity = false logger.xmlsig.name = org.apache.xml.security.signature.XMLSignature diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/audit/logfile/CapturingLogger.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/audit/logfile/CapturingLogger.java index ede18c8241b34..31881e7586e5a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/audit/logfile/CapturingLogger.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/audit/logfile/CapturingLogger.java @@ -30,6 +30,8 @@ */ public class CapturingLogger { + private static final String IMPLICIT_APPENDER_NAME = "__implicit"; + /** * Constructs a new {@link CapturingLogger} named as the fully qualified name of * the invoking method. One name can be assigned to a single logger globally, so @@ -52,42 +54,88 @@ public static Logger newCapturingLogger(final Level level, @Nullable StringLayou final String name = caller.getClassName() + "." + caller.getMethodName() + "." + level.toString(); final Logger logger = ESLoggerFactory.getLogger(name); Loggers.setLevel(logger, level); - final MockAppender appender = new MockAppender(name, layout); + attachNewMockAppender(logger, IMPLICIT_APPENDER_NAME, layout); + return logger; + } + + public static void attachNewMockAppender(final Logger logger, final String appenderName, @Nullable StringLayout layout) + throws IllegalAccessException { + final MockAppender appender = new MockAppender(buildAppenderName(logger.getName(), appenderName), layout); appender.start(); Loggers.addAppender(logger, appender); - return logger; } - private static MockAppender getMockAppender(final String name) { + private static String buildAppenderName(final String loggerName, final String appenderName) { + // appender name also has to be unique globally (logging context globally) + return loggerName + "." + appenderName; + } + + private static MockAppender getMockAppender(final String loggerName, final String appenderName) { final LoggerContext ctx = (LoggerContext) LogManager.getContext(false); final Configuration config = ctx.getConfiguration(); - final LoggerConfig loggerConfig = config.getLoggerConfig(name); - return (MockAppender) loggerConfig.getAppenders().get(name); + final LoggerConfig loggerConfig = config.getLoggerConfig(loggerName); + final String mockAppenderName = buildAppenderName(loggerName, appenderName); + return (MockAppender) loggerConfig.getAppenders().get(mockAppenderName); } /** - * Checks if the logger's appender has captured any events. + * Checks if the logger's appender(s) has captured any events. * - * @param name + * @param loggerName * The unique global name of the logger. + * @param appenderNames + * Names of other appenders nested under this same logger. * @return {@code true} if no event has been captured, {@code false} otherwise. */ - public static boolean isEmpty(final String name) { - final MockAppender appender = getMockAppender(name); - return appender.isEmpty(); + public static boolean isEmpty(final String loggerName, final String... appenderNames) { + // check if implicit appender is empty + final MockAppender implicitAppender = getMockAppender(loggerName, IMPLICIT_APPENDER_NAME); + assert implicitAppender != null; + if (false == implicitAppender.isEmpty()) { + return false; + } + if (null == appenderNames) { + return true; + } + // check if any named appenders are empty + for (String appenderName : appenderNames) { + final MockAppender namedAppender = getMockAppender(loggerName, appenderName); + if (namedAppender != null && false == namedAppender.isEmpty()) { + return false; + } + } + return true; + } + + /** + * Gets the captured events for a logger by its name. Events are those of the + * implicit appender of the logger. + * + * @param loggerName + * The unique global name of the logger. + * @param level + * The priority level of the captured events to be returned. + * @return A list of captured events formated to {@code String}. + */ + public static List output(final String loggerName, final Level level) { + return output(loggerName, IMPLICIT_APPENDER_NAME, level); } /** - * Gets the captured events for a logger by its name. + * Gets the captured events for a logger and an appender by their respective + * names. There is a one to many relationship between loggers and appenders. * - * @param name + * @param loggerName * The unique global name of the logger. + * @param appenderName + * The name of an appender associated with the {@code loggerName} + * logger. * @param level * The priority level of the captured events to be returned. * @return A list of captured events formated to {@code String}. */ - public static List output(final String name, final Level level) { - final MockAppender appender = getMockAppender(name); + public static List output(final String loggerName, final String appenderName, final Level level) { + final MockAppender appender = getMockAppender(loggerName, appenderName); return appender.output(level); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index 359fe7cd464a4..5dfc54618b003 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -611,6 +611,8 @@ LogEntryBuilder withRestUri(RestRequest request) { if (queryStringIndex > -1) { logEntry.with(URL_QUERY_FIELD_NAME, request.uri().substring(queryStringIndex + 1, queryStringLength)); } + // deprecated uri format + logEntry.with("deprecated.uri", request.uri()); return this; } @@ -626,10 +628,16 @@ LogEntryBuilder withRunAsSubject(Authentication authentication) { LogEntryBuilder withRestOrigin(RestRequest request) { assert LOCAL_ORIGIN_FIELD_VALUE.equals(logEntry.get(ORIGIN_TYPE_FIELD_NAME)); // this is the default - final InetSocketAddress socketAddress = request.getHttpChannel().getRemoteAddress(); + final String formattedAddress; + final SocketAddress socketAddress = request.getRemoteAddress(); + if (socketAddress instanceof InetSocketAddress) { + formattedAddress = NetworkAddress.format(((InetSocketAddress) socketAddress)); + } else { + formattedAddress = socketAddress.toString(); + } if (socketAddress != null) { logEntry.with(ORIGIN_TYPE_FIELD_NAME, REST_ORIGIN_FIELD_VALUE) - .with(ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(socketAddress)); + .with(ORIGIN_ADDRESS_FIELD_NAME, formattedAddress); } // fall through to local_node default return this; @@ -700,6 +708,8 @@ LogEntryBuilder with(String key, String value) { LogEntryBuilder with(String key, String[] values) { if (values != null) { logEntry.with(key, toQuotedJsonArray(values)); + // deprecated format required for bwc + logEntry.with("deprecated." + key, Strings.arrayToCommaDelimitedString(values)); } return this; } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java index 6c302c9311f3d..24958f250a598 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.mock.orig.Mockito; +import org.elasticsearch.node.Node; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; @@ -128,6 +129,7 @@ protected String expectedMessage() { } private static PatternLayout patternLayout; + private static PatternLayout deprecatedPatternLayout; private Settings settings; private DiscoveryNode localNode; private ClusterService clusterService; @@ -136,6 +138,13 @@ protected String expectedMessage() { private Map commonFields; private Logger logger; private LoggingAuditTrail auditTrail; + // from: + // https://www.myintervals.com/blog/2009/05/20/iso-8601-date-validation-that-doesnt-suck/ + // but stripped off the leading ^ and trailing $ anchors + private static final String ISO8601_DATE_REGEXP = "([\\+-]?\\d{4}(?!\\d{2}\\b))((-?)((0[1-9]|1[0-2])(\\3([12]\\d|0[1-9]|3[01]))?|" + + "W([0-4]\\d|5[0-2])(-?[1-7])?|(00[1-9]|0[1-9]\\d|[12]\\d{2}|3([0-5]\\d|6[1-6])))([T\\s]((([01]\\d|2[0-3])((:?)" + + "[0-5]\\d)?|24\\:?00)([\\.,]\\d+(?!:))?)?(\\17[0-5]\\d([\\.,]\\d+)?)?([zZ]|([\\+-])([01]\\d|2[0-3]):?([0-5]\\d)?)?)?)?"; + private static final String DEPRECATED_APPENDER_NAME = "deprecated"; @BeforeClass public static void lookupPatternLayout() throws Exception { @@ -150,16 +159,25 @@ public static void lookupPatternLayout() throws Exception { // attached to the LoggingAuditTrail.class logger. assertThat(properties.getProperty("logger.xpack_security_audit_logfile.name"), is(LoggingAuditTrail.class.getName())); assertThat(properties.getProperty("logger.xpack_security_audit_logfile.appenderRef.audit_rolling.ref"), is("audit_rolling")); + assertThat(properties.getProperty("logger.xpack_security_audit_logfile.appenderRef.deprecated_audit_rolling.ref"), + is("deprecated_audit_rolling")); assertThat(properties.getProperty("appender.audit_rolling.name"), is("audit_rolling")); + assertThat(properties.getProperty("appender.deprecated_audit_rolling.name"), is("deprecated_audit_rolling")); assertThat(properties.getProperty("appender.audit_rolling.layout.type"), is("PatternLayout")); + assertThat(properties.getProperty("appender.deprecated_audit_rolling.layout.type"), is("PatternLayout")); final String patternLayoutFormat = properties.getProperty("appender.audit_rolling.layout.pattern"); assertThat(patternLayoutFormat, is(notNullValue())); patternLayout = PatternLayout.newBuilder().withPattern(patternLayoutFormat).withCharset(StandardCharsets.UTF_8).build(); + final String deprecatedPatternLayoutFormat = properties.getProperty("appender.deprecated_audit_rolling.layout.pattern"); + assertThat(deprecatedPatternLayoutFormat, is(notNullValue())); + deprecatedPatternLayout = PatternLayout.newBuilder().withPattern(deprecatedPatternLayoutFormat).withCharset(StandardCharsets.UTF_8) + .build(); } @AfterClass public static void releasePatternLayout() { patternLayout = null; + deprecatedPatternLayout = null; } @Before @@ -169,7 +187,9 @@ public void init() throws Exception { .put("xpack.security.audit.logfile.prefix.emit_node_host_address", randomBoolean()) .put("xpack.security.audit.logfile.prefix.emit_node_host_name", randomBoolean()) .put("xpack.security.audit.logfile.prefix.emit_node_name", randomBoolean()) + .put("xpack.security.audit.logfile.prefix.emit_node_id", randomBoolean()) .put("xpack.security.audit.logfile.events.emit_request_body", includeRequestBody) + .put(Node.NODE_NAME_SETTING.getKey(), randomAlphaOfLength(6)) .build(); localNode = mock(DiscoveryNode.class); when(localNode.getAddress()).thenReturn(buildNewFakeTransportAddress()); @@ -188,12 +208,15 @@ public void init() throws Exception { threadContext.putHeader(Task.X_OPAQUE_ID, randomAlphaOfLengthBetween(1, 4)); } logger = CapturingLogger.newCapturingLogger(Level.INFO, patternLayout); + CapturingLogger.attachNewMockAppender(logger, DEPRECATED_APPENDER_NAME, deprecatedPatternLayout); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); } @After public void clearLog() throws Exception { + // clear both the implicit as well as the deprecated appender CapturingLogger.output(logger.getName(), Level.INFO).clear(); + CapturingLogger.output(logger.getName(), DEPRECATED_APPENDER_NAME, Level.INFO).clear(); } public void testAnonymousAccessDeniedTransport() throws Exception { @@ -209,16 +232,17 @@ public void testAnonymousAccessDeniedTransport() throws Exception { restOrTransportOrigin(message, threadContext, checkedFields); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); + assertDeprecatedMsg(logger, settings, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled - CapturingLogger.output(logger.getName(), Level.INFO).clear(); + clearLog(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "anonymous_access_denied") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.anonymousAccessDenied("_action", message); - assertEmptyLog(logger); + assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); } public void testAnonymousAccessDeniedRest() throws Exception { @@ -240,16 +264,17 @@ public void testAnonymousAccessDeniedRest() throws Exception { .put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, null); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap()); + assertDeprecatedMsg(logger, settings, checkedFields.immutableMap()); // test disabled - CapturingLogger.output(logger.getName(), Level.INFO).clear(); + clearLog(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "anonymous_access_denied") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.anonymousAccessDenied(request); - assertEmptyLog(logger); + assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); } public void testAuthenticationFailed() throws Exception { @@ -268,16 +293,17 @@ public void testAuthenticationFailed() throws Exception { indicesRequest(message, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); + assertDeprecatedMsg(logger, settings, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled - CapturingLogger.output(logger.getName(), Level.INFO).clear(); + clearLog(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "authentication_failed") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.authenticationFailed(new MockToken(), "_action", message); - assertEmptyLog(logger); + assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); } public void testAuthenticationFailedNoToken() throws Exception { @@ -294,16 +320,17 @@ public void testAuthenticationFailedNoToken() throws Exception { indicesRequest(message, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); + assertDeprecatedMsg(logger, settings, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled - CapturingLogger.output(logger.getName(), Level.INFO).clear(); + clearLog(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "authentication_failed") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.authenticationFailed("_action", message); - assertEmptyLog(logger); + assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); } public void testAuthenticationFailedRest() throws Exception { @@ -332,16 +359,17 @@ public void testAuthenticationFailedRest() throws Exception { .put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, params.isEmpty() ? null : "foo=bar"); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap()); + assertDeprecatedMsg(logger, settings, checkedFields.immutableMap()); // test disabled - CapturingLogger.output(logger.getName(), Level.INFO).clear(); + clearLog(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "authentication_failed") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.authenticationFailed(new MockToken(), request); - assertEmptyLog(logger); + assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); } public void testAuthenticationFailedRestNoToken() throws Exception { @@ -369,16 +397,17 @@ public void testAuthenticationFailedRestNoToken() throws Exception { .put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, params.isEmpty() ? null : "bar=baz"); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap()); + assertDeprecatedMsg(logger, settings, checkedFields.immutableMap()); // test disabled - CapturingLogger.output(logger.getName(), Level.INFO).clear(); + clearLog(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "authentication_failed") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.authenticationFailed(request); - assertEmptyLog(logger); + assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); } public void testAuthenticationFailedRealm() throws Exception { @@ -386,7 +415,7 @@ public void testAuthenticationFailedRealm() throws Exception { final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); final String realm = randomAlphaOfLengthBetween(1, 6); auditTrail.authenticationFailed(realm, mockToken, "_action", message); - assertEmptyLog(logger); + assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); // test enabled settings = Settings.builder() @@ -407,6 +436,7 @@ public void testAuthenticationFailedRealm() throws Exception { indicesRequest(message, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); + assertDeprecatedMsg(logger, settings, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); } public void testAuthenticationFailedRealmRest() throws Exception { @@ -422,7 +452,7 @@ public void testAuthenticationFailedRealmRest() throws Exception { final AuthenticationToken mockToken = new MockToken(); final String realm = randomAlphaOfLengthBetween(1, 6); auditTrail.authenticationFailed(realm, mockToken, request); - assertEmptyLog(logger); + assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); // test enabled settings = Settings.builder() @@ -445,6 +475,7 @@ public void testAuthenticationFailedRealmRest() throws Exception { .put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, params.isEmpty() ? null : "_param=baz"); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap()); + assertDeprecatedMsg(logger, settings, checkedFields.immutableMap()); } public void testAccessGranted() throws Exception { @@ -456,25 +487,26 @@ public void testAccessGranted() throws Exception { final MapBuilder checkedFields = new MapBuilder<>(commonFields); final MapBuilder checkedArrayFields = new MapBuilder<>(); checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) - .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "access_granted") - .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action") - .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()); + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "access_granted") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action") + .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()); checkedArrayFields.put(LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME, roles); subject(authentication, checkedFields); restOrTransportOrigin(message, threadContext, checkedFields); indicesRequest(message, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); + assertDeprecatedMsg(logger, settings, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled - CapturingLogger.output(logger.getName(), Level.INFO).clear(); + clearLog(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "access_granted") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.accessGranted(authentication, "_action", message, roles); - assertEmptyLog(logger); + assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); } public void testAccessGrantedInternalSystemAction() throws Exception { @@ -482,7 +514,7 @@ public void testAccessGrantedInternalSystemAction() throws Exception { final String[] roles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4)); final Authentication authentication = new Authentication(SystemUser.INSTANCE, new RealmRef("_reserved", "test", "foo"), null); auditTrail.accessGranted(authentication, "internal:_action", message, roles); - assertEmptyLog(logger); + assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); // test enabled settings = Settings.builder() @@ -504,6 +536,7 @@ public void testAccessGrantedInternalSystemAction() throws Exception { indicesRequest(message, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); + assertDeprecatedMsg(logger, settings, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); } public void testAccessGrantedInternalSystemActionNonSystemUser() throws Exception { @@ -524,16 +557,17 @@ public void testAccessGrantedInternalSystemActionNonSystemUser() throws Exceptio indicesRequest(message, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); + assertDeprecatedMsg(logger, settings, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled - CapturingLogger.output(logger.getName(), Level.INFO).clear(); + clearLog(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "access_granted") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.accessGranted(authentication, "internal:_action", message, roles); - assertEmptyLog(logger); + assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); } public void testAccessDenied() throws Exception { @@ -554,16 +588,17 @@ public void testAccessDenied() throws Exception { indicesRequest(message, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); + assertDeprecatedMsg(logger, settings, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled - CapturingLogger.output(logger.getName(), Level.INFO).clear(); + clearLog(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "access_denied") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.accessDenied(authentication, "_action", message, roles); - assertEmptyLog(logger); + assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); } public void testTamperedRequestRest() throws Exception { @@ -588,16 +623,17 @@ public void testTamperedRequestRest() throws Exception { .put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, params.isEmpty() ? null : "_param=baz"); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap()); + assertDeprecatedMsg(logger, settings, checkedFields.immutableMap()); // test disabled - CapturingLogger.output(logger.getName(), Level.INFO).clear(); + clearLog(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "tampered_request") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.tamperedRequest(request); - assertEmptyLog(logger); + assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); } public void testTamperedRequest() throws Exception { @@ -614,16 +650,17 @@ public void testTamperedRequest() throws Exception { indicesRequest(message, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); + assertDeprecatedMsg(logger, settings, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled - CapturingLogger.output(logger.getName(), Level.INFO).clear(); + clearLog(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "tampered_request") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.tamperedRequest("_action", message); - assertEmptyLog(logger); + assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); } public void testTamperedRequestWithUser() throws Exception { @@ -653,16 +690,17 @@ public void testTamperedRequestWithUser() throws Exception { indicesRequest(message, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); + assertDeprecatedMsg(logger, settings, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled - CapturingLogger.output(logger.getName(), Level.INFO).clear(); + clearLog(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "tampered_request") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.tamperedRequest(user, "_action", message); - assertEmptyLog(logger); + assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); } public void testConnectionDenied() throws Exception { @@ -680,16 +718,17 @@ public void testConnectionDenied() throws Exception { .put(LoggingAuditTrail.RULE_FIELD_NAME, "deny _all"); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap()); + assertDeprecatedMsg(logger, settings, checkedFields.immutableMap()); // test disabled - CapturingLogger.output(logger.getName(), Level.INFO).clear(); + clearLog(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "connection_denied") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.connectionDenied(inetAddress, profile, rule); - assertEmptyLog(logger); + assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); } public void testConnectionGranted() throws Exception { @@ -698,7 +737,7 @@ public void testConnectionGranted() throws Exception { final String profile = randomAlphaOfLengthBetween(1, 6); auditTrail.connectionGranted(inetAddress, profile, rule); - assertEmptyLog(logger); + assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); // test enabled settings = Settings.builder() @@ -723,8 +762,7 @@ public void testRunAsGranted() throws Exception { final String[] roles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4)); final Authentication authentication = new Authentication( new User("running as", new String[] { "r2" }, new User("_username", new String[] { "r1" })), - new RealmRef("authRealm", "test", "foo"), - new RealmRef("lookRealm", "up", "by")); + new RealmRef("authRealm", "test", "foo"), new RealmRef("lookRealm", "up", "by")); auditTrail.runAsGranted(authentication, "_action", message, roles); final MapBuilder checkedFields = new MapBuilder<>(commonFields); @@ -759,8 +797,7 @@ public void testRunAsDenied() throws Exception { final String[] roles = randomArray(0, 4, String[]::new, () -> randomAlphaOfLengthBetween(1, 4)); final Authentication authentication = new Authentication( new User("running as", new String[] { "r2" }, new User("_username", new String[] { "r1" })), - new RealmRef("authRealm", "test", "foo"), - new RealmRef("lookRealm", "up", "by")); + new RealmRef("authRealm", "test", "foo"), new RealmRef("lookRealm", "up", "by")); auditTrail.runAsDenied(authentication, "_action", message, roles); final MapBuilder checkedFields = new MapBuilder<>(commonFields); @@ -934,6 +971,107 @@ private void assertMsg(Logger logger, Map checkFields) { assertMsg(logger, checkFields, Collections.emptyMap()); } + private void assertDeprecatedMsg(Logger logger, Settings settings, Map checkFields) { + assertDeprecatedMsg(logger, settings, checkFields, Collections.emptyMap()); + } + + private void assertDeprecatedMsg(Logger logger, Settings settings, Map checkFields, + Map checkArrayFields) { + final List output = CapturingLogger.output(logger.getName(), DEPRECATED_APPENDER_NAME, Level.INFO); + assertThat("Exactly one log line expected. Found: " + output.size(), output.size(), is(1)); + if (checkFields == null) { + // only check msg existence + return; + } + final String logLine = output.get(0); + final StringBuilder logLineRegexpBuilder = new StringBuilder(); + if (LoggingAuditTrail.EMIT_HOST_ADDRESS_SETTING.get(settings)) { + logLineRegexpBuilder.append(" [").append(checkFields.get(LoggingAuditTrail.HOST_ADDRESS_FIELD_NAME)).append("]"); + } + if (LoggingAuditTrail.EMIT_HOST_NAME_SETTING.get(settings)) { + logLineRegexpBuilder.append(" [").append(checkFields.get(LoggingAuditTrail.HOST_NAME_FIELD_NAME)).append("]"); + } + if (LoggingAuditTrail.EMIT_NODE_NAME_SETTING.get(settings)) { + logLineRegexpBuilder.append(" [").append(checkFields.get(LoggingAuditTrail.NODE_NAME_FIELD_NAME)).append("]"); + } + logLineRegexpBuilder.append(" [").append(checkFields.get(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME)).append("]"); + logLineRegexpBuilder.append(" [").append(checkFields.get(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME)).append("]"); + logLineRegexpBuilder.append(" "); + if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.REALM_FIELD_NAME))) { + logLineRegexpBuilder.append("realm=[").append(checkFields.get(LoggingAuditTrail.REALM_FIELD_NAME)).append("], "); + } + logLineRegexpBuilder.append("origin_type=[").append(checkFields.get(LoggingAuditTrail.ORIGIN_TYPE_FIELD_NAME)).append("]"); + if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.ORIGIN_ADDRESS_FIELD_NAME))) { + logLineRegexpBuilder.append(", origin_address=[").append(checkFields.get(LoggingAuditTrail.ORIGIN_ADDRESS_FIELD_NAME)) + .append("]"); + } + if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.PRINCIPAL_FIELD_NAME))) { + logLineRegexpBuilder.append(", principal=[").append(checkFields.get(LoggingAuditTrail.PRINCIPAL_FIELD_NAME)).append("]"); + } + if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.PRINCIPAL_REALM_FIELD_NAME))) { + logLineRegexpBuilder.append(", realm=[").append(checkFields.get(LoggingAuditTrail.PRINCIPAL_REALM_FIELD_NAME)).append("]"); + } + if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.PRINCIPAL_RUN_BY_FIELD_NAME))) { + logLineRegexpBuilder.append(", run_by_principal=[").append(checkFields.get(LoggingAuditTrail.PRINCIPAL_RUN_BY_FIELD_NAME)) + .append("]"); + } + if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.PRINCIPAL_RUN_AS_FIELD_NAME))) { + logLineRegexpBuilder.append(", run_as_principal=[").append(checkFields.get(LoggingAuditTrail.PRINCIPAL_RUN_AS_FIELD_NAME)) + .append("]"); + } + if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.PRINCIPAL_RUN_BY_REALM_FIELD_NAME))) { + logLineRegexpBuilder.append(", run_by_realm=[").append(checkFields.get(LoggingAuditTrail.PRINCIPAL_RUN_BY_REALM_FIELD_NAME)) + .append("]"); + } + if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.PRINCIPAL_RUN_AS_REALM_FIELD_NAME))) { + logLineRegexpBuilder.append(", run_as_realm=[").append(checkFields.get(LoggingAuditTrail.PRINCIPAL_RUN_AS_REALM_FIELD_NAME)) + .append("]"); + } + final String[] roles = checkArrayFields.get(LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME); + if (null != roles && roles.length > 0) { + logLineRegexpBuilder.append(", roles=[") + .append(Strings.arrayToCommaDelimitedString(checkArrayFields.get(LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME))) + .append("]"); + } + if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.ACTION_FIELD_NAME))) { + logLineRegexpBuilder.append(", action=[").append(checkFields.get(LoggingAuditTrail.ACTION_FIELD_NAME)).append("]"); + } + final String[] indices = checkArrayFields.get(LoggingAuditTrail.INDICES_FIELD_NAME); + if (null != indices && indices.length > 0) { + logLineRegexpBuilder.append(", indices=[") + .append(Strings.arrayToCommaDelimitedString(checkArrayFields.get(LoggingAuditTrail.INDICES_FIELD_NAME))).append("]"); + } + if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME))) { + logLineRegexpBuilder.append(", request=[").append(checkFields.get(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME)).append("]"); + } + if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.TRANSPORT_PROFILE_FIELD_NAME))) { + logLineRegexpBuilder.append(", transport_profile=[").append(checkFields.get(LoggingAuditTrail.TRANSPORT_PROFILE_FIELD_NAME)) + .append("]"); + } + if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.RULE_FIELD_NAME))) { + logLineRegexpBuilder.append(", rule=[").append(checkFields.get(LoggingAuditTrail.RULE_FIELD_NAME)).append("]"); + } + if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.URL_PATH_FIELD_NAME))) { + String uri = checkFields.get(LoggingAuditTrail.URL_PATH_FIELD_NAME); + if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.URL_QUERY_FIELD_NAME))) { + uri += "?" + checkFields.get(LoggingAuditTrail.URL_QUERY_FIELD_NAME); + } + logLineRegexpBuilder.append(", uri=[").append(uri).append("]"); + } + if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.OPAQUE_ID_FIELD_NAME))) { + logLineRegexpBuilder.append(", opaque_id=[").append(checkFields.get(LoggingAuditTrail.OPAQUE_ID_FIELD_NAME)).append("]"); + } + if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.REQUEST_BODY_FIELD_NAME))) { + logLineRegexpBuilder.append(", request_body=[").append(checkFields.get(LoggingAuditTrail.REQUEST_BODY_FIELD_NAME)).append("]"); + } + logLineRegexpBuilder.append(System.lineSeparator()); + + // regex for the date field + final String logLineRegexp = "^\\[" + ISO8601_DATE_REGEXP + "\\]" + Pattern.quote(logLineRegexpBuilder.toString()) + "$"; + assertThat("The regexp does not match the log line: |" + logLineRegexp + "| |" + logLine + "|", + Pattern.matches(logLineRegexp, logLine), is(true)); + } + private void assertMsg(Logger logger, Map checkFields, Map checkArrayFields) { final List output = CapturingLogger.output(logger.getName(), Level.INFO); assertThat("Exactly one logEntry expected. Found: " + output.size(), output.size(), is(1)); @@ -976,13 +1114,13 @@ private void assertMsg(Logger logger, Map checkFields, Map prepareRestContent(String uri, InetSocketAddress remoteAddress) { From 259e3cc280fae9df98a9c71db36bb3b20ad25733 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 5 Oct 2018 08:47:31 +0300 Subject: [PATCH 03/10] Revert "Resurrect the previous format alongside the new one" This reverts commit 1ca9d19c1e7ebd8b25ae1f535c299a429336bdd9. --- .../core/src/main/config/log4j2.properties | 38 --- .../audit/logfile/CapturingLogger.java | 76 ++---- .../audit/logfile/LoggingAuditTrail.java | 14 +- .../audit/logfile/LoggingAuditTrailTests.java | 218 ++++-------------- 4 files changed, 56 insertions(+), 290 deletions(-) diff --git a/x-pack/plugin/core/src/main/config/log4j2.properties b/x-pack/plugin/core/src/main/config/log4j2.properties index afb1e9c02b032..1c9358f3cc490 100644 --- a/x-pack/plugin/core/src/main/config/log4j2.properties +++ b/x-pack/plugin/core/src/main/config/log4j2.properties @@ -1,40 +1,3 @@ -appender.deprecated_audit_rolling.type = RollingFile -appender.deprecated_audit_rolling.name = deprecated_audit_rolling -appender.deprecated_audit_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_access.log -appender.deprecated_audit_rolling.layout.type = PatternLayout -appender.deprecated_audit_rolling.layout.pattern = \ - [%d{ISO8601}]\ - %varsNotEmpty{ [%map{host.ip}]}\ - %varsNotEmpty{ [%map{host.name}]}\ - %varsNotEmpty{ [%map{node.name}]}\ - \ [%map{event.type}]\ - \ [%map{event.action}]\ - \ \ \ - %varsNotEmpty{realm=[%map{realm}], }\ - origin_type=[%map{origin.type}]\ - %varsNotEmpty{, origin_address=[%map{origin.address}]}\ - %varsNotEmpty{, principal=[%map{user.name}]}\ - %varsNotEmpty{, realm=[%map{user.realm}]}\ - %varsNotEmpty{, run_by_principal=[%map{user.run_by.name}]}\ - %varsNotEmpty{, run_as_principal=[%map{user.run_as.name}]}\ - %varsNotEmpty{, run_by_realm=[%map{user.run_by.realm}]}\ - %varsNotEmpty{, run_as_realm=[%map{user.run_as.realm}]}\ - %varsNotEmpty{, roles=[%map{deprecated.user.roles}]}\ - %varsNotEmpty{, action=[%map{action}]}\ - %varsNotEmpty{, indices=[%map{deprecated.indices}]}\ - %varsNotEmpty{, request=[%map{request.name}]}\ - %varsNotEmpty{, transport_profile=[%map{transport.profile}]}\ - %varsNotEmpty{, rule=[%map{rule}]}\ - %varsNotEmpty{, uri=[%map{deprecated.uri}]}\ - %varsNotEmpty{, opaque_id=[%map{opaque_id}]}\ - %varsNotEmpty{, request_body=[%map{request.body}]}\ - %n -appender.deprecated_audit_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_access-%d{yyyy-MM-dd}.log -appender.deprecated_audit_rolling.policies.type = Policies -appender.deprecated_audit_rolling.policies.time.type = TimeBasedTriggeringPolicy -appender.deprecated_audit_rolling.policies.time.interval = 1 -appender.deprecated_audit_rolling.policies.time.modulate = true - appender.audit_rolling.type = RollingFile appender.audit_rolling.name = audit_rolling appender.audit_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_audit.log @@ -104,7 +67,6 @@ appender.audit_rolling.policies.time.modulate = true logger.xpack_security_audit_logfile.name = org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail logger.xpack_security_audit_logfile.level = info logger.xpack_security_audit_logfile.appenderRef.audit_rolling.ref = audit_rolling -logger.xpack_security_audit_logfile.appenderRef.deprecated_audit_rolling.ref = deprecated_audit_rolling logger.xpack_security_audit_logfile.additivity = false logger.xmlsig.name = org.apache.xml.security.signature.XMLSignature diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/audit/logfile/CapturingLogger.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/audit/logfile/CapturingLogger.java index 31881e7586e5a..ede18c8241b34 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/audit/logfile/CapturingLogger.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/audit/logfile/CapturingLogger.java @@ -30,8 +30,6 @@ */ public class CapturingLogger { - private static final String IMPLICIT_APPENDER_NAME = "__implicit"; - /** * Constructs a new {@link CapturingLogger} named as the fully qualified name of * the invoking method. One name can be assigned to a single logger globally, so @@ -54,88 +52,42 @@ public static Logger newCapturingLogger(final Level level, @Nullable StringLayou final String name = caller.getClassName() + "." + caller.getMethodName() + "." + level.toString(); final Logger logger = ESLoggerFactory.getLogger(name); Loggers.setLevel(logger, level); - attachNewMockAppender(logger, IMPLICIT_APPENDER_NAME, layout); - return logger; - } - - public static void attachNewMockAppender(final Logger logger, final String appenderName, @Nullable StringLayout layout) - throws IllegalAccessException { - final MockAppender appender = new MockAppender(buildAppenderName(logger.getName(), appenderName), layout); + final MockAppender appender = new MockAppender(name, layout); appender.start(); Loggers.addAppender(logger, appender); + return logger; } - private static String buildAppenderName(final String loggerName, final String appenderName) { - // appender name also has to be unique globally (logging context globally) - return loggerName + "." + appenderName; - } - - private static MockAppender getMockAppender(final String loggerName, final String appenderName) { + private static MockAppender getMockAppender(final String name) { final LoggerContext ctx = (LoggerContext) LogManager.getContext(false); final Configuration config = ctx.getConfiguration(); - final LoggerConfig loggerConfig = config.getLoggerConfig(loggerName); - final String mockAppenderName = buildAppenderName(loggerName, appenderName); - return (MockAppender) loggerConfig.getAppenders().get(mockAppenderName); + final LoggerConfig loggerConfig = config.getLoggerConfig(name); + return (MockAppender) loggerConfig.getAppenders().get(name); } /** - * Checks if the logger's appender(s) has captured any events. + * Checks if the logger's appender has captured any events. * - * @param loggerName + * @param name * The unique global name of the logger. - * @param appenderNames - * Names of other appenders nested under this same logger. * @return {@code true} if no event has been captured, {@code false} otherwise. */ - public static boolean isEmpty(final String loggerName, final String... appenderNames) { - // check if implicit appender is empty - final MockAppender implicitAppender = getMockAppender(loggerName, IMPLICIT_APPENDER_NAME); - assert implicitAppender != null; - if (false == implicitAppender.isEmpty()) { - return false; - } - if (null == appenderNames) { - return true; - } - // check if any named appenders are empty - for (String appenderName : appenderNames) { - final MockAppender namedAppender = getMockAppender(loggerName, appenderName); - if (namedAppender != null && false == namedAppender.isEmpty()) { - return false; - } - } - return true; - } - - /** - * Gets the captured events for a logger by its name. Events are those of the - * implicit appender of the logger. - * - * @param loggerName - * The unique global name of the logger. - * @param level - * The priority level of the captured events to be returned. - * @return A list of captured events formated to {@code String}. - */ - public static List output(final String loggerName, final Level level) { - return output(loggerName, IMPLICIT_APPENDER_NAME, level); + public static boolean isEmpty(final String name) { + final MockAppender appender = getMockAppender(name); + return appender.isEmpty(); } /** - * Gets the captured events for a logger and an appender by their respective - * names. There is a one to many relationship between loggers and appenders. + * Gets the captured events for a logger by its name. * - * @param loggerName + * @param name * The unique global name of the logger. - * @param appenderName - * The name of an appender associated with the {@code loggerName} - * logger. * @param level * The priority level of the captured events to be returned. * @return A list of captured events formated to {@code String}. */ - public static List output(final String loggerName, final String appenderName, final Level level) { - final MockAppender appender = getMockAppender(loggerName, appenderName); + public static List output(final String name, final Level level) { + final MockAppender appender = getMockAppender(name); return appender.output(level); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index 5dfc54618b003..359fe7cd464a4 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -611,8 +611,6 @@ LogEntryBuilder withRestUri(RestRequest request) { if (queryStringIndex > -1) { logEntry.with(URL_QUERY_FIELD_NAME, request.uri().substring(queryStringIndex + 1, queryStringLength)); } - // deprecated uri format - logEntry.with("deprecated.uri", request.uri()); return this; } @@ -628,16 +626,10 @@ LogEntryBuilder withRunAsSubject(Authentication authentication) { LogEntryBuilder withRestOrigin(RestRequest request) { assert LOCAL_ORIGIN_FIELD_VALUE.equals(logEntry.get(ORIGIN_TYPE_FIELD_NAME)); // this is the default - final String formattedAddress; - final SocketAddress socketAddress = request.getRemoteAddress(); - if (socketAddress instanceof InetSocketAddress) { - formattedAddress = NetworkAddress.format(((InetSocketAddress) socketAddress)); - } else { - formattedAddress = socketAddress.toString(); - } + final InetSocketAddress socketAddress = request.getHttpChannel().getRemoteAddress(); if (socketAddress != null) { logEntry.with(ORIGIN_TYPE_FIELD_NAME, REST_ORIGIN_FIELD_VALUE) - .with(ORIGIN_ADDRESS_FIELD_NAME, formattedAddress); + .with(ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(socketAddress)); } // fall through to local_node default return this; @@ -708,8 +700,6 @@ LogEntryBuilder with(String key, String value) { LogEntryBuilder with(String key, String[] values) { if (values != null) { logEntry.with(key, toQuotedJsonArray(values)); - // deprecated format required for bwc - logEntry.with("deprecated." + key, Strings.arrayToCommaDelimitedString(values)); } return this; } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java index 24958f250a598..6c302c9311f3d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.mock.orig.Mockito; -import org.elasticsearch.node.Node; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; @@ -129,7 +128,6 @@ protected String expectedMessage() { } private static PatternLayout patternLayout; - private static PatternLayout deprecatedPatternLayout; private Settings settings; private DiscoveryNode localNode; private ClusterService clusterService; @@ -138,13 +136,6 @@ protected String expectedMessage() { private Map commonFields; private Logger logger; private LoggingAuditTrail auditTrail; - // from: - // https://www.myintervals.com/blog/2009/05/20/iso-8601-date-validation-that-doesnt-suck/ - // but stripped off the leading ^ and trailing $ anchors - private static final String ISO8601_DATE_REGEXP = "([\\+-]?\\d{4}(?!\\d{2}\\b))((-?)((0[1-9]|1[0-2])(\\3([12]\\d|0[1-9]|3[01]))?|" - + "W([0-4]\\d|5[0-2])(-?[1-7])?|(00[1-9]|0[1-9]\\d|[12]\\d{2}|3([0-5]\\d|6[1-6])))([T\\s]((([01]\\d|2[0-3])((:?)" - + "[0-5]\\d)?|24\\:?00)([\\.,]\\d+(?!:))?)?(\\17[0-5]\\d([\\.,]\\d+)?)?([zZ]|([\\+-])([01]\\d|2[0-3]):?([0-5]\\d)?)?)?)?"; - private static final String DEPRECATED_APPENDER_NAME = "deprecated"; @BeforeClass public static void lookupPatternLayout() throws Exception { @@ -159,25 +150,16 @@ public static void lookupPatternLayout() throws Exception { // attached to the LoggingAuditTrail.class logger. assertThat(properties.getProperty("logger.xpack_security_audit_logfile.name"), is(LoggingAuditTrail.class.getName())); assertThat(properties.getProperty("logger.xpack_security_audit_logfile.appenderRef.audit_rolling.ref"), is("audit_rolling")); - assertThat(properties.getProperty("logger.xpack_security_audit_logfile.appenderRef.deprecated_audit_rolling.ref"), - is("deprecated_audit_rolling")); assertThat(properties.getProperty("appender.audit_rolling.name"), is("audit_rolling")); - assertThat(properties.getProperty("appender.deprecated_audit_rolling.name"), is("deprecated_audit_rolling")); assertThat(properties.getProperty("appender.audit_rolling.layout.type"), is("PatternLayout")); - assertThat(properties.getProperty("appender.deprecated_audit_rolling.layout.type"), is("PatternLayout")); final String patternLayoutFormat = properties.getProperty("appender.audit_rolling.layout.pattern"); assertThat(patternLayoutFormat, is(notNullValue())); patternLayout = PatternLayout.newBuilder().withPattern(patternLayoutFormat).withCharset(StandardCharsets.UTF_8).build(); - final String deprecatedPatternLayoutFormat = properties.getProperty("appender.deprecated_audit_rolling.layout.pattern"); - assertThat(deprecatedPatternLayoutFormat, is(notNullValue())); - deprecatedPatternLayout = PatternLayout.newBuilder().withPattern(deprecatedPatternLayoutFormat).withCharset(StandardCharsets.UTF_8) - .build(); } @AfterClass public static void releasePatternLayout() { patternLayout = null; - deprecatedPatternLayout = null; } @Before @@ -187,9 +169,7 @@ public void init() throws Exception { .put("xpack.security.audit.logfile.prefix.emit_node_host_address", randomBoolean()) .put("xpack.security.audit.logfile.prefix.emit_node_host_name", randomBoolean()) .put("xpack.security.audit.logfile.prefix.emit_node_name", randomBoolean()) - .put("xpack.security.audit.logfile.prefix.emit_node_id", randomBoolean()) .put("xpack.security.audit.logfile.events.emit_request_body", includeRequestBody) - .put(Node.NODE_NAME_SETTING.getKey(), randomAlphaOfLength(6)) .build(); localNode = mock(DiscoveryNode.class); when(localNode.getAddress()).thenReturn(buildNewFakeTransportAddress()); @@ -208,15 +188,12 @@ public void init() throws Exception { threadContext.putHeader(Task.X_OPAQUE_ID, randomAlphaOfLengthBetween(1, 4)); } logger = CapturingLogger.newCapturingLogger(Level.INFO, patternLayout); - CapturingLogger.attachNewMockAppender(logger, DEPRECATED_APPENDER_NAME, deprecatedPatternLayout); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); } @After public void clearLog() throws Exception { - // clear both the implicit as well as the deprecated appender CapturingLogger.output(logger.getName(), Level.INFO).clear(); - CapturingLogger.output(logger.getName(), DEPRECATED_APPENDER_NAME, Level.INFO).clear(); } public void testAnonymousAccessDeniedTransport() throws Exception { @@ -232,17 +209,16 @@ public void testAnonymousAccessDeniedTransport() throws Exception { restOrTransportOrigin(message, threadContext, checkedFields); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); - assertDeprecatedMsg(logger, settings, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled - clearLog(); + CapturingLogger.output(logger.getName(), Level.INFO).clear(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "anonymous_access_denied") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.anonymousAccessDenied("_action", message); - assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); + assertEmptyLog(logger); } public void testAnonymousAccessDeniedRest() throws Exception { @@ -264,17 +240,16 @@ public void testAnonymousAccessDeniedRest() throws Exception { .put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, null); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap()); - assertDeprecatedMsg(logger, settings, checkedFields.immutableMap()); // test disabled - clearLog(); + CapturingLogger.output(logger.getName(), Level.INFO).clear(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "anonymous_access_denied") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.anonymousAccessDenied(request); - assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); + assertEmptyLog(logger); } public void testAuthenticationFailed() throws Exception { @@ -293,17 +268,16 @@ public void testAuthenticationFailed() throws Exception { indicesRequest(message, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); - assertDeprecatedMsg(logger, settings, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled - clearLog(); + CapturingLogger.output(logger.getName(), Level.INFO).clear(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "authentication_failed") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.authenticationFailed(new MockToken(), "_action", message); - assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); + assertEmptyLog(logger); } public void testAuthenticationFailedNoToken() throws Exception { @@ -320,17 +294,16 @@ public void testAuthenticationFailedNoToken() throws Exception { indicesRequest(message, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); - assertDeprecatedMsg(logger, settings, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled - clearLog(); + CapturingLogger.output(logger.getName(), Level.INFO).clear(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "authentication_failed") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.authenticationFailed("_action", message); - assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); + assertEmptyLog(logger); } public void testAuthenticationFailedRest() throws Exception { @@ -359,17 +332,16 @@ public void testAuthenticationFailedRest() throws Exception { .put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, params.isEmpty() ? null : "foo=bar"); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap()); - assertDeprecatedMsg(logger, settings, checkedFields.immutableMap()); // test disabled - clearLog(); + CapturingLogger.output(logger.getName(), Level.INFO).clear(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "authentication_failed") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.authenticationFailed(new MockToken(), request); - assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); + assertEmptyLog(logger); } public void testAuthenticationFailedRestNoToken() throws Exception { @@ -397,17 +369,16 @@ public void testAuthenticationFailedRestNoToken() throws Exception { .put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, params.isEmpty() ? null : "bar=baz"); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap()); - assertDeprecatedMsg(logger, settings, checkedFields.immutableMap()); // test disabled - clearLog(); + CapturingLogger.output(logger.getName(), Level.INFO).clear(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "authentication_failed") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.authenticationFailed(request); - assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); + assertEmptyLog(logger); } public void testAuthenticationFailedRealm() throws Exception { @@ -415,7 +386,7 @@ public void testAuthenticationFailedRealm() throws Exception { final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); final String realm = randomAlphaOfLengthBetween(1, 6); auditTrail.authenticationFailed(realm, mockToken, "_action", message); - assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); + assertEmptyLog(logger); // test enabled settings = Settings.builder() @@ -436,7 +407,6 @@ public void testAuthenticationFailedRealm() throws Exception { indicesRequest(message, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); - assertDeprecatedMsg(logger, settings, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); } public void testAuthenticationFailedRealmRest() throws Exception { @@ -452,7 +422,7 @@ public void testAuthenticationFailedRealmRest() throws Exception { final AuthenticationToken mockToken = new MockToken(); final String realm = randomAlphaOfLengthBetween(1, 6); auditTrail.authenticationFailed(realm, mockToken, request); - assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); + assertEmptyLog(logger); // test enabled settings = Settings.builder() @@ -475,7 +445,6 @@ public void testAuthenticationFailedRealmRest() throws Exception { .put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, params.isEmpty() ? null : "_param=baz"); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap()); - assertDeprecatedMsg(logger, settings, checkedFields.immutableMap()); } public void testAccessGranted() throws Exception { @@ -487,26 +456,25 @@ public void testAccessGranted() throws Exception { final MapBuilder checkedFields = new MapBuilder<>(commonFields); final MapBuilder checkedArrayFields = new MapBuilder<>(); checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) - .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "access_granted") - .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action") - .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()); + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "access_granted") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action") + .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName()); checkedArrayFields.put(LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME, roles); subject(authentication, checkedFields); restOrTransportOrigin(message, threadContext, checkedFields); indicesRequest(message, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); - assertDeprecatedMsg(logger, settings, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled - clearLog(); + CapturingLogger.output(logger.getName(), Level.INFO).clear(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "access_granted") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.accessGranted(authentication, "_action", message, roles); - assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); + assertEmptyLog(logger); } public void testAccessGrantedInternalSystemAction() throws Exception { @@ -514,7 +482,7 @@ public void testAccessGrantedInternalSystemAction() throws Exception { final String[] roles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4)); final Authentication authentication = new Authentication(SystemUser.INSTANCE, new RealmRef("_reserved", "test", "foo"), null); auditTrail.accessGranted(authentication, "internal:_action", message, roles); - assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); + assertEmptyLog(logger); // test enabled settings = Settings.builder() @@ -536,7 +504,6 @@ public void testAccessGrantedInternalSystemAction() throws Exception { indicesRequest(message, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); - assertDeprecatedMsg(logger, settings, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); } public void testAccessGrantedInternalSystemActionNonSystemUser() throws Exception { @@ -557,17 +524,16 @@ public void testAccessGrantedInternalSystemActionNonSystemUser() throws Exceptio indicesRequest(message, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); - assertDeprecatedMsg(logger, settings, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled - clearLog(); + CapturingLogger.output(logger.getName(), Level.INFO).clear(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "access_granted") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.accessGranted(authentication, "internal:_action", message, roles); - assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); + assertEmptyLog(logger); } public void testAccessDenied() throws Exception { @@ -588,17 +554,16 @@ public void testAccessDenied() throws Exception { indicesRequest(message, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); - assertDeprecatedMsg(logger, settings, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled - clearLog(); + CapturingLogger.output(logger.getName(), Level.INFO).clear(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "access_denied") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.accessDenied(authentication, "_action", message, roles); - assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); + assertEmptyLog(logger); } public void testTamperedRequestRest() throws Exception { @@ -623,17 +588,16 @@ public void testTamperedRequestRest() throws Exception { .put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, params.isEmpty() ? null : "_param=baz"); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap()); - assertDeprecatedMsg(logger, settings, checkedFields.immutableMap()); // test disabled - clearLog(); + CapturingLogger.output(logger.getName(), Level.INFO).clear(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "tampered_request") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.tamperedRequest(request); - assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); + assertEmptyLog(logger); } public void testTamperedRequest() throws Exception { @@ -650,17 +614,16 @@ public void testTamperedRequest() throws Exception { indicesRequest(message, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); - assertDeprecatedMsg(logger, settings, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled - clearLog(); + CapturingLogger.output(logger.getName(), Level.INFO).clear(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "tampered_request") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.tamperedRequest("_action", message); - assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); + assertEmptyLog(logger); } public void testTamperedRequestWithUser() throws Exception { @@ -690,17 +653,16 @@ public void testTamperedRequestWithUser() throws Exception { indicesRequest(message, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); - assertDeprecatedMsg(logger, settings, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled - clearLog(); + CapturingLogger.output(logger.getName(), Level.INFO).clear(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "tampered_request") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.tamperedRequest(user, "_action", message); - assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); + assertEmptyLog(logger); } public void testConnectionDenied() throws Exception { @@ -718,17 +680,16 @@ public void testConnectionDenied() throws Exception { .put(LoggingAuditTrail.RULE_FIELD_NAME, "deny _all"); opaqueId(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap()); - assertDeprecatedMsg(logger, settings, checkedFields.immutableMap()); // test disabled - clearLog(); + CapturingLogger.output(logger.getName(), Level.INFO).clear(); settings = Settings.builder() .put(settings) .put("xpack.security.audit.logfile.events.exclude", "connection_denied") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.connectionDenied(inetAddress, profile, rule); - assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); + assertEmptyLog(logger); } public void testConnectionGranted() throws Exception { @@ -737,7 +698,7 @@ public void testConnectionGranted() throws Exception { final String profile = randomAlphaOfLengthBetween(1, 6); auditTrail.connectionGranted(inetAddress, profile, rule); - assertEmptyLog(logger, DEPRECATED_APPENDER_NAME); + assertEmptyLog(logger); // test enabled settings = Settings.builder() @@ -762,7 +723,8 @@ public void testRunAsGranted() throws Exception { final String[] roles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4)); final Authentication authentication = new Authentication( new User("running as", new String[] { "r2" }, new User("_username", new String[] { "r1" })), - new RealmRef("authRealm", "test", "foo"), new RealmRef("lookRealm", "up", "by")); + new RealmRef("authRealm", "test", "foo"), + new RealmRef("lookRealm", "up", "by")); auditTrail.runAsGranted(authentication, "_action", message, roles); final MapBuilder checkedFields = new MapBuilder<>(commonFields); @@ -797,7 +759,8 @@ public void testRunAsDenied() throws Exception { final String[] roles = randomArray(0, 4, String[]::new, () -> randomAlphaOfLengthBetween(1, 4)); final Authentication authentication = new Authentication( new User("running as", new String[] { "r2" }, new User("_username", new String[] { "r1" })), - new RealmRef("authRealm", "test", "foo"), new RealmRef("lookRealm", "up", "by")); + new RealmRef("authRealm", "test", "foo"), + new RealmRef("lookRealm", "up", "by")); auditTrail.runAsDenied(authentication, "_action", message, roles); final MapBuilder checkedFields = new MapBuilder<>(commonFields); @@ -971,107 +934,6 @@ private void assertMsg(Logger logger, Map checkFields) { assertMsg(logger, checkFields, Collections.emptyMap()); } - private void assertDeprecatedMsg(Logger logger, Settings settings, Map checkFields) { - assertDeprecatedMsg(logger, settings, checkFields, Collections.emptyMap()); - } - - private void assertDeprecatedMsg(Logger logger, Settings settings, Map checkFields, - Map checkArrayFields) { - final List output = CapturingLogger.output(logger.getName(), DEPRECATED_APPENDER_NAME, Level.INFO); - assertThat("Exactly one log line expected. Found: " + output.size(), output.size(), is(1)); - if (checkFields == null) { - // only check msg existence - return; - } - final String logLine = output.get(0); - final StringBuilder logLineRegexpBuilder = new StringBuilder(); - if (LoggingAuditTrail.EMIT_HOST_ADDRESS_SETTING.get(settings)) { - logLineRegexpBuilder.append(" [").append(checkFields.get(LoggingAuditTrail.HOST_ADDRESS_FIELD_NAME)).append("]"); - } - if (LoggingAuditTrail.EMIT_HOST_NAME_SETTING.get(settings)) { - logLineRegexpBuilder.append(" [").append(checkFields.get(LoggingAuditTrail.HOST_NAME_FIELD_NAME)).append("]"); - } - if (LoggingAuditTrail.EMIT_NODE_NAME_SETTING.get(settings)) { - logLineRegexpBuilder.append(" [").append(checkFields.get(LoggingAuditTrail.NODE_NAME_FIELD_NAME)).append("]"); - } - logLineRegexpBuilder.append(" [").append(checkFields.get(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME)).append("]"); - logLineRegexpBuilder.append(" [").append(checkFields.get(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME)).append("]"); - logLineRegexpBuilder.append(" "); - if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.REALM_FIELD_NAME))) { - logLineRegexpBuilder.append("realm=[").append(checkFields.get(LoggingAuditTrail.REALM_FIELD_NAME)).append("], "); - } - logLineRegexpBuilder.append("origin_type=[").append(checkFields.get(LoggingAuditTrail.ORIGIN_TYPE_FIELD_NAME)).append("]"); - if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.ORIGIN_ADDRESS_FIELD_NAME))) { - logLineRegexpBuilder.append(", origin_address=[").append(checkFields.get(LoggingAuditTrail.ORIGIN_ADDRESS_FIELD_NAME)) - .append("]"); - } - if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.PRINCIPAL_FIELD_NAME))) { - logLineRegexpBuilder.append(", principal=[").append(checkFields.get(LoggingAuditTrail.PRINCIPAL_FIELD_NAME)).append("]"); - } - if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.PRINCIPAL_REALM_FIELD_NAME))) { - logLineRegexpBuilder.append(", realm=[").append(checkFields.get(LoggingAuditTrail.PRINCIPAL_REALM_FIELD_NAME)).append("]"); - } - if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.PRINCIPAL_RUN_BY_FIELD_NAME))) { - logLineRegexpBuilder.append(", run_by_principal=[").append(checkFields.get(LoggingAuditTrail.PRINCIPAL_RUN_BY_FIELD_NAME)) - .append("]"); - } - if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.PRINCIPAL_RUN_AS_FIELD_NAME))) { - logLineRegexpBuilder.append(", run_as_principal=[").append(checkFields.get(LoggingAuditTrail.PRINCIPAL_RUN_AS_FIELD_NAME)) - .append("]"); - } - if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.PRINCIPAL_RUN_BY_REALM_FIELD_NAME))) { - logLineRegexpBuilder.append(", run_by_realm=[").append(checkFields.get(LoggingAuditTrail.PRINCIPAL_RUN_BY_REALM_FIELD_NAME)) - .append("]"); - } - if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.PRINCIPAL_RUN_AS_REALM_FIELD_NAME))) { - logLineRegexpBuilder.append(", run_as_realm=[").append(checkFields.get(LoggingAuditTrail.PRINCIPAL_RUN_AS_REALM_FIELD_NAME)) - .append("]"); - } - final String[] roles = checkArrayFields.get(LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME); - if (null != roles && roles.length > 0) { - logLineRegexpBuilder.append(", roles=[") - .append(Strings.arrayToCommaDelimitedString(checkArrayFields.get(LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME))) - .append("]"); - } - if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.ACTION_FIELD_NAME))) { - logLineRegexpBuilder.append(", action=[").append(checkFields.get(LoggingAuditTrail.ACTION_FIELD_NAME)).append("]"); - } - final String[] indices = checkArrayFields.get(LoggingAuditTrail.INDICES_FIELD_NAME); - if (null != indices && indices.length > 0) { - logLineRegexpBuilder.append(", indices=[") - .append(Strings.arrayToCommaDelimitedString(checkArrayFields.get(LoggingAuditTrail.INDICES_FIELD_NAME))).append("]"); - } - if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME))) { - logLineRegexpBuilder.append(", request=[").append(checkFields.get(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME)).append("]"); - } - if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.TRANSPORT_PROFILE_FIELD_NAME))) { - logLineRegexpBuilder.append(", transport_profile=[").append(checkFields.get(LoggingAuditTrail.TRANSPORT_PROFILE_FIELD_NAME)) - .append("]"); - } - if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.RULE_FIELD_NAME))) { - logLineRegexpBuilder.append(", rule=[").append(checkFields.get(LoggingAuditTrail.RULE_FIELD_NAME)).append("]"); - } - if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.URL_PATH_FIELD_NAME))) { - String uri = checkFields.get(LoggingAuditTrail.URL_PATH_FIELD_NAME); - if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.URL_QUERY_FIELD_NAME))) { - uri += "?" + checkFields.get(LoggingAuditTrail.URL_QUERY_FIELD_NAME); - } - logLineRegexpBuilder.append(", uri=[").append(uri).append("]"); - } - if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.OPAQUE_ID_FIELD_NAME))) { - logLineRegexpBuilder.append(", opaque_id=[").append(checkFields.get(LoggingAuditTrail.OPAQUE_ID_FIELD_NAME)).append("]"); - } - if (false == Strings.isNullOrEmpty(checkFields.get(LoggingAuditTrail.REQUEST_BODY_FIELD_NAME))) { - logLineRegexpBuilder.append(", request_body=[").append(checkFields.get(LoggingAuditTrail.REQUEST_BODY_FIELD_NAME)).append("]"); - } - logLineRegexpBuilder.append(System.lineSeparator()); - - // regex for the date field - final String logLineRegexp = "^\\[" + ISO8601_DATE_REGEXP + "\\]" + Pattern.quote(logLineRegexpBuilder.toString()) + "$"; - assertThat("The regexp does not match the log line: |" + logLineRegexp + "| |" + logLine + "|", - Pattern.matches(logLineRegexp, logLine), is(true)); - } - private void assertMsg(Logger logger, Map checkFields, Map checkArrayFields) { final List output = CapturingLogger.output(logger.getName(), Level.INFO); assertThat("Exactly one logEntry expected. Found: " + output.size(), output.size(), is(1)); @@ -1114,13 +976,13 @@ private void assertMsg(Logger logger, Map checkFields, Map prepareRestContent(String uri, InetSocketAddress remoteAddress) { From b3adbdbf6de0a143d4a8e48240688c9719f47035 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 5 Oct 2018 09:28:45 +0300 Subject: [PATCH 04/10] network fromatting backport --- .../security/audit/logfile/LoggingAuditTrail.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index 359fe7cd464a4..358a4a0b12cb6 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -626,10 +626,16 @@ LogEntryBuilder withRunAsSubject(Authentication authentication) { LogEntryBuilder withRestOrigin(RestRequest request) { assert LOCAL_ORIGIN_FIELD_VALUE.equals(logEntry.get(ORIGIN_TYPE_FIELD_NAME)); // this is the default - final InetSocketAddress socketAddress = request.getHttpChannel().getRemoteAddress(); + final String formattedAddress; + final SocketAddress socketAddress = request.getRemoteAddress(); + if (socketAddress instanceof InetSocketAddress) { + formattedAddress = NetworkAddress.format(((InetSocketAddress) socketAddress)); + } else { + formattedAddress = socketAddress.toString(); + } if (socketAddress != null) { logEntry.with(ORIGIN_TYPE_FIELD_NAME, REST_ORIGIN_FIELD_VALUE) - .with(ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(socketAddress)); + .with(ORIGIN_ADDRESS_FIELD_NAME, formattedAddress); } // fall through to local_node default return this; From 0e69812b070832e6142bd5d22c847d08474a3b48 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 5 Oct 2018 10:32:57 +0300 Subject: [PATCH 05/10] Cherry-picked the old LoggingAuditTrail --- .../xpack/security/Security.java | 4 + .../logfile/DeprecatedLoggingAuditTrail.java | 629 ++++++++++++++++++ .../audit/logfile/LoggingAuditTrail.java | 26 +- .../xpack/security/SecurityTests.java | 13 +- 4 files changed, 652 insertions(+), 20 deletions(-) create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/DeprecatedLoggingAuditTrail.java diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 24bb7f9d0b5f9..5ffd50952bc2d 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -171,6 +171,7 @@ import org.elasticsearch.xpack.security.audit.AuditTrailService; import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail; import org.elasticsearch.xpack.security.audit.index.IndexNameResolver; +import org.elasticsearch.xpack.security.audit.logfile.DeprecatedLoggingAuditTrail; import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail; import org.elasticsearch.xpack.security.authc.AuthenticationService; import org.elasticsearch.xpack.security.authc.InternalRealms; @@ -417,6 +418,9 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste switch (output) { case LoggingAuditTrail.NAME: auditTrails.add(new LoggingAuditTrail(settings, clusterService, threadPool)); + // also enabling the deprecated format. To disable it, remove it's associated + // appender in the log4j2.properties file + auditTrails.add(new DeprecatedLoggingAuditTrail(settings, clusterService, threadPool)); break; case IndexAuditTrail.NAME: indexAuditTrail.set(new IndexAuditTrail(settings, client, threadPool, clusterService)); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/DeprecatedLoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/DeprecatedLoggingAuditTrail.java new file mode 100644 index 0000000000000..2d948919a84ce --- /dev/null +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/DeprecatedLoggingAuditTrail.java @@ -0,0 +1,629 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.security.audit.logfile; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.node.Node; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportMessage; +import org.elasticsearch.xpack.core.security.authc.Authentication; +import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; +import org.elasticsearch.xpack.core.security.user.SystemUser; +import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.xpack.core.security.user.XPackUser; +import org.elasticsearch.xpack.security.audit.AuditLevel; +import org.elasticsearch.xpack.security.audit.AuditTrail; +import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.AuditEventMetaInfo; +import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.EventFilterPolicy; +import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.EventFilterPolicyRegistry; +import org.elasticsearch.xpack.security.rest.RemoteHostHeader; +import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.Optional; + +import static org.elasticsearch.common.Strings.arrayToCommaDelimitedString; +import static org.elasticsearch.xpack.security.audit.AuditLevel.ACCESS_DENIED; +import static org.elasticsearch.xpack.security.audit.AuditLevel.ACCESS_GRANTED; +import static org.elasticsearch.xpack.security.audit.AuditLevel.ANONYMOUS_ACCESS_DENIED; +import static org.elasticsearch.xpack.security.audit.AuditLevel.AUTHENTICATION_FAILED; +import static org.elasticsearch.xpack.security.audit.AuditLevel.AUTHENTICATION_SUCCESS; +import static org.elasticsearch.xpack.security.audit.AuditLevel.CONNECTION_DENIED; +import static org.elasticsearch.xpack.security.audit.AuditLevel.CONNECTION_GRANTED; +import static org.elasticsearch.xpack.security.audit.AuditLevel.REALM_AUTHENTICATION_FAILED; +import static org.elasticsearch.xpack.security.audit.AuditLevel.RUN_AS_DENIED; +import static org.elasticsearch.xpack.security.audit.AuditLevel.RUN_AS_GRANTED; +import static org.elasticsearch.xpack.security.audit.AuditLevel.SYSTEM_ACCESS_GRANTED; +import static org.elasticsearch.xpack.security.audit.AuditLevel.TAMPERED_REQUEST; +import static org.elasticsearch.xpack.security.audit.AuditLevel.parse; +import static org.elasticsearch.xpack.security.audit.AuditUtil.restRequestContent; + +public class DeprecatedLoggingAuditTrail extends AbstractComponent implements AuditTrail, ClusterStateListener { + + public static final String NAME = "deprecatedLogfile"; + + private final Logger logger; + final EventFilterPolicyRegistry eventFilterPolicyRegistry; + private final ThreadContext threadContext; + // package for testing + volatile EnumSet events; + boolean includeRequestBody; + LocalNodeInfo localNodeInfo; + + @Override + public String name() { + return NAME; + } + + public DeprecatedLoggingAuditTrail(Settings settings, ClusterService clusterService, ThreadPool threadPool) { + this(settings, clusterService, Loggers.getLogger(DeprecatedLoggingAuditTrail.class), threadPool.getThreadContext()); + } + + DeprecatedLoggingAuditTrail(Settings settings, ClusterService clusterService, Logger logger, ThreadContext threadContext) { + super(settings); + this.logger = logger; + this.events = parse(LoggingAuditTrail.INCLUDE_EVENT_SETTINGS.get(settings), LoggingAuditTrail.EXCLUDE_EVENT_SETTINGS.get(settings)); + this.includeRequestBody = LoggingAuditTrail.INCLUDE_REQUEST_BODY.get(settings); + this.threadContext = threadContext; + this.localNodeInfo = new LocalNodeInfo(settings, null); + this.eventFilterPolicyRegistry = new EventFilterPolicyRegistry(settings); + clusterService.addListener(this); + clusterService.getClusterSettings().addSettingsUpdateConsumer(newSettings -> { + final LocalNodeInfo localNodeInfo = this.localNodeInfo; + final Settings.Builder builder = Settings.builder().put(localNodeInfo.settings).put(newSettings, false); + this.localNodeInfo = new LocalNodeInfo(builder.build(), localNodeInfo.localNode); + this.includeRequestBody = LoggingAuditTrail.INCLUDE_REQUEST_BODY.get(newSettings); + // `events` is a volatile field! Keep `events` write last so that + // `localNodeInfo` and `includeRequestBody` writes happen-before! `events` is + // always read before `localNodeInfo` and `includeRequestBody`. + this.events = parse(LoggingAuditTrail.INCLUDE_EVENT_SETTINGS.get(newSettings), + LoggingAuditTrail.EXCLUDE_EVENT_SETTINGS.get(newSettings)); + }, Arrays.asList(LoggingAuditTrail.EMIT_HOST_ADDRESS_SETTING, LoggingAuditTrail.EMIT_HOST_NAME_SETTING, + LoggingAuditTrail.EMIT_NODE_NAME_SETTING, LoggingAuditTrail.INCLUDE_EVENT_SETTINGS, + LoggingAuditTrail.EXCLUDE_EVENT_SETTINGS, LoggingAuditTrail.INCLUDE_REQUEST_BODY)); + clusterService.getClusterSettings().addAffixUpdateConsumer(LoggingAuditTrail.FILTER_POLICY_IGNORE_PRINCIPALS, + (policyName, filtersList) -> { + final Optional policy = eventFilterPolicyRegistry.get(policyName); + final EventFilterPolicy newPolicy = policy.orElse(new EventFilterPolicy(policyName, settings)) + .changePrincipalsFilter(filtersList); + this.eventFilterPolicyRegistry.set(policyName, newPolicy); + }, (policyName, filtersList) -> EventFilterPolicy.parsePredicate(filtersList)); + clusterService.getClusterSettings().addAffixUpdateConsumer(LoggingAuditTrail.FILTER_POLICY_IGNORE_REALMS, + (policyName, filtersList) -> { + final Optional policy = eventFilterPolicyRegistry.get(policyName); + final EventFilterPolicy newPolicy = policy.orElse(new EventFilterPolicy(policyName, settings)) + .changeRealmsFilter(filtersList); + this.eventFilterPolicyRegistry.set(policyName, newPolicy); + }, (policyName, filtersList) -> EventFilterPolicy.parsePredicate(filtersList)); + clusterService.getClusterSettings().addAffixUpdateConsumer(LoggingAuditTrail.FILTER_POLICY_IGNORE_ROLES, + (policyName, filtersList) -> { + final Optional policy = eventFilterPolicyRegistry.get(policyName); + final EventFilterPolicy newPolicy = policy.orElse(new EventFilterPolicy(policyName, settings)) + .changeRolesFilter(filtersList); + this.eventFilterPolicyRegistry.set(policyName, newPolicy); + }, (policyName, filtersList) -> EventFilterPolicy.parsePredicate(filtersList)); + clusterService.getClusterSettings().addAffixUpdateConsumer(LoggingAuditTrail.FILTER_POLICY_IGNORE_INDICES, + (policyName, filtersList) -> { + final Optional policy = eventFilterPolicyRegistry.get(policyName); + final EventFilterPolicy newPolicy = policy.orElse(new EventFilterPolicy(policyName, settings)) + .changeIndicesFilter(filtersList); + this.eventFilterPolicyRegistry.set(policyName, newPolicy); + }, (policyName, filtersList) -> EventFilterPolicy.parsePredicate(filtersList)); + } + + @Override + public void authenticationSuccess(String realm, User user, RestRequest request) { + if (events.contains(AUTHENTICATION_SUCCESS) && (eventFilterPolicyRegistry.ignorePredicate() + .test(new AuditEventMetaInfo(Optional.of(user), Optional.of(realm), Optional.empty(), Optional.empty())) == false)) { + if (includeRequestBody) { + logger.info("{}[rest] [authentication_success]\t{}, realm=[{}], uri=[{}], params=[{}]{}, request_body=[{}]", + localNodeInfo.prefix, principal(user), realm, request.uri(), request.params(), opaqueId(), + restRequestContent(request)); + } else { + logger.info("{}[rest] [authentication_success]\t{}, realm=[{}], uri=[{}], params=[{}]{}", localNodeInfo.prefix, + principal(user), realm, request.uri(), request.params(), opaqueId()); + } + } + } + + @Override + public void authenticationSuccess(String realm, User user, String action, TransportMessage message) { + if (events.contains(AUTHENTICATION_SUCCESS)) { + final Optional indices = indices(message); + if (eventFilterPolicyRegistry.ignorePredicate() + .test(new AuditEventMetaInfo(Optional.of(user), Optional.of(realm), Optional.empty(), indices)) == false) { + final LocalNodeInfo localNodeInfo = this.localNodeInfo; + if (indices.isPresent()) { + logger.info("{}[transport] [authentication_success]\t{}, {}, realm=[{}], action=[{}], indices=[{}], request=[{}]{}", + localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), principal(user), realm, action, + arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName(), opaqueId()); + } else { + logger.info("{}[transport] [authentication_success]\t{}, {}, realm=[{}], action=[{}], request=[{}]{}", + localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), principal(user), realm, action, + message.getClass().getSimpleName(), opaqueId()); + } + } + } + } + + @Override + public void anonymousAccessDenied(String action, TransportMessage message) { + if (events.contains(ANONYMOUS_ACCESS_DENIED)) { + final Optional indices = indices(message); + if (eventFilterPolicyRegistry.ignorePredicate() + .test(new AuditEventMetaInfo(Optional.empty(), Optional.empty(), indices)) == false) { + final LocalNodeInfo localNodeInfo = this.localNodeInfo; + if (indices.isPresent()) { + logger.info("{}[transport] [anonymous_access_denied]\t{}, action=[{}], indices=[{}], request=[{}]{}", + localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), action, + arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName(), opaqueId()); + } else { + logger.info("{}[transport] [anonymous_access_denied]\t{}, action=[{}], request=[{}]{}", localNodeInfo.prefix, + originAttributes(threadContext, message, localNodeInfo), action, message.getClass().getSimpleName(), + opaqueId()); + } + } + } + } + + @Override + public void anonymousAccessDenied(RestRequest request) { + if (events.contains(ANONYMOUS_ACCESS_DENIED) + && (eventFilterPolicyRegistry.ignorePredicate().test(AuditEventMetaInfo.EMPTY) == false)) { + if (includeRequestBody) { + logger.info("{}[rest] [anonymous_access_denied]\t{}, uri=[{}]{}, request_body=[{}]", localNodeInfo.prefix, + hostAttributes(request), request.uri(), opaqueId(), restRequestContent(request)); + } else { + logger.info("{}[rest] [anonymous_access_denied]\t{}, uri=[{}]{}", localNodeInfo.prefix, hostAttributes(request), + request.uri(), opaqueId()); + } + } + } + + @Override + public void authenticationFailed(AuthenticationToken token, String action, TransportMessage message) { + if (events.contains(AUTHENTICATION_FAILED)) { + final Optional indices = indices(message); + if (eventFilterPolicyRegistry.ignorePredicate() + .test(new AuditEventMetaInfo(Optional.of(token), Optional.empty(), indices)) == false) { + final LocalNodeInfo localNodeInfo = this.localNodeInfo; + if (indices.isPresent()) { + logger.info("{}[transport] [authentication_failed]\t{}, principal=[{}], action=[{}], indices=[{}], request=[{}]{}", + localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), token.principal(), action, + arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName(), opaqueId()); + } else { + logger.info("{}[transport] [authentication_failed]\t{}, principal=[{}], action=[{}], request=[{}]{}", + localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), token.principal(), action, + message.getClass().getSimpleName(), opaqueId()); + } + } + } + } + + @Override + public void authenticationFailed(RestRequest request) { + if (events.contains(AUTHENTICATION_FAILED) + && (eventFilterPolicyRegistry.ignorePredicate().test(AuditEventMetaInfo.EMPTY) == false)) { + if (includeRequestBody) { + logger.info("{}[rest] [authentication_failed]\t{}, uri=[{}]{}, request_body=[{}]", localNodeInfo.prefix, + hostAttributes(request), request.uri(), opaqueId(), restRequestContent(request)); + } else { + logger.info("{}[rest] [authentication_failed]\t{}, uri=[{}]{}", localNodeInfo.prefix, hostAttributes(request), + request.uri(), opaqueId()); + } + } + } + + @Override + public void authenticationFailed(String action, TransportMessage message) { + if (events.contains(AUTHENTICATION_FAILED)) { + final Optional indices = indices(message); + if (eventFilterPolicyRegistry.ignorePredicate() + .test(new AuditEventMetaInfo(Optional.empty(), Optional.empty(), indices)) == false) { + final LocalNodeInfo localNodeInfo = this.localNodeInfo; + if (indices.isPresent()) { + logger.info("{}[transport] [authentication_failed]\t{}, action=[{}], indices=[{}], request=[{}]{}", + localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), action, + arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName(), opaqueId()); + } else { + logger.info("{}[transport] [authentication_failed]\t{}, action=[{}], request=[{}]{}", localNodeInfo.prefix, + originAttributes(threadContext, message, localNodeInfo), action, message.getClass().getSimpleName(), + opaqueId()); + } + } + } + } + + @Override + public void authenticationFailed(AuthenticationToken token, RestRequest request) { + if (events.contains(AUTHENTICATION_FAILED) && (eventFilterPolicyRegistry.ignorePredicate() + .test(new AuditEventMetaInfo(Optional.of(token), Optional.empty(), Optional.empty())) == false)) { + if (includeRequestBody) { + logger.info("{}[rest] [authentication_failed]\t{}, principal=[{}], uri=[{}]{}, request_body=[{}]", localNodeInfo.prefix, + hostAttributes(request), token.principal(), request.uri(), opaqueId(), restRequestContent(request)); + } else { + logger.info("{}[rest] [authentication_failed]\t{}, principal=[{}], uri=[{}]{}", localNodeInfo.prefix, + hostAttributes(request), token.principal(), request.uri(), opaqueId()); + } + } + } + + @Override + public void authenticationFailed(String realm, AuthenticationToken token, String action, TransportMessage message) { + if (events.contains(REALM_AUTHENTICATION_FAILED)) { + final Optional indices = indices(message); + if (eventFilterPolicyRegistry.ignorePredicate() + .test(new AuditEventMetaInfo(Optional.of(token), Optional.of(realm), indices)) == false) { + final LocalNodeInfo localNodeInfo = this.localNodeInfo; + if (indices.isPresent()) { + logger.info( + "{}[transport] [realm_authentication_failed]\trealm=[{}], {}, principal=[{}], action=[{}], indices=[{}], " + + "request=[{}]{}", + localNodeInfo.prefix, realm, originAttributes(threadContext, message, localNodeInfo), token.principal(), action, + arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName(), opaqueId()); + } else { + logger.info("{}[transport] [realm_authentication_failed]\trealm=[{}], {}, principal=[{}], action=[{}], request=[{}]{}", + localNodeInfo.prefix, realm, originAttributes(threadContext, message, localNodeInfo), token.principal(), action, + message.getClass().getSimpleName(), opaqueId()); + } + } + } + } + + @Override + public void authenticationFailed(String realm, AuthenticationToken token, RestRequest request) { + if (events.contains(REALM_AUTHENTICATION_FAILED) && (eventFilterPolicyRegistry.ignorePredicate() + .test(new AuditEventMetaInfo(Optional.of(token), Optional.of(realm), Optional.empty())) == false)) { + if (includeRequestBody) { + logger.info("{}[rest] [realm_authentication_failed]\trealm=[{}], {}, principal=[{}], uri=[{}]{}, " + "request_body=[{}]", + localNodeInfo.prefix, realm, hostAttributes(request), token.principal(), request.uri(), opaqueId(), + restRequestContent(request)); + } else { + logger.info("{}[rest] [realm_authentication_failed]\trealm=[{}], {}, principal=[{}], uri=[{}]{}", localNodeInfo.prefix, + realm, hostAttributes(request), token.principal(), request.uri(), opaqueId()); + } + } + } + + @Override + public void accessGranted(Authentication authentication, String action, TransportMessage message, String[] roleNames) { + final User user = authentication.getUser(); + final boolean isSystem = SystemUser.is(user) || XPackUser.is(user); + if ((isSystem && events.contains(SYSTEM_ACCESS_GRANTED)) || ((isSystem == false) && events.contains(ACCESS_GRANTED))) { + final Optional indices = indices(message); + if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(user), + Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), indices)) == false) { + final LocalNodeInfo localNodeInfo = this.localNodeInfo; + if (indices.isPresent()) { + logger.info("{}[transport] [access_granted]\t{}, {}, roles=[{}], action=[{}], indices=[{}], request=[{}]{}", + localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), subject(authentication), + arrayToCommaDelimitedString(roleNames), action, arrayToCommaDelimitedString(indices.get()), + message.getClass().getSimpleName(), opaqueId()); + } else { + logger.info("{}[transport] [access_granted]\t{}, {}, roles=[{}], action=[{}], request=[{}]{}", localNodeInfo.prefix, + originAttributes(threadContext, message, localNodeInfo), subject(authentication), + arrayToCommaDelimitedString(roleNames), action, message.getClass().getSimpleName(), opaqueId()); + } + } + } + } + + @Override + public void accessDenied(Authentication authentication, String action, TransportMessage message, String[] roleNames) { + if (events.contains(ACCESS_DENIED)) { + final Optional indices = indices(message); + if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()), + Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), indices)) == false) { + final LocalNodeInfo localNodeInfo = this.localNodeInfo; + if (indices.isPresent()) { + logger.info("{}[transport] [access_denied]\t{}, {}, roles=[{}], action=[{}], indices=[{}], request=[{}]{}", + localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), subject(authentication), + arrayToCommaDelimitedString(roleNames), action, arrayToCommaDelimitedString(indices.get()), + message.getClass().getSimpleName(), opaqueId()); + } else { + logger.info("{}[transport] [access_denied]\t{}, {}, roles=[{}], action=[{}], request=[{}]{}", localNodeInfo.prefix, + originAttributes(threadContext, message, localNodeInfo), subject(authentication), + arrayToCommaDelimitedString(roleNames), action, message.getClass().getSimpleName(), opaqueId()); + } + } + } + } + + @Override + public void tamperedRequest(RestRequest request) { + if (events.contains(TAMPERED_REQUEST) && (eventFilterPolicyRegistry.ignorePredicate().test(AuditEventMetaInfo.EMPTY) == false)) { + if (includeRequestBody) { + logger.info("{}[rest] [tampered_request]\t{}, uri=[{}]{}, request_body=[{}]", localNodeInfo.prefix, hostAttributes(request), + request.uri(), opaqueId(), restRequestContent(request)); + } else { + logger.info("{}[rest] [tampered_request]\t{}, uri=[{}]{}", localNodeInfo.prefix, hostAttributes(request), request.uri(), + opaqueId()); + } + } + } + + @Override + public void tamperedRequest(String action, TransportMessage message) { + if (events.contains(TAMPERED_REQUEST)) { + final Optional indices = indices(message); + if (eventFilterPolicyRegistry.ignorePredicate() + .test(new AuditEventMetaInfo(Optional.empty(), Optional.empty(), indices)) == false) { + final LocalNodeInfo localNodeInfo = this.localNodeInfo; + if (indices.isPresent()) { + logger.info("{}[transport] [tampered_request]\t{}, action=[{}], indices=[{}], request=[{}]{}", localNodeInfo.prefix, + originAttributes(threadContext, message, localNodeInfo), action, arrayToCommaDelimitedString(indices.get()), + message.getClass().getSimpleName(), opaqueId()); + } else { + logger.info("{}[transport] [tampered_request]\t{}, action=[{}], request=[{}]{}", localNodeInfo.prefix, + originAttributes(threadContext, message, localNodeInfo), action, message.getClass().getSimpleName(), + opaqueId()); + } + } + } + } + + @Override + public void tamperedRequest(User user, String action, TransportMessage request) { + if (events.contains(TAMPERED_REQUEST)) { + final Optional indices = indices(request); + if (eventFilterPolicyRegistry.ignorePredicate() + .test(new AuditEventMetaInfo(Optional.of(user), Optional.empty(), Optional.empty(), indices)) == false) { + final LocalNodeInfo localNodeInfo = this.localNodeInfo; + if (indices.isPresent()) { + logger.info("{}[transport] [tampered_request]\t{}, {}, action=[{}], indices=[{}], request=[{}]{}", localNodeInfo.prefix, + originAttributes(threadContext, request, localNodeInfo), principal(user), action, + arrayToCommaDelimitedString(indices.get()), request.getClass().getSimpleName(), opaqueId()); + } else { + logger.info("{}[transport] [tampered_request]\t{}, {}, action=[{}], request=[{}]{}", localNodeInfo.prefix, + originAttributes(threadContext, request, localNodeInfo), principal(user), action, + request.getClass().getSimpleName(), opaqueId()); + } + } + } + } + + @Override + public void connectionGranted(InetAddress inetAddress, String profile, SecurityIpFilterRule rule) { + if (events.contains(CONNECTION_GRANTED) && (eventFilterPolicyRegistry.ignorePredicate().test(AuditEventMetaInfo.EMPTY) == false)) { + logger.info("{}[ip_filter] [connection_granted]\torigin_address=[{}], transport_profile=[{}], rule=[{}]{}", + localNodeInfo.prefix, NetworkAddress.format(inetAddress), profile, rule, opaqueId()); + } + } + + @Override + public void connectionDenied(InetAddress inetAddress, String profile, SecurityIpFilterRule rule) { + if (events.contains(CONNECTION_DENIED) && (eventFilterPolicyRegistry.ignorePredicate().test(AuditEventMetaInfo.EMPTY) == false)) { + logger.info("{}[ip_filter] [connection_denied]\torigin_address=[{}], transport_profile=[{}], rule=[{}]{}", localNodeInfo.prefix, + NetworkAddress.format(inetAddress), profile, rule, opaqueId()); + } + } + + @Override + public void runAsGranted(Authentication authentication, String action, TransportMessage message, String[] roleNames) { + if (events.contains(RUN_AS_GRANTED)) { + final Optional indices = indices(message); + if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()), + Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), indices)) == false) { + final LocalNodeInfo localNodeInfo = this.localNodeInfo; + if (indices.isPresent()) { + logger.info("{}[transport] [run_as_granted]\t{}, {}, roles=[{}], action=[{}], indices=[{}], request=[{}]{}", + localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), runAsSubject(authentication), + arrayToCommaDelimitedString(roleNames), action, arrayToCommaDelimitedString(indices.get()), + message.getClass().getSimpleName(), opaqueId()); + } else { + logger.info("{}[transport] [run_as_granted]\t{}, {}, roles=[{}], action=[{}], request=[{}]{}", localNodeInfo.prefix, + originAttributes(threadContext, message, localNodeInfo), runAsSubject(authentication), + arrayToCommaDelimitedString(roleNames), action, message.getClass().getSimpleName(), opaqueId()); + } + } + } + } + + @Override + public void runAsDenied(Authentication authentication, String action, TransportMessage message, String[] roleNames) { + if (events.contains(RUN_AS_DENIED)) { + final Optional indices = indices(message); + if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()), + Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), indices)) == false) { + final LocalNodeInfo localNodeInfo = this.localNodeInfo; + if (indices.isPresent()) { + logger.info("{}[transport] [run_as_denied]\t{}, {}, roles=[{}], action=[{}], indices=[{}], request=[{}]{}", + localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), runAsSubject(authentication), + arrayToCommaDelimitedString(roleNames), action, arrayToCommaDelimitedString(indices.get()), + message.getClass().getSimpleName(), opaqueId()); + } else { + logger.info("{}[transport] [run_as_denied]\t{}, {}, roles=[{}], action=[{}], request=[{}]{}", localNodeInfo.prefix, + originAttributes(threadContext, message, localNodeInfo), runAsSubject(authentication), + arrayToCommaDelimitedString(roleNames), action, message.getClass().getSimpleName(), opaqueId()); + } + } + } + } + + @Override + public void runAsDenied(Authentication authentication, RestRequest request, String[] roleNames) { + if (events.contains(RUN_AS_DENIED) + && (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()), + Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), Optional.empty())) == false)) { + if (includeRequestBody) { + logger.info("{}[rest] [run_as_denied]\t{}, {}, roles=[{}], uri=[{}], request_body=[{}]{}", localNodeInfo.prefix, + hostAttributes(request), runAsSubject(authentication), arrayToCommaDelimitedString(roleNames), request.uri(), + restRequestContent(request), opaqueId()); + } else { + logger.info("{}[rest] [run_as_denied]\t{}, {}, roles=[{}], uri=[{}]{}", localNodeInfo.prefix, hostAttributes(request), + runAsSubject(authentication), arrayToCommaDelimitedString(roleNames), request.uri(), opaqueId()); + } + } + } + + static String runAsSubject(Authentication authentication) { + final StringBuilder sb = new StringBuilder("principal=["); + sb.append(authentication.getUser().authenticatedUser().principal()); + sb.append("], realm=["); + sb.append(authentication.getAuthenticatedBy().getName()); + sb.append("], run_as_principal=["); + sb.append(authentication.getUser().principal()); + if (authentication.getLookedUpBy() != null) { + sb.append("], run_as_realm=[").append(authentication.getLookedUpBy().getName()); + } + sb.append("]"); + return sb.toString(); + } + + static String subject(Authentication authentication) { + final StringBuilder sb = new StringBuilder("principal=["); + sb.append(authentication.getUser().principal()).append("], realm=["); + if (authentication.getUser().isRunAs()) { + sb.append(authentication.getLookedUpBy().getName()).append("], run_by_principal=["); + sb.append(authentication.getUser().authenticatedUser().principal()).append("], run_by_realm=["); + } + sb.append(authentication.getAuthenticatedBy().getName()).append("]"); + return sb.toString(); + } + + private static String hostAttributes(RestRequest request) { + String formattedAddress; + final SocketAddress socketAddress = request.getRemoteAddress(); + if (socketAddress instanceof InetSocketAddress) { + formattedAddress = NetworkAddress.format(((InetSocketAddress) socketAddress).getAddress()); + } else { + formattedAddress = socketAddress.toString(); + } + return "origin_address=[" + formattedAddress + "]"; + } + + protected static String originAttributes(ThreadContext threadContext, TransportMessage message, LocalNodeInfo localNodeInfo) { + return restOriginTag(threadContext).orElse(transportOriginTag(message).orElse(localNodeInfo.localOriginTag)); + } + + private String opaqueId() { + String opaqueId = threadContext.getHeader(Task.X_OPAQUE_ID); + if (opaqueId != null) { + return ", opaque_id=[" + opaqueId + "]"; + } else { + return ""; + } + } + + private static Optional restOriginTag(ThreadContext threadContext) { + final InetSocketAddress restAddress = RemoteHostHeader.restRemoteAddress(threadContext); + if (restAddress == null) { + return Optional.empty(); + } + return Optional.of(new StringBuilder("origin_type=[rest], origin_address=[").append(NetworkAddress.format(restAddress.getAddress())) + .append("]").toString()); + } + + private static Optional transportOriginTag(TransportMessage message) { + final TransportAddress address = message.remoteAddress(); + if (address == null) { + return Optional.empty(); + } + return Optional.of(new StringBuilder("origin_type=[transport], origin_address=[") + .append(NetworkAddress.format(address.address().getAddress())).append("]").toString()); + } + + static Optional indices(TransportMessage message) { + if (message instanceof IndicesRequest) { + final String[] indices = ((IndicesRequest) message).indices(); + if ((indices != null) && (indices.length != 0)) { + return Optional.of(((IndicesRequest) message).indices()); + } + } + return Optional.empty(); + } + + static String effectiveRealmName(Authentication authentication) { + return authentication.getLookedUpBy() != null ? authentication.getLookedUpBy().getName() + : authentication.getAuthenticatedBy().getName(); + } + + static String principal(User user) { + final StringBuilder builder = new StringBuilder("principal=["); + builder.append(user.principal()); + if (user.isRunAs()) { + builder.append("], run_by_principal=[").append(user.authenticatedUser().principal()); + } + return builder.append("]").toString(); + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + updateLocalNodeInfo(event.state().getNodes().getLocalNode()); + } + + void updateLocalNodeInfo(DiscoveryNode newLocalNode) { + // check if local node changed + final LocalNodeInfo localNodeInfo = this.localNodeInfo; + if ((localNodeInfo.localNode == null) || (localNodeInfo.localNode.equals(newLocalNode) == false)) { + // no need to synchronize, called only from the cluster state applier thread + this.localNodeInfo = new LocalNodeInfo(localNodeInfo.settings, newLocalNode); + } + } + + static class LocalNodeInfo { + private final Settings settings; + private final DiscoveryNode localNode; + final String prefix; + private final String localOriginTag; + + LocalNodeInfo(Settings settings, @Nullable DiscoveryNode newLocalNode) { + this.settings = settings; + this.localNode = newLocalNode; + this.prefix = resolvePrefix(settings, newLocalNode); + this.localOriginTag = localOriginTag(newLocalNode); + } + + static String resolvePrefix(Settings settings, @Nullable DiscoveryNode localNode) { + final StringBuilder builder = new StringBuilder(); + if (LoggingAuditTrail.EMIT_HOST_ADDRESS_SETTING.get(settings)) { + final String address = localNode != null ? localNode.getHostAddress() : null; + if (address != null) { + builder.append("[").append(address).append("] "); + } + } + if (LoggingAuditTrail.EMIT_HOST_NAME_SETTING.get(settings)) { + final String hostName = localNode != null ? localNode.getHostName() : null; + if (hostName != null) { + builder.append("[").append(hostName).append("] "); + } + } + if (LoggingAuditTrail.EMIT_NODE_NAME_SETTING.get(settings)) { + final String name = Node.NODE_NAME_SETTING.get(settings); + if (name != null) { + builder.append("[").append(name).append("] "); + } + } + return builder.toString(); + } + + private static String localOriginTag(@Nullable DiscoveryNode localNode) { + if (localNode == null) { + return "origin_type=[local_node]"; + } + return new StringBuilder("origin_type=[local_node], origin_address=[").append(localNode.getHostAddress()).append("]") + .toString(); + } + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index 358a4a0b12cb6..02b6825d6634c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -127,20 +127,20 @@ public class LoggingAuditTrail extends AbstractComponent implements AuditTrail, Collections.emptyList(), Function.identity(), Property.NodeScope, Property.Dynamic); public static final Setting INCLUDE_REQUEST_BODY = Setting.boolSetting(setting("audit.logfile.events.emit_request_body"), false, Property.NodeScope, Property.Dynamic); - private static final String FILTER_POLICY_PREFIX = setting("audit.logfile.events.ignore_filters."); + public static final String FILTER_POLICY_PREFIX = setting("audit.logfile.events.ignore_filters."); // because of the default wildcard value (*) for the field filter, a policy with // an unspecified filter field will match events that have any value for that // particular field, as well as events with that particular field missing - private static final Setting.AffixSetting> FILTER_POLICY_IGNORE_PRINCIPALS = Setting.affixKeySetting(FILTER_POLICY_PREFIX, + public static final Setting.AffixSetting> FILTER_POLICY_IGNORE_PRINCIPALS = Setting.affixKeySetting(FILTER_POLICY_PREFIX, "users", (key) -> Setting.listSetting(key, Collections.singletonList("*"), Function.identity(), Property.NodeScope, Property.Dynamic)); - private static final Setting.AffixSetting> FILTER_POLICY_IGNORE_REALMS = Setting.affixKeySetting(FILTER_POLICY_PREFIX, + public static final Setting.AffixSetting> FILTER_POLICY_IGNORE_REALMS = Setting.affixKeySetting(FILTER_POLICY_PREFIX, "realms", (key) -> Setting.listSetting(key, Collections.singletonList("*"), Function.identity(), Property.NodeScope, Property.Dynamic)); - private static final Setting.AffixSetting> FILTER_POLICY_IGNORE_ROLES = Setting.affixKeySetting(FILTER_POLICY_PREFIX, + public static final Setting.AffixSetting> FILTER_POLICY_IGNORE_ROLES = Setting.affixKeySetting(FILTER_POLICY_PREFIX, "roles", (key) -> Setting.listSetting(key, Collections.singletonList("*"), Function.identity(), Property.NodeScope, Property.Dynamic)); - private static final Setting.AffixSetting> FILTER_POLICY_IGNORE_INDICES = Setting.affixKeySetting(FILTER_POLICY_PREFIX, + public static final Setting.AffixSetting> FILTER_POLICY_IGNORE_INDICES = Setting.affixKeySetting(FILTER_POLICY_PREFIX, "indices", (key) -> Setting.listSetting(key, Collections.singletonList("*"), Function.identity(), Property.NodeScope, Property.Dynamic)); @@ -772,7 +772,7 @@ public static void registerSettings(List> settings) { * Predicates on each field are ANDed together to form the filter predicate of * the policy. */ - private static final class EventFilterPolicy { + static final class EventFilterPolicy { private final String name; private final Predicate ignorePrincipalsPredicate; private final Predicate ignoreRealmsPredicate; @@ -800,22 +800,22 @@ private static final class EventFilterPolicy { this.ignoreIndicesPredicate = ignoreIndicesPredicate; } - private EventFilterPolicy changePrincipalsFilter(List filtersList) { + EventFilterPolicy changePrincipalsFilter(List filtersList) { return new EventFilterPolicy(name, parsePredicate(filtersList), ignoreRealmsPredicate, ignoreRolesPredicate, ignoreIndicesPredicate); } - private EventFilterPolicy changeRealmsFilter(List filtersList) { + EventFilterPolicy changeRealmsFilter(List filtersList) { return new EventFilterPolicy(name, ignorePrincipalsPredicate, parsePredicate(filtersList), ignoreRolesPredicate, ignoreIndicesPredicate); } - private EventFilterPolicy changeRolesFilter(List filtersList) { + EventFilterPolicy changeRolesFilter(List filtersList) { return new EventFilterPolicy(name, ignorePrincipalsPredicate, ignoreRealmsPredicate, parsePredicate(filtersList), ignoreIndicesPredicate); } - private EventFilterPolicy changeIndicesFilter(List filtersList) { + EventFilterPolicy changeIndicesFilter(List filtersList) { return new EventFilterPolicy(name, ignorePrincipalsPredicate, ignoreRealmsPredicate, ignoreRolesPredicate, parsePredicate(filtersList)); } @@ -862,7 +862,7 @@ static final class EventFilterPolicyRegistry { private volatile Map policyMap; private volatile Predicate predicate; - private EventFilterPolicyRegistry(Settings settings) { + EventFilterPolicyRegistry(Settings settings) { final MapBuilder mapBuilder = MapBuilder.newMapBuilder(); for (final String policyName : settings.getGroups(FILTER_POLICY_PREFIX, true).keySet()) { mapBuilder.put(policyName, new EventFilterPolicy(policyName, settings)); @@ -872,11 +872,11 @@ private EventFilterPolicyRegistry(Settings settings) { predicate = buildIgnorePredicate(policyMap); } - private Optional get(String policyName) { + Optional get(String policyName) { return Optional.ofNullable(policyMap.get(policyName)); } - private synchronized void set(String policyName, EventFilterPolicy eventFilterPolicy) { + synchronized void set(String policyName, EventFilterPolicy eventFilterPolicy) { policyMap = MapBuilder.newMapBuilder(policyMap).put(policyName, eventFilterPolicy).immutableMap(); // precompute predicate predicate = buildIgnorePredicate(policyMap); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java index 7594e0d8c7753..c83a01bba6930 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java @@ -44,6 +44,7 @@ import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.security.audit.AuditTrailService; import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail; +import org.elasticsearch.xpack.security.audit.logfile.DeprecatedLoggingAuditTrail; import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail; import org.elasticsearch.xpack.security.authc.Realms; import org.junit.Before; @@ -65,6 +66,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_FORMAT_SETTING; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_INDEX_FORMAT; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; @@ -163,8 +165,7 @@ public void testAuditEnabled() throws Exception { Collection components = createComponents(settings); AuditTrailService service = findComponent(AuditTrailService.class, components); assertNotNull(service); - assertEquals(1, service.getAuditTrails().size()); - assertEquals(LoggingAuditTrail.NAME, service.getAuditTrails().get(0).name()); + assertThat(service.getAuditTrails(), containsInAnyOrder(LoggingAuditTrail.NAME, DeprecatedLoggingAuditTrail.NAME)); } public void testDisabledByDefault() throws Exception { @@ -180,8 +181,7 @@ public void testIndexAuditTrail() throws Exception { Collection components = createComponents(settings); AuditTrailService service = findComponent(AuditTrailService.class, components); assertNotNull(service); - assertEquals(1, service.getAuditTrails().size()); - assertEquals(IndexAuditTrail.NAME, service.getAuditTrails().get(0).name()); + assertThat(service.getAuditTrails(), containsInAnyOrder(IndexAuditTrail.NAME)); } public void testIndexAndLoggingAuditTrail() throws Exception { @@ -191,9 +191,8 @@ public void testIndexAndLoggingAuditTrail() throws Exception { Collection components = createComponents(settings); AuditTrailService service = findComponent(AuditTrailService.class, components); assertNotNull(service); - assertEquals(2, service.getAuditTrails().size()); - assertEquals(IndexAuditTrail.NAME, service.getAuditTrails().get(0).name()); - assertEquals(LoggingAuditTrail.NAME, service.getAuditTrails().get(1).name()); + assertThat(service.getAuditTrails(), + containsInAnyOrder(LoggingAuditTrail.NAME, DeprecatedLoggingAuditTrail.NAME, IndexAuditTrail.NAME)); } public void testUnknownOutput() { From d93333310fe3ce0e85d1150413f5dd305c7e4aac Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 5 Oct 2018 11:24:47 +0300 Subject: [PATCH 06/10] Cherry-picked Deprecated Unit tests --- .../audit/logfile/CapturingLogger.java | 1 - .../DeprecatedLoggingAuditTrailTests.java | 885 ++++++++++++++++++ 2 files changed, 885 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/DeprecatedLoggingAuditTrailTests.java diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/audit/logfile/CapturingLogger.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/audit/logfile/CapturingLogger.java index 1c86154ab7915..3ea15d48f2b4f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/audit/logfile/CapturingLogger.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/audit/logfile/CapturingLogger.java @@ -17,7 +17,6 @@ import org.apache.logging.log4j.core.config.LoggerConfig; import org.apache.logging.log4j.core.filter.RegexFilter; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.logging.Loggers; import java.util.ArrayList; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/DeprecatedLoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/DeprecatedLoggingAuditTrailTests.java new file mode 100644 index 0000000000000..3d5c398bc6daf --- /dev/null +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/DeprecatedLoggingAuditTrailTests.java @@ -0,0 +1,885 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.security.audit.logfile; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.mock.orig.Mockito; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.FakeRestRequest; +import org.elasticsearch.test.rest.FakeRestRequest.Builder; +import org.elasticsearch.transport.TransportMessage; +import org.elasticsearch.xpack.core.security.audit.logfile.CapturingLogger; +import org.elasticsearch.xpack.core.security.authc.Authentication; +import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; +import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; +import org.elasticsearch.xpack.core.security.user.SystemUser; +import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.xpack.security.rest.RemoteHostHeader; +import org.elasticsearch.xpack.security.transport.filter.IPFilter; +import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule; +import org.junit.Before; +import org.mockito.stubbing.Answer; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.containsString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class DeprecatedLoggingAuditTrailTests extends ESTestCase { + + enum RestContent { + VALID() { + @Override + protected boolean hasContent() { + return true; + } + + @Override + protected BytesReference content() { + return new BytesArray("{ \"key\": \"value\" }"); + } + + @Override + protected String expectedMessage() { + return "{ \"key\": \"value\" }"; + } + }, + INVALID() { + @Override + protected boolean hasContent() { + return true; + } + + @Override + protected BytesReference content() { + return new BytesArray("{ \"key\": \"value\" "); + } + + @Override + protected String expectedMessage() { + return "{ \"key\": \"value\" "; + } + }, + EMPTY() { + @Override + protected boolean hasContent() { + return false; + } + + @Override + protected BytesReference content() { + throw new RuntimeException("should never be called"); + } + + @Override + protected String expectedMessage() { + return ""; + } + }; + + protected abstract boolean hasContent(); + protected abstract BytesReference content(); + protected abstract String expectedMessage(); + } + + private String prefix; + private Settings settings; + private DiscoveryNode localNode; + private ClusterService clusterService; + private ThreadContext threadContext; + private boolean includeRequestBody; + private String opaqueId; + + @Before + public void init() throws Exception { + includeRequestBody = randomBoolean(); + settings = Settings.builder() + .put("xpack.security.audit.logfile.prefix.emit_node_host_address", randomBoolean()) + .put("xpack.security.audit.logfile.prefix.emit_node_host_name", randomBoolean()) + .put("xpack.security.audit.logfile.prefix.emit_node_name", randomBoolean()) + .put("xpack.security.audit.logfile.events.emit_request_body", includeRequestBody) + .build(); + localNode = mock(DiscoveryNode.class); + when(localNode.getHostAddress()).thenReturn(buildNewFakeTransportAddress().toString()); + clusterService = mock(ClusterService.class); + when(clusterService.localNode()).thenReturn(localNode); + Mockito.doAnswer((Answer) invocation -> { + final DeprecatedLoggingAuditTrail arg0 = (DeprecatedLoggingAuditTrail) invocation.getArguments()[0]; + arg0.updateLocalNodeInfo(localNode); + return null; + }).when(clusterService).addListener(Mockito.isA(DeprecatedLoggingAuditTrail.class)); + final ClusterSettings clusterSettings = mockClusterSettings(); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + prefix = DeprecatedLoggingAuditTrail.LocalNodeInfo.resolvePrefix(settings, localNode); + threadContext = new ThreadContext(Settings.EMPTY); + if (randomBoolean()) { + String id = randomAlphaOfLength(10); + threadContext.putHeader(Task.X_OPAQUE_ID, id); + opaqueId = ", opaque_id=[" + id + "]"; + } else { + opaqueId = ""; + } + } + + public void testAnonymousAccessDeniedTransport() throws Exception { + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); + final String origins = DeprecatedLoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); + auditTrail.anonymousAccessDenied("_action", message); + if (message instanceof IndicesRequest) { + assertMsg(logger, Level.INFO, prefix + "[transport] [anonymous_access_denied]\t" + origins + + ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); + } else { + assertMsg(logger, Level.INFO, prefix + "[transport] [anonymous_access_denied]\t" + origins + + ", action=[_action], request=[MockMessage]" + opaqueId); + } + + // test disabled + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "anonymous_access_denied").build(); + auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.anonymousAccessDenied("_action", message); + assertEmptyLog(logger); + } + + public void testAnonymousAccessDeniedRest() throws Exception { + final InetAddress address = forge("_hostname", randomBoolean() ? "127.0.0.1" : "::1"); + final Tuple tuple = prepareRestContent("_uri", new InetSocketAddress(address, 9200)); + final String expectedMessage = tuple.v1().expectedMessage(); + final RestRequest request = tuple.v2(); + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.anonymousAccessDenied(request); + if (includeRequestBody) { + assertMsg(logger, Level.INFO, prefix + "[rest] [anonymous_access_denied]\torigin_address=[" + + NetworkAddress.format(address) + "], uri=[_uri]" + opaqueId + ", request_body=[" + expectedMessage + "]"); + } else { + assertMsg(logger, Level.INFO, prefix + "[rest] [anonymous_access_denied]\torigin_address=[" + + NetworkAddress.format(address) + "], uri=[_uri]" + opaqueId); + } + + // test disabled + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "anonymous_access_denied").build(); + auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.anonymousAccessDenied(request); + assertEmptyLog(logger); + } + + public void testAuthenticationFailed() throws Exception { + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); + final String origins = DeprecatedLoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); + auditTrail.authenticationFailed(new MockToken(), "_action", message); + if (message instanceof IndicesRequest) { + assertMsg(logger, Level.INFO, prefix + "[transport] [authentication_failed]\t" + origins + + ", principal=[_principal], action=[_action], indices=[" + indices(message) + + "], request=[MockIndicesRequest]" + opaqueId); + } else { + assertMsg(logger, Level.INFO, prefix + "[transport] [authentication_failed]\t" + origins + + ", principal=[_principal], action=[_action], request=[MockMessage]" + opaqueId); + } + + // test disabled + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "authentication_failed").build(); + auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.authenticationFailed(new MockToken(), "_action", message); + assertEmptyLog(logger); + } + + public void testAuthenticationFailedNoToken() throws Exception { + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); + final String origins = DeprecatedLoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); + auditTrail.authenticationFailed("_action", message); + if (message instanceof IndicesRequest) { + assertMsg(logger, Level.INFO, prefix + "[transport] [authentication_failed]\t" + origins + + ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); + } else { + assertMsg(logger, Level.INFO, prefix + "[transport] [authentication_failed]\t" + origins + + ", action=[_action], request=[MockMessage]" + opaqueId); + } + + // test disabled + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "authentication_failed").build(); + auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.authenticationFailed("_action", message); + assertEmptyLog(logger); + } + + public void testAuthenticationFailedRest() throws Exception { + final InetAddress address = forge("_hostname", randomBoolean() ? "127.0.0.1" : "::1"); + final Tuple tuple = prepareRestContent("_uri", new InetSocketAddress(address, 9200)); + final String expectedMessage = tuple.v1().expectedMessage(); + final RestRequest request = tuple.v2(); + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.authenticationFailed(new MockToken(), request); + if (includeRequestBody) { + assertMsg(logger, Level.INFO, prefix + "[rest] [authentication_failed]\torigin_address=[" + + NetworkAddress.format(address) + "], principal=[_principal], uri=[_uri]" + opaqueId + ", request_body=[" + + expectedMessage + "]"); + } else { + assertMsg(logger, Level.INFO, prefix + "[rest] [authentication_failed]\torigin_address=[" + + NetworkAddress.format(address) + "], principal=[_principal], uri=[_uri]" + opaqueId); + } + + // test disabled + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "authentication_failed").build(); + auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.authenticationFailed(new MockToken(), request); + assertEmptyLog(logger); + } + + public void testAuthenticationFailedRestNoToken() throws Exception { + final InetAddress address = forge("_hostname", randomBoolean() ? "127.0.0.1" : "::1"); + final Tuple tuple = prepareRestContent("_uri", new InetSocketAddress(address, 9200)); + final String expectedMessage = tuple.v1().expectedMessage(); + final RestRequest request = tuple.v2(); + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.authenticationFailed(request); + if (includeRequestBody) { + assertMsg(logger, Level.INFO, prefix + "[rest] [authentication_failed]\torigin_address=[" + + NetworkAddress.format(address) + "], uri=[_uri]" + opaqueId + ", request_body=[" + expectedMessage + "]"); + } else { + assertMsg(logger, Level.INFO, prefix + "[rest] [authentication_failed]\torigin_address=[" + + NetworkAddress.format(address) + "], uri=[_uri]" + opaqueId); + } + + // test disabled + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "authentication_failed").build(); + auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.authenticationFailed(request); + assertEmptyLog(logger); + } + + public void testAuthenticationFailedRealm() throws Exception { + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); + auditTrail.authenticationFailed("_realm", new MockToken(), "_action", message); + assertEmptyLog(logger); + + // test enabled + settings = + Settings.builder().put(settings).put("xpack.security.audit.logfile.events.include", "realm_authentication_failed").build(); + auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + final String origins = DeprecatedLoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); + auditTrail.authenticationFailed("_realm", new MockToken(), "_action", message); + if (message instanceof IndicesRequest) { + assertMsg(logger, Level.INFO, prefix + "[transport] [realm_authentication_failed]\trealm=[_realm], " + origins + + ", principal=[_principal], action=[_action], indices=[" + indices(message) + "], " + + "request=[MockIndicesRequest]" + opaqueId); + } else { + assertMsg(logger, Level.INFO, prefix + "[transport] [realm_authentication_failed]\trealm=[_realm], " + origins + + ", principal=[_principal], action=[_action], request=[MockMessage]" + opaqueId); + } + } + + public void testAuthenticationFailedRealmRest() throws Exception { + final InetAddress address = forge("_hostname", randomBoolean() ? "127.0.0.1" : "::1"); + final Tuple tuple = prepareRestContent("_uri", new InetSocketAddress(address, 9200)); + final String expectedMessage = tuple.v1().expectedMessage(); + final RestRequest request = tuple.v2(); + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.authenticationFailed("_realm", new MockToken(), request); + assertEmptyLog(logger); + + // test enabled + settings = + Settings.builder().put(settings).put("xpack.security.audit.logfile.events.include", "realm_authentication_failed").build(); + auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.authenticationFailed("_realm", new MockToken(), request); + if (includeRequestBody) { + assertMsg(logger, Level.INFO, prefix + "[rest] [realm_authentication_failed]\trealm=[_realm], origin_address=[" + + NetworkAddress.format(address) + "], principal=[_principal], uri=[_uri]" + opaqueId + ", request_body=[" + + expectedMessage + "]"); + } else { + assertMsg(logger, Level.INFO, prefix + "[rest] [realm_authentication_failed]\trealm=[_realm], origin_address=[" + + NetworkAddress.format(address) + "], principal=[_principal], uri=[_uri]" + opaqueId); + } + } + + public void testAccessGranted() throws Exception { + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); + final String origins = DeprecatedLoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); + final boolean runAs = randomBoolean(); + User user; + if (runAs) { + user = new User("running as", new String[]{"r2"}, new User("_username", new String[] {"r1"})); + } else { + user = new User("_username", new String[]{"r1"}); + } + final String role = randomAlphaOfLengthBetween(1, 6); + auditTrail.accessGranted(createAuthentication(user), "_action", message, new String[] { role }); + final String userInfo = (runAs ? "principal=[running as], realm=[lookRealm], run_by_principal=[_username], run_by_realm=[authRealm]" + : "principal=[_username], realm=[authRealm]") + ", roles=[" + role + "]"; + if (message instanceof IndicesRequest) { + assertMsg(logger, Level.INFO, prefix + "[transport] [access_granted]\t" + origins + ", " + userInfo + + ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); + } else { + assertMsg(logger, Level.INFO, prefix + "[transport] [access_granted]\t" + origins + ", " + userInfo + + ", action=[_action], request=[MockMessage]" + opaqueId); + } + + // test disabled + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "access_granted").build(); + auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.accessGranted(createAuthentication(user), "_action", message, new String[] { role }); + assertEmptyLog(logger); + } + + public void testAccessGrantedInternalSystemAction() throws Exception { + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); + final String role = randomAlphaOfLengthBetween(1, 6); + auditTrail.accessGranted(createAuthentication(SystemUser.INSTANCE), "internal:_action", message, new String[] { role }); + assertEmptyLog(logger); + + // test enabled + settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.include", "system_access_granted").build(); + auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + final String origins = DeprecatedLoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); + auditTrail.accessGranted(createAuthentication(SystemUser.INSTANCE), "internal:_action", message, new String[] { role }); + if (message instanceof IndicesRequest) { + assertMsg(logger, Level.INFO, prefix + "[transport] [access_granted]\t" + origins + ", principal=[" + + SystemUser.INSTANCE.principal() + + "], realm=[authRealm], roles=[" + role + "], action=[internal:_action], indices=[" + indices(message) + + "], request=[MockIndicesRequest]" + opaqueId); + } else { + assertMsg(logger, Level.INFO, prefix + "[transport] [access_granted]\t" + origins + ", principal=[" + + SystemUser.INSTANCE.principal() + "], realm=[authRealm], roles=[" + role + + "], action=[internal:_action], request=[MockMessage]" + opaqueId); + } + } + + public void testAccessGrantedInternalSystemActionNonSystemUser() throws Exception { + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); + final String origins = DeprecatedLoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); + final boolean runAs = randomBoolean(); + User user; + if (runAs) { + user = new User("running as", new String[]{"r2"}, new User("_username", new String[] {"r1"})); + } else { + user = new User("_username", new String[]{"r1"}); + } + final String role = randomAlphaOfLengthBetween(1, 6); + auditTrail.accessGranted(createAuthentication(user), "internal:_action", message, new String[] { role }); + final String userInfo = (runAs ? "principal=[running as], realm=[lookRealm], run_by_principal=[_username], run_by_realm=[authRealm]" + : "principal=[_username], realm=[authRealm]") + ", roles=[" + role + "]"; + if (message instanceof IndicesRequest) { + assertMsg(logger, Level.INFO, prefix + "[transport] [access_granted]\t" + origins + ", " + userInfo + + ", action=[internal:_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); + } else { + assertMsg(logger, Level.INFO, prefix + "[transport] [access_granted]\t" + origins + ", " + userInfo + + ", action=[internal:_action], request=[MockMessage]" + opaqueId); + } + + // test disabled + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "access_granted").build(); + auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.accessGranted(createAuthentication(user), "internal:_action", message, new String[] { role }); + assertEmptyLog(logger); + } + + public void testAccessDenied() throws Exception { + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); + final String origins = DeprecatedLoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); + final boolean runAs = randomBoolean(); + User user; + if (runAs) { + user = new User("running as", new String[]{"r2"}, new User("_username", new String[] {"r1"})); + } else { + user = new User("_username", new String[]{"r1"}); + } + final String role = randomAlphaOfLengthBetween(1, 6); + auditTrail.accessDenied(createAuthentication(user), "_action", message, new String[] { role }); + final String userInfo = (runAs ? "principal=[running as], realm=[lookRealm], run_by_principal=[_username], run_by_realm=[authRealm]" + : "principal=[_username], realm=[authRealm]") + ", roles=[" + role + "]"; + if (message instanceof IndicesRequest) { + assertMsg(logger, Level.INFO, prefix + "[transport] [access_denied]\t" + origins + ", " + userInfo + + ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); + } else { + assertMsg(logger, Level.INFO, prefix + "[transport] [access_denied]\t" + origins + ", " + userInfo + + ", action=[_action], request=[MockMessage]" + opaqueId); + } + + // test disabled + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "access_denied").build(); + auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.accessDenied(createAuthentication(user), "_action", message, new String[] { role }); + assertEmptyLog(logger); + } + + public void testTamperedRequestRest() throws Exception { + final InetAddress address = forge("_hostname", randomBoolean() ? "127.0.0.1" : "::1"); + final Tuple tuple = prepareRestContent("_uri", new InetSocketAddress(address, 9200)); + final String expectedMessage = tuple.v1().expectedMessage(); + final RestRequest request = tuple.v2(); + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.tamperedRequest(request); + if (includeRequestBody) { + assertMsg(logger, Level.INFO, prefix + "[rest] [tampered_request]\torigin_address=[" + + NetworkAddress.format(address) + "], uri=[_uri]" + opaqueId + ", request_body=[" + expectedMessage + "]"); + } else { + assertMsg(logger, Level.INFO, prefix + "[rest] [tampered_request]\torigin_address=[" + + NetworkAddress.format(address) + "], uri=[_uri]" + opaqueId); + } + + // test disabled + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "tampered_request").build(); + auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.tamperedRequest(request); + assertEmptyLog(logger); + } + + public void testTamperedRequest() throws Exception { + final String action = "_action"; + final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + final DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + final String origins = DeprecatedLoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); + auditTrail.tamperedRequest(action, message); + if (message instanceof IndicesRequest) { + assertMsg(logger, Level.INFO, prefix + "[transport] [tampered_request]\t" + origins + + ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); + } else { + assertMsg(logger, Level.INFO, prefix + "[transport] [tampered_request]\t" + origins + + ", action=[_action], request=[MockMessage]" + opaqueId); + } + + // test disabled + + } + + public void testTamperedRequestWithUser() throws Exception { + final String action = "_action"; + final boolean runAs = randomBoolean(); + User user; + if (runAs) { + user = new User("running as", new String[]{"r2"}, new User("_username", new String[] {"r1"})); + } else { + user = new User("_username", new String[]{"r1"}); + } + final String userInfo = runAs ? "principal=[running as], run_by_principal=[_username]" : "principal=[_username]"; + final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + final String origins = DeprecatedLoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); + auditTrail.tamperedRequest(user, action, message); + if (message instanceof IndicesRequest) { + assertMsg(logger, Level.INFO, prefix + "[transport] [tampered_request]\t" + origins + ", " + userInfo + + ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); + } else { + assertMsg(logger, Level.INFO, prefix + "[transport] [tampered_request]\t" + origins + ", " + userInfo + + ", action=[_action], request=[MockMessage]" + opaqueId); + } + + // test disabled + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "tampered_request").build(); + auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.tamperedRequest(user, action, message); + assertEmptyLog(logger); + } + + public void testConnectionDenied() throws Exception { + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + final InetAddress inetAddress = InetAddress.getLoopbackAddress(); + final SecurityIpFilterRule rule = new SecurityIpFilterRule(false, "_all"); + auditTrail.connectionDenied(inetAddress, "default", rule); + assertMsg(logger, Level.INFO, String.format(Locale.ROOT, prefix + + "[ip_filter] [connection_denied]\torigin_address=[%s], transport_profile=[%s], rule=[deny %s]" + opaqueId, + NetworkAddress.format(inetAddress), "default", "_all")); + + // test disabled + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "connection_denied").build(); + auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.connectionDenied(inetAddress, "default", rule); + assertEmptyLog(logger); + } + + public void testConnectionGranted() throws Exception { + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + final InetAddress inetAddress = InetAddress.getLoopbackAddress(); + final SecurityIpFilterRule rule = IPFilter.DEFAULT_PROFILE_ACCEPT_ALL; + auditTrail.connectionGranted(inetAddress, "default", rule); + assertEmptyLog(logger); + + // test enabled + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.include", "connection_granted").build(); + auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.connectionGranted(inetAddress, "default", rule); + assertMsg(logger, Level.INFO, String.format(Locale.ROOT, prefix + "[ip_filter] [connection_granted]\torigin_address=[%s], " + + "transport_profile=[default], rule=[allow default:accept_all]" + opaqueId, + NetworkAddress.format(inetAddress))); + } + + public void testRunAsGranted() throws Exception { + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); + final String origins = DeprecatedLoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); + final User user = new User("running as", new String[]{"r2"}, new User("_username", new String[] {"r1"})); + final String role = randomAlphaOfLengthBetween(1, 6); + auditTrail.runAsGranted(createAuthentication(user), "_action", message, new String[] { role }); + if (message instanceof IndicesRequest) { + assertMsg(logger, Level.INFO, + prefix + "[transport] [run_as_granted]\t" + origins + + ", principal=[_username], realm=[authRealm], run_as_principal=[running as], run_as_realm=[lookRealm], roles=[" + + role + "], action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); + } else { + assertMsg(logger, Level.INFO, + prefix + "[transport] [run_as_granted]\t" + origins + + ", principal=[_username], realm=[authRealm], run_as_principal=[running as], run_as_realm=[lookRealm], roles=[" + + role + "], action=[_action], request=[MockMessage]" + opaqueId); + } + + // test disabled + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "run_as_granted").build(); + auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.runAsGranted(createAuthentication(user), "_action", message, new String[] { role }); + assertEmptyLog(logger); + } + + public void testRunAsDenied() throws Exception { + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); + final String origins = DeprecatedLoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); + final User user = new User("running as", new String[]{"r2"}, new User("_username", new String[] {"r1"})); + final String role = randomAlphaOfLengthBetween(1, 6); + auditTrail.runAsDenied(createAuthentication(user), "_action", message, new String[] { role }); + if (message instanceof IndicesRequest) { + assertMsg(logger, Level.INFO, + prefix + "[transport] [run_as_denied]\t" + origins + + ", principal=[_username], realm=[authRealm], run_as_principal=[running as], run_as_realm=[lookRealm], roles=[" + + role + "], action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); + } else { + assertMsg(logger, Level.INFO, + prefix + "[transport] [run_as_denied]\t" + origins + + ", principal=[_username], realm=[authRealm], run_as_principal=[running as], run_as_realm=[lookRealm], roles=[" + + role + "], action=[_action], request=[MockMessage]" + opaqueId); + } + + // test disabled + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + settings = Settings.builder().put(settings).put("xpack.security.audit.logfile.events.exclude", "run_as_denied").build(); + auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.runAsDenied(createAuthentication(user), "_action", message, new String[] { role }); + assertEmptyLog(logger); + } + + public void testOriginAttributes() throws Exception { + final MockMessage message = new MockMessage(threadContext); + final DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + final String text = DeprecatedLoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); + final InetSocketAddress restAddress = RemoteHostHeader.restRemoteAddress(threadContext); + if (restAddress != null) { + assertThat(text, equalTo("origin_type=[rest], origin_address=[" + + NetworkAddress.format(restAddress.getAddress()) + "]")); + return; + } + final TransportAddress address = message.remoteAddress(); + if (address == null) { + assertThat(text, equalTo("origin_type=[local_node], origin_address=[" + localNode.getHostAddress() + "]")); + return; + } + + assertThat(text, equalTo("origin_type=[transport], origin_address=[" + + NetworkAddress.format(address.address().getAddress()) + "]")); + } + + public void testAuthenticationSuccessRest() throws Exception { + final Map params = new HashMap<>(); + params.put("foo", "bar"); + final InetAddress address = forge("_hostname", randomBoolean() ? "127.0.0.1" : "::1"); + final Tuple tuple = prepareRestContent("_uri", new InetSocketAddress(address, 9200), params); + final String expectedMessage = tuple.v1().expectedMessage(); + final RestRequest request = tuple.v2(); + final boolean runAs = randomBoolean(); + User user; + if (runAs) { + user = new User("running as", new String[]{"r2"}, new User("_username", new String[] {"r1"})); + } else { + user = new User("_username", new String[] { "r1" }); + } + final String userInfo = runAs ? "principal=[running as], run_by_principal=[_username]" : "principal=[_username]"; + final String realm = "_realm"; + + Settings settings = Settings.builder().put(this.settings) + .put("xpack.security.audit.logfile.events.include", "authentication_success") + .build(); + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.authenticationSuccess(realm, user, request); + if (includeRequestBody) { + assertMsg(logger, Level.INFO, + prefix + "[rest] [authentication_success]\t" + userInfo + ", realm=[_realm], uri=[_uri], params=[" + params + + "]" + opaqueId + ", request_body=[" + expectedMessage + "]"); + } else { + assertMsg(logger, Level.INFO, + prefix + "[rest] [authentication_success]\t" + userInfo + ", realm=[_realm], uri=[_uri], params=[" + params + + "]" + opaqueId); + } + + // test disabled + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + settings = Settings.builder().put(this.settings).put("xpack.security.audit.logfile.events.exclude", "authentication_success") + .build(); + auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.authenticationSuccess(realm, user, request); + assertEmptyLog(logger); + } + + public void testAuthenticationSuccessTransport() throws Exception { + Settings settings = Settings.builder().put(this.settings) + .put("xpack.security.audit.logfile.events.include", "authentication_success").build(); + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext); + final String origins = DeprecatedLoggingAuditTrail.originAttributes(threadContext, message, auditTrail.localNodeInfo); + final boolean runAs = randomBoolean(); + User user; + if (runAs) { + user = new User("running as", new String[]{"r2"}, new User("_username", new String[] {"r1"})); + } else { + user = new User("_username", new String[] { "r1" }); + } + final String userInfo = runAs ? "principal=[running as], run_by_principal=[_username]" : "principal=[_username]"; + final String realm = "_realm"; + auditTrail.authenticationSuccess(realm, user, "_action", message); + if (message instanceof IndicesRequest) { + assertMsg(logger, Level.INFO, prefix + "[transport] [authentication_success]\t" + origins + ", " + userInfo + + ", realm=[_realm], action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); + } else { + assertMsg(logger, Level.INFO, prefix + "[transport] [authentication_success]\t" + origins + ", " + userInfo + + ", realm=[_realm], action=[_action], request=[MockMessage]" + opaqueId); + } + + // test disabled + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + settings = Settings.builder() + .put(this.settings) + .put("xpack.security.audit.logfile.events.exclude", "authentication_success") + .build(); + auditTrail = new DeprecatedLoggingAuditTrail(settings, clusterService, logger, threadContext); + auditTrail.authenticationSuccess(realm, user, "_action", message); + assertEmptyLog(logger); + } + + public void testRequestsWithoutIndices() throws Exception { + final Logger logger = CapturingLogger.newCapturingLogger(Level.INFO, null); + final Settings allEventsSettings = Settings.builder() + .put(settings) + .put("xpack.security.audit.logfile.events.include", "_all") + .build(); + final DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(allEventsSettings, clusterService, logger, threadContext); + final User user = new User("_username", new String[] { "r1" }); + final String role = randomAlphaOfLengthBetween(1, 6); + final String realm = randomAlphaOfLengthBetween(1, 6); + // transport messages without indices + final TransportMessage[] messages = new TransportMessage[] { new MockMessage(threadContext), + new org.elasticsearch.action.MockIndicesRequest(IndicesOptions.strictExpandOpenAndForbidClosed(), new String[0]), + new org.elasticsearch.action.MockIndicesRequest(IndicesOptions.strictExpandOpenAndForbidClosed(), (String[]) null) }; + final List output = CapturingLogger.output(logger.getName(), Level.INFO); + int logEntriesCount = 1; + for (final TransportMessage message : messages) { + auditTrail.anonymousAccessDenied("_action", message); + assertThat(output.size(), is(logEntriesCount++)); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); + auditTrail.authenticationFailed(new MockToken(), "_action", message); + assertThat(output.size(), is(logEntriesCount++)); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); + auditTrail.authenticationFailed("_action", message); + assertThat(output.size(), is(logEntriesCount++)); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); + auditTrail.authenticationFailed(realm, new MockToken(), "_action", message); + assertThat(output.size(), is(logEntriesCount++)); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); + auditTrail.accessGranted(createAuthentication(user), "_action", message, new String[] { role }); + assertThat(output.size(), is(logEntriesCount++)); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); + auditTrail.accessDenied(createAuthentication(user), "_action", message, new String[] { role }); + assertThat(output.size(), is(logEntriesCount++)); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); + auditTrail.tamperedRequest("_action", message); + assertThat(output.size(), is(logEntriesCount++)); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); + auditTrail.tamperedRequest(user, "_action", message); + assertThat(output.size(), is(logEntriesCount++)); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); + auditTrail.runAsGranted(createAuthentication(user), "_action", message, new String[] { role }); + assertThat(output.size(), is(logEntriesCount++)); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); + auditTrail.runAsDenied(createAuthentication(user), "_action", message, new String[] { role }); + assertThat(output.size(), is(logEntriesCount++)); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); + auditTrail.authenticationSuccess(realm, user, "_action", message); + assertThat(output.size(), is(logEntriesCount++)); + assertThat(output.get(logEntriesCount - 2), not(containsString("indices=["))); + } + } + + private void assertMsg(Logger logger, Level level, String message) { + final List output = CapturingLogger.output(logger.getName(), level); + assertThat(output.size(), is(1)); + assertThat(output.get(0), equalTo(message)); + } + + private void assertEmptyLog(Logger logger) { + assertThat(CapturingLogger.isEmpty(logger.getName()), is(true)); + } + + protected Tuple prepareRestContent(String uri, InetSocketAddress remoteAddress) { + return prepareRestContent(uri, remoteAddress, Collections.emptyMap()); + } + + private Tuple prepareRestContent(String uri, InetSocketAddress remoteAddress, Map params) { + final RestContent content = randomFrom(RestContent.values()); + final FakeRestRequest.Builder builder = new Builder(NamedXContentRegistry.EMPTY); + if (content.hasContent()) { + builder.withContent(content.content(), XContentType.JSON); + } + builder.withPath(uri); + builder.withRemoteAddress(remoteAddress); + builder.withParams(params); + return new Tuple<>(content, builder.build()); + } + + /** creates address without any lookups. hostname can be null, for missing */ + protected static InetAddress forge(String hostname, String address) throws IOException { + final byte bytes[] = InetAddress.getByName(address).getAddress(); + return InetAddress.getByAddress(hostname, bytes); + } + + private static String indices(TransportMessage message) { + return Strings.arrayToCommaDelimitedString(((IndicesRequest) message).indices()); + } + + private static Authentication createAuthentication(User user) { + final RealmRef lookedUpBy = user.authenticatedUser() == user ? null : new RealmRef("lookRealm", "up", "by"); + return new Authentication(user, new RealmRef("authRealm", "test", "foo"), lookedUpBy); + } + + private ClusterSettings mockClusterSettings() { + final List> settingsList = new ArrayList<>(); + LoggingAuditTrail.registerSettings(settingsList); + settingsList.addAll(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + return new ClusterSettings(settings, new HashSet<>(settingsList)); + } + + static class MockMessage extends TransportMessage { + + MockMessage(ThreadContext threadContext) throws IOException { + if (randomBoolean()) { + if (randomBoolean()) { + remoteAddress(buildNewFakeTransportAddress()); + } else { + remoteAddress(new TransportAddress(InetAddress.getLoopbackAddress(), 1234)); + } + } + if (randomBoolean()) { + RemoteHostHeader.putRestRemoteAddress(threadContext, new InetSocketAddress(forge("localhost", "127.0.0.1"), 1234)); + } + } + } + + static class MockIndicesRequest extends org.elasticsearch.action.MockIndicesRequest { + + MockIndicesRequest(ThreadContext threadContext) throws IOException { + super(IndicesOptions.strictExpandOpenAndForbidClosed(), "idx1", "idx2"); + if (randomBoolean()) { + remoteAddress(buildNewFakeTransportAddress()); + } + if (randomBoolean()) { + RemoteHostHeader.putRestRemoteAddress(threadContext, new InetSocketAddress(forge("localhost", "127.0.0.1"), 1234)); + } + } + + @Override + public String toString() { + return "mock-message"; + } + } + + private static class MockToken implements AuthenticationToken { + @Override + public String principal() { + return "_principal"; + } + + @Override + public Object credentials() { + fail("it's not allowed to print the credentials of the auth token"); + return null; + } + + @Override + public void clearCredentials() { + + } + } + +} From c81b362c86a531fa7db9ee4b018e37be1375fa1f Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 5 Oct 2018 11:36:30 +0300 Subject: [PATCH 07/10] Log4j2properties ... sweeet! --- .../core/src/main/config/log4j2.properties | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/x-pack/plugin/core/src/main/config/log4j2.properties b/x-pack/plugin/core/src/main/config/log4j2.properties index 1c9358f3cc490..4f4bd19060cd4 100644 --- a/x-pack/plugin/core/src/main/config/log4j2.properties +++ b/x-pack/plugin/core/src/main/config/log4j2.properties @@ -64,11 +64,27 @@ appender.audit_rolling.policies.time.type = TimeBasedTriggeringPolicy appender.audit_rolling.policies.time.interval = 1 appender.audit_rolling.policies.time.modulate = true +appender.deprecated_audit_rolling.type = RollingFile +appender.deprecated_audit_rolling.name = deprecated_audit_rolling +appender.deprecated_audit_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_access.log +appender.deprecated_audit_rolling.layout.type = PatternLayout +appender.deprecated_audit_rolling.layout.pattern = [%d{ISO8601}] %m%n +appender.deprecated_audit_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_access-%d{yyyy-MM-dd}.log +appender.deprecated_audit_rolling.policies.type = Policies +appender.deprecated_audit_rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.deprecated_audit_rolling.policies.time.interval = 1 +appender.deprecated_audit_rolling.policies.time.modulate = true + logger.xpack_security_audit_logfile.name = org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail logger.xpack_security_audit_logfile.level = info logger.xpack_security_audit_logfile.appenderRef.audit_rolling.ref = audit_rolling logger.xpack_security_audit_logfile.additivity = false +logger.xpack_security_audit_deprecated_logfile.name = org.elasticsearch.xpack.security.audit.logfile.DeprecatedLoggingAuditTrail +logger.xpack_security_audit_deprecated_logfile.level = info +logger.xpack_security_audit_deprecated_logfile.appenderRef.deprecated_audit_rolling.ref = deprecated_audit_rolling +logger.xpack_security_audit_deprecated_logfile.additivity = false + logger.xmlsig.name = org.apache.xml.security.signature.XMLSignature logger.xmlsig.level = error logger.samlxml_decrypt.name = org.opensaml.xmlsec.encryption.support.Decrypter From 29e2aeac8e9e05bd190a3a424bbbdb4a55cd8f27 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Mon, 15 Oct 2018 14:19:00 +0300 Subject: [PATCH 08/10] Checkstyle --- .../audit/logfile/DeprecatedLoggingAuditTrailTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/DeprecatedLoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/DeprecatedLoggingAuditTrailTests.java index 3d5c398bc6daf..4f7c0a34bb723 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/DeprecatedLoggingAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/DeprecatedLoggingAuditTrailTests.java @@ -735,7 +735,8 @@ public void testRequestsWithoutIndices() throws Exception { .put(settings) .put("xpack.security.audit.logfile.events.include", "_all") .build(); - final DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(allEventsSettings, clusterService, logger, threadContext); + final DeprecatedLoggingAuditTrail auditTrail = new DeprecatedLoggingAuditTrail(allEventsSettings, clusterService, + logger, threadContext); final User user = new User("_username", new String[] { "r1" }); final String role = randomAlphaOfLengthBetween(1, 6); final String realm = randomAlphaOfLengthBetween(1, 6); From b49f5042f49f40b1dbdd1b08069db65e65ad5e4a Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Mon, 15 Oct 2018 18:41:18 +0300 Subject: [PATCH 09/10] Fix mistery tests --- .../org/elasticsearch/xpack/security/SecurityTests.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java index 310e5b9b94a1d..6a82b47710a45 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java @@ -65,6 +65,7 @@ import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Predicate; +import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_FORMAT_SETTING; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_INDEX_FORMAT; @@ -169,7 +170,8 @@ public void testAuditEnabled() throws Exception { Collection components = createComponents(settings); AuditTrailService service = findComponent(AuditTrailService.class, components); assertNotNull(service); - assertThat(service.getAuditTrails(), containsInAnyOrder(LoggingAuditTrail.NAME, DeprecatedLoggingAuditTrail.NAME)); + assertThat(service.getAuditTrails().stream().map(x -> x.name()).collect(Collectors.toList()), + containsInAnyOrder(LoggingAuditTrail.NAME, DeprecatedLoggingAuditTrail.NAME)); } public void testDisabledByDefault() throws Exception { @@ -185,7 +187,8 @@ public void testIndexAuditTrail() throws Exception { Collection components = createComponents(settings); AuditTrailService service = findComponent(AuditTrailService.class, components); assertNotNull(service); - assertThat(service.getAuditTrails(), containsInAnyOrder(IndexAuditTrail.NAME)); + assertThat(service.getAuditTrails().stream().map(x -> x.name()).collect(Collectors.toList()), + containsInAnyOrder(IndexAuditTrail.NAME)); } public void testIndexAndLoggingAuditTrail() throws Exception { @@ -195,7 +198,7 @@ public void testIndexAndLoggingAuditTrail() throws Exception { Collection components = createComponents(settings); AuditTrailService service = findComponent(AuditTrailService.class, components); assertNotNull(service); - assertThat(service.getAuditTrails(), + assertThat(service.getAuditTrails().stream().map(x -> x.name()).collect(Collectors.toList()), containsInAnyOrder(LoggingAuditTrail.NAME, DeprecatedLoggingAuditTrail.NAME, IndexAuditTrail.NAME)); } From 6bf666bb71fd19abc8a3831921aca7ccc1a0cd8e Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Wed, 17 Oct 2018 13:57:21 +0300 Subject: [PATCH 10/10] Comments as per review --- x-pack/plugin/core/src/main/config/log4j2.properties | 3 +++ x-pack/plugin/security/build.gradle | 3 +++ 2 files changed, 6 insertions(+) diff --git a/x-pack/plugin/core/src/main/config/log4j2.properties b/x-pack/plugin/core/src/main/config/log4j2.properties index 4f4bd19060cd4..d7b0181f8f971 100644 --- a/x-pack/plugin/core/src/main/config/log4j2.properties +++ b/x-pack/plugin/core/src/main/config/log4j2.properties @@ -81,6 +81,9 @@ logger.xpack_security_audit_logfile.appenderRef.audit_rolling.ref = audit_rollin logger.xpack_security_audit_logfile.additivity = false logger.xpack_security_audit_deprecated_logfile.name = org.elasticsearch.xpack.security.audit.logfile.DeprecatedLoggingAuditTrail +# set this to "off" instead of "info" to disable the deprecated appender +# in the 6.x releases both the new and the previous appenders are enabled +# for the logfile auditing logger.xpack_security_audit_deprecated_logfile.level = info logger.xpack_security_audit_deprecated_logfile.appenderRef.deprecated_audit_rolling.ref = deprecated_audit_rolling logger.xpack_security_audit_deprecated_logfile.additivity = false diff --git a/x-pack/plugin/security/build.gradle b/x-pack/plugin/security/build.gradle index b9a30771c0edb..8d22903824e41 100644 --- a/x-pack/plugin/security/build.gradle +++ b/x-pack/plugin/security/build.gradle @@ -146,6 +146,9 @@ processResources { sourceSets.test.resources { srcDir '../core/src/test/resources' + // this is required because audit logging tests (LoggingAuditTrailTests) + // parse the actual log4j2.properties config file to extract and test + // the log line pattern. These tests need access to the file as a resource. srcDir '../core/src/main/config' } dependencyLicenses {