From 0e7723317162074314728a2fcebcb0107a79ce5b Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 4 Jun 2018 20:29:58 -0700 Subject: [PATCH 1/6] Initial commit for HashProcessor It is useful to have a processor similar to https://www.elastic.co/guide/en/logstash/6.0/plugins-filters-fingerprint.html in Elasticsearch. A processor that leverages a variety of hashing algorithms to create cryptographically-secure one-way hashes of values in documents. --- .../core/security/authc/support/Hasher.java | 2 +- .../xpack/security/Security.java | 10 +- .../xpack/security/ingest/HashProcessor.java | 183 ++++++++++++++++++ .../ingest/HashProcessorFactoryTests.java | 114 +++++++++++ .../security/ingest/HashProcessorTests.java | 94 +++++++++ 5 files changed, 401 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java index 0d4a1d23e7910..18769b3c810b0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java @@ -277,7 +277,7 @@ public static Hasher resolve(String name) { public abstract boolean verify(SecureString data, char[] hash); - static final class SaltProvider { + public static final class SaltProvider { static final char[] ALPHABET = new char[]{ '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index dbb50a92f1088..41c8ff4e87eaf 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -173,6 +173,7 @@ import org.elasticsearch.xpack.security.authz.accesscontrol.OptOutQueryCache; import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore; import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; +import org.elasticsearch.xpack.security.ingest.HashProcessor; import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor; import org.elasticsearch.xpack.security.rest.SecurityRestFilter; import org.elasticsearch.xpack.security.rest.action.RestAuthenticateAction; @@ -572,6 +573,10 @@ public static List> getSettings(boolean transportClientMode, List getRestHandlers(Settings settings, RestController restC @Override public Map getProcessors(Processor.Parameters parameters) { - return Collections.singletonMap(SetSecurityUserProcessor.TYPE, new SetSecurityUserProcessor.Factory(parameters.threadContext)); + Map processors = new HashMap<>(); + processors.put(SetSecurityUserProcessor.TYPE, new SetSecurityUserProcessor.Factory(parameters.threadContext)); + processors.put(HashProcessor.TYPE, new HashProcessor.Factory(parameters.env.settings())); + return processors; } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java new file mode 100644 index 0000000000000..4c6fdeebf7a70 --- /dev/null +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java @@ -0,0 +1,183 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.security.ingest; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.settings.SecureSetting; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.ingest.AbstractProcessor; +import org.elasticsearch.ingest.ConfigurationUtils; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.Processor; +import org.elasticsearch.xpack.core.security.SecurityField; +import org.elasticsearch.xpack.core.security.authc.support.CharArrays; +import org.elasticsearch.xpack.core.security.authc.support.Hasher; + +import javax.crypto.Mac; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; +import java.nio.charset.StandardCharsets; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.spec.InvalidKeySpecException; +import java.util.Arrays; +import java.util.Base64; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException; + +/** + * A processor that hashes the contents of a field (or fields) using various hashing algorithms + */ +public final class HashProcessor extends AbstractProcessor { + enum Method { + SHA1("HmacSHA1"), + SHA256("HmacSHA256"), + SHA384("HmacSHA384"), + SHA512("HmacSHA512"); + + private final String algorithm; + + Method(String algorithm) { + this.algorithm = algorithm; + } + + public String getAlgorithm() { + return algorithm; + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + + public String hash(Mac mac, byte[] salt, String input) { + try { + byte[] encrypted = mac.doFinal(input.getBytes(StandardCharsets.UTF_8)); + byte[] messageWithSalt = new byte[salt.length + encrypted.length]; + System.arraycopy(salt, 0, messageWithSalt, 0, salt.length); + System.arraycopy(encrypted, 0, messageWithSalt, salt.length, encrypted.length); + return Base64.getEncoder().encodeToString(messageWithSalt); + } catch (IllegalStateException e) { + throw new ElasticsearchException("error hashing data", e); + } + } + + public static Method fromString(String processorTag, String propertyName, String type) { + try { + return Method.valueOf(type.toUpperCase(Locale.ROOT)); + } catch(IllegalArgumentException e) { + throw newConfigurationException(TYPE, processorTag, propertyName, "type [" + type + + "] not supported, cannot convert field. Valid hash methods: " + Arrays.toString(Method.values())); + } + } + } + + public static final String TYPE = "hash"; + public static final Setting.AffixSetting HMAC_KEY_SETTING = SecureSetting + .affixKeySetting(SecurityField.setting("ingest." + TYPE) + ".", "key", + (key) -> SecureSetting.secureString(key, null)); + + private final List fields; + private final String targetField; + private final Method method; + private final Mac mac; + private final byte[] salt; + + HashProcessor(String tag, List fields, String targetField, byte[] salt, Method method, @Nullable Mac mac) { + super(tag); + this.fields = fields; + this.targetField = targetField; + this.method = method; + this.mac = mac; + this.salt = salt; + } + + List getFields() { + return fields; + } + + String getTargetField() { + return targetField; + } + + byte[] getSalt() { + return salt; + } + + @Override + public void execute(IngestDocument document) { + Map hashedFieldValues = fields.stream().map(f -> { + try { + String value = document.getFieldValue(f, String.class); + return new Tuple<>(f, method.hash(mac, salt, value)); + } catch (Exception e) { + throw new IllegalArgumentException("field[" + f + "] could not be hashed", e); + } + }).collect(Collectors.toMap(Tuple::v1, Tuple::v2)); + if (hashedFieldValues.size() == 1) { + document.setFieldValue(targetField, hashedFieldValues.values().iterator().next()); + } else { + document.setFieldValue(targetField, hashedFieldValues); + } + } + + @Override + public String getType() { + return TYPE; + } + + public static final class Factory implements Processor.Factory { + + private final Settings settings; + + public Factory(Settings settings) { + this.settings = settings; + } + + private static Mac createMac(Method method, SecureString password, byte[] salt, int iterations) { + try { + SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2With" + method.getAlgorithm()); + PBEKeySpec keySpec = new PBEKeySpec(password.getChars(), salt, iterations, 128); + byte[] pbkdf2 = secretKeyFactory.generateSecret(keySpec).getEncoded(); + Mac mac = Mac.getInstance(method.getAlgorithm()); + mac.init(new SecretKeySpec(pbkdf2, method.getAlgorithm())); + return mac; + } catch (NoSuchAlgorithmException | InvalidKeySpecException | InvalidKeyException e) { + throw new IllegalArgumentException("invalid settings", e); + } + } + + @Override + public HashProcessor create(Map registry, String processorTag, Map config) { + List fields = ConfigurationUtils.readList(TYPE, processorTag, config, "fields"); + if (fields.isEmpty()) { + throw ConfigurationUtils.newConfigurationException(TYPE, processorTag, "fields", "must specify at least one field"); + } else if (fields.stream().anyMatch(Strings::isNullOrEmpty)) { + throw ConfigurationUtils.newConfigurationException(TYPE, processorTag, "fields", + "a field-name entry is either empty or null"); + } + String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field"); + String keySettingName = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "key_setting"); + SecureString key = HMAC_KEY_SETTING.getConcreteSetting(keySettingName).get(settings); + byte[] salt = CharArrays.toUtf8Bytes(Hasher.SaltProvider.salt(8)); + String methodProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "method", "SHA256"); + Method method = Method.fromString(processorTag, "method", methodProperty); + int iterations = ConfigurationUtils.readIntProperty(TYPE, processorTag, config, "iterations", 5); + Mac mac = createMac(method, key, salt, iterations); + return new HashProcessor(processorTag, fields, targetField, salt, method, mac); + } + } +} diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java new file mode 100644 index 0000000000000..8f8a986f7a1e0 --- /dev/null +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.security.ingest; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.settings.MockSecureSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class HashProcessorFactoryTests extends ESTestCase { + + public void testProcessor() { + MockSecureSettings mockSecureSettings = new MockSecureSettings(); + mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key"); + Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build(); + HashProcessor.Factory factory = new HashProcessor.Factory(settings); + Map config = new HashMap<>(); + config.put("fields", Collections.singletonList("_field")); + config.put("target_field", "_target"); + config.put("key_setting", "xpack.security.ingest.hash.processor.key"); + for (HashProcessor.Method method : HashProcessor.Method.values()) { + config.put("method", method.toString()); + HashProcessor processor = factory.create(null, "_tag", new HashMap<>(config)); + assertThat(processor.getFields(), equalTo(Collections.singletonList("_field"))); + assertThat(processor.getTargetField(), equalTo("_target")); + } + } + + public void testProcessorNoFields() { + MockSecureSettings mockSecureSettings = new MockSecureSettings(); + mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key"); + Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build(); + HashProcessor.Factory factory = new HashProcessor.Factory(settings); + Map config = new HashMap<>(); + config.put("salt", "_salt"); + config.put("target_field", "_target"); + config.put("key_setting", "xpack.security.ingest.hash.processor.key"); + config.put("method", HashProcessor.Method.SHA1.toString()); + ElasticsearchException e = expectThrows(ElasticsearchException.class, + () -> factory.create(null, "_tag", config)); + assertThat(e.getMessage(), equalTo("[fields] required property is missing")); + } + + public void testProcessorNoTargetField() { + MockSecureSettings mockSecureSettings = new MockSecureSettings(); + mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key"); + Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build(); + HashProcessor.Factory factory = new HashProcessor.Factory(settings); + Map config = new HashMap<>(); + config.put("fields", Collections.singletonList("_field")); + config.put("key_setting", "xpack.security.ingest.hash.processor.key"); + config.put("method", HashProcessor.Method.SHA1.toString()); + ElasticsearchException e = expectThrows(ElasticsearchException.class, + () -> factory.create(null, "_tag", config)); + assertThat(e.getMessage(), equalTo("[target_field] required property is missing")); + } + + public void testProcessorFieldsIsEmpty() { + MockSecureSettings mockSecureSettings = new MockSecureSettings(); + mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key"); + Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build(); + HashProcessor.Factory factory = new HashProcessor.Factory(settings); + Map config = new HashMap<>(); + config.put("fields", Collections.singletonList(randomBoolean() ? "" : null)); + config.put("target_field", "_target"); + config.put("key_setting", "xpack.security.ingest.hash.processor.key"); + config.put("method", HashProcessor.Method.SHA1.toString()); + ElasticsearchException e = expectThrows(ElasticsearchException.class, + () -> factory.create(null, "_tag", config)); + assertThat(e.getMessage(), equalTo("[fields] a field-name entry is either empty or null")); + } + + public void testProcessorInvalidMethod() { + MockSecureSettings mockSecureSettings = new MockSecureSettings(); + mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key"); + Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build(); + HashProcessor.Factory factory = new HashProcessor.Factory(settings); + Map config = new HashMap<>(); + config.put("fields", Collections.singletonList("_field")); + config.put("target_field", "_target"); + config.put("key_setting", "xpack.security.ingest.hash.processor.key"); + config.put("method", "invalid"); + ElasticsearchException e = expectThrows(ElasticsearchException.class, + () -> factory.create(null, "_tag", config)); + assertThat(e.getMessage(), equalTo("[method] type [invalid] not supported, cannot convert field. " + + "Valid hash methods: [sha1, sha256, sha384, sha512]")); + } + + public void testProcessorInvalidOrMissingKeySetting() { + Settings settings = Settings.builder().setSecureSettings(new MockSecureSettings()).build(); + HashProcessor.Factory factory = new HashProcessor.Factory(settings); + Map config = new HashMap<>(); + config.put("fields", Collections.singletonList("_field")); + config.put("target_field", "_target"); + config.put("key_setting", "invalid"); + config.put("method", HashProcessor.Method.SHA1.toString()); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> factory.create(null, "_tag", new HashMap<>(config))); + assertThat(e.getMessage(), equalTo("key [invalid] must match [xpack.security.ingest.hash.*.key] but didn't.")); + config.remove("key_setting"); + ElasticsearchException ex = expectThrows(ElasticsearchException.class, + () -> factory.create(null, "_tag", config)); + assertThat(ex.getMessage(), equalTo("[key_setting] required property is missing")); + } +} diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorTests.java new file mode 100644 index 0000000000000..20ab26ed6e244 --- /dev/null +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorTests.java @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.security.ingest; + +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.security.authc.support.CharArrays; +import org.elasticsearch.xpack.core.security.authc.support.Hasher; +import org.elasticsearch.xpack.security.ingest.HashProcessor.Method; + +import javax.crypto.Mac; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class HashProcessorTests extends ESTestCase { + public void testProcessorSingleField() throws Exception { + List fields = Collections.singletonList(randomAlphaOfLength(6)); + Map docFields = new HashMap<>(); + for (String field : fields) { + docFields.put(field, randomAlphaOfLengthBetween(2, 10)); + } + + String targetField = randomAlphaOfLength(6); + Method method = randomFrom(Method.values()); + Mac mac = createMac(method); + byte[] salt = CharArrays.toUtf8Bytes(Hasher.SaltProvider.salt(5)); + HashProcessor processor = new HashProcessor("_tag", fields, targetField, salt, method, mac); + IngestDocument ingestDocument = new IngestDocument(docFields, new HashMap<>()); + processor.execute(ingestDocument); + + String targetFieldValue = ingestDocument.getFieldValue(targetField, String.class); + Object expectedTargetFieldValue = method.hash(mac, salt, ingestDocument.getFieldValue(fields.get(0), String.class)); + assertThat(targetFieldValue, equalTo(expectedTargetFieldValue)); + byte[] bytes = Base64.getDecoder().decode(targetFieldValue); + byte[] actualSaltPrefix = new byte[salt.length]; + System.arraycopy(bytes, 0, actualSaltPrefix, 0, salt.length); + assertArrayEquals(salt, actualSaltPrefix); + } + + @SuppressWarnings("unchecked") + public void testProcessorMultipleFields() throws Exception { + List fields = new ArrayList<>(); + for (int i = 0; i < randomIntBetween(2, 10); i++) { + fields.add(randomAlphaOfLength(5 + i)); + } + Map docFields = new HashMap<>(); + for (String field : fields) { + docFields.put(field, randomAlphaOfLengthBetween(2, 10)); + } + + String targetField = randomAlphaOfLength(6); + Method method = randomFrom(Method.values()); + Mac mac = createMac(method); + byte[] salt = CharArrays.toUtf8Bytes(Hasher.SaltProvider.salt(5)); + HashProcessor processor = new HashProcessor("_tag", fields, targetField, salt, method, mac); + IngestDocument ingestDocument = new IngestDocument(docFields, new HashMap<>()); + processor.execute(ingestDocument); + + Map targetFieldMap = ingestDocument.getFieldValue(targetField, Map.class); + for (Map.Entry entry : targetFieldMap.entrySet()) { + Object expectedTargetFieldValue = method.hash(mac, salt, ingestDocument.getFieldValue(entry.getKey(), String.class)); + assertThat(entry.getValue(), equalTo(expectedTargetFieldValue)); + byte[] bytes = Base64.getDecoder().decode(entry.getValue()); + byte[] actualSaltPrefix = new byte[salt.length]; + System.arraycopy(bytes, 0, actualSaltPrefix, 0, salt.length); + assertArrayEquals(salt, actualSaltPrefix); + } + } + + private Mac createMac(Method method) throws Exception { + char[] password = randomAlphaOfLengthBetween(1, 10).toCharArray(); + byte[] salt = randomAlphaOfLength(5).getBytes(StandardCharsets.UTF_8); + int iterations = randomIntBetween(1, 10); + SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2With" + method.getAlgorithm()); + PBEKeySpec keySpec = new PBEKeySpec(password, salt, iterations, 128); + byte[] pbkdf2 = secretKeyFactory.generateSecret(keySpec).getEncoded(); + Mac mac = Mac.getInstance(method.getAlgorithm()); + mac.init(new SecretKeySpec(pbkdf2, method.getAlgorithm())); + return mac; + } +} From 80ca1ee5b385c7411ce15fa7af740cfd9a62d207 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 27 Jun 2018 22:33:52 -0700 Subject: [PATCH 2/6] add rest test, and add two params for salt & ignore_missing --- x-pack/plugin/build.gradle | 1 + .../xpack/security/ingest/HashProcessor.java | 109 ++++++++++-------- .../ingest/HashProcessorFactoryTests.java | 4 +- .../security/ingest/HashProcessorTests.java | 42 ++++++- .../test/hash_processor/10_basic.yml | 51 ++++++++ 5 files changed, 154 insertions(+), 53 deletions(-) create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/hash_processor/10_basic.yml diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 3822ef1d4d584..7454ba33fb590 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -151,6 +151,7 @@ integTestCluster { setting 'xpack.license.self_generated.type', 'trial' keystoreSetting 'bootstrap.password', 'x-pack-test-password' keystoreSetting 'xpack.security.transport.ssl.keystore.secure_password', 'keypass' + keystoreSetting 'xpack.security.ingest.hash.processor.test_key', 'hmackey' distribution = 'zip' // this is important since we use the reindex module in ML setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser' diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java index 4c6fdeebf7a70..e91fc15ef0a61 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java @@ -18,7 +18,6 @@ import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; import org.elasticsearch.xpack.core.security.SecurityField; -import org.elasticsearch.xpack.core.security.authc.support.CharArrays; import org.elasticsearch.xpack.core.security.authc.support.Hasher; import javax.crypto.Mac; @@ -34,6 +33,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException; @@ -42,49 +42,6 @@ * A processor that hashes the contents of a field (or fields) using various hashing algorithms */ public final class HashProcessor extends AbstractProcessor { - enum Method { - SHA1("HmacSHA1"), - SHA256("HmacSHA256"), - SHA384("HmacSHA384"), - SHA512("HmacSHA512"); - - private final String algorithm; - - Method(String algorithm) { - this.algorithm = algorithm; - } - - public String getAlgorithm() { - return algorithm; - } - - @Override - public String toString() { - return name().toLowerCase(Locale.ROOT); - } - - public String hash(Mac mac, byte[] salt, String input) { - try { - byte[] encrypted = mac.doFinal(input.getBytes(StandardCharsets.UTF_8)); - byte[] messageWithSalt = new byte[salt.length + encrypted.length]; - System.arraycopy(salt, 0, messageWithSalt, 0, salt.length); - System.arraycopy(encrypted, 0, messageWithSalt, salt.length, encrypted.length); - return Base64.getEncoder().encodeToString(messageWithSalt); - } catch (IllegalStateException e) { - throw new ElasticsearchException("error hashing data", e); - } - } - - public static Method fromString(String processorTag, String propertyName, String type) { - try { - return Method.valueOf(type.toUpperCase(Locale.ROOT)); - } catch(IllegalArgumentException e) { - throw newConfigurationException(TYPE, processorTag, propertyName, "type [" + type + - "] not supported, cannot convert field. Valid hash methods: " + Arrays.toString(Method.values())); - } - } - } - public static final String TYPE = "hash"; public static final Setting.AffixSetting HMAC_KEY_SETTING = SecureSetting .affixKeySetting(SecurityField.setting("ingest." + TYPE) + ".", "key", @@ -95,14 +52,16 @@ public static Method fromString(String processorTag, String propertyName, String private final Method method; private final Mac mac; private final byte[] salt; + private final boolean ignoreMissing; - HashProcessor(String tag, List fields, String targetField, byte[] salt, Method method, @Nullable Mac mac) { + HashProcessor(String tag, List fields, String targetField, byte[] salt, Method method, @Nullable Mac mac, boolean ignoreMissing) { super(tag); this.fields = fields; this.targetField = targetField; this.method = method; this.mac = mac; this.salt = salt; + this.ignoreMissing = ignoreMissing; } List getFields() { @@ -120,14 +79,18 @@ byte[] getSalt() { @Override public void execute(IngestDocument document) { Map hashedFieldValues = fields.stream().map(f -> { + String value = document.getFieldValue(f, String.class, ignoreMissing); + if (value == null && ignoreMissing) { + return new Tuple(null, null); + } + try { - String value = document.getFieldValue(f, String.class); return new Tuple<>(f, method.hash(mac, salt, value)); } catch (Exception e) { throw new IllegalArgumentException("field[" + f + "] could not be hashed", e); } - }).collect(Collectors.toMap(Tuple::v1, Tuple::v2)); - if (hashedFieldValues.size() == 1) { + }).filter(tuple -> Objects.nonNull(tuple.v1())).collect(Collectors.toMap(Tuple::v1, Tuple::v2)); + if (fields.size() == 1) { document.setFieldValue(targetField, hashedFieldValues.values().iterator().next()); } else { document.setFieldValue(targetField, hashedFieldValues); @@ -162,6 +125,7 @@ private static Mac createMac(Method method, SecureString password, byte[] salt, @Override public HashProcessor create(Map registry, String processorTag, Map config) { + boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); List fields = ConfigurationUtils.readList(TYPE, processorTag, config, "fields"); if (fields.isEmpty()) { throw ConfigurationUtils.newConfigurationException(TYPE, processorTag, "fields", "must specify at least one field"); @@ -172,12 +136,57 @@ public HashProcessor create(Map registry, String proc String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field"); String keySettingName = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "key_setting"); SecureString key = HMAC_KEY_SETTING.getConcreteSetting(keySettingName).get(settings); - byte[] salt = CharArrays.toUtf8Bytes(Hasher.SaltProvider.salt(8)); + String saltString = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "salt", + new String(Hasher.SaltProvider.salt(8))); + byte[] salt = saltString.getBytes(StandardCharsets.UTF_8); String methodProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "method", "SHA256"); Method method = Method.fromString(processorTag, "method", methodProperty); int iterations = ConfigurationUtils.readIntProperty(TYPE, processorTag, config, "iterations", 5); Mac mac = createMac(method, key, salt, iterations); - return new HashProcessor(processorTag, fields, targetField, salt, method, mac); + return new HashProcessor(processorTag, fields, targetField, salt, method, mac, ignoreMissing); + } + } + + enum Method { + SHA1("HmacSHA1"), + SHA256("HmacSHA256"), + SHA384("HmacSHA384"), + SHA512("HmacSHA512"); + + private final String algorithm; + + Method(String algorithm) { + this.algorithm = algorithm; + } + + public String getAlgorithm() { + return algorithm; + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + + public String hash(Mac mac, byte[] salt, String input) { + try { + byte[] encrypted = mac.doFinal(input.getBytes(StandardCharsets.UTF_8)); + byte[] messageWithSalt = new byte[salt.length + encrypted.length]; + System.arraycopy(salt, 0, messageWithSalt, 0, salt.length); + System.arraycopy(encrypted, 0, messageWithSalt, salt.length, encrypted.length); + return Base64.getEncoder().encodeToString(messageWithSalt); + } catch (IllegalStateException e) { + throw new ElasticsearchException("error hashing data", e); + } + } + + public static Method fromString(String processorTag, String propertyName, String type) { + try { + return Method.valueOf(type.toUpperCase(Locale.ROOT)); + } catch(IllegalArgumentException e) { + throw newConfigurationException(TYPE, processorTag, propertyName, "type [" + type + + "] not supported, cannot convert field. Valid hash methods: " + Arrays.toString(Method.values())); + } } } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java index 8f8a986f7a1e0..f16d82d40700f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -26,12 +27,14 @@ public void testProcessor() { Map config = new HashMap<>(); config.put("fields", Collections.singletonList("_field")); config.put("target_field", "_target"); + config.put("salt", "_salt"); config.put("key_setting", "xpack.security.ingest.hash.processor.key"); for (HashProcessor.Method method : HashProcessor.Method.values()) { config.put("method", method.toString()); HashProcessor processor = factory.create(null, "_tag", new HashMap<>(config)); assertThat(processor.getFields(), equalTo(Collections.singletonList("_field"))); assertThat(processor.getTargetField(), equalTo("_target")); + assertArrayEquals(processor.getSalt(), "_salt".getBytes(StandardCharsets.UTF_8)); } } @@ -41,7 +44,6 @@ public void testProcessorNoFields() { Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build(); HashProcessor.Factory factory = new HashProcessor.Factory(settings); Map config = new HashMap<>(); - config.put("salt", "_salt"); config.put("target_field", "_target"); config.put("key_setting", "xpack.security.ingest.hash.processor.key"); config.put("method", HashProcessor.Method.SHA1.toString()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorTests.java index 20ab26ed6e244..681ac67a22305 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorTests.java @@ -17,6 +17,7 @@ import javax.crypto.spec.SecretKeySpec; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.Base64; import java.util.Collections; import java.util.HashMap; @@ -26,6 +27,43 @@ import static org.hamcrest.Matchers.equalTo; public class HashProcessorTests extends ESTestCase { + + @SuppressWarnings("unchecked") + public void testIgnoreMissing() throws Exception { + Method method = randomFrom(Method.values()); + Mac mac = createMac(method); + Map fields = new HashMap<>(); + fields.put("one", "foo"); + HashProcessor processor = new HashProcessor("_tag", Arrays.asList("one", "two"), + "target", "_salt".getBytes(StandardCharsets.UTF_8), Method.SHA1, mac, true); + IngestDocument ingestDocument = new IngestDocument(fields, new HashMap<>()); + processor.execute(ingestDocument); + Map target = ingestDocument.getFieldValue("target", Map.class); + assertThat(target.size(), equalTo(1)); + assertNotNull(target.get("one")); + + HashProcessor failProcessor = new HashProcessor("_tag", Arrays.asList("one", "two"), + "target", "_salt".getBytes(StandardCharsets.UTF_8), Method.SHA1, mac, false); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> failProcessor.execute(ingestDocument)); + assertThat(exception.getMessage(), equalTo("field [two] not present as part of path [two]")); + } + + public void testStaticKeyAndSalt() throws Exception { + byte[] salt = "_salt".getBytes(StandardCharsets.UTF_8); + SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1"); + PBEKeySpec keySpec = new PBEKeySpec("hmackey".toCharArray(), salt, 5, 128); + byte[] pbkdf2 = secretKeyFactory.generateSecret(keySpec).getEncoded(); + Mac mac = Mac.getInstance(Method.SHA1.getAlgorithm()); + mac.init(new SecretKeySpec(pbkdf2, Method.SHA1.getAlgorithm())); + Map fields = new HashMap<>(); + fields.put("field", "0123456789"); + HashProcessor processor = new HashProcessor("_tag", Collections.singletonList("field"), + "target", salt, Method.SHA1, mac, false); + IngestDocument ingestDocument = new IngestDocument(fields, new HashMap<>()); + processor.execute(ingestDocument); + assertThat(ingestDocument.getFieldValue("target", String.class), equalTo("X3NhbHQMW0oHJGEEE9obGcGv5tGd7HFyDw==")); + } + public void testProcessorSingleField() throws Exception { List fields = Collections.singletonList(randomAlphaOfLength(6)); Map docFields = new HashMap<>(); @@ -37,7 +75,7 @@ public void testProcessorSingleField() throws Exception { Method method = randomFrom(Method.values()); Mac mac = createMac(method); byte[] salt = CharArrays.toUtf8Bytes(Hasher.SaltProvider.salt(5)); - HashProcessor processor = new HashProcessor("_tag", fields, targetField, salt, method, mac); + HashProcessor processor = new HashProcessor("_tag", fields, targetField, salt, method, mac, false); IngestDocument ingestDocument = new IngestDocument(docFields, new HashMap<>()); processor.execute(ingestDocument); @@ -65,7 +103,7 @@ public void testProcessorMultipleFields() throws Exception { Method method = randomFrom(Method.values()); Mac mac = createMac(method); byte[] salt = CharArrays.toUtf8Bytes(Hasher.SaltProvider.salt(5)); - HashProcessor processor = new HashProcessor("_tag", fields, targetField, salt, method, mac); + HashProcessor processor = new HashProcessor("_tag", fields, targetField, salt, method, mac, false); IngestDocument ingestDocument = new IngestDocument(docFields, new HashMap<>()); processor.execute(ingestDocument); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/hash_processor/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/hash_processor/10_basic.yml new file mode 100644 index 0000000000000..b85592adf7391 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/hash_processor/10_basic.yml @@ -0,0 +1,51 @@ +--- +teardown: + - do: + ingest.delete_pipeline: + id: "my_pipeline" + ignore: 404 + +--- +"Test Hash Processor": + + - do: + cluster.health: + wait_for_status: yellow + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "processors": [ + { + "hash" : { + "fields" : ["user_ssid"], + "target_field" : "anonymized", + "salt": "_salt", + "iterations": 5, + "method": "sha1", + "key_setting": "xpack.security.ingest.hash.processor.test_key" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + type: test + id: 1 + pipeline: "my_pipeline" + body: > + { + "user_ssid": "0123456789" + } + + - do: + get: + index: test + type: test + id: 1 + - match: { _source.anonymized: "X3NhbHQMW0oHJGEEE9obGcGv5tGd7HFyDw==" } + From 1676aa98d2a8259e40ebe12c704a75e993c0152e Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 28 Jun 2018 07:03:28 -0700 Subject: [PATCH 3/6] fix key resolution --- x-pack/plugin/build.gradle | 2 +- .../xpack/security/ingest/HashProcessor.java | 16 ++++++++++++++-- .../ingest/HashProcessorFactoryTests.java | 5 +++-- .../test/hash_processor/10_basic.yml | 2 +- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index ac6d7de9b28a6..919663c57cbfa 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -151,7 +151,7 @@ integTestCluster { setting 'xpack.license.self_generated.type', 'trial' keystoreSetting 'bootstrap.password', 'x-pack-test-password' keystoreSetting 'xpack.security.transport.ssl.keystore.secure_password', 'keypass' - keystoreSetting 'xpack.security.ingest.hash.processor.test_key', 'hmackey' + keystoreSetting 'xpack.security.ingest.hash.processor.key', 'hmackey' distribution = 'zip' // this is important since we use the reindex module in ML setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser' diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java index e91fc15ef0a61..7fb1d0d4cea49 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java @@ -30,6 +30,7 @@ import java.security.spec.InvalidKeySpecException; import java.util.Arrays; import java.util.Base64; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -54,7 +55,8 @@ public final class HashProcessor extends AbstractProcessor { private final byte[] salt; private final boolean ignoreMissing; - HashProcessor(String tag, List fields, String targetField, byte[] salt, Method method, @Nullable Mac mac, boolean ignoreMissing) { + HashProcessor(String tag, List fields, String targetField, byte[] salt, Method method, @Nullable Mac mac, + boolean ignoreMissing) { super(tag); this.fields = fields; this.targetField = targetField; @@ -105,9 +107,15 @@ public String getType() { public static final class Factory implements Processor.Factory { private final Settings settings; + private final Map secureKeys; public Factory(Settings settings) { this.settings = settings; + this.secureKeys = new HashMap<>(); + HMAC_KEY_SETTING.getAllConcreteSettings(settings).forEach(k -> { + secureKeys.put(k.getKey(), k.get(settings)); + }); + } private static Mac createMac(Method method, SecureString password, byte[] salt, int iterations) { @@ -135,7 +143,11 @@ public HashProcessor create(Map registry, String proc } String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field"); String keySettingName = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "key_setting"); - SecureString key = HMAC_KEY_SETTING.getConcreteSetting(keySettingName).get(settings); + SecureString key = secureKeys.get(keySettingName); + if (key == null) { + throw ConfigurationUtils.newConfigurationException(TYPE, processorTag, "key_setting", + "key [" + keySettingName + "] must match [xpack.security.ingest.hash.*.key]. It is not set"); + } String saltString = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "salt", new String(Hasher.SaltProvider.salt(8))); byte[] salt = saltString.getBytes(StandardCharsets.UTF_8); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java index f16d82d40700f..42c0703194558 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java @@ -105,9 +105,10 @@ public void testProcessorInvalidOrMissingKeySetting() { config.put("target_field", "_target"); config.put("key_setting", "invalid"); config.put("method", HashProcessor.Method.SHA1.toString()); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, "_tag", new HashMap<>(config))); - assertThat(e.getMessage(), equalTo("key [invalid] must match [xpack.security.ingest.hash.*.key] but didn't.")); + assertThat(e.getMessage(), + equalTo("[key_setting] key [invalid] must match [xpack.security.ingest.hash.*.key]. It is not set")); config.remove("key_setting"); ElasticsearchException ex = expectThrows(ElasticsearchException.class, () -> factory.create(null, "_tag", config)); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/hash_processor/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/hash_processor/10_basic.yml index b85592adf7391..ee84e02d2f498 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/hash_processor/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/hash_processor/10_basic.yml @@ -24,7 +24,7 @@ teardown: "salt": "_salt", "iterations": 5, "method": "sha1", - "key_setting": "xpack.security.ingest.hash.processor.test_key" + "key_setting": "xpack.security.ingest.hash.processor.key" } } ] From cad0dc5e7308b02f16973291bb5b900b8412c6f3 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 28 Jun 2018 10:34:53 -0700 Subject: [PATCH 4/6] make salt required --- .../xpack/security/ingest/HashProcessor.java | 4 +--- .../ingest/HashProcessorFactoryTests.java | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java index 7fb1d0d4cea49..5bb78b67b7b0f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java @@ -18,7 +18,6 @@ import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; import org.elasticsearch.xpack.core.security.SecurityField; -import org.elasticsearch.xpack.core.security.authc.support.Hasher; import javax.crypto.Mac; import javax.crypto.SecretKeyFactory; @@ -148,8 +147,7 @@ public HashProcessor create(Map registry, String proc throw ConfigurationUtils.newConfigurationException(TYPE, processorTag, "key_setting", "key [" + keySettingName + "] must match [xpack.security.ingest.hash.*.key]. It is not set"); } - String saltString = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "salt", - new String(Hasher.SaltProvider.salt(8))); + String saltString = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "salt"); byte[] salt = saltString.getBytes(StandardCharsets.UTF_8); String methodProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "method", "SHA256"); Method method = Method.fromString(processorTag, "method", methodProperty); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java index 42c0703194558..e9dda488e7216 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java @@ -45,6 +45,7 @@ public void testProcessorNoFields() { HashProcessor.Factory factory = new HashProcessor.Factory(settings); Map config = new HashMap<>(); config.put("target_field", "_target"); + config.put("salt", "_salt"); config.put("key_setting", "xpack.security.ingest.hash.processor.key"); config.put("method", HashProcessor.Method.SHA1.toString()); ElasticsearchException e = expectThrows(ElasticsearchException.class, @@ -59,6 +60,7 @@ public void testProcessorNoTargetField() { HashProcessor.Factory factory = new HashProcessor.Factory(settings); Map config = new HashMap<>(); config.put("fields", Collections.singletonList("_field")); + config.put("salt", "_salt"); config.put("key_setting", "xpack.security.ingest.hash.processor.key"); config.put("method", HashProcessor.Method.SHA1.toString()); ElasticsearchException e = expectThrows(ElasticsearchException.class, @@ -73,6 +75,7 @@ public void testProcessorFieldsIsEmpty() { HashProcessor.Factory factory = new HashProcessor.Factory(settings); Map config = new HashMap<>(); config.put("fields", Collections.singletonList(randomBoolean() ? "" : null)); + config.put("salt", "_salt"); config.put("target_field", "_target"); config.put("key_setting", "xpack.security.ingest.hash.processor.key"); config.put("method", HashProcessor.Method.SHA1.toString()); @@ -81,6 +84,20 @@ public void testProcessorFieldsIsEmpty() { assertThat(e.getMessage(), equalTo("[fields] a field-name entry is either empty or null")); } + public void testProcessorMissingSalt() { + MockSecureSettings mockSecureSettings = new MockSecureSettings(); + mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key"); + Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build(); + HashProcessor.Factory factory = new HashProcessor.Factory(settings); + Map config = new HashMap<>(); + config.put("fields", Collections.singletonList("_field")); + config.put("target_field", "_target"); + config.put("key_setting", "xpack.security.ingest.hash.processor.key"); + ElasticsearchException e = expectThrows(ElasticsearchException.class, + () -> factory.create(null, "_tag", config)); + assertThat(e.getMessage(), equalTo("[salt] required property is missing")); + } + public void testProcessorInvalidMethod() { MockSecureSettings mockSecureSettings = new MockSecureSettings(); mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key"); @@ -88,6 +105,7 @@ public void testProcessorInvalidMethod() { HashProcessor.Factory factory = new HashProcessor.Factory(settings); Map config = new HashMap<>(); config.put("fields", Collections.singletonList("_field")); + config.put("salt", "_salt"); config.put("target_field", "_target"); config.put("key_setting", "xpack.security.ingest.hash.processor.key"); config.put("method", "invalid"); @@ -102,6 +120,7 @@ public void testProcessorInvalidOrMissingKeySetting() { HashProcessor.Factory factory = new HashProcessor.Factory(settings); Map config = new HashMap<>(); config.put("fields", Collections.singletonList("_field")); + config.put("salt", "_salt"); config.put("target_field", "_target"); config.put("key_setting", "invalid"); config.put("method", HashProcessor.Method.SHA1.toString()); From 38f914109cd075b23806411cc265e35596048f78 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 28 Jun 2018 10:51:54 -0700 Subject: [PATCH 5/6] Fix merge master test --- .../xpack/security/ingest/HashProcessorTests.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorTests.java index 681ac67a22305..b3890600592f5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorTests.java @@ -7,8 +7,6 @@ import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.security.authc.support.CharArrays; -import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.security.ingest.HashProcessor.Method; import javax.crypto.Mac; @@ -74,7 +72,7 @@ public void testProcessorSingleField() throws Exception { String targetField = randomAlphaOfLength(6); Method method = randomFrom(Method.values()); Mac mac = createMac(method); - byte[] salt = CharArrays.toUtf8Bytes(Hasher.SaltProvider.salt(5)); + byte[] salt = randomByteArrayOfLength(5); HashProcessor processor = new HashProcessor("_tag", fields, targetField, salt, method, mac, false); IngestDocument ingestDocument = new IngestDocument(docFields, new HashMap<>()); processor.execute(ingestDocument); @@ -102,7 +100,7 @@ public void testProcessorMultipleFields() throws Exception { String targetField = randomAlphaOfLength(6); Method method = randomFrom(Method.values()); Mac mac = createMac(method); - byte[] salt = CharArrays.toUtf8Bytes(Hasher.SaltProvider.salt(5)); + byte[] salt = randomByteArrayOfLength(5); HashProcessor processor = new HashProcessor("_tag", fields, targetField, salt, method, mac, false); IngestDocument ingestDocument = new IngestDocument(docFields, new HashMap<>()); processor.execute(ingestDocument); From 278f23500456808399b836174fbdbdb8a65a1614 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Fri, 29 Jun 2018 10:07:39 -0400 Subject: [PATCH 6/6] remove extra lines --- .../org/elasticsearch/xpack/security/ingest/HashProcessor.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java index 5bb78b67b7b0f..fa49b843847ee 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java @@ -84,7 +84,6 @@ public void execute(IngestDocument document) { if (value == null && ignoreMissing) { return new Tuple(null, null); } - try { return new Tuple<>(f, method.hash(mac, salt, value)); } catch (Exception e) { @@ -114,7 +113,6 @@ public Factory(Settings settings) { HMAC_KEY_SETTING.getAllConcreteSettings(settings).forEach(k -> { secureKeys.put(k.getKey(), k.get(settings)); }); - } private static Mac createMac(Method method, SecureString password, byte[] salt, int iterations) {