Skip to content

Commit 0e77233

Browse files
talevytalevy
committed
Initial commit for HashProcessor
It is useful to have a processor similar to https://www.elastic.co/guide/en/logstash/6.0/plugins-filters-fingerprint.html in Elasticsearch. A processor that leverages a variety of hashing algorithms to create cryptographically-secure one-way hashes of values in documents.
1 parent 237650e commit 0e77233

File tree

5 files changed

+401
-2
lines changed

5 files changed

+401
-2
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -277,7 +277,7 @@ public static Hasher resolve(String name) {
277277

278278
public abstract boolean verify(SecureString data, char[] hash);
279279

280-
static final class SaltProvider {
280+
public static final class SaltProvider {
281281

282282
static final char[] ALPHABET = new char[]{
283283
'.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D',

‎x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,7 @@
173173
import org.elasticsearch.xpack.security.authz.accesscontrol.OptOutQueryCache;
174174
import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
175175
import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
176+
import org.elasticsearch.xpack.security.ingest.HashProcessor;
176177
import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor;
177178
import org.elasticsearch.xpack.security.rest.SecurityRestFilter;
178179
import org.elasticsearch.xpack.security.rest.action.RestAuthenticateAction;
@@ -572,6 +573,10 @@ public static List<Setting<?>> getSettings(boolean transportClientMode, List<Sec
572573
// hide settings
573574
settingsList.add(Setting.listSetting(SecurityField.setting("hide_settings"), Collections.emptyList(), Function.identity(),
574575
Property.NodeScope, Property.Filtered));
576+
577+
// ingest processor settings
578+
settingsList.add(HashProcessor.HMAC_KEY_SETTING);
579+
575580
return settingsList;
576581
}
577582

@@ -715,7 +720,10 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
715720

716721
@Override
717722
public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
718-
return Collections.singletonMap(SetSecurityUserProcessor.TYPE, new SetSecurityUserProcessor.Factory(parameters.threadContext));
723+
Map<String, Processor.Factory> processors = new HashMap<>();
724+
processors.put(SetSecurityUserProcessor.TYPE, new SetSecurityUserProcessor.Factory(parameters.threadContext));
725+
processors.put(HashProcessor.TYPE, new HashProcessor.Factory(parameters.env.settings()));
726+
return processors;
719727
}
720728

721729

‎x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.security.ingest;
7+
8+
import org.elasticsearch.ElasticsearchException;
9+
import org.elasticsearch.common.Nullable;
10+
import org.elasticsearch.common.Strings;
11+
import org.elasticsearch.common.collect.Tuple;
12+
import org.elasticsearch.common.settings.SecureSetting;
13+
import org.elasticsearch.common.settings.SecureString;
14+
import org.elasticsearch.common.settings.Setting;
15+
import org.elasticsearch.common.settings.Settings;
16+
import org.elasticsearch.ingest.AbstractProcessor;
17+
import org.elasticsearch.ingest.ConfigurationUtils;
18+
import org.elasticsearch.ingest.IngestDocument;
19+
import org.elasticsearch.ingest.Processor;
20+
import org.elasticsearch.xpack.core.security.SecurityField;
21+
import org.elasticsearch.xpack.core.security.authc.support.CharArrays;
22+
import org.elasticsearch.xpack.core.security.authc.support.Hasher;
23+
24+
import javax.crypto.Mac;
25+
import javax.crypto.SecretKeyFactory;
26+
import javax.crypto.spec.PBEKeySpec;
27+
import javax.crypto.spec.SecretKeySpec;
28+
import java.nio.charset.StandardCharsets;
29+
import java.security.InvalidKeyException;
30+
import java.security.NoSuchAlgorithmException;
31+
import java.security.spec.InvalidKeySpecException;
32+
import java.util.Arrays;
33+
import java.util.Base64;
34+
import java.util.List;
35+
import java.util.Locale;
36+
import java.util.Map;
37+
import java.util.stream.Collectors;
38+
39+
import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
40+
41+
/**
42+
* A processor that hashes the contents of a field (or fields) using various hashing algorithms
43+
*/
44+
public final class HashProcessor extends AbstractProcessor {
45+
enum Method {
46+
SHA1("HmacSHA1"),
47+
SHA256("HmacSHA256"),
48+
SHA384("HmacSHA384"),
49+
SHA512("HmacSHA512");
50+
51+
private final String algorithm;
52+
53+
Method(String algorithm) {
54+
this.algorithm = algorithm;
55+
}
56+
57+
public String getAlgorithm() {
58+
return algorithm;
59+
}
60+
61+
@Override
62+
public String toString() {
63+
return name().toLowerCase(Locale.ROOT);
64+
}
65+
66+
public String hash(Mac mac, byte[] salt, String input) {
67+
try {
68+
byte[] encrypted = mac.doFinal(input.getBytes(StandardCharsets.UTF_8));
69+
byte[] messageWithSalt = new byte[salt.length + encrypted.length];
70+
System.arraycopy(salt, 0, messageWithSalt, 0, salt.length);
71+
System.arraycopy(encrypted, 0, messageWithSalt, salt.length, encrypted.length);
72+
return Base64.getEncoder().encodeToString(messageWithSalt);
73+
} catch (IllegalStateException e) {
74+
throw new ElasticsearchException("error hashing data", e);
75+
}
76+
}
77+
78+
public static Method fromString(String processorTag, String propertyName, String type) {
79+
try {
80+
return Method.valueOf(type.toUpperCase(Locale.ROOT));
81+
} catch(IllegalArgumentException e) {
82+
throw newConfigurationException(TYPE, processorTag, propertyName, "type [" + type +
83+
"] not supported, cannot convert field. Valid hash methods: " + Arrays.toString(Method.values()));
84+
}
85+
}
86+
}
87+
88+
public static final String TYPE = "hash";
89+
public static final Setting.AffixSetting<SecureString> HMAC_KEY_SETTING = SecureSetting
90+
.affixKeySetting(SecurityField.setting("ingest." + TYPE) + ".", "key",
91+
(key) -> SecureSetting.secureString(key, null));
92+
93+
private final List<String> fields;
94+
private final String targetField;
95+
private final Method method;
96+
private final Mac mac;
97+
private final byte[] salt;
98+
99+
HashProcessor(String tag, List<String> fields, String targetField, byte[] salt, Method method, @Nullable Mac mac) {
100+
super(tag);
101+
this.fields = fields;
102+
this.targetField = targetField;
103+
this.method = method;
104+
this.mac = mac;
105+
this.salt = salt;
106+
}
107+
108+
List<String> getFields() {
109+
return fields;
110+
}
111+
112+
String getTargetField() {
113+
return targetField;
114+
}
115+
116+
byte[] getSalt() {
117+
return salt;
118+
}
119+
120+
@Override
121+
public void execute(IngestDocument document) {
122+
Map<String, String> hashedFieldValues = fields.stream().map(f -> {
123+
try {
124+
String value = document.getFieldValue(f, String.class);
125+
return new Tuple<>(f, method.hash(mac, salt, value));
126+
} catch (Exception e) {
127+
throw new IllegalArgumentException("field[" + f + "] could not be hashed", e);
128+
}
129+
}).collect(Collectors.toMap(Tuple::v1, Tuple::v2));
130+
if (hashedFieldValues.size() == 1) {
131+
document.setFieldValue(targetField, hashedFieldValues.values().iterator().next());
132+
} else {
133+
document.setFieldValue(targetField, hashedFieldValues);
134+
}
135+
}
136+
137+
@Override
138+
public String getType() {
139+
return TYPE;
140+
}
141+
142+
public static final class Factory implements Processor.Factory {
143+
144+
private final Settings settings;
145+
146+
public Factory(Settings settings) {
147+
this.settings = settings;
148+
}
149+
150+
private static Mac createMac(Method method, SecureString password, byte[] salt, int iterations) {
151+
try {
152+
SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2With" + method.getAlgorithm());
153+
PBEKeySpec keySpec = new PBEKeySpec(password.getChars(), salt, iterations, 128);
154+
byte[] pbkdf2 = secretKeyFactory.generateSecret(keySpec).getEncoded();
155+
Mac mac = Mac.getInstance(method.getAlgorithm());
156+
mac.init(new SecretKeySpec(pbkdf2, method.getAlgorithm()));
157+
return mac;
158+
} catch (NoSuchAlgorithmException | InvalidKeySpecException | InvalidKeyException e) {
159+
throw new IllegalArgumentException("invalid settings", e);
160+
}
161+
}
162+
163+
@Override
164+
public HashProcessor create(Map<String, Processor.Factory> registry, String processorTag, Map<String, Object> config) {
165+
List<String> fields = ConfigurationUtils.readList(TYPE, processorTag, config, "fields");
166+
if (fields.isEmpty()) {
167+
throw ConfigurationUtils.newConfigurationException(TYPE, processorTag, "fields", "must specify at least one field");
168+
} else if (fields.stream().anyMatch(Strings::isNullOrEmpty)) {
169+
throw ConfigurationUtils.newConfigurationException(TYPE, processorTag, "fields",
170+
"a field-name entry is either empty or null");
171+
}
172+
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field");
173+
String keySettingName = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "key_setting");
174+
SecureString key = HMAC_KEY_SETTING.getConcreteSetting(keySettingName).get(settings);
175+
byte[] salt = CharArrays.toUtf8Bytes(Hasher.SaltProvider.salt(8));
176+
String methodProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "method", "SHA256");
177+
Method method = Method.fromString(processorTag, "method", methodProperty);
178+
int iterations = ConfigurationUtils.readIntProperty(TYPE, processorTag, config, "iterations", 5);
179+
Mac mac = createMac(method, key, salt, iterations);
180+
return new HashProcessor(processorTag, fields, targetField, salt, method, mac);
181+
}
182+
}
183+
}

‎x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.security.ingest;
7+
8+
import org.elasticsearch.ElasticsearchException;
9+
import org.elasticsearch.common.settings.MockSecureSettings;
10+
import org.elasticsearch.common.settings.Settings;
11+
import org.elasticsearch.test.ESTestCase;
12+
13+
import java.util.Collections;
14+
import java.util.HashMap;
15+
import java.util.Map;
16+
17+
import static org.hamcrest.Matchers.equalTo;
18+
19+
public class HashProcessorFactoryTests extends ESTestCase {
20+
21+
public void testProcessor() {
22+
MockSecureSettings mockSecureSettings = new MockSecureSettings();
23+
mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key");
24+
Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build();
25+
HashProcessor.Factory factory = new HashProcessor.Factory(settings);
26+
Map<String, Object> config = new HashMap<>();
27+
config.put("fields", Collections.singletonList("_field"));
28+
config.put("target_field", "_target");
29+
config.put("key_setting", "xpack.security.ingest.hash.processor.key");
30+
for (HashProcessor.Method method : HashProcessor.Method.values()) {
31+
config.put("method", method.toString());
32+
HashProcessor processor = factory.create(null, "_tag", new HashMap<>(config));
33+
assertThat(processor.getFields(), equalTo(Collections.singletonList("_field")));
34+
assertThat(processor.getTargetField(), equalTo("_target"));
35+
}
36+
}
37+
38+
public void testProcessorNoFields() {
39+
MockSecureSettings mockSecureSettings = new MockSecureSettings();
40+
mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key");
41+
Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build();
42+
HashProcessor.Factory factory = new HashProcessor.Factory(settings);
43+
Map<String, Object> config = new HashMap<>();
44+
config.put("salt", "_salt");
45+
config.put("target_field", "_target");
46+
config.put("key_setting", "xpack.security.ingest.hash.processor.key");
47+
config.put("method", HashProcessor.Method.SHA1.toString());
48+
ElasticsearchException e = expectThrows(ElasticsearchException.class,
49+
() -> factory.create(null, "_tag", config));
50+
assertThat(e.getMessage(), equalTo("[fields] required property is missing"));
51+
}
52+
53+
public void testProcessorNoTargetField() {
54+
MockSecureSettings mockSecureSettings = new MockSecureSettings();
55+
mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key");
56+
Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build();
57+
HashProcessor.Factory factory = new HashProcessor.Factory(settings);
58+
Map<String, Object> config = new HashMap<>();
59+
config.put("fields", Collections.singletonList("_field"));
60+
config.put("key_setting", "xpack.security.ingest.hash.processor.key");
61+
config.put("method", HashProcessor.Method.SHA1.toString());
62+
ElasticsearchException e = expectThrows(ElasticsearchException.class,
63+
() -> factory.create(null, "_tag", config));
64+
assertThat(e.getMessage(), equalTo("[target_field] required property is missing"));
65+
}
66+
67+
public void testProcessorFieldsIsEmpty() {
68+
MockSecureSettings mockSecureSettings = new MockSecureSettings();
69+
mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key");
70+
Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build();
71+
HashProcessor.Factory factory = new HashProcessor.Factory(settings);
72+
Map<String, Object> config = new HashMap<>();
73+
config.put("fields", Collections.singletonList(randomBoolean() ? "" : null));
74+
config.put("target_field", "_target");
75+
config.put("key_setting", "xpack.security.ingest.hash.processor.key");
76+
config.put("method", HashProcessor.Method.SHA1.toString());
77+
ElasticsearchException e = expectThrows(ElasticsearchException.class,
78+
() -> factory.create(null, "_tag", config));
79+
assertThat(e.getMessage(), equalTo("[fields] a field-name entry is either empty or null"));
80+
}
81+
82+
public void testProcessorInvalidMethod() {
83+
MockSecureSettings mockSecureSettings = new MockSecureSettings();
84+
mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key");
85+
Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build();
86+
HashProcessor.Factory factory = new HashProcessor.Factory(settings);
87+
Map<String, Object> config = new HashMap<>();
88+
config.put("fields", Collections.singletonList("_field"));
89+
config.put("target_field", "_target");
90+
config.put("key_setting", "xpack.security.ingest.hash.processor.key");
91+
config.put("method", "invalid");
92+
ElasticsearchException e = expectThrows(ElasticsearchException.class,
93+
() -> factory.create(null, "_tag", config));
94+
assertThat(e.getMessage(), equalTo("[method] type [invalid] not supported, cannot convert field. " +
95+
"Valid hash methods: [sha1, sha256, sha384, sha512]"));
96+
}
97+
98+
public void testProcessorInvalidOrMissingKeySetting() {
99+
Settings settings = Settings.builder().setSecureSettings(new MockSecureSettings()).build();
100+
HashProcessor.Factory factory = new HashProcessor.Factory(settings);
101+
Map<String, Object> config = new HashMap<>();
102+
config.put("fields", Collections.singletonList("_field"));
103+
config.put("target_field", "_target");
104+
config.put("key_setting", "invalid");
105+
config.put("method", HashProcessor.Method.SHA1.toString());
106+
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
107+
() -> factory.create(null, "_tag", new HashMap<>(config)));
108+
assertThat(e.getMessage(), equalTo("key [invalid] must match [xpack.security.ingest.hash.*.key] but didn't."));
109+
config.remove("key_setting");
110+
ElasticsearchException ex = expectThrows(ElasticsearchException.class,
111+
() -> factory.create(null, "_tag", config));
112+
assertThat(ex.getMessage(), equalTo("[key_setting] required property is missing"));
113+
}
114+
}

0 commit comments

Comments
 (0)