From 25dcb2306c332a37c31918c60acc147aa8e740e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Thu, 25 Apr 2019 15:27:31 +0200 Subject: [PATCH 1/5] HDDS-1469. Generate default configuration fragments based on annotations. Contributed by Elek, Marton. --- hadoop-hdds/common/pom.xml | 5 + .../hadoop/hdds/conf/OzoneConfiguration.java | 27 +++- .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 12 -- .../src/main/resources/ozone-default.xml | 38 ------ .../hadoop/hdds/conf/SimpleConfiguration.java | 10 +- .../hdds/conf/TestOzoneConfiguration.java | 4 +- hadoop-hdds/config/pom.xml | 66 +++++++++ .../org/apache/hadoop/hdds/conf/Config.java | 12 ++ .../hadoop/hdds/conf/ConfigFileAppender.java | 127 ++++++++++++++++++ .../hadoop/hdds/conf/ConfigFileGenerator.java | 115 ++++++++++++++++ .../apache/hadoop/hdds/conf/ConfigGroup.java | 0 .../apache/hadoop/hdds/conf/ConfigTag.java | 44 ++++++ .../apache/hadoop/hdds/conf/ConfigType.java | 0 .../hdds/conf/ConfigurationException.java | 2 +- .../javax.annotation.processing.Processor | 1 + .../hdds/conf/ConfigurationExample.java | 83 ++++++++++++ .../hdds/conf/TestConfigFileAppender.java | 28 ++++ hadoop-hdds/pom.xml | 7 + .../scm/container/ReplicationManager.java | 35 ++++- .../scm/container/TestReplicationManager.java | 14 ++ 20 files changed, 564 insertions(+), 66 deletions(-) create mode 100644 hadoop-hdds/config/pom.xml rename hadoop-hdds/{common => config}/src/main/java/org/apache/hadoop/hdds/conf/Config.java (88%) create mode 100644 hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileAppender.java create mode 100644 hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileGenerator.java rename hadoop-hdds/{common => config}/src/main/java/org/apache/hadoop/hdds/conf/ConfigGroup.java (100%) create mode 100644 hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java rename hadoop-hdds/{common => config}/src/main/java/org/apache/hadoop/hdds/conf/ConfigType.java (100%) rename hadoop-hdds/{common => config}/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationException.java (95%) create mode 100644 hadoop-hdds/config/src/main/resources/META-INF/services/javax.annotation.processing.Processor create mode 100644 hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/ConfigurationExample.java create mode 100644 hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/TestConfigFileAppender.java diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml index 5d1bb52844dff..51560ca3af92d 100644 --- a/hadoop-hdds/common/pom.xml +++ b/hadoop-hdds/common/pom.xml @@ -36,6 +36,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> + + org.apache.hadoop + hadoop-hdds-config + + javax.annotation javax.annotation-api diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java index b4dc94a2e3639..99477d48fe946 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java @@ -18,9 +18,6 @@ package org.apache.hadoop.hdds.conf; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; - import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Unmarshaller; @@ -28,6 +25,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URL; @@ -36,6 +34,9 @@ import java.util.List; import java.util.Properties; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; + /** * Configuration for ozone. */ @@ -47,12 +48,31 @@ public class OzoneConfiguration extends Configuration { public OzoneConfiguration() { OzoneConfiguration.activate(); + loadDefaults(); } public OzoneConfiguration(Configuration conf) { super(conf); //load the configuration from the classloader of the original conf. setClassLoader(conf.getClassLoader()); + loadDefaults(); + } + + private void loadDefaults() { + try { + //there could be multiple ozone-default-generated.xml files on the + // classpath, which are generated by the annotation processor. + // Here we add all of them to the list of the available configuration. + Enumeration generatedDefaults = + OzoneConfiguration.class.getClassLoader().getResources( + "ozone-default-generated.xml"); + while (generatedDefaults.hasMoreElements()) { + addResource(generatedDefaults.nextElement()); + } + } catch (IOException e) { + e.printStackTrace(); + } + addResource("ozone-site.xml"); } public List readPropertyFromXml(URL url) throws JAXBException { @@ -265,7 +285,6 @@ public static void activate() { Configuration.addDefaultResource("hdfs-default.xml"); Configuration.addDefaultResource("hdfs-site.xml"); Configuration.addDefaultResource("ozone-default.xml"); - Configuration.addDefaultResource("ozone-site.xml"); } /** diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index 4dc60b3072fc1..ccfd7ca1ddc76 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -348,18 +348,6 @@ public final class ScmConfigKeys { public static final String HDDS_SCM_WATCHER_TIMEOUT_DEFAULT = "10m"; - public static final String HDDS_SCM_REPLICATION_THREAD_INTERVAL = - "hdds.scm.replication.thread.interval"; - - public static final String HDDS_SCM_REPLICATION_THREAD_INTERVAL_DEFAULT = - "5m"; - - public static final String HDDS_SCM_REPLICATION_EVENT_TIMEOUT = - "hdds.scm.replication.event.timeout"; - - public static final String HDDS_SCM_REPLICATION_EVENT_TIMEOUT_DEFAULT = - "10m"; - public static final String HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY = "hdds.scm.http.kerberos.principal"; diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 2cab7f383edad..fc9b7576365ca 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -970,25 +970,7 @@ Timeout value for the RPC from Datanode to SCM. - - ozone.scm.heartbeat.thread.interval - 3s - OZONE, MANAGEMENT - - When a heartbeat from the data node arrives on SCM, It is queued for - processing with the time stamp of when the heartbeat arrived. There is a - heartbeat processing thread inside SCM that runs at a specified interval. - This value controls how frequently this thread is run. - - There are some assumptions build into SCM such as this value should allow - the heartbeat processing thread to run at least three times more - frequently than heartbeats and at least five times more than stale node - detection time. If you specify a wrong value, SCM will gracefully refuse - to run. For more info look at the node manager tests in SCM. - In short, you don't need to change this. - - ozone.scm.http-address 0.0.0.0:9876 @@ -2385,26 +2367,6 @@ Request to flush the OM DB before taking checkpoint snapshot. - - hdds.scm.replication.thread.interval - 5m - OZONE, SCM - - There is a replication monitor thread running inside SCM which - takes care of replicating the containers in the cluster. This - property is used to configure the interval in which that thread - runs. - - - - hdds.scm.replication.event.timeout - 10m - OZONE, SCM - - Timeout for the container replication/deletion commands sent - to datanodes. After this timeout the command will be retried. - - hdds.tracing.enabled true diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/SimpleConfiguration.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/SimpleConfiguration.java index ac696b3640ace..cb8724796d2e8 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/SimpleConfiguration.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/SimpleConfiguration.java @@ -35,28 +35,28 @@ public class SimpleConfiguration { private long waitTime = 1; - @Config(key = "address") + @Config(key = "address", defaultValue = "localhost") public void setClientAddress(String clientAddress) { this.clientAddress = clientAddress; } - @Config(key = "bind.host") + @Config(key = "bind.host", defaultValue = "0.0.0.0") public void setBindHost(String bindHost) { this.bindHost = bindHost; } - @Config(key = "enabled") + @Config(key = "enabled", defaultValue = "true") public void setEnabled(boolean enabled) { this.enabled = enabled; } - @Config(key = "port") + @Config(key = "port", defaultValue = "9878") public void setPort(int port) { this.port = port; } @Config(key = "wait", type = ConfigType.TIME, timeUnit = - TimeUnit.SECONDS) + TimeUnit.SECONDS, defaultValue = "10m") public void setWaitTime(long waitTime) { this.waitTime = waitTime; } diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java index bf8ac04145680..0a8047837aa2b 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java @@ -124,8 +124,8 @@ public void getConfigurationObjectWithDefault() { SimpleConfiguration configuration = ozoneConfiguration.getObject(SimpleConfiguration.class); - Assert.assertEquals(false, configuration.isEnabled()); - Assert.assertEquals(9860, configuration.getPort()); + Assert.assertEquals(true, configuration.isEnabled()); + Assert.assertEquals(9878, configuration.getPort()); } diff --git a/hadoop-hdds/config/pom.xml b/hadoop-hdds/config/pom.xml new file mode 100644 index 0000000000000..075f587a35046 --- /dev/null +++ b/hadoop-hdds/config/pom.xml @@ -0,0 +1,66 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-hdds + 0.5.0-SNAPSHOT + + hadoop-hdds-config + 0.5.0-SNAPSHOT + Apache Hadoop Distributed Data Store Config Tools + Apache Hadoop HDDS Config + jar + + + + + + + + + junit + junit + test + + + + + + + + maven-compiler-plugin + 3.1 + + + default-compile + compile + + compile + + + + -proc:none + + + + + + + diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/Config.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/Config.java similarity index 88% rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/Config.java rename to hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/Config.java index 2d1e18a1c0cbd..15b60ad4de7a0 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/Config.java +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/Config.java @@ -35,6 +35,16 @@ */ String key(); + /** + * Default value to use if not set. + */ + String defaultValue(); + + /** + * Custom description as a help. + */ + String description() default ""; + /** * Type of configuration. Use AUTO to decide it based on the java type. */ @@ -44,4 +54,6 @@ * If type == TIME the unit should be defined with this attribute. */ TimeUnit timeUnit() default TimeUnit.MILLISECONDS; + + ConfigTag[] tags() default {ConfigTag.OZONE}; } diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileAppender.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileAppender.java new file mode 100644 index 0000000000000..d88554acea5e2 --- /dev/null +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileAppender.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.conf; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import java.io.InputStream; +import java.io.Writer; +import java.util.Arrays; +import java.util.stream.Collectors; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +/** + * Simple DOM based config file writer. + *

+ * This class can init/load existing ozone-site.xml fragments and append + * new entries and write to the file system. + */ +public class ConfigFileAppender { + + private Document document; + + private final DocumentBuilder builder; + + public ConfigFileAppender() { + try { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + builder = factory.newDocumentBuilder(); + } catch (Exception ex) { + throw new ConfigurationException("Can initialize new configuration", ex); + } + } + + /** + * Initialize a new ozone-site.xml structure with empty content. + */ + public void init() { + try { + document = builder.newDocument(); + document.appendChild(document.createElement("configuration")); + } catch (Exception ex) { + throw new ConfigurationException("Can initialize new configuration", ex); + } + } + + /** + * Load existing ozone-site.xml content and parse the DOM tree. + */ + public void load(InputStream stream) { + try { + document = builder.parse(stream); + } catch (Exception ex) { + throw new ConfigurationException("Can't load existing configuration", ex); + } + } + + /** + * Add configuration fragment. + */ + public void addConfig(String key, String defaultValue, String description, + ConfigTag[] tags) { + Element root = document.getDocumentElement(); + Element propertyElement = document.createElement("property"); + + addXmlElement(propertyElement, "name", key); + + addXmlElement(propertyElement, "value", defaultValue); + + addXmlElement(propertyElement, "description", description); + + String tagsAsString = Arrays.stream(tags).map(tag -> tag.name()) + .collect(Collectors.joining(", ")); + + addXmlElement(propertyElement, "tag", tagsAsString); + + root.appendChild(propertyElement); + } + + private void addXmlElement(Element parentElement, String tagValue, + String textValue) { + Element element = document.createElement(tagValue); + element.appendChild(document.createTextNode(textValue)); + parentElement.appendChild(element); + } + + /** + * Write out the XML content to a writer. + */ + public void write(Writer writer) { + try { + TransformerFactory transformerFactory = TransformerFactory.newInstance(); + Transformer transf = transformerFactory.newTransformer(); + + transf.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); + transf.setOutputProperty(OutputKeys.INDENT, "yes"); + transf + .setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2"); + + transf.transform(new DOMSource(document), new StreamResult(writer)); + } catch (TransformerException e) { + throw new ConfigurationException("Can't write the configuration xml", e); + } + } +} diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileGenerator.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileGenerator.java new file mode 100644 index 0000000000000..a6a5c9ac34af4 --- /dev/null +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileGenerator.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.conf; + +import javax.annotation.processing.AbstractProcessor; +import javax.annotation.processing.Filer; +import javax.annotation.processing.RoundEnvironment; +import javax.annotation.processing.SupportedAnnotationTypes; +import javax.lang.model.element.Element; +import javax.lang.model.element.ElementKind; +import javax.lang.model.element.TypeElement; +import javax.tools.Diagnostic.Kind; +import javax.tools.FileObject; +import javax.tools.StandardLocation; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.util.Set; + +/** + * Annotation processor to generate ozone-site-generated fragments from + * ozone-site.xml. + */ +@SupportedAnnotationTypes("org.apache.hadoop.hdds.conf.ConfigGroup") +public class ConfigFileGenerator extends AbstractProcessor { + + public static final String OUTPUT_FILE_NAME = "ozone-default-generated.xml"; + + @Override + public boolean process(Set annotations, + RoundEnvironment roundEnv) { + if (roundEnv.processingOver()) { + return false; + } + + Filer filer = processingEnv.getFiler(); + System.out.println("round"); + + try { + + //load existing generated config (if exists) + ConfigFileAppender appender = new ConfigFileAppender(); + try (InputStream input = filer + .getResource(StandardLocation.CLASS_OUTPUT, "", + OUTPUT_FILE_NAME).openInputStream()) { + appender.load(input); + } catch (FileNotFoundException ex) { + appender.init(); + } + + Set annotatedElements = + roundEnv.getElementsAnnotatedWith(ConfigGroup.class); + for (Element annotatedElement : annotatedElements) { + TypeElement configGroup = (TypeElement) annotatedElement; + + //check if any of the setters are annotated with @Config + for (Element element : configGroup.getEnclosedElements()) { + if (element.getKind() == ElementKind.METHOD) { + processingEnv.getMessager() + .printMessage(Kind.WARNING, element.getSimpleName().toString()); + if (element.getSimpleName().toString().startsWith("set") + && element.getAnnotation(Config.class) != null) { + + //update the ozone-site-generated.xml + Config configAnnotation = element.getAnnotation(Config.class); + ConfigGroup configGroupAnnotation = + configGroup.getAnnotation(ConfigGroup.class); + + String key = configGroupAnnotation.prefix() + "." + + configAnnotation.key(); + + appender.addConfig(key, + configAnnotation.defaultValue(), + configAnnotation.description(), + configAnnotation.tags()); + } + } + + } + FileObject resource = filer + .createResource(StandardLocation.CLASS_OUTPUT, "", + OUTPUT_FILE_NAME); + + try (Writer writer = new OutputStreamWriter( + resource.openOutputStream(), StandardCharsets.UTF_8)) { + appender.write(writer); + } + } + } catch (IOException e) { + processingEnv.getMessager().printMessage(Kind.ERROR, + "Can't generate the config file from annotation: " + e.getMessage()); + } + return false; + } + + +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/ConfigGroup.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigGroup.java similarity index 100% rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/ConfigGroup.java rename to hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigGroup.java diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java new file mode 100644 index 0000000000000..de50d2afe9eaf --- /dev/null +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.conf; + +/** + * Available config tags. + *

+ * Note: the values are defined in ozone-default.xml by hadoop.tags.custom. + */ +public enum ConfigTag { + OZONE, + MANAGEMENT, + SECURITY, + PERFORMANCE, + DEBUG, + CLIENT, + SERVER, + OM, + SCM, + CRITICAL, + RATIS, + CONTAINER, + REQUIRED, + REST, + STORAGE, + PIPELINE, + STANDALONE, + S3GATEWAY +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/ConfigType.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigType.java similarity index 100% rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/ConfigType.java rename to hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigType.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationException.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationException.java similarity index 95% rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationException.java rename to hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationException.java index 9c6b213fe9951..2e680126a0917 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationException.java +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationException.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdds.conf; /** - * Exeception to throw in case of a configuration problem. + * Exception to throw in case of a configuration problem. */ public class ConfigurationException extends RuntimeException { public ConfigurationException() { diff --git a/hadoop-hdds/config/src/main/resources/META-INF/services/javax.annotation.processing.Processor b/hadoop-hdds/config/src/main/resources/META-INF/services/javax.annotation.processing.Processor new file mode 100644 index 0000000000000..fa9740d164e8e --- /dev/null +++ b/hadoop-hdds/config/src/main/resources/META-INF/services/javax.annotation.processing.Processor @@ -0,0 +1 @@ +org.apache.hadoop.hdds.conf.ConfigFileGenerator diff --git a/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/ConfigurationExample.java b/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/ConfigurationExample.java new file mode 100644 index 0000000000000..8ef81913dbfdb --- /dev/null +++ b/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/ConfigurationExample.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.conf; + +import java.util.concurrent.TimeUnit; + +/** + * Example configuration to test the configuration injection. + */ +@ConfigGroup(prefix = "ozone.scm.client") +public class ConfigurationExample { + + private String clientAddress; + + private String bindHost; + + private boolean enabled; + + private int port = 1234; + + private long waitTime = 1; + + @Config(key = "address", defaultValue = "localhost") + public void setClientAddress(String clientAddress) { + this.clientAddress = clientAddress; + } + + @Config(key = "bind.host", defaultValue = "0.0.0.0") + public void setBindHost(String bindHost) { + this.bindHost = bindHost; + } + + @Config(key = "enabled", defaultValue = "true") + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + @Config(key = "port", defaultValue = "1234") + public void setPort(int port) { + this.port = port; + } + + @Config(key = "wait", type = ConfigType.TIME, timeUnit = + TimeUnit.SECONDS, defaultValue = "30m") + public void setWaitTime(long waitTime) { + this.waitTime = waitTime; + } + + public String getClientAddress() { + return clientAddress; + } + + public String getBindHost() { + return bindHost; + } + + public boolean isEnabled() { + return enabled; + } + + public int getPort() { + return port; + } + + public long getWaitTime() { + return waitTime; + } +} diff --git a/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/TestConfigFileAppender.java b/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/TestConfigFileAppender.java new file mode 100644 index 0000000000000..6b6c0e2e3e007 --- /dev/null +++ b/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/TestConfigFileAppender.java @@ -0,0 +1,28 @@ +package org.apache.hadoop.hdds.conf; + +import java.io.StringWriter; + +import org.junit.Assert; +import org.junit.Test; + +public class TestConfigFileAppender { + + @Test + public void testInit() { + ConfigFileAppender appender = new ConfigFileAppender(); + + appender.init(); + + appender.addConfig("hadoop.scm.enabled", "true", "desc", + new ConfigTag[] {ConfigTag.OZONE, ConfigTag.SECURITY}); + + StringWriter builder = new StringWriter(); + appender.write(builder); + + Assert.assertTrue("Generated config should contain property key entry", + builder.toString().contains("hadoop.scm.enabled")); + + Assert.assertTrue("Generated config should contain tags", + builder.toString().contains("OZONE, SECURITY")); + } +} \ No newline at end of file diff --git a/hadoop-hdds/pom.xml b/hadoop-hdds/pom.xml index 3e8132f6b94f0..f77fe9db9f5db 100644 --- a/hadoop-hdds/pom.xml +++ b/hadoop-hdds/pom.xml @@ -38,6 +38,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> server-scm tools docs + config @@ -115,6 +116,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> ${hdds.version} + + org.apache.hadoop + hadoop-hdds-config + ${hdds.version} + + org.apache.hadoop hadoop-hdds-container-service diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java index 8f62243e7d205..4b00562d3be98 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java @@ -21,6 +21,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.GeneratedMessage; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +import org.apache.hadoop.hdds.conf.ConfigTag; import org.apache.hadoop.hdds.conf.ConfigType; import org.apache.hadoop.hdds.conf.ConfigGroup; import org.apache.hadoop.hdds.conf.Config; @@ -40,6 +42,8 @@ import org.apache.hadoop.ozone.protocol.commands.SCMCommand; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.Time; + +import static org.apache.hadoop.hdds.conf.ConfigTag.*; import org.apache.ratis.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -760,14 +764,37 @@ public static class ReplicationManagerConfiguration { */ private long eventTimeout = 10 * 60 * 1000; - @Config(key = "thread.interval", type = ConfigType.TIME, timeUnit = - TimeUnit.MILLISECONDS) + @Config(key = "thread.interval", + type = ConfigType.TIME, + defaultValue = "3s", + tags = {SCM, OZONE}, + description = "When a heartbeat from the data node arrives on SCM, " + + "It is queued for processing with the time stamp of when the " + + "heartbeat arrived. There is a heartbeat processing thread " + + "inside " + + "SCM that runs at a specified interval. This value controls how " + + "frequently this thread is run.\n\n" + + "There are some assumptions build into SCM such as this " + + "value should allow the heartbeat processing thread to run at " + + "least three times more frequently than heartbeats and at least " + + "five times more than stale node detection time. " + + "If you specify a wrong value, SCM will gracefully refuse to " + + "run. " + + "For more info look at the node manager tests in SCM.\n" + + "\n" + + "In short, you don't need to change this." + ) public void setInterval(long interval) { this.interval = interval; } - @Config(key = "event.timeout", type = ConfigType.TIME, timeUnit = - TimeUnit.MILLISECONDS) + @Config(key = "event.timeout", + type = ConfigType.TIME, + defaultValue = "10m", + tags = {SCM, OZONE}, + description = "Timeout for the container replication/deletion commands " + + "sent to datanodes. After this timeout the command will be " + + "retried.") public void setEventTimeout(long eventTimeout) { this.eventTimeout = eventTimeout; } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java index 00b468483a5fd..5da057e10ee44 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java @@ -575,6 +575,20 @@ public void testHealthyClosedContainer() Assert.assertEquals(0, datanodeCommandHandler.getInvocation()); } + @Test + public void testGeneratedConfig() { + OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + + ReplicationManagerConfiguration rmc = + ozoneConfiguration.getObject(ReplicationManagerConfiguration.class); + + //default is not included in ozone-site.xml but generated from annotation + //to the ozone-site-generated.xml which should be loaded by the + // OzoneConfiguration. + Assert.assertEquals(600000, rmc.getEventTimeout()); + + } + @After public void teardown() throws IOException { containerStateManager.close(); From cef0f91f29c557c373284e97db9f5cf121ec11fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Fri, 26 Apr 2019 12:03:50 +0200 Subject: [PATCH 2/5] address review comments --- .../hadoop/hdds/conf/SimpleConfiguration.java | 15 +++++++---- .../org/apache/hadoop/hdds/conf/Config.java | 4 +-- .../hadoop/hdds/conf/ConfigFileAppender.java | 4 +-- .../hadoop/hdds/conf/ConfigFileGenerator.java | 3 +-- .../hdds/conf/ConfigurationExample.java | 26 ++++++++++++------- 5 files changed, 31 insertions(+), 21 deletions(-) diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/SimpleConfiguration.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/SimpleConfiguration.java index cb8724796d2e8..f18fd5e50b6f6 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/SimpleConfiguration.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/SimpleConfiguration.java @@ -35,28 +35,33 @@ public class SimpleConfiguration { private long waitTime = 1; - @Config(key = "address", defaultValue = "localhost") + @Config(key = "address", defaultValue = "localhost", description = "Just " + + "for testing", tags = ConfigTag.MANAGEMENT) public void setClientAddress(String clientAddress) { this.clientAddress = clientAddress; } - @Config(key = "bind.host", defaultValue = "0.0.0.0") + @Config(key = "bind.host", defaultValue = "0.0.0.0", description = "Just " + + "for testing", tags = ConfigTag.MANAGEMENT) public void setBindHost(String bindHost) { this.bindHost = bindHost; } - @Config(key = "enabled", defaultValue = "true") + @Config(key = "enabled", defaultValue = "true", description = "Just for " + + "testing", tags = ConfigTag.MANAGEMENT) public void setEnabled(boolean enabled) { this.enabled = enabled; } - @Config(key = "port", defaultValue = "9878") + @Config(key = "port", defaultValue = "9878", description = "Just for " + + "testing", tags = ConfigTag.MANAGEMENT) public void setPort(int port) { this.port = port; } @Config(key = "wait", type = ConfigType.TIME, timeUnit = - TimeUnit.SECONDS, defaultValue = "10m") + TimeUnit.SECONDS, defaultValue = "10m", description = "Just for " + + "testing", tags = ConfigTag.MANAGEMENT) public void setWaitTime(long waitTime) { this.waitTime = waitTime; } diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/Config.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/Config.java index 15b60ad4de7a0..70aa58d541756 100644 --- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/Config.java +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/Config.java @@ -43,7 +43,7 @@ /** * Custom description as a help. */ - String description() default ""; + String description(); /** * Type of configuration. Use AUTO to decide it based on the java type. @@ -55,5 +55,5 @@ */ TimeUnit timeUnit() default TimeUnit.MILLISECONDS; - ConfigTag[] tags() default {ConfigTag.OZONE}; + ConfigTag[] tags(); } diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileAppender.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileAppender.java index d88554acea5e2..9463f42909564 100644 --- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileAppender.java +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileAppender.java @@ -36,8 +36,8 @@ /** * Simple DOM based config file writer. *

- * This class can init/load existing ozone-site.xml fragments and append - * new entries and write to the file system. + * This class can init/load existing ozone-default-generated.xml fragments + * and append new entries and write to the file system. */ public class ConfigFileAppender { diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileGenerator.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileGenerator.java index a6a5c9ac34af4..76cf1f2440be8 100644 --- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileGenerator.java +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileGenerator.java @@ -37,7 +37,7 @@ /** * Annotation processor to generate ozone-site-generated fragments from - * ozone-site.xml. + * @Config annotations. */ @SupportedAnnotationTypes("org.apache.hadoop.hdds.conf.ConfigGroup") public class ConfigFileGenerator extends AbstractProcessor { @@ -52,7 +52,6 @@ public boolean process(Set annotations, } Filer filer = processingEnv.getFiler(); - System.out.println("round"); try { diff --git a/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/ConfigurationExample.java b/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/ConfigurationExample.java index 8ef81913dbfdb..2dd26696b2760 100644 --- a/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/ConfigurationExample.java +++ b/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/ConfigurationExample.java @@ -29,34 +29,40 @@ public class ConfigurationExample { private String bindHost; - private boolean enabled; + private boolean compressionEnabled; private int port = 1234; private long waitTime = 1; - @Config(key = "address", defaultValue = "localhost") + @Config(key = "address", defaultValue = "localhost", description = "Client " + + "addres (To test string injection).", tags = ConfigTag.MANAGEMENT) public void setClientAddress(String clientAddress) { this.clientAddress = clientAddress; } - @Config(key = "bind.host", defaultValue = "0.0.0.0") + @Config(key = "bind.host", defaultValue = "0.0.0.0", description = "Bind " + + "host(To test string injection).", tags = ConfigTag.MANAGEMENT) public void setBindHost(String bindHost) { this.bindHost = bindHost; } - @Config(key = "enabled", defaultValue = "true") - public void setEnabled(boolean enabled) { - this.enabled = enabled; + @Config(key = "compression.enabled", defaultValue = "true", description = + "Compression enabled. (Just to test boolean flag)", tags = + ConfigTag.MANAGEMENT) + public void setCompressionEnabled(boolean compressionEnabled) { + this.compressionEnabled = compressionEnabled; } - @Config(key = "port", defaultValue = "1234") + @Config(key = "port", defaultValue = "1234", description = "Port number " + + "config (To test in injection)", tags = ConfigTag.MANAGEMENT) public void setPort(int port) { this.port = port; } @Config(key = "wait", type = ConfigType.TIME, timeUnit = - TimeUnit.SECONDS, defaultValue = "30m") + TimeUnit.SECONDS, defaultValue = "30m", description = "Wait time (To " + + "test TIME config type)", tags = ConfigTag.MANAGEMENT) public void setWaitTime(long waitTime) { this.waitTime = waitTime; } @@ -69,8 +75,8 @@ public String getBindHost() { return bindHost; } - public boolean isEnabled() { - return enabled; + public boolean isCompressionEnabled() { + return compressionEnabled; } public int getPort() { From b3474833c5571522a5dc7110079deeb525c5f92f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Fri, 26 Apr 2019 12:05:43 +0200 Subject: [PATCH 3/5] RAT and javadoc fixes --- .../apache/hadoop/hdds/conf/package-info.java | 22 +++++++++++++++++ .../javax.annotation.processing.Processor | 15 ++++++++++++ .../hdds/conf/TestConfigFileAppender.java | 20 ++++++++++++++++ .../apache/hadoop/hdds/conf/package-info.java | 24 +++++++++++++++++++ .../scm/container/ReplicationManager.java | 2 -- 5 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/package-info.java create mode 100644 hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/package-info.java diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/package-info.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/package-info.java new file mode 100644 index 0000000000000..e789040d276d2 --- /dev/null +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Generic configuration annotations, tools and generators. + */ +package org.apache.hadoop.hdds.conf; diff --git a/hadoop-hdds/config/src/main/resources/META-INF/services/javax.annotation.processing.Processor b/hadoop-hdds/config/src/main/resources/META-INF/services/javax.annotation.processing.Processor index fa9740d164e8e..f29efdab384d1 100644 --- a/hadoop-hdds/config/src/main/resources/META-INF/services/javax.annotation.processing.Processor +++ b/hadoop-hdds/config/src/main/resources/META-INF/services/javax.annotation.processing.Processor @@ -1 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + org.apache.hadoop.hdds.conf.ConfigFileGenerator diff --git a/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/TestConfigFileAppender.java b/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/TestConfigFileAppender.java index 6b6c0e2e3e007..0edb01a02b403 100644 --- a/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/TestConfigFileAppender.java +++ b/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/TestConfigFileAppender.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hdds.conf; import java.io.StringWriter; @@ -5,6 +22,9 @@ import org.junit.Assert; import org.junit.Test; +/** + * Test the utility which loads/writes the config file fragments. + */ public class TestConfigFileAppender { @Test diff --git a/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/package-info.java b/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/package-info.java new file mode 100644 index 0000000000000..e8b310d109c23 --- /dev/null +++ b/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/package-info.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *

+ * Testing configuration tools. + */ + +/** + * Testing configuration tools. + */ +package org.apache.hadoop.hdds.conf; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java index 4b00562d3be98..e247e96cd99ce 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java @@ -22,7 +22,6 @@ import com.google.protobuf.GeneratedMessage; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import org.apache.hadoop.hdds.conf.ConfigTag; import org.apache.hadoop.hdds.conf.ConfigType; import org.apache.hadoop.hdds.conf.ConfigGroup; import org.apache.hadoop.hdds.conf.Config; @@ -58,7 +57,6 @@ import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.stream.Collectors; From 3a1991edc43223183291d0b56646f5e61899d87a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Fri, 26 Apr 2019 12:14:31 +0200 Subject: [PATCH 4/5] reestore ozone.scm.heartbeat.thread.interval --- .../src/main/resources/ozone-default.xml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index fc9b7576365ca..6aafb58a3bf48 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -970,7 +970,25 @@ Timeout value for the RPC from Datanode to SCM. + + ozone.scm.heartbeat.thread.interval + 3s + OZONE, MANAGEMENT + + When a heartbeat from the data node arrives on SCM, It is queued for + processing with the time stamp of when the heartbeat arrived. There is a + heartbeat processing thread inside SCM that runs at a specified interval. + This value controls how frequently this thread is run. + There are some assumptions build into SCM such as this value should allow + the heartbeat processing thread to run at least three times more + frequently than heartbeats and at least five times more than stale node + detection time. If you specify a wrong value, SCM will gracefully refuse + to run. For more info look at the node manager tests in SCM. + + In short, you don't need to change this. + + ozone.scm.http-address 0.0.0.0:9876 From 61a2bbc205699695429e9c085922cbd0b0d3796e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Fri, 26 Apr 2019 12:27:35 +0200 Subject: [PATCH 5/5] fix unit tests --- .../hadoop/hdds/conf/OzoneConfiguration.java | 4 +++- .../hadoop/hdds/conf/ConfigFileGenerator.java | 3 +-- .../src/test/resources/core-site.xml | 24 +++++++++++++++++++ .../src/test/resources/hdfs-site.xml | 24 +++++++++++++++++++ 4 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 hadoop-ozone/integration-test/src/test/resources/core-site.xml create mode 100644 hadoop-ozone/integration-test/src/test/resources/hdfs-site.xml diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java index 99477d48fe946..b32ad63353553 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java @@ -55,7 +55,9 @@ public OzoneConfiguration(Configuration conf) { super(conf); //load the configuration from the classloader of the original conf. setClassLoader(conf.getClassLoader()); - loadDefaults(); + if (!(conf instanceof OzoneConfiguration)) { + loadDefaults(); + } } private void loadDefaults() { diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileGenerator.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileGenerator.java index 76cf1f2440be8..e9e88a0898805 100644 --- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileGenerator.java +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileGenerator.java @@ -36,8 +36,7 @@ import java.util.Set; /** - * Annotation processor to generate ozone-site-generated fragments from - * @Config annotations. + * Annotation processor to generate config fragments from Config annotations. */ @SupportedAnnotationTypes("org.apache.hadoop.hdds.conf.ConfigGroup") public class ConfigFileGenerator extends AbstractProcessor { diff --git a/hadoop-ozone/integration-test/src/test/resources/core-site.xml b/hadoop-ozone/integration-test/src/test/resources/core-site.xml new file mode 100644 index 0000000000000..77dd7ef994026 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/resources/core-site.xml @@ -0,0 +1,24 @@ + + + + + + + + + diff --git a/hadoop-ozone/integration-test/src/test/resources/hdfs-site.xml b/hadoop-ozone/integration-test/src/test/resources/hdfs-site.xml new file mode 100644 index 0000000000000..77dd7ef994026 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/resources/hdfs-site.xml @@ -0,0 +1,24 @@ + + + + + + + + +