HDDS-1660. Use Picocli for Ozone Manager

Closes #925

Author: S O'Donnell

hadoop-ozone/common/src/main/bin/ozone

@@ -129,7 +129,7 @@ function ozonecmd_case
     ;;
     om)
       HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
-      HADOOP_CLASSNAME=org.apache.hadoop.ozone.om.OzoneManager
+      HADOOP_CLASSNAME=org.apache.hadoop.ozone.om.OzoneManagerStarter
       HDFS_OM_OPTS="${HDFS_OM_OPTS} -Dlog4j.configurationFile=${HADOOP_CONF_DIR}/om-audit-log4j2.properties"
       HADOOP_OPTS="${HADOOP_OPTS} ${HDFS_OM_OPTS}"
       OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-ozone-manager"

hadoop-ozone/dev-support/intellij/runConfigurations/OzoneManager.xml

@@ -16,7 +16,7 @@
 -->
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="OzoneManager" type="Application" factoryName="Application">
-    <option name="MAIN_CLASS_NAME" value="org.apache.hadoop.ozone.om.OzoneManager" />
+    <option name="MAIN_CLASS_NAME" value="org.apache.hadoop.ozone.om.OzoneManagerStarter" />
     <module name="hadoop-ozone-ozone-manager" />
     <option name="PROGRAM_PARAMETERS" value="-conf=hadoop-ozone/dev-support/intellij/ozone-site.xml" />
     <option name="VM_PARAMETERS" value="-Dlog4j.configuration=file:hadoop-ozone/dev-support/intellij/log4j.properties" />
@@ -30,4 +30,4 @@
       <option name="Make" enabled="true" />
     </method>
   </configuration>
-</component>
+</component>

hadoop-ozone/dev-support/intellij/runConfigurations/OzoneManagerInit.xml

@@ -16,7 +16,7 @@
 -->
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="OzoneManagerInit" type="Application" factoryName="Application">
-    <option name="MAIN_CLASS_NAME" value="org.apache.hadoop.ozone.om.OzoneManager" />
+    <option name="MAIN_CLASS_NAME" value="org.apache.hadoop.ozone.om.OzoneManagerStarter" />
     <module name="hadoop-ozone-ozone-manager" />
     <option name="PROGRAM_PARAMETERS" value="-conf=hadoop-ozone/dev-support/intellij/ozone-site.xml --init" />
     <option name="VM_PARAMETERS" value="-Dlog4j.configuration=file:hadoop-ozone/dev-support/intellij/log4j.properties" />
@@ -30,4 +30,4 @@
       <option name="Make" enabled="true" />
     </method>
   </configuration>
-</component>
+</component>

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java

@@ -517,7 +517,7 @@ OzoneManager createOM()
       configureOM();
       OMStorage omStore = new OMStorage(conf);
       initializeOmStorage(omStore);
-      return OzoneManager.createOm(null, conf);
+      return OzoneManager.createOm(conf);
     }
 
     /**

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java

@@ -211,7 +211,7 @@ private Map<String, OzoneManager> createOMService() throws IOException,
             OMStorage omStore = new OMStorage(conf);
             initializeOmStorage(omStore);
 
-            OzoneManager om = OzoneManager.createOm(null, conf);
+            OzoneManager om = OzoneManager.createOm(conf);
             om.setCertClient(certClient);
             omMap.put(nodeId, om);
 

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java

@@ -383,7 +383,7 @@ public void testSecureOMInitializationFailure() throws Exception {
     setupOm(conf);
     conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
         "[email protected]");
-    testCommonKerberosFailures(() -> OzoneManager.createOm(null, conf));
+    testCommonKerberosFailures(() -> OzoneManager.createOm(conf));
   }
 
   /**
@@ -662,7 +662,7 @@ private void setupOm(OzoneConfiguration config) throws Exception {
     // writes the version file properties
     omStore.initialize();
     OzoneManager.setTestSecureOmFlag(true);
-    om = OzoneManager.createOm(null, config);
+    om = OzoneManager.createOm(config);
   }
 
   @Test
@@ -717,6 +717,26 @@ public void testSecureOmReInit() throws Exception {
     LogCapturer omLogs =
         LogCapturer.captureLogs(OzoneManager.getLogger());
     omLogs.clearOutput();
+
+    /**
+     * As all these processes run inside the same JVM, there are issues around
+     * the Hadoop UGI if different processes run with different principals.
+     * In this test, the OM has to contact the SCM to download certs. SCM runs
+     * as scm/host@REALM, but the OM logs in as om/host@REALM, and then the test
+     * fails, and the OM is unable to contact the SCM due to kerberos login
+     * issues. To work around that, have the OM run as the same principal as the
+     * SCM, and then the test passes.
+     *
+     * TODO: Need to look into this further to see if there is a better way to
+     *       address this problem.
+     */
+    String realm = miniKdc.getRealm();
+    conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
+        "scm/" + host + "@" + realm);
+    omKeyTab = new File(workDir, "scm.keytab");
+    conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
+        omKeyTab.getAbsolutePath());
+
     initSCM();
     try {
       scm = StorageContainerManager.createSCM(conf);
@@ -725,7 +745,7 @@ public void testSecureOmReInit() throws Exception {
       OMStorage omStore = new OMStorage(conf);
       initializeOmStorage(omStore);
       OzoneManager.setTestSecureOmFlag(true);
-      om = OzoneManager.createOm(null, conf);
+      om = OzoneManager.createOm(conf);
 
       assertNull(om.getCertificateClient());
       assertFalse(omLogs.getOutput().contains("Init response: GETCERT"));
@@ -735,7 +755,7 @@ public void testSecureOmReInit() throws Exception {
       conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
       OzoneManager.omInit(conf);
       om.stop();
-      om = OzoneManager.createOm(null, conf);
+      om = OzoneManager.createOm(conf);
 
       Assert.assertNotNull(om.getCertificateClient());
       Assert.assertNotNull(om.getCertificateClient().getPublicKey());
@@ -771,7 +791,7 @@ public void testSecureOmInitSuccess() throws Exception {
       OMStorage omStore = new OMStorage(conf);
       initializeOmStorage(omStore);
       OzoneManager.setTestSecureOmFlag(true);
-      om = OzoneManager.createOm(null, conf);
+      om = OzoneManager.createOm(conf);
 
       Assert.assertNotNull(om.getCertificateClient());
       Assert.assertNotNull(om.getCertificateClient().getPublicKey());

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmInit.java

@@ -23,6 +23,7 @@
 import org.apache.hadoop.ozone.web.handlers.UserArgs;
 import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
 import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -90,10 +91,11 @@ public static void shutdown() {
 
   /**
    * Tests the OM Initialization.
-   * @throws IOException
+   * @throws IOException, AuthenticationException
    */
   @Test
-  public void testOmInitAgain() throws IOException {
+  public void testOmInitAgain() throws IOException,
+      AuthenticationException {
     // Stop the Ozone Manager
     cluster.getOzoneManager().stop();
     // Now try to init the OM again. It should succeed

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java

@@ -1326,7 +1326,7 @@ public void testOmInitializationFailure() throws Exception {
         conf.get(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY));
 
     OzoneTestUtils.expectOmException(ResultCodes.OM_NOT_INITIALIZED, () -> {
-      OzoneManager.createOm(null, config);
+      OzoneManager.createOm(config);
     });
 
     OzoneTestUtils
@@ -1336,7 +1336,7 @@ public void testOmInitializationFailure() throws Exception {
           omStore.setScmId("testScmId");
           // writes the version file properties
           omStore.initialize();
-          OzoneManager.createOm(null, config);
+          OzoneManager.createOm(config);
         });
   }
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStarterInterface.java

@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import java.io.IOException;
+
+/**
+ * This interface is used by the OzoneManagerStarter class to allow the
+ * dependencies to be injected to the CLI class.
+ */
+public interface OMStarterInterface {
+  void start(OzoneConfiguration conf) throws IOException,
+      AuthenticationException;
+  boolean init(OzoneConfiguration conf) throws IOException,
+      AuthenticationException;
+}