HDDS-1382. Create customized CSI server for Ozone.

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java

@@ -226,7 +226,12 @@ public static Optional<String> getHostName(String value) {
     if ((value == null) || value.isEmpty()) {
       return Optional.empty();
     }
-    return Optional.of(HostAndPort.fromString(value).getHostText());
+    String hostname = value.replaceAll("\\:[0-9]+$", "");
+    if (hostname.length() == 0) {
+      return Optional.empty();
+    } else {
+      return Optional.of(hostname);
+    }
   }
 
   /**

hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java

@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds;
+
+import java.util.Optional;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Testing HddsUtils.
+ */
+public class TestHddsUtils {
+
+  @Test
+  public void testGetHostName() {
+    Assert.assertEquals(Optional.of("localhost"),
+        HddsUtils.getHostName("localhost:1234"));
+
+    Assert.assertEquals(Optional.of("localhost"),
+        HddsUtils.getHostName("localhost"));
+
+    Assert.assertEquals(Optional.empty(),
+        HddsUtils.getHostName(":1234"));
+  }
+
+}

hadoop-ozone/common/src/main/bin/ozone

@@ -46,6 +46,7 @@ function hadoop_usage
   hadoop_add_subcommand "om" daemon "Ozone Manager"
   hadoop_add_subcommand "scm" daemon "run the Storage Container Manager service"
   hadoop_add_subcommand "s3g" daemon "run the S3 compatible REST gateway"
+  hadoop_add_subcommand "csi" daemon "run the standalone CSI daemon"
   hadoop_add_subcommand "recon" daemon "run the Recon service"
   hadoop_add_subcommand "scmcli" client "run the CLI of the Storage Container Manager"
   hadoop_add_subcommand "sh" client "command line interface for object store operations"
@@ -154,6 +155,11 @@ function ozonecmd_case
       HADOOP_CLASSNAME='org.apache.hadoop.ozone.s3.Gateway'
       OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-s3gateway"
     ;;
+    csi)
+      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+      HADOOP_CLASSNAME='org.apache.hadoop.ozone.csi.CsiServer'
+      OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-csi"
+    ;;
     recon)
       HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
       HADOOP_CLASSNAME='org.apache.hadoop.ozone.recon.ReconServer'

hadoop-ozone/csi/dev-support/findbugsExcludeFile.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<FindBugsFilter>
+  <Match>
+    <Package name="csi.v1"/>
+  </Match>
+</FindBugsFilter>

hadoop-ozone/csi/pom.xml

@@ -0,0 +1,169 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hadoop</groupId>
+    <artifactId>hadoop-ozone</artifactId>
+    <version>0.5.0-SNAPSHOT</version>
+  </parent>
+  <artifactId>hadoop-ozone-csi</artifactId>
+  <version>0.5.0-SNAPSHOT</version>
+  <description>Apache Hadoop Ozone CSI service</description>
+  <name>Apache Hadoop Ozone CSI service</name>
+  <packaging>jar</packaging>
+
+  <properties>
+    <grpc.version>1.17.1</grpc.version>
+  </properties>
+  <dependencies>
+    <dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java-util</artifactId>
+      <version>3.5.1</version>
+      <exclusions>
+        <exclusion>
+          <groupId>com.google.protobuf</groupId>
+          <artifactId>protobuf-java</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdds-config</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>26.0-android</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <version>3.5.1</version>
+    </dependency>
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-netty</artifactId>
+      <version>${grpc.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-transport-native-epoll</artifactId>
+      <version>4.1.30.Final</version>
+    </dependency>
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-protobuf</artifactId>
+      <version>${grpc.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>com.google.protobuf</groupId>
+          <artifactId>protobuf-java</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-stub</artifactId>
+      <version>${grpc.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-ozone-client</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.protobuf</groupId>
+          <artifactId>protobuf-java</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>io.netty</groupId>
+          <artifactId>netty-all</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+  </dependencies>
+
+
+  <build>
+    <extensions>
+      <extension>
+        <groupId>kr.motd.maven</groupId>
+        <artifactId>os-maven-plugin</artifactId>
+        <version>${os-maven-plugin.version}</version>
+      </extension>
+    </extensions>
+    <plugins>
+      <plugin>
+        <groupId>org.xolstice.maven.plugins</groupId>
+        <artifactId>protobuf-maven-plugin</artifactId>
+        <version>${protobuf-maven-plugin.version}</version>
+        <extensions>true</extensions>
+        <configuration>
+          <protocArtifact>
+            com.google.protobuf:protoc:${protobuf-compile.version}:exe:${os.detected.classifier}
+          </protocArtifact>
+          <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
+          <includes>
+            <include>csi.proto</include>
+          </includes>
+          <outputDirectory>target/generated-sources/java</outputDirectory>
+          <clearOutputDirectory>false</clearOutputDirectory>
+        </configuration>
+        <executions>
+          <execution>
+            <id>compile-protoc</id>
+            <goals>
+              <goal>compile</goal>
+              <goal>test-compile</goal>
+              <goal>compile-custom</goal>
+              <goal>test-compile-custom</goal>
+            </goals>
+            <configuration>
+              <pluginId>grpc-java</pluginId>
+              <pluginArtifact>
+                io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}
+              </pluginArtifact>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <artifactId>maven-enforcer-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>depcheck</id>
+            <phase></phase>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>findbugs-maven-plugin</artifactId>
+        <configuration>
+          <excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml
+          </excludeFilterFile>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+</project>

hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/ControllerService.java

@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.csi;
+
+import java.io.IOException;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+
+import csi.v1.ControllerGrpc.ControllerImplBase;
+import csi.v1.Csi.CapacityRange;
+import csi.v1.Csi.ControllerGetCapabilitiesRequest;
+import csi.v1.Csi.ControllerGetCapabilitiesResponse;
+import csi.v1.Csi.ControllerServiceCapability;
+import csi.v1.Csi.ControllerServiceCapability.RPC;
+import csi.v1.Csi.ControllerServiceCapability.RPC.Type;
+import csi.v1.Csi.CreateVolumeRequest;
+import csi.v1.Csi.CreateVolumeResponse;
+import csi.v1.Csi.DeleteVolumeRequest;
+import csi.v1.Csi.DeleteVolumeResponse;
+import csi.v1.Csi.Volume;
+import io.grpc.stub.StreamObserver;
+
+/**
+ * CSI controller service.
+ * <p>
+ * This service usually runs only once and responsible for the creation of
+ * the volume.
+ */
+public class ControllerService extends ControllerImplBase {
+
+  private final String volumeOwner;
+
+  private long defaultVolumeSize;
+
+  private OzoneClient ozoneClient;
+
+  public ControllerService(OzoneClient ozoneClient, long volumeSize,
+      String volumeOwner) {
+    this.volumeOwner = volumeOwner;
+    this.defaultVolumeSize = volumeSize;
+    this.ozoneClient = ozoneClient;
+  }
+
+  @Override
+  public void createVolume(CreateVolumeRequest request,
+      StreamObserver<CreateVolumeResponse> responseObserver) {
+    try {
+      ozoneClient.getObjectStore()
+          .createS3Bucket(volumeOwner, request.getName());
+
+      long size = findSize(request.getCapacityRange());
+
+      CreateVolumeResponse response = CreateVolumeResponse.newBuilder()
+          .setVolume(Volume.newBuilder()
+              .setVolumeId(request.getName())
+              .setCapacityBytes(size))
+          .build();
+
+      responseObserver.onNext(response);
+      responseObserver.onCompleted();
+    } catch (IOException e) {
+      responseObserver.onError(e);
+    }
+  }
+
+  private long findSize(CapacityRange capacityRange) {
+    if (capacityRange.getRequiredBytes() != 0) {
+      return capacityRange.getRequiredBytes();
+    } else {
+      if (capacityRange.getLimitBytes() != 0) {
+        return Math.min(defaultVolumeSize, capacityRange.getLimitBytes());
+      } else {
+        //~1 gig
+        return defaultVolumeSize;
+      }
+    }
+  }
+
+  @Override
+  public void deleteVolume(DeleteVolumeRequest request,
+      StreamObserver<DeleteVolumeResponse> responseObserver) {
+    try {
+      ozoneClient.getObjectStore().deleteS3Bucket(request.getVolumeId());
+
+      DeleteVolumeResponse response = DeleteVolumeResponse.newBuilder()
+          .build();
+
+      responseObserver.onNext(response);
+      responseObserver.onCompleted();
+    } catch (IOException e) {
+      responseObserver.onError(e);
+    }
+  }
+
+  @Override
+  public void controllerGetCapabilities(
+      ControllerGetCapabilitiesRequest request,
+      StreamObserver<ControllerGetCapabilitiesResponse> responseObserver) {
+    ControllerGetCapabilitiesResponse response =
+        ControllerGetCapabilitiesResponse.newBuilder()
+            .addCapabilities(
+                ControllerServiceCapability.newBuilder().setRpc(
+                    RPC.newBuilder().setType(Type.CREATE_DELETE_VOLUME)))
+            .build();
+    responseObserver.onNext(response);
+    responseObserver.onCompleted();
+  }
+}