aclList = new ArrayList<>();
-
- aclList.add(OzoneAcl.parseAcl("user:bilbo:r"));
- aclList.add(OzoneAcl.parseAcl("user:samwise:rw"));
- newBucketInfo.setAcls(aclList);
-
- assertNotNull(newBucketInfo.getAcls());
- assertEquals(2, newBucketInfo.getAcls().size());
- }
-
- @Test
- public void testBucketInfoVersionAndType() throws IOException {
- BucketInfo bucketInfo = new BucketInfo("volumeName", "bucketName");
- bucketInfo.setVersioning(OzoneConsts.Versioning.ENABLED);
- bucketInfo.setStorageType(StorageType.DISK);
-
- String bucketInfoString = bucketInfo.toDBString();
-
- BucketInfo newBucketInfo = BucketInfo.parse(bucketInfoString);
- assertEquals(bucketInfo, newBucketInfo);
- }
-
-}
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/web/TestQuota.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/web/TestQuota.java
deleted file mode 100644
index ba4a5acf0c6be..0000000000000
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/web/TestQuota.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.web;
-
-import org.apache.hadoop.ozone.web.request.OzoneQuota;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Set;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * Test Ozone Volume Quota.
- */
-public class TestQuota {
- @Test
- public void testParseQuota() {
- HashMap testMatrix;
- testMatrix = new HashMap();
-
- testMatrix.put("10TB", Boolean.TRUE);
- testMatrix.put("1 TB", Boolean.TRUE);
- testMatrix.put("0MB", Boolean.TRUE);
- testMatrix.put("0 TB", Boolean.TRUE);
- testMatrix.put(" 1000MB ", Boolean.TRUE);
-
- testMatrix.put(" 1000MBMB ", Boolean.FALSE);
- testMatrix.put(" 1000MB00 ", Boolean.FALSE);
- testMatrix.put("1000ZMB", Boolean.FALSE);
- testMatrix.put("MB1000", Boolean.FALSE);
- testMatrix.put("9999", Boolean.FALSE);
- testMatrix.put("1", Boolean.FALSE);
- testMatrix.put("remove", Boolean.FALSE);
- testMatrix.put("1UNDEFINED", Boolean.FALSE);
- testMatrix.put(null, Boolean.FALSE);
- testMatrix.put("", Boolean.FALSE);
- testMatrix.put("-1000MB", Boolean.FALSE);
- testMatrix.put("1024 bytes", Boolean.TRUE);
- testMatrix.put("1bytes", Boolean.TRUE);
- testMatrix.put("0bytes", Boolean.TRUE);
- testMatrix.put("10000 BYTES", Boolean.TRUE);
- testMatrix.put("BYTESbytes", Boolean.FALSE);
- testMatrix.put("bytes", Boolean.FALSE);
-
- Set keys = testMatrix.keySet();
- for (String key : keys) {
- if (testMatrix.get(key)) {
- OzoneQuota.parseQuota(key);
- } else {
- try {
- OzoneQuota.parseQuota(key);
- // should never get here since the isValid call will throw
- fail(key);
- fail("An exception was expected but did not happen.");
- } catch (IllegalArgumentException e) {
-
- }
- }
- }
- }
-
- @Test
- public void testVerifyQuota() {
- OzoneQuota qt = OzoneQuota.parseQuota("10TB");
- assertEquals(10, qt.getSize());
- assertEquals(OzoneQuota.Units.TB, qt.getUnit());
- assertEquals(10L * (1024L * 1024L * 1024L * 1024L), qt.sizeInBytes());
-
- qt = OzoneQuota.parseQuota("10MB");
- assertEquals(10, qt.getSize());
- assertEquals(OzoneQuota.Units.MB, qt.getUnit());
- assertEquals(10L * (1024L * 1024L), qt.sizeInBytes());
-
- qt = OzoneQuota.parseQuota("10GB");
- assertEquals(10, qt.getSize());
- assertEquals(OzoneQuota.Units.GB, qt.getUnit());
- assertEquals(10L * (1024L * 1024L * 1024L), qt.sizeInBytes());
-
- qt = OzoneQuota.parseQuota("10BYTES");
- assertEquals(10, qt.getSize());
- assertEquals(OzoneQuota.Units.BYTES, qt.getUnit());
- assertEquals(10L, qt.sizeInBytes());
-
- OzoneQuota emptyQuota = new OzoneQuota();
- assertEquals(-1L, emptyQuota.sizeInBytes());
- assertEquals(0, emptyQuota.getSize());
- assertEquals(OzoneQuota.Units.UNDEFINED, emptyQuota.getUnit());
- }
-
- @Test
- public void testVerifyRemove() {
- assertTrue(OzoneQuota.isRemove("remove"));
- assertFalse(OzoneQuota.isRemove("not remove"));
- assertFalse(OzoneQuota.isRemove(null));
- }
-}
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/web/TestUtils.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/web/TestUtils.java
deleted file mode 100644
index d3f8f5e659ca8..0000000000000
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/web/TestUtils.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web;
-
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Set;
-
-import static org.apache.hadoop.ozone.web.utils.OzoneUtils.getRequestID;
-import static org.apache.hadoop.ozone.web.utils.OzoneUtils.verifyResourceName;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * Test Ozone Utility operations like verifying resource name.
- */
-public class TestUtils {
-
- /**
- * Tests if the bucket name handling is correct.
- */
- @Test
- public void testValidBucketNames() {
- HashMap testMatrix;
- // Init the Table with Strings and Expected Return values
- testMatrix = new HashMap();
-
- testMatrix.put("bucket-.ozone.self", Boolean.FALSE);
- testMatrix.put("bucket.-ozone.self", Boolean.FALSE);
- testMatrix.put(".bucket.ozone.self", Boolean.FALSE);
- testMatrix.put("bucket.ozone.self.", Boolean.FALSE);
- testMatrix.put("bucket..ozone.self", Boolean.FALSE);
- testMatrix.put("192.1.1.1", Boolean.FALSE);
- testMatrix.put("ab", Boolean.FALSE);
- testMatrix.put("bucket.ozone.self.this.is.a.really.long.name.that."
- + "is.more.than.sixty.three.characters.long.for.sure", Boolean.FALSE);
- testMatrix.put(null, Boolean.FALSE);
- testMatrix.put("bucket@$", Boolean.FALSE);
- testMatrix.put("BUCKET", Boolean.FALSE);
- testMatrix.put("bucket .ozone.self", Boolean.FALSE);
- testMatrix.put(" bucket.ozone.self", Boolean.FALSE);
- testMatrix.put("bucket.ozone.self-", Boolean.FALSE);
- testMatrix.put("-bucket.ozone.self", Boolean.FALSE);
-
- testMatrix.put("bucket", Boolean.TRUE);
- testMatrix.put("bucket.ozone.self", Boolean.TRUE);
- testMatrix.put("bucket.ozone.self", Boolean.TRUE);
- testMatrix.put("bucket-name.ozone.self", Boolean.TRUE);
- testMatrix.put("bucket.1.ozone.self", Boolean.TRUE);
-
- Set keys = testMatrix.keySet();
- for (String key : keys) {
- if (testMatrix.get(key)) {
-
- // For valid names there should be no exceptions at all
- verifyResourceName(key);
- } else {
- try {
- verifyResourceName(key);
- // should never get here since the isValid call will throw
- fail("An exception was expected but did not happen.");
- } catch (IllegalArgumentException e) {
-
- }
- }
- }
- }
-
- /**
- * Just calls Request ID many times and assert we
- * got different values, ideally this should be
- * run under parallel threads. Since the function under
- * test has no external dependencies it is assumed
- * that this test is good enough.
- */
- @Test
- public void testRequestIDisRandom() {
- HashSet set = new HashSet<>();
- for (int i = 0; i < 1000; i++) {
- assertTrue(set.add(getRequestID()));
- }
- }
-}
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/web/TestVolumeStructs.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/web/TestVolumeStructs.java
deleted file mode 100644
index b433be6fb5104..0000000000000
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/web/TestVolumeStructs.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.hadoop.ozone.web;
-
-import org.apache.hadoop.ozone.web.response.ListVolumes;
-import org.apache.hadoop.ozone.web.response.VolumeInfo;
-import org.apache.hadoop.ozone.web.response.VolumeOwner;
-import org.junit.Test;
-
-import java.io.IOException;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Test Ozone Volume info structure.
- */
-public class TestVolumeStructs {
-
- @Test
- public void testVolumeInfoParse() throws IOException {
- VolumeInfo volInfo =
- new VolumeInfo("testvol", "Thu, Apr 9, 2015 10:23:45 GMT", "gandalf");
- VolumeOwner owner = new VolumeOwner("bilbo");
- volInfo.setOwner(owner);
- String jString = volInfo.toJsonString();
- VolumeInfo newVollInfo = VolumeInfo.parse(jString);
- String one = volInfo.toJsonString();
- String two = newVollInfo.toJsonString();
-
- assertEquals(volInfo.toJsonString(), newVollInfo.toJsonString());
- }
-
- @Test
- public void testVolumeInfoValue() throws IOException {
- String createdOn = "Thu, Apr 9, 2015 10:23:45 GMT";
- String createdBy = "gandalf";
- VolumeInfo volInfo = new VolumeInfo("testvol", createdOn, createdBy);
- assertEquals(volInfo.getCreatedBy(), createdBy);
- assertEquals(volInfo.getCreatedOn(), createdOn);
- }
-
-
- @Test
- public void testVolumeListParse() throws IOException {
- ListVolumes list = new ListVolumes();
- for (int x = 0; x < 100; x++) {
- VolumeInfo volInfo = new VolumeInfo("testvol" + Integer.toString(x),
- "Thu, Apr 9, 2015 10:23:45 GMT", "gandalf");
- list.addVolume(volInfo);
- }
- list.sort();
- String listString = list.toJsonString();
- ListVolumes newList = ListVolumes.parse(listString);
- assertEquals(list.toJsonString(), newList.toJsonString());
- }
-}
diff --git a/hadoop-ozone/datanode/pom.xml b/hadoop-ozone/datanode/pom.xml
index b1a48e3d10c41..0f5c3c49ef7f1 100644
--- a/hadoop-ozone/datanode/pom.xml
+++ b/hadoop-ozone/datanode/pom.xml
@@ -47,10 +47,6 @@
org.apache.hadoop
hadoop-hdds-container-service
-
- org.apache.hadoop
- hadoop-ozone-objectstore-service
-
com.sun.xml.bind
jaxb-impl
diff --git a/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh b/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
new file mode 100644
index 0000000000000..a4c6e9bf8d860
--- /dev/null
+++ b/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## generate summary txt file
+find "." -name 'TEST*.xml' -print0 \
+ | xargs -n1 -0 "grep" -l -E " "$SUMMARY_FILE"
+for TEST_RESULT_FILE in $(find "$REPORT_DIR" -name "*.txt" | grep -v output); do
+
+ FAILURES=$(cat $TEST_RESULT_FILE | grep FAILURE | grep "Tests run" | awk '{print $18}' | sort | uniq)
+
+ for FAILURE in $FAILURES; do
+ printf " $FAILURE\n" >> "$SUMMARY_FILE"
+ TEST_RESULT_LOCATION=$(realpath --relative-to=$REPORT_DIR $TEST_RESULT_FILE)
+ printf " (result)[./$TEST_RESULT_LOCATION]\n\n" >> "$SUMMARY_FILE"
+ done
+done
+printf "\n\n" >> "$SUMMARY_FILE"
+
+## generate counter
+wc -l "$REPORT_DIR/summary.txt" | awk '{print $1}'> "$REPORT_DIR/failures"
diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh b/hadoop-ozone/dev-support/checks/acceptance.sh
index 1e80ad4e48c2e..1b70cddae54ad 100755
--- a/hadoop-ozone/dev-support/checks/acceptance.sh
+++ b/hadoop-ozone/dev-support/checks/acceptance.sh
@@ -16,7 +16,19 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target"}
+mkdir -p "$REPORT_DIR"
+
OZONE_VERSION=$(grep "" "$DIR/../../pom.xml" | sed 's/<[^>]*>//g'| sed 's/^[ \t]*//')
-cd "$DIR/../../dist/target/ozone-$OZONE_VERSION/compose" || exit 1
+DIST_DIR="$DIR/../../dist/target/ozone-$OZONE_VERSION"
+
+if [ ! -d "$DIST_DIR" ]; then
+ echo "Distribution dir is missing. Doing a full build"
+ "$DIR/build.sh"
+fi
+
+cd "$DIST_DIR/compose" || exit 1
./test-all.sh
+cp results/* "$REPORT_DIR/"
+cp "$REPORT_DIR/log.html" "$REPORT_DIR/summary.html"
exit $?
diff --git a/hadoop-ozone/dev-support/checks/author.sh b/hadoop-ozone/dev-support/checks/author.sh
index f50a39627b4a8..efad4dbd127f9 100755
--- a/hadoop-ozone/dev-support/checks/author.sh
+++ b/hadoop-ozone/dev-support/checks/author.sh
@@ -16,12 +16,18 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
-#hide this tring to not confuse yetus
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target"}
+mkdir -p "$REPORT_DIR"
+REPORT_FILE="$REPORT_DIR/summary.txt"
+
+#hide this string to not confuse yetus
AUTHOR="uthor"
AUTHOR="@a${AUTHOR}"
-if grep -r --include="*.java" "$AUTHOR" .; then
- exit 1
-else
- exit 0
+grep -r --include="*.java" "$AUTHOR" . | tee "$REPORT_FILE"
+
+wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
+
+if [[ -s "${REPORT_FILE}" ]]; then
+ exit 1
fi
diff --git a/hadoop-ozone/dev-support/checks/checkstyle.sh b/hadoop-ozone/dev-support/checks/checkstyle.sh
index c4de5288238fd..9e72685006c36 100755
--- a/hadoop-ozone/dev-support/checks/checkstyle.sh
+++ b/hadoop-ozone/dev-support/checks/checkstyle.sh
@@ -16,14 +16,20 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target"}
+mkdir -p "$REPORT_DIR"
+REPORT_FILE="$REPORT_DIR/summary.txt"
+
mvn -B -fn checkstyle:check -f pom.ozone.xml
#Print out the exact violations with parsing XML results with sed
-find "." -name checkstyle-errors.xml -print0 | xargs -0 sed '$!N; //d'
+find "." -name checkstyle-errors.xml -print0 | xargs -0 sed '$!N; //d' | tee "$REPORT_FILE"
+
+## generate counter
+wc -l "$REPORT_DIR/summary.txt" | awk '{print $1}'> "$REPORT_DIR/failures"
-violations=$(grep -r error --include checkstyle-errors.xml .| wc -l)
-if [[ $violations -gt 0 ]]; then
- echo "There are $violations checkstyle violations"
- exit 1
+if [[ -s "${REPORT_FILE}" ]]; then
+ exit 1
fi
-exit 0
+
+
diff --git a/hadoop-ozone/dev-support/checks/findbugs.sh b/hadoop-ozone/dev-support/checks/findbugs.sh
index 545ad9fd7d4e9..0bda1b7318bb0 100755
--- a/hadoop-ozone/dev-support/checks/findbugs.sh
+++ b/hadoop-ozone/dev-support/checks/findbugs.sh
@@ -16,21 +16,20 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
-FINDBUGS_ALL_FILE=./target/findbugs-all.txt
+mvn -B compile -fn findbugs:check -Dfindbugs.failOnError=false -f pom.ozone.xml
-mkdir -p ./target
-rm "$FINDBUGS_ALL_FILE" || true
-touch "$FINDBUGS_ALL_FILE"
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target"}
+mkdir -p "$REPORT_DIR"
+REPORT_FILE="$REPORT_DIR/summary.txt"
-mvn -B compile -fn findbugs:check -Dfindbugs.failOnError=false -f pom.ozone.xml
+touch "$REPORT_FILE"
-find hadoop-ozone -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
-find hadoop-hdds -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
+find hadoop-ozone -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${REPORT_FILE}"
+find hadoop-hdds -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${REPORT_FILE}"
-bugs=$(wc -l < "$FINDBUGS_ALL_FILE")
+wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
-if [[ ${bugs} -gt 0 ]]; then
+if [[ -s "${REPORT_FILE}" ]]; then
exit 1
-else
- exit 0
fi
+
diff --git a/hadoop-ozone/dev-support/checks/integration.sh b/hadoop-ozone/dev-support/checks/integration.sh
index 8170c2e257da5..0d99790656c53 100755
--- a/hadoop-ozone/dev-support/checks/integration.sh
+++ b/hadoop-ozone/dev-support/checks/integration.sh
@@ -20,10 +20,13 @@ export MAVEN_OPTS="-Xmx4096m"
mvn -B install -f pom.ozone.xml -DskipTests
mvn -B -fn test -f pom.ozone.xml -pl :hadoop-ozone-integration-test,:hadoop-ozone-filesystem,:hadoop-ozone-tools \
-Dtest=\!TestMiniChaosOzoneCluster
-module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \
- | xargs -0 -n1 "grep" -l -E "/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
-mkdir -p target
-REPORT_FILE="$DIR/../../../target/rat-aggregated.txt"
-mkdir -p "$(dirname "$REPORT_FILE")"
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target"}
+mkdir -p "$REPORT_DIR"
+
+REPORT_FILE="$REPORT_DIR/summary.txt"
cd hadoop-hdds || exit 1
mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check
@@ -26,7 +27,11 @@ cd ../hadoop-ozone || exit 1
mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check
cd "$DIR/../../.." || exit 1
+
grep -r --include=rat.txt "!????" hadoop-hdds hadoop-ozone | tee "$REPORT_FILE"
+
+wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
+
if [[ -s "${REPORT_FILE}" ]]; then
exit 1
fi
diff --git a/hadoop-ozone/dev-support/checks/shellcheck.sh b/hadoop-ozone/dev-support/checks/shellcheck.sh
index 637a4f863caf8..c8e47b396978b 100755
--- a/hadoop-ozone/dev-support/checks/shellcheck.sh
+++ b/hadoop-ozone/dev-support/checks/shellcheck.sh
@@ -16,8 +16,10 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
-OUTPUT_FILE="$DIR/../../../target/shell-problems.txt"
-mkdir -p "$(dirname "$OUTPUT_FILE")"
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target"}
+mkdir -p "$REPORT_DIR"
+REPORT_FILE="$REPORT_DIR/summary.txt"
+
echo "" > "$OUTPUT_FILE"
if [[ "$(uname -s)" = "Darwin" ]]; then
find hadoop-hdds hadoop-ozone -type f -perm '-500'
@@ -26,8 +28,11 @@ else
fi \
| grep -v -e target/ -e node_modules/ -e '\.\(ico\|py\|yml\)$' \
| xargs -n1 shellcheck \
- | tee "$OUTPUT_FILE"
+ | tee "$REPORT_FILE"
+
+## generate counter
+wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
-if [ "$(cat "$OUTPUT_FILE")" ]; then
+if [[ -s "${REPORT_FILE}" ]]; then
exit 1
fi
diff --git a/hadoop-ozone/dev-support/checks/unit.sh b/hadoop-ozone/dev-support/checks/unit.sh
index 473f7147f0100..da0118aad1385 100755
--- a/hadoop-ozone/dev-support/checks/unit.sh
+++ b/hadoop-ozone/dev-support/checks/unit.sh
@@ -13,12 +13,18 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+set -x
export MAVEN_OPTS="-Xmx4096m"
mvn -fn test -f pom.ozone.xml -pl \!:hadoop-ozone-integration-test,\!:hadoop-ozone-filesystem,\!:hadoop-ozone-tools
-module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \
- | xargs -n1 -0 "grep" -l -E "
test
-
- org.apache.hadoop
- hadoop-ozone-objectstore-service
-
org.apache.hadoop
hadoop-ozone-s3gateway
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index 44c005c29ec99..9dd3f005c8f57 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -137,15 +137,6 @@ static Builder newHABuilder(OzoneConfiguration conf) {
*/
OzoneClient getRpcClient() throws IOException;
- /**
- * Returns an REST based {@link OzoneClient} to access the
- * {@link MiniOzoneCluster}.
- *
- * @return {@link OzoneClient}
- * @throws IOException
- */
- OzoneClient getRestClient() throws IOException;
-
/**
* Returns StorageContainerLocationClient to communicate with
* {@link StorageContainerManager} associated with the MiniOzoneCluster.
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index be1b77b2303ee..dc0ed9f717797 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -36,7 +36,6 @@
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.ozone.common.Storage.StorageState;
import org.apache.hadoop.ozone.container.common.utils.ContainerCache;
import org.apache.hadoop.ozone.om.OMConfigKeys;
@@ -67,7 +66,6 @@
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState
.HEALTHY;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_DATANODE_PLUGINS_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys
.DFS_CONTAINER_IPC_PORT;
import static org.apache.hadoop.ozone.OzoneConfigKeys
@@ -212,18 +210,6 @@ public OzoneClient getRpcClient() throws IOException {
return OzoneClientFactory.getRpcClient(conf);
}
- /**
- * Creates an {@link OzoneClient} connected to this cluster's REST
- * service. Callers take ownership of the client and must close it when done.
- *
- * @return OzoneRestClient connected to this cluster's REST service
- * @throws OzoneException if Ozone encounters an error creating the client
- */
- @Override
- public OzoneClient getRestClient() throws IOException {
- return OzoneClientFactory.getRestClient(conf);
- }
-
/**
* Returns an RPC proxy connected to this cluster's StorageContainerManager
* for accessing container location information. Callers take ownership of
@@ -619,8 +605,6 @@ private void configureOM() {
private void configureHddsDatanodes() {
conf.set(ScmConfigKeys.HDDS_REST_HTTP_ADDRESS_KEY, "0.0.0.0:0");
conf.set(HddsConfigKeys.HDDS_DATANODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
- conf.set(HDDS_DATANODE_PLUGINS_KEY,
- "org.apache.hadoop.ozone.web.OzoneHddsDatanodeService");
conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT,
randomContainerPort);
conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT,
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java
index a1243e87f7bb8..9b0f2f7419431 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java
@@ -18,15 +18,23 @@
package org.apache.hadoop.ozone;
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.ratis.RatisHelper;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.client.rpc.RpcClient;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.hdds.ratis.RatisHelper;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.ratis.RatisHelper.newRaftClient;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.rpc.RpcType;
@@ -35,16 +43,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
-import static org.apache.hadoop.hdds.ratis.RatisHelper.newRaftClient;
-
/**
* Helpers for Ratis tests.
*/
@@ -80,7 +78,7 @@ public MiniOzoneCluster getCluster() {
}
public ClientProtocol newOzoneClient()
- throws OzoneException, URISyntaxException, IOException {
+ throws IOException {
return new RpcClient(conf);
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java
new file mode 100644
index 0000000000000..83a5133144d62
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Scanner;
+
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.protocol.StorageType;
+import org.apache.hadoop.ozone.client.BucketArgs;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.VolumeArgs;
+
+import org.apache.commons.lang3.RandomStringUtils;
+
+/**
+ * Utility to help to generate test data.
+ */
+public final class TestDataUtil {
+
+ private TestDataUtil() {
+ }
+
+ public static OzoneBucket createVolumeAndBucket(MiniOzoneCluster cluster,
+ String volumeName, String bucketName) throws IOException {
+ String userName = "user" + RandomStringUtils.randomNumeric(5);
+ String adminName = "admin" + RandomStringUtils.randomNumeric(5);
+
+ OzoneClient client = cluster.getClient();
+
+ VolumeArgs volumeArgs = VolumeArgs.newBuilder()
+ .setAdmin(adminName)
+ .setOwner(userName)
+ .build();
+
+ ObjectStore objectStore = client.getObjectStore();
+
+ objectStore.createVolume(volumeName, volumeArgs);
+
+ OzoneVolume volume = objectStore.getVolume(volumeName);
+
+ BucketArgs omBucketArgs = BucketArgs.newBuilder()
+ .setStorageType(StorageType.DISK)
+ .build();
+
+ volume.createBucket(bucketName, omBucketArgs);
+ return volume.getBucket(bucketName);
+
+ }
+
+ public static void createKey(OzoneBucket bucket, String keyName,
+ String content) throws IOException {
+ try (OutputStream stream = bucket
+ .createKey(keyName, content.length(), ReplicationType.STAND_ALONE,
+ ReplicationFactor.ONE, new HashMap<>())) {
+ stream.write(content.getBytes());
+ }
+ }
+
+ public static String getKey(OzoneBucket bucket, String keyName)
+ throws IOException {
+ try (InputStream stream = bucket.readKey(keyName)) {
+ return new Scanner(stream).useDelimiter("\\A").next();
+ }
+ }
+
+ public static OzoneBucket createVolumeAndBucket(MiniOzoneCluster cluster)
+ throws IOException {
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+ String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+ return createVolumeAndBucket(cluster, volumeName, bucketName);
+ }
+}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneHelper.java
deleted file mode 100644
index 8a3d1a857558f..0000000000000
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneHelper.java
+++ /dev/null
@@ -1,413 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone;
-
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.util.Time;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-
-import javax.ws.rs.core.HttpHeaders;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.Locale;
-
-import static java.net.HttpURLConnection.HTTP_CREATED;
-import static java.net.HttpURLConnection.HTTP_OK;
-import static org.junit.Assert.assertEquals;
-
-/**
- * Helper functions to test Ozone.
- */
-public class TestOzoneHelper {
-
- public CloseableHttpClient createHttpClient() {
- return HttpClients.createDefault();
- }
- /**
- * Creates Volumes on Ozone Store.
- *
- * @throws IOException
- */
- public void testCreateVolumes(int port) throws IOException {
- SimpleDateFormat format =
- new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
- CloseableHttpClient client = createHttpClient();
- String volumeName = OzoneUtils.getRequestID().toLowerCase();
- try {
- HttpPost httppost = new HttpPost(
- String.format("http://localhost:%d/%s", port, volumeName));
-
- httppost.addHeader(Header.OZONE_VERSION_HEADER,
- Header.OZONE_V1_VERSION_HEADER);
- httppost.addHeader(HttpHeaders.DATE,
- format.format(new Date(Time.monotonicNow())));
- httppost.addHeader(HttpHeaders.AUTHORIZATION,
- Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
- OzoneConsts.OZONE_SIMPLE_HDFS_USER);
- httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
-
- HttpResponse response = client.execute(httppost);
- assertEquals(response.toString(), HTTP_CREATED,
- response.getStatusLine().getStatusCode());
- } finally {
- client.close();
- }
- }
-
- /**
- * Create Volumes with Quota.
- *
- * @throws IOException
- */
- public void testCreateVolumesWithQuota(int port) throws IOException {
- SimpleDateFormat format =
- new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
- CloseableHttpClient client = createHttpClient();
- String volumeName = OzoneUtils.getRequestID().toLowerCase();
- try {
- HttpPost httppost = new HttpPost(
- String.format("http://localhost:%d/%s?quota=10TB", port, volumeName));
-
- httppost.addHeader(Header.OZONE_VERSION_HEADER,
- Header.OZONE_V1_VERSION_HEADER);
- httppost.addHeader(HttpHeaders.DATE,
- format.format(new Date(Time.monotonicNow())));
- httppost.addHeader(HttpHeaders.AUTHORIZATION,
- Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
- OzoneConsts.OZONE_SIMPLE_HDFS_USER);
- httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
-
- HttpResponse response = client.execute(httppost);
- assertEquals(response.toString(), HTTP_CREATED,
- response.getStatusLine().getStatusCode());
- } finally {
- client.close();
- }
- }
-
- /**
- * Create Volumes with Invalid Quota.
- *
- * @throws IOException
- */
- public void testCreateVolumesWithInvalidQuota(int port) throws IOException {
- SimpleDateFormat format =
- new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
- CloseableHttpClient client = createHttpClient();
- String volumeName = OzoneUtils.getRequestID().toLowerCase();
- try {
- HttpPost httppost = new HttpPost(
- String.format("http://localhost:%d/%s?quota=NaN", port, volumeName));
-
- httppost.addHeader(Header.OZONE_VERSION_HEADER,
- Header.OZONE_V1_VERSION_HEADER);
- httppost.addHeader(HttpHeaders.DATE,
- format.format(new Date(Time.monotonicNow())));
- httppost.addHeader(HttpHeaders.AUTHORIZATION,
- Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
- OzoneConsts.OZONE_SIMPLE_HDFS_USER);
- httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
-
- HttpResponse response = client.execute(httppost);
- assertEquals(response.toString(), ErrorTable.MALFORMED_QUOTA
- .getHttpCode(),
- response.getStatusLine().getStatusCode());
- } finally {
- client.close();
- }
- }
-
- /**
- * To create a volume a user name must be specified using OZONE_USER header.
- * This test verifies that we get an error in case we call without a OZONE
- * user name.
- *
- * @throws IOException
- */
- public void testCreateVolumesWithInvalidUser(int port) throws IOException {
- SimpleDateFormat format =
- new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
- CloseableHttpClient client = createHttpClient();
- String volumeName = OzoneUtils.getRequestID().toLowerCase();
- try {
- HttpPost httppost = new HttpPost(
- String.format("http://localhost:%d/%s?quota=1TB", port, volumeName));
-
- httppost.addHeader(Header.OZONE_VERSION_HEADER,
- Header.OZONE_V1_VERSION_HEADER);
- httppost.addHeader(HttpHeaders.DATE,
- format.format(new Date(Time.monotonicNow())));
- httppost.addHeader(HttpHeaders.AUTHORIZATION,
- Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
- OzoneConsts.OZONE_SIMPLE_HDFS_USER);
-
- HttpResponse response = client.execute(httppost);
-
- assertEquals(response.toString(), ErrorTable.USER_NOT_FOUND.getHttpCode(),
- response.getStatusLine().getStatusCode());
- } finally {
- client.close();
- }
- }
-
- /**
- * Only Admins can create volumes in Ozone. This test uses simple userauth as
- * backend and hdfs and root are admin users in the simple backend.
- *
- * This test tries to create a volume as user bilbo.
- *
- * @throws IOException
- */
- public void testCreateVolumesWithOutAdminRights(int port) throws IOException {
- SimpleDateFormat format =
- new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
- CloseableHttpClient client = createHttpClient();
- String volumeName = OzoneUtils.getRequestID().toLowerCase();
- try {
- HttpPost httppost = new HttpPost(
- String.format("http://localhost:%d/%s?quota=NaN", port, volumeName));
-
- httppost.addHeader(Header.OZONE_VERSION_HEADER,
- Header.OZONE_V1_VERSION_HEADER);
- httppost.addHeader(HttpHeaders.DATE,
- format.format(new Date(Time.monotonicNow())));
- httppost.addHeader(HttpHeaders.AUTHORIZATION,
- Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
- "bilbo"); // This is not a root user in Simple Auth
- httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
-
- HttpResponse response = client.execute(httppost);
- assertEquals(response.toString(), ErrorTable.ACCESS_DENIED.getHttpCode(),
- response.getStatusLine().getStatusCode());
- } finally {
- client.close();
- }
- }
-
- /**
- * Create a bunch of volumes in a loop.
- *
- * @throws IOException
- */
- public void testCreateVolumesInLoop(int port) throws IOException {
- SimpleDateFormat format =
- new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
-
- for (int x = 0; x < 1000; x++) {
- CloseableHttpClient client = createHttpClient();
- String volumeName = OzoneUtils.getRequestID().toLowerCase();
- String userName = OzoneUtils.getRequestID().toLowerCase();
-
- HttpPost httppost = new HttpPost(
- String.format("http://localhost:%d/%s?quota=10TB", port, volumeName));
-
- httppost.addHeader(Header.OZONE_VERSION_HEADER,
- Header.OZONE_V1_VERSION_HEADER);
- httppost.addHeader(HttpHeaders.DATE,
- format.format(new Date(Time.monotonicNow())));
- httppost.addHeader(HttpHeaders.AUTHORIZATION,
- Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
- OzoneConsts.OZONE_SIMPLE_HDFS_USER);
- httppost.addHeader(Header.OZONE_USER, userName);
-
- HttpResponse response = client.execute(httppost);
- assertEquals(response.toString(), HTTP_CREATED,
- response.getStatusLine().getStatusCode());
- client.close();
- }
- }
- /**
- * Get volumes owned by the user.
- *
- * @throws IOException
- */
- public void testGetVolumesByUser(int port) throws IOException {
- SimpleDateFormat format =
- new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
- // We need to create a volume for this test to succeed.
- testCreateVolumes(port);
- CloseableHttpClient client = createHttpClient();
- try {
- HttpGet httpget =
- new HttpGet(String.format("http://localhost:%d/", port));
-
- httpget.addHeader(Header.OZONE_VERSION_HEADER,
- Header.OZONE_V1_VERSION_HEADER);
-
- httpget.addHeader(HttpHeaders.DATE,
- format.format(new Date(Time.monotonicNow())));
-
- httpget.addHeader(HttpHeaders.AUTHORIZATION,
- Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
- OzoneConsts.OZONE_SIMPLE_HDFS_USER);
-
- httpget.addHeader(Header.OZONE_USER,
- OzoneConsts.OZONE_SIMPLE_HDFS_USER);
-
- HttpResponse response = client.execute(httpget);
- assertEquals(response.toString(), HTTP_OK,
- response.getStatusLine().getStatusCode());
-
- } finally {
- client.close();
- }
- }
-
- /**
- * Admins can read volumes belonging to other users.
- *
- * @throws IOException
- */
- public void testGetVolumesOfAnotherUser(int port) throws IOException {
- SimpleDateFormat format =
- new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
-
- CloseableHttpClient client = createHttpClient();
- try {
- HttpGet httpget =
- new HttpGet(String.format("http://localhost:%d/", port));
-
- httpget.addHeader(Header.OZONE_VERSION_HEADER,
- Header.OZONE_V1_VERSION_HEADER);
- httpget.addHeader(HttpHeaders.DATE,
- format.format(new Date(Time.monotonicNow())));
-
- httpget.addHeader(HttpHeaders.AUTHORIZATION,
- Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
- OzoneConsts.OZONE_SIMPLE_ROOT_USER);
-
- // User Root is getting volumes belonging to user HDFS
- httpget.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
-
- HttpResponse response = client.execute(httpget);
- assertEquals(response.toString(), HTTP_OK,
- response.getStatusLine().getStatusCode());
-
- } finally {
- client.close();
- }
- }
-
- /**
- * if you try to read volumes belonging to another user,
- * then server always ignores it.
- *
- * @throws IOException
- */
- public void testGetVolumesOfAnotherUserShouldFail(int port)
- throws IOException {
- SimpleDateFormat format =
- new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
-
- CloseableHttpClient client = createHttpClient();
- String userName = OzoneUtils.getRequestID().toLowerCase();
- try {
- HttpGet httpget =
- new HttpGet(String.format("http://localhost:%d/", port));
-
- httpget.addHeader(Header.OZONE_VERSION_HEADER,
- Header.OZONE_V1_VERSION_HEADER);
- httpget.addHeader(HttpHeaders.DATE,
- format.format(new Date(Time.monotonicNow())));
-
- httpget.addHeader(HttpHeaders.AUTHORIZATION,
- Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
- userName);
-
- // userName is NOT a root user, hence he should NOT be able to read the
- // volumes of user HDFS
- httpget.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
-
- HttpResponse response = client.execute(httpget);
- // We will get an Error called userNotFound when using Simple Auth Scheme
- assertEquals(response.toString(), ErrorTable.USER_NOT_FOUND.getHttpCode(),
- response.getStatusLine().getStatusCode());
-
- } finally {
- client.close();
- }
- }
-
- public void testListKeyOnEmptyBucket(int port) throws IOException {
- SimpleDateFormat format =
- new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
- CloseableHttpClient client = createHttpClient();
- String volumeName = OzoneUtils.getRequestID().toLowerCase();
- String bucketName = OzoneUtils.getRequestID().toLowerCase() + "bucket";
- try {
-
- HttpPost httppost = new HttpPost(
- String.format("http://localhost:%d/%s", port, volumeName));
- httppost.addHeader(Header.OZONE_VERSION_HEADER,
- Header.OZONE_V1_VERSION_HEADER);
- httppost.addHeader(HttpHeaders.DATE,
- format.format(new Date(Time.monotonicNow())));
- httppost.addHeader(HttpHeaders.AUTHORIZATION,
- Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " "
- + OzoneConsts.OZONE_SIMPLE_HDFS_USER);
- httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
- HttpResponse response = client.execute(httppost);
- assertEquals(response.toString(), HTTP_CREATED,
- response.getStatusLine().getStatusCode());
- client.close();
-
- client = createHttpClient();
- httppost = new HttpPost(String
- .format("http://localhost:%d/%s/%s", port, volumeName, bucketName));
- httppost.addHeader(Header.OZONE_VERSION_HEADER,
- Header.OZONE_V1_VERSION_HEADER);
- httppost.addHeader(HttpHeaders.DATE,
- format.format(new Date(Time.monotonicNow())));
- httppost.addHeader(HttpHeaders.AUTHORIZATION,
- Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " "
- + OzoneConsts.OZONE_SIMPLE_HDFS_USER);
- httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
- response = client.execute(httppost);
- assertEquals(response.toString(), HTTP_CREATED,
- response.getStatusLine().getStatusCode());
- client.close();
-
- client = createHttpClient();
- HttpGet httpget = new HttpGet(String
- .format("http://localhost:%d/%s/%s", port, volumeName, bucketName));
- httpget.addHeader(Header.OZONE_VERSION_HEADER,
- Header.OZONE_V1_VERSION_HEADER);
- httpget.addHeader(HttpHeaders.DATE,
- format.format(new Date(Time.monotonicNow())));
- httpget.addHeader(HttpHeaders.AUTHORIZATION,
- Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " "
- + OzoneConsts.OZONE_SIMPLE_HDFS_USER);
- httpget.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
- response = client.execute(httpget);
- assertEquals(response.toString() + " " + response.getStatusLine()
- .getReasonPhrase(), HTTP_OK,
- response.getStatusLine().getStatusCode());
-
- } finally {
- client.close();
- }
- }
-
-}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
index bb0a3e69207c5..5aceda015b275 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
@@ -16,37 +16,30 @@
*/
package org.apache.hadoop.ozone;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import com.google.common.primitives.Longs;
-import org.apache.commons.lang3.RandomStringUtils;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.protocol.StorageType;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.KeyArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.utils.MetadataKeyFilters;
import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter;
-import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.primitives.Longs;
+import org.apache.commons.lang3.RandomStringUtils;
/**
* A helper class used by {@link TestStorageContainerManager} to generate
@@ -56,53 +49,32 @@ public class TestStorageContainerManagerHelper {
private final MiniOzoneCluster cluster;
private final Configuration conf;
- private final StorageHandler storageHandler;
public TestStorageContainerManagerHelper(MiniOzoneCluster cluster,
Configuration conf) throws IOException {
this.cluster = cluster;
this.conf = conf;
- storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
}
public Map createKeys(int numOfKeys, int keySize)
throws Exception {
Map keyLocationMap = Maps.newHashMap();
- String volume = "volume" + RandomStringUtils.randomNumeric(5);
- String bucket = "bucket" + RandomStringUtils.randomNumeric(5);
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
- null, null, null, null);
-
- VolumeArgs createVolumeArgs = new VolumeArgs(volume, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs);
-
- BucketArgs bucketArgs = new BucketArgs(bucket, createVolumeArgs);
- bucketArgs.setStorageType(StorageType.DISK);
- storageHandler.createBucket(bucketArgs);
-
- // Write 20 keys in bucket.
+
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
+ // Write 20 keys in bucketName.
Set keyNames = Sets.newHashSet();
- KeyArgs keyArgs;
for (int i = 0; i < numOfKeys; i++) {
String keyName = RandomStringUtils.randomAlphabetic(5) + i;
keyNames.add(keyName);
- keyArgs = new KeyArgs(keyName, bucketArgs);
- keyArgs.setSize(keySize);
- // Just for testing list keys call, so no need to write real data.
- OutputStream stream = storageHandler.newKeyWriter(keyArgs);
- stream.write(DFSUtil.string2Bytes(
- RandomStringUtils.randomAlphabetic(5)));
- stream.close();
+
+ TestDataUtil
+ .createKey(bucket, keyName, RandomStringUtils.randomAlphabetic(5));
}
for (String key : keyNames) {
OmKeyArgs arg = new OmKeyArgs.Builder()
- .setVolumeName(volume)
- .setBucketName(bucket)
+ .setVolumeName(bucket.getVolumeName())
+ .setBucketName(bucket.getName())
.setKeyName(key)
.setRefreshPipeline(true)
.build();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
index 84d17adc4ab21..6cced841678d4 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
@@ -58,6 +58,7 @@
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneKeyDetails;
@@ -67,7 +68,6 @@
import org.apache.hadoop.ozone.client.VolumeArgs;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.ozone.common.OzoneChecksumException;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
@@ -241,7 +241,7 @@ public void testOMClientProxyProvider() {
@Test
public void testSetVolumeQuota()
- throws IOException, OzoneException {
+ throws IOException {
String volumeName = UUID.randomUUID().toString();
store.createVolume(volumeName);
store.getVolume(volumeName).setQuota(
@@ -265,7 +265,7 @@ public void testDeleteVolume()
@Test
public void testCreateVolumeWithMetadata()
- throws IOException, OzoneException {
+ throws IOException, OzoneClientException {
String volumeName = UUID.randomUUID().toString();
VolumeArgs volumeArgs = VolumeArgs.newBuilder()
.addMetadata("key1", "val1")
@@ -279,7 +279,7 @@ public void testCreateVolumeWithMetadata()
@Test
public void testCreateBucketWithMetadata()
- throws IOException, OzoneException {
+ throws IOException, OzoneClientException {
long currentTime = Time.now();
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
@@ -298,7 +298,7 @@ public void testCreateBucketWithMetadata()
@Test
public void testCreateBucket()
- throws IOException, OzoneException {
+ throws IOException, OzoneClientException {
long currentTime = Time.now();
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
@@ -313,7 +313,7 @@ public void testCreateBucket()
@Test
public void testCreateS3Bucket()
- throws IOException, OzoneException {
+ throws IOException, OzoneClientException {
long currentTime = Time.now();
String userName = UserGroupInformation.getCurrentUser().getUserName();
String bucketName = UUID.randomUUID().toString();
@@ -346,7 +346,7 @@ public void testCreateSecureS3Bucket() throws IOException {
@Test
public void testListS3Buckets()
- throws IOException, OzoneException {
+ throws IOException, OzoneClientException {
String userName = "ozone100";
String bucketName1 = UUID.randomUUID().toString();
String bucketName2 = UUID.randomUUID().toString();
@@ -364,7 +364,7 @@ public void testListS3Buckets()
@Test
public void testListS3BucketsFail()
- throws IOException, OzoneException {
+ throws IOException, OzoneClientException {
String userName = "randomUser";
Iterator iterator = store.listS3Buckets(userName,
null);
@@ -403,7 +403,7 @@ public void testDeleteS3NonExistingBucket() {
@Test
public void testCreateS3BucketMapping()
- throws IOException, OzoneException {
+ throws IOException, OzoneClientException {
long currentTime = Time.now();
String userName = "ozone";
String bucketName = UUID.randomUUID().toString();
@@ -422,7 +422,7 @@ public void testCreateS3BucketMapping()
@Test
public void testCreateBucketWithVersioning()
- throws IOException, OzoneException {
+ throws IOException, OzoneClientException {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
store.createVolume(volumeName);
@@ -437,7 +437,7 @@ public void testCreateBucketWithVersioning()
@Test
public void testCreateBucketWithStorageType()
- throws IOException, OzoneException {
+ throws IOException, OzoneClientException {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
store.createVolume(volumeName);
@@ -452,7 +452,7 @@ public void testCreateBucketWithStorageType()
@Test
public void testCreateBucketWithAcls()
- throws IOException, OzoneException {
+ throws IOException, OzoneClientException {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
OzoneAcl userAcl = new OzoneAcl(USER, "test",
@@ -471,7 +471,7 @@ public void testCreateBucketWithAcls()
@Test
public void testCreateBucketWithAllArgument()
- throws IOException, OzoneException {
+ throws IOException, OzoneClientException {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
OzoneAcl userAcl = new OzoneAcl(USER, "test",
@@ -508,7 +508,7 @@ public void testInvalidBucketCreation() throws Exception {
@Test
public void testAddBucketAcl()
- throws IOException, OzoneException {
+ throws IOException, OzoneClientException {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
store.createVolume(volumeName);
@@ -527,7 +527,7 @@ public void testAddBucketAcl()
@Test
public void testRemoveBucketAcl()
- throws IOException, OzoneException {
+ throws IOException, OzoneClientException {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
OzoneAcl userAcl = new OzoneAcl(USER, "test",
@@ -582,7 +582,7 @@ public void testRemoveBucketAclUsingRpcClientRemoveAcl()
@Test
public void testSetBucketVersioning()
- throws IOException, OzoneException {
+ throws IOException, OzoneClientException {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
store.createVolume(volumeName);
@@ -619,7 +619,7 @@ public void testAclsAfterCallingSetBucketProperty() throws Exception {
@Test
public void testSetBucketStorageType()
- throws IOException, OzoneException {
+ throws IOException, OzoneClientException {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
store.createVolume(volumeName);
@@ -679,7 +679,7 @@ private boolean verifyRatisReplication(String volumeName, String bucketName,
@Test
public void testPutKey()
- throws IOException, OzoneException {
+ throws IOException, OzoneClientException {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
long currentTime = Time.now();
@@ -747,7 +747,7 @@ public void testValidateBlockLengthWithCommitKey() throws IOException {
@Test
public void testPutKeyRatisOneNode()
- throws IOException, OzoneException {
+ throws IOException, OzoneClientException {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
long currentTime = Time.now();
@@ -782,7 +782,7 @@ public void testPutKeyRatisOneNode()
@Test
public void testPutKeyRatisThreeNodes()
- throws IOException, OzoneException {
+ throws IOException, OzoneClientException {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
long currentTime = Time.now();
@@ -977,7 +977,7 @@ private void readKey(OzoneBucket bucket, String keyName, String data)
}
@Test
- public void testGetKeyDetails() throws IOException, OzoneException {
+ public void testGetKeyDetails() throws IOException, OzoneClientException {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
index 26d16d505a3ea..dfa9e0e1b29e0 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
@@ -46,7 +46,6 @@
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
-import org.apache.hadoop.ozone.ozShell.TestOzoneShell;
import org.apache.hadoop.ozone.protocol.commands.RetriableDatanodeEventWatcher;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
@@ -95,7 +94,7 @@ public static void init() throws Exception {
GenericTestUtils.setLogLevel(SCMBlockDeletingService.LOG, Level.DEBUG);
String path =
- GenericTestUtils.getTempPath(TestOzoneShell.class.getSimpleName());
+ GenericTestUtils.getTempPath(TestBlockDeletion.class.getSimpleName());
File baseDir = new File(path);
baseDir.mkdirs();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java
index b1fa6fb224547..74cde37bceaf1 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java
@@ -17,8 +17,15 @@
*/
package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
@@ -26,32 +33,25 @@
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE;
import org.apache.hadoop.test.GenericTestUtils;
+
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE;
import org.junit.Assert;
import org.junit.Test;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.concurrent.TimeoutException;
-
/**
* Test to behaviour of the datanode when recieve close container command.
*/
public class TestCloseContainerHandler {
@Test
- public void test() throws IOException, TimeoutException, InterruptedException,
- OzoneException {
+ public void test()
+ throws IOException, TimeoutException, InterruptedException {
//setup a cluster (1G free space is enough for a unit test)
OzoneConfiguration conf = new OzoneConfiguration();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
deleted file mode 100644
index 328d1be2a112a..0000000000000
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.om;
-
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.conf.StorageUnit;
-import org.apache.hadoop.hdds.protocol.StorageType;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
-import org.apache.hadoop.metrics2.MetricsRecordBuilder;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.KeyArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
-import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
-import static org.junit.Assert.assertEquals;
-
-/**
- * Test key write/read where a key can span multiple containers.
- */
-public class TestMultipleContainerReadWrite {
- private static MiniOzoneCluster cluster = null;
- private static StorageHandler storageHandler;
- private static UserArgs userArgs;
- private static OzoneConfiguration conf;
-
- @Rule
- public ExpectedException exception = ExpectedException.none();
-
- /**
- * Create a MiniDFSCluster for testing.
- *
- * Ozone is made active by setting OZONE_ENABLED = true
- *
- * @throws IOException
- */
- @BeforeClass
- public static void init() throws Exception {
- conf = new OzoneConfiguration();
- conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 1,
- StorageUnit.MB);
- conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, 5);
- cluster = MiniOzoneCluster.newBuilder(conf).build();
- cluster.waitForClusterToBeReady();
- storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
- userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
- null, null, null, null);
- }
-
- /**
- * Shutdown MiniDFSCluster.
- */
- @AfterClass
- public static void shutdown() {
- if (cluster != null) {
- cluster.shutdown();
- }
- }
-
- @Test
- public void testWriteRead() throws Exception {
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- String keyName = "key" + RandomStringUtils.randomNumeric(5);
-
- VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs);
-
- BucketArgs bucketArgs = new BucketArgs(bucketName, createVolumeArgs);
- bucketArgs.setStorageType(StorageType.DISK);
- storageHandler.createBucket(bucketArgs);
-
- String dataString = RandomStringUtils.randomAscii(3 * (int)OzoneConsts.MB);
- KeyArgs keyArgs = new KeyArgs(volumeName, bucketName, keyName, userArgs);
- keyArgs.setSize(3 * (int)OzoneConsts.MB);
- keyArgs.setUserName(userName);
-
- try (OutputStream outputStream = storageHandler.newKeyWriter(keyArgs)) {
- outputStream.write(dataString.getBytes());
- }
-
- byte[] data = new byte[dataString.length()];
- try (InputStream inputStream = storageHandler.newKeyReader(keyArgs)) {
- inputStream.read(data, 0, data.length);
- }
- assertEquals(dataString, new String(data));
- // checking whether container meta data has the chunk file persisted.
- MetricsRecordBuilder containerMetrics = getMetrics(
- "StorageContainerMetrics");
- assertCounter("numWriteChunk", 3L, containerMetrics);
- assertCounter("numReadChunk", 3L, containerMetrics);
- }
-
- // Disable this test, because this tests assumes writing beyond a specific
- // size is not allowed. Which is not true for now. Keeping this test in case
- // we add this restrict in the future.
- @Ignore
- @Test
- public void testErrorWrite() throws Exception {
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- String keyName = "key" + RandomStringUtils.randomNumeric(5);
-
- VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs);
-
- BucketArgs bucketArgs = new BucketArgs(bucketName, createVolumeArgs);
- bucketArgs.setStorageType(StorageType.DISK);
- storageHandler.createBucket(bucketArgs);
-
- String dataString1 = RandomStringUtils.randomAscii(100);
- String dataString2 = RandomStringUtils.randomAscii(500);
- KeyArgs keyArgs = new KeyArgs(volumeName, bucketName, keyName, userArgs);
- keyArgs.setSize(500);
-
- try (OutputStream outputStream = storageHandler.newKeyWriter(keyArgs)) {
- // first write will write succeed
- outputStream.write(dataString1.getBytes());
- // second write
- exception.expect(IOException.class);
- exception.expectMessage(
- "Can not write 500 bytes with only 400 byte space");
- outputStream.write(dataString2.getBytes());
- }
- }
-
- @Test
- public void testPartialRead() throws Exception {
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- String keyName = "key" + RandomStringUtils.randomNumeric(5);
-
- VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs);
-
- BucketArgs bucketArgs = new BucketArgs(bucketName, createVolumeArgs);
- bucketArgs.setStorageType(StorageType.DISK);
- storageHandler.createBucket(bucketArgs);
-
- String dataString = RandomStringUtils.randomAscii(500);
- KeyArgs keyArgs = new KeyArgs(volumeName, bucketName, keyName, userArgs);
- keyArgs.setSize(500);
- keyArgs.setUserName(userName);
-
- try (OutputStream outputStream = storageHandler.newKeyWriter(keyArgs)) {
- outputStream.write(dataString.getBytes());
- }
-
- byte[] data = new byte[600];
- try (InputStream inputStream = storageHandler.newKeyReader(keyArgs)) {
- int readLen = inputStream.read(data, 0, 340);
- assertEquals(340, readLen);
- assertEquals(dataString.substring(0, 340),
- new String(data).substring(0, 340));
-
- readLen = inputStream.read(data, 340, 260);
- assertEquals(160, readLen);
- assertEquals(dataString, new String(data).substring(0, 500));
-
- readLen = inputStream.read(data, 500, 1);
- assertEquals(-1, readLen);
- }
- }
-}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java
index 3b6389b8c82f5..6041f4c291a8d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java
@@ -19,29 +19,27 @@
import java.util.UUID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.StorageType;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneTestUtils;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.VolumeArgs;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.ozone.security.acl.IOzoneObj;
import org.apache.hadoop.ozone.security.acl.RequestContext;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.KeyArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.request.OzoneQuota;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.commons.lang3.RandomStringUtils;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS;
import org.junit.AfterClass;
import static org.junit.Assert.assertTrue;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
@@ -52,9 +50,8 @@
*/
public class TestOmAcls {
+ public static boolean aclAllow = true;
private static MiniOzoneCluster cluster = null;
- private static StorageHandler storageHandler;
- private static UserArgs userArgs;
private static OMMetrics omMetrics;
private static OzoneConfiguration conf;
private static String clusterId;
@@ -80,15 +77,13 @@ public static void init() throws Exception {
conf.setInt(OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, 2);
conf.setClass(OZONE_ACL_AUTHORIZER_CLASS, OzoneAccessAuthorizerTest.class,
IAccessAuthorizer.class);
+ conf.setStrings(OZONE_ADMINISTRATORS, OZONE_ADMINISTRATORS_WILDCARD);
cluster = MiniOzoneCluster.newBuilder(conf)
.setClusterId(clusterId)
.setScmId(scmId)
.setOmId(omId)
.build();
cluster.waitForClusterToBeReady();
- storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
- userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
- null, null, null, null);
omMetrics = cluster.getOzoneManager().getMetrics();
logCapturer =
GenericTestUtils.LogCapturer.captureLogs(OzoneManager.getLogger());
@@ -104,53 +99,41 @@ public static void shutdown() {
}
}
-
/**
* Tests the OM Initialization.
*/
+
@Test
- public void testOMAclsPermissionDenied() throws Exception {
- String user0 = "testListVolumes-user-0";
- String adminUser = "testListVolumes-admin";
- final VolumeArgs createVolumeArgs;
- int i = 100;
- String user0VolName = "Vol-" + user0 + "-" + i;
- createVolumeArgs = new VolumeArgs(user0VolName, userArgs);
- createVolumeArgs.setUserName(user0);
- createVolumeArgs.setAdminName(adminUser);
- createVolumeArgs.setQuota(new OzoneQuota(i, OzoneQuota.Units.GB));
- logCapturer.clearOutput();
- OzoneTestUtils.expectOmException(ResultCodes.PERMISSION_DENIED,
- () -> storageHandler.createVolume(createVolumeArgs));
- assertTrue(logCapturer.getOutput().contains("Only admin users are " +
- "authorized to create Ozone"));
+ public void testBucketCreationPermissionDenied() throws Exception {
- BucketArgs bucketArgs = new BucketArgs("bucket1", createVolumeArgs);
- bucketArgs.setStorageType(StorageType.DISK);
+ TestOmAcls.aclAllow = true;
+
+ String volumeName = RandomStringUtils.randomAlphabetic(5).toLowerCase();
+ String bucketName = RandomStringUtils.randomAlphabetic(5).toLowerCase();
+ cluster.getClient().getObjectStore().createVolume(volumeName);
+ OzoneVolume volume =
+ cluster.getClient().getObjectStore().getVolume(volumeName);
+
+ TestOmAcls.aclAllow = false;
OzoneTestUtils.expectOmException(ResultCodes.PERMISSION_DENIED,
- () -> storageHandler.createBucket(bucketArgs));
- assertTrue(logCapturer.getOutput().contains("Only admin users are" +
- " authorized to create Ozone"));
+ () -> volume.createBucket(bucketName));
+
+ assertTrue(logCapturer.getOutput()
+ .contains("doesn't have CREATE permission to access volume"));
}
@Test
public void testFailureInKeyOp() throws Exception {
final VolumeArgs createVolumeArgs;
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- createVolumeArgs = new VolumeArgs(userName, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- createVolumeArgs.setQuota(new OzoneQuota(100, OzoneQuota.Units.GB));
- BucketArgs bucketArgs = new BucketArgs("bucket1", createVolumeArgs);
- bucketArgs.setStorageType(StorageType.DISK);
+
+ TestOmAcls.aclAllow = true;
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
logCapturer.clearOutput();
+ ;
+ TestOmAcls.aclAllow = false;
- // write a key without specifying size at all
- String keyName = "testKey";
- KeyArgs keyArgs = new KeyArgs(keyName, bucketArgs);
OzoneTestUtils.expectOmException(ResultCodes.PERMISSION_DENIED,
- () -> storageHandler.newKeyWriter(keyArgs));
+ () -> TestDataUtil.createKey(bucket, "testKey", "testcontent"));
assertTrue(logCapturer.getOutput().contains("doesn't have WRITE " +
"permission to access key"));
}
@@ -163,6 +146,6 @@ class OzoneAccessAuthorizerTest implements IAccessAuthorizer {
@Override
public boolean checkAccess(IOzoneObj ozoneObject, RequestContext context) {
- return false;
+ return TestOmAcls.aclAllow;
}
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmBlockVersioning.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmBlockVersioning.java
index 0db78c75e141d..ff1cf039e0c4d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmBlockVersioning.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmBlockVersioning.java
@@ -16,50 +16,39 @@
*/
package org.apache.hadoop.ozone.om;
-import org.apache.commons.lang3.RandomStringUtils;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.KeyArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.security.UserGroupInformation;
+
+import org.apache.commons.lang3.RandomStringUtils;
import org.junit.AfterClass;
+import org.junit.Assert;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
-import org.junit.Assert;
import org.junit.rules.ExpectedException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
/**
* This class tests the versioning of blocks from OM side.
*/
public class TestOmBlockVersioning {
private static MiniOzoneCluster cluster = null;
- private static UserArgs userArgs;
private static OzoneConfiguration conf;
private static OzoneManager ozoneManager;
- private static StorageHandler storageHandler;
@Rule
public ExpectedException exception = ExpectedException.none();
@@ -76,9 +65,6 @@ public static void init() throws Exception {
conf = new OzoneConfiguration();
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
- storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
- userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
- null, null, null, null);
ozoneManager = cluster.getOzoneManager();
}
@@ -101,14 +87,7 @@ public void testAllocateCommit() throws Exception {
String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
String keyName = "key" + RandomStringUtils.randomNumeric(5);
- VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs);
-
- BucketArgs bucketArgs = new BucketArgs(bucketName, createVolumeArgs);
- bucketArgs.setStorageType(StorageType.DISK);
- storageHandler.createBucket(bucketArgs);
+ TestDataUtil.createVolumeAndBucket(cluster, volumeName, bucketName);
OmKeyArgs keyArgs = new OmKeyArgs.Builder()
.setVolumeName(volumeName)
@@ -200,14 +179,8 @@ public void testReadLatestVersion() throws Exception {
String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
String keyName = "key" + RandomStringUtils.randomNumeric(5);
- VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs);
-
- BucketArgs bucketArgs = new BucketArgs(bucketName, createVolumeArgs);
- bucketArgs.setStorageType(StorageType.DISK);
- storageHandler.createBucket(bucketArgs);
+ OzoneBucket bucket =
+ TestDataUtil.createVolumeAndBucket(cluster, volumeName, bucketName);
OmKeyArgs omKeyArgs = new OmKeyArgs.Builder()
.setVolumeName(volumeName)
@@ -218,48 +191,30 @@ public void testReadLatestVersion() throws Exception {
.build();
String dataString = RandomStringUtils.randomAlphabetic(100);
- KeyArgs keyArgs = new KeyArgs(volumeName, bucketName, keyName, userArgs);
- keyArgs.setUserName(userName);
- // this write will create 1st version with one block
- try (OutputStream stream = storageHandler.newKeyWriter(keyArgs)) {
- stream.write(dataString.getBytes());
- }
- byte[] data = new byte[dataString.length()];
- try (InputStream in = storageHandler.newKeyReader(keyArgs)) {
- in.read(data);
- }
+
+ TestDataUtil.createKey(bucket, keyName, dataString);
+ assertEquals(dataString, TestDataUtil.getKey(bucket, keyName));
OmKeyInfo keyInfo = ozoneManager.lookupKey(omKeyArgs);
- assertEquals(dataString, DFSUtil.bytes2String(data));
assertEquals(0, keyInfo.getLatestVersionLocations().getVersion());
assertEquals(1,
keyInfo.getLatestVersionLocations().getLocationList().size());
// this write will create 2nd version, 2nd version will contain block from
// version 1, and add a new block
- dataString = RandomStringUtils.randomAlphabetic(10);
- data = new byte[dataString.length()];
- try (OutputStream stream = storageHandler.newKeyWriter(keyArgs)) {
- stream.write(dataString.getBytes());
- }
- try (InputStream in = storageHandler.newKeyReader(keyArgs)) {
- in.read(data);
- }
+ TestDataUtil.createKey(bucket, keyName, dataString);
+
+
keyInfo = ozoneManager.lookupKey(omKeyArgs);
- assertEquals(dataString, DFSUtil.bytes2String(data));
+ assertEquals(dataString, TestDataUtil.getKey(bucket, keyName));
assertEquals(1, keyInfo.getLatestVersionLocations().getVersion());
assertEquals(2,
keyInfo.getLatestVersionLocations().getLocationList().size());
dataString = RandomStringUtils.randomAlphabetic(200);
- data = new byte[dataString.length()];
- try (OutputStream stream = storageHandler.newKeyWriter(keyArgs)) {
- stream.write(dataString.getBytes());
- }
- try (InputStream in = storageHandler.newKeyReader(keyArgs)) {
- in.read(data);
- }
+ TestDataUtil.createKey(bucket, keyName, dataString);
+
keyInfo = ozoneManager.lookupKey(omKeyArgs);
- assertEquals(dataString, DFSUtil.bytes2String(data));
+ assertEquals(dataString, TestDataUtil.getKey(bucket, keyName));
assertEquals(2, keyInfo.getLatestVersionLocations().getVersion());
assertEquals(3,
keyInfo.getLatestVersionLocations().getLocationList().size());
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmInit.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmInit.java
index 4b736c9436d64..de42fdccc0349 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmInit.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmInit.java
@@ -16,14 +16,14 @@
*/
package org.apache.hadoop.ozone.om;
+import java.io.IOException;
+import java.util.UUID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -31,18 +31,11 @@
import org.junit.Test;
import org.junit.rules.ExpectedException;
-import java.io.IOException;
-import java.util.UUID;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS;
-
/**
* Test Ozone Manager Init.
*/
public class TestOmInit {
private static MiniOzoneCluster cluster = null;
- private static StorageHandler storageHandler;
- private static UserArgs userArgs;
private static OMMetrics omMetrics;
private static OzoneConfiguration conf;
private static String clusterId;
@@ -72,9 +65,6 @@ public static void init() throws Exception {
.setOmId(omId)
.build();
cluster.waitForClusterToBeReady();
- storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
- userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
- null, null, null, null);
omMetrics = cluster.getOzoneManager().getMetrics();
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
deleted file mode 100644
index 1fb3b001ff25d..0000000000000
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
+++ /dev/null
@@ -1,1449 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.om;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.InetSocketAddress;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.text.ParseException;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-import java.util.UUID;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import org.apache.commons.codec.binary.StringUtils;
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.StorageType;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.ScmInfo;
-import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
-import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.OzoneTestUtils;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.ozone.om.codec.OmKeyInfoCodec;
-import org.apache.hadoop.ozone.om.exceptions.OMException;
-import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
-import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
-import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
-import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
-import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
-import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
-import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.VolumeList;
-import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType;
-import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType;
-import org.apache.hadoop.ozone.util.OzoneVersionInfo;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.KeyArgs;
-import org.apache.hadoop.ozone.web.handlers.ListArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.request.OzoneQuota;
-import org.apache.hadoop.ozone.web.response.BucketInfo;
-import org.apache.hadoop.ozone.web.response.KeyInfo;
-import org.apache.hadoop.ozone.web.response.ListBuckets;
-import org.apache.hadoop.ozone.web.response.ListKeys;
-import org.apache.hadoop.ozone.web.response.ListVolumes;
-import org.apache.hadoop.ozone.web.response.VolumeInfo;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.db.DBUpdatesWrapper;
-import org.apache.hadoop.utils.db.RDBStore;
-import org.apache.hadoop.utils.db.Table;
-import org.apache.hadoop.utils.db.Table.KeyValue;
-import org.apache.hadoop.utils.db.TableIterator;
-
-import org.apache.commons.lang3.RandomStringUtils;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
-import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
-import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
-import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND;
-import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.ALL;
-
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.rocksdb.RocksDB;
-
-/**
- * Test Ozone Manager operation in distributed handler scenario.
- */
-public class TestOzoneManager {
- private MiniOzoneCluster cluster = null;
- private StorageHandler storageHandler;
- private UserArgs userArgs;
- private OMMetrics omMetrics;
- private OzoneConfiguration conf;
- private String clusterId;
- private String scmId;
- private String omId;
-
- @Rule
- public Timeout timeout = new Timeout(60000);
-
- /**
- * Create a MiniDFSCluster for testing.
- *
- * Ozone is made active by setting OZONE_ENABLED = true
- *
- * @throws IOException
- */
- @Before
- public void init() throws Exception {
- conf = new OzoneConfiguration();
- clusterId = UUID.randomUUID().toString();
- scmId = UUID.randomUUID().toString();
- omId = UUID.randomUUID().toString();
- conf.setBoolean(OZONE_ACL_ENABLED, true);
- conf.setInt(OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, 2);
- conf.set(OZONE_ADMINISTRATORS, OZONE_ADMINISTRATORS_WILDCARD);
- cluster = MiniOzoneCluster.newBuilder(conf)
- .setClusterId(clusterId)
- .setScmId(scmId)
- .setOmId(omId)
- .build();
- cluster.waitForClusterToBeReady();
- storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
- userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
- null, null, null, null);
- omMetrics = cluster.getOzoneManager().getMetrics();
- }
-
- /**
- * Shutdown MiniDFSCluster.
- */
- @After
- public void shutdown() {
- if (cluster != null) {
- cluster.shutdown();
- }
- }
-
- // Create a volume and test its attribute after creating them
- @Test
- public void testCreateVolume() throws IOException, OzoneException {
- long volumeCreateFailCount = omMetrics.getNumVolumeCreateFails();
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
-
- VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs);
-
- VolumeArgs getVolumeArgs = new VolumeArgs(volumeName, userArgs);
- VolumeInfo retVolumeinfo = storageHandler.getVolumeInfo(getVolumeArgs);
- Assert.assertTrue(retVolumeinfo.getVolumeName().equals(volumeName));
- Assert.assertTrue(retVolumeinfo.getOwner().getName().equals(userName));
- Assert.assertEquals(volumeCreateFailCount,
- omMetrics.getNumVolumeCreateFails());
- }
-
- // Create a volume and modify the volume owner and then test its attributes
- @Test
- public void testChangeVolumeOwner() throws IOException, OzoneException {
- long volumeCreateFailCount = omMetrics.getNumVolumeCreateFails();
- long volumeInfoFailCount = omMetrics.getNumVolumeInfoFails();
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
-
- VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs);
-
- String newUserName = "user" + RandomStringUtils.randomNumeric(5);
- createVolumeArgs.setUserName(newUserName);
- storageHandler.setVolumeOwner(createVolumeArgs);
-
- VolumeArgs getVolumeArgs = new VolumeArgs(volumeName, userArgs);
- VolumeInfo retVolumeInfo = storageHandler.getVolumeInfo(getVolumeArgs);
-
- Assert.assertTrue(retVolumeInfo.getVolumeName().equals(volumeName));
- Assert.assertFalse(retVolumeInfo.getOwner().getName().equals(userName));
- Assert.assertTrue(retVolumeInfo.getOwner().getName().equals(newUserName));
- Assert.assertEquals(volumeCreateFailCount,
- omMetrics.getNumVolumeCreateFails());
- Assert.assertEquals(volumeInfoFailCount,
- omMetrics.getNumVolumeInfoFails());
- }
-
- // Create a volume and modify the volume owner and then test its attributes
- @Test
- public void testChangeVolumeQuota() throws IOException, OzoneException {
- long numVolumeCreateFail = omMetrics.getNumVolumeCreateFails();
- long numVolumeInfoFail = omMetrics.getNumVolumeInfoFails();
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- Random rand = new Random();
-
- // Create a new volume with a quota
- OzoneQuota createQuota =
- new OzoneQuota(rand.nextInt(100), OzoneQuota.Units.GB);
- VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- createVolumeArgs.setQuota(createQuota);
- storageHandler.createVolume(createVolumeArgs);
-
- VolumeArgs getVolumeArgs = new VolumeArgs(volumeName, userArgs);
- VolumeInfo retVolumeInfo = storageHandler.getVolumeInfo(getVolumeArgs);
- Assert.assertEquals(createQuota.sizeInBytes(),
- retVolumeInfo.getQuota().sizeInBytes());
-
- // Set a new quota and test it
- OzoneQuota setQuota =
- new OzoneQuota(rand.nextInt(100), OzoneQuota.Units.GB);
- createVolumeArgs.setQuota(setQuota);
- storageHandler.setVolumeQuota(createVolumeArgs, false);
- getVolumeArgs = new VolumeArgs(volumeName, userArgs);
- retVolumeInfo = storageHandler.getVolumeInfo(getVolumeArgs);
- Assert.assertEquals(setQuota.sizeInBytes(),
- retVolumeInfo.getQuota().sizeInBytes());
-
- // Remove the quota and test it again
- storageHandler.setVolumeQuota(createVolumeArgs, true);
- getVolumeArgs = new VolumeArgs(volumeName, userArgs);
- retVolumeInfo = storageHandler.getVolumeInfo(getVolumeArgs);
- Assert.assertEquals(OzoneConsts.MAX_QUOTA_IN_BYTES,
- retVolumeInfo.getQuota().sizeInBytes());
- Assert.assertEquals(numVolumeCreateFail,
- omMetrics.getNumVolumeCreateFails());
- Assert.assertEquals(numVolumeInfoFail,
- omMetrics.getNumVolumeInfoFails());
- }
-
- // Create a volume and then delete it and then check for deletion
- @Test
- public void testDeleteVolume() throws IOException, OzoneException {
- long volumeCreateFailCount = omMetrics.getNumVolumeCreateFails();
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String volumeName1 = volumeName + "_A";
- String volumeName2 = volumeName + "_AA";
- VolumeArgs volumeArgs = null;
- VolumeInfo volumeInfo = null;
-
- // Create 2 empty volumes with same prefix.
- volumeArgs = new VolumeArgs(volumeName1, userArgs);
- volumeArgs.setUserName(userName);
- volumeArgs.setAdminName(adminName);
- storageHandler.createVolume(volumeArgs);
-
- volumeArgs = new VolumeArgs(volumeName2, userArgs);
- volumeArgs.setUserName(userName);
- volumeArgs.setAdminName(adminName);
- storageHandler.createVolume(volumeArgs);
-
- volumeArgs = new VolumeArgs(volumeName1, userArgs);
- volumeInfo = storageHandler.getVolumeInfo(volumeArgs);
- Assert.assertTrue(volumeInfo.getVolumeName().equals(volumeName1));
- Assert.assertTrue(volumeInfo.getOwner().getName().equals(userName));
- Assert.assertEquals(volumeCreateFailCount,
- omMetrics.getNumVolumeCreateFails());
-
- // Volume with _A should be able to delete as it is empty.
- storageHandler.deleteVolume(volumeArgs);
-
- // Make sure volume with _AA suffix still exists.
- volumeArgs = new VolumeArgs(volumeName2, userArgs);
- volumeInfo = storageHandler.getVolumeInfo(volumeArgs);
- Assert.assertTrue(volumeInfo.getVolumeName().equals(volumeName2));
-
- // Make sure volume with _A suffix is successfully deleted.
- try {
- volumeArgs = new VolumeArgs(volumeName1, userArgs);
- storageHandler.getVolumeInfo(volumeArgs);
- Assert.fail("Volume is not deleted");
- } catch (OMException ex) {
- Assert.assertEquals(ResultCodes.VOLUME_NOT_FOUND, ex.getResult());
- }
- //delete the _AA volume, too
- storageHandler.deleteVolume(new VolumeArgs(volumeName2, userArgs));
-
- //Make sure there is no volume information for the specific user
- OMMetadataManager metadataManager =
- cluster.getOzoneManager().getMetadataManager();
-
- String userKey = metadataManager.getUserKey(userName);
- VolumeList volumes = metadataManager.getUserTable().get(userKey);
-
- //that was the last volume of the user, shouldn't be any record here
- Assert.assertNull(volumes);
-
-
- }
-
- // Create a volume and a bucket inside the volume,
- // then delete it and then check for deletion failure
- @Test
- public void testFailedDeleteVolume() throws IOException, OzoneException {
- long numVolumeCreateFails = omMetrics.getNumVolumeCreateFails();
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
-
- VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs);
-
- VolumeArgs getVolumeArgs = new VolumeArgs(volumeName, userArgs);
- VolumeInfo retVolumeInfo = storageHandler.getVolumeInfo(getVolumeArgs);
- Assert.assertTrue(retVolumeInfo.getVolumeName().equals(volumeName));
- Assert.assertTrue(retVolumeInfo.getOwner().getName().equals(userName));
- Assert.assertEquals(numVolumeCreateFails,
- omMetrics.getNumVolumeCreateFails());
-
- BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
- storageHandler.createBucket(bucketArgs);
-
- try {
- storageHandler.deleteVolume(createVolumeArgs);
- Assert.fail("Expecting deletion should fail "
- + "because volume is not empty");
- } catch (OMException ex) {
- Assert.assertEquals(ResultCodes.VOLUME_NOT_EMPTY, ex.getResult());
- }
- retVolumeInfo = storageHandler.getVolumeInfo(getVolumeArgs);
- Assert.assertTrue(retVolumeInfo.getVolumeName().equals(volumeName));
- Assert.assertTrue(retVolumeInfo.getOwner().getName().equals(userName));
- }
-
- // Create a volume and test Volume access for a different user
- @Test
- public void testAccessVolume() throws IOException, OzoneException {
- String userName = UserGroupInformation.getCurrentUser().getUserName();
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String[] groupName =
- {"group" + RandomStringUtils.randomNumeric(5)};
-
- VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- createVolumeArgs.setGroups(groupName);
- storageHandler.createVolume(createVolumeArgs);
-
- OzoneAcl userAcl = new OzoneAcl(ACLIdentityType.USER, userName,
- ACLType.READ, ACCESS);
- Assert.assertTrue(storageHandler.checkVolumeAccess(volumeName, userAcl));
- OzoneAcl group = new OzoneAcl(ACLIdentityType.GROUP, groupName[0],
- ACLType.READ, ACCESS);
- Assert.assertTrue(storageHandler.checkVolumeAccess(volumeName, group));
-
- // Create a different user and access should fail
- String falseUserName = "user" + RandomStringUtils.randomNumeric(5);
- OzoneAcl falseUserAcl =
- new OzoneAcl(ACLIdentityType.USER, falseUserName,
- ACLType.ALL, ACCESS);
- Assert.assertFalse(storageHandler
- .checkVolumeAccess(volumeName, falseUserAcl));
- // Checking access with user name and Group Type should fail
- OzoneAcl falseGroupAcl = new OzoneAcl(ACLIdentityType.GROUP, userName,
- ACLType.ALL, ACCESS);
- Assert.assertFalse(storageHandler
- .checkVolumeAccess(volumeName, falseGroupAcl));
-
- // Access for acl type world should also fail
- OzoneAcl worldAcl =
- new OzoneAcl(ACLIdentityType.WORLD, "", ACLType.READ, ACCESS);
- Assert.assertFalse(storageHandler.checkVolumeAccess(volumeName, worldAcl));
-
- Assert.assertEquals(0, omMetrics.getNumVolumeCheckAccessFails());
- Assert.assertEquals(0, omMetrics.getNumVolumeCreateFails());
- }
-
- @Test
- public void testCreateBucket() throws IOException, OzoneException {
- long numVolumeCreateFail = omMetrics.getNumVolumeCreateFails();
- long numBucketCreateFail = omMetrics.getNumBucketCreateFails();
- long numBucketInfoFail = omMetrics.getNumBucketInfoFails();
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
-
- VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
- volumeArgs.setUserName(userName);
- volumeArgs.setAdminName(adminName);
- storageHandler.createVolume(volumeArgs);
-
- BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
- storageHandler.createBucket(bucketArgs);
-
- BucketArgs getBucketArgs = new BucketArgs(volumeName, bucketName,
- userArgs);
- BucketInfo bucketInfo = storageHandler.getBucketInfo(getBucketArgs);
- Assert.assertTrue(bucketInfo.getVolumeName().equals(volumeName));
- Assert.assertTrue(bucketInfo.getBucketName().equals(bucketName));
- Assert.assertEquals(numVolumeCreateFail,
- omMetrics.getNumVolumeCreateFails());
- Assert.assertEquals(numBucketCreateFail,
- omMetrics.getNumBucketCreateFails());
- Assert.assertEquals(numBucketInfoFail,
- omMetrics.getNumBucketInfoFails());
- }
-
- @Test
- public void testDeleteBucket() throws Exception {
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
- volumeArgs.setUserName(userName);
- volumeArgs.setAdminName(adminName);
- storageHandler.createVolume(volumeArgs);
- BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
- storageHandler.createBucket(bucketArgs);
- BucketArgs getBucketArgs = new BucketArgs(volumeName, bucketName,
- userArgs);
- BucketInfo bucketInfo = storageHandler.getBucketInfo(getBucketArgs);
- Assert.assertTrue(bucketInfo.getVolumeName().equals(volumeName));
- Assert.assertTrue(bucketInfo.getBucketName().equals(bucketName));
- storageHandler.deleteBucket(bucketArgs);
-
- OzoneTestUtils.expectOmException(ResultCodes.BUCKET_NOT_FOUND,
- () -> storageHandler.getBucketInfo(getBucketArgs));
- }
-
- @Test
- public void testDeleteNonExistingBucket() throws Exception {
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
- volumeArgs.setUserName(userName);
- volumeArgs.setAdminName(adminName);
- storageHandler.createVolume(volumeArgs);
- BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
- storageHandler.createBucket(bucketArgs);
- BucketArgs getBucketArgs = new BucketArgs(volumeName, bucketName,
- userArgs);
- BucketInfo bucketInfo = storageHandler.getBucketInfo(getBucketArgs);
- Assert.assertTrue(bucketInfo.getVolumeName().equals(volumeName));
- Assert.assertTrue(bucketInfo.getBucketName().equals(bucketName));
- BucketArgs newBucketArgs = new BucketArgs(
- volumeName, bucketName + "_invalid", userArgs);
- OzoneTestUtils.expectOmException(ResultCodes.BUCKET_NOT_FOUND,
- () -> storageHandler.deleteBucket(newBucketArgs));
- }
-
-
- @Test
- public void testDeleteNonEmptyBucket() throws Exception {
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- String keyName = "key" + RandomStringUtils.randomNumeric(5);
- VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
- volumeArgs.setUserName(userName);
- volumeArgs.setAdminName(adminName);
- storageHandler.createVolume(volumeArgs);
- BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
- storageHandler.createBucket(bucketArgs);
- BucketArgs getBucketArgs = new BucketArgs(volumeName, bucketName,
- userArgs);
- BucketInfo bucketInfo = storageHandler.getBucketInfo(getBucketArgs);
- Assert.assertTrue(bucketInfo.getVolumeName().equals(volumeName));
- Assert.assertTrue(bucketInfo.getBucketName().equals(bucketName));
- String dataString = RandomStringUtils.randomAscii(100);
- KeyArgs keyArgs = new KeyArgs(volumeName, bucketName, keyName, userArgs);
- keyArgs.setSize(100);
- keyArgs.setUserName(userName);
- try (OutputStream stream = storageHandler.newKeyWriter(keyArgs)) {
- stream.write(dataString.getBytes());
- }
- OzoneTestUtils.expectOmException(ResultCodes.BUCKET_NOT_EMPTY,
- () -> storageHandler.deleteBucket(bucketArgs));
-
- }
-
- /**
- * Basic test of both putKey and getKey from OM, as one can not be tested
- * without the other.
- *
- * @throws IOException
- * @throws OzoneException
- */
- @Test
- public void testGetKeyWriterReader() throws IOException, OzoneException {
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- String keyName = "key" + RandomStringUtils.randomNumeric(5);
- long numKeyAllocates = omMetrics.getNumKeyAllocates();
- long numKeyLookups = omMetrics.getNumKeyLookups();
-
- VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs);
-
- BucketArgs bucketArgs = new BucketArgs(bucketName, createVolumeArgs);
- bucketArgs.setStorageType(StorageType.DISK);
- storageHandler.createBucket(bucketArgs);
-
- String dataString = RandomStringUtils.randomAscii(100);
- KeyArgs keyArgs = new KeyArgs(volumeName, bucketName, keyName, userArgs);
- keyArgs.setSize(100);
- keyArgs.setUserName(userName);
- try (OutputStream stream = storageHandler.newKeyWriter(keyArgs)) {
- stream.write(dataString.getBytes());
- }
- Assert.assertEquals(1 + numKeyAllocates, omMetrics.getNumKeyAllocates());
-
- byte[] data = new byte[dataString.length()];
- try (InputStream in = storageHandler.newKeyReader(keyArgs)) {
- in.read(data);
- }
- Assert.assertEquals(dataString, DFSUtil.bytes2String(data));
- Assert.assertEquals(1 + numKeyLookups, omMetrics.getNumKeyLookups());
- }
-
- /**
- * Test write the same key twice, the second write should fail, as currently
- * key overwrite is not supported.
- *
- * @throws IOException
- * @throws OzoneException
- */
- @Test
- public void testKeyOverwrite() throws IOException, OzoneException {
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- String keyName = "key" + RandomStringUtils.randomNumeric(5);
- long numKeyAllocateFails = omMetrics.getNumKeyAllocateFails();
-
- VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs);
-
- BucketArgs bucketArgs = new BucketArgs(bucketName, createVolumeArgs);
- bucketArgs.setStorageType(StorageType.DISK);
- storageHandler.createBucket(bucketArgs);
-
- KeyArgs keyArgs = new KeyArgs(volumeName, bucketName, keyName, userArgs);
- keyArgs.setSize(100);
- keyArgs.setUserName(userName);
- String dataString = RandomStringUtils.randomAscii(100);
- try (OutputStream stream = storageHandler.newKeyWriter(keyArgs)) {
- stream.write(dataString.getBytes());
- }
-
- // We allow the key overwrite to be successful. Please note : Till
- // HDFS-11922 is fixed this causes a data block leak on the data node side.
- // That is this overwrite only overwrites the keys on OM. We need to
- // garbage collect those blocks from datanode.
- KeyArgs keyArgs2 = new KeyArgs(volumeName, bucketName, keyName, userArgs);
- keyArgs2.setUserName(userName);
- storageHandler.newKeyWriter(keyArgs2);
- Assert
- .assertEquals(numKeyAllocateFails, omMetrics.getNumKeyAllocateFails());
- }
-
- /**
- * Test get a non-exiting key.
- *
- * @throws IOException
- * @throws OzoneException
- */
- @Test
- public void testGetNonExistKey() throws Exception {
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- String keyName = "key" + RandomStringUtils.randomNumeric(5);
- long numKeyLookupFails = omMetrics.getNumKeyLookupFails();
-
- VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs);
-
- BucketArgs bucketArgs = new BucketArgs(bucketName, createVolumeArgs);
- bucketArgs.setStorageType(StorageType.DISK);
- storageHandler.createBucket(bucketArgs);
-
- KeyArgs keyArgs = new KeyArgs(volumeName, bucketName, keyName, userArgs);
- // try to get the key, should fail as it hasn't been created
- OzoneTestUtils.expectOmException(KEY_NOT_FOUND,
- () -> storageHandler.newKeyReader(keyArgs));
- Assert.assertEquals(1 + numKeyLookupFails,
- omMetrics.getNumKeyLookupFails());
- }
-
- /**
- * Test delete keys for om.
- *
- * @throws IOException
- * @throws OzoneException
- */
- @Test
- public void testDeleteKey() throws Exception {
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- String keyName = "key" + RandomStringUtils.randomNumeric(5);
- long numKeyDeletes = omMetrics.getNumKeyDeletes();
- long numKeyDeleteFails = omMetrics.getNumKeyDeletesFails();
-
- VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs);
-
- BucketArgs bucketArgs = new BucketArgs(bucketName, createVolumeArgs);
- storageHandler.createBucket(bucketArgs);
-
- KeyArgs keyArgs = new KeyArgs(keyName, bucketArgs);
- keyArgs.setSize(100);
- String dataString = RandomStringUtils.randomAscii(100);
- try (OutputStream stream = storageHandler.newKeyWriter(keyArgs)) {
- stream.write(dataString.getBytes());
- }
-
- storageHandler.deleteKey(keyArgs);
- Assert.assertEquals(1 + numKeyDeletes, omMetrics.getNumKeyDeletes());
-
- // Make sure the deleted key has been moved to the deleted table.
- OMMetadataManager manager = cluster.getOzoneManager().
- getMetadataManager();
- try (TableIterator> iter =
- manager.getDeletedTable().iterator()) {
- iter.seekToFirst();
- Table.KeyValue kv = iter.next();
- Assert.assertNotNull(kv);
- }
-
- // Delete the key again to test deleting non-existing key.
- OzoneTestUtils.expectOmException(KEY_NOT_FOUND,
- () -> storageHandler.deleteKey(keyArgs));
- Assert.assertEquals(1 + numKeyDeleteFails,
- omMetrics.getNumKeyDeletesFails());
- }
-
- /**
- * Test rename key for om.
- *
- * @throws IOException
- * @throws OzoneException
- */
- @Test
- public void testRenameKey() throws IOException, OzoneException {
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- String keyName = "key" + RandomStringUtils.randomNumeric(5);
- long numKeyRenames = omMetrics.getNumKeyRenames();
- long numKeyRenameFails = omMetrics.getNumKeyRenameFails();
- int testRenameFails = 0;
- int testRenames = 0;
- OMException omException = null;
-
- VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs);
-
- BucketArgs bucketArgs = new BucketArgs(bucketName, createVolumeArgs);
- storageHandler.createBucket(bucketArgs);
-
- KeyArgs keyArgs = new KeyArgs(keyName, bucketArgs);
- keyArgs.setSize(100);
- String toKeyName = "key" + RandomStringUtils.randomNumeric(5);
-
- // Rename from non-existent key should fail
- try {
- testRenames++;
- storageHandler.renameKey(keyArgs, toKeyName);
- } catch (OMException e) {
- testRenameFails++;
- omException = e;
- }
- Assert.assertEquals(KEY_NOT_FOUND, omException.getResult());
-
- // Write the contents of the key to be renamed
- String dataString = RandomStringUtils.randomAscii(100);
- try (OutputStream stream = storageHandler.newKeyWriter(keyArgs)) {
- stream.write(dataString.getBytes());
- }
-
- // Rename the key
- toKeyName = "key" + RandomStringUtils.randomNumeric(5);
- testRenames++;
- storageHandler.renameKey(keyArgs, toKeyName);
- Assert.assertEquals(numKeyRenames + testRenames,
- omMetrics.getNumKeyRenames());
- Assert.assertEquals(numKeyRenameFails + testRenameFails,
- omMetrics.getNumKeyRenameFails());
-
- // Try to get the key, should fail as it has been renamed
- try {
- storageHandler.newKeyReader(keyArgs);
- } catch (OMException e) {
- omException = e;
- }
- Assert.assertEquals(KEY_NOT_FOUND, omException.getResult());
-
- // Verify the contents of the renamed key
- keyArgs = new KeyArgs(toKeyName, bucketArgs);
- InputStream in = storageHandler.newKeyReader(keyArgs);
- byte[] b = new byte[dataString.getBytes().length];
- in.read(b);
- Assert.assertEquals(new String(b), dataString);
-
- // Rewrite the renamed key. Rename to key which already exists should fail.
- keyArgs = new KeyArgs(keyName, bucketArgs);
- keyArgs.setSize(100);
- dataString = RandomStringUtils.randomAscii(100);
- try (OutputStream stream = storageHandler.newKeyWriter(keyArgs)) {
- stream.write(dataString.getBytes());
- stream.close();
- testRenames++;
- storageHandler.renameKey(keyArgs, toKeyName);
- } catch (OMException e) {
- testRenameFails++;
- omException = e;
- }
- Assert.assertEquals(ResultCodes.KEY_ALREADY_EXISTS,
- omException.getResult());
-
- // Rename to empty string should fail
- toKeyName = "";
- try {
- testRenames++;
- storageHandler.renameKey(keyArgs, toKeyName);
- } catch (OMException e) {
- testRenameFails++;
- omException = e;
- }
- Assert.assertEquals(ResultCodes.INVALID_KEY_NAME, omException.getResult());
-
- // Rename from empty string should fail
- keyArgs = new KeyArgs("", bucketArgs);
- toKeyName = "key" + RandomStringUtils.randomNumeric(5);
- try {
- testRenames++;
- storageHandler.renameKey(keyArgs, toKeyName);
- } catch (OMException e) {
- testRenameFails++;
- omException = e;
- }
- Assert.assertEquals(ResultCodes.INVALID_KEY_NAME, omException.getResult());
-
- Assert.assertEquals(numKeyRenames + testRenames,
- omMetrics.getNumKeyRenames());
- Assert.assertEquals(numKeyRenameFails + testRenameFails,
- omMetrics.getNumKeyRenameFails());
- }
-
- @Test
- public void testListBuckets() throws IOException, OzoneException {
- ListBuckets result = null;
- ListArgs listBucketArgs = null;
-
- // Create volume - volA.
- final String volAname = "volA";
- VolumeArgs volAArgs = new VolumeArgs(volAname, userArgs);
- volAArgs.setUserName("userA");
- volAArgs.setAdminName("adminA");
- storageHandler.createVolume(volAArgs);
-
- // Create 20 buckets in volA for tests.
- for (int i=0; i<10; i++) {
- // Create "/volA/aBucket_0" to "/volA/aBucket_9" buckets in volA volume.
- BucketArgs aBuckets = new BucketArgs(volAname,
- "aBucket_" + i, userArgs);
- if(i % 3 == 0) {
- aBuckets.setStorageType(StorageType.ARCHIVE);
- } else {
- aBuckets.setStorageType(StorageType.DISK);
- }
- storageHandler.createBucket(aBuckets);
-
- // Create "/volA/bBucket_0" to "/volA/bBucket_9" buckets in volA volume.
- BucketArgs bBuckets = new BucketArgs(volAname,
- "bBucket_" + i, userArgs);
- if(i % 3 == 0) {
- bBuckets.setStorageType(StorageType.RAM_DISK);
- } else {
- bBuckets.setStorageType(StorageType.SSD);
- }
- storageHandler.createBucket(bBuckets);
- }
-
- VolumeArgs volArgs = new VolumeArgs(volAname, userArgs);
-
- // List all buckets in volA.
- listBucketArgs = new ListArgs(volArgs, null, 100, null);
- result = storageHandler.listBuckets(listBucketArgs);
- Assert.assertEquals(20, result.getBuckets().size());
- List archiveBuckets = result.getBuckets().stream()
- .filter(item -> item.getStorageType() == StorageType.ARCHIVE)
- .collect(Collectors.toList());
- Assert.assertEquals(4, archiveBuckets.size());
-
- // List buckets with prefix "aBucket".
- listBucketArgs = new ListArgs(volArgs, "aBucket", 100, null);
- result = storageHandler.listBuckets(listBucketArgs);
- Assert.assertEquals(10, result.getBuckets().size());
- Assert.assertTrue(result.getBuckets().stream()
- .allMatch(entry -> entry.getBucketName().startsWith("aBucket")));
-
- // List a certain number of buckets.
- listBucketArgs = new ListArgs(volArgs, null, 3, null);
- result = storageHandler.listBuckets(listBucketArgs);
- Assert.assertEquals(3, result.getBuckets().size());
- Assert.assertEquals("aBucket_0",
- result.getBuckets().get(0).getBucketName());
- Assert.assertEquals("aBucket_1",
- result.getBuckets().get(1).getBucketName());
- Assert.assertEquals("aBucket_2",
- result.getBuckets().get(2).getBucketName());
-
- // List a certain number of buckets from the startKey.
- listBucketArgs = new ListArgs(volArgs, null, 2, "bBucket_3");
- result = storageHandler.listBuckets(listBucketArgs);
- Assert.assertEquals(2, result.getBuckets().size());
- Assert.assertEquals("bBucket_4",
- result.getBuckets().get(0).getBucketName());
- Assert.assertEquals("bBucket_5",
- result.getBuckets().get(1).getBucketName());
-
- // Provide an invalid bucket name as start key.
- listBucketArgs = new ListArgs(volArgs, null, 100, "unknown_bucket_name");
- ListBuckets buckets = storageHandler.listBuckets(listBucketArgs);
- Assert.assertEquals(buckets.getBuckets().size(), 0);
-
- // Use all arguments.
- listBucketArgs = new ListArgs(volArgs, "b", 5, "bBucket_7");
- result = storageHandler.listBuckets(listBucketArgs);
- Assert.assertEquals(2, result.getBuckets().size());
- Assert.assertEquals("bBucket_8",
- result.getBuckets().get(0).getBucketName());
- Assert.assertEquals("bBucket_9",
- result.getBuckets().get(1).getBucketName());
-
- // Provide an invalid maxKeys argument.
- try {
- listBucketArgs = new ListArgs(volArgs, null, -1, null);
- storageHandler.listBuckets(listBucketArgs);
- Assert.fail("Expecting an error when the given"
- + " maxKeys argument is invalid.");
- } catch (Exception e) {
- Assert.assertTrue(e.getMessage()
- .contains(String.format("the value must be in range (0, %d]",
- OzoneConsts.MAX_LISTBUCKETS_SIZE)));
- }
-
- // Provide an invalid volume name.
- VolumeArgs invalidVolArgs = new VolumeArgs("invalid_name", userArgs);
- try {
- listBucketArgs = new ListArgs(invalidVolArgs, null, 100, null);
- storageHandler.listBuckets(listBucketArgs);
- Assert.fail("Expecting an error when the given volume name is invalid.");
- } catch (OMException e) {
- Assert.assertEquals(VOLUME_NOT_FOUND, e.getResult());
- }
- }
-
- /**
- * Test list keys.
- * @throws IOException
- * @throws OzoneException
- */
- @Test
- public void testListKeys() throws Exception {
- ListKeys result = null;
- ListArgs listKeyArgs = null;
-
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
-
- VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs);
-
- BucketArgs bucketArgs = new BucketArgs(bucketName, createVolumeArgs);
- bucketArgs.setStorageType(StorageType.DISK);
- storageHandler.createBucket(bucketArgs);
-
- // Write 20 keys in bucket.
- int numKeys = 20;
- String keyName = "Key";
- KeyArgs keyArgs = null;
- for (int i = 0; i < numKeys; i++) {
- if (i % 2 == 0) {
- // Create /volume/bucket/aKey[0,2,4,...,18] in bucket.
- keyArgs = new KeyArgs("a" + keyName + i, bucketArgs);
- } else {
- // Create /volume/bucket/bKey[1,3,5,...,19] in bucket.
- keyArgs = new KeyArgs("b" + keyName + i, bucketArgs);
- }
- keyArgs.setSize(4096);
-
- // Just for testing list keys call, so no need to write real data.
- OutputStream stream = storageHandler.newKeyWriter(keyArgs);
- stream.close();
- }
-
- // List all keys in bucket.
- bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
- listKeyArgs = new ListArgs(bucketArgs, null, 100, null);
- result = storageHandler.listKeys(listKeyArgs);
- Assert.assertEquals(numKeys, result.getKeyList().size());
-
- // List keys with prefix "aKey".
- listKeyArgs = new ListArgs(bucketArgs, "aKey", 100, null);
- result = storageHandler.listKeys(listKeyArgs);
- Assert.assertEquals(numKeys / 2, result.getKeyList().size());
- Assert.assertTrue(result.getKeyList().stream()
- .allMatch(entry -> entry.getKeyName().startsWith("aKey")));
-
- // List a certain number of keys.
- listKeyArgs = new ListArgs(bucketArgs, null, 3, null);
- result = storageHandler.listKeys(listKeyArgs);
- Assert.assertEquals(3, result.getKeyList().size());
- Assert.assertEquals("aKey0",
- result.getKeyList().get(0).getKeyName());
- Assert.assertEquals("aKey10",
- result.getKeyList().get(1).getKeyName());
- Assert.assertEquals("aKey12",
- result.getKeyList().get(2).getKeyName());
-
- // List a certain number of keys from the startKey.
- listKeyArgs = new ListArgs(bucketArgs, null, 2, "bKey1");
- result = storageHandler.listKeys(listKeyArgs);
- Assert.assertEquals(2, result.getKeyList().size());
- Assert.assertEquals("bKey11",
- result.getKeyList().get(0).getKeyName());
- Assert.assertEquals("bKey13",
- result.getKeyList().get(1).getKeyName());
-
- // Provide an invalid key name as start key.
- listKeyArgs = new ListArgs(bucketArgs, null, 100, "invalid_start_key");
- ListKeys keys = storageHandler.listKeys(listKeyArgs);
- Assert.assertEquals(keys.getKeyList().size(), 0);
-
- // Provide an invalid maxKeys argument.
- try {
- listKeyArgs = new ListArgs(bucketArgs, null, -1, null);
- storageHandler.listBuckets(listKeyArgs);
- Assert.fail("Expecting an error when the given"
- + " maxKeys argument is invalid.");
- } catch (Exception e) {
- GenericTestUtils.assertExceptionContains(
- String.format("the value must be in range (0, %d]",
- OzoneConsts.MAX_LISTKEYS_SIZE), e);
- }
-
- OzoneTestUtils.expectOmException(ResultCodes.BUCKET_NOT_FOUND, () -> {
- // Provide an invalid bucket name.
- BucketArgs bucket = new BucketArgs("invalid_bucket", createVolumeArgs);
- ListArgs ks = new ListArgs(bucket, null, numKeys, null);
- storageHandler.listKeys(ks);
- });
- }
-
- @Test
- public void testListVolumes() throws IOException, OzoneException {
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- String user0 = ugi.getUserName();
- String user1 = "testListVolumes-user-1";
- String adminUser = "testListVolumes-admin";
- ListArgs listVolumeArgs;
- ListVolumes volumes;
-
- // Create 10 volumes by user0 and user1
- String[] user0vols = new String[10];
- String[] user1vols = new String[10];
- for (int i =0; i<10; i++) {
- VolumeArgs createVolumeArgs;
- String user0VolName = "Vol-" + user0 + "-" + i;
- user0vols[i] = user0VolName;
- createVolumeArgs = new VolumeArgs(user0VolName, userArgs);
- createVolumeArgs.setUserName(user0);
- createVolumeArgs.setAdminName(adminUser);
- createVolumeArgs.setQuota(new OzoneQuota(i, OzoneQuota.Units.GB));
- storageHandler.createVolume(createVolumeArgs);
-
- String user1VolName = "Vol-" + user1 + "-" + i;
- user1vols[i] = user1VolName;
- createVolumeArgs = new VolumeArgs(user1VolName, userArgs);
- createVolumeArgs.setUserName(user1);
- createVolumeArgs.setAdminName(adminUser);
- createVolumeArgs.setQuota(new OzoneQuota(i, OzoneQuota.Units.GB));
- storageHandler.createVolume(createVolumeArgs);
- }
-
- // Test list all volumes - Removed Support for this operation for time
- // being. TODO: we will need to bring this back if needed.
- UserArgs userArgs0 = new UserArgs(user0, OzoneUtils.getRequestID(),
- null, null, null, null);
- //listVolumeArgs = new ListArgs(userArgs0,"Vol-testListVolumes", 100, null);
- // listVolumeArgs.setRootScan(true);
- // volumes = storageHandler.listVolumes(listVolumeArgs);
- // Assert.assertEquals(20, volumes.getVolumes().size());
-
- // Test list all volumes belongs to an user
- listVolumeArgs = new ListArgs(userArgs0, null, 100, null);
- listVolumeArgs.setRootScan(false);
- volumes = storageHandler.listVolumes(listVolumeArgs);
- Assert.assertEquals(10, volumes.getVolumes().size());
-
- // Test prefix
- listVolumeArgs = new ListArgs(userArgs0,
- "Vol-" + user0 + "-3", 100, null);
- volumes = storageHandler.listVolumes(listVolumeArgs);
- Assert.assertEquals(1, volumes.getVolumes().size());
- Assert.assertEquals(user0vols[3],
- volumes.getVolumes().get(0).getVolumeName());
- Assert.assertEquals(user0,
- volumes.getVolumes().get(0).getOwner().getName());
-
- // Test list volumes by user
- UserArgs userArgs1 = new UserArgs(user1, OzoneUtils.getRequestID(),
- null, null, null, null);
- listVolumeArgs = new ListArgs(userArgs1, null, 100, null);
- listVolumeArgs.setRootScan(false);
- volumes = storageHandler.listVolumes(listVolumeArgs);
- Assert.assertEquals(0, volumes.getVolumes().size());
-
- // Make sure all available fields are returned
- final String user0vol4 = "Vol-" + user0 + "-4";
- final String user0vol5 = "Vol-" + user0 + "-5";
- listVolumeArgs = new ListArgs(userArgs0, null, 1, user0vol4);
- listVolumeArgs.setRootScan(false);
- volumes = storageHandler.listVolumes(listVolumeArgs);
- Assert.assertEquals(1, volumes.getVolumes().size());
- Assert.assertEquals(user0,
- volumes.getVolumes().get(0).getOwner().getName());
- Assert.assertEquals(user0vol5,
- volumes.getVolumes().get(0).getVolumeName());
- Assert.assertEquals(5,
- volumes.getVolumes().get(0).getQuota().getSize());
- Assert.assertEquals(OzoneQuota.Units.GB,
- volumes.getVolumes().get(0).getQuota().getUnit());
-
- // User doesn't have volumes
- UserArgs userArgsX = new UserArgs("unknwonUser", OzoneUtils.getRequestID(),
- null, null, null, null);
- listVolumeArgs = new ListArgs(userArgsX, null, 100, null);
- listVolumeArgs.setRootScan(false);
- volumes = storageHandler.listVolumes(listVolumeArgs);
- Assert.assertEquals(0, volumes.getVolumes().size());
- }
-
- /**
- * Test get key information.
- *
- * @throws IOException
- * @throws OzoneException
- */
- @Test
- public void testGetKeyInfo() throws IOException,
- OzoneException, ParseException {
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- long currentTime = Time.now();
-
- VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs);
-
- BucketArgs bucketArgs = new BucketArgs(bucketName, createVolumeArgs);
- bucketArgs.setStorageType(StorageType.DISK);
- storageHandler.createBucket(bucketArgs);
-
- String keyName = "testKey";
- KeyArgs keyArgs = new KeyArgs(keyName, bucketArgs);
- keyArgs.setSize(4096);
-
-
- OutputStream stream = storageHandler.newKeyWriter(keyArgs);
- stream.close();
-
- KeyInfo keyInfo = storageHandler.getKeyInfo(keyArgs);
- // Compare the time in second unit since the date string reparsed to
- // millisecond will lose precision.
- Assert.assertTrue(
- (HddsClientUtils.formatDateTime(keyInfo.getCreatedOn()) / 1000) >= (
- currentTime / 1000));
- Assert.assertTrue(
- (HddsClientUtils.formatDateTime(keyInfo.getModifiedOn()) / 1000) >= (
- currentTime / 1000));
- Assert.assertEquals(keyName, keyInfo.getKeyName());
- // with out data written, the size would be 0
- Assert.assertEquals(0, keyInfo.getSize());
- }
-
- /**
- * Test that the write can proceed without having to set the right size.
- *
- * @throws IOException
- */
- @Test
- public void testWriteSize() throws IOException, OzoneException {
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
-
- VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs);
-
- BucketArgs bucketArgs = new BucketArgs(bucketName, createVolumeArgs);
- bucketArgs.setStorageType(StorageType.DISK);
- storageHandler.createBucket(bucketArgs);
-
- String dataString = RandomStringUtils.randomAscii(100);
- // write a key without specifying size at all
- String keyName = "testKey";
- KeyArgs keyArgs = new KeyArgs(keyName, bucketArgs);
- try (OutputStream stream = storageHandler.newKeyWriter(keyArgs)) {
- stream.write(dataString.getBytes());
- }
- byte[] data = new byte[dataString.length()];
- try (InputStream in = storageHandler.newKeyReader(keyArgs)) {
- in.read(data);
- }
- Assert.assertEquals(dataString, DFSUtil.bytes2String(data));
-
- // write a key with a size, but write above it.
- String keyName1 = "testKey1";
- KeyArgs keyArgs1 = new KeyArgs(keyName1, bucketArgs);
- keyArgs1.setSize(30);
- try (OutputStream stream = storageHandler.newKeyWriter(keyArgs1)) {
- stream.write(dataString.getBytes());
- }
- byte[] data1 = new byte[dataString.length()];
- try (InputStream in = storageHandler.newKeyReader(keyArgs1)) {
- in.read(data1);
- }
- Assert.assertEquals(dataString, DFSUtil.bytes2String(data1));
- }
-
- /**
- * Tests the RPC call for getting scmId and clusterId from SCM.
- * @throws IOException
- */
- @Test
- public void testGetScmInfo() throws IOException {
- ScmInfo info = cluster.getOzoneManager().getScmInfo();
- Assert.assertEquals(clusterId, info.getClusterId());
- Assert.assertEquals(scmId, info.getScmId());
- }
-
-
- //Disabling this test
- @Ignore("Disabling this test until Open Key is fixed.")
- public void testExpiredOpenKey() throws Exception {
-// BackgroundService openKeyCleanUpService = ((BlockManagerImpl)cluster
-// .getOzoneManager().getBlockManager()).getOpenKeyCleanupService();
-
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
-
- VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
- createVolumeArgs.setUserName(userName);
- createVolumeArgs.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs);
-
- BucketArgs bucketArgs = new BucketArgs(bucketName, createVolumeArgs);
- bucketArgs.setStorageType(StorageType.DISK);
- storageHandler.createBucket(bucketArgs);
-
- // open some keys.
-
- KeyArgs keyArgs1 = new KeyArgs("testKey1", bucketArgs);
- KeyArgs keyArgs2 = new KeyArgs("testKey2", bucketArgs);
- KeyArgs keyArgs3 = new KeyArgs("testKey3", bucketArgs);
- KeyArgs keyArgs4 = new KeyArgs("testKey4", bucketArgs);
- List openKeys;
- storageHandler.newKeyWriter(keyArgs1);
- storageHandler.newKeyWriter(keyArgs2);
- storageHandler.newKeyWriter(keyArgs3);
- storageHandler.newKeyWriter(keyArgs4);
-
- Set expected = Stream.of(
- "testKey1", "testKey2", "testKey3", "testKey4")
- .collect(Collectors.toSet());
-
- // Now all k1-k4 should be in open state, so ExpiredOpenKeys should not
- // contain these values.
- openKeys = cluster.getOzoneManager()
- .getMetadataManager().getExpiredOpenKeys();
-
- for (BlockGroup bg : openKeys) {
- String[] subs = bg.getGroupID().split("/");
- String keyName = subs[subs.length - 1];
- Assert.assertFalse(expected.contains(keyName));
- }
-
- Thread.sleep(2000);
- // Now all k1-k4 should be in ExpiredOpenKeys
- openKeys = cluster.getOzoneManager()
- .getMetadataManager().getExpiredOpenKeys();
- for (BlockGroup bg : openKeys) {
- String[] subs = bg.getGroupID().split("/");
- String keyName = subs[subs.length - 1];
- if (expected.contains(keyName)) {
- expected.remove(keyName);
- }
- }
- Assert.assertEquals(0, expected.size());
-
- KeyArgs keyArgs5 = new KeyArgs("testKey5", bucketArgs);
- storageHandler.newKeyWriter(keyArgs5);
-
- //openKeyCleanUpService.triggerBackgroundTaskForTesting();
- Thread.sleep(2000);
- // now all k1-k4 should have been removed by the clean-up task, only k5
- // should be present in ExpiredOpenKeys.
- openKeys =
- cluster.getOzoneManager().getMetadataManager().getExpiredOpenKeys();
- System.out.println(openKeys);
- boolean key5found = false;
- Set removed = Stream.of(
- "testKey1", "testKey2", "testKey3", "testKey4")
- .collect(Collectors.toSet());
- for (BlockGroup bg : openKeys) {
- String[] subs = bg.getGroupID().split("/");
- String keyName = subs[subs.length - 1];
- Assert.assertFalse(removed.contains(keyName));
- if (keyName.equals("testKey5")) {
- key5found = true;
- }
- }
- Assert.assertTrue(key5found);
- }
-
- /**
- * Tests the OM Initialization.
- * @throws IOException
- */
- @Test
- public void testOmInitialization() throws IOException {
- // Read the version file info from OM version file
- OMStorage omStorage = cluster.getOzoneManager().getOmStorage();
- SCMStorageConfig scmStorageConfig = new SCMStorageConfig(conf);
- // asserts whether cluster Id and SCM ID are properly set in SCM Version
- // file.
- Assert.assertEquals(clusterId, scmStorageConfig.getClusterID());
- Assert.assertEquals(scmId, scmStorageConfig.getScmId());
- // asserts whether OM Id is properly set in OM Version file.
- Assert.assertEquals(omId, omStorage.getOmId());
- // asserts whether the SCM info is correct in OM Version file.
- Assert.assertEquals(clusterId, omStorage.getClusterID());
- Assert.assertEquals(scmId, omStorage.getScmId());
- }
-
- /**
- * Tests the OM Initialization Failure.
- * @throws IOException
- */
- @Test
- public void testOmInitializationFailure() throws Exception {
- OzoneConfiguration config = new OzoneConfiguration();
- final String path =
- GenericTestUtils.getTempPath(UUID.randomUUID().toString());
- Path metaDirPath = Paths.get(path, "om-meta");
- config.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDirPath.toString());
- config.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
- config.set(OZONE_OM_ADDRESS_KEY, "127.0.0.1:0");
- config.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "127.0.0.1:0");
- config.set(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY,
- conf.get(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY));
-
- OzoneTestUtils.expectOmException(ResultCodes.OM_NOT_INITIALIZED, () -> {
- OzoneManager.createOm(config);
- });
-
- OzoneTestUtils
- .expectOmException(ResultCodes.SCM_VERSION_MISMATCH_ERROR, () -> {
- OMStorage omStore = new OMStorage(config);
- omStore.setClusterId("testClusterId");
- omStore.setScmId("testScmId");
- // writes the version file properties
- omStore.initialize();
- OzoneManager.createOm(config);
- });
- }
-
- @Test
- public void testGetServiceList() throws IOException {
- long numGetServiceListCalls = omMetrics.getNumGetServiceLists();
- List services = cluster.getOzoneManager().getServiceList();
-
- Assert.assertEquals(numGetServiceListCalls + 1,
- omMetrics.getNumGetServiceLists());
-
- ServiceInfo omInfo = services.stream().filter(
- a -> a.getNodeType().equals(HddsProtos.NodeType.OM))
- .collect(Collectors.toList()).get(0);
- InetSocketAddress omAddress = new InetSocketAddress(omInfo.getHostname(),
- omInfo.getPort(ServicePort.Type.RPC));
- Assert.assertEquals(NetUtils.createSocketAddr(
- conf.get(OZONE_OM_ADDRESS_KEY)), omAddress);
-
- ServiceInfo scmInfo = services.stream().filter(
- a -> a.getNodeType().equals(HddsProtos.NodeType.SCM))
- .collect(Collectors.toList()).get(0);
- InetSocketAddress scmAddress = new InetSocketAddress(scmInfo.getHostname(),
- scmInfo.getPort(ServicePort.Type.RPC));
- Assert.assertEquals(NetUtils.createSocketAddr(
- conf.get(OZONE_SCM_CLIENT_ADDRESS_KEY)), scmAddress);
- }
-
- @Test
- public void testVersion() {
- String expectedVersion = OzoneVersionInfo.OZONE_VERSION_INFO.getVersion();
- String actualVersion = cluster.getOzoneManager().getSoftwareVersion();
- Assert.assertEquals(expectedVersion, actualVersion);
- }
-
- /**
- * Test if OM RocksDB keyMayExist API works for keys that are present.
- * Test added in this module since we need access to custom codec dependent
- * objects like OMKeyInfo.
- * @throws Exception if OM or RocksDB operations fail.
- */
- @Test
- public void testDBKeyMayExist() throws Exception {
- RDBStore rdbStore = (RDBStore) cluster.getOzoneManager()
- .getMetadataManager().getStore();
- RocksDB db = rdbStore.getDb();
-
- OmKeyInfo keyInfo = getNewOmKeyInfo();
- OmKeyInfoCodec omKeyInfoCodec = new OmKeyInfoCodec();
-
- db.put(StringUtils.getBytesUtf16("OMKey1"),
- omKeyInfoCodec.toPersistedFormat(keyInfo));
-
- StringBuilder sb = new StringBuilder();
- Assert.assertTrue(db.keyMayExist(StringUtils.getBytesUtf16("OMKey1"),
- sb));
- Assert.assertTrue(sb.length() > 0);
- }
-
-
- @Test
- public void testGetOMDBUpdates() throws IOException {
-
- DBUpdatesRequest dbUpdatesRequest =
- DBUpdatesRequest.newBuilder().setSequenceNumber(0).build();
-
- DBUpdatesWrapper dbUpdates =
- cluster.getOzoneManager().getDBUpdates(dbUpdatesRequest);
- Assert.assertTrue(dbUpdates.getData().isEmpty());
-
- //Write data to OM.
- OmKeyInfo keyInfo = getNewOmKeyInfo();
- Assert.assertNotNull(keyInfo);
- dbUpdates =
- cluster.getOzoneManager().getDBUpdates(dbUpdatesRequest);
- Assert.assertFalse(dbUpdates.getData().isEmpty());
-
- }
-
- private OmKeyInfo getNewOmKeyInfo() throws IOException {
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- OmVolumeArgs volumeArgs = OmVolumeArgs.newBuilder()
- .setVolume("vol1")
- .setAdminName("bilbo")
- .setOwnerName("bilbo")
- .build();
- cluster.getOzoneManager().createVolume(volumeArgs);
-
- OmBucketInfo bucketInfo = OmBucketInfo.newBuilder()
- .setVolumeName("vol1")
- .setBucketName("bucket1")
- .build();
- cluster.getOzoneManager().createBucket(bucketInfo);
-
- OmKeyArgs keyArgs = new OmKeyArgs.Builder()
- .setBucketName("bucket1")
- .setFactor(HddsProtos.ReplicationFactor.ONE)
- .setDataSize(0)
- .setType(HddsProtos.ReplicationType.STAND_ALONE)
- .setAcls(OzoneAclUtil.getAclList(ugi.getUserName(), ugi.getGroups(),
- ALL, ALL))
- .setVolumeName("vol1")
- .setKeyName(UUID.randomUUID().toString())
- .setDataSize(16 * 1024 * 1024 * 10)
- .build();
- OpenKeySession keySession = cluster.getOzoneManager().getKeyManager()
- .openKey(keyArgs);
- return keySession.getKeyInfo();
- }
-}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java
index 76841ddcf680f..443f3059fd11d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java
@@ -18,7 +18,10 @@
package org.apache.hadoop.ozone.om;
-import org.apache.commons.lang3.RandomStringUtils;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.UUID;
+
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -29,25 +32,20 @@
import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.UUID;
+import org.apache.commons.lang3.RandomStringUtils;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS;
+import org.junit.After;
+import org.junit.Assert;
import static org.junit.Assert.fail;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
/**
* Test some client operations after cluster starts. And perform restart and
@@ -55,7 +53,6 @@
*/
public class TestOzoneManagerRestart {
private MiniOzoneCluster cluster = null;
- private UserArgs userArgs;
private OzoneConfiguration conf;
private String clusterId;
private String scmId;
@@ -86,8 +83,7 @@ public void init() throws Exception {
.setOmId(omId)
.build();
cluster.waitForClusterToBeReady();
- userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
- null, null, null, null);
+
}
/**
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneDatanodeShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneDatanodeShell.java
index e2942a4f7e950..65bc2751423e0 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneDatanodeShell.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneDatanodeShell.java
@@ -17,13 +17,10 @@
*/
package org.apache.hadoop.ozone.ozShell;
-import com.google.common.base.Strings;
-
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintStream;
import java.util.Arrays;
-import java.util.Collection;
import java.util.List;
import org.apache.hadoop.fs.FileUtil;
@@ -31,19 +28,19 @@
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.client.rest.RestClient;
-import org.apache.hadoop.ozone.client.rpc.RpcClient;
import org.apache.hadoop.test.GenericTestUtils;
+
+import com.google.common.base.Strings;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
+import static org.junit.Assert.fail;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import picocli.CommandLine;
@@ -53,13 +50,9 @@
import picocli.CommandLine.ParseResult;
import picocli.CommandLine.RunLast;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
-import static org.junit.Assert.fail;
-
/**
* This test class specified for testing Ozone datanode shell command.
*/
-@RunWith(value = Parameterized.class)
public class TestOzoneDatanodeShell {
private static final Logger LOG =
@@ -81,17 +74,6 @@ public class TestOzoneDatanodeShell {
private static final PrintStream OLD_OUT = System.out;
private static final PrintStream OLD_ERR = System.err;
- @Parameterized.Parameters
- public static Collection clientProtocol() {
- Object[][] params = new Object[][]{
- {RpcClient.class},
- {RestClient.class}};
- return Arrays.asList(params);
- }
-
- @Parameterized.Parameter
- @SuppressWarnings("visibilitymodifier")
- public Class clientProtocol;
/**
* Create a MiniDFSCluster for testing with using distributed Ozone
* handler type.
@@ -103,7 +85,7 @@ public static void init() throws Exception {
conf = new OzoneConfiguration();
String path = GenericTestUtils.getTempPath(
- TestOzoneShell.class.getSimpleName());
+ TestOzoneDatanodeShell.class.getSimpleName());
baseDir = new File(path);
baseDir.mkdirs();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
deleted file mode 100644
index bdcb307196f86..0000000000000
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
+++ /dev/null
@@ -1,1266 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ozShell;
-
-import java.io.ByteArrayOutputStream;
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-import java.util.Random;
-import java.util.UUID;
-
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdds.cli.MissingSubcommandException;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.tracing.StringCodec;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneKey;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.VolumeArgs;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rpc.RpcClient;
-import org.apache.hadoop.ozone.om.exceptions.OMException;
-import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
-import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort;
-import org.apache.hadoop.ozone.web.ozShell.OzoneShell;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.request.OzoneQuota;
-import org.apache.hadoop.ozone.web.response.BucketInfo;
-import org.apache.hadoop.ozone.web.response.KeyInfo;
-import org.apache.hadoop.ozone.web.response.VolumeInfo;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.test.GenericTestUtils;
-
-import com.google.common.base.Strings;
-import org.apache.commons.lang3.RandomStringUtils;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
-import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND;
-import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
-import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.slf4j.event.Level.TRACE;
-
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import picocli.CommandLine;
-import picocli.CommandLine.ExecutionException;
-import picocli.CommandLine.IExceptionHandler2;
-import picocli.CommandLine.ParameterException;
-import picocli.CommandLine.ParseResult;
-import picocli.CommandLine.RunLast;
-
-/**
- * This test class specified for testing Ozone shell command.
- */
-public class TestOzoneShell {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestOzoneShell.class);
-
- /**
- * Set the timeout for every test.
- */
- @Rule
- public Timeout testTimeout = new Timeout(300000);
-
- private static String url;
- private static File baseDir;
- private static OzoneConfiguration conf = null;
- private static MiniOzoneCluster cluster = null;
- private static ClientProtocol client = null;
- private static Shell shell = null;
-
- private final ByteArrayOutputStream out = new ByteArrayOutputStream();
- private final ByteArrayOutputStream err = new ByteArrayOutputStream();
- private static final PrintStream OLD_OUT = System.out;
- private static final PrintStream OLD_ERR = System.err;
-
- /**
- * Create a MiniDFSCluster for testing with using distributed Ozone
- * handler type.
- *
- * @throws Exception
- */
- @BeforeClass
- public static void init() throws Exception {
- conf = new OzoneConfiguration();
-
- String path = GenericTestUtils.getTempPath(
- TestOzoneShell.class.getSimpleName());
- baseDir = new File(path);
- baseDir.mkdirs();
-
- shell = new OzoneShell();
-
- cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(3)
- .build();
- conf.setInt(OZONE_REPLICATION, ReplicationFactor.THREE.getValue());
- conf.setQuietMode(false);
- client = new RpcClient(conf);
- cluster.waitForClusterToBeReady();
- }
-
- /**
- * shutdown MiniDFSCluster.
- */
- @AfterClass
- public static void shutdown() {
- if (cluster != null) {
- cluster.shutdown();
- }
-
- if (baseDir != null) {
- FileUtil.fullyDelete(baseDir, true);
- }
- }
-
- @Before
- public void setup() {
- System.setOut(new PrintStream(out));
- System.setErr(new PrintStream(err));
- url = "o3://" + getOmAddress();
- }
-
- @After
- public void reset() {
- // reset stream after each unit test
- out.reset();
- err.reset();
-
- // restore system streams
- System.setOut(OLD_OUT);
- System.setErr(OLD_ERR);
- }
-
- @Test
- public void testCreateVolume() throws Exception {
- LOG.info("Running testCreateVolume");
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- testCreateVolume(volumeName, "");
- volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- testCreateVolume("/////" + volumeName, "");
- testCreateVolume("/////", "Volume name is required");
- testCreateVolume("/////vol/123",
- "Invalid volume name. Delimiters (/) not allowed in volume name");
- }
-
- private void testCreateVolume(String volumeName, String errorMsg)
- throws Exception {
- err.reset();
- String userName = "bilbo";
- String[] args = new String[] {"volume", "create", url + "/" + volumeName,
- "--user", userName, "--root"};
-
- if (Strings.isNullOrEmpty(errorMsg)) {
- execute(shell, args);
-
- } else {
- executeWithError(shell, args, errorMsg);
- return;
- }
-
- String truncatedVolumeName =
- volumeName.substring(volumeName.lastIndexOf('/') + 1);
- OzoneVolume volumeInfo = client.getVolumeDetails(truncatedVolumeName);
- assertEquals(truncatedVolumeName, volumeInfo.getName());
- assertEquals(userName, volumeInfo.getOwner());
- }
-
- private void execute(Shell ozoneShell, String[] args) {
- LOG.info("Executing shell command with args {}", Arrays.asList(args));
- CommandLine cmd = ozoneShell.getCmd();
-
- IExceptionHandler2> exceptionHandler =
- new IExceptionHandler2>() {
- @Override
- public List
-
- org.apache.hadoop
- hadoop-ozone-objectstore-service
- provided
-
com.google.code.findbugs
findbugs
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
index 913bb3ebd569e..2e9e3a4d41e12 100644
--- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
@@ -18,40 +18,36 @@
package org.apache.hadoop.fs.ozone;
-import org.apache.commons.lang3.RandomStringUtils;
+import java.io.IOException;
+import java.util.Arrays;
+
import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+
+import org.apache.commons.lang3.RandomStringUtils;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
-import java.io.IOException;
-import java.util.Arrays;
-
/**
* Test OzoneFSInputStream by reading through multiple interfaces.
*/
public class TestOzoneFSInputStream {
private static MiniOzoneCluster cluster = null;
private static FileSystem fs;
- private static StorageHandler storageHandler;
private static Path filePath = null;
private static byte[] data = null;
@@ -71,22 +67,9 @@ public static void init() throws Exception {
.setNumDatanodes(10)
.build();
cluster.waitForClusterToBeReady();
- storageHandler =
- new ObjectStoreHandler(conf).getStorageHandler();
// create a volume and a bucket to be used by OzoneFileSystem
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
- null, null, null, null);
- VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
- volumeArgs.setUserName(userName);
- volumeArgs.setAdminName(adminName);
- storageHandler.createVolume(volumeArgs);
- BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
- storageHandler.createBucket(bucketArgs);
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
// Fetch the host and port for File System init
DatanodeDetails datanodeDetails = cluster.getHddsDatanodes().get(0)
@@ -94,7 +77,7 @@ public static void init() throws Exception {
// Set the fs.defaultFS and start the filesystem
String uri = String.format("%s://%s.%s/",
- OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName);
+ OzoneConsts.OZONE_URI_SCHEME, bucket.getName(), bucket.getVolumeName());
conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, uri);
fs = FileSystem.get(conf);
int fileLen = 100 * 1024 * 1024;
@@ -111,7 +94,6 @@ public static void init() throws Exception {
@AfterClass
public static void shutdown() throws IOException {
fs.close();
- storageHandler.close();
cluster.shutdown();
}
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
index ca3e64f18b92c..aac45fa1d973c 100644
--- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
@@ -25,45 +25,40 @@
import java.util.Collection;
import java.util.List;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.om.OMMetrics;
-import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
-import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.RandomStringUtils;
-import org.junit.After;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.GlobalStorageStatistics;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.StorageStatistics;
-import org.apache.hadoop.fs.GlobalStorageStatistics;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.om.OMMetrics;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Time;
import static java.nio.charset.StandardCharsets.UTF_8;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import org.junit.After;
+import org.junit.Assert;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
/**
* Test OzoneFileSystem Interfaces.
@@ -103,8 +98,6 @@ public static Collection data() {
private static String bucketName;
- private static StorageHandler storageHandler;
-
private OzoneFSStorageStatistics statistics;
private OMMetrics omMetrics;
@@ -123,22 +116,10 @@ public void init() throws Exception {
.setNumDatanodes(3)
.build();
cluster.waitForClusterToBeReady();
- storageHandler =
- new ObjectStoreHandler(conf).getStorageHandler();
// create a volume and a bucket to be used by OzoneFileSystem
- userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
- null, null, null, null);
- VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
- volumeArgs.setUserName(userName);
- volumeArgs.setAdminName(adminName);
- storageHandler.createVolume(volumeArgs);
- BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
- storageHandler.createBucket(bucketArgs);
+ OzoneBucket bucket =
+ TestDataUtil.createVolumeAndBucket(cluster, volumeName, bucketName);
rootPath = String
.format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME, bucketName,
@@ -161,7 +142,6 @@ public void teardown() throws IOException {
cluster.shutdown();
}
IOUtils.closeQuietly(fs);
- IOUtils.closeQuietly(storageHandler);
}
@Test
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
index ac8f11fc9057b..9fb6bf962d9cd 100644
--- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
@@ -18,41 +18,34 @@
package org.apache.hadoop.fs.ozone;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.RandomStringUtils;
+import java.io.IOException;
+import java.util.Set;
+import java.util.TreeSet;
+
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.KeyArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.response.KeyInfo;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
import org.apache.hadoop.test.GenericTestUtils;
+
+import org.apache.commons.io.IOUtils;
import org.junit.After;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Set;
-import java.util.TreeSet;
-
import org.junit.rules.Timeout;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
/**
* Ozone file system tests that are not covered by contract tests.
*/
@@ -66,8 +59,6 @@ public class TestOzoneFileSystem {
private static FileSystem fs;
private static OzoneFileSystem o3fs;
- private static StorageHandler storageHandler;
- private static UserArgs userArgs;
private String volumeName;
private String bucketName;
private String userName;
@@ -80,25 +71,12 @@ public void init() throws Exception {
.setNumDatanodes(3)
.build();
cluster.waitForClusterToBeReady();
- storageHandler =
- new ObjectStoreHandler(conf).getStorageHandler();
// create a volume and a bucket to be used by OzoneFileSystem
- userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
- null, null, null, null);
- VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
- volumeArgs.setUserName(userName);
- volumeArgs.setAdminName(adminName);
- storageHandler.createVolume(volumeArgs);
- BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
- storageHandler.createBucket(bucketArgs);
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
rootPath = String.format("%s://%s.%s/",
- OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName);
+ OzoneConsts.OZONE_URI_SCHEME, bucket.getName(), bucket.getVolumeName());
// Set the fs.defaultFS and start the filesystem
conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
@@ -112,7 +90,6 @@ public void teardown() throws IOException {
cluster.shutdown();
}
IOUtils.closeQuietly(fs);
- IOUtils.closeQuietly(storageHandler);
}
@Test
@@ -129,8 +106,8 @@ public void testCreateDoesNotAddParentDirKeys() throws Exception {
Path child = new Path(parent, "child");
ContractTestUtils.touch(fs, child);
- KeyInfo key = getKey(child, false);
- assertEquals(key.getKeyName(), o3fs.pathToKey(child));
+ OzoneKeyDetails key = getKey(child, false);
+ assertEquals(key.getName(), o3fs.pathToKey(child));
// Creating a child should not add parent keys to the bucket
try {
@@ -167,8 +144,8 @@ public void testDeleteCreatesFakeParentDir() throws Exception {
// Deleting the only child should create the parent dir key if it does
// not exist
String parentKey = o3fs.pathToKey(parent) + "/";
- KeyInfo parentKeyInfo = getKey(parent, true);
- assertEquals(parentKey, parentKeyInfo.getKeyName());
+ OzoneKeyDetails parentKeyInfo = getKey(parent, true);
+ assertEquals(parentKey, parentKeyInfo.getName());
}
@Test
@@ -285,15 +262,14 @@ public void testListStatusOnSubDirs() throws Exception {
fileStatus2.equals(dir12.toString()));
}
- private KeyInfo getKey(Path keyPath, boolean isDirectory)
- throws IOException, OzoneException {
+ private OzoneKeyDetails getKey(Path keyPath, boolean isDirectory)
+ throws IOException, OzoneClientException {
String key = o3fs.pathToKey(keyPath);
if (isDirectory) {
key = key + "/";
}
- KeyArgs parentKeyArgs = new KeyArgs(volumeName, bucketName, key,
- userArgs);
- return storageHandler.getKeyInfo(parentKeyArgs);
+ return cluster.getClient().getObjectStore().getVolume(volumeName)
+ .getBucket(bucketName).getKey(key);
}
private void assertKeyNotFoundException(IOException ex) {
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsRenameDir.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsRenameDir.java
index 0c639e7dccdb3..1d584651bf98c 100644
--- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsRenameDir.java
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsRenameDir.java
@@ -18,30 +18,25 @@
package org.apache.hadoop.fs.ozone;
-import org.apache.commons.lang3.RandomStringUtils;
+import java.io.IOException;
+
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+
import org.junit.After;
+import static org.junit.Assert.assertTrue;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-
-import static org.junit.Assert.assertTrue;
-
/**
* Unit Test for verifying directory rename operation through OzoneFS.
*/
@@ -51,7 +46,6 @@ public class TestOzoneFsRenameDir {
private MiniOzoneCluster cluster = null;
private OzoneConfiguration conf = null;
- private static StorageHandler storageHandler;
private static FileSystem fs;
@Before
@@ -61,22 +55,9 @@ public void init() throws Exception {
.setNumDatanodes(1)
.build();
cluster.waitForClusterToBeReady();
- storageHandler =
- new ObjectStoreHandler(conf).getStorageHandler();
// create a volume and a bucket to be used by OzoneFileSystem
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
- null, null, null, null);
- VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
- volumeArgs.setUserName(userName);
- volumeArgs.setAdminName(adminName);
- storageHandler.createVolume(volumeArgs);
- BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
- storageHandler.createBucket(bucketArgs);
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
// Fetch the host and port for File System init
DatanodeDetails datanodeDetails = cluster.getHddsDatanodes().get(0)
@@ -84,7 +65,7 @@ public void init() throws Exception {
// Set the fs.defaultFS and start the filesystem
String uri = String.format("%s://%s.%s/",
- OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName);
+ OzoneConsts.OZONE_URI_SCHEME, bucket.getName(), bucket.getVolumeName());
conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, uri);
fs = FileSystem.get(conf);
LOG.info("fs.defaultFS=" + fs.getUri());
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
index b63e182a1e78b..56d63ac2f2580 100644
--- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
@@ -18,26 +18,21 @@
package org.apache.hadoop.fs.ozone.contract;
-import org.apache.commons.lang3.RandomStringUtils;
+import java.io.IOException;
+
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.om.OMConfigKeys;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.junit.Assert;
-import java.io.IOException;
+import org.junit.Assert;
/**
* The contract of Ozone: only enabled if the test bucket is provided.
@@ -45,7 +40,6 @@
class OzoneContract extends AbstractFSContract {
private static MiniOzoneCluster cluster;
- private static StorageHandler storageHandler;
private static final String CONTRACT_XML = "contract/ozone.xml";
OzoneContract(Configuration conf) {
@@ -75,7 +69,6 @@ public static void createCluster() throws IOException {
} catch (Exception e) {
throw new IOException(e);
}
- storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
}
private void copyClusterConfigs(String configKey) {
@@ -87,27 +80,10 @@ public FileSystem getTestFileSystem() throws IOException {
//assumes cluster is not null
Assert.assertNotNull("cluster not created", cluster);
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
-
-
- UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
- null, null, null, null);
- VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
- volumeArgs.setUserName(userName);
- volumeArgs.setAdminName(adminName);
- BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
- try {
- storageHandler.createVolume(volumeArgs);
- storageHandler.createBucket(bucketArgs);
- } catch (OzoneException e) {
- throw new IOException(e.getMessage());
- }
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
String uri = String.format("%s://%s.%s/",
- OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName);
+ OzoneConsts.OZONE_URI_SCHEME, bucket.getName(), bucket.getVolumeName());
getConf().set("fs.defaultFS", uri);
copyClusterConfigs(OMConfigKeys.OZONE_OM_ADDRESS_KEY);
copyClusterConfigs(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index 967303e5a55b0..9d00bc4d36e98 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -46,7 +46,6 @@
ozonefs-lib-legacy
tools
integration-test
- objectstore-service
datanode
s3gateway
dist
@@ -82,11 +81,6 @@
hadoop-ozone-ozone-manager
${ozone.version}
-
- org.apache.hadoop
- hadoop-ozone-objectstore-service
- ${ozone.version}
-
org.apache.hadoop
hadoop-ozone-s3gateway
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java
index 17cf7bca5a3df..252d87b307cba 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java
@@ -17,31 +17,30 @@
*/
package org.apache.hadoop.ozone.s3;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
-import org.apache.hadoop.test.LambdaTestUtils;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.mockito.Mockito;
-
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.core.MultivaluedHashMap;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.UriInfo;
-
import java.io.IOException;
import java.net.URI;
import java.util.Arrays;
import java.util.Collection;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.test.LambdaTestUtils;
+
import static org.apache.hadoop.ozone.s3.AWSAuthParser.AUTHORIZATION_HEADER;
import static org.apache.hadoop.ozone.s3.AWSAuthParser.CONTENT_MD5;
import static org.apache.hadoop.ozone.s3.AWSAuthParser.CONTENT_TYPE;
import static org.apache.hadoop.ozone.s3.AWSAuthParser.HOST_HEADER;
import static org.apache.hadoop.ozone.s3.AWSAuthParser.X_AMAZ_DATE;
import static org.apache.hadoop.ozone.s3.AWSAuthParser.X_AMZ_CONTENT_SHA256;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.Mockito;
/**
* Test class for @{@link OzoneClientProducer}.
@@ -87,8 +86,8 @@ public TestOzoneClientProducer(String authHeader, String contentMd5,
@Test
public void testGetClientFailure() throws Exception {
- LambdaTestUtils.intercept(IOException.class, "Couldn't create" +
- " protocol ", () -> producer.createClient());
+ LambdaTestUtils.intercept(IOException.class, "Couldn't create",
+ () -> producer.createClient());
}
private void setupContext() throws Exception {
@@ -143,4 +142,4 @@ public static Collection data() {
});
}
-}
\ No newline at end of file
+}
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java
index 88fcb7b687f50..2cdbf0d97cc03 100644
--- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java
@@ -16,28 +16,7 @@
*/
package org.apache.hadoop.ozone.om;
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.scm.cli.SQLCLI;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.KeyArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
import java.io.IOException;
-import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.sql.Connection;
@@ -52,11 +31,26 @@
import java.util.List;
import java.util.UUID;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.scm.cli.SQLCLI;
+import org.apache.hadoop.test.GenericTestUtils;
+
import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME;
+import org.junit.After;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
/**
* This class tests the CLI that transforms om.db into SQLite DB files.
@@ -64,8 +58,7 @@
@RunWith(Parameterized.class)
public class TestOmSQLCli {
private MiniOzoneCluster cluster = null;
- private StorageHandler storageHandler;
- private UserArgs userArgs;
+
private OzoneConfiguration conf;
private SQLCLI cli;
@@ -108,48 +101,17 @@ public void setup() throws Exception {
conf = new OzoneConfiguration();
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
- storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
- userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
- null, null, null, null);
- cluster.waitForClusterToBeReady();
-
- VolumeArgs createVolumeArgs0 = new VolumeArgs(volumeName0, userArgs);
- createVolumeArgs0.setUserName(userName);
- createVolumeArgs0.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs0);
- VolumeArgs createVolumeArgs1 = new VolumeArgs(volumeName1, userArgs);
- createVolumeArgs1.setUserName(userName);
- createVolumeArgs1.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs1);
-
- BucketArgs bucketArgs0 = new BucketArgs(volumeName0, bucketName0, userArgs);
- storageHandler.createBucket(bucketArgs0);
- BucketArgs bucketArgs1 = new BucketArgs(volumeName1, bucketName1, userArgs);
- storageHandler.createBucket(bucketArgs1);
- BucketArgs bucketArgs2 = new BucketArgs(volumeName0, bucketName2, userArgs);
- storageHandler.createBucket(bucketArgs2);
-
- KeyArgs keyArgs0 =
- new KeyArgs(volumeName0, bucketName0, keyName0, userArgs);
- keyArgs0.setSize(100);
- KeyArgs keyArgs1 =
- new KeyArgs(volumeName1, bucketName1, keyName1, userArgs);
- keyArgs1.setSize(200);
- KeyArgs keyArgs2 =
- new KeyArgs(volumeName0, bucketName2, keyName2, userArgs);
- keyArgs2.setSize(300);
- KeyArgs keyArgs3 =
- new KeyArgs(volumeName0, bucketName2, keyName3, userArgs);
- keyArgs3.setSize(400);
-
- OutputStream stream = storageHandler.newKeyWriter(keyArgs0);
- stream.close();
- stream = storageHandler.newKeyWriter(keyArgs1);
- stream.close();
- stream = storageHandler.newKeyWriter(keyArgs2);
- stream.close();
- stream = storageHandler.newKeyWriter(keyArgs3);
- stream.close();
+ OzoneBucket bucket0 =
+ TestDataUtil.createVolumeAndBucket(cluster, volumeName0, bucketName0);
+ OzoneBucket bucket1 =
+ TestDataUtil.createVolumeAndBucket(cluster, volumeName1, bucketName1);
+ OzoneBucket bucket2 =
+ TestDataUtil.createVolumeAndBucket(cluster, volumeName0, bucketName2);
+
+ TestDataUtil.createKey(bucket0, keyName0, "");
+ TestDataUtil.createKey(bucket1, keyName1, "");
+ TestDataUtil.createKey(bucket2, keyName2, "");
+ TestDataUtil.createKey(bucket2, keyName3, "");
cluster.getOzoneManager().stop();
cluster.getStorageContainerManager().stop();