diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt index 56755adac3f90..769e5da0f1ccd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt @@ -35,6 +35,7 @@ add_executable(hdfs_tool_tests hdfs-count-mock.cc hdfs-mkdir-mock.cc hdfs-rm-mock.cc + hdfs-get-mock.cc main.cc) target_include_directories(hdfs_tool_tests PRIVATE ../tools @@ -54,6 +55,7 @@ target_include_directories(hdfs_tool_tests PRIVATE ../../tools/hdfs-count ../../tools/hdfs-mkdir ../../tools/hdfs-rm + ../../tools/hdfs-get ../../tools/hdfs-cat) target_link_libraries(hdfs_tool_tests PRIVATE gmock_main @@ -72,5 +74,6 @@ target_link_libraries(hdfs_tool_tests PRIVATE hdfs_count_lib hdfs_mkdir_lib hdfs_rm_lib + hdfs_get_lib hdfs_cat_lib) add_test(hdfs_tool_tests hdfs_tool_tests) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.cc new file mode 100644 index 0000000000000..713564e45b160 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.cc @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-get-mock.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +GetMock::~GetMock() = default; + +void GetMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &Pass2Paths) { + const auto arg1 = args[0]; + const auto arg2 = args[1]; + EXPECT_CALL(*this, HandlePath(arg1, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.h new file mode 100644 index 0000000000000..535f7153f1f98 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.h @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_GET_MOCK +#define LIBHDFSPP_TOOLS_HDFS_GET_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-get.h" + +namespace hdfs::tools::test { +/** + * {@class GetMock} is an {@class Get} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class GetMock : public hdfs::tools::Get { +public: + /** + * {@inheritdoc} + */ + GetMock(const int argc, char **argv) : Get(argc, argv) {} + + // Abiding to the Rule of 5 + GetMock(const GetMock &) = delete; + GetMock(GetMock &&) = delete; + GetMock &operator=(const GetMock &) = delete; + GetMock &operator=(GetMock &&) = delete; + ~GetMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, (const std::string &, const std::string &), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc index 7678834a955ca..50d555aebda0c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc @@ -31,6 +31,7 @@ #include "hdfs-df-mock.h" #include "hdfs-disallow-snapshot-mock.h" #include "hdfs-du-mock.h" +#include "hdfs-get-mock.h" #include "hdfs-mkdir-mock.h" #include "hdfs-move-to-local-mock.h" #include "hdfs-rename-snapshot-mock.h" @@ -108,6 +109,11 @@ INSTANTIATE_TEST_SUITE_P( testing::Values(CallHelp, Pass2Paths)); +INSTANTIATE_TEST_SUITE_P( + HdfsGet, HdfsToolBasicTest, + testing::Values(CallHelp, + Pass2Paths)); + INSTANTIATE_TEST_SUITE_P( HdfsMoveToLocal, HdfsToolBasicTest, testing::Values(CallHelp, @@ -173,6 +179,10 @@ INSTANTIATE_TEST_SUITE_P( HdfsCopyToLocal, HdfsToolNegativeTestThrows, testing::Values(Pass3Paths)); +INSTANTIATE_TEST_SUITE_P( + HdfsGet, HdfsToolNegativeTestThrows, + testing::Values(Pass3Paths)); + INSTANTIATE_TEST_SUITE_P( HdfsMoveToLocal, HdfsToolNegativeTestThrows, testing::Values(Pass3Paths)); @@ -220,6 +230,10 @@ INSTANTIATE_TEST_SUITE_P( HdfsCopyToLocal, HdfsToolNegativeTestNoThrow, testing::Values(PassAPath)); +INSTANTIATE_TEST_SUITE_P( + HdfsGet, HdfsToolNegativeTestNoThrow, + testing::Values(PassAPath)); + INSTANTIATE_TEST_SUITE_P( HdfsDeleteSnapshot, HdfsToolNegativeTestNoThrow, testing::Values(PassAPath)); diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt index aa82c018b3871..bed78b7a5777d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt @@ -60,8 +60,7 @@ add_subdirectory(hdfs-df) add_subdirectory(hdfs-du) -add_executable(hdfs_get hdfs_get.cc) -target_link_libraries(hdfs_get tools_common hdfspp_static) +add_subdirectory(hdfs-get) add_subdirectory(hdfs-copy-to-local) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.cc index 9219b36ffd5da..7affa1fbdc92a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.cc @@ -64,7 +64,8 @@ bool CopyToLocal::ValidateConstraints() const { std::string CopyToLocal::GetDescription() const { std::stringstream desc; - desc << "Usage: hdfs_copyToLocal [OPTION] SRC_FILE DST_FILE" << std::endl + desc << "Usage: hdfs_" << GetToolName() << " [OPTION] SRC_FILE DST_FILE" + << std::endl << std::endl << "Copy SRC_FILE from hdfs to DST_FILE on the local file system." << std::endl @@ -72,16 +73,19 @@ std::string CopyToLocal::GetDescription() const { << " -h display this help and exit" << std::endl << std::endl << "Examples:" << std::endl - << "hdfs_copyToLocal hdfs://localhost.localdomain:8020/dir/file " + << "hdfs_" << GetToolName() + << " hdfs://localhost.localdomain:8020/dir/file " "/home/usr/myfile" << std::endl - << "hdfs_copyToLocal /dir/file /home/usr/dir/file" << std::endl; + << "hdfs_" << GetToolName() << " /dir/file /home/usr/dir/file" + << std::endl; return desc.str(); } bool CopyToLocal::Do() { if (!Initialize()) { - std::cerr << "Unable to initialize HDFS copyToLocal tool" << std::endl; + std::cerr << "Unable to initialize HDFS " << GetToolName() << " tool" + << std::endl; return false; } @@ -129,4 +133,6 @@ bool CopyToLocal::HandlePath(const std::string &source, std::fclose(dst_file); return true; } + +std::string CopyToLocal::GetToolName() const { return "copyToLocal"; } } // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.h index 6eb8cf1b922f9..0137f1e614d1c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.h @@ -81,6 +81,11 @@ class CopyToLocal : public HdfsTool { [[nodiscard]] virtual bool HandlePath(const std::string &source, const std::string &target) const; + /** + * @return The name of the tool. + */ + [[nodiscard]] virtual std::string GetToolName() const; + private: /** * A boost data-structure containing the description of positional arguments diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/CMakeLists.txt new file mode 100644 index 0000000000000..367bca6a0279c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_get_lib STATIC $ hdfs-get.cc) +target_include_directories(hdfs_get_lib PRIVATE ../../tools hdfs-get ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_get_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static hdfs_copyToLocal_lib) + +add_executable(hdfs_get main.cc) +target_include_directories(hdfs_get PRIVATE ../../tools) +target_link_libraries(hdfs_get PRIVATE hdfs_get_lib) + +install(TARGETS hdfs_get RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.cc new file mode 100644 index 0000000000000..e45c1d0215b26 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.cc @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "hdfs-get.h" + +namespace hdfs::tools { +Get::Get(const int argc, char **argv) : CopyToLocal(argc, argv) {} + +std::string Get::GetToolName() const { return "get"; } +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.h new file mode 100644 index 0000000000000..8153264a72a3e --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.h @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_GET +#define LIBHDFSPP_TOOLS_HDFS_GET + +#include "hdfs-copy-to-local/hdfs-copy-to-local.h" + +namespace hdfs::tools { +class Get : public CopyToLocal { +public: + /** + * {@inheritdoc} + */ + Get(int argc, char **argv); + + // Abiding to the Rule of 5 + Get(const Get &) = default; + Get(Get &&) = default; + Get &operator=(const Get &) = delete; + Get &operator=(Get &&) = delete; + ~Get() override = default; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetToolName() const override; +}; +} // namespace hdfs::tools + +#endif \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/main.cc new file mode 100644 index 0000000000000..916e60a4631c6 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/main.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "hdfs-get.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS " + "get tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::Get get(argc, argv); + auto success = false; + + try { + success = get.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc deleted file mode 100644 index 16dd72d24bf72..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc +++ /dev/null @@ -1,88 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_get [OPTION] SRC_FILE DST_FILE" - << std::endl - << std::endl << "Copy SRC_FILE from hdfs to DST_FILE on the local file system." - << std::endl - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_get hdfs://localhost.localdomain:8020/dir/file /home/usr/myfile" - << std::endl << "hdfs_get /dir/file /home/usr/dir/file" - << std::endl; -} - -int main(int argc, char *argv[]) { - if (argc > 4) { - usage(); - exit(EXIT_FAILURE); - } - - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "h")) != -1) { - switch (input) - { - case 'h': - usage(); - exit(EXIT_SUCCESS); - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - - std::string uri_path = argv[optind]; - std::string dest = argv[optind+1]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, false); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - std::FILE* dst_file = std::fopen(dest.c_str(), "wb"); - if(!dst_file){ - std::cerr << "Unable to open the destination file: " << dest << std::endl; - exit(EXIT_FAILURE); - } - readFile(fs, uri.get_path(), 0, dst_file, false); - std::fclose(dst_file); - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -}