From 5d4c972fd615c584e74083dfe17bafce2220738a Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Thu, 21 Mar 2024 19:22:21 +0000 Subject: [PATCH 1/9] feat: generate selected libraries --- library_generation/cli/generate_repo.py | 71 ++++++++++++++++++++ library_generation/generate_repo.py | 88 +++++++------------------ 2 files changed, 96 insertions(+), 63 deletions(-) create mode 100644 library_generation/cli/generate_repo.py diff --git a/library_generation/cli/generate_repo.py b/library_generation/cli/generate_repo.py new file mode 100644 index 0000000000..cebe773677 --- /dev/null +++ b/library_generation/cli/generate_repo.py @@ -0,0 +1,71 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import click + +from library_generation.generate_repo import generate_from_yaml + + +@click.group(invoke_without_command=False) +@click.pass_context +@click.version_option(message="%(version)s") +def main(ctx): + pass + + +@main.command() +@click.option( + "--generation-config-yaml", + required=True, + type=str, + help=""" + Path to generation_config.yaml that contains the metadata about + library generation + """, +) +@click.option( + "--target-library-names", + required=False, + default=None, + type=list[str], + help=""" + If specified, only the `library` whose library_name is in + target-library-names will be generated. + If not specified, all libraries in the configuration yaml will be generated. + """, +) +@click.option( + "--repository-path", + required=False, + default=".", + type=str, + help=""" + If specified, the generated files will be sent to this location. + If not specified, the repository will be generated to the current working + directory. + """, +) +def generate( + generation_config_yaml: str, + target_library_names: list[str], + repository_path: str, +): + generate_from_yaml( + generation_config_yaml=generation_config_yaml, + repository_path=repository_path, + target_library_names=target_library_names, + ) + + +if __name__ == "__main__": + main() diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index abe51c50f2..147539680a 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -13,87 +13,33 @@ # See the License for the specific language governing permissions and # limitations under the License. -import click import library_generation.utilities as util import os from library_generation.generate_composed_library import generate_composed_library +from library_generation.model.generation_config import GenerationConfig from library_generation.model.generation_config import from_yaml +from library_generation.model.library_config import LibraryConfig from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing -@click.group(invoke_without_command=False) -@click.pass_context -@click.version_option(message="%(version)s") -def main(ctx): - pass - - -@main.command() -@click.option( - "--generation-config-yaml", - required=True, - type=str, - help=""" - Path to generation_config.yaml that contains the metadata about - library generation - """, -) -@click.option( - "--target-library-api-shortname", - required=False, - type=str, - help=""" - If specified, only the `library` whose api_shortname equals to - target-library-api-shortname will be generated. - If not specified, all libraries in the configuration yaml will be generated. - """, -) -@click.option( - "--repository-path", - required=False, - default=".", - type=str, - help=""" - If specified, the generated files will be sent to this location. - If not specified, the repository will be generated to the current working - directory. - """, -) -def generate( - generation_config_yaml: str, - target_library_api_shortname: str, - repository_path: str, -): - generate_from_yaml( - generation_config_yaml=generation_config_yaml, - repository_path=repository_path, - target_library_api_shortname=target_library_api_shortname, - ) - - def generate_from_yaml( generation_config_yaml: str, repository_path: str, - target_library_api_shortname: str = None, + target_library_names: list[str] = None, ) -> None: """ Parses a config yaml and generates libraries via generate_composed_library.py """ - # convert paths to absolute paths so they can be correctly referenced in + # convert paths to absolute paths, so they can be correctly referenced in # downstream scripts generation_config_yaml = os.path.abspath(generation_config_yaml) repository_path = os.path.abspath(repository_path) config = from_yaml(generation_config_yaml) - target_libraries = config.libraries - if target_library_api_shortname is not None: - target_libraries = [ - library - for library in config.libraries - if library.api_shortname == target_library_api_shortname - ] - + target_libraries = __get_target_libraries( + config=config, target_library_names=target_library_names + ) repo_config = util.prepare_repo( gen_config=config, library_config=target_libraries, repo_path=repository_path ) @@ -118,5 +64,21 @@ def generate_from_yaml( ) -if __name__ == "__main__": - main() +def __get_target_libraries( + config: GenerationConfig, target_library_names: list[str] = None +) -> list[LibraryConfig]: + """ + Returns LibraryConfig objects whose library_name is in target_library_names. + + :param config: a GenerationConfig object. + :param target_library_names: library_name of target libraries. + :return: + """ + if target_library_names is None: + return config.libraries + target_libraries = set(target_library_names) + return [ + library + for library in config.libraries + if library.get_library_name() in target_libraries + ] From c0a0a19c6b75f3c01c8cd40e0c41ad1c194474cf Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Thu, 21 Mar 2024 19:40:49 +0000 Subject: [PATCH 2/9] add custom option --- library_generation/cli/generate_repo.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/library_generation/cli/generate_repo.py b/library_generation/cli/generate_repo.py index cebe773677..8fe35de48f 100644 --- a/library_generation/cli/generate_repo.py +++ b/library_generation/cli/generate_repo.py @@ -11,8 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import ast import click - from library_generation.generate_repo import generate_from_yaml @@ -23,6 +23,15 @@ def main(ctx): pass +# https://stackoverflow.com/questions/47631914/how-to-pass-several-list-of-arguments-to-click-option +class PythonLiteralOption(click.Option): + def type_cast_value(self, ctx, value): + try: + return ast.literal_eval(value) + except: + raise click.BadParameter(value) + + @main.command() @click.option( "--generation-config-yaml", @@ -35,6 +44,7 @@ def main(ctx): ) @click.option( "--target-library-names", + cls=PythonLiteralOption, required=False, default=None, type=list[str], From f32e32b15458e10310dcc6c8e3f522badde934c3 Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Thu, 21 Mar 2024 20:41:59 +0000 Subject: [PATCH 3/9] use get_library_name --- library_generation/utilities.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/library_generation/utilities.py b/library_generation/utilities.py index de3dd6ce44..fb7e8938d2 100755 --- a/library_generation/utilities.py +++ b/library_generation/utilities.py @@ -131,11 +131,7 @@ def prepare_repo( os.makedirs(output_folder, exist_ok=True) libraries = {} for library in library_config: - library_name = ( - f"{language}-{library.library_name}" - if library.library_name - else f"{language}-{library.api_shortname}" - ) + library_name = f"{language}-{library.get_library_name()}" library_path = ( f"{repo_path}/{library_name}" if gen_config.is_monorepo else f"{repo_path}" ) From fc128f97d799639bd605fb73ef2c0c2b0deab1d7 Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Thu, 21 Mar 2024 20:47:33 +0000 Subject: [PATCH 4/9] use string with comma --- library_generation/cli/generate_repo.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/library_generation/cli/generate_repo.py b/library_generation/cli/generate_repo.py index 8fe35de48f..00037b070a 100644 --- a/library_generation/cli/generate_repo.py +++ b/library_generation/cli/generate_repo.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import ast import click from library_generation.generate_repo import generate_from_yaml @@ -23,15 +22,6 @@ def main(ctx): pass -# https://stackoverflow.com/questions/47631914/how-to-pass-several-list-of-arguments-to-click-option -class PythonLiteralOption(click.Option): - def type_cast_value(self, ctx, value): - try: - return ast.literal_eval(value) - except: - raise click.BadParameter(value) - - @main.command() @click.option( "--generation-config-yaml", @@ -44,11 +34,16 @@ def type_cast_value(self, ctx, value): ) @click.option( "--target-library-names", - cls=PythonLiteralOption, required=False, default=None, - type=list[str], + type=str, help=""" + The input string will be parsed to a list of string with comma as the + separator. + + For example, apigeeconnect,alloydb-connectors will be parsed as a + list of two strings, apigeeconnect and alloydb-connectors. + If specified, only the `library` whose library_name is in target-library-names will be generated. If not specified, all libraries in the configuration yaml will be generated. @@ -67,13 +62,15 @@ def type_cast_value(self, ctx, value): ) def generate( generation_config_yaml: str, - target_library_names: list[str], + target_library_names: str, repository_path: str, ): generate_from_yaml( generation_config_yaml=generation_config_yaml, repository_path=repository_path, - target_library_names=target_library_names, + target_library_names=target_library_names.split(",") + if target_library_names is not None + else target_library_names, ) From 81a3edcb60ef904c06a2e80c9bcf6d026ae01d1c Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Thu, 21 Mar 2024 21:15:13 +0000 Subject: [PATCH 5/9] change comment --- library_generation/cli/generate_repo.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/library_generation/cli/generate_repo.py b/library_generation/cli/generate_repo.py index 00037b070a..5f366c9984 100644 --- a/library_generation/cli/generate_repo.py +++ b/library_generation/cli/generate_repo.py @@ -38,15 +38,17 @@ def main(ctx): default=None, type=str, help=""" + A list of libraries will be generated. + + If specified, only the `library` whose library_name is in + target-library-names will be generated. + If not specified, all libraries in the configuration yaml will be generated. + The input string will be parsed to a list of string with comma as the separator. For example, apigeeconnect,alloydb-connectors will be parsed as a list of two strings, apigeeconnect and alloydb-connectors. - - If specified, only the `library` whose library_name is in - target-library-names will be generated. - If not specified, all libraries in the configuration yaml will be generated. """, ) @click.option( From 4ceb816f00d5e7449ed1aabd372b51f95066b205 Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Sun, 24 Mar 2024 00:39:53 +0000 Subject: [PATCH 6/9] add unit tests --- library_generation/generate_repo.py | 4 +- .../test/generate_repo_unit_tests.py | 63 +++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 library_generation/test/generate_repo_unit_tests.py diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index 147539680a..fae1e27614 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -37,7 +37,7 @@ def generate_from_yaml( repository_path = os.path.abspath(repository_path) config = from_yaml(generation_config_yaml) - target_libraries = __get_target_libraries( + target_libraries = get_target_libraries( config=config, target_library_names=target_library_names ) repo_config = util.prepare_repo( @@ -64,7 +64,7 @@ def generate_from_yaml( ) -def __get_target_libraries( +def get_target_libraries( config: GenerationConfig, target_library_names: list[str] = None ) -> list[LibraryConfig]: """ diff --git a/library_generation/test/generate_repo_unit_tests.py b/library_generation/test/generate_repo_unit_tests.py new file mode 100644 index 0000000000..db8cb4eb63 --- /dev/null +++ b/library_generation/test/generate_repo_unit_tests.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from library_generation.generate_repo import get_target_libraries +from library_generation.model.generation_config import GenerationConfig +from library_generation.model.library_config import LibraryConfig + + +class GenerateRepoTest(unittest.TestCase): + def test_get_target_library_returns_selected_libraries(self): + one_library = GenerateRepoTest.__get_an_empty_library_config() + one_library.api_shortname = "one_library" + another_library = GenerateRepoTest.__get_an_empty_library_config() + another_library.api_shortname = "another_library" + config = GenerateRepoTest.__get_an_empty_generation_config() + config.libraries.extend([one_library, another_library]) + target_libraries = get_target_libraries(config, ["another_library"]) + self.assertEqual([another_library], target_libraries) + + def test_get_target_library_given_null_returns_all_libraries(self): + one_library = GenerateRepoTest.__get_an_empty_library_config() + one_library.api_shortname = "one_library" + another_library = GenerateRepoTest.__get_an_empty_library_config() + another_library.api_shortname = "another_library" + config = GenerateRepoTest.__get_an_empty_generation_config() + config.libraries.extend([one_library, another_library]) + target_libraries = get_target_libraries(config) + self.assertEqual([one_library, another_library], target_libraries) + + @staticmethod + def __get_an_empty_generation_config() -> GenerationConfig: + return GenerationConfig( + gapic_generator_version="", + googleapis_commitish="", + synthtool_commitish="", + owlbot_cli_image="", + template_excludes=[], + path_to_yaml="", + libraries=[], + ) + + @staticmethod + def __get_an_empty_library_config() -> LibraryConfig: + return LibraryConfig( + api_shortname="", + name_pretty="", + api_description="", + product_documentation="", + gapic_configs=[], + ) From 40dd5bf62fc486158b45a1d2e37c018dc1e40b28 Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Mon, 25 Mar 2024 22:57:47 +0000 Subject: [PATCH 7/9] add comment --- library_generation/generate_repo.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index fae1e27614..235ffc45d1 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -30,6 +30,16 @@ def generate_from_yaml( """ Parses a config yaml and generates libraries via generate_composed_library.py + :param generation_config_yaml: Path to generation_config.yaml that contains + the metadata about library generation + :param repository_path: If specified, the generated files will be sent to + this location. If not specified, the repository will be generated to the + current working directory. + :param target_library_names: a list of libraries to be generated. + If specified, only the library whose library_name is in + target-library-names will be generated. + If specified with an empty list, then no library will be generated. + If not specified, all libraries in the configuration yaml will be generated. """ # convert paths to absolute paths, so they can be correctly referenced in # downstream scripts From 2327a619018500e0e30f3209b12a05b9e66980de Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Mon, 25 Mar 2024 23:01:49 +0000 Subject: [PATCH 8/9] remove cli --- library_generation/cli/generate_repo.py | 80 ------------------------- library_generation/generate_repo.py | 62 +++++++++++++++++++ 2 files changed, 62 insertions(+), 80 deletions(-) delete mode 100644 library_generation/cli/generate_repo.py diff --git a/library_generation/cli/generate_repo.py b/library_generation/cli/generate_repo.py deleted file mode 100644 index 5f366c9984..0000000000 --- a/library_generation/cli/generate_repo.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import click -from library_generation.generate_repo import generate_from_yaml - - -@click.group(invoke_without_command=False) -@click.pass_context -@click.version_option(message="%(version)s") -def main(ctx): - pass - - -@main.command() -@click.option( - "--generation-config-yaml", - required=True, - type=str, - help=""" - Path to generation_config.yaml that contains the metadata about - library generation - """, -) -@click.option( - "--target-library-names", - required=False, - default=None, - type=str, - help=""" - A list of libraries will be generated. - - If specified, only the `library` whose library_name is in - target-library-names will be generated. - If not specified, all libraries in the configuration yaml will be generated. - - The input string will be parsed to a list of string with comma as the - separator. - - For example, apigeeconnect,alloydb-connectors will be parsed as a - list of two strings, apigeeconnect and alloydb-connectors. - """, -) -@click.option( - "--repository-path", - required=False, - default=".", - type=str, - help=""" - If specified, the generated files will be sent to this location. - If not specified, the repository will be generated to the current working - directory. - """, -) -def generate( - generation_config_yaml: str, - target_library_names: str, - repository_path: str, -): - generate_from_yaml( - generation_config_yaml=generation_config_yaml, - repository_path=repository_path, - target_library_names=target_library_names.split(",") - if target_library_names is not None - else target_library_names, - ) - - -if __name__ == "__main__": - main() diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index 235ffc45d1..878012741a 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -14,6 +14,7 @@ # limitations under the License. import library_generation.utilities as util +import click import os from library_generation.generate_composed_library import generate_composed_library from library_generation.model.generation_config import GenerationConfig @@ -22,6 +23,67 @@ from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing +@click.group(invoke_without_command=False) +@click.pass_context +@click.version_option(message="%(version)s") +def main(ctx): + pass + + +@main.command() +@click.option( + "--generation-config-yaml", + required=True, + type=str, + help=""" + Path to generation_config.yaml that contains the metadata about + library generation + """, +) +@click.option( + "--target-library-names", + required=False, + default=None, + type=str, + help=""" + A list of libraries will be generated. + + If specified, only the `library` whose library_name is in + target-library-names will be generated. + If not specified, all libraries in the configuration yaml will be generated. + + The input string will be parsed to a list of string with comma as the + separator. + + For example, apigeeconnect,alloydb-connectors will be parsed as a + list of two strings, apigeeconnect and alloydb-connectors. + """, +) +@click.option( + "--repository-path", + required=False, + default=".", + type=str, + help=""" + If specified, the generated files will be sent to this location. + If not specified, the repository will be generated to the current working + directory. + """, +) +def generate( + generation_config_yaml: str, + target_library_names: str, + repository_path: str, +): + generate_from_yaml( + generation_config_yaml=generation_config_yaml, + repository_path=repository_path, + target_library_names=target_library_names.split(",") + if target_library_names is not None + else target_library_names, + ) + + def generate_from_yaml( generation_config_yaml: str, repository_path: str, From d2f02c6c6fcdb88343e97fb977b932798f6dfaf0 Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Mon, 25 Mar 2024 23:22:17 +0000 Subject: [PATCH 9/9] add comment --- library_generation/generate_repo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index 878012741a..b33e8ba10a 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -144,7 +144,8 @@ def get_target_libraries( :param config: a GenerationConfig object. :param target_library_names: library_name of target libraries. - :return: + If not specified, all libraries in the given config will be returned. + :return: LibraryConfig objects. """ if target_library_names is None: return config.libraries