From ec03dd05b12c5fbce9d6bbc8289165cf31825359 Mon Sep 17 00:00:00 2001 From: Blake Li Date: Wed, 3 Apr 2024 16:38:38 -0400 Subject: [PATCH 1/4] refactor: Rename requirements.in to requirements.txt. Add environment setup before running hermetic build scripts. --- .../library_generation.Dockerfile | 2 +- .../workflows/verify_library_generation.yaml | 8 +++--- library_generation/README.md | 25 +++++++++++++------ library_generation/postprocess_library.sh | 2 +- .../{requirements.in => requirements.txt} | 0 5 files changed, 23 insertions(+), 14 deletions(-) rename library_generation/{requirements.in => requirements.txt} (100%) diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 4b3b1f4b1c..09d23a106b 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -26,7 +26,7 @@ RUN rm $(which python3) RUN ln -s $(which python3.11) /usr/local/bin/python RUN ln -s $(which python3.11) /usr/local/bin/python3 RUN python -m pip install --upgrade pip -RUN cd /src && python -m pip install -r requirements.in +RUN cd /src && python -m pip install -r requirements.txt RUN cd /src && python -m pip install . # set dummy git credentials for empty commit used in postprocessing diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index b5707b443d..e1cb855cb1 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -41,7 +41,7 @@ jobs: run: | set -ex pushd library_generation - pip install -r requirements.in + pip install -r requirements.txt pip install . popd - name: Run integration tests @@ -73,7 +73,7 @@ jobs: run: | set -ex pushd library_generation - pip install -r requirements.in + pip install -r requirements.txt popd - name: install synthtool shell: bash @@ -89,7 +89,7 @@ jobs: git reset --hard origin/no-java-templates python3 -m pip install -e . - python3 -m pip install -r requirements.in + python3 -m pip install -r requirements.txt - name: Run shell unit tests run: | set -x @@ -117,7 +117,7 @@ jobs: run: | set -ex pushd library_generation - pip install -r requirements.in + pip install -r requirements.txt popd - name: Lint shell: bash diff --git a/library_generation/README.md b/library_generation/README.md index 75fc32695d..62cd5bf44a 100644 --- a/library_generation/README.md +++ b/library_generation/README.md @@ -10,6 +10,8 @@ configuration file. - Java runtime environment (8 or above) - Apache Maven (used in formatting source code) - Python (3.11.6 or above) +- Docker +- Git ## Prerequisite @@ -184,19 +186,26 @@ libraries: - proto_path: google/cloud/asset/v1p7beta1 ``` -## An example to generate a repository using `generate_repo.py` +# Local Environment Setup before running `generate_repo.py` +1. Assuming Python 3 is installed, follow official guide from [Python.org](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments) to create a virtual environment. The virtual environment can be installed to any folder, usually it is recommended to be installed under the root folder of the project(`sdk-platform-java` in this case). +2. Assuming the virtual environment is installed under `sdk-platform-java`. Run the following command under the root folder of `sdk-platform-java` to install the dependencies of `library_generation` ```bash -# install python module (allows the `library_generation` module to be imported from anywhere) -python -m pip install -r library_generation/requirements.in -# generate the repository -python -m library_generation/generate_repo.py generate \ ---generation-config-yaml=/path/to/config-file \ ---repository-path=/path/to/repository +python -m pip install -r library_generation/requirements.txt +``` +3. Run the following command to install `library_generation` as a module, which allows the `library_generation` module to be imported from anywhere +```bash +python -m pip install library_generation/ ``` -## An example of generated repository using `generate_repo.py` +## An example to generate a repository using `generate_repo.py` +```bash +# generate the repository +python library_generation/generate_repo.py generate \ +--generation-config-yaml=absolute/path/to/config-file \ +--repository-path=absolute/path/to/repository +``` If you run `generate_repo.py` with the example [configuration](#an-example-of-generation-configuration) shown above, the repository structure is: ``` diff --git a/library_generation/postprocess_library.sh b/library_generation/postprocess_library.sh index c6a5b4020d..5025a70947 100755 --- a/library_generation/postprocess_library.sh +++ b/library_generation/postprocess_library.sh @@ -143,7 +143,7 @@ git fetch --all git reset --hard "${synthtool_commitish}" python3 -m pip install -e . -python3 -m pip install -r requirements.in +python3 -m pip install -r requirements.txt popd # synthtool popd # temp dir diff --git a/library_generation/requirements.in b/library_generation/requirements.txt similarity index 100% rename from library_generation/requirements.in rename to library_generation/requirements.txt From b3969faa10bd66f97f67bc4ddea041701332f667 Mon Sep 17 00:00:00 2001 From: Blake Li Date: Wed, 3 Apr 2024 18:38:23 -0400 Subject: [PATCH 2/4] Update verify_library_generation.yaml --- .github/workflows/verify_library_generation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index 8977895400..12b171a8f2 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -87,7 +87,7 @@ jobs: git reset --hard origin/no-java-templates python3 -m pip install -e . - python3 -m pip install -r requirements.txt + python3 -m pip install -r requirements.in - name: Run shell unit tests run: | set -x From 138b0386e61474c7a027bfa516f3cc2d50b8e34b Mon Sep 17 00:00:00 2001 From: Blake Li Date: Fri, 5 Apr 2024 16:32:27 -0400 Subject: [PATCH 3/4] refactor: Remove Java setup in library generation ci. --- .github/workflows/verify_library_generation.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index 12b171a8f2..c51f8ed7b0 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -10,17 +10,9 @@ on: name: verify_library_generation jobs: integration_tests: - strategy: - matrix: - java: [ 11 ] runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java }} - distribution: temurin - cache: maven - uses: actions/setup-python@v5 with: python-version: 3.11 From 3a557193d9234f90fcdc15c9f385b275b7605940 Mon Sep 17 00:00:00 2001 From: Blake Li Date: Fri, 5 Apr 2024 18:28:57 -0400 Subject: [PATCH 4/4] refactor: Revert the change in postprocessing. --- library_generation/postprocess_library.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library_generation/postprocess_library.sh b/library_generation/postprocess_library.sh index 8e4aa1691b..480b7e5170 100755 --- a/library_generation/postprocess_library.sh +++ b/library_generation/postprocess_library.sh @@ -143,7 +143,7 @@ git fetch --all git reset --hard "${synthtool_commitish}" python3 -m pip install -e . -python3 -m pip install -r requirements.txt +python3 -m pip install -r requirements.in popd # synthtool popd # temp dir