diff --git a/.github/workflows/cypher-reference-documentation.yml b/.github/workflows/cypher-reference-documentation.yml index c14594552..be4fd84ec 100644 --- a/.github/workflows/cypher-reference-documentation.yml +++ b/.github/workflows/cypher-reference-documentation.yml @@ -32,7 +32,14 @@ jobs: run: | ./../scripts/documentation/generateCypherReference.sh + - name: Use git to detect changes in the regenerated document and set generated_document_changed + run: git diff --quiet || echo "generated_document_changed=true" >> $GITHUB_ENV + + - name: Display generated_document_changed + run: echo "generated_document_changed=${{ env.generated_document_changed}}" + - name: Archive generated cypher reference document + if: env.generated_document_changed uses: actions/upload-artifact@v4 with: name: cypher-reference-document @@ -40,12 +47,6 @@ jobs: if-no-files-found: error retention-days: 5 - - name: Use git to detect changes in the regenerated document and set generated_document_changed - run: git diff --quiet || echo "generated_document_changed=true" >> $GITHUB_ENV - - - name: Display generated_document_changed - run: echo "generated_document_changed=${{ env.generated_document_changed}}" - - name: Commit generated cypher reference document if there were changes # Only run when a pull request gets merged or a commit is pushed to the main branch. # And only run when the generated document changed to avoid an empty commit or an error while committing. @@ -53,7 +54,9 @@ jobs: run: | git config --global user.name '${{ env.CI_COMMIT_AUTHOR }}' git config --global user.email 'joht@users.noreply.github.com' - git pull + set -o errexit -o pipefail -o xtrace git add ./cypher/CYPHER.md + git status git commit -m "${{ env.CI_COMMIT_MESSAGE }}" - git push + git status + git push --verbose diff --git a/.github/workflows/environment-variables-reference-documentation.yml b/.github/workflows/environment-variables-reference-documentation.yml index 6200c8fc0..c3e4bee84 100644 --- a/.github/workflows/environment-variables-reference-documentation.yml +++ b/.github/workflows/environment-variables-reference-documentation.yml @@ -32,7 +32,14 @@ jobs: run: | ./documentation/generateEnvironmentVariableReference.sh + - name: Use git to detect changes in the regenerated document and set generated_document_changed + run: git diff --quiet || echo "generated_document_changed=true" >> $GITHUB_ENV + + - name: Display generated_document_changed + run: echo "generated_document_changed=${{ env.generated_document_changed}}" + - name: Archive generated environment variables reference document + if: env.generated_document_changed uses: actions/upload-artifact@v4 with: name: environment-variables-reference-document @@ -40,12 +47,6 @@ jobs: if-no-files-found: error retention-days: 5 - - name: Use git to detect changes in the regenerated document and set generated_document_changed - run: git diff --quiet || echo "generated_document_changed=true" >> $GITHUB_ENV - - - name: Display generated_document_changed - run: echo "generated_document_changed=${{ env.generated_document_changed}}" - - name: Commit generated environment variables reference document if there were changes # Only run when a pull request gets merged or a commit is pushed to the main branch. # And only run when the generated document changed to avoid an empty commit or an error while committing. @@ -53,7 +54,9 @@ jobs: run: | git config --global user.name '${{ env.CI_COMMIT_AUTHOR }}' git config --global user.email 'joht@users.noreply.github.com' - git pull + set -o errexit -o pipefail -o xtrace git add ./scripts/ENVIRONMENT_VARIABLES.md + git status git commit -m "${{ env.CI_COMMIT_MESSAGE }}" - git push \ No newline at end of file + git status + git push --verbose \ No newline at end of file diff --git a/.github/workflows/java-code-analysis.yml b/.github/workflows/java-code-analysis.yml index fc711c005..e20753cbd 100644 --- a/.github/workflows/java-code-analysis.yml +++ b/.github/workflows/java-code-analysis.yml @@ -179,8 +179,9 @@ jobs: git config --global user.name '${{ env.CI_COMMIT_AUTHOR }}' git config --global user.email 'joht@users.noreply.github.com' git config --local http.postBuffer 524288000 - git pull + set -o errexit -o pipefail -o xtrace git add results git status git commit -m "${{ env.CI_COMMIT_MESSAGE }}" - git push + git status + git push --verbose diff --git a/.github/workflows/report-reference-documentation.yml b/.github/workflows/report-reference-documentation.yml new file mode 100644 index 000000000..08bee7fe4 --- /dev/null +++ b/.github/workflows/report-reference-documentation.yml @@ -0,0 +1,64 @@ +name: Generate report reference documentation + +on: + push: + branches: + - main + paths: + - 'results/**' # Only run results have been updates + - '!results/*.md' # Ignore when report reference documentation itself changes + - '.github/workflows/report-reference-documentation.yml' # or when this file was changed + pull_request: + branches: + - main + paths: + - 'results/**' # Only run results have been updates + - '!results/*.md' # Ignore when report reference documentation itself changes + - '.github/workflows/report-reference-documentation.yml' # or when this file was changed + +jobs: + reports: + runs-on: ubuntu-latest + env: + CI_COMMIT_MESSAGE: Automated report reference document generation (CI) + CI_COMMIT_AUTHOR: ${{ github.event.repository.name }} Continuous Integration + + steps: + - name: Checkout git repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.WORKFLOW_GIT_ACCESS_TOKEN }} + + - name: Generate report reference document + working-directory: results + run: | + ./../scripts/documentation/generateReportReferences.sh + + - name: Use git to detect changes in the regenerated documenta and set generated_document_changed + run: git diff --quiet || echo "generated_document_changed=true" >> $GITHUB_ENV + + - name: Display generated_document_changed + run: echo "generated_document_changed=${{ env.generated_document_changed}}" + + - name: Archive generated report reference document + if: env.generated_document_changed + uses: actions/upload-artifact@v4 + with: + name: report-reference-document + path: ./results/*.md + if-no-files-found: error + retention-days: 5 + + - name: Commit generated report reference document if there were changes + # Only run when a pull request gets merged or a commit is pushed to the main branch. + # And only run when the generated document changed to avoid an empty commit or an error while committing. + if: github.event_name == 'push' && env.generated_document_changed + run: | + git config --global user.name '${{ env.CI_COMMIT_AUTHOR }}' + git config --global user.email 'joht@users.noreply.github.com' + set -o errexit -o pipefail -o xtrace + git add ./results/*.md + git status + git commit -m "${{ env.CI_COMMIT_MESSAGE }}" + git status + git push --verbose \ No newline at end of file diff --git a/.github/workflows/scripts-reference-documentation.yml b/.github/workflows/scripts-reference-documentation.yml index 137850db5..a1b26ed0f 100644 --- a/.github/workflows/scripts-reference-documentation.yml +++ b/.github/workflows/scripts-reference-documentation.yml @@ -32,7 +32,14 @@ jobs: run: | ./documentation/generateScriptReference.sh + - name: Use git to detect changes in the regenerated document and set generated_document_changed + run: git diff --quiet || echo "generated_document_changed=true" >> $GITHUB_ENV + + - name: Display generated_document_changed + run: echo "generated_document_changed=${{ env.generated_document_changed}}" + - name: Archive generated scripts reference document + if: env.generated_document_changed uses: actions/upload-artifact@v4 with: name: scripts-reference-document @@ -40,12 +47,6 @@ jobs: if-no-files-found: error retention-days: 5 - - name: Use git to detect changes in the regenerated document and set generated_document_changed - run: git diff --quiet || echo "generated_document_changed=true" >> $GITHUB_ENV - - - name: Display generated_document_changed - run: echo "generated_document_changed=${{ env.generated_document_changed}}" - - name: Commit generated scripts reference document if there were changes # Only run when a pull request gets merged or a commit is pushed to the main branch. # And only run when the generated document changed to avoid an empty commit or an error while committing. @@ -53,7 +54,9 @@ jobs: run: | git config --global user.name '${{ env.CI_COMMIT_AUTHOR }}' git config --global user.email 'joht@users.noreply.github.com' - git pull + set -o errexit -o pipefail -o xtrace git add ./scripts/SCRIPTS.md + git status git commit -m "${{ env.CI_COMMIT_MESSAGE }}" - git push + git status + git push --verbose \ No newline at end of file diff --git a/.github/workflows/typescript-code-analysis.yml b/.github/workflows/typescript-code-analysis.yml index 8dab5711e..a2f3aa7b1 100644 --- a/.github/workflows/typescript-code-analysis.yml +++ b/.github/workflows/typescript-code-analysis.yml @@ -179,8 +179,9 @@ jobs: git config --global user.name '${{ env.CI_COMMIT_AUTHOR }}' git config --global user.email 'joht@users.noreply.github.com' git config --local http.postBuffer 524288000 - git pull + set -o errexit -o pipefail -o xtrace git add results git status git commit -m "${{ env.CI_COMMIT_MESSAGE }}" - git push + git status + git push --verbose diff --git a/scripts/copyReportsIntoResults.sh b/scripts/copyReportsIntoResults.sh index b643582dc..d356cbc8f 100755 --- a/scripts/copyReportsIntoResults.sh +++ b/scripts/copyReportsIntoResults.sh @@ -41,13 +41,4 @@ for report_source_folder in **/"${REPORTS_DIRECTORY}"; do rm -rf "${reportTargetDirectory}" cp -Rp "${report_source_folder}" "${reportTargetDirectory}" -done - -# Generate JUPYTER_REPORTS.md containing a reference to all Jupyter Notebook Markdown reports in the "results" directory and its subdirectories. -(cd "./../${RESULTS_DIRECTORY}" && exec "${SCRIPTS_DIR}/documentation/generateJupyterReportReference.sh") - -# Generate CSV_REPORTS.md containing a reference to all CSV cypher query reports in the "results" directory and its subdirectories. -(cd "./../${RESULTS_DIRECTORY}" && exec "${SCRIPTS_DIR}/documentation/generateCsvReportReference.sh") - -# Generate IMAGES.md containing a reference to all PNG images in the "results" directory and its subdirectories. -(cd "./../${RESULTS_DIRECTORY}" && exec "${SCRIPTS_DIR}/documentation/generateImageReference.sh") \ No newline at end of file +done \ No newline at end of file diff --git a/scripts/documentation/generateCsvReportReference.sh b/scripts/documentation/generateCsvReportReference.sh index 4591bd0b9..d6e213f8d 100755 --- a/scripts/documentation/generateCsvReportReference.sh +++ b/scripts/documentation/generateCsvReportReference.sh @@ -9,7 +9,7 @@ set -o errexit -o pipefail # Output Markdown file name -output_file="CSV_REPORTS.md" +markdown_file="CSV_REPORTS.md" # Function to count rows in a CSV file count_rows() { @@ -23,6 +23,8 @@ get_source_query() { echo "$source_query" } +echo "generateCsvReportReference: Generating ${markdown_file}..." + # Create the Markdown header { echo "# CSV Cypher Query Reports Reference" @@ -33,10 +35,10 @@ get_source_query() { # Create the Markdown table header echo "| CSV File | Analysis | Number of Rows | Source Query |" echo "| -------- | -------- | -------------- | ------------ |" -} > "$output_file" +} > "$markdown_file" # Find and process CSV files -find . -type f -name "*.csv" | while IFS= read -r csv_file; do +find . -type f -name "*.csv" | sort | while IFS= read -r csv_file; do num_rows=$(count_rows "$csv_file") source_query=$(get_source_query "$csv_file") @@ -53,7 +55,7 @@ find . -type f -name "*.csv" | while IFS= read -r csv_file; do source_query_link="./../cypher/$source_query" # Append a new row to the Markdown table - echo "| [$base_name]($csv_link) | $main_dir | $num_rows | [$source_query]($source_query_link) |" >> "$output_file" + echo "| [$base_name]($csv_link) | $main_dir | $num_rows | [$source_query]($source_query_link) |" >> "$markdown_file" done -echo "CSV table and header generated in $output_file" +echo "generateCsvReportReference: Successfully generated ${markdown_file}." diff --git a/scripts/documentation/generateCypherReference.sh b/scripts/documentation/generateCypherReference.sh index 85082b387..b8a86f02d 100755 --- a/scripts/documentation/generateCypherReference.sh +++ b/scripts/documentation/generateCypherReference.sh @@ -9,6 +9,8 @@ set -o errexit -o pipefail # Markdown file name markdown_file="CYPHER.md" +echo "generateCypherReference: Generating ${markdown_file}..." + { echo "# Cypher Reference" echo "" @@ -43,3 +45,5 @@ find . -type f -name "*.cypher" | sort | while read -r cypher_file; do # Add the script file and its description to the Markdown table echo "| ${link} | ${last_path_segment%%.} | ${description//|/\\|} |" >> ${markdown_file} done + +echo "generateCypherReference: Successfully generated ${markdown_file}." \ No newline at end of file diff --git a/scripts/documentation/generateEnvironmentVariableReference.sh b/scripts/documentation/generateEnvironmentVariableReference.sh index b9d6281a2..32deb7663 100755 --- a/scripts/documentation/generateEnvironmentVariableReference.sh +++ b/scripts/documentation/generateEnvironmentVariableReference.sh @@ -14,6 +14,8 @@ set -o errexit -o pipefail DOCUMENTATION_SCRIPTS_DIR=${DOCUMENTATION_SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the documentation generation scripts echo "generateEnvironmentVariablesReference: DOCUMENTATION_SCRIPTS_DIR=${DOCUMENTATION_SCRIPTS_DIR}" +echo "generateEnvironmentVariableReference: Generating ENVIRONMENT_VARIABLES.md..." + # Clear existing markdown document source "${DOCUMENTATION_SCRIPTS_DIR}/appendEnvironmentVariables.sh" "clear" || exit 1 @@ -21,4 +23,6 @@ source "${DOCUMENTATION_SCRIPTS_DIR}/appendEnvironmentVariables.sh" "clear" || e find . -type f -name "*.sh" | sort | while read -r scriptFile; do echo "generateEnvironmentVariablesReference: Searching for environment variables in ${scriptFile}" source "${DOCUMENTATION_SCRIPTS_DIR}/appendEnvironmentVariables.sh" "${scriptFile}" || exit 1 -done \ No newline at end of file +done + +echo "generateEnvironmentVariableReference: Successfully generated ENVIRONMENT_VARIABLES.md." \ No newline at end of file diff --git a/scripts/documentation/generateImageReference.sh b/scripts/documentation/generateImageReference.sh index 8c057cef8..bf33f2f19 100755 --- a/scripts/documentation/generateImageReference.sh +++ b/scripts/documentation/generateImageReference.sh @@ -8,6 +8,8 @@ set -o errexit -o pipefail # Markdown file name markdown_file="IMAGES.md" +echo "generateImageReference: Generating ${markdown_file}..." + { echo "# Image Reference" echo "" @@ -43,3 +45,5 @@ find . -type f -name "*.png" | sort | while read -r image_file; do # Add the script file and its description to the Markdown table echo "| ${link} | ${analysisname%%.} |" >> ${markdown_file} done + +echo "generateImageReference: Successfully generated ${markdown_file}." \ No newline at end of file diff --git a/scripts/documentation/generateJupyterReportReference.sh b/scripts/documentation/generateJupyterReportReference.sh index 64347e9d2..337b65001 100755 --- a/scripts/documentation/generateJupyterReportReference.sh +++ b/scripts/documentation/generateJupyterReportReference.sh @@ -9,6 +9,8 @@ set -o errexit -o pipefail # Markdown file name markdown_file="JUPYTER_REPORTS.md" +echo "generateJupyterReportReference: Generating ${markdown_file}..." + { echo "# Jupyter Notebook Reports Reference" echo "" @@ -50,3 +52,5 @@ find . -type f -name "*.md" | sort | while read -r report_file; do # Add the script file and its description to the Markdown table echo "| ${link} | ${analysisname%%.} | ${description} |" >> ${markdown_file} done + +echo "generateJupyterReportReference: Successfully generated ${markdown_file}." \ No newline at end of file diff --git a/scripts/documentation/generateReportReferences.sh b/scripts/documentation/generateReportReferences.sh new file mode 100755 index 000000000..d5386f5cc --- /dev/null +++ b/scripts/documentation/generateReportReferences.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Triggers the regeneration of all reference documentations for the reports inside the "results" directory. +# The generation to the reference documentations is delegated to the dedicated scripts. + +# Notice that this scripts needs to be executed within the "results" directory. + +# Requires generateJupyterReportReference.sh, generateCsvReportReference.sh, generateImageReference.sh + +# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands) +set -o errexit -o pipefail + +## Get this "scripts" directory if not already set +# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. +# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. +# This way non-standard tools like readlink aren't needed. +DOCUMENTATION_SCRIPT_DIR=${DOCUMENTATION_SCRIPT_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts +echo "generateReportReferences: SCRIPTS_DIR=${DOCUMENTATION_SCRIPT_DIR}" + +# Generate JUPYTER_REPORTS.md containing a reference to all Jupyter Notebook Markdown reports in the "results" directory and its subdirectories. +source "${DOCUMENTATION_SCRIPT_DIR}/generateJupyterReportReference.sh" + +# Generate CSV_REPORTS.md containing a reference to all CSV cypher query reports in the "results" directory and its subdirectories. +source "${DOCUMENTATION_SCRIPT_DIR}/generateCsvReportReference.sh" + +# Generate IMAGES.md containing a reference to all PNG images in the "results" directory and its subdirectories. +source "${DOCUMENTATION_SCRIPT_DIR}/generateImageReference.sh" \ No newline at end of file diff --git a/scripts/documentation/generateScriptReference.sh b/scripts/documentation/generateScriptReference.sh index 13d3b6150..3a04a5ac8 100755 --- a/scripts/documentation/generateScriptReference.sh +++ b/scripts/documentation/generateScriptReference.sh @@ -9,6 +9,8 @@ set -o errexit -o pipefail # Markdown file name markdown_file="SCRIPTS.md" +echo "generateScriptReference: Generating ${markdown_file}..." + { echo "# Scripts Reference" echo "" @@ -38,3 +40,5 @@ find . -type f -name "*.sh" | sort | while read -r script_file; do # Add the script file and its description to the Markdown table echo "| ${link} | ${last_path_segment%%.} | ${description//|/\\|} |" >> ${markdown_file} done + +echo "generateScriptReference: Successfully generated ${markdown_file}." \ No newline at end of file