Separate report reference documentation generation

.github/workflows/cypher-reference-documentation.yml

@@ -32,28 +32,31 @@ jobs:
       run: |
         ./../scripts/documentation/generateCypherReference.sh
 
+    - name: Use git to detect changes in the regenerated document and set generated_document_changed
+      run: git diff --quiet || echo "generated_document_changed=true" >> $GITHUB_ENV
+
+    - name: Display generated_document_changed
+      run: echo "generated_document_changed=${{ env.generated_document_changed}}"
+
     - name: Archive generated cypher reference document
+      if: env.generated_document_changed
       uses: actions/upload-artifact@v4
       with:
         name: cypher-reference-document
         path: ./cypher/CYPHER.md
         if-no-files-found: error
         retention-days: 5
 
-    - name: Use git to detect changes in the regenerated document and set generated_document_changed
-      run: git diff --quiet || echo "generated_document_changed=true" >> $GITHUB_ENV
-
-    - name: Display generated_document_changed
-      run: echo "generated_document_changed=${{ env.generated_document_changed}}"
-
     - name: Commit generated cypher reference document if there were changes
       # Only run when a pull request gets merged or a commit is pushed to the main branch.
       # And only run when the generated document changed to avoid an empty commit or an error while committing.
       if: github.event_name == 'push' && env.generated_document_changed
       run: |
         git config --global user.name '${{ env.CI_COMMIT_AUTHOR }}'
         git config --global user.email '[email protected]'
-        git pull
+        set -o errexit -o pipefail -o xtrace
         git add ./cypher/CYPHER.md
+        git status
         git commit -m "${{ env.CI_COMMIT_MESSAGE }}"
-        git push
+        git status
+        git push --verbose

.github/workflows/environment-variables-reference-documentation.yml

@@ -32,28 +32,31 @@ jobs:
       run: |
         ./documentation/generateEnvironmentVariableReference.sh
 
+    - name: Use git to detect changes in the regenerated document and set generated_document_changed
+      run: git diff --quiet || echo "generated_document_changed=true" >> $GITHUB_ENV
+
+    - name: Display generated_document_changed
+      run: echo "generated_document_changed=${{ env.generated_document_changed}}"
+
     - name: Archive generated environment variables reference document
+      if: env.generated_document_changed
       uses: actions/upload-artifact@v4
       with:
         name: environment-variables-reference-document
         path: ./scripts/ENVIRONMENT_VARIABLES.md
         if-no-files-found: error
         retention-days: 5
 
-    - name: Use git to detect changes in the regenerated document and set generated_document_changed
-      run: git diff --quiet || echo "generated_document_changed=true" >> $GITHUB_ENV
-
-    - name: Display generated_document_changed
-      run: echo "generated_document_changed=${{ env.generated_document_changed}}"
-
     - name: Commit generated environment variables reference document if there were changes
       # Only run when a pull request gets merged or a commit is pushed to the main branch.
       # And only run when the generated document changed to avoid an empty commit or an error while committing.
       if: github.event_name == 'push' && env.generated_document_changed
       run: |
         git config --global user.name '${{ env.CI_COMMIT_AUTHOR }}'
         git config --global user.email '[email protected]'
-        git pull
+        set -o errexit -o pipefail -o xtrace
         git add ./scripts/ENVIRONMENT_VARIABLES.md
+        git status
         git commit -m "${{ env.CI_COMMIT_MESSAGE }}"
-        git push
+        git status
+        git push --verbose

.github/workflows/java-code-analysis.yml

@@ -179,8 +179,9 @@ jobs:
         git config --global user.name '${{ env.CI_COMMIT_AUTHOR }}'
         git config --global user.email '[email protected]'
         git config --local http.postBuffer 524288000
-        git pull
+        set -o errexit -o pipefail -o xtrace
         git add results
         git status
         git commit -m "${{ env.CI_COMMIT_MESSAGE }}"
-        git push
+        git status
+        git push --verbose

.github/workflows/report-reference-documentation.yml

@@ -0,0 +1,64 @@
+name: Generate report reference documentation
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'results/**' # Only run results have been updates
+      - '!results/*.md'  # Ignore when report reference documentation itself changes
+      - '.github/workflows/report-reference-documentation.yml' # or when this file was changed
+  pull_request:
+    branches:
+      - main
+    paths: 
+      - 'results/**' # Only run results have been updates
+      - '!results/*.md'  # Ignore when report reference documentation itself changes
+      - '.github/workflows/report-reference-documentation.yml' # or when this file was changed
+
+jobs:
+  reports:
+    runs-on: ubuntu-latest
+    env: 
+      CI_COMMIT_MESSAGE: Automated report reference document generation (CI)
+      CI_COMMIT_AUTHOR: ${{ github.event.repository.name }} Continuous Integration
+
+    steps:
+    - name: Checkout git repository
+      uses: actions/checkout@v4
+      with:
+        token: ${{ secrets.WORKFLOW_GIT_ACCESS_TOKEN }}
+
+    - name: Generate report reference document
+      working-directory: results
+      run: |
+        ./../scripts/documentation/generateReportReferences.sh
+
+    - name: Use git to detect changes in the regenerated documenta and set generated_document_changed
+      run: git diff --quiet || echo "generated_document_changed=true" >> $GITHUB_ENV
+
+    - name: Display generated_document_changed
+      run: echo "generated_document_changed=${{ env.generated_document_changed}}"
+
+    - name: Archive generated report reference document
+      if: env.generated_document_changed
+      uses: actions/upload-artifact@v4
+      with:
+        name: report-reference-document
+        path: ./results/*.md
+        if-no-files-found: error
+        retention-days: 5
+
+    - name: Commit generated report reference document if there were changes
+      # Only run when a pull request gets merged or a commit is pushed to the main branch.
+      # And only run when the generated document changed to avoid an empty commit or an error while committing.
+      if: github.event_name == 'push' && env.generated_document_changed
+      run: |
+        git config --global user.name '${{ env.CI_COMMIT_AUTHOR }}'
+        git config --global user.email '[email protected]'
+        set -o errexit -o pipefail -o xtrace
+        git add ./results/*.md
+        git status
+        git commit -m "${{ env.CI_COMMIT_MESSAGE }}"
+        git status
+        git push --verbose

.github/workflows/scripts-reference-documentation.yml

@@ -32,28 +32,31 @@ jobs:
       run: |
         ./documentation/generateScriptReference.sh
 
+    - name: Use git to detect changes in the regenerated document and set generated_document_changed
+      run: git diff --quiet || echo "generated_document_changed=true" >> $GITHUB_ENV
+
+    - name: Display generated_document_changed
+      run: echo "generated_document_changed=${{ env.generated_document_changed}}"
+
     - name: Archive generated scripts reference document
+      if: env.generated_document_changed
       uses: actions/upload-artifact@v4
       with:
         name: scripts-reference-document
         path: ./scripts/SCRIPTS.md
         if-no-files-found: error
         retention-days: 5
 
-    - name: Use git to detect changes in the regenerated document and set generated_document_changed
-      run: git diff --quiet || echo "generated_document_changed=true" >> $GITHUB_ENV
-
-    - name: Display generated_document_changed
-      run: echo "generated_document_changed=${{ env.generated_document_changed}}"
-
     - name: Commit generated scripts reference document if there were changes
       # Only run when a pull request gets merged or a commit is pushed to the main branch.
       # And only run when the generated document changed to avoid an empty commit or an error while committing.
       if: github.event_name == 'push' && env.generated_document_changed
       run: |
         git config --global user.name '${{ env.CI_COMMIT_AUTHOR }}'
         git config --global user.email '[email protected]'
-        git pull
+        set -o errexit -o pipefail -o xtrace
         git add ./scripts/SCRIPTS.md
+        git status
         git commit -m "${{ env.CI_COMMIT_MESSAGE }}"
-        git push
+        git status
+        git push --verbose

.github/workflows/typescript-code-analysis.yml

@@ -179,8 +179,9 @@ jobs:
         git config --global user.name '${{ env.CI_COMMIT_AUTHOR }}'
         git config --global user.email '[email protected]'
         git config --local http.postBuffer 524288000
-        git pull
+        set -o errexit -o pipefail -o xtrace
         git add results
         git status
         git commit -m "${{ env.CI_COMMIT_MESSAGE }}"
-        git push
+        git status
+        git push --verbose

scripts/copyReportsIntoResults.sh

@@ -41,13 +41,4 @@ for report_source_folder in **/"${REPORTS_DIRECTORY}"; do
 
     rm -rf "${reportTargetDirectory}"
     cp -Rp "${report_source_folder}" "${reportTargetDirectory}"
-done
-
-# Generate JUPYTER_REPORTS.md containing a reference to all Jupyter Notebook Markdown reports in the "results" directory and its subdirectories.
-(cd "./../${RESULTS_DIRECTORY}" && exec "${SCRIPTS_DIR}/documentation/generateJupyterReportReference.sh")
-
-# Generate CSV_REPORTS.md containing a reference to all CSV cypher query reports in the "results" directory and its subdirectories.
-(cd "./../${RESULTS_DIRECTORY}" && exec "${SCRIPTS_DIR}/documentation/generateCsvReportReference.sh")
-
-# Generate IMAGES.md containing a reference to all PNG images in the "results" directory and its subdirectories.
-(cd "./../${RESULTS_DIRECTORY}" && exec "${SCRIPTS_DIR}/documentation/generateImageReference.sh")
+done

scripts/documentation/generateCsvReportReference.sh

@@ -9,7 +9,7 @@
 set -o errexit -o pipefail
 
 # Output Markdown file name
-output_file="CSV_REPORTS.md"
+markdown_file="CSV_REPORTS.md"
 
 # Function to count rows in a CSV file
 count_rows() {
@@ -23,6 +23,8 @@ get_source_query() {
   echo "$source_query"
 }
 
+echo "generateCsvReportReference: Generating ${markdown_file}..."
+
 # Create the Markdown header
 {
   echo "# CSV Cypher Query Reports Reference"
@@ -33,10 +35,10 @@ get_source_query() {
   # Create the Markdown table header
   echo "| CSV File | Analysis | Number of Rows | Source Query |"
   echo "| -------- | -------- | -------------- | ------------ |"
-} > "$output_file"
+} > "$markdown_file"
 
 # Find and process CSV files
-find . -type f -name "*.csv" | while IFS= read -r csv_file; do
+find . -type f -name "*.csv" | sort | while IFS= read -r csv_file; do
   num_rows=$(count_rows "$csv_file")
   source_query=$(get_source_query "$csv_file")
 
@@ -53,7 +55,7 @@ find . -type f -name "*.csv" | while IFS= read -r csv_file; do
   source_query_link="./../cypher/$source_query"
 
   # Append a new row to the Markdown table
-  echo "| [$base_name]($csv_link) | $main_dir | $num_rows | [$source_query]($source_query_link) |" >> "$output_file"
+  echo "| [$base_name]($csv_link) | $main_dir | $num_rows | [$source_query]($source_query_link) |" >> "$markdown_file"
 done
 
-echo "CSV table and header generated in $output_file"
+echo "generateCsvReportReference: Successfully generated ${markdown_file}."

scripts/documentation/generateCypherReference.sh

@@ -9,6 +9,8 @@ set -o errexit -o pipefail
 # Markdown file name
 markdown_file="CYPHER.md"
 
+echo "generateCypherReference: Generating ${markdown_file}..."
+
 { 
   echo "# Cypher Reference" 
   echo ""
@@ -43,3 +45,5 @@ find . -type f -name "*.cypher" | sort | while read -r cypher_file; do
     # Add the script file and its description to the Markdown table
     echo "| ${link} | ${last_path_segment%%.} | ${description//|/\\|} |" >> ${markdown_file}
 done
+
+echo "generateCypherReference: Successfully generated ${markdown_file}."

scripts/documentation/generateEnvironmentVariableReference.sh

@@ -14,11 +14,15 @@ set -o errexit -o pipefail
 DOCUMENTATION_SCRIPTS_DIR=${DOCUMENTATION_SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the documentation generation scripts
 echo "generateEnvironmentVariablesReference: DOCUMENTATION_SCRIPTS_DIR=${DOCUMENTATION_SCRIPTS_DIR}"
 
+echo "generateEnvironmentVariableReference: Generating ENVIRONMENT_VARIABLES.md..."
+
 # Clear existing markdown document
 source "${DOCUMENTATION_SCRIPTS_DIR}/appendEnvironmentVariables.sh" "clear" || exit 1
 
 # Loop through all script files in the current directory
 find . -type f -name "*.sh" | sort | while read -r scriptFile; do
   echo "generateEnvironmentVariablesReference: Searching for environment variables in ${scriptFile}"
   source "${DOCUMENTATION_SCRIPTS_DIR}/appendEnvironmentVariables.sh" "${scriptFile}" || exit 1
-done
+done
+
+echo "generateEnvironmentVariableReference: Successfully generated ENVIRONMENT_VARIABLES.md."

scripts/documentation/generateImageReference.sh

@@ -8,6 +8,8 @@ set -o errexit -o pipefail
 # Markdown file name
 markdown_file="IMAGES.md"
 
+echo "generateImageReference: Generating ${markdown_file}..."
+
 { 
   echo "# Image Reference" 
   echo ""
@@ -43,3 +45,5 @@ find . -type f -name "*.png" | sort | while read -r image_file; do
     # Add the script file and its description to the Markdown table
     echo "| ${link} | ${analysisname%%.} |" >> ${markdown_file}
 done
+
+echo "generateImageReference: Successfully generated ${markdown_file}."

scripts/documentation/generateJupyterReportReference.sh

@@ -9,6 +9,8 @@ set -o errexit -o pipefail
 # Markdown file name
 markdown_file="JUPYTER_REPORTS.md"
 
+echo "generateJupyterReportReference: Generating ${markdown_file}..."
+
 { 
   echo "# Jupyter Notebook Reports Reference" 
   echo ""
@@ -50,3 +52,5 @@ find . -type f -name "*.md" | sort | while read -r report_file; do
     # Add the script file and its description to the Markdown table
     echo "| ${link} | ${analysisname%%.} | ${description} |" >> ${markdown_file}
 done
+
+echo "generateJupyterReportReference: Successfully generated ${markdown_file}."

scripts/documentation/generateReportReferences.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Triggers the regeneration of all reference documentations for the reports inside the "results" directory.
+# The generation to the reference documentations is delegated to the dedicated scripts.
+
+# Notice that this scripts needs to be executed within the "results" directory.
+
+# Requires generateJupyterReportReference.sh, generateCsvReportReference.sh, generateImageReference.sh
+
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -o errexit -o pipefail
+
+## Get this "scripts" directory if not already set
+# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
+# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
+# This way non-standard tools like readlink aren't needed.
+DOCUMENTATION_SCRIPT_DIR=${DOCUMENTATION_SCRIPT_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts
+echo "generateReportReferences: SCRIPTS_DIR=${DOCUMENTATION_SCRIPT_DIR}"
+
+# Generate JUPYTER_REPORTS.md containing a reference to all Jupyter Notebook Markdown reports in the "results" directory and its subdirectories.
+source "${DOCUMENTATION_SCRIPT_DIR}/generateJupyterReportReference.sh"
+
+# Generate CSV_REPORTS.md containing a reference to all CSV cypher query reports in the "results" directory and its subdirectories.
+source "${DOCUMENTATION_SCRIPT_DIR}/generateCsvReportReference.sh"
+
+# Generate IMAGES.md containing a reference to all PNG images in the "results" directory and its subdirectories.
+source "${DOCUMENTATION_SCRIPT_DIR}/generateImageReference.sh"

scripts/documentation/generateScriptReference.sh

@@ -9,6 +9,8 @@ set -o errexit -o pipefail
 # Markdown file name
 markdown_file="SCRIPTS.md"
 
+echo "generateScriptReference: Generating ${markdown_file}..."
+
 { 
   echo "# Scripts Reference" 
   echo ""
@@ -38,3 +40,5 @@ find . -type f -name "*.sh" | sort | while read -r script_file; do
     # Add the script file and its description to the Markdown table
     echo "| ${link} | ${last_path_segment%%.} | ${description//|/\\|} |" >> ${markdown_file}
 done
+
+echo "generateScriptReference: Successfully generated ${markdown_file}."