From 339e65bfe37472d529fcd331a93390f407360bc4 Mon Sep 17 00:00:00 2001 From: JohT Date: Sat, 9 Dec 2023 21:50:22 +0100 Subject: [PATCH 1/7] Improve error handling by providing screenshots --- graph-visualization/renderVisualizations.js | 56 +++++++++++++++------ 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/graph-visualization/renderVisualizations.js b/graph-visualization/renderVisualizations.js index 1c3850890..c2ae05261 100644 --- a/graph-visualization/renderVisualizations.js +++ b/graph-visualization/renderVisualizations.js @@ -13,7 +13,7 @@ console.log(`renderVisualizations.js: dirname=${__dirname}`); /** * Crops the image in the buffer so that there is no empty frame around it. - * @param {Buffer} buffer + * @param {Buffer} buffer * @returns Buffer */ const autoCropImageBuffer = async (buffer) => { @@ -31,6 +31,30 @@ const autoCropImageBuffer = async (buffer) => { */ const camelToKebabCase = (str) => str.replace(/[A-Z]/g, (letter) => `-${letter.toLowerCase()}`); +/** + * Take a screenshot after an error happened. + * + * @param {string} htmlFilename + * @param {string} reason + */ +const makeScreenshotOfError = async (page, htmlFilename, reason) => { + const reportName = basename(htmlFilename, ".html"); + const directoryName = camelToKebabCase(reportName); + console.log(`Taking an error screenshot of report ${reportName}. Reason: ${reason}`); + await page.screenshot({ path: `./${directoryName}/error-${reportName}-${reason}.png`, omitBackground: false }); +}; + +/** + * Handle a catched error by taking a screenshot. + * + * @param {string} htmlFilename + * @param {string} reason + * @return + */ +const handleErrorWithScreenshot = (page, htmlFilename, reason) => async (error) => { + await makeScreenshotOfError(page, htmlFilename, reason); + throw new Error(error); +}; /** * Creates a new page and takes a screenshot of all canvas tags that are contained within a div. * Note: Don't run this in parallel. See https://github.com/puppeteer/puppeteer/issues/1479 @@ -38,12 +62,21 @@ const camelToKebabCase = (str) => str.replace(/[A-Z]/g, (letter) => `-${letter.t * @param {string} htmlFilename */ const takeCanvasScreenshots = async (browser, htmlFilename) => { + const reportName = basename(htmlFilename, ".html"); + const directoryName = camelToKebabCase(reportName); + if (!existsSync(directoryName)) { + mkdirSync(directoryName); + } + const page = await browser.newPage(); await page.setViewport({ width: 1500, height: 1000, isMobile: false, isLandscape: true, hasTouch: false, deviceScaleFactor: 1 }); console.log(`Loading ${htmlFilename}`); await page.goto(`file://${htmlFilename}`); + if (!process.env.NEO4J_INITIAL_PASSWORD) { + throw new Error("Missing environment variable NEO4J_INITIAL_PASSWORD"); + } // Login with Neo4j server password from the environment variable NEO4J_INITIAL_PASSWORD const loginButton = await page.waitForSelector("#neo4j-server-login"); await page.type("#neo4j-server-password", process.env.NEO4J_INITIAL_PASSWORD); @@ -51,21 +84,18 @@ const takeCanvasScreenshots = async (browser, htmlFilename) => { // Wait for the graph visualization to be rendered onto a HTML5 canvas console.log(`Waiting for visualizations to be finished`); - await page.waitForSelector(".visualization-finished", { timeout: 90_000 }); + await page + .waitForSelector(".visualization-finished", { timeout: 90_000 }) + .catch(handleErrorWithScreenshot(page, htmlFilename, "visualization-did-not-finish")); // Get all HTML canvas tag elements const canvasElements = await page.$$("canvas"); if (canvasElements.length <= 0) { - console.error(`No elements with CSS selector 'canvas' found in ${htmlFilename}`); + await makeScreenshotOfError(page, htmlFilename, "no-canvas-found"); } console.log(`Found ${canvasElements.length} visualizations`); // Take a png screenshot of every canvas element and save them with increasing indices - const reportName = basename(htmlFilename, ".html"); - const directoryName = camelToKebabCase(reportName); - if (!existsSync(directoryName)) { - mkdirSync(directoryName); - } await Promise.all( Array.from(canvasElements).map(async (canvasElement, index) => { console.log(`Exporting image ${reportName}-${index}.png...`); @@ -74,10 +104,8 @@ const takeCanvasScreenshots = async (browser, htmlFilename) => { }, canvasElement); let data = Buffer.from(dataUrl.split(",").pop(), "base64"); console.log(`Cropping image ${reportName}-${index}.png...`); - data = await autoCropImageBuffer(data); + data = await autoCropImageBuffer(data).catch(handleErrorWithScreenshot(page, htmlFilename, `failed-to-crop-image-${index}`)); writeFileSync(`./${directoryName}/${reportName}-${index}.png`, data); - // console.log(`Taking screenshot ${reportName} of canvas ${index} in ${htmlFilename} of element...`); - // await canvasElement.screenshot({ path: `./${directoryName}/${reportName}-${index}.png`, omitBackground: true }); }) ); }; @@ -89,13 +117,13 @@ let browser; * and takes a screenshot of the canvas elements using {@link takeCanvasScreenshots}. */ (async () => { - console.log('renderVisualizations.js: Starting headless browser...'); + console.log("renderVisualizations.js: Starting headless browser..."); browser = await puppeteer.launch({ headless: "new" }); // { headless: false } for testing // Get all *.html files in this (script) directory and its subdirectories - // The separate filter is needed to ignore the "node_modules" directory. + // The separate filter is needed to ignore the "node_modules" directory. // Glob's build-in filter doesn't seem to work on Windows. - const htmlFiles = globSync(`${__dirname}/**/*.html`, { absolute: true }).filter(file => !file.includes('node_modules')); + const htmlFiles = globSync(`${__dirname}/**/*.html`, { absolute: true }).filter((file) => !file.includes("node_modules")); for (const htmlFile of htmlFiles) { await takeCanvasScreenshots(browser, htmlFile); } From 9abd1ea5f747a18d1869d88bfcbd4d994bb5c703 Mon Sep 17 00:00:00 2001 From: JohT Date: Sat, 9 Dec 2023 21:59:41 +0100 Subject: [PATCH 2/7] Pass error to node to exit the script (fail fast) --- graph-visualization/renderVisualizations.js | 4 +--- scripts/reports/GraphVisualization.sh | 4 +++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/graph-visualization/renderVisualizations.js b/graph-visualization/renderVisualizations.js index c2ae05261..7723651e3 100644 --- a/graph-visualization/renderVisualizations.js +++ b/graph-visualization/renderVisualizations.js @@ -128,6 +128,4 @@ let browser; await takeCanvasScreenshots(browser, htmlFile); } console.log(`renderVisualizations.js: Successfully rendered ${htmlFiles.length} html file(s)`); -})() - .catch((err) => console.error(err)) - .finally(() => browser?.close()); +})().finally(() => browser?.close()); diff --git a/scripts/reports/GraphVisualization.sh b/scripts/reports/GraphVisualization.sh index 8b1f97f7b..99bde269c 100755 --- a/scripts/reports/GraphVisualization.sh +++ b/scripts/reports/GraphVisualization.sh @@ -33,4 +33,6 @@ if [ ! -d "${GRAPH_VISUALIZATION_DIRECTORY}/node_modules" ] ; then fi # Execute the node.js script to render the graph visualizations as image files -(cd "${REPORTS_DIRECTORY}" && exec node "${GRAPH_VISUALIZATION_DIRECTORY}/renderVisualizations.js") \ No newline at end of file +(cd "${REPORTS_DIRECTORY}" && exec node "${GRAPH_VISUALIZATION_DIRECTORY}/renderVisualizations.js") + +echo "GraphVisualization: Successfully finished" \ No newline at end of file From 321f19010237466f9614bda5d47b7c7ec23d5807 Mon Sep 17 00:00:00 2001 From: JohT Date: Sat, 9 Dec 2023 22:35:43 +0100 Subject: [PATCH 3/7] Add intermediate results to error archive --- .github/workflows/code-structure-analysis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/code-structure-analysis.yml b/.github/workflows/code-structure-analysis.yml index 08ec02bf2..6b8f72ffb 100644 --- a/.github/workflows/code-structure-analysis.yml +++ b/.github/workflows/code-structure-analysis.yml @@ -141,6 +141,7 @@ jobs: name: code-analysis-logs-java-${{ matrix.java }}-python-${{ matrix.python }}-mambaforge-${{ matrix.mambaforge }} path: | ./temp/**/runtime/* + ./temp/**/reports/* retention-days: 5 # Upload successful results in case they are needed for troubleshooting From f097acd22d1c4c5e2c6c97114e185bb7e032e5ab Mon Sep 17 00:00:00 2001 From: JohT Date: Sun, 10 Dec 2023 09:42:28 +0100 Subject: [PATCH 4/7] Improve error handling by forwarding browser logs --- graph-visualization/renderVisualizations.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/graph-visualization/renderVisualizations.js b/graph-visualization/renderVisualizations.js index 7723651e3..66804f45d 100644 --- a/graph-visualization/renderVisualizations.js +++ b/graph-visualization/renderVisualizations.js @@ -69,6 +69,7 @@ const takeCanvasScreenshots = async (browser, htmlFilename) => { } const page = await browser.newPage(); + page.on("requestfailed", (request) => console.log(`${request.failure().errorText} ${request.url()}`)); await page.setViewport({ width: 1500, height: 1000, isMobile: false, isLandscape: true, hasTouch: false, deviceScaleFactor: 1 }); console.log(`Loading ${htmlFilename}`); @@ -118,7 +119,7 @@ let browser; */ (async () => { console.log("renderVisualizations.js: Starting headless browser..."); - browser = await puppeteer.launch({ headless: "new" }); // { headless: false } for testing + browser = await puppeteer.launch({ headless: "new", dumpio: true }); // { headless: false } for testing // Get all *.html files in this (script) directory and its subdirectories // The separate filter is needed to ignore the "node_modules" directory. From 558a35f0365e4a7a351716d2491e21a04b84bb62 Mon Sep 17 00:00:00 2001 From: JohT Date: Wed, 13 Dec 2023 07:32:00 +0100 Subject: [PATCH 5/7] Disable unneeded features of the headless browser --- graph-visualization/renderVisualizations.js | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/graph-visualization/renderVisualizations.js b/graph-visualization/renderVisualizations.js index 66804f45d..35d3bd2c0 100644 --- a/graph-visualization/renderVisualizations.js +++ b/graph-visualization/renderVisualizations.js @@ -119,7 +119,26 @@ let browser; */ (async () => { console.log("renderVisualizations.js: Starting headless browser..."); - browser = await puppeteer.launch({ headless: "new", dumpio: true }); // { headless: false } for testing + browser = await puppeteer.launch({ + headless: "new", + dumpio: true, + args: [ + "--enable-logging", + "--disable-search-engine-choice-screen", + "--ash-no-nudges", + "--no-first-run", + "--no-default-browser-check", + "--hide-scrollbars", + "--disable-features=Translate", + "--disable-features=InterestFeedContentSuggestions", + "--disable-extensions", + "--disable-default-apps", + "--disable-component-extensions-with-background-pages", + "--disable-client-side-phishing-detection", + "--use-gl=disabled", + "--disable-features=Vulkan", + ], + }); // { headless: false } for testing // Get all *.html files in this (script) directory and its subdirectories // The separate filter is needed to ignore the "node_modules" directory. From 5de277935266ab6856132692c9f5213369d78b63 Mon Sep 17 00:00:00 2001 From: JohT Date: Wed, 13 Dec 2023 08:10:23 +0100 Subject: [PATCH 6/7] Add additional logs to troubleshoot visualization --- graph-visualization/visualization-pagination.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/graph-visualization/visualization-pagination.js b/graph-visualization/visualization-pagination.js index 8a40f4814..1fda9bf0d 100644 --- a/graph-visualization/visualization-pagination.js +++ b/graph-visualization/visualization-pagination.js @@ -26,11 +26,14 @@ function paginatedGraphVisualization({ /** * Marks the given element as finished when the visualization is completed. * @param {Element} indexedVisualizationElement + * @param {string} logDescription */ - function markVisualizationAsFinished(indexedVisualizationElement) { + function markVisualizationAsFinished(indexedVisualizationElement, logDescription) { indexedVisualizationElement.classList.add(classOfFinishedVisualization); const unfinishedVisualizations = document.querySelectorAll(`.${classOfIndexedVisualizationElement}:not(.${classOfFinishedVisualization})`); if (unfinishedVisualizations.length === 0) { + console.debug(`${logDescription}: Last visualization finished on element ${JSON.stringify(indexedVisualizationElement)}.`); + console.debug(`${logDescription}: Mark whole visualization as finished on parent element ${JSON.stringify( indexedVisualizationElement.parentElement)}`); indexedVisualizationElement.parentElement.classList.add(classOfFinishedVisualization); } } @@ -48,19 +51,23 @@ function paginatedGraphVisualization({ neoViz.registerOnEvent(NeoVis.NeoVisEvents.CompletionEvent, (event) => { if (event.recordCount == 0) { + if (index=0) { + log.error('No query results. Nothing to visualize. Check the query and if the nodes and properties have been written.') + } indexedVisualizationContainer.remove(); // remove an empty canvas - markVisualizationAsFinished(indexedVisualizationContainer); + markVisualizationAsFinished(indexedVisualizationContainer, 'No query results (anymore)'); } else { setTimeout(() => { neoViz.stabilize(); - markVisualizationAsFinished(indexedVisualizationContainer); + markVisualizationAsFinished(indexedVisualizationContainer, 'Visualization stabilized'); }, 5000); } }); neoViz.registerOnEvent(NeoVis.NeoVisEvents.ErrorEvent, (event) => { indexedVisualizationContainer.classList.add(classOfFailedVisualization); indexedVisualizationContainer.textContent = event.error.message; - markVisualizationAsFinished(indexedVisualizationContainer); + console.error(`Visualization Error: ${JSON.stringify(event.error)}`) + markVisualizationAsFinished(indexedVisualizationContainer, 'Error event'); }); const parameters = { blockSize: recordsPerVisualization, From 667522e3f594fc999700abea3df062feccb93ecc Mon Sep 17 00:00:00 2001 From: JohT Date: Sat, 16 Dec 2023 10:09:59 +0100 Subject: [PATCH 7/7] Run report groups in order with visualizations last --- scripts/reports/compilations/AllReports.sh | 15 +++++++-------- .../reports/compilations/VisualizationReports.sh | 1 + 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/reports/compilations/AllReports.sh b/scripts/reports/compilations/AllReports.sh index 773d216a1..896799321 100755 --- a/scripts/reports/compilations/AllReports.sh +++ b/scripts/reports/compilations/AllReports.sh @@ -15,11 +15,10 @@ set -o errexit -o pipefail REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} echo "AllReports: REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT_DIR}" -REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR:-$(dirname -- "${REPORT_COMPILATIONS_SCRIPT_DIR}")} -echo "AllReports: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}" - -# Run all report scripts -for report_script_file in "${REPORTS_SCRIPT_DIR}"/*.sh; do - echo "AllReports: Starting ${report_script_file}..."; - source "${report_script_file}" -done \ No newline at end of file +# The reports will not be generically searched as files anymore +# but will be processed in order. Especially the visualization +# needs to be done as a last step to be able to use properties +# and data written to the Graph in the CsvReports. +source "${REPORT_COMPILATIONS_SCRIPT_DIR}/CsvReports.sh" +source "${REPORT_COMPILATIONS_SCRIPT_DIR}/JupyterReports.sh" +source "${REPORT_COMPILATIONS_SCRIPT_DIR}/VisualizationReports.sh" \ No newline at end of file diff --git a/scripts/reports/compilations/VisualizationReports.sh b/scripts/reports/compilations/VisualizationReports.sh index fa8590ca1..691b2eee5 100755 --- a/scripts/reports/compilations/VisualizationReports.sh +++ b/scripts/reports/compilations/VisualizationReports.sh @@ -6,6 +6,7 @@ # Therefore these reports will take longer and require more ressources than just plain database queries/procedures. # Requires reports/*.sh +# Needs to run after reports/TopologySortCsv.sh that provides the property "topologicalSortIndex" to be queried. # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands) set -o errexit -o pipefail