feat: add audit output caching for execute plugin

examples/plugins/src/file-size/src/file-size.plugin.int.test.ts

@@ -63,7 +63,12 @@ describe('create', () => {
 
   it('should return PluginConfig that executes correctly', async () => {
     const pluginConfig = create(baseOptions);
-    await expect(executePlugin(pluginConfig)).resolves.toMatchObject({
+    await expect(
+      executePlugin(pluginConfig, {
+        persist: { outputDir: '.code-pushup' },
+        cache: { read: false, write: false },
+      }),
+    ).resolves.toMatchObject({
       description:
         'A plugin to measure and assert size of files in a directory.',
       slug,
@@ -79,7 +84,10 @@ describe('create', () => {
       ...baseOptions,
       pattern: /\.js$/,
     });
-    const { audits: auditOutputs } = await executePlugin(pluginConfig);
+    const { audits: auditOutputs } = await executePlugin(pluginConfig, {
+      persist: { outputDir: '.code-pushup' },
+      cache: { read: false, write: false },
+    });
 
     expect(auditOutputs).toHaveLength(1);
     expect(auditOutputs[0]?.score).toBe(1);
@@ -91,7 +99,10 @@ describe('create', () => {
       ...baseOptions,
       budget: 0,
     });
-    const { audits: auditOutputs } = await executePlugin(pluginConfig);
+    const { audits: auditOutputs } = await executePlugin(pluginConfig, {
+      persist: { outputDir: '.code-pushup' },
+      cache: { read: false, write: false },
+    });
 
     expect(auditOutputs).toHaveLength(1);
     expect(auditOutputs[0]?.score).toBe(0);

examples/plugins/src/package-json/src/package-json.plugin.int.test.ts

@@ -47,7 +47,10 @@ describe('create-package-json', () => {
 
   it('should return PluginConfig that executes correctly', async () => {
     const pluginConfig = create(baseOptions);
-    const pluginOutput = await executePlugin(pluginConfig);
+    const pluginOutput = await executePlugin(pluginConfig, {
+      persist: { outputDir: '.code-pushup' },
+      cache: { read: false, write: false },
+    });
 
     expect(() => pluginReportSchema.parse(pluginOutput)).not.toThrow();
     expect(pluginOutput).toMatchObject(
@@ -68,7 +71,10 @@ describe('create-package-json', () => {
       ...baseOptions,
       license: 'MIT',
     });
-    const { audits: auditOutputs } = await executePlugin(pluginConfig);
+    const { audits: auditOutputs } = await executePlugin(pluginConfig, {
+      persist: { outputDir: '.code-pushup' },
+      cache: { read: false, write: false },
+    });
 
     expect(auditOutputs[0]?.value).toBe(1);
     expect(auditOutputs[0]?.score).toBe(0);
@@ -85,7 +91,10 @@ describe('create-package-json', () => {
       ...baseOptions,
       type: 'module',
     });
-    const { audits: auditOutputs } = await executePlugin(pluginConfig);
+    const { audits: auditOutputs } = await executePlugin(pluginConfig, {
+      persist: { outputDir: '.code-pushup' },
+      cache: { read: false, write: false },
+    });
 
     expect(auditOutputs[1]?.slug).toBe('package-type');
     expect(auditOutputs[1]?.score).toBe(0);
@@ -104,7 +113,10 @@ describe('create-package-json', () => {
         test: '0',
       },
     });
-    const { audits: auditOutputs } = await executePlugin(pluginConfig);
+    const { audits: auditOutputs } = await executePlugin(pluginConfig, {
+      persist: { outputDir: '.code-pushup' },
+      cache: { read: false, write: false },
+    });
 
     expect(auditOutputs).toHaveLength(audits.length);
     expect(auditOutputs[2]?.slug).toBe('package-dependencies');
@@ -125,7 +137,10 @@ describe('create-package-json', () => {
         test: '0',
       },
     });
-    const { audits: auditOutputs } = await executePlugin(pluginConfig);
+    const { audits: auditOutputs } = await executePlugin(pluginConfig, {
+      persist: { outputDir: '.code-pushup' },
+      cache: { read: false, write: false },
+    });
 
     expect(auditOutputs).toHaveLength(audits.length);
     expect(auditOutputs[2]?.score).toBe(0);
@@ -146,7 +161,10 @@ describe('create-package-json', () => {
         test: '0',
       },
     });
-    const { audits: auditOutputs } = await executePlugin(pluginConfig);
+    const { audits: auditOutputs } = await executePlugin(pluginConfig, {
+      persist: { outputDir: '.code-pushup' },
+      cache: { read: false, write: false },
+    });
 
     expect(auditOutputs).toHaveLength(audits.length);
     expect(auditOutputs[2]?.score).toBe(0);

packages/core/src/index.ts

@@ -17,8 +17,8 @@ export type { ReportsToCompare } from './lib/implementation/compare-scorables.js
 export {
   executePlugin,
   executePlugins,
-  PluginOutputMissingAuditError,
 } from './lib/implementation/execute-plugin.js';
+export { AuditOutputsMissingAuditError } from './lib/implementation/runner.js';
 export {
   PersistDirError,
   PersistError,

packages/core/src/lib/implementation/collect.ts

@@ -1,22 +1,36 @@
 import { createRequire } from 'node:module';
-import type { CoreConfig, Report } from '@code-pushup/models';
+import {
+  type CoreConfig,
+  DEFAULT_PERSIST_OUTPUT_DIR,
+  type PersistConfig,
+  type Report,
+} from '@code-pushup/models';
 import { calcDuration, getLatestCommit } from '@code-pushup/utils';
 import type { GlobalOptions } from '../types.js';
 import { executePlugins } from './execute-plugin.js';
 
-export type CollectOptions = Pick<CoreConfig, 'plugins' | 'categories'> &
-  Partial<GlobalOptions>;
+export type CollectOptions = Pick<CoreConfig, 'plugins' | 'categories'> & {
+  persist?: Required<Pick<PersistConfig, 'outputDir'>>;
+} & Partial<GlobalOptions>;
 
 /**
  * Run audits, collect plugin output and aggregate it into a JSON object
  * @param options
  */
 export async function collect(options: CollectOptions): Promise<Report> {
-  const { plugins, categories } = options;
+  const { plugins, categories, persist, ...otherOptions } = options;
   const date = new Date().toISOString();
   const start = performance.now();
   const commit = await getLatestCommit();
-  const pluginOutputs = await executePlugins(plugins, options);
+  const pluginOutputs = await executePlugins(
+    {
+      plugins,
+      persist: { outputDir: DEFAULT_PERSIST_OUTPUT_DIR, ...persist },
+      // implement together with CLI option
+      cache: { read: false, write: false },
+    },
+    otherOptions,
+  );
   const packageJson = createRequire(import.meta.url)(
     '../../../package.json',
   ) as typeof import('../../../package.json');

packages/core/src/lib/implementation/execute-plugin.ts

@@ -1,12 +1,12 @@
 import { bold } from 'ansis';
-import {
-  type Audit,
-  type AuditOutput,
-  type AuditOutputs,
-  type AuditReport,
-  type PluginConfig,
-  type PluginReport,
-  auditOutputsSchema,
+import type {
+  Audit,
+  AuditOutput,
+  AuditReport,
+  CacheConfigObject,
+  PersistConfig,
+  PluginConfig,
+  PluginReport,
 } from '@code-pushup/models';
 import {
   type ProgressBar,
@@ -15,29 +15,20 @@ import {
   logMultipleResults,
   pluralizeToken,
 } from '@code-pushup/utils';
-import { normalizeAuditOutputs } from '../normalize.js';
-import { executeRunnerConfig, executeRunnerFunction } from './runner.js';
-
-/**
- * Error thrown when plugin output is invalid.
- */
-export class PluginOutputMissingAuditError extends Error {
-  constructor(auditSlug: string) {
-    super(
-      `Audit metadata not present in plugin config. Missing slug: ${bold(
-        auditSlug,
-      )}`,
-    );
-  }
-}
+import {
+  executePluginRunner,
+  readRunnerResults,
+  writeRunnerResults,
+} from './runner.js';
 
 /**
  * Execute a plugin.
  *
  * @public
  * @param pluginConfig - {@link ProcessConfig} object with runner and meta
+ * @param opt
  * @returns {Promise<AuditOutput[]>} - audit outputs from plugin runner
- * @throws {PluginOutputMissingAuditError} - if plugin runner output is invalid
+ * @throws {AuditOutputsMissingAuditError} - if plugin runner output is invalid
  *
  * @example
  * // plugin execution
@@ -54,7 +45,12 @@ export class PluginOutputMissingAuditError extends Error {
  */
 export async function executePlugin(
   pluginConfig: PluginConfig,
+  opt: {
+    cache: CacheConfigObject;
+    persist: Required<Pick<PersistConfig, 'outputDir'>>;
+  },
 ): Promise<PluginReport> {
+  const { cache, persist } = opt;
   const {
     runner,
     audits: pluginConfigAudits,
@@ -63,26 +59,25 @@ export async function executePlugin(
     groups,
     ...pluginMeta
   } = pluginConfig;
-
-  // execute plugin runner
-  const runnerResult =
-    typeof runner === 'object'
-      ? await executeRunnerConfig(runner)
-      : await executeRunnerFunction(runner);
-  const { audits: unvalidatedAuditOutputs, ...executionMeta } = runnerResult;
-
-  // validate auditOutputs
-  const result = auditOutputsSchema.safeParse(unvalidatedAuditOutputs);
-  if (!result.success) {
-    throw new Error(`Audit output is invalid: ${result.error.message}`);
+  const { write: cacheWrite = false, read: cacheRead = false } = cache;
+  const { outputDir } = persist;
+
+  const { audits, ...executionMeta } = cacheRead
+    ? // IF not null, take the result from cache
+      ((await readRunnerResults(pluginMeta.slug, outputDir)) ??
+      // ELSE execute the plugin runner
+      (await executePluginRunner(pluginConfig)))
+    : await executePluginRunner(pluginConfig);
+
+  if (cacheWrite) {
+    await writeRunnerResults(pluginMeta.slug, outputDir, {
+      ...executionMeta,
+      audits,
+    });
   }
-  const auditOutputs = result.data;
-  auditOutputsCorrelateWithPluginOutput(auditOutputs, pluginConfigAudits);
-
-  const normalizedAuditOutputs = await normalizeAuditOutputs(auditOutputs);
 
   // enrich `AuditOutputs` to `AuditReport`
-  const auditReports: AuditReport[] = normalizedAuditOutputs.map(
+  const auditReports: AuditReport[] = audits.map(
     (auditOutput: AuditOutput) => ({
       ...auditOutput,
       ...(pluginConfigAudits.find(
@@ -103,13 +98,18 @@ export async function executePlugin(
 }
 
 const wrapProgress = async (
-  pluginCfg: PluginConfig,
+  cfg: {
+    plugin: PluginConfig;
+    persist: Required<Pick<PersistConfig, 'outputDir'>>;
+    cache: CacheConfigObject;
+  },
   steps: number,
   progressBar: ProgressBar | null,
 ) => {
+  const { plugin: pluginCfg, ...rest } = cfg;
   progressBar?.updateTitle(`Executing ${bold(pluginCfg.title)}`);
   try {
-    const pluginReport = await executePlugin(pluginCfg);
+    const pluginReport = await executePlugin(pluginCfg, rest);
     progressBar?.incrementInSteps(steps);
     return pluginReport;
   } catch (error) {
@@ -127,7 +127,7 @@ const wrapProgress = async (
 /**
  * Execute multiple plugins and aggregates their output.
  * @public
- * @param plugins array of {@link PluginConfig} objects
+ * @param cfg
  * @param {Object} [options] execution options
  * @param {boolean} options.progress show progress bar
  * @returns {Promise<PluginReport[]>} plugin report
@@ -146,15 +146,24 @@ const wrapProgress = async (
  *
  */
 export async function executePlugins(
-  plugins: PluginConfig[],
+  cfg: {
+    plugins: PluginConfig[];
+    persist: Required<Pick<PersistConfig, 'outputDir'>>;
+    cache: CacheConfigObject;
+  },
   options?: { progress?: boolean },
 ): Promise<PluginReport[]> {
+  const { plugins, ...cacheCfg } = cfg;
   const { progress = false } = options ?? {};
 
   const progressBar = progress ? getProgressBar('Run plugins') : null;
 
   const pluginsResult = plugins.map(pluginCfg =>
-    wrapProgress(pluginCfg, plugins.length, progressBar),
+    wrapProgress(
+      { plugin: pluginCfg, ...cacheCfg },
+      plugins.length,
+      progressBar,
+    ),
   );
 
   const errorsTransform = ({ reason }: PromiseRejectedResult) => String(reason);
@@ -179,17 +188,3 @@ export async function executePlugins(
 
   return fulfilled.map(result => result.value);
 }
-
-function auditOutputsCorrelateWithPluginOutput(
-  auditOutputs: AuditOutputs,
-  pluginConfigAudits: PluginConfig['audits'],
-) {
-  auditOutputs.forEach(auditOutput => {
-    const auditMetadata = pluginConfigAudits.find(
-      audit => audit.slug === auditOutput.slug,
-    );
-    if (!auditMetadata) {
-      throw new PluginOutputMissingAuditError(auditOutput.slug);
-    }
-  });
-}