From e9014924f86a7f4b89ae5070dcf9d85812975ab5 Mon Sep 17 00:00:00 2001 From: Axel Sorenson Date: Tue, 31 Dec 2024 17:38:27 -0800 Subject: [PATCH] [PassBuilder] VectorizerEnd Extension Points Added an extension point after vectorizer passes in the PassBuilder. Additionally, added extension points before and after vectorizer passes in `buildLTODefaultPipeline`. Credit goes to @mshockwave for guiding me through my first LLVM contribution (and my first open source contribution in general!) :) - Implemented `registerVectorizerEndEPCallback` - Implemented `invokeVectorizerEndEPCallbacks` - Added `VectorizerEndEPCallbacks` SmallVector - Added a command line option `passes-ep-vectorizer-end` to `NewPMDriver.cpp` - `buildModuleOptimizationPipeline` now calls `invokeVectorizerEndEPCallbacks` - `buildO0DefaultPipeline` now calls `invokeVectorizerEndEPCallbacks` - `buildLTODefaultPipeline` now calls BOTH `invokeVectorizerStartEPCallbacks` and `invokeVectorizerEndEPCallbacks` - Added LIT tests to `new-pm-defaults.ll`, `new-pm-lto-defaults.ll`, `new-pm-O0-ep-callbacks.ll`, and `pass-pipeline-parsing.ll` - Renamed `CHECK-EP-Peephole` to `CHECK-EP-PEEPHOLE` in `new-pm-lto-defaults.ll` for consistency. --- llvm/include/llvm/Passes/PassBuilder.h | 15 +++++++++++++++ llvm/lib/Passes/PassBuilderPipelines.cpp | 18 ++++++++++++++++++ llvm/test/Other/new-pm-O0-ep-callbacks.ll | 1 + llvm/test/Other/new-pm-defaults.ll | 5 +++++ llvm/test/Other/new-pm-lto-defaults.ll | 16 ++++++++++++---- llvm/test/Other/pass-pipeline-parsing.ll | 3 +++ llvm/tools/opt/NewPMDriver.cpp | 11 +++++++++++ 7 files changed, 65 insertions(+), 4 deletions(-) diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index e7bc3a58f414f..1b54855a5c6f4 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -465,6 +465,17 @@ class PassBuilder { VectorizerStartEPCallbacks.push_back(C); } + /// Register a callback for a default optimizer pipeline extension + /// point + /// + /// This extension point allows adding optimization passes after the + /// vectorizer and other highly target specific optimization passes are + /// executed. + void registerVectorizerEndEPCallback( + const std::function &C) { + VectorizerEndEPCallbacks.push_back(C); + } + /// Register a callback for a default optimizer pipeline extension point. /// /// This extension point allows adding optimization once at the start of the @@ -631,6 +642,8 @@ class PassBuilder { OptimizationLevel Level); void invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM, OptimizationLevel Level); + void invokeVectorizerEndEPCallbacks(FunctionPassManager &FPM, + OptimizationLevel Level); void invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM, OptimizationLevel Level, ThinOrFullLTOPhase Phase); @@ -759,6 +772,8 @@ class PassBuilder { CGSCCOptimizerLateEPCallbacks; SmallVector, 2> VectorizerStartEPCallbacks; + SmallVector, 2> + VectorizerEndEPCallbacks; // Module callbacks SmallVector, diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index 4ec0fb8fc81ea..f5a82a243a07d 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -360,6 +360,11 @@ void PassBuilder::invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM, for (auto &C : VectorizerStartEPCallbacks) C(FPM, Level); } +void PassBuilder::invokeVectorizerEndEPCallbacks(FunctionPassManager &FPM, + OptimizationLevel Level) { + for (auto &C : VectorizerEndEPCallbacks) + C(FPM, Level); +} void PassBuilder::invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM, OptimizationLevel Level, ThinOrFullLTOPhase Phase) { @@ -1534,6 +1539,8 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level, addVectorPasses(Level, OptimizePM, /* IsFullLTO */ false); + invokeVectorizerEndEPCallbacks(OptimizePM, Level); + // LoopSink pass sinks instructions hoisted by LICM, which serves as a // canonicalization pass that enables other optimizations. As a result, // LoopSink pass needs to be a very late IR pass to avoid undoing LICM @@ -2048,6 +2055,8 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, MainFPM.addPass(MoveAutoInitPass()); MainFPM.addPass(MergedLoadStoreMotionPass()); + invokeVectorizerStartEPCallbacks(MainFPM, Level); + LoopPassManager LPM; if (EnableLoopFlatten && Level.getSpeedupLevel() > 1) LPM.addPass(LoopFlattenPass()); @@ -2068,6 +2077,8 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, addVectorPasses(Level, MainFPM, /* IsFullLTO */ true); + invokeVectorizerEndEPCallbacks(MainFPM, Level); + // Run the OpenMPOpt CGSCC pass again late. MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor( OpenMPOptCGSCCPass(ThinOrFullLTOPhase::FullLTOPostLink))); @@ -2231,6 +2242,13 @@ PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level, MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); } + if (!VectorizerEndEPCallbacks.empty()) { + FunctionPassManager FPM; + invokeVectorizerEndEPCallbacks(FPM, Level); + if (!FPM.isEmpty()) + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + } + ModulePassManager CoroPM; CoroPM.addPass(CoroEarlyPass()); CGSCCPassManager CGPM; diff --git a/llvm/test/Other/new-pm-O0-ep-callbacks.ll b/llvm/test/Other/new-pm-O0-ep-callbacks.ll index 2184a2ded5920..77ccddba56a62 100644 --- a/llvm/test/Other/new-pm-O0-ep-callbacks.ll +++ b/llvm/test/Other/new-pm-O0-ep-callbacks.ll @@ -3,6 +3,7 @@ ; RUN: opt -disable-output -debug-pass-manager -passes-ep-scalar-optimizer-late=no-op-function -passes='default' 2>&1 < %s | FileCheck %s ; RUN: opt -disable-output -debug-pass-manager -passes-ep-cgscc-optimizer-late=no-op-cgscc -passes='default' 2>&1 < %s | FileCheck %s ; RUN: opt -disable-output -debug-pass-manager -passes-ep-vectorizer-start=no-op-function -passes='default' 2>&1 < %s | FileCheck %s +; RUN: opt -disable-output -debug-pass-manager -passes-ep-vectorizer-end=no-op-function -passes='default' 2>&1 < %s | FileCheck %s ; RUN: opt -disable-output -debug-pass-manager -passes-ep-pipeline-start=no-op-module -passes='default' 2>&1 < %s | FileCheck %s ; RUN: opt -disable-output -debug-pass-manager -passes-ep-pipeline-early-simplification=no-op-module -passes='default' 2>&1 < %s | FileCheck %s ; RUN: opt -disable-output -debug-pass-manager -passes-ep-optimizer-early=no-op-module -passes='default' 2>&1 < %s | FileCheck %s diff --git a/llvm/test/Other/new-pm-defaults.ll b/llvm/test/Other/new-pm-defaults.ll index 7cf035b0c6f37..c554fdbf4c799 100644 --- a/llvm/test/Other/new-pm-defaults.ll +++ b/llvm/test/Other/new-pm-defaults.ll @@ -51,6 +51,10 @@ ; RUN: -passes='default' -S %s 2>&1 \ ; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-O3,%llvmcheckext,CHECK-EP-VECTORIZER-START,CHECK-O23SZ ; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \ +; RUN: -passes-ep-vectorizer-end='no-op-function' \ +; RUN: -passes='default' -S %s 2>&1 \ +; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-O3,%llvmcheckext,CHECK-EP-VECTORIZER-END,CHECK-O23SZ +; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \ ; RUN: -passes-ep-pipeline-start='no-op-module' \ ; RUN: -passes='default' -S %s 2>&1 \ ; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-O3,%llvmcheckext,CHECK-EP-PIPELINE-START,CHECK-O23SZ @@ -274,6 +278,7 @@ ; CHECK-O-NEXT: Running pass: LCSSAPass ; CHECK-O-NEXT: Running pass: LICMPass ; CHECK-O-NEXT: Running pass: AlignmentFromAssumptionsPass +; CHECK-EP-VECTORIZER-END-NEXT: Running pass: NoOpFunctionPass ; CHECK-O-NEXT: Running pass: LoopSinkPass ; CHECK-O-NEXT: Running pass: InstSimplifyPass ; CHECK-O-NEXT: Running pass: DivRemPairsPass diff --git a/llvm/test/Other/new-pm-lto-defaults.ll b/llvm/test/Other/new-pm-lto-defaults.ll index f788db1e338a1..0b0202d4a14b5 100644 --- a/llvm/test/Other/new-pm-lto-defaults.ll +++ b/llvm/test/Other/new-pm-lto-defaults.ll @@ -18,6 +18,12 @@ ; RUN: -passes='lto' -S %s 2>&1 \ ; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ ; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \ +; RUN: -passes='lto' -S %s -passes-ep-vectorizer-start='no-op-function' 2>&1 \ +; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ,CHECK-EP-VECTORIZER-START +; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \ +; RUN: -passes='lto' -S %s -passes-ep-vectorizer-end='no-op-function' 2>&1 \ +; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ,CHECK-EP-VECTORIZER-END +; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \ ; RUN: -passes='lto' -S %s 2>&1 \ ; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-OS,CHECK-O23SZ ; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \ @@ -25,7 +31,7 @@ ; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O23SZ ; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \ ; RUN: -passes='lto' -S %s -passes-ep-peephole='no-op-function' 2>&1 \ -; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ,CHECK-EP-Peephole +; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ,CHECK-EP-PEEPHOLE ; CHECK-EP: Running pass: NoOpModulePass ; CHECK-O: Running pass: CrossDSOCFIPass @@ -69,7 +75,7 @@ ; CHECK-O23SZ-NEXT: Running pass: InstCombinePass ; CHECK-O23SZ-NEXT: Running analysis: LastRunTrackingAnalysis ; CHECK-O23SZ-NEXT: Running pass: AggressiveInstCombinePass -; CHECK-EP-Peephole-NEXT: Running pass: NoOpFunctionPass +; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass ; CHECK-O23SZ-NEXT: Running pass: ExpandVariadicsPass ; CHECK-O23SZ-NEXT: Running pass: ModuleInlinerWrapperPass ; CHECK-O23SZ-NEXT: Running analysis: InlineAdvisorAnalysis @@ -81,7 +87,7 @@ ; CHECK-O23SZ-NEXT: Running pass: GlobalDCEPass ; CHECK-O23SZ-NEXT: Running pass: ArgumentPromotionPass ; CHECK-O23SZ-NEXT: Running pass: InstCombinePass -; CHECK-EP-Peephole-NEXT: Running pass: NoOpFunctionPass +; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass ; CHECK-O23SZ-NEXT: Running pass: ConstraintEliminationPass ; CHECK-O23SZ-NEXT: Running analysis: LoopAnalysis on foo ; CHECK-O23SZ-NEXT: Running analysis: ScalarEvolutionAnalysis on foo @@ -109,6 +115,7 @@ ; CHECK-O23SZ-NEXT: Running pass: DSEPass on foo ; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo ; CHECK-O23SZ-NEXT: Running pass: MergedLoadStoreMotionPass on foo +; CHECK-EP-VECTORIZER-START-NEXT: Running pass: NoOpFunctionPass on foo ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass on foo ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass on foo ; CHECK-O23SZ-NEXT: Running pass: IndVarSimplifyPass on loop @@ -137,7 +144,8 @@ ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass ; CHECK-O23SZ-NEXT: Running pass: AlignmentFromAssumptionsPass on foo -; CHECK-EP-Peephole-NEXT: Running pass: NoOpFunctionPass on foo +; CHECK-EP-VECTORIZER-END-NEXT: Running pass: NoOpFunctionPass on foo +; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass on foo ; CHECK-O23SZ-NEXT: Running pass: JumpThreadingPass on foo ; CHECK-O23SZ-NEXT: Running pass: LowerTypeTestsPass ; CHECK-O-NEXT: Running pass: LowerTypeTestsPass diff --git a/llvm/test/Other/pass-pipeline-parsing.ll b/llvm/test/Other/pass-pipeline-parsing.ll index fd65224b1ef0d..4253b646b69b0 100644 --- a/llvm/test/Other/pass-pipeline-parsing.ll +++ b/llvm/test/Other/pass-pipeline-parsing.ll @@ -264,6 +264,9 @@ ; RUN: opt -passes-ep-vectorizer-start=bad -passes=no-op-function \ ; RUN: /dev/null -disable-output 2>&1 | FileCheck %s -check-prefix=PASSES-EP-VECTORIZERSTART-ERR ; PASSES-EP-VECTORIZERSTART-ERR: Could not parse -passes-ep-vectorizer-start pipeline: unknown function pass 'bad' +; RUN: opt -passes-ep-vectorizer-end=bad -passes=no-op-function \ +; RUN: /dev/null -disable-output 2>&1 | FileCheck %s -check-prefix=PASSES-EP-VECTORIZEREND-ERR +; PASSES-EP-VECTORIZEREND-ERR: Could not parse -passes-ep-vectorizer-end pipeline: unknown function pass 'bad' ; RUN: opt -passes-ep-pipeline-start=bad -passes=no-op-function \ ; RUN: /dev/null -disable-output 2>&1 | FileCheck %s -check-prefix=PASSES-EP-PIPELINESTART-ERR ; PASSES-EP-PIPELINESTART-ERR: Could not parse -passes-ep-pipeline-start pipeline: unknown pass name 'bad' diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp index ea300fb3dbeb1..5ed42b054316b 100644 --- a/llvm/tools/opt/NewPMDriver.cpp +++ b/llvm/tools/opt/NewPMDriver.cpp @@ -122,6 +122,11 @@ static cl::opt VectorizerStartEPPipeline( cl::desc("A textual description of the function pass pipeline inserted at " "the VectorizerStart extension point into default pipelines"), cl::Hidden); +static cl::opt VectorizerEndEPPipeline( + "passes-ep-vectorizer-end", + cl::desc("A textual description of the function pass pipeline inserted at " + "the VectorizerEnd extension point into default pipelines"), + cl::Hidden); static cl::opt PipelineStartEPPipeline( "passes-ep-pipeline-start", cl::desc("A textual description of the module pass pipeline inserted at " @@ -285,6 +290,12 @@ static void registerEPCallbacks(PassBuilder &PB) { ExitOnError Err("Unable to parse VectorizerStartEP pipeline: "); Err(PB.parsePassPipeline(PM, VectorizerStartEPPipeline)); }); + if (tryParsePipelineText(PB, VectorizerEndEPPipeline)) + PB.registerVectorizerEndEPCallback( + [&PB](FunctionPassManager &PM, OptimizationLevel Level) { + ExitOnError Err("Unable to parse VectorizerEndEP pipeline: "); + Err(PB.parsePassPipeline(PM, VectorizerEndEPPipeline)); + }); if (tryParsePipelineText(PB, PipelineStartEPPipeline)) PB.registerPipelineStartEPCallback( [&PB](ModulePassManager &PM, OptimizationLevel) {