Benchmarks Writer CLI (#6567)

* initial mockup

* add and wipe

* track writes

* start to add to pipeline

* return all reads/writes

* Log reads and writes from bench db

* causes panic

* Allow multiple commits

* commit before ending benchmark

* doesn't work???

* fix

* Update lib.rs

* switch to struct for `BenchmarkResults`

* add to output

* fix test

* line width

* @kianenigma review

* Add Whitelist to DB Tracking in Benchmarks Pipeline (#6405)

* hardcoded whitelist

* Add whitelist to pipeline

* Remove whitelist pipeline from CLI, add to runtime

* clean-up unused db initialized whitelist

* Add regression analysis to DB Tracking (#6475)

* Add selector

* add tests

* debug formatter for easy formula

* initial idea

* use all benchmarks

* broken

* working without trait

* Make work for multiple pallets

* Fix merge issues

* writer appends to file

* implement () for balances weight trait

* update name of trait

* Weights to WeightInfo

* auto trait writer

* Heap pages are configurable

* clean out runtime changes

* more clean up

* Fix string generation

* Update comments

* Update bin/node/runtime/src/lib.rs

Co-authored-by: arkpar <[email protected]>

Author: shawntabrizi

Cargo.lock

@@ -1118,7 +1118,6 @@ impl_runtime_apis! {
 
 			let whitelist: Vec<Vec<u8>> = vec![
 				// Block Number
-				// frame_system::Number::<Runtime>::hashed_key().to_vec(),
 				hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef702a5c1b19ab7a04f536c519aca4983ac").to_vec(),
 				// Total Issuance
 				hex_literal::hex!("c2261276cc9d1f8598ea4b6a74b15c2f57c875e4cff74148e4628f264b974c80").to_vec(),
@@ -1137,25 +1136,25 @@ impl_runtime_apis! {
 			let mut batches = Vec::<BenchmarkBatch>::new();
 			let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat, &whitelist);
 
-			add_benchmark!(params, batches, b"babe", Babe);
-			add_benchmark!(params, batches, b"balances", Balances);
-			add_benchmark!(params, batches, b"collective", Council);
-			add_benchmark!(params, batches, b"democracy", Democracy);
-			add_benchmark!(params, batches, b"elections", Elections);
-			add_benchmark!(params, batches, b"identity", Identity);
-			add_benchmark!(params, batches, b"im-online", ImOnline);
-			add_benchmark!(params, batches, b"indices", Indices);
-			add_benchmark!(params, batches, b"multisig", Multisig);
-			add_benchmark!(params, batches, b"offences", OffencesBench::<Runtime>);
-			add_benchmark!(params, batches, b"proxy", Proxy);
-			add_benchmark!(params, batches, b"scheduler", Scheduler);
-			add_benchmark!(params, batches, b"session", SessionBench::<Runtime>);
-			add_benchmark!(params, batches, b"staking", Staking);
-			add_benchmark!(params, batches, b"system", SystemBench::<Runtime>);
-			add_benchmark!(params, batches, b"timestamp", Timestamp);
-			add_benchmark!(params, batches, b"treasury", Treasury);
-			add_benchmark!(params, batches, b"utility", Utility);
-			add_benchmark!(params, batches, b"vesting", Vesting);
+			add_benchmark!(params, batches, pallet_babe, Babe);
+			add_benchmark!(params, batches, pallet_balances, Balances);
+			add_benchmark!(params, batches, pallet_collective, Council);
+			add_benchmark!(params, batches, pallet_democracy, Democracy);
+			add_benchmark!(params, batches, pallet_elections_phragmen, Elections);
+			add_benchmark!(params, batches, pallet_identity, Identity);
+			add_benchmark!(params, batches, pallet_im_online, ImOnline);
+			add_benchmark!(params, batches, pallet_indices, Indices);
+			add_benchmark!(params, batches, pallet_multisig, Multisig);
+			add_benchmark!(params, batches, pallet_offences, OffencesBench::<Runtime>);
+			add_benchmark!(params, batches, pallet_proxy, Proxy);
+			add_benchmark!(params, batches, pallet_scheduler, Scheduler);
+			add_benchmark!(params, batches, pallet_session, SessionBench::<Runtime>);
+			add_benchmark!(params, batches, pallet_staking, Staking);
+			add_benchmark!(params, batches, frame_system, SystemBench::<Runtime>);
+			add_benchmark!(params, batches, pallet_timestamp, Timestamp);
+			add_benchmark!(params, batches, pallet_treasury, Treasury);
+			add_benchmark!(params, batches, pallet_utility, Utility);
+			add_benchmark!(params, batches, pallet_vesting, Vesting);
 
 			if batches.is_empty() { return Err("Benchmark not found for this pallet.".into()) }
 			Ok(batches)

bin/node/runtime/src/lib.rs

@@ -22,11 +22,11 @@ use linregress::{FormulaRegressionBuilder, RegressionDataBuilder, RegressionMode
 use crate::BenchmarkResults;
 
 pub struct Analysis {
-	base: u128,
-	slopes: Vec<u128>,
-	names: Vec<String>,
-	value_dists: Option<Vec<(Vec<u32>, u128, u128)>>,
-	model: Option<RegressionModel>,
+	pub base: u128,
+	pub slopes: Vec<u128>,
+	pub names: Vec<String>,
+	pub value_dists: Option<Vec<(Vec<u32>, u128, u128)>>,
+	pub model: Option<RegressionModel>,
 }
 
 pub enum BenchmarkSelector {

frame/benchmarking/src/analysis.rs

@@ -1158,31 +1158,46 @@ macro_rules! impl_benchmark_test {
 /// First create an object that holds in the input parameters for the benchmark:
 ///
 /// ```ignore
-/// let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat);
+/// let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat, &whitelist);
 /// ```
 ///
+/// The `whitelist` is a `Vec<Vec<u8>>` of storage keys that you would like to skip for DB tracking. For example:
+///
+/// ```ignore
+/// let whitelist: Vec<Vec<u8>> = vec![
+/// 	// Block Number
+/// 	hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef702a5c1b19ab7a04f536c519aca4983ac").to_vec(),
+/// 	// Total Issuance
+/// 	hex_literal::hex!("c2261276cc9d1f8598ea4b6a74b15c2f57c875e4cff74148e4628f264b974c80").to_vec(),
+/// 	// Execution Phase
+/// 	hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef7ff553b5a9862a516939d82b3d3d8661a").to_vec(),
+/// 	// Event Count
+/// 	hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef70a98fdbe9ce6c55837576c60c7af3850").to_vec(),
+/// ];
+///
 /// Then define a mutable local variable to hold your `BenchmarkBatch` object:
 ///
 /// ```ignore
 /// let mut batches = Vec::<BenchmarkBatch>::new();
 /// ````
 ///
-/// Then add the pallets you want to benchmark to this object, including the string
-/// you want to use target a particular pallet:
+/// Then add the pallets you want to benchmark to this object, using their crate name and generated
+/// module struct:
 ///
 /// ```ignore
-/// add_benchmark!(params, batches, b"balances", Balances);
-/// add_benchmark!(params, batches, b"identity", Identity);
-/// add_benchmark!(params, batches, b"session", SessionBench::<Runtime>);
+/// add_benchmark!(params, batches, pallet_balances, Balances);
+/// add_benchmark!(params, batches, pallet_session, SessionBench::<Runtime>);
+/// add_benchmark!(params, batches, frame_system, SystemBench::<Runtime>);
 /// ...
 /// ```
 ///
 /// At the end of `dispatch_benchmark`, you should return this batches object.
 #[macro_export]
 macro_rules! add_benchmark {
-	( $params:ident, $batches:ident, $name:literal, $( $location:tt )* ) => (
+	( $params:ident, $batches:ident, $name:ident, $( $location:tt )* ) => (
+		let name_string = stringify!($name).as_bytes();
 		let (pallet, benchmark, lowest_range_values, highest_range_values, steps, repeat, whitelist) = $params;
-		if &pallet[..] == &$name[..] || &pallet[..] == &b"*"[..] {
+		if &pallet[..] == &name_string[..] || &pallet[..] == &b"*"[..] {
 			if &pallet[..] == &b"*"[..] || &benchmark[..] == &b"*"[..] {
 				for benchmark in $( $location )*::benchmarks().into_iter() {
 					$batches.push($crate::BenchmarkBatch {
@@ -1194,7 +1209,7 @@ macro_rules! add_benchmark {
 							repeat,
 							whitelist,
 						)?,
-						pallet: $name.to_vec(),
+						pallet: name_string.to_vec(),
 						benchmark: benchmark.to_vec(),
 					});
 				}
@@ -1208,7 +1223,7 @@ macro_rules! add_benchmark {
 						repeat,
 						whitelist,
 					)?,
-					pallet: $name.to_vec(),
+					pallet: name_string.to_vec(),
 					benchmark: benchmark.clone(),
 				});
 			}

frame/benchmarking/src/lib.rs

@@ -12,6 +12,7 @@ description = "CLI for benchmarking FRAME"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
+Inflector = "0.11.4"
 frame-benchmarking = { version = "2.0.0-rc4", path = "../../../frame/benchmarking" }
 sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" }
 sc-service = { version = "0.8.0-rc4", default-features = false, path = "../../../client/service" }

utils/frame/benchmarking-cli/Cargo.toml

@@ -55,7 +55,7 @@ impl BenchmarkCmd {
 		let state = BenchmarkingState::<BB>::new(genesis_storage, cache_size)?;
 		let executor = NativeExecutor::<ExecDispatch>::new(
 			wasm_method,
-			None, // heap pages
+			self.heap_pages,
 			2, // The runtime instances cache size.
 		);
 
@@ -89,6 +89,16 @@ impl BenchmarkCmd {
 		let results = <std::result::Result<Vec<BenchmarkBatch>, String> as Decode>::decode(&mut &result[..])
 			.map_err(|e| format!("Failed to decode benchmark results: {:?}", e))?;
 
+		if self.output {
+			if self.weight_trait {
+				let mut file = crate::writer::open_file("traits.rs")?;
+				crate::writer::write_trait(&mut file, results.clone())?;
+			} else {
+				let mut file = crate::writer::open_file("benchmarks.rs")?;
+				crate::writer::write_results(&mut file, results.clone())?;
+			}
+		}
+
 		match results {
 			Ok(batches) => for batch in batches.into_iter() {
 				// Print benchmark metadata

utils/frame/benchmarking-cli/src/command.rs

@@ -16,6 +16,7 @@
 // limitations under the License.
 
 mod command;
+mod writer;
 
 use sc_cli::{ExecutionStrategy, WasmExecutionMethod};
 use std::fmt::Debug;
@@ -59,6 +60,18 @@ pub struct BenchmarkCmd {
 	#[structopt(long)]
 	pub no_min_squares: bool,
 
+	/// Output the benchmarks to a Rust file.
+	#[structopt(long)]
+	pub output: bool,
+
+	/// Output the trait definition to a Rust file.
+	#[structopt(long)]
+	pub weight_trait: bool,
+
+	/// Set the heap pages while running benchmarks.
+	#[structopt(long)]
+	pub heap_pages: Option<u64>,
+
 	#[allow(missing_docs)]
 	#[structopt(flatten)]
 	pub shared_params: sc_cli::SharedParams,