Skip to content

feat: fuzzer metrics #10988

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

feat: fuzzer metrics #10988

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
149 changes: 100 additions & 49 deletions crates/evm/evm/src/executors/invariant/corpus.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ use uuid::Uuid;

const METADATA_SUFFIX: &str = "metadata.json";
const JSON_EXTENSION: &str = ".json";
const FAVORABILITY_THRESHOLD: f64 = 0.3;

/// Possible mutation strategies to apply on a call sequence.
#[derive(Debug, Clone)]
Expand All @@ -45,7 +46,7 @@ enum MutationType {

/// Holds Corpus information.
#[derive(Serialize)]
struct Corpus {
struct CorpusEntry {
// Unique corpus identifier.
uuid: Uuid,
// Total mutations of corpus as primary source.
Expand All @@ -55,22 +56,31 @@ struct Corpus {
// Corpus call sequence.
#[serde(skip_serializing)]
tx_seq: Vec<BasicTxDetails>,
// Whether this corpus is favored, i.e. producing new finds more often than
// `FAVORABILITY_THRESHOLD`.
is_favored: bool,
}

impl Corpus {
impl CorpusEntry {
/// New corpus from given call sequence and corpus path to read uuid.
pub fn new(tx_seq: Vec<BasicTxDetails>, path: PathBuf) -> eyre::Result<Self> {
let uuid = if let Some(stem) = path.file_stem().and_then(|s| s.to_str()) {
Uuid::try_from(stem.strip_suffix(JSON_EXTENSION).unwrap_or(stem).to_string())?
} else {
Uuid::new_v4()
};
Ok(Self { uuid, total_mutations: 0, new_finds_produced: 0, tx_seq })
Ok(Self { uuid, total_mutations: 0, new_finds_produced: 0, tx_seq, is_favored: false })
}

/// New corpus with given call sequence and new uuid.
pub fn from_tx_seq(tx_seq: Vec<BasicTxDetails>) -> Self {
Self { uuid: Uuid::new_v4(), total_mutations: 0, new_finds_produced: 0, tx_seq }
Self {
uuid: Uuid::new_v4(),
total_mutations: 0,
new_finds_produced: 0,
tx_seq,
is_favored: false,
}
}
}

Expand All @@ -81,7 +91,7 @@ pub struct TxCorpusManager {
// Call sequence mutation strategy type generator.
mutation_generator: BoxedStrategy<MutationType>,
// Path to invariant corpus directory. If None, sequences with new coverage are not persisted.
corpus_dir: Option<PathBuf>,
corpus_dir: Option<PathBuf>, // TODO consolidate into config
// Whether corpus to use gzip file compression and decompression.
corpus_gzip: bool,
// Number of mutations until entry marked as eligible to be flushed from in-memory corpus.
Expand All @@ -91,11 +101,19 @@ pub struct TxCorpusManager {
corpus_min_size: usize,
// In-memory corpus, populated from persisted files and current runs.
// Mutation is performed on these.
in_memory_corpus: Vec<Corpus>,
in_memory_corpus: Vec<CorpusEntry>,
// Identifier of current mutated entry.
current_mutated: Option<Uuid>,
// Number of failed replays from persisted corpus.
failed_replays: usize,
// Number of edges seen during the invariant run.
pub(crate) cumulative_edges_seen: usize,
// Number of features (new hitcount bin of previously hit edge) seen during the invariant run.
pub(crate) cumulative_features_seen: usize,
// Nnumber of corpus entries.
pub(crate) corpus_count: usize,
// Number of corpus entries that are favored.
pub(crate) favored_items: usize,
}

impl TxCorpusManager {
Expand Down Expand Up @@ -134,6 +152,10 @@ impl TxCorpusManager {
in_memory_corpus,
current_mutated: None,
failed_replays,
cumulative_edges_seen: 0,
cumulative_features_seen: 0,
corpus_count: 0,
favored_items: 0,
});
};

Expand All @@ -145,6 +167,10 @@ impl TxCorpusManager {

let fuzzed_contracts = fuzzed_contracts.targets.lock();

let mut cumulative_edges_seen = 0;
let mut cumulative_features_seen = 0;
let mut corpus_count = 0;

for entry in std::fs::read_dir(&corpus_dir)? {
let path = entry?.path();
if path.is_file()
Expand All @@ -155,6 +181,7 @@ impl TxCorpusManager {
continue;
}
}
corpus_count += 1;

let read_corpus_result = match path.extension().and_then(|ext| ext.to_str()) {
Some("gz") => foundry_common::fs::read_json_gzip_file::<Vec<BasicTxDetails>>(&path),
Expand All @@ -180,7 +207,15 @@ impl TxCorpusManager {
.map_err(|e| eyre!(format!("Could not make raw evm call: {e}")))?;

if fuzzed_contracts.can_replay(tx) {
call_result.merge_edge_coverage(history_map);
let (new_coverage, is_edge) = call_result.merge_edge_coverage(history_map);
if new_coverage {
if is_edge {
cumulative_edges_seen += 1;
} else {
cumulative_features_seen += 1;
}
}

executor.commit(&mut call_result);
} else {
failed_replays += 1;
Expand All @@ -195,7 +230,7 @@ impl TxCorpusManager {
);

// Populate in memory corpus with sequence from corpus file.
in_memory_corpus.push(Corpus::new(tx_seq, path)?);
in_memory_corpus.push(CorpusEntry::new(tx_seq, path)?);
}
}

Expand All @@ -209,6 +244,10 @@ impl TxCorpusManager {
in_memory_corpus,
current_mutated: None,
failed_replays,
cumulative_edges_seen,
cumulative_features_seen,
favored_items: 0,
corpus_count,
})
}

Expand All @@ -229,6 +268,14 @@ impl TxCorpusManager {
if test_run.new_coverage {
corpus.new_finds_produced += 1
}
let is_favored = (corpus.new_finds_produced as f64 / corpus.total_mutations as f64)
< FAVORABILITY_THRESHOLD;
if is_favored && !corpus.is_favored {
self.favored_items += 1;
} else if !is_favored && corpus.is_favored {
self.favored_items -= 1;
}
corpus.is_favored = is_favored;

trace!(
target: "corpus",
Expand All @@ -245,7 +292,7 @@ impl TxCorpusManager {
return;
}

let corpus = Corpus::from_tx_seq(test_run.inputs.clone());
let corpus = CorpusEntry::from_tx_seq(test_run.inputs.clone());
let corpus_uuid = corpus.uuid;

// Persist to disk if corpus dir is configured.
Expand Down Expand Up @@ -273,6 +320,7 @@ impl TxCorpusManager {

// This includes reverting txs in the corpus and `can_continue` removes
// them. We want this as it is new coverage and may help reach the other branch.
self.corpus_count += 1;
self.in_memory_corpus.push(corpus);
}

Expand All @@ -291,29 +339,32 @@ impl TxCorpusManager {

if !self.in_memory_corpus.is_empty() {
// Flush oldest corpus mutated more than configured max mutations unless they are
// producing new finds more than 1/3 of the time.
// favored.
let should_evict = self.in_memory_corpus.len() > self.corpus_min_size.max(1);
if should_evict
&& let Some(index) = self.in_memory_corpus.iter().position(|corpus| {
corpus.total_mutations > self.corpus_min_mutations
&& (corpus.new_finds_produced as f64 / corpus.total_mutations as f64) < 0.3
})
{
let corpus = self.in_memory_corpus.get(index).unwrap();
let uuid = corpus.uuid;
debug!(target: "corpus", "evict corpus {uuid}");

// Flush to disk the seed metadata at the time of eviction.
let eviction_time = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Time went backwards")
.as_secs();
foundry_common::fs::write_json_file(
corpus_dir.join(format!("{uuid}-{eviction_time}-{METADATA_SUFFIX}")).as_path(),
&corpus,
)?;
// Remove corpus from memory.
self.in_memory_corpus.remove(index);
if should_evict {
if let Some(index) = self.in_memory_corpus.iter().position(|corpus| {
corpus.total_mutations > self.corpus_min_mutations && !corpus.is_favored
}) {
let corpus = self.in_memory_corpus.get(index).unwrap();

let uuid = corpus.uuid;
debug!(target: "corpus", "evict corpus {uuid}");

// Flush to disk the seed metadata at the time of eviction.
let eviction_time = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Time went backwards")
.as_secs();
foundry_common::fs::write_json_file(
corpus_dir
.join(format!("{uuid}-{eviction_time}-{METADATA_SUFFIX}"))
.as_path(),
&corpus,
)?;

// Remove corpus from memory.
self.in_memory_corpus.remove(index);
}
}

let mutation_type = self
Expand All @@ -329,9 +380,9 @@ impl TxCorpusManager {
match mutation_type {
MutationType::Splice => {
trace!(target: "corpus", "splice {} and {}", primary.uuid, secondary.uuid);
if should_evict {
self.current_mutated = Some(primary.uuid);
}

self.current_mutated = Some(primary.uuid);

let start1 = rng.random_range(0..primary.tx_seq.len());
let end1 = rng.random_range(start1..primary.tx_seq.len());

Expand All @@ -348,9 +399,9 @@ impl TxCorpusManager {
MutationType::Repeat => {
let corpus = if rng.random::<bool>() { primary } else { secondary };
trace!(target: "corpus", "repeat {}", corpus.uuid);
if should_evict {
self.current_mutated = Some(corpus.uuid);
}

self.current_mutated = Some(corpus.uuid);

new_seq = corpus.tx_seq.clone();
let start = rng.random_range(0..corpus.tx_seq.len());
let end = rng.random_range(start..corpus.tx_seq.len());
Expand All @@ -360,9 +411,9 @@ impl TxCorpusManager {
}
MutationType::Interleave => {
trace!(target: "corpus", "interleave {} with {}", primary.uuid, secondary.uuid);
if should_evict {
self.current_mutated = Some(primary.uuid);
}

self.current_mutated = Some(primary.uuid);

for (tx1, tx2) in primary.tx_seq.iter().zip(secondary.tx_seq.iter()) {
// chunks?
let tx = if rng.random::<bool>() { tx1.clone() } else { tx2.clone() };
Expand All @@ -372,9 +423,9 @@ impl TxCorpusManager {
MutationType::Prefix => {
let corpus = if rng.random::<bool>() { primary } else { secondary };
trace!(target: "corpus", "overwrite prefix of {}", corpus.uuid);
if should_evict {
self.current_mutated = Some(corpus.uuid);
}

self.current_mutated = Some(corpus.uuid);

new_seq = corpus.tx_seq.clone();
for i in 0..rng.random_range(0..=new_seq.len()) {
new_seq[i] = self.new_tx(test_runner)?;
Expand All @@ -383,9 +434,9 @@ impl TxCorpusManager {
MutationType::Suffix => {
let corpus = if rng.random::<bool>() { primary } else { secondary };
trace!(target: "corpus", "overwrite suffix of {}", corpus.uuid);
if should_evict {
self.current_mutated = Some(corpus.uuid);
}

self.current_mutated = Some(corpus.uuid);

new_seq = corpus.tx_seq.clone();
for i in new_seq.len() - rng.random_range(0..new_seq.len())..corpus.tx_seq.len()
{
Expand All @@ -396,9 +447,9 @@ impl TxCorpusManager {
let targets = test.targeted_contracts.targets.lock();
let corpus = if rng.random::<bool>() { primary } else { secondary };
trace!(target: "corpus", "ABI mutate args of {}", corpus.uuid);
if should_evict {
self.current_mutated = Some(corpus.uuid);
}

self.current_mutated = Some(corpus.uuid);

new_seq = corpus.tx_seq.clone();

let idx = rng.random_range(0..new_seq.len());
Expand Down
35 changes: 31 additions & 4 deletions crates/evm/evm/src/executors/invariant/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ use std::{
cell::RefCell,
collections::{HashMap as Map, btree_map::Entry},
sync::Arc,
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
};

mod error;
Expand Down Expand Up @@ -343,6 +344,8 @@ impl<'a> InvariantExecutor<'a> {
// Start timer for this invariant test.
let mut runs = 0;
let timer = FuzzTestTimer::new(self.config.timeout);
const DURATION_BETWEEN_METRICS_REPORT: Duration = Duration::from_secs(5);
let mut last_metrics_report = Instant::now();
let continue_campaign = |runs: u32| {
// If timeout is configured, then perform invariant runs until expires.
if self.config.timeout.is_some() {
Expand Down Expand Up @@ -404,10 +407,17 @@ impl<'a> InvariantExecutor<'a> {
invariant_test.merge_coverage(call_result.line_coverage.clone());
// If coverage guided fuzzing is enabled then merge edge count with current history
// map and set new coverage in current run.
if self.config.corpus_dir.is_some()
&& call_result.merge_edge_coverage(&mut self.history_map)
{
current_run.new_coverage = true;
if self.config.corpus_dir.is_some() {
let (new_coverage, is_edge) =
call_result.merge_edge_coverage(&mut self.history_map);
if new_coverage {
current_run.new_coverage = true;
if is_edge {
corpus_manager.cumulative_edges_seen += 1;
} else {
corpus_manager.cumulative_features_seen += 1;
}
}
}

if discarded {
Expand Down Expand Up @@ -513,6 +523,23 @@ impl<'a> InvariantExecutor<'a> {
}

runs += 1;

if last_metrics_report.elapsed() > DURATION_BETWEEN_METRICS_REPORT {
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
let corpus_count = corpus_manager.corpus_count;
let cumulative_edges_seen = corpus_manager.cumulative_edges_seen;
let cumulative_features_seen = corpus_manager.cumulative_features_seen;
let favored_items = corpus_manager.favored_items;
println!(
"{{\"timestamp\":{},\"cumulative_edges_seen\":{},\"cumulative_features_seen\":{},\"corpus_count\":{},\"favored_items\":{}}}",
now,
cumulative_edges_seen,
cumulative_features_seen,
corpus_count,
favored_items
);
last_metrics_report = Instant::now();
}
}

trace!(?fuzz_fixtures);
Expand Down
Loading
Loading