Skip to content

[SPARK-53077][CORE][TESTS][FOLLOWUP] Reduce insertion count in SparkBloomFilterSuite #51845

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[SPARK-53077][CORE][TESTS][FOLLOWUP] Reduce insertion count in SparkBloomFilterSuite #51845

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions common/sketch/src/test/java/org/apache/spark/util/sketch/SparkBloomFilterSuite.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,7 @@ public class SparkBloomFilterSuite {

// the implemented fpp limit is only approximating the hard boundary,
// so we'll need an error threshold for the assertion
final double FPP_EVEN_ODD_ERROR_FACTOR = 0.10;
final double FPP_RANDOM_ERROR_FACTOR = 0.10;
final double FPP_ACCEPTABLE_ERROR_FACTOR = 0.10;

final long ONE_GB = 1024L * 1024L * 1024L;
final long REQUIRED_HEAP_UPPER_BOUND_IN_BYTES = 4 * ONE_GB;
Expand Down Expand Up @@ -106,7 +105,7 @@ private static Stream<Arguments> dataPointProvider() {
// to reduce running time to acceptable levels, we test only one case,
// with the default FPP and the default seed only.
return Stream.of(
Arguments.of(1_000_000_000L, 0.03, BloomFilterImplV2.DEFAULT_SEED)
Arguments.of(350_000_000L, 0.03, BloomFilterImplV2.DEFAULT_SEED)
);
// preferable minimum parameter space for tests:
// {1_000_000L, 1_000_000_000L} for: long numItems
Expand Down Expand Up @@ -201,7 +200,7 @@ public void testAccuracyEvenOdd(
);

double actualFpp = mightContainOdd.doubleValue() / numItems;
double acceptableFpp = expectedFpp * (1 + FPP_EVEN_ODD_ERROR_FACTOR);
double acceptableFpp = expectedFpp * (1 + FPP_ACCEPTABLE_ERROR_FACTOR);

testOut.printf("expectedFpp: %f %%\n", 100 * expectedFpp);
testOut.printf("acceptableFpp: %f %%\n", 100 * acceptableFpp);
Expand Down Expand Up @@ -279,6 +278,7 @@ public void testAccuracyRandomDistribution(
deterministicSeed
);

// V1 ignores custom seed values, so the control filter must be at least V2
BloomFilter bloomFilterSecondary =
BloomFilter.create(
BloomFilter.Version.V2,
Expand Down Expand Up @@ -354,7 +354,7 @@ public void testAccuracyRandomDistribution(

double actualFpp =
mightContainOddIndexed.doubleValue() / confirmedAsNotInserted.doubleValue();
double acceptableFpp = expectedFpp * (1 + FPP_RANDOM_ERROR_FACTOR);
double acceptableFpp = expectedFpp * (1 + FPP_ACCEPTABLE_ERROR_FACTOR);

testOut.printf("mightContainOddIndexed: %10d\n", mightContainOddIndexed.longValue());
testOut.printf("confirmedAsNotInserted: %10d\n", confirmedAsNotInserted.longValue());
Expand Down