From 969f2ed8e9baf2ebb45fee285eb45dd13476fa4d Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Thu, 23 Feb 2023 17:33:31 -0800 Subject: [PATCH 01/32] add unchanged names to existence filter and modify spec test runner --- .../Example/Tests/SpecTests/FSTSpecTests.mm | 35 +++++- .../json/existence_filter_spec_test.json | 114 ++++++++++-------- .../Tests/SpecTests/json/limbo_spec_test.json | 24 ++-- .../Tests/SpecTests/json/limit_spec_test.json | 12 +- Firestore/core/src/remote/existence_filter.h | 18 ++- Firestore/core/src/remote/serializer.cc | 2 +- 6 files changed, 137 insertions(+), 68 deletions(-) diff --git a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm index c6c5ba015c9..9cabc78bb41 100644 --- a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm +++ b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm @@ -70,12 +70,16 @@ #include "Firestore/core/src/util/string_apple.h" #include "Firestore/core/src/util/to_string.h" #include "Firestore/core/test/unit/testutil/testutil.h" + #include "absl/memory/memory.h" +#include "absl/strings/escaping.h" #include "absl/types/optional.h" namespace objc = firebase::firestore::objc; using firebase::firestore::Error; using firebase::firestore::google_firestore_v1_ArrayValue; +using firebase::firestore::google_firestore_v1_BitSequence; +using firebase::firestore::google_firestore_v1_BloomFilter; using firebase::firestore::google_firestore_v1_Value; using firebase::firestore::api::LoadBundleTask; using firebase::firestore::bundle::BundleReader; @@ -325,6 +329,27 @@ - (SnapshotVersion)parseVersion:(NSNumber *_Nullable)version { return Version(version.longLongValue); } +- (google_firestore_v1_BloomFilter)parseBloomFilter:(NSDictionary *_Nullable)bloomFilterProto { + NSDictionary *bitsData = bloomFilterProto[@"bits"]; + NSString *bitmapData = bitsData[@"bitmap"]; + + // Decode base64 json string: bitmap + std::string bitmap; + absl::Base64Unescape([bitmapData UTF8String], &bitmap); + + int32_t padding = [bitsData[@"padding"] intValue]; + int32_t hashCount = [bloomFilterProto[@"hashCount"] intValue]; + + // Parse data into bloom filter proto type + google_firestore_v1_BitSequence bits; + bits.bitmap = nanopb::MakeBytesArray(bitmap); + bits.padding = padding; + google_firestore_v1_BloomFilter filter; + filter.hash_count = hashCount; + + return filter; +} + - (DocumentViewChange)parseChange:(NSDictionary *)jsonDoc ofType:(DocumentViewChange::Type)type { NSNumber *version = jsonDoc[@"version"]; NSDictionary *options = jsonDoc[@"options"]; @@ -463,13 +488,15 @@ - (void)doWatchEntity:(NSDictionary *)watchEntity { } } -- (void)doWatchFilter:(NSArray *)watchFilter { - NSArray *targets = watchFilter[0]; +- (void)doWatchFilter:(NSDictionary *)watchFilter { + NSArray *targets = watchFilter[@"targetIds"]; HARD_ASSERT(targets.count == 1, "ExistenceFilters currently support exactly one target only."); - int keyCount = watchFilter.count == 0 ? 0 : (int)watchFilter.count - 1; + NSArray *keys = watchFilter[@"keys"]; + int keyCount = keys ? keys.count : 0; - ExistenceFilter filter{keyCount}; + google_firestore_v1_BloomFilter bloomFilter = [self parseBloomFilter:watchFilter[@"bloomFilter"]]; + ExistenceFilter filter{keyCount, bloomFilter}; ExistenceFilterWatchChange change{filter, targets[0].intValue}; [self.driver receiveWatchChange:change snapshotVersion:SnapshotVersion::None()]; } diff --git a/Firestore/Example/Tests/SpecTests/json/existence_filter_spec_test.json b/Firestore/Example/Tests/SpecTests/json/existence_filter_spec_test.json index 3083b762224..75722e8ebcb 100644 --- a/Firestore/Example/Tests/SpecTests/json/existence_filter_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/existence_filter_spec_test.json @@ -128,12 +128,14 @@ ] }, { - "watchFilter": [ - [ - 2 + "watchFilter": { + "keys": [ + "collection/1" ], - "collection/1" - ] + "targetIds": [ + 2 + ] + } }, { "watchSnapshot": { @@ -341,13 +343,15 @@ ] }, { - "watchFilter": [ - [ - 2 + "watchFilter": { + "keys": [ + "collection/1", + "collection/2" ], - "collection/1", - "collection/2" - ] + "targetIds": [ + 2 + ] + } }, { "watchEntity": { @@ -694,11 +698,13 @@ } }, { - "watchFilter": [ - [ + "watchFilter": { + "keys": [ + ], + "targetIds": [ 2 ] - ] + } }, { "watchRemove": { @@ -871,12 +877,14 @@ ] }, { - "watchFilter": [ - [ - 2 + "watchFilter": { + "keys": [ + "collection/1" ], - "collection/1" - ] + "targetIds": [ + 2 + ] + } }, { "watchSnapshot": { @@ -1156,12 +1164,14 @@ ] }, { - "watchFilter": [ - [ - 2 + "watchFilter": { + "keys": [ + "collection/1" ], - "collection/1" - ] + "targetIds": [ + 2 + ] + } }, { "watchSnapshot": { @@ -1265,12 +1275,14 @@ } }, { - "watchFilter": [ - [ - 2 + "watchFilter": { + "keys": [ + "collection/1" ], - "collection/1" - ] + "targetIds": [ + 2 + ] + } }, { "watchSnapshot": { @@ -1436,12 +1448,14 @@ ] }, { - "watchFilter": [ - [ - 2 + "watchFilter": { + "keys": [ + "collection/1" ], - "collection/1" - ] + "targetIds": [ + 2 + ] + } }, { "watchSnapshot": { @@ -1783,12 +1797,14 @@ ] }, { - "watchFilter": [ - [ - 2 + "watchFilter": { + "keys": [ + "collection/1" ], - "collection/1" - ] + "targetIds": [ + 2 + ] + } }, { "watchSnapshot": { @@ -2078,11 +2094,13 @@ ] }, { - "watchFilter": [ - [ + "watchFilter": { + "keys": [ + ], + "targetIds": [ 2 ] - ] + } }, { "watchSnapshot": { @@ -2193,12 +2211,14 @@ ] }, { - "watchFilter": [ - [ - 2 + "watchFilter": { + "keys": [ + "collection/1" ], - "collection/1" - ] + "targetIds": [ + 2 + ] + } }, { "watchSnapshot": { diff --git a/Firestore/Example/Tests/SpecTests/json/limbo_spec_test.json b/Firestore/Example/Tests/SpecTests/json/limbo_spec_test.json index 0b6abe08a2b..214a9940fc9 100644 --- a/Firestore/Example/Tests/SpecTests/json/limbo_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/limbo_spec_test.json @@ -3386,11 +3386,13 @@ ] }, { - "watchFilter": [ - [ + "watchFilter": { + "keys": [ + ], + "targetIds": [ 1 ] - ] + } }, { "watchCurrent": [ @@ -8036,14 +8038,16 @@ } }, { - "watchFilter": [ - [ - 2 + "watchFilter": { + "keys": [ + "collection/b1", + "collection/b2", + "collection/b3" ], - "collection/b1", - "collection/b2", - "collection/b3" - ] + "targetIds": [ + 2 + ] + } }, { "watchSnapshot": { diff --git a/Firestore/Example/Tests/SpecTests/json/limit_spec_test.json b/Firestore/Example/Tests/SpecTests/json/limit_spec_test.json index 40e48205956..856e0505d91 100644 --- a/Firestore/Example/Tests/SpecTests/json/limit_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/limit_spec_test.json @@ -5455,12 +5455,14 @@ } }, { - "watchFilter": [ - [ - 2 + "watchFilter": { + "keys": [ + "collection/b" ], - "collection/b" - ] + "targetIds": [ + 2 + ] + } }, { "watchSnapshot": { diff --git a/Firestore/core/src/remote/existence_filter.h b/Firestore/core/src/remote/existence_filter.h index 3f4019c8987..7be936912cb 100644 --- a/Firestore/core/src/remote/existence_filter.h +++ b/Firestore/core/src/remote/existence_filter.h @@ -17,6 +17,8 @@ #ifndef FIRESTORE_CORE_SRC_REMOTE_EXISTENCE_FILTER_H_ #define FIRESTORE_CORE_SRC_REMOTE_EXISTENCE_FILTER_H_ +#include "Firestore/Protos/nanopb/google/firestore/v1/bloom_filter.nanopb.h" + namespace firebase { namespace firestore { namespace remote { @@ -27,16 +29,30 @@ class ExistenceFilter { explicit ExistenceFilter(int count) : count_{count} { } + explicit ExistenceFilter(int count, + google_firestore_v1_BloomFilter unchangedNames) + : count_{count}, unchanged_names_{unchangedNames} { + } + int count() const { return count_; } + google_firestore_v1_BloomFilter unchanged_names() const { + return unchanged_names_; + } + private: int count_ = 0; + google_firestore_v1_BloomFilter unchanged_names_; }; inline bool operator==(const ExistenceFilter& lhs, const ExistenceFilter& rhs) { - return lhs.count() == rhs.count(); + return lhs.count() == rhs.count() && + lhs.unchanged_names().hash_count == rhs.unchanged_names().hash_count && + lhs.unchanged_names().bits.padding == + rhs.unchanged_names().bits.padding && + lhs.unchanged_names().bits.bitmap == rhs.unchanged_names().bits.bitmap; } } // namespace remote diff --git a/Firestore/core/src/remote/serializer.cc b/Firestore/core/src/remote/serializer.cc index 402197739b9..dea5503347b 100644 --- a/Firestore/core/src/remote/serializer.cc +++ b/Firestore/core/src/remote/serializer.cc @@ -1425,7 +1425,7 @@ std::unique_ptr Serializer::DecodeDocumentRemove( std::unique_ptr Serializer::DecodeExistenceFilterWatchChange( ReadContext*, const google_firestore_v1_ExistenceFilter& filter) const { - ExistenceFilter existence_filter{filter.count}; + ExistenceFilter existence_filter{filter.count, filter.unchanged_names}; return absl::make_unique(existence_filter, filter.target_id); } From 1df9410ba9bb8351c4343b915319c78a804d011b Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Tue, 28 Feb 2023 12:24:56 -0800 Subject: [PATCH 02/32] use BloomFilter instead of proto --- .../Firestore.xcodeproj/project.pbxproj | 16 ++++++++++++++++ .../Example/Tests/SpecTests/FSTSpecTests.mm | 17 ++++++----------- Firestore/core/src/remote/bloom_filter.cc | 6 ++++++ Firestore/core/src/remote/bloom_filter.h | 14 +++++++++++++- Firestore/core/src/remote/existence_filter.h | 18 ++++++------------ Firestore/core/src/remote/serializer.cc | 8 +++++--- Firestore/core/src/remote/watch_change.h | 2 +- .../core/test/unit/local/local_store_test.cc | 2 +- .../core/test/unit/remote/remote_event_test.cc | 5 +++-- .../core/test/unit/remote/serializer_test.cc | 2 +- .../core/test/unit/remote/watch_change_test.cc | 2 +- 11 files changed, 59 insertions(+), 33 deletions(-) diff --git a/Firestore/Example/Firestore.xcodeproj/project.pbxproj b/Firestore/Example/Firestore.xcodeproj/project.pbxproj index cae3f9fb903..14a56e7864e 100644 --- a/Firestore/Example/Firestore.xcodeproj/project.pbxproj +++ b/Firestore/Example/Firestore.xcodeproj/project.pbxproj @@ -342,6 +342,7 @@ 3FFFC1FE083D8BE9C4D9A148 /* string_util_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = AB380CFC201A2EE200D97691 /* string_util_test.cc */; }; 40431BF2A368D0C891229F6E /* FSTMemorySpecTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E02F20213FFC00B64F25 /* FSTMemorySpecTests.mm */; }; 409C0F2BFC2E1BECFFAC4D32 /* testutil.cc in Sources */ = {isa = PBXBuildFile; fileRef = 54A0352820A3B3BD003E0143 /* testutil.cc */; }; + 416B653E930A609FA121C5E3 /* md5_testing.cc in Sources */ = {isa = PBXBuildFile; fileRef = 651321349CA7230B0E4377DA /* md5_testing.cc */; }; 4173B61CB74EB4CD1D89EE68 /* latlng.pb.cc in Sources */ = {isa = PBXBuildFile; fileRef = 618BBE9220B89AAC00B5BCE7 /* latlng.pb.cc */; }; 4194B7BB8B0352E1AC5D69B9 /* precondition_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 549CCA5520A36E1F00BCEB75 /* precondition_test.cc */; }; 41EAC526C543064B8F3F7EDA /* field_path_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = B686F2AD2023DDB20028D6BE /* field_path_test.cc */; }; @@ -445,6 +446,7 @@ 544129DD21C2DDC800EFB9CC /* document.pb.cc in Sources */ = {isa = PBXBuildFile; fileRef = 544129D821C2DDC800EFB9CC /* document.pb.cc */; }; 544129DE21C2DDC800EFB9CC /* write.pb.cc in Sources */ = {isa = PBXBuildFile; fileRef = 544129D921C2DDC800EFB9CC /* write.pb.cc */; }; 54511E8E209805F8005BD28F /* hashing_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 54511E8D209805F8005BD28F /* hashing_test.cc */; }; + 545D42F58BF326180B845350 /* md5_testing.cc in Sources */ = {isa = PBXBuildFile; fileRef = 651321349CA7230B0E4377DA /* md5_testing.cc */; }; 5467FB01203E5717009C9584 /* FIRFirestoreTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5467FAFF203E56F8009C9584 /* FIRFirestoreTests.mm */; }; 5467FB08203E6A44009C9584 /* app_testing.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5467FB07203E6A44009C9584 /* app_testing.mm */; }; 546877D52248206A005E3DE0 /* collection_spec_test.json in Resources */ = {isa = PBXBuildFile; fileRef = 54DA129C1F315EE100DD57A1 /* collection_spec_test.json */; }; @@ -883,6 +885,7 @@ 939C898FE9D129F6A2EA259C /* FSTHelpers.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E03A2021401F00B64F25 /* FSTHelpers.mm */; }; 93C8F772F4DC5A985FA3D815 /* task_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 899FC22684B0F7BEEAE13527 /* task_test.cc */; }; 93E5620E3884A431A14500B0 /* document_key_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = B6152AD5202A5385000E5744 /* document_key_test.cc */; }; + 9484B1C8E9447D397C54B100 /* md5_testing.cc in Sources */ = {isa = PBXBuildFile; fileRef = 651321349CA7230B0E4377DA /* md5_testing.cc */; }; 94854FAEAEA75A1AC77A0515 /* memory_bundle_cache_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = AB4AB1388538CD3CB19EB028 /* memory_bundle_cache_test.cc */; }; 94BBB23B93E449D03FA34F87 /* mutation_queue_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 3068AA9DFBBA86C1FE2A946E /* mutation_queue_test.cc */; }; 95C0F55813DA51E6B8C439E1 /* status_apple_test.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5493A423225F9990006DE7BA /* status_apple_test.mm */; }; @@ -977,6 +980,7 @@ AB38D93020236E21000A432D /* database_info_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = AB38D92E20235D22000A432D /* database_info_test.cc */; }; AB6B908420322E4D00CC290A /* document_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = AB6B908320322E4D00CC290A /* document_test.cc */; }; AB6D588EB21A2C8D40CEB408 /* byte_stream_cpp_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 01D10113ECC5B446DB35E96D /* byte_stream_cpp_test.cc */; }; + AB6FE7D0D8EDD7369E94C0A4 /* md5_testing.cc in Sources */ = {isa = PBXBuildFile; fileRef = 651321349CA7230B0E4377DA /* md5_testing.cc */; }; AB7BAB342012B519001E0872 /* geo_point_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = AB7BAB332012B519001E0872 /* geo_point_test.cc */; }; AB8209455BAA17850D5E196D /* http.pb.cc in Sources */ = {isa = PBXBuildFile; fileRef = 618BBE9720B89AAC00B5BCE7 /* http.pb.cc */; }; AB9FF792C60FC581909EF381 /* recovery_spec_test.json in Resources */ = {isa = PBXBuildFile; fileRef = 9C1AFCC9E616EC33D6E169CF /* recovery_spec_test.json */; }; @@ -1208,6 +1212,7 @@ D98A0B6007E271E32299C79D /* FIRGeoPointTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E048202154AA00B64F25 /* FIRGeoPointTests.mm */; }; D9DA467E7903412DC6AECDE4 /* grpc_connection_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = B6D9649021544D4F00EB9CFB /* grpc_connection_test.cc */; }; D9EF7FC0E3F8646B272B427E /* FSTAPIHelpers.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E04E202154AA00B64F25 /* FSTAPIHelpers.mm */; }; + DA179A5E064DDFE734DC205E /* md5_testing.cc in Sources */ = {isa = PBXBuildFile; fileRef = 651321349CA7230B0E4377DA /* md5_testing.cc */; }; DA1D665B12AA1062DCDEA6BD /* async_queue_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = B6FB467B208E9A8200554BA2 /* async_queue_test.cc */; }; DA4303684707606318E1914D /* target_id_generator_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = AB380CF82019382300D97691 /* target_id_generator_test.cc */; }; DABB9FB61B1733F985CBF713 /* executor_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = B6FB4688208F9B9100554BA2 /* executor_test.cc */; }; @@ -1314,6 +1319,7 @@ ED420D8F49DA5C41EEF93913 /* FIRSnapshotMetadataTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E04D202154AA00B64F25 /* FIRSnapshotMetadataTests.mm */; }; ED4E2AC80CAF2A8FDDAC3DEE /* field_mask_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 549CCA5320A36E1F00BCEB75 /* field_mask_test.cc */; }; ED9DF1EB20025227B38736EC /* message_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = CE37875365497FFA8687B745 /* message_test.cc */; }; + EE3512AC48C1E4B8A190FBB3 /* md5_testing.cc in Sources */ = {isa = PBXBuildFile; fileRef = 651321349CA7230B0E4377DA /* md5_testing.cc */; }; EE470CC3C8FBCDA5F70A8466 /* local_store_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 307FF03D0297024D59348EBD /* local_store_test.cc */; }; EE6DBFB0874A50578CE97A7F /* leveldb_remote_document_cache_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 0840319686A223CC4AD3FAB1 /* leveldb_remote_document_cache_test.cc */; }; EECC1EC64CA963A8376FA55C /* persistence_testing.cc in Sources */ = {isa = PBXBuildFile; fileRef = 9113B6F513D0473AEABBAF1F /* persistence_testing.cc */; }; @@ -1504,6 +1510,7 @@ 403DBF6EFB541DFD01582AA3 /* path_test.cc */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; path = path_test.cc; sourceTree = ""; }; 40F9D09063A07F710811A84F /* value_util_test.cc */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; path = value_util_test.cc; sourceTree = ""; }; 4132F30044D5DF1FB15B2A9D /* fake_credentials_provider.h */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.c.h; path = fake_credentials_provider.h; sourceTree = ""; }; + 4196784E9EDA47C4E68E3A20 /* md5_testing.h */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.c.h; path = md5_testing.h; sourceTree = ""; }; 432C71959255C5DBDF522F52 /* byte_stream_test.cc */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; path = byte_stream_test.cc; sourceTree = ""; }; 4334F87873015E3763954578 /* status_testing.h */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.c.h; path = status_testing.h; sourceTree = ""; }; 444B7AB3F5A2929070CB1363 /* hard_assert_test.cc */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; path = hard_assert_test.cc; sourceTree = ""; }; @@ -1658,6 +1665,7 @@ 62E103B28B48A81D682A0DE9 /* Pods_Firestore_Example_tvOS.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Firestore_Example_tvOS.framework; sourceTree = BUILT_PRODUCTS_DIR; }; 63136A2371C0C013EC7A540C /* target_index_matcher_test.cc */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; path = target_index_matcher_test.cc; sourceTree = ""; }; 64AA92CFA356A2360F3C5646 /* filesystem_testing.h */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.c.h; path = filesystem_testing.h; sourceTree = ""; }; + 651321349CA7230B0E4377DA /* md5_testing.cc */ = {isa = PBXFileReference; includeInIndex = 1; path = md5_testing.cc; sourceTree = ""; }; 69E6C311558EC77729A16CF1 /* Pods-Firestore_Example_iOS-Firestore_SwiftTests_iOS.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Firestore_Example_iOS-Firestore_SwiftTests_iOS.debug.xcconfig"; path = "Pods/Target Support Files/Pods-Firestore_Example_iOS-Firestore_SwiftTests_iOS/Pods-Firestore_Example_iOS-Firestore_SwiftTests_iOS.debug.xcconfig"; sourceTree = ""; }; 6AE927CDFC7A72BF825BE4CB /* Pods-Firestore_Tests_tvOS.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Firestore_Tests_tvOS.release.xcconfig"; path = "Pods/Target Support Files/Pods-Firestore_Tests_tvOS/Pods-Firestore_Tests_tvOS.release.xcconfig"; sourceTree = ""; }; 6E8302DE210222ED003E1EA3 /* FSTFuzzTestFieldPath.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FSTFuzzTestFieldPath.h; sourceTree = ""; }; @@ -2024,6 +2032,8 @@ CD422AF3E4515FB8E9BE67A0 /* equals_tester.h */, BA02DA2FCD0001CFC6EB08DA /* filesystem_testing.cc */, 64AA92CFA356A2360F3C5646 /* filesystem_testing.h */, + 651321349CA7230B0E4377DA /* md5_testing.cc */, + 4196784E9EDA47C4E68E3A20 /* md5_testing.h */, 3CAA33F964042646FDDAF9F9 /* status_testing.cc */, 4334F87873015E3763954578 /* status_testing.h */, 54A0352820A3B3BD003E0143 /* testutil.cc */, @@ -3749,6 +3759,7 @@ 3F6C9F8A993CF4B0CD51E7F0 /* lru_garbage_collector_test.cc in Sources */, 12158DFCEE09D24B7988A340 /* maybe_document.pb.cc in Sources */, 52BC453A24D8335C5A57C775 /* md5_test.cc in Sources */, + DA179A5E064DDFE734DC205E /* md5_testing.cc in Sources */, FA43BA0195DA90CE29B29D36 /* memory_bundle_cache_test.cc in Sources */, 8F2055702DB5EE8DA4BACD7C /* memory_document_overlay_cache_test.cc in Sources */, CFF1EBC60A00BA5109893C6E /* memory_index_manager_test.cc in Sources */, @@ -3959,6 +3970,7 @@ 1F56F51EB6DF0951B1F4F85B /* lru_garbage_collector_test.cc in Sources */, 88FD82A1FC5FEC5D56B481D8 /* maybe_document.pb.cc in Sources */, 88C1B719AC0AABE9EEEA71BA /* md5_test.cc in Sources */, + 9484B1C8E9447D397C54B100 /* md5_testing.cc in Sources */, 9611A0FAA2E10A6B1C1AC2EA /* memory_bundle_cache_test.cc in Sources */, 75C6CECF607CA94F56260BAB /* memory_document_overlay_cache_test.cc in Sources */, 3987A3E8534BAA496D966735 /* memory_index_manager_test.cc in Sources */, @@ -4185,6 +4197,7 @@ 913F6E57AF18F84C5ECFD414 /* lru_garbage_collector_test.cc in Sources */, 6F511ABFD023AEB81F92DB12 /* maybe_document.pb.cc in Sources */, 861EA75409AB15BADCD5793F /* md5_test.cc in Sources */, + 545D42F58BF326180B845350 /* md5_testing.cc in Sources */, FF6333B8BD9732C068157221 /* memory_bundle_cache_test.cc in Sources */, 5F6FD840AC2D729B50991CCB /* memory_document_overlay_cache_test.cc in Sources */, E6B825EE85BF20B88AF3E3CD /* memory_index_manager_test.cc in Sources */, @@ -4411,6 +4424,7 @@ 95CE3F5265B9BB7297EE5A6B /* lru_garbage_collector_test.cc in Sources */, C19214F5B43AA745A7FC2FC1 /* maybe_document.pb.cc in Sources */, 82473F290CC7D9579D64A175 /* md5_test.cc in Sources */, + 416B653E930A609FA121C5E3 /* md5_testing.cc in Sources */, 94854FAEAEA75A1AC77A0515 /* memory_bundle_cache_test.cc in Sources */, 053C11420E49AE1A77E21C20 /* memory_document_overlay_cache_test.cc in Sources */, 4D8367018652104A8803E8DB /* memory_index_manager_test.cc in Sources */, @@ -4631,6 +4645,7 @@ 1290FA77A922B76503AE407C /* lru_garbage_collector_test.cc in Sources */, 618BBEA720B89AAC00B5BCE7 /* maybe_document.pb.cc in Sources */, 4EC3518D09A8C37A28763052 /* md5_test.cc in Sources */, + AB6FE7D0D8EDD7369E94C0A4 /* md5_testing.cc in Sources */, A0E1C7F5C7093A498F65C5CF /* memory_bundle_cache_test.cc in Sources */, E56EEC9DAC455E2BE77D110A /* memory_document_overlay_cache_test.cc in Sources */, 3B47CC43DBA24434E215B8ED /* memory_index_manager_test.cc in Sources */, @@ -4876,6 +4891,7 @@ 4DF18D15AC926FB7A4888313 /* lru_garbage_collector_test.cc in Sources */, 12E04A12ABD5533B616D552A /* maybe_document.pb.cc in Sources */, 816375E636FD0EAECFA1E764 /* md5_test.cc in Sources */, + EE3512AC48C1E4B8A190FBB3 /* md5_testing.cc in Sources */, 479A392EAB42453D49435D28 /* memory_bundle_cache_test.cc in Sources */, 5CEB0E83DA68652927D2CF07 /* memory_document_overlay_cache_test.cc in Sources */, 90FE088B8FD9EC06EEED1F39 /* memory_index_manager_test.cc in Sources */, diff --git a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm index 9cabc78bb41..ce71327b244 100644 --- a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm +++ b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm @@ -56,6 +56,7 @@ #include "Firestore/core/src/model/types.h" #include "Firestore/core/src/nanopb/message.h" #include "Firestore/core/src/nanopb/nanopb_util.h" +#include "Firestore/core/src/remote/bloom_filter.h" #include "Firestore/core/src/remote/existence_filter.h" #include "Firestore/core/src/remote/serializer.h" #include "Firestore/core/src/remote/watch_change.h" @@ -78,8 +79,6 @@ namespace objc = firebase::firestore::objc; using firebase::firestore::Error; using firebase::firestore::google_firestore_v1_ArrayValue; -using firebase::firestore::google_firestore_v1_BitSequence; -using firebase::firestore::google_firestore_v1_BloomFilter; using firebase::firestore::google_firestore_v1_Value; using firebase::firestore::api::LoadBundleTask; using firebase::firestore::bundle::BundleReader; @@ -102,6 +101,7 @@ using firebase::firestore::nanopb::ByteString; using firebase::firestore::nanopb::MakeByteString; using firebase::firestore::nanopb::Message; +using firebase::firestore::remote::BloomFilter; using firebase::firestore::remote::DocumentWatchChange; using firebase::firestore::remote::ExistenceFilter; using firebase::firestore::remote::ExistenceFilterWatchChange; @@ -329,10 +329,10 @@ - (SnapshotVersion)parseVersion:(NSNumber *_Nullable)version { return Version(version.longLongValue); } -- (google_firestore_v1_BloomFilter)parseBloomFilter:(NSDictionary *_Nullable)bloomFilterProto { +- (BloomFilter)parseBloomFilter:(NSDictionary *_Nullable)bloomFilterProto { NSDictionary *bitsData = bloomFilterProto[@"bits"]; - NSString *bitmapData = bitsData[@"bitmap"]; + NSString *bitmapData = bitsData[@"bitmap"]; // Decode base64 json string: bitmap std::string bitmap; absl::Base64Unescape([bitmapData UTF8String], &bitmap); @@ -340,12 +340,7 @@ - (google_firestore_v1_BloomFilter)parseBloomFilter:(NSDictionary *_Nullable)blo int32_t padding = [bitsData[@"padding"] intValue]; int32_t hashCount = [bloomFilterProto[@"hashCount"] intValue]; - // Parse data into bloom filter proto type - google_firestore_v1_BitSequence bits; - bits.bitmap = nanopb::MakeBytesArray(bitmap); - bits.padding = padding; - google_firestore_v1_BloomFilter filter; - filter.hash_count = hashCount; + BloomFilter filter{bitmap, padding, hashCount}; return filter; } @@ -495,7 +490,7 @@ - (void)doWatchFilter:(NSDictionary *)watchFilter { NSArray *keys = watchFilter[@"keys"]; int keyCount = keys ? keys.count : 0; - google_firestore_v1_BloomFilter bloomFilter = [self parseBloomFilter:watchFilter[@"bloomFilter"]]; + BloomFilter bloomFilter = [self parseBloomFilter:watchFilter[@"bloomFilter"]]; ExistenceFilter filter{keyCount, bloomFilter}; ExistenceFilterWatchChange change{filter, targets[0].intValue}; [self.driver receiveWatchChange:change snapshotVersion:SnapshotVersion::None()]; diff --git a/Firestore/core/src/remote/bloom_filter.cc b/Firestore/core/src/remote/bloom_filter.cc index b42624a681c..d9ecd690a54 100644 --- a/Firestore/core/src/remote/bloom_filter.cc +++ b/Firestore/core/src/remote/bloom_filter.cc @@ -122,6 +122,12 @@ bool BloomFilter::MightContain(absl::string_view value) const { return true; } +bool operator==(const BloomFilter& lhs, const BloomFilter& rhs) { + return lhs.bit_count() == rhs.bit_count() && + lhs.hash_count() == rhs.hash_count() && + equal(lhs.bitmap().begin(), lhs.bitmap().end(), rhs.bitmap().begin()); +} + } // namespace remote } // namespace firestore } // namespace firebase diff --git a/Firestore/core/src/remote/bloom_filter.h b/Firestore/core/src/remote/bloom_filter.h index e406611b1cc..c04c5f69f85 100644 --- a/Firestore/core/src/remote/bloom_filter.h +++ b/Firestore/core/src/remote/bloom_filter.h @@ -57,11 +57,21 @@ class BloomFilter final { */ bool MightContain(absl::string_view value) const; - /** Get the `bit_count_` field. Used for testing purpose. */ + /** Get the `bit_count_` field. */ int32_t bit_count() const { return bit_count_; } + /** Get the `hash_count_` field. */ + int32_t hash_count() const { + return hash_count_; + } + + /** Get the `bitmap_` field. */ + const std::vector& bitmap() const { + return bitmap_; + } + private: /** * When checking membership of a key in bitmap, the first step is to generate @@ -104,6 +114,8 @@ class BloomFilter final { std::vector bitmap_; }; +bool operator==(const BloomFilter& lhs, const BloomFilter& rhs); + } // namespace remote } // namespace firestore } // namespace firebase diff --git a/Firestore/core/src/remote/existence_filter.h b/Firestore/core/src/remote/existence_filter.h index 7be936912cb..7ceade64faa 100644 --- a/Firestore/core/src/remote/existence_filter.h +++ b/Firestore/core/src/remote/existence_filter.h @@ -17,7 +17,7 @@ #ifndef FIRESTORE_CORE_SRC_REMOTE_EXISTENCE_FILTER_H_ #define FIRESTORE_CORE_SRC_REMOTE_EXISTENCE_FILTER_H_ -#include "Firestore/Protos/nanopb/google/firestore/v1/bloom_filter.nanopb.h" +#include "Firestore/core/src/remote/bloom_filter.h" namespace firebase { namespace firestore { @@ -26,33 +26,27 @@ namespace remote { class ExistenceFilter { public: ExistenceFilter() = default; - explicit ExistenceFilter(int count) : count_{count} { - } - explicit ExistenceFilter(int count, - google_firestore_v1_BloomFilter unchangedNames) - : count_{count}, unchanged_names_{unchangedNames} { + ExistenceFilter(int count, absl::optional unchangedNames) + : count_{count}, unchanged_names_{std::move(unchangedNames)} { } int count() const { return count_; } - google_firestore_v1_BloomFilter unchanged_names() const { + const absl::optional& unchanged_names() const { return unchanged_names_; } private: int count_ = 0; - google_firestore_v1_BloomFilter unchanged_names_; + absl::optional unchanged_names_; }; inline bool operator==(const ExistenceFilter& lhs, const ExistenceFilter& rhs) { return lhs.count() == rhs.count() && - lhs.unchanged_names().hash_count == rhs.unchanged_names().hash_count && - lhs.unchanged_names().bits.padding == - rhs.unchanged_names().bits.padding && - lhs.unchanged_names().bits.bitmap == rhs.unchanged_names().bits.bitmap; + lhs.unchanged_names() == rhs.unchanged_names(); } } // namespace remote diff --git a/Firestore/core/src/remote/serializer.cc b/Firestore/core/src/remote/serializer.cc index dea5503347b..35b9404f5b9 100644 --- a/Firestore/core/src/remote/serializer.cc +++ b/Firestore/core/src/remote/serializer.cc @@ -1425,9 +1425,11 @@ std::unique_ptr Serializer::DecodeDocumentRemove( std::unique_ptr Serializer::DecodeExistenceFilterWatchChange( ReadContext*, const google_firestore_v1_ExistenceFilter& filter) const { - ExistenceFilter existence_filter{filter.count, filter.unchanged_names}; - return absl::make_unique(existence_filter, - filter.target_id); + // TODO(Mila) + // BloomFilter bloomFilter{} + ExistenceFilter existence_filter{filter.count, absl::nullopt}; + return absl::make_unique( + std::move(existence_filter), filter.target_id); } bool Serializer::IsLocalResourceName(const ResourcePath& path) const { diff --git a/Firestore/core/src/remote/watch_change.h b/Firestore/core/src/remote/watch_change.h index ba2951c66c2..20e6121a7b5 100644 --- a/Firestore/core/src/remote/watch_change.h +++ b/Firestore/core/src/remote/watch_change.h @@ -115,7 +115,7 @@ bool operator==(const DocumentWatchChange& lhs, const DocumentWatchChange& rhs); class ExistenceFilterWatchChange : public WatchChange { public: ExistenceFilterWatchChange(ExistenceFilter filter, model::TargetId target_id) - : filter_{filter}, target_id_{target_id} { + : filter_{std::move(filter)}, target_id_{target_id} { } Type type() const override { diff --git a/Firestore/core/test/unit/local/local_store_test.cc b/Firestore/core/test/unit/local/local_store_test.cc index 0c3e1a1e595..683cc806eb0 100644 --- a/Firestore/core/test/unit/local/local_store_test.cc +++ b/Firestore/core/test/unit/local/local_store_test.cc @@ -146,7 +146,7 @@ RemoteEvent ExistenceFilterEvent(TargetId target_id, remote::FakeTargetMetadataProvider metadata_provider; metadata_provider.SetSyncedKeys(synced_keys, target_data); - ExistenceFilter existence_filter{remote_count}; + ExistenceFilter existence_filter{remote_count, absl::nullopt}; WatchChangeAggregator aggregator{&metadata_provider}; ExistenceFilterWatchChange existence_filter_watch_change{existence_filter, target_id}; diff --git a/Firestore/core/test/unit/remote/remote_event_test.cc b/Firestore/core/test/unit/remote/remote_event_test.cc index 8764a3aae90..355fd530f2b 100644 --- a/Firestore/core/test/unit/remote/remote_event_test.cc +++ b/Firestore/core/test/unit/remote/remote_event_test.cc @@ -547,7 +547,7 @@ TEST_F(RemoteEventTest, ExistenceFilterMismatchClearsTarget) { // The existence filter mismatch will remove the document from target 1, // but not synthesize a document delete. - ExistenceFilterWatchChange change4{ExistenceFilter{1}, 1}; + ExistenceFilterWatchChange change4{ExistenceFilter{1, absl::nullopt}, 1}; aggregator.HandleExistenceFilter(change4); event = aggregator.CreateRemoteEvent(testutil::Version(4)); @@ -578,7 +578,8 @@ TEST_F(RemoteEventTest, ExistenceFilterMismatchRemovesCurrentChanges) { // The existence filter mismatch will remove the document from target 1, but // not synthesize a document delete. - ExistenceFilterWatchChange existence_filter{ExistenceFilter{0}, 1}; + ExistenceFilterWatchChange existence_filter{ExistenceFilter{0, absl::nullopt}, + 1}; aggregator.HandleExistenceFilter(existence_filter); RemoteEvent event = aggregator.CreateRemoteEvent(testutil::Version(3)); diff --git a/Firestore/core/test/unit/remote/serializer_test.cc b/Firestore/core/test/unit/remote/serializer_test.cc index 5c8d08216ed..41b3120cfe3 100644 --- a/Firestore/core/test/unit/remote/serializer_test.cc +++ b/Firestore/core/test/unit/remote/serializer_test.cc @@ -1680,7 +1680,7 @@ TEST_F(SerializerTest, DecodesListenResponseWithDocumentRemove) { } TEST_F(SerializerTest, DecodesListenResponseWithExistenceFilter) { - ExistenceFilterWatchChange model(ExistenceFilter(2), 100); + ExistenceFilterWatchChange model(ExistenceFilter(2, absl::nullopt), 100); v1::ListenResponse proto; diff --git a/Firestore/core/test/unit/remote/watch_change_test.cc b/Firestore/core/test/unit/remote/watch_change_test.cc index 67a7e7d3a6f..d9459a656ac 100644 --- a/Firestore/core/test/unit/remote/watch_change_test.cc +++ b/Firestore/core/test/unit/remote/watch_change_test.cc @@ -41,7 +41,7 @@ TEST(WatchChangeTest, CanCreateDocumentWatchChange) { } TEST(WatchChangeTest, CanCreateExistenceFilterWatchChange) { - ExistenceFilter filter{7}; + ExistenceFilter filter{7, absl::nullopt}; ExistenceFilterWatchChange change{filter, 5}; EXPECT_EQ(change.filter().count(), 7); EXPECT_EQ(change.target_id(), 5); From 1e67033c932d90b2eb7d61713d839d49a23ab82d Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Tue, 28 Feb 2023 15:20:12 -0800 Subject: [PATCH 03/32] update serializer --- Firestore/Example/Tests/SpecTests/FSTSpecTests.mm | 5 +++-- Firestore/core/src/remote/existence_filter.h | 2 ++ Firestore/core/src/remote/serializer.cc | 10 ++++++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm index ce71327b244..86445e036e4 100644 --- a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm +++ b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm @@ -336,12 +336,12 @@ - (BloomFilter)parseBloomFilter:(NSDictionary *_Nullable)bloomFilterProto { // Decode base64 json string: bitmap std::string bitmap; absl::Base64Unescape([bitmapData UTF8String], &bitmap); + std::vector bitmapVector(bitmap.begin(), bitmap.end()); int32_t padding = [bitsData[@"padding"] intValue]; int32_t hashCount = [bloomFilterProto[@"hashCount"] intValue]; - BloomFilter filter{bitmap, padding, hashCount}; - + BloomFilter filter(bitmapVector, padding, hashCount); return filter; } @@ -491,6 +491,7 @@ - (void)doWatchFilter:(NSDictionary *)watchFilter { int keyCount = keys ? keys.count : 0; BloomFilter bloomFilter = [self parseBloomFilter:watchFilter[@"bloomFilter"]]; + ExistenceFilter filter{keyCount, bloomFilter}; ExistenceFilterWatchChange change{filter, targets[0].intValue}; [self.driver receiveWatchChange:change snapshotVersion:SnapshotVersion::None()]; diff --git a/Firestore/core/src/remote/existence_filter.h b/Firestore/core/src/remote/existence_filter.h index 7ceade64faa..76e3adaec89 100644 --- a/Firestore/core/src/remote/existence_filter.h +++ b/Firestore/core/src/remote/existence_filter.h @@ -19,6 +19,8 @@ #include "Firestore/core/src/remote/bloom_filter.h" +#include + namespace firebase { namespace firestore { namespace remote { diff --git a/Firestore/core/src/remote/serializer.cc b/Firestore/core/src/remote/serializer.cc index 35b9404f5b9..2ee82a9d983 100644 --- a/Firestore/core/src/remote/serializer.cc +++ b/Firestore/core/src/remote/serializer.cc @@ -26,6 +26,7 @@ #include #include #include +#include #include "Firestore/Protos/nanopb/google/firestore/v1/document.nanopb.h" #include "Firestore/Protos/nanopb/google/firestore/v1/firestore.nanopb.h" @@ -1425,8 +1426,13 @@ std::unique_ptr Serializer::DecodeDocumentRemove( std::unique_ptr Serializer::DecodeExistenceFilterWatchChange( ReadContext*, const google_firestore_v1_ExistenceFilter& filter) const { - // TODO(Mila) - // BloomFilter bloomFilter{} + + // TODO(Mila): convert pb_bytes_array_t into std::vector + ByteString bitmap_string(filter.unchanged_names.bits.bitmap); + std::vector bitmap = MakeVector(bitmap_string); + + BloomFilter bloomFilter(bitmap, filter.unchanged_names.bits.padding, + filter.unchanged_names.hash_count); ExistenceFilter existence_filter{filter.count, absl::nullopt}; return absl::make_unique( std::move(existence_filter), filter.target_id); From 899e3cc380444be74a6ac72c98a422c3db0ff6ca Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Thu, 2 Mar 2023 16:53:29 -0800 Subject: [PATCH 04/32] add bloomfilter equality test --- Firestore/core/test/unit/local/local_store_test.cc | 3 ++- .../core/test/unit/remote/bloom_filter_test.cc | 13 +++++++++++++ .../core/test/unit/remote/remote_event_test.cc | 7 ++++--- Firestore/core/test/unit/remote/serializer_test.cc | 3 ++- .../core/test/unit/remote/watch_change_test.cc | 2 +- 5 files changed, 22 insertions(+), 6 deletions(-) diff --git a/Firestore/core/test/unit/local/local_store_test.cc b/Firestore/core/test/unit/local/local_store_test.cc index 683cc806eb0..a9762c8fd5c 100644 --- a/Firestore/core/test/unit/local/local_store_test.cc +++ b/Firestore/core/test/unit/local/local_store_test.cc @@ -146,7 +146,8 @@ RemoteEvent ExistenceFilterEvent(TargetId target_id, remote::FakeTargetMetadataProvider metadata_provider; metadata_provider.SetSyncedKeys(synced_keys, target_data); - ExistenceFilter existence_filter{remote_count, absl::nullopt}; + ExistenceFilter existence_filter{remote_count, + /*unchangedNames=*/absl::nullopt}; WatchChangeAggregator aggregator{&metadata_provider}; ExistenceFilterWatchChange existence_filter_watch_change{existence_filter, target_id}; diff --git a/Firestore/core/test/unit/remote/bloom_filter_test.cc b/Firestore/core/test/unit/remote/bloom_filter_test.cc index 953cbf93260..a41a630132a 100644 --- a/Firestore/core/test/unit/remote/bloom_filter_test.cc +++ b/Firestore/core/test/unit/remote/bloom_filter_test.cc @@ -107,6 +107,19 @@ TEST(BloomFilterTest, CreateShouldReturnNotOKStatusIfPaddingIsTooLarge) { EXPECT_EQ(maybe_bloom_filter.status().error_message(), "Invalid padding: 8"); } +TEST(BloomFilterTest, CanCheckBloomFiltersEquality) { + { + BloomFilter bloom_filter1(std::vector{1}, 1, 1); + BloomFilter bloom_filter2(std::vector{1}, 1, 1); + EXPECT_TRUE(bloom_filter1 == bloom_filter2); + } + { + BloomFilter bloom_filter1(std::vector{1}, 1, 1); + BloomFilter bloom_filter2(std::vector{1}, 2, 1); + EXPECT_FALSE(bloom_filter1 == bloom_filter2); + } +} + TEST(BloomFilterTest, MightContainCanProcessNonStandardCharacters) { // A non-empty BloomFilter object with 1 insertion : "ÀÒ∑" BloomFilter bloom_filter(std::vector{237, 5}, 5, 8); diff --git a/Firestore/core/test/unit/remote/remote_event_test.cc b/Firestore/core/test/unit/remote/remote_event_test.cc index 355fd530f2b..fc8f956d4f6 100644 --- a/Firestore/core/test/unit/remote/remote_event_test.cc +++ b/Firestore/core/test/unit/remote/remote_event_test.cc @@ -547,7 +547,8 @@ TEST_F(RemoteEventTest, ExistenceFilterMismatchClearsTarget) { // The existence filter mismatch will remove the document from target 1, // but not synthesize a document delete. - ExistenceFilterWatchChange change4{ExistenceFilter{1, absl::nullopt}, 1}; + ExistenceFilterWatchChange change4{ + ExistenceFilter{1, /*unchangedNames=*/absl::nullopt}, 1}; aggregator.HandleExistenceFilter(change4); event = aggregator.CreateRemoteEvent(testutil::Version(4)); @@ -578,8 +579,8 @@ TEST_F(RemoteEventTest, ExistenceFilterMismatchRemovesCurrentChanges) { // The existence filter mismatch will remove the document from target 1, but // not synthesize a document delete. - ExistenceFilterWatchChange existence_filter{ExistenceFilter{0, absl::nullopt}, - 1}; + ExistenceFilterWatchChange existence_filter{ + ExistenceFilter{0, /*unchangedNames=*/absl::nullopt}, 1}; aggregator.HandleExistenceFilter(existence_filter); RemoteEvent event = aggregator.CreateRemoteEvent(testutil::Version(3)); diff --git a/Firestore/core/test/unit/remote/serializer_test.cc b/Firestore/core/test/unit/remote/serializer_test.cc index 41b3120cfe3..1000107ab20 100644 --- a/Firestore/core/test/unit/remote/serializer_test.cc +++ b/Firestore/core/test/unit/remote/serializer_test.cc @@ -1680,7 +1680,8 @@ TEST_F(SerializerTest, DecodesListenResponseWithDocumentRemove) { } TEST_F(SerializerTest, DecodesListenResponseWithExistenceFilter) { - ExistenceFilterWatchChange model(ExistenceFilter(2, absl::nullopt), 100); + ExistenceFilterWatchChange model( + ExistenceFilter(2, /*unchangedNames=*/absl::nullopt), 100); v1::ListenResponse proto; diff --git a/Firestore/core/test/unit/remote/watch_change_test.cc b/Firestore/core/test/unit/remote/watch_change_test.cc index d9459a656ac..9dca6e13e4a 100644 --- a/Firestore/core/test/unit/remote/watch_change_test.cc +++ b/Firestore/core/test/unit/remote/watch_change_test.cc @@ -41,7 +41,7 @@ TEST(WatchChangeTest, CanCreateDocumentWatchChange) { } TEST(WatchChangeTest, CanCreateExistenceFilterWatchChange) { - ExistenceFilter filter{7, absl::nullopt}; + ExistenceFilter filter{7, /*unchangedNames=*/absl::nullopt}; ExistenceFilterWatchChange change{filter, 5}; EXPECT_EQ(change.filter().count(), 7); EXPECT_EQ(change.target_id(), 5); From 468f967cc98c7968fb39cf4556485623b5928704 Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Tue, 7 Mar 2023 10:07:45 -0800 Subject: [PATCH 05/32] make unchanged_names optional in proto --- .../Example/Firestore.xcodeproj/project.pbxproj | 14 ++++---------- .../nanopb/google/firestore/v1/write.nanopb.cc | 8 +++++--- .../nanopb/google/firestore/v1/write.nanopb.h | 5 +++-- .../protos/google/firestore/v1/write.options | 4 ++++ Firestore/core/src/remote/existence_filter.h | 4 ++-- Firestore/core/src/remote/serializer.cc | 14 ++++++++------ 6 files changed, 26 insertions(+), 23 deletions(-) diff --git a/Firestore/Example/Firestore.xcodeproj/project.pbxproj b/Firestore/Example/Firestore.xcodeproj/project.pbxproj index fb1cf85d921..bf0365539d3 100644 --- a/Firestore/Example/Firestore.xcodeproj/project.pbxproj +++ b/Firestore/Example/Firestore.xcodeproj/project.pbxproj @@ -245,7 +245,6 @@ 264AAB492E24318C5EEB0649 /* memory_lru_garbage_collector_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 9765D47FA12FA283F4EFAD02 /* memory_lru_garbage_collector_test.cc */; }; 26777815544F549DD18D87AF /* message_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = CE37875365497FFA8687B745 /* message_test.cc */; }; 268FC3360157A2DCAF89F92D /* snapshot_version_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = ABA495B9202B7E79008A7851 /* snapshot_version_test.cc */; }; - 2695C8DE8DCAD5D6E9C9407C /* md5_testing.cc in Sources */ = {isa = PBXBuildFile; fileRef = 4007021A5284243AAE77292A /* md5_testing.cc */; }; 26B52236C9D049847042E1BD /* FSTMockDatastore.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E02D20213FFC00B64F25 /* FSTMockDatastore.mm */; }; 26B6F5D7571279F4FC06581A /* leveldb_query_engine_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = DB1F1E1B1ED15E8D042144B1 /* leveldb_query_engine_test.cc */; }; 26C4E52128C8E7B5B96BECC4 /* defer_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 8ABAC2E0402213D837F73DC3 /* defer_test.cc */; }; @@ -613,6 +612,7 @@ 55427A6CFFB22E069DCC0CC4 /* target_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 526D755F65AC676234F57125 /* target_test.cc */; }; 555161D6DB2DDC8B57F72A70 /* comparison_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 548DB928200D59F600E00ABC /* comparison_test.cc */; }; 5556B648B9B1C2F79A706B4F /* common.pb.cc in Sources */ = {isa = PBXBuildFile; fileRef = 544129D221C2DDC800EFB9CC /* common.pb.cc */; }; + 55CF4295DAB9B5E7D4DA3D67 /* Validation_BloomFilterTest_MD5_1_1_membership_test_result.json in Resources */ = {isa = PBXBuildFile; fileRef = 0DB24C79879C8290D46F106D /* Validation_BloomFilterTest_MD5_1_1_membership_test_result.json */; }; 55E84644D385A70E607A0F91 /* leveldb_local_store_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 5FF903AEFA7A3284660FA4C5 /* leveldb_local_store_test.cc */; }; 568EC1C0F68A7B95E57C8C6C /* leveldb_key_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 54995F6E205B6E12004EFFA0 /* leveldb_key_test.cc */; }; 56D85436D3C864B804851B15 /* string_format_apple_test.mm in Sources */ = {isa = PBXBuildFile; fileRef = 9CFD366B783AE27B9E79EE7A /* string_format_apple_test.mm */; }; @@ -964,7 +964,6 @@ 925BE64990449E93242A00A2 /* memory_mutation_queue_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 74FBEFA4FE4B12C435011763 /* memory_mutation_queue_test.cc */; }; 92D7081085679497DC112EDB /* persistence_testing.cc in Sources */ = {isa = PBXBuildFile; fileRef = 9113B6F513D0473AEABBAF1F /* persistence_testing.cc */; }; 92EFF0CC2993B43CBC7A61FF /* grpc_streaming_reader_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = B6D964922154AB8F00EB9CFB /* grpc_streaming_reader_test.cc */; }; - 937265DEC2441FA018104C27 /* md5_testing.cc in Sources */ = {isa = PBXBuildFile; fileRef = 4007021A5284243AAE77292A /* md5_testing.cc */; }; 9382BE7190E7750EE7CCCE7C /* write_spec_test.json in Resources */ = {isa = PBXBuildFile; fileRef = 54DA12A51F315EE100DD57A1 /* write_spec_test.json */; }; 938F2AF6EC5CD0B839300DB0 /* query.pb.cc in Sources */ = {isa = PBXBuildFile; fileRef = 544129D621C2DDC800EFB9CC /* query.pb.cc */; }; 939C898FE9D129F6A2EA259C /* FSTHelpers.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E03A2021401F00B64F25 /* FSTHelpers.mm */; }; @@ -1188,7 +1187,6 @@ BB1A6F7D8F06E74FB6E525C5 /* document_key_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = B6152AD5202A5385000E5744 /* document_key_test.cc */; }; BB3F35B1510FE5449E50EC8A /* bundle_cache_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = F7FC06E0A47D393DE1759AE1 /* bundle_cache_test.cc */; }; BB894A81FDF56EEC19CC29F8 /* FIRQuerySnapshotTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E04F202154AA00B64F25 /* FIRQuerySnapshotTests.mm */; }; - BBC5E4D41F51A8BB3F32A627 /* md5_testing.cc in Sources */ = {isa = PBXBuildFile; fileRef = 4007021A5284243AAE77292A /* md5_testing.cc */; }; BBDFE0000C4D7E529E296ED4 /* mutation.pb.cc in Sources */ = {isa = PBXBuildFile; fileRef = 618BBE8220B89AAC00B5BCE7 /* mutation.pb.cc */; }; BC0C98A9201E8F98B9A176A9 /* FIRWriteBatchTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E06F202154D600B64F25 /* FIRWriteBatchTests.mm */; }; BC0EF4474F799EFB1849FE26 /* Validation_BloomFilterTest_MD5_500_01_bloom_filter_proto.json in Resources */ = {isa = PBXBuildFile; fileRef = 3F8374AFD04A9FF22693AA19 /* Validation_BloomFilterTest_MD5_500_01_bloom_filter_proto.json */; }; @@ -1313,7 +1311,6 @@ D5B252EE3F4037405DB1ECE3 /* FIRNumericTransformTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = D5B25E7E7D6873CBA4571841 /* FIRNumericTransformTests.mm */; }; D5B25CBF07F65E885C9D68AB /* perf_spec_test.json in Resources */ = {isa = PBXBuildFile; fileRef = D5B2593BCB52957D62F1C9D3 /* perf_spec_test.json */; }; D5E9954FC1C5ABBC7A180B33 /* FSTSpecTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E03020213FFC00B64F25 /* FSTSpecTests.mm */; }; - D609BD9C3CB32EDFF46D7817 /* md5_testing.cc in Sources */ = {isa = PBXBuildFile; fileRef = 4007021A5284243AAE77292A /* md5_testing.cc */; }; D6486C7FFA8BE6F9C7D2F4C4 /* filesystem_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = F51859B394D01C0C507282F1 /* filesystem_test.cc */; }; D658E6DA5A218E08810E1688 /* byte_string_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 5342CDDB137B4E93E2E85CCA /* byte_string_test.cc */; }; D6962E598CEDABA312D87760 /* bundle_reader_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 6ECAF7DE28A19C69DF386D88 /* bundle_reader_test.cc */; }; @@ -1373,7 +1370,6 @@ DF7ABEB48A650117CBEBCD26 /* object_value_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 214877F52A705012D6720CA0 /* object_value_test.cc */; }; DF96816EC67F9B8DF19B0CFD /* document_overlay_cache_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = FFCA39825D9678A03D1845D0 /* document_overlay_cache_test.cc */; }; DF983A9C1FBF758AF3AF110D /* aggregation_result.pb.cc in Sources */ = {isa = PBXBuildFile; fileRef = D872D754B8AD88E28AF28B28 /* aggregation_result.pb.cc */; }; - E0008075F6D28D0D8BC46407 /* md5_testing.cc in Sources */ = {isa = PBXBuildFile; fileRef = 4007021A5284243AAE77292A /* md5_testing.cc */; }; E0239CE5666437FA1F353C41 /* Validation_BloomFilterTest_MD5_50000_01_membership_test_result.json in Resources */ = {isa = PBXBuildFile; fileRef = 2CF440979CE992E038A54427 /* Validation_BloomFilterTest_MD5_50000_01_membership_test_result.json */; }; E04607A1E2964684184E8AEA /* index_spec_test.json in Resources */ = {isa = PBXBuildFile; fileRef = 8C7278B604B8799F074F4E8C /* index_spec_test.json */; }; E08297B35E12106105F448EB /* ordered_code_benchmark.cc in Sources */ = {isa = PBXBuildFile; fileRef = 0473AFFF5567E667A125347B /* ordered_code_benchmark.cc */; }; @@ -1416,6 +1412,7 @@ E6B0FD6D70A7FFC14719B503 /* Validation_BloomFilterTest_MD5_500_1_bloom_filter_proto.json in Resources */ = {isa = PBXBuildFile; fileRef = 435B519FF670450EF8ECD6B4 /* Validation_BloomFilterTest_MD5_500_1_bloom_filter_proto.json */; }; E6B825EE85BF20B88AF3E3CD /* memory_index_manager_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = DB5A1E760451189DA36028B3 /* memory_index_manager_test.cc */; }; E6F8EB02A0E499F25160BB40 /* FIRFieldPathTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E04C202154AA00B64F25 /* FIRFieldPathTests.mm */; }; + E748D5354E230ABA231F4720 /* Validation_BloomFilterTest_MD5_500_0001_membership_test_result.json in Resources */ = {isa = PBXBuildFile; fileRef = 1C46787BE2535DD3B1D5CC71 /* Validation_BloomFilterTest_MD5_500_0001_membership_test_result.json */; }; E764F0F389E7119220EB212C /* target_id_generator_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = AB380CF82019382300D97691 /* target_id_generator_test.cc */; }; E7CE4B1ECD008983FAB90F44 /* string_format_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 54131E9620ADE678001DF3FF /* string_format_test.cc */; }; E7D415B8717701B952C344E5 /* executor_std_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = B6FB4687208F9B9100554BA2 /* executor_std_test.cc */; }; @@ -1523,7 +1520,6 @@ FABE084FA7DA6E216A41EE80 /* status_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 54A0352C20A3B3D7003E0143 /* status_test.cc */; }; FAD97B82766AEC29B7B5A1B7 /* index_manager_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = AE4A9E38D65688EE000EE2A1 /* index_manager_test.cc */; }; FAE5DA6ED3E1842DC21453EE /* fake_target_metadata_provider.cc in Sources */ = {isa = PBXBuildFile; fileRef = 71140E5D09C6E76F7C71B2FC /* fake_target_metadata_provider.cc */; }; - FB17BF00D4D6FA1A7E710250 /* md5_testing.cc in Sources */ = {isa = PBXBuildFile; fileRef = 4007021A5284243AAE77292A /* md5_testing.cc */; }; FB2111D9205822CC8E7368C2 /* FIRDocumentReferenceTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E049202154AA00B64F25 /* FIRDocumentReferenceTests.mm */; }; FB2D5208A6B5816A7244D77A /* query_engine_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = B8A853940305237AFDA8050B /* query_engine_test.cc */; }; FB3D9E01547436163C456A3C /* message_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = CE37875365497FFA8687B745 /* message_test.cc */; }; @@ -1663,7 +1659,6 @@ 3F0992A4B83C60841C52E960 /* Pods-Firestore_Example_iOS.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Firestore_Example_iOS.release.xcconfig"; path = "Pods/Target Support Files/Pods-Firestore_Example_iOS/Pods-Firestore_Example_iOS.release.xcconfig"; sourceTree = ""; }; 3F8374AFD04A9FF22693AA19 /* Validation_BloomFilterTest_MD5_500_01_bloom_filter_proto.json */ = {isa = PBXFileReference; includeInIndex = 1; name = Validation_BloomFilterTest_MD5_500_01_bloom_filter_proto.json; path = bloom_filter_golden_test_data/Validation_BloomFilterTest_MD5_500_01_bloom_filter_proto.json; sourceTree = ""; }; 3FBAA6F05C0B46A522E3B5A7 /* bundle_cache_test.h */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.c.h; path = bundle_cache_test.h; sourceTree = ""; }; - 4007021A5284243AAE77292A /* md5_testing.cc */ = {isa = PBXFileReference; includeInIndex = 1; path = md5_testing.cc; sourceTree = ""; }; 403DBF6EFB541DFD01582AA3 /* path_test.cc */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; path = path_test.cc; sourceTree = ""; }; 40F9D09063A07F710811A84F /* value_util_test.cc */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; path = value_util_test.cc; sourceTree = ""; }; 4132F30044D5DF1FB15B2A9D /* fake_credentials_provider.h */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.c.h; path = fake_credentials_provider.h; sourceTree = ""; }; @@ -1965,7 +1960,6 @@ CF39535F2C41AB0006FA6C0E /* create_noop_connectivity_monitor.cc */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; path = create_noop_connectivity_monitor.cc; sourceTree = ""; }; CF39ECA1293D21A0A2AB2626 /* FIRTransactionOptionsTests.mm */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.objcpp; path = FIRTransactionOptionsTests.mm; sourceTree = ""; }; D0A6E9136804A41CEC9D55D4 /* delayed_constructor_test.cc */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; path = delayed_constructor_test.cc; sourceTree = ""; }; - D225CE12C489975D3D758414 /* md5_testing.h */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.c.h; path = md5_testing.h; sourceTree = ""; }; D3CC3DC5338DCAF43A211155 /* README.md */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = net.daringfireball.markdown; name = README.md; path = ../README.md; sourceTree = ""; }; D5B2593BCB52957D62F1C9D3 /* perf_spec_test.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = perf_spec_test.json; sourceTree = ""; }; D5B25E7E7D6873CBA4571841 /* FIRNumericTransformTests.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = FIRNumericTransformTests.mm; sourceTree = ""; }; @@ -3411,7 +3405,7 @@ 67439E2FC35308F50C0643D9 /* Validation_BloomFilterTest_MD5_1_01_bloom_filter_proto.json in Resources */, E9ABBFCEC5A3D2AAEEC961B5 /* Validation_BloomFilterTest_MD5_1_01_membership_test_result.json in Resources */, 9D6CE441655A093BB2EC7693 /* Validation_BloomFilterTest_MD5_1_1_bloom_filter_proto.json in Resources */, - EE2404A60CE148E22EF44851 /* Validation_BloomFilterTest_MD5_1_1_membership_test_result.json in Resources */, + 55CF4295DAB9B5E7D4DA3D67 /* Validation_BloomFilterTest_MD5_1_1_membership_test_result.json in Resources */, 7F7D236E5DF9EBD84E831DAC /* Validation_BloomFilterTest_MD5_50000_0001_bloom_filter_proto.json in Resources */, 951F06B3CCE4EE1568A75B30 /* Validation_BloomFilterTest_MD5_50000_0001_membership_test_result.json in Resources */, 50ED640EA94CEC891C61A914 /* Validation_BloomFilterTest_MD5_50000_01_bloom_filter_proto.json in Resources */, @@ -3425,7 +3419,7 @@ 83A4D0CFF283CBD70EB11B2D /* Validation_BloomFilterTest_MD5_5000_1_bloom_filter_proto.json in Resources */, 0908F0C7C15A1974CF3D20C3 /* Validation_BloomFilterTest_MD5_5000_1_membership_test_result.json in Resources */, A31AC114D37771C1E430CDC4 /* Validation_BloomFilterTest_MD5_500_0001_bloom_filter_proto.json in Resources */, - 941B56B83A1C5771990E14DE /* Validation_BloomFilterTest_MD5_500_0001_membership_test_result.json in Resources */, + E748D5354E230ABA231F4720 /* Validation_BloomFilterTest_MD5_500_0001_membership_test_result.json in Resources */, E8805A3A1DFC1E4EDA65524F /* Validation_BloomFilterTest_MD5_500_01_bloom_filter_proto.json in Resources */, 2374EA106F4F320120852E81 /* Validation_BloomFilterTest_MD5_500_01_membership_test_result.json in Resources */, 012498C63EE01921671FA0B6 /* Validation_BloomFilterTest_MD5_500_1_bloom_filter_proto.json in Resources */, diff --git a/Firestore/Protos/nanopb/google/firestore/v1/write.nanopb.cc b/Firestore/Protos/nanopb/google/firestore/v1/write.nanopb.cc index 56275531bd4..407c3d9b38a 100644 --- a/Firestore/Protos/nanopb/google/firestore/v1/write.nanopb.cc +++ b/Firestore/Protos/nanopb/google/firestore/v1/write.nanopb.cc @@ -95,7 +95,7 @@ const pb_field_t google_firestore_v1_DocumentRemove_fields[4] = { const pb_field_t google_firestore_v1_ExistenceFilter_fields[4] = { PB_FIELD( 1, INT32 , SINGULAR, STATIC , FIRST, google_firestore_v1_ExistenceFilter, target_id, target_id, 0), PB_FIELD( 2, INT32 , SINGULAR, STATIC , OTHER, google_firestore_v1_ExistenceFilter, count, target_id, 0), - PB_FIELD( 3, MESSAGE , SINGULAR, STATIC , OTHER, google_firestore_v1_ExistenceFilter, unchanged_names, count, &google_firestore_v1_BloomFilter_fields), + PB_FIELD( 3, MESSAGE , OPTIONAL, STATIC , OTHER, google_firestore_v1_ExistenceFilter, unchanged_names, count, &google_firestore_v1_BloomFilter_fields), PB_LAST_FIELD }; @@ -317,8 +317,10 @@ std::string google_firestore_v1_ExistenceFilter::ToString(int indent) const { target_id, indent + 1, false); tostring_result += PrintPrimitiveField("count: ", count, indent + 1, false); - tostring_result += PrintMessageField("unchanged_names ", - unchanged_names, indent + 1, false); + if (has_unchanged_names) { + tostring_result += PrintMessageField("unchanged_names ", + unchanged_names, indent + 1, true); + } std::string tostring_tail = PrintTail(indent); return tostring_header + tostring_result + tostring_tail; diff --git a/Firestore/Protos/nanopb/google/firestore/v1/write.nanopb.h b/Firestore/Protos/nanopb/google/firestore/v1/write.nanopb.h index 3e22c2bc42a..f7dc73102dc 100644 --- a/Firestore/Protos/nanopb/google/firestore/v1/write.nanopb.h +++ b/Firestore/Protos/nanopb/google/firestore/v1/write.nanopb.h @@ -110,6 +110,7 @@ typedef struct _google_firestore_v1_DocumentTransform_FieldTransform { typedef struct _google_firestore_v1_ExistenceFilter { int32_t target_id; int32_t count; + bool has_unchanged_names; google_firestore_v1_BloomFilter unchanged_names; std::string ToString(int indent = 0) const; @@ -155,7 +156,7 @@ typedef struct _google_firestore_v1_WriteResult { #define google_firestore_v1_DocumentChange_init_default {google_firestore_v1_Document_init_default, 0, NULL, 0, NULL} #define google_firestore_v1_DocumentDelete_init_default {NULL, false, google_protobuf_Timestamp_init_default, 0, NULL} #define google_firestore_v1_DocumentRemove_init_default {NULL, 0, NULL, google_protobuf_Timestamp_init_default} -#define google_firestore_v1_ExistenceFilter_init_default {0, 0, google_firestore_v1_BloomFilter_init_default} +#define google_firestore_v1_ExistenceFilter_init_default {0, 0, false, google_firestore_v1_BloomFilter_init_default} #define google_firestore_v1_Write_init_zero {0, {google_firestore_v1_Document_init_zero}, false, google_firestore_v1_DocumentMask_init_zero, false, google_firestore_v1_Precondition_init_zero, 0, NULL} #define google_firestore_v1_DocumentTransform_init_zero {NULL, 0, NULL} #define google_firestore_v1_DocumentTransform_FieldTransform_init_zero {NULL, 0, {_google_firestore_v1_DocumentTransform_FieldTransform_ServerValue_MIN}} @@ -163,7 +164,7 @@ typedef struct _google_firestore_v1_WriteResult { #define google_firestore_v1_DocumentChange_init_zero {google_firestore_v1_Document_init_zero, 0, NULL, 0, NULL} #define google_firestore_v1_DocumentDelete_init_zero {NULL, false, google_protobuf_Timestamp_init_zero, 0, NULL} #define google_firestore_v1_DocumentRemove_init_zero {NULL, 0, NULL, google_protobuf_Timestamp_init_zero} -#define google_firestore_v1_ExistenceFilter_init_zero {0, 0, google_firestore_v1_BloomFilter_init_zero} +#define google_firestore_v1_ExistenceFilter_init_zero {0, 0, false, google_firestore_v1_BloomFilter_init_zero} /* Field tags (for use in manual encoding/decoding) */ #define google_firestore_v1_DocumentTransform_document_tag 1 diff --git a/Firestore/Protos/protos/google/firestore/v1/write.options b/Firestore/Protos/protos/google/firestore/v1/write.options index 91805bd55a9..b7955f853c6 100644 --- a/Firestore/Protos/protos/google/firestore/v1/write.options +++ b/Firestore/Protos/protos/google/firestore/v1/write.options @@ -29,3 +29,7 @@ google.firestore.v1.Write.update_mask proto3:false # update_time should not be set for deletes. google.firestore.v1.WriteResult.update_time proto3:false + +# `unchanged_names` field might not be set if ListenRequest doesn't have +# `expected_count` field. +google.firestore.v1.ExistenceFilter.unchanged_names proto3:false \ No newline at end of file diff --git a/Firestore/core/src/remote/existence_filter.h b/Firestore/core/src/remote/existence_filter.h index 76e3adaec89..5e7532e35b0 100644 --- a/Firestore/core/src/remote/existence_filter.h +++ b/Firestore/core/src/remote/existence_filter.h @@ -17,10 +17,10 @@ #ifndef FIRESTORE_CORE_SRC_REMOTE_EXISTENCE_FILTER_H_ #define FIRESTORE_CORE_SRC_REMOTE_EXISTENCE_FILTER_H_ -#include "Firestore/core/src/remote/bloom_filter.h" - #include +#include "Firestore/core/src/remote/bloom_filter.h" + namespace firebase { namespace firestore { namespace remote { diff --git a/Firestore/core/src/remote/serializer.cc b/Firestore/core/src/remote/serializer.cc index 2ee82a9d983..35c71dc0f39 100644 --- a/Firestore/core/src/remote/serializer.cc +++ b/Firestore/core/src/remote/serializer.cc @@ -1426,14 +1426,16 @@ std::unique_ptr Serializer::DecodeDocumentRemove( std::unique_ptr Serializer::DecodeExistenceFilterWatchChange( ReadContext*, const google_firestore_v1_ExistenceFilter& filter) const { + absl::optional bloom_filter = absl::nullopt; + if (filter.has_unchanged_names) { + ByteString bitmap_string(filter.unchanged_names.bits.bitmap); + std::vector bitmap = MakeVector(bitmap_string); - // TODO(Mila): convert pb_bytes_array_t into std::vector - ByteString bitmap_string(filter.unchanged_names.bits.bitmap); - std::vector bitmap = MakeVector(bitmap_string); + bloom_filter = BloomFilter(bitmap, filter.unchanged_names.bits.padding, + filter.unchanged_names.hash_count); + } - BloomFilter bloomFilter(bitmap, filter.unchanged_names.bits.padding, - filter.unchanged_names.hash_count); - ExistenceFilter existence_filter{filter.count, absl::nullopt}; + ExistenceFilter existence_filter{filter.count, bloom_filter}; return absl::make_unique( std::move(existence_filter), filter.target_id); } From 23f0fb19d86a9db120d4b410430e070d7699d225 Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Tue, 7 Mar 2023 15:44:54 -0800 Subject: [PATCH 06/32] handle invalid unchanged-names input with BloomFilter::Create --- .../Example/Tests/SpecTests/FSTSpecTests.mm | 29 ++++++++++++------- .../firestore/v1/bloom_filter.nanopb.cc | 6 ++-- .../google/firestore/v1/bloom_filter.nanopb.h | 5 ++-- .../google/firestore/v1/bloom_filter.options | 21 ++++++++++++++ Firestore/core/src/remote/serializer.cc | 27 ++++++++++++----- Firestore/core/src/remote/serializer.h | 3 ++ 6 files changed, 70 insertions(+), 21 deletions(-) create mode 100644 Firestore/Protos/protos/google/firestore/v1/bloom_filter.options diff --git a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm index 86445e036e4..05ca08ba9e8 100644 --- a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm +++ b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm @@ -119,6 +119,7 @@ using firebase::firestore::util::MakeStringPtr; using firebase::firestore::util::Path; using firebase::firestore::util::Status; +using firebase::firestore::util::StatusOr; using firebase::firestore::util::TimerId; using firebase::firestore::util::ToString; using firebase::firestore::util::WrapCompare; @@ -329,20 +330,28 @@ - (SnapshotVersion)parseVersion:(NSNumber *_Nullable)version { return Version(version.longLongValue); } -- (BloomFilter)parseBloomFilter:(NSDictionary *_Nullable)bloomFilterProto { +- (absl::optional)parseBloomFilter:(NSDictionary *_Nullable)bloomFilterProto { + if (bloomFilterProto == nil) { + return absl::nullopt; + } NSDictionary *bitsData = bloomFilterProto[@"bits"]; - NSString *bitmapData = bitsData[@"bitmap"]; - // Decode base64 json string: bitmap - std::string bitmap; - absl::Base64Unescape([bitmapData UTF8String], &bitmap); - std::vector bitmapVector(bitmap.begin(), bitmap.end()); + // Decode base64 string into uint8_t vector. If not specified, will default bitmap to empty + // vector. + NSString *bitmapEncoded = bitsData[@"bitmap"]; + std::string bitmapDecoded; + absl::Base64Unescape([bitmapEncoded UTF8String], &bitmapDecoded); + std::vector bitmap(bitmapDecoded.begin(), bitmapDecoded.end()); + // If not specified, will default padding and hashCount to 0. int32_t padding = [bitsData[@"padding"] intValue]; int32_t hashCount = [bloomFilterProto[@"hashCount"] intValue]; + StatusOr maybeBloomFilter = BloomFilter::Create(bitmap, padding, hashCount); + if (maybeBloomFilter.ok()) { + return maybeBloomFilter.ValueOrDie(); + } - BloomFilter filter(bitmapVector, padding, hashCount); - return filter; + return absl::nullopt; } - (DocumentViewChange)parseChange:(NSDictionary *)jsonDoc ofType:(DocumentViewChange::Type)type { @@ -488,9 +497,9 @@ - (void)doWatchFilter:(NSDictionary *)watchFilter { HARD_ASSERT(targets.count == 1, "ExistenceFilters currently support exactly one target only."); NSArray *keys = watchFilter[@"keys"]; - int keyCount = keys ? keys.count : 0; + int keyCount = keys ? (int)keys.count : 0; - BloomFilter bloomFilter = [self parseBloomFilter:watchFilter[@"bloomFilter"]]; + absl::optional bloomFilter = [self parseBloomFilter:watchFilter[@"bloomFilter"]]; ExistenceFilter filter{keyCount, bloomFilter}; ExistenceFilterWatchChange change{filter, targets[0].intValue}; diff --git a/Firestore/Protos/nanopb/google/firestore/v1/bloom_filter.nanopb.cc b/Firestore/Protos/nanopb/google/firestore/v1/bloom_filter.nanopb.cc index 2050bc400fd..45da032ef97 100644 --- a/Firestore/Protos/nanopb/google/firestore/v1/bloom_filter.nanopb.cc +++ b/Firestore/Protos/nanopb/google/firestore/v1/bloom_filter.nanopb.cc @@ -44,7 +44,7 @@ const pb_field_t google_firestore_v1_BitSequence_fields[3] = { }; const pb_field_t google_firestore_v1_BloomFilter_fields[3] = { - PB_FIELD( 1, MESSAGE , SINGULAR, STATIC , FIRST, google_firestore_v1_BloomFilter, bits, bits, &google_firestore_v1_BitSequence_fields), + PB_FIELD( 1, MESSAGE , OPTIONAL, STATIC , FIRST, google_firestore_v1_BloomFilter, bits, bits, &google_firestore_v1_BitSequence_fields), PB_FIELD( 2, INT32 , SINGULAR, STATIC , OTHER, google_firestore_v1_BloomFilter, hash_count, bits, 0), PB_LAST_FIELD }; @@ -96,7 +96,9 @@ std::string google_firestore_v1_BloomFilter::ToString(int indent) const { std::string tostring_header = PrintHeader(indent, "BloomFilter", this); std::string tostring_result; - tostring_result += PrintMessageField("bits ", bits, indent + 1, false); + if (has_bits) { + tostring_result += PrintMessageField("bits ", bits, indent + 1, true); + } tostring_result += PrintPrimitiveField("hash_count: ", hash_count, indent + 1, false); diff --git a/Firestore/Protos/nanopb/google/firestore/v1/bloom_filter.nanopb.h b/Firestore/Protos/nanopb/google/firestore/v1/bloom_filter.nanopb.h index e0347c466a5..925d70db281 100644 --- a/Firestore/Protos/nanopb/google/firestore/v1/bloom_filter.nanopb.h +++ b/Firestore/Protos/nanopb/google/firestore/v1/bloom_filter.nanopb.h @@ -42,6 +42,7 @@ typedef struct _google_firestore_v1_BitSequence { } google_firestore_v1_BitSequence; typedef struct _google_firestore_v1_BloomFilter { + bool has_bits; google_firestore_v1_BitSequence bits; int32_t hash_count; @@ -53,9 +54,9 @@ typedef struct _google_firestore_v1_BloomFilter { /* Initializer values for message structs */ #define google_firestore_v1_BitSequence_init_default {NULL, 0} -#define google_firestore_v1_BloomFilter_init_default {google_firestore_v1_BitSequence_init_default, 0} +#define google_firestore_v1_BloomFilter_init_default {false, google_firestore_v1_BitSequence_init_default, 0} #define google_firestore_v1_BitSequence_init_zero {NULL, 0} -#define google_firestore_v1_BloomFilter_init_zero {google_firestore_v1_BitSequence_init_zero, 0} +#define google_firestore_v1_BloomFilter_init_zero {false, google_firestore_v1_BitSequence_init_zero, 0} /* Field tags (for use in manual encoding/decoding) */ #define google_firestore_v1_BitSequence_bitmap_tag 1 diff --git a/Firestore/Protos/protos/google/firestore/v1/bloom_filter.options b/Firestore/Protos/protos/google/firestore/v1/bloom_filter.options new file mode 100644 index 00000000000..652655cd9d5 --- /dev/null +++ b/Firestore/Protos/protos/google/firestore/v1/bloom_filter.options @@ -0,0 +1,21 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# In proto3 mode, Nanopb doesn't allow distinguishing between unset fields and +# fields having default values, even for non-primitive types. Using the +# workaround suggested in +# https://github.com/nanopb/nanopb/issues/255#issuecomment-291842903 + +# `bits` might not be set if the BloomFilter is empty. +google.firestore.v1.BloomFilter.bits proto3:false \ No newline at end of file diff --git a/Firestore/core/src/remote/serializer.cc b/Firestore/core/src/remote/serializer.cc index 35c71dc0f39..9b746487ed1 100644 --- a/Firestore/core/src/remote/serializer.cc +++ b/Firestore/core/src/remote/serializer.cc @@ -1426,18 +1426,31 @@ std::unique_ptr Serializer::DecodeDocumentRemove( std::unique_ptr Serializer::DecodeExistenceFilterWatchChange( ReadContext*, const google_firestore_v1_ExistenceFilter& filter) const { + ExistenceFilter existence_filter = DecodeExistenceFilter(filter); + return absl::make_unique( + std::move(existence_filter), filter.target_id); +} + +ExistenceFilter Serializer::DecodeExistenceFilter( + const google_firestore_v1_ExistenceFilter& filter) const { + // Create bloom filter if there is an unchanged_names present in the filter + // and inputs are valid, otherwise keep it null. absl::optional bloom_filter = absl::nullopt; - if (filter.has_unchanged_names) { + if (filter.has_unchanged_names && filter.unchanged_names.has_bits) { + // TODO(Mila): Ensure bloom filter with invalid inputs are handled correctly + // in next PR. ByteString bitmap_string(filter.unchanged_names.bits.bitmap); std::vector bitmap = MakeVector(bitmap_string); - - bloom_filter = BloomFilter(bitmap, filter.unchanged_names.bits.padding, - filter.unchanged_names.hash_count); + int32_t padding = filter.unchanged_names.bits.padding; + int32_t hash_count = filter.unchanged_names.hash_count; + StatusOr maybe_bloom_filter = + BloomFilter::Create(bitmap, padding, hash_count); + if (maybe_bloom_filter.ok()) { + bloom_filter = maybe_bloom_filter.ValueOrDie(); + } } - ExistenceFilter existence_filter{filter.count, bloom_filter}; - return absl::make_unique( - std::move(existence_filter), filter.target_id); + return {filter.count, bloom_filter}; } bool Serializer::IsLocalResourceName(const ResourcePath& path) const { diff --git a/Firestore/core/src/remote/serializer.h b/Firestore/core/src/remote/serializer.h index ab9370b441a..c42c6c3ac1b 100644 --- a/Firestore/core/src/remote/serializer.h +++ b/Firestore/core/src/remote/serializer.h @@ -341,6 +341,9 @@ class Serializer { util::ReadContext* context, const google_firestore_v1_ExistenceFilter& filter) const; + ExistenceFilter DecodeExistenceFilter( + const google_firestore_v1_ExistenceFilter& filter) const; + model::DatabaseId database_id_; // TODO(varconst): Android caches the result of calling `EncodeDatabaseName` // as well, consider implementing that. From 475fffb4281b0f1dc4f31a10fcde049140c08d8b Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Tue, 7 Mar 2023 15:58:03 -0800 Subject: [PATCH 07/32] format the comments --- Firestore/core/src/remote/bloom_filter.h | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/Firestore/core/src/remote/bloom_filter.h b/Firestore/core/src/remote/bloom_filter.h index c04c5f69f85..52178784d2f 100644 --- a/Firestore/core/src/remote/bloom_filter.h +++ b/Firestore/core/src/remote/bloom_filter.h @@ -57,17 +57,24 @@ class BloomFilter final { */ bool MightContain(absl::string_view value) const; - /** Get the `bit_count_` field. */ + /** + * The number of bits in the bloom filter. Guaranteed to be non-negative, and + * less than the max number of bits `bitmap_` can represent, i.e., + * bitmap_.size() * 8. + */ int32_t bit_count() const { return bit_count_; } - /** Get the `hash_count_` field. */ + /** + * The number of hash functions used to construct the filter. Guaranteed to + * be non-negative. + */ int32_t hash_count() const { return hash_count_; } - /** Get the `bitmap_` field. */ + /** Bloom filter's bitmap. */ const std::vector& bitmap() const { return bitmap_; } @@ -97,20 +104,10 @@ class BloomFilter final { /** Return whether the bit at the given index in the bitmap is set to 1. */ bool IsBitSet(int32_t index) const; - /** - * The number of bits in the bloom filter. Guaranteed to be non-negative, and - * less than the max number of bits `bitmap_` can represent, i.e., - * bitmap_.size() * 8. - */ int32_t bit_count_ = 0; - /** - * The number of hash functions used to construct the filter. Guaranteed to - * be non-negative. - */ int32_t hash_count_ = 0; - /** Bloom filter's bitmap. */ std::vector bitmap_; }; From 11158e539316ab2c3a8cee49ff8bdf4caa87abd4 Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Mon, 13 Mar 2023 15:45:53 -0700 Subject: [PATCH 08/32] port spec tests --- .../SpecTests/json/bundle_spec_test.json | 3 +- .../json/existence_filter_spec_test.json | 6928 +++++++++++++++-- .../Tests/SpecTests/json/limbo_spec_test.json | 568 +- .../Tests/SpecTests/json/limit_spec_test.json | 39 +- .../SpecTests/json/listen_spec_test.json | 12 +- .../SpecTests/json/offline_spec_test.json | 6 +- .../SpecTests/json/recovery_spec_test.json | 12 +- 7 files changed, 6789 insertions(+), 779 deletions(-) diff --git a/Firestore/Example/Tests/SpecTests/json/bundle_spec_test.json b/Firestore/Example/Tests/SpecTests/json/bundle_spec_test.json index e45f01e937a..617f5946d5f 100644 --- a/Firestore/Example/Tests/SpecTests/json/bundle_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/bundle_spec_test.json @@ -1165,7 +1165,8 @@ "path": "collection/a" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ diff --git a/Firestore/Example/Tests/SpecTests/json/existence_filter_spec_test.json b/Firestore/Example/Tests/SpecTests/json/existence_filter_spec_test.json index 75722e8ebcb..131d47cd4a1 100644 --- a/Firestore/Example/Tests/SpecTests/json/existence_filter_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/existence_filter_spec_test.json @@ -1,9 +1,8 @@ { - "Existence filter clears resume token": { + "Bloom filter can process special characters in document name": { "describeName": "Existence Filters:", - "itName": "Existence filter clears resume token", + "itName": "Bloom filter can process special characters in document name", "tags": [ - "durable-persistence" ], "config": { "numClients": 1, @@ -47,7 +46,7 @@ "watchEntity": { "docs": [ { - "key": "collection/1", + "key": "collection/ÀÒ∑", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -58,13 +57,13 @@ "version": 1000 }, { - "key": "collection/2", + "key": "collection/À∑Ò", "options": { "hasCommittedMutations": false, "hasLocalMutations": false }, "value": { - "v": 2 + "v": 1 }, "version": 1000 } @@ -92,7 +91,7 @@ { "added": [ { - "key": "collection/1", + "key": "collection/ÀÒ∑", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -103,13 +102,13 @@ "version": 1000 }, { - "key": "collection/2", + "key": "collection/À∑Ò", "options": { "hasCommittedMutations": false, "hasLocalMutations": false }, "value": { - "v": 2 + "v": 1 }, "version": 1000 } @@ -129,8 +128,15 @@ }, { "watchFilter": { + "bloomFilter": { + "bits": { + "bitmap": "IIAAIIAIIAAIIAIIAA==", + "padding": 4 + }, + "hashCount": 10 + }, "keys": [ - "collection/1" + "collection/ÀÒ∑" ], "targetIds": [ 2 @@ -156,19 +162,52 @@ "path": "collection" } } - ] - }, - { - "restart": true, + ], "expectedState": { "activeLimboDocs": [ + "collection/À∑Ò" ], "activeTargets": { - }, - "enqueuedLimboDocs": [ - ] + "1": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/À∑Ò" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + } + } } - }, + } + ] + }, + "Bloom filter fills in default values for undefined padding and hashCount": { + "describeName": "Existence Filters:", + "itName": "Bloom filter fills in default values for undefined padding and hashCount", + "tags": [ + ], + "config": { + "numClients": 1, + "useGarbageCollection": true + }, + "steps": [ { "userListen": { "query": { @@ -180,11 +219,78 @@ }, "targetId": 2 }, + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + } + } + } + }, + { + "watchAck": [ + 2 + ] + }, + { + "watchEntity": { + "docs": [ + { + "key": "collection/a", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/b", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 1000 + } + ], + "targets": [ + 2 + ] + } + }, + { + "watchCurrent": [ + [ + 2 + ], + "resume-token-1000" + ] + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 1000 + }, "expectedSnapshotEvents": [ { "added": [ { - "key": "collection/1", + "key": "collection/a", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -195,7 +301,7 @@ "version": 1000 }, { - "key": "collection/2", + "key": "collection/b", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -206,6 +312,42 @@ "version": 1000 } ], + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ] + }, + { + "watchFilter": { + "bloomFilter": { + "bits": { + "bitmap": "AhAAApAAAIAEBIAABA==" + } + }, + "keys": [ + "collection/a" + ], + "targetIds": [ + 2 + ] + } + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 2000 + }, + "expectedSnapshotEvents": [ + { "errorCode": 0, "fromCache": true, "hasPendingWrites": false, @@ -230,16 +372,17 @@ "path": "collection" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 1 } } } } ] }, - "Existence filter handled at global snapshot": { + "Bloom filter is handled at global snapshot": { "describeName": "Existence Filters:", - "itName": "Existence filter handled at global snapshot", + "itName": "Bloom filter is handled at global snapshot", "tags": [ ], "config": { @@ -284,7 +427,7 @@ "watchEntity": { "docs": [ { - "key": "collection/1", + "key": "collection/a", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -293,6 +436,17 @@ "v": 1 }, "version": 1000 + }, + { + "key": "collection/b", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 2000 } ], "targets": [ @@ -318,7 +472,7 @@ { "added": [ { - "key": "collection/1", + "key": "collection/a", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -327,6 +481,17 @@ "v": 1 }, "version": 1000 + }, + { + "key": "collection/b", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 2000 } ], "errorCode": 0, @@ -344,9 +509,15 @@ }, { "watchFilter": { + "bloomFilter": { + "bits": { + "bitmap": "AhAAApAAAIAEBIAABA==", + "padding": 4 + }, + "hashCount": 10 + }, "keys": [ - "collection/1", - "collection/2" + "collection/a" ], "targetIds": [ 2 @@ -357,7 +528,7 @@ "watchEntity": { "docs": [ { - "key": "collection/3", + "key": "collection/c", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -383,7 +554,7 @@ { "added": [ { - "key": "collection/3", + "key": "collection/c", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -407,7 +578,23 @@ } ], "expectedState": { + "activeLimboDocs": [ + "collection/b" + ], "activeTargets": { + "1": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/b" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, "2": { "queries": [ { @@ -422,12 +609,45 @@ } } } - }, + } + ] + }, + "Bloom filter limbo resolution is denied": { + "describeName": "Existence Filters:", + "itName": "Bloom filter limbo resolution is denied", + "tags": [ + ], + "config": { + "numClients": 1, + "useGarbageCollection": true + }, + "steps": [ { - "watchRemove": { - "targetIds": [ - 2 - ] + "userListen": { + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + }, + "targetId": 2 + }, + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + } + } } }, { @@ -439,7 +659,7 @@ "watchEntity": { "docs": [ { - "key": "collection/1", + "key": "collection/a", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -450,26 +670,5188 @@ "version": 1000 }, { - "key": "collection/2", + "key": "collection/b", "options": { "hasCommittedMutations": false, "hasLocalMutations": false }, "value": { - "v": 2 + "v": 1 }, - "version": 2000 - }, - { - "key": "collection/3", - "options": { - "hasCommittedMutations": false, - "hasLocalMutations": false + "version": 1000 + } + ], + "targets": [ + 2 + ] + } + }, + { + "watchCurrent": [ + [ + 2 + ], + "resume-token-1000" + ] + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 1000 + }, + "expectedSnapshotEvents": [ + { + "added": [ + { + "key": "collection/a", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/b", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + } + ], + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ] + }, + { + "watchFilter": { + "bloomFilter": { + "bits": { + "bitmap": "AhAAApAAAIAEBIAABA==", + "padding": 4 + }, + "hashCount": 10 + }, + "keys": [ + "collection/a" + ], + "targetIds": [ + 2 + ] + } + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 2000 + }, + "expectedSnapshotEvents": [ + { + "errorCode": 0, + "fromCache": true, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ], + "expectedState": { + "activeLimboDocs": [ + "collection/b" + ], + "activeTargets": { + "1": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/b" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + } + } + } + }, + { + "watchRemove": { + "cause": { + "code": 7 + }, + "targetIds": [ + 1 + ] + }, + "expectedSnapshotEvents": [ + { + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + }, + "removed": [ + { + "key": "collection/b", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + } + ] + } + ], + "expectedState": { + "activeLimboDocs": [ + ], + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + } + } + } + } + ] + }, + "Bloom filter with large size works as expected": { + "describeName": "Existence Filters:", + "itName": "Bloom filter with large size works as expected", + "tags": [ + ], + "config": { + "numClients": 1, + "useGarbageCollection": true + }, + "steps": [ + { + "userListen": { + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + }, + "targetId": 2 + }, + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + } + } + } + }, + { + "watchAck": [ + 2 + ] + }, + { + "watchEntity": { + "docs": [ + { + "key": "collection/doc0", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc2", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc3", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc4", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc5", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc6", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc7", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc8", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc9", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc10", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc11", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc12", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc13", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc14", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc15", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc16", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc17", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc18", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc19", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc20", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc21", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc22", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc23", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc24", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc25", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc26", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc27", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc28", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc29", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc30", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc31", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc32", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc33", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc34", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc35", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc36", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc37", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc38", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc39", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc40", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc41", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc42", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc43", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc44", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc45", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc46", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc47", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc48", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc49", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc50", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc51", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc52", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc53", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc54", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc55", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc56", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc57", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc58", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc59", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc60", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc61", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc62", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc63", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc64", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc65", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc66", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc67", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc68", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc69", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc70", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc71", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc72", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc73", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc74", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc75", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc76", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc77", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc78", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc79", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc80", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc81", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc82", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc83", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc84", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc85", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc86", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc87", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc88", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc89", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc90", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc91", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc92", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc93", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc94", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc95", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc96", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc97", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc98", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc99", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + } + ], + "targets": [ + 2 + ] + } + }, + { + "watchCurrent": [ + [ + 2 + ], + "resume-token-1000" + ] + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 1000 + }, + "expectedSnapshotEvents": [ + { + "added": [ + { + "key": "collection/doc0", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc2", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc3", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc4", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc5", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc6", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc7", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc8", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc9", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc10", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc11", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc12", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc13", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc14", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc15", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc16", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc17", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc18", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc19", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc20", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc21", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc22", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc23", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc24", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc25", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc26", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc27", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc28", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc29", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc30", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc31", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc32", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc33", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc34", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc35", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc36", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc37", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc38", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc39", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc40", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc41", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc42", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc43", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc44", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc45", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc46", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc47", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc48", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc49", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc50", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc51", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc52", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc53", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc54", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc55", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc56", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc57", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc58", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc59", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc60", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc61", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc62", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc63", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc64", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc65", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc66", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc67", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc68", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc69", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc70", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc71", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc72", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc73", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc74", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc75", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc76", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc77", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc78", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc79", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc80", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc81", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc82", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc83", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc84", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc85", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc86", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc87", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc88", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc89", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc90", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc91", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc92", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc93", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc94", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc95", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc96", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc97", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc98", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/doc99", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + } + ], + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ] + }, + { + "watchFilter": { + "bloomFilter": { + "bits": { + "bitmap": "+9oMQXUptl274DOaET8sfebQ4aCu0Roiddbja3z8TfadKuyPV/9XWV5Ksv+vywRXTfZSNIn8z+xk/oq1+cbOPepeNvbXVOF6H92fCOAz/KiS3Mcw338R9tXE3Y7QB1L2kbvbvVHW3Kn/k3Vx8k9Oa19eWX6RYE97Q+oCcVU=", + "padding": 0 + }, + "hashCount": 16 + }, + "keys": [ + "collection/doc0", + "collection/doc1", + "collection/doc2", + "collection/doc3", + "collection/doc4", + "collection/doc5", + "collection/doc6", + "collection/doc7", + "collection/doc8", + "collection/doc9", + "collection/doc10", + "collection/doc11", + "collection/doc12", + "collection/doc13", + "collection/doc14", + "collection/doc15", + "collection/doc16", + "collection/doc17", + "collection/doc18", + "collection/doc19", + "collection/doc20", + "collection/doc21", + "collection/doc22", + "collection/doc23", + "collection/doc24", + "collection/doc25", + "collection/doc26", + "collection/doc27", + "collection/doc28", + "collection/doc29", + "collection/doc30", + "collection/doc31", + "collection/doc32", + "collection/doc33", + "collection/doc34", + "collection/doc35", + "collection/doc36", + "collection/doc37", + "collection/doc38", + "collection/doc39", + "collection/doc40", + "collection/doc41", + "collection/doc42", + "collection/doc43", + "collection/doc44", + "collection/doc45", + "collection/doc46", + "collection/doc47", + "collection/doc48", + "collection/doc49" + ], + "targetIds": [ + 2 + ] + } + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 2000 + }, + "expectedSnapshotEvents": [ + { + "errorCode": 0, + "fromCache": true, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ], + "expectedState": { + "activeLimboDocs": [ + "collection/doc50", + "collection/doc51", + "collection/doc52", + "collection/doc53", + "collection/doc54", + "collection/doc55", + "collection/doc56", + "collection/doc57", + "collection/doc58", + "collection/doc59", + "collection/doc60", + "collection/doc61", + "collection/doc62", + "collection/doc63", + "collection/doc64", + "collection/doc65", + "collection/doc66", + "collection/doc67", + "collection/doc68", + "collection/doc69", + "collection/doc70", + "collection/doc71", + "collection/doc72", + "collection/doc73", + "collection/doc74", + "collection/doc75", + "collection/doc76", + "collection/doc77", + "collection/doc78", + "collection/doc79", + "collection/doc80", + "collection/doc81", + "collection/doc82", + "collection/doc83", + "collection/doc84", + "collection/doc85", + "collection/doc86", + "collection/doc87", + "collection/doc88", + "collection/doc89", + "collection/doc90", + "collection/doc91", + "collection/doc92", + "collection/doc93", + "collection/doc94", + "collection/doc95", + "collection/doc96", + "collection/doc97", + "collection/doc98", + "collection/doc99" + ], + "activeTargets": { + "1": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc50" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "11": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc55" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "13": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc56" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "15": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc57" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "17": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc58" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "19": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc59" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + }, + "21": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc60" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "23": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc61" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "25": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc62" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "27": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc63" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "29": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc64" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "3": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc51" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "31": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc65" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "33": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc66" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "35": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc67" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "37": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc68" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "39": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc69" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "41": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc70" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "43": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc71" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "45": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc72" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "47": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc73" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "49": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc74" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "5": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc52" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "51": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc75" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "53": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc76" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "55": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc77" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "57": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc78" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "59": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc79" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "61": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc80" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "63": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc81" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "65": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc82" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "67": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc83" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "69": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc84" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "7": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc53" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "71": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc85" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "73": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc86" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "75": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc87" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "77": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc88" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "79": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc89" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "81": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc90" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "83": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc91" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "85": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc92" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "87": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc93" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "89": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc94" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "9": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc54" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "91": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc95" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "93": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc96" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "95": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc97" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "97": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc98" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "99": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/doc99" + } + ], + "resumeToken": "", + "targetPurpose": 3 + } + } + } + } + ] + }, + "Existence filter clears resume token": { + "describeName": "Existence Filters:", + "itName": "Existence filter clears resume token", + "tags": [ + "durable-persistence" + ], + "config": { + "numClients": 1, + "useGarbageCollection": true + }, + "steps": [ + { + "userListen": { + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + }, + "targetId": 2 + }, + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + } + } + } + }, + { + "watchAck": [ + 2 + ] + }, + { + "watchEntity": { + "docs": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/2", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 1000 + } + ], + "targets": [ + 2 + ] + } + }, + { + "watchCurrent": [ + [ + 2 + ], + "resume-token-1000" + ] + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 1000 + }, + "expectedSnapshotEvents": [ + { + "added": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/2", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 1000 + } + ], + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ] + }, + { + "watchFilter": { + "keys": [ + "collection/1" + ], + "targetIds": [ + 2 + ] + } + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 2000 + }, + "expectedSnapshotEvents": [ + { + "errorCode": 0, + "fromCache": true, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ], + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "", + "targetPurpose": 1 + } + } + } + }, + { + "restart": true, + "expectedState": { + "activeLimboDocs": [ + ], + "activeTargets": { + }, + "enqueuedLimboDocs": [ + ] + } + }, + { + "userListen": { + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + }, + "targetId": 2 + }, + "expectedSnapshotEvents": [ + { + "added": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/2", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 1000 + } + ], + "errorCode": 0, + "fromCache": true, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ], + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + } + } + } + } + ] + }, + "Existence filter handled at global snapshot": { + "describeName": "Existence Filters:", + "itName": "Existence filter handled at global snapshot", + "tags": [ + ], + "config": { + "numClients": 1, + "useGarbageCollection": true + }, + "steps": [ + { + "userListen": { + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + }, + "targetId": 2 + }, + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + } + } + } + }, + { + "watchAck": [ + 2 + ] + }, + { + "watchEntity": { + "docs": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + } + ], + "targets": [ + 2 + ] + } + }, + { + "watchCurrent": [ + [ + 2 + ], + "resume-token-1000" + ] + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 1000 + }, + "expectedSnapshotEvents": [ + { + "added": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + } + ], + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ] + }, + { + "watchFilter": { + "keys": [ + "collection/1", + "collection/2" + ], + "targetIds": [ + 2 + ] + } + }, + { + "watchEntity": { + "docs": [ + { + "key": "collection/3", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 3 + }, + "version": 3000 + } + ], + "targets": [ + 2 + ] + } + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 2000 + }, + "expectedSnapshotEvents": [ + { + "added": [ + { + "key": "collection/3", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 3 + }, + "version": 3000 + } + ], + "errorCode": 0, + "fromCache": true, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ], + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "", + "targetPurpose": 1 + } + } + } + }, + { + "watchRemove": { + "targetIds": [ + 2 + ] + } + }, + { + "watchAck": [ + 2 + ] + }, + { + "watchEntity": { + "docs": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/2", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 2000 + }, + { + "key": "collection/3", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 3 + }, + "version": 3000 + } + ], + "targets": [ + 2 + ] + } + }, + { + "watchCurrent": [ + [ + 2 + ], + "resume-token-3000" + ] + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 3000 + }, + "expectedSnapshotEvents": [ + { + "added": [ + { + "key": "collection/2", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 2000 + } + ], + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ] + } + ] + }, + "Existence filter ignored with pending target": { + "describeName": "Existence Filters:", + "itName": "Existence filter ignored with pending target", + "tags": [ + ], + "config": { + "numClients": 1, + "useGarbageCollection": false + }, + "steps": [ + { + "userListen": { + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + }, + "targetId": 2 + }, + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + } + } + } + }, + { + "watchAck": [ + 2 + ] + }, + { + "watchEntity": { + "docs": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 2000 + } + ], + "targets": [ + 2 + ] + } + }, + { + "watchCurrent": [ + [ + 2 + ], + "resume-token-1000" + ] + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 1000 + }, + "expectedSnapshotEvents": [ + { + "added": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 2000 + } + ], + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ] + }, + { + "userUnlisten": [ + 2, + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "expectedState": { + "activeTargets": { + } + } + }, + { + "userListen": { + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + }, + "targetId": 2 + }, + "expectedSnapshotEvents": [ + { + "added": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 2000 + } + ], + "errorCode": 0, + "fromCache": true, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ], + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "resume-token-1000" + } + } + } + }, + { + "watchFilter": { + "keys": [ + ], + "targetIds": [ + 2 + ] + } + }, + { + "watchRemove": { + "targetIds": [ + 2 + ] + } + }, + { + "watchAck": [ + 2 + ] + }, + { + "watchCurrent": [ + [ + 2 + ], + "resume-token-2000" + ] + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 2000 + }, + "expectedSnapshotEvents": [ + { + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ] + } + ] + }, + "Existence filter limbo resolution is denied": { + "describeName": "Existence Filters:", + "itName": "Existence filter limbo resolution is denied", + "tags": [ + ], + "config": { + "numClients": 1, + "useGarbageCollection": true + }, + "steps": [ + { + "userListen": { + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + }, + "targetId": 2 + }, + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + } + } + } + }, + { + "watchAck": [ + 2 + ] + }, + { + "watchEntity": { + "docs": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/2", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 1000 + } + ], + "targets": [ + 2 + ] + } + }, + { + "watchCurrent": [ + [ + 2 + ], + "resume-token-1000" + ] + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 1000 + }, + "expectedSnapshotEvents": [ + { + "added": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/2", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 1000 + } + ], + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ] + }, + { + "watchFilter": { + "keys": [ + "collection/1" + ], + "targetIds": [ + 2 + ] + } + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 2000 + }, + "expectedSnapshotEvents": [ + { + "errorCode": 0, + "fromCache": true, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ], + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "", + "targetPurpose": 1 + } + } + } + }, + { + "watchRemove": { + "targetIds": [ + 2 + ] + } + }, + { + "watchAck": [ + 2 + ] + }, + { + "watchEntity": { + "docs": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + } + ], + "targets": [ + 2 + ] + } + }, + { + "watchCurrent": [ + [ + 2 + ], + "resume-token-2000" + ] + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 2000 + }, + "expectedState": { + "activeLimboDocs": [ + "collection/2" + ], + "activeTargets": { + "1": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/2" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "", + "targetPurpose": 1 + } + } + } + }, + { + "watchRemove": { + "cause": { + "code": 7 + }, + "targetIds": [ + 1 + ] + }, + "expectedSnapshotEvents": [ + { + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + }, + "removed": [ + { + "key": "collection/2", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 1000 + } + ] + } + ], + "expectedState": { + "activeLimboDocs": [ + ], + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "", + "targetPurpose": 1 + } + } + } + } + ] + }, + "Existence filter match": { + "describeName": "Existence Filters:", + "itName": "Existence filter match", + "tags": [ + ], + "config": { + "numClients": 1, + "useGarbageCollection": true + }, + "steps": [ + { + "userListen": { + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + }, + "targetId": 2 + }, + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + } + } + } + }, + { + "watchAck": [ + 2 + ] + }, + { + "watchEntity": { + "docs": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + } + ], + "targets": [ + 2 + ] + } + }, + { + "watchCurrent": [ + [ + 2 + ], + "resume-token-1000" + ] + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 1000 + }, + "expectedSnapshotEvents": [ + { + "added": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + } + ], + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ] + }, + { + "watchFilter": { + "keys": [ + "collection/1" + ], + "targetIds": [ + 2 + ] + } + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 2000 + } + } + ] + }, + "Existence filter match after pending update": { + "describeName": "Existence Filters:", + "itName": "Existence filter match after pending update", + "tags": [ + ], + "config": { + "numClients": 1, + "useGarbageCollection": true + }, + "steps": [ + { + "userListen": { + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + }, + "targetId": 2 + }, + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + } + } + } + }, + { + "watchAck": [ + 2 + ] + }, + { + "watchCurrent": [ + [ + 2 + ], + "resume-token-1000" + ] + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 2000 + }, + "expectedSnapshotEvents": [ + { + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ] + }, + { + "watchEntity": { + "docs": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 2000 + } + ], + "targets": [ + 2 + ] + } + }, + { + "watchFilter": { + "keys": [ + "collection/1" + ], + "targetIds": [ + 2 + ] + } + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 2000 + }, + "expectedSnapshotEvents": [ + { + "added": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 2000 + } + ], + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ] + } + ] + }, + "Existence filter mismatch triggers re-run of query": { + "describeName": "Existence Filters:", + "itName": "Existence filter mismatch triggers re-run of query", + "tags": [ + ], + "config": { + "numClients": 1, + "useGarbageCollection": true + }, + "steps": [ + { + "userListen": { + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + }, + "targetId": 2 + }, + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + } + } + } + }, + { + "watchAck": [ + 2 + ] + }, + { + "watchEntity": { + "docs": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/2", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 1000 + } + ], + "targets": [ + 2 + ] + } + }, + { + "watchCurrent": [ + [ + 2 + ], + "resume-token-1000" + ] + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 1000 + }, + "expectedSnapshotEvents": [ + { + "added": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/2", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 1000 + } + ], + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ] + }, + { + "watchFilter": { + "keys": [ + "collection/1" + ], + "targetIds": [ + 2 + ] + } + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 2000 + }, + "expectedSnapshotEvents": [ + { + "errorCode": 0, + "fromCache": true, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ], + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "", + "targetPurpose": 1 + } + } + } + }, + { + "watchRemove": { + "targetIds": [ + 2 + ] + } + }, + { + "watchAck": [ + 2 + ] + }, + { + "watchEntity": { + "docs": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + } + ], + "targets": [ + 2 + ] + } + }, + { + "watchCurrent": [ + [ + 2 + ], + "resume-token-2000" + ] + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 2000 + }, + "expectedState": { + "activeLimboDocs": [ + "collection/2" + ], + "activeTargets": { + "1": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/2" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "", + "targetPurpose": 1 + } + } + } + }, + { + "watchAck": [ + 1 + ] + }, + { + "watchCurrent": [ + [ + 1 + ], + "resume-token-2000" + ] + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 2000 + }, + "expectedSnapshotEvents": [ + { + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + }, + "removed": [ + { + "key": "collection/2", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 1000 + } + ] + } + ], + "expectedState": { + "activeLimboDocs": [ + ], + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "", + "targetPurpose": 1 + } + } + } + } + ] + }, + "Existence filter mismatch will drop resume token": { + "describeName": "Existence Filters:", + "itName": "Existence filter mismatch will drop resume token", + "tags": [ + ], + "config": { + "numClients": 1, + "useGarbageCollection": true + }, + "steps": [ + { + "userListen": { + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + }, + "targetId": 2 + }, + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + } + } + } + }, + { + "watchAck": [ + 2 + ] + }, + { + "watchEntity": { + "docs": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/2", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 1000 + } + ], + "targets": [ + 2 + ] + } + }, + { + "watchCurrent": [ + [ + 2 + ], + "existence-filter-resume-token" + ] + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 1000 + }, + "expectedSnapshotEvents": [ + { + "added": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/2", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 1000 + } + ], + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ] + }, + { + "watchStreamClose": { + "error": { + "code": 14, + "message": "Simulated Backend Error" + }, + "runBackoffTimer": true + }, + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "existence-filter-resume-token" + } + } + } + }, + { + "watchAck": [ + 2 + ] + }, + { + "watchFilter": { + "keys": [ + "collection/1" + ], + "targetIds": [ + 2 + ] + } + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 2000 + }, + "expectedSnapshotEvents": [ + { + "errorCode": 0, + "fromCache": true, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ], + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "", + "targetPurpose": 1 + } + } + } + }, + { + "watchRemove": { + "targetIds": [ + 2 + ] + } + }, + { + "watchAck": [ + 2 + ] + }, + { + "watchEntity": { + "docs": [ + { + "key": "collection/1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false }, "value": { - "v": 3 + "v": 1 }, - "version": 3000 + "version": 1000 } ], "targets": [ @@ -482,18 +5864,81 @@ [ 2 ], - "resume-token-3000" + "resume-token-2000" ] }, { "watchSnapshot": { "targetIds": [ ], - "version": 3000 + "version": 2000 + }, + "expectedState": { + "activeLimboDocs": [ + "collection/2" + ], + "activeTargets": { + "1": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/2" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "", + "targetPurpose": 1 + } + } + } + }, + { + "watchAck": [ + 1 + ] + }, + { + "watchCurrent": [ + [ + 1 + ], + "resume-token-2000" + ] + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 2000 }, "expectedSnapshotEvents": [ { - "added": [ + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + }, + "removed": [ { "key": "collection/2", "options": { @@ -503,32 +5948,41 @@ "value": { "v": 2 }, - "version": 2000 + "version": 1000 } - ], - "errorCode": 0, - "fromCache": false, - "hasPendingWrites": false, - "query": { - "filters": [ - ], - "orderBys": [ + ] + } + ], + "expectedState": { + "activeLimboDocs": [ + ], + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } ], - "path": "collection" + "resumeToken": "", + "targetPurpose": 1 } } - ] + } } ] }, - "Existence filter ignored with pending target": { + "Existence filter synthesizes deletes": { "describeName": "Existence Filters:", - "itName": "Existence filter ignored with pending target", + "itName": "Existence filter synthesizes deletes", "tags": [ ], "config": { "numClients": 1, - "useGarbageCollection": false + "useGarbageCollection": true }, "steps": [ { @@ -538,7 +5992,7 @@ ], "orderBys": [ ], - "path": "collection" + "path": "collection/a" }, "targetId": 2 }, @@ -551,7 +6005,7 @@ ], "orderBys": [ ], - "path": "collection" + "path": "collection/a" } ], "resumeToken": "" @@ -568,15 +6022,15 @@ "watchEntity": { "docs": [ { - "key": "collection/1", + "key": "collection/a", "options": { "hasCommittedMutations": false, "hasLocalMutations": false }, "value": { - "v": 2 + "v": 1 }, - "version": 2000 + "version": 1000 } ], "targets": [ @@ -602,15 +6056,15 @@ { "added": [ { - "key": "collection/1", + "key": "collection/a", "options": { "hasCommittedMutations": false, "hasLocalMutations": false }, "value": { - "v": 2 + "v": 1 }, - "version": 2000 + "version": 1000 } ], "errorCode": 0, @@ -621,65 +6075,77 @@ ], "orderBys": [ ], - "path": "collection" + "path": "collection/a" } } ] }, { - "userUnlisten": [ - 2, - { - "filters": [ - ], - "orderBys": [ - ], - "path": "collection" - } - ], - "expectedState": { - "activeTargets": { - } + "watchFilter": { + "keys": [ + ], + "targetIds": [ + 2 + ] } }, { - "userListen": { - "query": { - "filters": [ - ], - "orderBys": [ - ], - "path": "collection" - }, - "targetId": 2 + "watchSnapshot": { + "targetIds": [ + ], + "version": 2000 }, "expectedSnapshotEvents": [ { - "added": [ + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/a" + }, + "removed": [ { - "key": "collection/1", + "key": "collection/a", "options": { "hasCommittedMutations": false, "hasLocalMutations": false }, "value": { - "v": 2 + "v": 1 }, - "version": 2000 + "version": 1000 } - ], - "errorCode": 0, - "fromCache": true, - "hasPendingWrites": false, - "query": { - "filters": [ - ], - "orderBys": [ - ], - "path": "collection" - } + ] } - ], + ] + } + ] + }, + "Existence filter with empty target": { + "describeName": "Existence Filters:", + "itName": "Existence filter with empty target", + "tags": [ + ], + "config": { + "numClients": 1, + "useGarbageCollection": true + }, + "steps": [ + { + "userListen": { + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + }, + "targetId": 2 + }, "expectedState": { "activeTargets": { "2": { @@ -692,39 +6158,54 @@ "path": "collection" } ], - "resumeToken": "resume-token-1000" + "resumeToken": "" } } } }, { - "watchFilter": { - "keys": [ - ], - "targetIds": [ - 2 - ] - } + "watchAck": [ + 2 + ] }, { - "watchRemove": { - "targetIds": [ + "watchCurrent": [ + [ 2 - ] - } + ], + "resume-token-1000" + ] }, { - "watchAck": [ - 2 + "watchSnapshot": { + "targetIds": [ + ], + "version": 2000 + }, + "expectedSnapshotEvents": [ + { + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } ] }, { - "watchCurrent": [ - [ - 2 + "watchFilter": { + "keys": [ + "collection/1" ], - "resume-token-2000" - ] + "targetIds": [ + 2 + ] + } }, { "watchSnapshot": { @@ -735,7 +6216,7 @@ "expectedSnapshotEvents": [ { "errorCode": 0, - "fromCache": false, + "fromCache": true, "hasPendingWrites": false, "query": { "filters": [ @@ -745,13 +6226,30 @@ "path": "collection" } } - ] + ], + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "", + "targetPurpose": 1 + } + } + } } ] }, - "Existence filter limbo resolution is denied": { + "Full re-query is skipped when bloom filter can identify documents deleted": { "describeName": "Existence Filters:", - "itName": "Existence filter limbo resolution is denied", + "itName": "Full re-query is skipped when bloom filter can identify documents deleted", "tags": [ ], "config": { @@ -796,7 +6294,7 @@ "watchEntity": { "docs": [ { - "key": "collection/1", + "key": "collection/a", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -807,7 +6305,7 @@ "version": 1000 }, { - "key": "collection/2", + "key": "collection/b", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -841,7 +6339,7 @@ { "added": [ { - "key": "collection/1", + "key": "collection/a", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -852,7 +6350,7 @@ "version": 1000 }, { - "key": "collection/2", + "key": "collection/b", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -878,8 +6376,15 @@ }, { "watchFilter": { + "bloomFilter": { + "bits": { + "bitmap": "AhAAApAAAIAEBIAABA==", + "padding": 4 + }, + "hashCount": 10 + }, "keys": [ - "collection/1" + "collection/a" ], "targetIds": [ 2 @@ -906,72 +6411,9 @@ } } ], - "expectedState": { - "activeTargets": { - "2": { - "queries": [ - { - "filters": [ - ], - "orderBys": [ - ], - "path": "collection" - } - ], - "resumeToken": "" - } - } - } - }, - { - "watchRemove": { - "targetIds": [ - 2 - ] - } - }, - { - "watchAck": [ - 2 - ] - }, - { - "watchEntity": { - "docs": [ - { - "key": "collection/1", - "options": { - "hasCommittedMutations": false, - "hasLocalMutations": false - }, - "value": { - "v": 1 - }, - "version": 1000 - } - ], - "targets": [ - 2 - ] - } - }, - { - "watchCurrent": [ - [ - 2 - ], - "resume-token-2000" - ] - }, - { - "watchSnapshot": { - "targetIds": [ - ], - "version": 2000 - }, "expectedState": { "activeLimboDocs": [ - "collection/2" + "collection/b" ], "activeTargets": { "1": { @@ -981,106 +6423,12 @@ ], "orderBys": [ ], - "path": "collection/2" - } - ], - "resumeToken": "" - }, - "2": { - "queries": [ - { - "filters": [ - ], - "orderBys": [ - ], - "path": "collection" + "path": "collection/b" } ], - "resumeToken": "" - } - } - } - }, - { - "watchRemove": { - "cause": { - "code": 7 - }, - "targetIds": [ - 1 - ] - }, - "expectedSnapshotEvents": [ - { - "errorCode": 0, - "fromCache": false, - "hasPendingWrites": false, - "query": { - "filters": [ - ], - "orderBys": [ - ], - "path": "collection" + "resumeToken": "", + "targetPurpose": 3 }, - "removed": [ - { - "key": "collection/2", - "options": { - "hasCommittedMutations": false, - "hasLocalMutations": false - }, - "value": { - "v": 2 - }, - "version": 1000 - } - ] - } - ], - "expectedState": { - "activeLimboDocs": [ - ], - "activeTargets": { - "2": { - "queries": [ - { - "filters": [ - ], - "orderBys": [ - ], - "path": "collection" - } - ], - "resumeToken": "" - } - } - } - } - ] - }, - "Existence filter match": { - "describeName": "Existence Filters:", - "itName": "Existence filter match", - "tags": [ - ], - "config": { - "numClients": 1, - "useGarbageCollection": true - }, - "steps": [ - { - "userListen": { - "query": { - "filters": [ - ], - "orderBys": [ - ], - "path": "collection" - }, - "targetId": 2 - }, - "expectedState": { - "activeTargets": { "2": { "queries": [ { @@ -1098,58 +6446,25 @@ }, { "watchAck": [ - 2 + 1 ] }, - { - "watchEntity": { - "docs": [ - { - "key": "collection/1", - "options": { - "hasCommittedMutations": false, - "hasLocalMutations": false - }, - "value": { - "v": 1 - }, - "version": 1000 - } - ], - "targets": [ - 2 - ] - } - }, { "watchCurrent": [ [ - 2 + 1 ], - "resume-token-1000" + "resume-token-2000" ] }, { "watchSnapshot": { "targetIds": [ ], - "version": 1000 + "version": 2000 }, "expectedSnapshotEvents": [ { - "added": [ - { - "key": "collection/1", - "options": { - "hasCommittedMutations": false, - "hasLocalMutations": false - }, - "value": { - "v": 1 - }, - "version": 1000 - } - ], "errorCode": 0, "fromCache": false, "hasPendingWrites": false, @@ -1158,34 +6473,50 @@ ], "orderBys": [ ], - "path": "collection" + "path": "collection" + }, + "removed": [ + { + "key": "collection/b", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 1000 + } + ] + } + ], + "expectedState": { + "activeLimboDocs": [ + ], + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" } } - ] - }, - { - "watchFilter": { - "keys": [ - "collection/1" - ], - "targetIds": [ - 2 - ] - } - }, - { - "watchSnapshot": { - "targetIds": [ - ], - "version": 2000 } } ] }, - "Existence filter match after pending update": { + "Full re-query is triggered when bloom filter bitmap is invalid": { "describeName": "Existence Filters:", - "itName": "Existence filter match after pending update", + "itName": "Full re-query is triggered when bloom filter bitmap is invalid", "tags": [ + "no-ios", + "no-android" ], "config": { "numClients": 1, @@ -1225,6 +6556,37 @@ 2 ] }, + { + "watchEntity": { + "docs": [ + { + "key": "collection/a", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/b", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + } + ], + "targets": [ + 2 + ] + } + }, { "watchCurrent": [ [ @@ -1237,10 +6599,34 @@ "watchSnapshot": { "targetIds": [ ], - "version": 2000 + "version": 1000 }, "expectedSnapshotEvents": [ { + "added": [ + { + "key": "collection/a", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/b", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + } + ], "errorCode": 0, "fromCache": false, "hasPendingWrites": false, @@ -1254,30 +6640,17 @@ } ] }, - { - "watchEntity": { - "docs": [ - { - "key": "collection/1", - "options": { - "hasCommittedMutations": false, - "hasLocalMutations": false - }, - "value": { - "v": 2 - }, - "version": 2000 - } - ], - "targets": [ - 2 - ] - } - }, { "watchFilter": { + "bloomFilter": { + "bits": { + "bitmap": "INVALID_BASE_64", + "padding": 4 + }, + "hashCount": 10 + }, "keys": [ - "collection/1" + "collection/a" ], "targetIds": [ 2 @@ -1292,21 +6665,8 @@ }, "expectedSnapshotEvents": [ { - "added": [ - { - "key": "collection/1", - "options": { - "hasCommittedMutations": false, - "hasLocalMutations": false - }, - "value": { - "v": 2 - }, - "version": 2000 - } - ], "errorCode": 0, - "fromCache": false, + "fromCache": true, "hasPendingWrites": false, "query": { "filters": [ @@ -1316,13 +6676,30 @@ "path": "collection" } } - ] + ], + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "", + "targetPurpose": 1 + } + } + } } ] }, - "Existence filter mismatch triggers re-run of query": { + "Full re-query is triggered when bloom filter can not identify documents deleted": { "describeName": "Existence Filters:", - "itName": "Existence filter mismatch triggers re-run of query", + "itName": "Full re-query is triggered when bloom filter can not identify documents deleted", "tags": [ ], "config": { @@ -1367,7 +6744,7 @@ "watchEntity": { "docs": [ { - "key": "collection/1", + "key": "collection/a", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -1378,7 +6755,18 @@ "version": 1000 }, { - "key": "collection/2", + "key": "collection/b", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 1000 + }, + { + "key": "collection/c", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -1412,7 +6800,7 @@ { "added": [ { - "key": "collection/1", + "key": "collection/a", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -1423,7 +6811,18 @@ "version": 1000 }, { - "key": "collection/2", + "key": "collection/b", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 1000 + }, + { + "key": "collection/c", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -1449,8 +6848,15 @@ }, { "watchFilter": { + "bloomFilter": { + "bits": { + "bitmap": "AxBIApBIAIAWBoCQBA==", + "padding": 4 + }, + "hashCount": 10 + }, "keys": [ - "collection/1" + "collection/a" ], "targetIds": [ 2 @@ -1489,16 +6895,50 @@ "path": "collection" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 2 } } } - }, + } + ] + }, + "Full re-query is triggered when bloom filter hashCount is invalid": { + "describeName": "Existence Filters:", + "itName": "Full re-query is triggered when bloom filter hashCount is invalid", + "tags": [ + ], + "config": { + "numClients": 1, + "useGarbageCollection": true + }, + "steps": [ { - "watchRemove": { - "targetIds": [ - 2 - ] + "userListen": { + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + }, + "targetId": 2 + }, + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + } + } } }, { @@ -1510,7 +6950,18 @@ "watchEntity": { "docs": [ { - "key": "collection/1", + "key": "collection/a", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/b", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -1531,59 +6982,70 @@ [ 2 ], - "resume-token-2000" + "resume-token-1000" ] }, { "watchSnapshot": { "targetIds": [ ], - "version": 2000 + "version": 1000 }, - "expectedState": { - "activeLimboDocs": [ - "collection/2" - ], - "activeTargets": { - "1": { - "queries": [ - { - "filters": [ - ], - "orderBys": [ - ], - "path": "collection/2" - } + "expectedSnapshotEvents": [ + { + "added": [ + { + "key": "collection/a", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/b", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + } + ], + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ ], - "resumeToken": "" - }, - "2": { - "queries": [ - { - "filters": [ - ], - "orderBys": [ - ], - "path": "collection" - } + "orderBys": [ ], - "resumeToken": "" + "path": "collection" } } - } - }, - { - "watchAck": [ - 1 ] }, { - "watchCurrent": [ - [ - 1 + "watchFilter": { + "bloomFilter": { + "bits": { + "bitmap": "AhAAApAAAIAEBIAABA==", + "padding": 4 + }, + "hashCount": -1 + }, + "keys": [ + "collection/a" ], - "resume-token-2000" - ] + "targetIds": [ + 2 + ] + } }, { "watchSnapshot": { @@ -1594,7 +7056,7 @@ "expectedSnapshotEvents": [ { "errorCode": 0, - "fromCache": false, + "fromCache": true, "hasPendingWrites": false, "query": { "filters": [ @@ -1602,25 +7064,10 @@ "orderBys": [ ], "path": "collection" - }, - "removed": [ - { - "key": "collection/2", - "options": { - "hasCommittedMutations": false, - "hasLocalMutations": false - }, - "value": { - "v": 2 - }, - "version": 1000 - } - ] + } } ], "expectedState": { - "activeLimboDocs": [ - ], "activeTargets": { "2": { "queries": [ @@ -1632,16 +7079,17 @@ "path": "collection" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 1 } } } } ] }, - "Existence filter mismatch will drop resume token": { + "Full re-query is triggered when bloom filter is empty": { "describeName": "Existence Filters:", - "itName": "Existence filter mismatch will drop resume token", + "itName": "Full re-query is triggered when bloom filter is empty", "tags": [ ], "config": { @@ -1686,7 +7134,7 @@ "watchEntity": { "docs": [ { - "key": "collection/1", + "key": "collection/a", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -1697,13 +7145,13 @@ "version": 1000 }, { - "key": "collection/2", + "key": "collection/b", "options": { "hasCommittedMutations": false, "hasLocalMutations": false }, "value": { - "v": 2 + "v": 1 }, "version": 1000 } @@ -1718,7 +7166,7 @@ [ 2 ], - "existence-filter-resume-token" + "resume-token-1000" ] }, { @@ -1731,7 +7179,7 @@ { "added": [ { - "key": "collection/1", + "key": "collection/a", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -1742,13 +7190,13 @@ "version": 1000 }, { - "key": "collection/2", + "key": "collection/b", "options": { "hasCommittedMutations": false, "hasLocalMutations": false }, "value": { - "v": 2 + "v": 1 }, "version": 1000 } @@ -1766,40 +7214,17 @@ } ] }, - { - "watchStreamClose": { - "error": { - "code": 14, - "message": "Simulated Backend Error" - }, - "runBackoffTimer": true - }, - "expectedState": { - "activeTargets": { - "2": { - "queries": [ - { - "filters": [ - ], - "orderBys": [ - ], - "path": "collection" - } - ], - "resumeToken": "existence-filter-resume-token" - } - } - } - }, - { - "watchAck": [ - 2 - ] - }, { "watchFilter": { + "bloomFilter": { + "bits": { + "bitmap": "", + "padding": 0 + }, + "hashCount": 0 + }, "keys": [ - "collection/1" + "collection/a" ], "targetIds": [ 2 @@ -1838,16 +7263,60 @@ "path": "collection" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 1 } } } - }, + } + ] + }, + "Same documents can have different bloom filters": { + "describeName": "Existence Filters:", + "itName": "Same documents can have different bloom filters", + "tags": [ + ], + "config": { + "numClients": 1, + "useGarbageCollection": true + }, + "steps": [ { - "watchRemove": { - "targetIds": [ - 2 - ] + "userListen": { + "query": { + "filters": [ + [ + "v", + "<=", + 2 + ] + ], + "orderBys": [ + ], + "path": "collection" + }, + "targetId": 2 + }, + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + [ + "v", + "<=", + 2 + ] + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + } + } } }, { @@ -1859,7 +7328,7 @@ "watchEntity": { "docs": [ { - "key": "collection/1", + "key": "collection/a", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -1868,6 +7337,17 @@ "v": 1 }, "version": 1000 + }, + { + "key": "collection/b", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 1000 } ], "targets": [ @@ -1880,81 +7360,80 @@ [ 2 ], - "resume-token-2000" - ] - }, - { - "watchSnapshot": { - "targetIds": [ - ], - "version": 2000 - }, - "expectedState": { - "activeLimboDocs": [ - "collection/2" - ], - "activeTargets": { - "1": { - "queries": [ - { - "filters": [ - ], - "orderBys": [ - ], - "path": "collection/2" - } - ], - "resumeToken": "" - }, - "2": { - "queries": [ - { - "filters": [ - ], - "orderBys": [ - ], - "path": "collection" - } - ], - "resumeToken": "" - } - } - } - }, - { - "watchAck": [ - 1 - ] - }, - { - "watchCurrent": [ - [ - 1 - ], - "resume-token-2000" + "resume-token-1000" ] }, { "watchSnapshot": { "targetIds": [ ], - "version": 2000 + "version": 1000 }, "expectedSnapshotEvents": [ { + "added": [ + { + "key": "collection/a", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 1 + }, + "version": 1000 + }, + { + "key": "collection/b", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 2 + }, + "version": 1000 + } + ], "errorCode": 0, "fromCache": false, "hasPendingWrites": false, "query": { "filters": [ + [ + "v", + "<=", + 2 + ] ], "orderBys": [ ], "path": "collection" - }, - "removed": [ + } + } + ] + }, + { + "userListen": { + "query": { + "filters": [ + [ + "v", + ">=", + 2 + ] + ], + "orderBys": [ + ], + "path": "collection" + }, + "targetId": 4 + }, + "expectedSnapshotEvents": [ + { + "added": [ { - "key": "collection/2", + "key": "collection/b", "options": { "hasCommittedMutations": false, "hasLocalMutations": false @@ -1964,17 +7443,35 @@ }, "version": 1000 } - ] + ], + "errorCode": 0, + "fromCache": true, + "hasPendingWrites": false, + "query": { + "filters": [ + [ + "v", + ">=", + 2 + ] + ], + "orderBys": [ + ], + "path": "collection" + } } ], "expectedState": { - "activeLimboDocs": [ - ], "activeTargets": { "2": { "queries": [ { "filters": [ + [ + "v", + "<=", + 2 + ] ], "orderBys": [ ], @@ -1982,43 +7479,20 @@ } ], "resumeToken": "" - } - } - } - } - ] - }, - "Existence filter synthesizes deletes": { - "describeName": "Existence Filters:", - "itName": "Existence filter synthesizes deletes", - "tags": [ - ], - "config": { - "numClients": 1, - "useGarbageCollection": true - }, - "steps": [ - { - "userListen": { - "query": { - "filters": [ - ], - "orderBys": [ - ], - "path": "collection/a" - }, - "targetId": 2 - }, - "expectedState": { - "activeTargets": { - "2": { + }, + "4": { "queries": [ { "filters": [ + [ + "v", + ">=", + 2 + ] ], "orderBys": [ ], - "path": "collection/a" + "path": "collection" } ], "resumeToken": "" @@ -2028,54 +7502,65 @@ }, { "watchAck": [ - 2 + 4 ] }, { "watchEntity": { "docs": [ { - "key": "collection/a", + "key": "collection/b", "options": { "hasCommittedMutations": false, "hasLocalMutations": false }, "value": { - "v": 1 + "v": 2 + }, + "version": 1000 + }, + { + "key": "collection/c", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "v": 3 }, "version": 1000 } ], "targets": [ - 2 + 4 ] } }, { "watchCurrent": [ [ - 2 + 4 ], - "resume-token-1000" + "resume-token-1001" ] }, { "watchSnapshot": { "targetIds": [ ], - "version": 1000 + "version": 1001 }, "expectedSnapshotEvents": [ { "added": [ { - "key": "collection/a", + "key": "collection/c", "options": { "hasCommittedMutations": false, "hasLocalMutations": false }, "value": { - "v": 1 + "v": 3 }, "version": 1000 } @@ -2085,17 +7570,30 @@ "hasPendingWrites": false, "query": { "filters": [ + [ + "v", + ">=", + 2 + ] ], "orderBys": [ ], - "path": "collection/a" + "path": "collection" } } ] }, { "watchFilter": { + "bloomFilter": { + "bits": { + "bitmap": "CQ==", + "padding": 3 + }, + "hashCount": 2 + }, "keys": [ + "collection/b" ], "targetIds": [ 2 @@ -2111,60 +7609,49 @@ "expectedSnapshotEvents": [ { "errorCode": 0, - "fromCache": false, + "fromCache": true, "hasPendingWrites": false, "query": { "filters": [ + [ + "v", + "<=", + 2 + ] ], "orderBys": [ ], - "path": "collection/a" - }, - "removed": [ - { - "key": "collection/a", - "options": { - "hasCommittedMutations": false, - "hasLocalMutations": false - }, - "value": { - "v": 1 - }, - "version": 1000 - } - ] + "path": "collection" + } } - ] - } - ] - }, - "Existence filter with empty target": { - "describeName": "Existence Filters:", - "itName": "Existence filter with empty target", - "tags": [ - ], - "config": { - "numClients": 1, - "useGarbageCollection": true - }, - "steps": [ - { - "userListen": { - "query": { - "filters": [ - ], - "orderBys": [ - ], - "path": "collection" - }, - "targetId": 2 - }, + ], "expectedState": { + "activeLimboDocs": [ + "collection/a" + ], "activeTargets": { + "1": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/a" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, "2": { "queries": [ { "filters": [ + [ + "v", + "<=", + 2 + ] ], "orderBys": [ ], @@ -2172,51 +7659,41 @@ } ], "resumeToken": "" - } - } - } - }, - { - "watchAck": [ - 2 - ] - }, - { - "watchCurrent": [ - [ - 2 - ], - "resume-token-1000" - ] - }, - { - "watchSnapshot": { - "targetIds": [ - ], - "version": 2000 - }, - "expectedSnapshotEvents": [ - { - "errorCode": 0, - "fromCache": false, - "hasPendingWrites": false, - "query": { - "filters": [ - ], - "orderBys": [ + }, + "4": { + "queries": [ + { + "filters": [ + [ + "v", + ">=", + 2 + ] + ], + "orderBys": [ + ], + "path": "collection" + } ], - "path": "collection" + "resumeToken": "" } } - ] + } }, { "watchFilter": { + "bloomFilter": { + "bits": { + "bitmap": "CA==", + "padding": 4 + }, + "hashCount": 1 + }, "keys": [ - "collection/1" + "collection/b" ], "targetIds": [ - 2 + 4 ] } }, @@ -2224,7 +7701,7 @@ "watchSnapshot": { "targetIds": [ ], - "version": 2000 + "version": 3000 }, "expectedSnapshotEvents": [ { @@ -2233,13 +7710,86 @@ "hasPendingWrites": false, "query": { "filters": [ + [ + "v", + ">=", + 2 + ] ], "orderBys": [ ], "path": "collection" } } - ] + ], + "expectedState": { + "activeLimboDocs": [ + "collection/a", + "collection/c" + ], + "activeTargets": { + "1": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/a" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "2": { + "queries": [ + { + "filters": [ + [ + "v", + "<=", + 2 + ] + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + }, + "3": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/c" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "4": { + "queries": [ + { + "filters": [ + [ + "v", + ">=", + 2 + ] + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + } + } + } } ] } diff --git a/Firestore/Example/Tests/SpecTests/json/limbo_spec_test.json b/Firestore/Example/Tests/SpecTests/json/limbo_spec_test.json index 214a9940fc9..de82951040a 100644 --- a/Firestore/Example/Tests/SpecTests/json/limbo_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/limbo_spec_test.json @@ -234,7 +234,8 @@ "path": "collection1/doc" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -279,7 +280,8 @@ "path": "collection1/doc" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -403,7 +405,8 @@ "path": "collection1/doc" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -485,7 +488,8 @@ "path": "collection1/doc" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -567,7 +571,8 @@ "path": "collection1/doc" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -669,7 +674,8 @@ "path": "collection1/doc" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "10": { "queries": [ @@ -768,7 +774,8 @@ "path": "collection1/doc" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "10": { "queries": [ @@ -858,7 +865,8 @@ "path": "collection1/doc" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -931,7 +939,8 @@ "path": "collection1/doc" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -1192,7 +1201,8 @@ "path": "collection1/doc1" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -1237,7 +1247,8 @@ "path": "collection1/doc1" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -1361,7 +1372,8 @@ "path": "collection1/doc1" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -1443,7 +1455,8 @@ "path": "collection1/doc1" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -1525,7 +1538,8 @@ "path": "collection1/doc1" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -1627,7 +1641,8 @@ "path": "collection1/doc1" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "10": { "queries": [ @@ -1726,7 +1741,8 @@ "path": "collection1/doc1" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "10": { "queries": [ @@ -2021,7 +2037,8 @@ "path": "collection/doc" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -2105,7 +2122,8 @@ "path": "collection/doc" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -2187,7 +2205,8 @@ "path": "collection/doc" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -2400,7 +2419,8 @@ "path": "collection/b" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ @@ -2813,7 +2833,8 @@ "path": "collection/a" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -3097,7 +3118,8 @@ "path": "collection/b" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ @@ -3363,7 +3385,8 @@ "path": "collection/a" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ @@ -3609,7 +3632,8 @@ "path": "collection/a" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ @@ -3896,7 +3920,8 @@ "path": "collection/b" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ @@ -4011,7 +4036,8 @@ "path": "collection/b" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ @@ -4271,7 +4297,8 @@ "path": "collection/a" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ @@ -4618,7 +4645,8 @@ "path": "collection/a" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ @@ -4996,7 +5024,8 @@ "path": "collection/b" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ @@ -5084,7 +5113,8 @@ "path": "collection/b" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ @@ -5455,7 +5485,8 @@ "path": "collection/b" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ @@ -5479,7 +5510,8 @@ "path": "collection/c" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 } } } @@ -5590,7 +5622,8 @@ "path": "collection/b" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ @@ -5614,7 +5647,8 @@ "path": "collection/c" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 } } } @@ -5668,7 +5702,8 @@ "path": "collection/c" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 } } } @@ -5786,7 +5821,8 @@ "path": "collection/c" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 } } } @@ -6218,7 +6254,8 @@ "path": "collection/b" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -6744,7 +6781,8 @@ "path": "collection/b" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -7070,7 +7108,8 @@ "path": "collection/a" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ @@ -7153,7 +7192,8 @@ "path": "collection/b" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -7471,7 +7511,8 @@ "path": "collection/a" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ @@ -7495,7 +7536,8 @@ "path": "collection/b" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -7603,7 +7645,8 @@ "path": "collection/c" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "7": { "queries": [ @@ -7615,7 +7658,8 @@ "path": "collection/d" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -7720,7 +7764,8 @@ "path": "collection/e" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -7796,6 +7841,388 @@ } ] }, + "Limbo resolution throttling with bloom filter application": { + "describeName": "Limbo Documents:", + "itName": "Limbo resolution throttling with bloom filter application", + "tags": [ + ], + "config": { + "maxConcurrentLimboResolutions": 2, + "numClients": 1, + "useGarbageCollection": true + }, + "steps": [ + { + "userListen": { + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + }, + "targetId": 2 + }, + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "" + } + } + } + }, + { + "watchAck": [ + 2 + ] + }, + { + "watchEntity": { + "docs": [ + { + "key": "collection/a1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "key": "a1" + }, + "version": 1000 + }, + { + "key": "collection/a2", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "key": "a2" + }, + "version": 1000 + }, + { + "key": "collection/a3", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "key": "a3" + }, + "version": 1000 + } + ], + "targets": [ + 2 + ] + } + }, + { + "watchCurrent": [ + [ + 2 + ], + "resume-token-1000" + ] + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 1000 + }, + "expectedSnapshotEvents": [ + { + "added": [ + { + "key": "collection/a1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "key": "a1" + }, + "version": 1000 + }, + { + "key": "collection/a2", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "key": "a2" + }, + "version": 1000 + }, + { + "key": "collection/a3", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "key": "a3" + }, + "version": 1000 + } + ], + "errorCode": 0, + "fromCache": false, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ] + }, + { + "enableNetwork": false, + "expectedSnapshotEvents": [ + { + "errorCode": 0, + "fromCache": true, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ], + "expectedState": { + "activeLimboDocs": [ + ], + "activeTargets": { + }, + "enqueuedLimboDocs": [ + ] + } + }, + { + "enableNetwork": true, + "expectedState": { + "activeTargets": { + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "resume-token-1000" + } + } + } + }, + { + "watchAck": [ + 2 + ] + }, + { + "watchEntity": { + "docs": [ + { + "key": "collection/b1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "key": "b1" + }, + "version": 1000 + }, + { + "key": "collection/b2", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "key": "b2" + }, + "version": 1000 + }, + { + "key": "collection/b3", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "key": "b3" + }, + "version": 1000 + } + ], + "targets": [ + 2 + ] + } + }, + { + "watchFilter": { + "bloomFilter": { + "bits": { + "bitmap": "yABCEAeZURNRGAkgAQ==", + "padding": 4 + }, + "hashCount": 10 + }, + "keys": [ + "collection/b1", + "collection/b2", + "collection/b3" + ], + "targetIds": [ + 2 + ] + } + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 1001 + }, + "expectedSnapshotEvents": [ + { + "added": [ + { + "key": "collection/b1", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "key": "b1" + }, + "version": 1000 + }, + { + "key": "collection/b2", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "key": "b2" + }, + "version": 1000 + }, + { + "key": "collection/b3", + "options": { + "hasCommittedMutations": false, + "hasLocalMutations": false + }, + "value": { + "key": "b3" + }, + "version": 1000 + } + ], + "errorCode": 0, + "fromCache": true, + "hasPendingWrites": false, + "query": { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + } + ] + }, + { + "watchCurrent": [ + [ + 2 + ], + "resume-token-1002" + ] + }, + { + "watchSnapshot": { + "targetIds": [ + ], + "version": 1002 + }, + "expectedState": { + "activeLimboDocs": [ + "collection/a1", + "collection/a2" + ], + "activeTargets": { + "1": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/a1" + } + ], + "resumeToken": "", + "targetPurpose": 3 + }, + "2": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection" + } + ], + "resumeToken": "resume-token-1000" + }, + "3": { + "queries": [ + { + "filters": [ + ], + "orderBys": [ + ], + "path": "collection/a2" + } + ], + "resumeToken": "", + "targetPurpose": 3 + } + }, + "enqueuedLimboDocs": [ + "collection/a3" + ] + } + } + ] + }, "Limbo resolution throttling with existence filter mismatch": { "describeName": "Limbo Documents:", "itName": "Limbo resolution throttling with existence filter mismatch", @@ -8116,7 +8543,8 @@ "path": "collection" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 1 } } } @@ -8205,7 +8633,8 @@ "path": "collection/a1" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ @@ -8217,7 +8646,8 @@ "path": "collection" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 1 }, "3": { "queries": [ @@ -8229,7 +8659,8 @@ "path": "collection/a2" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -8322,7 +8753,8 @@ "path": "collection" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 1 }, "5": { "queries": [ @@ -8334,7 +8766,8 @@ "path": "collection/a3" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -8401,7 +8834,8 @@ "path": "collection" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 1 } }, "enqueuedLimboDocs": [ @@ -8662,7 +9096,8 @@ "path": "collection/a" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ @@ -8686,7 +9121,8 @@ "path": "collection/b" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -8775,7 +9211,8 @@ "path": "collection/b" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "5": { "queries": [ @@ -8787,7 +9224,8 @@ "path": "collection/c" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -8870,7 +9308,8 @@ "path": "collection/c" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "7": { "queries": [ @@ -8882,7 +9321,8 @@ "path": "collection/d" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -8964,7 +9404,8 @@ "path": "collection/d" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "9": { "queries": [ @@ -8976,7 +9417,8 @@ "path": "collection/e" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -9056,7 +9498,8 @@ "path": "collection/e" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -9427,7 +9870,8 @@ "path": "collection/c" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ diff --git a/Firestore/Example/Tests/SpecTests/json/limit_spec_test.json b/Firestore/Example/Tests/SpecTests/json/limit_spec_test.json index 856e0505d91..ed2b461ce74 100644 --- a/Firestore/Example/Tests/SpecTests/json/limit_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/limit_spec_test.json @@ -213,7 +213,8 @@ "path": "collection/a" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ @@ -4332,7 +4333,8 @@ "path": "collection/a" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ @@ -4358,7 +4360,8 @@ "path": "collection/b" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -4491,7 +4494,8 @@ "path": "collection/b" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -4515,7 +4519,8 @@ "path": "collection/c" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 } } } @@ -4655,7 +4660,8 @@ "path": "collection/c" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "7": { "queries": [ @@ -4667,7 +4673,8 @@ "path": "collection/d" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 } } } @@ -4806,7 +4813,8 @@ "path": "collection/d" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 } } } @@ -5295,7 +5303,8 @@ "path": "collection/a" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -5488,7 +5497,8 @@ "path": "collection" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 1 } } } @@ -5554,7 +5564,8 @@ "path": "collection/a" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ @@ -5572,7 +5583,8 @@ "path": "collection" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 1 } } } @@ -5662,7 +5674,8 @@ "path": "collection" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 1 } } } diff --git a/Firestore/Example/Tests/SpecTests/json/listen_spec_test.json b/Firestore/Example/Tests/SpecTests/json/listen_spec_test.json index 85170a91d71..a126c226530 100644 --- a/Firestore/Example/Tests/SpecTests/json/listen_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/listen_spec_test.json @@ -3481,8 +3481,7 @@ "path": "collection" } ], - "resumeToken": "resume-token-1000", - "expectedCount": 1 + "resumeToken": "resume-token-1000" } } } @@ -12751,8 +12750,7 @@ "path": "collection" } ], - "resumeToken": "resume-token-1000", - "expectedCount": 1 + "resumeToken": "resume-token-1000" } } } @@ -12900,8 +12898,7 @@ "path": "collection" } ], - "resumeToken": "resume-token-1000", - "expectedCount": 0 + "resumeToken": "resume-token-1000" } } } @@ -13072,8 +13069,7 @@ "path": "collection" } ], - "resumeToken": "resume-token-2000", - "expectedCount": 2 + "resumeToken": "resume-token-2000" } } } diff --git a/Firestore/Example/Tests/SpecTests/json/offline_spec_test.json b/Firestore/Example/Tests/SpecTests/json/offline_spec_test.json index eb8a35f3074..91e4205e441 100644 --- a/Firestore/Example/Tests/SpecTests/json/offline_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/offline_spec_test.json @@ -1149,7 +1149,8 @@ "path": "collection/a" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ @@ -1186,7 +1187,8 @@ "path": "collection/a" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "2": { "queries": [ diff --git a/Firestore/Example/Tests/SpecTests/json/recovery_spec_test.json b/Firestore/Example/Tests/SpecTests/json/recovery_spec_test.json index 9ae9f2349ff..2e66d63b8d5 100644 --- a/Firestore/Example/Tests/SpecTests/json/recovery_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/recovery_spec_test.json @@ -3094,7 +3094,8 @@ "path": "collection/key1" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -3182,7 +3183,8 @@ "path": "collection/key1" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -3566,7 +3568,8 @@ "path": "collection/key1" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ @@ -3625,7 +3628,8 @@ "path": "collection/key1" } ], - "resumeToken": "" + "resumeToken": "", + "targetPurpose": 3 }, "4": { "queries": [ From 05caaebd40acd70c01f1f4df2ff9d269add5e535 Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Mon, 13 Mar 2023 16:07:04 -0700 Subject: [PATCH 09/32] change unchanged_names to Bloom_filter --- Firestore/core/src/remote/existence_filter.h | 13 ++++++------- Firestore/core/test/unit/local/local_store_test.cc | 2 +- .../core/test/unit/remote/remote_event_test.cc | 4 ++-- Firestore/core/test/unit/remote/serializer_test.cc | 2 +- .../core/test/unit/remote/watch_change_test.cc | 2 +- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/Firestore/core/src/remote/existence_filter.h b/Firestore/core/src/remote/existence_filter.h index 5e7532e35b0..968341df588 100644 --- a/Firestore/core/src/remote/existence_filter.h +++ b/Firestore/core/src/remote/existence_filter.h @@ -29,26 +29,25 @@ class ExistenceFilter { public: ExistenceFilter() = default; - ExistenceFilter(int count, absl::optional unchangedNames) - : count_{count}, unchanged_names_{std::move(unchangedNames)} { + ExistenceFilter(int count, absl::optional bloom_filter) + : count_{count}, bloom_filter_{std::move(bloom_filter)} { } int count() const { return count_; } - const absl::optional& unchanged_names() const { - return unchanged_names_; + const absl::optional& bloom_filter() const { + return bloom_filter_; } private: int count_ = 0; - absl::optional unchanged_names_; + absl::optional bloom_filter_; }; inline bool operator==(const ExistenceFilter& lhs, const ExistenceFilter& rhs) { - return lhs.count() == rhs.count() && - lhs.unchanged_names() == rhs.unchanged_names(); + return lhs.count() == rhs.count() && lhs.bloom_filter() == rhs.bloom_filter(); } } // namespace remote diff --git a/Firestore/core/test/unit/local/local_store_test.cc b/Firestore/core/test/unit/local/local_store_test.cc index a9762c8fd5c..834a20a364c 100644 --- a/Firestore/core/test/unit/local/local_store_test.cc +++ b/Firestore/core/test/unit/local/local_store_test.cc @@ -147,7 +147,7 @@ RemoteEvent ExistenceFilterEvent(TargetId target_id, metadata_provider.SetSyncedKeys(synced_keys, target_data); ExistenceFilter existence_filter{remote_count, - /*unchangedNames=*/absl::nullopt}; + /*bloom_filter=*/absl::nullopt}; WatchChangeAggregator aggregator{&metadata_provider}; ExistenceFilterWatchChange existence_filter_watch_change{existence_filter, target_id}; diff --git a/Firestore/core/test/unit/remote/remote_event_test.cc b/Firestore/core/test/unit/remote/remote_event_test.cc index fc8f956d4f6..05f06722578 100644 --- a/Firestore/core/test/unit/remote/remote_event_test.cc +++ b/Firestore/core/test/unit/remote/remote_event_test.cc @@ -548,7 +548,7 @@ TEST_F(RemoteEventTest, ExistenceFilterMismatchClearsTarget) { // The existence filter mismatch will remove the document from target 1, // but not synthesize a document delete. ExistenceFilterWatchChange change4{ - ExistenceFilter{1, /*unchangedNames=*/absl::nullopt}, 1}; + ExistenceFilter{1, /*bloom_filter=*/absl::nullopt}, 1}; aggregator.HandleExistenceFilter(change4); event = aggregator.CreateRemoteEvent(testutil::Version(4)); @@ -580,7 +580,7 @@ TEST_F(RemoteEventTest, ExistenceFilterMismatchRemovesCurrentChanges) { // The existence filter mismatch will remove the document from target 1, but // not synthesize a document delete. ExistenceFilterWatchChange existence_filter{ - ExistenceFilter{0, /*unchangedNames=*/absl::nullopt}, 1}; + ExistenceFilter{0, /*bloom_filter=*/absl::nullopt}, 1}; aggregator.HandleExistenceFilter(existence_filter); RemoteEvent event = aggregator.CreateRemoteEvent(testutil::Version(3)); diff --git a/Firestore/core/test/unit/remote/serializer_test.cc b/Firestore/core/test/unit/remote/serializer_test.cc index f67e45b6735..7bd10ff3838 100644 --- a/Firestore/core/test/unit/remote/serializer_test.cc +++ b/Firestore/core/test/unit/remote/serializer_test.cc @@ -1739,7 +1739,7 @@ TEST_F(SerializerTest, DecodesListenResponseWithDocumentRemove) { TEST_F(SerializerTest, DecodesListenResponseWithExistenceFilter) { ExistenceFilterWatchChange model( - ExistenceFilter(2, /*unchangedNames=*/absl::nullopt), 100); + ExistenceFilter(2, /*bloom_filter=*/absl::nullopt), 100); v1::ListenResponse proto; diff --git a/Firestore/core/test/unit/remote/watch_change_test.cc b/Firestore/core/test/unit/remote/watch_change_test.cc index 9dca6e13e4a..39c3a049c3f 100644 --- a/Firestore/core/test/unit/remote/watch_change_test.cc +++ b/Firestore/core/test/unit/remote/watch_change_test.cc @@ -41,7 +41,7 @@ TEST(WatchChangeTest, CanCreateDocumentWatchChange) { } TEST(WatchChangeTest, CanCreateExistenceFilterWatchChange) { - ExistenceFilter filter{7, /*unchangedNames=*/absl::nullopt}; + ExistenceFilter filter{7, /*bloom_filter=*/absl::nullopt}; ExistenceFilterWatchChange change{filter, 5}; EXPECT_EQ(change.filter().count(), 7); EXPECT_EQ(change.target_id(), 5); From b0bb8bfdeb0dc1e9b80e5557f6ec229453f9b0a8 Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Mon, 13 Mar 2023 21:00:09 -0700 Subject: [PATCH 10/32] add GetDatabaseId to the targetMetadataProvider --- Firestore/core/src/remote/datastore.cc | 1 + Firestore/core/src/remote/datastore.h | 5 ++ Firestore/core/src/remote/existence_filter.h | 13 ++--- Firestore/core/src/remote/remote_event.cc | 57 +++++++++++++++++-- Firestore/core/src/remote/remote_event.h | 15 +++++ Firestore/core/src/remote/remote_store.cc | 4 ++ Firestore/core/src/remote/remote_store.h | 1 + .../core/test/unit/local/local_store_test.cc | 2 +- .../remote/fake_target_metadata_provider.cc | 4 ++ .../remote/fake_target_metadata_provider.h | 1 + .../test/unit/remote/remote_event_test.cc | 4 +- .../core/test/unit/remote/serializer_test.cc | 2 +- .../test/unit/remote/watch_change_test.cc | 2 +- 13 files changed, 95 insertions(+), 16 deletions(-) diff --git a/Firestore/core/src/remote/datastore.cc b/Firestore/core/src/remote/datastore.cc index e3c664a9ed9..f837c7801a4 100644 --- a/Firestore/core/src/remote/datastore.cc +++ b/Firestore/core/src/remote/datastore.cc @@ -108,6 +108,7 @@ Datastore::Datastore( connectivity_monitor_{connectivity_monitor}, grpc_connection_{database_info, worker_queue, &grpc_queue_, connectivity_monitor_, firebase_metadata_provider}, + database_info_{database_info}, datastore_serializer_{database_info} { if (!database_info.ssl_enabled()) { GrpcConnection::UseInsecureChannel(database_info.host()); diff --git a/Firestore/core/src/remote/datastore.h b/Firestore/core/src/remote/datastore.h index d02885ee6f8..7afa403c1ee 100644 --- a/Firestore/core/src/remote/datastore.h +++ b/Firestore/core/src/remote/datastore.h @@ -139,6 +139,10 @@ class Datastore : public std::enable_shared_from_this { static std::string GetAllowlistedHeadersAsString( const GrpcCall::Metadata& headers); + const core::DatabaseInfo& database_info() const { + return database_info_; + } + Datastore(const Datastore& other) = delete; Datastore(Datastore&& other) = delete; Datastore& operator=(const Datastore& other) = delete; @@ -212,6 +216,7 @@ class Datastore : public std::enable_shared_from_this { GrpcConnection grpc_connection_; std::vector> active_calls_; + core::DatabaseInfo database_info_; DatastoreSerializer datastore_serializer_; }; diff --git a/Firestore/core/src/remote/existence_filter.h b/Firestore/core/src/remote/existence_filter.h index 5e7532e35b0..968341df588 100644 --- a/Firestore/core/src/remote/existence_filter.h +++ b/Firestore/core/src/remote/existence_filter.h @@ -29,26 +29,25 @@ class ExistenceFilter { public: ExistenceFilter() = default; - ExistenceFilter(int count, absl::optional unchangedNames) - : count_{count}, unchanged_names_{std::move(unchangedNames)} { + ExistenceFilter(int count, absl::optional bloom_filter) + : count_{count}, bloom_filter_{std::move(bloom_filter)} { } int count() const { return count_; } - const absl::optional& unchanged_names() const { - return unchanged_names_; + const absl::optional& bloom_filter() const { + return bloom_filter_; } private: int count_ = 0; - absl::optional unchanged_names_; + absl::optional bloom_filter_; }; inline bool operator==(const ExistenceFilter& lhs, const ExistenceFilter& rhs) { - return lhs.count() == rhs.count() && - lhs.unchanged_names() == rhs.unchanged_names(); + return lhs.count() == rhs.count() && lhs.bloom_filter() == rhs.bloom_filter(); } } // namespace remote diff --git a/Firestore/core/src/remote/remote_event.cc b/Firestore/core/src/remote/remote_event.cc index f6f5d86ccca..2a37aaa235b 100644 --- a/Firestore/core/src/remote/remote_event.cc +++ b/Firestore/core/src/remote/remote_event.cc @@ -28,6 +28,7 @@ using core::DocumentViewChange; using core::Target; using local::QueryPurpose; using local::TargetData; +using model::DatabaseId; using model::DocumentKey; using model::DocumentKeySet; using model::MutableDocument; @@ -235,15 +236,63 @@ void WatchChangeAggregator::HandleExistenceFilter( } else { int current_size = GetCurrentDocumentCountForTarget(target_id); if (current_size != expected_count) { - // Existence filter mismatch: We reset the mapping and raise a new - // snapshot with `isFromCache:true`. - ResetTarget(target_id); - pending_target_resets_.insert(target_id); + // Apply bloom filter to identify and mark removed documents. + bool bloomFilterApplied = + ApplyBloomFilter(existence_filter, current_size); + + if (!bloomFilterApplied) { + // If bloom filter application fails, we reset the mapping and + // trigger re-run of the query. + ResetTarget(target_id); + pending_target_resets_.insert(target_id); + } } } } } +/** Returns whether a bloom filter removed the deleted documents successfully. + */ +bool WatchChangeAggregator::ApplyBloomFilter( + ExistenceFilterWatchChange existence_filter, int current_count) { + int expected_count = existence_filter.filter().count(); + absl::optional bloom_filter = + existence_filter.filter().bloom_filter(); + + if (!bloom_filter.has_value()) { + return false; + } + + int removed_document_count = FilterRemovedDocuments( + bloom_filter.value(), existence_filter.target_id()); + + return expected_count == (current_count - removed_document_count); +} + +/** + * Filter out removed documents based on bloom filter membership result and + * return number of documents removed. + */ +int WatchChangeAggregator::FilterRemovedDocuments(BloomFilter bloom_filter, + int target_id) { + const DocumentKeySet& existing_keys = + target_metadata_provider_->GetRemoteKeysForTarget(target_id); + int removalCount = 0; + for (DocumentKey key : existing_keys) { + DatabaseId database_id = target_metadata_provider_->GetDatabaseId(); + std::string documentPath = util::StringFormat( + "projects/%s/databases/%s/documents/%s", database_id.project_id(), + database_id.database_id(), key.ToString()); + + if (!bloom_filter.MightContain(documentPath)) { + RemoveDocumentFromTarget(target_id, key, + /*updatedDocument=*/absl::nullopt); + removalCount++; + } + } + return removalCount; +} + RemoteEvent WatchChangeAggregator::CreateRemoteEvent( const SnapshotVersion& snapshot_version) { std::unordered_map target_changes; diff --git a/Firestore/core/src/remote/remote_event.h b/Firestore/core/src/remote/remote_event.h index bdec91a1703..5b35a325270 100644 --- a/Firestore/core/src/remote/remote_event.h +++ b/Firestore/core/src/remote/remote_event.h @@ -62,6 +62,9 @@ class TargetMetadataProvider { */ virtual absl::optional GetTargetDataForTarget( model::TargetId target_id) const = 0; + + /** Returns the database ID of the Firestore instance. */ + virtual model::DatabaseId GetDatabaseId() const = 0; }; /** @@ -410,6 +413,18 @@ class WatchChangeAggregator { bool TargetContainsDocument(model::TargetId target_id, const model::DocumentKey& key); + // TODO(Mila) + /** Returns whether a bloom filter removed the deleted documents successfully. + */ + bool ApplyBloomFilter(ExistenceFilterWatchChange existence_filter, + int current_count); + + /** + * Filter out removed documents based on bloom filter membership result and + * return number of documents removed. + */ + int FilterRemovedDocuments(BloomFilter bloom_filter, int target_id); + /** The internal state of all tracked targets. */ std::unordered_map target_states_; diff --git a/Firestore/core/src/remote/remote_store.cc b/Firestore/core/src/remote/remote_store.cc index cf98ff67296..87f2835c7bc 100644 --- a/Firestore/core/src/remote/remote_store.cc +++ b/Firestore/core/src/remote/remote_store.cc @@ -566,6 +566,10 @@ absl::optional RemoteStore::GetTargetDataForTarget( : absl::optional{}; } +model::DatabaseId RemoteStore::GetDatabaseId() const { + return datastore_->database_info().database_id(); +} + void RemoteStore::RestartNetwork() { is_network_enabled_ = false; DisableNetworkInternal(); diff --git a/Firestore/core/src/remote/remote_store.h b/Firestore/core/src/remote/remote_store.h index 469a8726a98..9b6452aaa76 100644 --- a/Firestore/core/src/remote/remote_store.h +++ b/Firestore/core/src/remote/remote_store.h @@ -193,6 +193,7 @@ class RemoteStore : public TargetMetadataProvider, model::TargetId target_id) const override; absl::optional GetTargetDataForTarget( model::TargetId target_id) const override; + model::DatabaseId GetDatabaseId() const override; void RunCountQuery(const core::Query& query, api::CountQueryCallback&& result_callback); diff --git a/Firestore/core/test/unit/local/local_store_test.cc b/Firestore/core/test/unit/local/local_store_test.cc index a9762c8fd5c..834a20a364c 100644 --- a/Firestore/core/test/unit/local/local_store_test.cc +++ b/Firestore/core/test/unit/local/local_store_test.cc @@ -147,7 +147,7 @@ RemoteEvent ExistenceFilterEvent(TargetId target_id, metadata_provider.SetSyncedKeys(synced_keys, target_data); ExistenceFilter existence_filter{remote_count, - /*unchangedNames=*/absl::nullopt}; + /*bloom_filter=*/absl::nullopt}; WatchChangeAggregator aggregator{&metadata_provider}; ExistenceFilterWatchChange existence_filter_watch_change{existence_filter, target_id}; diff --git a/Firestore/core/test/unit/remote/fake_target_metadata_provider.cc b/Firestore/core/test/unit/remote/fake_target_metadata_provider.cc index cf330a5100b..353952d79cf 100644 --- a/Firestore/core/test/unit/remote/fake_target_metadata_provider.cc +++ b/Firestore/core/test/unit/remote/fake_target_metadata_provider.cc @@ -102,6 +102,10 @@ absl::optional FakeTargetMetadataProvider::GetTargetDataForTarget( return it->second; } +model::DatabaseId FakeTargetMetadataProvider::GetDatabaseId() const { + return model::DatabaseId("test-project", "test-database"); +} + } // namespace remote } // namespace firestore } // namespace firebase diff --git a/Firestore/core/test/unit/remote/fake_target_metadata_provider.h b/Firestore/core/test/unit/remote/fake_target_metadata_provider.h index 6b231774578..6d1e0dc35eb 100644 --- a/Firestore/core/test/unit/remote/fake_target_metadata_provider.h +++ b/Firestore/core/test/unit/remote/fake_target_metadata_provider.h @@ -73,6 +73,7 @@ class FakeTargetMetadataProvider : public TargetMetadataProvider { model::TargetId target_id) const override; absl::optional GetTargetDataForTarget( model::TargetId target_id) const override; + model::DatabaseId GetDatabaseId() const override; private: std::unordered_map synced_keys_; diff --git a/Firestore/core/test/unit/remote/remote_event_test.cc b/Firestore/core/test/unit/remote/remote_event_test.cc index fc8f956d4f6..05f06722578 100644 --- a/Firestore/core/test/unit/remote/remote_event_test.cc +++ b/Firestore/core/test/unit/remote/remote_event_test.cc @@ -548,7 +548,7 @@ TEST_F(RemoteEventTest, ExistenceFilterMismatchClearsTarget) { // The existence filter mismatch will remove the document from target 1, // but not synthesize a document delete. ExistenceFilterWatchChange change4{ - ExistenceFilter{1, /*unchangedNames=*/absl::nullopt}, 1}; + ExistenceFilter{1, /*bloom_filter=*/absl::nullopt}, 1}; aggregator.HandleExistenceFilter(change4); event = aggregator.CreateRemoteEvent(testutil::Version(4)); @@ -580,7 +580,7 @@ TEST_F(RemoteEventTest, ExistenceFilterMismatchRemovesCurrentChanges) { // The existence filter mismatch will remove the document from target 1, but // not synthesize a document delete. ExistenceFilterWatchChange existence_filter{ - ExistenceFilter{0, /*unchangedNames=*/absl::nullopt}, 1}; + ExistenceFilter{0, /*bloom_filter=*/absl::nullopt}, 1}; aggregator.HandleExistenceFilter(existence_filter); RemoteEvent event = aggregator.CreateRemoteEvent(testutil::Version(3)); diff --git a/Firestore/core/test/unit/remote/serializer_test.cc b/Firestore/core/test/unit/remote/serializer_test.cc index f67e45b6735..7bd10ff3838 100644 --- a/Firestore/core/test/unit/remote/serializer_test.cc +++ b/Firestore/core/test/unit/remote/serializer_test.cc @@ -1739,7 +1739,7 @@ TEST_F(SerializerTest, DecodesListenResponseWithDocumentRemove) { TEST_F(SerializerTest, DecodesListenResponseWithExistenceFilter) { ExistenceFilterWatchChange model( - ExistenceFilter(2, /*unchangedNames=*/absl::nullopt), 100); + ExistenceFilter(2, /*bloom_filter=*/absl::nullopt), 100); v1::ListenResponse proto; diff --git a/Firestore/core/test/unit/remote/watch_change_test.cc b/Firestore/core/test/unit/remote/watch_change_test.cc index 9dca6e13e4a..39c3a049c3f 100644 --- a/Firestore/core/test/unit/remote/watch_change_test.cc +++ b/Firestore/core/test/unit/remote/watch_change_test.cc @@ -41,7 +41,7 @@ TEST(WatchChangeTest, CanCreateDocumentWatchChange) { } TEST(WatchChangeTest, CanCreateExistenceFilterWatchChange) { - ExistenceFilter filter{7, /*unchangedNames=*/absl::nullopt}; + ExistenceFilter filter{7, /*bloom_filter=*/absl::nullopt}; ExistenceFilterWatchChange change{filter, 5}; EXPECT_EQ(change.filter().count(), 7); EXPECT_EQ(change.target_id(), 5); From d56f4dfd3c9f641760263196938de31c427a7d1e Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Mon, 13 Mar 2023 21:23:31 -0700 Subject: [PATCH 11/32] remove the parsing unchanged_names to BloomFilter logic --- .../Example/Tests/SpecTests/FSTSpecTests.mm | 24 ++++--------------- Firestore/core/src/remote/serializer.cc | 12 ++++------ 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm index 0ee79ac7a2f..d14b4aceead 100644 --- a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm +++ b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm @@ -331,27 +331,11 @@ - (SnapshotVersion)parseVersion:(NSNumber *_Nullable)version { } - (absl::optional)parseBloomFilter:(NSDictionary *_Nullable)bloomFilterProto { - if (bloomFilterProto == nil) { - return absl::nullopt; - } - NSDictionary *bitsData = bloomFilterProto[@"bits"]; - - // Decode base64 string into uint8_t vector. If not specified, will default bitmap to empty - // vector. - NSString *bitmapEncoded = bitsData[@"bitmap"]; - std::string bitmapDecoded; - absl::Base64Unescape([bitmapEncoded UTF8String], &bitmapDecoded); - std::vector bitmap(bitmapDecoded.begin(), bitmapDecoded.end()); - - // If not specified, will default padding and hashCount to 0. - int32_t padding = [bitsData[@"padding"] intValue]; - int32_t hashCount = [bloomFilterProto[@"hashCount"] intValue]; - StatusOr maybeBloomFilter = BloomFilter::Create(bitmap, padding, hashCount); - if (maybeBloomFilter.ok()) { - return maybeBloomFilter.ValueOrDie(); - } - + // TODO(Mila): None of the ported spec tests actually has the bloom filter json string, so hard + // code a null value for now. Actual parsing code will be written in the next PR, where we can + // validate the parsing result. return absl::nullopt; + ; } - (DocumentViewChange)parseChange:(NSDictionary *)jsonDoc ofType:(DocumentViewChange::Type)type { diff --git a/Firestore/core/src/remote/serializer.cc b/Firestore/core/src/remote/serializer.cc index 82ca1eefb16..5c92d481e07 100644 --- a/Firestore/core/src/remote/serializer.cc +++ b/Firestore/core/src/remote/serializer.cc @@ -1441,14 +1441,12 @@ ExistenceFilter Serializer::DecodeExistenceFilter( // and inputs are valid, otherwise keep it null. absl::optional bloom_filter = absl::nullopt; if (filter.has_unchanged_names && filter.unchanged_names.has_bits) { - // TODO(Mila): Ensure bloom filter with invalid inputs are handled correctly - // in next PR. - ByteString bitmap_string(filter.unchanged_names.bits.bitmap); - std::vector bitmap = MakeVector(bitmap_string); - int32_t padding = filter.unchanged_names.bits.padding; - int32_t hash_count = filter.unchanged_names.hash_count; + // TODO(Mila): None of the ported spec tests here actually has the bloom + // filter json string, so hard code an empty bloom filter for now. The + // actual parsing code will be written in the next PR, where we can validate + // the parsing result. StatusOr maybe_bloom_filter = - BloomFilter::Create(bitmap, padding, hash_count); + BloomFilter::Create(std::vector{}, 0, 0); if (maybe_bloom_filter.ok()) { bloom_filter = maybe_bloom_filter.ValueOrDie(); } From 41583524885518eb368dcb61e7796c2518cac113 Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Tue, 14 Mar 2023 12:21:29 -0700 Subject: [PATCH 12/32] sort the actual and expected events in spec test before comparing them --- .../Example/Tests/SpecTests/FSTSpecTests.mm | 18 ++++++++++++++++-- Firestore/core/src/remote/remote_event.cc | 1 + 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm index 0ee79ac7a2f..cd3bd1cf667 100644 --- a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm +++ b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm @@ -711,8 +711,22 @@ - (void)validateEvent:(FSTQueryEvent *)actual matches:(NSDictionary *)expected { } XCTAssertEqual(actual.viewSnapshot.value().document_changes().size(), expectedChanges.size()); - for (size_t i = 0; i != expectedChanges.size(); ++i) { - XCTAssertTrue((actual.viewSnapshot.value().document_changes()[i] == expectedChanges[i])); + + std::vector expectedChangesSorted = expectedChanges; + std::sort(expectedChangesSorted.begin(), expectedChangesSorted.end(), + [](const DocumentViewChange &lhs, const DocumentViewChange &rhs) { + return lhs.document()->key() < rhs.document()->key(); + }); + + std::vector actualChangesSorted = + actual.viewSnapshot.value().document_changes(); + std::sort(actualChangesSorted.begin(), actualChangesSorted.end(), + [](const DocumentViewChange &lhs, const DocumentViewChange &rhs) { + return lhs.document()->key() < rhs.document()->key(); + }); + + for (size_t i = 0; i != expectedChangesSorted.size(); ++i) { + XCTAssertTrue((actualChangesSorted[i] == expectedChangesSorted[i])); } BOOL expectedHasPendingWrites = diff --git a/Firestore/core/src/remote/remote_event.cc b/Firestore/core/src/remote/remote_event.cc index 2a37aaa235b..9792ea8e25a 100644 --- a/Firestore/core/src/remote/remote_event.cc +++ b/Firestore/core/src/remote/remote_event.cc @@ -16,6 +16,7 @@ #include "Firestore/core/src/remote/remote_event.h" +#include #include #include "Firestore/core/src/local/target_data.h" From ae930ef9f9be3c8344748f22b1282e10796c20b2 Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Wed, 15 Mar 2023 11:03:05 -0700 Subject: [PATCH 13/32] Revert "Merge branch 'mila/BloomFilter-add--existence-filter-class' into mila/BloomFilter-apply-bloom-filter-on-existence-filter-mismatch" This reverts commit 2b8533cb1b20e1c186f50981a27b52e5aff66962, reversing changes made to 41583524885518eb368dcb61e7796c2518cac113. --- .../Example/Tests/SpecTests/FSTSpecTests.mm | 24 +++++++++++++++---- Firestore/core/src/remote/serializer.cc | 12 ++++++---- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm index bf65b463467..cd3bd1cf667 100644 --- a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm +++ b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm @@ -331,11 +331,27 @@ - (SnapshotVersion)parseVersion:(NSNumber *_Nullable)version { } - (absl::optional)parseBloomFilter:(NSDictionary *_Nullable)bloomFilterProto { - // TODO(Mila): None of the ported spec tests actually has the bloom filter json string, so hard - // code a null value for now. Actual parsing code will be written in the next PR, where we can - // validate the parsing result. + if (bloomFilterProto == nil) { + return absl::nullopt; + } + NSDictionary *bitsData = bloomFilterProto[@"bits"]; + + // Decode base64 string into uint8_t vector. If not specified, will default bitmap to empty + // vector. + NSString *bitmapEncoded = bitsData[@"bitmap"]; + std::string bitmapDecoded; + absl::Base64Unescape([bitmapEncoded UTF8String], &bitmapDecoded); + std::vector bitmap(bitmapDecoded.begin(), bitmapDecoded.end()); + + // If not specified, will default padding and hashCount to 0. + int32_t padding = [bitsData[@"padding"] intValue]; + int32_t hashCount = [bloomFilterProto[@"hashCount"] intValue]; + StatusOr maybeBloomFilter = BloomFilter::Create(bitmap, padding, hashCount); + if (maybeBloomFilter.ok()) { + return maybeBloomFilter.ValueOrDie(); + } + return absl::nullopt; - ; } - (DocumentViewChange)parseChange:(NSDictionary *)jsonDoc ofType:(DocumentViewChange::Type)type { diff --git a/Firestore/core/src/remote/serializer.cc b/Firestore/core/src/remote/serializer.cc index 5c92d481e07..82ca1eefb16 100644 --- a/Firestore/core/src/remote/serializer.cc +++ b/Firestore/core/src/remote/serializer.cc @@ -1441,12 +1441,14 @@ ExistenceFilter Serializer::DecodeExistenceFilter( // and inputs are valid, otherwise keep it null. absl::optional bloom_filter = absl::nullopt; if (filter.has_unchanged_names && filter.unchanged_names.has_bits) { - // TODO(Mila): None of the ported spec tests here actually has the bloom - // filter json string, so hard code an empty bloom filter for now. The - // actual parsing code will be written in the next PR, where we can validate - // the parsing result. + // TODO(Mila): Ensure bloom filter with invalid inputs are handled correctly + // in next PR. + ByteString bitmap_string(filter.unchanged_names.bits.bitmap); + std::vector bitmap = MakeVector(bitmap_string); + int32_t padding = filter.unchanged_names.bits.padding; + int32_t hash_count = filter.unchanged_names.hash_count; StatusOr maybe_bloom_filter = - BloomFilter::Create(std::vector{}, 0, 0); + BloomFilter::Create(bitmap, padding, hash_count); if (maybe_bloom_filter.ok()) { bloom_filter = maybe_bloom_filter.ValueOrDie(); } From 91a42e632364a57e7db19316085b19994f1b5ec4 Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Thu, 16 Mar 2023 14:56:24 -0700 Subject: [PATCH 14/32] resolve comments --- .../Example/Tests/SpecTests/FSTSpecTests.mm | 4 +- Firestore/core/src/remote/bloom_filter.cc | 22 ++++++++- Firestore/core/src/remote/bloom_filter.h | 8 ++- Firestore/core/src/remote/serializer.cc | 7 +-- .../test/unit/remote/bloom_filter_test.cc | 49 ++++++++++++++----- .../test/unit/remote/remote_event_test.cc | 1 + .../core/test/unit/remote/serializer_test.cc | 1 + .../test/unit/remote/watch_change_test.cc | 1 + 8 files changed, 73 insertions(+), 20 deletions(-) diff --git a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm index d14b4aceead..009aca8a542 100644 --- a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm +++ b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm @@ -485,8 +485,8 @@ - (void)doWatchFilter:(NSDictionary *)watchFilter { absl::optional bloomFilter = [self parseBloomFilter:watchFilter[@"bloomFilter"]]; - ExistenceFilter filter{keyCount, bloomFilter}; - ExistenceFilterWatchChange change{filter, targets[0].intValue}; + ExistenceFilter filter{keyCount, std::move(bloomFilter)}; + ExistenceFilterWatchChange change{std::move(filter), targets[0].intValue}; [self.driver receiveWatchChange:change snapshotVersion:SnapshotVersion::None()]; } diff --git a/Firestore/core/src/remote/bloom_filter.cc b/Firestore/core/src/remote/bloom_filter.cc index d893677bd1d..ad4c5bcbe7c 100644 --- a/Firestore/core/src/remote/bloom_filter.cc +++ b/Firestore/core/src/remote/bloom_filter.cc @@ -115,8 +115,26 @@ bool BloomFilter::MightContain(absl::string_view value) const { bool operator==(const BloomFilter& lhs, const BloomFilter& rhs) { return lhs.bit_count() == rhs.bit_count() && - lhs.hash_count() == rhs.hash_count() && - equal(lhs.bitmap().begin(), lhs.bitmap().end(), rhs.bitmap().begin()); + lhs.hash_count() == rhs.hash_count() && CompareBits(lhs, rhs); +} + +bool operator!=(const BloomFilter& lhs, const BloomFilter& rhs) { + return !(lhs == rhs); +} + +bool CompareBits(const BloomFilter& lhs, const BloomFilter& rhs) { + for (int32_t i = 0; i < lhs.bit_count(); ++i) { + int32_t offset = i % 8; + const uint8_t& lhs_byte = lhs.bitmap()[i / 8]; + const uint8_t& rhs_byte = rhs.bitmap()[i / 8]; + const bool bit1 = (lhs_byte & (static_cast(0x01) << offset)); + const bool bit2 = (rhs_byte & (static_cast(0x01) << offset)); + if (bit1 != bit2) { + return false; + } + } + + return true; } } // namespace remote diff --git a/Firestore/core/src/remote/bloom_filter.h b/Firestore/core/src/remote/bloom_filter.h index 52178784d2f..b5bb2003f93 100644 --- a/Firestore/core/src/remote/bloom_filter.h +++ b/Firestore/core/src/remote/bloom_filter.h @@ -59,8 +59,8 @@ class BloomFilter final { /** * The number of bits in the bloom filter. Guaranteed to be non-negative, and - * less than the max number of bits `bitmap_` can represent, i.e., - * bitmap_.size() * 8. + * less than the max number of bits the bitmap can represent, i.e., + * bitmap().size() * 8. */ int32_t bit_count() const { return bit_count_; @@ -113,6 +113,10 @@ class BloomFilter final { bool operator==(const BloomFilter& lhs, const BloomFilter& rhs); +bool operator!=(const BloomFilter& lhs, const BloomFilter& rhs); + +bool CompareBits(const BloomFilter& lhs, const BloomFilter& rhs); + } // namespace remote } // namespace firestore } // namespace firebase diff --git a/Firestore/core/src/remote/serializer.cc b/Firestore/core/src/remote/serializer.cc index 5c92d481e07..fbdc82e6eae 100644 --- a/Firestore/core/src/remote/serializer.cc +++ b/Firestore/core/src/remote/serializer.cc @@ -1439,20 +1439,21 @@ ExistenceFilter Serializer::DecodeExistenceFilter( const google_firestore_v1_ExistenceFilter& filter) const { // Create bloom filter if there is an unchanged_names present in the filter // and inputs are valid, otherwise keep it null. - absl::optional bloom_filter = absl::nullopt; + absl::optional bloom_filter; if (filter.has_unchanged_names && filter.unchanged_names.has_bits) { // TODO(Mila): None of the ported spec tests here actually has the bloom // filter json string, so hard code an empty bloom filter for now. The // actual parsing code will be written in the next PR, where we can validate // the parsing result. + // TODO(Mila): handle bloom filter creation failure. StatusOr maybe_bloom_filter = BloomFilter::Create(std::vector{}, 0, 0); if (maybe_bloom_filter.ok()) { - bloom_filter = maybe_bloom_filter.ValueOrDie(); + bloom_filter = std::move(maybe_bloom_filter).ValueOrDie(); } } - return {filter.count, bloom_filter}; + return {filter.count, std::move(bloom_filter)}; } bool Serializer::IsLocalResourceName(const ResourcePath& path) const { diff --git a/Firestore/core/test/unit/remote/bloom_filter_test.cc b/Firestore/core/test/unit/remote/bloom_filter_test.cc index 640061a72e6..93583f63d0f 100644 --- a/Firestore/core/test/unit/remote/bloom_filter_test.cc +++ b/Firestore/core/test/unit/remote/bloom_filter_test.cc @@ -118,17 +118,44 @@ TEST(BloomFilterUnitTest, CreateShouldReturnNotOKStatusIfPaddingIsTooLarge) { EXPECT_EQ(maybe_bloom_filter.status().error_message(), "Invalid padding: 8"); } -TEST(BloomFilterTest, CanCheckBloomFiltersEquality) { - { - BloomFilter bloom_filter1(std::vector{1}, 1, 1); - BloomFilter bloom_filter2(std::vector{1}, 1, 1); - EXPECT_TRUE(bloom_filter1 == bloom_filter2); - } - { - BloomFilter bloom_filter1(std::vector{1}, 1, 1); - BloomFilter bloom_filter2(std::vector{1}, 2, 1); - EXPECT_FALSE(bloom_filter1 == bloom_filter2); - } +TEST(BloomFilterTest, CheckBloomFiltersEqualityWithSameInput) { + BloomFilter bloom_filter1(std::vector{1}, 1, 1); + BloomFilter bloom_filter2(std::vector{1}, 1, 1); + EXPECT_TRUE(bloom_filter1 == bloom_filter2); + EXPECT_FALSE(bloom_filter1 != bloom_filter2); +} + +TEST(BloomFilterTest, CheckBloomFiltersEqualityWithDifferentBitmap) { + BloomFilter bloom_filter1(std::vector{1}, 1, 1); + BloomFilter bloom_filter2(std::vector{2}, 1, 1); + EXPECT_FALSE(bloom_filter1 == bloom_filter2); + EXPECT_TRUE(bloom_filter1 != bloom_filter2); +} + +TEST(BloomFilterTest, CheckBloomFiltersEqualityWithDifferentPadding) { + BloomFilter bloom_filter1(std::vector{1}, 1, 1); + BloomFilter bloom_filter2(std::vector{1}, 2, 1); + EXPECT_FALSE(bloom_filter1 == bloom_filter2); + EXPECT_TRUE(bloom_filter1 != bloom_filter2); +} + +TEST(BloomFilterTest, CheckBloomFiltersEqualityWithDifferentHashCount) { + BloomFilter bloom_filter1(std::vector{1}, 1, 1); + BloomFilter bloom_filter2(std::vector{1}, 1, 2); + EXPECT_FALSE(bloom_filter1 == bloom_filter2); + EXPECT_TRUE(bloom_filter1 != bloom_filter2); +} + +TEST(BloomFilterTest, + BloomFiltersEqualityCheckShouldIgnoreBitsInPaddingIndexes) { + // In BloomFilter bitmap, padding is guaranteed to be less than 8, and + // starts counting from the leftmost indexes of the last byte. + BloomFilter bloom_filter1(std::vector{255, 127}, 1, + 1); // bitmap -> 11111111 01111111 + BloomFilter bloom_filter2(std::vector{255, 255}, 1, + 1); // bitmap -> 11111111 11111111 + EXPECT_TRUE(bloom_filter1 == bloom_filter2); + EXPECT_FALSE(bloom_filter1 != bloom_filter2); } TEST(BloomFilterTest, MightContainCanProcessNonStandardCharacters) { diff --git a/Firestore/core/test/unit/remote/remote_event_test.cc b/Firestore/core/test/unit/remote/remote_event_test.cc index 05f06722578..5e68c22ed66 100644 --- a/Firestore/core/test/unit/remote/remote_event_test.cc +++ b/Firestore/core/test/unit/remote/remote_event_test.cc @@ -512,6 +512,7 @@ TEST_F(RemoteEventTest, NoChangeWillStillMarkTheAffectedTargets) { ASSERT_TRUE(event.target_changes().at(1) == target_change); } +// TODO(Mila): Add test coverage for when the bloom filter is not null TEST_F(RemoteEventTest, ExistenceFilterMismatchClearsTarget) { std::unordered_map target_map = ActiveQueries({1, 2}); diff --git a/Firestore/core/test/unit/remote/serializer_test.cc b/Firestore/core/test/unit/remote/serializer_test.cc index 7bd10ff3838..23e119f3922 100644 --- a/Firestore/core/test/unit/remote/serializer_test.cc +++ b/Firestore/core/test/unit/remote/serializer_test.cc @@ -1737,6 +1737,7 @@ TEST_F(SerializerTest, DecodesListenResponseWithDocumentRemove) { ExpectDeserializationRoundTrip(model, proto); } +// TODO(Mila): Add test coverage for when the bloom filter is not null TEST_F(SerializerTest, DecodesListenResponseWithExistenceFilter) { ExistenceFilterWatchChange model( ExistenceFilter(2, /*bloom_filter=*/absl::nullopt), 100); diff --git a/Firestore/core/test/unit/remote/watch_change_test.cc b/Firestore/core/test/unit/remote/watch_change_test.cc index 39c3a049c3f..c560d63a3f5 100644 --- a/Firestore/core/test/unit/remote/watch_change_test.cc +++ b/Firestore/core/test/unit/remote/watch_change_test.cc @@ -40,6 +40,7 @@ TEST(WatchChangeTest, CanCreateDocumentWatchChange) { EXPECT_EQ(change.new_document(), doc); } +// TODO(Mila): Add test coverage for when the bloom filter is not null TEST(WatchChangeTest, CanCreateExistenceFilterWatchChange) { ExistenceFilter filter{7, /*bloom_filter=*/absl::nullopt}; ExistenceFilterWatchChange change{filter, 5}; From db883fb24d540e581ec137bcd8d23e516aea9acd Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Thu, 16 Mar 2023 15:41:35 -0700 Subject: [PATCH 15/32] Merge branch 'mila/BloomFilter-add--existence-filter-class' into mila/BloomFilter-apply-bloom-filter-on-existence-filter-mismatch --- .../Example/Tests/SpecTests/FSTSpecTests.mm | 5 +- Firestore/core/src/remote/bloom_filter.cc | 22 ++++++++- Firestore/core/src/remote/bloom_filter.h | 8 ++- .../test/unit/remote/bloom_filter_test.cc | 49 ++++++++++++++----- .../test/unit/remote/remote_event_test.cc | 1 + .../core/test/unit/remote/serializer_test.cc | 1 + .../test/unit/remote/watch_change_test.cc | 1 + 7 files changed, 69 insertions(+), 18 deletions(-) diff --git a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm index cd3bd1cf667..0cc163b928a 100644 --- a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm +++ b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm @@ -71,7 +71,6 @@ #include "Firestore/core/src/util/string_apple.h" #include "Firestore/core/src/util/to_string.h" #include "Firestore/core/test/unit/testutil/testutil.h" - #include "absl/memory/memory.h" #include "absl/strings/escaping.h" #include "absl/types/optional.h" @@ -501,8 +500,8 @@ - (void)doWatchFilter:(NSDictionary *)watchFilter { absl::optional bloomFilter = [self parseBloomFilter:watchFilter[@"bloomFilter"]]; - ExistenceFilter filter{keyCount, bloomFilter}; - ExistenceFilterWatchChange change{filter, targets[0].intValue}; + ExistenceFilter filter{keyCount, std::move(bloomFilter)}; + ExistenceFilterWatchChange change{std::move(filter), targets[0].intValue}; [self.driver receiveWatchChange:change snapshotVersion:SnapshotVersion::None()]; } diff --git a/Firestore/core/src/remote/bloom_filter.cc b/Firestore/core/src/remote/bloom_filter.cc index d893677bd1d..ad4c5bcbe7c 100644 --- a/Firestore/core/src/remote/bloom_filter.cc +++ b/Firestore/core/src/remote/bloom_filter.cc @@ -115,8 +115,26 @@ bool BloomFilter::MightContain(absl::string_view value) const { bool operator==(const BloomFilter& lhs, const BloomFilter& rhs) { return lhs.bit_count() == rhs.bit_count() && - lhs.hash_count() == rhs.hash_count() && - equal(lhs.bitmap().begin(), lhs.bitmap().end(), rhs.bitmap().begin()); + lhs.hash_count() == rhs.hash_count() && CompareBits(lhs, rhs); +} + +bool operator!=(const BloomFilter& lhs, const BloomFilter& rhs) { + return !(lhs == rhs); +} + +bool CompareBits(const BloomFilter& lhs, const BloomFilter& rhs) { + for (int32_t i = 0; i < lhs.bit_count(); ++i) { + int32_t offset = i % 8; + const uint8_t& lhs_byte = lhs.bitmap()[i / 8]; + const uint8_t& rhs_byte = rhs.bitmap()[i / 8]; + const bool bit1 = (lhs_byte & (static_cast(0x01) << offset)); + const bool bit2 = (rhs_byte & (static_cast(0x01) << offset)); + if (bit1 != bit2) { + return false; + } + } + + return true; } } // namespace remote diff --git a/Firestore/core/src/remote/bloom_filter.h b/Firestore/core/src/remote/bloom_filter.h index 52178784d2f..b5bb2003f93 100644 --- a/Firestore/core/src/remote/bloom_filter.h +++ b/Firestore/core/src/remote/bloom_filter.h @@ -59,8 +59,8 @@ class BloomFilter final { /** * The number of bits in the bloom filter. Guaranteed to be non-negative, and - * less than the max number of bits `bitmap_` can represent, i.e., - * bitmap_.size() * 8. + * less than the max number of bits the bitmap can represent, i.e., + * bitmap().size() * 8. */ int32_t bit_count() const { return bit_count_; @@ -113,6 +113,10 @@ class BloomFilter final { bool operator==(const BloomFilter& lhs, const BloomFilter& rhs); +bool operator!=(const BloomFilter& lhs, const BloomFilter& rhs); + +bool CompareBits(const BloomFilter& lhs, const BloomFilter& rhs); + } // namespace remote } // namespace firestore } // namespace firebase diff --git a/Firestore/core/test/unit/remote/bloom_filter_test.cc b/Firestore/core/test/unit/remote/bloom_filter_test.cc index 640061a72e6..93583f63d0f 100644 --- a/Firestore/core/test/unit/remote/bloom_filter_test.cc +++ b/Firestore/core/test/unit/remote/bloom_filter_test.cc @@ -118,17 +118,44 @@ TEST(BloomFilterUnitTest, CreateShouldReturnNotOKStatusIfPaddingIsTooLarge) { EXPECT_EQ(maybe_bloom_filter.status().error_message(), "Invalid padding: 8"); } -TEST(BloomFilterTest, CanCheckBloomFiltersEquality) { - { - BloomFilter bloom_filter1(std::vector{1}, 1, 1); - BloomFilter bloom_filter2(std::vector{1}, 1, 1); - EXPECT_TRUE(bloom_filter1 == bloom_filter2); - } - { - BloomFilter bloom_filter1(std::vector{1}, 1, 1); - BloomFilter bloom_filter2(std::vector{1}, 2, 1); - EXPECT_FALSE(bloom_filter1 == bloom_filter2); - } +TEST(BloomFilterTest, CheckBloomFiltersEqualityWithSameInput) { + BloomFilter bloom_filter1(std::vector{1}, 1, 1); + BloomFilter bloom_filter2(std::vector{1}, 1, 1); + EXPECT_TRUE(bloom_filter1 == bloom_filter2); + EXPECT_FALSE(bloom_filter1 != bloom_filter2); +} + +TEST(BloomFilterTest, CheckBloomFiltersEqualityWithDifferentBitmap) { + BloomFilter bloom_filter1(std::vector{1}, 1, 1); + BloomFilter bloom_filter2(std::vector{2}, 1, 1); + EXPECT_FALSE(bloom_filter1 == bloom_filter2); + EXPECT_TRUE(bloom_filter1 != bloom_filter2); +} + +TEST(BloomFilterTest, CheckBloomFiltersEqualityWithDifferentPadding) { + BloomFilter bloom_filter1(std::vector{1}, 1, 1); + BloomFilter bloom_filter2(std::vector{1}, 2, 1); + EXPECT_FALSE(bloom_filter1 == bloom_filter2); + EXPECT_TRUE(bloom_filter1 != bloom_filter2); +} + +TEST(BloomFilterTest, CheckBloomFiltersEqualityWithDifferentHashCount) { + BloomFilter bloom_filter1(std::vector{1}, 1, 1); + BloomFilter bloom_filter2(std::vector{1}, 1, 2); + EXPECT_FALSE(bloom_filter1 == bloom_filter2); + EXPECT_TRUE(bloom_filter1 != bloom_filter2); +} + +TEST(BloomFilterTest, + BloomFiltersEqualityCheckShouldIgnoreBitsInPaddingIndexes) { + // In BloomFilter bitmap, padding is guaranteed to be less than 8, and + // starts counting from the leftmost indexes of the last byte. + BloomFilter bloom_filter1(std::vector{255, 127}, 1, + 1); // bitmap -> 11111111 01111111 + BloomFilter bloom_filter2(std::vector{255, 255}, 1, + 1); // bitmap -> 11111111 11111111 + EXPECT_TRUE(bloom_filter1 == bloom_filter2); + EXPECT_FALSE(bloom_filter1 != bloom_filter2); } TEST(BloomFilterTest, MightContainCanProcessNonStandardCharacters) { diff --git a/Firestore/core/test/unit/remote/remote_event_test.cc b/Firestore/core/test/unit/remote/remote_event_test.cc index 05f06722578..5e68c22ed66 100644 --- a/Firestore/core/test/unit/remote/remote_event_test.cc +++ b/Firestore/core/test/unit/remote/remote_event_test.cc @@ -512,6 +512,7 @@ TEST_F(RemoteEventTest, NoChangeWillStillMarkTheAffectedTargets) { ASSERT_TRUE(event.target_changes().at(1) == target_change); } +// TODO(Mila): Add test coverage for when the bloom filter is not null TEST_F(RemoteEventTest, ExistenceFilterMismatchClearsTarget) { std::unordered_map target_map = ActiveQueries({1, 2}); diff --git a/Firestore/core/test/unit/remote/serializer_test.cc b/Firestore/core/test/unit/remote/serializer_test.cc index 7bd10ff3838..23e119f3922 100644 --- a/Firestore/core/test/unit/remote/serializer_test.cc +++ b/Firestore/core/test/unit/remote/serializer_test.cc @@ -1737,6 +1737,7 @@ TEST_F(SerializerTest, DecodesListenResponseWithDocumentRemove) { ExpectDeserializationRoundTrip(model, proto); } +// TODO(Mila): Add test coverage for when the bloom filter is not null TEST_F(SerializerTest, DecodesListenResponseWithExistenceFilter) { ExistenceFilterWatchChange model( ExistenceFilter(2, /*bloom_filter=*/absl::nullopt), 100); diff --git a/Firestore/core/test/unit/remote/watch_change_test.cc b/Firestore/core/test/unit/remote/watch_change_test.cc index 39c3a049c3f..c560d63a3f5 100644 --- a/Firestore/core/test/unit/remote/watch_change_test.cc +++ b/Firestore/core/test/unit/remote/watch_change_test.cc @@ -40,6 +40,7 @@ TEST(WatchChangeTest, CanCreateDocumentWatchChange) { EXPECT_EQ(change.new_document(), doc); } +// TODO(Mila): Add test coverage for when the bloom filter is not null TEST(WatchChangeTest, CanCreateExistenceFilterWatchChange) { ExistenceFilter filter{7, /*bloom_filter=*/absl::nullopt}; ExistenceFilterWatchChange change{filter, 5}; From a2b7408d9dc87fabbb70edbc152a300dd0d7bde4 Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Thu, 16 Mar 2023 15:51:03 -0700 Subject: [PATCH 16/32] format --- Firestore/Example/Tests/SpecTests/FSTSpecTests.mm | 1 - Firestore/core/src/remote/bloom_filter.cc | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm index 08a5d28bb4b..703e3807df6 100644 --- a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm +++ b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm @@ -71,7 +71,6 @@ #include "Firestore/core/src/util/string_apple.h" #include "Firestore/core/src/util/to_string.h" #include "Firestore/core/test/unit/testutil/testutil.h" - #include "absl/memory/memory.h" #include "absl/strings/escaping.h" #include "absl/types/optional.h" diff --git a/Firestore/core/src/remote/bloom_filter.cc b/Firestore/core/src/remote/bloom_filter.cc index ad4c5bcbe7c..d7fa5eee7f6 100644 --- a/Firestore/core/src/remote/bloom_filter.cc +++ b/Firestore/core/src/remote/bloom_filter.cc @@ -123,6 +123,10 @@ bool operator!=(const BloomFilter& lhs, const BloomFilter& rhs) { } bool CompareBits(const BloomFilter& lhs, const BloomFilter& rhs) { + if (lhs.bit_count() != rhs.bit_count()) { + return false; + } + for (int32_t i = 0; i < lhs.bit_count(); ++i) { int32_t offset = i % 8; const uint8_t& lhs_byte = lhs.bitmap()[i / 8]; From f6b062f18eb280e56067ed78d0d1834ca307d8f2 Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Thu, 16 Mar 2023 20:19:05 -0700 Subject: [PATCH 17/32] log error message --- Firestore/core/src/remote/serializer.cc | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/Firestore/core/src/remote/serializer.cc b/Firestore/core/src/remote/serializer.cc index 82ca1eefb16..b346a2ef1c8 100644 --- a/Firestore/core/src/remote/serializer.cc +++ b/Firestore/core/src/remote/serializer.cc @@ -51,6 +51,7 @@ #include "Firestore/core/src/nanopb/writer.h" #include "Firestore/core/src/timestamp_internal.h" #include "Firestore/core/src/util/hard_assert.h" +#include "Firestore/core/src/util/log.h" #include "Firestore/core/src/util/status.h" #include "Firestore/core/src/util/statusor.h" #include "Firestore/core/src/util/string_format.h" @@ -1439,22 +1440,27 @@ ExistenceFilter Serializer::DecodeExistenceFilter( const google_firestore_v1_ExistenceFilter& filter) const { // Create bloom filter if there is an unchanged_names present in the filter // and inputs are valid, otherwise keep it null. - absl::optional bloom_filter = absl::nullopt; + absl::optional bloom_filter; + if (filter.has_unchanged_names && filter.unchanged_names.has_bits) { - // TODO(Mila): Ensure bloom filter with invalid inputs are handled correctly - // in next PR. - ByteString bitmap_string(filter.unchanged_names.bits.bitmap); - std::vector bitmap = MakeVector(bitmap_string); + pb_bytes_array_t* bitmap_ptr = filter.unchanged_names.bits.bitmap; + std::vector bitmap(bitmap_ptr->bytes, + bitmap_ptr->bytes + bitmap_ptr->size); + int32_t padding = filter.unchanged_names.bits.padding; int32_t hash_count = filter.unchanged_names.hash_count; + StatusOr maybe_bloom_filter = BloomFilter::Create(bitmap, padding, hash_count); if (maybe_bloom_filter.ok()) { - bloom_filter = maybe_bloom_filter.ValueOrDie(); + bloom_filter = std::move(maybe_bloom_filter).ValueOrDie(); + } else { + LOG_WARN("Creating BloomFilter failed: %s", + maybe_bloom_filter.status().error_message()); } } - return {filter.count, bloom_filter}; + return {filter.count, std::move(bloom_filter)}; } bool Serializer::IsLocalResourceName(const ResourcePath& path) const { From 01c332b65081e945d036178489a28aed79e6ba7e Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Thu, 16 Mar 2023 20:36:35 -0700 Subject: [PATCH 18/32] merge queryPurpose spec test check --- .../json/existence_filter_spec_test.json | 114 +++++++++--------- .../Tests/SpecTests/json/limbo_spec_test.json | 4 +- .../SpecTests/json/offline_spec_test.json | 4 +- 3 files changed, 61 insertions(+), 61 deletions(-) diff --git a/Firestore/Example/Tests/SpecTests/json/existence_filter_spec_test.json b/Firestore/Example/Tests/SpecTests/json/existence_filter_spec_test.json index 9a90f96d280..24f9d92cf53 100644 --- a/Firestore/Example/Tests/SpecTests/json/existence_filter_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/existence_filter_spec_test.json @@ -179,7 +179,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "2": { "queries": [ @@ -593,7 +593,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "2": { "queries": [ @@ -792,7 +792,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "2": { "queries": [ @@ -3300,7 +3300,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "11": { "queries": [ @@ -3313,7 +3313,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "13": { "queries": [ @@ -3326,7 +3326,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "15": { "queries": [ @@ -3339,7 +3339,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "17": { "queries": [ @@ -3352,7 +3352,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "19": { "queries": [ @@ -3365,7 +3365,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "2": { "queries": [ @@ -3390,7 +3390,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "23": { "queries": [ @@ -3403,7 +3403,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "25": { "queries": [ @@ -3416,7 +3416,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "27": { "queries": [ @@ -3429,7 +3429,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "29": { "queries": [ @@ -3442,7 +3442,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "3": { "queries": [ @@ -3455,7 +3455,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "31": { "queries": [ @@ -3468,7 +3468,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "33": { "queries": [ @@ -3481,7 +3481,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "35": { "queries": [ @@ -3494,7 +3494,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "37": { "queries": [ @@ -3507,7 +3507,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "39": { "queries": [ @@ -3520,7 +3520,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "41": { "queries": [ @@ -3533,7 +3533,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "43": { "queries": [ @@ -3546,7 +3546,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "45": { "queries": [ @@ -3559,7 +3559,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "47": { "queries": [ @@ -3572,7 +3572,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "49": { "queries": [ @@ -3585,7 +3585,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "5": { "queries": [ @@ -3598,7 +3598,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "51": { "queries": [ @@ -3611,7 +3611,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "53": { "queries": [ @@ -3624,7 +3624,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "55": { "queries": [ @@ -3637,7 +3637,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "57": { "queries": [ @@ -3650,7 +3650,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "59": { "queries": [ @@ -3663,7 +3663,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "61": { "queries": [ @@ -3676,7 +3676,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "63": { "queries": [ @@ -3689,7 +3689,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "65": { "queries": [ @@ -3702,7 +3702,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "67": { "queries": [ @@ -3715,7 +3715,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "69": { "queries": [ @@ -3728,7 +3728,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "7": { "queries": [ @@ -3741,7 +3741,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "71": { "queries": [ @@ -3754,7 +3754,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "73": { "queries": [ @@ -3767,7 +3767,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "75": { "queries": [ @@ -3780,7 +3780,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "77": { "queries": [ @@ -3793,7 +3793,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "79": { "queries": [ @@ -3806,7 +3806,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "81": { "queries": [ @@ -3819,7 +3819,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "83": { "queries": [ @@ -3832,7 +3832,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "85": { "queries": [ @@ -3845,7 +3845,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "87": { "queries": [ @@ -3858,7 +3858,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "89": { "queries": [ @@ -3871,7 +3871,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "9": { "queries": [ @@ -3884,7 +3884,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "91": { "queries": [ @@ -3897,7 +3897,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "93": { "queries": [ @@ -3910,7 +3910,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "95": { "queries": [ @@ -3923,7 +3923,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "97": { "queries": [ @@ -3936,7 +3936,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "99": { "queries": [ @@ -3949,7 +3949,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 } } } @@ -6427,7 +6427,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "2": { "queries": [ @@ -7641,7 +7641,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "2": { "queries": [ @@ -7739,7 +7739,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "2": { "queries": [ @@ -7769,7 +7769,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "4": { "queries": [ diff --git a/Firestore/Example/Tests/SpecTests/json/limbo_spec_test.json b/Firestore/Example/Tests/SpecTests/json/limbo_spec_test.json index 108d2016cb8..3958930b2dc 100644 --- a/Firestore/Example/Tests/SpecTests/json/limbo_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/limbo_spec_test.json @@ -8188,7 +8188,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "2": { "queries": [ @@ -8213,7 +8213,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 } }, "enqueuedLimboDocs": [ diff --git a/Firestore/Example/Tests/SpecTests/json/offline_spec_test.json b/Firestore/Example/Tests/SpecTests/json/offline_spec_test.json index 91e4205e441..1d1013681f9 100644 --- a/Firestore/Example/Tests/SpecTests/json/offline_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/offline_spec_test.json @@ -1150,7 +1150,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "2": { "queries": [ @@ -1188,7 +1188,7 @@ } ], "resumeToken": "", - "targetPurpose": 3 + "targetPurpose": 2 }, "2": { "queries": [ From 4e1e4899deb3656477657a7cdb578f2a8b16afd1 Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Fri, 17 Mar 2023 09:46:48 -0700 Subject: [PATCH 19/32] add back the expectedCount check get deleted --- .../SpecTests/json/listen_spec_test.json | 12 ++++++---- .../core/test/unit/remote/serializer_test.cc | 22 +++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/Firestore/Example/Tests/SpecTests/json/listen_spec_test.json b/Firestore/Example/Tests/SpecTests/json/listen_spec_test.json index a126c226530..85170a91d71 100644 --- a/Firestore/Example/Tests/SpecTests/json/listen_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/listen_spec_test.json @@ -3481,7 +3481,8 @@ "path": "collection" } ], - "resumeToken": "resume-token-1000" + "resumeToken": "resume-token-1000", + "expectedCount": 1 } } } @@ -12750,7 +12751,8 @@ "path": "collection" } ], - "resumeToken": "resume-token-1000" + "resumeToken": "resume-token-1000", + "expectedCount": 1 } } } @@ -12898,7 +12900,8 @@ "path": "collection" } ], - "resumeToken": "resume-token-1000" + "resumeToken": "resume-token-1000", + "expectedCount": 0 } } } @@ -13069,7 +13072,8 @@ "path": "collection" } ], - "resumeToken": "resume-token-2000" + "resumeToken": "resume-token-2000", + "expectedCount": 2 } } } diff --git a/Firestore/core/test/unit/remote/serializer_test.cc b/Firestore/core/test/unit/remote/serializer_test.cc index 23e119f3922..0ad9f6ecae1 100644 --- a/Firestore/core/test/unit/remote/serializer_test.cc +++ b/Firestore/core/test/unit/remote/serializer_test.cc @@ -1751,6 +1751,28 @@ TEST_F(SerializerTest, DecodesListenResponseWithExistenceFilter) { ExpectDeserializationRoundTrip(model, proto); } +TEST_F(SerializerTest, DecodesListenResponseWithExistenceFilterWhenBloomFilterNotNull) { + ExistenceFilterWatchChange model( + ExistenceFilter(2, BloomFilter(std::vector{1}, 2, 3)), 100); + + v1::ListenResponse proto; + + v1::BitSequence* bits = new v1::BitSequence; + std::vector vector{1}; + bits->set_bitmap(reinterpret_cast(vector.data()), vector.size()); + bits->set_padding(2); + v1::BloomFilter* bloom_filter = new v1::BloomFilter; + bloom_filter->set_hash_count(3); + bloom_filter->set_allocated_bits(bits); + + proto.mutable_filter()->set_count(2); + proto.mutable_filter()->set_allocated_unchanged_names(bloom_filter); + proto.mutable_filter()->set_target_id(100); + + SCOPED_TRACE("DecodesListenResponseWithExistenceFilter"); + ExpectDeserializationRoundTrip(model, proto); +} + TEST_F(SerializerTest, DecodesVersion) { auto version = Version(123456789); SnapshotVersion model(version.timestamp()); From 523fd6cb97063f3bf8f689dbc3897a7c6119bab8 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Fri, 17 Mar 2023 15:18:28 -0400 Subject: [PATCH 20/32] re-write CompareBits() --- Firestore/core/src/remote/bloom_filter.cc | 27 ++++++++++++++++------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/Firestore/core/src/remote/bloom_filter.cc b/Firestore/core/src/remote/bloom_filter.cc index ad4c5bcbe7c..1371ab23435 100644 --- a/Firestore/core/src/remote/bloom_filter.cc +++ b/Firestore/core/src/remote/bloom_filter.cc @@ -123,18 +123,29 @@ bool operator!=(const BloomFilter& lhs, const BloomFilter& rhs) { } bool CompareBits(const BloomFilter& lhs, const BloomFilter& rhs) { - for (int32_t i = 0; i < lhs.bit_count(); ++i) { - int32_t offset = i % 8; - const uint8_t& lhs_byte = lhs.bitmap()[i / 8]; - const uint8_t& rhs_byte = rhs.bitmap()[i / 8]; - const bool bit1 = (lhs_byte & (static_cast(0x01) << offset)); - const bool bit2 = (rhs_byte & (static_cast(0x01) << offset)); - if (bit1 != bit2) { + if (lhs.bit_count() != rhs.bit_count()) { + return false; + } + if (lhs.bit_count() == 0) { + return true; + } + + const std::vector& bitmap1 = lhs.bitmap(); + const std::vector& bitmap2 = rhs.bitmap(); + const auto byte_count = static_cast(bitmap1.size()); + + // Compare all bytes from the bitmap, except for the last byte. + for (int32_t i = 0; i < byte_count - 1; ++i) { + if (bitmap1[i] != bitmap2[i]) { return false; } } - return true; + // Compare the last bytes, ignoring the padding. + const int32_t padding = (byte_count * 8) - lhs.bit_count(); + const uint8_t last_byte1 = bitmap1[byte_count - 1] << padding; + const uint8_t last_byte2 = bitmap2[byte_count - 1] << padding; + return (last_byte1 == last_byte2); } } // namespace remote From 6bbc6b2852cdb8335e0027d53345b5547a8b7b90 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Fri, 17 Mar 2023 15:19:06 -0400 Subject: [PATCH 21/32] fix typo --- Firestore/core/src/remote/bloom_filter.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Firestore/core/src/remote/bloom_filter.cc b/Firestore/core/src/remote/bloom_filter.cc index 1371ab23435..2906c88adc4 100644 --- a/Firestore/core/src/remote/bloom_filter.cc +++ b/Firestore/core/src/remote/bloom_filter.cc @@ -141,7 +141,7 @@ bool CompareBits(const BloomFilter& lhs, const BloomFilter& rhs) { } } - // Compare the last bytes, ignoring the padding. + // Compare the last byte, ignoring the padding. const int32_t padding = (byte_count * 8) - lhs.bit_count(); const uint8_t last_byte1 = bitmap1[byte_count - 1] << padding; const uint8_t last_byte2 = bitmap2[byte_count - 1] << padding; From 2a71c206d5f288c86e61cf6c9bfa2380f76dd5a3 Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Fri, 17 Mar 2023 12:35:48 -0700 Subject: [PATCH 22/32] resolve comments --- Firestore/core/src/remote/bloom_filter.cc | 54 ++++++++++--------- Firestore/core/src/remote/bloom_filter.h | 6 +-- .../test/unit/remote/bloom_filter_test.cc | 38 +++++++++---- 3 files changed, 61 insertions(+), 37 deletions(-) diff --git a/Firestore/core/src/remote/bloom_filter.cc b/Firestore/core/src/remote/bloom_filter.cc index d7fa5eee7f6..8f3d45107ab 100644 --- a/Firestore/core/src/remote/bloom_filter.cc +++ b/Firestore/core/src/remote/bloom_filter.cc @@ -29,6 +29,35 @@ namespace remote { using util::Status; using util::StatusOr; +namespace { +bool HasSameBits(const BloomFilter& lhs, const BloomFilter& rhs) { + if (lhs.bit_count() != rhs.bit_count()) { + return false; + } + if (lhs.bit_count() == 0) { + return true; + } + + const std::vector& bitmap1 = lhs.bitmap(); + const std::vector& bitmap2 = rhs.bitmap(); + const auto byte_count = static_cast(bitmap1.size()); + + // Compare all bytes from the bitmap, except for the last byte. + for (int32_t i = 0; i < byte_count - 1; ++i) { + if (bitmap1[i] != bitmap2[i]) { + return false; + } + } + + // Compare the last byte, ignoring the padding. + const int32_t padding = (byte_count * 8) - lhs.bit_count(); + const uint8_t last_byte1 = bitmap1[byte_count - 1] << padding; + const uint8_t last_byte2 = bitmap2[byte_count - 1] << padding; + + return (last_byte1 == last_byte2); +} +} // namespace + BloomFilter::Hash BloomFilter::Md5HashDigest(absl::string_view key) const { std::array md5_digest{util::CalculateMd5Digest(key)}; @@ -115,30 +144,7 @@ bool BloomFilter::MightContain(absl::string_view value) const { bool operator==(const BloomFilter& lhs, const BloomFilter& rhs) { return lhs.bit_count() == rhs.bit_count() && - lhs.hash_count() == rhs.hash_count() && CompareBits(lhs, rhs); -} - -bool operator!=(const BloomFilter& lhs, const BloomFilter& rhs) { - return !(lhs == rhs); -} - -bool CompareBits(const BloomFilter& lhs, const BloomFilter& rhs) { - if (lhs.bit_count() != rhs.bit_count()) { - return false; - } - - for (int32_t i = 0; i < lhs.bit_count(); ++i) { - int32_t offset = i % 8; - const uint8_t& lhs_byte = lhs.bitmap()[i / 8]; - const uint8_t& rhs_byte = rhs.bitmap()[i / 8]; - const bool bit1 = (lhs_byte & (static_cast(0x01) << offset)); - const bool bit2 = (rhs_byte & (static_cast(0x01) << offset)); - if (bit1 != bit2) { - return false; - } - } - - return true; + lhs.hash_count() == rhs.hash_count() && HasSameBits(lhs, rhs); } } // namespace remote diff --git a/Firestore/core/src/remote/bloom_filter.h b/Firestore/core/src/remote/bloom_filter.h index b5bb2003f93..c88d5687865 100644 --- a/Firestore/core/src/remote/bloom_filter.h +++ b/Firestore/core/src/remote/bloom_filter.h @@ -113,9 +113,9 @@ class BloomFilter final { bool operator==(const BloomFilter& lhs, const BloomFilter& rhs); -bool operator!=(const BloomFilter& lhs, const BloomFilter& rhs); - -bool CompareBits(const BloomFilter& lhs, const BloomFilter& rhs); +inline bool operator!=(const BloomFilter& lhs, const BloomFilter& rhs) { + return !(lhs == rhs); +}; } // namespace remote } // namespace firestore diff --git a/Firestore/core/test/unit/remote/bloom_filter_test.cc b/Firestore/core/test/unit/remote/bloom_filter_test.cc index 93583f63d0f..43015e3981c 100644 --- a/Firestore/core/test/unit/remote/bloom_filter_test.cc +++ b/Firestore/core/test/unit/remote/bloom_filter_test.cc @@ -126,10 +126,18 @@ TEST(BloomFilterTest, CheckBloomFiltersEqualityWithSameInput) { } TEST(BloomFilterTest, CheckBloomFiltersEqualityWithDifferentBitmap) { - BloomFilter bloom_filter1(std::vector{1}, 1, 1); - BloomFilter bloom_filter2(std::vector{2}, 1, 1); - EXPECT_FALSE(bloom_filter1 == bloom_filter2); - EXPECT_TRUE(bloom_filter1 != bloom_filter2); + { + BloomFilter bloom_filter1(std::vector{1}, 1, 1); + BloomFilter bloom_filter2(std::vector{2}, 1, 1); + EXPECT_FALSE(bloom_filter1 == bloom_filter2); + EXPECT_TRUE(bloom_filter1 != bloom_filter2); + } + { + BloomFilter bloom_filter1(std::vector{1}, 1, 1); + BloomFilter bloom_filter2(std::vector{1, 1}, 1, 1); + EXPECT_FALSE(bloom_filter1 == bloom_filter2); + EXPECT_TRUE(bloom_filter1 != bloom_filter2); + } } TEST(BloomFilterTest, CheckBloomFiltersEqualityWithDifferentPadding) { @@ -150,12 +158,22 @@ TEST(BloomFilterTest, BloomFiltersEqualityCheckShouldIgnoreBitsInPaddingIndexes) { // In BloomFilter bitmap, padding is guaranteed to be less than 8, and // starts counting from the leftmost indexes of the last byte. - BloomFilter bloom_filter1(std::vector{255, 127}, 1, - 1); // bitmap -> 11111111 01111111 - BloomFilter bloom_filter2(std::vector{255, 255}, 1, - 1); // bitmap -> 11111111 11111111 - EXPECT_TRUE(bloom_filter1 == bloom_filter2); - EXPECT_FALSE(bloom_filter1 != bloom_filter2); + { + BloomFilter bloom_filter1(std::vector{255, 127}, 1, + 1); // bitmap -> 11111111 01111111 + BloomFilter bloom_filter2(std::vector{255, 255}, 1, + 1); // bitmap -> 11111111 11111111 + EXPECT_TRUE(bloom_filter1 == bloom_filter2); + EXPECT_FALSE(bloom_filter1 != bloom_filter2); + } + { + BloomFilter bloom_filter1(std::vector{255, 207}, 4, + 1); // bitmap -> 11111111 11001111 + BloomFilter bloom_filter2(std::vector{255, 255}, 4, + 1); // bitmap -> 11111111 11111111 + EXPECT_TRUE(bloom_filter1 == bloom_filter2); + EXPECT_FALSE(bloom_filter1 != bloom_filter2); + } } TEST(BloomFilterTest, MightContainCanProcessNonStandardCharacters) { From 4a976306933108f568c059db1f1ee2a37664d2ea Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Fri, 17 Mar 2023 14:16:45 -0700 Subject: [PATCH 23/32] resolve comments --- Firestore/Example/Tests/SpecTests/FSTSpecTests.mm | 1 - Firestore/core/src/remote/bloom_filter.h | 2 +- Firestore/core/src/remote/serializer.cc | 5 ++--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm index 703e3807df6..7ab419f357d 100644 --- a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm +++ b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm @@ -334,7 +334,6 @@ - (SnapshotVersion)parseVersion:(NSNumber *_Nullable)version { // code a null value for now. Actual parsing code will be written in the next PR, where we can // validate the parsing result. return absl::nullopt; - ; } - (DocumentViewChange)parseChange:(NSDictionary *)jsonDoc ofType:(DocumentViewChange::Type)type { diff --git a/Firestore/core/src/remote/bloom_filter.h b/Firestore/core/src/remote/bloom_filter.h index c88d5687865..6c0facaad10 100644 --- a/Firestore/core/src/remote/bloom_filter.h +++ b/Firestore/core/src/remote/bloom_filter.h @@ -115,7 +115,7 @@ bool operator==(const BloomFilter& lhs, const BloomFilter& rhs); inline bool operator!=(const BloomFilter& lhs, const BloomFilter& rhs) { return !(lhs == rhs); -}; +} } // namespace remote } // namespace firestore diff --git a/Firestore/core/src/remote/serializer.cc b/Firestore/core/src/remote/serializer.cc index fbdc82e6eae..2b9133bf422 100644 --- a/Firestore/core/src/remote/serializer.cc +++ b/Firestore/core/src/remote/serializer.cc @@ -1430,9 +1430,8 @@ std::unique_ptr Serializer::DecodeDocumentRemove( std::unique_ptr Serializer::DecodeExistenceFilterWatchChange( ReadContext*, const google_firestore_v1_ExistenceFilter& filter) const { - ExistenceFilter existence_filter = DecodeExistenceFilter(filter); return absl::make_unique( - std::move(existence_filter), filter.target_id); + DecodeExistenceFilter(filter), filter.target_id); } ExistenceFilter Serializer::DecodeExistenceFilter( @@ -1440,7 +1439,7 @@ ExistenceFilter Serializer::DecodeExistenceFilter( // Create bloom filter if there is an unchanged_names present in the filter // and inputs are valid, otherwise keep it null. absl::optional bloom_filter; - if (filter.has_unchanged_names && filter.unchanged_names.has_bits) { + if (filter.has_unchanged_names) { // TODO(Mila): None of the ported spec tests here actually has the bloom // filter json string, so hard code an empty bloom filter for now. The // actual parsing code will be written in the next PR, where we can validate From 3eec288085122d4962b4a34d2b290506fc5e05c3 Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Fri, 17 Mar 2023 14:17:27 -0700 Subject: [PATCH 24/32] Update bloom_filter.cc --- Firestore/core/src/remote/bloom_filter.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Firestore/core/src/remote/bloom_filter.cc b/Firestore/core/src/remote/bloom_filter.cc index 8f3d45107ab..7ca39e1b00d 100644 --- a/Firestore/core/src/remote/bloom_filter.cc +++ b/Firestore/core/src/remote/bloom_filter.cc @@ -143,8 +143,7 @@ bool BloomFilter::MightContain(absl::string_view value) const { } bool operator==(const BloomFilter& lhs, const BloomFilter& rhs) { - return lhs.bit_count() == rhs.bit_count() && - lhs.hash_count() == rhs.hash_count() && HasSameBits(lhs, rhs); + return lhs.hash_count() == rhs.hash_count() && HasSameBits(lhs, rhs); } } // namespace remote From ffeac2908c49a76115bb08d40ce074aada48d035 Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Fri, 17 Mar 2023 15:26:41 -0700 Subject: [PATCH 25/32] add test coverage for ExistenceFilter with not null BloomFilter --- Firestore/core/src/remote/remote_event.cc | 17 ++- Firestore/core/src/remote/serializer.cc | 1 - .../test/unit/remote/remote_event_test.cc | 103 +++++++++++++++++- .../core/test/unit/remote/serializer_test.cc | 26 ++--- .../test/unit/remote/watch_change_test.cc | 18 ++- 5 files changed, 137 insertions(+), 28 deletions(-) diff --git a/Firestore/core/src/remote/remote_event.cc b/Firestore/core/src/remote/remote_event.cc index 9792ea8e25a..6b46e10cd8d 100644 --- a/Firestore/core/src/remote/remote_event.cc +++ b/Firestore/core/src/remote/remote_event.cc @@ -238,10 +238,11 @@ void WatchChangeAggregator::HandleExistenceFilter( int current_size = GetCurrentDocumentCountForTarget(target_id); if (current_size != expected_count) { // Apply bloom filter to identify and mark removed documents. - bool bloomFilterApplied = + bool bloom_filter_applied = ApplyBloomFilter(existence_filter, current_size); - - if (!bloomFilterApplied) { + std::cout << "bloom_filter_applied" << bloom_filter_applied + << std::endl; + if (!bloom_filter_applied) { // If bloom filter application fails, we reset the mapping and // trigger re-run of the query. ResetTarget(target_id); @@ -266,6 +267,7 @@ bool WatchChangeAggregator::ApplyBloomFilter( int removed_document_count = FilterRemovedDocuments( bloom_filter.value(), existence_filter.target_id()); + std::cout << "removed_document_count" << removed_document_count << std::endl; return expected_count == (current_count - removed_document_count); } @@ -278,14 +280,17 @@ int WatchChangeAggregator::FilterRemovedDocuments(BloomFilter bloom_filter, int target_id) { const DocumentKeySet& existing_keys = target_metadata_provider_->GetRemoteKeysForTarget(target_id); + std::cout << "existing_keys" << existing_keys.size() << std::endl; + int removalCount = 0; - for (DocumentKey key : existing_keys) { + for (const DocumentKey& key : existing_keys) { DatabaseId database_id = target_metadata_provider_->GetDatabaseId(); - std::string documentPath = util::StringFormat( + std::string document_path = util::StringFormat( "projects/%s/databases/%s/documents/%s", database_id.project_id(), database_id.database_id(), key.ToString()); + std::cout << "documentPath" << document_path << std::endl; - if (!bloom_filter.MightContain(documentPath)) { + if (!bloom_filter.MightContain(document_path)) { RemoveDocumentFromTarget(target_id, key, /*updatedDocument=*/absl::nullopt); removalCount++; diff --git a/Firestore/core/src/remote/serializer.cc b/Firestore/core/src/remote/serializer.cc index ad6fda62c1e..318d12dfd87 100644 --- a/Firestore/core/src/remote/serializer.cc +++ b/Firestore/core/src/remote/serializer.cc @@ -1437,7 +1437,6 @@ std::unique_ptr Serializer::DecodeExistenceFilterWatchChange( ExistenceFilter Serializer::DecodeExistenceFilter( const google_firestore_v1_ExistenceFilter& filter) const { - absl::optional bloom_filter; // Create bloom filter if there is an unchanged_names present in the filter // and inputs are valid, otherwise keep it null. diff --git a/Firestore/core/test/unit/remote/remote_event_test.cc b/Firestore/core/test/unit/remote/remote_event_test.cc index 5e68c22ed66..2b0608f14d2 100644 --- a/Firestore/core/test/unit/remote/remote_event_test.cc +++ b/Firestore/core/test/unit/remote/remote_event_test.cc @@ -512,7 +512,6 @@ TEST_F(RemoteEventTest, NoChangeWillStillMarkTheAffectedTargets) { ASSERT_TRUE(event.target_changes().at(1) == target_change); } -// TODO(Mila): Add test coverage for when the bloom filter is not null TEST_F(RemoteEventTest, ExistenceFilterMismatchClearsTarget) { std::unordered_map target_map = ActiveQueries({1, 2}); @@ -564,6 +563,108 @@ TEST_F(RemoteEventTest, ExistenceFilterMismatchClearsTarget) { ASSERT_EQ(event.document_updates().size(), 0); } +TEST_F(RemoteEventTest, ExistenceFilterMismatchWithBloomFilterSuccess) { + std::unordered_map target_map = ActiveQueries({1, 2}); + + MutableDocument doc1 = Doc("docs/1", 1, Map("value", 1)); + auto change1 = MakeDocChange({1}, {}, doc1.key(), doc1); + MutableDocument doc2 = Doc("docs/2", 2, Map("value", 2)); + auto change2 = MakeDocChange({1}, {}, doc2.key(), doc2); + auto change3 = + MakeTargetChange(WatchTargetChangeState::Current, {1}, resume_token1_); + + WatchChangeAggregator aggregator = CreateAggregator( + target_map, no_outstanding_responses_, + DocumentKeySet{doc1.key(), doc2.key()}, + Changes(std::move(change1), std::move(change2), std::move(change3))); + + RemoteEvent event = aggregator.CreateRemoteEvent(testutil::Version(3)); + + ASSERT_EQ(event.snapshot_version(), testutil::Version(3)); + ASSERT_EQ(event.document_updates().size(), 2); + ASSERT_EQ(event.document_updates().at(doc1.key()), doc1); + ASSERT_EQ(event.document_updates().at(doc2.key()), doc2); + + ASSERT_EQ(event.target_changes().size(), 2); + + TargetChange target_change1{resume_token1_, true, DocumentKeySet{}, + DocumentKeySet{doc1.key(), doc2.key()}, + DocumentKeySet{}}; + ASSERT_TRUE(event.target_changes().at(1) == target_change1); + + TargetChange target_change2{resume_token1_, false, DocumentKeySet{}, + DocumentKeySet{}, DocumentKeySet{}}; + ASSERT_TRUE(event.target_changes().at(2) == target_change2); + + // The given BloomFilter will return false on MightContain(doc1) and true on + // MightContain(doc2). + ExistenceFilterWatchChange change4{ + ExistenceFilter{1, BloomFilter({0x0E, 0x0F}, 1, 7)}, 1}; + // The existence filter mismatch will identify that doc1 is deleted, and skips + // the full re-query. + aggregator.HandleExistenceFilter(change4); + + event = aggregator.CreateRemoteEvent(testutil::Version(4)); + + ASSERT_EQ(event.target_changes().size(), 1); + ASSERT_EQ(event.target_mismatches().size(), 0); + ASSERT_EQ(event.document_updates().size(), 0); +} + +TEST_F(RemoteEventTest, + ExistenceFilterMismatchWithBloomFilterFalsePositiveResult) { + std::unordered_map target_map = ActiveQueries({1, 2}); + + MutableDocument doc1 = Doc("docs/1", 1, Map("value", 1)); + auto change1 = MakeDocChange({1}, {}, doc1.key(), doc1); + MutableDocument doc2 = Doc("docs/2", 2, Map("value", 2)); + auto change2 = MakeDocChange({1}, {}, doc2.key(), doc2); + auto change3 = + MakeTargetChange(WatchTargetChangeState::Current, {1}, resume_token1_); + + WatchChangeAggregator aggregator = CreateAggregator( + target_map, no_outstanding_responses_, + DocumentKeySet{doc1.key(), doc2.key()}, + Changes(std::move(change1), std::move(change2), std::move(change3))); + + RemoteEvent event = aggregator.CreateRemoteEvent(testutil::Version(3)); + + ASSERT_EQ(event.snapshot_version(), testutil::Version(3)); + ASSERT_EQ(event.document_updates().size(), 2); + ASSERT_EQ(event.document_updates().at(doc1.key()), doc1); + ASSERT_EQ(event.document_updates().at(doc2.key()), doc2); + + ASSERT_EQ(event.target_changes().size(), 2); + + TargetChange target_change1{resume_token1_, true, DocumentKeySet{}, + DocumentKeySet{doc1.key(), doc2.key()}, + DocumentKeySet{}}; + ASSERT_TRUE(event.target_changes().at(1) == target_change1); + + TargetChange target_change2{resume_token1_, false, DocumentKeySet{}, + DocumentKeySet{}, DocumentKeySet{}}; + ASSERT_TRUE(event.target_changes().at(2) == target_change2); + + // The given BloomFilter will return false on both MightContain(doc1) and + // MightContain(doc2). + ExistenceFilterWatchChange change4{ + ExistenceFilter{1, BloomFilter({0x42, 0xFE}, 2, 7)}, 1}; + // The existence filter mismatch cannot identify which doc is deleted. It will + // remove the document from target 1, but not synthesize a document delete. + aggregator.HandleExistenceFilter(change4); + + event = aggregator.CreateRemoteEvent(testutil::Version(4)); + + TargetChange target_change3{ByteString(), false, DocumentKeySet{}, + DocumentKeySet{}, + DocumentKeySet{doc1.key(), doc2.key()}}; + ASSERT_TRUE(event.target_changes().at(1) == target_change3); + + ASSERT_EQ(event.target_changes().size(), 1); + ASSERT_EQ(event.target_mismatches().size(), 1); + ASSERT_EQ(event.document_updates().size(), 0); +} + TEST_F(RemoteEventTest, ExistenceFilterMismatchRemovesCurrentChanges) { std::unordered_map target_map = ActiveQueries({1}); diff --git a/Firestore/core/test/unit/remote/serializer_test.cc b/Firestore/core/test/unit/remote/serializer_test.cc index 0ad9f6ecae1..c355141eb00 100644 --- a/Firestore/core/test/unit/remote/serializer_test.cc +++ b/Firestore/core/test/unit/remote/serializer_test.cc @@ -1737,7 +1737,6 @@ TEST_F(SerializerTest, DecodesListenResponseWithDocumentRemove) { ExpectDeserializationRoundTrip(model, proto); } -// TODO(Mila): Add test coverage for when the bloom filter is not null TEST_F(SerializerTest, DecodesListenResponseWithExistenceFilter) { ExistenceFilterWatchChange model( ExistenceFilter(2, /*bloom_filter=*/absl::nullopt), 100); @@ -1751,23 +1750,20 @@ TEST_F(SerializerTest, DecodesListenResponseWithExistenceFilter) { ExpectDeserializationRoundTrip(model, proto); } -TEST_F(SerializerTest, DecodesListenResponseWithExistenceFilterWhenBloomFilterNotNull) { +TEST_F(SerializerTest, + DecodesListenResponseWithExistenceFilterWhenBloomFilterNotNull) { ExistenceFilterWatchChange model( - ExistenceFilter(2, BloomFilter(std::vector{1}, 2, 3)), 100); + ExistenceFilter(555, BloomFilter({0x42, 0xFE}, 7, 33)), 999); v1::ListenResponse proto; - - v1::BitSequence* bits = new v1::BitSequence; - std::vector vector{1}; - bits->set_bitmap(reinterpret_cast(vector.data()), vector.size()); - bits->set_padding(2); - v1::BloomFilter* bloom_filter = new v1::BloomFilter; - bloom_filter->set_hash_count(3); - bloom_filter->set_allocated_bits(bits); - - proto.mutable_filter()->set_count(2); - proto.mutable_filter()->set_allocated_unchanged_names(bloom_filter); - proto.mutable_filter()->set_target_id(100); + proto.mutable_filter()->set_count(555); + proto.mutable_filter()->set_target_id(999); + + v1::BloomFilter* bloom_filter = + proto.mutable_filter()->mutable_unchanged_names(); + bloom_filter->set_hash_count(33); + bloom_filter->mutable_bits()->set_padding(7); + bloom_filter->mutable_bits()->set_bitmap("\x42\xFE"); SCOPED_TRACE("DecodesListenResponseWithExistenceFilter"); ExpectDeserializationRoundTrip(model, proto); diff --git a/Firestore/core/test/unit/remote/watch_change_test.cc b/Firestore/core/test/unit/remote/watch_change_test.cc index c560d63a3f5..b92e57bfd23 100644 --- a/Firestore/core/test/unit/remote/watch_change_test.cc +++ b/Firestore/core/test/unit/remote/watch_change_test.cc @@ -40,12 +40,20 @@ TEST(WatchChangeTest, CanCreateDocumentWatchChange) { EXPECT_EQ(change.new_document(), doc); } -// TODO(Mila): Add test coverage for when the bloom filter is not null TEST(WatchChangeTest, CanCreateExistenceFilterWatchChange) { - ExistenceFilter filter{7, /*bloom_filter=*/absl::nullopt}; - ExistenceFilterWatchChange change{filter, 5}; - EXPECT_EQ(change.filter().count(), 7); - EXPECT_EQ(change.target_id(), 5); + { + ExistenceFilter filter{7, /*bloom_filter=*/absl::nullopt}; + ExistenceFilterWatchChange change{filter, 5}; + EXPECT_EQ(change.filter().count(), 7); + EXPECT_EQ(change.target_id(), 5); + } + { + ExistenceFilter filter{7, BloomFilter({0x42, 0xFE}, 7, 33)}; + ExistenceFilterWatchChange change{std::move(filter), 5}; + EXPECT_EQ(change.filter().count(), 7); + EXPECT_EQ(change.filter().bloom_filter(), BloomFilter({0x42, 0xFE}, 7, 33)); + EXPECT_EQ(change.target_id(), 5); + } } TEST(WatchChangeTest, CanCreateWatchTargetChange) { From dd0d0fd3e868febb1cfdfc3d89f1e79872a41e09 Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Mon, 20 Mar 2023 12:24:53 -0700 Subject: [PATCH 26/32] nit --- .../Example/Tests/SpecTests/FSTSpecTests.mm | 11 +++++---- Firestore/core/src/remote/datastore.cc | 8 +++---- Firestore/core/src/remote/datastore.h | 2 +- Firestore/core/src/remote/remote_event.cc | 23 +++++-------------- Firestore/core/src/remote/remote_event.h | 5 ++-- Firestore/core/src/remote/serializer.cc | 20 +++++++++------- .../core/test/unit/remote/serializer_test.cc | 1 - 7 files changed, 32 insertions(+), 38 deletions(-) diff --git a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm index 2f413dc7f1e..ff091299cc1 100644 --- a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm +++ b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm @@ -335,19 +335,22 @@ - (SnapshotVersion)parseVersion:(NSNumber *_Nullable)version { } NSDictionary *bitsData = bloomFilterProto[@"bits"]; - // Decode base64 string into uint8_t vector. If not specified, will default bitmap to empty - // vector. + // Decode base64 string into uint8_t vector. If bitmap is not specified in proto, use default + // empty string. NSString *bitmapEncoded = bitsData[@"bitmap"]; std::string bitmapDecoded; - absl::Base64Unescape([bitmapEncoded UTF8String], &bitmapDecoded); + absl::Base64Unescape([bitmapEncoded cStringUsingEncoding:NSASCIIStringEncoding], &bitmapDecoded); std::vector bitmap(bitmapDecoded.begin(), bitmapDecoded.end()); - // If not specified, will default padding and hashCount to 0. + // If not specified in proto, default padding and hashCount to 0. int32_t padding = [bitsData[@"padding"] intValue]; int32_t hashCount = [bloomFilterProto[@"hashCount"] intValue]; + StatusOr maybeBloomFilter = BloomFilter::Create(bitmap, padding, hashCount); if (maybeBloomFilter.ok()) { return maybeBloomFilter.ValueOrDie(); + } else { + LOG_WARN("Parsing BloomFilterProto failed: %s", maybeBloomFilter.status().error_message()); } return absl::nullopt; diff --git a/Firestore/core/src/remote/datastore.cc b/Firestore/core/src/remote/datastore.cc index f837c7801a4..0f67ba555ef 100644 --- a/Firestore/core/src/remote/datastore.cc +++ b/Firestore/core/src/remote/datastore.cc @@ -106,11 +106,11 @@ Datastore::Datastore( auth_credentials_{std::move(auth_credentials)}, rpc_executor_{CreateExecutor()}, connectivity_monitor_{connectivity_monitor}, - grpc_connection_{database_info, worker_queue, &grpc_queue_, + database_info_{std::move(database_info)}, + grpc_connection_{database_info_, worker_queue, &grpc_queue_, connectivity_monitor_, firebase_metadata_provider}, - database_info_{database_info}, - datastore_serializer_{database_info} { - if (!database_info.ssl_enabled()) { + datastore_serializer_{database_info_} { + if (!database_info_.ssl_enabled()) { GrpcConnection::UseInsecureChannel(database_info.host()); } } diff --git a/Firestore/core/src/remote/datastore.h b/Firestore/core/src/remote/datastore.h index 7afa403c1ee..4a2e2e3e9f4 100644 --- a/Firestore/core/src/remote/datastore.h +++ b/Firestore/core/src/remote/datastore.h @@ -213,10 +213,10 @@ class Datastore : public std::enable_shared_from_this { std::unique_ptr rpc_executor_; grpc::CompletionQueue grpc_queue_; ConnectivityMonitor* connectivity_monitor_ = nullptr; + core::DatabaseInfo database_info_; GrpcConnection grpc_connection_; std::vector> active_calls_; - core::DatabaseInfo database_info_; DatastoreSerializer datastore_serializer_; }; diff --git a/Firestore/core/src/remote/remote_event.cc b/Firestore/core/src/remote/remote_event.cc index 6b46e10cd8d..1eef9f88d35 100644 --- a/Firestore/core/src/remote/remote_event.cc +++ b/Firestore/core/src/remote/remote_event.cc @@ -237,11 +237,10 @@ void WatchChangeAggregator::HandleExistenceFilter( } else { int current_size = GetCurrentDocumentCountForTarget(target_id); if (current_size != expected_count) { + // TODO(Mila): Use application status instead of a boolean in next PR. // Apply bloom filter to identify and mark removed documents. bool bloom_filter_applied = ApplyBloomFilter(existence_filter, current_size); - std::cout << "bloom_filter_applied" << bloom_filter_applied - << std::endl; if (!bloom_filter_applied) { // If bloom filter application fails, we reset the mapping and // trigger re-run of the query. @@ -253,12 +252,10 @@ void WatchChangeAggregator::HandleExistenceFilter( } } -/** Returns whether a bloom filter removed the deleted documents successfully. - */ bool WatchChangeAggregator::ApplyBloomFilter( - ExistenceFilterWatchChange existence_filter, int current_count) { + const ExistenceFilterWatchChange& existence_filter, int current_count) { int expected_count = existence_filter.filter().count(); - absl::optional bloom_filter = + const absl::optional& bloom_filter = existence_filter.filter().bloom_filter(); if (!bloom_filter.has_value()) { @@ -267,28 +264,20 @@ bool WatchChangeAggregator::ApplyBloomFilter( int removed_document_count = FilterRemovedDocuments( bloom_filter.value(), existence_filter.target_id()); - std::cout << "removed_document_count" << removed_document_count << std::endl; return expected_count == (current_count - removed_document_count); } -/** - * Filter out removed documents based on bloom filter membership result and - * return number of documents removed. - */ -int WatchChangeAggregator::FilterRemovedDocuments(BloomFilter bloom_filter, - int target_id) { +int WatchChangeAggregator::FilterRemovedDocuments( + const BloomFilter& bloom_filter, int target_id) { const DocumentKeySet& existing_keys = target_metadata_provider_->GetRemoteKeysForTarget(target_id); - std::cout << "existing_keys" << existing_keys.size() << std::endl; - int removalCount = 0; for (const DocumentKey& key : existing_keys) { - DatabaseId database_id = target_metadata_provider_->GetDatabaseId(); + const DatabaseId& database_id = target_metadata_provider_->GetDatabaseId(); std::string document_path = util::StringFormat( "projects/%s/databases/%s/documents/%s", database_id.project_id(), database_id.database_id(), key.ToString()); - std::cout << "documentPath" << document_path << std::endl; if (!bloom_filter.MightContain(document_path)) { RemoveDocumentFromTarget(target_id, key, diff --git a/Firestore/core/src/remote/remote_event.h b/Firestore/core/src/remote/remote_event.h index 5b35a325270..547028f39dd 100644 --- a/Firestore/core/src/remote/remote_event.h +++ b/Firestore/core/src/remote/remote_event.h @@ -413,17 +413,16 @@ class WatchChangeAggregator { bool TargetContainsDocument(model::TargetId target_id, const model::DocumentKey& key); - // TODO(Mila) /** Returns whether a bloom filter removed the deleted documents successfully. */ - bool ApplyBloomFilter(ExistenceFilterWatchChange existence_filter, + bool ApplyBloomFilter(const ExistenceFilterWatchChange& existence_filter, int current_count); /** * Filter out removed documents based on bloom filter membership result and * return number of documents removed. */ - int FilterRemovedDocuments(BloomFilter bloom_filter, int target_id); + int FilterRemovedDocuments(const BloomFilter& bloom_filter, int target_id); /** The internal state of all tracked targets. */ std::unordered_map target_states_; diff --git a/Firestore/core/src/remote/serializer.cc b/Firestore/core/src/remote/serializer.cc index 73de562d212..48579790f43 100644 --- a/Firestore/core/src/remote/serializer.cc +++ b/Firestore/core/src/remote/serializer.cc @@ -1437,25 +1437,29 @@ std::unique_ptr Serializer::DecodeExistenceFilterWatchChange( ExistenceFilter Serializer::DecodeExistenceFilter( const google_firestore_v1_ExistenceFilter& filter) const { - absl::optional bloom_filter; - // Create bloom filter if there is an unchanged_names present in the filter - // and inputs are valid, otherwise keep it null. if (filter.has_unchanged_names) { - pb_bytes_array_t* bitmap_ptr = filter.unchanged_names.bits.bitmap; + pb_bytes_array_t* bitmap_ptr = filter.unchanged_names.has_bits + ? filter.unchanged_names.bits.bitmap + : nullptr; std::vector bitmap(bitmap_ptr->bytes, bitmap_ptr->bytes + bitmap_ptr->size); - int32_t padding = filter.unchanged_names.bits.padding; + + int32_t padding = filter.unchanged_names.has_bits + ? filter.unchanged_names.bits.padding + : 0; int32_t hash_count = filter.unchanged_names.hash_count; StatusOr maybe_bloom_filter = BloomFilter::Create(bitmap, padding, hash_count); + if (maybe_bloom_filter.ok()) { - bloom_filter = std::move(maybe_bloom_filter).ValueOrDie(); + return {filter.count, std::move(maybe_bloom_filter).ValueOrDie()}; } else { LOG_WARN("Creating BloomFilter failed: %s", maybe_bloom_filter.status().error_message()); - - return {filter.count, std::move(bloom_filter)}; + } + } + return {filter.count, absl::nullopt}; } bool Serializer::IsLocalResourceName(const ResourcePath& path) const { diff --git a/Firestore/core/test/unit/remote/serializer_test.cc b/Firestore/core/test/unit/remote/serializer_test.cc index b487a3794b1..c355141eb00 100644 --- a/Firestore/core/test/unit/remote/serializer_test.cc +++ b/Firestore/core/test/unit/remote/serializer_test.cc @@ -1737,7 +1737,6 @@ TEST_F(SerializerTest, DecodesListenResponseWithDocumentRemove) { ExpectDeserializationRoundTrip(model, proto); } -// TODO(Mila): Add test coverage for when the bloom filter is not null TEST_F(SerializerTest, DecodesListenResponseWithExistenceFilter) { ExistenceFilterWatchChange model( ExistenceFilter(2, /*bloom_filter=*/absl::nullopt), 100); From f412d2070c8978a53bec97a0b93c26cefe70db7d Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Mon, 20 Mar 2023 14:15:48 -0700 Subject: [PATCH 27/32] format --- Firestore/Example/Tests/SpecTests/FSTSpecTests.mm | 7 +++---- Firestore/core/src/remote/remote_event.cc | 3 +-- Firestore/core/test/unit/remote/remote_event_test.cc | 10 +++++----- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm index ff091299cc1..f712ab801c3 100644 --- a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm +++ b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm @@ -345,15 +345,14 @@ - (SnapshotVersion)parseVersion:(NSNumber *_Nullable)version { // If not specified in proto, default padding and hashCount to 0. int32_t padding = [bitsData[@"padding"] intValue]; int32_t hashCount = [bloomFilterProto[@"hashCount"] intValue]; - StatusOr maybeBloomFilter = BloomFilter::Create(bitmap, padding, hashCount); + if (maybeBloomFilter.ok()) { - return maybeBloomFilter.ValueOrDie(); + return std::move(maybeBloomFilter).ValueOrDie(); } else { LOG_WARN("Parsing BloomFilterProto failed: %s", maybeBloomFilter.status().error_message()); + return absl::nullopt; } - - return absl::nullopt; } - (DocumentViewChange)parseChange:(NSDictionary *)jsonDoc ofType:(DocumentViewChange::Type)type { diff --git a/Firestore/core/src/remote/remote_event.cc b/Firestore/core/src/remote/remote_event.cc index 1eef9f88d35..d35a884db73 100644 --- a/Firestore/core/src/remote/remote_event.cc +++ b/Firestore/core/src/remote/remote_event.cc @@ -254,10 +254,8 @@ void WatchChangeAggregator::HandleExistenceFilter( bool WatchChangeAggregator::ApplyBloomFilter( const ExistenceFilterWatchChange& existence_filter, int current_count) { - int expected_count = existence_filter.filter().count(); const absl::optional& bloom_filter = existence_filter.filter().bloom_filter(); - if (!bloom_filter.has_value()) { return false; } @@ -265,6 +263,7 @@ bool WatchChangeAggregator::ApplyBloomFilter( int removed_document_count = FilterRemovedDocuments( bloom_filter.value(), existence_filter.target_id()); + int expected_count = existence_filter.filter().count(); return expected_count == (current_count - removed_document_count); } diff --git a/Firestore/core/test/unit/remote/remote_event_test.cc b/Firestore/core/test/unit/remote/remote_event_test.cc index 2b0608f14d2..8464c1e79d1 100644 --- a/Firestore/core/test/unit/remote/remote_event_test.cc +++ b/Firestore/core/test/unit/remote/remote_event_test.cc @@ -600,8 +600,8 @@ TEST_F(RemoteEventTest, ExistenceFilterMismatchWithBloomFilterSuccess) { // MightContain(doc2). ExistenceFilterWatchChange change4{ ExistenceFilter{1, BloomFilter({0x0E, 0x0F}, 1, 7)}, 1}; - // The existence filter mismatch will identify that doc1 is deleted, and skips - // the full re-query. + // The existence filter identifies that doc1 is deleted, and skips the full + // re-query. aggregator.HandleExistenceFilter(change4); event = aggregator.CreateRemoteEvent(testutil::Version(4)); @@ -645,12 +645,12 @@ TEST_F(RemoteEventTest, DocumentKeySet{}, DocumentKeySet{}}; ASSERT_TRUE(event.target_changes().at(2) == target_change2); - // The given BloomFilter will return false on both MightContain(doc1) and + // The given BloomFilter will return true on both MightContain(doc1) and // MightContain(doc2). ExistenceFilterWatchChange change4{ ExistenceFilter{1, BloomFilter({0x42, 0xFE}, 2, 7)}, 1}; - // The existence filter mismatch cannot identify which doc is deleted. It will - // remove the document from target 1, but not synthesize a document delete. + // The existence filter cannot identify which doc is deleted. It will remove + // the document from target 1, but not synthesize a document delete. aggregator.HandleExistenceFilter(change4); event = aggregator.CreateRemoteEvent(testutil::Version(4)); From 2f9385163c4b9716f09fbd793e818ac06f172c5b Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Mon, 20 Mar 2023 22:07:12 -0700 Subject: [PATCH 28/32] initial code --- .../SpecTests/json/bundle_spec_test.json | 2 +- .../json/existence_filter_spec_test.json | 120 +++++++++--------- .../Tests/SpecTests/json/limbo_spec_test.json | 120 +++++++++--------- .../Tests/SpecTests/json/limit_spec_test.json | 20 +-- .../SpecTests/json/offline_spec_test.json | 4 +- .../SpecTests/json/recovery_spec_test.json | 8 +- Firestore/core/src/core/sync_engine.cc | 2 +- Firestore/core/src/local/target_data.cc | 2 + Firestore/core/src/local/target_data.h | 6 + Firestore/core/src/remote/remote_event.cc | 23 +++- Firestore/core/src/remote/remote_event.h | 21 +-- Firestore/core/src/remote/remote_store.cc | 6 +- Firestore/core/src/remote/serializer.cc | 2 + 13 files changed, 180 insertions(+), 156 deletions(-) diff --git a/Firestore/Example/Tests/SpecTests/json/bundle_spec_test.json b/Firestore/Example/Tests/SpecTests/json/bundle_spec_test.json index 94bc9cc0da1..617f5946d5f 100644 --- a/Firestore/Example/Tests/SpecTests/json/bundle_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/bundle_spec_test.json @@ -1166,7 +1166,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ diff --git a/Firestore/Example/Tests/SpecTests/json/existence_filter_spec_test.json b/Firestore/Example/Tests/SpecTests/json/existence_filter_spec_test.json index 24f9d92cf53..131d47cd4a1 100644 --- a/Firestore/Example/Tests/SpecTests/json/existence_filter_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/existence_filter_spec_test.json @@ -179,7 +179,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -593,7 +593,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -792,7 +792,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -3300,7 +3300,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "11": { "queries": [ @@ -3313,7 +3313,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "13": { "queries": [ @@ -3326,7 +3326,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "15": { "queries": [ @@ -3339,7 +3339,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "17": { "queries": [ @@ -3352,7 +3352,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "19": { "queries": [ @@ -3365,7 +3365,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -3390,7 +3390,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "23": { "queries": [ @@ -3403,7 +3403,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "25": { "queries": [ @@ -3416,7 +3416,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "27": { "queries": [ @@ -3429,7 +3429,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "29": { "queries": [ @@ -3442,7 +3442,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "3": { "queries": [ @@ -3455,7 +3455,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "31": { "queries": [ @@ -3468,7 +3468,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "33": { "queries": [ @@ -3481,7 +3481,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "35": { "queries": [ @@ -3494,7 +3494,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "37": { "queries": [ @@ -3507,7 +3507,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "39": { "queries": [ @@ -3520,7 +3520,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "41": { "queries": [ @@ -3533,7 +3533,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "43": { "queries": [ @@ -3546,7 +3546,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "45": { "queries": [ @@ -3559,7 +3559,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "47": { "queries": [ @@ -3572,7 +3572,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "49": { "queries": [ @@ -3585,7 +3585,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "5": { "queries": [ @@ -3598,7 +3598,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "51": { "queries": [ @@ -3611,7 +3611,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "53": { "queries": [ @@ -3624,7 +3624,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "55": { "queries": [ @@ -3637,7 +3637,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "57": { "queries": [ @@ -3650,7 +3650,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "59": { "queries": [ @@ -3663,7 +3663,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "61": { "queries": [ @@ -3676,7 +3676,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "63": { "queries": [ @@ -3689,7 +3689,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "65": { "queries": [ @@ -3702,7 +3702,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "67": { "queries": [ @@ -3715,7 +3715,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "69": { "queries": [ @@ -3728,7 +3728,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "7": { "queries": [ @@ -3741,7 +3741,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "71": { "queries": [ @@ -3754,7 +3754,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "73": { "queries": [ @@ -3767,7 +3767,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "75": { "queries": [ @@ -3780,7 +3780,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "77": { "queries": [ @@ -3793,7 +3793,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "79": { "queries": [ @@ -3806,7 +3806,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "81": { "queries": [ @@ -3819,7 +3819,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "83": { "queries": [ @@ -3832,7 +3832,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "85": { "queries": [ @@ -3845,7 +3845,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "87": { "queries": [ @@ -3858,7 +3858,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "89": { "queries": [ @@ -3871,7 +3871,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "9": { "queries": [ @@ -3884,7 +3884,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "91": { "queries": [ @@ -3897,7 +3897,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "93": { "queries": [ @@ -3910,7 +3910,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "95": { "queries": [ @@ -3923,7 +3923,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "97": { "queries": [ @@ -3936,7 +3936,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "99": { "queries": [ @@ -3949,7 +3949,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 } } } @@ -4961,7 +4961,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -5536,7 +5536,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -5889,7 +5889,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -6427,7 +6427,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -7641,7 +7641,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -7739,7 +7739,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -7769,7 +7769,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ diff --git a/Firestore/Example/Tests/SpecTests/json/limbo_spec_test.json b/Firestore/Example/Tests/SpecTests/json/limbo_spec_test.json index 3958930b2dc..de82951040a 100644 --- a/Firestore/Example/Tests/SpecTests/json/limbo_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/limbo_spec_test.json @@ -235,7 +235,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -281,7 +281,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -406,7 +406,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -489,7 +489,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -572,7 +572,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -675,7 +675,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "10": { "queries": [ @@ -775,7 +775,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "10": { "queries": [ @@ -866,7 +866,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -940,7 +940,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -1202,7 +1202,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -1248,7 +1248,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -1373,7 +1373,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -1456,7 +1456,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -1539,7 +1539,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -1642,7 +1642,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "10": { "queries": [ @@ -1742,7 +1742,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "10": { "queries": [ @@ -2038,7 +2038,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -2123,7 +2123,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -2206,7 +2206,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -2420,7 +2420,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -2834,7 +2834,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -3119,7 +3119,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -3386,7 +3386,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -3633,7 +3633,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -3921,7 +3921,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -4037,7 +4037,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -4298,7 +4298,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -4646,7 +4646,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -5025,7 +5025,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -5114,7 +5114,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -5486,7 +5486,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -5511,7 +5511,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 } } } @@ -5623,7 +5623,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -5648,7 +5648,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 } } } @@ -5703,7 +5703,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 } } } @@ -5822,7 +5822,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 } } } @@ -6255,7 +6255,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -6782,7 +6782,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -7109,7 +7109,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -7193,7 +7193,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -7512,7 +7512,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -7537,7 +7537,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -7646,7 +7646,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "7": { "queries": [ @@ -7659,7 +7659,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -7765,7 +7765,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -8188,7 +8188,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -8213,7 +8213,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -8634,7 +8634,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -8660,7 +8660,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -8767,7 +8767,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -9097,7 +9097,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -9122,7 +9122,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -9212,7 +9212,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "5": { "queries": [ @@ -9225,7 +9225,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -9309,7 +9309,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "7": { "queries": [ @@ -9322,7 +9322,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -9405,7 +9405,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "9": { "queries": [ @@ -9418,7 +9418,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -9499,7 +9499,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 } }, "enqueuedLimboDocs": [ @@ -9871,7 +9871,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ diff --git a/Firestore/Example/Tests/SpecTests/json/limit_spec_test.json b/Firestore/Example/Tests/SpecTests/json/limit_spec_test.json index 3b226ee1061..ed2b461ce74 100644 --- a/Firestore/Example/Tests/SpecTests/json/limit_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/limit_spec_test.json @@ -214,7 +214,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -4334,7 +4334,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -4361,7 +4361,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -4495,7 +4495,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -4520,7 +4520,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 } } } @@ -4661,7 +4661,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "7": { "queries": [ @@ -4674,7 +4674,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 } } } @@ -4814,7 +4814,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 } } } @@ -5304,7 +5304,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -5565,7 +5565,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ diff --git a/Firestore/Example/Tests/SpecTests/json/offline_spec_test.json b/Firestore/Example/Tests/SpecTests/json/offline_spec_test.json index 1d1013681f9..91e4205e441 100644 --- a/Firestore/Example/Tests/SpecTests/json/offline_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/offline_spec_test.json @@ -1150,7 +1150,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ @@ -1188,7 +1188,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "2": { "queries": [ diff --git a/Firestore/Example/Tests/SpecTests/json/recovery_spec_test.json b/Firestore/Example/Tests/SpecTests/json/recovery_spec_test.json index bb7306476a3..2e66d63b8d5 100644 --- a/Firestore/Example/Tests/SpecTests/json/recovery_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/recovery_spec_test.json @@ -3095,7 +3095,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -3184,7 +3184,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -3569,7 +3569,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ @@ -3629,7 +3629,7 @@ } ], "resumeToken": "", - "targetPurpose": 2 + "targetPurpose": 3 }, "4": { "queries": [ diff --git a/Firestore/core/src/core/sync_engine.cc b/Firestore/core/src/core/sync_engine.cc index 6899c236870..1ab3fdc4216 100644 --- a/Firestore/core/src/core/sync_engine.cc +++ b/Firestore/core/src/core/sync_engine.cc @@ -340,7 +340,7 @@ void SyncEngine::HandleRejectedListen(TargetId target_id, Status error) { // copy-initialization" error. DocumentKeySet limbo_documents{limbo_key}; RemoteEvent::TargetChangeMap target_changes; - RemoteEvent::TargetSet target_mismatches; + RemoteEvent::TargetMismatchMap target_mismatches; DocumentUpdateMap document_updates{{limbo_key, doc}}; RemoteEvent event{SnapshotVersion::None(), std::move(target_changes), diff --git a/Firestore/core/src/local/target_data.cc b/Firestore/core/src/local/target_data.cc index d056b41c38a..4512e2f5d89 100644 --- a/Firestore/core/src/local/target_data.cc +++ b/Firestore/core/src/local/target_data.cc @@ -39,6 +39,8 @@ const char* ToString(QueryPurpose purpose) { return "Listen"; case QueryPurpose::ExistenceFilterMismatch: return "ExistenceFilterMismatch"; + case QueryPurpose::ExistenceFilterMismatchBloom: + return "ExistenceFilterMismatchBloom"; case QueryPurpose::LimboResolution: return "LimboResolution"; } diff --git a/Firestore/core/src/local/target_data.h b/Firestore/core/src/local/target_data.h index 5bdb94a423e..829f352a285 100644 --- a/Firestore/core/src/local/target_data.h +++ b/Firestore/core/src/local/target_data.h @@ -41,6 +41,12 @@ enum class QueryPurpose { */ ExistenceFilterMismatch, + /** + * The query target was used if the query is the result of a false positive + * in the bloom filter. + */ + ExistenceFilterMismatchBloom, + /** The query was used to resolve a limbo document. */ LimboResolution, }; diff --git a/Firestore/core/src/remote/remote_event.cc b/Firestore/core/src/remote/remote_event.cc index d35a884db73..1a206b32954 100644 --- a/Firestore/core/src/remote/remote_event.cc +++ b/Firestore/core/src/remote/remote_event.cc @@ -239,32 +239,43 @@ void WatchChangeAggregator::HandleExistenceFilter( if (current_size != expected_count) { // TODO(Mila): Use application status instead of a boolean in next PR. // Apply bloom filter to identify and mark removed documents. - bool bloom_filter_applied = + local::BloomFilterApplicationStatus status = ApplyBloomFilter(existence_filter, current_size); - if (!bloom_filter_applied) { + if (status != local::BloomFilterApplicationStatus::Success) { // If bloom filter application fails, we reset the mapping and // trigger re-run of the query. ResetTarget(target_id); - pending_target_resets_.insert(target_id); + const local::QueryPurpose purpose = + status == local::BloomFilterApplicationStatus::FalsePositive + ? local::QueryPurpose::ExistenceFilterMismatchBloom + : local::QueryPurpose::ExistenceFilterMismatch; + pending_target_resets_.insert({target_id, purpose}); } } } } } -bool WatchChangeAggregator::ApplyBloomFilter( +local::BloomFilterApplicationStatus WatchChangeAggregator::ApplyBloomFilter( const ExistenceFilterWatchChange& existence_filter, int current_count) { const absl::optional& bloom_filter = existence_filter.filter().bloom_filter(); if (!bloom_filter.has_value()) { - return false; + return local::BloomFilterApplicationStatus::Skipped; + } + + if (bloom_filter.value().bit_count() == 0) { + return local::BloomFilterApplicationStatus::Skipped; } int removed_document_count = FilterRemovedDocuments( bloom_filter.value(), existence_filter.target_id()); int expected_count = existence_filter.filter().count(); - return expected_count == (current_count - removed_document_count); + if (expected_count != (current_count - removed_document_count)) { + return local::BloomFilterApplicationStatus::FalsePositive; + } + return local::BloomFilterApplicationStatus::Success; } int WatchChangeAggregator::FilterRemovedDocuments( diff --git a/Firestore/core/src/remote/remote_event.h b/Firestore/core/src/remote/remote_event.h index 547028f39dd..fc65ca92158 100644 --- a/Firestore/core/src/remote/remote_event.h +++ b/Firestore/core/src/remote/remote_event.h @@ -17,6 +17,7 @@ #ifndef FIRESTORE_CORE_SRC_REMOTE_REMOTE_EVENT_H_ #define FIRESTORE_CORE_SRC_REMOTE_REMOTE_EVENT_H_ +#include #include #include #include @@ -37,6 +38,7 @@ namespace firestore { namespace local { class TargetData; +enum class BloomFilterApplicationStatus { Success, Skipped, FalsePositive }; } // namespace local namespace remote { @@ -246,11 +248,11 @@ class TargetState { class RemoteEvent { public: using TargetChangeMap = std::unordered_map; - using TargetSet = std::unordered_set; + using TargetMismatchMap = std::map; RemoteEvent(model::SnapshotVersion snapshot_version, TargetChangeMap target_changes, - TargetSet target_mismatches, + TargetMismatchMap target_mismatches, model::DocumentUpdateMap document_updates, model::DocumentKeySet limbo_document_changes) : snapshot_version_{snapshot_version}, @@ -271,10 +273,11 @@ class RemoteEvent { } /** - * A set of targets that is known to be inconsistent. Listens for these - * targets should be re-established without resume tokens. + * A map of targets that is known to be inconsistent, and the purpose for + * re-listening. Listens for these targets should be re-established without + * resume tokens. */ - const TargetSet& target_mismatches() const { + const TargetMismatchMap& target_mismatches() const { return target_mismatches_; } @@ -296,7 +299,7 @@ class RemoteEvent { private: model::SnapshotVersion snapshot_version_; TargetChangeMap target_changes_; - TargetSet target_mismatches_; + TargetMismatchMap target_mismatches_; model::DocumentUpdateMap document_updates_; model::DocumentKeySet limbo_document_changes_; }; @@ -415,8 +418,8 @@ class WatchChangeAggregator { /** Returns whether a bloom filter removed the deleted documents successfully. */ - bool ApplyBloomFilter(const ExistenceFilterWatchChange& existence_filter, - int current_count); + local::BloomFilterApplicationStatus ApplyBloomFilter( + const ExistenceFilterWatchChange& existence_filter, int current_count); /** * Filter out removed documents based on bloom filter membership result and @@ -441,7 +444,7 @@ class WatchChangeAggregator { * to be inconsistent and their listens needs to be re-established by * `RemoteStore`. */ - RemoteEvent::TargetSet pending_target_resets_; + RemoteEvent::TargetMismatchMap pending_target_resets_; TargetMetadataProvider* target_metadata_provider_ = nullptr; }; diff --git a/Firestore/core/src/remote/remote_store.cc b/Firestore/core/src/remote/remote_store.cc index 87f2835c7bc..a63c690f46b 100644 --- a/Firestore/core/src/remote/remote_store.cc +++ b/Firestore/core/src/remote/remote_store.cc @@ -327,7 +327,8 @@ void RemoteStore::RaiseWatchSnapshot(const SnapshotVersion& snapshot_version) { // Re-establish listens for the targets that have been invalidated by // existence filter mismatches. - for (TargetId target_id : remote_event.target_mismatches()) { + for (const auto& entry : remote_event.target_mismatches()) { + const auto target_id = entry.first; auto found = listen_targets_.find(target_id); if (found == listen_targets_.end()) { // A watched target might have been removed already. @@ -351,8 +352,7 @@ void RemoteStore::RaiseWatchSnapshot(const SnapshotVersion& snapshot_version) { // that we flag the first re-listen this way without impacting future // listens of this target (that might happen e.g. on reconnect). TargetData request_target_data(target_data.target(), target_id, - target_data.sequence_number(), - QueryPurpose::ExistenceFilterMismatch); + target_data.sequence_number(), entry.second); SendWatchRequest(request_target_data); } diff --git a/Firestore/core/src/remote/serializer.cc b/Firestore/core/src/remote/serializer.cc index 48579790f43..ce8decd84f0 100644 --- a/Firestore/core/src/remote/serializer.cc +++ b/Firestore/core/src/remote/serializer.cc @@ -1277,6 +1277,8 @@ std::string Serializer::EncodeLabel(QueryPurpose purpose) const { return ""; case QueryPurpose::ExistenceFilterMismatch: return "existence-filter-mismatch"; + case QueryPurpose::ExistenceFilterMismatchBloom: + return "existence-filter-mismatch-bloom"; case QueryPurpose::LimboResolution: return "limbo-document"; } From 9079c2888d95ece8927d6f34e4a93801239786a5 Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Thu, 23 Mar 2023 18:38:58 -0700 Subject: [PATCH 29/32] format --- Firestore/core/src/local/target_data.h | 4 +-- Firestore/core/src/remote/datastore.cc | 4 +-- Firestore/core/src/remote/remote_event.cc | 34 +++++++++++++---------- Firestore/core/src/remote/remote_event.h | 7 +++-- Firestore/core/src/remote/remote_store.cc | 2 +- Firestore/core/src/remote/serializer.cc | 1 - 6 files changed, 28 insertions(+), 24 deletions(-) diff --git a/Firestore/core/src/local/target_data.h b/Firestore/core/src/local/target_data.h index 829f352a285..5a6a53370e0 100644 --- a/Firestore/core/src/local/target_data.h +++ b/Firestore/core/src/local/target_data.h @@ -42,8 +42,8 @@ enum class QueryPurpose { ExistenceFilterMismatch, /** - * The query target was used if the query is the result of a false positive - * in the bloom filter. + * The query was used refill a query if there is an existence filter mismatch + * and the bloom filter application returned a false positive result. */ ExistenceFilterMismatchBloom, diff --git a/Firestore/core/src/remote/datastore.cc b/Firestore/core/src/remote/datastore.cc index 38927bc0cb4..94691a4fad2 100644 --- a/Firestore/core/src/remote/datastore.cc +++ b/Firestore/core/src/remote/datastore.cc @@ -109,8 +109,8 @@ Datastore::Datastore( database_info_{database_info}, grpc_connection_{database_info, worker_queue, &grpc_queue_, connectivity_monitor_, firebase_metadata_provider}, - datastore_serializer_{database_info_} { - if (!database_info_.ssl_enabled()) { + datastore_serializer_{database_info} { + if (!database_info.ssl_enabled()) { GrpcConnection::UseInsecureChannel(database_info.host()); } } diff --git a/Firestore/core/src/remote/remote_event.cc b/Firestore/core/src/remote/remote_event.cc index 069946938e5..755f1072447 100644 --- a/Firestore/core/src/remote/remote_event.cc +++ b/Firestore/core/src/remote/remote_event.cc @@ -238,18 +238,17 @@ void WatchChangeAggregator::HandleExistenceFilter( } else { int current_size = GetCurrentDocumentCountForTarget(target_id); if (current_size != expected_count) { - // TODO(Mila): Use application status instead of a boolean in next PR. // Apply bloom filter to identify and mark removed documents. - local::BloomFilterApplicationStatus status = + BloomFilterApplicationStatus status = ApplyBloomFilter(existence_filter, current_size); - if (status != local::BloomFilterApplicationStatus::Success) { + if (status != BloomFilterApplicationStatus::Success) { // If bloom filter application fails, we reset the mapping and // trigger re-run of the query. ResetTarget(target_id); - const local::QueryPurpose purpose = - status == local::BloomFilterApplicationStatus::FalsePositive - ? local::QueryPurpose::ExistenceFilterMismatchBloom - : local::QueryPurpose::ExistenceFilterMismatch; + const QueryPurpose purpose = + status == BloomFilterApplicationStatus::FalsePositive + ? QueryPurpose::ExistenceFilterMismatchBloom + : QueryPurpose::ExistenceFilterMismatch; pending_target_resets_.insert({target_id, purpose}); } } @@ -257,13 +256,12 @@ void WatchChangeAggregator::HandleExistenceFilter( } } -local::BloomFilterApplicationStatus WatchChangeAggregator::ApplyBloomFilter( +BloomFilterApplicationStatus WatchChangeAggregator::ApplyBloomFilter( const ExistenceFilterWatchChange& existence_filter, int current_count) { const absl::optional& bloom_filter_parameters = existence_filter.filter().bloom_filter_parameters(); - if (!bloom_filter_parameters.has_value()) { - return local::BloomFilterApplicationStatus::Skipped; + return BloomFilterApplicationStatus::Skipped; } util::StatusOr maybe_bloom_filter = @@ -273,17 +271,23 @@ local::BloomFilterApplicationStatus WatchChangeAggregator::ApplyBloomFilter( if (!maybe_bloom_filter.ok()) { LOG_WARN("Creating BloomFilter failed: %s", maybe_bloom_filter.status().error_message()); - return local::BloomFilterApplicationStatus::Skipped; + return BloomFilterApplicationStatus::Skipped; + } + + BloomFilter bloom_filter = std::move(maybe_bloom_filter).ValueOrDie(); + + if (bloom_filter.bit_count() == 0) { + return BloomFilterApplicationStatus::Skipped; } - int removed_document_count = FilterRemovedDocuments( - maybe_bloom_filter.ValueOrDie(), existence_filter.target_id()); + int removed_document_count = + FilterRemovedDocuments(bloom_filter, existence_filter.target_id()); int expected_count = existence_filter.filter().count(); if (expected_count != (current_count - removed_document_count)) { - return local::BloomFilterApplicationStatus::FalsePositive; + return BloomFilterApplicationStatus::FalsePositive; } - return local::BloomFilterApplicationStatus::Success; + return BloomFilterApplicationStatus::Success; } int WatchChangeAggregator::FilterRemovedDocuments( diff --git a/Firestore/core/src/remote/remote_event.h b/Firestore/core/src/remote/remote_event.h index 6a70d1fe675..e5cdcce2101 100644 --- a/Firestore/core/src/remote/remote_event.h +++ b/Firestore/core/src/remote/remote_event.h @@ -38,11 +38,12 @@ namespace firestore { namespace local { class TargetData; -enum class BloomFilterApplicationStatus { Success, Skipped, FalsePositive }; } // namespace local namespace remote { +enum class BloomFilterApplicationStatus { Success, Skipped, FalsePositive }; + /** * Interface implemented by `RemoteStore` to expose target metadata to the * `WatchChangeAggregator`. @@ -418,7 +419,7 @@ class WatchChangeAggregator { /** Returns whether a bloom filter removed the deleted documents successfully. */ - local::BloomFilterApplicationStatus ApplyBloomFilter( + BloomFilterApplicationStatus ApplyBloomFilter( const ExistenceFilterWatchChange& existence_filter, int current_count); /** @@ -440,7 +441,7 @@ class WatchChangeAggregator { pending_document_target_mappings_; /** - * A list of targets with existence filter mismatches. These targets are known + * A map of targets with existence filter mismatches. These targets are known * to be inconsistent and their listens needs to be re-established by * `RemoteStore`. */ diff --git a/Firestore/core/src/remote/remote_store.cc b/Firestore/core/src/remote/remote_store.cc index 3fc65478d5d..4a5cb974de6 100644 --- a/Firestore/core/src/remote/remote_store.cc +++ b/Firestore/core/src/remote/remote_store.cc @@ -328,7 +328,7 @@ void RemoteStore::RaiseWatchSnapshot(const SnapshotVersion& snapshot_version) { // Re-establish listens for the targets that have been invalidated by // existence filter mismatches. for (const auto& entry : remote_event.target_mismatches()) { - const auto target_id = entry.first; + const TargetId& target_id = entry.first; auto found = listen_targets_.find(target_id); if (found == listen_targets_.end()) { // A watched target might have been removed already. diff --git a/Firestore/core/src/remote/serializer.cc b/Firestore/core/src/remote/serializer.cc index 62b7ca8a005..9c9a4cbae7e 100644 --- a/Firestore/core/src/remote/serializer.cc +++ b/Firestore/core/src/remote/serializer.cc @@ -53,7 +53,6 @@ #include "Firestore/core/src/timestamp_internal.h" #include "Firestore/core/src/util/comparison.h" #include "Firestore/core/src/util/hard_assert.h" -#include "Firestore/core/src/util/log.h" #include "Firestore/core/src/util/status.h" #include "Firestore/core/src/util/statusor.h" #include "Firestore/core/src/util/string_format.h" From 0e7af0b92ff82d0c95e701712c3d1f8cc6dc07cd Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Thu, 23 Mar 2023 18:44:54 -0700 Subject: [PATCH 30/32] change ApplyBloomFilter comment --- Firestore/core/src/remote/remote_event.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Firestore/core/src/remote/remote_event.h b/Firestore/core/src/remote/remote_event.h index e5cdcce2101..68f3874c0ee 100644 --- a/Firestore/core/src/remote/remote_event.h +++ b/Firestore/core/src/remote/remote_event.h @@ -417,7 +417,9 @@ class WatchChangeAggregator { bool TargetContainsDocument(model::TargetId target_id, const model::DocumentKey& key); - /** Returns whether a bloom filter removed the deleted documents successfully. + /** + * Apply bloom filter to remove the deleted documents, and return the + * application status. */ BloomFilterApplicationStatus ApplyBloomFilter( const ExistenceFilterWatchChange& existence_filter, int current_count); From 78d3fbb5723d5f2e1b471f07ffa4077e8547a082 Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Fri, 24 Mar 2023 09:58:02 -0700 Subject: [PATCH 31/32] resolve comments --- Firestore/core/src/remote/remote_event.cc | 14 +++++++------- Firestore/core/src/remote/remote_event.h | 6 +++--- Firestore/core/src/remote/remote_store.cc | 3 ++- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/Firestore/core/src/remote/remote_event.cc b/Firestore/core/src/remote/remote_event.cc index 755f1072447..82d0e923569 100644 --- a/Firestore/core/src/remote/remote_event.cc +++ b/Firestore/core/src/remote/remote_event.cc @@ -241,12 +241,12 @@ void WatchChangeAggregator::HandleExistenceFilter( // Apply bloom filter to identify and mark removed documents. BloomFilterApplicationStatus status = ApplyBloomFilter(existence_filter, current_size); - if (status != BloomFilterApplicationStatus::Success) { + if (status != BloomFilterApplicationStatus::kSuccess) { // If bloom filter application fails, we reset the mapping and // trigger re-run of the query. ResetTarget(target_id); const QueryPurpose purpose = - status == BloomFilterApplicationStatus::FalsePositive + status == BloomFilterApplicationStatus::kFalsePositive ? QueryPurpose::ExistenceFilterMismatchBloom : QueryPurpose::ExistenceFilterMismatch; pending_target_resets_.insert({target_id, purpose}); @@ -261,7 +261,7 @@ BloomFilterApplicationStatus WatchChangeAggregator::ApplyBloomFilter( const absl::optional& bloom_filter_parameters = existence_filter.filter().bloom_filter_parameters(); if (!bloom_filter_parameters.has_value()) { - return BloomFilterApplicationStatus::Skipped; + return BloomFilterApplicationStatus::kSkipped; } util::StatusOr maybe_bloom_filter = @@ -271,13 +271,13 @@ BloomFilterApplicationStatus WatchChangeAggregator::ApplyBloomFilter( if (!maybe_bloom_filter.ok()) { LOG_WARN("Creating BloomFilter failed: %s", maybe_bloom_filter.status().error_message()); - return BloomFilterApplicationStatus::Skipped; + return BloomFilterApplicationStatus::kSkipped; } BloomFilter bloom_filter = std::move(maybe_bloom_filter).ValueOrDie(); if (bloom_filter.bit_count() == 0) { - return BloomFilterApplicationStatus::Skipped; + return BloomFilterApplicationStatus::kSkipped; } int removed_document_count = @@ -285,9 +285,9 @@ BloomFilterApplicationStatus WatchChangeAggregator::ApplyBloomFilter( int expected_count = existence_filter.filter().count(); if (expected_count != (current_count - removed_document_count)) { - return BloomFilterApplicationStatus::FalsePositive; + return BloomFilterApplicationStatus::kFalsePositive; } - return BloomFilterApplicationStatus::Success; + return BloomFilterApplicationStatus::kSuccess; } int WatchChangeAggregator::FilterRemovedDocuments( diff --git a/Firestore/core/src/remote/remote_event.h b/Firestore/core/src/remote/remote_event.h index 68f3874c0ee..0d5331a0d7c 100644 --- a/Firestore/core/src/remote/remote_event.h +++ b/Firestore/core/src/remote/remote_event.h @@ -17,7 +17,6 @@ #ifndef FIRESTORE_CORE_SRC_REMOTE_REMOTE_EVENT_H_ #define FIRESTORE_CORE_SRC_REMOTE_REMOTE_EVENT_H_ -#include #include #include #include @@ -42,7 +41,7 @@ class TargetData; namespace remote { -enum class BloomFilterApplicationStatus { Success, Skipped, FalsePositive }; +enum class BloomFilterApplicationStatus { kSuccess, kSkipped, kFalsePositive }; /** * Interface implemented by `RemoteStore` to expose target metadata to the @@ -249,7 +248,8 @@ class TargetState { class RemoteEvent { public: using TargetChangeMap = std::unordered_map; - using TargetMismatchMap = std::map; + using TargetMismatchMap = + std::unordered_map; RemoteEvent(model::SnapshotVersion snapshot_version, TargetChangeMap target_changes, diff --git a/Firestore/core/src/remote/remote_store.cc b/Firestore/core/src/remote/remote_store.cc index 4a5cb974de6..fdc583c867a 100644 --- a/Firestore/core/src/remote/remote_store.cc +++ b/Firestore/core/src/remote/remote_store.cc @@ -347,12 +347,13 @@ void RemoteStore::RaiseWatchSnapshot(const SnapshotVersion& snapshot_version) { // deliberately don't send a resume token so that we get a full update. SendUnwatchRequest(target_id); + const QueryPurpose& purpose = entry.second; // Mark the query we send as being on behalf of an existence filter // mismatch, but don't actually retain that in listen_targets_. This ensures // that we flag the first re-listen this way without impacting future // listens of this target (that might happen e.g. on reconnect). TargetData request_target_data(target_data.target(), target_id, - target_data.sequence_number(), entry.second); + target_data.sequence_number(), purpose); SendWatchRequest(request_target_data); } From 5eda3aecea5938addc187bda29b1bbe21ef2cbf1 Mon Sep 17 00:00:00 2001 From: milaGGL <107142260+milaGGL@users.noreply.github.com> Date: Fri, 24 Mar 2023 13:00:55 -0700 Subject: [PATCH 32/32] resolve comment --- Firestore/core/src/remote/remote_store.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Firestore/core/src/remote/remote_store.cc b/Firestore/core/src/remote/remote_store.cc index fdc583c867a..25944595549 100644 --- a/Firestore/core/src/remote/remote_store.cc +++ b/Firestore/core/src/remote/remote_store.cc @@ -329,6 +329,8 @@ void RemoteStore::RaiseWatchSnapshot(const SnapshotVersion& snapshot_version) { // existence filter mismatches. for (const auto& entry : remote_event.target_mismatches()) { const TargetId& target_id = entry.first; + const QueryPurpose& purpose = entry.second; + auto found = listen_targets_.find(target_id); if (found == listen_targets_.end()) { // A watched target might have been removed already. @@ -347,7 +349,6 @@ void RemoteStore::RaiseWatchSnapshot(const SnapshotVersion& snapshot_version) { // deliberately don't send a resume token so that we get a full update. SendUnwatchRequest(target_id); - const QueryPurpose& purpose = entry.second; // Mark the query we send as being on behalf of an existence filter // mismatch, but don't actually retain that in listen_targets_. This ensures // that we flag the first re-listen this way without impacting future