From 7ccadd878086addd00f1f939f18dfaaa7b50ff69 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Tue, 15 Feb 2022 10:09:38 -0700 Subject: [PATCH 01/12] Implement Index Scanning --- packages/firestore/src/core/target.ts | 6 +- packages/firestore/src/index/index_entry.ts | 2 +- packages/firestore/src/local/index_manager.ts | 5 +- .../src/local/indexeddb_index_manager.ts | 436 ++++++++- .../src/local/memory_index_manager.ts | 11 +- packages/firestore/src/local/simple_db.ts | 18 + packages/firestore/src/model/field_index.ts | 7 + .../firestore/src/platform/node/base64.ts | 15 +- packages/firestore/src/util/byte_string.ts | 2 + .../firestore/test/unit/api/bytes.test.ts | 6 - .../test/unit/local/index_manager.test.ts | 914 +++++++++++++++++- .../test/unit/local/simple_db.test.ts | 13 + .../test/unit/local/test_index_manager.ts | 8 +- 13 files changed, 1390 insertions(+), 53 deletions(-) diff --git a/packages/firestore/src/core/target.ts b/packages/firestore/src/core/target.ts index af80bf6205d..5606e123148 100644 --- a/packages/firestore/src/core/target.ts +++ b/packages/firestore/src/core/target.ts @@ -333,9 +333,8 @@ export function targetGetLowerBound( filterValue = MIN_VALUE; break; case Operator.NOT_IN: - const length = (fieldFilter.value.arrayValue!.values || []).length; filterValue = { - arrayValue: { values: new Array(length).fill(MIN_VALUE) } + arrayValue: { values: [MIN_VALUE] } }; break; default: @@ -418,9 +417,8 @@ export function targetGetUpperBound( filterValue = MAX_VALUE; break; case Operator.NOT_IN: - const length = (fieldFilter.value.arrayValue!.values || []).length; filterValue = { - arrayValue: { values: new Array(length).fill(MIN_VALUE) } + arrayValue: { values: [MAX_VALUE] } }; break; default: diff --git a/packages/firestore/src/index/index_entry.ts b/packages/firestore/src/index/index_entry.ts index e6c4ee95c2a..13ec55fe900 100644 --- a/packages/firestore/src/index/index_entry.ts +++ b/packages/firestore/src/index/index_entry.ts @@ -49,7 +49,7 @@ export function indexEntryComparator( return compareByteArrays(left.directionalValue, right.directionalValue); } -function compareByteArrays(left: Uint8Array, right: Uint8Array): number { +export function compareByteArrays(left: Uint8Array, right: Uint8Array): number { for (let i = 0; i < left.length && i < right.length; ++i) { const compare = left[i] - right[i]; if (compare !== 0) { diff --git a/packages/firestore/src/local/index_manager.ts b/packages/firestore/src/local/index_manager.ts index 8cc84cdd221..2c3578c0bbd 100644 --- a/packages/firestore/src/local/index_manager.ts +++ b/packages/firestore/src/local/index_manager.ts @@ -103,13 +103,12 @@ export interface IndexManager { /** * Returns the documents that match the given target based on the provided - * index. + * index or `null` if the target does not have a matching index. */ getDocumentsMatchingTarget( transaction: PersistenceTransaction, - fieldIndex: FieldIndex, target: Target - ): PersistencePromise; + ): PersistencePromise; /** * Returns the next collection group to update. Returns `null` if no group diff --git a/packages/firestore/src/local/indexeddb_index_manager.ts b/packages/firestore/src/local/indexeddb_index_manager.ts index 9e9f2fd0e9f..f26ee4f242e 100644 --- a/packages/firestore/src/local/indexeddb_index_manager.ts +++ b/packages/firestore/src/local/indexeddb_index_manager.ts @@ -16,10 +16,25 @@ */ import { User } from '../auth/user'; -import { Target } from '../core/target'; +import { + Bound, + canonifyTarget, + FieldFilter, + Operator, + Target, + targetEquals, + targetGetArrayValues, + targetGetLowerBound, + targetGetNotInValues, + targetGetUpperBound +} from '../core/target'; import { FirestoreIndexValueWriter } from '../index/firestore_index_value_writer'; import { IndexByteEncoder } from '../index/index_byte_encoder'; -import { IndexEntry, indexEntryComparator } from '../index/index_entry'; +import { + compareByteArrays, + IndexEntry, + indexEntryComparator +} from '../index/index_entry'; import { documentKeySet, DocumentKeySet, @@ -31,15 +46,19 @@ import { FieldIndex, fieldIndexGetArraySegment, fieldIndexGetDirectionalSegments, + fieldIndexToString, IndexKind, - IndexOffset + IndexOffset, + IndexSegment } from '../model/field_index'; -import { ResourcePath } from '../model/path'; +import { FieldPath, ResourcePath } from '../model/path'; +import { TargetIndexMatcher } from '../model/target_index_matcher'; import { isArray } from '../model/values'; import { Value as ProtoValue } from '../protos/firestore_proto_api'; import { debugAssert } from '../util/assert'; import { logDebug } from '../util/log'; import { immediateSuccessor } from '../util/misc'; +import { ObjectMap } from '../util/obj_map'; import { diffSortedSets, SortedSet } from '../util/sorted_set'; import { @@ -70,6 +89,8 @@ import { SimpleDbStore } from './simple_db'; const LOG_TAG = 'IndexedDbIndexManager'; +const EMPTY_VALUE = new Uint8Array(0); + /** * A persisted implementation of IndexManager. * @@ -88,6 +109,15 @@ export class IndexedDbIndexManager implements IndexManager { private uid: string; + /** + * Maps from a target to its equivalent list of sub-targets. Each sub-target + * contains only one term from the target's disjunctive normal form (DNF). + */ + private targetToDnfSubTargets = new ObjectMap( + t => canonifyTarget(t), + (l, r) => targetEquals(l, r) + ); + constructor(private user: User) { this.uid = user.uid || ''; } @@ -196,19 +226,211 @@ export class IndexedDbIndexManager implements IndexManager { getDocumentsMatchingTarget( transaction: PersistenceTransaction, - fieldIndex: FieldIndex, target: Target - ): PersistencePromise { - // TODO(indexing): Implement - return PersistencePromise.resolve(documentKeySet()); + ): PersistencePromise { + const indexEntries = indexEntriesStore(transaction); + + let canServeTarget = true; + const indexes = new Map(); + + return PersistencePromise.forEach( + this.getSubTargets(target), + (subTarget: Target) => { + return this.getFieldIndex(transaction, subTarget).next(index => { + canServeTarget &&= !!index; + indexes.set(subTarget, index); + }); + } + ).next(() => { + if (!canServeTarget) { + return PersistencePromise.resolve(null as DocumentKeySet | null); + } else { + let result = documentKeySet(); + return PersistencePromise.forEach(indexes, (index, subTarget) => { + logDebug( + LOG_TAG, + `Using index ${fieldIndexToString( + index! + )} to execute ${canonifyTarget(target)}` + ); + + const arrayValues = targetGetArrayValues(subTarget, index!); + const notInValues = targetGetNotInValues(subTarget, index!); + const lowerBound = targetGetLowerBound(subTarget, index!); + const upperBound = targetGetUpperBound(subTarget, index!); + + const lowerBoundEncoded = this.encodeBound( + index!, + subTarget, + lowerBound + ); + const upperBoundEncoded = this.encodeBound( + index!, + subTarget, + upperBound + ); + const notInEncoded = this.encodeValues( + index!, + subTarget, + notInValues + ); + + const indexRanges = this.generateIndexRanges( + index!.indexId, + arrayValues, + lowerBoundEncoded, + !!lowerBound && lowerBound.inclusive, + upperBoundEncoded, + !!upperBound && upperBound.inclusive, + notInEncoded + ); + return PersistencePromise.forEach( + indexRanges, + (indexRange: IDBKeyRange) => { + return indexEntries + .loadFirst(indexRange, target.limit) + .next(entries => { + entries.forEach(entry => { + result = result.add( + new DocumentKey(decodeResourcePath(entry.documentKey)) + ); + }); + }); + } + ); + }).next(() => result as DocumentKeySet | null); + } + }); + } + + private getSubTargets(target: Target): Target[] { + let subTargets = this.targetToDnfSubTargets.get(target); + if (subTargets) { + return subTargets; + } + // TODO(orquery): Implement DNF transform + subTargets = [target]; + this.targetToDnfSubTargets.set(target, subTargets); + return subTargets; + } + + /** + * Constructs a key range query on `DbIndexEntry.store` that unions all + * bounds. + */ + private generateIndexRanges( + indexId: number, + arrayValues: ProtoValue[] | null, + lowerBounds: Uint8Array[] | null, + lowerBoundInclusive: boolean, + upperBounds: Uint8Array[] | null, + upperBoundInclusive: boolean, + notInValues: Uint8Array[] | null + ): IDBKeyRange[] { + // The number of total index scans we union together. This is similar to a + // distributed normal form, but adapted for array values. We create a single + // index range per value in an ARRAY_CONTAINS or ARRAY_CONTAINS_ANY filter + // combined with the values from the query bounds. + const totalScans = + (arrayValues != null ? arrayValues.length : 1) * + Math.max( + lowerBounds != null ? lowerBounds.length : 1, + upperBounds != null ? upperBounds.length : 1 + ); + const scansPerArrayElement = + totalScans / (arrayValues != null ? arrayValues.length : 1); + + const indexRanges: IDBKeyRange[] = []; + for (let i = 0; i < totalScans; ++i) { + const arrayValue = arrayValues + ? this.encodeSingleElement(arrayValues[i / scansPerArrayElement]) + : EMPTY_VALUE; + indexRanges.push( + IDBKeyRange.bound( + lowerBounds + ? this.generateLowerBound( + indexId, + arrayValue, + lowerBounds[i % scansPerArrayElement], + lowerBoundInclusive + ) + : this.generateEmptyBound(indexId), + upperBounds + ? this.generateUpperBound( + indexId, + arrayValue, + upperBounds[i % scansPerArrayElement], + upperBoundInclusive + ) + : this.generateEmptyBound(indexId + 1), + /* lowerOpen= */ false, + /* upperOpen= */ true + ) + ); + } + + return this.applyNotIn(indexRanges, notInValues); + } + + /** Generates the lower bound for `arrayValue` and `directionalValue`. */ + private generateLowerBound( + indexId: number, + arrayValue: Uint8Array, + directionalValue: Uint8Array, + inclusive: boolean + ): DbIndexEntryKey { + return [ + indexId, + this.uid, + arrayValue, + inclusive ? directionalValue : successor(directionalValue), + '' + ]; + } + + /** Generates the upper bound for `arrayValue` and `directionalValue`. */ + private generateUpperBound( + indexId: number, + arrayValue: Uint8Array, + directionalValue: Uint8Array, + inclusive: boolean + ): DbIndexEntryKey { + return [ + indexId, + this.uid, + arrayValue, + inclusive ? successor(directionalValue) : directionalValue, + '' + ]; + } + + /** + * Generates an empty bound that scopes the index scan to the current index + * and user. + */ + private generateEmptyBound(indexId: number): DbIndexEntryKey { + return [indexId, this.uid, EMPTY_VALUE, EMPTY_VALUE, '']; } getFieldIndex( transaction: PersistenceTransaction, target: Target ): PersistencePromise { - // TODO(indexing): Implement - return PersistencePromise.resolve(null); + const targetIndexMatcher = new TargetIndexMatcher(target); + const collectionGroup = + target.collectionGroup != null + ? target.collectionGroup + : target.path.lastSegment(); + + return this.getFieldIndexes(transaction, collectionGroup).next(indexes => { + const matchingIndexes = indexes.filter(i => + targetIndexMatcher.servedByIndex(i) + ); + + // Return the index that matches the most number of segments. + matchingIndexes.sort((l, r) => r.fields.length - l.fields.length); + return matchingIndexes.length > 0 ? matchingIndexes[0] : null; + }); } /** @@ -245,6 +467,101 @@ export class IndexedDbIndexManager implements IndexManager { return encoder.encodedBytes(); } + /** + * Encodes the given field values according to the specification in `target`. + * For IN queries, a list of possible values is returned. + */ + private encodeValues( + fieldIndex: FieldIndex, + target: Target, + bound: ProtoValue[] | null + ): Uint8Array[] | null { + if (bound == null) { + return null; + } + + let encoders: IndexByteEncoder[] = []; + encoders.push(new IndexByteEncoder()); + + let boundIdx = 0; + for (const segment of fieldIndexGetDirectionalSegments(fieldIndex)) { + const value = bound[boundIdx++]; + for (const encoder of encoders) { + if (this.isInFilter(target, segment.fieldPath) && isArray(value)) { + encoders = this.expandIndexValues(encoders, segment, value); + } else { + const directionalEncoder = encoder.forKind(segment.kind); + FirestoreIndexValueWriter.INSTANCE.writeIndexValue( + value, + directionalEncoder + ); + } + } + } + return this.getEncodedBytes(encoders); + } + + /** + * Encodes the given bounds according to the specification in `target`. For IN + * queries, a list of possible values is returned. + */ + private encodeBound( + fieldIndex: FieldIndex, + target: Target, + bound: Bound | null + ): Uint8Array[] | null { + if (bound == null) { + return null; + } + return this.encodeValues(fieldIndex, target, bound.position); + } + + /** Returns the byte representation for the provided encoders. */ + private getEncodedBytes(encoders: IndexByteEncoder[]): Uint8Array[] { + const result: Uint8Array[] = []; + for (let i = 0; i < encoders.length; ++i) { + result[i] = encoders[i].encodedBytes(); + } + return result; + } + + /** + * Creates a separate encoder for each element of an array. + * + * The method appends each value to all existing encoders (e.g. filter("a", + * "==", "a1").filter("b", "in", ["b1", "b2"]) becomes ["a1,b1", "a1,b2"]). A + * list of new encoders is returned. + */ + private expandIndexValues( + encoders: IndexByteEncoder[], + segment: IndexSegment, + value: ProtoValue + ): IndexByteEncoder[] { + const prefixes = [...encoders]; + const results: IndexByteEncoder[] = []; + for (const arrayElement of value.arrayValue!.values || []) { + for (const prefix of prefixes) { + const clonedEncoder = new IndexByteEncoder(); + clonedEncoder.seed(prefix.encodedBytes()); + FirestoreIndexValueWriter.INSTANCE.writeIndexValue( + arrayElement, + clonedEncoder.forKind(segment.kind) + ); + results.push(clonedEncoder); + } + } + return results; + } + + private isInFilter(target: Target, fieldPath: FieldPath): boolean { + for (const filter of target.filters) { + if (filter instanceof FieldFilter && filter.field.isEqual(fieldPath)) { + return filter.op === Operator.IN || filter.op === Operator.NOT_IN; + } + } + return false; + } + getFieldIndexes( transaction: PersistenceTransaction, collectionGroup?: string @@ -459,7 +776,7 @@ export class IndexedDbIndexManager implements IndexManager { new IndexEntry( fieldIndex.indexId, document.key, - new Uint8Array(), + EMPTY_VALUE, directionalValue ) ); @@ -516,6 +833,103 @@ export class IndexedDbIndexManager implements IndexManager { ) .next(() => nextSequenceNumber); } + + /** + * Applies notIn and != filters by taking the provided ranges and excluding + * any values that match the `notInValue` from these ranges. As an example, + * '[foo > 2 && foo != 3]` becomes `[foo > 2 && < 3, foo > 3]`. + */ + private applyNotIn( + indexRanges: IDBKeyRange[], + notInValues: Uint8Array[] | null + ): IDBKeyRange[] { + if (!notInValues) { + return indexRanges; + } + + // The values need to be sorted so that we can return a sorted set of + // non-overlapping ranges. + notInValues.sort((l, r) => compareByteArrays(l, r)); + + const notInRanges: IDBKeyRange[] = []; + for (const indexRange of indexRanges) { + // Use the existing bounds and interleave the notIn values. This means + // that we would split an existing range into multiple ranges that exclude + // the values from any notIn filter. + + // The first index range starts with the lower bound and ends at the + // first notIn value (exclusive). + notInRanges.push( + IDBKeyRange.bound( + indexRange.lower, + this.generateNotInBound(indexRange.lower, notInValues[0]), + indexRange.lowerOpen, + /* upperOpen= */ true + ) + ); + + for (let i = 1; i < notInValues.length - 1; ++i) { + // Each index range that we need to scan starts at the last notIn value + // and ends at the next. + notInRanges.push( + IDBKeyRange.bound( + this.generateNotInBound(indexRange.lower, notInValues[i - 1]), + this.generateNotInBound( + indexRange.lower, + successor(notInValues[i]) + ), + /* lowerOpen= */ false, + /* upperOpen= */ true + ) + ); + } + + // The last index range starts at tha last notIn value (exclusive) and + // ends at the upper bound; + notInRanges.push( + IDBKeyRange.bound( + this.generateNotInBound( + indexRange.lower, + successor(notInValues[notInValues.length - 1]) + ), + indexRange.upper, + /* lowerOpen= */ false, + indexRange.upperOpen + ) + ); + } + return notInRanges; + } + + /** + * Generates the index entry that can be used to create the cutoff for `value` + * on the provided index range. + */ + private generateNotInBound( + existingRange: DbIndexEntryKey, + value: Uint8Array + ): DbIndexEntryKey { + return [ + /* indexId= */ existingRange[0], + /* userId= */ existingRange[1], + /* arrayValue= */ existingRange[2], + value, + /* documentKey= */ '' + ]; + } +} + +/** Creates a Uint8Array value that sorts immediately after `value`. */ +function successor(value: Uint8Array): Uint8Array { + if (value.length === 0 || value[value.length - 1] === 255) { + const successor = new Uint8Array(value.length + 1); + successor.set(value, 0); + successor.set([0], value.length); + return successor; + } else { + ++value[value.length - 1]; + } + return value; } /** diff --git a/packages/firestore/src/local/memory_index_manager.ts b/packages/firestore/src/local/memory_index_manager.ts index 5cb34113411..7990e00dc97 100644 --- a/packages/firestore/src/local/memory_index_manager.ts +++ b/packages/firestore/src/local/memory_index_manager.ts @@ -16,11 +16,7 @@ */ import { Target } from '../core/target'; -import { - documentKeySet, - DocumentKeySet, - DocumentMap -} from '../model/collections'; +import { DocumentKeySet, DocumentMap } from '../model/collections'; import { FieldIndex, IndexOffset } from '../model/field_index'; import { ResourcePath } from '../model/path'; import { debugAssert } from '../util/assert'; @@ -71,11 +67,10 @@ export class MemoryIndexManager implements IndexManager { getDocumentsMatchingTarget( transaction: PersistenceTransaction, - fieldIndex: FieldIndex, target: Target - ): PersistencePromise { + ): PersistencePromise { // Field indices are not supported with memory persistence. - return PersistencePromise.resolve(documentKeySet()); + return PersistencePromise.resolve(null); } getFieldIndex( diff --git a/packages/firestore/src/local/simple_db.ts b/packages/firestore/src/local/simple_db.ts index 405707a119d..40d95865a1f 100644 --- a/packages/firestore/src/local/simple_db.ts +++ b/packages/firestore/src/local/simple_db.ts @@ -683,6 +683,24 @@ export class SimpleDbStore< } } + loadFirst( + range: IDBKeyRange, + count: number | null + ): PersistencePromise { + const request = this.store.getAll( + range, + count === null ? undefined : count + ); + return new PersistencePromise((resolve, reject) => { + request.onerror = (event: Event) => { + reject((event.target as IDBRequest).error!); + }; + request.onsuccess = (event: Event) => { + resolve((event.target as IDBRequest).result); + }; + }); + } + deleteAll(): PersistencePromise; deleteAll(range: IDBKeyRange): PersistencePromise; deleteAll(index: string, range: IDBKeyRange): PersistencePromise; diff --git a/packages/firestore/src/model/field_index.ts b/packages/firestore/src/model/field_index.ts index a8499e5a23a..71b2ce634b7 100644 --- a/packages/firestore/src/model/field_index.ts +++ b/packages/firestore/src/model/field_index.ts @@ -101,6 +101,13 @@ export function fieldIndexSemanticComparator( return primitiveComparator(left.fields.length, right.fields.length); } +/** Returns a debug representation of the field index */ +export function fieldIndexToString(fieldIndex: FieldIndex): string { + return `id=${fieldIndex.indexId}|cg=${ + fieldIndex.collectionGroup + }|f=${fieldIndex.fields.map(f => `${f.fieldPath}:${f.kind}`).join(',')}`; +} + /** The type of the index, e.g. for which type of query it can be used. */ export const enum IndexKind { // Note: The order of these values cannot be changed as the enum values are diff --git a/packages/firestore/src/platform/node/base64.ts b/packages/firestore/src/platform/node/base64.ts index 07f7fbb07b0..134afc5be16 100644 --- a/packages/firestore/src/platform/node/base64.ts +++ b/packages/firestore/src/platform/node/base64.ts @@ -16,23 +16,16 @@ */ /** Converts a Base64 encoded string to a binary string. */ -import { Code, FirestoreError } from '../../util/error'; export function decodeBase64(encoded: string): string { - // Node actually doesn't validate base64 strings. - // A quick sanity check that is not a fool-proof validation - if (/[^-A-Za-z0-9+/=]/.test(encoded)) { - throw new FirestoreError( - Code.INVALID_ARGUMENT, - 'Not a valid Base64 string: ' + encoded - ); - } - return new Buffer(encoded, 'base64').toString('binary'); + // Note: We used to validate the base64 string here via a regular expression. + // This was removed to improve the performance of indexing. + return Buffer.from(encoded, 'base64').toString('binary'); } /** Converts a binary string to a Base64 encoded string. */ export function encodeBase64(raw: string): string { - return new Buffer(raw, 'binary').toString('base64'); + return Buffer.from(raw, 'binary').toString('base64'); } /** True if and only if the Base64 conversion functions are available. */ diff --git a/packages/firestore/src/util/byte_string.ts b/packages/firestore/src/util/byte_string.ts index 73b5cd1fa1a..bd2573cc1fb 100644 --- a/packages/firestore/src/util/byte_string.ts +++ b/packages/firestore/src/util/byte_string.ts @@ -39,6 +39,8 @@ export class ByteString { } static fromUint8Array(array: Uint8Array): ByteString { + // TODO(indexing); Remove the copy of the byte string here as this method + // is frequently called during indexing. const binaryString = binaryStringFromUint8Array(array); return new ByteString(binaryString); } diff --git a/packages/firestore/test/unit/api/bytes.test.ts b/packages/firestore/test/unit/api/bytes.test.ts index d53885879b5..afc37400d8e 100644 --- a/packages/firestore/test/unit/api/bytes.test.ts +++ b/packages/firestore/test/unit/api/bytes.test.ts @@ -47,12 +47,6 @@ describe('Bytes', () => { }); }); - it('throws on invalid Base64 strings', () => { - expect(() => Bytes.fromBase64String('not-base64!')).to.throw( - /Failed to construct data from Base64 string:/ - ); - }); - it('works with instanceof checks', () => { expect(Bytes.fromBase64String('') instanceof Bytes).to.equal(true); }); diff --git a/packages/firestore/test/unit/local/index_manager.test.ts b/packages/firestore/test/unit/local/index_manager.test.ts index 5d3ef587226..c4588606cce 100644 --- a/packages/firestore/test/unit/local/index_manager.test.ts +++ b/packages/firestore/test/unit/local/index_manager.test.ts @@ -18,6 +18,17 @@ import { expect } from 'chai'; import { User } from '../../../src/auth/user'; +import { + LimitType, + newQueryForCollectionGroup, + Query, + queryToTarget, + queryWithAddedFilter, + queryWithAddedOrderBy, + queryWithEndAt, + queryWithLimit, + queryWithStartAt +} from '../../../src/core/query'; import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; import { INDEXING_SCHEMA_VERSION } from '../../../src/local/indexeddb_schema'; import { Persistence } from '../../../src/local/persistence'; @@ -29,17 +40,30 @@ import { IndexState } from '../../../src/model/field_index'; import { JsonObject } from '../../../src/model/object_value'; +import { canonicalId } from '../../../src/model/values'; import { addEqualityMatcher } from '../../util/equality_matcher'; -import { doc, fieldIndex, key, path, version } from '../../util/helpers'; +import { + bound, + deletedDoc, + doc, + fieldIndex, + filter, + key, + orderBy, + path, + query, + version, + wrap +} from '../../util/helpers'; import * as persistenceHelpers from './persistence_test_helpers'; import { TestIndexManager } from './test_index_manager'; -describe('MemoryIndexManager', () => { +describe('MemoryIndexManager', async () => { genericIndexManagerTests(persistenceHelpers.testMemoryEagerPersistence); }); -describe('IndexedDbIndexManager', () => { +describe('IndexedDbIndexManager', async () => { if (!IndexedDbPersistence.isAvailable()) { console.warn('No IndexedDB. Skipping IndexedDbIndexManager tests.'); return; @@ -171,6 +195,10 @@ describe('IndexedDbIndexManager', () => { nextCollectionGroup = await indexManager.getNextCollectionGroupToUpdate(); expect(nextCollectionGroup).to.equal('coll2'); + + await indexManager.updateCollectionGroup('coll2', IndexOffset.min()); + nextCollectionGroup = await indexManager.getNextCollectionGroupToUpdate(); + expect(nextCollectionGroup).to.equal('coll1'); }); it('deleting field index removes entry from collection group', async () => { @@ -208,6 +236,20 @@ describe('IndexedDbIndexManager', () => { expect(await indexManager.getNextCollectionGroupToUpdate()).to.equal(null); }); + it('persist index offset', async () => { + await indexManager.addFieldIndex( + fieldIndex('coll1', { fields: [['value', IndexKind.ASCENDING]] }) + ); + const offset = new IndexOffset(version(20), key('coll/doc'), 42); + await indexManager.updateCollectionGroup('coll1', offset); + + indexManager = await getIndexManager(User.UNAUTHENTICATED); + + const indexes = await indexManager.getFieldIndexes('coll1'); + expect(indexes).to.have.length(1); + expect(indexes[0].indexState.offset).to.deep.equal(offset); + }); + it('changes user', async () => { let indexManager = await getIndexManager(new User('user1')); @@ -243,6 +285,863 @@ describe('IndexedDbIndexManager', () => { await addDoc('coll/doc2', {}); }); + it('applies orderBy', async () => { + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['count', IndexKind.ASCENDING]] }) + ); + await addDoc('coll/val1', { 'count': 1 }); + await addDoc('coll/val2', { 'not-count': 3 }); + await addDoc('coll/val3', { 'count': 2 }); + const q = queryWithAddedOrderBy(query('coll'), orderBy('count')); + await verifyResults(q, 'coll/val1', 'coll/val3'); + }); + + it('applies equality filter', async () => { + await setUpSingleValueFilter(); + const q = queryWithAddedFilter(query('coll'), filter('count', '==', 2)); + await verifyResults(q, 'coll/val2'); + }); + + it('applies nested field equality filter', async () => { + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['a.b', IndexKind.ASCENDING]] }) + ); + await addDoc('coll/doc1', { 'a': { 'b': 1 } }); + await addDoc('coll/doc2', { 'a': { 'b': 2 } }); + const q = queryWithAddedFilter(query('coll'), filter('a.b', '==', 2)); + await verifyResults(q, 'coll/doc2'); + }); + + it('applies not equals filter', async () => { + await setUpSingleValueFilter(); + const q = queryWithAddedFilter(query('coll'), filter('count', '!=', 2)); + await verifyResults(q, 'coll/val1', 'coll/val3'); + }); + + it('applies equals with not equals filter', async () => { + await indexManager.addFieldIndex( + fieldIndex('coll', { + fields: [ + ['a', IndexKind.ASCENDING], + ['b', IndexKind.ASCENDING] + ] + }) + ); + await addDoc('coll/val1', { 'a': 1, 'b': 1 }); + await addDoc('coll/val2', { 'a': 1, 'b': 2 }); + await addDoc('coll/val3', { 'a': 2, 'b': 1 }); + await addDoc('coll/val4', { 'a': 2, 'b': 2 }); + + // Verifies that we apply the filter in the order of the field index + let q = queryWithAddedFilter( + queryWithAddedFilter(query('coll'), filter('a', '==', 1)), + filter('b', '!=', 1) + ); + await verifyResults(q, 'coll/val2'); + + q = queryWithAddedFilter( + queryWithAddedFilter(query('coll'), filter('b', '!=', 1)), + filter('a', '==', 1) + ); + await verifyResults(q, 'coll/val2'); + }); + + it('applies less than filter', async () => { + await setUpSingleValueFilter(); + const q = queryWithAddedFilter(query('coll'), filter('count', '<', 2)); + await verifyResults(q, 'coll/val1'); + }); + + it('applies less than or equals filter', async () => { + await setUpSingleValueFilter(); + const q = queryWithAddedFilter(query('coll'), filter('count', '<=', 2)); + await verifyResults(q, 'coll/val1', 'coll/val2'); + }); + + it('applies greater than or equals filter', async () => { + await setUpSingleValueFilter(); + const q = queryWithAddedFilter(query('coll'), filter('count', '>=', 2)); + await verifyResults(q, 'coll/val2', 'coll/val3'); + }); + + it('applies greater than filter', async () => { + await setUpSingleValueFilter(); + const q = queryWithAddedFilter(query('coll'), filter('count', '>', 2)); + await verifyResults(q, 'coll/val3'); + }); + + it('applies range filter', async () => { + await setUpSingleValueFilter(); + const q = queryWithAddedFilter( + queryWithAddedFilter(query('coll'), filter('count', '>', 1)), + filter('count', '<', 3) + ); + await verifyResults(q, 'coll/val2'); + }); + + it('applies startAt filter', async () => { + await setUpSingleValueFilter(); + const q = queryWithStartAt( + queryWithAddedOrderBy(query('coll'), orderBy('count')), + bound([2], true) + ); + await verifyResults(q, 'coll/val2', 'coll/val3'); + }); + + it('applies startAfter filter', async () => { + await setUpSingleValueFilter(); + const q = queryWithStartAt( + queryWithAddedOrderBy(query('coll'), orderBy('count')), + bound([2], false) + ); + await verifyResults(q, 'coll/val3'); + }); + + it('applies endAt filter', async () => { + await setUpSingleValueFilter(); + const q = queryWithEndAt( + queryWithAddedOrderBy(query('coll'), orderBy('count')), + bound([2], true) + ); + await verifyResults(q, 'coll/val1', 'coll/val2'); + }); + + it('applies endBefore filter', async () => { + await setUpSingleValueFilter(); + const q = queryWithEndAt( + queryWithAddedOrderBy(query('coll'), orderBy('count')), + bound([2], false) + ); + await verifyResults(q, 'coll/val1'); + }); + + it('applies range with bound filter', async () => { + await setUpSingleValueFilter(); + const startAt = queryWithEndAt( + queryWithStartAt( + queryWithAddedOrderBy( + queryWithAddedFilter( + queryWithAddedFilter(query('coll'), filter('count', '>=', 1)), + filter('count', '<=', 3) + ), + orderBy('count') + ), + bound([1], false) + ), + bound([2], true) + ); + await verifyResults(startAt, 'coll/val2'); + }); + + it('applies in filter', async () => { + await setUpSingleValueFilter(); + const q = queryWithAddedFilter( + query('coll'), + filter('count', 'in', [1, 3]) + ); + await verifyResults(q, 'coll/val1', 'coll/val3'); + }); + + it('applies notIn filter', async () => { + await setUpSingleValueFilter(); + const q = queryWithAddedFilter( + query('coll'), + filter('count', 'not-in', [1, 2]) + ); + await verifyResults(q, 'coll/val3'); + }); + + it('applies arrayContains filter', async () => { + await setUpArrayValueFilter(); + const q = queryWithAddedFilter( + query('coll'), + filter('values', 'array-contains', 1) + ); + await verifyResults(q, 'coll/arr1'); + }); + + it('applies arrayContainsAny filter', async () => { + await setUpArrayValueFilter(); + const q = queryWithAddedFilter( + query('coll'), + filter('values', 'array-contains-any', [1, 2, 4]) + ); + await verifyResults(q, 'coll/arr1', 'coll/arr2'); + }); + + it('validates that array contains filter only matches array', async () => { + // Set up two field indexes. This causes two index entries to be written, + // but our query should only use one index. + await setUpArrayValueFilter(); + await setUpSingleValueFilter(); + await addDoc('coll/nonmatching', { 'values': 1 }); + const q = queryWithAddedFilter( + query('coll'), + filter('values', 'array-contains-any', [1]) + ); + await verifyResults(q, 'coll/arr1'); + }); + + it('returns empty result when no index exists', async () => { + await setUpSingleValueFilter(); + const q = queryWithAddedFilter( + query('coll'), + filter('unknown', '==', true) + ); + expect(await indexManager.getFieldIndex(queryToTarget(q))).to.be.null; + expect(await indexManager.getDocumentsMatchingTarget(queryToTarget(q))).to + .be.null; + }); + + it('returns empty results when no matching documents exists', async () => { + await setUpSingleValueFilter(); + const q = queryWithAddedFilter(query('coll'), filter('count', '==', -1)); + await verifyResults(q); + }); + + it('filters by field type', async () => { + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['value', IndexKind.ASCENDING]] }) + ); + await addDoc('coll/boolean', { 'value': true }); + await addDoc('coll/string', { 'value': 'true' }); + await addDoc('coll/number', { 'value': 1 }); + const q = queryWithAddedFilter(query('coll'), filter('value', '==', true)); + await verifyResults(q, 'coll/boolean'); + }); + + it('supports collection group indexes', async () => { + await indexManager.addFieldIndex( + fieldIndex('coll1', { fields: [['value', IndexKind.ASCENDING]] }) + ); + await addDoc('coll1/doc1', { 'value': true }); + await addDoc('coll2/doc2/coll1/doc1', { 'value': true }); + await addDoc('coll2/doc2', { 'value': true }); + const q = queryWithAddedFilter( + newQueryForCollectionGroup('coll1'), + filter('value', '==', true) + ); + await verifyResults(q, 'coll1/doc1', 'coll2/doc2/coll1/doc1'); + }); + + it('applies limit filter', async () => { + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['value', IndexKind.ASCENDING]] }) + ); + await addDoc('coll/doc1', { 'value': 1 }); + await addDoc('coll/doc2', { 'value': 1 }); + await addDoc('coll/doc3', { 'value': 1 }); + const q = queryWithLimit( + queryWithAddedFilter(query('coll'), filter('value', '==', 1)), + 2, + LimitType.First + ); + await verifyResults(q, 'coll/doc1', 'coll/doc2'); + }); + + it('uses ordering for limit filter', async () => { + await indexManager.addFieldIndex( + fieldIndex('coll', { + fields: [ + ['value', IndexKind.CONTAINS], + ['value', IndexKind.ASCENDING] + ] + }) + ); + await addDoc('coll/doc1', { 'value': [1, 'foo'] }); + await addDoc('coll/doc2', { 'value': [3, 'foo'] }); + await addDoc('coll/doc3', { 'value': [2, 'foo'] }); + const q = queryWithLimit( + queryWithAddedOrderBy( + queryWithAddedFilter( + query('coll'), + filter('value', 'array-contains', 'foo') + ), + orderBy('value') + ), + 2, + LimitType.First + ); + await verifyResults(q, 'coll/doc1', 'coll/doc3'); + }); + + it('updates index entries', async () => { + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['value', IndexKind.ASCENDING]] }) + ); + const q = queryWithAddedOrderBy(query('coll'), orderBy('value')); + + await addDoc('coll/doc1', { 'value': true }); + await verifyResults(q, 'coll/doc1'); + + await addDocs( + doc('coll/doc1', 1, {}), + doc('coll/doc2', 1, { 'value': true }) + ); + await verifyResults(q, 'coll/doc2'); + }); + + it('removes index entries', async () => { + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['value', IndexKind.ASCENDING]] }) + ); + const q = queryWithAddedOrderBy(query('coll'), orderBy('value')); + + await addDoc('coll/doc1', { 'value': true }); + await verifyResults(q, 'coll/doc1'); + + await addDocs(deletedDoc('coll/doc1', 1)); + await verifyResults(q); + }); + + it('support advances queries', async () => { + // This test compares local query results with those received from the Java + // Server SDK. + + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['null', IndexKind.ASCENDING]] }) + ); + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['int', IndexKind.ASCENDING]] }) + ); + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['float', IndexKind.ASCENDING]] }) + ); + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['string', IndexKind.ASCENDING]] }) + ); + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['multi', IndexKind.ASCENDING]] }) + ); + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['array', IndexKind.ASCENDING]] }) + ); + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['array', IndexKind.DESCENDING]] }) + ); + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['array', IndexKind.CONTAINS]] }) + ); + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['map', IndexKind.ASCENDING]] }) + ); + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['map.field', IndexKind.ASCENDING]] }) + ); + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['prefix', IndexKind.ASCENDING]] }) + ); + await indexManager.addFieldIndex( + fieldIndex('coll', { + fields: [ + ['prefix', IndexKind.ASCENDING], + ['suffix', IndexKind.ASCENDING] + ] + }) + ); + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['a', IndexKind.ASCENDING]] }) + ); + await indexManager.addFieldIndex( + fieldIndex('coll', { + fields: [ + ['a', IndexKind.ASCENDING], + ['b', IndexKind.ASCENDING] + ] + }) + ); + await indexManager.addFieldIndex( + fieldIndex('coll', { + fields: [ + ['a', IndexKind.DESCENDING], + ['b', IndexKind.ASCENDING] + ] + }) + ); + await indexManager.addFieldIndex( + fieldIndex('coll', { + fields: [ + ['a', IndexKind.ASCENDING], + ['b', IndexKind.DESCENDING] + ] + }) + ); + await indexManager.addFieldIndex( + fieldIndex('coll', { + fields: [ + ['a', IndexKind.DESCENDING], + ['b', IndexKind.DESCENDING] + ] + }) + ); + await indexManager.addFieldIndex( + fieldIndex('coll', { + fields: [ + ['b', IndexKind.ASCENDING], + ['a', IndexKind.ASCENDING] + ] + }) + ); + + const docs = [ + {}, + { 'int': 1, 'array': [1, 'foo'] }, + { 'array': [2, 'foo'] }, + { 'int': 3, 'array': [3, 'foo'] }, + { 'array': 'foo' }, + { 'array': [1] }, + { 'float': -0.0, 'string': 'a' }, + { 'float': 0, 'string': 'ab' }, + { 'float': 0.0, 'string': 'b' }, + { 'float': NaN }, + { 'multi': true }, + { 'multi': 1 }, + { 'multi': 'string' }, + { 'multi': [] }, + { 'null': null }, + { 'prefix': [1, 2], 'suffix': null }, + { 'prefix': [1], 'suffix': 2 }, + { 'map': {} }, + { 'map': { 'field': true } }, + { 'map': { 'field': false } }, + { 'a': 0, 'b': 0 }, + { 'a': 0, 'b': 1 }, + { 'a': 1, 'b': 0 }, + { 'a': 1, 'b': 1 }, + { 'a': 2, 'b': 0 }, + { 'a': 2, 'b': 1 } + ]; + + for (const doc of docs) { + await addDoc('coll/' + canonicalId(wrap(doc)), doc); + } + + const q = query('coll'); + + await verifyResults( + queryWithAddedOrderBy(q, orderBy('int')), + 'coll/{array:[1,foo],int:1}', + 'coll/{array:[3,foo],int:3}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('float', '==', NaN)), + 'coll/{float:NaN}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('float', '==', -0.0)), + 'coll/{float:-0,string:a}', + 'coll/{float:0,string:ab}', + 'coll/{float:0,string:b}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('float', '==', 0)), + 'coll/{float:-0,string:a}', + 'coll/{float:0,string:ab}', + 'coll/{float:0,string:b}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('float', '==', 0.0)), + 'coll/{float:-0,string:a}', + 'coll/{float:0,string:ab}', + 'coll/{float:0,string:b}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('string', '==', 'a')), + 'coll/{float:-0,string:a}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('string', '>', 'a')), + 'coll/{float:0,string:ab}', + 'coll/{float:0,string:b}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('string', '>=', 'a')), + 'coll/{float:-0,string:a}', + 'coll/{float:0,string:ab}', + 'coll/{float:0,string:b}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('string', '<', 'b')), + 'coll/{float:-0,string:a}', + 'coll/{float:0,string:ab}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('string', '<', 'coll')), + 'coll/{float:-0,string:a}', + 'coll/{float:0,string:ab}', + 'coll/{float:0,string:b}' + ); + await verifyResults( + queryWithAddedFilter( + queryWithAddedFilter(q, filter('string', '>', 'a')), + filter('string', '<', 'b') + ), + 'coll/{float:0,string:ab}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('array', 'array-contains', 'foo')), + 'coll/{array:[1,foo],int:1}', + 'coll/{array:[2,foo]}', + 'coll/{array:[3,foo],int:3}' + ); + await verifyResults( + queryWithAddedFilter( + q, + filter('array', 'array-contains-any', [1, 'foo']) + ), + 'coll/{array:[1,foo],int:1}', + 'coll/{array:[2,foo]}', + 'coll/{array:[3,foo],int:3}', + 'coll/{array:[1]}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('multi', '>=', true)), + 'coll/{multi:true}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('multi', '>=', 0)), + 'coll/{multi:1}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('multi', '>=', '')), + 'coll/{multi:string}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('multi', '>=', [])), + 'coll/{multi:[]}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('multi', '!=', true)), + 'coll/{multi:1}', + 'coll/{multi:string}', + 'coll/{multi:[]}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('multi', 'in', [true, 1])), + 'coll/{multi:true}', + 'coll/{multi:1}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('multi', 'not-in', [true, 1])), + 'coll/{multi:string}', + 'coll/{multi:[]}' + ); + await verifyResults( + queryWithStartAt( + queryWithAddedOrderBy(q, orderBy('array')), + bound([[2]], true) + ), + 'coll/{array:[2,foo]}', + 'coll/{array:[3,foo],int:3}' + ); + await verifyResults( + queryWithStartAt( + queryWithAddedOrderBy(q, orderBy('array', 'desc')), + bound([[2]], true) + ), + 'coll/{array:[1,foo],int:1}', + 'coll/{array:foo}', + 'coll/{array:[1]}' + ); + await verifyResults( + queryWithLimit( + queryWithStartAt( + queryWithAddedOrderBy(q, orderBy('array', 'desc')), + bound([[2]], true) + ), + 2, + LimitType.First + ), + 'coll/{array:[1,foo],int:1}', + 'coll/{array:[1]}' + ); + await verifyResults( + queryWithStartAt( + queryWithAddedOrderBy(q, orderBy('array')), + bound([[2]], false) + ), + 'coll/{array:[2,foo]}', + 'coll/{array:[3,foo],int:3}' + ); + await verifyResults( + queryWithStartAt( + queryWithAddedOrderBy(q, orderBy('array', 'desc')), + bound([[2]], false) + ), + 'coll/{array:[1,foo],int:1}', + 'coll/{array:foo}', + 'coll/{array:[1]}' + ); + await verifyResults( + queryWithLimit( + queryWithStartAt( + queryWithAddedOrderBy(q, orderBy('array', 'desc')), + bound([[2]], true) + ), + 2, + LimitType.First + ), + 'coll/{array:[1,foo],int:1}', + 'coll/{array:[1]}' + ); + await verifyResults( + queryWithStartAt( + queryWithAddedOrderBy(q, orderBy('array')), + bound([[2, 'foo']], false) + ), + 'coll/{array:[3,foo],int:3}' + ); + await verifyResults( + queryWithStartAt( + queryWithAddedOrderBy(q, orderBy('array', 'desc')), + bound([[2, 'foo']], false) + ), + 'coll/{array:[1,foo],int:1}', + 'coll/{array:foo}', + 'coll/{array:[1]}' + ); + await verifyResults( + queryWithLimit( + queryWithStartAt( + queryWithAddedOrderBy(q, orderBy('array', 'desc')), + bound([[2, 'foo']], false) + ), + 2, + LimitType.First + ), + 'coll/{array:[1,foo],int:1}', + 'coll/{array:[1]}' + ); + await verifyResults( + queryWithEndAt( + queryWithAddedOrderBy(q, orderBy('array')), + bound([[2]], true) + ), + 'coll/{array:[1,foo],int:1}', + 'coll/{array:foo}', + 'coll/{array:[1]}' + ); + await verifyResults( + queryWithEndAt( + queryWithAddedOrderBy(q, orderBy('array', 'desc')), + bound([[2]], true) + ), + 'coll/{array:[2,foo]}', + 'coll/{array:[3,foo],int:3}' + ); + await verifyResults( + queryWithEndAt( + queryWithAddedOrderBy(q, orderBy('array')), + bound([[2]], false) + ), + 'coll/{array:[1,foo],int:1}', + 'coll/{array:foo}', + 'coll/{array:[1]}' + ); + await verifyResults( + queryWithLimit( + queryWithEndAt( + queryWithAddedOrderBy(q, orderBy('array')), + bound([[2]], false) + ), + 2, + LimitType.First + ), + 'coll/{array:foo}', + 'coll/{array:[1]}' + ); + await verifyResults( + queryWithEndAt( + queryWithAddedOrderBy(q, orderBy('array', 'desc')), + bound([[2]], false) + ), + 'coll/{array:[2,foo]}', + 'coll/{array:[3,foo],int:3}' + ); + await verifyResults( + queryWithEndAt( + queryWithAddedOrderBy(q, orderBy('array')), + bound([[2, 'foo']], false) + ), + 'coll/{array:[1,foo],int:1}', + 'coll/{array:foo}', + 'coll/{array:[1]}' + ); + await verifyResults( + queryWithLimit( + queryWithEndAt( + queryWithAddedOrderBy(q, orderBy('array')), + bound([[2, 'foo']], false) + ), + 2, + LimitType.First + ), + 'coll/{array:foo}', + 'coll/{array:[1]}' + ); + await verifyResults( + queryWithEndAt( + queryWithAddedOrderBy(q, orderBy('array', 'desc')), + bound([[2, 'foo']], false) + ), + 'coll/{array:[3,foo],int:3}' + ); + await verifyResults( + queryWithLimit( + queryWithAddedOrderBy( + queryWithAddedOrderBy(q, orderBy('a')), + orderBy('b') + ), + 1, + LimitType.First + ), + 'coll/{a:0,b:0}' + ); + await verifyResults( + queryWithLimit( + queryWithAddedOrderBy( + queryWithAddedOrderBy(q, orderBy('a', 'desc')), + orderBy('b') + ), + 1, + LimitType.First + ), + 'coll/{a:2,b:0}' + ); + await verifyResults( + queryWithLimit( + queryWithAddedOrderBy( + queryWithAddedOrderBy(q, orderBy('a')), + orderBy('b', 'desc') + ), + 1, + LimitType.First + ), + 'coll/{a:0,b:1}' + ); + await verifyResults( + queryWithLimit( + queryWithAddedOrderBy( + queryWithAddedOrderBy(q, orderBy('a', 'desc')), + orderBy('b', 'desc') + ), + 1, + LimitType.First + ), + 'coll/{a:2,b:1}' + ); + await verifyResults( + queryWithAddedFilter( + queryWithAddedFilter(q, filter('a', '>', 0)), + filter('b', '==', 1) + ), + 'coll/{a:1,b:1}', + 'coll/{a:2,b:1}' + ); + await verifyResults( + queryWithAddedFilter( + queryWithAddedFilter(q, filter('a', '==', 1)), + filter('b', '==', 1) + ), + 'coll/{a:1,b:1}' + ); + await verifyResults( + queryWithAddedFilter( + queryWithAddedFilter(q, filter('a', '!=', 0)), + filter('b', '==', 1) + ), + 'coll/{a:1,b:1}', + 'coll/{a:2,b:1}' + ); + await verifyResults( + queryWithAddedFilter( + queryWithAddedFilter(q, filter('b', '==', 1)), + filter('a', '!=', 0) + ), + 'coll/{a:1,b:1}', + 'coll/{a:2,b:1}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('a', 'not-in', [0, 1])), + 'coll/{a:2,b:0}', + 'coll/{a:2,b:1}' + ); + await verifyResults( + queryWithAddedFilter( + queryWithAddedFilter(q, filter('a', 'not-in', [0, 1])), + filter('b', '==', 1) + ), + 'coll/{a:2,b:1}' + ); + await verifyResults( + queryWithAddedFilter( + queryWithAddedFilter(q, filter('b', '==', 1)), + filter('a', 'not-in', [0, 1]) + ), + 'coll/{a:2,b:1}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('null', '==', null)), + 'coll/{null:null}' + ); + await verifyResults( + queryWithAddedOrderBy(q, orderBy('null')), + 'coll/{null:null}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('prefix', '==', [1, 2])), + 'coll/{prefix:[1,2],suffix:null}' + ); + await verifyResults( + queryWithAddedFilter( + queryWithAddedFilter(q, filter('prefix', '==', [1])), + filter('suffix', '==', 2) + ), + 'coll/{prefix:[1],suffix:2}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('map', '==', {})), + 'coll/{map:{}}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('map', '==', { 'field': true })), + 'coll/{map:{field:true}}' + ); + await verifyResults( + queryWithAddedFilter(q, filter('map.field', '==', true)), + 'coll/{map:{field:true}}' + ); + await verifyResults( + queryWithAddedOrderBy(q, orderBy('map')), + 'coll/{map:{}}', + 'coll/{map:{field:true}}', + 'coll/{map:{field:false}}' + ); + await verifyResults( + queryWithAddedOrderBy(q, orderBy('map.field')), + 'coll/{map:{field:true}}', + 'coll/{map:{field:false}}' + ); + }); + + async function setUpSingleValueFilter(): Promise { + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['count', IndexKind.ASCENDING]] }) + ); + await addDoc('coll/val1', { 'count': 1 }); + await addDoc('coll/val2', { 'count': 2 }); + await addDoc('coll/val3', { 'count': 3 }); + } + + async function setUpArrayValueFilter(): Promise { + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['values', IndexKind.CONTAINS]] }) + ); + await addDoc('coll/arr1', { 'values': [1, 2, 3] }); + await addDoc('coll/arr2', { 'values': [4, 5, 6] }); + await addDoc('coll/arr3', { 'values': [7, 8, 9] }); + } + function addDocs(...docs: Document[]): Promise { let data = documentMap(); for (const doc of docs) { @@ -254,6 +1153,15 @@ describe('IndexedDbIndexManager', () => { function addDoc(key: string, data: JsonObject): Promise { return addDocs(doc(key, 1, data)); } + + async function verifyResults(query: Query, ...keys: string[]): Promise { + const target = queryToTarget(query); + const actualResults = await indexManager.getDocumentsMatchingTarget(target); + expect(actualResults).to.not.equal(null, 'Expected successful query'); + const actualKeys: string[] = []; + actualResults!.forEach(v => actualKeys.push(v.path.toString())); + expect(actualKeys).to.have.members(keys); + } }); /** diff --git a/packages/firestore/test/unit/local/simple_db.test.ts b/packages/firestore/test/unit/local/simple_db.test.ts index 8a9f6076c68..61ac82f5ebb 100644 --- a/packages/firestore/test/unit/local/simple_db.test.ts +++ b/packages/firestore/test/unit/local/simple_db.test.ts @@ -263,6 +263,19 @@ describe('SimpleDb', () => { }); }); + it('loadFirst', async () => { + const range = IDBKeyRange.bound(3, 8); + await runTransaction(store => { + return store.loadFirst(range, 2).next(users => { + const expected = testData + .filter(user => user.id >= 3 && user.id <= 5) + .slice(0, 2); + expect(users.length).to.deep.equal(expected.length); + expect(users).to.deep.equal(expected); + }); + }); + }); + it('deleteAll', async () => { await runTransaction(store => { return store diff --git a/packages/firestore/test/unit/local/test_index_manager.ts b/packages/firestore/test/unit/local/test_index_manager.ts index bc90a5cddb4..99090340b06 100644 --- a/packages/firestore/test/unit/local/test_index_manager.ts +++ b/packages/firestore/test/unit/local/test_index_manager.ts @@ -76,15 +76,11 @@ export class TestIndexManager { ); } - getDocumentsMatchingTarget( - fieldIndex: FieldIndex, - target: Target - ): Promise { + getDocumentsMatchingTarget(target: Target): Promise { return this.persistence.runTransaction( 'getDocumentsMatchingTarget', 'readonly', - txn => - this.indexManager.getDocumentsMatchingTarget(txn, fieldIndex, target) + txn => this.indexManager.getDocumentsMatchingTarget(txn, target) ); } From 5e5f31f733dfe1b09e5d63a2f9e2e8e6a8d4549c Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Wed, 16 Feb 2022 15:35:10 -0700 Subject: [PATCH 02/12] Review --- packages/firestore/src/core/target.ts | 20 +--- .../src/local/indexeddb_index_manager.ts | 109 +++++++++++------- packages/firestore/src/local/simple_db.ts | 10 ++ packages/firestore/src/model/type_order.ts | 5 +- packages/firestore/src/model/values.ts | 13 ++- .../test/unit/local/index_manager.test.ts | 45 +++++++- .../test/unit/local/simple_db.test.ts | 12 +- 7 files changed, 144 insertions(+), 70 deletions(-) diff --git a/packages/firestore/src/core/target.ts b/packages/firestore/src/core/target.ts index 5606e123148..dd4c5a8351f 100644 --- a/packages/firestore/src/core/target.ts +++ b/packages/firestore/src/core/target.ts @@ -259,7 +259,7 @@ export function targetGetNotInValues( target: Target, fieldIndex: FieldIndex ): ProtoValue[] | null { - const values: ProtoValue[] = []; + const values = new Map(); for (const segment of fieldIndexGetDirectionalSegments(fieldIndex)) { for (const fieldFilter of targetGetFieldFiltersForPath( @@ -272,14 +272,14 @@ export function targetGetNotInValues( // Encode equality prefix, which is encoded in the index value before // the inequality (e.g. `a == 'a' && b != 'b'` is encoded to // `value != 'ab'`). - values.push(fieldFilter.value); + values.set(segment.fieldPath.canonicalString(), fieldFilter.value); break; case Operator.NOT_IN: case Operator.NOT_EQUAL: // NotIn/NotEqual is always a suffix. There cannot be any remaining // segments and hence we can return early here. - values.push(fieldFilter.value); - return values; + values.set(segment.fieldPath.canonicalString(), fieldFilter.value); + return Array.from(values.values()); default: // Remaining filters cannot be used as notIn bounds. } @@ -330,12 +330,8 @@ export function targetGetLowerBound( filterInclusive = false; break; case Operator.NOT_EQUAL: - filterValue = MIN_VALUE; - break; case Operator.NOT_IN: - filterValue = { - arrayValue: { values: [MIN_VALUE] } - }; + filterValue = MIN_VALUE; break; default: // Remaining filters cannot be used as lower bounds. @@ -414,12 +410,8 @@ export function targetGetUpperBound( filterInclusive = false; break; case Operator.NOT_EQUAL: - filterValue = MAX_VALUE; - break; case Operator.NOT_IN: - filterValue = { - arrayValue: { values: [MAX_VALUE] } - }; + filterValue = MAX_VALUE; break; default: // Remaining filters cannot be used as upper bounds. diff --git a/packages/firestore/src/local/indexeddb_index_manager.ts b/packages/firestore/src/local/indexeddb_index_manager.ts index f26ee4f242e..6c32619ba42 100644 --- a/packages/firestore/src/local/indexeddb_index_manager.ts +++ b/packages/firestore/src/local/indexeddb_index_manager.ts @@ -554,12 +554,12 @@ export class IndexedDbIndexManager implements IndexManager { } private isInFilter(target: Target, fieldPath: FieldPath): boolean { - for (const filter of target.filters) { - if (filter instanceof FieldFilter && filter.field.isEqual(fieldPath)) { - return filter.op === Operator.IN || filter.op === Operator.NOT_IN; - } - } - return false; + return !!target.filters.find( + f => + f instanceof FieldFilter && + f.field.isEqual(fieldPath) && + (f.op === Operator.IN || f.op === Operator.NOT_IN) + ); } getFieldIndexes( @@ -853,52 +853,75 @@ export class IndexedDbIndexManager implements IndexManager { const notInRanges: IDBKeyRange[] = []; for (const indexRange of indexRanges) { - // Use the existing bounds and interleave the notIn values. This means - // that we would split an existing range into multiple ranges that exclude - // the values from any notIn filter. - - // The first index range starts with the lower bound and ends at the - // first notIn value (exclusive). - notInRanges.push( - IDBKeyRange.bound( - indexRange.lower, - this.generateNotInBound(indexRange.lower, notInValues[0]), - indexRange.lowerOpen, - /* upperOpen= */ true - ) + // Remove notIn values that are not applicable to this index range + const filteredRanges = notInValues.filter( + v => + compareByteArrays(v, new Uint8Array(indexRange.lower[3])) >= 0 && + compareByteArrays(v, new Uint8Array(indexRange.upper[3])) <= 0 ); - for (let i = 1; i < notInValues.length - 1; ++i) { - // Each index range that we need to scan starts at the last notIn value - // and ends at the next. - notInRanges.push( - IDBKeyRange.bound( - this.generateNotInBound(indexRange.lower, notInValues[i - 1]), - this.generateNotInBound( - indexRange.lower, - successor(notInValues[i]) - ), - /* lowerOpen= */ false, - /* upperOpen= */ true - ) - ); + if (filteredRanges.length === 0) { + notInRanges.push(indexRange); + } else { + // Use the existing bounds and interleave the notIn values. This means + // that we would split an existing range into multiple ranges that exclude + // the values from any notIn filter. + notInRanges.push(...this.interleaveRanges(indexRange, filteredRanges)); } + } + return notInRanges; + } + + /** + * Splits up the range defined by `indexRange` and removes any values + * contained in `barrier`. As an example, if the original range is [1,4] and + * the barrier is [2,3], then this method would return [1,2), (2,3), (3,4]. + */ + private interleaveRanges( + indexRange: IDBKeyRange, + barriers: Uint8Array[] + ): IDBKeyRange[] { + const ranges: IDBKeyRange[] = []; + + // The first index range starts with the lower bound and ends before the + // first barrier. + ranges.push( + IDBKeyRange.bound( + indexRange.lower, + this.generateNotInBound(indexRange.lower, barriers[0]), + indexRange.lowerOpen, + /* upperOpen= */ true + ) + ); - // The last index range starts at tha last notIn value (exclusive) and - // ends at the upper bound; - notInRanges.push( + for (let i = 1; i < barriers.length - 1; ++i) { + // Each index range that we need to scan starts after the last barrier + // and ends before the next. + ranges.push( IDBKeyRange.bound( - this.generateNotInBound( - indexRange.lower, - successor(notInValues[notInValues.length - 1]) - ), - indexRange.upper, + this.generateNotInBound(indexRange.lower, barriers[i - 1]), + this.generateNotInBound(indexRange.lower, successor(barriers[i])), /* lowerOpen= */ false, - indexRange.upperOpen + /* upperOpen= */ true ) ); } - return notInRanges; + + // The last index range starts after the last barrier and ends at the upper + // bound + ranges.push( + IDBKeyRange.bound( + this.generateNotInBound( + indexRange.lower, + successor(barriers[barriers.length - 1]) + ), + indexRange.upper, + /* lowerOpen= */ false, + indexRange.upperOpen + ) + ); + + return ranges; } /** diff --git a/packages/firestore/src/local/simple_db.ts b/packages/firestore/src/local/simple_db.ts index 40d95865a1f..103c96ed819 100644 --- a/packages/firestore/src/local/simple_db.ts +++ b/packages/firestore/src/local/simple_db.ts @@ -652,8 +652,14 @@ export class SimpleDbStore< return wrapRequest(request); } + /** Loads all elements from the object store. */ loadAll(): PersistencePromise; + /** Loads all elements for the index range from the object store. */ loadAll(range: IDBKeyRange): PersistencePromise; + /** + * Loads all elements from the object store that fall into the provided in the + * index range for the given index. + */ loadAll(index: string, range: IDBKeyRange): PersistencePromise; loadAll( indexOrRange?: string | IDBKeyRange, @@ -683,6 +689,10 @@ export class SimpleDbStore< } } + /** + * Loads the first `count` elements from the provided index range. Loads all + * elements if no limit is provided. + */ loadFirst( range: IDBKeyRange, count: number | null diff --git a/packages/firestore/src/model/type_order.ts b/packages/firestore/src/model/type_order.ts index 63b94db3586..df4c6067a31 100644 --- a/packages/firestore/src/model/type_order.ts +++ b/packages/firestore/src/model/type_order.ts @@ -24,7 +24,7 @@ */ export const enum TypeOrder { // This order is based on the backend's ordering, but modified to support - // server timestamps. + // server timestamps and `MAX_VALUE`. NullValue = 0, BooleanValue = 1, NumberValue = 2, @@ -35,5 +35,6 @@ export const enum TypeOrder { RefValue = 7, GeoPointValue = 8, ArrayValue = 9, - ObjectValue = 10 + ObjectValue = 10, + MaxValue = 9007199254740991 // Number.MAX_SAFE_INTEGER } diff --git a/packages/firestore/src/model/values.ts b/packages/firestore/src/model/values.ts index bf2a31c323d..3c6c4b648ec 100644 --- a/packages/firestore/src/model/values.ts +++ b/packages/firestore/src/model/values.ts @@ -41,10 +41,11 @@ import { } from './server_timestamps'; import { TypeOrder } from './type_order'; +const MAX_VALUE_TYPE = '__max__'; export const MAX_VALUE: Value = { mapValue: { fields: { - '__type__': { stringValue: '__max___' } + '__type__': { stringValue: MAX_VALUE_TYPE } } } }; @@ -76,6 +77,8 @@ export function typeOrder(value: Value): TypeOrder { } else if ('mapValue' in value) { if (isServerTimestamp(value)) { return TypeOrder.ServerTimestampValue; + } else if (isMaxValue(value)) { + return TypeOrder.ArrayValue; } return TypeOrder.ObjectValue; } else { @@ -122,6 +125,8 @@ export function valueEquals(left: Value, right: Value): boolean { ); case TypeOrder.ObjectValue: return objectEquals(left, right); + case TypeOrder.MaxValue: + return true; default: return fail('Unexpected value type: ' + JSON.stringify(left)); } @@ -224,6 +229,7 @@ export function valueCompare(left: Value, right: Value): number { switch (leftType) { case TypeOrder.NullValue: + case TypeOrder.MaxValue: return 0; case TypeOrder.BooleanValue: return primitiveComparator(left.booleanValue!, right.booleanValue!); @@ -604,7 +610,10 @@ export function deepClone(source: Value): Value { /** Returns true if the Value represents the canonical {@link #MAX_VALUE} . */ export function isMaxValue(value: Value): boolean { - return valueEquals(value, MAX_VALUE); + return ( + (((value.mapValue || {}).fields || {})['__type__'] || {}).stringValue === + MAX_VALUE_TYPE + ); } /** Returns the lowest value for the given value type (inclusive). */ diff --git a/packages/firestore/test/unit/local/index_manager.test.ts b/packages/firestore/test/unit/local/index_manager.test.ts index c4588606cce..8449bab4c44 100644 --- a/packages/firestore/test/unit/local/index_manager.test.ts +++ b/packages/firestore/test/unit/local/index_manager.test.ts @@ -346,6 +346,27 @@ describe('IndexedDbIndexManager', async () => { await verifyResults(q, 'coll/val2'); }); + it('applies equals with not equals filter on same field', async () => { + await setUpSingleValueFilter(); + let q = queryWithAddedFilter( + queryWithAddedFilter(query('coll'), filter('count', '>', 1)), + filter('count', '!=', 2) + ); + await verifyResults(q, 'coll/val3'); + + q = queryWithAddedFilter( + queryWithAddedFilter(query('coll'), filter('count', '==', 1)), + filter('count', '!=', 2) + ); + await verifyResults(q, 'coll/val1'); + + q = queryWithAddedFilter( + queryWithAddedFilter(query('coll'), filter('count', '==', 1)), + filter('count', '!=', 1) + ); + await verifyResults(q); + }); + it('applies less than filter', async () => { await setUpSingleValueFilter(); const q = queryWithAddedFilter(query('coll'), filter('count', '<', 2)); @@ -442,7 +463,7 @@ describe('IndexedDbIndexManager', async () => { await verifyResults(q, 'coll/val1', 'coll/val3'); }); - it('applies notIn filter', async () => { + it('applies not in filter', async () => { await setUpSingleValueFilter(); const q = queryWithAddedFilter( query('coll'), @@ -451,7 +472,25 @@ describe('IndexedDbIndexManager', async () => { await verifyResults(q, 'coll/val3'); }); - it('applies arrayContains filter', async () => { + it('applies not in filter with greater than filter', async () => { + await setUpSingleValueFilter(); + const q = queryWithAddedFilter( + queryWithAddedFilter(query('coll'), filter('count', '>', 1)), + filter('count', 'not-in', [2]) + ); + await verifyResults(q, 'coll/val3'); + }); + + it('applies not in filter with out of bounds greater than filter', async () => { + await setUpSingleValueFilter(); + const q = queryWithAddedFilter( + queryWithAddedFilter(query('coll'), filter('count', '>', 2)), + filter('count', 'not-in', [1]) + ); + await verifyResults(q, 'coll/val3'); + }); + + it('applies array contains filter', async () => { await setUpArrayValueFilter(); const q = queryWithAddedFilter( query('coll'), @@ -460,7 +499,7 @@ describe('IndexedDbIndexManager', async () => { await verifyResults(q, 'coll/arr1'); }); - it('applies arrayContainsAny filter', async () => { + it('applies array contains any filter', async () => { await setUpArrayValueFilter(); const q = queryWithAddedFilter( query('coll'), diff --git a/packages/firestore/test/unit/local/simple_db.test.ts b/packages/firestore/test/unit/local/simple_db.test.ts index 61ac82f5ebb..b07bac1a73d 100644 --- a/packages/firestore/test/unit/local/simple_db.test.ts +++ b/packages/firestore/test/unit/local/simple_db.test.ts @@ -242,14 +242,14 @@ describe('SimpleDb', () => { await runTransaction(store => { return store.loadAll(range).next(users => { const expected = testData.filter(user => user.id >= 3 && user.id <= 5); - expect(users.length).to.deep.equal(expected.length); + expect(users.length).to.equal(expected.length); expect(users).to.deep.equal(expected); }); }); await runTransaction(store => { return store.loadAll().next(users => { const expected = testData; - expect(users.length).to.deep.equal(expected.length); + expect(users.length).to.equal(expected.length); expect(users).to.deep.equal(expected); }); }); @@ -257,7 +257,7 @@ describe('SimpleDb', () => { await runTransaction(store => { return store.loadAll('age-name', indexRange).next(users => { const expected = testData.filter(user => user.id >= 3 && user.id <= 6); - expect(users.length).to.deep.equal(expected.length); + expect(users.length).to.equal(expected.length); expect(users).to.deep.equal(expected); }); }); @@ -270,7 +270,7 @@ describe('SimpleDb', () => { const expected = testData .filter(user => user.id >= 3 && user.id <= 5) .slice(0, 2); - expect(users.length).to.deep.equal(expected.length); + expect(users.length).to.equal(expected.length); expect(users).to.deep.equal(expected); }); }); @@ -299,7 +299,7 @@ describe('SimpleDb', () => { }) .next(users => { const expected = testData.filter(user => user.id < 3 || user.id > 5); - expect(users.length).to.deep.equal(expected.length); + expect(users.length).to.equal(expected.length); expect(users).to.deep.equal(expected); }); }); @@ -315,7 +315,7 @@ describe('SimpleDb', () => { }) .next(users => { const expected = testData.filter(user => user.id < 3 || user.id > 6); - expect(users.length).to.deep.equal(expected.length); + expect(users.length).to.equal(expected.length); expect(users).to.deep.equal(expected); }); }); From e55c5e7d6d5079dafda78f7198c6c272df9473f1 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 17 Feb 2022 13:52:58 -0700 Subject: [PATCH 03/12] Cleanup --- .../src/local/indexeddb_index_manager.ts | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/packages/firestore/src/local/indexeddb_index_manager.ts b/packages/firestore/src/local/indexeddb_index_manager.ts index 6c32619ba42..bd40cea6783 100644 --- a/packages/firestore/src/local/indexeddb_index_manager.ts +++ b/packages/firestore/src/local/indexeddb_index_manager.ts @@ -847,29 +847,27 @@ export class IndexedDbIndexManager implements IndexManager { return indexRanges; } - // The values need to be sorted so that we can return a sorted set of - // non-overlapping ranges. - notInValues.sort((l, r) => compareByteArrays(l, r)); - - const notInRanges: IDBKeyRange[] = []; + const result: IDBKeyRange[] = []; for (const indexRange of indexRanges) { // Remove notIn values that are not applicable to this index range + const lowerBound = new Uint8Array(indexRange.lower[3]); + const upperBound = new Uint8Array(indexRange.upper[3]); const filteredRanges = notInValues.filter( v => - compareByteArrays(v, new Uint8Array(indexRange.lower[3])) >= 0 && - compareByteArrays(v, new Uint8Array(indexRange.upper[3])) <= 0 + compareByteArrays(v, lowerBound) >= 0 && + compareByteArrays(v, upperBound) <= 0 ); if (filteredRanges.length === 0) { - notInRanges.push(indexRange); + result.push(indexRange); } else { // Use the existing bounds and interleave the notIn values. This means // that we would split an existing range into multiple ranges that exclude // the values from any notIn filter. - notInRanges.push(...this.interleaveRanges(indexRange, filteredRanges)); + result.push(...this.interleaveRanges(indexRange, filteredRanges)); } } - return notInRanges; + return result; } /** @@ -881,6 +879,10 @@ export class IndexedDbIndexManager implements IndexManager { indexRange: IDBKeyRange, barriers: Uint8Array[] ): IDBKeyRange[] { + // The values need to be sorted so that we can return a sorted set of + // non-overlapping ranges. + barriers.sort((l, r) => compareByteArrays(l, r)); + const ranges: IDBKeyRange[] = []; // The first index range starts with the lower bound and ends before the From cf1880eee46144fa18508d0d46dd5835cdd84a42 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 17 Feb 2022 14:12:12 -0700 Subject: [PATCH 04/12] Simplify --- .../src/local/indexeddb_index_manager.ts | 100 ++++++------------ 1 file changed, 35 insertions(+), 65 deletions(-) diff --git a/packages/firestore/src/local/indexeddb_index_manager.ts b/packages/firestore/src/local/indexeddb_index_manager.ts index bd40cea6783..8d1633e6f26 100644 --- a/packages/firestore/src/local/indexeddb_index_manager.ts +++ b/packages/firestore/src/local/indexeddb_index_manager.ts @@ -847,82 +847,52 @@ export class IndexedDbIndexManager implements IndexManager { return indexRanges; } - const result: IDBKeyRange[] = []; - for (const indexRange of indexRanges) { - // Remove notIn values that are not applicable to this index range - const lowerBound = new Uint8Array(indexRange.lower[3]); - const upperBound = new Uint8Array(indexRange.upper[3]); - const filteredRanges = notInValues.filter( - v => - compareByteArrays(v, lowerBound) >= 0 && - compareByteArrays(v, upperBound) <= 0 - ); - - if (filteredRanges.length === 0) { - result.push(indexRange); - } else { - // Use the existing bounds and interleave the notIn values. This means - // that we would split an existing range into multiple ranges that exclude - // the values from any notIn filter. - result.push(...this.interleaveRanges(indexRange, filteredRanges)); - } - } - return result; - } - - /** - * Splits up the range defined by `indexRange` and removes any values - * contained in `barrier`. As an example, if the original range is [1,4] and - * the barrier is [2,3], then this method would return [1,2), (2,3), (3,4]. - */ - private interleaveRanges( - indexRange: IDBKeyRange, - barriers: Uint8Array[] - ): IDBKeyRange[] { // The values need to be sorted so that we can return a sorted set of // non-overlapping ranges. - barriers.sort((l, r) => compareByteArrays(l, r)); + notInValues.sort((l, r) => compareByteArrays(l, r)); const ranges: IDBKeyRange[] = []; - // The first index range starts with the lower bound and ends before the - // first barrier. - ranges.push( - IDBKeyRange.bound( - indexRange.lower, - this.generateNotInBound(indexRange.lower, barriers[0]), - indexRange.lowerOpen, - /* upperOpen= */ true - ) - ); + // Use the existing bounds and interleave the notIn values. This means + // that we would split an existing range into multiple ranges that exclude + // the values from any notIn filter. + for (const indexRange of indexRanges) { + const lowerBound = new Uint8Array(indexRange.lower[3]); + const upperBound = new Uint8Array(indexRange.upper[3]); + let lastLower = indexRange.lower; + let lastOpen = indexRange.lowerOpen; + + for (const notInValue of notInValues) { + if ( + compareByteArrays(notInValue, lowerBound) >= 0 && + compareByteArrays(notInValue, upperBound) <= 0 + ) { + ranges.push( + IDBKeyRange.bound( + lastLower, + this.generateNotInBound(indexRange.lower, notInValue), + lastOpen, + /* upperOpen= */ true + ) + ); + + lastLower = this.generateNotInBound( + indexRange.lower, + successor(notInValue) + ); + lastOpen = true; + } + } - for (let i = 1; i < barriers.length - 1; ++i) { - // Each index range that we need to scan starts after the last barrier - // and ends before the next. ranges.push( IDBKeyRange.bound( - this.generateNotInBound(indexRange.lower, barriers[i - 1]), - this.generateNotInBound(indexRange.lower, successor(barriers[i])), - /* lowerOpen= */ false, - /* upperOpen= */ true + lastLower, + indexRange.upper, + lastOpen, + indexRange.upperOpen ) ); } - - // The last index range starts after the last barrier and ends at the upper - // bound - ranges.push( - IDBKeyRange.bound( - this.generateNotInBound( - indexRange.lower, - successor(barriers[barriers.length - 1]) - ), - indexRange.upper, - /* lowerOpen= */ false, - indexRange.upperOpen - ) - ); - return ranges; } From 2a4af04a485ae06eb7dfe8f2032b652650700ce6 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 17 Feb 2022 15:30:19 -0700 Subject: [PATCH 05/12] Use cleaner code for barriers --- .../src/local/indexeddb_index_manager.ts | 43 +++++++---------- .../test/unit/local/index_manager.test.ts | 48 +++++++++++++++++++ 2 files changed, 66 insertions(+), 25 deletions(-) diff --git a/packages/firestore/src/local/indexeddb_index_manager.ts b/packages/firestore/src/local/indexeddb_index_manager.ts index 8d1633e6f26..819c13396a7 100644 --- a/packages/firestore/src/local/indexeddb_index_manager.ts +++ b/packages/firestore/src/local/indexeddb_index_manager.ts @@ -859,39 +859,32 @@ export class IndexedDbIndexManager implements IndexManager { for (const indexRange of indexRanges) { const lowerBound = new Uint8Array(indexRange.lower[3]); const upperBound = new Uint8Array(indexRange.upper[3]); - let lastLower = indexRange.lower; - let lastOpen = indexRange.lowerOpen; - for (const notInValue of notInValues) { + let lastLower = lowerBound; + let lowerOpen = indexRange.lowerOpen; + + const barriers = [...notInValues, upperBound]; + for (const barrier of barriers) { + // Verify that the range in the bound is sensible, as the bound may get + // rejected otherwise + const sortsAfter = compareByteArrays(barrier, lastLower); + const sortsBefore = compareByteArrays(barrier, upperBound); if ( - compareByteArrays(notInValue, lowerBound) >= 0 && - compareByteArrays(notInValue, upperBound) <= 0 + (lowerOpen ? sortsAfter > 0 : sortsAfter >= 0) && + sortsBefore <= 0 ) { ranges.push( IDBKeyRange.bound( - lastLower, - this.generateNotInBound(indexRange.lower, notInValue), - lastOpen, - /* upperOpen= */ true + this.generateBound(indexRange.lower, lastLower), + this.generateBound(indexRange.lower, barrier), + lowerOpen, + /* upperOpen= */ false ) ); - - lastLower = this.generateNotInBound( - indexRange.lower, - successor(notInValue) - ); - lastOpen = true; + lowerOpen = true; + lastLower = successor(barrier); } } - - ranges.push( - IDBKeyRange.bound( - lastLower, - indexRange.upper, - lastOpen, - indexRange.upperOpen - ) - ); } return ranges; } @@ -900,7 +893,7 @@ export class IndexedDbIndexManager implements IndexManager { * Generates the index entry that can be used to create the cutoff for `value` * on the provided index range. */ - private generateNotInBound( + private generateBound( existingRange: DbIndexEntryKey, value: Uint8Array ): DbIndexEntryKey { diff --git a/packages/firestore/test/unit/local/index_manager.test.ts b/packages/firestore/test/unit/local/index_manager.test.ts index 8449bab4c44..3f5fe16eff3 100644 --- a/packages/firestore/test/unit/local/index_manager.test.ts +++ b/packages/firestore/test/unit/local/index_manager.test.ts @@ -365,6 +365,54 @@ describe('IndexedDbIndexManager', async () => { filter('count', '!=', 1) ); await verifyResults(q); + + q = queryWithAddedFilter( + queryWithAddedFilter(query('coll'), filter('count', '>', 2)), + filter('count', '!=', 2) + ); + await verifyResults(q, 'coll/val3'); + + q = queryWithAddedFilter( + queryWithAddedFilter(query('coll'), filter('count', '>=', 2)), + filter('count', '!=', 2) + ); + await verifyResults(q, 'coll/val3'); + + q = queryWithAddedFilter( + queryWithAddedFilter(query('coll'), filter('count', '<=', 2)), + filter('count', '!=', 2) + ); + await verifyResults(q, 'coll/val1'); + + q = queryWithAddedFilter( + queryWithAddedFilter(query('coll'), filter('count', '<=', 2)), + filter('count', '!=', 1) + ); + await verifyResults(q, 'coll/val2'); + + q = queryWithAddedFilter( + queryWithAddedFilter(query('coll'), filter('count', '<', 2)), + filter('count', '!=', 2) + ); + await verifyResults(q, 'coll/val1'); + + q = queryWithAddedFilter( + queryWithAddedFilter(query('coll'), filter('count', '<', 2)), + filter('count', '!=', 1) + ); + await verifyResults(q); + + q = queryWithAddedFilter( + queryWithAddedFilter(query('coll'), filter('count', '>', 2)), + filter('count', 'not-in', [3]) + ); + await verifyResults(q); + + q = queryWithAddedFilter( + queryWithAddedFilter(query('coll'), filter('count', '>', 2)), + filter('count', 'not-in', [2, 2]) + ); + await verifyResults(q, 'coll/val3'); }); it('applies less than filter', async () => { From fc1bd3735fa0fe97c5accd8c97b8d0b686062c2f Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 17 Feb 2022 18:15:06 -0700 Subject: [PATCH 06/12] Better test --- .../src/local/indexeddb_index_manager.ts | 29 ++-- .../test/unit/local/index_manager.test.ts | 127 ++++++++---------- 2 files changed, 71 insertions(+), 85 deletions(-) diff --git a/packages/firestore/src/local/indexeddb_index_manager.ts b/packages/firestore/src/local/indexeddb_index_manager.ts index 819c13396a7..35fffbd9e60 100644 --- a/packages/firestore/src/local/indexeddb_index_manager.ts +++ b/packages/firestore/src/local/indexeddb_index_manager.ts @@ -860,29 +860,26 @@ export class IndexedDbIndexManager implements IndexManager { const lowerBound = new Uint8Array(indexRange.lower[3]); const upperBound = new Uint8Array(indexRange.upper[3]); - let lastLower = lowerBound; - let lowerOpen = indexRange.lowerOpen; + let lastBound = lowerBound; + let lastOpen = indexRange.lowerOpen; - const barriers = [...notInValues, upperBound]; - for (const barrier of barriers) { + const bounds = [...notInValues, upperBound]; + for (const currentBound of bounds) { // Verify that the range in the bound is sensible, as the bound may get // rejected otherwise - const sortsAfter = compareByteArrays(barrier, lastLower); - const sortsBefore = compareByteArrays(barrier, upperBound); - if ( - (lowerOpen ? sortsAfter > 0 : sortsAfter >= 0) && - sortsBefore <= 0 - ) { + const sortsAfter = compareByteArrays(currentBound, lastBound); + const sortsBefore = compareByteArrays(currentBound, upperBound); + if ((lastOpen ? sortsAfter > 0 : sortsAfter >= 0) && sortsBefore <= 0) { ranges.push( IDBKeyRange.bound( - this.generateBound(indexRange.lower, lastLower), - this.generateBound(indexRange.lower, barrier), - lowerOpen, - /* upperOpen= */ false + this.generateBound(indexRange.lower, lastBound), + this.generateBound(indexRange.lower, currentBound), + lastOpen, + /* upperOpen= */ indexRange.upperOpen ) ); - lowerOpen = true; - lastLower = successor(barrier); + lastOpen = true; + lastBound = successor(currentBound); } } } diff --git a/packages/firestore/test/unit/local/index_manager.test.ts b/packages/firestore/test/unit/local/index_manager.test.ts index 3f5fe16eff3..dfde007d72a 100644 --- a/packages/firestore/test/unit/local/index_manager.test.ts +++ b/packages/firestore/test/unit/local/index_manager.test.ts @@ -58,6 +58,7 @@ import { import * as persistenceHelpers from './persistence_test_helpers'; import { TestIndexManager } from './test_index_manager'; +import { FieldFilter } from '../../../src/core/target'; describe('MemoryIndexManager', async () => { genericIndexManagerTests(persistenceHelpers.testMemoryEagerPersistence); @@ -348,77 +349,65 @@ describe('IndexedDbIndexManager', async () => { it('applies equals with not equals filter on same field', async () => { await setUpSingleValueFilter(); - let q = queryWithAddedFilter( - queryWithAddedFilter(query('coll'), filter('count', '>', 1)), - filter('count', '!=', 2) - ); - await verifyResults(q, 'coll/val3'); - - q = queryWithAddedFilter( - queryWithAddedFilter(query('coll'), filter('count', '==', 1)), - filter('count', '!=', 2) - ); - await verifyResults(q, 'coll/val1'); - - q = queryWithAddedFilter( - queryWithAddedFilter(query('coll'), filter('count', '==', 1)), - filter('count', '!=', 1) - ); - await verifyResults(q); - - q = queryWithAddedFilter( - queryWithAddedFilter(query('coll'), filter('count', '>', 2)), - filter('count', '!=', 2) - ); - await verifyResults(q, 'coll/val3'); - q = queryWithAddedFilter( - queryWithAddedFilter(query('coll'), filter('count', '>=', 2)), - filter('count', '!=', 2) - ); - await verifyResults(q, 'coll/val3'); - - q = queryWithAddedFilter( - queryWithAddedFilter(query('coll'), filter('count', '<=', 2)), - filter('count', '!=', 2) - ); - await verifyResults(q, 'coll/val1'); - - q = queryWithAddedFilter( - queryWithAddedFilter(query('coll'), filter('count', '<=', 2)), - filter('count', '!=', 1) - ); - await verifyResults(q, 'coll/val2'); - - q = queryWithAddedFilter( - queryWithAddedFilter(query('coll'), filter('count', '<', 2)), - filter('count', '!=', 2) - ); - await verifyResults(q, 'coll/val1'); - - q = queryWithAddedFilter( - queryWithAddedFilter(query('coll'), filter('count', '<', 2)), - filter('count', '!=', 1) - ); - await verifyResults(q); - - q = queryWithAddedFilter( - queryWithAddedFilter(query('coll'), filter('count', '>', 2)), - filter('count', 'not-in', [3]) - ); - await verifyResults(q); - - q = queryWithAddedFilter( - queryWithAddedFilter(query('coll'), filter('count', '>', 2)), - filter('count', 'not-in', [2, 2]) - ); - await verifyResults(q, 'coll/val3'); - }); + const filtersAndResults: Array = [ + [filter('count', '>', 1), filter('count', '!=', 2)], + ['coll/val3'], + [filter('count', '==', 1), filter('count', '!=', 2)], + ['coll/val1'], + [filter('count', '==', 1), filter('count', '!=', 1)], + [], + [filter('count', '>', 2), filter('count', '!=', 2)], + ['coll/val3'], + [filter('count', '>=', 2), filter('count', '!=', 2)], + ['coll/val3'], + [filter('count', '<=', 2), filter('count', '!=', 2)], + ['coll/val1'], + [filter('count', '<=', 2), filter('count', '!=', 1)], + ['coll/val2'], + [filter('count', '<', 2), filter('count', '!=', 2)], + ['coll/val1'], + [filter('count', '<', 2), filter('count', '!=', 1)], + [], + [filter('count', '>', 2), filter('count', 'not-in', [3])], + [], + [filter('count', '>=', 2), filter('count', 'not-in', [3])], + ['coll/val2'], + [filter('count', '>=', 2), filter('count', 'not-in', [3, 3])], + ['coll/val2'], + [ + filter('count', '>', 1), + filter('count', '<', 3), + filter('count', '!=', 2) + ], + [], + [ + filter('count', '>=', 1), + filter('count', '<', 3), + filter('count', '!=', 2) + ], + ['coll/val1'], + [ + filter('count', '>=', 1), + filter('count', '<=', 3), + filter('count', '!=', 2) + ], + ['coll/val1', 'coll/val3'], + [ + filter('count', '>', 1), + filter('count', '<=', 3), + filter('count', '!=', 2) + ], + ['coll/val3'] + ]; - it('applies less than filter', async () => { - await setUpSingleValueFilter(); - const q = queryWithAddedFilter(query('coll'), filter('count', '<', 2)); - await verifyResults(q, 'coll/val1'); + for (let i = 0; i < filtersAndResults.length; i += 2) { + let q = query('coll'); + for (const filter of filtersAndResults[i] as FieldFilter[]) { + q = queryWithAddedFilter(q, filter); + } + await verifyResults(q, ...(filtersAndResults[i + 1] as string[])); + } }); it('applies less than or equals filter', async () => { From 00df97dfab9354a3afdb5be57317c50681eeb597 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 17 Feb 2022 19:35:54 -0700 Subject: [PATCH 07/12] Lint --- packages/firestore/test/unit/local/index_manager.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/firestore/test/unit/local/index_manager.test.ts b/packages/firestore/test/unit/local/index_manager.test.ts index dfde007d72a..da842139396 100644 --- a/packages/firestore/test/unit/local/index_manager.test.ts +++ b/packages/firestore/test/unit/local/index_manager.test.ts @@ -29,6 +29,7 @@ import { queryWithLimit, queryWithStartAt } from '../../../src/core/query'; +import { FieldFilter } from '../../../src/core/target'; import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; import { INDEXING_SCHEMA_VERSION } from '../../../src/local/indexeddb_schema'; import { Persistence } from '../../../src/local/persistence'; @@ -58,7 +59,6 @@ import { import * as persistenceHelpers from './persistence_test_helpers'; import { TestIndexManager } from './test_index_manager'; -import { FieldFilter } from '../../../src/core/target'; describe('MemoryIndexManager', async () => { genericIndexManagerTests(persistenceHelpers.testMemoryEagerPersistence); From b87e324da122afcadda78ffc12ded0443997c2d3 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 17 Feb 2022 19:58:18 -0700 Subject: [PATCH 08/12] Add assert --- .../firestore/src/local/indexeddb_index_manager.ts | 6 +++++- .../firestore/test/unit/local/index_manager.test.ts | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/firestore/src/local/indexeddb_index_manager.ts b/packages/firestore/src/local/indexeddb_index_manager.ts index 35fffbd9e60..dd14635b997 100644 --- a/packages/firestore/src/local/indexeddb_index_manager.ts +++ b/packages/firestore/src/local/indexeddb_index_manager.ts @@ -857,6 +857,10 @@ export class IndexedDbIndexManager implements IndexManager { // that we would split an existing range into multiple ranges that exclude // the values from any notIn filter. for (const indexRange of indexRanges) { + debugAssert( + indexRange.lower[0] === indexRange.upper[0], + 'Index ID must match for index range' + ); const lowerBound = new Uint8Array(indexRange.lower[3]); const upperBound = new Uint8Array(indexRange.upper[3]); @@ -873,7 +877,7 @@ export class IndexedDbIndexManager implements IndexManager { ranges.push( IDBKeyRange.bound( this.generateBound(indexRange.lower, lastBound), - this.generateBound(indexRange.lower, currentBound), + this.generateBound(indexRange.upper, currentBound), lastOpen, /* upperOpen= */ indexRange.upperOpen ) diff --git a/packages/firestore/test/unit/local/index_manager.test.ts b/packages/firestore/test/unit/local/index_manager.test.ts index da842139396..ce13340982b 100644 --- a/packages/firestore/test/unit/local/index_manager.test.ts +++ b/packages/firestore/test/unit/local/index_manager.test.ts @@ -446,6 +446,18 @@ describe('IndexedDbIndexManager', async () => { await verifyResults(q, 'coll/val2', 'coll/val3'); }); + it('applies startAt filter with notIn', async () => { + await setUpSingleValueFilter(); + const q = queryWithStartAt( + queryWithAddedOrderBy( + queryWithAddedFilter(query('coll'), filter('count', '!=', 2)), + orderBy('count') + ), + bound([2], true) + ); + await verifyResults(q, 'coll/val3'); + }); + it('applies startAfter filter', async () => { await setUpSingleValueFilter(); const q = queryWithStartAt( From ba1507faf5be9c046bfc6d68e6e08cbbd96996fd Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Fri, 18 Feb 2022 11:18:57 -0700 Subject: [PATCH 09/12] Use precompute bounds --- packages/firestore/src/core/target.ts | 2 +- packages/firestore/src/index/index_entry.ts | 33 ++- .../src/local/indexeddb_index_manager.ts | 221 +++++++++--------- .../test/unit/local/index_manager.test.ts | 59 ++++- 4 files changed, 196 insertions(+), 119 deletions(-) diff --git a/packages/firestore/src/core/target.ts b/packages/firestore/src/core/target.ts index dd4c5a8351f..706a1024535 100644 --- a/packages/firestore/src/core/target.ts +++ b/packages/firestore/src/core/target.ts @@ -440,7 +440,7 @@ export function targetGetUpperBound( } if (segmentValue === undefined) { - // No lower bound exists + // No upper bound exists return null; } values.push(segmentValue); diff --git a/packages/firestore/src/index/index_entry.ts b/packages/firestore/src/index/index_entry.ts index 13ec55fe900..ee80276f325 100644 --- a/packages/firestore/src/index/index_entry.ts +++ b/packages/firestore/src/index/index_entry.ts @@ -25,6 +25,33 @@ export class IndexEntry { readonly arrayValue: Uint8Array, readonly directionalValue: Uint8Array ) {} + + /** + * Returns an IndexEntry entry that sorts immediately after the current + * directional value. + */ + successor(): IndexEntry { + const currentLength = this.directionalValue.length; + const newLength = + currentLength === 0 || this.directionalValue[currentLength - 1] === 255 + ? currentLength + 1 + : currentLength; + + const successor = new Uint8Array(newLength); + successor.set(this.directionalValue, 0); + if (newLength !== currentLength) { + successor.set([0], this.directionalValue.length); + } else { + ++successor[successor.length - 1]; + } + + return new IndexEntry( + this.indexId, + this.documentKey, + this.arrayValue, + successor + ); + } } export function indexEntryComparator( @@ -36,17 +63,17 @@ export function indexEntryComparator( return cmp; } - cmp = DocumentKey.comparator(left.documentKey, right.documentKey); + cmp = compareByteArrays(left.arrayValue, right.arrayValue); if (cmp !== 0) { return cmp; } - cmp = compareByteArrays(left.arrayValue, right.arrayValue); + cmp = compareByteArrays(left.directionalValue, right.directionalValue); if (cmp !== 0) { return cmp; } - return compareByteArrays(left.directionalValue, right.directionalValue); + return DocumentKey.comparator(left.documentKey, right.documentKey); } export function compareByteArrays(left: Uint8Array, right: Uint8Array): number { diff --git a/packages/firestore/src/local/indexeddb_index_manager.ts b/packages/firestore/src/local/indexeddb_index_manager.ts index dd14635b997..93886c55b93 100644 --- a/packages/firestore/src/local/indexeddb_index_manager.ts +++ b/packages/firestore/src/local/indexeddb_index_manager.ts @@ -30,11 +30,7 @@ import { } from '../core/target'; import { FirestoreIndexValueWriter } from '../index/firestore_index_value_writer'; import { IndexByteEncoder } from '../index/index_byte_encoder'; -import { - compareByteArrays, - IndexEntry, - indexEntryComparator -} from '../index/index_entry'; +import { IndexEntry, indexEntryComparator } from '../index/index_entry'; import { documentKeySet, DocumentKeySet, @@ -325,7 +321,7 @@ export class IndexedDbIndexManager implements IndexManager { lowerBoundInclusive: boolean, upperBounds: Uint8Array[] | null, upperBoundInclusive: boolean, - notInValues: Uint8Array[] | null + notInValues: Uint8Array[] ): IDBKeyRange[] { // The number of total index scans we union together. This is similar to a // distributed normal form, but adapted for array values. We create a single @@ -345,31 +341,46 @@ export class IndexedDbIndexManager implements IndexManager { const arrayValue = arrayValues ? this.encodeSingleElement(arrayValues[i / scansPerArrayElement]) : EMPTY_VALUE; + + const lowerBound = lowerBounds + ? this.generateLowerBound( + indexId, + arrayValue, + lowerBounds[i % scansPerArrayElement], + lowerBoundInclusive + ) + : this.generateEmptyBound(indexId); + const upperBound = upperBounds + ? this.generateUpperBound( + indexId, + arrayValue, + upperBounds[i % scansPerArrayElement], + upperBoundInclusive + ) + : this.generateEmptyBound(indexId + 1); + indexRanges.push( - IDBKeyRange.bound( - lowerBounds - ? this.generateLowerBound( + ...this.createRange( + lowerBound, + /* lowerInclusive= */ true, + upperBound, + /* upperInclusive= */ false, + notInValues.map( + ( + notIn // make non-nullable + ) => + this.generateLowerBound( indexId, arrayValue, - lowerBounds[i % scansPerArrayElement], - lowerBoundInclusive + notIn, + /* inclusive= */ true ) - : this.generateEmptyBound(indexId), - upperBounds - ? this.generateUpperBound( - indexId, - arrayValue, - upperBounds[i % scansPerArrayElement], - upperBoundInclusive - ) - : this.generateEmptyBound(indexId + 1), - /* lowerOpen= */ false, - /* upperOpen= */ true + ) ) ); } - return this.applyNotIn(indexRanges, notInValues); + return indexRanges; } /** Generates the lower bound for `arrayValue` and `directionalValue`. */ @@ -378,14 +389,14 @@ export class IndexedDbIndexManager implements IndexManager { arrayValue: Uint8Array, directionalValue: Uint8Array, inclusive: boolean - ): DbIndexEntryKey { - return [ + ): IndexEntry { + const entry = new IndexEntry( indexId, - this.uid, + DocumentKey.empty(), arrayValue, - inclusive ? directionalValue : successor(directionalValue), - '' - ]; + directionalValue + ); + return inclusive ? entry : entry.successor(); } /** Generates the upper bound for `arrayValue` and `directionalValue`. */ @@ -394,22 +405,27 @@ export class IndexedDbIndexManager implements IndexManager { arrayValue: Uint8Array, directionalValue: Uint8Array, inclusive: boolean - ): DbIndexEntryKey { - return [ + ): IndexEntry { + const entry = new IndexEntry( indexId, - this.uid, + DocumentKey.empty(), arrayValue, - inclusive ? successor(directionalValue) : directionalValue, - '' - ]; + directionalValue + ); + return inclusive ? entry.successor() : entry; } /** * Generates an empty bound that scopes the index scan to the current index * and user. */ - private generateEmptyBound(indexId: number): DbIndexEntryKey { - return [indexId, this.uid, EMPTY_VALUE, EMPTY_VALUE, '']; + private generateEmptyBound(indexId: number): IndexEntry { + return new IndexEntry( + indexId, + DocumentKey.empty(), + EMPTY_VALUE, + EMPTY_VALUE + ); } getFieldIndex( @@ -475,9 +491,9 @@ export class IndexedDbIndexManager implements IndexManager { fieldIndex: FieldIndex, target: Target, bound: ProtoValue[] | null - ): Uint8Array[] | null { - if (bound == null) { - return null; + ): Uint8Array[] { + if (bound === null) { + return []; } let encoders: IndexByteEncoder[] = []; @@ -835,90 +851,69 @@ export class IndexedDbIndexManager implements IndexManager { } /** - * Applies notIn and != filters by taking the provided ranges and excluding + * Returns a new set of IDB ranges that splits the existing range and excludes * any values that match the `notInValue` from these ranges. As an example, * '[foo > 2 && foo != 3]` becomes `[foo > 2 && < 3, foo > 3]`. */ - private applyNotIn( - indexRanges: IDBKeyRange[], - notInValues: Uint8Array[] | null + private createRange( + lower: IndexEntry, + lowerInclusive: boolean, + upper: IndexEntry, + upperInclusive: boolean, + notInValues: IndexEntry[] ): IDBKeyRange[] { - if (!notInValues) { - return indexRanges; - } + // The notIb values need to be sorted and unique so that we can return a + // sorted set of non-overlapping ranges. + notInValues = notInValues + .sort((l, r) => indexEntryComparator(l, r)) + .filter( + (el, i, values) => !i || indexEntryComparator(el, values[i - 1]) !== 0 + ); - // The values need to be sorted so that we can return a sorted set of - // non-overlapping ranges. - notInValues.sort((l, r) => compareByteArrays(l, r)); + const bounds: IndexEntry[] = []; + bounds.push(lower); + for (const notInValue of notInValues) { + const sortsAfter = indexEntryComparator(notInValue, lower); + const sortsBefore = indexEntryComparator(notInValue, upper); + + if (sortsAfter > 0 && sortsBefore < 0) { + bounds.push(notInValue); + bounds.push(notInValue.successor()); + } else if (sortsAfter === 0) { + // The lowest value in the range is excluded + bounds[0] = lower.successor(); + } else if (sortsBefore === 0) { + // The largest value in the range is excluded + upperInclusive = false; + } + } + bounds.push(upper); const ranges: IDBKeyRange[] = []; - - // Use the existing bounds and interleave the notIn values. This means - // that we would split an existing range into multiple ranges that exclude - // the values from any notIn filter. - for (const indexRange of indexRanges) { - debugAssert( - indexRange.lower[0] === indexRange.upper[0], - 'Index ID must match for index range' + for (let i = 0; i < bounds.length; i += 2) { + ranges.push( + IDBKeyRange.bound( + [ + bounds[i].indexId, + this.uid, + bounds[i].arrayValue, + bounds[i].directionalValue, + '' + ], + [ + bounds[i + 1].indexId, + this.uid, + bounds[i + 1].arrayValue, + bounds[i + 1].directionalValue, + '' + ], + !lowerInclusive, + !upperInclusive + ) ); - const lowerBound = new Uint8Array(indexRange.lower[3]); - const upperBound = new Uint8Array(indexRange.upper[3]); - - let lastBound = lowerBound; - let lastOpen = indexRange.lowerOpen; - - const bounds = [...notInValues, upperBound]; - for (const currentBound of bounds) { - // Verify that the range in the bound is sensible, as the bound may get - // rejected otherwise - const sortsAfter = compareByteArrays(currentBound, lastBound); - const sortsBefore = compareByteArrays(currentBound, upperBound); - if ((lastOpen ? sortsAfter > 0 : sortsAfter >= 0) && sortsBefore <= 0) { - ranges.push( - IDBKeyRange.bound( - this.generateBound(indexRange.lower, lastBound), - this.generateBound(indexRange.upper, currentBound), - lastOpen, - /* upperOpen= */ indexRange.upperOpen - ) - ); - lastOpen = true; - lastBound = successor(currentBound); - } - } } return ranges; } - - /** - * Generates the index entry that can be used to create the cutoff for `value` - * on the provided index range. - */ - private generateBound( - existingRange: DbIndexEntryKey, - value: Uint8Array - ): DbIndexEntryKey { - return [ - /* indexId= */ existingRange[0], - /* userId= */ existingRange[1], - /* arrayValue= */ existingRange[2], - value, - /* documentKey= */ '' - ]; - } -} - -/** Creates a Uint8Array value that sorts immediately after `value`. */ -function successor(value: Uint8Array): Uint8Array { - if (value.length === 0 || value[value.length - 1] === 255) { - const successor = new Uint8Array(value.length + 1); - successor.set(value, 0); - successor.set([0], value.length); - return successor; - } else { - ++value[value.length - 1]; - } - return value; } /** diff --git a/packages/firestore/test/unit/local/index_manager.test.ts b/packages/firestore/test/unit/local/index_manager.test.ts index ce13340982b..da174587a62 100644 --- a/packages/firestore/test/unit/local/index_manager.test.ts +++ b/packages/firestore/test/unit/local/index_manager.test.ts @@ -291,12 +291,25 @@ describe('IndexedDbIndexManager', async () => { fieldIndex('coll', { fields: [['count', IndexKind.ASCENDING]] }) ); await addDoc('coll/val1', { 'count': 1 }); - await addDoc('coll/val2', { 'not-count': 3 }); - await addDoc('coll/val3', { 'count': 2 }); + await addDoc('coll/val2', { 'not-count': 2 }); + await addDoc('coll/val3', { 'count': 3 }); const q = queryWithAddedOrderBy(query('coll'), orderBy('count')); await verifyResults(q, 'coll/val1', 'coll/val3'); }); + it('applies orderBy with not equals filter', async () => { + await indexManager.addFieldIndex( + fieldIndex('coll', { fields: [['count', IndexKind.ASCENDING]] }) + ); + await addDoc('coll/val1', { 'count': 1 }); + await addDoc('coll/val2', { 'count': 2 }); + const q = queryWithAddedOrderBy( + queryWithAddedFilter(query('coll'), filter('count', '!=', 2)), + orderBy('count') + ); + await verifyResults(q, 'coll/val1'); + }); + it('applies equality filter', async () => { await setUpSingleValueFilter(); const q = queryWithAddedFilter(query('coll'), filter('count', '==', 2)); @@ -347,6 +360,48 @@ describe('IndexedDbIndexManager', async () => { await verifyResults(q, 'coll/val2'); }); + it('applies array contains with not equals filter', async () => { + await indexManager.addFieldIndex( + fieldIndex('coll', { + fields: [ + ['a', IndexKind.CONTAINS], + ['b', IndexKind.ASCENDING] + ] + }) + ); + await addDoc('coll/val1', { 'a': [1], 'b': 1 }); + await addDoc('coll/val2', { 'a': [1], 'b': 2 }); + await addDoc('coll/val3', { 'a': [2], 'b': 1 }); + await addDoc('coll/val4', { 'a': [2], 'b': 2 }); + + const q = queryWithAddedFilter( + queryWithAddedFilter(query('coll'), filter('a', 'array-contains', 1)), + filter('b', '!=', 1) + ); + await verifyResults(q, 'coll/val2'); + }); + + it('applies array contains with not equals filter on same field', async () => { + await indexManager.addFieldIndex( + fieldIndex('coll', { + fields: [ + ['a', IndexKind.CONTAINS], + ['a', IndexKind.ASCENDING] + ] + }) + ); + await addDoc('coll/val1', { 'a': [1, 1] }); + await addDoc('coll/val2', { 'a': [1, 2] }); + await addDoc('coll/val3', { 'a': [2, 1] }); + await addDoc('coll/val4', { 'a': [2, 2] }); + + const q = queryWithAddedFilter( + queryWithAddedFilter(query('coll'), filter('a', 'array-contains', 1)), + filter('a', '!=', [1, 2]) + ); + await verifyResults(q, 'coll/val1', 'coll/val3'); + }); + it('applies equals with not equals filter on same field', async () => { await setUpSingleValueFilter(); From 56e7441421be31066b3c4d1d80741bffd082d4f0 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Fri, 18 Feb 2022 16:51:49 -0700 Subject: [PATCH 10/12] Next round --- .../src/local/indexeddb_index_manager.ts | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/packages/firestore/src/local/indexeddb_index_manager.ts b/packages/firestore/src/local/indexeddb_index_manager.ts index 93886c55b93..065f9d95f98 100644 --- a/packages/firestore/src/local/indexeddb_index_manager.ts +++ b/packages/firestore/src/local/indexeddb_index_manager.ts @@ -873,18 +873,24 @@ export class IndexedDbIndexManager implements IndexManager { const bounds: IndexEntry[] = []; bounds.push(lower); for (const notInValue of notInValues) { - const sortsAfter = indexEntryComparator(notInValue, lower); - const sortsBefore = indexEntryComparator(notInValue, upper); + const cpmToLower = indexEntryComparator(notInValue, lower); + const cmpToUpper = indexEntryComparator(notInValue, upper); - if (sortsAfter > 0 && sortsBefore < 0) { + if (cpmToLower === 0) { + // `notInValue` is the lower bound. We therefore need to raise the bound + // to the next value. + bounds[0] = lower.successor(); + } else if (cpmToLower > 0 && cmpToUpper < 0) { + // `notInValue` is in the middle of the range bounds.push(notInValue); bounds.push(notInValue.successor()); - } else if (sortsAfter === 0) { - // The lowest value in the range is excluded - bounds[0] = lower.successor(); - } else if (sortsBefore === 0) { - // The largest value in the range is excluded + } else if (cmpToUpper === 0) { + // `notInValue` is the upper value. We therefore need to exclude the + // upper bound. upperInclusive = false; + } else { + // `notInValue` (and all following values) are out of the range + break; } } bounds.push(upper); From aa5c523ea32040e4d256f6fe907bec175106369f Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Tue, 22 Feb 2022 09:39:01 -0700 Subject: [PATCH 11/12] Update indexeddb_index_manager.ts --- packages/firestore/src/local/indexeddb_index_manager.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/firestore/src/local/indexeddb_index_manager.ts b/packages/firestore/src/local/indexeddb_index_manager.ts index 065f9d95f98..e65510204ed 100644 --- a/packages/firestore/src/local/indexeddb_index_manager.ts +++ b/packages/firestore/src/local/indexeddb_index_manager.ts @@ -873,14 +873,14 @@ export class IndexedDbIndexManager implements IndexManager { const bounds: IndexEntry[] = []; bounds.push(lower); for (const notInValue of notInValues) { - const cpmToLower = indexEntryComparator(notInValue, lower); + const cmpToLower = indexEntryComparator(notInValue, lower); const cmpToUpper = indexEntryComparator(notInValue, upper); - if (cpmToLower === 0) { + if (cmpToLower === 0) { // `notInValue` is the lower bound. We therefore need to raise the bound // to the next value. bounds[0] = lower.successor(); - } else if (cpmToLower > 0 && cmpToUpper < 0) { + } else if (cmpToLower > 0 && cmpToUpper < 0) { // `notInValue` is in the middle of the range bounds.push(notInValue); bounds.push(notInValue.successor()); From 6b88a47ca84009ed28742b9e0528821c45d70b9c Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Tue, 22 Feb 2022 15:19:08 -0700 Subject: [PATCH 12/12] Fix browser tests --- .../firestore/src/local/indexeddb_index_manager.ts | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/packages/firestore/src/local/indexeddb_index_manager.ts b/packages/firestore/src/local/indexeddb_index_manager.ts index e65510204ed..40176dea847 100644 --- a/packages/firestore/src/local/indexeddb_index_manager.ts +++ b/packages/firestore/src/local/indexeddb_index_manager.ts @@ -362,9 +362,7 @@ export class IndexedDbIndexManager implements IndexManager { indexRanges.push( ...this.createRange( lowerBound, - /* lowerInclusive= */ true, upperBound, - /* upperInclusive= */ false, notInValues.map( ( notIn // make non-nullable @@ -857,9 +855,7 @@ export class IndexedDbIndexManager implements IndexManager { */ private createRange( lower: IndexEntry, - lowerInclusive: boolean, upper: IndexEntry, - upperInclusive: boolean, notInValues: IndexEntry[] ): IDBKeyRange[] { // The notIb values need to be sorted and unique so that we can return a @@ -884,11 +880,7 @@ export class IndexedDbIndexManager implements IndexManager { // `notInValue` is in the middle of the range bounds.push(notInValue); bounds.push(notInValue.successor()); - } else if (cmpToUpper === 0) { - // `notInValue` is the upper value. We therefore need to exclude the - // upper bound. - upperInclusive = false; - } else { + } else if (cmpToUpper > 0) { // `notInValue` (and all following values) are out of the range break; } @@ -912,9 +904,7 @@ export class IndexedDbIndexManager implements IndexManager { bounds[i + 1].arrayValue, bounds[i + 1].directionalValue, '' - ], - !lowerInclusive, - !upperInclusive + ] ) ); }