feat: Add Redis caching for navigation pagination

.github/workflows/cicd.yml

@@ -66,6 +66,16 @@ jobs:
         ports:
           - 9202:9202
 
+      redis:
+        image: redis:7-alpine
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
     strategy:
       matrix:
         python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13"]
@@ -126,3 +136,6 @@ jobs:
           DATABASE_REFRESH: true
           ES_VERIFY_CERTS: false
           BACKEND: ${{ matrix.backend == 'elasticsearch7' && 'elasticsearch' || matrix.backend == 'elasticsearch8' && 'elasticsearch' || 'opensearch' }}
+          REDIS_ENABLE: true
+          REDIS_HOST: localhost
+          REDIS_PORT: 6379

CHANGELOG.md

@@ -9,8 +9,24 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 
+- Added Redis caching configuration for navigation pagination support, enabling proper `prev` and `next` links in paginated responses. [#466](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/466)
+
+### Changed
+
+### Fixed
+
+
+## [v6.4.0] - 2025-09-24
+
+### Added
+
+- GET `/collections` collection search free text extension ex. `/collections?q=sentinel`. [#470](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/470)
 - Added `USE_DATETIME` environment variable to configure datetime search behavior in SFEOS. [#452](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/452)
 - GET `/collections` collection search sort extension ex. `/collections?sortby=+id`. [#456](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/456)
+- GET `/collections` collection search fields extension ex. `/collections?fields=id,title`. [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465)
+- Improved error messages for sorting on unsortable fields in collection search, including guidance on how to make fields sortable. [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465)
+- Added field alias for `temporal` to enable easier sorting by temporal extent, alongside `extent.temporal.interval`. [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465)
+- Added `ENABLE_COLLECTIONS_SEARCH` environment variable to make collection search extensions optional (defaults to enabled). [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465)
 
 ### Changed
 
@@ -518,7 +534,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Use genexp in execute_search and get_all_collections to return results.
 - Added db_to_stac serializer to item_collection method in core.py.
 
-[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.3.0...main
+[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.4.0...main
+[v6.4.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.3.0...v6.4.0
 [v6.3.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.2.1...v6.3.0
 [v6.2.1]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.2.0...v6.2.1
 [v6.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.1.0...v6.2.0

Makefile

@@ -63,26 +63,28 @@ docker-shell-os:
 
 .PHONY: test-elasticsearch
 test-elasticsearch:
-	-$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest'
-	docker compose down
+	docker compose -f compose-redis.yml up -d
+	-$(run_es) /bin/bash -c 'export REDIS_ENABLE=true REDIS_HOST=redis REDIS_PORT=6379 && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest'
+	docker compose -f compose-redis.yml down
 
-.PHONY: test-opensearch
+.PHONY: test-opensearch  
 test-opensearch:
-	-$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest'
-	docker compose down
+	docker compose -f compose-redis.yml up -d
+	-$(run_os) /bin/bash -c 'export REDIS_ENABLE=true REDIS_HOST=redis REDIS_PORT=6379 && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest'
+	docker compose -f compose-redis.yml down
 
 .PHONY: test-datetime-filtering-es
 test-datetime-filtering-es:
-	-$(run_es) /bin/bash -c 'export ENABLE_DATETIME_INDEX_FILTERING=true && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest -s --cov=stac_fastapi --cov-report=term-missing -m datetime_filtering'
+	-$(run_es) /bin/bash -c 'pip install redis==6.4.0 && export ENABLE_DATETIME_INDEX_FILTERING=true && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest -s --cov=stac_fastapi --cov-report=term-missing -m datetime_filtering'
 	docker compose down
 
 .PHONY: test-datetime-filtering-os
 test-datetime-filtering-os:
-	-$(run_os) /bin/bash -c 'export ENABLE_DATETIME_INDEX_FILTERING=true && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest -s --cov=stac_fastapi --cov-report=term-missing -m datetime_filtering'
+	-$(run_os) /bin/bash -c 'pip install redis==6.4.0 && export ENABLE_DATETIME_INDEX_FILTERING=true && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest -s --cov=stac_fastapi --cov-report=term-missing -m datetime_filtering'
 	docker compose down
 
 .PHONY: test
-test: test-elasticsearch test-datetime-filtering-es test-opensearch test-datetime-filtering-os
+test: test-elasticsearch test-datetime-filtering-es test-opensearch test-datetime-filtering-os test-redis-es test-redis-os
 
 .PHONY: run-database-es
 run-database-es:
@@ -117,4 +119,16 @@ docs-image:
 .PHONY: docs
 docs: docs-image
 	docker compose -f compose.docs.yml \
-		run docs
+		run docs
+
+.PHONY: test-redis-es
+test-redis-es:
+	docker compose -f compose-redis.yml up -d
+	-$(run_es) /bin/bash -c 'export REDIS_ENABLE=true REDIS_HOST=redis REDIS_PORT=6379 && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest redis/ -v'
+	docker compose -f compose-redis.yml down
+
+.PHONY: test-redis-os
+test-redis-os:
+	docker compose -f compose-redis.yml up -d
+	-$(run_os) /bin/bash -c 'export REDIS_ENABLE=true REDIS_HOST=redis REDIS_PORT=6379 && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest redis/ -v'
+	docker compose -f compose-redis.yml down

README.md

@@ -36,11 +36,10 @@ SFEOS (stac-fastapi-elasticsearch-opensearch) is a high-performance, scalable AP
 - **Scale to millions of geospatial assets** with fast search performance through optimized spatial indexing and query capabilities
 - **Support OGC-compliant filtering** including spatial operations (intersects, contains, etc.) and temporal queries
 - **Perform geospatial aggregations** to analyze data distribution across space and time
+- **Enhanced collection search capabilities** with support for sorting and field selection
 
 This implementation builds on the STAC-FastAPI framework, providing a production-ready solution specifically optimized for Elasticsearch and OpenSearch databases. It's ideal for organizations managing large geospatial data catalogs who need efficient discovery and access capabilities through standardized APIs.
 
-
-
 ## Common Deployment Patterns
 
 stac-fastapi-elasticsearch-opensearch can be deployed in several ways depending on your needs:
@@ -72,6 +71,7 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI
   - [Common Deployment Patterns](#common-deployment-patterns)
   - [Technologies](#technologies)
   - [Table of Contents](#table-of-contents)
+  - [Collection Search Extensions](#collection-search-extensions)
   - [Documentation \& Resources](#documentation--resources)
   - [Package Structure](#package-structure)
   - [Examples](#examples)
@@ -113,6 +113,37 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI
   - [Gitter Chat](https://app.gitter.im/#/room/#stac-fastapi-elasticsearch_community:gitter.im) - For real-time discussions
   - [GitHub Discussions](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/discussions) - For longer-form questions and answers
 
+## Collection Search Extensions
+
+SFEOS implements extended capabilities for the `/collections` endpoint, allowing for more powerful collection discovery:
+
+- **Sorting**: Sort collections by sortable fields using the `sortby` parameter
+  - Example: `/collections?sortby=+id` (ascending sort by ID)
+  - Example: `/collections?sortby=-id` (descending sort by ID)
+  - Example: `/collections?sortby=-temporal` (descending sort by temporal extent)
+
+- **Field Selection**: Request only specific fields to be returned using the `fields` parameter
+  - Example: `/collections?fields=id,title,description`
+  - This helps reduce payload size when only certain fields are needed
+
+- **Free Text Search**: Search across collection text fields using the `q` parameter
+  - Example: `/collections?q=landsat`
+  - Searches across multiple text fields including title, description, and keywords
+  - Supports partial word matching and relevance-based sorting
+
+These extensions make it easier to build user interfaces that display and navigate through collections efficiently.
+
+> **Configuration**: Collection search extensions can be disabled by setting the `ENABLE_COLLECTIONS_SEARCH` environment variable to `false`. By default, these extensions are enabled.
+
+> **Note**: Sorting is only available on fields that are indexed for sorting in Elasticsearch/OpenSearch. With the default mappings, you can sort on:
+> - `id` (keyword field)
+> - `extent.temporal.interval` (date field)
+> - `temporal` (alias to extent.temporal.interval)
+>
+> Text fields like `title` and `description` are not sortable by default as they use text analysis for better search capabilities. Attempting to sort on these fields will result in a user-friendly error message explaining which fields are sortable and how to make additional fields sortable by updating the mappings.
+>
+> **Important**: Adding keyword fields to make text fields sortable can significantly increase the index size, especially for large text fields. Consider the storage implications when deciding which fields to make sortable.
+
 ## Package Structure
 
 This project is organized into several packages, each with a specific purpose:
@@ -243,6 +274,7 @@ You can customize additional settings in your `.env` file:
 | `ENABLE_DIRECT_RESPONSE`     | Enable direct response for maximum performance (disables all FastAPI dependencies, including authentication, custom status codes, and validation) | `false`                  | Optional                       |
 | `RAISE_ON_BULK_ERROR`        | Controls whether bulk insert operations raise exceptions on errors. If set to `true`, the operation will stop and raise an exception when an error occurs. If set to `false`, errors will be logged, and the operation will continue. **Note:** STAC Item and ItemCollection validation errors will always raise, regardless of this flag. | `false` | Optional |
 | `DATABASE_REFRESH`           | Controls whether database operations refresh the index immediately after changes. If set to `true`, changes will be immediately searchable. If set to `false`, changes may not be immediately visible but can improve performance for bulk operations. If set to `wait_for`, changes will wait for the next refresh cycle to become visible. | `false` | Optional |
+| `ENABLE_COLLECTIONS_SEARCH`  | Enable collection search extensions (sort, fields).                                 | `true`                   | Optional                                                                                    |
 | `ENABLE_TRANSACTIONS_EXTENSIONS` | Enables or disables the Transactions and Bulk Transactions API extensions. If set to `false`, the POST `/collections` route and related transaction endpoints (including bulk transaction operations) will be unavailable in the API. This is useful for deployments where mutating the catalog via the API should be prevented. | `true` | Optional |
 | `STAC_ITEM_LIMIT` | Sets the environment variable for result limiting to SFEOS for the number of returned items and STAC collections. | `10` | Optional |
 | `STAC_INDEX_ASSETS` | Controls if Assets are indexed when added to Elasticsearch/Opensearch. This allows asset fields to be included in search queries. | `false` | Optional |
@@ -252,6 +284,29 @@ You can customize additional settings in your `.env` file:
 > [!NOTE]
 > The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
 
+**Redis for Navigation:**
+These Redis configuration variables enable proper navigation functionality in STAC FastAPI. The Redis cache stores navigation state for paginated results, allowing the system to maintain previous page links using tokens. The configuration supports either Redis Sentinel or Redis:
+
+| Variable                     | Description                                                                          | Default                  | Required                                                                                     |
+|------------------------------|--------------------------------------------------------------------------------------|--------------------------|---------------------------------------------------------------------------------------------|
+| **Redis Sentinel**           |                                                                                      |                          |                                                                                             |
+| `REDIS_SENTINEL_HOSTS`       | Comma-separated list of Redis Sentinel hostnames/IP addresses.                       | `""`                     | Conditional (required if using Sentinel)                                                    |
+| `REDIS_SENTINEL_PORTS`       | Comma-separated list of Redis Sentinel ports (must match order).               | `"26379"`                | Conditional (required if using Sentinel)                                                    |
+| `REDIS_SENTINEL_MASTER_NAME` | Name of the Redis master node in Sentinel configuration.                             | `"master"`               | Conditional (required if using Sentinel)                                                    |
+| **Redis**                    |                                                                                      |                          |                                                                                             |
+| `REDIS_HOST`                 | Redis server hostname or IP address for Redis configuration.                    | `""`                     | Conditional (required for standalone Redis)                                                 |
+| `REDIS_PORT`                 | Redis server port for Redis configuration.                                      | `6379`                   | Conditional (required for standalone Redis)                                                 |
+| **Both**                     |                                                                                      |                          |                                                                                             |
+| `REDIS_DB`                   | Redis database number to use for caching.                                            | `0` (Sentinel) / `0` (Standalone) | Optional                                                                                    |
+| `REDIS_MAX_CONNECTIONS`      | Maximum number of connections in the Redis connection pool.                          | `10`                     | Optional                                                                                    |
+| `REDIS_RETRY_TIMEOUT`        | Enable retry on timeout for Redis operations.                                        | `true`                   | Optional                                                                                    |
+| `REDIS_DECODE_RESPONSES`     | Automatically decode Redis responses to strings.                                     | `true`                   | Optional                                                                                    |
+| `REDIS_CLIENT_NAME`          | Client name identifier for Redis connections.                                        | `"stac-fastapi-app"`     | Optional                                                                                    |
+| `REDIS_HEALTH_CHECK_INTERVAL`| Interval in seconds for Redis health checks.                                         | `30`                     | Optional                                                                                    |
+
+> [!NOTE]
+> Use either the Sentinel configuration (`REDIS_SENTINEL_HOSTS`, `REDIS_SENTINEL_PORTS`, `REDIS_SENTINEL_MASTER_NAME`) OR the Redis configuration (`REDIS_HOST`, `REDIS_PORT`), but not both.
+
 ## Datetime-Based Index Management
 
 ### Overview
@@ -389,6 +444,10 @@ The system uses a precise naming convention:
 - **Root Path Configuration**: The application root path is the base URL by default.
   - For AWS Lambda with Gateway API: Set `STAC_FASTAPI_ROOT_PATH` to match the Gateway API stage name (e.g., `/v1`)
 
+- **Feature Configuration**: Control which features are enabled:
+  - `ENABLE_COLLECTIONS_SEARCH`: Set to `true` (default) to enable collection search extensions (sort, fields). Set to `false` to disable.
+  - `ENABLE_TRANSACTIONS_EXTENSIONS`: Set to `true` (default) to enable transaction extensions. Set to `false` to disable.
+
 
 ## Collection Pagination
 

compose-redis.yml

@@ -0,0 +1,13 @@
+version: '3.8'
+
+services:
+  redis:
+    image: redis:7-alpine
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis_test_data:/data
+    command: redis-server --appendonly yes
+
+volumes:
+  redis_test_data:

docker-compose.redis.yml

@@ -0,0 +1,27 @@
+version: '3.8'
+
+services:
+  redis:
+    image: redis:7-alpine
+    container_name: stac-fastapi-redis
+    ports:
+      - "6379:6379"
+    command: redis-server --appendonly yes
+    volumes:
+      - redis_data:/data
+    environment:
+      - REDIS_PORT=6379
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+    networks:
+      - stac-network
+
+volumes:
+  redis_data:
+
+networks:
+  stac-network:
+    driver: bridge

mypy.ini

@@ -0,0 +1,3 @@
+[mypy]
+[mypy-redis.*]
+ignore_missing_imports = True

stac_fastapi/core/setup.py

@@ -19,6 +19,7 @@
     "pygeofilter~=0.3.1",
     "jsonschema~=4.0.0",
     "slowapi~=0.1.9",
+    "redis==6.4.0",
 ]
 
 setup(