diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 043695a..02caf57 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,6 +19,8 @@ jobs: - uses: hecrj/setup-rust-action@v2 with: rust-version: ${{ matrix.rust }} + - name: Install protoc + uses: arduino/setup-protoc@v3 - name: Build SDK run: cargo build - name: Build documentation diff --git a/Cargo.lock b/Cargo.lock index 67cdf8b..c889daf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "anyhow" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" + [[package]] name = "ascii-canvas" version = "3.0.0" @@ -237,6 +243,28 @@ dependencies = [ "wasm-bindgen-futures", ] +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "async-task" version = "4.7.1" @@ -266,6 +294,51 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.29", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper 0.1.2", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.72" @@ -670,6 +743,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.5" @@ -682,13 +774,19 @@ dependencies = [ "futures-core", "futures-sink", "http 1.1.0", - "indexmap", + "indexmap 2.2.6", "slab", "tokio", "tokio-util", "tracing", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -819,6 +917,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -841,7 +940,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2", + "h2 0.4.5", "http 1.1.0", "http-body 1.0.0", "httparse", @@ -862,13 +961,25 @@ dependencies = [ "http 1.1.0", "hyper 1.3.1", "hyper-util", - "rustls", + "rustls 0.23.10", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.0", "tower-service", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper 0.14.29", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "hyper-tls" version = "0.6.0" @@ -915,6 +1026,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.2.6" @@ -922,7 +1043,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.14.5", ] [[package]] @@ -960,6 +1081,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -993,7 +1123,7 @@ dependencies = [ "ascii-canvas", "bit-set", "ena", - "itertools", + "itertools 0.11.0", "lalrpop-util", "petgraph", "pico-args", @@ -1074,6 +1204,12 @@ dependencies = [ "value-bag", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "memchr" version = "2.7.2" @@ -1116,6 +1252,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "multimap" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" + [[package]] name = "native-tls" version = "0.2.12" @@ -1261,7 +1403,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 2.2.6", ] [[package]] @@ -1316,7 +1458,10 @@ name = "pinecone_sdk" version = "0.1.0" dependencies = [ "httpmock", + "once_cell", "openapi", + "prost", + "prost-types", "rand", "regex", "reqwest", @@ -1324,6 +1469,8 @@ dependencies = [ "snafu", "temp-env", "tokio", + "tonic", + "tonic-build", ] [[package]] @@ -1386,6 +1533,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "prettyplease" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" +dependencies = [ + "proc-macro2", + "syn 2.0.66", +] + [[package]] name = "proc-macro2" version = "1.0.84" @@ -1395,6 +1552,59 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" +dependencies = [ + "bytes", + "heck", + "itertools 0.12.1", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.66", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +dependencies = [ + "anyhow", + "itertools 0.12.1", + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "prost-types" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.36" @@ -1494,7 +1704,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", + "h2 0.4.5", "http 1.1.0", "http-body 1.0.0", "http-body-util", @@ -1515,7 +1725,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 1.0.1", "system-configuration", "tokio", "tokio-native-tls", @@ -1575,6 +1785,20 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +dependencies = [ + "log", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + [[package]] name = "rustls" version = "0.23.10" @@ -1588,6 +1812,19 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "2.1.2" @@ -1851,6 +2088,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "sync_wrapper" version = "1.0.1" @@ -1973,6 +2216,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.3.0" @@ -1994,17 +2247,39 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +dependencies = [ + "rustls 0.22.4", + "rustls-pki-types", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls", + "rustls 0.23.10", "rustls-pki-types", "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.11" @@ -2018,6 +2293,50 @@ dependencies = [ "tokio", ] +[[package]] +name = "tonic" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.21.7", + "bytes", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.29", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "rustls-native-certs", + "rustls-pemfile", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.25.0", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4ef6dd70a610078cb4e338a0f79d06bc759ff1b22d2120c2ff02ae264ba9c2" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "quote", + "syn 2.0.66", +] + [[package]] name = "tower" version = "0.4.13" @@ -2026,11 +2345,16 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", + "indexmap 1.9.3", "pin-project", "pin-project-lite", + "rand", + "slab", "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -2052,9 +2376,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "tracing-core" version = "0.1.32" diff --git a/Cargo.toml b/Cargo.toml index 93bc083..dc62ba5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [workspace] members = [ "openapi", - "pinecone_sdk", + "pinecone_sdk" ] exclude = [ "codegen" diff --git a/README.md b/README.md index c1b7f0a..aeabd9a 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,12 @@ - You need [Docker Desktop](https://www.docker.com/products/docker-desktop/) in order to generate code using openapi. Look at `codegen/build-oas.sh` to see how this is used. - Install [homebrew](https://brew.sh/) if you haven't already -- Instasll `brew install just`. just is a command line tool used to run commands defined inside the project `justfile`. You run commands by passing the command name, for example `just build-openapi`. +- Install `brew install just`. just is a command line tool used to run commands defined inside the project `justfile`. You run commands by passing the command name, for example `just build-openapi`. + +Dependencies for generating code from OpenAPI and protobuf specifications: +- Follow setup instructions for the `apis` repository +- `brew install protobuf` +- `cargo install protobuf-codegen` and add it to path: `PATH="$HOME/.cargo/bin:$PATH"` # Clone the repository @@ -20,4 +25,19 @@ Or, if you have already cloned the repository, you can update the submodules wit ``` git submodule update --init --recursive -``` \ No newline at end of file +``` + +# Build and run + +OpenAPI +- The generated code is comitted to the repository, but to regenerate run `just build-openapi` +- References the spec files from the `codegen/apis` submodule +- Outputs the generated code to `openapi` + +Protobuf +- Code is generated from protobuf during the project build process (`cargo build`) using the build script `build.rs` +- The generated code is outputted to `/target/debug/build/pinecone_sdk-{hash}/out` + +Build and run the project: +- `cargo build` builds the project +- `cargo test` builds the project and runs tests diff --git a/pinecone_sdk/Cargo.toml b/pinecone_sdk/Cargo.toml index 8d55ad5..9c2c892 100644 --- a/pinecone_sdk/Cargo.toml +++ b/pinecone_sdk/Cargo.toml @@ -12,8 +12,15 @@ regex = "1.10.4" serde_json = "1.0.117" snafu = "0.8.3" rand = "0.8.5" +tonic = { version = "0.11.0", features = ["tls", "transport", "tls-roots"] } +prost = "0.12.6" +prost-types = "0.12.6" reqwest = "0.12.5" +once_cell = "1.19.0" [dev-dependencies] temp-env = "0.3.6" httpmock = "0.7.0-rc.1" + +[build-dependencies] +tonic-build = "0.11.0" diff --git a/pinecone_sdk/build.rs b/pinecone_sdk/build.rs new file mode 100644 index 0000000..cb8679d --- /dev/null +++ b/pinecone_sdk/build.rs @@ -0,0 +1,18 @@ +use std::path::Path; + +fn main() -> Result<(), Box> { + let proto_path: &Path = "../protos/data_2024-07.proto".as_ref(); + + // directory the main .proto file resides in + let proto_dir = proto_path + .parent() + .expect("proto file should reside in a directory"); + + let include_dirs = [proto_dir]; + + tonic_build::configure() + .protoc_arg("--experimental_allow_proto3_optional") + .compile(&[proto_path], &include_dirs[..])?; + + Ok(()) +} diff --git a/pinecone_sdk/src/lib.rs b/pinecone_sdk/src/lib.rs index b0f956d..a743196 100644 --- a/pinecone_sdk/src/lib.rs +++ b/pinecone_sdk/src/lib.rs @@ -9,9 +9,6 @@ /// Defines configurations for the Pinecone SDK. pub mod config; -/// Control plane module. -pub mod control; - /// Defines the main entrypoint of the Pinecone SDK. pub mod pinecone; diff --git a/pinecone_sdk/src/control.rs b/pinecone_sdk/src/pinecone/control.rs similarity index 98% rename from pinecone_sdk/src/control.rs rename to pinecone_sdk/src/pinecone/control.rs index da2ca03..32af66f 100644 --- a/pinecone_sdk/src/control.rs +++ b/pinecone_sdk/src/pinecone/control.rs @@ -45,9 +45,8 @@ impl PineconeClient { /// /// ### Example /// ```no_run - /// use pinecone_sdk::pinecone::PineconeClient; + /// use pinecone_sdk::pinecone::{PineconeClient, control::{Metric, Cloud, WaitPolicy, IndexModel}}; /// use pinecone_sdk::utils::errors::PineconeError; - /// use pinecone_sdk::control::{Metric, Cloud, WaitPolicy, IndexModel}; /// /// # #[tokio::main] /// # async fn main() -> Result<(), PineconeError>{ @@ -123,9 +122,8 @@ impl PineconeClient { /// /// ### Example /// ```no_run - /// use pinecone_sdk::pinecone::PineconeClient; + /// use pinecone_sdk::pinecone::{PineconeClient, control::{Metric, Cloud, WaitPolicy, IndexModel}}; /// use pinecone_sdk::utils::errors::PineconeError; - /// use pinecone_sdk::control::{Metric, Cloud, WaitPolicy, IndexModel}; /// use std::time::Duration; /// /// # #[tokio::main] @@ -263,9 +261,8 @@ impl PineconeClient { /// /// ### Example /// ```no_run - /// use pinecone_sdk::pinecone::PineconeClient; + /// use pinecone_sdk::pinecone::{PineconeClient, control::IndexModel}; /// use pinecone_sdk::utils::errors::PineconeError; - /// use pinecone_sdk::control::IndexModel; /// /// # #[tokio::main] /// # async fn main() -> Result<(), PineconeError>{ @@ -297,9 +294,8 @@ impl PineconeClient { /// /// ### Example /// ```no_run - /// use pinecone_sdk::pinecone::PineconeClient; + /// use pinecone_sdk::pinecone::{PineconeClient, control::IndexList}; /// use pinecone_sdk::utils::errors::PineconeError; - /// use pinecone_sdk::control::IndexList; /// /// # #[tokio::main] /// # async fn main() -> Result<(), PineconeError>{ @@ -412,9 +408,8 @@ impl PineconeClient { /// /// ### Example /// ```no_run - /// use pinecone_sdk::pinecone::PineconeClient; + /// use pinecone_sdk::pinecone::{PineconeClient, control::CollectionModel}; /// use pinecone_sdk::utils::errors::PineconeError; - /// use pinecone_sdk::control::CollectionModel; /// /// # #[tokio::main] /// # async fn main() -> Result<(), PineconeError>{ @@ -868,21 +863,21 @@ mod tests { .header("content-type", "application/json") .body( r#"{ - "dimension": 1536, - "host": "mock-host", - "metric": "cosine", - "name": "serverless-index", - "spec": { - "serverless": { - "cloud": "aws", - "region": "us-east-1" + "dimension": 1536, + "host": "mock-host", + "metric": "cosine", + "name": "serverless-index", + "spec": { + "serverless": { + "cloud": "aws", + "region": "us-east-1" + } + }, + "status": { + "ready": true, + "state": "Ready" } - }, - "status": { - "ready": true, - "state": "Ready" - } - }"#, + }"#, ); }); diff --git a/pinecone_sdk/src/pinecone/data.rs b/pinecone_sdk/src/pinecone/data.rs new file mode 100644 index 0000000..2ddf264 --- /dev/null +++ b/pinecone_sdk/src/pinecone/data.rs @@ -0,0 +1,270 @@ +use crate::pinecone::PineconeClient; +use crate::utils::errors::PineconeError; +use once_cell::sync::Lazy; +use pb::vector_service_client::VectorServiceClient; +use tonic::metadata::{Ascii, MetadataValue as TonicMetadataVal}; +use tonic::service::interceptor::InterceptedService; +use tonic::service::Interceptor; +use tonic::transport::Channel; +use tonic::{Request, Status}; + +pub use pb::{UpsertResponse, Vector}; + +/// Generated protobuf module for data plane. +pub mod pb { + tonic::include_proto!("_"); +} + +#[derive(Debug, Clone)] +struct ApiKeyInterceptor { + api_token: TonicMetadataVal, +} + +impl Interceptor for ApiKeyInterceptor { + fn call(&mut self, mut request: Request<()>) -> Result, Status> { + // TODO: replace `api_token` with an `Option`, and do a proper `if_some`. + if !self.api_token.is_empty() { + request + .metadata_mut() + .insert("api-key", self.api_token.clone()); + } + Ok(request) + } +} + +/// A client for interacting with a Pinecone index. +#[derive(Debug)] +pub struct Index { + /// The name of the index. + host: String, + connection: VectorServiceClient>, +} + +impl Index { + /// The upsert operation writes vectors into a namespace. + /// If a new value is upserted for an existing vector id, it will overwrite the previous value. + /// + /// ### Arguments + /// * `vectors: Vec` - A list of vectors to upsert. + /// + /// ### Return + /// * `Result` - A response object. + /// + /// ### Example + /// ```no_run + /// use pinecone_sdk::pinecone::PineconeClient; + /// use pinecone_sdk::pinecone::data::pb::Vector; + /// # use pinecone_sdk::utils::errors::PineconeError; + /// + /// # #[tokio::main] + /// # async fn main() -> Result<(), PineconeError>{ + /// let pinecone = PineconeClient::new(None, None, None, None).unwrap(); + /// + /// let mut index = pinecone.index("index-host").await.unwrap(); + /// + /// let vectors = vec![Vector { + /// id: "vector-id".to_string(), + /// values: vec![1.0, 2.0, 3.0, 4.0], + /// sparse_values: None, + /// metadata: None, + /// }]; + /// let response = index.upsert(vectors, None).await.unwrap(); + /// # Ok(()) + /// # } + /// ``` + pub async fn upsert( + &mut self, + vectors: Vec, + namespace: Option, + ) -> Result { + let request = pb::UpsertRequest { + vectors, + namespace: namespace.unwrap_or_default(), + }; + + let response = self + .connection + .upsert(request) + .await + .map_err(|e| PineconeError::UpsertError { inner: Box::new(e) })? + .into_inner(); + + Ok(response) + } +} + +impl PineconeClient { + /// Match the scheme in a host string. + /// + /// ### Arguments + /// * `host: &str` - The host string to match. + /// + /// ### Return + /// * `bool` - True if the host string contains a scheme, false otherwise. + fn has_scheme(host: &str) -> bool { + static RE: Lazy = Lazy::new(|| regex::Regex::new(r"^[a-zA-Z]+://").unwrap()); + RE.is_match(host) + } + + /// Match the port in a host string. + /// + /// ### Arguments + /// * `host: &str` - The host string to match. + /// + /// ### Return + /// * `bool` - True if the host string contains a port, false otherwise. + fn has_port(host: &str) -> bool { + static RE: Lazy = Lazy::new(|| regex::Regex::new(r":\d+$").unwrap()); + RE.is_match(host) + } + + /// Target an index for data operations. + /// + /// ### Arguments + /// * `host: &str` - The host of the index to target. If the host does not contain a scheme, it will default to `https://`. If the host does not contain a port, it will default to `443`. + /// + /// ### Return + /// * `Result` - A Pinecone index object. + /// + /// ### Example + /// + /// ```no_run + /// use pinecone_sdk::pinecone::PineconeClient; + /// # use pinecone_sdk::utils::errors::PineconeError; + /// + /// # #[tokio::main] + /// # async fn main() -> Result<(), PineconeError>{ + /// let pinecone = PineconeClient::new(None, None, None, None).unwrap(); + /// + /// let index = pinecone.index("index-host").await.unwrap(); + /// # Ok(()) + /// # } + /// ``` + pub async fn index(&self, host: &str) -> Result { + let endpoint = host.to_string(); + + let endpoint = if PineconeClient::has_scheme(&endpoint) { + endpoint + } else { + format!("https://{}", endpoint) + }; + + let endpoint = if PineconeClient::has_port(&endpoint) { + endpoint + } else { + format!("{}:443", endpoint) + }; + + let index = Index { + host: endpoint.clone(), + connection: self.new_index_connection(endpoint).await?, + }; + + Ok(index) + } + + async fn new_index_connection( + &self, + host: String, + ) -> Result>, PineconeError> + { + let tls_config = tonic::transport::ClientTlsConfig::default(); + + // connect to server + let endpoint = Channel::from_shared(host) + .map_err(|e| PineconeError::ConnectionError { inner: Box::new(e) })? + .tls_config(tls_config) + .map_err(|e| PineconeError::ConnectionError { inner: Box::new(e) })?; + + let channel = endpoint + .connect() + .await + .map_err(|e| PineconeError::ConnectionError { inner: Box::new(e) })?; + + // add api key in metadata through interceptor + let token: TonicMetadataVal<_> = self.api_key.parse().unwrap(); + let add_api_key_interceptor = ApiKeyInterceptor { api_token: token }; + let inner = VectorServiceClient::with_interceptor(channel, add_api_key_interceptor); + + Ok(inner) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use httpmock::prelude::*; + + #[tokio::test] + async fn test_index_full_endpoint() { + let server = MockServer::start(); + + // server url contains scheme and port + let _mock = server.mock(|_when, then| { + then.status(200); + }); + + let pinecone = PineconeClient::new(None, None, None, None).unwrap(); + + let index = pinecone.index(server.base_url().as_str()).await.unwrap(); + + assert_eq!(index.host, server.base_url()); + } + + #[tokio::test] + async fn test_index_no_scheme() { + let server = MockServer::start(); + + // server url contains no scheme + let _mock = server.mock(|_when, then| { + then.status(200); + }); + + let pinecone = PineconeClient::new(None, None, None, None).unwrap(); + + let addr = server.address().to_string(); + + let _index = pinecone + .index(addr.as_str()) + .await + .expect_err("Expected connection error"); + } + + #[tokio::test] + async fn test_index_no_port() { + let server = MockServer::start(); + + // server url contains no port + let _mock = server.mock(|_when, then| { + then.status(200); + }); + + let pinecone = PineconeClient::new(None, None, None, None).unwrap(); + + let scheme_host = format!("http://{}", server.host()); + + let _index = pinecone + .index(scheme_host.as_str()) + .await + .expect_err("Expected connection error"); + } + + #[tokio::test] + async fn test_index_no_scheme_no_port() { + let server = MockServer::start(); + + // server url contains no scheme and no port + let _mock = server.mock(|_when, then| { + then.status(200); + }); + + let pinecone = PineconeClient::new(None, None, None, None).unwrap(); + + let host = server.host(); + + let _index = pinecone + .index(host.as_str()) + .await + .expect_err("Expected connection error"); + } +} diff --git a/pinecone_sdk/src/pinecone.rs b/pinecone_sdk/src/pinecone/mod.rs similarity index 99% rename from pinecone_sdk/src/pinecone.rs rename to pinecone_sdk/src/pinecone/mod.rs index 5ff6b5f..826d261 100644 --- a/pinecone_sdk/src/pinecone.rs +++ b/pinecone_sdk/src/pinecone/mod.rs @@ -6,6 +6,12 @@ use openapi::apis::configuration::Configuration; use serde_json; use std::collections::HashMap; +/// Control plane module. +pub mod control; + +/// Data plane module. +pub mod data; + /// The `PineconeClient` struct is the main entry point for interacting with Pinecone via this Rust SDK. #[derive(Debug, Clone)] pub struct PineconeClient { diff --git a/pinecone_sdk/src/utils/errors.rs b/pinecone_sdk/src/utils/errors.rs index 7be1d16..6d4bd3e 100644 --- a/pinecone_sdk/src/utils/errors.rs +++ b/pinecone_sdk/src/utils/errors.rs @@ -22,6 +22,12 @@ pub enum PineconeError { message: String, }, + /// ConnectionError: Failed to establish a connection. + ConnectionError { + /// inner: Error object for connection error. + inner: Box, + }, + /// ReqwestError: Error caused by Reqwest ReqwestError { /// HTTP status code. @@ -145,6 +151,12 @@ pub enum PineconeError { /// Error message. message: String, }, + + /// UpsertError: Failed to upsert data. + UpsertError { + /// inner: Error object for tonic error. + inner: Box, + }, } // Implement the conversion from OpenApiError to PineconeError for CreateIndexError. diff --git a/pinecone_sdk/tests/integration_test.rs b/pinecone_sdk/tests/integration_test.rs index 23a23fd..f99bb76 100644 --- a/pinecone_sdk/tests/integration_test.rs +++ b/pinecone_sdk/tests/integration_test.rs @@ -1,10 +1,11 @@ -use std::time::Duration; - use openapi::models::index_model::Metric as OpenApiMetric; use openapi::models::serverless_spec::Cloud as OpenApiCloud; -use pinecone_sdk::control::{Cloud, Metric, WaitPolicy}; +use pinecone_sdk::pinecone::control::{Cloud, Metric, WaitPolicy}; +use pinecone_sdk::pinecone::data::Vector; use pinecone_sdk::pinecone::PineconeClient; use pinecone_sdk::utils::errors::PineconeError; +use std::time::Duration; +use std::vec; // helpers to generate random test/collection names fn generate_random_string() -> String { @@ -433,3 +434,32 @@ async fn test_delete_collection_err() -> Result<(), PineconeError> { Ok(()) } + +#[tokio::test] +async fn test_upsert() -> Result<(), PineconeError> { + let pinecone = PineconeClient::new(None, None, None, None).unwrap(); + + let host = pinecone + .describe_index(&get_serverless_index()) + .await + .unwrap() + .host; + + let mut index = pinecone + .index(host.as_str()) + .await + .expect("Failed to target index"); + + let vectors = vec![Vector { + id: "1".to_string(), + values: vec![1.0, 2.0, 3.0, 5.5], + sparse_values: None, + metadata: None, + }]; + + let upsert_response = index.upsert(vectors, None).await.expect("Failed to upsert"); + + assert_eq!(upsert_response.upserted_count, 1); + + Ok(()) +} diff --git a/protos/data_2024-07.proto b/protos/data_2024-07.proto new file mode 100644 index 0000000..ab6c360 --- /dev/null +++ b/protos/data_2024-07.proto @@ -0,0 +1,404 @@ +syntax = "proto3"; + +option go_package = "github.com/pinecone-io/go-pinecone/internal/gen/data"; +option java_package = "io.pinecone.proto"; +option java_multiple_files = true; + +import "google/protobuf/struct.proto"; +import "google/api/annotations.proto"; +import "google/api/field_behavior.proto"; + +message SparseValues { + repeated uint32 indices = 1 [ + (google.api.field_behavior) = REQUIRED + ]; + repeated float values = 2 [ + (google.api.field_behavior) = REQUIRED + ]; +} + +message Vector { + + // This is the vector's unique id. + string id = 1 [ + (google.api.field_behavior) = REQUIRED + ]; + + // This is the vector data included in the request. + repeated float values = 2 [ + (google.api.field_behavior) = REQUIRED + ]; + + SparseValues sparse_values = 4; + + // This is the metadata included in the request. + google.protobuf.Struct metadata = 3; +} + +message ScoredVector { + + // This is the vector's unique id. + string id = 1 [ + (google.api.field_behavior) = REQUIRED + ]; + + // This is a measure of similarity between this vector and the query vector. The higher the score, the more they are similar. + float score = 2; + + // This is the vector data, if it is requested. + repeated float values = 3; + + // This is the sparse data, if it is requested. + SparseValues sparse_values = 5; + + // This is the metadata, if it is requested. + google.protobuf.Struct metadata = 4; +} + +// This is a container to hold mutating vector requests. This is not actually used +// in any public APIs. +message RequestUnion { + oneof RequestUnionInner { + UpsertRequest upsert = 1; + DeleteRequest delete = 2; + UpdateRequest update = 3; + } +} + +// The request for the `upsert` operation. +message UpsertRequest { + // An array containing the vectors to upsert. Recommended batch limit is 100 vectors. + repeated Vector vectors = 1 [ + (google.api.field_behavior) = REQUIRED + ]; + + // The namespace where you upsert vectors. + string namespace = 2; +} + +// The response for the `upsert` operation. +message UpsertResponse { + // The number of vectors upserted. + uint32 upserted_count = 1; +} + +// The request for the `Delete` operation. +message DeleteRequest { + // Vectors to delete. + repeated string ids = 1; + + // This indicates that all vectors in the index namespace should be deleted. + bool delete_all = 2; + + // The namespace to delete vectors from, if applicable. + string namespace = 3; + + // If specified, the metadata filter here will be used to select the vectors to delete. This is mutually exclusive + // with specifying ids to delete in the ids param or using `delete_all=True`. + // For guidance and examples, see [Filter with metadata](https://docs.pinecone.io/guides/data/filter-with-metadata). + // Serverless indexes do not support delete by metadata. Instead, you can use the `list` operation to fetch the vector IDs based on their common ID prefix and then delete the records by ID. + google.protobuf.Struct filter = 4; +} + +// The response for the `Delete` operation. +message DeleteResponse {} + +// The request for the `fetch` operation. +message FetchRequest { + // The vector IDs to fetch. Does not accept values containing spaces. + repeated string ids = 1 [ + (google.api.field_behavior) = REQUIRED + ]; + + string namespace = 2; +} + +// The response for the `fetch` operation. +message FetchResponse { + // The fetched vectors, in the form of a map between the fetched ids and the fetched vectors + map vectors = 1; + + // The namespace of the vectors. + string namespace = 2; + + // The usage for this operation. + optional Usage usage = 3; +} + +// The request for the `List` operation. +message ListRequest { + // The vector IDs to fetch. Does not accept values containing spaces. + optional string prefix = 1; + + // Max number of ids to return + optional uint32 limit = 2; + // Pagination token to continue a previous listing operation + optional string pagination_token = 3; + string namespace = 4; +} + +message Pagination { + string next = 1; +} + +message ListItem { + string id = 1; +} + +// The response for the `List` operation. +message ListResponse { + // A list of ids + repeated ListItem vectors = 1; + + // Pagination token to continue past this listing + optional Pagination pagination = 2; + + // The namespace of the vectors. + string namespace = 3; + + // The usage for this operation. + optional Usage usage = 4; +} + +// A single query vector within a `QueryRequest`. +message QueryVector { + + // The query vector values. This should be the same length as the dimension of the index being queried. + repeated float values = 1 [ + (google.api.field_behavior) = REQUIRED + ]; + + // The query sparse values. + SparseValues sparse_values = 5; + + // An override for the number of results to return for this query vector. + uint32 top_k = 2; + + // An override the namespace to search. + string namespace = 3; + + // An override for the metadata filter to apply. This replaces the request-level filter. + google.protobuf.Struct filter = 4; +} + +// The request for the `query` operation. +message QueryRequest { + // The namespace to query. + string namespace = 1; + + // The number of results to return for each query. + uint32 top_k = 2 [ + (google.api.field_behavior) = REQUIRED + ]; + + // The filter to apply. You can use vector metadata to limit your search. See [Filter with metadata](https://docs.pinecone.io/guides/data/filter-with-metadata). + google.protobuf.Struct filter = 3; + + // Indicates whether vector values are included in the response. + bool include_values = 4; + + // Indicates whether metadata is included in the response as well as the ids. + bool include_metadata = 5; + + // DEPRECATED. The query vectors. Each `query()` request can contain only one of the parameters `queries`, `vector`, or `id`. + repeated QueryVector queries = 6 [ + deprecated = true + ]; + + + // The query vector. This should be the same length as the dimension of the index being queried. Each `query()` request can contain only one of the parameters `id` or `vector`. + repeated float vector = 7; + + // The query sparse values. + SparseValues sparse_vector = 9; + + // The unique ID of the vector to be used as a query vector. Each `query()` request can contain only one of the parameters `queries`, `vector`, or `id`. + string id = 8; +} + +// The query results for a single `QueryVector` +message SingleQueryResults { + // The matches for the vectors. + repeated ScoredVector matches = 1; + + // The namespace for the vectors. + string namespace = 2; +} + +// The response for the `query` operation. These are the matches found for a particular query vector. The matches are ordered from most similar to least similar. +message QueryResponse { + // DEPRECATED. The results of each query. The order is the same as `QueryRequest.queries`. + repeated SingleQueryResults results = 1 [deprecated=true]; + + // The matches for the vectors. + repeated ScoredVector matches = 2; + + // The namespace for the vectors. + string namespace = 3; + + // The usage for this operation. + optional Usage usage = 4; +} + +message Usage { + // The number of read units consumed by this operation. + optional uint32 read_units = 1; +} + +// The request for the `update` operation. +message UpdateRequest { + // Vector's unique id. + string id = 1 [ + (google.api.field_behavior) = REQUIRED + ]; + + // Vector data. + repeated float values = 2; + + SparseValues sparse_values = 5; + + // Metadata to set for the vector. + google.protobuf.Struct set_metadata = 3; + + // The namespace containing the vector to update. + string namespace = 4; +} + +// The response for the `update` operation. +message UpdateResponse {} + +// The request for the `describe_index_stats` operation. +message DescribeIndexStatsRequest { + + + // If this parameter is present, the operation only returns statistics + // for vectors that satisfy the filter. + // See https://docs.pinecone.io/guides/data/filtering-with-metadata. + google.protobuf.Struct filter = 1; +} + +// A summary of the contents of a namespace. +message NamespaceSummary { + // The number of vectors stored in this namespace. Note that updates to this field may lag behind updates to the + // underlying index and corresponding query results, etc. + uint32 vector_count = 1; +} + +// The response for the `describe_index_stats` operation. +message DescribeIndexStatsResponse { + // A mapping for each namespace in the index from the namespace name to a + // summary of its contents. If a metadata filter expression is present, the + // summary will reflect only vectors matching that expression. + map namespaces = 1; + + // The dimension of the indexed vectors. + uint32 dimension = 2; + + // The fullness of the index, regardless of whether a metadata filter expression was passed. The granularity of this metric is 10%. + // + // Serverless indexes scale automatically as needed, so index fullness is relevant only for pod-based indexes. + // + // The index fullness result may be inaccurate during pod resizing; to get the status of a pod resizing process, use [`describe_index`](https://docs.pinecone.io/reference/api/control-plane/describe_index). + float index_fullness = 3; + + // The total number of vectors in the index, regardless of whether a metadata filter expression was passed + uint32 total_vector_count = 4; +} + +// The `VectorService` interface is exposed by Pinecone's vector index services. +// This service could also be called a `gRPC` service or a `REST`-like api. +service VectorService { + // Upsert vectors + // + // The `upsert` operation writes vectors into a namespace. If a new value is upserted for an existing vector ID, it will overwrite the previous value. + // + // For guidance and examples, see [Upsert data](https://docs.pinecone.io/guides/data/upsert-data). + rpc Upsert(UpsertRequest) returns (UpsertResponse) { + option (google.api.http) = { + post: "/vectors/upsert" + body: "*" + }; + } + + // Delete vectors + // + // The `delete` operation deletes vectors, by id, from a single namespace. + // + // For guidance and examples, see [Delete data](https://docs.pinecone.io/guides/data/delete-data). + rpc Delete(DeleteRequest) returns (DeleteResponse) { + option (google.api.http) = { + post: "/vectors/delete" + body: "*" + additional_bindings { + delete: "/vectors/delete" + } + }; + } + + // Fetch vectors + // + // The `fetch` operation looks up and returns vectors, by ID, from a single namespace. The returned vectors include the vector data and/or metadata. + // + // For guidance and examples, see [Fetch data](https://docs.pinecone.io/guides/data/fetch-data). + rpc Fetch(FetchRequest) returns (FetchResponse) { + option (google.api.http) = { + get: "/vectors/fetch" + }; + } + + // List vector IDs + // + // The `list` operation lists the IDs of vectors in a single namespace of a serverless index. An optional prefix can be passed to limit the results to IDs with a common prefix. + // + // `list` returns up to 100 IDs at a time by default in sorted order (bitwise/"C" collation). If the `limit` parameter is set, `list` returns up to that number of IDs instead. Whenever there are additional IDs to return, the response also includes a `pagination_token` that you can use to get the next batch of IDs. When the response does not include a `pagination_token`, there are no more IDs to return. + // + // For guidance and examples, see [List record IDs](https://docs.pinecone.io/guides/data/list-record-ids). + // + // **Note:** `list` is supported only for serverless indexes. + rpc List(ListRequest) returns (ListResponse) { + option (google.api.http) = { + get: "/vectors/list" + }; + } + + // Query vectors + // + // The `query` operation searches a namespace, using a query vector. It retrieves the ids of the most similar items in a namespace, along with their similarity scores. + // + // For guidance and examples, see [Query data](https://docs.pinecone.io/guides/data/query-data). + rpc Query(QueryRequest) returns (QueryResponse) { + option (google.api.http) = { + post: "/query" + body: "*" + }; + } + + // Update a vector + // + // The `update` operation updates a vector in a namespace. If a value is included, it will overwrite the previous value. If a `set_metadata` is included, the values of the fields specified in it will be added or overwrite the previous value. + // + // For guidance and examples, see [Update data](https://docs.pinecone.io/guides/data/update-data). + rpc Update(UpdateRequest) returns (UpdateResponse) { + option (google.api.http) = { + post: "/vectors/update" + body: "*" + }; + } + + // Get index stats + // + // The `describe_index_stats` operation returns statistics about the contents of an index, including the vector count per namespace, the number of dimensions, and the index fullness. + // + // Serverless indexes scale automatically as needed, so index fullness is relevant only for pod-based indexes. + // + // For pod-based indexes, the index fullness result may be inaccurate during pod resizing; to get the status of a pod resizing process, use [`describe_index`](https://docs.pinecone.io/reference/api/control-plane/describe_index). + rpc DescribeIndexStats(DescribeIndexStatsRequest) returns (DescribeIndexStatsResponse) { + option (google.api.http) = { + post: "/describe_index_stats" + body: "*" + additional_bindings { + get: "/describe_index_stats" + } + }; + } +} diff --git a/protos/google/api/annotations.proto b/protos/google/api/annotations.proto new file mode 100644 index 0000000..b17b345 --- /dev/null +++ b/protos/google/api/annotations.proto @@ -0,0 +1,31 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +import "google/api/http.proto"; +import "google/protobuf/descriptor.proto"; + +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "AnnotationsProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +extend google.protobuf.MethodOptions { + // See `HttpRule`. + HttpRule http = 72295728; +} \ No newline at end of file diff --git a/protos/google/api/field_behavior.proto b/protos/google/api/field_behavior.proto new file mode 100644 index 0000000..2b73d36 --- /dev/null +++ b/protos/google/api/field_behavior.proto @@ -0,0 +1,104 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +import "google/protobuf/descriptor.proto"; + +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "FieldBehaviorProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +extend google.protobuf.FieldOptions { + // A designation of a specific field behavior (required, output only, etc.) + // in protobuf messages. + // + // Examples: + // + // string name = 1 [(google.api.field_behavior) = REQUIRED]; + // State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + // google.protobuf.Duration ttl = 1 + // [(google.api.field_behavior) = INPUT_ONLY]; + // google.protobuf.Timestamp expire_time = 1 + // [(google.api.field_behavior) = OUTPUT_ONLY, + // (google.api.field_behavior) = IMMUTABLE]; + repeated google.api.FieldBehavior field_behavior = 1052 [packed = false]; +} + +// An indicator of the behavior of a given field (for example, that a field +// is required in requests, or given as output but ignored as input). +// This **does not** change the behavior in protocol buffers itself; it only +// denotes the behavior and may affect how API tooling handles the field. +// +// Note: This enum **may** receive new values in the future. +enum FieldBehavior { + // Conventional default for enums. Do not use this. + FIELD_BEHAVIOR_UNSPECIFIED = 0; + + // Specifically denotes a field as optional. + // While all fields in protocol buffers are optional, this may be specified + // for emphasis if appropriate. + OPTIONAL = 1; + + // Denotes a field as required. + // This indicates that the field **must** be provided as part of the request, + // and failure to do so will cause an error (usually `INVALID_ARGUMENT`). + REQUIRED = 2; + + // Denotes a field as output only. + // This indicates that the field is provided in responses, but including the + // field in a request does nothing (the server *must* ignore it and + // *must not* throw an error as a result of the field's presence). + OUTPUT_ONLY = 3; + + // Denotes a field as input only. + // This indicates that the field is provided in requests, and the + // corresponding field is not included in output. + INPUT_ONLY = 4; + + // Denotes a field as immutable. + // This indicates that the field may be set once in a request to create a + // resource, but may not be changed thereafter. + IMMUTABLE = 5; + + // Denotes that a (repeated) field is an unordered list. + // This indicates that the service may provide the elements of the list + // in any arbitrary order, rather than the order the user originally + // provided. Additionally, the list's order may or may not be stable. + UNORDERED_LIST = 6; + + // Denotes that this field returns a non-empty default value if not set. + // This indicates that if the user provides the empty value in a request, + // a non-empty value will be returned. The user will not be aware of what + // non-empty value to expect. + NON_EMPTY_DEFAULT = 7; + + // Denotes that the field in a resource (a message annotated with + // google.api.resource) is used in the resource name to uniquely identify the + // resource. For AIP-compliant APIs, this should only be applied to the + // `name` field on the resource. + // + // This behavior should not be applied to references to other resources within + // the message. + // + // The identifier field of resources often have different field behavior + // depending on the request it is embedded in (e.g. for Create methods name + // is optional and unused, while for Update methods it is required). Instead + // of method-specific annotations, only `IDENTIFIER` is required. + IDENTIFIER = 8; +} \ No newline at end of file diff --git a/protos/google/api/http.proto b/protos/google/api/http.proto new file mode 100644 index 0000000..cc6ba3c --- /dev/null +++ b/protos/google/api/http.proto @@ -0,0 +1,379 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "HttpProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +// Defines the HTTP configuration for an API service. It contains a list of +// [HttpRule][google.api.HttpRule], each specifying the mapping of an RPC method +// to one or more HTTP REST API methods. +message Http { + // A list of HTTP configuration rules that apply to individual API methods. + // + // **NOTE:** All service configuration rules follow "last one wins" order. + repeated HttpRule rules = 1; + + // When set to true, URL path parameters will be fully URI-decoded except in + // cases of single segment matches in reserved expansion, where "%2F" will be + // left encoded. + // + // The default behavior is to not decode RFC 6570 reserved characters in multi + // segment matches. + bool fully_decode_reserved_expansion = 2; +} + +// # gRPC Transcoding +// +// gRPC Transcoding is a feature for mapping between a gRPC method and one or +// more HTTP REST endpoints. It allows developers to build a single API service +// that supports both gRPC APIs and REST APIs. Many systems, including [Google +// APIs](https://github.com/googleapis/googleapis), +// [Cloud Endpoints](https://cloud.google.com/endpoints), [gRPC +// Gateway](https://github.com/grpc-ecosystem/grpc-gateway), +// and [Envoy](https://github.com/envoyproxy/envoy) proxy support this feature +// and use it for large scale production services. +// +// `HttpRule` defines the schema of the gRPC/REST mapping. The mapping specifies +// how different portions of the gRPC request message are mapped to the URL +// path, URL query parameters, and HTTP request body. It also controls how the +// gRPC response message is mapped to the HTTP response body. `HttpRule` is +// typically specified as an `google.api.http` annotation on the gRPC method. +// +// Each mapping specifies a URL path template and an HTTP method. The path +// template may refer to one or more fields in the gRPC request message, as long +// as each field is a non-repeated field with a primitive (non-message) type. +// The path template controls how fields of the request message are mapped to +// the URL path. +// +// Example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get: "/v1/{name=messages/*}" +// }; +// } +// } +// message GetMessageRequest { +// string name = 1; // Mapped to URL path. +// } +// message Message { +// string text = 1; // The resource content. +// } +// +// This enables an HTTP REST to gRPC mapping as below: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456` | `GetMessage(name: "messages/123456")` +// +// Any fields in the request message which are not bound by the path template +// automatically become HTTP query parameters if there is no HTTP request body. +// For example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get:"/v1/messages/{message_id}" +// }; +// } +// } +// message GetMessageRequest { +// message SubMessage { +// string subfield = 1; +// } +// string message_id = 1; // Mapped to URL path. +// int64 revision = 2; // Mapped to URL query parameter `revision`. +// SubMessage sub = 3; // Mapped to URL query parameter `sub.subfield`. +// } +// +// This enables a HTTP JSON to RPC mapping as below: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456?revision=2&sub.subfield=foo` | +// `GetMessage(message_id: "123456" revision: 2 sub: SubMessage(subfield: +// "foo"))` +// +// Note that fields which are mapped to URL query parameters must have a +// primitive type or a repeated primitive type or a non-repeated message type. +// In the case of a repeated type, the parameter can be repeated in the URL +// as `...?param=A¶m=B`. In the case of a message type, each field of the +// message is mapped to a separate parameter, such as +// `...?foo.a=A&foo.b=B&foo.c=C`. +// +// For HTTP methods that allow a request body, the `body` field +// specifies the mapping. Consider a REST update method on the +// message resource collection: +// +// service Messaging { +// rpc UpdateMessage(UpdateMessageRequest) returns (Message) { +// option (google.api.http) = { +// patch: "/v1/messages/{message_id}" +// body: "message" +// }; +// } +// } +// message UpdateMessageRequest { +// string message_id = 1; // mapped to the URL +// Message message = 2; // mapped to the body +// } +// +// The following HTTP JSON to RPC mapping is enabled, where the +// representation of the JSON in the request body is determined by +// protos JSON encoding: +// +// HTTP | gRPC +// -----|----- +// `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: +// "123456" message { text: "Hi!" })` +// +// The special name `*` can be used in the body mapping to define that +// every field not bound by the path template should be mapped to the +// request body. This enables the following alternative definition of +// the update method: +// +// service Messaging { +// rpc UpdateMessage(Message) returns (Message) { +// option (google.api.http) = { +// patch: "/v1/messages/{message_id}" +// body: "*" +// }; +// } +// } +// message Message { +// string message_id = 1; +// string text = 2; +// } +// +// +// The following HTTP JSON to RPC mapping is enabled: +// +// HTTP | gRPC +// -----|----- +// `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: +// "123456" text: "Hi!")` +// +// Note that when using `*` in the body mapping, it is not possible to +// have HTTP parameters, as all fields not bound by the path end in +// the body. This makes this option more rarely used in practice when +// defining REST APIs. The common usage of `*` is in custom methods +// which don't use the URL at all for transferring data. +// +// It is possible to define multiple HTTP methods for one RPC by using +// the `additional_bindings` option. Example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get: "/v1/messages/{message_id}" +// additional_bindings { +// get: "/v1/users/{user_id}/messages/{message_id}" +// } +// }; +// } +// } +// message GetMessageRequest { +// string message_id = 1; +// string user_id = 2; +// } +// +// This enables the following two alternative HTTP JSON to RPC mappings: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456` | `GetMessage(message_id: "123456")` +// `GET /v1/users/me/messages/123456` | `GetMessage(user_id: "me" message_id: +// "123456")` +// +// ## Rules for HTTP mapping +// +// 1. Leaf request fields (recursive expansion nested messages in the request +// message) are classified into three categories: +// - Fields referred by the path template. They are passed via the URL path. +// - Fields referred by the [HttpRule.body][google.api.HttpRule.body]. They +// are passed via the HTTP +// request body. +// - All other fields are passed via the URL query parameters, and the +// parameter name is the field path in the request message. A repeated +// field can be represented as multiple query parameters under the same +// name. +// 2. If [HttpRule.body][google.api.HttpRule.body] is "*", there is no URL +// query parameter, all fields +// are passed via URL path and HTTP request body. +// 3. If [HttpRule.body][google.api.HttpRule.body] is omitted, there is no HTTP +// request body, all +// fields are passed via URL path and URL query parameters. +// +// ### Path template syntax +// +// Template = "/" Segments [ Verb ] ; +// Segments = Segment { "/" Segment } ; +// Segment = "*" | "**" | LITERAL | Variable ; +// Variable = "{" FieldPath [ "=" Segments ] "}" ; +// FieldPath = IDENT { "." IDENT } ; +// Verb = ":" LITERAL ; +// +// The syntax `*` matches a single URL path segment. The syntax `**` matches +// zero or more URL path segments, which must be the last part of the URL path +// except the `Verb`. +// +// The syntax `Variable` matches part of the URL path as specified by its +// template. A variable template must not contain other variables. If a variable +// matches a single path segment, its template may be omitted, e.g. `{var}` +// is equivalent to `{var=*}`. +// +// The syntax `LITERAL` matches literal text in the URL path. If the `LITERAL` +// contains any reserved character, such characters should be percent-encoded +// before the matching. +// +// If a variable contains exactly one path segment, such as `"{var}"` or +// `"{var=*}"`, when such a variable is expanded into a URL path on the client +// side, all characters except `[-_.~0-9a-zA-Z]` are percent-encoded. The +// server side does the reverse decoding. Such variables show up in the +// [Discovery +// Document](https://developers.google.com/discovery/v1/reference/apis) as +// `{var}`. +// +// If a variable contains multiple path segments, such as `"{var=foo/*}"` +// or `"{var=**}"`, when such a variable is expanded into a URL path on the +// client side, all characters except `[-_.~/0-9a-zA-Z]` are percent-encoded. +// The server side does the reverse decoding, except "%2F" and "%2f" are left +// unchanged. Such variables show up in the +// [Discovery +// Document](https://developers.google.com/discovery/v1/reference/apis) as +// `{+var}`. +// +// ## Using gRPC API Service Configuration +// +// gRPC API Service Configuration (service config) is a configuration language +// for configuring a gRPC service to become a user-facing product. The +// service config is simply the YAML representation of the `google.api.Service` +// proto message. +// +// As an alternative to annotating your proto file, you can configure gRPC +// transcoding in your service config YAML files. You do this by specifying a +// `HttpRule` that maps the gRPC method to a REST endpoint, achieving the same +// effect as the proto annotation. This can be particularly useful if you +// have a proto that is reused in multiple services. Note that any transcoding +// specified in the service config will override any matching transcoding +// configuration in the proto. +// +// Example: +// +// http: +// rules: +// # Selects a gRPC method and applies HttpRule to it. +// - selector: example.v1.Messaging.GetMessage +// get: /v1/messages/{message_id}/{sub.subfield} +// +// ## Special notes +// +// When gRPC Transcoding is used to map a gRPC to JSON REST endpoints, the +// proto to JSON conversion must follow the [proto3 +// specification](https://developers.google.com/protocol-buffers/docs/proto3#json). +// +// While the single segment variable follows the semantics of +// [RFC 6570](https://tools.ietf.org/html/rfc6570) Section 3.2.2 Simple String +// Expansion, the multi segment variable **does not** follow RFC 6570 Section +// 3.2.3 Reserved Expansion. The reason is that the Reserved Expansion +// does not expand special characters like `?` and `#`, which would lead +// to invalid URLs. As the result, gRPC Transcoding uses a custom encoding +// for multi segment variables. +// +// The path variables **must not** refer to any repeated or mapped field, +// because client libraries are not capable of handling such variable expansion. +// +// The path variables **must not** capture the leading "/" character. The reason +// is that the most common use case "{var}" does not capture the leading "/" +// character. For consistency, all path variables must share the same behavior. +// +// Repeated message fields must not be mapped to URL query parameters, because +// no client library can support such complicated mapping. +// +// If an API needs to use a JSON array for request or response body, it can map +// the request or response body to a repeated field. However, some gRPC +// Transcoding implementations may not support this feature. +message HttpRule { + // Selects a method to which this rule applies. + // + // Refer to [selector][google.api.DocumentationRule.selector] for syntax + // details. + string selector = 1; + + // Determines the URL pattern is matched by this rules. This pattern can be + // used with any of the {get|put|post|delete|patch} methods. A custom method + // can be defined using the 'custom' field. + oneof pattern { + // Maps to HTTP GET. Used for listing and getting information about + // resources. + string get = 2; + + // Maps to HTTP PUT. Used for replacing a resource. + string put = 3; + + // Maps to HTTP POST. Used for creating a resource or performing an action. + string post = 4; + + // Maps to HTTP DELETE. Used for deleting a resource. + string delete = 5; + + // Maps to HTTP PATCH. Used for updating a resource. + string patch = 6; + + // The custom pattern is used for specifying an HTTP method that is not + // included in the `pattern` field, such as HEAD, or "*" to leave the + // HTTP method unspecified for this rule. The wild-card rule is useful + // for services that provide content to Web (HTML) clients. + CustomHttpPattern custom = 8; + } + + // The name of the request field whose value is mapped to the HTTP request + // body, or `*` for mapping all request fields not captured by the path + // pattern to the HTTP body, or omitted for not having any HTTP request body. + // + // NOTE: the referred field must be present at the top-level of the request + // message type. + string body = 7; + + // Optional. The name of the response field whose value is mapped to the HTTP + // response body. When omitted, the entire response message will be used + // as the HTTP response body. + // + // NOTE: The referred field must be present at the top-level of the response + // message type. + string response_body = 12; + + // Additional HTTP bindings for the selector. Nested bindings must + // not contain an `additional_bindings` field themselves (that is, + // the nesting may only be one level deep). + repeated HttpRule additional_bindings = 11; +} + +// A custom pattern is used for defining custom HTTP verb. +message CustomHttpPattern { + // The name of this custom HTTP verb. + string kind = 1; + + // The path matched by this custom verb. + string path = 2; +} \ No newline at end of file