Merge branch 'main' into jdahlgren/get-snapshot-status-missing-stats-for-restarted-nodes

Author: JeremyDahlgren

.github/validate-pr/index.js

@@ -141,12 +141,10 @@ async function run() {
     }
     comment += `\nYou can validate ${table.length === 1 ? 'this' : 'these'} API${table.length === 1 ? '' : 's'} yourself by using the ${tick}make validate${tick} target.\n`
 
-    await octokit.rest.issues.createComment({
-      owner: 'elastic',
-      repo: 'elasticsearch-specification',
-      issue_number: context.payload.pull_request.number,
-      body: comment
-    })
+    core.setOutput('has_results', 'true')
+    core.setOutput('comment_body', comment)
+  } else {
+    core.setOutput('has_results', 'false')
   }
 
   core.info('Done!')

.github/workflows/update-rest-api-json.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        branch: ['main', '9.0', '8.19', '8.18', '8.17', '8.16', '7.17']
+        branch: ['main', '9.0', '8.19', '8.18', '8.17']
 
     steps:
       - uses: actions/checkout@v4
@@ -28,8 +28,7 @@ jobs:
       - name: Install deps
         run: |
           npm install --prefix .github/download-artifacts
-          npm install --prefix compiler
-          npm install --prefix typescript-generator
+          make setup
 
       - name: Download artifacts
         run: |
@@ -53,12 +52,3 @@ jobs:
           delete-branch: true
           reviewers: Anaethelion,ezimuel,flobernd,JoshMock,l-trotta,miguelgrinberg,picandocodigo,pquentin,swallez,technige
           branch: automated/rest-api-spec-update-${{ matrix.branch }}
-
-      - name: Open an issue if the action fails
-        if: ${{ failure() }}
-        uses: nashmaniac/[email protected]
-        with:
-          title: rest-api-spec update failed
-          token: ${{ secrets.GITHUB_TOKEN }}
-          labels: bug
-          body: The rest-api-spec action is currently failing, see [here](https://github.com/elastic/elasticsearch-specification/actions/workflows/update-rest-api-json.yml).

.github/workflows/validate-pr.yml

@@ -60,18 +60,26 @@ jobs:
           fi
           node scripts/upload-recording/download.js --branch $branch --git
           node scripts/clone-elasticsearch/index.js --branch $branch
-        env:
-          GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }}
-
-      - name: Remove previous comment
-        uses: maheshrayas/action-pr-comment-delete@v1
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          org: elastic
-          repo: elasticsearch-specification
-          user: github-actions[bot]
-          issue: ${{ github.event.number }}
 
       - name: Run validation
+        id: validation
         working-directory: ./elasticsearch-specification
         run: node .github/validate-pr --token ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Find existing comment
+        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
+        id: find-comment
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: 'Following you can find the validation results'
+
+      - name: Create or update comment
+        if: steps.validation.outputs.has_results == 'true'
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          comment-id: ${{ steps.find-comment.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body: ${{ steps.validation.outputs.comment_body }}
+          edit-mode: replace

.gitignore

@@ -69,4 +69,6 @@ compiler/test/**/output/
 output/openapi/elasticsearch-serverless-openapi.tmp*.json
 output/openapi/elasticsearch-serverless-openapi.examples.json
 output/openapi/elasticsearch-openapi.tmp*.json
-output/openapi/elasticsearch-openapi.examples.json
+output/openapi/elasticsearch-openapi.examples.json
+output/openapi/elasticsearch-serverless-openapi-docs.json
+output/openapi/elasticsearch-openapi-docs.json

Makefile

@@ -43,6 +43,7 @@ setup:	## Install dependencies for contrib target
 	@npm install --prefix validator
 	@npm install --prefix specification
 	@npm install @redocly/cli
+	@npm install --prefix docs/examples
 
 clean-dep:	## Clean npm dependencies
 	@rm -rf compiler/node_modules
@@ -56,7 +57,10 @@ transform-to-openapi: ## Generate the OpenAPI definition from the compiled schem
 	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor serverless --output output/openapi/elasticsearch-serverless-openapi.json
 
 transform-to-openapi-for-docs: ## Generate the OpenAPI definition tailored for API docs generation
-	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor stack --lift-enum-descriptions --merge-multipath-endpoints --output output/openapi/elasticsearch-openapi-docs.json
+	@make generate-language-examples
+	@make generate
+	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor stack --lift-enum-descriptions --merge-multipath-endpoints --multipath-redirects --include-language-examples --output output/openapi/elasticsearch-openapi-docs.json
+	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor serverless --lift-enum-descriptions --merge-multipath-endpoints --multipath-redirects --include-language-examples --output output/openapi/elasticsearch-serverless-openapi-docs.json
 
 filter-for-serverless: ## Generate the serverless version from the compiled schema
 	@npm run --prefix compiler filter-by-availability -- --serverless --visibility=public --input ../output/schema/schema.json --output ../output/output/openapi/elasticsearch-serverless-openapi.json
@@ -74,6 +78,14 @@ overlay-docs: ## Apply overlays to OpenAPI documents
 	rm output/openapi/elasticsearch-serverless-openapi.tmp*.json
 	rm output/openapi/elasticsearch-openapi.tmp*.json
 
+generate-language-examples:
+	@node docs/examples/generate-language-examples.js
+	@npm run format:fix-examples --prefix compiler
+
+generate-language-examples-with-java:
+	@node docs/examples/generate-language-examples.js java
+	@npm run format:fix-examples --prefix compiler
+
 lint-docs: ## Lint the OpenAPI documents after overlays
 	@npx @redocly/cli lint "output/openapi/elasticsearch-*.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
 

README.md

@@ -60,6 +60,13 @@ Follow the steps to generate the JSON representation, then:
 ```
 # Generate the OpenAPI representation
 $ make transform-to-openapi
+```
+
+To generate the JSON representation that is used for documentation purposes, the commands are different:
+
+```
+# Generate the OpenAPI files
+$ make transform-to-openapi-for-docs
 
 # Apply fixes
 $ make overlay-docs

api-design-guidelines/data-modelling.md

@@ -150,7 +150,7 @@ Or better yet, to completely avoid using dynamic keys the user-defined value can
 
 ### Model object variants in a consumable way
 
-Sometimes an API accepts an object but the keys are determined by what "kind/variant" of the object is intended. An example of this is aggregations, queries, and pipeline steps. There are two ways the Elasticsearch API handles this situation. The first method is using an **internal variant type** property like "type" with analyzers:
+Sometimes an API accepts an object but the keys are determined by what "kind/variant" of the object is intended. An example of this is aggregations, queries, and pipeline steps. There are two ways the Elasticsearch API handles this situation. The first method is using **internal tagging** with property like "type" with analyzers:
 
 ```yaml
 {
@@ -159,7 +159,7 @@ Sometimes an API accepts an object but the keys are determined by what "kind/var
 }
 ```
 
-The second is using **external variants** where the inner object is wrapped with an object with a single key containing the kind of the inner object. This example changes the analyzer from above to use an external variant:
+The second is using **external tagging** where the inner object is wrapped with an object with a single key containing the kind of the inner object. This example changes the analyzer from above to use an external variant:
 
 ```yaml
 {
@@ -169,7 +169,7 @@ The second is using **external variants** where the inner object is wrapped with
 }
 ```
 
-When choosing between these two possibilities **favor using external variants** as it removes the requirement to buffer key-value pairs until the internal variant property is found. Using external variants also improves traversability of the API (ie auto-complete) as properties can be anticipated without waiting for the discriminant property.
+Internal tagging is a common way to identify variants and should usually be preferred. However, it may have performance implications when used to deserialize large objects in strongly-typed languages. In that case, external tagging should be considered instead, as it removes the requirement to buffer key-value pairs until the internal variant property is found. Using external tagging also improves traversability of the API (ie auto-complete) as properties can be anticipated without waiting for the discriminant property.
 
 ## Model enumerations in a portable way
 

api-design-guidelines/requests-responses.md

@@ -97,34 +97,6 @@ It's best to use the body for large or high cardinality variable-length request
 - Large objects (Query DSL, aggregations, machine learning models)
 - High cardinality variable length values (list of fields, indices, shards, document IDs)
 
-## Adhere to the robustness principle
-
-The robustness principle states:
-
-> "be conservative in what you do, be liberal in what you accept from others"
-
-This principle can apply in a number of ways, but in particular it can determine the design of API requests and responses. An API request may be built flexibly so as to allow both verbose detailed payloads and terse convenience payloads. The corresponding response, however, should always be structured predictably, and not depend on a particular syntax used within the request.
-
-As an example, consider an API that looks up entities based on their ID. The request may be structured to allow either a single ID or a collection of IDs to be passed in. However, the API response should always return a collection of entities, even if that collection only contains one entity.
-
-This principle allows for simpler consumer code that neither has to remember state between request and response, nor needs to "sniff" the output to determine its structure. If multiple variants of the response are truly desired, this may suggest that multiple API endpoints should be introduced, for example called `get_single_entity` and `get_multiple_entities`.
-
-An example of this is the datafeeds API which accepts either a string or list of strings for indices but always returns a list of strings:
-
-```yaml
-PUT /_ml/datafeeds/feed-id
-{
-  "indices": "index-name", // Input is a string.
-  ...
-}
-
-GET /_ml/datafeeds/feed-id
-{
-  "indices": ["index-name"], // Always returns a list of strings.
-  ...
-}
-```
-
 ## Consider how client functions would wrap the API endpoint
 
 It is common within client-side architecture to provide a one-to-one mapping between API endpoint and client language function. This simplifies implementation and documentation, provides good developer experience, and makes tracking of endpoint usage straightforward.

compiler-rs/clients_schema/src/lib.rs

@@ -58,6 +58,7 @@ pub trait Documented {
 pub trait ExternalDocument {
     fn ext_doc_id(&self) -> Option<&str>;
     fn ext_doc_url(&self) -> Option<&str>;
+    fn ext_previous_version_doc_url(&self) -> Option<&str>;
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
@@ -322,6 +323,9 @@ pub struct Property {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub ext_doc_url: Option<String>,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub ext_previous_version_doc_url: Option<String>,
+
     #[serde(skip_serializing_if = "Option::is_none")]
     pub ext_doc_id: Option<String>,
 
@@ -371,6 +375,10 @@ impl ExternalDocument for Property {
         self.ext_doc_url.as_deref()
     }
 
+    fn ext_previous_version_doc_url(&self) -> Option<&str> {
+        self.ext_previous_version_doc_url.as_deref()
+    }
+
     fn ext_doc_id(&self) -> Option<&str> {
         self.ext_doc_id.as_deref()
     }
@@ -484,20 +492,28 @@ impl TypeDefinition {
 
 /// The Example type is used for both requests and responses.
 ///
-/// This type definition is taken from the OpenAPI spec
+/// This type definition is based on the OpenAPI spec
 ///     https://spec.openapis.org/oas/v3.1.0#example-object
-/// with the exception of using String as the 'value' type.
+/// with the exception of using String as the 'value' type,
+/// and some custom additions.
 ///
 /// The OpenAPI v3 spec also defines the 'Example' type, so
 /// to distinguish them, this type is called SchemaExample.
 
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ExampleAlternative {
+    pub language: String,
+    pub code: String,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct SchemaExample {
     pub summary: Option<String>,
     pub method_request: Option<String>,
     pub description: Option<String>,
     pub value: Option<String>,
     pub external_value: Option<String>,
+    pub alternatives: Option<Vec<ExampleAlternative>>,
 }
 
 /// Common attributes for all type definitions
@@ -520,6 +536,9 @@ pub struct BaseType {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub ext_doc_url: Option<String>,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub ext_previous_version_doc_url: Option<String>,
+
     #[serde(skip_serializing_if = "Option::is_none")]
     pub ext_doc_id: Option<String>,
 
@@ -560,6 +579,7 @@ impl BaseType {
             spec_location: None,
             ext_doc_id: None,
             ext_doc_url: None,
+            ext_previous_version_doc_url: None,
         }
     }
 }
@@ -583,6 +603,10 @@ impl ExternalDocument for BaseType {
         self.ext_doc_url.as_deref()
     }
 
+    fn ext_previous_version_doc_url(&self) -> Option<&str> {
+        self.ext_previous_version_doc_url.as_deref()
+    }
+
     fn ext_doc_id(&self) -> Option<&str> {
         self.ext_doc_id.as_deref()
     }
@@ -611,6 +635,10 @@ impl<T: WithBaseType> ExternalDocument for T {
         self.base().doc_url()
     }
 
+    fn ext_previous_version_doc_url(&self) -> Option<&str> {
+        self.base().ext_previous_version_doc_url()
+    }
+
     fn ext_doc_id(&self) -> Option<&str> {
         self.base().doc_id()
     }
@@ -887,6 +915,9 @@ pub struct Endpoint {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub ext_doc_url: Option<String>,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub ext_previous_version_doc_url: Option<String>,
+
     #[serde(skip_serializing_if = "Option::is_none")]
     pub deprecation: Option<Deprecation>,
 
@@ -937,6 +968,10 @@ impl ExternalDocument for Endpoint {
         self.ext_doc_url.as_deref()
     }
 
+    fn ext_previous_version_doc_url(&self) -> Option<&str> {
+        self.ext_previous_version_doc_url.as_deref()
+    }
+
     fn ext_doc_id(&self) -> Option<&str> {
         self.ext_doc_id.as_deref()
     }