Include date in data stream backing index names (#65205)

Author: danhermann

docs/reference/data-streams/change-mappings-and-settings.asciidoc

@@ -546,10 +546,9 @@ The following get data stream API request retrieves information about
 GET /_data_stream/my-data-stream
 ----
 
-The API returns the following response. Note the `indices` property contains an
-array of the stream's current backing indices. The first item in the array
-contains information about the stream's oldest backing index,
-`.ds-my-data-stream-000001`.
+The response's `indices` property contains an array of the stream's current
+backing indices. The first item in the array contains information about the
+stream's oldest backing index.
 
 [source,console-result]
 ----
@@ -562,11 +561,11 @@ contains information about the stream's oldest backing index,
       },
       "indices": [
         {
-          "index_name": ".ds-my-data-stream-000001",        <1>
+          "index_name": ".ds-my-data-stream-2099.03.07-000001", <1>
           "index_uuid": "Gpdiyq8sRuK9WuthvAdFbw"
         },
         {
-          "index_name": ".ds-my-data-stream-000002",
+          "index_name": ".ds-my-data-stream-2099.03.08-000002",
           "index_uuid": "_eEfRrFHS9OyhqWntkgHAQ"
         }
       ],
@@ -580,29 +579,33 @@ contains information about the stream's oldest backing index,
 ----
 // TESTRESPONSE[s/"index_uuid": "Gpdiyq8sRuK9WuthvAdFbw"/"index_uuid": $body.data_streams.0.indices.0.index_uuid/]
 // TESTRESPONSE[s/"index_uuid": "_eEfRrFHS9OyhqWntkgHAQ"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/]
+// TESTRESPONSE[s/"index_name": ".ds-my-data-stream-2099.03.07-000001"/"index_name": $body.data_streams.0.indices.0.index_name/]
+// TESTRESPONSE[s/"index_name": ".ds-my-data-stream-2099.03.08-000002"/"index_name": $body.data_streams.0.indices.1.index_name/]
 // TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW"/]
 
-<1> First item in the `indices` array for `my-data-stream`. This
-item contains information about the stream's oldest backing index,
-`.ds-my-data-stream-000001`.
+<1> First item in the `indices` array for `my-data-stream`. This item contains
+information about the stream's oldest backing index,
+`.ds-my-data-stream-2099.03.07-000001`.
 
 The following <<docs-reindex,reindex API>> request copies documents from
-`.ds-my-data-stream-000001` to `new-data-stream`. Note the request's `op_type`
-is `create`.
+`.ds-my-data-stream-2099.03.07-000001` to `new-data-stream`. The request's
+`op_type` is `create`.
 
 [source,console]
 ----
 POST /_reindex
 {
   "source": {
-    "index": ".ds-my-data-stream-000001"
+    "index": ".ds-my-data-stream-2099.03.07-000001"
   },
   "dest": {
     "index": "new-data-stream",
     "op_type": "create"
   }
 }
 ----
+// TEST[setup:my_index]
+// TEST[s/.ds-my-data-stream-2099.03.07-000001/my-index-000001/]
 --
 +
 You can also use a query to reindex only a subset of documents with each

docs/reference/data-streams/data-streams.asciidoc

@@ -95,12 +95,13 @@ convention:
 
 [source,text]
 ----
-.ds-<data-stream>-<generation>
+.ds-<data-stream>-<yyyy.MM.dd>-<generation>
 ----
 
-Backing indices with a higher generation contain more recent data. For example,
-the `web-server-logs` data stream has a generation of `34`. The stream's most
-recent backing index is named `.ds-web-server-logs-000034`.
+`<yyyy.MM.dd>` is the backing index's creation date. Backing indices with a
+higher generation contain more recent data. For example, the `web-server-logs`
+data stream has a generation of `34`. The stream's most recent backing index,
+created on 7 March 2099, is named `.ds-web-server-logs-2099.03.07-000034`.
 
 Some operations, such as a <<indices-shrink-index,shrink>> or
 <<snapshots-restore-snapshot,restore>>, can change a backing index's name.

docs/reference/data-streams/set-up-a-data-stream.asciidoc

@@ -137,7 +137,7 @@ name must match one of your template's index patterns.
 ----
 POST /my-data-stream/_doc/
 {
-  "@timestamp": "2020-12-06T11:04:05.000Z",
+  "@timestamp": "2099-03-07T11:04:05.000Z",
   "user": {
     "id": "vlb44hny"
   },

docs/reference/data-streams/use-a-data-stream.asciidoc

@@ -32,7 +32,7 @@ POST /my-data-stream/_rollover/
 
 PUT /my-data-stream/_create/bfspvnIBr7VVZlfp2lqX?refresh=wait_for
 {
-  "@timestamp": "2020-12-07T11:06:07.000Z",
+  "@timestamp": "2099-03-08T11:06:07.000Z",
   "user": {
     "id": "yWIumJd7"
   },
@@ -61,7 +61,7 @@ To add an individual document, use the <<docs-index_,index API>>.
 ----
 POST /my-data-stream/_doc/
 {
-  "@timestamp": "2020-12-07T11:06:07.000Z",
+  "@timestamp": "2099-03-08T11:06:07.000Z",
   "user": {
     "id": "8a4f500d"
   },
@@ -81,11 +81,11 @@ Only `create` actions are supported.
 ----
 PUT /my-data-stream/_bulk?refresh
 {"create":{ }}
-{ "@timestamp": "2020-12-08T11:04:05.000Z", "user": { "id": "vlb44hny" }, "message": "Login attempt failed" }
+{ "@timestamp": "2099-03-08T11:04:05.000Z", "user": { "id": "vlb44hny" }, "message": "Login attempt failed" }
 {"create":{ }}
-{ "@timestamp": "2020-12-08T11:06:07.000Z", "user": { "id": "8a4f500d" }, "message": "Login successful" }
+{ "@timestamp": "2099-03-08T11:06:07.000Z", "user": { "id": "8a4f500d" }, "message": "Login successful" }
 {"create":{ }}
-{ "@timestamp": "2020-12-09T11:07:08.000Z", "user": { "id": "l7gk7f82" }, "message": "Logout successful" }
+{ "@timestamp": "2099-03-09T11:07:08.000Z", "user": { "id": "l7gk7f82" }, "message": "Logout successful" }
 ----
 
 [discrete]
@@ -137,8 +137,10 @@ index API request>> directly to the index:
 
 [source,console]
 ----
-POST /.ds-my-data-stream-000001/_open/
+POST /.ds-my-data-stream-2099.03.07-000001/_open/
 ----
+// TEST[setup:my_index]
+// TEST[s/.ds-my-data-stream-2099.03.07-000001/my-index-000001/]
 
 To re-open all closed backing indices for a data stream, submit an open index
 API request to the stream:
@@ -163,13 +165,13 @@ stream.
 ----
 PUT /_bulk?refresh=wait_for
 {"create":{"_index" : "archive_1"}}
-{ "@timestamp": "2020-12-08T11:04:05.000Z" }
+{ "@timestamp": "2099-03-08T11:04:05.000Z" }
 {"create":{"_index" : "archive_2"}}
-{ "@timestamp": "2020-12-08T11:06:07.000Z" }
+{ "@timestamp": "2099-03-08T11:06:07.000Z" }
 {"create":{"_index" : "archive_2"}}
-{ "@timestamp": "2020-12-09T11:07:08.000Z" }
+{ "@timestamp": "2099-03-09T11:07:08.000Z" }
 {"create":{"_index" : "archive_2"}}
-{ "@timestamp": "2020-12-09T11:07:08.000Z" }
+{ "@timestamp": "2099-03-09T11:07:08.000Z" }
 
 POST /_aliases
 {
@@ -288,13 +290,13 @@ Response:
     "max_score": 0.2876821,
     "hits": [
       {
-        "_index": ".ds-my-data-stream-000003",      <1>
+        "_index": ".ds-my-data-stream-2099.03.08-000003",      <1>
         "_id": "bfspvnIBr7VVZlfp2lqX",              <2>
         "_seq_no": 0,                               <3>
         "_primary_term": 1,                         <4>
         "_score": 0.2876821,
         "_source": {
-          "@timestamp": "2020-12-07T11:06:07.000Z",
+          "@timestamp": "2099-03-08T11:06:07.000Z",
           "user": {
             "id": "yWIumJd7"
           },
@@ -307,6 +309,7 @@ Response:
 ----
 // TESTRESPONSE[s/"took": 20/"took": $body.took/]
 // TESTRESPONSE[s/"max_score": 0.2876821/"max_score": $body.hits.max_score/]
+// TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.03.08-000003"/"_index": $body.hits.hits.0._index/]
 // TESTRESPONSE[s/"_score": 0.2876821/"_score": $body.hits.hits.0._score/]
 
 <1> Backing index containing the matching document
@@ -319,22 +322,29 @@ To update the document, use an <<docs-index_,index API>> request with valid
 
 [source,console]
 ----
-PUT /.ds-my-data-stream-000003/_doc/bfspvnIBr7VVZlfp2lqX?if_seq_no=0&if_primary_term=1
+PUT /.ds-my-data-stream-2099-03-08-000003/_doc/bfspvnIBr7VVZlfp2lqX?if_seq_no=0&if_primary_term=1
 {
-  "@timestamp": "2020-12-07T11:06:07.000Z",
+  "@timestamp": "2099-03-08T11:06:07.000Z",
   "user": {
     "id": "8a4f500d"
   },
   "message": "Login successful"
 }
 ----
+// TEST[setup:my_index]
+// TEST[s/.ds-my-data-stream-2099.03.08-000003/my-index-000001/]
+// TEST[s/bfspvnIBr7VVZlfp2lqX/1/]
+// TEST[s/if_seq_no=0/if_seq_no=1/]
 
 To delete the document, use the <<docs-delete,delete API>>:
 
 [source,console]
 ----
-DELETE /.ds-my-data-stream-000003/_doc/bfspvnIBr7VVZlfp2lqX
+DELETE /.ds-my-data-stream-2099.03.08-000003/_doc/bfspvnIBr7VVZlfp2lqX
 ----
+// TEST[setup:my_index]
+// TEST[s/.ds-my-data-stream-2099.03.08-000003/my-index-000001/]
+// TEST[s/bfspvnIBr7VVZlfp2lqX/1/]
 
 To delete or update multiple documents with a single request, use the
 <<docs-bulk,bulk API>>'s `delete`, `index`, and `update` actions. For `index`
@@ -344,7 +354,10 @@ actions, include valid <<bulk-optimistic-concurrency-control,`if_seq_no` and
 [source,console]
 ----
 PUT /_bulk?refresh
-{ "index": { "_index": ".ds-my-data-stream-000003", "_id": "bfspvnIBr7VVZlfp2lqX", "if_seq_no": 0, "if_primary_term": 1 } }
-{ "@timestamp": "2020-12-07T11:06:07.000Z", "user": { "id": "8a4f500d" }, "message": "Login successful" }
+{ "index": { "_index": ".ds-my-data-stream-2099.03.08-000003", "_id": "bfspvnIBr7VVZlfp2lqX", "if_seq_no": 0, "if_primary_term": 1 } }
+{ "@timestamp": "2099-03-08T11:06:07.000Z", "user": { "id": "8a4f500d" }, "message": "Login successful" }
 ----
+// TEST[setup:my_index]
+// TEST[s/.ds-my-data-stream-2099.03.08-000003/my-index-000001/]
+// TEST[s/bfspvnIBr7VVZlfp2lqX/1/]
 

docs/reference/ilm/ilm-tutorial.asciidoc

@@ -31,7 +31,7 @@ indices.
 . <<ilm-gs-check-progress, Verify indices are moving through the lifecycle phases>>
 as expected.
 
-For an introduction to rolling indices, see <<index-rollover>>. 
+For an introduction to rolling indices, see <<index-rollover>>.
 
 IMPORTANT: When you enable {ilm} for {beats} or the {ls} {es} output plugin,
 lifecycle policies are set up automatically.
@@ -46,17 +46,17 @@ or the {ilm-init} APIs.
 
 A lifecycle policy specifies the phases in the index lifecycle
 and the actions to perform in each phase. A lifecycle can have up to four phases:
-`hot`, `warm`, `cold`, and `delete`. 
+`hot`, `warm`, `cold`, and `delete`.
 
 For example, you might define a `timeseries_policy` that has two phases:
- 
-* A `hot` phase that defines a rollover action to specify that an index rolls over when it 
+
+* A `hot` phase that defines a rollover action to specify that an index rolls over when it
 reaches either a `max_size` of 50 gigabytes or a `max_age` of 30 days.
 * A `delete` phase that sets `min_age` to remove the index 90 days after rollover.
-Note that this value is relative to the rollover time, not the index creation time. 
+Note that this value is relative to the rollover time, not the index creation time.
 
-You can create the policy through {kib} or with the 
-<<ilm-put-lifecycle, put policy>> API. 
+You can create the policy through {kib} or with the
+<<ilm-put-lifecycle, put policy>> API.
 To create the policy from {kib}, open the menu and go to *Stack Management >
 Index Lifecycle Policies*. Click *Index Lifecycle Policies*.
 
@@ -154,8 +154,8 @@ request automatically creates a corresponding data stream with a single backing
 {es} automatically indexes the request's documents into this backing index, which also
 acts as the stream's <<data-stream-write-index,write index>>.
 
-For example, the following request creates the `timeseries` data stream and the first generation
-backing index called `.ds-timeseries-000001`.
+For example, the following request creates the `timeseries` data stream and the
+first generation backing index called `.ds-timeseries-2099.03.08-000001`.
 
 [source,console]
 -----------------------
@@ -170,10 +170,13 @@ POST timeseries/_doc
 
 When a rollover condition in the lifecycle policy is met, the `rollover` action:
 
-* Creates the second generation backing index, named `.ds-timeseries-000002`.
-Because it is a backing index of the `timeseries` data stream, the configuration from the `timeseries_template` index template is applied to the new index.
-* As it is the latest generation index of the `timeseries` data stream, the newly created
-backing index `.ds-timeseries-000002` becomes the data stream's write index.
+* Creates the second generation backing index, named
+`.ds-timeseries-2099.03.08-000002`. Because it is a backing index of the
+`timeseries` data stream, the configuration from the `timeseries_template` index
+template is applied to the new index.
+* As it is the latest generation index of the `timeseries` data stream, the
+newly created backing index `.ds-timeseries-2099.03.08-000002` becomes the data
+stream's write index.
 
 This process repeats each time a rollover condition is met.
 You can search across all of the data stream's backing indices, managed by the `timeseries_policy`,
@@ -211,8 +214,8 @@ is met.
 --------------------------------------------------
 {
   "indices": {
-    ".ds-timeseries-000001": {
-      "index": ".ds-timeseries-000001",
+    ".ds-timeseries-2099.03.07-000001": {
+      "index": ".ds-timeseries-2099.03.07-000001",
       "managed": true,
       "policy": "timeseries_policy",             <1>
       "lifecycle_date_millis": 1538475653281,

docs/reference/images/data-streams/data-streams-diagram.svg

@@ -176,10 +176,10 @@ its shards can be force-merged to a single segment.
 
 [source,console]
 --------------------------------------------------
-POST /.ds-logs-000001/_forcemerge?max_num_segments=1
+POST /.ds-my-data-stream-2099.03.07-000001/_forcemerge?max_num_segments=1
 --------------------------------------------------
 // TEST[setup:my_index]
-// TEST[s/.ds-logs-000001/my-index-000001/]
+// TEST[s/.ds-my-data-stream-2099.03.07-000001/my-index-000001/]
 
 This can be a good idea because single-segment shards can sometimes use simpler
 and more efficient data structures to perform searches.

docs/reference/images/data-streams/data-streams-index-request.svg

@@ -50,7 +50,7 @@ PUT /_data_stream/my-data-stream
 
 POST /my-data-stream/_rollover
 
-PUT /_data_stream/my-data-stream_two
+PUT /_data_stream/my-data-stream-two
 ----
 // TESTSETUP
 ////
@@ -207,11 +207,11 @@ The API returns the following response:
       },
       "indices": [
         {
-          "index_name": ".ds-my-data-stream-000001",
+          "index_name": ".ds-my-data-stream-2099.03.07-000001",
           "index_uuid": "xCEhwsp8Tey0-FLNFYVwSg"
         },
         {
-          "index_name": ".ds-my-data-stream-000002",
+          "index_name": ".ds-my-data-stream-2099.03.08-000002",
           "index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"
         }
       ],
@@ -225,13 +225,13 @@ The API returns the following response:
       "hidden": false
     },
     {
-      "name": "my-data-stream_two",
+      "name": "my-data-stream-two",
       "timestamp_field": {
         "name": "@timestamp"
       },
       "indices": [
         {
-          "index_name": ".ds-my-data-stream_two-000001",
+          "index_name": ".ds-my-data-stream-two-2099.03.08-000001",
           "index_uuid": "3liBu2SYS5axasRt6fUIpA"
         }
       ],
@@ -247,7 +247,10 @@ The API returns the following response:
   ]
 }
 ----
+// TESTRESPONSE[s/"index_name": ".ds-my-data-stream-2099.03.07-000001"/"index_name": $body.data_streams.0.indices.0.index_name/]
 // TESTRESPONSE[s/"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg"/"index_uuid": $body.data_streams.0.indices.0.index_uuid/]
+// TESTRESPONSE[s/"index_name": ".ds-my-data-stream-2099.03.08-000002"/"index_name": $body.data_streams.0.indices.1.index_name/]
 // TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/]
+// TESTRESPONSE[s/"index_name": ".ds-my-data-stream-two-2099.03.08-000001"/"index_name": $body.data_streams.1.indices.0.index_name/]
 // TESTRESPONSE[s/"index_uuid": "3liBu2SYS5axasRt6fUIpA"/"index_uuid": $body.data_streams.1.indices.0.index_uuid/]
 // TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW"/]