From 39105e38c2d1b8c21e633f557d0cb2f8d1a94818 Mon Sep 17 00:00:00 2001 From: lcawl Date: Thu, 19 Sep 2019 09:56:59 -0700 Subject: [PATCH 1/3] [DOCS] Group rollup and transform content --- docs/reference/data-operations.asciidoc | 16 ++++++++++++ docs/reference/index.asciidoc | 4 +-- docs/reference/rollup/api-quickref.asciidoc | 11 +++++--- docs/reference/rollup/index.asciidoc | 25 ++++++++----------- docs/reference/rollup/overview.asciidoc | 13 ++++++---- .../rollup/rollup-agg-limitations.asciidoc | 4 +-- .../rollup/rollup-getting-started.asciidoc | 13 ++++++---- .../rollup/rollup-search-limitations.asciidoc | 12 ++++----- .../rollup/understanding-groups.asciidoc | 6 ++--- 9 files changed, 63 insertions(+), 41 deletions(-) create mode 100644 docs/reference/data-operations.asciidoc diff --git a/docs/reference/data-operations.asciidoc b/docs/reference/data-operations.asciidoc new file mode 100644 index 0000000000000..13bb6d896eae0 --- /dev/null +++ b/docs/reference/data-operations.asciidoc @@ -0,0 +1,16 @@ +[[data-operations]] += Roll up or transform your data + +[partintro] +-- + +{es} offers the following methods for manipulating your data: + +* <> ++ +include::rollup/index.asciidoc[tag=rollup-intro] +* {stack-ov}/ml-dataframes.html[Transforming your data] + +-- + +include::rollup/index.asciidoc[] diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc index 3b7c2d988af72..cf41b08572e71 100644 --- a/docs/reference/index.asciidoc +++ b/docs/reference/index.asciidoc @@ -48,10 +48,10 @@ include::sql/index.asciidoc[] include::monitoring/index.asciidoc[] -include::rollup/index.asciidoc[] - include::frozen-indices.asciidoc[] +include::data-operations.asciidoc[] + include::high-availability.asciidoc[] include::commands/index.asciidoc[] diff --git a/docs/reference/rollup/api-quickref.asciidoc b/docs/reference/rollup/api-quickref.asciidoc index d6be3e4e5b69c..8a64d9df17f35 100644 --- a/docs/reference/rollup/api-quickref.asciidoc +++ b/docs/reference/rollup/api-quickref.asciidoc @@ -1,7 +1,10 @@ [role="xpack"] [testenv="basic"] [[rollup-api-quickref]] -== API Quick Reference +=== {rollup-cap} API quick reference +++++ +API quick reference +++++ experimental[] @@ -15,7 +18,7 @@ Most rollup endpoints have the following base: [float] [[rollup-api-jobs]] -=== /job/ +==== /job/ * {ref}/rollup-put-job.html[PUT /_rollup/job/+++]: Create a {rollup-job} * {ref}/rollup-get-job.html[GET /_rollup/job]: List {rollup-jobs} @@ -26,13 +29,13 @@ Most rollup endpoints have the following base: [float] [[rollup-api-data]] -=== /data/ +==== /data/ * {ref}/rollup-get-rollup-caps.html[GET /_rollup/data//_rollup_caps+++]: Get Rollup Capabilities * {ref}/rollup-get-rollup-index-caps.html[GET //_rollup/data/+++]: Get Rollup Index Capabilities [float] [[rollup-api-index]] -=== // +==== // * {ref}/rollup-search.html[GET //_rollup_search]: Search rollup data diff --git a/docs/reference/rollup/index.asciidoc b/docs/reference/rollup/index.asciidoc index 64dc233f82f6e..99180e2f32d45 100644 --- a/docs/reference/rollup/index.asciidoc +++ b/docs/reference/rollup/index.asciidoc @@ -1,10 +1,7 @@ [role="xpack"] [testenv="basic"] [[xpack-rollup]] -= Rolling up historical data - -[partintro] --- +== Rolling up historical data experimental[] @@ -12,19 +9,19 @@ Keeping historical data around for analysis is extremely useful but often avoide archiving massive amounts of data. Retention periods are thus driven by financial realities rather than by the usefulness of extensive historical data. -The Rollup feature in {xpack} provides a means to summarize and store historical data so that it can still be used -for analysis, but at a fraction of the storage cost of raw data. - +// tag::rollup-intro[] +The {stack} {rollup-features} provide a means to summarize and store historical +data so that it can still be used for analysis, but at a fraction of the storage +cost of raw data. +// end::rollup-intro[] -* <> -* <> -* <> -* <> +* <> +* <> +* <> +* <> * <> -* <> - +* <> --- include::overview.asciidoc[] include::api-quickref.asciidoc[] diff --git a/docs/reference/rollup/overview.asciidoc b/docs/reference/rollup/overview.asciidoc index 90c5e20a850c1..eb8815d8ad162 100644 --- a/docs/reference/rollup/overview.asciidoc +++ b/docs/reference/rollup/overview.asciidoc @@ -1,7 +1,10 @@ [role="xpack"] [testenv="basic"] [[rollup-overview]] -== Overview +=== {rollup-cap} overview +++++ +Overview +++++ experimental[] @@ -23,7 +26,7 @@ reading often diminishes with time. It's not useless -- it could easily contrib value often leads to deletion rather than paying the fixed storage cost. [float] -=== Rollup store historical data at reduced granularity +==== Rollup store historical data at reduced granularity That's where Rollup comes into play. The Rollup functionality summarizes old, high-granularity data into a reduced granularity format for long-term storage. By "rolling" the data up into a single summary document, historical data @@ -39,7 +42,7 @@ automates this process of summarizing historical data. Details about setting up and configuring Rollup are covered in <> [float] -=== Rollup uses standard query DSL +==== Rollup uses standard query DSL The Rollup feature exposes a new search endpoint (`/_rollup_search` vs the standard `/_search`) which knows how to search over rolled-up data. Importantly, this endpoint accepts 100% normal {es} Query DSL. Your application does not need to learn @@ -53,7 +56,7 @@ But if your queries, aggregations and dashboards only use the available function data is trivial. [float] -=== Rollup merges "live" and "rolled" data +==== Rollup merges "live" and "rolled" data A useful feature of Rollup is the ability to query both "live", realtime data in addition to historical "rolled" data in a single query. @@ -67,7 +70,7 @@ It will take the results from both data sources and merge them together. If the "rolled" data, live data is preferred to increase accuracy. [float] -=== Rollup is multi-interval aware +==== Rollup is multi-interval aware Finally, Rollup is capable of intelligently utilizing the best interval available. If you've worked with summarizing features of other products, you'll find that they can be limiting. If you configure rollups at daily intervals... your diff --git a/docs/reference/rollup/rollup-agg-limitations.asciidoc b/docs/reference/rollup/rollup-agg-limitations.asciidoc index 9f8b6f66adeeb..6f9f949bf8b6c 100644 --- a/docs/reference/rollup/rollup-agg-limitations.asciidoc +++ b/docs/reference/rollup/rollup-agg-limitations.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="basic"] [[rollup-agg-limitations]] -== Rollup Aggregation Limitations +=== {rollup-cap} aggregation limitations experimental[] @@ -9,7 +9,7 @@ There are some limitations to how fields can be rolled up / aggregated. This pa you are aware of them. [float] -=== Limited aggregation components +==== Limited aggregation components The Rollup functionality allows fields to be grouped with the following aggregations: diff --git a/docs/reference/rollup/rollup-getting-started.asciidoc b/docs/reference/rollup/rollup-getting-started.asciidoc index 27f9d9cd40603..3b57e968a9e5e 100644 --- a/docs/reference/rollup/rollup-getting-started.asciidoc +++ b/docs/reference/rollup/rollup-getting-started.asciidoc @@ -1,7 +1,10 @@ [role="xpack"] [testenv="basic"] [[rollup-getting-started]] -== Getting Started +=== Getting started with {rollups} +++++ +Getting started +++++ experimental[] @@ -23,7 +26,7 @@ look like this: // NOTCONSOLE [float] -=== Creating a Rollup Job +==== Creating a rollup job We'd like to rollup these documents into hourly summaries, which will allow us to generate reports and dashboards with any time interval one hour or greater. A rollup job might look like this: @@ -103,7 +106,7 @@ After you execute the above command and create the job, you'll receive the follo ---- [float] -=== Starting the job +==== Starting the job After the job is created, it will be sitting in an inactive state. Jobs need to be started before they begin processing data (this allows you to stop them later as a way to temporarily pause, without deleting the configuration). @@ -117,7 +120,7 @@ POST _rollup/job/sensor/_start // TEST[setup:sensor_rollup_job] [float] -=== Searching the Rolled results +==== Searching the rolled results After the job has run and processed some data, we can use the <> endpoint to do some searching. The Rollup feature is designed so that you can use the same Query DSL syntax that you are accustomed to... it just happens to run on the rolled up data instead. @@ -292,7 +295,7 @@ In addition to being more complicated (date histogram and a terms aggregation, p the date_histogram uses a `7d` interval instead of `60m`. [float] -=== Conclusion +==== Conclusion This quickstart should have provided a concise overview of the core functionality that Rollup exposes. There are more tips and things to consider when setting up Rollups, which you can find throughout the rest of this section. You may also explore the <> diff --git a/docs/reference/rollup/rollup-search-limitations.asciidoc b/docs/reference/rollup/rollup-search-limitations.asciidoc index d55787f3cec53..f6315e12a3007 100644 --- a/docs/reference/rollup/rollup-search-limitations.asciidoc +++ b/docs/reference/rollup/rollup-search-limitations.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="basic"] [[rollup-search-limitations]] -== Rollup Search Limitations +=== {rollup-cap} search limitations experimental[] @@ -11,7 +11,7 @@ live data is thrown away, you will always lose some flexibility. This page highlights the major limitations so that you are aware of them. [float] -=== Only one Rollup index per search +==== Only one {rollup} index per search When using the <> endpoint, the `index` parameter accepts one or more indices. These can be a mix of regular, non-rollup indices and rollup indices. However, only one rollup index can be specified. The exact list of rules for the `index` parameter are as @@ -33,7 +33,7 @@ may be able to open this up to multiple rollup jobs. [float] [[aggregate-stored-only]] -=== Can only aggregate what's been stored +==== Can only aggregate what's been stored A perhaps obvious limitation, but rollups can only aggregate on data that has been stored in the rollups. If you don't configure the rollup job to store metrics about the `price` field, you won't be able to use the `price` field in any query or aggregation. @@ -81,7 +81,7 @@ The response will tell you that the field and aggregation were not possible, bec // TESTRESPONSE[s/"stack_trace": \.\.\./"stack_trace": $body.$_path/] [float] -=== Interval Granularity +==== Interval granularity Rollups are stored at a certain granularity, as defined by the `date_histogram` group in the configuration. This means you can only search/aggregate the rollup data with an interval that is greater-than or equal to the configured rollup interval. @@ -111,7 +111,7 @@ That said, if multiple jobs are present in a single rollup index with varying in with the largest interval to satisfy the search request. [float] -=== Limited querying components +==== Limited querying components The Rollup functionality allows `query`'s in the search request, but with a limited subset of components. The queries currently allowed are: @@ -128,7 +128,7 @@ If you attempt to use an unsupported query, or the query references a field that thrown. We expect the list of support queries to grow over time as more are implemented. [float] -=== Timezones +==== Timezones Rollup documents are stored in the timezone of the `date_histogram` group configuration in the job. If no timezone is specified, the default is to rollup timestamps in `UTC`. diff --git a/docs/reference/rollup/understanding-groups.asciidoc b/docs/reference/rollup/understanding-groups.asciidoc index a59c19fbf5cc6..eb1b47e8a16d0 100644 --- a/docs/reference/rollup/understanding-groups.asciidoc +++ b/docs/reference/rollup/understanding-groups.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="basic"] [[rollup-understanding-groups]] -== Understanding Groups +=== Understanding groups experimental[] @@ -121,7 +121,7 @@ Ultimately, when configuring `groups` for a job, think in terms of how you might then include those in the config. Because Rollup Search allows any order or combination of the grouped fields, you just need to decide if a field is useful for aggregating later, and how you might wish to use it (terms, histogram, etc) -=== Grouping Limitations with heterogeneous indices +==== Grouping limitations with heterogeneous indices There was previously a limitation in how Rollup could handle indices that had heterogeneous mappings (multiple, unrelated/non-overlapping mappings). The recommendation at the time was to configure a separate job per data "type". For example, you might configure a separate @@ -192,7 +192,7 @@ PUT _rollup/job/combined -------------------------------------------------- // NOTCONSOLE -=== Doc counts and overlapping jobs +==== Doc counts and overlapping jobs There was previously an issue with document counts on "overlapping" job configurations, driven by the same internal implementation detail. If there were two Rollup jobs saving to the same index, where one job is a "subset" of another job, it was possible that document counts From ccfd0a18e3a2d729018ae6ff7497f2e790e59bff Mon Sep 17 00:00:00 2001 From: lcawl Date: Fri, 20 Sep 2019 13:13:52 -0700 Subject: [PATCH 2/3] [DOCS] Renames rollup transform landing page --- ...{data-operations.asciidoc => data-rollup-transform.asciidoc} | 2 +- docs/reference/index.asciidoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename docs/reference/{data-operations.asciidoc => data-rollup-transform.asciidoc} (92%) diff --git a/docs/reference/data-operations.asciidoc b/docs/reference/data-rollup-transform.asciidoc similarity index 92% rename from docs/reference/data-operations.asciidoc rename to docs/reference/data-rollup-transform.asciidoc index 13bb6d896eae0..5fe08d6f0d577 100644 --- a/docs/reference/data-operations.asciidoc +++ b/docs/reference/data-rollup-transform.asciidoc @@ -1,4 +1,4 @@ -[[data-operations]] +[[data-rollup-transform]] = Roll up or transform your data [partintro] diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc index cf41b08572e71..4ba4e7bfdd9b2 100644 --- a/docs/reference/index.asciidoc +++ b/docs/reference/index.asciidoc @@ -50,7 +50,7 @@ include::monitoring/index.asciidoc[] include::frozen-indices.asciidoc[] -include::data-operations.asciidoc[] +include::data-rollup-transform.asciidoc[] include::high-availability.asciidoc[] From b8e8da99dcabad66a5b10995847e871993779da1 Mon Sep 17 00:00:00 2001 From: lcawl Date: Fri, 20 Sep 2019 13:22:55 -0700 Subject: [PATCH 3/3] [DOCS] Fixes typo in rollup overview --- docs/reference/rollup/overview.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/rollup/overview.asciidoc b/docs/reference/rollup/overview.asciidoc index eb8815d8ad162..843cd5c058496 100644 --- a/docs/reference/rollup/overview.asciidoc +++ b/docs/reference/rollup/overview.asciidoc @@ -26,7 +26,7 @@ reading often diminishes with time. It's not useless -- it could easily contrib value often leads to deletion rather than paying the fixed storage cost. [float] -==== Rollup store historical data at reduced granularity +==== Rollup stores historical data at reduced granularity That's where Rollup comes into play. The Rollup functionality summarizes old, high-granularity data into a reduced granularity format for long-term storage. By "rolling" the data up into a single summary document, historical data