Merge branch 'master' into feature/analysis-pipeline

Author: tveasey

bin/autodetect/CCmdLineParser.cc

@@ -50,7 +50,6 @@ bool CCmdLineParser::parse(int argc,
                            bool& isPersistFileNamedPipe,
                            size_t& maxAnomalyRecords,
                            bool& memoryUsage,
-                           std::size_t& bucketResultsDelay,
                            bool& multivariateByFields,
                            TStrVec& clauseTokens) {
     try {
@@ -111,8 +110,6 @@ bool CCmdLineParser::parse(int argc,
                         "The maximum number of records to be outputted for each bucket. Defaults to 100, a value 0 removes the limit.")
             ("memoryUsage",
                         "Log the model memory usage at the end of the job")
-            ("resultFinalizationWindow", boost::program_options::value<std::size_t>(),
-                        "The numer of half buckets to store before choosing which overlapping bucket has the biggest anomaly")
             ("multivariateByFields",
                         "Optional flag to enable multi-variate analysis of correlated by fields")
         ;
@@ -222,9 +219,6 @@ bool CCmdLineParser::parse(int argc,
         if (vm.count("memoryUsage") > 0) {
             memoryUsage = true;
         }
-        if (vm.count("resultFinalizationWindow") > 0) {
-            bucketResultsDelay = vm["resultFinalizationWindow"].as<std::size_t>();
-        }
         if (vm.count("multivariateByFields") > 0) {
             multivariateByFields = true;
         }

bin/autodetect/CCmdLineParser.h

@@ -62,7 +62,6 @@ class CCmdLineParser {
                       bool& isPersistFileNamedPipe,
                       size_t& maxAnomalyRecords,
                       bool& memoryUsage,
-                      std::size_t& bucketResultsDelay,
                       bool& multivariateByFields,
                       TStrVec& clauseTokens);
 

bin/autodetect/Main.cc

@@ -86,7 +86,6 @@ int main(int argc, char** argv) {
     bool isPersistFileNamedPipe(false);
     size_t maxAnomalyRecords(100u);
     bool memoryUsage(false);
-    std::size_t bucketResultsDelay(0);
     bool multivariateByFields(false);
     TStrVec clauseTokens;
     if (ml::autodetect::CCmdLineParser::parse(
@@ -96,8 +95,8 @@ int main(int argc, char** argv) {
             timeFormat, quantilesStateFile, deleteStateFiles, persistInterval,
             maxQuantileInterval, inputFileName, isInputFileNamedPipe, outputFileName,
             isOutputFileNamedPipe, restoreFileName, isRestoreFileNamedPipe,
-            persistFileName, isPersistFileNamedPipe, maxAnomalyRecords, memoryUsage,
-            bucketResultsDelay, multivariateByFields, clauseTokens) == false) {
+            persistFileName, isPersistFileNamedPipe, maxAnomalyRecords,
+            memoryUsage, multivariateByFields, clauseTokens) == false) {
         return EXIT_FAILURE;
     }
 
@@ -143,8 +142,7 @@ int main(int argc, char** argv) {
         summaryCountFieldName.empty() ? ml::model_t::E_None : ml::model_t::E_Manual);
     ml::model::CAnomalyDetectorModelConfig modelConfig =
         ml::model::CAnomalyDetectorModelConfig::defaultConfig(
-            bucketSpan, summaryMode, summaryCountFieldName, latency,
-            bucketResultsDelay, multivariateByFields);
+            bucketSpan, summaryMode, summaryCountFieldName, latency, multivariateByFields);
     modelConfig.detectionRules(ml::model::CAnomalyDetectorModelConfig::TIntDetectionRuleVecUMapCRef(
         fieldConfig.detectionRules()));
     modelConfig.scheduledEvents(ml::model::CAnomalyDetectorModelConfig::TStrDetectionRulePrVecCRef(

build.gradle

@@ -190,16 +190,21 @@ task build(dependsOn: [check, assemble]) {
   description = 'Assemble and test the C++ part of Machine Learning'
 }
 
-task wrapper(type: Wrapper) {
-    distributionType = DistributionType.ALL
-
+/*
+ * This breaks when ml-cpp is in elasticsearch-extra
+ * and elasticsearch is still using Gradle 4.x
+ * TODO: Uncomment next time Gradle is upgraded
+ *
+wrapper {
+    distributionType = 'ALL'
     doLast {
         final DistributionLocator locator = new DistributionLocator()
         final GradleVersion version = GradleVersion.version(wrapper.gradleVersion)
         final URI distributionUri = locator.getDistributionFor(version, wrapper.distributionType.name().toLowerCase(Locale.ENGLISH))
         final URI sha256Uri = new URI(distributionUri.toString() + ".sha256")
         final String sha256Sum = new String(sha256Uri.toURL().bytes)
         wrapper.getPropertiesFile() << "distributionSha256Sum=${sha256Sum}\n"
+        println "Added checksum to wrapper properties"
     }
 }
-
+*/

docs/CHANGELOG.asciidoc

@@ -28,6 +28,30 @@
 
 //=== Regressions
 
+ == {es} version 6.6.0
+
+=== Breaking Changes
+
+=== Deprecations
+
+=== New Features
+
+=== Enhancements
+
+=== Bug Fixes
+
+Fix cause of "Sample out of bounds" error message (See {ml-pull}355[355].}
+
+=== Regressions
+
+ == {es} version 6.5.3
+
+=== Bug Fixes
+
+Correct query times for model plot and forecast in the bucket to match the times we assign
+the samples we add to the model for each bucket. For long bucket lengths, this could result
+in apparently shifted model plot with respect to the data and increased errors in forecasts. 
+
  == {es} version 6.5.0
 
 //=== Breaking Changes

gradle/wrapper/gradle-wrapper.jar

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.10-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.0-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=fc049dcbcb245d5892bebae143bd515a78f6a5a93cec99d489b312dc0ce4aad9
+distributionSha256Sum=17847c8e12b2bcfce26a79f425f082c31d4ded822f99a66127eee2d96bf18216

gradle/wrapper/gradle-wrapper.properties

@@ -28,7 +28,7 @@ APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
+DEFAULT_JVM_OPTS='"-Xmx64m"'
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"

gradlew

@@ -14,7 +14,7 @@ set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
 @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
+set DEFAULT_JVM_OPTS="-Xmx64m"
 
 @rem Find java.exe
 if defined JAVA_HOME goto findJavaFromJavaHome

gradlew.bat

@@ -12,13 +12,11 @@
 
 #include <model/CAnomalyDetector.h>
 #include <model/CAnomalyDetectorModelConfig.h>
-#include <model/CBucketQueue.h>
 #include <model/CHierarchicalResults.h>
 #include <model/CHierarchicalResultsAggregator.h>
 #include <model/CHierarchicalResultsNormalizer.h>
 #include <model/CInterimBucketCorrector.h>
 #include <model/CResourceMonitor.h>
-#include <model/CResultsQueue.h>
 #include <model/CSearchKey.h>
 
 #include <api/CDataProcessor.h>
@@ -117,26 +115,20 @@ class API_EXPORT CAnomalyJob : public CDataProcessor {
         std::pair<model::CSearchKey::TStrCRefKeyCRefPr, TAnomalyDetectorPtr>;
     using TKeyCRefAnomalyDetectorPtrPrVec = std::vector<TKeyCRefAnomalyDetectorPtrPr>;
     using TModelPlotDataVec = model::CAnomalyDetector::TModelPlotDataVec;
-    using TModelPlotDataVecCItr = TModelPlotDataVec::const_iterator;
-    using TModelPlotDataVecQueue = model::CBucketQueue<TModelPlotDataVec>;
 
     struct API_EXPORT SRestoredStateDetail {
         ERestoreStateStatus s_RestoredStateStatus;
         boost::optional<std::string> s_Extra;
     };
 
     struct SBackgroundPersistArgs {
-        SBackgroundPersistArgs(const model::CResultsQueue& resultsQueue,
-                               const TModelPlotDataVecQueue& modelPlotQueue,
-                               core_t::TTime time,
+        SBackgroundPersistArgs(core_t::TTime time,
                                const model::CResourceMonitor::SResults& modelSizeStats,
                                const model::CInterimBucketCorrector& interimBucketCorrector,
                                const model::CHierarchicalResultsAggregator& aggregator,
                                core_t::TTime latestRecordTime,
                                core_t::TTime lastResultsTime);
 
-        model::CResultsQueue s_ResultsQueue;
-        TModelPlotDataVecQueue s_ModelPlotQueue;
         core_t::TTime s_Time;
         model::CResourceMonitor::SResults s_ModelSizeStats;
         model::CInterimBucketCorrector s_InterimBucketCorrector;
@@ -224,14 +216,11 @@ class API_EXPORT CAnomalyJob : public CDataProcessor {
     void outputInterimResults(core_t::TTime bucketStartTime);
 
     //! Helper function for outputResults.
-    //! \p processingTimer is the processing time can be written to the bucket
-    //! \p sumPastProcessingTime is the total time previously spent processing
-    //! but resulted in no bucket being outputted.
+    //! \p processingTime is the processing time of the bucket
     void writeOutResults(bool interim,
                          model::CHierarchicalResults& results,
                          core_t::TTime bucketTime,
-                         uint64_t processingTime,
-                         uint64_t sumPastProcessingTime);
+                         uint64_t processingTime);
 
     //! Reset buckets in the range specified by the control message.
     void resetBuckets(const std::string& controlMessage);
@@ -259,8 +248,6 @@ class API_EXPORT CAnomalyJob : public CDataProcessor {
 
     //! Persist the detectors to a stream.
     bool persistState(const std::string& descriptionPrefix,
-                      const model::CResultsQueue& resultsQueue,
-                      const TModelPlotDataVecQueue& modelPlotQueue,
                       core_t::TTime time,
                       const TKeyCRefAnomalyDetectorPtrPrVec& detectors,
                       const model::CResourceMonitor::SResults& modelSizeStats,
@@ -296,16 +283,9 @@ class API_EXPORT CAnomalyJob : public CDataProcessor {
     //! \param[in] endTime The end of the time interval to skip sampling.
     void skipSampling(core_t::TTime endTime);
 
-    //! Outputs queued results and resets the queue to the given \p startTime
-    void flushAndResetResultsQueue(core_t::TTime startTime);
-
     //! Roll time forward to \p time
     void timeNow(core_t::TTime time);
 
-    //! Get the bucketLength, or half the bucketLength if
-    //! out-of-phase buckets are active
-    core_t::TTime effectiveBucketLength() const;
-
     //! Update configuration
     void updateConfig(const std::string& config);
 
@@ -333,15 +313,12 @@ class API_EXPORT CAnomalyJob : public CDataProcessor {
     //! specified time range.
     void generateModelPlot(core_t::TTime startTime,
                            core_t::TTime endTime,
-                           const model::CAnomalyDetector& detector);
+                           const model::CAnomalyDetector& detector,
+                           TModelPlotDataVec& modelPlotData);
 
     //! Write the pre-generated model plot to the output stream of the user's
     //! choosing: either file or streamed to the API
-    void writeOutModelPlot(core_t::TTime resultsTime);
-
-    //! Write the pre-generated model plot to the output stream of the user's
-    //! choosing: either file or streamed to the API
-    void writeOutModelPlot(core_t::TTime, CModelPlotDataJsonWriter& writer);
+    void writeOutModelPlot(const TModelPlotDataVec& modelPlotData);
 
     //! Persist one detector to a stream.
     //! This method is static so that there is no danger of it accessing
@@ -477,15 +454,6 @@ class API_EXPORT CAnomalyJob : public CDataProcessor {
     //! The hierarchical results normalizer.
     model::CHierarchicalResultsNormalizer m_Normalizer;
 
-    //! Store the last N half-buckets' results in order
-    //! to choose the best result
-    model::CResultsQueue m_ResultsQueue;
-
-    //! Also store the model plot for the buckets for each
-    //! result time - these will be output when the corresponding
-    //! result is output
-    TModelPlotDataVecQueue m_ModelPlotQueue;
-
     friend class ::CBackgroundPersisterTest;
     friend class ::CAnomalyJobTest;
 };