[ML] Remove old per-partition normalization code (#184)

Per-partition normalization is an old, undocumented feature that was
never used by clients. It has been superseded by per-partition maximum
scoring (see #32748).

This PR removes the now redundant code.

Relates elastic/elasticsearch#32816

Author: edsavage

bin/autodetect/CCmdLineParser.cc

@@ -52,7 +52,6 @@ bool CCmdLineParser::parse(int argc,
                            bool& memoryUsage,
                            std::size_t& bucketResultsDelay,
                            bool& multivariateByFields,
-                           bool& perPartitionNormalization,
                            TStrVec& clauseTokens) {
     try {
         boost::program_options::options_description desc(DESCRIPTION);
@@ -116,8 +115,6 @@ bool CCmdLineParser::parse(int argc,
                         "The numer of half buckets to store before choosing which overlapping bucket has the biggest anomaly")
             ("multivariateByFields",
                         "Optional flag to enable multi-variate analysis of correlated by fields")
-            ("perPartitionNormalization",
-                        "Optional flag to enable per partition normalization")
         ;
         // clang-format on
 
@@ -231,9 +228,6 @@ bool CCmdLineParser::parse(int argc,
         if (vm.count("multivariateByFields") > 0) {
             multivariateByFields = true;
         }
-        if (vm.count("perPartitionNormalization") > 0) {
-            perPartitionNormalization = true;
-        }
 
         boost::program_options::collect_unrecognized(
             parsed.options, boost::program_options::include_positional)

bin/autodetect/CCmdLineParser.h

@@ -64,7 +64,6 @@ class CCmdLineParser {
                       bool& memoryUsage,
                       std::size_t& bucketResultsDelay,
                       bool& multivariateByFields,
-                      bool& perPartitionNormalization,
                       TStrVec& clauseTokens);
 
 private:

bin/autodetect/Main.cc

@@ -88,7 +88,6 @@ int main(int argc, char** argv) {
     bool memoryUsage(false);
     std::size_t bucketResultsDelay(0);
     bool multivariateByFields(false);
-    bool perPartitionNormalization(false);
     TStrVec clauseTokens;
     if (ml::autodetect::CCmdLineParser::parse(
             argc, argv, limitConfigFile, modelConfigFile, fieldConfigFile,
@@ -98,7 +97,7 @@ int main(int argc, char** argv) {
             maxQuantileInterval, inputFileName, isInputFileNamedPipe, outputFileName,
             isOutputFileNamedPipe, restoreFileName, isRestoreFileNamedPipe, persistFileName,
             isPersistFileNamedPipe, maxAnomalyRecords, memoryUsage, bucketResultsDelay,
-            multivariateByFields, perPartitionNormalization, clauseTokens) == false) {
+            multivariateByFields, clauseTokens) == false) {
         return EXIT_FAILURE;
     }
 
@@ -146,7 +145,6 @@ int main(int argc, char** argv) {
         ml::model::CAnomalyDetectorModelConfig::defaultConfig(
             bucketSpan, summaryMode, summaryCountFieldName, latency,
             bucketResultsDelay, multivariateByFields);
-    modelConfig.perPartitionNormalization(perPartitionNormalization);
     modelConfig.detectionRules(ml::model::CAnomalyDetectorModelConfig::TIntDetectionRuleVecUMapCRef(
         fieldConfig.detectionRules()));
     modelConfig.scheduledEvents(ml::model::CAnomalyDetectorModelConfig::TStrDetectionRulePrVecCRef(

bin/normalize/CCmdLineParser.cc

@@ -30,8 +30,7 @@ bool CCmdLineParser::parse(int argc,
                            bool& isOutputFileNamedPipe,
                            std::string& quantilesState,
                            bool& deleteStateFiles,
-                           bool& writeCsv,
-                           bool& perPartitionNormalization) {
+                           bool& writeCsv) {
     try {
         boost::program_options::options_description desc(DESCRIPTION);
         // clang-format off
@@ -60,8 +59,6 @@ bool CCmdLineParser::parse(int argc,
                         "If this flag is set then delete the normalizer state files once they have been read")
             ("writeCsv",
                         "Write the results in CSV format (default is lineified JSON)")
-            ("perPartitionNormalization",
-                        "Optional flag to enable per partition normalization")
         ;
         // clang-format on
 
@@ -114,9 +111,6 @@ bool CCmdLineParser::parse(int argc,
         if (vm.count("writeCsv") > 0) {
             writeCsv = true;
         }
-        if (vm.count("perPartitionNormalization") > 0) {
-            perPartitionNormalization = true;
-        }
     } catch (std::exception& e) {
         std::cerr << "Error processing command line: " << e.what() << std::endl;
         return false;

bin/normalize/CCmdLineParser.h

@@ -43,8 +43,7 @@ class CCmdLineParser {
                       bool& isOutputFileNamedPipe,
                       std::string& quantilesState,
                       bool& deleteStateFiles,
-                      bool& writeCsv,
-                      bool& perPartitionNormalization);
+                      bool& writeCsv);
 
 private:
     static const std::string DESCRIPTION;

bin/normalize/Main.cc

@@ -54,12 +54,10 @@ int main(int argc, char** argv) {
     std::string quantilesStateFile;
     bool deleteStateFiles(false);
     bool writeCsv(false);
-    bool perPartitionNormalization(false);
     if (ml::normalize::CCmdLineParser::parse(
-            argc, argv, modelConfigFile, logProperties, logPipe, bucketSpan,
-            lengthEncodedInput, inputFileName, isInputFileNamedPipe,
-            outputFileName, isOutputFileNamedPipe, quantilesStateFile,
-            deleteStateFiles, writeCsv, perPartitionNormalization) == false) {
+            argc, argv, modelConfigFile, logProperties, logPipe, bucketSpan, lengthEncodedInput,
+            inputFileName, isInputFileNamedPipe, outputFileName, isOutputFileNamedPipe,
+            quantilesStateFile, deleteStateFiles, writeCsv) == false) {
         return EXIT_FAILURE;
     }
 
@@ -93,7 +91,6 @@ int main(int argc, char** argv) {
         LOG_FATAL(<< "Ml model config file '" << modelConfigFile << "' could not be loaded");
         return EXIT_FAILURE;
     }
-    modelConfig.perPartitionNormalization(perPartitionNormalization);
 
     // There's a choice of input and output formats for the numbers to be normalised
     using TInputParserUPtr = std::unique_ptr<ml::api::CInputParser>;

include/api/CHierarchicalResultsWriter.h

@@ -51,12 +51,7 @@ class API_EXPORT CHierarchicalResultsWriter : public model::CHierarchicalResults
     using TStr1Vec = core::CSmallVector<std::string, 1>;
 
 public:
-    enum EResultType {
-        E_SimpleCountResult,
-        E_PopulationResult,
-        E_PartitionResult,
-        E_Result
-    };
+    enum EResultType { E_SimpleCountResult, E_PopulationResult, E_Result };
     //! Type which wraps up the results of anomaly detection.
     struct API_EXPORT SResults {
         //! Construct for population results
@@ -168,9 +163,6 @@ class API_EXPORT CHierarchicalResultsWriter : public model::CHierarchicalResults
     //! pivot.
     void writePivotResult(const model::CHierarchicalResults& results, const TNode& node);
 
-    //! Write partition result if \p node is a partition level result
-    void writePartitionResult(const model::CHierarchicalResults& results, const TNode& node);
-
     //! Write out a simple count result if \p node is simple
     //! count.
     void writeSimpleCountResult(const TNode& node);

include/api/CJsonOutputWriter.h

@@ -162,9 +162,6 @@ class API_EXPORT CJsonOutputWriter : public COutputHandler {
         // when the number to write is limited
         double s_LowestBucketInfluencerScore;
 
-        //! Partition scores
-        TDocumentWeakPtrVec s_PartitionScoreDocuments;
-
         //! scheduled event descriptions
         TStr1Vec s_ScheduledEventDescriptions;
     };
@@ -304,10 +301,6 @@ class API_EXPORT CJsonOutputWriter : public COutputHandler {
     void addInfluences(const CHierarchicalResultsWriter::TStoredStringPtrStoredStringPtrPrDoublePrVec& influenceResults,
                        TDocumentWeakPtr weakDoc);
 
-    //! Write partition score & probability
-    void addPartitionScores(const CHierarchicalResultsWriter::TResults& results,
-                            TDocumentWeakPtr weakDoc);
-
 private:
     //! The job ID
     std::string m_JobId;

include/api/CResultNormalizer.h

@@ -93,15 +93,6 @@ class API_EXPORT CResultNormalizer {
                          std::string& valueFieldName,
                          double& probability);
 
-    bool parseDataFields(const TStrStrUMap& dataRowFields,
-                         std::string& level,
-                         std::string& partition,
-                         std::string& partitionValue,
-                         std::string& person,
-                         std::string& function,
-                         std::string& valueFieldName,
-                         double& probability);
-
     template<typename T>
     bool parseDataField(const TStrStrUMap& dataRowFields,
                         const std::string& fieldName,

include/model/CAnomalyDetectorModelConfig.h

@@ -418,12 +418,6 @@ class MODEL_EXPORT CAnomalyDetectorModelConfig {
     const TDoubleDoublePrVec& normalizedScoreKnotPoints() const;
     //@}
 
-    //! Check if we should create one normalizer per partition field value.
-    bool perPartitionNormalization() const;
-
-    //! Set whether we should create one normalizer per partition field value.
-    void perPartitionNormalization(bool value);
-
     //! Sets the reference to the detection rules map
     void detectionRules(TIntDetectionRuleVecUMapCRef detectionRules);
 
@@ -494,9 +488,6 @@ class MODEL_EXPORT CAnomalyDetectorModelConfig {
     //! and the normalized anomaly score with these knot points.
     //! \see DEFAULT_NORMALIZED_SCORE_KNOT_POINTS for details.
     TDoubleDoublePrVec m_NormalizedScoreKnotPoints;
-
-    //! If true then create one normalizer per partition field value.
-    bool m_PerPartitionNormalisation;
     //@}
 
     //! A reference to the map containing detection rules per