[ML] Add new categorization stats to model_size_stats (#989)

This change adds support for the following new model_size_stats
fields:

- categorized_doc_count
- total_category_count
- frequent_category_count
- rare_category_count
- dead_category_count
- categorization_status

Relates elastic/elasticsearch#50749

Author: David Roberts

docs/CHANGELOG.asciidoc

@@ -46,6 +46,8 @@ progress, memory usage, etc. (See {ml-pull}906[#906].)
 
 * Improve initialization of learn rate for better and more stable results in regression
 and classification. (See {ml-pull}948[#948].)
+* Add new model_size_stats fields to instrument categorization.  (See {ml-pull}948[#948]
+and {pull}51879[#51879], issue: {issue}50794[#50749].)
 
 === Bug Fixes
 

include/api/CAnomalyJob.h

@@ -120,14 +120,14 @@ class API_EXPORT CAnomalyJob : public CDataProcessor {
 
     struct SBackgroundPersistArgs {
         SBackgroundPersistArgs(core_t::TTime time,
-                               const model::CResourceMonitor::SResults& modelSizeStats,
+                               const model::CResourceMonitor::SModelSizeStats& modelSizeStats,
                                const model::CInterimBucketCorrector& interimBucketCorrector,
                                const model::CHierarchicalResultsAggregator& aggregator,
                                core_t::TTime latestRecordTime,
                                core_t::TTime lastResultsTime);
 
         core_t::TTime s_Time;
-        model::CResourceMonitor::SResults s_ModelSizeStats;
+        model::CResourceMonitor::SModelSizeStats s_ModelSizeStats;
         model::CInterimBucketCorrector s_InterimBucketCorrector;
         model::CHierarchicalResultsAggregator s_Aggregator;
         std::string s_NormalizerState;
@@ -258,7 +258,7 @@ class API_EXPORT CAnomalyJob : public CDataProcessor {
     bool persistCopiedState(const std::string& descriptionPrefix,
                             core_t::TTime time,
                             const TKeyCRefAnomalyDetectorPtrPrVec& detectors,
-                            const model::CResourceMonitor::SResults& modelSizeStats,
+                            const model::CResourceMonitor::SModelSizeStats& modelSizeStats,
                             const model::CInterimBucketCorrector& interimBucketCorrector,
                             const model::CHierarchicalResultsAggregator& aggregator,
                             const std::string& normalizerState,

include/api/CJsonOutputWriter.h

@@ -232,7 +232,7 @@ class API_EXPORT CJsonOutputWriter : public COutputHandler {
 
     //! Report the current levels of resource usage, as given to us
     //! from the CResourceMonitor via a callback
-    void reportMemoryUsage(const model::CResourceMonitor::SResults& results);
+    void reportMemoryUsage(const model::CResourceMonitor::SModelSizeStats& modelSizeStats);
 
     //! Acknowledge a flush request by echoing back the flush ID
     void acknowledgeFlush(const std::string& flushId, core_t::TTime lastFinalizedBucketEnd);

include/api/CModelSizeStatsJsonWriter.h

@@ -24,7 +24,7 @@ class API_EXPORT CModelSizeStatsJsonWriter : private core::CNonInstantiatable {
 public:
     //! Writes the model size stats in the \p results in JSON format.
     static void write(const std::string& jobId,
-                      const model::CResourceMonitor::SResults& results,
+                      const model::CResourceMonitor::SModelSizeStats& results,
                       core::CRapidJsonConcurrentLineWriter& writer);
 };
 }

include/api/CModelSnapshotJsonWriter.h

@@ -33,7 +33,7 @@ class API_EXPORT CModelSnapshotJsonWriter {
         std::string s_Description;
         std::string s_SnapshotId;
         size_t s_NumDocs;
-        model::CResourceMonitor::SResults s_ModelSizeStats;
+        model::CResourceMonitor::SModelSizeStats s_ModelSizeStats;
         std::string s_NormalizerState;
         core_t::TTime s_LatestRecordTime;
         core_t::TTime s_LatestFinalResultTime;

include/model/CAnomalyDetector.h

@@ -270,9 +270,9 @@ class MODEL_EXPORT CAnomalyDetector : public CMonitoredResource {
     //! Prune the model.
     void prune(std::size_t maximumAge) override;
 
-    //! Update the overall model memory stats results with stats from this
-    //! anomaly detector.
-    void updateMemoryResults(CResourceMonitor::SResults& results) const override;
+    //! Update the overall model size stats with information from this anomaly
+    //! detector.
+    void updateModelSizeStats(CResourceMonitor::SModelSizeStats& modelSizeStats) const override;
 
     //! Get end of the last complete bucket we've observed.
     const core_t::TTime& lastBucketEndTime() const;

include/model/CMonitoredResource.h

@@ -58,9 +58,10 @@ class MODEL_EXPORT CMonitoredResource {
     //! discarding the least recently seen entities it knows about.
     virtual void prune(std::size_t maximumAge);
 
-    //! Update the overall model memory stats results with stats from this
+    //! Update the overall model size stats results with stats from this
     //! monitored resource.
-    virtual void updateMemoryResults(CResourceMonitor::SResults& results) const = 0;
+    virtual void
+    updateModelSizeStats(CResourceMonitor::SModelSizeStats& modelSizeStats) const = 0;
 };
 }
 }

include/model/CResourceMonitor.h

@@ -38,23 +38,30 @@ class CMonitoredResource;
 //! Assess memory used by models and decide on further memory allocations.
 class MODEL_EXPORT CResourceMonitor {
 public:
-    struct MODEL_EXPORT SResults {
-        std::size_t s_Usage;
-        std::size_t s_AdjustedUsage;
-        std::size_t s_ByFields;
-        std::size_t s_PartitionFields;
-        std::size_t s_OverFields;
-        std::size_t s_AllocationFailures;
-        model_t::EMemoryStatus s_MemoryStatus;
-        core_t::TTime s_BucketStartTime;
-        std::size_t s_BytesExceeded;
-        std::size_t s_BytesMemoryLimit;
+    struct MODEL_EXPORT SModelSizeStats {
+        std::size_t s_Usage = 0;
+        std::size_t s_AdjustedUsage = 0;
+        std::size_t s_ByFields = 0;
+        std::size_t s_PartitionFields = 0;
+        std::size_t s_OverFields = 0;
+        std::size_t s_AllocationFailures = 0;
+        model_t::EMemoryStatus s_MemoryStatus = model_t::E_MemoryStatusOk;
+        core_t::TTime s_BucketStartTime = 0;
+        std::size_t s_BytesExceeded = 0;
+        std::size_t s_BytesMemoryLimit = 0;
+        std::size_t s_CategorizedMessages = 0;
+        std::size_t s_TotalCategories = 0;
+        std::size_t s_FrequentCategories = 0;
+        std::size_t s_RareCategories = 0;
+        std::size_t s_DeadCategories = 0;
+        model_t::ECategorizationStatus s_CategorizationStatus = model_t::E_CategorizationStatusOk;
     };
 
 public:
     using TMonitoredResourcePtrSizeUMap =
         boost::unordered_map<CMonitoredResource*, std::size_t>;
-    using TMemoryUsageReporterFunc = std::function<void(const CResourceMonitor::SResults&)>;
+    using TMemoryUsageReporterFunc =
+        std::function<void(const CResourceMonitor::SModelSizeStats&)>;
     using TTimeSizeMap = std::map<core_t::TTime, std::size_t>;
 
     //! The minimum time between prunes
@@ -109,7 +116,7 @@ class MODEL_EXPORT CResourceMonitor {
     void sendMemoryUsageReport(core_t::TTime bucketStartTime);
 
     //! Create a memory usage report
-    SResults createMemoryUsageReport(core_t::TTime bucketStartTime);
+    SModelSizeStats createMemoryUsageReport(core_t::TTime bucketStartTime);
 
     //! We are being told that a class has failed to allocate memory
     //! based on the resource limits, and we will report this to the

include/model/CTokenListCategory.h

@@ -10,6 +10,7 @@
 
 #include <model/ImportExport.h>
 
+#include <algorithm>
 #include <map>
 #include <string>
 #include <utility>
@@ -94,16 +95,38 @@ class MODEL_EXPORT CTokenListCategory {
     //! this category's common unique tokens?
     std::size_t missingCommonTokenWeight(const TSizeSizeMap& uniqueTokenIds) const;
 
-    //! Is the weight of tokens in a given map that are missing from this
-    //! category's common unique tokens equal to zero?  It is possible to test:
+    //! Is the weight of tokens in the provided container that are missing from
+    //! this category's common unique tokens equal to zero?  It is possible to
+    //! test:
     //!     if (category.missingCommonTokenWeight(uniqueTokenIds) == 0)
     //! instead of calling this method.  However, this method is much faster
     //! as it can return false as soon as a mismatch occurs.
-    bool isMissingCommonTokenWeightZero(const TSizeSizeMap& uniqueTokenIds) const;
+    //! \param uniqueTokenIds A container of pairs where the first element is
+    //!                       a token ID and the container is sorted into
+    //!                       ascending token ID order.
+    template<typename PAIR_CONTAINER>
+    bool isMissingCommonTokenWeightZero(const PAIR_CONTAINER& uniqueTokenIds) const {
+
+        auto testIter = uniqueTokenIds.begin();
+        for (const auto& commonItem : m_CommonUniqueTokenIds) {
+            testIter = std::find_if(testIter, uniqueTokenIds.end(),
+                                    [&commonItem](const auto& testItem) {
+                                        return testItem.first >= commonItem.first;
+                                    });
+            if (testIter == uniqueTokenIds.end() ||
+                testIter->first != commonItem.first ||
+                testIter->second != commonItem.second) {
+                return false;
+            }
+            ++testIter;
+        }
+
+        return true;
+    }
 
     //! Does the supplied token vector contain all our common tokens in the
     //! same order as our base token vector?
-    bool containsCommonTokensInOrder(const TSizeSizePrVec& tokenIds) const;
+    bool containsCommonInOrderTokensInOrder(const TSizeSizePrVec& tokenIds) const;
 
     //! \return Does the supplied token ID represent a common unique token?
     bool isTokenCommon(std::size_t tokenId) const;

include/model/CTokenListDataCategorizer.h

@@ -81,12 +81,6 @@ class CTokenListDataCategorizer : public CTokenListDataCategorizerBase {
     //! Get the static size of this object - used for virtual hierarchies
     std::size_t staticSize() const override { return sizeof(*this); }
 
-    //! Currently the overall model memory stats do not contain any categorizer
-    //! stats fields.
-    void updateMemoryResults(CResourceMonitor::SResults& /*results*/) const override {
-        // NO-OP
-    }
-
 protected:
     //! Split the string into a list of tokens.  The result of the
     //! tokenisation is returned in \p tokenIds, \p tokenUniqueIds and