[ML] Instrumentation for outlier detection (#1068) (#1073)

This PR implements the initial instrumentation of the outlier detection code. Instrumentation information is posted only once on completion of the outlier score computation.

Backport of #1068.

Author: valeriy42

docs/CHANGELOG.asciidoc

@@ -57,6 +57,8 @@ the build from version 2.20 to 2.34.  (See {ml-pull}1013[#1013].)
 model training. (See {ml-pull}1034[#1034].)
 * Add instrumentation information for supervised learning data frame analytics jobs.
 (See {ml-pull}1031[#1031].)
+* Add instrumentation information for outlier detection data frame analytics jobs.
+(See {ml-pull}1068[#1068].)
 
 === Bug Fixes
 

include/api/CDataFrameAnalysisInstrumentation.h

@@ -116,12 +116,22 @@ class API_EXPORT CDataFrameOutliersInstrumentation final
 public:
     explicit CDataFrameOutliersInstrumentation(const std::string& jobId)
         : CDataFrameAnalysisInstrumentation(jobId) {}
+    void parameters(const maths::COutliers::SComputeParameters& parameters) override;
+    void elapsedTime(std::uint64_t time) override;
+    void featureInfluenceThreshold(double featureInfluenceThreshold) override;
 
 protected:
     counter_t::ECounterTypes memoryCounterType() override;
 
 private:
     void writeAnalysisStats(std::int64_t timestamp) override;
+    void writeTimingStats(rapidjson::Value& parentObject);
+    void writeParameters(rapidjson::Value& parentObject);
+
+private:
+    maths::COutliers::SComputeParameters m_Parameters;
+    std::uint64_t m_ElapsedTime;
+    double m_FeatureInfluenceThreshold = -1.0;
 };
 
 //! \brief Instrumentation class for Supervised Learning jobs.

include/maths/CDataFrameAnalysisInstrumentationInterface.h

@@ -8,6 +8,7 @@
 #define INCLUDED_ml_maths_CDataFrameAnalysisInstrumentationInterface_h
 
 #include <maths/CBoostedTree.h>
+#include <maths/COutliers.h>
 #include <maths/ImportExport.h>
 
 #include <cstdint>
@@ -30,7 +31,7 @@ class MATHS_EXPORT CDataFrameAnalysisInstrumentationInterface {
     virtual ~CDataFrameAnalysisInstrumentationInterface() = default;
     //! Adds \p delta to the memory usage statistics.
     virtual void updateMemoryUsage(std::int64_t delta) = 0;
-    //! This adds \p fractionalProgess to the current progress.
+    //! This adds \p fractionalProgress to the current progress.
     //!
     //! \note The caller should try to ensure that the sum of the values added
     //! at the end of the analysis is equal to one.
@@ -59,7 +60,12 @@ class MATHS_EXPORT CDataFrameAnalysisInstrumentationInterface {
 };
 
 class MATHS_EXPORT CDataFrameOutliersInstrumentationInterface
-    : virtual public CDataFrameAnalysisInstrumentationInterface {};
+    : virtual public CDataFrameAnalysisInstrumentationInterface {
+public:
+    virtual void parameters(const maths::COutliers::SComputeParameters& parameters) = 0;
+    virtual void elapsedTime(std::uint64_t time) = 0;
+    virtual void featureInfluenceThreshold(double featureInfluenceThreshold) = 0;
+};
 
 //! \brief Instrumentation interface for Supervised Learning jobs.
 //!
@@ -105,7 +111,7 @@ class MATHS_EXPORT CDataFrameTrainBoostedTreeInstrumentationInterface
 
 public:
     virtual ~CDataFrameTrainBoostedTreeInstrumentationInterface() = default;
-    //! Supevised learning job \p type, can be E_Regression or E_Classification.
+    //! Supervised learning job \p type, can be E_Regression or E_Classification.
     virtual void type(EStatsType type) = 0;
     //! Current \p iteration number.
     virtual void iteration(std::size_t iteration) = 0;
@@ -126,6 +132,9 @@ class MATHS_EXPORT CDataFrameOutliersInstrumentationStub
     void updateMemoryUsage(std::int64_t) override {}
     void updateProgress(double) override {}
     void nextStep(const std::string& /* phase */) override {}
+    void parameters(const maths::COutliers::SComputeParameters& /* parameters */) override {}
+    void elapsedTime(std::uint64_t /* time */) override {}
+    void featureInfluenceThreshold(double /* featureInfluenceThreshold */) override {}
 };
 
 //! \brief Dummies out all instrumentation for supervised learning.

include/maths/COutliers.h

@@ -14,7 +14,6 @@
 #include <core/Concurrency.h>
 
 #include <maths/CBasicStatistics.h>
-#include <maths/CDataFrameAnalysisInstrumentationInterface.h>
 #include <maths/CKdTree.h>
 #include <maths/CLinearAlgebraShims.h>
 #include <maths/COrthogonaliser.h>
@@ -34,6 +33,9 @@
 
 namespace ml {
 namespace maths {
+
+class CDataFrameOutliersInstrumentationInterface;
+
 namespace outliers_detail {
 using TDoubleVec = std::vector<double>;
 using TDouble1Vec = core::CSmallVector<double, 1>;
@@ -657,6 +659,12 @@ class MATHS_EXPORT COutliers : private core::CNonInstantiatable {
     template<typename POINT>
     using TAnnotatedPoint = CAnnotatedVector<POINT, std::size_t>;
 
+    static const std::string LOF;
+    static const std::string LDOF;
+    static const std::string DISTANCE_KNN;
+    static const std::string TOTAL_DISTANCE_KNN;
+    static const std::string ENSEMBLE;
+
     //! The outlier detection methods which are available.
     enum EMethod {
         E_Lof,
@@ -680,7 +688,7 @@ class MATHS_EXPORT COutliers : private core::CNonInstantiatable {
         std::size_t s_NumberNeighbours;
         //! If true also compute the feature influence.
         bool s_ComputeFeatureInfluence;
-        //! The fraction of true outliers amoung the points.
+        //! The fraction of true outliers among the points.
         double s_OutlierFraction;
     };
 
@@ -710,6 +718,9 @@ class MATHS_EXPORT COutliers : private core::CNonInstantiatable {
                                                    std::size_t partitionNumberPoints,
                                                    std::size_t dimension);
 
+    //! Return string representation of the \p method.
+    static std::string print(EMethod method);
+
     //! \name Test Interface
     //@{
     //! Compute the normalized LOF scores for \p points.

lib/api/CDataFrameAnalysisInstrumentation.cc

@@ -24,6 +24,7 @@ const std::string ITERATION_TAG{"iteration"};
 const std::string JOB_ID_TAG{"job_id"};
 const std::string MEMORY_TYPE_TAG{"analytics_memory_usage"};
 const std::string OUTLIER_DETECTION_STATS{"outlier_detection_stats"};
+const std::string PARAMETERS_TAG{"parameters"};
 const std::string PEAK_MEMORY_USAGE_TAG{"peak_usage_bytes"};
 const std::string PROGRESS_TAG{"progress"};
 const std::string REGRESSION_STATS_TAG{"regression_stats"};
@@ -59,7 +60,7 @@ const std::string REGULARIZATION_TREE_SIZE_PENALTY_MULTIPLIER_TAG{"regularizatio
 
 // Outlier detection parameters
 const std::string N_NEIGHBORS{"n_neighbors"};
-const std::string METHODS{"methods"};
+const std::string METHOD{"method"};
 const std::string COMPUTE_FEATURE_INFLUENCE{"compute_feature_influence"};
 const std::string FEATURE_INFLUENCE_THRESHOLD{"feature_influence_threshold"};
 const std::string OUTLIER_FRACTION{"outlier_fraction"};
@@ -114,7 +115,7 @@ void CDataFrameAnalysisInstrumentation::resetProgress() {
 }
 
 void CDataFrameAnalysisInstrumentation::nextStep(const std::string& /* phase */) {
-    // TODO reactivate once Java part is ready
+    // reactivate once java side is ready
     // this->writeState();
 }
 
@@ -171,10 +172,69 @@ void CDataFrameOutliersInstrumentation::writeAnalysisStats(std::int64_t timestam
         writer->String(this->jobId());
         writer->Key(TIMESTAMP_TAG);
         writer->Int64(timestamp);
+
+        rapidjson::Value parametersObject{writer->makeObject()};
+        this->writeParameters(parametersObject);
+        writer->Key(PARAMETERS_TAG);
+        writer->write(parametersObject);
+
+        rapidjson::Value timingStatsObject{writer->makeObject()};
+        this->writeTimingStats(timingStatsObject);
+        writer->Key(TIMING_STATS_TAG);
+        writer->write(timingStatsObject);
+
         writer->EndObject();
     }
 }
 
+void CDataFrameOutliersInstrumentation::parameters(const maths::COutliers::SComputeParameters& parameters) {
+    m_Parameters = parameters;
+}
+
+void CDataFrameOutliersInstrumentation::elapsedTime(std::uint64_t time) {
+    m_ElapsedTime = time;
+}
+
+void CDataFrameOutliersInstrumentation::featureInfluenceThreshold(double featureInfluenceThreshold) {
+    m_FeatureInfluenceThreshold = featureInfluenceThreshold;
+}
+
+void CDataFrameOutliersInstrumentation::writeTimingStats(rapidjson::Value& parentObject) {
+    auto* writer = this->writer();
+    if (writer != nullptr) {
+        writer->addMember(TIMING_ELAPSED_TIME_TAG,
+                          rapidjson::Value(m_ElapsedTime).Move(), parentObject);
+    }
+}
+
+void CDataFrameOutliersInstrumentation::writeParameters(rapidjson::Value& parentObject) {
+    auto* writer = this->writer();
+
+    if (writer != nullptr) {
+
+        writer->addMember(
+            N_NEIGHBORS,
+            rapidjson::Value(static_cast<std::uint64_t>(this->m_Parameters.s_NumberNeighbours))
+                .Move(),
+            parentObject);
+        writer->addMember(
+            COMPUTE_FEATURE_INFLUENCE,
+            rapidjson::Value(this->m_Parameters.s_ComputeFeatureInfluence).Move(),
+            parentObject);
+        writer->addMember(OUTLIER_FRACTION,
+                          rapidjson::Value(this->m_Parameters.s_OutlierFraction).Move(),
+                          parentObject);
+        writer->addMember(FEATURE_INFLUENCE_THRESHOLD,
+                          rapidjson::Value(this->m_FeatureInfluenceThreshold).Move(),
+                          parentObject);
+        writer->addMember(
+            STANDARDIZATION_ENABLED,
+            rapidjson::Value(this->m_Parameters.s_StandardizeColumns).Move(), parentObject);
+        writer->addMember(METHOD, maths::COutliers::print(this->m_Parameters.s_Method),
+                          parentObject);
+    }
+}
+
 void CDataFrameTrainBoostedTreeInstrumentation::type(EStatsType type) {
     m_Type = type;
 }

lib/api/CDataFrameOutliersRunner.cc

@@ -28,21 +28,17 @@ namespace api {
 namespace {
 const CDataFrameAnalysisConfigReader& parameterReader() {
     static const CDataFrameAnalysisConfigReader PARAMETER_READER{[] {
-        const std::string lof{"lof"};
-        const std::string ldof{"ldof"};
-        const std::string knn{"distance_kth_nn"};
-        const std::string tnn{"distance_knn"};
         CDataFrameAnalysisConfigReader theReader;
         theReader.addParameter(CDataFrameOutliersRunner::STANDARDIZATION_ENABLED,
                                CDataFrameAnalysisConfigReader::E_OptionalParameter);
         theReader.addParameter(CDataFrameOutliersRunner::N_NEIGHBORS,
                                CDataFrameAnalysisConfigReader::E_OptionalParameter);
-        theReader.addParameter(CDataFrameOutliersRunner::METHOD,
-                               CDataFrameAnalysisConfigReader::E_OptionalParameter,
-                               {{lof, int{maths::COutliers::E_Lof}},
-                                {ldof, int{maths::COutliers::E_Ldof}},
-                                {knn, int{maths::COutliers::E_DistancekNN}},
-                                {tnn, int{maths::COutliers::E_TotalDistancekNN}}});
+        theReader.addParameter(
+            CDataFrameOutliersRunner::METHOD, CDataFrameAnalysisConfigReader::E_OptionalParameter,
+            {{maths::COutliers::LOF, int{maths::COutliers::E_Lof}},
+             {maths::COutliers::LDOF, int{maths::COutliers::E_Ldof}},
+             {maths::COutliers::DISTANCE_KNN, int{maths::COutliers::E_DistancekNN}},
+             {maths::COutliers::TOTAL_DISTANCE_KNN, int{maths::COutliers::E_TotalDistancekNN}}});
         theReader.addParameter(CDataFrameOutliersRunner::COMPUTE_FEATURE_INFLUENCE,
                                CDataFrameAnalysisConfigReader::E_OptionalParameter);
         theReader.addParameter(CDataFrameOutliersRunner::FEATURE_INFLUENCE_THRESHOLD,
@@ -69,6 +65,8 @@ CDataFrameOutliersRunner::CDataFrameOutliersRunner(const CDataFrameAnalysisSpeci
     m_ComputeFeatureInfluence = parameters[COMPUTE_FEATURE_INFLUENCE].fallback(true);
     m_FeatureInfluenceThreshold = parameters[FEATURE_INFLUENCE_THRESHOLD].fallback(0.1);
     m_OutlierFraction = parameters[OUTLIER_FRACTION].fallback(0.05);
+
+    m_Instrumentation.featureInfluenceThreshold(m_FeatureInfluenceThreshold);
 }
 
 CDataFrameOutliersRunner::CDataFrameOutliersRunner(const CDataFrameAnalysisSpecification& spec)