[ML] Pad estimated memory usage in anticipation of multi-bucket analysis (#173)

Author: tveasey

lib/api/unittest/CAnomalyJobLimitTest.cc

@@ -392,10 +392,10 @@ void CAnomalyJobLimitTest::testModelledEntityCountForFixedMemoryLimit() {
         LOG_DEBUG(<< "Memory status = " << used.s_MemoryStatus);
         LOG_DEBUG(<< "Memory usage bytes = " << used.s_Usage);
         LOG_DEBUG(<< "Memory limit bytes = " << memoryLimit * 1024 * 1024);
-        CPPUNIT_ASSERT(used.s_ByFields > 650 && used.s_ByFields < 850);
+        CPPUNIT_ASSERT(used.s_ByFields > 550 && used.s_ByFields < 750);
         CPPUNIT_ASSERT_EQUAL(std::size_t(2), used.s_PartitionFields);
         CPPUNIT_ASSERT_DOUBLES_EQUAL(memoryLimit * 1024 * 1024 / 2, used.s_Usage,
-                                     memoryLimit * 1024 * 1024 / 33); // Within 6%.
+                                     memoryLimit * 1024 * 1024 / 20); // Within 10%.
     }
 
     LOG_DEBUG(<< "**** Test partition ****");
@@ -438,7 +438,7 @@ void CAnomalyJobLimitTest::testModelledEntityCountForFixedMemoryLimit() {
         LOG_DEBUG(<< "# partition = " << used.s_PartitionFields);
         LOG_DEBUG(<< "Memory status = " << used.s_MemoryStatus);
         LOG_DEBUG(<< "Memory usage = " << used.s_Usage);
-        CPPUNIT_ASSERT(used.s_PartitionFields > 370 && used.s_PartitionFields < 470);
+        CPPUNIT_ASSERT(used.s_PartitionFields > 300 && used.s_PartitionFields < 400);
         CPPUNIT_ASSERT(static_cast<double>(used.s_ByFields) >
                        0.97 * static_cast<double>(used.s_PartitionFields));
         CPPUNIT_ASSERT_DOUBLES_EQUAL(memoryLimit * 1024 * 1024 / 2, used.s_Usage,

lib/maths/CTimeSeriesModel.cc

@@ -1167,7 +1167,14 @@ void CUnivariateTimeSeriesModel::debugMemoryUsage(core::CMemoryUsage::TMemoryUsa
     mem->setName("CUnivariateTimeSeriesModel");
     core::CMemoryDebug::dynamicSize("m_Controllers", m_Controllers, mem);
     core::CMemoryDebug::dynamicSize("m_TrendModel", m_TrendModel, mem);
+    // We made various memory improvements in 6.4 partly in preparation for multi-bucket
+    // analysis. This is not going to be available until 6.5; however, we will account for
+    // its memory now. The memory usage is used, for example, to allocate jobs to nodes and
+    // to decide if there is sufficient memory to create jobs. Operationally, we therefore
+    // want to avoid unnecessary changes in model memory between versions. An extra residual
+    // model is a good proxy for the amount of memory multi-bucket will consume.
     core::CMemoryDebug::dynamicSize("m_ResidualModel", m_ResidualModel, mem);
+    core::CMemoryDebug::dynamicSize("m_ResidualModelPad", m_ResidualModel, mem);
     core::CMemoryDebug::dynamicSize("m_AnomalyModel", m_AnomalyModel, mem);
     core::CMemoryDebug::dynamicSize("m_ChangeDetector", m_ChangeDetector, mem);
     core::CMemoryDebug::dynamicSize("m_SlidingWindow", m_SlidingWindow, mem);
@@ -1176,7 +1183,8 @@ void CUnivariateTimeSeriesModel::debugMemoryUsage(core::CMemoryUsage::TMemoryUsa
 std::size_t CUnivariateTimeSeriesModel::memoryUsage() const {
     return core::CMemory::dynamicSize(m_Controllers) +
            core::CMemory::dynamicSize(m_TrendModel) +
-           core::CMemory::dynamicSize(m_ResidualModel) +
+           /*see above*/
+           2 * core::CMemory::dynamicSize(m_ResidualModel) +
            core::CMemory::dynamicSize(m_AnomalyModel) +
            core::CMemory::dynamicSize(m_ChangeDetector) +
            core::CMemory::dynamicSize(m_SlidingWindow);
@@ -2440,15 +2448,23 @@ void CMultivariateTimeSeriesModel::debugMemoryUsage(core::CMemoryUsage::TMemoryU
     mem->setName("CUnivariateTimeSeriesModel");
     core::CMemoryDebug::dynamicSize("m_Controllers", m_Controllers, mem);
     core::CMemoryDebug::dynamicSize("m_TrendModel", m_TrendModel, mem);
+    // We made various memory improvements in 6.4 partly in preparation for multi-bucket
+    // analysis. This is not going to be available until 6.5; however, we will account for
+    // its memory now. The memory usage is used, for example, to allocate jobs to nodes and
+    // to decide if there is sufficient memory to create jobs. Operationally, we therefore
+    // want to avoid unnecessary changes in model memory between versions. An extra residual
+    // model is a good proxy for the amount of memory multi-bucket will consume.
     core::CMemoryDebug::dynamicSize("m_ResidualModel", m_ResidualModel, mem);
+    core::CMemoryDebug::dynamicSize("m_ResidualModelPad", m_ResidualModel, mem);
     core::CMemoryDebug::dynamicSize("m_AnomalyModel", m_AnomalyModel, mem);
     core::CMemoryDebug::dynamicSize("m_SlidingWindow", m_SlidingWindow, mem);
 }
 
 std::size_t CMultivariateTimeSeriesModel::memoryUsage() const {
     return core::CMemory::dynamicSize(m_Controllers) +
            core::CMemory::dynamicSize(m_TrendModel) +
-           core::CMemory::dynamicSize(m_ResidualModel) +
+           /*see above*/
+           2 * core::CMemory::dynamicSize(m_ResidualModel) +
            core::CMemory::dynamicSize(m_AnomalyModel) +
            core::CMemory::dynamicSize(m_SlidingWindow);
 }