From 32633b6a42793f9cc3748ba89171585293df3934 Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Wed, 21 Feb 2018 18:40:18 +0000 Subject: [PATCH 1/4] Improvements to trend modelling and periodicity testing for forecasting. --- include/maths/CAdaptiveBucketing.h | 38 +- include/maths/CBasicStatistics.h | 23 + .../CCalendarComponentAdaptiveBucketing.h | 13 +- include/maths/CDecompositionComponent.h | 24 +- include/maths/CExpandingWindow.h | 135 + include/maths/CGammaRateConjugate.h | 23 +- include/maths/CLogNormalMeanPrecConjugate.h | 21 +- include/maths/CModel.h | 8 +- include/maths/CPeriodicityHypothesisTests.h | 439 +++ include/maths/CPrior.h | 29 +- include/maths/CRegression.h | 91 +- include/maths/CRegressionDetail.h | 7 + include/maths/CSeasonalComponent.h | 38 +- .../CSeasonalComponentAdaptiveBucketing.h | 70 +- include/maths/CSeasonalTime.h | 42 +- include/maths/CTimeSeriesDecomposition.h | 66 +- .../maths/CTimeSeriesDecompositionDetail.h | 482 +-- .../maths/CTimeSeriesDecompositionInterface.h | 75 +- include/maths/CTimeSeriesDecompositionStub.h | 19 +- include/maths/CTimeSeriesModel.h | 39 +- include/maths/CTools.h | 23 + include/maths/CTrendComponent.h | 207 ++ include/maths/CTrendTests.h | 586 ---- include/maths/Constants.h | 58 +- include/model/CAnomalyDetectorModelConfig.h | 17 - include/model/CModelParams.h | 9 - lib/maths/CAdaptiveBucketing.cc | 64 +- .../CCalendarComponentAdaptiveBucketing.cc | 13 +- lib/maths/CExpandingWindow.cc | 200 ++ lib/maths/CGammaRateConjugate.cc | 30 +- lib/maths/CLogNormalMeanPrecConjugate.cc | 30 +- lib/maths/CModel.cc | 4 +- lib/maths/CMultimodalPrior.cc | 5 +- lib/maths/COneOfNPrior.cc | 5 +- lib/maths/CPeriodicityHypothesisTests.cc | 1949 ++++++++++++ lib/maths/CPrior.cc | 20 +- lib/maths/CQuantileSketch.cc | 5 +- lib/maths/CRegression.cc | 2 - lib/maths/CSeasonalComponent.cc | 60 +- .../CSeasonalComponentAdaptiveBucketing.cc | 458 +-- lib/maths/CSeasonalTime.cc | 41 +- lib/maths/CSignal.cc | 11 +- lib/maths/CStatisticalTests.cc | 16 + lib/maths/CTimeSeriesDecomposition.cc | 422 ++- lib/maths/CTimeSeriesDecompositionDetail.cc | 2036 ++++-------- .../CTimeSeriesDecompositionInterface.cc | 152 - lib/maths/CTimeSeriesDecompositionStub.cc | 24 +- lib/maths/CTimeSeriesModel.cc | 580 ++-- lib/maths/CTrendComponent.cc | 561 ++++ lib/maths/CTrendTests.cc | 1982 +----------- lib/maths/CXMeansOnline1d.cc | 118 +- lib/maths/Makefile | 4 +- lib/maths/unittest/CForecastTest.cc | 592 ++-- lib/maths/unittest/CForecastTest.h | 35 +- lib/maths/unittest/CGammaRateConjugateTest.cc | 9 +- .../CLogNormalMeanPrecConjugateTest.cc | 13 +- lib/maths/unittest/CMathsMemoryTest.cc | 1 - lib/maths/unittest/CMultimodalPriorTest.cc | 13 +- lib/maths/unittest/COneOfNPriorTest.cc | 6 +- .../CPeriodicityHypothesisTestsTest.cc | 720 +++++ .../CPeriodicityHypothesisTestsTest.h | 33 + lib/maths/unittest/CRegressionTest.cc | 8 +- ...CSeasonalComponentAdaptiveBucketingTest.cc | 112 +- .../CSeasonalComponentAdaptiveBucketingTest.h | 1 + lib/maths/unittest/CSeasonalComponentTest.cc | 32 +- .../unittest/CTimeSeriesDecompositionTest.cc | 507 +-- .../unittest/CTimeSeriesDecompositionTest.h | 5 +- lib/maths/unittest/CTimeSeriesModelTest.cc | 698 ++-- lib/maths/unittest/CTimeSeriesModelTest.h | 1 + lib/maths/unittest/CTrendComponentTest.cc | 477 +++ lib/maths/unittest/CTrendComponentTest.h | 32 + lib/maths/unittest/CTrendTestsTest.cc | 1111 +------ lib/maths/unittest/CTrendTestsTest.h | 5 - lib/maths/unittest/Main.cc | 4 + lib/maths/unittest/Makefile | 4 +- lib/maths/unittest/TestUtils.cc | 111 +- lib/maths/unittest/TestUtils.h | 120 +- ...TimeSeriesModel.6.2.expected_intervals.txt | 1 + ...CMultivariateTimeSeriesModel.6.2.state.xml | 2800 +++++++++++++++++ ...alComponentAdaptiveBucketing.6.2.state.xml | 49 + ...mposition.6.2.seasonal.expected_scales.txt | 1 + ...mposition.6.2.seasonal.expected_values.txt | 1 + ...SeriesDecomposition.6.2.seasonal.state.xml | 1198 +++++++ ...6.2.trend_and_seasonal.expected_scales.txt | 1 + ...6.2.trend_and_seasonal.expected_values.txt | 1 + ...mposition.6.2.trend_and_seasonal.state.xml | 560 ++++ ...TimeSeriesModel.6.2.expected_intervals.txt | 1 + .../CUnivariateTimeSeriesModel.6.2.state.xml | 986 ++++++ lib/model/CAnomalyDetector.cc | 2 +- lib/model/CAnomalyDetectorModel.cc | 6 +- lib/model/CAnomalyDetectorModelConfig.cc | 3 - lib/model/CEventRateModel.cc | 2 +- lib/model/CEventRateModelFactory.cc | 6 +- lib/model/CEventRatePopulationModel.cc | 12 +- lib/model/CEventRatePopulationModelFactory.cc | 6 +- lib/model/CMetricModelFactory.cc | 4 +- lib/model/CMetricPopulationModel.cc | 20 +- lib/model/CMetricPopulationModelFactory.cc | 4 +- lib/model/CModelDetailsView.cc | 2 +- lib/model/CModelParams.cc | 3 - .../unittest/CEventRateAnomalyDetectorTest.cc | 40 +- lib/model/unittest/CEventRateModelTest.cc | 48 +- .../unittest/CEventRatePopulationModelTest.cc | 13 +- .../unittest/CMetricAnomalyDetectorTest.cc | 2 +- lib/model/unittest/CMetricModelTest.cc | 94 +- .../unittest/CMetricPopulationModelTest.cc | 7 +- 106 files changed, 13969 insertions(+), 7928 deletions(-) create mode 100644 include/maths/CExpandingWindow.h create mode 100644 include/maths/CPeriodicityHypothesisTests.h create mode 100644 include/maths/CTrendComponent.h create mode 100644 lib/maths/CExpandingWindow.cc create mode 100644 lib/maths/CPeriodicityHypothesisTests.cc delete mode 100644 lib/maths/CTimeSeriesDecompositionInterface.cc create mode 100644 lib/maths/CTrendComponent.cc create mode 100644 lib/maths/unittest/CPeriodicityHypothesisTestsTest.cc create mode 100644 lib/maths/unittest/CPeriodicityHypothesisTestsTest.h create mode 100644 lib/maths/unittest/CTrendComponentTest.cc create mode 100644 lib/maths/unittest/CTrendComponentTest.h create mode 100644 lib/maths/unittest/testfiles/CMultivariateTimeSeriesModel.6.2.expected_intervals.txt create mode 100644 lib/maths/unittest/testfiles/CMultivariateTimeSeriesModel.6.2.state.xml create mode 100644 lib/maths/unittest/testfiles/CSeasonalComponentAdaptiveBucketing.6.2.state.xml create mode 100644 lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.seasonal.expected_scales.txt create mode 100644 lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.seasonal.expected_values.txt create mode 100644 lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.seasonal.state.xml create mode 100644 lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.trend_and_seasonal.expected_scales.txt create mode 100644 lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.trend_and_seasonal.expected_values.txt create mode 100644 lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.trend_and_seasonal.state.xml create mode 100644 lib/maths/unittest/testfiles/CUnivariateTimeSeriesModel.6.2.expected_intervals.txt create mode 100644 lib/maths/unittest/testfiles/CUnivariateTimeSeriesModel.6.2.state.xml diff --git a/include/maths/CAdaptiveBucketing.h b/include/maths/CAdaptiveBucketing.h index 0187452406..60fa23fc00 100644 --- a/include/maths/CAdaptiveBucketing.h +++ b/include/maths/CAdaptiveBucketing.h @@ -80,12 +80,10 @@ namespace maths class MATHS_EXPORT CAdaptiveBucketing { public: - typedef std::vector TDoubleVec; - typedef std::vector TFloatVec; - typedef std::pair TTimeTimePr; - typedef CBasicStatistics::SSampleMeanVar::TAccumulator TDoubleMeanVarAccumulator; - typedef std::pair TTimeTimePrMeanVarPr; - typedef std::vector TTimeTimePrMeanVarPrVec; + using TDoubleVec = std::vector; + using TFloatVec = std::vector; + using TFloatMeanAccumulator = CBasicStatistics::SSampleMean::TAccumulator; + using TFloatMeanAccumulatorVec = std::vector; public: //! Restore by traversing a state document @@ -116,14 +114,17 @@ class MATHS_EXPORT CAdaptiveBucketing //! \param[in] n The number of buckets. bool initialize(double a, double b, std::size_t n); - //! Add the function moments \f$([a_i,b_i], S_i)\f$ where - //! \f$S_i\f$ are the means and variances of the function - //! in the time intervals \f$([a_i,b_i])\f$. + //! Add the function mean values \f$([a_i,b_i], m_i)\f$ where + //! \f$m_i\f$ are the means of the function in the time intervals + //! \f$([a+(i-1)l,b+il])\f$, \f$i\in[n]\f$ and \f$l=(b-a)/n\f$. //! - //! \param[in] time The start of the period including \p values. - //! \param[in] values Time ranges and the corresponding function - //! value moments. - void initialValues(core_t::TTime time, const TTimeTimePrMeanVarPrVec &values); + //! \param[in] startTime The start of the period. + //! \param[in] endTime The start of the period. + //! \param[in] values The mean values in a regular subdivision + //! of [\p start,\p end]. + void initialValues(core_t::TTime startTime, + core_t::TTime endTime, + const TFloatMeanAccumulatorVec &values); //! Get the number of buckets. std::size_t size(void) const; @@ -204,9 +205,6 @@ class MATHS_EXPORT CAdaptiveBucketing //! Get the memory used by this component std::size_t memoryUsage(void) const; - private: - typedef CBasicStatistics::SSampleMean::TAccumulator TFloatMeanAccumulator; - private: //! Compute the values corresponding to the change in end //! points from \p endpoints. The values are assigned based @@ -214,11 +212,11 @@ class MATHS_EXPORT CAdaptiveBucketing //! bucket configuration. virtual void refresh(const TFloatVec &endpoints) = 0; + //! Check if \p time is in the this component's window. + virtual bool inWindow(core_t::TTime time) const = 0; + //! Add the function value at \p time. - virtual void add(std::size_t bucket, - core_t::TTime time, - double offset, - const TDoubleMeanVarAccumulator &value) = 0; + virtual void add(std::size_t bucket, core_t::TTime time, double value, double weight) = 0; //! Get the offset w.r.t. the start of the bucketing of \p time. virtual double offset(core_t::TTime time) const = 0; diff --git a/include/maths/CBasicStatistics.h b/include/maths/CBasicStatistics.h index 986ca0d74c..51fbca5b13 100644 --- a/include/maths/CBasicStatistics.h +++ b/include/maths/CBasicStatistics.h @@ -85,6 +85,23 @@ class MATHS_EXPORT CBasicStatistics //! Compute the sample median. static double median(const TDoubleVec &dataIn); + //! Compute the maximum of \p first, \p second and \p third. + template + static T max(T first, T second, T third) + { + return first >= second ? + (third >= first ? third : first) : + (third >= second ? third : second); + } + + //! Compute the minimum of \p first, \p second and \p third. + template + static T min(T first, T second, T third) + { + return first <= second ? + (third <= first ? third : first) : + (third <= second ? third : second); + } /////////////////////////// ACCUMULATORS /////////////////////////// @@ -1620,6 +1637,12 @@ class MATHS_EXPORT CBasicStatistics return m_Max[0]; } + //! Get the range. + T range(void) const + { + return m_Max[0] - m_Min[0]; + } + //! Get the margin by which all the values have the same sign. T signMargin(void) const { diff --git a/include/maths/CCalendarComponentAdaptiveBucketing.h b/include/maths/CCalendarComponentAdaptiveBucketing.h index 4e8250d97e..89a6f360f7 100644 --- a/include/maths/CCalendarComponentAdaptiveBucketing.h +++ b/include/maths/CCalendarComponentAdaptiveBucketing.h @@ -47,8 +47,7 @@ class CSeasonalTime; class MATHS_EXPORT CCalendarComponentAdaptiveBucketing : private CAdaptiveBucketing { public: - typedef CAdaptiveBucketing::TTimeTimePrMeanVarPrVec TTimeTimePrMeanVarPrVec; - typedef CBasicStatistics::SSampleMeanVar::TAccumulator TFloatMeanVarAccumulator; + using TFloatMeanVarAccumulator = CBasicStatistics::SSampleMeanVar::TAccumulator; public: CCalendarComponentAdaptiveBucketing(void); @@ -160,7 +159,7 @@ class MATHS_EXPORT CCalendarComponentAdaptiveBucketing : private CAdaptiveBucket //@} private: - typedef std::vector TFloatMeanVarVec; + using TFloatMeanVarVec = std::vector; private: //! Restore by traversing a state document @@ -174,11 +173,11 @@ class MATHS_EXPORT CCalendarComponentAdaptiveBucketing : private CAdaptiveBucket //! \param[in] endpoints The old end points. void refresh(const TFloatVec &endpoints); + //! Check if \p time is in the this component's window. + virtual bool inWindow(core_t::TTime time) const; + //! Add the function value to \p bucket. - virtual void add(std::size_t bucket, - core_t::TTime time, - double offset, - const TDoubleMeanVarAccumulator &value); + virtual void add(std::size_t bucket, core_t::TTime time, double value, double weight); //! Get the offset w.r.t. the start of the bucketing of \p time. virtual double offset(core_t::TTime time) const; diff --git a/include/maths/CDecompositionComponent.h b/include/maths/CDecompositionComponent.h index 1acf61b7c2..e3451417a1 100644 --- a/include/maths/CDecompositionComponent.h +++ b/include/maths/CDecompositionComponent.h @@ -43,15 +43,15 @@ namespace maths class MATHS_EXPORT CDecompositionComponent { public: - typedef maths_t::TDoubleDoublePr TDoubleDoublePr; - typedef std::vector TDoubleVec; - typedef std::vector TFloatVec; - typedef CSpline, - boost::reference_wrapper, - boost::reference_wrapper > TSplineCRef; - typedef CSpline, - boost::reference_wrapper, - boost::reference_wrapper > TSplineRef; + using TDoubleDoublePr = maths_t::TDoubleDoublePr; + using TDoubleVec = std::vector; + using TFloatVec = std::vector; + using TSplineCRef = CSpline, + boost::reference_wrapper, + boost::reference_wrapper>; + using TSplineRef = CSpline, + boost::reference_wrapper, + boost::reference_wrapper>; public: //! Persist state by passing information to \p inserter. @@ -72,9 +72,9 @@ class MATHS_EXPORT CDecompositionComponent }; public: - typedef boost::array TTypeArray; - typedef boost::array TFloatVecArray; - typedef boost::array TDoubleVecArray; + using TTypeArray = boost::array; + using TFloatVecArray = boost::array; + using TDoubleVecArray = boost::array; public: CPackedSplines(CSplineTypes::EType valueInterpolationType, diff --git a/include/maths/CExpandingWindow.h b/include/maths/CExpandingWindow.h new file mode 100644 index 0000000000..d9b3413174 --- /dev/null +++ b/include/maths/CExpandingWindow.h @@ -0,0 +1,135 @@ +/* + * ELASTICSEARCH CONFIDENTIAL + * + * Copyright (c) 2018 Elasticsearch BV. All Rights Reserved. + * + * Notice: this software, and all information contained + * therein, is the exclusive property of Elasticsearch BV + * and its licensors, if any, and is protected under applicable + * domestic and foreign law, and international treaties. + * + * Reproduction, republication or distribution without the + * express written consent of Elasticsearch BV is + * strictly prohibited. + */ + +#ifndef INCLUDED_ml_maths_CExpandingWindow_h +#define INCLUDED_ml_maths_CExpandingWindow_h + +#include +#include +#include + +#include +#include + +#include +#include +#include + +namespace ml +{ +namespace core +{ +class CStatePersistInserter; +class CStateRestoreTraverser; +} + +namespace maths +{ + +//! \brief Implements a fixed memory expanding time window. +//! +//! DESCRIPTION:\n +//! As the window expands it compresses by merging adjacent values +//! and maintaining means of merged values. It cycles through a +//! sequence of increasing compression factors, which are determined +//! by a sequence of increasing bucketing lengths supplied to the +//! constructor. At the point it overflows, i.e. time since the +//! beginning of the window exceeds "size" x "maximum bucket length", +//! it will re-initialize the bucketing and update the start time. +class MATHS_EXPORT CExpandingWindow +{ + public: + using TDoubleVec = std::vector; + using TTimeVec = std::vector; + using TTimeCRng = core::CVectorRange; + using TFloatMeanAccumulator = CBasicStatistics::SSampleMean::TAccumulator; + using TFloatMeanAccumulatorVec = std::vector; + using TPredictor = std::function; + + public: + CExpandingWindow(core_t::TTime bucketLength, + TTimeCRng bucketLengths, + std::size_t size, + double decayRate = 0.0); + + //! Initialize by reading state from \p traverser. + bool acceptRestoreTraverser(core::CStateRestoreTraverser &traverser); + + //! Persist state by passing information to \p inserter. + void acceptPersistInserter(core::CStatePersistInserter &inserter) const; + + //! Get the start time of the sketch. + core_t::TTime startTime() const; + + //! Get the end time of the sketch. + core_t::TTime endTime() const; + + //! Get the current bucket length. + core_t::TTime bucketLength() const; + + //! Get the bucket values. + const TFloatMeanAccumulatorVec &values() const; + + //! Get the bucket values minus the values from \p trend. + TFloatMeanAccumulatorVec valuesMinusPrediction(const TPredictor &predictor) const; + + //! Set the start time to \p time. + void initialize(core_t::TTime time); + + //! Age the bucket values to account for \p time elapsed time. + void propagateForwardsByTime(double time); + + //! Add \p value at \p time. + void add(core_t::TTime time, double value, double weight = 1.0); + + //! Check if we need to compress by increasing the bucket span. + bool needToCompress(core_t::TTime time) const; + + //! Get a checksum for this object. + uint64_t checksum(uint64_t seed = 0) const; + + //! Debug the memory used by this object. + void debugMemoryUsage(core::CMemoryUsage::TMemoryUsagePtr mem) const; + + //! Get the memory used by this object. + std::size_t memoryUsage() const; + + private: + //! The rate at which the bucket values are aged. + double m_DecayRate; + + //! The data bucketing length. + core_t::TTime m_BucketLength; + + //! The bucket lengths to test. + TTimeCRng m_BucketLengths; + + //! The index in m_BucketLengths of the current bucketing interval. + std::size_t m_BucketLengthIndex; + + //! The time of the first data point. + core_t::TTime m_StartTime; + + //! The bucket values. + TFloatMeanAccumulatorVec m_BucketValues; + + //! The mean value time modulo the data bucketing length. + TFloatMeanAccumulator m_MeanOffset; +}; + +} +} + +#endif // INCLUDED_ml_maths_CExpandingWindow_h diff --git a/include/maths/CGammaRateConjugate.h b/include/maths/CGammaRateConjugate.h index a9530ba837..065ec73081 100644 --- a/include/maths/CGammaRateConjugate.h +++ b/include/maths/CGammaRateConjugate.h @@ -85,15 +85,19 @@ class MATHS_EXPORT CGammaRateConjugate : public CPrior //! \param[in] priorShape The shape parameter of the gamma prior. //! \param[in] priorRate The rate parameter of the gamma prior. //! \param[in] decayRate The rate at which to revert to non-informative. + //! \param[in] offsetMargin The margin between the smallest value and the support + //! left end. CGammaRateConjugate(maths_t::EDataType dataType, double offset, double priorShape, double priorRate, - double decayRate = 0.0); + double decayRate = 0.0, + double offsetMargin = GAMMA_OFFSET_MARGIN); //! Construct by traversing a state document. CGammaRateConjugate(const SDistributionRestoreParams ¶ms, - core::CStateRestoreTraverser &traverser); + core::CStateRestoreTraverser &traverser, + double offsetMargin = GAMMA_OFFSET_MARGIN); // Default copy constructor and assignment operator work. @@ -103,10 +107,13 @@ class MATHS_EXPORT CGammaRateConjugate : public CPrior //! for details). //! \param[in] offset The offset to apply to the data. //! \param[in] decayRate The rate at which to revert to the non-informative prior. + //! \param[in] offsetMargin The margin between the smallest value and the support + //! left end. //! \return A non-informative prior. static CGammaRateConjugate nonInformativePrior(maths_t::EDataType dataType, double offset = 0.0, - double decayRate = 0.0); + double decayRate = 0.0, + double offsetMargin = GAMMA_OFFSET_MARGIN); //@} //! \name Prior Contract @@ -123,7 +130,12 @@ class MATHS_EXPORT CGammaRateConjugate : public CPrior //! Reset the prior to non-informative. virtual void setToNonInformative(double offset = 0.0, double decayRate = 0.0); - //! Returns false. + //! Get the margin between the smallest value and the support left + //! end. Priors with non-negative support, automatically adjust the + //! offset if a value is seen which is smaller than offset + margin. + virtual double offsetMargin(void) const; + + //! Returns true. virtual bool needsOffset(void) const; //! Reset m_Offset so the smallest sample is not within some minimum @@ -399,6 +411,9 @@ class MATHS_EXPORT CGammaRateConjugate : public CPrior //! us to model data with negative values greater than \f$-u\f$. double m_Offset; + //! The margin between the smallest value and the support left end. + double m_OffsetMargin; + //! The maximum likelihood estimate of the shape parameter. double m_LikelihoodShape; diff --git a/include/maths/CLogNormalMeanPrecConjugate.h b/include/maths/CLogNormalMeanPrecConjugate.h index 70e07c08e6..6d7d662038 100644 --- a/include/maths/CLogNormalMeanPrecConjugate.h +++ b/include/maths/CLogNormalMeanPrecConjugate.h @@ -84,17 +84,21 @@ class MATHS_EXPORT CLogNormalMeanPrecConjugate : public CPrior //! \param[in] gammaRate The rate parameter of the gamma component of the //! prior. //! \param[in] decayRate The rate at which to revert to non-informative. + //! \param[in] offsetMargin The margin between the smallest value and the support + //! left end. CLogNormalMeanPrecConjugate(maths_t::EDataType dataType, double offset, double gaussianMean, double gaussianPrecision, double gammaShape, double gammaRate, - double decayRate = 0.0); + double decayRate = 0.0, + double offsetMargin = LOG_NORMAL_OFFSET_MARGIN); //! Construct from part of a state document. CLogNormalMeanPrecConjugate(const SDistributionRestoreParams ¶ms, - core::CStateRestoreTraverser &traverser); + core::CStateRestoreTraverser &traverser, + double offsetMargin = LOG_NORMAL_OFFSET_MARGIN); // Default copy constructor and assignment operator work. @@ -104,10 +108,13 @@ class MATHS_EXPORT CLogNormalMeanPrecConjugate : public CPrior //! for details). //! \param[in] offset The offset to apply to the data. //! \param[in] decayRate The rate at which to revert to the non-informative prior. + //! \param[in] offsetMargin The margin between the smallest value and the support + //! left end. //! \return A non-informative prior. static CLogNormalMeanPrecConjugate nonInformativePrior(maths_t::EDataType dataType, double offset = 0.0, - double decayRate = 0.0); + double decayRate = 0.0, + double offsetMargin = LOG_NORMAL_OFFSET_MARGIN); //@} //! \name Prior Contract @@ -124,6 +131,11 @@ class MATHS_EXPORT CLogNormalMeanPrecConjugate : public CPrior //! Reset the prior to non-informative. virtual void setToNonInformative(double offset = 0.0, double decayRate = 0.0); + //! Get the margin between the smallest value and the support left + //! end. Priors with non-negative support, automatically adjust the + //! offset if a value is seen which is smaller than offset + margin. + virtual double offsetMargin(void) const; + //! Returns true. virtual bool needsOffset(void) const; @@ -419,6 +431,9 @@ class MATHS_EXPORT CLogNormalMeanPrecConjugate : public CPrior //! allows us to model data with negative values greater than \f$-u\f$. double m_Offset; + //! The margin between the smallest value and the support left end. + double m_OffsetMargin; + //! The mean of the prior conditional distribution for the mean of the //! exponentiated normal (conditioned on its precision). double m_GaussianMean; diff --git a/include/maths/CModel.h b/include/maths/CModel.h index eb1d1c5dc4..131e4bba56 100644 --- a/include/maths/CModel.h +++ b/include/maths/CModel.h @@ -356,9 +356,9 @@ class MATHS_EXPORT CModel //! Get the prediction and \p confidenceInterval percentage //! confidence interval for the time series at \p time. virtual TDouble2Vec3Vec confidenceInterval(core_t::TTime time, + double confidenceInterval, const maths_t::TWeightStyleVec &weightStyles, - const TDouble2Vec4Vec &weights, - double confidenceInterval) const = 0; + const TDouble2Vec4Vec &weights) const = 0; //! Forecast the time series and get its \p confidenceInterval //! percentage confidence interval between \p startTime and @@ -516,9 +516,9 @@ class MATHS_EXPORT CModelStub : public CModel //! Returns empty. virtual TDouble2Vec3Vec confidenceInterval(core_t::TTime time, + double confidenceInterval, const maths_t::TWeightStyleVec &weightStyles, - const TDouble2Vec4Vec &weights, - double percentage) const; + const TDouble2Vec4Vec &weights) const; //! Returns empty. virtual bool forecast(core_t::TTime startTime, core_t::TTime endTime, diff --git a/include/maths/CPeriodicityHypothesisTests.h b/include/maths/CPeriodicityHypothesisTests.h new file mode 100644 index 0000000000..3d0b3e4a4a --- /dev/null +++ b/include/maths/CPeriodicityHypothesisTests.h @@ -0,0 +1,439 @@ +/* + * ELASTICSEARCH CONFIDENTIAL + * + * Copyright (c) 2018 Elasticsearch BV. All Rights Reserved. + * + * Notice: this software, and all information contained + * therein, is the exclusive property of Elasticsearch BV + * and its licensors, if any, and is protected under applicable + * domestic and foreign law, and international treaties. + * + * Reproduction, republication or distribution without the + * express written consent of Elasticsearch BV is + * strictly prohibited. + */ + +#ifndef INCLUDED_ml_maths_CPeriodicityHypothesisTests_h +#define INCLUDED_ml_maths_CPeriodicityHypothesisTests_h + +#include +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include + +namespace ml +{ +namespace maths +{ +class CSeasonalTime; + +//! \brief Represents the result of running the periodicity +//! hypothesis tests. +class MATHS_EXPORT CPeriodicityHypothesisTestsResult : boost::equality_comparable > +{ + public: + using TTimeTimePr = std::pair; + + public: + //! \brief Component data. + struct MATHS_EXPORT SComponent + { + SComponent(); + SComponent(const std::string &description, + bool diurnal, + core_t::TTime startOfPartition, + core_t::TTime period, + const TTimeTimePr &window, + double precedence = 1.0); + + //! Check if this is equal to \p other. + bool operator==(const SComponent &other) const; + + //! Get a seasonal time for the specified results. + //! + //! \warning The caller owns the returned object. + CSeasonalTime *seasonalTime() const; + + //! An identifier for the component used by the test. + std::string s_Description; + //! True if this is a diurnal component false otherwise. + bool s_Diurnal; + //! The start of the partition. + core_t::TTime s_StartOfPartition; + //! The period of the component. + core_t::TTime s_Period; + //! The component window. + TTimeTimePr s_Window; + //! The precedence to apply to this component when + //! deciding which to keep. + double s_Precedence; + }; + + using TComponent5Vec = core::CSmallVector; + + public: + //! Check if this is equal to \p other. + bool operator==(const CPeriodicityHypothesisTestsResult &other) const; + + //! Sets to the union of the periodic components present. + //! + //! \warning This only makes sense if the this and the + //! other result share the start of the partition time. + const CPeriodicityHypothesisTestsResult &operator+=(const CPeriodicityHypothesisTestsResult &other); + + //! Add a component. + void add(const std::string &description, + bool diurnal, + core_t::TTime startOfWeek, + core_t::TTime period, + const TTimeTimePr &window, + double precedence = 1.0); + + //! Remove the component with \p description. + void remove(const std::string &description); + + //! Check if there are any periodic components. + bool periodic() const; + + //! Get the binary representation of the periodic components. + const TComponent5Vec &components() const; + + //! Get a human readable description of the result. + std::string print() const; + + private: + //! The periodic components. + TComponent5Vec m_Components; +}; + +//! \brief Configures the periodicity testing. +class MATHS_EXPORT CPeriodicityHypothesisTestsConfig +{ + public: + CPeriodicityHypothesisTestsConfig(); + + //! Disable diurnal periodicity tests. + void disableDiurnal(); + //! Test given we know there is daily periodic component. + void hasDaily(bool value); + //! Test given we know there is a weekend. + void hasWeekend(bool value); + //! Test given we know there is a weekly periodic component. + void hasWeekly(bool value); + //! Set the start of the week. + void startOfWeek(core_t::TTime value); + + //! Check if we should test for diurnal periodic components. + bool testForDiurnal() const; + //! Check if we know there is a daily component. + bool hasDaily() const; + //! Check if we know there is a weekend. + bool hasWeekend() const; + //! Check if we know there is a weekly component. + bool hasWeekly() const; + //! Get the start of the week. + core_t::TTime startOfWeek() const; + + private: + //! True if we should test for diurnal periodicity. + bool m_TestForDiurnal; + //! True if we know there is a daily component. + bool m_HasDaily; + //! True if we know there is a weekend. + bool m_HasWeekend; + //! True if we know there is a weekly component. + bool m_HasWeekly; + //! The start of the week. + core_t::TTime m_StartOfWeek; +}; + +//! \brief Implements a set of hypothesis tests to discover the +//! most plausible explanation of the periodic patterns in a +//! time window of a time series. +//! +//! DESCRIPTION:\n +//! This tests whether there are daily and/or weekly components +//! in a time series. It also checks to see if there is a partition +//! of the time series into disjoint weekend and weekday intervals. +//! Tests include various forms of analysis of variance and a test +//! of the amplitude. It also compares these possibilities with a +//! specified period (typically found by examining the cyclic +//! autocorrelation). +class MATHS_EXPORT CPeriodicityHypothesisTests +{ + public: + using TDouble2Vec = core::CSmallVector; + using TTimeTimePr = std::pair; + using TTimeTimePr2Vec = core::CSmallVector; + using TMeanVarAccumulator = CBasicStatistics::SSampleMeanVar::TAccumulator; + using TMeanVarAccumulatorVec = std::vector; + using TTimeTimePrMeanVarAccumulatorPr = std::pair; + using TTimeTimePrMeanVarAccumulatorPrVec = std::vector; + using TTimeTimePrMeanVarAccumulatorPrVecVec = std::vector; + using TFloatMeanAccumulator = CBasicStatistics::SSampleMean::TAccumulator; + using TFloatMeanAccumulatorVec = std::vector; + using TTime2Vec = core::CSmallVector; + using TComponent = CPeriodicityHypothesisTestsResult::SComponent; + + public: + CPeriodicityHypothesisTests(); + explicit CPeriodicityHypothesisTests(const CPeriodicityHypothesisTestsConfig &config); + + //! Check if the test is initialized. + bool initialized() const; + + //! Initialize the bucket values. + void initialize(core_t::TTime bucketLength, + core_t::TTime window, + core_t::TTime period); + + //! Add \p value at \p time. + void add(core_t::TTime time, double value, double weight = 1.0); + + //! Check if there periodic components and, if there are, + //! which best describe the periodic patterns in the data. + CPeriodicityHypothesisTestsResult test() const; + + private: + using TDoubleVec = std::vector; + using TDoubleVec2Vec = core::CSmallVector; + using TFloatMeanAccumulatorCRng = core::CVectorRange; + using TMinMaxAccumulator = maths::CBasicStatistics::CMinMax; + + //! \brief A collection of statistics used during testing. + struct STestStats + { + STestStats(); + //! Set the various test thresholds. + void setThresholds(double vt, double at, double Rt); + //! Check if the null hypothesis is good enough to not need an + //! alternative. + bool nullHypothesisGoodEnough() const; + //! True if a known periodic component is tested. + bool s_HasPeriod; + //! True if a known repeating partition is tested. + bool s_HasPartition; + //! The maximum variance to accept the alternative hypothesis. + double s_Vt; + //! The minimum amplitude to accept the alternative hypothesis. + double s_At; + //! The minimum autocorrelation to accept the alternative + //! hypothesis. + double s_Rt; + //! The data range. + double s_Range; + //! The number of buckets with at least one measurement. + double s_B; + //! The average number of measurements per bucket value. + double s_M; + //! The null hypothesis periodic components. + CPeriodicityHypothesisTestsResult s_H0; + //! The variance estimate of H0. + double s_V0; + //! The degrees of freedom in the variance estimate of H0. + double s_DF0; + //! The trend for the null hypothesis. + TDoubleVec2Vec s_T0; + //! The partition for the null hypothesis. + TTimeTimePr2Vec s_Partition; + //! The start of the repeating partition. + core_t::TTime s_StartOfPartition; + }; + + //! \brief Manages the testing of a set of nested hypotheses. + class CNestedHypotheses + { + public: + using TTestFunc = std::function; + + //! \brief Manages the building of a collection of nested + //! hypotheses. + class CBuilder + { + public: + explicit CBuilder(CNestedHypotheses &hypothesis); + CBuilder &addNested(TTestFunc test); + CBuilder &addAlternative(TTestFunc test); + CBuilder &finishedNested(); + + private: + using TNestedHypothesesPtrVec = std::vector; + + private: + TNestedHypothesesPtrVec m_Levels; + }; + + public: + explicit CNestedHypotheses(TTestFunc test = 0); + + //! Set the null hypothesis. + CBuilder null(TTestFunc test); + //! Add a nested hypothesis for \p test. + CNestedHypotheses &addNested(TTestFunc test); + //! Test the hypotheses. + CPeriodicityHypothesisTestsResult test(STestStats &stats) const; + + private: + using THypothesisVec = std::vector; + + private: + //! The test. + TTestFunc m_Test; + //! If true always test the nested hypotheses. + bool m_AlwaysTestNested; + //! The nested hypotheses to test. + THypothesisVec m_Nested; + }; + + using TNestedHypothesesVec = std::vector; + + private: + //! Get the hypotheses to test for period/daily/weekly components. + void hypothesesForWeekly(const TTimeTimePr2Vec &windowForTestingWeekly, + const TFloatMeanAccumulatorCRng &bucketsForTestingWeekly, + const TTimeTimePr2Vec &windowForTestingPeriod, + const TFloatMeanAccumulatorCRng &bucketsForTestingPeriod, + TNestedHypothesesVec &hypotheses) const; + + //! Get the hypotheses to test for period/daily components. + void hypothesesForDaily(const TTimeTimePr2Vec &windowForTestingDaily, + const TFloatMeanAccumulatorCRng &bucketsForTestingDaily, + const TTimeTimePr2Vec &windowForTestingPeriod, + const TFloatMeanAccumulatorCRng &bucketsForTestingPeriod, + TNestedHypothesesVec &hypotheses) const; + + //! Get the hypotheses to test for period components. + void hypothesesForPeriod(const TTimeTimePr2Vec &windows, + const TFloatMeanAccumulatorCRng &buckets, + TNestedHypothesesVec &hypotheses) const; + + //! Extract the best hypothesis. + CPeriodicityHypothesisTestsResult best(const TNestedHypothesesVec &hypotheses) const; + + //! The null hypothesis of the various tests. + CPeriodicityHypothesisTestsResult + testForNull(const TTimeTimePr2Vec &window, + const TFloatMeanAccumulatorCRng &buckets, + STestStats &stats) const; + + //! Test for a daily periodic component. + CPeriodicityHypothesisTestsResult + testForDaily(const TTimeTimePr2Vec &window, + const TFloatMeanAccumulatorCRng &buckets, + STestStats &stats) const; + + //! Test for a weekly periodic component. + CPeriodicityHypothesisTestsResult + testForWeekly(const TTimeTimePr2Vec &window, + const TFloatMeanAccumulatorCRng &buckets, + STestStats &stats) const; + + //! Test for a weekday/end partition. + CPeriodicityHypothesisTestsResult + testForDailyWithWeekend(const TFloatMeanAccumulatorCRng &buckets, + STestStats &stats) const; + + //! Test for a weekday/end partition with weekly . + CPeriodicityHypothesisTestsResult + testForWeeklyGivenDailyWithWeekend(const TTimeTimePr2Vec &window, + const TFloatMeanAccumulatorCRng &buckets, + STestStats &stats) const; + + //! Test for the specified period given we think there is + //! some diurnal periodicity. + CPeriodicityHypothesisTestsResult + testForPeriod(const TTimeTimePr2Vec &window, + const TFloatMeanAccumulatorCRng &buckets, + STestStats &stats) const; + + //! Check we've seen sufficient data to test accurately. + bool seenSufficientDataToTest(core_t::TTime period, + const TFloatMeanAccumulatorCRng &buckets) const; + + //! Compute various ancillary statistics for testing. + bool testStatisticsFor(const TFloatMeanAccumulatorCRng &buckets, + STestStats &stats) const; + + //! Get the variance and degrees freedom for the null hypothesis + //! that there is no trend or repeating partition of any kind. + void nullHypothesis(const TTimeTimePr2Vec &window, + const TFloatMeanAccumulatorCRng &buckets, + STestStats &stats) const; + + //! Compute the variance and degrees freedom for the hypothesis. + void hypothesis(const TTime2Vec &periods, + const TFloatMeanAccumulatorCRng &buckets, + STestStats &stats) const; + + //! Condition \p buckets assuming the null hypothesis is true. + //! + //! This removes any trend associated with the null hypothesis. + void conditionOnHypothesis(const TTimeTimePr2Vec &windows, + const STestStats &stats, + TFloatMeanAccumulatorVec &buckets) const; + + //! Test to see if there is significant evidence for a component + //! with period \p period. + bool testPeriod(const TTimeTimePr2Vec &window, + const TFloatMeanAccumulatorCRng &buckets, + core_t::TTime period, STestStats &stats) const; + + //! Test to see if there is significant evidence for a repeating + //! partition of the data into windows defined by \p partition. + bool testPartition(const TTimeTimePr2Vec &partition, + const TFloatMeanAccumulatorCRng &buckets, + core_t::TTime period, + double correction, STestStats &stats) const; + + private: + //! The minimum proportion of populated buckets for which + //! the test is accurate. + static const double ACCURATE_TEST_POPULATED_FRACTION; + + //! The minimum coefficient of variation to bother to test. + static const double MINIMUM_COEFFICIENT_OF_VARIATION; + + private: + //! Configures the tests to run. + CPeriodicityHypothesisTestsConfig m_Config; + + //! The bucketing interval. + core_t::TTime m_BucketLength; + + //! The window length for which to maintain bucket values. + core_t::TTime m_WindowLength; + + //! The specified period to test. + core_t::TTime m_Period; + + //! The time range of values added to the test. + TMinMaxAccumulator m_TimeRange; + + //! The mean bucket values. + TFloatMeanAccumulatorVec m_BucketValues; +}; + +using TFloatMeanAccumulator = CBasicStatistics::SSampleMean::TAccumulator; +using TFloatMeanAccumulatorVec = std::vector; + +//! Test for periodic components in \p values. +MATHS_EXPORT +CPeriodicityHypothesisTestsResult testForPeriods(const CPeriodicityHypothesisTestsConfig &config, + core_t::TTime startTime, + core_t::TTime bucketLength, + const TFloatMeanAccumulatorVec &values); + +} +} + +#endif // INCLUDED_ml_maths_CPeriodicityHypothesisTests_h diff --git a/include/maths/CPrior.h b/include/maths/CPrior.h index 166e887f3d..89c15aa25d 100644 --- a/include/maths/CPrior.h +++ b/include/maths/CPrior.h @@ -150,15 +150,6 @@ class MATHS_EXPORT CPrior void swap(CPrior &other); //@} - //! Mark the prior as being used for forecasting. - //! - //! \warning This is an irreversible action so if the prior - //! is still need it should be copied first. - void forForecasting(void); - - //! Check if this prior is being used for forecasting. - bool isForForecasting(void) const; - //! Check if the prior is being used to model discrete data. bool isDiscrete(void) const; @@ -187,16 +178,6 @@ class MATHS_EXPORT CPrior //! Set the rate at which the prior returns to non-informative. virtual void decayRate(double value); - //! Get the margin between the smallest value and the support left - //! end. Priors with non-negative support, automatically adjust the - //! offset if a value is seen which is smaller than offset + margin. - //! This is to avoid the numerical instability caused by adding - //! values close to zero. - //! - //! \note This is overridden by CPriorTestInterface so don't replace - //! it by a static constant in the calling functions. - virtual double offsetMargin(void) const; - //! Reset the prior to non-informative. virtual void setToNonInformative(double offset = 0.0, double decayRate = 0.0) = 0; @@ -204,6 +185,11 @@ class MATHS_EXPORT CPrior //! Remove models marked by \p filter. virtual void removeModels(CModelFilter &filter); + //! Get the margin between the smallest value and the support left + //! end. Priors with non-negative support, automatically adjust the + //! offset if a value is seen which is smaller than offset + margin. + virtual double offsetMargin(void) const; + //! Check if the prior needs an offset to be applied. virtual bool needsOffset(void) const = 0; @@ -584,11 +570,6 @@ class MATHS_EXPORT CPrior virtual std::string debug(void) const; private: - //! Set to true if this model is being used for forecasting. Note - //! we don't have any need to persist forecast models so this is - //! is not persisted. - bool m_Forecasting; - //! If this is true then the prior is being used to model discrete //! data. Note that this is not persisted and deduced from context. maths_t::EDataType m_DataType; diff --git a/include/maths/CRegression.h b/include/maths/CRegression.h index a607ebfce9..2ae8a88cb9 100644 --- a/include/maths/CRegression.h +++ b/include/maths/CRegression.h @@ -125,7 +125,7 @@ class MATHS_EXPORT CRegression static const std::string STATISTIC_TAG; public: - CLeastSquaresOnline(void) : m_S() {} + CLeastSquaresOnline() : m_S() {} template CLeastSquaresOnline(const CLeastSquaresOnline &other) : m_S(other.statistic()) @@ -166,12 +166,30 @@ class MATHS_EXPORT CRegression return *this; } + //! Differences two regressions. + //! + //! This creates a regression which is fit on just the points + //! add to this and not \p rhs. + //! + //! \param[in] rhs The regression fit to combine. + //! \note This is only meaningful if they have the same time + //! origin and the values added to \p rhs are a subset of the + //! values add to this. + template + const CLeastSquaresOnline &operator-=(const CLeastSquaresOnline &rhs) + { + m_S -= rhs.statistic(); + return *this; + } + //! Combines two regressions. //! //! This creates the regression fit on the points fit with - //! \p lhs and the points fit with this regression. + //! \p rhs and the points fit with this regression. //! //! \param[in] rhs The regression fit to combine. + //! \note This is only meaningful if they have the same time + //! origin. template const CLeastSquaresOnline &operator+=(const CLeastSquaresOnline &rhs) { @@ -240,6 +258,9 @@ class MATHS_EXPORT CRegression return *this; } + //! Get the predicted value at \p x. + double predict(double x, double maxCondition = regression_detail::CMaxCondition::VALUE) const; + //! Get the regression parameters. //! //! i.e. The intercept, slope, curvature, etc. @@ -248,17 +269,16 @@ class MATHS_EXPORT CRegression //! the Gramian this will consider solving. If the condition //! is worse than this it'll fit a lower order polynomial. //! \param[out] result Filled in with the regression parameters. - bool parameters(TArray &result, - double maxCondition = regression_detail::CMaxCondition::VALUE) const; + bool parameters(TArray &result, double maxCondition = regression_detail::CMaxCondition::VALUE) const; //! Get the predicted value of the regression parameters at \p x. //! //! \note Returns array of zeros if getting the parameters fails. - TArray parameters(double x) const + TArray parameters(double x, double maxCondition = regression_detail::CMaxCondition::VALUE) const { TArray result; TArray params; - if (this->parameters(params)) + if (this->parameters(params, maxCondition)) { std::ptrdiff_t n = static_cast(params.size()); double xi = x; @@ -301,17 +321,17 @@ class MATHS_EXPORT CRegression //! Get the safe prediction horizon based on the spread //! of the abscissa added to the model so far. - double range(void) const + double range() const { + // The magic 12 comes from assuming the independent + // variable X is uniform over the range (for our uses + // it typically is). We maintain mean X^2 and X. For + // a uniform variable on a range [a, b] we have that + // E[(X - E(X))^2] = E[X^2] - E[X]^2 = (b - a)^2 / 12. + double x1 = CBasicStatistics::mean(m_S)(1); double x2 = CBasicStatistics::mean(m_S)(2); - return ::sqrt(12.0 * std::max(x2 - x1 * x1, 0.0)); - } - - //! Check if the regression has sufficient history to predict. - bool sufficientHistoryToPredict(void) const - { - return this->range() >= MINIMUM_RANGE_TO_PREDICT; + return std::sqrt(12.0 * std::max(x2 - x1 * x1, 0.0)); } //! Age out the old points. @@ -330,13 +350,13 @@ class MATHS_EXPORT CRegression } //! Get the effective number of points being fitted. - double count(void) const + double count() const { return CBasicStatistics::count(m_S); } //! Get the mean value of the ordinates. - double mean(void) const + double mean() const { return CBasicStatistics::mean(m_S)(2*N-1); } @@ -377,19 +397,19 @@ class MATHS_EXPORT CRegression } //! Get the vector statistic. - const TVectorMeanAccumulator &statistic(void) const + const TVectorMeanAccumulator &statistic() const { return m_S; } //! Get a checksum for this object. - std::uint64_t checksum(void) const + std::uint64_t checksum() const { return m_S.checksum(); } //! Print this regression out for debug. - std::string print(void) const; + std::string print() const; private: //! Get the first \p n regression parameters. @@ -430,17 +450,9 @@ class MATHS_EXPORT CRegression }; //! Get the predicted value of \p r at \p x. - template - static double predict(const CLeastSquaresOnline &r, double x) + template + static double predict(const boost::array ¶ms, double x) { - if (!r.sufficientHistoryToPredict()) - { - return r.mean(); - } - - typename CLeastSquaresOnline::TArray params; - r.parameters(params); - double result = params[0]; double xi = x; for (std::size_t i = 1u; i < params.size(); ++i, xi *= x) @@ -457,6 +469,7 @@ class MATHS_EXPORT CRegression { public: using TVector = CVectorNx1; + using TMatrix = CSymmetricMatrixNxN; public: static const std::string UNIT_TIME_COVARIANCES_TAG; @@ -499,6 +512,9 @@ class MATHS_EXPORT CRegression m_UnitTimeCovariances.age(factor); } + //! Get the process covariance matrix. + TMatrix covariance() const; + //! Compute the variance of the mean zero normal distribution //! due to the drift in the regression parameters over \p time. double predictionVariance(double time) const @@ -523,10 +539,10 @@ class MATHS_EXPORT CRegression } //! Get a checksum for this object. - uint64_t checksum(void) const; + uint64_t checksum() const; //! Print this process out for debug. - std::string print(void) const; + std::string print() const; private: using TCovarianceAccumulator = CBasicStatistics::SSampleCovariances; @@ -536,13 +552,16 @@ class MATHS_EXPORT CRegression //! covariance matrix. TCovarianceAccumulator m_UnitTimeCovariances; }; - - private: - //! The minimum range of the predictor variable for which we'll - //! forward predict using the higher order terms. - static const double MINIMUM_RANGE_TO_PREDICT; }; +template +double CRegression::CLeastSquaresOnline::predict(double x, double maxCondition) const +{ + TArray params; + this->parameters(params, maxCondition); + return CRegression::predict(params, x); +} + template const std::string CRegression::CLeastSquaresOnline::STATISTIC_TAG("a"); template diff --git a/include/maths/CRegressionDetail.h b/include/maths/CRegressionDetail.h index 416fe9b4f8..f4f4d1e8ba 100644 --- a/include/maths/CRegressionDetail.h +++ b/include/maths/CRegressionDetail.h @@ -305,6 +305,13 @@ void CRegression::CLeastSquaresOnlineParameterProcess::acceptPersistInsert inserter.insertValue(UNIT_TIME_COVARIANCES_TAG, m_UnitTimeCovariances.toDelimited()); } +template +typename CRegression::CLeastSquaresOnlineParameterProcess::TMatrix +CRegression::CLeastSquaresOnlineParameterProcess::covariance() const +{ + return CBasicStatistics::covariances(m_UnitTimeCovariances); +} + template uint64_t CRegression::CLeastSquaresOnlineParameterProcess::checksum(void) const { diff --git a/include/maths/CSeasonalComponent.h b/include/maths/CSeasonalComponent.h index 9e109c1da1..dc678efad8 100644 --- a/include/maths/CSeasonalComponent.h +++ b/include/maths/CSeasonalComponent.h @@ -53,11 +53,9 @@ namespace maths class MATHS_EXPORT CSeasonalComponent : private CDecompositionComponent { public: - typedef CBasicStatistics::SSampleMeanVar::TAccumulator TMeanVarAccumulator; - typedef std::pair TTimeTimePr; - typedef std::pair TTimeTimePrMeanVarPr; - typedef std::vector TTimeTimePrMeanVarPrVec; - typedef CSymmetricMatrixNxN TMatrix; + using TMatrix = CSymmetricMatrixNxN; + using TFloatMeanAccumulator = CBasicStatistics::SSampleMean::TAccumulator; + using TFloatMeanAccumulatorVec = std::vector; public: //! \param[in] time The time provider. @@ -96,9 +94,9 @@ class MATHS_EXPORT CSeasonalComponent : private CDecompositionComponent bool initialized(void) const; //! Initialize the adaptive bucketing. - bool initialize(core_t::TTime startTime, - core_t::TTime endTime, - const TTimeTimePrMeanVarPrVec &values); + bool initialize(core_t::TTime startTime = 0, + core_t::TTime endTime = 0, + const TFloatMeanAccumulatorVec &values = TFloatMeanAccumulatorVec()); //! Get the size of this component. std::size_t size(void) const; @@ -151,16 +149,18 @@ class MATHS_EXPORT CSeasonalComponent : private CDecompositionComponent //! Get the mean value of the component. double meanValue(void) const; - //! Get the difference from the mean of repeats at \p period. + //! This computes the delta to apply to the component with \p period. //! - //! This computes the quantity: - //! 
-        //!    \f$\sum_i f(t + p i)\f$
-        //!
+ //! This is used to adjust the decomposition when it contains components + //! whose periods are divisors of one another to get the most efficient + //! representation. //! - //! where \f$t\f$ is \p time and \f$p\f$ is \p period and must divide - //! this component's period \f$P\f$. The sum ranges over \f$[P/p]\f$. - double differenceFromMean(core_t::TTime time, core_t::TTime period) const; + //! \param[in] time The time at which to compute the delta. + //! \param[in] shortPeriod The period of the short component. + //! \param[in] shortPeriodValue The short component value at \p time. + double delta(core_t::TTime time, + core_t::TTime shortPeriod, + double shortPeriodValue) const; //! Get the variance of the residual about the prediction at \p time. //! @@ -176,10 +176,6 @@ class MATHS_EXPORT CSeasonalComponent : private CDecompositionComponent //! residual variance. double heteroscedasticity(void) const; - //! Get the variance in the prediction due to drift in the regression - //! model parameters expected by \p time. - double varianceDueToParameterDrift(core_t::TTime time) const; - //! Get the covariance matrix of the regression parameters' at \p time. //! //! \param[in] time The time of interest. @@ -194,7 +190,7 @@ class MATHS_EXPORT CSeasonalComponent : private CDecompositionComponent double slope(void) const; //! Check if the bucket regression models have enough history to predict. - bool sufficientHistoryToPredict(core_t::TTime time) const; + bool slopeAccurate(core_t::TTime time) const; //! Get a checksum for this object. uint64_t checksum(uint64_t seed = 0) const; diff --git a/include/maths/CSeasonalComponentAdaptiveBucketing.h b/include/maths/CSeasonalComponentAdaptiveBucketing.h index 6e6d485af2..54af5944f7 100644 --- a/include/maths/CSeasonalComponentAdaptiveBucketing.h +++ b/include/maths/CSeasonalComponentAdaptiveBucketing.h @@ -46,7 +46,7 @@ class CSeasonalTime; class MATHS_EXPORT CSeasonalComponentAdaptiveBucketing : private CAdaptiveBucketing { public: - using TTimeTimePrMeanVarPrVec = CAdaptiveBucketing::TTimeTimePrMeanVarPrVec; + using CAdaptiveBucketing::TFloatMeanAccumulatorVec; using TDoubleRegression = CRegression::CLeastSquaresOnline<1, double>; using TRegression = CRegression::CLeastSquaresOnline<1, CFloatStorage>; @@ -88,7 +88,7 @@ class MATHS_EXPORT CSeasonalComponentAdaptiveBucketing : private CAdaptiveBucket //! value moments. void initialValues(core_t::TTime startTime, core_t::TTime endTime, - const TTimeTimePrMeanVarPrVec &values); + const TFloatMeanAccumulatorVec &values); //! Get the number of buckets. std::size_t size(void) const; @@ -113,10 +113,7 @@ class MATHS_EXPORT CSeasonalComponentAdaptiveBucketing : private CAdaptiveBucket //! \param[in] prediction The prediction for \p value. //! \param[in] weight The weight of function point. The smaller //! this is the less influence it has on the bucket. - void add(core_t::TTime time, - double value, - double prediction, - double weight = 1.0); + void add(core_t::TTime time, double value, double prediction, double weight = 1.0); //! Get the time provider. const CSeasonalTime &time(void) const; @@ -165,11 +162,7 @@ class MATHS_EXPORT CSeasonalComponentAdaptiveBucketing : private CAdaptiveBucket double slope(void) const; //! Check if this regression models have enough history to predict. - bool sufficientHistoryToPredict(core_t::TTime time) const; - - //! Get the variance in the prediction due to drift in the regression - //! model parameters expected by \p time. - double varianceDueToParameterDrift(core_t::TTime time) const; + bool slopeAccurate(core_t::TTime time) const; //! Get a checksum for this object. uint64_t checksum(uint64_t seed = 0) const; @@ -188,7 +181,7 @@ class MATHS_EXPORT CSeasonalComponentAdaptiveBucketing : private CAdaptiveBucket //! Get the total count of in the bucketing. double count(void) const; - //! Get the bucket regressions. + //! Get the bucket regression predictions at \p time. TDoubleVec values(core_t::TTime time) const; //! Get the bucket variances. @@ -196,10 +189,28 @@ class MATHS_EXPORT CSeasonalComponentAdaptiveBucketing : private CAdaptiveBucket //@} private: - using TTimeVec = std::vector; - using TRegressionVec = std::vector; using TSeasonalTimePtr = boost::shared_ptr; - using TRegressionParameterProcess = CRegression::CLeastSquaresOnlineParameterProcess<2, double>; + + //! \brief The state maintained for each bucket. + struct SBucket + { + SBucket(void); + SBucket(const TRegression ®ression, + double variance, + core_t::TTime firstUpdate, + core_t::TTime lastUpdate); + + bool acceptRestoreTraverser(core::CStateRestoreTraverser &traverser); + void acceptPersistInserter(core::CStatePersistInserter &inserter) const; + + uint64_t checksum(uint64_t seed) const; + + TRegression s_Regression; + CFloatStorage s_Variance; + core_t::TTime s_FirstUpdate; + core_t::TTime s_LastUpdate; + }; + using TBucketVec = std::vector; private: //! Restore by traversing a state document @@ -213,11 +224,11 @@ class MATHS_EXPORT CSeasonalComponentAdaptiveBucketing : private CAdaptiveBucket //! \param[in] endpoints The old end points. void refresh(const TFloatVec &endpoints); + //! Check if \p time is in the this component's window. + virtual bool inWindow(core_t::TTime time) const; + //! Add the function value at \p time. - virtual void add(std::size_t bucket, - core_t::TTime time, - double offset, - const TDoubleMeanVarAccumulator &value); + virtual void add(std::size_t bucket, core_t::TTime time, double value, double weight); //! Get the offset w.r.t. the start of the bucketing of \p time. virtual double offset(core_t::TTime time) const; @@ -231,28 +242,15 @@ class MATHS_EXPORT CSeasonalComponentAdaptiveBucketing : private CAdaptiveBucket //! Get the variance of \p bucket. virtual double variance(std::size_t bucket) const; - //! Get the age of the bucketing at \p time. - double bucketingAgeAt(core_t::TTime time) const; + //! Get the interval which has been observed at \p time. + double observedInterval(core_t::TTime time) const; private: //! The time provider. TSeasonalTimePtr m_Time; - //! The time that the bucketing was initialized. - core_t::TTime m_InitialTime; - - //! The bucket regressions. - TRegressionVec m_Regressions; - - //! The bucket variances. - TFloatVec m_Variances; - - //! The time that the regressions were last updated. - TTimeVec m_LastUpdates; - - //! The Wiener process which describes the evolution of the - //! regression model parameters. - TRegressionParameterProcess m_ParameterProcess; + //! The buckets. + TBucketVec m_Buckets; }; //! Create a free function which will be found by Koenig lookup. diff --git a/include/maths/CSeasonalTime.h b/include/maths/CSeasonalTime.h index 49e9a2b0f4..f7bd7de285 100644 --- a/include/maths/CSeasonalTime.h +++ b/include/maths/CSeasonalTime.h @@ -39,12 +39,15 @@ namespace maths class MATHS_EXPORT CSeasonalTime { public: - typedef std::pair TTimeTimePr; + using TTimeTimePr = std::pair; public: CSeasonalTime(void); - CSeasonalTime(core_t::TTime period); - virtual ~CSeasonalTime(void); + CSeasonalTime(core_t::TTime period, double precedence); + virtual ~CSeasonalTime(void) = default; + + //! A total order on seasonal times. + bool operator<(const CSeasonalTime &rhs) const; //! Get a copy of this time. //! @@ -124,15 +127,12 @@ class MATHS_EXPORT CSeasonalTime double fractionInWindow(void) const; //@} - //! Get the decay rate scaled from \p fromPeriod period to - //! \p toPeriod period. - //! - //! \param[in] decayRate The decay rate for \p fromPeriod. - //! \param[in] fromPeriod The period of \p decayRate. - //! \param[in] toPeriod The desired period decay rate. - static double scaleDecayRate(double decayRate, - core_t::TTime fromPeriod, - core_t::TTime toPeriod); + //! Check whether this time's seasonal component time excludes + //! modeling \p other's. + bool excludes(const CSeasonalTime &other) const; + + //! True if this has a weekend and false otherwise. + virtual bool hasWeekend(void) const = 0; //! Get a checksum for this object. virtual uint64_t checksum(uint64_t seed = 0) const = 0; @@ -155,6 +155,9 @@ class MATHS_EXPORT CSeasonalTime //! The origin of the time coordinates used to maintain //! a reasonably conditioned Gramian of the design matrix. core_t::TTime m_RegressionOrigin; + //! The precedence of the corresponding component when + //! deciding which to keep amongst alternatives. + double m_Precedence; }; //! \brief Provides times for daily and weekly period seasonal @@ -166,7 +169,8 @@ class MATHS_EXPORT CDiurnalTime : public CSeasonalTime CDiurnalTime(core_t::TTime startOfWeek, core_t::TTime windowStart, core_t::TTime windowEnd, - core_t::TTime period); + core_t::TTime period, + double precedence = 1.0); //! Get a copy of this time. CDiurnalTime *clone(void) const; @@ -189,6 +193,9 @@ class MATHS_EXPORT CDiurnalTime : public CSeasonalTime //! Get the end of the window. virtual core_t::TTime windowEnd(void) const; + //! True if this has a weekend and false otherwise. + virtual bool hasWeekend(void) const; + //! Get a checksum for this object. virtual uint64_t checksum(uint64_t seed = 0) const; @@ -210,8 +217,8 @@ class MATHS_EXPORT CDiurnalTime : public CSeasonalTime class MATHS_EXPORT CGeneralPeriodTime : public CSeasonalTime { public: - CGeneralPeriodTime(void); - CGeneralPeriodTime(core_t::TTime period); + CGeneralPeriodTime(void) = default; + CGeneralPeriodTime(core_t::TTime period, double precedence = 1.0); //! Get a copy of this time. CGeneralPeriodTime *clone(void) const; @@ -234,6 +241,9 @@ class MATHS_EXPORT CGeneralPeriodTime : public CSeasonalTime //! Returns the period. virtual core_t::TTime windowEnd(void) const; + //! Returns false. + virtual bool hasWeekend(void) const; + //! Get a checksum for this object. virtual uint64_t checksum(uint64_t seed = 0) const; @@ -253,7 +263,7 @@ class MATHS_EXPORT CSeasonalTimeStateSerializer public: //! Shared pointer to the CTimeSeriesDecompositionInterface abstract //! base class. - typedef boost::shared_ptr TSeasonalTimePtr; + using TSeasonalTimePtr = boost::shared_ptr; public: //! Construct the appropriate CSeasonalTime sub-class from its state diff --git a/include/maths/CTimeSeriesDecomposition.h b/include/maths/CTimeSeriesDecomposition.h index ce5aa3ff32..635d97473e 100644 --- a/include/maths/CTimeSeriesDecomposition.h +++ b/include/maths/CTimeSeriesDecomposition.h @@ -16,7 +16,6 @@ #ifndef INCLUDED_ml_maths_CTimeSeriesDecomposition_h #define INCLUDED_ml_maths_CTimeSeriesDecomposition_h -#include #include #include #include @@ -34,6 +33,7 @@ class CStateRestoreTraverser; } namespace maths { +class CPrior; //! \brief Decomposes a time series into a linear combination //! of periodic functions and a stationary random process. @@ -63,8 +63,6 @@ class MATHS_EXPORT CTimeSeriesDecomposition : public CTimeSeriesDecompositionInt { public: using TSizeVec = std::vector; - using TMinAccumulator = CBasicStatistics::COrderStatisticsStack; - using TMaxAccumulator = CBasicStatistics::COrderStatisticsStack>; public: //! The default size to use for the seasonal components. @@ -106,9 +104,6 @@ class MATHS_EXPORT CTimeSeriesDecomposition : public CTimeSeriesDecompositionInt //! Get the decay rate. virtual double decayRate(void) const; - //! Switch to using this trend to forecast. - virtual void forForecasting(void); - //! Check if the decomposition has any initialized components. virtual bool initialized(void) const; @@ -132,31 +127,36 @@ class MATHS_EXPORT CTimeSeriesDecomposition : public CTimeSeriesDecompositionInt //! Propagate the decomposition forwards to \p time. void propagateForwardsTo(core_t::TTime time); - //! May be test to see if there are any new seasonal components - //! and interpolate. - //! - //! \param[in] time The current time. - //! \return True if the number of seasonal components changed - //! and false otherwise. - virtual bool testAndInterpolate(core_t::TTime time); - //! Get the mean value of the baseline in the vicinity of \p time. virtual double mean(core_t::TTime time) const; //! Get the value of the time series baseline at \p time. //! //! \param[in] time The time of interest. - //! \param[in] predictionConfidence The symmetric confidence interval - //! for the prediction the baseline as a percentage. - //! \param[in] forecastConfidence The symmetric confidence interval - //! for long range forecasts as a percentage. + //! \param[in] confidence The symmetric confidence interval for the prediction + //! the baseline as a percentage. //! \param[in] components The components to include in the baseline. virtual maths_t::TDoubleDoublePr baseline(core_t::TTime time, - double predictionConfidence = 0.0, - double forecastConfidence = 0.0, - EComponents components = E_All, + double confidence = 0.0, + int components = E_All, bool smooth = true) const; + //! Forecast from \p start to \p end at \p dt intervals. + //! + //! \param[in] startTime The start of the forecast. + //! \param[in] endTime The end of the forecast. + //! \param[in] step The time increment. + //! \param[in] confidence The forecast confidence interval. + //! \param[in] minimumScale The minimum permitted seasonal scale. + //! \param[in] result Filled in with the forecast lower bound, prediction + //! and upper bound. + virtual void forecast(core_t::TTime startTime, + core_t::TTime endTime, + core_t::TTime step, + double confidence, + double minimumScale, + TDouble3VecVec &result); + //! Detrend \p value from the time series being modeled by removing //! any trend and periodic component at \p time. virtual double detrend(core_t::TTime time, double value, double confidence) const; @@ -207,23 +207,20 @@ class MATHS_EXPORT CTimeSeriesDecomposition : public CTimeSeriesDecompositionInt //! Create from part of a state document. bool acceptRestoreTraverser(core::CStateRestoreTraverser &traverser); - //! Check if we are forecasting the model. - bool forecasting(void) const; - //! The correction to produce a smooth join between periodic //! repeats and partitions. template maths_t::TDoubleDoublePr smooth(const F &f, core_t::TTime time, - EComponents components) const; + int components) const; //! Check if \p component has been selected. bool selected(core_t::TTime time, - EComponents components, + int components, const CSeasonalComponent &component) const; //! Check if \p components match \p component. - bool matches(EComponents components, const CSeasonalComponent &component) const; + bool matches(int components, const CSeasonalComponent &component) const; private: //! The time over which discontinuities between weekdays @@ -231,9 +228,6 @@ class MATHS_EXPORT CTimeSeriesDecomposition : public CTimeSeriesDecompositionInt static const core_t::TTime SMOOTHING_INTERVAL; private: - //! Controls the rate at which information is lost. - double m_DecayRate; - //! The time of the latest value added. core_t::TTime m_LastValueTime; @@ -244,22 +238,14 @@ class MATHS_EXPORT CTimeSeriesDecomposition : public CTimeSeriesDecompositionInt //! components. TMediatorPtr m_Mediator; - //! The test for a large time scale smooth trend. - CLongTermTrendTest m_LongTermTrendTest; - - //! The test for daily and weekly seasonal components. - CDiurnalTest m_DiurnalTest; - - //! The test for general seasonal components. - CNonDiurnalTest m_GeneralSeasonalityTest; + //! The test for seasonal components. + CPeriodicityTest m_PeriodicityTest; //! The test for calendar cyclic components. CCalendarTest m_CalendarCyclicTest; //! The state for modeling the components of the decomposition. CComponents m_Components; - - friend class ::CTimeSeriesDecompositionTest; }; } diff --git a/include/maths/CTimeSeriesDecompositionDetail.h b/include/maths/CTimeSeriesDecompositionDetail.h index b4e32a8775..d0c7114ffc 100644 --- a/include/maths/CTimeSeriesDecompositionDetail.h +++ b/include/maths/CTimeSeriesDecompositionDetail.h @@ -24,44 +24,47 @@ #include #include #include +#include +#include #include #include #include #include +#include #include namespace ml { namespace maths { +class CExpandingWindow; +class CTimeSeriesDecomposition; //! \brief Utilities for computing the decomposition. class MATHS_EXPORT CTimeSeriesDecompositionDetail { public: + using TPredictor = std::function; using TDoubleVec = std::vector; using TTimeVec = std::vector; - using TRegression = CRegression::CLeastSquaresOnline<3, double>; - using TRegressionParameterProcess = CRegression::CLeastSquaresOnlineParameterProcess<4, double>; class CMediator; //! \brief The base message passed. struct MATHS_EXPORT SMessage { - SMessage(void); SMessage(core_t::TTime time, core_t::TTime lastTime); //! The message time. core_t::TTime s_Time; - //! The last update time. core_t::TTime s_LastTime; }; //! \brief The message passed to add a point. - struct MATHS_EXPORT SAddValue : public SMessage + struct MATHS_EXPORT SAddValue : public SMessage, + private core::CNonCopyable { SAddValue(core_t::TTime time, core_t::TTime lastTime, @@ -69,82 +72,65 @@ class MATHS_EXPORT CTimeSeriesDecompositionDetail const maths_t::TWeightStyleVec &weightStyles, const maths_t::TDouble4Vec &weights, double trend, - double nonDiurnal, double seasonal, - double calendar); + double calendar, + const TPredictor &predictor, + const CPeriodicityHypothesisTestsConfig &periodicityTestConfig); + //! The value to add. double s_Value; - //! The styles of the weights. Both the count and the Winsorisation - //! weight styles have an effect. See maths_t::ESampleWeightStyle - //! for more details. + //! The styles of the weights. const maths_t::TWeightStyleVec &s_WeightStyles; - //! The weights of associated with the value. The smaller the count - //! weight the less influence the value has on the trend and it's - //! local variance. + //! The weights of associated with the value. const maths_t::TDouble4Vec &s_Weights; //! The trend component prediction at the value's time. double s_Trend; - //! The non daily/weekly seasonal components' prediction at the - //! value's time. - double s_NonDiurnal; //! The seasonal component prediction at the value's time. double s_Seasonal; //! The calendar component prediction at the value's time. double s_Calendar; + //! The predictor for value. + TPredictor s_Predictor; + //! The periodicity test configuration. + CPeriodicityHypothesisTestsConfig s_PeriodicityTestConfig; }; - //! \brief The message passed to indicate a trend has been detected. - struct MATHS_EXPORT SDetectedTrend : public SMessage + //! \brief The message passed to indicate periodic components have + //! been detected. + struct MATHS_EXPORT SDetectedSeasonal : public SMessage { - SDetectedTrend(core_t::TTime time, - core_t::TTime lastTime, - const CTrendTest &test); - const CTrendTest &s_Test; - }; - - //! \brief The message passed to indicate diurnal periodic components - //! have been detected. - struct MATHS_EXPORT SDetectedDiurnal : public SMessage - { - SDetectedDiurnal(core_t::TTime time, - core_t::TTime lastTime, - const CPeriodicityTestResult &result, - const CDiurnalPeriodicityTest &test); - CPeriodicityTestResult s_Result; - const CDiurnalPeriodicityTest &s_Test; - }; - - //! \brief The message passed to indicate general periodic components - //! have been detected. - struct MATHS_EXPORT SDetectedNonDiurnal : public SMessage - { - SDetectedNonDiurnal(core_t::TTime time, - core_t::TTime lastTime, - bool discardLongTermTrend, - const CPeriodicityTestResult &result, - const CGeneralPeriodicityTest &test); - bool s_DiscardLongTermTrend; - CPeriodicityTestResult s_Result; - const CGeneralPeriodicityTest &s_Test; + SDetectedSeasonal(core_t::TTime time, + core_t::TTime lastTime, + const CPeriodicityHypothesisTestsResult &result, + const CExpandingWindow &window, + const TPredictor &predictor); + + //! The components found. + CPeriodicityHypothesisTestsResult s_Result; + //! The window tested. + const CExpandingWindow &s_Window; + //! The predictor for window values. + TPredictor s_Predictor; }; - //! \brief The mssage passed to indicate calendar components have been - //! detected. + //! \brief The message passed to indicate calendar components have + //! been detected. struct MATHS_EXPORT SDetectedCalendar : public SMessage { SDetectedCalendar(core_t::TTime time, core_t::TTime lastTime, CCalendarFeature feature); + + //! The calendar feature found. CCalendarFeature s_Feature; }; - //! \brief The message passed to indicate new diurnal components are - //! being modeled. + //! \brief The message passed to indicate new components are being + //! modeled. struct MATHS_EXPORT SNewComponents : public SMessage { enum EComponent { - E_Trend, E_DiurnalSeasonal, E_GeneralSeasonal, E_CalendarCyclic @@ -169,14 +155,8 @@ class MATHS_EXPORT CTimeSeriesDecompositionDetail //! Add a value. virtual void handle(const SAddValue &message); - //! Handle when a trend is detected. - virtual void handle(const SDetectedTrend &message); - //! Handle when a diurnal component is detected. - virtual void handle(const SDetectedDiurnal &message); - - //! Handle when a non-diurnal seasonal component is detected. - virtual void handle(const SDetectedNonDiurnal &message); + virtual void handle(const SDetectedSeasonal &message); //! Handle when a calendar component is detected. virtual void handle(const SDetectedCalendar &message); @@ -221,177 +201,13 @@ class MATHS_EXPORT CTimeSeriesDecompositionDetail THandlerRefVec m_Handlers; }; - //! \brief Tests for a long term trend. - class MATHS_EXPORT CLongTermTrendTest : public CHandler - { - public: - CLongTermTrendTest(double decayRate); - CLongTermTrendTest(const CLongTermTrendTest &other); - - //! Initialize by reading state from \p traverser. - bool acceptRestoreTraverser(core::CStateRestoreTraverser &traverser); - - //! Persist state by passing information to \p inserter. - void acceptPersistInserter(core::CStatePersistInserter &inserter) const; - - //! Efficiently swap the state of this and \p other. - void swap(CLongTermTrendTest &other); - - //! Update the test with a new value. - virtual void handle(const SAddValue &message); - - //! Reset the test if still testing. - virtual void handle(const SNewComponents &message); - - //! Check if the time series has shifted level. - void test(const SMessage &message); - - //! Set the decay rate. - void decayRate(double decayRate); - - //! Age the test to account for the interval \p end - \p start - //! elapsed time. - void propagateForwards(core_t::TTime start, core_t::TTime end); - - //! Roll time forwards by \p skipInterval. - void skipTime(core_t::TTime skipInterval); - - //! Get a checksum for this object. - uint64_t checksum(uint64_t seed = 0) const; - - //! Debug the memory used by this object. - void debugMemoryUsage(core::CMemoryUsage::TMemoryUsagePtr mem) const; - - //! Get the memory used by this object. - std::size_t memoryUsage(void) const; - - private: - using TTrendTestPtr = boost::shared_ptr; - - private: - //! Handle \p symbol. - void apply(std::size_t symbol, const SMessage &message); - - //! Check if we should run a test. - bool shouldTest(core_t::TTime time); - - //! Get the interval between tests. - core_t::TTime testInterval(void) const; - - private: - //! The state machine. - core::CStateMachine m_Machine; - - //! The maximum rate at which information is lost. - double m_MaximumDecayRate; - - //! The next time to test for a long term trend. - core_t::TTime m_NextTestTime; - - //! The test for a long term trend. - TTrendTestPtr m_Test; - }; - - //! \brief Tests for daily and weekly periodic components and weekend - //! weekday splits of the time series. - class MATHS_EXPORT CDiurnalTest : public CHandler - { - public: - CDiurnalTest(double decayRate, core_t::TTime bucketLength); - CDiurnalTest(const CDiurnalTest &other); - - //! Initialize by reading state from \p traverser. - bool acceptRestoreTraverser(core::CStateRestoreTraverser &traverser); - - //! Persist state by passing information to \p inserter. - void acceptPersistInserter(core::CStatePersistInserter &inserter) const; - - //! Efficiently swap the state of this and \p other. - void swap(CDiurnalTest &other); - - //! Update the test with a new value. - virtual void handle(const SAddValue &message); - - //! Reset the test. - virtual void handle(const SNewComponents &message); - - //! Test to see whether any seasonal components are present. - void test(const SMessage &message); - - //! Age the test to account for the interval \p end - \p start - //! elapsed time. - void propagateForwards(core_t::TTime start, core_t::TTime end); - - //! Roll time forwards by \p skipInterval. - void skipTime(core_t::TTime skipInterval); - - //! Get a checksum for this object. - uint64_t checksum(uint64_t seed = 0) const; - - //! Debug the memory used by this object. - void debugMemoryUsage(core::CMemoryUsage::TMemoryUsagePtr mem) const; - - //! Get the memory used by this object. - std::size_t memoryUsage(void) const; - - private: - using TRandomizedPeriodicityTestPtr = boost::shared_ptr; - using TPeriodicityTestPtr = boost::shared_ptr; - - private: - //! Handle \p symbol. - void apply(std::size_t symbol, const SMessage &message); - - //! Check if we should run a test. - bool shouldTest(core_t::TTime time); - - //! Get the interval between tests. - core_t::TTime testInterval(void) const; - - //! Get the time at which to time out the regular test. - core_t::TTime timeOutRegularTest(void) const; - - //! Account for memory that is not yet allocated - //! during the initial state - std::size_t extraMemoryOnInitialization(void) const; - - private: - //! The state machine. - core::CStateMachine m_Machine; - - //! Controls the rate at which information is lost. - double m_DecayRate; - - //! The raw data bucketing interval. - core_t::TTime m_BucketLength; - - //! The next time to test for periodic components. - core_t::TTime m_NextTestTime; - - //! The time at which we began regular testing. - core_t::TTime m_StartedRegularTest; - - //! The time at which we switch to the small test to save memory. - core_t::TTime m_TimeOutRegularTest; - - //! The test for periodic components. - TPeriodicityTestPtr m_RegularTest; - - //! A small but slower test for periodic components that is used - //! after a while if the regular test is inconclusive. - TRandomizedPeriodicityTestPtr m_SmallTest; - - //! The result of the last test for periodic components. - CPeriodicityTestResult m_Periods; - }; - - //! \brief Scans through increasingly low frequencies looking for the - //! the largest amplitude seasonal components. - class MATHS_EXPORT CNonDiurnalTest : public CHandler + //! \brief Scans through increasingly low frequencies looking for custom + //! diurnal and any other large amplitude seasonal components. + class MATHS_EXPORT CPeriodicityTest : public CHandler { public: - CNonDiurnalTest(double decayRate, core_t::TTime bucketLength); - CNonDiurnalTest(const CNonDiurnalTest &other); + CPeriodicityTest(double decayRate, core_t::TTime bucketLength); + CPeriodicityTest(const CPeriodicityTest &other); //! Initialize by reading state from \p traverser. bool acceptRestoreTraverser(core::CStateRestoreTraverser &traverser); @@ -400,7 +216,7 @@ class MATHS_EXPORT CTimeSeriesDecompositionDetail void acceptPersistInserter(core::CStatePersistInserter &inserter) const; //! Efficiently swap the state of this and \p other. - void swap(CNonDiurnalTest &other); + void swap(CPeriodicityTest &other); //! Update the test with a new value. virtual void handle(const SAddValue &message); @@ -409,15 +225,12 @@ class MATHS_EXPORT CTimeSeriesDecompositionDetail virtual void handle(const SNewComponents &message); //! Test to see whether any seasonal components are present. - void test(const SMessage &message); + void test(const SAddValue &message); //! Age the test to account for the interval \p end - \p start //! elapsed time. void propagateForwards(core_t::TTime start, core_t::TTime end); - //! Roll time forwards by \p skipInterval. - void skipTime(core_t::TTime skipInterval); - //! Get a checksum for this object. uint64_t checksum(uint64_t seed = 0) const; @@ -429,16 +242,13 @@ class MATHS_EXPORT CTimeSeriesDecompositionDetail private: using TTimeAry = boost::array; - using TScanningPeriodicityTestPtr = boost::shared_ptr; - using TScanningPeriodicityTestPtrAry = boost::array; + using TExpandingWindowPtr = boost::shared_ptr; + using TExpandingWindowPtrAry = boost::array; //! Test types (categorised as short and long period tests). enum ETest { E_Short, E_Long }; private: - //! The size of the periodicity test. - static const std::size_t TEST_SIZE; - //! The bucket lengths to use to test for short period components. static const TTimeVec SHORT_BUCKET_LENGTHS; @@ -447,12 +257,13 @@ class MATHS_EXPORT CTimeSeriesDecompositionDetail private: //! Handle \p symbol. - void apply(std::size_t symbol, - const SMessage &message, - const TTimeAry &offsets = {{345600, 345600}}); // 4 * DAY + void apply(std::size_t symbol, const SMessage &message); + + //! Check if we should run the periodicity test on \p window. + bool shouldTest(const TExpandingWindowPtr &window, core_t::TTime time) const; //! Get a new \p test. (Warning owned by the caller.) - CScanningPeriodicityTest *newTest(ETest test) const; + CExpandingWindow *newWindow(ETest test) const; //! Account for memory that is not yet allocated //! during the initial state @@ -468,8 +279,8 @@ class MATHS_EXPORT CTimeSeriesDecompositionDetail //! The raw data bucketing interval. core_t::TTime m_BucketLength; - //! The test for arbitrary periodic components. - TScanningPeriodicityTestPtrAry m_Tests; + //! Expanding windows on the "recent" time series values. + TExpandingWindowPtrAry m_Windows; }; //! \brief Tests for cyclic calendar components explaining large prediction @@ -502,9 +313,6 @@ class MATHS_EXPORT CTimeSeriesDecompositionDetail //! elapsed time. void propagateForwards(core_t::TTime start, core_t::TTime end); - //! Roll time forwards to \p time. - void advanceTimeTo(core_t::TTime time); - //! Get a checksum for this object. uint64_t checksum(uint64_t seed = 0) const; @@ -544,66 +352,6 @@ class MATHS_EXPORT CTimeSeriesDecompositionDetail TCalendarCyclicTestPtr m_Test; }; - //! \brief A reference to the long term trend. - class MATHS_EXPORT CTrendCRef - { - public: - using TMatrix = CSymmetricMatrixNxN; - - public: - CTrendCRef(void); - CTrendCRef(const TRegression ®ression, - double variance, - core_t::TTime timeOrigin, - core_t::TTime lastUpdate, - const TRegressionParameterProcess &process); - - //! Check if the trend has been initialized. - bool initialized(void) const; - - //! Get the count of values added to the trend. - double count(void) const; - - //! Predict the long term trend at \p time with confidence - //! interval \p confidence. - maths_t::TDoubleDoublePr prediction(core_t::TTime time, double confidence) const; - - //! Get the variance about the long term trend. - double variance(void) const; - - //! Get the covariance matrix of the regression parameters' - //! at \p time. - //! - //! \param[out] result Filled in with the regression parameters' - //! covariance matrix. - bool covariances(TMatrix &result) const; - - //! Get the variance in the prediction due to drift in the - //! regression model parameters expected by \p time. - double varianceDueToParameterDrift(core_t::TTime time) const; - - //! Get the time at which to evaluate the regression model - //! of the trend. - double time(core_t::TTime time) const; - - private: - //! The regression model of the trend. - const TRegression *m_Trend; - - //! The variance of the prediction residuals. - double m_Variance; - - //! The origin of the time coordinate system. - core_t::TTime m_TimeOrigin; - - //! The time of the last update of the regression model. - core_t::TTime m_LastUpdate; - - //! The Wiener process which describes the evolution of the - //! regression model parameters. - const TRegressionParameterProcess *m_ParameterProcess; - }; - //! \brief Holds and updates the components of the decomposition. class MATHS_EXPORT CComponents : public CHandler { @@ -643,52 +391,52 @@ class MATHS_EXPORT CTimeSeriesDecompositionDetail //! Update the components with a new value. virtual void handle(const SAddValue &message); - //! Create a new trend component. - virtual void handle(const SDetectedTrend &message); - - //! Create new diurnal components. - virtual void handle(const SDetectedDiurnal &message); - - //! Create new general seasonal component. - virtual void handle(const SDetectedNonDiurnal &message); + //! Create new seasonal components. + virtual void handle(const SDetectedSeasonal &message); //! Create a new calendar component. virtual void handle(const SDetectedCalendar &message); //! Maybe re-interpolate the components. - void interpolate(const SMessage &message); + void interpolate(const SMessage &message, bool refine = true); //! Set the decay rate. void decayRate(double decayRate); + //! Get the decay rate. + double decayRate(void) const; + //! Age the components to account for the interval \p end - \p start //! elapsed time. void propagateForwards(core_t::TTime start, core_t::TTime end); - //! Check if we're forecasting. - bool forecasting(void) const; - - //! Start forecasting. - void forecast(void); - //! Check if the decomposition has any initialized components. bool initialized(void) const; //! Get the long term trend. - CTrendCRef trend(void) const; + const CTrendComponent &trend(void) const; - //! Get the components. + //! Get the seasonal components. const maths_t::TSeasonalComponentVec &seasonal(void) const; //! Get the calendar components. const maths_t::TCalendarComponentVec &calendar(void) const; + //! Return true if we're using the trend for prediction. + bool usingTrendForPrediction(void) const; + + //! Get configuration for the periodicity test. + CPeriodicityHypothesisTestsConfig periodicityTestConfig(void) const; + //! Get the mean value of the baseline in the vicinity of \p time. double meanValue(core_t::TTime time) const; //! Get the mean variance of the baseline. double meanVariance(void) const; + //! Get the mean error variance scale for the components. + double meanVarianceScale(void) const; + //! Get a checksum for this object. uint64_t checksum(uint64_t seed = 0) const; @@ -704,7 +452,6 @@ class MATHS_EXPORT CTimeSeriesDecompositionDetail using TSeasonalComponentPtrVec = std::vector; using TCalendarComponentPtrVec = std::vector; using TFloatMeanAccumulator = CBasicStatistics::SSampleMean::TAccumulator; - using TVector = CVectorNx1; //! \brief Tracks prediction errors with and without components. //! @@ -759,45 +506,6 @@ class MATHS_EXPORT CTimeSeriesDecompositionDetail using TComponentErrorsVec = std::vector; using TComponentErrorsPtrVec = std::vector; - //! \brief The long term trend. - struct MATHS_EXPORT STrend - { - STrend(void); - - //! Initialize by reading state from \p traverser. - bool acceptRestoreTraverser(core::CStateRestoreTraverser &traverser); - - //! Persist state by passing information to \p inserter. - void acceptPersistInserter(core::CStatePersistInserter &inserter) const; - - //! Get a reference to this trend. - CTrendCRef reference(void) const; - - //! Shift the regression's time origin to \p time. - void shiftOrigin(core_t::TTime time); - - //! Get a checksum for this object. - uint64_t checksum(uint64_t seed = 0) const; - - //! The regression model of the trend. - TRegression s_Regression; - - //! The variance of the trend. - double s_Variance; - - //! The origin of the time coordinate system. - core_t::TTime s_TimeOrigin; - - //! The time of the last update of the regression model. - core_t::TTime s_LastUpdate; - - //! The Wiener process which describes the evolution of the - //! regression model parameters. - TRegressionParameterProcess s_ParameterProcess; - }; - - using TTrendPtr = boost::shared_ptr; - //! \brief The seasonal components of the decomposition. struct MATHS_EXPORT SSeasonal { @@ -819,9 +527,6 @@ class MATHS_EXPORT CTimeSeriesDecompositionDetail //! Get the combined size of the seasonal components. std::size_t size(void) const; - //! Check if there is already a component with \p period. - bool haveComponent(core_t::TTime period) const; - //! Get the state to update. void componentsErrorsAndDeltas(core_t::TTime time, TSeasonalComponentPtrVec &components, @@ -928,14 +633,15 @@ class MATHS_EXPORT CTimeSeriesDecompositionDetail std::size_t maxSize(void) const; //! Add new seasonal components to \p components. - void addSeasonalComponents(const CPeriodicityTest &test, - const CPeriodicityTestResult &result, - core_t::TTime time, + bool addSeasonalComponents(const CPeriodicityHypothesisTestsResult &result, + const CExpandingWindow &window, + const TPredictor &predictor, + CTrendComponent &trend, maths_t::TSeasonalComponentVec &components, TComponentErrorsVec &errors) const; //! Add a new calendar component to \p components. - void addCalendarComponent(const CCalendarFeature &feature, + bool addCalendarComponent(const CCalendarFeature &feature, core_t::TTime time, maths_t::TCalendarComponentVec &components, TComponentErrorsVec &errors) const; @@ -980,7 +686,7 @@ class MATHS_EXPORT CTimeSeriesDecompositionDetail std::size_t m_CalendarComponentSize; //! The long term trend. - TTrendPtr m_Trend; + CTrendComponent m_Trend; //! The seasonal components. TSeasonalPtr m_Seasonal; @@ -988,28 +694,26 @@ class MATHS_EXPORT CTimeSeriesDecompositionDetail //! The calendar components. TCalendarPtr m_Calendar; + //! The mean error variance scale for the components. + TFloatMeanAccumulator m_MeanVarianceScale; + + //! The moments of the values added. + TMeanVarAccumulator m_Moments; + + //! The moments of the values added after subtracting a trend. + TMeanVarAccumulator m_MomentsMinusTrend; + + //! Set to true if the trend model should be used for prediction. + bool m_UsingTrendForPrediction; + //! Set to true if non-null when the seasonal components change. bool *m_Watcher; }; }; //! Create a free function which will be found by Koenig lookup. -inline void swap(CTimeSeriesDecompositionDetail::CLongTermTrendTest &lhs, - CTimeSeriesDecompositionDetail::CLongTermTrendTest &rhs) -{ - lhs.swap(rhs); -} - -//! Create a free function which will be found by Koenig lookup. -inline void swap(CTimeSeriesDecompositionDetail::CDiurnalTest &lhs, - CTimeSeriesDecompositionDetail::CDiurnalTest &rhs) -{ - lhs.swap(rhs); -} - -//! Create a free function which will be found by Koenig lookup. -inline void swap(CTimeSeriesDecompositionDetail::CNonDiurnalTest &lhs, - CTimeSeriesDecompositionDetail::CNonDiurnalTest &rhs) +inline void swap(CTimeSeriesDecompositionDetail::CPeriodicityTest &lhs, + CTimeSeriesDecompositionDetail::CPeriodicityTest &rhs) { lhs.swap(rhs); } diff --git a/include/maths/CTimeSeriesDecompositionInterface.h b/include/maths/CTimeSeriesDecompositionInterface.h index f7db586c6c..72e782b091 100644 --- a/include/maths/CTimeSeriesDecompositionInterface.h +++ b/include/maths/CTimeSeriesDecompositionInterface.h @@ -45,18 +45,23 @@ class CSeasonalComponent; class MATHS_EXPORT CTimeSeriesDecompositionInterface { public: + using TDouble3Vec = core::CSmallVector; + using TDouble3VecVec = std::vector; using TDoubleAry = boost::array; using TWeights = CConstantWeights; //! The components of the decomposition. enum EComponents { - E_Diurnal = 0x1, - E_NonDiurnal = 0x2, - E_Seasonal = 0x3, - E_Trend = 0x4, - E_Calendar = 0x8, - E_All = 0xf + E_Diurnal = 0x1, + E_NonDiurnal = 0x2, + E_Seasonal = 0x3, + E_Trend = 0x4, + E_Calendar = 0x8, + E_All = 0xf, + E_TrendForced = 0x10 //!< Force get the trend component (if + //!< it's not being used for prediction). + //!< This needs to be bigger than E_All. }; public: @@ -71,12 +76,6 @@ class MATHS_EXPORT CTimeSeriesDecompositionInterface //! Get the decay rate. virtual double decayRate(void) const = 0; - //! Switch to using this trend to forecast. - //! - //! \warning This is an irreversible action so if the trend - //! is still need it should be copied first. - virtual void forForecasting(void) = 0; - //! Check if this is initialized. virtual bool initialized(void) const = 0; @@ -100,31 +99,36 @@ class MATHS_EXPORT CTimeSeriesDecompositionInterface //! Propagate the decomposition forwards to \p time. virtual void propagateForwardsTo(core_t::TTime time) = 0; - //! May be test to see if there are any new seasonal components - //! and interpolate. - //! - //! \param[in] time The current time. - //! \return True if the number of seasonal components changed - //! and false otherwise. - virtual bool testAndInterpolate(core_t::TTime time) = 0; - //! Get the mean value of the baseline in the vicinity of \p time. virtual double mean(core_t::TTime time) const = 0; //! Get the value of the time series baseline at \p time. //! //! \param[in] time The time of interest. - //! \param[in] predictionConfidence The symmetric confidence interval - //! for the prediction the baseline as a percentage. - //! \param[in] forecastConfidence The symmetric confidence interval - //! for long range forecasts as a percentage. + //! \param[in] confidence The symmetric confidence interval for the prediction + //! the baseline as a percentage. //! \param[in] components The components to include in the baseline. virtual maths_t::TDoubleDoublePr baseline(core_t::TTime time, - double predictionConfidence = 0.0, - double forecastConfidence = 0.0, - EComponents components = E_All, + double confidence = 0.0, + int components = E_All, bool smooth = true) const = 0; + //! Forecast from \p start to \p end at \p dt intervals. + //! + //! \param[in] startTime The start of the forecast. + //! \param[in] endTime The end of the forecast. + //! \param[in] step The time increment. + //! \param[in] confidence The forecast confidence interval. + //! \param[in] minimumScale The minimum permitted seasonal scale. + //! \param[in] result Filled in with the forecast lower bound, prediction + //! and upper bound. + virtual void forecast(core_t::TTime startTime, + core_t::TTime endTime, + core_t::TTime step, + double confidence, + double minimumScale, + TDouble3VecVec &result) = 0; + //! Detrend \p value from the time series being modeled by removing //! any periodic component at \p time. //! @@ -167,23 +171,6 @@ class MATHS_EXPORT CTimeSeriesDecompositionInterface virtual core_t::TTime lastValueTime(void) const = 0; }; -using TDecompositionPtr = boost::shared_ptr; -using TDecompositionPtr10Vec = core::CSmallVector; - -//! Initialize a univariate prior to match the moments of \p decomposition. -MATHS_EXPORT -bool initializePrior(core_t::TTime bucketLength, - double learnRate, - const CTimeSeriesDecompositionInterface &decomposition, - CPrior &prior); - -//! Initialize a multivariate prior to match the moments of \p decomposition. -MATHS_EXPORT -bool initializePrior(core_t::TTime bucketLength, - double learnRate, - const TDecompositionPtr10Vec &decomposition, - CMultivariatePrior &prior); - } } diff --git a/include/maths/CTimeSeriesDecompositionStub.h b/include/maths/CTimeSeriesDecompositionStub.h index 35e60d1ed9..8924153305 100644 --- a/include/maths/CTimeSeriesDecompositionStub.h +++ b/include/maths/CTimeSeriesDecompositionStub.h @@ -43,9 +43,6 @@ class MATHS_EXPORT CTimeSeriesDecompositionStub : public CTimeSeriesDecompositio //! Get the decay rate. virtual double decayRate(void) const; - //! No-op. - virtual void forForecasting(void); - //! Returns false. virtual bool initialized(void) const; @@ -58,19 +55,23 @@ class MATHS_EXPORT CTimeSeriesDecompositionStub : public CTimeSeriesDecompositio //! No-op. virtual void propagateForwardsTo(core_t::TTime time); - //! No-op returning false. - virtual bool testAndInterpolate(core_t::TTime time); - //! Returns 0. virtual double mean(core_t::TTime time) const; //! Returns (0.0, 0.0). virtual maths_t::TDoubleDoublePr baseline(core_t::TTime time, - double predictionConfidence = 0.0, - double forecastConfidence = 0.0, - EComponents components = E_All, + double confidence = 0.0, + int components = E_All, bool smooth = true) const; + //! Clears \p result. + virtual void forecast(core_t::TTime startTime, + core_t::TTime endTime, + core_t::TTime step, + double confidence, + double minimumScale, + TDouble3VecVec &result); + //! Returns \p value. virtual double detrend(core_t::TTime time, double value, double confidence) const; diff --git a/include/maths/CTimeSeriesModel.h b/include/maths/CTimeSeriesModel.h index 56bb83fc70..a2bba1d60c 100644 --- a/include/maths/CTimeSeriesModel.h +++ b/include/maths/CTimeSeriesModel.h @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -42,6 +43,8 @@ struct SModelRestoreParams; class MATHS_EXPORT CUnivariateTimeSeriesModel : public CModel { public: + using TTimeDoublePr = std::pair; + using TTimeDoublePrCBuf = boost::circular_buffer; using TDecayRateController2Ary = boost::array; public: @@ -122,9 +125,9 @@ class MATHS_EXPORT CUnivariateTimeSeriesModel : public CModel //! Get the prediction and \p confidenceInterval percentage //! confidence interval for the time series at \p time. virtual TDouble2Vec3Vec confidenceInterval(core_t::TTime time, + double confidenceInterval, const maths_t::TWeightStyleVec &weightStyles, - const TDouble2Vec4Vec &weights, - double confidenceInterval) const; + const TDouble2Vec4Vec &weights) const; //! Forecast the time series and get its \p confidenceInterval //! percentage confidence interval between \p startTime and @@ -173,11 +176,17 @@ class MATHS_EXPORT CUnivariateTimeSeriesModel : public CModel //! Get the type of data being modeled. virtual maths_t::EDataType dataType(void) const; + //! \name Test Functions + //@{ + //! Get the sliding window of recent values. + const TTimeDoublePrCBuf &slidingWindow(void) const; + //! Get the trend. const CTimeSeriesDecompositionInterface &trend(void) const; //! Get the prior. const CPrior &prior(void) const; + //@} private: using TDouble1Vec = core::CSmallVector; @@ -221,6 +230,9 @@ class MATHS_EXPORT CUnivariateTimeSeriesModel : public CModel //! True if the model can be forecast. bool m_IsForecastable; + //! A random number generator for sampling the sliding window. + CPRNG::CXorOShiro128Plus m_Rng; + //! These control the trend and prior decay rates (see CDecayRateController //! for more details). TDecayRateController2AryPtr m_Controllers; @@ -235,6 +247,10 @@ class MATHS_EXPORT CUnivariateTimeSeriesModel : public CModel //! of the time series. TAnomalyModelPtr m_AnomalyModel; + //! A sliding window of the recent samples (used to reinitialize the + //! residual model when a new trend component is detected). + TTimeDoublePrCBuf m_SlidingWindow; + //! Models the correlations between time series. CTimeSeriesCorrelations *m_Correlations; }; @@ -444,6 +460,8 @@ class MATHS_EXPORT CTimeSeriesCorrelations class MATHS_EXPORT CMultivariateTimeSeriesModel : public CModel { public: + using TTimeDouble2VecPr = std::pair; + using TTimeDouble2VecPrCBuf = boost::circular_buffer; using TDecompositionPtr = boost::shared_ptr; using TDecompositionPtr10Vec = core::CSmallVector; using TDecayRateController2Ary = boost::array; @@ -523,9 +541,9 @@ class MATHS_EXPORT CMultivariateTimeSeriesModel : public CModel //! Get the prediction and \p confidenceInterval percentage //! confidence interval for the time series at \p time. virtual TDouble2Vec3Vec confidenceInterval(core_t::TTime time, + double confidenceInterval, const maths_t::TWeightStyleVec &weightStyles, - const TDouble2Vec4Vec &weights, - double confidenceInterval) const; + const TDouble2Vec4Vec &weights) const; //! Not currently supported. virtual bool forecast(core_t::TTime startTime, @@ -572,11 +590,17 @@ class MATHS_EXPORT CMultivariateTimeSeriesModel : public CModel //! Get the type of data being modeled. virtual maths_t::EDataType dataType(void) const; + //! \name Test Functions + //@{ + //! Get the sliding window of recent values. + const TTimeDouble2VecPrCBuf &slidingWindow(void) const; + //! Get the trend. const TDecompositionPtr10Vec &trend(void) const; //! Get the prior. const CMultivariatePrior &prior(void) const; + //@} private: using TDouble1Vec = core::CSmallVector; @@ -606,6 +630,9 @@ class MATHS_EXPORT CMultivariateTimeSeriesModel : public CModel //! True if the data are non-negative. bool m_IsNonNegative; + //! A random number generator for sampling the sliding window. + CPRNG::CXorOShiro128Plus m_Rng; + //! These control the trend and prior decay rates (see CDecayRateController //! for more details). TDecayRateController2AryPtr m_Controllers; @@ -619,6 +646,10 @@ class MATHS_EXPORT CMultivariateTimeSeriesModel : public CModel //! A model for time periods when the basic model can't predict the value //! of the time series. TAnomalyModelPtr m_AnomalyModel; + + //! A sliding window of the recent samples (used to reinitialize the + //! residual model when a new trend component is detected). + TTimeDouble2VecPrCBuf m_SlidingWindow; }; } diff --git a/include/maths/CTools.h b/include/maths/CTools.h index 47e964a7fd..80204047f5 100644 --- a/include/maths/CTools.h +++ b/include/maths/CTools.h @@ -725,6 +725,29 @@ class MATHS_EXPORT CTools : private core::CNonInstantiatable //! Shift \p x to the right by \p eps times \p x. static double shiftRight(double x, double eps = std::numeric_limits::epsilon()); + + //! Sigmoid function of \p p. + static double sigmoid(double p) + { + return 1.0 / (1.0 + 1.0 / p); + } + + //! A smooth Heaviside function centred at one. + //! + //! This is a smooth version of the Heaviside function implemented + //! as \f$sigmoid\left(\frac{sign (x - 1)}{wb}\right)\f$ normalized + //! to the range [0, 1], where \f$b\f$ is \p boundary and \f$w\f$ + //! is \p width. Note, if \p sign is one this is a step up and if + //! it is -1 it is a step down. + //! + //! \param[in] x The argument. + //! \param[in] width The step width. + //! \param[in] sign Determines whether it's a step up or down. + static double smoothHeaviside(double x, double width, double sign = 1.0) + { + return sigmoid(std::exp(sign * (x - 1.0) / width)) + / sigmoid(std::exp(1.0 / width)); + } }; } diff --git a/include/maths/CTrendComponent.h b/include/maths/CTrendComponent.h new file mode 100644 index 0000000000..7f9cdc7065 --- /dev/null +++ b/include/maths/CTrendComponent.h @@ -0,0 +1,207 @@ +/* + * ELASTICSEARCH CONFIDENTIAL + * + * Copyright (c) 2017 Elasticsearch BV. All Rights Reserved. + * + * Notice: this software, and all information contained + * therein, is the exclusive property of Elasticsearch BV + * and its licensors, if any, and is protected under applicable + * domestic and foreign law, and international treaties. + * + * Reproduction, republication or distribution without the + * express written consent of Elasticsearch BV is + * strictly prohibited. + */ + +#ifndef INCLUDED_ml_maths_CTrendComponent_h +#define INCLUDED_ml_maths_CTrendComponent_h + +#include + +#include +#include +#include +#include +#include + +#include + +namespace ml +{ +namespace maths +{ + +//! \brief Models the trend component of a time series. +//! +//! DESCRIPTION:\n +//! This is an ensemble of trend models fitted over different time scales. +//! In particular, we age data at different rates from each of the models. +//! A prediction is then a weighted average of the different models. We +//! adjust the weighting of components based on the difference in their +//! decay rate and the target decay rate. +//! +//! The key advantage of this approach is that we can also adjust the +//! weighting over a forecast based on how far ahead we are predicting. +//! This means at each time scale we can revert to the trend for that time +//! scale. It also allows us to accurately estimate confidence intervals +//! (since these can be estimated from the variation of observed values +//! we see w.r.t. the predictions from the next longer time scale component). +//! This produces plausible looking and this sort of mean reversion is common +//! in many real world time series. +class MATHS_EXPORT CTrendComponent +{ + public: + using TDoubleDoublePr = maths_t::TDoubleDoublePr; + using TDoubleDoublePrVec = std::vector; + using TDoubleVec = std::vector; + using TDoubleVecVec = std::vector; + using TDouble3Vec = core::CSmallVector; + using TDouble3VecVec = std::vector; + using TVector = CVectorNx1; + using TVectorVec = std::vector; + using TVectorVecVec = std::vector; + using TMatrix = CSymmetricMatrixNxN; + using TMatrixVec = std::vector; + + public: + CTrendComponent(double decayRate); + + //! Efficiently swap the state of this and \p other. + void swap(CTrendComponent &other); + + //! Persist state by passing information to \p inserter. + void acceptPersistInserter(core::CStatePersistInserter &inserter) const; + + //! Initialize by reading state from \p traverser. + bool acceptRestoreTraverser(core::CStateRestoreTraverser &traverser); + + //! Check if the trend has been estimated. + bool initialized() const; + + //! Clear all data. + void clear(); + + //! Shift the regression models' time origins to \p time. + void shiftOrigin(core_t::TTime time); + + //! Shift the slope of all regression models' whose decay rate is + //! greater than \p decayRate. + void shiftSlope(double decayRate, double shift); + + //! Adds a value \f$(t, f(t))\f$ to this component. + //! + //! \param[in] time The time of the point. + //! \param[in] value The value at \p time. + //! \param[in] weight The weight of \p value. The smaller this is the + //! less influence it has on the component. + void add(core_t::TTime time, double value, double weight = 1.0); + + //! Get the base rate at which models lose information. + double defaultDecayRate() const; + + //! Set the rate base rate at which models lose information. + void decayRate(double decayRate); + + //! Age the trend to account for \p interval elapsed time. + void propagateForwardsByTime(core_t::TTime interval); + + //! Get the predicted value at \p time. + //! + //! \param[in] time The time of interest. + //! \param[in] confidence The symmetric confidence interval for the variance + //! as a percentage. + TDoubleDoublePr value(core_t::TTime time, double confidence) const; + + //! Get the variance of the residual about the predicted value at \p time. + //! + //! \param[in] confidence The symmetric confidence interval for the + //! variance as a percentage. + TDoubleDoublePr variance(double confidence) const; + + //! Create \p n sample forecast paths. + void forecast(core_t::TTime startTime, + core_t::TTime endTime, + core_t::TTime step, + double confidence, + TDouble3VecVec &result) const; + + //! Get the interval which has been observed so far. + core_t::TTime observedInterval() const; + + //! Get the number of parameters used to describe the trend. + double parameters() const; + + //! Get a checksum for this object. + uint64_t checksum(uint64_t seed = 0) const; + + //! Get a debug description of this object. + std::string print() const; + + private: + using TRegression = CRegression::CLeastSquaresOnline<2, double>; + using TRegressionArray = TRegression::TArray; + using TRegressionArrayVec = std::vector; + using TMeanAccumulator = CBasicStatistics::SSampleMean::TAccumulator; + using TMeanAccumulatorVec = std::vector; + using TMeanVarAccumulator = CBasicStatistics::SSampleMeanVar::TAccumulator; + + //! \brief A model of the trend at a specific time scale. + struct SModel + { + explicit SModel(double weight); + void acceptPersistInserter(core::CStatePersistInserter &inserter) const; + bool acceptRestoreTraverser(core::CStateRestoreTraverser &traverser); + uint64_t checksum(uint64_t seed) const; + TMeanAccumulator s_Weight; + TRegression s_Regression; + TMeanVarAccumulator s_ResidualMoments; + }; + using TModelVec = std::vector; + + private: + //! Get the factors by which to age the different regression models. + TDoubleVec factors(core_t::TTime interval) const; + + //! Get the initial weights to use for forecast predictions. + TDoubleVec initialForecastModelWeights() const; + + //! Get the initial weights to use for forecast prediction errors. + TDoubleVec initialForecastErrorWeights() const; + + //! Get the mean count of samples in the prediction. + double count() const; + + //! Get the predicted value at \p time. + double value(const TDoubleVec &weights, + const TRegressionArrayVec &models, + double time) const; + + //! Get the weight to assign to the prediction verses the long term mean. + double weightOfPrediction(core_t::TTime time) const; + + private: + //! The default rate at which information is aged out of the trend models. + double m_DefaultDecayRate; + + //! The target rate at which information is aged out of the ensemble. + double m_TargetDecayRate; + + //! The time the model was first updated. + core_t::TTime m_FirstUpdate; + //! The time the model was last updated. + core_t::TTime m_LastUpdate; + + //! The start time of the regression models. + core_t::TTime m_RegressionOrigin; + //! The regression models (we have them for multiple time scales). + TModelVec m_Models; + //! The variance of the prediction errors. + double m_PredictionErrorVariance; + //! The mean and variance of the values added to the trend component. + TMeanVarAccumulator m_ValueMoments; +}; + +} +} + +#endif // INCLUDED_ml_maths_CTrendComponent_h diff --git a/include/maths/CTrendTests.h b/include/maths/CTrendTests.h index 0e2f525000..d1d0a1c5ce 100644 --- a/include/maths/CTrendTests.h +++ b/include/maths/CTrendTests.h @@ -48,101 +48,6 @@ namespace maths { class CSeasonalTime; -//! \brief Implements a simple test for whether a random -//! process has a trend. -//! -//! DESCRIPTION:\n -//! A process with a trend is defined as one which can be -//! modeled as follows: -//! 
-//!   \f$Y_i = f(t_i) + X_i\f$
-//!
-//! Here, \f$f(.)\f$ is some smoothly varying function and -//! the \f$X_i = X\f$ are IID. The approach we take is to -//! perform a variance ratio test on \f${ Y_i - f(t_i) }\f$ -//! verses \f${ Y_i }\f$. We are interested in the case that -//! modeling f(.), using an exponentially decaying cubic -//! regression with the current decay rate, will materially -//! affect our results. We therefore test to see if the -//! reduction in variance, as a proxy for the full model -//! confidence bounds, is both large enough and statistically -//! significant. -class MATHS_EXPORT CTrendTest -{ - public: - //! The order of the trend regression. - static const std::size_t ORDER = 3u; - - public: - using TRegression = CRegression::CLeastSquaresOnline; - - public: - explicit CTrendTest(double decayRate = 0.0); - - //! Initialize by reading state from \p traverser. - bool acceptRestoreTraverser(core::CStateRestoreTraverser &traverser); - - //! Persist state by passing information to \p inserter. - void acceptPersistInserter(core::CStatePersistInserter &inserter) const; - - //! Set the decay rate. - void decayRate(double decayRate); - - //! Age the state to account for \p time elapsed time. - void propagateForwardsByTime(double time); - - //! Add a new value \p value at \p time. - void add(core_t::TTime time, double value, double weight = 1.0); - - //! Capture the variance in the prediction error. - void captureVariance(core_t::TTime time, double value, double weight = 1.0); - - //! Translate the trend by \p shift. - void shift(double shift); - - //! Test whether there is a trend. - bool test(void) const; - - //! Get the regression model of the trend. - const TRegression &trend(void) const; - - //! Get the origin of the time coordinate system. - core_t::TTime origin(void) const; - - //! Get the variance after removing the trend. - double variance(void) const; - - //! Get a checksum for this object. - uint64_t checksum(uint64_t seed = 0) const; - - private: - //! The smallest decrease in variance after removing the trend - //! which is consider significant. - static const double MAXIMUM_TREND_VARIANCE_RATIO; - - private: - using TVector = CVectorNx1; - using TVectorMeanVarAccumulator = CBasicStatistics::SSampleMeanVar::TAccumulator; - - private: - //! Get the time at which to evaluate the regression model - //! of the trend. - double time(core_t::TTime time) const; - - private: - //! The rate at which the regression model is aged. - double m_DecayRate; - - //! The origin of the time coordinate system. - core_t::TTime m_TimeOrigin; - - //! The current regression model. - TRegression m_Trend; - - //! The values' mean and variance. - TVectorMeanVarAccumulator m_Variances; -}; - //! \brief A low memory footprint randomized test for probability. //! //! This is based on the idea of random projection in a Hilbert @@ -266,497 +171,6 @@ class MATHS_EXPORT CRandomizedPeriodicityTest friend class ::CTrendTestsTest; }; -//! \brief Represents the result of running a periodicity test. -class MATHS_EXPORT CPeriodicityTestResult : boost::equality_comparable > -{ - public: - using TTimeTimePr = std::pair; - - public: - //! \brief Component data. - struct MATHS_EXPORT SComponent - { - SComponent(void); - SComponent(unsigned int id, - core_t::TTime startOfPartition, - core_t::TTime period, - const TTimeTimePr &window); - - //! Initialize from a string created by persist. - static bool fromString(const std::string &value, SComponent &result); - //! Convert to a string. - static std::string toString(const SComponent &component); - - //! Check if this is equal to \p other. - bool operator==(const SComponent &other) const; - - //! Get a checksum of this object. - uint64_t checksum(void) const; - - //! An identifier for the component used by the test. - unsigned int s_Id; - //! The start of the partition. - core_t::TTime s_StartOfPartition; - //! The period of the component. - core_t::TTime s_Period; - //! The component window. - TTimeTimePr s_Window; - }; - - using TComponent4Vec = core::CSmallVector; - - public: - //! Initialize by reading state from \p traverser. - bool acceptRestoreTraverser(core::CStateRestoreTraverser &traverser); - - //! Persist state by passing information to \p inserter. - void acceptPersistInserter(core::CStatePersistInserter &inserter) const; - - //! Check if this is equal to \p other. - bool operator==(const CPeriodicityTestResult &other) const; - - //! Sets to the union of the periodic components present. - //! - //! \warning This only makes sense if the this and the other result - //! share the start of the partition time. - const CPeriodicityTestResult &operator+=(const CPeriodicityTestResult &other); - - //! Add a component if and only if \p hasPeriod is true. - void add(unsigned int id, - core_t::TTime startOfWeek, - core_t::TTime period, - const TTimeTimePr &window); - - //! Check if there are any periodic components. - bool periodic(void) const; - - //! Get the binary representation of the periodic components. - const TComponent4Vec &components(void) const; - - //! Get a checksum for this object. - uint64_t checksum(void) const; - - private: - //! The periodic components. - TComponent4Vec m_Components; -}; - -//! \brief Implements shared functionality for our periodicity tests. -class MATHS_EXPORT CPeriodicityTest -{ - public: - using TDouble2Vec = core::CSmallVector; - using TTimeTimePr = std::pair; - using TTimeTimePr2Vec = core::CSmallVector; - using TMeanVarAccumulator = CBasicStatistics::SSampleMeanVar::TAccumulator; - using TMeanVarAccumulatorVec = std::vector; - using TTimeTimePrMeanVarAccumulatorPr = std::pair; - using TTimeTimePrMeanVarAccumulatorPrVec = std::vector; - using TTimeTimePrMeanVarAccumulatorPrVecVec = std::vector; - using TFloatMeanAccumulator = CBasicStatistics::SSampleMean::TAccumulator; - using TFloatMeanAccumulatorVec = std::vector; - using TTime2Vec = core::CSmallVector; - using TComponent = CPeriodicityTestResult::SComponent; - - public: - CPeriodicityTest(void); - explicit CPeriodicityTest(double decayRate); - virtual ~CPeriodicityTest(void) = default; - - //! Check if the test is initialized. - bool initialized(void) const; - - //! Age the bucket values to account for \p time elapsed time. - void propagateForwardsByTime(double time); - - //! Add \p value at \p time. - void add(core_t::TTime time, double value, double weight = 1.0); - - //! Get the periods being tested. - virtual TTime2Vec periods(void) const = 0; - - //! Get a seasonal time for the specified results. - //! - //! \warning The caller owns the returned object. - virtual CSeasonalTime *seasonalTime(const TComponent &component) const = 0; - - //! Get the periodic trends corresponding to \p periods. - virtual void trends(const CPeriodicityTestResult &periods, - TTimeTimePrMeanVarAccumulatorPrVecVec &result) const = 0; - - //! Get the fraction of populated test slots - double populatedRatio(void) const; - - //! Check we've seen sufficient data to test accurately. - bool seenSufficientData(void) const; - - //! Debug the memory used by this object. - virtual void debugMemoryUsage(core::CMemoryUsage::TMemoryUsagePtr mem) const; - - //! Get the memory used by this object. - virtual std::size_t memoryUsage(void) const; - - //! Get the static size of this object. - virtual std::size_t staticSize(void) const; - - //! Print a description of \p result. - virtual std::string print(const CPeriodicityTestResult &result) const = 0; - - protected: - using TDoubleVec = std::vector; - using TDoubleVec2Vec = core::CSmallVector; - - //! \brief A collection of statistics used during testing. - struct MATHS_EXPORT STestStats - { - STestStats(double vt, double at, double Rt); - //! The maximum variance to accept the alternative hypothesis. - double s_Vt; - //! The minimum amplitude to accept the alternative hypothesis. - double s_At; - //! The minimum autocorrelation to accept the alternative - //! hypothesis. - double s_Rt; - //! The data range. - double s_Range; - //! The number of buckets with at least one measurement. - double s_B; - //! The average number of measurements per bucket value. - double s_M; - //! The variance estimate of H0. - double s_V0; - //! The degrees of freedom in the variance estimate of H0. - double s_DF0; - //! The trend for the null hypothesis. - TDoubleVec2Vec s_T0; - //! The partition for the null hypothesis. - TTimeTimePr2Vec s_Partition; - //! The start of the repeating partition. - core_t::TTime s_StartOfPartition; - }; - - protected: - //! The minimum proportion of populated buckets for which - //! the test is accurate. - static const double ACCURATE_TEST_POPULATED_FRACTION; - - protected: - //! Get the bucket length. - core_t::TTime bucketLength(void) const; - - //! Get the window length. - core_t::TTime windowLength(void) const; - - //! Get the bucket values. - const TFloatMeanAccumulatorVec &bucketValues(void) const; - - //! Initialize by reading state from \p traverser. - bool acceptRestoreTraverser(core::CStateRestoreTraverser &traverser); - - //! Persist state by passing information to \p inserter. - void acceptPersistInserter(core::CStatePersistInserter &inserter) const; - - //! Get a checksum for this object. - uint64_t checksum(uint64_t seed) const; - - //! Initialize the bucket values. - void initialize(core_t::TTime bucketLength, - core_t::TTime window, - const TFloatMeanAccumulatorVec &initial); - - //! Compute various ancillary statistics for testing. - bool initializeTestStatistics(STestStats &stats) const; - - //! Get the variance and degrees freedom for the null hypothesis - //! that there is no trend or repeating partition of any kind. - bool nullHypothesis(STestStats &stats) const; - - //! Test to see if there is significant evidence for a component - //! with period \p period. - bool testPeriod(const TTimeTimePr2Vec &window, - core_t::TTime period, STestStats &stats) const; - - //! Test to see if there is significant evidence for a repeating - //! partition of the data into windows defined by \p partition. - bool testPartition(const TTimeTimePr2Vec &partition, - core_t::TTime period, - double correction, STestStats &stats) const; - - //! Condition \p values assuming the null hypothesis is true. - //! - //! This removes any trend associated with the null hypothesis. - void conditionOnNullHypothesis(const TTimeTimePr2Vec &window, - const STestStats &stats, - TFloatMeanAccumulatorVec &values) const; - - //! Get the trend for period \p period. - void periodicBucketing(core_t::TTime period, - const TTimeTimePr2Vec &windows, - TTimeTimePrMeanVarAccumulatorPrVec &trend) const; - - //! Get the decomposition into a short and long period trend. - void periodicBucketing(TTime2Vec periods, - const TTimeTimePr2Vec &windows, - TTimeTimePrMeanVarAccumulatorPrVec &shortTrend, - TTimeTimePrMeanVarAccumulatorPrVec &longTrend) const; - - //! Initialize the buckets in \p trend. - void initializeBuckets(std::size_t period, - const TTimeTimePr2Vec &windows, - TTimeTimePrMeanVarAccumulatorPrVec &trend) const; - - private: - //! The minimum coefficient of variation to bother to test. - static const double MINIMUM_COEFFICIENT_OF_VARIATION; - - private: - //! The rate at which the bucket values are aged. - double m_DecayRate; - - //! The bucketing interval. - core_t::TTime m_BucketLength; - - //! The window length for which to maintain bucket values. - core_t::TTime m_WindowLength; - - //! The mean bucket values. - TFloatMeanAccumulatorVec m_BucketValues; -}; - -//! \brief Implements test for diurnal periodic components. -//! -//! DESCRIPTION:\n -//! This tests whether there are daily and/or weekly components -//! in a time series. It also checks to see if there is a partition -//! of the time series into disjoint weekend and weekday intervals. -//! Tests include various forms of analysis of variance and a test -//! of the amplitude. -class MATHS_EXPORT CDiurnalPeriodicityTest : public CPeriodicityTest -{ - public: - explicit CDiurnalPeriodicityTest(double decayRate = 0.0); - - //! \name Persistence - //@{ - //! Initialize by reading state from \p traverser. - bool acceptRestoreTraverser(core::CStateRestoreTraverser &traverser); - - //! Persist state by passing information to \p inserter. - void acceptPersistInserter(core::CStatePersistInserter &inserter) const; - //@} - - //! Get a test for daily and weekly periodic components. - //! - //! \warning The caller owns the returned object. - static CDiurnalPeriodicityTest *create(core_t::TTime bucketLength, - double decayRate = 0.0); - - //! Check if there periodic components. - CPeriodicityTestResult test(void) const; - - //! Get the periods being tested. - virtual TTime2Vec periods(void) const; - - //! Get a seasonal time for the specified results. - //! - //! \warning The caller owns the returned object. - virtual CSeasonalTime *seasonalTime(const TComponent &component) const; - - //! Get the periodic trends corresponding to \p required. - virtual void trends(const CPeriodicityTestResult &required, - TTimeTimePrMeanVarAccumulatorPrVecVec &result) const; - - //! Get the static size of this object. - virtual std::size_t staticSize(void) const; - - //! Get a checksum for this object. - uint64_t checksum(uint64_t seed = 0) const; - - //! Print a description of \p result. - virtual std::string print(const CPeriodicityTestResult &result) const; - - private: - //! Initialize the bucket values. - bool initialize(core_t::TTime bucketLength, - core_t::TTime window, - const double (&corrections)[2]); - - //! Test for weekly components given a daily periodic component - //! and weekday/end partition. - CPeriodicityTestResult testWeeklyGivenDailyAndWeekend(STestStats &stats) const; - - //! Test for a daily periodic component. - bool testDaily(STestStats &stats) const; - //! Test for a weekday/end partition. - bool testWeekend(bool daily, STestStats &stats) const; - //! Test for a weekly periodic component. - bool testWeekly(const TTimeTimePr2Vec &window, STestStats &stats) const; - - private: - //! The minimum proportion of explained variance including the - //! partition into weekdays and weekends for the test to pass. - static const double MAXIMUM_PARTITION_VARIANCE; - //! The minimum proportion of explained variance including the - //! periodic component for the test to pass. - static const double MAXIMUM_PERIOD_VARIANCE; - //! The threshold for the amplitude of the periodic component, - //! as a multiple of the residual standard deviation, used to - //! test for the presence of periodic spikes. - static const double MINIMUM_PERIOD_AMPLITUDE; - //! The minimum permitted autocorrelation for the test to pass. - static const double MINIMUM_AUTOCORRELATION; - - private: - //! The scales to apply when to the weekend variance when using - //! {simple partition, partition and daily periodicity}. - double m_VarianceCorrections[2]; -}; - -//! \brief Tests to see whether there is a specified periodic component. -class MATHS_EXPORT CGeneralPeriodicityTest : public CPeriodicityTest -{ - public: - //! An empty collection of bucket values. - static const TFloatMeanAccumulatorVec NO_BUCKET_VALUES; - - public: - //! Initialize the bucket values. - bool initialize(core_t::TTime bucketLength, - core_t::TTime window, - core_t::TTime period, - const TFloatMeanAccumulatorVec &initial = NO_BUCKET_VALUES); - - //! Check if there periodic components. - CPeriodicityTestResult test(void) const; - - //! Get the periods being tested. - virtual TTime2Vec periods(void) const; - - //! Get a seasonal time for the specified results. - //! - //! \warning The caller owns the returned object. - CSeasonalTime *seasonalTime(const TComponent &component) const; - - //! Get the periodic trend corresponding to \p required. - virtual void trends(const CPeriodicityTestResult &required, - TTimeTimePrMeanVarAccumulatorPrVecVec &result) const; - - //! Get the static size of this object. - virtual std::size_t staticSize(void) const; - - //! Get a checksum for this object. - uint64_t checksum(uint64_t seed = 0) const; - - //! Print a description of \p result. - virtual std::string print(const CPeriodicityTestResult &result) const; - - private: - //! The minimum proportion of explained variance including the - //! periodic component for the test to pass. - static const double MAXIMUM_PERIOD_VARIANCE; - //! The threshold for the amplitude of the periodic component, - //! as a multiple of the residual standard deviation, used to - //! test for the presence of periodic spikes. - static const double MINIMUM_PERIOD_AMPLITUDE; - //! The minimum permitted autocorrelation for the test to pass. - static const double MINIMUM_AUTOCORRELATION; - - private: - //! The candidate period. - core_t::TTime m_Period; -}; - -//! \brief Implements a test that scans through a range of frequencies -//! looking for different periodic components in the data. -//! -//! DESCRIPTION:\n -//! This performs a scan for increasingly low frequency periodic -//! components maintaining a fixed size buffer. We find the most -//! promising candidate periods using linear autocorrelation and -//! then test them using our standard periodicity test. -//! -//! In order to maintain a fixed space the bucket length is increased -//! as soon as the observed data span exceeds the test size multiplied -//! by the current bucket span. -class MATHS_EXPORT CScanningPeriodicityTest -{ - public: - using TDoubleVec = std::vector; - using TTimeVec = std::vector; - using TTimeCRng = core::CVectorRange; - using TPeriodicityResultPr = std::pair; - - public: - CScanningPeriodicityTest(TTimeCRng bucketLengths, - std::size_t size, - double decayRate = 0.0); - - //! Initialize by reading state from \p traverser. - bool acceptRestoreTraverser(core::CStateRestoreTraverser &traverser); - - //! Persist state by passing information to \p inserter. - void acceptPersistInserter(core::CStatePersistInserter &inserter) const; - - //! Set the start time to \p time. - void initialize(core_t::TTime time); - - //! Age the bucket values to account for \p time elapsed time. - void propagateForwardsByTime(double time); - - //! Add \p value at \p time. - void add(core_t::TTime time, double value, double weight = 1.0); - - //! Check if we need to compress by increasing the bucket span. - bool needToCompress(core_t::TTime) const; - - //! Check if there periodic components. - TPeriodicityResultPr test(void) const; - - //! Roll time forwards by \p skipInterval. - void skipTime(core_t::TTime skipInterval); - - //! Get a checksum for this object. - uint64_t checksum(uint64_t seed = 0) const; - - //! Debug the memory used by this object. - void debugMemoryUsage(core::CMemoryUsage::TMemoryUsagePtr mem) const; - - //! Get the memory used by this object. - std::size_t memoryUsage(void) const; - - private: - using TFloatMeanAccumulator = CBasicStatistics::SSampleMean::TAccumulator; - using TFloatMeanAccumulatorVec = std::vector; - - private: - //! Resize the bucket values to \p size. - void resize(std::size_t size, TFloatMeanAccumulatorVec &values) const; - - //! Compute the mean of the autocorrelation at integer multiples - //! of \p period. - double meanForPeriodicOffsets(const TDoubleVec &correlations, std::size_t period) const; - - //! Correct the autocorrelation calculated on padded data. - double correctForPad(double correlation, std::size_t offset) const; - - private: - //! The rate at which the bucket values are aged. - double m_DecayRate; - - //! The bucket lengths to test. - TTimeCRng m_BucketLengths; - - //! The index in m_BucketLengths of the current bucketing interval. - std::size_t m_BucketLengthIndex; - - //! The time of the first data point. - core_t::TTime m_StartTime; - - //! The bucket values. - TFloatMeanAccumulatorVec m_BucketValues; -}; - //! \brief The basic idea of this test is to see if there is stronger //! than expected temporal correlation between large prediction errors //! and calendar features. diff --git a/include/maths/Constants.h b/include/maths/Constants.h index 06676f29c4..ad1a5b4d07 100644 --- a/include/maths/Constants.h +++ b/include/maths/Constants.h @@ -49,32 +49,62 @@ const double SMALL_PROBABILITY{1e-4}; //! similar score. const double MINUSCULE_PROBABILITY{1e-50}; +//! The margin between the smallest value and the support left end +//! to use for the gamma distribution. +const double GAMMA_OFFSET_MARGIN{0.1}; + +//! The margin between the smallest value and the support left end +//! to use for the log-normal distribution. +const double LOG_NORMAL_OFFSET_MARGIN{1.0}; + +//! The minimum amount by which a trend decomposition component can +//! reduce the prediction error variance and still be worthwhile +//! modeling. We have different thresholds because we have inductive +//! bias for particular types of components. +const double SIGNIFICANT_VARIANCE_REDUCTION[]{0.7, 0.5}; + +//! The minimum repeated amplitude of a seasonal component, as a +//! multiple of error standard deviation, to be worthwhile modeling. +//! We have different thresholds because we have inductive bias for +//! particular types of components. +const double SIGNIFICANT_AMPLITUDE[]{1.0, 2.0}; + +//! The minimum autocorrelation of a seasonal component to be +//! worthwhile modeling. We have different thresholds because we +//! have inductive bias for particular types of components. +const double SIGNIFICANT_AUTOCORRELATION[]{0.5, 0.7}; + +//! The maximum significance of a test statistic to choose to model +//! a trend decomposition component. +const double MAXIMUM_SIGNIFICANCE{0.001}; + //! The minimum variance scale for which the likelihood function -//! can be accurately adjusted. For smaller scales there errors -//! are introduced for some priors. +//! can be accurately adjusted. For smaller scales errors are +//! introduced for some priors. const double MINIMUM_ACCURATE_VARIANCE_SCALE{0.5}; //! The maximum variance scale for which the likelihood function -//! can be accurately adjusted. For larger scales there errors -//! are introduced for some priors. +//! can be accurately adjusted. For larger scales errors are +//! introduced for some priors. const double MAXIMUM_ACCURATE_VARIANCE_SCALE{2.0}; -//! The confidence interval to compute for the seasonal trend and +//! The confidence interval to use for the seasonal trend and //! variation. We detrend to the nearest point in the confidence //! interval and use the upper confidence interval variance when //! scaling the likelihood function so that we don't get transient -//! anomalies after detecting a periodic trend. +//! anomalies after detecting a periodic trend (when the trend +//! can be in significant error). const double DEFAULT_SEASONAL_CONFIDENCE_INTERVAL{50.0}; //! \brief A collection of weight styles and weights. class MATHS_EXPORT CConstantWeights { public: - typedef core::CSmallVector TDouble2Vec; - typedef core::CSmallVector TDouble4Vec; - typedef core::CSmallVector TDouble2Vec4Vec; - typedef core::CSmallVector TDouble4Vec1Vec; - typedef core::CSmallVector TDouble2Vec4Vec1Vec; + using TDouble2Vec = core::CSmallVector; + using TDouble4Vec = core::CSmallVector; + using TDouble2Vec4Vec = core::CSmallVector; + using TDouble4Vec1Vec = core::CSmallVector; + using TDouble2Vec4Vec1Vec = core::CSmallVector; public: //! A single count weight style. @@ -103,13 +133,13 @@ class MATHS_EXPORT CConstantWeights }; //! The minimum fractional count of points in a cluster. -const double MINIMUM_CLUSTER_SPLIT_FRACTION = 0.0; +const double MINIMUM_CLUSTER_SPLIT_FRACTION{0.0}; //! The default minimum count of points in a cluster. -const double MINIMUM_CLUSTER_SPLIT_COUNT = 24.0; +const double MINIMUM_CLUSTER_SPLIT_COUNT{24.0}; //! The minimum count of a category in the sketch to cluster. -const double MINIMUM_CATEGORY_COUNT = 0.5; +const double MINIMUM_CATEGORY_COUNT{0.5}; //! Get the maximum amount we'll penalize a model in addSamples. MATHS_EXPORT double maxModelPenalty(double numberSamples); diff --git a/include/model/CAnomalyDetectorModelConfig.h b/include/model/CAnomalyDetectorModelConfig.h index 2871db650d..dfbc173cc4 100644 --- a/include/model/CAnomalyDetectorModelConfig.h +++ b/include/model/CAnomalyDetectorModelConfig.h @@ -227,23 +227,6 @@ class MODEL_EXPORT CAnomalyDetectorModelConfig static const TDoubleDoublePr DEFAULT_NORMALIZED_SCORE_KNOT_POINTS[9]; //@} - //! For priors whose support isn't the whole real line we use an - //! offset, which is just a translation of the prior, to extend - //! the range over which it is valid. This is the default offset - //! for the log-normal distribution prior. - static const double DEFAULT_LOG_NORMAL_PRIOR_OFFSET; - //! For priors whose support isn't the whole real line we use an - //! offset, which is just a translation of the prior, to extend - //! the range over which it is valid. This is the default offset - //! for the gamma distribution prior. - static const double DEFAULT_GAMMA_PRIOR_OFFSET; - - //! For priors whose support isn't the whole real line we use an - //! offset, which is just a translation of the prior, to extend - //! the range over which it is valid. This is the default offset - //! for the Poisson distribution prior. - static const double DEFAULT_POISSON_PRIOR_OFFSET; - //! The maximum number of samples we use when re-sampling a prior. static const std::size_t DEFAULT_RESAMPLING_MAX_SAMPLES; diff --git a/include/model/CModelParams.h b/include/model/CModelParams.h index 62b7f53342..5d04a73bf8 100644 --- a/include/model/CModelParams.h +++ b/include/model/CModelParams.h @@ -80,15 +80,6 @@ struct MODEL_EXPORT SModelParams //! feature. std::string s_MultivariateComponentDelimiter; - //! The log-normal prior offset. - double s_LogNormalOffset; - - //! The gamma prior offset. - double s_GammaOffset; - - //! The Poisson prior offset. - double s_PoissonOffset; - //! The rate at which the model learns per bucket. double s_LearnRate; diff --git a/lib/maths/CAdaptiveBucketing.cc b/lib/maths/CAdaptiveBucketing.cc index 974b362fa1..943d0036c0 100644 --- a/lib/maths/CAdaptiveBucketing.cc +++ b/lib/maths/CAdaptiveBucketing.cc @@ -16,22 +16,18 @@ #include #include -#include #include #include #include #include -#include #include #include #include -#include -#include -#include #include #include +#include #include #include @@ -155,53 +151,37 @@ bool CAdaptiveBucketing::initialize(double a, double b, std::size_t n) return true; } -void CAdaptiveBucketing::initialValues(core_t::TTime time, const TTimeTimePrMeanVarPrVec &values) +void CAdaptiveBucketing::initialValues(core_t::TTime start, + core_t::TTime end, + const TFloatMeanAccumulatorVec &values) { if (!this->initialized()) { return; } - for (std::size_t i = 0u; i < values.size(); ++i) - { - double ai{this->offset(values[i].first.first)}; - double bi{this->offset(values[i].first.second - 1) + 1.0}; - const TDoubleMeanVarAccumulator &vi{values[i].second}; + core_t::TTime size{static_cast(values.size())}; + core_t::TTime dT{(end - start) / size}; + core_t::TTime dt{static_cast( + CTools::truncate(m_MinimumBucketLength, 1.0, static_cast(dT)))}; - std::size_t n{m_Endpoints.size()}; - if (ai < m_Endpoints[0] || bi > m_Endpoints[n-1]) - { - LOG_ERROR("t = [" << ai << "," << bi << "]" - << " out of range [" << m_Endpoints[0] - << "," << m_Endpoints[n-1] << ")"); - return; - } + double scale{std::pow(static_cast(dt) / static_cast(dT), 2.0)}; - std::size_t ka(std::upper_bound(m_Endpoints.begin(), - m_Endpoints.end(), ai) - m_Endpoints.begin()); - std::size_t kb(std::lower_bound(m_Endpoints.begin(), - m_Endpoints.end(), bi) - m_Endpoints.begin()); - - double length{bi - ai}; - for (std::size_t k = ka; k <= kb; ++k) + for (core_t::TTime time = start + dt/2; time < end; time += dt) + { + if (this->inWindow(time)) { - double xl{m_Endpoints[k-1]}; - double xr{m_Endpoints[k]}; - double ak{std::max(ai, xl)}; - double bk{std::min(bi, xr)}; - double w{(bk - ak) / length}; - if (w > 0.0) + core_t::TTime i{(time - start) / dT}; + double value{CBasicStatistics::mean(values[i])}; + double weight{scale * CBasicStatistics::count(values[i])}; + if (weight > 0.0) { - TDoubleMeanVarAccumulator vk{vi}; - vk.age(w * w); - double nk = CBasicStatistics::count(vk); - - auto centre = CBasicStatistics::accumulator(this->count(k-1), - static_cast(m_Centres[k-1])) - + CBasicStatistics::accumulator(nk, (ak + bk) / 2.0); - - m_Centres[k-1] = CBasicStatistics::mean(centre); - this->add(k-1, time, xl, vk); + std::size_t bucket; + if (this->bucket(time, bucket)) + { + this->add(bucket, time, weight); + this->add(bucket, time, value, weight); + } } } } diff --git a/lib/maths/CCalendarComponentAdaptiveBucketing.cc b/lib/maths/CCalendarComponentAdaptiveBucketing.cc index fbc30f917a..6c441e8422 100644 --- a/lib/maths/CCalendarComponentAdaptiveBucketing.cc +++ b/lib/maths/CCalendarComponentAdaptiveBucketing.cc @@ -289,6 +289,7 @@ void CCalendarComponentAdaptiveBucketing::refresh(const TFloatVec &endpoints) // This might be worth considering at some point. using TDoubleMeanAccumulator = CBasicStatistics::SSampleMean::TAccumulator; + using TDoubleMeanVarAccumulator = CBasicStatistics::SSampleMeanVar::TAccumulator; std::size_t m{m_Values.size()}; std::size_t n{endpoints.size()}; @@ -390,12 +391,14 @@ void CCalendarComponentAdaptiveBucketing::refresh(const TFloatVec &endpoints) m_Centres.swap(centres); } -void CCalendarComponentAdaptiveBucketing::add(std::size_t bucket, - core_t::TTime /*time*/, - double /*offset*/, - const TDoubleMeanVarAccumulator &value) +bool CCalendarComponentAdaptiveBucketing::inWindow(core_t::TTime time) const { - m_Values[bucket] += value; + return m_Feature.inWindow(time); +} + +void CCalendarComponentAdaptiveBucketing::add(std::size_t bucket, core_t::TTime /*time*/, double value, double weight) +{ + m_Values[bucket].add(value, weight); } double CCalendarComponentAdaptiveBucketing::offset(core_t::TTime time) const diff --git a/lib/maths/CExpandingWindow.cc b/lib/maths/CExpandingWindow.cc new file mode 100644 index 0000000000..6e21a32f3d --- /dev/null +++ b/lib/maths/CExpandingWindow.cc @@ -0,0 +1,200 @@ +/* + * ELASTICSEARCH CONFIDENTIAL + * + * Copyright (c) 2018 Elasticsearch BV. All Rights Reserved. + * + * Notice: this software, and all information contained + * therein, is the exclusive property of Elasticsearch BV + * and its licensors, if any, and is protected under applicable + * domestic and foreign law, and international treaties. + * + * Reproduction, republication or distribution without the + * express written consent of Elasticsearch BV is + * strictly prohibited. + */ + +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +namespace ml +{ +namespace maths +{ +namespace +{ +const std::string BUCKET_LENGTH_INDEX_TAG("a"); +const std::string BUCKET_VALUES_TAG("b"); +const std::string START_TIME_TAG("c"); +} + +CExpandingWindow::CExpandingWindow(core_t::TTime bucketLength, + TTimeCRng bucketLengths, + std::size_t size, + double decayRate) : + m_DecayRate(decayRate), + m_BucketLength(bucketLength), + m_BucketLengths(bucketLengths), + m_BucketLengthIndex(0), + m_StartTime(boost::numeric::bounds::lowest()), + m_BucketValues(size % 2 == 0 ? size : size + 1) +{} + +bool CExpandingWindow::acceptRestoreTraverser(core::CStateRestoreTraverser &traverser) +{ + m_BucketValues.clear(); + do + { + const std::string &name = traverser.name(); + RESTORE_BUILT_IN(BUCKET_LENGTH_INDEX_TAG, m_BucketLengthIndex) + RESTORE_BUILT_IN(START_TIME_TAG, m_StartTime) + RESTORE(BUCKET_VALUES_TAG, core::CPersistUtils::restore(BUCKET_VALUES_TAG, m_BucketValues, traverser)); + } + while (traverser.next()); + return true; +} + +void CExpandingWindow::acceptPersistInserter(core::CStatePersistInserter &inserter) const +{ + inserter.insertValue(BUCKET_LENGTH_INDEX_TAG, m_BucketLengthIndex); + inserter.insertValue(START_TIME_TAG, m_StartTime); + core::CPersistUtils::persist(BUCKET_VALUES_TAG, m_BucketValues, inserter); +} + +core_t::TTime CExpandingWindow::startTime() const +{ + return m_StartTime; +} + +core_t::TTime CExpandingWindow::endTime() const +{ + return m_StartTime + ( static_cast(m_BucketValues.size()) + * m_BucketLengths[m_BucketLengthIndex]); +} + +core_t::TTime CExpandingWindow::bucketLength() const +{ + return m_BucketLengths[m_BucketLengthIndex]; +} + +const CExpandingWindow::TFloatMeanAccumulatorVec &CExpandingWindow::values() const +{ + return m_BucketValues; +} + +CExpandingWindow::TFloatMeanAccumulatorVec CExpandingWindow::valuesMinusPrediction(const TPredictor &predictor) const +{ + core_t::TTime start{CIntegerTools::floor(this->startTime(), m_BucketLength)}; + core_t::TTime end{CIntegerTools::ceil(this->endTime(), m_BucketLength)}; + core_t::TTime size{static_cast(m_BucketValues.size())}; + core_t::TTime offset{static_cast(CBasicStatistics::mean(m_MeanOffset) + 0.5)}; + + TFloatMeanAccumulatorVec predictions(size); + for (core_t::TTime time = start + offset; time < end; time += m_BucketLength) + { + core_t::TTime bucket{(time - start) / m_BucketLengths[m_BucketLengthIndex]}; + if (bucket >= 0 && bucket < size) + { + predictions[bucket].add(predictor(time)); + } + } + + TFloatMeanAccumulatorVec result(m_BucketValues); + for (core_t::TTime i = 0; i < size; ++i) + { + if (CBasicStatistics::count(result[i]) > 0.0) + { + CBasicStatistics::moment<0>(result[i]) -= CBasicStatistics::mean(predictions[i]); + } + } + + return result; +} + +void CExpandingWindow::initialize(core_t::TTime time) +{ + m_StartTime = CIntegerTools::floor(time, m_BucketLengths[0]); +} + +void CExpandingWindow::propagateForwardsByTime(double time) +{ + if (!CMathsFuncs::isFinite(time) || time < 0.0) + { + LOG_ERROR("Bad propagation time " << time); + } + double factor = std::exp(-m_DecayRate * time); + for (auto &value : m_BucketValues) + { + value.age(factor); + } +} + +void CExpandingWindow::add(core_t::TTime time, double value, double weight) +{ + if (time >= m_StartTime) + { + while (this->needToCompress(time)) + { + m_BucketLengthIndex = (m_BucketLengthIndex + 1) % m_BucketLengths.size(); + auto end = m_BucketValues.begin(); + + if (m_BucketLengthIndex == 0) + { + m_StartTime = CIntegerTools::floor(time, m_BucketLengths[0]); + } + else + { + std::size_t compression = m_BucketLengths[m_BucketLengthIndex] + / m_BucketLengths[m_BucketLengthIndex - 1]; + for (std::size_t i = 0u; i < m_BucketValues.size(); i += compression, ++end) + { + std::swap(*end, m_BucketValues[i]); + for (std::size_t j = 1u; j < compression && i + j < m_BucketValues.size(); ++j) + { + *end += m_BucketValues[i + j]; + } + } + } + std::fill(end, m_BucketValues.end(), TFloatMeanAccumulator()); + } + + m_BucketValues[(time - m_StartTime) / m_BucketLengths[m_BucketLengthIndex]].add(value, weight); + m_MeanOffset.add(static_cast(time % m_BucketLength)); + } +} + +bool CExpandingWindow::needToCompress(core_t::TTime time) const +{ + return time >= this->endTime(); +} + +uint64_t CExpandingWindow::checksum(uint64_t seed) const +{ + seed = CChecksum::calculate(seed, m_BucketLengthIndex); + seed = CChecksum::calculate(seed, m_StartTime); + return CChecksum::calculate(seed, m_BucketValues); +} + +void CExpandingWindow::debugMemoryUsage(core::CMemoryUsage::TMemoryUsagePtr mem) const +{ + mem->setName("CScanningPeriodicityTest"); + core::CMemoryDebug::dynamicSize("m_BucketValues", m_BucketValues, mem); +} + +std::size_t CExpandingWindow::memoryUsage() const +{ + return core::CMemory::dynamicSize(m_BucketValues); +} + +} +} diff --git a/lib/maths/CGammaRateConjugate.cc b/lib/maths/CGammaRateConjugate.cc index 3950430da1..11859c9bf3 100644 --- a/lib/maths/CGammaRateConjugate.cc +++ b/lib/maths/CGammaRateConjugate.cc @@ -881,19 +881,22 @@ CGammaRateConjugate::CGammaRateConjugate(maths_t::EDataType dataType, double offset, double shape, double rate, - double decayRate) : + double decayRate, + double offsetMargin) : CPrior(dataType, decayRate), m_Offset(offset), + m_OffsetMargin(offsetMargin), m_LikelihoodShape(1.0), m_PriorShape(shape), m_PriorRate(rate) -{ -} +{} CGammaRateConjugate::CGammaRateConjugate(const SDistributionRestoreParams ¶ms, - core::CStateRestoreTraverser &traverser) : + core::CStateRestoreTraverser &traverser, + double offsetMargin) : CPrior(params.s_DataType, 0.0), m_Offset(0.0), + m_OffsetMargin(offsetMargin), m_LikelihoodShape(1.0), m_PriorShape(0.0), m_PriorRate(0.0) @@ -928,12 +931,12 @@ bool CGammaRateConjugate::acceptRestoreTraverser(core::CStateRestoreTraverser &t CGammaRateConjugate CGammaRateConjugate::nonInformativePrior(maths_t::EDataType dataType, double offset, - double decayRate) + double decayRate, + double offsetMargin) { - return CGammaRateConjugate(dataType, offset, - NON_INFORMATIVE_SHAPE, - NON_INFORMATIVE_RATE, - decayRate); + return CGammaRateConjugate(dataType, offset + offsetMargin, + NON_INFORMATIVE_SHAPE, NON_INFORMATIVE_RATE, + decayRate, offsetMargin); } CGammaRateConjugate::EPrior CGammaRateConjugate::type(void) const @@ -949,7 +952,14 @@ CGammaRateConjugate *CGammaRateConjugate::clone(void) const void CGammaRateConjugate::setToNonInformative(double offset, double decayRate) { - *this = nonInformativePrior(this->dataType(), offset, decayRate); + *this = nonInformativePrior(this->dataType(), + offset + this->offsetMargin(), + decayRate, this->offsetMargin()); +} + +double CGammaRateConjugate::offsetMargin(void) const +{ + return m_OffsetMargin; } bool CGammaRateConjugate::needsOffset(void) const diff --git a/lib/maths/CLogNormalMeanPrecConjugate.cc b/lib/maths/CLogNormalMeanPrecConjugate.cc index d9b8f6ee21..a6f846e023 100644 --- a/lib/maths/CLogNormalMeanPrecConjugate.cc +++ b/lib/maths/CLogNormalMeanPrecConjugate.cc @@ -751,9 +751,11 @@ CLogNormalMeanPrecConjugate::CLogNormalMeanPrecConjugate(maths_t::EDataType data double gaussianPrecision, double gammaShape, double gammaRate, - double decayRate/*= 0.0*/) : + double decayRate, + double offsetMargin) : CPrior(dataType, decayRate), m_Offset(offset), + m_OffsetMargin(offsetMargin), m_GaussianMean(gaussianMean), m_GaussianPrecision(gaussianPrecision), m_GammaShape(gammaShape), @@ -761,9 +763,11 @@ CLogNormalMeanPrecConjugate::CLogNormalMeanPrecConjugate(maths_t::EDataType data {} CLogNormalMeanPrecConjugate::CLogNormalMeanPrecConjugate(const SDistributionRestoreParams ¶ms, - core::CStateRestoreTraverser &traverser) : + core::CStateRestoreTraverser &traverser, + double offsetMargin) : CPrior(params.s_DataType, params.s_DecayRate), m_Offset(0.0), + m_OffsetMargin(offsetMargin), m_GaussianMean(0.0), m_GaussianPrecision(0.0), m_GammaShape(0.0), @@ -798,14 +802,13 @@ bool CLogNormalMeanPrecConjugate::acceptRestoreTraverser(core::CStateRestoreTrav CLogNormalMeanPrecConjugate CLogNormalMeanPrecConjugate::nonInformativePrior(maths_t::EDataType dataType, double offset, - double decayRate) + double decayRate, + double offsetMargin) { - return CLogNormalMeanPrecConjugate(dataType, offset, - NON_INFORMATIVE_MEAN, - NON_INFORMATIVE_PRECISION, - NON_INFORMATIVE_SHAPE, - NON_INFORMATIVE_RATE, - decayRate); + return CLogNormalMeanPrecConjugate(dataType, offset + offsetMargin, + NON_INFORMATIVE_MEAN, NON_INFORMATIVE_PRECISION, + NON_INFORMATIVE_SHAPE, NON_INFORMATIVE_RATE, + decayRate, offsetMargin); } CLogNormalMeanPrecConjugate::EPrior CLogNormalMeanPrecConjugate::type(void) const @@ -821,7 +824,14 @@ CLogNormalMeanPrecConjugate *CLogNormalMeanPrecConjugate::clone(void) const void CLogNormalMeanPrecConjugate::setToNonInformative(double offset, double decayRate) { - *this = nonInformativePrior(this->dataType(), offset, decayRate); + *this = nonInformativePrior(this->dataType(), + offset + this->offsetMargin(), + decayRate, this->offsetMargin()); +} + +double CLogNormalMeanPrecConjugate::offsetMargin(void) const +{ + return m_OffsetMargin; } bool CLogNormalMeanPrecConjugate::needsOffset(void) const diff --git a/lib/maths/CModel.cc b/lib/maths/CModel.cc index 07ced4ae1a..0b8cc3a871 100644 --- a/lib/maths/CModel.cc +++ b/lib/maths/CModel.cc @@ -478,9 +478,9 @@ CModelStub::TDouble2Vec CModelStub::predict(core_t::TTime /*time*/, } CModelStub::TDouble2Vec3Vec CModelStub::confidenceInterval(core_t::TTime /*time*/, + double /*confidenceInterval*/, const maths_t::TWeightStyleVec &/*weightStyles*/, - const TDouble2Vec4Vec &/*weights*/, - double /*confidence*/) const + const TDouble2Vec4Vec &/*weights*/) const { return TDouble2Vec3Vec(); } diff --git a/lib/maths/CMultimodalPrior.cc b/lib/maths/CMultimodalPrior.cc index 807ade88e9..22c0434aec 100644 --- a/lib/maths/CMultimodalPrior.cc +++ b/lib/maths/CMultimodalPrior.cc @@ -462,10 +462,7 @@ void CMultimodalPrior::propagateForwardsByTime(double time) // where w(i) is its weight we can achieve this by multiplying // all weights by some factor f in the range [0, 1]. - if (!this->isForForecasting()) - { - m_Clusterer->propagateForwardsByTime(time); - } + m_Clusterer->propagateForwardsByTime(time); for (const auto &mode : m_Modes) { mode.s_Prior->propagateForwardsByTime(time); diff --git a/lib/maths/COneOfNPrior.cc b/lib/maths/COneOfNPrior.cc index 54800a6388..11e5c0da1e 100644 --- a/lib/maths/COneOfNPrior.cc +++ b/lib/maths/COneOfNPrior.cc @@ -499,10 +499,7 @@ void COneOfNPrior::propagateForwardsByTime(double time) for (auto &&model : m_Models) { - if (!this->isForForecasting()) - { - model.first.age(alpha); - } + model.first.age(alpha); model.second->propagateForwardsByTime(time); } diff --git a/lib/maths/CPeriodicityHypothesisTests.cc b/lib/maths/CPeriodicityHypothesisTests.cc new file mode 100644 index 0000000000..aa50b945aa --- /dev/null +++ b/lib/maths/CPeriodicityHypothesisTests.cc @@ -0,0 +1,1949 @@ +/* + * ELASTICSEARCH CONFIDENTIAL + * + * Copyright (c) 2018 Elasticsearch BV. All Rights Reserved. + * + * Notice: this software, and all information contained + * therein, is the exclusive property of Elasticsearch BV + * and its licensors, if any, and is protected under applicable + * domestic and foreign law, and international treaties. + * + * Reproduction, republication or distribution without the + * express written consent of Elasticsearch BV is + * strictly prohibited. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace ml +{ +namespace maths +{ +namespace +{ + +using TDoubleVec = std::vector; +using TTimeVec = std::vector; +using TSizeSizePr = std::pair; +using TSizeSizePr2Vec = core::CSmallVector; +using TMeanAccumulator = CBasicStatistics::SSampleMean::TAccumulator; +using TMeanAccumulatorVec = std::vector; +using TFloatMeanAccumulator = CBasicStatistics::SSampleMean::TAccumulator; +using TFloatMeanAccumulatorVec = std::vector; +using TMeanVarAccumulator = CBasicStatistics::SSampleMeanVar::TAccumulator; +using TMeanVarAccumulatorVec = std::vector; +using TTimeTimePr = std::pair; +using TTimeTimePr2Vec = core::CSmallVector; +using TTimeTimePrMeanVarAccumulatorPr = std::pair; + +//! \brief Accumulates the minimum amplitude. +class CMinAmplitude +{ + public: + CMinAmplitude(std::size_t n, double level) : + m_Level(level), + m_Count(0), + m_Min(std::max(n, MINIMUM_COUNT_TO_TEST)), + m_Max(std::max(n, MINIMUM_COUNT_TO_TEST)) + {} + + void add(double x, double n) + { + if (n > 0.0) + { + ++m_Count; + m_Min.add(x - m_Level); + m_Max.add(x - m_Level); + } + } + + double amplitude() const + { + if (this->count() >= MINIMUM_COUNT_TO_TEST) + { + return std::max(std::max(-m_Min.biggest(), 0.0), + std::max( m_Max.biggest(), 0.0)); + } + return 0.0; + } + + double significance(const boost::math::normal &normal) const + { + if (this->count() < MINIMUM_COUNT_TO_TEST) + { + return 1.0; + } + + double F{2.0 * CTools::safeCdf(normal, -this->amplitude())}; + if (F == 0.0) + { + return 0.0; + } + + double n{static_cast(this->count())}; + boost::math::binomial binomial(static_cast(m_Count), F); + return CTools::safeCdfComplement(binomial, n - 1.0); + } + + private: + using TMinAccumulator = CBasicStatistics::COrderStatisticsHeap; + using TMaxAccumulator = CBasicStatistics::COrderStatisticsHeap>; + + private: + std::size_t count() const { return m_Min.count(); } + + private: + //! The minimum number of repeats for which we'll test. + static const std::size_t MINIMUM_COUNT_TO_TEST; + + private: + //! The mean of the trend. + double m_Level; + //! The total count of values added. + std::size_t m_Count; + //! The smallest values. + TMinAccumulator m_Min; + //! The largest values. + TMaxAccumulator m_Max; +}; + +const std::size_t CMinAmplitude::MINIMUM_COUNT_TO_TEST{4}; + +using TMinAmplitudeVec = std::vector; + +//! \brief Holds the relevant summary for choosing between alternative +//! (non-nested) hypotheses. +struct SHypothesisSummary +{ + SHypothesisSummary(double v, double DF, const CPeriodicityHypothesisTestsResult &H) : + s_V(v), s_DF(DF), s_H(H) + {} + + double s_V; + double s_DF; + CPeriodicityHypothesisTestsResult s_H; +}; + +using THypothesisSummaryVec = std::vector; + +enum EDiurnalComponents +{ + E_WeekendDay, + E_WeekendWeek, + E_WeekdayDay, + E_WeekdayWeek, + E_Day, + E_Week, +}; + +using TComponent4Vec = core::CSmallVector; + +enum EThreshold +{ + E_LowThreshold, + E_HighThreshold +}; + +// Copy constants into scope. +const core_t::TTime DAY{core::constants::DAY}; +const core_t::TTime WEEKEND{core::constants::WEEKEND}; +const core_t::TTime WEEK{core::constants::WEEK}; +//! The periods of the diurnal components. +const core_t::TTime DIURNAL_PERIODS[]{DAY, WEEK}; +//! The weekend/day windows. +const TTimeTimePr DIURNAL_WINDOWS[]{{0, WEEKEND}, {WEEKEND, WEEK}, {0, WEEK}}; +//! The names of the the diurnal periodic components. +const std::string DIURNAL_COMPONENT_NAMES[] = + { + "weekend daily", + "weekend weekly", + "weekday daily", + "weekday weekly", + "daily", + "weekly" + }; + +//! The confidence interval used for test statistic values. +const double CONFIDENCE_INTERVAL{80.0}; +//! A high priority for components we want to take precendence. +double HIGH_PRIORITY{2.0}; + +//! Fit and remove a linear trend from \p values. +void removeLinearTrend(TFloatMeanAccumulatorVec &values) +{ + using TRegression = CRegression::CLeastSquaresOnline<1, double>; + + TRegression trend; + double time{0.0}; + double dt{10.0 / static_cast(values.size())}; + for (const auto &value : values) + { + trend.add(time, CBasicStatistics::mean(value), CBasicStatistics::count(value)); + time += dt; + } + time = dt / 2.0; + for (auto &&value : values) + { + CBasicStatistics::moment<0>(value) -= trend.predict(time); + time += dt; + } +} + +//! Get the correction to apply to the partition variance test +//! statistic if there are \p bucketsPerRepeat buckets in the +//! in one repeat of the partitioning pattern. +double weekendPartitionVarianceCorrection(std::size_t bucketsPerWeek) +{ + static const std::size_t BUCKETS_PER_WEEK[]{7, 14, 21, 28, 42, 56, 84, 168}; + static const double CORRECTIONS[]{1.0, 1.0, 1.0, 1.12, 1.31, 1.31, 1.31, 1.31}; + std::ptrdiff_t index{std::min( std::lower_bound(boost::begin(BUCKETS_PER_WEEK), + boost::end(BUCKETS_PER_WEEK), + bucketsPerWeek) + - boost::begin(BUCKETS_PER_WEEK), + std::ptrdiff_t(boost::size(BUCKETS_PER_WEEK) - 1))}; + return CORRECTIONS[index]; +} + +//! Compute the \p percentage % variance for a chi-squared random +//! variance with \p df degrees of freedom. +double varianceAtPercentile(double variance, double df, double percentage) +{ + try + { + boost::math::chi_squared chi(df); + return boost::math::quantile(chi, percentage / 100.0) / df * variance; + } + catch (const std::exception &e) + { + LOG_ERROR("Bad input: " << e.what() + << ", df = " << df + << ", percentage = " << percentage); + } + return variance; +} + +//! Compute the \p percentage % autocorrelation for a F distributed +//! random autocorrelation with parameters \p n - 1 and \p n - 1. +double autocorrelationAtPercentile(double autocorrelation, double n, double percentage) +{ + try + { + boost::math::fisher_f f(n - 1.0, n - 1.0); + return boost::math::quantile(f, percentage / 100.0) * autocorrelation; + } + catch (const std::exception &e) + { + LOG_ERROR("Bad input: " << e.what() + << ", n = " << n + << ", percentage = " << percentage); + } + return autocorrelation; +} + +//! Get the length of the \p window. +template +T length(const std::pair &window) +{ + return window.second - window.first; +} + +//! Get the total length of the \p windows. +template +T length(const core::CSmallVector, 2> &windows) +{ + return std::accumulate(windows.begin(), windows.end(), 0, + [](core_t::TTime length_, const TTimeTimePr &window) + { return length_ + length(window); }); +} + +//! Get the length of \p buckets. +template +core_t::TTime length(const T &buckets, core_t::TTime bucketLength) +{ + return static_cast(buckets.size()) * bucketLength; +} + +//! Compute the windows at repeat \p repeat with length \p length. +TTimeTimePr2Vec calculateWindows(core_t::TTime startOfWeek, + core_t::TTime window, + core_t::TTime repeat, + const TTimeTimePr &interval) +{ + core_t::TTime a{startOfWeek + interval.first}; + core_t::TTime b{startOfWeek + window}; + core_t::TTime l{length(interval)}; + TTimeTimePr2Vec result; + result.reserve((b - a) / repeat); + for (core_t::TTime time = a; time < b; time += repeat) + { + result.emplace_back(time, time + l); + } + return result; +} + +//! Get the index ranges corresponding to \p windows. +std::size_t calculateIndexWindows(const TTimeTimePr2Vec &windows, + core_t::TTime bucketLength, + TSizeSizePr2Vec &result) +{ + std::size_t l(0); + result.reserve(windows.size()); + for (const auto &window : windows) + { + core_t::TTime a{window.first / bucketLength}; + core_t::TTime b{window.second / bucketLength}; + result.emplace_back(a, b); + l += b - a; + } + return l; +} + +//! Compute the projection of \p values to \p windows. +void project(const TFloatMeanAccumulatorVec &values, + const TTimeTimePr2Vec &windows_, + core_t::TTime bucketLength, + TFloatMeanAccumulatorVec &result) +{ + result.clear(); + if (!values.empty()) + { + TSizeSizePr2Vec windows; + calculateIndexWindows(windows_, bucketLength, windows); + result.reserve(length(windows)); + std::size_t n{values.size()}; + for (std::size_t i = 0u; i < windows.size(); ++i) + { + std::size_t a{windows[i].first}; + std::size_t b{windows[i].second}; + for (std::size_t j = a; j < b; ++j) + { + const TFloatMeanAccumulator &value{values[j % n]}; + result.push_back(value); + } + } + } +} + +//! Compute the periodic trend from \p values falling in \p windows. +template +void periodicTrend(const U &values, + const TSizeSizePr2Vec &windows_, + core_t::TTime bucketLength, + V &trend) +{ + if (!trend.empty()) + { + TSizeSizePr2Vec windows; + calculateIndexWindows(windows_, bucketLength, windows); + std::size_t period{trend.size()}; + std::size_t n{values.size()}; + for (std::size_t i = 0u; i < windows.size(); ++i) + { + std::size_t a{windows[i].first}; + std::size_t b{windows[i].second}; + for (std::size_t j = a; j < b; ++j) + { + const TFloatMeanAccumulator &value{values[j % n]}; + trend[(j - a) % period].add(CBasicStatistics::mean(value), + CBasicStatistics::count(value)); + } + } + } +} + +//! Compute the average of the values at \p times. +void averageValue(const TFloatMeanAccumulatorVec &values, + const TTimeVec ×, + core_t::TTime bucketLength, + TMeanVarAccumulator &value) +{ + for (const auto time : times) + { + std::size_t index(time / bucketLength); + value.add(CBasicStatistics::mean(values[index]), + CBasicStatistics::count(values[index])); + } +} + +//! Get the maximum residual of \p trend. +template +double trendAmplitude(const T &trend) +{ + using TMaxAccumulator = CBasicStatistics::SMax::TAccumulator; + + TMeanAccumulator level; + for (const auto &bucket : trend) + { + level.add(mean(bucket), count(bucket)); + } + + TMaxAccumulator result; + result.add(0.0); + for (const auto &bucket : trend) + { + if (count(bucket) > 0.0) + { + result.add(std::fabs(mean(bucket) - CBasicStatistics::mean(level))); + } + } + + return result[0]; +} + +//! Extract the residual variance from the mean of a collection +//! of residual variances. +double residualVariance(const TMeanAccumulator &mean) +{ + double n{CBasicStatistics::count(mean)}; + return n <= 1.0 ? 0.0 : n / (n - 1.0) * std::max(CBasicStatistics::mean(mean), 0.0); +} + +//! Extract the residual variance of \p bucket of a trend. +TMeanAccumulator residualVariance(const TMeanVarAccumulator &bucket, + double scale) +{ + return CBasicStatistics::accumulator(scale * CBasicStatistics::count(bucket), + CBasicStatistics::maximumLikelihoodVariance(bucket)); +} + +//! \brief Partially specialized helper class to get the trend +//! residual variance as a specified type. +template struct SResidualVarianceImpl {}; + +//! \brief Get the residual variance as a double. +template<> +struct SResidualVarianceImpl +{ + static double get(const TMeanAccumulator &mean) + { + return residualVariance(mean); + } +}; + +//! \brief Get the residual variance as a mean accumulator. +template<> +struct SResidualVarianceImpl +{ + static TMeanAccumulator get(const TMeanAccumulator &mean) + { + return mean; + } +}; + +//! Compute the residual variance of the trend \p trend. +template +R residualVariance(const T &trend, double scale) +{ + TMeanAccumulator result; + for (const auto &bucket : trend) + { + result.add(CBasicStatistics::maximumLikelihoodVariance(bucket), + CBasicStatistics::count(bucket)); + } + result.s_Count *= scale; + return SResidualVarianceImpl::get(result); +} + +} + +bool CPeriodicityHypothesisTestsResult::operator==(const CPeriodicityHypothesisTestsResult &other) const +{ + return m_Components == other.m_Components; +} + +const CPeriodicityHypothesisTestsResult & +CPeriodicityHypothesisTestsResult::operator+=(const CPeriodicityHypothesisTestsResult &other) +{ + m_Components.insert(m_Components.end(), + other.m_Components.begin(), + other.m_Components.end()); + return *this; +} + +void CPeriodicityHypothesisTestsResult::add(const std::string &description, + bool diurnal, + core_t::TTime startOfPartition, + core_t::TTime period, + const TTimeTimePr &window, + double precedence) +{ + m_Components.emplace_back(description, diurnal, startOfPartition, period, window, precedence); +} + +void CPeriodicityHypothesisTestsResult::remove(const std::string &description) +{ + auto i = std::find_if(m_Components.begin(), m_Components.end(), + [&description](const SComponent &component) + { + return component.s_Description == description; + }); + if (i != m_Components.end()) + { + m_Components.erase(i); + } +} + +bool CPeriodicityHypothesisTestsResult::periodic() const +{ + return m_Components.size() > 0; +} + +const CPeriodicityHypothesisTestsResult::TComponent5Vec & +CPeriodicityHypothesisTestsResult::components() const +{ + return m_Components; +} + +std::string CPeriodicityHypothesisTestsResult::print() const +{ + std::string result("{"); + for (const auto &component : m_Components) + { + result += " '" + component.s_Description + "'"; + } + result += " }"; + return result; +} + +CPeriodicityHypothesisTestsResult::SComponent::SComponent() : + s_Description(""), + s_Diurnal(false), + s_StartOfPartition(0), + s_Period(0), + s_Precedence(0.0) +{} + +CPeriodicityHypothesisTestsResult::SComponent::SComponent(const std::string &description, + bool diurnal, + core_t::TTime startOfPartition, + core_t::TTime period, + const TTimeTimePr &window, + double precedence) : + s_Description(description), + s_Diurnal(diurnal), + s_StartOfPartition(startOfPartition), + s_Period(period), + s_Window(window), + s_Precedence(precedence) +{} + +bool CPeriodicityHypothesisTestsResult::SComponent::operator==(const SComponent &other) const +{ + return s_Description == other.s_Description + && s_StartOfPartition == other.s_StartOfPartition; +} + +CSeasonalTime *CPeriodicityHypothesisTestsResult::SComponent::seasonalTime() const +{ + if (s_Diurnal) + { + return new CDiurnalTime(s_StartOfPartition, + s_Window.first, + s_Window.second, + s_Period, s_Precedence); + } + return new CGeneralPeriodTime(s_Period, s_Precedence); +} + + +CPeriodicityHypothesisTestsConfig::CPeriodicityHypothesisTestsConfig() : + m_TestForDiurnal(true), + m_HasDaily(false), + m_HasWeekend(false), + m_HasWeekly(false), + m_StartOfWeek(0) +{} + +void CPeriodicityHypothesisTestsConfig::disableDiurnal() +{ + m_TestForDiurnal = false; +} + +void CPeriodicityHypothesisTestsConfig::hasDaily(bool value) +{ + m_HasDaily = value; +} + +void CPeriodicityHypothesisTestsConfig::hasWeekend(bool value) +{ + m_HasWeekend = value; +} + +void CPeriodicityHypothesisTestsConfig::hasWeekly(bool value) +{ + m_HasWeekly = value; +} + +void CPeriodicityHypothesisTestsConfig::startOfWeek(core_t::TTime value) +{ + m_StartOfWeek = value; +} + +bool CPeriodicityHypothesisTestsConfig::testForDiurnal() const +{ + return m_TestForDiurnal; +} + +bool CPeriodicityHypothesisTestsConfig::hasDaily() const +{ + return m_HasDaily; +} + +bool CPeriodicityHypothesisTestsConfig::hasWeekend() const +{ + return m_HasWeekend; +} + +bool CPeriodicityHypothesisTestsConfig::hasWeekly() const +{ + return m_HasWeekly; +} + +core_t::TTime CPeriodicityHypothesisTestsConfig::startOfWeek() const +{ + return m_StartOfWeek; +} + + +CPeriodicityHypothesisTests::CPeriodicityHypothesisTests() : + m_BucketLength(0), m_WindowLength(0), m_Period(0) +{} +CPeriodicityHypothesisTests::CPeriodicityHypothesisTests(const CPeriodicityHypothesisTestsConfig &config) : + m_Config(config), m_BucketLength(0), m_WindowLength(0), m_Period(0) +{} + +bool CPeriodicityHypothesisTests::initialized() const +{ + return m_BucketValues.size() > 0; +} + +void CPeriodicityHypothesisTests::initialize(core_t::TTime bucketLength, + core_t::TTime windowLength, + core_t::TTime period) +{ + m_BucketLength = bucketLength; + m_WindowLength = windowLength; + m_BucketValues.resize(static_cast(windowLength / m_BucketLength)); + m_Period = period; +} + +void CPeriodicityHypothesisTests::add(core_t::TTime time, double value, double weight) +{ + if (!m_BucketValues.empty()) + { + std::size_t i((time % m_WindowLength) / m_BucketLength); + m_BucketValues[i].add(value, weight); + if (weight > 0.0) + { + m_TimeRange.add(time); + } + } +} + +CPeriodicityHypothesisTestsResult CPeriodicityHypothesisTests::test() const +{ + // We perform a series of tests of nested hypotheses about + // the periodic components and weekday/end patterns. To test + // for periodic components we compare the residual variance + // with and without trend. This must be reduced in significant + // absolute sense to make it worthwhile modelling and in a + // statistical sense. We use an F-test for this purpose. Note + // that since the buckets contain the mean of multiple samples + // we expect them to tend to Gaussian. We also test the amplitude. + // Again this must be significant in both an absolute and + // statistical sense. We assume the bucket values are Gaussian, + // with the same rationale, for the purpose of the statistical + // test. Each time we accept a simpler hypothesis about the + // data we test the nested hypothesis w.r.t. this. This entails + // removing any periodic component we've already found from the + // data. + + if (!this->initialized()) + { + return CPeriodicityHypothesisTestsResult(); + } + + auto window = [this](core_t::TTime period) + { + std::size_t bucketsPerPeriod(period / m_BucketLength); + std::size_t repeats{bucketsPerPeriod == 0 ? + 0 : m_BucketValues.size() / bucketsPerPeriod}; + core_t::TTime windowLength{static_cast(repeats) * period}; + return TTimeTimePr2Vec{{0, windowLength}}; + }; + auto buckets = [this](core_t::TTime period) + { + std::size_t bucketsPerPeriod(period / m_BucketLength); + std::size_t repeats{bucketsPerPeriod == 0 ? + 0 : m_BucketValues.size() / bucketsPerPeriod}; + return bucketsPerPeriod * repeats; + }; + + TFloatMeanAccumulatorVec detrendedBucketValues(m_BucketValues); + removeLinearTrend(detrendedBucketValues); + + TTimeTimePr2Vec windowForTestingDaily(window(DAY)); + TTimeTimePr2Vec windowForTestingWeekly(window(WEEK)); + TTimeTimePr2Vec windowForTestingPeriod(window(m_Period)); + TFloatMeanAccumulatorCRng bucketsForTestingDaily[]{{m_BucketValues, 0, buckets(DAY)}, + {detrendedBucketValues, 0, buckets(DAY)}}; + TFloatMeanAccumulatorCRng bucketsForTestingWeekly[]{{m_BucketValues, 0, buckets(WEEK)}, + {detrendedBucketValues, 0, buckets(WEEK)}}; + TFloatMeanAccumulatorCRng bucketsForTestingPeriod[]{{m_BucketValues, 0, buckets(m_Period)}, + {detrendedBucketValues, 0, buckets(m_Period)}}; + + LOG_TRACE("Testing periodicity hypotheses"); + LOG_TRACE("window for daily = " << core::CContainerPrinter::print(windowForTestingDaily)); + LOG_TRACE("window for weekly = " << core::CContainerPrinter::print(windowForTestingWeekly)); + LOG_TRACE("window for period = " << core::CContainerPrinter::print(windowForTestingPeriod)); + + TNestedHypothesesVec hypotheses; + + for (std::size_t i : {0, 1}) + { + TNestedHypothesesVec hypotheses_; + + if (this->seenSufficientDataToTest(WEEK, bucketsForTestingWeekly[i])) + { + this->hypothesesForWeekly(windowForTestingWeekly, + bucketsForTestingWeekly[i], + windowForTestingPeriod, + bucketsForTestingPeriod[i], + hypotheses_); + } + else if (this->seenSufficientDataToTest(DAY, bucketsForTestingDaily[i])) + { + this->hypothesesForDaily(windowForTestingDaily, + bucketsForTestingDaily[i], + windowForTestingPeriod, + bucketsForTestingPeriod[i], + hypotheses_); + } + else if (this->seenSufficientDataToTest(m_Period, bucketsForTestingPeriod[i])) + { + this->hypothesesForPeriod(windowForTestingPeriod, + bucketsForTestingPeriod[i], + hypotheses_); + } + + hypotheses.insert(hypotheses.end(), hypotheses_.begin(), hypotheses_.end()); + } + + return this->best(hypotheses); +} + +void CPeriodicityHypothesisTests::hypothesesForWeekly(const TTimeTimePr2Vec &windowForTestingWeekly, + const TFloatMeanAccumulatorCRng &bucketsForTestingWeekly, + const TTimeTimePr2Vec &windowForTestingPeriod, + const TFloatMeanAccumulatorCRng &bucketsForTestingPeriod, + TNestedHypothesesVec &hypotheses) const +{ + if (WEEK % m_Period == 0) + { + auto testForNull = boost::bind(&CPeriodicityHypothesisTests::testForNull, + this, boost::cref(windowForTestingWeekly), + boost::cref(bucketsForTestingWeekly), _1); + auto testForPeriod = boost::bind(&CPeriodicityHypothesisTests::testForPeriod, + this, boost::cref(windowForTestingWeekly), + boost::cref(bucketsForTestingWeekly), _1); + auto testForDaily = boost::bind(&CPeriodicityHypothesisTests::testForDaily, + this, boost::cref(windowForTestingWeekly), + boost::cref(bucketsForTestingWeekly), _1); + auto testForWeekly = boost::bind(&CPeriodicityHypothesisTests::testForWeekly, + this, boost::cref(windowForTestingWeekly), + boost::cref(bucketsForTestingWeekly), _1); + auto testForDailyWithWeekend = boost::bind(&CPeriodicityHypothesisTests::testForDailyWithWeekend, + this, boost::cref(bucketsForTestingWeekly), _1); + auto testForWeeklyGivenWeekend = boost::bind(&CPeriodicityHypothesisTests::testForWeeklyGivenDailyWithWeekend, + this, boost::cref(windowForTestingWeekly), + boost::cref(bucketsForTestingWeekly), _1); + + hypotheses.resize(1); + if (DAY % m_Period == 0) + { + hypotheses[0].null(testForNull) + .addNested(testForPeriod) + .addNested(testForDaily) + .addNested(testForDailyWithWeekend) + .addNested(testForWeeklyGivenWeekend) + .finishedNested() + .addAlternative(testForWeekly) + .finishedNested() + .finishedNested() + .addAlternative(testForDaily) + .addNested(testForDailyWithWeekend) + .addNested(testForWeeklyGivenWeekend) + .finishedNested() + .addAlternative(testForWeekly) + .finishedNested() + .addAlternative(testForDailyWithWeekend) + .addNested(testForWeeklyGivenWeekend) + .finishedNested() + .addAlternative(testForWeekly); + } + else + { + hypotheses[0].null(testForNull) + .addNested(testForDaily) + .addNested(testForDailyWithWeekend) + .addNested(testForWeeklyGivenWeekend) + .finishedNested() + .addAlternative(testForWeekly) + .finishedNested() + .addAlternative(testForDailyWithWeekend) + .addNested(testForWeeklyGivenWeekend) + .finishedNested() + .addAlternative(testForPeriod) + .addNested(testForWeekly) + .finishedNested() + .addAlternative(testForWeekly); + } + } + else if (m_Period % WEEK == 0) + { + auto testForNull = boost::bind(&CPeriodicityHypothesisTests::testForNull, + this, boost::cref(windowForTestingPeriod), + boost::cref(bucketsForTestingPeriod), _1); + auto testForPeriod = boost::bind(&CPeriodicityHypothesisTests::testForPeriod, + this, boost::cref(windowForTestingPeriod), + boost::cref(bucketsForTestingPeriod), _1); + auto testForDaily = boost::bind(&CPeriodicityHypothesisTests::testForDaily, + this, boost::cref(windowForTestingPeriod), + boost::cref(bucketsForTestingPeriod), _1); + auto testForWeekly = boost::bind(&CPeriodicityHypothesisTests::testForWeekly, + this, boost::cref(windowForTestingPeriod), + boost::cref(bucketsForTestingPeriod), _1); + auto testForDailyWithWeekend = boost::bind(&CPeriodicityHypothesisTests::testForDailyWithWeekend, + this, boost::cref(bucketsForTestingPeriod), _1); + auto testForWeeklyGivenWeekend = boost::bind(&CPeriodicityHypothesisTests::testForWeeklyGivenDailyWithWeekend, + this, boost::cref(windowForTestingPeriod), + boost::cref(bucketsForTestingPeriod), _1); + + hypotheses.resize(1); + hypotheses[0].null(testForNull) + .addNested(testForDaily) + .addNested(testForDailyWithWeekend) + .addNested(testForWeeklyGivenWeekend) + .addNested(testForPeriod) + .finishedNested() + .finishedNested() + .addAlternative(testForWeekly) + .addNested(testForPeriod) + .finishedNested() + .finishedNested() + .addAlternative(testForDailyWithWeekend) + .addNested(testForWeeklyGivenWeekend) + .addNested(testForPeriod) + .finishedNested() + .finishedNested() + .addAlternative(testForWeekly) + .addNested(testForPeriod) + .finishedNested() + .addAlternative(testForPeriod); + } + else + { + { + auto testForNull = boost::bind(&CPeriodicityHypothesisTests::testForNull, + this, boost::cref(windowForTestingWeekly), + boost::cref(bucketsForTestingWeekly), _1); + auto testForDaily = boost::bind(&CPeriodicityHypothesisTests::testForDaily, + this, boost::cref(windowForTestingWeekly), + boost::cref(bucketsForTestingWeekly), _1); + auto testForWeekly = boost::bind(&CPeriodicityHypothesisTests::testForWeekly, + this, boost::cref(windowForTestingWeekly), + boost::cref(bucketsForTestingWeekly), _1); + auto testForDailyWithWeekend = boost::bind(&CPeriodicityHypothesisTests::testForDailyWithWeekend, + this, boost::cref(bucketsForTestingWeekly), _1); + auto testForWeeklyGivenWeekend = boost::bind(&CPeriodicityHypothesisTests::testForWeeklyGivenDailyWithWeekend, + this, boost::cref(windowForTestingWeekly), + boost::cref(bucketsForTestingWeekly), _1); + + hypotheses.resize(2); + hypotheses[0].null(testForNull) + .addNested(testForDaily) + .addNested(testForDailyWithWeekend) + .addNested(testForWeeklyGivenWeekend) + .finishedNested() + .addAlternative(testForWeekly) + .finishedNested() + .addAlternative(testForDailyWithWeekend) + .addNested(testForWeeklyGivenWeekend) + .finishedNested() + .addAlternative(testForWeekly); + } + if (m_Period % DAY == 0) + { + auto testForNull = boost::bind(&CPeriodicityHypothesisTests::testForNull, + this, boost::cref(windowForTestingPeriod), + boost::cref(bucketsForTestingPeriod), _1); + auto testForDaily = boost::bind(&CPeriodicityHypothesisTests::testForDaily, + this, boost::cref(windowForTestingPeriod), + boost::cref(bucketsForTestingPeriod), _1); + auto testForPeriod = boost::bind(&CPeriodicityHypothesisTests::testForPeriod, + this, boost::cref(windowForTestingPeriod), + boost::cref(bucketsForTestingPeriod), _1); + + hypotheses[1].null(testForNull) + .addNested(testForDaily) + .addNested(testForPeriod) + .finishedNested() + .addAlternative(testForPeriod); + } + else + { + auto testForNull = boost::bind(&CPeriodicityHypothesisTests::testForNull, + this, boost::cref(windowForTestingPeriod), + boost::cref(bucketsForTestingPeriod), _1); + auto testForPeriod = boost::bind(&CPeriodicityHypothesisTests::testForPeriod, + this, boost::cref(windowForTestingPeriod), + boost::cref(bucketsForTestingPeriod), _1); + + hypotheses[1].null(testForNull) + .addNested(testForPeriod); + } + } +} + +void CPeriodicityHypothesisTests::hypothesesForDaily(const TTimeTimePr2Vec &windowForTestingDaily, + const TFloatMeanAccumulatorCRng &bucketsForTestingDaily, + const TTimeTimePr2Vec &windowForTestingPeriod, + const TFloatMeanAccumulatorCRng &bucketsForTestingPeriod, + TNestedHypothesesVec &hypotheses) const +{ + if (DAY % m_Period == 0) + { + auto testForNull = boost::bind(&CPeriodicityHypothesisTests::testForNull, + this, boost::cref(windowForTestingDaily), + boost::cref(bucketsForTestingDaily), _1); + auto testForPeriod = boost::bind(&CPeriodicityHypothesisTests::testForPeriod, + this, boost::cref(windowForTestingDaily), + boost::cref(bucketsForTestingDaily), _1); + auto testForDaily = boost::bind(&CPeriodicityHypothesisTests::testForDaily, + this, boost::cref(windowForTestingDaily), + boost::cref(bucketsForTestingDaily), _1); + + hypotheses.resize(1); + hypotheses[0].null(testForNull) + .addNested(testForPeriod) + .addNested(testForDaily) + .finishedNested() + .addAlternative(testForDaily); + } + else if (m_Period % DAY == 0) + { + auto testForNull = boost::bind(&CPeriodicityHypothesisTests::testForNull, + this, boost::cref(windowForTestingPeriod), + boost::cref(bucketsForTestingPeriod), _1); + auto testForPeriod = boost::bind(&CPeriodicityHypothesisTests::testForPeriod, + this, boost::cref(windowForTestingPeriod), + boost::cref(bucketsForTestingPeriod), _1); + auto testForDaily = boost::bind(&CPeriodicityHypothesisTests::testForDaily, + this, boost::cref(windowForTestingPeriod), + boost::cref(bucketsForTestingPeriod), _1); + + hypotheses.resize(1); + hypotheses[0].null(testForNull) + .addNested(testForDaily) + .addNested(testForPeriod); + } + else + { + { + auto testForNull = boost::bind(&CPeriodicityHypothesisTests::testForNull, + this, boost::cref(windowForTestingDaily), + boost::cref(bucketsForTestingDaily), _1); + auto testForDaily = boost::bind(&CPeriodicityHypothesisTests::testForDaily, + this, boost::cref(windowForTestingDaily), + boost::cref(bucketsForTestingDaily), _1); + + hypotheses.resize(2); + hypotheses[0].null(testForNull) + .addNested(testForDaily); + } + { + auto testForNull = boost::bind(&CPeriodicityHypothesisTests::testForNull, + this, boost::cref(windowForTestingPeriod), + boost::cref(bucketsForTestingPeriod), _1); + auto testForPeriod = boost::bind(&CPeriodicityHypothesisTests::testForPeriod, + this, boost::cref(windowForTestingPeriod), + boost::cref(bucketsForTestingPeriod), _1); + hypotheses[1].null(testForNull) + .addNested(testForPeriod); + } + } +} + +void CPeriodicityHypothesisTests::hypothesesForPeriod(const TTimeTimePr2Vec &windows, + const TFloatMeanAccumulatorCRng &buckets, + TNestedHypothesesVec &hypotheses) const +{ + auto testForNull = boost::bind(&CPeriodicityHypothesisTests::testForNull, + this, boost::cref(windows), boost::cref(buckets), _1); + auto testForPeriod = boost::bind(&CPeriodicityHypothesisTests::testForPeriod, + this, boost::cref(windows), boost::cref(buckets), _1); + + hypotheses.resize(1); + hypotheses[0].null(testForNull) + .addNested(testForPeriod); +} + +CPeriodicityHypothesisTestsResult +CPeriodicityHypothesisTests::best(const TNestedHypothesesVec &hypotheses) const +{ + // Note if there isn't a clear cut best hypothesis for variance + // reduction we choose the simplest hypothesis, i.e. with maximum + // degrees-of-freedom. + + using TMinAccumulator = CBasicStatistics::SMin::TAccumulator; + + LOG_TRACE("# hypotheses = " << hypotheses.size()); + + CPeriodicityHypothesisTestsResult result; + + THypothesisSummaryVec summaries; + summaries.reserve(hypotheses.size()); + + for (const auto &hypothesis : hypotheses) + { + STestStats stats; + CPeriodicityHypothesisTestsResult resultForHypothesis{hypothesis.test(stats)}; + summaries.emplace_back(stats.s_V0, stats.s_B - stats.s_DF0, + std::move(resultForHypothesis)); + } + + TMinAccumulator vCutoff; + for (const auto &summary : summaries) + { + vCutoff.add(varianceAtPercentile(summary.s_V, summary.s_DF, + 50.0 + CONFIDENCE_INTERVAL / 2.0)); + } + LOG_TRACE("variance cutoff = " << vCutoff[0]); + + TMinAccumulator df; + for (const auto &summary : summaries) + { + double v{varianceAtPercentile(summary.s_V, summary.s_DF, + 50.0 - CONFIDENCE_INTERVAL / 2.0)}; + if (v <= vCutoff[0] && df.add(-summary.s_DF)) + { + result = summary.s_H; + } + } + + return result; +} + +CPeriodicityHypothesisTestsResult +CPeriodicityHypothesisTests::testForNull(const TTimeTimePr2Vec &window, + const TFloatMeanAccumulatorCRng &buckets, + STestStats &stats) const +{ + LOG_TRACE("Testing null on " << core::CContainerPrinter::print(window)); + this->nullHypothesis(window, buckets, stats); + return CPeriodicityHypothesisTestsResult(); +} + +CPeriodicityHypothesisTestsResult +CPeriodicityHypothesisTests::testForDaily(const TTimeTimePr2Vec &windows, + const TFloatMeanAccumulatorCRng &buckets, + STestStats &stats) const +{ + LOG_TRACE("Testing daily on " << core::CContainerPrinter::print(windows)); + + CPeriodicityHypothesisTestsResult result{stats.s_H0}; + + stats.s_HasPeriod = m_Config.hasDaily(); + stats.setThresholds(SIGNIFICANT_VARIANCE_REDUCTION[E_LowThreshold], + SIGNIFICANT_AMPLITUDE[E_LowThreshold], + SIGNIFICANT_AUTOCORRELATION[E_LowThreshold]); + + if ( m_Config.testForDiurnal() + && m_BucketLength <= DAY / 4 + && this->seenSufficientDataToTest(DAY, buckets) + && this->testPeriod(windows, buckets, DAY, stats)) + { + this->hypothesis({DAY}, buckets, stats); + result.add(DIURNAL_COMPONENT_NAMES[E_Day], true, 0, + DIURNAL_PERIODS[static_cast(E_Day) % 2], + DIURNAL_WINDOWS[static_cast(E_Day) / 2]); + } + + return result; +} + +CPeriodicityHypothesisTestsResult +CPeriodicityHypothesisTests::testForWeekly(const TTimeTimePr2Vec &windows, + const TFloatMeanAccumulatorCRng &buckets, + STestStats &stats) const +{ + LOG_TRACE("Testing weekly on " << core::CContainerPrinter::print(windows)); + + CPeriodicityHypothesisTestsResult result{stats.s_H0}; + + stats.s_HasPeriod = m_Config.hasWeekly(); + stats.setThresholds(SIGNIFICANT_VARIANCE_REDUCTION[E_LowThreshold], + SIGNIFICANT_AMPLITUDE[E_LowThreshold], + SIGNIFICANT_AUTOCORRELATION[E_LowThreshold]); + + if ( m_Config.testForDiurnal() + && m_BucketLength <= WEEK / 4 + && this->seenSufficientDataToTest(WEEK, buckets) + && this->testPeriod(windows, buckets, WEEK, stats)) + { + stats.s_StartOfPartition = 0; + stats.s_Partition.assign(1, {0, length(buckets, m_BucketLength)}); + this->hypothesis({WEEK}, buckets, stats); + result.add(DIURNAL_COMPONENT_NAMES[E_Week], true, 0, + DIURNAL_PERIODS[static_cast(E_Week) % 2], + DIURNAL_WINDOWS[static_cast(E_Week) / 2]); + } + + return result; +} + +CPeriodicityHypothesisTestsResult +CPeriodicityHypothesisTests::testForDailyWithWeekend(const TFloatMeanAccumulatorCRng &buckets, + STestStats &stats) const +{ + LOG_TRACE("Testing for weekend"); + + CPeriodicityHypothesisTestsResult result{stats.s_H0}; + + stats.s_HasPartition = m_Config.hasWeekend(); + stats.s_StartOfPartition = m_Config.hasWeekend() ? m_Config.startOfWeek() : 0; + stats.setThresholds(SIGNIFICANT_VARIANCE_REDUCTION[E_HighThreshold], + SIGNIFICANT_AMPLITUDE[E_HighThreshold], + SIGNIFICANT_AUTOCORRELATION[E_HighThreshold]); + + TTimeTimePr2Vec partition{{0, WEEKEND}, {WEEKEND, WEEK}}; + std::size_t bucketsPerWeek(WEEK / m_BucketLength); + + if ( m_Config.testForDiurnal() + && m_BucketLength <= DAY / 4 + && this->seenSufficientDataToTest(WEEK, buckets) + && this->testPartition(partition, buckets, DAY, + weekendPartitionVarianceCorrection(bucketsPerWeek), + stats)) + { + stats.s_Partition = partition; + this->hypothesis({DAY, DAY}, buckets, stats); + core_t::TTime startOfWeek{stats.s_StartOfPartition}; + result.remove(DIURNAL_COMPONENT_NAMES[E_Day]); + result.add(DIURNAL_COMPONENT_NAMES[E_WeekendDay], true, startOfWeek, + DIURNAL_PERIODS[static_cast(E_WeekendDay) % 2], + DIURNAL_WINDOWS[static_cast(E_WeekendDay) / 2], + HIGH_PRIORITY); + result.add(DIURNAL_COMPONENT_NAMES[E_WeekdayDay], true, startOfWeek, + DIURNAL_PERIODS[static_cast(E_WeekdayDay) % 2], + DIURNAL_WINDOWS[static_cast(E_WeekdayDay) / 2], + HIGH_PRIORITY); + } + + return result; +} + +CPeriodicityHypothesisTestsResult +CPeriodicityHypothesisTests::testForWeeklyGivenDailyWithWeekend(const TTimeTimePr2Vec &windows, + const TFloatMeanAccumulatorCRng &buckets, + STestStats &stats) const +{ + LOG_TRACE("Testing for weekly given weekend on " << core::CContainerPrinter::print(windows)); + + CPeriodicityHypothesisTestsResult result(stats.s_H0); + + if (!m_Config.testForDiurnal()) + { + return result; + } + + core_t::TTime startOfWeek{stats.s_StartOfPartition}; + + CPeriodicityHypothesisTestsResult resultForWeekly{this->testForWeekly(windows, buckets, stats)}; + if (resultForWeekly != result) + { + // Note that testForWeekly sets up the hypothesis for us. + result.add(DIURNAL_COMPONENT_NAMES[E_WeekendWeek], true, startOfWeek, + DIURNAL_PERIODS[static_cast(E_WeekendWeek) % 2], + DIURNAL_WINDOWS[static_cast(E_WeekendWeek) / 2], + HIGH_PRIORITY); + result.add(DIURNAL_COMPONENT_NAMES[E_WeekdayWeek], true, startOfWeek, + DIURNAL_PERIODS[static_cast(E_WeekdayWeek) % 2], + DIURNAL_WINDOWS[static_cast(E_WeekdayWeek) / 2], + HIGH_PRIORITY); + return result; + } + + core_t::TTime windowLength{length(windows)}; + TTimeTimePr2Vec partition{{0, WEEKEND}, {WEEKEND, WEEK}}; + + TTimeTimePr2Vec weekday(calculateWindows(startOfWeek, windowLength, WEEK, {WEEKEND, WEEK})); + CPeriodicityHypothesisTestsResult resultForWeekday{this->testForWeekly(weekday, buckets, stats)}; + if (resultForWeekday != result) + { + stats.s_StartOfPartition = startOfWeek; + stats.s_Partition = partition; + this->hypothesis({DAY, WEEK}, buckets, stats); + result.add(DIURNAL_COMPONENT_NAMES[E_WeekdayWeek], true, startOfWeek, + DIURNAL_PERIODS[static_cast(E_WeekdayWeek) % 2], + DIURNAL_WINDOWS[static_cast(E_WeekdayWeek) / 2], + HIGH_PRIORITY); + return result; + } + + TTimeTimePr2Vec weekend(calculateWindows(startOfWeek, windowLength, WEEK, {0, WEEKEND})); + CPeriodicityHypothesisTestsResult resultForWeekend{this->testForWeekly(weekend, buckets, stats)}; + if (resultForWeekend != result) + { + stats.s_StartOfPartition = startOfWeek; + stats.s_Partition = partition; + this->hypothesis({WEEK, DAY}, buckets, stats); + result.add(DIURNAL_COMPONENT_NAMES[E_WeekendWeek], true, startOfWeek, + DIURNAL_PERIODS[static_cast(E_WeekendWeek) % 2], + DIURNAL_WINDOWS[static_cast(E_WeekendWeek) / 2], + HIGH_PRIORITY); + } + + return result; +} + +CPeriodicityHypothesisTestsResult +CPeriodicityHypothesisTests::testForPeriod(const TTimeTimePr2Vec &windows, + const TFloatMeanAccumulatorCRng &buckets, + STestStats &stats) const +{ + LOG_TRACE("Testing for " << m_Period << " on " << core::CContainerPrinter::print(windows)); + + CPeriodicityHypothesisTestsResult result{stats.s_H0}; + + if ( m_Period != DAY + && m_Period != WEEK + && m_BucketLength <= m_Period / 4 + && this->seenSufficientDataToTest(m_Period, buckets)) + { + stats.s_HasPeriod = false; + EThreshold index{m_Period % DAY == 0 ? E_LowThreshold : E_HighThreshold}; + stats.setThresholds(SIGNIFICANT_VARIANCE_REDUCTION[index], + SIGNIFICANT_AMPLITUDE[index], + SIGNIFICANT_AUTOCORRELATION[index]); + if (this->testPeriod(windows, buckets, m_Period, stats)) + { + stats.s_StartOfPartition = 0; + stats.s_Partition.assign(1, {0, length(buckets, m_BucketLength)}); + this->hypothesis({m_Period}, buckets, stats); + result.add(core::CStringUtils::typeToString(m_Period), + false, 0, m_Period, {0, m_Period}); + } + } + + return result; +} + +bool CPeriodicityHypothesisTests::seenSufficientDataToTest(core_t::TTime period, + const TFloatMeanAccumulatorCRng &buckets) const +{ + return (buckets.size() * m_BucketLength) / period >= 2 + && m_TimeRange.initialized() + && static_cast(m_TimeRange.range()) + >= 2.0 * ACCURATE_TEST_POPULATED_FRACTION * static_cast(period); +} + +bool CPeriodicityHypothesisTests::testStatisticsFor(const TFloatMeanAccumulatorCRng &buckets, + STestStats &stats) const +{ + CBasicStatistics::CMinMax range; + double populated{0.0}; + double count{0.0}; + for (std::size_t i = 0u; i < buckets.size(); ++i) + { + double ni{CBasicStatistics::count(buckets[i])}; + count += ni; + if (ni > 0.0) + { + populated += 1.0; + range.add(static_cast(i)); + } + } + + if (populated == 0.0) + { + return false; + } + + LOG_TRACE("populated = " << 100.0 * populated << "%"); + + stats.s_Range = range.max() - range.min(); + stats.s_B = populated; + stats.s_M = count / stats.s_B; + LOG_TRACE("range = " << stats.s_Range + << ", populatedBuckets = " << stats.s_B + << ", valuesPerBucket = " << stats.s_M); + + return true; +} + +void CPeriodicityHypothesisTests::nullHypothesis(const TTimeTimePr2Vec &window, + const TFloatMeanAccumulatorCRng &buckets, + STestStats &stats) const +{ + if (this->testStatisticsFor(buckets, stats)) + { + TMeanVarAccumulatorVec trend(1); + periodicTrend(buckets, window, m_BucketLength, trend); + double mean{CBasicStatistics::mean(trend[0])}; + double v0{CBasicStatistics::variance(trend[0])}; + LOG_TRACE("mean = " << mean); + LOG_TRACE("variance = " << v0); + stats.s_DF0 = 1.0; + stats.s_V0 = v0; + stats.s_T0.assign(1, TDoubleVec{mean}); + stats.s_Partition = window; + } +} + +void CPeriodicityHypothesisTests::hypothesis(const TTime2Vec &periods, + const TFloatMeanAccumulatorCRng &buckets, + STestStats &stats) const +{ + if (this->testStatisticsFor(buckets, stats)) + { + stats.s_V0 = 0.0; + stats.s_DF0 = 0.0; + stats.s_T0 = TDoubleVec2Vec(stats.s_Partition.size()); + for (std::size_t i = 0u; i < stats.s_Partition.size(); ++i) + { + core_t::TTime period_{ std::min(periods[i], length(stats.s_Partition[i])) + / m_BucketLength}; + TTimeTimePr2Vec windows(calculateWindows(stats.s_StartOfPartition, + length(buckets, m_BucketLength), + length(stats.s_Partition), + stats.s_Partition[i])); + TMeanVarAccumulatorVec trend(periods[i] / m_BucketLength); + periodicTrend(buckets, windows, m_BucketLength, trend); + stats.s_V0 += residualVariance(trend, 1.0 / stats.s_M); + stats.s_DF0 += static_cast( + std::count_if(trend.begin(), trend.end(), + [](const TMeanVarAccumulator &value) + { return CBasicStatistics::count(value) > 0.0; })); + stats.s_T0[i].reserve(period_); + std::for_each(trend.begin(), trend.end(), + [&stats, i](const TMeanVarAccumulator &value) + { stats.s_T0[i].push_back(CBasicStatistics::mean(value)); }); + } + stats.s_V0 /= static_cast(periods.size()); + } +} + +void CPeriodicityHypothesisTests::conditionOnHypothesis(const TTimeTimePr2Vec &windows, + const STestStats &stats, + TFloatMeanAccumulatorVec &buckets) const +{ + std::size_t n{buckets.size()}; + core_t::TTime windowLength{static_cast(n) * m_BucketLength}; + for (std::size_t i = 0u; i < stats.s_Partition.size(); ++i) + { + TTimeTimePr2Vec windows_(calculateWindows(stats.s_StartOfPartition, + windowLength, + length(stats.s_Partition), + stats.s_Partition[i])); + TSizeSizePr2Vec indexWindows; + calculateIndexWindows(windows_, m_BucketLength, indexWindows); + + std::size_t period{stats.s_T0[i].size()}; + LOG_TRACE("Conditioning on period = " << period + << " in windows = " << core::CContainerPrinter::print(windows_)); + for (const auto &window : indexWindows) + { + std::size_t a{window.first}; + std::size_t b{window.second}; + for (std::size_t j = a; j < b; ++j) + { + CBasicStatistics::moment<0>(buckets[j % n]) -= stats.s_T0[i][(j - a) % period]; + } + } + } + + if (length(windows) < windowLength) + { + LOG_TRACE("Projecting onto " << core::CContainerPrinter::print(windows)); + TFloatMeanAccumulatorVec projection; + project(buckets, windows, m_BucketLength, projection); + buckets = std::move(projection); + LOG_TRACE("# values = " << buckets.size()); + } +} + +bool CPeriodicityHypothesisTests::testPeriod(const TTimeTimePr2Vec &windows, + const TFloatMeanAccumulatorCRng &buckets, + core_t::TTime period_, + STestStats &stats) const +{ + // We use two tests to check for the period: + // 1) That it explains both a non-negligible absolute and statistically + // significant amount of variance and the cyclic autocorrelation at + // that repeat is high enough OR + // 2) There is a large absolute and statistically significant periodic + // spike or trough. + + LOG_TRACE("Testing period " << period_); + + if (!this->testStatisticsFor(buckets, stats) || stats.nullHypothesisGoodEnough()) + { + return false; + } + + period_ = std::min(period_, length(windows[0])); + std::size_t period{static_cast(period_ / m_BucketLength)}; + + // We need to observe a minimum number of repeated values to test with + // an acceptable false positive rate. + double repeats{0.0}; + for (std::size_t i = 0u; i < period; ++i) + { + for (std::size_t j = i + period; j < buckets.size(); j += period) + { + if ( CBasicStatistics::count(buckets[j]) + * CBasicStatistics::count(buckets[j - period]) > 0.0) + { + repeats += 1.0; + break; + } + } + } + LOG_TRACE(" repeated values = " << repeats); + if (repeats < static_cast(period) * ACCURATE_TEST_POPULATED_FRACTION / 3.0) + { + return false; + } + if (stats.s_HasPeriod) + { + return true; + } + + TTimeTimePr2Vec window{{0, length(windows)}}; + double M{stats.s_M}; + double B{stats.s_B}; + double scale{1.0 / M}; + double df0{B - stats.s_DF0}; + double v0{varianceAtPercentile(stats.s_V0, df0, 50.0 + CONFIDENCE_INTERVAL / 2.0)}; + double vt{stats.s_Vt * v0}; + double at{stats.s_At * std::sqrt(v0 / scale)}; + LOG_TRACE(" M = " << M); + + TFloatMeanAccumulatorVec values(buckets.begin(), buckets.end()); + this->conditionOnHypothesis(window, stats, values); + + // The variance test. + + TMeanVarAccumulatorVec trend(period); + periodicTrend(values, window, m_BucketLength, trend); + double b{static_cast( + std::count_if(trend.begin(), trend.end(), + [](const TMeanVarAccumulator &value) + { return CBasicStatistics::count(value) > 0.0; }))}; + LOG_TRACE(" populated = " << b); + + double df1{B - b}; + double v1{varianceAtPercentile(residualVariance(trend, scale), df1, + 50.0 + CONFIDENCE_INTERVAL / 2.0)}; + LOG_TRACE(" variance = " << v1); + LOG_TRACE(" varianceThreshold = " << vt); + LOG_TRACE(" significance = " << CStatisticalTests::leftTailFTest(v1 / v0, df1, df0)); + + double Rt{stats.s_Rt * CTools::truncate(1.0 - 0.5 * (vt - v1) / vt, 0.9, 1.0)}; + if (v1 < vt && CStatisticalTests::leftTailFTest(v1 / v0, df1, df0) <= MAXIMUM_SIGNIFICANCE) + { + double R{CSignal::autocorrelation(period, values)}; + R = autocorrelationAtPercentile(R, B, 50.0 - CONFIDENCE_INTERVAL / 2.0); + LOG_TRACE(" autocorrelation = " << R); + LOG_TRACE(" autocorrelationThreshold = " << Rt); + if (R > Rt) + { + return true; + } + } + + // The amplitude test. + + double F1{1.0}; + if (v1 > 0.0) + { + try + { + std::size_t n{static_cast( + std::ceil(Rt * static_cast(length(window) / period_)))}; + TMeanAccumulator level; + for (const auto &value : values) + { + if (CBasicStatistics::count(value) > 0.0) + { + level.add(CBasicStatistics::mean(value)); + } + } + TMinAmplitudeVec amplitudes(period, {n, CBasicStatistics::mean(level)}); + periodicTrend(values, window, m_BucketLength, amplitudes); + boost::math::normal normal(0.0, std::sqrt(v1)); + std::for_each(amplitudes.begin(), amplitudes.end(), + [&F1, &normal, at](CMinAmplitude &x) + { + if (x.amplitude() >= at) + { + F1 = std::min(F1, x.significance(normal)); + } + }); + } + catch (const std::exception &e) + { + LOG_ERROR("Unable to compute significance of amplitude: " << e.what()); + } + } + LOG_TRACE(" F(amplitude) = " << F1); + + if (1.0 - std::pow(1.0 - F1, b) <= MAXIMUM_SIGNIFICANCE) + { + return true; + } + return false; +} + +bool CPeriodicityHypothesisTests::testPartition(const TTimeTimePr2Vec &partition, + const TFloatMeanAccumulatorCRng &buckets, + core_t::TTime period_, + double correction, + STestStats &stats) const +{ + using TDoubleTimePr = std::pair; + using TDoubleTimePrVec = std::vector; + using TMinAccumulator = CBasicStatistics::COrderStatisticsStack; + using TMeanVarAccumulatorBuffer = boost::circular_buffer; + + LOG_TRACE("Testing partition " << core::CContainerPrinter::print(partition) + << " with period " << period_); + + if (!this->testStatisticsFor(buckets, stats) || stats.nullHypothesisGoodEnough()) + { + return false; + } + if (stats.s_HasPartition) + { + return true; + } + + // Find the partition of the data such that the residual variance + // w.r.t. the period is minimized and check if there is significant + // evidence that it reduces the residual variance and repeats. + + core_t::TTime windowLength{length(buckets, m_BucketLength)}; + std::size_t period{static_cast(period_ / m_BucketLength)}; + core_t::TTime repeat{length(partition)}; + core_t::TTime startOfPartition{stats.s_StartOfPartition}; + double B{stats.s_B}; + double scale{1.0 / stats.s_M}; + double df0{B - stats.s_DF0}; + double v0{varianceAtPercentile(stats.s_V0, df0, 50.0 + CONFIDENCE_INTERVAL / 2.0)}; + double vt{stats.s_Vt * v0}; + LOG_TRACE("period = " << period); + LOG_TRACE("scale = " << scale); + + TFloatMeanAccumulatorVec values(buckets.begin(), buckets.end()); + this->conditionOnHypothesis({{0, windowLength}}, stats, values); + + TTimeTimePr2Vec windows[]{calculateWindows(startOfPartition, windowLength, repeat, partition[0]), + calculateWindows(startOfPartition, windowLength, repeat, partition[1])}; + LOG_TRACE("windows = " << core::CContainerPrinter::print(windows)); + + TTimeVec deltas[2]; + deltas[0].reserve((length(partition[0]) * windowLength) / (period_ * repeat)); + deltas[1].reserve((length(partition[1]) * windowLength) / (period_ * repeat)); + for (std::size_t i = 0u; i < 2; ++i) + { + for (const auto &window : windows[i]) + { + core_t::TTime a_{window.first}; + core_t::TTime b_{window.second}; + for (core_t::TTime t = a_ + period_; t <= b_; t += period_) + { + deltas[i].push_back(t - m_BucketLength); + } + } + } + LOG_TRACE("deltas = " << core::CContainerPrinter::print(deltas)); + + TMeanVarAccumulatorBuffer trends[] + { + TMeanVarAccumulatorBuffer(period, TMeanVarAccumulator()), + TMeanVarAccumulatorBuffer(period, TMeanVarAccumulator()) + }; + periodicTrend(values, windows[0], m_BucketLength, trends[0]); + periodicTrend(values, windows[1], m_BucketLength, trends[1]); + + TMeanAccumulator variances[] + { + residualVariance(trends[0], scale), + residualVariance(trends[1], scale) + }; + LOG_TRACE("variances = " << core::CContainerPrinter::print(variances)); + + TMinAccumulator minimum; + minimum.add({( residualVariance(variances[0]) + + residualVariance(variances[1])) / 2.0, 0}); + + TDoubleTimePrVec candidates; + candidates.reserve(period); + for (core_t::TTime time = m_BucketLength; + time < repeat; + time += m_BucketLength) + { + for (std::size_t i = 0u; i < 2; ++i) + { + for (auto &&delta : deltas[i]) + { + delta = (delta + m_BucketLength) % windowLength; + } + TMeanVarAccumulator oldBucket{trends[i].front()}; + TMeanVarAccumulator newBucket; + averageValue(values, deltas[i], m_BucketLength, newBucket); + + trends[i].pop_front(); + trends[i].push_back(newBucket); + variances[i] -= residualVariance(oldBucket, scale); + variances[i] += residualVariance(newBucket, scale); + } + double variance{( residualVariance(variances[0]) + + residualVariance(variances[1])) / 2.0}; + minimum.add({variance, time}); + if (variance <= 1.05 * minimum[0].first) + { + candidates.emplace_back(variance, time); + } + } + + double b{0.0}; + TMinAccumulator best; + + TTimeTimePr2Vec candidateWindows; + for (const auto &candidate : candidates) + { + if (candidate.first <= 1.05 * minimum[0].first) + { + core_t::TTime candidateStartOfPartition{candidate.second}; + candidateWindows = calculateWindows(candidateStartOfPartition, + windowLength, + repeat, partition[0]); + TMeanAccumulator cost; + for (const auto &window : candidateWindows) + { + core_t::TTime a_{window.first / m_BucketLength}; + core_t::TTime b_{window.second / m_BucketLength - 1}; + double va{CBasicStatistics::mean(values[a_ % values.size()])}; + double vb{CBasicStatistics::mean(values[b_ % values.size()])}; + cost.add(std::fabs(va) + std::fabs(vb) + std::fabs(vb - va)); + } + if (best.add({CBasicStatistics::mean(cost), candidateStartOfPartition})) + { + b = 0.0; + for (std::size_t i = 0u; i < 2; ++i) + { + candidateWindows = calculateWindows(candidateStartOfPartition, + windowLength, + repeat, partition[i]); + + TMeanVarAccumulatorVec trend(period); + periodicTrend(values, candidateWindows, m_BucketLength, trend); + + b += static_cast( + std::count_if(trend.begin(), trend.end(), + [](const TMeanVarAccumulator &value) + { return CBasicStatistics::count(value) > 0.0; })); + } + } + } + } + + double df1{B - b}; + double variance{correction * minimum[0].first}; + double v1{varianceAtPercentile(variance, df1, 50.0 + CONFIDENCE_INTERVAL / 2.0)}; + LOG_TRACE(" variance = " << v1); + LOG_TRACE(" varianceThreshold = " << vt); + LOG_TRACE(" significance = " << CStatisticalTests::leftTailFTest(v1 / v0, df1, df0)); + + if (v1 <= vt && CStatisticalTests::leftTailFTest(v1 / v0, df1, df0) <= MAXIMUM_SIGNIFICANCE) + { + double R{-1.0}; + double Rt{stats.s_Rt * CTools::truncate(1.0 - 0.5 * (vt - v1) / vt, 0.9, 1.0)}; + + startOfPartition = best[0].second; + windows[0] = calculateWindows(startOfPartition, windowLength, repeat, partition[0]); + windows[1] = calculateWindows(startOfPartition, windowLength, repeat, partition[1]); + for (const auto &windows_ : windows) + { + TFloatMeanAccumulatorVec partitionValues; + project(values, windows_, m_BucketLength, partitionValues); + std::size_t windowLength_(length(windows_[0]) / m_BucketLength); + double BW{std::accumulate(partitionValues.begin(), partitionValues.end(), 0.0, + [](double n, const TFloatMeanAccumulator &value) + { return n + (CBasicStatistics::count(value) > 0.0 ? 1.0 : 0.0); })}; + R = std::max(R, autocorrelationAtPercentile(CSignal::autocorrelation( + windowLength_ + period, partitionValues), + BW, 50.0 - CONFIDENCE_INTERVAL / 2.0)); + LOG_TRACE(" autocorrelation = " << R); + LOG_TRACE(" autocorrelationThreshold = " << Rt); + } + + if (R > Rt) + { + stats.s_StartOfPartition = startOfPartition; + return true; + } + } + return false; +} + +const double CPeriodicityHypothesisTests::ACCURATE_TEST_POPULATED_FRACTION{0.9}; +const double CPeriodicityHypothesisTests::MINIMUM_COEFFICIENT_OF_VARIATION{1e-4}; + +CPeriodicityHypothesisTests::STestStats::STestStats() : + s_HasPeriod(false), s_HasPartition(false), + s_Vt(0.0), s_At(0.0), s_Rt(0.0), + s_Range(0.0), s_B(0.0), s_M(0.0), s_V0(0.0), s_DF0(0.0), + s_StartOfPartition(0) +{} + +void CPeriodicityHypothesisTests::STestStats::setThresholds(double vt, double at, double Rt) +{ + s_Vt = vt; + s_At = at; + s_Rt = Rt; +} + +bool CPeriodicityHypothesisTests::STestStats::nullHypothesisGoodEnough() const +{ + TMeanAccumulator mean; + for (const auto &t : s_T0) + { + mean += std::accumulate(t.begin(), t.end(), TMeanAccumulator(), + [](TMeanAccumulator m, double x) + { + m.add(std::fabs(x)); + return m; + }); + } + return std::sqrt(s_V0) <= MINIMUM_COEFFICIENT_OF_VARIATION * CBasicStatistics::mean(mean); +} + +CPeriodicityHypothesisTests::CNestedHypotheses::CNestedHypotheses(TTestFunc test) : + m_Test(test), m_AlwaysTestNested(false) +{} + +CPeriodicityHypothesisTests::CNestedHypotheses::CBuilder +CPeriodicityHypothesisTests::CNestedHypotheses::null(TTestFunc test) +{ + m_Test = test; + m_AlwaysTestNested = true; + return CBuilder(*this); +} + +CPeriodicityHypothesisTests::CNestedHypotheses & +CPeriodicityHypothesisTests::CNestedHypotheses::addNested(TTestFunc test) +{ + m_Nested.emplace_back(test); + return m_Nested.back(); +} + +CPeriodicityHypothesisTestsResult +CPeriodicityHypothesisTests::CNestedHypotheses::test(STestStats &stats) const +{ + CPeriodicityHypothesisTestsResult result{m_Test(stats)}; + if (m_AlwaysTestNested || result != stats.s_H0) + { + stats.s_H0 = result; + for (const auto &child : m_Nested) + { + CPeriodicityHypothesisTestsResult childResult{child.test(stats)}; + if (result != childResult) + { + return childResult; + } + } + } + + return result; +} + +CPeriodicityHypothesisTests::CNestedHypotheses::CBuilder::CBuilder(CNestedHypotheses &hypothesis) +{ + m_Levels.push_back(&hypothesis); +} + +CPeriodicityHypothesisTests::CNestedHypotheses::CBuilder & +CPeriodicityHypothesisTests::CNestedHypotheses::CBuilder::addNested(TTestFunc test) +{ + m_Levels.push_back(&m_Levels.back()->addNested(test)); + return *this; +} + +CPeriodicityHypothesisTests::CNestedHypotheses::CBuilder & +CPeriodicityHypothesisTests::CNestedHypotheses::CBuilder::addAlternative(TTestFunc test) +{ + m_Levels.pop_back(); + return this->addNested(test); +} + +CPeriodicityHypothesisTests::CNestedHypotheses::CBuilder & +CPeriodicityHypothesisTests::CNestedHypotheses::CBuilder::finishedNested() +{ + m_Levels.pop_back(); + return *this; +} + +namespace +{ + +//! Compute the mean of the autocorrelation for \f${P, 2P, ...}\f$ +//! where \f$P\f$ is \p period. +double meanAutocorrelationForPeriodicOffsets(const TDoubleVec &correlations, + std::size_t window, + std::size_t period) +{ + auto correctForPad = [window](double correlation, std::size_t offset) + { + return correlation * static_cast(window) + / static_cast(window - offset); + }; + TMeanAccumulator result; + for (std::size_t offset = period; offset < correlations.size(); offset += period) + { + result.add(correctForPad(correlations[offset - 1], offset)); + } + return CBasicStatistics::mean(result); +} + +//! Find the single periodic component which explains the most +//! cyclic autocorrelation. +std::size_t mostSignificantPeriodicComponent(TFloatMeanAccumulatorVec values) +{ + using TSizeVec = std::vector; + using TDoubleSizePr = std::pair; + using TMaxAccumulator = CBasicStatistics::COrderStatisticsHeap>; + using TFloatMeanAccumulatorCRng = core::CVectorRange; + + std::size_t n{values.size()}; + std::size_t pad{n / 3}; + + // Compute the serial autocorrelations padding to the maximum offset + // to avoid windowing effects. + TDoubleVec correlations; + values.resize(n + pad); + CSignal::autocorrelations(values, correlations); + values.resize(n); + + // We retain the top 15 serial autocorrelations so we have a high + // chance of finding the highest cyclic autocorrelation. Note, we + // average over offsets which are integer multiples of the period + // since these should have high autocorrelation if the signal is + // periodic. + TMaxAccumulator candidates(15); + correlations.resize(pad); + for (std::size_t p = 4u; p < correlations.size(); ++p) + { + double correlation{meanAutocorrelationForPeriodicOffsets(correlations, n, p)}; + LOG_TRACE("correlation(" << p << ") = " << correlation); + candidates.add({correlation, p}); + } + + // Sort by decreasing cyclic autocorrelation. + TSizeVec candidatePeriods(15); + std::transform(candidates.begin(), candidates.end(), + candidatePeriods.begin(), + [](const TDoubleSizePr &candidate_) { return candidate_.second; }); + candidates.clear(); + for (const auto period : candidatePeriods) + { + TFloatMeanAccumulatorCRng window(values, 0, period * (values.size() / period)); + candidates.add({CSignal::autocorrelation(period, window), period}); + } + candidates.sort(); + LOG_TRACE("candidate periods = " << candidates.print()); + + // We prefer shorter periods if the decision is close because + // if there is some periodic component with period p in the + // signal it is possible by chance that n * p + eps for n > 1 + // ends up with higher autocorrelation due to additive noise. + std::size_t result{candidates[0].second}; + double cutoff{0.9 * candidates[0].first}; + for (auto i = candidates.begin() + 1; i != candidates.end() && i->first > cutoff; ++i) + { + if (i->second < result && candidates[0].second % i->second == 0) + { + result = i->second; + } + } + + return result; +} + +} + +CPeriodicityHypothesisTestsResult testForPeriods(const CPeriodicityHypothesisTestsConfig &config, + core_t::TTime startTime, + core_t::TTime bucketLength, + const TFloatMeanAccumulatorVec &values) +{ + // Find the single periodic component which explains the + // most cyclic autocorrelation. + std::size_t period_{mostSignificantPeriodicComponent(values)}; + core_t::TTime window{static_cast(values.size()) * bucketLength}; + core_t::TTime period{static_cast(period_) * bucketLength}; + LOG_TRACE("bucket length = " << bucketLength + << ", window = " << window + << ", periods to test = " << period + << ", # values = " << values.size()); + + // Set up the hypothesis tests. + CPeriodicityHypothesisTests test{config}; + test.initialize(bucketLength, window, period); + core_t::TTime time{startTime + bucketLength / 2}; + for (const auto &value : values) + { + test.add(time, CBasicStatistics::mean(value), CBasicStatistics::count(value)); + time += bucketLength; + } + + return test.test(); +} + +} +} diff --git a/lib/maths/CPrior.cc b/lib/maths/CPrior.cc index 7b57d24fb9..f5147c3e35 100644 --- a/lib/maths/CPrior.cc +++ b/lib/maths/CPrior.cc @@ -68,14 +68,12 @@ const std::size_t ADJUST_OFFSET_TRIALS = 20; } CPrior::CPrior(void) : - m_Forecasting(false), m_DataType(maths_t::E_DiscreteData), m_DecayRate(0.0), m_NumberSamples(0) {} CPrior::CPrior(maths_t::EDataType dataType, double decayRate) : - m_Forecasting(false), m_DataType(dataType), m_NumberSamples(0) { @@ -84,22 +82,11 @@ CPrior::CPrior(maths_t::EDataType dataType, double decayRate) : void CPrior::swap(CPrior &other) { - std::swap(m_Forecasting, other.m_Forecasting); std::swap(m_DataType, other.m_DataType); std::swap(m_DecayRate, other.m_DecayRate); std::swap(m_NumberSamples, other.m_NumberSamples); } -void CPrior::forForecasting(void) -{ - m_Forecasting = true; -} - -bool CPrior::isForForecasting(void) const -{ - return m_Forecasting; -} - bool CPrior::isDiscrete(void) const { return m_DataType == maths_t::E_DiscreteData || m_DataType == maths_t::E_IntegerData; @@ -130,13 +117,13 @@ void CPrior::decayRate(double value) detail::setDecayRate(value, FALLBACK_DECAY_RATE, m_DecayRate); } -double CPrior::offsetMargin(void) const +void CPrior::removeModels(CModelFilter &/*filter*/) { - return 0.2; } -void CPrior::removeModels(CModelFilter &/*filter*/) +double CPrior::offsetMargin(void) const { + return 0.0; } void CPrior::addSamples(const TWeightStyleVec &weightStyles, @@ -246,7 +233,6 @@ CPrior::SPlot CPrior::marginalLikelihoodPlot(unsigned int numberPoints, double w uint64_t CPrior::checksum(uint64_t seed) const { - seed = CChecksum::calculate(seed, m_Forecasting); seed = CChecksum::calculate(seed, m_DataType); seed = CChecksum::calculate(seed, m_DecayRate); return CChecksum::calculate(seed, m_NumberSamples); diff --git a/lib/maths/CQuantileSketch.cc b/lib/maths/CQuantileSketch.cc index 43369009b1..9402cf01c6 100644 --- a/lib/maths/CQuantileSketch.cc +++ b/lib/maths/CQuantileSketch.cc @@ -414,10 +414,7 @@ double CQuantileSketch::count(void) const uint64_t CQuantileSketch::checksum(uint64_t seed) const { seed = CChecksum::calculate(seed, m_MaxSize); - for (std::size_t i = 0u; i < m_Knots.size(); ++i) - { - seed = CChecksum::calculate(seed, m_Knots[i]); - } + seed = CChecksum::calculate(seed, m_Knots); return CChecksum::calculate(seed, m_Count); } diff --git a/lib/maths/CRegression.cc b/lib/maths/CRegression.cc index 88a234d5ee..c8b5ce724a 100644 --- a/lib/maths/CRegression.cc +++ b/lib/maths/CRegression.cc @@ -23,7 +23,5 @@ namespace regression_detail { const double CMaxCondition::VALUE = 1e7; } - -const double CRegression::MINIMUM_RANGE_TO_PREDICT = 1.0; } } diff --git a/lib/maths/CSeasonalComponent.cc b/lib/maths/CSeasonalComponent.cc index 2a1ecdddbc..63f3de1e7c 100644 --- a/lib/maths/CSeasonalComponent.cc +++ b/lib/maths/CSeasonalComponent.cc @@ -42,7 +42,7 @@ namespace maths namespace { -typedef maths_t::TDoubleDoublePr TDoubleDoublePr; +using TDoubleDoublePr = maths_t::TDoubleDoublePr; const std::string DECOMPOSITION_COMPONENT_TAG{"a"}; const std::string RNG_TAG{"b"}; @@ -118,7 +118,7 @@ bool CSeasonalComponent::initialized(void) const bool CSeasonalComponent::initialize(core_t::TTime startTime, core_t::TTime endTime, - const TTimeTimePrMeanVarPrVec &values) + const TFloatMeanAccumulatorVec &values) { this->clear(); @@ -217,24 +217,61 @@ double CSeasonalComponent::meanValue(void) const return this->CDecompositionComponent::meanValue(); } -double CSeasonalComponent::differenceFromMean(core_t::TTime time, core_t::TTime shortPeriod) const -{ +double CSeasonalComponent::delta(core_t::TTime time, + core_t::TTime shortPeriod, + double shortPeriodValue) const +{ + using TMinAccumulator = CBasicStatistics::SMin::TAccumulator; + using TMinMaxAccumulator = CBasicStatistics::CMinMax; + + // This is used to adjust how periodic patterns in the trend are + // represented in the case that we have two periodic components + // one of which is a divisor of the other. We are interested in + // two situations: + // 1) The long component has a bias at this time, w.r.t. its + // mean, for all repeats of short component, + // 2) There is a large difference between the values and the + // mean of the long component for all repeats of the short + // period at this time. + // In the first case we can represent the bias using the short + // seasonal component; we prefer to do this since the resolution + // is better. In the second case we have a bad decomposition of + // periodic features at the long period into terms which cancel + // out or reinforce. In this case we want to just represent the + // periodic features in long component. We can achieve this by + // reducing the value in the short seasonal component. + const CSeasonalTime &time_{this->time()}; core_t::TTime longPeriod{time_.period()}; if (longPeriod > shortPeriod && longPeriod % shortPeriod == 0) { - CBasicStatistics::CMinMax minmax; - double mean = this->CDecompositionComponent::meanValue(); + TMinAccumulator min; + TMinMaxAccumulator minmax; + double mean{this->CDecompositionComponent::meanValue()}; for (core_t::TTime t = time; t < time + longPeriod; t += shortPeriod) { if (time_.inWindow(t)) { double difference{CBasicStatistics::mean(this->value(t, 0.0)) - mean}; + min.add(std::fabs(difference)); minmax.add(difference); } } - return minmax.signMargin(); + + if (std::fabs(minmax.signMargin()) > 0.0) + { + return minmax.signMargin(); + } + + // We have a smooth decision boundary for whether to apply + // a delta for the case that the difference from the mean + // is 1/3 of the range. We force the delta to zero for values + // significantly smaller than this. + double scale{CTools::smoothHeaviside(3.0 * min[0] / minmax.range(), 1.0 / 12.0)}; + scale = CTools::truncate(1.002 * scale - 0.001, 0.0, 1.0); + + return -scale * min[0] * CTools::sign(shortPeriodValue); } return 0.0; @@ -257,11 +294,6 @@ double CSeasonalComponent::heteroscedasticity(void) const return this->CDecompositionComponent::heteroscedasticity(); } -double CSeasonalComponent::varianceDueToParameterDrift(core_t::TTime time) const -{ - return this->initialized() ? m_Bucketing.varianceDueToParameterDrift(time) : 0.0; -} - bool CSeasonalComponent::covariances(core_t::TTime time, TMatrix &result) const { result = TMatrix(0.0); @@ -290,9 +322,9 @@ double CSeasonalComponent::slope(void) const return m_Bucketing.slope(); } -bool CSeasonalComponent::sufficientHistoryToPredict(core_t::TTime time) const +bool CSeasonalComponent::slopeAccurate(core_t::TTime time) const { - return m_Bucketing.sufficientHistoryToPredict(time); + return m_Bucketing.slopeAccurate(time); } uint64_t CSeasonalComponent::checksum(uint64_t seed) const diff --git a/lib/maths/CSeasonalComponentAdaptiveBucketing.cc b/lib/maths/CSeasonalComponentAdaptiveBucketing.cc index 0762d59ab0..263319ce51 100644 --- a/lib/maths/CSeasonalComponentAdaptiveBucketing.cc +++ b/lib/maths/CSeasonalComponentAdaptiveBucketing.cc @@ -49,10 +49,9 @@ namespace maths namespace { +using TDoubleMeanVarAccumulator = CBasicStatistics::SSampleMeanVar::TAccumulator; using TRegression = CSeasonalComponentAdaptiveBucketing::TRegression; -const double SUFFICIENT_HISTORY_TO_PREDICT{2.5}; - //! Clear a vector and recover its memory. template void clearAndShrink(std::vector &vector) @@ -61,45 +60,52 @@ void clearAndShrink(std::vector &vector) empty.swap(vector); } -//! Get the predicted value of \p r at \p t. -double predict_(const TRegression &r, double t, double age) +//! Get the gradient of \p r. +double gradient(const TRegression &r) { - return age < SUFFICIENT_HISTORY_TO_PREDICT ? r.mean() : CRegression::predict(r, t); + TRegression::TArray params; + r.parameters(params); + return params[1]; } -const std::string ADAPTIVE_BUCKETING_TAG{"a"}; -const std::string TIME_TAG{"b"}; -const std::string INITIAL_TIME_TAG{"c"}; -const std::string REGRESSION_TAG{"d"}; -const std::string VARIANCES_TAG{"e"}; -const std::string LAST_UPDATES_TAG{"f"}; -const std::string PARAMETER_PROCESS_TAG{"g"}; +// Version 6.3 +const std::string VERSION_6_3_TAG("6.3"); +const std::string ADAPTIVE_BUCKETING_6_3_TAG{"a"}; +const std::string TIME_6_3_TAG{"b"}; +const std::string BUCKETS_6_3_TAG{"e"}; +const std::string REGRESSION_6_3_TAG{"e"}; +const std::string VARIANCE_6_3_TAG{"f"}; +const std::string FIRST_UPDATE_6_3_TAG{"g"}; +const std::string LAST_UPDATE_6_3_TAG{"h"}; +// Version < 6.3 +const std::string ADAPTIVE_BUCKETING_OLD_TAG{"a"}; +const std::string TIME_OLD_TAG{"b"}; +const std::string INITIAL_TIME_OLD_TAG{"c"}; +const std::string REGRESSION_OLD_TAG{"d"}; +const std::string VARIANCES_OLD_TAG{"e"}; +const std::string LAST_UPDATES_OLD_TAG{"f"}; + const std::string EMPTY_STRING; -const core_t::TTime UNSET_LAST_UPDATE{0}; +const core_t::TTime UNSET_TIME{0}; +const double SUFFICIENT_INTERVAL_TO_ESTIMATE_SLOPE{2.5}; } CSeasonalComponentAdaptiveBucketing::CSeasonalComponentAdaptiveBucketing(void) : - CAdaptiveBucketing{0.0, 0.0}, - m_InitialTime{boost::numeric::bounds::lowest()} + CAdaptiveBucketing{0.0, 0.0} {} CSeasonalComponentAdaptiveBucketing::CSeasonalComponentAdaptiveBucketing(const CSeasonalTime &time, double decayRate, double minimumBucketLength) : CAdaptiveBucketing{decayRate, minimumBucketLength}, - m_Time{time.clone()}, - m_InitialTime{boost::numeric::bounds::lowest()} + m_Time{time.clone()} {} CSeasonalComponentAdaptiveBucketing::CSeasonalComponentAdaptiveBucketing(const CSeasonalComponentAdaptiveBucketing &other) : CAdaptiveBucketing(other), m_Time{other.m_Time->clone()}, - m_InitialTime{other.m_InitialTime}, - m_Regressions(other.m_Regressions), - m_Variances(other.m_Variances), - m_LastUpdates(other.m_LastUpdates), - m_ParameterProcess(other.m_ParameterProcess) + m_Buckets(other.m_Buckets) {} CSeasonalComponentAdaptiveBucketing::CSeasonalComponentAdaptiveBucketing(double decayRate, @@ -123,32 +129,20 @@ CSeasonalComponentAdaptiveBucketing::operator=(const CSeasonalComponentAdaptiveB void CSeasonalComponentAdaptiveBucketing::acceptPersistInserter(core::CStatePersistInserter &inserter) const { - inserter.insertLevel(ADAPTIVE_BUCKETING_TAG, + inserter.insertValue(VERSION_6_3_TAG, ""); + inserter.insertLevel(ADAPTIVE_BUCKETING_6_3_TAG, boost::bind(&CAdaptiveBucketing::acceptPersistInserter, static_cast(this), _1)); - inserter.insertLevel(TIME_TAG, boost::bind(&CSeasonalTimeStateSerializer::acceptPersistInserter, - boost::cref(*m_Time), _1)); - inserter.insertValue(INITIAL_TIME_TAG, m_InitialTime); - for (const auto ®ression : m_Regressions) - { - inserter.insertLevel(REGRESSION_TAG, boost::bind(&TRegression::acceptPersistInserter, - ®ression, _1)); - } - inserter.insertValue(VARIANCES_TAG, core::CPersistUtils::toString(m_Variances)); - inserter.insertValue(LAST_UPDATES_TAG, core::CPersistUtils::toString(m_LastUpdates)); - inserter.insertLevel(PARAMETER_PROCESS_TAG, boost::bind(&TRegressionParameterProcess::acceptPersistInserter, - &m_ParameterProcess, _1)); + inserter.insertLevel(TIME_6_3_TAG, boost::bind(&CSeasonalTimeStateSerializer::acceptPersistInserter, + boost::cref(*m_Time), _1)); + core::CPersistUtils::persist(BUCKETS_6_3_TAG, m_Buckets, inserter); } void CSeasonalComponentAdaptiveBucketing::swap(CSeasonalComponentAdaptiveBucketing &other) { this->CAdaptiveBucketing::swap(other); m_Time.swap(other.m_Time); - std::swap(m_InitialTime, other.m_InitialTime); - m_Regressions.swap(other.m_Regressions); - m_Variances.swap(other.m_Variances); - m_LastUpdates.swap(other.m_LastUpdates); - std::swap(m_ParameterProcess, other.m_ParameterProcess); + m_Buckets.swap(other.m_Buckets); } bool CSeasonalComponentAdaptiveBucketing::initialized(void) const @@ -165,12 +159,7 @@ bool CSeasonalComponentAdaptiveBucketing::initialize(std::size_t n) if (this->CAdaptiveBucketing::initialize(a, b, n)) { n = this->size(); - m_Regressions.clear(); - m_Regressions.resize(n); - m_Variances.clear(); - m_Variances.resize(n); - m_LastUpdates.clear(); - m_LastUpdates.resize(n, UNSET_LAST_UPDATE); + m_Buckets.assign(n, SBucket()); return true; } return false; @@ -178,14 +167,16 @@ bool CSeasonalComponentAdaptiveBucketing::initialize(std::size_t n) void CSeasonalComponentAdaptiveBucketing::initialValues(core_t::TTime startTime, core_t::TTime endTime, - const TTimeTimePrMeanVarPrVec &values) + const TFloatMeanAccumulatorVec &values) { if (this->initialized()) { this->shiftOrigin(startTime); - m_InitialTime = m_Time->startOfWindowRepeat(endTime); - this->CAdaptiveBucketing::initialValues(startTime, values); - m_LastUpdates.assign(this->size(), endTime); + if (!values.empty()) + { + this->CAdaptiveBucketing::initialValues(startTime, endTime, values); + this->shiftSlope(-this->slope()); + } } } @@ -197,10 +188,7 @@ std::size_t CSeasonalComponentAdaptiveBucketing::size(void) const void CSeasonalComponentAdaptiveBucketing::clear(void) { this->CAdaptiveBucketing::clear(); - clearAndShrink(m_Regressions); - clearAndShrink(m_Variances); - clearAndShrink(m_LastUpdates); - m_ParameterProcess = TRegressionParameterProcess(); + clearAndShrink(m_Buckets); } void CSeasonalComponentAdaptiveBucketing::shiftOrigin(core_t::TTime time) @@ -209,9 +197,9 @@ void CSeasonalComponentAdaptiveBucketing::shiftOrigin(core_t::TTime time) double shift{m_Time->regression(time)}; if (shift > 0.0) { - for (auto &®ression : m_Regressions) + for (auto &bucket : m_Buckets) { - regression.shiftAbscissa(-shift); + bucket.s_Regression.shiftAbscissa(-shift); } m_Time->regressionOrigin(time); } @@ -219,17 +207,17 @@ void CSeasonalComponentAdaptiveBucketing::shiftOrigin(core_t::TTime time) void CSeasonalComponentAdaptiveBucketing::shiftLevel(double shift) { - for (auto &®ression : m_Regressions) + for (auto &bucket : m_Buckets) { - regression.shiftOrdinate(shift); + bucket.s_Regression.shiftOrdinate(shift); } } void CSeasonalComponentAdaptiveBucketing::shiftSlope(double shift) { - for (auto &®ression : m_Regressions) + for (auto &bucket : m_Buckets) { - regression.shiftGradient(shift); + bucket.s_Regression.shiftGradient(shift); } } @@ -244,37 +232,34 @@ void CSeasonalComponentAdaptiveBucketing::add(core_t::TTime time, return; } - using TVector = CVectorNx1; - this->CAdaptiveBucketing::add(bucket, time, weight); + SBucket &bucket_{m_Buckets[bucket]}; double t{m_Time->regression(time)}; - TRegression ®ression{m_Regressions[bucket]}; + TRegression ®ression{bucket_.s_Regression}; TDoubleMeanVarAccumulator moments = CBasicStatistics::accumulator(regression.count(), prediction, - static_cast(m_Variances[bucket])); + static_cast(bucket_.s_Variance)); moments.add(value, weight * weight); - // Note this condition can change as a result adding the new - // value we need to check before as well. - bool sufficientHistoryBeforeUpdate{regression.sufficientHistoryToPredict()}; - TVector paramsDrift(regression.parameters(t)); - regression.add(t, value, weight); - m_Variances[bucket] = CBasicStatistics::maximumLikelihoodVariance(moments); - - paramsDrift -= TVector(regression.parameters(t)); + bucket_.s_Variance = CBasicStatistics::maximumLikelihoodVariance(moments); - if ( sufficientHistoryBeforeUpdate - && regression.sufficientHistoryToPredict() - && m_LastUpdates[bucket] != UNSET_LAST_UPDATE) + if (m_Time->regressionInterval(bucket_.s_FirstUpdate, + bucket_.s_LastUpdate) < SUFFICIENT_INTERVAL_TO_ESTIMATE_SLOPE) { - double interval{m_Time->regressionInterval(m_LastUpdates[bucket], time)}; - m_ParameterProcess.add(interval, paramsDrift, TVector(weight * interval)); + double delta{regression.predict(t)}; + regression.shiftGradient(-gradient(regression)); + delta -= regression.predict(t); + regression.shiftOrdinate(delta); } - m_LastUpdates[bucket] = time; + + bucket_.s_FirstUpdate = bucket_.s_FirstUpdate == UNSET_TIME ? + time : std::min(bucket_.s_FirstUpdate, time); + bucket_.s_LastUpdate = bucket_.s_LastUpdate == UNSET_TIME ? + time : std::max(bucket_.s_LastUpdate, time); } const CSeasonalTime &CSeasonalComponentAdaptiveBucketing::time(void) const @@ -302,11 +287,10 @@ void CSeasonalComponentAdaptiveBucketing::propagateForwardsByTime(double time, b { double factor{std::exp(-this->CAdaptiveBucketing::decayRate() * time)}; this->CAdaptiveBucketing::age(factor); - for (auto &®ression : m_Regressions) + for (auto &bucket : m_Buckets) { - regression.age(factor, meanRevert); + bucket.s_Regression.age(factor, meanRevert); } - m_ParameterProcess.age(factor); } } @@ -333,8 +317,8 @@ const TRegression *CSeasonalComponentAdaptiveBucketing::regression(core_t::TTime { std::size_t bucket{0}; this->bucket(time, bucket); - bucket = CTools::truncate(bucket, std::size_t(0), m_Regressions.size() - 1); - result = &m_Regressions[bucket]; + bucket = CTools::truncate(bucket, std::size_t(0), m_Buckets.size() - 1); + result = &m_Buckets[bucket].s_Regression; } return result; } @@ -351,48 +335,26 @@ bool CSeasonalComponentAdaptiveBucketing::knots(core_t::TTime time, double CSeasonalComponentAdaptiveBucketing::slope(void) const { CBasicStatistics::CMinMax minmax; - for (const auto ®ression : m_Regressions) + for (const auto &bucket : m_Buckets) { - if (regression.count() > 0.0) + if (bucket.s_Regression.count() > 0.0) { - TRegression::TArray params; - regression.parameters(params); - minmax.add(params[1]); + minmax.add(gradient(bucket.s_Regression)); } } return minmax.initialized() ? minmax.signMargin() : 0.0; } -bool CSeasonalComponentAdaptiveBucketing::sufficientHistoryToPredict(core_t::TTime time) const +bool CSeasonalComponentAdaptiveBucketing::slopeAccurate(core_t::TTime time) const { - return this->bucketingAgeAt(time) >= SUFFICIENT_HISTORY_TO_PREDICT; -} - -double CSeasonalComponentAdaptiveBucketing::varianceDueToParameterDrift(core_t::TTime time) const -{ - double result{0.0}; - if (this->initialized()) - { - core_t::TTime last{*std::max_element(m_LastUpdates.begin(), m_LastUpdates.end())}; - core_t::TTime current{std::max(time - m_Time->period(), last)}; - if (current > last) - { - double interval{m_Time->regressionInterval(last, current)}; - result = m_ParameterProcess.predictionVariance(interval); - } - } - return result; + return this->observedInterval(time) >= SUFFICIENT_INTERVAL_TO_ESTIMATE_SLOPE; } uint64_t CSeasonalComponentAdaptiveBucketing::checksum(uint64_t seed) const { seed = this->CAdaptiveBucketing::checksum(seed); seed = CChecksum::calculate(seed, m_Time); - seed = CChecksum::calculate(seed, m_InitialTime); - seed = CChecksum::calculate(seed, m_Regressions); - seed = CChecksum::calculate(seed, m_Variances); - seed = CChecksum::calculate(seed, m_LastUpdates); - return CChecksum::calculate(seed, m_ParameterProcess); + return CChecksum::calculate(seed, m_Buckets); } void CSeasonalComponentAdaptiveBucketing::debugMemoryUsage(core::CMemoryUsage::TMemoryUsagePtr mem) const @@ -400,18 +362,13 @@ void CSeasonalComponentAdaptiveBucketing::debugMemoryUsage(core::CMemoryUsage::T mem->setName("CSeasonalComponentAdaptiveBucketing"); core::CMemoryDebug::dynamicSize("m_Endpoints", this->CAdaptiveBucketing::endpoints(), mem); core::CMemoryDebug::dynamicSize("m_Centres", this->CAdaptiveBucketing::centres(), mem); - core::CMemoryDebug::dynamicSize("m_Regressions", m_Regressions, mem); - core::CMemoryDebug::dynamicSize("m_Variances", m_Variances, mem); - core::CMemoryDebug::dynamicSize("m_LastUpdates", m_LastUpdates, mem); + core::CMemoryDebug::dynamicSize("m_Buckets", m_Buckets, mem); } std::size_t CSeasonalComponentAdaptiveBucketing::memoryUsage(void) const { - std::size_t mem{this->CAdaptiveBucketing::memoryUsage()}; - mem += core::CMemory::dynamicSize(m_Regressions); - mem += core::CMemory::dynamicSize(m_Variances); - mem += core::CMemory::dynamicSize(m_LastUpdates); - return mem; + return this->CAdaptiveBucketing::memoryUsage() + + core::CMemory::dynamicSize(m_Buckets); } const CSeasonalComponentAdaptiveBucketing::TFloatVec &CSeasonalComponentAdaptiveBucketing::endpoints(void) const @@ -436,33 +393,59 @@ CSeasonalComponentAdaptiveBucketing::TDoubleVec CSeasonalComponentAdaptiveBucket bool CSeasonalComponentAdaptiveBucketing::acceptRestoreTraverser(core::CStateRestoreTraverser &traverser) { - do + if (traverser.name() == VERSION_6_3_TAG) { - const std::string &name{traverser.name()}; - RESTORE(ADAPTIVE_BUCKETING_TAG, traverser.traverseSubLevel( - boost::bind(&CAdaptiveBucketing::acceptRestoreTraverser, - static_cast(this), _1))); - RESTORE(TIME_TAG, traverser.traverseSubLevel( - boost::bind(&CSeasonalTimeStateSerializer::acceptRestoreTraverser, boost::ref(m_Time), _1))) - RESTORE_BUILT_IN(INITIAL_TIME_TAG, m_InitialTime) - RESTORE_SETUP_TEARDOWN(REGRESSION_TAG, - TRegression regression, - traverser.traverseSubLevel(boost::bind(&TRegression::acceptRestoreTraverser, - ®ression, _1)), - m_Regressions.push_back(regression)) - RESTORE(VARIANCES_TAG, core::CPersistUtils::fromString(traverser.value(), m_Variances)) - RESTORE(LAST_UPDATES_TAG, core::CPersistUtils::fromString(traverser.value(), m_LastUpdates)) - RESTORE(PARAMETER_PROCESS_TAG, traverser.traverseSubLevel( - boost::bind(&TRegressionParameterProcess::acceptRestoreTraverser, &m_ParameterProcess, _1))) + while (traverser.next()) + { + const std::string &name{traverser.name()}; + RESTORE(ADAPTIVE_BUCKETING_6_3_TAG, traverser.traverseSubLevel( + boost::bind(&CAdaptiveBucketing::acceptRestoreTraverser, + static_cast(this), _1))); + RESTORE(TIME_6_3_TAG, traverser.traverseSubLevel( + boost::bind(&CSeasonalTimeStateSerializer::acceptRestoreTraverser, boost::ref(m_Time), _1))) + RESTORE(BUCKETS_6_3_TAG, core::CPersistUtils::restore(BUCKETS_6_3_TAG, m_Buckets, traverser)) + } } - while (traverser.next()); - - TRegressionVec(m_Regressions).swap(m_Regressions); - if (m_LastUpdates.empty()) + else { - m_LastUpdates.resize(this->size(), UNSET_LAST_UPDATE); + // There is no version string this is historic state. + + using TTimeVec = std::vector; + using TRegressionVec = std::vector; + + core_t::TTime initialTime; + TRegressionVec regressions; + TFloatVec variances; + TTimeVec lastUpdates; + do + { + const std::string &name{traverser.name()}; + RESTORE(ADAPTIVE_BUCKETING_OLD_TAG, traverser.traverseSubLevel( + boost::bind(&CAdaptiveBucketing::acceptRestoreTraverser, + static_cast(this), _1))); + RESTORE(TIME_OLD_TAG, traverser.traverseSubLevel( + boost::bind(&CSeasonalTimeStateSerializer::acceptRestoreTraverser, boost::ref(m_Time), _1))) + RESTORE_BUILT_IN(INITIAL_TIME_OLD_TAG, initialTime) + RESTORE_SETUP_TEARDOWN(REGRESSION_OLD_TAG, + TRegression regression, + traverser.traverseSubLevel(boost::bind(&TRegression::acceptRestoreTraverser, + ®ression, _1)), + regressions.push_back(regression)) + RESTORE(VARIANCES_OLD_TAG, core::CPersistUtils::fromString(traverser.value(), variances)) + RESTORE(LAST_UPDATES_OLD_TAG, core::CPersistUtils::fromString(traverser.value(), lastUpdates)) + } + while (traverser.next()); + + m_Buckets.clear(); + m_Buckets.reserve(regressions.size()); + for (std::size_t i = 0u; i < regressions.size(); ++i) + { + m_Buckets.emplace_back(regressions[i], variances[i], initialTime, lastUpdates[i]); + } } + m_Buckets.shrink_to_fit(); + return true; } @@ -492,8 +475,9 @@ void CSeasonalComponentAdaptiveBucketing::refresh(const TFloatVec &endpoints) // This might be worth considering at some point. using TDoubleMeanAccumulator = CBasicStatistics::SSampleMean::TAccumulator; + using TMinAccumulator = CBasicStatistics::SMin::TAccumulator; - std::size_t m{m_Regressions.size()}; + std::size_t m{m_Buckets.size()}; std::size_t n{endpoints.size()}; if (m+1 != n) { @@ -504,12 +488,10 @@ void CSeasonalComponentAdaptiveBucketing::refresh(const TFloatVec &endpoints) TFloatVec &m_Endpoints{this->CAdaptiveBucketing::endpoints()}; TFloatVec &m_Centres{this->CAdaptiveBucketing::centres()}; - TRegressionVec regressions; + TBucketVec buckets; TFloatVec centres; - TFloatVec variances; - regressions.reserve(m); + buckets.reserve(m); centres.reserve(m); - variances.reserve(m); for (std::size_t i = 1u; i < n; ++i) { @@ -533,48 +515,63 @@ void CSeasonalComponentAdaptiveBucketing::refresh(const TFloatVec &endpoints) { double interval{m_Endpoints[i] - m_Endpoints[i-1]}; double w{CTools::truncate(interval / (xr - xl), 0.0, 1.0)}; - regressions.push_back(m_Regressions[l-1].scaled(w * w)); + const SBucket &bucket{m_Buckets[l-1]}; + buckets.emplace_back(bucket.s_Regression.scaled(w * w), + bucket.s_Variance, + bucket.s_FirstUpdate, + bucket.s_LastUpdate); centres.push_back(CTools::truncate(static_cast(m_Centres[l-1]), yl, yr)); - variances.push_back(m_Variances[l-1]); } else { double interval{xr - m_Endpoints[i-1]}; double w{CTools::truncate(interval / (xr - xl), 0.0, 1.0)}; - TDoubleRegression regression{m_Regressions[l-1].scaled(w)}; + const SBucket *bucket{&m_Buckets[l-1]}; + TMinAccumulator firstUpdate; + TMinAccumulator lastUpdate; + TDoubleRegression regression{bucket->s_Regression.scaled(w)}; + TDoubleMeanVarAccumulator variance{ + CBasicStatistics::accumulator(w * bucket->s_Regression.count(), + bucket->s_Regression.mean(), + static_cast(bucket->s_Variance))}; + firstUpdate.add(bucket->s_FirstUpdate); + lastUpdate.add(bucket->s_LastUpdate); TDoubleMeanAccumulator centre{ - CBasicStatistics::accumulator(w * m_Regressions[l-1].count(), + CBasicStatistics::accumulator(w * bucket->s_Regression.count(), static_cast(m_Centres[l-1]))}; - TDoubleMeanVarAccumulator variance{ - CBasicStatistics::accumulator(w * m_Regressions[l-1].count(), - m_Regressions[l-1].mean(), - static_cast(m_Variances[l-1]))}; - double count{w * w * m_Regressions[l-1].count()}; + double count{w * w * bucket->s_Regression.count()}; while (++l < r) { - regression += m_Regressions[l-1]; - centre += CBasicStatistics::accumulator(m_Regressions[l-1].count(), + bucket = &m_Buckets[l-1]; + regression += bucket->s_Regression; + variance += CBasicStatistics::accumulator(bucket->s_Regression.count(), + bucket->s_Regression.mean(), + static_cast(bucket->s_Variance)); + firstUpdate.add(bucket->s_FirstUpdate); + lastUpdate.add(bucket->s_LastUpdate); + centre += CBasicStatistics::accumulator(bucket->s_Regression.count(), static_cast(m_Centres[l-1])); - variance += CBasicStatistics::accumulator(m_Regressions[l-1].count(), - m_Regressions[l-1].mean(), - static_cast(m_Variances[l-1])); - count += m_Regressions[l-1].count(); + count += bucket->s_Regression.count(); } xl = endpoints[l-1]; xr = endpoints[l]; + bucket = &m_Buckets[l-1]; interval = m_Endpoints[i] - xl; w = CTools::truncate(interval / (xr - xl), 0.0, 1.0); - regression += m_Regressions[l-1].scaled(w); - centre += CBasicStatistics::accumulator(w * m_Regressions[l-1].count(), + regression += bucket->s_Regression.scaled(w); + variance += CBasicStatistics::accumulator(w * bucket->s_Regression.count(), + bucket->s_Regression.mean(), + static_cast(bucket->s_Variance)); + firstUpdate.add(bucket->s_FirstUpdate); + lastUpdate.add(bucket->s_LastUpdate); + centre += CBasicStatistics::accumulator(w * bucket->s_Regression.count(), static_cast(m_Centres[l-1])); - variance += CBasicStatistics::accumulator(w * m_Regressions[l-1].count(), - m_Regressions[l-1].mean(), - static_cast(m_Variances[l-1])); - count += w * w * m_Regressions[l-1].count(); + count += w * w * bucket->s_Regression.count(); double scale{count == regression.count() ? 1.0 : count / regression.count()}; - regressions.push_back(regression.scaled(scale)); + buckets.emplace_back(regression.scaled(scale), + CBasicStatistics::maximumLikelihoodVariance(variance), + firstUpdate[0], lastUpdate[0]); centres.push_back(CTools::truncate(CBasicStatistics::mean(centre), yl, yr)); - variances.push_back(CBasicStatistics::maximumLikelihoodVariance(variance)); } } @@ -583,51 +580,44 @@ void CSeasonalComponentAdaptiveBucketing::refresh(const TFloatVec &endpoints) // that is equal to the number of points they will receive in one // period. double count{0.0}; - for (const auto ®ression : regressions) + for (const auto &bucket : buckets) { - count += regression.count(); + count += bucket.s_Regression.count(); } count /= (endpoints[m] - endpoints[0]); for (std::size_t i = 0u; i < m; ++i) { - double c{regressions[i].count()}; + double c{buckets[i].s_Regression.count()}; if (c > 0.0) { - regressions[i].scale(count * (endpoints[i+1] - endpoints[i]) / c); + buckets[i].s_Regression.scale(count * (endpoints[i+1] - endpoints[i]) / c); } } LOG_TRACE("old endpoints = " << core::CContainerPrinter::print(endpoints)); - LOG_TRACE("old regressions = " << core::CContainerPrinter::print(m_Regressions)); LOG_TRACE("old centres = " << core::CContainerPrinter::print(m_Centres)); - LOG_TRACE("old variances = " << core::CContainerPrinter::print(m_Variances)); LOG_TRACE("new endpoints = " << core::CContainerPrinter::print(m_Endpoints)); - LOG_TRACE("new regressions = " << core::CContainerPrinter::print(regressions)); LOG_TRACE("new centres = " << core::CContainerPrinter::print(centres)); - LOG_TRACE("new variances = " << core::CContainerPrinter::print(variances)); - m_Regressions.swap(regressions); + m_Buckets.swap(buckets); m_Centres.swap(centres); - m_Variances.swap(variances); } -void CSeasonalComponentAdaptiveBucketing::add(std::size_t bucket, - core_t::TTime time, - double offset, - const TDoubleMeanVarAccumulator &value) +bool CSeasonalComponentAdaptiveBucketing::inWindow(core_t::TTime time) const { - TRegression ®ression{m_Regressions[bucket]}; - CFloatStorage &variance{m_Variances[bucket]}; + return m_Time->inWindow(time); +} - core_t::TTime tk{time + (m_Time->windowStart() + static_cast(offset + 0.5)) - % m_Time->windowRepeat()}; +void CSeasonalComponentAdaptiveBucketing::add(std::size_t bucket, core_t::TTime time, double value, double weight) +{ + SBucket &bucket_{m_Buckets[bucket]}; + TRegression ®ression{bucket_.s_Regression}; + CFloatStorage &variance{bucket_.s_Variance}; TDoubleMeanVarAccumulator variance_{ CBasicStatistics::accumulator(regression.count(), regression.mean(), - static_cast(variance)) + value}; - - regression.add(m_Time->regression(tk), - CBasicStatistics::mean(value), - CBasicStatistics::count(value)); + static_cast(variance))}; + variance_.add(value, weight); + regression.add(m_Time->regression(time), value, weight); variance = CBasicStatistics::maximumLikelihoodVariance(variance_); } @@ -638,23 +628,97 @@ double CSeasonalComponentAdaptiveBucketing::offset(core_t::TTime time) const double CSeasonalComponentAdaptiveBucketing::count(std::size_t bucket) const { - return m_Regressions[bucket].count(); + return m_Buckets[bucket].s_Regression.count(); } double CSeasonalComponentAdaptiveBucketing::predict(std::size_t bucket, core_t::TTime time, double offset) const { + const SBucket &bucket_{m_Buckets[bucket]}; + core_t::TTime firstUpdate{bucket_.s_FirstUpdate}; + core_t::TTime lastUpdate{bucket_.s_LastUpdate}; + const TRegression ®ression{bucket_.s_Regression}; + + double interval{static_cast(lastUpdate - firstUpdate)}; + if (interval == 0) + { + return regression.mean(); + } + double t{m_Time->regression(time + static_cast(offset + 0.5))}; - return predict_(m_Regressions[bucket], t, this->bucketingAgeAt(time)); + + double extrapolateInterval{static_cast( + CBasicStatistics::max(time - lastUpdate, firstUpdate - time, core_t::TTime(0)))}; + if (extrapolateInterval == 0.0) + { + return regression.predict(t); + } + + // We mean revert our predictions if trying to predict much further + // ahead than the observed interval for the data. + double alpha{CTools::smoothHeaviside(extrapolateInterval / interval, 1.0 / 12.0, -1.0)}; + double beta{1.0 - alpha}; + return alpha * regression.predict(t) + beta * regression.mean(); } double CSeasonalComponentAdaptiveBucketing::variance(std::size_t bucket) const { - return m_Variances[bucket]; + return m_Buckets[bucket].s_Variance; +} + +double CSeasonalComponentAdaptiveBucketing::observedInterval(core_t::TTime time) const +{ + return m_Time->regressionInterval(std::min_element( + m_Buckets.begin(), m_Buckets.end(), + [](const SBucket &lhs, const SBucket &rhs) + { return lhs.s_FirstUpdate < rhs.s_FirstUpdate; })->s_FirstUpdate, time); +} + +CSeasonalComponentAdaptiveBucketing::SBucket::SBucket(void) : + s_Variance{0.0}, + s_FirstUpdate{UNSET_TIME}, + s_LastUpdate{UNSET_TIME} +{} + +CSeasonalComponentAdaptiveBucketing::SBucket::SBucket(const TRegression ®ression, + double variance, + core_t::TTime firstUpdate, + core_t::TTime lastUpdate) : + s_Regression{regression}, + s_Variance{variance}, + s_FirstUpdate{firstUpdate}, + s_LastUpdate{lastUpdate} +{} + +bool CSeasonalComponentAdaptiveBucketing::SBucket::acceptRestoreTraverser(core::CStateRestoreTraverser &traverser) +{ + do + { + const std::string &name{traverser.name()}; + RESTORE(REGRESSION_6_3_TAG, traverser.traverseSubLevel(boost::bind(&TRegression::acceptRestoreTraverser, + &s_Regression, _1))) + RESTORE(VARIANCE_6_3_TAG, s_Variance.fromString(traverser.value())) + RESTORE_BUILT_IN(FIRST_UPDATE_6_3_TAG, s_FirstUpdate) + RESTORE_BUILT_IN(LAST_UPDATE_6_3_TAG, s_LastUpdate) + } + while (traverser.next()); + return true; +} + +void CSeasonalComponentAdaptiveBucketing::SBucket::acceptPersistInserter(core::CStatePersistInserter &inserter) const +{ + inserter.insertLevel(REGRESSION_6_3_TAG, boost::bind(&TRegression::acceptPersistInserter, + &s_Regression, _1)); + inserter.insertValue(VARIANCE_6_3_TAG, s_Variance.toString()); + inserter.insertValue(FIRST_UPDATE_6_3_TAG, s_FirstUpdate); + inserter.insertValue(LAST_UPDATE_6_3_TAG, s_LastUpdate); } -double CSeasonalComponentAdaptiveBucketing::bucketingAgeAt(core_t::TTime time) const +uint64_t CSeasonalComponentAdaptiveBucketing::SBucket::checksum(uint64_t seed) const { - return static_cast(time - m_InitialTime) / static_cast(core::constants::WEEK); + seed = CChecksum::calculate(seed, s_Regression); + seed = CChecksum::calculate(seed, s_Variance); + seed = CChecksum::calculate(seed, s_FirstUpdate); + return CChecksum::calculate(seed, s_LastUpdate); } } diff --git a/lib/maths/CSeasonalTime.cc b/lib/maths/CSeasonalTime.cc index 8fddd27f30..54eed35ea2 100644 --- a/lib/maths/CSeasonalTime.cc +++ b/lib/maths/CSeasonalTime.cc @@ -44,14 +44,18 @@ const std::string ARBITRARY_PERIOD_TIME_TAG("b"); //////// CSeasonalTime //////// CSeasonalTime::CSeasonalTime(void) : - m_Period(0), m_RegressionOrigin(0) + m_Period(0), m_RegressionOrigin(0), m_Precedence(0) {} -CSeasonalTime::CSeasonalTime(core_t::TTime period) : - m_Period(period), m_RegressionOrigin(0) +CSeasonalTime::CSeasonalTime(core_t::TTime period, double precedence) : + m_Period(period), m_RegressionOrigin(0), m_Precedence(precedence) {} -CSeasonalTime::~CSeasonalTime(void) {} +bool CSeasonalTime::operator<(const CSeasonalTime &rhs) const +{ + return COrderings::lexicographical_compare(m_Period, -m_Precedence, + rhs.m_Period, -rhs.m_Precedence); +} double CSeasonalTime::periodic(core_t::TTime time) const { @@ -127,11 +131,10 @@ double CSeasonalTime::fractionInWindow(void) const / static_cast(this->windowRepeat()); } -double CSeasonalTime::scaleDecayRate(double decayRate, - core_t::TTime fromPeriod, - core_t::TTime toPeriod) +bool CSeasonalTime::excludes(const CSeasonalTime &other) const { - return static_cast(fromPeriod) / static_cast(toPeriod) * decayRate; + return std::abs(other.m_Period - m_Period) < std::max(other.m_Period, m_Period) / 20 + && m_Precedence >= other.m_Precedence; } core_t::TTime CSeasonalTime::startOfWindowRepeat(core_t::TTime offset, core_t::TTime time) const @@ -148,8 +151,9 @@ CDiurnalTime::CDiurnalTime(void) : CDiurnalTime::CDiurnalTime(core_t::TTime startOfWeek, core_t::TTime windowStart, core_t::TTime windowEnd, - core_t::TTime period) : - CSeasonalTime(period), + core_t::TTime period, + double precedence) : + CSeasonalTime(period, precedence), m_StartOfWeek(startOfWeek), m_WindowStart(windowStart), m_WindowEnd(windowEnd) @@ -206,6 +210,12 @@ core_t::TTime CDiurnalTime::windowEnd(void) const return m_WindowEnd; } +bool CDiurnalTime::hasWeekend(void) const +{ + return this->windowLength() == core::constants::WEEKEND + || this->windowLength() == core::constants::WEEKDAYS; +} + uint64_t CDiurnalTime::checksum(uint64_t seed) const { seed = CChecksum::calculate(seed, m_StartOfWeek); @@ -221,10 +231,8 @@ core_t::TTime CDiurnalTime::regressionTimeScale(void) const //////// CGeneralPeriodTime //////// -CGeneralPeriodTime::CGeneralPeriodTime(void) {} - -CGeneralPeriodTime::CGeneralPeriodTime(core_t::TTime period) : - CSeasonalTime(period) +CGeneralPeriodTime::CGeneralPeriodTime(core_t::TTime period, double precedence) : + CSeasonalTime(period, precedence) {} CGeneralPeriodTime *CGeneralPeriodTime::clone(void) const @@ -272,6 +280,11 @@ core_t::TTime CGeneralPeriodTime::windowEnd(void) const return this->period(); } +bool CGeneralPeriodTime::hasWeekend(void) const +{ + return false; +} + uint64_t CGeneralPeriodTime::checksum(uint64_t seed) const { return CChecksum::calculate(seed, this->period()); diff --git a/lib/maths/CSignal.cc b/lib/maths/CSignal.cc index 0264cca34c..01d7fd6f7e 100644 --- a/lib/maths/CSignal.cc +++ b/lib/maths/CSignal.cc @@ -181,8 +181,9 @@ double CSignal::autocorrelation(std::size_t offset, TFloatMeanAccumulatorCRng va } } - TMeanAccumulator autocorrelation; double mean = CBasicStatistics::mean(moments); + + TMeanAccumulator autocorrelation; for (std::size_t i = 0u; i < values.size(); ++i) { std::size_t j = (i + offset) % n; @@ -195,8 +196,10 @@ double CSignal::autocorrelation(std::size_t offset, TFloatMeanAccumulatorCRng va } } - return CBasicStatistics::mean(autocorrelation) == CBasicStatistics::variance(moments) ? - 1.0 : CBasicStatistics::mean(autocorrelation) / CBasicStatistics::variance(moments); + double a = CBasicStatistics::mean(autocorrelation); + double v = CBasicStatistics::maximumLikelihoodVariance(moments); + + return a == v ? 1.0 : a / v; } void CSignal::autocorrelations(const TFloatMeanAccumulatorVec &values, TDoubleVec &result) @@ -217,7 +220,7 @@ void CSignal::autocorrelations(const TFloatMeanAccumulatorVec &values, TDoubleVe } } double mean = CBasicStatistics::mean(moments); - double variance = CBasicStatistics::variance(moments); + double variance = CBasicStatistics::maximumLikelihoodVariance(moments); TComplexVec f; f.reserve(n); diff --git a/lib/maths/CStatisticalTests.cc b/lib/maths/CStatisticalTests.cc index 3bece30943..eebb113cdb 100644 --- a/lib/maths/CStatisticalTests.cc +++ b/lib/maths/CStatisticalTests.cc @@ -84,6 +84,14 @@ const std::string EMPTY_STRING; double CStatisticalTests::leftTailFTest(double x, double d1, double d2) { + if (x < 0.0) + { + return 0.0; + } + if (boost::math::isinf(x)) + { + return 1.0; + } try { boost::math::fisher_f_distribution<> F(d1, d2); @@ -99,6 +107,14 @@ double CStatisticalTests::leftTailFTest(double x, double d1, double d2) double CStatisticalTests::rightTailFTest(double x, double d1, double d2) { + if (x < 0.0) + { + return 1.0; + } + if (boost::math::isinf(x)) + { + return 0.0; + } try { boost::math::fisher_f_distribution<> F(d1, d2); diff --git a/lib/maths/CTimeSeriesDecomposition.cc b/lib/maths/CTimeSeriesDecomposition.cc index 733549929e..0fcc8f4a3f 100644 --- a/lib/maths/CTimeSeriesDecomposition.cc +++ b/lib/maths/CTimeSeriesDecomposition.cc @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -63,54 +64,45 @@ TVector2x1 vector2x1(const TDoubleDoublePr &p) //! Convert a 2x1 vector to a double pair. TDoubleDoublePr pair(const TVector2x1 &v) { - return std::make_pair(v(0), v(1)); + return {v(0), v(1)}; } -//! Update the confidence interval to reflect the initial errors -//! in the regression coefficients. -template -void forecastErrors(double varianceDueToParameterDrift, - const CSymmetricMatrixNxN &m, - double dt, double confidence, TVector2x1 &result) +//! Get the normal confidence interval. +TDoubleDoublePr confidenceInterval(double confidence, double variance) { - double variance{varianceDueToParameterDrift}; - double ti{dt}; - for (std::size_t i = 1; dt > 0.0 && i < m.rows(); ++i, ti *= dt) + if (variance > 0.0) { - if (m(i,i) == 0.0) + try { - LOG_TRACE("Ignoring t^" << i << " as variance of coefficient is zero"); - continue; - } - variance += m(i,i) * ti * ti; - } - try - { - if (variance > 0.0) - { - boost::math::normal normal{0.0, std::sqrt(variance)}; - result(0) += boost::math::quantile(normal, (100.0 - confidence) / 200.0); - result(1) += boost::math::quantile(normal, (100.0 + confidence) / 200.0); + boost::math::normal normal(0.0, std::sqrt(variance)); + double ql{boost::math::quantile(normal, (100.0 - confidence) / 200.0)}; + double qu{boost::math::quantile(normal, (100.0 + confidence) / 200.0)}; + return {ql, qu}; } - else + catch (const std::exception &e) { - result(0) = result(1) = 0.0; + LOG_ERROR("Failed calculating confidence interval: " << e.what() + << ", variance = " << variance + << ", confidence = " << confidence); } } - catch (const std::exception &e) - { - LOG_ERROR("Unable to compute error: " << e.what()); - } + return {0.0, 0.0}; } -const std::string DECAY_RATE_TAG{"a"}; -const std::string LAST_VALUE_TIME_TAG{"b"}; -const std::string LONG_TERM_TREND_TEST_TAG{"c"}; -const std::string DIURNAL_TEST_TAG{"d"}; -const std::string GENERAL_SEASONALITY_TEST_TAG{"e"}; -const std::string CALENDAR_CYCLIC_TEST_TAG{"f"}; -const std::string SEASONAL_COMPONENTS_TAG{"g"}; -const std::string LAST_PROPAGATION_TIME_TAG{"h"}; +// Version 6.3 +const std::string VERSION_6_3_TAG("6.3"); +const std::string LAST_VALUE_TIME_6_3_TAG{"a"}; +const std::string LAST_PROPAGATION_TIME_6_3_TAG{"b"}; +const std::string PERIODICITY_TEST_6_3_TAG{"c"}; +const std::string CALENDAR_CYCLIC_TEST_6_3_TAG{"d"}; +const std::string COMPONENTS_6_3_TAG{"e"}; +// Version < 6.3 +const std::string DECAY_RATE_OLD_TAG{"a"}; +const std::string LAST_VALUE_TIME_OLD_TAG{"b"}; +const std::string CALENDAR_CYCLIC_TEST_OLD_TAG{"f"}; +const std::string COMPONENTS_OLD_TAG{"g"}; +const std::string LAST_PROPAGATION_TIME_OLD_TAG{"h"}; + const std::string EMPTY_STRING; } @@ -118,12 +110,9 @@ const std::string EMPTY_STRING; CTimeSeriesDecomposition::CTimeSeriesDecomposition(double decayRate, core_t::TTime bucketLength, std::size_t seasonalComponentSize) : - m_DecayRate{decayRate}, m_LastValueTime{0}, m_LastPropagationTime{0}, - m_LongTermTrendTest{decayRate}, - m_DiurnalTest{decayRate, bucketLength}, - m_GeneralSeasonalityTest{decayRate, bucketLength}, + m_PeriodicityTest{decayRate, bucketLength}, m_CalendarCyclicTest{decayRate, bucketLength}, m_Components{decayRate, bucketLength, seasonalComponentSize} { @@ -134,12 +123,9 @@ CTimeSeriesDecomposition::CTimeSeriesDecomposition(double decayRate, core_t::TTime bucketLength, std::size_t seasonalComponentSize, core::CStateRestoreTraverser &traverser) : - m_DecayRate{decayRate}, m_LastValueTime{0}, m_LastPropagationTime{0}, - m_LongTermTrendTest{decayRate}, - m_DiurnalTest{decayRate, bucketLength}, - m_GeneralSeasonalityTest{decayRate, bucketLength}, + m_PeriodicityTest{decayRate, bucketLength}, m_CalendarCyclicTest{decayRate, bucketLength}, m_Components{decayRate, bucketLength, seasonalComponentSize} { @@ -148,12 +134,9 @@ CTimeSeriesDecomposition::CTimeSeriesDecomposition(double decayRate, } CTimeSeriesDecomposition::CTimeSeriesDecomposition(const CTimeSeriesDecomposition &other) : - m_DecayRate{other.m_DecayRate}, m_LastValueTime{other.m_LastValueTime}, m_LastPropagationTime{other.m_LastPropagationTime}, - m_LongTermTrendTest{other.m_LongTermTrendTest}, - m_DiurnalTest{other.m_DiurnalTest}, - m_GeneralSeasonalityTest{other.m_GeneralSeasonalityTest}, + m_PeriodicityTest{other.m_PeriodicityTest}, m_CalendarCyclicTest{other.m_CalendarCyclicTest}, m_Components{other.m_Components} { @@ -162,48 +145,52 @@ CTimeSeriesDecomposition::CTimeSeriesDecomposition(const CTimeSeriesDecompositio bool CTimeSeriesDecomposition::acceptRestoreTraverser(core::CStateRestoreTraverser &traverser) { - do + if (traverser.name() == VERSION_6_3_TAG) { - const std::string &name{traverser.name()}; - RESTORE_BUILT_IN(DECAY_RATE_TAG, m_DecayRate); - RESTORE_BUILT_IN(LAST_VALUE_TIME_TAG, m_LastValueTime) - RESTORE_BUILT_IN(LAST_PROPAGATION_TIME_TAG, m_LastPropagationTime) - RESTORE(LONG_TERM_TREND_TEST_TAG, traverser.traverseSubLevel( - boost::bind(&CLongTermTrendTest::acceptRestoreTraverser, - &m_LongTermTrendTest, _1))); - RESTORE(DIURNAL_TEST_TAG, traverser.traverseSubLevel( - boost::bind(&CDiurnalTest::acceptRestoreTraverser, - &m_DiurnalTest, _1))); - RESTORE(GENERAL_SEASONALITY_TEST_TAG, traverser.traverseSubLevel( - boost::bind(&CNonDiurnalTest::acceptRestoreTraverser, - &m_GeneralSeasonalityTest, _1))) - RESTORE(CALENDAR_CYCLIC_TEST_TAG, traverser.traverseSubLevel( - boost::bind(&CCalendarTest::acceptRestoreTraverser, - &m_CalendarCyclicTest, _1))) - RESTORE(SEASONAL_COMPONENTS_TAG, traverser.traverseSubLevel( - boost::bind(&CComponents::acceptRestoreTraverser, - &m_Components, _1))) + while (traverser.next()) + { + const std::string &name{traverser.name()}; + RESTORE_BUILT_IN(LAST_VALUE_TIME_6_3_TAG, m_LastValueTime) + RESTORE_BUILT_IN(LAST_PROPAGATION_TIME_6_3_TAG, m_LastPropagationTime) + RESTORE(PERIODICITY_TEST_6_3_TAG, traverser.traverseSubLevel( + boost::bind(&CPeriodicityTest::acceptRestoreTraverser, + &m_PeriodicityTest, _1))) + RESTORE(CALENDAR_CYCLIC_TEST_6_3_TAG, traverser.traverseSubLevel( + boost::bind(&CCalendarTest::acceptRestoreTraverser, + &m_CalendarCyclicTest, _1))) + RESTORE(COMPONENTS_6_3_TAG, traverser.traverseSubLevel( + boost::bind(&CComponents::acceptRestoreTraverser, + &m_Components, _1))) + } } - while (traverser.next()); - - m_LongTermTrendTest.decayRate(m_DecayRate); - m_Components.decayRate(m_DecayRate); - if (m_LastPropagationTime == 0) + else { - m_LastPropagationTime = m_LastValueTime; + // There is no version string this is historic state. + double decayRate{0.012}; + do + { + const std::string &name{traverser.name()}; + RESTORE_BUILT_IN(DECAY_RATE_OLD_TAG, decayRate) + RESTORE_BUILT_IN(LAST_VALUE_TIME_OLD_TAG, m_LastValueTime) + RESTORE_BUILT_IN(LAST_PROPAGATION_TIME_OLD_TAG, m_LastPropagationTime) + RESTORE(CALENDAR_CYCLIC_TEST_OLD_TAG, traverser.traverseSubLevel( + boost::bind(&CCalendarTest::acceptRestoreTraverser, + &m_CalendarCyclicTest, _1))) + RESTORE(COMPONENTS_OLD_TAG, traverser.traverseSubLevel( + boost::bind(&CComponents::acceptRestoreTraverser, + &m_Components, _1))) + } + while (traverser.next()); + this->decayRate(decayRate); } - return true; } void CTimeSeriesDecomposition::swap(CTimeSeriesDecomposition &other) { - std::swap(m_DecayRate, other.m_DecayRate); std::swap(m_LastValueTime, other.m_LastValueTime); std::swap(m_LastPropagationTime, other.m_LastPropagationTime); - m_LongTermTrendTest.swap(other.m_LongTermTrendTest); - m_DiurnalTest.swap(other.m_DiurnalTest); - m_GeneralSeasonalityTest.swap(other.m_GeneralSeasonalityTest); + m_PeriodicityTest.swap(other.m_PeriodicityTest); m_CalendarCyclicTest.swap(other.m_CalendarCyclicTest); m_Components.swap(other.m_Components); } @@ -220,19 +207,15 @@ CTimeSeriesDecomposition &CTimeSeriesDecomposition::operator=(const CTimeSeriesD void CTimeSeriesDecomposition::acceptPersistInserter(core::CStatePersistInserter &inserter) const { - inserter.insertValue(DECAY_RATE_TAG, this->decayRate(), core::CIEEE754::E_SinglePrecision); - inserter.insertValue(LAST_VALUE_TIME_TAG, m_LastValueTime); - inserter.insertValue(LAST_PROPAGATION_TIME_TAG, m_LastPropagationTime); - inserter.insertLevel(LONG_TERM_TREND_TEST_TAG, boost::bind(&CLongTermTrendTest::acceptPersistInserter, - &m_LongTermTrendTest, _1)); - inserter.insertLevel(DIURNAL_TEST_TAG, boost::bind(&CDiurnalTest::acceptPersistInserter, - &m_DiurnalTest, _1)); - inserter.insertLevel(GENERAL_SEASONALITY_TEST_TAG, boost::bind(&CNonDiurnalTest::acceptPersistInserter, - &m_GeneralSeasonalityTest, _1)); - inserter.insertLevel(CALENDAR_CYCLIC_TEST_TAG, boost::bind(&CCalendarTest::acceptPersistInserter, - &m_CalendarCyclicTest, _1)); - inserter.insertLevel(SEASONAL_COMPONENTS_TAG, boost::bind(&CComponents::acceptPersistInserter, - &m_Components, _1)); + inserter.insertValue(VERSION_6_3_TAG, ""); + inserter.insertValue(LAST_VALUE_TIME_6_3_TAG, m_LastValueTime); + inserter.insertValue(LAST_PROPAGATION_TIME_6_3_TAG, m_LastPropagationTime); + inserter.insertLevel(PERIODICITY_TEST_6_3_TAG, boost::bind(&CPeriodicityTest::acceptPersistInserter, + &m_PeriodicityTest, _1)); + inserter.insertLevel(CALENDAR_CYCLIC_TEST_6_3_TAG, boost::bind(&CCalendarTest::acceptPersistInserter, + &m_CalendarCyclicTest, _1)); + inserter.insertLevel(COMPONENTS_6_3_TAG, boost::bind(&CComponents::acceptPersistInserter, + &m_Components, _1)); } CTimeSeriesDecomposition *CTimeSeriesDecomposition::clone(void) const @@ -243,19 +226,12 @@ CTimeSeriesDecomposition *CTimeSeriesDecomposition::clone(void) const void CTimeSeriesDecomposition::decayRate(double decayRate) { // Periodic component tests use a fixed decay rate. - m_DecayRate = decayRate; - m_LongTermTrendTest.decayRate(decayRate); m_Components.decayRate(decayRate); } double CTimeSeriesDecomposition::decayRate(void) const { - return m_DecayRate; -} - -void CTimeSeriesDecomposition::forForecasting(void) -{ - m_Components.forecast(); + return m_Components.decayRate(); } bool CTimeSeriesDecomposition::initialized(void) const @@ -276,31 +252,18 @@ bool CTimeSeriesDecomposition::addPoint(core_t::TTime time, this->propagateForwardsTo(time); SAddValue message{time, lastTime, value, weightStyles, weights, - CBasicStatistics::mean(this->baseline(time, 0.0, 0.0, E_Trend)), - CBasicStatistics::mean(this->baseline(time, 0.0, 0.0, E_NonDiurnal)), - CBasicStatistics::mean(this->baseline(time, 0.0, 0.0, E_Seasonal)), - CBasicStatistics::mean(this->baseline(time, 0.0, 0.0, E_Calendar))}; + CBasicStatistics::mean(this->baseline(time, 0.0, E_TrendForced)), + CBasicStatistics::mean(this->baseline(time, 0.0, E_Seasonal)), + CBasicStatistics::mean(this->baseline(time, 0.0, E_Calendar)), + [this](core_t::TTime time_) + { + return CBasicStatistics::mean(this->baseline( + time_, 0.0, E_Seasonal | E_Calendar)); + }, + m_Components.periodicityTestConfig()}; m_Components.handle(message); - - m_LongTermTrendTest.handle(message); - if (result.changed()) - { - message.s_Trend = CBasicStatistics::mean(this->baseline(time, 0.0, 0.0, E_Trend)); - } - - m_DiurnalTest.handle(message); - if (result.changed()) - { - message.s_Seasonal = CBasicStatistics::mean(this->baseline(time, 0.0, 0.0, E_Seasonal)); - } - - m_GeneralSeasonalityTest.handle(message); - if (result.changed()) - { - message.s_NonDiurnal = CBasicStatistics::mean(this->baseline(time, 0.0, 0.0, E_NonDiurnal)); - } - + m_PeriodicityTest.handle(message); m_CalendarCyclicTest.handle(message); return result.changed(); @@ -310,67 +273,34 @@ void CTimeSeriesDecomposition::propagateForwardsTo(core_t::TTime time) { if (time > m_LastPropagationTime) { - if (!this->forecasting()) - { - m_LongTermTrendTest.propagateForwards(m_LastPropagationTime, time); - m_DiurnalTest.propagateForwards(m_LastPropagationTime, time); - m_CalendarCyclicTest.propagateForwards(m_LastPropagationTime, time); - } + m_PeriodicityTest.propagateForwards(m_LastPropagationTime, time); + m_CalendarCyclicTest.propagateForwards(m_LastPropagationTime, time); m_Components.propagateForwards(m_LastPropagationTime, time); } m_LastPropagationTime = std::max(m_LastPropagationTime, time); } -bool CTimeSeriesDecomposition::testAndInterpolate(core_t::TTime time) -{ - CComponents::CScopeNotifyOnStateChange result(m_Components); - - SMessage message{time, std::max(m_LastValueTime, m_LastPropagationTime)}; - if (!this->forecasting()) - { - m_LongTermTrendTest.test(message); - m_DiurnalTest.test(message); - m_GeneralSeasonalityTest.test(message); - m_CalendarCyclicTest.test(message); - } - m_Components.interpolate(message); - - return result.changed(); -} - double CTimeSeriesDecomposition::mean(core_t::TTime time) const { return m_Components.meanValue(time); } TDoubleDoublePr CTimeSeriesDecomposition::baseline(core_t::TTime time, - double predictionConfidence, - double forecastConfidence, - EComponents components, + double confidence, + int components, bool smooth) const { - if (!this->initialized()) - { - return {0.0, 0.0}; - } - TVector2x1 baseline{0.0}; - if (components & E_Trend) + if (components & E_TrendForced) { - CTrendCRef trend = m_Components.trend(); - - baseline += vector2x1(trend.prediction(time, predictionConfidence)); - - if (this->forecasting()) + baseline += vector2x1(m_Components.trend().value(time, confidence)); + } + else if (components & E_Trend) + { + if (m_Components.usingTrendForPrediction()) { - CTrendCRef::TMatrix m; - if (trend.covariances(m)) - { - double dt{std::max(trend.time(time) - trend.time(m_LastValueTime), 0.0)}; - forecastErrors(trend.varianceDueToParameterDrift(time), - m, dt, forecastConfidence, baseline); - } + baseline += vector2x1(m_Components.trend().value(time, confidence)); } } @@ -380,19 +310,7 @@ TDoubleDoublePr CTimeSeriesDecomposition::baseline(core_t::TTime time, { if (this->selected(time, components, component)) { - baseline += vector2x1(component.value(time, predictionConfidence)); - if (this->forecasting()) - { - CSeasonalComponent::TMatrix m; - if (component.covariances(time, m)) - { - const CSeasonalTime &seasonalTime{component.time()}; - double dt{std::max( seasonalTime.regression(time) - - seasonalTime.regression(m_LastValueTime), 0.0)}; - forecastErrors(component.varianceDueToParameterDrift(time), - m, dt, forecastConfidence, baseline); - } - } + baseline += vector2x1(component.value(time, confidence)); } } } @@ -401,9 +319,9 @@ TDoubleDoublePr CTimeSeriesDecomposition::baseline(core_t::TTime time, { for (const auto &component : m_Components.calendar()) { - if (component.feature().inWindow(time)) + if (component.initialized() && component.feature().inWindow(time)) { - baseline += vector2x1(component.value(time, predictionConfidence)); + baseline += vector2x1(component.value(time, confidence)); } } } @@ -412,13 +330,78 @@ TDoubleDoublePr CTimeSeriesDecomposition::baseline(core_t::TTime time, { baseline += vector2x1(this->smooth( boost::bind(&CTimeSeriesDecomposition::baseline, - this, _1, predictionConfidence, - forecastConfidence, E_All, false), time, components)); + this, _1, confidence, components & E_Seasonal, false), + time, components)); } return pair(baseline); } +void CTimeSeriesDecomposition::forecast(core_t::TTime startTime, + core_t::TTime endTime, + core_t::TTime step, + double confidence, + double minimumScale, + TDouble3VecVec &result) +{ + if (endTime < startTime) + { + LOG_ERROR("Bad forecast range: [" << startTime << "," << endTime << "]"); + return; + } + if (confidence < 0.0 || confidence >= 100.0) + { + LOG_ERROR("Bad confidence interval: " << confidence << "%"); + return; + } + + auto predictor = [this, confidence](core_t::TTime time) + { + TVector2x1 prediction(0.0); + for (const auto &component : m_Components.seasonal()) + { + if (component.initialized() && component.time().inWindow(time)) + { + prediction += vector2x1(component.value(time, confidence)); + } + } + for (const auto &component : m_Components.calendar()) + { + if (component.initialized() && component.feature().inWindow(time)) + { + prediction += vector2x1(component.value(time, confidence)); + } + } + return pair(prediction); + }; + + endTime = startTime + CIntegerTools::ceil(endTime - startTime, step); + + double trendVariance{CBasicStatistics::mean(m_Components.trend().variance(0.0))}; + double seasonalVariance{m_Components.meanVariance() - trendVariance}; + double variance{this->meanVariance()}; + + double scale0{std::sqrt(std::max(CBasicStatistics::mean( + this->scale(startTime, variance, 0.0)), minimumScale))}; + TVector2x1 i0{vector2x1(confidenceInterval(confidence, seasonalVariance))}; + + m_Components.trend().forecast(startTime, endTime, step, confidence, result); + for (core_t::TTime time = startTime; time < endTime; time += step) + { + double scale{std::sqrt(std::max(CBasicStatistics::mean( + this->scale(time, variance, 0.0)), minimumScale))}; + TVector2x1 prediction{ vector2x1(predictor(time)) + + vector2x1(this->smooth(predictor, time, E_Seasonal)) + + (scale - scale0) * i0}; + + core_t::TTime index{(time - startTime) / step}; + result[index][0] += prediction(0); + result[index][1] += (prediction(0) + prediction(1)) / 2.0; + result[index][2] += prediction(1); + m_Components.interpolate(SMessage{time, time - step}, false); + } +} + double CTimeSeriesDecomposition::detrend(core_t::TTime time, double value, double confidence) const { if (!this->initialized()) @@ -431,7 +414,7 @@ double CTimeSeriesDecomposition::detrend(core_t::TTime time, double value, doubl double CTimeSeriesDecomposition::meanVariance(void) const { - return m_Components.meanVariance(); + return m_Components.meanVarianceScale() * m_Components.meanVariance(); } TDoubleDoublePr CTimeSeriesDecomposition::scale(core_t::TTime time, @@ -450,12 +433,18 @@ TDoubleDoublePr CTimeSeriesDecomposition::scale(core_t::TTime time, return {1.0, 1.0}; } - TVector2x1 scale{m_Components.trend().variance()}; + double components{0.0}; + TVector2x1 scale(0.0); + if (m_Components.usingTrendForPrediction()) + { + scale += vector2x1(m_Components.trend().variance(confidence)); + } for (const auto &component : m_Components.seasonal()) { if (component.initialized() && component.time().inWindow(time)) { scale += vector2x1(component.variance(time, confidence)); + components += 1.0; } } for (const auto &component : m_Components.calendar()) @@ -463,12 +452,21 @@ TDoubleDoublePr CTimeSeriesDecomposition::scale(core_t::TTime time, if (component.initialized() && component.feature().inWindow(time)) { scale += vector2x1(component.variance(time, confidence)); + components += 1.0; } } - LOG_TRACE("mean = " << mean << " variance = " << core::CContainerPrinter::print(scale)); double bias{std::min(2.0 * mean / variance, 1.0)}; - scale /= mean; + if (m_Components.usingTrendForPrediction()) + { + bias *= (components + 1.0) / std::max(components, 1.0); + } + LOG_TRACE("mean = " << mean + << " variance = " << variance + << " bias = " << bias + << " scale = " << core::CContainerPrinter::print(scale)); + + scale *= m_Components.meanVarianceScale() / mean; scale = TVector2x1{1.0} + bias * (scale - TVector2x1{1.0}); if (smooth) @@ -485,20 +483,13 @@ void CTimeSeriesDecomposition::skipTime(core_t::TTime skipInterval) { m_LastValueTime += skipInterval; m_LastPropagationTime += skipInterval; - m_LongTermTrendTest.skipTime(skipInterval); - m_DiurnalTest.skipTime(skipInterval); - m_GeneralSeasonalityTest.skipTime(skipInterval); - m_CalendarCyclicTest.advanceTimeTo(m_LastPropagationTime); } uint64_t CTimeSeriesDecomposition::checksum(uint64_t seed) const { - seed = CChecksum::calculate(seed, m_DecayRate); seed = CChecksum::calculate(seed, m_LastValueTime); seed = CChecksum::calculate(seed, m_LastPropagationTime); - seed = CChecksum::calculate(seed, m_LongTermTrendTest); - seed = CChecksum::calculate(seed, m_DiurnalTest); - seed = CChecksum::calculate(seed, m_GeneralSeasonalityTest); + seed = CChecksum::calculate(seed, m_PeriodicityTest); seed = CChecksum::calculate(seed, m_CalendarCyclicTest); return CChecksum::calculate(seed, m_Components); } @@ -507,9 +498,7 @@ void CTimeSeriesDecomposition::debugMemoryUsage(core::CMemoryUsage::TMemoryUsage { mem->setName("CTimeSeriesDecomposition"); core::CMemoryDebug::dynamicSize("m_Mediator", m_Mediator, mem); - core::CMemoryDebug::dynamicSize("m_LongTermTrendTest", m_LongTermTrendTest, mem); - core::CMemoryDebug::dynamicSize("m_DiurnalTest", m_DiurnalTest, mem); - core::CMemoryDebug::dynamicSize("m_GeneralSeasonalityTest", m_GeneralSeasonalityTest, mem); + core::CMemoryDebug::dynamicSize("m_PeriodicityTest", m_PeriodicityTest, mem); core::CMemoryDebug::dynamicSize("m_CalendarCyclicTest", m_CalendarCyclicTest, mem); core::CMemoryDebug::dynamicSize("m_Components", m_Components, mem); } @@ -517,9 +506,7 @@ void CTimeSeriesDecomposition::debugMemoryUsage(core::CMemoryUsage::TMemoryUsage std::size_t CTimeSeriesDecomposition::memoryUsage(void) const { return core::CMemory::dynamicSize(m_Mediator) - + core::CMemory::dynamicSize(m_LongTermTrendTest) - + core::CMemory::dynamicSize(m_DiurnalTest) - + core::CMemory::dynamicSize(m_GeneralSeasonalityTest) + + core::CMemory::dynamicSize(m_PeriodicityTest) + core::CMemory::dynamicSize(m_CalendarCyclicTest) + core::CMemory::dynamicSize(m_Components); } @@ -537,31 +524,24 @@ const maths_t::TSeasonalComponentVec &CTimeSeriesDecomposition::seasonalComponen void CTimeSeriesDecomposition::initializeMediator(void) { m_Mediator = boost::make_shared(); - m_Mediator->registerHandler(m_LongTermTrendTest); - m_Mediator->registerHandler(m_DiurnalTest); - m_Mediator->registerHandler(m_GeneralSeasonalityTest); + m_Mediator->registerHandler(m_PeriodicityTest); m_Mediator->registerHandler(m_CalendarCyclicTest); m_Mediator->registerHandler(m_Components); } -bool CTimeSeriesDecomposition::forecasting(void) const -{ - return m_Components.forecasting(); -} - template TDoubleDoublePr CTimeSeriesDecomposition::smooth(const F &f, core_t::TTime time, - EComponents components) const + int components) const { auto offset = [&f, time](core_t::TTime discontinuity) - { - TVector2x1 baselineMinusEps{vector2x1(f(discontinuity - 1))}; - TVector2x1 baselinePlusEps{ vector2x1(f(discontinuity + 1))}; - return 0.5 * (1.0 - static_cast(std::abs(time - discontinuity)) - / static_cast(SMOOTHING_INTERVAL)) - * (baselinePlusEps - baselineMinusEps); - }; + { + TVector2x1 baselineMinusEps{vector2x1(f(discontinuity - 1))}; + TVector2x1 baselinePlusEps{ vector2x1(f(discontinuity + 1))}; + return 0.5 * (1.0 - static_cast(std::abs(time - discontinuity)) + / static_cast(SMOOTHING_INTERVAL)) + * (baselinePlusEps - baselineMinusEps); + }; for (const auto &component : m_Components.seasonal()) { @@ -598,7 +578,7 @@ TDoubleDoublePr CTimeSeriesDecomposition::smooth(const F &f, } bool CTimeSeriesDecomposition::selected(core_t::TTime time, - EComponents components, + int components, const CSeasonalComponent &component) const { return component.initialized() @@ -606,7 +586,7 @@ bool CTimeSeriesDecomposition::selected(core_t::TTime time, && component.time().inWindow(time); } -bool CTimeSeriesDecomposition::matches(EComponents components, +bool CTimeSeriesDecomposition::matches(int components, const CSeasonalComponent &component) const { int seasonal{components & E_Seasonal}; diff --git a/lib/maths/CTimeSeriesDecompositionDetail.cc b/lib/maths/CTimeSeriesDecompositionDetail.cc index e3ecbc4760..34d963c7e8 100644 --- a/lib/maths/CTimeSeriesDecompositionDetail.cc +++ b/lib/maths/CTimeSeriesDecompositionDetail.cc @@ -29,13 +29,18 @@ #include #include #include +#include #include #include #include +#include #include +#include #include #include #include +#include +#include #include #include @@ -72,12 +77,10 @@ using TTimeTimePr = std::pair; using TTimeTimePrVec = std::vector; using TTimeTimePrDoubleFMap = boost::container::flat_map; using TTimeTimePrSizeFMap = boost::container::flat_map; +using TComponent5Vec = CPeriodicityHypothesisTestsResult::TComponent5Vec; using TSeasonalComponentPtrVec = std::vector; using TCalendarComponentPtrVec = std::vector; -using TRegression = CTimeSeriesDecompositionDetail::TRegression; -using TTrendCRef = CTimeSeriesDecompositionDetail::CTrendCRef; -const core_t::TTime HOUR = core::constants::HOUR; const core_t::TTime DAY = core::constants::DAY; const core_t::TTime WEEK = core::constants::WEEK; const core_t::TTime MONTH = 4 * WEEK; @@ -88,18 +91,6 @@ double pow2(double x) return x * x; } -//! Check if a period is daily oe weekly. -bool isDiurnal(core_t::TTime period) -{ - return period % DAY == 0 || period % WEEK == 0; -} - -//! Scale \p interval to account for \p bucketLength. -core_t::TTime scale(core_t::TTime interval, core_t::TTime bucketLength) -{ - return interval * std::max(bucketLength / HOUR, core_t::TTime(1)); -} - //! Compute the mean of \p mean of \p components. template double meanOf(MEAN_FUNCTION mean, const TSeasonalComponentVec &components) @@ -149,25 +140,29 @@ double meanOf(MEAN_FUNCTION mean, const TSeasonalComponentVec &components) //! \param[in] trend The long term trend. //! \param[in] seasonal The seasonal components. //! \param[in] calendar The calendar components. +//! \param[in] time The time of value to decompose. //! \param[in] deltas The delta offset to apply to the difference //! between each component value and its mean, used to minimize //! slope in the longer periods. -//! \param[in] time The time of value to decompose. //! \param[in,out] decomposition Updated to contain the value to //! add to each by component. -void decompose(const TTrendCRef &trend, +//! \param[out] predictions Filled in with the component predictions. +//! \param[out] error Filled in with the prediction error. +//! \param[out] scale Filled in with the normalization scaling. +void decompose(const CTrendComponent &trend, const TSeasonalComponentPtrVec &seasonal, const TCalendarComponentPtrVec &calendar, - const TDoubleVec &deltas, core_t::TTime time, - double &error, + const TDoubleVec &deltas, TDoubleVec &decomposition, - TDoubleVec &predictions) + TDoubleVec &predictions, + double &error, + double &scale) { std::size_t m{seasonal.size()}; std::size_t n{calendar.size()}; - double x0{CBasicStatistics::mean(trend.prediction(time, 0.0))}; + double x0{CBasicStatistics::mean(trend.value(time, 0.0))}; TDoubleVec x(m + n); double xhat{x0}; for (std::size_t i = 0u; i < m; ++i) @@ -181,61 +176,64 @@ void decompose(const TTrendCRef &trend, xhat += x[i]; } - double w0{trend.initialized() ? 1.0 : 0.0}; - double Z{static_cast(m + n) + w0}; + // Note we are adding on the a proportion of the error to the + // target value for each component. This constant controls the + // proportion of the overall error we add. There is no need + // to arrange for the sum error added to all components to be + // equal to the actual error to avoid bias: noise will still + // average down to zero (since the errors will be both positive + // and negative). It will however affect the variance in the + // limit the trend has been fit. This can be thought of as a + // trade off between the rate at which each component reacts + // to errors verses the error variance in the steady state with + // smaller values of Z corresponding to greater responsiveness. + double Z{std::max(0.5 * static_cast(m + n + 1), 1.0)}; error = decomposition[0] - xhat; - decomposition[0] = x0 + (decomposition[0] - xhat) * w0 / Z; - double lastDelta{0.0}; + decomposition[0] = x0 + (decomposition[0] - xhat) / Z; for (std::size_t i = 0u; i < m; ++i) { - double d{deltas[i] - lastDelta}; - lastDelta = deltas[i]; predictions[i] = x[i] - seasonal[i]->meanValue(); - decomposition[i + 1] = x[i] + (decomposition[i + 1] - xhat) / Z + d; + decomposition[i + 1] = x[i] + (decomposition[i + 1] - xhat) / Z + deltas[i]; } for (std::size_t i = m; i < m + n; ++i) { predictions[i] = x[i] - calendar[i - m]->meanValue(); decomposition[i + 1] = x[i] + (decomposition[i + 1] - xhat) / Z; } -} -//! Convert the propagation decay rate into the corresponding regular -//! periodicity test decay rate. -double regularTestDecayRate(double decayRate) -{ - return CSeasonalTime::scaleDecayRate(decayRate, DAY, WEEK); + // Because we add in more than the prediction error across the + // different components, i.e. because Z < m + n + 1, we end up + // with a bias in our variance estimates. We can mostly correct + // the bias by scaling the variance estimate, but need to calculate + // the scale. + scale = Z / static_cast(m + n + 1); } -//! Propagate a test forwards to account for \p end - \p start elapsed -//! time. -template -void propagateTestForwards(core_t::TTime start, - core_t::TTime end, - core_t::TTime interval, - const TEST &test) +//! Propagate a test forwards to account for \p end - \p start +//! elapsed time in steps or size \p step. +template +void stepwisePropagateForwards(core_t::TTime step, + core_t::TTime start, + core_t::TTime end, + const T &target) { - if (test) + if (target) { - start = CIntegerTools::floor(start, interval); - end = CIntegerTools::floor(end, interval); + start = CIntegerTools::floor(start, step); + end = CIntegerTools::floor(end, step); if (end > start) { - double time{static_cast(end - start) / static_cast(interval)}; - test->propagateForwardsByTime(time); + double time{static_cast(end - start) / static_cast(step)}; + target->propagateForwardsByTime(time); } } } -//! Get the time of \p time suitable for use in a trend regression model. -double regressionTime(core_t::TTime time, core_t::TTime origin) -{ - return static_cast(time - origin) / static_cast(WEEK); -} - //! Apply the common shift to the slope of \p trend. -void shiftSlope(const TTimeTimePrDoubleFMap &slopes, TRegression &trend) +void shiftSlope(const TTimeTimePrDoubleFMap &slopes, + double decayRate, + CTrendComponent &trend) { CBasicStatistics::CMinMax minmax; for (const auto &slope : slopes) @@ -245,75 +243,27 @@ void shiftSlope(const TTimeTimePrDoubleFMap &slopes, TRegression &trend) double shift{minmax.signMargin()}; if (shift != 0.0) { - trend.shiftGradient(shift); + trend.shiftSlope(decayRate, shift); } } - -// Long Term Trend Test State Machine - -// States -const std::size_t LT_INITIAL = 0; -const std::size_t LT_TEST = 1; -const std::size_t LT_NOT_TESTING = 2; -const std::size_t LT_ERROR = 3; -const TStrVec LT_STATES{"INITIAL", "TEST", "NOT_TESTING", "ERROR"}; -// Alphabet -const std::size_t LT_NEW_VALUE = 0; -const std::size_t LT_FINISH_TEST = 1; -const std::size_t LT_RESET = 2; -const TStrVec LT_ALPHABET{"NEW_VALUE", "FINISH_TEST", "RESET"}; -// Transition Function -const TSizeVecVec LT_TRANSITION_FUNCTION - { - TSizeVec{LT_TEST, LT_TEST, LT_NOT_TESTING, LT_ERROR }, - TSizeVec{LT_NOT_TESTING, LT_NOT_TESTING, LT_NOT_TESTING, LT_ERROR }, - TSizeVec{LT_INITIAL, LT_INITIAL, LT_INITIAL, LT_INITIAL} - }; - -// Diurnal Test State Machine - -// States -const std::size_t DW_INITIAL = 0; -const std::size_t DW_SMALL_TEST = 1; -const std::size_t DW_REGULAR_TEST = 2; -const std::size_t DW_NOT_TESTING = 3; -const std::size_t DW_ERROR = 4; -const TStrVec DW_STATES{"INITIAL", "SMALL_TEST", "REGULAR_TEST", "NOT_TESTING", "ERROR"}; -// Alphabet -const std::size_t DW_NEW_VALUE = 0; -const std::size_t DW_SMALL_TEST_TRUE = 1; -const std::size_t DW_REGULAR_TEST_TIMED_OUT = 2; -const std::size_t DW_FINISH_TEST = 3; -const std::size_t DW_RESET = 4; -const TStrVec DW_ALPHABET{"NEW_VALUE", "SMALL_TEST_TRUE", "REGULAR_TEST_TIMED_OUT", "FINISH_TEST", "RESET"}; -// Transition Function -const TSizeVecVec DW_TRANSITION_FUNCTION - { - TSizeVec{DW_REGULAR_TEST, DW_SMALL_TEST, DW_REGULAR_TEST, DW_NOT_TESTING, DW_ERROR }, - TSizeVec{DW_ERROR, DW_REGULAR_TEST, DW_ERROR, DW_NOT_TESTING, DW_ERROR }, - TSizeVec{DW_ERROR, DW_ERROR, DW_SMALL_TEST, DW_NOT_TESTING, DW_ERROR }, - TSizeVec{DW_NOT_TESTING, DW_NOT_TESTING, DW_NOT_TESTING, DW_NOT_TESTING, DW_ERROR }, - TSizeVec{DW_INITIAL, DW_INITIAL, DW_INITIAL, DW_NOT_TESTING, DW_INITIAL} - }; - -// General Seasonality Test State Machine +// Periodicity Test State Machine // States -const std::size_t GS_INITIAL = 0; -const std::size_t GS_TEST = 1; -const std::size_t GS_NOT_TESTING = 2; -const std::size_t GS_ERROR = 3; -const TStrVec GS_STATES{"INITIAL", "TEST", "NOT_TESTING", "ERROR" }; +const std::size_t PT_INITIAL = 0; +const std::size_t PT_TEST = 1; +const std::size_t PT_NOT_TESTING = 2; +const std::size_t PT_ERROR = 3; +const TStrVec PT_STATES{"INITIAL", "TEST", "NOT_TESTING", "ERROR" }; // Alphabet -const std::size_t GS_NEW_VALUE = 0; -const std::size_t GS_RESET = 1; -const TStrVec GS_ALPHABET{"NEW_VALUE", "RESET"}; +const std::size_t PT_NEW_VALUE = 0; +const std::size_t PT_RESET = 1; +const TStrVec PT_ALPHABET{"NEW_VALUE", "RESET"}; // Transition Function -const TSizeVecVec GS_TRANSITION_FUNCTION +const TSizeVecVec PT_TRANSITION_FUNCTION { - TSizeVec{GS_TEST, GS_TEST, GS_NOT_TESTING, GS_ERROR }, - TSizeVec{GS_INITIAL, GS_INITIAL, GS_NOT_TESTING, GS_INITIAL} + TSizeVec{PT_TEST, PT_TEST, PT_NOT_TESTING, PT_ERROR }, + TSizeVec{PT_INITIAL, PT_INITIAL, PT_NOT_TESTING, PT_INITIAL} }; // Calendar Cyclic Test State Machine @@ -340,66 +290,112 @@ const TSizeVecVec CC_TRANSITION_FUNCTION // States const std::size_t SC_NEW_COMPONENTS = 0; const std::size_t SC_NORMAL = 1; -const std::size_t SC_FORECASTING = 2; -const std::size_t SC_DISABLED = 3; -const std::size_t SC_ERROR = 4; -const TStrVec SC_STATES{"NEW_COMPONENTS", "NORMAL", "FORECASTING", "DISABLED", "ERROR"}; +const std::size_t SC_DISABLED = 2; +const std::size_t SC_ERROR = 3; +const TStrVec SC_STATES{"NEW_COMPONENTS", "NORMAL", "DISABLED", "ERROR"}; // Alphabet const std::size_t SC_ADDED_COMPONENTS = 0; const std::size_t SC_INTERPOLATED = 1; -const std::size_t SC_FORECAST = 2; -const std::size_t SC_RESET = 3; -const TStrVec SC_ALPHABET{"ADDED_COMPONENTS", "INTERPOLATED", "FORECAST", "RESET"}; +const std::size_t SC_RESET = 2; +const TStrVec SC_ALPHABET{"ADDED_COMPONENTS", "INTERPOLATED", "RESET"}; // Transition Function const TSizeVecVec SC_TRANSITION_FUNCTION { - TSizeVec{SC_NEW_COMPONENTS, SC_NEW_COMPONENTS, SC_ERROR, SC_DISABLED, SC_ERROR }, - TSizeVec{SC_NORMAL, SC_NORMAL, SC_FORECASTING, SC_DISABLED, SC_ERROR }, - TSizeVec{SC_ERROR, SC_FORECASTING, SC_FORECASTING, SC_DISABLED, SC_ERROR }, - TSizeVec{SC_NORMAL, SC_NORMAL, SC_NORMAL, SC_NORMAL, SC_NORMAL} + TSizeVec{SC_NEW_COMPONENTS, SC_NEW_COMPONENTS, SC_DISABLED, SC_ERROR }, + TSizeVec{SC_NORMAL, SC_NORMAL, SC_DISABLED, SC_ERROR }, + TSizeVec{SC_NORMAL, SC_NORMAL, SC_NORMAL, SC_NORMAL} }; -// Long Term Trend Test Tags -const std::string MACHINE_TAG{"a"}; -const std::string NEXT_TEST_TIME_TAG{"b"}; -const std::string TEST_TAG{"c"}; - -// Diurnal Test Tags -//const std::string MACHINE_TAG{"a"}; -//const std::string NEXT_TEST_TIME_TAG{"b"}; -const std::string STARTED_REGULAR_TEST_TAG{"c"}; -const std::string TIME_OUT_REGULAR_TEST_TAG{"d"}; -const std::string REGULAR_TEST_TAG{"e"}; -const std::string SMALL_TEST_TAG{"f"}; -const std::string PERIODS_TAG{"g"}; - -// General Seasonality Tags -//const std::string MACHINE_TAG{"a"}; -const std::string SHORT_TEST_TAG{"d"}; -const std::string LONG_TEST_TAG{"e"}; +const std::string VERSION_6_3_TAG("6.3"); + +// Periodicity Test Tags +// Version 6.3 +const std::string PERIODICITY_TEST_MACHINE_6_3_TAG{"a"}; +const std::string SHORT_WINDOW_6_3_TAG{"b"}; +const std::string LONG_WINDOW_6_3_TAG{"c"}; +// Old versions can't be restored. // Calendar Cyclic Test Tags -//const std::string MACHINE_TAG{"a"}; -const std::string LAST_MONTH_TAG{"b"}; -//const std::string TEST_TAG{"c"}; - -// Seasonal Components Tags -//const std::string MACHINE_TAG{"a"}; -const std::string TREND_TAG{"b"}; -const std::string SEASONAL_TAG{"c"}; -const std::string CALENDAR_TAG{"d"}; -const std::string COMPONENT_TAG{"e"}; -const std::string ERRORS_TAG{"f"}; -const std::string REGRESSION_TAG{"g"}; -const std::string VARIANCE_TAG{"h"}; -const std::string TIME_ORIGIN_TAG{"i"}; -const std::string LAST_UPDATE_TAG{"j"}; -const std::string PARAMETER_PROCESS_TAG{"k"}; +// Version 6.3 +const std::string CALENDAR_TEST_MACHINE_6_3_TAG{"a"}; +const std::string LAST_MONTH_6_3_TAG{"b"}; +const std::string CALENDAR_TEST_6_3_TAG{"c"}; +// These work for all versions. + +// Components Tags +// Version 6.3 +const std::string COMPONENTS_MACHINE_6_3_TAG{"a"}; +const std::string DECAY_RATE_6_3_TAG{"b"}; +const std::string TREND_6_3_TAG{"c"}; +const std::string SEASONAL_6_3_TAG{"d"}; +const std::string CALENDAR_6_3_TAG{"e"}; +const std::string COMPONENT_6_3_TAG{"f"}; +const std::string ERRORS_6_3_TAG{"g"}; +const std::string MEAN_VARIANCE_SCALE_6_3_TAG{"h"}; +const std::string MOMENTS_6_3_TAG{"i"}; +const std::string MOMENTS_MINUS_TREND_6_3_TAG{"j"}; +const std::string USING_TREND_FOR_PREDICTION_6_3_TAG{"k"}; +// Version < 6.3 +const std::string COMPONENTS_MACHINE_OLD_TAG{"a"}; +const std::string TREND_OLD_TAG{"b"}; +const std::string SEASONAL_OLD_TAG{"c"}; +const std::string CALENDAR_OLD_TAG{"d"}; +const std::string COMPONENT_OLD_TAG{"e"}; +const std::string ERRORS_OLD_TAG{"f"}; +const std::string REGRESSION_OLD_TAG{"g"}; +const std::string VARIANCE_OLD_TAG{"h"}; +const std::string TIME_ORIGIN_OLD_TAG{"i"}; +const std::string LAST_UPDATE_OLD_TAG{"j"}; + +//////////////////////// Upgrade to Version 6.3 //////////////////////// + +const double MODEL_WEIGHT_UPGRADING_TO_VERSION_6p3{48.0}; + +bool upgradeTrendModelToVersion6p3(const core_t::TTime bucketLength, + CTrendComponent &trend, + core::CStateRestoreTraverser &traverser) +{ + using TRegression = CRegression::CLeastSquaresOnline<3, double>; + + TRegression regression; + double variance{0.0}; + core_t::TTime origin{0}; + core_t::TTime lastUpdate{0}; + do + { + const std::string &name{traverser.name()}; + RESTORE(REGRESSION_OLD_TAG, traverser.traverseSubLevel(boost::bind( + &TRegression::acceptRestoreTraverser, ®ression, _1))) + RESTORE_BUILT_IN(VARIANCE_OLD_TAG, variance) + RESTORE_BUILT_IN(TIME_ORIGIN_OLD_TAG, origin) + RESTORE_BUILT_IN(LAST_UPDATE_OLD_TAG, lastUpdate) + } + while (traverser.next()); + + // Generate some samples from the old trend model. + double weight{MODEL_WEIGHT_UPGRADING_TO_VERSION_6p3 * static_cast(bucketLength) + / static_cast(4 * WEEK)}; + + CPRNG::CXorOShiro128Plus rng; + for (core_t::TTime time = lastUpdate - 4 * WEEK; + time < lastUpdate; + time += bucketLength) + { + double time_{static_cast(time - origin) / static_cast(WEEK)}; + double sample{ regression.predict(time_) + + CSampling::normalSample(rng, 0.0, variance)}; + trend.add(time, sample, weight); + } + + return true; +} + +//////////////////////////////////////////////////////////////////////// + +// Constants const core_t::TTime FOREVER{boost::numeric::bounds::highest()}; -const core_t::TTime UNSET_LAST_UPDATE{0}; -const std::size_t MAXIMUM_COMPONENTS = 8; -const TTrendCRef NO_TREND; +const std::size_t MAXIMUM_COMPONENTS{8}; const TSeasonalComponentVec NO_SEASONAL_COMPONENTS; const TCalendarComponentVec NO_CALENDAR_COMPONENTS; @@ -407,7 +403,6 @@ const TCalendarComponentVec NO_CALENDAR_COMPONENTS; //////// SMessage //////// -CTimeSeriesDecompositionDetail::SMessage::SMessage(void) : s_Time{}, s_LastTime{} {} CTimeSeriesDecompositionDetail::SMessage::SMessage(core_t::TTime time, core_t::TTime lastTime) : s_Time{time}, s_LastTime{lastTime} {} @@ -420,46 +415,32 @@ CTimeSeriesDecompositionDetail::SAddValue::SAddValue(core_t::TTime time, const maths_t::TWeightStyleVec &weightStyles, const maths_t::TDouble4Vec &weights, double trend, - double nonDiurnal, double seasonal, - double calendar) : + double calendar, + const TPredictor &predictor, + const CPeriodicityHypothesisTestsConfig &periodicityTestConfig) : SMessage{time, lastTime}, s_Value{value}, s_WeightStyles{weightStyles}, s_Weights{weights}, s_Trend{trend}, - s_NonDiurnal{nonDiurnal}, s_Seasonal{seasonal}, - s_Calendar{calendar} -{} - -//////// SDetectedTrend //////// - -CTimeSeriesDecompositionDetail::SDetectedTrend::SDetectedTrend(core_t::TTime time, - core_t::TTime lastTime, - const CTrendTest &test) : - SMessage{time, lastTime}, s_Test{test} -{} - -//////// SDetectedDiurnal //////// - -CTimeSeriesDecompositionDetail::SDetectedDiurnal::SDetectedDiurnal(core_t::TTime time, - core_t::TTime lastTime, - const CPeriodicityTestResult &result, - const CDiurnalPeriodicityTest &test) : - SMessage{time, lastTime}, s_Result{result}, s_Test{test} + s_Calendar{calendar}, + s_Predictor{predictor}, + s_PeriodicityTestConfig{periodicityTestConfig} {} -//////// SDetectedNonDiurnal //////// +//////// SDetectedSeasonal //////// -CTimeSeriesDecompositionDetail::SDetectedNonDiurnal::SDetectedNonDiurnal(core_t::TTime time, - core_t::TTime lastTime, - bool discardLongTermTrend, - const CPeriodicityTestResult &result, - const CGeneralPeriodicityTest &test) : +CTimeSeriesDecompositionDetail::SDetectedSeasonal::SDetectedSeasonal(core_t::TTime time, + core_t::TTime lastTime, + const CPeriodicityHypothesisTestsResult &result, + const CExpandingWindow &window, + const TPredictor &predictor) : SMessage{time, lastTime}, - s_DiscardLongTermTrend{discardLongTermTrend}, - s_Result{result}, s_Test{test} + s_Result{result}, + s_Window{window}, + s_Predictor{predictor} {} //////// SDetectedCalendar //////// @@ -485,11 +466,7 @@ CTimeSeriesDecompositionDetail::CHandler::~CHandler(void) {} void CTimeSeriesDecompositionDetail::CHandler::handle(const SAddValue &/*message*/) {} -void CTimeSeriesDecompositionDetail::CHandler::handle(const SDetectedTrend &/*message*/) {} - -void CTimeSeriesDecompositionDetail::CHandler::handle(const SDetectedDiurnal &/*message*/) {} - -void CTimeSeriesDecompositionDetail::CHandler::handle(const SDetectedNonDiurnal &/*message*/) {} +void CTimeSeriesDecompositionDetail::CHandler::handle(const SDetectedSeasonal &/*message*/) {} void CTimeSeriesDecompositionDetail::CHandler::handle(const SDetectedCalendar &/*message*/) {} @@ -533,477 +510,204 @@ std::size_t CTimeSeriesDecompositionDetail::CMediator::memoryUsage(void) const return core::CMemory::dynamicSize(m_Handlers); } -//////// CLongTermTrendTest //////// +//////// CPeriodicityTest //////// -CTimeSeriesDecompositionDetail::CLongTermTrendTest::CLongTermTrendTest(double decayRate) : - m_Machine{core::CStateMachine::create(LT_ALPHABET, LT_STATES, LT_TRANSITION_FUNCTION, LT_INITIAL)}, - m_MaximumDecayRate{decayRate}, - m_NextTestTime{}, - m_Test{new CTrendTest{decayRate}} +CTimeSeriesDecompositionDetail::CPeriodicityTest::CPeriodicityTest(double decayRate, + core_t::TTime bucketLength) : + m_Machine{core::CStateMachine::create( + PT_ALPHABET, PT_STATES, PT_TRANSITION_FUNCTION, + bucketLength > LONG_BUCKET_LENGTHS.back() ? PT_NOT_TESTING : PT_INITIAL)}, + m_DecayRate{decayRate}, + m_BucketLength{bucketLength} {} -CTimeSeriesDecompositionDetail::CLongTermTrendTest::CLongTermTrendTest(const CLongTermTrendTest &other) : +CTimeSeriesDecompositionDetail::CPeriodicityTest::CPeriodicityTest(const CPeriodicityTest &other) : m_Machine{other.m_Machine}, - m_MaximumDecayRate{other.m_MaximumDecayRate}, - m_NextTestTime{other.m_NextTestTime}, - m_Test{other.m_Test ? new CTrendTest{*other.m_Test} : 0} -{} - -bool CTimeSeriesDecompositionDetail::CLongTermTrendTest::acceptRestoreTraverser(core::CStateRestoreTraverser &traverser) -{ - do - { - const std::string &name{traverser.name()}; - RESTORE(MACHINE_TAG, traverser.traverseSubLevel( - boost::bind(&core::CStateMachine::acceptRestoreTraverser, &m_Machine, _1))); - RESTORE_BUILT_IN(NEXT_TEST_TIME_TAG, m_NextTestTime) - RESTORE_SETUP_TEARDOWN(TEST_TAG, - m_Test.reset(new CTrendTest(m_MaximumDecayRate)), - traverser.traverseSubLevel( - boost::bind(&CTrendTest::acceptRestoreTraverser, m_Test.get(), _1)), - /**/) - } - while (traverser.next()); - return true; -} - -void CTimeSeriesDecompositionDetail::CLongTermTrendTest::acceptPersistInserter(core::CStatePersistInserter &inserter) const -{ - inserter.insertLevel(MACHINE_TAG, boost::bind(&core::CStateMachine::acceptPersistInserter, &m_Machine, _1)); - inserter.insertValue(NEXT_TEST_TIME_TAG, m_NextTestTime); - if (m_Test) - { - inserter.insertLevel(TEST_TAG, boost::bind(&CTrendTest::acceptPersistInserter, m_Test.get(), _1)); - } -} - -void CTimeSeriesDecompositionDetail::CLongTermTrendTest::swap(CLongTermTrendTest &other) -{ - std::swap(m_Machine, other.m_Machine); - std::swap(m_MaximumDecayRate, other.m_MaximumDecayRate); - std::swap(m_NextTestTime, other.m_NextTestTime); - m_Test.swap(other.m_Test); -} - -void CTimeSeriesDecompositionDetail::CLongTermTrendTest::handle(const SAddValue &message) -{ - core_t::TTime time{message.s_Time}; - double value{message.s_Value - message.s_Seasonal - message.s_Calendar}; - double count{maths_t::countForUpdate(message.s_WeightStyles, message.s_Weights)}; - - this->test(message); - - if (time >= m_NextTestTime - this->testInterval()) - { - switch (m_Machine.state()) - { - case LT_NOT_TESTING: - break; - case LT_TEST: - m_Test->add(time, value, count); - m_Test->captureVariance(time, value, count); - break; - case LT_INITIAL: - this->apply(LT_NEW_VALUE, message); - this->handle(message); - break; - default: - LOG_ERROR("Test in a bad state: " << m_Machine.state()); - this->apply(LT_RESET, message); - break; - } - } -} - -void CTimeSeriesDecompositionDetail::CLongTermTrendTest::handle(const SNewComponents &message) -{ - switch (message.s_Component) - { - case SNewComponents::E_DiurnalSeasonal: - case SNewComponents::E_GeneralSeasonal: - if (m_Machine.state() != LT_NOT_TESTING) - { - this->apply(LT_RESET, message); - } - break; - case SNewComponents::E_Trend: - case SNewComponents::E_CalendarCyclic: - break; - } -} - -void CTimeSeriesDecompositionDetail::CLongTermTrendTest::test(const SMessage &message) -{ - core_t::TTime time{message.s_Time}; - core_t::TTime lastTime{message.s_LastTime}; - - if (this->shouldTest(time)) - { - switch (m_Machine.state()) - { - case LT_NOT_TESTING: - case LT_INITIAL: - break; - case LT_TEST: - if (m_Test->test()) - { - this->mediator()->forward(SDetectedTrend(time, lastTime, *m_Test)); - this->apply(LT_FINISH_TEST, message); - } - break; - default: - LOG_ERROR("Test in a bad state: " << m_Machine.state()); - this->apply(LT_RESET, message); - break; - } - } -} - -void CTimeSeriesDecompositionDetail::CLongTermTrendTest::decayRate(double decayRate) -{ - if (m_Test) - { - m_Test->decayRate(std::min(decayRate, m_MaximumDecayRate)); - } -} - -void CTimeSeriesDecompositionDetail::CLongTermTrendTest::propagateForwards(core_t::TTime start, - core_t::TTime end) -{ - if (m_Test) - { - double time{static_cast(end - start) / static_cast(DAY)}; - m_Test->propagateForwardsByTime(time); - } -} - -void CTimeSeriesDecompositionDetail::CLongTermTrendTest::skipTime(core_t::TTime skipInterval) -{ - core_t::TTime testInterval{this->testInterval()}; - m_NextTestTime = CIntegerTools::floor(m_NextTestTime + skipInterval + testInterval, testInterval); -} - -uint64_t CTimeSeriesDecompositionDetail::CLongTermTrendTest::checksum(uint64_t seed) const -{ - seed = CChecksum::calculate(seed, m_Machine); - seed = CChecksum::calculate(seed, m_MaximumDecayRate); - seed = CChecksum::calculate(seed, m_NextTestTime); - return CChecksum::calculate(seed, m_Test); -} - -void CTimeSeriesDecompositionDetail::CLongTermTrendTest::debugMemoryUsage(core::CMemoryUsage::TMemoryUsagePtr mem) const -{ - mem->setName("CLongTermTrendTest"); - core::CMemoryDebug::dynamicSize("m_Test", m_Test, mem); -} - -std::size_t CTimeSeriesDecompositionDetail::CLongTermTrendTest::memoryUsage(void) const -{ - return core::CMemory::dynamicSize(m_Test); -} - -void CTimeSeriesDecompositionDetail::CLongTermTrendTest::apply(std::size_t symbol, const SMessage &message) + m_DecayRate{other.m_DecayRate}, + m_BucketLength{other.m_BucketLength} { - core_t::TTime time{message.s_Time}; - - std::size_t old{m_Machine.state()}; - m_Machine.apply(symbol); - std::size_t state{m_Machine.state()}; - - if (state != old) + // Note that m_Windows is an array. + for (std::size_t i = 0u; i < other.m_Windows.size(); ++i) { - LOG_TRACE(LT_STATES[old] << "," << LT_ALPHABET[symbol] << " -> " << LT_STATES[state]); - - if (old == LT_INITIAL) - { - m_NextTestTime = time + 3 * WEEK; - } - - switch (state) + if (other.m_Windows[i]) { - case LT_TEST: - break; - case LT_NOT_TESTING: - m_NextTestTime = core_t::TTime{}; - m_Test.reset(); - break; - case LT_INITIAL: - m_NextTestTime = core_t::TTime{}; - m_Test.reset(new CTrendTest(m_MaximumDecayRate)); - break; - default: - LOG_ERROR("Test in a bad state: " << state); - this->apply(LT_RESET, message); - break; + m_Windows[i] = boost::make_shared(*other.m_Windows[i]); } } } -bool CTimeSeriesDecompositionDetail::CLongTermTrendTest::shouldTest(core_t::TTime time) -{ - if (time >= m_NextTestTime) - { - m_NextTestTime = CIntegerTools::ceil(time + 1, this->testInterval()); - return true; - } - return false; -} - -core_t::TTime CTimeSeriesDecompositionDetail::CLongTermTrendTest::testInterval(void) const -{ - return DAY; -} - -//////// CDiurnalTest //////// - -CTimeSeriesDecompositionDetail::CDiurnalTest::CDiurnalTest(double decayRate, - core_t::TTime bucketLength) : - m_Machine{core::CStateMachine::create(DW_ALPHABET, DW_STATES, DW_TRANSITION_FUNCTION, DW_INITIAL)}, - m_DecayRate{decayRate}, - m_BucketLength{bucketLength}, - m_NextTestTime{}, - m_StartedRegularTest{}, - m_TimeOutRegularTest{} -{} - -CTimeSeriesDecompositionDetail::CDiurnalTest::CDiurnalTest(const CDiurnalTest &other) : - m_Machine{other.m_Machine}, - m_DecayRate{other.m_DecayRate}, - m_BucketLength{other.m_BucketLength}, - m_NextTestTime{other.m_NextTestTime}, - m_StartedRegularTest{other.m_StartedRegularTest}, - m_TimeOutRegularTest{other.m_TimeOutRegularTest}, - m_RegularTest{other.m_RegularTest ? new CDiurnalPeriodicityTest{*other.m_RegularTest} : 0}, - m_SmallTest{other.m_SmallTest ? new CRandomizedPeriodicityTest{*other.m_SmallTest} : 0}, - m_Periods{other.m_Periods} -{} - -bool CTimeSeriesDecompositionDetail::CDiurnalTest::acceptRestoreTraverser(core::CStateRestoreTraverser &traverser) +bool CTimeSeriesDecompositionDetail::CPeriodicityTest::acceptRestoreTraverser(core::CStateRestoreTraverser &traverser) { do { const std::string &name{traverser.name()}; - RESTORE(MACHINE_TAG, traverser.traverseSubLevel( - boost::bind(&core::CStateMachine::acceptRestoreTraverser, &m_Machine, _1))); - RESTORE_BUILT_IN(NEXT_TEST_TIME_TAG, m_NextTestTime) - RESTORE_BUILT_IN(STARTED_REGULAR_TEST_TAG, m_StartedRegularTest) - RESTORE_BUILT_IN(TIME_OUT_REGULAR_TEST_TAG, m_TimeOutRegularTest) - RESTORE_SETUP_TEARDOWN(REGULAR_TEST_TAG, - m_RegularTest.reset(new CDiurnalPeriodicityTest(regularTestDecayRate(m_DecayRate))), - traverser.traverseSubLevel( - boost::bind(&CDiurnalPeriodicityTest::acceptRestoreTraverser, m_RegularTest.get(), _1)), + RESTORE(PERIODICITY_TEST_MACHINE_6_3_TAG, traverser.traverseSubLevel( + boost::bind(&core::CStateMachine::acceptRestoreTraverser, &m_Machine, _1))) + RESTORE_SETUP_TEARDOWN(SHORT_WINDOW_6_3_TAG, + m_Windows[E_Short].reset(this->newWindow(E_Short)), + m_Windows[E_Short] && traverser.traverseSubLevel( + boost::bind(&CExpandingWindow::acceptRestoreTraverser, + m_Windows[E_Short].get(), _1)), /**/) - RESTORE_SETUP_TEARDOWN(SMALL_TEST_TAG, - m_SmallTest.reset(new CRandomizedPeriodicityTest), - traverser.traverseSubLevel( - boost::bind(&CRandomizedPeriodicityTest::acceptRestoreTraverser, m_SmallTest.get(), _1)), + RESTORE_SETUP_TEARDOWN(LONG_WINDOW_6_3_TAG, + m_Windows[E_Long].reset(this->newWindow(E_Long)), + m_Windows[E_Long] && traverser.traverseSubLevel( + boost::bind(&CExpandingWindow::acceptRestoreTraverser, + m_Windows[E_Long].get(), _1)), /**/) - RESTORE(PERIODS_TAG, traverser.traverseSubLevel( - boost::bind(&CPeriodicityTestResult::acceptRestoreTraverser, &m_Periods, _1))) } while (traverser.next()); return true; } -void CTimeSeriesDecompositionDetail::CDiurnalTest::acceptPersistInserter(core::CStatePersistInserter &inserter) const +void CTimeSeriesDecompositionDetail::CPeriodicityTest::acceptPersistInserter(core::CStatePersistInserter &inserter) const { - inserter.insertLevel(MACHINE_TAG, boost::bind(&core::CStateMachine::acceptPersistInserter, &m_Machine, _1)); - inserter.insertValue(NEXT_TEST_TIME_TAG, m_NextTestTime); - inserter.insertValue(STARTED_REGULAR_TEST_TAG, m_StartedRegularTest); - inserter.insertValue(TIME_OUT_REGULAR_TEST_TAG, m_TimeOutRegularTest); - if (m_RegularTest) + inserter.insertLevel(PERIODICITY_TEST_MACHINE_6_3_TAG, + boost::bind(&core::CStateMachine::acceptPersistInserter, &m_Machine, _1)); + if (m_Windows[E_Short]) { - inserter.insertLevel(REGULAR_TEST_TAG, boost::bind( - &CDiurnalPeriodicityTest::acceptPersistInserter, m_RegularTest.get(), _1)); + inserter.insertLevel(SHORT_WINDOW_6_3_TAG, boost::bind( + &CExpandingWindow::acceptPersistInserter, m_Windows[E_Short].get(), _1)); } - if (m_SmallTest) + if (m_Windows[E_Long]) { - inserter.insertLevel(SMALL_TEST_TAG, boost::bind( - &CRandomizedPeriodicityTest::acceptPersistInserter, m_SmallTest.get(), _1)); + inserter.insertLevel(LONG_WINDOW_6_3_TAG, boost::bind( + &CExpandingWindow::acceptPersistInserter, m_Windows[E_Long].get(), _1)); } - inserter.insertLevel(PERIODS_TAG, boost::bind( - &CPeriodicityTestResult::acceptPersistInserter, &m_Periods, _1)); } -void CTimeSeriesDecompositionDetail::CDiurnalTest::swap(CDiurnalTest &other) +void CTimeSeriesDecompositionDetail::CPeriodicityTest::swap(CPeriodicityTest &other) { std::swap(m_Machine, other.m_Machine); std::swap(m_DecayRate, other.m_DecayRate); std::swap(m_BucketLength, other.m_BucketLength); - std::swap(m_NextTestTime, other.m_NextTestTime); - std::swap(m_StartedRegularTest, other.m_StartedRegularTest); - std::swap(m_TimeOutRegularTest, other.m_TimeOutRegularTest); - m_RegularTest.swap(other.m_RegularTest); - m_SmallTest.swap(other.m_SmallTest); - std::swap(m_Periods, other.m_Periods); + m_Windows[E_Short].swap(other.m_Windows[E_Short]); + m_Windows[E_Long].swap(other.m_Windows[E_Long]); } -void CTimeSeriesDecompositionDetail::CDiurnalTest::handle(const SAddValue &message) +void CTimeSeriesDecompositionDetail::CPeriodicityTest::handle(const SAddValue &message) { core_t::TTime time{message.s_Time}; - double value{message.s_Value - message.s_Trend - message.s_NonDiurnal - message.s_Calendar}; + double value{message.s_Value}; const maths_t::TWeightStyleVec &weightStyles{message.s_WeightStyles}; const maths_t::TDouble4Vec &weights{message.s_Weights}; + double weight{maths_t::countForUpdate(weightStyles, weights)}; this->test(message); switch (m_Machine.state()) { - case DW_NOT_TESTING: - break; - case DW_SMALL_TEST: - m_SmallTest->add(time, value); - break; - case DW_REGULAR_TEST: - if (time < this->timeOutRegularTest()) + case PT_TEST: + for (auto &&window : m_Windows) { - m_RegularTest->add(time, value, maths_t::countForUpdate(weightStyles, weights)); - } - else - { - LOG_TRACE("Switching to small test at " << time); - this->apply(DW_REGULAR_TEST_TIMED_OUT, message); - this->handle(message); + if (window) + { + window->add(time, value, weight); + } } break; - case DW_INITIAL: - this->apply(DW_NEW_VALUE, message); + case PT_NOT_TESTING: + break; + case PT_INITIAL: + this->apply(PT_NEW_VALUE, message); this->handle(message); break; default: LOG_ERROR("Test in a bad state: " << m_Machine.state()); - this->apply(DW_RESET, message); + this->apply(PT_RESET, message); break; } } -void CTimeSeriesDecompositionDetail::CDiurnalTest::handle(const SNewComponents &message) +void CTimeSeriesDecompositionDetail::CPeriodicityTest::handle(const SNewComponents &/*message*/) { - switch (message.s_Component) - { - case SNewComponents::E_GeneralSeasonal: - case SNewComponents::E_Trend: - if (m_Machine.state() != DW_NOT_TESTING) - { - this->apply(DW_RESET, message); - } - break; - case SNewComponents::E_DiurnalSeasonal: - case SNewComponents::E_CalendarCyclic: - break; - } + // This can be a no-op because we always maintain the raw time + // series values in the windows and apply corrections for other + // components only when we test. } -void CTimeSeriesDecompositionDetail::CDiurnalTest::test(const SMessage &message) +void CTimeSeriesDecompositionDetail::CPeriodicityTest::test(const SAddValue &message) { core_t::TTime time{message.s_Time}; - core_t::TTime lastTime{message.s_Time}; + core_t::TTime lastTime{message.s_LastTime}; + const TPredictor &predictor{message.s_Predictor}; + const CPeriodicityHypothesisTestsConfig &config{message.s_PeriodicityTestConfig}; switch (m_Machine.state()) { - case DW_NOT_TESTING: - case DW_INITIAL: - break; - case DW_SMALL_TEST: - if (this->shouldTest(time)) + case PT_TEST: + for (const auto &window : m_Windows) { - LOG_TRACE("Small testing at " << time); - if (m_SmallTest->test()) - { - LOG_TRACE("Switching to full test at " << time); - this->apply(DW_SMALL_TEST_TRUE, message); - } - } - break; - case DW_REGULAR_TEST: - if (this->shouldTest(time)) - { - LOG_TRACE("Regular testing at " << time); - CPeriodicityTestResult result{m_RegularTest->test()}; - if (result.periodic() && result != m_Periods) - { - this->mediator()->forward(SDetectedDiurnal(time, lastTime, result, *m_RegularTest)); - m_Periods = result; - } - if (result.periodic()) + if (this->shouldTest(window, time)) { - if (m_RegularTest->seenSufficientData()) - { - LOG_TRACE("Finished testing"); - this->apply(DW_FINISH_TEST, message); - } - else + TFloatMeanAccumulatorVec values(window->valuesMinusPrediction(predictor)); + core_t::TTime start{CIntegerTools::floor(window->startTime(), m_BucketLength)}; + core_t::TTime bucketLength{window->bucketLength()}; + CPeriodicityHypothesisTestsResult result{testForPeriods(config, start, bucketLength, values)}; + if (result.periodic()) { - m_NextTestTime = std::max(CIntegerTools::ceil( - m_StartedRegularTest + 2 * WEEK, this->testInterval()), - m_NextTestTime); + this->mediator()->forward(SDetectedSeasonal{time, lastTime, result, *window, predictor}); } } } break; + case PT_NOT_TESTING: + case PT_INITIAL: + break; default: LOG_ERROR("Test in a bad state: " << m_Machine.state()); - this->apply(DW_RESET, message); + this->apply(PT_RESET, message); break; } } -void CTimeSeriesDecompositionDetail::CDiurnalTest::propagateForwards(core_t::TTime start, - core_t::TTime end) +void CTimeSeriesDecompositionDetail::CPeriodicityTest::propagateForwards(core_t::TTime start, + core_t::TTime end) { - propagateTestForwards(start, end, DAY, m_RegularTest); + stepwisePropagateForwards(DAY, start, end, m_Windows[E_Short]); + stepwisePropagateForwards(WEEK, start, end, m_Windows[E_Long]); } -void CTimeSeriesDecompositionDetail::CDiurnalTest::skipTime(core_t::TTime skipInterval) -{ - core_t::TTime testInterval{this->testInterval()}; - m_NextTestTime = CIntegerTools::floor(m_NextTestTime + skipInterval + testInterval, testInterval); - m_TimeOutRegularTest += skipInterval; -} - -uint64_t CTimeSeriesDecompositionDetail::CDiurnalTest::checksum(uint64_t seed) const +uint64_t CTimeSeriesDecompositionDetail::CPeriodicityTest::checksum(uint64_t seed) const { seed = CChecksum::calculate(seed, m_Machine); seed = CChecksum::calculate(seed, m_DecayRate); seed = CChecksum::calculate(seed, m_BucketLength); - seed = CChecksum::calculate(seed, m_NextTestTime); - seed = CChecksum::calculate(seed, m_StartedRegularTest); - seed = CChecksum::calculate(seed, m_TimeOutRegularTest); - seed = CChecksum::calculate(seed, m_RegularTest); - seed = CChecksum::calculate(seed, m_SmallTest); - return CChecksum::calculate(seed, m_Periods); + return CChecksum::calculate(seed, m_Windows); } -void CTimeSeriesDecompositionDetail::CDiurnalTest::debugMemoryUsage(core::CMemoryUsage::TMemoryUsagePtr mem) const +void CTimeSeriesDecompositionDetail::CPeriodicityTest::debugMemoryUsage(core::CMemoryUsage::TMemoryUsagePtr mem) const { - mem->setName("CDiurnalTest"); - core::CMemoryDebug::dynamicSize("m_RegularTest", m_RegularTest, mem); - core::CMemoryDebug::dynamicSize("m_SmallTest", m_SmallTest, mem); + mem->setName("CPeriodicityTest"); + core::CMemoryDebug::dynamicSize("m_Windows", m_Windows, mem); } -std::size_t CTimeSeriesDecompositionDetail::CDiurnalTest::memoryUsage(void) const +std::size_t CTimeSeriesDecompositionDetail::CPeriodicityTest::memoryUsage(void) const { - std::size_t usage{core::CMemory::dynamicSize(m_RegularTest) + core::CMemory::dynamicSize(m_SmallTest)}; - if (m_Machine.state() == DW_INITIAL) + std::size_t usage{core::CMemory::dynamicSize(m_Windows)}; + if (m_Machine.state() == PT_INITIAL) { usage += this->extraMemoryOnInitialization(); } return usage; } -std::size_t CTimeSeriesDecompositionDetail::CDiurnalTest::extraMemoryOnInitialization(void) const +std::size_t CTimeSeriesDecompositionDetail::CPeriodicityTest::extraMemoryOnInitialization(void) const { static std::size_t result{0}; if (result == 0) { - TPeriodicityTestPtr regularTest(CDiurnalPeriodicityTest::create( - m_BucketLength, regularTestDecayRate(m_DecayRate))); - result = core::CMemory::dynamicSize(regularTest); + for (auto i : {E_Short, E_Long}) + { + TExpandingWindowPtr window(this->newWindow(i)); + result += core::CMemory::dynamicSize(window); + } } return result; } -void CTimeSeriesDecompositionDetail::CDiurnalTest::apply(std::size_t symbol, const SMessage &message) +void CTimeSeriesDecompositionDetail::CPeriodicityTest::apply(std::size_t symbol, + const SMessage &message) { core_t::TTime time{message.s_Time}; @@ -1013,396 +717,100 @@ void CTimeSeriesDecompositionDetail::CDiurnalTest::apply(std::size_t symbol, con if (state != old) { - LOG_TRACE(DW_STATES[old] << "," << DW_ALPHABET[symbol] << " -> " << DW_STATES[state]); + LOG_TRACE(PT_STATES[old] << "," << PT_ALPHABET[symbol] << " -> " << PT_STATES[state]); - if (old == DW_INITIAL) - { - m_NextTestTime = time; - m_TimeOutRegularTest = time + scale(5 * WEEK, m_BucketLength); - } + auto initialize = [this](core_t::TTime time_) + { + for (auto i : {E_Short, E_Long}) + { + m_Windows[i].reset(this->newWindow(i)); + if (m_Windows[i]) + { + m_Windows[i]->initialize(time_); + } + } + }; switch (state) { - case DW_SMALL_TEST: - if (m_RegularTest) + case PT_TEST: + if (std::all_of(m_Windows.begin(), m_Windows.end(), + [](const TExpandingWindowPtr &window) { return !window; })) { - m_RegularTest.reset(); - } - if (!m_SmallTest) - { - m_SmallTest.reset(new CRandomizedPeriodicityTest); + initialize(time); } break; - case DW_REGULAR_TEST: - if (m_SmallTest) - { - m_TimeOutRegularTest = time + scale(9 * WEEK, m_BucketLength); - m_SmallTest.reset(); - } - if (!m_RegularTest) - { - m_StartedRegularTest = time; - m_RegularTest.reset(CDiurnalPeriodicityTest::create( - m_BucketLength, regularTestDecayRate(m_DecayRate))); - if (!m_RegularTest) - { - this->apply(DW_NOT_TESTING, message); - } - } + case PT_INITIAL: + initialize(time); break; - case DW_NOT_TESTING: - case DW_INITIAL: - m_NextTestTime = core_t::TTime{}; - m_TimeOutRegularTest = core_t::TTime{}; - m_SmallTest.reset(); - m_RegularTest.reset(); + case PT_NOT_TESTING: + m_Windows[0].reset(); + m_Windows[1].reset(); break; default: LOG_ERROR("Test in a bad state: " << state); - this->apply(DW_RESET, message); - break; - } - } -} - -bool CTimeSeriesDecompositionDetail::CDiurnalTest::shouldTest(core_t::TTime time) -{ - if (time >= m_NextTestTime) - { - m_NextTestTime = CIntegerTools::ceil(time + 1, this->testInterval()); - return true; - } - return false; -} - -core_t::TTime CTimeSeriesDecompositionDetail::CDiurnalTest::testInterval(void) const -{ - switch (m_Machine.state()) - { - case DW_SMALL_TEST: - return DAY; - case DW_REGULAR_TEST: - return m_NextTestTime > m_StartedRegularTest + 2 * WEEK ? 2 * WEEK : DAY; - default: - break; - } - return FOREVER; -} - -core_t::TTime CTimeSeriesDecompositionDetail::CDiurnalTest::timeOutRegularTest(void) const -{ - return m_TimeOutRegularTest + static_cast( - 6.0 * static_cast(WEEK) - * (1.0 - m_RegularTest->populatedRatio())); -} - -//////// CNonDiurnalTest //////// - -CTimeSeriesDecompositionDetail::CNonDiurnalTest::CNonDiurnalTest(double decayRate, - core_t::TTime bucketLength) : - m_Machine{core::CStateMachine::create( - GS_ALPHABET, GS_STATES, GS_TRANSITION_FUNCTION, - bucketLength > LONG_BUCKET_LENGTHS.back() ? GS_NOT_TESTING : GS_INITIAL)}, - m_DecayRate{decayRate}, - m_BucketLength{bucketLength} -{} - -CTimeSeriesDecompositionDetail::CNonDiurnalTest::CNonDiurnalTest(const CNonDiurnalTest &other) : - m_Machine{other.m_Machine}, - m_DecayRate{other.m_DecayRate}, - m_BucketLength{other.m_BucketLength} -{ - for (std::size_t i = 0u; i < other.m_Tests.size(); ++i) - { - if (other.m_Tests[i]) - { - m_Tests[i] = boost::make_shared(*other.m_Tests[i]); - } - } -} - -bool CTimeSeriesDecompositionDetail::CNonDiurnalTest::acceptRestoreTraverser(core::CStateRestoreTraverser &traverser) -{ - do - { - const std::string &name{traverser.name()}; - RESTORE(MACHINE_TAG, traverser.traverseSubLevel( - boost::bind(&core::CStateMachine::acceptRestoreTraverser, &m_Machine, _1))) - RESTORE_SETUP_TEARDOWN(SHORT_TEST_TAG, - m_Tests[E_Short].reset(this->newTest(E_Short)), - m_Tests[E_Short] && traverser.traverseSubLevel( - boost::bind(&CScanningPeriodicityTest::acceptRestoreTraverser, - m_Tests[E_Short].get(), _1)), - /**/) - RESTORE_SETUP_TEARDOWN(LONG_TEST_TAG, - m_Tests[E_Long].reset(this->newTest(E_Long)), - m_Tests[E_Long] && traverser.traverseSubLevel( - boost::bind(&CScanningPeriodicityTest::acceptRestoreTraverser, - m_Tests[E_Long].get(), _1)), - /**/) - } - while (traverser.next()); - return true; -} - -void CTimeSeriesDecompositionDetail::CNonDiurnalTest::acceptPersistInserter(core::CStatePersistInserter &inserter) const -{ - inserter.insertLevel(MACHINE_TAG, boost::bind(&core::CStateMachine::acceptPersistInserter, &m_Machine, _1)); - if (m_Tests[E_Short]) - { - inserter.insertLevel(SHORT_TEST_TAG, boost::bind( - &CScanningPeriodicityTest::acceptPersistInserter, m_Tests[E_Short].get(), _1)); - } - if (m_Tests[E_Long]) - { - inserter.insertLevel(LONG_TEST_TAG, boost::bind( - &CScanningPeriodicityTest::acceptPersistInserter, m_Tests[E_Long].get(), _1)); - } -} - -void CTimeSeriesDecompositionDetail::CNonDiurnalTest::swap(CNonDiurnalTest &other) -{ - std::swap(m_Machine, other.m_Machine); - std::swap(m_DecayRate, other.m_DecayRate); - std::swap(m_BucketLength, other.m_BucketLength); - m_Tests[0].swap(other.m_Tests[0]); - m_Tests[1].swap(other.m_Tests[1]); -} - -void CTimeSeriesDecompositionDetail::CNonDiurnalTest::handle(const SAddValue &message) -{ - core_t::TTime time{message.s_Time}; - double values[2]; - double aperiodic{message.s_Value - message.s_Seasonal - message.s_Calendar}; - values[E_Short] = aperiodic - message.s_Trend; - values[E_Long] = aperiodic; - const maths_t::TWeightStyleVec &weightStyles{message.s_WeightStyles}; - const maths_t::TDouble4Vec &weights{message.s_Weights}; - - this->test(message); - - switch (m_Machine.state()) - { - case GS_TEST: - for (auto test : {E_Short, E_Long}) - { - if (m_Tests[test]) - { - m_Tests[test]->add(time, values[test], - maths_t::countForUpdate(weightStyles, weights)); - } - } - break; - case GS_NOT_TESTING: - break; - case GS_INITIAL: - this->apply(GS_NEW_VALUE, message); - this->handle(message); - break; - default: - LOG_ERROR("Test in a bad state: " << m_Machine.state()); - this->apply(GS_RESET, message); - break; - } -} - -void CTimeSeriesDecompositionDetail::CNonDiurnalTest::handle(const SNewComponents &message) -{ - if (m_Machine.state() != GS_NOT_TESTING) - { - switch (message.s_Component) - { - case SNewComponents::E_GeneralSeasonal: - case SNewComponents::E_Trend: - this->apply(GS_RESET, message, {{0 * WEEK, 12 * WEEK}}); - break; - case SNewComponents::E_DiurnalSeasonal: - this->apply(GS_RESET, message, {{3 * WEEK, 12 * WEEK}}); - break; - case SNewComponents::E_CalendarCyclic: + this->apply(PT_RESET, message); break; } } } -void CTimeSeriesDecompositionDetail::CNonDiurnalTest::test(const SMessage &message) +bool CTimeSeriesDecompositionDetail::CPeriodicityTest::shouldTest(const TExpandingWindowPtr &window, + core_t::TTime time) const { - core_t::TTime time{message.s_Time}; - core_t::TTime lastTime{message.s_LastTime}; + // We need to test more frequently than when we compress, because + // this only happens after we've seen 336 buckets, this would thus + // significantly delay when we first detect a daily periodic for + // longer bucket lengths otherwise. - switch (m_Machine.state()) - { - case GS_TEST: - for (auto test_ : {E_Short, E_Long}) + auto shouldTest = [this, time](const TExpandingWindowPtr &window_) { - if (m_Tests[test_] && m_Tests[test_]->needToCompress(time)) + core_t::TTime length{time - window_->startTime()}; + for (auto lengthToTest : {3 * DAY, 1 * WEEK, 2 * WEEK}) { - CGeneralPeriodicityTest test; - CPeriodicityTestResult result; - boost::tie(test, result) = m_Tests[test_]->test(); - if (result.periodic()) + if (length >= lengthToTest && length < lengthToTest + m_BucketLength) { - this->mediator()->forward(SDetectedNonDiurnal( - time, lastTime, test_ == E_Long, result, test)); + return true; } } - } - break; - case GS_NOT_TESTING: - case GS_INITIAL: - break; - default: - LOG_ERROR("Test in a bad state: " << m_Machine.state()); - this->apply(GS_RESET, message); - break; - } -} - -void CTimeSeriesDecompositionDetail::CNonDiurnalTest::propagateForwards(core_t::TTime start, - core_t::TTime end) -{ - propagateTestForwards(start, end, DAY, m_Tests[E_Short]); - propagateTestForwards(start, end, WEEK, m_Tests[E_Long]); -} - -void CTimeSeriesDecompositionDetail::CNonDiurnalTest::skipTime(core_t::TTime skipInterval) -{ - for (const auto &test : m_Tests) - { - if (test) - { - test->skipTime(skipInterval); - } - } -} - -uint64_t CTimeSeriesDecompositionDetail::CNonDiurnalTest::checksum(uint64_t seed) const -{ - seed = CChecksum::calculate(seed, m_Machine); - seed = CChecksum::calculate(seed, m_DecayRate); - seed = CChecksum::calculate(seed, m_BucketLength); - return CChecksum::calculate(seed, m_Tests); -} - -void CTimeSeriesDecompositionDetail::CNonDiurnalTest::debugMemoryUsage(core::CMemoryUsage::TMemoryUsagePtr mem) const -{ - mem->setName("CNonDiurnalTest"); - core::CMemoryDebug::dynamicSize("m_Tests", m_Tests, mem); -} - -std::size_t CTimeSeriesDecompositionDetail::CNonDiurnalTest::memoryUsage(void) const -{ - std::size_t usage{core::CMemory::dynamicSize(m_Tests)}; - if (m_Machine.state() == GS_INITIAL) - { - usage += this->extraMemoryOnInitialization(); - } - return usage; -} - -std::size_t CTimeSeriesDecompositionDetail::CNonDiurnalTest::extraMemoryOnInitialization(void) const -{ - static std::size_t result{0}; - if (result == 0) - { - TScanningPeriodicityTestPtr shortTest(this->newTest(E_Short)); - TScanningPeriodicityTestPtr longTest(this->newTest(E_Long)); - result = core::CMemory::dynamicSize(shortTest) + core::CMemory::dynamicSize(longTest); - } - return result; + return false; + }; + return window && (window->needToCompress(time) || shouldTest(window)); } -void CTimeSeriesDecompositionDetail::CNonDiurnalTest::apply(std::size_t symbol, - const SMessage &message, - const TTimeAry &offsets) +CExpandingWindow *CTimeSeriesDecompositionDetail::CPeriodicityTest::newWindow(ETest test) const { - core_t::TTime time{message.s_Time}; + using TTimeCRng = CExpandingWindow::TTimeCRng; - std::size_t old{m_Machine.state()}; - m_Machine.apply(symbol); - std::size_t state{m_Machine.state()}; - - if (state != old) - { - LOG_TRACE(GS_STATES[old] << "," << GS_ALPHABET[symbol] << " -> " << GS_STATES[state]); - - switch (state) + auto newWindow = [this](const TTimeVec &bucketLengths) { - case GS_TEST: - if (std::all_of(m_Tests.begin(), m_Tests.end(), - [](const TScanningPeriodicityTestPtr &test) { return !test; })) - { - for (auto test : {E_Short, E_Long}) - { - m_Tests[test].reset(this->newTest(test)); - if (m_Tests[test]) - { - m_Tests[test]->initialize(time + offsets[test]); - } - } - } - break; - case GS_INITIAL: - for (auto test : {E_Short, E_Long}) + if (m_BucketLength <= bucketLengths.back()) { - m_Tests[test].reset(this->newTest(test)); - if (m_Tests[test]) - { - m_Tests[test]->initialize(time + offsets[test]); - } + std::ptrdiff_t a{std::lower_bound(bucketLengths.begin(), + bucketLengths.end(), + m_BucketLength) - bucketLengths.begin()}; + std::size_t b{bucketLengths.size()}; + TTimeCRng bucketLengths_(bucketLengths, a, b); + return new CExpandingWindow(m_BucketLength, bucketLengths_, 336, m_DecayRate); } - break; - case GS_NOT_TESTING: - m_Tests[0].reset(); - m_Tests[1].reset(); - break; - default: - LOG_ERROR("Test in a bad state: " << state); - this->apply(GS_RESET, message, offsets); - break; - } - } -} + return static_cast(0); + }; -CScanningPeriodicityTest *CTimeSeriesDecompositionDetail::CNonDiurnalTest::newTest(ETest test) const -{ - using TTimeCRng = CScanningPeriodicityTest::TTimeCRng; switch (test) { - case E_Short: - if (m_BucketLength < SHORT_BUCKET_LENGTHS.back()) - { - std::ptrdiff_t a{std::lower_bound(SHORT_BUCKET_LENGTHS.begin(), - SHORT_BUCKET_LENGTHS.end(), - m_BucketLength) - SHORT_BUCKET_LENGTHS.begin()}; - std::size_t b{SHORT_BUCKET_LENGTHS.size()}; - TTimeCRng bucketLengths(SHORT_BUCKET_LENGTHS, a, b); - return new CScanningPeriodicityTest(bucketLengths, TEST_SIZE, m_DecayRate); - } - break; - case E_Long: - if (m_BucketLength < LONG_BUCKET_LENGTHS.back()) - { - std::ptrdiff_t a{std::lower_bound(LONG_BUCKET_LENGTHS.begin(), - LONG_BUCKET_LENGTHS.end(), - m_BucketLength) - LONG_BUCKET_LENGTHS.begin()}; - std::size_t b{LONG_BUCKET_LENGTHS.size()}; - TTimeCRng bucketLengths(LONG_BUCKET_LENGTHS, a, b); - return new CScanningPeriodicityTest(bucketLengths, TEST_SIZE, m_DecayRate); - } - break; + case E_Short: return newWindow(SHORT_BUCKET_LENGTHS); + case E_Long: return newWindow(LONG_BUCKET_LENGTHS); } return 0; } -const std::size_t CTimeSeriesDecompositionDetail::CNonDiurnalTest::TEST_SIZE{168}; -const TTimeVec CTimeSeriesDecompositionDetail::CNonDiurnalTest::SHORT_BUCKET_LENGTHS +const TTimeVec CTimeSeriesDecompositionDetail::CPeriodicityTest::SHORT_BUCKET_LENGTHS { - 1, 5, 10, 30, 60, 300, 600, 1800, 3600, 7200 + 1, 5, 10, 30, 60, 300, 600, 1800, 3600 }; -const TTimeVec CTimeSeriesDecompositionDetail::CNonDiurnalTest::LONG_BUCKET_LENGTHS +const TTimeVec CTimeSeriesDecompositionDetail::CPeriodicityTest::LONG_BUCKET_LENGTHS { - 21600, 43200, 86400, 172800, 345600, 691200 + 7200, 21600, 43200, 86400, 172800, 345600 }; //////// CCalendarCyclic //////// @@ -1427,10 +835,10 @@ bool CTimeSeriesDecompositionDetail::CCalendarTest::acceptRestoreTraverser(core: do { const std::string &name{traverser.name()}; - RESTORE(MACHINE_TAG, traverser.traverseSubLevel( - boost::bind(&core::CStateMachine::acceptRestoreTraverser, &m_Machine, _1))) - RESTORE_BUILT_IN(LAST_MONTH_TAG, m_LastMonth); - RESTORE_SETUP_TEARDOWN(TEST_TAG, + RESTORE(CALENDAR_TEST_MACHINE_6_3_TAG, traverser.traverseSubLevel( + boost::bind(&core::CStateMachine::acceptRestoreTraverser, &m_Machine, _1))) + RESTORE_BUILT_IN(LAST_MONTH_6_3_TAG, m_LastMonth); + RESTORE_SETUP_TEARDOWN(CALENDAR_TEST_6_3_TAG, m_Test.reset(new CCalendarCyclicTest(m_DecayRate)), traverser.traverseSubLevel( boost::bind(&CCalendarCyclicTest::acceptRestoreTraverser, m_Test.get(), _1)), @@ -1442,12 +850,13 @@ bool CTimeSeriesDecompositionDetail::CCalendarTest::acceptRestoreTraverser(core: void CTimeSeriesDecompositionDetail::CCalendarTest::acceptPersistInserter(core::CStatePersistInserter &inserter) const { - inserter.insertLevel(MACHINE_TAG, boost::bind(&core::CStateMachine::acceptPersistInserter, &m_Machine, _1)); - inserter.insertValue(LAST_MONTH_TAG, m_LastMonth); + inserter.insertLevel(CALENDAR_TEST_MACHINE_6_3_TAG, + boost::bind(&core::CStateMachine::acceptPersistInserter, &m_Machine, _1)); + inserter.insertValue(LAST_MONTH_6_3_TAG, m_LastMonth); if (m_Test) { - inserter.insertLevel(TEST_TAG, boost::bind( - &CCalendarCyclicTest::acceptPersistInserter, m_Test.get(), _1)); + inserter.insertLevel(CALENDAR_TEST_6_3_TAG, boost::bind( + &CCalendarCyclicTest::acceptPersistInserter, m_Test.get(), _1)); } } @@ -1494,8 +903,7 @@ void CTimeSeriesDecompositionDetail::CCalendarTest::handle(const SNewComponents { case SNewComponents::E_GeneralSeasonal: case SNewComponents::E_DiurnalSeasonal: - case SNewComponents::E_Trend: - this->apply(GS_RESET, message); + this->apply(CC_RESET, message); break; case SNewComponents::E_CalendarCyclic: break; @@ -1534,18 +942,14 @@ void CTimeSeriesDecompositionDetail::CCalendarTest::test(const SMessage &message void CTimeSeriesDecompositionDetail::CCalendarTest::propagateForwards(core_t::TTime start, core_t::TTime end) { - propagateTestForwards(start, end, DAY, m_Test); -} - -void CTimeSeriesDecompositionDetail::CCalendarTest::advanceTimeTo(core_t::TTime time) -{ - m_LastMonth = this->month(time); + stepwisePropagateForwards(DAY, start, end, m_Test); } uint64_t CTimeSeriesDecompositionDetail::CCalendarTest::checksum(uint64_t seed) const { seed = CChecksum::calculate(seed, m_Machine); seed = CChecksum::calculate(seed, m_DecayRate); + seed = CChecksum::calculate(seed, m_LastMonth); return CChecksum::calculate(seed, m_Test); } @@ -1629,86 +1033,6 @@ int CTimeSeriesDecompositionDetail::CCalendarTest::month(core_t::TTime time) con return month; } -//////// CTrendCRef //////// - -CTimeSeriesDecompositionDetail::CTrendCRef::CTrendCRef(void) : - m_Trend{0}, m_Variance{0.0}, m_TimeOrigin{0}, - m_LastUpdate{0}, m_ParameterProcess{0} -{} - -CTimeSeriesDecompositionDetail::CTrendCRef::CTrendCRef(const TRegression &trend, - double variance, - core_t::TTime timeOrigin, - core_t::TTime lastUpdate, - const TRegressionParameterProcess &process) : - m_Trend{&trend}, - m_Variance{variance}, - m_TimeOrigin{timeOrigin}, - m_LastUpdate{lastUpdate}, - m_ParameterProcess{&process} -{} - -bool CTimeSeriesDecompositionDetail::CTrendCRef::initialized(void) const -{ - return m_Trend != 0; -} - -double CTimeSeriesDecompositionDetail::CTrendCRef::count(void) const -{ - return m_Trend ? m_Trend->count() : 0.0; -} - -TDoubleDoublePr CTimeSeriesDecompositionDetail::CTrendCRef::prediction(core_t::TTime time, double confidence) const -{ - if (!m_Trend) - { - return {0.0, 0.0}; - } - - double m{CRegression::predict(*m_Trend, this->time(time))}; - - if (confidence > 0.0 && m_Variance > 0.0) - { - double sd{std::sqrt(m_Variance) / std::max(m_Trend->count(), 1.0)}; - - try - { - boost::math::normal normal{m, sd}; - double ql{boost::math::quantile(normal, (100.0 - confidence) / 200.0)}; - double qu{boost::math::quantile(normal, (100.0 + confidence) / 200.0)}; - return {ql, qu}; - } - catch (const std::exception &e) - { - LOG_ERROR("Failed calculating confidence interval: " << e.what() - << ", m = " << m - << ", sd = " << sd - << ", confidence = " << confidence); - } - } - return {m, m}; -} - -double CTimeSeriesDecompositionDetail::CTrendCRef::variance(void) const -{ - return m_Variance; -} - -bool CTimeSeriesDecompositionDetail::CTrendCRef::covariances(TMatrix &result) const -{ - return m_Trend ? m_Trend->covariances(m_Variance, result) : false; -} - -double CTimeSeriesDecompositionDetail::CTrendCRef::varianceDueToParameterDrift(core_t::TTime time) const -{ - return m_ParameterProcess ? m_ParameterProcess->predictionVariance(regressionTime(time, m_LastUpdate)) : 0.0; -} - -double CTimeSeriesDecompositionDetail::CTrendCRef::time(core_t::TTime time) const -{ - return m_Trend ? regressionTime(time, m_TimeOrigin) : 0.0; -} - //////// CComponents //////// CTimeSeriesDecompositionDetail::CComponents::CComponents(double decayRate, @@ -1719,6 +1043,8 @@ CTimeSeriesDecompositionDetail::CComponents::CComponents(double decayRate, m_BucketLength{bucketLength}, m_SeasonalComponentSize{seasonalComponentSize}, m_CalendarComponentSize{seasonalComponentSize / 3}, + m_Trend{decayRate}, + m_UsingTrendForPrediction{false}, m_Watcher{0} {} @@ -1728,57 +1054,101 @@ CTimeSeriesDecompositionDetail::CComponents::CComponents(const CComponents &othe m_BucketLength{other.m_BucketLength}, m_SeasonalComponentSize{other.m_SeasonalComponentSize}, m_CalendarComponentSize{other.m_CalendarComponentSize}, - m_Trend{other.m_Trend ? new STrend(*other.m_Trend) : 0}, + m_Trend{other.m_Trend}, m_Seasonal{other.m_Seasonal ? new SSeasonal{*other.m_Seasonal} : 0}, m_Calendar{other.m_Calendar ? new SCalendar{*other.m_Calendar} : 0}, + m_MeanVarianceScale{other.m_MeanVarianceScale}, + m_Moments{other.m_Moments}, + m_MomentsMinusTrend{other.m_MomentsMinusTrend}, + m_UsingTrendForPrediction{other.m_UsingTrendForPrediction}, m_Watcher{0} - {} +{} bool CTimeSeriesDecompositionDetail::CComponents::acceptRestoreTraverser(core::CStateRestoreTraverser &traverser) { - do + if (traverser.name() == VERSION_6_3_TAG) { - const std::string &name{traverser.name()}; - RESTORE(MACHINE_TAG, traverser.traverseSubLevel( - boost::bind(&core::CStateMachine::acceptRestoreTraverser, &m_Machine, _1))); - RESTORE_SETUP_TEARDOWN(TREND_TAG, - m_Trend.reset(new STrend), - traverser.traverseSubLevel(boost::bind(&STrend::acceptRestoreTraverser, - m_Trend.get(), _1)), - /**/) - RESTORE_SETUP_TEARDOWN(SEASONAL_TAG, - m_Seasonal.reset(new SSeasonal), - traverser.traverseSubLevel(boost::bind(&SSeasonal::acceptRestoreTraverser, - m_Seasonal.get(), - m_DecayRate, m_BucketLength, _1)), - /**/) - RESTORE_SETUP_TEARDOWN(CALENDAR_TAG, - m_Calendar.reset(new SCalendar), - traverser.traverseSubLevel(boost::bind(&SCalendar::acceptRestoreTraverser, - m_Calendar.get(), - m_DecayRate, m_BucketLength, _1)), - /**/) + while (traverser.next()) + { + const std::string &name{traverser.name()}; + RESTORE(COMPONENTS_MACHINE_6_3_TAG, traverser.traverseSubLevel( + boost::bind(&core::CStateMachine::acceptRestoreTraverser, &m_Machine, _1))); + RESTORE_BUILT_IN(DECAY_RATE_6_3_TAG, m_DecayRate); + RESTORE(TREND_6_3_TAG, traverser.traverseSubLevel(boost::bind( + &CTrendComponent::acceptRestoreTraverser, &m_Trend, _1))) + RESTORE_SETUP_TEARDOWN(SEASONAL_6_3_TAG, + m_Seasonal.reset(new SSeasonal), + traverser.traverseSubLevel(boost::bind( + &SSeasonal::acceptRestoreTraverser, + m_Seasonal.get(), m_DecayRate, m_BucketLength, _1)), + /**/) + RESTORE_SETUP_TEARDOWN(CALENDAR_6_3_TAG, + m_Calendar.reset(new SCalendar), + traverser.traverseSubLevel(boost::bind( + &SCalendar::acceptRestoreTraverser, + m_Calendar.get(), m_DecayRate, m_BucketLength, _1)), + /**/) + RESTORE(MEAN_VARIANCE_SCALE_6_3_TAG, m_MeanVarianceScale.fromDelimited(traverser.value())) + RESTORE(MOMENTS_6_3_TAG, m_Moments.fromDelimited(traverser.value())); + RESTORE(MOMENTS_MINUS_TREND_6_3_TAG, m_MomentsMinusTrend.fromDelimited(traverser.value())); + RESTORE_BUILT_IN(USING_TREND_FOR_PREDICTION_6_3_TAG, m_UsingTrendForPrediction) + } + + this->decayRate(m_DecayRate); } - while (traverser.next()); + else + { + // There is no version string this is historic state. + do + { + const std::string &name{traverser.name()}; + RESTORE(COMPONENTS_MACHINE_OLD_TAG, traverser.traverseSubLevel( + boost::bind(&core::CStateMachine::acceptRestoreTraverser, &m_Machine, _1))); + RESTORE_SETUP_TEARDOWN(TREND_OLD_TAG, + /**/, + traverser.traverseSubLevel(boost::bind( + upgradeTrendModelToVersion6p3, + m_BucketLength, boost::ref(m_Trend), _1)), + m_UsingTrendForPrediction = true) + RESTORE_SETUP_TEARDOWN(SEASONAL_OLD_TAG, + m_Seasonal.reset(new SSeasonal), + traverser.traverseSubLevel(boost::bind( + &SSeasonal::acceptRestoreTraverser, + m_Seasonal.get(), m_DecayRate, m_BucketLength, _1)), + /**/) + RESTORE_SETUP_TEARDOWN(CALENDAR_OLD_TAG, + m_Calendar.reset(new SCalendar), + traverser.traverseSubLevel(boost::bind( + &SCalendar::acceptRestoreTraverser, + m_Calendar.get(), m_DecayRate, m_BucketLength, _1)), + /**/) + } + while (traverser.next()); + m_MeanVarianceScale.add(1.0, MODEL_WEIGHT_UPGRADING_TO_VERSION_6p3); + } return true; } void CTimeSeriesDecompositionDetail::CComponents::acceptPersistInserter(core::CStatePersistInserter &inserter) const { - inserter.insertLevel(MACHINE_TAG, boost::bind(&core::CStateMachine::acceptPersistInserter, &m_Machine, _1)); - if (m_Trend) - { - inserter.insertLevel(TREND_TAG, boost::bind(&STrend::acceptPersistInserter, m_Trend.get(), _1)); - } + inserter.insertValue(VERSION_6_3_TAG, ""); + inserter.insertLevel(COMPONENTS_MACHINE_6_3_TAG, + boost::bind(&core::CStateMachine::acceptPersistInserter, &m_Machine, _1)); + inserter.insertValue(DECAY_RATE_6_3_TAG, m_DecayRate, core::CIEEE754::E_SinglePrecision); + inserter.insertLevel(TREND_6_3_TAG, boost::bind(&CTrendComponent::acceptPersistInserter, m_Trend, _1)); if (m_Seasonal) { - inserter.insertLevel(SEASONAL_TAG, boost::bind(&SSeasonal::acceptPersistInserter, m_Seasonal.get(), _1)); + inserter.insertLevel(SEASONAL_6_3_TAG, boost::bind(&SSeasonal::acceptPersistInserter, m_Seasonal.get(), _1)); } if (m_Calendar) { - inserter.insertLevel(CALENDAR_TAG, boost::bind(&SCalendar::acceptPersistInserter, m_Calendar.get(), _1)); + inserter.insertLevel(CALENDAR_6_3_TAG, boost::bind(&SCalendar::acceptPersistInserter, m_Calendar.get(), _1)); } + inserter.insertValue(MEAN_VARIANCE_SCALE_6_3_TAG, m_MeanVarianceScale.toDelimited()); + inserter.insertValue(MOMENTS_6_3_TAG, m_Moments.toDelimited()); + inserter.insertValue(MOMENTS_MINUS_TREND_6_3_TAG, m_MomentsMinusTrend.toDelimited()); + inserter.insertValue(USING_TREND_FOR_PREDICTION_6_3_TAG, m_UsingTrendForPrediction); } void CTimeSeriesDecompositionDetail::CComponents::swap(CComponents &other) @@ -1791,6 +1161,10 @@ void CTimeSeriesDecompositionDetail::CComponents::swap(CComponents &other) m_Trend.swap(other.m_Trend); m_Seasonal.swap(other.m_Seasonal); m_Calendar.swap(other.m_Calendar); + std::swap(m_MeanVarianceScale, other.m_MeanVarianceScale); + std::swap(m_Moments, other.m_Moments); + std::swap(m_MomentsMinusTrend, other.m_MomentsMinusTrend); + std::swap(m_UsingTrendForPrediction, other.m_UsingTrendForPrediction); } void CTimeSeriesDecompositionDetail::CComponents::handle(const SAddValue &message) @@ -1799,71 +1173,49 @@ void CTimeSeriesDecompositionDetail::CComponents::handle(const SAddValue &messag { case SC_NORMAL: case SC_NEW_COMPONENTS: - if (m_Trend || m_Seasonal) { this->interpolate(message); core_t::TTime time{message.s_Time}; double value{message.s_Value}; + double trend{message.s_Trend}; + double seasonal{message.s_Seasonal}; + double calendar{message.s_Calendar}; const maths_t::TWeightStyleVec &weightStyles{message.s_WeightStyles}; const maths_t::TDouble4Vec &weights{message.s_Weights}; - TSeasonalComponentPtrVec seasonal; - TCalendarComponentPtrVec calendar; + TSeasonalComponentPtrVec seasonalComponents; + TCalendarComponentPtrVec calendarComponents; TComponentErrorsPtrVec seasonalErrors; TComponentErrorsPtrVec calendarErrors; TDoubleVec deltas; if (m_Seasonal) { - m_Seasonal->componentsErrorsAndDeltas(time, seasonal, seasonalErrors, deltas); + m_Seasonal->componentsErrorsAndDeltas(time, seasonalComponents, seasonalErrors, deltas); } if (m_Calendar) { - m_Calendar->componentsAndErrors(time, calendar, calendarErrors); + m_Calendar->componentsAndErrors(time, calendarComponents, calendarErrors); } double weight{maths_t::countForUpdate(weightStyles, weights)}; - std::size_t m{seasonal.size()}; - std::size_t n{calendar.size()}; + std::size_t m{seasonalComponents.size()}; + std::size_t n{calendarComponents.size()}; - CTrendCRef trend{this->trend()}; - double error; TDoubleVec values(m + n + 1, value); TDoubleVec predictions(m + n); - decompose(trend, seasonal, calendar, deltas, time, error, values, predictions); + double error; + double scale; + decompose(m_Trend, seasonalComponents, calendarComponents, + time, deltas, values, predictions, error, scale); - if (m_Trend) - { - TMeanVarAccumulator moments{ - CBasicStatistics::accumulator(trend.count(), - CBasicStatistics::mean(trend.prediction(time, 0.0)), - m_Trend->s_Variance)}; - moments.add(values[0], weight); - double t{trend.time(time)}; - - // Note this condition can change as a result adding the new - // value we need to check before as well. - bool sufficientHistoryBeforeUpdate{m_Trend->s_Regression.sufficientHistoryToPredict()}; - TVector paramsDrift(m_Trend->s_Regression.parameters(t)); - - m_Trend->s_Regression.add(t, values[0], weight); - m_Trend->s_Variance = CBasicStatistics::maximumLikelihoodVariance(moments); - - paramsDrift -= TVector(m_Trend->s_Regression.parameters(t)); - - if ( sufficientHistoryBeforeUpdate - && m_Trend->s_Regression.sufficientHistoryToPredict() - && m_Trend->s_LastUpdate != UNSET_LAST_UPDATE) - { - double interval{regressionTime(time, m_Trend->s_LastUpdate)}; - m_Trend->s_ParameterProcess.add(interval, paramsDrift, TVector(weight * interval)); - } - m_Trend->s_LastUpdate = time; - } + core_t::TTime observedInterval{m_Trend.observedInterval()}; + + m_Trend.add(time, values[0], weight); for (std::size_t i = 1u; i <= m; ++i) { - CSeasonalComponent *component{seasonal[i - 1]}; + CSeasonalComponent *component{seasonalComponents[i - 1]}; CComponentErrors *error_{seasonalErrors[i - 1]}; double wi{weight / component->time().fractionInWindow()}; component->add(time, values[i], wi); @@ -1871,54 +1223,30 @@ void CTimeSeriesDecompositionDetail::CComponents::handle(const SAddValue &messag } for (std::size_t i = m + 1; i <= m + n; ++i) { - CCalendarComponent *component{calendar[i - m - 1]}; + CCalendarComponent *component{calendarComponents[i - m - 1]}; CComponentErrors *error_{calendarErrors[i - m - 1]}; component->add(time, values[i], weight); error_->add(error, predictions[i - 1], weight); } - } - break; - case SC_FORECASTING: - case SC_DISABLED: - break; - default: - LOG_ERROR("Components in a bad state: " << m_Machine.state()); - this->apply(SC_RESET, message); - break; - } -} - -void CTimeSeriesDecompositionDetail::CComponents::handle(const SDetectedTrend &message) -{ - switch (m_Machine.state()) - { - case SC_NORMAL: - case SC_NEW_COMPONENTS: - { - if (m_Watcher) - { - *m_Watcher = true; - } - LOG_DEBUG("Detected long term trend at '" << message.s_Time << "'"); - core_t::TTime time{message.s_Time}; - core_t::TTime lastTime{message.s_LastTime}; - const TRegression &trend{message.s_Test.trend()}; - double variance{message.s_Test.variance()}; - core_t::TTime origin{message.s_Test.origin()}; + m_MeanVarianceScale.add(scale, weight); + m_Moments.add(value - seasonal - calendar, weight); + m_MomentsMinusTrend.add(value - trend - seasonal - calendar, weight); - m_Trend.reset(new STrend); - m_Trend->s_Regression = trend; - m_Trend->s_Variance = variance; - m_Trend->s_TimeOrigin = origin; - m_Trend->s_LastUpdate = time; - - this->clearComponentErrors(); - this->apply(SC_ADDED_COMPONENTS, message); - this->mediator()->forward(SNewComponents(time, lastTime, SNewComponents::E_Trend)); + if (!m_UsingTrendForPrediction && observedInterval > 6 * m_BucketLength) + { + double v0{CBasicStatistics::variance(m_Moments)}; + double v1{CBasicStatistics::variance(m_MomentsMinusTrend)}; + double df0{CBasicStatistics::count(m_Moments) - 1.0}; + double df1{CBasicStatistics::count(m_MomentsMinusTrend) - m_Trend.parameters()}; + m_UsingTrendForPrediction = + v1 < SIGNIFICANT_VARIANCE_REDUCTION[0] * v0 + && df0 > 0.0 && df1 > 0.0 + && CStatisticalTests::leftTailFTest(v1 / v0, df1, df0) <= MAXIMUM_SIGNIFICANCE; + *m_Watcher = m_UsingTrendForPrediction; + } + } break; - } - case SC_FORECASTING: case SC_DISABLED: break; default: @@ -1928,7 +1256,7 @@ void CTimeSeriesDecompositionDetail::CComponents::handle(const SDetectedTrend &m } } -void CTimeSeriesDecompositionDetail::CComponents::handle(const SDetectedDiurnal &message) +void CTimeSeriesDecompositionDetail::CComponents::handle(const SDetectedSeasonal &message) { if (this->size() + m_SeasonalComponentSize > this->maxSize()) { @@ -1944,67 +1272,17 @@ void CTimeSeriesDecompositionDetail::CComponents::handle(const SDetectedDiurnal { m_Seasonal.reset(new SSeasonal); } - if (m_Watcher) - { - *m_Watcher = true; - } core_t::TTime time{message.s_Time}; core_t::TTime lastTime{message.s_LastTime}; - const CDiurnalPeriodicityTest &test{message.s_Test}; - const CPeriodicityTestResult &result{message.s_Result}; + const CPeriodicityHypothesisTestsResult &result{message.s_Result}; + const CExpandingWindow &window{message.s_Window}; + const TPredictor &predictor{message.s_Predictor}; TSeasonalComponentVec &components{m_Seasonal->s_Components}; TComponentErrorsVec &errors{m_Seasonal->s_PredictionErrors}; - components.erase(std::remove_if(components.begin(), components.end(), - [](const CSeasonalComponent &component) - { - return isDiurnal(component.time().period()); - }), components.end()); - - CPeriodicityTest::TTimeTimePrMeanVarAccumulatorPrVecVec trends; - test.trends(result, trends); - - this->clearComponentErrors(); - this->addSeasonalComponents(test, result, time, components, errors); - this->apply(SC_ADDED_COMPONENTS, message); - this->mediator()->forward(SNewComponents(time, lastTime, SNewComponents::E_DiurnalSeasonal)); - break; - } - case SC_FORECASTING: - case SC_DISABLED: - break; - default: - LOG_ERROR("Components in a bad state: " << m_Machine.state()); - this->apply(SC_RESET, message); - break; - } -} - -void CTimeSeriesDecompositionDetail::CComponents::handle(const SDetectedNonDiurnal &message) -{ - if (this->size() + m_SeasonalComponentSize > this->maxSize()) - { - return; - } - - switch (m_Machine.state()) - { - case SC_NORMAL: - case SC_NEW_COMPONENTS: - { - if (!m_Seasonal) - { - m_Seasonal.reset(new SSeasonal); - } - - core_t::TTime time{message.s_Time}; - core_t::TTime lastTime{message.s_LastTime}; - core_t::TTime period{message.s_Test.periods()[0]}; - const CGeneralPeriodicityTest &test{message.s_Test}; - CPeriodicityTestResult result{message.s_Result}; - if (m_Seasonal->haveComponent(period)) + if (!this->addSeasonalComponents(result, window, predictor, m_Trend, components, errors)) { break; } @@ -2012,21 +1290,14 @@ void CTimeSeriesDecompositionDetail::CComponents::handle(const SDetectedNonDiurn { *m_Watcher = true; } + LOG_DEBUG("Detected seasonal components at " << time); - TSeasonalComponentVec &components{m_Seasonal->s_Components}; - TComponentErrorsVec &errors{m_Seasonal->s_PredictionErrors}; - + m_UsingTrendForPrediction = true; this->clearComponentErrors(); - this->addSeasonalComponents(test, result, time, components, errors); this->apply(SC_ADDED_COMPONENTS, message); - if (message.s_DiscardLongTermTrend) - { - m_Trend.reset(); - } this->mediator()->forward(SNewComponents(time, lastTime, SNewComponents::E_GeneralSeasonal)); break; } - case SC_FORECASTING: case SC_DISABLED: break; default: @@ -2070,7 +1341,6 @@ void CTimeSeriesDecompositionDetail::CComponents::handle(const SDetectedCalendar this->mediator()->forward(SNewComponents(time, lastTime, SNewComponents::E_CalendarCyclic)); break; } - case SC_FORECASTING: case SC_DISABLED: break; default: @@ -2080,7 +1350,7 @@ void CTimeSeriesDecompositionDetail::CComponents::handle(const SDetectedCalendar } } -void CTimeSeriesDecompositionDetail::CComponents::interpolate(const SMessage &message) +void CTimeSeriesDecompositionDetail::CComponents::interpolate(const SMessage &message, bool refine) { core_t::TTime time{message.s_Time}; core_t::TTime lastTime{message.s_LastTime}; @@ -2091,7 +1361,6 @@ void CTimeSeriesDecompositionDetail::CComponents::interpolate(const SMessage &me { case SC_NORMAL: case SC_NEW_COMPONENTS: - case SC_FORECASTING: this->canonicalize(time); if (this->shouldInterpolate(time, lastTime)) { @@ -2099,11 +1368,11 @@ void CTimeSeriesDecompositionDetail::CComponents::interpolate(const SMessage &me if (m_Seasonal) { - m_Seasonal->interpolate(time, lastTime, !this->forecasting()); + m_Seasonal->interpolate(time, lastTime, refine); } if (m_Calendar) { - m_Calendar->interpolate(time, lastTime, !this->forecasting()); + m_Calendar->interpolate(time, lastTime, refine); } this->apply(SC_INTERPOLATED, message); @@ -2121,6 +1390,7 @@ void CTimeSeriesDecompositionDetail::CComponents::interpolate(const SMessage &me void CTimeSeriesDecompositionDetail::CComponents::decayRate(double decayRate) { m_DecayRate = decayRate; + m_Trend.decayRate(decayRate); if (m_Seasonal) { m_Seasonal->decayRate(decayRate); @@ -2131,16 +1401,15 @@ void CTimeSeriesDecompositionDetail::CComponents::decayRate(double decayRate) } } +double CTimeSeriesDecompositionDetail::CComponents::decayRate(void) const +{ + return m_DecayRate; +} + void CTimeSeriesDecompositionDetail::CComponents::propagateForwards(core_t::TTime start, core_t::TTime end) { - if (m_Trend) - { - double time{static_cast(end - start) / static_cast(DAY)}; - double factor{std::exp(-m_DecayRate * time)}; - m_Trend->s_Regression.age(factor); - m_Trend->s_ParameterProcess.age(factor); - } + m_Trend.propagateForwardsByTime(end - start); if (m_Seasonal) { m_Seasonal->propagateForwards(start, end); @@ -2149,28 +1418,24 @@ void CTimeSeriesDecompositionDetail::CComponents::propagateForwards(core_t::TTim { m_Calendar->propagateForwards(start, end); } -} - -bool CTimeSeriesDecompositionDetail::CComponents::forecasting(void) const -{ - return m_Machine.state() == SC_FORECASTING; -} - -void CTimeSeriesDecompositionDetail::CComponents::forecast(void) -{ - this->apply(SC_FORECAST, SMessage()); + double factor{std::exp(-m_DecayRate * static_cast(end - start) + / static_cast(DAY))}; + m_MeanVarianceScale.age(factor); + m_Moments.age(factor); + m_MomentsMinusTrend.age(factor); } bool CTimeSeriesDecompositionDetail::CComponents::initialized(void) const { - return m_Trend ? true : + return m_UsingTrendForPrediction && m_Trend.initialized() ? true : + (m_Seasonal && m_Calendar ? m_Seasonal->initialized() || m_Calendar->initialized() : (m_Seasonal ? m_Seasonal->initialized() : - (m_Calendar ? m_Calendar->initialized() : false)); + (m_Calendar ? m_Calendar->initialized() : false))); } -TTrendCRef CTimeSeriesDecompositionDetail::CComponents::trend(void) const +const CTrendComponent &CTimeSeriesDecompositionDetail::CComponents::trend(void) const { - return m_Trend ? m_Trend->reference() : NO_TREND; + return m_Trend; } const TSeasonalComponentVec &CTimeSeriesDecompositionDetail::CComponents::seasonal(void) const @@ -2183,16 +1448,45 @@ const maths_t::TCalendarComponentVec &CTimeSeriesDecompositionDetail::CComponent return m_Calendar ? m_Calendar->s_Components : NO_CALENDAR_COMPONENTS; } +bool CTimeSeriesDecompositionDetail::CComponents::usingTrendForPrediction(void) const +{ + return m_UsingTrendForPrediction; +} + +CPeriodicityHypothesisTestsConfig CTimeSeriesDecompositionDetail::CComponents::periodicityTestConfig(void) const +{ + CPeriodicityHypothesisTestsConfig result; + for (const auto &component : this->seasonal()) + { + const CSeasonalTime &time{component.time()}; + result.hasDaily( result.hasDaily() || time.period() == DAY); + result.hasWeekend(result.hasWeekend() || time.hasWeekend()); + result.hasWeekly( result.hasWeekly() || time.period() == WEEK); + if (time.hasWeekend()) + { + result.startOfWeek(time.windowRepeatStart()); + } + } + return result; +} + double CTimeSeriesDecompositionDetail::CComponents::meanValue(core_t::TTime time) const { - return this->initialized() ? ( CBasicStatistics::mean(this->trend().prediction(time, 0.0)) + return this->initialized() ? ( (m_UsingTrendForPrediction ? + CBasicStatistics::mean(m_Trend.value(time, 0.0)) : 0.0) + meanOf(&CSeasonalComponent::meanValue, this->seasonal())) : 0.0; } double CTimeSeriesDecompositionDetail::CComponents::meanVariance(void) const { - return this->initialized() ? this->trend().variance() - + meanOf(&CSeasonalComponent::meanVariance, this->seasonal()) : 0.0; + return this->initialized() ? ( (m_UsingTrendForPrediction ? + CBasicStatistics::mean(this->trend().variance(0.0)) : 0.0) + + meanOf(&CSeasonalComponent::meanVariance, this->seasonal())) : 0.0; +} + +double CTimeSeriesDecompositionDetail::CComponents::meanVarianceScale(void) const +{ + return CBasicStatistics::mean(m_MeanVarianceScale); } uint64_t CTimeSeriesDecompositionDetail::CComponents::checksum(uint64_t seed) const @@ -2204,7 +1498,11 @@ uint64_t CTimeSeriesDecompositionDetail::CComponents::checksum(uint64_t seed) co seed = CChecksum::calculate(seed, m_CalendarComponentSize); seed = CChecksum::calculate(seed, m_Trend); seed = CChecksum::calculate(seed, m_Seasonal); - return CChecksum::calculate(seed, m_Calendar); + seed = CChecksum::calculate(seed, m_Calendar); + seed = CChecksum::calculate(seed, m_MeanVarianceScale); + seed = CChecksum::calculate(seed, m_Moments); + seed = CChecksum::calculate(seed, m_MomentsMinusTrend); + return CChecksum::calculate(seed, m_UsingTrendForPrediction); } void CTimeSeriesDecompositionDetail::CComponents::debugMemoryUsage(core::CMemoryUsage::TMemoryUsagePtr mem) const @@ -2232,40 +1530,86 @@ std::size_t CTimeSeriesDecompositionDetail::CComponents::maxSize(void) const return MAXIMUM_COMPONENTS * m_SeasonalComponentSize; } -void CTimeSeriesDecompositionDetail::CComponents::addSeasonalComponents(const CPeriodicityTest &test, - const CPeriodicityTestResult &result, - core_t::TTime time, +bool CTimeSeriesDecompositionDetail::CComponents::addSeasonalComponents(const CPeriodicityHypothesisTestsResult &result, + const CExpandingWindow &window, + const TPredictor &predictor, + CTrendComponent &trend, TSeasonalComponentVec &components, TComponentErrorsVec &errors) const { - using TSeasonalTimePtr = boost::scoped_ptr; + using TSeasonalTimePtr = boost::shared_ptr; + using TSeasonalTimePtrVec = std::vector; - double bucketLength{static_cast(m_BucketLength)}; + TSeasonalTimePtrVec newSeasonalTimes; - CPeriodicityTest::TTimeTimePrMeanVarAccumulatorPrVecVec trends; - test.trends(result, trends); - components.reserve(components.size() + result.components().size()); - for (std::size_t i = 0u; i < result.components().size(); ++i) + for (const auto &candidate_ : result.components()) { - TSeasonalTimePtr seasonalTime(test.seasonalTime(result.components()[i])); - components.emplace_back(*seasonalTime, m_SeasonalComponentSize, - m_DecayRate, bucketLength, CSplineTypes::E_Natural); - components.back().initialize(seasonalTime->startOfWindowRepeat(time), time, trends[i]); + TSeasonalTimePtr seasonalTime(candidate_.seasonalTime()); + if (std::find_if(components.begin(), components.end(), + [&seasonalTime](const CSeasonalComponent &component) + { + return component.time().excludes(*seasonalTime); + }) == components.end()) + { + LOG_DEBUG("Detected '" << candidate_.s_Description << "'"); + newSeasonalTimes.push_back(seasonalTime); + } } - errors.resize(components.size()); - LOG_DEBUG("Detected " << test.print(result)); - LOG_DEBUG("Estimated new periods at '" << time << "'"); - LOG_TRACE("# components = " << components.size()); + if (newSeasonalTimes.size() > 0) + { + for (const auto &seasonalTime : newSeasonalTimes) + { + components.erase(std::remove_if(components.begin(), components.end(), + [&seasonalTime](const CSeasonalComponent &component) + { + return seasonalTime->excludes(component.time()); + }), components.end()); + } + + std::sort(newSeasonalTimes.begin(), newSeasonalTimes.end(), maths::COrderings::SLess()); - COrderings::simultaneousSort(components, errors, - [](const CSeasonalComponent &lhs, const CSeasonalComponent &rhs) - { - return lhs.time().period() < rhs.time().period(); - }); + TFloatMeanAccumulatorVec values; + for (const auto &seasonalTime : newSeasonalTimes) + { + values = window.valuesMinusPrediction(predictor); + components.emplace_back(*seasonalTime, m_SeasonalComponentSize, + m_DecayRate, static_cast(m_BucketLength), + CSplineTypes::E_Natural); + components.back().initialize(window.startTime(), window.endTime(), values); + components.back().interpolate(CIntegerTools::floor(window.endTime(), + seasonalTime->period())); + } + + CTrendComponent windowTrend{trend.defaultDecayRate()}; + values = window.valuesMinusPrediction(predictor); + core_t::TTime time{window.startTime() + window.bucketLength() / 2}; + for (const auto &value : values) + { + // Because we now test before the window is fully compressed + // we can get a run of unset values at the end of the window, + // we should just ignore these. + if (CBasicStatistics::count(value) > 0.0) + { + windowTrend.add(time, CBasicStatistics::mean(value), CBasicStatistics::count(value)); + windowTrend.propagateForwardsByTime(window.bucketLength()); + } + time += window.bucketLength(); + } + trend.swap(windowTrend); + + errors.resize(components.size()); + COrderings::simultaneousSort(components, errors, + [](const CSeasonalComponent &lhs, const CSeasonalComponent &rhs) + { + return lhs.time() < rhs.time(); + }); + } + + return newSeasonalTimes.size() > 0; } -void CTimeSeriesDecompositionDetail::CComponents::addCalendarComponent(const CCalendarFeature &feature, +bool CTimeSeriesDecompositionDetail::CComponents::addCalendarComponent(const CCalendarFeature &feature, core_t::TTime time, maths_t::TCalendarComponentVec &components, TComponentErrorsVec &errors) const @@ -2275,23 +1619,22 @@ void CTimeSeriesDecompositionDetail::CComponents::addCalendarComponent(const CCa m_DecayRate, bucketLength, CSplineTypes::E_Natural); components.back().initialize(); errors.resize(components.size()); - - LOG_DEBUG("Detected feature '" << feature.print() << "'"); - LOG_DEBUG("Estimated new calendar component at '" << time << "'"); + LOG_DEBUG("Detected feature '" << feature.print() << "' at " << time); + return true; } void CTimeSeriesDecompositionDetail::CComponents::clearComponentErrors(void) { if (m_Seasonal) { - for (auto &&errors : m_Seasonal->s_PredictionErrors) + for (auto &errors : m_Seasonal->s_PredictionErrors) { errors.clear(); } } if (m_Calendar) { - for (auto &&errors : m_Calendar->s_PredictionErrors) + for (auto &errors : m_Calendar->s_PredictionErrors) { errors.clear(); } @@ -2302,7 +1645,7 @@ void CTimeSeriesDecompositionDetail::CComponents::apply(std::size_t symbol, cons { if (symbol == SC_RESET) { - m_Trend.reset(); + m_Trend.clear(); m_Seasonal.reset(); m_Calendar.reset(); } @@ -2318,12 +1661,11 @@ void CTimeSeriesDecompositionDetail::CComponents::apply(std::size_t symbol, cons switch (state) { case SC_NORMAL: - case SC_FORECASTING: case SC_NEW_COMPONENTS: this->interpolate(message); break; case SC_DISABLED: - m_Trend.reset(); + m_Trend.clear(); m_Seasonal.reset(); m_Calendar.reset(); break; @@ -2338,29 +1680,18 @@ void CTimeSeriesDecompositionDetail::CComponents::apply(std::size_t symbol, cons bool CTimeSeriesDecompositionDetail::CComponents::shouldInterpolate(core_t::TTime time, core_t::TTime last) { - std::size_t state{m_Machine.state()}; - if (state == SC_NEW_COMPONENTS) - { - return true; - } - return this->forecasting() + return m_Machine.state() == SC_NEW_COMPONENTS || (m_Seasonal && m_Seasonal->shouldInterpolate(time, last)) || (m_Calendar && m_Calendar->shouldInterpolate(time, last)); } void CTimeSeriesDecompositionDetail::CComponents::shiftOrigin(core_t::TTime time) { - if (!this->forecasting()) + time -= static_cast(static_cast(DAY) / m_DecayRate / 2.0); + m_Trend.shiftOrigin(time); + if (m_Seasonal) { - time -= static_cast(static_cast(DAY) / m_DecayRate / 2.0); - if (m_Trend) - { - m_Trend->shiftOrigin(time); - } - if (m_Seasonal) - { - m_Seasonal->shiftOrigin(time); - } + m_Seasonal->shiftOrigin(time); } } @@ -2377,16 +1708,16 @@ void CTimeSeriesDecompositionDetail::CComponents::canonicalize(core_t::TTime tim m_Calendar.reset(); } - if (m_Seasonal && m_Trend) + if (m_Seasonal) { TSeasonalComponentVec &seasonal{m_Seasonal->s_Components}; TTimeTimePrDoubleFMap slope; slope.reserve(seasonal.size()); - for (auto &&component : seasonal) + for (auto &component : seasonal) { - if (component.sufficientHistoryToPredict(time)) + if (component.slopeAccurate(time)) { const CSeasonalTime &time_{component.time()}; double si{component.slope()}; @@ -2396,7 +1727,7 @@ void CTimeSeriesDecompositionDetail::CComponents::canonicalize(core_t::TTime tim } LOG_TRACE("slope = " << core::CContainerPrinter::print(slope)); - shiftSlope(slope, m_Trend->s_Regression); + shiftSlope(slope, m_DecayRate, m_Trend); } } @@ -2506,91 +1837,52 @@ double CTimeSeriesDecompositionDetail::CComponents::CComponentErrors::winsorise( std::min(squareError, 36.0 * CBasicStatistics::mean(variance)) : squareError; } -CTimeSeriesDecompositionDetail::CComponents::STrend::STrend(void) : - s_Variance{0.0}, s_TimeOrigin{0}, s_LastUpdate{UNSET_LAST_UPDATE} -{} - -bool CTimeSeriesDecompositionDetail::CComponents::STrend::acceptRestoreTraverser(core::CStateRestoreTraverser &traverser) -{ - s_LastUpdate = UNSET_LAST_UPDATE; - do - { - const std::string &name{traverser.name()}; - RESTORE(REGRESSION_TAG, traverser.traverseSubLevel( - boost::bind(&TRegression::acceptRestoreTraverser, &s_Regression, _1))) - RESTORE_BUILT_IN(VARIANCE_TAG, s_Variance) - RESTORE_BUILT_IN(TIME_ORIGIN_TAG, s_TimeOrigin) - RESTORE_BUILT_IN(LAST_UPDATE_TAG, s_LastUpdate) - RESTORE(PARAMETER_PROCESS_TAG, - traverser.traverseSubLevel( - boost::bind(&TRegressionParameterProcess::acceptRestoreTraverser, &s_ParameterProcess, _1))) - } - while (traverser.next()); - return true; -} - -void CTimeSeriesDecompositionDetail::CComponents::STrend::acceptPersistInserter(core::CStatePersistInserter &inserter) const -{ - inserter.insertLevel(REGRESSION_TAG, boost::bind(&TRegression::acceptPersistInserter, &s_Regression, _1)); - inserter.insertValue(VARIANCE_TAG, s_Variance, core::CIEEE754::E_SinglePrecision); - inserter.insertValue(TIME_ORIGIN_TAG, s_TimeOrigin); - inserter.insertValue(LAST_UPDATE_TAG, s_LastUpdate); - inserter.insertLevel(PARAMETER_PROCESS_TAG, - boost::bind(&TRegressionParameterProcess::acceptPersistInserter, &s_ParameterProcess, _1)); -} - -CTimeSeriesDecompositionDetail::CTrendCRef -CTimeSeriesDecompositionDetail::CComponents::STrend::reference(void) const -{ - return CTrendCRef(s_Regression, s_Variance, s_TimeOrigin, s_LastUpdate, s_ParameterProcess); -} - -void CTimeSeriesDecompositionDetail::CComponents::STrend::shiftOrigin(core_t::TTime time) -{ - time = CIntegerTools::floor(time, WEEK); - double shift{regressionTime(time, s_TimeOrigin)}; - if (shift > 0.0) - { - s_Regression.shiftAbscissa(-shift); - s_TimeOrigin = time; - } -} - -uint64_t CTimeSeriesDecompositionDetail::CComponents::STrend::checksum(uint64_t seed) const -{ - seed = CChecksum::calculate(seed, s_Regression); - seed = CChecksum::calculate(seed, s_Variance); - return CChecksum::calculate(seed, s_TimeOrigin); -} - bool CTimeSeriesDecompositionDetail::CComponents::SSeasonal::acceptRestoreTraverser(double decayRate, - core_t::TTime bucketLength, + core_t::TTime bucketLength_, core::CStateRestoreTraverser &traverser) { - do + double bucketLength{static_cast(bucketLength_)}; + if (traverser.name() == VERSION_6_3_TAG) { - const std::string &name{traverser.name()}; - RESTORE_NO_ERROR(COMPONENT_TAG, s_Components.emplace_back( - decayRate, static_cast(bucketLength), traverser)) - RESTORE(ERRORS_TAG, core::CPersistUtils::restore(ERRORS_TAG, s_PredictionErrors, traverser)) + while (traverser.next()) + { + const std::string &name{traverser.name()}; + RESTORE_NO_ERROR(COMPONENT_6_3_TAG, s_Components.emplace_back( + decayRate, bucketLength, traverser)) + RESTORE(ERRORS_6_3_TAG, core::CPersistUtils::restore( + ERRORS_6_3_TAG, s_PredictionErrors, traverser)) + } + } + else + { + // There is no version string this is historic state. + do + { + const std::string &name{traverser.name()}; + RESTORE_NO_ERROR(COMPONENT_OLD_TAG, s_Components.emplace_back( + decayRate, bucketLength, traverser)) + RESTORE(ERRORS_OLD_TAG, core::CPersistUtils::restore( + ERRORS_OLD_TAG, s_PredictionErrors, traverser)) + } + while (traverser.next()); } - while (traverser.next()); return true; } void CTimeSeriesDecompositionDetail::CComponents::SSeasonal::acceptPersistInserter(core::CStatePersistInserter &inserter) const { + inserter.insertValue(VERSION_6_3_TAG, ""); for (const auto &component : s_Components) { - inserter.insertLevel(COMPONENT_TAG, boost::bind( + inserter.insertLevel(COMPONENT_6_3_TAG, boost::bind( &CSeasonalComponent::acceptPersistInserter, &component, _1)); } - core::CPersistUtils::persist(ERRORS_TAG, s_PredictionErrors, inserter); + core::CPersistUtils::persist(ERRORS_6_3_TAG, s_PredictionErrors, inserter); } void CTimeSeriesDecompositionDetail::CComponents::SSeasonal::decayRate(double decayRate) { - for (auto &&component : s_Components) + for (auto &component : s_Components) { component.decayRate(decayRate); } @@ -2624,25 +1916,12 @@ std::size_t CTimeSeriesDecompositionDetail::CComponents::SSeasonal::size(void) c return result; } -bool CTimeSeriesDecompositionDetail::CComponents::SSeasonal::haveComponent(core_t::TTime period) const -{ - for (const auto &component : s_Components) - { - core_t::TTime reference{component.time().period()}; - if (std::abs(period - reference) < reference / 10) - { - return true; - } - } - return false; -} - void CTimeSeriesDecompositionDetail::CComponents::SSeasonal::componentsErrorsAndDeltas(core_t::TTime time, TSeasonalComponentPtrVec &components, TComponentErrorsPtrVec &errors, TDoubleVec &deltas) { - std::size_t n = s_Components.size(); + std::size_t n{s_Components.size()}; components.reserve(n); errors.reserve(n); @@ -2659,8 +1938,19 @@ void CTimeSeriesDecompositionDetail::CComponents::SSeasonal::componentsErrorsAnd deltas.resize(components.size(), 0.0); for (std::size_t i = 1u; i < components.size(); ++i) { - core_t::TTime period{components[i-1]->time().period()}; - deltas[i-1] = 0.2 * components[i]->differenceFromMean(time, period); + int j{static_cast(i - 1)}; + for (core_t::TTime period{components[i]->time().period()}; j > -1; --j) + { + core_t::TTime period_{components[j]->time().period()}; + if (period % period_ == 0) + { + double value{CBasicStatistics::mean(components[j]->value(time, 0.0))}; + double delta{0.2 * components[i]->delta(time, period_, value)}; + deltas[j] += delta; + deltas[i] -= delta; + break; + } + } } } @@ -2684,7 +1974,7 @@ void CTimeSeriesDecompositionDetail::CComponents::SSeasonal::interpolate(core_t: core_t::TTime last, bool refine) { - for (auto &&component : s_Components) + for (auto &component : s_Components) { core_t::TTime period{component.time().period()}; core_t::TTime a{CIntegerTools::floor(last, period)}; @@ -2749,7 +2039,7 @@ bool CTimeSeriesDecompositionDetail::CComponents::SSeasonal::prune(core_t::TTime CSetTools::simultaneousRemoveIf( remove, s_Components, s_PredictionErrors, [](bool remove_) { return remove_; }); - for (auto &&shift : shifts) + for (auto &shift : shifts) { if (windowed.count(shift.first) > 0) { @@ -2801,7 +2091,7 @@ bool CTimeSeriesDecompositionDetail::CComponents::SSeasonal::prune(core_t::TTime void CTimeSeriesDecompositionDetail::CComponents::SSeasonal::shiftOrigin(core_t::TTime time) { - for (auto &&component : s_Components) + for (auto &component : s_Components) { component.shiftOrigin(time); } @@ -2826,33 +2116,51 @@ std::size_t CTimeSeriesDecompositionDetail::CComponents::SSeasonal::memoryUsage( } bool CTimeSeriesDecompositionDetail::CComponents::SCalendar::acceptRestoreTraverser(double decayRate, - core_t::TTime bucketLength, + core_t::TTime bucketLength_, core::CStateRestoreTraverser &traverser) { - do + double bucketLength{static_cast(bucketLength_)}; + if (traverser.name() == VERSION_6_3_TAG) { - const std::string &name{traverser.name()}; - RESTORE_NO_ERROR(COMPONENT_TAG, s_Components.emplace_back( - decayRate, static_cast(bucketLength), traverser)) - RESTORE(ERRORS_TAG, core::CPersistUtils::restore(ERRORS_TAG, s_PredictionErrors, traverser)) + while (traverser.next()) + { + const std::string &name{traverser.name()}; + RESTORE_NO_ERROR(COMPONENT_6_3_TAG, s_Components.emplace_back( + decayRate, bucketLength, traverser)) + RESTORE(ERRORS_6_3_TAG, core::CPersistUtils::restore( + ERRORS_6_3_TAG, s_PredictionErrors, traverser)) + } + } + else + { + // There is no version string this is historic state. + do + { + const std::string &name{traverser.name()}; + RESTORE_NO_ERROR(COMPONENT_OLD_TAG, s_Components.emplace_back( + decayRate, bucketLength, traverser)) + RESTORE(ERRORS_OLD_TAG, core::CPersistUtils::restore( + ERRORS_OLD_TAG, s_PredictionErrors, traverser)) + } + while (traverser.next()); } - while (traverser.next()); return true; } void CTimeSeriesDecompositionDetail::CComponents::SCalendar::acceptPersistInserter(core::CStatePersistInserter &inserter) const { + inserter.insertValue(VERSION_6_3_TAG, ""); for (const auto &component : s_Components) { - inserter.insertLevel(COMPONENT_TAG, boost::bind( + inserter.insertLevel(COMPONENT_6_3_TAG, boost::bind( &CCalendarComponent::acceptPersistInserter, &component, _1)); } - core::CPersistUtils::persist(ERRORS_TAG, s_PredictionErrors, inserter); + core::CPersistUtils::persist(ERRORS_6_3_TAG, s_PredictionErrors, inserter); } void CTimeSeriesDecompositionDetail::CComponents::SCalendar::decayRate(double decayRate) { - for (auto &&component : s_Components) + for (auto &component : s_Components) { component.decayRate(decayRate); } @@ -2931,7 +2239,7 @@ void CTimeSeriesDecompositionDetail::CComponents::SCalendar::interpolate(core_t: core_t::TTime last, bool refine) { - for (auto &&component : s_Components) + for (auto &component : s_Components) { CCalendarFeature feature = component.feature(); if (!feature.inWindow(time) && feature.inWindow(last)) diff --git a/lib/maths/CTimeSeriesDecompositionInterface.cc b/lib/maths/CTimeSeriesDecompositionInterface.cc deleted file mode 100644 index 9be8e7b291..0000000000 --- a/lib/maths/CTimeSeriesDecompositionInterface.cc +++ /dev/null @@ -1,152 +0,0 @@ -/* - * ELASTICSEARCH CONFIDENTIAL - * - * Copyright (c) 2016 Elasticsearch BV. All Rights Reserved. - * - * Notice: this software, and all information contained - * therein, is the exclusive property of Elasticsearch BV - * and its licensors, if any, and is protected under applicable - * domestic and foreign law, and international treaties. - * - * Reproduction, republication or distribution without the - * express written consent of Elasticsearch BV is - * strictly prohibited. - */ - -#include - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include - -namespace ml -{ -namespace maths -{ -namespace -{ - -using TDoubleVec = std::vector; -using TDouble4Vec = core::CSmallVector; -const std::size_t NUMBER_SAMPLES{100u}; - -} - -bool initializePrior(core_t::TTime bucketLength, - double learnRate, - const CTimeSeriesDecompositionInterface &decomposition, - CPrior &prior) -{ - using TDouble4Vec1Vec = core::CSmallVector; - - if (!decomposition.initialized()) - { - return false; - } - - // Note our estimate of variance is approximate so we use a - // small tolerance when generating the samples with which to - // initialize the prior. - - double variance{2.25 * decomposition.meanVariance()}; - double sd{std::sqrt(variance)}; - - TDoubleVec samples; - CPRNG::CXorOShiro128Plus rng; - CSampling::normalSample(rng, 0.0, variance, NUMBER_SAMPLES, samples); - std::sort(samples.begin(), samples.end()); - - double weight{0.2 * static_cast(core::constants::DAY) - / static_cast(std::min(bucketLength, core::constants::HOUR)) - / static_cast(NUMBER_SAMPLES) * learnRate}; - TDouble4Vec1Vec weights(samples.size(), TDouble4Vec(1, weight)); - - // Use an offset which ensures that the samples lie in the - // supports of all priors. - double offset{std::max(-samples[0] + 0.01 * sd, prior.offsetMargin())}; - prior.setToNonInformative(offset, prior.decayRate()); - prior.removeModels(CPrior::CModelFilter().remove(CPrior::E_Poisson)); - - LOG_DEBUG("sd = " << sd << ", offset = " << offset << ", weight = " << weight); - - prior.addSamples(CConstantWeights::COUNT, samples, weights); - - return true; -} - -bool initializePrior(core_t::TTime bucketLength, - double learnRate, - const TDecompositionPtr10Vec &decomposition, - CMultivariatePrior &prior) -{ - using TDoubleVecVec = std::vector; - using TDouble10Vec = core::CSmallVector; - using TDouble10Vec4Vec = core::CSmallVector; - using TDouble10Vec4Vec1Vec = core::CSmallVector; - - if (decomposition.empty()) - { - return false; - } - - // Note our estimate of variance is approximate so we use a - // small tolerance when generating the samples with which to - // initialize the prior. - - std::size_t dimension{decomposition.size()}; - - TDoubleVec mean(dimension, 0.0); - TDoubleVecVec covariance(dimension, TDoubleVec(dimension, 0.0)); - double sd{0.0}; - for (std::size_t i = 0u; i < dimension; ++i) - { - if (!decomposition[i]->initialized()) - { - return false; - } - covariance[i][i] = 2.25 * decomposition[i]->meanVariance(); - sd += covariance[i][i]; - } - sd /= static_cast(dimension); - sd = std::sqrt(sd); - - TDoubleVecVec samples; - samples.reserve(NUMBER_SAMPLES); - CPRNG::CXorOShiro128Plus rng; - CSampling::multivariateNormalSample(rng, mean, covariance, NUMBER_SAMPLES, samples); - std::sort(samples.begin(), samples.end()); - - double weight{0.2 * static_cast(core::constants::DAY) - / static_cast(std::min(bucketLength, core::constants::HOUR)) - / static_cast(NUMBER_SAMPLES) * learnRate}; - TDouble10Vec4Vec1Vec weights(samples.size(), TDouble10Vec4Vec(1, TDouble10Vec(dimension, weight))); - - // Use an offset which ensures that the samples lie in the - // supports of all priors. - - double offset{prior.offsetMargin()}; - for (auto &&sample : samples) - { - double min = *std::min_element(sample.begin(), sample.end()); - offset = std::max(offset, -min + 0.01 * sd); - } - prior.setToNonInformative(offset, prior.decayRate()); - - LOG_DEBUG("sd = " << sd << ", offset = " << offset << ", weight = " << weight); - - prior.addSamples(CConstantWeights::COUNT, samples, weights); - - return true; -} - -} -} diff --git a/lib/maths/CTimeSeriesDecompositionStub.cc b/lib/maths/CTimeSeriesDecompositionStub.cc index f99ce49e6c..ee5ab9ea70 100644 --- a/lib/maths/CTimeSeriesDecompositionStub.cc +++ b/lib/maths/CTimeSeriesDecompositionStub.cc @@ -40,10 +40,6 @@ double CTimeSeriesDecompositionStub::decayRate(void) const return 0.0; } -void CTimeSeriesDecompositionStub::forForecasting(void) -{ -} - bool CTimeSeriesDecompositionStub::initialized(void) const { return false; @@ -61,25 +57,29 @@ void CTimeSeriesDecompositionStub::propagateForwardsTo(core_t::TTime /*time*/) { } -bool CTimeSeriesDecompositionStub::testAndInterpolate(core_t::TTime /*time*/) -{ - return false; -} - double CTimeSeriesDecompositionStub::mean(core_t::TTime /*time*/) const { return 0.0; } maths_t::TDoubleDoublePr CTimeSeriesDecompositionStub::baseline(core_t::TTime /*time*/, - double /*predictionConfidence*/, - double /*forecastConfidence*/, - EComponents /*components*/, + double /*confidence*/, + int /*components*/, bool /*smooth*/) const { return {0.0, 0.0}; } +void CTimeSeriesDecompositionStub::forecast(core_t::TTime /*startTime*/, + core_t::TTime /*endTime*/, + core_t::TTime /*step*/, + double /*confidence*/, + double /*minimumScale*/, + TDouble3VecVec &result) +{ + result.clear(); +} + double CTimeSeriesDecompositionStub::detrend(core_t::TTime /*time*/, double value, double /*confidence*/) const diff --git a/lib/maths/CTimeSeriesModel.cc b/lib/maths/CTimeSeriesModel.cc index 2da13b926d..6f563cd1d0 100644 --- a/lib/maths/CTimeSeriesModel.cc +++ b/lib/maths/CTimeSeriesModel.cc @@ -82,17 +82,13 @@ double computeWinsorisationWeight(const CPrior &prior, double derate, double sca double lowerBound; double upperBound; if (!prior.minusLogJointCdf(CConstantWeights::SEASONAL_VARIANCE, - TDouble1Vec{value}, - TDouble4Vec1Vec{TDouble4Vec{scale}}, - lowerBound, upperBound)) + {value}, {{scale}}, lowerBound, upperBound)) { return 1.0; } if ( upperBound < MINUS_LOG_TOLERANCE && !prior.minusLogJointCdfComplement(CConstantWeights::SEASONAL_VARIANCE, - TDouble1Vec{value}, - TDouble4Vec1Vec{TDouble4Vec{scale}}, - lowerBound, upperBound)) + {value}, {{scale}}, lowerBound, upperBound)) { return 1.0; } @@ -173,22 +169,37 @@ enum EDecayRateController }; // Models -const std::string ID_TAG{"a"}; -const std::string CONTROLLER_TAG{"b"}; -const std::string TREND_TAG{"c"}; -const std::string PRIOR_TAG{"d"}; -const std::string ANOMALY_MODEL_TAG{"e"}; -//const std::string MEAN_DECAY_RATES_TAG{"f"}; No longer used. -const std::string IS_NON_NEGATIVE_TAG{"g"}; -const std::string IS_FORECASTABLE_TAG{"h"}; + +// Version 6.3 +const std::string VERSION_6_3_TAG("6.3"); +const std::string ID_6_3_TAG{"a"}; +const std::string IS_NON_NEGATIVE_6_3_TAG{"b"}; +const std::string IS_FORECASTABLE_6_3_TAG{"c"}; +const std::string RNG_6_3_TAG{"d"}; +const std::string CONTROLLER_6_3_TAG{"e"}; +const std::string TREND_6_3_TAG{"f"}; +const std::string PRIOR_6_3_TAG{"g"}; +const std::string ANOMALY_MODEL_6_3_TAG{"h"}; +const std::string SLIDING_WINDOW_6_3_TAG{"i"}; +// Version < 6.3 +const std::string ID_OLD_TAG{"a"}; +const std::string CONTROLLER_OLD_TAG{"b"}; +const std::string TREND_OLD_TAG{"c"}; +const std::string PRIOR_OLD_TAG{"d"}; +const std::string ANOMALY_MODEL_OLD_TAG{"e"}; +const std::string IS_NON_NEGATIVE_OLD_TAG{"g"}; +const std::string IS_FORECASTABLE_OLD_TAG{"h"}; + // Anomaly model const std::string MEAN_ERROR_TAG{"a"}; const std::string ANOMALIES_TAG{"b"}; +const std::string PRIOR_TAG{"d"}; // Anomaly model nested const std::string TAG_TAG{"a"}; const std::string OPEN_TIME_TAG{"b"}; const std::string SIGN_TAG{"c"}; const std::string MEAN_ERROR_NORM_TAG{"d"}; + // Correlations const std::string K_MOST_CORRELATED_TAG{"a"}; const std::string CORRELATED_LOOKUP_TAG{"b"}; @@ -201,17 +212,14 @@ const std::string CORRELATION_TAG{"d"}; const std::size_t MAXIMUM_CORRELATIONS{5000}; const double MINIMUM_CORRELATE_PRIOR_SAMPLE_COUNT{24.0}; +const std::size_t SLIDING_WINDOW_SIZE{12}; const TSize10Vec NOTHING_TO_MARGINALIZE; const TSizeDoublePr10Vec NOTHING_TO_CONDITION; namespace forecast { const std::string INFO_INSUFFICIENT_HISTORY("Insufficient history to forecast"); -const std::string INFO_INSUFFICIENT_ACCURACY_STOPPING_EARLY( - "Unable to forecast for entire duration as confidence fell outside of acceptable limits"); const std::string ERROR_MULTIVARIATE("Forecast not supported for multivariate features"); -const double RELATIVE_ERROR_TOLERANCE{0.1}; -const double MAXIMUM_ERROR_GROWTH{10.0}; } } @@ -306,8 +314,8 @@ class CTimeSeriesAnomalyModel //! Get the feature vector for this anomaly. TDouble10Vec features(core_t::TTime time) const { - return TDouble10Vec{static_cast(time - m_OpenTime), - CBasicStatistics::mean(m_MeanErrorNorm)}; + return {static_cast(time - m_OpenTime), + CBasicStatistics::mean(m_MeanErrorNorm)}; } //! Compute a checksum for this object. @@ -376,7 +384,7 @@ class CTimeSeriesAnomalyModel std::size_t index(anomaly.positive() ? 0 : 1); TDouble10Vec1Vec features{anomaly.features(this->scale(time))}; m_Priors[index].addSamples(CConstantWeights::COUNT, features, - TDouble10Vec4Vec1Vec{TDouble10Vec4Vec{TDouble10Vec(2, weight)}}); + {{TDouble10Vec(2, weight)}}); } //! Get the scaled time. @@ -430,7 +438,7 @@ void CTimeSeriesAnomalyModel::updateAnomaly(const CModelProbabilityParams ¶m [](double n, double x) { return n + x*x; }))); double scale{CBasicStatistics::mean(m_MeanError)}; - for (auto &&error : errors) + for (auto &error : errors) { error = scale == 0.0 ? 1.0 : error / scale; } @@ -467,7 +475,7 @@ void CTimeSeriesAnomalyModel::sampleAnomaly(const CModelProbabilityParams ¶m void CTimeSeriesAnomalyModel::reset(void) { m_MeanError = TMeanAccumulator(); - for (auto &&prior : m_Priors) + for (auto &prior : m_Priors) { prior = TMultivariateNormalConjugate::nonInformativePrior(maths_t::E_ContinuousData, prior.decayRate()); } @@ -584,6 +592,7 @@ CUnivariateTimeSeriesModel::CUnivariateTimeSeriesModel(const CModelParams ¶m boost::make_shared(params.bucketLength(), params.decayRate()) : TAnomalyModelPtr()), + m_SlidingWindow(SLIDING_WINDOW_SIZE), m_Correlations(0) { if (controllers) @@ -596,6 +605,7 @@ CUnivariateTimeSeriesModel::CUnivariateTimeSeriesModel(const SModelRestoreParams core::CStateRestoreTraverser &traverser): CModel(params.s_Params), m_IsForecastable(false), + m_SlidingWindow(SLIDING_WINDOW_SIZE), m_Correlations(0) { traverser.traverseSubLevel(boost::bind(&CUnivariateTimeSeriesModel::acceptRestoreTraverser, @@ -632,10 +642,7 @@ CUnivariateTimeSeriesModel *CUnivariateTimeSeriesModel::cloneForPersistence(void CUnivariateTimeSeriesModel *CUnivariateTimeSeriesModel::cloneForForecast(void) const { - CUnivariateTimeSeriesModel *forecastModel{new CUnivariateTimeSeriesModel{*this, m_Id}}; - forecastModel->m_Prior->forForecasting(); - forecastModel->m_Trend->forForecasting(); - return forecastModel; + return new CUnivariateTimeSeriesModel{*this, m_Id}; } bool CUnivariateTimeSeriesModel::isForecastPossible(void) const @@ -673,7 +680,7 @@ void CUnivariateTimeSeriesModel::addBucketValue(const TTimeDouble2VecSizeTrVec & for (const auto &value : values) { m_Prior->adjustOffset(CConstantWeights::COUNT, - TDouble1Vec{m_Trend->detrend(value.first, value.second[0], 0.0)}, + {m_Trend->detrend(value.first, value.second[0], 0.0)}, CConstantWeights::SINGLE_UNIT); } } @@ -688,21 +695,35 @@ CUnivariateTimeSeriesModel::addSamples(const CModelAddSamplesParams ¶ms, } using TMeanAccumulator = CBasicStatistics::SSampleMean::TAccumulator; + using TOptionalTimeDoublePr = boost::optional; + + TSizeVec valueorder(samples.size()); + std::iota(valueorder.begin(), valueorder.end(), 0); + std::stable_sort(valueorder.begin(), valueorder.end(), + [&samples](std::size_t lhs, std::size_t rhs) + { + return samples[lhs].second < samples[rhs].second; + }); + + TOptionalTimeDoublePr randomSample; + + double p{SLIDING_WINDOW_SIZE * static_cast(this->params().bucketLength()) + / static_cast(core::constants::DAY)}; + if (p >= 1.0 || CSampling::uniformSample(m_Rng, 0.0, 1.0) < p) + { + std::size_t i{CSampling::uniformSample(m_Rng, 0, samples.size())}; + randomSample.reset({samples[valueorder[i]].first, samples[valueorder[i]].second[0]}); + } m_IsNonNegative = params.isNonNegative(); EUpdateResult result{this->updateTrend(params.weightStyles(), samples, params.trendWeights())}; - TMeanAccumulator averageTime; for (auto &sample : samples) { - core_t::TTime time{sample.first}; - sample.second[0] = m_Trend->detrend(time, sample.second[0], 0.0); - averageTime.add(static_cast(time)); + sample.second[0] = m_Trend->detrend(sample.first, sample.second[0], 0.0); } - TSizeVec valueorder(samples.size()); - std::iota(valueorder.begin(), valueorder.end(), 0); std::stable_sort(valueorder.begin(), valueorder.end(), [&samples](std::size_t lhs, std::size_t rhs) { @@ -716,6 +737,7 @@ CUnivariateTimeSeriesModel::addSamples(const CModelAddSamplesParams ¶ms, TDouble4Vec1Vec weights; samples_.reserve(samples.size()); weights.reserve(samples.size()); + TMeanAccumulator averageTime; for (auto i : valueorder) { @@ -727,6 +749,7 @@ CUnivariateTimeSeriesModel::addSamples(const CModelAddSamplesParams ¶ms, wi[0].push_back(weight[0]); } weights.push_back(wi[0]); + averageTime.add(static_cast(samples[i].first)); } m_Prior->addSamples(params.weightStyles(), samples_, weights); @@ -781,6 +804,11 @@ CUnivariateTimeSeriesModel::addSamples(const CModelAddSamplesParams ¶ms, m_Correlations->addSamples(m_Id, type, samples, weights, params.propagationInterval(), multiplier); } + if (randomSample) + { + m_SlidingWindow.push_back({randomSample->first, randomSample->second}); + } + return result; } @@ -800,8 +828,8 @@ CUnivariateTimeSeriesModel::mode(core_t::TTime time, { weights.push_back(weight[0]); } - return TDouble2Vec{ m_Prior->marginalLikelihoodMode(weightStyles, weights) - + CBasicStatistics::mean(m_Trend->baseline(time))}; + return { m_Prior->marginalLikelihoodMode(weightStyles, weights) + + CBasicStatistics::mean(m_Trend->baseline(time))}; } CUnivariateTimeSeriesModel::TDouble2Vec1Vec @@ -861,7 +889,7 @@ CUnivariateTimeSeriesModel::residualModes(const maths_t::TWeightStyleVec &weight result.reserve(modes.size()); for (auto mode : modes) { - result.push_back(TDouble2Vec{mode}); + result.push_back({mode}); } return result; @@ -951,14 +979,14 @@ CUnivariateTimeSeriesModel::predict(core_t::TTime time, m_Prior->nearestMarginalLikelihoodMean(hint[0]))}; double result{scale * (seasonalOffset + median + correlateCorrection)}; - return TDouble2Vec{m_IsNonNegative ? std::max(result, 0.0) : result}; + return {m_IsNonNegative ? std::max(result, 0.0) : result}; } CUnivariateTimeSeriesModel::TDouble2Vec3Vec CUnivariateTimeSeriesModel::confidenceInterval(core_t::TTime time, + double confidenceInterval, const maths_t::TWeightStyleVec &weightStyles, - const TDouble2Vec4Vec &weights_, - double confidenceInterval) const + const TDouble2Vec4Vec &weights_) const { if (m_Prior->isNonInformative()) { @@ -967,14 +995,8 @@ CUnivariateTimeSeriesModel::confidenceInterval(core_t::TTime time, double scale{1.0 - this->params().probabilityBucketEmpty()}; - double seasonalOffset[]{0.0, 0.0, 0.0}; - if (m_Trend->initialized()) - { - TDoubleDoublePr seasonalOffset_{m_Trend->baseline(time, 0.0, confidenceInterval)}; - seasonalOffset[0] = seasonalOffset_.first; - seasonalOffset[1] = CBasicStatistics::mean(seasonalOffset_); - seasonalOffset[2] = seasonalOffset_.second; - } + double seasonalOffset{m_Trend->initialized() ? + CBasicStatistics::mean(m_Trend->baseline(time, confidenceInterval)) : 0.0}; TDouble4Vec weights; weights.reserve(weights_.size()); @@ -988,13 +1010,13 @@ CUnivariateTimeSeriesModel::confidenceInterval(core_t::TTime time, TDoubleDoublePr interval{ m_Prior->marginalLikelihoodConfidenceInterval(confidenceInterval, weightStyles, weights)}; - double result[]{scale * (seasonalOffset[0] + interval.first), - scale * (seasonalOffset[1] + median), - scale * (seasonalOffset[2] + interval.second)}; + double result[]{scale * (seasonalOffset + interval.first), + scale * (seasonalOffset + median), + scale * (seasonalOffset + interval.second)}; - return TDouble2Vec3Vec{TDouble2Vec{m_IsNonNegative ? std::max(result[0], 0.0) : result[0]}, - TDouble2Vec{m_IsNonNegative ? std::max(result[1], 0.0) : result[1]}, - TDouble2Vec{m_IsNonNegative ? std::max(result[2], 0.0) : result[2]}}; + return {{m_IsNonNegative ? std::max(result[0], 0.0) : result[0]}, + {m_IsNonNegative ? std::max(result[1], 0.0) : result[1]}, + {m_IsNonNegative ? std::max(result[2], 0.0) : result[2]}}; } bool CUnivariateTimeSeriesModel::forecast(core_t::TTime startTime, @@ -1005,64 +1027,38 @@ bool CUnivariateTimeSeriesModel::forecast(core_t::TTime startTime, const TForecastPushDatapointFunc &forecastPushDataPointFunc, std::string &messageOut) { - // Should not happen if model has been cloned with cloneForForecast. - if (!m_Prior->isForForecasting()) - { - LOG_ERROR("Unexpected error: model hasn't been prepared for forecasting."); - return false; - } - if (m_Prior->isNonInformative()) { messageOut = forecast::INFO_INSUFFICIENT_HISTORY; return true; } + using TDouble3Vec = core::CSmallVector; + using TDouble3VecVec = std::vector; + + core_t::TTime bucketLength{this->params().bucketLength()}; double minimum{m_IsNonNegative ? std::max(minimum_[0], 0.0) : minimum_[0]}; double maximum{m_IsNonNegative ? std::max(maximum_[0], 0.0) : maximum_[0]}; - double cutoffInterval{forecast::RELATIVE_ERROR_TOLERANCE * m_Trend->mean(startTime)}; - { - TDouble2Vec3Vec prediction(this->confidenceInterval(startTime, - CConstantWeights::COUNT, - CConstantWeights::unit(2), - confidenceInterval)); - if (prediction.size() == 3) - { - cutoffInterval = std::max(cutoffInterval, forecast::MAXIMUM_ERROR_GROWTH - * (prediction[2][0] - prediction[0][0])); - } - } + TDouble3VecVec predictions; + m_Trend->forecast(startTime, endTime, bucketLength, confidenceInterval, + this->params().minimumSeasonalVarianceScale(), predictions); - core_t::TTime bucketLength{this->params().bucketLength()}; - for (core_t::TTime time = startTime; time < endTime; time += bucketLength) - { - TDouble2Vec4Vec weights{this->seasonalWeight(DEFAULT_SEASONAL_CONFIDENCE_INTERVAL, time)}; - TDouble2Vec3Vec prediction(this->confidenceInterval(time, - CConstantWeights::SEASONAL_VARIANCE, - weights, confidenceInterval)); - double interval{prediction.size() == 3 ? - prediction[2][0] - prediction[0][0] : - boost::numeric::bounds::highest()}; - - if (interval < cutoffInterval) - { - SErrorBar errorBar; - errorBar.s_Time = time; - errorBar.s_BucketLength = bucketLength; - errorBar.s_LowerBound = CTools::truncate(prediction[0][0], minimum, maximum); - errorBar.s_Predicted = CTools::truncate(prediction[1][0], minimum, maximum); - errorBar.s_UpperBound = CTools::truncate(prediction[2][0], minimum, maximum); - forecastPushDataPointFunc(errorBar); - } - else - { - messageOut = forecast::INFO_INSUFFICIENT_ACCURACY_STOPPING_EARLY; - return true; - } - - m_Trend->testAndInterpolate(time + bucketLength); - m_Prior->propagateForwardsByTime(1.0); + core_t::TTime time{startTime}; + for (const auto &prediction : predictions) + { + SErrorBar errorBar; + errorBar.s_Time = time; + errorBar.s_BucketLength = bucketLength; + errorBar.s_LowerBound = CTools::truncate(prediction[0], + minimum, + maximum + prediction[0] - prediction[1]); + errorBar.s_Predicted = CTools::truncate(prediction[1], minimum, maximum); + errorBar.s_UpperBound = CTools::truncate(prediction[2], + minimum + prediction[2] - prediction[1], + maximum); + forecastPushDataPointFunc(errorBar); + time += bucketLength; } return true; @@ -1252,14 +1248,14 @@ CUnivariateTimeSeriesModel::winsorisationWeight(double derate, { double scale{this->seasonalWeight(0.0, time)[0]}; double sample{m_Trend->detrend(time, value[0], 0.0)}; - return TDouble2Vec{computeWinsorisationWeight(*m_Prior, derate, scale, sample)}; + return {computeWinsorisationWeight(*m_Prior, derate, scale, sample)}; } CUnivariateTimeSeriesModel::TDouble2Vec CUnivariateTimeSeriesModel::seasonalWeight(double confidence, core_t::TTime time) const { double scale{m_Trend->scale(time, m_Prior->marginalLikelihoodVariance(), confidence).second}; - return TDouble2Vec{std::max(scale, this->params().minimumSeasonalVarianceScale())}; + return {std::max(scale, this->params().minimumSeasonalVarianceScale())}; } uint64_t CUnivariateTimeSeriesModel::checksum(uint64_t seed) const @@ -1269,6 +1265,7 @@ uint64_t CUnivariateTimeSeriesModel::checksum(uint64_t seed) const seed = CChecksum::calculate(seed, m_Trend); seed = CChecksum::calculate(seed, m_Prior); seed = CChecksum::calculate(seed, m_AnomalyModel); + seed = CChecksum::calculate(seed, m_SlidingWindow); return CChecksum::calculate(seed, m_Correlations != 0); } @@ -1279,6 +1276,7 @@ void CUnivariateTimeSeriesModel::debugMemoryUsage(core::CMemoryUsage::TMemoryUsa core::CMemoryDebug::dynamicSize("m_Trend", m_Trend, mem); core::CMemoryDebug::dynamicSize("m_Prior", m_Prior, mem); core::CMemoryDebug::dynamicSize("m_AnomalyModel", m_AnomalyModel, mem); + core::CMemoryDebug::dynamicSize("m_SlidingWindow", m_SlidingWindow, mem); } std::size_t CUnivariateTimeSeriesModel::memoryUsage(void) const @@ -1286,57 +1284,99 @@ std::size_t CUnivariateTimeSeriesModel::memoryUsage(void) const return core::CMemory::dynamicSize(m_Controllers) + core::CMemory::dynamicSize(m_Trend) + core::CMemory::dynamicSize(m_Prior) - + core::CMemory::dynamicSize(m_AnomalyModel); + + core::CMemory::dynamicSize(m_AnomalyModel) + + core::CMemory::dynamicSize(m_SlidingWindow); } bool CUnivariateTimeSeriesModel::acceptRestoreTraverser(const SModelRestoreParams ¶ms, core::CStateRestoreTraverser &traverser) { - do + if (traverser.name() == VERSION_6_3_TAG) + { + while (traverser.next()) + { + const std::string &name{traverser.name()}; + RESTORE_BUILT_IN(ID_6_3_TAG, m_Id) + RESTORE_BOOL(IS_NON_NEGATIVE_6_3_TAG, m_IsNonNegative) + RESTORE_BOOL(IS_FORECASTABLE_6_3_TAG, m_IsForecastable) + RESTORE(RNG_6_3_TAG, m_Rng.fromString(traverser.value())) + RESTORE_SETUP_TEARDOWN(CONTROLLER_6_3_TAG, + m_Controllers = boost::make_shared(), + core::CPersistUtils::restore(CONTROLLER_6_3_TAG, *m_Controllers, traverser), + /**/) + RESTORE(TREND_6_3_TAG, traverser.traverseSubLevel(boost::bind( + CTimeSeriesDecompositionStateSerialiser(), + boost::cref(params.s_DecompositionParams), + boost::ref(m_Trend), _1))) + RESTORE(PRIOR_6_3_TAG, traverser.traverseSubLevel(boost::bind( + CPriorStateSerialiser(), + boost::cref(params.s_DistributionParams), + boost::ref(m_Prior), _1))) + RESTORE_SETUP_TEARDOWN(ANOMALY_MODEL_6_3_TAG, + m_AnomalyModel = boost::make_shared(), + traverser.traverseSubLevel(boost::bind(&CTimeSeriesAnomalyModel::acceptRestoreTraverser, + m_AnomalyModel.get(), boost::cref(params), _1)), + /**/) + RESTORE(SLIDING_WINDOW_6_3_TAG, + core::CPersistUtils::restore(SLIDING_WINDOW_6_3_TAG, m_SlidingWindow, traverser)) + } + } + else { - const std::string &name{traverser.name()}; - RESTORE_BUILT_IN(ID_TAG, m_Id) - RESTORE_BOOL(IS_NON_NEGATIVE_TAG, m_IsNonNegative) - RESTORE_BOOL(IS_FORECASTABLE_TAG, m_IsForecastable) - RESTORE_SETUP_TEARDOWN(CONTROLLER_TAG, - m_Controllers = boost::make_shared(), - core::CPersistUtils::restore(CONTROLLER_TAG, *m_Controllers, traverser), - /**/) - RESTORE(TREND_TAG, traverser.traverseSubLevel(boost::bind(CTimeSeriesDecompositionStateSerialiser(), - boost::cref(params.s_DecompositionParams), - boost::ref(m_Trend), _1))) - RESTORE(PRIOR_TAG, traverser.traverseSubLevel(boost::bind(CPriorStateSerialiser(), - boost::cref(params.s_DistributionParams), - boost::ref(m_Prior), _1))) - RESTORE_SETUP_TEARDOWN(ANOMALY_MODEL_TAG, - m_AnomalyModel = boost::make_shared(), - traverser.traverseSubLevel(boost::bind(&CTimeSeriesAnomalyModel::acceptRestoreTraverser, - m_AnomalyModel.get(), boost::cref(params), _1)), - /**/) + // There is no version string this is historic state. + do + { + const std::string &name{traverser.name()}; + RESTORE_BUILT_IN(ID_OLD_TAG, m_Id) + RESTORE_BOOL(IS_NON_NEGATIVE_OLD_TAG, m_IsNonNegative) + RESTORE_BOOL(IS_FORECASTABLE_OLD_TAG, m_IsForecastable) + RESTORE_SETUP_TEARDOWN(CONTROLLER_OLD_TAG, + m_Controllers = boost::make_shared(), + core::CPersistUtils::restore(CONTROLLER_OLD_TAG, *m_Controllers, traverser), + /**/) + RESTORE(TREND_OLD_TAG, traverser.traverseSubLevel(boost::bind( + CTimeSeriesDecompositionStateSerialiser(), + boost::cref(params.s_DecompositionParams), + boost::ref(m_Trend), _1))) + RESTORE(PRIOR_OLD_TAG, traverser.traverseSubLevel(boost::bind( + CPriorStateSerialiser(), + boost::cref(params.s_DistributionParams), + boost::ref(m_Prior), _1))) + RESTORE_SETUP_TEARDOWN(ANOMALY_MODEL_OLD_TAG, + m_AnomalyModel = boost::make_shared(), + traverser.traverseSubLevel(boost::bind(&CTimeSeriesAnomalyModel::acceptRestoreTraverser, + m_AnomalyModel.get(), boost::cref(params), _1)), + /**/) + } + while (traverser.next()); } - while (traverser.next()); return true; } void CUnivariateTimeSeriesModel::acceptPersistInserter(core::CStatePersistInserter &inserter) const { - // Note we don't persist this->params() or the correlations because - // that state is reinitialized. - inserter.insertValue(ID_TAG, m_Id); - inserter.insertValue(IS_NON_NEGATIVE_TAG, static_cast(m_IsNonNegative)); - inserter.insertValue(IS_FORECASTABLE_TAG, static_cast(m_IsForecastable)); + // Note that we don't persist this->params() or the correlations + // because that state is reinitialized. + inserter.insertValue(VERSION_6_3_TAG, ""); + inserter.insertValue(ID_6_3_TAG, m_Id); + inserter.insertValue(IS_NON_NEGATIVE_6_3_TAG, static_cast(m_IsNonNegative)); + inserter.insertValue(IS_FORECASTABLE_6_3_TAG, static_cast(m_IsForecastable)); + inserter.insertValue(RNG_6_3_TAG, m_Rng.toString()); if (m_Controllers) { - core::CPersistUtils::persist(CONTROLLER_TAG, *m_Controllers, inserter); + core::CPersistUtils::persist(CONTROLLER_6_3_TAG, *m_Controllers, inserter); } - inserter.insertLevel(TREND_TAG, boost::bind(CTimeSeriesDecompositionStateSerialiser(), - boost::cref(*m_Trend), _1)); - inserter.insertLevel(PRIOR_TAG, boost::bind(CPriorStateSerialiser(), boost::cref(*m_Prior), _1)); + inserter.insertLevel(TREND_6_3_TAG, boost::bind(CTimeSeriesDecompositionStateSerialiser(), + boost::cref(*m_Trend), _1)); + inserter.insertLevel(PRIOR_6_3_TAG, boost::bind(CPriorStateSerialiser(), + boost::cref(*m_Prior), _1)); if (m_AnomalyModel) { - inserter.insertLevel(ANOMALY_MODEL_TAG, boost::bind(&CTimeSeriesAnomalyModel::acceptPersistInserter, - m_AnomalyModel.get(), _1)); + inserter.insertLevel(ANOMALY_MODEL_6_3_TAG, + boost::bind(&CTimeSeriesAnomalyModel::acceptPersistInserter, + m_AnomalyModel.get(), _1)); } + core::CPersistUtils::persist(SLIDING_WINDOW_6_3_TAG, m_SlidingWindow, inserter); } maths_t::EDataType CUnivariateTimeSeriesModel::dataType(void) const @@ -1344,6 +1384,11 @@ maths_t::EDataType CUnivariateTimeSeriesModel::dataType(void) const return m_Prior->dataType(); } +const CUnivariateTimeSeriesModel::TTimeDoublePrCBuf &CUnivariateTimeSeriesModel::slidingWindow(void) const +{ + return m_SlidingWindow; +} + const CTimeSeriesDecompositionInterface &CUnivariateTimeSeriesModel::trend(void) const { return *m_Trend; @@ -1360,11 +1405,13 @@ CUnivariateTimeSeriesModel::CUnivariateTimeSeriesModel(const CUnivariateTimeSeri m_Id(id), m_IsNonNegative(other.m_IsNonNegative), m_IsForecastable(other.m_IsForecastable), + m_Rng(other.m_Rng), m_Trend(other.m_Trend->clone()), m_Prior(other.m_Prior->clone()), m_AnomalyModel(other.m_AnomalyModel ? boost::make_shared(*other.m_AnomalyModel) : TAnomalyModelPtr()), + m_SlidingWindow(other.m_SlidingWindow), m_Correlations(0) { if (other.m_Controllers) @@ -1387,6 +1434,8 @@ CUnivariateTimeSeriesModel::updateTrend(const maths_t::TWeightStyleVec &weightSt } } + // Time order is not reliable, for example if the data are polled + // or for count feature, the times of all samples will be the same. TSizeVec timeorder(samples.size()); std::iota(timeorder.begin(), timeorder.end(), 0); std::stable_sort(timeorder.begin(), timeorder.end(), @@ -1399,28 +1448,31 @@ CUnivariateTimeSeriesModel::updateTrend(const maths_t::TWeightStyleVec &weightSt }); EUpdateResult result = E_Success; - TDouble4Vec weight(weightStyles.size()); - - for (auto i : timeorder) { - core_t::TTime time{samples[i].first}; - double value{samples[i].second[0]}; - for (std::size_t j = 0u; j < weights[i].size(); ++j) - { - weight[j] = weights[i][j][0]; - } - if (m_Trend->addPoint(time, value, weightStyles, weight)) + TDouble4Vec weight(weightStyles.size()); + for (auto i : timeorder) { - result = E_Reset; + core_t::TTime time{samples[i].first}; + double value{samples[i].second[0]}; + for (std::size_t j = 0u; j < weights[i].size(); ++j) + { + weight[j] = weights[i][j][0]; + } + if (m_Trend->addPoint(time, value, weightStyles, weight)) + { + result = E_Reset; + } } } - if (result == E_Reset) { - if (initializePrior(this->params().bucketLength(), - this->params().learnRate(), *m_Trend, *m_Prior)) + m_Prior->setToNonInformative(0.0, m_Prior->decayRate()); + TDouble4Vec1Vec weight{{std::max(this->params().learnRate(), + 5.0 / static_cast(SLIDING_WINDOW_SIZE))}}; + for (const auto &value : m_SlidingWindow) { - m_Prior->decayRate(this->params().decayRate()); + TDouble1Vec sample{m_Trend->detrend(value.first, value.second, 0.0)}; + m_Prior->addSamples(CConstantWeights::COUNT, sample, weight); } if (m_Correlations) { @@ -1428,9 +1480,11 @@ CUnivariateTimeSeriesModel::updateTrend(const maths_t::TWeightStyleVec &weightSt } if (m_Controllers) { + m_Prior->decayRate( m_Prior->decayRate() + / (*m_Controllers)[E_PriorControl].multiplier()); m_Trend->decayRate( m_Trend->decayRate() / (*m_Controllers)[E_TrendControl].multiplier()); - for (auto &&controller : *m_Controllers) + for (auto &controller : *m_Controllers) { controller.reset(); } @@ -1859,7 +1913,7 @@ void CTimeSeriesCorrelations::removeTimeSeries(std::size_t id) } this->refreshLookup(); } - m_Correlations.removeVariables(TSizeVec{id}); + m_Correlations.removeVariables({id}); m_TimeSeriesModels[id] = 0; } @@ -1962,7 +2016,7 @@ void CTimeSeriesCorrelations::refreshLookup(void) m_CorrelatedLookup[x0].push_back(x1); m_CorrelatedLookup[x1].push_back(x0); } - for (auto &&prior : m_CorrelatedLookup) + for (auto &prior : m_CorrelatedLookup) { std::sort(prior.second.begin(), prior.second.end()); } @@ -1980,7 +2034,8 @@ CMultivariateTimeSeriesModel::CMultivariateTimeSeriesModel(const CModelParams &p m_AnomalyModel(modelAnomalies ? boost::make_shared(params.bucketLength(), params.decayRate()) : - TAnomalyModelPtr()) + TAnomalyModelPtr()), + m_SlidingWindow(SLIDING_WINDOW_SIZE) { if (controllers) { @@ -1998,7 +2053,8 @@ CMultivariateTimeSeriesModel::CMultivariateTimeSeriesModel(const CMultivariateTi m_Prior(other.m_Prior->clone()), m_AnomalyModel(other.m_AnomalyModel ? boost::make_shared(*other.m_AnomalyModel) : - TAnomalyModelPtr()) + TAnomalyModelPtr()), + m_SlidingWindow(other.m_SlidingWindow) { if (other.m_Controllers) { @@ -2013,7 +2069,8 @@ CMultivariateTimeSeriesModel::CMultivariateTimeSeriesModel(const CMultivariateTi CMultivariateTimeSeriesModel::CMultivariateTimeSeriesModel(const SModelRestoreParams ¶ms, core::CStateRestoreTraverser &traverser) : - CModel(params.s_Params) + CModel(params.s_Params), + m_SlidingWindow(SLIDING_WINDOW_SIZE) { traverser.traverseSubLevel(boost::bind(&CMultivariateTimeSeriesModel::acceptRestoreTraverser, this, boost::cref(params), _1)); @@ -2070,6 +2127,25 @@ CMultivariateTimeSeriesModel::addSamples(const CModelAddSamplesParams ¶ms, } using TMeanAccumulator = CBasicStatistics::SSampleMean::TAccumulator; + using TOptionalTimeDouble2VecPr = boost::optional; + + TSizeVec valueorder(samples.size()); + std::iota(valueorder.begin(), valueorder.end(), 0); + std::stable_sort(valueorder.begin(), valueorder.end(), + [&samples](std::size_t lhs, std::size_t rhs) + { + return samples[lhs].second < samples[rhs].second; + }); + + TOptionalTimeDouble2VecPr randomSample; + + double p{SLIDING_WINDOW_SIZE * static_cast(this->params().bucketLength()) + / static_cast(core::constants::DAY)}; + if (p >= 1.0 || CSampling::uniformSample(m_Rng, 0.0, 1.0) < p) + { + std::size_t i{CSampling::uniformSample(m_Rng, 0, samples.size())}; + randomSample.reset({samples[valueorder[i]].first, samples[valueorder[i]].second}); + } m_IsNonNegative = params.isNonNegative(); @@ -2077,7 +2153,6 @@ CMultivariateTimeSeriesModel::addSamples(const CModelAddSamplesParams ¶ms, EUpdateResult result{this->updateTrend(params.weightStyles(), samples, params.trendWeights())}; - TMeanAccumulator averageTime; for (auto &sample : samples) { if (sample.second.size() != dimension) @@ -2091,11 +2166,8 @@ CMultivariateTimeSeriesModel::addSamples(const CModelAddSamplesParams ¶ms, { sample.second[d] = m_Trend[d]->detrend(time, sample.second[d], 0.0); } - averageTime.add(static_cast(time)); } - TSizeVec valueorder(samples.size()); - std::iota(valueorder.begin(), valueorder.end(), 0); std::stable_sort(valueorder.begin(), valueorder.end(), [&samples](std::size_t lhs, std::size_t rhs) { @@ -2109,6 +2181,7 @@ CMultivariateTimeSeriesModel::addSamples(const CModelAddSamplesParams ¶ms, TDouble10Vec4Vec1Vec weights; samples_.reserve(samples.size()); weights.reserve(samples.size()); + TMeanAccumulator averageTime; for (auto i : valueorder) { @@ -2123,6 +2196,7 @@ CMultivariateTimeSeriesModel::addSamples(const CModelAddSamplesParams ¶ms, } } weights.push_back(wi); + averageTime.add(static_cast(samples[i].first)); } m_Prior->addSamples(params.weightStyles(), samples_, weights); @@ -2177,6 +2251,11 @@ CMultivariateTimeSeriesModel::addSamples(const CModelAddSamplesParams ¶ms, } } + if (randomSample) + { + m_SlidingWindow.push_back({randomSample->first, randomSample->second}); + } + return result; } @@ -2307,9 +2386,9 @@ CMultivariateTimeSeriesModel::predict(core_t::TTime time, CMultivariateTimeSeriesModel::TDouble2Vec3Vec CMultivariateTimeSeriesModel::confidenceInterval(core_t::TTime time, + double confidenceInterval, const maths_t::TWeightStyleVec &weightStyles, - const TDouble2Vec4Vec &weights_, - double confidenceInterval) const + const TDouble2Vec4Vec &weights_) const { if (m_Prior->isNonInformative()) { @@ -2331,13 +2410,11 @@ CMultivariateTimeSeriesModel::confidenceInterval(core_t::TTime time, TDouble4Vec weights; for (std::size_t d = 0u; d < dimension; --marginalize[std::min(d, dimension - 2)], ++d) { - double seasonalOffset{0.0}; - if (m_Trend[d]->initialized()) - { - seasonalOffset = CBasicStatistics::mean( - m_Trend[d]->baseline(time, 0.0, confidenceInterval)); - } + double seasonalOffset{m_Trend[d]->initialized() ? + CBasicStatistics::mean( + m_Trend[d]->baseline(time, confidenceInterval)) : 0.0}; + weights.clear(); weights.reserve(weights_.size()); for (const auto &weight : weights_) { @@ -2517,7 +2594,8 @@ uint64_t CMultivariateTimeSeriesModel::checksum(uint64_t seed) const seed = CChecksum::calculate(seed, m_Controllers); seed = CChecksum::calculate(seed, m_Trend); seed = CChecksum::calculate(seed, m_Prior); - return CChecksum::calculate(seed, m_AnomalyModel); + seed = CChecksum::calculate(seed, m_AnomalyModel); + return CChecksum::calculate(seed, m_SlidingWindow); } void CMultivariateTimeSeriesModel::debugMemoryUsage(core::CMemoryUsage::TMemoryUsagePtr mem) const @@ -2527,6 +2605,7 @@ void CMultivariateTimeSeriesModel::debugMemoryUsage(core::CMemoryUsage::TMemoryU core::CMemoryDebug::dynamicSize("m_Trend", m_Trend, mem); core::CMemoryDebug::dynamicSize("m_Prior", m_Prior, mem); core::CMemoryDebug::dynamicSize("m_AnomalyModel", m_AnomalyModel, mem); + core::CMemoryDebug::dynamicSize("m_SlidingWindow", m_SlidingWindow, mem); } std::size_t CMultivariateTimeSeriesModel::memoryUsage(void) const @@ -2534,58 +2613,100 @@ std::size_t CMultivariateTimeSeriesModel::memoryUsage(void) const return core::CMemory::dynamicSize(m_Controllers) + core::CMemory::dynamicSize(m_Trend) + core::CMemory::dynamicSize(m_Prior) - + core::CMemory::dynamicSize(m_AnomalyModel); + + core::CMemory::dynamicSize(m_AnomalyModel) + + core::CMemory::dynamicSize(m_SlidingWindow); } bool CMultivariateTimeSeriesModel::acceptRestoreTraverser(const SModelRestoreParams ¶ms, core::CStateRestoreTraverser &traverser) { - do + if (traverser.name() == VERSION_6_3_TAG) + { + while (traverser.next()) + { + const std::string &name{traverser.name()}; + RESTORE_BOOL(IS_NON_NEGATIVE_6_3_TAG, m_IsNonNegative) + RESTORE(RNG_6_3_TAG, m_Rng.fromString(traverser.value())) + RESTORE_SETUP_TEARDOWN(CONTROLLER_6_3_TAG, + m_Controllers = boost::make_shared(), + core::CPersistUtils::restore(CONTROLLER_6_3_TAG, *m_Controllers, traverser), + /**/) + RESTORE_SETUP_TEARDOWN(TREND_6_3_TAG, + m_Trend.push_back(TDecompositionPtr()), + traverser.traverseSubLevel(boost::bind( + CTimeSeriesDecompositionStateSerialiser(), + boost::cref(params.s_DecompositionParams), + boost::ref(m_Trend.back()), _1)), + /**/) + RESTORE(PRIOR_6_3_TAG, traverser.traverseSubLevel(boost::bind( + CPriorStateSerialiser(), + boost::cref(params.s_DistributionParams), + boost::ref(m_Prior), _1))) + RESTORE_SETUP_TEARDOWN(ANOMALY_MODEL_6_3_TAG, + m_AnomalyModel = boost::make_shared(), + traverser.traverseSubLevel(boost::bind(&CTimeSeriesAnomalyModel::acceptRestoreTraverser, + m_AnomalyModel.get(), boost::cref(params), _1)), + /**/) + RESTORE(SLIDING_WINDOW_6_3_TAG, + core::CPersistUtils::restore(SLIDING_WINDOW_6_3_TAG, m_SlidingWindow, traverser)) + } + } + else { - const std::string &name{traverser.name()}; - RESTORE_BOOL(IS_NON_NEGATIVE_TAG, m_IsNonNegative) - RESTORE_SETUP_TEARDOWN(CONTROLLER_TAG, - m_Controllers = boost::make_shared(), - core::CPersistUtils::restore(CONTROLLER_TAG, *m_Controllers, traverser), - /**/) - RESTORE_SETUP_TEARDOWN(TREND_TAG, - m_Trend.push_back(TDecompositionPtr()), - traverser.traverseSubLevel(boost::bind(CTimeSeriesDecompositionStateSerialiser(), - boost::cref(params.s_DecompositionParams), - boost::ref(m_Trend.back()), _1)), - /**/) - RESTORE(PRIOR_TAG, traverser.traverseSubLevel(boost::bind(CPriorStateSerialiser(), - boost::cref(params.s_DistributionParams), - boost::ref(m_Prior), _1))) - RESTORE_SETUP_TEARDOWN(ANOMALY_MODEL_TAG, - m_AnomalyModel = boost::make_shared(), - traverser.traverseSubLevel(boost::bind(&CTimeSeriesAnomalyModel::acceptRestoreTraverser, - m_AnomalyModel.get(), boost::cref(params), _1)), - /**/) + do + { + const std::string &name{traverser.name()}; + RESTORE_BOOL(IS_NON_NEGATIVE_OLD_TAG, m_IsNonNegative) + RESTORE_SETUP_TEARDOWN(CONTROLLER_OLD_TAG, + m_Controllers = boost::make_shared(), + core::CPersistUtils::restore(CONTROLLER_6_3_TAG, *m_Controllers, traverser), + /**/) + RESTORE_SETUP_TEARDOWN(TREND_OLD_TAG, + m_Trend.push_back(TDecompositionPtr()), + traverser.traverseSubLevel(boost::bind( + CTimeSeriesDecompositionStateSerialiser(), + boost::cref(params.s_DecompositionParams), + boost::ref(m_Trend.back()), _1)), + /**/) + RESTORE(PRIOR_OLD_TAG, traverser.traverseSubLevel(boost::bind( + CPriorStateSerialiser(), + boost::cref(params.s_DistributionParams), + boost::ref(m_Prior), _1))) + RESTORE_SETUP_TEARDOWN(ANOMALY_MODEL_OLD_TAG, + m_AnomalyModel = boost::make_shared(), + traverser.traverseSubLevel(boost::bind(&CTimeSeriesAnomalyModel::acceptRestoreTraverser, + m_AnomalyModel.get(), boost::cref(params), _1)), + /**/) + } + while (traverser.next()); } - while (traverser.next()); return true; } void CMultivariateTimeSeriesModel::acceptPersistInserter(core::CStatePersistInserter &inserter) const { - // Note we don't persist this->params() because that state is reinitialized. - inserter.insertValue(IS_NON_NEGATIVE_TAG, static_cast(m_IsNonNegative)); + // Note that we don't persist this->params() because that state + // is reinitialized. + inserter.insertValue(VERSION_6_3_TAG, ""); + inserter.insertValue(IS_NON_NEGATIVE_6_3_TAG, static_cast(m_IsNonNegative)); if (m_Controllers) { - core::CPersistUtils::persist(CONTROLLER_TAG, *m_Controllers, inserter); + core::CPersistUtils::persist(CONTROLLER_6_3_TAG, *m_Controllers, inserter); } for (const auto &trend : m_Trend) { - inserter.insertLevel(TREND_TAG, boost::bind(CTimeSeriesDecompositionStateSerialiser(), - boost::cref(*trend), _1)); + inserter.insertLevel(TREND_6_3_TAG, boost::bind(CTimeSeriesDecompositionStateSerialiser(), + boost::cref(*trend), _1)); } - inserter.insertLevel(PRIOR_TAG, boost::bind(CPriorStateSerialiser(), boost::cref(*m_Prior), _1)); + inserter.insertLevel(PRIOR_6_3_TAG, boost::bind(CPriorStateSerialiser(), + boost::cref(*m_Prior), _1)); if (m_AnomalyModel) { - inserter.insertLevel(ANOMALY_MODEL_TAG, boost::bind(&CTimeSeriesAnomalyModel::acceptPersistInserter, - m_AnomalyModel.get(), _1)); + inserter.insertLevel(ANOMALY_MODEL_6_3_TAG, + boost::bind(&CTimeSeriesAnomalyModel::acceptPersistInserter, + m_AnomalyModel.get(), _1)); } + core::CPersistUtils::persist(SLIDING_WINDOW_6_3_TAG, m_SlidingWindow, inserter); } maths_t::EDataType CMultivariateTimeSeriesModel::dataType(void) const @@ -2593,7 +2714,12 @@ maths_t::EDataType CMultivariateTimeSeriesModel::dataType(void) const return m_Prior->dataType(); } -const CMultivariateTimeSeriesModel::TDecompositionPtr10Vec &CMultivariateTimeSeriesModel::trend(void) const +const CMultivariateTimeSeriesModel::TTimeDouble2VecPrCBuf &CMultivariateTimeSeriesModel::slidingWindow(void) const +{ + return m_SlidingWindow; +} + +const CMultivariateTimeSeriesModel::CMultivariateTimeSeriesModel::TDecompositionPtr10Vec &CMultivariateTimeSeriesModel::trend(void) const { return m_Trend; } @@ -2620,6 +2746,8 @@ CMultivariateTimeSeriesModel::updateTrend(const maths_t::TWeightStyleVec &weight } } + // Time order is not reliable, for example if the data are polled + // or for count feature, the times of all samples will be the same. TSizeVec timeorder(samples.size()); std::iota(timeorder.begin(), timeorder.end(), 0); std::stable_sort(timeorder.begin(), timeorder.end(), @@ -2632,40 +2760,50 @@ CMultivariateTimeSeriesModel::updateTrend(const maths_t::TWeightStyleVec &weight }); EUpdateResult result{E_Success}; - TDouble4Vec weight(weightStyles.size()); - - for (auto i : timeorder) { - core_t::TTime time{samples[i].first}; - TDouble10Vec value(samples[i].second); - for (std::size_t d = 0u; d < dimension; ++d) + TDouble4Vec weight(weightStyles.size()); + for (auto i : timeorder) { - for (std::size_t j = 0u; j < weights[i].size(); ++j) - { - weight[j] = weights[i][j][d]; - } - if (m_Trend[d]->addPoint(time, value[d], weightStyles, weight)) + core_t::TTime time{samples[i].first}; + TDouble10Vec value(samples[i].second); + for (std::size_t d = 0u; d < dimension; ++d) { - result = E_Reset; + for (std::size_t j = 0u; j < weights[i].size(); ++j) + { + weight[j] = weights[i][j][d]; + } + if (m_Trend[d]->addPoint(time, value[d], weightStyles, weight)) + { + result = E_Reset; + } } } } - if (result == E_Reset) { - if (initializePrior(this->params().bucketLength(), - this->params().learnRate(), m_Trend, *m_Prior)) + m_Prior->setToNonInformative(0.0, m_Prior->decayRate()); + TDouble10Vec4Vec1Vec weight{{TDouble10Vec( + dimension, std::max(this->params().learnRate(), + 5.0 / static_cast(SLIDING_WINDOW_SIZE)))}}; + for (const auto &value : m_SlidingWindow) { - m_Prior->decayRate(this->params().decayRate()); + TDouble10Vec1Vec sample{TDouble10Vec(dimension)}; + for (std::size_t i = 0u; i < dimension; ++i) + { + sample[0][i] = m_Trend[i]->detrend(value.first, value.second[i], 0.0); + } + m_Prior->addSamples(CConstantWeights::COUNT, sample, weight); } if (m_Controllers) { - for (auto &&trend : m_Trend) + m_Prior->decayRate( m_Prior->decayRate() + / (*m_Controllers)[E_PriorControl].multiplier()); + for (auto &trend : m_Trend) { trend->decayRate( trend->decayRate() / (*m_Controllers)[E_TrendControl].multiplier()); } - for (auto &&controller : *m_Controllers) + for (auto &controller : *m_Controllers) { controller.reset(); } diff --git a/lib/maths/CTrendComponent.cc b/lib/maths/CTrendComponent.cc new file mode 100644 index 0000000000..7d8ec29418 --- /dev/null +++ b/lib/maths/CTrendComponent.cc @@ -0,0 +1,561 @@ +/* + * ELASTICSEARCH CONFIDENTIAL + * + * Copyright (c) 2017 Elasticsearch BV. All Rights Reserved. + * + * Notice: this software, and all information contained + * therein, is the exclusive property of Elasticsearch BV + * and its licensors, if any, and is protected under applicable + * domestic and foreign law, and international treaties. + * + * Reproduction, republication or distribution without the + * express written consent of Elasticsearch BV is + * strictly prohibited. + */ + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +namespace ml +{ +namespace maths +{ +namespace +{ + +//! Get the desired weight for the regression model. +double modelWeight(double targetDecayRate, double modelDecayRate) +{ + return targetDecayRate == modelDecayRate ? + 1.0 : std::min(targetDecayRate, modelDecayRate) + / std::max(targetDecayRate, modelDecayRate); +} + +//! We scale the time used for the regression model to improve +//! the condition of the design matrix. +double scaleTime(core_t::TTime time, core_t::TTime origin) +{ + return static_cast(time - origin) / static_cast(core::constants::WEEK); +} + +const std::string TARGET_DECAY_RATE_TAG{"a"}; +const std::string FIRST_UPDATE_TAG{"b"}; +const std::string LAST_UPDATE_TAG{"c"}; +const std::string REGRESSION_ORIGIN_TAG{"d"}; +const std::string MODEL_TAG{"e"}; +const std::string PREDICTION_ERROR_VARIANCE_TAG{"f"}; +const std::string VALUE_MOMENTS_TAG{"g"}; +const std::string WEIGHT_TAG{"a"}; +const std::string REGRESSION_TAG{"b"}; +const std::string RESIDUAL_MOMENTS_TAG{"c"}; + +const double TIME_SCALES[]{144.0, 72.0, 36.0, 12.0, 4.0, 1.0, 0.25, 0.05}; +const std::size_t NUMBER_MODELS{boost::size(TIME_SCALES)}; +const double MAX_CONDITION{1e12}; +const core_t::TTime UNSET_TIME{0}; + +} + +CTrendComponent::CTrendComponent(double decayRate) : + m_DefaultDecayRate(decayRate), + m_TargetDecayRate(decayRate), + m_FirstUpdate(UNSET_TIME), + m_LastUpdate(UNSET_TIME), + m_RegressionOrigin(UNSET_TIME), + m_PredictionErrorVariance(0.0) +{ + for (std::size_t i = 0u; i < NUMBER_MODELS; ++i) + { + m_Models.emplace_back(modelWeight(1.0, TIME_SCALES[i])); + } +} + +void CTrendComponent::swap(CTrendComponent &other) +{ + std::swap(m_DefaultDecayRate, other.m_DefaultDecayRate); + std::swap(m_TargetDecayRate, other.m_TargetDecayRate); + std::swap(m_FirstUpdate, other.m_FirstUpdate); + std::swap(m_LastUpdate, other.m_LastUpdate); + std::swap(m_RegressionOrigin, other.m_RegressionOrigin); + m_Models.swap(other.m_Models); + std::swap(m_PredictionErrorVariance, other.m_PredictionErrorVariance); + std::swap(m_ValueMoments, other.m_ValueMoments); +} + +void CTrendComponent::acceptPersistInserter(core::CStatePersistInserter &inserter) const +{ + inserter.insertValue(TARGET_DECAY_RATE_TAG, m_TargetDecayRate); + inserter.insertValue(FIRST_UPDATE_TAG, m_FirstUpdate); + inserter.insertValue(LAST_UPDATE_TAG, m_LastUpdate); + inserter.insertValue(REGRESSION_ORIGIN_TAG, m_RegressionOrigin); + for (const auto &model : m_Models) + { + inserter.insertLevel(MODEL_TAG, boost::bind(&SModel::acceptPersistInserter, &model, _1)); + } + inserter.insertValue(PREDICTION_ERROR_VARIANCE_TAG, + m_PredictionErrorVariance, + core::CIEEE754::E_DoublePrecision); + inserter.insertValue(VALUE_MOMENTS_TAG, m_ValueMoments.toDelimited()); +} + +bool CTrendComponent::acceptRestoreTraverser(core::CStateRestoreTraverser &traverser) +{ + std::size_t i{0}; + do + { + const std::string &name{traverser.name()}; + RESTORE_BUILT_IN(TARGET_DECAY_RATE_TAG, m_TargetDecayRate) + RESTORE_BUILT_IN(FIRST_UPDATE_TAG, m_FirstUpdate) + RESTORE_BUILT_IN(LAST_UPDATE_TAG, m_LastUpdate) + RESTORE_BUILT_IN(REGRESSION_ORIGIN_TAG, m_RegressionOrigin) + RESTORE(MODEL_TAG, traverser.traverseSubLevel(boost::bind( + &SModel::acceptRestoreTraverser, &m_Models[i++], _1))) + RESTORE_BUILT_IN(PREDICTION_ERROR_VARIANCE_TAG, m_PredictionErrorVariance) + RESTORE(VALUE_MOMENTS_TAG, m_ValueMoments.fromDelimited(traverser.value())) + } + while (traverser.next()); + return true; +} + +bool CTrendComponent::initialized() const +{ + return m_LastUpdate != UNSET_TIME; +} + +void CTrendComponent::clear() +{ + m_FirstUpdate = UNSET_TIME; + m_LastUpdate = UNSET_TIME; + m_RegressionOrigin = UNSET_TIME; + for (std::size_t i = 0u; i < NUMBER_MODELS; ++i) + { + m_Models[i] = SModel(modelWeight(1.0, TIME_SCALES[i])); + } + m_PredictionErrorVariance = 0.0; + m_ValueMoments = TMeanVarAccumulator(); +} + +void CTrendComponent::shiftOrigin(core_t::TTime time) +{ + time = CIntegerTools::floor(time, core::constants::WEEK); + double scaledShift{scaleTime(time, m_RegressionOrigin)}; + if (scaledShift > 0.0) + { + for (auto &&model : m_Models) + { + model.s_Regression.shiftAbscissa(-scaledShift); + } + m_RegressionOrigin = time; + } +} + +void CTrendComponent::shiftSlope(double decayRate, double shift) +{ + for (std::size_t i = 0u; i < NUMBER_MODELS; ++i) + { + double shift_{std::min(m_DefaultDecayRate * TIME_SCALES[i] + / decayRate, 1.0) * shift}; + m_Models[i].s_Regression.shiftGradient(shift_); + } +} + +void CTrendComponent::add(core_t::TTime time, double value, double weight) +{ + // Update the model weights: we weight the components based on the + // relative difference in the component scale and the target scale. + + for (std::size_t i = 0u; i < NUMBER_MODELS; ++i) + { + m_Models[i].s_Weight.add(modelWeight(m_TargetDecayRate, + m_DefaultDecayRate * TIME_SCALES[i])); + } + + // Update the models. + + if (m_FirstUpdate == UNSET_TIME) + { + m_RegressionOrigin = CIntegerTools::floor(time, core::constants::WEEK); + } + + double prediction{CBasicStatistics::mean(this->value(time, 0.0))}; + + double count{this->count()}; + if (count > 0.0) + { + TMeanVarAccumulator moments{CBasicStatistics::accumulator( + count, prediction, m_PredictionErrorVariance)}; + moments.add(value, weight); + m_PredictionErrorVariance = CBasicStatistics::maximumLikelihoodVariance(moments); + } + + double scaledTime{scaleTime(time, m_RegressionOrigin)}; + for (auto &&model : m_Models) + { + model.s_Regression.add(scaledTime, value, weight); + model.s_ResidualMoments.add(value - model.s_Regression.predict(scaledTime, MAX_CONDITION)); + } + m_ValueMoments.add(value); + + m_FirstUpdate = m_FirstUpdate == UNSET_TIME ? time : std::min(m_FirstUpdate, time); + m_LastUpdate = std::max(m_LastUpdate, time); +} + +double CTrendComponent::defaultDecayRate() const +{ + return m_DefaultDecayRate; +} + +void CTrendComponent::decayRate(double decayRate) +{ + m_TargetDecayRate = decayRate; +} + +void CTrendComponent::propagateForwardsByTime(core_t::TTime interval) +{ + TDoubleVec factors(this->factors(interval)); + double median{CBasicStatistics::median(factors)}; + for (std::size_t i = 0u; i < NUMBER_MODELS; ++i) + { + m_Models[i].s_Weight.age(median); + m_Models[i].s_Regression.age(factors[i]); + m_Models[i].s_ResidualMoments.age(factors[i]); + } +} + +CTrendComponent::TDoubleDoublePr CTrendComponent::value(core_t::TTime time, double confidence) const +{ + if (!this->initialized()) + { + return {0.0, 0.0}; + } + + double a{this->weightOfPrediction(time)}; + double b{1.0 - a}; + double scaledTime{scaleTime(time, m_RegressionOrigin)}; + + TMeanAccumulator prediction_; + { + TDoubleVec factors(this->factors(std::abs(time - m_LastUpdate))); + for (std::size_t i = 0u; i < NUMBER_MODELS; ++i) + { + prediction_.add(m_Models[i].s_Regression.predict(scaledTime, MAX_CONDITION), + factors[i] * CBasicStatistics::mean(m_Models[i].s_Weight)); + } + } + + double prediction{ a * CBasicStatistics::mean(prediction_) + + b * CBasicStatistics::mean(m_ValueMoments)}; + + if (confidence > 0.0 && m_PredictionErrorVariance > 0.0) + { + double variance{ a * m_PredictionErrorVariance + / std::max(this->count(), 1.0) + + b * CBasicStatistics::variance(m_ValueMoments) + / std::max(CBasicStatistics::count(m_ValueMoments), 1.0)}; + try + { + boost::math::normal normal{prediction, std::sqrt(variance)}; + double ql{boost::math::quantile(normal, (100.0 - confidence) / 200.0)}; + double qu{boost::math::quantile(normal, (100.0 + confidence) / 200.0)}; + return {ql, qu}; + } + catch (const std::exception &e) + { + LOG_ERROR("Failed calculating confidence interval: " << e.what() + << ", prediction = " << prediction + << ", variance = " << variance + << ", confidence = " << confidence); + } + } + + return {prediction, prediction}; +} + +CTrendComponent::TDoubleDoublePr CTrendComponent::variance(double confidence) const +{ + if (!this->initialized()) + { + return {0.0, 0.0}; + } + + double variance{m_PredictionErrorVariance}; + + if (confidence > 0.0 && m_PredictionErrorVariance > 0.0) + { + double df{std::max(this->count(), 2.0) - 1.0}; + try + { + boost::math::chi_squared chi{df}; + double ql{boost::math::quantile(chi, (100.0 - confidence) / 200.0)}; + double qu{boost::math::quantile(chi, (100.0 + confidence) / 200.0)}; + return {ql * variance / df, qu * variance / df}; + } + catch (const std::exception &e) + { + LOG_ERROR("Failed calculating confidence interval: " << e.what() + << ", df = " << df + << ", confidence = " << confidence); + } + } + + return {variance, variance}; +} + +void CTrendComponent::forecast(core_t::TTime startTime, + core_t::TTime endTime, + core_t::TTime step, + double confidence, + TDouble3VecVec &result) const +{ + result.clear(); + + if (endTime < startTime) + { + LOG_ERROR("Bad forecast range: [" << startTime << "," << endTime << "]"); + return; + } + if (confidence < 0.0 || confidence >= 100.0) + { + LOG_ERROR("Bad confidence interval: " << confidence << "%"); + return; + } + + endTime = startTime + CIntegerTools::ceil(endTime - startTime, step); + + core_t::TTime steps{(endTime - startTime) / step}; + result.resize(steps, TDouble3Vec(3)); + + TDoubleVec factors(this->factors(step)); + + TDoubleVec modelWeights(this->initialForecastModelWeights()); + TRegressionArrayVec models(NUMBER_MODELS); + TMatrixVec modelCovariances(NUMBER_MODELS); + TDoubleVec residualVarianceWeights(this->initialForecastErrorWeights()); + TDoubleVec residualVariances(NUMBER_MODELS); + for (std::size_t i = 0u; i < NUMBER_MODELS; ++i) + { + m_Models[i].s_Regression.parameters(models[i], MAX_CONDITION); + m_Models[i].s_Regression.covariances(m_PredictionErrorVariance, modelCovariances[i], MAX_CONDITION); + modelCovariances[i] /= std::max(m_Models[i].s_Regression.count(), 1.0); + residualVariances[i] = std::pow(CBasicStatistics::mean(m_Models[i].s_ResidualMoments), 2.0) + + CBasicStatistics::variance(m_Models[i].s_ResidualMoments); + LOG_TRACE("params = " << core::CContainerPrinter::print(models[i])); + LOG_TRACE("covariances = " << modelCovariances[i].toDelimited()) + LOG_TRACE("variances = " << residualVariances[i]); + } + LOG_TRACE("long time variance = " << CBasicStatistics::variance(m_ValueMoments)); + + for (core_t::TTime time = startTime; time < endTime; time += step) + { + core_t::TTime pillar{(time - startTime) / step}; + double scaledDt{scaleTime(time, startTime)}; + TVector times({0.0, scaledDt, scaledDt * scaledDt}); + + double a{this->weightOfPrediction(time)}; + double b{1.0 - a}; + + for (std::size_t j = 0u; j < NUMBER_MODELS; ++j) + { + modelWeights[j] *= factors[j]; + residualVarianceWeights[j] *= std::pow(factors[j], 2.0); + } + + TMeanAccumulator variance_; + std::size_t last{NUMBER_MODELS - 1}; + for (std::size_t j = 0u; j < last; ++j) + { + variance_.add( times.inner(modelCovariances[j + 1] * times) + + residualVariances[j], residualVarianceWeights[j]); + } + variance_.add(residualVariances[last], residualVarianceWeights[last]); + variance_.add(CBasicStatistics::variance(m_ValueMoments), + residualVarianceWeights[NUMBER_MODELS]); + + double prediction{this->value(modelWeights, models, scaleTime(time, m_RegressionOrigin))}; + double ql{0.0}; + double qu{0.0}; + double variance{ a * CBasicStatistics::mean(variance_) + + b * CBasicStatistics::variance(m_ValueMoments)}; + try + { + boost::math::normal normal{0.0, std::sqrt(variance)}; + ql = boost::math::quantile(normal, (100.0 - confidence) / 200.0); + qu = boost::math::quantile(normal, (100.0 + confidence) / 200.0); + } + catch (const std::exception &e) + { + LOG_ERROR("Failed calculating confidence interval: " << e.what() + << ", variance = " << variance + << ", confidence = " << confidence); + } + + result[pillar][0] = prediction + ql; + result[pillar][1] = prediction; + result[pillar][2] = prediction + qu; + } +} + +core_t::TTime CTrendComponent::observedInterval() const +{ + return m_LastUpdate - m_FirstUpdate; +} + +double CTrendComponent::parameters() const +{ + return static_cast(TRegression::N); +} + +uint64_t CTrendComponent::checksum(uint64_t seed) const +{ + seed = CChecksum::calculate(seed, m_TargetDecayRate); + seed = CChecksum::calculate(seed, m_FirstUpdate); + seed = CChecksum::calculate(seed, m_LastUpdate); + seed = CChecksum::calculate(seed, m_Models); + seed = CChecksum::calculate(seed, m_PredictionErrorVariance); + return CChecksum::calculate(seed, m_ValueMoments); +} + +std::string CTrendComponent::print() const +{ + std::ostringstream result; + for (const auto &model : m_Models) + { + result << model.s_Regression.print() << "\n"; + } + return result.str(); +} + +CTrendComponent::TDoubleVec CTrendComponent::factors(core_t::TTime interval) const +{ + TDoubleVec result(NUMBER_MODELS); + double factor{ m_DefaultDecayRate + * static_cast(interval) + / static_cast(core::constants::DAY)}; + for (std::size_t i = 0u; i < NUMBER_MODELS; ++i) + { + result[i] = std::exp(-TIME_SCALES[i] * factor); + } + return result; +} + +CTrendComponent::TDoubleVec CTrendComponent::initialForecastModelWeights() const +{ + TDoubleVec result(NUMBER_MODELS); + for (std::size_t i = 0u; i < NUMBER_MODELS; ++i) + { + result[i] = std::exp( static_cast(NUMBER_MODELS / 2) + - static_cast(i)); + } + return result; +} + +CTrendComponent::TDoubleVec CTrendComponent::initialForecastErrorWeights() const +{ + TDoubleVec result(NUMBER_MODELS + 1, 1.0); + result[0] = std::exp(1.0); + for (std::size_t i = 0u; i < NUMBER_MODELS; ++i) + { + result[i] *= std::exp( static_cast(NUMBER_MODELS / 2) + - static_cast(i)); + } + result[NUMBER_MODELS] = result[NUMBER_MODELS - 1] / std::exp(1.0); + return result; +} + +double CTrendComponent::count() const +{ + TMeanAccumulator result; + for (const auto &model : m_Models) + { + result.add(CTools::fastLog(model.s_Regression.count()), + CBasicStatistics::mean(model.s_Weight)); + } + return std::exp(CBasicStatistics::mean(result)); +} + +double CTrendComponent::value(const TDoubleVec &weights, + const TRegressionArrayVec &models, + double time) const +{ + TMeanAccumulator prediction; + for (std::size_t i = 0u; i < models.size(); ++i) + { + prediction.add(CRegression::predict(models[i], time), weights[i]); + } + return CBasicStatistics::mean(prediction); +} + +double CTrendComponent::weightOfPrediction(core_t::TTime time) const +{ + double interval{static_cast(m_LastUpdate - m_FirstUpdate)}; + if (interval == 0.0) + { + return 0.0; + } + + double extrapolateInterval{static_cast( + CBasicStatistics::max(time - m_LastUpdate, m_FirstUpdate - time, core_t::TTime(0)))}; + if (extrapolateInterval == 0.0) + { + return 1.0; + } + + return CTools::smoothHeaviside(extrapolateInterval / interval, 1.0 / 12.0, -1.0); +} + +CTrendComponent::SModel::SModel(double weight) +{ + s_Weight.add(weight, 0.01); +} + +void CTrendComponent::SModel::acceptPersistInserter(core::CStatePersistInserter &inserter) const +{ + inserter.insertValue(WEIGHT_TAG, s_Weight.toDelimited()); + inserter.insertLevel(REGRESSION_TAG, boost::bind(&TRegression::acceptPersistInserter, + &s_Regression, _1)); + inserter.insertValue(RESIDUAL_MOMENTS_TAG, s_ResidualMoments.toDelimited()); +} + +bool CTrendComponent::SModel::acceptRestoreTraverser(core::CStateRestoreTraverser &traverser) +{ + do + { + const std::string &name{traverser.name()}; + RESTORE(WEIGHT_TAG, s_Weight.fromDelimited(traverser.value())) + RESTORE(REGRESSION_TAG, traverser.traverseSubLevel(boost::bind( + &TRegression::acceptRestoreTraverser, &s_Regression, _1))) + RESTORE(RESIDUAL_MOMENTS_TAG, s_ResidualMoments.fromDelimited(traverser.value())) + } + while (traverser.next()); + return true; +} + +uint64_t CTrendComponent::SModel::checksum(uint64_t seed) const +{ + seed = CChecksum::calculate(seed, s_Weight); + seed = CChecksum::calculate(seed, s_Regression); + return CChecksum::calculate(seed, s_ResidualMoments); +} + +} +} diff --git a/lib/maths/CTrendTests.cc b/lib/maths/CTrendTests.cc index a8f6e5ef40..43408c4c65 100644 --- a/lib/maths/CTrendTests.cc +++ b/lib/maths/CTrendTests.cc @@ -65,42 +65,8 @@ namespace { using TDoubleVec = std::vector; -using TTimeVec = std::vector; -using TSizeSizePr = std::pair; -using TSizeSizePr2Vec = core::CSmallVector; using TMeanAccumulator = CBasicStatistics::SSampleMean::TAccumulator; -using TMeanAccumulatorVec = std::vector; -using TFloatMeanAccumulator = CBasicStatistics::SSampleMean::TAccumulator; -using TFloatMeanAccumulatorVec = std::vector; -using TMeanVarAccumulator = CBasicStatistics::SSampleMeanVar::TAccumulator; -using TMeanVarAccumulatorVec = std::vector; -using TTimeTimePr = std::pair; -using TTimeTimePr2Vec = core::CSmallVector; -using TTimeTimePrMeanVarAccumulatorPr = std::pair; - -const core_t::TTime HOUR = core::constants::HOUR; -const core_t::TTime DAY = core::constants::DAY; -const core_t::TTime WEEKEND = core::constants::WEEKEND; -const core_t::TTime WEEK = core::constants::WEEK; - -//! The diurnal components. -enum EDiurnalComponents -{ - E_WeekendDay, - E_WeekendWeek, - E_WeekdayDay, - E_WeekdayWeek, - E_Day, - E_Week, -}; -//! The periods of the diurnal components. -const core_t::TTime DIURNAL_PERIODS[]{DAY, WEEK}; -//! The weekend/day windows. -const TTimeTimePr DIURNAL_WINDOWS[]{{0, WEEKEND}, {WEEKEND, WEEK}, {0, WEEK}}; -//! The names of the the diurnal periodic components. -const std::string DIURNAL_PERIOD_NAMES[]{"daily", "weekly", ""}; -//! The names of the weekday/end partitions for diurnal components. -const std::string DIURNAL_WINDOW_NAMES[]{"weekend", "weekday", ""}; +using TTimeVec = std::vector; //! \brief Sets the timezone to a specified value in a constructor //! call so it can be called once by static initialisation. @@ -112,81 +78,6 @@ struct SSetTimeZone } }; -//! \brief Accumulates the minimum amplitude. -class CMinAmplitude -{ - public: - CMinAmplitude(std::size_t n, double level) : - m_Level(level), - m_Count(0), - m_Min(std::max(n, MINIMUM_COUNT_TO_TEST)), - m_Max(std::max(n, MINIMUM_COUNT_TO_TEST)) - {} - - void add(double x, double n) - { - if (n > 0.0) - { - ++m_Count; - m_Min.add(x - m_Level); - m_Max.add(x - m_Level); - } - } - - double amplitude(void) const - { - if (this->count() >= MINIMUM_COUNT_TO_TEST) - { - return std::max(std::max(-m_Min.biggest(), 0.0), - std::max( m_Max.biggest(), 0.0)); - } - return 0.0; - } - - double significance(const boost::math::normal &normal) const - { - if (this->count() < MINIMUM_COUNT_TO_TEST) - { - return 1.0; - } - - double F{2.0 * CTools::safeCdf(normal, -this->amplitude())}; - if (F == 0.0) - { - return 0.0; - } - - double n{static_cast(this->count())}; - boost::math::binomial binomial(static_cast(m_Count), F); - return CTools::safeCdfComplement(binomial, n - 1.0); - } - - private: - using TMinAccumulator = CBasicStatistics::COrderStatisticsHeap; - using TMaxAccumulator = CBasicStatistics::COrderStatisticsHeap>; - - private: - std::size_t count(void) const { return m_Min.count(); } - - private: - //! The minimum number of repeats for which we'll test. - static const std::size_t MINIMUM_COUNT_TO_TEST; - - private: - //! The mean of the trend. - double m_Level; - //! The total count of values added. - std::size_t m_Count; - //! The smallest values. - TMinAccumulator m_Min; - //! The largest values. - TMaxAccumulator m_Max; -}; - -const std::size_t CMinAmplitude::MINIMUM_COUNT_TO_TEST{3}; - -using TMinAmplitudeVec = std::vector; - //! Generate \p n samples uniformly in the interval [\p a, \p b]. template void generateUniformSamples(boost::random::mt19937_64 &rng, @@ -213,281 +104,6 @@ void zeroMean(TDoubleVec &samples) } } -//! Compute the \p percentage % variance for a chi-squared random -//! variance with \p df degrees of freedom. -double varianceAtPercentile(double variance, double df, double percentage) -{ - try - { - boost::math::chi_squared chi(df); - return boost::math::quantile(chi, percentage / 100.0) / df * variance; - } - catch (const std::exception &e) - { - LOG_ERROR("Bad input: " << e.what() - << ", df = " << df - << ", percentage = " << percentage); - } - return variance; -} - -//! Compute the \p percentage % autocorrelation for a F distributed -//! random autocorrelation with parameters \p n - 1 and \p n - 1. -double autocorrelationAtPercentile(double autocorrelation, double n, double percentage) -{ - try - { - boost::math::fisher_f f(n - 1.0, n - 1.0); - return boost::math::quantile(f, percentage / 100.0) * autocorrelation; - } - catch (const std::exception &e) - { - LOG_ERROR("Bad input: " << e.what() - << ", n = " << n - << ", percentage = " << percentage); - } - return autocorrelation; -} - -//! Get the length of the \p window. -template -T length(const std::pair &window) -{ - return window.second - window.first; -} - -//! Get the total length of the \p windows. -template -T length(const core::CSmallVector, 2> &windows) -{ - return std::accumulate(windows.begin(), windows.end(), 0, - [](core_t::TTime length_, const TTimeTimePr &window) - { return length_ + length(window); }); -} - -//! Compute the windows at repeat \p repeat with length \p length. -TTimeTimePr2Vec calculateWindows(core_t::TTime startOfWeek, - core_t::TTime window, - core_t::TTime repeat, - const TTimeTimePr &interval) -{ - core_t::TTime a{startOfWeek + interval.first}; - core_t::TTime b{startOfWeek + window}; - core_t::TTime l{length(interval)}; - TTimeTimePr2Vec result; - result.reserve((b - a) / repeat); - for (core_t::TTime time = a; time < b; time += repeat) - { - result.emplace_back(time, time + l); - } - return result; -} - -//! Get the index ranges corresponding to \p windows. -std::size_t calculateIndexWindows(const TTimeTimePr2Vec &windows, - core_t::TTime bucketLength, - TSizeSizePr2Vec &result) -{ - std::size_t l(0); - result.reserve(windows.size()); - for (const auto &window : windows) - { - core_t::TTime a{window.first / bucketLength}; - core_t::TTime b{window.second / bucketLength}; - result.emplace_back(a, b); - l += b - a; - } - return l; -} - -//! Compute the projection of \p values to \p windows. -void project(const TFloatMeanAccumulatorVec &values, - const TTimeTimePr2Vec &windows_, - core_t::TTime bucketLength, - TFloatMeanAccumulatorVec &result) -{ - TSizeSizePr2Vec windows; - calculateIndexWindows(windows_, bucketLength, windows); - result.clear(); - result.reserve(length(windows)); - std::size_t n{values.size()}; - for (std::size_t i = 0u; i < windows.size(); ++i) - { - std::size_t a{windows[i].first}; - std::size_t b{windows[i].second}; - for (std::size_t j = a; j < b; ++j) - { - const TFloatMeanAccumulator &value{values[j % n]}; - result.push_back(value); - } - } -} - -//! Compute the periodic trend from \p values falling in \p windows. -template -void periodicTrend(const TFloatMeanAccumulatorVec &values, - const TSizeSizePr2Vec &windows_, - core_t::TTime bucketLength, T &trend) -{ - if (!trend.empty()) - { - TSizeSizePr2Vec windows; - calculateIndexWindows(windows_, bucketLength, windows); - std::size_t period{trend.size()}; - std::size_t n{values.size()}; - for (std::size_t i = 0u; i < windows.size(); ++i) - { - std::size_t a{windows[i].first}; - std::size_t b{windows[i].second}; - for (std::size_t j = a; j < b; ++j) - { - const TFloatMeanAccumulator &value{values[j % n]}; - trend[(j - a) % period].add(CBasicStatistics::mean(value), - CBasicStatistics::count(value)); - } - } - } -} - -//! Compute the average of the values at \p times. -void averageValue(const TFloatMeanAccumulatorVec &values, - const TTimeVec ×, - core_t::TTime bucketLength, - TMeanVarAccumulator &value) -{ - for (auto time : times) - { - std::size_t index(time / bucketLength); - value.add(CBasicStatistics::mean(values[index]), - CBasicStatistics::count(values[index])); - } -} - -//! Compute the variance of the \p trend values. -template -double trendVariance(const T &trend) -{ - TMeanVarAccumulator result; - for (const auto &value : trend) - { - result.add(CBasicStatistics::mean(value), - CBasicStatistics::count(value)); - } - return CBasicStatistics::variance(result); -} - -//! Get the maximum residual of \p trend. -template -double trendAmplitude(const T &trend) -{ - using TMaxAccumulator = CBasicStatistics::SMax::TAccumulator; - - TMeanAccumulator level; - for (const auto &bucket : trend) - { - level.add(mean(bucket), count(bucket)); - } - - TMaxAccumulator result; - result.add(0.0); - for (const auto &bucket : trend) - { - if (count(bucket) > 0.0) - { - result.add(std::fabs(mean(bucket) - CBasicStatistics::mean(level))); - } - } - - return result[0]; -} - -//! Extract the residual variance from the mean of a collection -//! of residual variances. -double residualVariance(const TMeanAccumulator &mean) -{ - double n{CBasicStatistics::count(mean)}; - return std::max(n / (n - 1.0) * CBasicStatistics::mean(mean), 0.0); -} - -//! Extract the residual variance of \p bucket of a trend. -TMeanAccumulator residualVariance(const TMeanVarAccumulator &bucket, - double scale) -{ - return CBasicStatistics::accumulator(scale * CBasicStatistics::count(bucket), - CBasicStatistics::maximumLikelihoodVariance(bucket)); -} - -//! \brief Partially specialized helper class to get the trend -//! residual variance as a specified type. -template struct SResidualVarianceImpl {}; - -//! \brief Get the residual variance as a double. -template<> -struct SResidualVarianceImpl -{ - static double get(const TMeanAccumulator &mean) - { - return residualVariance(mean); - } -}; - -//! \brief Get the residual variance as a mean accumulator. -template<> -struct SResidualVarianceImpl -{ - static TMeanAccumulator get(const TMeanAccumulator &mean) - { - return mean; - } -}; - -//! Compute the residual variance of the trend \p trend. -template -R residualVariance(const T &trend, double scale) -{ - TMeanAccumulator result; - for (const auto &bucket : trend) - { - result.add(CBasicStatistics::maximumLikelihoodVariance(bucket), - CBasicStatistics::count(bucket)); - } - result.s_Count *= scale; - return SResidualVarianceImpl::get(result); -} - -//! Get a diurnal result. -CPeriodicityTestResult diurnalResult(core::CSmallVector components, - core_t::TTime startOfWeek = 0) -{ - CPeriodicityTestResult result; - for (auto component : components) - { - result.add(component, startOfWeek, - DIURNAL_PERIODS[static_cast(component) % 2], - DIURNAL_WINDOWS[static_cast(component) / 2]); - - } - return result; -} - -//! Cyclic permutation of \p values with shift \p shift. -void cyclicShift(std::size_t shift, TFloatMeanAccumulatorVec &values) -{ - std::size_t n = values.size(); - TFloatMeanAccumulatorVec result(n); - for (std::size_t i = 0u; i < n; ++i) - { - result[(i + shift) % n] = values[i]; - } - values.swap(result); -} - -// CTrendTest -const std::string DECAY_RATE_TAG("a"); -const std::string TIME_ORIGIN_TAG("b"); -const std::string TREND_TAG("c"); -const std::string VARIANCES_TAG("d"); - // CRandomizedPeriodicityTest // statics const std::string RNG_TAG("a"); @@ -506,19 +122,6 @@ const std::string WEEK_PROJECTIONS_TAG("d"); const std::string WEEK_STATISTICS_TAG("e"); const std::string WEEK_REFRESHED_PROJECTIONS_TAG("f"); -// CPeriodicityTestResult -const std::string COMPONENTS_TAG("a"); - -// CPeriodicityTest -const std::string WINDOW_LENGTH_TAG("a"); -const std::string BUCKET_LENGTH_TAG("b"); -const std::string BUCKET_VALUE_TAG("c"); - -// CScanningPeriodicityTest -const std::string BUCKET_LENGTH_INDEX_TAG("a"); -//const std::string BUCKET_VALUE_TAG("c"); -const std::string START_TIME_TAG("d"); - // CCalendarCyclicTest const std::string ERROR_QUANTILES_TAG("a"); const std::string BUCKET_TAG("c"); @@ -527,139 +130,13 @@ const std::string ERROR_SUMS_TAG("e"); //! The maximum significance of a test statistic. const double MAXIMUM_SIGNIFICANCE = 0.001; -//! The confidence interval used for test statistic values. -const double CONFIDENCE_INTERVAL = 80.0; -//! The test bucket lengths for the diurnal periodicity test. -//! The data bucketing interval is snapped to the least longer -//! bucket length if there is one. -const core_t::TTime DIURNAL_PERMITTED_BUCKET_LENGTHS[] - { - HOUR, 2 * HOUR, 3 * HOUR, 4 * HOUR, 6 * HOUR, 8 * HOUR, 12 * HOUR, DAY - }; -//! Scales to apply to the minimum partitioned variance when -//! testing the significance. -const double DIURNAL_VARIANCE_CORRECTIONS[][8] - { - { 1.08, 1.08, 1.08, 1.08, 1.09, 1.1, 1.11, 1.15 }, - { 1.31, 1.31, 1.31, 1.31, 1.12, 1.0, 1.0, 1.0 } - }; - -} - -//////// CTrendTest //////// - -CTrendTest::CTrendTest(double decayRate) : - m_DecayRate(decayRate), m_TimeOrigin(0) -{} - -bool CTrendTest::acceptRestoreTraverser(core::CStateRestoreTraverser &traverser) -{ - do - { - const std::string &name = traverser.name(); - RESTORE_BUILT_IN(DECAY_RATE_TAG, m_DecayRate) - RESTORE_BUILT_IN(TIME_ORIGIN_TAG, m_TimeOrigin) - RESTORE(TREND_TAG, traverser.traverseSubLevel( - boost::bind(&TRegression::acceptRestoreTraverser, &m_Trend, _1))) - RESTORE(VARIANCES_TAG, m_Variances.fromDelimited(traverser.value())) - } - while (traverser.next()); - return true; -} - -void CTrendTest::acceptPersistInserter(core::CStatePersistInserter &inserter) const -{ - inserter.insertValue(DECAY_RATE_TAG, m_DecayRate); - inserter.insertValue(TIME_ORIGIN_TAG, m_TimeOrigin); - inserter.insertLevel(TREND_TAG, boost::bind(&TRegression::acceptPersistInserter, &m_Trend, _1)); - inserter.insertValue(VARIANCES_TAG, m_Variances.toDelimited()); -} - -void CTrendTest::decayRate(double decayRate) -{ - m_DecayRate = decayRate; -} - -void CTrendTest::propagateForwardsByTime(double time) -{ - if (!CMathsFuncs::isFinite(time) || time < 0.0) - { - LOG_ERROR("Bad propagation time " << time); - return; - } - double factor = std::exp(-m_DecayRate * time); - m_Trend.age(factor); - m_Variances.age(factor); -} - -void CTrendTest::add(core_t::TTime time, double value, double weight) -{ - core_t::TTime origin = CIntegerTools::floor(time, WEEK); - double shift = this->time(origin); - if (shift > 0.0) - { - m_Trend.shiftAbscissa(-shift); - m_TimeOrigin = origin; - } - m_Trend.add(this->time(time), value, weight); -} - -void CTrendTest::captureVariance(core_t::TTime time, double value, double weight) -{ - double prediction = CRegression::predict(m_Trend, this->time(time)); - TVector values; - values(0) = value; - values(1) = value - prediction; - m_Variances.add(values, weight); -} - -void CTrendTest::shift(double shift) -{ - m_Trend.shiftOrdinate(shift); -} - -bool CTrendTest::test(void) const -{ - double n = CBasicStatistics::count(m_Variances); - double df0 = n - 1.0; - double df1 = n - ORDER - 1.0; - double v0 = CBasicStatistics::maximumLikelihoodVariance(m_Variances)(0); - double v1 = CBasicStatistics::maximumLikelihoodVariance(m_Variances)(1); - return n > 3 * ORDER - && varianceAtPercentile(v1, df1, 80.0) < MAXIMUM_TREND_VARIANCE_RATIO * v0 - && CStatisticalTests::leftTailFTest(v1 / v0, df1, df0) <= MAXIMUM_SIGNIFICANCE; -} - -const CTrendTest::TRegression &CTrendTest::trend(void) const -{ - return m_Trend; -} - -core_t::TTime CTrendTest::origin(void) const -{ - return m_TimeOrigin; -} - -double CTrendTest::variance(void) const -{ - return CBasicStatistics::maximumLikelihoodVariance(m_Variances)(1); -} - -uint64_t CTrendTest::checksum(uint64_t seed) const -{ - seed = CChecksum::calculate(seed, m_DecayRate); - seed = CChecksum::calculate(seed, m_TimeOrigin); - seed = CChecksum::calculate(seed, m_Trend); - return CChecksum::calculate(seed, m_Variances); -} +//! Forward day in seconds into scope. +const core_t::TTime DAY = core::constants::DAY; +//! Forward day in seconds into scope. +const core_t::TTime WEEK = core::constants::WEEK; -double CTrendTest::time(core_t::TTime time) const -{ - return static_cast(time - m_TimeOrigin) / static_cast(WEEK); } -const double CTrendTest::MAXIMUM_TREND_VARIANCE_RATIO{0.5}; - //////// CRandomizedPeriodicitytest //////// CRandomizedPeriodicityTest::CRandomizedPeriodicityTest(void) : @@ -1006,1455 +483,6 @@ TDoubleVec CRandomizedPeriodicityTest::ms_WeekPeriodicProjections[N] = {}; std::atomic CRandomizedPeriodicityTest::ms_WeekResampled(-WEEK_RESAMPLE_INTERVAL); core::CMutex CRandomizedPeriodicityTest::ms_Lock; -//////// CPeriodicityResultTest //////// - -bool CPeriodicityTestResult::acceptRestoreTraverser(core::CStateRestoreTraverser &traverser) -{ - do - { - const std::string name = traverser.name(); - RESTORE(COMPONENTS_TAG, core::CPersistUtils::fromString(traverser.value(), - SComponent::fromString, - m_Components)) - } - while (traverser.next()); - return true; -} - -void CPeriodicityTestResult::acceptPersistInserter(core::CStatePersistInserter &inserter) const -{ - inserter.insertValue(COMPONENTS_TAG, core::CPersistUtils::toString(m_Components, - SComponent::toString)); -} - -bool CPeriodicityTestResult::operator==(const CPeriodicityTestResult &other) const -{ - return m_Components == other.m_Components; -} - -const CPeriodicityTestResult &CPeriodicityTestResult::operator+=(const CPeriodicityTestResult &other) -{ - m_Components.insert(m_Components.end(), other.m_Components.begin(), other.m_Components.end()); - return *this; -} - -void CPeriodicityTestResult::add(unsigned int id, - core_t::TTime startOfPartition, - core_t::TTime period, - const TTimeTimePr &window) -{ - m_Components.emplace_back(id, startOfPartition, period, window); -} - -bool CPeriodicityTestResult::periodic(void) const -{ - return m_Components.size() > 0; -} - -const CPeriodicityTestResult::TComponent4Vec &CPeriodicityTestResult::components(void) const -{ - return m_Components; -} - -uint64_t CPeriodicityTestResult::checksum(void) const -{ - return CChecksum::calculate(0, m_Components); -} - -CPeriodicityTestResult::SComponent::SComponent(void) : - s_Id(0), s_StartOfPartition(0), s_Period(0) -{} - -CPeriodicityTestResult::SComponent::SComponent(unsigned int id, - core_t::TTime startOfPartition, - core_t::TTime period, - const TTimeTimePr &window) : - s_Id(id), - s_StartOfPartition(startOfPartition), - s_Period(period), - s_Window(window) -{} - -bool CPeriodicityTestResult::SComponent::fromString(const std::string &value, - SComponent &result) -{ - boost::array state; - if (core::CPersistUtils::fromString(value, state, core::CPersistUtils::PAIR_DELIMITER)) - { - result.s_Id = static_cast(state[0]); - result.s_StartOfPartition = static_cast(state[1]); - result.s_Period = static_cast(state[2]); - result.s_Window.first = static_cast(state[3]); - result.s_Window.second = static_cast(state[4]); - return true; - } - return false; -} - -std::string CPeriodicityTestResult::SComponent::toString(const SComponent &component) -{ - boost::array state; - state[0] = static_cast(component.s_Id); - state[1] = component.s_StartOfPartition; - state[2] = component.s_Period; - state[3] = component.s_Window.first; - state[4] = component.s_Window.second; - return core::CPersistUtils::toString(state, core::CPersistUtils::PAIR_DELIMITER); -} - -bool CPeriodicityTestResult::SComponent::operator==(const SComponent &other) const -{ - return s_Id == other.s_Id && s_StartOfPartition == other.s_StartOfPartition; -} - -uint64_t CPeriodicityTestResult::SComponent::checksum(void) const -{ - uint64_t seed = CChecksum::calculate(0, s_Id); - seed = CChecksum::calculate(seed, s_StartOfPartition); - seed = CChecksum::calculate(seed, s_Period); - return CChecksum::calculate(seed, s_Window); -} - -//////// CPeriodicityTest //////// - -CPeriodicityTest::CPeriodicityTest(void) : - m_DecayRate(0.0), - m_BucketLength(0), - m_WindowLength(0) -{} - -CPeriodicityTest::CPeriodicityTest(double decayRate) : - m_DecayRate(decayRate), - m_BucketLength(0), - m_WindowLength(0) -{} - -bool CPeriodicityTest::initialized(void) const -{ - return m_BucketValues.size() > 0; -} - -void CPeriodicityTest::propagateForwardsByTime(double time) -{ - if (!CMathsFuncs::isFinite(time) || time < 0.0) - { - LOG_ERROR("Bad propagation time " << time); - return; - } - if (this->initialized()) - { - double factor = std::exp(-m_DecayRate * time); - std::for_each(m_BucketValues.begin(), m_BucketValues.end(), - [factor](TFloatMeanAccumulator &value) { value.age(factor); }); - } -} - -void CPeriodicityTest::add(core_t::TTime time, double value, double weight) -{ - if (!m_BucketValues.empty()) - { - std::size_t i((time % m_WindowLength) / m_BucketLength); - m_BucketValues[i].add(value, weight); - } -} - -double CPeriodicityTest::populatedRatio(void) const -{ - if (!m_BucketValues.empty()) - { - return static_cast( - std::count_if(m_BucketValues.begin(), m_BucketValues.end(), - [](const TFloatMeanAccumulator &value) - { return CBasicStatistics::count(value) > 0.0; }) - / static_cast(m_BucketValues.size())); - } - return 0.0; -} - -bool CPeriodicityTest::seenSufficientData(void) const -{ - double populated{0.0}; - CBasicStatistics::CMinMax range; - for (std::size_t i = 0u; i < m_BucketValues.size(); ++i) - { - if (CBasicStatistics::count(m_BucketValues[i]) > 0.0) - { - populated += 1.0; - range.add(static_cast(i)); - } - } - return range.max() - range.min() >= ACCURATE_TEST_POPULATED_FRACTION - * static_cast(m_WindowLength / m_BucketLength) - && populated > ACCURATE_TEST_POPULATED_FRACTION - * static_cast(m_BucketValues.size()); -} - -void CPeriodicityTest::debugMemoryUsage(core::CMemoryUsage::TMemoryUsagePtr mem) const -{ - mem->setName("CPeriodicityTest"); - core::CMemoryDebug::dynamicSize("m_BucketValues", m_BucketValues, mem); -} - -std::size_t CPeriodicityTest::memoryUsage(void) const -{ - return core::CMemory::dynamicSize(m_BucketValues); -} - -std::size_t CPeriodicityTest::staticSize(void) const -{ - return sizeof(*this); -} - -core_t::TTime CPeriodicityTest::bucketLength(void) const -{ - return m_BucketLength; -} - -core_t::TTime CPeriodicityTest::windowLength(void) const -{ - return m_WindowLength; -} - -const CPeriodicityTest::TFloatMeanAccumulatorVec &CPeriodicityTest::bucketValues(void) const -{ - return m_BucketValues; -} - -bool CPeriodicityTest::acceptRestoreTraverser(core::CStateRestoreTraverser &traverser) -{ - m_BucketValues.clear(); - do - { - const std::string &name = traverser.name(); - RESTORE_BUILT_IN(BUCKET_LENGTH_TAG, m_BucketLength) - RESTORE_BUILT_IN(WINDOW_LENGTH_TAG, m_WindowLength) - RESTORE(BUCKET_VALUE_TAG, core::CPersistUtils::restore(BUCKET_VALUE_TAG, m_BucketValues, traverser)) - } - while (traverser.next()); - return true; -} - -void CPeriodicityTest::acceptPersistInserter(core::CStatePersistInserter &inserter) const -{ - inserter.insertValue(BUCKET_LENGTH_TAG, m_BucketLength); - inserter.insertValue(WINDOW_LENGTH_TAG, m_WindowLength); - core::CPersistUtils::persist(BUCKET_VALUE_TAG, m_BucketValues, inserter); -} - -uint64_t CPeriodicityTest::checksum(uint64_t seed) const -{ - seed = CChecksum::calculate(seed, m_DecayRate); - seed = CChecksum::calculate(seed, m_BucketLength); - seed = CChecksum::calculate(seed, m_WindowLength); - return CChecksum::calculate(seed, m_BucketValues); -} - -void CPeriodicityTest::initialize(core_t::TTime bucketLength, - core_t::TTime windowLength, - const TFloatMeanAccumulatorVec &initial) -{ - m_BucketLength = bucketLength; - m_WindowLength = windowLength; - m_BucketValues.resize(static_cast(windowLength / m_BucketLength)); - std::copy(initial.begin(), - initial.begin() + std::min(initial.size(), m_BucketValues.size()), - m_BucketValues.begin()); -} - -bool CPeriodicityTest::initializeTestStatistics(STestStats &stats) const -{ - CBasicStatistics::CMinMax range; - double populated{0.0}; - double count{0.0}; - for (std::size_t i = 0u; i < m_BucketValues.size(); ++i) - { - double ni{CBasicStatistics::count(m_BucketValues[i])}; - count += ni; - if (ni > 0.0) - { - populated += 1.0; - range.add(static_cast(i)); - } - } - - if (populated == 0.0) - { - return false; - } - - LOG_TRACE("populated = " << 100.0 * populated << "%"); - - stats.s_Range = range.max() - range.min(); - stats.s_B = populated; - stats.s_M = count / stats.s_B; - LOG_TRACE("range = " << stats.s_Range - << ", populatedBuckets = " << stats.s_B - << ", valuesPerBucket = " << stats.s_M); - - return true; -} - -bool CPeriodicityTest::nullHypothesis(STestStats &stats) const -{ - TMeanVarAccumulatorVec trend(1); - TTimeTimePr2Vec window{{0, this->windowLength()}}; - periodicTrend(m_BucketValues, window, m_BucketLength, trend); - double mean{CBasicStatistics::mean(trend[0])}; - double v0{CBasicStatistics::variance(trend[0])}; - LOG_TRACE("mean = " << mean); - LOG_TRACE("variance = " << v0); - if (std::sqrt(v0) <= MINIMUM_COEFFICIENT_OF_VARIATION * std::fabs(mean)) - { - return false; - } - stats.s_DF0 = stats.s_B - 1.0; - stats.s_V0 = varianceAtPercentile(v0, stats.s_DF0, 50.0 + CONFIDENCE_INTERVAL / 2.0); - stats.s_T0.assign(1, TDoubleVec{mean}); - stats.s_Partition = window; - return true; -} - -bool CPeriodicityTest::testPeriod(const TTimeTimePr2Vec &windows, - core_t::TTime period_, - STestStats &stats) const -{ - // We use two tests to check for the period: - // 1) That it explains both a non-negligible absolute and statistically - // significant amount of variance and the cyclic autocorrelation at - // that repeat is high enough OR - // 2) There is a large absolute and statistically significant periodic - // spike or trough. - - LOG_TRACE("Testing period " << period_); - - period_ = std::min(period_, length(windows[0])); - std::size_t period{static_cast(period_ / m_BucketLength)}; - - // We need to observe a minimum number of repeated values to test with - // an acceptable false positive rate. - double repeats{0.0}; - for (std::size_t i = 0u; i < period; ++i) - { - for (std::size_t j = i + period; j < m_BucketValues.size(); j += period) - { - if ( CBasicStatistics::count(m_BucketValues[j]) - * CBasicStatistics::count(m_BucketValues[j - period]) > 0.0) - { - repeats += 1.0; - break; - } - } - } - LOG_TRACE("repeats = " << repeats); - if (repeats < static_cast(period) * ACCURATE_TEST_POPULATED_FRACTION / 3.0) - { - return false; - } - - TTimeTimePr2Vec window{{0, length(windows)}}; - double M{stats.s_M}; - double scale{1.0 / M}; - double v0{stats.s_V0}, df0{stats.s_DF0}; - double vt{stats.s_Vt * v0}; - double at{stats.s_At * std::sqrt(v0 / scale)}; - LOG_TRACE("M = " << M); - - TFloatMeanAccumulatorVec values(m_BucketValues); - this->conditionOnNullHypothesis(windows, stats, values); - double B{static_cast( - std::count_if(values.begin(), values.end(), - [](const TFloatMeanAccumulator &value) - { return CBasicStatistics::count(value) > 0.0; }))}; - - // The variance test. - - TMeanVarAccumulatorVec trend(period); - periodicTrend(values, window, m_BucketLength, trend); - double b{static_cast( - std::count_if(trend.begin(), trend.end(), - [](const TMeanVarAccumulator &value) - { return CBasicStatistics::count(value) > 0.0; }))}; - LOG_TRACE("populated = " << b); - - double df1{B - b}; - double v1{varianceAtPercentile(residualVariance(trend, scale), - df1, 50.0 + CONFIDENCE_INTERVAL / 2.0)}; - LOG_TRACE(" variance = " << v1); - LOG_TRACE(" varianceThreshold = " << vt); - LOG_TRACE(" significance = " << CStatisticalTests::leftTailFTest(v1 / v0, df1, df0)); - - double Rt{stats.s_Rt * CTools::truncate(1.0 - 0.5 * (vt - v1) / vt, 0.9, 1.0)}; - if (v1 < vt && CStatisticalTests::leftTailFTest(v1 / v0, df1, df0) <= MAXIMUM_SIGNIFICANCE) - { - double R{CSignal::autocorrelation(period, values)}; - R = autocorrelationAtPercentile(R, B, 50.0 - CONFIDENCE_INTERVAL / 2.0); - LOG_TRACE(" autocorrelation = " << R); - LOG_TRACE(" autocorrelationThreshold = " << Rt); - if (R > Rt) - { - stats.s_V0 = v1; - stats.s_DF0 = df1; - return true; - } - } - - // The amplitude test. - - double F1{1.0}; - if (v1 > 0.0) - { - try - { - std::size_t n{static_cast( - std::ceil(Rt * static_cast(length(windows) / period_)))}; - TMeanAccumulator level; - for (const auto &value : values) - { - if (CBasicStatistics::count(value) > 0.0) - { - level.add(CBasicStatistics::mean(value)); - } - } - TMinAmplitudeVec amplitudes(period, {n, CBasicStatistics::mean(level)}); - periodicTrend(values, window, m_BucketLength, amplitudes); - boost::math::normal normal(0.0, std::sqrt(v1)); - std::for_each(amplitudes.begin(), amplitudes.end(), - [&F1, &normal, at](CMinAmplitude &x) - { - if (x.amplitude() >= at) - { - F1 = std::min(F1, x.significance(normal)); - } - }); - } - catch (const std::exception &e) - { - LOG_ERROR("Unable to compute significance of amplitude: " << e.what()); - } - } - LOG_TRACE(" F(amplitude) = " << F1); - - if (1.0 - std::pow(1.0 - F1, b) <= MAXIMUM_SIGNIFICANCE) - { - stats.s_V0 = v1; - stats.s_DF0 = df1; - return true; - } - return false; -} - -bool CPeriodicityTest::testPartition(const TTimeTimePr2Vec &partition, - core_t::TTime period_, - double correction, - STestStats &stats) const -{ - using TDoubleTimePr = std::pair; - using TDoubleTimePrVec = std::vector; - using TMinAccumulator = CBasicStatistics::COrderStatisticsStack; - using TMeanVarAccumulatorBuffer = boost::circular_buffer; - - LOG_TRACE("Testing partition " << core::CContainerPrinter::print(partition) - << " with period " << period_); - - // Find the partition of the data such that the residual variance - // w.r.t. the period is minimized and check if there is significant - // evidence that it reduces the residual variance and repeats. - - std::size_t period{static_cast(period_ / m_BucketLength)}; - core_t::TTime repeat{length(partition)}; - double b{static_cast(period)}; - double B{stats.s_B}; - double scale{1.0 / stats.s_M}; - double v0{stats.s_V0}, df0{stats.s_DF0}; - double vt{stats.s_Vt * v0}; - LOG_TRACE("period = " << period); - LOG_TRACE("scale = " << scale); - - TFloatMeanAccumulatorVec values(m_BucketValues); - this->conditionOnNullHypothesis(TTimeTimePr2Vec{{0, m_WindowLength}}, stats, values); - - double df1{B - 2.0 * b}; - - TTimeTimePr2Vec windows[]{calculateWindows(0, m_WindowLength, repeat, partition[0]), - calculateWindows(0, m_WindowLength, repeat, partition[1])}; - LOG_TRACE("windows = " << core::CContainerPrinter::print(windows)); - - TTimeVec deltas[2]; - deltas[0].reserve((length(partition[0]) * m_WindowLength) / (period_ * repeat)); - deltas[1].reserve((length(partition[1]) * m_WindowLength) / (period_ * repeat)); - for (std::size_t i = 0u; i < 2; ++i) - { - for (const auto &window : windows[i]) - { - core_t::TTime a_{window.first}; - core_t::TTime b_{window.second}; - for (core_t::TTime t = a_ + period_; t <= b_; t += period_) - { - deltas[i].push_back(t - m_BucketLength); - } - } - } - LOG_TRACE("deltas = " << core::CContainerPrinter::print(deltas)); - - TMeanVarAccumulatorBuffer trends[] - { - TMeanVarAccumulatorBuffer(period, TMeanVarAccumulator()), - TMeanVarAccumulatorBuffer(period, TMeanVarAccumulator()) - }; - periodicTrend(values, windows[0], m_BucketLength, trends[0]); - periodicTrend(values, windows[1], m_BucketLength, trends[1]); - - TMeanAccumulator variances[] - { - residualVariance(trends[0], scale), - residualVariance(trends[1], scale) - }; - LOG_TRACE("variances = " << core::CContainerPrinter::print(variances)); - - TMinAccumulator minimum; - minimum.add({( residualVariance(variances[0]) - + residualVariance(variances[1])) / 2.0, 0}); - - TDoubleTimePrVec candidates; - candidates.reserve(period); - for (core_t::TTime time = m_BucketLength; - time < repeat; - time += m_BucketLength) - { - for (std::size_t i = 0u; i < 2; ++i) - { - for (auto &&delta : deltas[i]) - { - delta = (delta + m_BucketLength) % m_WindowLength; - } - TMeanVarAccumulator oldBucket{trends[i].front()}; - TMeanVarAccumulator newBucket; - averageValue(values, deltas[i], m_BucketLength, newBucket); - - trends[i].pop_front(); - trends[i].push_back(newBucket); - variances[i] -= residualVariance(oldBucket, scale); - variances[i] += residualVariance(newBucket, scale); - } - double variance{( residualVariance(variances[0]) - + residualVariance(variances[1])) / 2.0}; - minimum.add({variance, time}); - if (variance <= 1.05 * minimum[0].first) - { - candidates.emplace_back(variance, time); - } - } - - TMinAccumulator lowest; - for (const auto &candidate : candidates) - { - if (candidate.first <= 1.05 * minimum[0].first) - { - core_t::TTime time{candidate.second}; - std::size_t j{static_cast(time / m_BucketLength)}; - lowest.add({std::fabs(CBasicStatistics::mean(values[j])), time}); - } - } - - double v1{correction * minimum[0].first}; - LOG_TRACE(" variance = " << v1); - LOG_TRACE(" varianceThreshold = " << vt); - LOG_TRACE(" significance = " << CStatisticalTests::leftTailFTest(v1 / v0, df1, df0)); - - if (v1 <= vt && CStatisticalTests::leftTailFTest(v1 / v0, df1, df0) <= MAXIMUM_SIGNIFICANCE) - { - double R{-1.0}; - double Rt{stats.s_Rt * CTools::truncate(1.0 - 0.5 * (vt - v1) / vt, 0.9, 1.0)}; - - core_t::TTime startOfPartition{lowest[0].second}; - windows[0] = calculateWindows(startOfPartition, m_WindowLength, repeat, partition[0]); - windows[1] = calculateWindows(startOfPartition, m_WindowLength, repeat, partition[1]); - for (const auto &windows_ : windows) - { - TFloatMeanAccumulatorVec partitionValues; - project(values, windows_, m_BucketLength, partitionValues); - std::size_t windowLength(length(windows_[0]) / m_BucketLength); - double BW{std::accumulate(partitionValues.begin(), partitionValues.end(), 0.0, - [](double n, const TFloatMeanAccumulator &value) - { return n + (CBasicStatistics::count(value) > 0.0 ? 1.0 : 0.0); })}; - R = std::max(R, autocorrelationAtPercentile(CSignal::autocorrelation( - windowLength + period, partitionValues), - BW, 50.0 - CONFIDENCE_INTERVAL / 2.0)); - LOG_TRACE(" autocorrelation = " << R); - LOG_TRACE(" autocorrelationThreshold = " << Rt); - } - - if (R > Rt) - { - stats.s_V0 = v1; - stats.s_DF0 = df1; - stats.s_StartOfPartition = startOfPartition; - return true; - } - } - return false; -} - -void CPeriodicityTest::conditionOnNullHypothesis(const TTimeTimePr2Vec &windows, - const STestStats &stats, - TFloatMeanAccumulatorVec &values) const -{ - std::size_t n{values.size()}; - for (std::size_t i = 0u; i < stats.s_Partition.size(); ++i) - { - TTimeTimePr2Vec windows_(calculateWindows(stats.s_StartOfPartition, - m_WindowLength, - length(stats.s_Partition), - stats.s_Partition[i])); - TSizeSizePr2Vec indexWindows; - calculateIndexWindows(windows_, m_BucketLength, indexWindows); - - std::size_t period{stats.s_T0[i].size()}; - for (const auto &window : indexWindows) - { - std::size_t a{window.first}; - std::size_t b{window.second}; - for (std::size_t j = a; j < b; ++j) - { - CBasicStatistics::moment<0>(values[j % n]) -= stats.s_T0[i][(j - a) % period]; - } - } - } - if (length(windows) < m_WindowLength) - { - LOG_TRACE("Projecting onto " << core::CContainerPrinter::print(windows)); - TFloatMeanAccumulatorVec projection; - project(values, windows, m_BucketLength, projection); - values = std::move(projection); - LOG_TRACE("# values = " << values.size()); - } -} - -void CPeriodicityTest::periodicBucketing(core_t::TTime period_, - const TTimeTimePr2Vec &windows, - TTimeTimePrMeanVarAccumulatorPrVec &trend) const -{ - trend.clear(); - - if (windows.empty()) - { - return; - } - - period_ = std::min(period_, length(windows[0])); - std::size_t period(period_ / m_BucketLength); - - initializeBuckets(period, windows, trend); - TMeanAccumulatorVec varianceScales(trend.size()); - std::size_t n{m_BucketValues.size()}; - for (std::size_t i = 0u, j = 0u; i < windows.size(); ++i) - { - std::size_t a{static_cast(windows[i].first / m_BucketLength)}; - std::size_t b{static_cast(windows[i].second / m_BucketLength)}; - for (std::size_t k = a; k < b; ++j, ++k) - { - const TFloatMeanAccumulator &bucket{m_BucketValues[k % n]}; - double count{CBasicStatistics::count(bucket)}; - double mean{CBasicStatistics::mean(bucket)}; - if (count > 0.0) - { - std::size_t pj{j % period}; - trend[pj].second.add(mean, count); - varianceScales[pj].add(1.0 / count); - } - } - } - for (std::size_t i = 0u; i < trend.size(); ++i) - { - CBasicStatistics::moment<1>(trend[i].second) /= CBasicStatistics::mean(varianceScales[i]); - } -} - -void CPeriodicityTest::periodicBucketing(TTime2Vec periods_, - const TTimeTimePr2Vec &windows, - TTimeTimePrMeanVarAccumulatorPrVec &shortTrend, - TTimeTimePrMeanVarAccumulatorPrVec &longTrend) const -{ - // For periods P1 and P2 then, given the window is a whole number - // of P2, we have that - // x(i) = d(i mod P2, j) * m2'(j) + d(i mod P1, j) * m1'(j) - // - // where d(.) denotes the Kronecker delta, and m1' and m2' are the - // adjusted periodic baselines, respectively. This gives an over - // determined matrix equation which can be solved using the standard - // least squares approach, i.e. using the Moore-Penrose pseudo-inverse. - // There is one complication which is that the matrix is singular. - // This is because there is a degeneracy among possible solutions. - // Specifically, if we add c(j) to m1'(j) and subtract c(j) from - // m(j + k * D) we end up with the same total baseline. One strategy - // is to add eps * I and let eps -> 0 which gives a well defined linear - // combination of {x(i)} to compute the mean, i.e. for the j'th bucket - // in period P2 - // m1'(j) = (N/N1) / (N/N1 + N/N2) * m1(j) - // m2'(j) = m2(j) - (N/N1) / (N/N1 + N/N2) * m1(j mod n1) - // - // Since we have lower resolution to model m2' we prefer to subsequently - // adjust m1' to make m2' as smooth as possible. - - shortTrend.clear(); - longTrend.clear(); - - if (windows.empty()) - { - return; - } - - core_t::TTime window{length(windows)}; - - core_t::TTime w0{length(windows[0])}; - periods_[0] = std::min(periods_[0], w0); - periods_[1] = std::min(periods_[1], w0); - - std::size_t periods[2]; - periods[0] = static_cast(periods_[0] / m_BucketLength); - periods[1] = static_cast(periods_[1] / m_BucketLength); - - std::size_t length = m_BucketValues.size(); - double S{static_cast(window / periods_[0])}; - double L{static_cast(window / periods_[1])}; - double scale{S / (S + L)}; - - TMeanAccumulatorVec trends[]{TMeanAccumulatorVec(periods[0]), - TMeanAccumulatorVec(periods[1])}; - periodicTrend(m_BucketValues, windows, m_BucketLength, trends[0]); - periodicTrend(m_BucketValues, windows, m_BucketLength, trends[1]); - - for (auto &&bucket : trends[0]) - { - CBasicStatistics::moment<0>(bucket) *= scale; - } - for (std::size_t i = 0u; i < trends[1].size(); /**/) - { - for (std::size_t j = 0u; i < trends[1].size() && j < trends[0].size(); ++i, ++j) - { - TMeanAccumulator &bucket{trends[1][i]}; - if (CBasicStatistics::count(bucket) > 0.0) - { - CBasicStatistics::moment<0>(bucket) -= CBasicStatistics::mean(trends[0][j]); - } - } - } - - TMeanAccumulatorVec shifts(periods[0]); - for (std::size_t i = 0u; i < trends[1].size(); /**/) - { - for (std::size_t j = 0u; i < trends[1].size() && j < shifts.size(); ++i, ++j) - { - const TMeanAccumulator &bucket{trends[1][i]}; - if (CBasicStatistics::count(bucket) > 0.0) - { - shifts[j].add(CBasicStatistics::mean(bucket)); - } - } - } - for (std::size_t i = 0u; i < trends[0].size(); ++i) - { - double shift{CBasicStatistics::mean(shifts[i])}; - if (shift != 0.0) - { - CBasicStatistics::moment<0>(trends[0][i]) += shift; - for (std::size_t j = 0u; j < trends[1].size(); j += trends[0].size()) - { - TMeanAccumulator &bucket{trends[1][i + j]}; - if (CBasicStatistics::count(bucket) > 0.0) - { - CBasicStatistics::moment<0>(bucket) -= shift; - } - } - } - } - - this->initializeBuckets(periods[0], windows, shortTrend); - this->initializeBuckets(periods[1], windows, longTrend); - TMeanAccumulatorVec varianceScales(shortTrend.size()); - for (std::size_t i = 0u, j = 0u, k = 0u; i < windows.size(); ++i) - { - std::size_t a(windows[i].first / m_BucketLength); - std::size_t b(windows[i].second / m_BucketLength); - for (std::size_t l = a; l < b; ++j, ++k, ++l) - { - const TFloatMeanAccumulator &bucket{m_BucketValues[l % length]}; - double count{CBasicStatistics::count(bucket)}; - double mean{CBasicStatistics::mean(bucket)}; - if (count > 0.0) - { - std::size_t pj{j % periods[0]}; - std::size_t pk{k % periods[1]}; - shortTrend[pj].second.add(mean - CBasicStatistics::mean(trends[1][pk]), count); - varianceScales[pj].add(1.0 / count); - } - } - } - for (std::size_t i = 0u; i < shortTrend.size(); ++i) - { - CBasicStatistics::moment<1>(shortTrend[i].second) /= CBasicStatistics::mean(varianceScales[i]); - } - for (std::size_t i = 0u; i < trends[1].size(); ++i) - { - longTrend[i].second.add(CBasicStatistics::mean(trends[1][i])); - } -} - -void CPeriodicityTest::initializeBuckets(std::size_t period, - const TTimeTimePr2Vec &windows, - TTimeTimePrMeanVarAccumulatorPrVec &trend) const -{ - trend.resize(period); - core_t::TTime bucket = windows[0].first; - for (auto &&value : trend) - { - value.first = {bucket, bucket + m_BucketLength}; - bucket += m_BucketLength; - } -} - -const double CPeriodicityTest::ACCURATE_TEST_POPULATED_FRACTION{0.9}; -const double CPeriodicityTest::MINIMUM_COEFFICIENT_OF_VARIATION{1e-4}; - -CPeriodicityTest::STestStats::STestStats(double vt, double at, double Rt) : - s_Vt(vt), s_At(at), s_Rt(Rt), - s_Range(0.0), s_B(0.0), s_M(0.0), s_V0(0.0), s_DF0(0.0), - s_StartOfPartition(0) -{} - -//////// CDiurnalPeriodicityTest //////// - -CDiurnalPeriodicityTest::CDiurnalPeriodicityTest(double decayRate) : - CPeriodicityTest(decayRate) -{ - std::fill_n(m_VarianceCorrections, 2, 1.0); -} - -bool CDiurnalPeriodicityTest::acceptRestoreTraverser(core::CStateRestoreTraverser &traverser) -{ - bool result{this->CPeriodicityTest::acceptRestoreTraverser(traverser)}; - if (result) - { - std::ptrdiff_t i{ std::find(boost::begin(DIURNAL_PERMITTED_BUCKET_LENGTHS), - boost::end(DIURNAL_PERMITTED_BUCKET_LENGTHS), - this->bucketLength()) - - boost::begin(DIURNAL_PERMITTED_BUCKET_LENGTHS)}; - m_VarianceCorrections[0] = DIURNAL_VARIANCE_CORRECTIONS[0][i]; - m_VarianceCorrections[1] = DIURNAL_VARIANCE_CORRECTIONS[1][i]; - } - return result; -} - -void CDiurnalPeriodicityTest::acceptPersistInserter(core::CStatePersistInserter &inserter) const -{ - this->CPeriodicityTest::acceptPersistInserter(inserter); -} - -CDiurnalPeriodicityTest *CDiurnalPeriodicityTest::create(core_t::TTime bucketLength, double decayRate) -{ - std::size_t n{boost::size(DIURNAL_PERMITTED_BUCKET_LENGTHS)}; - if (bucketLength > DIURNAL_PERMITTED_BUCKET_LENGTHS[n - 1]) - { - return 0; - } - - std::ptrdiff_t index{ std::lower_bound(DIURNAL_PERMITTED_BUCKET_LENGTHS, - DIURNAL_PERMITTED_BUCKET_LENGTHS + n, - bucketLength) - - DIURNAL_PERMITTED_BUCKET_LENGTHS}; - bucketLength = DIURNAL_PERMITTED_BUCKET_LENGTHS[index]; - double corrections[]{DIURNAL_VARIANCE_CORRECTIONS[0][index], - DIURNAL_VARIANCE_CORRECTIONS[1][index]}; - - CDiurnalPeriodicityTest *result{new CDiurnalPeriodicityTest(decayRate)}; - core_t::TTime window{std::min(2 * WEEK * (bucketLength / HOUR), 8 * WEEK)}; - if (!result->initialize(bucketLength, window, corrections)) - { - delete result; - result = 0; - } - return result; -} - -CPeriodicityTestResult CDiurnalPeriodicityTest::test(void) const -{ - // We perform a series of tests of nested hypotheses about - // the periodic components and weekday/end patterns. To test - // for periodic components we compare the residual variance - // with and without trend. This must be reduced in significant - // absolute sense to make it worthwhile modelling and in a - // statistical sense. We use an F-test for this purpose. Note - // that since the buckets contain the mean of multiple samples - // we expect them to tend to Gaussian over time. We also test - // the amplitude. Again this must be significant in both an - // absolute and statistical sense. We again assume the bucket - // values are Gaussian for the purpose of the statistical test. - // Each time we accept a simpler hypothesis about the data we - // test the nested hypothesis w.r.t. this. This entails removing - // any periodic component we've already found from the data. - - if (this->bucketLength() > DAY) - { - return CPeriodicityTestResult(); - } - - STestStats stats(MAXIMUM_PERIOD_VARIANCE, - MINIMUM_PERIOD_AMPLITUDE, - MINIMUM_AUTOCORRELATION); - if (!this->initializeTestStatistics(stats) - || stats.s_Range < 2.9 * static_cast(DAY / this->bucketLength()) - || !this->nullHypothesis(stats)) - { - return CPeriodicityTestResult(); - } - - TTimeTimePr2Vec window{{0, this->windowLength()}}; - if (this->testDaily(stats)) - { - if (this->testWeekend(true, stats)) - { - return this->testWeeklyGivenDailyAndWeekend(stats); - } - if (this->testWeekly(window, stats)) - { - return diurnalResult({E_Day, E_Week}); - } - return diurnalResult({E_Day}); - } - - if (this->testWeekend(false, stats)) - { - if (this->testWeekend(true, stats)) - { - return this->testWeeklyGivenDailyAndWeekend(stats); - } - if (this->testWeekly(window, stats)) - { - return diurnalResult({E_Week}); - } - return diurnalResult({E_WeekdayWeek, E_WeekendWeek}, stats.s_StartOfPartition); - } - - if (this->testWeekend(true, stats)) - { - return this->testWeeklyGivenDailyAndWeekend(stats); - } - - if (this->testWeekly(window, stats)) - { - return diurnalResult({E_Week}); - } - - return CPeriodicityTestResult(); -} - -CDiurnalPeriodicityTest::TTime2Vec CDiurnalPeriodicityTest::periods(void) const -{ - return TTime2Vec{DAY, WEEK}; -} - -CSeasonalTime *CDiurnalPeriodicityTest::seasonalTime(const TComponent &component) const -{ - return new CDiurnalTime(component.s_StartOfPartition, - component.s_Window.first, - component.s_Window.second, - component.s_Period); -} - -void CDiurnalPeriodicityTest::trends(const CPeriodicityTestResult &required, - TTimeTimePrMeanVarAccumulatorPrVecVec &result) const -{ - auto &components = required.components(); - - result.resize(components.size()); - - std::size_t table[6]; - std::fill_n(boost::begin(table), 6, result.size()); - for (std::size_t i = 0u; i < components.size(); ++i) - { - table[components[i].s_Id] = i; - } - - for (std::size_t i = 0u; i < 6; i += 2) - { - if (table[i] < result.size() && table[i+1] < result.size()) - { - std::size_t day{table[i]}; - std::size_t week{table[i+1]}; - core_t::TTime startOfWeek{components[day].s_StartOfPartition}; - const TTimeTimePr &window{components[day].s_Window}; - this->CPeriodicityTest::periodicBucketing(this->periods(), - calculateWindows(startOfWeek, - this->windowLength(), - WEEK, window), - result[day], result[week]); - - } - else if (table[i] < result.size() || table[i+1] < result.size()) - { - std::size_t period{std::min(table[i], table[i+1])}; - core_t::TTime startOfWeek{components[period].s_StartOfPartition}; - const TTimeTimePr &window{components[period].s_Window}; - this->CPeriodicityTest::periodicBucketing(components[period].s_Period, - calculateWindows(startOfWeek, - this->windowLength(), - WEEK, window), - result[period]); - } - } -} - -std::size_t CDiurnalPeriodicityTest::staticSize(void) const -{ - return sizeof(*this); -} - -uint64_t CDiurnalPeriodicityTest::checksum(uint64_t seed) const -{ - return this->CPeriodicityTest::checksum(seed); -} - -std::string CDiurnalPeriodicityTest::print(const CPeriodicityTestResult &result) const -{ - std::string desc = "{"; - for (const auto &component : result.components()) - { - std::string partition{DIURNAL_WINDOW_NAMES[component.s_Id / 2]}; - std::string period{DIURNAL_PERIOD_NAMES[component.s_Id % 2]}; - desc += " '" + partition + (partition.empty() ? "" : " ") + period + "'"; - } - desc += " }"; - return desc; -} - -bool CDiurnalPeriodicityTest::initialize(core_t::TTime bucketLength, - core_t::TTime window, - const double (&corrections)[2]) -{ - // The following conditions need to hold: - // - The window needs to be at least two weeks, - // - The window needs to be a whole number of weeks, - // - The periods needs to be multiples of the bucket length. - if ( window < 2 * WEEK - || window % WEEK != 0 - || DAY % bucketLength != 0 - || WEEK % bucketLength != 0) - { - return false; - } - - TFloatMeanAccumulatorVec initial; - this->CPeriodicityTest::initialize(bucketLength, window, initial); - std::copy(corrections, corrections + 2, m_VarianceCorrections); - - return true; -} - -CPeriodicityTestResult -CDiurnalPeriodicityTest::testWeeklyGivenDailyAndWeekend(STestStats &stats) const -{ - core_t::TTime startOfWeek{stats.s_StartOfPartition}; - TTimeTimePr2Vec window{{0, this->windowLength()}}; - if (this->testWeekly(window, stats)) - { - return diurnalResult({E_WeekendDay, E_WeekendWeek, E_WeekdayDay, E_WeekdayWeek}, startOfWeek); - } - TTimeTimePr2Vec weekday( - calculateWindows(startOfWeek, this->windowLength(), WEEK, {WEEKEND, WEEK})); - if (this->testWeekly(weekday, stats)) - { - return diurnalResult({E_WeekendDay, E_WeekdayDay, E_WeekdayWeek}, startOfWeek); - } - TTimeTimePr2Vec weekend( - calculateWindows(startOfWeek, this->windowLength(), WEEK, {0, WEEKEND})); - if (this->testWeekly(weekend, stats)) - { - return diurnalResult({E_WeekendDay, E_WeekendWeek, E_WeekdayDay}, startOfWeek); - - } - return diurnalResult({E_WeekendDay, E_WeekdayDay}, startOfWeek); -} - -bool CDiurnalPeriodicityTest::testDaily(STestStats &stats) const -{ - std::size_t period(DAY / this->bucketLength()); - TTimeTimePr2Vec window{{0, this->windowLength()}}; - if ( this->bucketLength() <= DAY / 4 - && stats.s_Range >= 2.9 * static_cast(period) - && this->testPeriod(window, DAY, stats)) - { - TMeanAccumulatorVec trend(period); - periodicTrend(this->bucketValues(), window, this->bucketLength(), trend); - stats.s_T0 = TDoubleVec2Vec(1); - stats.s_T0[0].reserve(period); - std::for_each(trend.begin(), trend.end(), - [&stats](const TMeanAccumulator &value) - { stats.s_T0[0].push_back(CBasicStatistics::mean(value)); }); - return true; - } - return false; -} - -bool CDiurnalPeriodicityTest::testWeekend(bool daily, STestStats &stats) const -{ - core_t::TTime period_{daily ? DAY : this->bucketLength()}; - std::size_t period(period_ / this->bucketLength()); - TTimeTimePr2Vec partition{{0, WEEKEND}, {WEEKEND, WEEK}}; - if ( (!daily || this->bucketLength() <= DAY / 4) - && this->seenSufficientData() - && this->testPartition( - partition, period_, m_VarianceCorrections[daily ? 0 : 1], stats)) - { - - stats.s_Partition = partition; - stats.s_T0 = TDoubleVec2Vec(2); - for (std::size_t i = 0u; i < partition.size(); ++i) - { - TMeanAccumulatorVec trend(period); - TTimeTimePr2Vec windows(calculateWindows(stats.s_StartOfPartition, - this->windowLength(), - WEEK, partition[i])); - periodicTrend(this->bucketValues(), windows, this->bucketLength(), trend); - stats.s_T0[i].reserve(period); - std::for_each(trend.begin(), trend.end(), - [&stats, i](const TMeanAccumulator &value) - { stats.s_T0[i].push_back(CBasicStatistics::mean(value)); }); - } - return true; - } - return false; -} - -bool CDiurnalPeriodicityTest::testWeekly(const TTimeTimePr2Vec &window, - STestStats &stats) const -{ - return stats.s_Range >= ACCURATE_TEST_POPULATED_FRACTION - * static_cast(2 * WEEK / this->bucketLength()) - && this->testPeriod(window, WEEK, stats); -} - -const double CDiurnalPeriodicityTest::MAXIMUM_PARTITION_VARIANCE{0.5}; -const double CDiurnalPeriodicityTest::MAXIMUM_PERIOD_VARIANCE{0.7}; -const double CDiurnalPeriodicityTest::MINIMUM_PERIOD_AMPLITUDE{1.0}; -const double CDiurnalPeriodicityTest::MINIMUM_AUTOCORRELATION{0.5}; - -//////// CGeneralPeriodicityTest //////// - -bool CGeneralPeriodicityTest::initialize(core_t::TTime bucketLength, - core_t::TTime window, - core_t::TTime period, - const TFloatMeanAccumulatorVec &initial) -{ - // The following conditions need to hold: - // - The window needs to be at least two periods, - // - The window needs to be a whole number of periods, - // - The period needs to be multiples of the bucket length. - if ( window < 2 * period - || window % period != 0 - || period % bucketLength != 0) - { - return false; - } - - this->CPeriodicityTest::initialize(bucketLength, window, initial); - m_Period = period; - return true; -} - -CPeriodicityTestResult CGeneralPeriodicityTest::test(void) const -{ - std::size_t period = static_cast(m_Period / this->bucketLength()); - - STestStats stats(MAXIMUM_PERIOD_VARIANCE, - MINIMUM_PERIOD_AMPLITUDE, - MINIMUM_AUTOCORRELATION); - if (!this->initializeTestStatistics(stats) - || stats.s_B < 2.9 * static_cast(period) - || !this->nullHypothesis(stats)) - { - return CPeriodicityTestResult(); - } - - LOG_TRACE("Testing " << m_Period); - - CPeriodicityTestResult result; - TTimeTimePr2Vec window{{0, this->windowLength()}}; - if (this->testPeriod(window, m_Period, stats)) - { - result.add(0, 0, m_Period, {0, m_Period}); - } - return result; -} - -CGeneralPeriodicityTest::TTime2Vec CGeneralPeriodicityTest::periods(void) const -{ - return TTime2Vec{m_Period}; -} - -CSeasonalTime *CGeneralPeriodicityTest::seasonalTime(const TComponent &/*component*/) const -{ - return new CGeneralPeriodTime(m_Period); -} - -void CGeneralPeriodicityTest::trends(const CPeriodicityTestResult &required, - TTimeTimePrMeanVarAccumulatorPrVecVec &result) const -{ - if (required.periodic()) - { - result.resize(1); - TTimeTimePr2Vec window{{0, this->windowLength()}}; - this->CPeriodicityTest::periodicBucketing(m_Period, window, result[0]); - } -} - -std::size_t CGeneralPeriodicityTest::staticSize(void) const -{ - return sizeof(*this); -} - -uint64_t CGeneralPeriodicityTest::checksum(uint64_t seed) const -{ - seed = this->CPeriodicityTest::checksum(seed); - return CChecksum::calculate(seed, m_Period); -} - -std::string CGeneralPeriodicityTest::print(const CPeriodicityTestResult &result) const -{ - return result.periodic() ? - "{ " + core::CStringUtils::typeToString(result.components()[0].s_Period) + " }" : "{ }"; -} - -const TFloatMeanAccumulatorVec CGeneralPeriodicityTest::NO_BUCKET_VALUES{}; -const double CGeneralPeriodicityTest::MAXIMUM_PERIOD_VARIANCE{0.5}; -const double CGeneralPeriodicityTest::MINIMUM_PERIOD_AMPLITUDE{2.0}; -const double CGeneralPeriodicityTest::MINIMUM_AUTOCORRELATION{0.7}; - -//////// CScanningPeriodicityTest //////// - -CScanningPeriodicityTest::CScanningPeriodicityTest(TTimeCRng bucketLengths, - std::size_t size, - double decayRate) : - m_DecayRate(decayRate), - m_BucketLengths(bucketLengths), - m_BucketLengthIndex(0), - m_StartTime(boost::numeric::bounds::lowest()), - m_BucketValues(size % 2 == 0 ? size : size + 1) -{} - -bool CScanningPeriodicityTest::acceptRestoreTraverser(core::CStateRestoreTraverser &traverser) -{ - m_BucketValues.clear(); - do - { - const std::string &name = traverser.name(); - RESTORE_BUILT_IN(BUCKET_LENGTH_INDEX_TAG, m_BucketLengthIndex) - RESTORE_BUILT_IN(START_TIME_TAG, m_StartTime) - RESTORE(BUCKET_VALUE_TAG, core::CPersistUtils::restore(BUCKET_VALUE_TAG, m_BucketValues, traverser)); - } - while (traverser.next()); - return true; -} - -void CScanningPeriodicityTest::acceptPersistInserter(core::CStatePersistInserter &inserter) const -{ - inserter.insertValue(BUCKET_LENGTH_INDEX_TAG, m_BucketLengthIndex); - inserter.insertValue(START_TIME_TAG, m_StartTime); - core::CPersistUtils::persist(BUCKET_VALUE_TAG, m_BucketValues, inserter); -} - -void CScanningPeriodicityTest::initialize(core_t::TTime time) -{ - m_StartTime = time; -} - -void CScanningPeriodicityTest::propagateForwardsByTime(double time) -{ - if (!CMathsFuncs::isFinite(time) || time < 0.0) - { - LOG_ERROR("Bad propagation time " << time); - } - double factor = std::exp(-m_DecayRate * time); - for (auto &&value : m_BucketValues) - { - value.age(factor); - } -} - -void CScanningPeriodicityTest::add(core_t::TTime time, double value, double weight) -{ - if (time >= m_StartTime) - { - while (this->needToCompress(time)) - { - m_BucketLengthIndex = (m_BucketLengthIndex + 1) % m_BucketLengths.size(); - auto end = m_BucketValues.begin(); - - if (m_BucketLengthIndex == 0) - { - m_StartTime = CIntegerTools::floor(time, m_BucketLengths[0]); - } - else - { - std::size_t compression = m_BucketLengths[m_BucketLengthIndex] - / m_BucketLengths[m_BucketLengthIndex - 1]; - for (std::size_t i = 0u; i < m_BucketValues.size(); i += compression, ++end) - { - std::swap(*end, m_BucketValues[i]); - for (std::size_t j = 1u; j < compression && i + j < m_BucketValues.size(); ++j) - { - *end += m_BucketValues[i + j]; - } - } - } - std::fill(end, m_BucketValues.end(), TFloatMeanAccumulator()); - } - - m_BucketValues[(time - m_StartTime) / m_BucketLengths[m_BucketLengthIndex]].add(value, weight); - } -} - -bool CScanningPeriodicityTest::needToCompress(core_t::TTime time) const -{ - return time >= m_StartTime + static_cast(m_BucketValues.size()) - * m_BucketLengths[m_BucketLengthIndex]; -} - -CScanningPeriodicityTest::TPeriodicityResultPr CScanningPeriodicityTest::test(void) const -{ - using TSizeVec = std::vector; - using TDoubleSizePr = std::pair; - using TMaxAccumulator = CBasicStatistics::COrderStatisticsHeap>; - - // Compute the serial autocorrelations padding to the maximum offset - // to avoid windowing effects. - - core_t::TTime bucketLength = m_BucketLengths[m_BucketLengthIndex]; - - LOG_TRACE("Testing with bucket length " << bucketLength); - - std::size_t n = m_BucketValues.size(); - std::size_t pad = n / 3; - - TFloatMeanAccumulatorVec values(m_BucketValues); - - TDoubleVec correlations; - values.resize(n + pad); - CSignal::autocorrelations(values, correlations); - values.resize(n); - - // We retain the top 15 serial autocorrelations so we have a high - // chance of finding the highest cyclic autocorrelation. Note, we - // average over offsets which are integer multiples of the period - // since these should have high autocorrelation if the signal is - // periodic. - - TMaxAccumulator candidates(15); - correlations.resize(pad); - for (std::size_t p = 4u; p < correlations.size(); ++p) - { - double correlation = this->meanForPeriodicOffsets(correlations, p); - LOG_TRACE("correlation(" << p << ") = " << correlation); - candidates.add({correlation, p}); - } - - TSizeVec candidatePeriods(15); - std::transform(candidates.begin(), candidates.end(), - candidatePeriods.begin(), - [](const TDoubleSizePr &candidate_) { return candidate_.second; }); - candidates.clear(); - for (auto period : candidatePeriods) - { - this->resize(n - n % period, values); - candidates.add({CSignal::autocorrelation(period, values), period}); - } - candidates.sort(); - LOG_TRACE("candidate periods = " << candidates.print()); - - std::size_t period_ = candidates[0].second; - double cutoff = 0.9 * candidates[0].first; - for (auto i = candidates.begin() + 1; i != candidates.end() && i->first > cutoff; ++i) - { - if (i->second < period_ && candidates[0].second % i->second == 0) - { - period_ = i->second; - } - } - - // Configure the full periodicity test. - - std::size_t window_ = n - n % period_; - core_t::TTime window = static_cast(window_) * bucketLength; - core_t::TTime period = static_cast(period_) * bucketLength; - this->resize(window_, values); - - // We define times relative to an integer multiple of the period so need - // to shift values to account for the offset of m_StartTime relative to - // this pattern. - std::size_t offset = static_cast( - (m_StartTime - CIntegerTools::floor(m_StartTime, period)) / bucketLength); - cyclicShift(offset, values); - - LOG_TRACE("bucket length = " << bucketLength - << ", window = " << window - << ", periods to test = " << period - << ", # values = " << values.size()); - - CGeneralPeriodicityTest test; - test.initialize(bucketLength, window, period, values); - - return {test, test.test()}; -} - -void CScanningPeriodicityTest::skipTime(core_t::TTime skipInterval) -{ - m_StartTime += skipInterval; -} - -uint64_t CScanningPeriodicityTest::checksum(uint64_t seed) const -{ - seed = CChecksum::calculate(seed, m_BucketLengthIndex); - seed = CChecksum::calculate(seed, m_StartTime); - return CChecksum::calculate(seed, m_BucketValues); -} - -void CScanningPeriodicityTest::debugMemoryUsage(core::CMemoryUsage::TMemoryUsagePtr mem) const -{ - mem->setName("CScanningPeriodicityTest"); - core::CMemoryDebug::dynamicSize("m_BucketValues", m_BucketValues, mem); -} - -std::size_t CScanningPeriodicityTest::memoryUsage(void) const -{ - return core::CMemory::dynamicSize(m_BucketValues); -} - -void CScanningPeriodicityTest::resize(std::size_t size, TFloatMeanAccumulatorVec &values) const -{ - std::size_t n = values.size(); - values.resize(size); - for (std::size_t i = n; i < size; ++i) - { - values[i] = m_BucketValues[i]; - } -} - -double CScanningPeriodicityTest::meanForPeriodicOffsets(const TDoubleVec &correlations, - std::size_t period) const -{ - TMeanAccumulator result; - for (std::size_t offset = period; offset < correlations.size(); offset += period) - { - result.add(this->correctForPad(correlations[offset - 1], offset)); - } - return CBasicStatistics::mean(result); -} - -double CScanningPeriodicityTest::correctForPad(double correlation, std::size_t offset) const -{ - return correlation * static_cast(m_BucketValues.size()) - / static_cast(m_BucketValues.size() - offset); -} - //////// CCalendarCyclicTest //////// CCalendarCyclicTest::CCalendarCyclicTest(double decayRate) : diff --git a/lib/maths/CXMeansOnline1d.cc b/lib/maths/CXMeansOnline1d.cc index 56bcfa8337..881d7c0011 100644 --- a/lib/maths/CXMeansOnline1d.cc +++ b/lib/maths/CXMeansOnline1d.cc @@ -92,7 +92,7 @@ struct SClusterCentreLess }; //! Get \p x time \p x. -double square(double x) +double pow2(double x) { return x * x; } @@ -112,16 +112,11 @@ logLikelihoodFromCluster(double point, { result = core::constants::LOG_MIN_DOUBLE - 1.0; - static const maths_t::TWeightStyleVec COUNT_WEIGHT(1, maths_t::E_SampleCountWeight); - static const TDouble4Vec1Vec UNIT_WEIGHT(1, TDouble4Vec(1, 1.0)); - double likelihood; maths_t::EFloatingPointErrorStatus status = - normal.jointLogMarginalLikelihood(COUNT_WEIGHT, - TDouble1Vec(1, point), - UNIT_WEIGHT, - likelihood); + normal.jointLogMarginalLikelihood(CConstantWeights::COUNT, {point}, + CConstantWeights::SINGLE_UNIT, likelihood); if (status & maths_t::E_FpFailed) { LOG_ERROR("Unable to compute likelihood for: " << point); @@ -267,12 +262,15 @@ void BICGain(maths_t::EDataType dataType, TMeanVarAccumulator mvl; TMeanVarAccumulator mvr; candidates(categories, start, split, end, mv, mvl, mvr); - double offset = std::max(0.0, 0.2 - smallest); + double logNormalOffset = std::max(0.0, GAMMA_OFFSET_MARGIN - smallest); + double gammaOffset = std::max(0.0, LOG_NORMAL_OFFSET_MARGIN - smallest); for (std::size_t i = start; i < end; ++i) { - offset = std::max(offset, 0.2 - mean(dataType, categories[i])); + double x = mean(dataType, categories[i]); + logNormalOffset = std::max(logNormalOffset, LOG_NORMAL_OFFSET_MARGIN - x); + gammaOffset = std::max(gammaOffset, GAMMA_OFFSET_MARGIN - x); } - LOG_TRACE("offset = " << offset); + LOG_TRACE("offsets = [" << gammaOffset << "," << logNormalOffset << "]"); distance = 0.0; nl = CBasicStatistics::count(mvl); @@ -280,9 +278,9 @@ void BICGain(maths_t::EDataType dataType, // Compute the BIC gain for splitting the mode. - double ll1n = 0.0; - double ll1l = 0.0; - double ll1g = 0.0; + double ll1n = 0.0; + double ll1l = 0.0; + double ll1g = 0.0; double ll2nl = 0.0; double ll2ll = 0.0; double ll2gl = 0.0; @@ -300,11 +298,11 @@ void BICGain(maths_t::EDataType dataType, } // Log-normal (method of moments) - double s = ::log(1.0 + v / square(m + offset)); - double l = ::log(m + offset) - s / 2.0; + double s = ::log(1.0 + v / pow2(m + logNormalOffset)); + double l = ::log(m + logNormalOffset) - s / 2.0; // Gamma (method of moments) - double a = square(m + offset) / v; - double b = (m + offset) / v; + double a = pow2(m + gammaOffset) / v; + double b = (m + gammaOffset) / v; double vmin = std::min(MIN_RELATIVE_VARIANCE * v, MIN_ABSOLUTE_VARIANCE); @@ -319,23 +317,23 @@ void BICGain(maths_t::EDataType dataType, try { // Mixture of log-normals (method of moments) - double sl = ::log(1.0 + vl / square(ml + offset)); - double ll = ::log(ml + offset) - sl / 2.0; - double sr = ::log(1.0 + vr / square(mr + offset)); - double lr = ::log(mr + offset) - sr / 2.0; + double sl = ::log(1.0 + vl / pow2(ml + logNormalOffset)); + double ll = ::log(ml + logNormalOffset) - sl / 2.0; + double sr = ::log(1.0 + vr / pow2(mr + logNormalOffset)); + double lr = ::log(mr + logNormalOffset) - sr / 2.0; // Mixture of gammas (method of moments) - double al = square(ml + offset) / vl; - double bl = (ml + offset) / vl; - double ar = square(mr + offset) / vr; - double br = (mr + offset) / vr; + double al = pow2(ml + gammaOffset) / vl; + double bl = (ml + gammaOffset) / vl; + double ar = pow2(mr + gammaOffset) / vr; + double br = (mr + gammaOffset) / vr; double log2piv = ::log(boost::math::double_constants::two_pi * v); double log2pis = ::log(boost::math::double_constants::two_pi * s); double loggn = boost::math::lgamma(a) - a * ::log(b); - double log2pivl = ::log(boost::math::double_constants::two_pi * vl / square(wl)); - double log2pivr = ::log(boost::math::double_constants::two_pi * vr / square(wr)); - double log2pisl = ::log(boost::math::double_constants::two_pi * sl / square(wl)); - double log2pisr = ::log(boost::math::double_constants::two_pi * sr / square(wr)); + double log2pivl = ::log(boost::math::double_constants::two_pi * vl / pow2(wl)); + double log2pivr = ::log(boost::math::double_constants::two_pi * vr / pow2(wr)); + double log2pisl = ::log(boost::math::double_constants::two_pi * sl / pow2(wl)); + double log2pisr = ::log(boost::math::double_constants::two_pi * sr / pow2(wr)); double loggnl = boost::math::lgamma(al) - al * ::log(bl) - ::log(wl); double loggnr = boost::math::lgamma(ar) - ar * ::log(br) - ::log(wr); @@ -347,24 +345,24 @@ void BICGain(maths_t::EDataType dataType, if (vi == 0.0) { - double li = ::log(mi + offset); - ll1n += ni * ((vi + square(mi - m)) / v + log2piv); - ll1l += ni * (square(li - l) / s + 2.0 * li + log2pis); - ll1g += ni * 2.0 * (b * (mi + offset) - (a - 1.0) * li + loggn); - ll2nl += ni * ((vi + square(mi - ml)) / vl + log2pivl); - ll2ll += ni * (square(li - ll) / sl + 2.0 * li + log2pisl); - ll2gl += ni * 2.0 * (bl * (mi + offset) - (al - 1.0) * li + loggnl); + double li = ::log(mi + logNormalOffset); + ll1n += ni * ((vi + pow2(mi - m)) / v + log2piv); + ll1l += ni * (pow2(li - l) / s + 2.0 * li + log2pis); + ll1g += ni * 2.0 * (b * (mi + gammaOffset) - (a - 1.0) * li + loggn); + ll2nl += ni * ((vi + pow2(mi - ml)) / vl + log2pivl); + ll2ll += ni * (pow2(li - ll) / sl + 2.0 * li + log2pisl); + ll2gl += ni * 2.0 * (bl * (mi + gammaOffset) - (al - 1.0) * li + loggnl); } else { - double si = ::log(1.0 + vi / square(mi + offset)); - double li = ::log(mi + offset) - si / 2.0; - ll1n += ni * ((vi + square(mi - m)) / v + log2piv); - ll1l += ni * ((si + square(li - l)) / s + 2.0 * li + log2pis); - ll1g += ni * 2.0 * (b * (mi + offset) - (a - 1.0) * li + loggn); - ll2nl += ni * ((vi + square(mi - ml)) / vl + log2pivl); - ll2ll += ni * ((si + square(li - ll)) / sl + 2.0 * li + log2pisl); - ll2gl += ni * 2.0 * (bl * (mi + offset) - (al - 1.0) * li + loggnl); + double si = ::log(1.0 + vi / pow2(mi + logNormalOffset)); + double li = ::log(mi + logNormalOffset) - si / 2.0; + ll1n += ni * ((vi + pow2(mi - m)) / v + log2piv); + ll1l += ni * ((si + pow2(li - l)) / s + 2.0 * li + log2pis); + ll1g += ni * 2.0 * (b * (mi + gammaOffset) - (a - 1.0) * li + loggn); + ll2nl += ni * ((vi + pow2(mi - ml)) / vl + log2pivl); + ll2ll += ni * ((si + pow2(li - ll)) / sl + 2.0 * li + log2pisl); + ll2gl += ni * 2.0 * (bl * (mi + gammaOffset) - (al - 1.0) * li + loggnl); } } @@ -376,24 +374,24 @@ void BICGain(maths_t::EDataType dataType, if (vi == 0.0) { - double li = ::log(mi + offset); - ll1n += ni * ((vi + square(mi - m)) / v + log2piv); - ll1l += ni * (square(li - l) / s + 2.0 * li + log2pis); - ll1g += ni * 2.0 * (b * (mi + offset) - (a - 1.0) * li + loggn); - ll2nr += ni * ((vi + square(mi - mr)) / vr + log2pivr); - ll2lr += ni * (square(li - lr) / sr + 2.0 * li + log2pisr); - ll2gr += ni * 2.0 * (br * (mi + offset) - (ar - 1.0) * li + loggnr); + double li = ::log(mi + logNormalOffset); + ll1n += ni * ((vi + pow2(mi - m)) / v + log2piv); + ll1l += ni * (pow2(li - l) / s + 2.0 * li + log2pis); + ll1g += ni * 2.0 * (b * (mi + gammaOffset) - (a - 1.0) * li + loggn); + ll2nr += ni * ((vi + pow2(mi - mr)) / vr + log2pivr); + ll2lr += ni * (pow2(li - lr) / sr + 2.0 * li + log2pisr); + ll2gr += ni * 2.0 * (br * (mi + gammaOffset) - (ar - 1.0) * li + loggnr); } else { - double si = ::log(1.0 + vi / square(mi + offset)); - double li = ::log(mi + offset) - si / 2.0; - ll1n += ni * ((vi + square(mi - m)) / v + log2piv); - ll1l += ni * ((si + square(li - l)) / s + 2.0 * li + log2pis); - ll1g += ni * 2.0 * (b * (mi + offset) - (a - 1.0) * li + loggn); - ll2nr += ni * ((vi + square(mi - mr)) / vr + log2pivr); - ll2lr += ni * ((si + square(li - lr)) / sr + 2.0 * li + log2pisr); - ll2gr += ni * 2.0 * (br * (mi + offset) - (ar - 1.0) * li + loggnr); + double si = ::log(1.0 + vi / pow2(mi + logNormalOffset)); + double li = ::log(mi + logNormalOffset) - si / 2.0; + ll1n += ni * ((vi + pow2(mi - m)) / v + log2piv); + ll1l += ni * ((si + pow2(li - l)) / s + 2.0 * li + log2pis); + ll1g += ni * 2.0 * (b * (mi + gammaOffset) - (a - 1.0) * li + loggn); + ll2nr += ni * ((vi + pow2(mi - mr)) / vr + log2pivr); + ll2lr += ni * ((si + pow2(li - lr)) / sr + 2.0 * li + log2pisr); + ll2gr += ni * 2.0 * (br * (mi + gammaOffset) - (ar - 1.0) * li + loggnr); } } } diff --git a/lib/maths/Makefile b/lib/maths/Makefile index 580f429a0b..fad5cde0ae 100644 --- a/lib/maths/Makefile +++ b/lib/maths/Makefile @@ -41,6 +41,7 @@ CCountMinSketch.cc \ CDecayRateController.cc \ CDecompositionComponent.cc \ CEntropySketch.cc \ +CExpandingWindow.cc \ CGammaRateConjugate.cc \ CGradientDescent.cc \ CGramSchmidt.cc \ @@ -75,6 +76,7 @@ Constants.cc \ COrderings.cc \ COrdinal.cc \ CPackedBitVector.cc \ +CPeriodicityHypothesisTests.cc \ CPoissonMeanConjugate.cc \ CPrior.cc \ CPriorStateSerialiser.cc \ @@ -94,11 +96,11 @@ CSpline.cc \ CStatisticalTests.cc \ CTimeSeriesDecomposition.cc \ CTimeSeriesDecompositionDetail.cc \ -CTimeSeriesDecompositionInterface.cc \ CTimeSeriesDecompositionStateSerialiser.cc \ CTimeSeriesDecompositionStub.cc \ CTimeSeriesModel.cc \ CTools.cc \ +CTrendComponent.cc \ CTrendTests.cc \ CXMeansOnline1d.cc \ CXMeansOnlineFactory.cc \ diff --git a/lib/maths/unittest/CForecastTest.cc b/lib/maths/unittest/CForecastTest.cc index 19e2f7407b..18571e6610 100644 --- a/lib/maths/unittest/CForecastTest.cc +++ b/lib/maths/unittest/CForecastTest.cc @@ -56,7 +56,6 @@ using TErrorBarVec = std::vector; using TMeanAccumulator = maths::CBasicStatistics::SSampleMean::TAccumulator; using TModelPtr = boost::shared_ptr; -const double MINIMUM_SEASONAL_SCALE{0.25}; const double DECAY_RATE{0.0005}; const std::size_t TAG{0u}; const TDouble2Vec MINIMUM_VALUE{boost::numeric::bounds::lowest()}; @@ -67,7 +66,7 @@ maths::CModelParams params(core_t::TTime bucketLength) using TTimeDoubleMap = std::map; static TTimeDoubleMap learnRates; learnRates[bucketLength] = static_cast(bucketLength) / 1800.0; - double minimumSeasonalVarianceScale{MINIMUM_SEASONAL_SCALE}; + double minimumSeasonalVarianceScale{0.25}; return maths::CModelParams{bucketLength, learnRates[bucketLength], DECAY_RATE, minimumSeasonalVarianceScale}; } @@ -99,195 +98,36 @@ void CForecastTest::testDailyNoLongTermTrend(void) test::CRandomNumbers rng; - maths::CTimeSeriesDecomposition trend(0.012, 3600, 24); - maths::CNormalMeanPrecConjugate prior = - maths::CNormalMeanPrecConjugate::nonInformativePrior(maths_t::E_ContinuousData, 0.0005); - maths::CUnivariateTimeSeriesModel::TDecayRateController2Ary controllers{decayRateControllers()}; - maths::CUnivariateTimeSeriesModel model(params(bucketLength), 0, trend, prior, &controllers); - - //std::ofstream file; - //file.open("results.m"); - //TDoubleVec actual; - //TDoubleVec ly; - //TDoubleVec my; - //TDoubleVec uy; - - LOG_DEBUG("*** learn ***"); - - core_t::TTime time{0}; - TDouble2Vec4VecVec weights{TDouble2Vec4Vec{TDouble2Vec{1.0}}}; - for (std::size_t d = 0u; d < 60; ++d) - { - TDoubleVec noise; - rng.generateNormalSamples(40.0, 64.0, 6 * y.size(), noise); - for (std::size_t i = 0u; i < 6 * y.size(); ++i, time += bucketLength) + auto trend = [&y, bucketLength](core_t::TTime time, double noise) { - maths::CModelAddSamplesParams params; - params.integer(false) - .propagationInterval(1.0) - .weightStyles(maths::CConstantWeights::COUNT) - .trendWeights(weights) - .priorWeights(weights); + core_t::TTime i{(time % 86400) / bucketLength}; double alpha{static_cast(i % 6) / 6.0}; double beta{1.0 - alpha}; - double yi{alpha * y[i/6] + beta * y[(i/6 + 1) % y.size()] + noise[i]}; - model.addSamples(params, TTimeDouble2VecSizeTrVec{ - core::make_triple(time, TDouble2Vec{yi}, TAG)}); - //actual.push_back(yi); - } - } - - LOG_DEBUG("*** forecast ***"); - - TErrorBarVec prediction; - core_t::TTime start{time}; - core_t::TTime end{time + 2 * core::constants::WEEK}; - TModelPtr forecastModel(model.cloneForForecast()); - std::string m; - forecastModel->forecast(start, end, 80.0, - MINIMUM_VALUE, MAXIMUM_VALUE, - boost::bind(&mockSink, _1, boost::ref(prediction)), - m); + return 40.0 + alpha * y[i/6] + beta * y[(i/6 + 1) % y.size()] + noise; + }; - std::size_t outOfBounds{0}; - std::size_t count{0}; - TMeanAccumulator error; - - for (std::size_t i = 0u; i < prediction.size(); /**/) - { - TDoubleVec noise; - rng.generateNormalSamples(40.0, 64.0, 6 * y.size(), noise); - TDoubleVec day; - for (std::size_t j = 0u; - i < prediction.size() && j < 6 * y.size(); - ++i, ++j, time += bucketLength) - { - double alpha{static_cast(i % 6) / 6.0}; - double beta{1.0 - alpha}; - double yj{alpha * y[j/6] + beta * y[(j/6 + 1) % y.size()] + noise[j]}; - day.push_back(yj); - outOfBounds += ( yj < prediction[i].s_LowerBound - || yj > prediction[i].s_UpperBound ? 1 : 0); - ++count; - error.add(std::fabs(yj - prediction[i].s_Predicted) / std::fabs(yj)); - //actual.push_back(yj); - //ly.push_back(prediction[i].s_LowerBound); - //my.push_back(prediction[i].s_Predicted); - //uy.push_back(prediction[i].s_UpperBound); - } - } - - double percentageOutOfBounds{100.0 * static_cast(outOfBounds) / static_cast(count)}; - LOG_DEBUG("% out of bounds = " << percentageOutOfBounds); - LOG_DEBUG("error = " << maths::CBasicStatistics::mean(error)); - - //file << "actual = " << core::CContainerPrinter::print(actual) << ";\n"; - //file << "ly = " << core::CContainerPrinter::print(ly) << ";\n"; - //file << "my = " << core::CContainerPrinter::print(my) << ";\n"; - //file << "uy = " << core::CContainerPrinter::print(uy) << ";\n"; - - CPPUNIT_ASSERT(percentageOutOfBounds < 10.0); - CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.15); + this->test(trend, bucketLength, 60, 64.0, 4.0, 0.13); } void CForecastTest::testDailyConstantLongTermTrend(void) { - LOG_DEBUG("+-----------------------------------------------------+"); - LOG_DEBUG("| CForecastTest::testDailyPlusConstantLongTermTrend |"); - LOG_DEBUG("+-----------------------------------------------------+"); + LOG_DEBUG("+-------------------------------------------------+"); + LOG_DEBUG("| CForecastTest::testDailyConstantLongTermTrend |"); + LOG_DEBUG("+-------------------------------------------------+"); core_t::TTime bucketLength{3600}; TDoubleVec y{ 0.0, 2.0, 2.0, 4.0, 8.0, 10.0, 15.0, 20.0, 80.0, 100.0, 110.0, 120.0, 110.0, 100.0, 90.0, 80.0, 30.0, 15.0, 10.0, 8.0, 5.0, 3.0, 2.0, 0.0}; - test::CRandomNumbers rng; - - maths::CTimeSeriesDecomposition trend(0.012, 3600, 24); - maths::CNormalMeanPrecConjugate prior = - maths::CNormalMeanPrecConjugate::nonInformativePrior(maths_t::E_ContinuousData, 0.0005); - maths::CUnivariateTimeSeriesModel::TDecayRateController2Ary controllers{decayRateControllers()}; - maths::CUnivariateTimeSeriesModel model(params(bucketLength), 0, trend, prior, &controllers); - - //std::ofstream file; - //file.open("results.m"); - //TDoubleVec actual; - //TDoubleVec ly; - //TDoubleVec my; - //TDoubleVec uy; - - LOG_DEBUG("*** learn ***"); - - core_t::TTime time{0}; - double dy{0.0}; - TDouble2Vec4VecVec weights{TDouble2Vec4Vec{TDouble2Vec{1.0}}}; - for (std::size_t d = 0u; d < 60; ++d) - { - TDoubleVec noise; - rng.generateNormalSamples(0.0, 64.0, y.size(), noise); - - for (std::size_t i = 0u; i < y.size(); ++i, time += bucketLength, dy += 0.25) + auto trend = [&y, bucketLength](core_t::TTime time, double noise) { - maths::CModelAddSamplesParams params; - params.integer(false) - .propagationInterval(1.0) - .weightStyles(maths::CConstantWeights::COUNT) - .trendWeights(weights) - .priorWeights(weights); - double yi{dy + y[i] + noise[i]}; - model.addSamples(params, TTimeDouble2VecSizeTrVec{ - core::make_triple(time, TDouble2Vec{yi}, TAG)}); - //actual.push_back(yi); - } - } + core_t::TTime i{(time % 86400) / bucketLength}; + return 0.25 * static_cast(time) + / static_cast(bucketLength) + y[i] + noise; + }; - LOG_DEBUG("*** forecast ***"); - - TErrorBarVec prediction; - core_t::TTime start{time}; - core_t::TTime end{time + 2 * core::constants::WEEK}; - std::string m; - TModelPtr forecastModel(model.cloneForForecast()); - forecastModel->forecast(start, end, 80.0, - MINIMUM_VALUE, MAXIMUM_VALUE, - boost::bind(&mockSink, _1, boost::ref(prediction)), - m); - - std::size_t outOfBounds{0}; - std::size_t count{0}; - TMeanAccumulator error; - - for (std::size_t i = 0u; i < prediction.size(); /**/) - { - TDoubleVec noise; - rng.generateNormalSamples(0.0, 64.0, y.size(), noise); - for (std::size_t j = 0u; - i < prediction.size() && j < y.size(); - ++i, ++j, time += bucketLength, dy += 0.25) - { - double yj{dy + y[j] + noise[j]}; - outOfBounds += ( yj < prediction[i].s_LowerBound - || yj > prediction[i].s_UpperBound ? 1 : 0); - ++count; - error.add(std::fabs(yj - prediction[i].s_Predicted) / std::fabs(yj)); - //actual.push_back(yj); - //ly.push_back(prediction[i].s_LowerBound); - //my.push_back(prediction[i].s_Predicted); - //uy.push_back(prediction[i].s_UpperBound); - } - } - - double percentageOutOfBounds{100.0 * static_cast(outOfBounds) / static_cast(count)}; - LOG_DEBUG("% out of bounds = " << percentageOutOfBounds); - LOG_DEBUG("error = " << maths::CBasicStatistics::mean(error)); - - //file << "actual = " << core::CContainerPrinter::print(actual) << ";\n"; - //file << "ly = " << core::CContainerPrinter::print(ly) << ";\n"; - //file << "my = " << core::CContainerPrinter::print(my) << ";\n"; - //file << "uy = " << core::CContainerPrinter::print(uy) << ";\n"; - - CPPUNIT_ASSERT(percentageOutOfBounds < 5.0); - CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.02); + this->test(trend, bucketLength, 60, 64.0, 4.0, 0.02); } void CForecastTest::testDailyVaryingLongTermTrend(void) @@ -308,117 +148,15 @@ void CForecastTest::testDailyVaryingLongTermTrend(void) maths::CSpline<> trend_(maths::CSplineTypes::E_Cubic); trend_.interpolate(times, values, maths::CSplineTypes::E_Natural); - test::CRandomNumbers rng; - - maths::CTimeSeriesDecomposition trend(0.012, 3600, 24); - maths::CNormalMeanPrecConjugate prior = - maths::CNormalMeanPrecConjugate::nonInformativePrior(maths_t::E_ContinuousData, 0.0005); - maths::CUnivariateTimeSeriesModel::TDecayRateController2Ary controllers{decayRateControllers()}; - maths::CUnivariateTimeSeriesModel model(params(bucketLength), 0, trend, prior, &controllers); - - //std::ofstream file; - //file.open("results.m"); - //TDoubleVec actual; - //TDoubleVec ly; - //TDoubleVec my; - //TDoubleVec uy; - - LOG_DEBUG("*** learn ***"); - - core_t::TTime time{0}; - TDouble2Vec4VecVec weights{TDouble2Vec4Vec{TDouble2Vec{1.0}}}; - for (std::size_t d = 0u; d < 104; ++d) - { - TDoubleVec noise; - rng.generateNormalSamples(0.0, 9.0, 24, noise); - - for (std::size_t i = 0u; i < 24; ++i, time += bucketLength) + auto trend = [&trend_](core_t::TTime time, double noise) { - maths::CModelAddSamplesParams params; - params.integer(false) - .propagationInterval(1.0) - .weightStyles(maths::CConstantWeights::COUNT) - .trendWeights(weights) - .priorWeights(weights); - double t{static_cast(time)}; - double yi{ trend_.value(t) - + 8.0 * std::sin(boost::math::double_constants::two_pi * t / 43200.0) - + noise[i]}; - model.addSamples(params, TTimeDouble2VecSizeTrVec{ - core::make_triple(time, TDouble2Vec{yi}, TAG)}); - //actual.push_back(yi); - } - } + double time_{static_cast(time)}; + return trend_.value(time_) + + 8.0 * std::sin(boost::math::double_constants::two_pi * time_ / 43200.0) + + noise; + }; - LOG_DEBUG("*** forecast ***"); - - TErrorBarVec prediction; - { - core_t::TTime start{time}; - core_t::TTime end{time + 10 * core::constants::DAY}; - std::string m; - TModelPtr forecastModel(model.cloneForForecast()); - forecastModel->forecast(start, end, 80.0, - MINIMUM_VALUE, MAXIMUM_VALUE, - boost::bind(&mockSink, _1, boost::ref(prediction)), - m); - } - - std::size_t outOfBounds{0}; - std::size_t count{0}; - TMeanAccumulator error; - - for (std::size_t i = 0u; i < prediction.size(); /**/) - { - TDoubleVec noise; - rng.generateNormalSamples(0.0, 9.0, 24, noise); - for (std::size_t j = 0u; - i < prediction.size() && j < 24; - ++i, ++j, time += bucketLength) - { - double t{static_cast(time)}; - double yj{ trend_.value(t) - + 8.0 * std::sin(boost::math::double_constants::two_pi * t / 43200.0) - + noise[j]}; - outOfBounds += ( yj < prediction[i].s_LowerBound - || yj > prediction[i].s_UpperBound ? 1 : 0); - ++count; - error.add(std::fabs(yj - prediction[i].s_Predicted) / std::fabs(yj)); - //actual.push_back(yj); - //ly.push_back(prediction[i].s_LowerBound); - //my.push_back(prediction[i].s_Predicted); - //uy.push_back(prediction[i].s_UpperBound); - } - } - - double percentageOutOfBounds{100.0 * static_cast(outOfBounds) / static_cast(count)}; - LOG_DEBUG("% out of bounds = " << percentageOutOfBounds); - LOG_DEBUG("error = " << maths::CBasicStatistics::mean(error)); - - //file << "actual = " << core::CContainerPrinter::print(actual) << ";\n"; - //file << "ly = " << core::CContainerPrinter::print(ly) << ";\n"; - //file << "my = " << core::CContainerPrinter::print(my) << ";\n"; - //file << "uy = " << core::CContainerPrinter::print(uy) << ";\n"; - - CPPUNIT_ASSERT(percentageOutOfBounds < 26.0); - CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.06); - - prediction.clear(); - { - core_t::TTime start{time}; - core_t::TTime end{time + 50 * core::constants::DAY}; - std::string m; - TModelPtr forecastModel(model.cloneForForecast()); - forecastModel->forecast(start, end, 80.0, - MINIMUM_VALUE, MAXIMUM_VALUE, - boost::bind(&mockSink, _1, boost::ref(prediction)), - m); - LOG_DEBUG(m); - LOG_DEBUG("horizon = " << prediction.size() << " hrs"); - CPPUNIT_ASSERT(prediction.size() < static_cast( - 50 * core::constants::DAY / bucketLength)); - CPPUNIT_ASSERT(!m.empty()); - } + this->test(trend, bucketLength, 100, 9.0, 11.0, 0.04); } void CForecastTest::testComplexNoLongTermTrend(void) @@ -433,92 +171,14 @@ void CForecastTest::testComplexNoLongTermTrend(void) 60.0, 40.0, 30.0, 20.0, 10.0, 10.0, 5.0, 0.0}; TDoubleVec scale{1.0, 1.1, 1.05, 0.95, 0.9, 0.3, 0.2}; - test::CRandomNumbers rng; - - maths::CTimeSeriesDecomposition trend(0.012, 3600, 24); - maths::CNormalMeanPrecConjugate prior = - maths::CNormalMeanPrecConjugate::nonInformativePrior(maths_t::E_ContinuousData); - maths::CUnivariateTimeSeriesModel::TDecayRateController2Ary controllers{decayRateControllers()}; - maths::CUnivariateTimeSeriesModel model(params(bucketLength), 0, trend, prior, &controllers); - - //std::ofstream file; - //file.open("results.m"); - //TDoubleVec actual; - //TDoubleVec ly; - //TDoubleVec my; - //TDoubleVec uy; - - LOG_DEBUG("*** learn ***"); - - core_t::TTime time{0}; - TDouble2Vec4VecVec weights{TDouble2Vec4Vec{TDouble2Vec{1.0}}}; - for (std::size_t d = 0u; d < 60; ++d) - { - TDoubleVec noise; - rng.generateNormalSamples(0.0, 24.0, y.size(), noise); - - for (std::size_t i = 0u; i < y.size(); ++i, time += bucketLength) - { - maths::CModelAddSamplesParams params; - params.integer(false) - .propagationInterval(1.0) - .weightStyles(maths::CConstantWeights::COUNT) - .trendWeights(weights) - .priorWeights(weights); - double yi{scale[d % 7] * (20.0 + y[i] + noise[i])}; - model.addSamples(params, TTimeDouble2VecSizeTrVec{ - core::make_triple(time, TDouble2Vec{yi}, TAG)}); - //actual.push_back(yi); - } - } - - LOG_DEBUG("*** forecast ***"); - - TErrorBarVec prediction; - core_t::TTime start{time}; - core_t::TTime end{time + 2 * core::constants::WEEK}; - TModelPtr forecastModel(model.cloneForForecast()); - std::string m; - forecastModel->forecast(start, end, 80.0, - MINIMUM_VALUE, MAXIMUM_VALUE, - boost::bind(&mockSink, _1, boost::ref(prediction)), - m); - - std::size_t outOfBounds{0}; - std::size_t count{0}; - TMeanAccumulator error; - - for (std::size_t i = 0u, d = 60; i < prediction.size(); ++d) - { - TDoubleVec noise; - rng.generateNormalSamples(0.0, 24.0, y.size(), noise); - for (std::size_t j = 0u; - i < prediction.size() && j < y.size(); - ++i, ++j, time += bucketLength) + auto trend = [&y, &scale, bucketLength](core_t::TTime time, double noise) { - double yj{scale[d % 7] * (20.0 + y[j] + noise[j])}; - outOfBounds += ( yj < prediction[i].s_LowerBound - || yj > prediction[i].s_UpperBound ? 1 : 0); - ++count; - error.add(std::fabs(yj - prediction[i].s_Predicted) / std::fabs(yj)); - //actual.push_back(yj); - //ly.push_back(prediction[i].s_LowerBound); - //my.push_back(prediction[i].s_Predicted); - //uy.push_back(prediction[i].s_UpperBound); - } - } - - double percentageOutOfBounds{100.0 * static_cast(outOfBounds) / static_cast(count)}; - LOG_DEBUG("% out of bounds = " << percentageOutOfBounds); - LOG_DEBUG("error = " << maths::CBasicStatistics::mean(error)); - - //file << "actual = " << core::CContainerPrinter::print(actual) << ";\n"; - //file << "ly = " << core::CContainerPrinter::print(ly) << ";\n"; - //file << "my = " << core::CContainerPrinter::print(my) << ";\n"; - //file << "uy = " << core::CContainerPrinter::print(uy) << ";\n"; + core_t::TTime d{(time % 604800) / 86400}; + core_t::TTime h{(time % 86400) / bucketLength}; + return scale[d] * (20.0 + y[h] + noise); + }; - CPPUNIT_ASSERT(percentageOutOfBounds < 2.0); - CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.14); + this->test(trend, bucketLength, 60, 24.0, 32.0, 0.13); } void CForecastTest::testComplexConstantLongTermTrend(void) @@ -526,6 +186,22 @@ void CForecastTest::testComplexConstantLongTermTrend(void) LOG_DEBUG("+---------------------------------------------------+"); LOG_DEBUG("| CForecastTest::testComplexConstantLongTermTrend |"); LOG_DEBUG("+---------------------------------------------------+"); + + core_t::TTime bucketLength{3600}; + TDoubleVec y{ 0.0, 10.0, 20.0, 20.0, 30.0, 40.0, 50.0, 60.0, + 80.0, 100.0, 110.0, 120.0, 110.0, 100.0, 90.0, 80.0, + 60.0, 40.0, 30.0, 20.0, 10.0, 10.0, 5.0, 0.0}; + TDoubleVec scale{1.0, 1.1, 1.05, 0.95, 0.9, 0.3, 0.2}; + + auto trend = [&y, &scale, bucketLength](core_t::TTime time, double noise) + { + core_t::TTime d{(time % 604800) / 86400}; + core_t::TTime h{(time % 86400) / bucketLength}; + return 0.25 * static_cast(time) + / static_cast(bucketLength) + scale[d] * (20.0 + y[h] + noise); + }; + + this->test(trend, bucketLength, 60, 24.0, 17.0, 0.04); } void CForecastTest::testComplexVaryingLongTermTrend(void) @@ -533,6 +209,32 @@ void CForecastTest::testComplexVaryingLongTermTrend(void) LOG_DEBUG("+--------------------------------------------------+"); LOG_DEBUG("| CForecastTest::testComplexVaryingLongTermTrend |"); LOG_DEBUG("+--------------------------------------------------+"); + + core_t::TTime bucketLength{3600}; + double day{86400.0}; + TDoubleVec times{ 0.0 , 5.0 * day, 10.0 * day, 15.0 * day, 20.0 * day, 25.0 * day, + 30.0 * day, 35.0 * day, 40.0 * day, 45.0 * day, 50.0 * day, 55.0 * day, + 60.0 * day, 65.0 * day, 70.0 * day, 75.0 * day, 80.0 * day, 85.0 * day, + 90.0 * day, 95.0 * day, 100.0 * day, 105.0 * day, 110.0 * day, 115.0 * day}; + TDoubleVec values{20.0, 30.0, 25.0, 35.0, 45.0, 40.0, 38.0, 36.0, 35.0, 25.0, 35.0, 45.0, + 55.0, 62.0, 70.0, 76.0, 79.0, 82.0, 86.0, 90.0, 95.0, 100.0, 106.0, 112.0}; + TDoubleVec y{0.0, 1.0, 2.0, 2.0, 3.0, 4.0, 5.0, 6.0, + 8.0, 10.0, 11.0, 12.0, 11.0, 10.0, 9.0, 8.0, + 6.0, 4.0, 3.0, 2.0, 1.0, 1.0, 0.5, 0.0}; + TDoubleVec scale{1.0, 1.1, 1.05, 0.95, 0.9, 0.3, 0.2}; + + maths::CSpline<> trend_(maths::CSplineTypes::E_Cubic); + trend_.interpolate(times, values, maths::CSplineTypes::E_Natural); + + auto trend = [&trend_, &y, &scale, bucketLength](core_t::TTime time, double noise) + { + core_t::TTime d{(time % 604800) / 86400}; + core_t::TTime h{(time % 86400) / bucketLength}; + double time_{static_cast(time)}; + return trend_.value(time_) + scale[d] * (20.0 + y[h] + noise); + }; + + this->test(trend, bucketLength, 60, 4.0, 19.0, 0.05); } void CForecastTest::testNonNegative(void) @@ -545,11 +247,11 @@ void CForecastTest::testNonNegative(void) test::CRandomNumbers rng; - maths::CTimeSeriesDecomposition trend(0.012, 1800, 24); + maths::CTimeSeriesDecomposition trend(0.012, bucketLength); maths::CNormalMeanPrecConjugate prior = - maths::CNormalMeanPrecConjugate::nonInformativePrior(maths_t::E_ContinuousData, 0.0005); + maths::CNormalMeanPrecConjugate::nonInformativePrior(maths_t::E_ContinuousData, DECAY_RATE); maths::CUnivariateTimeSeriesModel::TDecayRateController2Ary controllers{decayRateControllers()}; - maths::CUnivariateTimeSeriesModel model(params(bucketLength), 0, trend, prior, &controllers); + maths::CUnivariateTimeSeriesModel model(params(bucketLength), TAG, trend, prior, &controllers); LOG_DEBUG("*** learn ***"); @@ -561,8 +263,8 @@ void CForecastTest::testNonNegative(void) //TDoubleVec uy; core_t::TTime time{0}; - TDouble2Vec4VecVec weights{TDouble2Vec4Vec{TDouble2Vec{1.0}}}; - for (std::size_t d = 0u; d < 10; ++d) + TDouble2Vec4VecVec weights{{{1.0}}}; + for (std::size_t d = 0u; d < 20; ++d) { TDoubleVec noise; rng.generateNormalSamples(2.0, 3.0, 48, noise); @@ -576,8 +278,7 @@ void CForecastTest::testNonNegative(void) .trendWeights(weights) .priorWeights(weights); double y{std::max(*value, 0.0)}; - model.addSamples(params, TTimeDouble2VecSizeTrVec{ - core::make_triple(time, TDouble2Vec{y}, TAG)}); + model.addSamples(params, {core::make_triple(time, TDouble2Vec{y}, TAG)}); //actual.push_back(y); } } @@ -586,13 +287,12 @@ void CForecastTest::testNonNegative(void) TErrorBarVec prediction; core_t::TTime start{time}; - core_t::TTime end{time + 5 * core::constants::DAY}; + core_t::TTime end{time + 20 * core::constants::DAY}; std::string m; TModelPtr forecastModel(model.cloneForForecast()); forecastModel->forecast(start, end, 95.0, MINIMUM_VALUE, MAXIMUM_VALUE, - boost::bind(&mockSink, _1, boost::ref(prediction)), - m); + boost::bind(&mockSink, _1, boost::ref(prediction)), m); std::size_t outOfBounds{0}; std::size_t count{0}; @@ -620,7 +320,8 @@ void CForecastTest::testNonNegative(void) } } - double percentageOutOfBounds{100.0 * static_cast(outOfBounds) / static_cast(count)}; + double percentageOutOfBounds{100.0 * static_cast(outOfBounds) + / static_cast(count)}; LOG_DEBUG("% out of bounds = " << percentageOutOfBounds); //file << "actual = " << core::CContainerPrinter::print(actual) << ";\n"; @@ -628,14 +329,14 @@ void CForecastTest::testNonNegative(void) //file << "my = " << core::CContainerPrinter::print(my) << ";\n"; //file << "uy = " << core::CContainerPrinter::print(uy) << ";\n"; - CPPUNIT_ASSERT(percentageOutOfBounds < 5.0); + CPPUNIT_ASSERT(percentageOutOfBounds < 8.0); } void CForecastTest::testFinancialIndex(void) { - LOG_DEBUG("+--------------------------+"); + LOG_DEBUG("+-------------------------------------+"); LOG_DEBUG("| CForecastTest::testFinancialIndex |"); - LOG_DEBUG("+--------------------------+"); + LOG_DEBUG("+-------------------------------------+"); core_t::TTime bucketLength{1800}; @@ -653,11 +354,11 @@ void CForecastTest::testFinancialIndex(void) timeseries.begin() + 10) << " ..."); - maths::CTimeSeriesDecomposition trend(0.012, 1800, 24); + maths::CTimeSeriesDecomposition trend(0.012, bucketLength); maths::CNormalMeanPrecConjugate prior = - maths::CNormalMeanPrecConjugate::nonInformativePrior(maths_t::E_ContinuousData, 0.0005); + maths::CNormalMeanPrecConjugate::nonInformativePrior(maths_t::E_ContinuousData, DECAY_RATE); maths::CUnivariateTimeSeriesModel::TDecayRateController2Ary controllers{decayRateControllers()}; - maths::CUnivariateTimeSeriesModel model(params(bucketLength), 0, trend, prior, &controllers); + maths::CUnivariateTimeSeriesModel model(params(bucketLength), TAG, trend, prior, &controllers); LOG_DEBUG("*** learn ***"); @@ -668,9 +369,9 @@ void CForecastTest::testFinancialIndex(void) //TDoubleVec my; //TDoubleVec uy; - std::size_t n{9 * timeseries.size() / 10}; + std::size_t n{5 * timeseries.size() / 6}; - TDouble2Vec4VecVec weights{TDouble2Vec4Vec{TDouble2Vec{1.0}}}; + TDouble2Vec4VecVec weights{{{1.0}}}; for (std::size_t i = 0u; i < n; ++i) { maths::CModelAddSamplesParams params; @@ -679,10 +380,9 @@ void CForecastTest::testFinancialIndex(void) .weightStyles(maths::CConstantWeights::COUNT) .trendWeights(weights) .priorWeights(weights); - model.addSamples(params, TTimeDouble2VecSizeTrVec{ - core::make_triple(timeseries[i].first, - TDouble2Vec{timeseries[i].second}, - TAG)}); + model.addSamples(params, {core::make_triple(timeseries[i].first, + TDouble2Vec{timeseries[i].second}, + TAG)}); //actual.push_back(timeseries[i].second); } @@ -693,10 +393,9 @@ void CForecastTest::testFinancialIndex(void) core_t::TTime end{timeseries[timeseries.size() - 1].first}; std::string m; TModelPtr forecastModel(model.cloneForForecast()); - forecastModel->forecast(start, end, 95.0, + forecastModel->forecast(start, end, 99.0, MINIMUM_VALUE, MAXIMUM_VALUE, - boost::bind(&mockSink, _1, boost::ref(prediction)), - m); + boost::bind(&mockSink, _1, boost::ref(prediction)), m); std::size_t outOfBounds{0}; std::size_t count{0}; @@ -717,7 +416,8 @@ void CForecastTest::testFinancialIndex(void) //uy.push_back(prediction[j].s_UpperBound); } - double percentageOutOfBounds{100.0 * static_cast(outOfBounds) / static_cast(count)}; + double percentageOutOfBounds{100.0 * static_cast(outOfBounds) + / static_cast(count)}; LOG_DEBUG("% out of bounds = " << percentageOutOfBounds); LOG_DEBUG("error = " << maths::CBasicStatistics::mean(error)); @@ -726,8 +426,8 @@ void CForecastTest::testFinancialIndex(void) //file << "my = " << core::CContainerPrinter::print(my) << ";\n"; //file << "uy = " << core::CContainerPrinter::print(uy) << ";\n"; - CPPUNIT_ASSERT(percentageOutOfBounds < 1.0); - CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.05); + CPPUNIT_ASSERT(percentageOutOfBounds < 50.0); + CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.1); } CppUnit::Test *CForecastTest::suite(void) @@ -761,3 +461,101 @@ CppUnit::Test *CForecastTest::suite(void) return suiteOfTests; } + +void CForecastTest::test(TTrend trend, + core_t::TTime bucketLength, + std::size_t daysToLearn, + double noiseVariance, + double maximumPercentageOutOfBounds, + double maximumError) +{ + + //std::ofstream file; + //file.open("results.m"); + //TDoubleVec actual; + //TDoubleVec ly; + //TDoubleVec my; + //TDoubleVec uy; + + LOG_DEBUG("*** learn ***"); + + test::CRandomNumbers rng; + + maths::CUnivariateTimeSeriesModel::TDecayRateController2Ary controllers{decayRateControllers()}; + maths::CUnivariateTimeSeriesModel model( + params(bucketLength), TAG, + maths::CTimeSeriesDecomposition(0.012, bucketLength), + maths::CNormalMeanPrecConjugate::nonInformativePrior(maths_t::E_ContinuousData, DECAY_RATE), + &controllers); + + core_t::TTime time{0}; + TDouble2Vec4VecVec weights{{{1.0}}}; + for (std::size_t d = 0u; d < daysToLearn; ++d) + { + TDoubleVec noise; + rng.generateNormalSamples(0.0, noiseVariance, 86400 / bucketLength, noise); + + for (std::size_t i = 0u; i < noise.size(); ++i, time += bucketLength) + { + maths::CModelAddSamplesParams params; + params.integer(false) + .propagationInterval(1.0) + .weightStyles(maths::CConstantWeights::COUNT) + .trendWeights(weights) + .priorWeights(weights); + double yi{trend(time, noise[i])}; + model.addSamples(params, {core::make_triple(time, TDouble2Vec{yi}, TAG)}); + //actual.push_back(yi); + } + } + + LOG_DEBUG("*** forecast ***"); + + TErrorBarVec prediction; + core_t::TTime start{time}; + core_t::TTime end{time + 2 * core::constants::WEEK}; + TModelPtr forecastModel(model.cloneForForecast()); + std::string m; + forecastModel->forecast(start, end, 80.0, + MINIMUM_VALUE, MAXIMUM_VALUE, + boost::bind(&mockSink, _1, boost::ref(prediction)), m); + + std::size_t outOfBounds{0}; + std::size_t count{0}; + TMeanAccumulator error; + + for (std::size_t i = 0u; i < prediction.size(); /**/) + { + TDoubleVec noise; + rng.generateNormalSamples(0.0, noiseVariance, 86400 / bucketLength, noise); + TDoubleVec day; + for (std::size_t j = 0u; + i < prediction.size() && j < noise.size(); + ++i, ++j, time += bucketLength) + { + double yj{trend(time, noise[j])}; + day.push_back(yj); + outOfBounds += ( yj < prediction[i].s_LowerBound + || yj > prediction[i].s_UpperBound ? 1 : 0); + ++count; + error.add(std::fabs(yj - prediction[i].s_Predicted) / std::fabs(yj)); + //actual.push_back(yj); + //ly.push_back(prediction[i].s_LowerBound); + //my.push_back(prediction[i].s_Predicted); + //uy.push_back(prediction[i].s_UpperBound); + } + } + + double percentageOutOfBounds{100.0 * static_cast(outOfBounds) + / static_cast(count)}; + LOG_DEBUG("% out of bounds = " << percentageOutOfBounds); + LOG_DEBUG("error = " << maths::CBasicStatistics::mean(error)); + + //file << "actual = " << core::CContainerPrinter::print(actual) << ";\n"; + //file << "ly = " << core::CContainerPrinter::print(ly) << ";\n"; + //file << "my = " << core::CContainerPrinter::print(my) << ";\n"; + //file << "uy = " << core::CContainerPrinter::print(uy) << ";\n"; + + CPPUNIT_ASSERT(percentageOutOfBounds < maximumPercentageOutOfBounds); + CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < maximumError); +} diff --git a/lib/maths/unittest/CForecastTest.h b/lib/maths/unittest/CForecastTest.h index 48eb650665..4ea4592e0c 100644 --- a/lib/maths/unittest/CForecastTest.h +++ b/lib/maths/unittest/CForecastTest.h @@ -18,19 +18,34 @@ #include +#include + +#include + class CForecastTest : public CppUnit::TestFixture { public: - void testDailyNoLongTermTrend(void); - void testDailyConstantLongTermTrend(void); - void testDailyVaryingLongTermTrend(void); - void testComplexNoLongTermTrend(void); - void testComplexConstantLongTermTrend(void); - void testComplexVaryingLongTermTrend(void); - void testNonNegative(void); - void testFinancialIndex(void); - - static CppUnit::Test *suite(void); + void testDailyNoLongTermTrend(); + void testDailyConstantLongTermTrend(); + void testDailyVaryingLongTermTrend(); + void testComplexNoLongTermTrend(); + void testComplexConstantLongTermTrend(); + void testComplexVaryingLongTermTrend(); + void testNonNegative(); + void testFinancialIndex(); + + static CppUnit::Test *suite(); + + private: + using TTrend = std::function; + + private: + void test(TTrend trend, + ml::core_t::TTime bucketLength, + std::size_t daysToLearn, + double noiseVariance, + double maximumPercentageOutOfBounds, + double maximumError); }; #endif // INCLUDED_CForecastTest_h diff --git a/lib/maths/unittest/CGammaRateConjugateTest.cc b/lib/maths/unittest/CGammaRateConjugateTest.cc index 6a017c0caf..d077d7729f 100644 --- a/lib/maths/unittest/CGammaRateConjugateTest.cc +++ b/lib/maths/unittest/CGammaRateConjugateTest.cc @@ -57,7 +57,7 @@ CGammaRateConjugate makePrior(maths_t::EDataType dataType = maths_t::E_Continuou const double &offset = 0.0, const double &decayRate = 0.0) { - return CGammaRateConjugate::nonInformativePrior(dataType, offset, decayRate); + return CGammaRateConjugate::nonInformativePrior(dataType, offset, decayRate, 0.0); } } @@ -1791,12 +1791,9 @@ void CGammaRateConjugateTest::testNegativeSample(void) rng.generateGammaSamples(shape, scale, 100, samples); CGammaRateConjugate filter1( - CGammaRateConjugate::nonInformativePrior(maths_t::E_ContinuousData, 0.0)); + CGammaRateConjugate::nonInformativePrior(maths_t::E_ContinuousData, 0.0, 0.0, 0.2)); CGammaRateConjugate filter2( - CGammaRateConjugate::nonInformativePrior(maths_t::E_ContinuousData, 1.2586)); - - filter1.setOffset(0.2); - filter2.setOffset(0.2); + CGammaRateConjugate::nonInformativePrior(maths_t::E_ContinuousData, 1.2586, 0.0, 0.2)); filter1.addSamples(samples); filter2.addSamples(samples); diff --git a/lib/maths/unittest/CLogNormalMeanPrecConjugateTest.cc b/lib/maths/unittest/CLogNormalMeanPrecConjugateTest.cc index 45dc1d35e3..b0aedf1c03 100644 --- a/lib/maths/unittest/CLogNormalMeanPrecConjugateTest.cc +++ b/lib/maths/unittest/CLogNormalMeanPrecConjugateTest.cc @@ -55,10 +55,10 @@ typedef maths::CBasicStatistics::SSampleMeanVar::TAccumulator TMeanVarAc typedef CPriorTestInterfaceMixin CLogNormalMeanPrecConjugate; CLogNormalMeanPrecConjugate makePrior(maths_t::EDataType dataType = maths_t::E_ContinuousData, - const double &offset = 0.0, - const double &decayRate = 0.0) + const double &offset = 0.0, + const double &decayRate = 0.0) { - return CLogNormalMeanPrecConjugate::nonInformativePrior(dataType, offset, decayRate); + return CLogNormalMeanPrecConjugate::nonInformativePrior(dataType, offset, decayRate, 0.0); } } @@ -1907,12 +1907,9 @@ void CLogNormalMeanPrecConjugateTest::testNegativeSample(void) rng.generateLogNormalSamples(location, squareScale, 100, samples); CLogNormalMeanPrecConjugate filter1 = - CLogNormalMeanPrecConjugate::nonInformativePrior(maths_t::E_ContinuousData, 0.0); + CLogNormalMeanPrecConjugate::nonInformativePrior(maths_t::E_ContinuousData, 0.0, 0.0, 0.2); CLogNormalMeanPrecConjugate filter2 = - CLogNormalMeanPrecConjugate::nonInformativePrior(maths_t::E_ContinuousData, 1.74524); - - filter1.setOffset(0.2); - filter2.setOffset(0.2); + CLogNormalMeanPrecConjugate::nonInformativePrior(maths_t::E_ContinuousData, 1.74524, 0.0, 0.2); filter1.addSamples(samples); filter2.addSamples(samples); diff --git a/lib/maths/unittest/CMathsMemoryTest.cc b/lib/maths/unittest/CMathsMemoryTest.cc index e72e210e45..8eee314b06 100644 --- a/lib/maths/unittest/CMathsMemoryTest.cc +++ b/lib/maths/unittest/CMathsMemoryTest.cc @@ -42,7 +42,6 @@ void CMathsMemoryTest::testTimeSeriesDecompositions(void) for (unsigned i = 0; i < 600000; i += 600) { decomp.addPoint(time + i, (0.55 * (0.2 + (i % 86400)))); - decomp.testAndInterpolate(time + i); } core::CMemoryUsage mem; diff --git a/lib/maths/unittest/CMultimodalPriorTest.cc b/lib/maths/unittest/CMultimodalPriorTest.cc index ae4d8dc9f6..895eb94205 100644 --- a/lib/maths/unittest/CMultimodalPriorTest.cc +++ b/lib/maths/unittest/CMultimodalPriorTest.cc @@ -65,16 +65,11 @@ typedef CPriorTestInterfaceMixin COneOfNPrior; COneOfNPrior makeModePrior(const double &decayRate = 0.0) { CGammaRateConjugate gamma( - maths::CGammaRateConjugate::nonInformativePrior(maths_t::E_ContinuousData, - 0.01, - decayRate)); + maths::CGammaRateConjugate::nonInformativePrior(maths_t::E_ContinuousData, 0.01, decayRate, 0.0)); CLogNormalMeanPrecConjugate logNormal( - maths::CLogNormalMeanPrecConjugate::nonInformativePrior(maths_t::E_ContinuousData, - 0.01, - decayRate)); + maths::CLogNormalMeanPrecConjugate::nonInformativePrior(maths_t::E_ContinuousData, 0.01, decayRate, 0.0)); CNormalMeanPrecConjugate normal( - maths::CNormalMeanPrecConjugate::nonInformativePrior(maths_t::E_ContinuousData, - decayRate)); + maths::CNormalMeanPrecConjugate::nonInformativePrior(maths_t::E_ContinuousData, decayRate)); COneOfNPrior::TPriorPtrVec priors; priors.push_back(COneOfNPrior::TPriorPtr(gamma.clone())); @@ -459,7 +454,7 @@ void CMultimodalPriorTest::testSingleMode(void) << ", differential entropy " << differentialEntropy); CPPUNIT_ASSERT( maths::CBasicStatistics::mean(L1G) - / maths::CBasicStatistics::mean(differentialEntropy) < 0.05); + / maths::CBasicStatistics::mean(differentialEntropy) < 0.1); } } diff --git a/lib/maths/unittest/COneOfNPriorTest.cc b/lib/maths/unittest/COneOfNPriorTest.cc index 03fda4194c..2c8193062f 100644 --- a/lib/maths/unittest/COneOfNPriorTest.cc +++ b/lib/maths/unittest/COneOfNPriorTest.cc @@ -109,9 +109,9 @@ void COneOfNPriorTest::testFilter(void) LOG_DEBUG("+--------------------------------+"); TPriorPtrVec models; - models.push_back(TPriorPtr(maths::CGammaRateConjugate::nonInformativePrior(E_IntegerData).clone())); - models.push_back(TPriorPtr(maths::CLogNormalMeanPrecConjugate::nonInformativePrior(E_IntegerData).clone())); - models.push_back(TPriorPtr(maths::CNormalMeanPrecConjugate::nonInformativePrior(E_IntegerData).clone())); + models.push_back(TPriorPtr(maths::CGammaRateConjugate::nonInformativePrior(E_ContinuousData).clone())); + models.push_back(TPriorPtr(maths::CLogNormalMeanPrecConjugate::nonInformativePrior(E_ContinuousData).clone())); + models.push_back(TPriorPtr(maths::CNormalMeanPrecConjugate::nonInformativePrior(E_ContinuousData).clone())); models.push_back(TPriorPtr(maths::CPoissonMeanConjugate::nonInformativePrior().clone())); COneOfNPrior filter(maths::COneOfNPrior(clone(models), E_ContinuousData)); diff --git a/lib/maths/unittest/CPeriodicityHypothesisTestsTest.cc b/lib/maths/unittest/CPeriodicityHypothesisTestsTest.cc new file mode 100644 index 0000000000..cf51c7d8bc --- /dev/null +++ b/lib/maths/unittest/CPeriodicityHypothesisTestsTest.cc @@ -0,0 +1,720 @@ +/* + * ELASTICSEARCH CONFIDENTIAL + * + * Copyright (c) 2017 Elasticsearch BV. All Rights Reserved. + * + * Notice: this software, and all information contained + * therein, is the exclusive property of Elasticsearch BV + * and its licensors, if any, and is protected under applicable + * domestic and foreign law, and international treaties. + * + * Reproduction, republication or distribution without the + * express written consent of Elasticsearch BV is + * strictly prohibited. + */ + +#include "CPeriodicityHypothesisTestsTest.h" + +#include +#include +#include + +#include +#include + +#include +#include + +#include "TestUtils.h" + +#include + +#include +#include + +using namespace ml; +using namespace handy_typedefs; + +namespace +{ +using TDoubleVec = std::vector; +using TSizeVec = std::vector; +using TTimeVec = std::vector; +using TTimeDoublePr = std::pair; +using TTimeDoublePrVec = std::vector; +using TStrVec = std::vector; + +const core_t::TTime TEN_MINS{600}; +const core_t::TTime HALF_HOUR{core::constants::HOUR / 2}; +const core_t::TTime HOUR{core::constants::HOUR}; +const core_t::TTime DAY{core::constants::DAY}; +const core_t::TTime WEEK{core::constants::WEEK}; +} + +void CPeriodicityHypothesisTestsTest::testNonPeriodic() +{ + LOG_DEBUG("+----------------------------------------------------+"); + LOG_DEBUG("| CPeriodicityHypothesisTestsTest::testNonPeriodic |"); + LOG_DEBUG("+----------------------------------------------------+"); + + // Test a variety of synthetic non-periodic signals. + + TTimeVec windows{WEEK, 2 * WEEK, 16 * DAY, 4 * WEEK}; + TTimeVec bucketLengths{TEN_MINS, HALF_HOUR}; + TGeneratorVec generators{constant, ramp, markov}; + + test::CRandomNumbers rng; + + TDoubleVec noise; + TSizeVec index; + TSizeVec repeats; + + double FP{0.0}; + double TN{0.0}; + + for (std::size_t test = 0u; test < 50; ++test) + { + if (test % 10 == 0) + { + LOG_DEBUG("test " << test << " / 50"); + } + for (auto window : windows) + { + for (auto bucketLength : bucketLengths) + { + switch (test % 3) + { + case 0: rng.generateNormalSamples(0.0, 0.4, window / bucketLength, noise); break; + case 1: rng.generateGammaSamples(1.0, 5.0, window / bucketLength, noise); break; + case 2: rng.generateLogNormalSamples(0.2, 0.3, window / bucketLength, noise); break; + } + rng.generateUniformSamples(0, generators.size(), 1, index); + rng.generateUniformSamples(3, 20, 1, repeats); + + maths::CPeriodicityHypothesisTests hypotheses; + hypotheses.initialize(bucketLength, window, + window / static_cast(repeats[0])); + + for (core_t::TTime time = 10000; time < 10000 + window; time += bucketLength) + { + hypotheses.add(time, generators[index[0]](time) + + noise[(time - 10000) / bucketLength]); + } + + maths::CPeriodicityHypothesisTestsResult result{hypotheses.test()}; + if (result.periodic()) + { + LOG_DEBUG("result = " << result.print()); + } + FP += result.periodic() ? 1.0 : 0.0; + TN += result.periodic() ? 0.0 : 1.0; + } + } + } + + LOG_DEBUG("True negative rate = " << TN / (FP + TN)); + CPPUNIT_ASSERT(TN / (FP + TN) > 0.995); +} + +void CPeriodicityHypothesisTestsTest::testDiurnal() +{ + LOG_DEBUG("+------------------------------------------------+"); + LOG_DEBUG("| CPeriodicityHypothesisTestsTest::testDiurnal |"); + LOG_DEBUG("+------------------------------------------------+"); + + // Test the recall for a variety of synthetic periodic signals + // and for a number of real data examples. + + LOG_DEBUG("Random diurnal"); + { + TTimeVec windows{WEEK, 2 * WEEK, 16 * DAY, 4 * WEEK}; + TTimeVec bucketLengths{TEN_MINS, HALF_HOUR}; + TSizeVec permittedGenerators{2, 4, 4, 5}; + TGeneratorVec generators{smoothDaily, spikeyDaily, smoothWeekly, weekends, spikeyWeekly}; + TStrVec expected + { + "{ 'daily' }", + "{ 'daily' }", + "{ 'weekly' }", + "{ 'weekend daily' 'weekday daily' 'weekend weekly' 'weekday weekly' }", + "{ 'daily' 'weekly' }" + }; + + test::CRandomNumbers rng; + + TDoubleVec noise; + TSizeVec index; + TSizeVec repeats; + + double TP{0.0}; + double FN{0.0}; + + for (std::size_t test = 0u; test < 100; ++test) + { + if (test % 10 == 0) + { + LOG_DEBUG("test " << test << " / 100"); + } + for (std::size_t i = 0u; i < windows.size(); ++i) + { + core_t::TTime window{windows[i]}; + + for (auto bucketLength : bucketLengths) + { + switch (test % 3) + { + case 0: rng.generateNormalSamples(0.0, 1.0, window / bucketLength, noise); break; + case 1: rng.generateGammaSamples(1.0, 1.0, window / bucketLength, noise); break; + case 2: rng.generateLogNormalSamples(0.2, 0.3, window / bucketLength, noise); break; + } + rng.generateUniformSamples(0, permittedGenerators[i], 1, index); + rng.generateUniformSamples(3, 20, 1, repeats); + + maths::CPeriodicityHypothesisTests hypotheses; + hypotheses.initialize(bucketLength, window, + window / static_cast(repeats[0])); + + for (core_t::TTime time = 10000; time < 10000 + window; time += bucketLength) + { + hypotheses.add(time, 20.0 * generators[index[0]](time) + + noise[(time - 10000) / bucketLength]); + } + + maths::CPeriodicityHypothesisTestsResult result{hypotheses.test()}; + if (result.print() != expected[index[0]]) + { + LOG_DEBUG("result = " << result.print() + << " expected " << expected[index[0]]); + } + TP += result.print() == expected[index[0]] ? 1.0 : 0.0; + FN += result.print() == expected[index[0]] ? 0.0 : 1.0; + } + } + } + + LOG_DEBUG("Recall = " << TP / (TP + FN)); + CPPUNIT_ASSERT(TP / (TP + FN) > 0.99); + } + + LOG_DEBUG(""); + LOG_DEBUG("*** Spikey: daily ***"); + { + TTimeDoublePrVec timeseries; + core_t::TTime startTime; + core_t::TTime endTime; + CPPUNIT_ASSERT(test::CTimeSeriesTestData::parse("testfiles/spikey_data.csv", + timeseries, + startTime, + endTime, + test::CTimeSeriesTestData::CSV_UNIX_REGEX)); + CPPUNIT_ASSERT(!timeseries.empty()); + + LOG_DEBUG("timeseries = " << core::CContainerPrinter::print(timeseries.begin(), + timeseries.begin() + 10) + << " ..."); + + TTimeVec lastTests{timeseries[0].first, timeseries[0].first}; + TTimeVec windows{4 * DAY, 14 * DAY}; + + maths::CPeriodicityHypothesisTests hypotheses[2]; + hypotheses[0].initialize(HOUR, windows[0], DAY); + hypotheses[1].initialize(HOUR, windows[1], DAY); + + for (std::size_t i = 0u; i < timeseries.size(); ++i) + { + core_t::TTime time{timeseries[i].first}; + for (std::size_t j = 0u; j < 2; ++j) + { + if (time > lastTests[j] + windows[j]) + { + maths::CPeriodicityHypothesisTestsResult result{hypotheses[j].test()}; + CPPUNIT_ASSERT_EQUAL(std::string("{ 'daily' }"), result.print()); + hypotheses[j] = maths::CPeriodicityHypothesisTests(); + hypotheses[j].initialize(HOUR, windows[j], DAY); + lastTests[j] += windows[j]; + } + hypotheses[j].add(time, timeseries[i].second); + } + } + } + + LOG_DEBUG(""); + LOG_DEBUG("*** Diurnal: daily and weekends ***"); + { + TTimeDoublePrVec timeseries; + core_t::TTime startTime; + core_t::TTime endTime; + CPPUNIT_ASSERT(test::CTimeSeriesTestData::parse("testfiles/diurnal.csv", + timeseries, + startTime, + endTime, + test::CTimeSeriesTestData::CSV_UNIX_REGEX)); + CPPUNIT_ASSERT(!timeseries.empty()); + + LOG_DEBUG("timeseries = " << core::CContainerPrinter::print(timeseries.begin(), + timeseries.begin() + 10) + << " ..."); + + core_t::TTime lastTest{timeseries[0].first}; + core_t::TTime window{14 * DAY}; + + maths::CPeriodicityHypothesisTests hypotheses; + hypotheses.initialize(HOUR, window, DAY); + + for (std::size_t i = 0u; i < timeseries.size(); ++i) + { + core_t::TTime time{timeseries[i].first}; + if (time > lastTest + window) + { + maths::CPeriodicityHypothesisTestsResult result{hypotheses.test()}; + CPPUNIT_ASSERT_EQUAL(std::string("{ 'weekend daily' 'weekday daily' }"), result.print()); + hypotheses = maths::CPeriodicityHypothesisTests(); + hypotheses.initialize(HOUR, window, DAY); + lastTest += window; + } + hypotheses.add(time, timeseries[i].second); + } + } + + LOG_DEBUG(""); + LOG_DEBUG("*** Switching: no periods ***"); + { + TTimeDoublePrVec timeseries; + core_t::TTime startTime; + core_t::TTime endTime; + CPPUNIT_ASSERT(test::CTimeSeriesTestData::parse("testfiles/no_periods.csv", + timeseries, + startTime, + endTime, + test::CTimeSeriesTestData::CSV_ISO8601_REGEX, + test::CTimeSeriesTestData::CSV_ISO8601_DATE_FORMAT)); + CPPUNIT_ASSERT(!timeseries.empty()); + + LOG_DEBUG("timeseries = " << core::CContainerPrinter::print(timeseries.begin(), + timeseries.begin() + 10) + << " ..."); + + core_t::TTime lastTest{timeseries[0].first}; + core_t::TTime window{14 * DAY}; + + maths::CPeriodicityHypothesisTests hypotheses; + hypotheses.initialize(HOUR, window, DAY); + + for (std::size_t i = 0u; i < timeseries.size(); ++i) + { + core_t::TTime time{timeseries[i].first}; + if (time > lastTest + window) + { + maths::CPeriodicityHypothesisTestsResult result{hypotheses.test()}; + CPPUNIT_ASSERT_EQUAL(std::string("{ }"), result.print()); + hypotheses = maths::CPeriodicityHypothesisTests(); + hypotheses.initialize(HOUR, window, DAY); + lastTest += window; + } + hypotheses.add(time, timeseries[i].second); + } + } + + LOG_DEBUG(""); + LOG_DEBUG("*** Diurnal: daily, weekly and weekends ***"); + { + TTimeDoublePrVec timeseries; + core_t::TTime startTime; + core_t::TTime endTime; + CPPUNIT_ASSERT(test::CTimeSeriesTestData::parse("testfiles/thirty_minute_samples.csv", + timeseries, + startTime, + endTime, + test::CTimeSeriesTestData::CSV_ISO8601_REGEX, + test::CTimeSeriesTestData::CSV_ISO8601_DATE_FORMAT)); + CPPUNIT_ASSERT(!timeseries.empty()); + + LOG_DEBUG("timeseries = " << core::CContainerPrinter::print(timeseries.begin(), + timeseries.begin() + 10) + << " ..."); + + core_t::TTime lastTest{timeseries[0].first}; + core_t::TTime window{14 * DAY}; + + maths::CPeriodicityHypothesisTests hypotheses; + hypotheses.initialize(HOUR, window, DAY); + + for (std::size_t i = 0u; i < timeseries.size(); ++i) + { + core_t::TTime time{timeseries[i].first}; + if (time > lastTest + window) + { + maths::CPeriodicityHypothesisTestsResult result{hypotheses.test()}; + CPPUNIT_ASSERT( result.print() == "{ 'weekend daily' 'weekday daily' }" + || result.print() == "{ 'weekend daily' 'weekday daily' 'weekend weekly' 'weekday weekly' }"); + hypotheses = maths::CPeriodicityHypothesisTests(); + hypotheses.initialize(HOUR, window, DAY); + lastTest += window; + } + hypotheses.add(time, timeseries[i].second); + } + } +} + +void CPeriodicityHypothesisTestsTest::testNonDiurnal() +{ + LOG_DEBUG("+---------------------------------------------------+"); + LOG_DEBUG("| CPeriodicityHypothesisTestsTest::testNonDiurnal |"); + LOG_DEBUG("+---------------------------------------------------+"); + + // Test the recall for periods in the range [DAY / 5, 5 * DAY]. + + TTimeVec windows{WEEK, 2 * WEEK, 16 * DAY, 4 * WEEK}; + TTimeVec bucketLengths{TEN_MINS, HALF_HOUR}; + TGeneratorVec generators{smoothDaily, spikeyDaily}; + TSizeVec permittedGenerators{2, 1}; + + test::CRandomNumbers rng; + + TDoubleVec noise; + TSizeVec index; + TSizeVec repeats; + + double TP{0.0}; + double FN{0.0}; + + for (std::size_t test = 0u; test < 100; ++test) + { + if (test % 10 == 0) + { + LOG_DEBUG("test " << test << " / 100"); + } + for (std::size_t i = 0u; i < windows.size(); ++i) + { + core_t::TTime window{windows[i]}; + + TDoubleVec scaling_; + rng.generateUniformSamples(1.0, 5.0, 1, scaling_); + double scaling{test % 2 == 0 ? scaling_[0] : 1.0 / scaling_[0]}; + + for (std::size_t j = 0u; j < bucketLengths.size(); ++j) + { + core_t::TTime bucketLength{bucketLengths[j]}; + core_t::TTime period{maths::CIntegerTools::floor( + static_cast(static_cast(DAY) / scaling), + bucketLength)}; + scaling = static_cast(DAY) / static_cast(period); + if (scaling == 1.0 || window < 3 * period) + { + continue; + } + + maths::CPeriodicityHypothesisTestsResult expected; + expected.add(core::CStringUtils::typeToString(period), false, 0, period, {0, period}); + + switch (test % 3) + { + case 0: rng.generateNormalSamples(0.0, 1.0, window / bucketLength, noise); break; + case 1: rng.generateGammaSamples(1.0, 1.0, window / bucketLength, noise); break; + case 2: rng.generateLogNormalSamples(0.2, 0.3, window / bucketLength, noise); break; + } + rng.generateUniformSamples(0, permittedGenerators[j], 1, index); + rng.generateUniformSamples(3, 20, 1, repeats); + + maths::CPeriodicityHypothesisTests hypotheses; + hypotheses.initialize(bucketLength, window, period); + + for (core_t::TTime time = 10000; time < 10000 + window; time += bucketLength) + { + hypotheses.add(time, 20.0 * scale(scaling, time, generators[index[0]]) + + noise[(time - 10000) / bucketLength]); + } + + maths::CPeriodicityHypothesisTestsResult result{hypotheses.test()}; + if (result.print() != expected.print()) + { + LOG_DEBUG("result = " << result.print() + << " expected " << expected.print()); + } + TP += result.print() == expected.print() ? 1.0 : 0.0; + FN += result.print() == expected.print() ? 0.0 : 1.0; + } + } + } + + LOG_DEBUG("Recall = " << TP / (TP + FN)); + CPPUNIT_ASSERT(TP / (TP + FN) > 0.99); +} + +void CPeriodicityHypothesisTestsTest::testWithSparseData() +{ + LOG_DEBUG("+-----------------------------------------------------------+"); + LOG_DEBUG("| CPeriodicityHypothesisTestsTest::testTestWithSparseData |"); + LOG_DEBUG("+-----------------------------------------------------------+"); + + test::CRandomNumbers rng; + + LOG_DEBUG("Daily Periodic") + { + maths::CPeriodicityHypothesisTests hypotheses; + hypotheses.initialize(HALF_HOUR, WEEK, DAY); + + core_t::TTime time = 0; + for (std::size_t t = 0u; t < 7; ++t) + { + for (auto value : { 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, + 10.0, 10.0, 8.0, 4.0, 3.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 1.0}) + { + if (value > 0.0) + { + hypotheses.add(time, value); + } + time += HALF_HOUR; + } + if (t > 3) + { + maths::CPeriodicityHypothesisTestsResult result{hypotheses.test()}; + LOG_DEBUG("result = " << result.print()); + CPPUNIT_ASSERT_EQUAL(std::string("{ 'daily' }"), result.print()); + } + } + } + + LOG_DEBUG("Daily Not Periodic") + { + maths::CPeriodicityHypothesisTests hypotheses; + hypotheses.initialize(HALF_HOUR, WEEK, DAY); + + core_t::TTime time = 0; + for (std::size_t t = 0u; t < 7; ++t) + { + for (auto value : { 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, + 10.0, 10.0, 8.0, 4.0, 3.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 1.0}) + { + if (value > 0.0) + { + TDoubleVec rand; + rng.generateUniformSamples(-1.0, 1.0, 1, rand); + hypotheses.add(time, rand[0]); + } + time += HALF_HOUR; + } + + maths::CPeriodicityHypothesisTestsResult result{hypotheses.test()}; + LOG_DEBUG("result = " << result.print()); + CPPUNIT_ASSERT_EQUAL(std::string("{ }"), result.print()); + } + } + + LOG_DEBUG("Weekly") + { + maths::CPeriodicityHypothesisTests hypotheses; + hypotheses.initialize(HOUR, 2 * WEEK, WEEK); + + core_t::TTime time = 0; + for (std::size_t t = 0u; t < 4; ++t) + { + for (auto value : { 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 10.0, 10.0, 8.0, 4.0, 3.0, 1.0, 1.0, 3.0, 0.0, 0.0, 0.0, 0.0, + 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, + 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, + 10.0, 10.0, 8.0, 4.0, 3.0, 1.0, 1.0, 3.0, 0.0, 0.0, 0.0, 0.0, + 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, + 10.0, 10.0, 8.0, 4.0, 3.0, 1.0, 1.0, 3.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}) + { + if (value > 0.0) + { + hypotheses.add(time, value); + } + time += HOUR; + } + + if (t >= 2) + { + maths::CPeriodicityHypothesisTestsResult result{hypotheses.test()}; + LOG_DEBUG("result = " << result.print()); + CPPUNIT_ASSERT_EQUAL(std::string("{ 'daily' 'weekly' }"), result.print()); + } + } + } + + LOG_DEBUG("Weekly Not Periodic") + { + maths::CPeriodicityHypothesisTests hypotheses; + hypotheses.initialize(HOUR, 4 * WEEK, WEEK); + + core_t::TTime time = 0; + for (std::size_t t = 0u; t < 4; ++t) + { + for (auto value : { 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 10.0, 10.0, 8.0, 4.0, 3.0, 1.0, 1.0, 3.0, 0.0, 0.0, 0.0, 0.0, + 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, + 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, + 10.0, 10.0, 8.0, 4.0, 3.0, 1.0, 1.0, 3.0, 0.0, 0.0, 0.0, 0.0, + 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, + 10.0, 10.0, 8.0, 4.0, 3.0, 1.0, 1.0, 3.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}) + { + if (value > 0.0) + { + TDoubleVec rand; + rng.generateUniformSamples(-1.0, 1.0, 1, rand); + hypotheses.add(time, rand[0]); + } + time += HOUR; + } + + maths::CPeriodicityHypothesisTestsResult result{hypotheses.test()}; + LOG_DEBUG("result = " << result.print()); + CPPUNIT_ASSERT_EQUAL(std::string("{ }"), result.print()); + } + } +} + +void CPeriodicityHypothesisTestsTest::testTestForPeriods() +{ + LOG_DEBUG("+-------------------------------------------------------+"); + LOG_DEBUG("| CPeriodicityHypothesisTestsTest::testTestForPeriods |"); + LOG_DEBUG("+-------------------------------------------------------+"); + + // Test the ability to correctly find and test for periodic + // signals without being told the periods to test a-priori. + + TTimeVec windows{WEEK, 2 * WEEK, 16 * DAY, 4 * WEEK}; + TTimeVec bucketLengths{TEN_MINS, HALF_HOUR}; + TGeneratorVec generators{smoothDaily, spikeyDaily}; + TSizeVec permittedGenerators{2, 1}; + core_t::TTime startTime{10000}; + + test::CRandomNumbers rng; + + TDoubleVec noise; + TSizeVec index; + TSizeVec repeats; + + TDoubleVec TP{0.0, 0.0, 0.0}; + TDoubleVec FN{0.0, 0.0, 0.0}; + + for (std::size_t test = 0u; test < 100; ++test) + { + if (test % 10 == 0) + { + LOG_DEBUG("test " << test << " / 100"); + } + for (std::size_t i = 0u; i < windows.size(); ++i) + { + core_t::TTime window{windows[i]}; + + TDoubleVec scaling_; + rng.generateUniformSamples(1.0, 5.0, 1, scaling_); + double scaling{test % 2 == 0 ? scaling_[0] : 1.0 / scaling_[0]}; + + for (std::size_t j = 0u; j < bucketLengths.size(); ++j) + { + core_t::TTime bucketLength{bucketLengths[j]}; + core_t::TTime period{maths::CIntegerTools::floor( + static_cast(static_cast(DAY) / scaling), + bucketLength)}; + scaling = static_cast(DAY) / static_cast(period); + if (scaling == 1.0 || window < 3 * period) + { + continue; + } + + maths::CPeriodicityHypothesisTestsResult expected; + expected.add(core::CStringUtils::typeToString(period), false, 0, period, {0, period}); + + switch (test % 3) + { + case 0: rng.generateNormalSamples(0.0, 1.0, window / bucketLength, noise); break; + case 1: rng.generateGammaSamples(1.0, 1.0, window / bucketLength, noise); break; + case 2: rng.generateLogNormalSamples(0.2, 0.3, window / bucketLength, noise); break; + } + rng.generateUniformSamples(0, permittedGenerators[j], 1, index); + rng.generateUniformSamples(3, 20, 1, repeats); + + maths::CPeriodicityHypothesisTests hypotheses; + hypotheses.initialize(bucketLength, window, period); + + maths::TFloatMeanAccumulatorVec values(window / bucketLength); + for (core_t::TTime time = startTime; time < startTime + window; time += bucketLength) + { + std::size_t bucket((time - startTime) / bucketLength); + double value{20.0 * scale(scaling, time, generators[index[0]]) + noise[bucket]}; + values[bucket].add(value); + } + + maths::CPeriodicityHypothesisTestsConfig config; + maths::CPeriodicityHypothesisTestsResult result{ + maths::testForPeriods(config, startTime, bucketLength, values)}; + if (result.print() != expected.print()) + { + LOG_DEBUG("result = " << result.print() + << " expected " << expected.print()); + } + + TP[0] += result.print() == expected.print() ? 1.0 : 0.0; + FN[0] += result.print() == expected.print() ? 0.0 : 1.0; + if (result.components().size() == 1) + { + core_t::TTime modp{result.components()[0].s_Period % period}; + double error{ static_cast(std::min(modp, std::abs(period - modp))) + / static_cast(period)}; + TP[1] += error < 0.01 ? 1.0 : 0.0; + FN[1] += error < 0.01 ? 0.0 : 1.0; + TP[2] += error < 0.05 ? 1.0 : 0.0; + FN[2] += error < 0.05 ? 0.0 : 1.0; + } + else + { + FN[0] += 1.0; + FN[1] += 1.0; + FN[2] += 1.0; + } + } + } + } + + LOG_DEBUG("Recall at 0% error = " << TP[0] / (TP[0] + FN[0])); + LOG_DEBUG("Recall at 1% error = " << TP[1] / (TP[1] + FN[1])); + LOG_DEBUG("Recall at 5% error = " << TP[2] / (TP[2] + FN[2])); + CPPUNIT_ASSERT(TP[0] / (TP[0] + FN[0]) > 0.91); + CPPUNIT_ASSERT(TP[1] / (TP[1] + FN[1]) > 0.99); + CPPUNIT_ASSERT(TP[2] / (TP[2] + FN[2]) > 0.99); +} + +CppUnit::Test *CPeriodicityHypothesisTestsTest::suite() +{ + CppUnit::TestSuite *suiteOfTests = new CppUnit::TestSuite("CPeriodicityHypothesisTestsTest"); + + suiteOfTests->addTest( new CppUnit::TestCaller( + "CPeriodicityHypothesisTestsTest::testNonPeriodic", + &CPeriodicityHypothesisTestsTest::testNonPeriodic) ); + suiteOfTests->addTest( new CppUnit::TestCaller( + "CPeriodicityHypothesisTestsTest::testDiurnal", + &CPeriodicityHypothesisTestsTest::testDiurnal) ); + suiteOfTests->addTest( new CppUnit::TestCaller( + "CPeriodicityHypothesisTestsTest::testNonDiurnal", + &CPeriodicityHypothesisTestsTest::testNonDiurnal) ); + suiteOfTests->addTest( new CppUnit::TestCaller( + "CPeriodicityHypothesisTestsTest::testWithSparseData", + &CPeriodicityHypothesisTestsTest::testWithSparseData) ); + suiteOfTests->addTest( new CppUnit::TestCaller( + "CPeriodicityHypothesisTestsTest::testTestForPeriods", + &CPeriodicityHypothesisTestsTest::testTestForPeriods) ); + + return suiteOfTests; + +} diff --git a/lib/maths/unittest/CPeriodicityHypothesisTestsTest.h b/lib/maths/unittest/CPeriodicityHypothesisTestsTest.h new file mode 100644 index 0000000000..31f17ea117 --- /dev/null +++ b/lib/maths/unittest/CPeriodicityHypothesisTestsTest.h @@ -0,0 +1,33 @@ +/* + * ELASTICSEARCH CONFIDENTIAL + * + * Copyright (c) 2017 Elasticsearch BV. All Rights Reserved. + * + * Notice: this software, and all information contained + * therein, is the exclusive property of Elasticsearch BV + * and its licensors, if any, and is protected under applicable + * domestic and foreign law, and international treaties. + * + * Reproduction, republication or distribution without the + * express written consent of Elasticsearch BV is + * strictly prohibited. + */ + +#ifndef INCLUDED_CPeriodicityHypothesisTestsTest_h +#define INCLUDED_CPeriodicityHypothesisTestsTest_h + +#include + +class CPeriodicityHypothesisTestsTest : public CppUnit::TestFixture +{ + public: + void testNonPeriodic(); + void testDiurnal(); + void testNonDiurnal(); + void testWithSparseData(); + void testTestForPeriods(); + + static CppUnit::Test *suite(); +}; + +#endif // INCLUDED_CPeriodicityHypothesisTestsTest_h diff --git a/lib/maths/unittest/CRegressionTest.cc b/lib/maths/unittest/CRegressionTest.cc index a3a8bba3bb..e9e3032cc7 100644 --- a/lib/maths/unittest/CRegressionTest.cc +++ b/lib/maths/unittest/CRegressionTest.cc @@ -829,7 +829,7 @@ class CRegressionPrediction bool operator()(double x, double &result) const { - result = maths::CRegression::predict(m_Regression, x); + result = m_Regression.predict(x); return true; } @@ -1110,10 +1110,12 @@ void CRegressionTest::testParameterProcess(void) v += a * 0.05; a += da_; } + + bool sufficientHistoryBeforeUpdate = regression.range() >= 1.0; TVector paramsDrift(regression.parameters(t + dt)); regression.add(t + dt, x); paramsDrift -= TVector(regression.parameters(t + dt)); - if (regression.sufficientHistoryToPredict()) + if (sufficientHistoryBeforeUpdate && regression.range() >= 1.0) { parameterProcess.add(t + dt, paramsDrift, TVector(dt)); } @@ -1163,7 +1165,7 @@ void CRegressionTest::testParameterProcess(void) } LOG_DEBUG("error = " << maths::CBasicStatistics::mean(error)); - CPPUNIT_ASSERT(std::fabs(maths::CBasicStatistics::mean(error)) < 0.06); + CPPUNIT_ASSERT(std::fabs(maths::CBasicStatistics::mean(error)) < 0.08); } CppUnit::Test *CRegressionTest::suite(void) diff --git a/lib/maths/unittest/CSeasonalComponentAdaptiveBucketingTest.cc b/lib/maths/unittest/CSeasonalComponentAdaptiveBucketingTest.cc index aca35a11a7..7177ee8430 100644 --- a/lib/maths/unittest/CSeasonalComponentAdaptiveBucketingTest.cc +++ b/lib/maths/unittest/CSeasonalComponentAdaptiveBucketingTest.cc @@ -29,6 +29,7 @@ #include #include +#include #include #include @@ -439,7 +440,7 @@ void CSeasonalComponentAdaptiveBucketingTest::testMinimumBucketLength(void) } LOG_DEBUG("minimumTotalError = " << minimumTotalError); LOG_DEBUG("totalError = " << totalError); - CPPUNIT_ASSERT(totalError <= 6.0 * minimumTotalError); + CPPUNIT_ASSERT(totalError <= 7.5 * minimumTotalError); } } @@ -548,7 +549,8 @@ void CSeasonalComponentAdaptiveBucketingTest::testKnots(void) LOG_DEBUG("meanError = " << maths::CBasicStatistics::mean(meanError)); LOG_DEBUG("meanValue = " << maths::CBasicStatistics::mean(meanValue)); CPPUNIT_ASSERT( maths::CBasicStatistics::mean(meanError) - / maths::CBasicStatistics::mean(meanValue) < 0.32 / static_cast(p+1)); + / maths::CBasicStatistics::mean(meanValue) + < 0.1 / ::sqrt(static_cast(p+1))); } } LOG_DEBUG("*** Variances ***"); @@ -599,8 +601,7 @@ void CSeasonalComponentAdaptiveBucketingTest::testKnots(void) LOG_DEBUG("meanError = " << maths::CBasicStatistics::mean(meanError)); LOG_DEBUG("meanVariance = " << maths::CBasicStatistics::mean(meanVariance)); CPPUNIT_ASSERT( maths::CBasicStatistics::mean(meanError) - / maths::CBasicStatistics::mean(meanVariance) - < 0.35 / ::sqrt(static_cast((p+1))/10)); + / maths::CBasicStatistics::mean(meanVariance) < 0.2); } } } @@ -618,7 +619,7 @@ void CSeasonalComponentAdaptiveBucketingTest::testLongTermTrendKnots(void) maths::CDiurnalTime time(0, 0, 86400, 86400); maths::CSeasonalComponentAdaptiveBucketing bucketing(time, 0.1, 864.0); - maths::CSeasonalComponentAdaptiveBucketing::TTimeTimePrMeanVarPrVec empty; + maths::CSeasonalComponentAdaptiveBucketing::TFloatMeanAccumulatorVec empty; bucketing.initialize(20); bucketing.initialValues(0, 0, empty); @@ -683,7 +684,7 @@ void CSeasonalComponentAdaptiveBucketingTest::testShiftValue(void) maths::CDiurnalTime time(0, 0, 86400, 86400); maths::CSeasonalComponentAdaptiveBucketing bucketing(time, 0.1, 600.0); - maths::CSeasonalComponentAdaptiveBucketing::TTimeTimePrMeanVarPrVec empty; + maths::CSeasonalComponentAdaptiveBucketing::TFloatMeanAccumulatorVec empty; bucketing.initialize(20); bucketing.initialValues(0, 0, empty); @@ -733,13 +734,13 @@ void CSeasonalComponentAdaptiveBucketingTest::testSlope(void) maths::CDiurnalTime time(0, 0, 86400, 86400); maths::CSeasonalComponentAdaptiveBucketing bucketing(time, 0.1, 600.0); - maths::CSeasonalComponentAdaptiveBucketing::TTimeTimePrMeanVarPrVec empty; + maths::CSeasonalComponentAdaptiveBucketing::TFloatMeanAccumulatorVec empty; bucketing.initialize(20); bucketing.initialValues(0, 0, empty); core_t::TTime t = 0; - for (/**/; t < 30 * 86400; t += 600) + for (/**/; t < 60 * 86400; t += 600) { double x = static_cast(t) / 86400.0; double y = x + 20.0 + 20.0 * ::sin(boost::math::double_constants::two_pi * x); @@ -754,14 +755,14 @@ void CSeasonalComponentAdaptiveBucketingTest::testSlope(void) double slopeBefore = bucketing.slope(); LOG_DEBUG("slope = " << slopeBefore); - CPPUNIT_ASSERT_DOUBLES_EQUAL(7.0, slopeBefore, 0.15); + CPPUNIT_ASSERT_DOUBLES_EQUAL(7.0, slopeBefore, 0.25); bucketing.shiftSlope(10.0); double slopeAfter = bucketing.slope(); LOG_DEBUG("slope = " << slopeAfter); - CPPUNIT_ASSERT_DOUBLES_EQUAL(slopeBefore + 10.0, slopeAfter, 1e-5); + CPPUNIT_ASSERT_DOUBLES_EQUAL(slopeBefore + 10.0, slopeAfter, 1e-4); } void CSeasonalComponentAdaptiveBucketingTest::testPersist(void) @@ -776,9 +777,9 @@ void CSeasonalComponentAdaptiveBucketingTest::testPersist(void) double minimumBucketLength = 1.0; maths::CDiurnalTime time(0, 0, 86400, 86400); - maths::CSeasonalComponentAdaptiveBucketing bucketing(time, decayRate, minimumBucketLength); + maths::CSeasonalComponentAdaptiveBucketing origBucketing(time, decayRate, minimumBucketLength); - bucketing.initialize(10); + origBucketing.initialize(10); for (std::size_t p = 0; p < 10; ++p) { for (std::size_t i = 0u; i < 100; ++i) @@ -786,17 +787,17 @@ void CSeasonalComponentAdaptiveBucketingTest::testPersist(void) core_t::TTime x = static_cast(p * 86400 + 864 * i); double y = 0.02 * (static_cast(i) - 50.0) * (static_cast(i) - 50.0); - bucketing.add(x, y, y); + origBucketing.add(x, y, y); } - bucketing.refine(static_cast(86400 * (p + 1))); + origBucketing.refine(static_cast(86400 * (p + 1))); } - uint64_t checksum = bucketing.checksum(); + uint64_t checksum = origBucketing.checksum(); std::string origXml; { core::CRapidXmlStatePersistInserter inserter("root"); - bucketing.acceptPersistInserter(inserter); + origBucketing.acceptPersistInserter(inserter); inserter.toXml(origXml); } @@ -826,6 +827,82 @@ void CSeasonalComponentAdaptiveBucketingTest::testPersist(void) CPPUNIT_ASSERT_EQUAL(origXml, newXml); } +void CSeasonalComponentAdaptiveBucketingTest::testUpgrade(void) +{ + LOG_DEBUG("+--------------------------------------------------------+"); + LOG_DEBUG("| CSeasonalComponentAdaptiveBucketingTest::testUpgrade |"); + LOG_DEBUG("+--------------------------------------------------------+"); + + // Check we can validly upgrade existing state. + + double decayRate = 0.1; + double minimumBucketLength = 1.0; + + maths::CDiurnalTime time(0, 0, 86400, 86400); + maths::CSeasonalComponentAdaptiveBucketing expectedBucketing(time, decayRate, minimumBucketLength); + + expectedBucketing.initialize(10); + for (std::size_t p = 0; p < 10; ++p) + { + for (std::size_t i = 0u; i < 100; ++i) + { + core_t::TTime x = static_cast(p * 86400 + 864 * i); + double y = 0.02 * (static_cast(i) - 50.0) + * (static_cast(i) - 50.0); + expectedBucketing.add(x, y, y); + } + expectedBucketing.refine(static_cast(86400 * (p + 1))); + } + + std::ifstream file; + file.open("testfiles/CSeasonalComponentAdaptiveBucketing.6.2.state.xml"); + std::stringbuf buf; + file >> &buf; + std::string xml{buf.str()}; + LOG_DEBUG("Saved state size = " << xml.size()); + + core::CRapidXmlParser parser; + CPPUNIT_ASSERT(parser.parseStringIgnoreCdata(xml)); + core::CRapidXmlStateRestoreTraverser traverser(parser); + + // Restore the XML into a new bucketing. + maths::CSeasonalComponentAdaptiveBucketing restoredBucketing(decayRate + 0.1, + minimumBucketLength, + traverser); + + // Check that the knots points we get back are very nearly + // those we expect. + + TDoubleVec expectedKnots; + TDoubleVec expectedValues; + TDoubleVec expectedVariances; + expectedBucketing.knots(863136, maths::CSplineTypes::E_Periodic, + expectedKnots, expectedValues, expectedVariances); + + TDoubleVec restoredKnots; + TDoubleVec restoredValues; + TDoubleVec restoredVariances; + expectedBucketing.knots(863136, maths::CSplineTypes::E_Periodic, + restoredKnots, restoredValues, restoredVariances); + + CPPUNIT_ASSERT_EQUAL(expectedBucketing.decayRate(), restoredBucketing.decayRate()); + + LOG_DEBUG("expected knots = " << core::CContainerPrinter::print(expectedKnots)); + LOG_DEBUG("restored knots = " << core::CContainerPrinter::print(restoredKnots)); + CPPUNIT_ASSERT_EQUAL(core::CContainerPrinter::print(expectedKnots), + core::CContainerPrinter::print(restoredKnots)); + + LOG_DEBUG("expected values = " << core::CContainerPrinter::print(expectedValues)); + LOG_DEBUG("restored values = " << core::CContainerPrinter::print(restoredValues)); + CPPUNIT_ASSERT_EQUAL(core::CContainerPrinter::print(expectedValues), + core::CContainerPrinter::print(restoredValues)); + + LOG_DEBUG("expected variances = " << core::CContainerPrinter::print(expectedVariances)); + LOG_DEBUG("restored variances = " << core::CContainerPrinter::print(restoredVariances)); + CPPUNIT_ASSERT_EQUAL(core::CContainerPrinter::print(expectedVariances), + core::CContainerPrinter::print(restoredVariances)); +} + CppUnit::Test *CSeasonalComponentAdaptiveBucketingTest::suite(void) { CppUnit::TestSuite *suiteOfTests = new CppUnit::TestSuite("CSeasonalComponentAdaptiveBucketingTest"); @@ -863,6 +940,9 @@ CppUnit::Test *CSeasonalComponentAdaptiveBucketingTest::suite(void) suiteOfTests->addTest( new CppUnit::TestCaller( "CSeasonalComponentAdaptiveBucketingTest::testPersist", &CSeasonalComponentAdaptiveBucketingTest::testPersist) ); + suiteOfTests->addTest( new CppUnit::TestCaller( + "CSeasonalComponentAdaptiveBucketingTest::testUpgrade", + &CSeasonalComponentAdaptiveBucketingTest::testUpgrade) ); return suiteOfTests; } diff --git a/lib/maths/unittest/CSeasonalComponentAdaptiveBucketingTest.h b/lib/maths/unittest/CSeasonalComponentAdaptiveBucketingTest.h index de505af6d6..bd82c0fd8c 100644 --- a/lib/maths/unittest/CSeasonalComponentAdaptiveBucketingTest.h +++ b/lib/maths/unittest/CSeasonalComponentAdaptiveBucketingTest.h @@ -32,6 +32,7 @@ class CSeasonalComponentAdaptiveBucketingTest : public CppUnit::TestFixture void testShiftValue(void); void testSlope(void); void testPersist(void); + void testUpgrade(void); static CppUnit::Test *suite(void); }; diff --git a/lib/maths/unittest/CSeasonalComponentTest.cc b/lib/maths/unittest/CSeasonalComponentTest.cc index 0bcd348dad..e7bad50e65 100644 --- a/lib/maths/unittest/CSeasonalComponentTest.cc +++ b/lib/maths/unittest/CSeasonalComponentTest.cc @@ -71,12 +71,6 @@ class CTestSeasonalComponent : public maths::CSeasonalComponent m_StartTime(startTime) {} - bool initialize(core_t::TTime time) - { - TTimeTimePrMeanVarPrVec empty; - return this->maths::CSeasonalComponent::initialize(time, time, empty); - } - void addPoint(core_t::TTime time, double value, double weight = 1.0) @@ -176,7 +170,7 @@ void CSeasonalComponentTest::testNoPeriodicity(void) double residualMean = maths::CBasicStatistics::mean(residuals); CTestSeasonalComponent seasonal(startTime, core::constants::DAY, core::constants::DAY, 24); - seasonal.initialize(startTime); + seasonal.initialize(); //std::ofstream file; //file.open("results.m"); @@ -284,7 +278,7 @@ void CSeasonalComponentTest::testConstantPeriodic(void) double residualMean = maths::CBasicStatistics::mean(residuals); CTestSeasonalComponent seasonal(startTime, core::constants::DAY, core::constants::DAY, 24, 0.01); - seasonal.initialize(startTime); + seasonal.initialize(); //std::ofstream file; //file.open("results.m"); @@ -353,7 +347,7 @@ void CSeasonalComponentTest::testConstantPeriodic(void) totalError2 /= 30.0; LOG_DEBUG("totalError1 = " << totalError1); LOG_DEBUG("totalError2 = " << totalError2); - CPPUNIT_ASSERT(totalError1 < 0.6); + CPPUNIT_ASSERT(totalError1 < 0.5); CPPUNIT_ASSERT(totalError2 < 0.01); } @@ -430,7 +424,7 @@ void CSeasonalComponentTest::testConstantPeriodic(void) double residualMean = maths::CBasicStatistics::mean(residuals); CTestSeasonalComponent seasonal(startTime, core::constants::DAY, core::constants::DAY, 24, 0.01); - seasonal.initialize(startTime); + seasonal.initialize(); //std::ofstream file; //file.open("results.m"); @@ -482,7 +476,7 @@ void CSeasonalComponentTest::testConstantPeriodic(void) LOG_DEBUG("error1 = " << error1); LOG_DEBUG("error2 = " << error2); CPPUNIT_ASSERT(error1 < 11.0); - CPPUNIT_ASSERT(error2 < 4.5); + CPPUNIT_ASSERT(error2 < 4.6); totalError1 += error1; totalError2 += error2; @@ -500,8 +494,8 @@ void CSeasonalComponentTest::testConstantPeriodic(void) totalError2 /= 40.0; LOG_DEBUG("totalError1 = " << totalError1); LOG_DEBUG("totalError2 = " << totalError2); - CPPUNIT_ASSERT(totalError1 < 6.4); - CPPUNIT_ASSERT(totalError2 < 3.1); + CPPUNIT_ASSERT(totalError1 < 7.3); + CPPUNIT_ASSERT(totalError2 < 4.2); } } @@ -574,7 +568,7 @@ void CSeasonalComponentTest::testTimeVaryingPeriodic(void) test::CRandomNumbers rng; CTestSeasonalComponent seasonal(startTime, core::constants::DAY, core::constants::DAY, 24, 0.048); - seasonal.initialize(startTime); + seasonal.initialize(); core_t::TTime time = startTime; @@ -642,7 +636,7 @@ void CSeasonalComponentTest::testTimeVaryingPeriodic(void) error2 /= static_cast(function.size()); LOG_DEBUG("error1 = " << error1); LOG_DEBUG("error2 = " << error2); - CPPUNIT_ASSERT(error1 < 29.0); + CPPUNIT_ASSERT(error1 < 42.0); CPPUNIT_ASSERT(error2 < 20.0); totalError1 += error1; totalError2 += error2; @@ -748,7 +742,7 @@ void CSeasonalComponentTest::testVeryLowVariation(void) error2 /= static_cast(function.size()); LOG_DEBUG("deviation = " << deviation); LOG_DEBUG("error1 = " << error1 << ", error2 = " << error2); - CPPUNIT_ASSERT_DOUBLES_EQUAL(0.0, error1, 0.5 * deviation); + CPPUNIT_ASSERT_DOUBLES_EQUAL(0.0, error1, 1.0 * deviation); CPPUNIT_ASSERT_DOUBLES_EQUAL(0.0, error2, 0.1 * deviation); totalError1 += error1; totalError2 += error2; @@ -765,7 +759,7 @@ void CSeasonalComponentTest::testVeryLowVariation(void) totalError2 /= 30.0; LOG_DEBUG("deviation = " << deviation); LOG_DEBUG("totalError1 = " << totalError1 << ", totalError2 = " << totalError2); - CPPUNIT_ASSERT_DOUBLES_EQUAL(totalError1, 0.0, 0.15 * deviation); + CPPUNIT_ASSERT_DOUBLES_EQUAL(totalError1, 0.0, 0.20 * deviation); CPPUNIT_ASSERT_DOUBLES_EQUAL(totalError2, 0.0, 0.04 * deviation); } @@ -815,13 +809,13 @@ void CSeasonalComponentTest::testVariance(void) << ", v = " << core::CContainerPrinter::print(vv) << ", relative error = " << ::fabs(v - v_) / v_); - CPPUNIT_ASSERT_DOUBLES_EQUAL(v_, v, 0.3 * v_); + CPPUNIT_ASSERT_DOUBLES_EQUAL(v_, v, 0.4 * v_); CPPUNIT_ASSERT(v_ > vv.first && v_ < vv.second); error.add(::fabs(v - v_) / v_); } LOG_DEBUG("mean relative error = " << maths::CBasicStatistics::mean(error)); - CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.1); + CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.11); } void CSeasonalComponentTest::testPersist(void) diff --git a/lib/maths/unittest/CTimeSeriesDecompositionTest.cc b/lib/maths/unittest/CTimeSeriesDecompositionTest.cc index b26a9fcf0c..aa16ae8fc3 100644 --- a/lib/maths/unittest/CTimeSeriesDecompositionTest.cc +++ b/lib/maths/unittest/CTimeSeriesDecompositionTest.cc @@ -86,8 +86,6 @@ void CTimeSeriesDecompositionTest::testSuperpositionOfSines(void) trend.push_back(weekly * daily); } - LOG_DEBUG(core::CContainerPrinter::print(trend.begin(), trend.begin() + 48)); - test::CRandomNumbers rng; TDoubleVec noise; rng.generateNormalSamples(0.0, 400.0, times.size(), noise); @@ -129,7 +127,7 @@ void CTimeSeriesDecompositionTest::testSuperpositionOfSines(void) for (core_t::TTime t = lastWeek; t < lastWeek + WEEK; t += HALF_HOUR) { - TDoubleDoublePr baseline = decomposition.baseline(t + WEEK, 70.0); + TDoubleDoublePr baseline = decomposition.baseline(t, 70.0); double residual = ::fabs(trend[t / HALF_HOUR] - mean(baseline)); sumResidual += residual; maxResidual = std::max(maxResidual, residual); @@ -147,9 +145,9 @@ void CTimeSeriesDecompositionTest::testSuperpositionOfSines(void) if (time >= 2 * WEEK) { - CPPUNIT_ASSERT(sumResidual < 0.049 * sumValue); - CPPUNIT_ASSERT(maxResidual < 0.061 * maxValue); - CPPUNIT_ASSERT(percentileError < 0.03 * sumValue); + CPPUNIT_ASSERT(sumResidual < 0.04 * sumValue); + CPPUNIT_ASSERT(maxResidual < 0.04 * maxValue); + CPPUNIT_ASSERT(percentileError < 0.02 * sumValue); totalSumResidual += sumResidual; totalMaxResidual += maxResidual; totalSumValue += sumValue; @@ -168,18 +166,18 @@ void CTimeSeriesDecompositionTest::testSuperpositionOfSines(void) //file << "fe = " << core::CContainerPrinter::print(f) << ";\n"; //file << "r = " << core::CContainerPrinter::print(r) << ";\n"; //file << "plot(t(1:length(fe)), fe, 'r');\n"; - //file << "scatter(t(1:length(r)), r, 15, 'k', 'x');\n"; + //file << "plot(t(1:length(r)), r, 'k');\n"; - CPPUNIT_ASSERT(totalSumResidual < 0.023 * totalSumValue); - CPPUNIT_ASSERT(totalMaxResidual < 0.028 * totalMaxValue); - CPPUNIT_ASSERT(totalPercentileError < 0.012 * totalSumValue); + CPPUNIT_ASSERT(totalSumResidual < 0.018 * totalSumValue); + CPPUNIT_ASSERT(totalMaxResidual < 0.021 * totalMaxValue); + CPPUNIT_ASSERT(totalPercentileError < 0.01 * totalSumValue); } -void CTimeSeriesDecompositionTest::testMinimizeComponents(void) +void CTimeSeriesDecompositionTest::testDistortedPeriodic(void) { - LOG_DEBUG("+----------------------------------------------+"); - LOG_DEBUG("| CTimeSeriesDecompositionTest::testMinimizeComponents |"); - LOG_DEBUG("+----------------------------------------------+"); + LOG_DEBUG("+-------------------------------------------------------+"); + LOG_DEBUG("| CTimeSeriesDecompositionTest::testDistortedPeriodic |"); + LOG_DEBUG("+-------------------------------------------------------+"); const core_t::TTime bucketLength = HOUR; const core_t::TTime startTime = 0; @@ -284,7 +282,6 @@ void CTimeSeriesDecompositionTest::testMinimizeComponents(void) //TDoubleVec t; //TDoubleVec f; //TDoubleVec fe; - //TDoubleVec r; double sumResidual = 0.0; double maxResidual = 0.0; @@ -297,7 +294,7 @@ void CTimeSeriesDecompositionTest::testMinimizeComponents(void) static_cast(tt / HOUR) < boost::size(timeseries); tt += HOUR) { - TDoubleDoublePr baseline = decomposition.baseline(tt + WEEK, 70.0); + TDoubleDoublePr baseline = decomposition.baseline(tt, 70.0); double residual = ::fabs(timeseries[tt / HOUR] - mean(baseline)); sumResidual += residual; @@ -310,7 +307,6 @@ void CTimeSeriesDecompositionTest::testMinimizeComponents(void) //t.push_back(tt); //f.push_back(timeseries[tt / HOUR]); //fe.push_back(mean(baseline)); - //r.push_back(mean(baseline) - timeseries[tt / HOUR]); } LOG_DEBUG("'sum residual' / 'sum value' = " << sumResidual / sumValue); @@ -319,9 +315,9 @@ void CTimeSeriesDecompositionTest::testMinimizeComponents(void) if (time >= 2 * WEEK) { - CPPUNIT_ASSERT(sumResidual < 0.27 * sumValue); + CPPUNIT_ASSERT(sumResidual < 0.30 * sumValue); CPPUNIT_ASSERT(maxResidual < 0.56 * maxValue); - CPPUNIT_ASSERT(percentileError < 0.19 * sumValue); + CPPUNIT_ASSERT(percentileError < 0.21 * sumValue); totalSumResidual += sumResidual; totalMaxResidual += maxResidual; @@ -333,10 +329,8 @@ void CTimeSeriesDecompositionTest::testMinimizeComponents(void) //file << "t = " << core::CContainerPrinter::print(t) << ";\n"; //file << "f = " << core::CContainerPrinter::print(f) << ";\n"; //file << "fe = " << core::CContainerPrinter::print(fe) << ";\n"; - //file << "r = " << core::CContainerPrinter::print(r) << ";\n"; //file << "plot(t, f);\n"; //file << "plot(t, fe, 'r');\n"; - //file << "scatter(t, r, 15, 'k', 'x');\n"; lastWeek += WEEK; } @@ -346,9 +340,9 @@ void CTimeSeriesDecompositionTest::testMinimizeComponents(void) LOG_DEBUG("total 'max residual' / 'max value' = " << totalMaxResidual / totalMaxValue); LOG_DEBUG("total 70% error = " << totalPercentileError / totalSumValue); - CPPUNIT_ASSERT(totalSumResidual < 0.18 * totalSumValue); - CPPUNIT_ASSERT(totalMaxResidual < 0.26 * totalMaxValue); - CPPUNIT_ASSERT(totalPercentileError < 0.12 * totalSumValue); + CPPUNIT_ASSERT(totalSumResidual < 0.17 * totalSumValue); + CPPUNIT_ASSERT(totalMaxResidual < 0.23 * totalMaxValue); + CPPUNIT_ASSERT(totalPercentileError < 0.03 * totalSumValue); } void CTimeSeriesDecompositionTest::testMinimizeLongComponents(void) @@ -414,7 +408,7 @@ void CTimeSeriesDecompositionTest::testMinimizeLongComponents(void) for (core_t::TTime t = lastWeek; t < lastWeek + WEEK; t += HALF_HOUR) { - TDoubleDoublePr baseline = decomposition.baseline(t + WEEK, 70.0); + TDoubleDoublePr baseline = decomposition.baseline(t, 70.0); double residual = ::fabs(trend[t / HALF_HOUR] - mean(baseline)); sumResidual += residual; @@ -434,9 +428,9 @@ void CTimeSeriesDecompositionTest::testMinimizeLongComponents(void) if (time >= 2 * WEEK) { - CPPUNIT_ASSERT(sumResidual < 0.15 * sumValue); - CPPUNIT_ASSERT(maxResidual < 0.45 * maxValue); - CPPUNIT_ASSERT(percentileError < 0.18 * sumValue); + CPPUNIT_ASSERT(sumResidual < 0.16 * sumValue); + CPPUNIT_ASSERT(maxResidual < 0.35 * maxValue); + CPPUNIT_ASSERT(percentileError < 0.05 * sumValue); totalSumResidual += sumResidual; totalMaxResidual += maxResidual; @@ -469,11 +463,11 @@ void CTimeSeriesDecompositionTest::testMinimizeLongComponents(void) //file << "fe = " << core::CContainerPrinter::print(f) << ";\n"; //file << "r = " << core::CContainerPrinter::print(r) << ";\n"; //file << "plot(t(1:length(fe)), fe, 'r');\n"; - //file << "scatter(t(1:length(r)), r, 15, 'k', 'x');\n"; + //file << "plot(t(1:length(r)), r, 'k');\n"; CPPUNIT_ASSERT(totalSumResidual < 0.06 * totalSumValue); - CPPUNIT_ASSERT(totalMaxResidual < 0.25 * totalMaxValue); - CPPUNIT_ASSERT(totalPercentileError < 0.04 * totalSumValue); + CPPUNIT_ASSERT(totalMaxResidual < 0.27 * totalMaxValue); + CPPUNIT_ASSERT(totalPercentileError < 0.03 * totalSumValue); meanSlope /= refinements; LOG_DEBUG("mean weekly |slope| = " << meanSlope); @@ -510,7 +504,7 @@ void CTimeSeriesDecompositionTest::testWeekend(void) //file.open("results.m"); //file << "hold on;\n"; //file << "t = " << core::CContainerPrinter::print(times) << ";\n"; - //file << "f = " << core::CContainerPrinter::print(timeseries) << ";\n"; + //file << "f = " << core::CContainerPrinter::print(trend) << ";\n"; //file << "plot(t, f);"; //TDoubleVec f; //TDoubleVec r; @@ -541,7 +535,7 @@ void CTimeSeriesDecompositionTest::testWeekend(void) for (core_t::TTime t = lastWeek; t < lastWeek + WEEK; t += HALF_HOUR) { - TDoubleDoublePr baseline = decomposition.baseline(t + WEEK, 70.0); + TDoubleDoublePr baseline = decomposition.baseline(t, 70.0); double residual = ::fabs(trend[t / HALF_HOUR] - mean(baseline)); sumResidual += residual; @@ -579,7 +573,7 @@ void CTimeSeriesDecompositionTest::testWeekend(void) //file << "fe = " << core::CContainerPrinter::print(f) << ";\n"; //file << "r = " << core::CContainerPrinter::print(r) << ";\n"; //file << "plot(t(1:length(fe)), fe, 'r');\n"; - //file << "scatter(t(1:length(r)), r, 15, 'k', 'x');\n"; + //file << "plot(t(1:length(r)), r, 'k');\n"; LOG_DEBUG("total 'sum residual' / 'sum value' = " << totalSumResidual / totalSumValue); LOG_DEBUG("total 'max residual' / 'max value' = " << totalMaxResidual / totalMaxValue); @@ -587,7 +581,7 @@ void CTimeSeriesDecompositionTest::testWeekend(void) CPPUNIT_ASSERT(totalSumResidual < 0.027 * totalSumValue); CPPUNIT_ASSERT(totalMaxResidual < 0.12 * totalMaxValue); - CPPUNIT_ASSERT(totalPercentileError < 0.015 * totalSumValue); + CPPUNIT_ASSERT(totalPercentileError < 0.012 * totalSumValue); } void CTimeSeriesDecompositionTest::testSinglePeriodicity(void) @@ -652,7 +646,7 @@ void CTimeSeriesDecompositionTest::testSinglePeriodicity(void) t < lastWeek + WEEK; t += HALF_HOUR) { - TDoubleDoublePr baseline = decomposition.baseline(t + WEEK, 70.0); + TDoubleDoublePr baseline = decomposition.baseline(t, 70.0); double residual = ::fabs(trend[t / HALF_HOUR] + noiseMean - mean(baseline)); sumResidual += residual; @@ -672,8 +666,8 @@ void CTimeSeriesDecompositionTest::testSinglePeriodicity(void) if (time >= 1 * WEEK) { - CPPUNIT_ASSERT(sumResidual < 0.023 * sumValue); - CPPUNIT_ASSERT(maxResidual < 0.046 * maxValue); + CPPUNIT_ASSERT(sumResidual < 0.06 * sumValue); + CPPUNIT_ASSERT(maxResidual < 0.08 * maxValue); CPPUNIT_ASSERT(percentileError < 0.02 * sumValue); totalSumResidual += sumResidual; @@ -696,8 +690,8 @@ void CTimeSeriesDecompositionTest::testSinglePeriodicity(void) LOG_DEBUG("total 'sum residual' / 'sum value' = " << totalSumResidual / totalSumValue); LOG_DEBUG("total 'max residual' / 'max value' = " << totalMaxResidual / totalMaxValue); LOG_DEBUG("total 70% error = " << totalPercentileError / totalSumValue); - CPPUNIT_ASSERT(totalSumResidual < 0.013 * totalSumValue); - CPPUNIT_ASSERT(totalMaxResidual < 0.037 * totalMaxValue); + CPPUNIT_ASSERT(totalSumResidual < 0.015 * totalSumValue); + CPPUNIT_ASSERT(totalMaxResidual < 0.042 * totalMaxValue); CPPUNIT_ASSERT(totalPercentileError < 0.01 * totalSumValue); // Check that only the daily component has been initialized. @@ -709,7 +703,7 @@ void CTimeSeriesDecompositionTest::testSinglePeriodicity(void) //file << "fe = " << core::CContainerPrinter::print(f) << ";\n"; //file << "r = " << core::CContainerPrinter::print(r) << ";\n"; //file << "plot(t(1:length(fe)), fe, 'r');\n"; - //file << "scatter(t(1:length(r)), r, 15, 'k', 'x');\n"; + //file << "plot(t(1:length(r)), r, 'k');\n"; } void CTimeSeriesDecompositionTest::testSeasonalOnset(void) @@ -784,7 +778,7 @@ void CTimeSeriesDecompositionTest::testSeasonalOnset(void) double percentileError = 0.0; for (core_t::TTime t = lastWeek; t < lastWeek + WEEK; t += HOUR) { - TDoubleDoublePr baseline = decomposition.baseline(t + WEEK, 70.0); + TDoubleDoublePr baseline = decomposition.baseline(t, 70.0); double residual = ::fabs(trend[t / HOUR] - mean(baseline)); sumResidual += residual; @@ -810,17 +804,18 @@ void CTimeSeriesDecompositionTest::testSeasonalOnset(void) totalPercentileError += percentileError; const TSeasonalComponentVec &components = decomposition.seasonalComponents(); - if (time > 14 * WEEK) + if (time > 11 * WEEK) { // Check that both components have been initialized. - CPPUNIT_ASSERT(components.size() > 1); + CPPUNIT_ASSERT(components.size() > 2); CPPUNIT_ASSERT(components[0].initialized()); CPPUNIT_ASSERT(components[1].initialized()); + CPPUNIT_ASSERT(components[2].initialized()); } - else if (time > 12 * WEEK) + else if (time > 10 * WEEK) { // Check that both components have been initialized. - CPPUNIT_ASSERT(components.size() == 1); + CPPUNIT_ASSERT_EQUAL(std::size_t(1), components.size()); CPPUNIT_ASSERT(components[0].initialized()); } else @@ -835,14 +830,14 @@ void CTimeSeriesDecompositionTest::testSeasonalOnset(void) //file << "fe = " << core::CContainerPrinter::print(f) << ";\n"; //file << "r = " << core::CContainerPrinter::print(r) << ";\n"; //file << "plot(t(1:length(fe)), fe);\n"; - //file << "scatter(t(1:length(r)), r, 15, 'k', 'x');\n"; + //file << "plot(t(1:length(r)), r, 'k');\n"; LOG_DEBUG("total 'sum residual' / 'sum value' = " << totalSumResidual / totalSumValue); LOG_DEBUG("total 'max residual' / 'max value' = " << totalMaxResidual / totalMaxValue); LOG_DEBUG("total 70% error = " << totalPercentileError / totalSumValue); - CPPUNIT_ASSERT(totalSumResidual < 0.09 * totalSumValue); - CPPUNIT_ASSERT(totalMaxResidual < 0.13 * totalMaxValue); - CPPUNIT_ASSERT(totalPercentileError < 0.06 * totalSumValue); + CPPUNIT_ASSERT(totalSumResidual < 0.07 * totalSumValue); + CPPUNIT_ASSERT(totalMaxResidual < 0.09 * totalMaxValue); + CPPUNIT_ASSERT(totalPercentileError < 0.03 * totalSumValue); } void CTimeSeriesDecompositionTest::testVarianceScale(void) @@ -905,8 +900,8 @@ void CTimeSeriesDecompositionTest::testVarianceScale(void) LOG_DEBUG("mean error = " << maths::CBasicStatistics::mean(error)); LOG_DEBUG("mean 70% error = " << maths::CBasicStatistics::mean(percentileError)) LOG_DEBUG("mean scale = " << maths::CBasicStatistics::mean(meanScale)); - CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.27); - CPPUNIT_ASSERT(maths::CBasicStatistics::mean(percentileError) < 0.055); + CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.29); + CPPUNIT_ASSERT(maths::CBasicStatistics::mean(percentileError) < 0.05); CPPUNIT_ASSERT_DOUBLES_EQUAL(1.0, maths::CBasicStatistics::mean(meanScale), 0.04); } LOG_DEBUG("Smoothly Varying Variance"); @@ -960,8 +955,8 @@ void CTimeSeriesDecompositionTest::testVarianceScale(void) LOG_DEBUG("mean error = " << maths::CBasicStatistics::mean(error)); LOG_DEBUG("mean 70% error = " << maths::CBasicStatistics::mean(percentileError)); LOG_DEBUG("mean scale = " << maths::CBasicStatistics::mean(meanScale)); - CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.27); - CPPUNIT_ASSERT(maths::CBasicStatistics::mean(percentileError) < 0.13); + CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.22); + CPPUNIT_ASSERT(maths::CBasicStatistics::mean(percentileError) < 0.1); CPPUNIT_ASSERT_DOUBLES_EQUAL(1.0, maths::CBasicStatistics::mean(meanScale), 0.01); } LOG_DEBUG("Long Term Trend"); @@ -1008,9 +1003,9 @@ void CTimeSeriesDecompositionTest::testVarianceScale(void) void CTimeSeriesDecompositionTest::testSpikeyDataProblemCase(void) { - LOG_DEBUG("+----------------------------------------------------+"); + LOG_DEBUG("+-----------------------------------------------------------+"); LOG_DEBUG("| CTimeSeriesDecompositionTest::testSpikeyDataProblemCase |"); - LOG_DEBUG("+----------------------------------------------------+"); + LOG_DEBUG("+-----------------------------------------------------------+"); TTimeDoublePrVec timeseries; core_t::TTime startTime; @@ -1102,7 +1097,7 @@ void CTimeSeriesDecompositionTest::testSpikeyDataProblemCase(void) LOG_DEBUG("total 'max residual' / 'max value' = " << totalMaxResidual / totalMaxValue); LOG_DEBUG("total 70% error = " << totalPercentileError / totalSumValue); - CPPUNIT_ASSERT(totalSumResidual < 0.2 * totalSumValue); + CPPUNIT_ASSERT(totalSumResidual < 0.19 * totalSumValue); CPPUNIT_ASSERT(totalMaxResidual < 0.33 * totalMaxValue); CPPUNIT_ASSERT(totalPercentileError < 0.14 * totalSumValue); @@ -1166,9 +1161,9 @@ void CTimeSeriesDecompositionTest::testSpikeyDataProblemCase(void) void CTimeSeriesDecompositionTest::testDiurnalProblemCase(void) { - LOG_DEBUG("+---------------------------------------------------------+"); + LOG_DEBUG("+--------------------------------------------------------+"); LOG_DEBUG("| CTimeSeriesDecompositionTest::testDiurnalProblemCase |"); - LOG_DEBUG("+---------------------------------------------------------+"); + LOG_DEBUG("+--------------------------------------------------------+"); TTimeDoublePrVec timeseries; core_t::TTime startTime; @@ -1264,9 +1259,9 @@ void CTimeSeriesDecompositionTest::testDiurnalProblemCase(void) << totalMaxResidual / totalMaxValue); LOG_DEBUG("total 70% error = " << totalPercentileError / totalSumValue); - CPPUNIT_ASSERT(totalSumResidual < 0.28 * totalSumValue); - CPPUNIT_ASSERT(totalMaxResidual < 0.71 * totalMaxValue); - CPPUNIT_ASSERT(totalPercentileError < 0.18 * totalSumValue); + CPPUNIT_ASSERT(totalSumResidual < 0.27 * totalSumValue); + CPPUNIT_ASSERT(totalMaxResidual < 0.72 * totalMaxValue); + CPPUNIT_ASSERT(totalPercentileError < 0.16 * totalSumValue); //file << "hold on;\n"; //file << "t = " << core::CContainerPrinter::print(times) << ";\n"; @@ -1275,7 +1270,7 @@ void CTimeSeriesDecompositionTest::testDiurnalProblemCase(void) //file << "fe = " << core::CContainerPrinter::print(f) << ";\n"; //file << "r = " << core::CContainerPrinter::print(r) << ";\n"; //file << "plot(t(1:length(fe)), fe);\n"; - //file << "scatter(t(1:length(r)), r, 15, 'k', 'x');\n"; + //file << "plot(t(1:length(r)), r, 'k');\n"; TMeanAccumulator scale; double variance = decomposition.meanVariance(); @@ -1289,11 +1284,11 @@ void CTimeSeriesDecompositionTest::testDiurnalProblemCase(void) CPPUNIT_ASSERT_DOUBLES_EQUAL(1.0, maths::CBasicStatistics::mean(scale), 0.07); } -void CTimeSeriesDecompositionTest::testThirtyMinuteSamplingProblemCase(void) +void CTimeSeriesDecompositionTest::testComplexDiurnalProblemCase(void) { - LOG_DEBUG("+------------------------------------------------------+"); - LOG_DEBUG("| CTimeSeriesDecompositionTest::testThirtyMinuteSamplingProblemCase |"); - LOG_DEBUG("+------------------------------------------------------+"); + LOG_DEBUG("+---------------------------------------------------------------+"); + LOG_DEBUG("| CTimeSeriesDecompositionTest::testComplexDiurnalProblemCase |"); + LOG_DEBUG("+---------------------------------------------------------------+"); TTimeDoublePrVec timeseries; core_t::TTime startTime; @@ -1390,9 +1385,9 @@ void CTimeSeriesDecompositionTest::testThirtyMinuteSamplingProblemCase(void) LOG_DEBUG("total 'max residual' / 'max value' = " << totalMaxResidual / totalMaxValue); LOG_DEBUG("total 70% error = " << totalPercentileError / totalSumValue); - CPPUNIT_ASSERT(totalSumResidual < 0.15 * totalSumValue); - CPPUNIT_ASSERT(totalMaxResidual < 0.40 * totalMaxValue); - CPPUNIT_ASSERT(totalPercentileError < 0.06 * totalSumValue); + CPPUNIT_ASSERT(totalSumResidual < 0.18 * totalSumValue); + CPPUNIT_ASSERT(totalMaxResidual < 0.42 * totalMaxValue); + CPPUNIT_ASSERT(totalPercentileError < 0.08 * totalSumValue); //file << "hold on;\n"; //file << "t = " << core::CContainerPrinter::print(times) << ";\n"; @@ -1401,7 +1396,7 @@ void CTimeSeriesDecompositionTest::testThirtyMinuteSamplingProblemCase(void) //file << "fe = " << core::CContainerPrinter::print(f) << ";\n"; //file << "r = " << core::CContainerPrinter::print(r) << ";\n"; //file << "plot(t(1:length(fe)), fe);\n"; - //file << "scatter(t(1:length(r)), r, 15, 'k', 'x');\n"; + //file << "plot(t(1:length(r)), r, 'k');\n"; } void CTimeSeriesDecompositionTest::testDiurnalPeriodicityWithMissingValues(void) @@ -1451,7 +1446,7 @@ void CTimeSeriesDecompositionTest::testDiurnalPeriodicityWithMissingValues(void) } LOG_DEBUG("mean error = " << maths::CBasicStatistics::mean(error)); - CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.092); + CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.1); //file << "hold on;\n"; //file << "t = " << core::CContainerPrinter::print(times) << ";\n"; @@ -1510,7 +1505,7 @@ void CTimeSeriesDecompositionTest::testDiurnalPeriodicityWithMissingValues(void) } LOG_DEBUG("mean error = " << maths::CBasicStatistics::mean(error)) - CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.11); + CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.1); //file << "hold on;\n"; //file << "t = " << core::CContainerPrinter::print(times) << ";\n"; @@ -1546,7 +1541,7 @@ void CTimeSeriesDecompositionTest::testLongTermTrend(void) for (core_t::TTime time = 0; time < length; time += HALF_HOUR) { times.push_back(time); - trend.push_back(static_cast(time) / static_cast(DAY)); + trend.push_back(5.0 + static_cast(time) / static_cast(DAY)); } maths::CTimeSeriesDecomposition decomposition(0.024, HALF_HOUR); @@ -1595,8 +1590,8 @@ void CTimeSeriesDecompositionTest::testLongTermTrend(void) totalSumValue += sumValue; totalMaxValue += maxValue; - CPPUNIT_ASSERT(sumResidual / sumValue < 0.06); - CPPUNIT_ASSERT(maxResidual / maxValue < 0.21); + CPPUNIT_ASSERT(sumResidual / sumValue < 0.05); + CPPUNIT_ASSERT(maxResidual / maxValue < 0.05); } lastDay += DAY; } @@ -1614,8 +1609,8 @@ void CTimeSeriesDecompositionTest::testLongTermTrend(void) //file << "plot(t, f, 'r');\n"; //file << "plot(t, fe);\n"; - CPPUNIT_ASSERT(totalSumResidual / totalSumValue < 0.013); - CPPUNIT_ASSERT(totalMaxResidual / totalMaxValue < 0.04); + CPPUNIT_ASSERT(totalSumResidual / totalSumValue < 0.01); + CPPUNIT_ASSERT(totalMaxResidual / totalMaxValue < 0.01); } LOG_DEBUG("Saw Tooth Not Periodic"); @@ -1702,8 +1697,8 @@ void CTimeSeriesDecompositionTest::testLongTermTrend(void) //file << "plot(t, f, 'r');\n"; //file << "plot(t, fe);\n"; - CPPUNIT_ASSERT(totalSumResidual / totalSumValue < 0.39); - CPPUNIT_ASSERT(totalMaxResidual / totalMaxValue < 0.46); + CPPUNIT_ASSERT(totalSumResidual / totalSumValue < 0.38); + CPPUNIT_ASSERT(totalMaxResidual / totalMaxValue < 0.42); } } @@ -1715,11 +1710,10 @@ void CTimeSeriesDecompositionTest::testLongTermTrendAndPeriodicity(void) // Test long term mean reverting component plus daily periodic component. - const core_t::TTime length = 120 * DAY; TTimeVec times; TDoubleVec trend; - + const core_t::TTime length = 120 * DAY; for (core_t::TTime time = 0; time < length; time += HALF_HOUR) { times.push_back(time); @@ -1732,7 +1726,6 @@ void CTimeSeriesDecompositionTest::testLongTermTrendAndPeriodicity(void) } test::CRandomNumbers rng; - TDoubleVec noise; rng.generateNormalSamples(0.0, 4.0, times.size(), noise); @@ -1787,8 +1780,8 @@ void CTimeSeriesDecompositionTest::testLongTermTrendAndPeriodicity(void) totalSumValue += sumValue; totalMaxValue += maxValue; - CPPUNIT_ASSERT(sumResidual / sumValue < 0.5); - CPPUNIT_ASSERT(maxResidual / maxValue < 0.5); + CPPUNIT_ASSERT(sumResidual / sumValue < 0.4); + CPPUNIT_ASSERT(maxResidual / maxValue < 0.4); } lastDay += DAY; } @@ -1805,8 +1798,8 @@ void CTimeSeriesDecompositionTest::testLongTermTrendAndPeriodicity(void) //file << "plot(t, f, 'r');\n"; //file << "plot(t, fe);\n"; - CPPUNIT_ASSERT(totalSumResidual / totalSumValue < 0.053); - CPPUNIT_ASSERT(totalMaxResidual / totalMaxValue < 0.069); + CPPUNIT_ASSERT(totalSumResidual / totalSumValue < 0.04); + CPPUNIT_ASSERT(totalMaxResidual / totalMaxValue < 0.05); } void CTimeSeriesDecompositionTest::testNonDiurnal(void) @@ -1825,7 +1818,6 @@ void CTimeSeriesDecompositionTest::testNonDiurnal(void) TTimeVec times; TDoubleVec trends[2]{ TDoubleVec(), TDoubleVec(8 * DAY / FIVE_MINS) }; - for (core_t::TTime time = 0; time < length; time += FIVE_MINS) { times.push_back(time); @@ -1836,7 +1828,8 @@ void CTimeSeriesDecompositionTest::testNonDiurnal(void) TDoubleVec noise; rng.generateNormalSamples(0.0, 1.0, trends[1].size(), noise); - TDoubleVec thresholds[]{ TDoubleVec{0.06, 0.06}, TDoubleVec{0.22, 0.2} }; + core_t::TTime startTesting[]{3 * HOUR, 16 * DAY}; + TDoubleVec thresholds[]{ TDoubleVec{0.07, 0.06}, TDoubleVec{0.18, 0.13} }; for (std::size_t t = 0u; t < 2; ++t) { @@ -1861,7 +1854,7 @@ void CTimeSeriesDecompositionTest::testNonDiurnal(void) { LOG_DEBUG("Processing hour " << times[i] / HOUR); - if (decomposition.initialized()) + if (times[i] > startTesting[t]) { double sumResidual = 0.0; double maxResidual = 0.0; @@ -1891,8 +1884,8 @@ void CTimeSeriesDecompositionTest::testNonDiurnal(void) totalSumValue += sumValue; totalMaxValue += maxValue; - CPPUNIT_ASSERT(sumResidual / sumValue < 0.53); - CPPUNIT_ASSERT(maxResidual / maxValue < 0.51); + CPPUNIT_ASSERT(sumResidual / sumValue < 0.33); + CPPUNIT_ASSERT(maxResidual / maxValue < 0.28); } lastHour += HOUR; } @@ -1922,7 +1915,6 @@ void CTimeSeriesDecompositionTest::testNonDiurnal(void) TTimeVec times; TDoubleVec trend; - for (core_t::TTime time = 0; time < length; time += TEN_MINS) { times.push_back(time); @@ -1937,6 +1929,7 @@ void CTimeSeriesDecompositionTest::testNonDiurnal(void) //TDoubleVec f; //TDoubleVec values; + core_t::TTime startTesting{14 * DAY}; maths::CTimeSeriesDecomposition decomposition(0.01, TEN_MINS); double totalSumResidual = 0.0; @@ -1953,7 +1946,7 @@ void CTimeSeriesDecompositionTest::testNonDiurnal(void) { LOG_DEBUG("Processing two days " << times[i] / 2 * DAY); - if (decomposition.initialized()) + if (times[i] > startTesting) { double sumResidual = 0.0; double maxResidual = 0.0; @@ -2001,7 +1994,7 @@ void CTimeSeriesDecompositionTest::testNonDiurnal(void) //file << "plot(t, f, 'r');\n"; //file << "plot(t, fe);\n"; - CPPUNIT_ASSERT(totalSumResidual / totalSumValue < 0.09); + CPPUNIT_ASSERT(totalSumResidual / totalSumValue < 0.1); CPPUNIT_ASSERT(totalMaxResidual / totalMaxValue < 0.18); } } @@ -2022,7 +2015,7 @@ void CTimeSeriesDecompositionTest::testYearly(void) | maths::CDecayRateController::E_PredictionErrorIncrease, 1); TDoubleVec noise; core_t::TTime time = 0; - for (/**/; time < 5 * YEAR; time += 4 * HOUR) + for (/**/; time < 4 * YEAR; time += 4 * HOUR) { double trend = 15.0 * (2.0 + ::sin( boost::math::double_constants::two_pi * static_cast(time) @@ -2042,15 +2035,15 @@ void CTimeSeriesDecompositionTest::testYearly(void) } } - //std::ofstream file; - //file.open("results.m"); - //TDoubleVec f; - //TTimeVec times; - //TDoubleVec values; + std::ofstream file; + file.open("results.m"); + TDoubleVec f; + TTimeVec times; + TDoubleVec values; // Predict over one year and check we get reasonable accuracy. TMeanAccumulator meanError; - for (/**/; time < 6 * YEAR; time += 4 * HOUR) + for (/**/; time < 5 * YEAR; time += 4 * HOUR) { double trend = 15.0 * (2.0 + ::sin( boost::math::double_constants::two_pi * static_cast(time) @@ -2061,14 +2054,14 @@ void CTimeSeriesDecompositionTest::testYearly(void) double prediction = maths::CBasicStatistics::mean(decomposition.baseline(time, 0.0)); double error = ::fabs((prediction - trend) / trend); meanError.add(error); - //times.push_back(time); - //values.push_back(trend); - //f.push_back(prediction); + times.push_back(time); + values.push_back(trend); + f.push_back(prediction); if (time / HOUR % 40 == 0) { LOG_DEBUG("error = " << error); } - CPPUNIT_ASSERT(error < 0.6); + CPPUNIT_ASSERT(error < 0.1); } //file << "t = " << core::CContainerPrinter::print(times) << ";\n"; @@ -2078,7 +2071,7 @@ void CTimeSeriesDecompositionTest::testYearly(void) //file << "plot(t, fe);\n"; LOG_DEBUG("mean error = " << maths::CBasicStatistics::mean(meanError)); - CPPUNIT_ASSERT(maths::CBasicStatistics::mean(meanError) < 0.15); + CPPUNIT_ASSERT(maths::CBasicStatistics::mean(meanError) < 0.02); } void CTimeSeriesDecompositionTest::testCalendar(void) @@ -2099,11 +2092,13 @@ void CTimeSeriesDecompositionTest::testCalendar(void) 18230400, // Fri 31st Jul 18316800 }; core_t::TTime end = months.back(); - TDoubleVec errors{ 5.0, 10.0, 25.0, 22.0, 19.0, 28.0, 15.0, 8.0, 1.0 }; + TDoubleVec errors{ 5.0, 15.0, 35.0, 32.0, 25.0, 36.0, 22.0, 12.0, 3.0 }; auto trend = [&months, &errors](core_t::TTime t) { - double result = 10.0 + 5.0 * ::sin(boost::math::double_constants::two_pi * t / DAY); + double result = 20.0 + 10.0 * ::sin( boost::math::double_constants::two_pi + * static_cast(t) + / static_cast(DAY)); auto i = std::lower_bound(months.begin(), months.end(), t - DAY); if ( t >= *i + 7200 && t < *i + 7200 + static_cast(errors.size()) * HALF_HOUR) @@ -2151,8 +2146,8 @@ void CTimeSeriesDecompositionTest::testCalendar(void) } LOG_DEBUG("large error count = " << largeErrorCount); - CPPUNIT_ASSERT(++count > 4 || largeErrorCount > 10); - CPPUNIT_ASSERT( count < 5 || largeErrorCount < 6); + CPPUNIT_ASSERT(++count > 4 || largeErrorCount > 15); + CPPUNIT_ASSERT( count < 5 || largeErrorCount <= 5); } //times.push_back(time); @@ -2175,7 +2170,7 @@ void CTimeSeriesDecompositionTest::testConditionOfTrend(void) auto trend = [](core_t::TTime time) { - return std::pow(static_cast(time) / static_cast(WEEK), 3.0); + return std::pow(static_cast(time) / static_cast(WEEK), 2.0); }; const core_t::TTime bucketLength = 6 * HOUR; @@ -2186,11 +2181,11 @@ void CTimeSeriesDecompositionTest::testConditionOfTrend(void) TDoubleVec noise; for (core_t::TTime time = 0; time < 10 * YEAR; time += 6 * HOUR) { - rng.generateNormalSamples(0.0, 1.0, 1, noise); + rng.generateNormalSamples(0.0, 3.0, 1, noise); decomposition.addPoint(time, trend(time) + noise[0]); if (time > 10 * WEEK) { - CPPUNIT_ASSERT(::fabs(decomposition.detrend(time, trend(time), 0.0)) < 1.0); + CPPUNIT_ASSERT(::fabs(decomposition.detrend(time, trend(time), 0.0)) < 3.0); } } } @@ -2251,109 +2246,231 @@ void CTimeSeriesDecompositionTest::testPersist(void) const double decayRate = 0.01; const core_t::TTime bucketLength = HALF_HOUR; + TTimeVec times; + TDoubleVec trend; + for (core_t::TTime time = 0; time < 10 * WEEK + 1; time += HALF_HOUR) { - // First with some known periodic data - TTimeVec times; - TDoubleVec trend; - for (core_t::TTime time = 0; time < 10 * WEEK + 1; time += HALF_HOUR) - { - double daily = 15.0 + 10.0 * ::sin(boost::math::double_constants::two_pi - * static_cast(time) - / static_cast(DAY)); - times.push_back(time); - trend.push_back(daily); - } + double daily = 15.0 + 10.0 * ::sin(boost::math::double_constants::two_pi + * static_cast(time) + / static_cast(DAY)); + times.push_back(time); + trend.push_back(daily); + } - test::CRandomNumbers rng; - TDoubleVec noise; - rng.generateNormalSamples(20.0, 16.0, times.size(), noise); + test::CRandomNumbers rng; + TDoubleVec noise; + rng.generateNormalSamples(20.0, 16.0, times.size(), noise); - maths::CTimeSeriesDecomposition origDecomposition(decayRate, bucketLength); + maths::CTimeSeriesDecomposition origDecomposition(decayRate, bucketLength); - for (std::size_t i = 0u; i < times.size(); ++i) - { - origDecomposition.addPoint(times[i], trend[i] + noise[i]); - } + for (std::size_t i = 0u; i < times.size(); ++i) + { + origDecomposition.addPoint(times[i], trend[i] + noise[i]); + } - std::string origXml; + std::string origXml; + { + ml::core::CRapidXmlStatePersistInserter inserter("root"); + origDecomposition.acceptPersistInserter(inserter); + inserter.toXml(origXml); + } + + LOG_TRACE("Decomposition XML representation:\n" << origXml); + + // Restore the XML into a new decomposition + core::CRapidXmlParser parser; + CPPUNIT_ASSERT(parser.parseStringIgnoreCdata(origXml)); + core::CRapidXmlStateRestoreTraverser traverser(parser); + + maths::CTimeSeriesDecomposition restoredDecomposition(decayRate + 0.1, + bucketLength, + maths::CTimeSeriesDecomposition::DEFAULT_COMPONENT_SIZE, + traverser); + + std::string newXml; + { + core::CRapidXmlStatePersistInserter inserter("root"); + restoredDecomposition.acceptPersistInserter(inserter); + inserter.toXml(newXml); + } + CPPUNIT_ASSERT_EQUAL(origXml, newXml); +} + +void CTimeSeriesDecompositionTest::testUpgrade(void) +{ + LOG_DEBUG("+---------------------------------------------+"); + LOG_DEBUG("| CTimeSeriesDecompositionTest::testUpgrade |"); + LOG_DEBUG("+---------------------------------------------+"); + + // Check we can validly upgrade existing state. + + using TStrVec = std::vector; + auto load = [](const std::string &name, std::string &result) { - ml::core::CRapidXmlStatePersistInserter inserter("root"); - origDecomposition.acceptPersistInserter(inserter); - inserter.toXml(origXml); - } + std::ifstream file; + file.open(name); + std::stringbuf buf; + file >> &buf; + result = buf.str(); + }; + auto stringToPair = [](const std::string &str) + { + double first; + double second; + std::size_t n{str.find(",")}; + CPPUNIT_ASSERT(n != std::string::npos); + core::CStringUtils::stringToType(str.substr(0, n), first); + core::CStringUtils::stringToType(str.substr(n + 1), second); + return TDoubleDoublePr{first, second}; + }; - LOG_DEBUG("decomposition XML representation:\n" << origXml); + std::string empty; + + LOG_DEBUG("*** Seasonal and Calendar Components ***"); + { + std::string xml; + load("testfiles/CTimeSeriesDecomposition.6.2.seasonal.state.xml", xml); + LOG_DEBUG("Saved state size = " << xml.size()); + + std::string values; + load("testfiles/CTimeSeriesDecomposition.6.2.seasonal.expected_values.txt", values); + LOG_DEBUG("Expected values size = " << values.size()); + TStrVec expectedValues; + core::CStringUtils::tokenise(";", values, expectedValues, empty); + + std::string scales; + load("testfiles/CTimeSeriesDecomposition.6.2.seasonal.expected_scales.txt", scales); + LOG_DEBUG("Expected scales size = " << scales.size()); + TStrVec expectedScales; + core::CStringUtils::tokenise(";", scales, expectedScales, empty); + + CPPUNIT_ASSERT_EQUAL(expectedValues.size(), expectedScales.size()); - // Restore the XML into a new filter core::CRapidXmlParser parser; - CPPUNIT_ASSERT(parser.parseStringIgnoreCdata(origXml)); + CPPUNIT_ASSERT(parser.parseStringIgnoreCdata(xml)); core::CRapidXmlStateRestoreTraverser traverser(parser); - maths::CTimeSeriesDecomposition restoredDecomposition(decayRate + 0.1, - bucketLength, - maths::CTimeSeriesDecomposition::DEFAULT_COMPONENT_SIZE, - traverser); + maths::CTimeSeriesDecomposition decomposition(0.1, HALF_HOUR, + maths::CTimeSeriesDecomposition::DEFAULT_COMPONENT_SIZE, + traverser); - std::string newXml; - { - core::CRapidXmlStatePersistInserter inserter("root"); - restoredDecomposition.acceptPersistInserter(inserter); - inserter.toXml(newXml); - } - CPPUNIT_ASSERT_EQUAL(origXml, newXml); - } - { - // Then with non-periodic data that drops down to the - // small CRandomizedPeriodic test - TTimeVec times; - TDoubleVec trend; - for (core_t::TTime time = 0; time < 10 * WEEK + 1; time += HALF_HOUR) - { - double daily = 15.0; - times.push_back(time); - trend.push_back(daily); - } + // Check that the decay rates match and the values and variances + // predictions match the values obtained from 6.2. - test::CRandomNumbers rng; - TDoubleVec noise; - rng.generateNormalSamples(20.0, 16.0, times.size(), noise); + CPPUNIT_ASSERT_EQUAL(0.01, decomposition.decayRate()); - maths::CTimeSeriesDecomposition origDecomposition(decayRate, bucketLength); + double meanValue{decomposition.mean(60480000)}; + double meanVariance{decomposition.meanVariance()}; + LOG_DEBUG("restored mean value = " << meanValue); + LOG_DEBUG("restored mean variance = " << meanVariance); + CPPUNIT_ASSERT_DOUBLES_EQUAL(5994.36, meanValue, 0.005); + CPPUNIT_ASSERT_DOUBLES_EQUAL(286374.0, meanVariance, 0.5); - for (std::size_t i = 0u; i < times.size(); ++i) + for (core_t::TTime time = 60480000, i = 0; + i < static_cast(expectedValues.size()); + time += HALF_HOUR, ++i) { - origDecomposition.addPoint(times[i], trend[i] + noise[i]); + TDoubleDoublePr expectedValue{stringToPair(expectedValues[i])}; + TDoubleDoublePr expectedScale{stringToPair(expectedScales[i])}; + TDoubleDoublePr value{decomposition.baseline(time, 10.0)}; + TDoubleDoublePr scale{decomposition.scale(time, 286374.0, 10.0)}; + CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedValue.first, + value.first, + 0.005 * std::fabs(expectedValue.first)); + CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedValue.second, + value.second, + 0.005 * std::fabs(expectedValue.second)); + CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedScale.first, + scale.first, + 0.005 * expectedScale.first); + CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedScale.second, + scale.second, + 0.005 * std::max(expectedScale.second, 0.4)); } + } - std::string origXml; - { - ml::core::CRapidXmlStatePersistInserter inserter("root"); - origDecomposition.acceptPersistInserter(inserter); - inserter.toXml(origXml); - } + LOG_DEBUG("*** Trend and Seasonal Components ***"); + { + std::string xml; + load("testfiles/CTimeSeriesDecomposition.6.2.trend_and_seasonal.state.xml", xml); + LOG_DEBUG("Saved state size = " << xml.size()); + + std::string values; + load("testfiles/CTimeSeriesDecomposition.6.2.trend_and_seasonal.expected_values.txt", values); + LOG_DEBUG("Expected values size = " << values.size()); + TStrVec expectedValues; + core::CStringUtils::tokenise(";", values, expectedValues, empty); - LOG_DEBUG("decomposition XML representation:\n" << origXml); + std::string scales; + load("testfiles/CTimeSeriesDecomposition.6.2.trend_and_seasonal.expected_scales.txt", scales); + LOG_DEBUG("Expected scales size = " << scales.size()); + TStrVec expectedScales; + core::CStringUtils::tokenise(";", scales, expectedScales, empty); + + CPPUNIT_ASSERT_EQUAL(expectedValues.size(), expectedScales.size()); - // Restore the XML into a new filter core::CRapidXmlParser parser; - CPPUNIT_ASSERT(parser.parseStringIgnoreCdata(origXml)); + CPPUNIT_ASSERT(parser.parseStringIgnoreCdata(xml)); core::CRapidXmlStateRestoreTraverser traverser(parser); - maths::CTimeSeriesDecomposition restoredDecomposition(decayRate + 0.1, - bucketLength, - maths::CTimeSeriesDecomposition::DEFAULT_COMPONENT_SIZE, - traverser); - - std::string newXml; + maths::CTimeSeriesDecomposition decomposition(0.1, HALF_HOUR, + maths::CTimeSeriesDecomposition::DEFAULT_COMPONENT_SIZE, + traverser); + + // Check that the decay rates match and the values and variances + // predictions are close to the values obtained from 6.2. We can't + // update the state exactly in this case so the tolerances in this + // test are significantly larger. + + CPPUNIT_ASSERT_EQUAL(0.024, decomposition.decayRate()); + + double meanValue{decomposition.mean(10366200)}; + double meanVariance{decomposition.meanVariance()}; + LOG_DEBUG("restored mean value = " << meanValue); + LOG_DEBUG("restored mean variance = " << meanVariance); + CPPUNIT_ASSERT_DOUBLES_EQUAL(133.207, meanValue, 4.0); + CPPUNIT_ASSERT_DOUBLES_EQUAL(96.1654, meanVariance, 4.0); + + TMeanAccumulator meanValueError; + TMeanAccumulator meanScaleError; + for (core_t::TTime time = 10366200, i = 0; + i < static_cast(expectedValues.size()); + time += HALF_HOUR, ++i) { - core::CRapidXmlStatePersistInserter inserter("root"); - restoredDecomposition.acceptPersistInserter(inserter); - inserter.toXml(newXml); + TDoubleDoublePr expectedValue{stringToPair(expectedValues[i])}; + TDoubleDoublePr expectedScale{stringToPair(expectedScales[i])}; + TDoubleDoublePr value{decomposition.baseline(time, 10.0)}; + TDoubleDoublePr scale{decomposition.scale(time, 96.1654, 10.0)}; + CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedValue.first, + value.first, + 0.1 * std::fabs(expectedValue.first)); + CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedValue.second, + value.second, + 0.1 * std::fabs(expectedValue.second)); + CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedScale.first, + scale.first, + 0.3 * expectedScale.first); + CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedScale.second, + scale.second, + 0.3 * expectedScale.second); + meanValueError.add( std::fabs(expectedValue.first - value.first) + / std::fabs(expectedValue.first)); + meanValueError.add( std::fabs(expectedValue.second - value.second) + / std::fabs(expectedValue.second)); + meanScaleError.add( std::fabs(expectedScale.first - scale.first) + / expectedScale.first); + meanScaleError.add( std::fabs(expectedScale.second - scale.second) + / expectedScale.second); } - CPPUNIT_ASSERT_EQUAL(origXml, newXml); + + LOG_DEBUG("Mean value error = " << maths::CBasicStatistics::mean(meanValueError)); + LOG_DEBUG("Mean scale error = " << maths::CBasicStatistics::mean(meanScaleError)); + CPPUNIT_ASSERT(maths::CBasicStatistics::mean(meanValueError) < 0.06); + CPPUNIT_ASSERT(maths::CBasicStatistics::mean(meanScaleError) < 0.07); } } + CppUnit::Test *CTimeSeriesDecompositionTest::suite(void) { CppUnit::TestSuite *suiteOfTests = new CppUnit::TestSuite("CTimeSeriesDecompositionTest"); @@ -2362,8 +2479,8 @@ CppUnit::Test *CTimeSeriesDecompositionTest::suite(void) "CTimeSeriesDecompositionTest::testSuperpositionOfSines", &CTimeSeriesDecompositionTest::testSuperpositionOfSines) ); suiteOfTests->addTest( new CppUnit::TestCaller( - "CTimeSeriesDecompositionTest::testMinimizeComponents", - &CTimeSeriesDecompositionTest::testMinimizeComponents) ); + "CTimeSeriesDecompositionTest::testDistortedPeriodic", + &CTimeSeriesDecompositionTest::testDistortedPeriodic) ); suiteOfTests->addTest( new CppUnit::TestCaller( "CTimeSeriesDecompositionTest::testMinimizeLongComponents", &CTimeSeriesDecompositionTest::testMinimizeLongComponents) ); @@ -2386,8 +2503,8 @@ CppUnit::Test *CTimeSeriesDecompositionTest::suite(void) "CTimeSeriesDecompositionTest::testDiurnalProblemCase", &CTimeSeriesDecompositionTest::testDiurnalProblemCase) ); suiteOfTests->addTest( new CppUnit::TestCaller( - "CTimeSeriesDecompositionTest::testThirtyMinuteSamplingProblemCase", - &CTimeSeriesDecompositionTest::testThirtyMinuteSamplingProblemCase) ); + "CTimeSeriesDecompositionTest::testComplexDiurnalProblemCase", + &CTimeSeriesDecompositionTest::testComplexDiurnalProblemCase) ); suiteOfTests->addTest( new CppUnit::TestCaller( "CTimeSeriesDecompositionTest::testDiurnalPeriodicityWithMissingValues", &CTimeSeriesDecompositionTest::testDiurnalPeriodicityWithMissingValues) ); @@ -2415,5 +2532,9 @@ CppUnit::Test *CTimeSeriesDecompositionTest::suite(void) suiteOfTests->addTest( new CppUnit::TestCaller( "CTimeSeriesDecompositionTest::testPersist", &CTimeSeriesDecompositionTest::testPersist) ); + suiteOfTests->addTest( new CppUnit::TestCaller( + "CTimeSeriesDecompositionTest::testUpgrade", + &CTimeSeriesDecompositionTest::testUpgrade) ); + return suiteOfTests; } diff --git a/lib/maths/unittest/CTimeSeriesDecompositionTest.h b/lib/maths/unittest/CTimeSeriesDecompositionTest.h index af5f6c5d58..207d2d2040 100644 --- a/lib/maths/unittest/CTimeSeriesDecompositionTest.h +++ b/lib/maths/unittest/CTimeSeriesDecompositionTest.h @@ -22,7 +22,7 @@ class CTimeSeriesDecompositionTest : public CppUnit::TestFixture { public: void testSuperpositionOfSines(void); - void testMinimizeComponents(void); + void testDistortedPeriodic(void); void testMinimizeLongComponents(void); void testWeekend(void); void testSinglePeriodicity(void); @@ -30,7 +30,7 @@ class CTimeSeriesDecompositionTest : public CppUnit::TestFixture void testVarianceScale(void); void testSpikeyDataProblemCase(void); void testDiurnalProblemCase(void); - void testThirtyMinuteSamplingProblemCase(void); + void testComplexDiurnalProblemCase(void); void testDiurnalPeriodicityWithMissingValues(void); void testLongTermTrend(void); void testLongTermTrendAndPeriodicity(void); @@ -40,6 +40,7 @@ class CTimeSeriesDecompositionTest : public CppUnit::TestFixture void testConditionOfTrend(void); void testSwap(void); void testPersist(void); + void testUpgrade(void); static CppUnit::Test *suite(void); }; diff --git a/lib/maths/unittest/CTimeSeriesModelTest.cc b/lib/maths/unittest/CTimeSeriesModelTest.cc index 3f8c017732..b152c96473 100644 --- a/lib/maths/unittest/CTimeSeriesModelTest.cc +++ b/lib/maths/unittest/CTimeSeriesModelTest.cc @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -36,6 +37,8 @@ #include +#include + using namespace ml; namespace @@ -62,8 +65,9 @@ using TTail10Vec = core::CSmallVector; using TTimeDouble2VecSizeTrVec = maths::CModel::TTimeDouble2VecSizeTrVec; using TMeanAccumulator = maths::CBasicStatistics::SSampleMean::TAccumulator; using TMeanAccumulator2Vec = core::CSmallVector; -using TDecompositionPtr = boost::shared_ptr; +using TDecompositionPtr = boost::shared_ptr; using TDecompositionPtr10Vec = core::CSmallVector; +using TDecayRateController2Ary = maths::CUnivariateTimeSeriesModel::TDecayRateController2Ary; const double MINIMUM_SEASONAL_SCALE{0.25}; const double MINIMUM_SIGNIFICANT_CORRELATION{0.4}; @@ -162,6 +166,36 @@ maths::CUnivariateTimeSeriesModel::TDecayRateController2Ary decayRateControllers | maths::CDecayRateController::E_PredictionErrorDecrease, dimension)}}; } + +void reinitializePrior(double learnRate, + const maths::CMultivariateTimeSeriesModel &model, + TDecompositionPtr10Vec &trends, + maths::CMultivariatePrior &prior, + TDecayRateController2Ary *controllers = 0) +{ + prior.setToNonInformative(0.0, prior.decayRate()); + TDouble10Vec1Vec detrended_{TDouble10Vec(3)}; + for (const auto &value : model.slidingWindow()) + { + for (std::size_t i = 0u; i < value.second.size(); ++i) + { + detrended_[0][i] = trends[i]->detrend(value.first, value.second[i], 0.0); + } + prior.addSamples(maths::CConstantWeights::COUNT, + detrended_, + {{TDouble10Vec(value.second.size(), learnRate)}}); + } + if (controllers) + { + for (auto &&trend : trends) + { + trend->decayRate(trend->decayRate() / (*controllers)[0].multiplier()); + } + prior.decayRate(prior.decayRate() / (*controllers)[1].multiplier()); + (*controllers)[0].reset(); + (*controllers)[1].reset(); + } +} } void CTimeSeriesModelTest::testClone(void) @@ -187,7 +221,7 @@ void CTimeSeriesModelTest::testClone(void) TDoubleVec samples; rng.generateNormalSamples(1.0, 4.0, 1000, samples); - TDouble2Vec4Vec weight{TDouble2Vec{1.0}}; + TDouble2Vec4Vec weight{{1.0}}; TDouble2Vec4VecVec weights{weight}; core_t::TTime time{0}; for (auto sample : samples) @@ -267,17 +301,19 @@ void CTimeSeriesModelTest::testMode(void) maths::CNormalMeanPrecConjugate prior{univariateNormal()}; maths::CUnivariateTimeSeriesModel model{params(bucketLength), 0, trend, prior}; + core_t::TTime time{0}; for (auto sample : samples) { - prior.addSamples(maths::CConstantWeights::COUNT, - TDouble1Vec{sample}, - maths::CConstantWeights::SINGLE_UNIT); + trend.addPoint(time, sample); + TDouble1Vec sample_{trend.detrend(time, sample, 0.0)}; + prior.addSamples(maths::CConstantWeights::COUNT, sample_, maths::CConstantWeights::SINGLE_UNIT); prior.propagateForwardsByTime(1.0); + time += bucketLength; } - TDouble2Vec4Vec weight{TDouble2Vec{1.0}}; + TDouble2Vec4Vec weight{{1.0}}; TDouble2Vec4VecVec weights{weight}; - core_t::TTime time{0}; + time = 0; for (auto sample : samples) { maths::CModelAddSamplesParams params; @@ -289,12 +325,14 @@ void CTimeSeriesModelTest::testMode(void) model.addSamples(params, {core::make_triple(time, TDouble2Vec{sample}, TAG)}); time += bucketLength; } + double expectedMode{ maths::CBasicStatistics::mean(trend.baseline(time)) + + prior.marginalLikelihoodMode()}; TDouble2Vec mode(model.mode(time, maths::CConstantWeights::COUNT, weight)); - LOG_DEBUG("expected mode = " << prior.marginalLikelihoodMode()); + LOG_DEBUG("expected mode = " << expectedMode); LOG_DEBUG("mode = " << mode[0]); CPPUNIT_ASSERT_EQUAL(std::size_t(1), mode.size()); - CPPUNIT_ASSERT_EQUAL(prior.marginalLikelihoodMode(), mode[0]); + CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedMode, mode[0], 1e-3 * expectedMode); } LOG_DEBUG("Univariate trend"); @@ -302,6 +340,7 @@ void CTimeSeriesModelTest::testMode(void) TDoubleVec samples; rng.generateNormalSamples(1.0, 4.0, 1000, samples); + double learnRate{params(bucketLength).learnRate()}; maths::CTimeSeriesDecomposition trend{24.0 * DECAY_RATE, bucketLength}; maths::CNormalMeanPrecConjugate prior{univariateNormal()}; maths::CUnivariateTimeSeriesModel model{params(bucketLength), 0, trend, prior}; @@ -312,17 +351,10 @@ void CTimeSeriesModelTest::testMode(void) sample += 20.0 + 10.0 * ::sin( boost::math::double_constants::two_pi * static_cast(time) / static_cast(core::constants::DAY)); - if (trend.addPoint(time, sample)) - { - maths::initializePrior(bucketLength, params(bucketLength).learnRate(), trend, prior); - } - TDouble1Vec sample_{trend.detrend(time, sample, 0.0)}; - prior.addSamples(maths::CConstantWeights::COUNT, sample_, maths::CConstantWeights::SINGLE_UNIT); - prior.propagateForwardsByTime(1.0); time += bucketLength; } - TDouble2Vec4Vec weight{TDouble2Vec{1.0}}; + TDouble2Vec4Vec weight{{1.0}}; TDouble2Vec4VecVec weights{weight}; time = 0; for (auto sample : samples) @@ -334,6 +366,19 @@ void CTimeSeriesModelTest::testMode(void) .trendWeights(weights) .priorWeights(weights); model.addSamples(params, {core::make_triple(time, TDouble2Vec{sample}, TAG)}); + if (trend.addPoint(time, sample)) + { + prior.setToNonInformative(0.0, DECAY_RATE); + for (const auto &value : model.slidingWindow()) + { + prior.addSamples(maths::CConstantWeights::COUNT, + {trend.detrend(value.first, value.second, 0.0)}, + {{learnRate}}); + } + } + TDouble1Vec sample_{trend.detrend(time, sample, 0.0)}; + prior.addSamples(maths::CConstantWeights::COUNT, sample_, maths::CConstantWeights::SINGLE_UNIT); + prior.propagateForwardsByTime(1.0); time += bucketLength; } @@ -344,7 +389,7 @@ void CTimeSeriesModelTest::testMode(void) LOG_DEBUG("expected mode = " << expectedMode); LOG_DEBUG("mode = " << mode[0]); CPPUNIT_ASSERT_EQUAL(std::size_t(1), mode.size()); - CPPUNIT_ASSERT_EQUAL(expectedMode, mode[0]); + CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedMode, mode[0], 1e-3 * expectedMode); } LOG_DEBUG("Multivariate no trend"); @@ -354,20 +399,29 @@ void CTimeSeriesModelTest::testMode(void) TDoubleVecVec samples; rng.generateMultivariateNormalSamples(mean, covariance, 1000, samples); - maths::CTimeSeriesDecomposition trend{DECAY_RATE, bucketLength}; + TDecompositionPtr10Vec trends{TDecompositionPtr{new maths::CTimeSeriesDecomposition{DECAY_RATE, bucketLength}}, + TDecompositionPtr{new maths::CTimeSeriesDecomposition{DECAY_RATE, bucketLength}}, + TDecompositionPtr{new maths::CTimeSeriesDecomposition{DECAY_RATE, bucketLength}}}; maths::CMultivariateNormalConjugate<3> prior{multivariateNormal()}; - maths::CMultivariateTimeSeriesModel model{params(bucketLength), trend, prior}; + maths::CMultivariateTimeSeriesModel model{params(bucketLength), *trends[0], prior}; + core_t::TTime time{0}; for (const auto &sample : samples) { + TDouble10Vec1Vec detrended{TDouble10Vec(3)}; + for (std::size_t i = 0u; i < sample.size(); ++i) + { + trends[i]->addPoint(time, sample[i]); + detrended[0][i] = trends[i]->detrend(time, sample[i], 0.0); + } prior.addSamples(maths::CConstantWeights::COUNT, - TDouble10Vec1Vec{sample}, + detrended, maths::CConstantWeights::singleUnit(3)); prior.propagateForwardsByTime(1.0); } TDouble2Vec4VecVec weights{maths::CConstantWeights::unit(3)}; - core_t::TTime time{0}; + time = 0; for (const auto &sample : samples) { maths::CModelAddSamplesParams params; @@ -381,6 +435,10 @@ void CTimeSeriesModelTest::testMode(void) } TDouble2Vec expectedMode(prior.marginalLikelihoodMode(maths::CConstantWeights::COUNT, maths::CConstantWeights::unit(3))); + for (std::size_t i = 0u; i < trends.size(); ++i) + { + expectedMode[i] += maths::CBasicStatistics::mean(trends[i]->baseline(time)); + } TDouble2Vec mode(model.mode(time, maths::CConstantWeights::COUNT, maths::CConstantWeights::unit(3))); @@ -388,9 +446,9 @@ void CTimeSeriesModelTest::testMode(void) LOG_DEBUG("expected mode = " << expectedMode); LOG_DEBUG("mode = " << mode); CPPUNIT_ASSERT_EQUAL(std::size_t(3), mode.size()); - CPPUNIT_ASSERT_EQUAL(expectedMode[0], mode[0]); - CPPUNIT_ASSERT_EQUAL(expectedMode[1], mode[1]); - CPPUNIT_ASSERT_EQUAL(expectedMode[2], mode[2]); + CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedMode[0], mode[0], 0.02 * expectedMode[0]); + CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedMode[1], mode[1], 0.02 * expectedMode[0]); + CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedMode[2], mode[2], 0.02 * expectedMode[0]); } LOG_DEBUG("Multivariate trend"); @@ -400,36 +458,24 @@ void CTimeSeriesModelTest::testMode(void) TDoubleVecVec samples; rng.generateMultivariateNormalSamples(mean, covariance, 1000, samples); + double learnRate{params(bucketLength).learnRate()}; TDecompositionPtr10Vec trends{TDecompositionPtr{new maths::CTimeSeriesDecomposition{DECAY_RATE, bucketLength}}, TDecompositionPtr{new maths::CTimeSeriesDecomposition{DECAY_RATE, bucketLength}}, TDecompositionPtr{new maths::CTimeSeriesDecomposition{DECAY_RATE, bucketLength}}}; maths::CMultivariateNormalConjugate<3> prior{multivariateNormal()}; maths::CMultivariateTimeSeriesModel model{params(bucketLength), *trends[0], prior}; + core_t::TTime time{0}; for (auto &&sample : samples) { - bool reinitialize{false}; - TDouble10Vec1Vec detrended{TDouble10Vec(3)}; - double amplitude{10.0}; for (std::size_t i = 0u; i < sample.size(); ++i) { sample[i] += 30.0 + amplitude * ::sin( boost::math::double_constants::two_pi * static_cast(time) / static_cast(core::constants::DAY)); - reinitialize |= trends[i]->addPoint(time, sample[i]); - detrended[0][i] = trends[i]->detrend(time, sample[i], 0.0); amplitude += 4.0; } - - if (reinitialize) - { - maths::initializePrior(bucketLength, params(bucketLength).learnRate(), trends, prior); - } - prior.addSamples(maths::CConstantWeights::COUNT, - detrended, - maths::CConstantWeights::singleUnit(3)); - prior.propagateForwardsByTime(1.0); time += bucketLength; } @@ -444,6 +490,23 @@ void CTimeSeriesModelTest::testMode(void) .trendWeights(weights) .priorWeights(weights); model.addSamples(params, {core::make_triple(time, TDouble2Vec(sample), TAG)}); + + bool reinitialize{false}; + TDouble10Vec1Vec detrended{TDouble10Vec(3)}; + for (std::size_t i = 0u; i < sample.size(); ++i) + { + reinitialize |= trends[i]->addPoint(time, sample[i]); + detrended[0][i] = trends[i]->detrend(time, sample[i], 0.0); + } + if (reinitialize) + { + reinitializePrior(learnRate, model, trends, prior); + } + prior.addSamples(maths::CConstantWeights::COUNT, + detrended, + maths::CConstantWeights::singleUnit(3)); + prior.propagateForwardsByTime(1.0); + time += bucketLength; } TDouble2Vec expectedMode(prior.marginalLikelihoodMode(maths::CConstantWeights::COUNT, @@ -459,9 +522,9 @@ void CTimeSeriesModelTest::testMode(void) LOG_DEBUG("expected mode = " << expectedMode); LOG_DEBUG("mode = " << mode); CPPUNIT_ASSERT_EQUAL(std::size_t(3), mode.size()); - CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedMode[0], mode[0], 0.01); - CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedMode[1], mode[1], 0.01); - CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedMode[2], mode[2], 0.01); + CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedMode[0], mode[0], 1e-3 * expectedMode[0]); + CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedMode[1], mode[1], 1e-3 * expectedMode[1]); + CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedMode[2], mode[2], 1e-3 * expectedMode[2]); } } @@ -475,7 +538,7 @@ void CTimeSeriesModelTest::testAddBucketValue(void) // for negative bucket values. core_t::TTime bucketLength{600}; - maths::CTimeSeriesDecomposition trend{24.0 * DECAY_RATE, bucketLength}; + maths::CTimeSeriesDecompositionStub trend; maths::CLogNormalMeanPrecConjugate prior{univariateLogNormal()}; maths::CUnivariateTimeSeriesModel model{params(bucketLength), 0, trend, prior}; @@ -483,21 +546,17 @@ void CTimeSeriesModelTest::testAddBucketValue(void) core::make_triple(core_t::TTime{12}, TDouble2Vec{3.9}, TAG), core::make_triple(core_t::TTime{18}, TDouble2Vec{2.1}, TAG), core::make_triple(core_t::TTime{12}, TDouble2Vec{1.2}, TAG),}; - TDouble2Vec4VecVec weights{TDouble2Vec4Vec{TDouble2Vec{1.0}}, - TDouble2Vec4Vec{TDouble2Vec{1.5}}, - TDouble2Vec4Vec{TDouble2Vec{0.9}}, - TDouble2Vec4Vec{TDouble2Vec{1.9}}}; + TDouble2Vec4VecVec weights{{{1.0}}, {{1.5}}, {{0.9}}, {{1.9}}}; for (std::size_t i = 0u; i < samples.size(); ++i) { prior.addSamples(maths::CConstantWeights::COUNT, - TDouble1Vec{samples[i].second[0]}, - TDouble4Vec1Vec{TDouble4Vec{weights[i][0][0]}}); + {samples[i].second[0]}, + {{weights[i][0][0]}}); } prior.propagateForwardsByTime(1.0); prior.adjustOffset(maths::CConstantWeights::COUNT, - TDouble1Vec{-1.0}, - maths::CConstantWeights::SINGLE_UNIT); + {-1.0}, maths::CConstantWeights::SINGLE_UNIT); maths::CModelAddSamplesParams params; params.integer(false) @@ -527,16 +586,14 @@ void CTimeSeriesModelTest::testAddSamples(void) LOG_DEBUG("Multiple samples univariate"); { - maths::CTimeSeriesDecomposition trend{24.0 * DECAY_RATE, bucketLength}; + maths::CTimeSeriesDecompositionStub trend; maths::CNormalMeanPrecConjugate prior{univariateNormal()}; maths::CUnivariateTimeSeriesModel model{params(bucketLength), 0, trend, prior}; TTimeDouble2VecSizeTrVec samples{core::make_triple(core_t::TTime{20}, TDouble2Vec{3.5}, TAG), core::make_triple(core_t::TTime{12}, TDouble2Vec{3.9}, TAG), core::make_triple(core_t::TTime{18}, TDouble2Vec{2.1}, TAG)}; - TDouble2Vec4VecVec weights{TDouble2Vec4Vec{TDouble2Vec{1.0}}, - TDouble2Vec4Vec{TDouble2Vec{1.5}}, - TDouble2Vec4Vec{TDouble2Vec{0.9}}}; + TDouble2Vec4VecVec weights{{{1.0}}, {{1.5}}, {{0.9}}}; maths::CModelAddSamplesParams params; params.integer(false) @@ -567,18 +624,16 @@ void CTimeSeriesModelTest::testAddSamples(void) LOG_DEBUG("Multiple samples multivariate"); { - TDecompositionPtr10Vec trends{TDecompositionPtr{new maths::CTimeSeriesDecomposition{DECAY_RATE, bucketLength}}, - TDecompositionPtr{new maths::CTimeSeriesDecomposition{DECAY_RATE, bucketLength}}, - TDecompositionPtr{new maths::CTimeSeriesDecomposition{DECAY_RATE, bucketLength}}}; + TDecompositionPtr10Vec trends{TDecompositionPtr{new maths::CTimeSeriesDecompositionStub{}}, + TDecompositionPtr{new maths::CTimeSeriesDecompositionStub{}}, + TDecompositionPtr{new maths::CTimeSeriesDecompositionStub{}}}; maths::CMultivariateNormalConjugate<3> prior{multivariateNormal()}; maths::CMultivariateTimeSeriesModel model{params(bucketLength), *trends[0], prior}; TTimeDouble2VecSizeTrVec samples{core::make_triple(core_t::TTime{20}, TDouble2Vec{3.5, 3.4, 3.3}, TAG), core::make_triple(core_t::TTime{12}, TDouble2Vec{3.9, 3.8, 3.7}, TAG), core::make_triple(core_t::TTime{18}, TDouble2Vec{2.1, 2.0, 1.9}, TAG)}; - TDouble2Vec4VecVec weights{TDouble2Vec4Vec{TDouble2Vec{1.0, 1.1, 1.2}}, - TDouble2Vec4Vec{TDouble2Vec{1.5, 1.6, 1.7}}, - TDouble2Vec4Vec{TDouble2Vec{0.9, 1.0, 1.1}}}; + TDouble2Vec4VecVec weights{{{1.0, 1.1, 1.2}}, {{1.5, 1.6, 1.7}}, {{0.9, 1.0, 1.1}}}; maths::CModelAddSamplesParams params; params.integer(false) @@ -599,9 +654,7 @@ void CTimeSeriesModelTest::testAddSamples(void) maths::CConstantWeights::COUNT, TDouble4Vec{weights[0][0][i]}); } TDouble10Vec1Vec samples_{samples[2].second, samples[0].second, samples[1].second}; - TDouble10Vec4Vec1Vec weights_{TDouble10Vec4Vec{weights[2][0]}, - TDouble10Vec4Vec{weights[0][0]}, - TDouble10Vec4Vec{weights[1][0]}}; + TDouble10Vec4Vec1Vec weights_{{weights[2][0]}, {weights[0][0]}, {weights[1][0]}}; prior.addSamples(maths::CConstantWeights::COUNT, samples_, weights_); prior.propagateForwardsByTime(1.0); @@ -623,13 +676,13 @@ void CTimeSeriesModelTest::testAddSamples(void) maths_t::TWeightStyleVec weightStyles{maths_t::E_SampleWinsorisationWeight, maths_t::E_SampleCountWeight, maths_t::E_SampleCountVarianceScaleWeight}; - maths::CTimeSeriesDecomposition trend{24.0 * DECAY_RATE, bucketLength}; + maths::CTimeSeriesDecompositionStub trend; maths::CNormalMeanPrecConjugate prior{univariateNormal()}; maths::CUnivariateTimeSeriesModel model{params(bucketLength), 0, trend, prior}; double interval[]{1.0, 1.1, 0.4}; - TDouble2Vec samples[]{TDouble2Vec{10.0}, TDouble2Vec{13.9}, TDouble2Vec{27.1}}; - TDouble2Vec4VecVec weights{TDouble2Vec4Vec{TDouble2Vec{0.9}, TDouble2Vec{1.5}, TDouble2Vec{1.1}}}; + TDouble2Vec samples[]{{10.0}, {13.9}, {27.1}}; + TDouble2Vec4VecVec weights{{{0.9}, {1.5}, {1.1}}}; core_t::TTime time{0}; for (std::size_t i = 0u; i < 3; ++i) @@ -644,7 +697,7 @@ void CTimeSeriesModelTest::testAddSamples(void) model.addSamples(params, sample); TDouble4Vec weight{weights[0][0][0], weights[0][1][0], weights[0][2][0]}; - prior.addSamples(weightStyles, samples[i], TDouble4Vec1Vec{weight}); + prior.addSamples(weightStyles, samples[i], {weight}); prior.propagateForwardsByTime(interval[i]); uint64_t checksum1{prior.checksum()}; @@ -658,9 +711,9 @@ void CTimeSeriesModelTest::testAddSamples(void) LOG_DEBUG("Propagation interval multivariate"); { - TDecompositionPtr10Vec trends{TDecompositionPtr{new maths::CTimeSeriesDecomposition{DECAY_RATE, bucketLength}}, - TDecompositionPtr{new maths::CTimeSeriesDecomposition{DECAY_RATE, bucketLength}}, - TDecompositionPtr{new maths::CTimeSeriesDecomposition{DECAY_RATE, bucketLength}}}; + TDecompositionPtr10Vec trends{TDecompositionPtr{new maths::CTimeSeriesDecompositionStub{}}, + TDecompositionPtr{new maths::CTimeSeriesDecompositionStub{}}, + TDecompositionPtr{new maths::CTimeSeriesDecompositionStub{}}}; maths::CMultivariateNormalConjugate<3> prior{multivariateNormal()}; maths::CMultivariateTimeSeriesModel model{params(bucketLength), *trends[0], prior}; @@ -668,18 +721,12 @@ void CTimeSeriesModelTest::testAddSamples(void) maths_t::E_SampleCountWeight, maths_t::E_SampleCountVarianceScaleWeight}; double interval[]{1.0, 1.1, 0.4}; - TDouble2Vec samples[]{TDouble2Vec{13.5, 13.4, 13.3}, - TDouble2Vec{13.9, 13.8, 13.7}, - TDouble2Vec{20.1, 20.0, 10.9}}; - TDouble2Vec4VecVec weights{TDouble2Vec4Vec{TDouble2Vec{0.1, 0.1, 0.2}, - TDouble2Vec{1.0, 1.1, 1.2}, - TDouble2Vec{2.0, 2.1, 2.2}}, - TDouble2Vec4Vec{TDouble2Vec{0.5, 0.6, 0.7}, - TDouble2Vec{2.0, 2.1, 2.2}, - TDouble2Vec{1.0, 1.1, 1.2}}, - TDouble2Vec4Vec{TDouble2Vec{0.9, 1.0, 1.0}, - TDouble2Vec{0.9, 1.0, 1.0}, - TDouble2Vec{1.9, 2.0, 2.0}}}; + TDouble2Vec samples[]{{13.5, 13.4, 13.3}, + {13.9, 13.8, 13.7}, + {20.1, 20.0, 10.9}}; + TDouble2Vec4VecVec weights{{{0.1, 0.1, 0.2}, {1.0, 1.1, 1.2}, {2.0, 2.1, 2.2}}, + {{0.5, 0.6, 0.7}, {2.0, 2.1, 2.2}, {1.0, 1.1, 1.2}}, + {{0.9, 1.0, 1.0}, {0.9, 1.0, 1.0}, {1.9, 2.0, 2.0}}}; core_t::TTime time{0}; for (std::size_t i = 0u; i < 3; ++i) @@ -697,8 +744,8 @@ void CTimeSeriesModelTest::testAddSamples(void) TDouble10Vec(weights[0][1]), TDouble10Vec(weights[0][2])}; prior.addSamples(weightStyles, - TDouble10Vec1Vec{TDouble10Vec(samples[i])}, - TDouble10Vec4Vec1Vec{weight}); + {TDouble10Vec(samples[i])}, + {weight}); prior.propagateForwardsByTime(interval[i]); uint64_t checksum1{prior.checksum()}; @@ -712,6 +759,7 @@ void CTimeSeriesModelTest::testAddSamples(void) LOG_DEBUG("Decay rate control univariate"); { + double learnRate{params(bucketLength).learnRate()}; maths::CTimeSeriesDecomposition trend{DECAY_RATE, bucketLength}; maths::CNormalMeanPrecConjugate prior{univariateNormal()}; auto controllers = decayRateControllers(1); @@ -720,7 +768,7 @@ void CTimeSeriesModelTest::testAddSamples(void) TDoubleVec samples; rng.generateNormalSamples(1.0, 4.0, 2000, samples); - TDouble4Vec1Vec weight{TDouble4Vec{1.0}}; + TDouble4Vec1Vec weight{{1.0}}; TDouble2Vec4VecVec weights{maths::CConstantWeights::unit(1)}; core_t::TTime time{0}; @@ -741,13 +789,20 @@ void CTimeSeriesModelTest::testAddSamples(void) if (trend.addPoint(time, sample)) { + trend.decayRate(trend.decayRate() / controllers[0].multiplier()); + prior.setToNonInformative(0.0, prior.decayRate()); + for (const auto &value : model.slidingWindow()) + { + prior.addSamples(maths::CConstantWeights::COUNT, + {trend.detrend(value.first, value.second, 0.0)}, + {{learnRate}}); + } + prior.decayRate(prior.decayRate() / controllers[1].multiplier()); controllers[0].reset(); controllers[1].reset(); - maths::initializePrior(bucketLength, model.params().learnRate(), trend, prior); - prior.decayRate(DECAY_RATE); } double detrended{trend.detrend(time, sample, 0.0)}; - prior.addSamples(maths::CConstantWeights::COUNT, TDouble1Vec{detrended}, weight); + prior.addSamples(maths::CConstantWeights::COUNT, {detrended}, weight); prior.propagateForwardsByTime(1.0); if (trend.initialized()) @@ -782,6 +837,7 @@ void CTimeSeriesModelTest::testAddSamples(void) LOG_DEBUG("Decay rate control multivariate"); { + double learnRate{params(bucketLength).learnRate()}; TDecompositionPtr10Vec trends{TDecompositionPtr{new maths::CTimeSeriesDecomposition{DECAY_RATE, bucketLength}}, TDecompositionPtr{new maths::CTimeSeriesDecomposition{DECAY_RATE, bucketLength}}, TDecompositionPtr{new maths::CTimeSeriesDecomposition{DECAY_RATE, bucketLength}}}; @@ -796,8 +852,8 @@ void CTimeSeriesModelTest::testAddSamples(void) rng.generateMultivariateNormalSamples(mean, covariance, 1000, samples); } - TDouble10Vec4Vec1Vec weight{TDouble10Vec4Vec{TDouble10Vec{1.0, 1.0, 1.0}}}; - TDouble2Vec4VecVec weights{TDouble2Vec4Vec{TDouble2Vec{1.0, 1.0, 1.0}}}; + TDouble10Vec4Vec1Vec weight{{{1.0, 1.0, 1.0}}}; + TDouble2Vec4VecVec weights{{{1.0, 1.0, 1.0}}}; core_t::TTime time{0}; for (auto &&sample : samples) @@ -831,10 +887,7 @@ void CTimeSeriesModelTest::testAddSamples(void) if (reinitialize) { - controllers[0].reset(); - controllers[1].reset(); - maths::initializePrior(bucketLength, params(bucketLength).learnRate(), trends, prior); - prior.decayRate(DECAY_RATE); + reinitializePrior(learnRate, model, trends, prior, &controllers); } prior.addSamples(maths::CConstantWeights::COUNT, detrended, weight); prior.propagateForwardsByTime(1.0); @@ -895,6 +948,7 @@ void CTimeSeriesModelTest::testPredict(void) LOG_DEBUG("Univariate seasonal"); { + double learnRate{params(bucketLength).learnRate()}; maths::CTimeSeriesDecomposition trend{24.0 * DECAY_RATE, bucketLength}; maths::CNormalMeanPrecConjugate prior{univariateNormal()}; auto controllers = decayRateControllers(1); @@ -909,15 +963,6 @@ void CTimeSeriesModelTest::testPredict(void) sample += 10.0 + 5.0 * ::sin( boost::math::double_constants::two_pi * static_cast(time) / 86400.0); - if (trend.addPoint(time, sample)) - { - maths::initializePrior(bucketLength, model.params().learnRate(), trend, prior); - } - prior.addSamples(maths::CConstantWeights::COUNT, - TDouble1Vec{trend.detrend(time, sample, 0.0)}, - maths::CConstantWeights::SINGLE_UNIT); - prior.propagateForwardsByTime(1.0); - maths::CModelAddSamplesParams params; params.integer(false) .propagationInterval(1.0) @@ -926,9 +971,25 @@ void CTimeSeriesModelTest::testPredict(void) .priorWeights(weights); model.addSamples(params, {core::make_triple(time, TDouble2Vec{sample}, TAG)}); + if (trend.addPoint(time, sample)) + { + prior.setToNonInformative(0.0, DECAY_RATE); + for (const auto &value : model.slidingWindow()) + { + prior.addSamples(maths::CConstantWeights::COUNT, + {trend.detrend(value.first, value.second, 0.0)}, + {{learnRate}}); + } + } + prior.addSamples(maths::CConstantWeights::COUNT, + {trend.detrend(time, sample, 0.0)}, + maths::CConstantWeights::SINGLE_UNIT); + prior.propagateForwardsByTime(1.0); + time += bucketLength; } + TMeanAccumulator meanError; for (core_t::TTime time_ = time; time_ < time + 86400; time_ += 3600) { double trend_{10.0 + 5.0 * ::sin( boost::math::double_constants::two_pi @@ -939,14 +1000,18 @@ void CTimeSeriesModelTest::testPredict(void) LOG_DEBUG("expected = " << expected << " predicted = " << predicted << " (trend = " << trend_ << ")"); - CPPUNIT_ASSERT_EQUAL(expected, predicted); - CPPUNIT_ASSERT(::fabs(trend_ - predicted) / trend_ < 0.1); + CPPUNIT_ASSERT_DOUBLES_EQUAL(expected, predicted, 1e-3 * expected); + CPPUNIT_ASSERT(std::fabs(trend_ - predicted) / trend_ < 0.3); + meanError.add(std::fabs(trend_ - predicted) / trend_); } + + LOG_DEBUG("mean error = " << maths::CBasicStatistics::mean(meanError)); + CPPUNIT_ASSERT(maths::CBasicStatistics::mean(meanError) < 0.06); } LOG_DEBUG("Univariate nearest mode"); { - maths::CTimeSeriesDecomposition trend{24.0 * DECAY_RATE, bucketLength}; + maths::CTimeSeriesDecompositionStub trend; maths::CMultimodalPrior prior{univariateMultimodal()}; maths::CUnivariateTimeSeriesModel model{params(bucketLength), 0, trend, prior}; @@ -974,8 +1039,8 @@ void CTimeSeriesModelTest::testPredict(void) } maths::CModel::TSizeDoublePr1Vec empty; - double predicted[]{model.predict(time, empty, TDouble2Vec{-2.0})[0], - model.predict(time, empty, TDouble2Vec{12.0})[0]}; + double predicted[]{model.predict(time, empty, {-2.0})[0], + model.predict(time, empty, {12.0})[0]}; LOG_DEBUG("expected(0) = " << maths::CBasicStatistics::mean(modes[0]) << " actual(0) = " << predicted[0]); @@ -991,6 +1056,7 @@ void CTimeSeriesModelTest::testPredict(void) LOG_DEBUG("Multivariate Seasonal"); { + double learnRate{params(bucketLength).learnRate()}; TDecompositionPtr10Vec trends{TDecompositionPtr{new maths::CTimeSeriesDecomposition{24.0 * DECAY_RATE, bucketLength}}, TDecompositionPtr{new maths::CTimeSeriesDecomposition{24.0 * DECAY_RATE, bucketLength}}, TDecompositionPtr{new maths::CTimeSeriesDecomposition{24.0 * DECAY_RATE, bucketLength}}}; @@ -1022,10 +1088,10 @@ void CTimeSeriesModelTest::testPredict(void) } if (reinitialize) { - maths::initializePrior(bucketLength, model.params().learnRate(), trends, prior); + reinitializePrior(learnRate, model, trends, prior); } prior.addSamples(maths::CConstantWeights::COUNT, - TDouble10Vec1Vec{detrended}, + {detrended}, maths::CConstantWeights::singleUnit(3)); prior.propagateForwardsByTime(1.0); @@ -1052,28 +1118,28 @@ void CTimeSeriesModelTest::testPredict(void) double expected{ maths::CBasicStatistics::mean(trends[i]->baseline(time_)) + maths::CBasicStatistics::mean(margin->marginalLikelihoodConfidenceInterval(0.0))}; double predicted{model.predict(time_)[i]}; - --marginalize[std::min(i, marginalize.size() -1)]; + --marginalize[std::min(i, marginalize.size() - 1)]; LOG_DEBUG("expected = " << expected << " predicted = " << predicted << " (trend = " << trend_ << ")"); - CPPUNIT_ASSERT_EQUAL(expected, predicted); - CPPUNIT_ASSERT(::fabs(trend_ - predicted) / trend_ < 0.2); + CPPUNIT_ASSERT_DOUBLES_EQUAL(expected, predicted, 1e-3 * expected); + CPPUNIT_ASSERT(std::fabs(trend_ - predicted) / trend_ < 0.3); } } } LOG_DEBUG("Multivariate nearest mode"); { - TDecompositionPtr10Vec trends{TDecompositionPtr{new maths::CTimeSeriesDecomposition{24.0 * DECAY_RATE, bucketLength}}, - TDecompositionPtr{new maths::CTimeSeriesDecomposition{24.0 * DECAY_RATE, bucketLength}}, - TDecompositionPtr{new maths::CTimeSeriesDecomposition{24.0 * DECAY_RATE, bucketLength}}}; + TDecompositionPtr10Vec trends{TDecompositionPtr{new maths::CTimeSeriesDecompositionStub{}}, + TDecompositionPtr{new maths::CTimeSeriesDecompositionStub{}}, + TDecompositionPtr{new maths::CTimeSeriesDecompositionStub{}}}; maths::CMultivariateMultimodalPrior<3> prior{multivariateMultimodal()}; maths::CMultivariateTimeSeriesModel model{maths::CMultivariateTimeSeriesModel{params(bucketLength), *trends[0], prior}}; TMeanAccumulator2Vec modes[2]{TMeanAccumulator2Vec(3), TMeanAccumulator2Vec(3)}; TDoubleVecVec samples; { - TDoubleVec means[]{TDoubleVec{0.0, 2.0, 1.0}, TDoubleVec{10.0, 15.0, 12.0}}; + TDoubleVec means[]{{0.0, 2.0, 1.0}, {10.0, 15.0, 12.0}}; TDoubleVecVec covariance{{3.0, 2.9, 0.5}, {2.9, 2.6, 0.1}, {0.5, 0.1, 2.0}}; rng.generateMultivariateNormalSamples(means[0], covariance, 500, samples); TDoubleVecVec samples_; @@ -1113,14 +1179,14 @@ void CTimeSeriesModelTest::testPredict(void) maths::CModel::TSizeDoublePr1Vec empty; TDouble2Vec expected[]{maths::CBasicStatistics::mean(modes[0]), maths::CBasicStatistics::mean(modes[1])}; - TDouble2Vec predicted[]{model.predict(time, empty, TDouble2Vec{ 0.0, 0.0, 0.0}), - model.predict(time, empty, TDouble2Vec{10.0, 10.0, 10.0})}; + TDouble2Vec predicted[]{model.predict(time, empty, { 0.0, 0.0, 0.0}), + model.predict(time, empty, {10.0, 10.0, 10.0})}; for (std::size_t i = 0u; i < 3; ++i) { LOG_DEBUG("expected(0) = " << expected[0][i] << " actual(0) = " << predicted[0][i]); LOG_DEBUG("expected(1) = " << expected[1][i] << " actual(1) = " << predicted[1][i]); - CPPUNIT_ASSERT_DOUBLES_EQUAL(expected[0][i], predicted[0][i], ::fabs(0.2 * expected[0][i])); - CPPUNIT_ASSERT_DOUBLES_EQUAL(expected[1][i], predicted[1][i], ::fabs(0.01 * expected[1][i])); + CPPUNIT_ASSERT_DOUBLES_EQUAL(expected[0][i], predicted[0][i], std::fabs(0.2 * expected[0][i])); + CPPUNIT_ASSERT_DOUBLES_EQUAL(expected[1][i], predicted[1][i], std::fabs(0.01 * expected[1][i])); } } } @@ -1144,60 +1210,48 @@ void CTimeSeriesModelTest::testProbability(void) LOG_DEBUG("Univariate"); { - maths::CTimeSeriesDecomposition trend{24.0 * DECAY_RATE, bucketLength}; - maths::CNormalMeanPrecConjugate priors[]{univariateNormal(), univariateNormal()}; - maths::CUnivariateTimeSeriesModel models[]{maths::CUnivariateTimeSeriesModel{params(bucketLength), 1, trend, priors[0], 0, false}, - maths::CUnivariateTimeSeriesModel{params(bucketLength), 1, trend, priors[1], 0, false}}; + maths::CUnivariateTimeSeriesModel models[]{ + maths::CUnivariateTimeSeriesModel{params(bucketLength), 1, + maths::CTimeSeriesDecompositionStub{}, + univariateNormal(), 0, false}, + maths::CUnivariateTimeSeriesModel{params(bucketLength), 1, + maths::CTimeSeriesDecomposition{24.0 * DECAY_RATE, bucketLength}, + univariateNormal(), 0, false}}; TDoubleVec samples; rng.generateNormalSamples(10.0, 4.0, 1000, samples); core_t::TTime time{0}; + const TDouble2Vec4VecVec weight{maths::CConstantWeights::unit(1)}; for (auto sample : samples) { - static const TDouble2Vec4VecVec UNIT{maths::CConstantWeights::unit(1)}; - double season{5.0 + 5.0 * ::sin( boost::math::double_constants::two_pi - * static_cast(time) / 86400.0)}; - - priors[0].addSamples(maths::CConstantWeights::COUNT, - TDouble1Vec{sample}, - maths::CConstantWeights::SINGLE_UNIT); - priors[0].propagateForwardsByTime(1.0); - if (trend.addPoint(time, season + sample)) - { - maths::initializePrior(bucketLength, models[1].params().learnRate(), trend, priors[1]); - } - priors[1].addSamples(maths::CConstantWeights::COUNT, - TDouble1Vec{trend.detrend(time, season + sample, 0.0)}, - maths::CConstantWeights::SINGLE_UNIT); - priors[1].propagateForwardsByTime(1.0); - maths::CModelAddSamplesParams params; params.integer(false) .propagationInterval(1.0) .weightStyles(maths::CConstantWeights::COUNT) - .trendWeights(UNIT) - .priorWeights(UNIT); - models[0].addSamples(params, TTimeDouble2VecSizeTrVec{ - core::make_triple(time, TDouble2Vec{sample}, TAG)}); - models[1].addSamples(params, TTimeDouble2VecSizeTrVec{ - core::make_triple(time, TDouble2Vec{season + sample}, TAG)}); + .trendWeights(weight) + .priorWeights(weight); + + double trend{5.0 + 5.0 * ::sin( boost::math::double_constants::two_pi + * static_cast(time) / 86400.0)}; + + models[0].addSamples(params, {core::make_triple(time, TDouble2Vec{sample}, TAG)}); + models[1].addSamples(params, {core::make_triple(time, TDouble2Vec{trend + sample}, TAG)}); time += bucketLength; } - TTime2Vec1Vec time_{TTime2Vec{time}}; + TTime2Vec1Vec time_{{time}}; TDouble2Vec sample{15.0}; maths_t::EProbabilityCalculation calculations[]{maths_t::E_TwoSided, maths_t::E_OneSidedAbove}; double confidences[]{0.0, 20.0, 50.0}; bool empties[]{true, false}; - maths_t::TWeightStyleVec weightStyles[]{maths_t::TWeightStyleVec{maths_t::E_SampleCountVarianceScaleWeight}, - maths_t::TWeightStyleVec{maths_t::E_SampleCountVarianceScaleWeight, - maths_t::E_SampleSeasonalVarianceScaleWeight}}; - TDouble2Vec4Vec weights[]{TDouble2Vec4Vec{TDouble2Vec{0.9}}, - TDouble2Vec4Vec{TDouble2Vec{1.1}, TDouble2Vec{1.8}}}; + maths_t::TWeightStyleVec weightStyles[]{{maths_t::E_SampleCountVarianceScaleWeight}, + {maths_t::E_SampleCountVarianceScaleWeight, + maths_t::E_SampleSeasonalVarianceScaleWeight}}; + TDouble2Vec4Vec weights[]{{{0.9}}, {{1.1}, {1.8}}}; for (auto calculation : calculations) { @@ -1215,20 +1269,22 @@ void CTimeSeriesModelTest::testProbability(void) maths_t::ETail expectedTail[2]; { TDouble4Vec weights_; - for (const auto &weight : weights[i]) + for (const auto &weight_ : weights[i]) { - weights_.push_back(weight[0]); + weights_.push_back(weight_[0]); } double lb[2], ub[2]; - priors[0].probabilityOfLessLikelySamples(calculation, - weightStyles[i], - sample, TDouble4Vec1Vec{weights_}, - lb[0], ub[0], expectedTail[0]); - priors[1].probabilityOfLessLikelySamples(calculation, - weightStyles[i], - TDouble1Vec{trend.detrend(time, sample[0], confidence)}, - TDouble4Vec1Vec{weights_}, - lb[1], ub[1], expectedTail[1]); + models[0].prior().probabilityOfLessLikelySamples( + calculation, + weightStyles[i], + sample, {weights_}, + lb[0], ub[0], expectedTail[0]); + models[1].prior().probabilityOfLessLikelySamples( + calculation, + weightStyles[i], + {models[1].trend().detrend(time, sample[0], confidence)}, + {weights_}, + lb[1], ub[1], expectedTail[1]); expectedProbability[0] = (lb[0] + ub[0]) / 2.0; expectedProbability[1] = (lb[1] + ub[1]) / 2.0; } @@ -1239,15 +1295,15 @@ void CTimeSeriesModelTest::testProbability(void) maths::CModelProbabilityParams params; params.addCalculation(calculation) .seasonalConfidenceInterval(confidence) - .addBucketEmpty(TBool2Vec{empty}) + .addBucketEmpty({empty}) .weightStyles(weightStyles[i]) .addWeights(weights[i]); bool conditional; TSize1Vec mostAnomalousCorrelate; - models[0].probability(params, time_, TDouble2Vec1Vec{sample}, + models[0].probability(params, time_, {sample}, probability[0], tail[0], conditional, mostAnomalousCorrelate); - models[1].probability(params, time_, TDouble2Vec1Vec{sample}, + models[1].probability(params, time_, {sample}, probability[1], tail[1], conditional, mostAnomalousCorrelate); } @@ -1264,12 +1320,13 @@ void CTimeSeriesModelTest::testProbability(void) LOG_DEBUG("Multivariate"); { - TDecompositionPtr10Vec trends{TDecompositionPtr{new maths::CTimeSeriesDecomposition{24.0 * DECAY_RATE, bucketLength}}, - TDecompositionPtr{new maths::CTimeSeriesDecomposition{24.0 * DECAY_RATE, bucketLength}}, - TDecompositionPtr{new maths::CTimeSeriesDecomposition{24.0 * DECAY_RATE, bucketLength}}}; - maths::CMultivariateNormalConjugate<3> priors[]{multivariateNormal(), multivariateNormal()}; - maths::CMultivariateTimeSeriesModel models[]{maths::CMultivariateTimeSeriesModel{params(bucketLength), *trends[0], priors[0], 0, false}, - maths::CMultivariateTimeSeriesModel{params(bucketLength), *trends[1], priors[1], 0, false}}; + maths::CMultivariateTimeSeriesModel models[]{ + maths::CMultivariateTimeSeriesModel{params(bucketLength), + maths::CTimeSeriesDecompositionStub{}, + multivariateNormal(), 0, false}, + maths::CMultivariateTimeSeriesModel{params(bucketLength), + maths::CTimeSeriesDecomposition{24.0 * DECAY_RATE, bucketLength}, + multivariateNormal(), 0, false}}; TDoubleVecVec samples; { @@ -1279,62 +1336,42 @@ void CTimeSeriesModelTest::testProbability(void) } core_t::TTime time{0}; + const TDouble2Vec4VecVec weight{maths::CConstantWeights::unit(3)}; for (auto &&sample : samples) { - static const TDouble2Vec4VecVec UNIT{maths::CConstantWeights::unit(3)}; - maths::CModelAddSamplesParams params; params.integer(false) .propagationInterval(1.0) .weightStyles(maths::CConstantWeights::COUNT) - .trendWeights(UNIT) - .priorWeights(UNIT); - - double season{5.0 + 5.0 * ::sin( boost::math::double_constants::two_pi - * static_cast(time) / 86400.0)}; + .trendWeights(weight) + .priorWeights(weight); - priors[0].addSamples(maths::CConstantWeights::COUNT, - TDouble10Vec1Vec{TDouble10Vec(sample)}, - maths::CConstantWeights::singleUnit(3)); - priors[0].propagateForwardsByTime(1.0); - models[0].addSamples(params, TTimeDouble2VecSizeTrVec{ - core::make_triple(time, TDouble2Vec(sample), TAG)}); + TDouble2Vec sample_(sample); + models[0].addSamples(params, {core::make_triple(time, sample_, TAG)}); - bool reinitialize{false}; - TDouble10Vec detrended; - for (std::size_t i = 0u; i < sample.size(); ++i) - { - sample[i] += season; - reinitialize |= trends[i]->addPoint(time, sample[i]); - detrended.push_back(trends[i]->detrend(time, sample[i], 0.0)); - } - if (reinitialize) + double trend{5.0 + 5.0 * ::sin( boost::math::double_constants::two_pi + * static_cast(time) / 86400.0)}; + for (auto &&component : sample_) { - maths::initializePrior(bucketLength, models[1].params().learnRate(), trends, priors[1]); + component += trend; } - priors[1].addSamples(maths::CConstantWeights::COUNT, - TDouble10Vec1Vec{detrended}, - maths::CConstantWeights::singleUnit(3)); - priors[1].propagateForwardsByTime(1.0); - models[1].addSamples(params, TTimeDouble2VecSizeTrVec{ - core::make_triple(time, TDouble2Vec(sample), TAG)}); + + models[1].addSamples(params, {core::make_triple(time, sample_, TAG)}); time += bucketLength; } - TTime2Vec1Vec time_{TTime2Vec{time}}; + TTime2Vec1Vec time_{{time}}; TDouble2Vec sample{15.0, 14.0, 16.0}; maths_t::EProbabilityCalculation calculations[]{maths_t::E_TwoSided, maths_t::E_OneSidedAbove}; double confidences[]{0.0, 20.0, 50.0}; bool empties[]{true, false}; - maths_t::TWeightStyleVec weightStyles[]{maths_t::TWeightStyleVec{maths_t::E_SampleCountVarianceScaleWeight}, - maths_t::TWeightStyleVec{maths_t::E_SampleCountVarianceScaleWeight, - maths_t::E_SampleSeasonalVarianceScaleWeight}}; - TDouble2Vec4Vec weights[]{TDouble2Vec4Vec{TDouble2Vec{0.9, 0.9, 0.8}}, - TDouble2Vec4Vec{TDouble2Vec{1.1, 1.0, 1.2}, - TDouble2Vec{1.8, 1.7, 1.6}}}; + maths_t::TWeightStyleVec weightStyles[]{{maths_t::E_SampleCountVarianceScaleWeight}, + {maths_t::E_SampleCountVarianceScaleWeight, + maths_t::E_SampleSeasonalVarianceScaleWeight}}; + TDouble2Vec4Vec weights[]{{{0.9, 0.9, 0.8}}, {{1.1, 1.0, 1.2}, {1.8, 1.7, 1.6}}}; for (auto calculation : calculations) { @@ -1352,26 +1389,26 @@ void CTimeSeriesModelTest::testProbability(void) TTail10Vec expectedTail[2]; { TDouble10Vec4Vec weights_; - for (const auto &weight : weights[i]) + for (const auto &weight_ : weights[i]) { - weights_.push_back(weight); + weights_.push_back(weight_); } double lb[2], ub[2]; - priors[0].probabilityOfLessLikelySamples(calculation, - weightStyles[i], - TDouble10Vec1Vec{TDouble10Vec(sample)}, - TDouble10Vec4Vec1Vec{weights_}, - lb[0], ub[0], expectedTail[0]); + models[0].prior().probabilityOfLessLikelySamples(calculation, + weightStyles[i], + {TDouble10Vec(sample)}, + {weights_}, + lb[0], ub[0], expectedTail[0]); TDouble10Vec detrended; for (std::size_t j = 0u; j < sample.size(); ++j) { - detrended.push_back(trends[j]->detrend(time, sample[j], confidence)); + detrended.push_back(models[1].trend()[j]->detrend(time, sample[j], confidence)); } - priors[1].probabilityOfLessLikelySamples(calculation, - weightStyles[i], - TDouble10Vec1Vec{detrended}, - TDouble10Vec4Vec1Vec{weights_}, - lb[1], ub[1], expectedTail[1]); + models[1].prior().probabilityOfLessLikelySamples(calculation, + weightStyles[i], + {detrended}, + {weights_}, + lb[1], ub[1], expectedTail[1]); expectedProbability[0] = (lb[0] + ub[0]) / 2.0; expectedProbability[1] = (lb[1] + ub[1]) / 2.0; } @@ -1382,15 +1419,15 @@ void CTimeSeriesModelTest::testProbability(void) maths::CModelProbabilityParams params; params.addCalculation(calculation) .seasonalConfidenceInterval(confidence) - .addBucketEmpty(TBool2Vec{empty}) + .addBucketEmpty({empty}) .weightStyles(weightStyles[i]) .addWeights(weights[i]); bool conditional; TSize1Vec mostAnomalousCorrelate; - models[0].probability(params, time_, TDouble2Vec1Vec{sample}, + models[0].probability(params, time_, {sample}, probability[0], tail[0], conditional, mostAnomalousCorrelate); - models[1].probability(params, time_, TDouble2Vec1Vec{sample}, + models[1].probability(params, time_, {sample}, probability[1], tail[1], conditional, mostAnomalousCorrelate); } @@ -1438,22 +1475,20 @@ void CTimeSeriesModelTest::testProbability(void) .weightStyles(maths::CConstantWeights::COUNT) .trendWeights(weights) .priorWeights(weights); - model.addSamples(params, TTimeDouble2VecSizeTrVec{core::make_triple(time, TDouble2Vec{sample}, TAG)}); + model.addSamples(params, {core::make_triple(time, TDouble2Vec{sample}, TAG)}); } { maths::CModelProbabilityParams params; params.addCalculation(maths_t::E_TwoSided) .seasonalConfidenceInterval(50.0) - .addBucketEmpty(TBool2Vec{false}) + .addBucketEmpty({false}) .weightStyles(maths::CConstantWeights::COUNT) .addWeights(weight); TTail2Vec tail; double probability; bool conditional; TSize1Vec mostAnomalousCorrelate; - model.probability(params, - TTime2Vec1Vec{TTime2Vec{time}}, - TDouble2Vec1Vec{TDouble2Vec{sample}}, + model.probability(params, {{time}}, {{sample}}, probability, tail, conditional, mostAnomalousCorrelate); smallest.add({probability, bucket - 1}); @@ -1492,22 +1527,14 @@ void CTimeSeriesModelTest::testWeights(void) TDoubleVec samples; rng.generateNormalSamples(0.0, 4.0, 1008, samples); - TDouble2Vec4VecVec weights{TDouble2Vec4Vec{TDouble2Vec{1.0}}}; + TDouble2Vec4VecVec weights{{{1.0}}}; core_t::TTime time{0}; for (auto sample : samples) { double scale{10.0 + 5.0 * ::sin( boost::math::double_constants::two_pi - * static_cast(time) / 86400.0)}; + * static_cast(time) / 86400.0)}; sample = scale * (1.0 + 0.1 * sample); - if (trend.addPoint(time, sample)) - { - maths::initializePrior(bucketLength, model.params().learnRate(), trend, prior); - } - prior.addSamples(maths::CConstantWeights::COUNT, - TDouble1Vec{trend.detrend(time, sample, 0.0)}, - maths::CConstantWeights::SINGLE_UNIT); - maths::CModelAddSamplesParams params; params.integer(false) .propagationInterval(1.0) @@ -1516,6 +1543,20 @@ void CTimeSeriesModelTest::testWeights(void) .priorWeights(weights); model.addSamples(params, {core::make_triple(time, TDouble2Vec{sample}, TAG)}); + if (trend.addPoint(time, sample)) + { + prior.setToNonInformative(0.0, DECAY_RATE); + for (const auto &value : model.slidingWindow()) + { + prior.addSamples(maths::CConstantWeights::COUNT, + {trend.detrend(value.first, value.second, 0.0)}, + maths::CConstantWeights::SINGLE_UNIT); + } + } + prior.addSamples(maths::CConstantWeights::COUNT, + {trend.detrend(time, sample, 0.0)}, + maths::CConstantWeights::SINGLE_UNIT); + time += bucketLength; } @@ -1534,10 +1575,10 @@ void CTimeSeriesModelTest::testWeights(void) << " (data weight = " << dataScale << ")"); CPPUNIT_ASSERT_EQUAL(std::max(expectedScale, MINIMUM_SEASONAL_SCALE), scale); - error.add(::fabs(scale - dataScale) / dataScale); + error.add(std::fabs(scale - dataScale) / dataScale); } LOG_DEBUG("error = " << maths::CBasicStatistics::mean(error)); - CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.17); + CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.2); LOG_DEBUG("Winsorisation"); TDouble2Vec prediction(model.predict(time)); @@ -1554,6 +1595,7 @@ void CTimeSeriesModelTest::testWeights(void) LOG_DEBUG("Multivariate"); { + double learnRate{params(bucketLength).learnRate()}; TDecompositionPtr10Vec trends{TDecompositionPtr{new maths::CTimeSeriesDecomposition{DECAY_RATE, bucketLength}}, TDecompositionPtr{new maths::CTimeSeriesDecomposition{DECAY_RATE, bucketLength}}, TDecompositionPtr{new maths::CTimeSeriesDecomposition{DECAY_RATE, bucketLength}}}; @@ -1564,11 +1606,11 @@ void CTimeSeriesModelTest::testWeights(void) { TDoubleVec mean{10.0, 15.0, 11.0}; TDoubleVecVec covariance{{3.0, 2.9, 0.5}, {2.9, 2.6, 0.1}, {0.5, 0.1, 2.0}}; - rng.generateMultivariateNormalSamples(mean, covariance, 1000, samples); + rng.generateMultivariateNormalSamples(mean, covariance, 1008, samples); } - TDouble10Vec4Vec1Vec weight{TDouble10Vec4Vec{TDouble10Vec{1.0, 1.0, 1.0}}}; - TDouble2Vec4VecVec weights{TDouble2Vec4Vec{TDouble2Vec{1.0, 1.0, 1.0}}}; + TDouble10Vec4Vec1Vec weight{{{1.0, 1.0, 1.0}}}; + TDouble2Vec4VecVec weights{{{1.0, 1.0, 1.0}}}; core_t::TTime time{0}; for (auto &&sample : samples) { @@ -1585,7 +1627,7 @@ void CTimeSeriesModelTest::testWeights(void) } if (reinitialize) { - maths::initializePrior(bucketLength, model.params().learnRate(), trends, prior); + reinitializePrior(learnRate, model, trends, prior); } prior.addSamples(maths::CConstantWeights::COUNT, detrended, weight); @@ -1615,11 +1657,11 @@ void CTimeSeriesModelTest::testWeights(void) << ", weight = " << scale << " (data weight = " << dataScale << ")"); CPPUNIT_ASSERT_EQUAL(std::max(expectedScale, MINIMUM_SEASONAL_SCALE), scale); - error.add(::fabs(scale - dataScale) / dataScale); + error.add(std::fabs(scale - dataScale) / dataScale); } } LOG_DEBUG("error = " << maths::CBasicStatistics::mean(error)); - CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.23); + CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.21); LOG_DEBUG("Winsorisation"); TDouble2Vec prediction(model.predict(time)); @@ -1658,7 +1700,7 @@ void CTimeSeriesModelTest::testMemoryUsage(void) TDoubleVec samples; rng.generateNormalSamples(1.0, 4.0, 1000, samples); - TDouble2Vec4Vec weight{TDouble2Vec{1.0}}; + TDouble2Vec4Vec weight{{1.0}}; TDouble2Vec4VecVec weights{weight}; core_t::TTime time{0}; for (auto sample : samples) @@ -1755,7 +1797,7 @@ void CTimeSeriesModelTest::testPersist(void) TDoubleVec samples; rng.generateNormalSamples(1.0, 4.0, 1000, samples); - TDouble2Vec4Vec weight{TDouble2Vec{1.0}}; + TDouble2Vec4Vec weight{{1.0}}; TDouble2Vec4VecVec weights{weight}; core_t::TTime time{0}; for (auto sample : samples) @@ -1847,6 +1889,134 @@ void CTimeSeriesModelTest::testPersist(void) // TODO LOG_DEBUG("Correlates"); } +void CTimeSeriesModelTest::testUpgrade(void) +{ + LOG_DEBUG("+-------------------------------------+"); + LOG_DEBUG("| CTimeSeriesModelTest::testUpgrade |"); + LOG_DEBUG("+-------------------------------------+"); + + using TStrVec = std::vector; + auto load = [](const std::string &name, std::string &result) + { + std::ifstream file; + file.open(name); + std::stringbuf buf; + file >> &buf; + result = buf.str(); + }; + + core_t::TTime bucketLength{600}; + core_t::TTime halfHour{1800}; + maths::CModelParams params_{params(bucketLength)}; + std::string empty; + + LOG_DEBUG("Univariate"); + { + std::string xml; + load("testfiles/CUnivariateTimeSeriesModel.6.2.state.xml", xml); + LOG_DEBUG("Saved state size = " << xml.size()); + + std::string intervals; + load("testfiles/CUnivariateTimeSeriesModel.6.2.expected_intervals.txt", intervals); + LOG_DEBUG("Expected intervals size = " << intervals.size()); + TStrVec expectedIntervals; + core::CStringUtils::tokenise(";", intervals, expectedIntervals, empty); + + core::CRapidXmlParser parser; + CPPUNIT_ASSERT(parser.parseStringIgnoreCdata(xml)); + core::CRapidXmlStateRestoreTraverser traverser(parser); + + maths::STimeSeriesDecompositionRestoreParams decompositionParams{24.0 * DECAY_RATE, + bucketLength, + maths::CTimeSeriesDecomposition::DEFAULT_COMPONENT_SIZE}; + maths::SDistributionRestoreParams distributionParams{maths_t::E_ContinuousData, DECAY_RATE, 0.5, 24.0, 12}; + maths::SModelRestoreParams restoreParams{params_, decompositionParams, distributionParams}; + maths::CUnivariateTimeSeriesModel restoredModel{restoreParams, traverser}; + + TStrVec expectedInterval; + TStrVec interval; + for (core_t::TTime time = 600000, i = 0; + i < static_cast(expectedIntervals.size()); + time += halfHour, ++i) + { + expectedInterval.clear(); + interval.clear(); + + core::CStringUtils::tokenise(",", expectedIntervals[i], expectedInterval, empty); + std::string interval_{core::CContainerPrinter::print( + restoredModel.confidenceInterval(time, 90.0, + maths::CConstantWeights::COUNT, + maths::CConstantWeights::unit(1)))}; + core::CStringUtils::replace("[", "", interval_); + core::CStringUtils::replace("]", "", interval_); + core::CStringUtils::replace(" ", "", interval_); + interval_ += ","; + core::CStringUtils::tokenise(",", interval_, interval, empty); + + CPPUNIT_ASSERT_EQUAL(expectedInterval.size(), interval.size()); + for (std::size_t j = 0u; j < expectedInterval.size(); ++j) + { + CPPUNIT_ASSERT_DOUBLES_EQUAL(boost::lexical_cast(expectedInterval[j]), + boost::lexical_cast(interval[j]), + 0.0001); + } + } + } + + LOG_DEBUG("Multivariate"); + { + std::string xml; + load("testfiles/CMultivariateTimeSeriesModel.6.2.state.xml", xml); + LOG_DEBUG("Saved state size = " << xml.size()); + + std::string intervals; + load("testfiles/CMultivariateTimeSeriesModel.6.2.expected_intervals.txt", intervals); + LOG_DEBUG("Expected intervals size = " << intervals.size()); + TStrVec expectedIntervals; + core::CStringUtils::tokenise(";", intervals, expectedIntervals, empty); + + core::CRapidXmlParser parser; + CPPUNIT_ASSERT(parser.parseStringIgnoreCdata(xml)); + core::CRapidXmlStateRestoreTraverser traverser(parser); + + maths::STimeSeriesDecompositionRestoreParams decompositionParams{24.0 * DECAY_RATE, + bucketLength, + maths::CTimeSeriesDecomposition::DEFAULT_COMPONENT_SIZE}; + maths::SDistributionRestoreParams distributionParams{maths_t::E_ContinuousData, DECAY_RATE, 0.5, 24.0, 12}; + maths::SModelRestoreParams restoreParams{params_, decompositionParams, distributionParams}; + maths::CMultivariateTimeSeriesModel restoredModel{restoreParams, traverser}; + + TStrVec expectedInterval; + TStrVec interval; + for (core_t::TTime time = 600000, i = 0; + i < static_cast(expectedIntervals.size()); + time += halfHour, ++i) + { + expectedInterval.clear(); + interval.clear(); + + core::CStringUtils::tokenise(",", expectedIntervals[i], expectedInterval, empty); + std::string interval_{core::CContainerPrinter::print( + restoredModel.confidenceInterval(time, 90.0, + maths::CConstantWeights::COUNT, + maths::CConstantWeights::unit(3)))}; + core::CStringUtils::replace("[", "", interval_); + core::CStringUtils::replace("]", "", interval_); + core::CStringUtils::replace(" ", "", interval_); + interval_ += ","; + core::CStringUtils::tokenise(",", interval_, interval, empty); + + CPPUNIT_ASSERT_EQUAL(expectedInterval.size(), interval.size()); + for (std::size_t j = 0u; j < expectedInterval.size(); ++j) + { + CPPUNIT_ASSERT_DOUBLES_EQUAL(boost::lexical_cast(expectedInterval[j]), + boost::lexical_cast(interval[j]), + 0.0001); + } + } + } +} + void CTimeSeriesModelTest::testAddSamplesWithCorrelations(void) { LOG_DEBUG("+--------------------------------------------------------+"); @@ -1874,7 +2044,7 @@ void CTimeSeriesModelTest::testAddSamplesWithCorrelations(void) models[1].modelCorrelations(correlations); CTimeSeriesCorrelateModelAllocator allocator; - TDouble2Vec4VecVec weights{TDouble2Vec4Vec{TDouble2Vec{1.0}}}; + TDouble2Vec4VecVec weights{{{1.0}}}; core_t::TTime time{0}; for (auto sample : samples) { @@ -1955,24 +2125,21 @@ void CTimeSeriesModelTest::testAnomalyModel(void) .weightStyles(maths::CConstantWeights::COUNT) .trendWeights(weights) .priorWeights(weights); - model.addSamples(params, TTimeDouble2VecSizeTrVec{core::make_triple(time, TDouble2Vec{sample}, TAG)}); + model.addSamples(params, {core::make_triple(time, TDouble2Vec{sample}, TAG)}); } { maths::CModelProbabilityParams params; params.addCalculation(maths_t::E_TwoSided) .seasonalConfidenceInterval(50.0) - .addBucketEmpty(TBool2Vec{false}) + .addBucketEmpty({false}) .weightStyles(maths::CConstantWeights::COUNT) .addWeights(weight); TTail2Vec tail; double probability; bool conditional; TSize1Vec mostAnomalousCorrelate; - model.probability(params, - TTime2Vec1Vec{TTime2Vec{time}}, - TDouble2Vec1Vec{TDouble2Vec{sample}}, - probability, tail, - conditional, mostAnomalousCorrelate); + model.probability(params, {{time}}, {{sample}}, + probability, tail, conditional, mostAnomalousCorrelate); mostAnomalous.add({::log(probability), bucket}); //scores.push_back(maths::CTools::deviation(probability)); } @@ -2059,7 +2226,7 @@ void CTimeSeriesModelTest::testAnomalyModel(void) maths::CModelProbabilityParams params; params.addCalculation(maths_t::E_TwoSided) .seasonalConfidenceInterval(50.0) - .addBucketEmpty(TBool2Vec{false}) + .addBucketEmpty({false}) .weightStyles(maths::CConstantWeights::COUNT) .addWeights(weight); TTail2Vec tail; @@ -2067,8 +2234,8 @@ void CTimeSeriesModelTest::testAnomalyModel(void) bool conditional; TSize1Vec mostAnomalousCorrelate; model.probability(params, - TTime2Vec1Vec{TTime2Vec{time}}, - TDouble2Vec1Vec{TDouble2Vec(sample)}, + {{time}}, + {(sample)}, probability, tail, conditional, mostAnomalousCorrelate); mostAnomalous.add({::log(probability), bucket}); @@ -2088,8 +2255,6 @@ void CTimeSeriesModelTest::testAnomalyModel(void) } LOG_DEBUG("anomalies = " << core::CContainerPrinter::print(anomalyBuckets)); LOG_DEBUG("probabilities = " << core::CContainerPrinter::print(anomalyProbabilities)); - CPPUNIT_ASSERT(std::find(anomalyBuckets.begin(), - anomalyBuckets.end(), 1907) != anomalyBuckets.end()); CPPUNIT_ASSERT(std::find(anomalyBuckets.begin(), anomalyBuckets.end(), 1908) != anomalyBuckets.end()); @@ -2143,6 +2308,9 @@ CppUnit::Test *CTimeSeriesModelTest::suite(void) suiteOfTests->addTest( new CppUnit::TestCaller( "CTimeSeriesModelTest::testPersist", &CTimeSeriesModelTest::testPersist) ); + suiteOfTests->addTest( new CppUnit::TestCaller( + "CTimeSeriesModelTest::testUpgrade", + &CTimeSeriesModelTest::testUpgrade) ); suiteOfTests->addTest( new CppUnit::TestCaller( "CTimeSeriesModelTest::testAddSamplesWithCorrelations", &CTimeSeriesModelTest::testAddSamplesWithCorrelations) ); diff --git a/lib/maths/unittest/CTimeSeriesModelTest.h b/lib/maths/unittest/CTimeSeriesModelTest.h index ac689d00f0..bc25a7754b 100644 --- a/lib/maths/unittest/CTimeSeriesModelTest.h +++ b/lib/maths/unittest/CTimeSeriesModelTest.h @@ -30,6 +30,7 @@ class CTimeSeriesModelTest : public CppUnit::TestFixture void testWeights(void); void testMemoryUsage(void); void testPersist(void); + void testUpgrade(void); void testAddSamplesWithCorrelations(void); void testProbabilityWithCorrelations(void); void testAnomalyModel(void); diff --git a/lib/maths/unittest/CTrendComponentTest.cc b/lib/maths/unittest/CTrendComponentTest.cc new file mode 100644 index 0000000000..148a2ba62c --- /dev/null +++ b/lib/maths/unittest/CTrendComponentTest.cc @@ -0,0 +1,477 @@ +/* + * ELASTICSEARCH CONFIDENTIAL + * + * Copyright (c) 2017 Elasticsearch BV. All Rights Reserved. + * + * Notice: this software, and all information contained + * therein, is the exclusive property of Elasticsearch BV + * and its licensors, if any, and is protected under applicable + * domestic and foreign law, and international treaties. + * + * Reproduction, republication or distribution without the + * express written consent of Elasticsearch BV is + * strictly prohibited. + */ + +#include "CTrendComponentTest.h" + +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include + +#include +#include + +using namespace ml; + +namespace +{ + +using TDoubleVec = std::vector; +using TDoubleVecVec = std::vector; +using TDouble1Vec = core::CSmallVector; +using TDouble3Vec = core::CSmallVector; +using TDouble3VecVec = std::vector; +using TGenerator = TDoubleVec (*)(test::CRandomNumbers &, + core_t::TTime, core_t::TTime, core_t::TTime); +using TMeanAccumulator = maths::CBasicStatistics::SSampleMean::TAccumulator; +using TMeanVarAccumulator = maths::CBasicStatistics::SSampleMeanVar::TAccumulator; +using TRegression = maths::CRegression::CLeastSquaresOnline<2, double>; + +TDoubleVec multiscaleRandomWalk(test::CRandomNumbers &rng, + core_t::TTime bucketLength, + core_t::TTime start, core_t::TTime end) +{ + TDoubleVecVec noise(4); + + core_t::TTime buckets{(end - start) / bucketLength + 1}; + rng.generateNormalSamples(0.0, 0.2, buckets, noise[0]); + rng.generateNormalSamples(0.0, 0.5, buckets, noise[1]); + rng.generateNormalSamples(0.0, 1.0, buckets, noise[2]); + rng.generateNormalSamples(0.0, 5.0, buckets, noise[3]); + for (core_t::TTime i = 1; i < buckets; ++i) + { + noise[0][i] = 0.998 * noise[0][i-1] + 0.002 * noise[0][i]; + noise[1][i] = 0.99 * noise[1][i-1] + 0.01 * noise[1][i]; + noise[2][i] = 0.9 * noise[2][i-1] + 0.1 * noise[2][i]; + } + + TDoubleVec result; + result.reserve(buckets); + + TDoubleVec rw{0.0, 0.0, 0.0}; + for (core_t::TTime i = 0; i < buckets; ++i) + { + rw[0] = rw[0] + noise[0][i]; + rw[1] = rw[1] + noise[1][i]; + rw[2] = rw[2] + noise[2][i]; + double ramp{0.05 * static_cast(i)}; + result.push_back(ramp + rw[0] + rw[1] + rw[2] + noise[3][i]); + } + + return result; +} + +TDoubleVec piecewiseLinear(test::CRandomNumbers &rng, + core_t::TTime bucketLength, + core_t::TTime start, core_t::TTime end) +{ + core_t::TTime buckets{(end - start) / bucketLength + 1}; + + TDoubleVec knots; + rng.generateUniformSamples(100.0, 500.0, buckets / 200, knots); + knots.insert(knots.begin(), 0.0); + std::partial_sum(knots.begin(), knots.end(), knots.begin()); + + TDoubleVec slopes; + rng.generateUniformSamples(-1.0, 2.0, knots.size(), slopes); + + TDoubleVec result; + result.reserve(buckets); + + double value{0.0}; + + auto knot = knots.begin(); + auto slope = slopes.begin(); + for (core_t::TTime time = start; time < end; time += bucketLength) + { + if (time > start + static_cast(bucketLength * *knot)) + { + ++knot; + ++slope; + } + value += *slope; + result.push_back(value); + } + + return result; +} + +TDoubleVec staircase(test::CRandomNumbers &rng, + core_t::TTime bucketLength, + core_t::TTime start, core_t::TTime end) +{ + core_t::TTime buckets{(end - start) / bucketLength + 1}; + + TDoubleVec knots; + rng.generateUniformSamples(200.0, 400.0, buckets / 200, knots); + knots.insert(knots.begin(), 0.0); + std::partial_sum(knots.begin(), knots.end(), knots.begin()); + + TDoubleVec steps; + rng.generateUniformSamples(1.0, 20.0, knots.size(), steps); + + TDoubleVec result; + result.reserve(buckets); + + double value{0.0}; + + auto knot = knots.begin(); + auto step = steps.begin(); + for (core_t::TTime time = start; time < end; time += bucketLength) + { + if (time > start + static_cast(bucketLength * *knot)) + { + value += *step; + ++knot; + ++step; + } + result.push_back(value); + } + + return result; +} + +TDoubleVec switching(test::CRandomNumbers &rng, + core_t::TTime bucketLength, + core_t::TTime start, core_t::TTime end) +{ + core_t::TTime buckets{(end - start) / bucketLength + 1}; + + TDoubleVec knots; + rng.generateUniformSamples(400.0, 800.0, buckets / 400, knots); + knots.insert(knots.begin(), 0.0); + std::partial_sum(knots.begin(), knots.end(), knots.begin()); + + TDoubleVec steps; + rng.generateUniformSamples(-10.0, 10.0, knots.size(), steps); + + TDoubleVec result; + result.reserve(buckets); + + double value{0.0}; + + auto knot = knots.begin(); + auto step = steps.begin(); + for (core_t::TTime time = start; time < end; time += bucketLength) + { + if (time > start + static_cast(bucketLength * *knot)) + { + value += *step; + ++knot; + ++step; + } + result.push_back(value); + } + + return result; +} + +} + +void CTrendComponentTest::testValueAndVariance() +{ + LOG_DEBUG("+---------------------------------------------+"); + LOG_DEBUG("| CTrendComponentTest::testValueAndVariance |"); + LOG_DEBUG("+---------------------------------------------+"); + + // Check that the prediction bias is small in the long run + // and that the predicted variance approximately matches the + // variance observed in prediction errors. + + test::CRandomNumbers rng; + + core_t::TTime bucketLength{600}; + core_t::TTime start{1000000}; + core_t::TTime end{3000000}; + + TDoubleVec values(multiscaleRandomWalk(rng, bucketLength, start, end)); + + maths::CTrendComponent component{0.012}; + maths::CDecayRateController controller( maths::CDecayRateController::E_PredictionBias + | maths::CDecayRateController::E_PredictionErrorIncrease, 1); + + TMeanVarAccumulator normalisedResiduals; + for (core_t::TTime time = start; time < end; time += bucketLength) + { + double value{values[(time - start) / bucketLength]}; + double prediction{maths::CBasicStatistics::mean(component.value(time, 0.0))}; + + if (time > start + bucketLength) + { + double variance{maths::CBasicStatistics::mean(component.variance(0.0))}; + normalisedResiduals.add((value - prediction) / std::sqrt(variance)); + } + + component.add(time, value); + controller.multiplier({prediction}, + {{values[(time - start) / bucketLength] - prediction}}, + bucketLength, 1.0, 0.012); + component.decayRate(0.012 * controller.multiplier()); + component.propagateForwardsByTime(bucketLength); + } + + LOG_DEBUG("normalised error moments = " << normalisedResiduals); + CPPUNIT_ASSERT(std::fabs(maths::CBasicStatistics::mean(normalisedResiduals)) < 0.5); + CPPUNIT_ASSERT(std::fabs(maths::CBasicStatistics::variance(normalisedResiduals) - 1.0) < 0.2); +} + +void CTrendComponentTest::testDecayRate() +{ + LOG_DEBUG("+--------------------------------------+"); + LOG_DEBUG("| CTrendComponentTest::testDecayRate |"); + LOG_DEBUG("+--------------------------------------+"); + + // Test that the trend short range predictions approximately + // match a regression model with the same decay rate. + + test::CRandomNumbers rng; + + core_t::TTime bucketLength{600}; + core_t::TTime start{0}; + core_t::TTime end{3000000}; + + //std::ofstream file; + //file.open("results.m"); + //TDoubleVec predictions; + //TDoubleVec expectedPredictions; + + TDoubleVec values(multiscaleRandomWalk(rng, bucketLength, start, end)); + + maths::CTrendComponent component{0.012}; + TRegression regression; + maths::CDecayRateController controller( maths::CDecayRateController::E_PredictionBias + | maths::CDecayRateController::E_PredictionErrorIncrease, 1); + + TMeanAccumulator error; + TMeanAccumulator level; + for (core_t::TTime time = start; time < end; time += bucketLength) + { + double value{values[(time - start) / bucketLength]}; + component.add(time, value); + regression.add(time / 604800.0, value); + + double expectedPrediction{regression.predict(time / 604800.0)}; + double prediction{maths::CBasicStatistics::mean(component.value(time, 0.0))}; + error.add(std::fabs(prediction - expectedPrediction)); + level.add(value); + + controller.multiplier({prediction}, + {{values[(time - start) / bucketLength] - prediction}}, + bucketLength, 1.0, 0.012); + component.decayRate(0.012 * controller.multiplier()); + component.propagateForwardsByTime(bucketLength); + regression.age(std::exp(-0.012 * controller.multiplier() * 600.0 / 86400.0)); + + //predictions.push_back(prediction); + //expectedPredictions.push_back(expectedPrediction); + } + + double relativeError{ maths::CBasicStatistics::mean(error) + / std::fabs(maths::CBasicStatistics::mean(level))}; + LOG_DEBUG("relative error = " << relativeError); + + //file << "f = " << core::CContainerPrinter::print(values) << ";" << std::endl; + //file << "p = " << core::CContainerPrinter::print(predictions) << ";" << std::endl; + //file << "pe = " << core::CContainerPrinter::print(expectedPredictions) << ";" << std::endl; +} + +void CTrendComponentTest::testForecast() +{ + LOG_DEBUG("+-------------------------------------+"); + LOG_DEBUG("| CTrendComponentTest::testForecast |"); + LOG_DEBUG("+-------------------------------------+"); + + // Check the forecast errors for a variety of signals. + + test::CRandomNumbers rng; + + auto testForecast = [&rng](TGenerator generate, + core_t::TTime start, + core_t::TTime end) + { + //std::ofstream file; + //file.open("results.m"); + //TDoubleVec predictions; + //TDoubleVec forecastPredictions; + //TDoubleVec forecastLower; + //TDoubleVec forecastUpper; + + core_t::TTime bucketLength{600}; + TDoubleVec values(generate(rng, bucketLength, start, end + 1000 * bucketLength)); + + maths::CTrendComponent component{0.012}; + maths::CDecayRateController controller( maths::CDecayRateController::E_PredictionBias + | maths::CDecayRateController::E_PredictionErrorIncrease, 1); + + core_t::TTime time{0}; + for (/**/; time < end; time += bucketLength) + { + component.add(time, values[time / bucketLength]); + component.propagateForwardsByTime(bucketLength); + + double prediction{maths::CBasicStatistics::mean(component.value(time, 0.0))}; + controller.multiplier({prediction}, + {{values[time / bucketLength] - prediction}}, + bucketLength, 0.3, 0.012); + component.decayRate(0.012 * controller.multiplier()); + //predictions.push_back(prediction); + } + + component.shiftOrigin(time); + + TDouble3VecVec forecast; + component.forecast(time, time + 1000 * bucketLength, 3600, 95.0, forecast); + + TMeanAccumulator meanError; + TMeanAccumulator meanErrorAt95; + for (auto &&errorbar : forecast) + { + core_t::TTime bucket{(time - start) / bucketLength}; + meanError.add( std::fabs((values[bucket] - errorbar[1]) + / std::fabs(values[bucket]))); + meanErrorAt95.add( std::max(std::max(values[bucket] - errorbar[2], + errorbar[0] - values[bucket]), 0.0) + / std::fabs(values[bucket])); + //forecastLower.push_back(errorbar[0]); + //forecastPredictions.push_back(errorbar[1]); + //forecastUpper.push_back(errorbar[2]); + } + + //file << "f = " << core::CContainerPrinter::print(values) << ";" << std::endl; + //file << "p = " << core::CContainerPrinter::print(predictions) << ";" << std::endl; + //file << "fl = " << core::CContainerPrinter::print(forecastLower) << ";" << std::endl; + //file << "fm = " << core::CContainerPrinter::print(forecastPredictions) << ";" << std::endl; + //file << "fu = " << core::CContainerPrinter::print(forecastUpper) << ";" << std::endl; + + LOG_DEBUG("error = " << maths::CBasicStatistics::mean(meanError)); + LOG_DEBUG("error @ 95% = " << maths::CBasicStatistics::mean(meanErrorAt95)); + + return std::make_pair(maths::CBasicStatistics::mean(meanError), + maths::CBasicStatistics::mean(meanErrorAt95)); + }; + + double error; + double errorAt95; + + LOG_DEBUG("Random Walk"); + { + boost::tie(error, errorAt95) = testForecast(multiscaleRandomWalk, 0, 3000000); + CPPUNIT_ASSERT(error < 0.16); + CPPUNIT_ASSERT(errorAt95 < 0.001); + } + + LOG_DEBUG("Piecewise Linear"); + { + boost::tie(error, errorAt95) = testForecast(piecewiseLinear, 0, 3200000); + CPPUNIT_ASSERT(error < 0.17); + CPPUNIT_ASSERT(errorAt95 < 0.05); + } + + LOG_DEBUG("Staircase"); + { + boost::tie(error, errorAt95) = testForecast(staircase, 0, 2000000); + CPPUNIT_ASSERT(error < 0.03); + CPPUNIT_ASSERT(errorAt95 < 0.01); + } + + LOG_DEBUG("Switching"); + { + boost::tie(error, errorAt95) = testForecast(switching, 0, 3000000); + CPPUNIT_ASSERT(error < 0.06); + CPPUNIT_ASSERT(errorAt95 < 0.01); + } +} + +void CTrendComponentTest::testPersist() +{ + LOG_DEBUG("+------------------------------------+"); + LOG_DEBUG("| CTrendComponentTest::testPersist |"); + LOG_DEBUG("+------------------------------------+"); + + // Check that serialization is idempotent. + + test::CRandomNumbers rng; + + core_t::TTime bucketLength{600}; + core_t::TTime start{1200}; + core_t::TTime end{200000}; + + TDoubleVec values(multiscaleRandomWalk(rng, bucketLength, start, end)); + + maths::CTrendComponent origComponent{0.012}; + + for (core_t::TTime time = start; time < end; time += bucketLength) + { + double value{values[(time - start) / bucketLength]}; + origComponent.add(time, value); + origComponent.propagateForwardsByTime(bucketLength); + } + + std::string origXml; + { + ml::core::CRapidXmlStatePersistInserter inserter("root"); + origComponent.acceptPersistInserter(inserter); + inserter.toXml(origXml); + } + + LOG_DEBUG("decomposition XML representation:\n" << origXml); + + // Restore the XML into a new filter + core::CRapidXmlParser parser; + CPPUNIT_ASSERT(parser.parseStringIgnoreCdata(origXml)); + core::CRapidXmlStateRestoreTraverser traverser(parser); + + maths::CTrendComponent restoredComponent{0.1}; + traverser.traverseSubLevel(boost::bind(&maths::CTrendComponent::acceptRestoreTraverser, + &restoredComponent, _1)); + + CPPUNIT_ASSERT_EQUAL(origComponent.checksum(), restoredComponent.checksum()); + + std::string newXml; + { + core::CRapidXmlStatePersistInserter inserter("root"); + restoredComponent.acceptPersistInserter(inserter); + inserter.toXml(newXml); + } + CPPUNIT_ASSERT_EQUAL(origXml, newXml); +} + +CppUnit::Test *CTrendComponentTest::suite() +{ + CppUnit::TestSuite *suiteOfTests = new CppUnit::TestSuite("CTrendComponentTest"); + + suiteOfTests->addTest( new CppUnit::TestCaller( + "CTrendComponentTest::testValueAndVariance", + &CTrendComponentTest::testValueAndVariance) ); + suiteOfTests->addTest( new CppUnit::TestCaller( + "CTrendComponentTest::testDecayRate", + &CTrendComponentTest::testDecayRate) ); + suiteOfTests->addTest( new CppUnit::TestCaller( + "CTrendComponentTest::testForecast", + &CTrendComponentTest::testForecast) ); + suiteOfTests->addTest( new CppUnit::TestCaller( + "CTrendComponentTest::testPersist", + &CTrendComponentTest::testPersist) ); + + return suiteOfTests; +} diff --git a/lib/maths/unittest/CTrendComponentTest.h b/lib/maths/unittest/CTrendComponentTest.h new file mode 100644 index 0000000000..225070f2cb --- /dev/null +++ b/lib/maths/unittest/CTrendComponentTest.h @@ -0,0 +1,32 @@ +/* + * ELASTICSEARCH CONFIDENTIAL + * + * Copyright (c) 2017 Elasticsearch BV. All Rights Reserved. + * + * Notice: this software, and all information contained + * therein, is the exclusive property of Elasticsearch BV + * and its licensors, if any, and is protected under applicable + * domestic and foreign law, and international treaties. + * + * Reproduction, republication or distribution without the + * express written consent of Elasticsearch BV is + * strictly prohibited. + */ + +#ifndef INCLUDED_CTrendComponentTest_h +#define INCLUDED_CTrendComponentTest_h + +#include + +class CTrendComponentTest : public CppUnit::TestFixture +{ + public: + void testValueAndVariance(); + void testDecayRate(); + void testForecast(); + void testPersist(); + + static CppUnit::Test *suite(); +}; + +#endif // INCLUDED_CTrendComponentTest_h diff --git a/lib/maths/unittest/CTrendTestsTest.cc b/lib/maths/unittest/CTrendTestsTest.cc index 90c783bebd..3e98e09abb 100644 --- a/lib/maths/unittest/CTrendTestsTest.cc +++ b/lib/maths/unittest/CTrendTestsTest.cc @@ -29,12 +29,11 @@ #include #include +#include "TestUtils.h" + #include -#include #include #include -#include -#include #include @@ -42,330 +41,14 @@ using namespace ml; namespace { - using TDoubleVec = std::vector; using TTimeVec = std::vector; using TTimeDoublePr = std::pair; using TTimeDoublePrVec = std::vector; -using TDiurnalPeriodicityTestPtr = boost::shared_ptr; - -class CDiurnalPeriodicityTestInspector : public maths::CDiurnalPeriodicityTest -{ - public: - CDiurnalPeriodicityTestInspector(const maths::CDiurnalPeriodicityTest &test) : - maths::CDiurnalPeriodicityTest(test) - {} - core_t::TTime bucketLength(void) const - { - return this->maths::CDiurnalPeriodicityTest::bucketLength(); - } -}; - -const core_t::TTime FIVE_MINS = 300; const core_t::TTime HALF_HOUR = core::constants::HOUR / 2; -const core_t::TTime HOUR = core::constants::HOUR; -const core_t::TTime DAY = core::constants::DAY; -const core_t::TTime WEEK = core::constants::WEEK; - -double constant(core_t::TTime /*time*/) -{ - return 0.0; -} - -double ramp(core_t::TTime time) -{ - return 0.1 * static_cast(time) / static_cast(WEEK); -} - -double markov(core_t::TTime time) -{ - static double state = 0.2; - if (time % WEEK == 0) - { - core::CHashing::CMurmurHash2BT hasher; - state = 2.0 * static_cast(hasher(time)) - / static_cast(std::numeric_limits::max()); - } - return state; -} - -double smoothDaily(core_t::TTime time) -{ - return ::sin( boost::math::double_constants::two_pi - * static_cast(time) - / static_cast(DAY)); -} - -double smoothWeekly(core_t::TTime time) -{ - return ::sin( boost::math::double_constants::two_pi - * static_cast(time) - / static_cast(WEEK)); -} - -double spikeyDaily(core_t::TTime time) -{ - double pattern[] = - { - 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, - 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.1 - }; - return pattern[(time % DAY) / HALF_HOUR]; -} - -double spikeyWeekly(core_t::TTime time) -{ - double pattern[] = - { - 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.0, 0.1, 0.1, 0.1, 0.1, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.0, 0.1, 0.1, 0.1, 0.1, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.0, 0.1, 0.1, 0.1, 0.1, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.0, 0.1, 0.1, 0.1, 0.1, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.0, 0.1, 0.1, 0.1, 0.1, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.0, 0.1, 0.1, 0.1, 0.1, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, - 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.1 - }; - return pattern[(time % WEEK) / HALF_HOUR]; -} - -double weekends(core_t::TTime time) -{ - double amplitude[] = { 1.0, 0.9, 0.9, 0.9, 1.0, 0.2, 0.1 }; - return amplitude[(time % WEEK) / DAY] - * ::sin( boost::math::double_constants::two_pi - * static_cast(time) - / static_cast(DAY)); -} - -const TTimeVec BUCKET_LENGTHS_ - { - 1, 5, 10, 30, 60, 300, 600, 1800, 3600, 7200, 21600, 43200, 86400, 172800, 345600, 691200 - }; -const maths::CScanningPeriodicityTest::TTimeCRng BUCKET_LENGTHS(BUCKET_LENGTHS_, 7, 16); - -} - -void CTrendTestsTest::testDiurnalInitialisation(void) -{ - LOG_DEBUG("+----------------------------------------------+"); - LOG_DEBUG("| CTrendTestsTest::testDiurnalInitialisation |"); - LOG_DEBUG("+----------------------------------------------+"); - - TDiurnalPeriodicityTestPtr test( - maths::CDiurnalPeriodicityTest::create(DAY * 2, 0.1)); - CPPUNIT_ASSERT(nullptr == test); - - test.reset(maths::CDiurnalPeriodicityTest::create(DAY, 0.1)); - CPPUNIT_ASSERT(nullptr != test); - CPPUNIT_ASSERT_EQUAL(DAY, CDiurnalPeriodicityTestInspector(*test).bucketLength()); - - test.reset(maths::CDiurnalPeriodicityTest::create(600, 0.1)); - CPPUNIT_ASSERT(nullptr != test); - CPPUNIT_ASSERT_EQUAL(core_t::TTime(3600), - CDiurnalPeriodicityTestInspector(*test).bucketLength()); -} - -void CTrendTestsTest::testTrend(void) -{ - LOG_DEBUG("+------------------------------+"); - LOG_DEBUG("| CTrendTestsTest::testTrend |"); - LOG_DEBUG("+------------------------------+"); - - // Test that there is a high probability that a stationary - // time series tests negative for having a trend and a high - // probability that a time series with a trend tests positive - // for having a trend. - // - // For the sake of concreteness, in the following test we - // assume that the null hypothesis is that the series has - // no trend. In the first test case the time series have - // no trend so we are testing probability of type I error. - // In the second test case the time series have a significant - // trend so we are testing the probability of type II error. - - test::CRandomNumbers rng; - - double pFalsePositive = 0.0; - double meanTimeToDetect = 0.0; - - for (std::size_t trend = 0u; trend < 2; ++trend) - { - LOG_DEBUG("*** trend = " << trend << " ***"); - - int trendCount = 0; - core_t::TTime timeToDetect = 0; - - for (std::size_t t = 0u; t < 100; ++t) - { - TDoubleVec samples; - rng.generateNormalSamples(1.0, 5.0, 600, samples); - double scale = 0.03 * ::sqrt(5.0); - - maths::CTrendTest trendTest(0.001); - - core_t::TTime testTimeToDetect = 7200 * samples.size(); - for (std::size_t i = 0u; i < samples.size(); ++i) - { - core_t::TTime time = static_cast(i) * 7200; - double x = (trend == 0 ? 0.0 : scale * static_cast(i)) + samples[i]; - trendTest.add(time, x); - trendTest.captureVariance(time, x); - trendTest.propagateForwardsByTime(2.0); - if (trendTest.test()) - { - testTimeToDetect = std::min(testTimeToDetect, time); - ++trendCount; - } - } - timeToDetect += testTimeToDetect; - if (trend == 1) - { - CPPUNIT_ASSERT_EQUAL(true, trendTest.test()); - } - } - - if (trend == 0) - { - pFalsePositive = trendCount / 60000.0; - } - if (trend == 1) - { - meanTimeToDetect = timeToDetect / 100; - } - } - - LOG_DEBUG("[P(false positive)] = " << pFalsePositive); - LOG_DEBUG("time to detect = " << meanTimeToDetect); - CPPUNIT_ASSERT(pFalsePositive < 1e-4); - CPPUNIT_ASSERT(meanTimeToDetect < 12 * DAY); - - for (std::size_t trend = 0u; trend < 2; ++trend) - { - LOG_DEBUG("*** trend = " << trend << " ***"); - - std::size_t trendCount = 0u; - core_t::TTime timeToDetect = 0; - - for (std::size_t t = 0u; t < 100; ++t) - { - TDoubleVec samples; - rng.generateGammaSamples(50.0, 2.0, 600, samples); - double scale = 0.03 * ::sqrt(200.0); - - maths::CTrendTest trendTest(0.001); - - core_t::TTime testTimeToDetect = 7200 * samples.size(); - for (std::size_t i = 0u; i < samples.size(); ++i) - { - core_t::TTime time = static_cast(i) * 7200; - double x = (trend == 0 ? 0.0 : scale * static_cast(i)) + samples[i]; - trendTest.add(time, x); - trendTest.captureVariance(time, x); - trendTest.propagateForwardsByTime(2.0); - if (trendTest.test()) - { - testTimeToDetect = std::min(testTimeToDetect, time); - ++trendCount; - } - } - timeToDetect += testTimeToDetect; - if (trend == 1) - { - CPPUNIT_ASSERT_EQUAL(true, trendTest.test()); - } - } - - if (trend == 0) - { - pFalsePositive = trendCount / 60000.0; - } - if (trend == 1) - { - meanTimeToDetect = timeToDetect / 100; - } - } - - LOG_DEBUG("[P(false positive)] = " << pFalsePositive); - LOG_DEBUG("time to detect = " << meanTimeToDetect); - CPPUNIT_ASSERT(pFalsePositive < 1e-4); - CPPUNIT_ASSERT(meanTimeToDetect < 12 * DAY); - - for (std::size_t trend = 0u; trend < 2; ++trend) - { - LOG_DEBUG("*** trend = " << trend << " ***"); - - std::size_t trendCount = 0u; - core_t::TTime timeToDetect = 0; - - for (std::size_t t = 0u; t < 100; ++t) - { - TDoubleVec samples; - rng.generateLogNormalSamples(2.5, 1.0, 600, samples); - double scale = 3.0 * ::sqrt(693.20); - - maths::CTrendTest trendTest(0.001); - - core_t::TTime testTimeToDetect = 7200 * samples.size(); - for (std::size_t i = 0u; i < samples.size(); ++i) - { - core_t::TTime time = static_cast(i) * 7200; - double x = (trend == 0 ? 0.0 : scale * ::sin( boost::math::double_constants::two_pi - * static_cast(i) / 600.0)) + samples[i]; - trendTest.add(time, x); - trendTest.captureVariance(time, x); - trendTest.propagateForwardsByTime(2.0); - if (trendTest.test()) - { - testTimeToDetect = std::min(testTimeToDetect, time); - ++trendCount; - } - } - - timeToDetect += testTimeToDetect; - if (trend == 1) - { - CPPUNIT_ASSERT_EQUAL(true, trendTest.test()); - } - } - - if (trend == 0) - { - pFalsePositive = trendCount / 60000.0; - } - if (trend == 1) - { - meanTimeToDetect = timeToDetect / 100; - } - } - - LOG_DEBUG("[P(false positive)] = " << pFalsePositive); - LOG_DEBUG("time to detect = " << meanTimeToDetect); - CPPUNIT_ASSERT(pFalsePositive < 1e-4); - CPPUNIT_ASSERT(meanTimeToDetect < 23 * DAY); +const core_t::TTime DAY = core::constants::DAY; +const core_t::TTime WEEK = core::constants::WEEK; } void CTrendTestsTest::testRandomizedPeriodicity(void) @@ -475,635 +158,6 @@ void CTrendTestsTest::testRandomizedPeriodicity(void) CPPUNIT_ASSERT(maths::CBasicStatistics::mean(typeII) < 0.05); } -void CTrendTestsTest::testDiurnalPeriodicity(void) -{ - LOG_DEBUG("+-------------------------------------------+"); - LOG_DEBUG("| CTrendTestsTest::testDiurnalPeriodicity |"); - LOG_DEBUG("+-------------------------------------------+"); - - using TResult = maths::CPeriodicityTestResult; - - test::CRandomNumbers rng; - - LOG_DEBUG(""); - LOG_DEBUG("*** Synthetic: no periods ***"); - { - TDoubleVec samples; - rng.generateNormalSamples(1.0, 5.0, 16128, samples); - - TDiurnalPeriodicityTestPtr test(maths::CDiurnalPeriodicityTest::create(FIVE_MINS)); - - core_t::TTime time = 0; - core_t::TTime day = DAY; - core_t::TTime week = 13 * DAY; - - for (std::size_t i = 0u; i < samples.size(); ++i) - { - if (time > day && time < 13 * DAY) - { - TResult result = test->test(); - LOG_DEBUG("detected = " << test->print(result)); - CPPUNIT_ASSERT(!result.periodic()); - day += DAY; - } - if (time >= week) - { - TResult result = test->test(); - LOG_DEBUG("detected = " << test->print(result)); - CPPUNIT_ASSERT(!result.periodic()); - week += WEEK; - } - test->add(time, samples[i]); - time += HALF_HOUR; - } - } - - LOG_DEBUG(""); - LOG_DEBUG("*** Ramp ***"); - { - TDiurnalPeriodicityTestPtr test(maths::CDiurnalPeriodicityTest::create(HALF_HOUR)); - - for (core_t::TTime time = 0; time < 10 * WEEK; time += HALF_HOUR) - { - test->add(time, static_cast(time)); - - if (time > DAY && time < 2 * WEEK && time % DAY == 0) - { - TResult result = test->test(); - LOG_DEBUG("detected = " << test->print(result)); - CPPUNIT_ASSERT(!result.periodic()); - } - if (time > 2 * WEEK && time % (2 * WEEK) == 0) - { - TResult result = test->test(); - LOG_DEBUG("detected = " << test->print(result)); - CPPUNIT_ASSERT(!result.periodic()); - } - } - } - - LOG_DEBUG(""); - LOG_DEBUG("*** Synthetic: one period ***"); - { - TDoubleVec samples; - rng.generateNormalSamples(1.0, 5.0, 4032, samples); - - TDiurnalPeriodicityTestPtr test(maths::CDiurnalPeriodicityTest::create(FIVE_MINS)); - - core_t::TTime time = 0; - core_t::TTime day = 3 * DAY; - core_t::TTime week = 13 * DAY; - - for (std::size_t i = 0u; i < samples.size(); ++i) - { - if (time > day && time < 13 * DAY) - { - TResult result = test->test(); - LOG_DEBUG("detected = " << test->print(result)); - CPPUNIT_ASSERT_EQUAL(std::string("{ 'daily' }"), test->print(result)); - day += DAY; - } - if (time >= week) - { - TResult result = test->test(); - LOG_DEBUG("detected = " << test->print(result)); - CPPUNIT_ASSERT_EQUAL(std::string("{ 'daily' }"), test->print(result)); - week += WEEK; - } - double x = 2.0 * ::sqrt(5.0) * sin( static_cast(i) - / 48.0 * boost::math::double_constants::two_pi); - test->add(time, x + samples[i]); - time += HALF_HOUR; - } - - CPPUNIT_ASSERT_EQUAL(std::string("{ 'daily' }"), test->print(test->test())); - } - - LOG_DEBUG(""); - LOG_DEBUG("*** Synthetic: daily weekday/weekend ***"); - { - TDoubleVec samples; - rng.generateNormalSamples(1.0, 5.0, 4032, samples); - - TDiurnalPeriodicityTestPtr test(maths::CDiurnalPeriodicityTest::create(FIVE_MINS)); - - core_t::TTime time = 0; - core_t::TTime day = 3 * DAY; - core_t::TTime week = 13 * DAY; - - for (std::size_t i = 0u; i < samples.size(); ++i) - { - if (time > day && time < 13 * DAY) - { - TResult result = test->test(); - LOG_DEBUG("detected = " << test->print(result)); - CPPUNIT_ASSERT_EQUAL(std::string("{ 'daily' }"), test->print(result)); - day += DAY; - } - if (time > week) - { - TResult result = test->test(); - LOG_DEBUG("detected = " << test->print(result)); - CPPUNIT_ASSERT_EQUAL(std::string("{ 'weekend daily' 'weekday daily' }"), - test->print(result)); - week += WEEK; - } - double scale = 1.0; - switch (((time + DAY) % WEEK) / DAY) - { - case 0: - case 1: - scale = 0.1; break; - default: - break; - } - double x = 10.0 * ::sqrt(5.0) * sin( static_cast(i) - / 48.0 * boost::math::double_constants::two_pi); - test->add(time, scale * (x + samples[i])); - time += HALF_HOUR; - } - - CPPUNIT_ASSERT_EQUAL(std::string("{ 'weekend daily' 'weekday daily' }"), - test->print(test->test())); - } - - LOG_DEBUG(""); - LOG_DEBUG("*** Synthetic: weekly ***"); - { - TDoubleVec samples; - rng.generateNormalSamples(1.0, 5.0, 4032, samples); - - TDiurnalPeriodicityTestPtr test(maths::CDiurnalPeriodicityTest::create(FIVE_MINS)); - - core_t::TTime time = 0; - core_t::TTime day = DAY; - core_t::TTime week = 13 * DAY; - std::size_t errors = 0u; - std::size_t calls = 0u; - - for (std::size_t i = 0u; i < samples.size(); ++i) - { - if (time > day && time < 13 * DAY) - { - TResult result = test->test(); - LOG_DEBUG("detected = " << test->print(result)); - CPPUNIT_ASSERT(!result.periodic()); - day += DAY; - } - if (time >= week) - { - TResult result = test->test(); - LOG_DEBUG("detected = " << test->print(result)); - if (test->print(result) != "{ 'weekly' }") - { - ++errors; - } - ++calls; - week += WEEK; - } - double x = 3.0 * ::sqrt(5.0) * sin( static_cast(i) - / 336.0 * boost::math::double_constants::two_pi); - test->add(time, x + samples[i]); - time += HALF_HOUR; - } - LOG_DEBUG("errors = " << static_cast(errors) / static_cast(calls)); - CPPUNIT_ASSERT_EQUAL(std::string("{ 'weekly' }"), test->print(test->test())); - CPPUNIT_ASSERT_DOUBLES_EQUAL(0.0, static_cast(errors) / static_cast(calls), 0.1); - } - - LOG_DEBUG(""); - LOG_DEBUG("*** daily ***"); - { - TTimeDoublePrVec timeseries; - core_t::TTime startTime; - core_t::TTime endTime; - CPPUNIT_ASSERT(test::CTimeSeriesTestData::parse("testfiles/spikey_data.csv", - timeseries, - startTime, - endTime, - test::CTimeSeriesTestData::CSV_UNIX_REGEX)); - CPPUNIT_ASSERT(!timeseries.empty()); - - LOG_DEBUG("timeseries = " << core::CContainerPrinter::print(timeseries.begin(), - timeseries.begin() + 10) - << " ..."); - - TDiurnalPeriodicityTestPtr test(maths::CDiurnalPeriodicityTest::create(FIVE_MINS)); - - core_t::TTime day = startTime + 3 * DAY; - core_t::TTime week = startTime + 13 * DAY; - - for (std::size_t i = 0u; i < timeseries.size(); ++i) - { - if (timeseries[i].first > day && timeseries[i].first < startTime + 13 * DAY) - { - TResult result = test->test(); - LOG_DEBUG("detected = " << test->print(result)); - CPPUNIT_ASSERT_EQUAL(std::string("{ 'daily' }"), test->print(result)); - day += DAY; - } - if (timeseries[i].first >= week) - { - TResult result = test->test(); - LOG_DEBUG("detected = " << test->print(result)); - CPPUNIT_ASSERT_EQUAL(std::string("{ 'daily' }"), test->print(result)); - week += WEEK; - } - test->add(timeseries[i].first, timeseries[i].second, 1.0 / 6.0); - } - - CPPUNIT_ASSERT_EQUAL(std::string("{ 'daily' }"), test->print(test->test())); - } - - LOG_DEBUG(""); - LOG_DEBUG("*** daily and weekends ***"); - { - TTimeDoublePrVec timeseries; - core_t::TTime startTime; - core_t::TTime endTime; - CPPUNIT_ASSERT(test::CTimeSeriesTestData::parse("testfiles/diurnal.csv", - timeseries, - startTime, - endTime, - test::CTimeSeriesTestData::CSV_UNIX_REGEX)); - CPPUNIT_ASSERT(!timeseries.empty()); - - LOG_DEBUG("timeseries = " << core::CContainerPrinter::print(timeseries.begin(), - timeseries.begin() + 10) - << " ..."); - - TDiurnalPeriodicityTestPtr test(maths::CDiurnalPeriodicityTest::create(FIVE_MINS)); - - core_t::TTime day = startTime + 5 * DAY; - core_t::TTime week = startTime + 13 * DAY; - - for (std::size_t i = 0u; i < timeseries.size(); ++i) - { - if (timeseries[i].first > 2 * day && timeseries[i].first < startTime + 13 * DAY) - { - TResult result = test->test(); - LOG_DEBUG("detected = " << test->print(result)); - CPPUNIT_ASSERT_EQUAL(std::string("{ 'daily' }"), test->print(result)); - day += DAY; - } - if (timeseries[i].first > week) - { - TResult result = test->test(); - LOG_DEBUG("detected = " << test->print(result)); - CPPUNIT_ASSERT_EQUAL(std::string("{ 'weekend daily' 'weekend weekly' 'weekday daily' }"), - test->print(result)); - week += WEEK; - } - test->add(timeseries[i].first, timeseries[i].second, 1.0 / 6.0); - } - - CPPUNIT_ASSERT_EQUAL(std::string("{ 'weekend daily' 'weekend weekly' 'weekday daily' }"), - test->print(test->test())); - } - - LOG_DEBUG(""); - LOG_DEBUG("*** no periods ***"); - { - TTimeDoublePrVec timeseries; - core_t::TTime startTime; - core_t::TTime endTime; - CPPUNIT_ASSERT(test::CTimeSeriesTestData::parse("testfiles/no_periods.csv", - timeseries, - startTime, - endTime, - test::CTimeSeriesTestData::CSV_ISO8601_REGEX, - test::CTimeSeriesTestData::CSV_ISO8601_DATE_FORMAT)); - CPPUNIT_ASSERT(!timeseries.empty()); - - LOG_DEBUG("timeseries = " << core::CContainerPrinter::print(timeseries.begin(), - timeseries.begin() + 10) - << " ..."); - - TDiurnalPeriodicityTestPtr test(maths::CDiurnalPeriodicityTest::create(HALF_HOUR)); - - core_t::TTime day = startTime + DAY; - core_t::TTime week = startTime + WEEK; - - for (std::size_t i = 0u; i < timeseries.size(); ++i) - { - if (timeseries[i].first > day && timeseries[i].first < startTime + 12 * DAY) - { - TResult result = test->test(); - LOG_DEBUG("detected = " << test->print(result)); - CPPUNIT_ASSERT(!result.periodic()); - day += DAY; - } - if (timeseries[i].first > week) - { - TResult result = test->test(); - LOG_DEBUG("detected = " << test->print(result)); - CPPUNIT_ASSERT(!result.periodic()); - week += WEEK; - } - test->add(timeseries[i].first, timeseries[i].second); - } - } - - LOG_DEBUG(""); - LOG_DEBUG("*** daily weekly and weekends ***"); - { - TTimeDoublePrVec timeseries; - core_t::TTime startTime; - core_t::TTime endTime; - CPPUNIT_ASSERT(test::CTimeSeriesTestData::parse("testfiles/thirty_minute_samples.csv", - timeseries, - startTime, - endTime, - test::CTimeSeriesTestData::CSV_ISO8601_REGEX, - test::CTimeSeriesTestData::CSV_ISO8601_DATE_FORMAT)); - CPPUNIT_ASSERT(!timeseries.empty()); - - LOG_DEBUG("timeseries = " << core::CContainerPrinter::print(timeseries.begin(), - timeseries.begin() + 10) - << " ..."); - - TDiurnalPeriodicityTestPtr test(maths::CDiurnalPeriodicityTest::create(HALF_HOUR)); - - core_t::TTime day = startTime + DAY; - core_t::TTime week = startTime + 2 * WEEK; - - for (std::size_t i = 0u; i < timeseries.size(); ++i) - { - if (timeseries[i].first > day && timeseries[i].first < startTime + 7 * DAY) - { - if (timeseries[i].first > startTime + 3 * DAY) - { - TResult result = test->test(); - LOG_DEBUG("detected = " << test->print(result)); - CPPUNIT_ASSERT_EQUAL(std::string("{ 'daily' }"), test->print(result)); - } - day += DAY; - } - if (timeseries[i].first > week) - { - TResult result = test->test(); - LOG_DEBUG("detected = " << test->print(result)); - CPPUNIT_ASSERT_EQUAL(std::string("{ 'weekend daily' 'weekend weekly' 'weekday daily' }"), test->print(result)); - week += WEEK; - } - test->add(timeseries[i].first, timeseries[i].second); - } - - CPPUNIT_ASSERT_EQUAL(std::string("{ 'weekend daily' 'weekend weekly' 'weekday daily' }"), - test->print(test->test())); - } -} - -void CTrendTestsTest::testDiurnalPeriodicityWithMissingValues(void) -{ - LOG_DEBUG("+------------------------------------------------------------+"); - LOG_DEBUG("| CTrendTestsTest::testDiurnalPeriodicityWithMissingValues |"); - LOG_DEBUG("+------------------------------------------------------------+"); - - test::CRandomNumbers rng; - - LOG_DEBUG("Daily Periodic") - { - TDiurnalPeriodicityTestPtr test(maths::CDiurnalPeriodicityTest::create(HALF_HOUR)); - core_t::TTime time = 0; - for (std::size_t t = 0u; t < 7; ++t) - { - for (auto value : { 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, - 10.0, 10.0, 8.0, 4.0, 3.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 1.0}) - { - if (value > 0.0) - { - test->add(time, value); - } - time += HALF_HOUR; - } - maths::CPeriodicityTestResult result{test->test()}; - LOG_DEBUG("result = " << test->print(result)); - if (t > 3) - { - CPPUNIT_ASSERT_EQUAL(std::string("{ 'daily' }"), test->print(result)); - } - } - } - LOG_DEBUG("Daily Not Periodic") - { - TDiurnalPeriodicityTestPtr test(maths::CDiurnalPeriodicityTest::create(HALF_HOUR)); - core_t::TTime time = 0; - for (std::size_t t = 0u; t < 7; ++t) - { - for (auto value : { 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, - 10.0, 10.0, 8.0, 4.0, 3.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 1.0}) - { - if (value > 0.0) - { - TDoubleVec rand; - rng.generateUniformSamples(-1.0, 1.0, 1, rand); - test->add(time, rand[0]); - } - time += HALF_HOUR; - } - maths::CPeriodicityTestResult result{test->test()}; - LOG_DEBUG("result = " << test->print(result)); - if (t > 3) - { - CPPUNIT_ASSERT_EQUAL(std::string("{ }"), test->print(result)); - } - } - } - LOG_DEBUG("Weekly") - { - TDiurnalPeriodicityTestPtr test(maths::CDiurnalPeriodicityTest::create(HALF_HOUR)); - core_t::TTime time = 0; - for (std::size_t t = 0u; t < 4; ++t) - { - for (auto value : { 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 10.0, 10.0, 8.0, 4.0, 3.0, 1.0, 1.0, 3.0, 0.0, 0.0, 0.0, 0.0, - 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, - 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, - 10.0, 10.0, 8.0, 4.0, 3.0, 1.0, 1.0, 3.0, 0.0, 0.0, 0.0, 0.0, - 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, - 10.0, 10.0, 8.0, 4.0, 3.0, 1.0, 1.0, 3.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}) - { - if (value > 0.0) - { - test->add(time, value); - } - time += HOUR; - } - maths::CPeriodicityTestResult result{test->test()}; - LOG_DEBUG("result = " << test->print(result)); - if (t > 3) - { - CPPUNIT_ASSERT_EQUAL(std::string("{ 'daily' 'weekly' }"), test->print(result)); - } - } - } - LOG_DEBUG("Weekly Not Periodic") - { - TDiurnalPeriodicityTestPtr test(maths::CDiurnalPeriodicityTest::create(HALF_HOUR)); - core_t::TTime time = 0; - for (std::size_t t = 0u; t < 4; ++t) - { - for (auto value : { 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 10.0, 10.0, 8.0, 4.0, 3.0, 1.0, 1.0, 3.0, 0.0, 0.0, 0.0, 0.0, - 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, - 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, - 10.0, 10.0, 8.0, 4.0, 3.0, 1.0, 1.0, 3.0, 0.0, 0.0, 0.0, 0.0, - 20.0, 18.0, 10.0, 4.0, 4.0, 4.0, 4.0, 5.0, 6.0, 8.0, 9.0, 9.0, - 10.0, 10.0, 8.0, 4.0, 3.0, 1.0, 1.0, 3.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}) - { - if (value > 0.0) - { - TDoubleVec rand; - rng.generateUniformSamples(-1.0, 1.0, 1, rand); - test->add(time, rand[0]); - } - time += HOUR; - } - maths::CPeriodicityTestResult result{test->test()}; - LOG_DEBUG("result = " << test->print(result)); - if (t > 3) - { - CPPUNIT_ASSERT_EQUAL(std::string("{ }"), test->print(result)); - } - } - } -} - -void CTrendTestsTest::testScanningPeriodicity(void) -{ - LOG_DEBUG("+--------------------------------------------+"); - LOG_DEBUG("| CTrendTestsTest::testScanningPeriodicity |"); - LOG_DEBUG("+--------------------------------------------+"); - - using TPeriodicityResultPr = maths::CScanningPeriodicityTest::TPeriodicityResultPr; - - test::CRandomNumbers rng; - - LOG_DEBUG("Smooth") - { - TDoubleVec timeseries; - for (core_t::TTime time = 0; time <= 2 * WEEK; time += HALF_HOUR) - { - timeseries.push_back(15.0 + 10.0 * ::sin(0.7 * boost::math::double_constants::two_pi - * static_cast(time) - / static_cast(DAY))); - } - - maths::CScanningPeriodicityTest test(BUCKET_LENGTHS, 240); - test.initialize(0); - - core_t::TTime time = 0; - for (std::size_t i = 0u; i < timeseries.size(); ++i, time += HALF_HOUR) - { - if (test.needToCompress(time)) - { - TPeriodicityResultPr result = test.test(); - LOG_DEBUG("time = " << time); - LOG_DEBUG("periods = " << result.first.print(result.second)); - CPPUNIT_ASSERT(result.second.periodic()); - CPPUNIT_ASSERT_DOUBLES_EQUAL(static_cast(DAY) / 0.7, - static_cast(result.first.periods()[0]), - 900.0); - break; - } - test.add(time, timeseries[i]); - } - } - - LOG_DEBUG("Smooth + Noise") - { - TDoubleVec timeseries; - for (core_t::TTime time = 0; time <= 2 * WEEK; time += HALF_HOUR) - { - timeseries.push_back(15.0 + 10.0 * ::sin(0.4 * boost::math::double_constants::two_pi - * static_cast(time) - / static_cast(DAY))); - } - - TDoubleVec noise; - rng.generateNormalSamples(0.0, 4.0, timeseries.size(), noise); - - maths::CScanningPeriodicityTest test(BUCKET_LENGTHS, 240); - test.initialize(0); - - core_t::TTime time = 0; - std::size_t periodic = 0; - for (std::size_t i = 0u; i < timeseries.size(); ++i, time += HALF_HOUR) - { - if (test.needToCompress(time)) - { - TPeriodicityResultPr result = test.test(); - if (result.second.periodic()) - { - LOG_DEBUG("time = " << time); - LOG_DEBUG("periods = " << result.first.print(result.second)); - CPPUNIT_ASSERT_EQUAL(static_cast(DAY) / 0.4, - static_cast(result.first.periods()[0])); - ++periodic; - break; - } - } - test.add(time, timeseries[i] + noise[i]); - } - CPPUNIT_ASSERT_EQUAL(std::size_t(1), periodic); - } - - LOG_DEBUG("Long term behaviour") - { - TDoubleVec timeseries; - for (core_t::TTime time = 0; time <= 300 * WEEK; time += HALF_HOUR) - { - timeseries.push_back(15.0 + 10.0 * ::sin(0.1 * boost::math::double_constants::two_pi - * static_cast(time) - / static_cast(DAY))); - } - - TDoubleVec noise; - rng.generateNormalSamples(0.0, 4.0, timeseries.size(), noise); - - maths::CScanningPeriodicityTest test(BUCKET_LENGTHS, 240); - test.initialize(0); - - core_t::TTime time = 0; - std::size_t periodic = 0; - for (std::size_t i = 0u; i < timeseries.size(); ++i, time += HALF_HOUR) - { - if (test.needToCompress(time)) - { - TPeriodicityResultPr result = test.test(); - if (result.second.periodic()) - { - LOG_DEBUG("time = " << time); - LOG_DEBUG("periods = " << result.first.print(result.second)); - CPPUNIT_ASSERT(result.first.periods()[0] % (10 * DAY) == 0); - ++periodic; - } - } - test.add(time, timeseries[i] + noise[i]); - } - CPPUNIT_ASSERT_EQUAL(std::size_t(8), periodic); - } -} - void CTrendTestsTest::testCalendarCyclic(void) { LOG_DEBUG("+---------------------------------------+"); @@ -1288,57 +342,6 @@ void CTrendTestsTest::testPersist(void) // Check that persistence is idempotent. - LOG_DEBUG("Test CTrendTest"); - { - TDoubleVec timeseries; - for (core_t::TTime time = 0; time <= 2 * WEEK; time += HALF_HOUR) - { - double daily = 15.0 + 10.0 * ::sin( boost::math::double_constants::two_pi - * static_cast(time) - / static_cast(DAY)); - timeseries.push_back(daily); - } - - test::CRandomNumbers rng; - TDoubleVec noise; - rng.generateNormalSamples(20.0, 16.0, timeseries.size(), noise); - - maths::CTrendTest orig; - core_t::TTime time = 0; - for (std::size_t i = 0u; i < timeseries.size(); ++i, time += HALF_HOUR) - { - orig.add(time, timeseries[i] + noise[i]); - orig.captureVariance(time, timeseries[i] + noise[i]); - } - - std::string origXml; - { - core::CRapidXmlStatePersistInserter inserter("root"); - orig.acceptPersistInserter(inserter); - inserter.toXml(origXml); - } - - LOG_DEBUG("XML representation:\n" << origXml); - - maths::CTrendTest restored; - { - core::CRapidXmlParser parser; - CPPUNIT_ASSERT(parser.parseStringIgnoreCdata(origXml)); - core::CRapidXmlStateRestoreTraverser traverser(parser); - CPPUNIT_ASSERT(traverser.traverseSubLevel(boost::bind( - &maths::CTrendTest::acceptRestoreTraverser, &restored, _1))); - } - CPPUNIT_ASSERT_EQUAL(orig.checksum(), restored.checksum()); - - std::string newXml; - { - core::CRapidXmlStatePersistInserter inserter("root"); - restored.acceptPersistInserter(inserter); - inserter.toXml(newXml); - } - CPPUNIT_ASSERT_EQUAL(origXml, newXml); - } - LOG_DEBUG("Test CRandomizedPeriodicityTest"); { maths::CRandomizedPeriodicityTest test; @@ -1402,97 +405,6 @@ void CTrendTestsTest::testPersist(void) CPPUNIT_ASSERT_EQUAL(origNextRandom, newNextRandom); } - LOG_DEBUG("Test CDiurnalPeriodicityTest"); - { - TDoubleVec timeseries; - for (core_t::TTime time = 0; time <= 2 * WEEK; time += HALF_HOUR) - { - double daily = 15.0 + 10.0 * ::sin( boost::math::double_constants::two_pi - * static_cast(time) - / static_cast(DAY)); - timeseries.push_back(daily); - } - - test::CRandomNumbers rng; - TDoubleVec noise; - rng.generateNormalSamples(20.0, 16.0, timeseries.size(), noise); - - boost::scoped_ptr orig(maths::CDiurnalPeriodicityTest::create(HALF_HOUR)); - - core_t::TTime time = 0; - for (std::size_t i = 0u; i < timeseries.size(); ++i, time += HALF_HOUR) - { - orig->add(time, timeseries[i] + noise[i]); - } - - std::string origXml; - { - core::CRapidXmlStatePersistInserter inserter("root"); - orig->acceptPersistInserter(inserter); - inserter.toXml(origXml); - } - - LOG_DEBUG("XML representation:\n" << origXml); - - maths::CDiurnalPeriodicityTest restored; - { - core::CRapidXmlParser parser; - CPPUNIT_ASSERT(parser.parseStringIgnoreCdata(origXml)); - core::CRapidXmlStateRestoreTraverser traverser(parser); - CPPUNIT_ASSERT(traverser.traverseSubLevel(boost::bind( - &maths::CDiurnalPeriodicityTest::acceptRestoreTraverser, &restored, _1))); - } - - CPPUNIT_ASSERT_EQUAL(orig->checksum(), restored.checksum()); - - std::string newXml; - { - core::CRapidXmlStatePersistInserter inserter("root"); - restored.acceptPersistInserter(inserter); - inserter.toXml(newXml); - } - CPPUNIT_ASSERT_EQUAL(origXml, newXml); - } - - LOG_DEBUG("Test CScanningPeriodicityTest"); - { - maths::CScanningPeriodicityTest orig(BUCKET_LENGTHS, 120); - orig.initialize(0); - for (core_t::TTime time = 0; time <= 2 * WEEK; time += HALF_HOUR) - { - orig.add(time, 15.0 + 10.0 * ::sin( boost::math::double_constants::two_pi - * static_cast(time) - / static_cast(DAY))); - } - - std::string origXml; - { - core::CRapidXmlStatePersistInserter inserter("root"); - orig.acceptPersistInserter(inserter); - inserter.toXml(origXml); - } - - LOG_DEBUG("XML representation:\n" << origXml); - - maths::CScanningPeriodicityTest restored(BUCKET_LENGTHS, 10); - { - core::CRapidXmlParser parser; - CPPUNIT_ASSERT(parser.parseStringIgnoreCdata(origXml)); - core::CRapidXmlStateRestoreTraverser traverser(parser); - CPPUNIT_ASSERT(traverser.traverseSubLevel(boost::bind( - &maths::CScanningPeriodicityTest::acceptRestoreTraverser, &restored, _1))); - } - CPPUNIT_ASSERT_EQUAL(orig.checksum(), restored.checksum()); - - std::string newXml; - { - core::CRapidXmlStatePersistInserter inserter("root"); - restored.acceptPersistInserter(inserter); - inserter.toXml(newXml); - } - CPPUNIT_ASSERT_EQUAL(origXml, newXml); - } - LOG_DEBUG("Test CCalendarCyclicTest"); { test::CRandomNumbers rng; @@ -1539,30 +451,15 @@ CppUnit::Test *CTrendTestsTest::suite(void) { CppUnit::TestSuite *suiteOfTests = new CppUnit::TestSuite("CTrendTestsTest"); - suiteOfTests->addTest( new CppUnit::TestCaller( - "CTrendTestsTest::testTrend", - &CTrendTestsTest::testTrend) ); suiteOfTests->addTest( new CppUnit::TestCaller( "CTrendTestsTest::testRandomizedPeriodicity", &CTrendTestsTest::testRandomizedPeriodicity) ); - suiteOfTests->addTest( new CppUnit::TestCaller( - "CTrendTestsTest::testDiurnalPeriodicity", - &CTrendTestsTest::testDiurnalPeriodicity) ); - suiteOfTests->addTest( new CppUnit::TestCaller( - "CTrendTestsTest::testDiurnalPeriodicityWithMissingValues", - &CTrendTestsTest::testDiurnalPeriodicityWithMissingValues) ); - suiteOfTests->addTest( new CppUnit::TestCaller( - "CTrendTestsTest::testScanningPeriodicity", - &CTrendTestsTest::testScanningPeriodicity) ); suiteOfTests->addTest( new CppUnit::TestCaller( "CTrendTestsTest::testCalendarCyclic", &CTrendTestsTest::testCalendarCyclic) ); suiteOfTests->addTest( new CppUnit::TestCaller( "CTrendTestsTest::testPersist", &CTrendTestsTest::testPersist) ); - suiteOfTests->addTest( new CppUnit::TestCaller( - "CTrendTestsTest::testDiurnalInitialisation", - &CTrendTestsTest::testDiurnalInitialisation) ); return suiteOfTests; } diff --git a/lib/maths/unittest/CTrendTestsTest.h b/lib/maths/unittest/CTrendTestsTest.h index 0eb6b438ac..4cc88aa36a 100644 --- a/lib/maths/unittest/CTrendTestsTest.h +++ b/lib/maths/unittest/CTrendTestsTest.h @@ -21,12 +21,7 @@ class CTrendTestsTest : public CppUnit::TestFixture { public: - void testDiurnalInitialisation(void); - void testTrend(void); void testRandomizedPeriodicity(void); - void testDiurnalPeriodicity(void); - void testDiurnalPeriodicityWithMissingValues(void); - void testScanningPeriodicity(void); void testCalendarCyclic(void); void testPersist(void); diff --git a/lib/maths/unittest/Main.cc b/lib/maths/unittest/Main.cc index 888a507f98..0a5347290b 100644 --- a/lib/maths/unittest/Main.cc +++ b/lib/maths/unittest/Main.cc @@ -60,6 +60,7 @@ #include "COrderingsTest.h" #include "COrdinalTest.h" #include "CPackedBitVectorTest.h" +#include "CPeriodicityHypothesisTestsTest.h" #include "CPoissonMeanConjugateTest.h" #include "CPriorTest.h" #include "CPRNGTest.h" @@ -81,6 +82,7 @@ #include "CTimeSeriesDecompositionTest.h" #include "CTimeSeriesModelTest.h" #include "CToolsTest.h" +#include "CTrendComponentTest.h" #include "CTrendTestsTest.h" #include "CXMeansTest.h" #include "CXMeansOnlineTest.h" @@ -135,6 +137,7 @@ int main(int argc, const char **argv) runner.addTest( COrderingsTest::suite() ); runner.addTest( COrdinalTest::suite() ); runner.addTest( CPackedBitVectorTest::suite() ); + runner.addTest( CPeriodicityHypothesisTestsTest::suite() ); runner.addTest( CPoissonMeanConjugateTest::suite() ); runner.addTest( CPriorTest::suite() ); runner.addTest( CPRNGTest::suite() ); @@ -156,6 +159,7 @@ int main(int argc, const char **argv) runner.addTest( CTimeSeriesDecompositionTest::suite() ); runner.addTest( CTimeSeriesModelTest::suite() ); runner.addTest( CToolsTest::suite() ); + runner.addTest( CTrendComponentTest::suite() ); runner.addTest( CTrendTestsTest::suite() ); runner.addTest( CXMeansTest::suite() ); runner.addTest( CXMeansOnlineTest::suite() ); diff --git a/lib/maths/unittest/Makefile b/lib/maths/unittest/Makefile index 25e89207fc..73323a7a89 100644 --- a/lib/maths/unittest/Makefile +++ b/lib/maths/unittest/Makefile @@ -59,7 +59,7 @@ SRCS=\ CMathsMemoryTest.cc \ CMixtureDistributionTest.cc \ CModelTest.cc \ - CMultimodalPriorTest.cc\ + CMultimodalPriorTest.cc \ CMultinomialConjugateTest.cc \ CMultivariateConstantPriorTest.cc \ CMultivariateMultimodalPriorTest.cc \ @@ -71,6 +71,7 @@ SRCS=\ COrderingsTest.cc \ COrdinalTest.cc \ CPackedBitVectorTest.cc \ + CPeriodicityHypothesisTestsTest.cc \ CPoissonMeanConjugateTest.cc \ CPriorTest.cc \ CPRNGTest.cc \ @@ -92,6 +93,7 @@ SRCS=\ CTimeSeriesDecompositionTest.cc \ CTimeSeriesModelTest.cc \ CToolsTest.cc \ + CTrendComponentTest.cc \ CTrendTestsTest.cc \ CXMeansOnlineTest.cc \ CXMeansOnline1dTest.cc \ diff --git a/lib/maths/unittest/TestUtils.cc b/lib/maths/unittest/TestUtils.cc index 42c09c9d9e..b09967fcc9 100644 --- a/lib/maths/unittest/TestUtils.cc +++ b/lib/maths/unittest/TestUtils.cc @@ -21,6 +21,8 @@ #include #include +#include + namespace ml { using namespace maths; @@ -28,6 +30,9 @@ using namespace handy_typedefs; namespace { +const core_t::TTime HALF_HOUR{core::constants::HOUR / 2}; +const core_t::TTime DAY{core::constants::DAY}; +const core_t::TTime WEEK{core::constants::WEEK}; //! \brief Computes the c.d.f. of the prior minus the target supplied //! to its constructor at specific locations. @@ -68,11 +73,11 @@ class CCdf : public std::unary_function switch (m_Style) { - case E_Lower: return ::exp(-lowerBound) - m_Target; - case E_Upper: return ::exp(-upperBound) - m_Target; - case E_GeometricMean: return ::exp(-(lowerBound + upperBound) / 2.0) - m_Target; + case E_Lower: return std::exp(-lowerBound) - m_Target; + case E_Upper: return std::exp(-upperBound) - m_Target; + case E_GeometricMean: return std::exp(-(lowerBound + upperBound) / 2.0) - m_Target; } - return ::exp(-(lowerBound + upperBound) / 2.0) - m_Target; + return std::exp(-(lowerBound + upperBound) / 2.0) - m_Target; } private: @@ -350,5 +355,103 @@ bool CPriorTestInterface::marginalLikelihoodVarianceForTest(double &result) cons return true; } +double constant(core_t::TTime /*time*/) +{ + return 4.0; +} + +double ramp(core_t::TTime time) +{ + return 0.1 * static_cast(time) / static_cast(WEEK); +} + +double markov(core_t::TTime time) +{ + static double state{0.2}; + if (time % WEEK == 0) + { + core::CHashing::CMurmurHash2BT hasher; + state = 2.0 * static_cast(hasher(time)) + / static_cast(std::numeric_limits::max()); + } + return state; +} + +double smoothDaily(core_t::TTime time) +{ + return std::sin( boost::math::double_constants::two_pi + * static_cast(time) + / static_cast(DAY)); +} + +double smoothWeekly(core_t::TTime time) +{ + return std::sin( boost::math::double_constants::two_pi + * static_cast(time) + / static_cast(WEEK)); +} + +double spikeyDaily(core_t::TTime time) +{ + double pattern[] + { + 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, + 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.1 + }; + return pattern[(time % DAY) / HALF_HOUR]; +} + +double spikeyWeekly(core_t::TTime time) +{ + double pattern[] + { + 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.1, + 0.0, 0.1, 0.1, 0.1, 0.1, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.1, + 0.0, 0.1, 0.1, 0.1, 0.1, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.1, + 0.0, 0.1, 0.1, 0.1, 0.1, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.1, + 0.0, 0.1, 0.1, 0.1, 0.1, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.1, + 0.0, 0.1, 0.1, 0.1, 0.1, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.1, + 0.0, 0.1, 0.1, 0.1, 0.1, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.2, + 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.1, 0.1, 0.1, 0.1, 0.1 + }; + return pattern[(time % WEEK) / HALF_HOUR]; +} + +double weekends(core_t::TTime time) +{ + double amplitude[] = { 1.0, 0.9, 0.8, 0.9, 1.1, 0.2, 0.05 }; + return amplitude[(time % WEEK) / DAY] + * std::sin( boost::math::double_constants::two_pi + * static_cast(time) + / static_cast(DAY)); +} + +double scale(double scale, core_t::TTime time, TGenerator generator) +{ + return generator(static_cast(scale * static_cast(time))); +} + } diff --git a/lib/maths/unittest/TestUtils.h b/lib/maths/unittest/TestUtils.h index 1f54fb092b..b4ab94001e 100644 --- a/lib/maths/unittest/TestUtils.h +++ b/lib/maths/unittest/TestUtils.h @@ -16,6 +16,7 @@ #ifndef INCLUDED_ml_TestUtils_h #define INCLUDED_ml_TestUtils_h +#include #include #include @@ -24,26 +25,29 @@ #include #include +#include namespace ml { namespace handy_typedefs { -typedef core::CSmallVector TDouble1Vec; -typedef core::CSmallVector TDouble4Vec; -typedef core::CSmallVector TDouble10Vec; -typedef core::CSmallVector TDouble4Vec1Vec; -typedef core::CSmallVector TDouble10Vec1Vec; -typedef core::CSmallVector TDouble10Vec4Vec; -typedef core::CSmallVector TDouble10Vec10Vec; -typedef core::CSmallVector TDouble10Vec4Vec1Vec; -typedef maths::CVectorNx1 TVector2; -typedef std::vector TVector2Vec; -typedef std::vector TVector2VecVec; -typedef maths::CSymmetricMatrixNxN TMatrix2; -typedef std::vector TMatrix2Vec; -typedef maths::CVectorNx1 TVector3; -typedef maths::CSymmetricMatrixNxN TMatrix3; +using TDouble1Vec = core::CSmallVector; +using TDouble4Vec = core::CSmallVector; +using TDouble10Vec = core::CSmallVector; +using TDouble4Vec1Vec = core::CSmallVector; +using TDouble10Vec1Vec = core::CSmallVector; +using TDouble10Vec4Vec = core::CSmallVector; +using TDouble10Vec10Vec = core::CSmallVector; +using TDouble10Vec4Vec1Vec = core::CSmallVector; +using TVector2 = maths::CVectorNx1; +using TVector2Vec = std::vector; +using TVector2VecVec = std::vector; +using TMatrix2 = maths::CSymmetricMatrixNxN; +using TMatrix2Vec = std::vector; +using TVector3 = maths::CVectorNx1; +using TMatrix3 = maths::CSymmetricMatrixNxN; +using TGenerator = double (*)(core_t::TTime); +using TGeneratorVec = std::vector; } //! \brief A set of test and utility functions for use in testing only. @@ -53,10 +57,10 @@ typedef maths::CSymmetricMatrixNxN TMatrix3; class CPriorTestInterface { public: - typedef std::pair TDoubleDoublePr; - typedef core::CSmallVector TDoubleDoublePr1Vec; - typedef maths_t::TWeightStyleVec TWeightStyleVec; - typedef maths::CConstantWeights TWeights; + using TDoubleDoublePr = std::pair; + using TDoubleDoublePr1Vec = core::CSmallVector; + using TWeightStyleVec = maths_t::TWeightStyleVec; + using TWeights = maths::CConstantWeights; public: explicit CPriorTestInterface(maths::CPrior &prior); @@ -173,17 +177,13 @@ class CPriorTestInterfaceMixin : public PRIOR, public CPriorTestInterface public: CPriorTestInterfaceMixin(const PRIOR &prior) : PRIOR(prior), - CPriorTestInterface(static_cast(*this)), - m_Offset(0.0) - { - } + CPriorTestInterface(static_cast(*this)) + {} CPriorTestInterfaceMixin(const CPriorTestInterfaceMixin &other) : PRIOR(static_cast(other)), - CPriorTestInterface(static_cast(*this)), - m_Offset(0.0) - { - } + CPriorTestInterface(static_cast(*this)) + {} virtual ~CPriorTestInterfaceMixin(void) {} @@ -193,39 +193,11 @@ class CPriorTestInterfaceMixin : public PRIOR, public CPriorTestInterface this->PRIOR::swap(other); } - //! Overload assignment. - CPriorTestInterfaceMixin &operator=(const CPriorTestInterfaceMixin &other) - { - if (this != &other) - { - // This intentionally slices! We don't want to copy the - // CPriorTestInterface state. - static_cast(*this) = static_cast(other); - m_Offset = other.m_Offset; - } - return *this; - } - //! Clone the object. virtual CPriorTestInterfaceMixin *clone(void) const { return new CPriorTestInterfaceMixin(*this); } - - //! Set the offset margin. - void setOffset(double offset) - { - m_Offset = offset; - } - - private: - //! Override to zero for nearly all testing. - virtual double offsetMargin(void) const - { - return m_Offset; - } - - double m_Offset; }; @@ -256,6 +228,7 @@ class CVarianceKernel double m_Mean; }; +//! \brief A constant unit kernel. template class CUnitKernel { @@ -269,7 +242,7 @@ class CUnitKernel { m_X[0].assign(x.begin(), x.end()); m_Prior->jointLogMarginalLikelihood(maths::CConstantWeights::COUNT, m_X, SINGLE_UNIT, result); - result = ::exp(result); + result = std::exp(result); return true; } @@ -284,7 +257,7 @@ class CUnitKernel template handy_typedefs::TDouble10Vec4Vec1Vec CUnitKernel::SINGLE_UNIT(1, handy_typedefs::TDouble10Vec4Vec(1, handy_typedefs::TDouble10Vec(N, 1.0))); - +//! \brief The kernel for computing the mean of a multivariate prior. template class CMeanKernel { @@ -300,7 +273,7 @@ class CMeanKernel m_X[0].assign(x.begin(), x.end()); double likelihood; m_Prior->jointLogMarginalLikelihood(maths::CConstantWeights::COUNT, m_X, SINGLE_UNIT, likelihood); - likelihood = ::exp(likelihood); + likelihood = std::exp(likelihood); result = x * likelihood; return true; } @@ -316,7 +289,7 @@ class CMeanKernel template handy_typedefs::TDouble10Vec4Vec1Vec CMeanKernel::SINGLE_UNIT(1, handy_typedefs::TDouble10Vec4Vec(1, handy_typedefs::TDouble10Vec(N, 1.0))); - +//! \brief The kernel for computing the variance of a multivariate prior. template class CCovarianceKernel { @@ -334,7 +307,7 @@ class CCovarianceKernel m_X[0].assign(x.begin(), x.end()); double likelihood; m_Prior->jointLogMarginalLikelihood(maths::CConstantWeights::COUNT, m_X, SINGLE_UNIT, likelihood); - likelihood = ::exp(likelihood); + likelihood = std::exp(likelihood); result = (x - m_Mean).outer() * likelihood; return true; } @@ -351,6 +324,33 @@ class CCovarianceKernel template handy_typedefs::TDouble10Vec4Vec1Vec CCovarianceKernel::SINGLE_UNIT(1, handy_typedefs::TDouble10Vec4Vec(1, handy_typedefs::TDouble10Vec(N, 1.0))); +//! A constant function. +double constant(core_t::TTime time); + +//! A linear ramp. +double ramp(core_t::TTime time); + +//! A Markov process. +double markov(core_t::TTime time); + +//! Smooth daily periodic. +double smoothDaily(core_t::TTime time); + +//! Smooth weekly periodic. +double smoothWeekly(core_t::TTime time); + +//! Spikey daily periodic. +double spikeyDaily(core_t::TTime time); + +//! Spikey weekly periodic. +double spikeyWeekly(core_t::TTime time); + +//! Weekday/weekend periodic. +double weekends(core_t::TTime time); + +//! Scales time input to \p generator. +double scale(double scale, core_t::TTime time, handy_typedefs::TGenerator generator); + } #endif // INCLUDED_ml_TestUtils_h diff --git a/lib/maths/unittest/testfiles/CMultivariateTimeSeriesModel.6.2.expected_intervals.txt b/lib/maths/unittest/testfiles/CMultivariateTimeSeriesModel.6.2.expected_intervals.txt new file mode 100644 index 0000000000..cc782dc9b8 --- /dev/null +++ b/lib/maths/unittest/testfiles/CMultivariateTimeSeriesModel.6.2.expected_intervals.txt @@ -0,0 +1 @@ +3.457267,3.065089,6.146333,6.845776,5.881191,8.556494,10.23429,8.697292,10.96665,;4.45187,4.349719,7.123246,7.84038,7.16582,9.533407,11.22889,9.981922,11.94357,;5.238844,4.93916,7.970138,8.627353,7.755262,10.3803,12.01586,10.57136,12.79046,;8.496956,8.115735,9.919255,11.88547,10.93184,12.32942,15.27398,13.74794,14.73958,;9.758549,9.145305,10.95926,13.14706,11.96141,13.36943,16.53557,14.77751,15.77959,;10.24652,9.820546,12.40078,13.63503,12.63665,14.81094,17.02354,15.45275,17.22111,;11.20936,10.93118,13.54465,14.59787,13.74728,15.95481,17.98638,16.56338,18.36497,;13.66281,13.3159,15.14944,17.05132,16.132,17.5596,20.43983,18.9481,19.96976,;14.38378,13.73874,16.61078,17.77229,16.55484,19.02094,21.1608,19.37094,21.4311,;14.70039,14.27166,16.71515,18.0889,17.08776,19.12531,21.47741,19.90386,21.53547,;15.47749,15.21465,17.41244,18.866,18.03075,19.8226,22.25451,20.84685,22.23276,;16.41683,16.11741,18.41585,19.80534,18.93351,20.82601,23.19385,21.74961,23.23617,;17.0581,16.71791,18.89777,20.44661,19.53401,21.30793,23.83512,22.35011,23.71809,;17.29124,17.11799,19.37293,20.67975,19.93409,21.78309,24.06826,22.75019,24.19325,;17.27504,17.34414,19.62177,20.66355,20.16024,22.03193,24.05206,22.97634,24.4421,;17.20832,17.34421,19.43679,20.59683,20.16031,21.84695,23.98534,22.97642,24.25711,;17.12314,17.02005,19.09208,20.51165,19.83616,21.50224,23.90016,22.65226,23.9124,;17.06487,16.59156,18.76175,20.45338,19.40766,21.17191,23.84188,22.22376,23.58207,;16.96128,16.69207,18.49776,20.34979,19.50817,20.90792,23.7383,22.32427,23.31808,;16.21027,16.13993,18.20247,19.59878,18.95603,20.61263,22.98729,21.77213,23.02279,;14.92526,14.13089,17.6112,18.31377,16.94699,20.02136,21.70228,19.7631,22.43152,;14.22105,13.52887,16.50778,17.60956,16.34497,18.91794,20.99807,19.16107,21.3281,;13.88172,13.48694,15.51333,17.27023,16.30304,17.92349,20.65874,19.11914,20.33365,;13.12246,12.54903,14.15357,16.51097,15.36513,16.56373,19.89947,18.18123,18.97389,;11.48349,10.96693,12.60667,14.87199,13.78303,15.01683,18.2605,16.59913,17.42699,;9.521387,9.283737,11.59819,12.9099,12.09984,14.00835,16.29841,14.91594,16.41851,;8.375495,8.101113,10.49694,11.764,10.91721,12.9071,15.15251,13.73332,15.31726,;6.930475,6.578518,9.287013,10.31898,9.394619,11.69717,13.70749,12.21072,14.10733,;5.704726,5.265178,8.29861,9.093236,8.081279,10.70877,12.48175,10.89738,13.11893,;4.723245,4.285852,7.39421,8.111755,7.101953,9.804371,11.50026,9.918055,12.21453,;3.360139,3.011989,5.721903,6.748649,5.82809,8.132063,10.13716,8.644191,10.54222,;2.122611,1.945631,3.698522,5.511121,4.761732,6.108682,8.899631,7.577834,8.518843,;1.441638,1.129074,2.986961,4.830148,3.945175,5.397122,8.218658,6.761276,7.807283,;0.5315632,0.1561281,2.551305,3.920073,2.972229,4.961466,7.308583,5.788331,7.371627,;-0.4197438,-0.8761183,1.540553,2.968766,1.939983,3.950714,6.357276,4.756084,6.360874,;-0.9097442,-1.58621,0.5565663,2.478766,1.229892,2.966727,5.867275,4.045993,5.376888,;-1.313094,-1.880557,0.2102114,2.075416,0.9355441,2.620372,5.463926,3.751645,5.030533,;-1.725898,-2.151863,-0.03909628,1.662612,0.6642381,2.371065,5.051122,3.480339,4.781225,;-2.115575,-2.620171,-0.01902707,1.272935,0.1959305,2.391134,4.661444,3.012032,4.801295,;-2.412717,-3.054235,0.3001471,0.9757928,-0.2381334,2.710308,4.364303,2.577968,5.120469,;-2.372926,-3.003442,0.5915033,1.015584,-0.187341,3.001664,4.404094,2.62876,5.411825,;-1.908949,-2.389102,0.6368664,1.47956,0.4269988,3.047027,4.86807,3.2431,5.457188,;-1.272513,-1.991996,0.4963225,2.115997,0.8241052,2.906483,5.504507,3.640206,5.316644,;-0.6206085,-1.438895,1.284361,2.767901,1.377206,3.694522,6.156411,4.193307,6.104683,;0.09649325,-0.2969013,2.699005,3.485003,2.5192,5.109165,6.873513,5.335301,7.519326,;1.000493,0.593017,2.845431,4.389003,3.409118,5.255592,7.777512,6.22522,7.665753,;1.485238,1.34156,3.375257,4.873748,4.157661,5.785418,8.262258,6.973763,8.195578,;2.287702,2.192013,4.866177,5.676212,5.008114,7.276338,9.064722,7.824215,9.686498,;3.849966,3.446241,6.382532,7.238476,6.262342,8.792693,10.62699,9.078443,11.20285,;5.139093,5.016734,7.536594,8.527603,7.832835,9.946755,11.91611,10.64894,12.35692,;6.220591,5.892039,8.560634,9.609101,8.70814,10.97079,12.99761,11.52424,13.38096,;8.496956,8.115735,9.919255,11.88547,10.93184,12.32942,15.27398,13.74794,14.73958,;9.758549,9.145305,10.95926,13.14706,11.96141,13.36943,16.53557,14.77751,15.77959,;10.24652,9.820546,12.40078,13.63503,12.63665,14.81094,17.02354,15.45275,17.22111,;11.20936,10.93118,13.54465,14.59787,13.74728,15.95481,17.98638,16.56338,18.36497,;13.66281,13.3159,15.14944,17.05132,16.132,17.5596,20.43983,18.9481,19.96976,;14.38378,13.73874,16.61078,17.77229,16.55484,19.02094,21.1608,19.37094,21.4311,;14.70039,14.27166,16.71515,18.0889,17.08776,19.12531,21.47741,19.90386,21.53547,;15.47749,15.21465,17.41244,18.866,18.03075,19.8226,22.25451,20.84685,22.23276,;16.41683,16.11741,18.41585,19.80534,18.93351,20.82601,23.19385,21.74961,23.23617,;17.0581,16.71791,18.89777,20.44661,19.53401,21.30793,23.83512,22.35011,23.71809,;17.29124,17.11799,19.37293,20.67975,19.93409,21.78309,24.06826,22.75019,24.19325,;17.27504,17.34414,19.62177,20.66355,20.16024,22.03193,24.05206,22.97634,24.4421,;17.20832,17.34421,19.43679,20.59683,20.16031,21.84695,23.98534,22.97642,24.25711,;17.12314,17.02005,19.09208,20.51165,19.83616,21.50224,23.90016,22.65226,23.9124,;17.06487,16.59156,18.76175,20.45338,19.40766,21.17191,23.84188,22.22376,23.58207,;16.96128,16.69207,18.49776,20.34979,19.50817,20.90792,23.7383,22.32427,23.31808,;16.21027,16.13993,18.20247,19.59878,18.95603,20.61263,22.98729,21.77213,23.02279,;14.92526,14.13089,17.6112,18.31377,16.94699,20.02136,21.70228,19.7631,22.43152,;14.22105,13.52887,16.50778,17.60956,16.34497,18.91794,20.99807,19.16107,21.3281,;13.88172,13.48694,15.51333,17.27023,16.30304,17.92349,20.65874,19.11914,20.33365,;13.12246,12.54903,14.15357,16.51097,15.36513,16.56373,19.89947,18.18123,18.97389,;11.48349,10.96693,12.60667,14.87199,13.78303,15.01683,18.2605,16.59913,17.42699,;9.521387,9.283737,11.59819,12.9099,12.09984,14.00835,16.29841,14.91594,16.41851,;8.375495,8.101113,10.49694,11.764,10.91721,12.9071,15.15251,13.73332,15.31726,;6.930475,6.578518,9.287013,10.31898,9.394619,11.69717,13.70749,12.21072,14.10733,;5.704726,5.265178,8.29861,9.093236,8.081279,10.70877,12.48175,10.89738,13.11893,;4.723245,4.285852,7.39421,8.111755,7.101953,9.804371,11.50026,9.918055,12.21453,;3.360139,3.011989,5.721903,6.748649,5.82809,8.132063,10.13716,8.644191,10.54222,;2.122611,1.945631,3.698522,5.511121,4.761732,6.108682,8.899631,7.577834,8.518843,;1.441638,1.129074,2.986961,4.830148,3.945175,5.397122,8.218658,6.761276,7.807283,;0.5315632,0.1561281,2.551305,3.920073,2.972229,4.961466,7.308583,5.788331,7.371627,;-0.4197438,-0.8761183,1.540553,2.968766,1.939983,3.950714,6.357276,4.756084,6.360874,;-0.9097442,-1.58621,0.5565663,2.478766,1.229892,2.966727,5.867275,4.045993,5.376888,;-1.313094,-1.880557,0.2102114,2.075416,0.9355441,2.620372,5.463926,3.751645,5.030533,;-1.725898,-2.151863,-0.03909628,1.662612,0.6642381,2.371065,5.051122,3.480339,4.781225,;-2.115575,-2.620171,-0.01902707,1.272935,0.1959305,2.391134,4.661444,3.012032,4.801295,;-2.412717,-3.054235,0.3001471,0.9757928,-0.2381334,2.710308,4.364303,2.577968,5.120469,;-2.372926,-3.003442,0.5915033,1.015584,-0.187341,3.001664,4.404094,2.62876,5.411825,;-1.908949,-2.389102,0.6368664,1.47956,0.4269988,3.047027,4.86807,3.2431,5.457188,;-1.272513,-1.991996,0.4963225,2.115997,0.8241052,2.906483,5.504507,3.640206,5.316644,;-0.6206085,-1.438895,1.284361,2.767901,1.377206,3.694522,6.156411,4.193307,6.104683,;0.09649325,-0.2969013,2.699005,3.485003,2.5192,5.109165,6.873513,5.335301,7.519326,;1.000493,0.593017,2.845431,4.389003,3.409118,5.255592,7.777512,6.22522,7.665753,;1.485238,1.34156,3.375257,4.873748,4.157661,5.785418,8.262258,6.973763,8.195578,;2.287702,2.192013,4.866177,5.676212,5.008114,7.276338,9.064722,7.824215,9.686498,;3.849966,3.446241,6.382532,7.238476,6.262342,8.792693,10.62699,9.078443,11.20285,;5.139093,5.016734,7.536594,8.527603,7.832835,9.946755,11.91611,10.64894,12.35692,;6.220591,5.892039,8.560634,9.609101,8.70814,10.97079,12.99761,11.52424,13.38096,;8.496956,8.115735,9.919255,11.88547,10.93184,12.32942,15.27398,13.74794,14.73958,;9.758549,9.145305,10.95926,13.14706,11.96141,13.36943,16.53557,14.77751,15.77959,;10.24652,9.820546,12.40078,13.63503,12.63665,14.81094,17.02354,15.45275,17.22111,;11.20936,10.93118,13.54465,14.59787,13.74728,15.95481,17.98638,16.56338,18.36497,;13.66281,13.3159,15.14944,17.05132,16.132,17.5596,20.43983,18.9481,19.96976,;14.38378,13.73874,16.61078,17.77229,16.55484,19.02094,21.1608,19.37094,21.4311,;14.70039,14.27166,16.71515,18.0889,17.08776,19.12531,21.47741,19.90386,21.53547,;15.47749,15.21465,17.41244,18.866,18.03075,19.8226,22.25451,20.84685,22.23276,;16.41683,16.11741,18.41585,19.80534,18.93351,20.82601,23.19385,21.74961,23.23617,;17.0581,16.71791,18.89777,20.44661,19.53401,21.30793,23.83512,22.35011,23.71809,;17.29124,17.11799,19.37293,20.67975,19.93409,21.78309,24.06826,22.75019,24.19325,;17.27504,17.34414,19.62177,20.66355,20.16024,22.03193,24.05206,22.97634,24.4421,;17.20832,17.34421,19.43679,20.59683,20.16031,21.84695,23.98534,22.97642,24.25711,;17.12314,17.02005,19.09208,20.51165,19.83616,21.50224,23.90016,22.65226,23.9124,;17.06487,16.59156,18.76175,20.45338,19.40766,21.17191,23.84188,22.22376,23.58207,;16.96128,16.69207,18.49776,20.34979,19.50817,20.90792,23.7383,22.32427,23.31808,;16.21027,16.13993,18.20247,19.59878,18.95603,20.61263,22.98729,21.77213,23.02279,;14.92526,14.13089,17.6112,18.31377,16.94699,20.02136,21.70228,19.7631,22.43152,;14.22105,13.52887,16.50778,17.60956,16.34497,18.91794,20.99807,19.16107,21.3281,;13.88172,13.48694,15.51333,17.27023,16.30304,17.92349,20.65874,19.11914,20.33365,;13.12246,12.54903,14.15357,16.51097,15.36513,16.56373,19.89947,18.18123,18.97389,;11.48349,10.96693,12.60667,14.87199,13.78303,15.01683,18.2605,16.59913,17.42699,;9.521387,9.283737,11.59819,12.9099,12.09984,14.00835,16.29841,14.91594,16.41851,;8.375495,8.101113,10.49694,11.764,10.91721,12.9071,15.15251,13.73332,15.31726,;6.930475,6.578518,9.287013,10.31898,9.394619,11.69717,13.70749,12.21072,14.10733,;5.704726,5.265178,8.29861,9.093236,8.081279,10.70877,12.48175,10.89738,13.11893,;4.723245,4.285852,7.39421,8.111755,7.101953,9.804371,11.50026,9.918055,12.21453,;3.360139,3.011989,5.721903,6.748649,5.82809,8.132063,10.13716,8.644191,10.54222,;2.122611,1.945631,3.698522,5.511121,4.761732,6.108682,8.899631,7.577834,8.518843,;1.441638,1.129074,2.986961,4.830148,3.945175,5.397122,8.218658,6.761276,7.807283,;0.5315632,0.1561281,2.551305,3.920073,2.972229,4.961466,7.308583,5.788331,7.371627,;-0.4197438,-0.8761183,1.540553,2.968766,1.939983,3.950714,6.357276,4.756084,6.360874,;-0.9097442,-1.58621,0.5565663,2.478766,1.229892,2.966727,5.867275,4.045993,5.376888,;-1.313094,-1.880557,0.2102114,2.075416,0.9355441,2.620372,5.463926,3.751645,5.030533,;-1.725898,-2.151863,-0.03909628,1.662612,0.6642381,2.371065,5.051122,3.480339,4.781225,;-2.115575,-2.620171,-0.01902707,1.272935,0.1959305,2.391134,4.661444,3.012032,4.801295,;-2.412717,-3.054235,0.3001471,0.9757928,-0.2381334,2.710308,4.364303,2.577968,5.120469,;-2.372926,-3.003442,0.5915033,1.015584,-0.187341,3.001664,4.404094,2.62876,5.411825,;-1.908949,-2.389102,0.6368664,1.47956,0.4269988,3.047027,4.86807,3.2431,5.457188,;-1.272513,-1.991996,0.4963225,2.115997,0.8241052,2.906483,5.504507,3.640206,5.316644,;-0.6206085,-1.438895,1.284361,2.767901,1.377206,3.694522,6.156411,4.193307,6.104683,;0.09649325,-0.2969013,2.699005,3.485003,2.5192,5.109165,6.873513,5.335301,7.519326,;1.000493,0.593017,2.845431,4.389003,3.409118,5.255592,7.777512,6.22522,7.665753,;1.485238,1.34156,3.375257,4.873748,4.157661,5.785418,8.262258,6.973763,8.195578,;2.287702,2.192013,4.866177,5.676212,5.008114,7.276338,9.064722,7.824215,9.686498,;3.849966,3.446241,6.382532,7.238476,6.262342,8.792693,10.62699,9.078443,11.20285,;5.139093,5.016734,7.536594,8.527603,7.832835,9.946755,11.91611,10.64894,12.35692,;6.220591,5.892039,8.560634,9.609101,8.70814,10.97079,12.99761,11.52424,13.38096,;8.496956,8.115735,9.919255,11.88547,10.93184,12.32942,15.27398,13.74794,14.73958,;9.758549,9.145305,10.95926,13.14706,11.96141,13.36943,16.53557,14.77751,15.77959,;10.24652,9.820546,12.40078,13.63503,12.63665,14.81094,17.02354,15.45275,17.22111,;11.20936,10.93118,13.54465,14.59787,13.74728,15.95481,17.98638,16.56338,18.36497,;13.66281,13.3159,15.14944,17.05132,16.132,17.5596,20.43983,18.9481,19.96976,;14.38378,13.73874,16.61078,17.77229,16.55484,19.02094,21.1608,19.37094,21.4311,;14.70039,14.27166,16.71515,18.0889,17.08776,19.12531,21.47741,19.90386,21.53547,;15.47749,15.21465,17.41244,18.866,18.03075,19.8226,22.25451,20.84685,22.23276,;16.41683,16.11741,18.41585,19.80534,18.93351,20.82601,23.19385,21.74961,23.23617,;17.0581,16.71791,18.89777,20.44661,19.53401,21.30793,23.83512,22.35011,23.71809,;17.29124,17.11799,19.37293,20.67975,19.93409,21.78309,24.06826,22.75019,24.19325,;17.27504,17.34414,19.62177,20.66355,20.16024,22.03193,24.05206,22.97634,24.4421,;17.20832,17.34421,19.43679,20.59683,20.16031,21.84695,23.98534,22.97642,24.25711,;17.12314,17.02005,19.09208,20.51165,19.83616,21.50224,23.90016,22.65226,23.9124,;17.06487,16.59156,18.76175,20.45338,19.40766,21.17191,23.84188,22.22376,23.58207,;16.96128,16.69207,18.49776,20.34979,19.50817,20.90792,23.7383,22.32427,23.31808,;16.21027,16.13993,18.20247,19.59878,18.95603,20.61263,22.98729,21.77213,23.02279,;14.92526,14.13089,17.6112,18.31377,16.94699,20.02136,21.70228,19.7631,22.43152,;14.22105,13.52887,16.50778,17.60956,16.34497,18.91794,20.99807,19.16107,21.3281,;13.88172,13.48694,15.51333,17.27023,16.30304,17.92349,20.65874,19.11914,20.33365,;13.12246,12.54903,14.15357,16.51097,15.36513,16.56373,19.89947,18.18123,18.97389,;11.48349,10.96693,12.60667,14.87199,13.78303,15.01683,18.2605,16.59913,17.42699,;9.521387,9.283737,11.59819,12.9099,12.09984,14.00835,16.29841,14.91594,16.41851,;8.375495,8.101113,10.49694,11.764,10.91721,12.9071,15.15251,13.73332,15.31726,;6.930475,6.578518,9.287013,10.31898,9.394619,11.69717,13.70749,12.21072,14.10733,;5.704726,5.265178,8.29861,9.093236,8.081279,10.70877,12.48175,10.89738,13.11893,;4.723245,4.285852,7.39421,8.111755,7.101953,9.804371,11.50026,9.918055,12.21453,;3.360139,3.011989,5.721903,6.748649,5.82809,8.132063,10.13716,8.644191,10.54222,;2.122611,1.945631,3.698522,5.511121,4.761732,6.108682,8.899631,7.577834,8.518843,;1.441638,1.129074,2.986961,4.830148,3.945175,5.397122,8.218658,6.761276,7.807283,;0.5315632,0.1561281,2.551305,3.920073,2.972229,4.961466,7.308583,5.788331,7.371627,;-0.4197438,-0.8761183,1.540553,2.968766,1.939983,3.950714,6.357276,4.756084,6.360874,;-0.9097442,-1.58621,0.5565663,2.478766,1.229892,2.966727,5.867275,4.045993,5.376888,;-1.313094,-1.880557,0.2102114,2.075416,0.9355441,2.620372,5.463926,3.751645,5.030533,;-1.725898,-2.151863,-0.03909628,1.662612,0.6642381,2.371065,5.051122,3.480339,4.781225,;-2.115575,-2.620171,-0.01902707,1.272935,0.1959305,2.391134,4.661444,3.012032,4.801295,;-2.412717,-3.054235,0.3001471,0.9757928,-0.2381334,2.710308,4.364303,2.577968,5.120469,;-2.372926,-3.003442,0.5915033,1.015584,-0.187341,3.001664,4.404094,2.62876,5.411825,;-1.908949,-2.389102,0.6368664,1.47956,0.4269988,3.047027,4.86807,3.2431,5.457188,;-1.272513,-1.991996,0.4963225,2.115997,0.8241052,2.906483,5.504507,3.640206,5.316644,;-0.6206085,-1.438895,1.284361,2.767901,1.377206,3.694522,6.156411,4.193307,6.104683,;0.09649325,-0.2969013,2.699005,3.485003,2.5192,5.109165,6.873513,5.335301,7.519326,;1.000493,0.593017,2.845431,4.389003,3.409118,5.255592,7.777512,6.22522,7.665753,;1.485238,1.34156,3.375257,4.873748,4.157661,5.785418,8.262258,6.973763,8.195578,;2.287702,2.192013,4.866177,5.676212,5.008114,7.276338,9.064722,7.824215,9.686498,;3.849966,3.446241,6.382532,7.238476,6.262342,8.792693,10.62699,9.078443,11.20285,;5.139093,5.016734,7.536594,8.527603,7.832835,9.946755,11.91611,10.64894,12.35692,;6.220591,5.892039,8.560634,9.609101,8.70814,10.97079,12.99761,11.52424,13.38096,;8.496956,8.115735,9.919255,11.88547,10.93184,12.32942,15.27398,13.74794,14.73958,;9.758549,9.145305,10.95926,13.14706,11.96141,13.36943,16.53557,14.77751,15.77959,;10.24652,9.820546,12.40078,13.63503,12.63665,14.81094,17.02354,15.45275,17.22111,;11.20936,10.93118,13.54465,14.59787,13.74728,15.95481,17.98638,16.56338,18.36497,;13.66281,13.3159,15.14944,17.05132,16.132,17.5596,20.43983,18.9481,19.96976,;14.38378,13.73874,16.61078,17.77229,16.55484,19.02094,21.1608,19.37094,21.4311,;14.70039,14.27166,16.71515,18.0889,17.08776,19.12531,21.47741,19.90386,21.53547,;15.47749,15.21465,17.41244,18.866,18.03075,19.8226,22.25451,20.84685,22.23276,;16.41683,16.11741,18.41585,19.80534,18.93351,20.82601,23.19385,21.74961,23.23617,;17.0581,16.71791,18.89777,20.44661,19.53401,21.30793,23.83512,22.35011,23.71809,;17.29124,17.11799,19.37293,20.67975,19.93409,21.78309,24.06826,22.75019,24.19325,;17.27504,17.34414,19.62177,20.66355,20.16024,22.03193,24.05206,22.97634,24.4421,;17.20832,17.34421,19.43679,20.59683,20.16031,21.84695,23.98534,22.97642,24.25711,;17.12314,17.02005,19.09208,20.51165,19.83616,21.50224,23.90016,22.65226,23.9124,;17.06487,16.59156,18.76175,20.45338,19.40766,21.17191,23.84188,22.22376,23.58207,;16.96128,16.69207,18.49776,20.34979,19.50817,20.90792,23.7383,22.32427,23.31808,;16.21027,16.13993,18.20247,19.59878,18.95603,20.61263,22.98729,21.77213,23.02279,;14.92526,14.13089,17.6112,18.31377,16.94699,20.02136,21.70228,19.7631,22.43152,;14.22105,13.52887,16.50778,17.60956,16.34497,18.91794,20.99807,19.16107,21.3281,;13.88172,13.48694,15.51333,17.27023,16.30304,17.92349,20.65874,19.11914,20.33365,;13.12246,12.54903,14.15357,16.51097,15.36513,16.56373,19.89947,18.18123,18.97389,;11.48349,10.96693,12.60667,14.87199,13.78303,15.01683,18.2605,16.59913,17.42699,;9.521387,9.283737,11.59819,12.9099,12.09984,14.00835,16.29841,14.91594,16.41851,;8.375495,8.101113,10.49694,11.764,10.91721,12.9071,15.15251,13.73332,15.31726,;6.930475,6.578518,9.287013,10.31898,9.394619,11.69717,13.70749,12.21072,14.10733,;5.704726,5.265178,8.29861,9.093236,8.081279,10.70877,12.48175,10.89738,13.11893,;4.723245,4.285852,7.39421,8.111755,7.101953,9.804371,11.50026,9.918055,12.21453,;3.360139,3.011989,5.721903,6.748649,5.82809,8.132063,10.13716,8.644191,10.54222,;2.122611,1.945631,3.698522,5.511121,4.761732,6.108682,8.899631,7.577834,8.518843,;1.441638,1.129074,2.986961,4.830148,3.945175,5.397122,8.218658,6.761276,7.807283,;0.5315632,0.1561281,2.551305,3.920073,2.972229,4.961466,7.308583,5.788331,7.371627,;-0.4197438,-0.8761183,1.540553,2.968766,1.939983,3.950714,6.357276,4.756084,6.360874,;-0.9097442,-1.58621,0.5565663,2.478766,1.229892,2.966727,5.867275,4.045993,5.376888,;-1.313094,-1.880557,0.2102114,2.075416,0.9355441,2.620372,5.463926,3.751645,5.030533,;-1.725898,-2.151863,-0.03909628,1.662612,0.6642381,2.371065,5.051122,3.480339,4.781225,;-2.115575,-2.620171,-0.01902707,1.272935,0.1959305,2.391134,4.661444,3.012032,4.801295,;-2.412717,-3.054235,0.3001471,0.9757928,-0.2381334,2.710308,4.364303,2.577968,5.120469,;-2.372926,-3.003442,0.5915033,1.015584,-0.187341,3.001664,4.404094,2.62876,5.411825,;-1.908949,-2.389102,0.6368664,1.47956,0.4269988,3.047027,4.86807,3.2431,5.457188,;-1.272513,-1.991996,0.4963225,2.115997,0.8241052,2.906483,5.504507,3.640206,5.316644,;-0.6206085,-1.438895,1.284361,2.767901,1.377206,3.694522,6.156411,4.193307,6.104683,;0.09649325,-0.2969013,2.699005,3.485003,2.5192,5.109165,6.873513,5.335301,7.519326,;1.000493,0.593017,2.845431,4.389003,3.409118,5.255592,7.777512,6.22522,7.665753,;1.485238,1.34156,3.375257,4.873748,4.157661,5.785418,8.262258,6.973763,8.195578,;2.287702,2.192013,4.866177,5.676212,5.008114,7.276338,9.064722,7.824215,9.686498,;3.849966,3.446241,6.382532,7.238476,6.262342,8.792693,10.62699,9.078443,11.20285,;5.139093,5.016734,7.536594,8.527603,7.832835,9.946755,11.91611,10.64894,12.35692,;6.220591,5.892039,8.560634,9.609101,8.70814,10.97079,12.99761,11.52424,13.38096,;8.496956,8.115735,9.919255,11.88547,10.93184,12.32942,15.27398,13.74794,14.73958,;9.758549,9.145305,10.95926,13.14706,11.96141,13.36943,16.53557,14.77751,15.77959,;10.24652,9.820546,12.40078,13.63503,12.63665,14.81094,17.02354,15.45275,17.22111,;11.20936,10.93118,13.54465,14.59787,13.74728,15.95481,17.98638,16.56338,18.36497,;13.66281,13.3159,15.14944,17.05132,16.132,17.5596,20.43983,18.9481,19.96976,;14.38378,13.73874,16.61078,17.77229,16.55484,19.02094,21.1608,19.37094,21.4311,;14.70039,14.27166,16.71515,18.0889,17.08776,19.12531,21.47741,19.90386,21.53547,;15.47749,15.21465,17.41244,18.866,18.03075,19.8226,22.25451,20.84685,22.23276,;16.41683,16.11741,18.41585,19.80534,18.93351,20.82601,23.19385,21.74961,23.23617,;17.0581,16.71791,18.89777,20.44661,19.53401,21.30793,23.83512,22.35011,23.71809,;17.29124,17.11799,19.37293,20.67975,19.93409,21.78309,24.06826,22.75019,24.19325,;17.27504,17.34414,19.62177,20.66355,20.16024,22.03193,24.05206,22.97634,24.4421,;17.20832,17.34421,19.43679,20.59683,20.16031,21.84695,23.98534,22.97642,24.25711,;17.12314,17.02005,19.09208,20.51165,19.83616,21.50224,23.90016,22.65226,23.9124,;17.06487,16.59156,18.76175,20.45338,19.40766,21.17191,23.84188,22.22376,23.58207,;16.96128,16.69207,18.49776,20.34979,19.50817,20.90792,23.7383,22.32427,23.31808,;16.21027,16.13993,18.20247,19.59878,18.95603,20.61263,22.98729,21.77213,23.02279,;14.92526,14.13089,17.6112,18.31377,16.94699,20.02136,21.70228,19.7631,22.43152,;14.22105,13.52887,16.50778,17.60956,16.34497,18.91794,20.99807,19.16107,21.3281,;13.88172,13.48694,15.51333,17.27023,16.30304,17.92349,20.65874,19.11914,20.33365,;13.12246,12.54903,14.15357,16.51097,15.36513,16.56373,19.89947,18.18123,18.97389,;11.48349,10.96693,12.60667,14.87199,13.78303,15.01683,18.2605,16.59913,17.42699,;9.521387,9.283737,11.59819,12.9099,12.09984,14.00835,16.29841,14.91594,16.41851,;8.375495,8.101113,10.49694,11.764,10.91721,12.9071,15.15251,13.73332,15.31726,;6.930475,6.578518,9.287013,10.31898,9.394619,11.69717,13.70749,12.21072,14.10733,;5.704726,5.265178,8.29861,9.093236,8.081279,10.70877,12.48175,10.89738,13.11893,;4.723245,4.285852,7.39421,8.111755,7.101953,9.804371,11.50026,9.918055,12.21453,;3.360139,3.011989,5.721903,6.748649,5.82809,8.132063,10.13716,8.644191,10.54222,;2.122611,1.945631,3.698522,5.511121,4.761732,6.108682,8.899631,7.577834,8.518843,;1.441638,1.129074,2.986961,4.830148,3.945175,5.397122,8.218658,6.761276,7.807283,;0.5315632,0.1561281,2.551305,3.920073,2.972229,4.961466,7.308583,5.788331,7.371627,;-0.4197438,-0.8761183,1.540553,2.968766,1.939983,3.950714,6.357276,4.756084,6.360874,;-0.9097442,-1.58621,0.5565663,2.478766,1.229892,2.966727,5.867275,4.045993,5.376888,;-1.313094,-1.880557,0.2102114,2.075416,0.9355441,2.620372,5.463926,3.751645,5.030533,;-1.725898,-2.151863,-0.03909628,1.662612,0.6642381,2.371065,5.051122,3.480339,4.781225,;-2.115575,-2.620171,-0.01902707,1.272935,0.1959305,2.391134,4.661444,3.012032,4.801295,;-2.412717,-3.054235,0.3001471,0.9757928,-0.2381334,2.710308,4.364303,2.577968,5.120469,;-2.372926,-3.003442,0.5915033,1.015584,-0.187341,3.001664,4.404094,2.62876,5.411825,;-1.908949,-2.389102,0.6368664,1.47956,0.4269988,3.047027,4.86807,3.2431,5.457188,;-1.272513,-1.991996,0.4963225,2.115997,0.8241052,2.906483,5.504507,3.640206,5.316644,;-0.6206085,-1.438895,1.284361,2.767901,1.377206,3.694522,6.156411,4.193307,6.104683,;0.09649325,-0.2969013,2.699005,3.485003,2.5192,5.109165,6.873513,5.335301,7.519326,;1.000493,0.593017,2.845431,4.389003,3.409118,5.255592,7.777512,6.22522,7.665753,;1.485238,1.34156,3.375257,4.873748,4.157661,5.785418,8.262258,6.973763,8.195578,;2.287702,2.192013,4.866177,5.676212,5.008114,7.276338,9.064722,7.824215,9.686498,;3.849966,3.446241,6.382532,7.238476,6.262342,8.792693,10.62699,9.078443,11.20285,;5.139093,5.016734,7.536594,8.527603,7.832835,9.946755,11.91611,10.64894,12.35692,;6.220591,5.892039,8.560634,9.609101,8.70814,10.97079,12.99761,11.52424,13.38096,;8.496956,8.115735,9.919255,11.88547,10.93184,12.32942,15.27398,13.74794,14.73958,;9.758549,9.145305,10.95926,13.14706,11.96141,13.36943,16.53557,14.77751,15.77959,;10.24652,9.820546,12.40078,13.63503,12.63665,14.81094,17.02354,15.45275,17.22111,;11.20936,10.93118,13.54465,14.59787,13.74728,15.95481,17.98638,16.56338,18.36497,;13.66281,13.3159,15.14944,17.05132,16.132,17.5596,20.43983,18.9481,19.96976,;14.38378,13.73874,16.61078,17.77229,16.55484,19.02094,21.1608,19.37094,21.4311,;14.70039,14.27166,16.71515,18.0889,17.08776,19.12531,21.47741,19.90386,21.53547,;15.47749,15.21465,17.41244,18.866,18.03075,19.8226,22.25451,20.84685,22.23276,;16.41683,16.11741,18.41585,19.80534,18.93351,20.82601,23.19385,21.74961,23.23617,;17.0581,16.71791,18.89777,20.44661,19.53401,21.30793,23.83512,22.35011,23.71809,;17.29124,17.11799,19.37293,20.67975,19.93409,21.78309,24.06826,22.75019,24.19325,;17.27504,17.34414,19.62177,20.66355,20.16024,22.03193,24.05206,22.97634,24.4421,;17.20832,17.34421,19.43679,20.59683,20.16031,21.84695,23.98534,22.97642,24.25711,;17.12314,17.02005,19.09208,20.51165,19.83616,21.50224,23.90016,22.65226,23.9124,;17.06487,16.59156,18.76175,20.45338,19.40766,21.17191,23.84188,22.22376,23.58207,;16.96128,16.69207,18.49776,20.34979,19.50817,20.90792,23.7383,22.32427,23.31808,;16.21027,16.13993,18.20247,19.59878,18.95603,20.61263,22.98729,21.77213,23.02279,;14.92526,14.13089,17.6112,18.31377,16.94699,20.02136,21.70228,19.7631,22.43152,;14.22105,13.52887,16.50778,17.60956,16.34497,18.91794,20.99807,19.16107,21.3281,;13.88172,13.48694,15.51333,17.27023,16.30304,17.92349,20.65874,19.11914,20.33365,;13.12246,12.54903,14.15357,16.51097,15.36513,16.56373,19.89947,18.18123,18.97389,;11.48349,10.96693,12.60667,14.87199,13.78303,15.01683,18.2605,16.59913,17.42699,;9.521387,9.283737,11.59819,12.9099,12.09984,14.00835,16.29841,14.91594,16.41851,;8.375495,8.101113,10.49694,11.764,10.91721,12.9071,15.15251,13.73332,15.31726,;6.930475,6.578518,9.287013,10.31898,9.394619,11.69717,13.70749,12.21072,14.10733,;5.704726,5.265178,8.29861,9.093236,8.081279,10.70877,12.48175,10.89738,13.11893,;4.723245,4.285852,7.39421,8.111755,7.101953,9.804371,11.50026,9.918055,12.21453,;3.360139,3.011989,5.721903,6.748649,5.82809,8.132063,10.13716,8.644191,10.54222,;2.122611,1.945631,3.698522,5.511121,4.761732,6.108682,8.899631,7.577834,8.518843,;1.441638,1.129074,2.986961,4.830148,3.945175,5.397122,8.218658,6.761276,7.807283,;0.5315632,0.1561281,2.551305,3.920073,2.972229,4.961466,7.308583,5.788331,7.371627,;-0.4197438,-0.8761183,1.540553,2.968766,1.939983,3.950714,6.357276,4.756084,6.360874,;-0.9097442,-1.58621,0.5565663,2.478766,1.229892,2.966727,5.867275,4.045993,5.376888,;-1.313094,-1.880557,0.2102114,2.075416,0.9355441,2.620372,5.463926,3.751645,5.030533,;-1.725898,-2.151863,-0.03909628,1.662612,0.6642381,2.371065,5.051122,3.480339,4.781225,;-2.115575,-2.620171,-0.01902707,1.272935,0.1959305,2.391134,4.661444,3.012032,4.801295,;-2.412717,-3.054235,0.3001471,0.9757928,-0.2381334,2.710308,4.364303,2.577968,5.120469,;-2.372926,-3.003442,0.5915033,1.015584,-0.187341,3.001664,4.404094,2.62876,5.411825,;-1.908949,-2.389102,0.6368664,1.47956,0.4269988,3.047027,4.86807,3.2431,5.457188,;-1.272513,-1.991996,0.4963225,2.115997,0.8241052,2.906483,5.504507,3.640206,5.316644,;-0.6206085,-1.438895,1.284361,2.767901,1.377206,3.694522,6.156411,4.193307,6.104683,;0.09649325,-0.2969013,2.699005,3.485003,2.5192,5.109165,6.873513,5.335301,7.519326,;1.000493,0.593017,2.845431,4.389003,3.409118,5.255592,7.777512,6.22522,7.665753,;1.485238,1.34156,3.375257,4.873748,4.157661,5.785418,8.262258,6.973763,8.195578,;2.189528,2.096725,4.807127,5.578037,4.912826,7.217288,8.966547,7.728927,9.627449,; diff --git a/lib/maths/unittest/testfiles/CMultivariateTimeSeriesModel.6.2.state.xml b/lib/maths/unittest/testfiles/CMultivariateTimeSeriesModel.6.2.state.xml new file mode 100644 index 0000000000..25618a35a7 --- /dev/null +++ b/lib/maths/unittest/testfiles/CMultivariateTimeSeriesModel.6.2.state.xml @@ -0,0 +1,2800 @@ + + 0 + + 2 + + 1.000000 + 1:1 + 17570810039929444164:4496958953345110445 + + 3 + 100.9787:11.01138 + 100.9787:10.00121 + 100.9787:12.0222 + + + 3 + 26.47849:-4.889154e-2 + 26.47849:-1.188766e-2 + 26.47849:-2.386449e-2 + + + 3 + 26.47849:1.595815 + 26.47849:1.220594 + 26.47849:1.116212 + + + 3 + 100.9787:1.640196 + 100.9787:1.311445 + 100.9787:1.169313 + + + + 1.000000 + 1:1 + 15082986415760524671:18412454960531526315 + + 3 + 100.1638:1.065868e-2 + 100.1638:6.121291e-2 + 100.1638:-7.698335e-2 + + + 3 + 26.47557:-5.153934e-2 + 26.47557:-5.132269e-2 + 26.47557:-2.522258e-2 + + + 3 + 26.47557:1.594071 + 26.47557:1.217627 + 26.47557:1.117153 + + + 3 + 100.1638:1.632956 + 100.1638:1.300019 + 100.1638:1.167165 + + + + + + 1.2e-2 + 599400 + 599400 + + + 0 + 1 + + 2074200 + + 0.012000 + 0 + + 0:0,0,0,0,0,0,0,0,0,0,0 + + 0:0,0:0,0 + + + + + 1 + 2 + + 1209600 + 0 + 3024000 + + 3600 + 1209600 + + 336 + 5.938603:12.22945 + 5.938603:14.32173 + 5.938603:17.2191 + 5.938603:18.04727 + 5.938603:19.27789 + 5.938603:19.87772 + 5.938603:20.04129 + 5.938603:20.48034 + 5.938603:19.14971 + 5.938603:18.11358 + 5.938603:14.36026 + 5.938603:10.82368 + 5.938603:10.42216 + 5.938603:6.290776 + 5.938603:7.129382 + 5.938603:3.271674 + 5.938603:2.358605 + 5.938603:6.39548e-1 + 5.938603:-5.392803e-1 + 5.938603:2.04939 + 5.938603:2.923434 + 5.938603:4.016687 + 5.938603:7.453522 + 5.938603:10.9128 + 5.948792:12.69577 + 5.948792:15.00822 + 5.948792:18.16217 + 5.948792:18.89222 + 5.948792:20.59734 + 5.948792:19.20771 + 5.948792:20.23743 + 5.948792:19.06582 + 5.948792:19.73045 + 5.948792:16.76169 + 5.948792:14.3052 + 5.948792:13.1676 + 5.948792:9.707491 + 5.948792:6.208685 + 5.948792:3.862444 + 5.948792:3.095824 + 5.948792:2.688135 + 5.948792:1.468634 + 5.948792:4.209245e-1 + 5.948792:1.837605 + 5.948792:3.325111 + 5.948792:5.854228 + 5.948792:5.910984 + 5.948792:10.62389 + 5.958999:12.86436 + 5.958999:15.69732 + 5.958999:16.41929 + 5.958999:19.34313 + 5.958999:20.40804 + 5.958999:21.30992 + 5.958999:21.52436 + 5.958999:19.97502 + 5.958999:18.43804 + 5.958999:17.13613 + 5.958999:15.78658 + 5.958999:12.02771 + 5.958999:10.10472 + 5.958999:8.428082 + 5.958999:4.759506 + 5.958999:4.383882 + 5.958999:1.880875 + 5.958999:2.743031 + 5.958999:1.607084e-1 + 5.958999:9.922436e-1 + 5.958999:2.259369 + 5.958999:5.27984 + 5.958999:8.420159 + 5.958999:10.29963 + 5.969223:11.21646 + 5.969223:15.00902 + 5.969223:17.26268 + 5.969223:19.05879 + 5.969223:19.56757 + 5.969223:21.79676 + 5.969223:20.86945 + 5.969223:20.7433 + 5.969223:18.31111 + 5.969223:17.94045 + 5.969223:16.07072 + 5.969223:12.19213 + 5.969223:10.28051 + 5.969223:8.797435 + 5.969223:5.128645 + 5.969223:4.086825 + 5.969223:2.812014 + 5.969223:1.058348 + 5.969223:1.555016 + 5.969223:1.563751 + 5.969223:2.329493 + 5.969223:3.010607 + 5.969223:6.749099 + 5.969223:10.07613 + 5.979464:11.97813 + 5.979464:13.83097 + 5.979464:17.68171 + 5.979464:17.43135 + 5.979464:20.17145 + 5.979464:21.12093 + 5.979464:20.04399 + 5.979464:20.53443 + 5.979464:19.44004 + 5.979464:16.20558 + 5.979464:16.05906 + 5.979464:12.26813 + 5.979464:9.960791 + 5.979464:6.525975 + 5.979464:3.926669 + 5.979464:2.673707 + 5.979464:2.298069 + 5.979464:1.002956 + 5.979464:9.38545e-1 + 5.979464:1.037944 + 5.979464:3.698119 + 5.979464:4.403436 + 5.979464:6.042991 + 5.979464:8.44635 + 5.989723:13.2492 + 5.989723:14.159 + 5.989723:17.70345 + 5.989723:18.5507 + 5.989723:20.34288 + 5.989723:21.63313 + 5.989723:20.21939 + 5.989723:20.92202 + 5.989723:18.54827 + 5.989723:18.35219 + 5.989723:15.78189 + 5.989723:12.30869 + 5.989723:8.954588 + 5.989723:7.24049 + 5.989723:5.707849 + 5.989723:3.588516 + 5.989723:1.061881 + 5.989723:2.017426 + 5.989723:1.44435 + 5.989723:2.97195 + 5.989723:3.317296 + 5.989723:5.015072 + 5.989723:6.436425 + 5.989723:8.079546 + 6:13.27096 + 6:15.14513 + 6:18.2525 + 6:18.36507 + 6:18.47144 + 6:22.93664 + 6:20.88083 + 6:20.51248 + 6:19.13054 + 6:17.86622 + 6:15.35033 + 6:11.33194 + 6:10.14184 + 6:6.667609 + 6:4.098285 + 6:3.543013 + 6:3.02948 + 6:1.537444 + 6:-3.808537e-1 + 6:5.863404e-1 + 6:3.696506 + 6:5.186293 + 4:5.685474 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + + + + 4;0;86400;0;604800 + + + + + 2 + 1 + + + 0 + 2073600 + + 168 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + + + + 0 + 7516800 + + 168 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + + + + + + 2 + 1 + + 2 + + 518400 + + 0 + 3.28283e-3;1:1.360629e-1;55.99475:3.953881e-1;38.33727:6.011828e-1;42.38342:7.748663e-1;44.86595:9.775177e-1;48.98225:1.290021;56.15848:1.625067;53.91111:1.890138;40.34933:2.141794;22.44448:2.400405;33.39662:2.584102;16.66899:2.832344;24.66828:3.228825;32.49077:3.808226;20.56248:4.381586;11.82235:4.908168;6.834281:5.732914;5.904859:6.787525;1 + 557.7757 + + 44:2097296:3145872:1048712 + 345600;11.30527:432000;16.95104:518400;6.056703 + + + + + 3 + 1 + + + + + 36 + 0 + + 1 + 0:1026.582:3116.897:5231.259:7304.936:9420.301:11738.12:14207.3:17220.03:19982.96:22809.23:25785.81:28760.26:31436.75:33945.29:36147.62:38268.86:40448.05:42546.84:44430.07:46698.14:48807.84:51028.52:53380.57:55917.22:58351.17:61020.47:63838.29:66614.28:69458.76:72001.3:74605.88:76981.05:79113.97:81254.02:83336.92:85200:86400 + 11.96935:11.96935:13.66543:13.76066:16.54758:17.74504:18.19381:19.4191:20.54135:20.67879:20.57272:20.45728:19.98206:18.17822:17.43718:16.78879:15.02006:12.70224:11.41723:9.790198:8.550868:7.062588:5.50027:4.600643:3.177143:2.449587:1.844487:1.228904:9.521402e-1:1.638672:2.557643:3.586718:4.682845:5.219392:6.96189:8.484973:9.612823:9.612823 + 6.550615:6.550615:3.843127:3.519941:6.724232:3.159359:3.413064:3.26024:4.139425:3.510112:2.844913:3.002919:4.517162:4.308549:5.170891:3.918164:6.480599:4.668423:2.396291:3.535941:5.213679:4.957764:6.012614:5.319609:5.456476:3.871075:4.497241:4.105316:1.777678:2.386994:3.009257:2.680742:3.581918:4.027559:4.111287:4.158289:2.728887:2.728887 + + + 9094411231511854520:13258687326618484610 + + + 1.2e-2 + 0:2020.29:4055.415:6127.674:8270.605:10492.42:12878.26:15553.06:18538.63:21487.09:24396.34:27319.8:30172.48:32730.34:35066.89:37258.08:39369.66:41431.55:43487.89:45567.44:47692.14:49871.19:52133.87:54529.76:57068.53:59659.26:62417.09:65351.31:68092.25:70704.19:73300.44:75783.02:78081.79:80260.44:82357.27:84392.54:86400 + 987.4259:3087.511:5190.983:7279.403:9385.331:11727.52:14179.7:17104.71:19932.25:22806.6:25790.2:28773.01:31454.55:33931.9:36194.04:38301.84:40464.25:42560.27:44422.51:46637.4:48835.62:51021.73:53301.66:55879.98:58391.9:61077.3:63854.31:66609.89:69416.31:72000.89:74632.62:77016.56:79135.45:81151.7:83336.92:85200 + 3.85995:1596.673 + 3.85995:37970.73 + + + 0:0:604800:86400:0 + + 0 + + 12.93104:1,5.772041e-1,4.216148e-1,12.25414,7.178575 + + + 11.93394:1,5.653537e-1,4.026058e-1,13.84859,7.880185 + + + 13.03584:1,5.776699e-1,4.195317e-1,14.28144,8.260672 + + + 12.3296:1,5.559704e-1,3.967503e-1,16.4697,9.184388 + + + 13.76041:1,5.850688e-1,4.274673e-1,18.0194,10.67022 + + + 14.38726:1,5.825539e-1,4.245109e-1,18.27967,10.65166 + + + 15.55023:1,5.77093e-1,4.173154e-1,18.83249,10.69487 + + + 18.21157:1,6.045655e-1,4.425525e-1,20.33859,12.30958 + + + 18.03758:1,6.038855e-1,4.451831e-1,21.43641,13.20661 + + + 17.54268:1,6.168976e-1,4.592137e-1,20.69822,12.79684 + + + 16.17844:1,6.238278e-1,4.712861e-1,20.53335,12.86518 + + + 15.5885:1,6.376105e-1,4.885831e-1,20.22736,13.02503 + + + 14.21408:1,6.212105e-1,4.694421e-1,17.98145,11.05836 + + + 13.53561:1,6.179018e-1,4.688552e-1,17.54651,10.90604 + + + 13.13119:1,6.329644e-1,4.885535e-1,16.72482,10.65399 + + + 11.92948:1,6.151822e-1,4.656253e-1,14.8194,9.112947 + + + 12.82257:1,6.384595e-1,4.937147e-1,12.28235,7.695634 + + + 11.87279:1,6.202887e-1,4.721845e-1,11.6191,7.228788 + + + 11.93274:1,6.117563e-1,4.655108e-1,9.727486,5.848292 + + + 13.04995:1,6.542777e-1,5.128518e-1,8.290334,5.305095 + + + 13.26825:1,6.475014e-1,5.053909e-1,6.994665,4.498858 + + + 12.62116:1,6.315556e-1,4.833131e-1,5.055216,3.032004 + + + 14.22354:1,6.524647e-1,5.108565e-1,4.587682,3.001048 + + + 15.73617:1,6.820865e-1,5.456874e-1,3.231804,2.142596 + + + 14.61508:1,6.608164e-1,5.202972e-1,2.420468,1.539997 + + + 15.79609:1,6.793111e-1,5.464679e-1,2.052848,1.473458 + + + 15.4063:1,6.697738e-1,5.314978e-1,1.126382,6.883445e-1 + + + 16.27836:1,6.90209e-1,5.578469e-1,6.531608e-1,4.316119e-1 + + + 14.96058:1,6.702192e-1,5.346555e-1,1.283004,7.953243e-1 + + + 15.74141:1,7.172111e-1,5.917699e-1,2.570656,1.916273 + + + 14.07348:1,6.828066e-1,5.54974e-1,3.638569,2.56003 + + + 13.39493:1,6.901719e-1,5.638655e-1,4.995776,3.571062 + + + 12.01298:1,6.84704e-1,5.561264e-1,5.046951,3.338219 + + + 10.8293:1,6.591108e-1,5.198342e-1,6.945261,4.484179 + + + 8.733809:1,5.784242e-1,4.061064e-1,8.484973,4.749748 + + + 8.751767:1,5.503663e-1,3.836052e-1,9.612823,5.034814 + + 6.629004:5.168701:3.123913:5.625604:3.314742:4.605895:4.376786:3.229295:5.998186:3.316691:3.398127:5.234853:4.019297:4.452399:2.974124:5.321921:4.300025:2.866863:3.788174:4.926954:4.376476:5.752486:4.13043:4.653016:3.11399:4.84796:3.817516:1.707136:3.025557:2.766412:2.566775:3.171733:3.945025:4.10339:4.158289:2.728887 + 520200:522000:524400:526200:528600:531000:533400:536400:539400:542400:545400:548400:550800:553200:555600:557400:559800:561600:563400:565800:568200:570000:572400:575400:577800:580800:583200:586200:588600:591600:594000:596400:598200:599400:516000:517800 + + 2.97619047619048e-2,2.97619047619048e-2,2.95453198273302,4.99906528522253,126.383027202696,214.756081026524,364.968358133274 + + + + + 1 + 556.811:4.206614;556.811:52.31532; + + + + + + + + 1.2e-2 + 599400 + 599400 + + + 0 + 1 + + 2074200 + + 0.012000 + 0 + + 0:0,0,0,0,0,0,0,0,0,0,0 + + 0:0,0:0,0 + + + + + 1 + 2 + + 1209600 + 0 + 3024000 + + 3600 + 1209600 + + 336 + 5.938603:11.11864 + 5.938603:13.02348 + 5.938603:15.3574 + 5.938603:17.25426 + 5.938603:18.17257 + 5.938603:19.5384 + 5.938603:19.32845 + 5.938603:19.0914 + 5.938603:18.55198 + 5.938603:16.67941 + 5.938603:14.01237 + 5.938603:9.99258 + 5.938603:9.527705 + 5.938603:5.773245 + 5.938603:5.78929 + 5.938603:2.20544 + 5.938603:1.384659 + 5.938603:-3.883223e-1 + 5.938603:-1.253107 + 5.938603:1.319585e-1 + 5.938603:1.546283 + 5.938603:3.130431 + 5.938603:6.443229 + 5.938603:9.630959 + 5.948792:11.31623 + 5.948792:13.76481 + 5.948792:16.14958 + 5.948792:17.98628 + 5.948792:19.74916 + 5.948792:18.81137 + 5.948792:19.43665 + 5.948792:18.97883 + 5.948792:18.49123 + 5.948792:15.79001 + 5.948792:13.61882 + 5.948792:12.27529 + 5.948792:8.839604 + 5.948792:6.002207 + 5.948792:3.032425 + 5.948792:1.936481 + 5.948792:1.301257 + 5.948792:-2.110939e-2 + 5.948792:-7.397629e-1 + 5.948792:3.557993e-1 + 5.948792:2.545373 + 5.948792:4.791344 + 5.948792:4.557694 + 5.948792:9.184454 + 5.958999:12.20975 + 5.958999:14.15008 + 5.958999:15.71616 + 5.958999:18.21251 + 5.958999:19.21719 + 5.958999:20.30669 + 5.958999:19.81222 + 5.958999:19.24012 + 5.958999:17.3682 + 5.958999:16.16728 + 5.958999:14.61925 + 5.958999:10.94899 + 5.958999:9.132174 + 5.958999:7.205705 + 5.958999:3.654331 + 5.958999:2.588185 + 5.958999:9.132288e-1 + 5.958999:1.073596 + 5.958999:-7.664828e-1 + 5.958999:1.504747e-1 + 5.958999:1.302147 + 5.958999:4.377644 + 5.958999:7.180356 + 5.958999:8.91069 + 5.969223:9.988378 + 5.969223:14.17367 + 5.969223:16.32056 + 5.969223:18.63007 + 5.969223:18.53746 + 5.969223:20.81869 + 5.969223:20.04202 + 5.969223:19.59295 + 5.969223:16.83781 + 5.969223:17.14522 + 5.969223:14.90919 + 5.969223:11.54201 + 5.969223:9.30621 + 5.969223:7.539864 + 5.969223:3.219122 + 5.969223:2.730919 + 5.969223:1.336624 + 5.969223:1.007133e-1 + 5.969223:7.377021e-1 + 5.969223:8.235351e-1 + 5.969223:1.503211 + 5.969223:2.884641 + 5.969223:5.964855 + 5.969223:8.919652 + 5.979464:11.08073 + 5.979464:12.85962 + 5.979464:16.44128 + 5.979464:16.22314 + 5.979464:19.70646 + 5.979464:20.12242 + 5.979464:19.4058 + 5.979464:19.62494 + 5.979464:18.38927 + 5.979464:15.35956 + 5.979464:14.89005 + 5.979464:11.13383 + 5.979464:9.16378 + 5.979464:5.76144 + 5.979464:3.361221 + 5.979464:1.628195 + 5.979464:1.120487 + 5.979464:2.892461e-1 + 5.979464:-7.891318e-1 + 5.979464:4.72888e-2 + 5.979464:2.236467 + 5.979464:3.632168 + 5.979464:5.684954 + 5.979464:7.964773 + 5.989723:11.83846 + 5.989723:13.31179 + 5.989723:16.81229 + 5.989723:17.47882 + 5.989723:19.36336 + 5.989723:21.22986 + 5.989723:19.7791 + 5.989723:19.95551 + 5.989723:17.70488 + 5.989723:16.73627 + 5.989723:14.55351 + 5.989723:11.49932 + 5.989723:7.874352 + 5.989723:5.999305 + 5.989723:5.015462 + 5.989723:2.808143 + 5.989723:1.687334e-1 + 5.989723:1.352995 + 5.989723:-8.485115e-2 + 5.989723:1.161877 + 5.989723:2.171371 + 5.989723:3.981488 + 5.989723:5.248373 + 5.989723:7.530467 + 6:11.68235 + 6:13.70832 + 6:16.77509 + 6:17.41961 + 6:18.18026 + 6:21.38137 + 6:19.86634 + 6:19.81809 + 6:18.33732 + 6:16.32297 + 6:14.21246 + 6:10.7797 + 6:9.208052 + 6:5.821321 + 6:3.528982 + 6:3.076262 + 6:1.601845 + 6:2.816389e-1 + 6:-1.018912 + 6:-2.546187e-2 + 6:2.846622 + 6:3.740311 + 4:4.556145 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + + + + 4;0;86400;0;604800 + + + + + 2 + 1 + + + 0 + 2073600 + + 168 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + + + + 0 + 7516800 + + 168 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + + + + + + 2 + 1 + + 2 + + 518400 + + 0 + 3.711699e-3;9.646403e-1:1.2298e-1;54.11191:3.464809e-1;57.92318:5.621577e-1;56.97047:7.354424e-1;43.39591:9.08304e-1;45.0889:1.110118;41.2424:1.367487;59.90046:1.643492;27.46762:1.854797;36.26612:2.061915;27.57384:2.284266;26.53891:2.543395;21.50299:2.820441;14.69256:3.067519;14.75135:3.452802;8.786853:3.830364;6.834564:4.537972;12.77492:5.398485;9.880717e-1 + 557.7757 + + 44:1048720:3145872:1048712 + 345600;4.469707:432000;14.39436:518400;4.890023 + + + + + 3 + 1 + + + + + 36 + 0 + + 1 + 0:1028.209:3201.338:5386.713:7481.754:9665.898:12068.86:14586.64:17305.43:20505.17:23435.01:26501.08:29093.55:31542.21:34070.59:36242.7:38316.14:40562.67:42592.83:44580.85:46869.02:48920.95:51067.65:53408.2:55793.29:58269.67:61127.59:63993.41:66793.65:69651.3:72112.7:74509.03:76879.3:78956.88:81084.74:83199.54:85200.01:86400 + 10.96325:10.96325:12.46108:12.99874:15.82616:16.52982:17.38262:18.72019:19.62989:20.13516:19.94514:19.34232:19.17044:16.63572:16.38264:15.58405:13.83239:11.78828:10.46601:8.679382:7.400974:5.988424:4.695963:3.618752:2.232642:1.179676:7.612669e-1:5.458564e-2:-2.992638e-1:5.916563e-1:1.113591:2.553144:3.655159:4.556406:5.803113:7.644737:8.676103:8.676103 + 5.497946:5.497946:2.766031:2.054935:4.299906:2.12329:2.425665:2.274054:3.3501:2.250415:1.620111:1.923992:4.124959:2.777456:3.758187:1.739032:4.445366:2.953074:2.162628:3.037565:3.471119:3.436738:4.606715:3.010836:4.101185:3.741751:3.143497:2.359675:1.786849:2.183908:1.241995:2.411795:2.364605:2.512632:3.870003:2.126389:9.572408e-1:9.572408e-1 + + + 9094411231511854520:13258687326618484610 + + + 1.2e-2 + 0:2098.075:4190.262:6319.74:8515.873:10790.55:13211.87:15882.4:18792.71:22023.38:25046.43:27795.26:30376.3:32856.03:35165.94:37360.05:39475.75:41542.46:43609.73:45691.56:47804.23:49960.55:52182.82:54515.53:57027.31:59700.75:62499.22:65444.96:68242.8:70816.66:73317.86:75697.88:77952.61:80129.92:82253.44:84330.39:86400 + 993.9225:3158.065:5310.76:7486.5:9652.498:12049.1:14612.23:17328.84:20481.24:23426.35:26473.65:29095.08:31532.01:34027.51:36257.7:38334.12:40545.77:42594.38:44612.9:46854.19:48915.41:51053.6:53331.26:55795.16:58322.17:61146.87:64046.46:66816.91:69637.26:72157.41:74556.73:76862.84:78941.84:81025.34:83199.54:85200.01 + 3.85995:7930.355 + 3.85995:25321.05 + + + 0:0:604800:86400:0 + + 0 + + 14.95723:1,5.5669e-1,4.001396e-1,10.97629,6.124944 + + + 13.95829:1,5.477125e-1,3.837683e-1,12.59545,6.918186 + + + 15.09941:1,5.559895e-1,3.964965e-1,13.28471,7.392062 + + + 15.37879:1,5.600104e-1,4.032077e-1,15.57184,8.75283 + + + 14.75277:1,5.486614e-1,3.859316e-1,16.82753,9.381665 + + + 17.42408:1,5.850147e-1,4.282905e-1,17.42713,10.19453 + + + 17.7188:1,5.65625e-1,4.034783e-1,18.37437,10.30792 + + + 20.20143:1,5.824487e-1,4.228781e-1,19.59614,11.47078 + + + 21.9768:1,5.885747e-1,4.237661e-1,20.33984,12.1171 + + + 20.20208:1,6.019836e-1,4.407935e-1,20.0578,12.13089 + + + 18.4258:1,6.010367e-1,4.474323e-1,19.36028,11.66223 + + + 16.86442:1,5.9193e-1,4.361378e-1,19.54185,11.80305 + + + 16.55346:1,5.981389e-1,4.438562e-1,16.63271,9.826719 + + + 15.83937:1,6.034307e-1,4.538957e-1,16.36662,9.9033 + + + 15.28525:1,6.21064e-1,4.698868e-1,15.50124,9.65614 + + + 13.99247:1,5.774539e-1,4.247206e-1,13.67656,7.853074 + + + 14.83644:1,6.325215e-1,4.838201e-1,11.56713,7.239044 + + + 13.84307:1,5.860063e-1,4.346133e-1,10.59008,6.229441 + + + 14.86784:1,6.294718e-1,4.862935e-1,8.646584,5.341553 + + + 13.96393:1,6.193019e-1,4.678293e-1,7.32076,4.424037 + + + 15.13319:1,6.173599e-1,4.733606e-1,5.895943,3.57307 + + + 14.44493:1,6.180112e-1,4.670992e-1,4.480606,2.722212 + + + 16.02206:1,6.387405e-1,4.937044e-1,3.587009,2.32652 + + + 17.91619:1,6.497892e-1,5.098447e-1,2.429379,1.601858 + + + 17.55015:1,6.39564e-1,4.928954e-1,1.164916,6.890491e-1 + + + 18.77652:1,6.711406e-1,5.310642e-1,8.250089e-1,6.20692e-1 + + + 19.47085:1,6.651057e-1,5.262142e-1,4.634029e-2,5.568746e-3 + + + 18.27962:1,6.486449e-1,5.036101e-1,-4.738159e-1,-3.301849e-1 + + + 18.26476:1,6.824807e-1,5.500306e-1,3.162841e-1,1.987019e-1 + + + 16.75546:1,6.630508e-1,5.256892e-1,1.507293,1.14898 + + + 16.01501:1,6.739722e-1,5.40677e-1,2.525459,1.7391 + + + 14.45252:1,6.414519e-1,5.026737e-1,3.62354,2.334751 + + + 15.12958:1,6.925958e-1,5.639349e-1,4.4425,3.017113 + + + 12.95191:1,6.233245e-1,4.767059e-1,5.820678,3.524113 + + + 10.82555:1,5.953762e-1,4.265636e-1,7.644737,4.472349 + + + 10.81361:1,5.45099e-1,3.786331e-1,8.676103,4.580457 + + 4.473383:3.042295:1.747516:3.831976:2.109376:3.282854:2.524965:2.613281:3.481879:1.439824:2.325656:4.274657:2.519516:3.379746:1.528509:3.662614:2.649827:2.162102:3.116345:3.307164:2.964322:3.92821:2.449426:3.625827:3.204999:3.104917:2.0374:1.835245:2.164561:1.505351:2.011621:1.978743:2.57951:3.477066:2.126389:9.572408e-1 + 520200:522000:524400:526800:528600:531600:534000:537000:540000:543000:546000:548400:550800:553200:555600:557400:559800:561600:564000:565800:568200:570000:572400:575400:577800:580800:583800:586200:589200:591600:594000:595800:598200:599400:516000:517800 + + 3.57142857142857e-2,3.57142857142857e-2,1.12331236187416,1.85667695795805,41.0775182375181,68.7632521427758,115.133081843007 + + + + + 1 + 556.811:2.855049;556.811:52.26762; + + + + + + + + 1.2e-2 + 599400 + 599400 + + + 0 + 1 + + 2074200 + + 0.012000 + 0 + + 0:0,0,0,0,0,0,0,0,0,0,0 + + 0:0,0:0,0 + + + + + 1 + 2 + + 1209600 + 0 + 3024000 + + 3600 + 1209600 + + 336 + 5.938603:14.41493 + 5.938603:16.97233 + 5.938603:19.00052 + 5.938603:19.34693 + 5.938603:21.23162 + 5.938603:21.57721 + 5.938603:22.37519 + 5.938603:20.67838 + 5.938603:20.58496 + 5.938603:18.80667 + 5.938603:15.61338 + 5.938603:13.19109 + 5.938603:10.64538 + 5.938603:7.752373 + 5.938603:7.290094 + 5.938603:4.427566 + 5.938603:3.507984 + 5.938603:2.2383 + 5.938603:2.293033 + 5.938603:3.217484 + 5.938603:4.724331 + 5.938603:4.95948 + 5.938603:8.527335 + 5.938603:10.48936 + 5.948792:12.55229 + 5.948792:16.07236 + 5.948792:18.12323 + 5.948792:20.34106 + 5.948792:20.53125 + 5.948792:21.8414 + 5.948792:21.7989 + 5.948792:20.97049 + 5.948792:19.70294 + 5.948792:18.81372 + 5.948792:15.51204 + 5.948792:13.06896 + 5.948792:11.63529 + 5.948792:8.085461 + 5.948792:6.858862 + 5.948792:4.28266 + 5.948792:3.433843 + 5.948792:2.456749 + 5.948792:1.994734 + 5.948792:3.304781 + 5.948792:3.123468 + 5.948792:5.426206 + 5.948792:8.69908 + 5.948792:10.02537 + 5.958999:13.21521 + 5.958999:15.49807 + 5.958999:17.81228 + 5.958999:20.09425 + 5.958999:21.97865 + 5.958999:21.91336 + 5.958999:22.76776 + 5.958999:20.26478 + 5.958999:20.1477 + 5.958999:18.18572 + 5.958999:16.07406 + 5.958999:13.45976 + 5.958999:11.332 + 5.958999:7.435594 + 5.958999:6.363955 + 5.958999:4.904237 + 5.958999:3.197159 + 5.958999:1.629321 + 5.958999:1.208565 + 5.958999:2.131581 + 5.958999:3.385123 + 5.958999:6.945215 + 5.958999:8.019678 + 5.958999:9.995172 + 5.969223:13.78662 + 5.969223:15.06681 + 5.969223:17.28115 + 5.969223:19.23172 + 5.969223:21.26333 + 5.969223:21.66243 + 5.969223:21.75682 + 5.969223:21.50475 + 5.969223:19.97231 + 5.969223:18.52508 + 5.969223:15.65271 + 5.969223:12.87568 + 5.969223:9.706975 + 5.969223:9.371907 + 5.969223:6.37455 + 5.969223:3.76117 + 5.969223:1.868181 + 5.969223:2.665698 + 5.969223:3.110426 + 5.969223:1.741419 + 5.969223:4.314768 + 5.969223:5.345867 + 5.969223:7.46347 + 5.969223:10.87163 + 5.979464:11.59576 + 5.979464:14.79271 + 5.979464:17.80008 + 5.979464:19.40249 + 5.979464:19.89157 + 5.979464:22.66646 + 5.979464:21.37339 + 5.979464:20.95234 + 5.979464:20.09312 + 5.979464:18.01308 + 5.979464:16.12319 + 5.979464:13.46643 + 5.979464:11.80921 + 5.979464:8.246732 + 5.979464:6.274331 + 5.979464:3.911715 + 5.979464:2.122329 + 5.979464:2.265075 + 5.979464:3.337031 + 5.979464:2.902636 + 5.979464:4.731161 + 5.979464:5.59203 + 5.979464:7.96878 + 5.979464:11.11889 + 5.989723:12.76447 + 5.989723:16.32565 + 5.989723:18.61706 + 5.989723:20.04677 + 5.989723:21.41486 + 5.989723:22.06694 + 5.989723:21.699 + 5.989723:21.01515 + 5.989723:20.59638 + 5.989723:18.94432 + 5.989723:15.82601 + 5.989723:13.12309 + 5.989723:12.24925 + 5.989723:9.199264 + 5.989723:4.870469 + 5.989723:4.71382 + 5.989723:2.346282 + 5.989723:2.589999 + 5.989723:3.028055 + 5.989723:3.831053 + 5.989723:4.721169 + 5.989723:5.057173 + 5.989723:8.277921 + 5.989723:10.35941 + 6:14.14883 + 6:16.53365 + 6:17.91117 + 6:20.25526 + 6:20.98663 + 6:21.25829 + 6:20.93706 + 6:22.07995 + 6:20.48187 + 6:18.08239 + 6:15.81079 + 6:13.04558 + 6:10.89183 + 6:9.040415 + 6:5.772973 + 6:4.104272 + 6:2.679933 + 6:2.562256 + 6:2.043345 + 6:2.604173 + 6:3.779477 + 6:6.336036 + 4:7.336605 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + + + + 4;0;86400;0;604800 + + + + + 2 + 1 + + + 0 + 2073600 + + 168 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + + + + 0 + 7516800 + + 168 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + + + + + + 2 + 1 + + 2 + + 518400 + + 0 + 8.860623e-3;9.880717e-1:1.181899e-1;48.95882:2.814025e-1;46.13633:4.154721e-1;36.43198:5.312592e-1;45.13689:6.9221e-1;45.3367:8.653983e-1;44.19469:1.090963;57.94675:1.28321;30.39676:1.460073;35.38427:1.688391;35.39662:1.890194;32.39662:2.1148;32.39634:2.329221;19.5977:2.527407;14.72792:2.801277;11.79892:3.289783;13.68049:4.003262;5.905141:4.51427;9.646403e-1 + 557.7757 + + 44:1048720:2097296:1048712 + 345600;4.137167:432000;7.873276:518400;4.134641 + + + + + 3 + 1 + + + + + 36 + 0 + + 1 + 0:1026.771:3136.748:5280.305:7413.406:9490.247:11842.73:14380.92:16990.73:19827.84:22689.02:25698.17:28600.53:31296.41:33602.9:35673.21:37861.02:39991.96:42061.83:44116.5:46320.58:48543.78:50740.38:52973.15:55288.32:57588.95:59910.03:62793.53:65865.8:69154.45:71956.25:74406.93:76741.34:78910.1:81026.28:83170.02:85200.01:86400 + 12.3865:12.3865:14.05123:15.4598:17.16657:18.97497:19.17555:20.54934:21.35213:22.00459:21.75598:21.18682:20.75575:19.97149:18.54772:17.37197:15.40593:14.12064:12.86416:11.4995:10.36259:8.892316:6.319352:5.361618:4.657799:3.209356:2.631897:2.328421:2.798829:3.032838:3.228298:5.110494:5.281898:6.460663:8.374456:9.786043:10.96904:10.96904 + 3.94851:3.94851:3.581489:2.349854:2.691002:2.329499:1.772607:1.695194:1.923861:1.800037:1.213721:6.521156e-1:1.816027:2.670886:1.245022:1.554308:1.69054:1.56805:2.172423:2.180933:3.356893:1.507976:3.750927:2.729686:1.895456:1.550026:1.796994:1.191052:2.048688:1.574212:1.560969:2.601721:3.382655:1.875951:2.586627:2.455737:1.26448:1.26448 + + + 9094411231511854520:13258687326618484610 + + + 1.2e-2 + 0:2095.596:4173.253:6273.787:8447.421:10719.58:13126.22:15717.48:18422.44:21193.5:24154.04:27204.11:30011.04:32462.41:34702.12:36847.77:38941.84:41017.31:43121.97:45256.25:47400.61:49562.94:51754.13:53988.98:56286.85:58674.61:61213.43:64180.64:67683.1:70675.02:73235.12:75612.7:77881.99:80080.16:82210.91:84301.63:86400 + 992.9803:3107.293:5232.395:7435.856:9512.302:11808.35:14386.06:17014.77:19820.71:22716.17:25722.66:28651.49:31344.92:33602.31:35680.23:37847.94:39967.25:42048.61:44112.16:46367.57:48555.5:50729.99:52937.98:55191.68:57518.73:59934.76:62774.3:65824.66:69116.61:71968.27:74478.53:76753.17:78907.5:80976.78:83170.02:85200.01 + 3.85995:-10080.7 + 3.85995:19164.09 + + + 0:0:604800:86400:0 + + 0 + + 15.00671:1,5.559707e-1,4.013165e-1,12.58489,6.928656 + + + 13.92778:1,5.297467e-1,3.697866e-1,14.52525,7.811477 + + + 15.05345:1,5.591444e-1,4.041846e-1,15.6218,8.759267 + + + 15.42892:1,5.604574e-1,4.06841e-1,17.25176,9.685762 + + + 14.92886:1,5.256309e-1,3.677243e-1,18.91465,10.02024 + + + 16.60779:1,5.614215e-1,4.041401e-1,19.26327,10.81712 + + + 18.55682:1,5.91459e-1,4.334854e-1,20.64576,12.24807 + + + 18.17884:1,5.596057e-1,4.006923e-1,21.3509,11.97501 + + + 19.53205:1,5.959194e-1,4.370151e-1,21.84573,13.00555 + + + 20.44081:1,5.841212e-1,4.293371e-1,21.49539,12.44261 + + + 20.78524:1,5.946287e-1,4.388656e-1,21.41198,12.7901 + + + 19.57053:1,6.110217e-1,4.573598e-1,20.82745,12.78048 + + + 16.78801:1,5.941623e-1,4.419358e-1,20.00934,11.95881 + + + 14.73751:1,5.571243e-1,4.052331e-1,18.33202,10.11347 + + + 15.26717:1,6.184584e-1,4.692113e-1,17.46185,10.77198 + + + 13.99466:1,5.866008e-1,4.331856e-1,15.4207,9.050819 + + + 14.88725:1,6.168723e-1,4.729614e-1,14.18674,8.761604 + + + 14.03041:1,5.825818e-1,4.345371e-1,12.65371,7.272476 + + + 15.19267:1,6.20778e-1,4.777214e-1,11.38562,7.099001 + + + 15.27348:1,6.339943e-1,4.895391e-1,10.25702,6.585161 + + + 14.39208:1,5.989185e-1,4.481987e-1,9.035614,5.560112 + + + 15.54823:1,6.376876e-1,4.94352e-1,6.274665,3.810715 + + + 14.77036:1,6.177037e-1,4.690388e-1,5.410281,3.234654 + + + 16.07251:1,6.290236e-1,4.88434e-1,4.643083,2.904634 + + + 16.46016:1,6.261008e-1,4.824392e-1,3.111292,1.844833 + + + 18.19218:1,6.639981e-1,5.281008e-1,2.697856,1.781827 + + + 19.45487:1,6.400123e-1,4.932594e-1,2.40683,1.533744 + + + 24.178:1,6.644016e-1,5.261485e-1,2.61592,1.763416 + + + 20.41298:1,6.667499e-1,5.315301e-1,2.84267,1.923975 + + + 18.20536:1,6.91691e-1,5.613155e-1,3.233781,2.25479 + + + 16.37297:1,6.503502e-1,5.160963e-1,4.898393,3.250814 + + + 14.87548:1,6.502207e-1,5.145497e-1,5.599165,3.723383 + + + 15.52294:1,6.854298e-1,5.574278e-1,6.56157,4.49966 + + + 13.18229:1,6.157472e-1,4.707615e-1,8.192416,4.961989 + + + 10.97948:1,5.923025e-1,4.258453e-1,9.786043,5.84877 + + + 11.01764:1,5.410964e-1,3.771789e-1,10.96904,6.05859 + + 3.145671:3.911074:2.995116:2.587317:1.914562:2.231871:1.62894:1.984055:1.573592:1.613945:1.507788:1.781308:2.404483:1.205845:1.307109:1.557873:1.186662:2.191862:2.018205:3.190985:1.45333:2.884701:2.234948:1.600031:1.343595:1.84876:1.150508:1.900306:1.821944:1.724174:2.234307:3.203002:1.598277:2.607306:2.455737:1.26448 + 520200:522000:524400:526800:528600:531000:534000:536400:539400:542400:545400:548400:550800:552600:555000:556800:559200:561000:563400:565800:567600:570000:571800:574200:576600:579600:582000:585600:588600:591600:594000:595800:598200:599400:516000:517800 + + 6.05158730158729e-2,6.05158730158729e-2,2.59703579199887e-1,3.97445300252325e-1,59.550574800093,98.8264960357523,164.046998775628 + + + + + 1 + 556.811:2.132757;556.811:51.25344; + + + + + + + + 5e-4 + 567.284 + -3.72149702899643e-3,3.20387557922956e-2,1.75299380557383e-3 + 567.283992117364,567.283992117364,567.283992117364 + 567.283992117364 + 2403.24805683075,1739.47821359901,1659.88366674936,265.438231425312,68.2595403358334,1215.83084942762 + + + + 0:0 + + 0 + + + 2.5e-5 + 0 + 0,0 + 0,0 + 0 + 0,0,0 + + + 2.5e-5 + 0 + 0,0 + 0,0 + 0 + 0,0,0 + + + + diff --git a/lib/maths/unittest/testfiles/CSeasonalComponentAdaptiveBucketing.6.2.state.xml b/lib/maths/unittest/testfiles/CSeasonalComponentAdaptiveBucketing.6.2.state.xml new file mode 100644 index 0000000000..dace016b16 --- /dev/null +++ b/lib/maths/unittest/testfiles/CSeasonalComponentAdaptiveBucketing.6.2.state.xml @@ -0,0 +1,49 @@ + + + 1e-1 + 0:6664.981:13553.2:21815.09:32112.34:43036.12:52791.59:61537.79:70254.37:78862.69:86400 + 3332.747:10234.17:17653.79:26926.7:37561.86:47895.59:57167.65:65933.6:74634.44:82304.52 + 10:-3268.727 + 10:3268.727 + + + 0:0:86400:86400:0 + + 0 + + 69.70497:1,6.992375e-1,6.46919e-1,42.71123,30.08379 + + + 72.01623:1,7.000664e-1,6.516068e-1,29.34301,20.64653 + + + 85.79748:1,7.20356e-1,6.824e-1,17.81582,12.74652 + + + 106.7681:1,7.422521e-1,7.082695e-1,7.513468,5.386452 + + + 113.7603:1,7.378312e-1,7.025079e-1,1.219779,8.165162e-1 + + + 101.6499:1,7.736393e-1,7.604023e-1,8.841862e-1,6.667031e-1 + + + 91.20244:1,7.814465e-1,7.697524e-1,5.461958,4.309042 + + + 90.80847:1,7.938154e-1,7.894413e-1,14.07379,11.25277 + + + 89.57156:1,8.111495e-1,8.180668e-1,26.6939,21.77247 + + + 78.63993:1,7.909374e-1,7.904267e-1,41.09013,32.71338 + + 3.210121e-28:13.99402:15.72525:8.157597:1.004211:6.717919e-1:2.123492:4.330707:6.95912:3.767426e-28 + 783648:790560:799200:809568:819936:830304:838944:847584:856224:863136 + + 2.96999999999999,2.96999999999999,-4.88019313958899e-2,-6.46984908135304e-2,2.2851553986519,2.93342010023463,3.7954843138499 + + + diff --git a/lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.seasonal.expected_scales.txt b/lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.seasonal.expected_scales.txt new file mode 100644 index 0000000000..6494849f0b --- /dev/null +++ b/lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.seasonal.expected_scales.txt @@ -0,0 +1 @@ +3.27334,3.36121;3.88769,3.99136;4.56367,4.68503;5.12145,5.25738;5.31639,5.45742;5.51707,5.66336;5.70673,5.86037;5.90277,6.06167;6.1045,6.26882;5.78345,5.93916;5.15154,5.29031;4.51624,4.64425;3.8877,3.99798;3.25323,3.34559;3.22365,3.31524;3.20274,3.29393;3.1738,3.26437;3.1439,3.23371;3.30582,3.40026;3.51406,3.61441;3.74798,3.85021;3.97513,4.08369;4.13089,4.24371;3.95285,4.06086;3.76439,3.86713;3.58357,3.67786;3.416,3.50593;3.22995,3.31494;2.88227,2.95813;2.38442,2.44727;1.87235,1.92168;1.3796,1.41637;0.893226,0.917277;0.410988,0.422362;0.364726,0.374895;0.319765,0.32881;0.266963,0.274591;0.225958,0.232513;0.198156,0.204078;0.191618,0.197531;0.206956,0.213951;0.31002,0.320715;0.44897,0.464483;0.579446,0.599816;0.477252,0.49459;0.691818,0.717757;0.836642,0.869273;1.03932,1.08092;1.12783,1.17298;0.938837,0.976689;0.851452,0.885758;1.03735,1.07865;0.697232,0.725144;0.702735,0.731129;0.510846,0.532039;0.372699,0.388137;0.282486,0.293762;0.162633,0.16867;0.103756,0.107579;0.0795184,0.082436;0.0732333,0.075876;0.0834188,0.086133;0.106703,0.110114;0.137834,0.142171;0.176818,0.182748;0.317845,0.328965;0.468466,0.484894;0.482449,0.499545;0.462905,0.479486;0.505068,0.523632;0.724416,0.752069;1.04388,1.08511;1.06946,1.11211;1.02006,1.06039;1.00001,1.03954;0.771944,0.801908;0.5999,0.622582;0.436486,0.452627;0.32372,0.335324;0.234209,0.242517;0.173445,0.179432;0.118756,0.122887;0.0783572,0.081203;0.0466936,0.0484632;0.0425969,0.0440641;0.044494,0.0459348;0.0725032,0.0748481;0.121043,0.124833;0.191382,0.197735;0.29771,0.308023;0.433068,0.448103;0.553218,0.572747;0.441847,0.457877;0.648012,0.67246;0.78619,0.816797;0.979946,1.01914;1.07851,1.12172;0.901069,0.937198;0.804508,0.836746;0.971971,1.01005;0.60293,0.626192;0.520102,0.53946;0.286743,0.297072;0.163963,0.169836;0.142062,0.146889;0.101668,0.105123;0.0727427,0.0751819;0.0480615,0.0497258;0.0476225,0.0492989;0.0681692,0.0703814;0.101891,0.105177;0.141084,0.145585;0.184946,0.191215;0.330815,0.342493;0.481433,0.498416;0.486577,0.503916;0.458465,0.475123;0.50755,0.526489;0.747037,0.776018;1.0845,1.12782;1.08289,1.1263;1.00273,1.04258;0.961727,0.999909;0.719291,0.74692;0.532884,0.55261;0.358624,0.371288;0.251731,0.260083;0.18062,0.186626;0.135517,0.139925;0.0947223,0.0977856;0.0683349,0.0705799;0.057098,0.0589894;0.0680457,0.0702732;0.0778986,0.0804262;0.0871478,0.0900547;0.16385,0.16929;0.279041,0.288265;0.376912,0.389203;0.459037,0.474172;0.518589,0.53534;0.579011,0.597407;0.659284,0.679861;0.719817,0.741773;0.784071,0.807507;0.849744,0.874546;0.861669,0.88599;0.82175,0.844835;0.770936,0.792491;0.725914,0.746927;0.68618,0.70604;0.636966,0.655395;0.555503,0.57163;0.478379,0.492332;0.403713,0.415677;0.332199,0.34214;0.360346,0.371091;0.516947,0.532134;0.66818,0.686896;0.82944,0.852564;0.998753,1.02665;1.19128,1.22449;1.39285,1.43159;1.5913,1.63349;1.78035,1.82738;1.99137,2.04405;2.1239,2.18021;2.01021,2.06367;1.88681,1.93706;1.75411,1.79924;1.62531,1.66702;1.51235,1.55134;1.37919,1.41474;1.23853,1.27047;1.10041,1.1288;0.948997,0.972095;0.816097,0.835952;0.689228,0.706127;0.562189,0.576079;0.526515,0.539596;0.515914,0.528842;0.49861,0.511239;0.480593,0.491798;0.460541,0.471386;0.481274,0.493097;0.507966,0.521243;0.527712,0.542041;0.531605,0.546735;0.512911,0.527634;0.495088,0.509431;0.47786,0.490832;0.439352,0.451184;0.404617,0.415353;0.37125,0.380832;0.284056,0.290634;0.258059,0.263924;0.22582,0.230805;0.207525,0.212066;0.201803,0.206168;0.18689,0.190814;0.176297,0.179934;0.170404,0.1738;0.16697,0.170301;0.16673,0.169877;0.167771,0.17099;0.170071,0.173414;0.166405,0.169725;0.173225,0.176815;0.188099,0.19232;0.195005,0.199562;0.20089,0.205723;0.214623,0.219752;0.22119,0.226374;0.24974,0.255788;0.276114,0.282967;0.292793,0.30004;0.299768,0.307058;0.296324,0.303206;0.297271,0.304881;0.314727,0.322807;0.313788,0.321666;0.310918,0.318548;0.314733,0.322268;0.392377,0.401099;0.489638,0.500123;0.59288,0.605478;0.695943,0.710599;0.799974,0.820042;0.908668,0.931514;1.01066,1.03614;1.12497,1.15327;1.26583,1.29768;1.44738,1.48423;1.63453,1.67735;1.81514,1.86299;1.98003,2.03266;2.05531,2.10983;1.89205,1.94246;1.72871,1.77501;1.54572,1.58594;1.36554,1.401;1.18673,1.21738;0.958124,0.982214;0.873483,0.895315;0.782602,0.802008;0.698047,0.715333;0.619503,0.634461;0.531463,0.544235;0.447743,0.458484;0.483681,0.495249;0.612369,0.626981;0.744112,0.761859;0.878027,0.897937;1.01256,1.03555;1.14113,1.16705;1.28019,1.30934;1.3783,1.40993;1.43956,1.47273;1.4998,1.53445;1.55642,1.59103;1.60435,1.63997;1.67425,1.71169;1.74167,1.78086;1.79317,1.83357;1.67921,1.71711;1.55484,1.58979;1.43548,1.4677;1.33148,1.36235;1.20972,1.23772;1.08602,1.11112;0.964861,0.98712;0.85267,0.872094;0.842853,0.861914;0.839065,0.858053;0.835098,0.853958;0.83401,0.853486;0.835999,0.855623;0.831286,0.850911;0.825193,0.844666;0.820302,0.839732;0.856111,0.876933;0.897965,0.920556;0.93255,0.956951;0.951734,0.977255;0.948303,0.97377;0.925724,0.950657;0.870755,0.894389;0.795238,0.816751;0.723496,0.742931;0.652922,0.670487;0.528819,0.542373;0.466026,0.477864;0.468857,0.480542;0.495583,0.507845;0.528051,0.54104;0.551658,0.564826;0.579416,0.593189;0.6118,0.62631;0.64672,0.662052;0.709308,0.726131;0.774455,0.792838;0.840862,0.860858;0.901705,0.922622;0.97266,0.995298;1.05167,1.07639;1.11243,1.1387;1.02462,1.04901;0.932212,0.954541;0.83201,0.85208;0.754538,0.772138;0.673769,0.689947;0.583232,0.597576;0.48299,0.49515;0.416258,0.426719;0.386195,0.39585;0.372045,0.381508;0.339701,0.348306;0.305756,0.313272;0.274199,0.280906;0.228427,0.233815;0.202224,0.20693;0.198975,0.203633;0.211529,0.216414;0.227401,0.232658;0.245909,0.251672;0.257716,0.263836;0.26759,0.274607;0.276651,0.283961;0.360655,0.370502;0.538485,0.55332;0.709373,0.728796;0.864532,0.888295;0.995537,1.0245;1.13082,1.16338;1.28036,1.31689;1.94709,2.00091;2.6176,2.68875 diff --git a/lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.seasonal.expected_values.txt b/lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.seasonal.expected_values.txt new file mode 100644 index 0000000000..757a298784 --- /dev/null +++ b/lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.seasonal.expected_values.txt @@ -0,0 +1 @@ +6507.55,6532.17;7321.96,7346.79;8079.19,8105.29;8856.21,8883.06;9615.09,9642.13;10340.7,10368;11056.8,11084.4;11731.4,11759.2;12340.9,12369;12884.1,12911.6;13351,13377.4;13723.8,13750.1;13981.6,14006.6;14111.1,14134.3;14145.6,14169.2;14028.7,14052.9;13681.3,13705.8;13252.6,13277.3;12704.4,12729.7;12018.7,12044.6;11317,11343.1;10431.8,10458.9;9530.11,9557.77;8597.19,8624.36;7609.79,7636.16;6732.48,6757.49;5837.76,5862.28;4981.22,5005.02;4089.3,4111.87;3191.57,3212.39;2473.05,2491.45;1868.71,1884.99;1295.3,1309.29;740.831,751.203;454.915,464.645;392.97,402.165;364.996,373.352;330.444,338.439;478.766,486.744;793.755,801.938;1272.64,1281.57;1985.65,1996.04;2899.83,2912.08;3982.02,3996.1;4996.75,5010.6;5907.99,5924.51;7160.75,7179.46;8495.44,8516.34;9833.92,9855.31;11216,11236;12575,12594.4;13942.5,13963.9;15302.9,15321.9;16459.1,16478.5;17555.1,17571.8;18423.7,18437.8;19146.3,19158.5;19942.1,19951.1;20540.4,20547.7;20853,20859.3;21006.6,21012.5;21027.7,21033.6;20842.9,20849.3;20476.2,20483.1;19947.5,19955.4;19235.3,19245.5;18254.8,18266.6;17058.1,17070.3;15975.4,15987.6;15038.5,15051.5;13851.8,13867.3;12437.1,12455.6;10900.7,10919.8;9433.89,9452.6;8141.35,8160.1;6819.65,6836.84;5372.96,5388.38;4196.21,4209.71;3158.72,3170.33;2265.43,2275.3;1508.41,1516.83;853.757,860.879;367.533,373.516;105.804,110.353;78.963,83.2731;98.3158,102.62;349.794,355.091;812.312,818.611;1451.01,1458.9;2242.94,2252.67;3227.87,3239.29;4401.39,4414.21;5559.55,5571.63;6635.96,6650.77;8019.31,8035.82;9406.75,9425.24;10768.2,10787.6;12193.5,12211.5;13606.2,13623.4;14986.5,15004.9;16307,16323;17401.4,17416.3;18409.9,18421;19198.1,19206.6;19865.9,19873.5;20538.6,20545.2;20934.9,20940.5;21046.9,21051.7;21026.7,21031.4;20875.8,20881.2;20518.7,20525;19981.7,19988.8;19297.5,19305.9;18462.9,18473.8;17411,17423.5;16190.4,16203;15090.4,15103;14100.5,14114.2;12830.4,12847.4;11368.2,11388.8;9873.83,9894;8501,8519.75;7307.23,7325.21;6085.57,6100.86;4755.57,4768.48;3703.09,3713.67;2791.21,2799.84;2020.53,2028.08;1373.28,1380.04;812.681,818.65;408.464,413.793;226.5,231.205;245.828,250.977;298.942,304.505;516.509,522.649;772.717,781.037;1203.19,1214.09;1777.01,1789.42;2471.73,2485.54;3244.74,3259.16;4061.53,4076.53;4962.25,4977.94;5934.79,5950.74;6960.06,6976.32;8021.45,8037.73;9005.33,9020.43;9904.22,9918.8;10786.3,10800.2;11605.1,11618.8;12337.1,12350.4;12997.1,13009.7;13546,13557.7;13960.4,13971.4;14242.8,14253.2;14395.9,14405.8;14421,14431.3;14316.2,14328.1;14091.8,14104.6;13801.3,13815.3;13394,13409.6;12795.3,12812.1;12164,12182;11472.9,11491.2;10697.7,10716.7;9946.01,9966.23;9037.72,9058.75;8126.4,8147.14;7191.84,7212.07;6217.63,6236.61;5375.57,5393.69;4546.72,4564.52;3794.45,3811.49;3046.76,3062.95;2316.93,2332.25;1766.15,1779.58;1304.36,1316.83;823.857,835.666;356.647,367.629;74.9961,85.8321;-37.3839,-26.4024;-294.852,-283.814;-314.692,-304.395;-163.711,-153.444;16.8906,27.9591;345.859,357.947;782.428,795.081;1280.64,1293.82;1786.09,1799.13;2253.54,2266.43;2717.11,2729.13;3176.95,3188.47;3632.97,3643.95;4087,4097.3;4444.03,4452.49;4714.38,4722.3;4982.71,4989.91;5218.03,5224.88;5410.93,5417.59;5589.35,5595.49;5725.14,5730.91;5799.35,5804.7;5816.55,5821.86;5782.05,5787.25;5699.66,5705.01;5568.55,5574.13;5396.94,5402.57;5237.38,5243.34;5037.92,5044.53;4722.08,4728.99;4441.39,4448.51;4158.7,4165.98;3842.71,3850.01;3594.21,3602.07;3226.21,3234.56;2883.43,2891.98;2533.34,2541.9;2146.13,2154.39;1880.36,1889.24;1604.62,1613.75;1368.69,1377.64;1093.08,1101.81;794.823,803.38;637.252,645.769;525.696,534.336;344.581,353.811;118.582,128.252;12.1893,23.8417;4.12566,16.5756;-226.264,-213.135;-302.031,-288.405;-287.17,-272.832;-308.93,-292.884;-233.134,-215.202;-85.1387,-65.9781;103.151,123.488;287.323,307.909;431.045,451.044;575.739,595.122;730.456,748.459;902.355,919.273;1101.43,1117.1;1240.66,1253.88;1335.78,1348.22;1468.94,1480.42;1604.8,1615.6;1730.77,1740.8;1872.46,1881.56;1999.75,2008.02;2091.12,2099.47;2149.94,2159.09;2180.59,2190.6;2186.3,2196.66;2164.64,2175.83;2123,2134.83;2112.94,2125.64;2081.49,2095.19;1950.4,1964.62;1868.88,1883.52;1797.38,1811.9;1699.59,1714.22;1671.37,1686.67;1520.65,1536.52;1387.03,1403.17;1233.89,1249.62;1029.66,1044.66;932.391,946.768;808.841,823.244;708.529,722.243;550.921,563.889;352.163,364.369;277.274,288.308;237.838,248.517;121.256,131.944;-46.0516,-35.4525;-97.1937,-86.2467;-52.718,-41.5267;-230.933,-219.584;-253.157,-241.869;-185.519,-174.154;-154.808,-142.334;-23.7795,-10.1754;184.905,199.379;442.595,457.718;707.574,722.806;946.592,961.702;1203.03,1217.85;1484.39,1498.53;1796.71,1810.1;2148.21,2160.87;2450.53,2461.12;2720.4,2730.22;3047.31,3056.7;3395.56,3404.93;3747.52,3756.96;4123.97,4133.02;4489.48,4498.42;4817.88,4826.86;5106.77,5115.96;5354.04,5363.71;5556.76,5566.96;5707.05,5717.82;5806.23,5817.18;5899.69,5911.29;5928.59,5941.16;5809.52,5822.65;5689.48,5702.44;5532.98,5545.62;5310.09,5322.4;5122.93,5134.8;4784.91,4796.69;4441.93,4453.3;4062.41,4073.12;3617.4,3627.38;3267.47,3277.06;2882.67,2892.26;2515.31,2524.48;2088.25,2096.82;1620.83,1628.96;1281.55,1288.75;988.379,995.069;640.661,647.366;276.338,283.114;72.5939,79.6219;18.658,26.085;-193.621,-185.886;-175.882,-167.362;12.6202,21.367;236.108,246.418;621.996,634.516;1142.6,1156.71;1763.95,1779.52;2438.97,2456.11;3129.9,3147.89;3876.46,3895.33;4681.89,4703.4;5540.36,5563.93 diff --git a/lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.seasonal.state.xml b/lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.seasonal.state.xml new file mode 100644 index 0000000000..009eaacad1 --- /dev/null +++ b/lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.seasonal.state.xml @@ -0,0 +1,1198 @@ + + 1e-2 + 60480000 + 60480000 + + + 0 + 1 + + 60566400 + + 0.010000 + 60480000 + + 4390.264:1,-11.00117,198.9572,-4472.915,113137.5,-3075130,8.766135e7,4.567685,-31.80872,374.363,-7075.814 + + 4390.37:4.56764769438948,-1.37555043547669:27847.8764486777,27770.9172426487 + + + + + 1 + 3 + + 0 + 0 + 0 + + 0;64800;86400;0;172800:1;64800;604800;0;172800:2;64800;86400;172800;604800:3;64800;604800;172800;604800 + + + + + 2 + 1 + + + 0 + 60480000 + + 168 + 1:-498.1854 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + + + + 3 + 44755200 + + 168 + 96:3.720367 + 96:-35.88307 + 96:10.41097 + 96:19.10994 + 96:10.87921 + 96:-16.93822 + 96:-4.274967 + 96:10.48392 + 96:-38.03411 + 96:9.836008 + 96:17.90978 + 96:19.95323 + 96:-13.26831 + 96:-6.089562 + 96:6.487684 + 96:-47.09082 + 96:14.85965 + 96:24.44639 + 96:9.412406 + 96:-14.38557 + 96:-5.78003 + 96:18.99668 + 96:-46.71292 + 96:17.67111 + 96:27.60615 + 96:9.734477 + 96:-11.87728 + 96:-10.64841 + 96:17.28211 + 96:-43.7168 + 96:20.87388 + 96:33.4024 + 96:4.182891 + 96:-11.14236 + 96:1.640738 + 96:13.07003 + 96:-40.35469 + 96:18.71054 + 96:34.72885 + 96:10.73478 + 96:-10.93142 + 96:-1.535446 + 96:17.82857 + 96:-38.36657 + 96:17.86586 + 96:37.74407 + 96:4.146163 + 96:-11.71128 + 96:7.216434 + 96:19.97486 + 96:-41.78338 + 96:16.69088 + 96:39.67858 + 96:5.378158 + 96:-13.65542 + 96:5.050858 + 96:26.4882 + 96:-39.09408 + 96:22.88231 + 96:40.14479 + 96:-1.715438 + 96:-11.93286 + 96:8.266279 + 96:28.03196 + 96:-41.64379 + 96:23.27399 + 96:46.34034 + 96:-1.073905 + 96:-12.82995 + 96:16.12961 + 96:30.96347 + 96:-46.07127 + 96:22.71378 + 96:39.71739 + 96:-5.418464 + 96:-15.23559 + 96:16.25636 + 96:35.78307 + 96:-43.60847 + 96:26.63418 + 96:46.31608 + 96:-9.194983 + 96:-17.50459 + 96:14.33755 + 96:29.73532 + 96:-50.05243 + 96:19.89827 + 96:42.85352 + 96:5.974119 + 96:-21.58294 + 96:13.65861 + 1:-498.1854 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + + + + + + 2 + 1 + + 11 + + 60480000 + + 0 + 6.085628e-2;6.065308e-1:6.226253;313.4706:19.50897;330.2422:34.76368;347.166:53.20958;380.118:73.32223;346.8714:95.37843;393.1543:119.9886;428.4194:146.6317;399.4634:176.4141;400.5096:206.0797;324.299:233.6587;235.8362:262.2072;168.5054:294.0104;116.8712:334.0826;107.8649:380.8156;68.16639:447.8285;54.78033:562.1304;25.91533:667.0167;7.046882e-1 + 4442.964 + + 48:48:48:48:2097200:48:48:48:48:48:48:3145776:48:48:48:48:48:48:3145776:48:48:48:48:48:48:2097200:48:48:48:48:48:48:3145776:48:48:48:48:48:48:3145776:48:48:48:48:48:48:3145776:48:48:48:48:48:48:3145776:48:48:48:48:48:48:4194352:48:48:48:48:48:48:2097200:48:48:48:1048624:48:48:5242928:48:48:48:48:48:48:4194352:48:48:48:48:48:48:7340080:48:48:48:48:48:48:3145776:48:48:48:48:48:48:8388656:48:48:48:48:48:48:4194352:48:48:48:48:48:48:8388656:48:48:48:48:48:48:1048577 + 50198400;950.2728:50803200;1382.749:51408000;1406.476:52012800;914.305:52617600;1397.647:53222400;1382.768:53827200;1412.052:54432000;1415.533:55036800;1872.63:55641600;950.4237:55987200;450.3738:56246400;2373.396:56851200;1886.845:57456000;3361.585:58060800;1466.612:58665600;3929.761:59270400;1996.988:59875200;4066.847:60480000;498.1854 + + + + + 3 + 1 + + + + + 36 + 0 + + 1 + 0:936.7401:4537.078:8009.072:10509.48:12531.18:14352.83:16230.06:18608.06:20667.64:22460.6:24501.47:26729.8:28705.32:30495.13:33193.79:35774.4:38612.75:42299.41:45076.11:47771.61:51093.21:53488.14:55705.33:57544.05:59426.02:61572.72:63736.95:65741:67727.52:69870.52:71938.14:73759.41:76361.7:78869.79:81870.21:84531.26:86400 + -1048.584:-1048.584:-546.3181:649.7096:1900.305:3146.843:4221.029:5248.91:7100.246:8788.73:10325.2:12057.09:13869.14:15420.23:16532.73:17789.97:18395.32:19192.09:19722.98:19689.07:19165.73:17994.08:16765.76:15289.07:14173.11:13182.39:11680.1:9834.165:8034.28:6458.573:4902.414:3307.696:2239.417:1026.009:142.4738:-705.1736:-1070.479:-1070.479 + 3791.207:3791.207:18151.44:56045.34:110392:152212:115540.6:177668:229715.2:304797.4:298818.2:191972.6:288191.4:149951.1:120452.4:35999.53:32134.84:16930.47:2942.023:10732.1:24364.65:45926.52:126226:130062:122014.3:133390:210602:332634.6:267965.5:301159.9:213211:151460.7:99367.52:54294.88:36528.77:11714.25:1748.898:1748.898 + + + 17879339885978285686:2140934814581142352 + + + 1e-2 + 0:3284.313:6361.221:8994.329:11380.08:13683.43:15859.91:17893.45:19798.3:21600.32:23400.32:25211.58:27109.45:29171.15:31531.64:34266.16:37258.67:40284.06:43345.86:46572.21:49628.09:52192.61:54497.11:56752.2:58950.09:61030.57:62965.83:64783.18:66583.18:68414.7:70367.02:72467.16:74781.48:77407.59:80283.45:83263.18:86400 + 936.7401:4537.078:8009.072:10509.48:12531.18:14352.83:16230.06:18608.06:20667.64:22460.6:24501.47:26729.8:28705.32:30495.13:33193.79:35774.4:38612.75:42299.41:45076.11:47771.61:51093.21:53488.14:55705.33:57544.05:59426.02:61572.72:63736.95:65741:67727.52:69870.52:71938.14:73759.41:76361.7:78869.79:81870.21:84531.26 + 51.59188:-143.8902 + 51.59188:404.1996 + + + 64800:0:172800:86400:55641600 + + 669600 + + 152.3999:1,-3.150924,123.4292,-864.5687,851.4329 + + + 142.7997:1,-2.543437,109.3789,-353.8012,-977.8658 + + + 122.2017:1,-2.012811,101.6948,820.0032,-3308.995 + + + 110.7167:1,-5.933211,222.4633,2083.194,-14814.98 + + + 106.8797:1,-7.472714,279.4158,3238.815,-25529.92 + + + 100.978:1,-6.591704,240.6421,4317.305,-29757.36 + + + 94.34453:1,-5.336124,185.7223,5438.621,-31253.57 + + + 88.37724:1,-9.7662e-1,81.34733,7236.268,-8281.114 + + + 83.63614:1,-4.556041,158.4885,9023.18,-43674.74 + + + 83.51932:1,-4.157113,152.0063,10533.62,-46092.06 + + + 84.05691:1,-1.259623,91.24393,12174.46,-16466.76 + + + 88.06377:1,-3.739115,143.1579,13989.53,-53628.29 + + + 95.66406:1,-6.296775,238.194,15526.96,-99247.1 + + + 109.5327:1,-8.272849,319.8353,16568.36,-137616.2 + + + 126.8994:1,-1.872784,104.1732,17733.48,-32638.17 + + + 138.8358:1,-10.58594,393.7397,18493.43,-197252.5 + + + 140.2804:1,-2.491524,114.3813,19228.61,-48282.76 + + + 141.9703:1,-2.51728,112.5556,19825.62,-50936.43 + + + 149.7243:1,-12.42791,502.947,19880.46,-250325.9 + + + 141.867:1,-2.724705,118.0331,19243.15,-53224.48 + + + 119.035:1,-3.016602,128.786,18105.68,-55820.84 + + + 106.9464:1,-3.887334,158.2923,16944.89,-68012.02 + + + 104.6397:1,-7.22431,271.8215,15653.69,-118315.6 + + + 102.0003:1,-6.741898,246.1488,14524,-102665.7 + + + 96.58987:1,-5.80534,201.907,13374.39,-79965.66 + + + 89.86085:1,-2.799666,118.2701,11764.37,-33789.91 + + + 84.35814:1,-1.882388,111.2553,9964.837,-20166.9 + + + 83.51932:1,-3.902875,143.3966,8292.22,-35115.73 + + + 84.93052:1,-2.28721,100.0165,6671.024,-17194.46 + + + 90.51298:1,-3.38207,146.7927,5037.778,-18631.68 + + + 97.37831:1,-6.259639,237.1922,3618.518,-26930.98 + + + 107.3546:1,-7.581152,284.8436,2623.733,-25455.55 + + + 121.8479:1,-2.390924,112.147,1244.836,-5190.762 + + + 133.4312:1,-7.64085,296.7179,391.5509,-6755.835 + + + 138.2444:1,-2.441981,115.6321,-533.16,-481.5182 + + + 145.5302:1,-12.39149,511.2014,-799.9346,5199.518 + + 3791.207:18151.44:56045.34:110392:152212:115540.6:177668:229715.2:304797.4:298818.2:191972.6:288191.4:149951.1:120452.4:35999.53:32134.84:16930.47:2942.023:10732.1:24364.65:45926.52:126226:130062:122014.3:133390:210602:332634.6:267965.5:301159.9:213211:151460.7:99367.52:54294.88:36528.77:11714.25:1748.898 + 60028200:60031800:60035400:60037200:60039000:60040800:60042600:60046200:60048000:60049800:60051600:60053400:60055200:60057000:60060600:60062400:60066000:60069600:60071400:60075000:60076800:60080400:60082200:60084000:60085800:60087600:60091200:60093000:60094800:60096600:60098400:60100200:60103800:60105600:60109200:60111000 + + 1751.71021501026,1751.71021501026,6.05901012517789e-1,4.19918383576832e-2,2307.53242774043,147.941986172909,33.3349073791161 + + + + + + 36 + 0 + + 1 + 0:918.2796:3624.152:5467.019:7786.826:9913.493:11785.26:13592.39:15422.07:17308.84:19333.79:21550.4:23372.68:25207.65:27856.15:30575.86:33328.2:36982.09:40506.39:43149.39:45822.27:48464.16:50996.48:53469.76:55646.46:57568.54:60020.87:62471.51:64623.02:66482.99:68705.35:71238.42:73808.95:76034.59:78476.05:81848.34:84523.67:86400 + 1572.479:1572.479:1837.247:2071.711:2577.376:3171.632:3750.246:4279.056:4792.645:5327.351:5882.755:6492.716:6888.54:7190.836:7600.896:7911.968:8157.092:8266.788:8146.133:7930.004:7634.51:7333.881:6865.32:6479.335:6082.769:5811.794:5303.779:4847.1:4363.366:4064.755:3695.733:3301.681:2791.101:2518.994:2238.285:1763.721:1733.516:1733.516 + 14434.46:14434.46:13382.36:25051.54:41951.85:53695.7:56645.14:52564.2:57567.5:50644.45:44664.21:39494.78:18561.33:16127.24:9929.318:8990.536:3884.94:2418.332:4203.632:6770.932:6716.912:14923.61:19229.85:22808.27:21035.38:27020.1:33652.3:34291.83:29435.12:26561.91:29307.65:23286.03:19005.34:10116.73:10641.49:11166.57:14336.25:14336.25 + + + 4305797209917071696:12377736236737676023 + + + 1e-2 + 0:2577.599:4868.495:7021.657:9000.608:10803.44:12603.44:14403.44:16203.44:18003.44:19868.24:21872.88:24060.71:26492.12:29198.77:32101.05:35151.79:38347.5:41572.86:44665.4:47403.18:49847.61:52196.48:54525.71:56820.13:59042.04:61196.4:63352.95:65593.91:67895.59:70194.27:72521.88:74890.5:77396.87:80302.52:83457.42:86400 + 918.2796:3624.152:5467.019:7786.826:9913.493:11785.26:13592.39:15422.07:17308.84:19333.79:21551.13:23372.68:25207.65:27856.15:30575.86:33328.2:36982.09:40506.39:43149.39:45822.27:48464.16:50996.48:53469.76:55646.46:57568.54:60020.87:62471.51:64623.02:66482.99:68705.35:71238.42:73808.95:76034.59:78476.05:81848.34:84523.67 + 51.59188:-136.1805 + 51.59188:310.9886 + + + 64800:172800:604800:86400:55641600 + + 669600 + + 119.6134:1,-9.028208e-2,62.45866,1543.426,84.88435 + + + 106.3321:1,-5.632437,184.6011,1753.529,-8938.227 + + + 99.92892:1,-4.766377,157.1039,2018.724,-9064.621 + + + 91.81389:1,-6.055017e-1,72.14771,2543.74,-1260.089 + + + 83.64412:1,-2.255949,100.6681,3130.172,-6675.741 + + + 83.52734:1,-2.961351,114.2853,3688.968,-10335.51 + + + 83.52734:1,-2.972203,114.8768,4213.795,-11894.82 + + + 83.52734:1,-2.636518,106.5136,4730.902,-11896.55 + + + 83.52729:1,-2.188819,102.4259,5259.21,-10860.33 + + + 86.5407:1,-8.618826e-1,76.75774,5820.262,-4482.265 + + + 94.46265:1,-4.09988,142.2059,6386.391,-25137.42 + + + 101.5773:1,-5.42858,176.5821,6768.127,-35425.96 + + + 112.8698:1,-6.505198,201.8808,7077.861,-44803.68 + + + 125.6156:1,-8.430725e-1,75.25011,7520.194,-5663.345 + + + 134.6355:1,-8.268665,258.3628,7809.67,-63384.57 + + + 141.4632:1,-1.524495,83.68519,8087.773,-11741.04 + + + 148.2231:1,-1.977831,92.59888,8206.104,-15694.19 + + + 149.7141:1,-2.408524,105.7029,8096.354,-19025.54 + + + 143.5463:1,-9.871436,315.5005,7869.085,-76938.84 + + + 126.9898:1,-9.97848e-1,81.66508,7628.301,-7556.688 + + + 113.3826:1,-6.142684,187.8629,7308.367,-44623.68 + + + 109.0005:1,-1.323961,69.82803,6881.609,-9228.842 + + + 108.1108:1,-3.116112,144.639,6478.74,-20181.31 + + + 106.4876:1,-5.643942,177.5344,6133.026,-35147.47 + + + 103.1113:1,-4.897641,145.449,5818.202,-28555.36 + + + 99.97396:1,-6.989211e-1,63.30838,5321.43,-3845.288 + + + 100.0921:1,-2.332477,127.702,4851.184,-11363.12 + + + 104.0486:1,-5.373729,171.5482,4440.385,-24676.56 + + + 106.8574:1,-5.446235,166.876,4103.454,-22740.08 + + + 106.6453:1,-3.225763,106.5837,3673.481,-11661.03 + + + 107.9678:1,-3.995176e-1,73.78207,3241.079,-771.0342 + + + 109.899:1,-5.955635,186.8205,2737.918,-15734.27 + + + 116.3165:1,-3.938793,137.0166,2454.95,-9024.551 + + + 134.851:1,-2.732002,110.7226,2162.874,-5192.071 + + + 146.3784:1,-1.886982,93.43559,1716.367,-2814.113 + + + 136.5099:1,-9.080403,310.0894,1549.179,-11630.41 + + 14434.46:13382.36:25051.54:41951.85:53695.7:56645.14:52564.2:57567.5:50644.45:44664.21:39932.3:18561.33:16127.24:9929.318:8990.536:3884.94:2418.332:4203.632:6770.932:6716.912:14923.61:19229.85:22808.27:21035.38:27020.1:33652.3:34291.83:29435.12:26561.91:29307.65:23286.03:19005.34:10116.73:10641.49:11166.57:14336.25 + 60460200:60462000:60463800:60467400:60469200:60471000:60472800:60474600:60476400:60478200:60480000:60395400:60397200:60400800:60402600:60406200:60409800:60413400:60415200:60418800:60420600:60422400:60426000:60427800:60429600:60431400:60435000:60436800:60438600:60440400:60444000:60445800:60447600:60451200:60454800:60456600 + + 718.855407649011,718.855407649011,5.98406734793441e-1,2.78691505432283e-2,172.218867090804,14.4969670308643,4.18223881330658 + + + + + + 36 + 0 + + 1 + 0:2953.472:9889.494:15509.4:20088.11:24335.38:27847.75:31146.23:34617.01:38429.43:42538.96:47732.76:53707.65:59864.99:65874.54:70666.55:74572.05:78801.3:83210.81:88043.51:93422.18:98949.88:104235.5:108899.5:113007.4:116593.6:120100:123680.5:128025.5:133605.9:139474.1:144721.7:149277.5:153548.3:158248.3:163511.3:169049.3:172800 + 910.4948:910.4948:692.4498:565.2307:255.1247:-134.4964:-314.5481:-233.1825:297.4768:897.5973:1081.637:1357.175:1677.788:1727.266:1971.499:2059.436:1806.471:1403.645:1136.66:1009.747:1046.411:1127.404:1351.255:1044.765:843.7838:730.9653:993.3024:1471.169:1295.063:1016.613:885.1838:830.6299:762.4136:1101.637:1340.052:1226.986:1094.804:1094.804 + 2354.379:2354.379:3735.039:17434.78:26658.26:19494.92:35045.32:95594.09:68023.88:13672.21:13256.16:7419.019:7118.017:12997.09:14644.62:22535.67:28897.56:15447.66:12707.28:3935.345:7012.773:9708.504:13196.37:14538.7:12317.24:36603.27:12570.42:6674.606:5651.271:8409.773:12290.93:9306.442:25755.76:7129.233:3009.838:1641.251:866.7905:866.7905 + + + 17879339885978285686:2140934814581142352 + + + 1e-2 + 0:6699.546:12714.45:17906:22273.68:26010.8:29347.88:32656.16:36261.93:40252.87:44885.58:50526.62:56801.18:63221.35:68466.86:72543.74:76511.55:80838.59:85428.69:90528.38:96448.01:102035.3:106672.9:111004.6:114980.9:118628:122089.3:125735.9:130340.8:136330.6:142307.2:147212.4:151606.2:155830:160319.5:165972.9:172800 + 2953.472:9889.494:15509.4:20088.11:24335.38:27847.75:31146.23:34617.01:38429.43:42538.96:47732.76:53707.65:59864.99:65874.54:70666.55:74572.05:78801.3:83210.81:88043.51:93422.18:98949.88:104235.5:108899.5:113007.4:116593.6:120100:123680.5:128025.5:133605.9:139474.1:144721.7:149277.5:153548.3:158248.3:163511.3:169049.3 + 44.46429:6300.229 + 44.46429:39594.34 + + + 64800:0:172800:604800:55641600 + + 669600 + + 117.0922:1,-13.94213,458.2404,622.4716,-5215.821 + + + 105.2541:1,-13.53592,476.0489,451.8746,-2847.874 + + + 90.69752:1,-16.92971,596.1518,337.3521,-2884.757 + + + 76.08128:1,-14.46858,499.6193,40.061,2194.756 + + + 65.10555:1,-15.52892,547.0093,-337.5491,7876.831 + + + 58.20322:1,-16.00008,582.9759,-470.1579,9638.508 + + + 57.77425:1,-15.53481,535.1026,-272.3527,4718.821 + + + 63.02028:1,-15.21861,533.2859,246.6688,-3095.429 + + + 69.89057:1,-14.5798,500.9417,738.9327,-8752.838 + + + 81.26492:1,-18.37007,646.7825,887.0159,-14017.69 + + + 98.99745:1,-13.03278,455.1181,1173.635,-12815.72 + + + 110.0534:1,-17.32183,613.0522,1365.849,-19816.56 + + + 112.3017:1,-14.13816,492.4769,1465.254,-17268.53 + + + 91.21946:1,-14.3308,498.063,1682.922,-20353.58 + + + 70.97612:1,-16.28482,570.8325,1682.447,-22676.46 + + + 69.39727:1,-16.66526,589.1434,1420.292,-18818.05 + + + 75.74843:1,-13.69487,471.4475,1128.202,-11867.65 + + + 80.33296:1,-16.87616,595.0536,853.8349,-10901.54 + + + 89.22693:1,-14.73695,510.5396,741.9537,-7500.929 + + + 103.2077:1,-15.89822,559.9186,767.8516,-8650.12 + + + 96.97681:1,-15.17393,535.822,916.8895,-11156.42 + + + 80.66811:1,-14.37275,527.7707,1088.398,-11898.42 + + + 75.40509:1,-15.33448,513.0295,802.5839,-9445.232 + + + 69.2765:1,-15.84016,539.5375,617.7229,-7069.253 + + + 63.84754:1,-15.94174,564.4217,632.1392,-8806.853 + + + 60.68193:1,-15.53098,556.1161,973.8852,-14867.71 + + + 63.83185:1,-15.82032,569.0646,1255.884,-17011.9 + + + 80.47993:1,-12.97731,435.8091,1079.307,-11283.73 + + + 104.2947:1,-15.9018,560.298,811.216,-10282.13 + + + 103.7402:1,-15.4255,563.9893,628.7944,-6165.749 + + + 85.5298:1,-14.67857,502.1506,523.4214,-3840.062 + + + 76.98244:1,-15.40743,559.8364,533.4574,-5098.147 + + + 74.06737:1,-14.71057,496.6653,837.6816,-9101.387 + + + 78.59241:1,-13.65689,489.0083,1035.457,-9937.422 + + + 98.81671:1,-15.44388,526.2051,878.9273,-9351.534 + + + 119.2574:1,-18.97829,733.8035,786.6446,-10705.17 + + 2354.379:3735.039:17434.78:26658.26:19494.92:35045.32:95594.09:68023.88:13672.21:13256.16:7419.019:7118.017:12997.09:14644.62:22535.67:28897.56:15447.66:12707.28:3935.345:7012.773:9708.504:13196.37:14538.7:12317.24:36603.27:12570.42:6674.606:5651.271:8409.773:12290.93:9306.442:25755.76:7129.233:3009.838:1641.251:866.7905 + 59945400:59952600:59956200:59961600:59965200:59968800:59972400:59976000:59979600:59983200:59990400:59995800:60003000:60008400:60012000:60015600:60019200:60024600:60030000:60035400:60040800:60046200:60049800:60053400:60057000:60060600:60064200:60069600:60075000:60082200:60085800:60091200:60094800:60100200:60105600:60111000 + + 2224.20274598133,2224.20274598133,1.21127755635235e-2,-6.35122441720404e-2,641.503931231617,280.240265748566,172.993356397543 + + + + + + 36 + 0 + + 1 + 0:4586.55:14470.09:23471.13:32413.89:40615.55:49039.48:58962.18:69759.56:81365.55:95341.95:114251.2:138681.3:160119:174005.2:185006.3:196117.7:207791.2:220267.6:233931.6:248006.3:260971.6:272910.6:284753.6:297079.5:309482.8:322953.8:337474.3:350494.9:361755.8:372181.2:382273.3:390601.8:398188.6:406512.4:416679.9:426395.5:432000 + -1486.18:-1486.18:-604.1959:2005.813:4859.792:6318.098:6138.804:3890.382:553.9976:-1477.984:-2135.999:-2574.006:-2350.135:-2135.075:-2019.911:-3771.384:-5799.03:-6362.763:-5526.892:-3888.293:-2510.3:-2075.589:-3657.679:-4673.414:-3344.561:-1375.805:-849.4881:-1530.975:-1874.537:-1240.964:1059.718:4036.127:6002.401:6025.415:3987.487:781.8337:-1320.885:-1320.885 + 12430.55:12430.55:107110.5:223564.7:168767.7:69555.95:275285.6:583861.3:374188.6:135000.2:96168.91:43239.58:31577.32:66193.87:294792.4:549807:257106.8:112898.4:372167.6:484467.5:232830.6:222630.2:218130.2:113595.3:182811:304015.6:92613.22:41610.64:64890.64:306849.6:1460462:1791717:934018.4:884051.3:1167003:877955.6:110941.6:110941.6 + + + 4305797209917071696:12377736236737676023 + + + 1e-2 + 0:9273.413:18481.92:27698.03:36242.22:44618.02:53813.14:63954.95:74848.16:87089.2:102339.9:125111.8:152824.2:168840.1:179752.1:190319.1:201629.6:213542.4:226711:241046.7:254577.1:267195.8:279179.9:290652.7:302664.3:315341.8:329852.7:345675.4:357644.4:367298.1:376846:386032.6:394400.8:402839.2:412195.5:422218.5:432000 + 4586.55:14470.09:23471.13:32413.89:40615.55:49039.48:58962.18:69759.56:81365.55:95341.95:114251.2:138681.3:160119:174005.2:185006.3:196117.7:207791.2:220267.6:233931.6:248006.3:260971.6:272910.6:284753.6:297079.5:309482.8:322953.8:337474.3:350494.9:361784.7:372181.2:382273.3:390601.8:398188.6:406512.4:416679.9:426395.5 + 44.46429:15742.55 + 44.46429:41850.78 + + + 64800:172800:604800:604800:55641600 + + 669600 + + 181.0008:1,-15.30598,581.0538,-1177.985,13445.93 + + + 179.7906:1,-18.66079,779.5819,-144.8248,-4723.167 + + + 179.9456:1,-18.62781,775.3233,2342.567,-49045.96 + + + 166.8285:1,-17.48009,716.1053,4969.522,-88632.04 + + + 163.5241:1,-20.4187,861.9767,6243.463,-126317.3 + + + 179.5019:1,-18.31644,750.9658,5836.356,-102141 + + + 198.0092:1,-17.01969,686.989,3571.213,-55732.12 + + + 212.7376:1,-18.34008,760.6952,655.9884,-13666.78 + + + 239.167:1,-17.93616,742.6577,-1061.338,12308.94 + + + 297.9984:1,-18.87808,779.5159,-1820.965,29445.75 + + + 444.4532:1,-17.54365,719.2187,-2279.186,35271.37 + + + 540.8452:1,-18.06406,737.7038,-2237.708,38663 + + + 312.9283:1,-18.03842,744.242,-1981.29,33290.44 + + + 213.1204:1,-18.24233,756.1293,-1833.541,30474.09 + + + 206.2741:1,-17.47268,713.954,-3387.345,53097.93 + + + 220.6968:1,-18.30103,754.7618,-5304.368,89275.47 + + + 232.5054:1,-17.67559,740.546,-6007.729,100348.4 + + + 257.2058:1,-17.82572,724.0578,-5354.784,92782.88 + + + 279.7976:1,-17.95573,744.6683,-3862.039,68924.88 + + + 264.0945:1,-17.53174,721.9521,-2448.738,41946.78 + + + 246.8018:1,-17.82511,734.8415,-1965.045,33271.05 + + + 234.6863:1,-18.02131,744.7393,-3035.958,44848.72 + + + 224.711:1,-17.84067,732.4085,-3930.321,58423.95 + + + 235.3015:1,-17.37838,712.4959,-3019.912,47329.8 + + + 248.3526:1,-18.13612,754.2668,-1488.456,28792.94 + + + 284.2384:1,-17.74905,732.0732,-926.2106,17656.74 + + + 309.0565:1,-17.54298,724.2068,-1249.427,17426.48 + + + 232.9793:1,-17.76094,731.8377,-1420.608,18055.68 + + + 188.9714:1,-17.42779,729.535,-821.1015,7627.806 + + + 187.0731:1,-18.86893,769.95,1461.686,-33634.09 + + + 181.3046:1,-18.63458,769.4051,4306.386,-84432.02 + + + 164.1492:1,-17.86577,731.1778,5874.808,-102975.1 + + + 163.8697:1,-19.92569,824.5882,5913.558,-116158.6 + + + 180.5124:1,-17.03296,691.3312,4314.901,-78605.87 + + + 194.4606:1,-20.1262,849.2705,1313.023,-34614.83 + + + 190.7946:1,-19.88557,845.5448,-824.6146,8585.028 + + 12430.55:107110.5:223564.7:168767.7:69555.95:275285.6:583861.3:374188.6:135000.2:96168.91:43239.58:31577.32:66193.87:294792.4:549807:257106.8:112898.4:372167.6:484467.5:232830.6:222630.2:218130.2:113595.3:182811:304015.6:92613.22:41610.64:64890.64:305530.6:1460462:1791717:934018.4:884051.3:1167003:877955.6:110941.6 + 60121800:60130800:60139800:60148800:60156000:60165000:60175800:60186600:60199200:60213600:60237000:60264000:60280200:60291000:60301800:60312600:60325200:60337800:60352200:60366600:60379200:60391800:60402600:60415200:60427800:60442200:60458400:60469200:60480000:59884200:59893200:59902200:59909400:59920200:59929200:59938200 + + 2221.79711803412,2221.79711803412,-4.37408322237771e-1,-9.93477568063273e-2,3936.15102240582,1804.66800498102,1198.02536255792 + + + + + + 36 + 0 + + 1 + 0:12193.68:35797.8:61919.09:93253.2:133544.9:172247.9:207209.9:239223.2:266798.4:291697.6:317550.5:346673.6:381945.6:426213.2:478358.6:527788:566087.8:596304:621811.3:647587.2:678843.5:718248.1:755603.5:793332.8:832109.1:864185.6:891304.8:919025.8:948235.1:979916.6:1015445:1057970:1102988:1142975:1172164:1196063:1209600 + 233.258:233.258:558.7869:264.9304:152.7257:115.5334:83.3082:102.5678:194.827:92.53857:-134.7057:-52.84145:65.7438:124.6542:158.1524:192.6141:212.4394:147.498:80.19669:401.8564:450.8602:215.9216:126.838:101.3764:90.30961:131.1399:125.1167:-146.6861:-56.78666:50.10894:148.6482:236.1386:142.3618:225.5474:246.4328:162.3415:24.26862:24.26862 + 5272.663:5272.663:10542.43:12940.6:4423.579:1929.954:1588.578:2344.794:5325.227:11365.21:11486.73:9991.312:8383.01:5334.481:4247.421:3354.775:2667.394:3058.679:5306.493:19403.21:12385.26:9353.625:3016.794:1555.696:1348.572:2808.337:5041.103:10197.87:7626.104:6361.777:4494.575:5350.004:2437.358:3526.178:2129.238:3792.31:3788.388:3788.388 + + + 5576312830972368443:13246837206183887008 + + + 1e-2 + 0:21805.93:45505.6:72698.97:106977.7:152504.7:195150.1:228783.9:255739:279112.6:302498.9:328597.1:359631:399643:451508.8:509103.6:552126.9:583873.6:609974.6:633998.6:659257.7:691342.2:734228.6:777180.4:819472.8:853381.6:879366.1:904117.6:931812.9:963297.1:996893.3:1034183:1082320:1127222:1159965:1186519:1209600 + 12034.02:35797.8:61919.09:93253.2:133544.9:172247.9:207209.9:239223.2:266798.4:291697.6:317550.5:346673.6:381945.6:426213.2:478358.6:527788:566087.8:596304:621811.3:647587.2:678843.5:718248.1:755603.5:793332.8:832109.1:864185.6:891304.8:919025.8:948235.1:979916.6:1015445:1057970:1102988:1142975:1172164:1196063 + 14.51161:-424741.7 + 14.51161:848474.3 + + + 1209600:55641600 + + 37497600 + + 76.37052:1,-2.099008,25.75725,134.3513,41.64057 + + + 82.15195:1,-2.239855,26.90755,368.5548,-161.2893 + + + 94.68034:1,-2.286711,27.19185,188.5545,-166.5029 + + + 119.4457:1,-2.246899,26.68697,105.8525,-77.45677 + + + 157.4576:1,-2.140613,25.93566,75.4697,-24.69386 + + + 141.7607:1,-2.061097,25.33407,50.74478,6.094574 + + + 106.1087:1,-1.958582,24.98741,62.22939,17.30761 + + + 85.68337:1,-2.022675,25.57369,130.1115,-39.67626 + + + 77.46091:1,-2.012048,25.68156,48.86966,53.2454 + + + 78.93953:1,-2.004435,25.72254,-108.2641,125.1179 + + + 88.41419:1,-2.037208,26.04684,-65.84857,179.3574 + + + 103.8679:1,-2.024487,25.80559,24.58846,91.77513 + + + 128.5648:1,-1.940052,25.31229,83.39919,-19.69365 + + + 162.7664:1,-1.859506,24.84006,106.9222,-22.47976 + + + 189.6031:1,-1.701109,23.95205,130.56,-7.754473 + + + 145.9273:1,-1.730978,24.21066,149.8356,-44.01663 + + + 108.5709:1,-1.772878,25.06271,97.43912,3.072704 + + + 88.92345:1,-1.745609,24.96021,55.77113,-11.55973 + + + 81.29551:1,-1.78791,25.19193,276.8912,-58.9001 + + + 85.62186:1,-1.730201,24.98217,290.9313,57.8887 + + + 109.939:1,-1.769973,25.0703,144.7601,-9.647258 + + + 147.4653:1,-1.732827,24.91696,81.56346,15.48742 + + + 140.7388:1,-1.639132,24.25067,59.16972,48.31477 + + + 132.9965:1,-1.498267,23.48747,54.10478,43.90745 + + + 112.7145:1,-1.373907,22.90626,88.62956,25.63057 + + + 86.14164:1,-1.471433,23.44891,66.43815,104.1365 + + + 81.26155:1,-1.425414,23.61343,-121.3088,84.03823 + + + 89.73572:1,-1.455221,23.95249,-72.40976,160.26 + + + 102.7477:1,-1.486422,24.09266,16.22383,94.14198 + + + 110.8977:1,-1.373969,23.48196,93.65491,63.35173 + + + 122.626:1,-1.33603,23.19691,162.1315,39.987 + + + 161.4265:1,-1.277209,22.56918,108.5567,-23.59329 + + + 151.2642:1,-1.257344,22.47218,160.1276,20.20079 + + + 109.9769:1,-1.247792,22.88626,163.3794,82.19346 + + + 90.16071:1,-1.330769,23.80629,129.0141,-55.11602 + + + 79.19389:1,-1.414338,24.40629,-8.700216,127.6699 + + 5595.612:10542.43:12940.6:4423.579:1929.954:1588.578:2344.794:5325.227:11365.21:11486.73:9991.312:8383.01:5334.481:4247.421:3354.775:2667.394:3058.679:5306.493:19403.21:12385.26:9353.625:3016.794:1555.696:1348.572:2808.337:5041.103:10197.87:7626.104:6361.777:4494.575:5350.004:2437.358:3526.178:2129.238:3792.31:3788.388 + 60480000:59317200:59346000:59380200:59428800:59470200:59502600:59527800:59551200:59574600:59601600:59632200:59671800:59720400:59778000:59821200:59853600:59880600:59904000:59931000:59963400:60008400:60049800:60089400:60123600:60148800:60174000:60201000:60231600:60264000:60301800:60350400:60395400:60427800:60454800:60478200 + + 514.870235480273,514.870235480273,-8.22100853376075e-2,-1.21178223089578e-2,132.915051906945,55.1018356190892,33.3067602419617 + + + + + 5 + 4364.128:9732.331;4364.128:5.608692e7; + 4474.112:35566.09;4474.112:5355942; + 3019.781:9449.2;3019.781:273769.2; + 7549.77:36022.12;7549.77:1.021911e7; + 10516.71:28438.74;10516.71:28585.84; + + + + + + 12 + 0 + + 1 + 0:2460.282:8146.763:13870.04:21040.14:28545.24:35459.91:42367.75:49872.03:58700.9:68139.48:76250.36:82651.05:86400 + -182.6609:-182.6609:-66.38694:12.331:106.2038:158.9741:159.6649:107.0884:29.56036:-18.8622:18.06953:-84.21854:-146.8488:-146.8488 + 2429.384:2429.384:2398.067:1826.141:1964.757:789.4703:1030.021:1622.533:826.6129:569.966:1581.522:2069.256:1056.502:1056.502 + + + + + 1e-2 + 0:5351.383:10687.99:16902.41:24404.03:31904.28:39031.43:46261.74:54097.12:63510.34:72769.94:80211.59:86400 + 2460.282:8146.763:13870.04:21040.14:28545.24:35459.91:42367.75:49872.03:58700.9:68139.48:76250.36:82651.05 + 8.973129:-10235.71 + 8.973129:17096.87 + + 3:27 + + 12 + 21.08349:-182.6609:2429.384 + 21.01215:-66.38694:2398.067 + 24.20266:12.331:1826.141 + 29.11081:106.2038:1964.757 + 29.28073:158.9741:789.4703 + 28.04483:159.6649:1030.021 + 28.56414:107.0884:1622.533 + 30.83901:29.56036:826.6129 + 37.17579:-18.8622:569.966 + 36.6405:18.06953:1581.522 + 29.47567:-84.21854:2069.256 + 24.44864:-146.8488:1056.502 + + + + + + 12 + 0 + + 1 + 0:2767.718:10273.52:18078.37:25758.89:33009.44:39934.04:46703.16:53411.4:60300:67348.12:74393.8:81744.24:86400 + -83.48159:-83.48159:-1.077249:35.10725:49.61652:107.6665:140.6021:100.1887:22.18724:-75.3891:-30.81896:-30.69588:2.982394:2.982394 + 739.644:739.644:418.5691:9.785093:688.1915:870.2864:494.9797:1667.811:1341.228:1757.836:115.5179:431.963:1091.564:1091.564 + + + + + 1e-2 + 0:7268.361:14790.22:22670.86:30357.5:37496.69:44379.71:51155.88:57891.3:64651.59:71687.27:79040.8:86400 + 2767.718:10273.52:18078.37:25758.89:33009.44:39934.04:46703.16:53411.4:60300:67348.12:74393.8:81744.24 + 9.801987e-1:24760.67 + 9.801987e-1:24760.67 + + 4:4 + + 12 + 3.383481:-83.48159:739.644 + 3.383481:-1.077249:418.5691 + 3.383481:35.10725:9.785093 + 3.383481:49.61652:688.1915 + 3.383481:107.6665:870.2864 + 3.383481:140.6021:494.9797 + 3.383481:100.1887:1667.811 + 3.383481:22.18724:1341.228 + 3.383481:-75.3891:1757.836 + 3.383481:-30.81896:115.5179 + 3.383481:-30.69588:431.963 + 3.383481:2.982394:1091.564 + + + + + 2 + 410.6316:12652.24;410.6316:18080.95; + 47.52239:90613.36;47.52239:90613.36; + + + + + diff --git a/lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.trend_and_seasonal.expected_scales.txt b/lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.trend_and_seasonal.expected_scales.txt new file mode 100644 index 0000000000..b9dd1219d1 --- /dev/null +++ b/lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.trend_and_seasonal.expected_scales.txt @@ -0,0 +1 @@ +0.886239,0.93872;1.09523,1.17329;1.00221,1.06251;0.867651,0.913012;0.873291,0.919419;0.908863,0.960655;0.979067,1.03602;0.987334,1.04321;0.98218,1.03276;0.969392,1.01873;0.960937,1.00763;0.94507,0.987285;0.92621,0.966787;0.949833,0.994085;0.961915,1.00726;0.957065,1.00509;0.950649,0.997168;0.94412,0.990021;0.937448,0.986189;0.953585,1.00442;0.980173,1.03374;0.997753,1.05726;0.976347,1.03429;0.936809,0.990334;0.946461,1.00096;0.990645,1.04994;0.99347,1.04973;0.95097,1.00275;0.92904,0.980052;0.939574,0.991598;0.954331,1.0078;0.955157,1.00482;0.950147,0.999313;0.955175,1.0021;0.951683,0.994538;0.940695,0.982594;0.922974,0.965505;0.917699,0.959747;0.908753,0.953941;0.902218,0.947055;0.92132,0.964307;0.934781,0.979019;0.929164,0.974828;0.944401,0.991545;0.968229,1.02199;0.999936,1.05935;1.00787,1.07251;0.901335,0.955568;0.886239,0.93872;1.09523,1.17329;1.00221,1.06251;0.867651,0.913012;0.873291,0.919419;0.908863,0.960655;0.979067,1.03602;0.987334,1.04321;0.98218,1.03276;0.969392,1.01873;0.960937,1.00763;0.94507,0.987285;0.92621,0.966787;0.949833,0.994085;0.961915,1.00726;0.957065,1.00509;0.950649,0.997168;0.94412,0.990021;0.937448,0.986189;0.953585,1.00442;0.980173,1.03374;0.997753,1.05726;0.976347,1.03429;0.936809,0.990334;0.946461,1.00096;0.990645,1.04994;0.99347,1.04973;0.95097,1.00275;0.92904,0.980052;0.939574,0.991598;0.954331,1.0078;0.955157,1.00482;0.950147,0.999313;0.955175,1.0021;0.951683,0.994538;0.940695,0.982594;0.922974,0.965505;0.917699,0.959747;0.908753,0.953941;0.902218,0.947055;0.92132,0.964307;0.934781,0.979019;0.929164,0.974828;0.944401,0.991545;0.968229,1.02199;0.999936,1.05935;1.00787,1.07251;0.901335,0.955568;0.886239,0.93872;1.09523,1.17329;1.00221,1.06251;0.867651,0.913012;0.873291,0.919419;0.908863,0.960655;0.979067,1.03602;0.987334,1.04321;0.98218,1.03276;0.969392,1.01873;0.960937,1.00763;0.94507,0.987285;0.92621,0.966787;0.949833,0.994085;0.961915,1.00726;0.957065,1.00509;0.950649,0.997168;0.94412,0.990021;0.937448,0.986189;0.953585,1.00442;0.980173,1.03374;0.997753,1.05726;0.976347,1.03429;0.936809,0.990334;0.946461,1.00096;0.990645,1.04994;0.99347,1.04973;0.95097,1.00275;0.92904,0.980052;0.939574,0.991598;0.954331,1.0078;0.955157,1.00482;0.950147,0.999313;0.955175,1.0021;0.951683,0.994538;0.940695,0.982594;0.922974,0.965505;0.917699,0.959747;0.908753,0.953941;0.902218,0.947055;0.92132,0.964307;0.934781,0.979019;0.929164,0.974828;0.944401,0.991545;0.968229,1.02199;0.999936,1.05935;1.00787,1.07251;0.901335,0.955568;0.886239,0.93872;1.09523,1.17329;1.00221,1.06251;0.867651,0.913012;0.873291,0.919419;0.908863,0.960655;0.979067,1.03602;0.987334,1.04321;0.98218,1.03276;0.969392,1.01873;0.960937,1.00763;0.94507,0.987285;0.92621,0.966787;0.949833,0.994085;0.961915,1.00726;0.957065,1.00509;0.950649,0.997168;0.94412,0.990021;0.937448,0.986189;0.953585,1.00442;0.980173,1.03374;0.997753,1.05726;0.976347,1.03429;0.936809,0.990334;0.946461,1.00096;0.990645,1.04994;0.99347,1.04973;0.95097,1.00275;0.92904,0.980052;0.939574,0.991598;0.954331,1.0078;0.955157,1.00482;0.950147,0.999313;0.955175,1.0021;0.951683,0.994538;0.940695,0.982594;0.922974,0.965505;0.917699,0.959747;0.908753,0.953941;0.902218,0.947055;0.92132,0.964307;0.934781,0.979019;0.929164,0.974828;0.944401,0.991545;0.968229,1.02199;0.999936,1.05935;1.00787,1.07251;0.901335,0.955568;0.886239,0.93872;1.09523,1.17329;1.00221,1.06251;0.867651,0.913012;0.873291,0.919419;0.908863,0.960655;0.979067,1.03602;0.987334,1.04321;0.98218,1.03276;0.969392,1.01873;0.960937,1.00763;0.94507,0.987285;0.92621,0.966787;0.949833,0.994085;0.961915,1.00726;0.957065,1.00509;0.950649,0.997168;0.94412,0.990021;0.937448,0.986189;0.953585,1.00442;0.980173,1.03374;0.997753,1.05726;0.976347,1.03429;0.936809,0.990334;0.946461,1.00096;0.990645,1.04994;0.99347,1.04973;0.95097,1.00275;0.92904,0.980052;0.939574,0.991598;0.954331,1.0078;0.955157,1.00482;0.950147,0.999313;0.955175,1.0021;0.951683,0.994538;0.940695,0.982594;0.922974,0.965505;0.917699,0.959747;0.908753,0.953941;0.902218,0.947055;0.92132,0.964307;0.934781,0.979019;0.929164,0.974828;0.944401,0.991545;0.968229,1.02199;0.999936,1.05935;1.00787,1.07251;0.901335,0.955568;0.886239,0.93872;1.09523,1.17329;1.00221,1.06251;0.867651,0.913012;0.873291,0.919419;0.908863,0.960655;0.979067,1.03602;0.987334,1.04321;0.98218,1.03276;0.969392,1.01873;0.960937,1.00763;0.94507,0.987285;0.92621,0.966787;0.949833,0.994085;0.961915,1.00726;0.957065,1.00509;0.950649,0.997168;0.94412,0.990021;0.937448,0.986189;0.953585,1.00442;0.980173,1.03374;0.997753,1.05726;0.976347,1.03429;0.936809,0.990334;0.946461,1.00096;0.990645,1.04994;0.99347,1.04973;0.95097,1.00275;0.92904,0.980052;0.939574,0.991598;0.954331,1.0078;0.955157,1.00482;0.950147,0.999313;0.955175,1.0021;0.951683,0.994538;0.940695,0.982594;0.922974,0.965505;0.917699,0.959747;0.908753,0.953941;0.902218,0.947055;0.92132,0.964307;0.934781,0.979019;0.929164,0.974828;0.944401,0.991545;0.968229,1.02199;0.999936,1.05935;0.981748,1.04319;0.849086,0.896926;0.807866,0.850757;1.09523,1.17329;1.00221,1.06251;0.867651,0.913012;0.873291,0.919419;0.908863,0.960655;0.979067,1.03602;0.987334,1.04321;0.98218,1.03276;0.969392,1.01873;0.960937,1.00763;0.94507,0.987285;0.92621,0.966787;0.949833,0.994085;0.961915,1.00726;0.957065,1.00509;0.950649,0.997168;0.94412,0.990021;0.937448,0.986189;0.953585,1.00442;0.980173,1.03374;0.997753,1.05726;0.976347,1.03429;0.936809,0.990334;0.946461,1.00096;0.990645,1.04994;0.99347,1.04973;0.95097,1.00275;0.92904,0.980052;0.939574,0.991598;0.954331,1.0078;0.955157,1.00482;0.950147,0.999313;0.955175,1.0021;0.951683,0.994538;0.940695,0.982594;0.922974,0.965505;0.917699,0.959747;0.908753,0.953941;0.902218,0.947055;0.92132,0.964307;0.934781,0.979019;0.929164,0.974828;0.944401,0.991545;0.968229,1.02199;0.999936,1.05935;1.00787,1.07251;0.901335,0.955568; \ No newline at end of file diff --git a/lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.trend_and_seasonal.expected_values.txt b/lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.trend_and_seasonal.expected_values.txt new file mode 100644 index 0000000000..f17e029ea4 --- /dev/null +++ b/lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.trend_and_seasonal.expected_values.txt @@ -0,0 +1 @@ +132.877,133.397;138.668,139.301;137.352,137.886;135.316,135.78;137.637,138.105;139.014,139.517;137.846,138.363;137.542,138.046;141.332,141.794;145.131,145.587;145.371,145.807;143.419,143.822;142.486,142.881;144.769,145.188;146.756,147.18;146.507,146.956;144.824,145.263;142.787,143.223;142.296,142.76;144.757,145.232;144.1,144.588;140.342,140.872;138.51,139.035;138.828,139.334;140.775,141.285;138.693,139.224;135.293,135.798;133.857,134.341;133.896,134.383;134.206,134.697;133.201,133.698;131.571,132.035;130.81,131.273;130.98,131.42;130.604,131.011;130.021,130.422;129.986,130.4;130.688,131.1;131.432,131.875;131.86,132.303;132.491,132.91;133.082,133.507;133.366,133.805;133.599,134.045;133.895,134.389;134.689,135.218;136.883,137.449;140.422,140.95;141.904,142.424;147.709,148.342;146.406,146.941;144.384,144.848;146.719,147.187;148.11,148.612;146.956,147.472;146.665,147.169;150.469,150.931;154.281,154.737;154.535,154.972;152.597,153;151.678,152.073;153.974,154.393;155.975,156.399;155.74,156.189;154.071,154.51;152.048,152.484;151.571,152.034;154.045,154.52;153.402,153.89;149.658,150.188;147.84,148.365;148.172,148.677;150.133,150.642;148.064,148.595;144.678,145.183;143.256,143.74;143.309,143.796;143.632,144.124;142.642,143.139;141.025,141.49;140.279,140.741;140.462,140.903;140.101,140.507;139.531,139.933;139.511,139.925;140.226,140.638;140.984,141.427;141.426,141.869;142.071,142.49;142.676,143.101;142.974,143.413;143.221,143.667;143.531,144.025;144.339,144.867;146.546,147.113;150.1,150.628;151.596,152.116;157.414,158.047;156.126,156.661;154.118,154.582;156.467,156.935;157.872,158.374;156.732,157.248;156.455,156.959;160.273,160.735;164.099,164.555;164.367,164.804;162.443,162.846;161.538,161.933;163.849,164.268;165.863,166.288;165.642,166.092;163.988,164.427;161.979,162.415;161.516,161.979;164.004,164.479;163.375,163.863;159.646,160.176;157.841,158.367;158.188,158.693;160.163,160.672;158.108,158.639;154.736,155.241;153.328,153.813;153.396,153.882;153.733,154.224;152.757,153.254;151.155,151.619;150.422,150.885;150.62,151.06;150.273,150.679;149.717,150.119;149.711,150.125;150.441,150.853;151.213,151.656;151.669,152.113;152.328,152.747;152.948,153.373;153.26,153.699;153.521,153.968;153.845,154.34;154.668,155.196;156.89,157.456;160.458,160.985;161.968,162.488;167.801,168.434;166.527,167.061;164.533,164.997;166.896,167.364;168.315,168.818;167.189,167.706;166.927,167.431;170.76,171.222;174.6,175.056;174.883,175.319;172.973,173.376;172.082,172.477;174.407,174.826;176.436,176.861;176.23,176.679;174.59,175.028;172.595,173.031;172.146,172.61;174.65,175.125;174.035,174.523;170.32,170.85;168.53,169.055;168.891,169.396;170.88,171.39;168.84,169.371;165.483,165.988;164.089,164.574;164.171,164.658;164.523,165.014;163.561,164.059;161.974,162.438;161.256,161.718;161.468,161.908;161.135,161.541;160.595,160.996;160.603,161.017;161.348,161.759;162.134,162.577;162.605,163.048;163.278,163.697;163.913,164.338;164.239,164.679;164.515,164.962;164.854,165.348;165.691,166.219;167.927,168.494;171.51,172.038;173.035,173.555;178.882,179.515;177.623,178.157;175.644,176.108;178.021,178.489;179.455,179.958;178.344,178.861;178.097,178.601;181.944,182.406;185.799,186.255;186.097,186.533;184.201,184.604;183.325,183.72;185.665,186.084;187.709,188.133;187.517,187.967;185.892,186.331;183.912,184.348;183.478,183.942;185.996,186.471;185.396,185.884;181.696,182.226;179.92,180.446;180.296,180.802;182.301,182.81;180.275,180.806;176.933,177.438;175.554,176.038;175.651,176.138;176.018,176.509;175.07,175.568;173.498,173.962;172.795,173.257;173.022,173.462;172.704,173.11;172.178,172.58;172.201,172.615;172.961,173.372;173.762,174.205;174.248,174.691;174.936,175.355;175.586,176.01;175.927,176.366;176.217,176.664;176.572,177.066;177.424,177.952;179.675,180.241;183.273,183.8;184.812,185.332;190.675,191.308;189.43,189.965;187.466,187.93;189.859,190.327;191.308,191.81;190.212,190.728;189.979,190.483;193.841,194.303;197.712,198.168;198.024,198.46;196.144,196.547;195.283,195.678;197.638,198.057;199.697,200.121;199.52,199.969;197.909,198.348;195.945,196.381;195.526,195.989;198.059,198.534;197.474,197.962;193.789,194.319;192.029,192.554;192.419,192.925;194.439,194.948;192.429,192.96;189.101,189.606;187.738,188.222;187.849,188.336;188.232,188.723;187.3,187.797;185.742,186.206;185.054,185.516;185.296,185.737;184.994,185.4;184.483,184.885;184.521,184.935;185.296,185.708;186.112,186.556;186.614,187.057;187.317,187.736;187.982,188.407;188.338,188.778;188.644,189.091;189.014,189.508;189.881,190.409;191.446,191.998;194.358,194.857;195.212,195.689;203.193,203.826;201.964,202.498;200.015,200.479;202.423,202.891;203.887,204.39;202.806,203.323;202.589,203.093;206.467,206.929;210.352,210.808;210.68,211.116;208.815,209.218;207.969,208.364;210.34,210.759;212.414,212.838;212.252,212.702;210.657,211.096;208.708,209.144;208.305,208.768;210.853,211.328;210.284,210.771;206.614,207.144;204.869,205.395;205.275,205.781;207.31,207.82;205.316,205.846;202.004,202.508;200.655,201.14;200.783,201.27;201.18,201.671;200.264,200.761;198.722,199.186;198.049,198.512;198.307,198.747;198.02,198.426;197.525,197.926;197.578,197.992;198.369,198.78;199.2,199.644;199.717,200.161;200.436,200.855;201.117,201.541;201.488,201.928;201.81,202.257;202.195,202.689;203.078,203.606;205.36,205.926;208.988,209.516; \ No newline at end of file diff --git a/lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.trend_and_seasonal.state.xml b/lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.trend_and_seasonal.state.xml new file mode 100644 index 0000000000..da5c28330d --- /dev/null +++ b/lib/maths/unittest/testfiles/CTimeSeriesDecomposition.6.2.trend_and_seasonal.state.xml @@ -0,0 +1,560 @@ + + 2.4e-2 + 10366200 + 10366200 + + + 0 + 2 + + 10368000 + + + + 1 + 3 + + 0 + 0 + 0 + + 4;0;86400;0;604800 + + + + + 2 + 1 + + + 0 + 10281600 + + 168 + 1:12.60541 + 1:14.19556 + 1:17.79921 + 1:17.15353 + 1:17.21997 + 1:19.92679 + 1:20.77021 + 1:18.01883 + 1:15.42434 + 1:14.19508 + 1:17.02139 + 1:17.83361 + 1:17.46921 + 1:15.77641 + 1:15.326 + 1:16.88908 + 1:18.26118 + 1:18.10428 + 1:15.33726 + 1:15.28679 + 1:16.96092 + 1:19.07243 + 1:17.59718 + 1:14.36802 + 1:14.60532 + 1:16.7252 + 1:18.0339 + 1:17.18947 + 1:15.25461 + 1:15.40931 + 1:17.18618 + 1:16.26258 + 1:15.71746 + 1:15.99754 + 1:17.54311 + 1:17.46786 + 1:15.54406 + 1:15.04921 + 1:15.34437 + 1:15.20612 + 1:16.26483 + 1:16.86064 + 1:16.13064 + 1:17.85892 + 1:18.22312 + 1:17.88932 + 1:15.67078 + 1:14.56746 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + + + + 0 + 10281600 + + 168 + 12:-56.81403 + 12:-53.81668 + 12:-51.51041 + 12:-49.17507 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + + + + + + 2 + 1 + + 3 + + 10281600 + + 0 + 1.9358e-3;5.488116e-1:9.015685e-1;265.6828:2.916767;162.3924:4.949073;119.9475:6.730366;106.4132:8.685481;103.7199:10.73572;105.4709:12.69486;106.3114:14.57687;105.7416:16.27713;147.0379:17.90802;187.6309:19.40618;172.0567:21.05325;91.47071:22.25773;4.743615:23.1986;45.88255:25.23945;29.93542:27.50158;4.853569:32.22551;9.323101e-1:34.86858;1.331872e-1 + 1760.905 + + 41:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:1048624:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48:48 + 4752000;29.10957 + + + + + 3 + 1 + + + + 1805.242:1,-1.31307676805491,14.1645495833669,-85.747961630886,726.693607433893,-6047.70138318784,53788.2457901516,-72.286303474959,-59.678409485381,-238.473339712036,782.113411846747 + + 44.15356 + 8467200 + 10366200 + + 5.17930805474757,5.17930805474757,5.17930805474757,5.17930805474757,-5.67465018142549e-1,-5.35716772556194e-1,-1.25103748715301e-1,-7.53846204263943e-3,9.10525425485549e-1,2.39039853516001,7.25790489451945,1.91780148527838,6.44345933757583,6.15752827162214,4.94366009670206e-1,1.7873357158638,1.79833703225809,5.44281201997451e-1 + + + + + + 36 + 0 + + 1 + 0:796.1544:3103.068:5145.687:7264.795:9221.888:11238.55:13609.88:16694.63:19641.51:22479.63:25470.38:28193.85:30716.07:33211.39:35638.52:37599.84:39535.21:41787.07:44039.98:46106.15:48156.89:50356.64:52633.34:54965.66:57791.12:60701.03:63403.94:65997.18:68742.09:71541.09:74050.07:76398.61:78783.52:80868.63:82800:84600:86400 + 202.966:202.966:199.2326:200.9624:202.5078:200.9788:201.1597:206.7328:207.4613:204.6042:207.9323:207.9314:204.7552:203.4639:206.0556:201.5912:199.0093:198.9453:200.7357:196.5981:193.6814:193.0561:193.2495:191.6139:189.5025:189.2047:188.0453:187.7046:188.5875:189.0178:189.7183:189.89:189.9661:190.4491:192.4683:196.0171:197.3001:197.3001 + 69.43143:69.43143:43.83319:43.89956:48.70205:56.58526:56.20063:54.21448:52.26087:48.31939:52.52658:52.38759:50.99003:50.68694:53.33718:58.14928:56.08868:51.30906:52.58602:59.77145:53.59012:49.99472:51.3345:53.30206:51.85392:52.09013:50.49984:48.16302:47.78701:46.79853:49.91314:49.46422:51.96391:57.37207:60.28613:48.26214:46.63048:46.63048 + + + 17726390493245370771:7552890861703612417 + + + 2.4e-2 + 0:1800:3746.56:5888.302:7922.598:10121.43:12452.4:15016.9:17956.98:21121.5:24070.75:26839.08:29534.71:32063.77:34380.23:36551.72:38640.04:40717.26:42799.26:44893.77:47044.17:49244.45:51454.98:53683.51:56073.32:58932.04:62085.44:64935.52:67535.84:70160.47:73067.07:75824.62:78513.04:80822.76:82800:84600:86400 + 723.6114:3040.777:5165.438:7259.475:9205.13:11207.32:13596.46:16666.38:19566.43:22481.83:25453.94:28159.28:30708.31:33223.41:35666.21:37615.54:39540.29:41756.8:43974.57:46077.02:48189.86:50359.86:52601.39:54957.15:57779.88:60660.32:63459.17:66036.52:68719.51:71490.75:74130.09:76469.7:78815.13:80879.83:82800:84600 + 21.29515:-19140.54 + 21.29515:48515.75 + + + 0:0:604800:86400:8467200 + + 0 + + 10.97493:1,1.882674,5.489721,200.2413,381.4913 + + + 12.94137:1,1.287621,5.057968,198.2252,260.3673 + + + 12.87549:1,1.046113,4.622236,198.7841,212.614 + + + 12.17932:1,1.13465,4.403981,199.7749,231.998 + + + 13.24073:1,8.775443e-1,4.894278,200.2584,178.2472 + + + 14.04309:1,7.00282e-1,5.358633,201.6064,141.1813 + + + 16.37716:1,1.374408,5.628109,205.408,287.0013 + + + 17.509:1,1.266969,4.843598,204.9155,264.1443 + + + 19.7539:1,9.662842e-1,4.855563,204.6179,199.8511 + + + 18.49534:1,1.235422,5.413564,206.5344,259.7873 + + + 16.44668:1,1.246199,5.032931,205.5585,260.7482 + + + 16.99754:1,1.149296,5.019363,204.2957,238.2598 + + + 14.96432:1,8.463035e-1,5.131957,202.8609,173.5886 + + + 14.74019:1,1.564794,5.512224,204.099,325.0095 + + + 13.05034:1,9.226088e-1,5.164697,200.4297,187.937 + + + 12.7499:1,8.18933e-1,5.374825,198.8011,164.5745 + + + 12.74612:1,1.058556,5.10446,197.7862,212.6676 + + + 12.78953:1,1.511615,5.339962,198.1986,304.7751 + + + 12.8423:1,1.100566,5.261093,195.2566,218.6886 + + + 14.15577:1,1.060271,5.423214,193.6232,209.1045 + + + 13.44162:1,1.070454,5.004012,191.8267,208.5039 + + + 13.49871:1,1.303108,5.222371,191.384,253.3997 + + + 13.56302:1,1.168716,5.300786,189.8088,226.0446 + + + 15.42069:1,1.251023,5.289417,188.7017,240.375 + + + 17.01463:1,1.099964,5.172003,187.1745,209.89 + + + 19.70075:1,1.175224,5.292406,186.6889,223.8666 + + + 17.96332:1,1.230117,5.316032,186.4933,233.8322 + + + 15.32267:1,1.250178,5.166911,186.4809,237.3046 + + + 15.14447:1,1.221678,5.151577,186.8134,232.6072 + + + 17.5226:1,1.251653,5.480423,188.1931,240.4815 + + + 16.24424:1,1.176315,5.635084,188.997,226.6704 + + + 14.08487:1,1.00505,5.385236,189.1129,192.6042 + + + 13.17233:1,1.062038,5.583666,189.8996,203.8778 + + + 11.73478:1,1.175259,5.624537,191.3635,228.5566 + + + 10.98446:1,1.263752,5.376832,193.9006,250.2887 + + + 10.95077:1,1.271388,5.345518,195.1211,253.2147 + + 66.13342:46.07275:45.76432:50.52668:59.94428:58.36333:56.98878:52.61569:51.40335:53.06348:52.35333:54.11626:51.60847:54.10194:59.94865:57.62322:53.2299:52.65767:61.1862:54.96964:51.37902:51.69194:52.88792:53.64557:52.44284:52.2388:49.58831:48.02882:47.35268:50.82274:51.22694:53.39392:58.97668:59.28835:47.87883:46.69099 + 10281600:10285200:10287000:10288800:10290600:10292400:10296000:10297800:10301400:10305000:10306800:10310400:10312200:10315800:10317600:10319400:10321200:10323000:10324800:10328400:10330200:10332000:10333800:10337400:10339200:10342800:10346400:10348200:10350000:10353600:10357200:10359000:10360800:10362600:10364400:10366200 + + 202.364245290825,202.364245290825,-1.8191269569635,-5.37785947081112e-1,4.45319885243975,1.34788793731147,4.42339075387864e-1 + + + + + 1 + 1760.772:183.2568;1760.772:235.8358; + + + + + diff --git a/lib/maths/unittest/testfiles/CUnivariateTimeSeriesModel.6.2.expected_intervals.txt b/lib/maths/unittest/testfiles/CUnivariateTimeSeriesModel.6.2.expected_intervals.txt new file mode 100644 index 0000000000..3d5a116586 --- /dev/null +++ b/lib/maths/unittest/testfiles/CUnivariateTimeSeriesModel.6.2.expected_intervals.txt @@ -0,0 +1 @@ +-6.251348,-2.901241,0.4488662,;-5.343162,-1.993055,1.357052,;-3.516185,-0.1660782,3.184029,;-1.449993,1.900114,5.250221,;-1.039323,2.310784,5.660891,;0.4458539,3.795961,7.146068,;2.026607,5.376714,8.726821,;2.716834,6.066941,9.417048,;3.970657,7.320764,10.67087,;4.95068,8.300787,11.65089,;5.685371,9.035478,12.38558,;6.74014,10.09025,13.44035,;7.143804,10.49391,13.84402,;7.580485,10.93059,14.2807,;8.256265,11.60637,14.95648,;8.447602,11.79771,15.14782,;7.983014,11.33312,14.68323,;7.463431,10.81354,14.16364,;7.37704,10.72715,14.07725,;6.237444,9.587551,12.93766,;4.69587,8.045977,11.39608,;4.151903,7.50201,10.85212,;3.819442,7.169549,10.51966,;2.822216,6.172324,9.522431,;0.9585956,4.308703,7.65881,;-0.7692924,2.580815,5.930922,;-1.553635,1.796472,5.146579,;-3.237453,0.1126545,3.462761,;-4.560702,-1.210594,2.139513,;-5.144293,-1.794186,1.555922,;-5.889181,-2.539074,0.8110331,;-7.006773,-3.656666,-0.3065586,;-7.875526,-4.525419,-1.175312,;-9.725742,-6.375635,-3.025528,;-11.07814,-7.728029,-4.377922,;-11.02259,-7.672484,-4.322377,;-11.93138,-8.581273,-5.231166,;-12.08928,-8.739175,-5.389068,;-11.91153,-8.561418,-5.211311,;-12.38578,-9.035671,-5.685564,;-12.5863,-9.236191,-5.886084,;-12.32126,-8.971158,-5.62105,;-11.80748,-8.457372,-5.107265,;-10.6366,-7.286496,-3.936389,;-9.57477,-6.224663,-2.874556,;-9.200827,-5.85072,-2.500613,;-7.877398,-4.527291,-1.177184,;-6.584278,-3.234171,0.1159364,;-6.006206,-2.656099,0.6940076,;-4.914165,-1.564058,1.786049,;-2.903332,0.4467753,3.796882,;-1.449993,1.900114,5.250221,;-1.039323,2.310784,5.660891,;0.4458539,3.795961,7.146068,;2.026607,5.376714,8.726821,;2.716834,6.066941,9.417048,;3.970657,7.320764,10.67087,;4.95068,8.300787,11.65089,;5.685371,9.035478,12.38558,;6.74014,10.09025,13.44035,;7.143804,10.49391,13.84402,;7.580485,10.93059,14.2807,;8.256265,11.60637,14.95648,;8.447602,11.79771,15.14782,;7.983014,11.33312,14.68323,;7.463431,10.81354,14.16364,;7.37704,10.72715,14.07725,;6.237444,9.587551,12.93766,;4.69587,8.045977,11.39608,;4.151903,7.50201,10.85212,;3.819442,7.169549,10.51966,;2.822216,6.172324,9.522431,;0.9585956,4.308703,7.65881,;-0.7692924,2.580815,5.930922,;-1.553635,1.796472,5.146579,;-3.237453,0.1126545,3.462761,;-4.560702,-1.210594,2.139513,;-5.144293,-1.794186,1.555922,;-5.889181,-2.539074,0.8110331,;-7.006773,-3.656666,-0.3065586,;-7.875526,-4.525419,-1.175312,;-9.725742,-6.375635,-3.025528,;-11.07814,-7.728029,-4.377922,;-11.02259,-7.672484,-4.322377,;-11.93138,-8.581273,-5.231166,;-12.08928,-8.739175,-5.389068,;-11.91153,-8.561418,-5.211311,;-12.38578,-9.035671,-5.685564,;-12.5863,-9.236191,-5.886084,;-12.32126,-8.971158,-5.62105,;-11.80748,-8.457372,-5.107265,;-10.6366,-7.286496,-3.936389,;-9.57477,-6.224663,-2.874556,;-9.200827,-5.85072,-2.500613,;-7.877398,-4.527291,-1.177184,;-6.584278,-3.234171,0.1159364,;-6.006206,-2.656099,0.6940076,;-4.914165,-1.564058,1.786049,;-2.903332,0.4467753,3.796882,;-1.449993,1.900114,5.250221,;-1.039323,2.310784,5.660891,;0.4458539,3.795961,7.146068,;2.026607,5.376714,8.726821,;2.716834,6.066941,9.417048,;3.970657,7.320764,10.67087,;4.95068,8.300787,11.65089,;5.685371,9.035478,12.38558,;6.74014,10.09025,13.44035,;7.143804,10.49391,13.84402,;7.580485,10.93059,14.2807,;8.256265,11.60637,14.95648,;8.447602,11.79771,15.14782,;7.983014,11.33312,14.68323,;7.463431,10.81354,14.16364,;7.37704,10.72715,14.07725,;6.237444,9.587551,12.93766,;4.69587,8.045977,11.39608,;4.151903,7.50201,10.85212,;3.819442,7.169549,10.51966,;2.822216,6.172324,9.522431,;0.9585956,4.308703,7.65881,;-0.7692924,2.580815,5.930922,;-1.553635,1.796472,5.146579,;-3.237453,0.1126545,3.462761,;-4.560702,-1.210594,2.139513,;-5.144293,-1.794186,1.555922,;-5.889181,-2.539074,0.8110331,;-7.006773,-3.656666,-0.3065586,;-7.875526,-4.525419,-1.175312,;-9.725742,-6.375635,-3.025528,;-11.07814,-7.728029,-4.377922,;-11.02259,-7.672484,-4.322377,;-11.93138,-8.581273,-5.231166,;-12.08928,-8.739175,-5.389068,;-11.91153,-8.561418,-5.211311,;-12.38578,-9.035671,-5.685564,;-12.5863,-9.236191,-5.886084,;-12.32126,-8.971158,-5.62105,;-11.80748,-8.457372,-5.107265,;-10.6366,-7.286496,-3.936389,;-9.57477,-6.224663,-2.874556,;-9.200827,-5.85072,-2.500613,;-7.877398,-4.527291,-1.177184,;-6.584278,-3.234171,0.1159364,;-6.006206,-2.656099,0.6940076,;-4.914165,-1.564058,1.786049,;-2.903332,0.4467753,3.796882,;-1.449993,1.900114,5.250221,;-1.039323,2.310784,5.660891,;0.4458539,3.795961,7.146068,;2.026607,5.376714,8.726821,;2.716834,6.066941,9.417048,;3.970657,7.320764,10.67087,;4.95068,8.300787,11.65089,;5.685371,9.035478,12.38558,;6.74014,10.09025,13.44035,;7.143804,10.49391,13.84402,;7.580485,10.93059,14.2807,;8.256265,11.60637,14.95648,;8.447602,11.79771,15.14782,;7.983014,11.33312,14.68323,;7.463431,10.81354,14.16364,;7.37704,10.72715,14.07725,;6.237444,9.587551,12.93766,;4.69587,8.045977,11.39608,;4.151903,7.50201,10.85212,;3.819442,7.169549,10.51966,;2.822216,6.172324,9.522431,;0.9585956,4.308703,7.65881,;-0.7692924,2.580815,5.930922,;-1.553635,1.796472,5.146579,;-3.237453,0.1126545,3.462761,;-4.560702,-1.210594,2.139513,;-5.144293,-1.794186,1.555922,;-5.889181,-2.539074,0.8110331,;-7.006773,-3.656666,-0.3065586,;-7.875526,-4.525419,-1.175312,;-9.725742,-6.375635,-3.025528,;-11.07814,-7.728029,-4.377922,;-11.02259,-7.672484,-4.322377,;-11.93138,-8.581273,-5.231166,;-12.08928,-8.739175,-5.389068,;-11.91153,-8.561418,-5.211311,;-12.38578,-9.035671,-5.685564,;-12.5863,-9.236191,-5.886084,;-12.32126,-8.971158,-5.62105,;-11.80748,-8.457372,-5.107265,;-10.6366,-7.286496,-3.936389,;-9.57477,-6.224663,-2.874556,;-9.200827,-5.85072,-2.500613,;-7.877398,-4.527291,-1.177184,;-6.584278,-3.234171,0.1159364,;-6.006206,-2.656099,0.6940076,;-4.914165,-1.564058,1.786049,;-2.903332,0.4467753,3.796882,;-1.449993,1.900114,5.250221,;-1.039323,2.310784,5.660891,;0.4458539,3.795961,7.146068,;2.026607,5.376714,8.726821,;2.716834,6.066941,9.417048,;3.970657,7.320764,10.67087,;4.95068,8.300787,11.65089,;5.685371,9.035478,12.38558,;6.74014,10.09025,13.44035,;7.143804,10.49391,13.84402,;7.580485,10.93059,14.2807,;8.256265,11.60637,14.95648,;8.447602,11.79771,15.14782,;7.983014,11.33312,14.68323,;7.463431,10.81354,14.16364,;7.37704,10.72715,14.07725,;6.237444,9.587551,12.93766,;4.69587,8.045977,11.39608,;4.151903,7.50201,10.85212,;3.819442,7.169549,10.51966,;2.822216,6.172324,9.522431,;0.9585956,4.308703,7.65881,;-0.7692924,2.580815,5.930922,;-1.553635,1.796472,5.146579,;-3.237453,0.1126545,3.462761,;-4.560702,-1.210594,2.139513,;-5.144293,-1.794186,1.555922,;-5.889181,-2.539074,0.8110331,;-7.006773,-3.656666,-0.3065586,;-7.875526,-4.525419,-1.175312,;-9.725742,-6.375635,-3.025528,;-11.07814,-7.728029,-4.377922,;-11.02259,-7.672484,-4.322377,;-11.93138,-8.581273,-5.231166,;-12.08928,-8.739175,-5.389068,;-11.91153,-8.561418,-5.211311,;-12.38578,-9.035671,-5.685564,;-12.5863,-9.236191,-5.886084,;-12.32126,-8.971158,-5.62105,;-11.80748,-8.457372,-5.107265,;-10.6366,-7.286496,-3.936389,;-9.57477,-6.224663,-2.874556,;-9.200827,-5.85072,-2.500613,;-7.877398,-4.527291,-1.177184,;-6.584278,-3.234171,0.1159364,;-6.006206,-2.656099,0.6940076,;-4.914165,-1.564058,1.786049,;-2.903332,0.4467753,3.796882,;-1.449993,1.900114,5.250221,;-1.039323,2.310784,5.660891,;0.4458539,3.795961,7.146068,;2.026607,5.376714,8.726821,;2.716834,6.066941,9.417048,;3.970657,7.320764,10.67087,;4.95068,8.300787,11.65089,;5.685371,9.035478,12.38558,;6.74014,10.09025,13.44035,;7.143804,10.49391,13.84402,;7.580485,10.93059,14.2807,;8.256265,11.60637,14.95648,;8.447602,11.79771,15.14782,;7.983014,11.33312,14.68323,;7.463431,10.81354,14.16364,;7.37704,10.72715,14.07725,;6.237444,9.587551,12.93766,;4.69587,8.045977,11.39608,;4.151903,7.50201,10.85212,;3.819442,7.169549,10.51966,;2.822216,6.172324,9.522431,;0.9585956,4.308703,7.65881,;-0.7692924,2.580815,5.930922,;-1.553635,1.796472,5.146579,;-3.237453,0.1126545,3.462761,;-4.560702,-1.210594,2.139513,;-5.144293,-1.794186,1.555922,;-5.889181,-2.539074,0.8110331,;-7.006773,-3.656666,-0.3065586,;-7.875526,-4.525419,-1.175312,;-9.725742,-6.375635,-3.025528,;-11.07814,-7.728029,-4.377922,;-11.02259,-7.672484,-4.322377,;-11.93138,-8.581273,-5.231166,;-12.08928,-8.739175,-5.389068,;-11.91153,-8.561418,-5.211311,;-12.38578,-9.035671,-5.685564,;-12.5863,-9.236191,-5.886084,;-12.32126,-8.971158,-5.62105,;-11.80748,-8.457372,-5.107265,;-10.6366,-7.286496,-3.936389,;-9.57477,-6.224663,-2.874556,;-9.200827,-5.85072,-2.500613,;-7.877398,-4.527291,-1.177184,;-6.584278,-3.234171,0.1159364,;-6.006206,-2.656099,0.6940076,;-4.914165,-1.564058,1.786049,;-2.903332,0.4467753,3.796882,;-1.449993,1.900114,5.250221,;-1.039323,2.310784,5.660891,;0.4458539,3.795961,7.146068,;2.026607,5.376714,8.726821,;2.716834,6.066941,9.417048,;3.970657,7.320764,10.67087,;4.95068,8.300787,11.65089,;5.685371,9.035478,12.38558,;6.74014,10.09025,13.44035,;7.143804,10.49391,13.84402,;7.580485,10.93059,14.2807,;8.256265,11.60637,14.95648,;8.447602,11.79771,15.14782,;7.983014,11.33312,14.68323,;7.463431,10.81354,14.16364,;7.37704,10.72715,14.07725,;6.237444,9.587551,12.93766,;4.69587,8.045977,11.39608,;4.151903,7.50201,10.85212,;3.819442,7.169549,10.51966,;2.822216,6.172324,9.522431,;0.9585956,4.308703,7.65881,;-0.7692924,2.580815,5.930922,;-1.553635,1.796472,5.146579,;-3.237453,0.1126545,3.462761,;-4.560702,-1.210594,2.139513,;-5.144293,-1.794186,1.555922,;-5.889181,-2.539074,0.8110331,;-7.006773,-3.656666,-0.3065586,;-7.875526,-4.525419,-1.175312,;-9.725742,-6.375635,-3.025528,;-11.07814,-7.728029,-4.377922,;-11.02259,-7.672484,-4.322377,;-11.93138,-8.581273,-5.231166,;-12.08928,-8.739175,-5.389068,;-11.91153,-8.561418,-5.211311,;-12.38578,-9.035671,-5.685564,;-12.5863,-9.236191,-5.886084,;-12.32126,-8.971158,-5.62105,;-11.80748,-8.457372,-5.107265,;-10.6366,-7.286496,-3.936389,;-9.57477,-6.224663,-2.874556,;-9.200827,-5.85072,-2.500613,;-7.877398,-4.527291,-1.177184,;-6.645563,-3.295456,0.05465104,; diff --git a/lib/maths/unittest/testfiles/CUnivariateTimeSeriesModel.6.2.state.xml b/lib/maths/unittest/testfiles/CUnivariateTimeSeriesModel.6.2.state.xml new file mode 100644 index 0000000000..24814283e9 --- /dev/null +++ b/lib/maths/unittest/testfiles/CUnivariateTimeSeriesModel.6.2.state.xml @@ -0,0 +1,986 @@ + + 1 + 0 + 1 + + 2 + + 1.000000 + 1:1 + 12787433637658295908:13498075975421300259 + + 1 + 100.9787:1.077826 + + + 1 + 26.47849:1.955571e-2 + + + 1 + 26.47849:1.514082 + + + 1 + 100.9787:1.602193 + + + + 1.000000 + 1:1 + 16293136932282329476:14199107704794469554 + + 1 + 100.1638:9.874476e-2 + + + 1 + 26.47557:1.489309e-2 + + + 1 + 26.47557:1.513646 + + + 1 + 100.1638:1.603113 + + + + + + 1.2e-2 + 599400 + 599400 + + + 0 + 1 + + 2074200 + + 0.012000 + 0 + + 0:0,0,0,0,0,0,0,0,0,0,0 + + 0:0,0:0,0 + + + + + 1 + 2 + + 1209600 + 0 + 3024000 + + 3600 + 1209600 + + 336 + 5.938603:1.926519 + 5.938603:4.642855 + 5.938603:7.053182 + 5.938603:8.745275 + 5.938603:10.6177 + 5.938603:10.8463 + 5.938603:11.61586 + 5.938603:10.49274 + 5.938603:8.914948 + 5.938603:7.699262 + 5.938603:4.494287 + 5.938603:1.008308 + 5.938603:8.578432e-1 + 5.938603:-1.868162 + 5.938603:-5.534989 + 5.938603:-7.109211 + 5.938603:-9.33582 + 5.938603:-9.984402 + 5.938603:-8.328712 + 5.938603:-8.370218 + 5.938603:-6.639109 + 5.938603:-5.043692 + 5.938603:-2.178768 + 5.938603:-6.711516e-1 + 5.948792:2.151453 + 5.948792:4.447317 + 5.948792:5.010306 + 5.948792:9.272408 + 5.948792:10.17816 + 5.948792:11.97338 + 5.948792:12.59517 + 5.948792:10.87819 + 5.948792:9.655095 + 5.948792:7.506748 + 5.948792:4.888617 + 5.948792:1.636357 + 5.948792:-1.469332 + 5.948792:-2.356663 + 5.948792:-4.629245 + 5.948792:-7.350151 + 5.948792:-8.362033 + 5.948792:-6.897859 + 5.948792:-8.374001 + 5.948792:-10.19356 + 5.948792:-7.243155 + 5.948792:-6.059096 + 5.948792:-3.997766 + 5.948792:-2.388081e-1 + 5.958999:2.650002 + 5.958999:4.099511 + 5.958999:7.502212 + 5.958999:7.528317 + 5.958999:10.09051 + 5.958999:12.03159 + 5.958999:10.60233 + 5.958999:10.50653 + 5.958999:9.450271 + 5.958999:6.596671 + 5.958999:3.477213 + 5.958999:1.533457 + 5.958999:-1.192633 + 5.958999:-1.451209 + 5.958999:-4.548854 + 5.958999:-6.749701 + 5.958999:-9.105805 + 5.958999:-8.914743 + 5.958999:-8.151886 + 5.958999:-9.101657 + 5.958999:-6.576452 + 5.958999:-4.335184 + 5.958999:-5.95366e-1 + 5.958999:-5.335736e-1 + 5.969223:3.474705 + 5.969223:4.087154 + 5.969223:8.438519 + 5.969223:8.361912 + 5.969223:9.990691 + 5.969223:13.43703 + 5.969223:11.40553 + 5.969223:10.75946 + 5.969223:8.52168 + 5.969223:7.387879 + 5.969223:6.382613 + 5.969223:3.025739 + 5.969223:-1.012645 + 5.969223:-3.384315 + 5.969223:-4.716345 + 5.969223:-7.510514 + 5.969223:-8.32226 + 5.969223:-8.470715 + 5.969223:-8.098967 + 5.969223:-6.46659 + 5.969223:-7.276778 + 5.969223:-5.847695 + 5.969223:-2.056774 + 5.969223:-3.909559e-1 + 5.979464:1.068067 + 5.979464:5.573017 + 5.979464:6.03499 + 5.979464:8.261374 + 5.979464:11.10119 + 5.979464:10.61676 + 5.979464:11.3202 + 5.979464:11.41288 + 5.979464:9.073486 + 5.979464:7.51921 + 5.979464:5.639875 + 5.979464:9.86387e-1 + 5.979464:-7.156203e-1 + 5.979464:-2.360445 + 5.979464:-3.558376 + 5.979464:-5.33939 + 5.979464:-7.835542 + 5.979464:-9.474154 + 5.979464:-8.633908 + 5.979464:-9.026299 + 5.979464:-7.20318 + 5.979464:-5.739718 + 5.979464:-2.3219 + 5.979464:-1.07765 + 5.989723:2.005866 + 5.989723:4.307028 + 5.989723:6.11962 + 5.989723:9.735407 + 5.989723:9.428868 + 5.989723:10.9435 + 5.989723:12.71083 + 5.989723:9.875322 + 5.989723:7.935678 + 5.989723:7.765068 + 5.989723:4.289617 + 5.989723:2.883468 + 5.989723:-4.433285e-2 + 5.989723:-1.726437 + 5.989723:-4.355645 + 5.989723:-7.142166 + 5.989723:-8.571531 + 5.989723:-8.090042 + 5.989723:-10.87016 + 5.989723:-9.035573 + 5.989723:-6.592824 + 5.989723:-4.308862 + 5.989723:-3.368046 + 5.989723:-8.368479e-1 + 6:2.131955 + 6:3.492808 + 6:7.110198 + 6:7.281185 + 6:9.949496 + 6:10.28035 + 6:12.33449 + 6:10.4962 + 6:9.022582 + 6:6.913308 + 6:4.690897 + 6:2.470929 + 6:2.413207 + 6:-2.400813 + 6:-5.549529 + 6:-7.2252 + 6:-7.925957 + 6:-9.315027 + 6:-8.24064 + 6:-8.986718 + 6:-5.816041 + 6:-5.028866 + 4:-3.203069 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + + + + 4;0;86400;0;604800 + + + + + 2 + 1 + + + 0 + 2073600 + + 168 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + + + + 0 + 7516800 + + 168 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + 0:0 + + + + + + 2 + 1 + + 2 + + 518400 + + 0 + 1.129151e-2;9.762858e-1:1.077715e-1;44.17169:3.077746e-1;45.13689:5.219688e-1;46.13633:7.101985e-1;37.36127:9.161772e-1;44.11304:1.15978;48.2309:1.435587;42.17098:1.683852;44.26584:1.978311;39.2194:2.270838;32.59685:2.583336;28.5027:2.907891;32.4317:3.217954;23.53792:3.572623;17.64527:4.105782;14.65692:4.664355;7.763844:5.53581;7.881569:6.979822;9.762858e-1 + 557.7757 + + 44:2097296:144:136 + 345600;12.90696 + + + + + 3 + 1 + + + + + 36 + 0 + + 1 + 0:1033.835:3300.155:5656.222:8003.972:10345.38:12781.76:15184.04:17673.55:20299.6:22936.12:25712.35:28289.29:30738.72:33323:35625.79:37740.81:40014.09:42228.91:44379.94:46644.14:48892.86:51192.68:53595.82:55758.11:57923.25:60470.66:63199.91:65949.18:68977.27:71630.45:73997.23:76279.22:78421.18:80745.29:83093.16:85200.01:86400 + 1.915139:1.915139:2.913442:5.166112:6.165317:7.833686:8.80989:10.16487:10.63173:11.5743:11.6775:10.82066:10.47961:8.341686:7.452612:6.832656:5.062077:2.69481:1.646859:-4.510463e-1:-1.576537:-2.373214:-3.761695:-5.158612:-7.552042:-7.618279:-8.758415:-8.537061:-9.168698:-8.978564:-8.017194:-6.360502:-5.819217:-4.170598:-2.853662:-1.898178:4.439902e-1:4.439902e-1 + 3.506185:3.506185:2.917839:4.83047:5.110361:3.388759:4.278941:3.467121:6.254621:3.284995:2.759995:3.052363:4.195419:3.073952:2.797014:3.380045:5.136346:4.231707:5.812066:3.939432:2.205449:3.255504:3.352469:4.466878:3.345408:1.982607:4.950573:5.033471:3.927552:4.568439:3.859872:1.875085:3.371665:4.182261:5.124764:3.550905:2.291908:2.291908 + + + 9094411231511854520:13258687326618484610 + + + 1.2e-2 + 0:2252.438:4500.487:6809.01:9165.376:11545.63:13996.27:16478.81:18932.17:21577.62:24346.31:27003.76:29558:32000.98:34342.52:36616.16:38830.06:41052.8:43302.39:45554.04:47839.58:50145.97:52427.55:54646.29:56851.38:59132.55:61753.04:64631.15:67467.36:70271.82:72757.16:75072.28:77375.71:79669.61:81916.2:84139.84:86400 + 1002.67:3300.119:5666.233:8025.586:10379.94:12807.64:15209.25:17679.39:20258.06:22903.85:25733.72:28333.94:30774.15:33317.79:35642.98:37751.95:39987.27:42245.48:44383.67:46610.85:48894.5:51217.5:53620.45:55766.04:57917.86:60435.06:63153.54:65961.02:68982.59:71645.55:74020.8:76264.73:78393.27:80681.01:83093.16:85200.01 + 3.85995:-6635.755 + 3.85995:31196.98 + + + 0:0:604800:86400:0 + + 0 + + 17.17756:1,5.302488e-1,3.756404e-1,2.026463,1.038958 + + + 17.1394:1,5.446092e-1,3.847068e-1,2.754142,1.405206 + + + 17.49278:1,5.447988e-1,3.851581e-1,4.856719,2.643582 + + + 17.77121:1,5.455059e-1,3.863419e-1,6.413713,3.525767 + + + 17.89388:1,5.481072e-1,3.893774e-1,7.714934,4.246958 + + + 18.27554:1,5.477664e-1,3.894637e-1,8.417198,4.575468 + + + 18.40219:1,5.498415e-1,3.920969e-1,10.22912,5.661761 + + + 18.12415:1,5.640053e-1,4.028711e-1,10.2962,5.632142 + + + 19.21993:1,5.580856e-1,3.982378e-1,11.40751,6.249243 + + + 21.09234:1,5.789787e-1,4.219534e-1,11.84804,6.959854 + + + 20.50768:1,5.853069e-1,4.286535e-1,10.65209,6.147687 + + + 18.87674:1,5.829183e-1,4.266177e-1,10.67645,6.291884 + + + 18.20615:1,5.805121e-1,4.255527e-1,8.313559,4.711836 + + + 17.64597:1,5.905066e-1,4.364882e-1,7.495087,4.388415 + + + 17.26705:1,5.981896e-1,4.45451e-1,6.588676,3.911763 + + + 15.93215:1,5.527382e-1,3.997929e-1,4.832919,2.66516 + + + 17.01286:1,5.970373e-1,4.5033e-1,2.817058,1.695637 + + + 17.16413:1,6.063976e-1,4.570898e-1,1.813437,1.274951 + + + 16.14662:1,5.751942e-1,4.203654e-1,2.458399e-1,3.918334e-1 + + + 17.3186:1,6.010724e-1,4.548982e-1,-1.311789,-7.403951e-1 + + + 17.43389:1,6.168044e-1,4.688024e-1,-2.469505,-1.52231 + + + 17.29456:1,6.233283e-1,4.763633e-1,-4.061237,-2.549126 + + + 16.9218:1,6.315611e-1,4.862154e-1,-5.462671,-3.472212 + + + 15.83477:1,5.848529e-1,4.359069e-1,-7.604956,-4.571466 + + + 17.26967:1,6.333009e-1,4.893083e-1,-7.623354,-4.76559 + + + 19.17475:1,6.25781e-1,4.782192e-1,-8.764505,-5.487319 + + + 21.55417:1,6.417059e-1,4.98942e-1,-8.638981,-5.572731 + + + 21.44988:1,6.619292e-1,5.199164e-1,-8.829242,-5.883996 + + + 21.33047:1,6.607155e-1,5.234539e-1,-8.954837,-6.015957 + + + 18.42859:1,6.47298e-1,5.070258e-1,-8.122644,-5.24137 + + + 17.44539:1,6.592287e-1,5.22605e-1,-6.123041,-3.897032 + + + 16.40051:1,6.116247e-1,4.683009e-1,-5.760595,-3.456032 + + + 17.36623:1,6.665299e-1,5.324088e-1,-4.109718,-2.690103 + + + 16.11354:1,6.533712e-1,5.135997e-1,-2.996593,-2.133454 + + + 12.98776:1,5.808824e-1,4.122114e-1,-1.898178,-1.157176 + + + 13.21294:1,5.272114e-1,3.61395e-1,4.439902e-1,3.737124e-1 + + 4.217019:2.481257:4.570785:4.371468:2.852921:4.136422:2.928734:5.591805:2.830932:3.416563:3.31386:3.913303:2.710952:2.609013:3.016661:4.773147:4.112317:5.159906:5.817554:3.455071:2.966635:3.526828:4.165422:3.76725:1.779262:4.284017:4.554064:4.378734:3.939807:3.489252:2.137831:3.129925:3.374792:4.93202:3.550905:2.291908 + 520200:522600:525000:527400:529800:532200:534600:537000:539400:542400:545400:547800:550200:552600:555000:556800:559200:561600:563400:565800:568200:570600:573000:574800:577200:579600:582600:585600:588600:591000:593400:595200:597600:599400:516000:517800 + + 6.24999999999999e-2,6.24999999999999e-2,-4.90552083956277e-1,-7.9470879817161e-1,78.3194622862958,129.059942640838,212.741543517541 + + + + + 1 + 556.811:4.155342;556.811:56.46648; + + + + + + + + 5e-4 + 2.789018e-3 + 501.5697 + 251.7848 + 1042.383 + 501.5697 + -5.63977 + 6.979822 + + + + 0:0 + + 0 + + + 2.5e-5 + 0 + 0,0 + 0,0 + 0 + 0,0,0 + + + 2.5e-5 + 0 + 0,0 + 0,0 + 0 + 0,0,0 + + + + diff --git a/lib/model/CAnomalyDetector.cc b/lib/model/CAnomalyDetector.cc index 5539dfcd75..0ff8b0443d 100644 --- a/lib/model/CAnomalyDetector.cc +++ b/lib/model/CAnomalyDetector.cc @@ -91,7 +91,7 @@ CAnomalyDetector::TModelPtr makeModel(const CAnomalyDetector::TModelFactoryCPtr // Increment this every time a change to the state is made that requires // existing state to be discarded -const std::string CAnomalyDetector::STATE_VERSION("34"); +const std::string CAnomalyDetector::STATE_VERSION("35"); const std::string CAnomalyDetector::COUNT_NAME("count"); const std::string CAnomalyDetector::TIME_NAME("time"); diff --git a/lib/model/CAnomalyDetectorModel.cc b/lib/model/CAnomalyDetectorModel.cc index bfb0f44f1f..c9182b7442 100644 --- a/lib/model/CAnomalyDetectorModel.cc +++ b/lib/model/CAnomalyDetectorModel.cc @@ -101,8 +101,8 @@ bool checkScheduledEvents(const SModelParams::TStrDetectionRulePrVec &scheduledE } CAnomalyDetectorModel::CAnomalyDetectorModel(const SModelParams ¶ms, - const TDataGathererPtr &dataGatherer, - const TFeatureInfluenceCalculatorCPtrPrVecVec &influenceCalculators) : + const TDataGathererPtr &dataGatherer, + const TFeatureInfluenceCalculatorCPtrPrVecVec &influenceCalculators) : m_Params(params), m_DataGatherer(dataGatherer), m_BucketCount(0.0), @@ -150,7 +150,7 @@ const std::string &CAnomalyDetectorModel::personName(std::size_t pid) const } const std::string &CAnomalyDetectorModel::personName(std::size_t pid, - const std::string &fallback) const + const std::string &fallback) const { return m_DataGatherer->personName(pid, fallback); } diff --git a/lib/model/CAnomalyDetectorModelConfig.cc b/lib/model/CAnomalyDetectorModelConfig.cc index 43ab72c52a..6ba511f7f8 100644 --- a/lib/model/CAnomalyDetectorModelConfig.cc +++ b/lib/model/CAnomalyDetectorModelConfig.cc @@ -112,9 +112,6 @@ const CAnomalyDetectorModelConfig::TDoubleDoublePr CAnomalyDetectorModelConfig:: CAnomalyDetectorModelConfig::TDoubleDoublePr(99.9, 90.0), CAnomalyDetectorModelConfig::TDoubleDoublePr(100.0, 100.0) }; -const double CAnomalyDetectorModelConfig::DEFAULT_LOG_NORMAL_PRIOR_OFFSET(1.0); -const double CAnomalyDetectorModelConfig::DEFAULT_GAMMA_PRIOR_OFFSET(0.1); -const double CAnomalyDetectorModelConfig::DEFAULT_POISSON_PRIOR_OFFSET(0.0); const std::size_t CAnomalyDetectorModelConfig::DEFAULT_RESAMPLING_MAX_SAMPLES(40u); const double CAnomalyDetectorModelConfig::DEFAULT_PRUNE_WINDOW_SCALE_MINIMUM(0.25); const double CAnomalyDetectorModelConfig::DEFAULT_PRUNE_WINDOW_SCALE_MAXIMUM(4.0); diff --git a/lib/model/CEventRateModel.cc b/lib/model/CEventRateModel.cc index 18c6a4764a..39f6dc6d32 100644 --- a/lib/model/CEventRateModel.cc +++ b/lib/model/CEventRateModel.cc @@ -799,7 +799,7 @@ void CEventRateModel::fill(model_t::EFeature feature, value_ += correction; this->currentBucketInterimCorrections().emplace( core::make_triple(feature, pid, params.s_Correlated[i]), - TDouble1Vec(1, correction[params.s_Variables[i][0]])); + TDouble1Vec{correction[params.s_Variables[i][0]]}); } } } diff --git a/lib/model/CEventRateModelFactory.cc b/lib/model/CEventRateModelFactory.cc index d1a7b8e813..123b0bc2ba 100644 --- a/lib/model/CEventRateModelFactory.cc +++ b/lib/model/CEventRateModelFactory.cc @@ -179,16 +179,16 @@ CEventRateModelFactory::TPriorPtr CEventRateModelFactory::defaultPrior(model_t:: maths_t::EDataType dataType = this->dataType(); maths::CGammaRateConjugate gammaPrior = - maths::CGammaRateConjugate::nonInformativePrior(dataType, params.s_GammaOffset, params.s_DecayRate); + maths::CGammaRateConjugate::nonInformativePrior(dataType, 0.0, params.s_DecayRate); maths::CLogNormalMeanPrecConjugate logNormalPrior = - maths::CLogNormalMeanPrecConjugate::nonInformativePrior(dataType, params.s_LogNormalOffset, params.s_DecayRate); + maths::CLogNormalMeanPrecConjugate::nonInformativePrior(dataType, 0.0, params.s_DecayRate); maths::CNormalMeanPrecConjugate normalPrior = maths::CNormalMeanPrecConjugate::nonInformativePrior(dataType, params.s_DecayRate); maths::CPoissonMeanConjugate poissonPrior = - maths::CPoissonMeanConjugate::nonInformativePrior(params.s_PoissonOffset, params.s_DecayRate); + maths::CPoissonMeanConjugate::nonInformativePrior(0.0, params.s_DecayRate); // Create the component priors. TPriorPtrVec priors; diff --git a/lib/model/CEventRatePopulationModel.cc b/lib/model/CEventRatePopulationModel.cc index 21b68b13d9..ce0dde44b4 100644 --- a/lib/model/CEventRatePopulationModel.cc +++ b/lib/model/CEventRatePopulationModel.cc @@ -453,7 +453,7 @@ void CEventRatePopulationModel::sample(core_t::TTime startTime, { std::size_t cid = CDataGatherer::extractAttributeId(data_); uint64_t count = CDataGatherer::extractData(data_).s_Count; - fuzzy[cid].add(TDouble2Vec{static_cast(count)}); + fuzzy[cid].add({static_cast(count)}); } for (auto &&fuzzy_ : fuzzy) { @@ -496,7 +496,7 @@ void CEventRatePopulationModel::sample(core_t::TTime startTime, SValuesAndWeights &attribute = attributes[cid]; std::size_t duplicate = data.size() >= this->params().s_MinimumToDeduplicate ? - fuzzy[cid].duplicate(sampleTime, TDouble2Vec{value}) : + fuzzy[cid].duplicate(sampleTime, {value}) : attribute.s_Values.size(); if (duplicate < attribute.s_Values.size()) @@ -508,11 +508,9 @@ void CEventRatePopulationModel::sample(core_t::TTime startTime, { attribute.s_Values.emplace_back(sampleTime, TDouble2Vec{value}, pid); attribute.s_Weights.emplace_back( - TDouble2Vec4Vec{TDouble2Vec{ this->sampleRateWeight(pid, cid) - * this->learnRate(feature)}, - model->winsorisationWeight(1.0, // no derate - sampleTime, - TDouble2Vec{value})}); + TDouble2Vec4Vec{{ this->sampleRateWeight(pid, cid) + * this->learnRate(feature)}, + model->winsorisationWeight(1.0, sampleTime, {value})}); } } diff --git a/lib/model/CEventRatePopulationModelFactory.cc b/lib/model/CEventRatePopulationModelFactory.cc index e760f2c106..635779b668 100644 --- a/lib/model/CEventRatePopulationModelFactory.cc +++ b/lib/model/CEventRatePopulationModelFactory.cc @@ -179,16 +179,16 @@ CEventRatePopulationModelFactory::TPriorPtr maths_t::EDataType dataType = this->dataType(); maths::CGammaRateConjugate gammaPrior = - maths::CGammaRateConjugate::nonInformativePrior(dataType, params.s_GammaOffset, params.s_DecayRate); + maths::CGammaRateConjugate::nonInformativePrior(dataType, 0.0, params.s_DecayRate); maths::CLogNormalMeanPrecConjugate logNormalPrior = - maths::CLogNormalMeanPrecConjugate::nonInformativePrior(dataType, params.s_LogNormalOffset, params.s_DecayRate); + maths::CLogNormalMeanPrecConjugate::nonInformativePrior(dataType, 0.0, params.s_DecayRate); maths::CNormalMeanPrecConjugate normalPrior = maths::CNormalMeanPrecConjugate::nonInformativePrior(dataType, params.s_DecayRate); maths::CPoissonMeanConjugate poissonPrior = - maths::CPoissonMeanConjugate::nonInformativePrior(params.s_PoissonOffset, params.s_DecayRate); + maths::CPoissonMeanConjugate::nonInformativePrior(0.0, params.s_DecayRate); // Create the component priors. TPriorPtrVec priors; diff --git a/lib/model/CMetricModelFactory.cc b/lib/model/CMetricModelFactory.cc index 4cc86088a7..6b174075a5 100644 --- a/lib/model/CMetricModelFactory.cc +++ b/lib/model/CMetricModelFactory.cc @@ -175,10 +175,10 @@ CMetricModelFactory::TPriorPtr maths_t::EDataType dataType = this->dataType(); maths::CGammaRateConjugate gammaPrior = - maths::CGammaRateConjugate::nonInformativePrior(dataType, params.s_GammaOffset, params.s_DecayRate); + maths::CGammaRateConjugate::nonInformativePrior(dataType, 0.0, params.s_DecayRate); maths::CLogNormalMeanPrecConjugate logNormalPrior = - maths::CLogNormalMeanPrecConjugate::nonInformativePrior(dataType, params.s_LogNormalOffset, params.s_DecayRate); + maths::CLogNormalMeanPrecConjugate::nonInformativePrior(dataType, 0.0, params.s_DecayRate); maths::CNormalMeanPrecConjugate normalPrior = maths::CNormalMeanPrecConjugate::nonInformativePrior(dataType, params.s_DecayRate); diff --git a/lib/model/CMetricPopulationModel.cc b/lib/model/CMetricPopulationModel.cc index 17e479e5ac..a970905ee2 100644 --- a/lib/model/CMetricPopulationModel.cc +++ b/lib/model/CMetricPopulationModel.cc @@ -382,7 +382,7 @@ void CMetricPopulationModel::sample(core_t::TTime startTime, const TTimeVec &attributeLastBucketTimes = this->attributeLastBucketTimes(); - for (auto &&featureData_ : featureData) + for (auto &featureData_ : featureData) { model_t::EFeature feature = featureData_.first; std::size_t dimension = model_t::dimension(feature); @@ -406,7 +406,7 @@ void CMetricPopulationModel::sample(core_t::TTime startTime, fuzzy[cid].add(TDouble2Vec(sample.value(dimension))); } } - for (auto &&fuzzy_ : fuzzy) + for (auto &fuzzy_ : fuzzy) { fuzzy_.second.computeEpsilons(bucketLength, this->params().s_MinimumToDeduplicate); @@ -441,7 +441,6 @@ void CMetricPopulationModel::sample(core_t::TTime startTime, // multiple times (once per person) attributeLastBucketTimesMap[cid] = sampleTime; } - continue; } @@ -460,7 +459,8 @@ void CMetricPopulationModel::sample(core_t::TTime startTime, } std::size_t n = std::count_if(samples.begin(), samples.end(), - [cutoff](const CSample &sample) { return sample.time() >= cutoff; }); + [cutoff](const CSample &sample) + { return sample.time() >= cutoff; }); double updatesPerBucket = this->params().s_MaximumUpdatesPerBucket; double countWeight = this->sampleRateWeight(pid, cid) * this->learnRate(feature) @@ -494,13 +494,13 @@ void CMetricPopulationModel::sample(core_t::TTime startTime, { attribute.s_Values.emplace_back(sample.time(), value, pid); attribute.s_TrendWeights.push_back( - TDouble2Vec4Vec{TDouble2Vec(dimension, countWeight / vs), - model->winsorisationWeight(1.0, sample.time(), value), - TDouble2Vec(dimension, vs)}); + {TDouble2Vec(dimension, countWeight / vs), + model->winsorisationWeight(1.0, sample.time(), value), + TDouble2Vec(dimension, vs)}); attribute.s_PriorWeights.push_back( - TDouble2Vec4Vec{TDouble2Vec(dimension, countWeight), - model->winsorisationWeight(1.0, sample.time(), value), - TDouble2Vec(dimension, vs)}); + {TDouble2Vec(dimension, countWeight), + model->winsorisationWeight(1.0, sample.time(), value), + TDouble2Vec(dimension, vs)}); } } } diff --git a/lib/model/CMetricPopulationModelFactory.cc b/lib/model/CMetricPopulationModelFactory.cc index 72ba86605a..190f57e702 100644 --- a/lib/model/CMetricPopulationModelFactory.cc +++ b/lib/model/CMetricPopulationModelFactory.cc @@ -174,10 +174,10 @@ CMetricPopulationModelFactory::TPriorPtr maths_t::EDataType dataType = this->dataType(); maths::CGammaRateConjugate gammaPrior = - maths::CGammaRateConjugate::nonInformativePrior(dataType, params.s_GammaOffset, params.s_DecayRate); + maths::CGammaRateConjugate::nonInformativePrior(dataType, 0.0, params.s_DecayRate); maths::CLogNormalMeanPrecConjugate logNormalPrior = - maths::CLogNormalMeanPrecConjugate::nonInformativePrior(dataType, params.s_LogNormalOffset, params.s_DecayRate); + maths::CLogNormalMeanPrecConjugate::nonInformativePrior(dataType, 0.0, params.s_DecayRate); maths::CNormalMeanPrecConjugate normalPrior = maths::CNormalMeanPrecConjugate::nonInformativePrior(dataType, params.s_DecayRate); diff --git a/lib/model/CModelDetailsView.cc b/lib/model/CModelDetailsView.cc index fbf50d59e1..292ecc2233 100644 --- a/lib/model/CModelDetailsView.cc +++ b/lib/model/CModelDetailsView.cc @@ -121,7 +121,7 @@ void CModelDetailsView::modelPlotForByFieldId(core_t::TTime time, TDouble2Vec supportLower(support.first); TDouble2Vec supportUpper(support.second); - TDouble2Vec3Vec interval(model->confidenceInterval(time, WEIGHT_STYLES, weights, boundsPercentile)); + TDouble2Vec3Vec interval(model->confidenceInterval(time, boundsPercentile, WEIGHT_STYLES, weights)); if (interval.size() == 3) { diff --git a/lib/model/CModelParams.cc b/lib/model/CModelParams.cc index 6d11c9e0f4..8d06c1a4d7 100644 --- a/lib/model/CModelParams.cc +++ b/lib/model/CModelParams.cc @@ -40,9 +40,6 @@ const core_t::TTime SAMPLING_AGE_CUTOFF_DEFAULT(2 * core::constants::DAY); SModelParams::SModelParams(core_t::TTime bucketLength) : s_BucketLength(bucketLength), s_MultivariateComponentDelimiter(CAnomalyDetectorModelConfig::DEFAULT_MULTIVARIATE_COMPONENT_DELIMITER), - s_LogNormalOffset(CAnomalyDetectorModelConfig::DEFAULT_LOG_NORMAL_PRIOR_OFFSET), - s_GammaOffset(CAnomalyDetectorModelConfig::DEFAULT_GAMMA_PRIOR_OFFSET), - s_PoissonOffset(CAnomalyDetectorModelConfig::DEFAULT_POISSON_PRIOR_OFFSET), s_LearnRate(1.0), s_DecayRate(0.0), s_InitialDecayRateMultiplier(CAnomalyDetectorModelConfig::DEFAULT_INITIAL_DECAY_RATE_MULTIPLIER), diff --git a/lib/model/unittest/CEventRateAnomalyDetectorTest.cc b/lib/model/unittest/CEventRateAnomalyDetectorTest.cc index f85445b815..d13af1df3b 100644 --- a/lib/model/unittest/CEventRateAnomalyDetectorTest.cc +++ b/lib/model/unittest/CEventRateAnomalyDetectorTest.cc @@ -229,7 +229,7 @@ void importData(ml::core_t::TTime firstTime, void CEventRateAnomalyDetectorTest::testAnomalies(void) { - static const size_t EXPECTED_ANOMALOUS_HOURS(13); + static const size_t EXPECTED_ANOMALOUS_HOURS(12); static const ml::core_t::TTime FIRST_TIME(1346713620); static const ml::core_t::TTime LAST_TIME(1347317974); static const ml::core_t::TTime BUCKET_SIZE(1800); @@ -240,16 +240,16 @@ void CEventRateAnomalyDetectorTest::testAnomalies(void) ml::model::CLimits limits; ml::model::CSearchKey key(1, // identifier - ml::model::function_t::E_IndividualRareCount, - false, - ml::model_t::E_XF_None, - EMPTY_STRING, "status"); + ml::model::function_t::E_IndividualRareCount, + false, + ml::model_t::E_XF_None, + EMPTY_STRING, "status"); ml::model::CAnomalyDetector detector(1, // identifier - limits, - modelConfig, - EMPTY_STRING, - FIRST_TIME, - modelConfig.factory(key)); + limits, + modelConfig, + EMPTY_STRING, + FIRST_TIME, + modelConfig.factory(key)); CResultWriter writer(modelConfig, limits); TStrVec files; files.push_back("testfiles/status200.txt"); @@ -263,13 +263,11 @@ void CEventRateAnomalyDetectorTest::testAnomalies(void) const TTimeDoubleMap &anomalyScores = writer.anomalyScores(); TTimeVec peaks; - for (TTimeDoubleMapCItr itr = anomalyScores.begin(); - itr != anomalyScores.end(); - ++itr) + for (const auto &score : anomalyScores) { - if (itr->second > HIGH_ANOMALY_SCORE) + if (score.second > HIGH_ANOMALY_SCORE) { - peaks.push_back(itr->first); + peaks.push_back(score.first); } } @@ -331,15 +329,15 @@ void CEventRateAnomalyDetectorTest::testPersist(void) inserter.toXml(origXml); } - LOG_DEBUG("Event rate detector XML representation:\n" << origXml); + LOG_TRACE("Event rate detector XML representation:\n" << origXml); // Restore the XML into a new detector ml::model::CAnomalyDetector restoredDetector(1, // identifier - limits, - modelConfig, - "", - 0, - modelConfig.factory(key)); + limits, + modelConfig, + "", + 0, + modelConfig.factory(key)); { ml::core::CRapidXmlParser parser; CPPUNIT_ASSERT(parser.parseStringIgnoreCdata(origXml)); diff --git a/lib/model/unittest/CEventRateModelTest.cc b/lib/model/unittest/CEventRateModelTest.cc index 875e59a28e..b79316cc87 100644 --- a/lib/model/unittest/CEventRateModelTest.cc +++ b/lib/model/unittest/CEventRateModelTest.cc @@ -85,15 +85,14 @@ typedef maths::CBasicStatistics::SSampleMean::TAccumulator TMeanAccumula const std::string EMPTY_STRING; -TUInt64Vec rawEventCounts() +TUInt64Vec rawEventCounts(std::size_t copies = 1) { - uint64_t counts[] = - { - 54, 67, 39, 58, 46, 50, 42, 48, 53, 51, 50, 57, 53, 49 - }; - - TUInt64Vec result(boost::begin(counts), boost::end(counts)); - + uint64_t counts[] = { 54, 67, 39, 58, 46, 50, 42, 48, 53, 51, 50, 57, 53, 49 }; + TUInt64Vec result; + for (std::size_t i = 0u; i < copies; ++i) + { + result.insert(result.end(), boost::begin(counts), boost::end(counts)); + } return result; } @@ -381,7 +380,7 @@ void CEventRateModelTest::testOnlineCountSample(void) // Generate some events. TTimeVec eventTimes; - TUInt64Vec expectedEventCounts = rawEventCounts(); + TUInt64Vec expectedEventCounts(rawEventCounts()); generateEvents(startTime, bucketLength, expectedEventCounts, eventTimes); core_t::TTime endTime = (eventTimes.back() / bucketLength + 1) * bucketLength; LOG_DEBUG("startTime = " << startTime @@ -439,7 +438,7 @@ void CEventRateModelTest::testOnlineCountSample(void) LOG_TRACE("origXml = " << origXml); LOG_DEBUG("origXml size = " << origXml.size()); - CPPUNIT_ASSERT(origXml.size() < 36500); + CPPUNIT_ASSERT(origXml.size() < 41000); // Restore the XML into a new filter core::CRapidXmlParser parser; @@ -604,9 +603,9 @@ void CEventRateModelTest::testOnlineRare(void) inserter.toXml(origXml); } - LOG_DEBUG("origXml = " << origXml); + LOG_TRACE("origXml = " << origXml); LOG_DEBUG("size = " << origXml.size()); - CPPUNIT_ASSERT(origXml.size() < 15700); + CPPUNIT_ASSERT(origXml.size() < 21000); // Restore the XML into a new filter core::CRapidXmlParser parser; @@ -636,7 +635,7 @@ void CEventRateModelTest::testOnlineProbabilityCalculation(void) const core_t::TTime startTime = 1346968800; const core_t::TTime bucketLength = 3600; - const std::size_t anomalousBucket = 12u; + const std::size_t anomalousBucket = 25u; SModelParams params(bucketLength); params.s_DecayRate = 0.001; @@ -647,12 +646,11 @@ void CEventRateModelTest::testOnlineProbabilityCalculation(void) makeModel(factory, features, m_ResourceMonitor, startTime, bucketLength, gatherer, model_, 1); CEventRateModel *model = dynamic_cast(model_.get()); - TMinAccumulator minProbabilities(2u); // Generate some events. TTimeVec eventTimes; - TUInt64Vec expectedEventCounts = rawEventCounts(); + TUInt64Vec expectedEventCounts = rawEventCounts(2); expectedEventCounts[anomalousBucket] *= 3; generateEvents(startTime, bucketLength, expectedEventCounts, eventTimes); core_t::TTime endTime = (eventTimes.back() / bucketLength + 1) * bucketLength; @@ -938,9 +936,9 @@ void CEventRateModelTest::testOnlineCorrelatedNoTrend(void) inserter.toXml(origXml); } - //LOG_DEBUG("origXml = " << origXml); + LOG_TRACE("origXml = " << origXml); LOG_DEBUG("size = " << origXml.size()); - CPPUNIT_ASSERT(origXml.size() < 134000); + CPPUNIT_ASSERT(origXml.size() < 151000); core::CRapidXmlParser parser; CPPUNIT_ASSERT(parser.parseStringIgnoreCdata(origXml)); @@ -2567,8 +2565,8 @@ void CEventRateModelTest::testInterimCorrections(void) CPPUNIT_ASSERT(annotatedProbability2.s_Probability < 0.05); CPPUNIT_ASSERT(annotatedProbability3.s_Probability < 0.05); CPPUNIT_ASSERT(p1Baseline[0] > 44.0 && p1Baseline[0] < 46.0); - CPPUNIT_ASSERT(p2Baseline[0] > 44.0 && p2Baseline[0] < 46.0); - CPPUNIT_ASSERT(p3Baseline[0] > 59.0 && p3Baseline[0] < 61.0); + CPPUNIT_ASSERT(p2Baseline[0] > 43.0 && p2Baseline[0] < 47.0); + CPPUNIT_ASSERT(p3Baseline[0] > 57.0 && p3Baseline[0] < 62.0); for (std::size_t i = 0; i < 25; ++i) { @@ -2608,9 +2606,9 @@ void CEventRateModelTest::testInterimCorrections(void) CPPUNIT_ASSERT(annotatedProbability1.s_Probability > 0.75); CPPUNIT_ASSERT(annotatedProbability2.s_Probability > 0.9); CPPUNIT_ASSERT(annotatedProbability3.s_Probability < 0.05); - CPPUNIT_ASSERT(p1Baseline[0] > 58.0 && p1Baseline[0] < 61.0); - CPPUNIT_ASSERT(p2Baseline[0] > 58.0 && p2Baseline[0] < 61.0); - CPPUNIT_ASSERT(p3Baseline[0] > 58.0 && p3Baseline[0] < 61.0); + CPPUNIT_ASSERT(p1Baseline[0] > 58.0 && p1Baseline[0] < 62.0); + CPPUNIT_ASSERT(p2Baseline[0] > 58.0 && p2Baseline[0] < 62.0); + CPPUNIT_ASSERT(p3Baseline[0] > 58.0 && p3Baseline[0] < 62.0); } void CEventRateModelTest::testInterimCorrectionsWithCorrelations(void) @@ -2949,8 +2947,8 @@ void CEventRateModelTest::testDecayRateControl(void) } LOG_DEBUG("mean = " << maths::CBasicStatistics::mean(meanPredictionError)); LOG_DEBUG("reference = " << maths::CBasicStatistics::mean(meanReferencePredictionError)); - CPPUNIT_ASSERT( maths::CBasicStatistics::mean(meanPredictionError) - < 0.9 * maths::CBasicStatistics::mean(meanReferencePredictionError)); + CPPUNIT_ASSERT( maths::CBasicStatistics::mean(meanPredictionError) + < 0.94 * maths::CBasicStatistics::mean(meanReferencePredictionError)); } LOG_DEBUG("*** Test unmodelled cyclic component ***"); @@ -3011,7 +3009,7 @@ void CEventRateModelTest::testDecayRateControl(void) LOG_DEBUG("mean = " << maths::CBasicStatistics::mean(meanPredictionError)); LOG_DEBUG("reference = " << maths::CBasicStatistics::mean(meanReferencePredictionError)); CPPUNIT_ASSERT( maths::CBasicStatistics::mean(meanPredictionError) - < 0.5 * maths::CBasicStatistics::mean(meanReferencePredictionError)); + < 0.7 * maths::CBasicStatistics::mean(meanReferencePredictionError)); } } diff --git a/lib/model/unittest/CEventRatePopulationModelTest.cc b/lib/model/unittest/CEventRatePopulationModelTest.cc index bfe8737625..a2f14fce9b 100644 --- a/lib/model/unittest/CEventRatePopulationModelTest.cc +++ b/lib/model/unittest/CEventRatePopulationModelTest.cc @@ -405,9 +405,8 @@ void CEventRatePopulationModelTest::testFeatures(void) SModelParams params(bucketLength); params.s_InitialDecayRateMultiplier = 1.0; CEventRatePopulationModelFactory factory(params); - CModelFactory::TFeatureVec features; - features.push_back(model_t::E_PopulationCountByBucketPersonAndAttribute); - features.push_back(model_t::E_PopulationUniquePersonCountByAttribute); + CModelFactory::TFeatureVec features{model_t::E_PopulationCountByBucketPersonAndAttribute, + model_t::E_PopulationUniquePersonCountByAttribute}; factory.features(features); CModelFactory::SGathererInitializationData gathererInitData(startTime); CModelFactory::TDataGathererPtr gatherer(dynamic_cast(factory.makeDataGatherer(gathererInitData))); @@ -443,7 +442,7 @@ void CEventRatePopulationModelTest::testFeatures(void) numberAttributes = std::max(numberAttributes, cid + 1); numberPeople = std::max(numberPeople, pid + 1); attributePeople[cid].insert(pid); - expectedNonZeroCounts[std::make_pair(pid, cid)] = count.second; + expectedNonZeroCounts[{pid, cid}] = count.second; } TSizeDouble2VecVecDouble2Vec4VecVecPrMap populationSamples; @@ -462,12 +461,12 @@ void CEventRatePopulationModelTest::testFeatures(void) } TDoubleVec sample(1, count); - TDouble2Vec4Vec weight{TDouble2Vec{model->sampleRateWeight(pid, cid)}, + TDouble2Vec4Vec weight{{model->sampleRateWeight(pid, cid)}, model_->winsorisationWeight(1.0, time, sample)}; - populationSamples[cid].first.push_back(TDouble2Vec{sample[0]}); + populationSamples[cid].first.push_back({sample[0]}); populationSamples[cid].second.push_back(weight); } - for (auto &&samples_ : populationSamples) + for (auto &samples_ : populationSamples) { std::size_t cid = samples_.first; TDouble2Vec4VecVec &weights = samples_.second.second; diff --git a/lib/model/unittest/CMetricAnomalyDetectorTest.cc b/lib/model/unittest/CMetricAnomalyDetectorTest.cc index b0a8ee2ff4..b587252bc5 100644 --- a/lib/model/unittest/CMetricAnomalyDetectorTest.cc +++ b/lib/model/unittest/CMetricAnomalyDetectorTest.cc @@ -443,7 +443,7 @@ void CMetricAnomalyDetectorTest::testPersist(void) inserter.toXml(origXml); } - LOG_DEBUG("Event rate detector XML representation:\n" << origXml); + LOG_TRACE("Event rate detector XML representation:\n" << origXml); // Restore the XML into a new detector model::CAnomalyDetector restoredDetector(1, // identifier diff --git a/lib/model/unittest/CMetricModelTest.cc b/lib/model/unittest/CMetricModelTest.cc index 5a585b2502..7b448b88b2 100644 --- a/lib/model/unittest/CMetricModelTest.cc +++ b/lib/model/unittest/CMetricModelTest.cc @@ -563,92 +563,6 @@ void CMetricModelTest::testSample(void) CPPUNIT_ASSERT(maths::CBasicStatistics::mean(baselineMeanError) < 0.25); } } - - // Check we correctly handle negative values. - { - CDataGatherer::TFeatureVec features(1, model_t::E_IndividualMeanByPerson); - CModelFactory::TDataGathererPtr gatherer; - CAnomalyDetectorModel::TModelPtr model_; - unsigned int sampleCount = 1; - makeModel(factory, features, startTime, bucketLength, gatherer, model_, &sampleCount); - CMetricModel &model = static_cast(*model_.get()); - CPPUNIT_ASSERT_EQUAL(std::size_t(0), addPerson("p", gatherer, m_ResourceMonitor)); - - TTimeDoublePr data[] = - { - TTimeDoublePr(45, 1.0), - TTimeDoublePr(46, 0.3), - TTimeDoublePr(48, 0.8), - TTimeDoublePr(49, 0.5), - TTimeDoublePr(50, 1.2), - TTimeDoublePr(51, 0.1), - TTimeDoublePr(52, 0.2), - TTimeDoublePr(53, 0.5), - TTimeDoublePr(54, 1.3), - TTimeDoublePr(55, 0.9), - TTimeDoublePr(56, 1.6), - TTimeDoublePr(58, 0.7), - TTimeDoublePr(59, 0.9), - TTimeDoublePr(60, 0.8), - TTimeDoublePr(61, 1.4), - TTimeDoublePr(62, 1.2), - TTimeDoublePr(63, 0.3), - TTimeDoublePr(64, 0.9), - TTimeDoublePr(65, -1.19), - TTimeDoublePr(66, 0.4) - }; - - core_t::TTime time = startTime; - - for (std::size_t i = 0u; i < boost::size(data); ++i) - { - if (data[i].first >= time + bucketLength) - { - LOG_DEBUG("Sampling [" << time << ", " << time + bucketLength << ")"); - model.sample(time, time + bucketLength, m_ResourceMonitor); - time += bucketLength; - } - - LOG_DEBUG("Adding " << data[i].second << " at " << data[i].first); - addArrival(*gatherer, m_ResourceMonitor, data[i].first, "p", data[i].second); - } - - LOG_DEBUG("Sampling [" << time << ", " << time + bucketLength << ")"); - model.sample(time, time + bucketLength, m_ResourceMonitor); - - model::SModelParams expectedParams(bucketLength); - params.s_GammaOffset = 2.8; - params.s_LogNormalOffset = 2.8; - params.s_DecayRate = 0.0; - params.s_MinimumModeFraction = 0.01; - params.s_MinimumModeCount = 0.8; - TPriorPtr expectedPrior(factory.defaultPrior(model_t::E_IndividualMeanByPerson, expectedParams)); - for (std::size_t i = 0u; i < boost::size(data); ++i) - { - expectedPrior->addSamples(COUNT_WEIGHT, TDouble1Vec(1, data[i].second), UNIT_WEIGHT); - } - const maths::CPrior &prior = dynamic_cast( - model.details()->model(model_t::E_IndividualMeanByPerson, 0))->prior(); - - double confidenceIntervals[] = { 25.0, 50.0, 75.0, 99.0 }; - for (std::size_t i = 0; i < boost::size(confidenceIntervals); ++i) - { - TDoubleDoublePr expectedInterval = - expectedPrior->marginalLikelihoodConfidenceInterval(confidenceIntervals[i]); - TDoubleDoublePr interval = - prior.marginalLikelihoodConfidenceInterval(confidenceIntervals[i]); - - LOG_DEBUG("Testing " << confidenceIntervals[i] << "% interval"); - LOG_DEBUG("expected interval = " << core::CContainerPrinter::print(expectedInterval)); - LOG_DEBUG("Interval = " << core::CContainerPrinter::print(interval)); - CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedInterval.first, - interval.first, - 0.07 * (expectedInterval.second - expectedInterval.first)); - CPPUNIT_ASSERT_DOUBLES_EQUAL(expectedInterval.second, - interval.second, - 0.07 * (expectedInterval.second - expectedInterval.first)); - } - } } void CMetricModelTest::testMultivariateSample(void) @@ -1517,7 +1431,7 @@ void CMetricModelTest::testInfluence(void) { }, { }, { }, - { core::make_triple(std::string{"i1"}, 0.8, 0.9), + { core::make_triple(std::string{"i1"}, 0.9, 1.0), core::make_triple(std::string{"i3"}, 0.9, 1.0) }, { core::make_triple(std::string{"i1"}, 0.9, 1.0) }, { core::make_triple(std::string{"i5"}, 0.9, 1.0) }, @@ -2528,8 +2442,8 @@ void CMetricModelTest::testDecayRateControl(void) } LOG_DEBUG("mean = " << maths::CBasicStatistics::mean(meanPredictionError)); LOG_DEBUG("reference = " << maths::CBasicStatistics::mean(meanReferencePredictionError)); - CPPUNIT_ASSERT( maths::CBasicStatistics::mean(meanPredictionError) - < 0.9 * maths::CBasicStatistics::mean(meanReferencePredictionError)); + CPPUNIT_ASSERT( maths::CBasicStatistics::mean(meanPredictionError) + < 0.94 * maths::CBasicStatistics::mean(meanReferencePredictionError)); } LOG_DEBUG("*** Test unmodelled cyclic component ***"); @@ -2587,7 +2501,7 @@ void CMetricModelTest::testDecayRateControl(void) LOG_DEBUG("mean = " << maths::CBasicStatistics::mean(meanPredictionError)); LOG_DEBUG("reference = " << maths::CBasicStatistics::mean(meanReferencePredictionError)); CPPUNIT_ASSERT( maths::CBasicStatistics::mean(meanPredictionError) - < 0.5 * maths::CBasicStatistics::mean(meanReferencePredictionError)); + < 0.7 * maths::CBasicStatistics::mean(meanReferencePredictionError)); } } diff --git a/lib/model/unittest/CMetricPopulationModelTest.cc b/lib/model/unittest/CMetricPopulationModelTest.cc index e07f4acbaa..b6280a5167 100644 --- a/lib/model/unittest/CMetricPopulationModelTest.cc +++ b/lib/model/unittest/CMetricPopulationModelTest.cc @@ -530,10 +530,9 @@ void CMetricPopulationModelTest::testMinMaxAndMean(void) params.s_InitialDecayRateMultiplier = 1.0; params.s_MaximumUpdatesPerBucket = 0.0; CMetricPopulationModelFactory factory(params); - CModelFactory::TFeatureVec features; - features.push_back(model_t::E_PopulationMeanByPersonAndAttribute); - features.push_back(model_t::E_PopulationMinByPersonAndAttribute); - features.push_back(model_t::E_PopulationMaxByPersonAndAttribute); + CModelFactory::TFeatureVec features{model_t::E_PopulationMeanByPersonAndAttribute, + model_t::E_PopulationMinByPersonAndAttribute, + model_t::E_PopulationMaxByPersonAndAttribute}; factory.features(features); CModelFactory::SGathererInitializationData gathererInitData(startTime); CModelFactory::TDataGathererPtr gatherer(dynamic_cast(factory.makeDataGatherer(gathererInitData))); From b638ac78804af5be41962f90b3de526f92e82ab0 Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Tue, 27 Feb 2018 11:35:52 -0800 Subject: [PATCH 2/4] Revert model version bump --- lib/model/CAnomalyDetector.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/model/CAnomalyDetector.cc b/lib/model/CAnomalyDetector.cc index 0ff8b0443d..5539dfcd75 100644 --- a/lib/model/CAnomalyDetector.cc +++ b/lib/model/CAnomalyDetector.cc @@ -91,7 +91,7 @@ CAnomalyDetector::TModelPtr makeModel(const CAnomalyDetector::TModelFactoryCPtr // Increment this every time a change to the state is made that requires // existing state to be discarded -const std::string CAnomalyDetector::STATE_VERSION("35"); +const std::string CAnomalyDetector::STATE_VERSION("34"); const std::string CAnomalyDetector::COUNT_NAME("count"); const std::string CAnomalyDetector::TIME_NAME("time"); From 307c0dc547619df60e5e9d27fb8526ead4fac0f5 Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Tue, 27 Feb 2018 12:00:15 -0800 Subject: [PATCH 3/4] Fix time of day/month dependent test failure --- lib/maths/unittest/CCalendarFeatureTest.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/maths/unittest/CCalendarFeatureTest.cc b/lib/maths/unittest/CCalendarFeatureTest.cc index 2afeccb605..7b714423e9 100644 --- a/lib/maths/unittest/CCalendarFeatureTest.cc +++ b/lib/maths/unittest/CCalendarFeatureTest.cc @@ -111,6 +111,9 @@ void CCalendarFeatureTest::testComparison(void) features.insert(features.end(), fi.begin(), fi.end()); } + std::sort(features.begin(), features.end()); + features.erase(std::unique(features.begin(), features.end()), features.end()); + for (std::size_t i = 0u; i < features.size(); ++i) { CPPUNIT_ASSERT(features[i] == features[i]); From 582cb1c0e228969f28d87e95472b5fb6fd800b1c Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Tue, 27 Feb 2018 13:39:00 -0800 Subject: [PATCH 4/4] Be a bit more careful with forecast variance calculation --- lib/maths/CTrendComponent.cc | 40 +++++++++++++---------- lib/maths/unittest/CForecastTest.cc | 10 +++--- lib/maths/unittest/CTrendComponentTest.cc | 2 +- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/lib/maths/CTrendComponent.cc b/lib/maths/CTrendComponent.cc index 7d8ec29418..0e26b8ef77 100644 --- a/lib/maths/CTrendComponent.cc +++ b/lib/maths/CTrendComponent.cc @@ -161,7 +161,7 @@ void CTrendComponent::shiftOrigin(core_t::TTime time) double scaledShift{scaleTime(time, m_RegressionOrigin)}; if (scaledShift > 0.0) { - for (auto &&model : m_Models) + for (auto &model : m_Models) { model.s_Regression.shiftAbscissa(-scaledShift); } @@ -209,7 +209,7 @@ void CTrendComponent::add(core_t::TTime time, double value, double weight) } double scaledTime{scaleTime(time, m_RegressionOrigin)}; - for (auto &&model : m_Models) + for (auto &model : m_Models) { model.s_Regression.add(scaledTime, value, weight); model.s_ResidualMoments.add(value - model.s_Regression.predict(scaledTime, MAX_CONDITION)); @@ -238,7 +238,7 @@ void CTrendComponent::propagateForwardsByTime(core_t::TTime interval) { m_Models[i].s_Weight.age(median); m_Models[i].s_Regression.age(factors[i]); - m_Models[i].s_ResidualMoments.age(factors[i]); + m_Models[i].s_ResidualMoments.age(std::sqrt(factors[i])); } } @@ -342,15 +342,18 @@ void CTrendComponent::forecast(core_t::TTime startTime, endTime = startTime + CIntegerTools::ceil(endTime - startTime, step); + core_t::TTime steps{(endTime - startTime) / step}; result.resize(steps, TDouble3Vec(3)); + LOG_TRACE("forecasting = " << this->print()); + TDoubleVec factors(this->factors(step)); TDoubleVec modelWeights(this->initialForecastModelWeights()); + TDoubleVec errorWeights(this->initialForecastErrorWeights()); TRegressionArrayVec models(NUMBER_MODELS); TMatrixVec modelCovariances(NUMBER_MODELS); - TDoubleVec residualVarianceWeights(this->initialForecastErrorWeights()); TDoubleVec residualVariances(NUMBER_MODELS); for (std::size_t i = 0u; i < NUMBER_MODELS; ++i) { @@ -361,10 +364,10 @@ void CTrendComponent::forecast(core_t::TTime startTime, + CBasicStatistics::variance(m_Models[i].s_ResidualMoments); LOG_TRACE("params = " << core::CContainerPrinter::print(models[i])); LOG_TRACE("covariances = " << modelCovariances[i].toDelimited()) - LOG_TRACE("variances = " << residualVariances[i]); } LOG_TRACE("long time variance = " << CBasicStatistics::variance(m_ValueMoments)); + TDoubleVec variances(NUMBER_MODELS + 1); for (core_t::TTime time = startTime; time < endTime; time += step) { core_t::TTime pillar{(time - startTime) / step}; @@ -377,19 +380,23 @@ void CTrendComponent::forecast(core_t::TTime startTime, for (std::size_t j = 0u; j < NUMBER_MODELS; ++j) { modelWeights[j] *= factors[j]; - residualVarianceWeights[j] *= std::pow(factors[j], 2.0); + errorWeights[j] *= std::pow(factors[j], 2.0); } + for (std::size_t j = 0u; j < NUMBER_MODELS; ++j) + { + variances[j] = times.inner(modelCovariances[j] * times) + residualVariances[j]; + } + variances[NUMBER_MODELS] = CBasicStatistics::variance(m_ValueMoments); + for (auto v = variances.rbegin(); v != variances.rend(); ++v) + { + *v = *std::min_element(variances.rbegin(), v+1); + } TMeanAccumulator variance_; - std::size_t last{NUMBER_MODELS - 1}; - for (std::size_t j = 0u; j < last; ++j) + for (std::size_t j = 0u; j < NUMBER_MODELS; ++j) { - variance_.add( times.inner(modelCovariances[j + 1] * times) - + residualVariances[j], residualVarianceWeights[j]); + variance_.add(variances[j], errorWeights[j]); } - variance_.add(residualVariances[last], residualVarianceWeights[last]); - variance_.add(CBasicStatistics::variance(m_ValueMoments), - residualVarianceWeights[NUMBER_MODELS]); double prediction{this->value(modelWeights, models, scaleTime(time, m_RegressionOrigin))}; double ql{0.0}; @@ -471,12 +478,11 @@ CTrendComponent::TDoubleVec CTrendComponent::initialForecastModelWeights() const CTrendComponent::TDoubleVec CTrendComponent::initialForecastErrorWeights() const { - TDoubleVec result(NUMBER_MODELS + 1, 1.0); - result[0] = std::exp(1.0); + TDoubleVec result(NUMBER_MODELS + 1); for (std::size_t i = 0u; i < NUMBER_MODELS; ++i) { - result[i] *= std::exp( static_cast(NUMBER_MODELS / 2) - - static_cast(i)); + result[i] = std::exp( static_cast(NUMBER_MODELS / 2) + - static_cast(i)); } result[NUMBER_MODELS] = result[NUMBER_MODELS - 1] / std::exp(1.0); return result; diff --git a/lib/maths/unittest/CForecastTest.cc b/lib/maths/unittest/CForecastTest.cc index 18571e6610..8c6a3ac8bf 100644 --- a/lib/maths/unittest/CForecastTest.cc +++ b/lib/maths/unittest/CForecastTest.cc @@ -127,7 +127,7 @@ void CForecastTest::testDailyConstantLongTermTrend(void) / static_cast(bucketLength) + y[i] + noise; }; - this->test(trend, bucketLength, 60, 64.0, 4.0, 0.02); + this->test(trend, bucketLength, 60, 64.0, 15.0, 0.02); } void CForecastTest::testDailyVaryingLongTermTrend(void) @@ -156,7 +156,7 @@ void CForecastTest::testDailyVaryingLongTermTrend(void) + noise; }; - this->test(trend, bucketLength, 100, 9.0, 11.0, 0.04); + this->test(trend, bucketLength, 100, 9.0, 13.0, 0.04); } void CForecastTest::testComplexNoLongTermTrend(void) @@ -178,7 +178,7 @@ void CForecastTest::testComplexNoLongTermTrend(void) return scale[d] * (20.0 + y[h] + noise); }; - this->test(trend, bucketLength, 60, 24.0, 32.0, 0.13); + this->test(trend, bucketLength, 60, 24.0, 34.0, 0.13); } void CForecastTest::testComplexConstantLongTermTrend(void) @@ -234,7 +234,7 @@ void CForecastTest::testComplexVaryingLongTermTrend(void) return trend_.value(time_) + scale[d] * (20.0 + y[h] + noise); }; - this->test(trend, bucketLength, 60, 4.0, 19.0, 0.05); + this->test(trend, bucketLength, 60, 4.0, 23.0, 0.05); } void CForecastTest::testNonNegative(void) @@ -426,7 +426,7 @@ void CForecastTest::testFinancialIndex(void) //file << "my = " << core::CContainerPrinter::print(my) << ";\n"; //file << "uy = " << core::CContainerPrinter::print(uy) << ";\n"; - CPPUNIT_ASSERT(percentageOutOfBounds < 50.0); + CPPUNIT_ASSERT(percentageOutOfBounds < 53.0); CPPUNIT_ASSERT(maths::CBasicStatistics::mean(error) < 0.1); } diff --git a/lib/maths/unittest/CTrendComponentTest.cc b/lib/maths/unittest/CTrendComponentTest.cc index 148a2ba62c..89e297fbce 100644 --- a/lib/maths/unittest/CTrendComponentTest.cc +++ b/lib/maths/unittest/CTrendComponentTest.cc @@ -384,7 +384,7 @@ void CTrendComponentTest::testForecast() { boost::tie(error, errorAt95) = testForecast(piecewiseLinear, 0, 3200000); CPPUNIT_ASSERT(error < 0.17); - CPPUNIT_ASSERT(errorAt95 < 0.05); + CPPUNIT_ASSERT(errorAt95 < 0.07); } LOG_DEBUG("Staircase");