Skip to content

[ML] Encapsulate timestamp conversion from epoch seconds to epoch millis #16

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 14, 2018
Merged

[ML] Encapsulate timestamp conversion from epoch seconds to epoch millis #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c122c28
[ML] Encapsulate timestamp conversion from epoch seconds to epoch millis
dimitris-athanasiou Mar 14, 2018
4a3b872
Fix review points and also use in model snapshot writer
dimitris-athanasiou Mar 14, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions include/core/CRapidJsonWriterBase.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include <core/ImportExport.h>
#include <core/CFunctional.h>
#include <core/CRapidJsonPoolAllocator.h>
#include <core/CTimeUtils.h>

#include <rapidjson/document.h>
#include <rapidjson/stringbuffer.h>
Expand Down Expand Up @@ -192,6 +193,11 @@ class CRapidJsonWriterBase : public JSON_WRITER<OUTPUT_STREAM, SOURCE_ENCODING,
return TRapidJsonWriterBase::Double(d);
}

//! Writes an epoch second timestamp as an epoch millis timestamp
bool Time(core_t::TTime t)
{
return this->Int64(CTimeUtils::toEpochMs(t));
}

//! Push a constant string into a supplied rapidjson object value
//! \p[in] value constant string
Expand Down Expand Up @@ -400,8 +406,7 @@ class CRapidJsonWriterBase : public JSON_WRITER<OUTPUT_STREAM, SOURCE_ENCODING,
core_t::TTime value,
TValue &obj) const
{
int64_t javaTimestamp = int64_t(value) * 1000;
TValue v(javaTimestamp);
TValue v(CTimeUtils::toEpochMs(value));
Copy link

@droberts195 droberts195 Mar 14, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Lower down this file, around line 550-555 value * int64_t(1000) can also be changed.

this->addMember(fieldName, v, obj);
}

Expand Down Expand Up @@ -547,7 +552,7 @@ class CRapidJsonWriterBase : public JSON_WRITER<OUTPUT_STREAM, SOURCE_ENCODING,

for (const auto &value: values)
{
this->pushBack(value * int64_t(1000), array);
this->pushBack(CTimeUtils::toEpochMs(value), array);
}

this->addMember(fieldName, array, obj);
Expand Down
2 changes: 2 additions & 0 deletions include/core/CTimeUtils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ class CORE_EXPORT CTimeUtils : private CNonInstantiatable
//! E.g. 19:20:30
static std::string toTimeString(core_t::TTime t);

//! Converts an epoch seconds timestamp to epoch millis
static int64_t toEpochMs(core_t::TTime t);
//! strptime interface
//! NOTE: the time returned here is a UTC value
static bool strptime(const std::string &format,
Expand Down
10 changes: 5 additions & 5 deletions lib/api/CJsonOutputWriter.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -500,7 +500,7 @@ void CJsonOutputWriter::writeBucket(bool isInterim,
m_Writer.addIntFieldToObj(DETECTOR_INDEX, detectorIndex, *docPtr);
m_Writer.addIntFieldToObj(BUCKET_SPAN, bucketData.s_BucketSpan, *docPtr);
m_Writer.addStringFieldCopyToObj(JOB_ID, m_JobId, *docPtr);
m_Writer.addIntFieldToObj(TIMESTAMP, bucketTime * 1000, *docPtr);
m_Writer.addTimeFieldToObj(TIMESTAMP, bucketTime, *docPtr);

if (isInterim)
{
Expand Down Expand Up @@ -531,7 +531,7 @@ void CJsonOutputWriter::writeBucket(bool isInterim,
}

m_Writer.addStringFieldCopyToObj(JOB_ID, m_JobId, *docPtr);
m_Writer.addIntFieldToObj(TIMESTAMP, bucketTime * 1000, *docPtr);
m_Writer.addTimeFieldToObj(TIMESTAMP, bucketTime, *docPtr);
if (isInterim)
{
m_Writer.addBoolFieldToObj(IS_INTERIM, isInterim, *docPtr);
Expand All @@ -551,7 +551,7 @@ void CJsonOutputWriter::writeBucket(bool isInterim,
m_Writer.String(JOB_ID);
m_Writer.String(m_JobId);
m_Writer.String(TIMESTAMP);
m_Writer.Int64(bucketTime * 1000);
m_Writer.Time(bucketTime);

m_Writer.String(ANOMALY_SCORE);
m_Writer.Double(bucketData.s_MaxBucketInfluencerNormalizedAnomalyScore);
Expand Down Expand Up @@ -586,7 +586,7 @@ void CJsonOutputWriter::writeBucket(bool isInterim,


m_Writer.addStringFieldCopyToObj(JOB_ID, m_JobId, *docPtr);
m_Writer.addIntFieldToObj(TIMESTAMP, bucketTime * 1000, *docPtr);
m_Writer.addTimeFieldToObj(TIMESTAMP, bucketTime, *docPtr);
m_Writer.addIntFieldToObj(BUCKET_SPAN, bucketData.s_BucketSpan, *docPtr);
if (isInterim)
{
Expand Down Expand Up @@ -984,7 +984,7 @@ void CJsonOutputWriter::acknowledgeFlush(const std::string &flushId, core_t::TTi
m_Writer.String(ID);
m_Writer.String(flushId);
m_Writer.String(LAST_FINALIZED_BUCKET_END);
m_Writer.Int64(lastFinalizedBucketEnd * 1000);
m_Writer.Time(lastFinalizedBucketEnd);

m_Writer.EndObject();
m_Writer.EndObject();
Expand Down
2 changes: 1 addition & 1 deletion lib/api/CModelPlotDataJsonWriter.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ void CModelPlotDataJsonWriter::writeFlatRow(core_t::TTime time,
m_Writer.addIntFieldToObj(DETECTOR_INDEX, detectorIndex, doc);
m_Writer.addStringFieldCopyToObj(FEATURE, feature, doc, true);
// time is in Java format - milliseconds since the epoch
m_Writer.addIntFieldToObj(TIME, time * 1000, doc);
m_Writer.addTimeFieldToObj(TIME, time, doc);
m_Writer.addIntFieldToObj(BUCKET_SPAN, bucketSpan, doc);
if (!partitionFieldName.empty())
{
Expand Down
4 changes: 2 additions & 2 deletions lib/api/CModelSizeStatsJsonWriter.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,10 @@ void CModelSizeStatsJsonWriter::write(const std::string &jobId,
writer.String(print(results.s_MemoryStatus));

writer.String(TIMESTAMP);
writer.Int64(results.s_BucketStartTime * 1000);
writer.Time(results.s_BucketStartTime);

writer.String(LOG_TIME);
writer.Int64(core::CTimeUtils::now() * 1000);
writer.Time(core::CTimeUtils::now());

writer.EndObject();
}
Expand Down
15 changes: 4 additions & 11 deletions lib/api/CModelSnapshotJsonWriter.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,11 +60,8 @@ void CModelSnapshotJsonWriter::write(const SModelSnapshotReport &report)
m_Writer.String(SNAPSHOT_DOC_COUNT);
m_Writer.Uint64(report.s_NumDocs);

// Write as a Java timestamp - ms since the epoch rather than seconds
int64_t javaTimestamp = int64_t(report.s_SnapshotTimestamp) * 1000;

m_Writer.String(TIMESTAMP);
m_Writer.Int64(javaTimestamp);
m_Writer.Time(report.s_SnapshotTimestamp);

m_Writer.String(DESCRIPTION);
m_Writer.String(report.s_Description);
Expand All @@ -73,17 +70,13 @@ void CModelSnapshotJsonWriter::write(const SModelSnapshotReport &report)

if (report.s_LatestRecordTime > 0)
{
javaTimestamp = int64_t(report.s_LatestRecordTime) * 1000;

m_Writer.String(LATEST_RECORD_TIME);
m_Writer.Int64(javaTimestamp);
m_Writer.Time(report.s_LatestRecordTime);
}
if (report.s_LatestFinalResultTime > 0)
{
javaTimestamp = int64_t(report.s_LatestFinalResultTime) * 1000;

m_Writer.String(LATEST_RESULT_TIME);
m_Writer.Int64(javaTimestamp);
m_Writer.Time(report.s_LatestFinalResultTime);
}

// write normalizerState here
Expand Down Expand Up @@ -111,7 +104,7 @@ void CModelSnapshotJsonWriter::writeQuantileState(const std::string &jobId,
writer.String(QUANTILE_STATE);
writer.String(state);
writer.String(TIMESTAMP);
writer.Int64(time * 1000);
writer.Time(time);
writer.EndObject();
}

Expand Down
5 changes: 5 additions & 0 deletions lib/core/CTimeUtils.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,11 @@ std::string CTimeUtils::toTimeString(core_t::TTime t)
return result;
}

int64_t CTimeUtils::toEpochMs(core_t::TTime t)
{
return static_cast<int64_t>(t) * 1000;
}

bool CTimeUtils::strptime(const std::string &format,
const std::string &dateTime,
core_t::TTime &preTime)
Expand Down
3 changes: 3 additions & 0 deletions lib/core/unittest/CRapidJsonWriterBaseTest.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ const std::string NAN_NAME("nan");
const std::string INFINITY_NAME("infinity");
const std::string BOOL_NAME("bool");
const std::string INT_NAME("int");
const std::string TIME_NAME("time");
const std::string UINT_NAME("uint");
const std::string STR_ARRAY_NAME("str[]");
const std::string DOUBLE_ARRAY_NAME("double[]");
Expand All @@ -75,6 +76,7 @@ void CRapidJsonWriterBaseTest::testAddFields(void)
writer.addDoubleFieldToObj(INFINITY_NAME, std::numeric_limits<double>::infinity(), doc);
writer.addBoolFieldToObj(BOOL_NAME, false, doc);
writer.addIntFieldToObj(INT_NAME, -9, doc);
writer.addTimeFieldToObj(TIME_NAME, ml::core_t::TTime(1521035866), doc);
writer.addUIntFieldToObj(UINT_NAME, 999999999999999ull, doc);
writer.addStringArrayFieldToObj(STR_ARRAY_NAME, TGenericLineWriter::TStrVec(3, "blah"), doc);
writer.addDoubleArrayFieldToObj(DOUBLE_ARRAY_NAME, TGenericLineWriter::TDoubleVec(10, 1.5), doc);
Expand All @@ -97,6 +99,7 @@ void CRapidJsonWriterBaseTest::testAddFields(void)
"\"infinity\":0,"
"\"bool\":false,"
"\"int\":-9,"
"\"time\":1521035866000,"
"\"uint\":999999999999999,"
"\"str[]\":[\"blah\",\"blah\",\"blah\"],"
"\"double[]\":[1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5],"
Expand Down
11 changes: 11 additions & 0 deletions lib/core/unittest/CTimeUtilsTest.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ CppUnit::Test *CTimeUtilsTest::suite()
suiteOfTests->addTest( new CppUnit::TestCaller<CTimeUtilsTest>(
"CTimeUtilsTest::testToLocal",
&CTimeUtilsTest::testToLocal) );
suiteOfTests->addTest( new CppUnit::TestCaller<CTimeUtilsTest>(
"CTimeUtilsTest::testToEpochMs",
&CTimeUtilsTest::testToEpochMs) );
suiteOfTests->addTest( new CppUnit::TestCaller<CTimeUtilsTest>(
"CTimeUtilsTest::testStrptime",
&CTimeUtilsTest::testStrptime) );
Expand Down Expand Up @@ -114,6 +117,14 @@ void CTimeUtilsTest::testToLocal(void)
}
}

void CTimeUtilsTest::testToEpochMs(void)
{
CPPUNIT_ASSERT_EQUAL(int64_t(1000), ml::core::CTimeUtils::toEpochMs(ml::core_t::TTime(1)));
CPPUNIT_ASSERT_EQUAL(int64_t(-1000), ml::core::CTimeUtils::toEpochMs(ml::core_t::TTime(-1)));
CPPUNIT_ASSERT_EQUAL(int64_t(1521035866000), ml::core::CTimeUtils::toEpochMs(ml::core_t::TTime(1521035866)));
CPPUNIT_ASSERT_EQUAL(int64_t(-1521035866000), ml::core::CTimeUtils::toEpochMs(ml::core_t::TTime(-1521035866)));
}

void CTimeUtilsTest::testStrptime(void)
{
// These tests assume UK time. In case they're ever run outside the UK,
Expand Down
1 change: 1 addition & 0 deletions lib/core/unittest/CTimeUtilsTest.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ class CTimeUtilsTest : public CppUnit::TestFixture
void testNow(void);
void testToIso8601(void);
void testToLocal(void);
void testToEpochMs(void);
void testStrptime(void);
void testTimezone(void);
void testDateWords(void);
Expand Down