Skip to content

[ML] Add include categorical filter type to detector rules #27

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 9, 2018
Merged

[ML] Add include categorical filter type to detector rules #27

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
34c45e3
Add whitelist rule condition type
davidkyle Mar 27, 2018
f91ca5f
Enable BWC for parsing categorical rules
davidkyle Apr 4, 2018
a5192b0
Fix compilation after rebase
davidkyle Apr 5, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion include/model/CRuleCondition.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ class MODEL_EXPORT CRuleCondition {
using TPatternSetCRef = boost::reference_wrapper<const core::CPatternSet>;

public:
enum ERuleConditionType { E_Categorical, E_NumericalActual, E_NumericalTypical, E_NumericalDiffAbs, E_Time };
enum ERuleConditionType { E_CategoricalMatch, E_CategoricalComplement, E_NumericalActual, E_NumericalTypical, E_NumericalDiffAbs, E_Time };

enum EConditionOperator { E_LT, E_LTE, E_GT, E_GTE };

Expand Down Expand Up @@ -76,6 +76,8 @@ class MODEL_EXPORT CRuleCondition {
void valueFilter(const core::CPatternSet& valueFilter);

//! Is the condition categorical?
//! Categorical conditions are pattern match conditions i.e.
//! E_CategoricalMatch and E_CategoricalComplement
bool isCategorical() const;

//! Is the condition numerical?
Expand Down
8 changes: 6 additions & 2 deletions lib/api/CDetectionRulesJsonParser.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ const std::string TARGET_FIELD_NAME("target_field_name");
const std::string TARGET_FIELD_VALUE("target_field_value");
const std::string TYPE("type");
const std::string CATEGORICAL("categorical");
const std::string CATEGORICAL_MATCH("categorical_match");
const std::string CATEGORICAL_COMPLEMENT("categorical_complement");
const std::string NUMERICAL_ACTUAL("numerical_actual");
const std::string NUMERICAL_TYPICAL("numerical_typical");
const std::string NUMERICAL_DIFF_ABS("numerical_diff_abs");
Expand Down Expand Up @@ -237,8 +239,10 @@ bool CDetectionRulesJsonParser::parseRuleConditionType(const rapidjson::Value& r
}

const std::string& type = ruleConditionObject[TYPE.c_str()].GetString();
if (type == CATEGORICAL) {
ruleCondition.type(model::CRuleCondition::E_Categorical);
if (type == CATEGORICAL_MATCH || type == CATEGORICAL){
ruleCondition.type(model::CRuleCondition::E_CategoricalMatch);
} else if (type == CATEGORICAL_COMPLEMENT) {
ruleCondition.type(model::CRuleCondition::E_CategoricalComplement);
} else if (type == NUMERICAL_ACTUAL) {
ruleCondition.type(model::CRuleCondition::E_NumericalActual);
} else if (type == NUMERICAL_TYPICAL) {
Expand Down
69 changes: 65 additions & 4 deletions lib/api/unittest/CDetectionRulesJsonParserTest.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,13 +76,19 @@ CppUnit::Test* CDetectionRulesJsonParserTest::suite() {
new CppUnit::TestCaller<CDetectionRulesJsonParserTest>("CDetectionRulesJsonParserTest::testParseRulesGivenMultipleRules",
&CDetectionRulesJsonParserTest::testParseRulesGivenMultipleRules));
suiteOfTests->addTest(
new CppUnit::TestCaller<CDetectionRulesJsonParserTest>("CDetectionRulesJsonParserTest::testParseRulesGivenCategoricalRule",
&CDetectionRulesJsonParserTest::testParseRulesGivenCategoricalRule));
new CppUnit::TestCaller<CDetectionRulesJsonParserTest>("CDetectionRulesJsonParserTest::testParseRulesGivenCategoricalMatchRule",
&CDetectionRulesJsonParserTest::testParseRulesGivenCategoricalMatchRule));
suiteOfTests->addTest(
new CppUnit::TestCaller<CDetectionRulesJsonParserTest>("CDetectionRulesJsonParserTest::testParseRulesGivenCategoricalComplementRule",
&CDetectionRulesJsonParserTest::testParseRulesGivenCategoricalComplementRule));
suiteOfTests->addTest(new CppUnit::TestCaller<CDetectionRulesJsonParserTest>(
"CDetectionRulesJsonParserTest::testParseRulesGivenTimeRule", &CDetectionRulesJsonParserTest::testParseRulesGivenTimeRule));
suiteOfTests->addTest(
new CppUnit::TestCaller<CDetectionRulesJsonParserTest>("CDetectionRulesJsonParserTest::testParseRulesGivenDifferentActions",
&CDetectionRulesJsonParserTest::testParseRulesGivenDifferentActions));
suiteOfTests->addTest(
new CppUnit::TestCaller<CDetectionRulesJsonParserTest>("CDetectionRulesJsonParserTest::testParseRulesGivenOldStyleCategoricalRule",
&CDetectionRulesJsonParserTest::testParseRulesGivenOldStyleCategoricalRule));
return suiteOfTests;
}

Expand Down Expand Up @@ -356,8 +362,37 @@ void CDetectionRulesJsonParserTest::testParseRulesGivenMultipleRules() {
CPPUNIT_ASSERT_EQUAL(std::string("SKIP_SAMPLING (id:42) IF ACTUAL < 2.000000"), rules[1].print());
}

void CDetectionRulesJsonParserTest::testParseRulesGivenCategoricalRule() {
LOG_DEBUG("*** testParseRulesGivenCategoricalRule ***");
void CDetectionRulesJsonParserTest::testParseRulesGivenCategoricalMatchRule() {
LOG_DEBUG("*** testParseRulesGivenCategoricalMatchRule ***");

TStrPatternSetUMap filtersById;
core::CPatternSet filter;
filter.initFromJson("[\"b\", \"a\"]");
filtersById["filter1"] = filter;

CDetectionRulesJsonParser parser(filtersById);
CDetectionRulesJsonParser::TDetectionRuleVec rules;
std::string rulesJson = "[";
rulesJson += "{";
rulesJson += " \"actions\":[\"filter_results\"],";
rulesJson += " \"conditions_connective\":\"or\",";
rulesJson += " \"conditions\": [";
rulesJson += " {\"type\":\"categorical_match\", \"field_name\":\"foo\", \"filter_id\":\"filter1\"}";
rulesJson += " ]";
rulesJson += "}";
rulesJson += "]";

CPPUNIT_ASSERT(parser.parseRules(rulesJson, rules));

CPPUNIT_ASSERT_EQUAL(std::size_t(1), rules.size());
CPPUNIT_ASSERT_EQUAL(std::string("FILTER_RESULTS IF (foo) IN FILTER"), rules[0].print());
}

void CDetectionRulesJsonParserTest::testParseRulesGivenOldStyleCategoricalRule() {
LOG_DEBUG("*** testParseRulesGivenOldStyleCategoricalRule ***");

// Tests that the rule type can be parsed as categorical_match
// when the type is categorical

TStrPatternSetUMap filtersById;
core::CPatternSet filter;
Expand All @@ -382,6 +417,32 @@ void CDetectionRulesJsonParserTest::testParseRulesGivenCategoricalRule() {
CPPUNIT_ASSERT_EQUAL(std::string("FILTER_RESULTS IF (foo) IN FILTER"), rules[0].print());
}

void CDetectionRulesJsonParserTest::testParseRulesGivenCategoricalComplementRule() {
LOG_DEBUG("*** testParseRulesGivenCategoricalComplementRule ***");

TStrPatternSetUMap filtersById;
core::CPatternSet filter;
filter.initFromJson("[\"b\", \"a\"]");
filtersById["filter1"] = filter;

CDetectionRulesJsonParser parser(filtersById);
CDetectionRulesJsonParser::TDetectionRuleVec rules;
std::string rulesJson = "[";
rulesJson += "{";
rulesJson += " \"actions\":[\"filter_results\"],";
rulesJson += " \"conditions_connective\":\"or\",";
rulesJson += " \"conditions\": [";
rulesJson += " {\"type\":\"categorical_complement\", \"field_name\":\"foo\", \"filter_id\":\"filter1\"}";
rulesJson += " ]";
rulesJson += "}";
rulesJson += "]";

CPPUNIT_ASSERT(parser.parseRules(rulesJson, rules));

CPPUNIT_ASSERT_EQUAL(std::size_t(1), rules.size());
CPPUNIT_ASSERT_EQUAL(std::string("FILTER_RESULTS IF (foo) NOT IN FILTER"), rules[0].print());
}

void CDetectionRulesJsonParserTest::testParseRulesGivenTimeRule() {
LOG_DEBUG("*** testParseRulesGivenTimeRule ***");

Expand Down
4 changes: 3 additions & 1 deletion lib/api/unittest/CDetectionRulesJsonParserTest.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,11 @@ class CDetectionRulesJsonParserTest : public CppUnit::TestFixture {
void testParseRulesGivenNumericalActualRuleWithConnectiveOr();
void testParseRulesGivenNumericalTypicalAndDiffAbsRuleWithConnectiveAnd();
void testParseRulesGivenMultipleRules();
void testParseRulesGivenCategoricalRule();
void testParseRulesGivenCategoricalMatchRule();
void testParseRulesGivenCategoricalComplementRule();
void testParseRulesGivenTimeRule();
void testParseRulesGivenDifferentActions();
void testParseRulesGivenOldStyleCategoricalRule();
static CppUnit::Test* suite();
};

Expand Down
21 changes: 15 additions & 6 deletions lib/model/CRuleCondition.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ void CRuleCondition::valueFilter(const core::CPatternSet& valueFilter) {
}

bool CRuleCondition::isCategorical() const {
return m_Type == E_Categorical;
return m_Type == E_CategoricalMatch || m_Type == E_CategoricalComplement;
}

bool CRuleCondition::isNumerical() const {
Expand All @@ -90,16 +90,19 @@ bool CRuleCondition::test(const CAnomalyDetectorModel& model,
const CDataGatherer& gatherer = model.dataGatherer();

if (this->isCategorical()) {
bool containsValue{false};
if (m_FieldName == gatherer.partitionFieldName()) {
return m_ValueFilter.get().contains(gatherer.partitionFieldValue());
containsValue = m_ValueFilter.get().contains(gatherer.partitionFieldValue());
} else if (m_FieldName == gatherer.personFieldName()) {
return m_ValueFilter.get().contains(gatherer.personName(pid));
containsValue = m_ValueFilter.get().contains(gatherer.personName(pid));
} else if (m_FieldName == gatherer.attributeFieldName()) {
return m_ValueFilter.get().contains(gatherer.attributeName(cid));
containsValue = m_ValueFilter.get().contains(gatherer.attributeName(cid));
} else {
LOG_ERROR("Unexpected fieldName = " << m_FieldName);
return false;
}

return (m_Type == E_CategoricalComplement) ? !containsValue : containsValue;
} else {
if (m_FieldValue.empty() == false) {
if (isScoped) {
Expand Down Expand Up @@ -138,7 +141,8 @@ bool CRuleCondition::checkCondition(const CAnomalyDetectorModel& model,
core_t::TTime time) const {
TDouble1Vec value;
switch (m_Type) {
case E_Categorical: {
case E_CategoricalMatch:
case E_CategoricalComplement: {
LOG_ERROR("Should never check numerical condition for categorical rule condition");
return false;
}
Expand Down Expand Up @@ -198,7 +202,11 @@ std::string CRuleCondition::print() const {
result += ")";
}
result += " ";

if (this->isCategorical()) {
if (m_Type == E_CategoricalComplement) {
result += "NOT ";
}
result += "IN FILTER";
} else {
result += this->print(m_Condition.s_Op) + " " + core::CStringUtils::typeToString(m_Condition.s_Threshold);
Expand All @@ -208,7 +216,8 @@ std::string CRuleCondition::print() const {

std::string CRuleCondition::print(ERuleConditionType type) const {
switch (type) {
case E_Categorical:
case E_CategoricalMatch:
case E_CategoricalComplement:
return "";
case E_NumericalActual:
return "ACTUAL";
Expand Down
Loading