Skip to content

Commit 9a84bff

Browse files
mohamedzeidan2021Mohamed Zeidan
andauthored
fix: allow Constraints.set_monitoring to disable monitoring for non-string features (#6312)
set_monitoring() unconditionally read feature["string_constraints"], which raised KeyError for non-string (e.g. Integral/Fractional) features and nested the override one level too deep even for string features. Per the Model Monitor constraints.json schema, monitoring_config_overrides belongs at the feature level. Write it there directly so it works for any feature type. Fixes #2745 Co-authored-by: Mohamed Zeidan <zeidmo@amazon.com>
1 parent 593aaad commit 9a84bff

2 files changed

Lines changed: 118 additions & 4 deletions

File tree

‎sagemaker-core/src/sagemaker/core/model_monitor/monitoring_files.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -363,10 +363,9 @@ def set_monitoring(self, enable_monitoring, feature_name=None):
363363
else:
364364
for feature in self.body_dict["features"]:
365365
if feature["name"] == feature_name:
366-
string_constraints = feature["string_constraints"]
367-
if string_constraints.get("monitoring_config_overrides") is None:
368-
string_constraints["monitoring_config_overrides"] = {}
369-
string_constraints["monitoring_config_overrides"]["evaluate_constraints"] = flag
366+
if feature.get("monitoring_config_overrides") is None:
367+
feature["monitoring_config_overrides"] = {}
368+
feature["monitoring_config_overrides"]["evaluate_constraints"] = flag
370369

371370

372371
class ConstraintViolations(ModelMonitoringFile):
Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
"""Unit tests for the Constraints.set_monitoring feature-level override."""
14+
15+
from __future__ import absolute_import
16+
17+
import pytest
18+
19+
from sagemaker.core.model_monitor.monitoring_files import Constraints
20+
21+
CONSTRAINTS_S3_URI = "s3://bucket/constraints.json"
22+
23+
24+
def _body_dict():
25+
"""A constraints body with one non-string (binary/integral) and one string feature."""
26+
return {
27+
"version": 0.0,
28+
"features": [
29+
{
30+
"name": "Churn",
31+
"inferred_type": "Integral",
32+
"num_constraints": {"is_non_negative": True},
33+
},
34+
{
35+
"name": "State",
36+
"inferred_type": "String",
37+
"string_constraints": {"domains": ["OH", "NJ"]},
38+
},
39+
],
40+
"monitoring_config": {"evaluate_constraints": "Enabled"},
41+
}
42+
43+
44+
def _make_constraints():
45+
return Constraints(body_dict=_body_dict(), constraints_file_s3_uri=CONSTRAINTS_S3_URI)
46+
47+
48+
def _feature(constraints, name):
49+
return next(f for f in constraints.body_dict["features"] if f["name"] == name)
50+
51+
52+
def test_set_monitoring_disables_non_string_feature():
53+
# Regression test for GitHub issue #2745: set_monitoring must work for a
54+
# non-string (e.g. binary/integral) feature that has no string_constraints section.
55+
constraints = _make_constraints()
56+
57+
constraints.set_monitoring(False, feature_name="Churn")
58+
59+
churn = _feature(constraints, "Churn")
60+
assert churn["monitoring_config_overrides"] == {"evaluate_constraints": "Disabled"}
61+
# The override must live at the feature level, not nested inside num_constraints.
62+
assert "monitoring_config_overrides" not in churn["num_constraints"]
63+
64+
65+
def test_set_monitoring_override_is_feature_level_for_string_feature():
66+
# Even for a string feature, the override belongs at the feature level (per the
67+
# constraints.json schema), not inside the string_constraints section.
68+
constraints = _make_constraints()
69+
70+
constraints.set_monitoring(False, feature_name="State")
71+
72+
state = _feature(constraints, "State")
73+
assert state["monitoring_config_overrides"] == {"evaluate_constraints": "Disabled"}
74+
assert "monitoring_config_overrides" not in state["string_constraints"]
75+
76+
77+
def test_set_monitoring_enable_maps_to_enabled():
78+
constraints = _make_constraints()
79+
80+
constraints.set_monitoring(True, feature_name="Churn")
81+
82+
assert _feature(constraints, "Churn")["monitoring_config_overrides"] == {
83+
"evaluate_constraints": "Enabled"
84+
}
85+
86+
87+
def test_set_monitoring_preserves_existing_overrides():
88+
body = _body_dict()
89+
body["features"][0]["monitoring_config_overrides"] = {"some_other_flag": "keep"}
90+
constraints = Constraints(body_dict=body, constraints_file_s3_uri=CONSTRAINTS_S3_URI)
91+
92+
constraints.set_monitoring(False, feature_name="Churn")
93+
94+
overrides = _feature(constraints, "Churn")["monitoring_config_overrides"]
95+
assert overrides == {"some_other_flag": "keep", "evaluate_constraints": "Disabled"}
96+
97+
98+
def test_set_monitoring_without_feature_name_sets_top_level_flag():
99+
constraints = _make_constraints()
100+
101+
constraints.set_monitoring(False)
102+
103+
assert constraints.body_dict["monitoring_config"]["evaluate_constraints"] == "Disabled"
104+
105+
106+
@pytest.mark.parametrize("enable,expected", [(True, "Enabled"), (False, "Disabled")])
107+
def test_monitoring_api_map(enable, expected):
108+
constraints = _make_constraints()
109+
110+
constraints.set_monitoring(enable, feature_name="Churn")
111+
112+
assert (
113+
_feature(constraints, "Churn")["monitoring_config_overrides"]["evaluate_constraints"]
114+
== expected
115+
)

0 commit comments

Comments
 (0)