Skip to content

Add red outline visual indicators for infeasible arms in ScatterPlot and ArmEffectsPlot #5026

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
sdaulton wants to merge 1 commit into from
Closed

Add red outline visual indicators for infeasible arms in ScatterPlot and ArmEffectsPlot #5026

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions ax/analysis/plotly/arm_effects.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,13 @@
get_arm_tooltip,
get_trial_statuses_with_fallback,
get_trial_trace_name,
INFEASIBLE_LEGEND_NAME,
INFEASIBLE_OUTLINE_COLOR,
INFEASIBLE_OUTLINE_WIDTH,
LEGEND_BASE_OFFSET,
LEGEND_POSITION,
MARGIN_REDUCUTION,
MINIMUM_P_FEASIBLE,
MULTIPLE_CANDIDATE_TRIALS_LEGEND,
SINGLE_CANDIDATE_TRIAL_LEGEND,
trial_index_to_color,
Expand Down Expand Up @@ -494,6 +498,32 @@ def _prepare_figure(
)
)

# Add toggle-able infeasible indicators if any arms are infeasible
infeasible_df = df[df["p_feasible_mean"] < MINIMUM_P_FEASIBLE]
infeasible_df = infeasible_df[~infeasible_df[f"{metric_name}_mean"].isna()]
if is_relative and status_quo_arm_name is not None:
infeasible_df = infeasible_df[infeasible_df["arm_name"] != status_quo_arm_name]
if not infeasible_df.empty:
figure.add_trace(
go.Scatter(
x=infeasible_df["x_key_order"],
y=infeasible_df[f"{metric_name}_mean"],
mode="markers",
marker={
"color": "rgba(0,0,0,0)",
"size": 10,
"line": {
"color": INFEASIBLE_OUTLINE_COLOR,
"width": INFEASIBLE_OUTLINE_WIDTH,
},
},
name=INFEASIBLE_LEGEND_NAME,
showlegend=True,
hoverinfo="skip",
legendgroup="infeasible",
)
)

# Add a horizontal line for the status quo.
if status_quo_arm_name in df["arm_name"].values:
# In relativized plots the status quo is always 0% on the y-axis.
Expand Down
31 changes: 31 additions & 0 deletions ax/analysis/plotly/scatter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,12 @@
get_arm_tooltip,
get_trial_statuses_with_fallback,
get_trial_trace_name,
INFEASIBLE_LEGEND_NAME,
INFEASIBLE_OUTLINE_COLOR,
INFEASIBLE_OUTLINE_WIDTH,
LEGEND_POSITION,
MARGIN_REDUCUTION,
MINIMUM_P_FEASIBLE,
MULTIPLE_CANDIDATE_TRIALS_LEGEND,
SINGLE_CANDIDATE_TRIAL_LEGEND,
trial_index_to_color,
Expand Down Expand Up @@ -522,6 +526,33 @@ def _prepare_figure(
)
)

# Add toggle-able infeasible indicators if any arms are infeasible
infeasible_df = df[df["p_feasible_mean"] < MINIMUM_P_FEASIBLE]
if not infeasible_df.empty:
mean_x = infeasible_df[f"{x_metric_name}_mean"]
mean_y = infeasible_df[f"{y_metric_name}_mean"]
valid = ~(mean_x.isna() & mean_y.isna())
if valid.any():
figure.add_trace(
go.Scatter(
x=mean_x[valid],
y=mean_y[valid],
mode="markers",
marker={
"color": "rgba(0,0,0,0)",
"size": 10,
"line": {
"color": INFEASIBLE_OUTLINE_COLOR,
"width": INFEASIBLE_OUTLINE_WIDTH,
},
},
name=INFEASIBLE_LEGEND_NAME,
showlegend=True,
hoverinfo="skip",
legendgroup="infeasible",
)
)

# Add horizontal and vertical lines for the status quo.
if "status_quo" in df["arm_name"].values:
x = df[df["arm_name"] == "status_quo"][f"{x_metric_name}_mean"].iloc[0]
Expand Down
102 changes: 102 additions & 0 deletions ax/analysis/plotly/tests/test_arm_effects.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,108 @@ def test_offline(self) -> None:
)


class TestArmEffectsPlotInfeasibility(TestCase):
def setUp(self) -> None:
super().setUp()

self.client = Client()
self.client.configure_experiment(
name="test_infeasibility",
parameters=[
RangeParameterConfig(
name="x1",
parameter_type="float",
bounds=(0, 1),
),
RangeParameterConfig(
name="x2",
parameter_type="float",
bounds=(0, 1),
),
],
)
# Constraint on "bar" (non-objective) so it stays as an OutcomeConstraint.
self.client.configure_optimization(
objective="foo",
outcome_constraints=["bar >= 0.5"],
)

# Trial data: (foo, bar). Arms with bar < 0.5 are infeasible.
trial_data = [
{"foo": 1.0, "bar": (0.9, 0.01)}, # feasible
{"foo": 0.5, "bar": (0.8, 0.01)}, # feasible
{"foo": 0.8, "bar": (0.1, 0.01)}, # infeasible
{"foo": 0.3, "bar": (0.2, 0.01)}, # infeasible
]

for raw_data in trial_data:
for trial_index, _ in self.client.get_next_trials(max_trials=1).items():
self.client.complete_trial(trial_index=trial_index, raw_data=raw_data)

def test_infeasible_arms_have_red_outline(self) -> None:
card = ArmEffectsPlot(metric_name="bar", use_model_predictions=False).compute(
experiment=self.client._experiment,
generation_strategy=self.client._generation_strategy,
)
fig_data = json.loads(none_throws(card.blob))

# All infeasible arms are in a single trace with legendgroup="infeasible"
infeasible_traces = [
t for t in fig_data["data"] if t.get("legendgroup") == "infeasible"
]
# Single trace containing all infeasible arms
self.assertEqual(len(infeasible_traces), 1)
trace = infeasible_traces[0]
self.assertEqual(trace["marker"]["line"]["color"], "red")
self.assertGreater(trace["marker"]["line"]["width"], 0)
# The trace should contain 2 infeasible points
self.assertEqual(len([x for x in trace["x"] if x is not None]), 2)
# Legend entry is on the same trace
self.assertTrue(trace["showlegend"])
self.assertEqual(trace["legendgroup"], "infeasible")

def test_no_infeasible_legend_when_all_feasible(self) -> None:
# Create an experiment where all arms satisfy the constraint
client = Client()
client.configure_experiment(
name="all_feasible",
parameters=[
RangeParameterConfig(
name="x1",
parameter_type="float",
bounds=(0, 1),
),
RangeParameterConfig(
name="x2",
parameter_type="float",
bounds=(0, 1),
),
],
)
client.configure_optimization(
objective="foo",
outcome_constraints=["bar >= 0.5"],
)

for bar_val in [0.9, 0.8, 0.7, 0.6]:
for trial_index, _ in client.get_next_trials(max_trials=1).items():
client.complete_trial(
trial_index=trial_index,
raw_data={"foo": 1.0, "bar": (bar_val, 0.01)},
)

card = ArmEffectsPlot(metric_name="bar", use_model_predictions=False).compute(
experiment=client._experiment,
generation_strategy=client._generation_strategy,
)
fig_data = json.loads(none_throws(card.blob))

legend_traces = [
t for t in fig_data["data"] if t.get("name") == "Likely Infeasible"
]
self.assertEqual(len(legend_traces), 0)


class TestArmEffectsPlotRel(TestCase):
def setUp(self) -> None:
super().setUp()
Expand Down
167 changes: 167 additions & 0 deletions ax/analysis/plotly/tests/test_scatter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -364,3 +364,170 @@ def test_offline(self) -> None:
experiment=experiment,
adapter=adapter,
)


class TestScatterPlotInfeasibility(TestCase):
def setUp(self) -> None:
super().setUp()

self.client = Client()
self.client.configure_experiment(
name="test_infeasibility",
parameters=[
RangeParameterConfig(
name="x1",
parameter_type="float",
bounds=(0, 1),
),
RangeParameterConfig(
name="x2",
parameter_type="float",
bounds=(0, 1),
),
],
)
# Constraint on "bar" (non-objective) so it stays as an OutcomeConstraint.
self.client.configure_optimization(
objective="foo",
outcome_constraints=["bar >= 0.5"],
)

# Trial data: (foo, bar). Arms with bar < 0.5 are infeasible.
trial_data = [
{"foo": 1.0, "bar": (0.9, 0.01)}, # feasible
{"foo": 0.5, "bar": (0.8, 0.01)}, # feasible
{"foo": 0.8, "bar": (0.1, 0.01)}, # infeasible
{"foo": 0.3, "bar": (0.2, 0.01)}, # infeasible
]

for raw_data in trial_data:
for trial_index, _ in self.client.get_next_trials(max_trials=1).items():
self.client.complete_trial(trial_index=trial_index, raw_data=raw_data)

def test_infeasible_arms_have_red_outline(self) -> None:
card = ScatterPlot(
x_metric_name="foo",
y_metric_name="bar",
use_model_predictions=False,
).compute(
experiment=self.client._experiment,
generation_strategy=self.client._generation_strategy,
)
fig_data = json.loads(none_throws(card.blob))

# All infeasible arms are in a single trace with legendgroup="infeasible"
infeasible_traces = [
t for t in fig_data["data"] if t.get("legendgroup") == "infeasible"
]
# Single trace containing all infeasible arms
self.assertEqual(len(infeasible_traces), 1)
trace = infeasible_traces[0]
self.assertEqual(trace["marker"]["line"]["color"], "red")
self.assertGreater(trace["marker"]["line"]["width"], 0)
# The trace should contain 2 infeasible points
self.assertEqual(len([x for x in trace["x"] if x is not None]), 2)
# Legend entry is on the same trace
self.assertTrue(trace["showlegend"])
self.assertEqual(trace["legendgroup"], "infeasible")

def test_no_infeasible_legend_when_all_feasible(self) -> None:
client = Client()
client.configure_experiment(
name="all_feasible",
parameters=[
RangeParameterConfig(
name="x1",
parameter_type="float",
bounds=(0, 1),
),
RangeParameterConfig(
name="x2",
parameter_type="float",
bounds=(0, 1),
),
],
)
client.configure_optimization(
objective="foo",
outcome_constraints=["bar >= 0.5"],
)

for bar_val in [0.9, 0.8, 0.7, 0.6]:
for trial_index, _ in client.get_next_trials(max_trials=1).items():
client.complete_trial(
trial_index=trial_index,
raw_data={"foo": 1.0, "bar": (bar_val, 0.01)},
)

card = ScatterPlot(
x_metric_name="foo",
y_metric_name="bar",
use_model_predictions=False,
).compute(
experiment=client._experiment,
generation_strategy=client._generation_strategy,
)
fig_data = json.loads(none_throws(card.blob))

legend_traces = [
t for t in fig_data["data"] if t.get("name") == "Likely Infeasible"
]
self.assertEqual(len(legend_traces), 0)

def test_infeasible_when_constraint_metric_not_plotted(self) -> None:
"""Red outlines should appear even when the constraint metric is not
one of the plotted metrics."""
client = Client()
client.configure_experiment(
name="constraint_not_plotted",
parameters=[
RangeParameterConfig(
name="x1",
parameter_type="float",
bounds=(0, 1),
),
RangeParameterConfig(
name="x2",
parameter_type="float",
bounds=(0, 1),
),
],
)
# Constraint on "bar", but we will plot "foo" vs "baz"
client.configure_optimization(
objective="foo",
outcome_constraints=["bar >= 0.5"],
)
client.configure_tracking_metrics(metric_names=["baz"])

trial_data = [
{"foo": 1.0, "baz": 5.0, "bar": (0.9, 0.01)}, # feasible
{"foo": 0.5, "baz": 3.0, "bar": (0.8, 0.01)}, # feasible
{"foo": 0.8, "baz": 4.0, "bar": (0.1, 0.01)}, # infeasible
{"foo": 0.3, "baz": 2.0, "bar": (0.2, 0.01)}, # infeasible
]

for raw_data in trial_data:
for trial_index, _ in client.get_next_trials(max_trials=1).items():
client.complete_trial(trial_index=trial_index, raw_data=raw_data)

card = ScatterPlot(
x_metric_name="foo",
y_metric_name="baz",
use_model_predictions=False,
).compute(
experiment=client._experiment,
generation_strategy=client._generation_strategy,
)
fig_data = json.loads(none_throws(card.blob))

# All infeasible arms are in a single trace with legendgroup="infeasible"
infeasible_traces = [
t for t in fig_data["data"] if t.get("legendgroup") == "infeasible"
]
# Single trace containing all infeasible arms
self.assertEqual(len(infeasible_traces), 1)
# The trace should contain 2 infeasible points
self.assertEqual(
len([x for x in infeasible_traces[0]["x"] if x is not None]), 2
)
Loading
Loading