ANL-CEEESA/MIPLearn
1
0
Fork 0
mirror of https://github.com/ANL-CEEESA/MIPLearn.git synced 2025-12-06 09:28:51 -06:00
Code Issues Packages Projects Releases Wiki Activity

Redesign component.evaluate

Browse Source
This commit is contained in:
Alinson S. Xavier
2021-04-02 08:09:35 -05:00
parent 0c687692f7
commit 0bce2051a8
9 changed files with 221 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
miplearn/components/__init__.py
View File

@@ -1,12 +1,13 @@
# MIPLearn: Extensible Framework for Learning-Enhanced Mixed-Integer Optimization
# Copyright (C) 2020, UChicago Argonne, LLC. All rights reserved.
# Released under the modified BSD license. See COPYING.md for more details.
from typing import Dict
def classifier_evaluation_dict(tp, tn, fp, fn):
def classifier_evaluation_dict(tp: int, tn: int, fp: int, fn: int) -> Dict:
p = tp + fn
n = fp + tn
d = {
d: Dict = {
"Predicted positive": fp + tp,
"Predicted negative": fn + tn,
"Condition positive": p,

14
miplearn/components/component.py
View File

@@ -106,7 +106,7 @@ class Component:
return
@staticmethod
def xy(
def sample_xy(
features: Features,
sample: TrainingSample,
) -> Tuple[Dict, Dict]:
@@ -127,7 +127,7 @@ class Component:
for instance in InstanceIterator(instances):
assert isinstance(instance, Instance)
for sample in instance.training_data:
xy = self.xy(instance.features, sample)
xy = self.sample_xy(instance.features, sample)
if xy is None:
continue
x_sample, y_sample = xy
@@ -191,3 +191,13 @@ class Component:
model: Any,
) -> None:
return
def evaluate(self, instances: Union[List[str], List[Instance]]) -> List:
ev = []
for instance in InstanceIterator(instances):
for sample in instance.training_data:
ev += [self.sample_evaluate(instance.features, sample)]
return ev
def sample_evaluate(self, features: Features, sample: TrainingSample) -> Dict:
return {}

2
miplearn/components/lazy_static.py
View File

@@ -205,7 +205,7 @@ class StaticLazyConstraintsComponent(Component):
return result
@staticmethod
def xy(
def sample_xy(
features: Features,
sample: TrainingSample,
) -> Tuple[Dict, Dict]:

74
miplearn/components/objective.py
View File

@@ -116,45 +116,45 @@ class ObjectiveValueComponent(Component):
"Upper bound": np.array(ub),
}
def evaluate(
self,
instances: Union[List[str], List[Instance]],
) -> Dict[str, Dict[str, float]]:
y_pred = self.predict(instances)
y_true = np.array(
[
[
inst.training_data[0]["Lower bound"],
inst.training_data[0]["Upper bound"],
]
for inst in InstanceIterator(instances)
]
)
y_pred_lb = y_pred["Lower bound"]
y_pred_ub = y_pred["Upper bound"]
y_true_lb, y_true_ub = y_true[:, 1], y_true[:, 1]
ev = {
"Lower bound": {
"Mean squared error": mean_squared_error(y_true_lb, y_pred_lb),
"Explained variance": explained_variance_score(y_true_lb, y_pred_lb),
"Max error": max_error(y_true_lb, y_pred_lb),
"Mean absolute error": mean_absolute_error(y_true_lb, y_pred_lb),
"R2": r2_score(y_true_lb, y_pred_lb),
"Median absolute error": mean_absolute_error(y_true_lb, y_pred_lb),
},
"Upper bound": {
"Mean squared error": mean_squared_error(y_true_ub, y_pred_ub),
"Explained variance": explained_variance_score(y_true_ub, y_pred_ub),
"Max error": max_error(y_true_ub, y_pred_ub),
"Mean absolute error": mean_absolute_error(y_true_ub, y_pred_ub),
"R2": r2_score(y_true_ub, y_pred_ub),
"Median absolute error": mean_absolute_error(y_true_ub, y_pred_ub),
},
}
return ev
# def evaluate(
# self,
# instances: Union[List[str], List[Instance]],
# ) -> Dict[str, Dict[str, float]]:
# y_pred = self.predict(instances)
# y_true = np.array(
# [
# [
# inst.training_data[0]["Lower bound"],
# inst.training_data[0]["Upper bound"],
# ]
# for inst in InstanceIterator(instances)
# ]
# )
# y_pred_lb = y_pred["Lower bound"]
# y_pred_ub = y_pred["Upper bound"]
# y_true_lb, y_true_ub = y_true[:, 1], y_true[:, 1]
# ev = {
# "Lower bound": {
# "Mean squared error": mean_squared_error(y_true_lb, y_pred_lb),
# "Explained variance": explained_variance_score(y_true_lb, y_pred_lb),
# "Max error": max_error(y_true_lb, y_pred_lb),
# "Mean absolute error": mean_absolute_error(y_true_lb, y_pred_lb),
# "R2": r2_score(y_true_lb, y_pred_lb),
# "Median absolute error": mean_absolute_error(y_true_lb, y_pred_lb),
# },
# "Upper bound": {
# "Mean squared error": mean_squared_error(y_true_ub, y_pred_ub),
# "Explained variance": explained_variance_score(y_true_ub, y_pred_ub),
# "Max error": max_error(y_true_ub, y_pred_ub),
# "Mean absolute error": mean_absolute_error(y_true_ub, y_pred_ub),
# "R2": r2_score(y_true_ub, y_pred_ub),
# "Median absolute error": mean_absolute_error(y_true_ub, y_pred_ub),
# },
# }
# return ev
@staticmethod
def xy(
def sample_xy(
features: Features,
sample: TrainingSample,
) -> Tuple[Dict, Dict]:

149
miplearn/components/primal.py
View File

@@ -4,20 +4,16 @@
import logging
from typing import (
Union,
Dict,
Callable,
List,
Hashable,
Optional,
Any,
TYPE_CHECKING,
Tuple,
cast,
)
import numpy as np
from tqdm.auto import tqdm
from miplearn.classifiers import Classifier
from miplearn.classifiers.adaptive import AdaptiveClassifier
@@ -72,12 +68,18 @@ class PrimalSolutionComponent(Component):
features: Features,
training_data: TrainingSample,
) -> None:
if len(self.thresholds) > 0:
# Do nothing if models are not trained
if len(self.classifiers) == 0:
return
# Predict solution and provide it to the solver
logger.info("Predicting MIP solution...")
solution = self.predict(
instance.features,
instance.training_data[-1],
)
solution = self.sample_predict(features, training_data)
assert solver.internal_solver is not None
if self.mode == "heuristic":
solver.internal_solver.fix(solution)
else:
solver.internal_solver.set_warm_start(solution)
# Update statistics
stats["Primal: Free"] = 0
@@ -98,27 +100,7 @@ class PrimalSolutionComponent(Component):
f"one: {stats['Primal: One']}"
)
# Provide solution to the solver
assert solver.internal_solver is not None
if self.mode == "heuristic":
solver.internal_solver.fix(solution)
else:
solver.internal_solver.set_warm_start(solution)
def fit_xy(
self,
x: Dict[str, np.ndarray],
y: Dict[str, np.ndarray],
) -> None:
for category in x.keys():
clf = self.classifier_prototype.clone()
thr = self.threshold_prototype.clone()
clf.fit(x[category], y[category])
thr.fit(clf, x[category], y[category])
self.classifiers[category] = clf
self.thresholds[category] = thr
def predict(
def sample_predict(
self,
features: Features,
sample: TrainingSample,
@@ -131,7 +113,7 @@ class PrimalSolutionComponent(Component):
solution[var_name][idx] = None
# Compute y_pred
x, _ = self.xy(features, sample)
x, _ = self.sample_xy(features, sample)
y_pred = {}
for category in x.keys():
assert category in self.classifiers, (
@@ -162,55 +144,8 @@ class PrimalSolutionComponent(Component):
return solution
def evaluate(self, instances):
ev = {"Fix zero": {}, "Fix one": {}}
for instance_idx in tqdm(
range(len(instances)),
desc="Evaluate (primal)",
):
instance = instances[instance_idx]
solution_actual = instance.training_data[0]["Solution"]
solution_pred = self.predict(instance, instance.training_data[0])
vars_all, vars_one, vars_zero = set(), set(), set()
pred_one_positive, pred_zero_positive = set(), set()
for (varname, var_dict) in solution_actual.items():
if varname not in solution_pred.keys():
continue
for (idx, value) in var_dict.items():
vars_all.add((varname, idx))
if value > 0.5:
vars_one.add((varname, idx))
else:
vars_zero.add((varname, idx))
if solution_pred[varname][idx] is not None:
if solution_pred[varname][idx] > 0.5:
pred_one_positive.add((varname, idx))
else:
pred_zero_positive.add((varname, idx))
pred_one_negative = vars_all - pred_one_positive
pred_zero_negative = vars_all - pred_zero_positive
tp_zero = len(pred_zero_positive & vars_zero)
fp_zero = len(pred_zero_positive & vars_one)
tn_zero = len(pred_zero_negative & vars_one)
fn_zero = len(pred_zero_negative & vars_zero)
tp_one = len(pred_one_positive & vars_one)
fp_one = len(pred_one_positive & vars_zero)
tn_one = len(pred_one_negative & vars_zero)
fn_one = len(pred_one_negative & vars_one)
ev["Fix zero"][instance_idx] = classifier_evaluation_dict(
tp_zero, tn_zero, fp_zero, fn_zero
)
ev["Fix one"][instance_idx] = classifier_evaluation_dict(
tp_one, tn_one, fp_one, fn_one
)
return ev
@staticmethod
def xy(
def sample_xy(
features: Features,
sample: TrainingSample,
) -> Tuple[Dict, Dict]:
@@ -246,3 +181,59 @@ class PrimalSolutionComponent(Component):
)
y[category] += [[opt_value < 0.5, opt_value >= 0.5]]
return x, y
def sample_evaluate(
self,
features: Features,
sample: TrainingSample,
) -> Dict:
solution_actual = sample["Solution"]
assert solution_actual is not None
solution_pred = self.sample_predict(features, sample)
vars_all, vars_one, vars_zero = set(), set(), set()
pred_one_positive, pred_zero_positive = set(), set()
for (varname, var_dict) in solution_actual.items():
if varname not in solution_pred.keys():
continue
for (idx, value_actual) in var_dict.items():
assert value_actual is not None
vars_all.add((varname, idx))
if value_actual > 0.5:
vars_one.add((varname, idx))
else:
vars_zero.add((varname, idx))
value_pred = solution_pred[varname][idx]
if value_pred is not None:
if value_pred > 0.5:
pred_one_positive.add((varname, idx))
else:
pred_zero_positive.add((varname, idx))
pred_one_negative = vars_all - pred_one_positive
pred_zero_negative = vars_all - pred_zero_positive
return {
0: classifier_evaluation_dict(
tp=len(pred_zero_positive & vars_zero),
tn=len(pred_zero_negative & vars_one),
fp=len(pred_zero_positive & vars_one),
fn=len(pred_zero_negative & vars_zero),
),
1: classifier_evaluation_dict(
tp=len(pred_one_positive & vars_one),
tn=len(pred_one_negative & vars_zero),
fp=len(pred_one_positive & vars_zero),
fn=len(pred_one_negative & vars_one),
),
}
def fit_xy(
self,
x: Dict[str, np.ndarray],
y: Dict[str, np.ndarray],
) -> None:
for category in x.keys():
clf = self.classifier_prototype.clone()
thr = self.threshold_prototype.clone()
clf.fit(x[category], y[category])
thr.fit(clf, x[category], y[category])
self.classifiers[category] = clf
self.thresholds[category] = thr

4
tests/components/test_component.py
View File

@@ -7,7 +7,7 @@ from miplearn import Component, Instance
def test_xy_instance():
def _xy_sample(features, sample):
def _sample_xy(features, sample):
x = {
"s1": {
"category_a": [
@@ -57,7 +57,7 @@ def test_xy_instance():
instance_2 = Mock(spec=Instance)
instance_2.training_data = ["s3"]
instance_2.features = {}
comp.xy = _xy_sample
comp.sample_xy = _sample_xy
x_expected = {
"category_a": [
[1, 2, 3],

2
tests/components/test_lazy_static.py
View File

@@ -293,7 +293,7 @@ def test_xy_sample() -> None:
[False, True],
],
}
xy = StaticLazyConstraintsComponent.xy(features, sample)
xy = StaticLazyConstraintsComponent.sample_xy(features, sample)
assert xy is not None
x_actual, y_actual = xy
assert x_actual == x_expected

62
tests/components/test_objective.py
View File

@@ -75,35 +75,35 @@ def test_x_y_predict() -> None:
}
def test_obj_evaluate():
instances, models = get_test_pyomo_instances()
reg = Mock(spec=Regressor)
reg.predict = Mock(return_value=np.array([[1000.0], [1000.0]]))
reg.clone = lambda: reg
comp = ObjectiveValueComponent(
lb_regressor=reg,
ub_regressor=reg,
)
comp.fit(instances)
ev = comp.evaluate(instances)
assert ev == {
"Lower bound": {
"Explained variance": 0.0,
"Max error": 183.0,
"Mean absolute error": 126.5,
"Mean squared error": 19194.5,
"Median absolute error": 126.5,
"R2": -5.012843605607331,
},
"Upper bound": {
"Explained variance": 0.0,
"Max error": 183.0,
"Mean absolute error": 126.5,
"Mean squared error": 19194.5,
"Median absolute error": 126.5,
"R2": -5.012843605607331,
},
}
# def test_obj_evaluate():
# instances, models = get_test_pyomo_instances()
# reg = Mock(spec=Regressor)
# reg.predict = Mock(return_value=np.array([[1000.0], [1000.0]]))
# reg.clone = lambda: reg
# comp = ObjectiveValueComponent(
# lb_regressor=reg,
# ub_regressor=reg,
# )
# comp.fit(instances)
# ev = comp.evaluate(instances)
# assert ev == {
# "Lower bound": {
# "Explained variance": 0.0,
# "Max error": 183.0,
# "Mean absolute error": 126.5,
# "Mean squared error": 19194.5,
# "Median absolute error": 126.5,
# "R2": -5.012843605607331,
# },
# "Upper bound": {
# "Explained variance": 0.0,
# "Max error": 183.0,
# "Mean absolute error": 126.5,
# "Mean squared error": 19194.5,
# "Median absolute error": 126.5,
# "R2": -5.012843605607331,
# },
# }
def test_xy_sample_with_lp() -> None:
@@ -125,7 +125,7 @@ def test_xy_sample_with_lp() -> None:
"Lower bound": [[1.0]],
"Upper bound": [[2.0]],
}
xy = ObjectiveValueComponent.xy(features, sample)
xy = ObjectiveValueComponent.sample_xy(features, sample)
assert xy is not None
x_actual, y_actual = xy
assert x_actual == x_expected
@@ -150,7 +150,7 @@ def test_xy_sample_without_lp() -> None:
"Lower bound": [[1.0]],
"Upper bound": [[2.0]],
}
xy = ObjectiveValueComponent.xy(features, sample)
xy = ObjectiveValueComponent.sample_xy(features, sample)
assert xy is not None
x_actual, y_actual = xy
assert x_actual == x_expected

51
tests/components/test_primal.py
View File

@@ -1,7 +1,7 @@
# MIPLearn: Extensible Framework for Learning-Enhanced Mixed-Integer Optimization
# Copyright (C) 2020, UChicago Argonne, LLC. All rights reserved.
# Released under the modified BSD license. See COPYING.md for more details.
from typing import Dict
from unittest.mock import Mock
import numpy as np
@@ -10,6 +10,7 @@ from scipy.stats import randint
from miplearn import Classifier, LearningSolver
from miplearn.classifiers.threshold import Threshold
from miplearn.components import classifier_evaluation_dict
from miplearn.components.primal import PrimalSolutionComponent
from miplearn.problems.tsp import TravelingSalesmanGenerator
from miplearn.types import TrainingSample, Features
@@ -69,7 +70,7 @@ def test_xy() -> None:
[True, False],
]
}
xy = PrimalSolutionComponent.xy(features, sample)
xy = PrimalSolutionComponent.sample_xy(features, sample)
assert xy is not None
x_actual, y_actual = xy
assert x_actual == x_expected
@@ -122,7 +123,7 @@ def test_xy_without_lp_solution() -> None:
[True, False],
]
}
xy = PrimalSolutionComponent.xy(features, sample)
xy = PrimalSolutionComponent.sample_xy(features, sample)
assert xy is not None
x_actual, y_actual = xy
assert x_actual == x_expected
@@ -169,11 +170,11 @@ def test_predict() -> None:
}
}
}
x, _ = PrimalSolutionComponent.xy(features, sample)
x, _ = PrimalSolutionComponent.sample_xy(features, sample)
comp = PrimalSolutionComponent()
comp.classifiers = {"default": clf}
comp.thresholds = {"default": thr}
solution_actual = comp.predict(features, sample)
solution_actual = comp.sample_predict(features, sample)
clf.predict_proba.assert_called_once()
assert_array_equal(x["default"], clf.predict_proba.call_args[0][0])
thr.predict.assert_called_once()
@@ -229,3 +230,43 @@ def test_usage():
assert stats["Primal: Free"] == 0
assert stats["Primal: One"] + stats["Primal: Zero"] == 10
assert stats["Lower bound"] == stats["Warm start value"]
def test_evaluate() -> None:
comp = PrimalSolutionComponent()
comp.sample_predict = lambda _, __: { # type: ignore
"x": {
0: 1.0,
1: 0.0,
2: 0.0,
3: None,
4: 1.0,
}
}
features: Features = {
"Variables": {
"x": {
0: {},
1: {},
2: {},
3: {},
4: {},
}
}
}
sample: TrainingSample = {
"Solution": {
"x": {
0: 1.0,
1: 1.0,
2: 0.0,
3: 1.0,
4: 1.0,
}
}
}
ev = comp.sample_evaluate(features, sample)
assert ev == {
0: classifier_evaluation_dict(tp=1, fp=1, tn=3, fn=0),
1: classifier_evaluation_dict(tp=2, fp=0, tn=1, fn=2),
}