diff --git a/miplearn/components/component.py b/miplearn/components/component.py index 474b284..e73f329 100644 --- a/miplearn/components/component.py +++ b/miplearn/components/component.py @@ -106,8 +106,8 @@ class Component: """ return - @staticmethod def sample_xy( + self, instance: Instance, sample: TrainingSample, ) -> Tuple[Dict, Dict]: diff --git a/miplearn/components/lazy_dynamic.py b/miplearn/components/lazy_dynamic.py index 5334ef5..0de3c47 100644 --- a/miplearn/components/lazy_dynamic.py +++ b/miplearn/components/lazy_dynamic.py @@ -3,17 +3,16 @@ # Released under the modified BSD license. See COPYING.md for more details. import logging -import sys -from typing import Any, Dict, List, TYPE_CHECKING, Hashable +from typing import Dict, List, TYPE_CHECKING, Hashable, Tuple import numpy as np -from tqdm.auto import tqdm from miplearn.classifiers import Classifier from miplearn.classifiers.counting import CountingClassifier +from miplearn.classifiers.threshold import MinProbabilityThreshold, Threshold from miplearn.components import classifier_evaluation_dict from miplearn.components.component import Component -from miplearn.extractors import InstanceFeaturesExtractor +from miplearn.features import TrainingSample logger = logging.getLogger(__name__) @@ -29,14 +28,21 @@ class DynamicLazyConstraintsComponent(Component): def __init__( self, classifier: Classifier = CountingClassifier(), - threshold: float = 0.05, + threshold: Threshold = MinProbabilityThreshold([0, 0.05]), ): assert isinstance(classifier, Classifier) - self.threshold: float = threshold + self.threshold_prototype: Threshold = threshold self.classifier_prototype: Classifier = classifier - self.classifiers: Dict[Any, Classifier] = {} + self.classifiers: Dict[Hashable, Classifier] = {} + self.thresholds: Dict[Hashable, Threshold] = {} self.known_cids: List[str] = [] + @staticmethod + def enforce(cids, instance, model, solver): + for cid in cids: + cobj = instance.build_lazy_constraint(model, cid) + solver.internal_solver.add_constraint(cobj) + def before_solve_mip( self, solver, @@ -46,86 +52,91 @@ class DynamicLazyConstraintsComponent(Component): features, training_data, ): - instance.found_violated_lazy_constraints = [] + training_data.lazy_enforced = set() logger.info("Predicting violated lazy constraints...") - violations = self.predict(instance) - logger.info("Enforcing %d lazy constraints..." % len(violations)) - for v in violations: - cut = instance.build_lazy_constraint(model, v) - solver.internal_solver.add_constraint(cut) + cids = self.sample_predict(instance, training_data) + logger.info("Enforcing %d lazy constraints..." % len(cids)) + self.enforce(cids, instance, model, solver) def iteration_cb(self, solver, instance, model): - logger.debug("Finding violated (dynamic) lazy constraints...") - violations = instance.find_violated_lazy_constraints(model) - if len(violations) == 0: + logger.debug("Finding violated lazy constraints...") + cids = instance.find_violated_lazy_constraints(model) + if len(cids) == 0: + logger.debug("No violations found") return False - instance.found_violated_lazy_constraints += violations - logger.debug(" %d violations found" % len(violations)) - for v in violations: - cut = instance.build_lazy_constraint(model, v) - solver.internal_solver.add_constraint(cut) - return True - - def fit(self, training_instances): - logger.debug("Fitting...") - features = InstanceFeaturesExtractor().extract(training_instances) - - self.classifiers = {} - violation_to_instance_idx = {} - for (idx, instance) in enumerate(training_instances): - for v in instance.found_violated_lazy_constraints: - if isinstance(v, list): - v = tuple(v) - if v not in self.classifiers: - self.classifiers[v] = self.classifier_prototype.clone() - violation_to_instance_idx[v] = [] - violation_to_instance_idx[v] += [idx] - - for (v, classifier) in tqdm( - self.classifiers.items(), - desc="Fit (lazy)", - disable=not sys.stdout.isatty(), - ): - logger.debug("Training: %s" % (str(v))) - label = [[True, False] for i in training_instances] - for idx in violation_to_instance_idx[v]: - label[idx] = [False, True] - label = np.array(label, dtype=np.bool8) - classifier.fit(features, label) - - def predict(self, instance): - violations = [] - features = InstanceFeaturesExtractor().extract([instance]) - for (v, classifier) in self.classifiers.items(): - proba = classifier.predict_proba(features) - if proba[0][1] > self.threshold: - violations += [v] - return violations - - def evaluate(self, instances): - results = {} - all_violations = set() - for instance in instances: - all_violations |= set(instance.found_violated_lazy_constraints) - for idx in tqdm( - range(len(instances)), - desc="Evaluate (lazy)", - disable=not sys.stdout.isatty(), - ): - instance = instances[idx] - condition_positive = set(instance.found_violated_lazy_constraints) - condition_negative = all_violations - condition_positive - pred_positive = set(self.predict(instance)) & all_violations - pred_negative = all_violations - pred_positive - tp = len(pred_positive & condition_positive) - tn = len(pred_negative & condition_negative) - fp = len(pred_positive & condition_negative) - fn = len(pred_negative & condition_positive) - results[idx] = classifier_evaluation_dict(tp, tn, fp, fn) - return results - - def fit_new(self, training_instances: List["Instance"]) -> None: - # Update known_cids + else: + instance.training_data[-1].lazy_enforced |= set(cids) + logger.debug(" %d violations found" % len(cids)) + self.enforce(cids, instance, model, solver) + return True + + def sample_xy_with_cids( + self, + instance: "Instance", + sample: TrainingSample, + ) -> Tuple[ + Dict[Hashable, List[List[float]]], + Dict[Hashable, List[List[bool]]], + Dict[Hashable, List[str]], + ]: + x: Dict[Hashable, List[List[float]]] = {} + y: Dict[Hashable, List[List[bool]]] = {} + cids: Dict[Hashable, List[str]] = {} + for cid in self.known_cids: + category = instance.get_constraint_category(cid) + if category is None: + continue + if category not in x: + x[category] = [] + y[category] = [] + cids[category] = [] + assert instance.features.instance is not None + assert instance.features.instance.user_features is not None + cfeatures = instance.get_constraint_features(cid) + assert cfeatures is not None + assert isinstance(cfeatures, list) + for ci in cfeatures: + assert isinstance(ci, float) + f = list(instance.features.instance.user_features) + f += cfeatures + x[category] += [f] + cids[category] += [cid] + if sample.lazy_enforced is not None: + if cid in sample.lazy_enforced: + y[category] += [[False, True]] + else: + y[category] += [[True, False]] + return x, y, cids + + def sample_xy( + self, + instance: "Instance", + sample: TrainingSample, + ) -> Tuple[Dict, Dict]: + x, y, _ = self.sample_xy_with_cids(instance, sample) + return x, y + + def sample_predict( + self, + instance: "Instance", + sample: TrainingSample, + ) -> List[str]: + pred: List[str] = [] + x, _, cids = self.sample_xy_with_cids(instance, sample) + for category in x.keys(): + assert category in self.classifiers + assert category in self.thresholds + clf = self.classifiers[category] + thr = self.thresholds[category] + nx = np.array(x[category]) + proba = clf.predict_proba(nx) + t = thr.predict(nx) + for i in range(proba.shape[0]): + if proba[i][1] > t[1]: + pred += [cids[category][i]] + return pred + + def fit(self, training_instances: List["Instance"]) -> None: self.known_cids.clear() for instance in training_instances: for sample in instance.training_data: @@ -133,40 +144,57 @@ class DynamicLazyConstraintsComponent(Component): continue self.known_cids += list(sample.lazy_enforced) self.known_cids = sorted(set(self.known_cids)) + super().fit(training_instances) - # Build x and y matrices - x: Dict[Hashable, List[List[float]]] = {} - y: Dict[Hashable, List[List[bool]]] = {} - for instance in training_instances: - for sample in instance.training_data: - if sample.lazy_enforced is None: - continue - for cid in self.known_cids: - category = instance.get_constraint_category(cid) - if category is None: - continue - if category not in x: - x[category] = [] - y[category] = [] - assert instance.features.instance is not None - assert instance.features.instance.user_features is not None - cfeatures = instance.get_constraint_features(cid) - assert cfeatures is not None - assert isinstance(cfeatures, list) - for ci in cfeatures: - assert isinstance(ci, float) - f = list(instance.features.instance.user_features) - f += cfeatures - x[category] += [f] - if cid in sample.lazy_enforced: - y[category] += [[False, True]] - else: - y[category] += [[True, False]] - - # Train classifiers + def fit_xy( + self, + x: Dict[Hashable, np.ndarray], + y: Dict[Hashable, np.ndarray], + ) -> None: for category in x.keys(): self.classifiers[category] = self.classifier_prototype.clone() - self.classifiers[category].fit( - np.array(x[category]), - np.array(y[category]), + self.thresholds[category] = self.threshold_prototype.clone() + npx = np.array(x[category]) + npy = np.array(y[category]) + self.classifiers[category].fit(npx, npy) + self.thresholds[category].fit(self.classifiers[category], npx, npy) + + def sample_evaluate( + self, + instance: "Instance", + sample: TrainingSample, + ) -> Dict[Hashable, Dict[str, float]]: + assert sample.lazy_enforced is not None + pred = set(self.sample_predict(instance, sample)) + tp: Dict[Hashable, int] = {} + tn: Dict[Hashable, int] = {} + fp: Dict[Hashable, int] = {} + fn: Dict[Hashable, int] = {} + for cid in self.known_cids: + category = instance.get_constraint_category(cid) + if category is None: + continue + if category not in tp.keys(): + tp[category] = 0 + tn[category] = 0 + fp[category] = 0 + fn[category] = 0 + if cid in pred: + if cid in sample.lazy_enforced: + tp[category] += 1 + else: + fp[category] += 1 + else: + if cid in sample.lazy_enforced: + fn[category] += 1 + else: + tn[category] += 1 + return { + category: classifier_evaluation_dict( + tp=tp[category], + tn=tn[category], + fp=fp[category], + fn=fn[category], ) + for category in tp.keys() + } diff --git a/tests/components/test_lazy_dynamic.py b/tests/components/test_lazy_dynamic.py index e9a862b..cb2f82c 100644 --- a/tests/components/test_lazy_dynamic.py +++ b/tests/components/test_lazy_dynamic.py @@ -6,181 +6,22 @@ from unittest.mock import Mock import numpy as np import pytest -from numpy.linalg import norm from numpy.testing import assert_array_equal from miplearn import Instance from miplearn.classifiers import Classifier +from miplearn.classifiers.threshold import MinProbabilityThreshold +from miplearn.components import classifier_evaluation_dict from miplearn.components.lazy_dynamic import DynamicLazyConstraintsComponent from miplearn.features import ( TrainingSample, Features, - ConstraintFeatures, InstanceFeatures, ) -from miplearn.solvers.internal import InternalSolver -from miplearn.solvers.learning import LearningSolver -from tests.fixtures.knapsack import get_test_pyomo_instances E = 0.1 -def test_lazy_fit(): - instances, models = get_test_pyomo_instances() - instances[0].found_violated_lazy_constraints = ["a", "b"] - instances[1].found_violated_lazy_constraints = ["b", "c"] - classifier = Mock(spec=Classifier) - classifier.clone = lambda: Mock(spec=Classifier) - component = DynamicLazyConstraintsComponent(classifier=classifier) - - component.fit(instances) - - # Should create one classifier for each violation - assert "a" in component.classifiers - assert "b" in component.classifiers - assert "c" in component.classifiers - - # Should provide correct x_train to each classifier - expected_x_train_a = np.array([[67.0, 21.75, 1287.92], [70.0, 23.75, 1199.83]]) - expected_x_train_b = np.array([[67.0, 21.75, 1287.92], [70.0, 23.75, 1199.83]]) - expected_x_train_c = np.array([[67.0, 21.75, 1287.92], [70.0, 23.75, 1199.83]]) - actual_x_train_a = component.classifiers["a"].fit.call_args[0][0] - actual_x_train_b = component.classifiers["b"].fit.call_args[0][0] - actual_x_train_c = component.classifiers["c"].fit.call_args[0][0] - assert norm(expected_x_train_a - actual_x_train_a) < E - assert norm(expected_x_train_b - actual_x_train_b) < E - assert norm(expected_x_train_c - actual_x_train_c) < E - - # Should provide correct y_train to each classifier - expected_y_train_a = np.array( - [ - [False, True], - [True, False], - ] - ) - expected_y_train_b = np.array( - [ - [False, True], - [False, True], - ] - ) - expected_y_train_c = np.array( - [ - [True, False], - [False, True], - ] - ) - assert_array_equal( - component.classifiers["a"].fit.call_args[0][1], - expected_y_train_a, - ) - assert_array_equal( - component.classifiers["b"].fit.call_args[0][1], - expected_y_train_b, - ) - assert_array_equal( - component.classifiers["c"].fit.call_args[0][1], - expected_y_train_c, - ) - - -def test_lazy_before(): - instances, models = get_test_pyomo_instances() - instances[0].build_lazy_constraint = Mock(return_value="c1") - solver = LearningSolver() - solver.internal_solver = Mock(spec=InternalSolver) - component = DynamicLazyConstraintsComponent(threshold=0.10) - component.classifiers = {"a": Mock(spec=Classifier), "b": Mock(spec=Classifier)} - component.classifiers["a"].predict_proba = Mock(return_value=[[0.95, 0.05]]) - component.classifiers["b"].predict_proba = Mock(return_value=[[0.02, 0.80]]) - - component.before_solve_mip( - solver=solver, - instance=instances[0], - model=models[0], - stats=None, - features=None, - training_data=None, - ) - - # Should ask classifier likelihood of each constraint being violated - expected_x_test_a = np.array([[67.0, 21.75, 1287.92]]) - expected_x_test_b = np.array([[67.0, 21.75, 1287.92]]) - actual_x_test_a = component.classifiers["a"].predict_proba.call_args[0][0] - actual_x_test_b = component.classifiers["b"].predict_proba.call_args[0][0] - assert norm(expected_x_test_a - actual_x_test_a) < E - assert norm(expected_x_test_b - actual_x_test_b) < E - - # Should ask instance to generate cut for constraints whose likelihood - # of being violated exceeds the threshold - instances[0].build_lazy_constraint.assert_called_once_with(models[0], "b") - - # Should ask internal solver to add generated constraint - solver.internal_solver.add_constraint.assert_called_once_with("c1") - - -def test_lazy_evaluate(): - instances, models = get_test_pyomo_instances() - component = DynamicLazyConstraintsComponent() - component.classifiers = { - "a": Mock(spec=Classifier), - "b": Mock(spec=Classifier), - "c": Mock(spec=Classifier), - } - component.classifiers["a"].predict_proba = Mock(return_value=[[1.0, 0.0]]) - component.classifiers["b"].predict_proba = Mock(return_value=[[0.0, 1.0]]) - component.classifiers["c"].predict_proba = Mock(return_value=[[0.0, 1.0]]) - - instances[0].found_violated_lazy_constraints = ["a", "b", "c"] - instances[1].found_violated_lazy_constraints = ["b", "d"] - assert component.evaluate(instances) == { - 0: { - "Accuracy": 0.75, - "F1 score": 0.8, - "Precision": 1.0, - "Recall": 2 / 3.0, - "Predicted positive": 2, - "Predicted negative": 2, - "Condition positive": 3, - "Condition negative": 1, - "False negative": 1, - "False positive": 0, - "True negative": 1, - "True positive": 2, - "Predicted positive (%)": 50.0, - "Predicted negative (%)": 50.0, - "Condition positive (%)": 75.0, - "Condition negative (%)": 25.0, - "False negative (%)": 25.0, - "False positive (%)": 0, - "True negative (%)": 25.0, - "True positive (%)": 50.0, - }, - 1: { - "Accuracy": 0.5, - "F1 score": 0.5, - "Precision": 0.5, - "Recall": 0.5, - "Predicted positive": 2, - "Predicted negative": 2, - "Condition positive": 2, - "Condition negative": 2, - "False negative": 1, - "False positive": 1, - "True negative": 1, - "True positive": 1, - "Predicted positive (%)": 50.0, - "Predicted negative (%)": 50.0, - "Condition positive (%)": 50.0, - "Condition negative (%)": 50.0, - "False negative (%)": 25.0, - "False positive (%)": 25.0, - "True negative (%)": 25.0, - "True positive (%)": 25.0, - }, - } - - @pytest.fixture def training_instances() -> List[Instance]: instances = [cast(Instance, Mock(spec=Instance)) for _ in range(2)] @@ -235,11 +76,11 @@ def training_instances() -> List[Instance]: return instances -def test_fit_new(training_instances: List[Instance]) -> None: +def test_fit(training_instances: List[Instance]) -> None: clf = Mock(spec=Classifier) clf.clone = Mock(side_effect=lambda: Mock(spec=Classifier)) comp = DynamicLazyConstraintsComponent(classifier=clf) - comp.fit_new(training_instances) + comp.fit(training_instances) assert clf.clone.call_count == 2 assert "type-a" in comp.classifiers @@ -299,3 +140,32 @@ def test_fit_new(training_instances: List[Instance]) -> None: ] ), ) + + +def test_sample_predict_evaluate(training_instances: List[Instance]) -> None: + comp = DynamicLazyConstraintsComponent() + comp.known_cids = ["c1", "c2", "c3", "c4"] + comp.thresholds["type-a"] = MinProbabilityThreshold([0.5, 0.5]) + comp.thresholds["type-b"] = MinProbabilityThreshold([0.5, 0.5]) + comp.classifiers["type-a"] = Mock(spec=Classifier) + comp.classifiers["type-b"] = Mock(spec=Classifier) + comp.classifiers["type-a"].predict_proba = Mock( # type: ignore + side_effect=lambda _: np.array([[0.1, 0.9], [0.8, 0.2]]) + ) + comp.classifiers["type-b"].predict_proba = Mock( # type: ignore + side_effect=lambda _: np.array([[0.9, 0.1], [0.1, 0.9]]) + ) + pred = comp.sample_predict( + training_instances[0], + training_instances[0].training_data[0], + ) + assert pred == ["c1", "c4"] + ev = comp.sample_evaluate( + training_instances[0], + training_instances[0].training_data[0], + ) + print(ev) + assert ev == { + "type-a": classifier_evaluation_dict(tp=1, fp=0, tn=0, fn=1), + "type-b": classifier_evaluation_dict(tp=0, fp=1, tn=1, fn=0), + } diff --git a/tests/problems/test_tsp.py b/tests/problems/test_tsp.py index 1155991..908f560 100644 --- a/tests/problems/test_tsp.py +++ b/tests/problems/test_tsp.py @@ -66,7 +66,7 @@ def test_subtour(): instance = TravelingSalesmanInstance(n_cities, distances) solver = LearningSolver() solver.solve(instance) - assert hasattr(instance, "found_violated_lazy_constraints") + assert len(instance.training_data[0].lazy_enforced) > 0 assert hasattr(instance, "found_violated_user_cuts") x = instance.training_data[0].solution["x"] assert x[0, 1] == 1.0