Finish DynamicLazyConstraintsComponent rewrite

miplearn/components/component.py

@@ -106,8 +106,8 @@ class Component:
         """
         return
 
-    @staticmethod
     def sample_xy(
+        self,
         instance: Instance,
         sample: TrainingSample,
     ) -> Tuple[Dict, Dict]:

miplearn/components/lazy_dynamic.py

@@ -3,17 +3,16 @@
 #  Released under the modified BSD license. See COPYING.md for more details.
 
 import logging
-import sys
-from typing import Any, Dict, List, TYPE_CHECKING, Hashable
+from typing import Dict, List, TYPE_CHECKING, Hashable, Tuple
 
 import numpy as np
-from tqdm.auto import tqdm
 
 from miplearn.classifiers import Classifier
 from miplearn.classifiers.counting import CountingClassifier
+from miplearn.classifiers.threshold import MinProbabilityThreshold, Threshold
 from miplearn.components import classifier_evaluation_dict
 from miplearn.components.component import Component
-from miplearn.extractors import InstanceFeaturesExtractor
+from miplearn.features import TrainingSample
 
 logger = logging.getLogger(__name__)
 
@@ -29,14 +28,21 @@ class DynamicLazyConstraintsComponent(Component):
     def __init__(
         self,
         classifier: Classifier = CountingClassifier(),
-        threshold: float = 0.05,
+        threshold: Threshold = MinProbabilityThreshold([0, 0.05]),
     ):
         assert isinstance(classifier, Classifier)
-        self.threshold: float = threshold
+        self.threshold_prototype: Threshold = threshold
         self.classifier_prototype: Classifier = classifier
-        self.classifiers: Dict[Any, Classifier] = {}
+        self.classifiers: Dict[Hashable, Classifier] = {}
+        self.thresholds: Dict[Hashable, Threshold] = {}
         self.known_cids: List[str] = []
 
+    @staticmethod
+    def enforce(cids, instance, model, solver):
+        for cid in cids:
+            cobj = instance.build_lazy_constraint(model, cid)
+            solver.internal_solver.add_constraint(cobj)
+
     def before_solve_mip(
         self,
         solver,
@@ -46,101 +52,36 @@ class DynamicLazyConstraintsComponent(Component):
         features,
         training_data,
     ):
-        instance.found_violated_lazy_constraints = []
+        training_data.lazy_enforced = set()
         logger.info("Predicting violated lazy constraints...")
-        violations = self.predict(instance)
-        logger.info("Enforcing %d lazy constraints..." % len(violations))
-        for v in violations:
-            cut = instance.build_lazy_constraint(model, v)
-            solver.internal_solver.add_constraint(cut)
+        cids = self.sample_predict(instance, training_data)
+        logger.info("Enforcing %d lazy constraints..." % len(cids))
+        self.enforce(cids, instance, model, solver)
 
     def iteration_cb(self, solver, instance, model):
-        logger.debug("Finding violated (dynamic) lazy constraints...")
-        violations = instance.find_violated_lazy_constraints(model)
-        if len(violations) == 0:
+        logger.debug("Finding violated lazy constraints...")
+        cids = instance.find_violated_lazy_constraints(model)
+        if len(cids) == 0:
+            logger.debug("No violations found")
             return False
-        instance.found_violated_lazy_constraints += violations
-        logger.debug("    %d violations found" % len(violations))
-        for v in violations:
-            cut = instance.build_lazy_constraint(model, v)
-            solver.internal_solver.add_constraint(cut)
+        else:
+            instance.training_data[-1].lazy_enforced |= set(cids)
+            logger.debug("    %d violations found" % len(cids))
+            self.enforce(cids, instance, model, solver)
             return True
 
-    def fit(self, training_instances):
-        logger.debug("Fitting...")
-        features = InstanceFeaturesExtractor().extract(training_instances)
-
-        self.classifiers = {}
-        violation_to_instance_idx = {}
-        for (idx, instance) in enumerate(training_instances):
-            for v in instance.found_violated_lazy_constraints:
-                if isinstance(v, list):
-                    v = tuple(v)
-                if v not in self.classifiers:
-                    self.classifiers[v] = self.classifier_prototype.clone()
-                    violation_to_instance_idx[v] = []
-                violation_to_instance_idx[v] += [idx]
-
-        for (v, classifier) in tqdm(
-            self.classifiers.items(),
-            desc="Fit (lazy)",
-            disable=not sys.stdout.isatty(),
-        ):
-            logger.debug("Training: %s" % (str(v)))
-            label = [[True, False] for i in training_instances]
-            for idx in violation_to_instance_idx[v]:
-                label[idx] = [False, True]
-            label = np.array(label, dtype=np.bool8)
-            classifier.fit(features, label)
-
-    def predict(self, instance):
-        violations = []
-        features = InstanceFeaturesExtractor().extract([instance])
-        for (v, classifier) in self.classifiers.items():
-            proba = classifier.predict_proba(features)
-            if proba[0][1] > self.threshold:
-                violations += [v]
-        return violations
-
-    def evaluate(self, instances):
-        results = {}
-        all_violations = set()
-        for instance in instances:
-            all_violations |= set(instance.found_violated_lazy_constraints)
-        for idx in tqdm(
-            range(len(instances)),
-            desc="Evaluate (lazy)",
-            disable=not sys.stdout.isatty(),
-        ):
-            instance = instances[idx]
-            condition_positive = set(instance.found_violated_lazy_constraints)
-            condition_negative = all_violations - condition_positive
-            pred_positive = set(self.predict(instance)) & all_violations
-            pred_negative = all_violations - pred_positive
-            tp = len(pred_positive & condition_positive)
-            tn = len(pred_negative & condition_negative)
-            fp = len(pred_positive & condition_negative)
-            fn = len(pred_negative & condition_positive)
-            results[idx] = classifier_evaluation_dict(tp, tn, fp, fn)
-        return results
-
-    def fit_new(self, training_instances: List["Instance"]) -> None:
-        # Update known_cids
-        self.known_cids.clear()
-        for instance in training_instances:
-            for sample in instance.training_data:
-                if sample.lazy_enforced is None:
-                    continue
-                self.known_cids += list(sample.lazy_enforced)
-        self.known_cids = sorted(set(self.known_cids))
-
-        # Build x and y matrices
+    def sample_xy_with_cids(
+        self,
+        instance: "Instance",
+        sample: TrainingSample,
+    ) -> Tuple[
+        Dict[Hashable, List[List[float]]],
+        Dict[Hashable, List[List[bool]]],
+        Dict[Hashable, List[str]],
+    ]:
         x: Dict[Hashable, List[List[float]]] = {}
         y: Dict[Hashable, List[List[bool]]] = {}
-        for instance in training_instances:
-            for sample in instance.training_data:
-                if sample.lazy_enforced is None:
-                    continue
+        cids: Dict[Hashable, List[str]] = {}
         for cid in self.known_cids:
             category = instance.get_constraint_category(cid)
             if category is None:
@@ -148,6 +89,7 @@ class DynamicLazyConstraintsComponent(Component):
             if category not in x:
                 x[category] = []
                 y[category] = []
+                cids[category] = []
             assert instance.features.instance is not None
             assert instance.features.instance.user_features is not None
             cfeatures = instance.get_constraint_features(cid)
@@ -158,15 +100,101 @@ class DynamicLazyConstraintsComponent(Component):
             f = list(instance.features.instance.user_features)
             f += cfeatures
             x[category] += [f]
+            cids[category] += [cid]
+            if sample.lazy_enforced is not None:
                 if cid in sample.lazy_enforced:
                     y[category] += [[False, True]]
                 else:
                     y[category] += [[True, False]]
+        return x, y, cids
+
+    def sample_xy(
+        self,
+        instance: "Instance",
+        sample: TrainingSample,
+    ) -> Tuple[Dict, Dict]:
+        x, y, _ = self.sample_xy_with_cids(instance, sample)
+        return x, y
 
-        # Train classifiers
+    def sample_predict(
+        self,
+        instance: "Instance",
+        sample: TrainingSample,
+    ) -> List[str]:
+        pred: List[str] = []
+        x, _, cids = self.sample_xy_with_cids(instance, sample)
+        for category in x.keys():
+            assert category in self.classifiers
+            assert category in self.thresholds
+            clf = self.classifiers[category]
+            thr = self.thresholds[category]
+            nx = np.array(x[category])
+            proba = clf.predict_proba(nx)
+            t = thr.predict(nx)
+            for i in range(proba.shape[0]):
+                if proba[i][1] > t[1]:
+                    pred += [cids[category][i]]
+        return pred
+
+    def fit(self, training_instances: List["Instance"]) -> None:
+        self.known_cids.clear()
+        for instance in training_instances:
+            for sample in instance.training_data:
+                if sample.lazy_enforced is None:
+                    continue
+                self.known_cids += list(sample.lazy_enforced)
+        self.known_cids = sorted(set(self.known_cids))
+        super().fit(training_instances)
+
+    def fit_xy(
+        self,
+        x: Dict[Hashable, np.ndarray],
+        y: Dict[Hashable, np.ndarray],
+    ) -> None:
         for category in x.keys():
             self.classifiers[category] = self.classifier_prototype.clone()
-            self.classifiers[category].fit(
-                np.array(x[category]),
-                np.array(y[category]),
+            self.thresholds[category] = self.threshold_prototype.clone()
+            npx = np.array(x[category])
+            npy = np.array(y[category])
+            self.classifiers[category].fit(npx, npy)
+            self.thresholds[category].fit(self.classifiers[category], npx, npy)
+
+    def sample_evaluate(
+        self,
+        instance: "Instance",
+        sample: TrainingSample,
+    ) -> Dict[Hashable, Dict[str, float]]:
+        assert sample.lazy_enforced is not None
+        pred = set(self.sample_predict(instance, sample))
+        tp: Dict[Hashable, int] = {}
+        tn: Dict[Hashable, int] = {}
+        fp: Dict[Hashable, int] = {}
+        fn: Dict[Hashable, int] = {}
+        for cid in self.known_cids:
+            category = instance.get_constraint_category(cid)
+            if category is None:
+                continue
+            if category not in tp.keys():
+                tp[category] = 0
+                tn[category] = 0
+                fp[category] = 0
+                fn[category] = 0
+            if cid in pred:
+                if cid in sample.lazy_enforced:
+                    tp[category] += 1
+                else:
+                    fp[category] += 1
+            else:
+                if cid in sample.lazy_enforced:
+                    fn[category] += 1
+                else:
+                    tn[category] += 1
+        return {
+            category: classifier_evaluation_dict(
+                tp=tp[category],
+                tn=tn[category],
+                fp=fp[category],
+                fn=fn[category],
             )
+            for category in tp.keys()
+        }

tests/components/test_lazy_dynamic.py

@@ -6,181 +6,22 @@ from unittest.mock import Mock
 
 import numpy as np
 import pytest
-from numpy.linalg import norm
 from numpy.testing import assert_array_equal
 
 from miplearn import Instance
 from miplearn.classifiers import Classifier
+from miplearn.classifiers.threshold import MinProbabilityThreshold
+from miplearn.components import classifier_evaluation_dict
 from miplearn.components.lazy_dynamic import DynamicLazyConstraintsComponent
 from miplearn.features import (
     TrainingSample,
     Features,
-    ConstraintFeatures,
     InstanceFeatures,
 )
-from miplearn.solvers.internal import InternalSolver
-from miplearn.solvers.learning import LearningSolver
-from tests.fixtures.knapsack import get_test_pyomo_instances
 
 E = 0.1
 
 
-def test_lazy_fit():
-    instances, models = get_test_pyomo_instances()
-    instances[0].found_violated_lazy_constraints = ["a", "b"]
-    instances[1].found_violated_lazy_constraints = ["b", "c"]
-    classifier = Mock(spec=Classifier)
-    classifier.clone = lambda: Mock(spec=Classifier)
-    component = DynamicLazyConstraintsComponent(classifier=classifier)
-
-    component.fit(instances)
-
-    # Should create one classifier for each violation
-    assert "a" in component.classifiers
-    assert "b" in component.classifiers
-    assert "c" in component.classifiers
-
-    # Should provide correct x_train to each classifier
-    expected_x_train_a = np.array([[67.0, 21.75, 1287.92], [70.0, 23.75, 1199.83]])
-    expected_x_train_b = np.array([[67.0, 21.75, 1287.92], [70.0, 23.75, 1199.83]])
-    expected_x_train_c = np.array([[67.0, 21.75, 1287.92], [70.0, 23.75, 1199.83]])
-    actual_x_train_a = component.classifiers["a"].fit.call_args[0][0]
-    actual_x_train_b = component.classifiers["b"].fit.call_args[0][0]
-    actual_x_train_c = component.classifiers["c"].fit.call_args[0][0]
-    assert norm(expected_x_train_a - actual_x_train_a) < E
-    assert norm(expected_x_train_b - actual_x_train_b) < E
-    assert norm(expected_x_train_c - actual_x_train_c) < E
-
-    # Should provide correct y_train to each classifier
-    expected_y_train_a = np.array(
-        [
-            [False, True],
-            [True, False],
-        ]
-    )
-    expected_y_train_b = np.array(
-        [
-            [False, True],
-            [False, True],
-        ]
-    )
-    expected_y_train_c = np.array(
-        [
-            [True, False],
-            [False, True],
-        ]
-    )
-    assert_array_equal(
-        component.classifiers["a"].fit.call_args[0][1],
-        expected_y_train_a,
-    )
-    assert_array_equal(
-        component.classifiers["b"].fit.call_args[0][1],
-        expected_y_train_b,
-    )
-    assert_array_equal(
-        component.classifiers["c"].fit.call_args[0][1],
-        expected_y_train_c,
-    )
-
-
-def test_lazy_before():
-    instances, models = get_test_pyomo_instances()
-    instances[0].build_lazy_constraint = Mock(return_value="c1")
-    solver = LearningSolver()
-    solver.internal_solver = Mock(spec=InternalSolver)
-    component = DynamicLazyConstraintsComponent(threshold=0.10)
-    component.classifiers = {"a": Mock(spec=Classifier), "b": Mock(spec=Classifier)}
-    component.classifiers["a"].predict_proba = Mock(return_value=[[0.95, 0.05]])
-    component.classifiers["b"].predict_proba = Mock(return_value=[[0.02, 0.80]])
-
-    component.before_solve_mip(
-        solver=solver,
-        instance=instances[0],
-        model=models[0],
-        stats=None,
-        features=None,
-        training_data=None,
-    )
-
-    # Should ask classifier likelihood of each constraint being violated
-    expected_x_test_a = np.array([[67.0, 21.75, 1287.92]])
-    expected_x_test_b = np.array([[67.0, 21.75, 1287.92]])
-    actual_x_test_a = component.classifiers["a"].predict_proba.call_args[0][0]
-    actual_x_test_b = component.classifiers["b"].predict_proba.call_args[0][0]
-    assert norm(expected_x_test_a - actual_x_test_a) < E
-    assert norm(expected_x_test_b - actual_x_test_b) < E
-
-    # Should ask instance to generate cut for constraints whose likelihood
-    # of being violated exceeds the threshold
-    instances[0].build_lazy_constraint.assert_called_once_with(models[0], "b")
-
-    # Should ask internal solver to add generated constraint
-    solver.internal_solver.add_constraint.assert_called_once_with("c1")
-
-
-def test_lazy_evaluate():
-    instances, models = get_test_pyomo_instances()
-    component = DynamicLazyConstraintsComponent()
-    component.classifiers = {
-        "a": Mock(spec=Classifier),
-        "b": Mock(spec=Classifier),
-        "c": Mock(spec=Classifier),
-    }
-    component.classifiers["a"].predict_proba = Mock(return_value=[[1.0, 0.0]])
-    component.classifiers["b"].predict_proba = Mock(return_value=[[0.0, 1.0]])
-    component.classifiers["c"].predict_proba = Mock(return_value=[[0.0, 1.0]])
-
-    instances[0].found_violated_lazy_constraints = ["a", "b", "c"]
-    instances[1].found_violated_lazy_constraints = ["b", "d"]
-    assert component.evaluate(instances) == {
-        0: {
-            "Accuracy": 0.75,
-            "F1 score": 0.8,
-            "Precision": 1.0,
-            "Recall": 2 / 3.0,
-            "Predicted positive": 2,
-            "Predicted negative": 2,
-            "Condition positive": 3,
-            "Condition negative": 1,
-            "False negative": 1,
-            "False positive": 0,
-            "True negative": 1,
-            "True positive": 2,
-            "Predicted positive (%)": 50.0,
-            "Predicted negative (%)": 50.0,
-            "Condition positive (%)": 75.0,
-            "Condition negative (%)": 25.0,
-            "False negative (%)": 25.0,
-            "False positive (%)": 0,
-            "True negative (%)": 25.0,
-            "True positive (%)": 50.0,
-        },
-        1: {
-            "Accuracy": 0.5,
-            "F1 score": 0.5,
-            "Precision": 0.5,
-            "Recall": 0.5,
-            "Predicted positive": 2,
-            "Predicted negative": 2,
-            "Condition positive": 2,
-            "Condition negative": 2,
-            "False negative": 1,
-            "False positive": 1,
-            "True negative": 1,
-            "True positive": 1,
-            "Predicted positive (%)": 50.0,
-            "Predicted negative (%)": 50.0,
-            "Condition positive (%)": 50.0,
-            "Condition negative (%)": 50.0,
-            "False negative (%)": 25.0,
-            "False positive (%)": 25.0,
-            "True negative (%)": 25.0,
-            "True positive (%)": 25.0,
-        },
-    }
-
-
 @pytest.fixture
 def training_instances() -> List[Instance]:
     instances = [cast(Instance, Mock(spec=Instance)) for _ in range(2)]
@@ -235,11 +76,11 @@ def training_instances() -> List[Instance]:
     return instances
 
 
-def test_fit_new(training_instances: List[Instance]) -> None:
+def test_fit(training_instances: List[Instance]) -> None:
     clf = Mock(spec=Classifier)
     clf.clone = Mock(side_effect=lambda: Mock(spec=Classifier))
     comp = DynamicLazyConstraintsComponent(classifier=clf)
-    comp.fit_new(training_instances)
+    comp.fit(training_instances)
     assert clf.clone.call_count == 2
 
     assert "type-a" in comp.classifiers
@@ -299,3 +140,32 @@ def test_fit_new(training_instances: List[Instance]) -> None:
             ]
         ),
     )
+
+
+def test_sample_predict_evaluate(training_instances: List[Instance]) -> None:
+    comp = DynamicLazyConstraintsComponent()
+    comp.known_cids = ["c1", "c2", "c3", "c4"]
+    comp.thresholds["type-a"] = MinProbabilityThreshold([0.5, 0.5])
+    comp.thresholds["type-b"] = MinProbabilityThreshold([0.5, 0.5])
+    comp.classifiers["type-a"] = Mock(spec=Classifier)
+    comp.classifiers["type-b"] = Mock(spec=Classifier)
+    comp.classifiers["type-a"].predict_proba = Mock(  # type: ignore
+        side_effect=lambda _: np.array([[0.1, 0.9], [0.8, 0.2]])
+    )
+    comp.classifiers["type-b"].predict_proba = Mock(  # type: ignore
+        side_effect=lambda _: np.array([[0.9, 0.1], [0.1, 0.9]])
+    )
+    pred = comp.sample_predict(
+        training_instances[0],
+        training_instances[0].training_data[0],
+    )
+    assert pred == ["c1", "c4"]
+    ev = comp.sample_evaluate(
+        training_instances[0],
+        training_instances[0].training_data[0],
+    )
+    print(ev)
+    assert ev == {
+        "type-a": classifier_evaluation_dict(tp=1, fp=0, tn=0, fn=1),
+        "type-b": classifier_evaluation_dict(tp=0, fp=1, tn=1, fn=0),
+    }

tests/problems/test_tsp.py

@@ -66,7 +66,7 @@ def test_subtour():
     instance = TravelingSalesmanInstance(n_cities, distances)
     solver = LearningSolver()
     solver.solve(instance)
-    assert hasattr(instance, "found_violated_lazy_constraints")
+    assert len(instance.training_data[0].lazy_enforced) > 0
     assert hasattr(instance, "found_violated_user_cuts")
     x = instance.training_data[0].solution["x"]
     assert x[0, 1] == 1.0