Refactor thresholds

docs/customization.md

@@ -63,16 +63,16 @@ solver2 = LearningSolver(components=[
 
 ### Adjusting component aggressiveness
 
-The aggressiveness of classification components (such as `PrimalSolutionComponent` and `LazyConstraintComponent`) can
+The aggressiveness of classification components, such as `PrimalSolutionComponent` and `LazyConstraintComponent`, can be adjusted through the `threshold` constructor argument. Internally, these components ask the machine learning models how confident are they on each prediction they make, then automatically discard all predictions that have low confidence. The `threshold` argument specifies how confident should the ML models be for a prediction to be considered trustworthy. Lowering a component's threshold increases its aggressiveness, while raising a component's threshold makes it more conservative.
-be adjusted through the `threshold` constructor argument. Internally, these components ask the ML models how confident
-they are on each prediction (through the `predict_proba` method in the sklearn API), and only take into account
-predictions which have probabilities above the threshold. Lowering a component's threshold increases its aggressiveness,
-while raising a component's threshold makes it more conservative. 
 
-MIPLearn also includes `MinPrecisionThreshold`, a dynamic threshold which adjusts itself automatically during training
+For example, if the ML model predicts that a certain binary variable will assume value `1.0` in the optimal solution with 75% confidence, and if the `PrimalSolutionComponent` is configured to discard all predictions with less than 90% confidence, then this variable will not be included in the predicted MIP start.
-to achieve a minimum desired true positive rate (also known as precision). The example below shows how to initialize
+
-a `PrimalSolutionComponent` which achieves 95% precision, possibly at the cost of a lower recall. To make the component
+MIPLearn currently provides two types of thresholds:
-more aggressive, this precision may be lowered.
+
+* `MinProbabilityThreshold(p: float)` A threshold which indicates that a prediction is trustworthy if its probability of being correct, as computed by the machine learning model, is above a fixed value `p`.
+* `MinPrecisionThreshold(p: float)` A dynamic threshold which automatically adjusts itself during training to ensure that the component achieves at least a given precision `p` on the training data set. Note that increasing a component's precision may reduce its recall.
+
+The example below shows how to configure `PrimalSolutionComponent` to achieve at least 95% precision. Other components are configured similarly.
 
 ```python
 PrimalSolutionComponent(threshold=MinPrecisionThreshold(0.95))

miplearn/classifiers/threshold.py

@@ -3,6 +3,7 @@
 #  Released under the modified BSD license. See COPYING.md for more details.
 
 from abc import abstractmethod, ABC
+from typing import Optional
 
 import numpy as np
 from sklearn.metrics._ranking import _binary_clf_curve
@@ -10,47 +11,83 @@ from sklearn.metrics._ranking import _binary_clf_curve
 from miplearn.classifiers import Classifier
 
 
-class DynamicThreshold(ABC):
+class Threshold(ABC):
+    """
+    Solver components ask the machine learning models how confident are they on each
+    prediction they make, then automatically discard all predictions that have low
+    confidence. A Threshold specifies how confident should the ML models be for a
+    prediction to be considered trustworthy.
+
+    To model dynamic thresholds, which automatically adjust themselves during
+    training to reach some desired target (such as minimum precision, or minimum
+    recall), thresholds behave somewhat similar to ML models themselves, with `fit`
+    and `predict` methods.
+    """
+
     @abstractmethod
-    def find(
+    def fit(
         self,
         clf: Classifier,
         x_train: np.ndarray,
         y_train: np.ndarray,
-    ) -> float:
+    ) -> None:
         """
-        Given a trained binary classifier `clf` and a training data set,
+        Given a trained binary classifier `clf`, calibrates itself based on the
-        returns the numerical threshold (float) satisfying some criterea.
+        classifier's performance on the given training data set.
+        """
+        assert isinstance(clf, Classifier)
+        assert isinstance(x_train, np.ndarray)
+        assert isinstance(y_train, np.ndarray)
+        n_samples = x_train.shape[0]
+        assert y_train.shape[0] == n_samples
+
+    @abstractmethod
+    def predict(self, x_test: np.ndarray) -> float:
+        """
+        Returns the minimum probability for a machine learning prediction to be
+        considered trustworthy.
         """
         pass
 
 
-class MinPrecisionThreshold(DynamicThreshold):
+class MinProbabilityThreshold(Threshold):
     """
-    The smallest possible threshold satisfying a minimum acceptable true
+    A threshold which considers predictions trustworthy if their probability of being
-    positive rate (also known as precision).
+    correct, as computed by the machine learning models, are above a fixed value.
+    """
+
+    def __init__(self, min_probability: float):
+        self.min_probability = min_probability
+
+    def fit(self, clf: Classifier, x_train: np.ndarray, y_train: np.ndarray) -> None:
+        pass
+
+    def predict(self, x_test: np.ndarray) -> float:
+        return self.min_probability
+
+
+class MinPrecisionThreshold(Threshold):
+    """
+    A dynamic threshold which automatically adjusts itself during training to ensure
+    that the component achieves at least a given precision `p` on the training data
+    set. Note that increasing a component's minimum precision may reduce its recall.
     """
 
     def __init__(self, min_precision: float) -> None:
         self.min_precision = min_precision
+        self._computed_threshold: Optional[float] = None
 
-    def find(self, clf, x_train, y_train):
+    def fit(self, clf: Classifier, x_train: np.ndarray, y_train: np.ndarray) -> None:
+        super().fit(clf, x_train, y_train)
         proba = clf.predict_proba(x_train)
-
-        assert isinstance(proba, np.ndarray), "classifier should return numpy array"
-        assert proba.shape == (
-            x_train.shape[0],
-            2,
-        ), "classifier should return (%d,%d)-shaped array, not %s" % (
-            x_train.shape[0],
-            2,
-            str(proba.shape),
-        )
-
         fps, tps, thresholds = _binary_clf_curve(y_train, proba[:, 1])
         precision = tps / (tps + fps)
-
         for k in reversed(range(len(precision))):
             if precision[k] >= self.min_precision:
-                return thresholds[k]
+                self._computed_threshold = thresholds[k]
-        return 2.0
+                return
+        self._computed_threshold = float("inf")
+
+    def predict(self, x_test: np.ndarray) -> float:
+        assert self._computed_threshold is not None
+        return self._computed_threshold

miplearn/components/primal.py

@@ -11,7 +11,7 @@ from tqdm.auto import tqdm
 
 from miplearn.classifiers import Classifier
 from miplearn.classifiers.adaptive import AdaptiveClassifier
-from miplearn.classifiers.threshold import MinPrecisionThreshold, DynamicThreshold
+from miplearn.classifiers.threshold import MinPrecisionThreshold, Threshold
 from miplearn.components import classifier_evaluation_dict
 from miplearn.components.component import Component
 from miplearn.extractors import VariableFeaturesExtractor, SolutionExtractor, Extractor
@@ -28,11 +28,11 @@ class PrimalSolutionComponent(Component):
         self,
         classifier: Classifier = AdaptiveClassifier(),
         mode: str = "exact",
-        threshold: Union[float, DynamicThreshold] = MinPrecisionThreshold(0.98),
+        threshold: Union[float, Threshold] = MinPrecisionThreshold(0.98),
     ) -> None:
         self.mode = mode
         self.classifiers: Dict[Any, Classifier] = {}
-        self.thresholds: Dict[Any, Union[float, DynamicThreshold]] = {}
+        self.thresholds: Dict[Any, Union[float, Threshold]] = {}
         self.threshold_prototype = threshold
         self.classifier_prototype = classifier
 
@@ -89,8 +89,8 @@ class PrimalSolutionComponent(Component):
                 clf.fit(x_train, y_train)
 
                 # Find threshold (dynamic or static)
-                if isinstance(self.threshold_prototype, DynamicThreshold):
+                if isinstance(self.threshold_prototype, Threshold):
-                    self.thresholds[category, label] = self.threshold_prototype.find(
+                    self.thresholds[category, label] = self.threshold_prototype.fit(
                         clf,
                         x_train,
                         y_train,

tests/classifiers/test_threshold.py

@@ -26,13 +26,17 @@ def test_threshold_dynamic():
     y_train = np.array([1, 1, 0, 0])
 
     threshold = MinPrecisionThreshold(min_precision=1.0)
-    assert threshold.find(clf, x_train, y_train) == 0.90
+    threshold.fit(clf, x_train, y_train)
+    assert threshold.predict(x_train) == 0.90
 
     threshold = MinPrecisionThreshold(min_precision=0.65)
-    assert threshold.find(clf, x_train, y_train) == 0.80
+    threshold.fit(clf, x_train, y_train)
+    assert threshold.predict(x_train) == 0.80
 
     threshold = MinPrecisionThreshold(min_precision=0.50)
-    assert threshold.find(clf, x_train, y_train) == 0.70
+    threshold.fit(clf, x_train, y_train)
+    assert threshold.predict(x_train) == 0.70
 
     threshold = MinPrecisionThreshold(min_precision=0.00)
-    assert threshold.find(clf, x_train, y_train) == 0.70
+    threshold.fit(clf, x_train, y_train)
+    assert threshold.predict(x_train) == 0.70