diff --git a/docs/customization.md b/docs/customization.md index d9bbedd..02e3a72 100644 --- a/docs/customization.md +++ b/docs/customization.md @@ -63,16 +63,16 @@ solver2 = LearningSolver(components=[ ### Adjusting component aggressiveness -The aggressiveness of classification components (such as `PrimalSolutionComponent` and `LazyConstraintComponent`) can -be adjusted through the `threshold` constructor argument. Internally, these components ask the ML models how confident -they are on each prediction (through the `predict_proba` method in the sklearn API), and only take into account -predictions which have probabilities above the threshold. Lowering a component's threshold increases its aggressiveness, -while raising a component's threshold makes it more conservative. - -MIPLearn also includes `MinPrecisionThreshold`, a dynamic threshold which adjusts itself automatically during training -to achieve a minimum desired true positive rate (also known as precision). The example below shows how to initialize -a `PrimalSolutionComponent` which achieves 95% precision, possibly at the cost of a lower recall. To make the component -more aggressive, this precision may be lowered. +The aggressiveness of classification components, such as `PrimalSolutionComponent` and `LazyConstraintComponent`, can be adjusted through the `threshold` constructor argument. Internally, these components ask the machine learning models how confident are they on each prediction they make, then automatically discard all predictions that have low confidence. The `threshold` argument specifies how confident should the ML models be for a prediction to be considered trustworthy. Lowering a component's threshold increases its aggressiveness, while raising a component's threshold makes it more conservative. + +For example, if the ML model predicts that a certain binary variable will assume value `1.0` in the optimal solution with 75% confidence, and if the `PrimalSolutionComponent` is configured to discard all predictions with less than 90% confidence, then this variable will not be included in the predicted MIP start. + +MIPLearn currently provides two types of thresholds: + +* `MinProbabilityThreshold(p: float)` A threshold which indicates that a prediction is trustworthy if its probability of being correct, as computed by the machine learning model, is above a fixed value `p`. +* `MinPrecisionThreshold(p: float)` A dynamic threshold which automatically adjusts itself during training to ensure that the component achieves at least a given precision `p` on the training data set. Note that increasing a component's precision may reduce its recall. + +The example below shows how to configure `PrimalSolutionComponent` to achieve at least 95% precision. Other components are configured similarly. ```python PrimalSolutionComponent(threshold=MinPrecisionThreshold(0.95)) diff --git a/miplearn/classifiers/threshold.py b/miplearn/classifiers/threshold.py index a3e5f02..6008e11 100644 --- a/miplearn/classifiers/threshold.py +++ b/miplearn/classifiers/threshold.py @@ -3,6 +3,7 @@ # Released under the modified BSD license. See COPYING.md for more details. from abc import abstractmethod, ABC +from typing import Optional import numpy as np from sklearn.metrics._ranking import _binary_clf_curve @@ -10,47 +11,83 @@ from sklearn.metrics._ranking import _binary_clf_curve from miplearn.classifiers import Classifier -class DynamicThreshold(ABC): +class Threshold(ABC): + """ + Solver components ask the machine learning models how confident are they on each + prediction they make, then automatically discard all predictions that have low + confidence. A Threshold specifies how confident should the ML models be for a + prediction to be considered trustworthy. + + To model dynamic thresholds, which automatically adjust themselves during + training to reach some desired target (such as minimum precision, or minimum + recall), thresholds behave somewhat similar to ML models themselves, with `fit` + and `predict` methods. + """ + @abstractmethod - def find( + def fit( self, clf: Classifier, x_train: np.ndarray, y_train: np.ndarray, - ) -> float: + ) -> None: + """ + Given a trained binary classifier `clf`, calibrates itself based on the + classifier's performance on the given training data set. """ - Given a trained binary classifier `clf` and a training data set, - returns the numerical threshold (float) satisfying some criterea. + assert isinstance(clf, Classifier) + assert isinstance(x_train, np.ndarray) + assert isinstance(y_train, np.ndarray) + n_samples = x_train.shape[0] + assert y_train.shape[0] == n_samples + + @abstractmethod + def predict(self, x_test: np.ndarray) -> float: + """ + Returns the minimum probability for a machine learning prediction to be + considered trustworthy. """ pass -class MinPrecisionThreshold(DynamicThreshold): +class MinProbabilityThreshold(Threshold): + """ + A threshold which considers predictions trustworthy if their probability of being + correct, as computed by the machine learning models, are above a fixed value. + """ + + def __init__(self, min_probability: float): + self.min_probability = min_probability + + def fit(self, clf: Classifier, x_train: np.ndarray, y_train: np.ndarray) -> None: + pass + + def predict(self, x_test: np.ndarray) -> float: + return self.min_probability + + +class MinPrecisionThreshold(Threshold): """ - The smallest possible threshold satisfying a minimum acceptable true - positive rate (also known as precision). + A dynamic threshold which automatically adjusts itself during training to ensure + that the component achieves at least a given precision `p` on the training data + set. Note that increasing a component's minimum precision may reduce its recall. """ def __init__(self, min_precision: float) -> None: self.min_precision = min_precision + self._computed_threshold: Optional[float] = None - def find(self, clf, x_train, y_train): + def fit(self, clf: Classifier, x_train: np.ndarray, y_train: np.ndarray) -> None: + super().fit(clf, x_train, y_train) proba = clf.predict_proba(x_train) - - assert isinstance(proba, np.ndarray), "classifier should return numpy array" - assert proba.shape == ( - x_train.shape[0], - 2, - ), "classifier should return (%d,%d)-shaped array, not %s" % ( - x_train.shape[0], - 2, - str(proba.shape), - ) - fps, tps, thresholds = _binary_clf_curve(y_train, proba[:, 1]) precision = tps / (tps + fps) - for k in reversed(range(len(precision))): if precision[k] >= self.min_precision: - return thresholds[k] - return 2.0 + self._computed_threshold = thresholds[k] + return + self._computed_threshold = float("inf") + + def predict(self, x_test: np.ndarray) -> float: + assert self._computed_threshold is not None + return self._computed_threshold diff --git a/miplearn/components/primal.py b/miplearn/components/primal.py index fe479a8..fa3a8ff 100644 --- a/miplearn/components/primal.py +++ b/miplearn/components/primal.py @@ -11,7 +11,7 @@ from tqdm.auto import tqdm from miplearn.classifiers import Classifier from miplearn.classifiers.adaptive import AdaptiveClassifier -from miplearn.classifiers.threshold import MinPrecisionThreshold, DynamicThreshold +from miplearn.classifiers.threshold import MinPrecisionThreshold, Threshold from miplearn.components import classifier_evaluation_dict from miplearn.components.component import Component from miplearn.extractors import VariableFeaturesExtractor, SolutionExtractor, Extractor @@ -28,11 +28,11 @@ class PrimalSolutionComponent(Component): self, classifier: Classifier = AdaptiveClassifier(), mode: str = "exact", - threshold: Union[float, DynamicThreshold] = MinPrecisionThreshold(0.98), + threshold: Union[float, Threshold] = MinPrecisionThreshold(0.98), ) -> None: self.mode = mode self.classifiers: Dict[Any, Classifier] = {} - self.thresholds: Dict[Any, Union[float, DynamicThreshold]] = {} + self.thresholds: Dict[Any, Union[float, Threshold]] = {} self.threshold_prototype = threshold self.classifier_prototype = classifier @@ -89,8 +89,8 @@ class PrimalSolutionComponent(Component): clf.fit(x_train, y_train) # Find threshold (dynamic or static) - if isinstance(self.threshold_prototype, DynamicThreshold): - self.thresholds[category, label] = self.threshold_prototype.find( + if isinstance(self.threshold_prototype, Threshold): + self.thresholds[category, label] = self.threshold_prototype.fit( clf, x_train, y_train, diff --git a/tests/classifiers/test_threshold.py b/tests/classifiers/test_threshold.py index a96c326..c37578e 100644 --- a/tests/classifiers/test_threshold.py +++ b/tests/classifiers/test_threshold.py @@ -26,13 +26,17 @@ def test_threshold_dynamic(): y_train = np.array([1, 1, 0, 0]) threshold = MinPrecisionThreshold(min_precision=1.0) - assert threshold.find(clf, x_train, y_train) == 0.90 + threshold.fit(clf, x_train, y_train) + assert threshold.predict(x_train) == 0.90 threshold = MinPrecisionThreshold(min_precision=0.65) - assert threshold.find(clf, x_train, y_train) == 0.80 + threshold.fit(clf, x_train, y_train) + assert threshold.predict(x_train) == 0.80 threshold = MinPrecisionThreshold(min_precision=0.50) - assert threshold.find(clf, x_train, y_train) == 0.70 + threshold.fit(clf, x_train, y_train) + assert threshold.predict(x_train) == 0.70 threshold = MinPrecisionThreshold(min_precision=0.00) - assert threshold.find(clf, x_train, y_train) == 0.70 + threshold.fit(clf, x_train, y_train) + assert threshold.predict(x_train) == 0.70