Extract instance, var and constr features into sample

miplearn/features.py

@@ -4,6 +4,7 @@
 
 import collections
 import numbers
+from copy import copy
 from dataclasses import dataclass
 from math import log, isfinite
 from typing import TYPE_CHECKING, Dict, Optional, List, Hashable, Tuple, Any
@@ -176,6 +177,95 @@ class FeaturesExtractor:
         self.with_sa = with_sa
         self.with_lhs = with_lhs
 
+    def extract_after_load_features(
+        self,
+        instance: "Instance",
+        solver: "InternalSolver",
+        sample: Sample,
+    ) -> None:
+        variables = solver.get_variables(with_static=True)
+        constraints = solver.get_constraints(with_static=True, with_lhs=self.with_lhs)
+        sample.put("var_lower_bounds", variables.lower_bounds)
+        sample.put("var_names", variables.names)
+        sample.put("var_obj_coeffs", variables.obj_coeffs)
+        sample.put("var_types", variables.types)
+        sample.put("var_upper_bounds", variables.upper_bounds)
+        sample.put("constr_names", constraints.names)
+        sample.put("constr_lhs", constraints.lhs)
+        sample.put("constr_rhs", constraints.rhs)
+        sample.put("constr_senses", constraints.senses)
+        self._extract_user_features_vars(instance, sample)
+        self._extract_user_features_constrs(instance, sample)
+        self._extract_user_features_instance(instance, sample)
+        self._extract_var_features_AlvLouWeh2017(sample)
+        sample.put(
+            "var_features",
+            self._combine(
+                sample,
+                [
+                    "var_features_AlvLouWeh2017",
+                    "var_features_user",
+                    "var_lower_bounds",
+                    "var_obj_coeffs",
+                    "var_upper_bounds",
+                ],
+            ),
+        )
+
+    def extract_after_lp_features(
+        self,
+        solver: "InternalSolver",
+        sample: Sample,
+    ) -> None:
+        variables = solver.get_variables(with_static=False, with_sa=self.with_sa)
+        constraints = solver.get_constraints(with_static=False, with_sa=self.with_sa)
+        sample.put("lp_var_basis_status", variables.basis_status)
+        sample.put("lp_var_reduced_costs", variables.reduced_costs)
+        sample.put("lp_var_sa_lb_down", variables.sa_lb_down)
+        sample.put("lp_var_sa_lb_up", variables.sa_lb_up)
+        sample.put("lp_var_sa_obj_down", variables.sa_obj_down)
+        sample.put("lp_var_sa_obj_up", variables.sa_obj_up)
+        sample.put("lp_var_sa_ub_down", variables.sa_ub_down)
+        sample.put("lp_var_sa_ub_up", variables.sa_ub_up)
+        sample.put("lp_var_values", variables.values)
+        sample.put("lp_constr_basis_status", constraints.basis_status)
+        sample.put("lp_constr_dual_values", constraints.dual_values)
+        sample.put("lp_constr_sa_rhs_down", constraints.sa_rhs_down)
+        sample.put("lp_constr_sa_rhs_up", constraints.sa_rhs_up)
+        sample.put("lp_constr_slacks", constraints.slacks)
+        self._extract_var_features_AlvLouWeh2017(sample, prefix="lp_")
+        sample.put(
+            "lp_var_features",
+            self._combine(
+                sample,
+                [
+                    "lp_var_features_AlvLouWeh2017",
+                    "lp_var_reduced_costs",
+                    "lp_var_sa_lb_down",
+                    "lp_var_sa_lb_up",
+                    "lp_var_sa_obj_down",
+                    "lp_var_sa_obj_up",
+                    "lp_var_sa_ub_down",
+                    "lp_var_sa_ub_up",
+                    "lp_var_values",
+                    "var_features_user",
+                    "var_lower_bounds",
+                    "var_obj_coeffs",
+                    "var_upper_bounds",
+                ],
+            ),
+        )
+
+    def extract_after_mip_features(
+        self,
+        solver: "InternalSolver",
+        sample: Sample,
+    ) -> None:
+        variables = solver.get_variables(with_static=False, with_sa=False)
+        constraints = solver.get_constraints(with_static=False, with_sa=False)
+        sample.put("mip_var_values", variables.values)
+        sample.put("mip_constr_slacks", constraints.slacks)
+
     def extract(
         self,
         instance: "Instance",
@@ -193,13 +283,107 @@ class FeaturesExtractor:
             with_lhs=self.with_lhs,
         )
         if with_static:
-            self._extract_user_features_vars(instance, features)
+            self._extract_user_features_vars_old(instance, features)
-            self._extract_user_features_constrs(instance, features)
+            self._extract_user_features_constrs_old(instance, features)
-            self._extract_user_features_instance(instance, features)
+            self._extract_user_features_instance_old(instance, features)
-            self._extract_alvarez_2017(features)
+            self._extract_alvarez_2017_old(features)
         return features
 
     def _extract_user_features_vars(
+        self,
+        instance: "Instance",
+        sample: Sample,
+    ) -> None:
+        categories: List[Optional[Hashable]] = []
+        user_features: List[Optional[List[float]]] = []
+        var_features_dict = instance.get_variable_features()
+        var_categories_dict = instance.get_variable_categories()
+        var_names = sample.get("var_names")
+        assert var_names is not None
+        for (i, var_name) in enumerate(var_names):
+            if var_name not in var_categories_dict:
+                user_features.append(None)
+                categories.append(None)
+                continue
+            category: Hashable = var_categories_dict[var_name]
+            assert isinstance(category, collections.Hashable), (
+                f"Variable category must be be hashable. "
+                f"Found {type(category).__name__} instead for var={var_name}."
+            )
+            categories.append(category)
+            user_features_i: Optional[List[float]] = None
+            if var_name in var_features_dict:
+                user_features_i = var_features_dict[var_name]
+                if isinstance(user_features_i, np.ndarray):
+                    user_features_i = user_features_i.tolist()
+                assert isinstance(user_features_i, list), (
+                    f"Variable features must be a list. "
+                    f"Found {type(user_features_i).__name__} instead for "
+                    f"var={var_name}."
+                )
+                for v in user_features_i:
+                    assert isinstance(v, numbers.Real), (
+                        f"Variable features must be a list of numbers. "
+                        f"Found {type(v).__name__} instead "
+                        f"for var={var_name}."
+                    )
+                user_features_i = list(user_features_i)
+            user_features.append(user_features_i)
+        sample.put("var_categories", categories)
+        sample.put("var_features_user", user_features)
+
+    def _extract_user_features_constrs(
+        self,
+        instance: "Instance",
+        sample: Sample,
+    ) -> None:
+        has_static_lazy = instance.has_static_lazy_constraints()
+        user_features: List[Optional[List[float]]] = []
+        categories: List[Optional[Hashable]] = []
+        lazy: List[bool] = []
+        constr_categories_dict = instance.get_constraint_categories()
+        constr_features_dict = instance.get_constraint_features()
+        constr_names = sample.get("constr_names")
+        assert constr_names is not None
+
+        for (cidx, cname) in enumerate(constr_names):
+            category: Optional[Hashable] = cname
+            if cname in constr_categories_dict:
+                category = constr_categories_dict[cname]
+            if category is None:
+                user_features.append(None)
+                categories.append(None)
+                continue
+            assert isinstance(category, collections.Hashable), (
+                f"Constraint category must be hashable. "
+                f"Found {type(category).__name__} instead for cname={cname}.",
+            )
+            categories.append(category)
+            cf: Optional[List[float]] = None
+            if cname in constr_features_dict:
+                cf = constr_features_dict[cname]
+                if isinstance(cf, np.ndarray):
+                    cf = cf.tolist()
+                assert isinstance(cf, list), (
+                    f"Constraint features must be a list. "
+                    f"Found {type(cf).__name__} instead for cname={cname}."
+                )
+                for f in cf:
+                    assert isinstance(f, numbers.Real), (
+                        f"Constraint features must be a list of numbers. "
+                        f"Found {type(f).__name__} instead for cname={cname}."
+                    )
+                cf = list(cf)
+            user_features.append(cf)
+            if has_static_lazy:
+                lazy.append(instance.is_constraint_lazy(cname))
+            else:
+                lazy.append(False)
+        sample.put("constr_features_user", user_features)
+        sample.put("constr_lazy", lazy)
+        sample.put("constr_categories", categories)
+
+    def _extract_user_features_vars_old(
         self,
         instance: "Instance",
         features: Features,
@@ -243,7 +427,7 @@ class FeaturesExtractor:
         features.variables.categories = categories
         features.variables.user_features = user_features
 
-    def _extract_user_features_constrs(
+    def _extract_user_features_constrs_old(
         self,
         instance: "Instance",
         features: Features,
@@ -295,6 +479,28 @@ class FeaturesExtractor:
         features.constraints.categories = categories
 
     def _extract_user_features_instance(
+        self,
+        instance: "Instance",
+        sample: Sample,
+    ) -> None:
+        user_features = instance.get_instance_features()
+        if isinstance(user_features, np.ndarray):
+            user_features = user_features.tolist()
+        assert isinstance(user_features, list), (
+            f"Instance features must be a list. "
+            f"Found {type(user_features).__name__} instead."
+        )
+        for v in user_features:
+            assert isinstance(v, numbers.Real), (
+                f"Instance features must be a list of numbers. "
+                f"Found {type(v).__name__} instead."
+            )
+        constr_lazy = sample.get("constr_lazy")
+        assert constr_lazy is not None
+        sample.put("instance_features_user", user_features)
+        sample.put("static_lazy_count", sum(constr_lazy))
+
+    def _extract_user_features_instance_old(
         self,
         instance: "Instance",
         features: Features,
@@ -318,7 +524,7 @@ class FeaturesExtractor:
             lazy_constraint_count=sum(features.constraints.lazy),
         )
 
-    def _extract_alvarez_2017(self, features: Features) -> None:
+    def _extract_alvarez_2017_old(self, features: Features) -> None:
         assert features.variables is not None
         assert features.variables.names is not None
 
@@ -394,6 +600,114 @@ class FeaturesExtractor:
                 assert isfinite(v), f"non-finite elements detected: {f}"
             features.variables.alvarez_2017.append(f)
 
+    # Alvarez, A. M., Louveaux, Q., & Wehenkel, L. (2017). A machine learning-based
+    # approximation of strong branching. INFORMS Journal on Computing, 29(1), 185-195.
+    def _extract_var_features_AlvLouWeh2017(
+        self,
+        sample: Sample,
+        prefix: str = "",
+    ) -> None:
+        obj_coeffs = sample.get("var_obj_coeffs")
+        obj_sa_down = sample.get("lp_var_sa_obj_down")
+        obj_sa_up = sample.get("lp_var_sa_obj_up")
+        values = sample.get(f"lp_var_values")
+        assert obj_coeffs is not None
+
+        pos_obj_coeff_sum = 0.0
+        neg_obj_coeff_sum = 0.0
+        for coeff in obj_coeffs:
+            if coeff > 0:
+                pos_obj_coeff_sum += coeff
+            if coeff < 0:
+                neg_obj_coeff_sum += -coeff
+
+        features = []
+        for i in range(len(obj_coeffs)):
+            f: List[float] = []
+            if obj_coeffs is not None:
+                # Feature 1
+                f.append(np.sign(obj_coeffs[i]))
+
+                # Feature 2
+                if pos_obj_coeff_sum > 0:
+                    f.append(abs(obj_coeffs[i]) / pos_obj_coeff_sum)
+                else:
+                    f.append(0.0)
+
+                # Feature 3
+                if neg_obj_coeff_sum > 0:
+                    f.append(abs(obj_coeffs[i]) / neg_obj_coeff_sum)
+                else:
+                    f.append(0.0)
+
+            if values is not None:
+                # Feature 37
+                f.append(
+                    min(
+                        values[i] - np.floor(values[i]),
+                        np.ceil(values[i]) - values[i],
+                    )
+                )
+
+            if obj_sa_up is not None:
+                assert obj_sa_down is not None
+                assert obj_coeffs is not None
+
+                # Convert inf into large finite numbers
+                sd = max(-1e20, obj_sa_down[i])
+                su = min(1e20, obj_sa_up[i])
+                obj = obj_coeffs[i]
+
+                # Features 44 and 46
+                f.append(np.sign(obj_sa_up[i]))
+                f.append(np.sign(obj_sa_down[i]))
+
+                # Feature 47
+                csign = np.sign(obj)
+                if csign != 0 and ((obj - sd) / csign) > 0.001:
+                    f.append(log((obj - sd) / csign))
+                else:
+                    f.append(0.0)
+
+                # Feature 48
+                if csign != 0 and ((su - obj) / csign) > 0.001:
+                    f.append(log((su - obj) / csign))
+                else:
+                    f.append(0.0)
+
+            for v in f:
+                assert isfinite(v), f"non-finite elements detected: {f}"
+            features.append(f)
+        sample.put(f"{prefix}var_features_AlvLouWeh2017", features)
+
+    def _combine(
+        self,
+        sample: Sample,
+        attrs: List[str],
+    ) -> List[List[float]]:
+        combined: List[List[float]] = []
+        for attr in attrs:
+            series = sample.get(attr)
+            if series is None:
+                continue
+            if len(combined) == 0:
+                for i in range(len(series)):
+                    combined.append([])
+            for (i, s) in enumerate(series):
+                if s is None:
+                    continue
+                elif isinstance(s, list):
+                    combined[i].extend([_clipf(sj) for sj in s])
+                else:
+                    combined[i].append(_clipf(s))
+        return combined
+
+
+def _clipf(vi: float) -> float:
+    if not isfinite(vi):
+        return max(min(vi, 1e20), -1e20)
+    return vi
+
 
 def _clip(v: List[float]) -> None:
     for (i, vi) in enumerate(v):

miplearn/solvers/learning.py

@@ -168,6 +168,9 @@ class LearningSolver:
         # -------------------------------------------------------
         logger.info("Extracting features (after-load)...")
         initial_time = time.time()
+        self.extractor.extract_after_load_features(
+            instance, self.internal_solver, sample
+        )
         features = self.extractor.extract(instance, self.internal_solver)
         logger.info(
             "Features (after-load) extracted in %.2f seconds"

tests/test_features.py

@@ -9,6 +9,7 @@ from miplearn.features import (
     InstanceFeatures,
     VariableFeatures,
     ConstraintFeatures,
+    Sample,
 )
 from miplearn.solvers.gurobi import GurobiSolver
 from miplearn.solvers.tests import assert_equals
@@ -21,55 +22,40 @@ def test_knapsack() -> None:
     instance = solver.build_test_instance_knapsack()
     model = instance.to_model()
     solver.set_instance(instance, model)
-    solver.solve_lp()
+    extractor = FeaturesExtractor()
-
+    sample = Sample()
-    features = FeaturesExtractor().extract(instance, solver)
-    assert features.variables is not None
-    assert features.instance is not None
 
+    # after-load
+    # -------------------------------------------------------
+    extractor.extract_after_load_features(instance, solver, sample)
+    assert_equals(sample.get("var_names"), ["x[0]", "x[1]", "x[2]", "x[3]", "z"])
+    assert_equals(sample.get("var_lower_bounds"), [0.0, 0.0, 0.0, 0.0, 0.0])
+    assert_equals(sample.get("var_obj_coeffs"), [505.0, 352.0, 458.0, 220.0, 0.0])
+    assert_equals(sample.get("var_types"), ["B", "B", "B", "B", "C"])
+    assert_equals(sample.get("var_upper_bounds"), [1.0, 1.0, 1.0, 1.0, 67.0])
     assert_equals(
-        features.variables,
+        sample.get("var_categories"),
-        VariableFeatures(
+        ["default", "default", "default", "default", None],
-            names=["x[0]", "x[1]", "x[2]", "x[3]", "z"],
-            basis_status=["U", "B", "U", "L", "U"],
-            categories=["default", "default", "default", "default", None],
-            lower_bounds=[0.0, 0.0, 0.0, 0.0, 0.0],
-            obj_coeffs=[505.0, 352.0, 458.0, 220.0, 0.0],
-            reduced_costs=[193.615385, 0.0, 187.230769, -23.692308, 13.538462],
-            sa_lb_down=[-inf, -inf, -inf, -0.111111, -inf],
-            sa_lb_up=[1.0, 0.923077, 1.0, 1.0, 67.0],
-            sa_obj_down=[311.384615, 317.777778, 270.769231, -inf, -13.538462],
-            sa_obj_up=[inf, 570.869565, inf, 243.692308, inf],
-            sa_ub_down=[0.913043, 0.923077, 0.9, 0.0, 43.0],
-            sa_ub_up=[2.043478, inf, 2.2, inf, 69.0],
-            types=["B", "B", "B", "B", "C"],
-            upper_bounds=[1.0, 1.0, 1.0, 1.0, 67.0],
-            user_features=[
-                [23.0, 505.0],
-                [26.0, 352.0],
-                [20.0, 458.0],
-                [18.0, 220.0],
-                None,
-            ],
-            values=[1.0, 0.923077, 1.0, 0.0, 67.0],
-            alvarez_2017=[
-                [1.0, 0.32899, 0.0, 0.0, 1.0, 1.0, 5.265874, 46.051702],
-                [1.0, 0.229316, 0.0, 0.076923, 1.0, 1.0, 3.532875, 5.388476],
-                [1.0, 0.298371, 0.0, 0.0, 1.0, 1.0, 5.232342, 46.051702],
-                [1.0, 0.143322, 0.0, 0.0, 1.0, -1.0, 46.051702, 3.16515],
-                [0.0, 0.0, 0.0, 0.0, 1.0, -1.0, 0.0, 0.0],
-            ],
-        ),
     )
     assert_equals(
-        features.constraints,
+        sample.get("var_features_user"),
-        ConstraintFeatures(
+        [[23.0, 505.0], [26.0, 352.0], [20.0, 458.0], [18.0, 220.0], None],
-            basis_status=["N"],
+    )
-            categories=["eq_capacity"],
+    assert_equals(
-            dual_values=[13.538462],
+        sample.get("var_features_AlvLouWeh2017"),
-            names=["eq_capacity"],
+        [
-            lazy=[False],
+            [1.0, 0.32899, 0.0],
-            lhs=[
+            [1.0, 0.229316, 0.0],
+            [1.0, 0.298371, 0.0],
+            [1.0, 0.143322, 0.0],
+            [0.0, 0.0, 0.0],
+        ],
+    )
+    assert sample.get("var_features") is not None
+    assert_equals(sample.get("constr_names"), ["eq_capacity"])
+    assert_equals(
+        sample.get("constr_lhs"),
+        [
             [
                 ("x[0]", 23.0),
                 ("x[1]", 26.0),
@@ -78,14 +64,71 @@ def test_knapsack() -> None:
                 ("z", -1.0),
             ],
         ],
-            rhs=[0.0],
-            sa_rhs_down=[-24.0],
-            sa_rhs_up=[2.0],
-            senses=["="],
-            slacks=[0.0],
-            user_features=[None],
-        ),
     )
+    assert_equals(sample.get("constr_rhs"), [0.0])
+    assert_equals(sample.get("constr_senses"), ["="])
+    assert_equals(sample.get("constr_features_user"), [None])
+    assert_equals(sample.get("constr_categories"), ["eq_capacity"])
+    assert_equals(sample.get("constr_lazy"), [False])
+    assert_equals(sample.get("instance_features_user"), [67.0, 21.75])
+    assert_equals(sample.get("static_lazy_count"), 0)
+
+    # after-lp
+    # -------------------------------------------------------
+    solver.solve_lp()
+    extractor.extract_after_lp_features(solver, sample)
+    assert_equals(
+        sample.get("lp_var_basis_status"),
+        ["U", "B", "U", "L", "U"],
+    )
+    assert_equals(
+        sample.get("lp_var_reduced_costs"),
+        [193.615385, 0.0, 187.230769, -23.692308, 13.538462],
+    )
+    assert_equals(
+        sample.get("lp_var_sa_lb_down"),
+        [-inf, -inf, -inf, -0.111111, -inf],
+    )
+    assert_equals(
+        sample.get("lp_var_sa_lb_up"),
+        [1.0, 0.923077, 1.0, 1.0, 67.0],
+    )
+    assert_equals(
+        sample.get("lp_var_sa_obj_down"),
+        [311.384615, 317.777778, 270.769231, -inf, -13.538462],
+    )
+    assert_equals(
+        sample.get("lp_var_sa_obj_up"),
+        [inf, 570.869565, inf, 243.692308, inf],
+    )
+    assert_equals(sample.get("lp_var_sa_ub_down"), [0.913043, 0.923077, 0.9, 0.0, 43.0])
+    assert_equals(sample.get("lp_var_sa_ub_up"), [2.043478, inf, 2.2, inf, 69.0])
+    assert_equals(sample.get("lp_var_values"), [1.0, 0.923077, 1.0, 0.0, 67.0])
+    assert_equals(
+        sample.get("lp_var_features_AlvLouWeh2017"),
+        [
+            [1.0, 0.32899, 0.0, 0.0, 1.0, 1.0, 5.265874, 46.051702],
+            [1.0, 0.229316, 0.0, 0.076923, 1.0, 1.0, 3.532875, 5.388476],
+            [1.0, 0.298371, 0.0, 0.0, 1.0, 1.0, 5.232342, 46.051702],
+            [1.0, 0.143322, 0.0, 0.0, 1.0, -1.0, 46.051702, 3.16515],
+            [0.0, 0.0, 0.0, 0.0, 1.0, -1.0, 0.0, 0.0],
+        ],
+    )
+    assert sample.get("lp_var_features") is not None
+    assert_equals(sample.get("lp_constr_basis_status"), ["N"])
+    assert_equals(sample.get("lp_constr_dual_values"), [13.538462])
+    assert_equals(sample.get("lp_constr_sa_rhs_down"), [-24.0])
+    assert_equals(sample.get("lp_constr_sa_rhs_up"), [2.0])
+    assert_equals(sample.get("lp_constr_slacks"), [0.0])
+
+    # after-mip
+    # -------------------------------------------------------
+    solver.solve()
+    extractor.extract_after_mip_features(solver, sample)
+    assert_equals(sample.get("mip_var_values"), [1.0, 0.0, 1.0, 1.0, 61.0])
+    assert_equals(sample.get("mip_constr_slacks"), [0.0])
+
+    features = extractor.extract(instance, solver)
     assert_equals(
         features.instance,
         InstanceFeatures(
@@ -138,10 +181,7 @@ def test_constraint_getindex() -> None:
 def test_assert_equals() -> None:
     assert_equals("hello", "hello")
     assert_equals([1.0, 2.0], [1.0, 2.0])
-    assert_equals(
+    assert_equals(np.array([1.0, 2.0]), np.array([1.0, 2.0]))
-        np.array([1.0, 2.0]),
-        np.array([1.0, 2.0]),
-    )
     assert_equals(
         np.array([[1.0, 2.0], [3.0, 4.0]]),
         np.array([[1.0, 2.0], [3.0, 4.0]]),
@@ -150,9 +190,6 @@ def test_assert_equals() -> None:
         VariableFeatures(values=np.array([1.0, 2.0])),  # type: ignore
         VariableFeatures(values=np.array([1.0, 2.0])),  # type: ignore
     )
-    assert_equals(
+    assert_equals(np.array([True, True]), [True, True])
-        np.array([True, True]),
-        [True, True],
-    )
     assert_equals((1.0,), (1.0,))
     assert_equals({"x": 10}, {"x": 10})