Merge branch 'feature/files' into dev

docs/usage.md

@@ -115,8 +115,9 @@ For more significant performance benefits, `LearningSolver` can also be configur
 !!! danger
     The `heuristic` mode provides no optimality guarantees, and therefore should only be used if the solver is first trained on a large and representative set of training instances. Training on a small or non-representative set of instances may produce low-quality solutions, or make the solver incorrectly classify new instances as infeasible.
 
+## 6. Scaling Up
 
-## 6. Saving and loading solver state
+### 6.1 Saving and loading solver state
 
 After solving a large number of training instances, it may be desirable to save the current state of `LearningSolver` to disk, so that the solver can still use the acquired knowledge after the application restarts. This can be accomplished by using the standard `pickle` module, as the following example illustrates:
 
@@ -134,12 +135,14 @@ for instance in training_instances:
 solver.fit(training_instances)
 
 # Save trained solver to disk
-pickle.dump(solver, open("solver.pickle", "wb"))
+with open("solver.pickle", "wb") as file:
+    pickle.dump(solver, file)
 
 # Application restarts...
 
 # Load trained solver from disk
-solver = pickle.load(open("solver.pickle", "rb"))
+with open("solver.pickle", "rb") as file:
+    solver = pickle.load(file)
 
 # Solve additional instances
 test_instances = [...]
@@ -148,9 +151,9 @@ for instance in test_instances:
 ```
 
 
-## 7. Solving training instances in parallel
+### 6.2 Solving instances in parallel
 
-In many situations, training and test instances can be solved in parallel to accelerate the training process. `LearningSolver` provides the method `parallel_solve(instances)` to easily achieve this:
+In many situations, instances can be solved in parallel to accelerate the training process. `LearningSolver` provides the method `parallel_solve(instances)` to easily achieve this:
 
 ```python
 from miplearn import LearningSolver
@@ -166,6 +169,55 @@ solver.parallel_solve(test_instances)
 ```
 
 
-## 8. Current Limitations
+### 6.3 Solving instances from the disk
 
-* Only binary and continuous decision variables are currently supported.
+In all examples above, we have assumed that instances are available as Python objects, stored in memory. When problem instances are very large, or when there is a large number of problem instances, this approach may require an excessive amount of memory. To reduce memory requirements, MIPLearn can also operate on instances that are stored on disk. More precisely, the methods `fit`, `solve` and `parallel_solve` in `LearningSolver` can operate on filenames (or lists of filenames) instead of instance objects, as the next example illustrates.
+Instance files must be pickled instance objects. The method `solve` loads at most one instance to memory at a time, while `parallel_solve` loads at most `n_jobs` instances.
+
+
+```python
+from miplearn import LearningSolver
+
+# Construct and pickle 600 problem instances
+for i in range(600):
+    instance = CustomInstance([...])
+    with open("instance_%03d.pkl" % i, "w") as file:
+        pickle.dump(instance, obj)
+        
+# Split instances into training and test
+test_instances  = ["instance_%03d.pkl" % i for i in range(500)]
+train_instances = ["instance_%03d.pkl" % i for i in range(500, 600)]
+
+# Create solver
+solver = LearningSolver([...])
+
+# Solve training instances 
+solver.parallel_solve(train_instances, n_jobs=4)
+
+# Train ML models
+solver.fit(train_instances)
+
+# Solve test instances 
+solver.parallel_solve(test_instances, n_jobs=4)
+```
+
+
+By default, `solve` and `parallel_solve` modify files in place. That is, after the instances are loaded from disk and solved, MIPLearn writes them back to the disk, overwriting the original files. To write to an alternative file instead, the argument `output` may be used. In `solve`, this argument should be a single filename. In `parallel_solve`, it should be a list, containing exactly as many filenames as instances. If `output` is `None`, the modifications are simply discarded. This can be useful, for example, during benchmarks.
+
+```python
+# Solve a single instance file and store the output to another file
+solver.solve("knapsack_1.orig.pkl", output="knapsack_1.solved.pkl")
+
+# Solve a list of instance files
+instances = ["knapsack_%03d.orig.pkl" % i for i in range(100)]
+output = ["knapsack_%03d.solved.pkl" % i for i in range(100)]
+solver.parallel_solve(instances, output=output)
+
+# Solve instances and discard solutions and training data
+solver.parallel_solve(instances, output=None)
+```
+
+
+## 7. Current Limitations
+
+* Only binary and continuous decision variables are currently supported. General integer variables are not currently supported by all solver components.

miplearn/benchmark.py

@@ -37,7 +37,8 @@ class BenchmarkRunner:
         for (solver_name, solver) in self.solvers.items():
             results = solver.parallel_solve(trials,
                                             n_jobs=n_jobs,
-                                            label="Solve (%s)" % solver_name)
+                                            label="Solve (%s)" % solver_name,
+                                            output=None)
             for i in range(len(trials)):
                 idx = (i % len(instances)) + index_offset
                 self._push_result(results[i],

miplearn/components/primal.py

@@ -53,7 +53,6 @@ class PrimalSolutionComponent(Component):
 
         for category in tqdm(features.keys(),
                              desc="Fit (primal)",
-                             disable=not sys.stdout.isatty(),
                             ):
             x_train = features[category]
             for label in [0, 1]:
@@ -110,7 +109,6 @@ class PrimalSolutionComponent(Component):
               "Fix one": {}}
         for instance_idx in tqdm(range(len(instances)),
                                  desc="Evaluate (primal)",
-                                 disable=not sys.stdout.isatty(),
                                 ):
             instance = instances[instance_idx]
             solution_actual = instance.solution

miplearn/components/relaxation.py

@@ -4,6 +4,8 @@
 
 import logging
 import sys
+import numpy as np
+
 from copy import deepcopy
 
 from tqdm import tqdm
@@ -12,6 +14,7 @@ from miplearn import Component
 from miplearn.classifiers.counting import CountingClassifier
 from miplearn.components import classifier_evaluation_dict
 from miplearn.components.lazy_static import LazyConstraint
+from miplearn.extractors import InstanceIterator
 
 logger = logging.getLogger(__name__)
 
@@ -83,16 +86,12 @@ class RelaxationComponent(Component):
         instance.slacks = solver.internal_solver.get_constraint_slacks()
 
     def fit(self, training_instances):
-        training_instances = [instance
-                              for instance in training_instances
-                              if hasattr(instance, "slacks")]
         logger.debug("Extracting x and y...")
         x = self.x(training_instances)
         y = self.y(training_instances)
         logger.debug("Fitting...")
         for category in tqdm(x.keys(),
-                             desc="Fit (relaxation)",
+                             desc="Fit (relaxation)"):
-                             disable=not sys.stdout.isatty()):
             if category not in self.classifiers:
                 self.classifiers[category] = deepcopy(self.classifier_prototype)
             self.classifiers[category].fit(x[category], y[category])
@@ -103,7 +102,9 @@ class RelaxationComponent(Component):
           return_constraints=False):
         x = {}
         constraints = {}
-        for instance in instances:
+        for instance in tqdm(InstanceIterator(instances),
+                             desc="Extract (relaxation:x)",
+                             disable=len(instances) < 5):
             if constraint_ids is not None:
                 cids = constraint_ids
             else:
@@ -124,7 +125,9 @@ class RelaxationComponent(Component):
 
     def y(self, instances):
         y = {}
-        for instance in instances:
+        for instance in tqdm(InstanceIterator(instances),
+                             desc="Extract (relaxation:y)",
+                             disable=len(instances) < 5):
             for (cid, slack) in instance.slacks.items():
                 category = instance.get_constraint_category(cid)
                 if category is None:

miplearn/extractors.py

@@ -3,14 +3,41 @@
 #  Released under the modified BSD license. See COPYING.md for more details.
 
 import logging
-from abc import ABC, abstractmethod
+import pickle
+import gzip
 
 import numpy as np
-from tqdm import tqdm
+
+from tqdm.auto import tqdm
+from abc import ABC, abstractmethod
 
 logger = logging.getLogger(__name__)
 
 
+class InstanceIterator:
+    def __init__(self, instances):
+        self.instances = instances
+        self.current = 0
+        
+    def __iter__(self):
+        return self
+    
+    def __next__(self):
+        if self.current >= len(self.instances):
+            raise StopIteration
+        result = self.instances[self.current]
+        self.current += 1
+        if isinstance(result, str):
+            logger.info("Read: %s" % result)
+            if result.endswith(".gz"):
+                with gzip.GzipFile(result, "rb") as file:
+                    result = pickle.load(file)
+            else:
+                with open(result, "rb") as file:
+                    result = pickle.load(file)
+        return result
+
+
 class Extractor(ABC):
     @abstractmethod
     def extract(self, instances,):
@@ -34,7 +61,7 @@ class Extractor(ABC):
 class VariableFeaturesExtractor(Extractor):
     def extract(self, instances):
         result = {}
-        for instance in tqdm(instances,
+        for instance in tqdm(InstanceIterator(instances),
                              desc="Extract (vars)",
                              disable=len(instances) < 5):
             instance_features = instance.get_instance_features()
@@ -59,7 +86,7 @@ class SolutionExtractor(Extractor):
         
     def extract(self, instances):
         result = {}
-        for instance in tqdm(instances,
+        for instance in tqdm(InstanceIterator(instances),
                              desc="Extract (solution)",
                              disable=len(instances) < 5):
             var_split = self.split_variables(instance)
@@ -87,7 +114,7 @@ class InstanceFeaturesExtractor(Extractor):
                 instance.get_instance_features(),
                 instance.lp_value,
             ])
-            for instance in instances
+            for instance in InstanceIterator(instances)
         ])
     
     
@@ -98,8 +125,11 @@ class ObjectiveValueExtractor(Extractor):
         
     def extract(self, instances):
         if self.kind == "lower bound":
-            return np.array([[instance.lower_bound] for instance in instances])
+            return np.array([[instance.lower_bound]
+                             for instance in InstanceIterator(instances)])
         if self.kind == "upper bound":
-            return np.array([[instance.upper_bound] for instance in instances])
+            return np.array([[instance.upper_bound]
+                             for instance in InstanceIterator(instances)])
         if self.kind == "lp":
-            return np.array([[instance.lp_value] for instance in instances])
+            return np.array([[instance.lp_value]
+                             for instance in InstanceIterator(instances)])

miplearn/solvers/learning.py

@@ -3,6 +3,11 @@
 #  Released under the modified BSD license. See COPYING.md for more details.
 
 import logging
+import pickle
+import os
+import tempfile
+import gzip
+
 from copy import deepcopy
 from typing import Optional, List
 from p_tqdm import p_map
@@ -20,26 +25,21 @@ logger = logging.getLogger(__name__)
 # Global memory for multiprocessing
 SOLVER = [None]  # type: List[Optional[LearningSolver]]
 INSTANCES = [None]  # type: List[Optional[dict]]
+OUTPUTS = [None]
 
 
-def _parallel_solve(instance_idx):
+def _parallel_solve(idx):
     solver = deepcopy(SOLVER[0])
-    instance = INSTANCES[0][instance_idx]
+    if OUTPUTS[0] is None:
-    if not hasattr(instance, "found_violated_lazy_constraints"):
+        output = None
-        instance.found_violated_lazy_constraints = []
+    elif len(OUTPUTS[0]) == 0:
-    if not hasattr(instance, "found_violated_user_cuts"):
+        output = ""
-        instance.found_violated_user_cuts = []
+    else:
-    if not hasattr(instance, "slacks"):
+        output = OUTPUTS[0][idx]
-        instance.slacks = {}
+    instance = INSTANCES[0][idx]
-    solver_results = solver.solve(instance)
+    print(instance)
-    return {
+    stats = solver.solve(instance, output=output)
-        "solver_results": solver_results,
+    return (stats, instance)
-        "solution": instance.solution,
-        "lp_solution": instance.lp_solution,
-        "found_violated_lazy_constraints": instance.found_violated_lazy_constraints,
-        "found_violated_user_cuts": instance.found_violated_user_cuts,
-        "slacks": instance.slacks
-    }
 
 
 class LearningSolver:
@@ -145,31 +145,43 @@ class LearningSolver:
     def solve(self,
               instance,
               model=None,
+              output="",
               tee=False):
         """
         Solves the given instance. If trained machine-learning models are
         available, they will be used to accelerate the solution process.
         
+        The argument `instance` may be either an Instance object or a
+        filename pointing to a pickled Instance object. 
+
         This method modifies the instance object. Specifically, the following
         properties are set:
+        
             - instance.lp_solution
             - instance.lp_value
             - instance.lower_bound
             - instance.upper_bound
             - instance.solution
             - instance.solver_log
+            
         Additional solver components may set additional properties. Please
-        see their documentation for more details.
+        see their documentation for more details. If a filename is provided,
+        then the file is modified in-place. That is, the original file is
+        overwritten.
 
         If `solver.solve_lp_first` is False, the properties lp_solution and
         lp_value will be set to dummy values.
 
         Parameters
         ----------
-        instance: miplearn.Instance
+        instance: miplearn.Instance or str
-            The instance to be solved
+            The instance to be solved, or a filename.
         model: pyomo.core.ConcreteModel
             The corresponding Pyomo model. If not provided, it will be created.
+        output: str or None
+            If instance is a filename and output is provided, write the modified
+            instance to this file, instead of replacing the original file. If
+            output is None, discard modified instance.
         tee: bool
             If true, prints solver log to screen.
 
@@ -186,6 +198,20 @@ class LearningSolver:
             details.
         """
         
+        filename = None
+        fileformat = None
+        if isinstance(instance, str):
+            filename = instance
+            logger.info("Reading: %s" % filename)
+            if filename.endswith(".gz"):
+                fileformat = "pickle-gz"
+                with gzip.GzipFile(filename, "rb") as file:
+                    instance = pickle.load(file)
+            else:
+                fileformat = "pickle"
+                with open(filename, "rb") as file:
+                    instance = pickle.load(file)
+                
         if model is None:
             model = instance.to_model()
 
@@ -237,34 +263,59 @@ class LearningSolver:
         for component in self.components.values():
             component.after_solve(self, instance, model, results)
             
+        if filename is not None and output is not None:
+            output_filename = output
+            if len(output) == 0:
+                output_filename = filename
+            logger.info("Writing: %s" % output_filename)
+            if fileformat == "pickle":
+                with open(output_filename, "wb") as file:
+                    pickle.dump(instance, file)
+            else:
+                with gzip.GzipFile(output_filename, "wb") as file:
+                    pickle.dump(instance, file)
+
         return results
 
-    def parallel_solve(self,
+    def parallel_solve(self, instances, n_jobs=4, label="Solve", output=[]):
-                       instances,
+        """
-                       n_jobs=4,
+        Solves multiple instances in parallel.
-                       label="Solve"):
+        
+        This method is equivalent to calling `solve` for each item on the list,
+        but it processes multiple instances at the same time. Like `solve`, this
+        method modifies each instance in place. Also like `solve`, a list of
+        filenames may be provided.
+        
+        Parameters
+        ----------
+        instances: [miplearn.Instance] or [str]
+            The instances to be solved
+        n_jobs: int
+            Number of instances to solve in parallel at a time.
+            
+        Returns
+        -------
+        Returns a list of dictionaries, with one entry for each provided instance.
+        This dictionary is the same you would obtain by calling:
+        
+            [solver.solve(p) for p in instances]
         
+        """
         self.internal_solver = None
         self._silence_miplearn_logger()
         SOLVER[0] = self
+        OUTPUTS[0] = output
         INSTANCES[0] = instances
-        p_map_results = p_map(_parallel_solve,
+        results = p_map(_parallel_solve,
                         list(range(len(instances))),
                         num_cpus=n_jobs,
                         desc=label)
-        results = [p["solver_results"] for p in p_map_results]
+        stats = []
-        for (idx, r) in enumerate(p_map_results):
+        for (idx, (s, instance)) in enumerate(results):
-            instances[idx].solution = r["solution"]
+            stats.append(s)
-            instances[idx].lp_solution = r["lp_solution"]
+            instances[idx] = instance
-            instances[idx].lp_value = r["solver_results"]["LP value"]
-            instances[idx].lower_bound = r["solver_results"]["Lower bound"]
-            instances[idx].upper_bound = r["solver_results"]["Upper bound"]
-            instances[idx].found_violated_lazy_constraints = r["found_violated_lazy_constraints"]
-            instances[idx].found_violated_user_cuts = r["found_violated_user_cuts"]
-            instances[idx].slacks = r["slacks"]
-            instances[idx].solver_log = r["solver_results"]["Log"]
         self._restore_miplearn_logger()
-        return results
+        return stats
 
     def fit(self, training_instances):
         if len(training_instances) == 0:

miplearn/solvers/tests/test_learning_solver.py

@@ -5,6 +5,7 @@
 import logging
 import pickle
 import tempfile
+import os
 
 from miplearn import DynamicLazyConstraintsComponent
 from miplearn import LearningSolver
@@ -65,3 +66,45 @@ def test_add_components():
     solver.add(DynamicLazyConstraintsComponent())
     assert len(solver.components) == 1
     assert "DynamicLazyConstraintsComponent" in solver.components
+
+
+def test_solve_fit_from_disk():
+    for internal_solver in _get_internal_solvers():
+        # Create instances and pickle them
+        filenames = []
+        for k in range(3):
+            instance = _get_instance(internal_solver)
+            with tempfile.NamedTemporaryFile(suffix=".pkl",
+                                             delete=False) as file:
+                filenames += [file.name]
+                pickle.dump(instance, file)
+        
+        # Test: solve
+        solver = LearningSolver(solver=internal_solver)
+        solver.solve(filenames[0])
+        with open(filenames[0], "rb") as file:
+            instance = pickle.load(file)
+            assert hasattr(instance, "solution")
+            
+        # Test: parallel_solve
+        solver.parallel_solve(filenames)
+        for filename in filenames:
+            with open(filename, "rb") as file:
+                instance = pickle.load(file)
+                assert hasattr(instance, "solution")
+                
+        # Test: solve (with specified output)
+        output = [f + ".out" for f in filenames]
+        solver.solve(filenames[0], output=output[0])
+        assert os.path.isfile(output[0])
+            
+        # Test: parallel_solve (with specified output)
+        solver.parallel_solve(filenames, output=output)
+        for filename in output:
+            assert os.path.isfile(filename)
+            
+        # Delete temporary files
+        for filename in filenames:
+            os.remove(filename)
+        for filename in output:
+            os.remove(filename)