BenchmarkRunner: add solve method

miplearn/benchmark.py

@@ -3,7 +3,9 @@
 # Written by Alinson S. Xavier <axavier@anl.gov>
 
 from .solvers import LearningSolver
+from copy import deepcopy
 import pandas as pd
+from tqdm.auto import tqdm
 
 class BenchmarkRunner:
     def __init__(self, solvers):
@@ -13,7 +15,42 @@ class BenchmarkRunner:
         self.solvers = solvers
         self.results = None
         
+    def solve(self, instances, fit=True, tee=False):
+        for (name, solver) in self.solvers.items():
+            for i in tqdm(range(len((instances)))):
+                results = solver.solve(deepcopy(instances[i]), tee=tee)
+                self._push_result(results, solver=solver, name=name, instance=i)
+                if fit:
+                    solver.fit()
+
     def parallel_solve(self, instances, n_jobs=1, n_trials=1):
+        instances = instances * n_trials
+        for (name, solver) in self.solvers.items():
+            results = solver.parallel_solve(instances,
+                                            n_jobs=n_jobs,
+                                            label="Solve (%s)" % name,
+                                            collect_training_data=False)
+            for i in range(len(instances)):
+                self._push_result(results[i], solver=solver, name=name, instance=i)
+    
+    def raw_results(self):
+        return self.results
+    
+    def save_results(self, filename):
+        self.results.to_csv(filename)
+        
+    def load_results(self, filename):
+        self.results = pd.read_csv(filename, index_col=0)
+        
+    def load_state(self, filename):
+        for (name, solver) in self.solvers.items():
+            solver.load_state(filename)
+
+    def fit(self):
+        for (name, solver) in self.solvers.items():
+            solver.fit()
+            
+    def _push_result(self, result, solver, name, instance):
         if self.results is None:
             self.results = pd.DataFrame(columns=["Solver",
                                                  "Instance",
@@ -22,33 +59,28 @@ class BenchmarkRunner:
                                                  "Upper Bound",
                                                  "Gap",
                                                  "Nodes",
+                                                 "Mode",
                                                 ])
-        instances = instances * n_trials
-        for (name, solver) in self.solvers.items():
-            results = solver.parallel_solve(instances,
-                                            n_jobs=n_jobs,
-                                            label=name,
-                                            collect_training_data=False)
-            for i in range(len(instances)):
         wallclock_time = None
         for key in ["Time", "Wall time", "Wallclock time"]:
-                    if key not in results[i]["Solver"][0].keys():
+            if key not in result["Solver"][0].keys():
                 continue
-                    if str(results[i]["Solver"][0][key]) == "<undefined>":
+            if str(result["Solver"][0][key]) == "<undefined>":
                 continue
-                    wallclock_time = float(results[i]["Solver"][0][key])
-                nodes = results[i]["Solver"][0]["Nodes"]
-                lb = results[i]["Problem"][0]["Lower bound"]
-                ub = results[i]["Problem"][0]["Upper bound"]
+            wallclock_time = float(result["Solver"][0][key])
+        nodes = result["Solver"][0]["Nodes"]
+        lb = result["Problem"][0]["Lower bound"]
+        ub = result["Problem"][0]["Upper bound"]
         gap = (ub - lb) / lb
         self.results = self.results.append({
             "Solver": name,
-                    "Instance": i,
+            "Instance": instance,
             "Wallclock Time": wallclock_time,
             "Lower Bound": lb,
             "Upper Bound": ub,
             "Gap": gap,
             "Nodes": nodes,
+            "Mode": solver.mode,
         }, ignore_index=True)
         groups = self.results.groupby("Instance")
         best_lower_bound = groups["Lower Bound"].transform("max")
@@ -66,20 +98,3 @@ class BenchmarkRunner:
                 self.results["Gap"] / best_gap
         self.results["Relative Nodes"] = \
                 self.results["Nodes"] / best_nodes
-    
-    def raw_results(self):
-        return self.results
-    
-    def save_results(self, filename):
-        self.results.to_csv(filename)
-        
-    def load_results(self, filename):
-        self.results = pd.read_csv(filename, index_col=0)
-        
-    def load_state(self, filename):
-        for (name, solver) in self.solvers.items():
-            solver.load_state(filename)
-
-    def fit(self):
-        for (name, solver) in self.solvers.items():
-            solver.fit()