From ffb29d2bbb35140930c3ef415739df6b57fc16ea Mon Sep 17 00:00:00 2001 From: Alinson S Xavier Date: Fri, 21 Feb 2020 11:21:01 -0600 Subject: [PATCH] BenchmarkRunner: add solve method --- miplearn/benchmark.py | 107 ++++++++++++++++++++++++------------------ 1 file changed, 61 insertions(+), 46 deletions(-) diff --git a/miplearn/benchmark.py b/miplearn/benchmark.py index 0146772..51f28dd 100644 --- a/miplearn/benchmark.py +++ b/miplearn/benchmark.py @@ -3,7 +3,9 @@ # Written by Alinson S. Xavier from .solvers import LearningSolver +from copy import deepcopy import pandas as pd +from tqdm.auto import tqdm class BenchmarkRunner: def __init__(self, solvers): @@ -13,59 +15,23 @@ class BenchmarkRunner: self.solvers = solvers self.results = None + def solve(self, instances, fit=True, tee=False): + for (name, solver) in self.solvers.items(): + for i in tqdm(range(len((instances)))): + results = solver.solve(deepcopy(instances[i]), tee=tee) + self._push_result(results, solver=solver, name=name, instance=i) + if fit: + solver.fit() + def parallel_solve(self, instances, n_jobs=1, n_trials=1): - if self.results is None: - self.results = pd.DataFrame(columns=["Solver", - "Instance", - "Wallclock Time", - "Lower Bound", - "Upper Bound", - "Gap", - "Nodes", - ]) instances = instances * n_trials for (name, solver) in self.solvers.items(): results = solver.parallel_solve(instances, n_jobs=n_jobs, - label=name, + label="Solve (%s)" % name, collect_training_data=False) for i in range(len(instances)): - wallclock_time = None - for key in ["Time", "Wall time", "Wallclock time"]: - if key not in results[i]["Solver"][0].keys(): - continue - if str(results[i]["Solver"][0][key]) == "": - continue - wallclock_time = float(results[i]["Solver"][0][key]) - nodes = results[i]["Solver"][0]["Nodes"] - lb = results[i]["Problem"][0]["Lower bound"] - ub = results[i]["Problem"][0]["Upper bound"] - gap = (ub - lb) / lb - self.results = self.results.append({ - "Solver": name, - "Instance": i, - "Wallclock Time": wallclock_time, - "Lower Bound": lb, - "Upper Bound": ub, - "Gap": gap, - "Nodes": nodes, - }, ignore_index=True) - groups = self.results.groupby("Instance") - best_lower_bound = groups["Lower Bound"].transform("max") - best_upper_bound = groups["Upper Bound"].transform("min") - best_gap = groups["Gap"].transform("min") - best_nodes = groups["Nodes"].transform("min") - best_wallclock_time = groups["Wallclock Time"].transform("min") - self.results["Relative Lower Bound"] = \ - self.results["Lower Bound"] / best_lower_bound - self.results["Relative Upper Bound"] = \ - self.results["Upper Bound"] / best_upper_bound - self.results["Relative Wallclock Time"] = \ - self.results["Wallclock Time"] / best_wallclock_time - self.results["Relative Gap"] = \ - self.results["Gap"] / best_gap - self.results["Relative Nodes"] = \ - self.results["Nodes"] / best_nodes + self._push_result(results[i], solver=solver, name=name, instance=i) def raw_results(self): return self.results @@ -83,3 +49,52 @@ class BenchmarkRunner: def fit(self): for (name, solver) in self.solvers.items(): solver.fit() + + def _push_result(self, result, solver, name, instance): + if self.results is None: + self.results = pd.DataFrame(columns=["Solver", + "Instance", + "Wallclock Time", + "Lower Bound", + "Upper Bound", + "Gap", + "Nodes", + "Mode", + ]) + wallclock_time = None + for key in ["Time", "Wall time", "Wallclock time"]: + if key not in result["Solver"][0].keys(): + continue + if str(result["Solver"][0][key]) == "": + continue + wallclock_time = float(result["Solver"][0][key]) + nodes = result["Solver"][0]["Nodes"] + lb = result["Problem"][0]["Lower bound"] + ub = result["Problem"][0]["Upper bound"] + gap = (ub - lb) / lb + self.results = self.results.append({ + "Solver": name, + "Instance": instance, + "Wallclock Time": wallclock_time, + "Lower Bound": lb, + "Upper Bound": ub, + "Gap": gap, + "Nodes": nodes, + "Mode": solver.mode, + }, ignore_index=True) + groups = self.results.groupby("Instance") + best_lower_bound = groups["Lower Bound"].transform("max") + best_upper_bound = groups["Upper Bound"].transform("min") + best_gap = groups["Gap"].transform("min") + best_nodes = groups["Nodes"].transform("min") + best_wallclock_time = groups["Wallclock Time"].transform("min") + self.results["Relative Lower Bound"] = \ + self.results["Lower Bound"] / best_lower_bound + self.results["Relative Upper Bound"] = \ + self.results["Upper Bound"] / best_upper_bound + self.results["Relative Wallclock Time"] = \ + self.results["Wallclock Time"] / best_wallclock_time + self.results["Relative Gap"] = \ + self.results["Gap"] / best_gap + self.results["Relative Nodes"] = \ + self.results["Nodes"] / best_nodes