Module miplearn.benchmark
Expand source code
# MIPLearn: Extensible Framework for Learning-Enhanced Mixed-Integer Optimization
# Copyright (C) 2020, UChicago Argonne, LLC. All rights reserved.
# Released under the modified BSD license. See COPYING.md for more details.
import logging
import os
from typing import Dict, Union, List
import pandas as pd
from miplearn.instance import Instance
from miplearn.solvers.learning import LearningSolver
from miplearn.types import LearningSolveStats
class BenchmarkRunner:
"""
Utility class that simplifies the task of comparing the performance of different
solvers.
Example
-------
```python
benchmark = BenchmarkRunner({
"Baseline": LearningSolver(...),
"Strategy A": LearningSolver(...),
"Strategy B": LearningSolver(...),
"Strategy C": LearningSolver(...),
})
benchmark.fit(train_instances)
benchmark.parallel_solve(test_instances, n_jobs=5)
benchmark.save_results("result.csv")
```
Parameters
----------
solvers: Dict[str, LearningSolver]
Dictionary containing the solvers to compare. Solvers may have different
arguments and components. The key should be the name of the solver. It
appears in the exported tables of results.
"""
def __init__(self, solvers: Dict[str, LearningSolver]) -> None:
self.solvers: Dict[str, LearningSolver] = solvers
self.results = pd.DataFrame(
columns=[
"Solver",
"Instance",
]
)
def parallel_solve(
self,
instances: Union[List[str], List[Instance]],
n_jobs: int = 1,
n_trials: int = 3,
) -> None:
"""
Solves the given instances in parallel and collect benchmark statistics.
Parameters
----------
instances: Union[List[str], List[Instance]]
List of instances to solve. This can either be a list of instances
already loaded in memory, or a list of filenames pointing to pickled (and
optionally gzipped) files.
n_jobs: int
List of instances to solve in parallel at a time.
n_trials: int
How many times each instance should be solved.
"""
self._silence_miplearn_logger()
trials = instances * n_trials
for (solver_name, solver) in self.solvers.items():
results = solver.parallel_solve(
trials,
n_jobs=n_jobs,
label="Solve (%s)" % solver_name,
discard_outputs=True,
)
for i in range(len(trials)):
idx = i % len(instances)
results[i]["Solver"] = solver_name
results[i]["Instance"] = idx
self.results = self.results.append(pd.DataFrame([results[i]]))
self._restore_miplearn_logger()
def write_csv(self, filename: str) -> None:
"""
Writes the collected results to a CSV file.
Parameters
----------
filename: str
The name of the file.
"""
os.makedirs(os.path.dirname(filename), exist_ok=True)
self.results.to_csv(filename)
def fit(self, instances: Union[List[str], List[Instance]]) -> None:
"""
Trains all solvers with the provided training instances.
Parameters
----------
instances: Union[List[str], List[Instance]]
List of training instances. This can either be a list of instances
already loaded in memory, or a list of filenames pointing to pickled (and
optionally gzipped) files.
"""
for (solver_name, solver) in self.solvers.items():
solver.fit(instances)
def _silence_miplearn_logger(self) -> None:
miplearn_logger = logging.getLogger("miplearn")
self.prev_log_level = miplearn_logger.getEffectiveLevel()
miplearn_logger.setLevel(logging.WARNING)
def _restore_miplearn_logger(self) -> None:
miplearn_logger = logging.getLogger("miplearn")
miplearn_logger.setLevel(self.prev_log_level)Classes
class BenchmarkRunner (solvers)-
Utility class that simplifies the task of comparing the performance of different solvers.
Example
benchmark = BenchmarkRunner({ "Baseline": LearningSolver(...), "Strategy A": LearningSolver(...), "Strategy B": LearningSolver(...), "Strategy C": LearningSolver(...), }) benchmark.fit(train_instances) benchmark.parallel_solve(test_instances, n_jobs=5) benchmark.save_results("result.csv")Parameters
solvers:Dict[str,LearningSolver]- Dictionary containing the solvers to compare. Solvers may have different arguments and components. The key should be the name of the solver. It appears in the exported tables of results.
Expand source code
class BenchmarkRunner: """ Utility class that simplifies the task of comparing the performance of different solvers. Example ------- ```python benchmark = BenchmarkRunner({ "Baseline": LearningSolver(...), "Strategy A": LearningSolver(...), "Strategy B": LearningSolver(...), "Strategy C": LearningSolver(...), }) benchmark.fit(train_instances) benchmark.parallel_solve(test_instances, n_jobs=5) benchmark.save_results("result.csv") ``` Parameters ---------- solvers: Dict[str, LearningSolver] Dictionary containing the solvers to compare. Solvers may have different arguments and components. The key should be the name of the solver. It appears in the exported tables of results. """ def __init__(self, solvers: Dict[str, LearningSolver]) -> None: self.solvers: Dict[str, LearningSolver] = solvers self.results = pd.DataFrame( columns=[ "Solver", "Instance", ] ) def parallel_solve( self, instances: Union[List[str], List[Instance]], n_jobs: int = 1, n_trials: int = 3, ) -> None: """ Solves the given instances in parallel and collect benchmark statistics. Parameters ---------- instances: Union[List[str], List[Instance]] List of instances to solve. This can either be a list of instances already loaded in memory, or a list of filenames pointing to pickled (and optionally gzipped) files. n_jobs: int List of instances to solve in parallel at a time. n_trials: int How many times each instance should be solved. """ self._silence_miplearn_logger() trials = instances * n_trials for (solver_name, solver) in self.solvers.items(): results = solver.parallel_solve( trials, n_jobs=n_jobs, label="Solve (%s)" % solver_name, discard_outputs=True, ) for i in range(len(trials)): idx = i % len(instances) results[i]["Solver"] = solver_name results[i]["Instance"] = idx self.results = self.results.append(pd.DataFrame([results[i]])) self._restore_miplearn_logger() def write_csv(self, filename: str) -> None: """ Writes the collected results to a CSV file. Parameters ---------- filename: str The name of the file. """ os.makedirs(os.path.dirname(filename), exist_ok=True) self.results.to_csv(filename) def fit(self, instances: Union[List[str], List[Instance]]) -> None: """ Trains all solvers with the provided training instances. Parameters ---------- instances: Union[List[str], List[Instance]] List of training instances. This can either be a list of instances already loaded in memory, or a list of filenames pointing to pickled (and optionally gzipped) files. """ for (solver_name, solver) in self.solvers.items(): solver.fit(instances) def _silence_miplearn_logger(self) -> None: miplearn_logger = logging.getLogger("miplearn") self.prev_log_level = miplearn_logger.getEffectiveLevel() miplearn_logger.setLevel(logging.WARNING) def _restore_miplearn_logger(self) -> None: miplearn_logger = logging.getLogger("miplearn") miplearn_logger.setLevel(self.prev_log_level)Methods
def fit(self, instances)-
Trains all solvers with the provided training instances.
Parameters
instances:Union[List[str],List[Instance]]- List of training instances. This can either be a list of instances already loaded in memory, or a list of filenames pointing to pickled (and optionally gzipped) files.
Expand source code
def fit(self, instances: Union[List[str], List[Instance]]) -> None: """ Trains all solvers with the provided training instances. Parameters ---------- instances: Union[List[str], List[Instance]] List of training instances. This can either be a list of instances already loaded in memory, or a list of filenames pointing to pickled (and optionally gzipped) files. """ for (solver_name, solver) in self.solvers.items(): solver.fit(instances) def parallel_solve(self, instances, n_jobs=1, n_trials=3)-
Solves the given instances in parallel and collect benchmark statistics.
Parameters
instances:Union[List[str],List[Instance]]- List of instances to solve. This can either be a list of instances already loaded in memory, or a list of filenames pointing to pickled (and optionally gzipped) files.
n_jobs:int- List of instances to solve in parallel at a time.
n_trials:int- How many times each instance should be solved.
Expand source code
def parallel_solve( self, instances: Union[List[str], List[Instance]], n_jobs: int = 1, n_trials: int = 3, ) -> None: """ Solves the given instances in parallel and collect benchmark statistics. Parameters ---------- instances: Union[List[str], List[Instance]] List of instances to solve. This can either be a list of instances already loaded in memory, or a list of filenames pointing to pickled (and optionally gzipped) files. n_jobs: int List of instances to solve in parallel at a time. n_trials: int How many times each instance should be solved. """ self._silence_miplearn_logger() trials = instances * n_trials for (solver_name, solver) in self.solvers.items(): results = solver.parallel_solve( trials, n_jobs=n_jobs, label="Solve (%s)" % solver_name, discard_outputs=True, ) for i in range(len(trials)): idx = i % len(instances) results[i]["Solver"] = solver_name results[i]["Instance"] = idx self.results = self.results.append(pd.DataFrame([results[i]])) self._restore_miplearn_logger() def write_csv(self, filename)-
Writes the collected results to a CSV file.
Parameters
filename:str- The name of the file.
Expand source code
def write_csv(self, filename: str) -> None: """ Writes the collected results to a CSV file. Parameters ---------- filename: str The name of the file. """ os.makedirs(os.path.dirname(filename), exist_ok=True) self.results.to_csv(filename)