parent
2a405f7ce3
commit
beb15f7667
@ -1,31 +0,0 @@
|
||||
# MIPLearn: Extensible Framework for Learning-Enhanced Mixed-Integer Optimization
|
||||
# Copyright (C) 2020, UChicago Argonne, LLC. All rights reserved.
|
||||
# Released under the modified BSD license. See COPYING.md for more details.
|
||||
# Written by Alinson S. Xavier <axavier@anl.gov>
|
||||
|
||||
CHALLENGES := \
|
||||
stab/ChallengeA \
|
||||
knapsack/ChallengeA \
|
||||
tsp/ChallengeA
|
||||
|
||||
test: $(addsuffix /performance.png, $(CHALLENGES))
|
||||
|
||||
train: $(addsuffix /train/done, $(CHALLENGES))
|
||||
|
||||
%/train/done:
|
||||
python benchmark.py train $*
|
||||
|
||||
%/benchmark_baseline.csv: %/train/done
|
||||
python benchmark.py test-baseline $*
|
||||
|
||||
%/benchmark_ml.csv: %/benchmark_baseline.csv
|
||||
python benchmark.py test-ml $*
|
||||
|
||||
%/performance.png: %/benchmark_ml.csv
|
||||
python benchmark.py charts $*
|
||||
|
||||
clean:
|
||||
rm -rvf $(CHALLENGES)
|
||||
|
||||
.PHONY: clean
|
||||
.SECONDARY:
|
||||
@ -1,268 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# MIPLearn: Extensible Framework for Learning-Enhanced Mixed-Integer Optimization
|
||||
# Copyright (C) 2020-2021, UChicago Argonne, LLC. All rights reserved.
|
||||
# Released under the modified BSD license. See COPYING.md for more details.
|
||||
|
||||
"""MIPLearn Benchmark Scripts
|
||||
|
||||
Usage:
|
||||
benchmark.py train [options] <challenge>
|
||||
benchmark.py test-baseline [options] <challenge>
|
||||
benchmark.py test-ml [options] <challenge>
|
||||
benchmark.py charts <challenge>
|
||||
|
||||
Options:
|
||||
-h --help Show this screen
|
||||
--train-jobs=<n> Number of instances to solve in parallel during training [default: 10]
|
||||
--train-time-limit=<n> Solver time limit during training in seconds [default: 900]
|
||||
--test-jobs=<n> Number of instances to solve in parallel during test [default: 5]
|
||||
--test-time-limit=<n> Solver time limit during test in seconds [default: 900]
|
||||
--solver-threads=<n> Number of threads the solver is allowed to use [default: 4]
|
||||
"""
|
||||
import glob
|
||||
import importlib
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import pandas as pd
|
||||
import seaborn as sns
|
||||
from docopt import docopt
|
||||
from numpy import median
|
||||
|
||||
from miplearn import (
|
||||
LearningSolver,
|
||||
BenchmarkRunner,
|
||||
GurobiPyomoSolver,
|
||||
setup_logger,
|
||||
PickleGzInstance,
|
||||
write_pickle_gz_multiple,
|
||||
Instance,
|
||||
)
|
||||
|
||||
setup_logger()
|
||||
logging.getLogger("gurobipy").setLevel(logging.ERROR)
|
||||
logging.getLogger("pyomo.core").setLevel(logging.ERROR)
|
||||
logger = logging.getLogger("benchmark")
|
||||
|
||||
|
||||
def train(args: Dict) -> None:
|
||||
basepath = args["<challenge>"]
|
||||
problem_name, challenge_name = args["<challenge>"].split("/")
|
||||
pkg = importlib.import_module(f"miplearn.problems.{problem_name}")
|
||||
challenge = getattr(pkg, challenge_name)()
|
||||
|
||||
if not os.path.isdir(f"{basepath}/train"):
|
||||
write_pickle_gz_multiple(challenge.training_instances, f"{basepath}/train")
|
||||
write_pickle_gz_multiple(challenge.test_instances, f"{basepath}/test")
|
||||
|
||||
done_filename = f"{basepath}/train/done"
|
||||
if not os.path.isfile(done_filename):
|
||||
train_instances: List[Instance] = [
|
||||
PickleGzInstance(f) for f in glob.glob(f"{basepath}/train/*.gz")
|
||||
]
|
||||
solver = LearningSolver(
|
||||
solver=GurobiPyomoSolver(
|
||||
params={
|
||||
"TimeLimit": int(args["--train-time-limit"]),
|
||||
"Threads": int(args["--solver-threads"]),
|
||||
}
|
||||
),
|
||||
)
|
||||
solver.parallel_solve(
|
||||
train_instances,
|
||||
n_jobs=int(args["--train-jobs"]),
|
||||
)
|
||||
Path(done_filename).touch(exist_ok=True)
|
||||
|
||||
|
||||
def test_baseline(args: Dict) -> None:
|
||||
basepath = args["<challenge>"]
|
||||
test_instances: List[Instance] = [
|
||||
PickleGzInstance(f) for f in glob.glob(f"{basepath}/test/*.gz")
|
||||
]
|
||||
csv_filename = f"{basepath}/benchmark_baseline.csv"
|
||||
if not os.path.isfile(csv_filename):
|
||||
solvers = {
|
||||
"baseline": LearningSolver(
|
||||
solver=GurobiPyomoSolver(
|
||||
params={
|
||||
"TimeLimit": int(args["--test-time-limit"]),
|
||||
"Threads": int(args["--solver-threads"]),
|
||||
}
|
||||
),
|
||||
),
|
||||
}
|
||||
benchmark = BenchmarkRunner(solvers)
|
||||
benchmark.parallel_solve(
|
||||
test_instances,
|
||||
n_jobs=int(args["--test-jobs"]),
|
||||
)
|
||||
benchmark.write_csv(csv_filename)
|
||||
|
||||
|
||||
def test_ml(args: Dict) -> None:
|
||||
basepath = args["<challenge>"]
|
||||
test_instances: List[Instance] = [
|
||||
PickleGzInstance(f) for f in glob.glob(f"{basepath}/test/*.gz")
|
||||
]
|
||||
train_instances: List[Instance] = [
|
||||
PickleGzInstance(f) for f in glob.glob(f"{basepath}/train/*.gz")
|
||||
]
|
||||
csv_filename = f"{basepath}/benchmark_ml.csv"
|
||||
if not os.path.isfile(csv_filename):
|
||||
solvers = {
|
||||
"ml-exact": LearningSolver(
|
||||
solver=GurobiPyomoSolver(
|
||||
params={
|
||||
"TimeLimit": int(args["--test-time-limit"]),
|
||||
"Threads": int(args["--solver-threads"]),
|
||||
}
|
||||
),
|
||||
),
|
||||
"ml-heuristic": LearningSolver(
|
||||
solver=GurobiPyomoSolver(
|
||||
params={
|
||||
"TimeLimit": int(args["--test-time-limit"]),
|
||||
"Threads": int(args["--solver-threads"]),
|
||||
}
|
||||
),
|
||||
mode="heuristic",
|
||||
),
|
||||
}
|
||||
benchmark = BenchmarkRunner(solvers)
|
||||
benchmark.fit(train_instances)
|
||||
benchmark.parallel_solve(
|
||||
test_instances,
|
||||
n_jobs=int(args["--test-jobs"]),
|
||||
)
|
||||
benchmark.write_csv(csv_filename)
|
||||
|
||||
|
||||
def charts(args: Dict) -> None:
|
||||
basepath = args["<challenge>"]
|
||||
sns.set_style("whitegrid")
|
||||
sns.set_palette("Blues_r")
|
||||
|
||||
csv_files = [
|
||||
f"{basepath}/benchmark_baseline.csv",
|
||||
f"{basepath}/benchmark_ml.csv",
|
||||
]
|
||||
results = pd.concat(map(pd.read_csv, csv_files))
|
||||
groups = results.groupby("Instance")
|
||||
best_lower_bound = groups["Lower bound"].transform("max")
|
||||
best_upper_bound = groups["Upper bound"].transform("min")
|
||||
results["Relative lower bound"] = results["Lower bound"] / best_lower_bound
|
||||
results["Relative upper bound"] = results["Upper bound"] / best_upper_bound
|
||||
|
||||
sense = results.loc[0, "Sense"]
|
||||
if (sense == "min").any():
|
||||
primal_column = "Relative upper bound"
|
||||
obj_column = "Upper bound"
|
||||
predicted_obj_column = "Objective: Predicted upper bound"
|
||||
else:
|
||||
primal_column = "Relative lower bound"
|
||||
obj_column = "Lower bound"
|
||||
predicted_obj_column = "Objective: Predicted lower bound"
|
||||
|
||||
palette = {"baseline": "#9b59b6", "ml-exact": "#3498db", "ml-heuristic": "#95a5a6"}
|
||||
fig, (ax1, ax2, ax3, ax4) = plt.subplots(
|
||||
nrows=1,
|
||||
ncols=4,
|
||||
figsize=(12, 4),
|
||||
gridspec_kw={"width_ratios": [2, 1, 1, 2]},
|
||||
)
|
||||
|
||||
# Wallclock time
|
||||
sns.stripplot(
|
||||
x="Solver",
|
||||
y="Wallclock time",
|
||||
data=results,
|
||||
ax=ax1,
|
||||
jitter=0.25,
|
||||
palette=palette,
|
||||
size=4.0,
|
||||
)
|
||||
sns.barplot(
|
||||
x="Solver",
|
||||
y="Wallclock time",
|
||||
data=results,
|
||||
ax=ax1,
|
||||
errwidth=0.0,
|
||||
alpha=0.4,
|
||||
palette=palette,
|
||||
estimator=median,
|
||||
)
|
||||
ax1.set(ylabel="Wallclock time (s)")
|
||||
|
||||
# Gap
|
||||
ax2.set_ylim(-0.5, 5.5)
|
||||
sns.stripplot(
|
||||
x="Solver",
|
||||
y="Gap",
|
||||
jitter=0.25,
|
||||
data=results[results["Solver"] != "ml-heuristic"],
|
||||
ax=ax2,
|
||||
palette=palette,
|
||||
size=4.0,
|
||||
)
|
||||
|
||||
# Relative primal bound
|
||||
ax3.set_ylim(0.95, 1.05)
|
||||
sns.stripplot(
|
||||
x="Solver",
|
||||
y=primal_column,
|
||||
jitter=0.25,
|
||||
data=results[results["Solver"] == "ml-heuristic"],
|
||||
ax=ax3,
|
||||
palette=palette,
|
||||
)
|
||||
sns.scatterplot(
|
||||
x=obj_column,
|
||||
y=predicted_obj_column,
|
||||
hue="Solver",
|
||||
data=results[results["Solver"] == "ml-exact"],
|
||||
ax=ax4,
|
||||
palette=palette,
|
||||
)
|
||||
|
||||
# Predicted vs actual primal bound
|
||||
xlim, ylim = ax4.get_xlim(), ax4.get_ylim()
|
||||
ax4.plot(
|
||||
[-1e10, 1e10],
|
||||
[-1e10, 1e10],
|
||||
ls="-",
|
||||
color="#cccccc",
|
||||
)
|
||||
ax4.set_xlim(xlim)
|
||||
ax4.set_ylim(ylim)
|
||||
ax4.get_legend().remove()
|
||||
ax4.set(
|
||||
ylabel="Predicted value",
|
||||
xlabel="Actual value",
|
||||
)
|
||||
|
||||
fig.tight_layout()
|
||||
plt.savefig(
|
||||
f"{basepath}/performance.png",
|
||||
bbox_inches="tight",
|
||||
dpi=150,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = docopt(__doc__)
|
||||
if args["train"]:
|
||||
train(args)
|
||||
if args["test-baseline"]:
|
||||
test_baseline(args)
|
||||
if args["test-ml"]:
|
||||
test_ml(args)
|
||||
if args["charts"]:
|
||||
charts(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in new issue