ANL-CEEESA
/
MIPLearn
mirror of https://github.com/ANL-CEEESA/MIPLearn.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Benchmark: Add extra columns to CSV

Browse Source
master
Alinson S. Xavier 5 years ago
parent f77d1d5de9
commit c9ad7a3f56
No known key found for this signature in database
GPG Key ID: A796166E4E218E02
5 changed files with 60 additions and 51 deletions
Show Stats Download Patch File Download Diff File

73
miplearn/benchmark.py
Unescape View File

@ -77,56 +77,37 @@ class BenchmarkRunner:
def _push_result(self, result, solver, solver_name, instance): def _push_result(self, result, solver, solver_name, instance):
if self.results is None: if self.results is None:
self.results = pd.DataFrame( self.results = pd.DataFrame(
# Show the following columns first in the CSV file
columns=[ columns=[
"Solver", "Solver",
"Instance", "Instance",
"Wallclock Time",
"Lower Bound",
"Upper Bound",
"Gap",
"Nodes",
"Mode",
"Sense",
"Predicted LB",
"Predicted UB",
] ]
) )
lb = result["Lower bound"] lb = result["Lower bound"]
ub = result["Upper bound"] ub = result["Upper bound"]
gap = (ub - lb) / lb result["Solver"] = solver_name
if "Predicted LB" not in result: result["Instance"] = instance
result["Predicted LB"] = float("nan") result["Gap"] = (ub - lb) / lb
result["Predicted UB"] = float("nan") result["Mode"] = solver.mode
self.results = self.results.append( del result["Log"]
{ self.results = self.results.append(pd.DataFrame([result]))
"Solver": solver_name,
"Instance": instance, # Compute relative statistics
"Wallclock Time": result["Wallclock time"],
"Lower Bound": lb,
"Upper Bound": ub,
"Gap": gap,
"Nodes": result["Nodes"],
"Mode": solver.mode,
"Sense": result["Sense"],
"Predicted LB": result["Predicted LB"],
"Predicted UB": result["Predicted UB"],
},
ignore_index=True,
)
groups = self.results.groupby("Instance") groups = self.results.groupby("Instance")
best_lower_bound = groups["Lower Bound"].transform("max") best_lower_bound = groups["Lower bound"].transform("max")
best_upper_bound = groups["Upper Bound"].transform("min") best_upper_bound = groups["Upper bound"].transform("min")
best_gap = groups["Gap"].transform("min") best_gap = groups["Gap"].transform("min")
best_nodes = np.maximum(1, groups["Nodes"].transform("min")) best_nodes = np.maximum(1, groups["Nodes"].transform("min"))
best_wallclock_time = groups["Wallclock Time"].transform("min") best_wallclock_time = groups["Wallclock time"].transform("min")
self.results["Relative Lower Bound"] = ( self.results["Relative lower bound"] = (
self.results["Lower Bound"] / best_lower_bound self.results["Lower bound"] / best_lower_bound
) )
self.results["Relative Upper Bound"] = ( self.results["Relative upper bound"] = (
self.results["Upper Bound"] / best_upper_bound self.results["Upper bound"] / best_upper_bound
) )
self.results["Relative Wallclock Time"] = ( self.results["Relative wallclock time"] = (
self.results["Wallclock Time"] / best_wallclock_time self.results["Wallclock time"] / best_wallclock_time
) )
self.results["Relative Gap"] = self.results["Gap"] / best_gap self.results["Relative Gap"] = self.results["Gap"] / best_gap
self.results["Relative Nodes"] = self.results["Nodes"] / best_nodes self.results["Relative Nodes"] = self.results["Nodes"] / best_nodes
@ -143,12 +124,12 @@ class BenchmarkRunner:
sense = results.loc[0, "Sense"] sense = results.loc[0, "Sense"]
if sense == "min": if sense == "min":
primal_column = "Relative Upper Bound" primal_column = "Relative upper bound"
obj_column = "Upper Bound" obj_column = "Upper bound"
predicted_obj_column = "Predicted UB" predicted_obj_column = "Predicted UB"
else: else:
primal_column = "Relative Lower Bound" primal_column = "Relative lower bound"
obj_column = "Lower Bound" obj_column = "Lower bound"
predicted_obj_column = "Predicted LB" predicted_obj_column = "Predicted LB"
fig, (ax1, ax2, ax3, ax4) = plt.subplots( fig, (ax1, ax2, ax3, ax4) = plt.subplots(
@ -158,10 +139,10 @@ class BenchmarkRunner:
gridspec_kw={"width_ratios": [2, 1, 1, 2]}, gridspec_kw={"width_ratios": [2, 1, 1, 2]},
) )
# Figure 1: Solver x Wallclock Time # Figure 1: Solver x Wallclock time
sns.stripplot( sns.stripplot(
x="Solver", x="Solver",
y="Wallclock Time", y="Wallclock time",
data=results, data=results,
ax=ax1, ax=ax1,
jitter=0.25, jitter=0.25,
@ -169,14 +150,14 @@ class BenchmarkRunner:
) )
sns.barplot( sns.barplot(
x="Solver", x="Solver",
y="Wallclock Time", y="Wallclock time",
data=results, data=results,
ax=ax1, ax=ax1,
errwidth=0.0, errwidth=0.0,
alpha=0.4, alpha=0.4,
estimator=median, estimator=median,
) )
ax1.set(ylabel="Wallclock Time (s)") ax1.set(ylabel="Wallclock time (s)")
# Figure 2: Solver x Gap (%) # Figure 2: Solver x Gap (%)
ax2.set_ylim(-0.5, 5.5) ax2.set_ylim(-0.5, 5.5)

16
miplearn/components/steps/convert_tight.py
Unescape View File

@ -51,6 +51,11 @@ class ConvertTightIneqsIntoEqsStep(Component):
return_constraints=True, return_constraints=True,
) )
y = self.predict(x) y = self.predict(x)
self.total_converted = 0
self.total_restored = 0
self.total_kept = 0
self.total_iterations = 0
for category in y.keys(): for category in y.keys():
for i in range(len(y[category])): for i in range(len(y[category])):
if y[category][i][0] == 1: if y[category][i][0] == 1:
@ -59,10 +64,17 @@ class ConvertTightIneqsIntoEqsStep(Component):
self.original_sense[cid] = s self.original_sense[cid] = s
solver.internal_solver.set_constraint_sense(cid, "=") solver.internal_solver.set_constraint_sense(cid, "=")
self.converted += [cid] self.converted += [cid]
logger.info(f"Converted {len(self.converted)} inequalities") self.total_converted += 1
else:
self.total_kept += 1
logger.info(f"Converted {self.total_converted} inequalities")
def after_solve(self, solver, instance, model, results): def after_solve(self, solver, instance, model, results):
instance.slacks = solver.internal_solver.get_inequality_slacks() instance.slacks = solver.internal_solver.get_inequality_slacks()
results["ConvertTight: Kept"] = self.total_kept
results["ConvertTight: Converted"] = self.total_converted
results["ConvertTight: Restored"] = self.total_restored
results["ConvertTight: Iterations"] = self.total_iterations
def fit(self, training_instances): def fit(self, training_instances):
logger.debug("Extracting x and y...") logger.debug("Extracting x and y...")
@ -173,7 +185,9 @@ class ConvertTightIneqsIntoEqsStep(Component):
for cid in restored: for cid in restored:
self.converted.remove(cid) self.converted.remove(cid)
if len(restored) > 0: if len(restored) > 0:
self.total_restored += len(restored)
logger.info(f"Restored {len(restored)} inequalities") logger.info(f"Restored {len(restored)} inequalities")
self.total_iterations += 1
return True return True
else: else:
return False return False

16
miplearn/components/steps/drop_redundant.py
Unescape View File

@ -57,6 +57,11 @@ class DropRedundantInequalitiesStep(Component):
return_constraints=True, return_constraints=True,
) )
y = self.predict(x) y = self.predict(x)
self.total_dropped = 0
self.total_restored = 0
self.total_kept = 0
self.total_iterations = 0
for category in y.keys(): for category in y.keys():
for i in range(len(y[category])): for i in range(len(y[category])):
if y[category][i][0] == 1: if y[category][i][0] == 1:
@ -66,10 +71,17 @@ class DropRedundantInequalitiesStep(Component):
obj=solver.internal_solver.extract_constraint(cid), obj=solver.internal_solver.extract_constraint(cid),
) )
self.pool += [c] self.pool += [c]
logger.info("Extracted %d predicted constraints" % len(self.pool)) self.total_dropped += 1
else:
self.total_kept += 1
logger.info(f"Extracted {self.total_dropped} predicted constraints")
def after_solve(self, solver, instance, model, results): def after_solve(self, solver, instance, model, results):
instance.slacks = solver.internal_solver.get_inequality_slacks() instance.slacks = solver.internal_solver.get_inequality_slacks()
results["DropRedundant: Kept"] = self.total_kept
results["DropRedundant: Dropped"] = self.total_dropped
results["DropRedundant: Restored"] = self.total_restored
results["DropRedundant: Iterations"] = self.total_iterations
def fit(self, training_instances): def fit(self, training_instances):
logger.debug("Extracting x and y...") logger.debug("Extracting x and y...")
@ -180,10 +192,12 @@ class DropRedundantInequalitiesStep(Component):
self.pool.remove(c) self.pool.remove(c)
solver.internal_solver.add_constraint(c.obj) solver.internal_solver.add_constraint(c.obj)
if len(constraints_to_add) > 0: if len(constraints_to_add) > 0:
self.total_restored += len(constraints_to_add)
logger.info( logger.info(
"%8d constraints %8d in the pool" "%8d constraints %8d in the pool"
% (len(constraints_to_add), len(self.pool)) % (len(constraints_to_add), len(self.pool))
) )
self.total_iterations += 1
return True return True
else: else:
return False return False

2
miplearn/components/tests/test_relaxation.py
Unescape View File

@ -115,7 +115,7 @@ def test_drop_redundant():
) )
# LearningSolver calls after_solve # LearningSolver calls after_solve
component.after_solve(solver, instance, None, None) component.after_solve(solver, instance, None, {})
# Should query slack for all inequalities # Should query slack for all inequalities
internal.get_inequality_slacks.assert_called_once() internal.get_inequality_slacks.assert_called_once()

4
miplearn/tests/test_benchmark.py
Unescape View File

@ -27,11 +27,11 @@ def test_benchmark():
benchmark = BenchmarkRunner(test_solvers) benchmark = BenchmarkRunner(test_solvers)
benchmark.fit(train_instances) benchmark.fit(train_instances)
benchmark.parallel_solve(test_instances, n_jobs=2, n_trials=2) benchmark.parallel_solve(test_instances, n_jobs=2, n_trials=2)
assert benchmark.raw_results().values.shape == (12, 16) assert benchmark.raw_results().values.shape == (12, 18)
benchmark.save_results("/tmp/benchmark.csv") benchmark.save_results("/tmp/benchmark.csv")
assert os.path.isfile("/tmp/benchmark.csv") assert os.path.isfile("/tmp/benchmark.csv")
benchmark = BenchmarkRunner(test_solvers) benchmark = BenchmarkRunner(test_solvers)
benchmark.load_results("/tmp/benchmark.csv") benchmark.load_results("/tmp/benchmark.csv")
assert benchmark.raw_results().values.shape == (12, 16) assert benchmark.raw_results().values.shape == (12, 18)

Write Preview
Loading…
Cancel
Save