ANL-CEEESA
/
UnitCommitment.jl
mirror of https://github.com/ANL-CEEESA/UnitCommitment.jl.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update benchmark scripts

Browse Source
bugfix/formulations
Alinson S. Xavier 4 years ago
parent f01562e37f
commit 7db8d723f7
6 changed files with 245 additions and 221 deletions
Show Stats Download Patch File Download Diff File

2
Makefile
Unescape View File

@ -8,7 +8,7 @@ VERSION := 0.2
build/sysimage.so: src/utils/sysimage.jl Project.toml Manifest.toml build/sysimage.so: src/utils/sysimage.jl Project.toml Manifest.toml
mkdir -p build mkdir -p build
mkdir -p benchmark/results/test mkdir -p benchmark/results/test
cd benchmark; $(JULIA) --trace-compile=../build/precompile.jl run.jl test/case14.1.sol.json cd benchmark; $(JULIA) --trace-compile=../build/precompile.jl benchmark.jl test/case14
$(JULIA) src/utils/sysimage.jl $(JULIA) src/utils/sysimage.jl
clean: clean:

105
benchmark/Makefile
Unescape View File

@ -1,105 +0,0 @@
# UnitCommitment.jl: Optimization Package for Security-Constrained Unit Commitment
# Copyright (C) 2020, UChicago Argonne, LLC. All rights reserved.
# Released under the modified BSD license. See COPYING.md for more details.
SHELL := /bin/bash
JULIA := julia --project=. --sysimage ../build/sysimage.so
TIMESTAMP := $(shell date "+%Y-%m-%d %H:%M")
SRC_FILES := $(wildcard ../src/*.jl)
INSTANCES_PGLIB := \
pglib-uc/ca/2014-09-01_reserves_0 \
pglib-uc/ca/2014-09-01_reserves_1 \
pglib-uc/ca/2015-03-01_reserves_0 \
pglib-uc/ca/2015-06-01_reserves_0 \
pglib-uc/ca/Scenario400_reserves_1 \
pglib-uc/ferc/2015-01-01_lw \
pglib-uc/ferc/2015-05-01_lw \
pglib-uc/ferc/2015-07-01_hw \
pglib-uc/ferc/2015-10-01_lw \
pglib-uc/ferc/2015-12-01_lw \
pglib-uc/rts_gmlc/2020-04-03 \
pglib-uc/rts_gmlc/2020-09-20 \
pglib-uc/rts_gmlc/2020-10-27 \
pglib-uc/rts_gmlc/2020-11-25 \
pglib-uc/rts_gmlc/2020-12-23
INSTANCES_MATPOWER := \
matpower/case118/2017-02-01 \
matpower/case118/2017-08-01 \
matpower/case300/2017-02-01 \
matpower/case300/2017-08-01 \
matpower/case1354pegase/2017-02-01 \
matpower/case1888rte/2017-02-01 \
matpower/case1951rte/2017-08-01 \
matpower/case2848rte/2017-02-01 \
matpower/case2868rte/2017-08-01 \
matpower/case3375wp/2017-08-01 \
matpower/case6468rte/2017-08-01 \
matpower/case6515rte/2017-08-01
INSTANCES_ORLIB := \
or-lib/20_0_1_w \
or-lib/20_0_5_w \
or-lib/50_0_2_w \
or-lib/75_0_2_w \
or-lib/100_0_1_w \
or-lib/100_0_4_w \
or-lib/100_0_5_w \
or-lib/200_0_3_w \
or-lib/200_0_7_w \
or-lib/200_0_9_w
INSTANCES_TEJADA19 := \
tejada19/UC_24h_290g \
tejada19/UC_24h_623g \
tejada19/UC_24h_959g \
tejada19/UC_24h_1577g \
tejada19/UC_24h_1888g \
tejada19/UC_168h_72g \
tejada19/UC_168h_86g \
tejada19/UC_168h_130g \
tejada19/UC_168h_131g \
tejada19/UC_168h_199g
SAMPLES := 1 2 3 4 5
SOLUTIONS_MATPOWER := $(foreach s,$(SAMPLES),$(addprefix results/,$(addsuffix .$(s).sol.json,$(INSTANCES_MATPOWER))))
SOLUTIONS_PGLIB := $(foreach s,$(SAMPLES),$(addprefix results/,$(addsuffix .$(s).sol.json,$(INSTANCES_PGLIB))))
SOLUTIONS_ORLIB := $(foreach s,$(SAMPLES),$(addprefix results/,$(addsuffix .$(s).sol.json,$(INSTANCES_ORLIB))))
SOLUTIONS_TEJADA19 := $(foreach s,$(SAMPLES),$(addprefix results/,$(addsuffix .$(s).sol.json,$(INSTANCES_TEJADA19))))
.PHONY: matpower pglib orlib tejada19 clean clean-mps clean-sol save tables
all: matpower pglib orlib tejada19
matpower: $(SOLUTIONS_MATPOWER)
pglib: $(SOLUTIONS_PGLIB)
orlib: $(SOLUTIONS_ORLIB)
tejada19: $(SOLUTIONS_TEJADA19)
clean:
@rm -rf tables/benchmark* tables/compare* results
clean-mps:
@rm -fv results/*/*.mps.gz results/*/*/*.mps.gz
clean-sol:
@rm -rf results/*/*.sol.* results/*/*/*.sol.*
save:
mkdir -p "runs/$(TIMESTAMP)"
rsync -avP results tables "runs/$(TIMESTAMP)/"
results/%.sol.json: run.jl
@echo "run $*"
@mkdir -p $(dir results/$*)
@$(JULIA) run.jl $* 2>&1 | cat > results/$*.log
@echo "run $* [done]"
tables:
@mkdir -p tables
@python scripts/table.py
#@python scripts/compare.py tables/reference.csv tables/benchmark.csv

162
benchmark/benchmark.jl
Unescape View File

@ -0,0 +1,162 @@
# UnitCommitment.jl: Optimization Package for Security-Constrained Unit Commitment
# Copyright (C) 2020, UChicago Argonne, LLC. All rights reserved.
# Released under the modified BSD license. See COPYING.md for more details.
using Distributed
using Pkg
Pkg.activate(".")
@everywhere using Pkg
@everywhere Pkg.activate(".")
@everywhere using UnitCommitment
@everywhere using JuMP
@everywhere using Gurobi
@everywhere using JSON
@everywhere using Logging
@everywhere using Printf
@everywhere using LinearAlgebra
@everywhere using Random
@everywhere UnitCommitment._setup_logger()
function main()
cases = [
"pglib-uc/ca/2014-09-01_reserves_0",
"pglib-uc/ca/2014-09-01_reserves_1",
"pglib-uc/ca/2015-03-01_reserves_0",
"pglib-uc/ca/2015-06-01_reserves_0",
"pglib-uc/ca/Scenario400_reserves_1",
"pglib-uc/ferc/2015-01-01_lw",
"pglib-uc/ferc/2015-05-01_lw",
"pglib-uc/ferc/2015-07-01_hw",
"pglib-uc/ferc/2015-10-01_lw",
"pglib-uc/ferc/2015-12-01_lw",
"pglib-uc/rts_gmlc/2020-04-03",
"pglib-uc/rts_gmlc/2020-09-20",
"pglib-uc/rts_gmlc/2020-10-27",
"pglib-uc/rts_gmlc/2020-11-25",
"pglib-uc/rts_gmlc/2020-12-23",
"or-lib/20_0_1_w",
"or-lib/20_0_5_w",
"or-lib/50_0_2_w",
"or-lib/75_0_2_w",
"or-lib/100_0_1_w",
"or-lib/100_0_4_w",
"or-lib/100_0_5_w",
"or-lib/200_0_3_w",
"or-lib/200_0_7_w",
"or-lib/200_0_9_w",
"tejada19/UC_24h_290g",
"tejada19/UC_24h_623g",
"tejada19/UC_24h_959g",
"tejada19/UC_24h_1577g",
"tejada19/UC_24h_1888g",
"tejada19/UC_168h_72g",
"tejada19/UC_168h_86g",
"tejada19/UC_168h_130g",
"tejada19/UC_168h_131g",
"tejada19/UC_168h_199g",
]
formulations = Dict(
# "ArrCon00" => UnitCommitment.Formulation(
# ramping=UnitCommitment._ArrCon00(),
# ),
"DamKucRajAta16" => UnitCommitment.Formulation(
ramping = UnitCommitment.DamKucRajAta16(),
),
"MorLatRam13" => UnitCommitment.Formulation(
ramping = UnitCommitment.MorLatRam13(),
),
)
trials = [i for i in 1:5]
combinations = [
(c, f.first, f.second, t) for c in cases for f in formulations for
t in trials
]
shuffle!(combinations)
@sync @distributed for c in combinations
_run_combination(c...)
end
end
@everywhere function _run_combination(
case,
formulation_name,
formulation,
trial,
)
name = "$formulation_name/$case"
dirname = "results/$name"
mkpath(dirname)
if isfile("$dirname/$trial.json")
@info @sprintf(
"%-8s %-20s %-40s",
"skip",
formulation_name,
"$case/$trial",
)
return
end
@info @sprintf(
"%-8s %-20s %-40s",
"start",
formulation_name,
"$case/$trial",
)
time = @elapsed open("$dirname/$trial.log", "w") do file
redirect_stdout(file) do
redirect_stderr(file) do
return _run_sample(case, formulation, "$dirname/$trial")
end
end
end
@info @sprintf(
"%-8s %-20s %-40s %12.3f",
"finish",
formulation_name,
"$case/$trial",
time
)
end
@everywhere function _run_sample(case, formulation, prefix)
total_time = @elapsed begin
@info "Reading: $case"
time_read = @elapsed begin
instance = UnitCommitment.read_benchmark(case)
end
@info @sprintf("Read problem in %.2f seconds", time_read)
BLAS.set_num_threads(4)
model = UnitCommitment._build_model(
instance,
formulation,
optimizer = optimizer_with_attributes(
Gurobi.Optimizer,
"Threads" => 4,
"Seed" => rand(1:1000),
),
variable_names = true,
)
@info "Optimizing..."
BLAS.set_num_threads(1)
UnitCommitment.optimize!(
model,
UnitCommitment.XavQiuWanThi19(time_limit = 3600.0, gap_limit=1e-4),
)
end
@info @sprintf("Total time was %.2f seconds", total_time)
@info "Writing solution: $prefix.json"
solution = UnitCommitment.solution(model)
UnitCommitment.write("$prefix.json", solution)
@info "Verifying solution..."
return UnitCommitment.validate(instance, solution)
# @info "Exporting model..."
# return JuMP.write_to_file(model, model_filename)
end
if length(ARGS) > 0
_run_sample(ARGS[1], UnitCommitment.Formulation(), "tmp")
else
main()
end

54
benchmark/run.jl
Unescape View File

@ -1,54 +0,0 @@
# UnitCommitment.jl: Optimization Package for Security-Constrained Unit Commitment
# Copyright (C) 2020, UChicago Argonne, LLC. All rights reserved.
# Released under the modified BSD license. See COPYING.md for more details.
using UnitCommitment
using JuMP
using Gurobi
using JSON
using Logging
using Printf
using LinearAlgebra
UnitCommitment._setup_logger()
function main()
basename, suffix = split(ARGS[1], ".")
solution_filename = "results/$basename.$suffix.sol.json"
model_filename = "results/$basename.$suffix.mps.gz"
BLAS.set_num_threads(4)
total_time = @elapsed begin
@info "Reading: $basename"
time_read = @elapsed begin
instance = UnitCommitment.read_benchmark(basename)
end
@info @sprintf("Read problem in %.2f seconds", time_read)
model = UnitCommitment.build_model(
instance = instance,
optimizer = optimizer_with_attributes(
Gurobi.Optimizer,
"Threads" => 4,
"Seed" => rand(1:1000),
),
variable_names = true,
)
@info "Optimizing..."
BLAS.set_num_threads(1)
UnitCommitment.optimize!(
model,
UnitCommitment._XaQiWaTh19(time_limit = 3600.0),
)
end
@info @sprintf("Total time was %.2f seconds", total_time)
@info "Writing: $solution_filename"
solution = UnitCommitment.solution(model)
open(solution_filename, "w") do file
return JSON.print(file, solution, 2)
end
@info "Verifying solution..."
UnitCommitment.validate(instance, solution)
@info "Exporting model..."
return JuMP.write_to_file(model, model_filename)
end
main()

111
benchmark/scripts/compare.py
Unescape View File

@ -5,71 +5,84 @@
import pandas as pd import pandas as pd
import numpy as np import numpy as np
import seaborn as sns import seaborn as sns
import matplotlib
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import sys import sys
# easy_cutoff = 120 matplotlib.use("Agg")
sns.set("talk")
b1 = pd.read_csv(sys.argv[1], index_col=0) sns.set_palette(
b2 = pd.read_csv(sys.argv[2], index_col=0) [
"#9b59b6",
"#3498db",
"#95a5a6",
"#e74c3c",
"#34495e",
"#2ecc71",
]
)
c1 = b1.groupby(["Group", "Instance", "Sample"])[ filename = sys.argv[1]
["Optimization time (s)", "Primal bound"] m1 = sys.argv[2]
].mean() m2 = sys.argv[3]
c2 = b2.groupby(["Group", "Instance", "Sample"])[
["Optimization time (s)", "Primal bound"]
].mean()
c1.columns = ["A Time (s)", "A Value"]
c2.columns = ["B Time (s)", "B Value"]
merged = pd.concat([c1, c2], axis=1) # Prepare data
merged["Speedup"] = merged["A Time (s)"] / merged["B Time (s)"] data = pd.read_csv(filename, index_col=0)
merged["Time diff (s)"] = merged["B Time (s)"] - merged["A Time (s)"] b1 = (
merged["Value diff (%)"] = np.round( data[data["Group"] == m1]
(merged["B Value"] - merged["A Value"]) / merged["A Value"] * 100.0, 5 .groupby(["Instance", "Sample"])
.mean()[["Optimization time (s)"]]
) )
merged.loc[merged.loc[:, "B Time (s)"] <= 0, "Speedup"] = float("nan") b2 = (
merged.loc[merged.loc[:, "B Time (s)"] <= 0, "Time diff (s)"] = float("nan") data[data["Group"] == m2]
# merged = merged[(merged["A Time (s)"] >= easy_cutoff) | (merged["B Time (s)"] >= easy_cutoff)] .groupby(["Instance", "Sample"])
merged.reset_index(inplace=True) .mean()[["Optimization time (s)"]]
merged["Name"] = merged["Group"] + "/" + merged["Instance"] )
# merged = merged.sort_values(by="Speedup", ascending=False) b1.columns = [f"{m1} time (s)"]
b2.columns = [f"{m2} time (s)"]
merged = pd.merge(b1, b2, left_index=True, right_index=True).reset_index().dropna()
merged["Speedup"] = merged[f"{m1} time (s)"] / merged[f"{m2} time (s)"]
merged["Group"] = merged["Instance"].str.replace(r"\/.*", "", regex=True)
merged = merged.sort_values(by=["Instance", "Sample"], ascending=True)
merged = merged[(merged[f"{m1} time (s)"] > 0) & (merged[f"{m2} time (s)"] > 0)]
k = len(merged.groupby("Name")) # Plot results
plt.figure(figsize=(12, 0.50 * k)) k1 = len(merged.groupby("Instance").mean())
plt.rcParams["xtick.bottom"] = plt.rcParams["xtick.labelbottom"] = True k2 = len(merged.groupby("Group").mean())
plt.rcParams["xtick.top"] = plt.rcParams["xtick.labeltop"] = True k = k1 + k2
sns.set_style("whitegrid") fig = plt.figure(
sns.set_palette("Set1") constrained_layout=True,
figsize=(15, max(5, 0.75 * k)),
)
plt.suptitle(f"{m1} vs {m2}")
gs1 = fig.add_gridspec(nrows=k, ncols=1)
ax1 = fig.add_subplot(gs1[0:k1, 0:1])
ax2 = fig.add_subplot(gs1[k1:, 0:1], sharex=ax1)
sns.barplot(
data=merged,
x="Speedup",
y="Instance",
color="tab:purple",
capsize=0.15,
errcolor="k",
errwidth=1.25,
ax=ax1,
)
sns.barplot( sns.barplot(
data=merged, data=merged,
x="Speedup", x="Speedup",
y="Name", y="Group",
color="tab:red", color="tab:purple",
capsize=0.15, capsize=0.15,
errcolor="k", errcolor="k",
errwidth=1.25, errwidth=1.25,
ax=ax2,
) )
plt.axvline(1.0, linestyle="--", color="k") ax1.axvline(1.0, linestyle="--", color="k")
plt.tight_layout() ax2.axvline(1.0, linestyle="--", color="k")
print("Writing tables/compare.png") print("Writing tables/compare.png")
plt.savefig("tables/compare.png", dpi=150) plt.savefig("tables/compare.png", dpi=150)
print("Writing tables/compare.csv") print("Writing tables/compare.csv")
merged.loc[ merged.to_csv("tables/compare.csv", index_label="Index")
:,
[
"Group",
"Instance",
"Sample",
"A Time (s)",
"B Time (s)",
"Speedup",
"Time diff (s)",
"A Value",
"B Value",
"Value diff (%)",
],
].to_csv("tables/compare.csv", index_label="Index")

32
benchmark/scripts/table.py
Unescape View File

@ -9,8 +9,7 @@ from tabulate import tabulate
def process_all_log_files(): def process_all_log_files():
pathlist = list(Path(".").glob("results/*/*/*.log")) pathlist = list(Path(".").glob("results/**/*.log"))
pathlist += list(Path(".").glob("results/*/*.log"))
rows = [] rows = []
for path in pathlist: for path in pathlist:
if ".ipy" in str(path): if ".ipy" in str(path):
@ -26,9 +25,9 @@ def process_all_log_files():
def process(filename): def process(filename):
parts = filename.replace(".log", "").split("/") parts = filename.replace(".log", "").split("/")
group_name = "/".join(parts[1:-1]) group_name = parts[1]
instance_name = parts[-1] instance_name = "/".join(parts[2:-1])
instance_name, sample_name = instance_name.split(".") sample_name = parts[-1]
nodes = 0.0 nodes = 0.0
optimize_time = 0.0 optimize_time = 0.0
simplex_iterations = 0.0 simplex_iterations = 0.0
@ -174,28 +173,37 @@ def process(filename):
def generate_chart(): def generate_chart():
import pandas as pd import pandas as pd
import matplotlib
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import seaborn as sns import seaborn as sns
matplotlib.use("Agg")
sns.set("talk")
sns.set_palette(
[
"#9b59b6",
"#3498db",
"#95a5a6",
"#e74c3c",
"#34495e",
"#2ecc71",
]
)
tables = [] tables = []
files = ["tables/benchmark.csv"] files = ["tables/benchmark.csv"]
for f in files: for f in files:
table = pd.read_csv(f, index_col=0) table = pd.read_csv(f, index_col=0)
table.loc[:, "Instance"] = (
table.loc[:, "Group"] + "/" + table.loc[:, "Instance"]
)
table.loc[:, "Filename"] = f table.loc[:, "Filename"] = f
tables += [table] tables += [table]
benchmark = pd.concat(tables, sort=True) benchmark = pd.concat(tables, sort=True)
benchmark = benchmark.sort_values(by="Instance") benchmark = benchmark.sort_values(by="Instance")
k = len(benchmark.groupby("Instance")) k = len(benchmark.groupby("Instance"))
plt.figure(figsize=(12, 0.50 * k)) plt.figure(figsize=(12, k))
sns.set_style("whitegrid")
sns.set_palette("Set1")
sns.barplot( sns.barplot(
y="Instance", y="Instance",
x="Total time (s)", x="Total time (s)",
color="tab:red", hue="Group",
capsize=0.15, capsize=0.15,
errcolor="k", errcolor="k",
errwidth=1.25, errwidth=1.25,

Write Preview
Loading…
Cancel
Save