From f235333551b6b1d3b15a1802d3a57a4ab818136b Mon Sep 17 00:00:00 2001 From: Alinson S Xavier Date: Tue, 31 Aug 2021 07:56:25 -0500 Subject: [PATCH] Improve benchmark scripts --- Project.toml | 1 + benchmark/Project.toml | 1 + benchmark/benchmark.jl | 158 ------------------------------- benchmark/run.jl | 207 +++++++++++++++++++++++++++++++++++++++++ src/UnitCommitment.jl | 1 + src/utils/benchmark.jl | 116 +++++++++++++++++++++++ 6 files changed, 326 insertions(+), 158 deletions(-) delete mode 100644 benchmark/benchmark.jl create mode 100644 benchmark/run.jl create mode 100644 src/utils/benchmark.jl diff --git a/Project.toml b/Project.toml index f288cd8..cd9584f 100644 --- a/Project.toml +++ b/Project.toml @@ -6,6 +6,7 @@ version = "0.2.2" [deps] DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" GZip = "92fee26a-97fe-5a0c-ad85-20a5f3185b63" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" diff --git a/benchmark/Project.toml b/benchmark/Project.toml index 454b1b1..7f595fd 100644 --- a/benchmark/Project.toml +++ b/benchmark/Project.toml @@ -1,4 +1,5 @@ [deps] +DocOpt = "968ba79b-81e4-546f-ab3a-2eecfa62a9db" Gurobi = "2e9cd046-0924-5485-92f1-d5272153d98b" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" JuMP = "4076af6c-e467-56ae-b986-b466b2749572" diff --git a/benchmark/benchmark.jl b/benchmark/benchmark.jl deleted file mode 100644 index 212a55b..0000000 --- a/benchmark/benchmark.jl +++ /dev/null @@ -1,158 +0,0 @@ -# UnitCommitment.jl: Optimization Package for Security-Constrained Unit Commitment -# Copyright (C) 2020, UChicago Argonne, LLC. All rights reserved. -# Released under the modified BSD license. See COPYING.md for more details. - -using Distributed -using Pkg -Pkg.activate(".") - -@everywhere using Pkg -@everywhere Pkg.activate(".") - -@everywhere using UnitCommitment -@everywhere using JuMP -@everywhere using Gurobi -@everywhere using JSON -@everywhere using Logging -@everywhere using Printf -@everywhere using LinearAlgebra -@everywhere using Random - -@everywhere import UnitCommitment: - ArrCon2000, - CarArr2006, - DamKucRajAta2016, - Formulation, - Gar1962, - KnuOstWat2018, - MorLatRam2013, - PanGua2016, - XavQiuWanThi2019 - -@everywhere UnitCommitment._setup_logger() - -function main() - cases = [ - "pglib-uc/ca/2014-09-01_reserves_0", - "pglib-uc/ca/2014-09-01_reserves_1", - "pglib-uc/ca/2015-03-01_reserves_0", - "pglib-uc/ca/2015-06-01_reserves_0", - "pglib-uc/ca/Scenario400_reserves_1", - "pglib-uc/ferc/2015-01-01_lw", - "pglib-uc/ferc/2015-05-01_lw", - "pglib-uc/ferc/2015-07-01_hw", - "pglib-uc/ferc/2015-10-01_lw", - "pglib-uc/ferc/2015-12-01_lw", - "pglib-uc/rts_gmlc/2020-04-03", - "pglib-uc/rts_gmlc/2020-09-20", - "pglib-uc/rts_gmlc/2020-10-27", - "pglib-uc/rts_gmlc/2020-11-25", - "pglib-uc/rts_gmlc/2020-12-23", - "or-lib/20_0_1_w", - "or-lib/20_0_5_w", - "or-lib/50_0_2_w", - "or-lib/75_0_2_w", - "or-lib/100_0_1_w", - "or-lib/100_0_4_w", - "or-lib/100_0_5_w", - "or-lib/200_0_3_w", - "or-lib/200_0_7_w", - "or-lib/200_0_9_w", - "tejada19/UC_24h_290g", - "tejada19/UC_24h_623g", - "tejada19/UC_24h_959g", - "tejada19/UC_24h_1577g", - "tejada19/UC_24h_1888g", - "tejada19/UC_168h_72g", - "tejada19/UC_168h_86g", - "tejada19/UC_168h_130g", - "tejada19/UC_168h_131g", - "tejada19/UC_168h_199g", - ] - formulations = Dict( - "Default" => Formulation(), - "ArrCon2000" => Formulation(ramping = ArrCon2000.Ramping()), - "CarArr2006" => Formulation(pwl_costs = CarArr2006.PwlCosts()), - "DamKucRajAta2016" => - Formulation(ramping = DamKucRajAta2016.Ramping()), - "Gar1962" => Formulation(pwl_costs = Gar1962.PwlCosts()), - "KnuOstWat2018" => - Formulation(pwl_costs = KnuOstWat2018.PwlCosts()), - "MorLatRam2013" => Formulation(ramping = MorLatRam2013.Ramping()), - "PanGua2016" => Formulation(ramping = PanGua2016.Ramping()), - ) - trials = [i for i in 1:5] - combinations = [ - (c, f.first, f.second, t) for c in cases for f in formulations for - t in trials - ] - shuffle!(combinations) - @sync @distributed for c in combinations - _run_combination(c...) - end -end - -@everywhere function _run_combination( - case, - formulation_name, - formulation, - trial, -) - name = "$formulation_name/$case" - dirname = "results/$name" - mkpath(dirname) - if isfile("$dirname/$trial.json") - @info @sprintf("%-4s %-16s %s", "skip", formulation_name, case) - return - end - @info @sprintf("%-4s %-16s %s", "run", formulation_name, case) - open("$dirname/$trial.log", "w") do file - redirect_stdout(file) do - redirect_stderr(file) do - return _run_sample(case, formulation, "$dirname/$trial") - end - end - end - @info @sprintf("%-4s %-16s %s", "done", formulation_name, case) -end - -@everywhere function _run_sample(case, formulation, prefix) - total_time = @elapsed begin - @info "Reading: $case" - time_read = @elapsed begin - instance = UnitCommitment.read_benchmark(case) - end - @info @sprintf("Read problem in %.2f seconds", time_read) - BLAS.set_num_threads(4) - model = UnitCommitment.build_model( - instance = instance, - formulation = formulation, - optimizer = optimizer_with_attributes( - Gurobi.Optimizer, - "Threads" => 4, - "Seed" => rand(1:1000), - ), - variable_names = true, - ) - @info "Optimizing..." - BLAS.set_num_threads(1) - UnitCommitment.optimize!( - model, - XavQiuWanThi2019.Method(time_limit = 3600.0, gap_limit = 1e-4), - ) - end - @info @sprintf("Total time was %.2f seconds", total_time) - @info "Writing solution: $prefix.json" - solution = UnitCommitment.solution(model) - UnitCommitment.write("$prefix.json", solution) - @info "Verifying solution..." - return UnitCommitment.validate(instance, solution) - # @info "Exporting model..." - # return JuMP.write_to_file(model, model_filename) -end - -if length(ARGS) > 0 - _run_sample(ARGS[1], UnitCommitment.Formulation(), "tmp") -else - main() -end diff --git a/benchmark/run.jl b/benchmark/run.jl new file mode 100644 index 0000000..7c1e2a4 --- /dev/null +++ b/benchmark/run.jl @@ -0,0 +1,207 @@ +# UnitCommitment.jl: Optimization Package for Security-Constrained Unit Commitment +# Copyright (C) 2020, UChicago Argonne, LLC. All rights reserved. +# Released under the modified BSD license. See COPYING.md for more details. + +doc = """UnitCommitment.jl Benchmark Runner + +Usage: + run.jl [-s ARG]... [-m ARG]... [-c ARG]... [-f ARG]... [options] + +Examples: + + 1. Benchmark all solvers, methods and formulations: + + julia run.jl + + 2. Benchmark formulations "default" and "ArrCon200" using Gurobi: + + julia run.jl -s gurobi -f default -f ArrCon2000 + + 3. Benchmark a few test cases, using all solvers, methods and formulations: + + julia run.jl -c or-lib/20_0_1_w -c matpower/case1888rte/2017-02-01 + + 4. Solve 4 test cases in parallel, with 2 threads available per worker: + + JULIA_NUM_THREADS=2 julia --procs 4 run.jl + +Options: + -h --help Show this screen. + -s --solver=ARG Mixed-integer linear solver (e.g. gurobi) + -c --case=ARG Unit commitment test case (e.g. or-lib/20_0_1_w) + -m --method=ARG Solution method (e.g. default) + -f --formulation=ARG Formulation (e.g. ArrCon2000) + --time-limit=ARG Time limit in seconds [default: 3600] + --gap=ARG Relative MIP gap tolerance [default: 0.001] + --trials=ARG Number of trials [default: 5] +""" + +using Distributed +using Pkg +Pkg.activate(".") +@everywhere using Pkg +@everywhere Pkg.activate(".") + +using DocOpt +args = docopt(doc) + +@everywhere using UnitCommitment +@everywhere UnitCommitment._setup_logger() + +using UnitCommitment +using Gurobi +using Logging +using JuMP + +import UnitCommitment: + ArrCon2000, + CarArr2006, + DamKucRajAta2016, + Formulation, + Gar1962, + KnuOstWat2018, + MorLatRam2013, + PanGua2016, + XavQiuWanThi2019 + +# Benchmark test cases +# ----------------------------------------------------------------------------- +cases = [ + "pglib-uc/ca/2014-09-01_reserves_0", + "pglib-uc/ca/2014-09-01_reserves_1", + "pglib-uc/ca/2015-03-01_reserves_0", + "pglib-uc/ca/2015-06-01_reserves_0", + "pglib-uc/ca/Scenario400_reserves_1", + "pglib-uc/ferc/2015-01-01_lw", + "pglib-uc/ferc/2015-05-01_lw", + "pglib-uc/ferc/2015-07-01_hw", + "pglib-uc/ferc/2015-10-01_lw", + "pglib-uc/ferc/2015-12-01_lw", + "pglib-uc/rts_gmlc/2020-04-03", + "pglib-uc/rts_gmlc/2020-09-20", + "pglib-uc/rts_gmlc/2020-10-27", + "pglib-uc/rts_gmlc/2020-11-25", + "pglib-uc/rts_gmlc/2020-12-23", + "or-lib/20_0_1_w", + "or-lib/20_0_5_w", + "or-lib/50_0_2_w", + "or-lib/75_0_2_w", + "or-lib/100_0_1_w", + "or-lib/100_0_4_w", + "or-lib/100_0_5_w", + "or-lib/200_0_3_w", + "or-lib/200_0_7_w", + "or-lib/200_0_9_w", + "tejada19/UC_24h_290g", + "tejada19/UC_24h_623g", + "tejada19/UC_24h_959g", + "tejada19/UC_24h_1577g", + "tejada19/UC_24h_1888g", + "tejada19/UC_168h_72g", + "tejada19/UC_168h_86g", + "tejada19/UC_168h_130g", + "tejada19/UC_168h_131g", + "tejada19/UC_168h_199g", + "matpower/case1888rte/2017-02-01", + "matpower/case1951rte/2017-02-01", + "matpower/case2848rte/2017-02-01", + "matpower/case3012wp/2017-02-01", + "matpower/case3375wp/2017-02-01", + "matpower/case6468rte/2017-02-01", + "matpower/case6515rte/2017-02-01", +] + +# Formulations +# ----------------------------------------------------------------------------- +formulations = Dict( + "default" => Formulation(), + "ArrCon2000" => Formulation(ramping = ArrCon2000.Ramping()), + "CarArr2006" => Formulation(pwl_costs = CarArr2006.PwlCosts()), + "DamKucRajAta2016" => Formulation(ramping = DamKucRajAta2016.Ramping()), + "Gar1962" => Formulation(pwl_costs = Gar1962.PwlCosts()), + "KnuOstWat2018" => Formulation(pwl_costs = KnuOstWat2018.PwlCosts()), + "MorLatRam2013" => Formulation(ramping = MorLatRam2013.Ramping()), + "PanGua2016" => Formulation(ramping = PanGua2016.Ramping()), +) + +# Solution methods +# ----------------------------------------------------------------------------- +methods = Dict( + "default" => + XavQiuWanThi2019.Method(time_limit = 3600.0, gap_limit = 1e-4), +) + +# MIP solvers +# ----------------------------------------------------------------------------- +optimizers = Dict( + "gurobi" => optimizer_with_attributes( + Gurobi.Optimizer, + "Threads" => Threads.nthreads(), + ), +) + +# Parse command line arguments +# ----------------------------------------------------------------------------- +if !isempty(args["--case"]) + cases = args["--case"] +end +if !isempty(args["--formulation"]) + formulations = filter(p -> p.first in args["--formulation"], formulations) +end +if !isempty(args["--method"]) + methods = filter(p -> p.first in args["--method"], methods) +end +if !isempty(args["--solver"]) + optimizers = filter(p -> p.first in args["--solver"], optimizers) +end +const time_limit = parse(Float64, args["--time-limit"]) +const gap_limit = parse(Float64, args["--gap"]) +const ntrials = parse(Int, args["--trials"]) + +# Print benchmark settings +# ----------------------------------------------------------------------------- +function printlist(d::Dict) + for key in keys(d) + @info " - $key" + end +end + +function printlist(d::Vector) + for key in d + @info " - $key" + end +end + +@info "Computational environment:" +@info " - CPU: $(Sys.cpu_info()[1].model)" +@info " - Logical CPU cores: $(length(Sys.cpu_info()))" +@info " - System memory: $(round(Sys.total_memory() / 2^30, digits=2)) GiB" +@info " - Available workers: $(nworkers())" +@info " - Available threads per worker: $(Threads.nthreads())" + +@info "Parameters:" +@info " - Number of trials: $ntrials" +@info " - Time limit (s): $time_limit" +@info " - Relative MIP gap tolerance: $gap_limit" + +@info "Solvers:" +printlist(optimizers) + +@info "Methods:" +printlist(methods) + +@info "Formulations:" +printlist(formulations) + +@info "Cases:" +printlist(cases) + +# Run benchmarks +# ----------------------------------------------------------------------------- +UnitCommitment._run_benchmarks( + cases = cases, + formulations = formulations, + methods = methods, + optimizers = optimizers, + trials = 1:ntrials, +) diff --git a/src/UnitCommitment.jl b/src/UnitCommitment.jl index 6eb5638..9ada14c 100644 --- a/src/UnitCommitment.jl +++ b/src/UnitCommitment.jl @@ -50,6 +50,7 @@ include("transform/initcond.jl") include("transform/slice.jl") include("transform/randomize/XavQiuAhm2021.jl") include("utils/log.jl") +include("utils/benchmark.jl") include("validation/repair.jl") include("validation/validate.jl") diff --git a/src/utils/benchmark.jl b/src/utils/benchmark.jl new file mode 100644 index 0000000..ba26f1d --- /dev/null +++ b/src/utils/benchmark.jl @@ -0,0 +1,116 @@ +# UnitCommitment.jl: Optimization Package for Security-Constrained Unit Commitment +# Copyright (C) 2020, UChicago Argonne, LLC. All rights reserved. +# Released under the modified BSD license. See COPYING.md for more details. + +using Distributed +using Random + +function _run_benchmark_sample(; + case::String, + method::SolutionMethod, + formulation::Formulation, + solution_filename::String, + optimizer, +)::Nothing + total_time = @elapsed begin + @info "Reading: $case" + time_read = @elapsed begin + instance = read_benchmark(case) + end + @info @sprintf("Read problem in %.2f seconds", time_read) + BLAS.set_num_threads(Threads.nthreads()) + model = build_model( + instance = instance, + formulation = formulation, + optimizer = optimizer, + variable_names = true, + ) + @info "Optimizing..." + BLAS.set_num_threads(1) + optimize!(model, method) + end + @info @sprintf("Total time was %.2f seconds", total_time) + + @info "Writing solution: $solution_filename" + solution = UnitCommitment.solution(model) + write("$solution_filename", solution) + + @info "Verifying solution..." + validate(instance, solution) + return +end + +function _run_benchmark_combination( + case::String, + optimizer_name::String, + optimizer, + method_name::String, + method::SolutionMethod, + formulation_name::String, + formulation::Formulation, + trial, +) + dirname = "results/$optimizer_name/$method_name/$formulation_name/$case" + function info(msg) + @info @sprintf( + "%-8s %-16s %-16s %-16s %-8s %s", + msg, + optimizer_name, + method_name, + formulation_name, + trial, + case + ) + end + mkpath(dirname) + trial_filename = @sprintf("%s/%03d.json", dirname, trial) + if isfile(trial_filename) + info("skip") + return + end + info("run") + open("$trial_filename.log", "w") do file + redirect_stdout(file) do + redirect_stderr(file) do + return _run_benchmark_sample( + case = case, + method = method, + formulation = formulation, + solution_filename = trial_filename, + optimizer = optimizer, + ) + end + end + end + return info("done") +end + +function _run_benchmarks(; + cases::Vector{String}, + optimizers::Dict, + formulations::Dict, + methods::Dict, + trials, +) + combinations = [ + (c, s.first, s.second, m.first, m.second, f.first, f.second, t) for + c in cases for s in optimizers for f in formulations for + m in methods for t in trials + ] + shuffle!(combinations) + if nworkers() > 1 + @printf("%24s", "") + end + @info @sprintf( + "%-8s %-16s %-16s %-16s %-8s %s", + "STATUS", + "SOLVER", + "METHOD", + "FORMULATION", + "TRIAL", + "CASE" + ) + @sync @distributed for c in combinations + _run_benchmark_combination(c...) + end +end