Improve benchmark scripts

Project.toml

@@ -6,6 +6,7 @@ version = "0.2.2"
 
 [deps]
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
+Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 GZip = "92fee26a-97fe-5a0c-ad85-20a5f3185b63"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"

benchmark/Project.toml

@@ -1,4 +1,5 @@
 [deps]
+DocOpt = "968ba79b-81e4-546f-ab3a-2eecfa62a9db"
 Gurobi = "2e9cd046-0924-5485-92f1-d5272153d98b"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 JuMP = "4076af6c-e467-56ae-b986-b466b2749572"

benchmark/benchmark.jl

@@ -1,158 +0,0 @@
-# UnitCommitment.jl: Optimization Package for Security-Constrained Unit Commitment
-# Copyright (C) 2020, UChicago Argonne, LLC. All rights reserved.
-# Released under the modified BSD license. See COPYING.md for more details.
-
-using Distributed
-using Pkg
-Pkg.activate(".")
-
-@everywhere using Pkg
-@everywhere Pkg.activate(".")
-
-@everywhere using UnitCommitment
-@everywhere using JuMP
-@everywhere using Gurobi
-@everywhere using JSON
-@everywhere using Logging
-@everywhere using Printf
-@everywhere using LinearAlgebra
-@everywhere using Random
-
-@everywhere import UnitCommitment:
-    ArrCon2000,
-    CarArr2006,
-    DamKucRajAta2016,
-    Formulation,
-    Gar1962,
-    KnuOstWat2018,
-    MorLatRam2013,
-    PanGua2016,
-    XavQiuWanThi2019
-
-@everywhere UnitCommitment._setup_logger()
-
-function main()
-    cases = [
-        "pglib-uc/ca/2014-09-01_reserves_0",
-        "pglib-uc/ca/2014-09-01_reserves_1",
-        "pglib-uc/ca/2015-03-01_reserves_0",
-        "pglib-uc/ca/2015-06-01_reserves_0",
-        "pglib-uc/ca/Scenario400_reserves_1",
-        "pglib-uc/ferc/2015-01-01_lw",
-        "pglib-uc/ferc/2015-05-01_lw",
-        "pglib-uc/ferc/2015-07-01_hw",
-        "pglib-uc/ferc/2015-10-01_lw",
-        "pglib-uc/ferc/2015-12-01_lw",
-        "pglib-uc/rts_gmlc/2020-04-03",
-        "pglib-uc/rts_gmlc/2020-09-20",
-        "pglib-uc/rts_gmlc/2020-10-27",
-        "pglib-uc/rts_gmlc/2020-11-25",
-        "pglib-uc/rts_gmlc/2020-12-23",
-        "or-lib/20_0_1_w",
-        "or-lib/20_0_5_w",
-        "or-lib/50_0_2_w",
-        "or-lib/75_0_2_w",
-        "or-lib/100_0_1_w",
-        "or-lib/100_0_4_w",
-        "or-lib/100_0_5_w",
-        "or-lib/200_0_3_w",
-        "or-lib/200_0_7_w",
-        "or-lib/200_0_9_w",
-        "tejada19/UC_24h_290g",
-        "tejada19/UC_24h_623g",
-        "tejada19/UC_24h_959g",
-        "tejada19/UC_24h_1577g",
-        "tejada19/UC_24h_1888g",
-        "tejada19/UC_168h_72g",
-        "tejada19/UC_168h_86g",
-        "tejada19/UC_168h_130g",
-        "tejada19/UC_168h_131g",
-        "tejada19/UC_168h_199g",
-    ]
-    formulations = Dict(
-        "Default" => Formulation(),
-        "ArrCon2000" => Formulation(ramping = ArrCon2000.Ramping()),
-        "CarArr2006" => Formulation(pwl_costs = CarArr2006.PwlCosts()),
-        "DamKucRajAta2016" =>
-            Formulation(ramping = DamKucRajAta2016.Ramping()),
-        "Gar1962" => Formulation(pwl_costs = Gar1962.PwlCosts()),
-        "KnuOstWat2018" =>
-            Formulation(pwl_costs = KnuOstWat2018.PwlCosts()),
-        "MorLatRam2013" => Formulation(ramping = MorLatRam2013.Ramping()),
-        "PanGua2016" => Formulation(ramping = PanGua2016.Ramping()),
-    )
-    trials = [i for i in 1:5]
-    combinations = [
-        (c, f.first, f.second, t) for c in cases for f in formulations for
-        t in trials
-    ]
-    shuffle!(combinations)
-    @sync @distributed for c in combinations
-        _run_combination(c...)
-    end
-end
-
-@everywhere function _run_combination(
-    case,
-    formulation_name,
-    formulation,
-    trial,
-)
-    name = "$formulation_name/$case"
-    dirname = "results/$name"
-    mkpath(dirname)
-    if isfile("$dirname/$trial.json")
-        @info @sprintf("%-4s %-16s %s", "skip", formulation_name, case)
-        return
-    end
-    @info @sprintf("%-4s %-16s %s", "run", formulation_name, case)
-    open("$dirname/$trial.log", "w") do file
-        redirect_stdout(file) do
-            redirect_stderr(file) do
-                return _run_sample(case, formulation, "$dirname/$trial")
-            end
-        end
-    end
-    @info @sprintf("%-4s %-16s %s", "done", formulation_name, case)
-end
-
-@everywhere function _run_sample(case, formulation, prefix)
-    total_time = @elapsed begin
-        @info "Reading: $case"
-        time_read = @elapsed begin
-            instance = UnitCommitment.read_benchmark(case)
-        end
-        @info @sprintf("Read problem in %.2f seconds", time_read)
-        BLAS.set_num_threads(4)
-        model = UnitCommitment.build_model(
-            instance = instance,
-            formulation = formulation,
-            optimizer = optimizer_with_attributes(
-                Gurobi.Optimizer,
-                "Threads" => 4,
-                "Seed" => rand(1:1000),
-            ),
-            variable_names = true,
-        )
-        @info "Optimizing..."
-        BLAS.set_num_threads(1)
-        UnitCommitment.optimize!(
-            model,
-            XavQiuWanThi2019.Method(time_limit = 3600.0, gap_limit = 1e-4),
-        )
-    end
-    @info @sprintf("Total time was %.2f seconds", total_time)
-    @info "Writing solution: $prefix.json"
-    solution = UnitCommitment.solution(model)
-    UnitCommitment.write("$prefix.json", solution)
-    @info "Verifying solution..."
-    return UnitCommitment.validate(instance, solution)
-    # @info "Exporting model..."
-    # return JuMP.write_to_file(model, model_filename)
-end
-
-if length(ARGS) > 0
-    _run_sample(ARGS[1], UnitCommitment.Formulation(), "tmp")
-else
-    main()
-end

benchmark/run.jl

@@ -0,0 +1,207 @@
+# UnitCommitment.jl: Optimization Package for Security-Constrained Unit Commitment
+# Copyright (C) 2020, UChicago Argonne, LLC. All rights reserved.
+# Released under the modified BSD license. See COPYING.md for more details.
+
+doc = """UnitCommitment.jl Benchmark Runner
+
+Usage:
+  run.jl [-s ARG]... [-m ARG]... [-c ARG]... [-f ARG]... [options]
+
+Examples:
+
+    1. Benchmark all solvers, methods and formulations:
+
+        julia run.jl
+
+    2. Benchmark formulations "default" and "ArrCon200" using Gurobi:
+
+        julia run.jl -s gurobi -f default -f ArrCon2000
+
+    3. Benchmark a few test cases, using all solvers, methods and formulations:
+
+        julia run.jl -c or-lib/20_0_1_w -c matpower/case1888rte/2017-02-01
+
+    4. Solve 4 test cases in parallel, with 2 threads available per worker:
+
+        JULIA_NUM_THREADS=2 julia --procs 4 run.jl
+
+Options:
+  -h --help             Show this screen.
+  -s --solver=ARG       Mixed-integer linear solver (e.g. gurobi)
+  -c --case=ARG         Unit commitment test case (e.g. or-lib/20_0_1_w)
+  -m --method=ARG       Solution method (e.g. default)
+  -f --formulation=ARG  Formulation (e.g. ArrCon2000)
+  --time-limit=ARG      Time limit in seconds [default: 3600]
+  --gap=ARG             Relative MIP gap tolerance [default: 0.001]
+  --trials=ARG          Number of trials [default: 5]
+"""
+
+using Distributed
+using Pkg
+Pkg.activate(".")
+@everywhere using Pkg
+@everywhere Pkg.activate(".")
+
+using DocOpt
+args = docopt(doc)
+
+@everywhere using UnitCommitment
+@everywhere UnitCommitment._setup_logger()
+
+using UnitCommitment
+using Gurobi
+using Logging
+using JuMP
+
+import UnitCommitment:
+    ArrCon2000,
+    CarArr2006,
+    DamKucRajAta2016,
+    Formulation,
+    Gar1962,
+    KnuOstWat2018,
+    MorLatRam2013,
+    PanGua2016,
+    XavQiuWanThi2019
+
+# Benchmark test cases
+# -----------------------------------------------------------------------------
+cases = [
+    "pglib-uc/ca/2014-09-01_reserves_0",
+    "pglib-uc/ca/2014-09-01_reserves_1",
+    "pglib-uc/ca/2015-03-01_reserves_0",
+    "pglib-uc/ca/2015-06-01_reserves_0",
+    "pglib-uc/ca/Scenario400_reserves_1",
+    "pglib-uc/ferc/2015-01-01_lw",
+    "pglib-uc/ferc/2015-05-01_lw",
+    "pglib-uc/ferc/2015-07-01_hw",
+    "pglib-uc/ferc/2015-10-01_lw",
+    "pglib-uc/ferc/2015-12-01_lw",
+    "pglib-uc/rts_gmlc/2020-04-03",
+    "pglib-uc/rts_gmlc/2020-09-20",
+    "pglib-uc/rts_gmlc/2020-10-27",
+    "pglib-uc/rts_gmlc/2020-11-25",
+    "pglib-uc/rts_gmlc/2020-12-23",
+    "or-lib/20_0_1_w",
+    "or-lib/20_0_5_w",
+    "or-lib/50_0_2_w",
+    "or-lib/75_0_2_w",
+    "or-lib/100_0_1_w",
+    "or-lib/100_0_4_w",
+    "or-lib/100_0_5_w",
+    "or-lib/200_0_3_w",
+    "or-lib/200_0_7_w",
+    "or-lib/200_0_9_w",
+    "tejada19/UC_24h_290g",
+    "tejada19/UC_24h_623g",
+    "tejada19/UC_24h_959g",
+    "tejada19/UC_24h_1577g",
+    "tejada19/UC_24h_1888g",
+    "tejada19/UC_168h_72g",
+    "tejada19/UC_168h_86g",
+    "tejada19/UC_168h_130g",
+    "tejada19/UC_168h_131g",
+    "tejada19/UC_168h_199g",
+    "matpower/case1888rte/2017-02-01",
+    "matpower/case1951rte/2017-02-01",
+    "matpower/case2848rte/2017-02-01",
+    "matpower/case3012wp/2017-02-01",
+    "matpower/case3375wp/2017-02-01",
+    "matpower/case6468rte/2017-02-01",
+    "matpower/case6515rte/2017-02-01",
+]
+
+# Formulations
+# -----------------------------------------------------------------------------
+formulations = Dict(
+    "default" => Formulation(),
+    "ArrCon2000" => Formulation(ramping = ArrCon2000.Ramping()),
+    "CarArr2006" => Formulation(pwl_costs = CarArr2006.PwlCosts()),
+    "DamKucRajAta2016" => Formulation(ramping = DamKucRajAta2016.Ramping()),
+    "Gar1962" => Formulation(pwl_costs = Gar1962.PwlCosts()),
+    "KnuOstWat2018" => Formulation(pwl_costs = KnuOstWat2018.PwlCosts()),
+    "MorLatRam2013" => Formulation(ramping = MorLatRam2013.Ramping()),
+    "PanGua2016" => Formulation(ramping = PanGua2016.Ramping()),
+)
+
+# Solution methods
+# -----------------------------------------------------------------------------
+methods = Dict(
+    "default" =>
+        XavQiuWanThi2019.Method(time_limit = 3600.0, gap_limit = 1e-4),
+)
+
+# MIP solvers
+# -----------------------------------------------------------------------------
+optimizers = Dict(
+    "gurobi" => optimizer_with_attributes(
+        Gurobi.Optimizer,
+        "Threads" => Threads.nthreads(),
+    ),
+)
+
+# Parse command line arguments
+# -----------------------------------------------------------------------------
+if !isempty(args["--case"])
+    cases = args["--case"]
+end
+if !isempty(args["--formulation"])
+    formulations = filter(p -> p.first in args["--formulation"], formulations)
+end
+if !isempty(args["--method"])
+    methods = filter(p -> p.first in args["--method"], methods)
+end
+if !isempty(args["--solver"])
+    optimizers = filter(p -> p.first in args["--solver"], optimizers)
+end
+const time_limit = parse(Float64, args["--time-limit"])
+const gap_limit = parse(Float64, args["--gap"])
+const ntrials = parse(Int, args["--trials"])
+
+# Print benchmark settings
+# -----------------------------------------------------------------------------
+function printlist(d::Dict)
+    for key in keys(d)
+        @info "  - $key"
+    end
+end
+
+function printlist(d::Vector)
+    for key in d
+        @info "  - $key"
+    end
+end
+
+@info "Computational environment:"
+@info "  - CPU: $(Sys.cpu_info()[1].model)"
+@info "  - Logical CPU cores: $(length(Sys.cpu_info()))"
+@info "  - System memory: $(round(Sys.total_memory() / 2^30, digits=2)) GiB"
+@info "  - Available workers: $(nworkers())"
+@info "  - Available threads per worker: $(Threads.nthreads())"
+
+@info "Parameters:"
+@info "  - Number of trials: $ntrials"
+@info "  - Time limit (s): $time_limit"
+@info "  - Relative MIP gap tolerance: $gap_limit"
+
+@info "Solvers:"
+printlist(optimizers)
+
+@info "Methods:"
+printlist(methods)
+
+@info "Formulations:"
+printlist(formulations)
+
+@info "Cases:"
+printlist(cases)
+
+# Run benchmarks
+# -----------------------------------------------------------------------------
+UnitCommitment._run_benchmarks(
+    cases = cases,
+    formulations = formulations,
+    methods = methods,
+    optimizers = optimizers,
+    trials = 1:ntrials,
+)

src/UnitCommitment.jl

@@ -50,6 +50,7 @@ include("transform/initcond.jl")
 include("transform/slice.jl")
 include("transform/randomize/XavQiuAhm2021.jl")
 include("utils/log.jl")
+include("utils/benchmark.jl")
 include("validation/repair.jl")
 include("validation/validate.jl")
 

src/utils/benchmark.jl

@@ -0,0 +1,116 @@
+# UnitCommitment.jl: Optimization Package for Security-Constrained Unit Commitment
+# Copyright (C) 2020, UChicago Argonne, LLC. All rights reserved.
+# Released under the modified BSD license. See COPYING.md for more details.
+
+using Distributed
+using Random
+
+function _run_benchmark_sample(;
+    case::String,
+    method::SolutionMethod,
+    formulation::Formulation,
+    solution_filename::String,
+    optimizer,
+)::Nothing
+    total_time = @elapsed begin
+        @info "Reading: $case"
+        time_read = @elapsed begin
+            instance = read_benchmark(case)
+        end
+        @info @sprintf("Read problem in %.2f seconds", time_read)
+        BLAS.set_num_threads(Threads.nthreads())
+        model = build_model(
+            instance = instance,
+            formulation = formulation,
+            optimizer = optimizer,
+            variable_names = true,
+        )
+        @info "Optimizing..."
+        BLAS.set_num_threads(1)
+        optimize!(model, method)
+    end
+    @info @sprintf("Total time was %.2f seconds", total_time)
+
+    @info "Writing solution: $solution_filename"
+    solution = UnitCommitment.solution(model)
+    write("$solution_filename", solution)
+
+    @info "Verifying solution..."
+    validate(instance, solution)
+    return
+end
+
+function _run_benchmark_combination(
+    case::String,
+    optimizer_name::String,
+    optimizer,
+    method_name::String,
+    method::SolutionMethod,
+    formulation_name::String,
+    formulation::Formulation,
+    trial,
+)
+    dirname = "results/$optimizer_name/$method_name/$formulation_name/$case"
+    function info(msg)
+        @info @sprintf(
+            "%-8s %-16s %-16s %-16s %-8s %s",
+            msg,
+            optimizer_name,
+            method_name,
+            formulation_name,
+            trial,
+            case
+        )
+    end
+    mkpath(dirname)
+    trial_filename = @sprintf("%s/%03d.json", dirname, trial)
+    if isfile(trial_filename)
+        info("skip")
+        return
+    end
+    info("run")
+    open("$trial_filename.log", "w") do file
+        redirect_stdout(file) do
+            redirect_stderr(file) do
+                return _run_benchmark_sample(
+                    case = case,
+                    method = method,
+                    formulation = formulation,
+                    solution_filename = trial_filename,
+                    optimizer = optimizer,
+                )
+            end
+        end
+    end
+    return info("done")
+end
+
+function _run_benchmarks(;
+    cases::Vector{String},
+    optimizers::Dict,
+    formulations::Dict,
+    methods::Dict,
+    trials,
+)
+    combinations = [
+        (c, s.first, s.second, m.first, m.second, f.first, f.second, t) for
+        c in cases for s in optimizers for f in formulations for
+        m in methods for t in trials
+    ]
+    shuffle!(combinations)
+    if nworkers() > 1
+        @printf("%24s", "")
+    end
+    @info @sprintf(
+        "%-8s %-16s %-16s %-16s %-8s %s",
+        "STATUS",
+        "SOLVER",
+        "METHOD",
+        "FORMULATION",
+        "TRIAL",
+        "CASE"
+    )
+    @sync @distributed for c in combinations
+        _run_benchmark_combination(c...)
+    end
+end