Reformat source code

Fix duplicated startup constraint

.github/workflows/test.yml

@@ -9,8 +9,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        julia-version: ['1.3', '1.4', '1.5', '1.6']
-        julia-arch: [x64, x86]
+        julia-version: ['1.6', '1.7']
+        julia-arch: [x64]
         os: [ubuntu-latest, windows-latest, macOS-latest]
         exclude:
           - os: macOS-latest

.gitignore

@@ -1,19 +1,38 @@
 *.bak
 *.gz
-*.lastrun
-*.so
-*.mps
 *.ipynb
+*.lastrun
+*.mps
+*.so
+*/Manifest.toml
+.AppleDB
+.AppleDesktop
+.AppleDouble
+.DS_Store
+.DocumentRevisions-V100
+.LSOverride
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+._*
+.apdisk
+.com.apple.timemachine.donotpresent
+.fseventsd
 .ipy*
+.vscode
+Icon
+Manifest.toml
+Network Trash Folder
+TODO.md
+Temporary Items
 benchmark/results
 benchmark/runs
 benchmark/tables
+benchmark/tmp.json
 build
+docs/_build
 instances/**/*.json
 instances/_source
 local
 notebooks
-TODO.md
-docs/_build
-.vscode
-Manifest.toml

CHANGELOG.md

@@ -4,18 +4,40 @@ All notable changes to this project will be documented in this file.
 
 - The format is based on [Keep a Changelog][changelog].
 - This project adheres to [Semantic Versioning][semver].
-- For versions before 1.0, we follow [the Pkg.jl convention][pkjjl]
+- For versions before 1.0, we follow the [Pkg.jl convention][pkjjl]
   that `0.a.b` is compatible with `0.a.c`.
 
 [changelog]: https://keepachangelog.com/en/1.0.0/
 [semver]: https://semver.org/spec/v2.0.0.html
 [pkjjl]: https://pkgdocs.julialang.org/v1/compatibility/#compat-pre-1.0
 
+## [Unreleased]
+### Added
+- Add multiple reserve products
+
+### Changed
+- To support multiple reserve products, the input data format has been modified as follows:
+    - In `Generators`, replace `Provides spinning reserves?` by `Reserve eligibility`
+    - In `Parameters`, remove `Reserve shortfall penalty`
+    - Revise `Reserves` section
+
+
+## [0.2.2] - 2021-07-21
+### Fixed
+- Fix small bug in validation scripts related to startup costs
+- Fix duplicated startup constraints (@mtanneau, #12)
+
+## [0.2.1] - 2021-06-02
+### Added
+- Add multiple ramping formulations (ArrCon2000, MorLatRam2013, DamKucRajAta2016, PanGua2016)
+- Add multiple piecewise-linear costs formulations (Garver1962, CarArr2006, KnuOstWat2018)
+- Allow benchmark scripts to compare multiple formulations
+
 ## [0.2.0] - 2021-05-28
 ### Added
 - Add sub-hourly unit commitment.
 - Add `UnitCommitment.write(filename, solution)`.
-- Add mathematical formulation to the documentation.
+- Add current mathematical formulation to the documentation.
 
 ### Changed
 - Rename "Time (h)" parameter to "Time horizon (h)".

Makefile

@@ -2,31 +2,22 @@
 # Copyright (C) 2020, UChicago Argonne, LLC. All rights reserved.
 # Released under the modified BSD license. See COPYING.md for more details.
 
-JULIA := julia --color=yes --project=@.
 VERSION := 0.2
 
-build/sysimage.so: src/sysimage.jl Project.toml Manifest.toml
-	mkdir -p build
-	mkdir -p benchmark/results/test
-	cd benchmark; $(JULIA) --trace-compile=../build/precompile.jl run.jl test/case14.1.sol.json
-	$(JULIA) src/sysimage.jl
-
 clean:
-	rm -rf build/*
+	rm -rfv build Manifest.toml test/Manifest.toml deps/formatter/build deps/formatter/Manifest.toml
 
 docs:
 	cd docs; make clean; make dirhtml
 	rsync -avP --delete-after docs/_build/dirhtml/ ../docs/$(VERSION)/
 	
-test: build/sysimage.so
-	@echo Running tests...
-	$(JULIA) --sysimage build/sysimage.so -e 'using Pkg; Pkg.test("UnitCommitment")' | tee build/test.log
-
-
 format:
-	julia -e 'using JuliaFormatter; format("src"); format("test"); format("benchmark")'
+	cd deps/formatter; ../../juliaw format.jl
 
-install-deps:
-	julia -e 'using Pkg; Pkg.add(PackageSpec(name="JuliaFormatter", version="0.14.4"))'
+test: test/Manifest.toml
+	./juliaw test/runtests.jl
+
+test/Manifest.toml: test/Project.toml
+	julia --project=test -e "using Pkg; Pkg.instantiate()"
 
 .PHONY: docs test format install-deps

Project.toml

@@ -2,10 +2,12 @@ name = "UnitCommitment"
 uuid = "64606440-39ea-11e9-0f29-3303a1d3d877"
 authors = ["Santos Xavier, Alinson <axavier@anl.gov>"]
 repo = "https://github.com/ANL-CEEESA/UnitCommitment.jl"
-version = "0.2.0"
+version = "0.3.0"
 
 [deps]
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
+Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
+Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 GZip = "92fee26a-97fe-5a0c-ad85-20a5f3185b63"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 JuMP = "4076af6c-e467-56ae-b986-b466b2749572"
@@ -14,21 +16,16 @@ Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 MathOptInterface = "b8f27783-ece8-5eb3-8dc8-9495eed66fee"
 PackageCompiler = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 
 [compat]
-Cbc = "0.7"
 DataStructures = "0.18"
+Distributions = "0.25"
 GZip = "0.5"
 JSON = "0.21"
-JuMP = "0.21"
-MathOptInterface = "0.9"
+JuMP = "1"
+MathOptInterface = "1"
 PackageCompiler = "1"
 julia = "1"
-
-[extras]
-Cbc = "9961bab8-2fa3-5c5a-9d89-47fab24efd76"
-Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-
-[targets]
-test = ["Cbc", "Test"]

README.md

@@ -14,40 +14,132 @@
   </a>
 </p>
 
-**UnitCommitment.jl** (UC.jl) is an optimization package for the Security-Constrained Unit Commitment Problem (SCUC), a fundamental optimization problem in power systems used, for example, to clear the day-ahead electricity markets. The package provides benchmark instances for the problem and JuMP implementations of state-of-the-art mixed-integer programming formulations.
+**UnitCommitment.jl** (UC.jl) is an optimization package for the Security-Constrained Unit Commitment Problem (SCUC), a fundamental optimization problem in power systems used, for example, to clear the day-ahead electricity markets. The package provides benchmark instances for the problem and Julia/JuMP implementations of state-of-the-art mixed-integer programming formulations.
 
-### Package Components
+## Package Components
 
-* **Data Format:** The package proposes an extensible and fully-documented JSON-based data specification format for SCUC, developed in collaboration with Independent System Operators (ISOs), which describes the most important aspects of the problem. The format supports all the most common generator characteristics (including ramping, piecewise-linear production cost curves and time-dependent startup costs), as well as operating reserves, price-sensitive loads, transmission networks and contingencies.
-* **Benchmark Instances:** The package provides a diverse collection of large-scale benchmark instances collected from the literature and extended to make them more challenging and realistic.
-* **Model Implementation**: The package provides a Julia/JuMP implementation of state-of-the-art formulations and solution methods for SCUC. Our goal is to keep this implementation up-to-date, as new methods are proposed in the literature.
+* **Data Format:** The package proposes an extensible and fully-documented JSON-based data format for SCUC, developed in collaboration with Independent System Operators (ISOs), which describes the most important aspects of the problem. The format supports the most common generator characteristics (including ramping, piecewise-linear production cost curves and time-dependent startup costs), as well as operating reserves, price-sensitive loads, transmission networks and contingencies.
+* **Benchmark Instances:** The package provides a diverse collection of large-scale benchmark instances collected from the literature, converted into a common data format, and extended using data-driven methods to make them more challenging and realistic.
+* **Model Implementation**: The package provides Julia/JuMP implementations of state-of-the-art formulations and solution methods for SCUC, including multiple ramping formulations ([ArrCon2000][ArrCon2000], [MorLatRam2013][MorLatRam2013], [DamKucRajAta2016][DamKucRajAta2016], [PanGua2016][PanGua2016]), multiple piecewise-linear costs formulations ([Gar1962][Gar1962], [CarArr2006][CarArr2006], [KnuOstWat2018][KnuOstWat2018]) and contingency screening methods ([XavQiuWanThi2019][XavQiuWanThi2019]). Our goal is to keep these implementations up-to-date as new methods are proposed in the literature.
 * **Benchmark Tools:** The package provides automated benchmark scripts to accurately evaluate the performance impact of proposed code changes.
 
-### Documentation
+[ArrCon2000]: https://doi.org/10.1109/59.871739
+[CarArr2006]: https://doi.org/10.1109/TPWRS.2006.876672
+[DamKucRajAta2016]: https://doi.org/10.1007/s10107-015-0919-9
+[Gar1962]: https://doi.org/10.1109/AIEEPAS.1962.4501405
+[KnuOstWat2018]: https://doi.org/10.1109/TPWRS.2017.2783850
+[MorLatRam2013]: https://doi.org/10.1109/TPWRS.2013.2251373
+[PanGua2016]: https://doi.org/10.1287/opre.2016.1520
+[XavQiuWanThi2019]: https://doi.org/10.1109/TPWRS.2019.2892620
 
-* [Usage](https://anl-ceeesa.github.io/UnitCommitment.jl/0.1/usage/)
-* [Data Format](https://anl-ceeesa.github.io/UnitCommitment.jl/0.1/format/)
-* [Instances](https://anl-ceeesa.github.io/UnitCommitment.jl/0.1/instances/)
+## Sample Usage
 
-### Authors
-* **Alinson Santos Xavier** (Argonne National Laboratory)
+```julia
+using Cbc
+using JuMP
+using UnitCommitment
+
+import UnitCommitment:
+    Formulation,
+    KnuOstWat2018,
+    MorLatRam2013,
+    ShiftFactorsFormulation
+
+# Read benchmark instance
+instance = UnitCommitment.read_benchmark(
+    "matpower/case118/2017-02-01",
+)
+
+# Construct model (using state-of-the-art defaults)
+model = UnitCommitment.build_model(
+    instance = instance,
+    optimizer = Cbc.Optimizer,
+)
+
+# Construct model (using customized formulation)
+model = UnitCommitment.build_model(
+    instance = instance,
+    optimizer = Cbc.Optimizer,
+    formulation = Formulation(
+        pwl_costs = KnuOstWat2018.PwlCosts(),
+        ramping = MorLatRam2013.Ramping(),
+        startup_costs = MorLatRam2013.StartupCosts(),
+        transmission = ShiftFactorsFormulation(
+            isf_cutoff = 0.005,
+            lodf_cutoff = 0.001,
+        ),
+    ),
+)
+
+# Modify the model (e.g. add custom constraints)
+@constraint(
+    model,
+    model[:is_on]["g3", 1] + model[:is_on]["g4", 1] <= 1,
+)
+
+# Solve model
+UnitCommitment.optimize!(model)
+
+# Extract solution
+solution = UnitCommitment.solution(model)
+UnitCommitment.write("/tmp/output.json", solution)
+```
+
+## Documentation
+
+1. [Usage](https://anl-ceeesa.github.io/UnitCommitment.jl/0.2/usage/)
+2. [Data Format](https://anl-ceeesa.github.io/UnitCommitment.jl/0.2/format/)
+3. [Instances](https://anl-ceeesa.github.io/UnitCommitment.jl/0.2/instances/)
+4. [JuMP Model](https://anl-ceeesa.github.io/UnitCommitment.jl/0.2/model/)
+
+## Authors
+* **Alinson S. Xavier** (Argonne National Laboratory)
+* **Aleksandr M. Kazachkov** (University of Florida)
+* **Ogün Yurdakul** (Technische Universität Berlin)
 * **Feng Qiu** (Argonne National Laboratory)
 
-### Acknowledgments
+## Acknowledgments
 
-* We would like to thank **Aleksandr M. Kazachkov** (University of Florida), **Yonghong Chen** (Midcontinent Independent System Operator), **Feng Pan** (Pacific Northwest National Laboratory) for valuable feedback on early versions of this package.
+* We would like to **Yonghong Chen** (Midcontinent Independent System Operator), **Feng Pan** (Pacific Northwest National Laboratory) for valuable feedback on early versions of this package.
 
-* Based upon work supported by **Laboratory Directed Research and Development** (LDRD) funding from Argonne National Laboratory, provided by the Director, Office of Science, of the U.S. Department of Energy under Contract No. DE-AC02-06CH11357, and the U.S. Department of Energy **Advanced Grid Modeling Program** under Grant DE-OE0000875
+* Based upon work supported by **Laboratory Directed Research and Development** (LDRD) funding from Argonne National Laboratory, provided by the Director, Office of Science, of the U.S. Department of Energy under Contract No. DE-AC02-06CH11357
 
-### Citing
+* Based upon work supported by the **U.S. Department of Energy Advanced Grid Modeling Program** under Grant DE-OE0000875.
 
-If you use UnitCommitment.jl in your research, we request that you cite the package as follows:
+## Citing
 
-* **Alinson S. Xavier, Feng Qiu**. "UnitCommitment.jl: A Julia/JuMP Optimization Package for Security-Constrained Unit Commitment". Zenodo (2020). [DOI: 10.5281/zenodo.4269874](https://doi.org/10.5281/zenodo.4269874).
+If you use UnitCommitment.jl in your research (instances, models or algorithms), we kindly request that you cite the package as follows:
 
-If you make use of the provided instances files, we request that you additionally cite the original sources, as described in the [instances page](https://anl-ceeesa.github.io/UnitCommitment.jl/0.1/instances/).
+* **Alinson S. Xavier, Aleksandr M. Kazachkov, Feng Qiu**. "UnitCommitment.jl: A Julia/JuMP Optimization Package for Security-Constrained Unit Commitment". Zenodo (2020). [DOI: 10.5281/zenodo.4269874](https://doi.org/10.5281/zenodo.4269874).
 
-### License
+If you use the instances, we additionally request that you cite the original sources, as described in the [instances page](docs/instances.md).
 
-Released under the modified BSD license. See `LICENSE.md` for more details.
+## License
+
+```text
+UnitCommitment.jl: A Julia/JuMP Optimization Package for Security-Constrained Unit Commitment
+Copyright © 2020-2021, UChicago Argonne, LLC. All Rights Reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted
+provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of
+   conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice, this list of
+   conditions and the following disclaimer in the documentation and/or other materials provided
+   with the distribution.
+3. Neither the name of the copyright holder nor the names of its contributors may be used to
+   endorse or promote products derived from this software without specific prior written
+   permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+```
 

benchmark/Makefile

@@ -1,105 +0,0 @@
-# UnitCommitment.jl: Optimization Package for Security-Constrained Unit Commitment
-# Copyright (C) 2020, UChicago Argonne, LLC. All rights reserved.
-# Released under the modified BSD license. See COPYING.md for more details.
-
-SHELL := /bin/bash
-JULIA := julia --project=. --sysimage ../build/sysimage.so
-TIMESTAMP := $(shell date "+%Y-%m-%d %H:%M")
-SRC_FILES := $(wildcard ../src/*.jl)
-
-INSTANCES_PGLIB := \
-        pglib-uc/ca/2014-09-01_reserves_0 \
-        pglib-uc/ca/2014-09-01_reserves_1 \
-        pglib-uc/ca/2015-03-01_reserves_0 \
-        pglib-uc/ca/2015-06-01_reserves_0 \
-        pglib-uc/ca/Scenario400_reserves_1 \
-        pglib-uc/ferc/2015-01-01_lw \
-        pglib-uc/ferc/2015-05-01_lw \
-        pglib-uc/ferc/2015-07-01_hw \
-        pglib-uc/ferc/2015-10-01_lw \
-        pglib-uc/ferc/2015-12-01_lw \
-        pglib-uc/rts_gmlc/2020-04-03 \
-        pglib-uc/rts_gmlc/2020-09-20 \
-        pglib-uc/rts_gmlc/2020-10-27 \
-        pglib-uc/rts_gmlc/2020-11-25 \
-        pglib-uc/rts_gmlc/2020-12-23
-
-INSTANCES_MATPOWER := \
-	matpower/case118/2017-02-01 \
-	matpower/case118/2017-08-01 \
-	matpower/case300/2017-02-01 \
-	matpower/case300/2017-08-01 \
-	matpower/case1354pegase/2017-02-01 \
-	matpower/case1888rte/2017-02-01 \
-	matpower/case1951rte/2017-08-01 \
-	matpower/case2848rte/2017-02-01 \
-	matpower/case2868rte/2017-08-01 \
-	matpower/case3375wp/2017-08-01 \
-	matpower/case6468rte/2017-08-01 \
-	matpower/case6515rte/2017-08-01
-
-INSTANCES_ORLIB := \
-	or-lib/20_0_1_w \
-	or-lib/20_0_5_w \
-	or-lib/50_0_2_w \
-	or-lib/75_0_2_w \
-	or-lib/100_0_1_w \
-	or-lib/100_0_4_w \
-	or-lib/100_0_5_w \
-	or-lib/200_0_3_w \
-	or-lib/200_0_7_w \
-	or-lib/200_0_9_w
-
-INSTANCES_TEJADA19 := \
-	tejada19/UC_24h_290g \
-	tejada19/UC_24h_623g \
-	tejada19/UC_24h_959g \
-	tejada19/UC_24h_1577g \
-	tejada19/UC_24h_1888g \
-	tejada19/UC_168h_72g \
-	tejada19/UC_168h_86g \
-	tejada19/UC_168h_130g \
-	tejada19/UC_168h_131g \
-	tejada19/UC_168h_199g
-
-SAMPLES := 1 2 3 4 5
-SOLUTIONS_MATPOWER := $(foreach s,$(SAMPLES),$(addprefix results/,$(addsuffix .$(s).sol.json,$(INSTANCES_MATPOWER))))
-SOLUTIONS_PGLIB := $(foreach s,$(SAMPLES),$(addprefix results/,$(addsuffix .$(s).sol.json,$(INSTANCES_PGLIB))))
-SOLUTIONS_ORLIB := $(foreach s,$(SAMPLES),$(addprefix results/,$(addsuffix .$(s).sol.json,$(INSTANCES_ORLIB))))
-SOLUTIONS_TEJADA19 := $(foreach s,$(SAMPLES),$(addprefix results/,$(addsuffix .$(s).sol.json,$(INSTANCES_TEJADA19))))
-
-.PHONY: tables save small large clean-mps matpower pglib orlib
-
-all: matpower pglib orlib tejada19
-
-matpower: $(SOLUTIONS_MATPOWER)
-
-pglib: $(SOLUTIONS_PGLIB)
-
-orlib: $(SOLUTIONS_ORLIB)
-
-tejada19: $(SOLUTIONS_TEJADA19)
-
-clean:
-	@rm -rf tables/benchmark* tables/compare* results
-    
-clean-mps:
-	@rm -fv results/*/*.mps.gz results/*/*/*.mps.gz
-
-clean-sol:
-	@rm -rf results/*/*.sol.* results/*/*/*.sol.*
-        
-save:
-	mkdir -p "runs/$(TIMESTAMP)"
-	rsync -avP results tables "runs/$(TIMESTAMP)/"
-
-results/%.sol.json: run.jl
-	@echo "run  $*"
-	@mkdir -p $(dir results/$*)
-	@$(JULIA) run.jl $* 2>&1 | cat > results/$*.log
-	@echo "run  $* [done]"
-
-tables:
-	@mkdir -p tables
-	@python scripts/table.py
-	#@python scripts/compare.py tables/reference.csv tables/benchmark.csv

benchmark/Project.toml

@@ -1,4 +1,5 @@
 [deps]
+DocOpt = "968ba79b-81e4-546f-ab3a-2eecfa62a9db"
 Gurobi = "2e9cd046-0924-5485-92f1-d5272153d98b"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 JuMP = "4076af6c-e467-56ae-b986-b466b2749572"

benchmark/run.jl

@@ -2,65 +2,208 @@
 # Copyright (C) 2020, UChicago Argonne, LLC. All rights reserved.
 # Released under the modified BSD license. See COPYING.md for more details.
 
+doc = """UnitCommitment.jl Benchmark Runner
+
+Usage:
+  run.jl [-s ARG]... [-m ARG]... [-c ARG]... [-f ARG]... [options]
+
+Examples:
+
+    1. Benchmark all solvers, methods and formulations:
+
+        julia run.jl
+
+    2. Benchmark formulations "default" and "ArrCon200" using Gurobi:
+
+        julia run.jl -s gurobi -f default -f ArrCon2000
+
+    3. Benchmark a few test cases, using all solvers, methods and formulations:
+
+        julia run.jl -c or-lib/20_0_1_w -c matpower/case1888rte/2017-02-01
+
+    4. Solve 4 test cases in parallel, with 2 threads available per worker:
+
+        JULIA_NUM_THREADS=2 julia --procs 4 run.jl
+
+Options:
+  -h --help             Show this screen.
+  -s --solver=ARG       Mixed-integer linear solver (e.g. gurobi)
+  -c --case=ARG         Unit commitment test case (e.g. or-lib/20_0_1_w)
+  -m --method=ARG       Solution method (e.g. default)
+  -f --formulation=ARG  Formulation (e.g. ArrCon2000)
+  --time-limit=ARG      Time limit in seconds [default: 3600]
+  --gap=ARG             Relative MIP gap tolerance [default: 0.001]
+  --trials=ARG          Number of trials [default: 5]
+"""
+
+using Distributed
+using Pkg
+Pkg.activate(".")
+@everywhere using Pkg
+@everywhere Pkg.activate(".")
+
+using DocOpt
+args = docopt(doc)
+
+@everywhere using UnitCommitment
+@everywhere UnitCommitment._setup_logger()
+
 using UnitCommitment
-using JuMP
 using Gurobi
-using JSON
 using Logging
-using Printf
-using LinearAlgebra
+using JuMP
 
-UnitCommitment._setup_logger()
+import UnitCommitment:
+    ArrCon2000,
+    CarArr2006,
+    DamKucRajAta2016,
+    Formulation,
+    Gar1962,
+    KnuOstWat2018,
+    MorLatRam2013,
+    PanGua2016,
+    XavQiuWanThi2019
 
-function main()
-    basename, suffix = split(ARGS[1], ".")
-    solution_filename = "results/$basename.$suffix.sol.json"
-    model_filename = "results/$basename.$suffix.mps.gz"
+# Benchmark test cases
+# -----------------------------------------------------------------------------
+cases = [
+    "pglib-uc/ca/2014-09-01_reserves_0",
+    "pglib-uc/ca/2014-09-01_reserves_1",
+    "pglib-uc/ca/2015-03-01_reserves_0",
+    "pglib-uc/ca/2015-06-01_reserves_0",
+    "pglib-uc/ca/Scenario400_reserves_1",
+    "pglib-uc/ferc/2015-01-01_lw",
+    "pglib-uc/ferc/2015-05-01_lw",
+    "pglib-uc/ferc/2015-07-01_hw",
+    "pglib-uc/ferc/2015-10-01_lw",
+    "pglib-uc/ferc/2015-12-01_lw",
+    "pglib-uc/rts_gmlc/2020-04-03",
+    "pglib-uc/rts_gmlc/2020-09-20",
+    "pglib-uc/rts_gmlc/2020-10-27",
+    "pglib-uc/rts_gmlc/2020-11-25",
+    "pglib-uc/rts_gmlc/2020-12-23",
+    "or-lib/20_0_1_w",
+    "or-lib/20_0_5_w",
+    "or-lib/50_0_2_w",
+    "or-lib/75_0_2_w",
+    "or-lib/100_0_1_w",
+    "or-lib/100_0_4_w",
+    "or-lib/100_0_5_w",
+    "or-lib/200_0_3_w",
+    "or-lib/200_0_7_w",
+    "or-lib/200_0_9_w",
+    "tejada19/UC_24h_290g",
+    "tejada19/UC_24h_623g",
+    "tejada19/UC_24h_959g",
+    "tejada19/UC_24h_1577g",
+    "tejada19/UC_24h_1888g",
+    "tejada19/UC_168h_72g",
+    "tejada19/UC_168h_86g",
+    "tejada19/UC_168h_130g",
+    "tejada19/UC_168h_131g",
+    "tejada19/UC_168h_199g",
+    "matpower/case1888rte/2017-02-01",
+    "matpower/case1951rte/2017-02-01",
+    "matpower/case2848rte/2017-02-01",
+    "matpower/case3012wp/2017-02-01",
+    "matpower/case3375wp/2017-02-01",
+    "matpower/case6468rte/2017-02-01",
+    "matpower/case6515rte/2017-02-01",
+]
 
-    time_limit = 60 * 20
+# Formulations
+# -----------------------------------------------------------------------------
+formulations = Dict(
+    "default" => Formulation(),
+    "ArrCon2000" => Formulation(ramping = ArrCon2000.Ramping()),
+    "CarArr2006" => Formulation(pwl_costs = CarArr2006.PwlCosts()),
+    "DamKucRajAta2016" => Formulation(ramping = DamKucRajAta2016.Ramping()),
+    "Gar1962" => Formulation(pwl_costs = Gar1962.PwlCosts()),
+    "KnuOstWat2018" => Formulation(pwl_costs = KnuOstWat2018.PwlCosts()),
+    "MorLatRam2013" => Formulation(ramping = MorLatRam2013.Ramping()),
+    "PanGua2016" => Formulation(ramping = PanGua2016.Ramping()),
+)
 
-    BLAS.set_num_threads(4)
+# Solution methods
+# -----------------------------------------------------------------------------
+const gap_limit = parse(Float64, args["--gap"])
+const time_limit = parse(Float64, args["--time-limit"])
+methods = Dict(
+    "default" => XavQiuWanThi2019.Method(
+        time_limit = time_limit,
+        gap_limit = gap_limit,
+    ),
+)
 
-    total_time = @elapsed begin
-        @info "Reading: $basename"
-        time_read = @elapsed begin
-            instance = UnitCommitment.read_benchmark(basename)
-        end
-        @info @sprintf("Read problem in %.2f seconds", time_read)
+# MIP solvers
+# -----------------------------------------------------------------------------
+optimizers = Dict(
+    "gurobi" => optimizer_with_attributes(
+        Gurobi.Optimizer,
+        "Threads" => Threads.nthreads(),
+    ),
+)
 
-        time_model = @elapsed begin
-            model = build_model(
-                instance = instance,
-                optimizer = optimizer_with_attributes(
-                    Gurobi.Optimizer,
-                    "Threads" => 4,
-                    "Seed" => rand(1:1000),
-                ),
-                variable_names = true,
-            )
-        end
+# Parse command line arguments
+# -----------------------------------------------------------------------------
+if !isempty(args["--case"])
+    cases = args["--case"]
+end
+if !isempty(args["--formulation"])
+    formulations = filter(p -> p.first in args["--formulation"], formulations)
+end
+if !isempty(args["--method"])
+    methods = filter(p -> p.first in args["--method"], methods)
+end
+if !isempty(args["--solver"])
+    optimizers = filter(p -> p.first in args["--solver"], optimizers)
+end
+const ntrials = parse(Int, args["--trials"])
 
-        @info "Optimizing..."
-        BLAS.set_num_threads(1)
-        UnitCommitment.optimize!(
-            model,
-            time_limit = time_limit,
-            gap_limit = 1e-3,
-        )
+# Print benchmark settings
+# -----------------------------------------------------------------------------
+function printlist(d::Dict)
+    for key in keys(d)
+        @info "  - $key"
     end
-    @info @sprintf("Total time was %.2f seconds", total_time)
-
-    @info "Writing: $solution_filename"
-    solution = UnitCommitment.solution(model)
-    open(solution_filename, "w") do file
-        return JSON.print(file, solution, 2)
-    end
-
-    @info "Verifying solution..."
-    UnitCommitment.validate(instance, solution)
-
-    @info "Exporting model..."
-    return JuMP.write_to_file(model, model_filename)
 end
 
-main()
+function printlist(d::Vector)
+    for key in d
+        @info "  - $key"
+    end
+end
+
+@info "Computational environment:"
+@info "  - CPU: $(Sys.cpu_info()[1].model)"
+@info "  - Logical CPU cores: $(length(Sys.cpu_info()))"
+@info "  - System memory: $(round(Sys.total_memory() / 2^30, digits=2)) GiB"
+@info "  - Available workers: $(nworkers())"
+@info "  - Available threads per worker: $(Threads.nthreads())"
+
+@info "Parameters:"
+@info "  - Number of trials: $ntrials"
+@info "  - Time limit (s): $time_limit"
+@info "  - Relative MIP gap tolerance: $gap_limit"
+
+@info "Solvers:"
+printlist(optimizers)
+
+@info "Methods:"
+printlist(methods)
+
+@info "Formulations:"
+printlist(formulations)
+
+@info "Cases:"
+printlist(cases)
+
+# Run benchmarks
+# -----------------------------------------------------------------------------
+UnitCommitment._run_benchmarks(
+    cases = cases,
+    formulations = formulations,
+    methods = methods,
+    optimizers = optimizers,
+    trials = 1:ntrials,
+)

benchmark/scripts/compare.py

@@ -5,71 +5,82 @@
 import pandas as pd
 import numpy as np
 import seaborn as sns
+import matplotlib
 import matplotlib.pyplot as plt
 import sys
 
-# easy_cutoff = 120
-
-b1 = pd.read_csv(sys.argv[1], index_col=0)
-b2 = pd.read_csv(sys.argv[2], index_col=0)
-
-c1 = b1.groupby(["Group", "Instance", "Sample"])[
-    ["Optimization time (s)", "Primal bound"]
-].mean()
-c2 = b2.groupby(["Group", "Instance", "Sample"])[
-    ["Optimization time (s)", "Primal bound"]
-].mean()
-c1.columns = ["A Time (s)", "A Value"]
-c2.columns = ["B Time (s)", "B Value"]
-
-merged = pd.concat([c1, c2], axis=1)
-merged["Speedup"] = merged["A Time (s)"] / merged["B Time (s)"]
-merged["Time diff (s)"] = merged["B Time (s)"] - merged["A Time (s)"]
-merged["Value diff (%)"] = np.round(
-    (merged["B Value"] - merged["A Value"]) / merged["A Value"] * 100.0, 5
+matplotlib.use("Agg")
+sns.set("talk")
+sns.set_palette(
+    [
+        "#9b59b6",
+        "#3498db",
+        "#95a5a6",
+        "#e74c3c",
+        "#34495e",
+        "#2ecc71",
+    ]
 )
-merged.loc[merged.loc[:, "B Time (s)"] <= 0, "Speedup"] = float("nan")
-merged.loc[merged.loc[:, "B Time (s)"] <= 0, "Time diff (s)"] = float("nan")
-# merged = merged[(merged["A Time (s)"] >= easy_cutoff) | (merged["B Time (s)"] >= easy_cutoff)]
-merged.reset_index(inplace=True)
-merged["Name"] = merged["Group"] + "/" + merged["Instance"]
-# merged = merged.sort_values(by="Speedup", ascending=False)
 
+filename = sys.argv[1]
+m1 = sys.argv[2]
+m2 = sys.argv[3]
 
-k = len(merged.groupby("Name"))
-plt.figure(figsize=(12, 0.50 * k))
-plt.rcParams["xtick.bottom"] = plt.rcParams["xtick.labelbottom"] = True
-plt.rcParams["xtick.top"] = plt.rcParams["xtick.labeltop"] = True
-sns.set_style("whitegrid")
-sns.set_palette("Set1")
+# Prepare data
+data = pd.read_csv(filename, index_col=0)
+b1 = (
+    data[data["Group"] == m1]
+    .groupby(["Instance", "Sample"])
+    .mean()[["Optimization time (s)"]]
+)
+b2 = (
+    data[data["Group"] == m2]
+    .groupby(["Instance", "Sample"])
+    .mean()[["Optimization time (s)"]]
+)
+b1.columns = [f"{m1} time (s)"]
+b2.columns = [f"{m2} time (s)"]
+merged = pd.merge(b1, b2, left_index=True, right_index=True).reset_index().dropna()
+merged["Speedup"] = merged[f"{m1} time (s)"] / merged[f"{m2} time (s)"]
+merged["Group"] = merged["Instance"].str.replace(r"\/.*", "", regex=True)
+merged = merged.sort_values(by=["Instance", "Sample"], ascending=True)
+merged = merged[(merged[f"{m1} time (s)"] > 0) & (merged[f"{m2} time (s)"] > 0)]
+
+# Plot results
+k1 = len(merged.groupby("Instance").mean())
+k2 = len(merged.groupby("Group").mean())
+k = k1 + k2
+fig = plt.figure(
+    constrained_layout=True,
+    figsize=(15, max(5, 0.75 * k)),
+)
+plt.suptitle(f"{m1} vs {m2}")
+gs1 = fig.add_gridspec(nrows=k, ncols=1)
+ax1 = fig.add_subplot(gs1[0:k1, 0:1])
+ax2 = fig.add_subplot(gs1[k1:, 0:1], sharex=ax1)
 sns.barplot(
     data=merged,
     x="Speedup",
-    y="Name",
-    color="tab:red",
-    capsize=0.15,
+    y="Instance",
+    color="tab:purple",
     errcolor="k",
     errwidth=1.25,
+    ax=ax1,
 )
-plt.axvline(1.0, linestyle="--", color="k")
-plt.tight_layout()
+sns.barplot(
+    data=merged,
+    x="Speedup",
+    y="Group",
+    color="tab:purple",
+    errcolor="k",
+    errwidth=1.25,
+    ax=ax2,
+)
+ax1.axvline(1.0, linestyle="--", color="k")
+ax2.axvline(1.0, linestyle="--", color="k")
 
 print("Writing tables/compare.png")
 plt.savefig("tables/compare.png", dpi=150)
 
 print("Writing tables/compare.csv")
-merged.loc[
-    :,
-    [
-        "Group",
-        "Instance",
-        "Sample",
-        "A Time (s)",
-        "B Time (s)",
-        "Speedup",
-        "Time diff (s)",
-        "A Value",
-        "B Value",
-        "Value diff (%)",
-    ],
-].to_csv("tables/compare.csv", index_label="Index")
+merged.to_csv("tables/compare.csv", index_label="Index")

benchmark/scripts/table.py

@@ -6,11 +6,13 @@ from pathlib import Path
 import pandas as pd
 import re
 from tabulate import tabulate
+from colorama import init, Fore, Back, Style
+
+init()
 
 
 def process_all_log_files():
-    pathlist = list(Path(".").glob("results/*/*/*.log"))
-    pathlist += list(Path(".").glob("results/*/*.log"))
+    pathlist = list(Path(".").glob("results/**/*.log"))
     rows = []
     for path in pathlist:
         if ".ipy" in str(path):
@@ -26,9 +28,9 @@ def process_all_log_files():
 
 def process(filename):
     parts = filename.replace(".log", "").split("/")
-    group_name = "/".join(parts[1:-1])
-    instance_name = parts[-1]
-    instance_name, sample_name = instance_name.split(".")
+    group_name = parts[1]
+    instance_name = "/".join(parts[2:-1])
+    sample_name = parts[-1]
     nodes = 0.0
     optimize_time = 0.0
     simplex_iterations = 0.0
@@ -49,6 +51,7 @@ def process(filename):
     # m = re.search("case([0-9]*)", instance_name)
     # n_buses = int(m.group(1))
     n_buses = 0
+    validation_errors = 0
 
     with open(filename) as file:
         for line in file.readlines():
@@ -138,6 +141,14 @@ def process(filename):
             if m is not None:
                 transmission_count += 1
 
+            m = re.search(r".*Found ([0-9]*) validation errors", line)
+            if m is not None:
+                validation_errors += int(m.group(1))
+                print(
+                    f"{Fore.YELLOW}{Style.BRIGHT}Warning:{Style.RESET_ALL} {validation_errors:8d} "
+                    f"{Style.DIM}validation errors in {Style.RESET_ALL}{group_name}/{instance_name}/{sample_name}"
+                )
+
     return {
         "Group": group_name,
         "Instance": instance_name,
@@ -169,34 +180,44 @@ def process(filename):
         "Transmission screening constraints": transmission_count,
         "Transmission screening time": transmission_time,
         "Transmission screening calls": transmission_calls,
+        "Validation errors": validation_errors,
     }
 
 
 def generate_chart():
     import pandas as pd
+    import matplotlib
     import matplotlib.pyplot as plt
     import seaborn as sns
 
+    matplotlib.use("Agg")
+    sns.set("talk")
+    sns.set_palette(
+        [
+            "#9b59b6",
+            "#3498db",
+            "#95a5a6",
+            "#e74c3c",
+            "#34495e",
+            "#2ecc71",
+        ]
+    )
+
     tables = []
     files = ["tables/benchmark.csv"]
     for f in files:
         table = pd.read_csv(f, index_col=0)
-        table.loc[:, "Instance"] = (
-            table.loc[:, "Group"] + "/" + table.loc[:, "Instance"]
-        )
         table.loc[:, "Filename"] = f
         tables += [table]
     benchmark = pd.concat(tables, sort=True)
-    benchmark = benchmark.sort_values(by="Instance")
-    k = len(benchmark.groupby("Instance"))
-    plt.figure(figsize=(12, 0.50 * k))
-    sns.set_style("whitegrid")
-    sns.set_palette("Set1")
+    benchmark = benchmark.sort_values(by=["Group", "Instance"])
+    k1 = len(benchmark.groupby("Instance"))
+    k2 = len(benchmark.groupby("Group"))
+    plt.figure(figsize=(12, 0.25 * k1 * k2))
     sns.barplot(
         y="Instance",
         x="Total time (s)",
-        color="tab:red",
-        capsize=0.15,
+        hue="Group",
         errcolor="k",
         errwidth=1.25,
         data=benchmark,

deps/formatter/Project.toml

@@ -0,0 +1,5 @@
+[deps]
+JuliaFormatter = "98e50ef6-434e-11e9-1051-2b60c6c9e899"
+
+[compat]
+JuliaFormatter = "0.14.4"

deps/formatter/format.jl

@@ -0,0 +1,9 @@
+using JuliaFormatter
+format(
+    [
+        "../../src", 
+        "../../test",
+        "../../benchmark/run.jl",
+    ],
+    verbose=true,
+)

docs/format.md

@@ -28,11 +28,11 @@ Each section is described in detail below. For a complete example, see [case14](
 
 ### Parameters
 
-This section describes system-wide parameters, such as power balance penalties,  optimization parameters, such as the length of the planning horizon and the time.
+This section describes system-wide parameters, such as power balance penalty, and optimization parameters, such as the length of the planning horizon and the time.
 
 | Key                            | Description                                       | Default  | Time series?
 | :----------------------------- | :------------------------------------------------ | :------: | :------------:
-| `Time horizon (h)`                     | Length of the planning horizon (in hours). | Required | N
+| `Time horizon (h)` | Length of the planning horizon (in hours). | Required | N
 | `Time step (min)` | Length of each time step (in minutes). Must be a divisor of 60 (e.g. 60, 30, 20, 15, etc). | `60` | N
 | `Power balance penalty ($/MW)` | Penalty for system-wide shortage or surplus in production (in $/MW). This is charged per time step. For example, if there is a shortage of 1 MW for three time steps, three times this amount will be charged. | `1000.0` | Y
 
@@ -42,7 +42,7 @@ This section describes system-wide parameters, such as power balance penalties,
 {
     "Parameters": {
         "Time horizon (h)": 4,
-        "Power balance penalty ($/MW)": 1000.0
+        "Power balance penalty ($/MW)": 1000.0,
     }
 }
 ```
@@ -94,7 +94,7 @@ This section describes all generators in the system, including thermal units, re
 | `Initial status (h)`  | If set to a positive number, indicates the amount of time (in hours) the generator has been on at the beginning of the simulation, and if set to a negative number, the amount of time the generator has been off. For example, if `Initial status (h)` is `-2`, this means that the generator was off since `-02:00` (h:min). The simulation starts at time `00:00`. If `Initial status (h)` is `3`, this means that the generator was on since `-03:00`. A value of zero is not acceptable. | Required | N
 | `Initial power (MW)`  | Amount of power the generator at time step `-1`, immediately before the planning horizon starts. | Required | N
 | `Must run?`               | If `true`, the generator should be committed, even if that is not economical (Boolean). | `false` | Y
-| `Provides spinning reserves?`    | If `true`, this generator may provide spinning reserves (Boolean). | `true` | Y
+| `Reserve eligibility` | List of reserve products this generator is eligibe to provide. By default, the generator is not eligible to provide any reserves. | `[]` | N
 
 #### Production costs and limits
 
@@ -133,13 +133,13 @@ Note that this curve also specifies the production limits. Specifically, the fir
             "Minimum uptime (h)": 4,
             "Initial status (h)": 12,
             "Must run?": false,
-            "Provides spinning reserves?": true,
+            "Reserve eligibility": ["r1"],
         },
         "gen2": {
             "Bus": "b5",
             "Production cost curve (MW)": [0.0, [10.0, 8.0, 0.0, 3.0]],
             "Production cost curve ($)": [0.0, 0.0],
-            "Provides spinning reserves?": true,
+            "Reserve eligibility": ["r1", "r2"],
         }
     }
 }
@@ -204,24 +204,39 @@ This section describes the characteristics of transmission system, such as its t
 
 ### Reserves
 
-This section describes the hourly amount of operating reserves required.
+This section describes the hourly amount of reserves required.
 
 
 | Key                   | Description                                        | Default   |  Time series?
 | :-------------------- | :------------------------------------------------- | --------- |  :----:
-| `Spinning (MW)`       | Minimum amount of system-wide spinning reserves (in MW). Only generators which are online may provide this reserve. | `0.0` | Y
+| `Type` | Type of reserve product. Must be either "spinning" or "flexiramp". | Required | N
+| `Amount (MW)` | Amount of reserves required. | Required | Y
+| `Shortfall penalty ($/MW)` | Penalty for shortage in meeting the reserve requirements (in $/MW). This is charged per time step. Negative value implies reserve constraints must always be satisfied. | `-1` | Y
 
-#### Example
+#### Example 1
 
 ```json
 {
     "Reserves": {
-        "Spinning (MW)": [
-            57.30552,
-            53.88429,
-            51.31838,
-            50.46307
-        ]
+        "r1": {
+            "Type": "spinning",
+            "Amount (MW)": [
+                57.30552,
+                53.88429,
+                51.31838,
+                50.46307
+            ],
+            "Shortfall penalty ($/MW)": 5.0
+        },
+        "r2": {
+            "Type": "flexiramp",
+            "Amount (MW)": [
+                20.31042,
+                23.65273,
+                27.41784,
+                25.34057
+            ],
+        }
     }
 }
 ```
@@ -284,9 +299,7 @@ The output data format is also JSON-based, but it is not currently documented si
 Current limitations
 -------------------
 
-* All reserves are system-wide. Zonal reserves are not currently supported.
 * Network topology remains the same for all time periods
 * Only N-1 transmission contingencies are supported. Generator contingencies are not currently supported.
 * Time-varying minimum production amounts are not currently compatible with ramp/startup/shutdown limits.
-
-
+* Flexible ramping products can only be acquired under the `WanHob2016` formulation, which does not support spinning reserves. 

docs/index.md

@@ -2,20 +2,30 @@
 
 **UnitCommitment.jl** (UC.jl) is a Julia/JuMP optimization package for the Security-Constrained Unit Commitment Problem (SCUC), a fundamental optimization problem in power systems used, for example, to clear the day-ahead electricity markets. The package provides benchmark instances for the problem and Julia/JuMP implementations of state-of-the-art mixed-integer programming formulations.
 
-### Package Components
+## Package Components
 
 * **Data Format:** The package proposes an extensible and fully-documented JSON-based data specification format for SCUC, developed in collaboration with Independent System Operators (ISOs), which describes the most important aspects of the problem. The format supports all the most common generator characteristics (including ramping, piecewise-linear production cost curves and time-dependent startup costs), as well as operating reserves, price-sensitive loads, transmission networks and contingencies.
-* **Benchmark Instances:** The package provides a diverse collection of large-scale benchmark instances collected from the literature and extended to make them more challenging and realistic.
-* **Model Implementation**: The package provides a Julia/JuMP implementation of state-of-the-art formulations and solution methods for SCUC. Our goal is to keep this implementation up-to-date, as new methods are proposed in the literature.
+* **Benchmark Instances:** The package provides a diverse collection of large-scale benchmark instances collected from the literature, converted into a common data format, and extended using data-driven methods to make them more challenging and realistic.
+* **Model Implementation**: The package provides a Julia/JuMP implementations of state-of-the-art formulations and solution methods for SCUC, including multiple ramping formulations ([ArrCon2000][ArrCon2000], [MorLatRam2013][MorLatRam2013], [DamKucRajAta2016][DamKucRajAta2016], [PanGua2016][PanGua2016]), multiple piecewise-linear costs formulations ([Gar1962][Gar1962], [CarArr2006][CarArr2006], [KnuOstWat2018][KnuOstWat2018]) and contingency screening methods ([XavQiuWanThi2019][XavQiuWanThi2019]). Our goal is to keep these implementations up-to-date as new methods are proposed in the literature.
 * **Benchmark Tools:** The package provides automated benchmark scripts to accurately evaluate the performance impact of proposed code changes.
 
+[ArrCon2000]: https://doi.org/10.1109/59.871739
+[CarArr2006]: https://doi.org/10.1109/TPWRS.2006.876672
+[DamKucRajAta2016]: https://doi.org/10.1007/s10107-015-0919-9
+[Gar1962]: https://doi.org/10.1109/AIEEPAS.1962.4501405
+[KnuOstWat2018]: https://doi.org/10.1109/TPWRS.2017.2783850
+[MorLatRam2013]: https://doi.org/10.1109/TPWRS.2013.2251373
+[PanGua2016]: https://doi.org/10.1287/opre.2016.1520
+[XavQiuWanThi2019]: https://doi.org/10.1109/TPWRS.2019.2892620
+
 ### Authors
-* **Alinson Santos Xavier** (Argonne National Laboratory)
+* **Alinson S. Xavier** (Argonne National Laboratory)
+* **Aleksandr M. Kazachkov** (University of Florida)
 * **Feng Qiu** (Argonne National Laboratory)
 
 ### Acknowledgments
 
-* We would like to thank **Aleksandr M. Kazachkov** (University of Florida), **Yonghong Chen** (Midcontinent Independent System Operator), **Feng Pan** (Pacific Northwest National Laboratory) for valuable feedback on early versions of this package.
+* We would like to thank **Yonghong Chen** (Midcontinent Independent System Operator), **Feng Pan** (Pacific Northwest National Laboratory) for valuable feedback on early versions of this package.
 
 * Based upon work supported by **Laboratory Directed Research and Development** (LDRD) funding from Argonne National Laboratory, provided by the Director, Office of Science, of the U.S. Department of Energy under Contract No. DE-AC02-06CH11357
 
@@ -25,7 +35,7 @@
 
 If you use UnitCommitment.jl in your research (instances, models or algorithms), we kindly request that you cite the package as follows:
 
-* **Alinson S. Xavier, Feng Qiu**, "UnitCommitment.jl: A Julia/JuMP Optimization Package for Security-Constrained Unit Commitment". Zenodo (2020). [DOI: 10.5281/zenodo.4269874](https://doi.org/10.5281/zenodo.4269874).
+* **Alinson S. Xavier, Aleksandr M. Kazachkov, Feng Qiu**, "UnitCommitment.jl: A Julia/JuMP Optimization Package for Security-Constrained Unit Commitment". Zenodo (2020). [DOI: 10.5281/zenodo.4269874](https://doi.org/10.5281/zenodo.4269874).
 
 If you use the instances, we additionally request that you cite the original sources, as described in the [instances page](instances.md).
 

docs/instances.md

@@ -9,12 +9,11 @@ suffix: .
 Instances
 =========
 
-UnitCommitment.jl provides a large collection of benchmark instances collected
-from the literature and converted to a [common data format](format.md). In some cases, as indicated below, the original instances have been extended, with realistic parameters, using data-driven methods. 
-If you use these instances in your research, we request that you cite UnitCommitment.jl, as well as the original sources.
+UnitCommitment.jl provides a large collection of benchmark instances collected from the literature and converted to a [common data format](format.md). In some cases, as indicated below, the original instances have been extended, with realistic parameters, using data-driven methods. If you use these instances in your research, we request that you cite UnitCommitment.jl, as well as the original sources, as listed below. Benchmark instances can be loaded with `UnitCommitment.read_benchmark(name)`, as explained in the [usage section](usage.md).
 
-Raw instances files are [available at our GitHub repository](https://github.com/ANL-CEEESA/UnitCommitment.jl/tree/dev/instances). Benchmark instances can also be loaded with
-`UnitCommitment.read_benchmark(name)`, as explained in the [usage section](usage.md).
+```{warning}
+The instances included in UC.jl are still under development and may change in the future. If you use these instances in your research, for reproducibility, you should specify what version of UC.jl they came from.
+```
 
 
 MATPOWER
@@ -43,15 +42,10 @@ A variety of smaller IEEE test cases, [compiled by University of Washington](htt
 | Name | Buses | Generators | Lines | Contingencies | References |
 |------|-------|------------|-------|---------------|--------|
 | `matpower/case14/2017-02-01` | 14 | 5 | 20 | 19 | [MTPWR, PSTCA]
-| `matpower/case14/2017-08-01` | 14 | 5 | 20 | 19 | [MTPWR, PSTCA]
 | `matpower/case30/2017-02-01` | 30 | 6 | 41 | 38 | [MTPWR, PSTCA]
-| `matpower/case30/2017-08-01` | 30 | 6 | 41 | 38 | [MTPWR, PSTCA]
 | `matpower/case57/2017-02-01` | 57 | 7 | 80 | 79 | [MTPWR, PSTCA]
-| `matpower/case57/2017-08-01` | 57 | 7 | 80 | 79 | [MTPWR, PSTCA]
 | `matpower/case118/2017-02-01` | 118 | 54 | 186 | 177 | [MTPWR, PSTCA]
-| `matpower/case118/2017-08-01` | 118 | 54 | 186 | 177 | [MTPWR, PSTCA]
 | `matpower/case300/2017-02-01` | 300 | 69 | 411 | 320 | [MTPWR, PSTCA]
-| `matpower/case300/2017-08-01` | 300 | 69 | 411 | 320 | [MTPWR, PSTCA]
 
 
 ### MATPOWER/Polish
@@ -61,21 +55,13 @@ Test cases based on the Polish 400, 220 and 110 kV networks, originally provided
 | Name | Buses | Generators | Lines | Contingencies | References |
 |------|-------|------------|-------|---------------|--------|
 | `matpower/case2383wp/2017-02-01` | 2383 | 323 | 2896 | 2240 | [MTPWR]
-| `matpower/case2383wp/2017-08-01` | 2383 | 323 | 2896 | 2240 | [MTPWR]
 | `matpower/case2736sp/2017-02-01` | 2736 | 289 | 3504 | 3159 | [MTPWR]
-| `matpower/case2736sp/2017-08-01` | 2736 | 289 | 3504 | 3159 | [MTPWR]
 | `matpower/case2737sop/2017-02-01` | 2737 | 267 | 3506 | 3161 | [MTPWR]
-| `matpower/case2737sop/2017-08-01` | 2737 | 267 | 3506 | 3161 | [MTPWR]
 | `matpower/case2746wop/2017-02-01` | 2746 | 443 | 3514 | 3155 | [MTPWR]
-| `matpower/case2746wop/2017-08-01` | 2746 | 443 | 3514 | 3155 | [MTPWR]
 | `matpower/case2746wp/2017-02-01` | 2746 | 457 | 3514 | 3156 | [MTPWR]
-| `matpower/case2746wp/2017-08-01` | 2746 | 457 | 3514 | 3156 | [MTPWR]
 | `matpower/case3012wp/2017-02-01` | 3012 | 496 | 3572 | 2854 | [MTPWR]
-| `matpower/case3012wp/2017-08-01` | 3012 | 496 | 3572 | 2854 | [MTPWR]
 | `matpower/case3120sp/2017-02-01` | 3120 | 483 | 3693 | 2950 | [MTPWR]
-| `matpower/case3120sp/2017-08-01` | 3120 | 483 | 3693 | 2950 | [MTPWR]
 | `matpower/case3375wp/2017-02-01` | 3374 | 590 | 4161 | 3245 | [MTPWR]
-| `matpower/case3375wp/2017-08-01` | 3374 | 590 | 4161 | 3245 | [MTPWR]
 
 ### MATPOWER/PEGASE
 
@@ -84,15 +70,10 @@ Test cases from the [Pan European Grid Advanced Simulation and State Estimation
 | Name | Buses | Generators | Lines | Contingencies | References |
 |------|-------|------------|-------|---------------|--------|
 | `matpower/case89pegase/2017-02-01` | 89 | 12 | 210 | 192 | [JoFlMa16, FlPaCa13, MTPWR]
-| `matpower/case89pegase/2017-08-01` | 89 | 12 | 210 | 192 | [JoFlMa16, FlPaCa13, MTPWR]
 | `matpower/case1354pegase/2017-02-01` | 1354 | 260 | 1991 | 1288 | [JoFlMa16, FlPaCa13, MTPWR]
-| `matpower/case1354pegase/2017-08-01` | 1354 | 260 | 1991 | 1288 | [JoFlMa16, FlPaCa13, MTPWR]
 | `matpower/case2869pegase/2017-02-01` | 2869 | 510 | 4582 | 3579 | [JoFlMa16, FlPaCa13, MTPWR]
-| `matpower/case2869pegase/2017-08-01` | 2869 | 510 | 4582 | 3579 | [JoFlMa16, FlPaCa13, MTPWR]
 | `matpower/case9241pegase/2017-02-01` | 9241 | 1445 | 16049 | 13932 | [JoFlMa16, FlPaCa13, MTPWR]
-| `matpower/case9241pegase/2017-08-01` | 9241 | 1445 | 16049 | 13932 | [JoFlMa16, FlPaCa13, MTPWR]
 | `matpower/case13659pegase/2017-02-01` | 13659 | 4092 | 20467 | 13932 | [JoFlMa16, FlPaCa13, MTPWR]
-| `matpower/case13659pegase/2017-08-01` | 13659 | 4092 | 20467 | 13932 | [JoFlMa16, FlPaCa13, MTPWR]
 
 ### MATPOWER/RTE
 
@@ -101,21 +82,13 @@ Test cases from the R&D Division at [Reseau de Transport d'Electricite](https://
 | Name | Buses | Generators | Lines | Contingencies | References |
 |------|-------|------------|-------|---------------|--------|
 | `matpower/case1888rte/2017-02-01` | 1888 | 296 | 2531 | 1484 | [MTPWR, JoFlMa16]
-| `matpower/case1888rte/2017-08-01` | 1888 | 296 | 2531 | 1484 | [MTPWR, JoFlMa16]
 | `matpower/case1951rte/2017-02-01` | 1951 | 390 | 2596 | 1497 | [MTPWR, JoFlMa16]
-| `matpower/case1951rte/2017-08-01` | 1951 | 390 | 2596 | 1497 | [MTPWR, JoFlMa16]
 | `matpower/case2848rte/2017-02-01` | 2848 | 544 | 3776 | 2242 | [MTPWR, JoFlMa16]
-| `matpower/case2848rte/2017-08-01` | 2848 | 544 | 3776 | 2242 | [MTPWR, JoFlMa16]
 | `matpower/case2868rte/2017-02-01` | 2868 | 596 | 3808 | 2260 | [MTPWR, JoFlMa16]
-| `matpower/case2868rte/2017-08-01` | 2868 | 596 | 3808 | 2260 | [MTPWR, JoFlMa16]
 | `matpower/case6468rte/2017-02-01` | 6468 | 1262 | 9000 | 6094 | [MTPWR, JoFlMa16]
-| `matpower/case6468rte/2017-08-01` | 6468 | 1262 | 9000 | 6094 | [MTPWR, JoFlMa16]
 | `matpower/case6470rte/2017-02-01` | 6470 | 1306 | 9005 | 6085 | [MTPWR, JoFlMa16]
-| `matpower/case6470rte/2017-08-01` | 6470 | 1306 | 9005 | 6085 | [MTPWR, JoFlMa16]
 | `matpower/case6495rte/2017-02-01` | 6495 | 1352 | 9019 | 6060 | [MTPWR, JoFlMa16]
-| `matpower/case6495rte/2017-08-01` | 6495 | 1352 | 9019 | 6060 | [MTPWR, JoFlMa16]
 | `matpower/case6515rte/2017-02-01` | 6515 | 1368 | 9037 | 6063 | [MTPWR, JoFlMa16]
-| `matpower/case6515rte/2017-08-01` | 6515 | 1368 | 9037 | 6063 | [MTPWR, JoFlMa16]
 
 
 PGLIB-UC Instances
@@ -315,7 +288,7 @@ Tejada19
 References
 ----------
 
-* [UCJL] **Alinson S. Xavier, Feng Qiu.** "UnitCommitment.jl: A Julia/JuMP Optimization Package for Security-Constrained Unit Commitment". Zenodo (2020). [DOI: 10.5281/zenodo.4269874](https://doi.org/10.5281/zenodo.4269874)
+* [UCJL] **Alinson S. Xavier, Aleksandr M. Kazachkov, Feng Qiu.** "UnitCommitment.jl: A Julia/JuMP Optimization Package for Security-Constrained Unit Commitment". Zenodo (2020). [DOI: 10.5281/zenodo.4269874](https://doi.org/10.5281/zenodo.4269874)
 
 * [KnOsWa20] **Bernard Knueven, James Ostrowski and Jean-Paul Watson.** "On Mixed-Integer Programming Formulations for the Unit Commitment Problem". INFORMS Journal on Computing (2020). [DOI: 10.1287/ijoc.2019.0944](https://doi.org/10.1287/ijoc.2019.0944)
 

docs/model.md

@@ -23,7 +23,7 @@ Name | Symbol | Description | Unit
 `switch_off[g,t]` | $w_{g}(t)$ | True if generator `g` switches off at time `t`. | Binary
 `prod_above[g,t]` |$p'_{g}(t)$ | Amount of power produced by generator `g` above its minimum power output at time `t`. For example, if the minimum power of generator `g` is 100 MW and `g` is producing 115 MW of power at time `t`, then `prod_above[g,t]` equals `15.0`. | MW
 `segprod[g,t,k]` | $p^k_g(t)$ | Amount of power from piecewise linear segment `k` produced by generator `g` at time `t`. For example, if cost curve for generator `g` is defined by the points `(100, 1400)`, `(110, 1600)`, `(130, 2200)` and `(135, 2400)`, and if the generator is producing 115 MW of power at time `t`, then `segprod[g,t,:]` equals `[10.0, 5.0, 0.0]`.| MW
-`reserve[g,t]` | $r_g(t)$ | Amount of reserves provided by generator `g` at time `t`. | MW
+`reserve[r,g,t]` | $r_g(t)$ | Amount of reserve `r` provided by unit `g` at time `t`. | MW
 `startup[g,t,s]` | $\delta^s_g(t)$ | True if generator `g` switches on at time `t` incurring start-up costs from start-up category `s`. | Binary
 
 
@@ -148,7 +148,7 @@ for g in instance.units
 end
 ```
 
-### Modifying the model
+### Fixing variables, modifying objective function and adding constraints
 
 Since we now have a direct reference to the JuMP decision variables, it is possible to fix variables, change the coefficients in the objective function, or even add new constraints to the model before solving it. The script below shows how can this be accomplished. For more information on modifying an existing model, [see the JuMP documentation](https://jump.dev/JuMP.jl/stable/manual/variables/).
 
@@ -190,6 +190,54 @@ JuMP.set_objective_coefficient(
 UnitCommitment.optimize!(model)
 ```
 
+### Adding new component to a bus
+
+The following snippet shows how to add a new grid component to a particular bus. For each time step, we create decision variables for the new grid component, add these variables to the objective function, then attach the component to a particular bus by modifying some existing model constraints. 
+
+```julia
+using Cbc
+using JuMP
+using UnitCommitment
+
+# Load instance and build base model
+instance = UnitCommitment.read_benchmark("matpower/case118/2017-02-01")
+model = UnitCommitment.build_model(
+    instance=instance,
+    optimizer=Cbc.Optimizer,
+)
+
+# Get the number of time steps in the original instance
+T = instance.time
+
+# Create decision variables for the new grid component.
+# In this example, we assume that the new component can
+# inject up to 10 MW of power at each time step, so we
+# create new continuous variables 0 ≤ x[t] ≤ 10.
+@variable(model, x[1:T], lower_bound=0.0, upper_bound=10.0)
+
+# For each time step
+for t in 1:T
+
+    # Add production costs to the objective function.
+    # In this example, we assume a cost of $5/MW.
+    set_objective_coefficient(model, x[t], 5.0)
+
+    # Attach the new component to bus b1, by modifying the
+    # constraint `eq_net_injection`.
+    set_normalized_coefficient(
+        model[:eq_net_injection]["b1", t],
+        x[t],
+        1.0,
+    )
+end
+
+# Solve the model
+UnitCommitment.optimize!(model)
+
+# Show optimal values for the x variables
+@show value.(x)
+```
+
 References
 ----------
 * [KnOsWa20] **Bernard Knueven, James Ostrowski and Jean-Paul Watson.** "On Mixed-Integer Programming Formulations for the Unit Commitment Problem". INFORMS Journal on Computing (2020). [DOI: 10.1287/ijoc.2019.0944](https://doi.org/10.1287/ijoc.2019.0944)

docs/usage.md

@@ -33,35 +33,33 @@ Typical Usage
 
 ### Solving user-provided instances
 
-The first step to use UC.jl is to construct a JSON file describing your unit commitment instance. See the [data format page]() for a complete description of the data format UC.jl expects. The next steps, as shown below, are to read the instance from file, construct the optimization model, run the optimization and extract the optimal solution.
+The first step to use UC.jl is to construct a JSON file describing your unit commitment instance. See [Data Format](format.md) for a complete description of the data format UC.jl expects. The next steps, as shown below, are to: (1) read the instance from file; (2) construct the optimization model; (3) run the optimization; and (4) extract the optimal solution.
 
 ```julia
 using Cbc
 using JSON
 using UnitCommitment
 
-# Read instance
+# 1. Read instance
 instance = UnitCommitment.read("/path/to/input.json")
 
-# Construct optimization model
+# 2. Construct optimization model
 model = UnitCommitment.build_model(
     instance=instance,
     optimizer=Cbc.Optimizer,
 )
 
-# Solve model
+# 3. Solve model
 UnitCommitment.optimize!(model)
 
-# Extract solution
+# 4. Write solution to a file
 solution = UnitCommitment.solution(model)
-
-# Write solution to a file
 UnitCommitment.write("/path/to/output.json", solution)
 ```
 
 ### Solving benchmark instances
 
-As described in the [Instances page](instances.md), UnitCommitment.jl contains a number of benchmark instances collected from the literature. To solve one of these instances individually, instead of constructing your own, the function `read_benchmark` can be used:
+UnitCommitment.jl contains a large number of benchmark instances collected from the literature and converted into a common data format. To solve one of these instances individually, instead of constructing your own, the function `read_benchmark` can be used, as shown below. See [Instances](instances.md) for the complete list of available instances.
 
 ```julia
 using UnitCommitment
@@ -71,10 +69,38 @@ instance = UnitCommitment.read_benchmark("matpower/case3375wp/2017-02-01")
 Advanced usage
 --------------
 
+### Customizing the formulation
 
-### Modifying the formulation
+By default, `build_model` uses a formulation that combines modeling components from different publications, and that has been carefully tested, using our own benchmark scripts, to provide good performance across a wide variety of instances. This default formulation is expected to change over time, as new methods are proposed in the literature. You can, however, construct your own formulation, based on the modeling components that you choose, as shown in the next example.
 
-For the time being, the recommended way of modifying the MILP formulation used by UC.jl is to create a local copy of our git repository and directly modify the source code of the package. In a future version, it will be possible to switch between multiple formulations, or to simply add/remove constraints after the model has been generated.
+```julia
+using Cbc
+using UnitCommitment
+
+import UnitCommitment:
+    Formulation,
+    KnuOstWat2018,
+    MorLatRam2013,
+    ShiftFactorsFormulation
+
+instance = UnitCommitment.read_benchmark(
+    "matpower/case118/2017-02-01",
+)
+
+model = UnitCommitment.build_model(
+    instance = instance,
+    optimizer = Cbc.Optimizer,
+    formulation = Formulation(
+        pwl_costs = KnuOstWat2018.PwlCosts(),
+        ramping = MorLatRam2013.Ramping(),
+        startup_costs = MorLatRam2013.StartupCosts(),
+        transmission = ShiftFactorsFormulation(
+            isf_cutoff = 0.005,
+            lodf_cutoff = 0.001,
+        ),
+    ),
+)
+```
 
 ### Generating initial conditions
 
@@ -93,7 +119,10 @@ instance = UnitCommitment.read("instance.json")
 UnitCommitment.generate_initial_conditions!(instance, Cbc.Optimizer)
 
 # Construct and solve optimization model
-model = UnitCommitment.build_model(instance, Cbc.Optimizer)
+model = UnitCommitment.build_model(
+    instance=instance,
+    optimizer=Cbc.Optimizer,
+)
 UnitCommitment.optimize!(model)
 ```
 

instances/README.md

@@ -1,53 +0,0 @@
-Instances
-=========
-
-UnitCommitment.jl provides a large collection of benchmark instances collected
-from the literature and converted to a common data format. If you use these instances in your research, we request that you cite UnitCommitment.jl, as well as the original sources, as listed below. [See documentation for more details](https://anl-ceeesa.github.io/UnitCommitment.jl/).
-
-References
-----------
-
-### UnitCommitment.jl
-
-* [UCJL] **Alinson S. Xavier, Feng Qiu.** "UnitCommitment.jl: A Julia/JuMP Optimization Package for Security-Constrained Unit Commitment". Zenodo (2020). [DOI: 10.5281/zenodo.4269874](https://doi.org/10.5281/zenodo.4269874)
-
-
-### MATPOWER
-
-* [MTPWR] **D. Zimmerman, C. E. Murillo-Sandnchez and R. J. Thomas.** "Matpower:  Steady-state  operations,  planning,  and  analysis  tools  forpower systems research and education", IEEE Transactions on PowerSystems, vol. 26, no. 1, pp. 12 –19, Feb. 2011. [DOI: 10.1109/TPWRS.2010.2051168](https://doi.org/10.1109/TPWRS.2010.2051168)
-
-* [PSTCA] **University of Washington, Dept. of Electrical Engineering.** "Power Systems Test Case Archive". Available at: <http://www.ee.washington.edu/research/pstca/> (Accessed: Nov 14, 2020)
-
-* [JoFlMa16] **C. Josz, S. Fliscounakis, J. Maeght, and P. Panciatici.** "AC Power Flow
-Data in MATPOWER and QCQP Format: iTesla, RTE Snapshots, and PEGASE". [ArXiv (2016)](https://arxiv.org/abs/1603.01533).
-
-* [FlPaCa13] **S. Fliscounakis, P. Panciatici, F. Capitanescu, and L. Wehenkel.**
-"Contingency ranking with respect to overloads in very large power
-systems taking into account uncertainty, preventive and corrective
-actions", Power Systems, IEEE Trans. on, (28)4:4909-4917, 2013.
-[DOI: 10.1109/TPWRS.2013.2251015](https://doi.org/10.1109/TPWRS.2013.2251015)
-
-
-### PGLIB-UC
-
-* [PGLIB] **Carleton Coffrin and Bernard Knueven.** "Power Grid Lib - Unit Commitment". Available at: <https://github.com/power-grid-lib/pglib-uc> (Accessed: Nov 14, 2020)
-
-* [KrHiOn12] **Eric Krall, Michael Higgins and Richard P. O’Neill.** "RTO unit commitment test system." Federal Energy Regulatory Commission. Available at: <https://www.ferc.gov/industries-data/electric/power-sales-and-markets/increasing-efficiency-through-improved-software-1> (Accessed: Nov 14, 2020)
-
-* [KnOsWa20] **Bernard Knueven, James Ostrowski and Jean-Paul Watson.** "On Mixed-Integer Programming Formulations for the Unit Commitment Problem". INFORMS Journal on Computing (2020). [DOI: 10.1287/ijoc.2019.0944](https://doi.org/10.1287/ijoc.2019.0944)
-
-### RTS-GMLC
-
-* https://github.com/GridMod/RTS-GMLC
-
-* [BaBlEh19] **Clayton Barrows, Aaron Bloom, Ali Ehlen, Jussi Ikaheimo, Jennie Jorgenson, Dheepak Krishnamurthy, Jessica Lau et al.** "The IEEE Reliability Test System: A Proposed 2019 Update." IEEE Transactions on Power Systems (2019). [DOI: 10.1109/TPWRS.2019.2925557](https://doi.org/10.1109/TPWRS.2019.2925557)
-
-### OR-LIB
-
-* [ORLIB] **J.E.Beasley.** "OR-Library: distributing test problems by electronic mail", Journal of the Operational Research Society 41(11) (1990). [DOI: 10.2307/2582903](https://doi.org/10.2307/2582903)
-
-* [FrGe06] **A. Frangioni, C. Gentile.** "Solving nonlinear single-unit commitment problems with ramping constraints" Operations Research 54(4), p. 767 - 775, 2006. [DOI: 10.1287/opre.1060.0309](https://doi.org/10.1287/opre.1060.0309)
-
-### Tejada19
-
-* [TeLuSa19] **D. A. Tejada-Arango, S. Lumbreras, P. Sanchez-Martin and A. Ramos.** "Which Unit-Commitment Formulation is Best? A Systematic Comparison," in IEEE Transactions on Power Systems. [DOI: 10.1109/TPWRS.2019.2962024](https://ieeexplore.ieee.org/document/8941313/).