11 Commits

13 changed files with 734 additions and 36 deletions

15
.github/workflows/tagbot.yml vendored Normal file
View File

@@ -0,0 +1,15 @@
name: TagBot
on:
issue_comment:
types:
- created
workflow_dispatch:
jobs:
TagBot:
if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot'
runs-on: ubuntu-latest
steps:
- uses: JuliaRegistries/TagBot@v1
with:
token: ${{ secrets.GITHUB_TOKEN }}
ssh: ${{ secrets.DOCUMENTER_KEY }}

3
.gitignore vendored
View File

@@ -9,3 +9,6 @@ notebooks
.idea
*.lp
Manifest.toml
data
build
benchmark

View File

@@ -1,4 +1,4 @@
# Version 0.5.0 (TBD)
# Version 0.5.0 (Jan 6, 2021)
- Allow plants to store input material for processing in later years

View File

@@ -4,11 +4,13 @@ authors = ["Alinson S Xavier <axavier@anl.gov>"]
version = "0.5.0"
[deps]
CRC = "44b605c4-b955-5f2b-9b6d-d2bd01d3d205"
CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
Cbc = "9961bab8-2fa3-5c5a-9d89-47fab24efd76"
Clp = "e2554f3b-3117-50c0-817c-e040a3ddf72d"
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
GZip = "92fee26a-97fe-5a0c-ad85-20a5f3185b63"
Geodesy = "0ef565a4-170c-5f04-8de2-149903a85f3d"
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
@@ -16,13 +18,17 @@ JSONSchema = "7d188eb4-7ad8-530c-ae41-71a32a6d4692"
JuMP = "4076af6c-e467-56ae-b986-b466b2749572"
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
MathOptInterface = "b8f27783-ece8-5eb3-8dc8-9495eed66fee"
OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
PackageCompiler = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d"
Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
ProgressBars = "49802e3a-d2f1-5c88-81d8-b72133a6f568"
Shapefile = "8e980c4a-a4fe-5da2-b3a7-4b4b0353a2f4"
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
ZipFile = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea"
[compat]
CRC = "4"
CSV = "0.7"
Cbc = "0.6"
Clp = "0.8"
@@ -31,9 +37,11 @@ DataStructures = "0.17"
GZip = "0.5"
Geodesy = "0.5"
JSON = "0.21"
JSONSchema = "0.2"
JSONSchema = "0.3"
JuMP = "0.21"
MathOptInterface = "0.9"
PackageCompiler = "1"
ProgressBars = "0.6"
Shapefile = "0.7"
ZipFile = "0.9"
julia = "1"

348
instances/s2.json Normal file
View File

@@ -0,0 +1,348 @@
{
"parameters": {
"time horizon (years)": 2
},
"products": {
"P1": {
"transportation cost ($/km/tonne)": [
0.015,
0.015
],
"transportation energy (J/km/tonne)": [
0.12,
0.11
],
"transportation emissions (tonne/km/tonne)": {
"CO2": [
0.052,
0.050
],
"CH4": [
0.003,
0.002
]
},
"initial amounts": {
"C1": {
"location": "2018-us-county:17043",
"amount (tonne)": [
934.56,
934.56
]
},
"C2": {
"latitude (deg)": 7.0,
"longitude (deg)": 19.0,
"amount (tonne)": [
198.95,
198.95
]
},
"C3": {
"latitude (deg)": 84.0,
"longitude (deg)": 76.0,
"amount (tonne)": [
212.97,
212.97
]
},
"C4": {
"latitude (deg)": 21.0,
"longitude (deg)": 16.0,
"amount (tonne)": [
352.19,
352.19
]
},
"C5": {
"latitude (deg)": 32.0,
"longitude (deg)": 92.0,
"amount (tonne)": [
510.33,
510.33
]
},
"C6": {
"latitude (deg)": 14.0,
"longitude (deg)": 62.0,
"amount (tonne)": [
471.66,
471.66
]
},
"C7": {
"latitude (deg)": 30.0,
"longitude (deg)": 83.0,
"amount (tonne)": [
785.21,
785.21
]
},
"C8": {
"latitude (deg)": 35.0,
"longitude (deg)": 40.0,
"amount (tonne)": [
706.17,
706.17
]
},
"C9": {
"latitude (deg)": 74.0,
"longitude (deg)": 52.0,
"amount (tonne)": [
30.08,
30.08
]
},
"C10": {
"latitude (deg)": 22.0,
"longitude (deg)": 54.0,
"amount (tonne)": [
536.52,
536.52
]
}
}
},
"P2": {
"transportation cost ($/km/tonne)": [
0.02,
0.02
]
},
"P3": {
"transportation cost ($/km/tonne)": [
0.0125,
0.0125
]
},
"P4": {
"transportation cost ($/km/tonne)": [
0.0175,
0.0175
]
}
},
"plants": {
"F1": {
"input": "P1",
"outputs (tonne/tonne)": {
"P2": 0.2,
"P3": 0.5
},
"energy (GJ/tonne)": [
0.12,
0.11
],
"emissions (tonne/tonne)": {
"CO2": [
0.052,
0.050
],
"CH4": [
0.003,
0.002
]
},
"locations": {
"L1": {
"latitude (deg)": 0.0,
"longitude (deg)": 0.0,
"disposal": {
"P2": {
"cost ($/tonne)": [
-10.0,
-10.0
],
"limit (tonne)": [
1.0,
1.0
]
},
"P3": {
"cost ($/tonne)": [
-10.0,
-10.0
],
"limit (tonne)": [
1.0,
1.0
]
}
},
"capacities (tonne)": {
"250.0": {
"opening cost ($)": [
500.0,
500.0
],
"fixed operating cost ($)": [
30.0,
30.0
],
"variable operating cost ($/tonne)": [
30.0,
30.0
]
},
"1000.0": {
"opening cost ($)": [
1250.0,
1250.0
],
"fixed operating cost ($)": [
30.0,
30.0
],
"variable operating cost ($/tonne)": [
30.0,
30.0
]
}
}
},
"L2": {
"latitude (deg)": 0.5,
"longitude (deg)": 0.5,
"capacities (tonne)": {
"0.0": {
"opening cost ($)": [
1000,
1000
],
"fixed operating cost ($)": [
50.0,
50.0
],
"variable operating cost ($/tonne)": [
50.0,
50.0
]
},
"10000.0": {
"opening cost ($)": [
10000,
10000
],
"fixed operating cost ($)": [
50.0,
50.0
],
"variable operating cost ($/tonne)": [
50.0,
50.0
]
}
}
}
}
},
"F2": {
"input": "P2",
"outputs (tonne/tonne)": {
"P3": 0.05,
"P4": 0.80
},
"locations": {
"L3": {
"latitude (deg)": 25.0,
"longitude (deg)": 65.0,
"disposal": {
"P3": {
"cost ($/tonne)": [
100.0,
100.0
]
}
},
"capacities (tonne)": {
"1000.0": {
"opening cost ($)": [
3000,
3000
],
"fixed operating cost ($)": [
50.0,
50.0
],
"variable operating cost ($/tonne)": [
50.0,
50.0
]
}
}
},
"L4": {
"latitude (deg)": 0.75,
"longitude (deg)": 0.20,
"capacities (tonne)": {
"10000": {
"opening cost ($)": [
3000,
3000
],
"fixed operating cost ($)": [
50.0,
50.0
],
"variable operating cost ($/tonne)": [
50.0,
50.0
]
}
}
}
}
},
"F3": {
"input": "P4",
"locations": {
"L5": {
"latitude (deg)": 100.0,
"longitude (deg)": 100.0,
"capacities (tonne)": {
"15000": {
"opening cost ($)": [
0.0,
0.0
],
"fixed operating cost ($)": [
0.0,
0.0
],
"variable operating cost ($/tonne)": [
-15.0,
-15.0
]
}
}
}
}
},
"F4": {
"input": "P3",
"locations": {
"L6": {
"latitude (deg)": 50.0,
"longitude (deg)": 50.0,
"capacities (tonne)": {
"10000": {
"opening cost ($)": [
0.0,
0.0
],
"fixed operating cost ($)": [
0.0,
0.0
],
"variable operating cost ($/tonne)": [
-15.0,
-15.0
]
}
}
}
}
}
}
}

View File

@@ -11,6 +11,7 @@ include("graph/structs.jl")
include("graph/build.jl")
include("graph/csv.jl")
include("instance/compress.jl")
include("instance/geodb.jl")
include("instance/parse.jl")
include("instance/validate.jl")
include("model/build.jl")

229
src/instance/geodb.jl Normal file
View File

@@ -0,0 +1,229 @@
# RELOG: Reverse Logistics Optimization
# Copyright (C) 2020, UChicago Argonne, LLC. All rights reserved.
# Released under the modified BSD license. See COPYING.md for more details.
using CRC
using CSV
using DataFrames
using Shapefile
using Statistics
using ZipFile
using ProgressBars
using OrderedCollections
import Downloads: download
import Base: parse
crc32 = crc(CRC_32)
struct GeoPoint
lat::Float64
lon::Float64
end
struct GeoRegion
centroid::GeoPoint
population::Int
GeoRegion(; centroid, population) = new(centroid, population)
end
DB_CACHE = Dict{String,Dict{String,GeoRegion}}()
function centroid(geom::Shapefile.Polygon)::GeoPoint
x_max, x_min, y_max, y_min = -Inf, Inf, -Inf, Inf
for p in geom.points
x_max = max(x_max, p.x)
x_min = min(x_min, p.x)
y_max = max(y_max, p.y)
y_min = min(y_min, p.y)
end
x_center = (x_max + x_min) / 2.0
y_center = (y_max + y_min) / 2.0
return GeoPoint(round(y_center, digits = 5), round(x_center, digits = 5))
end
function _download(url, output, expected_crc32)::Nothing
if isfile(output)
return
end
mkpath(dirname(output))
@info "Downloading: $url"
fname = download(url)
actual_crc32 = open(crc32, fname)
expected_crc32 == actual_crc32 || error("CRC32 mismatch")
cp(fname, output)
return
end
function _download_zip(url, outputdir, expected_crc32)::Nothing
if isdir(outputdir)
return
end
mkpath(outputdir)
@info "Downloading: $url"
zip_filename = _download(url)
actual_crc32 = open(crc32, zip_filename)
expected_crc32 == actual_crc32 || error("CRC32 mismatch")
open(zip_filename) do zip_file
zr = ZipFile.Reader(zip_file)
for file in zr.files
open(joinpath(outputdir, file.name), "w") do output_file
write(output_file, read(file))
end
end
end
return
end
function _geodb_load_gov_census(;
db_name,
extract_cols,
shp_crc32,
shp_filename,
shp_url,
population_url,
population_crc32,
population_col,
population_preprocess,
population_join,
)::Dict{String,GeoRegion}
basedir = joinpath(dirname(@__FILE__), "..", "..", "data", db_name)
csv_filename = "$basedir/locations.csv"
if !isfile(csv_filename)
# Download required files
_download(population_url, "$basedir/population.csv", population_crc32)
_download_zip(shp_url, basedir, shp_crc32)
# Read shapefile
@info "Processing: $shp_filename"
table = Shapefile.Table(joinpath(basedir, shp_filename))
geoms = Shapefile.shapes(table)
# Build empty dataframe
df = DataFrame()
cols = extract_cols(table, 1)
for k in keys(cols)
df[!, k] = []
end
df[!, "latitude"] = Float64[]
df[!, "longitude"] = Float64[]
# Add regions to dataframe
for (i, geom) in tqdm(enumerate(geoms))
c = centroid(geom)
cols = extract_cols(table, i)
push!(df, [values(cols)..., c.lat, c.lon])
end
sort!(df)
# Join with population data
population = DataFrame(CSV.File("$basedir/population.csv"))
population_preprocess(population)
population = population[:, [population_join, population_col]]
rename!(population, population_col => "population")
df = leftjoin(df, population, on = population_join)
# Write output
CSV.write(csv_filename, df)
end
if db_name keys(DB_CACHE)
csv = CSV.File(csv_filename)
DB_CACHE[db_name] = Dict(
string(row.id) => GeoRegion(
centroid = GeoPoint(row.latitude, row.longitude),
population = (row.population === missing ? 0 : row.population),
) for row in csv
)
end
return DB_CACHE[db_name]
end
# 2018 US counties
# -----------------------------------------------------------------------------
function _extract_cols_2018_us_county(table::Shapefile.Table, i::Int)::OrderedDict{String,Any}
return OrderedDict(
"id" => table.STATEFP[i] * table.COUNTYFP[i],
"statefp" => table.STATEFP[i],
"countyfp" => table.COUNTYFP[i],
"name" => table.NAME[i]
)
end
function _population_preprocess_2018_us_county(df)
df[!, "id"] = [@sprintf("%02d%03d", row.STATE, row.COUNTY) for row in eachrow(df)]
end
function _geodb_load_2018_us_county()::Dict{String,GeoRegion}
return _geodb_load_gov_census(
db_name = "2018-us-county",
extract_cols = _extract_cols_2018_us_county,
shp_crc32 = 0x83eaec6d,
shp_filename = "cb_2018_us_county_500k.shp",
shp_url = "https://www2.census.gov/geo/tiger/GENZ2018/shp/cb_2018_us_county_500k.zip",
population_url = "https://www2.census.gov/programs-surveys/popest/datasets/2010-2019/counties/totals/co-est2019-alldata.csv",
population_crc32 = 0xf85b0405,
population_col = "POPESTIMATE2019",
population_join = "id",
population_preprocess = _population_preprocess_2018_us_county,
)
end
# # 2018 US ZIP codes
# # -----------------------------------------------------------------------------
# function _extract_cols_2018_us_zcta(table::Shapefile.Table, i::Int)::OrderedDict{String,Any}
# return OrderedDict("id" => table.ZCTA5CE10[i])
# end
# function _geodb_load_2018_us_zcta()::Dict{String,GeoRegion}
# return _geodb_load_gov_census(
# db_name = "2018-us-zcta",
# extract_cols = _extract_cols_2018_us_zcta,
# shp_crc32 = 0x6391f5fc,
# shp_filename = "cb_2018_us_zcta510_500k.shp",
# shp_url = "https://www2.census.gov/geo/tiger/GENZ2018/shp/cb_2018_us_zcta510_500k.zip",
# population_url = "http://www2.census.gov/programs-surveys/popest/datasets/2010-2019/national/totals/nst-est2019-alldata.csv",
# population_crc32 = 0x191cc64c,
# population_col = "POPESTIMATE2019",
# )
# end
# US States
# -----------------------------------------------------------------------------
function _extract_cols_us_state(table::Shapefile.Table, i::Int)::OrderedDict{String,Any}
return OrderedDict(
"id" => table.STUSPS[i],
"statefp" => parse(Int, table.STATEFP[i]),
"name" => table.NAME[i],
)
end
function _population_preprocess_us_state(df)
rename!(df, "STATE" => "statefp")
end
function _geodb_load_us_state()::Dict{String,GeoRegion}
return _geodb_load_gov_census(
db_name = "us-state",
extract_cols = _extract_cols_us_state,
shp_crc32 = 0x9469e5ca,
shp_filename = "cb_2018_us_state_500k.shp",
shp_url = "https://www2.census.gov/geo/tiger/GENZ2018/shp/cb_2018_us_state_500k.zip",
population_url = "http://www2.census.gov/programs-surveys/popest/datasets/2010-2019/national/totals/nst-est2019-alldata.csv",
population_crc32 = 0x191cc64c,
population_col = "POPESTIMATE2019",
population_join = "statefp",
population_preprocess = _population_preprocess_us_state,
)
end
function geodb_load(db_name::AbstractString)::Dict{String,GeoRegion}
db_name == "2018-us-county" && return _geodb_load_2018_us_county()
db_name == "2018-us-zcta" && return _geodb_load_2018_us_zcta()
db_name == "us-state" && return _geodb_load_us_state()
error("Unknown database: $db_name")
end
function geodb_query(name)::GeoRegion
db_name, id = split(name, ":")
return geodb_load(db_name)[id]
end

View File

@@ -53,6 +53,11 @@ function parse(json)::Instance
# Create collection centers
if "initial amounts" in keys(product_dict)
for (center_name, center_dict) in product_dict["initial amounts"]
if "location" in keys(center_dict)
region = geodb_query(center_dict["location"])
center_dict["latitude (deg)"] = region.centroid.lat
center_dict["longitude (deg)"] = region.centroid.lon
end
center = CollectionCenter(
length(collection_centers) + 1,
center_name,

View File

@@ -4,6 +4,7 @@
using DataFrames
using CSV
import Base: write
function write(solution::AbstractDict, filename::AbstractString)
@info "Writing solution: $filename"

View File

@@ -12,7 +12,9 @@
"Parameters": {
"type": "object",
"properties": {
"time horizon (years)": { "type": "number" }
"time horizon (years)": {
"type": "number"
}
},
"required": [
"time horizon (years)"
@@ -23,17 +25,27 @@
"additionalProperties": {
"type": "object",
"properties": {
"input": { "type": "string" },
"input": {
"type": "string"
},
"outputs (tonne/tonne)": {
"type": "object",
"additionalProperties": { "type": "number" }
"additionalProperties": {
"type": "number"
}
},
"energy (GJ/tonne)": {
"$ref": "#/definitions/TimeSeries"
},
"energy (GJ/tonne)": { "$ref": "#/definitions/TimeSeries" },
"emissions (tonne/tonne)": {
"type": "object",
"additionalProperties": { "$ref": "#/definitions/TimeSeries" }
"additionalProperties": {
"$ref": "#/definitions/TimeSeries"
}
},
"locations": { "$ref": "#/definitions/PlantLocation" }
"locations": {
"$ref": "#/definitions/PlantLocation"
}
},
"required": [
"input",
@@ -46,15 +58,26 @@
"additionalProperties": {
"type": "object",
"properties": {
"latitude (deg)": { "type": "number" },
"longitude (deg)": { "type": "number" },
"location": {
"type": "string"
},
"latitude (deg)": {
"type": "number"
},
"longitude (deg)": {
"type": "number"
},
"disposal": {
"type": "object",
"additionalProperties": {
"type": "object",
"properties": {
"cost ($/tonne)": { "$ref": "#/definitions/TimeSeries" },
"limit (tonne)": { "$ref": "#/definitions/TimeSeries" }
"cost ($/tonne)": {
"$ref": "#/definitions/TimeSeries"
},
"limit (tonne)": {
"$ref": "#/definitions/TimeSeries"
}
},
"required": [
"cost ($/tonne)"
@@ -64,22 +87,32 @@
"storage": {
"type": "object",
"properties": {
"cost ($/tonne)": { "$ref": "#/definitions/TimeSeries" },
"limit (tonne)": { "type": "number" }
"cost ($/tonne)": {
"$ref": "#/definitions/TimeSeries"
},
"limit (tonne)": {
"type": "number"
}
},
"required": [
"cost ($/tonne)",
"limit (tonne)"
]
},
},
"capacities (tonne)": {
"type": "object",
"additionalProperties": {
"type": "object",
"properties": {
"variable operating cost ($/tonne)": { "$ref": "#/definitions/TimeSeries" },
"fixed operating cost ($)": { "$ref": "#/definitions/TimeSeries" },
"opening cost ($)": { "$ref": "#/definitions/TimeSeries" }
"variable operating cost ($/tonne)": {
"$ref": "#/definitions/TimeSeries"
},
"fixed operating cost ($)": {
"$ref": "#/definitions/TimeSeries"
},
"opening cost ($)": {
"$ref": "#/definitions/TimeSeries"
}
},
"required": [
"variable operating cost ($/tonne)",
@@ -87,11 +120,9 @@
"opening cost ($)"
]
}
}
}
},
"required": [
"latitude (deg)",
"longitude (deg)",
"capacities (tonne)"
]
}
@@ -101,13 +132,20 @@
"additionalProperties": {
"type": "object",
"properties": {
"latitude (deg)": { "type": "number" },
"longitude (deg)": { "type": "number" },
"amount (tonne)": { "$ref": "#/definitions/TimeSeries" }
"location": {
"type": "string"
},
"latitude (deg)": {
"type": "number"
},
"longitude (deg)": {
"type": "number"
},
"amount (tonne)": {
"$ref": "#/definitions/TimeSeries"
}
},
"required": [
"latitude (deg)",
"longitude (deg)",
"amount (tonne)"
]
}
@@ -117,25 +155,39 @@
"additionalProperties": {
"type": "object",
"properties": {
"transportation cost ($/km/tonne)": { "$ref": "#/definitions/TimeSeries" },
"transportation energy (J/km/tonne)": { "$ref": "#/definitions/TimeSeries" },
"transportation cost ($/km/tonne)": {
"$ref": "#/definitions/TimeSeries"
},
"transportation energy (J/km/tonne)": {
"$ref": "#/definitions/TimeSeries"
},
"transportation emissions (tonne/km/tonne)": {
"type": "object",
"additionalProperties": { "$ref": "#/definitions/TimeSeries" }
"additionalProperties": {
"$ref": "#/definitions/TimeSeries"
}
},
"initial amounts": { "$ref": "#/definitions/InitialAmount" }
"initial amounts": {
"$ref": "#/definitions/InitialAmount"
}
},
"required": [
"transportation cost ($/km/tonne)"
]
}
}
}
},
"type": "object",
"properties": {
"parameters": { "$ref": "#/definitions/Parameters" },
"plants": { "$ref": "#/definitions/Plant" },
"products": { "$ref": "#/definitions/Product" }
"parameters": {
"$ref": "#/definitions/Parameters"
},
"plants": {
"$ref": "#/definitions/Plant"
},
"products": {
"$ref": "#/definitions/Product"
}
},
"required": [
"parameters",

View File

@@ -0,0 +1,25 @@
# RELOG: Reverse Logistics Optimization
# Copyright (C) 2020, UChicago Argonne, LLC. All rights reserved.
# Released under the modified BSD license. See COPYING.md for more details.
using RELOG
@testset "geodb_query (2018-us-county)" begin
region = RELOG.geodb_query("2018-us-county:17043")
@test region.centroid.lat == 41.83956
@test region.centroid.lon == -88.08857
@test region.population == 922_921
end
# @testset "geodb_query (2018-us-zcta)" begin
# region = RELOG.geodb_query("2018-us-zcta:60439")
# @test region.centroid.lat == 41.68241
# @test region.centroid.lon == -87.98954
# end
@testset "geodb_query (us-state)" begin
region = RELOG.geodb_query("us-state:IL")
@test region.centroid.lat == 39.73939
@test region.centroid.lon == -89.50414
@test region.population == 12_671_821
end

View File

@@ -70,7 +70,17 @@ using RELOG
@test plant.disposal_limit[p4] == [0, 0]
end
@testset "parse (invalid)" begin
@testset "parse (geodb)" begin
basedir = dirname(@__FILE__)
@test_throws String RELOG.parsefile("$basedir/../fixtures/s1-wrong-length.json")
instance = RELOG.parsefile("$basedir/../../instances/s2.json")
centers = instance.collection_centers
@test centers[1].name == "C1"
@test centers[1].latitude == 41.83956
@test centers[1].longitude == -88.08857
end
# @testset "parse (invalid)" begin
# basedir = dirname(@__FILE__)
# @test_throws ErrorException RELOG.parsefile("$basedir/../fixtures/s1-wrong-length.json")
# end

View File

@@ -6,6 +6,7 @@ using Test
@testset "RELOG" begin
@testset "Instance" begin
include("instance/compress_test.jl")
include("instance/geodb_test.jl")
include("instance/parse_test.jl")
end
@testset "Graph" begin