From 39072a6290a5652096ebc1a0ce1c1821deaf94e5 Mon Sep 17 00:00:00 2001 From: Alinson S Xavier Date: Wed, 18 Aug 2021 05:36:51 -0500 Subject: [PATCH] Make compatible with MIPLearn 5b3a56f0; reformat source code --- Makefile | 25 ++++ Manifest.toml | 6 + Project.toml | 1 + src/MIPLearn.jl | 25 +++- src/instance/abstract.jl | 5 +- src/instance/file.jl | 32 ++-- src/instance/jump.jl | 59 ++++++-- src/solvers/jump.jl | 272 ++++++++++++++-------------------- src/solvers/learning.jl | 60 +++----- src/solvers/macros.jl | 13 +- src/utils/benchmark.jl | 26 +--- src/utils/exceptions.jl | 2 +- src/utils/log.jl | 44 +++--- src/utils/sysimage.jl | 44 ++++++ test/fixtures/knapsack.jl | 8 +- test/instance/file_test.jl | 4 +- test/solvers/learning_test.jl | 2 +- test/utils/benchmark_test.jl | 28 +--- 18 files changed, 342 insertions(+), 314 deletions(-) create mode 100644 Makefile create mode 100644 src/utils/sysimage.jl diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..95a030b --- /dev/null +++ b/Makefile @@ -0,0 +1,25 @@ +# UnitCommitment.jl: Optimization Package for Security-Constrained Unit Commitment +# Copyright (C) 2020, UChicago Argonne, LLC. All rights reserved. +# Released under the modified BSD license. See COPYING.md for more details. + +JULIA := julia --color=yes --project=@. +VERSION := 0.2 + +build/sysimage.so: src/utils/sysimage.jl Project.toml Manifest.toml + mkdir -p build + $(JULIA) --trace-compile=build/precompile.jl test/runtests.jl + $(JULIA) src/utils/sysimage.jl + +clean: + rm -rf build/* + +test: build/sysimage.so + $(JULIA) --sysimage build/sysimage.so test/runtests.jl + +format: + julia -e 'using JuliaFormatter; format(["src", "test", "benchmark"], verbose=true);' + +install-deps: + julia -e 'using Pkg; Pkg.add(PackageSpec(name="JuliaFormatter", version="0.14.4"))' + +.PHONY: docs test format install-deps diff --git a/Manifest.toml b/Manifest.toml index bd1d3f8..5aa38b6 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -361,6 +361,12 @@ git-tree-sha1 = "6a9967c4394858f38b7fc49787b983ba3847e73d" uuid = "7da25872-d9ce-5375-a4d3-7a845f58efdd" version = "0.108.6+2" +[[PackageCompiler]] +deps = ["Libdl", "Pkg", "UUIDs"] +git-tree-sha1 = "bb40ed7cb3aac2b4cdf42f898c26a58ab797ac62" +uuid = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d" +version = "1.3.0" + [[Parsers]] deps = ["Dates"] git-tree-sha1 = "c8abc88faa3f7a3950832ac5d6e690881590d6dc" diff --git a/Project.toml b/Project.toml index b4f017d..f151d97 100644 --- a/Project.toml +++ b/Project.toml @@ -15,6 +15,7 @@ JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" JuMP = "4076af6c-e467-56ae-b986-b466b2749572" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" MathOptInterface = "b8f27783-ece8-5eb3-8dc8-9495eed66fee" +PackageCompiler = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" diff --git a/src/MIPLearn.jl b/src/MIPLearn.jl index 2d00658..04adc8e 100644 --- a/src/MIPLearn.jl +++ b/src/MIPLearn.jl @@ -44,13 +44,28 @@ function __init__() __init_PyFileInstance__() __init_PyJuMPInstance__() __init_JuMPSolver__() + + py""" + import numpy as np + + def to_str_array(values): + if values is None: + return None + return np.array(values, dtype="S") + + def from_str_array(values): + return [v.decode() for v in values] + """ end +to_str_array(values) = py"to_str_array"(values) +from_str_array(values) = py"from_str_array"(values) + export DynamicLazyConstraintsComponent, - UserCutsComponent, - ObjectiveValueComponent, - PrimalSolutionComponent, - StaticLazyConstraintsComponent, - MinPrecisionThreshold + UserCutsComponent, + ObjectiveValueComponent, + PrimalSolutionComponent, + StaticLazyConstraintsComponent, + MinPrecisionThreshold end # module diff --git a/src/instance/abstract.jl b/src/instance/abstract.jl index 439477e..f06d03c 100644 --- a/src/instance/abstract.jl +++ b/src/instance/abstract.jl @@ -2,7 +2,4 @@ # Copyright (C) 2020-2021, UChicago Argonne, LLC. All rights reserved. # Released under the modified BSD license. See COPYING.md for more details. -abstract type Instance -end - - +abstract type Instance end diff --git a/src/instance/file.jl b/src/instance/file.jl index 6f34b17..8cf3e2f 100644 --- a/src/instance/file.jl +++ b/src/instance/file.jl @@ -6,7 +6,7 @@ import Base: flush mutable struct FileInstance <: Instance py::Union{Nothing,PyCall.PyObject} - loaded::Union{Nothing, JuMPInstance} + loaded::Union{Nothing,JuMPInstance} filename::AbstractString h5::PyCall.PyObject @@ -21,9 +21,14 @@ end to_model(instance::FileInstance) = to_model(instance.loaded) get_instance_features(instance::FileInstance) = get_instance_features(instance.loaded) -get_variable_features(instance::FileInstance) = get_variable_features(instance.loaded) -get_variable_categories(instance::FileInstance) = get_variable_categories(instance.loaded) -get_constraint_features(instance::FileInstance) = get_constraint_features(instance.loaded) +get_variable_features(instance::FileInstance, names) = + get_variable_features(instance.loaded, names) +get_variable_categories(instance::FileInstance, names) = + get_variable_categories(instance.loaded, names) +get_constraint_features(instance::FileInstance, names) = + get_constraint_features(instance.loaded, names) +get_constraint_categories(instance::FileInstance, names) = + get_constraint_categories(instance.loaded, names) function get_samples(instance::FileInstance) return [instance.h5] @@ -33,10 +38,6 @@ function create_sample!(instance::FileInstance) return instance.h5 end -function get_constraint_categories(instance::FileInstance) - return get_constraint_categories(instance.loaded) -end - function load(instance::FileInstance) if instance.loaded === nothing instance.loaded = load_instance(instance.filename) @@ -49,8 +50,7 @@ function free(instance::FileInstance) GC.gc() end -function flush(instance::FileInstance) -end +function flush(instance::FileInstance) end function __init_PyFileInstance__() @pydef mutable struct Class <: miplearn.Instance @@ -59,10 +59,14 @@ function __init_PyFileInstance__() end to_model(self) = to_model(self.jl) get_instance_features(self) = get_instance_features(self.jl) - get_variable_features(self) = get_variable_features(self.jl) - get_variable_categories(self) = get_variable_categories(self.jl) - get_constraint_features(self) = get_constraint_features(self.jl) - get_constraint_categories(self) = get_constraint_categories(self.jl) + get_variable_features(self, names) = + get_variable_features(self.jl, from_str_array(names)) + get_variable_categories(self, names) = + to_str_array(get_variable_categories(self.jl, from_str_array(names))) + get_constraint_features(self, names) = + get_constraint_features(self.jl, from_str_array(names)) + get_constraint_categories(self, names) = + to_str_array(get_constraint_categories(self.jl, from_str_array(names))) get_samples(self) = get_samples(self.jl) create_sample(self) = create_sample!(self.jl) load(self) = load(self.jl) diff --git a/src/instance/jump.jl b/src/instance/jump.jl index 5ae2c2b..ec2f282 100644 --- a/src/instance/jump.jl +++ b/src/instance/jump.jl @@ -38,11 +38,44 @@ function to_model(instance::JuMPInstance)::JuMP.Model return instance.model end -get_instance_features(instance::JuMPInstance) = instance.ext["instance_features"] -get_variable_features(instance::JuMPInstance) = instance.ext["variable_features"] -get_variable_categories(instance::JuMPInstance) = instance.ext["variable_categories"] -get_constraint_features(instance::JuMPInstance) = instance.ext["constraint_features"] -get_constraint_categories(instance::JuMPInstance) = instance.ext["constraint_categories"] +function get_instance_features(instance::JuMPInstance)::Union{Vector{Float64},Nothing} + return instance.ext["instance_features"] +end + +function _concat_features(dict, names)::Matrix{Float64} + if isempty(dict) + return zeros(length(names), 1) + end + ncols = length(first(dict).second) + return vcat([n in keys(dict) ? dict[n]' : zeros(ncols) for n in names]...) +end + +function _concat_categories(dict, names)::Vector{String} + return String[n in keys(dict) ? dict[n] : n for n in names] +end + +function get_variable_features( + instance::JuMPInstance, + names::Vector{String}, +)::Matrix{Float64} + return _concat_features(instance.ext["variable_features"], names) +end + +function get_variable_categories(instance::JuMPInstance, names::Vector{String}) + return _concat_categories(instance.ext["variable_categories"], names) +end + +function get_constraint_features( + instance::JuMPInstance, + names::Vector{String}, +)::Matrix{Float64} + return _concat_features(instance.ext["constraint_features"], names) +end + +function get_constraint_categories(instance::JuMPInstance, names::Vector{String}) + return _concat_categories(instance.ext["constraint_categories"], names) +end + get_samples(instance::JuMPInstance) = instance.samples function create_sample!(instance::JuMPInstance) @@ -58,10 +91,14 @@ function __init_PyJuMPInstance__() end to_model(self) = to_model(self.jl) get_instance_features(self) = get_instance_features(self.jl) - get_variable_features(self) = get_variable_features(self.jl) - get_variable_categories(self) = get_variable_categories(self.jl) - get_constraint_features(self,) = get_constraint_features(self.jl) - get_constraint_categories(self) = get_constraint_categories(self.jl) + get_variable_features(self, names) = + get_variable_features(self.jl, from_str_array(names)) + get_variable_categories(self, names) = + to_str_array(get_variable_categories(self.jl, from_str_array(names))) + get_constraint_features(self, names) = + get_constraint_features(self.jl, from_str_array(names)) + get_constraint_categories(self, names) = + to_str_array(get_constraint_categories(self.jl, from_str_array(names))) get_samples(self) = get_samples(self.jl) create_sample(self) = create_sample!(self.jl) end @@ -76,7 +113,7 @@ function save(filename::AbstractString, instance::JuMPInstance)::Nothing mps = read(mps_filename) # Generate HDF5 - h5 = Hdf5Sample(filename, mode="w") + h5 = Hdf5Sample(filename, mode = "w") h5.put_scalar("miplearn_version", "0002") h5.put_bytes("mps", mps) h5.put_scalar("jump_ext", JSON.json(model.ext[:miplearn])) @@ -89,7 +126,7 @@ function _check_miplearn_version(h5) "The file you are trying to load has been generated by " * "MIPLearn $(v) and you are currently running MIPLearn 0002 " * "Reading files generated by different versions of MIPLearn is " * - "not currently supported." + "not currently supported.", ) end diff --git a/src/solvers/jump.jl b/src/solvers/jump.jl index 9fade0a..56a6b24 100644 --- a/src/solvers/jump.jl +++ b/src/solvers/jump.jl @@ -9,9 +9,8 @@ using MathOptInterface using TimerOutputs const MOI = MathOptInterface - mutable struct JuMPSolverData - optimizer_factory + optimizer_factory::Any varname_to_var::Dict{String,VariableRef} cname_to_constr::Dict{String,JuMP.ConstraintRef} instance::Union{Nothing,PyObject} @@ -29,10 +28,10 @@ end Optimizes a given JuMP model while capturing the solver log, then returns that log. If tee=true, prints the solver log to the standard output as the optimization takes place. """ -function _optimize_and_capture_output!(model; tee::Bool=false) +function _optimize_and_capture_output!(model; tee::Bool = false) logname = tempname() logfile = open(logname, "w") - redirect_stdout(logfile) do + redirect_stdout(logfile) do JuMP.optimize!(model) Base.Libc.flush_cstdio() end @@ -50,10 +49,7 @@ end function _update_solution!(data::JuMPSolverData) vars = JuMP.all_variables(data.model) - data.solution = Dict( - var => JuMP.value(var) - for var in vars - ) + data.solution = Dict(var => JuMP.value(var) for var in vars) # Reduced costs if has_duals(data.model) @@ -89,7 +85,7 @@ end function add_constraints( data::JuMPSolverData; - lhs::Vector{Vector{Tuple{String, Float64}}}, + lhs::Vector{Vector{Tuple{String,Float64}}}, rhs::Vector{Float64}, senses::Vector{String}, names::Vector{String}, @@ -104,8 +100,10 @@ function add_constraints( constr = @constraint(data.model, lhs_expr <= rhs[i]) elseif sense == ">" constr = @constraint(data.model, lhs_expr >= rhs[i]) - else + elseif sense == "=" constr = @constraint(data.model, lhs_expr == rhs[i]) + else + error("unknown sense: $(sense)") end set_name(constr, names[i]) data.cname_to_constr[names[i]] = constr @@ -116,10 +114,10 @@ end function are_constraints_satisfied( data::JuMPSolverData; - lhs::Vector{Vector{Tuple{String, Float64}}}, + lhs::Vector{Vector{Tuple{String,Float64}}}, rhs::Vector{Float64}, senses::Vector{String}, - tol::Float64=1e-5, + tol::Float64 = 1e-5, )::Vector{Bool} result = [] for (i, sense) in enumerate(senses) @@ -132,8 +130,10 @@ function are_constraints_satisfied( push!(result, lhs_value <= rhs[i] + tol) elseif sense == ">" push!(result, lhs_value >= rhs[i] - tol) - else + elseif sense == "=" push!(result, abs(lhs_value - rhs[i]) <= tol) + else + error("unknown sense: $(sense)") end end return result @@ -148,9 +148,9 @@ function build_test_instance_knapsack() model = Model() n = length(weights) @variable(model, x[0:n-1], Bin) - @variable(model, z, lower_bound=0.0, upper_bound=capacity) - @objective(model, Max, sum(x[i-1] * prices[i] for i in 1:n)) - @constraint(model, eq_capacity, sum(x[i-1] * weights[i] for i in 1:n) - z == 0) + @variable(model, z, lower_bound = 0.0, upper_bound = capacity) + @objective(model, Max, sum(x[i-1] * prices[i] for i = 1:n)) + @constraint(model, eq_capacity, sum(x[i-1] * weights[i] for i = 1:n) - z == 0) return JuMPInstance(model).py end @@ -165,10 +165,7 @@ function build_test_instance_infeasible() end -function remove_constraints( - data::JuMPSolverData, - names::Vector{String}, -)::Nothing +function remove_constraints(data::JuMPSolverData, names::Vector{String})::Nothing for name in names constr = data.cname_to_constr[name] delete(data.model, constr) @@ -178,17 +175,13 @@ function remove_constraints( end -function solve( - data::JuMPSolverData; - tee::Bool=false, - iteration_cb=nothing, -) +function solve(data::JuMPSolverData; tee::Bool = false, iteration_cb = nothing) model = data.model wallclock_time = 0 log = "" while true wallclock_time += @elapsed begin - log *= _optimize_and_capture_output!(model, tee=tee) + log *= _optimize_and_capture_output!(model, tee = tee) end if iteration_cb !== nothing iteration_cb() || break @@ -215,18 +208,18 @@ function solve( upper_bound = dual_bound end return miplearn.solvers.internal.MIPSolveStats( - mip_lower_bound=lower_bound, - mip_upper_bound=upper_bound, - mip_sense=sense, - mip_wallclock_time=wallclock_time, - mip_nodes=1, - mip_log=log, - mip_warm_start_value=nothing, + mip_lower_bound = lower_bound, + mip_upper_bound = upper_bound, + mip_sense = sense, + mip_wallclock_time = wallclock_time, + mip_nodes = 1, + mip_log = log, + mip_warm_start_value = nothing, ) end -function solve_lp(data::JuMPSolverData; tee::Bool=false) +function solve_lp(data::JuMPSolverData; tee::Bool = false) model, bin_vars = data.model, data.bin_vars for var in bin_vars ~is_fixed(var) || continue @@ -242,7 +235,7 @@ function solve_lp(data::JuMPSolverData; tee::Bool=false) set_optimizer(model, Clp.Optimizer) end wallclock_time = @elapsed begin - log = _optimize_and_capture_output!(model, tee=tee) + log = _optimize_and_capture_output!(model, tee = tee) end if is_infeasible(data) data.solution = Dict() @@ -259,9 +252,9 @@ function solve_lp(data::JuMPSolverData; tee::Bool=false) set_binary(var) end return miplearn.solvers.internal.LPSolveStats( - lp_value=obj_value, - lp_log=log, - lp_wallclock_time=wallclock_time, + lp_value = obj_value, + lp_log = log, + lp_wallclock_time = wallclock_time, ) end @@ -276,15 +269,8 @@ function set_instance!( model = instance.to_model() end data.model = model - data.bin_vars = [ - var - for var in JuMP.all_variables(model) - if JuMP.is_binary(var) - ] - data.varname_to_var = Dict( - JuMP.name(var) => var - for var in JuMP.all_variables(model) - ) + data.bin_vars = [var for var in JuMP.all_variables(model) if JuMP.is_binary(var)] + data.varname_to_var = Dict(JuMP.name(var) => var for var in JuMP.all_variables(model)) JuMP.set_optimizer(model, data.optimizer_factory) data.cname_to_constr = Dict() for (ftype, stype) in JuMP.list_of_constraint_types(model) @@ -302,7 +288,7 @@ function fix!(data::JuMPSolverData, solution) for (varname, value) in solution value !== nothing || continue var = data.varname_to_var[varname] - JuMP.fix(var, value, force=true) + JuMP.fix(var, value, force = true) end end @@ -317,17 +303,12 @@ end function is_infeasible(data::JuMPSolverData) - return JuMP.termination_status(data.model) in [ - MOI.INFEASIBLE, - MOI.INFEASIBLE_OR_UNBOUNDED, - ] + return JuMP.termination_status(data.model) in + [MOI.INFEASIBLE, MOI.INFEASIBLE_OR_UNBOUNDED] end -function get_variables( - data::JuMPSolverData; - with_static::Bool, -) +function get_variables(data::JuMPSolverData; with_static::Bool) vars = JuMP.all_variables(data.model) lb, ub, types, obj_coeffs = nothing, nothing, nothing, nothing values, rc = nothing, nothing @@ -343,46 +324,32 @@ function get_variables( if with_static # Lower bounds lb = [ - JuMP.is_binary(v) ? 0.0 : - JuMP.has_lower_bound(v) ? JuMP.lower_bound(v) : - -Inf - for v in vars + JuMP.is_binary(v) ? 0.0 : JuMP.has_lower_bound(v) ? JuMP.lower_bound(v) : -Inf for v in vars ] # Upper bounds ub = [ - JuMP.is_binary(v) ? 1.0 : - JuMP.has_upper_bound(v) ? JuMP.upper_bound(v) : - Inf - for v in vars + JuMP.is_binary(v) ? 1.0 : JuMP.has_upper_bound(v) ? JuMP.upper_bound(v) : Inf for v in vars ] # Variable types - types = [ - JuMP.is_binary(v) ? "B" : - JuMP.is_integer(v) ? "I" : - "C" - for v in vars - ] + types = [JuMP.is_binary(v) ? "B" : JuMP.is_integer(v) ? "I" : "C" for v in vars] # Objective function coefficients obj = objective_function(data.model) - obj_coeffs = [ - v ∈ keys(obj.terms) ? obj.terms[v] : 0.0 - for v in vars - ] + obj_coeffs = [v ∈ keys(obj.terms) ? obj.terms[v] : 0.0 for v in vars] end rc = isempty(data.reduced_costs) ? nothing : data.reduced_costs vf = miplearn.solvers.internal.Variables( - names=names, - lower_bounds=lb, - upper_bounds=ub, - types=types, - obj_coeffs=obj_coeffs, - reduced_costs=rc, - values=values, + names = to_str_array(names), + lower_bounds = lb, + upper_bounds = ub, + types = to_str_array(types), + obj_coeffs = obj_coeffs, + reduced_costs = rc, + values = values, ) return vf end @@ -394,26 +361,23 @@ function get_constraints( with_sa::Bool, with_lhs::Bool, ) - names = [] + names = String[] senses, lhs, rhs = nothing, nothing, nothing dual_values = nothing if !isempty(data.dual_values) - dual_values = [] + dual_values = Float64[] end if with_static - senses, lhs, rhs = [], [], [] + senses, lhs, rhs = String[], [], Float64[] end for (ftype, stype) in JuMP.list_of_constraint_types(data.model) - ftype in [JuMP.AffExpr, JuMP.VariableRef] || error("Unsupported constraint type: ($ftype, $stype)") + ftype in [JuMP.AffExpr, JuMP.VariableRef] || + error("Unsupported constraint type: ($ftype, $stype)") for constr in JuMP.all_constraints(data.model, ftype, stype) - cset = MOI.get( - constr.model.moi_backend, - MOI.ConstraintSet(), - constr.index, - ) + cset = MOI.get(constr.model.moi_backend, MOI.ConstraintSet(), constr.index) name = JuMP.name(constr) length(name) > 0 || continue push!(names, name) @@ -429,19 +393,21 @@ function get_constraints( lhs, [ ( - MOI.get( - constr.model.moi_backend, - MOI.VariableName(), - term.variable_index + pybytes( + MOI.get( + constr.model.moi_backend, + MOI.VariableName(), + term.variable_index, + ), ), term.coefficient, - ) - for term in MOI.get( + ) for term in + MOI.get( constr.model.moi_backend, MOI.ConstraintFunction(), constr.index, ).terms - ] + ], ) end if stype == MOI.EqualTo{Float64} @@ -464,11 +430,11 @@ function get_constraints( end return miplearn.solvers.internal.Constraints( - names=names, - senses=senses, - lhs=lhs, - rhs=rhs, - dual_values=dual_values, + names = to_str_array(names), + senses = to_str_array(senses), + lhs = lhs, + rhs = rhs, + dual_values = dual_values, ) end @@ -497,14 +463,14 @@ function __init_JuMPSolver__() end add_constraints( self.data, - lhs=lhs, - rhs=cf.rhs, - senses=cf.senses, - names=cf.names, + lhs = lhs, + rhs = cf.rhs, + senses = from_str_array(cf.senses), + names = from_str_array(cf.names), ) end - function are_constraints_satisfied(self, cf; tol=1e-5) + function are_constraints_satisfied(self, cf; tol = 1e-5) lhs = cf.lhs if lhs isa Matrix # Undo incorrect automatic conversion performed by PyCall @@ -512,38 +478,30 @@ function __init_JuMPSolver__() end return are_constraints_satisfied( self.data, - lhs=lhs, - rhs=cf.rhs, - senses=cf.senses, - tol=tol, + lhs = lhs, + rhs = cf.rhs, + senses = from_str_array(cf.senses), + tol = tol, ) end - build_test_instance_infeasible(self) = - build_test_instance_infeasible() + build_test_instance_infeasible(self) = build_test_instance_infeasible() + + build_test_instance_knapsack(self) = build_test_instance_knapsack() - build_test_instance_knapsack(self) = - build_test_instance_knapsack() - clone(self) = JuMPSolver(self.data.optimizer_factory) - fix(self, solution) = - fix!(self.data, solution) - - get_solution(self) = - isempty(self.data.solution) ? nothing : self.data.solution + fix(self, solution) = fix!(self.data, solution) - get_constraints( - self; - with_static=true, - with_sa=true, - with_lhs=true, - ) = get_constraints( - self.data, - with_static=with_static, - with_sa=with_sa, - with_lhs=with_lhs, - ) + get_solution(self) = isempty(self.data.solution) ? nothing : self.data.solution + + get_constraints(self; with_static = true, with_sa = true, with_lhs = true) = + get_constraints( + self.data, + with_static = with_static, + with_sa = with_sa, + with_lhs = with_lhs, + ) get_constraint_attrs(self) = [ # "basis_status", @@ -559,14 +517,11 @@ function __init_JuMPSolver__() # "slacks", "user_features", ] - - get_variables( - self; - with_static=true, - with_sa=true, - ) = get_variables(self.data; with_static=with_static) - - get_variable_attrs(self) = [ + + get_variables(self; with_static = true, with_sa = true) = + get_variables(self.data; with_static = with_static) + + get_variable_attrs(self) = [ "names", # "basis_status", "categories", @@ -585,35 +540,24 @@ function __init_JuMPSolver__() "values", ] - is_infeasible(self) = - is_infeasible(self.data) + is_infeasible(self) = is_infeasible(self.data) - remove_constraints(self, names) = - remove_constraints( - self.data, - [n for n in names], - ) + remove_constraints(self, names) = remove_constraints(self.data, [n for n in names]) + + set_instance(self, instance, model = nothing) = + set_instance!(self.data, instance, model = model) - set_instance(self, instance, model=nothing) = - set_instance!(self.data, instance, model=model) - - set_warm_start(self, solution) = - set_warm_start!(self.data, solution) + set_warm_start(self, solution) = set_warm_start!(self.data, solution) solve( self; - tee=false, - iteration_cb=nothing, - lazy_cb=nothing, - user_cut_cb=nothing, - ) = solve( - self.data, - tee=tee, - iteration_cb=iteration_cb, - ) - - solve_lp(self; tee=false) = - solve_lp(self.data, tee=tee) + tee = false, + iteration_cb = nothing, + lazy_cb = nothing, + user_cut_cb = nothing, + ) = solve(self.data, tee = tee, iteration_cb = iteration_cb) + + solve_lp(self; tee = false) = solve_lp(self.data, tee = tee) end copy!(JuMPSolver, Class) end diff --git a/src/solvers/learning.jl b/src/solvers/learning.jl index 9ee9b0b..bdbf616 100644 --- a/src/solvers/learning.jl +++ b/src/solvers/learning.jl @@ -8,7 +8,7 @@ using JLD2 struct LearningSolver py::PyCall.PyObject - optimizer_factory + optimizer_factory::Any end @@ -23,13 +23,13 @@ function LearningSolver( )::LearningSolver return LearningSolver( miplearn.LearningSolver( - solver=JuMPSolver(optimizer_factory), - mode=mode, - solve_lp=solve_lp, - simulate_perfect=simulate_perfect, - components=components, - extract_lhs=extract_lhs, - extract_sa=extract_sa, + solver = JuMPSolver(optimizer_factory), + mode = mode, + solve_lp = solve_lp, + simulate_perfect = simulate_perfect, + components = components, + extract_lhs = extract_lhs, + extract_sa = extract_sa, ), optimizer_factory, ) @@ -44,8 +44,8 @@ function solve!( ) return @python_call solver.py.solve( instance.py, - tee=tee, - discard_output=discard_output, + tee = tee, + discard_output = discard_output, ) end @@ -56,19 +56,11 @@ function fit!(solver::LearningSolver, instances::Vector{<:Instance}) end -function _solve( - solver_filename, - instance_filename; - discard_output::Bool, -) +function _solve(solver_filename, instance_filename; discard_output::Bool) @info "solve $instance_filename" solver = load_solver(solver_filename) solver.py._silence_miplearn_logger() - stats = solve!( - solver, - FileInstance(instance_filename), - discard_output = discard_output, - ) + stats = solve!(solver, FileInstance(instance_filename), discard_output = discard_output) solver.py._restore_miplearn_logger() GC.gc() @info "solve $instance_filename [done]" @@ -85,13 +77,10 @@ function parallel_solve!( solver_filename = tempname() save(solver_filename, solver) return pmap( - instance_filename -> _solve( - solver_filename, - instance_filename, - discard_output = discard_output, - ), + instance_filename -> + _solve(solver_filename, instance_filename, discard_output = discard_output), instance_filenames, - on_error=identity, + on_error = identity, ) end @@ -108,9 +97,9 @@ function save(filename::AbstractString, solver::LearningSolver) solver.py.internal_solver_prototype = internal_solver_prototype jldsave( filename; - miplearn_version="0.2", - solver_py=solver_py, - optimizer_factory=solver.optimizer_factory, + miplearn_version = "0.2", + solver_py = solver_py, + optimizer_factory = solver.optimizer_factory, ) return end @@ -123,18 +112,9 @@ function load_solver(filename::AbstractString)::LearningSolver solver_py = miplearn.read_pickle_gz(solve_py_filename) internal_solver = JuMPSolver(file["optimizer_factory"]) solver_py.internal_solver_prototype = internal_solver - return LearningSolver( - solver_py, - file["optimizer_factory"], - ) + return LearningSolver(solver_py, file["optimizer_factory"]) end end -export Instance, - LearningSolver, - solve!, - fit!, - parallel_solve!, - save, - load_solver +export Instance, LearningSolver, solve!, fit!, parallel_solve!, save, load_solver diff --git a/src/solvers/macros.jl b/src/solvers/macros.jl index fb57ecc..1df1c8d 100644 --- a/src/solvers/macros.jl +++ b/src/solvers/macros.jl @@ -6,10 +6,10 @@ function init_miplearn_ext(model)::Dict if :miplearn ∉ keys(model.ext) model.ext[:miplearn] = Dict() model.ext[:miplearn]["instance_features"] = [0.0] - model.ext[:miplearn]["variable_features"] = Dict{AbstractString, Vector{Float64}}() - model.ext[:miplearn]["variable_categories"] = Dict{AbstractString, String}() - model.ext[:miplearn]["constraint_features"] = Dict{AbstractString, Vector{Float64}}() - model.ext[:miplearn]["constraint_categories"] = Dict{AbstractString, String}() + model.ext[:miplearn]["variable_features"] = Dict{AbstractString,Vector{Float64}}() + model.ext[:miplearn]["variable_categories"] = Dict{AbstractString,String}() + model.ext[:miplearn]["constraint_features"] = Dict{AbstractString,Vector{Float64}}() + model.ext[:miplearn]["constraint_categories"] = Dict{AbstractString,String}() end return model.ext[:miplearn] end @@ -71,11 +71,10 @@ function _get_and_check_name(obj) n = name(obj) length(n) > 0 || error( "Features and categories can only be assigned to variables and " * - "constraints that have names. Unnamed model element detected." + "constraints that have names. Unnamed model element detected.", ) return n end -export @feature, - @category +export @feature, @category diff --git a/src/utils/benchmark.jl b/src/utils/benchmark.jl index 19ae467..0c131a1 100644 --- a/src/utils/benchmark.jl +++ b/src/utils/benchmark.jl @@ -16,11 +16,8 @@ mutable struct BenchmarkRunner solvers, nothing, # results miplearn.BenchmarkRunner( - Dict( - sname => solver.py - for (sname, solver) in solvers - ) - ) + Dict(sname => solver.py for (sname, solver) in solvers), + ), ) end end @@ -33,7 +30,7 @@ function parallel_solve!( instances = repeat(instances, n_trials) for (solver_name, solver) in runner.solvers @info "benchmark $solver_name" - stats = parallel_solve!(solver, instances, discard_output=true) + stats = parallel_solve!(solver, instances, discard_output = true) for (i, s) in enumerate(stats) s["Solver"] = solver_name s["Instance"] = instances[i].filename @@ -41,29 +38,20 @@ function parallel_solve!( if runner.results === nothing runner.results = DataFrame(s) else - push!(runner.results, s, cols=:union) + push!(runner.results, s, cols = :union) end end @info "benchmark $solver_name [done]" end end -function fit!( - runner::BenchmarkRunner, - instances::Vector{FileInstance} -)::Nothing +function fit!(runner::BenchmarkRunner, instances::Vector{FileInstance})::Nothing @python_call runner.py.fit([instance.py for instance in instances]) end -function write_csv!( - runner::BenchmarkRunner, - filename::AbstractString, -)::Nothing +function write_csv!(runner::BenchmarkRunner, filename::AbstractString)::Nothing CSV.write(filename, runner.results) return end -export BenchmarkRunner, - parallel_solve!, - fit!, - write_csv! +export BenchmarkRunner, parallel_solve!, fit!, write_csv! diff --git a/src/utils/exceptions.jl b/src/utils/exceptions.jl index e9234b8..8e0d480 100644 --- a/src/utils/exceptions.jl +++ b/src/utils/exceptions.jl @@ -11,7 +11,7 @@ macro python_call(expr) return $(esc(expr)) catch e if isa(e, PyCall.PyError) - printstyled("Uncaught Python exception:\n", bold=true, color=:red) + printstyled("Uncaught Python exception:\n", bold = true, color = :red) traceback.print_exception(e.T, e.val, e.traceback) end rethrow() diff --git a/src/utils/log.jl b/src/utils/log.jl index f0f8342..ebc106c 100644 --- a/src/utils/log.jl +++ b/src/utils/log.jl @@ -7,35 +7,37 @@ using Base.CoreLogging, Logging, Printf struct TimeLogger <: AbstractLogger initial_time::Float64 - file::Union{Nothing, IOStream} - screen_log_level - io_log_level + file::Union{Nothing,IOStream} + screen_log_level::Any + io_log_level::Any end function TimeLogger(; - initial_time::Float64, - file::Union{Nothing, IOStream} = nothing, - screen_log_level = CoreLogging.Info, - io_log_level = CoreLogging.Info, - ) :: TimeLogger + initial_time::Float64, + file::Union{Nothing,IOStream} = nothing, + screen_log_level = CoreLogging.Info, + io_log_level = CoreLogging.Info, +)::TimeLogger return TimeLogger(initial_time, file, screen_log_level, io_log_level) end min_enabled_level(logger::TimeLogger) = logger.io_log_level shouldlog(logger::TimeLogger, level, _module, group, id) = true -function handle_message(logger::TimeLogger, - level, - message, - _module, - group, - id, - filepath, - line; - kwargs...) +function handle_message( + logger::TimeLogger, + level, + message, + _module, + group, + id, + filepath, + line; + kwargs..., +) elapsed_time = time() - logger.initial_time time_string = @sprintf("[%12.3f] ", elapsed_time) - + if level >= Logging.Error color = :light_red elseif level >= Logging.Warn @@ -43,12 +45,12 @@ function handle_message(logger::TimeLogger, else color = :light_green end - + flush(stdout) flush(stderr) Base.Libc.flush_cstdio() if level >= logger.screen_log_level - printstyled(time_string, color=color) + printstyled(time_string, color = color) println(message) end if logger.file !== nothing && level >= logger.io_log_level @@ -64,7 +66,7 @@ end function setup_logger() initial_time = time() - global_logger(TimeLogger(initial_time=initial_time)) + global_logger(TimeLogger(initial_time = initial_time)) miplearn = pyimport("miplearn") miplearn.setup_logger(initial_time) end diff --git a/src/utils/sysimage.jl b/src/utils/sysimage.jl new file mode 100644 index 0000000..9b26b4b --- /dev/null +++ b/src/utils/sysimage.jl @@ -0,0 +1,44 @@ +# MIPLearn: Extensible Framework for Learning-Enhanced Mixed-Integer Optimization +# Copyright (C) 2020-2021, UChicago Argonne, LLC. All rights reserved. +# Released under the modified BSD license. See COPYING.md for more details. + +using PackageCompiler + +using CSV +using Cbc +using Clp +using Conda +using DataFrames +using Distributed +using JLD2 +using JSON +using JuMP +using Logging +using MathOptInterface +using Printf +using PyCall +using TimerOutputs + +pkg = [ + :CSV + :Cbc + :Clp + :Conda + :DataFrames + :Distributed + :JLD2 + :JSON + :JuMP + :Logging + :MathOptInterface + :Printf + :PyCall + :TimerOutputs +] + +@info "Building system image..." +create_sysimage( + pkg, + precompile_statements_file = "build/precompile.jl", + sysimage_path = "build/sysimage.so", +) diff --git a/test/fixtures/knapsack.jl b/test/fixtures/knapsack.jl index 1872a9f..25fb2bb 100644 --- a/test/fixtures/knapsack.jl +++ b/test/fixtures/knapsack.jl @@ -15,14 +15,14 @@ function build_knapsack_model() n = length(weights) @variable(model, x[1:n], Bin) - @objective(model, Max, sum(x[i] * prices[i] for i in 1:n)) - @constraint(model, c1, sum(x[i] * weights[i] for i in 1:n) <= capacity) + @objective(model, Max, sum(x[i] * prices[i] for i = 1:n)) + @constraint(model, c1, sum(x[i] * weights[i] for i = 1:n) <= capacity) # Add ML information to the model @feature(model, [5.0]) @feature(c1, [1.0, 2.0, 3.0]) @category(c1, "c1") - for i in 1:n + for i = 1:n @feature(x[i], [weights[i]; prices[i]]) @category(x[i], "type-$i") end @@ -36,7 +36,7 @@ function build_knapsack_model() @test model.ext[:miplearn]["variable_categories"]["x[3]"] == "type-3" @test model.ext[:miplearn]["constraint_features"]["c1"] == [1.0, 2.0, 3.0] @test model.ext[:miplearn]["constraint_categories"]["c1"] == "c1" - @test model.ext[:miplearn]["instance_features"] == [5.0] + @test model.ext[:miplearn]["instance_features"] == [5.0] return model end diff --git a/test/instance/file_test.jl b/test/instance/file_test.jl index 8552d98..22b4df0 100644 --- a/test/instance/file_test.jl +++ b/test/instance/file_test.jl @@ -23,6 +23,6 @@ using Cbc solver = LearningSolver(Cbc.Optimizer) solve!(solver, file_instance) - @test length(h5.get_vector("mip_var_values")) == 3 - end + @test length(h5.get_array("mip_var_values")) == 3 + end end diff --git a/test/solvers/learning_test.jl b/test/solvers/learning_test.jl index e08ca0c..e5a440c 100644 --- a/test/solvers/learning_test.jl +++ b/test/solvers/learning_test.jl @@ -39,7 +39,7 @@ using MIPLearn @testset "Discard output" begin instance = build_knapsack_file_instance() solver = LearningSolver(Cbc.Optimizer) - solve!(solver, instance, discard_output=true) + solve!(solver, instance, discard_output = true) loaded = load_instance(instance.filename) @test length(loaded.samples) == 0 end diff --git a/test/utils/benchmark_test.jl b/test/utils/benchmark_test.jl index c549391..2f5c6e1 100644 --- a/test/utils/benchmark_test.jl +++ b/test/utils/benchmark_test.jl @@ -9,38 +9,24 @@ using DataFrames @testset "BenchmarkRunner" begin @info "Building training data..." - instances = [ - build_knapsack_file_instance(), - build_knapsack_file_instance(), - ] - stats = parallel_solve!( - LearningSolver(Cbc.Optimizer), - instances, - ) + instances = [build_knapsack_file_instance(), build_knapsack_file_instance()] + stats = parallel_solve!(LearningSolver(Cbc.Optimizer), instances) @test length(stats) == 2 @test stats[1] !== nothing @test stats[2] !== nothing benchmark = BenchmarkRunner( - solvers=Dict( - "baseline" => LearningSolver( - Cbc.Optimizer, - components=[], - ), - "ml-exact" => LearningSolver( - Cbc.Optimizer, - ), - "ml-heur" => LearningSolver( - Cbc.Optimizer, - mode="heuristic", - ), + solvers = Dict( + "baseline" => LearningSolver(Cbc.Optimizer, components = []), + "ml-exact" => LearningSolver(Cbc.Optimizer), + "ml-heur" => LearningSolver(Cbc.Optimizer, mode = "heuristic"), ), ) @info "Fitting..." fit!(benchmark, instances) @info "Benchmarking..." - parallel_solve!(benchmark, instances, n_trials=2) + parallel_solve!(benchmark, instances, n_trials = 2) csv_filename = tempname() write_csv!(benchmark, csv_filename)