CpxCuts: Use sparse arrays in H5 file

Project.toml

@@ -5,9 +5,12 @@ version = "0.3.0"
 
 [deps]
 CPLEX = "a076750e-1247-5638-91d2-ce28b192dca0"
+Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d"
 HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
 JuMP = "4076af6c-e467-56ae-b986-b466b2749572"
 MathOptInterface = "b8f27783-ece8-5eb3-8dc8-9495eed66fee"
+PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
+SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 
 [compat]
 JuMP = "1"

deps/formatter/Project.toml

@@ -1,5 +0,0 @@
-[deps]
-JuliaFormatter = "98e50ef6-434e-11e9-1051-2b60c6c9e899"
-
-[compat]
-JuliaFormatter = "0.14.4"

deps/formatter/format.jl

@@ -1,8 +0,0 @@
-using JuliaFormatter
-format(
-    [
-        "../../src", 
-        "../../test",
-    ],
-    verbose=true,
-)

src/Cuts/BlackBox/cplex.jl

@@ -36,13 +36,13 @@ function collect(
     CPXsetintparam(env, CPX_PARAM_NODELIM, 0)
 
     # Parameter: Make cutting plane generation more aggresive
-    CPXsetintparam(env, CPX_PARAM_AGGCUTLIM, 100)
-    CPXsetintparam(env, CPX_PARAM_FRACCAND, 1000)
     CPXsetintparam(env, CPX_PARAM_FRACCUTS, 2)
-    CPXsetintparam(env, CPX_PARAM_FRACPASS, 100)
-    CPXsetintparam(env, CPX_PARAM_GUBCOVERS, 100)
     CPXsetintparam(env, CPX_PARAM_MIRCUTS, 2)
     CPXsetintparam(env, CPX_PARAM_ZEROHALFCUTS, 2)
+    # CPXsetintparam(env, CPX_PARAM_AGGCUTLIM, 100)
+    # CPXsetintparam(env, CPX_PARAM_FRACCAND, 1000)
+    # CPXsetintparam(env, CPX_PARAM_FRACPASS, 100)
+    # CPXsetintparam(env, CPX_PARAM_GUBCOVERS, 100)
 
     # Load problem
     lp = CPXcreateprob(env, status_p, "problem")
@@ -70,37 +70,35 @@ function collect(
     # Load generated MPS file
     model = JuMP.read_from_file("$tempdir/root.mps")
 
+    function select(cr)
+        return name(cr)[begin] in ['i', 'f', 'm', 'r', 'L', 'z', 'v'] &&  isdigit(name(cr)[begin+1])
+    end
+
     # Parse cuts
-    cuts_lhs::Vector{Vector{Float64}} = []
-    cuts_rhs::Vector{Float64} = []
-    nvars = num_variables(model)
     constraints = all_constraints(model, GenericAffExpr{Float64,VariableRef}, MOI.LessThan{Float64})
+    nvars = num_variables(model)
+    ncuts = length([cr for cr in constraints if select(cr)])
+    cuts_lhs = spzeros(ncuts, nvars)
+    cuts_rhs = Float64[]
+
+    offset = 1
     for conRef in constraints
-        if name(conRef)[begin] in ['i', 'f', 'm', 'r', 'L', 'z', 'v'] &&
-            isdigit(name(conRef)[begin+1])
+        if select(conRef)
             c = constraint_object(conRef)
             cset = MOI.get(conRef.model.moi_backend, MOI.ConstraintSet(), conRef.index)
-            lhs = zeros(nvars)
             for (key, val) in c.func.terms
-                lhs[key.index.value] = val
+                cuts_lhs[offset, key.index.value] = val
             end
-            push!(cuts_lhs, lhs)
             push!(cuts_rhs, cset.upper)
+            offset += 1
         end
     end
-    @info "$(length(cuts_lhs)) CPLEX cuts collected"
-    cuts_lhs_matrix::Matrix{Float64} = vcat(cuts_lhs'...)
-
-    # Store cuts in HDF5 file
-    h5open(h5_filename, "r+") do h5
-        for key in ["cuts_cpx_lhs", "cuts_cpx_rhs"]
-            if haskey(h5, key)
-                delete_object(h5, key)
-            end
-        end
-        write(h5, "cuts_cpx_lhs", cuts_lhs_matrix)
-        write(h5, "cuts_cpx_rhs", cuts_rhs)
-    end
+    
+    @info "Storing $(length(cuts_rhs)) CPLEX cuts..."
+    h5 = Hdf5Sample(h5_filename)
+    h5.put_sparse("cuts_cpx_lhs", cuts_lhs)
+    h5.put_array("cuts_cpx_rhs", cuts_rhs)
+    h5.file.close()
 
     return
 end

src/MIPLearn.jl

@@ -4,6 +4,46 @@
 
 module MIPLearn
 
+using PyCall
+using SparseArrays
+
+global miplearn = PyNULL()
+global Hdf5Sample = PyNULL()
+
+to_str_array(values) = py"to_str_array"(values)
+
+from_str_array(values) = py"from_str_array"(values)
+
+function __init__()
+    copy!(miplearn, pyimport("miplearn"))
+    copy!(Hdf5Sample, miplearn.features.sample.Hdf5Sample)
+
+    py"""
+    import numpy as np
+
+    def to_str_array(values):
+        if values is None:
+            return None
+        return np.array(values, dtype="S")
+
+    def from_str_array(values):
+        return [v.decode() for v in values]
+    """    
+end
+
+function convert(::Type{SparseMatrixCSC}, o::PyObject)
+    I, J, V = pyimport("scipy.sparse").find(o)
+    return sparse(I .+ 1, J .+ 1, V, o.shape...)
+end
+
+function PyObject(m::SparseMatrixCSC)
+    pyimport("scipy.sparse").csc_matrix(
+        (m.nzval, m.rowval .- 1, m.colptr .- 1),
+        shape = size(m),
+    ).tocoo()
+end
+
+
 include("Cuts/BlackBox/cplex.jl")
 
 end # module