Make python classes available in Julia

src/MIPLearn.jl

@@ -7,15 +7,24 @@ module MIPLearn
 using PyCall
 using SparseArrays
 
-include("problems/setcover.jl")
+include("Cuts/BlackBox/cplex.jl")
+
+include("collectors.jl")
+include("components.jl")
+include("extractors.jl")
 include("io.jl")
+include("problems/setcover.jl")
 include("solvers/jump.jl")
-include("Cuts/BlackBox/cplex.jl")
+include("solvers/learning.jl")
 
 function __init__()
-    __init_problems_setcover__()
+    __init_collectors__()
+    __init_components__()
+    __init_extractors__()
     __init_io__()
+    __init_problems_setcover__()
     __init_solvers_jump__()
+    __init_solvers_learning__()
 end
 
 end # module

src/collectors.jl

@@ -0,0 +1,7 @@
+global BasicCollector = PyNULL()
+
+function __init_collectors__()
+    copy!(BasicCollector, pyimport("miplearn.collectors.basic").BasicCollector)
+end
+
+export BasicCollector

src/components.jl

@@ -0,0 +1,65 @@
+
+global MinProbabilityClassifier = PyNULL()
+global SingleClassFix = PyNULL()
+global PrimalComponentAction = PyNULL()
+global SetWarmStart = PyNULL()
+global FixVariables = PyNULL()
+global EnforceProximity = PyNULL()
+global ExpertPrimalComponent = PyNULL()
+global IndependentVarsPrimalComponent = PyNULL()
+global JointVarsPrimalComponent = PyNULL()
+global SolutionConstructor = PyNULL()
+global MemorizingPrimalComponent = PyNULL()
+global SelectTopSolutions = PyNULL()
+global MergeTopSolutions = PyNULL()
+
+function __init_components__()
+    copy!(
+        MinProbabilityClassifier,
+        pyimport("miplearn.classifiers.minprob").MinProbabilityClassifier,
+    )
+    copy!(SingleClassFix, pyimport("miplearn.classifiers.singleclass").SingleClassFix)
+    copy!(
+        PrimalComponentAction,
+        pyimport("miplearn.components.primal.actions").PrimalComponentAction,
+    )
+    copy!(SetWarmStart, pyimport("miplearn.components.primal.actions").SetWarmStart)
+    copy!(FixVariables, pyimport("miplearn.components.primal.actions").FixVariables)
+    copy!(EnforceProximity, pyimport("miplearn.components.primal.actions").EnforceProximity)
+    copy!(
+        ExpertPrimalComponent,
+        pyimport("miplearn.components.primal.expert").ExpertPrimalComponent,
+    )
+    copy!(
+        IndependentVarsPrimalComponent,
+        pyimport("miplearn.components.primal.indep").IndependentVarsPrimalComponent,
+    )
+    copy!(
+        JointVarsPrimalComponent,
+        pyimport("miplearn.components.primal.joint").JointVarsPrimalComponent,
+    )
+    copy!(
+        SolutionConstructor,
+        pyimport("miplearn.components.primal.mem").SolutionConstructor,
+    )
+    copy!(
+        MemorizingPrimalComponent,
+        pyimport("miplearn.components.primal.mem").MemorizingPrimalComponent,
+    )
+    copy!(SelectTopSolutions, pyimport("miplearn.components.primal.mem").SelectTopSolutions)
+    copy!(MergeTopSolutions, pyimport("miplearn.components.primal.mem").MergeTopSolutions)
+end
+
+export MinProbabilityClassifier,
+    SingleClassFix,
+    PrimalComponentAction,
+    SetWarmStart,
+    FixVariables,
+    EnforceProximity,
+    ExpertPrimalComponent,
+    IndependentVarsPrimalComponent,
+    JointVarsPrimalComponent,
+    SolutionConstructor,
+    MemorizingPrimalComponent,
+    SelectTopSolutions,
+    MergeTopSolutions

src/extractors.jl

@@ -0,0 +1,18 @@
+
+global FeaturesExtractor = PyNULL()
+global AlvLouWeh2017Extractor = PyNULL()
+global DummyExtractor = PyNULL()
+global H5FieldsExtractor = PyNULL()
+
+function __init_extractors__()
+    copy!(FeaturesExtractor, pyimport("miplearn.extractors.abstract").FeaturesExtractor)
+    copy!(
+        AlvLouWeh2017Extractor,
+        pyimport("miplearn.extractors.AlvLouWeh2017").AlvLouWeh2017Extractor,
+    )
+    copy!(DummyExtractor, pyimport("miplearn.extractors.dummy").DummyExtractor)
+    copy!(H5FieldsExtractor, pyimport("miplearn.extractors.fields").H5FieldsExtractor)
+
+end
+
+export FeaturesExtractor, AlvLouWeh2017Extractor, DummyExtractor, H5FieldsExtractor

src/io.jl

@@ -1,4 +1,6 @@
 global H5File = PyNULL()
+global write_pkl_gz = PyNULL()
+global read_pkl_gz = PyNULL()
 
 to_str_array(values) = py"to_str_array"(values)
 
@@ -6,6 +8,8 @@ from_str_array(values) = py"from_str_array"(values)
 
 function __init_io__()
     copy!(H5File, pyimport("miplearn.h5").H5File)
+    copy!(write_pkl_gz, pyimport("miplearn.io").write_pkl_gz)
+    copy!(read_pkl_gz, pyimport("miplearn.io").read_pkl_gz)
 
     py"""
     import numpy as np
@@ -32,4 +36,4 @@ function PyObject(m::SparseMatrixCSC)
     ).tocoo()
 end
 
-export H5File
+export H5File, write_pkl_gz, read_pkl_gz

src/problems/setcover.jl

@@ -1,15 +1,18 @@
-global SetCoverData = PyNULL()
-global SetCoverGenerator = PyNULL()
-
 using JuMP
 using HiGHS
 
+global SetCoverData = PyNULL()
+global SetCoverGenerator = PyNULL()
+
 function __init_problems_setcover__()
     copy!(SetCoverData, pyimport("miplearn.problems.setcover").SetCoverData)
     copy!(SetCoverGenerator, pyimport("miplearn.problems.setcover").SetCoverGenerator)
 end
 
-function build_setcover_model(data; optimizer = HiGHS.Optimizer)
+function build_setcover_model(data::Any; optimizer = HiGHS.Optimizer)
+    if data isa String
+        data = read_pkl_gz(data)
+    end
     model = Model(optimizer)
     set_silent(model)
     n_elements, n_sets = size(data.incidence_matrix)

src/solvers/jump.jl

@@ -250,12 +250,14 @@ end
 function _fix_variables(model::JuMP.Model, var_names, var_values, stats)
     vars = [variable_by_name(model, v) for v in var_names]
     for (i, var) in enumerate(vars)
-        fix(var, var_values[i], force=true)
+        fix(var, var_values[i], force = true)
     end
 end
 
 function _optimize(model::JuMP.Model)
     optimize!(model)
+    flush(stdout)
+    Libc.flush_cstdio()
 end
 
 function _relax(model::JuMP.Model)
@@ -325,3 +327,5 @@ function __init_solvers_jump__()
     end
     copy!(JumpModel, Class)
 end
+
+export JumpModel

src/solvers/learning.jl

@@ -0,0 +1,7 @@
+global LearningSolver = PyNULL()
+
+function __init_solvers_learning__()
+    copy!(LearningSolver, pyimport("miplearn.solvers.learning").LearningSolver)
+end
+
+export LearningSolver

test/Project.toml

@@ -4,6 +4,7 @@ authors = ["Alinson S. Xavier <git@axavier.org>"]
 version = "0.1.0"
 
 [deps]
+Glob = "c27321d9-0574-5035-807b-f59d2c89b15c"
 HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
 HiGHS = "87dc4568-4c63-4d18-b0c0-bb2238e4078b"
 JuMP = "4076af6c-e467-56ae-b986-b466b2749572"

test/src/MIPLearnT.jl

@@ -4,21 +4,26 @@ using Test
 using Logging
 using JuliaFormatter
 using HiGHS
+using Glob
 
 BASEDIR = dirname(@__FILE__)
 FIXTURES = "$BASEDIR/../fixtures"
 
+include("fixtures.jl")
+
 include("Cuts/BlackBox/test_cplex.jl")
 include("problems/test_setcover.jl")
-include("test_h5.jl")
+include("test_io.jl")
 include("solvers/test_jump.jl")
+include("test_usage.jl")
 
 function runtests()
     @testset "MIPLearn" begin
         test_cuts_blackbox_cplex()
-        test_h5()
+        test_io()
         test_problems_setcover()
         test_solvers_jump()
+        test_usage()
     end
 end
 

test/src/fixtures.jl

@@ -0,0 +1,14 @@
+function fixture_setcover_data()
+    return SetCoverData(
+        costs = [5, 10, 12, 6, 8],
+        incidence_matrix = [
+            1 0 0 1 0
+            1 1 0 0 0
+            0 0 1 1 1
+        ],
+    )
+end
+
+function fixture_setcover_model()
+    return build_setcover_model(fixture_setcover_data())
+end

test/src/solvers/test_jump.jl

@@ -1,18 +1,6 @@
 using JuMP
 import MIPLearn: from_str_array, to_str_array
 
-function build_model()
-    data = SetCoverData(
-        costs = [5, 10, 12, 6, 8],
-        incidence_matrix = [
-            1 0 0 1 0
-            1 1 0 0 0
-            0 0 1 1 1
-        ],
-    )
-    return build_setcover_model(data)
-end
-
 function test_solvers_jump()
     test_solvers_jump_extract()
     test_solvers_jump_add_constrs()
@@ -51,7 +39,7 @@ function test_solvers_jump_extract()
         @test all(actual .≈ expected)
     end
 
-    model = build_model()
+    model = fixture_setcover_model()
     model.extract_after_load(h5)
     test_sparse(
         "static_constr_lhs",
@@ -106,7 +94,7 @@ end
 
 function test_solvers_jump_add_constrs()
     h5 = H5File(tempname(), "w")
-    model = build_model()
+    model = fixture_setcover_model()
     model.extract_after_load(h5)
     model.add_constrs(
         to_str_array(["x[2]", "x[3]"]),
@@ -124,12 +112,9 @@ end
 
 function test_solvers_jump_fix_vars()
     h5 = H5File(tempname(), "w")
-    model = build_model()
+    model = fixture_setcover_model()
     model.extract_after_load(h5)
-    model.fix_variables(
-        to_str_array(["x[2]", "x[3]"]),
-        [0, 0],
-    )
+    model.fix_variables(to_str_array(["x[2]", "x[3]"]), [0, 0])
     model.optimize()
     model.extract_after_mip(h5)
     @test all(h5.get_array("mip_var_values") .≈ [1, 0, 0, 0, 1])
@@ -138,7 +123,7 @@ end
 function test_solvers_jump_warm_starts()
     # TODO: Check presence of warm start on log file
     h5 = H5File(tempname(), "w")
-    model = build_model()
+    model = fixture_setcover_model()
     model.extract_after_load(h5)
     model.set_warm_starts(
         to_str_array(["x[0]", "x[1]", "x[2]", "x[3]", "x[4]"]),
@@ -149,8 +134,8 @@ end
 
 function test_solvers_jump_write()
     mps_filename = "$(tempname()).mps"
-    model = build_model()
+    model = fixture_setcover_model()
     model.write(mps_filename)
     @test isfile(mps_filename)
     rm(mps_filename)
-end
+end

test/src/test_io.jl

@@ -1,5 +1,18 @@
 using MIPLearn
 
+function test_io()
+    test_pkl_gz()
+    test_h5()
+end
+
+function test_pkl_gz()
+    original = Dict("K1" => 1, "K2" => [0, 1, 2], "K3" => "Hello")
+    dirname = tempdir()
+    MIPLearn.write_pkl_gz([original], dirname)
+    recovered = MIPLearn.read_pkl_gz("$dirname/00000.pkl.gz")
+    @test recovered == original
+end
+
 function test_h5()
     h5 = H5File(tempname(), "w")
     _test_roundtrip_scalar(h5, "A")

test/src/test_usage.jl

@@ -0,0 +1,40 @@
+
+function test_usage()
+    LogisticRegression = pyimport("sklearn.linear_model").LogisticRegression
+
+    @debug "Generating data files..."
+    dirname = tempdir()
+    data = [fixture_setcover_data()]
+    data_filenames = write_pkl_gz(data, dirname)
+    h5_filenames = ["$(f).h5" for f in data_filenames]
+
+    @debug "Setting up LearningSolver..."
+    solver = LearningSolver(
+        components = [
+            IndependentVarsPrimalComponent(
+                base_clf = SingleClassFix(
+                    MinProbabilityClassifier(
+                        base_clf = LogisticRegression(),
+                        thresholds = [0.95, 0.95],
+                    ),
+                ),
+                extractor = AlvLouWeh2017Extractor(),
+                action = SetWarmStart(),
+            ),
+        ],
+    )
+
+    @debug "Collecting training data..."
+    bc = BasicCollector()
+    bc.collect(data_filenames, build_setcover_model)
+
+    @debug "Training models..."
+    solver.fit(data_filenames)
+
+    @debug "Solving model..."
+    solver.optimize(data_filenames[1], build_setcover_model)
+
+    @debug "Checking solution..."
+    h5 = H5File(h5_filenames[1])
+    @test h5.get_scalar("mip_obj_value") == 11.0
+end