ANL-CEEESA/MIPLearn
1
0
Fork 0
mirror of https://github.com/ANL-CEEESA/MIPLearn.git synced 2025-12-06 01:18:52 -06:00
Code Issues Packages Projects Releases Wiki Activity

Update 0.3 docs

Browse Source
This commit is contained in:
Alinson S. Xavier
2023-06-08 09:18:49 -05:00
parent 3d4a6a87a8
commit d9d44ce4b2
45 changed files with 7925 additions and 3158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

358
0.3/guide/collectors/index.html
View File

@@ -5,7 +5,7 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>2. Training Data Collectors &#8212; MIPLearn 0.3</title>
<title>5. Training Data Collectors &#8212; MIPLearn 0.3</title>
<link href="../../_static/css/theme.css" rel="stylesheet" />
<link href="../../_static/css/index.c5995385ac14fb8791e8eb36b4908be2.css" rel="stylesheet" />
@@ -24,6 +24,7 @@
<link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../../_static/sphinx-book-theme.acff12b8f9c144ce68a297486a2fa670.css" type="text/css" />
<link rel="stylesheet" type="text/css" href="../../_static/nbsphinx-code-cells.css" />
<link rel="stylesheet" type="text/css" href="../../_static/custom.css" />
<link rel="preload" as="script" href="../../_static/js/index.1c5a1a01449ed65a7b51.js">
@@ -38,8 +39,8 @@
<script type="text/x-mathjax-config">MathJax.Hub.Config({"tex2jax": {"inlineMath": [["\\(", "\\)"]], "displayMath": [["\\[", "\\]"]], "processRefs": false, "processEnvironments": false}})</script>
<link rel="index" title="Index" href="../../genindex/" />
<link rel="search" title="Search" href="../../search/" />
<link rel="next" title="3. Feature Extractors" href="../features/" />
<link rel="prev" title="1. Benchmark Problems" href="../problems/" />
<link rel="next" title="6. Feature Extractors" href="../features/" />
<link rel="prev" title="4. Benchmark Problems" href="../problems/" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="docsearch:language" content="en" />
@@ -68,6 +69,28 @@
</form><nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
<p class="caption">
<span class="caption-text">
Tutorials
</span>
</p>
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference internal" href="../../tutorials/getting-started-pyomo/">
1. Getting started (Pyomo)
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../../tutorials/getting-started-gurobipy/">
2. Getting started (Gurobipy)
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../../tutorials/getting-started-jump/">
3. Getting started (JuMP)
</a>
</li>
</ul>
<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -75,59 +98,59 @@
<ul class="current nav bd-sidenav">
<li class="toctree-l1">
<a class="reference internal" href="../problems/">
1. Benchmark Problems
4. Benchmark Problems
</a>
</li>
<li class="toctree-l1 current active">
<a class="current reference internal" href="#">
2. Training Data Collectors
5. Training Data Collectors
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../features/">
3. Feature Extractors
6. Feature Extractors
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../primal/">
4. Primal Components
7. Primal Components
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../solvers/">
5. Solvers
8. Solvers
</a>
</li>
</ul>
<p class="caption">
<span class="caption-text">
API Reference
Python API Reference
</span>
</p>
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference internal" href="../../api/problems/">
6. Benchmark Problems
9. Benchmark Problems
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../../api/collectors/">
7. Collectors &amp; Extractors
10. Collectors &amp; Extractors
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../../api/components/">
8. Components
11. Components
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../../api/solvers/">
9. Solvers
12. Solvers
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../../api/helpers/">
10. Helpers
13. Helpers
</a>
</li>
</ul>
@@ -199,12 +222,12 @@
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#Overview">
2.1. Overview
5.1. Overview
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#HDF5-Format">
2.2. HDF5 Format
5.2. HDF5 Format
</a>
<ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry">
@@ -216,7 +239,7 @@
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#Basic-collector">
2.3. Basic collector
5.3. Basic collector
</a>
<ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry">
@@ -242,279 +265,22 @@
<div>
<style>
/* CSS for nbsphinx extension */
/* remove conflicting styling from Sphinx themes */
div.nbinput.container div.prompt *,
div.nboutput.container div.prompt *,
div.nbinput.container div.input_area pre,
div.nboutput.container div.output_area pre,
div.nbinput.container div.input_area .highlight,
div.nboutput.container div.output_area .highlight {
border: none;
padding: 0;
margin: 0;
box-shadow: none;
}
div.nbinput.container > div[class*=highlight],
div.nboutput.container > div[class*=highlight] {
margin: 0;
}
div.nbinput.container div.prompt *,
div.nboutput.container div.prompt * {
background: none;
}
div.nboutput.container div.output_area .highlight,
div.nboutput.container div.output_area pre {
background: unset;
}
div.nboutput.container div.output_area div.highlight {
color: unset; /* override Pygments text color */
}
/* avoid gaps between output lines */
div.nboutput.container div[class*=highlight] pre {
line-height: normal;
}
/* input/output containers */
div.nbinput.container,
div.nboutput.container {
display: -webkit-flex;
display: flex;
align-items: flex-start;
margin: 0;
width: 100%;
}
@media (max-width: 540px) {
div.nbinput.container,
div.nboutput.container {
flex-direction: column;
}
}
/* input container */
div.nbinput.container {
padding-top: 5px;
}
/* last container */
div.nblast.container {
padding-bottom: 5px;
}
/* input prompt */
div.nbinput.container div.prompt pre {
color: #307FC1;
}
/* output prompt */
div.nboutput.container div.prompt pre {
color: #BF5B3D;
}
/* all prompts */
div.nbinput.container div.prompt,
div.nboutput.container div.prompt {
width: 4.5ex;
padding-top: 5px;
position: relative;
user-select: none;
}
div.nbinput.container div.prompt > div,
div.nboutput.container div.prompt > div {
position: absolute;
right: 0;
margin-right: 0.3ex;
}
@media (max-width: 540px) {
div.nbinput.container div.prompt,
div.nboutput.container div.prompt {
width: unset;
text-align: left;
padding: 0.4em;
}
div.nboutput.container div.prompt.empty {
padding: 0;
}
div.nbinput.container div.prompt > div,
div.nboutput.container div.prompt > div {
position: unset;
}
}
/* disable scrollbars and line breaks on prompts */
div.nbinput.container div.prompt pre,
div.nboutput.container div.prompt pre {
overflow: hidden;
white-space: pre;
}
/* input/output area */
div.nbinput.container div.input_area,
div.nboutput.container div.output_area {
-webkit-flex: 1;
flex: 1;
overflow: auto;
}
@media (max-width: 540px) {
div.nbinput.container div.input_area,
div.nboutput.container div.output_area {
width: 100%;
}
}
/* input area */
div.nbinput.container div.input_area {
border: 1px solid #e0e0e0;
border-radius: 2px;
/*background: #f5f5f5;*/
}
/* override MathJax center alignment in output cells */
div.nboutput.container div[class*=MathJax] {
text-align: left !important;
}
/* override sphinx.ext.imgmath center alignment in output cells */
div.nboutput.container div.math p {
text-align: left;
}
/* standard error */
div.nboutput.container div.output_area.stderr {
background: #fdd;
}
/* ANSI colors */
.ansi-black-fg { color: #3E424D; }
.ansi-black-bg { background-color: #3E424D; }
.ansi-black-intense-fg { color: #282C36; }
.ansi-black-intense-bg { background-color: #282C36; }
.ansi-red-fg { color: #E75C58; }
.ansi-red-bg { background-color: #E75C58; }
.ansi-red-intense-fg { color: #B22B31; }
.ansi-red-intense-bg { background-color: #B22B31; }
.ansi-green-fg { color: #00A250; }
.ansi-green-bg { background-color: #00A250; }
.ansi-green-intense-fg { color: #007427; }
.ansi-green-intense-bg { background-color: #007427; }
.ansi-yellow-fg { color: #DDB62B; }
.ansi-yellow-bg { background-color: #DDB62B; }
.ansi-yellow-intense-fg { color: #B27D12; }
.ansi-yellow-intense-bg { background-color: #B27D12; }
.ansi-blue-fg { color: #208FFB; }
.ansi-blue-bg { background-color: #208FFB; }
.ansi-blue-intense-fg { color: #0065CA; }
.ansi-blue-intense-bg { background-color: #0065CA; }
.ansi-magenta-fg { color: #D160C4; }
.ansi-magenta-bg { background-color: #D160C4; }
.ansi-magenta-intense-fg { color: #A03196; }
.ansi-magenta-intense-bg { background-color: #A03196; }
.ansi-cyan-fg { color: #60C6C8; }
.ansi-cyan-bg { background-color: #60C6C8; }
.ansi-cyan-intense-fg { color: #258F8F; }
.ansi-cyan-intense-bg { background-color: #258F8F; }
.ansi-white-fg { color: #C5C1B4; }
.ansi-white-bg { background-color: #C5C1B4; }
.ansi-white-intense-fg { color: #A1A6B2; }
.ansi-white-intense-bg { background-color: #A1A6B2; }
.ansi-default-inverse-fg { color: #FFFFFF; }
.ansi-default-inverse-bg { background-color: #000000; }
.ansi-bold { font-weight: bold; }
.ansi-underline { text-decoration: underline; }
div.nbinput.container div.input_area div[class*=highlight] > pre,
div.nboutput.container div.output_area div[class*=highlight] > pre,
div.nboutput.container div.output_area div[class*=highlight].math,
div.nboutput.container div.output_area.rendered_html,
div.nboutput.container div.output_area > div.output_javascript,
div.nboutput.container div.output_area:not(.rendered_html) > img{
padding: 5px;
margin: 0;
}
/* fix copybtn overflow problem in chromium (needed for 'sphinx_copybutton') */
div.nbinput.container div.input_area > div[class^='highlight'],
div.nboutput.container div.output_area > div[class^='highlight']{
overflow-y: hidden;
}
/* hide copybtn icon on prompts (needed for 'sphinx_copybutton') */
.prompt .copybtn {
display: none;
}
/* Some additional styling taken form the Jupyter notebook CSS */
.jp-RenderedHTMLCommon table,
div.rendered_html table {
border: none;
border-collapse: collapse;
border-spacing: 0;
color: black;
font-size: 12px;
table-layout: fixed;
}
.jp-RenderedHTMLCommon thead,
div.rendered_html thead {
border-bottom: 1px solid black;
vertical-align: bottom;
}
.jp-RenderedHTMLCommon tr,
.jp-RenderedHTMLCommon th,
.jp-RenderedHTMLCommon td,
div.rendered_html tr,
div.rendered_html th,
div.rendered_html td {
text-align: right;
vertical-align: middle;
padding: 0.5em 0.5em;
line-height: normal;
white-space: normal;
max-width: none;
border: none;
}
.jp-RenderedHTMLCommon th,
div.rendered_html th {
font-weight: bold;
}
.jp-RenderedHTMLCommon tbody tr:nth-child(odd),
div.rendered_html tbody tr:nth-child(odd) {
background: #f5f5f5;
}
.jp-RenderedHTMLCommon tbody tr:hover,
div.rendered_html tbody tr:hover {
background: rgba(66, 165, 245, 0.2);
}
</style>
<div class="section" id="Training-Data-Collectors">
<h1><span class="section-number">2. </span>Training Data Collectors<a class="headerlink" href="#Training-Data-Collectors" title="Permalink to this headline"></a></h1>
<div class="section" id="Training-Data-Collectors">
<h1><span class="section-number">5. </span>Training Data Collectors<a class="headerlink" href="#Training-Data-Collectors" title="Permalink to this headline"></a></h1>
<p>The first step in solving mixed-integer optimization problems with the assistance of supervised machine learning methods is solving a large set of training instances and collecting the raw training data. In this section, we describe the various training data collectors included in MIPLearn. Additionally, the framework follows the convention of storing all training data in files with a specific data format (namely, HDF5). In this section, we briefly describe this format and the rationale for
choosing it.</p>
<div class="section" id="Overview">
<h2><span class="section-number">2.1. </span>Overview<a class="headerlink" href="#Overview" title="Permalink to this headline"></a></h2>
<h2><span class="section-number">5.1. </span>Overview<a class="headerlink" href="#Overview" title="Permalink to this headline"></a></h2>
<p>In MIPLearn, a <strong>collector</strong> is a class that solves or analyzes the problem and collects raw data which may be later useful for machine learning methods. Collectors, by convention, take as input: (i) a list of problem data filenames, in gzipped pickle format, ending with <code class="docutils literal notranslate"><span class="pre">.pkl.gz</span></code>; (ii) a function that builds the optimization model, such as <code class="docutils literal notranslate"><span class="pre">build_tsp_model</span></code>. After processing is done, collectors store the training data in a HDF5 file located alongside with the problem data. For example, if
the problem data is stored in file <code class="docutils literal notranslate"><span class="pre">problem.pkl.gz</span></code>, then the collector writes to <code class="docutils literal notranslate"><span class="pre">problem.h5</span></code>. Collectors are, in general, very time consuming, as they may need to solve the problem to optimality, potentially multiple times.</p>
</div>
<div class="section" id="HDF5-Format">
<h2><span class="section-number">2.2. </span>HDF5 Format<a class="headerlink" href="#HDF5-Format" title="Permalink to this headline"></a></h2>
<h2><span class="section-number">5.2. </span>HDF5 Format<a class="headerlink" href="#HDF5-Format" title="Permalink to this headline"></a></h2>
<p>MIPLearn stores all training data in <a class="reference external" href="HDF5">HDF5</a> (Hierarchical Data Format, Version 5) files. The HDF format was originally developed by the <a class="reference external" href="https://en.wikipedia.org/wiki/National_Center_for_Supercomputing_Applications">National Center for Supercomputing Applications</a> (NCSA) for storing and organizing large amounts of data, and supports a variety of data types, including integers, floating-point numbers, strings, and arrays. Compared to other formats, such as CSV, JSON or SQLite, the
HDF5 format provides several advantages for MIPLearn, including:</p>
<ul class="simple">
<li><p><em>Storage of multiple scalars, vectors and matrices in a single file</em> — This allows MIPLearn to store all training data related to a given problem instance in a single file, which makes training data easier to store, organize and transfer.</p></li>
<li><p><em>High-performance partial I/O</em> — Partial I/O allows MIPLearn to read a single element from the training data (e.g. value of the optimal solution) without loading the entire file to memory or reading it from beginning to end, which dramatically improves performance and reduces memory requirements. This is especially important when processing a large number of training data files.</p></li>
<li><p><em>High-performance partial I/O</em> — Partial I/O allows MIPLearn to read a single element from the training data (e.g. value of the optimal solution) without loading the entire file to memory or reading it from beginning to end, which dramatically improves performance and reduces memory requirements. This is especially important when processing a large number of training data files.</p></li>
<li><p><em>On-the-fly compression</em> — HDF5 files can be transparently compressed, using the gzip method, which reduces storage requirements and accelerates network transfers.</p></li>
<li><p><em>Stable, portable and well-supported data format</em> — Training data files are typically expensive to generate. Having a stable and well supported data format ensures that these files remain usable in the future, potentially even by other non-Python MIP/ML frameworks.</p></li>
</ul>
@@ -524,7 +290,7 @@ automatically perform type-checking and gzip compression. The example below show
<div class="section" id="Example">
<h3>Example<a class="headerlink" href="#Example" title="Permalink to this headline"></a></h3>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[1]:
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[3]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
@@ -587,7 +353,7 @@ x5 = (2, 3) 0.68030757
</div>
</div>
<div class="section" id="Basic-collector">
<h2><span class="section-number">2.3. </span>Basic collector<a class="headerlink" href="#Basic-collector" title="Permalink to this headline"></a></h2>
<h2><span class="section-number">5.3. </span>Basic collector<a class="headerlink" href="#Basic-collector" title="Permalink to this headline"></a></h2>
<p><a class="reference external" href="../../api/collectors/#miplearn.collectors.basic.BasicCollector">BasicCollector</a> is the most fundamental collector, and performs the following steps:</p>
<ol class="arabic simple">
<li><p>Extracts all model data, such as objective function and constraint right-hand sides into numpy arrays, which can later be easily and efficiently accessed without rebuilding the model or invoking the solver;</p></li>
@@ -729,7 +495,7 @@ x5 = (2, 3) 0.68030757
<h3>Example<a class="headerlink" href="#id1" title="Permalink to this headline"></a></h3>
<p>The example below shows how to generate a few random instances of the traveling salesman problem, store its problem data, run the collector and print some of the training data to screen.</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[2]:
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[4]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">random</span>
@@ -741,7 +507,7 @@ x5 = (2, 3) 0.68030757
<span class="n">TravelingSalesmanGenerator</span><span class="p">,</span>
<span class="n">build_tsp_model</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span> <span class="nn">miplearn.io</span> <span class="kn">import</span> <span class="n">save</span>
<span class="kn">from</span> <span class="nn">miplearn.io</span> <span class="kn">import</span> <span class="n">write_pkl_gz</span>
<span class="kn">from</span> <span class="nn">miplearn.h5</span> <span class="kn">import</span> <span class="n">H5File</span>
<span class="kn">from</span> <span class="nn">miplearn.collectors.basic</span> <span class="kn">import</span> <span class="n">BasicCollector</span>
@@ -760,11 +526,11 @@ x5 = (2, 3) 0.68030757
<span class="p">)</span><span class="o">.</span><span class="n">generate</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span>
<span class="c1"># Save instance data to data/tsp/00000.pkl.gz, data/tsp/00001.pkl.gz, ...</span>
<span class="n">save</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="s2">&quot;data/tsp&quot;</span><span class="p">)</span>
<span class="n">write_pkl_gz</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="s2">&quot;data/tsp&quot;</span><span class="p">)</span>
<span class="c1"># Solve all instances and collect basic solution information. Process at most four</span>
<span class="c1"># instances in parallel, with a per-instance time limit of one hour.</span>
<span class="n">bc</span> <span class="o">=</span> <span class="n">BasicCollector</span><span class="p">(</span><span class="n">time_limit_sec</span><span class="o">=</span><span class="mi">3600</span><span class="p">)</span>
<span class="c1"># Solve all instances and collect basic solution information.</span>
<span class="c1"># Process at most four instances in parallel.</span>
<span class="n">bc</span> <span class="o">=</span> <span class="n">BasicCollector</span><span class="p">()</span>
<span class="n">bc</span><span class="o">.</span><span class="n">collect</span><span class="p">(</span><span class="n">glob</span><span class="p">(</span><span class="s2">&quot;data/tsp/*.pkl.gz&quot;</span><span class="p">),</span> <span class="n">build_tsp_model</span><span class="p">,</span> <span class="n">n_jobs</span><span class="o">=</span><span class="mi">4</span><span class="p">)</span>
<span class="c1"># Read and print some training data for the first instance.</span>
@@ -774,27 +540,17 @@ x5 = (2, 3) 0.68030757
</pre></div>
</div>
</div>
<div class="nboutput docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area stderr docutils container">
<div class="highlight"><pre>
/home/axavier/Software/anaconda3/envs/miplearn/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
from .autonotebook import tqdm as notebook_tqdm
</pre></div></div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Removing empty/corrupted h5 file: data/tsp/00000.h5
lp_obj_value = 2909.0
mip_obj_value = 2921.0
</pre></div></div>
</div>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[2]:
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[ ]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span>
@@ -811,8 +567,8 @@ mip_obj_value = 2921.0
<div class='prev-next-bottom'>
<a class='left-prev' id="prev-link" href="../problems/" title="previous page"><span class="section-number">1. </span>Benchmark Problems</a>
<a class='right-next' id="next-link" href="../features/" title="next page"><span class="section-number">3. </span>Feature Extractors</a>
<a class='left-prev' id="prev-link" href="../problems/" title="previous page"><span class="section-number">4. </span>Benchmark Problems</a>
<a class='right-next' id="next-link" href="../features/" title="next page"><span class="section-number">6. </span>Feature Extractors</a>
</div>
@@ -822,7 +578,7 @@ mip_obj_value = 2921.0
<div class="container">
<p>
&copy; Copyright 2020-2022, UChicago Argonne, LLC.<br/>
&copy; Copyright 2020-2023, UChicago Argonne, LLC.<br/>
</p>
</div>
</footer>