{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LLEPE: Liquid-Liquid Equilibrium Parameter Estimator\n",
    "\n",
    "Copyright © 2020, UChicago Argonne, LLC. All rights reserved.\n",
    "\n",
    "Released under the modified BSD license. See LICENSE for more details."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This script performs the filtered fit method and shows the parity plots for each of the species. This code requires that the elementz.xml file in the data folder has been copied into Cantera's data folder. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Filtering single component data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llepe import LLEPE, lmse_perturbed_obj, ind_lmse_perturbed_obj\n",
    "import numpy as np\n",
    "import itertools\n",
    "import pandas as pd\n",
    "import matplotlib\n",
    "from scipy.optimize import curve_fit\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "font = {'family': 'sans serif',\n",
    "        'size': 24}\n",
    "matplotlib.rc('font', **font)\n",
    "plt.rc('xtick', labelsize=18)\n",
    "plt.rc('ytick', labelsize=18)\n",
    "plt.rcParams['lines.linewidth'] = 4\n",
    "matplotlib.rcParams['lines.markersize'] = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nd\n",
      "# of removed data points = 1\n",
      "Pr\n",
      "# of removed data points = 3\n",
      "Ce\n",
      "# of removed data points = 0\n",
      "La\n",
      "# of removed data points = 0\n",
      "Dy\n",
      "# of removed data points = 0\n",
      "Sm\n",
      "# of removed data points = 0\n",
      "Y\n",
      "# of removed data points = 0\n"
     ]
    }
   ],
   "source": [
    "species_list = 'Nd,Pr,Ce,La,Dy,Sm,Y'.split(',')\n",
    "outlier_dfs = {}\n",
    "exp_dfs = {}\n",
    "for species in species_list:\n",
    "    print(species)\n",
    "    lower_species = species.lower()\n",
    "    labeled_data = pd.read_csv(\"../../data/csvs/labeled_zeroes_removed_{0}.csv\".format(lower_species))\n",
    "    exp_data = labeled_data.drop(labeled_data.columns[0], axis=1)\n",
    "\n",
    "    xml_file = '../../data/xmls/PC88A_HCL_{0}.xml'.format(species)\n",
    "    x_guess = [-4e6, -1e6]\n",
    "\n",
    "    len_outliers = 100\n",
    "    outlier_df = None\n",
    "    count = 0\n",
    "    while len_outliers > 0:\n",
    "        info_dict = {'{0}(H(A)2)3(org)_h0'.format(species): {'upper_element_name': 'species',\n",
    "                                       'upper_attrib_name': 'name',\n",
    "                                       'upper_attrib_value': '{0}(H(A)2)3(org)'.format(species),\n",
    "                                       'lower_element_name': 'h0',\n",
    "                                       'lower_attrib_name': None,\n",
    "                                       'lower_attrib_value': None,\n",
    "                                       'input_format': '{0}',\n",
    "                                       'input_value': x_guess[0]},\n",
    "                     '(HA)2(org)_h0': {'upper_element_name': 'species',\n",
    "                                       'upper_attrib_name': 'name',\n",
    "                                       'upper_attrib_value': '(HA)2(org)',\n",
    "                                       'lower_element_name': 'h0',\n",
    "                                       'lower_attrib_name': None,\n",
    "                                       'lower_attrib_value': None,\n",
    "                                       'input_format': '{0}',\n",
    "                                       'input_value': x_guess[1]}}\n",
    "        reeps_params = {\n",
    "            'exp_data': exp_data,\n",
    "            'phases_xml_filename': xml_file,\n",
    "            'opt_dict': info_dict,\n",
    "            'phase_names': ['HCl_electrolyte', 'PC88A_liquid'],\n",
    "            'aq_solvent_name': 'H2O(L)',\n",
    "            'extractant_name': '(HA)2(org)',\n",
    "            'diluant_name': 'dodecane',\n",
    "            'complex_names': ['{0}(H(A)2)3(org)'.format(species)],\n",
    "            'extracted_species_ion_names': ['{0}+++'.format(species)],\n",
    "            'aq_solvent_rho': 1000.0,\n",
    "            'extractant_rho': 960.0,\n",
    "            'diluant_rho': 750.0,\n",
    "            'objective_function': lmse_perturbed_obj\n",
    "        }\n",
    "        estimator = LLEPE(**reeps_params)\n",
    "        estimator.update_xml(reeps_params['opt_dict'])\n",
    "        obj_kwargs = {'species_list': [species], 'epsilon': 1e-100}\n",
    "        optimizer_kwargs = {\"method\": 'l-bfgs-b',\n",
    "                            \"bounds\": [(1e-1, 1e1), (1e-1, 1e1)]}\n",
    "        opt_dict, obj_value = estimator.fit(\n",
    "            objective_kwargs=obj_kwargs,\n",
    "            optimizer_kwargs=optimizer_kwargs)\n",
    "        estimator.update_xml(opt_dict)\n",
    "        \n",
    "        pred_dict = estimator.get_predicted_dict()\n",
    "        exp_df = estimator.get_exp_df()\n",
    "        objectives = ind_lmse_perturbed_obj(pred_dict, exp_df, [species])\n",
    "        lower_quartile = np.percentile(objectives, 25)\n",
    "        upper_quartile = np.percentile(objectives, 75)\n",
    "        iqr = upper_quartile - lower_quartile\n",
    "        hi_bound = upper_quartile + 3 * iqr\n",
    "        cols = []\n",
    "        for col in exp_df.columns:\n",
    "            if \"org\" not in col:\n",
    "                cols.append(col)\n",
    "        exp_df['objective'] = objectives\n",
    "        if np.any(objectives > hi_bound) or np.median(objectives) > 10:\n",
    "            outlier_max = exp_df['objective'].max()\n",
    "            if outlier_df is None:\n",
    "                outlier_df = exp_df[exp_df['objective'] == outlier_max]\n",
    "            else:\n",
    "                outlier_df = pd.concat(\n",
    "                    [outlier_df,\n",
    "                     exp_df[exp_df['objective'] == outlier_max]])\n",
    "            exp_data = exp_df[exp_df['objective'] < outlier_max][cols]\n",
    "            len_outliers = 1\n",
    "            count += 1\n",
    "        else:\n",
    "            len_outliers = 0\n",
    "    \n",
    "    if outlier_df is not None:\n",
    "        outlier_dfs[species] = outlier_df\n",
    "        print(\"# of removed data points = {0}\".format(len(outlier_df)))\n",
    "    else:\n",
    "        print(\"# of removed data points = 0\")\n",
    "        \n",
    "    exp_dfs[species] = exp_df.iloc[:, 0:7]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fitting linear parameters to filtered single component data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nd\n",
      "slope = 3.00\n",
      "interecept = -800025.40\n",
      "Pr\n",
      "slope = 3.00\n",
      "interecept = -806414.23\n",
      "Ce\n",
      "slope = 3.00\n",
      "interecept = -799495.56\n",
      "La\n",
      "slope = 3.00\n",
      "interecept = -798511.25\n",
      "Dy\n",
      "slope = 3.00\n",
      "interecept = -807223.22\n",
      "Sm\n",
      "slope = 3.00\n",
      "interecept = -797560.15\n",
      "Y\n",
      "slope = 3.00\n",
      "interecept = -823279.05\n",
      "  species  slope      intercept\n",
      "0      Nd    3.0 -800025.401609\n",
      "1      Pr    3.0 -806414.234550\n",
      "2      Ce    3.0 -799495.559690\n",
      "3      La    3.0 -798511.252566\n",
      "4      Dy    3.0 -807223.215960\n",
      "5      Sm    3.0 -797560.154414\n",
      "6       Y    3.0 -823279.048921\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def linear(x, a, b):\n",
    "    return a * x + b\n",
    "\n",
    "\n",
    "lp_dict = {'species': [],\n",
    "           'slope': [],\n",
    "           'intercept': []}\n",
    "species_list = 'Nd,Pr,Ce,La,Dy,Sm,Y'.split(',')\n",
    "\n",
    "n_guesses = 11\n",
    "complex_guesses = -4e6 * np.logspace(-1, 1, n_guesses)\n",
    "extractant_guesses = -1e6 * np.logspace(-1, 1, n_guesses)\n",
    "x_guesses = list(itertools.product(*[complex_guesses, extractant_guesses]))\n",
    "guess_ext_h0 = 0\n",
    "for species in species_list:\n",
    "    print(species)\n",
    "    lower_species = species.lower()\n",
    "    \n",
    "    exp_data = exp_dfs[species]\n",
    "    xml_file = '../../data/xmls/PC88A_HCL_{0}.xml'.format(species)\n",
    "    opt_values = {\n",
    "        'optimizer': [],\n",
    "        'species': [],\n",
    "        'parameter': [],\n",
    "        'value': [],\n",
    "        'obj_value': [],\n",
    "        'guess': []}\n",
    "    new_x_guesses = []\n",
    "\n",
    "    for x_guess in x_guesses:\n",
    "        info_dict = {'{0}(H(A)2)3(org)_h0'.format(species): {'upper_element_name': 'species',\n",
    "                                       'upper_attrib_name': 'name',\n",
    "                                       'upper_attrib_value': '{0}(H(A)2)3(org)'.format(species),\n",
    "                                       'lower_element_name': 'h0',\n",
    "                                       'lower_attrib_name': None,\n",
    "                                       'lower_attrib_value': None,\n",
    "                                       'input_format': '{0}',\n",
    "                                       'input_value': x_guess[0]},\n",
    "                     '(HA)2(org)_h0': {'upper_element_name': 'species',\n",
    "                                       'upper_attrib_name': 'name',\n",
    "                                       'upper_attrib_value': '(HA)2(org)',\n",
    "                                       'lower_element_name': 'h0',\n",
    "                                       'lower_attrib_name': None,\n",
    "                                       'lower_attrib_value': None,\n",
    "                                       'input_format': '{0}',\n",
    "                                       'input_value': x_guess[1]}}\n",
    "        reeps_params = {\n",
    "            'exp_data': exp_data,\n",
    "            'phases_xml_filename': xml_file,\n",
    "            'opt_dict': info_dict,\n",
    "            'phase_names': ['HCl_electrolyte', 'PC88A_liquid'],\n",
    "            'aq_solvent_name': 'H2O(L)',\n",
    "            'extractant_name': '(HA)2(org)',\n",
    "            'diluant_name': 'dodecane',\n",
    "            'complex_names': ['{0}(H(A)2)3(org)'.format(species)],\n",
    "            'extracted_species_ion_names': ['{0}+++'.format(species)],\n",
    "            'aq_solvent_rho': 1000.0,\n",
    "            'extractant_rho': 960.0,\n",
    "            'diluant_rho': 750.0,\n",
    "            'objective_function': lmse_perturbed_obj\n",
    "        }\n",
    "        estimator = LLEPE(**reeps_params)\n",
    "        estimator.update_xml(reeps_params['opt_dict'])\n",
    "        exp_df = estimator.get_exp_df()\n",
    "        pred_dict = estimator.get_predicted_dict()\n",
    "        init_obj_value = lmse_perturbed_obj(pred_dict, exp_df, [species], 1e-100)\n",
    "        obj_kwargs = {'species_list': [species], 'epsilon': 1e-100}\n",
    "        optimizer_kwargs = {\"method\": 'l-bfgs-b',\n",
    "                            \"bounds\": [(1e-1, 1e1), (1e-1, 1e1)]}\n",
    "        opt_dict, obj_value = estimator.fit(objective_kwargs=obj_kwargs,\n",
    "                                            optimizer_kwargs=optimizer_kwargs)\n",
    "        estimator.update_xml(opt_dict)\n",
    "        \n",
    "        keys = list(opt_dict.keys())\n",
    "        info1 = ['lbfgsb',\n",
    "                 keys[0],\n",
    "                 'h0',\n",
    "                 opt_dict[keys[0]]['input_value'],\n",
    "                 obj_value,\n",
    "                 x_guess[0]\n",
    "                 ]\n",
    "        info2 = ['lbfgsb',\n",
    "                 keys[1],\n",
    "                 'h0',\n",
    "                 opt_dict[keys[1]]['input_value'],\n",
    "                 obj_value,\n",
    "                 x_guess[1]\n",
    "                 ]\n",
    "        if obj_value > init_obj_value*0.95:\n",
    "            continue\n",
    "        opt_values_keys = opt_values.keys()\n",
    "        for ind, key in enumerate(opt_values_keys):\n",
    "            opt_values[key].append(info1[ind])\n",
    "        for ind, key in enumerate(opt_values_keys):\n",
    "            opt_values[key].append(info2[ind])\n",
    "        new_x_guesses.append(x_guess)\n",
    "    x_guesses = new_x_guesses\n",
    "    opt_df = pd.DataFrame(opt_values)\n",
    "    min_ext_h0 = opt_df[opt_df['species'] == '(HA)2(org)_h0']['value'].min()\n",
    "    if min_ext_h0 < guess_ext_h0:\n",
    "        guess_ext_h0 = min_ext_h0\n",
    "    complex_df = opt_df[opt_df['species'] != '(HA)2(org)_h0']\n",
    "    ext_df = opt_df[opt_df['species'] == '(HA)2(org)_h0']\n",
    "    p_opt, p_cov = curve_fit(linear, ext_df['value'].values,\n",
    "                             complex_df['value'].values)\n",
    "    slope, intercept = p_opt\n",
    "    lp_dict['species'].append(species)\n",
    "    lp_dict['slope'].append(slope)\n",
    "    lp_dict['intercept'].append(intercept)\n",
    "    print(\"slope = {0:.2f}\".format(slope))\n",
    "    print(\"interecept = {0:.2f}\".format(intercept))\n",
    "\n",
    "lp_df = pd.DataFrame(lp_dict)\n",
    "print(lp_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fitting extractant enthalpy to unfiltered combined single and multicomponent dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\tq220\\documents\\tits things\\anl research\\gitlab\\parameter-estimation\\llepe\\llepe.py:988: RuntimeWarning: invalid value encountered in true_divide\n",
      "  d_array = extracted_species_org_array / extracted_species_aq_array\n"
     ]
    }
   ],
   "source": [
    "def ext_to_complex(h0, dummy_holder_for_custom_dict, species, linear_params):\n",
    "    row = linear_params[linear_params['species'] == species]\n",
    "\n",
    "    return row['slope'].values[0] * h0[0] + row['intercept'].values[0]\n",
    "\n",
    "labeled_data = pd.read_csv(\"../../data/csvs/zeroes_removed_PC88A_HCL_NdPrCeLaDySmY.csv\")\n",
    "exp_data = labeled_data.drop(labeled_data.columns[0], axis=1)\n",
    "xml_file = '../../data/xmls/PC88A_HCL_NdPrCeLaDySmY.xml'\n",
    "complex_names = ['{0}(H(A)2)3(org)'.format(species)\n",
    "                 for species in species_list]\n",
    "dependant_params_dict = {}\n",
    "for species, complex_name in zip(species_list, complex_names):\n",
    "    inner_dict = {'upper_element_name': 'species',\n",
    "                  'upper_attrib_name': 'name',\n",
    "                  'upper_attrib_value': complex_name,\n",
    "                  'lower_element_name': 'h0',\n",
    "                  'lower_attrib_name': None,\n",
    "                  'lower_attrib_value': None,\n",
    "                  'input_format': '{0}',\n",
    "                  'function': ext_to_complex,\n",
    "                  'kwargs': {\"species\": species,\n",
    "                             \"linear_params\": lp_df},\n",
    "                  'independent_params': '(HA)2(org)_h0'}\n",
    "    dependant_params_dict['{0}_h0'.format(complex_name)] = inner_dict\n",
    "info_dict = {'(HA)2(org)_h0': {'upper_element_name': 'species',\n",
    "                                  'upper_attrib_name': 'name',\n",
    "                                  'upper_attrib_value': '(HA)2(org)',\n",
    "                                  'lower_element_name': 'h0',\n",
    "                                  'lower_attrib_name': None,\n",
    "                                  'lower_attrib_value': None,\n",
    "                                  'input_format': '{0}',\n",
    "                                  'input_value': guess_ext_h0},\n",
    "                }\n",
    "reeps_params = {\n",
    "            'exp_data': exp_data,\n",
    "            'phases_xml_filename': xml_file,\n",
    "            'opt_dict': info_dict,\n",
    "            'phase_names': ['HCl_electrolyte', 'PC88A_liquid'],\n",
    "            'aq_solvent_name': 'H2O(L)',\n",
    "            'extractant_name': '(HA)2(org)',\n",
    "            'diluant_name': 'dodecane',\n",
    "            'complex_names': complex_names,\n",
    "            'extracted_species_ion_names': ['{0}+++'.format(species)\n",
    "                                            for species in species_list],\n",
    "            'aq_solvent_rho': 1000.0,\n",
    "            'extractant_rho': 960.0,\n",
    "            'diluant_rho': 750.0,\n",
    "            'objective_function': lmse_perturbed_obj\n",
    "        }\n",
    "estimator = LLEPE(**reeps_params)\n",
    "estimator.set_dependant_params_dict(dependant_params_dict)\n",
    "estimator.update_xml(reeps_params['opt_dict'],\n",
    "                     dependant_params_dict=dependant_params_dict)\n",
    "obj_kwargs = {'species_list': species_list,\n",
    "              'epsilon': 1e-100}\n",
    "optimizer_kwargs = {\"method\": 'l-bfgs-b',\n",
    "                    \"bounds\": [(1e-1, 1e1)]}\n",
    "opt_dict, obj_value = estimator.fit(\n",
    "    objective_kwargs=obj_kwargs,\n",
    "    optimizer_kwargs=optimizer_kwargs)\n",
    "estimator.update_xml(opt_dict,\n",
    "                     dependant_params_dict=dependant_params_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Estimated standard enthalpy values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PC88A:\t\t\t-1059541.40 J/mol\n",
      "Nd-PC88A complex:\t-3978649.62 J/mol\n",
      "Pr-PC88A complex:\t-3985038.44 J/mol\n",
      "Ce-PC88A complex:\t-3978119.77 J/mol\n",
      "La-PC88A complex:\t-3977135.47 J/mol\n",
      "Dy-PC88A complex:\t-3985847.43 J/mol\n",
      "Sm-PC88A complex:\t-3976184.36 J/mol\n",
      "Y-PC88A complex:\t-4001903.25 J/mol\n"
     ]
    }
   ],
   "source": [
    "print('PC88A:\\t\\t\\t{0:.2f} J/mol'.format(opt_dict['(HA)2(org)_h0']['input_value']))\n",
    "for species in species_list:\n",
    "    est_ext_h0 = opt_dict['(HA)2(org)_h0']['input_value']\n",
    "    est_comp_h0 = ext_to_complex([est_ext_h0], 'bla', species, lp_df)\n",
    "    print('{0}-PC88A complex:\\t{1:.2f} J/mol'.format(species, est_comp_h0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plotting organic phase parity plots for complete unfiltered dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 65292 characters]\n",
      "text/plain": [
       "<Figure size 743.226x572.185 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 58144 characters]\n",
      "text/plain": [
       "<Figure size 743.226x572.185 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 55556 characters]\n",
      "text/plain": [
       "<Figure size 743.226x572.185 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 53120 characters]\n",
      "text/plain": [
       "<Figure size 743.226x572.185 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 57584 characters]\n",
      "text/plain": [
       "<Figure size 743.226x572.185 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 55984 characters]\n",
      "text/plain": [
       "<Figure size 743.226x572.185 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 59848 characters]\n",
      "text/plain": [
       "<Figure size 743.226x572.185 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "feed_cols = []\n",
    "labeled_complete_data = pd.read_csv(\"../../data/csvs/PC88A_HCL_NdPrCeLaDySmY.csv\")\n",
    "complete_exp_df = labeled_data.drop(labeled_data.columns[0], axis=1)\n",
    "estimator.set_exp_df(complete_exp_df)\n",
    "exp_data = estimator.get_exp_df()\n",
    "for col in exp_data.columns:\n",
    "    if 'aq_i' in col:\n",
    "        feed_cols.append(col)\n",
    "exp_data['total_re'] = exp_data[feed_cols].sum(axis=1)\n",
    "for species in species_list:\n",
    "    fig, ax = estimator.parity_plot('{0}_org_eq'.format(species),\n",
    "                                    c_data=\n",
    "                                    exp_data['total_re'].values,\n",
    "                                    c_label='Feed total RE '\n",
    "                                            'molarity (mol/L)',\n",
    "                                    print_r_squared=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
