From 12723eed6b4dc74b688fb5cedeebde74739e0681 Mon Sep 17 00:00:00 2001
From: titusquah <46580668+titusquah@users.noreply.github.com>
Date: Wed, 15 Jul 2020 20:25:27 -0600
Subject: [PATCH] Updated iterative_fitter.py
---
.idea/dictionaries/Titus.xml | 1 +
.idea/workspace.xml | 54 ++--
.../xmls/PC88A_HCL_NdPrCeLaDySmY_w_pitzer.xml | 58 ++--
docs/Examples/iterative_fitter.py | 286 ++++++++++--------
llepe/llepe.py | 9 +-
5 files changed, 217 insertions(+), 191 deletions(-)
diff --git a/.idea/dictionaries/Titus.xml b/.idea/dictionaries/Titus.xml
index ff0cf04..d2062e7 100644
--- a/.idea/dictionaries/Titus.xml
+++ b/.idea/dictionaries/Titus.xml
@@ -18,6 +18,7 @@
maxiter
molality
ndarray
+ pitzer
pred
quah
rbfopt
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index c433b0b..7e2fcce 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -2,11 +2,10 @@
+
-
-
-
+
@@ -350,7 +349,14 @@
1594785960460
-
+
+ 1594852623912
+
+
+
+ 1594852623912
+
+
@@ -384,7 +390,8 @@
-
+
+
@@ -426,46 +433,46 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
-
+
+
-
-
+
+
-
-
+
+
-
+
@@ -517,11 +524,6 @@
9
-
- file://$PROJECT_DIR$/docs/examples/iterative_fitter.py
- 26
-
-
diff --git a/data/xmls/PC88A_HCL_NdPrCeLaDySmY_w_pitzer.xml b/data/xmls/PC88A_HCL_NdPrCeLaDySmY_w_pitzer.xml
index 934bb8a..00ea6ee 100644
--- a/data/xmls/PC88A_HCL_NdPrCeLaDySmY_w_pitzer.xml
+++ b/data/xmls/PC88A_HCL_NdPrCeLaDySmY_w_pitzer.xml
@@ -33,7 +33,7 @@
298.14999999999998
- -1459900.0
+ -1376877.1544988335
558.9824
0.0
@@ -50,7 +50,7 @@
298.14999999999998
- -5178185.714285715
+ -4926549.797810851
1117.965
0.0
@@ -67,7 +67,7 @@
298.14999999999998
- -5177400.0
+ -4935519.640701385
1117.965
0.0
@@ -85,7 +85,7 @@
298.14999999999998
- -5177400.0
+ -4928317.781440989
1117.965
0.0
@@ -103,7 +103,7 @@
298.14999999999998
- -5177400.0
+ -4927428.65973482
1117.965
0.0
@@ -120,7 +120,7 @@
298.14999999999998
- -5177400.0
+ -4935155.356789877
1117.965
0.0
@@ -138,7 +138,7 @@
298.14999999999998
- -5178028.571428572
+ -4944228.17930387
1117.965
0.0
@@ -157,7 +157,7 @@
298.14999999999998
- -5177400.0
+ -4925606.187988869
1117.965
0.0
@@ -245,10 +245,10 @@
- 1.229318338775545, 0.0, 0.0, 0.0, 0.0
- 7.705236327609295, 0.0, 0.0, 0.0, 0.0
+ 0.5879108393945309, 0.0, 0.0, 0.0, 0.0
+ 5.4483234694357385, 0.0, 0.0, 0.0, 0.0
0.0, 0.0, 0.0, 0.0, 0.0
- -0.020355000251503143, 0.0, 0.0, 0.0, 0.0
+ -0.02066999867229882, 0.0, 0.0, 0.0, 0.0
2
0
@@ -261,10 +261,10 @@
- 0.5877, 0.0, 0.0, 0.0, 0.0
- 5.206, 0.0, 0.0, 0.0, 0.0
+ 0.7459775851223264, 0.0, 0.0, 0.0, 0.0
+ 7.68392131299453, 0.0, 0.0, 0.0, 0.0
0.0, 0.0, 0.0, 0.0, 0.0
- -0.01969, 0.0, 0.0, 0.0, 0.0
+ -0.01963615126026457, 0.0, 0.0, 0.0, 0.0
2
0
@@ -277,10 +277,10 @@
- 1.199616118936196, 0.0, 0.0, 0.0, 0.0
- 13.377018086968107, 0.0, 0.0, 0.0, 0.0
+ 0.5929999713109059, 0.0, 0.0, 0.0, 0.0
+ 5.2769992484445485, 0.0, 0.0, 0.0, 0.0
0.0, 0.0, 0.0, 0.0, 0.0
- -0.023995730879246852, 0.0, 0.0, 0.0, 0.0
+ -0.024339999997603376, 0.0, 0.0, 0.0, 0.0
2
0
@@ -293,10 +293,10 @@
- 0.6129037619837437, 0.0, 0.0, 0.0, 0.0
- 5.359070120380359, 0.0, 0.0, 0.0, 0.0
+ 0.612950248542597, 0.0, 0.0, 0.0, 0.0
+ 5.3599329374557385, 0.0, 0.0, 0.0, 0.0
0.0, 0.0, 0.0, 0.0, 0.0
- -0.01969999915450973, 0.0, 0.0, 0.0, 0.0
+ -0.019699989216349984, 0.0, 0.0, 0.0, 0.0
2
0
@@ -309,10 +309,10 @@
- 0.4845924897449592, 0.0, 0.0, 0.0, 0.0
- 0.49100000000000005, 0.0, 0.0, 0.0, 0.0
+ 0.6005000182766684, 0.0, 0.0, 0.0, 0.0
+ 4.910000193972841, 0.0, 0.0, 0.0, 0.0
0.0, 0.0, 0.0, 0.0, 0.0
- -0.026227452256417736, 0.0, 0.0, 0.0, 0.0
+ -0.02618999999473301, 0.0, 0.0, 0.0, 0.0
2
0
@@ -325,10 +325,10 @@
- 0.6248, 0.0, 0.0, 0.0, 0.0
- 5.66, 0.0, 0.0, 0.0, 0.0
+ 0.9656352678202726, 0.0, 0.0, 0.0, 0.0
+ 8.514180603037651, 0.0, 0.0, 0.0, 0.0
0.0, 0.0, 0.0, 0.0, 0.0
- -0.01563, 0.0, 0.0, 0.0, 0.0
+ -0.015467323909969704, 0.0, 0.0, 0.0, 0.0
2
0
@@ -341,10 +341,10 @@
- 0.5977999158894437, 0.0, 0.0, 0.0, 0.0
- 5.278997937589517, 0.0, 0.0, 0.0, 0.0
+ 0.5977991228263208, 0.0, 0.0, 0.0, 0.0
+ 5.278978156246855, 0.0, 0.0, 0.0, 0.0
0.0, 0.0, 0.0, 0.0, 0.0
- -0.019920000010410537, 0.0, 0.0, 0.0, 0.0
+ -0.019920000110321332, 0.0, 0.0, 0.0, 0.0
2
0
diff --git a/docs/Examples/iterative_fitter.py b/docs/Examples/iterative_fitter.py
index 9092deb..a91ee43 100644
--- a/docs/Examples/iterative_fitter.py
+++ b/docs/Examples/iterative_fitter.py
@@ -1,36 +1,92 @@
-from scipy.optimize import curve_fit
import llepe
import pandas as pd
import numpy as np
+import json
-def linear(x, a, b):
- return a * x + b
+def mod_lin_param_df(lp_df, input_val, mini_species, mini_lin_param):
+ new_lp_df = lp_df.copy()
+ index = new_lp_df.index[new_lp_df['species'] == mini_species].tolist()[0]
+ new_lp_df.at[index, mini_lin_param] = input_val
+ return new_lp_df
species_list = 'Nd,Pr,Ce,La,Dy,Sm,Y'.split(',')
-pitzer_param_list = ['beta0', 'beta1', 'Cphi']
+pitzer_param_list = ['beta0', 'beta1']
+lin_param_list = ['slope', 'intercept']
meas_pitzer_param_df = pd.read_csv("../../data/csvs/may_pitzer_params.csv")
+pitzer_params_filename = "../../data/jsons/min_h0_pitzer_params.txt"
+with open(pitzer_params_filename) as file:
+ pitzer_params_dict = json.load(file)
+ext_h0_filename = "../../data/jsons/min_h0_guess_ext_h0.txt"
+with open(ext_h0_filename) as file:
+ ext_h0_dict = json.load(file)
labeled_data = pd.read_csv("../../data/csvs/"
"zeroes_removed_PC88A_HCL_NdPrCeLaDySmY.csv")
exp_data = labeled_data.drop(labeled_data.columns[0], axis=1)
xml_file = "PC88A_HCL_NdPrCeLaDySmY_w_pitzer.xml"
+lin_param_df = pd.read_csv("../../data/csvs"
+ "/zeroes_removed_min_h0_pitzer_lin_params.csv")
+estimator_params = {'exp_data': exp_data,
+ 'phases_xml_filename': xml_file,
+ 'phase_names': ['HCl_electrolyte', 'PC88A_liquid'],
+ 'aq_solvent_name': 'H2O(L)',
+ 'extractant_name': '(HA)2(org)',
+ 'diluant_name': 'dodecane',
+ 'complex_names': ['{0}(H(A)2)3(org)'.format(species)
+ for species in species_list],
+ 'extracted_species_ion_names': ['{0}+++'.format(species)
+ for species in
+ species_list],
+ 'aq_solvent_rho': 1000.0,
+ 'extractant_rho': 960.0,
+ 'diluant_rho': 750.0,
+ 'temp_xml_file_path': 'outputs/temp.xml',
+ 'objective_function': llepe.lmse_perturbed_obj
+ }
+estimator = llepe.LLEPE(**estimator_params)
+
+
+def ext_to_complex(h0, custom_obj_dict, mini_species):
+ linear_params = custom_obj_dict['lin_param_df']
+ row = linear_params[linear_params['species'] == mini_species]
+ return row['slope'].values[0] * h0[0] + row['intercept'].values[0]
+
+
+dependant_params_dict = {}
+for species, complex_name in zip(species_list,
+ estimator_params['complex_names']):
+ inner_dict = {'upper_element_name': 'species',
+ 'upper_attrib_name': 'name',
+ 'upper_attrib_value': complex_name,
+ 'lower_element_name': 'h0',
+ 'lower_attrib_name': None,
+ 'lower_attrib_value': None,
+ 'input_format': '{0}',
+ 'function': ext_to_complex,
+ 'kwargs': {"mini_species": species},
+ 'independent_params': '(HA)2(org)_h0'}
+ dependant_params_dict['{0}_h0'.format(complex_name)] = inner_dict
+estimator.update_xml(pitzer_params_dict)
+estimator.set_custom_objects_dict({'lin_param_df': lin_param_df})
+estimator.set_dependant_params_dict(dependant_params_dict)
+estimator.update_xml(ext_h0_dict,
+ dependant_params_dict=dependant_params_dict)
eps = 1e-4
mini_eps = 1e-8
-x_guesses = [[-5178500.0, -1459500.0],
- [-5178342.857142857, -1460300.0],
- [-5178342.857142857, -1459500.0],
- [-5178342.857142857, -1458300.0],
- [-5178185.714285715, -1459900.0],
- [-5178185.714285715, -1459500.0],
- [-5178185.714285715, -1459100.0],
- [-5178185.714285715, -1458300.0],
- [-5178028.571428572, -1459900.0],
- [-5178028.571428572, -1459100.0],
- [-5178028.571428572, -1458300.0],
- [-5177557.142857143, -1459900.0],
- [-5177400.0, -1460300.0]]
-pitzer_guess_df = meas_pitzer_param_df.copy()
+pitzer_guess_dict = {'species': [],
+ 'beta0': [],
+ 'beta1': []}
+for species in species_list:
+ pitzer_guess_dict['species'].append(species)
+ for param in pitzer_param_list:
+ mini_dict = pitzer_params_dict['{0}_{1}'.format(species, param)]
+ value = mini_dict['input_value']
+ pitzer_guess_dict[param].append(value)
+pitzer_guess_df = pd.DataFrame(pitzer_guess_dict)
+ext_h0_guess = ext_h0_dict['(HA)2(org)_h0']['input_value']
+lin_guess_df = lin_param_df.copy()
+
ignore_list = []
optimizer = 'scipy_minimize'
output_dict = {'iter': [0],
@@ -49,127 +105,78 @@ while rel_diff > 1e-4:
best_obj = 1e20
output_dict['iter'].append(i)
for species in species_list:
+ print(species)
lower_species = species.lower()
- opt_values = {
- '(HA)2(org)_h0': [],
- '{0}(H(A)2)3(org)_h0'.format(species): [],
- 'beta0': [],
- 'beta1': [],
- 'Cphi': [],
- 'obj_value': [],
- 'guess': []}
- for ind1, x_guess in enumerate(x_guesses):
- print(ind1)
- info_dict = {'(HA)2(org)_h0': {'upper_element_name': 'species',
- 'upper_attrib_name': 'name',
- 'upper_attrib_value': '(HA)2(org)',
- 'lower_element_name': 'h0',
- 'lower_attrib_name': None,
- 'lower_attrib_value': None,
- 'input_format': '{0}',
- 'input_value': x_guess[1]},
- '{0}(H(A)2)3(org)_h0'.format(species): {
- 'upper_element_name': 'species',
- 'upper_attrib_name': 'name',
- 'upper_attrib_value': '{0}(H(A)2)3(org)'.format(
- species),
- 'lower_element_name': 'h0',
- 'lower_attrib_name': None,
- 'lower_attrib_value': None,
- 'input_format': '{0}',
- 'input_value': x_guess[0]},
- }
- for pitzer_param in pitzer_param_list:
- if '{0}_{1}'.format(species, pitzer_param) not in ignore_list:
- pitzer_row = pitzer_guess_df[
- pitzer_guess_df['species'] == species]
- inner_dict = {'upper_element_name': 'binarySaltParameters',
- 'upper_attrib_name': 'cation',
- 'upper_attrib_value':
- '{0}+++'.format(species),
- 'lower_element_name': pitzer_param,
- 'lower_attrib_name': None,
- 'lower_attrib_value': None,
- 'input_format': ' {0}, 0.0, 0.0, 0.0, 0.0 ',
- 'input_value':
- pitzer_row[pitzer_param].values[0]
- }
- info_dict['{0}_{1}'.format(
- species, pitzer_param)] = inner_dict
- llepe_params = {
- 'exp_data': exp_data,
- 'phases_xml_filename': xml_file,
- 'opt_dict': info_dict,
- 'phase_names': ['HCl_electrolyte', 'PC88A_liquid'],
- 'aq_solvent_name': 'H2O(L)',
- 'extractant_name': '(HA)2(org)',
- 'diluant_name': 'dodecane',
- 'complex_names': ['{0}(H(A)2)3(org)'.format(species)
- for species in species_list],
- 'extracted_species_ion_names': ['{0}+++'.format(species)
- for species in species_list],
- 'aq_solvent_rho': 1000.0,
- 'extractant_rho': 960.0,
- 'diluant_rho': 750.0,
- 'objective_function': llepe.lmse_perturbed_obj,
- 'optimizer': optimizer,
- 'temp_xml_file_path': 'outputs/temp.xml'
- }
- estimator = llepe.LLEPE(**llepe_params)
- estimator.update_xml(llepe_params['opt_dict'])
- obj_kwargs = {'species_list': [species], 'epsilon': 1e-100}
- bounds = [(1e-1, 1e1)] * len(info_dict)
- optimizer_kwargs = {"method": 'l-bfgs-b',
- "bounds": bounds}
- opt_dict, obj_value = estimator.fit(
- objective_kwargs=obj_kwargs,
- optimizer_kwargs=optimizer_kwargs)
- if obj_value < best_obj:
- best_obj = obj_value
- keys = list(opt_dict.keys())
- info1 = [opt_dict[key]['input_value'] for key in keys]
- info1.append(obj_value)
- info1.append(x_guess)
- opt_values_keys = opt_values.keys()
- for ind, key in enumerate(opt_values_keys):
- opt_values[key].append(info1[ind])
- opt_value_df = pd.DataFrame(opt_values)
- p_opt, p_cov = curve_fit(linear,
- opt_value_df['(HA)2(org)_h0'].values,
- opt_value_df['{0}(H(A)2)3(org)_h0'.format(
- species)].values)
- slope, intercept = p_opt
- output_dict['{0}_slope'.format(species)].append(slope)
- output_dict['{0}_intercept'.format(species)].append(intercept)
- min_h0_df = opt_value_df[
- opt_value_df['(HA)2(org)_h0']
- == opt_value_df['(HA)2(org)_h0'].min()]
- update_pitzer_dict = {}
- for pitzer_param in pitzer_param_list:
- key_name = '{0}_{1}'.format(species, pitzer_param)
- output_dict[key_name].append(min_h0_df[pitzer_param].values[0])
-
- inner_dict = {'upper_element_name': 'binarySaltParameters',
- 'upper_attrib_name': 'cation',
- 'upper_attrib_value':
- '{0}+++'.format(species),
- 'lower_element_name': pitzer_param,
- 'lower_attrib_name': None,
- 'lower_attrib_value': None,
- 'input_format': ' {0}, 0.0, 0.0, 0.0, 0.0 ',
- 'input_value':
- min_h0_df[pitzer_param].values[0]
- }
- update_pitzer_dict['{0}_{1}'.format(
- species, pitzer_param)] = inner_dict
- estimator.update_xml(update_pitzer_dict)
+ info_dict = {'(HA)2(org)_h0': {'upper_element_name': 'species',
+ 'upper_attrib_name': 'name',
+ 'upper_attrib_value': '(HA)2(org)',
+ 'lower_element_name': 'h0',
+ 'lower_attrib_name': None,
+ 'lower_attrib_value': None,
+ 'input_format': '{0}',
+ 'input_value': ext_h0_guess}}
+ for pitzer_param in pitzer_param_list:
+ if '{0}_{1}'.format(species, pitzer_param) not in ignore_list:
+ pitzer_row = pitzer_guess_df[
+ pitzer_guess_df['species'] == species]
+ inner_dict = {'upper_element_name': 'binarySaltParameters',
+ 'upper_attrib_name': 'cation',
+ 'upper_attrib_value':
+ '{0}+++'.format(species),
+ 'lower_element_name': pitzer_param,
+ 'lower_attrib_name': None,
+ 'lower_attrib_value': None,
+ 'input_format': ' {0}, 0.0, 0.0, 0.0, 0.0 ',
+ 'input_value':
+ pitzer_row[pitzer_param].values[0]
+ }
+ info_dict['{0}_{1}'.format(
+ species, pitzer_param)] = inner_dict
+ for lin_param in lin_param_list:
+ if '{0}_{1}'.format(species, lin_param) not in ignore_list:
+ lin_row = lin_guess_df[lin_guess_df['species'] == species]
+ inner_dict = {'custom_object_name': 'lin_param_df',
+ 'function': mod_lin_param_df,
+ 'kwargs': {'mini_species': species,
+ 'mini_lin_param': lin_param},
+ 'input_value': lin_row[lin_param].values[0]
+ }
+ info_dict['{0}_{1}'.format(
+ species, lin_param)] = inner_dict
+ estimator.set_opt_dict(info_dict)
+ estimator.update_custom_objects_dict(info_dict)
+ estimator.update_xml(info_dict)
+ obj_kwargs = {'species_list': [species], 'epsilon': 1e-100}
+ bounds = [(1e-1, 1e1)] * len(info_dict)
+ optimizer_kwargs = {"method": 'l-bfgs-b',
+ "bounds": bounds}
+ opt_dict, obj_value = estimator.fit(
+ objective_kwargs=obj_kwargs,
+ optimizer_kwargs=optimizer_kwargs)
+ if obj_value < best_obj:
+ best_obj = obj_value
+ keys = list(opt_dict.keys())
+ for lin_param in lin_param_list:
+ mini_dict = opt_dict['{0}_{1}'.format(species, lin_param)]
+ value = mini_dict['input_value']
+ output_dict['{0}_{1}'.format(species, lin_param)].append(value)
+ for pitzer_param in pitzer_param_list:
+ mini_dict = opt_dict['{0}_{1}'.format(species, pitzer_param)]
+ value = mini_dict['input_value']
+ output_dict['{0}_{1}'.format(species, pitzer_param)].append(value)
+ estimator.update_custom_objects_dict(info_dict)
+ estimator.update_xml(opt_dict)
pitzer_guess_dict = {'species': [],
'beta0': [],
'beta1': [],
- 'Cphi': []}
+ }
+ lin_guess_dict = {'species': [],
+ 'slope': [],
+ 'intercept:': []}
for species in species_list:
pitzer_guess_dict['species'].append(species)
+ lin_guess_dict['species'].append(species)
for pitzer_param in pitzer_param_list:
value_list = output_dict['{0}_{1}'.format(species, pitzer_param)]
value = value_list[-1]
@@ -182,7 +189,20 @@ while rel_diff > 1e-4:
np.abs(value_list[-3]))
if mini_rel_diff1 < mini_eps and mini_rel_diff2 < mini_eps:
ignore_list.append('{0}_{1}'.format(species, pitzer_param))
+ for lin_param in lin_param_list:
+ value_list = output_dict['{0}_{1}'.format(species, lin_param)]
+ value = value_list[-1]
+ lin_guess_dict[lin_param].append(value)
+ if i > 2:
+ mini_rel_diff1 = np.abs(value_list[-1]
+ - value_list[-2]) / (
+ np.abs(value_list[-2]))
+ mini_rel_diff2 = np.abs(value_list[-2] - value_list[-3]) / (
+ np.abs(value_list[-3]))
+ if mini_rel_diff1 < mini_eps and mini_rel_diff2 < mini_eps:
+ ignore_list.append('{0}_{1}'.format(species, lin_param))
pitzer_guess_df = pd.DataFrame(pitzer_guess_dict)
+ lin_guess_df = pd.DataFrame(lin_guess_dict)
output_dict['best_obj'].append(best_obj)
output_df = pd.DataFrame(output_dict)
@@ -191,4 +211,4 @@ while rel_diff > 1e-4:
rel_diff = np.sum(np.abs(new_row - old_row) / np.abs(old_row))
output_dict['rel_diff'].append(rel_diff)
output_df = pd.DataFrame(output_dict)
- output_df.to_csv('outputs/iterative_fitter_output_df.csv')
+ output_df.to_csv('outputs/iterative_fitter_output.csv')
diff --git a/llepe/llepe.py b/llepe/llepe.py
index d397ce6..2b49ef7 100644
--- a/llepe/llepe.py
+++ b/llepe/llepe.py
@@ -1105,7 +1105,8 @@ class LLEPE:
if phases_xml_filename is None:
phases_xml_filename = self._phases_xml_filename
new_dict = copy.deepcopy(info_dict)
- dep_dict = dependant_params_dict
+ dep_dict = copy.deepcopy(dependant_params_dict)
+ custom_objects_dict = copy.deepcopy(self._custom_objects_dict)
if dep_dict is not None:
new_dict.update(dep_dict)
@@ -1123,10 +1124,12 @@ class LLEPE:
ind_vals = [new_dict[ind_param_name]['input_value']
for ind_param_name in ind_param_names]
if mod_kwargs is None:
- new_dict[param_name]['input_value'] = mod_func(ind_vals)
- else:
new_dict[param_name]['input_value'] = \
mod_func(ind_vals,
+ custom_objects_dict)
+ else:
+ new_dict[param_name]['input_value'] = \
+ mod_func(ind_vals, custom_objects_dict,
**mod_kwargs)
tree = ET.parse(phases_xml_filename)
root = tree.getroot()