Merge remote-tracking branch 'origin'

Author: jtwhite79

.gitignore

@@ -77,3 +77,9 @@ pest++/sweep/sweep
 
 /dev_tests
 /benchmarks/g07
+
+# Sphinx / Doxygen docs build artifacts
+docs/_build/
+docs/_doxygen/
+docs/_manual_assets/
+docs/users_manual_processed.md

.readthedocs.yaml

@@ -0,0 +1,25 @@
+version: 2
+
+build:
+  os: ubuntu-24.04
+  tools:
+    python: "3.12"
+  apt_packages:
+    - doxygen
+  jobs:
+    pre_build:
+      # Generate Doxygen XML that Breathe will consume
+      - doxygen Doxyfile
+      # Preprocess users manual: convert HTML anchors to MyST targets
+      - python docs/preprocess_manual.py documentation/pestpp_users_manual.md docs/users_manual_processed.md
+      # Copy users manual media assets so Sphinx places them at _build/html/media/
+      - mkdir -p docs/_manual_assets/media
+      - cp -r documentation/media/. docs/_manual_assets/media/
+
+sphinx:
+  configuration: docs/conf.py
+  fail_on_warning: false
+
+python:
+  install:
+    - requirements: docs/requirements.txt

CMakeLists.txt

@@ -1,6 +1,6 @@
 # This is the top-level CMake file for PEST++
 cmake_minimum_required(VERSION 3.15)
-
+set(CMAKE_VERBOSE_MAKEFILE,ON)
 if(CMAKE_BUILD_TYPE STREQUAL "Debug")
   add_compile_definitions(_DEBUG)
 endif()

Doxyfile

@@ -0,0 +1,56 @@
+# Doxyfile for PEST++
+# Doxygen generates XML consumed by Breathe/Sphinx for ReadTheDocs.
+
+PROJECT_NAME           = "PEST++"
+PROJECT_NUMBER         = 5
+PROJECT_BRIEF          = "Parameter ESTimation tool ++ - uncertainty analysis and data assimilation"
+
+OUTPUT_DIRECTORY       = docs/_doxygen
+
+# Input source directories (project code only, not third-party Eigen)
+INPUT                  = src/libs/common \
+                         src/libs/pestpp_common \
+                         src/libs/run_managers \
+                         src/programs/gsa \
+                         src/programs/pest++ \
+                         src/programs/pestpp-da \
+                         src/programs/pestpp-ies \
+                         src/programs/pestpp-mou \
+                         src/programs/pestpp-opt \
+                         src/programs/pestpp-sqp \
+                         src/programs/sweep
+
+RECURSIVE              = YES
+FILE_PATTERNS          = *.cpp *.h
+
+# Exclude third-party libraries
+EXCLUDE                = src/libs/Eigen \
+                         src/libs/mio \
+                         src/libs/propack
+
+EXCLUDE_PATTERNS       = */Eigen/* */mio/* */propack/*
+
+# Extract everything — even undocumented items
+EXTRACT_ALL            = YES
+EXTRACT_PRIVATE        = NO
+EXTRACT_STATIC         = YES
+
+# Doxygen output: XML only (consumed by Breathe)
+GENERATE_HTML          = NO
+GENERATE_LATEX         = NO
+GENERATE_XML           = YES
+XML_OUTPUT             = xml
+XML_PROGRAMLISTING     = NO
+
+# Enable Doxygen to follow #include chains
+ENABLE_PREPROCESSING   = YES
+MACRO_EXPANSION        = YES
+EXPAND_ONLY_PREDEF     = NO
+
+# Source browsing
+SOURCE_BROWSER         = NO
+INLINE_SOURCES         = NO
+
+# Warnings
+WARN_IF_UNDOCUMENTED   = NO
+QUIET                  = YES

benchmarks/basic_tests.py

@@ -1403,6 +1403,7 @@ def fr_timeout_test():
     pst.pestpp_options["ies_num_reals"] = 10
     pst.pestpp_options["ies_include_base"] = False
     pst.pestpp_options["max_run_fail"] = 1
+    pst.pestpp_options["panther_ping_interval_secs"] = 2
 
     #pst.pestpp_options["panther_transfer_on_fail"] = "10par_xsec.list"
     pst.pestpp_options["panther_agent_freeze_on_fail"] = False

benchmarks/glm_test_base.py

@@ -801,13 +801,13 @@ def tenpar_fosm_external_stdev_test():
 
 if __name__ == "__main__":
     #freyberg_stress_test()
-    tenpar_xsec_stress_test()
+    #tenpar_xsec_stress_test()
     #tenpar_base_test()
     #tenpar_fosm_external_stdev_test()
     #tenpar_superpar_restart_test()
     #freyberg_basic_restart_test()
     # jac_diff_invest()
-    #new_fmt_load_test()
+    new_fmt_load_test()
     #tenpar_hotstart_test()
     #tenpar_normalform_test()
     #freyberg_stress_test()

benchmarks/ies_test_part4.py

@@ -2025,6 +2025,7 @@ def tenpar_adjust_weights_test():
     obs.loc[:,"standard_deviation"] = 0.1
     obs.loc[pst.obs_names[12:],"standard_deviation"] = 1e-11
     obs.loc[pst.obs_names[12:],"obsval"] = 1e-9
+    #these values are hard coded later...
     with open(os.path.join(template_d,"phi.csv"),'w') as f:
         f.write("og1,0.333333\n")
         f.write("og3,0.333333\n")
@@ -2044,6 +2045,29 @@ def tenpar_adjust_weights_test():
     pst.pestpp_options['ies_verbose_level'] = 4
     pst.pestpp_options["ies_bad_phi_sigma"] = -1.5
 
+    pst.control_data.noptmax = 0
+    pst.pestpp_options["ies_drop_conflicts"] = False
+    pst.pestpp_options["ies_phi_factor_file"] = "phi.csv"
+    pst_name = "pest_adj.pst"
+    pst.write(os.path.join(template_d,pst_name),version=2)
+    if os.path.exists(test_d):
+        shutil.rmtree(test_d)
+    shutil.copytree(template_d, test_d)
+    pyemu.os_utils.run("{0} {1}".format(exe_path, pst_name), cwd=test_d)
+    pst.set_res(os.path.join(test_d,pst_name.replace(".pst",".base.rei")))
+    print(pst.phi_components)
+    obs = pst.observation_data
+    aobs = pd.read_csv(os.path.join(test_d,pst_name.replace(".pst",".0.adjusted.obs_data.csv")),index_col=0)
+    obs.loc[aobs.index,"weight"] = aobs.weight.values
+    print(pst.phi_components)
+    print(pst.phi_components_normalized)
+    pcn = pst.phi_components_normalized
+    assert np.isclose(pcn["og3blahblah"],0.33333,1e-3)
+    assert np.isclose(pcn["og4yadayada"],0.33333,1e-3)
+    
+
+    pst.pestpp_options.pop("ies_phi_factor_file")
+
 
     pst.control_data.noptmax = 2
     pst.pestpp_options["ies_drop_conflicts"] = False
@@ -5007,10 +5031,48 @@ def large_invest():
     pyemu.os_utils.run("{0} pest.pst /e".format(exe_path),cwd=t_d)
 
 
+def tenpar_xsec_combined_autoadaloc_mm_stress_test():
+    """testing combined matrix + autoadaloc"""
+    model_d = "ies_10par_xsec"
+    test_d = os.path.join(model_d, "master_comb_aal_test1")
+    template_d = os.path.join(model_d, "test_template")
+
+    if not os.path.exists(template_d):
+        raise Exception("template_d {0} not found".format(template_d))
+    pst_name = os.path.join(template_d, "pest.pst")
+    pst = pyemu.Pst(pst_name)
+
+    if os.path.exists(test_d):
+        shutil.rmtree(test_d)
+    shutil.copytree(template_d, test_d)
+    pst.pestpp_options = {}
+    pst.pestpp_options["ies_num_reals"] = 30
+    
+    mat = pyemu.Matrix.from_names(pst.nnz_obs_names, pst.adj_par_names).to_dataframe()
+    mat.loc[:, :] = 1
+    mat.loc[:, pst.adj_par_names[::2]] = 0
+    pyemu.Matrix.from_dataframe(mat).to_ascii(os.path.join(template_d, "loc.mat"))
 
+    pst.pestpp_options["ies_localizer"] = "loc.mat"
+    pst.pestpp_options["ies_autoadaloc"] = True
+    pst.pestpp_options["ies_verbose_level"] = 3
+    pst.pestpp_options["ies_debug_fail_remainder"] = True
+    pst.pestpp_options["ies_debug_fail_subset"] = True
+    pst.pestpp_options["ies_debug_bad_phi"] = True
+    pst.pestpp_options["ies_multimodal_alpha"] = 0.99
+    
+    pst.control_data.noptmax = 3
 
+    pst.write(os.path.join(template_d, "pest_aal_restart.pst"))
+    pyemu.os_utils.start_workers(template_d, exe_path, "pest_aal_restart.pst", num_workers=10,
+                                 master_dir=test_d, verbose=True, worker_root=model_d,
+                                 port=port)
+    
 
 if __name__ == "__main__":
+    tenpar_xsec_combined_autoadaloc_mm_stress_test()
+
+    #tenpar_adjust_weights_test()
     #large_invest()
     #tenpar_fixed_transform_test()
     #tenpar_reg_factor_test()
@@ -5031,7 +5093,7 @@ def large_invest():
     #tenpar_reg_factor_test()
     #tenpar_high_phi_test()
     #tenpar_iqr_bad_phi_sigma_test()
-    multimodal_test()
+    #multimodal_test()
     #plot_mm1_sweep_results()
     #plot_mm1_results()
     #plot_mm1_results_seq()

benchmarks/mou_tests_2.py

@@ -2703,19 +2703,8 @@ def pi_output_test():
     assert len([f for f in os.listdir(m1) if "pi_pop" in f]) > 0
 
 
-def mou_bound_handling_test():
-    test_d = "freyberg_mou"
-    org_t_d = os.path.join(test_d,"template")
-    new_t_d = org_t_d.replace("template","boundtest")
-    if os.path.exists(new_t_d):
-        shutil.rmtree(new_t_d)
-    shutil.copytree(org_t_d,new_t_d)
-    
-    pyemu.os_utils.run("{0} test1.pst".format(exe_path),cwd=new_t_d)
-
 
 if __name__ == "__main__":
-    #mou_bound_handling_test()
     basic_pso_test()
     #test_restart_all()
     #chance_consistency_test()

docs/api/common.rst

@@ -0,0 +1,64 @@
+Common Library
+==============
+
+Core utilities, networking, and error handling shared across all PEST++ tools.
+
+Utilities
+---------
+
+.. doxygenfile:: utilities.cpp
+   :project: pestpp
+
+.. doxygenfile:: utilities.h
+   :project: pestpp
+
+Transformable
+-------------
+
+.. doxygenfile:: Transformable.cpp
+   :project: pestpp
+
+.. doxygenfile:: Transformable.h
+   :project: pestpp
+
+Network Package
+---------------
+
+.. doxygenfile:: network_package.cpp
+   :project: pestpp
+
+.. doxygenfile:: network_package.h
+   :project: pestpp
+
+Network Wrapper
+---------------
+
+.. doxygenfile:: network_wrapper.cpp
+   :project: pestpp
+
+.. doxygenfile:: network_wrapper.h
+   :project: pestpp
+
+Error Handling
+--------------
+
+.. doxygenfile:: pest_error.cpp
+   :project: pestpp
+
+.. doxygenfile:: pest_error.h
+   :project: pestpp
+
+System Variables
+----------------
+
+.. doxygenfile:: system_variables.cpp
+   :project: pestpp
+
+.. doxygenfile:: system_variables.h
+   :project: pestpp
+
+Fortran Wrappers
+----------------
+
+.. doxygenfile:: fortran_wrappers.cpp
+   :project: pestpp

docs/api/index.rst

@@ -0,0 +1,16 @@
+C++ API Reference
+=================
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Libraries
+
+   common
+   pestpp_common
+   run_managers
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Programs
+
+   programs