Merge pull request #426 from jtwhite79/develop

general bug fixes

Author: jtwhite79

.gitignore

@@ -82,3 +82,4 @@ pest++/sweep/sweep
 docs/_build/
 docs/_doxygen/
 docs/_manual_assets/
+docs/users_manual_processed.md

.readthedocs.yaml

@@ -10,6 +10,8 @@ build:
     pre_build:
       # Generate Doxygen XML that Breathe will consume
       - doxygen Doxyfile
+      # Preprocess users manual: convert HTML anchors to MyST targets
+      - python docs/preprocess_manual.py documentation/pestpp_users_manual.md docs/users_manual_processed.md
       # Copy users manual media assets so Sphinx places them at _build/html/media/
       - mkdir -p docs/_manual_assets/media
       - cp -r documentation/media/. docs/_manual_assets/media/

CMakeLists.txt

@@ -1,6 +1,6 @@
 # This is the top-level CMake file for PEST++
 cmake_minimum_required(VERSION 3.15)
-
+set(CMAKE_VERBOSE_MAKEFILE,ON)
 if(CMAKE_BUILD_TYPE STREQUAL "Debug")
   add_compile_definitions(_DEBUG)
 endif()

benchmarks/glm_test_base.py

@@ -801,13 +801,13 @@ def tenpar_fosm_external_stdev_test():
 
 if __name__ == "__main__":
     #freyberg_stress_test()
-    tenpar_xsec_stress_test()
+    #tenpar_xsec_stress_test()
     #tenpar_base_test()
     #tenpar_fosm_external_stdev_test()
     #tenpar_superpar_restart_test()
     #freyberg_basic_restart_test()
     # jac_diff_invest()
-    #new_fmt_load_test()
+    new_fmt_load_test()
     #tenpar_hotstart_test()
     #tenpar_normalform_test()
     #freyberg_stress_test()

benchmarks/ies_test_part4.py

@@ -2025,6 +2025,7 @@ def tenpar_adjust_weights_test():
     obs.loc[:,"standard_deviation"] = 0.1
     obs.loc[pst.obs_names[12:],"standard_deviation"] = 1e-11
     obs.loc[pst.obs_names[12:],"obsval"] = 1e-9
+    #these values are hard coded later...
     with open(os.path.join(template_d,"phi.csv"),'w') as f:
         f.write("og1,0.333333\n")
         f.write("og3,0.333333\n")
@@ -2044,6 +2045,29 @@ def tenpar_adjust_weights_test():
     pst.pestpp_options['ies_verbose_level'] = 4
     pst.pestpp_options["ies_bad_phi_sigma"] = -1.5
 
+    pst.control_data.noptmax = 0
+    pst.pestpp_options["ies_drop_conflicts"] = False
+    pst.pestpp_options["ies_phi_factor_file"] = "phi.csv"
+    pst_name = "pest_adj.pst"
+    pst.write(os.path.join(template_d,pst_name),version=2)
+    if os.path.exists(test_d):
+        shutil.rmtree(test_d)
+    shutil.copytree(template_d, test_d)
+    pyemu.os_utils.run("{0} {1}".format(exe_path, pst_name), cwd=test_d)
+    pst.set_res(os.path.join(test_d,pst_name.replace(".pst",".base.rei")))
+    print(pst.phi_components)
+    obs = pst.observation_data
+    aobs = pd.read_csv(os.path.join(test_d,pst_name.replace(".pst",".0.adjusted.obs_data.csv")),index_col=0)
+    obs.loc[aobs.index,"weight"] = aobs.weight.values
+    print(pst.phi_components)
+    print(pst.phi_components_normalized)
+    pcn = pst.phi_components_normalized
+    assert np.isclose(pcn["og3blahblah"],0.33333,1e-3)
+    assert np.isclose(pcn["og4yadayada"],0.33333,1e-3)
+    
+
+    pst.pestpp_options.pop("ies_phi_factor_file")
+
 
     pst.control_data.noptmax = 2
     pst.pestpp_options["ies_drop_conflicts"] = False
@@ -5007,10 +5031,48 @@ def large_invest():
     pyemu.os_utils.run("{0} pest.pst /e".format(exe_path),cwd=t_d)
 
 
+def tenpar_xsec_combined_autoadaloc_mm_stress_test():
+    """testing combined matrix + autoadaloc"""
+    model_d = "ies_10par_xsec"
+    test_d = os.path.join(model_d, "master_comb_aal_test1")
+    template_d = os.path.join(model_d, "test_template")
+
+    if not os.path.exists(template_d):
+        raise Exception("template_d {0} not found".format(template_d))
+    pst_name = os.path.join(template_d, "pest.pst")
+    pst = pyemu.Pst(pst_name)
+
+    if os.path.exists(test_d):
+        shutil.rmtree(test_d)
+    shutil.copytree(template_d, test_d)
+    pst.pestpp_options = {}
+    pst.pestpp_options["ies_num_reals"] = 30
+    
+    mat = pyemu.Matrix.from_names(pst.nnz_obs_names, pst.adj_par_names).to_dataframe()
+    mat.loc[:, :] = 1
+    mat.loc[:, pst.adj_par_names[::2]] = 0
+    pyemu.Matrix.from_dataframe(mat).to_ascii(os.path.join(template_d, "loc.mat"))
 
+    pst.pestpp_options["ies_localizer"] = "loc.mat"
+    pst.pestpp_options["ies_autoadaloc"] = True
+    pst.pestpp_options["ies_verbose_level"] = 3
+    pst.pestpp_options["ies_debug_fail_remainder"] = True
+    pst.pestpp_options["ies_debug_fail_subset"] = True
+    pst.pestpp_options["ies_debug_bad_phi"] = True
+    pst.pestpp_options["ies_multimodal_alpha"] = 0.99
+    
+    pst.control_data.noptmax = 3
 
+    pst.write(os.path.join(template_d, "pest_aal_restart.pst"))
+    pyemu.os_utils.start_workers(template_d, exe_path, "pest_aal_restart.pst", num_workers=10,
+                                 master_dir=test_d, verbose=True, worker_root=model_d,
+                                 port=port)
+    
 
 if __name__ == "__main__":
+    tenpar_xsec_combined_autoadaloc_mm_stress_test()
+
+    #tenpar_adjust_weights_test()
     #large_invest()
     #tenpar_fixed_transform_test()
     #tenpar_reg_factor_test()
@@ -5031,7 +5093,7 @@ def large_invest():
     #tenpar_reg_factor_test()
     #tenpar_high_phi_test()
     #tenpar_iqr_bad_phi_sigma_test()
-    multimodal_test()
+    #multimodal_test()
     #plot_mm1_sweep_results()
     #plot_mm1_results()
     #plot_mm1_results_seq()

docs/conf.py

@@ -40,7 +40,7 @@
 html_extra_path = ["_manual_assets"]
 
 # -- MyST (Markdown support) -------------------------------------------------
-myst_enable_extensions = ["colon_fence"]
+myst_enable_extensions = ["colon_fence", "html_image"]
 source_suffix = {
     ".rst": "restructuredtext",
     ".md": "markdown",

docs/preprocess_manual.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""
+Preprocess pestpp_users_manual.md for Sphinx/MyST compatibility.
+
+Converts HTML anchor tags in headings to MyST named targets so that
+internal links like [Section](#s5-1) resolve correctly in Sphinx.
+
+  Before: ## <a id='s5-1' />1.1 PEST++ and PEST
+  After:  (s5-1)=
+          ## 1.1 PEST++ and PEST
+"""
+import re
+import sys
+from pathlib import Path
+
+
+def preprocess(input_path, output_path):
+    content = Path(input_path).read_text(encoding="utf-8")
+
+    # Convert:  # <a id='s5' />1. Introduction
+    # To:       (s5)=
+    #           # 1. Introduction
+    def replace_heading_anchor(m):
+        hashes = m.group(1)
+        anchor_id = m.group(2)
+        heading_text = m.group(3).strip()
+        return f"({anchor_id})=\n{hashes} {heading_text}"
+
+    content = re.sub(
+        r'^(#{1,6})[ \t]*<a\s+id=[\'"]([^\'"]+)[\'"][ \t]*/?>[ \t]*(.*)$',
+        replace_heading_anchor,
+        content,
+        flags=re.MULTILINE,
+    )
+
+    Path(output_path).write_text(content, encoding="utf-8")
+
+
+if __name__ == "__main__":
+    preprocess(sys.argv[1], sys.argv[2])

docs/users_guide.md

@@ -1,4 +1,4 @@
 # Users Guide
 
-```{include} ../documentation/pestpp_users_manual.md
+```{include} users_manual_processed.md
 ```

scripts/build_pestpp_mac_arm.sh

@@ -11,7 +11,7 @@ cd "$script_path"/..
 rm -rfd build
 mkdir build
 cd build
-cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=g++ -DFORCE_STATIC=OFF ..
+cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-fsanitize=address -fno-omit-frame-pointer" -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address" -DCMAKE_CXX_COMPILER=g++ -DFORCE_STATIC=OFF ..
 make -j
 cpack -G TGZ
 cp *.tar.gz ../

src/libs/common/utilities.cpp

@@ -96,6 +96,45 @@ double get_duration_sec(std::chrono::system_clock::time_point start_time)
 	return (double)std::chrono::duration_cast<std::chrono::milliseconds>(dt).count() / 1000.0;
 }
 
+// Split `input` on whitespace into tokens.  Single and double quotes may be
+// nested inside each other; all quote characters (at every nesting level) are
+// stripped from the output.  Empty and whitespace-only tokens are omitted.
+// Throws std::runtime_error if a quoted section is never closed.
+std::vector<std::string> tokenize_w_quotes(const std::string& input) {
+    std::vector<std::string> tokens;
+    std::string current;
+    std::vector<char> quote_stack; // innermost open-quote char at back
+
+    for (char c : input) {
+        if (!quote_stack.empty() && c == quote_stack.back()) {
+            // Closing quote matches the innermost open quote — pop it.
+            // The quote character itself is not copied into the token.
+            quote_stack.pop_back();
+        } else if (c == '"' || c == '\'') {
+            // Opening quote (or an inner quote of the other type) — push it.
+            // The quote character itself is not copied into the token.
+            quote_stack.push_back(c);
+        } else if (quote_stack.empty() && (c == ' ' || c == '\t' || c == '\n' || c == '\r')) {
+            // Whitespace outside all quotes — flush non-empty token.
+            if (!current.empty()) {
+                tokens.push_back(current);
+                current.clear();
+            }
+        } else {
+            current += c;
+        }
+    }
+
+    if (!quote_stack.empty()) {
+        throw std::runtime_error("unterminated quoted string: '" + input + "'");
+    }
+    if (!current.empty()) {
+        tokens.push_back(current);
+    }
+
+    return tokens;
+}
+
 template < class ContainerT >
 void tokenize(const std::string& str, ContainerT& tokens, const std::string& delimiters, const bool trimEmpty)
 {