Merge pull request #191 from jtwhite79/feat_partial

Feat partial

Author: jtwhite79

.github/workflows/ci.yml

@@ -9,7 +9,7 @@ on:
 jobs:
   pestppCI:
     name: benchmarks
-    runs-on: ubuntu-latest #${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
@@ -34,11 +34,14 @@ jobs:
           - { test_dir: "pestpp-ies_benchmarks", test_repo: "https://github.com/pestpp/pestpp-ies_benchmarks", test_script: "ies_test_part4.py"}
           - { test_dir: "pestpp-glm_benchmarks", test_repo: "https://github.com/pestpp/pestpp-glm_benchmarks", test_script: "glm_test_base.py"}
           - { test_dir: "pestpp-mou_benchmarks", test_repo: "https://github.com/pestpp/pestpp-mou_benchmarks", test_script: "mou_tests.py"}
-          - { test_dir: "pestpp-mou_benchmarks", test_repo: "https://github.com/pestpp/pestpp-mou_benchmarks", test_script: "mou_tests_2.py"}
+          #- { test_dir: "pestpp-mou_benchmarks", test_repo: "https://github.com/pestpp/pestpp-mou_benchmarks", test_script: "mou_tests_2.py"}
           - { test_dir: "pestpp-mou_benchmarks", test_repo: "https://github.com/pestpp/pestpp-mou_benchmarks", test_script: "henry_test.py"}
           - { test_dir: "pestpp-da_benchmarks", test_repo: "https://github.com/pestpp/pestpp-da_benchmarks", test_script: "da_tests.py"}
           - { test_dir: "pestpp-da_benchmarks", test_repo: "https://github.com/pestpp/pestpp-da_benchmarks", test_script: "lorenz96_tests.py"}
           - { test_dir: "pestpp-sqp_benchmarks", test_repo: "https://github.com/pestpp/pestpp-sqp_benchmarks", test_script: "sqp_tests.py"}
+        include: 
+            - os: ubuntu-latest
+              env: { test_dir: "pestpp-mou_benchmarks", test_repo: "https://github.com/pestpp/pestpp-mou_benchmarks", test_script: "mou_tests_2.py"}
     env: ${{ matrix.env }}
 
     steps:
@@ -61,49 +64,53 @@ jobs:
       if: runner.os == 'Windows'
       uses: ilammy/msvc-dev-cmd@v1
 
-    # - name: Get specific version CMake, v3.19
-    #   if: ${{ runner.os == 'Windows' }} 
-    #   uses: lukka/get-cmake@latest
-
-    - name: clone test repo ${{ env.test_repo }}
-      shell: bash -l {0}
-      run: |
-        git clone ${{ env.test_repo }}
-
-    - name: PESTPP exe install
-      shell: bash -l {0}
-      run: |
-        mkdir build && cd build
-        if [[ "$RUNNER_OS" == "Windows" ]]; then
-            cmake -GNinja -DCMAKE_CXX_COMPILER=cl -DCMAKE_C_COMPILER=cl -DCMAKE_BUILD_TYPE=Release ..
-            ninja
-        else
-            cmake -DCMAKE_BUILD_TYPE=Release ..
-            make -j2
-        fi
-        cd ..
+    
 
-   
     # Standard python fails on windows without GDAL installation
     # Using custom bash shell ("shell: bash -l {0}") with Miniconda
     - name: Setup Miniconda
-      uses: conda-incubator/setup-miniconda@v2.0.0
+      uses: conda-incubator/setup-miniconda@v2
       with:
+        # python-version: ${{ matrix.python-version }}
+        # #mamba-version: "0.20.0"
+        # channels: conda-forge
+        # auto-update-conda: true
+        # activate-environment: pyemu
+        # use-only-tar-bz2: true
+        miniforge-version: latest
+        # miniconda-version: "latest"
         python-version: ${{ matrix.python-version }}
-        mamba-version: "0.20.0"
-        channels: conda-forge
-        auto-update-conda: true
+        # mamba-version: "*"
+        # channels: conda-forge
+        miniforge-variant: Mambaforge
+        # auto-update-conda: true
         activate-environment: pyemu
-        use-only-tar-bz2: true
+        use-mamba: true
+        # environment-file: etc/environment.yml
+        # use-only-tar-bz2: true
+
+    # - name: Add packages to pyemu environment using mamba or conda
+    #   shell: bash -l {0}
+    #   run: |
+    #     if [ "$RUNNER_OS" == "Windows" ]; then
+    #       conda env update --name pyemu --file etc/environment.yml
+    #     else
+    #       conda env update --name pyemu --file etc/environment.yml
+    #     fi
 
-    - name: Add packages to pyemu environment using mamba or conda
+    - name: Add packages to pyemu environment using conda
+      if: ${{ matrix.python-version < 3.8 }}
+      # if: ${{ runner.os == 'Windows' || matrix.python-version < 3.8 }}
       shell: bash -l {0}
       run: |
-        if [ "$RUNNER_OS" == "Windows" ]; then
-          conda env update --name pyemu --file etc/environment.yml
-        else
           conda env update --name pyemu --file etc/environment.yml
-        fi
+    
+    - name: Add packages to pyemu environment using mamba
+      # if: ${{ runner.os != 'Windows' && matrix.python-version >= 3.8 }}
+      if: ${{ matrix.python-version >= 3.8 }}
+      shell: bash -l {0}
+      run: |
+          mamba env update --name pyemu --file etc/environment.yml
     
     - name: Install Flopy & pyemu?
       shell: bash -l {0}
@@ -117,24 +124,53 @@ jobs:
         cd pyemu
         python setup.py install
         cd ..
+
+    # - name: Get specific version CMake, v3.19
+    #   if: ${{ runner.os == 'Windows' }} 
+    #   uses: lukka/get-cmake@latest
+
+    - name: clone test repo ${{ env.test_repo }}
+      shell: bash -l {0}
+      run: |
+        git clone ${{ env.test_repo }}
+
+    - name: PESTPP exe install
+      shell: bash -l {0}
+      run: |
+        mkdir build && cd build
+        if [[ "$RUNNER_OS" == "Windows" ]]; then
+            cmake -GNinja -DCMAKE_CXX_COMPILER=cl -DCMAKE_C_COMPILER=cl -DCMAKE_BUILD_TYPE=Release ..
+            ninja
+        else
+            cmake -DCMAKE_BUILD_TYPE=Release ..
+            make -j2
+        fi
+        cd ..
+
     
     - name: aux bin path hackery
       shell: bash -l {0}
       run: |
+        cp -r benchmarks/test_bin/win $HOME/bin
+        echo "../../test_bin/win" >> $GITHUB_PATH
         if [[ "$RUNNER_OS" == "Windows" ]]; then
-            echo "..\..\test_bin\win" >> $GITHUB_PATH        
+            cp -r benchmarks/test_bin/win $HOME/bin
+            echo "../../test_bin/win" >> $GITHUB_PATH  
         else
             echo "../../test_bin/linux" >> $GITHUB_PATH
+            cp -r benchmarks/test_bin/win $HOME/bin
         fi
+        echo "$HOME/.local/bin" >> $GITHUB_PATH
 
     - name: run test script ${{ env.test_script }} in ${{ env.test_dir }}
       shell: bash -l {0}
       run: |
+        
         ls -l
         cd ${{ env.test_dir }}
         nosetests -v ${{ env.test_script }}
         cd
-
+    
 
 
 

benchmarks/basic_tests.py

@@ -733,6 +733,10 @@ def tplins1_test():
     jco = pyemu.Jco.from_binary(os.path.join(t_d,"pest.jcb")).to_dataframe().apply(np.abs)
     assert jco.sum().sum() == 0, jco.sum()
 
+    pst.control_data.noptmax = 0
+    pst.write(os.path.join(t_d,"pest.pst"))
+    pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d)
+
     # check the input file - the last two number should be the same
     arr = np.loadtxt(os.path.join(t_d,"hk_Layer_1.ref"))
     assert arr[-2] == arr[-1],arr[-2] - arr[-1]
@@ -742,6 +746,7 @@ def tplins1_test():
     assert len(lines_tpl) - 1 == len(lines_in)
 
     pst = pyemu.Pst(os.path.join(t_d,"pest.pst"))
+    pst.control_data.noptmax = -1
     pst.pestpp_options["fill_tpl_zeros"] = True
     pst.write(os.path.join(t_d,"pest_fill.pst"))
     pyemu.os_utils.run("{0} pest_fill.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d)
@@ -758,6 +763,10 @@ def tplins1_test():
     jco = pyemu.Jco.from_binary(os.path.join(t_d,"pest_fill.jcb")).to_dataframe().apply(np.abs)
     assert jco.sum().sum() == 0, jco.sum()
 
+    pst.control_data.noptmax = 0
+    pst.write(os.path.join(t_d,"pest.pst"))
+    pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d)
+
     # check the input file - the last two number should be the same
     arr = np.loadtxt(os.path.join(t_d,"hk_Layer_1.ref"))
     assert arr[-2] == arr[-1]
@@ -773,6 +782,7 @@ def tplins1_test():
     pst.model_output_data = pd.DataFrame({"pest_file":"out1dum.dat.ins",
                                           "model_file":'out1.dat'},index=["out1dum.dat.ins"])
     #pst.instruction_files = ['out1dum.dat.ins']
+    pst.control_data.noptmax = -1
     pst.write(os.path.join(t_d, "pest_dum.pst"))
     pyemu.os_utils.run("{0} pest_dum.pst".format(exe_path.replace("-ies", "-glm")), cwd=t_d)
     obf_df = pd.read_csv(os.path.join(t_d, "out1.dat.obf"), delim_whitespace=True, header=None,
@@ -872,6 +882,20 @@ def mf6_v5_ies_test():
     pyemu.os_utils.run("{0} freyberg6_run_ies.pst".format(exe_path),cwd=t_d)
 
     pst.control_data.noptmax = 3
+
+    m_d = os.path.join(model_d, "master_ies_glm_noloc_standard")
+    if os.path.exists(m_d):
+        shutil.rmtree(m_d)
+    pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst"))
+    pst.pestpp_options.pop("ies_localizer",None)
+    pst.pestpp_options.pop("ies_autoadaloc",None)
+    pst.pestpp_options["ies_bad_phi_sigma"] = 2.5
+    pst.control_data.noptmax = 3
+    pst.write(os.path.join(t_d, "freyberg6_run_ies_glm_noloc_standard.pst"))
+    pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_glm_noloc_standard.pst", num_workers=15,
+                                 master_dir=m_d, worker_root=model_d, port=port)
+
+    return
     pst.write(os.path.join(t_d,"freyberg6_run_ies_glm_loc.pst"))
 
     m_d = os.path.join(model_d, "master_ies_glm_covloc")
@@ -1106,7 +1130,7 @@ def fr_fail_test():
     assert not os.path.exists(oe_file)
     m_d = os.path.join(model_d,"fr_fail_master")
     try:
-        pyemu.os_utils.start_workers(new_d,exe_path,"pest.pst",num_workers=1,worker_root=model_d,master_dir=m_d)
+        pyemu.os_utils.start_workers(new_d,exe_path,"pest.pst",num_workers=5,worker_root=model_d,master_dir=m_d)
     except:
         pass
     else:
@@ -1175,29 +1199,29 @@ def fr_timeout_test():
         f.write("import os\nimport time\nimport pyemu\npyemu.os_utils.run('mfnwt 10par_xsec.nam')\n")
         f.write("if not os.path.exists('run.info'):\n    exit()\n")
         f.write("lines = open('run.info','r').readlines()\nrnum = int(lines[-1].split()[-1].split('=')[-1])\n")
-        f.write("if rnum % 2 == 0:\n    time.sleep(100000)\n")
+        f.write("if rnum % 2 == 0:\n    time.sleep(10000000)\n")
     pst.model_command = "python run.py"
     oe_file = os.path.join(new_d, "pest.0.obs.csv")
     if os.path.exists(oe_file):
         os.remove(oe_file)
     pst.control_data.noptmax = -1
     pst.pestpp_options["overdue_giveup_fac"] = 1.0e+10
-    pst.pestpp_options["overdue_giveup_minutes"] = 0.1
+    pst.pestpp_options["overdue_giveup_minutes"] = 0.25
     pst.pestpp_options["ies_num_reals"] = 10
+    pst.pestpp_options["ies_include_base"] = False
+    pst.pestpp_options["max_run_fail"] = 1
 
     #pst.pestpp_options["panther_transfer_on_fail"] = "10par_xsec.list"
-    pst.pestpp_options["panther_agent_freeze_on_fail"] = True
+    pst.pestpp_options["panther_agent_freeze_on_fail"] = False
     pst.write(os.path.join(new_d, "pest.pst"))
-    pyemu.os_utils.run("{0} pest.pst".format(exe_path),cwd=new_d)
 
-    assert os.path.exists(oe_file)
     m_d = os.path.join(model_d,"fr_timeout_master")
-    pyemu.os_utils.start_workers(new_d,exe_path,"pest.pst",num_workers=1,worker_root=model_d,master_dir=m_d)
+    pyemu.os_utils.start_workers(new_d,exe_path,"pest.pst",num_workers=5,worker_root=model_d,master_dir=m_d)
     oe_file = os.path.join(m_d, "pest.0.obs.csv")
     assert os.path.exists(oe_file)
     oe = pd.read_csv(oe_file,index_col=0)
     print(oe.shape)
-    assert oe.shape[0] == 4
+    assert oe.shape[0] == 5,oe.shape
 
 def ins_missing_e_test():
     import os
@@ -1248,7 +1272,7 @@ def ins_missing_e_test():
     #salib_verf()
     #tplins1_test()
     #secondary_marker_test()
-    ext_stdcol_test()
+    #ext_stdcol_test()
 
     # parallel_consist_test()
     # ext_stdcol_test()
@@ -1261,17 +1285,20 @@ def ins_missing_e_test():
 
     #da_prep_4_mf6_freyberg_seq_tbl()
     #da_mf6_freyberg_test_2()
-    #shutil.copy2(os.path.join("..","exe","windows","x64","Debug","pestpp-ies.exe"),os.path.join("..","bin","win","pestpp-ies.exe"))
+    shutil.copy2(os.path.join("..","exe","windows","x64","Debug","pestpp-ies.exe"),os.path.join("..","bin","win","pestpp-ies.exe"))
     #tplins1_test()
     #mf6_v5_ies_test()
     #mf6_v5_sen_test()
 
+
+
     #shutil.copy2(os.path.join("..","exe","windows","x64","Debug","pestpp-opt.exe"),os.path.join("..","bin","win","pestpp-opt.exe"))
     #mf6_v5_opt_stack_test()
     #mf6_v5_glm_test()
     #mf6_v5_ies_test()
     #cmdline_test()
     #basic_sqp_test()
     #mf6_v5_ies_test()
-    #fr_timeout_test()
+    fr_timeout_test()
     #fr_fail_test()
+    #tplins1_test()