diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4fb88c2dc67..94078c40720 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,9 +7,6 @@ updates: schedule: interval: "weekly" assignees: - - "rljacob" + - "bartgol" reviewers: - "mahf708" - - "bartgol" - labels: - - "AT: Integrate Without Testing" diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index 5c6ff081f73..d973b90dc05 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -2,7 +2,9 @@ name: gh on: pull_request: - branches: [ master ] + branches: + - master + - maint-3.0 paths: # first, yes to these - '.github/workflows/e3sm-gh-ci-cime-tests.yml' @@ -40,7 +42,7 @@ jobs: - SMS_D_Ln5_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu - ERS_Ld5_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu.eamxx-prod container: - image: ghcr.io/e3sm-project/containers-ghci:ghci-0.2.0 + image: ghcr.io/e3sm-project/containers-ghci:ghci-0.2.1 steps: - diff --git a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml index f51aa88a34c..0e30fc4e0a8 100644 --- a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml @@ -2,7 +2,9 @@ name: gh-w on: pull_request: - branches: [ master ] + branches: + - master + - maint-3.0 paths-ignore: - 'mkdocs.yaml' - 'docs/**' @@ -27,7 +29,7 @@ jobs: - SMS_D_Ld1_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu - ERS_Ld3_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu.allactive-wcprod_1850 container: - image: ghcr.io/e3sm-project/containers-ghci:ghci-0.2.0 + image: ghcr.io/e3sm-project/containers-ghci:ghci-0.2.1 steps: - diff --git a/.github/workflows/e3sm-gh-md-linter.yml b/.github/workflows/e3sm-gh-md-linter.yml index ad24487695e..8be6f87893b 100644 --- a/.github/workflows/e3sm-gh-md-linter.yml +++ b/.github/workflows/e3sm-gh-md-linter.yml @@ -7,8 +7,6 @@ on: branches: ["master"] paths: - '**/*.md' - # for now let's not lint files in eamxx - - '!components/eamxx/**/*.md' concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} @@ -27,7 +25,7 @@ jobs: with: files: '**/*.md' separator: "," - - uses: DavidAnson/markdownlint-cli2-action@v18 + - uses: DavidAnson/markdownlint-cli2-action@v19 if: steps.changed-files.outputs.any_changed == 'true' with: config: 'docs/.markdownlint.json' diff --git a/.github/workflows/e3sm-gh-pages.yml b/.github/workflows/e3sm-gh-pages.yml index dec9bc696bf..543295a9030 100644 --- a/.github/workflows/e3sm-gh-pages.yml +++ b/.github/workflows/e3sm-gh-pages.yml @@ -4,9 +4,27 @@ on: # Runs every time master branch is updated push: branches: ["master"] + # But only if docs-related files are touched + paths: + - .github/workflows/e3sm-gh-pages.yml + - ./mkdocs.yml + - ./tools/*/mkdocs.yml + - ./tools/docs/** + - components/*/mkdocs.yaml + - components/*/docs/** + - components/eamxx/cime_config/namelist_defaults_scream.xml # Runs every time a PR is open against master pull_request: branches: ["master"] + # But only if docs-related files are touched + paths: + - .github/workflows/e3sm-gh-pages.yml + - ./mkdocs.yml + - ./tools/*/mkdocs.yml + - ./tools/docs/** + - components/*/mkdocs.yaml + - components/*/docs/** + - components/eamxx/cime_config/namelist_defaults_scream.xml workflow_dispatch: concurrency: diff --git a/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml b/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml deleted file mode 100644 index cacb951b8a8..00000000000 --- a/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: mkatmsrffile - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - paths: - - 'components/eam/tools/mkatmsrffile/mkatmsrffile.py' - schedule: - - cron: '00 15 * * 2' - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: true - -jobs: - mkatmsrffile-test: - if: ${{ github.repository == 'E3SM-Project/E3SM' }} - runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} - outputs: - event_name: ${{ github.event_name }} - steps: - - - name: Repository checkout - uses: actions/checkout@v4 - with: - show-progress: false - submodules: false - - - name: Conda setup - uses: conda-incubator/setup-miniconda@v3 - with: - activate-environment: "envmkatmsrffile" - miniforge-version: latest - channel-priority: strict - auto-update-conda: true - python-version: 3.11 - - - name: Install dependencies - run: | - echo $CONDA_PREFIX - conda install -y nco xarray numba numpy netcdf4 -c conda-forge - - - name: Run tests - working-directory: components/eam/tools/mkatmsrffile - run: | - echo $CONDA_PREFIX - wget https://web.lcrc.anl.gov/public/e3sm/inputdata/atm/cam/chem/trop_mozart/dvel/clim_soilw.nc - wget https://web.lcrc.anl.gov/public/e3sm/inputdata/atm/cam/chem/trop_mozart/dvel/regrid_vegetation.nc - wget https://web.lcrc.anl.gov/public/e3sm/inputdata/atm/cam/chem/trop_mozart/dvel/map_1x1_to_ne30pg2_traave_c20240903.nc - python mkatmsrffile.py --map_file=map_1x1_to_ne30pg2_traave_c20240903.nc --vegetation_file=regrid_vegetation.nc --soil_water_file=clim_soilw.nc --dst_grid=ne30pg2 - - mkatmsrffile-notify: - needs: mkatmsrffile-test - if: ${{ failure() && needs.mkatmsrffile-test.outputs.event_name != 'pull_request' }} - runs-on: ubuntu-latest - steps: - - name: Create issue - run: | - previous_issue_number=$(gh issue list \ - --label "$LABELS" \ - --json number \ - --jq '.[0].number') - if [[ -n $previous_issue_number ]]; then - gh issue comment "$previous_issue_number" \ - --body "$BODY" - else - gh issue create \ - --title "$TITLE" \ - --assignee "$ASSIGNEES" \ - --label "$LABELS" \ - --body "$BODY" - fi - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} - TITLE: mkatmsrffile test failure - ASSIGNEES: whannah1 - LABELS: bug,notify-mkatmsrffile-gh-action - BODY: | - Workflow failed! There's likely an issue in the mkatmsrffile tool! For more information, please see: - - Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} (number ${{ github.run_number }}, attempt ${{ github.run_attempt }}) - - Workflow SHA: ${{ github.sha }} diff --git a/.github/workflows/eamxx-gh-ci-standalone.yml b/.github/workflows/eamxx-gh-ci-standalone.yml deleted file mode 100644 index 19a2ec9cd8e..00000000000 --- a/.github/workflows/eamxx-gh-ci-standalone.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: gh-standalone - -on: - pull_request: - branches: [ master ] - paths: - # first, yes to these - - '.github/workflows/eamxx-gh-ci-standalone.yml' - - 'cime_config/machine/config_machines.xml' - - 'components/eamxx/**' - - 'components/homme/**' - # second, no to these - - '!components/eamxx/docs/**' - - '!components/eamxx/mkdocs.yml' - - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: true - -jobs: - - ci: - if: ${{ github.repository == 'E3SM-Project/E3SM' }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - test: - - sp - - opt - - dbg - - fpe - container: - image: ghcr.io/e3sm-project/containers-standalone-ghci:standalone-ghci-0.1.0 - - steps: - - - name: Checkout - uses: actions/checkout@v4 - with: - show-progress: false - submodules: recursive - - - name: standalone - env: - SHELL: sh - run: | - # TODO: get rid of this extra line if we can? - git config --global safe.directory '*' - ./components/eamxx/scripts/test-all-scream -m ghci-oci -t ${{ matrix.test }} -c BUILD_SHARED_LIBS=ON - - - name: Artifacts - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - name: ${{ matrix.test }} - path: | - components/eamxx/ctest-build/*/Testing/Temporary/Last*.log diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml deleted file mode 100644 index 2e763c544cd..00000000000 --- a/.github/workflows/eamxx-gh-pages.yml +++ /dev/null @@ -1,89 +0,0 @@ -# This workflow aims to automatically rebuild eamxx documentation -# every time the master branch is updated on github and within every PR - -name: EAMxx Docs - -on: - # Runs every time master branch is updated - push: - branches: [ master ] - # Only if docs-related files are touched - paths: - - components/eamxx/mkdocs.yaml - - components/eamxx/docs/** - - components/eamxx/cime_config/namelist_defaults_scream.xml - # Runs every time a PR is open against master - pull_request: - branches: [ master ] - # Only if docs-related files are touched - paths: - - components/eamxx/mkdocs.yaml - - components/eamxx/docs/** - - components/eamxx/cime_config/namelist_defaults_scream.xml - - label: - types: - - created - - workflow_dispatch: - -concurrency: - # Prevent 2+ copies of this workflow from running concurrently - group: eamxx-docs-action - -jobs: - - eamxx-docs: - if: ${{ github.repository == 'E3SM-Project/scream' }} - runs-on: ubuntu-latest - - steps: - - name: Check out the repository - uses: actions/checkout@v4 - with: - persist-credentials: false - show-progress: false - # TODO: git rid of dependency on CIME - # TODO: another option to investigate is a sparse checkout. - # In the scream repo, all other components do not need to be checked out. - # And even in the upstream, we mainly need only components/xyz/docs (and a few more places). - submodules: true - - - name: Show action trigger - run: | - echo "= The job was automatically triggered by a ${{github.event_name}} event." - - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install Python deps - run: | - pip install mkdocs pymdown-extensions mkdocs-material mdutils mkdocs-bibtex - - - name: Generate EAMxx params docs - working-directory: components/eamxx/scripts - run: | - ./eamxx-params-docs-autogen - - - name: Build docs - working-directory: components/eamxx - run: | - mkdocs build --strict --verbose - - # only deploy to the main github page when there is a push to master - - if: ${{ github.event_name == 'push' }} - name: GitHub Pages action - uses: JamesIves/github-pages-deploy-action@v4 - with: - # Do not remove existing pr-preview pages - clean-exclude: pr-preview - folder: ./components/eamxx/site - - # If it's a PR from within the same repo, deploy to a preview page - - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} - name: Preview docs - uses: rossjrw/pr-preview-action@v1 - with: - source-dir: components/eamxx/site/ diff --git a/.github/workflows/eamxx-sa-testing.yml b/.github/workflows/eamxx-sa-testing.yml index a4397d4fdba..7bc20aeaca5 100644 --- a/.github/workflows/eamxx-sa-testing.yml +++ b/.github/workflows/eamxx-sa-testing.yml @@ -5,6 +5,19 @@ on: pull_request: branches: [ master ] types: [opened, synchronize, ready_for_review, reopened] + paths: + # first, yes to these + - '.github/workflows/eamxx-sa-testing.yml' + - 'cime_config/machine/config_machines.xml' + - 'components/eamxx/**' + - 'components/homme/**' + - 'externals/ekat' + - 'externals/mam4xx' + - 'externals/haero' + - 'externals/scorpio' + # second, no to these + - '!components/eamxx/docs/**' + - '!components/eamxx/mkdocs.yml' # Manual run is used to bless workflow_dispatch: @@ -40,83 +53,18 @@ concurrency: env: # Submit to cdash only for nightlies or if the user explicitly forced a submission via workflow dispatch submit: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.submit) }} + generate: ${{ github.event_name == 'workflow_dispatch' && inputs.bless }} jobs: - pre_process_pr: - if: ${{ github.event_name == 'pull_request' }} - runs-on: ubuntu-latest # This job can run anywhere - outputs: - relevant_paths: ${{ steps.check_paths.outputs.value }} - labels: ${{ steps.get_labels.outputs.labels }} - steps: - - name: Check files modified by PR - id: check_paths - run: | - paths=( - components/eamxx - components/eam/src/physics/rrtmgp - components/eam/src/physics/p3/scream - components/eam/src/physics/cam - components/eam/src/physics/rrtmgp/external - externals/ekat - externals/scorpio - externals/haero - externals/YAKL - .github/workflows/eamxx-sa-testing.yml - ) - pattern=$(IFS=\|; echo "${paths[*]}") - - # Use the GitHub API to get the list of changed files - # There are page size limits, so do it in chunks - page=1 - while true; do - response=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - "https://api.github.com/repos/E3SM-Project/scream/pulls/${{ github.event.number }}/files?per_page=100&page=$page") - - # Check if the response is empty, and break if it is - [ -z "$response" ] && break - - changed_files+=$(echo "$response" | grep -o '"filename": *"[^"]*"' | sed 's/"filename": *//; s/"//g')$'\n' - - # Check if there are more pages, and quite if there aren't - [[ $(echo "$response" | jq '. | length') -lt 100 ]] && break - - page=$((page + 1)) - done - - # Check for matches and echo the matching files (or "" if none) - matching_files=$(echo "$changed_files" | grep -E "^($pattern)" || echo "") - if [[ -n "$matching_files" ]]; then - echo "Found relevant files: $matching_files" - echo "value=true" >> $GITHUB_OUTPUT - else - echo "No relevant files touched by this PR." - echo "value=false" >> $GITHUB_OUTPUT - fi - - name: Retrieve PR labels - id: get_labels - run: | - labels="${{ join(github.event.pull_request.labels.*.name, ',') }}" - echo "labels=${labels}" >> $GITHUB_OUTPUT gcc-openmp: - needs: [pre_process_pr] if: | - !failure() && !cancelled() && - ( - github.event_name == 'schedule' || + ${{ + github.event_name != 'workflow_dispatch' || ( - github.event_name == 'pull_request' && - needs.pre_process_pr.outputs.relevant_paths=='true' && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip gcc') && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip openmp') && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-sa') && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-all') - ) || ( - github.event_name == 'workflow_dispatch' && github.event.inputs.job_to_run == 'gcc-openmp' || github.event.inputs.job_to_run == 'all' ) - ) + }} runs-on: [self-hosted, ghci-snl-cpu, gcc] strategy: fail-fast: false @@ -132,14 +80,6 @@ jobs: submodules: recursive - name: Show action trigger uses: ./.github/actions/show-workflow-trigger - - name: Set test-all inputs based on event specs - run: | - echo "generate=false" >> $GITHUB_ENV - if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - if [ "${{ inputs.bless }}" == "true" ]; then - echo "generate=true" >> $GITHUB_ENV - fi - fi - name: Run tests uses: ./.github/actions/test-all-scream with: @@ -149,24 +89,14 @@ jobs: submit: ${{ env.submit }} cmake-configs: Kokkos_ENABLE_OPENMP=ON gcc-cuda: - needs: [pre_process_pr] if: | - !failure() && !cancelled() && - ( - github.event_name == 'schedule' || + ${{ + github.event_name != 'workflow_dispatch' || ( - github.event_name == 'pull_request' && - needs.pre_process_pr.outputs.relevant_paths=='true' && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip gcc') && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip cuda') && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-sa') && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-all') - ) || ( - github.event_name == 'workflow_dispatch' && - github.event.inputs.job_to_run == 'gcc-cuda' || + github.event.inputs.job_to_run == 'gcc-cuda' || github.event.inputs.job_to_run == 'all' ) - ) + }} runs-on: [self-hosted, ghci-snl-cuda, cuda, gcc] strategy: fail-fast: false @@ -182,14 +112,6 @@ jobs: submodules: recursive - name: Show action trigger uses: ./.github/actions/show-workflow-trigger - - name: Set test-all inputs based on event specs - run: | - echo "generate=false" >> $GITHUB_ENV - if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - if [ "${{ inputs.bless }}" == "true" ]; then - echo "generate=true" >> $GITHUB_ENV - fi - fi - name: Get CUDA Arch run: | # Ensure nvidia-smi is available diff --git a/.github/workflows/eamxx-scripts-tests.yml b/.github/workflows/eamxx-scripts-tests.yml index a14cdc4f350..2cdd6f8758f 100644 --- a/.github/workflows/eamxx-scripts-tests.yml +++ b/.github/workflows/eamxx-scripts-tests.yml @@ -5,6 +5,10 @@ on: pull_request: branches: [ master ] types: [opened, synchronize, ready_for_review, reopened] + paths: + - 'components/eamxx/scripts/**' + - 'components/eamxx/cime_config/**' + - '.github/workflows/eamxx-scripts-tests.yml' # Manual run for debug purposes only workflow_dispatch: @@ -30,68 +34,7 @@ env: submit: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.submit) }} jobs: - pre_process_pr: - if: ${{ github.event_name == 'pull_request' }} - runs-on: ubuntu-latest # This job can run anywhere - outputs: - relevant_paths: ${{ steps.check_paths.outputs.value}} - labels: ${{ steps.get_labels.outputs.labels }} - steps: - - name: Check files modified by PR - id: check_paths - run: | - paths=( - components/eamxx/scripts - components/eamxx/cime_config/eamxx - components/eamxx/cime_config/build - components/eamxx/cime_config/yaml_utils.py - .github/workflows/eamxx-scripts-tests.yml - ) - pattern=$(IFS=\|; echo "${paths[*]}") - - # Use the GitHub API to get the list of changed files - # There are page size limits, so do it in chunks - page=1 - while true; do - response=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - "https://api.github.com/repos/E3SM-Project/scream/pulls/${{ github.event.number }}/files?per_page=100&page=$page") - - # Check if the response is empty, and break if it is - [ -z "$response" ] && break - - changed_files+=$(echo "$response" | grep -o '"filename": *"[^"]*"' | sed 's/"filename": *//; s/"//g')$'\n' - - # Check if there are more pages, and quite if there aren't - [[ $(echo "$response" | jq '. | length') -lt 100 ]] && break - - page=$((page + 1)) - done - - # Check for matches and echo the matching files (or "" if none) - matching_files=$(echo "$changed_files" | grep -E "^($pattern)" || echo "") - if [[ -n "$matching_files" ]]; then - echo "Found relevant files: $matching_files" - echo "value=true" >> $GITHUB_OUTPUT - else - echo "No relevant files touched by this PR." - echo "value=false" >> $GITHUB_OUTPUT - fi - - name: Retrieve PR labels - id: get_labels - run: | - labels="${{ join(github.event.pull_request.labels.*.name, ',') }}" - echo "labels=${labels}" >> $GITHUB_OUTPUT cpu-gcc: - needs: [pre_process_pr] - if: | - !failure() && !cancelled() && - ( - github.event_name != 'pull_request' || - ( - needs.pre_process_pr.outputs.relevant_paths == 'true' && - !contains(needs.pre_process_pr.outputs.labels, 'CI: skip eamxx-all') - ) - ) runs-on: [self-hosted, gcc, ghci-snl-cpu] steps: - name: Check out the repository diff --git a/.github/workflows/eamxx-v1-testing.yml b/.github/workflows/eamxx-v1-testing.yml index d55ed8252a5..e738a4adcb9 100644 --- a/.github/workflows/eamxx-v1-testing.yml +++ b/.github/workflows/eamxx-v1-testing.yml @@ -5,6 +5,19 @@ on: pull_request: branches: [ master ] types: [opened, synchronize, ready_for_review, reopened] + paths: + # first, yes to these + - '.github/workflows/eamxx-v1-testing.yml' + - 'cime_config/machine/config_machines.xml' + - 'components/eamxx/**' + - 'components/homme/**' + - 'externals/ekat' + - 'externals/mam4xx' + - 'externals/haero' + - 'externals/scorpio' + # second, no to these + - '!components/eamxx/docs/**' + - '!components/eamxx/mkdocs.yml' # Manual run is used to bless workflow_dispatch: @@ -28,93 +41,40 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -jobs: - pre_process_pr: - if: ${{ github.event_name == 'pull_request' }} - runs-on: ubuntu-latest # This job can run anywhere - outputs: - relevant_paths: ${{ steps.check_paths.outputs.value }} - labels: ${{ steps.get_labels.outputs.labels }} - steps: - - name: Check files modified by PR - id: check_paths - run: | - paths=( - components/eamxx - components/eam/src/physics/rrtmgp - components/eam/src/physics/p3/scream - components/eam/src/physics/cam - components/eam/src/physics/rrtmgp/external - externals/ekat - externals/scorpio - externals/haero - externals/YAKL - .github/workflows/eamxx-v1-testing.yml - ) - pattern=$(IFS=\|; echo "${paths[*]}") - - # Use the GitHub API to get the list of changed files - # There are page size limits, so do it in chunks - page=1 - while true; do - response=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - "https://api.github.com/repos/E3SM-Project/scream/pulls/${{ github.event.number }}/files?per_page=100&page=$page") - - # Check if the response is empty, and break if it is - [ -z "$response" ] && break - - changed_files+=$(echo "$response" | grep -o '"filename": *"[^"]*"' | sed 's/"filename": *//; s/"//g')$'\n' - - # Check if there are more pages, and quite if there aren't - [[ $(echo "$response" | jq '. | length') -lt 100 ]] && break +env: + # Submit to cdash only for nightlies or if the user explicitly forced a submission via workflow dispatch + submit: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.submit) }} + # Generate only if user requested via workflow_dispatch + generate: ${{ github.event_name == 'workflow_dispatch' && inputs.bless }} + # Correct case folder suffix for generate/compare, used to find files to upload as artifacts + folder_suffix: ${{ github.event_name == 'workflow_dispatch' && inputs.bless && '.G' || '.C' }} + # Compare/generate flags for create_test + flags: ${{ github.event_name == 'workflow_dispatch' && inputs.bless && '-o -g -b master' || '-c -b master' }} - page=$((page + 1)) - done - - # Check for matches and echo the matching files (or "" if none) - matching_files=$(echo "$changed_files" | grep -E "^($pattern)" || echo "") - if [[ -n "$matching_files" ]]; then - echo "Found relevant files: $matching_files" - echo "value=true" >> $GITHUB_OUTPUT - else - echo "No relevant files touched by this PR." - echo "value=false" >> $GITHUB_OUTPUT - fi - - name: Retrieve PR labels - id: get_labels - run: | - labels="${{ join(github.event.pull_request.labels.*.name, ',') }}" - echo "labels=${labels}" >> $GITHUB_OUTPUT +jobs: cpu-gcc: - needs: [pre_process_pr] if: | - !failure() && !cancelled() && - ( - github.event_name == 'schedule' || + ${{ + github.event_name != 'workflow_dispatch' || ( - github.event_name == 'pull_request' && - needs.pre_process_pr.outputs.relevant_paths=='true' && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip gcc') && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-v1') && - !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-all') - ) || ( - github.event_name == 'workflow_dispatch' && - github.event.inputs.job_to_run == 'cpu-gcc' || + github.event.inputs.job_to_run == 'cpu-gcc' || github.event.inputs.job_to_run == 'all' ) - ) + }} runs-on: [self-hosted, gcc, ghci-snl-cpu] strategy: matrix: test: - - full_name: ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.ghci-snl-cpu_gnu.scream-output-preset-2 - short_name: ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.scream-output-preset-2 - - full_name: ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.ghci-snl-cpu_gnu.scream-dpxx-arm97 - short_name: ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-arm97 - - full_name: ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.ghci-snl-cpu_gnu.scream-small_kernels--scream-output-preset-5 - short_name: ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5 - - full_name: SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-snl-cpu_gnu.scream-mam4xx-all_mam4xx_procs - short_name: SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-all_mam4xx_procs + - full_name: ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.ghci-snl-cpu_gnu.eamxx-output-preset-2 + short_name: ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.eamxx-output-preset-2 + - full_name: ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.ghci-snl-cpu_gnu.eamxx-dpxx-arm97 + short_name: ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.eamxx-dpxx-arm97 + - full_name: ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.ghci-snl-cpu_gnu.eamxx-small_kernels--eamxx-output-preset-5 + short_name: ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.eamxx-small_kernels--eamxx-output-preset-5 + - full_name: SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-snl-cpu_gnu.eamxx-mam4xx-all_mam4xx_procs + short_name: SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.eamxx-mam4xx-all_mam4xx_procs + - full_name: SMS_Ln3_P4.ne4pg2_oQU480.F2010-MMF2.ghci-snl-cpu_gnu + short_name: SMS_Ln3_P4.ne4pg2_oQU480.F2010-MMF2 fail-fast: false name: cpu-gcc / ${{ matrix.test.short_name }} steps: @@ -142,18 +102,6 @@ jobs: echo "Unsupported Linux distribution" exit 1 fi - - name: Establish cmp/gen flag - run: | - dir_suffix=".C" - cmp_gen_flag="-c" - if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - if [ ${{ inputs.bless }} ]; then - cmp_gen_flag="-o -g" - dir_suffix=".G" - fi - fi - echo "flags=$cmp_gen_flag -b master" >> $GITHUB_ENV - echo "folder_suffix=$dir_suffix" >> $GITHUB_ENV - name: Run test run: | ./cime/scripts/create_test ${{ matrix.test.full_name }} ${{ env.flags }} --wait diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml deleted file mode 100644 index 38c528306c4..00000000000 --- a/.github/workflows/eamxx_default_files.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: inputdata - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - paths: - - 'components/eamxx/cime_config/namelist_defaults_scream.xml' - schedule: - - cron: '00 00 * * *' - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: true - -jobs: - scream-defaults: - if: ${{ github.repository == 'E3SM-Project/E3SM' }} - runs-on: ubuntu-latest - outputs: - event_name: ${{ github.event_name }} - steps: - - name: Check out the repository - uses: actions/checkout@v4 - with: - show-progress: false - submodules: false - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - name: Run unit tests - working-directory: components/eamxx/cime_config/ - run: | - python -m unittest tests/eamxx_default_files.py -v - - notify-scream-defaults: - needs: scream-defaults - if: ${{ failure() && needs.scream-defaults.outputs.event_name != 'pull_request' }} - runs-on: ubuntu-latest - steps: - - name: Create issue - run: | - previous_issue_number=$(gh issue list \ - --label "$LABELS" \ - --json number \ - --jq '.[0].number') - if [[ -n $previous_issue_number ]]; then - gh issue comment "$previous_issue_number" \ - --body "$BODY" - else - gh issue create \ - --title "$TITLE" \ - --assignee "$ASSIGNEES" \ - --label "$LABELS" \ - --body "$BODY" - fi - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} - TITLE: Inputdata server file missing - ASSIGNEES: mahf708,bartgol - LABELS: bug,input file,notify-file-gh-action - BODY: | - Workflow failed! There's likely a missing file specified in the configs! For more information, please see: - - Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} (number ${{ github.run_number }}, attempt ${{ github.run_attempt }}) - - Workflow SHA: ${{ github.sha }} diff --git a/.mergify.yml b/.mergify.yml deleted file mode 100644 index 89fcc821e57..00000000000 --- a/.mergify.yml +++ /dev/null @@ -1,53 +0,0 @@ -merge_protections: - - name: Enforce checks passing - description: Make sure that checks are not failing on the PR, and reviewers approved - if: - - base = master - success_conditions: - - "#approved-reviews-by >= 1" # At least 1 approval - - "#changes-requested-reviews-by == 0" # No reviewer asked for changes - - or: - - and: - - check-success="gcc-openmp / dbg" - - check-success="gcc-openmp / sp" - - check-success="gcc-openmp / fpe" - - check-success="gcc-openmp / opt" - - check-skipped={% raw %}gcc-openmp / ${{ matrix.build_type }}{% endraw %} - - or: - - and: - - check-success="gcc-cuda / dbg" - - check-success="gcc-cuda / sp" - - check-success="gcc-cuda / opt" - - check-skipped={% raw %}gcc-cuda / ${{ matrix.build_type }}{% endraw %} - - or: - - and: - - check-success="cpu-gcc / ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.scream-output-preset-2" - - check-success="cpu-gcc / ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-arm97" - - check-success="cpu-gcc / ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5" - - check-success="cpu-gcc / SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-all_mam4xx_procs" - - check-skipped={% raw %}cpu-gcc / ${{ matrix.test.short_name }}{% endraw %} - - or: - - check-success=cpu-gcc - - check-skipped=cpu-gcc - -pull_request_rules: - - name: dismiss stale reviews - conditions: - - base=master - actions: - dismiss_reviews: - when: synchronize # Dismiss reviews when synchronize event happens - - name: Automatic merge when CI passes and approved - conditions: - - "label=CI: automerge" - - base=master - actions: - merge: - method: merge - commit_message_template: | - Merge pull request #{{number}} from {{head}} - - Automatically merged using mergify - PR title: {{title}} - PR author: {{author}} - PR labels: {{label}} diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index 0d5f2456596..a12a89d5220 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -1169,14 +1169,14 @@ improv: any compset on ne30np4 grid - -4 - -4 - -4 - -4 - -4 - -4 - -4 - -4 + -6 + -6 + -6 + -6 + -6 + -6 + -6 + -6 @@ -2065,6 +2065,21 @@ + + + improv+allactive: RRM-WCYCL on 6 nodes + + -6 + -6 + -6 + -6 + -6 + -6 + -6 + -6 + + + cmod016b64x1 s=2.4 @@ -2202,6 +2217,149 @@ + + + + allactive+chrysalis: v3.NARRM tri-grid on 10 nodes ~1 sypd + + 576 + 576 + 576 + 576 + 576 + 64 + + + 576 + + + + allactive+chrysalis: v3.NARRM tri-grid on 20 nodes ~2 sypd + + 1152 + 1152 + 768 + 768 + 384 + 128 + + + 1152 + 384 + 384 + + + + allactive+chrysalis: v3.NARRM tri-grid on 30 nodes ~3 sypd + + 1792 + 1792 + 1280 + 1280 + 512 + 128 + + + 1792 + 512 + 512 + + + + allactive+chrysalis: v3.NARRM tri-grid on 40 nodes ~4 sypd + + 2368 + 2368 + 1408 + 1408 + 960 + 192 + + + 2368 + 1408 + + + + allactive+chrysalis: v3.NARRM tri-grid on 50 nodes ~5 sypd + + 3008 + 3008 + 1856 + 1856 + 1152 + 192 + + + 3008 + 1152 + 1152 + + + + allactive+chrysalis: v3.NARRM tri-grid on 64 nodes ~6 sypd + + 3840 + 3840 + 2304 + 2304 + 1536 + 256 + + + 3840 + 1536 + 1536 + + + + + + allactive+anvil: v3.NARRM tri-grid on 64 nodes ~1.8 sypd + + 2160 + 2160 + 2160 + 2160 + 2160 + 144 + + + 2160 + + + + allactive+anvil: v3.NARRM tri-grid on 96 nodes ~2.5 sypd + + 3240 + 3240 + 1080 + 1080 + 2160 + 216 + + + 3240 + 2160 + 2160 + + + + allactive+anvil: v3.NARRM tri-grid on 128 nodes ~3.2 sypd + + 4320 + 4320 + 4320 + 4320 + 4320 + 288 + + + 4320 + + + + diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index b093285e59d..45f886588e2 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -1959,6 +1959,16 @@ IcoswISC30E3r5 + + TL319 + TL319 + oQU240wLI + JRA025 + mpas.gis20km + null + oQU240wLI + + TL319 TL319 @@ -1969,6 +1979,16 @@ IcoswISC30E3r5 + + TL319 + TL319 + IcoswISC30E3r5 + JRA025 + mpas.gis1to10kmR2 + null + IcoswISC30E3r5 + + ne30np4.pg2 r05 @@ -5660,7 +5680,7 @@ - cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc + cpl/cpl6/map_r05_to_SOwISC12to30E3r3_r250e1250_58NS.cstmnn.20241120.nc cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc @@ -5761,15 +5781,15 @@ - cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_mpas.aisgis20km_aave.190403.nc - cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_mpas.aisgis20km_bilin.190403.nc + cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_mpas.aisgis20km_aave.190403.nc + cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_mpas.aisgis20km_bilin.190403.nc cpl/gridmaps/mpas.aisgis20km/map_mpas.aisgis20km_to_oEC60to30v3_aave.190403.nc cpl/gridmaps/mpas.aisgis20km/map_mpas.aisgis20km_to_oEC60to30v3_bilin.190403.nc - cpl/gridmaps/oEC60to30v3wLI/map_oEC60to30v3wLI_to_mpas.aisgis20km_aave.190713.nc - cpl/gridmaps/oEC60to30v3wLI/map_oEC60to30v3wLI_to_mpas.aisgis20km_bilin.190713.nc + cpl/gridmaps/oEC60to30v3wLI/map_oEC60to30v3wLI_to_mpas.aisgis20km_aave.190713.nc + cpl/gridmaps/oEC60to30v3wLI/map_oEC60to30v3wLI_to_mpas.aisgis20km_bilin.190713.nc cpl/gridmaps/mpas.aisgis20km/map_mpas.aisgis20km_to_oEC60to30v3wLI_aave.190713.nc cpl/gridmaps/mpas.aisgis20km/map_mpas.aisgis20km_to_oEC60to30v3wLI_bilin.190713.nc @@ -5786,8 +5806,8 @@ - cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_mpas.gis20km_aave.181115.nc - cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_mpas.gis20km_bilin.181115.nc + cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_mpas.gis20km_aave.181115.nc + cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_mpas.gis20km_bilin.181115.nc cpl/gridmaps/mpas.gis20km/map_mpas.gis20km_to_oEC60to30v3_aave.181115.nc cpl/gridmaps/mpas.gis20km/map_mpas.gis20km_to_oEC60to30v3_aave.181115.nc @@ -5813,9 +5833,21 @@ cpl/gridmaps/mpas.gis20km/map_gis20km_to_TL319_traave.20240404.nc + + cpl/gridmaps/oQU240wLI/map_oQU240wLI_to_gis20km_esmfaave.20240919.nc + cpl/gridmaps/oQU240wLI/map_oQU240wLI_to_gis20km_esmfbilin.20240919.nc + cpl/gridmaps/oQU240wLI/map_oQU240wLI_to_gis20km_esmfneareststod.20240919.deeperThan300m.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_oQU240wLI_esmfaave.20240919.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_oQU240wLI_esmfaave.20240919.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_oQU240wLI_esmfaave.20240919.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_oQU240wLI_esmfaave.20240919.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_oQU240wLI_esmfaave.20240919.nc + cpl/gridmaps/mpas.gis20km/map_gis20km_to_oQU240wLI_esmfaave.20240919.nc + + - cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis20km_aave.230510.nc - cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis20km_bilin.230510.nc + cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis20km_aave.230510.nc + cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis20km_bilin.230510.nc cpl/gridmaps/mpas.gis20km/map_gis20km_to_EC30to60E2r2_aave.230510.nc cpl/gridmaps/mpas.gis20km/map_gis20km_to_EC30to60E2r2_aave.230510.nc cpl/gridmaps/mpas.gis20km/map_gis20km_to_EC30to60E2r2_aave.230510.nc @@ -5825,8 +5857,9 @@ - cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis20km_esmfaave.20240403.nc - cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis20km_esmfbilin.20240403.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis20km_esmfaave.20240403.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis20km_esmfbilin.20240403.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis20km_esmfneareststod.20240422.deeperThan300m.nc cpl/gridmaps/mpas.gis20km/map_gis20km_to_IcoswISC30E3r5_esmfaave.20240403.nc cpl/gridmaps/mpas.gis20km/map_gis20km_to_IcoswISC30E3r5_esmfaave.20240403.nc cpl/gridmaps/mpas.gis20km/map_gis20km_to_IcoswISC30E3r5_esmfaave.20240403.nc @@ -5861,8 +5894,8 @@ - cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_gis1to10km_aave.200602.nc - cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_gis1to10km_bilin.200602.nc + cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_gis1to10km_aave.200602.nc + cpl/gridmaps/oEC60to30v3/map_oEC60to30v3_to_gis1to10km_bilin.200602.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10km_to_oEC60to30v3_aave.200602.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10km_to_oEC60to30v3_aave.200602.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10km_to_oEC60to30v3_aave.200602.nc @@ -5872,8 +5905,8 @@ - cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis1to10km_aave.210304.nc - cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis1to10km_bilin.210304.nc + cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis1to10km_aave.210304.nc + cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis1to10km_bilin.210304.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10km_to_EC30to60E2r2_aave.210304.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10km_to_EC30to60E2r2_aave.210304.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10km_to_EC30to60E2r2_aave.210304.nc @@ -5915,8 +5948,8 @@ - cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis1to10r02_aave.230725.nc - cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis1to10r02_bilin.230725.nc + cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis1to10r02_aave.230725.nc + cpl/gridmaps/EC30to60E2r2/map_EC30to60E2r2_to_gis1to10r02_bilin.230725.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10r02_to_EC30to60E2r2_aave.230725.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10r02_to_EC30to60E2r2_aave.230725.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10r02_to_EC30to60E2r2_aave.230725.nc @@ -5926,8 +5959,9 @@ - cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis1to10kmR2_esmfaave.20240403.nc - cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis1to10kmR2_esmfbilin.20240403.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis1to10kmR2_esmfaave.20240403.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis1to10kmR2_esmfbilin.20240403.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_gis1to10kmR2_esmfneareststod.20240422.deeperThan300m.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_IcoswISC30E3r5_esmfaave.20240403.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_IcoswISC30E3r5_esmfaave.20240403.nc cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_IcoswISC30E3r5_esmfaave.20240403.nc @@ -5955,8 +5989,8 @@ - cpl/gridmaps/oQU240wLI/map_oQU240wLI-nomask_to_ais20km_esmfaave.20240509.nc - cpl/gridmaps/oQU240wLI/map_oQU240wLI-nomask_to_ais20km_esmfbilin.20240509.nc + cpl/gridmaps/oQU240wLI/map_oQU240wLI-nomask_to_ais20km_esmfaave.20240509.nc + cpl/gridmaps/oQU240wLI/map_oQU240wLI-nomask_to_ais20km_esmfbilin.20240509.nc cpl/gridmaps/mpas.ais20km/map_ais20km_to_oQU240wLI-nomask_esmfaave.20240509.nc cpl/gridmaps/mpas.ais20km/map_ais20km_to_oQU240wLI-nomask_esmfbilin.20240509.nc cpl/gridmaps/mpas.ais20km/map_ais20km_to_oQU240wLI-nomask_esmfaave.20240509.nc @@ -5984,8 +6018,8 @@ - cpl/gridmaps/oQU240wLI/map_oQU240wLI-nomask_to_ais8to30_esmfaave.20240701.nc - cpl/gridmaps/oQU240wLI/map_oQU240wLI-nomask_to_ais8to30_esmfbilin.20240701.nc + cpl/gridmaps/oQU240wLI/map_oQU240wLI-nomask_to_ais8to30_esmfaave.20240701.nc + cpl/gridmaps/oQU240wLI/map_oQU240wLI-nomask_to_ais8to30_esmfbilin.20240701.nc cpl/gridmaps/mpas.ais8to30km/map_ais8to30_to_oQU240wLI-nomask_esmfaave.20240701.nc cpl/gridmaps/mpas.ais8to30km/map_ais8to30_to_oQU240wLI-nomask_esmfbilin.20240701.nc cpl/gridmaps/mpas.ais8to30km/map_ais8to30_to_oQU240wLI-nomask_esmfaave.20240701.nc @@ -5995,8 +6029,8 @@ - cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5-nomask_to_ais8to30_esmfaave.20240701.nc - cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5-nomask_to_ais8to30_esmfbilin.20240701.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5-nomask_to_ais8to30_esmfaave.20240701.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5-nomask_to_ais8to30_esmfbilin.20240701.nc cpl/gridmaps/mpas.ais8to30km/map_ais8to30_to_IcoswISC30E3r5-nomask_esmfaave.20240701.nc cpl/gridmaps/mpas.ais8to30km/map_ais8to30_to_IcoswISC30E3r5-nomask_esmfbilin.20240701.nc cpl/gridmaps/mpas.ais8to30km/map_ais8to30_to_IcoswISC30E3r5-nomask_esmfaave.20240701.nc @@ -6024,8 +6058,8 @@ - cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5-nomask_to_ais4to20_esmfaave.20240701.nc - cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5-nomask_to_ais4to20_esmfbilin.20240701.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5-nomask_to_ais4to20_esmfaave.20240701.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5-nomask_to_ais4to20_esmfbilin.20240701.nc cpl/gridmaps/mpas.ais4to20km/map_ais4to20_to_IcoswISC30E3r5-nomask_esmfaave.20240701.nc cpl/gridmaps/mpas.ais4to20km/map_ais4to20_to_IcoswISC30E3r5-nomask_esmfbilin.20240701.nc cpl/gridmaps/mpas.ais4to20km/map_ais4to20_to_IcoswISC30E3r5-nomask_esmfaave.20240701.nc @@ -6142,11 +6176,11 @@ cpl/gridmaps/mpas.ais20km/map_ais20km_to_oQU240_aave.151209.nc - cpl/gridmaps/oQU240/map_oQU240_to_ais20km_aave.151209.nc + cpl/gridmaps/oQU240/map_oQU240_to_ais20km_aave.151209.nc cpl/gridmaps/mpas.ais20km/map_ais20km_to_oQU240_nearestdtos.151209.nc cpl/gridmaps/mpas.ais20km/map_ais20km_to_oQU240_nearestdtos.151209.nc cpl/gridmaps/mpas.ais20km/map_ais20km_to_oQU240_nearestdtos.151209.nc - cpl/gridmaps/oQU240/map_oQU240_to_ais20km_nearestdtos.151209.nc + cpl/gridmaps/oQU240/map_oQU240_to_ais20km_nearestdtos.151209.nc @@ -6156,8 +6190,8 @@ cpl/gridmaps/mpas.ais20km/map_ais20km_to_oQU120_nearestdtos.160331.nc cpl/gridmaps/mpas.ais20km/map_ais20km_to_oQU120_nearestdtos.160331.nc cpl/gridmaps/mpas.ais20km/map_ais20km_to_oQU120_nearestdtos.160331.nc - cpl/gridmaps/oQU120/map_oQU120_to_ais20km_aave.160331.nc - cpl/gridmaps/oQU120/map_oQU120_to_ais20km_neareststod.160331.nc + cpl/gridmaps/oQU120/map_oQU120_to_ais20km_aave.160331.nc + cpl/gridmaps/oQU120/map_oQU120_to_ais20km_neareststod.160331.nc @@ -6167,8 +6201,8 @@ cpl/gridmaps/mpas.ais20km/map_ais20km_to_oEC60to30v3wLI_nomask_nearestdtos.190207.nc cpl/gridmaps/mpas.ais20km/map_ais20km_to_oEC60to30v3wLI_nomask_nearestdtos.190207.nc cpl/gridmaps/mpas.ais20km/map_ais20km_to_oEC60to30v3wLI_nomask_nearestdtos.190207.nc - cpl/gridmaps/oEC60to30v3wLI/map_oEC60to30v3wLI_nomask_to_ais20km_aave.190207.nc - cpl/gridmaps/oEC60to30v3wLI/map_oEC60to30v3wLI_nomask_to_ais20km_neareststod.190207.nc + cpl/gridmaps/oEC60to30v3wLI/map_oEC60to30v3wLI_nomask_to_ais20km_aave.190207.nc + cpl/gridmaps/oEC60to30v3wLI/map_oEC60to30v3wLI_nomask_to_ais20km_neareststod.190207.nc diff --git a/cime_config/machines/Depends.crayclanggpu.cmake b/cime_config/machines/Depends.crayclanggpu.cmake index eaff237a27f..3b0881ccd04 100644 --- a/cime_config/machines/Depends.crayclanggpu.cmake +++ b/cime_config/machines/Depends.crayclanggpu.cmake @@ -4,6 +4,8 @@ list(APPEND NOOPT_FILES elm/src/data_types/VegetationDataType.F90 elm/src/biogeochem/CNNitrogenFluxType.F90 elm/src/biogeochem/CNCarbonFluxType.F90 + mosart/src/wrm/WRM_subw_IO_mod.F90 + mosart/src/riverroute/RtmMod.F90 ) # Files added below to mitigate excessive compilation times diff --git a/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake b/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake index 92567416c56..49463844347 100644 --- a/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake +++ b/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake @@ -1,6 +1,5 @@ set(MPICC "cc") set(MPICXX "mpicxx") -#set(MPICXX "CC") set(MPIFC "ftn") set(SCC "cc") set(SCXX "hipcc") @@ -34,7 +33,7 @@ set(HAS_F2008_CONTIGUOUS "TRUE") # -Wl,--allow-shlib-undefined was added to address rocm 5.4.3 Fortran linker issue: # /opt/rocm-5.4.3/lib/libhsa-runtime64.so.1: undefined reference to `std::condition_variable::wait(std::unique_lock&)@GLIBCXX_3.4.30' # AMD started building with GCC 12.2.0, which brings in a GLIBCXX symbol that isn't in CCE's default GCC toolchain. -#string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition -Wl,--allow-shlib-undefined") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-shlib-undefined -Wl,--allow-multiple-definition") # Switching to O3 for performance benchmarking # Will revisit any failing tests diff --git a/cime_config/machines/cmake_macros/gnu_chicoma-gpu.cmake b/cime_config/machines/cmake_macros/gnu_chicoma-gpu.cmake index 807c7d0211e..a6c13942620 100644 --- a/cime_config/machines/cmake_macros/gnu_chicoma-gpu.cmake +++ b/cime_config/machines/cmake_macros/gnu_chicoma-gpu.cmake @@ -2,15 +2,10 @@ string(APPEND CONFIG_ARGS " --host=cray") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY") endif() -string(APPEND SLIBS " -lblas -llapack") -set(CXX_LINKER "FORTRAN") -if (NOT DEBUG) - string(APPEND CFLAGS " -O2 -g") -endif() -if (NOT DEBUG) - string(APPEND FFLAGS " -O2 -g") -endif() -string(APPEND CXX_LIBS " -lstdc++") +set(PIO_FILESYSTEM_HINTS "lustre") +string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -g") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -g") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--enable-new-dtags") set(MPICC "cc") set(MPICXX "CC") set(MPIFC "ftn") diff --git a/cime_config/machines/cmake_macros/oneapi-ifx.cmake b/cime_config/machines/cmake_macros/oneapi-ifx.cmake index 9ab0cdda7d5..e590456e9f3 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifx.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifx.cmake @@ -4,15 +4,15 @@ if (compile_threaded) string(APPEND CMAKE_CXX_FLAGS " -qopenmp") string(APPEND CMAKE_EXE_LINKER_FLAGS " -qopenmp") endif() -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -fpe0") +string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -gline-tables-only -g") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -gline-tables-only -g") +string(APPEND CMAKE_CXX_FLAGS_RELEASE " -fp-model precise -O2 -gline-tables-only -g") +string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created") string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g") string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g") -string(APPEND CMAKE_C_FLAGS " -fp-model precise -std=gnu99") -string(APPEND CMAKE_CXX_FLAGS " -fp-model precise") -string(APPEND CMAKE_Fortran_FLAGS " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise") +string(APPEND CMAKE_C_FLAGS " -fp-model precise -std=gnu99 -gline-tables-only -g") +string(APPEND CMAKE_CXX_FLAGS " -fp-model precise -gline-tables-only -g") +string(APPEND CMAKE_Fortran_FLAGS " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise -gline-tables-only -g") string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL -DHAVE_SLASHPROC -DHIDE_MPI") string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -fixed -132") string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -free") @@ -23,4 +23,6 @@ set(MPICXX "mpicxx") set(SCC "icx") set(SCXX "icpx") set(SFC "ifx") -set(E3SM_LINK_WITH_FORTRAN "TRUE") + + +#set(E3SM_LINK_WITH_FORTRAN "TRUE") diff --git a/cime_config/machines/cmake_macros/oneapi-ifx_auroracpu.cmake b/cime_config/machines/cmake_macros/oneapi-ifx_auroracpu.cmake new file mode 100644 index 00000000000..bd6ec8ed913 --- /dev/null +++ b/cime_config/machines/cmake_macros/oneapi-ifx_auroracpu.cmake @@ -0,0 +1,19 @@ + +string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core") +if (compile_threaded) + string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64") +endif() + +string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off") + +#set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "") + + + + + + + + + + diff --git a/cime_config/machines/cmake_macros/oneapi-ifx_sunspotcpu.cmake b/cime_config/machines/cmake_macros/oneapi-ifx_sunspotcpu.cmake new file mode 100644 index 00000000000..bd6ec8ed913 --- /dev/null +++ b/cime_config/machines/cmake_macros/oneapi-ifx_sunspotcpu.cmake @@ -0,0 +1,19 @@ + +string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core") +if (compile_threaded) + string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64") +endif() + +string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off") + +#set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "") + + + + + + + + + + diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake index d7dfae00219..faf8748217a 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake @@ -4,15 +4,17 @@ if (compile_threaded) string(APPEND CMAKE_CXX_FLAGS " -qopenmp") string(APPEND CMAKE_EXE_LINKER_FLAGS " -qopenmp") endif() -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -fpe0") + +#adding -g here leads to linker internal errors +string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -g -gline-tables-only") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -fpscomp logicals -g -gline-tables-only") +string(APPEND CMAKE_CXX_FLAGS_RELEASE " -fp-model precise -O2 -g -gline-tables-only") +string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -fpscomp logicals -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created") string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g") string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g") string(APPEND CMAKE_C_FLAGS " -fp-model precise -std=gnu99") string(APPEND CMAKE_CXX_FLAGS " -fp-model precise") -string(APPEND CMAKE_Fortran_FLAGS " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise") +string(APPEND CMAKE_Fortran_FLAGS " -fpscomp logicals -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise") string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL -DHAVE_SLASHPROC -DHIDE_MPI") string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -fixed -132") string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -free") diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake index 6835515164f..e5c486f216e 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake @@ -1,7 +1,12 @@ -string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -fsycl-device-code-split=per_kernel -fsycl-max-parallel-link-jobs=16 -Wl,--no-relax") if (compile_threaded) string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64") endif() + +string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ARCH_INTEL_PVC=On -DKokkos_ENABLE_SYCL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off") string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"") +string(APPEND OMEGA_SYCL_EXE_LINKER_FLAGS " -Xsycl-target-backend \"-device 12.60.7\" ") string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions") + +set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "") diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot-gen.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot-gen.cmake new file mode 100644 index 00000000000..9c9eb97add1 --- /dev/null +++ b/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot-gen.cmake @@ -0,0 +1,30 @@ + +set(CXX_LINKER "CXX") + +execute_process(COMMAND $ENV{NETCDF_PATH}/bin/nf-config --flibs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE) + +string(APPEND SLIBS " ${SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0} -Wl,-rpath -Wl,$ENV{NETCDF_PATH}/lib -lmkl_intel_lp64 -lmkl_sequential -lmkl_core") + +execute_process(COMMAND $ENV{NETCDF_PATH}/bin/nc-config --libs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE) + +string(APPEND SLIBS " ${SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0}") +string(APPEND SLIBS " -fiopenmp -fopenmp-targets=spir64") + +set(NETCDF_PATH "$ENV{NETCDF_PATH}") +set(PNETCDF_PATH "$ENV{PNETCDF_PATH}") + +set(USE_SYCL "TRUE") + +string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ARCH_INTEL_GEN=On -DKokkos_ENABLE_SYCL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off") + +string(APPEND SYCL_FLAGS " -\-intel -Xclang -fsycl-allow-virtual-functions -fsycl -mlong-double-64 -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") + +#string(APPEND SYCL_FLAGS " -\-intel -fsycl") +string(APPEND CXX_LDFLAGS " -Wl,-\-defsym,main=MAIN_\_ -lifcore -\-intel -Xclang -fsycl-allow-virtual-functions -fsycl -lsycl -mlong-double-64 -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64") + +SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") +SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +SET(CMAKE_FORTRAN_COMPILER "mpifort" CACHE STRING "") + + + diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot-pvc.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot-pvc.cmake new file mode 100644 index 00000000000..c6afa7c2329 --- /dev/null +++ b/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot-pvc.cmake @@ -0,0 +1,9 @@ + +string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -fsycl-device-code-split=per_kernel -fsycl-max-parallel-link-jobs=16 -Wl,--no-relax") +if (compile_threaded) + string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64") +endif() +string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ARCH_INTEL_PVC=On -DKokkos_ENABLE_SYCL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off") +string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"") + +set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "") diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake index 6835515164f..1ad0d6e6b61 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake @@ -3,5 +3,6 @@ string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_c if (compile_threaded) string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64") endif() -string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"") +string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 ") +string(APPEND OMEGA_SYCL_EXE_LINKER_FLAGS " -Xsycl-target-backend \"-device 12.60.7\" ") string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions") diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml index ee271d1f9dc..dcdf52178ef 100644 --- a/cime_config/machines/config_batch.xml +++ b/cime_config/machines/config_batch.xml @@ -547,6 +547,30 @@ + + /lus/gila/projects/CSC249ADSE15_CNDA/tools/qsub/throttle + + workq + debug + + + + + /lus/gila/projects/CSC249ADSE15_CNDA/tools/qsub/throttle + + workq + debug + + + + + /lus/gila/projects/CSC249ADSE15_CNDA/tools/qsub/throttle + + workq + debug + + + /lus/flare/projects/CSC249ADSE15_CNDA/tools/qsub/throttle @@ -555,6 +579,15 @@ workq + + + /lus/flare/projects/CSC249ADSE15_CNDA/tools/qsub/throttle + + EarlyAppAccess + workq-route + workq + + /grand/E3SMinput/soft/qsub/throttle diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index fce20fca185..cbd857ee154 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1069,7 +1069,7 @@ /usr/share/lmod/lmod/libexec/lmod python - Core Core/24.07 + Core Core/24.00 PrgEnv-cray PrgEnv-cray/8.3.3 cce cce/15.0.1 @@ -1082,16 +1082,22 @@ - Core Core/24.07 + Core Core/24.00 PrgEnv-cray PrgEnv-amd/8.3.3 amd amd/5.4.0 + + + - Core Core/24.07 + Core Core/24.00 PrgEnv-cray PrgEnv-gnu/8.3.3 gcc gcc/12.2.0 @@ -1100,9 +1106,9 @@ rocm/5.4.0 - cray-python/3.11.5 + cray-python/3.9.13.1 cray-libsci - cmake/3.27.9 + cmake/3.21.3 subversion git zlib @@ -2852,7 +2858,7 @@ /lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/openmpi/4.1.6/gcc-12.3.0/bin:/lcrc/group/e3sm/soft/perl/improv/bin:$ENV{PATH} $SHELL{lp=/lcrc/group/e3sm/soft/improv/netlib-lapack/3.12.0/gcc-12.3.0:/lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6/lib:/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6/lib:/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6/lib:/opt/pbs/lib:/lcrc/group/e3sm/soft/improv/openmpi/4.1.6/gcc-12.3.0/lib; if [ -z "$LD_LIBRARY_PATH" ]; then echo $lp; else echo "$lp:$LD_LIBRARY_PATH"; fi} $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /lcrc/soft/climate/moab/improv/gnu; else echo "$MOAB_ROOT"; fi} - ^lockedfile + ^lockedfile,individual 128M @@ -3116,6 +3122,148 @@ + + + ANL Sunspot Test and Development System (TDS), batch system is pbspro + uan-.* + LINUX + oneapi-ifxgpu + mpich + CSC249ADSE15_CNDA + /gila/CSC249ADSE15_CNDA/performance_archive + .* + /lus/gila/projects/CSC249ADSE15_CNDA/$USER/scratch + /lus/gila/projects/CSC249ADSE15_CNDA/inputdata + /lus/gila/projects/CSC249ADSE15_CNDA/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + /lus/gila/projects/CSC249ADSE15_CNDA/baselines/$COMPILER + /lus/gila/projects/CSC249ADSE15_CNDA/tools/cprnc/cprnc + 16 + e3sm_developer + 4 + pbspro + e3sm + 12 + 12 + 12 + 12 + FALSE + + mpiexec + + + -np {{ total_tasks }} --label + -ppn {{ tasks_per_node }} + --no-vni --cpu-bind $ENV{RANKS_BIND} -envall + -d $ENV{OMP_NUM_THREADS} + $ENV{GPU_TILE_COMPACT} + + + + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh + /usr/share/lmod/lmod/init/env_modules_python.py + module + module + /usr/share/lmod/lmod/libexec/lmod python + + + /soft/modulefiles + /soft/restricted/CNDA/updates/modulefiles + spack-pe-gcc/0.7.0-24.086.0 cmake + python/3.10.11 + + + oneapi/eng-compiler/2024.07.30.002 mpich/icc-all-pmix-gpu/20240717 + + + cray-pals/1.4.0 + libfabric/1.20.1 + + + $CIME_OUTPUT_ROOT/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/bld + + /lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf + /lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf + /lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/pnetcdf + /lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf/lib:$ENV{LD_LIBRARY_PATH} + /lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf/bin:$ENV{PATH} + + list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203 + + + 1 + + + + + + 1 + 0 + + + + 1 + 1 + 1 + + 131072 + 20 + + disabled + 8388608 + + 240 + 240 + + disable + disable + + level_zero:gpu + 1 + + 4000MB + 0 + + /soft/tools/mpi_wrapper_utils/gpu_tile_compact.sh + + + + + + verbose,granularity=thread,balanced + 128M + + + -1 + + + + + + ALCF Polaris 560 nodes, 2.8 GHz AMD EPYC Milan 7543P 32c CPU, 4 NVIDIA A100 GPUs polaris-* @@ -3211,6 +3359,201 @@ + + + ANL Sunspot Test and Development System (TDS), batch system is pbspro + uan-.* + LINUX + oneapi-ifx + mpich + CSC249ADSE15_CNDA + /gila/CSC249ADSE15_CNDA/performance_archive + .* + /lus/gila/projects/CSC249ADSE15_CNDA/$USER/scratch + /lus/gila/projects/CSC249ADSE15_CNDA/inputdata + /lus/gila/projects/CSC249ADSE15_CNDA/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + /lus/gila/projects/CSC249ADSE15_CNDA/baselines/$COMPILER + /lus/gila/projects/CSC249ADSE15_CNDA/tools/cprnc/cprnc + 16 + e3sm_developer + 4 + pbspro + e3sm + 208 + 104 + FALSE + + mpiexec + + + -np {{ total_tasks }} --label + -ppn {{ tasks_per_node }} + --cpu-bind depth -envall + -d $ENV{OMP_NUM_THREADS} + + + + + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh + /usr/share/lmod/lmod/init/env_modules_python.py + module + module + /usr/share/lmod/lmod/libexec/lmod python + + + + spack-pe-gcc/0.7.0-24.086.0 cmake python/3.10.11 + + + + oneapi/eng-compiler/2024.04.15.002 + mpich/icc-all-pmix-gpu/20231026 + + + + + cray-pals + + libfabric/1.15.2.0 + + + $CIME_OUTPUT_ROOT/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/bld + + /lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf + /lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf + /lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/pnetcdf + /lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf/lib:$ENV{LD_LIBRARY_PATH} + /lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf/bin:$ENV{PATH} + list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203 + + + 1 + + + DISABLED + + + + + + 131072 + 20 + 0 + + + verbose,granularity=thread,balanced + 128M + + + -1 + + + + + + + + + + ANL Sunspot Test and Development System (TDS), batch system is pbspro + uan-.* + LINUX + oneapi-ifxgpu + mpich + CSC249ADSE15_CNDA + /gila/CSC249ADSE15_CNDA/performance_archive + .* + /lus/gila/projects/CSC249ADSE15_CNDA/$USER/scratch + /lus/gila/projects/CSC249ADSE15_CNDA/inputdata + /lus/gila/projects/CSC249ADSE15_CNDA/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + /lus/gila/projects/CSC249ADSE15_CNDA/baselines/$COMPILER + /lus/gila/projects/CSC249ADSE15_CNDA/tools/cprnc/cprnc + 16 + e3sm_developer + 4 + pbspro + e3sm + 12 + 12 + 12 + 12 + FALSE + + mpiexec + + + -np {{ total_tasks }} --label + -ppn {{ tasks_per_node }} + --cpu-bind depth -envall + -d $ENV{OMP_NUM_THREADS} + $ENV{GPU_TILE_COMPACT} + + + + /soft/packaging/lmod/lmod/init/sh + /soft/packaging/lmod/lmod/init/csh + /soft/packaging/lmod/lmod/init/env_modules_python.py + module + module + /soft/packaging/lmod/lmod/libexec/lmod python + + + /soft/modulefiles + spack cmake/3.24.2 python/3.9.13-gcc-11.2.0-76jlbxs + /soft/restricted/CNDA/updates/modulefiles + + + oneapi/eng-compiler/2023.10.15.002 + mpich/52.2-256/icc-all-pmix-gpu + + + + + + + + cray-pals + append-deps/default + libfabric/1.15.2.0 + + + $CIME_OUTPUT_ROOT/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/bld + + /lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf + /lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf + /lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/pnetcdf + + + 1 + + + level_zero:gpu + NO_GPU + 0 + disable + disable + 1 + 4000MB + 0 + /soft/tools/mpi_wrapper_utils/gpu_tile_compact.sh + 131072 + 20 + + + verbose,granularity=thread,balanced + 128M + + + -1 + + + + ANL Sunspot Test and Development System (TDS), batch system is pbspro uan-.* @@ -3293,7 +3636,6 @@ /soft/tools/mpi_wrapper_utils/gpu_tile_compact.sh 131072 20 - $ENV{KOKKOS_ROOT} 1 0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4 @@ -3339,9 +3681,9 @@ pbspro e3sm 208 - 104 + 12 104 - 48 + 12 FALSE mpiexec @@ -3349,7 +3691,7 @@ -np {{ total_tasks }} --label -ppn {{ tasks_per_node }} - -envall + --no-vni --cpu-bind $ENV{RANKS_BIND} -envall -d $ENV{OMP_NUM_THREADS} $ENV{GPU_TILE_COMPACT} @@ -3362,17 +3704,18 @@ module /soft/sunspot_migrate/soft/packaging/lmod/lmod/libexec/lmod python - cmake + + /soft/modulefiles + /soft/restricted/CNDA/updates/modulefiles + spack-pe-gcc/0.7.0-24.086.0 cmake + python/3.10.11 - oneapi/eng-compiler/2024.04.15.002 + oneapi/eng-compiler/2024.07.30.002 mpich/icc-all-pmix-gpu/20240717 - - kokkos/git.7ff87a5-omp-sycl - - - spack-pe-gcc cmake - gcc/10.3.0 + + cray-pals/1.4.0 + libfabric/1.20.1 $CIME_OUTPUT_ROOT/$CASE/run @@ -3381,28 +3724,167 @@ /lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002 /lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002 /lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002 - /lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002/lib:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002/lib:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002/lib:$ENV{LD_LIBRARY_PATH} - /lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002/bin:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002/bin:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002/bin:$ENV{PATH} + /opt/cray/pe/gcc-libs:/opt/cray/pe/python/3.9.13.1/lib:/lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002/lib:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002/lib:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002/lib:$ENV{LD_LIBRARY_PATH} + /opt/cray/pe/python/3.9.13.1/bin:/lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002/bin:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002/bin:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002/bin:$ENV{PATH} + list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203 1 + + + - level_zero:gpu - NO_GPU - 0 - disable - disable - 1 + + 1 + 0 + + + + 1 + 1 + 1 + + 131072 + 20 + cxi + disabled + 8388608 + + 240 + 240 + + disable + disable + + level_zero:gpu + 1 + 4000MB 0 - /soft/tools/mpi_wrapper_utils/gpu_tile_compact.sh + + /soft/tools/mpi_wrapper_utils/gpu_tile_compact.sh + + + + + + + 0 + DISABLED 131072 20 - $ENV{KOKKOS_ROOT} - 1 - 0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4 + 0 + + + + + verbose,granularity=thread,balanced + 128M + + + + threads + 128M + + + -1 + + + + + + + ALCF Aurora, 10624 nodes, 2x52c SPR, 6x2s PVC, 2x512GB DDR5, 2x64GB CPU-HBM, 6x128GB GPU-HBM, Slingshot 11, PBSPro + aurora-uan-.* + LINUX + oneapi-ifx + mpich + CSC249ADSE15_CNDA + /lus/flare/projects/CSC249ADSE15_CNDA/performance_archive + .* + /lus/flare/projects/CSC249ADSE15_CNDA/$USER/scratch + /lus/flare/projects/CSC249ADSE15_CNDA/inputdata + /lus/flare/projects/CSC249ADSE15_CNDA/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + /lus/flare/projects/CSC249ADSE15_CNDA/baselines/$COMPILER + /lus/flare/projects/CSC249ADSE15_CNDA/tools/cprnc/cprnc + 16 + e3sm_developer + 4 + pbspro + e3sm + 208 + 104 + FALSE + + mpiexec + + + -np {{ total_tasks }} --label + -ppn {{ tasks_per_node }} + --cpu-bind $ENV{RANKS_BIND} -envall + -d $ENV{OMP_NUM_THREADS} + $ENV{GPU_TILE_COMPACT} + + + + /lus/flare/projects/CSC249ADSE15_CNDA/modules/lmod.sh + /soft/sunspot_migrate/soft/packaging/lmod/lmod/init/csh + /soft/sunspot_migrate/soft/packaging/lmod/lmod/init/env_modules_python.py + module + module + /soft/sunspot_migrate/soft/packaging/lmod/lmod/libexec/lmod python + + + /soft/modulefiles + /soft/restricted/CNDA/updates/modulefiles + spack-pe-gcc/0.6.1-23.275.2 cmake + python/3.10.10 + + + oneapi/release/2023.12.15.001 + + + cray-pals/1.3.3 + libfabric/1.15.2.0 + + + $CIME_OUTPUT_ROOT/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/bld + + /lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2023.05.15.007 + /lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2023.05.15.007 + /lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2023.05.15.007 + /opt/cray/pe/gcc-libs:/opt/cray/pe/python/3.9.13.1/lib:/lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2023.05.15.007/lib:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2023.05.15.007/lib:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2023.05.15.007/lib:$ENV{LD_LIBRARY_PATH} + /opt/cray/pe/python/3.9.13.1/bin:/lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2023.05.15.007/bin:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2023.05.15.007/bin:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2023.05.15.007/bin:$ENV{PATH} + list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203 + + + 1 + 0 DISABLED @@ -3424,6 +3906,15 @@ + + + + + + + + + PNL cluster, OS is Linux, batch system is SLURM sooty @@ -4140,8 +4631,7 @@ - Chicoma GPU nodes at LANL IC. Each GPU node has single -AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' + Chicoma GPU nodes at LANL IC. Each GPU node has single AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' ch-fe* Linux gnugpu,gnu,nvidiagpu,nvidia @@ -4151,7 +4641,7 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' /usr/projects/e3sm/inputdata/atm/datm7 /lustre/scratch5/$ENV{USER}/E3SM/archive/$CASE /lustre/scratch5/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER - /usr/projects/climate/SHARED_CLIMATE/software/badger/cprnc + /usr/projects/e3sm/software/chicoma-cpu/cprnc 10 e3sm_developer 4 @@ -4175,11 +4665,11 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' - /usr/share/lmod/8.3.1/init/perl + /usr/share/lmod/lmod/init/perl - /usr/share/lmod/8.3.1/init/python - /usr/share/lmod/8.3.1/init/sh - /usr/share/lmod/8.3.1/init/csh + /usr/share/lmod/lmod/init/python + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module @@ -4191,32 +4681,35 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' cray-parallel-netcdf cray-netcdf cray-hdf5 - PrgEnv-gnu - PrgEnv-intel - PrgEnv-nvidia - PrgEnv-cray - PrgEnv-aocc intel intel-oneapi nvidia aocc cudatoolkit climate-utils + cray-libsci craype-accel-nvidia80 craype-accel-host perftools-base perftools darshan + PrgEnv-gnu + PrgEnv-intel + PrgEnv-nvidia + PrgEnv-cray + PrgEnv-aocc - PrgEnv-gnu/8.4.0 - gcc/11.2.0 + PrgEnv-gnu/8.5.0 + gcc/12.2.0 + cray-libsci/23.05.1.4 PrgEnv-nvidia/8.4.0 nvidia/22.7 + cray-libsci/23.05.1.4 @@ -4239,14 +4732,13 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' - cray-libsci/23.05.1.4 + craype-accel-host craype/2.7.21 cray-mpich/8.1.26 - libfabric/1.15.2.0 cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 - cmake/3.25.1 + cmake/3.27.7 @@ -4269,6 +4761,9 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' $ENV{CRAY_PARALLEL_NETCDF_PREFIX} /usr/projects/e3sm/cudatoolkit:$ENV{PKG_CONFIG_PATH} + + /opt/cray/pe/gcc/12.2.0/snos/lib64:$ENV{LD_LIBRARY_PATH} + -1 diff --git a/cime_config/testmods_dirs/config_pes_tests.xml b/cime_config/testmods_dirs/config_pes_tests.xml index 1ac88ac5fe6..eafc8e2a1fe 100644 --- a/cime_config/testmods_dirs/config_pes_tests.xml +++ b/cime_config/testmods_dirs/config_pes_tests.xml @@ -161,31 +161,41 @@ - tests+anvil: --compset WCYCL* --res ne30pg2_IcoswISC30E3r5 on 16 nodes pure-MPI + tests+anvil: --compset WCYCL* --res ne30pg2_IcoswISC30E3r5 on 25 nodes pure-MPI - 396 - 396 - 396 - 396 - 180 - 396 + 675 + 324 + 324 + 360 + 216 + 684 - 396 + 0 + 360 + 360 + 0 + 684 + 0 tests+anvil: --compset BGC* --res ne30pg2_r05_IcoswISC30E3r5 on 30 nodes pure-MPI 675 - 684 - 684 - 684 - 396 + 324 + 324 + 360 + 216 684 + 0 + 360 + 360 + 0 684 + 0 @@ -203,6 +213,17 @@ -6 + + "tests+anvil, F compset, 6 nodes" + + -16 + -16 + -16 + -16 + -16 + -16 + + @@ -241,17 +262,17 @@ - tests+anvil: --compset WCYCL1850 --res northamericax4v1pg2_WC14to60E2r3 on 64 nodes pure-MPI, 2.133 sypd + tests+anvil: --compset WCYCL1850 --res northamericax4v1pg2_WC14to60E2r3 on 69 nodes pure-MPI, 2.046 sypd - 1800 - 1800 - 1800 - 1800 + 1980 + 1980 + 1980 + 1944 504 - 1800 + 1980 - 1800 + 1980 diff --git a/cime_config/tests.py b/cime_config/tests.py index 46894e6ea96..d67f15e7469 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -177,7 +177,7 @@ ) }, - "e3sm_p3_developer" : { + "e3sm_p3_developer" : { "tests" : ( "ERP.ne4pg2_oQU480.F2010.eam-p3", "REP_Ln5.ne4pg2_oQU480.F2010.eam-p3", @@ -189,6 +189,17 @@ "ERS.ne4pg2_oQU480.F2010.eam-p3" ) }, + + "e3sm_orodrag_developer" : { + "tests" : ( + "ERP.ne4pg2_oQU480.F2010.eam-orodrag_ne4pg2", + "REP_Ln5.ne4pg2_oQU480.F2010.eam-orodrag_ne4pg2", + "PET.ne4pg2_oQU480.F2010.eam-orodrag_ne4pg2", + "PEM_Ln18.ne4pg2_oQU480.F2010.eam-orodrag_ne4pg2", + "SMS_Ln5.ne30pg2_EC30to60E2r2.F2010.eam-orodrag_ne30pg2", + "SMS_D_Ln5.ne4pg2_oQU480.F2010.eam-orodrag_ne4pg2" + ) + }, "e3sm_atm_integration" : { "inherit" : ("eam_preqx", "eam_theta"), @@ -269,6 +280,8 @@ "SMS_D_Ld1.T62_oQU240.GMPAS-IAF.mpaso-harmonic_mean_drag", "SMS_D_Ld1.T62_oQU240.GMPAS-IAF.mpaso-upwind_advection", "ERS_Ld5_D.T62_oQU240.GMPAS-IAF.mpaso-conservation_check", + "ERS_Ld5_PS.ne30pg2_r05_IcoswISC30E3r5.CRYO1850-DISMF.mpaso-scaled_dib_dismf", + "ERS_Ld5.TL319_oQU240wLI_gis20.MPAS_LISIO_JRA1p5.mpaso-ocn_glc_tf_coupling", ) }, @@ -310,6 +323,8 @@ "ERS_Ld5.TL319_oQU240wLI_ais8to30.MPAS_LISIO_JRA1p5.mpaso-ocn_glcshelf", "SMS_P12x2.ne4pg2_oQU480.WCYCL1850NS.allactive-mach_mods", "ERS_Ln9.ne4pg2_ne4pg2.F2010-MMF1.eam-mmf_crmout", + "SMS_Lh4.ne4_ne4.F2010-SCREAMv1.eamxx-output-preset-1", + "SMS_Lh4.ne4pg2_ne4pg2.F2010-SCREAMv1.eamxx-output-preset-1--eamxx-prod", ) }, @@ -520,6 +535,7 @@ "SMS.ne4pg2_oQU480.F2010.eam-thetanh_ftype2", "SMS.ne4pg2_oQU480.F2010.eam-thetanh_ftype4", "SMS.ne4pg2_oQU480.F2010.eam-thetahy_sl", + "ERS.ne4pg2_oQU480.F2010.eam-thetahy_sl_nsubstep2", "ERS.ne4pg2_oQU480.F2010.eam-thetahy_ftype2", "ERS.ne4pg2_oQU480.F2010.eam-thetanh_ftype2", ) @@ -639,17 +655,17 @@ "time" : "03:00:00", }, - "e3sm_scream" : { + "e3sm_eamxx" : { "time" : "03:00:00", - "inherit" : ("e3sm_scream_v0"), + "inherit" : ("e3sm_eamxx_v0"), }, - "e3sm_scream_v0" : { + "e3sm_eamxx_v0" : { "time" : "03:00:00", - "inherit" : ("e3sm_scream_v0_lowres"), + "inherit" : ("e3sm_eamxx_v0_lowres"), }, - "e3sm_scream_v0_lowres" : { + "e3sm_eamxx_v0_lowres" : { "time" : "03:00:00", "tests" : ( "SMS_D.ne4pg2_ne4pg2.F2010-SCREAM-HR", @@ -660,69 +676,70 @@ ) }, - "e3sm_scream_v1" : { + "e3sm_eamxx_v1" : { "time" : "03:00:00", - "inherit" : ("e3sm_scream_v1_lowres", "e3sm_scream_v1_medres", "e3sm_scream_v1_mpassi"), + "inherit" : ("e3sm_eamxx_v1_lowres", "e3sm_eamxx_v1_medres", "e3sm_eamxx_v1_mpassi"), }, - "e3sm_scream_v1_lowres" : { + "e3sm_eamxx_v1_lowres" : { "time" : "01:00:00", - "inherit" : ("e3sm_scream_mam4xx_v1_lowres"), + "inherit" : ("e3sm_eamxx_mam4xx_v1_lowres"), "tests" : ( - "ERP_D_Lh4.ne4_ne4.F2010-SCREAMv1.scream-output-preset-1", - "ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.scream-output-preset-2", - "SMS_D_Ln9.ne4_ne4.F2010-SCREAMv1-noAero.scream-output-preset-3", - "ERP_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-output-preset-4", - "ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-rad_frequency_2--scream-output-preset-5", - "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5", - "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels_p3--scream-output-preset-5", - "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels_shoc--scream-output-preset-5", - "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-all_mam4xx_procs", + "ERP_D_Lh4.ne4_ne4.F2010-SCREAMv1.eamxx-output-preset-1", + "ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.eamxx-output-preset-2", + "SMS_D_Ln9.ne4_ne4.F2010-SCREAMv1-noAero.eamxx-output-preset-3", + "ERP_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.eamxx-output-preset-4", + "ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.eamxx-rad_frequency_2--eamxx-output-preset-5", + "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.eamxx-small_kernels--eamxx-output-preset-5", + "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.eamxx-small_kernels_p3--eamxx-output-preset-5", + "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.eamxx-small_kernels_shoc--eamxx-output-preset-5", + "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.eamxx-mam4xx-all_mam4xx_procs", ) }, - "e3sm_scream_v1_dp-eamxx" : { + "e3sm_eamxx_v1_dp-eamxx" : { "time" : "01:00:00", # each test runs with 225 dynamics and 100 physics columns, roughly size of ne2 "tests" : ( - "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-dycomsrf01", - "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-arm97", - "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-comble", + "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.eamxx-dpxx-dycomsrf01", + "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.eamxx-dpxx-arm97", + "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.eamxx-dpxx-comble", "ERS_P16_Ln22.ne30pg2_ne30pg2.FRCE-SCREAMv1-DP", ) }, - # Tests run on exclusively on mappy for scream AT testing. These tests + # Tests run on exclusively on mappy for eamxx AT testing. These tests # should be fast, so we limit it to low res and add some thread tests # specifically for mappy. - "e3sm_scream_v1_at" : { - "inherit" : ("e3sm_scream_v1_lowres", "e3sm_scream_v1_dp-eamxx"), - "tests" : ("PET_Ln9_P32x2.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-output-preset-1") + "e3sm_eamxx_v1_at" : { + "inherit" : ("e3sm_eamxx_v1_lowres", "e3sm_eamxx_v1_dp-eamxx"), + "tests" : ("PET_Ln9_P32x2.ne4pg2_ne4pg2.F2010-SCREAMv1.eamxx-output-preset-1") }, - "e3sm_scream_v1_medres" : { + "e3sm_eamxx_v1_medres" : { "time" : "02:00:00", "tests" : ( # "SMS_D_Ln2.ne30_ne30.F2000-SCREAMv1-AQP1", # Uncomment once IC file for ne30 is ready - "ERS_Ln22.ne30_ne30.F2010-SCREAMv1.scream-internal_diagnostics_level--scream-output-preset-3", - "PEM_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-spa_remap--scream-output-preset-4", - "ERS_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5", - "ERP_Ln22.conusx4v1pg2_r05_oECv3.F2010-SCREAMv1-noAero.scream-bfbhash--scream-output-preset-6", - "ERS_Ln22.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-L128--scream-output-preset-4", - "REP_Ld5.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-L128--scream-output-preset-6" + "ERS_Ln22.ne30_ne30.F2010-SCREAMv1.eamxx-internal_diagnostics_level--eamxx-output-preset-3", + "PEM_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.eamxx-spa_remap--eamxx-output-preset-4", + "ERS_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.eamxx-small_kernels--eamxx-output-preset-5", + "ERP_Ln22.conusx4v1pg2_r05_oECv3.F2010-SCREAMv1-noAero.eamxx-bfbhash--eamxx-output-preset-6", + "ERS_Ln22.ne30pg2_ne30pg2.F2010-SCREAMv1.eamxx-L128--eamxx-output-preset-4", + "REP_Ld5.ne30pg2_ne30pg2.F2010-SCREAMv1.eamxx-L128--eamxx-output-preset-6", + "ERS_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.eamxx-L128--eamxx-sl_nsubstep2", ) }, # Used to track performance - "e3sm_scream_v1_hires" : { + "e3sm_eamxx_v1_hires" : { "time" : "01:00:00", "tests" : ( - "SMS_Ln300.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-perf_test--scream-output-preset-1" + "SMS_Ln300.ne30pg2_ne30pg2.F2010-SCREAMv1.eamxx-perf_test--eamxx-output-preset-1" ) }, - "e3sm_scream_v1_mpassi" : { + "e3sm_eamxx_v1_mpassi" : { "time" : "01:00:00", "tests" : ( # "ERP_D_Ln9.ne4_oQU240.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off", @@ -730,14 +747,14 @@ # Disable the two 111422-commented tests b/c they fail on pm-gpu and # we're not using MPASSI right now. #111422 "ERP_Ln22.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off", - "ERS_D_Ln22.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off--scream-output-preset-1", + "ERS_D_Ln22.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off--eamxx-output-preset-1", # "ERS_Ln22.ne30_oECv3.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off", #111422 "PEM_Ln90.ne30pg2_EC30to60E2r2.F2010-SCREAMv1-MPASSI", # "ERS_Ln22.ne30pg2_EC30to60E2r2.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off", ) }, - "e3sm_scream_v1_long" : { + "e3sm_eamxx_v1_long" : { "time" : "01:00:00", "tests" : ( "ERP_D_Lh182.ne4pg2_ne4pg2.F2010-SCREAMv1", @@ -745,7 +762,7 @@ ) }, - "e3sm_scream_v1_long_crusher" : { + "e3sm_eamxx_v1_long_crusher" : { # _D builds take a long longer on crusher than ascent or pm-gpu, so # don't run the long _D test. "time" : "01:00:00", @@ -754,17 +771,31 @@ ) }, - "e3sm_scream_mam4xx_v1_lowres" : { + "e3sm_eamxx_mam4xx_v1_lowres" : { "time" : "01:00:00", "tests" : ( - "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-optics", - "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-aci", - "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-wetscav", - "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-drydep", - "SMS_D_Ln5.ne30pg2_oECv3.F2010-SCREAMv1-MPASSI.scream-mam4xx-remap_emiss_ne4_ne30" + "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.eamxx-mam4xx-optics", + "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.eamxx-mam4xx-aci", + "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.eamxx-mam4xx-wetscav", + "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.eamxx-mam4xx-drydep", + "SMS_D_Ln5.ne30pg2_oECv3.F2010-SCREAMv1-MPASSI.eamxx-mam4xx-remap_emiss_ne4_ne30" ) }, + "e3sm_moab_dev" : { + "time" : "01:00:00", + "tests" : ( + "ERS_Vmoab_Ld3.ne4pg2_r05_oQU480.WCYCL1850NS", + "ERS_Vmoab_Ld3.ne4pg2_oQU480.WCYCL1850NS", + "ERS_Vmoab_Ld3.ne4pg2_oQU480.F1850", + "ERS_Vmoab_Ld3.ne4pg2_ne4pg2.I1850CNPRDCTCBCTOP", + "ERS_Vmoab_Ld3.T62_oQU240wLI.GMPAS-IAF", + "ERS_Vmoab_Ld3.T62_oQU120.CMPASO-NYF", + "ERS_Vmoab_Ld3.r05_r05.RMOSGPCC", + ) + }, + + "e3sm_gpuacc" : { "tests" : ( diff --git a/components/data_comps/dice/src/ice_comp_mct.F90 b/components/data_comps/dice/src/ice_comp_mct.F90 index f8c48c89240..3cd5557f1f2 100644 --- a/components/data_comps/dice/src/ice_comp_mct.F90 +++ b/components/data_comps/dice/src/ice_comp_mct.F90 @@ -79,16 +79,6 @@ subroutine ice_init_mct( EClock, cdata, x2i, i2x, NLFilename ) logical :: scmMode = .false. ! single column mode real(R8) :: scmLat = shr_const_SPVAL ! single column lat real(R8) :: scmLon = shr_const_SPVAL ! single column lon -#ifdef HAVE_MOAB - character(CL) :: filePath ! generic file path - character(CL) :: fileName ! generic file name - character(CS) :: timeName ! domain file: time variable name - character(CS) :: lonName ! domain file: lon variable name - character(CS) :: latName ! domain file: lat variable name - character(CS) :: hgtName ! domain file: hgt variable name - character(CS) :: maskName ! domain file: mask variable name - character(CS) :: areaName ! domain file: area variable name -#endif character(*), parameter :: subName = "(ice_init_mct) " !------------------------------------------------------------------------------- @@ -171,12 +161,9 @@ subroutine ice_init_mct( EClock, cdata, x2i, i2x, NLFilename ) #ifdef HAVE_MOAB if (my_task == master_task) then - call shr_stream_getDomainInfo(SDICE%stream(1), filePath,fileName,timeName,lonName, & - latName,hgtName,maskName,areaName) - call shr_stream_getFile(filePath,fileName) ! send path of ice domain to MOAB coupler. - call seq_infodata_PutData( infodata, ice_domain=fileName) - write(logunit,*), ' filename: ', filename + write(logunit,*), ' file used for ice domain ', SDICE%domainFile + call seq_infodata_PutData( infodata, ice_domain=SDICE%domainFile) endif #endif !---------------------------------------------------------------------------- diff --git a/components/data_comps/docn/cime_config/config_component.xml b/components/data_comps/docn/cime_config/config_component.xml index 431d358f995..68ec6a18c9f 100644 --- a/components/data_comps/docn/cime_config/config_component.xml +++ b/components/data_comps/docn/cime_config/config_component.xml @@ -13,10 +13,11 @@ This file may have ocn desc entries. --> - DOCN + DOCN null mode prescribed ocean mode slab ocean mode + relaxed slab ocean mode aquaplanet slab ocean mode interannual mode aquaplanet mode: @@ -45,12 +46,13 @@ char - prescribed,sst_aquap1,sst_aquap2,sst_aquap3,sst_aquap4,sst_aquap5,sst_aquap6,sst_aquap7,sst_aquap8,sst_aquap9,sst_aquap10,sst_aquap11,sst_aquap12,sst_aquap13,sst_aquap14,sst_aquap15,sst_aquapfile,som,som_aquap,sst_aquap_constant,interannual,null + prescribed,sst_aquap1,sst_aquap2,sst_aquap3,sst_aquap4,sst_aquap5,sst_aquap6,sst_aquap7,sst_aquap8,sst_aquap9,sst_aquap10,sst_aquap11,sst_aquap12,sst_aquap13,sst_aquap14,sst_aquap15,sst_aquapfile,som,rso,som_aquap,sst_aquap_constant,interannual,null prescribed null prescribed som + rso som_aquap interannual sst_aquap1 diff --git a/components/data_comps/docn/cime_config/namelist_definition_docn.xml b/components/data_comps/docn/cime_config/namelist_definition_docn.xml index a191d088d7f..fb28b7471c5 100644 --- a/components/data_comps/docn/cime_config/namelist_definition_docn.xml +++ b/components/data_comps/docn/cime_config/namelist_definition_docn.xml @@ -32,6 +32,7 @@ Currently the following datamods are supported prescribed SSTDATA (Run with prescribed SST, ICE_COV) som SOM (Slab ocean model) + rso RSO (Relaxed slab ocean model) null NULL (NULL mode) --> @@ -59,6 +60,7 @@ aquapfile '' som + rso som interannual @@ -93,6 +95,7 @@ null $DIN_LOC_ROOT/ocn/docn7/AQUAPLANET/ $DIN_LOC_ROOT/ocn/docn7/SOM + / $DIN_LOC_ROOT/atm/cam/sst @@ -106,6 +109,7 @@ null $DOCN_AQP_FILENAME $DOCN_SOM_FILENAME + $SSTICE_GRID_FILENAME sst_HadOIBl_bc_1x1_1850_2014_c150416.nc @@ -145,6 +149,7 @@ null $DIN_LOC_ROOT/ocn/docn7/AQUAPLANET $DIN_LOC_ROOT/ocn/docn7/SOM + / $DIN_LOC_ROOT/atm/cam/sst @@ -158,6 +163,7 @@ null $DOCN_AQP_FILENAME $DOCN_SOM_FILENAME + $SSTICE_DATA_FILENAME sst_HadOIBl_bc_1x1_1850_2014_c150416.nc @@ -181,6 +187,10 @@ hblt h qdp qbot + + SST_cpl t + hblt h + SST_cpl t @@ -213,6 +223,7 @@ $SSTICE_YEAR_ALIGN 0 1 + $SSTICE_YEAR_ALIGN 1 @@ -227,6 +238,7 @@ $SSTICE_YEAR_START 0 1 + $SSTICE_YEAR_START 1850 @@ -241,6 +253,7 @@ $SSTICE_YEAR_END 0 1 + $SSTICE_YEAR_END 2014 @@ -257,7 +270,7 @@ char streams shr_strdata_nml - SSTDATA,SST_AQUAP1,SST_AQUAP2,SST_AQUAP3,SST_AQUAP4,SST_AQUAP5,SST_AQUAP6,SST_AQUAP7,SST_AQUAP8,SST_AQUAP9,SST_AQUAP10,SST_AQUAP11,SST_AQUAP12,SST_AQUAP13,SST_AQUAP14,SST_AQUAP15,SST_AQUAPFILE,SST_AQUAP_CONSTANT,SOM,SOM_AQUAP,IAF,NULL,COPYALL + SSTDATA,SST_AQUAP1,SST_AQUAP2,SST_AQUAP3,SST_AQUAP4,SST_AQUAP5,SST_AQUAP6,SST_AQUAP7,SST_AQUAP8,SST_AQUAP9,SST_AQUAP10,SST_AQUAP11,SST_AQUAP12,SST_AQUAP13,SST_AQUAP14,SST_AQUAP15,SST_AQUAPFILE,SST_AQUAP_CONSTANT,SOM,RSO,SOM_AQUAP,IAF,NULL,COPYALL General method that operates on the data. This is generally implemented in the data models but is set in the strdata method for @@ -331,6 +344,7 @@ SST_AQUAPFILE SST_AQUAP_CONSTANT SOM + RSO SOM_AQUAP IAF @@ -662,4 +676,29 @@ + + real(30) + docn + docn_nml + + Relaxation timescale for relaxed slab ocean (RSO) mode + + + 691200 + + + + + real(30) + docn + docn_nml + + globally fixed mixed layer depth (MLD) for relaxed slab ocean (RSO) mode + use -1 to disable - input data file should have hblt field to override this + + + 50 + + + diff --git a/components/data_comps/docn/src/docn_comp_mod.F90 b/components/data_comps/docn/src/docn_comp_mod.F90 index 43bac32bff7..00a7ef4338c 100644 --- a/components/data_comps/docn/src/docn_comp_mod.F90 +++ b/components/data_comps/docn/src/docn_comp_mod.F90 @@ -29,6 +29,8 @@ module docn_comp_mod use docn_shr_mod , only: rest_file ! namelist input use docn_shr_mod , only: rest_file_strm ! namelist input use docn_shr_mod , only: sst_constant_value ! namelist input + use docn_shr_mod , only: RSO_relax_tau ! namelist input for relaxed slab ocean (RSO) + use docn_shr_mod , only: RSO_fixed_MLD ! namelist input for relaxed slab ocean (RSO) use docn_shr_mod , only: nullstr #ifdef HAVE_MOAB @@ -70,6 +72,8 @@ module docn_comp_mod integer(IN) :: kt,ks,ku,kv,kdhdx,kdhdy,kq,kswp ! field indices integer(IN) :: kswnet,klwup,klwdn,ksen,klat,kmelth,ksnow,krofi integer(IN) :: kh,kqbot + integer(IN) :: k10uu ! index for u10 + integer(IN) :: kRSO_bckgrd_sst ! index for background SST (relaxed slab ocean) integer(IN) :: index_lat, index_lon integer(IN) :: kmask, kfrac ! frac and mask field indices of docn domain integer(IN) :: ksomask ! So_omask field index @@ -93,7 +97,7 @@ module docn_comp_mod character(12) , parameter :: avofld(1:ktrans) = & (/ "So_t ","So_u ","So_v ","So_dhdx ",& "So_dhdy ","So_s ","strm_h ","strm_qbot "/) - character(len=*), parameter :: flds_strm = 'strm_h:strm_qbot' + character(len=*),parameter :: flds_strm = 'strm_h:strm_qbot:So_t' !-------------------------------------------------------------------------- !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -281,7 +285,7 @@ subroutine docn_comp_init(Eclock, x2o, o2x, & kdhdx = mct_aVect_indexRA(o2x,'So_dhdx') kdhdy = mct_aVect_indexRA(o2x,'So_dhdy') kswp = mct_aVect_indexRA(o2x,'So_fswpen', perrwith='quiet') - kq = mct_aVect_indexRA(o2x,'Fioo_q') + kq = mct_aVect_indexRA(o2x,'Fioo_q') ! ocn freezing melting potential call mct_aVect_init(x2o, rList=seq_flds_x2o_fields, lsize=lsize) call mct_aVect_zero(x2o) @@ -294,12 +298,14 @@ subroutine docn_comp_init(Eclock, x2o, o2x, & klwdn = mct_aVect_indexRA(x2o,'Faxa_lwdn') ksnow = mct_aVect_indexRA(x2o,'Faxa_snow') kmelth = mct_aVect_indexRA(x2o,'Fioi_melth') + k10uu = mct_aVect_indexRA(x2o,'So_duu10n') call mct_aVect_init(avstrm, rList=flds_strm, lsize=lsize) call mct_aVect_zero(avstrm) kh = mct_aVect_indexRA(avstrm,'strm_h') kqbot = mct_aVect_indexRA(avstrm,'strm_qbot') + kRSO_bckgrd_sst = mct_aVect_indexRA(avstrm,'So_t') allocate(somtp(lsize)) allocate(tfreeze(lsize)) @@ -472,14 +478,17 @@ subroutine docn_comp_init(Eclock, x2o, o2x, & call shr_mpi_bcast(exists,mpicom,'exists') call shr_mpi_bcast(exists1,mpicom,'exists1') - if (trim(datamode) == 'SOM' .or. trim(datamode) == 'SOM_AQUAP') then - if (exists1) then - if (my_task == master_task) write(logunit,F00) ' reading ',trim(rest_file) - call shr_pcdf_readwrite('read',SDOCN%pio_subsystem, SDOCN%io_type, & - trim(rest_file), mpicom, gsmap=gsmap, rf1=somtp, rf1n='somtp', io_format=SDOCN%io_format) - else - if (my_task == master_task) write(logunit,F00) ' file not found, skipping ',trim(rest_file) - endif + if ( trim(datamode) == 'SOM' & ! Traditional slab ocean + .or. trim(datamode) == 'RSO' & ! Relaxed slab ocean + .or. trim(datamode) == 'SOM_AQUAP' & ! Aquaplanet slab ocean + ) then + if (exists1) then + if (my_task == master_task) write(logunit,F00) ' reading ',trim(rest_file) + call shr_pcdf_readwrite('read',SDOCN%pio_subsystem, SDOCN%io_type, & + trim(rest_file), mpicom, gsmap=gsmap, rf1=somtp, rf1n='somtp', io_format=SDOCN%io_format) + else + if (my_task == master_task) write(logunit,F00) ' file not found, skipping ',trim(rest_file) + endif endif if (exists) then @@ -562,7 +571,18 @@ subroutine docn_comp_run(EClock, x2o, o2x, & integer(IN) :: idt ! integer timestep real(R8) :: dt ! timestep integer(IN) :: nu ! unit number - real(R8) :: hn ! h field + real(R8) :: hn ! h field - mixed layer depth (MLD) + ! relaxed slab ocean mode variables + real(R8) :: RSO_bckgrd_sst ! background SST + real(R8) :: RSO_X_cool ! logistics function weight + real(R8) :: u10 ! 10 m wind + ! relaxed slab ocean fixed parameters + integer, parameter :: RSO_slab_option = 0 ! Option for setting RSO_X_cool + real(R8), parameter :: RSO_R_cool = 11.75_r8/86400._r8 ! base cooling rate [K/s] + real(R8), parameter :: RSO_Tdeep = 271.0_r8 ! deep water temperature [K] + real(R8), parameter :: RSO_dT_o = 27.0_r8 ! scaling temperature gradient + real(R8), parameter :: RSO_h_o = 30.0_r8 ! scaling mixed layer depth + character(len=18) :: date_str character(len=CL) :: local_case_name real(R8), parameter :: & @@ -768,6 +788,65 @@ subroutine docn_comp_run(EClock, x2o, o2x, & enddo endif ! firstcall + ! Relaxed Slab Ocean based on Zarzycki(2016) + ! Zarzycki, C. M., 2016: Tropical Cyclone Intensity Errors Associated with Lack of Two-Way Ocean Coupling in High-Resolution Global Simulations. J. Climate, 29, 8589–8610. + ! https://journals.ametsoc.org/view/journals/clim/29/23/jcli-d-16-0273.1.xml + case('RSO') + lsize = mct_avect_lsize(o2x) + do n = 1,SDOCN%nstreams + call shr_dmodel_translateAV(SDOCN%avs(n),avstrm,avifld,avofld,rearr) + enddo + if (firstcall) then + do n = 1,lsize + if (.not. read_restart) then + somtp(n) = o2x%rAttr(kt,n) + TkFrz + endif + o2x%rAttr(kt,n) = somtp(n) + o2x%rAttr(kq,n) = 0.0_R8 + enddo + else ! firstcall + tfreeze = shr_frz_freezetemp(o2x%rAttr(ks,:)) + TkFrz + do n = 1,lsize + if (imask(n) /= 0) then + !******************************************************************* + if (RSO_fixed_MLD>=0) then + hn = RSO_fixed_MLD + else + hn = avstrm%rAttr(kh,n) + endif + ! Get "background" temperature for relaxation + RSO_bckgrd_sst = avstrm%rAttr(kRSO_bckgrd_sst,n) + TkFrz + u10 = SQRT(x2o%rAttr(k10uu,n)) + !******************************************************************* + ! Calculate scaling function - see Eq 3 in Zarzycki (2016) + if (RSO_slab_option==0) RSO_X_cool = 1._r8/(1._r8+EXP(-0.5_r8*(u10-30._r8)) ) ! SLAB1 + if (RSO_slab_option==1) RSO_X_cool =(1._r8/(1._r8+EXP(-0.2_r8*(u10-30._r8)) ))*(u10*2.4_r8/80._r8) ! SLAB2 + if (RSO_slab_option==2) RSO_X_cool = 0.0_r8 ! THERMO + !******************************************************************* + ! compute new ocean surface temperature + o2x%rAttr(kt,n) = somtp(n) & + ! Thermodynamic terms + +( x2o%rAttr(kswnet,n) & ! shortwave net + +x2o%rAttr(klwup ,n) & ! longwave up + +x2o%rAttr(klwdn ,n) & ! longwave down + +x2o%rAttr(ksen ,n) & ! sfc sensible heat flux + +x2o%rAttr(klat ,n) & ! sfc latent heat flux + -x2o%rAttr(ksnow ,n)*latice & ! latent heat from snow + -x2o%rAttr(krofi ,n)*latice & ! latent heat from runoff + ) * dt/(cpsw*rhosw*hn) & + - RSO_X_cool*RSO_R_cool*((somtp(n)-RSO_Tdeep)/RSO_dT_o)*(RSO_h_o/hn)*dt & ! Turb mixing + + (1_r8/RSO_relax_tau)*(RSO_bckgrd_sst - somtp(n))*dt ! Newtonian Relaxation + !******************************************************************* + ! Ignore ice formed or melt potential + o2x%rAttr(kq,n) = 0.0 + ! Cap SSTs to freezing + o2x%rAttr(kt,n) = max( TkFrzSw, o2x%rAttr(kt,n) ) + ! Save temperature to send back to coupler + somtp(n) = o2x%rAttr(kt,n) + endif ! imask /= 0 + enddo ! lsize + endif ! firstcall + case('SOM_AQUAP') lsize = mct_avect_lsize(o2x) do n = 1,SDOCN%nstreams @@ -912,7 +991,10 @@ subroutine docn_comp_run(EClock, x2o, o2x, & close(nu) call shr_file_freeUnit(nu) endif - if (trim(datamode) == 'SOM' .or. trim(datamode) == 'SOM_AQUAP') then + if ( trim(datamode) == 'SOM' & ! Traditional slab ocean + .or. trim(datamode) == 'RSO' & ! Relaxed slab ocean + .or. trim(datamode) == 'SOM_AQUAP' & ! Aquaplanet slab ocean + ) then if (my_task == master_task) write(logunit,F04) ' writing ',trim(rest_file),target_ymd,target_tod call shr_pcdf_readwrite('write', SDOCN%pio_subsystem, SDOCN%io_type,& trim(rest_file), mpicom, gsmap, clobber=.true., rf1=somtp,rf1n='somtp') diff --git a/components/data_comps/docn/src/docn_shr_mod.F90 b/components/data_comps/docn/src/docn_shr_mod.F90 index cf59dbc7f62..fef714c83b4 100644 --- a/components/data_comps/docn/src/docn_shr_mod.F90 +++ b/components/data_comps/docn/src/docn_shr_mod.F90 @@ -35,6 +35,8 @@ module docn_shr_mod character(CL) , public :: datamode ! mode integer(IN) , public :: aquap_option real(R8) , public :: sst_constant_value + real(R8) , public :: RSO_relax_tau ! relaxed slab ocean relaxation timescale [sec] + real(R8) , public :: RSO_fixed_MLD ! relaxed slab ocean globally fixed mixed layer depth (MLD) character(len=*), public, parameter :: nullstr = 'undefined' !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CONTAINS @@ -77,7 +79,8 @@ subroutine docn_shr_read_namelists(mpicom, my_task, master_task, & !----- define namelist ----- namelist / docn_nml / & - decomp, restfilm, restfils, force_prognostic_true, sst_constant_value + decomp, restfilm, restfils, force_prognostic_true, sst_constant_value, & + RSO_fixed_MLD, RSO_relax_tau !---------------------------------------------------------------------------- ! Determine input filenamname @@ -110,12 +113,16 @@ subroutine docn_shr_read_namelists(mpicom, my_task, master_task, & write(logunit,F00)' restfils = ',trim(restfils) write(logunit,F0L)' force_prognostic_true = ',force_prognostic_true write(logunit,*) ' sst_constant_value = ',sst_constant_value + write(logunit,*) ' RSO_fixed_MLD = ',RSO_fixed_MLD + write(logunit,*) ' RSO_relax_tau = ',RSO_relax_tau endif call shr_mpi_bcast(decomp ,mpicom,'decomp') call shr_mpi_bcast(restfilm,mpicom,'restfilm') call shr_mpi_bcast(restfils,mpicom,'restfils') call shr_mpi_bcast(force_prognostic_true,mpicom,'force_prognostic_true') call shr_mpi_bcast(sst_constant_value ,mpicom,'sst_constant_value') + call shr_mpi_bcast(RSO_fixed_MLD ,mpicom,'RSO_fixed_MLD') + call shr_mpi_bcast(RSO_relax_tau ,mpicom,'RSO_relax_tau') rest_file = trim(restfilm) rest_file_strm = trim(restfils) @@ -152,8 +159,9 @@ subroutine docn_shr_read_namelists(mpicom, my_task, master_task, & trim(datamode) == 'SST_AQUAP_CONSTANT' .or. & trim(datamode) == 'COPYALL' .or. & trim(datamode) == 'IAF' .or. & - trim(datamode) == 'SOM' .or. & - trim(datamode) == 'SOM_AQUAP') then + trim(datamode) == 'SOM' .or. & ! Traditional slab ocean + trim(datamode) == 'RSO' .or. & ! Relaxed slab ocean + trim(datamode) == 'SOM_AQUAP') then ! Aquaplanet slab ocean if (my_task == master_task) then write(logunit,F00) ' docn datamode = ',trim(datamode) end if @@ -181,9 +189,9 @@ subroutine docn_shr_read_namelists(mpicom, my_task, master_task, & ocn_prognostic = .true. ocnrof_prognostic = .true. endif - if (trim(datamode) == 'SOM' .or. trim(datamode) == 'SOM_AQUAP') then - ocn_prognostic = .true. - endif + if (trim(datamode) == 'SOM') ocn_prognostic = .true. ! Traditional slab ocean + if (trim(datamode) == 'RSO') ocn_prognostic = .true. ! Relaxed slab ocean + if (trim(datamode) == 'SOM_AQUAP') ocn_prognostic = .true. ! Aquaplanet slab ocean end subroutine docn_shr_read_namelists diff --git a/components/data_comps/docs/index.md b/components/data_comps/docs/index.md new file mode 100644 index 00000000000..adef0842673 --- /dev/null +++ b/components/data_comps/docs/index.md @@ -0,0 +1,13 @@ +# Data Models + +The E3SM data models are key components to support many different scenarios: + +- AMIP style experiments with observed sea surface temperatures +- component model spin-up, such as land forced by atmospheric reanalysis +- idealized experiments, such as aquaplanet or radiative-convective equilibrium + +More details can be found for each component below. + +- [Data Atmosphere](user-guide/data-atmos.md) +- [Data Land](user-guide/data-land.md) +- [Data Ocean](user-guide/data-ocean.md) diff --git a/components/data_comps/docs/user-guide/data-atmos.md b/components/data_comps/docs/user-guide/data-atmos.md new file mode 100644 index 00000000000..d74710598ee --- /dev/null +++ b/components/data_comps/docs/user-guide/data-atmos.md @@ -0,0 +1,4 @@ +# The E3SM Data Atmosphere Model + +!!!WARNING + This page is still under construction diff --git a/components/data_comps/docs/user-guide/data-land.md b/components/data_comps/docs/user-guide/data-land.md new file mode 100644 index 00000000000..3b07f59793e --- /dev/null +++ b/components/data_comps/docs/user-guide/data-land.md @@ -0,0 +1,4 @@ +# The E3SM Data Land Model + +!!!WARNING + This page is still under construction diff --git a/components/data_comps/docs/user-guide/data-ocean.md b/components/data_comps/docs/user-guide/data-ocean.md new file mode 100644 index 00000000000..de232d665e8 --- /dev/null +++ b/components/data_comps/docs/user-guide/data-ocean.md @@ -0,0 +1,123 @@ +# The E3SM Data Ocean Model + + + + +The E3SM data ocean has several different modes to support various realistic and idealized experiments. Sea surface temperatures (SST) can be either prescribed or prognostic. Prescribed SSTs are specified either through a data stream or analytically. Prognostic modes allow the SST field to evolve and respond to atmospheric conditions. The guides below provide more details on how to use these capabilities. + +- Prescribed + - [SST from Observations](#sst-from-observations) + - [Idealized SST](#idealized-sst) +- Prognostic + - [Traditional Slab Ocean Model](#traditional-slab-ocean-model) (SOM) + - [Relaxed Slab Ocean](#relaxed-slab-ocean) (RSO) + +## SST from Observations + +Using SST data derived from observations is the most common use of the data ocean model, often for AMIP style experiments to reproduce historical periods. + +Example compsets that use this capability are `F2010` and `F20TR`. These compsets use the `_DOCN%DOM_` compset modifier, which sets the `DOCN_MODE` variable in `env_run.xml` to "prescribed". + +Several additional XML variables need to be set in order to use this capability, which are set to defaults for common configurations, such as `F2010` at `ne30pg2` atmospheric resolution. + +```text +SSTICE_DATA_FILENAME Prescribed SST and ice coverage data file name +SSTICE_GRID_FILENAME Grid file in "domain" format corresponding to SSTICE_DATA_FILENAME +SSTICE_YEAR_ALIGN The model year that corresponds to SSTICE_YEAR_START on the data file +SSTICE_YEAR_START The first year of data to use from SSTICE_DATA_FILENAME +SSTICE_YEAR_END The last year of data to use from SSTICE_DATA_FILENAME +``` + +Most users will not need to edit these values from their defaults, but many scenarios require non-standard SST data, such as tropical cyclone hindcasts where the daily evolution of high-resolution SST data may be desireable. + +## Idealized SST + +The two main uses of idealized SST modes are aquaplanet (AQP) and radiative-convective equilibrium (RCE). The latter is just a special case of an aquaplanet where the SST is [usually] a constant value everywhere, traditionally used in conjunction with special modifications to homogenize radiation and disable rotation. There are several analytically specified SST patterns established by model intercomparison projects such as the Aqua-Planet Experiment (APE)[@blackburn_APE_context_2013] and RCEMIP[@wing_rcemip1_2018,@wing_rcemip2_2024]. + +### Idealized SST compsets + +The following list shows the currently defined E3SM compsets that utilize idealized SST. + +```text +FAQP +FAQP-MMF1 +FAQP-MMF2 +F-SCREAM-LR-AQP1 +F-SCREAM-HR-AQP1 +FRCE +FRCE-MMF1 +FRCE-MMF2 +FRCE-MW_295dT1p25 +FRCE-MW_300dT0p625 +FRCE-MW_300dT1p25 +FRCE-MW_300dT2p5 +FRCE-MW_305dT1p25 +FRCE-MW-MMF1_295dT1p25 +FRCE-MW-MMF1_300dT0p625 +FRCE-MW-MMF1_300dT1p25 +FRCE-MW-MMF1_300dT2p5 +FRCE-MW-MMF1_305dT1p25 +``` + +These all use "analytic" SST patterns that are specified via the `docn_comp_run()` subroutine in `components/data_comps/docn/src/docn_comp_mod.F90`. The `AQP` compsets currently only use the basic aquaplanet pattern that is symmetric about the equator. Other APE patterns introduce different meridional gradients and/or asymmetries. The various analytic SST patterns can be selected by changing the data ocean specifier: `_DOCN%AQP1_`. + +The first 10 analytic aquaplanet SST patterns correspond to the aqua-planet experiment (APE) protocol as follows + +```text +AQP1 = control symmetric SST pattern +AQP2 = Flat +AQP3 = Qobs = average of AQP1 and AQP2 +AQP4 = Peaked +AQP5 = Control+5N +AQP6 = 1KEQ - small warm pool +AQP7 = 3KEQ - small warm pool +AQP8 = 3KW1 - large warm pool +AQP9 = Control+10N +AQP10 = Control+15N +``` + +!!!NOTE + When using aquaplanet mode the orbital parameters will take on the idealized values shown below such that there are no seasonal variations, but there is still a diurnal cycle. + ```text + orb_eccen = 0 + orb_obliq = 0 + orb_mvelp = 0 + orb_mode = "fixed_parameters" + ``` + +The basic RCE compsets use the `_DOCN%AQPCONST_` modifier to produce a globally constant SST value, which is set by the `DOCN_AQPCONST_VALUE` variable in `env_run.xml`. The "FRCE-MW" compsets were designed for RCEMIP-II to produce a "mock walker-cell" configuration, in which sinusoidal SST variations are applied to promote a coherent large-scale circulation. + +### SST Data File + +In addition to the analytic SST modes the user can also specify an idealized aquaplanet SST pattern via the `_DOCN%AQPFILE_` option. The `aquapfile` namelist variable is used to specify the SST pattern in this mode. Note that this option has not been used or tested recently, so the user may experience difficulty trying to use this feature. + +## Traditional Slab Ocean Model + +A slab ocean model (SOM) allows responsive SSTs to address the "infinite heat source" problem associated with prescribed SSTs, but is much cheaper than running with a full ocean model. The traditional SOM appraoch requires special inputs, such as a specified mixed layer depth pattern that can vary in time and a prescribed heat flux to account for the missing effects of ocean dynamics often referred to as "Q-flux". The Q-flux data is often estimated from a fully coupled simulation with active ocean and sea-ice so that the SOM simulation will resemble the full model. + +Currently, we do not have Q-flux data to drive the SOM in E3SM. An alternative appraoch is to use a "relaxed" slab ocean (RSO) in which a specified relaxation time scale is used to bring the SST field back to a target SST field. The RSO mode is much simpler to use, but carries caveats that the user should be aware of before using. See [Data Ocean - Relaxed Slab Ocean](#relaxed-slab-ocean) for more information. + +## Relaxed Slab Ocean + +The relaxed slab ocean (RSO) is similar in many ways to the [traditional slab ocean model](#traditional-slab-ocean-model), but uses a specified relaxation time scale to avoid the need for specified "Q-flux" data to represent the effects of ocean transport. The RSO implementation in E3SM was inspired by Zarzycki (2016)[@Zarzycki_TC-ocn-cpl_2016]. + +A key consideration for the user is whether they need to use a realistic distribution of mixed layer depths (MLD), or whether their use case can benefit from the simplicity of a globally uniform MLD. + +The RSO mode has the following namelist variables to influence the ocean behavior: + +```text +RSO_relax_tau SST relaxation timescale +RSO_fixed_MLD globally uniform MLD value (use -1 for realistic MLD) +``` + +Other RSO parameter values are hardcoded in `components/data_comps/docn/src/docn_comp_mod.F90`. + +```text +RSO_slab_option = 0 ! Option for setting RSO_X_cool +RSO_R_cool = 11.75/86400 ! base cooling rate [K/s] +RSO_Tdeep = 271.00 ! deep water temperature [K] +RSO_dT_o = 27.0 ! scaling temperature gradient +RSO_h_o = 30.0 ! scaling mixed layer depth +``` + +The RSO mode uses the `SSTICE_DATA_FILENAME` in `env_run.xml` for its data stream. For a globally uniform MLD this file only need to contain a `SST_cpl` variable for the SST that will act as the target SST value for relaxation. If a realistic MLD pattern is desired then the `hblt` variable must also be present. This data can be derived a number of ways, but we currently do not have a dedicated tool or workflow. diff --git a/components/data_comps/mkdocs.yml b/components/data_comps/mkdocs.yml new file mode 100644 index 00000000000..89f5e6dca7b --- /dev/null +++ b/components/data_comps/mkdocs.yml @@ -0,0 +1,7 @@ +site_name: Data-Models + +nav: + - Introduction: 'index.md' + - Atmosphere: user-guide/data-atmos.md + - Land: user-guide/data-land.md + - Ocean: user-guide/data-ocean.md diff --git a/components/eam/bld/build-namelist b/components/eam/bld/build-namelist index 8dc532b6a3d..ee450573c36 100755 --- a/components/eam/bld/build-namelist +++ b/components/eam/bld/build-namelist @@ -830,6 +830,10 @@ add_default($nl,'use_hetfrz_classnuc'); add_default($nl,'hist_hetfrz_classnuc'); add_default($nl,'gw_convect_hcf') if (get_default_value('gw_convect_hcf')); add_default($nl,'hdepth_scaling_factor') if (get_default_value('hdepth_scaling_factor')); +add_default($nl,'gw_convect_hdepth_min') if (get_default_value('gw_convect_hdepth_min')); +add_default($nl,'gw_convect_storm_speed_min') if (get_default_value('gw_convect_storm_speed_min')); +add_default($nl,'gw_convect_plev_src_wind') if (get_default_value('gw_convect_plev_src_wind')); +add_default($nl,'use_gw_convect_old', 'val'=>'.true.'); add_default($nl,'linoz_psc_T'); if ($cfg->get('microphys') =~ /^mg2/) { add_default($nl,'micro_mg_dcs_tdep'); @@ -4089,13 +4093,24 @@ if ($waccm_phys or $cfg->get('nlev') >= 60) { add_default($nl, 'use_gw_oro' , 'val'=>'.true.'); add_default($nl, 'use_gw_front' , 'val'=>'.true.'); add_default($nl, 'use_gw_convect', 'val'=>'.true.'); + add_default($nl, 'use_od_ls', 'val'=>'.false.'); + add_default($nl, 'use_od_bl', 'val'=>'.false.'); + add_default($nl, 'use_od_ss', 'val'=>'.false.'); + add_default($nl, 'use_od_fd', 'val'=>'.false.'); } else { add_default($nl, 'use_gw_oro' , 'val'=>'.true.'); add_default($nl, 'use_gw_front' , 'val'=>'.false.'); add_default($nl, 'use_gw_convect', 'val'=>'.false.'); + add_default($nl, 'use_od_ls', 'val'=>'.false.'); + add_default($nl, 'use_od_bl', 'val'=>'.false.'); + add_default($nl, 'use_od_ss', 'val'=>'.false.'); + add_default($nl, 'use_od_fd', 'val'=>'.false.'); } add_default($nl, 'pgwv', 'val'=>'32'); add_default($nl, 'gw_dc','val'=>'2.5D0'); +add_default($nl, 'od_ls_ncleff' ,'val'=>'3.D0'); +add_default($nl, 'od_bl_ncd' ,'val'=>'3.D0'); +add_default($nl, 'od_ss_sncleff','val'=>'1.D0'); if ($nl->get_value('use_gw_oro') =~ /$TRUE/io) { add_default($nl, 'effgw_oro'); diff --git a/components/eam/bld/namelist_files/namelist_defaults_eam.xml b/components/eam/bld/namelist_files/namelist_defaults_eam.xml index 44e4e7e65ae..7c1a7cd3fd5 100755 --- a/components/eam/bld/namelist_files/namelist_defaults_eam.xml +++ b/components/eam/bld/namelist_files/namelist_defaults_eam.xml @@ -127,7 +127,6 @@ atm/cam/topo/USGS-gtopo30_64x128_c050520.nc - atm/cam/topo/USGS-gtopo30_ne4np4_16x.c20160612.nc atm/cam/topo/USGS-gtopo30_ne4np4pg2_16x_converted.c20200527.nc atm/cam/topo/USGS-gtopo30_ne11np4_16xconsistentSGH.c20160612.nc @@ -827,7 +826,7 @@ 0.5D0 7.0D0 0.001D0 - 0.04D0 + 0.08D0 2.65D0 0.02D0 0.1D0 @@ -1890,8 +1889,14 @@ with se_tstep, dt_remap_factor, dt_tracer_factor set to -1 10.0 0.50 1.0 +2.5 +10.0 +70000 0.375 .true. + 3.D0 + 3.D0 + 1.D0 2.5D0 268.15D0 13.8D0 diff --git a/components/eam/bld/namelist_files/namelist_definition.xml b/components/eam/bld/namelist_files/namelist_definition.xml index 8228c7d8d2e..848e43425b1 100644 --- a/components/eam/bld/namelist_files/namelist_definition.xml +++ b/components/eam/bld/namelist_files/namelist_definition.xml @@ -1078,6 +1078,48 @@ Whether or not to enable GWD brute-force energy fix. Default: set by build-namelist. + +Whether or not to enable nonlinear orographic gravity wave drag (oGWD). +Default: set by build-namelist. + + + +Whether or not to enable flow-blocking drag (FBD). +Default: set by build-namelist. + + + +Whether or not to enable small-scale orographic GWD drag (sGWD). +Default: set by build-namelist. + + + +Whether or not to enable turbulent orographic form drag (TOFD). +Default: set by build-namelist. + + + +Tuning parameter of orographic GWD (oGWD). See use_od_ls. +Default: set by build-namelist. + + + +Tuning parameter of flow-blocking drag (FBD). See use_od_bl. +Default: set by build-namelist. + + + +Tuning parameter of small-scale GWD (sGWD). See use_od_ss. +Default: set by build-namelist. + + Gravity wave spectrum dimension (wave numbers are from -pgwv to pgwv). @@ -1100,16 +1142,40 @@ Default: set by build-namelist. -Heating rate conversion factor associated with convective gravity waves +Heating rate conversion factor associated with convective GWD [unitless]. +Also often interpretted as "convective fraction" [%]. Default: 20.0 -Scaling factor for the heating depth +Scaling factor for the heating depth [unitless] Default: 1.0 + +minimum hdepth for for convective GWD spectrum lookup table [km] +Default: 2.5 km + + + +minimum convective storm speed in m/s for convective GWD +Default: 10.0 m/s + + + +Reference pressure value used for convective storm speed in m/s for convective GWD +Default: 70000 Pa + + + +switch to revert to old calculation of Beres scheme for heating depth and max + + Efficiency associated with convective gravity waves from the Beres @@ -6743,6 +6809,36 @@ example, to apply hyperviscosity to moisture, the first tracer, set the value to Default: (set by dycore) + +Number of element haloes to include in trajectory search. -1 triggers an +automatic calculation of max(1, dt_tracer_factor/3). This is based on the +advective CFL condition that governs the dynamics time step. +Default: -1 (set by dycore) + + + +Number of substeps to take in computing semi-Lagrangian transport +trajectories. 0 triggers the original algorithm; 1 or larger triggers the +enhanced-trajectory algorithm. +Default: 0 (set by dycore) + + + +Number of velocity snapshots to store for use when computing the enhanced +trajectories. -1 triggers an automatic calculation. 0, 1, 2 all become 2, the +minimum. +Default: -1 (set by dycore) + + + +Optional diagnostic output from transport module. +Default: 0 (set by dycore) + + FAQP-MMF2 2000_EAM%AQP-MMF2_SLND_SICE_DOCN%AQP1_SROF_SGLC_SWAV + + + + + + + F2010-RSO + 2010_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%PRES_DOCN%RSO_MOSART_SGLC_SWAV + diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/orodrag_ne30pg2/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/orodrag_ne30pg2/user_nl_eam new file mode 100644 index 00000000000..8ab37d27978 --- /dev/null +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/orodrag_ne30pg2/user_nl_eam @@ -0,0 +1,8 @@ +use_gw_oro=.false. +use_od_ls=.true. +use_od_bl=.true. +use_od_ss=.true. +use_od_fd=.true. + + +bnd_topo='$DIN_LOC_ROOT/atm/cam/topo/USGS-gtopo30_ne30np4pg2_x6t-SGH_forOroDrag.c20241001.nc' diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/orodrag_ne4pg2/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/orodrag_ne4pg2/user_nl_eam new file mode 100644 index 00000000000..f32cc8a6f93 --- /dev/null +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/orodrag_ne4pg2/user_nl_eam @@ -0,0 +1,7 @@ +use_gw_oro=.false. +use_od_ls=.true. +use_od_bl=.true. +use_od_ss=.true. +use_od_fd=.true. + +bnd_topo='$DIN_LOC_ROOT/atm/cam/topo/USGS-gtopo30_ne4np4pg2_16x_converted_forOroDrag.c20241019.nc' diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_sl_nsubstep2/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_sl_nsubstep2/user_nl_eam new file mode 100644 index 00000000000..7cfe7f1cb6e --- /dev/null +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_sl_nsubstep2/user_nl_eam @@ -0,0 +1,3 @@ +semi_lagrange_diagnostics = 1 +semi_lagrange_trajectory_nsubstep = 2 +semi_lagrange_trajectory_nvelocity = 3 diff --git a/components/eam/docs/figures/orodrag.png b/components/eam/docs/figures/orodrag.png new file mode 100644 index 00000000000..de6e5cd8407 Binary files /dev/null and b/components/eam/docs/figures/orodrag.png differ diff --git a/components/eam/docs/tech-guide/index.md b/components/eam/docs/tech-guide/index.md index f93b7715570..10e66936799 100644 --- a/components/eam/docs/tech-guide/index.md +++ b/components/eam/docs/tech-guide/index.md @@ -16,6 +16,8 @@ This Technical Guide describes the physics of version 3 of the E3SM Atmospheric - [RRTMG](rrtmg.md): Parameterization of radiation. +- [ORODRAG](orodrag.md): Parameterization of orographic drag + - [MAM](mam.md): Primary parameterization schemes used to represent aerosols. - [VBS](vbs.md): Parameterization of secondary organic aerosols. diff --git a/components/eam/docs/tech-guide/orodrag.md b/components/eam/docs/tech-guide/orodrag.md new file mode 100644 index 00000000000..a0be96e4f75 --- /dev/null +++ b/components/eam/docs/tech-guide/orodrag.md @@ -0,0 +1,35 @@ +# Orographic drag scheme + +## Overview + +The orographic drag schemes includes two main options: the default Gravity Wave Drag scheme of McFarlane (1987)[@mcfarlane_the_1987] and a new suite of orographic drag parameterization schemes. The new suite includes 4 components all combined in one module (i.e. subroutine gwdo2d). The schemes include orographic gravity wave drag (oGWD, Xie et al.,2020)[@xie_an_2020], flow-blocking drag (FBD, Xie et al.,2020)[@xie_an_2020], small-scale GWD (sGWD, Tsiringakis et al.,2017)[@tsiringakis_small_2020], and turbulent-scale orographic form drag (TOFD, Beljaars et al., 2004)[@beljaars_a_2020]. The oGWD and TOFD schemes are used to replace the default oGWD (McFarlane, 1987)[@mcfarlane_the_1987] and Turbulent Mountain Stress (TMS, documented on Richter et al., 2010)[@richter_the_2010] module in EAM, while the FBD and sGWD are added enhanced drag schemes. Each of the schemes are used to predict and add enhanced drags from surface to high level that would help decelerate the wind especially over the mountainous regions among the globe. The oGWD, FBD, and sGWD are implemented in gw_drag.F90, while TOFD is implemented in clubb_intr.F90 to join the vertical diffusion process. There are also new topographic parameters (orographic asymmetry [OA], orographic convexity [OC], effective orographic length [OL]) are input into the model for the new oGWD and FBD scheme implmented. The concept of each scheme is illustrate in the figure below. Currently, only the scheme of McFarlane (1987) is turned on as default in E3SMv3.0. + +![orodrag figure](../figures/orodrag.png) + +### Default oGWD scheme + +The current default oGWD scheme in E3SMv3.0 is from McFarlane (1987)[@mcfarlane_the_1987]. It is a linear orographic gravity wave drag scheme that parameterizes the subgrid process of vertical propagation of gravity wave originating from orographic source. The output from this scheme is a vertical profile of drag (or deceleration terms) when gravity wave breaks at higher levels and deposits momemtum flux to that level. This scheme is shown to improve the excessive westerly wind bias in the extratropics and the wind bias in the polar region. This scheme is turned on by default in E3SMv3.0. + +### Default TMS scheme + +The turbulent mountain stress (TMS) scheme is documented on Richter et al.,(2010)[@richter_the_2010]. It parameterizes the turbulent scale form drag (TOFD) through adding enhanced effective roughness length to the model. This method is justified by the observation that area-averaged wind sufficiently far above hilly terrain behave logarithmically. It leads to significant decrease of overspeed surface wind in the model. It is turned OFF in E3SMv3.0. + +### New oGWD scheme + +The Xie et al.(2020)[@xie_an_2020] oGWD scheme is a nonlinear orographic gravity wave drag scheme that parameterizes the subgrid oGWD (>5km). It includes both the linear oGWD like that of McFarlane (1987)[@mcfarlane_the_1987] and nonlinear oGWD like low-level wave breaking (LLWB). The LLWB downstream often results in downslope windstorm that can enhance the drag by several times that of linear oGWD. The incorporation of the nonlinear drag is by adding an "enhancement factor" that is a function of the higher moments of the orography, e.g. orographic asymmetry (OA), orographic convexity (OC), effective orographic length (OL). + +### FBD scheme + +The Xie et al.(2020)[@xie_an_2020] FBD scheme parameterizes the drag by flow blocked on the mountain flanks or flowing around the mountain under upstream stable conditions (>5km). It occurs when the mean flow does not have enough kinetic energy to traverse an obstacle and either stops upstream or diverges around the obstacle. This provides drag near the surface where the blocking occurs in addition to the oGWD. + +### TOFD scheme + +The TOFD scheme (Beljaars et al.,2004)[@beljaars_a_2020] parameterizes the drag generated by the shear stresses in the boundary layer when the flow encounters smaller obstacles (<5km). As airflow encounters an obstacle, it is disrupted, leading to the formation of eddies and vortices. These turbulent structures enhance mixing, allowing different layers of air to interact over a short distance. The intensity of the mixing is typically higher close to the obstacle, with rapid changes in velocity and direction of the airflow and can exhibit large control over the surface wind. It is an alternative scheme to the current TMS in E3SMv3. An important difference between the TOFD and TMS is that TOFD explicitly calculate the stress profile while TMS uses an enhanced effective roughtness length approach. + +### sGWD scheme + +The Tsiringakis et al.(2017)[@tsiringakis_small_2020] sGWD scheme parameterizes the small-scale orographic gravity wave drag (sGWD) within the relatively shallow stable boundary layer. Models applying the form drag such as TOFD schemes usually lack sufficient cyclonic filling and do not accurately represent the development of cyclones over land (Holstag 2006)[@holtslag_preface_2006]. This caused significant temperature bias that occurs due to runaway cooling at the surface. To bridge the gap of this missing drag in the boundary layer, studies have hypothesized that the missing drag can be generated by sGWD. sGWD occurs under the relative stable boundary layer with low winds where the turbulence is significantly reduced. + +## Namelist parameters + +[orodrag Namelist Parameters](../user-guide/namelist_parameters.md#orographic-drag-schemes) diff --git a/components/eam/docs/user-guide/namelist_parameters.md b/components/eam/docs/user-guide/namelist_parameters.md index b09e8c44ffd..8aa4b40299c 100644 --- a/components/eam/docs/user-guide/namelist_parameters.md +++ b/components/eam/docs/user-guide/namelist_parameters.md @@ -156,3 +156,20 @@ | Parameter | Description | Default value | | ------------------------- | ----------------------------------------------------------------- | ---------------------- | | `cosp_lite` | This namelist sets cosp_ncolumns=10 and cosp_nradsteps=3 (appropriate for COSP statistics derived from seasonal averages), and runs MISR, ISCCP, MODIS, and CALIPSO lidar simulators (cosp_lmisr_sim=.true.,cosp_lisccp_sim=.true., cosp_lmodis_sim=.true.,cosp_llidar_sim=.true.). | `false` | + +## Orographic drag schemes + +| Parameter | Description | Default value | +| ------------------------- | ----------------------------------------------------------------- | ---------------------- | +| `use_gw_oro` | This namelist controls the default linear orographic gravity wave drag (oGWD) for E3SM, if used, the default oGWD is turned on. | `true` | +| `do_tms` | This namelist controls the default Turbulent Mountain Stress (TMS) for E3SM, if used, the default TMS is turned on. | `false` | +| `effgw_oro` | Efficiency associated with orographic gravity waves. | `0.375` | +| `tms_orocnst` | Turbulent mountain stress parameter used when TMS calculation is turned on | `1.0` | +| `tms_z0fac` | Factor determining z_0 from orographic standard deviation [ no unit ] for TMS. | `0.75` | +| `use_od_ls` | This namelist controls the new nonlinear oGWD, if used, the nonlinear oGWD is turned on. use_od_ls should not be used at the same time with use_gw_oro. | `false` | +| `use_od_bl` | This namelist controls the Flow-blocking drag (FBD) scheme, if used, the FBD scheme is turned on. | `false` | +| `use_od_ss` | This namelist controls the small-scale GWD (sGWD) scheme, if used, the sGWD scheme is turned on. | `false` | +| `use_od_fd` | This namelist controls the Turbulent orographic form drag (TOFD) scheme, if used, the TOFD scheme is turned on. | `false` | +| `od_ls_ncleff` | Tuning parameter of nonlinear oGWD. Stands for effective resolution of the grid for oGWD. Scales the magnitude of nonlinear oGWD. | `3` | +| `od_bl_ncd` | Tuning parameter of FBD. Stands for bulk drag coefficient. Scales the magnitude of FBD. | `3` | +| `od_ss_sncleff` | Tuning parameter of sGWD. Stands for effective resolution of the grid for sGWD.Scales the magnitude of sGWD. | `1` | diff --git a/components/eam/mkdocs.yml b/components/eam/mkdocs.yml index e41cb614387..d0e23a6e7e8 100644 --- a/components/eam/mkdocs.yml +++ b/components/eam/mkdocs.yml @@ -16,6 +16,7 @@ nav: - tech-guide/clubb.md - tech-guide/zm.md - RRTMG: tech-guide/rrtmg.md + - tech-guide/orodrag.md - tech-guide/mam.md - tech-guide/vbs.md - tech-guide/dust.md diff --git a/components/eam/src/chemistry/mozart/rate_diags.F90 b/components/eam/src/chemistry/mozart/rate_diags.F90 index 457ede70307..4ad21392f54 100644 --- a/components/eam/src/chemistry/mozart/rate_diags.F90 +++ b/components/eam/src/chemistry/mozart/rate_diags.F90 @@ -126,7 +126,7 @@ subroutine rate_diags_calc( rxt_rates, vmr, m, ncol, lchnk, pver, pdeldry, mbar rxt_rates(:ncol,:,rxt_tag_map(i)) = rxt_rates(:ncol,:,rxt_tag_map(i)) * m(:,:) call outfld( rate_names(i), rxt_rates(:ncol,:,rxt_tag_map(i)), ncol, lchnk ) - if ( .not. history_UCIgaschmbudget_2D .and. .not. history_UCIgaschmbudget_2D_levels) return + if (history_UCIgaschmbudget_2D .or. history_UCIgaschmbudget_2D_levels) then if (rate_names(i) .eq. 'r_lch4') then !kg/m2/sec @@ -166,6 +166,8 @@ subroutine rate_diags_calc( rxt_rates, vmr, m, ncol, lchnk, pver, pdeldry, mbar endif endif + + endif enddo end subroutine rate_diags_calc diff --git a/components/eam/src/control/runtime_opts.F90 b/components/eam/src/control/runtime_opts.F90 index c52c02a5c23..1fcd84806b7 100644 --- a/components/eam/src/control/runtime_opts.F90 +++ b/components/eam/src/control/runtime_opts.F90 @@ -246,6 +246,7 @@ subroutine read_namelist(single_column_in, scmlon_in, scmlat_in, scm_multcols_in use uwshcu, only: uwshcu_readnl use pkg_cld_sediment, only: cld_sediment_readnl use gw_drag, only: gw_drag_readnl + use od_common, only: oro_drag_readnl use qbo, only: qbo_readnl use iondrag, only: iondrag_readnl use phys_debug_util, only: phys_debug_readnl @@ -516,6 +517,7 @@ subroutine read_namelist(single_column_in, scmlon_in, scmlat_in, scm_multcols_in call uwshcu_readnl(nlfilename) call cld_sediment_readnl(nlfilename) call gw_drag_readnl(nlfilename) + call oro_drag_readnl(nlfilename) call qbo_readnl(nlfilename) call iondrag_readnl(nlfilename) call phys_debug_readnl(nlfilename) diff --git a/components/eam/src/physics/cam/clubb_intr.F90 b/components/eam/src/physics/cam/clubb_intr.F90 index a93331fabdd..33815759e7c 100644 --- a/components/eam/src/physics/cam/clubb_intr.F90 +++ b/components/eam/src/physics/cam/clubb_intr.F90 @@ -20,7 +20,8 @@ module clubb_intr use shr_kind_mod, only: r8=>shr_kind_r8 use shr_log_mod , only: errMsg => shr_log_errMsg use ppgrid, only: pver, pverp - use phys_control, only: phys_getopts + use phys_control, only: phys_getopts, use_od_ss, use_od_fd + use od_common, only: od_ls_ncleff, od_bl_ncd, od_ss_sncleff use physconst, only: rair, cpair, gravit, latvap, latice, zvir, rh2o, karman, & tms_orocnst, tms_z0fac, pi use cam_logfile, only: iulog @@ -631,7 +632,8 @@ subroutine clubb_ini_cam(pbuf2d, dp1_in) use constituents, only: cnst_get_ind use phys_control, only: phys_getopts - use parameters_tunable, only: params_list + use parameters_tunable, only: params_list + use cam_abortutils, only: endrun #endif @@ -927,7 +929,25 @@ subroutine clubb_ini_cam(pbuf2d, dp1_in) call addfld ('VMAGDP', horiz_only, 'A', '-', 'ZM gustiness enhancement') call addfld ('VMAGCL', horiz_only, 'A', '-', 'CLUBB gustiness enhancement') call addfld ('TPERTBLT', horiz_only, 'A', 'K', 'perturbation temperature at PBL top') - + ! + if (use_od_fd) then + !added for turbulent orographic form drag (TOFD) output + call addfld ('DTAUX3_FD',(/'lev'/),'A','m/s2','U tendency - fd orographic drag') + call addfld ('DTAUY3_FD',(/'lev'/),'A','m/s2','V tendency - fd orographic drag') + call addfld ('DUSFC_FD',horiz_only,'A','N/m2','fd zonal oro surface stress') + call addfld ('DVSFC_FD',horiz_only,'A','N/m2','fd merio oro surface stress') + call add_default('DTAUX3_FD', 1, ' ') + call add_default('DTAUY3_FD', 1, ' ') + call add_default('DUSFC_FD', 1, ' ') + call add_default('DVSFC_FD', 1, ' ') + if (masterproc) then + write(iulog,*)'Using turbulent orographic form drag scheme (TOFD)' + end if + if (use_od_fd.and.do_tms) then + call endrun("clubb_intr: Both TMS and TOFD are turned on, please turn one off& + &by setting use_od_fd or do_tms as .false.") + end if + end if ! Initialize statistics, below are dummy variables dum1 = 300._r8 dum2 = 1200._r8 @@ -1155,7 +1175,9 @@ subroutine clubb_tend_cam( & use model_flags, only: ipdf_call_placement use advance_clubb_core_module, only: ipdf_post_advance_fields #endif - + use od_common, only: grid_size, oro_drag_interface + use hycoef, only: etamid + use physconst, only: rh2o,pi,rearth,r_universal implicit none ! --------------- ! @@ -1519,6 +1541,28 @@ subroutine clubb_tend_cam( & real(r8) :: sfc_v_diff_tau(pcols) ! Response to tau perturbation, m/s real(r8), parameter :: pert_tau = 0.1_r8 ! tau perturbation, Pa + !variables for turbulent orographic form drag (TOFD) interface + real(r8) :: dtaux3_fd(pcols,pver) + real(r8) :: dtauy3_fd(pcols,pver) + real(r8) :: dusfc_fd(pcols) + real(r8) :: dvsfc_fd(pcols) + logical :: gwd_ls,gwd_bl,gwd_ss,gwd_fd + real(r8) :: dummy_nm(pcols,pver) + real(r8) :: dummy_utgw(pcols,pver) + real(r8) :: dummy_vtgw(pcols,pver) + real(r8) :: dummy_ttgw(pcols,pver) + real(r8) :: dummx_ls(pcols,pver) + real(r8) :: dummx_bl(pcols,pver) + real(r8) :: dummx_ss(pcols,pver) + real(r8) :: dummy_ls(pcols,pver) + real(r8) :: dummy_bl(pcols,pver) + real(r8) :: dummy_ss(pcols,pver) + real(r8) :: dummx3_ls(pcols,pver) + real(r8) :: dummx3_bl(pcols,pver) + real(r8) :: dummx3_ss(pcols,pver) + real(r8) :: dummy3_ls(pcols,pver) + real(r8) :: dummy3_bl(pcols,pver) + real(r8) :: dummy3_ss(pcols,pver) real(r8) :: inv_exner_clubb_surf @@ -1947,6 +1991,35 @@ subroutine clubb_tend_cam( & call t_stopf('compute_tms') endif + if (use_od_fd) then + gwd_ls =.false. + gwd_bl =.false. + gwd_ss =.false. + gwd_fd =use_od_fd + dummy_nm =0.0_r8 + dummy_utgw=0.0_r8 + dummy_vtgw=0.0_r8 + dummy_ttgw=0.0_r8 + !sgh30 as the input for turbulent orographic form drag (TOFD) instead of sgh + call oro_drag_interface(state,cam_in,sgh30,pbuf,hdtime,dummy_nm,& + gwd_ls,gwd_bl,gwd_ss,gwd_fd,& + od_ls_ncleff,od_bl_ncd,od_ss_sncleff,& + dummy_utgw,dummy_vtgw,dummy_ttgw,& + dtaux3_ls=dummx3_ls,dtauy3_ls=dummy3_ls,& + dtaux3_bl=dummx3_bl,dtauy3_bl=dummy3_bl,& + dtaux3_ss=dummx3_ss,dtauy3_ss=dummy3_ss,& + dtaux3_fd=dtaux3_fd,dtauy3_fd=dtauy3_fd,& + dusfc_ls=dummx_ls,dvsfc_ls=dummy_ls,& + dusfc_bl=dummx_bl,dvsfc_bl=dummy_bl,& + dusfc_ss=dummx_ss,dvsfc_ss=dummy_ss,& + dusfc_fd=dusfc_fd,dvsfc_fd=dvsfc_fd) + + call outfld ('DTAUX3_FD', dtaux3_fd, pcols, lchnk) + call outfld ('DTAUY3_FD', dtauy3_fd, pcols, lchnk) + call outfld ('DUSFC_FD', dusfc_fd, pcols, lchnk) + call outfld ('DVSFC_FD', dvsfc_fd, pcols, lchnk) + endif + if (micro_do_icesupersat) then call physics_ptend_init(ptend_loc,state%psetcols, 'clubb_ice3', ls=.true., lu=.true., lv=.true., lq=lq) endif @@ -2067,7 +2140,14 @@ subroutine clubb_tend_cam( & dum_core_rknd = real((ksrftms(i)*state1%v(i,pver)), kind = core_rknd) vpwp_sfc = vpwp_sfc-(dum_core_rknd/rho_ds_zm(1)) endif - + ! ------------------------------------------------- ! + ! Apply TOFD + ! ------------------------------------------------- ! + ! tendency is flipped already + if (use_od_fd) then + um_forcing(2:pverp)=dtaux3_fd(i,pver:1:-1) + vm_forcing(2:pverp)=dtauy3_fd(i,pver:1:-1) + endif ! Need to flip arrays around for CLUBB core do k=1,pverp um_in(k) = real(um(i,pverp-k+1), kind = core_rknd) @@ -3091,7 +3171,7 @@ end subroutine clubb_tend_cam ! ! ! =============================================================================== ! - subroutine clubb_surface (state, cam_in, ustar, obklen) + subroutine clubb_surface (state, cam_in, pbuf, ustar, obklen) !------------------------------------------------------------------------------- ! Description: Provide the obukhov length and the surface friction velocity @@ -3108,10 +3188,12 @@ subroutine clubb_surface (state, cam_in, ustar, obklen) !------------------------------------------------------------------------------- use physics_types, only: physics_state - use physconst, only: zvir + use physconst, only: zvir,gravit use ppgrid, only: pver, pcols use constituents, only: cnst_get_ind use camsrfexch, only: cam_in_t + use hb_diff, only: pblintd_ri + use physics_buffer, only: pbuf_get_index, pbuf_get_field, physics_buffer_desc implicit none @@ -3119,8 +3201,9 @@ subroutine clubb_surface (state, cam_in, ustar, obklen) ! Input Auguments ! ! --------------- ! - type(physics_state), intent(inout) :: state ! Physics state variables - type(cam_in_t), intent(in) :: cam_in + type(physics_state), intent(inout) :: state ! Physics state variables + type(cam_in_t), intent(in) :: cam_in + type(physics_buffer_desc), pointer, intent(in) :: pbuf(:) ! ---------------- ! ! Output Auguments ! @@ -3136,16 +3219,23 @@ subroutine clubb_surface (state, cam_in, ustar, obklen) ! --------------- ! integer :: i ! indicees + integer :: k integer :: ncol ! # of atmospheric columns real(r8) :: th(pcols) ! surface potential temperature real(r8) :: thv(pcols) ! surface virtual potential temperature + real(r8) :: th_lv(pcols,pver) ! level potential temperature + real(r8) :: thv_lv(pcols,pver) ! level virtual potential temperature real(r8) :: kinheat ! kinematic surface heat flux real(r8) :: kinwat ! kinematic surface vapor flux real(r8) :: kbfs ! kinematic surface buoyancy flux + real(r8) :: kbfs_pcol(pcols) ! kinematic surface buoyancy flux stored for all pcols integer :: ixq,ixcldliq !PMA fix for thv real(r8) :: rrho ! Inverse air density + integer :: oro_drag_ribulk_idx ! pbuf index of bulk richardson number for oro drag + real(r8), pointer :: oro_drag_ribulk(:) ! pbuf pointer for bulk richardson number + #endif obklen(pcols) = 0.0_r8 @@ -3181,6 +3271,43 @@ subroutine clubb_surface (state, cam_in, ustar, obklen) kinheat, kinwat, kbfs, obklen(i) ) enddo + if (use_od_ss) then + !add calculation of bulk richardson number here + !compute the whole level th and thv for diagnose of bulk richardson number + thv_lv=0.0_r8 + th_lv =0.0_r8 + + !use the same virtual potential temperature formula as above (thv) except for all vertical levels + !used for bulk richardson number below in pblintd_ri + do i=1,ncol + do k=1,pver + th_lv(i,k) = state%t(i,k)*state%exner(i,k) + if (use_sgv) then + thv_lv(i,k) = th_lv(i,k)*(1.0_r8+zvir*state%q(i,k,ixq) & + - state%q(i,k,ixcldliq)) + else + thv_lv(i,k) = th_lv(i,k)*(1.0_r8+zvir*state%q(i,k,ixq)) + end if + enddo + enddo + + !recalculate the kbfs stored in kbfs_pcol for bulk richardson number in pblintd_ri + kbfs_pcol=0.0_r8 + do i=1,ncol + call calc_ustar( state%t(i,pver), state%pmid(i,pver), cam_in%wsx(i), cam_in%wsy(i), rrho, ustar(i) ) + call calc_obklen( th(i), thv(i), cam_in%cflx(i,1), cam_in%shf(i), rrho, ustar(i), & + kinheat, kinwat, kbfs, obklen(i) ) + kbfs_pcol(i)=kbfs + enddo + + oro_drag_ribulk_idx = pbuf_get_index('oro_drag_ribulk') + call pbuf_get_field(pbuf, oro_drag_ribulk_idx, oro_drag_ribulk) + + !calculate the bulk richardson number + call pblintd_ri(ncol, gravit, thv_lv, state%zm, state%u, state%v, & + ustar, obklen, kbfs_pcol, oro_drag_ribulk) + endif + return #endif diff --git a/components/eam/src/physics/cam/gw_convect.F90 b/components/eam/src/physics/cam/gw_convect.F90 index fc2fce99214..dff87eec89c 100644 --- a/components/eam/src/physics/cam/gw_convect.F90 +++ b/components/eam/src/physics/cam/gw_convect.F90 @@ -4,10 +4,10 @@ module gw_convect ! This module handles gravity waves from convection, and was extracted from ! gw_drag in May 2013. ! - -use gw_utils, only: r8 - -use gw_common, only: pver, pgwv +use cam_logfile, only: iulog +use spmd_utils, only: masterproc +use gw_utils, only: r8 +use gw_common, only: pver, pgwv implicit none private @@ -21,8 +21,8 @@ module gw_convect ! Dimension for mean wind in heating. integer :: maxuh -! Index for level at 700 mb. -integer :: k700 +! Index for level for storm/steering flow (usually 700 mb) +integer :: k_src_wind ! Table of source spectra. real(r8), allocatable :: mfcc(:,:,:) @@ -31,19 +31,21 @@ module gw_convect !========================================================================== -subroutine gw_convect_init(k700_in, mfcc_in, errstring) - ! Index at 700 mb. - integer, intent(in) :: k700_in - ! Source spectra to keep as table. - real(r8), intent(in) :: mfcc_in(:,:,:) - ! Report any errors from this routine. - character(len=*), intent(out) :: errstring - +subroutine gw_convect_init( plev_src_wind, mfcc_in, errstring) + use ref_pres, only: pref_edge + real(r8), intent(in) :: plev_src_wind ! reference pressure value [Pa] to set k_src_wind (previously hardcoded to 70000._r8) + real(r8), intent(in) :: mfcc_in(:,:,:) ! Source spectra to keep as table + character(len=*), intent(out) :: errstring ! Report any errors from this routine integer :: ierr + integer :: k errstring = "" - k700 = k700_in + do k = 0, pver + if ( pref_edge(k+1) < plev_src_wind ) k_src_wind = k+1 + end do + + if (masterproc) write (iulog,*) 'gw_convect: steering flow level = ',k_src_wind ! First dimension is maxh. maxh = size(mfcc_in,1) @@ -60,7 +62,9 @@ end subroutine gw_convect_init subroutine gw_beres_src(ncol, ngwv, lat, u, v, netdt, & zm, src_level, tend_level, tau, ubm, ubi, xv, yv, c, & - hdepth, maxq0, CF, hdepth_scaling_factor) + hdepth, maxq0_out, maxq0_conversion_factor, hdepth_scaling_factor, & + hdepth_min, storm_speed_min, & + use_gw_convect_old) !----------------------------------------------------------------------- ! Driver for multiple gravity wave drag parameterization. ! @@ -90,11 +94,20 @@ subroutine gw_beres_src(ncol, ngwv, lat, u, v, netdt, & real(r8), intent(in) :: zm(ncol,pver) ! Heating conversion factor - real(r8), intent(in) :: CF + real(r8), intent(in) :: maxq0_conversion_factor ! Scaling factor for the heating depth real(r8), intent(in) :: hdepth_scaling_factor + ! minimum hdepth for for spectrum lookup table + real(r8), intent(in) :: hdepth_min + + ! minimum convective storm speed + real(r8), intent(in) :: storm_speed_min + + ! switch for restoring legacy method + logical, intent(in) :: use_gw_convect_old + ! Indices of top gravity wave source level and lowest level where wind ! tendencies are allowed. integer, intent(out) :: src_level(ncol) @@ -110,19 +123,19 @@ subroutine gw_beres_src(ncol, ngwv, lat, u, v, netdt, & real(r8), intent(out) :: c(ncol,-pgwv:pgwv) ! Heating depth and maximum heating in each column. - real(r8), intent(out) :: hdepth(ncol), maxq0(ncol) + real(r8), intent(out) :: hdepth(ncol), maxq0_out(ncol) !---------------------------Local Storage------------------------------- ! Column and level indices. integer :: i, k - ! Zonal/meridional wind at 700mb. - real(r8) :: u700(ncol), v700(ncol) + ! Zonal/meridional source wind + real(r8) :: u_src(ncol), v_src(ncol) ! 3.14... real(r8), parameter :: pi = 4._r8*atan(1._r8) ! Maximum heating rate. - real(r8) :: q0(ncol) + real(r8) :: maxq0(ncol) ! Bottom/top heating range index. integer :: mini(ncol), maxi(ncol) @@ -135,36 +148,40 @@ subroutine gw_beres_src(ncol, ngwv, lat, u, v, netdt, & ! Source level tau for a column. real(r8) :: tau0(-PGWV:PGWV) ! Speed of convective cells relative to storm. - integer :: CS(ncol) + integer :: storm_speed(ncol) ! Index to shift spectra relative to ground. integer :: shift - ! Heating rate conversion factor. Change to take the value from caller and controllable by namelist (to tune QBO) - ! real(r8), parameter :: CF = 20._r8 - ! Averaging length. - real(r8), parameter :: AL = 1.0e5_r8 + ! fixed parameters (we may want to expose these in the namelist for tuning) + real(r8), parameter :: tau_avg_length = 100e3 ! spectrum averaging length [m] + real(r8), parameter :: heating_altitude_max = 20e3 ! max altitude [m] to check for max heating + + ! note: the heating_altitude_max is probably not needed because there is + ! rarely any convective heating above this level and the performance impact + ! of skipping the iteration over higher levels is likely negilible. + + integer :: ndepth_pos + integer :: ndepth_tot !---------------------------------------------------------------------- ! Initialize tau array !---------------------------------------------------------------------- - tau = 0.0_r8 + tau = 0.0_r8 hdepth = 0.0_r8 - q0 = 0.0_r8 - tau0 = 0.0_r8 + maxq0 = 0.0_r8 + tau0 = 0.0_r8 !------------------------------------------------------------------------ - ! Determine 700 mb layer wind and unit vectors, then project winds. + ! Determine source layer wind and unit vectors, then project winds. !------------------------------------------------------------------------ - ! Just use the 700 mb interface values for the source wind speed and - ! direction (unit vector). - - u700 = u(:,k700) - v700 = v(:,k700) + ! source wind speed and direction + u_src = u(:,k_src_wind) + v_src = v(:,k_src_wind) ! Get the unit vector components and magnitude at the surface. - call get_unit_vector(u700, v700, xv, yv, ubi(:,k700)) + call get_unit_vector(u_src, v_src, xv, yv, ubi(:,k_src_wind)) ! Project the local wind at midpoints onto the source wind. do k = 1, pver @@ -184,33 +201,62 @@ subroutine gw_beres_src(ncol, ngwv, lat, u, v, netdt, & ! which heating rate is continuously positive. !----------------------------------------------------------------------- - ! First find the indices for the top and bottom of the heating range. + ! Find indices for the top and bottom of the heating range. mini = 0 maxi = 0 - do k = pver, 1, -1 - do i = 1, ncol + + if (use_gw_convect_old) then + !--------------------------------------------------------------------- + ! original version used in CAM4/5/6 and EAMv1/2/3 + do k = pver, 1, -1 + do i = 1, ncol if (mini(i) == 0) then - ! Detect if we are outside the maximum range (where z = 20 km). - if (zm(i,k) >= 20000._r8) then - mini(i) = k - maxi(i) = k - else - ! First spot where heating rate is positive. - if (netdt(i,k) > 0.0_r8) mini(i) = k - end if + ! Detect if we are outside the maximum range (where z = 20 km). + if (zm(i,k) >= heating_altitude_max) then + mini(i) = k + maxi(i) = k + else + ! First spot where heating rate is positive. + if (netdt(i,k) > 0.0_r8) mini(i) = k + end if else if (maxi(i) == 0) then - ! Detect if we are outside the maximum range (z = 20 km). - if (zm(i,k) >= 20000._r8) then - maxi(i) = k - else - ! First spot where heating rate is no longer positive. - if (.not. (netdt(i,k) > 0.0_r8)) maxi(i) = k - end if + ! Detect if we are outside the maximum range (z = 20 km). + if (zm(i,k) >= heating_altitude_max) then + maxi(i) = k + else + ! First spot where heating rate is no longer positive. + if (.not. (netdt(i,k) > 0.0_r8)) maxi(i) = k + end if end if - end do - ! When all done, exit - if (all(maxi /= 0)) exit - end do + end do + ! When all done, exit + if (all(maxi /= 0)) exit + end do + !--------------------------------------------------------------------- + else + !--------------------------------------------------------------------- + ! cleaner version that addresses bug in original where heating max and + ! depth were too low whenever heating <=0 occurred in the middle of + ! the heating profile (ex. at the melting level) + do i = 1, ncol + do k = pver, 1, -1 + if ( zm(i,k) < heating_altitude_max ) then + if ( netdt(i,k) > 0.0_r8 ) then + ! Set mini as first spot where heating rate is positive + if ( mini(i)==0 ) mini(i) = k + ! Set maxi to current level + maxi(i) = k + end if + else + ! above the max check if indices were found + if ( mini(i)==0 ) mini(i) = k + if ( maxi(i)==0 ) maxi(i) = k + end if + end do + end do + !--------------------------------------------------------------------- + end if + ! Heating depth in km. hdepth = [ ( (zm(i,maxi(i))-zm(i,mini(i)))/1000._r8, i = 1, ncol ) ] @@ -223,19 +269,19 @@ subroutine gw_beres_src(ncol, ngwv, lat, u, v, netdt, & ! Maximum heating rate. do k = minval(maxi), maxval(mini) where (k >= maxi .and. k <= mini) - q0 = max(q0, netdt(:ncol,k)) + maxq0 = max(maxq0, netdt(:ncol,k)) end where end do !output max heating rate in K/day - maxq0 = q0*24._r8*3600._r8 + maxq0_out = maxq0*24._r8*3600._r8 ! Multipy by conversion factor - q0 = q0 * CF + maxq0 = maxq0 * maxq0_conversion_factor - ! Taking ubm at 700 mb to be the storm speed, find the cell speed where - ! the storm speed is > 10 m/s. - CS = int(sign(max(abs(ubm(:,k700))-10._r8, 0._r8), ubm(:,k700))) + ! Taking ubm at assumed source level to be the storm speed, + ! find the cell speed where the storm speed is > storm_speed_min + storm_speed = int(sign(max(abs(ubm(:,k_src_wind))-storm_speed_min, 0._r8), ubm(:,k_src_wind))) uh = 0._r8 do k = minval(maxi), maxval(mini) @@ -244,7 +290,7 @@ subroutine gw_beres_src(ncol, ngwv, lat, u, v, netdt, & end where end do - uh = uh - real(CS, r8) + uh = uh - real(storm_speed, r8) ! Limit uh to table range. uh = min(uh, real(maxuh, r8)) @@ -270,8 +316,19 @@ subroutine gw_beres_src(ncol, ngwv, lat, u, v, netdt, & !--------------------------------------------------------------------- ! Look up spectrum only if depth >= 2.5 km, else set tau0 = 0. !--------------------------------------------------------------------- - - if ((hdepth(i) >= 2.5_r8) .and. (abs(lat(i)) < (pi/2._r8))) then + if ((hdepth(i) >= hdepth_min)) then + ndepth_pos = 0 + ndepth_tot = 0 + do k = 1,pver + if ( k>=maxi(i).and.k<=mini(i) ) then + ndepth_tot = ndepth_tot + 1 + if ( netdt(i,k)>0 ) ndepth_pos = ndepth_pos + 1 + end if + end do + ! write (iulog,*) 'WHDEBUG - i: ',i,' Hd: ',hdepth(i),' Hn: ',ndepth_pos,' N: ',ndepth_tot + end if + + if ((hdepth(i) >= hdepth_min) .and. (abs(lat(i)) < (pi/2._r8))) then !------------------------------------------------------------------ ! Look up the spectrum using depth and uh. @@ -280,11 +337,11 @@ subroutine gw_beres_src(ncol, ngwv, lat, u, v, netdt, & tau0 = mfcc(NINT(hdepth(i)),NINT(uh(i)),:) ! Shift spectrum so that it is relative to the ground. - shift = -nint(real(CS(i), r8)/dc) + shift = -nint(storm_speed(i)/dc) tau0 = cshift(tau0,shift) ! Adjust magnitude. - tau0 = tau0*q0(i)*q0(i)/AL + tau0 = tau0*maxq0(i)*maxq0(i)/tau_avg_length ! Adjust for critical level filtering. Umini = max(nint(Umin(i)/dc),-PGWV) diff --git a/components/eam/src/physics/cam/gw_drag.F90 b/components/eam/src/physics/cam/gw_drag.F90 index 1c6b7920e85..2f3c1652a68 100644 --- a/components/eam/src/physics/cam/gw_drag.F90 +++ b/components/eam/src/physics/cam/gw_drag.F90 @@ -25,18 +25,21 @@ module gw_drag use shr_kind_mod, only: r8 => shr_kind_r8 use ppgrid, only: pcols, pver + use hycoef, only: hyai, hybi, hyam, hybm, etamid use constituents, only: pcnst use physics_types, only: physics_state, physics_ptend, physics_ptend_init use spmd_utils, only: masterproc use cam_history, only: outfld, hist_fld_active use cam_logfile, only: iulog - use cam_abortutils, only: endrun + use cam_abortutils,only: endrun use ref_pres, only: do_molec_diff, ntop_molec, nbot_molec - use physconst, only: cpair + use physconst, only: cpair, rh2o, zvir, pi, rearth, r_universal!zvir is the ep1 in wrf,rearth is the radius of earth(m),r_universal is the gas constant ! These are the actual switches for different gravity wave sources. - use phys_control, only: use_gw_oro, use_gw_front, use_gw_convect, use_gw_energy_fix + ! The orographic control switches are also here + use phys_control, only: use_gw_oro, use_gw_front, use_gw_convect, use_gw_energy_fix, use_od_ls, use_od_bl, use_od_ss + use od_common, only: od_ls_ncleff, od_bl_ncd, od_ss_sncleff ! Typical module header implicit none @@ -47,6 +50,7 @@ module gw_drag ! PUBLIC: interfaces ! public :: gw_drag_readnl ! Read namelist + public :: gw_register ! Register pbuf variables public :: gw_init ! Initialization public :: gw_tend ! interface to actual parameterization @@ -118,6 +122,11 @@ module gw_drag ! namelist logical :: history_amwg ! output the variables used by the AMWG diag package + logical :: use_gw_convect_old ! switch to enable legacy behavior + real(r8) :: gw_convect_plev_src_wind ! reference pressure level for source wind for convective GWD [Pa] + real(r8) :: gw_convect_hdepth_min ! minimum hdepth for for convective GWD spectrum lookup table [km] + real(r8) :: gw_convect_storm_speed_min ! minimum convective storm speed for convective GWD [m/s] + !========================================================================== contains !========================================================================== @@ -141,7 +150,9 @@ subroutine gw_drag_readnl(nlfile) namelist /gw_drag_nl/ pgwv, gw_dc, tau_0_ubc, effgw_beres, effgw_cm, & effgw_oro, fcrit2, frontgfc, gw_drag_file, taubgnd, gw_convect_hcf, & - hdepth_scaling_factor + hdepth_scaling_factor, gw_convect_hdepth_min, & + gw_convect_storm_speed_min, gw_convect_plev_src_wind, & + use_gw_convect_old !---------------------------------------------------------------------- if (masterproc) then @@ -170,8 +181,12 @@ subroutine gw_drag_readnl(nlfile) call mpibcast(frontgfc, 1, mpir8, 0, mpicom) call mpibcast(taubgnd, 1, mpir8, 0, mpicom) call mpibcast(gw_drag_file, len(gw_drag_file), mpichar, 0, mpicom) - call mpibcast(gw_convect_hcf, 1, mpir8, 0, mpicom) - call mpibcast(hdepth_scaling_factor, 1, mpir8, 0, mpicom) + call mpibcast(gw_convect_hcf, 1, mpir8, 0, mpicom) + call mpibcast(hdepth_scaling_factor, 1, mpir8, 0, mpicom) + call mpibcast(gw_convect_hdepth_min, 1, mpir8, 0, mpicom) + call mpibcast(gw_convect_storm_speed_min, 1, mpir8, 0, mpicom) + call mpibcast(gw_convect_plev_src_wind, 1, mpir8, 0, mpicom) + call mpibcast(use_gw_convect_old, 1, mpilog, 0, mpicom) #endif dc = gw_dc @@ -196,7 +211,16 @@ end subroutine gw_drag_readnl !========================================================================== -subroutine gw_init() +subroutine gw_register() + use od_common, only: oro_drag_register + + call oro_drag_register() + +end subroutine gw_register + +!========================================================================== + +subroutine gw_init(pbuf2d) !----------------------------------------------------------------------- ! Time independent initialization for multiple gravity wave ! parameterization. @@ -205,7 +229,7 @@ subroutine gw_init() use cam_history, only: addfld, horiz_only, add_default use interpolate_data, only: lininterp use phys_control, only: phys_getopts - use physics_buffer, only: pbuf_get_index + use physics_buffer, only: pbuf_get_index, physics_buffer_desc use ref_pres, only: pref_edge use physconst, only: gravit, rair @@ -215,6 +239,9 @@ subroutine gw_init() use gw_front, only: gw_front_init use gw_convect, only: gw_convect_init + use od_common, only: oro_drag_init + !------------------------------Arguments-------------------------------- + type(physics_buffer_desc), pointer :: pbuf2d(:,:) !---------------------------Local storage------------------------------- integer :: l, k @@ -224,7 +251,6 @@ subroutine gw_init() ! Index for levels at specific pressures. integer :: kfront - integer :: k700 ! output tendencies and state variables for CAM4 temperature, ! water vapor, cloud ice and cloud liquid budgets. @@ -288,6 +314,8 @@ subroutine gw_init() !----------------------------------------------------------------------- + call oro_drag_init(pbuf2d) + ! Set model flags. do_spectral_waves = (pgwv > 0 .and. (use_gw_front .or. use_gw_convect)) orographic_only = (use_gw_oro .and. .not. do_spectral_waves) @@ -362,13 +390,21 @@ subroutine gw_init() errstring) if (trim(errstring) /= "") call endrun("gw_common_init: "//errstring) - if (use_gw_oro) then - - if (effgw_oro == unset_r8) then + if (use_gw_oro.or.& + use_od_ls.or.& + use_od_bl.or.& + use_od_ss) then + ! + if (use_gw_oro.and.effgw_oro == unset_r8) then call endrun("gw_drag_init: Orographic gravity waves enabled, & &but effgw_oro was not set.") end if - + ! + if (use_gw_oro.and.use_od_ls) then + call endrun("gw_drag_init: Both orographic gravity waves schemes are turned on, & + &please turn one off by setting use_gw_oro or use_od_ls as .false.") + end if + ! call gw_oro_init(errstring) if (trim(errstring) /= "") call endrun("gw_oro_init: "//errstring) @@ -383,6 +419,36 @@ subroutine gw_init() 'Zonal gravity wave surface stress') call addfld ('TAUGWY',horiz_only, 'A','N/m2', & 'Meridional gravity wave surface stress') + if (use_od_ls.or.& + use_od_bl.or.& + use_od_ss) then + !added for orographic drag + call addfld ('DTAUX3_LS',(/'lev'/),'A','m/s2','U tendency - ls orographic drag') + call addfld ('DTAUY3_LS',(/'lev'/),'A','m/s2','V tendency - ls orographic drag') + call addfld ('DTAUX3_BL',(/'lev'/),'A','m/s2','U tendency - bl orographic drag') + call addfld ('DTAUY3_BL',(/'lev'/),'A','m/s2','V tendency - bl orographic drag') + call addfld ('DTAUX3_SS',(/'lev'/),'A','m/s2','U tendency - ss orographic drag') + call addfld ('DTAUY3_SS',(/'lev'/),'A','m/s2','V tendency - ss orographic drag') + call addfld ('DUSFC_LS',horiz_only,'A', 'N/m2', 'ls zonal oro surface stress') + call addfld ('DVSFC_LS',horiz_only,'A', 'N/m2', 'ls merio oro surface stress') + call addfld ('DUSFC_BL',horiz_only,'A', 'N/m2', 'bl zonal oro surface stress') + call addfld ('DVSFC_BL',horiz_only,'A', 'N/m2', 'bl merio oro surface stress') + call addfld ('DUSFC_SS',horiz_only,'A', 'N/m2', 'ss zonal oro surface stress') + call addfld ('DVSFC_SS',horiz_only,'A', 'N/m2', 'ss merio oro surface stress') + call add_default('DTAUX3_LS ', 1,' ') + call add_default('DTAUY3_LS ', 1,' ') + call add_default('DTAUX3_BL ', 1,' ') + call add_default('DTAUY3_BL ', 1,' ') + call add_default('DTAUX3_SS ', 1,' ') + call add_default('DTAUY3_SS ', 1,' ') + call add_default ('DUSFC_LS ', 1,' ') + call add_default ('DVSFC_LS ', 1,' ') + call add_default ('DUSFC_BL ', 1,' ') + call add_default ('DVSFC_BL ', 1,' ') + call add_default ('DUSFC_SS ', 1,' ') + call add_default ('DVSFC_SS ', 1,' ') + !added for orographic drag output + endif if (history_amwg) then call add_default('TAUGWX ', 1, ' ') @@ -451,20 +517,10 @@ subroutine gw_init() ttend_dp_idx = pbuf_get_index('TTEND_DP') - do k = 0, pver - ! 700 hPa index - if (pref_edge(k+1) < 70000._r8) k700 = k+1 - end do - - if (masterproc) then - write (iulog,*) 'K700 =',k700 - end if - ! Initialization of Beres' parameterization parameters call gw_init_beres(mfcc) - call gw_convect_init(k700, mfcc, errstring) - if (trim(errstring) /= "") & - call endrun("gw_convect_init: "//errstring) + call gw_convect_init(gw_convect_plev_src_wind, mfcc, errstring) + if (trim(errstring) /= "") call endrun("gw_convect_init: "//errstring) ! Output for gravity waves from the Beres scheme. call gw_spec_addflds(prefix=beres_pf, scheme="Beres", & @@ -583,12 +639,15 @@ subroutine gw_tend(state, sgh, pbuf, dt, ptend, cam_in) use camsrfexch, only: cam_in_t ! Location-dependent cpair use physconst, only: cpairv + use od_common, only: oro_drag_interface use gw_common, only: gw_prof, momentum_energy_conservation, & gw_drag_prof use gw_oro, only: gw_oro_src use gw_front, only: gw_cm_src use gw_convect, only: gw_beres_src use dycore, only: dycore_is + use phys_grid, only: get_rlat_all_p + use physconst, only: gravit,rair !------------------------------Arguments-------------------------------- type(physics_state), intent(in) :: state ! physics state structure ! Standard deviation of orography. @@ -598,6 +657,26 @@ subroutine gw_tend(state, sgh, pbuf, dt, ptend, cam_in) ! Parameterization net tendencies. type(physics_ptend), intent(out):: ptend type(cam_in_t), intent(in) :: cam_in + !locally added gw and bl drag + real(r8) :: dtaux3_ls(pcols,pver) + real(r8) :: dtauy3_ls(pcols,pver) + real(r8) :: dtaux3_bl(pcols,pver) + real(r8) :: dtauy3_bl(pcols,pver) + real(r8) :: dtaux3_ss(pcols,pver) + real(r8) :: dtauy3_ss(pcols,pver) + real(r8) :: dummx3_fd(pcols,pver) + real(r8) :: dummy3_fd(pcols,pver) + ! + real(r8) :: dusfc_ls(pcols) + real(r8) :: dvsfc_ls(pcols) + real(r8) :: dusfc_bl(pcols) + real(r8) :: dvsfc_bl(pcols) + real(r8) :: dusfc_ss(pcols) + real(r8) :: dvsfc_ss(pcols) + real(r8) :: dummx_fd(pcols) + real(r8) :: dummy_fd(pcols) + ! + real(r8) :: dx(pcols),dy(pcols) !---------------------------Local storage------------------------------- @@ -765,7 +844,9 @@ subroutine gw_tend(state, sgh, pbuf, dt, ptend, cam_in) ! Determine wave sources for Beres04 scheme call gw_beres_src(ncol, pgwv, state1%lat(:ncol), u, v, ttend_dp, & zm, src_level, tend_level, tau, ubm, ubi, xv, yv, c, & - hdepth, maxq0, gw_convect_hcf, hdepth_scaling_factor) + hdepth, maxq0, gw_convect_hcf, hdepth_scaling_factor, & + gw_convect_hdepth_min, gw_convect_storm_speed_min, & + use_gw_convect_old) do_latitude_taper = .false. @@ -879,7 +960,6 @@ subroutine gw_tend(state, sgh, pbuf, dt, ptend, cam_in) !--------------------------------------------------------------------- ! Orographic stationary gravity waves !--------------------------------------------------------------------- - ! Determine the orographic wave source call gw_oro_src(ncol, & u, v, t, sgh(:ncol), pmid, pint, dpm, zm, nm, & @@ -893,11 +973,35 @@ subroutine gw_tend(state, sgh, pbuf, dt, ptend, cam_in) piln, rhoi, nm, ni, ubm, ubi, xv, yv, & effgw_oro, c, kvtt, q, dse, tau, utgw, vtgw, & ttgw, qtgw, taucd, egwdffi, gwut(:,:,0:0), dttdf, dttke) - - ! Add the orographic tendencies to the spectrum tendencies - ! Compute the temperature tendency from energy conservation - ! (includes spectrum). - + endif + ! + if ( use_od_ls .or. use_od_bl .or. use_od_ss) then + utgw=0.0_r8 + vtgw=0.0_r8 + ttgw=0.0_r8 + call oro_drag_interface(state,cam_in,sgh,pbuf,dt,nm,& + use_od_ls,use_od_bl,use_od_ss,.false.,& + od_ls_ncleff,od_bl_ncd,od_ss_sncleff,& + utgw,vtgw,ttgw,& + dtaux3_ls=dtaux3_ls,dtauy3_ls=dtauy3_ls,& + dtaux3_bl=dtaux3_bl,dtauy3_bl=dtauy3_bl,& + dtaux3_ss=dtaux3_ss,dtauy3_ss=dtauy3_ss,& + dtaux3_fd=dummx3_fd,dtauy3_fd=dummy3_fd,& + dusfc_ls=dusfc_ls,dvsfc_ls=dvsfc_ls,& + dusfc_bl=dusfc_bl,dvsfc_bl=dvsfc_bl,& + dusfc_ss=dusfc_ss,dvsfc_ss=dvsfc_ss,& + dusfc_fd=dummx_fd,dvsfc_fd=dummy_fd) + endif + ! + ! Add the orographic tendencies to the spectrum tendencies + ! Compute the temperature tendency from energy conservation + ! (includes spectrum). + ! both old and new gwd scheme will add the tendency to circulation + ! + if (use_gw_oro.or.& + use_od_ls .or.& + use_od_bl .or.& + use_od_ss) then if(.not. use_gw_energy_fix) then !original do k = 1, pver @@ -906,11 +1010,11 @@ subroutine gw_tend(state, sgh, pbuf, dt, ptend, cam_in) vtgw(:,k) = vtgw(:,k) * cam_in%landfrac(:ncol) ptend%v(:ncol,k) = ptend%v(:ncol,k) + vtgw(:,k) ptend%s(:ncol,k) = ptend%s(:ncol,k) + ttgw(:,k) & - -(ptend%u(:ncol,k) * (u(:,k) + ptend%u(:ncol,k)*0.5_r8*dt) & - +ptend%v(:ncol,k) * (v(:,k) + ptend%v(:ncol,k)*0.5_r8*dt)) + -(ptend%u(:ncol,k) * (u(:,k) + ptend%u(:ncol,k)*0.5_r8*dt) & + +ptend%v(:ncol,k) * (v(:,k) + ptend%v(:ncol,k)*0.5_r8*dt)) ttgw(:,k) = ttgw(:,k) & - -(ptend%u(:ncol,k) * (u(:,k) + ptend%u(:ncol,k)*0.5_r8*dt) & - +ptend%v(:ncol,k) * (v(:,k) + ptend%v(:ncol,k)*0.5_r8*dt)) + -(ptend%u(:ncol,k) * (u(:,k) + ptend%u(:ncol,k)*0.5_r8*dt) & + +ptend%v(:ncol,k) * (v(:,k) + ptend%v(:ncol,k)*0.5_r8*dt)) ttgw(:,k) = ttgw(:,k) / cpairv(:ncol, k, lchnk) end do else @@ -947,12 +1051,34 @@ subroutine gw_tend(state, sgh, pbuf, dt, ptend, cam_in) call outfld('UTGWORO', utgw, ncol, lchnk) call outfld('VTGWORO', vtgw, ncol, lchnk) call outfld('TTGWORO', ttgw, ncol, lchnk) + ! + if (use_gw_oro) then + !old gwd scheme tau0x = tau(:,0,pver) * xv * effgw_oro tau0y = tau(:,0,pver) * yv * effgw_oro call outfld('TAUGWX', tau0x, ncol, lchnk) call outfld('TAUGWY', tau0y, ncol, lchnk) + endif + ! call outfld('SGH ', sgh,pcols, lchnk) - + ! + if (use_od_ls.or.& + use_od_bl.or.& + use_od_ss) then + call outfld ('DTAUX3_LS', dtaux3_ls, pcols, lchnk) + call outfld ('DTAUY3_LS', dtauy3_ls, pcols, lchnk) + call outfld ('DTAUX3_BL', dtaux3_bl, pcols, lchnk) + call outfld ('DTAUY3_BL', dtauy3_bl, pcols, lchnk) + call outfld ('DTAUX3_SS', dtaux3_ss, pcols, lchnk) + call outfld ('DTAUY3_SS', dtauy3_ss, pcols, lchnk) + call outfld ('DUSFC_LS', dusfc_ls, pcols, lchnk) + call outfld ('DVSFC_LS', dvsfc_ls, pcols, lchnk) + call outfld ('DUSFC_BL', dusfc_bl, pcols, lchnk) + call outfld ('DVSFC_BL', dvsfc_bl, pcols, lchnk) + call outfld ('DUSFC_SS', dusfc_ss, pcols, lchnk) + call outfld ('DVSFC_SS', dvsfc_ss, pcols, lchnk) + endif + ! end if ! Convert the tendencies for the dry constituents to dry air basis. diff --git a/components/eam/src/physics/cam/hb_diff.F90 b/components/eam/src/physics/cam/hb_diff.F90 index fdebeb1ee93..3d18ce50280 100644 --- a/components/eam/src/physics/cam/hb_diff.F90 +++ b/components/eam/src/physics/cam/hb_diff.F90 @@ -36,6 +36,8 @@ module hb_diff public init_hb_diff public compute_hb_diff public pblintd + !added for separation calculation of monin-obklen length + public pblintd_ri ! ! PBL limits ! @@ -764,5 +766,121 @@ subroutine austausch_pbl(lchnk ,ncol , & end do return end subroutine austausch_pbl + !=============================================================================== + subroutine pblintd_ri(ncol ,gravit , & + thv ,z ,u ,v , & + ustar ,obklen ,kbfs ,rino_bulk) + use pbl_utils, only: virtem, calc_ustar, calc_obklen + integer, intent(in) :: ncol ! number of atmospheric columns + real(r8), intent(in) :: gravit + real(r8), intent(in) :: thv(pcols,pver) ! virtual temperature + real(r8), intent(in) :: z(pcols,pver) ! height above surface [m] + real(r8), intent(in) :: u(pcols,pver) ! windspeed x-direction [m/s] + real(r8), intent(in) :: v(pcols,pver) ! windspeed y-direction [m/s] + real(r8), intent(in) :: ustar(pcols) ! surface friction velocity [m/s] + real(r8), intent(in) :: obklen(pcols) ! Obukhov length + real(r8), intent(in) :: kbfs(pcols) ! sfc kinematic buoyancy flux [m^2/s^3] + ! + ! Output arguments + ! + real(r8) :: wstar(pcols) ! convective sclae velocity [m/s] + real(r8) :: pblh(pcols) ! boundary-layer height [m] + real(r8) :: bge(pcols) ! buoyancy gradient enhancment + real(r8), intent(out) :: rino_bulk(pcols) ! bulk Richardson no. surface level + ! + !---------------------------Local parameters---------------------------- + ! + real(r8), parameter :: tiny = 1.e-36_r8 ! lower bound for wind magnitude + real(r8), parameter :: fac = 100._r8 ! ustar parameter in height diagnosis + ! + !---------------------------Local workspace----------------------------- + ! + integer :: i ! longitude index + integer :: k ! level index + real(r8) :: phiminv(pcols) ! inverse phi function for momentum + real(r8) :: phihinv(pcols) ! inverse phi function for heat + real(r8) :: rino(pcols,pver) ! bulk Richardson no. from level to ref lev + real(r8) :: tlv(pcols) ! ref. level pot tmp + tmp excess + real(r8) :: tref(pcols) ! ref. level pot tmp + real(r8) :: vvk ! velocity magnitude squared + logical :: unstbl(pcols) ! pts w/unstbl pbl (positive virtual ht flx) + logical :: check(pcols) ! True=>chk if Richardson no.>critcal + ! + do i=1,ncol + check(i) = .true. + rino(i,pver) = 0.0_r8 + rino_bulk(i) = 0.0_r8 + pblh(i) = z(i,pver) + tref(i) = thv(i,pver)!if not excess then tref is equal to lowest level thv_lv + end do + ! + ! PBL height calculation: Scan upward until the Richardson number between + ! the first level and the current level exceeds the "critical" value. + ! + do k=pver-1,pver-npbl+1,-1 + do i=1,ncol + if (check(i)) then + vvk = (u(i,k) - u(i,pver))**2 + (v(i,k) - v(i,pver))**2 + fac*ustar(i)**2 + vvk = max(vvk,tiny) + rino(i,k) = gravit*(thv(i,k) - thv(i,pver))*(z(i,k)-z(i,pver))/(thv(i,pver)*vvk) + if (rino(i,k) >= ricr) then + pblh(i) = z(i,k+1) + (ricr - rino(i,k+1))/(rino(i,k) - rino(i,k+1)) * & + (z(i,k) - z(i,k+1)) + check(i) = .false. + end if + end if + end do + end do + ! + ! Estimate an effective surface temperature to account for surface fluctuations + ! + do i=1,ncol + if (check(i)) pblh(i) = z(i,pverp-npbl) + unstbl(i) = (kbfs(i) > 0._r8) + check(i) = (kbfs(i) > 0._r8) + if (check(i)) then + phiminv(i) = (1._r8 - binm*pblh(i)/obklen(i))**onet + rino(i,pver) = 0.0_r8 + tlv(i) = thv(i,pver) + kbfs(i)*fak/( ustar(i)*phiminv(i) ) + tref(i) = tlv(i) + end if + end do + ! + ! Improve pblh estimate for unstable conditions using the convective temperature excess: + ! + do i = 1,ncol + bge(i) = 1.e-8_r8 + end do + do k=pver-1,pver-npbl+1,-1 + do i=1,ncol + if (check(i)) then + vvk = (u(i,k) - u(i,pver))**2 + (v(i,k) - v(i,pver))**2 + fac*ustar(i)**2 + vvk = max(vvk,tiny) + rino(i,k) = gravit*(thv(i,k) - tlv(i))*(z(i,k)-z(i,pver))/(thv(i,pver)*vvk) + if (rino(i,k) >= ricr) then + pblh(i) = z(i,k+1) + (ricr - rino(i,k+1))/(rino(i,k) - rino(i,k+1))* & + (z(i,k) - z(i,k+1)) + bge(i) = 2._r8*gravit/(thv(i,k)+thv(i,k+1))*(thv(i,k)-thv(i,k+1))/(z(i,k)-z(i,k+1))*pblh(i) + if (bge(i).lt.0._r8) then + bge(i) = 1.e-8_r8 + endif + check(i) = .false. + end if + end if + end do + end do + ! + !calculate bulk richardson number in the surface layer + !following Holstag and Boville (1993) equation (2.8) + ! + do i=1,ncol + vvk = u(i,pver)**2 + v(i,pver)**2 + fac*ustar(i)**2 + vvk = max(vvk,tiny) + rino_bulk(i)=gravit*(thv(i,pver) - tref(i))*z(i,pver)/(thv(i,pver)*vvk) + enddo + ! + return + end subroutine pblintd_ri + !=============================================================================== end module hb_diff diff --git a/components/eam/src/physics/cam/od_common.F90 b/components/eam/src/physics/cam/od_common.F90 new file mode 100644 index 00000000000..52658d2dbec --- /dev/null +++ b/components/eam/src/physics/cam/od_common.F90 @@ -0,0 +1,1693 @@ +module od_common +!========================================================================== +! This module contains code common to different orographic drag +! parameterizations. +! It includes 4 parts: +! orographic gravity wave drag (Xie et al.,2020), +! flow-blocking drag (Xie et al.,2020), +! small-scale orographic gravity wave drag (Tsiringakis et al. 2017), +! turbulent orographic form drag (Beljaars et al.,2004). +!========================================================================== +use shr_kind_mod, only: i8 => shr_kind_i8, r8 => shr_kind_r8 +use shr_sys_mod, only: shr_sys_flush +use ppgrid, only: pcols, pver, begchunk, endchunk +use cam_logfile, only: iulog +use cam_abortutils,only: endrun +use spmd_utils, only: masterproc +use pio, only: file_desc_t +use phys_control, only: use_od_ls, use_od_bl, use_od_ss +use physics_buffer,only: dtype_r8, physics_buffer_desc, pbuf_get_chunk +use physics_buffer,only: pbuf_get_index, pbuf_get_field, pbuf_add_field, pbuf_set_field + +implicit none +private +save + +! Public interface. +public :: oro_drag_readnl +public :: oro_drag_register +public :: oro_drag_init +public :: oro_drag_interface +public :: od_gsd,pblh_get_level_idx,grid_size + +type(file_desc_t), pointer :: topo_file_ncid + +! dimensions for topo shape data +integer, parameter :: ndir_asymmetry = 2+1 ! add 1 to avoid bug reading file - not sure why this happens +integer, parameter :: ndir_efflength = 180 ! 1-degree resolution with opposite directions mirrored + +! pbuf indices for data read in from topo data file +integer :: oro_drag_convexity_idx = -1 ! Convexity +integer :: oro_drag_asymmetry_idx = -1 ! Asymmetry +integer :: oro_drag_efflength_idx = -1 ! Effective length +integer :: oro_drag_ribulk_idx = -1 ! bulk richardson number (calculated in CLUBB) + +!tunable parameter to the od schemes +real(r8),public, protected :: od_ls_ncleff = 3._r8 !tunable parameter for oGWD +real(r8),public, protected :: od_bl_ncd = 3._r8 !tunable parameter for FBD +real(r8),public, protected :: od_ss_sncleff= 1._r8 !tunable parameter for sGWD + +contains + +!========================================================================== + +subroutine oro_drag_readnl(nlfile) + + use namelist_utils, only: find_group_name + use units, only: getunit, freeunit + use mpishorthand + + ! File containing namelist input. + character(len=*), intent(in) :: nlfile + + ! Local variables + integer :: unitn, ierr + character(len=*), parameter :: subname = 'oro_drag_readnl' + + ! More specific name for dc to prevent a name clash or confusion in the + ! namelist. + + namelist /oro_drag_nl/ od_ls_ncleff, od_bl_ncd, od_ss_sncleff + !--------------------------------------------------------------------- + !read oro_drag_nl only when use the od schemes + if (use_od_ls.or.use_od_bl.or.use_od_ss) then + if (masterproc) then + unitn = getunit() + open( unitn, file=trim(nlfile), status='old' ) + call find_group_name(unitn, 'oro_drag_nl', status=ierr) + if (ierr == 0) then + read(unitn, oro_drag_nl, iostat=ierr) + if (ierr /= 0) then + call endrun(subname // ':: ERROR reading namelist') + end if + end if + close(unitn) + call freeunit(unitn) + end if + + if (masterproc) write(iulog,*) "oro_drag_readnl od_ls_ncleff, od_bl_ncd, od_ss_sncleff ",od_ls_ncleff,od_bl_ncd,od_ss_sncleff + +#ifdef SPMD + ! Broadcast namelist variables + call mpibcast(od_ls_ncleff, 1, mpir8, 0, mpicom) + call mpibcast(od_bl_ncd, 1, mpir8, 0, mpicom) + call mpibcast(od_ss_sncleff, 1, mpir8, 0, mpicom) +#endif + ! + endif + +end subroutine oro_drag_readnl + +!========================================================================== + +subroutine oro_drag_open_topo_file() + use filenames, only: bnd_topo + use ioFileMod, only: getfil + use cam_pio_utils,only: cam_pio_openfile + use pio, only: pio_nowrite + include 'netcdf.inc' + !----------------------------------------------------------------------- + character(len=256) :: bnd_topo_loc ! filepath of topo file on local disk + allocate(topo_file_ncid) + call getfil(bnd_topo, bnd_topo_loc) + call cam_pio_openfile(topo_file_ncid, bnd_topo_loc, PIO_NOWRITE) +end subroutine oro_drag_open_topo_file + +!========================================================================== + +subroutine oro_drag_close_topo_file + use pio, only: pio_closefile + call pio_closefile(topo_file_ncid) + deallocate(topo_file_ncid) + nullify(topo_file_ncid) +end subroutine oro_drag_close_topo_file + +!========================================================================== + +subroutine oro_drag_register() + !----------------------------------------------------------------------- + ! Register pbuf variables for orographic drag parameterizations + !----------------------------------------------------------------------- + ! create pbuf variables to hold oro drag data + if (use_od_ls.or.use_od_bl) then + call pbuf_add_field('oro_drag_convexity','physpkg',dtype_r8,(/pcols/), oro_drag_convexity_idx) + call pbuf_add_field('oro_drag_asymmetry','physpkg',dtype_r8,(/pcols,ndir_asymmetry/),oro_drag_asymmetry_idx) + call pbuf_add_field('oro_drag_efflength','physpkg',dtype_r8,(/pcols,ndir_efflength/),oro_drag_efflength_idx) + end if + if (use_od_ss) then + call pbuf_add_field('oro_drag_ribulk', 'physpkg',dtype_r8,(/pcols/), oro_drag_ribulk_idx) + end if + +end subroutine oro_drag_register + +!========================================================================== + +subroutine oro_drag_init(pbuf2d) + !----------------------------------------------------------------------- + ! Initialization for orographic drag parameterizations + !----------------------------------------------------------------------- + use pio, only: file_desc_t + use ncdio_atm, only: infld + use cam_grid_support, only: cam_grid_check, cam_grid_get_decomp, cam_grid_id,cam_grid_get_dim_names + use infnan, only: nan, assignment(=) + !----------------------------------------------------------------------- + type(physics_buffer_desc), pointer :: pbuf2d(:,:) + !----------------------------------------------------------------------- + logical :: found + character(len=8) :: dim1name, dim2name + character*11 :: subname='oro_drag_init' + integer :: grid_id + integer :: c + + real(r8), allocatable:: oro_drag_convexity_tmp(:,:) + real(r8), allocatable:: oro_drag_asymmetry_tmp(:,:,:) + real(r8), allocatable:: oro_drag_efflength_tmp(:,:,:) + + type(physics_buffer_desc), pointer :: pbuf_chunk(:) ! temporary pbuf pointer for single chunk + !----------------------------------------------------------------------- + if (.not.(use_od_ls.or.use_od_bl)) return + + grid_id = cam_grid_id('physgrid') + if (.not. cam_grid_check(grid_id)) then + call endrun(trim(subname)//': Internal error, no "physgrid" grid') + end if + + ! Alocate variables for reading oro drag data + allocate( oro_drag_convexity_tmp(pcols,begchunk:endchunk) ) + allocate( oro_drag_asymmetry_tmp(pcols,ndir_asymmetry,begchunk:endchunk) ) + allocate( oro_drag_efflength_tmp(pcols,ndir_efflength,begchunk:endchunk) ) + oro_drag_convexity_tmp(:,:) = nan + oro_drag_asymmetry_tmp(:,:,:) = nan + oro_drag_efflength_tmp(:,:,:) = nan + + ! Read special orographic shape fields from topo file + call cam_grid_get_dim_names(grid_id, dim1name, dim2name) + call oro_drag_open_topo_file() + + found=.false. + call infld( 'OC', topo_file_ncid, dim1name, dim2name, 1, pcols, & + begchunk, endchunk, oro_drag_convexity_tmp(:,:), found, gridname='physgrid') + if(.not. found) call endrun('ERROR - oro_drag_init: topo file read error - OC') + + found=.false. + call infld( 'OA', topo_file_ncid, dim1name, 'ndir_asymmetry', dim2name, 1, pcols, 1, ndir_asymmetry, & + begchunk, endchunk, oro_drag_asymmetry_tmp(:,:,:), found, gridname='physgrid') + if(.not. found) call endrun('ERROR - oro_drag_init: topo file read error - OA') + + found=.false. + call infld( 'OL', topo_file_ncid, dim1name, 'ndir_efflength', dim2name, 1, pcols, 1, ndir_efflength, & + begchunk, endchunk, oro_drag_efflength_tmp(:,:,:), found, gridname='physgrid') + if(.not. found) call endrun('ERROR - oro_drag_init: topo file read error - OL') + + call oro_drag_close_topo_file() + + ! copy the oro drag data in pbuf + do c=begchunk,endchunk + pbuf_chunk => pbuf_get_chunk(pbuf2d, c) + call pbuf_set_field(pbuf_chunk, oro_drag_convexity_idx, oro_drag_convexity_tmp(:,c) ) + call pbuf_set_field(pbuf_chunk, oro_drag_asymmetry_idx, oro_drag_asymmetry_tmp(:,:,c) ) + call pbuf_set_field(pbuf_chunk, oro_drag_efflength_idx, oro_drag_efflength_tmp(:,:,c) ) + end do + + deallocate(oro_drag_convexity_tmp) + deallocate(oro_drag_asymmetry_tmp) + deallocate(oro_drag_efflength_tmp) + +end subroutine oro_drag_init +!========================================================================== + +subroutine oro_drag_interface(state, cam_in, sgh, pbuf, dtime, nm, & + gwd_ls, gwd_bl, gwd_ss, gwd_fd, & + od_ls_ncleff, od_bl_ncd,od_ss_sncleff, & + utgw, vtgw, ttgw, & + dtaux3_ls,dtauy3_ls,dtaux3_bl,dtauy3_bl, & + dtaux3_ss,dtauy3_ss,dtaux3_fd,dtauy3_fd, & + dusfc_ls, dvsfc_ls ,dusfc_bl, dvsfc_bl, & + dusfc_ss, dvsfc_ss ,dusfc_fd, dvsfc_fd) + use physics_types, only: physics_state + use camsrfexch, only: cam_in_t + use ppgrid, only: pcols,pver,pverp + use physconst, only: gravit,rair,cpair,rh2o,zvir,pi + use hycoef, only: etamid + !----------------------------------------------------------------------- + type(physics_state), intent(in) :: state ! physics state structure + type(cam_in_t), intent(in) :: cam_in + real(r8), intent(in) :: sgh(pcols) + type(physics_buffer_desc), pointer :: pbuf(:) ! Physics buffer + real(r8), intent(in) :: dtime + real(r8), intent(in) :: nm(state%ncol,pver) ! midpoint Brunt-Vaisalla frequency + !options for the 4 schemes + logical , intent(in) :: gwd_ls + logical , intent(in) :: gwd_bl + logical , intent(in) :: gwd_ss + logical , intent(in) :: gwd_fd + !tunable parameter from namelist + real(r8), intent(in) :: od_ls_ncleff + real(r8), intent(in) :: od_bl_ncd + real(r8), intent(in) :: od_ss_sncleff + !vertical profile of the momentum tendencies + real(r8), intent(out), optional :: utgw(state%ncol,pver) + real(r8), intent(out), optional :: vtgw(state%ncol,pver) + real(r8), intent(out), optional :: ttgw(state%ncol,pver) + !output drag terms in 3D and surface + real(r8), intent(out), optional :: dtaux3_ls(pcols,pver) + real(r8), intent(out), optional :: dtauy3_ls(pcols,pver) + real(r8), intent(out), optional :: dtaux3_bl(pcols,pver) + real(r8), intent(out), optional :: dtauy3_bl(pcols,pver) + real(r8), intent(out), optional :: dtaux3_ss(pcols,pver) + real(r8), intent(out), optional :: dtauy3_ss(pcols,pver) + real(r8), intent(out), optional :: dtaux3_fd(pcols,pver) + real(r8), intent(out), optional :: dtauy3_fd(pcols,pver) + real(r8), intent(out), optional :: dusfc_ls(pcols) + real(r8), intent(out), optional :: dvsfc_ls(pcols) + real(r8), intent(out), optional :: dusfc_bl(pcols) + real(r8), intent(out), optional :: dvsfc_bl(pcols) + real(r8), intent(out), optional :: dusfc_ss(pcols) + real(r8), intent(out), optional :: dvsfc_ss(pcols) + real(r8), intent(out), optional :: dusfc_fd(pcols) + real(r8), intent(out), optional :: dvsfc_fd(pcols) + + real(r8) :: ztop(pcols,pver) ! top interface height asl (m) + real(r8) :: zbot(pcols,pver) ! bottom interface height asl (m) + real(r8) :: zmid(pcols,pver) ! middle interface height asl (m) + real(r8) :: dz(pcols,pver) ! model layer height + + integer :: pblh_idx = 0 + integer :: kpbl2d_in(pcols) + integer :: kpbl2d_reverse_in(pcols) + real(r8), pointer :: pblh(:) + real(r8) :: dx(pcols),dy(pcols) + + real(r8), pointer :: oro_drag_convexity(:) + real(r8), pointer :: oro_drag_asymmetry(:,:) + real(r8), pointer :: oro_drag_efflength(:,:) + real(r8), pointer :: oro_drag_ribulk(:) ! pbuf pointer for bulk richardson number + + integer :: ncol + integer :: i + integer :: k + !----------------------------------------------------------------------- + + ncol=state%ncol + !convert heights above surface to heights above sea level + !obtain z,dz,dx,dy,and k for pblh + kpbl2d_in=0_r8 + kpbl2d_reverse_in=0_r8 + ztop=0._r8 + zbot=0._r8 + zmid=0._r8 + dusfc_ls=0._r8 + dvsfc_ls=0._r8 + dusfc_bl=0._r8 + dvsfc_bl=0._r8 + dusfc_ss=0._r8 + dvsfc_ss=0._r8 + dusfc_fd=0._r8 + dvsfc_fd=0._r8 + dtaux3_ls=0._r8 + dtaux3_bl=0._r8 + dtauy3_ls=0._r8 + dtauy3_bl=0._r8 + dtaux3_ss=0._r8 + dtaux3_fd=0._r8 + dtauy3_ss=0._r8 + dtauy3_fd=0._r8 + + do k=1,pver + do i=1,ncol + ! assign values for level top/bottom + ztop(i,k)=state%zi(i,k) + zbot(i,k)=state%zi(i,k+1) + enddo + end do + + !transform adding the pressure + !transfer from surface to sea level + do k=1,pver + do i=1,ncol + ztop(i,k)=ztop(i,k)+state%phis(i)/gravit + zbot(i,k)=zbot(i,k)+state%phis(i)/gravit + zmid(i,k)=state%zm(i,k)+state%phis(i)/gravit + !dz is from bottom to top already for gw_drag + dz(i,k)=ztop(i,k)-zbot(i,k) + end do + end do + !get the layer index of pblh in layer for input in drag scheme + pblh_idx = pbuf_get_index('pblh') + call pbuf_get_field(pbuf, pblh_idx, pblh) + do i=1,pcols + kpbl2d_in(i)=pblh_get_level_idx(zbot(i,:)-(state%phis(i)/gravit),pblh(i)) + kpbl2d_reverse_in(i)=pverp-kpbl2d_in(i)!pverp-k + end do + + call pbuf_get_field(pbuf, oro_drag_convexity_idx, oro_drag_convexity ) + call pbuf_get_field(pbuf, oro_drag_asymmetry_idx, oro_drag_asymmetry ) + call pbuf_get_field(pbuf, oro_drag_efflength_idx, oro_drag_efflength ) + call pbuf_get_field(pbuf, oro_drag_ribulk_idx, oro_drag_ribulk) + + !get grid size for dx,dy + call grid_size(state,dx,dy) + + !interface for orographic drag + call od_gsd(u3d=state%u(:ncol,pver:1:-1),v3d=state%v(:ncol,pver:1:-1),t3d=state%t(:ncol,pver:1:-1),& + qv3d=state%q(:ncol,pver:1:-1,1),p3d=state%pmid(:ncol,pver:1:-1),p3di=state%pint(:ncol,pver+1:1:-1),& + pi3d=state%exner(:ncol,pver:1:-1),z=zbot(:ncol,pver:1:-1),& + od_ls_ncleff=od_ls_ncleff,od_bl_ncd=od_bl_ncd,od_ss_sncleff=od_ss_sncleff,& + rublten=utgw(:ncol,pver:1:-1),rvblten=vtgw(:ncol,pver:1:-1),rthblten=ttgw(:ncol,pver:1:-1),& + dtaux3d_ls=dtaux3_ls(:ncol,pver:1:-1),dtauy3d_ls=dtauy3_ls(:ncol,pver:1:-1),& + dtaux3d_bl=dtaux3_bl(:ncol,pver:1:-1),dtauy3d_bl=dtauy3_bl(:ncol,pver:1:-1),& + dtaux3d_ss=dtaux3_ss(:ncol,pver:1:-1),dtauy3d_ss=dtauy3_ss(:ncol,pver:1:-1),& + dtaux3d_fd=dtaux3_fd(:ncol,pver:1:-1),dtauy3d_fd=dtauy3_fd(:ncol,pver:1:-1),& + dusfcg_ls=dusfc_ls(:ncol),dvsfcg_ls=dvsfc_ls(:ncol),& + dusfcg_bl=dusfc_bl(:ncol),dvsfcg_bl=dvsfc_bl(:ncol),& + dusfcg_ss=dusfc_ss(:ncol),dvsfcg_ss=dvsfc_ss(:ncol),& + dusfcg_fd=dusfc_fd(:ncol),dvsfcg_fd=dvsfc_fd(:ncol),& + xland=cam_in%landfrac,br=oro_drag_ribulk(:ncol),& + var2d=sgh(:ncol),& + oc12d=oro_drag_convexity(:ncol),& + oa2d=oro_drag_asymmetry(:ncol,:),& + ol2d=oro_drag_efflength(:ncol,:),& + znu=etamid(pver:1:-1),dz=dz(:ncol,pver:1:-1),pblh=pblh(:ncol),& + cp=cpair,g=gravit,rd=rair,rv=rh2o,ep1=zvir,pi=pi,bnvbg=nm(:ncol,pver:1:-1),& + dt=dtime,dx=dx,dy=dy,& + kpbl2d=kpbl2d_reverse_in,gwd_opt=0,& + ids=1,ide=ncol,jds=0,jde=0,kds=1,kde=pver, & + ims=1,ime=ncol,jms=0,jme=0,kms=1,kme=pver, & + its=1,ite=ncol,jts=0,jte=0,kts=1,kte=pver, & + gwd_ls=gwd_ls,gwd_bl=gwd_bl,gwd_ss=gwd_ss,gwd_fd=gwd_fd ) + +end subroutine oro_drag_interface + +!========================================================================== + +function pblh_get_level_idx(height_array,pblheight) + implicit none + real(r8),intent(in),dimension(pver) :: height_array + real(r8),intent(in) :: pblheight + integer :: pblh_get_level_idx + !local + integer :: k + logical :: found + + pblh_get_level_idx = -1 + found=.false. + !get the pblh level index and return + do k = 1, pver + if((pblheight >= height_array(k+1).and.pblheight 300._r8) then + kpbl2 = k + IF (k == kpbl(i)) then + hpbl2 = hpbl(i)+10._r8 + ELSE + hpbl2 = za(i,k)+10._r8 + ENDIF + exit + ENDIF + enddo + + if(xland1(i).gt.0._r8 .and. 2._r8*var(i).le.hpbl(i))then + if(br1(i).gt.0._r8 .and. thvx(i,kpbl2)-thvx(i,kts) > 0._r8)then + cleff = sqrt(dxy(i)**2_r8 + dxyp(i)**2_r8) + cleff = (2.0_r8/sncleff) * max(dxmax_ss,cleff) + coefm(i) = (1._r8 + ol(i)) ** (oa1(i)+1._r8) + xlinv(i) = coefm(i) / cleff + govrth(i)=g/(0.5_r8*(thvx(i,kpbl2)+thvx(i,kts))) + bnrf=sqrt(govrth(i)*(thvx(i,kpbl2)-thvx(i,kts))/hpbl2) + + if(abs(bnrf/u1(i,kpbl2)).gt.xlinv(i))then + tauwavex0=0.5_r8*bnrf*xlinv(i)*(2._r8*MIN(var(i),varmax))**2_r8*ro(i,kvar)*u1(i,kvar) + tauwavex0=tauwavex0*ss_taper ! "Scale-awareness" + else + tauwavex0=0._r8 + endif + + if(abs(bnrf/v1(i,kpbl2)).gt.xlinv(i))then + tauwavey0=0.5_r8*bnrf*xlinv(i)*(2._r8*MIN(var(i),varmax))**2._r8*ro(i,kvar)*v1(i,kvar) + tauwavey0=tauwavey0*ss_taper ! "Scale-awareness" + else + tauwavey0=0._r8 + endif + + do k=kts,kpbl(i) !MIN(kpbl2+1,kte-1) + utendwave(i,k)=-1._r8*tauwavex0*2._r8*max((1._r8-za(i,k)/hpbl2),0._r8)/hpbl2 + vtendwave(i,k)=-1._r8*tauwavey0*2._r8*max((1._r8-za(i,k)/hpbl2),0._r8)/hpbl2 + enddo + endif + endif + enddo ! end i loop + + do k = kts,kte + do i = its,ite + dudt(i,k) = dudt(i,k) + utendwave(i,k) + dvdt(i,k) = dvdt(i,k) + vtendwave(i,k) + dtaux2d_ss(i,k) = utendwave(i,k) + dtauy2d_ss(i,k) = vtendwave(i,k) + dusfc_ss(i) = dusfc_ss(i) + utendwave(i,k) * del(i,k) + dvsfc_ss(i) = dvsfc_ss(i) + vtendwave(i,k) * del(i,k) + enddo + enddo + + ENDIF ! end if gsd_gwd_ss == .true. + !================================================================ + !add Beljaars et al. (2004, QJRMS, equ. 16) form drag: + !================================================================ + IF (gsd_gwd_fd.and.(ss_taper.GT.1.E-02) ) THEN + + utendform=0._r8 + vtendform=0._r8 + zq=0._r8 + + if (.not.gsd_gwd_ss.and.(ss_taper.GT.1.E-02) ) THEN + ! Defining layer height. This is already done above is small-scale GWD is used + do k = kts,kte + do i = its,ite + zq(i,k+1) = dz2(i,k)+zq(i,k) + enddo + enddo + + do k = kts,kte + do i = its,ite + za(i,k) = 0.5_r8*(zq(i,k)+zq(i,k+1)) + enddo + enddo + endif + + do i=its,ite + if (xland1(i) .gt. 0..and.2._r8*var(i).gt.0) then + ! refer to Beljaars (2004) eq.16. + a1=0.00026615161_r8*var(i)**2_r8 + a2=a1*0.005363_r8 + do k=kts,kte + wsp=SQRT(u1(i,k)**2_r8 + v1(i,k)**2_r8) + ! refer to Beljaars (2004) eq.16. + ! alpha*beta*Cmd*Ccorr*2.109 = 12.*1.*0.005*0.6*2.109 = 0.0759 + utendform(i,k)=-0.0759_r8*wsp*u1(i,k)* & + EXP(-(za(i,k)/1500._r8)**1.5_r8)*a2*za(i,k)**(-1.2_r8)*ss_taper + vtendform(i,k)=-0.0759_r8*wsp*v1(i,k)* & + EXP(-(za(i,k)/1500._r8)**1.5_r8)*a2*za(i,k)**(-1.2_r8)*ss_taper + enddo + endif + enddo + + do k = kts,kte + do i = its,ite + dudt(i,k) = dudt(i,k) + utendform(i,k) + dvdt(i,k) = dvdt(i,k) + vtendform(i,k) + !limit drag tendency + !some tendency is likely to even overturn the wind, + !making wind reverse in 1 timestep and reverse again in next, + !this limitation may help to make model stable, + !and no more wind reversal due to drag, + !which is suppose to decelerate, not accelerate + utendform(i,k) = sign(min(abs(utendform(i,k)),abs(u1(i,k))/deltim),utendform(i,k)) + vtendform(i,k) = sign(min(abs(vtendform(i,k)),abs(v1(i,k))/deltim),vtendform(i,k)) + dtaux2d_fd(i,k) = utendform(i,k) + dtauy2d_fd(i,k) = vtendform(i,k) + dusfc_fd(i) = dusfc_fd(i) + utendform(i,k) * del(i,k) + dvsfc_fd(i) = dvsfc_fd(i) + vtendform(i,k) * del(i,k) + enddo + enddo + ENDIF ! end if gsd_gwd_fd == .true. + !======================================================= + ! More for the large-scale gwd component + !======================================================= + IF (gsd_gwd_ls.and.(ls_taper.GT.1.E-02) ) THEN + ! + ! now compute vertical structure of the stress. + ! + do k = kts,kpblmax + do i = its,ite + if (k .le. kbl(i)) taup(i,k) = taub(i) + enddo + enddo + + if (scorer_on) then + ! + !determination of the interface height for scorer adjustment + ! + do i=its,ite + iint=.false. + do k=kpblmin,kte-1 + if (k.gt.kbl(i).and.usqj(i,k)-usqj(i,k-1).lt.0.and.(.not.iint)) then + iint=.true. + zl_hint(i)=zl(i,k+1) + endif + enddo + enddo + endif + + do k = kpblmin, kte-1 ! vertical level k loop! + kp1 = k + 1 + do i = its,ite + ! + ! unstablelayer if ri < ric + ! unstable layer if upper air vel comp along surf vel <=0 (crit lay) + ! at (u-c)=0. crit layer exists and bit vector should be set (.le.) + ! + if (k .ge. kbl(i)) then + !we modify the criteria for unstable layer + !that the lv is critical under 0.25 + !while we keep wave breaking ric for + !other larger lv + icrilv(i) = icrilv(i) .or. ( usqj(i,k) .lt. ric_rig)& + .or. (velco(i,k) .le. 0.0_r8) + brvf(i) = max(bnv2(i,k),bnv2min) ! brunt-vaisala frequency squared + brvf(i) = sqrt(brvf(i)) ! brunt-vaisala frequency + endif + enddo + + do i = its,ite + if (k .ge. kbl(i) .and. (.not. ldrag(i))) then + if (.not.icrilv(i) .and. taup(i,k) .gt. 0.0_r8 ) then + temv = 1.0_r8 / velco(i,k) + tem1 = coefm(i)/(dxy(i)/ncleff)*(ro(i,kp1)+ro(i,k))*brvf(i)*velco(i,k)*0.5_r8 + hd = sqrt(taup(i,k) / tem1) + fro = brvf(i) * hd * temv + ! + ! rim is the minimum-richardson number by shutts (1985) + ! + tem2 = sqrt(usqj(i,k)) + tem = 1._r8 + tem2 * fro + rim = usqj(i,k) * (1._r8-fro) / (tem * tem) + + ! + ! check stability to employ the 'saturation hypothesis' + ! of lindzen (1981) except at tropospheric downstream regions + ! + if (rim .le. ric) then ! saturation hypothesis! + if ((oa1(i) .le. 0._r8).or.(kp1 .ge. kpblmin )) then + temc = 2.0_r8 + 1.0_r8 / tem2 + hd = velco(i,k) * (2.0_r8*sqrt(temc)-temc) / brvf(i) + taup(i,kp1) = tem1 * hd * hd + ! + ! taup is restricted to monotoncally decrease + ! to avoid unexpected high taup in calculation + ! + taup(i,kp1)=min(tem1*hd*hd,taup(i,k)) + ! + ! add vertical decrease at low level below hint (Kim and Doyle 2005) + ! where Ri first decreases + ! + if (scorer_on.and.k.gt.klowtop(i).and.zl(i,k).le.zl_hint(i).and.k.lt.kte-1) then + l1=(9.81_r8*bnv2(i,kp1)/velco(i,kp1)**2) + l2=(9.81_r8*bnv2(i,k)/velco(i,k)**2) + taup(i,kp1)=min(taup(i,k),taup(i,k)*(l1/l2),tem1*hd*hd) + endif + endif + else ! no wavebreaking! + taup(i,kp1) = taup(i,k) + endif + endif + endif + enddo + enddo + + if(lcap.lt.kte) then + do klcap = lcapp1,kte + do i = its,ite + taup(i,klcap) = prsi(i,klcap) / prsi(i,lcap) * taup(i,lcap) + enddo + enddo + endif + + ENDIF !END LARGE-SCALE TAU CALCULATION + !=============================================================== + !COMPUTE BLOCKING COMPONENT + !=============================================================== + IF (gsd_gwd_bl.and.(ls_taper .GT. 1.E-02)) THEN + + do i = its,ite + if(.not.ldrag(i)) then + ! + !------- determine the height of flow-blocking layer + ! + kblk = 0 + pe = 0.0_r8 + + do k = kte, kpblmin, -1 + if(kblk.eq.0 .and. k.le.komax(i)) then + !flow block appears within the reference level + !compare potential energy and kinetic energy + !divided by g*ro is to turn del(pa) into height + pe = pe + bnv2(i,k)*(zl(i,komax(i))-zl(i,k))*del(i,k)/g/ro(i,k) + ke = 0.5_r8*((rcs*u1(i,k))**2._r8+(rcs*v1(i,k))**2._r8) + ! + !---------- apply flow-blocking drag when pe >= ke + ! + if(pe.ge.ke) then + kblk = k + kblk = min(kblk,kbl(i)) + zblk = zl(i,kblk)-zl(i,kts) + endif + endif + enddo + + if(kblk.ne.0) then + ! + !--------- compute flow-blocking stress + ! + + !dxmax_ls is different than the usual one + !because the taper is very different + !dxy is a length scale mostly in the direction of the flow to the ridge + !so it is good and not needed for an uneven grid area + !ref Lott and Miller (1997) original scheme + cd = max(2.0_r8-1.0_r8/od(i),0.0_r8) + ! + !tuning of the drag magnitude + cd=ncd*cd + ! + taufb(i,kts) = 0.5_r8 * roll(i) * coefm(i) / max(dxmax_ls,dxy(i))**2 * cd * dxyp(i) & + * olp(i) * zblk * ulow(i)**2 + !changed grid box area into dy*dy + tautem = taufb(i,kts)/float(kblk-kts) + do k = kts+1, kblk + taufb(i,k) = taufb(i,k-1) - tautem + enddo + + ! + !----------sum orographic GW stress and flow-blocking stress + ! + !taup(i,:) = taup(i,:) + taufb(i,:) ! Keep taup and taufb separate for now + endif + endif + enddo + + ENDIF ! end blocking drag +!=========================================================== + IF (gsd_gwd_ls.OR.gsd_gwd_bl.and.(ls_taper .GT. 1.E-02)) THEN + ! + ! calculate - (g)*d(tau)/d(pressure) and deceleration terms dtaux, dtauy + ! + do k = kts,kte + do i = its,ite + taud_ls(i,k) = 1._r8 * (taup(i,k+1) - taup(i,k)) * csg / del(i,k) + taud_bl(i,k) = 1._r8 * (taufb(i,k+1) - taufb(i,k)) * csg / del(i,k) + enddo + enddo + ! + ! limit de-acceleration (momentum deposition ) at top to 1/2 value + ! the idea is some stuff must go out the 'top' + ! + do klcap = lcap,kte + do i = its,ite + taud_ls(i,klcap) = taud_ls(i,klcap) * factop + taud_bl(i,klcap) = taud_bl(i,klcap) * factop + enddo + enddo + ! + ! if the gravity wave drag would force a critical line + ! in the lower ksmm1 layers during the next deltim timestep, + ! then only apply drag until that critical line is reached. + ! + do k = kts,kpblmax-1 + do i = its,ite + if (k .le. kbl(i)) then + if((taud_ls(i,k)+taud_bl(i,k)).ne.0._r8) & + dtfac(i) = min(dtfac(i),abs(velco(i,k) & + /(deltim*rcs*(taud_ls(i,k)+taud_bl(i,k))))) + endif + enddo + enddo + + do k = kts,kte + do i = its,ite + taud_ls(i,k) = taud_ls(i,k) * dtfac(i) * ls_taper + !apply limiter for ogwd + !1.dudt < |c-u|/dt, so u-c cannot change sign(u^n+1 = u^n + du/dt * dt) + !2.dudtchk if Richardson no.>critcal -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call pblintd_f(& - shcol,nlev,nlevi,npbl,& ! Input - z,zi,thl,ql,& ! Input - q,u,v,& ! Input - ustar,obklen,kbfs,cldn,& ! Input - pblh) ! Output - return - endif -#endif - ! ! Compute Obukhov length virtual temperature flux and various arrays for use later: ! @@ -4343,9 +3927,6 @@ subroutine pblintd_init_pot(& shcol,nlev,& ! Input thl,ql,q,& ! Input thv) ! Output -#ifdef SCREAM_CONFIG_IS_CMAKE - use shoc_iso_f, only: shoc_pblintd_init_pot_f -#endif !------------------------------Arguments-------------------------------- ! ! Input arguments @@ -4364,13 +3945,6 @@ subroutine pblintd_init_pot(& integer :: k ! level index real(rtype) :: th -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call shoc_pblintd_init_pot_f(shcol,nlev,thl,ql,q,& ! Input - thv) ! Output - return - endif -#endif ! Compute virtual potential temperature do k=1,nlev do i=1,shcol @@ -4418,10 +3992,6 @@ subroutine pblintd_height(& thv,thv_ref,& ! Input pblh,rino,check) ! Output -#ifdef SCREAM_CONFIG_IS_CMAKE - use shoc_iso_f, only: pblintd_height_f -#endif - !------------------------------Arguments-------------------------------- ! ! Input arguments @@ -4450,14 +4020,6 @@ subroutine pblintd_height(& integer :: k ! level index real(rtype) :: vvk ! velocity magnitude squared -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call pblintd_height_f(shcol,nlev,npbl,z,u,v,ustar,thv,thv_ref,& ! Input - pblh,rino,check) ! Output - return - endif -#endif - ! ! PBL height calculation: Scan upward until the Richardson number between ! the first level and the current level exceeds the "critical" value. @@ -4485,10 +4047,6 @@ subroutine pblintd_surf_temp(& tlv,& ! Output pblh,check,rino) ! InOutput -#ifdef SCREAM_CONFIG_IS_CMAKE - use shoc_iso_f, only: pblintd_surf_temp_f -#endif - !------------------------------Arguments-------------------------------- ! Input arguments ! @@ -4519,15 +4077,6 @@ subroutine pblintd_surf_temp(& real(rtype), parameter :: sffrac= 0.1_rtype ! Surface layer fraction of boundary layer real(rtype), parameter :: binm = betam*sffrac ! betam * sffrac -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call pblintd_surf_temp_f(shcol,nlev,nlevi,& ! Input - z,ustar,obklen,kbfs,thv,& ! Input - tlv,pblh,check,rino) ! InOutput - return - endif -#endif - ! ! Estimate an effective surface temperature to account for surface ! fluctuations @@ -4550,10 +4099,6 @@ subroutine pblintd_check_pblh(& z,ustar,check,& ! Input pblh) ! Output -#ifdef SCREAM_CONFIG_IS_CMAKE - use shoc_iso_f, only: pblintd_check_pblh_f -#endif - !------------------------------Arguments-------------------------------- ! Input arguments ! @@ -4573,13 +4118,6 @@ subroutine pblintd_check_pblh(& ! integer :: i ! longitude index -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call pblintd_check_pblh_f(shcol,nlev,nlevi,npbl,z,ustar,check,pblh) - return - endif -#endif - ! ! PBL height must be greater than some minimum mechanical mixing depth ! Several investigators have proposed minimum mechanical mixing depth @@ -4606,10 +4144,6 @@ subroutine pblintd_cldcheck( & zi,cldn, & ! Input pblh) ! InOutput -#ifdef SCREAM_CONFIG_IS_CMAKE - use shoc_iso_f, only: shoc_pblintd_cldcheck_f -#endif - !------------------------------Arguments-------------------------------- ! Input arguments ! @@ -4629,13 +4163,6 @@ subroutine pblintd_cldcheck( & integer :: i ! longitude index logical(btype) :: cldcheck(shcol) ! True=>if cloud in lowest layer -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call shoc_pblintd_cldcheck_f(shcol, nlev, nlevi, zi, cldn, pblh) - return - endif -#endif - ! ! Final requirement on PBL heightis that it must be greater than the depth ! of the lowest model level if there is any cloud diagnosed in @@ -4660,10 +4187,6 @@ end subroutine pblintd_cldcheck subroutine linear_interp(x1,x2,y1,y2,km1,km2,ncol,minthresh) -#ifdef SCREAM_CONFIG_IS_CMAKE - use shoc_iso_f, only: linear_interp_f -#endif - implicit none integer, intent(in) :: km1, km2 @@ -4675,13 +4198,6 @@ subroutine linear_interp(x1,x2,y1,y2,km1,km2,ncol,minthresh) integer :: k1, k2, i -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call linear_interp_f(x1,x2,y1,y2,km1,km2,ncol,minthresh) - return - endif -#endif - #if 1 !i = check_grid(x1,x2,km1,km2,ncol) if (km1 .eq. km2+1) then @@ -4749,10 +4265,6 @@ subroutine compute_brunt_shoc_length(nlev,nlevi,shcol,dz_zt,thv,thv_zi,brunt) ! ! Computes the brunt_visala frequency -#ifdef SCREAM_CONFIG_IS_CMAKE - use shoc_iso_f, only: compute_brunt_shoc_length_f -#endif - implicit none integer, intent(in) :: nlev, nlevi, shcol ! Grid difference centereted on thermo grid [m] @@ -4765,13 +4277,6 @@ subroutine compute_brunt_shoc_length(nlev,nlevi,shcol,dz_zt,thv,thv_zi,brunt) real(rtype), intent(out) :: brunt(shcol, nlev) integer k, i -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call compute_brunt_shoc_length_f(nlev,nlevi,shcol,dz_zt,thv,thv_zi,brunt) - return - endif -#endif - do k=1,nlev do i=1,shcol brunt(i,k) = (ggr/thv(i,k)) * (thv_zi(i,k) - thv_zi(i,k+1))/dz_zt(i,k) @@ -4785,10 +4290,6 @@ subroutine compute_l_inf_shoc_length(nlev,shcol,zt_grid,dz_zt,tke,l_inf) !========================================================= ! -#ifdef SCREAM_CONFIG_IS_CMAKE - use shoc_iso_f, only: compute_l_inf_shoc_length_f -#endif - implicit none integer, intent(in) :: nlev, shcol real(rtype), intent(in) :: zt_grid(shcol,nlev), dz_zt(shcol,nlev), tke(shcol,nlev) @@ -4796,13 +4297,6 @@ subroutine compute_l_inf_shoc_length(nlev,shcol,zt_grid,dz_zt,tke,l_inf) real(rtype) :: tkes, numer(shcol), denom(shcol) integer k, i -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call compute_l_inf_shoc_length_f(nlev,shcol,zt_grid,dz_zt,tke,l_inf) - return - endif -#endif - numer(:) = 0._rtype denom(:) = 0._rtype @@ -4822,10 +4316,6 @@ end subroutine compute_l_inf_shoc_length subroutine compute_shoc_mix_shoc_length(nlev,shcol,tke,brunt,zt_grid,l_inf,shoc_mix) -#ifdef SCREAM_CONFIG_IS_CMAKE - use shoc_iso_f, only: compute_shoc_mix_shoc_length_f -#endif - implicit none integer, intent(in) :: nlev, shcol @@ -4848,14 +4338,6 @@ subroutine compute_shoc_mix_shoc_length(nlev,shcol,tke,brunt,zt_grid,l_inf,shoc_ ! Turnover timescale [s] real(rtype), parameter :: tscale = 400._rtype -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call compute_shoc_mix_shoc_length_f(nlev,shcol,tke,brunt,zt_grid,l_inf,& !Input - shoc_mix) ! Ouptut - return - endif -#endif - brunt2(:,:) = 0.0 do k=1,nlev @@ -4876,10 +4358,6 @@ subroutine check_length_scale_shoc_length(nlev,shcol,host_dx,host_dy,shoc_mix) ! Do checks on the length scale. Make sure it is not ! larger than the grid mesh of the host model. -#ifdef SCREAM_CONFIG_IS_CMAKE - use shoc_iso_f, only: check_length_scale_shoc_length_f -#endif - implicit none integer, intent(in) :: nlev, shcol real(rtype), intent(in) :: host_dx(shcol), host_dy(shcol) @@ -4887,13 +4365,6 @@ subroutine check_length_scale_shoc_length(nlev,shcol,host_dx,host_dy,shoc_mix) real(rtype), intent(inout) :: shoc_mix(shcol, nlev) integer k, i -#ifdef SCREAM_CONFIG_IS_CMAKE - if (use_cxx) then - call check_length_scale_shoc_length_f(nlev,shcol,host_dx,host_dy,shoc_mix) - return - endif -#endif - do k=1,nlev do i=1,shcol shoc_mix(i,k)=min(maxlen,shoc_mix(i,k)) diff --git a/components/eam/src/physics/clubb/advance_windm_edsclrm_module.F90 b/components/eam/src/physics/clubb/advance_windm_edsclrm_module.F90 index 72d2e4d214b..970de0b4d7f 100644 --- a/components/eam/src/physics/clubb/advance_windm_edsclrm_module.F90 +++ b/components/eam/src/physics/clubb/advance_windm_edsclrm_module.F90 @@ -3,6 +3,9 @@ !=============================================================================== module advance_windm_edsclrm_module + !options for turbulent orographic form drag in wind forcing + use phys_control, only: use_od_fd + implicit none private ! Set Default Scope @@ -1571,8 +1574,12 @@ subroutine compute_uv_tndcy( solve_type, fcor, perp_wind_m, perp_wind_g, xm_forc endif else ! implemented in a host model. - - xm_tndcy = 0.0_core_rknd + !use forcing when using turbulent orographic form drag (TOFD) forcing + if (use_od_fd) then + xm_tndcy(1:gr%nz) = xm_forcing(1:gr%nz) + else + xm_tndcy = 0.0_core_rknd + endif endif diff --git a/components/eam/src/physics/crm/dummy_modules/clubb_intr.F90 b/components/eam/src/physics/crm/dummy_modules/clubb_intr.F90 new file mode 100644 index 00000000000..d6729e33f6b --- /dev/null +++ b/components/eam/src/physics/crm/dummy_modules/clubb_intr.F90 @@ -0,0 +1,31 @@ +module clubb_intr +!------------------------------------------------------------------------------- +! Dummy module to override src/physics/cam/clubb_intr.F90 +!------------------------------------------------------------------------------- +use shr_kind_mod, only: r8=>shr_kind_r8 +public :: clubb_implements_cnst +public :: clubb_init_cnst +public :: clubb_readnl +contains +!=============================================================================== +function clubb_implements_cnst(name) + ! Return true if specified constituent is implemented + character(len=*), intent(in) :: name ! constituent name + logical :: clubb_implements_cnst ! return value + clubb_implements_cnst = .false. +end function clubb_implements_cnst +!=============================================================================== +subroutine clubb_init_cnst(name, q, gcid) + ! Initialize the state if clubb_do_adv + character(len=*), intent(in) :: name ! constituent name + real(r8), intent(out) :: q(:,:) ! mass mixing ratio (gcol, plev) + integer, intent(in) :: gcid(:) ! global column id + return +end subroutine clubb_init_cnst +!=============================================================================== +subroutine clubb_readnl(nlfile) + character(len=*), intent(in) :: nlfile ! filepath for file containing namelist input + return +end subroutine clubb_readnl +!=============================================================================== +end module clubb_intr diff --git a/components/eam/src/physics/crm/pam/external b/components/eam/src/physics/crm/pam/external index 1c37054d1ff..6b162a8892e 160000 --- a/components/eam/src/physics/crm/pam/external +++ b/components/eam/src/physics/crm/pam/external @@ -1 +1 @@ -Subproject commit 1c37054d1ff9b160290cc286dcbd3cdc6cd7e7f6 +Subproject commit 6b162a8892e788d0a672d519374a3c82a0de2a5b diff --git a/components/eam/src/physics/crm/pam/pam_driver.cpp b/components/eam/src/physics/crm/pam/pam_driver.cpp index c7038343f08..e3070cce240 100644 --- a/components/eam/src/physics/crm/pam/pam_driver.cpp +++ b/components/eam/src/physics/crm/pam/pam_driver.cpp @@ -20,7 +20,6 @@ // Needed for p3_init #include "p3_functions.hpp" -#include "p3_f90.hpp" #include "pam_debug.h" bool constexpr enable_check_state = false; @@ -203,7 +202,8 @@ extern "C" void pam_driver() { #if defined(P3_CXX) if (is_first_step || is_restart) { auto am_i_root = coupler.get_option("am_i_root"); - scream::p3::p3_init(/*write_tables=*/false, am_i_root); + using P3F = scream::p3::Functions; + P3F::p3_init(/*write_tables=*/false, am_i_root); pam::p3_init_lookup_tables(); // Load P3 lookup table data - avoid re-loading every CRM call } #endif diff --git a/components/eam/src/physics/crm/physpkg.F90 b/components/eam/src/physics/crm/physpkg.F90 index 33fab14066d..6ad53894f4e 100644 --- a/components/eam/src/physics/crm/physpkg.F90 +++ b/components/eam/src/physics/crm/physpkg.F90 @@ -638,7 +638,7 @@ subroutine phys_init( phys_state, phys_tend, pbuf2d, cam_out ) if (co2_transport()) call co2_init() call co2_diags_init(phys_state) - call gw_init() + call gw_init(pbuf2d) call rayleigh_friction_init() diff --git a/components/eam/src/physics/p3/eam/micro_p3.F90 b/components/eam/src/physics/p3/eam/micro_p3.F90 index e668cbaa494..e82c4fa0a96 100644 --- a/components/eam/src/physics/p3/eam/micro_p3.F90 +++ b/components/eam/src/physics/p3/eam/micro_p3.F90 @@ -4350,7 +4350,6 @@ subroutine ice_sedimentation(kts,kte,ktop,kbot,kdir, & dt_left, prt_accum, inv_dz, inv_rho, rho, num_arrays, vs, fluxes, qnr, dt_sub) do k = k_qxbot,k_qxtop,kdir - precip_ice_flux(k+1) = precip_ice_flux(k+1) + flux_qit(k)*dt_sub ! shanyp sflx(k+1) = sflx(k+1) + flux_qit(k)*dt_sub enddo @@ -4363,7 +4362,6 @@ subroutine ice_sedimentation(kts,kte,ktop,kbot,kdir, & bm_incld(:) = bm(:)/cld_frac_i(:) enddo substep_sedi_i - precip_ice_flux(:)=precip_ice_flux(:)*inv_dt sflx(:)=sflx(:)*inv_dt precip_ice_surf = precip_ice_surf + prt_accum*inv_rho_h2o*inv_dt diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 07c3cda7672..cb49f41ddf2 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -97,6 +97,7 @@ option (Kokkos_ENABLE_SERIAL "" ON) set (EAMXX_ENABLE_GPU FALSE CACHE BOOL "") set (CUDA_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there? set (HIP_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there? +set (SYCL_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there? # Determine if this is a Cuda build. if (Kokkos_ENABLE_CUDA) @@ -106,7 +107,7 @@ if (Kokkos_ENABLE_CUDA) set (CUDA_BUILD TRUE CACHE BOOL "" FORCE) #needed for yakl if kokkos vars are not visible there? endif () -# Determine if this is a Cuda build. +# Determine if this is a HIP build. if (Kokkos_ENABLE_HIP) # Add CUDA as a language for CUDA builds enable_language(HIP) @@ -114,6 +115,13 @@ if (Kokkos_ENABLE_HIP) set (HIP_BUILD TRUE CACHE BOOL "" FORCE) #needed for yakl if kokkos vars are not visible there? endif () +# Determine if this is a sycl build. +if (Kokkos_ENABLE_SYCL) + #enable_language(SYCL) + set (EAMXX_ENABLE_GPU TRUE CACHE BOOL "" FORCE) + set (SYCL_BUILD TRUE CACHE BOOL "" FORCE) #needed for yakl if kokkos vars are not visible there? +endif () + if( NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" ) set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -cpp") endif() @@ -178,7 +186,7 @@ elseif(MACH) endif() set(DEFAULT_SMALL_KERNELS FALSE) -if (Kokkos_ENABLE_HIP) +if (Kokkos_ENABLE_HIP OR Kokkos_ENABLE_SYCL) set(DEFAULT_SMALL_KERNELS TRUE) endif() @@ -197,7 +205,7 @@ set(NetCDF_Fortran_PATH ${DEFAULT_NetCDF_Fortran_PATH} CACHE FILEPATH "Path to n set(NetCDF_C_PATH ${DEFAULT_NetCDF_C_PATH} CACHE FILEPATH "Path to netcdf C installation") set(SCREAM_MACHINE ${DEFAULT_SCREAM_MACHINE} CACHE STRING "The CIME/SCREAM name for the current machine") option(SCREAM_MPI_ON_DEVICE "Whether to use device pointers for MPI calls" ON) -option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" ON) +option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" OFF) set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for ALL components that support them") set(SCREAM_P3_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for P3 only") set(SCREAM_SHOC_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for SHOC only") @@ -212,8 +220,8 @@ endif() # #cmakedefine RRTMGP_EXPENSIVE_CHECKS option (SCREAM_RRTMGP_DEBUG "Turn on extra debug checks in RRTMGP" ${SCREAM_DEBUG}) -option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" TRUE) -option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" FALSE) +option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" FALSE) +option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" TRUE) if (SCREAM_RRTMGP_ENABLE_YAKL) add_definitions("-DRRTMGP_ENABLE_YAKL") endif() @@ -453,10 +461,6 @@ if (NOT SCREAM_LIB_ONLY) # for LONG use 100. It is *completely* up to the test to decide what short, medium, and long mean. if (EKAT_ENABLE_COVERAGE OR EKAT_ENABLE_CUDA_MEMCHECK OR EKAT_ENABLE_VALGRIND OR EKAT_ENABLE_COMPUTE_SANITIZER) set (SCREAM_TEST_SIZE_DEFAULT SHORT) - # also set thread_ing=$max_thread - 1, so we test at most 2 threading configurations - if (SCREAM_TEST_MAX_THREADS GREATER 1) - math (EXPR SCREAM_TEST_THREAD_INC ${SCREAM_TEST_MAX_THREADS}-1) - endif() else() set (SCREAM_TEST_SIZE_DEFAULT MEDIUM) endif() @@ -588,9 +592,6 @@ if (NOT DEFINED ENV{SCREAM_FAKE_ONLY}) ${CMAKE_CURRENT_BINARY_DIR}/src/scream_config.h F90_FILE ${CMAKE_CURRENT_BINARY_DIR}/src/scream_config.f) - # Build any tools in the scripts/ dir. - add_subdirectory(scripts) - # Generate scream_config.h and scream_config.f include (EkatUtils) EkatConfigFile(${CMAKE_CURRENT_SOURCE_DIR}/src/scream_config.h.in diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 6d4c36b81bd..f7778cea5fd 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -199,10 +199,10 @@ be lost if SCREAM_HACK_XML is not enabled. 740.0e3 ${DIN_LOC_ROOT}/atm/scream/tables/p3_lookup_table_1.dat-v4.1.1, - ${DIN_LOC_ROOT}/atm/scream/tables/mu_r_table_vals.dat8, - ${DIN_LOC_ROOT}/atm/scream/tables/revap_table_vals.dat8, - ${DIN_LOC_ROOT}/atm/scream/tables/vn_table_vals.dat8, - ${DIN_LOC_ROOT}/atm/scream/tables/vm_table_vals.dat8 + ${DIN_LOC_ROOT}/atm/scream/tables/mu_r_table_vals_v2.dat8, + ${DIN_LOC_ROOT}/atm/scream/tables/revap_table_vals_v2.dat8, + ${DIN_LOC_ROOT}/atm/scream/tables/vn_table_vals_v2.dat8, + ${DIN_LOC_ROOT}/atm/scream/tables/vm_table_vals_v2.dat8 1350.0 2.47 @@ -223,6 +223,10 @@ be lost if SCREAM_HACK_XML is not enabled. 0.304 1.0 true + + false + false + false @@ -230,7 +234,7 @@ be lost if SCREAM_HACK_XML is not enabled. false false 0.001 - 0.04 + 0.08 2.65 0.02 1.0 @@ -427,6 +431,9 @@ be lost if SCREAM_HACK_XML is not enabled. false + + + 1,2 @@ -556,6 +563,7 @@ be lost if SCREAM_HACK_XML is not enabled. mac_aero_mic,rrtmgp + iop_forcing,mac_aero_mic,rrtmgp @@ -717,6 +725,7 @@ be lost if SCREAM_HACK_XML is not enabled. 0 + false @@ -855,6 +864,7 @@ be lost if SCREAM_HACK_XML is not enabled. 6 6 12 + 0 2 none diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/L128/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/L128/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/L128/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/L128/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/L72/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/L72/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/L72/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/L72/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/bfbhash/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/bfbhash/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/bfbhash/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/bfbhash/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/dpxx/arm97/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/dpxx/arm97/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/dpxx/comble/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/dpxx/comble/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/dpxx/dycomsrf01/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/dpxx/dycomsrf01/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/internal_diagnostics_level/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/internal_diagnostics_level/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/aci/shell_commands similarity index 95% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/aci/shell_commands index 492bda019fc..4928859e4e8 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/aci/shell_commands @@ -3,7 +3,7 @@ # MAM4xx adds additionaltracers to the simulation # Increase number of tracers for MAM4xx simulations #------------------------------------------------------ -$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/update_eamxx_num_tracers.sh # Add spa as RRTMG needs spa $CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,spa,rrtmgp" -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aero_microphysics/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/aero_microphysics/shell_commands similarity index 92% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aero_microphysics/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/aero_microphysics/shell_commands index ac1709f7dca..81b3d44b221 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aero_microphysics/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/aero_microphysics/shell_commands @@ -5,7 +5,7 @@ # Increase number of tracers for MAM4xx simulations #------------------------------------------------------ -$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/update_eamxx_num_tracers.sh -b #------------------------------------------------------ # Add microphysics process diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/all_mam4xx_procs/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/all_mam4xx_procs/shell_commands similarity index 96% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/all_mam4xx_procs/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/all_mam4xx_procs/shell_commands index 8ef80aef633..366f9afdd28 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/all_mam4xx_procs/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/all_mam4xx_procs/shell_commands @@ -2,7 +2,7 @@ # MAM4xx adds additional tracers to the simulation # Increase the number of tracers for MAM4xx simulations #------------------------------------------------------ -$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/update_eamxx_num_tracers.sh #modify initial condition file to get aerosol species ICs $CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/drydep/shell_commands similarity index 92% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/drydep/shell_commands index ea2d56ab53f..02b4594dbc9 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/drydep/shell_commands @@ -5,7 +5,7 @@ # Increase number of tracers for MAM4xx simulations #------------------------------------------------------ -$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/update_eamxx_num_tracers.sh -b #------------------------------------------------------ # Add drydep process diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/optics/shell_commands similarity index 90% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/optics/shell_commands index 9bbcfaea898..b62620e7fe0 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/optics/shell_commands @@ -2,7 +2,7 @@ # MAM4xx adds additionaltracers to the simulation # Increase number of tracers for MAM4xx simulations #------------------------------------------------------ -$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/update_eamxx_num_tracers.sh $CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,mam4_optics,rrtmgp" -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/remap_emiss_ne4_ne30/shell_commands similarity index 98% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/remap_emiss_ne4_ne30/shell_commands index b2d0286b870..9d0bbd39d97 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/remap_emiss_ne4_ne30/shell_commands @@ -5,7 +5,7 @@ # Increase number of tracers for MAM4xx simulations #------------------------------------------------------ -$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/update_eamxx_num_tracers.sh -b #------------------------------------------------------ # Add aerosol microphysics process, force ne4pg2 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands similarity index 92% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands index 500d0bf660a..249d7c2371e 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands @@ -5,7 +5,7 @@ # Increase number of tracers for MAM4xx simulations #------------------------------------------------------ -$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/update_eamxx_num_tracers.sh -b #------------------------------------------------------ # Add the processes diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/update_eamxx_num_tracers.sh similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/update_eamxx_num_tracers.sh diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/wetscav/shell_commands similarity index 92% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/wetscav/shell_commands index 2cdf303a582..d8436d38450 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/wetscav/shell_commands @@ -4,7 +4,7 @@ # Increase number of tracers for MAM4xx simulations #------------------------------------------------------ -$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/mam4xx/update_eamxx_num_tracers.sh -b #------------------------------------------------------ # Add wetscav process diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/README b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/README similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/README rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/README diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/diags/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/diags/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/hremap_to_ne4/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/hremap_to_ne4/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/hremap_to_ne4/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/hremap_to_ne4/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/phys/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/phys/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/phys_dyn/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/phys_dyn/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/1/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/1/shell_commands similarity index 92% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/1/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/1/shell_commands index 74bd2a4d021..216f5c73628 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/1/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/1/shell_commands @@ -1,7 +1,7 @@ # This preset uses the three output streams (phys_dyn, phys, and diags) # It does not add remap, and uses INSTANT output -SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output # Add the three streams . $SCRIPTS_DIR/phys/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/2/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/2/shell_commands similarity index 93% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/2/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/2/shell_commands index aea7feb5bbd..24b71ba0aa9 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/2/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/2/shell_commands @@ -1,7 +1,7 @@ # This preset uses the three output streams (phys_dyn, phys, and diags) # It does not add remap, and uses AVERAGE output -SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output # Add the three streams . $SCRIPTS_DIR/phys/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/3/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/3/shell_commands similarity index 93% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/3/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/3/shell_commands index da1d02dc5de..5d27192d525 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/3/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/3/shell_commands @@ -1,7 +1,7 @@ # This preset uses the three output streams (phys, and diags) # It adds horiz remap, and uses INSTANT output -SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output # Add the phys/diags streams (cannot add phys_dyn, b/c we use horiz remap) . $SCRIPTS_DIR/phys/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/4/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/4/shell_commands similarity index 94% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/4/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/4/shell_commands index 521ada7f508..6746cf1d46e 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/4/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/4/shell_commands @@ -1,7 +1,7 @@ # This preset uses the three output streams (phys, and diags) # It adds horizontal remap, and uses AVERAGE output -SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output # Add the phys/diags streams (cannot add phys_dyn, b/c we use horiz remap) . $SCRIPTS_DIR/phys/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/5/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/5/shell_commands similarity index 94% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/5/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/5/shell_commands index bd60a0472a8..3c63c1abd8f 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/5/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/5/shell_commands @@ -1,7 +1,7 @@ # This preset uses the three output streams (phys_dyn, phys, and diags) # It adds vertical remap, and uses AVERAGE output -SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output # Add the phys/dyn/diags streams . $SCRIPTS_DIR/phys/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/6/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/6/shell_commands similarity index 94% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/6/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/6/shell_commands index 937d1646851..442f8b9df8f 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/6/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/preset/6/shell_commands @@ -1,7 +1,7 @@ # This preset uses the three output streams (phys, and diags) # It adds horizontal and remap, and uses AVERAGE output -SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output # Add the phys/diags streams (cannot add phys_dyn, b/c we use horiz remap) . $SCRIPTS_DIR/phys/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/set_avg/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/set_avg/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/set_avg/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/set_avg/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/vremap/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/vremap/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/vremap/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/output/vremap/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/perf_test/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/perf_test/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/perf_test/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/perf_test/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/rad_frequency_2/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/rad_frequency_2/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/rad_frequency_2/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/rad_frequency_2/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/scream_example_testmod_atmchange/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/scream_example_testmod_atmchange/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/scream_example_testmod_atmchange/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/scream_example_testmod_atmchange/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/scream_example_testmod_cmake/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/scream_example_testmod_cmake/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/scream_example_testmod_cmake/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/scream_example_testmod_cmake/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/sl_nsubstep2/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/sl_nsubstep2/shell_commands new file mode 100644 index 00000000000..71cdd9b691c --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/sl_nsubstep2/shell_commands @@ -0,0 +1,3 @@ +ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange + +$ATMCHANGE semi_lagrange_trajectory_nsubstep=2 -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/small_kernels/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/small_kernels/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/small_kernels_p3/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/small_kernels_p3/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/small_kernels_shoc/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/small_kernels_shoc/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/spa_remap/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/spa_remap/shell_commands similarity index 100% rename from components/eamxx/cime_config/testdefs/testmods_dirs/scream/spa_remap/shell_commands rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/spa_remap/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/sunspot_run/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/sunspot_run/shell_commands new file mode 100644 index 00000000000..6ca33b486f9 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/sunspot_run/shell_commands @@ -0,0 +1,6 @@ + +$CIMEROOT/../components/eamxx/scripts/atmchange transport_alg=0 -b +$CIMEROOT/../components/eamxx/scripts/atmchange hypervis_subcycle_q=1 -b +$CIMEROOT/../components/eamxx/scripts/atmchange dt_tracer_factor=1 -b +$CIMEROOT/../components/eamxx/scripts/atmchange tstep_type=9 -b +$CIMEROOT/../components/eamxx/scripts/atmchange theta_hydrostatic_mode=False -b diff --git a/components/eamxx/cime_config/tests/eamxx_default_files.py b/components/eamxx/cime_config/tests/eamxx_default_files.py deleted file mode 100644 index b39d2d5c155..00000000000 --- a/components/eamxx/cime_config/tests/eamxx_default_files.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python3 - -import os -import http -import pathlib -import unittest -import urllib.request -import xml.etree.ElementTree as ET - - -class testNamelistDefaultsScream(unittest.TestCase): - def setUp(self): - """ - Set up the environment for the test by setting the DIN_LOC_ROOT - environment variable. Parse the 'namelist_defaults_scream.xml' - file and extract the files of interest based on the DIN_LOC_ROOT - variable or the array(file) type. Assign the extracted files - to the 'my_files' attribute of the test instance. - """ - - os.environ["DIN_LOC_ROOT"] = "https://web.lcrc.anl.gov/public/e3sm/inputdata/" - - scream_defaults_path = pathlib.Path(__file__) - tree = ET.parse(f"{scream_defaults_path.parent.parent}/namelist_defaults_scream.xml") - root = tree.getroot() - - files_of_interest = [ - child.text for child in root.findall(".//") - if child.text and child.text.startswith("${DIN_LOC_ROOT}") - ] - - more_files_of_interest = [ - child.text for child in root.findall(".//") - if child.text and "type" in child.attrib.keys() and child.attrib["type"]=="array(file)" - ] - - files_of_interest.extend( - text.strip() for text_list in more_files_of_interest for text in text_list.split(",") - if text.strip().startswith("${DIN_LOC_ROOT}") - ) - - self.my_files = [ - file.replace("${DIN_LOC_ROOT}/", "") - for file in files_of_interest - ] - - self.my_lines = [] - with open( - f"{scream_defaults_path.parent.parent}/namelist_defaults_scream.xml", - "r" - ) as the_file: - for a_line in the_file: - self.my_lines.append(a_line) - - def test_ascii_lines(self): - """ - Test that all lines are ASCII - """ - - for i_line, a_line in enumerate(self.my_lines): - with self.subTest(i_line=i_line): - self.assertTrue( - a_line.isascii(), - msg=f"\nERROR! This line is not ASCII!\n{a_line}" - ) - - def test_opening_files(self): - """ - Test the opening of files from the inputdata server. - """ - - for i_file in range(len(self.my_files)): - with self.subTest(i_file=i_file): - try: - request_return = urllib.request.urlopen( - f"{os.environ['DIN_LOC_ROOT']}{self.my_files[i_file]}" - ) - self.assertIsInstance(request_return, http.client.HTTPResponse) - except urllib.error.HTTPError: - file_name = f"{os.environ['DIN_LOC_ROOT']}{self.my_files[i_file]}" - self.assertTrue( - False, - msg=f"\nERROR! This file doesn't exist!\n{file_name}" - ) - - def test_expected_fail(self): - """ - Test an expected failure by manipulating the file name. - """ - - with self.assertRaises(urllib.error.HTTPError): - some_phony_file = f"{self.my_files[5][:-5]}some_phony_file.nc" - urllib.request.urlopen( - f"{os.environ['DIN_LOC_ROOT']}{some_phony_file}" - ) - - -if __name__ == '__main__': - unittest.main() diff --git a/components/eamxx/cmake/BuildCprnc.cmake b/components/eamxx/cmake/BuildCprnc.cmake index 287956c5a9d..05db9b60690 100644 --- a/components/eamxx/cmake/BuildCprnc.cmake +++ b/components/eamxx/cmake/BuildCprnc.cmake @@ -18,7 +18,9 @@ macro(BuildCprnc) configure_file (${SCREAM_BASE_DIR}/cmake/CprncTest.cmake.in ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake @ONLY) else() - message(WARNING "Path ${CCSM_CPRNC} does not exist, so we will try to build it") + if (NOT "${CCSM_CPRNC}" STREQUAL "") + message(WARNING "Path ${CCSM_CPRNC} does not exist, so we will try to build it") + endif() # Make sure this is built only once if (NOT TARGET cprnc) if (SCREAM_CIME_BUILD) diff --git a/components/eamxx/cmake/machine-files/aurora.cmake b/components/eamxx/cmake/machine-files/aurora.cmake index 59157285bab..e6a32fc72c0 100644 --- a/components/eamxx/cmake/machine-files/aurora.cmake +++ b/components/eamxx/cmake/machine-files/aurora.cmake @@ -1,14 +1,29 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() -include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake) +include (${EKAT_MACH_FILES_PATH}/kokkos/intel-pvc.cmake) include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) - set(EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "" FORCE) set(EKAT_MPI_NP_FLAG "-np" CACHE STRING "" FORCE) set(EKAT_MPI_EXTRA_ARGS "--label --cpu-bind depth -envall" CACHE STRING "") set(EKAT_MPI_THREAD_FLAG "-d" CACHE STRING "") -set(NETCDF_C_PATH "$ENV{NETCDF_C_PATH}") -set(NETCDF_FORTRAN_PATH "$ENV{NETCDF_FORTRAN_PATH}") -set(PNETCDF_PATH "$ENV{PNETCDF_PATH}") +SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"") + +set(CMAKE_CXX_FLAGS " -\-intel -Xclang -fsycl-allow-virtual-functions -mlong-double-64 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS "-O3 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_EXE_LINKER_FLAGS " -lifcore -\-intel -Xclang -fsycl-allow-virtual-functions -lsycl -mlong-double-64 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "" FORCE) + +#this is needed for cime builds! +set(NETCDF_PATH "/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002") +set(NETCDF_DIR "/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002") +set(NETCDF_C_PATH "/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002") +set(NETCDF_C "/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002") +#this one is for rrtmgp +set(NetCDF_C_PATH "/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002" CACHE STRING "") +set(NETCDF_FORTRAN_PATH "/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002") +set(PNETCDF_PATH "/lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002") + + diff --git a/components/eamxx/cmake/machine-files/auroracpu.cmake b/components/eamxx/cmake/machine-files/auroracpu.cmake new file mode 100644 index 00000000000..d620dcb3a0d --- /dev/null +++ b/components/eamxx/cmake/machine-files/auroracpu.cmake @@ -0,0 +1,31 @@ +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) +common_setup() + +include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake) +# kokkos sycl is on in the above file +#include (${EKAT_MACH_FILES_PATH}/kokkos/sycl.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) + +#AB flags from ekat +# -fsycl -fsycl-unnamed-lambda -sycl-std=2020 -qopenmp-simd -Wsycl-strict -fsycl-device-code-split=per_kernel + +#SET(MPICH_DIR "/soft/restricted/CNDA/updates/mpich/52.2/mpich-ofi-all-icc-default-pmix-gpu-drop52/" CACHE STRING "") + +set(CMAKE_CXX_FLAGS " -\-intel -Xclang -fsycl-allow-virtual-functions -mlong-double-64 -O3 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS "-O3 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_EXE_LINKER_FLAGS " -lifcore -\-intel -Xclang -mlong-double-64 -DNDEBUG -fortlib" CACHE STRING "" FORCE) +#set(CMAKE_EXE_LINKER_FLAGS " -Wl,-\-defsym,main=MAIN_\_ -lifcore -\-intel -Xclang -fsycl-allow-virtual-functions -lsycl -mlong-double-64 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib -L${MPICH_DIR}/lib" CACHE STRING "" FORCE) + +#this is needed for cime builds! +set(NETCDF_PATH "/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2023.05.15.007") +set(NETCDF_DIR "/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2023.05.15.007") +set(NETCDF_C_PATH "/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2023.05.15.007") +set(NETCDF_C "/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2023.05.15.007") +#this one is for rrtmgp +set(NetCDF_C_PATH "/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2023.05.15.007" CACHE STRING "") +set(NETCDF_FORTRAN_PATH "/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2023.05.15.007") +set(PNETCDF_PATH "/lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2023.05.15.007") + + + diff --git a/components/eamxx/cmake/machine-files/ghci-snl-cuda.cmake b/components/eamxx/cmake/machine-files/ghci-snl-cuda.cmake index 72a352f09d7..83d1d20c627 100644 --- a/components/eamxx/cmake/machine-files/ghci-snl-cuda.cmake +++ b/components/eamxx/cmake/machine-files/ghci-snl-cuda.cmake @@ -12,3 +12,6 @@ set(EKAT_MPI_NP_FLAG "-n" CACHE STRING "The mpirun flag for designating the tota # TODO: rebuild cuda image with cuda-aware MPI, so we can set this to ON option(SCREAM_MPI_ON_DEVICE "Whether to use device pointers for MPI calls" OFF) + +# Currently, we have 2 GPUs/node on Blake, and we run a SINGLE build per node, so we can fit 2 ranks there +set(SCREAM_TEST_MAX_RANKS 2 CACHE STRING "Upper limit on ranks for mpi tests") diff --git a/components/eamxx/cmake/machine-files/ghci-snl.cmake b/components/eamxx/cmake/machine-files/ghci-snl.cmake index ac4db2a39f1..9782036d92c 100644 --- a/components/eamxx/cmake/machine-files/ghci-snl.cmake +++ b/components/eamxx/cmake/machine-files/ghci-snl.cmake @@ -17,3 +17,5 @@ option (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES "" ON) # Needed by EkatCreateUnitTest set (EKAT_MPIRUN_EXE "mpirun" CACHE STRING "") set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "") + +set(EKAT_VALGRIND_SUPPRESSION_FILE "/projects/e3sm/baselines/scream/ghci-snl-cpu/eamxx-valgrind.supp" CACHE FILEPATH "Use this valgrind suppression file if valgrind is enabled.") diff --git a/components/eamxx/cmake/machine-files/sunspot-gen.cmake b/components/eamxx/cmake/machine-files/sunspot-gen.cmake new file mode 100644 index 00000000000..3e33ac7b461 --- /dev/null +++ b/components/eamxx/cmake/machine-files/sunspot-gen.cmake @@ -0,0 +1,31 @@ +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) +common_setup() + +include (${EKAT_MACH_FILES_PATH}/kokkos/intel-gen.cmake) +include (${EKAT_MACH_FILES_PATH}/kokkos/sycl.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) + +#AB flags from ekat +# -fsycl -fsycl-unnamed-lambda -sycl-std=2020 -qopenmp-simd -Wsycl-strict -fsycl-device-code-split=per_kernel +SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64") + +#SET(MPICH_DIR "/soft/restricted/CNDA/updates/mpich/52.2/mpich-ofi-all-icc-default-pmix-gpu-drop52/" CACHE STRING "") + +set(CMAKE_CXX_FLAGS " -\-intel -Xclang -fsycl-allow-virtual-functions -mlong-double-64 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS "-O3 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_EXE_LINKER_FLAGS " -lifcore -\-intel -Xclang -fsycl-allow-virtual-functions -lsycl -mlong-double-64 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "" FORCE) +#set(CMAKE_EXE_LINKER_FLAGS " -Wl,-\-defsym,main=MAIN_\_ -lifcore -\-intel -Xclang -fsycl-allow-virtual-functions -lsycl -mlong-double-64 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib -L${MPICH_DIR}/lib" CACHE STRING "" FORCE) + + + +set(NETCDF_PATH "$ENV{NETCDF_PATH}") +set(NETCDF_C_PATH "$ENV{NETCDF_PATH}") +#this one is for rrtmgp +set(NetCDF_C_PATH "$ENV{NETCDF_PATH}" CACHE STRING "") +set(NETCDF_FORTRAN_PATH "$ENV{NETCDF_PATH}") +set(PNETCDF_PATH "$ENV{PNETCDF_PATH}") + + + diff --git a/components/eamxx/cmake/machine-files/sunspot-pvc.cmake b/components/eamxx/cmake/machine-files/sunspot-pvc.cmake new file mode 100644 index 00000000000..53740d87e0e --- /dev/null +++ b/components/eamxx/cmake/machine-files/sunspot-pvc.cmake @@ -0,0 +1,28 @@ +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) +common_setup() + +include (${EKAT_MACH_FILES_PATH}/kokkos/intel-pvc.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) +set(EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "" FORCE) +set(EKAT_MPI_NP_FLAG "-np" CACHE STRING "" FORCE) +set(EKAT_MPI_EXTRA_ARGS "--label --cpu-bind depth -envall" CACHE STRING "") +set(EKAT_MPI_THREAD_FLAG "-d" CACHE STRING "") + +SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"") + +set(CMAKE_CXX_FLAGS " -\-intel -Xclang -fsycl-allow-virtual-functions -mlong-double-64 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS "-O3 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_EXE_LINKER_FLAGS " -lifcore -\-intel -Xclang -fsycl-allow-virtual-functions -lsycl -mlong-double-64 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "" FORCE) + +set(NETCDF_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") +set(NETCDF_DIR "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") +set(NETCDF_C_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") +set(NETCDF_C "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") +#this one is for rrtmgp +set(NetCDF_C_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") +set(NETCDF_FORTRAN_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") +set(PNETCDF_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/pnetcdf" CACHE STRING "") + + diff --git a/components/eamxx/cmake/machine-files/sunspotcpu.cmake b/components/eamxx/cmake/machine-files/sunspotcpu.cmake new file mode 100644 index 00000000000..ff7773daae1 --- /dev/null +++ b/components/eamxx/cmake/machine-files/sunspotcpu.cmake @@ -0,0 +1,34 @@ +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) +common_setup() + +include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake) +# kokkos sycl is on in the above file +#include (${EKAT_MACH_FILES_PATH}/kokkos/sycl.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) + +#AB flags from ekat +# -fsycl -fsycl-unnamed-lambda -sycl-std=2020 -qopenmp-simd -Wsycl-strict -fsycl-device-code-split=per_kernel + +#SET(MPICH_DIR "/soft/restricted/CNDA/updates/mpich/52.2/mpich-ofi-all-icc-default-pmix-gpu-drop52/" CACHE STRING "") + +set(CMAKE_CXX_FLAGS " -\-intel -Xclang -fsycl-allow-virtual-functions -mlong-double-64 -O3 -DNDEBUG -gline-tables-only -g" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g -gline-tables-only" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS "-O3 -DNDEBUG -gline-tables-only -g" CACHE STRING "" FORCE) +set(CMAKE_EXE_LINKER_FLAGS " -lifcore -\-intel -Xclang -mlong-double-64 -DNDEBUG -fortlib" CACHE STRING "" FORCE) +#set(CMAKE_EXE_LINKER_FLAGS " -Wl,-\-defsym,main=MAIN_\_ -lifcore -\-intel -Xclang -fsycl-allow-virtual-functions -lsycl -mlong-double-64 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib -L${MPICH_DIR}/lib" CACHE STRING "" FORCE) + + + +set(NETCDF_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") +set(NETCDF_DIR "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") +set(NETCDF_C_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") +set(NETCDF_C "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") +#this one is for rrtmgp +set(NetCDF_C_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") +set(NETCDF_FORTRAN_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") +set(PNETCDF_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/pnetcdf" CACHE STRING "") + + +set(PNETCDF_DIR "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/pnetcdf" CACHE STRING "") + + diff --git a/components/eamxx/docs/developer/index.md b/components/eamxx/docs/developer/index.md index 69673b12ebd..7216d6057cb 100644 --- a/components/eamxx/docs/developer/index.md +++ b/components/eamxx/docs/developer/index.md @@ -1 +1,16 @@ -# SCREAM Developer Guide +# EAMxx Developer Guide + +### [Installation](../common/installation.md) +### [Style Guide](style_guide.md) +### [Kokkos and EKAT](kokkos_ekat.md) +### [Source Tree](source_tree.md) +### Important Data Structures + * [Fields](field.md) + * [Grids and Remappers](grid.md) + * [Atmosphere Processes](processes.md) + * [Managers](managers.md) +### [I/O](io.md) +### Testing + * [Standalone](standalone_testing.md) + * [Full Model](cime_testing.md) + * [CI and Nightly Testing](ci_nightly.md) diff --git a/components/eamxx/docs/developer/kokkos_ekat.md b/components/eamxx/docs/developer/kokkos_ekat.md index a736384b2c6..6ebf23ba1aa 100644 --- a/components/eamxx/docs/developer/kokkos_ekat.md +++ b/components/eamxx/docs/developer/kokkos_ekat.md @@ -57,8 +57,8 @@ where run-time dimensions). E.g., a 2D view of doubles will have `DataType = double**`. There is also an ability to define compile-time dimensions by using `[]`, see - [Kokkos wiki section on views] - (). + [Kokkos wiki section on views] + (_f)" + lambda *x : "The f90 to cxx function declaration(_host)" )), ("cxx_f2c_bind_impl" , ( - lambda phys, sub, gb: f"{phys}_functions_f90.cpp", + lambda phys, sub, gb: f"tests/infra/{phys}_test_data.cpp", lambda phys, sub, gb: expect_exists(phys, sub, gb, "cxx_f2c_bind_impl"), lambda phys, sub, gb: get_namespace_close_regex(phys), # insert at end of namespace - lambda phys, sub, gb: get_cxx_function_begin_regex(sub + "_f"), # cxx_f + lambda phys, sub, gb: get_cxx_function_begin_regex(sub + "_host"), # cxx_f lambda phys, sub, gb: get_cxx_close_block_regex(at_line_start=True), # terminating } - lambda *x : "The f90 to cxx function implementation(_f)" + lambda *x : "The f90 to cxx function implementation(_host)" )), ("cxx_func_decl", ( @@ -455,6 +455,12 @@ def get_cxx_struct_begin_regex(struct): struct_regex_str = fr"^\s*struct\s+{struct}([\W]|$)" return re.compile(struct_regex_str) +############################################################################### +def get_plain_comment_regex(comment): +############################################################################### + comment_regex_str = fr"^\s*//\s*{comment}" + return re.compile(comment_regex_str) + ############################################################################### def get_data_struct_name(sub): ############################################################################### @@ -1169,6 +1175,21 @@ def split_by_type(arg_data): return reals, ints, logicals +############################################################################### +def split_by_scalar_vs_view(arg_data): +############################################################################### + """ + Take arg data and split into two lists of names based on scalar/not-scalar: [scalars] [non-scalars] + """ + scalars, non_scalars = [], [] + for name, _, _, dims in arg_data: + if dims is not None: + non_scalars.append(name) + else: + scalars.append(name) + + return scalars, non_scalars + ############################################################################### def gen_cxx_data_args(physics, arg_data): ############################################################################### @@ -1441,6 +1462,30 @@ def check_existing_piece(lines, begin_regex, end_regex): return None if begin_idx is None else (begin_idx, end_idx+1) +############################################################################### +def get_data_by_name(arg_data, arg_name, data_idx): +############################################################################### + for name, a, b, c in arg_data: + if name == arg_name: + return [name, a, b, c][data_idx] + + expect(False, f"Name {arg_name} not found") + +############################################################################### +def get_rank_map(arg_data, arg_names): +############################################################################### + # Create map of rank -> [args] + rank_map = {} + for arg in arg_names: + dims = get_data_by_name(arg_data, arg, ARG_DIMS) + rank = len(dims) + if rank in rank_map: + rank_map[rank].append(arg) + else: + rank_map[rank] = [arg] + + return rank_map + # # Main classes # @@ -1505,10 +1550,10 @@ def _get_db(self, phys): db = parse_origin(origin_file.open(encoding="utf-8").read(), self._subs) self._db[phys].update(db) if self._verbose: - print("For physics {}, found:") + print(f"For physics {phys}, found:") for sub in self._subs: if sub in db: - print(" For subroutine {}, found args:") + print(f" For subroutine {sub}, found args:") for name, argtype, intent, dims in db[sub]: print(" name:{} type:{} intent:{} dims:({})".\ format(name, argtype, intent, ",".join(dims) if dims else "scalar")) @@ -1729,7 +1774,7 @@ def gen_cxx_f2c_bind_decl(self, phys, sub, force_arg_data=None): arg_data = force_arg_data if force_arg_data else self._get_arg_data(phys, sub) arg_decls = gen_arg_cxx_decls(arg_data) - return f"void {sub}_f({', '.join(arg_decls)});" + return f"void {sub}_host({', '.join(arg_decls)});" ########################################################################### def gen_cxx_f2c_bind_impl(self, phys, sub, force_arg_data=None): @@ -1809,8 +1854,166 @@ def gen_cxx_f2c_bind_impl(self, phys, sub, force_arg_data=None): impl = "" if has_arrays(arg_data): - # TODO - impl += " // TODO" + # + # Steps: + # 1) Set up typedefs + # 2) Sync to device + # 3) Unpack view array + # 4) Get nk_pack and policy + # 5) Get subviews + # 6) Call fn + # 7) Sync back to host + # + inputs, inouts, outputs = split_by_intent(arg_data) + reals, ints, bools = split_by_type(arg_data) + scalars, views = split_by_scalar_vs_view(arg_data) + all_inputs = inputs + inouts + all_outputs = inouts + outputs + + vreals = list(sorted(set(reals) & set(views))) + vints = list(sorted(set(ints) & set(views))) + vbools = list(sorted(set(bools) & set(views))) + + sreals = list(sorted(set(reals) & set(scalars))) + sints = list(sorted(set(ints) & set(scalars))) + sbools = list(sorted(set(bools) & set(scalars))) + + ivreals = list(sorted(set(vreals) & set(all_inputs))) + ivints = list(sorted(set(vints) & set(all_inputs))) + ivbools = list(sorted(set(vbools) & set(all_inputs))) + + ovreals = list(sorted(set(vreals) & set(all_outputs))) + ovints = list(sorted(set(vints) & set(all_outputs))) + ovbools = list(sorted(set(vbools) & set(all_outputs))) + + isreals = list(sorted(set(sreals) & set(all_inputs))) + isints = list(sorted(set(sints) & set(all_inputs))) + isbools = list(sorted(set(sbools) & set(all_inputs))) + + osreals = list(sorted(set(sreals) & set(all_outputs))) + osints = list(sorted(set(sints) & set(all_outputs))) + osbools = list(sorted(set(sbools) & set(all_outputs))) + + # + # 1) Set up typedefs + # + + # set up basics + impl += "#if 0\n" # There's no way to guarantee this code compiles + impl += " using SHF = Functions;\n" + impl += " using Scalar = typename SHF::Scalar;\n" + impl += " using Spack = typename SHF::Spack;\n" + impl += " using KT = typename SHF::KT;\n" + impl += " using ExeSpace = typename KT::ExeSpace;\n" + impl += " using MemberType = typename SHF::MemberType;\n\n" + + prefix_list = ["", "i", "b"] + type_list = ["Real", "Int", "bool"] + ktype_list = ["Spack", "Int", "bool"] + + # make necessary view types. Anything that's an array needs a view type + for view_group, prefix_char, typename in zip([vreals, vints, vbools], prefix_list, type_list): + if view_group: + rank_map = get_rank_map(arg_data, view_group) + for rank in rank_map: + if typename == "Real" and rank > 1: + # Probably this should be packed data + impl += f" using {prefix_char}view_{rank}d = typename SHF::view_{rank}d;\n" + else: + impl += f" using {prefix_char}view_{rank}d = typename SHF::view_{rank}d<{typename}>;\n" + + impl += "\n" + + # + # 2) Sync to device. Do ALL views, not just inputs + # + + for input_group, prefix_char, typename in zip([vreals, vints, vbools], prefix_list, type_list): + if input_group: + rank_map = get_rank_map(arg_data, input_group) + + for rank, arg_list in rank_map.items(): + impl += f" static constexpr Int {prefix_char}num_arrays_{rank} = {len(arg_list)};\n" + impl += f" std::vector<{prefix_char}view_{rank}d> {prefix_char}temp_d_{rank}({prefix_char}num_arrays_{rank});\n" + for rank_itr in range(rank): + dims = [get_data_by_name(arg_data, arg_name, ARG_DIMS)[rank_itr] for arg_name in arg_list] + impl += f" std::vector {prefix_char}dim_{rank}_{rank_itr}_sizes = {{{', '.join(dims)}}};\n" + dim_vectors = [f"{prefix_char}dim_{rank}_{rank_itr}_sizes" for rank_itr in range(rank)] + funcname = "ekat::host_to_device" if (typename == "Real" and rank > 1) else "ScreamDeepCopy::copy_to_device" + impl += f" {funcname}({{{', '.join(arg_list)}}}, {', '.join(dim_vectors)}, {prefix_char}temp_d_{rank});\n\n" + + # + # 3) Unpack view array + # + + for input_group, prefix_char, typename in zip([vreals, vints, vbools], prefix_list, type_list): + if input_group: + rank_map = get_rank_map(arg_data, input_group) + + for rank, arg_list in rank_map.items(): + impl += f" {prefix_char}view_{rank}d\n" + for idx, input_item in enumerate(arg_list): + impl += f" {input_item}_d({prefix_char}temp_d_{rank}[{idx}]){';' if idx == len(arg_list) - 1 else ','}\n" + impl += "\n" + + + # + # 4) Get nk_pack and policy, launch kernel + # + impl += " const Int nk_pack = ekat::npack(nlev);\n" + impl += " const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(shcol, nk_pack);\n" + impl += " Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) {\n" + impl += " const Int i = team.league_rank();\n\n" + + # + # 5) Get subviews + # + for view_group, prefix_char, typename in zip([vreals, vints, vbools], prefix_list, type_list): + if view_group: + for view_arg in view_group: + dims = get_data_by_name(arg_data, view_arg, ARG_DIMS) + if "shcol" in dims: + if len(dims) == 1: + impl += f" const Scalar {view_arg}_s = {view_arg}_d(i);\n" + else: + impl += f" const auto {view_arg}_s = ekat::subview({view_arg}_d, i);\n" + + impl += "\n" + + # + # 6) Call fn + # + kernel_arg_names = [] + for arg_name in arg_names: + if arg_name in views: + if "shcol" in dims: + kernel_arg_names.append(f"{arg_name}_s") + else: + kernel_arg_names.append(f"{arg_name}_d") + else: + kernel_arg_names.append(arg_name) + + impl += f" SHF::{sub}({', '.join(kernel_arg_names)});\n" + impl += " });\n" + + # + # 7) Sync back to host + # + for output_group, prefix_char, typename in zip([ovreals, ovints, ovbools], prefix_list, type_list): + if output_group: + rank_map = get_rank_map(arg_data, output_group) + + for rank, arg_list in rank_map.items(): + impl += f" std::vector<{prefix_char}view_{rank}d> {prefix_char}tempout_d_{rank}({prefix_char}num_arrays_{rank});\n" + for rank_itr in range(rank): + dims = [get_data_by_name(arg_data, arg_name, ARG_DIMS)[rank_itr] for arg_name in arg_list] + impl += f" std::vector {prefix_char}dim_{rank}_{rank_itr}_out_sizes = {{{', '.join(dims)}}};\n" + dim_vectors = [f"{prefix_char}dim_{rank}_{rank_itr}_out_sizes" for rank_itr in range(rank)] + funcname = "ekat::device_to_host" if (typename == "Real" and rank > 1) else "ScreamDeepCopy::copy_to_host" + impl += f" {funcname}({{{', '.join(arg_list)}}}, {', '.join(dim_vectors)}, {prefix_char}tempout_d_{rank});\n\n" + + impl += "#endif\n" + else: inputs, inouts, outputs = split_by_intent(arg_data) reals, ints, logicals = split_by_type(arg_data) diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index c9c9f973022..dbb9363381c 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -124,7 +124,7 @@ def setup_pm(cls,partition): cls.batch += "--time 00:30:00 --nodes=1 -q debug" else: cls.batch += "--time 02:00:00 --nodes=4 --gpus-per-node=4 --gpu-bind=none --exclusive -q regular" - + cls.baselines_dir = f"/global/cfs/cdirs/e3sm/baselines/{compiler}/scream/{cls.name}" ############################################################################### @@ -214,6 +214,7 @@ class GHCISNLCPU(Machine): def setup(cls): super().setup_base("ghci-snl-cpu") cls.baselines_dir = "/projects/e3sm/baselines/scream/ghci-snl-cpu" + cls.env_setup = ["export GATOR_INITIAL_MB=4000MB"] ############################################################################### class GHCISNLCuda(Machine): diff --git a/components/eamxx/scripts/CMakeLists.txt b/components/eamxx/scripts/query-cf-database/CMakeLists.txt similarity index 67% rename from components/eamxx/scripts/CMakeLists.txt rename to components/eamxx/scripts/query-cf-database/CMakeLists.txt index 377452229b9..91200760602 100644 --- a/components/eamxx/scripts/CMakeLists.txt +++ b/components/eamxx/scripts/query-cf-database/CMakeLists.txt @@ -1,8 +1,8 @@ -# Generate the source file for the CF validator and build it. +# Build the CF validator tool -set (CF_STANDARD_NAME_FILE "${PROJECT_SOURCE_DIR}/data/cf-standard-name-table.yaml" +set (CF_STANDARD_NAME_FILE "${CMAKE_CURRENT_SOURCE_DIR}/cf-standard-name-table.yaml" CACHE STRING "Location of the cf standard name yaml table") -set (CF_SCREAM_NAME_FILE "${PROJECT_SOURCE_DIR}/data/cf-scream-name-table.yaml" +set (CF_SCREAM_NAME_FILE "${CMAKE_CURRENT_SOURCE_DIR}/cf-scream-name-table.yaml" CACHE STRING "Location of the scream-specific cf name yaml table") add_executable(query-cf-database query-cf-database.cpp) @@ -10,5 +10,6 @@ target_compile_definitions(query-cf-database PUBLIC CF_STANDARD_NAME_FILE=${CF_STANDARD_NAME_FILE} CF_SCREAM_NAME_FILE=${CF_SCREAM_NAME_FILE}) +find_package (ekat HINTS ${EKAT_ROOT}) find_package (yaml-cpp HINTS ${YAML_CPP_ROOT}) target_link_libraries(query-cf-database ekat yaml-cpp) diff --git a/components/eamxx/data/cf-scream-name-table.yaml b/components/eamxx/scripts/query-cf-database/cf-scream-name-table.yaml similarity index 100% rename from components/eamxx/data/cf-scream-name-table.yaml rename to components/eamxx/scripts/query-cf-database/cf-scream-name-table.yaml diff --git a/components/eamxx/data/cf-standard-name-table.yaml b/components/eamxx/scripts/query-cf-database/cf-standard-name-table.yaml similarity index 100% rename from components/eamxx/data/cf-standard-name-table.yaml rename to components/eamxx/scripts/query-cf-database/cf-standard-name-table.yaml diff --git a/components/eamxx/scripts/cf-xml-to-yaml b/components/eamxx/scripts/query-cf-database/cf-xml-to-yaml similarity index 100% rename from components/eamxx/scripts/cf-xml-to-yaml rename to components/eamxx/scripts/query-cf-database/cf-xml-to-yaml diff --git a/components/eamxx/scripts/query-cf-database.cpp b/components/eamxx/scripts/query-cf-database/query-cf-database.cpp similarity index 100% rename from components/eamxx/scripts/query-cf-database.cpp rename to components/eamxx/scripts/query-cf-database/query-cf-database.cpp diff --git a/components/eamxx/scripts/scripts-tests b/components/eamxx/scripts/scripts-tests index e83ce9308ae..67416b23d2b 100755 --- a/components/eamxx/scripts/scripts-tests +++ b/components/eamxx/scripts/scripts-tests @@ -292,12 +292,13 @@ class TestTestAllScream(TestBaseOuter.TestBase): self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) builddir = "compute_sanitizer_memcheck" if self._machine.gpu_arch=="cuda" else "valgrind" - test_cmake_cache_contents(self, builddir, "CMAKE_BUILD_TYPE", "Debug") test_cmake_cache_contents(self, builddir, "SCREAM_TEST_SIZE", "SHORT") if self._machine.gpu_arch=="cuda": + test_cmake_cache_contents(self, builddir, "CMAKE_BUILD_TYPE", "Debug") test_cmake_cache_contents(self, builddir, "EKAT_ENABLE_COMPUTE_SANITIZER", "TRUE") test_cmake_cache_contents(self, builddir, "EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=memcheck") else: + test_cmake_cache_contents(self, builddir, "CMAKE_BUILD_TYPE", "RelWithDebInfo") test_cmake_cache_contents(self, builddir, "EKAT_ENABLE_VALGRIND", "TRUE") else: self.skipTest("Skipping config-only run for jenkins test") diff --git a/components/eamxx/scripts/test_factory.py b/components/eamxx/scripts/test_factory.py index a4d7bceea82..47ae4736721 100644 --- a/components/eamxx/scripts/test_factory.py +++ b/components/eamxx/scripts/test_factory.py @@ -153,8 +153,11 @@ def __init__(self, tas): TestProperty.__init__( self, "valgrind", - "debug with valgrind", - [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_VALGRIND", "True")], + "Release build where tests run through valgrind", + [("CMAKE_BUILD_TYPE", "RelWithDebInfo"), + ("EKAT_ENABLE_VALGRIND", "True"), + ("SCREAM_PACK_SIZE", "1"), + ("SCREAM_TEST_MAX_THREADS", "2")], uses_baselines=False, on_by_default=False, default_test_len="short" diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index 0f1cb1e31ab..23da9630b27 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -4,6 +4,7 @@ #include "physics/share/physics_constants.hpp" +#include "share/scream_config.hpp" #include "share/atm_process/atmosphere_process_group.hpp" #include "share/atm_process/atmosphere_process_dag.hpp" #include "share/field/field_utils.hpp" @@ -180,24 +181,24 @@ init_time_stamps (const util::TimeStamp& run_t0, const util::TimeStamp& case_t0, } void AtmosphereDriver:: -setup_iop () +setup_iop_data_manager () { // At this point, must have comm, params, initialized timestamps created. check_ad_status(s_comm_set | s_params_set | s_ts_inited); // Check to make sure iop is not already initialized - EKAT_REQUIRE_MSG(not m_iop, "Error! setup_iop() is called, but IOP already set up.\n"); + EKAT_REQUIRE_MSG(not m_iop_data_manager, "Error! setup_iop_data_manager() is called, but IOP already set up.\n"); // This function should only be called if we are enabling IOP const bool enable_iop = m_atm_params.sublist("driver_options").get("enable_iop", false); - EKAT_REQUIRE_MSG(enable_iop, "Error! setup_iop() is called, but enable_iop=false " + EKAT_REQUIRE_MSG(enable_iop, "Error! setup_iop_data_manager() is called, but enable_iop=false " "in driver_options parameters.\n"); // Params must include iop_options sublist. const auto iop_sublist_exists = m_atm_params.isSublist("iop_options"); EKAT_REQUIRE_MSG(iop_sublist_exists, - "Error! setup_iop() is called, but no iop_options " + "Error! setup_iop_data_manager() is called, but no iop_options " "defined in parameters.\n"); const auto iop_params = m_atm_params.sublist("iop_options"); @@ -206,15 +207,15 @@ setup_iop () const auto hyam = phys_grid->get_geometry_data("hyam"); const auto hybm = phys_grid->get_geometry_data("hybm"); - m_iop = std::make_shared(m_atm_comm, - iop_params, - m_run_t0, - nlevs, - hyam, - hybm); + m_iop_data_manager = std::make_shared(m_atm_comm, + iop_params, + m_run_t0, + nlevs, + hyam, + hybm); // Set IOP object in atm processes - m_atm_process_group->set_iop(m_iop); + m_atm_process_group->set_iop_data_manager(m_iop_data_manager); } void AtmosphereDriver::create_atm_processes() @@ -295,7 +296,7 @@ void AtmosphereDriver::create_grids() const bool enable_iop = m_atm_params.sublist("driver_options").get("enable_iop", false); if (enable_iop) { - setup_iop (); + setup_iop_data_manager (); } // Set the grids in the processes. Do this by passing the grids manager. @@ -685,8 +686,68 @@ void AtmosphereDriver::create_fields() fm->add_to_group(fid.name(),"RESTART"); } + auto& driver_options_pl = m_atm_params.sublist("driver_options"); + const int verb_lvl = driver_options_pl.get("atmosphere_dag_verbosity_level",-1); + if (verb_lvl>0) { + // now that we've got fields, generate a DAG with fields and dependencies + // NOTE: at this point, fields provided by initial conditions may (will) + // appear as unmet dependencies + AtmProcDAG dag; + // First, add all atm processes + dag.create_dag(*m_atm_process_group); + // Write a dot file for visualizing the DAG + if (m_atm_comm.am_i_root()) { + std::string filename = "scream_atm_createField_dag"; + if (is_scream_standalone()) { + filename += ".np" + std::to_string(m_atm_comm.size()); + } + filename += ".dot"; + dag.write_dag(filename, verb_lvl); + } + } + m_ad_status |= s_fields_created; + // If the user requested it, we can save a dictionary of the FM fields to file + if (driver_options_pl.get("save_field_manager_content",false)) { + auto pg = m_grids_manager->get_grid("Physics"); + const auto& fm = m_field_mgrs.at(pg->name()); + ekat::ParameterList pl_out("field_manager_content"); + pl_out.sublist("provenance") = m_atm_params.sublist("provenance"); + DefaultMetadata std_names; + std::string desc; + desc = "content of the EAMxx FieldManager corresponding to the 'Physics' grid.\n" + "The dict keys are the field names as used in EAMxx.\n" + "For each field, we add the following entries:\n" + " - standard_name: the name commonly used to refer to this field in atm sciences (if applicable)\n" + " - units: the units for this field used in EAMxx\n" + " - layout: the names of the dimensions for this field (time excluded)\n" + " - providers: the atm processes that update/compute this field\n" + " - customers: the atm processes that require this field as an input\n"; + pl_out.set("description", desc); + auto& dict = pl_out.sublist("fields"); + for (const auto& it : *fm) { + const auto& fid = it.second->get_header().get_identifier(); + auto& pl = dict.sublist(fid.name()); + + pl.set("units",fid.get_units().to_string()); + pl.set("layout",fid.get_layout().names()); + pl.set("standard_name",std_names.get_standardname(fid.name())); + std::vector providers,customers; + const auto& track = it.second->get_header().get_tracking(); + for (auto ap : track.get_providers()) { + providers.push_back(ap.lock()->name()); + } + for (auto ap : track.get_customers()) { + customers.push_back(ap.lock()->name()); + } + pl.set("providers",providers); + pl.set("customers",customers); + } + + ekat::write_yaml_file("eamxx_field_manager_content.yaml",pl_out); + } + stop_timer("EAMxx::create_fields"); stop_timer("EAMxx::init"); m_atm_logger->info("[EAMxx] create_fields ... done!"); @@ -861,28 +922,6 @@ initialize_fields () TraceGasesWorkaround::singleton().run_type = m_run_type; } - // See if we need to print a DAG. We do this first, cause if any input - // field is missing from the initial condition file, an error will be thrown. - // By printing the DAG first, we give the user the possibility of seeing - // what fields are inputs to the atm time step, so he/she can fix the i.c. file. - // TODO: would be nice to do the IC input first, and mark the fields in the - // DAG node "Begin of atm time step" in red if there's no initialization - // mechanism set for them. That is, allow field XYZ to not be found in - // the IC file, and throw an error when the dag is created. - - auto& driver_options_pl = m_atm_params.sublist("driver_options"); - const int verb_lvl = driver_options_pl.get("atmosphere_dag_verbosity_level",-1); - if (verb_lvl>0) { - // Check the atm DAG for missing stuff - AtmProcDAG dag; - - // First, add all atm processes - dag.create_dag(*m_atm_process_group); - - // Write a dot file for visualization - dag.write_dag("scream_atm_dag.dot",std::max(verb_lvl,0)); - } - // Initialize fields if (m_run_type==RunType::Restart) { restart_model (); @@ -1060,7 +1099,6 @@ void AtmosphereDriver::set_initial_conditions () // Check which fields need to have an initial condition. std::map> ic_fields_names; std::vector ic_fields_to_copy; - std::map> fields_inited; // Check which fields should be loaded from the topography file std::map> topography_file_fields_names; @@ -1089,7 +1127,7 @@ void AtmosphereDriver::set_initial_conditions () EKAT_ERROR_MSG ("ERROR: invalid assignment for variable " + fname + ", only scalar " "double or string, or vector double arguments are allowed"); } - fields_inited[grid_name].push_back(fname); + m_fields_inited[grid_name].push_back(fname); } else if (fname == "phis" or fname == "sgh30") { // Both phis and sgh30 need to be loaded from the topography file auto& this_grid_topo_file_fnames = topography_file_fields_names[grid_name]; @@ -1105,7 +1143,7 @@ void AtmosphereDriver::set_initial_conditions () grid_name == "Point Grid") { this_grid_topo_file_fnames.push_back("PHIS_d"); this_grid_topo_eamxx_fnames.push_back(fname); - fields_inited[grid_name].push_back(fname); + m_fields_inited[grid_name].push_back(fname); } else { EKAT_ERROR_MSG ("Error! Requesting phis on an unknown grid: " + grid_name + ".\n"); } @@ -1117,7 +1155,7 @@ void AtmosphereDriver::set_initial_conditions () " topo file only has sgh30 for Physics PG2.\n"); topography_file_fields_names[grid_name].push_back("SGH30"); topography_eamxx_fields_names[grid_name].push_back(fname); - fields_inited[grid_name].push_back(fname); + m_fields_inited[grid_name].push_back(fname); } } else if (not (fvphyshack and grid_name == "Physics PG2")) { // The IC file is written for the GLL grid, so we only load @@ -1129,7 +1167,7 @@ void AtmosphereDriver::set_initial_conditions () // If this field is the parent of other subfields, we only read from file the subfields. if (not ekat::contains(this_grid_ic_fnames,fname)) { this_grid_ic_fnames.push_back(fname); - fields_inited[grid_name].push_back(fname); + m_fields_inited[grid_name].push_back(fname); } } else if (fvphyshack and grid_name == "Physics GLL") { // [CGLL ICs in pg2] I tried doing something like this in @@ -1146,7 +1184,7 @@ void AtmosphereDriver::set_initial_conditions () } else { this_grid_ic_fnames.push_back(fname); } - fields_inited[grid_name].push_back(fname); + m_fields_inited[grid_name].push_back(fname); } } } @@ -1192,7 +1230,7 @@ void AtmosphereDriver::set_initial_conditions () auto p = f.get_header().get_parent().lock(); if (p) { const auto& pname = p->get_identifier().name(); - if (ekat::contains(fields_inited[grid_name],pname)) { + if (ekat::contains(m_fields_inited[grid_name],pname)) { // The parent is already inited. No need to init this field as well. names.erase(it2); run_again = true; @@ -1203,7 +1241,7 @@ void AtmosphereDriver::set_initial_conditions () } } - if (m_iop) { + if (m_iop_data_manager) { // For runs with IOP, call to setup io grids and lat // lon information needed for reading from file // We use a single topo file for both GLL and PG2 runs. All @@ -1213,13 +1251,13 @@ void AtmosphereDriver::set_initial_conditions () for (const auto& it : m_field_mgrs) { const auto& grid_name = it.first; if (ic_fields_names[grid_name].size() > 0 or - topography_eamxx_fields_names[grid_name].size() > 0) { + topography_eamxx_fields_names[grid_name].size() > 0) { const auto& file_name = grid_name == "Physics GLL" ? ic_pl.get("Filename") : ic_pl.get("topography_filename"); - m_iop->setup_io_info(file_name, it.second->get_grid()); + m_iop_data_manager->setup_io_info(file_name, it.second->get_grid()); } } } @@ -1231,12 +1269,12 @@ void AtmosphereDriver::set_initial_conditions () m_atm_logger->info(" [EAMxx] IC filename: " + file_name); for (const auto& it : m_field_mgrs) { const auto& grid_name = it.first; - if (not m_iop) { + if (not m_iop_data_manager) { read_fields_from_file (ic_fields_names[grid_name],it.second->get_grid(),file_name,m_current_ts); } else { // For IOP enabled, we load from file and copy data from the closest // lat/lon column to every other column - m_iop->read_fields_from_file_for_iop(file_name, + m_iop_data_manager->read_fields_from_file_for_iop(file_name, ic_fields_names[grid_name], m_current_ts, it.second); @@ -1306,7 +1344,7 @@ void AtmosphereDriver::set_initial_conditions () m_atm_logger->info(" filename: " + file_name); for (const auto& it : m_field_mgrs) { const auto& grid_name = it.first; - if (not m_iop) { + if (not m_iop_data_manager) { // Topography files always use "ncol_d" for the GLL grid value of ncol. // To ensure we read in the correct value, we must change the name for that dimension auto io_grid = it.second->get_grid(); @@ -1322,7 +1360,7 @@ void AtmosphereDriver::set_initial_conditions () } else { // For IOP enabled, we load from file and copy data from the closest // lat/lon column to every other column - m_iop->read_fields_from_file_for_iop(file_name, + m_iop_data_manager->read_fields_from_file_for_iop(file_name, topography_file_fields_names[grid_name], topography_eamxx_fields_names[grid_name], m_current_ts, @@ -1347,16 +1385,16 @@ void AtmosphereDriver::set_initial_conditions () m_atm_params.sublist("provenance").set("topography_file","NONE"); } - if (m_iop) { + if (m_iop_data_manager) { // Load IOP data file data for initial time stamp - m_iop->read_iop_file_data(m_current_ts); + m_iop_data_manager->read_iop_file_data(m_current_ts); // Now that ICs are processed, set appropriate fields using IOP file data. // Since ICs are loaded on GLL grid, we set those fields only and dynamics // will take care of the rest (for PG2 case). if (m_field_mgrs.count("Physics GLL") > 0) { const auto& fm = m_field_mgrs.at("Physics GLL"); - m_iop->set_fields_from_iop_data(fm); + m_iop_data_manager->set_fields_from_iop_data(fm); } } @@ -1415,7 +1453,7 @@ void AtmosphereDriver::set_initial_conditions () // Loop through fields and apply perturbation. for (size_t f=0; fget_grid()->name()], fname), + EKAT_REQUIRE_MSG(ekat::contains(m_fields_inited[fm->get_grid()->name()], fname), "Error! Attempting to apply perturbation to field not in initial_conditions.\n" " - Field: "+fname+"\n" " - Grid: "+fm->get_grid()->name()+"\n"); @@ -1614,6 +1652,28 @@ void AtmosphereDriver::initialize_atm_procs () m_atm_logger->info("[EAMxx] initialize_atm_procs ... done!"); report_res_dep_memory_footprint (); + + auto& driver_options_pl = m_atm_params.sublist("driver_options"); + const int verb_lvl = driver_options_pl.get("atmosphere_dag_verbosity_level",-1); + if (verb_lvl>0) { + // now that we've got fields, generate a DAG with fields and dependencies + // NOTE: at this point, fields provided by initial conditions may (will) + // appear as unmet dependencies + AtmProcDAG dag; + // First, add all atm processes + dag.create_dag(*m_atm_process_group); + // process the initial conditions to maybe fulfill unmet dependencies + dag.process_initial_conditions(m_fields_inited); + // Write a dot file for visualizing the DAG + if (m_atm_comm.am_i_root()) { + std::string filename = "scream_atm_initProc_dag"; + if (is_scream_standalone()) { + filename += ".np" + std::to_string(m_atm_comm.size()); + } + filename += ".dot"; + dag.write_dag(filename, verb_lvl); + } + } } void AtmosphereDriver:: @@ -1754,7 +1814,7 @@ void AtmosphereDriver::finalize ( /* inputs? */ ) { } // Destroy iop - m_iop = nullptr; + m_iop_data_manager = nullptr; // Destroy the buffer manager m_memory_buffer = nullptr; diff --git a/components/eamxx/src/control/atmosphere_driver.hpp b/components/eamxx/src/control/atmosphere_driver.hpp index a3acfba5d94..408b82437e4 100644 --- a/components/eamxx/src/control/atmosphere_driver.hpp +++ b/components/eamxx/src/control/atmosphere_driver.hpp @@ -2,7 +2,6 @@ #define SCREAM_ATMOSPHERE_DRIVER_HPP #include "control/surface_coupling_utils.hpp" -#include "share/iop/intensive_observation_period.hpp" #include "share/field/field_manager.hpp" #include "share/grid/grids_manager.hpp" #include "share/util/scream_time_stamp.hpp" @@ -11,6 +10,7 @@ #include "share/io/scorpio_input.hpp" #include "share/atm_process/ATMBufferManager.hpp" #include "share/atm_process/SCDataManager.hpp" +#include "share/atm_process/IOPDataManager.hpp" #include "ekat/logging/ekat_logger.hpp" #include "ekat/mpi/ekat_comm.hpp" @@ -72,8 +72,8 @@ class AtmosphereDriver // Set AD params void init_scorpio (const int atm_id = 0); - // Setup IntensiveObservationPeriod - void setup_iop (); + // Setup IOPDataManager + void setup_iop_data_manager (); // Create atm processes, without initializing them void create_atm_processes (); @@ -217,7 +217,7 @@ class AtmosphereDriver std::shared_ptr m_surface_coupling_import_data_manager; std::shared_ptr m_surface_coupling_export_data_manager; - std::shared_ptr m_iop; + std::shared_ptr m_iop_data_manager; // This is the time stamp at the beginning of the time step. util::TimeStamp m_current_ts; @@ -264,6 +264,8 @@ class AtmosphereDriver // Current simulation casename std::string m_casename; + // maps grid name to a vector of its initialized fields + std::map> m_fields_inited; }; } // namespace control diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp index ca7772fdf1b..6471974bebb 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp @@ -23,7 +23,7 @@ namespace scream */ // enum to track how exported fields will be set. -enum ExportType { +enum ExportType:int { FROM_MODEL = 0, // Variable will be derived from atmosphere model state FROM_FILE = 1, // Variable will be set given data from a file CONSTANT = 2 // Set variable to a constant value diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp index 2c3360a3b4f..385e57cae55 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp @@ -208,8 +208,8 @@ void SurfaceCouplingImporter::do_import(const bool called_during_initialization) }); #endif - if (m_iop) { - if (m_iop->get_params().get("iop_srf_prop")) { + if (m_iop_data_manager) { + if (m_iop_data_manager->get_params().get("iop_srf_prop")) { // Overwrite imports with data from IOP file overwrite_iop_imports(called_during_initialization); } @@ -221,9 +221,12 @@ void SurfaceCouplingImporter::overwrite_iop_imports (const bool called_during_in using policy_type = KokkosTypes::RangePolicy; using C = physics::Constants; - const auto has_lhflx = m_iop->has_iop_field("lhflx"); - const auto has_shflx = m_iop->has_iop_field("shflx"); - const auto has_Tg = m_iop->has_iop_field("Tg"); + const auto has_lhflx = m_iop_data_manager->has_iop_field("lhflx"); + const auto has_shflx = m_iop_data_manager->has_iop_field("shflx"); + const auto has_Tg = m_iop_data_manager->has_iop_field("Tg"); + + // Read IOP file for current time step, if necessary + m_iop_data_manager->read_iop_file_data(timestamp()); static constexpr Real latvap = C::LatVap; static constexpr Real stebol = C::stebol; @@ -243,19 +246,19 @@ void SurfaceCouplingImporter::overwrite_iop_imports (const bool called_during_in // Store IOP surf data into col_val Real col_val(std::nan("")); if (fname == "surf_evap" && has_lhflx) { - const auto f = m_iop->get_iop_field("lhflx"); + const auto f = m_iop_data_manager->get_iop_field("lhflx"); f.sync_to_host(); col_val = f.get_view()()/latvap; } else if (fname == "surf_sens_flux" && has_shflx) { - const auto f = m_iop->get_iop_field("shflx"); + const auto f = m_iop_data_manager->get_iop_field("shflx"); f.sync_to_host(); col_val = f.get_view()(); } else if (fname == "surf_radiative_T" && has_Tg) { - const auto f = m_iop->get_iop_field("Tg"); + const auto f = m_iop_data_manager->get_iop_field("Tg"); f.sync_to_host(); col_val = f.get_view()(); } else if (fname == "surf_lw_flux_up" && has_Tg) { - const auto f = m_iop->get_iop_field("Tg"); + const auto f = m_iop_data_manager->get_iop_field("Tg"); f.sync_to_host(); col_val = stebol*std::pow(f.get_view()(), 4); } else { diff --git a/components/eamxx/src/diagnostics/CMakeLists.txt b/components/eamxx/src/diagnostics/CMakeLists.txt index be51f434615..0795f9e5469 100644 --- a/components/eamxx/src/diagnostics/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/CMakeLists.txt @@ -1,11 +1,16 @@ set(DIAGNOSTIC_SRCS + aerocom_cld.cpp + aodvis.cpp + atm_backtend.cpp atm_density.cpp dry_static_energy.cpp exner.cpp field_at_height.cpp field_at_level.cpp field_at_pressure_level.cpp + horiz_avg.cpp longwave_cloud_forcing.cpp + number_path.cpp potential_temperature.cpp precip_surf_mass_flux.cpp relative_humidity.cpp @@ -17,15 +22,11 @@ set(DIAGNOSTIC_SRCS virtual_temperature.cpp water_path.cpp wind_speed.cpp - aodvis.cpp - number_path.cpp - aerocom_cld.cpp - atm_backtend.cpp ) add_library(diagnostics ${DIAGNOSTIC_SRCS}) target_link_libraries(diagnostics PUBLIC scream_share) -if (NOT SCREAM_LIB_ONLY) +if (NOT SCREAM_LIB_ONLY AND NOT SCREAM_ONLY_GENERATE_BASELINES) add_subdirectory(tests) endif() diff --git a/components/eamxx/src/diagnostics/aerocom_cld.cpp b/components/eamxx/src/diagnostics/aerocom_cld.cpp index 8606e065b9e..cca55f6c19f 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.cpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.cpp @@ -22,8 +22,6 @@ AeroComCld::AeroComCld(const ekat::Comm &comm, "to be 'Bot' or 'Top' in its input parameters.\n"); } -std::string AeroComCld::name() const { return "AeroComCld" + m_topbot; } - void AeroComCld::set_grids( const std::shared_ptr grids_manager) { using namespace ekat::units; @@ -76,7 +74,8 @@ void AeroComCld::set_grids( m_dz.allocate_view(); // Construct and allocate the output field - FieldIdentifier fid(name(), vector1d_layout, nondim, grid_name); + + FieldIdentifier fid("AeroComCld"+m_topbot, vector1d_layout, nondim, grid_name); m_diagnostic_output = Field(fid); m_diagnostic_output.allocate_view(); diff --git a/components/eamxx/src/diagnostics/aerocom_cld.hpp b/components/eamxx/src/diagnostics/aerocom_cld.hpp index 694eed76f09..53a9a7f8e1b 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.hpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.hpp @@ -15,7 +15,7 @@ class AeroComCld : public AtmosphereDiagnostic { AeroComCld(const ekat::Comm &comm, const ekat::ParameterList ¶ms); // The name of the diagnostic - std::string name() const override; + std::string name() const override { return "AeroComCld"; } // Set the grid void set_grids( diff --git a/components/eamxx/src/diagnostics/field_at_height.cpp b/components/eamxx/src/diagnostics/field_at_height.cpp index f61cd3a76c1..db780f5a79f 100644 --- a/components/eamxx/src/diagnostics/field_at_height.cpp +++ b/components/eamxx/src/diagnostics/field_at_height.cpp @@ -45,21 +45,16 @@ FieldAtHeight (const ekat::Comm& comm, const ekat::ParameterList& params) " - surface reference: " + surf_ref + "\n" " - valid options: sealevel, surface\n"); m_z_name = (surf_ref == "sealevel") ? "z" : "height"; - const auto& location = m_params.get("vertical_location"); - auto chars_start = location.find_first_not_of("0123456789."); - EKAT_REQUIRE_MSG (chars_start!=0 && chars_start!=std::string::npos, - "Error! Invalid string for height value for FieldAtHeight.\n" - " - input string : " + location + "\n" - " - expected format: Nm, with N integer\n"); - const auto z_str = location.substr(0,chars_start); - m_z = std::stod(z_str); - - const auto units = location.substr(chars_start); + + const auto units = m_params.get("height_units"); EKAT_REQUIRE_MSG (units=="m", - "Error! Invalid string for height value for FieldAtHeight.\n" - " - input string : " + location + "\n" - " - expected format: Nm, with N integer\n"); - m_diag_name = m_field_name + "_at_" + m_params.get("vertical_location") + "_above_" + surf_ref; + "Error! Invalid units for FieldAtHeight.\n" + " - input units: " + units + "\n" + " - valid units: m\n"); + + auto z_val = m_params.get("height_value"); + m_z = std::stod(z_val); + m_diag_name = m_field_name + "_at_" + z_val + units + "_above_" + surf_ref; } void FieldAtHeight:: @@ -89,7 +84,9 @@ initialize_impl (const RunType /*run_type*/) EKAT_REQUIRE_MSG (layout.rank()>=2 && layout.rank()<=3, "Error! Field rank not supported by FieldAtHeight.\n" " - field name: " + fid.name() + "\n" - " - field layout: " + layout.to_string() + "\n"); + " - field layout: " + layout.to_string() + "\n" + "NOTE: if you requested something like 'field_horiz_avg_at_Y',\n" + " you can avoid this error by requesting 'fieldX_at_Y_horiz_avg' instead.\n"); const auto tag = layout.tags().back(); EKAT_REQUIRE_MSG (tag==LEV || tag==ILEV, "Error! FieldAtHeight diagnostic expects a layout ending with 'LEV'/'ILEV' tag.\n" diff --git a/components/eamxx/src/diagnostics/field_at_height.hpp b/components/eamxx/src/diagnostics/field_at_height.hpp index e6198153f94..91f6ae3eb1b 100644 --- a/components/eamxx/src/diagnostics/field_at_height.hpp +++ b/components/eamxx/src/diagnostics/field_at_height.hpp @@ -18,7 +18,7 @@ class FieldAtHeight : public AtmosphereDiagnostic FieldAtHeight (const ekat::Comm& comm, const ekat::ParameterList& params); // The name of the diagnostic - std::string name () const { return m_diag_name; } + std::string name () const { return "FieldAtHeight"; } // Set the grid void set_grids (const std::shared_ptr grids_manager); diff --git a/components/eamxx/src/diagnostics/field_at_level.cpp b/components/eamxx/src/diagnostics/field_at_level.cpp index 87ecf7ad910..b842429a8d6 100644 --- a/components/eamxx/src/diagnostics/field_at_level.cpp +++ b/components/eamxx/src/diagnostics/field_at_level.cpp @@ -30,10 +30,12 @@ initialize_impl (const RunType /*run_type*/) using namespace ShortFieldTagsNames; const auto& fid = f.get_header().get_identifier(); const auto& layout = fid.get_layout(); - EKAT_REQUIRE_MSG (layout.rank()>1 && layout.rank()<=6, + EKAT_REQUIRE_MSG (layout.rank()>=2 && layout.rank()<=6, "Error! Field rank not supported by FieldAtLevel.\n" " - field name: " + fid.name() + "\n" - " - field layout: " + layout.to_string() + "\n"); + " - field layout: " + layout.to_string() + "\n" + "NOTE: if you requested something like 'field_horiz_avg_at_Y',\n" + " you can avoid this error by requesting 'fieldX_at_Y_horiz_avg' instead.\n"); const auto tag = layout.tags().back(); EKAT_REQUIRE_MSG (tag==LEV || tag==ILEV, "Error! FieldAtLevel diagnostic expects a layout ending with 'LEV'/'ILEV' tag.\n" diff --git a/components/eamxx/src/diagnostics/field_at_level.hpp b/components/eamxx/src/diagnostics/field_at_level.hpp index b63cda0a1a3..3ab9bee1557 100644 --- a/components/eamxx/src/diagnostics/field_at_level.hpp +++ b/components/eamxx/src/diagnostics/field_at_level.hpp @@ -21,7 +21,7 @@ class FieldAtLevel : public AtmosphereDiagnostic FieldAtLevel (const ekat::Comm& comm, const ekat::ParameterList& params); // The name of the diagnostic - std::string name () const { return m_diag_name; } + std::string name () const { return "FieldAtLevel"; } // Set the grid void set_grids (const std::shared_ptr grids_manager); diff --git a/components/eamxx/src/diagnostics/field_at_pressure_level.cpp b/components/eamxx/src/diagnostics/field_at_pressure_level.cpp index 21c1ac78dd9..78ed921e758 100644 --- a/components/eamxx/src/diagnostics/field_at_pressure_level.cpp +++ b/components/eamxx/src/diagnostics/field_at_pressure_level.cpp @@ -15,30 +15,22 @@ FieldAtPressureLevel (const ekat::Comm& comm, const ekat::ParameterList& params) { m_field_name = m_params.get("field_name"); - // Figure out the pressure value - const auto& location = m_params.get("vertical_location"); - auto chars_start = location.find_first_not_of("0123456789."); - EKAT_REQUIRE_MSG (chars_start!=0 && chars_start!=std::string::npos, - "Error! Invalid string for pressure value for FieldAtPressureLevel.\n" - " - input string : " + location + "\n" - " - expected format: Nxyz, with N integer, and xyz='mb', 'hPa', or 'Pa'\n"); - const auto press_str = location.substr(0,chars_start); - m_pressure_level = std::stod(press_str); - - const auto units = location.substr(chars_start); + const auto units = m_params.get("pressure_units"); EKAT_REQUIRE_MSG (units=="mb" or units=="hPa" or units=="Pa", - "Error! Invalid string for pressure value for FieldAtPressureLevel.\n" - " - input string : " + location + "\n" - " - expected format: Nxyz, with N integer, and xyz='mb', 'hPa', or 'Pa'\n"); + "Error! Invalid units for FieldAtPressureLevel.\n" + " - input units: " + units + "\n" + " - valid units: 'mb', 'hPa', 'Pa'\n"); + + // Figure out the pressure value, and convert to Pa if needed + auto p_value = m_params.get("pressure_value"); - // Convert pressure level to Pa, the units of pressure in the simulation if (units=="mb" || units=="hPa") { - m_pressure_level *= 100; + m_pressure_level = std::stod(p_value)*100; + } else { + m_pressure_level = std::stod(p_value); } - m_mask_val = m_params.get("mask_value",Real(constants::DefaultFillValue::value)); - - m_diag_name = m_field_name + "_at_" + location; + m_diag_name = m_field_name + "_at_" + p_value + units; } void FieldAtPressureLevel:: @@ -64,7 +56,9 @@ initialize_impl (const RunType /*run_type*/) EKAT_REQUIRE_MSG (layout.rank()>=2 && layout.rank()<=3, "Error! Field rank not supported by FieldAtPressureLevel.\n" " - field name: " + fid.name() + "\n" - " - field layout: " + layout.to_string() + "\n"); + " - field layout: " + layout.to_string() + "\n" + "NOTE: if you requested something like 'field_horiz_avg_at_Y',\n" + " you can avoid this error by requesting 'fieldX_at_Y_horiz_avg' instead.\n"); const auto tag = layout.tags().back(); EKAT_REQUIRE_MSG (tag==LEV || tag==ILEV, "Error! FieldAtPressureLevel diagnostic expects a layout ending with 'LEV'/'ILEV' tag.\n" @@ -91,6 +85,8 @@ initialize_impl (const RunType /*run_type*/) // Add a field representing the mask as extra data to the diagnostic field. auto nondim = ekat::units::Units::nondimensional(); const auto& gname = fid.get_grid_name(); + m_mask_val = m_params.get("mask_value",Real(constants::DefaultFillValue::value)); + std::string mask_name = name() + " mask"; FieldLayout mask_layout( {COL}, {num_cols}); diff --git a/components/eamxx/src/diagnostics/field_at_pressure_level.hpp b/components/eamxx/src/diagnostics/field_at_pressure_level.hpp index 950c0c5e2ee..58e476ec83b 100644 --- a/components/eamxx/src/diagnostics/field_at_pressure_level.hpp +++ b/components/eamxx/src/diagnostics/field_at_pressure_level.hpp @@ -20,7 +20,7 @@ class FieldAtPressureLevel : public AtmosphereDiagnostic FieldAtPressureLevel (const ekat::Comm& comm, const ekat::ParameterList& params); // The name of the diagnostic - std::string name () const { return m_diag_name; } + std::string name () const { return "FieldAtPressureLevel"; } // Set the grid void set_grids (const std::shared_ptr grids_manager); diff --git a/components/eamxx/src/diagnostics/horiz_avg.cpp b/components/eamxx/src/diagnostics/horiz_avg.cpp new file mode 100644 index 00000000000..8bfe068c635 --- /dev/null +++ b/components/eamxx/src/diagnostics/horiz_avg.cpp @@ -0,0 +1,65 @@ +#include "diagnostics/horiz_avg.hpp" + +#include "share/field/field_utils.hpp" + +namespace scream { + +HorizAvgDiag::HorizAvgDiag(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) + : AtmosphereDiagnostic(comm, params) { + const auto &fname = m_params.get("field_name"); + m_diag_name = fname + "_horiz_avg"; +} + +void HorizAvgDiag::set_grids( + const std::shared_ptr grids_manager) { + const auto &fn = m_params.get("field_name"); + const auto &gn = m_params.get("grid_name"); + const auto g = grids_manager->get_grid("Physics"); + + add_field(fn, gn); + + // first clone the area unscaled, we will scale it later in initialize_impl + m_scaled_area = g->get_geometry_data("area").clone(); +} + +void HorizAvgDiag::initialize_impl(const RunType /*run_type*/) { + using namespace ShortFieldTagsNames; + const auto &f = get_fields_in().front(); + const auto &fid = f.get_header().get_identifier(); + const auto &layout = fid.get_layout(); + + EKAT_REQUIRE_MSG(layout.rank() >= 1 && layout.rank() <= 3, + "Error! Field rank not supported by HorizAvgDiag.\n" + " - field name: " + + fid.name() + + "\n" + " - field layout: " + + layout.to_string() + "\n"); + EKAT_REQUIRE_MSG(layout.tags()[0] == COL, + "Error! HorizAvgDiag diagnostic expects a layout starting " + "with the 'COL' tag.\n" + " - field name : " + + fid.name() + + "\n" + " - field layout: " + + layout.to_string() + "\n"); + + FieldIdentifier d_fid(m_diag_name, layout.clone().strip_dim(COL), + fid.get_units(), fid.get_grid_name()); + m_diagnostic_output = Field(d_fid); + m_diagnostic_output.allocate_view(); + + // scale the area field + auto total_area = field_sum(m_scaled_area, &m_comm); + m_scaled_area.scale(sp(1.0) / total_area); +} + +void HorizAvgDiag::compute_diagnostic_impl() { + const auto &f = get_fields_in().front(); + const auto &d = m_diagnostic_output; + // Call the horiz_contraction impl that will take care of everything + horiz_contraction(d, f, m_scaled_area, &m_comm); +} + +} // namespace scream diff --git a/components/eamxx/src/diagnostics/horiz_avg.hpp b/components/eamxx/src/diagnostics/horiz_avg.hpp new file mode 100644 index 00000000000..6ceac09103b --- /dev/null +++ b/components/eamxx/src/diagnostics/horiz_avg.hpp @@ -0,0 +1,43 @@ +#ifndef EAMXX_HORIZ_AVERAGE_HPP +#define EAMXX_HORIZ_AVERAGE_HPP + +#include "share/atm_process/atmosphere_diagnostic.hpp" + +namespace scream { + +/* + * This diagnostic will calculate the area-weighted average of a field + * across the COL tag dimension, producing an N-1 dimensional field + * that is area-weighted average of the input field. + */ + +class HorizAvgDiag : public AtmosphereDiagnostic { + public: + // Constructors + HorizAvgDiag(const ekat::Comm &comm, const ekat::ParameterList ¶ms); + + // The name of the diagnostic + std::string name() const { return m_diag_name; } + + // Set the grid + void set_grids(const std::shared_ptr grids_manager); + + protected: +#ifdef KOKKOS_ENABLE_CUDA + public: +#endif + void compute_diagnostic_impl(); + + protected: + void initialize_impl(const RunType /*run_type*/); + + // Name of each field (because the diagnostic impl is generic) + std::string m_diag_name; + + // Need area field, let's store it scaled by its norm + Field m_scaled_area; +}; + +} // namespace scream + +#endif // EAMXX_HORIZ_AVERAGE_HPP diff --git a/components/eamxx/src/diagnostics/number_path.cpp b/components/eamxx/src/diagnostics/number_path.cpp index d4df2f1bc22..70e18c6dee3 100644 --- a/components/eamxx/src/diagnostics/number_path.cpp +++ b/components/eamxx/src/diagnostics/number_path.cpp @@ -33,8 +33,6 @@ NumberPathDiagnostic::NumberPathDiagnostic(const ekat::Comm &comm, } } -std::string NumberPathDiagnostic::name() const { return m_kind + "NumberPath"; } - void NumberPathDiagnostic::set_grids( const std::shared_ptr grids_manager) { using namespace ekat::units; @@ -55,7 +53,7 @@ void NumberPathDiagnostic::set_grids( add_field(m_nname, scalar3d, 1 / kg, grid_name); // Construct and allocate the diagnostic field - FieldIdentifier fid(name(), scalar2d, kg/(kg*m2), grid_name); + FieldIdentifier fid(m_kind + "NumberPath", scalar2d, kg/(kg*m2), grid_name); m_diagnostic_output = Field(fid); m_diagnostic_output.allocate_view(); } diff --git a/components/eamxx/src/diagnostics/number_path.hpp b/components/eamxx/src/diagnostics/number_path.hpp index 4888d3601f4..30b383b9452 100644 --- a/components/eamxx/src/diagnostics/number_path.hpp +++ b/components/eamxx/src/diagnostics/number_path.hpp @@ -16,7 +16,7 @@ class NumberPathDiagnostic : public AtmosphereDiagnostic { const ekat::ParameterList ¶ms); // The name of the diagnostic - std::string name() const; + std::string name() const override { return "NumberPath"; } // Set the grid void set_grids(const std::shared_ptr grids_manager); diff --git a/components/eamxx/src/diagnostics/potential_temperature.cpp b/components/eamxx/src/diagnostics/potential_temperature.cpp index 67260e647f6..8cc45078de7 100644 --- a/components/eamxx/src/diagnostics/potential_temperature.cpp +++ b/components/eamxx/src/diagnostics/potential_temperature.cpp @@ -24,11 +24,6 @@ PotentialTemperatureDiagnostic::PotentialTemperatureDiagnostic (const ekat::Comm } } -std::string PotentialTemperatureDiagnostic::name() const -{ - return m_ptype; -} - // ========================================================================================= void PotentialTemperatureDiagnostic::set_grids(const std::shared_ptr grids_manager) { @@ -51,7 +46,7 @@ void PotentialTemperatureDiagnostic::set_grids(const std::shared_ptr("qc", scalar3d_layout_mid, kg/kg, grid_name, ps); // Construct and allocate the diagnostic field - FieldIdentifier fid (name(), scalar3d_layout_mid, K, grid_name); + FieldIdentifier fid (m_ptype, scalar3d_layout_mid, K, grid_name); m_diagnostic_output = Field(fid); auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); C_ap.request_allocation(ps); diff --git a/components/eamxx/src/diagnostics/potential_temperature.hpp b/components/eamxx/src/diagnostics/potential_temperature.hpp index 37fd1a30806..0ac1a1201d8 100644 --- a/components/eamxx/src/diagnostics/potential_temperature.hpp +++ b/components/eamxx/src/diagnostics/potential_temperature.hpp @@ -22,7 +22,7 @@ class PotentialTemperatureDiagnostic : public AtmosphereDiagnostic PotentialTemperatureDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); // The name of the diagnostic - std::string name () const; + std::string name () const override { return "PotentialTemperature"; } // Set the grid void set_grids (const std::shared_ptr grids_manager); diff --git a/components/eamxx/src/diagnostics/precip_surf_mass_flux.cpp b/components/eamxx/src/diagnostics/precip_surf_mass_flux.cpp index 068456f0522..329e83b6e9c 100644 --- a/components/eamxx/src/diagnostics/precip_surf_mass_flux.cpp +++ b/components/eamxx/src/diagnostics/precip_surf_mass_flux.cpp @@ -47,7 +47,7 @@ set_grids(const std::shared_ptr grids_manager) } // Construct and allocate the diagnostic field - FieldIdentifier fid(name(), scalar2d_layout_mid, m/s, grid_name); + FieldIdentifier fid(m_name, scalar2d_layout_mid, m/s, grid_name); m_diagnostic_output = Field(fid); m_diagnostic_output.get_header().get_alloc_properties().request_allocation(); m_diagnostic_output.allocate_view(); diff --git a/components/eamxx/src/diagnostics/precip_surf_mass_flux.hpp b/components/eamxx/src/diagnostics/precip_surf_mass_flux.hpp index 68dd1251646..6ff3458398d 100644 --- a/components/eamxx/src/diagnostics/precip_surf_mass_flux.hpp +++ b/components/eamxx/src/diagnostics/precip_surf_mass_flux.hpp @@ -17,7 +17,7 @@ class PrecipSurfMassFlux : public AtmosphereDiagnostic PrecipSurfMassFlux (const ekat::Comm& comm, const ekat::ParameterList& params); // The name of the diagnostic - std::string name () const { return m_name; } + std::string name () const { return "PrecipSurfMassFlux"; } // Set the grid void set_grids (const std::shared_ptr grids_manager); diff --git a/components/eamxx/src/diagnostics/register_diagnostics.hpp b/components/eamxx/src/diagnostics/register_diagnostics.hpp index 45a04eebeb1..67119416705 100644 --- a/components/eamxx/src/diagnostics/register_diagnostics.hpp +++ b/components/eamxx/src/diagnostics/register_diagnostics.hpp @@ -24,6 +24,7 @@ #include "diagnostics/number_path.hpp" #include "diagnostics/aerocom_cld.hpp" #include "diagnostics/atm_backtend.hpp" +#include "diagnostics/horiz_avg.hpp" namespace scream { @@ -36,13 +37,6 @@ inline void register_diagnostics () { diag_factory.register_product("AtmosphereDensity",&create_atmosphere_diagnostic); diag_factory.register_product("Exner",&create_atmosphere_diagnostic); diag_factory.register_product("VirtualTemperature",&create_atmosphere_diagnostic); - diag_factory.register_product("z_int",&create_atmosphere_diagnostic); - diag_factory.register_product("z_mid",&create_atmosphere_diagnostic); - diag_factory.register_product("geopotential_int",&create_atmosphere_diagnostic); - diag_factory.register_product("geopotential_mid",&create_atmosphere_diagnostic); - diag_factory.register_product("height_int",&create_atmosphere_diagnostic); - diag_factory.register_product("height_mid",&create_atmosphere_diagnostic); - diag_factory.register_product("dz",&create_atmosphere_diagnostic); diag_factory.register_product("DryStaticEnergy",&create_atmosphere_diagnostic); diag_factory.register_product("SeaLevelPressure",&create_atmosphere_diagnostic); diag_factory.register_product("WaterPath",&create_atmosphere_diagnostic); @@ -50,6 +44,7 @@ inline void register_diagnostics () { diag_factory.register_product("LongwaveCloudForcing",&create_atmosphere_diagnostic); diag_factory.register_product("RelativeHumidity",&create_atmosphere_diagnostic); diag_factory.register_product("VaporFlux",&create_atmosphere_diagnostic); + diag_factory.register_product("VerticalLayer",&create_atmosphere_diagnostic); diag_factory.register_product("precip_surf_mass_flux",&create_atmosphere_diagnostic); diag_factory.register_product("surface_upward_latent_heat_flux",&create_atmosphere_diagnostic); diag_factory.register_product("wind_speed",&create_atmosphere_diagnostic); @@ -57,7 +52,9 @@ inline void register_diagnostics () { diag_factory.register_product("NumberPath",&create_atmosphere_diagnostic); diag_factory.register_product("AeroComCld",&create_atmosphere_diagnostic); diag_factory.register_product("AtmBackTendDiag",&create_atmosphere_diagnostic); + diag_factory.register_product("HorizAvgDiag",&create_atmosphere_diagnostic); } } // namespace scream + #endif // SCREAM_REGISTER_DIAGNOSTICS_HPP diff --git a/components/eamxx/src/diagnostics/tests/CMakeLists.txt b/components/eamxx/src/diagnostics/tests/CMakeLists.txt index a684aa248fc..736253f9bee 100644 --- a/components/eamxx/src/diagnostics/tests/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/tests/CMakeLists.txt @@ -1,4 +1,4 @@ -# NOTE: tests inside this if statement won't be built in a baselines-only build +include(ScreamUtils) function (createDiagTest test_name test_srcs) CreateUnitTest(${test_name} "${test_srcs}" @@ -6,72 +6,71 @@ function (createDiagTest test_name test_srcs) LABELS diagnostics) endfunction () -if (NOT SCREAM_ONLY_GENERATE_BASELINES) - include(ScreamUtils) +# Test extracting a single level of a field +CreateDiagTest(field_at_level "field_at_level_tests.cpp") - # Test extracting a single level of a field - CreateDiagTest(field_at_level "field_at_level_tests.cpp") +# Test interpolating a field onto a single pressure level +CreateDiagTest(field_at_pressure_level "field_at_pressure_level_tests.cpp") - # Test interpolating a field onto a single pressure level - CreateDiagTest(field_at_pressure_level "field_at_pressure_level_tests.cpp") - # Test interpolating a field at a specific height - CreateDiagTest(field_at_height "field_at_height_tests.cpp") +# Test interpolating a field at a specific height +CreateDiagTest(field_at_height "field_at_height_tests.cpp") - # Test potential temperature diagnostic - CreateDiagTest(potential_temperature "potential_temperature_test.cpp") +# Test potential temperature diagnostic +CreateDiagTest(potential_temperature "potential_temperature_test.cpp") - # Test exner diagnostic - CreateDiagTest(exner_function "exner_test.cpp") +# Test exner diagnostic +CreateDiagTest(exner_function "exner_test.cpp") - # Test virtual temperature - CreateDiagTest(virtual_temperature "virtual_temperature_test.cpp") +# Test virtual temperature +CreateDiagTest(virtual_temperature "virtual_temperature_test.cpp") - # Test atmosphere density - CreateDiagTest(atmosphere_density "atm_density_test.cpp") +# Test atmosphere density +CreateDiagTest(atmosphere_density "atm_density_test.cpp") - # Test vertical layer (dz, z_int, z_mid) - CreateDiagTest(vertical_layer "vertical_layer_tests.cpp") +# Test vertical layer (dz, z_int, z_mid) +CreateDiagTest(vertical_layer "vertical_layer_tests.cpp") - # Test dry static energy - CreateDiagTest(dry_static_energy "dry_static_energy_test.cpp") +# Test dry static energy +CreateDiagTest(dry_static_energy "dry_static_energy_test.cpp") - # Test sea level pressure - CreateDiagTest(sea_level_pressure "sea_level_pressure_test.cpp") +# Test sea level pressure +CreateDiagTest(sea_level_pressure "sea_level_pressure_test.cpp") - # Test total water path - CreateDiagTest(water_path "water_path_tests.cpp") +# Test total water path +CreateDiagTest(water_path "water_path_tests.cpp") - # Test shortwave cloud forcing - CreateDiagTest(shortwave_cloud_forcing "shortwave_cloud_forcing_tests.cpp") +# Test shortwave cloud forcing +CreateDiagTest(shortwave_cloud_forcing "shortwave_cloud_forcing_tests.cpp") - # Test longwave cloud forcing - CreateDiagTest(longwave_cloud_forcing "longwave_cloud_forcing_tests.cpp") +# Test longwave cloud forcing +CreateDiagTest(longwave_cloud_forcing "longwave_cloud_forcing_tests.cpp") - # Test Relative Humidity - CreateDiagTest(relative_humidity "relative_humidity_tests.cpp") +# Test Relative Humidity +CreateDiagTest(relative_humidity "relative_humidity_tests.cpp") - # Test Vapor Flux - CreateDiagTest(vapor_flux "vapor_flux_tests.cpp") +# Test Vapor Flux +CreateDiagTest(vapor_flux "vapor_flux_tests.cpp") - # Test precipitation mass surface flux - CreateDiagTest(precip_surf_mass_flux "precip_surf_mass_flux_tests.cpp") +# Test precipitation mass surface flux +CreateDiagTest(precip_surf_mass_flux "precip_surf_mass_flux_tests.cpp") - # Test surface latent heat flux - CreateDiagTest(surface_upward_latent_heat_flux "surf_upward_latent_heat_flux_tests.cpp") +# Test surface latent heat flux +CreateDiagTest(surface_upward_latent_heat_flux "surf_upward_latent_heat_flux_tests.cpp") - # Test wind speed diagnostic - CreateDiagTest(wind_speed "wind_speed_tests.cpp") +# Test wind speed diagnostic +CreateDiagTest(wind_speed "wind_speed_tests.cpp") - # Test AODVIS - CreateDiagTest(aodvis "aodvis_test.cpp") +# Test AODVIS +CreateDiagTest(aodvis "aodvis_test.cpp") - # Test "number" paths - CreateDiagTest(number_paths "number_paths_tests.cpp") +# Test "number" paths +CreateDiagTest(number_paths "number_paths_tests.cpp") - # Test AEROCOM_CLD - CreateDiagTest(aerocom_cld "aerocom_cld_test.cpp") +# Test AEROCOM_CLD +CreateDiagTest(aerocom_cld "aerocom_cld_test.cpp") - # Test atm_tend - CreateDiagTest(atm_backtend "atm_backtend_test.cpp") +# Test atm_tend +CreateDiagTest(atm_backtend "atm_backtend_test.cpp") -endif() +# Test horizontal averaging +CreateDiagTest(horiz_avg "horiz_avg_test.cpp") diff --git a/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp b/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp index e26d34dd1a5..6e071e4b201 100644 --- a/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp @@ -104,12 +104,13 @@ TEST_CASE("field_at_height") // Lambda to create and run a diag, and return output auto run_diag = [&](const Field& f, const Field& z, - const std::string& loc, const std::string& surf_ref) { + const double h, const std::string& surf_ref) { util::TimeStamp t0 ({2022,1,1},{0,0,0}); auto& factory = AtmosphereDiagnosticFactory::instance(); ekat::ParameterList pl; pl.set("surface_reference",surf_ref); - pl.set("vertical_location",loc); + pl.set("height_value",std::to_string(h)); + pl.set("height_units",std::string("m")); pl.set("field_name",f.name()); pl.set("grid_name",grid->name()); auto diag = factory.create("FieldAtheight",comm,pl); @@ -173,13 +174,12 @@ TEST_CASE("field_at_height") // Make sure that an unsupported reference height throws an error. print(" -> Testing throws error with unsupported reference height...\n"); { - REQUIRE_THROWS(run_diag (s_mid,h_mid,"1m","foobar")); + REQUIRE_THROWS(run_diag (s_mid,h_mid,1.0,"foobar")); } print(" -> Testing throws error with unsupported reference height... OK\n"); // Run many times int z_tgt; - std::string loc; for (std::string surf_ref : {"sealevel","surface"}) { printf(" -> Testing for a reference height above %s...\n",surf_ref.c_str()); const auto mid_src = surf_ref == "sealevel" ? z_mid : h_mid; @@ -197,32 +197,31 @@ TEST_CASE("field_at_height") // Set target z-slice for testing to a random value. z_tgt = pdf_levs(engine)+max_surf_4test; - loc = std::to_string(z_tgt) + "m"; - printf(" -> test at height of %s.............\n",loc.c_str()); + printf(" -> test at height of %dm............\n",z_tgt); { print(" -> scalar midpoint field...............\n"); - auto d = run_diag(s_mid,mid_src,loc,surf_ref); + auto d = run_diag(s_mid,mid_src,z_tgt,surf_ref); f_z_tgt(inter,slope,z_tgt,mid_src,s_tgt); REQUIRE (views_are_approx_equal(d,s_tgt,tol)); print(" -> scalar midpoint field............... OK!\n"); } { print(" -> scalar interface field...............\n"); - auto d = run_diag (s_int,int_src,loc,surf_ref); + auto d = run_diag (s_int,int_src,z_tgt,surf_ref); f_z_tgt(inter,slope,z_tgt,int_src,s_tgt); REQUIRE (views_are_approx_equal(d,s_tgt,tol)); print(" -> scalar interface field............... OK!\n"); } { print(" -> vector midpoint field...............\n"); - auto d = run_diag (v_mid,mid_src,loc,surf_ref); + auto d = run_diag (v_mid,mid_src,z_tgt,surf_ref); f_z_tgt(inter,slope,z_tgt,mid_src,v_tgt); REQUIRE (views_are_approx_equal(d,v_tgt,tol)); print(" -> vector midpoint field............... OK!\n"); } { print(" -> vector interface field...............\n"); - auto d = run_diag (v_int,int_src,loc,surf_ref); + auto d = run_diag (v_int,int_src,z_tgt,surf_ref); f_z_tgt(inter,slope,z_tgt,int_src,v_tgt); REQUIRE (views_are_approx_equal(d,v_tgt,tol)); print(" -> vector interface field............... OK!\n"); @@ -230,8 +229,7 @@ TEST_CASE("field_at_height") { print(" -> Forced fail, give incorrect location...............\n"); const int z_tgt_adj = (z_tgt+max_surf_4test)/2; - std::string loc_err = std::to_string(z_tgt_adj) + "m"; - auto d = run_diag(s_int,int_src,loc_err,surf_ref); + auto d = run_diag(s_int,int_src,z_tgt_adj,surf_ref); f_z_tgt(inter,slope,z_tgt,int_src,s_tgt); REQUIRE (!views_are_approx_equal(d,s_tgt,tol,false)); print(" -> Forced fail, give incorrect location............... OK!\n"); @@ -243,15 +241,13 @@ TEST_CASE("field_at_height") auto inter = pdf_y0(engine); f_z_src(inter, slope, int_src, s_int); z_tgt = 2*z_top; - std::string loc = std::to_string(z_tgt) + "m"; - auto dtop = run_diag(s_int,int_src,loc,surf_ref); + auto dtop = run_diag(s_int,int_src,z_tgt,surf_ref); f_z_tgt(inter,slope,z_tgt,int_src,s_tgt); REQUIRE (views_are_approx_equal(dtop,s_tgt,tol)); print(" -> Forced extrapolation at top............... OK!\n"); print(" -> Forced extrapolation at bot...............\n"); z_tgt = 0; - loc = std::to_string(z_tgt) + "m"; - auto dbot = run_diag(s_int,int_src,loc,surf_ref); + auto dbot = run_diag(s_int,int_src,z_tgt,surf_ref); f_z_tgt(inter,slope,z_tgt,int_src,s_tgt); REQUIRE (views_are_approx_equal(dbot,s_tgt,tol)); print(" -> Forced extrapolation at bot............... OK!\n"); diff --git a/components/eamxx/src/diagnostics/tests/field_at_pressure_level_tests.cpp b/components/eamxx/src/diagnostics/tests/field_at_pressure_level_tests.cpp index 4e0deab1dfc..ba733980cad 100644 --- a/components/eamxx/src/diagnostics/tests/field_at_pressure_level_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/field_at_pressure_level_tests.cpp @@ -224,7 +224,8 @@ get_test_diag(const ekat::Comm& comm, std::shared_ptr fm, st ekat::ParameterList params; params.set("field_name",field.name()); params.set("grid_name",fm->get_grid()->name()); - params.set("vertical_location",std::to_string(plevel) + "Pa"); + params.set("pressure_value",std::to_string(plevel)); + params.set("pressure_units",std::string("Pa")); auto diag = std::make_shared(comm,params); diag->set_grids(gm); for (const auto& req : diag->get_required_field_requests()) { diff --git a/components/eamxx/src/diagnostics/tests/horiz_avg_test.cpp b/components/eamxx/src/diagnostics/tests/horiz_avg_test.cpp new file mode 100644 index 00000000000..9b9ec2022fa --- /dev/null +++ b/components/eamxx/src/diagnostics/tests/horiz_avg_test.cpp @@ -0,0 +1,166 @@ +#include "catch2/catch.hpp" +#include "diagnostics/register_diagnostics.hpp" +#include "share/field/field_utils.hpp" +#include "share/grid/mesh_free_grids_manager.hpp" +#include "share/util/scream_setup_random_test.hpp" +#include "share/util/scream_universal_constants.hpp" + +namespace scream { + +std::shared_ptr create_gm(const ekat::Comm &comm, const int ncols, + const int nlevs) { + const int num_global_cols = ncols * comm.size(); + + using vos_t = std::vector; + ekat::ParameterList gm_params; + gm_params.set("grids_names", vos_t{"Point Grid"}); + auto &pl = gm_params.sublist("Point Grid"); + pl.set("type", "point_grid"); + pl.set("aliases", vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); + + auto gm = create_mesh_free_grids_manager(comm, gm_params); + gm->build_grids(); + + return gm; +} + +TEST_CASE("horiz_avg") { + using namespace ShortFieldTagsNames; + using namespace ekat::units; + + // A numerical tolerance + auto tol = std::numeric_limits::epsilon() * 100; + + // A world comm + ekat::Comm comm(MPI_COMM_WORLD); + + // A time stamp + util::TimeStamp t0({2024, 1, 1}, {0, 0, 0}); + + // Create a grids manager - single column for these tests + constexpr int nlevs = 3; + constexpr int dim3 = 4; + const int ngcols = 6 * comm.size(); + + auto gm = create_gm(comm, ngcols, nlevs); + auto grid = gm->get_grid("Physics"); + + // Input (randomized) qc + FieldLayout scalar1d_layout{{COL}, {ngcols}}; + FieldLayout scalar2d_layout{{COL, LEV}, {ngcols, nlevs}}; + FieldLayout scalar3d_layout{{COL, CMP, LEV}, {ngcols, dim3, nlevs}}; + + FieldIdentifier qc1_fid("qc", scalar1d_layout, kg / kg, grid->name()); + FieldIdentifier qc2_fid("qc", scalar2d_layout, kg / kg, grid->name()); + FieldIdentifier qc3_fid("qc", scalar3d_layout, kg / kg, grid->name()); + + Field qc1(qc1_fid); + Field qc2(qc2_fid); + Field qc3(qc3_fid); + + qc1.allocate_view(); + qc2.allocate_view(); + qc3.allocate_view(); + + // Construct random number generator stuff + using RPDF = std::uniform_real_distribution; + RPDF pdf(sp(0.0), sp(200.0)); + auto engine = scream::setup_random_test(); + + // Construct the Diagnostics + std::map> diags; + auto &diag_factory = AtmosphereDiagnosticFactory::instance(); + register_diagnostics(); + + ekat::ParameterList params; + REQUIRE_THROWS(diag_factory.create("HorizAvgDiag", comm, + params)); // No 'field_name' parameter + + // Set time for qc and randomize its values + qc1.get_header().get_tracking().update_time_stamp(t0); + qc2.get_header().get_tracking().update_time_stamp(t0); + qc3.get_header().get_tracking().update_time_stamp(t0); + randomize(qc1, engine, pdf); + randomize(qc2, engine, pdf); + randomize(qc3, engine, pdf); + + // Create and set up the diagnostic + params.set("grid_name", grid->name()); + params.set("field_name", "qc"); + auto diag1 = diag_factory.create("HorizAvgDiag", comm, params); + auto diag2 = diag_factory.create("HorizAvgDiag", comm, params); + auto diag3 = diag_factory.create("HorizAvgDiag", comm, params); + diag1->set_grids(gm); + diag2->set_grids(gm); + diag3->set_grids(gm); + + // Clone the area field + auto area = grid->get_geometry_data("area").clone(); + + // Test the horiz contraction of qc1 + // Get the diagnostic field + diag1->set_required_field(qc1); + diag1->initialize(t0, RunType::Initial); + diag1->compute_diagnostic(); + auto diag1_f = diag1->get_diagnostic(); + + // Manual calculation + FieldIdentifier diag0_fid("qc_horiz_avg_manual", + scalar1d_layout.clone().strip_dim(COL), kg / kg, + grid->name()); + Field diag0(diag0_fid); + diag0.allocate_view(); + + // calculate total area + Real atot = field_sum(area, &comm); + // scale the area field + area.scale(1 / atot); + + // calculate weighted avg + horiz_contraction(diag0, qc1, area, &comm); + // Compare + REQUIRE(views_are_equal(diag1_f, diag0)); + + // Try other known cases + // Set qc1_v to 1.0 to get weighted average of 1.0 + Real wavg = 1; + qc1.deep_copy(wavg); + diag1->compute_diagnostic(); + auto diag1_v2_host = diag1_f.get_view(); + REQUIRE_THAT(diag1_v2_host(), + Catch::Matchers::WithinRel( + wavg, tol)); // Catch2's floating point comparison + + // other diags + // Set qc2_v to 5.0 to get weighted average of 5.0 + wavg = sp(5.0); + qc2.deep_copy(wavg); + diag2->set_required_field(qc2); + diag2->initialize(t0, RunType::Initial); + diag2->compute_diagnostic(); + auto diag2_f = diag2->get_diagnostic(); + + auto diag2_v_host = diag2_f.get_view(); + + for(int i = 0; i < nlevs; ++i) { + REQUIRE_THAT(diag2_v_host(i), Catch::Matchers::WithinRel(wavg, tol)); + } + + // Try a random case with qc3 + auto qc3_v = qc3.get_view(); + FieldIdentifier diag3_manual_fid("qc_horiz_avg_manual", + scalar3d_layout.clone().strip_dim(COL), + kg / kg, grid->name()); + Field diag3_manual(diag3_manual_fid); + diag3_manual.allocate_view(); + horiz_contraction(diag3_manual, qc3, area, &comm); + diag3->set_required_field(qc3); + diag3->initialize(t0, RunType::Initial); + diag3->compute_diagnostic(); + auto diag3_f = diag3->get_diagnostic(); + REQUIRE(views_are_equal(diag3_f, diag3_manual)); +} + +} // namespace scream diff --git a/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp b/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp index fe75114611d..631cb5acd8a 100644 --- a/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp @@ -50,14 +50,10 @@ void run (const std::string& diag_name, const std::string& location) // Construct the Diagnostic ekat::ParameterList params; - std::string name = diag_name; - if (location=="midpoints") { - name += "_mid"; - } else if (location=="interfaces") { - name += "_int"; - } - params.set("diag_name", name); - auto diag = diag_factory.create(name,comm,params); + + params.set("diag_name", diag_name); + params.set("vert_location",location); + auto diag = diag_factory.create("VerticalLayer",comm,params); diag->set_grids(gm); const bool needs_phis = diag_name=="z" or diag_name=="geopotential"; @@ -180,7 +176,7 @@ TEST_CASE("vertical_layer_test", "vertical_layer_test]"){ std::string msg = " -> Testing diag=dz "; std::string dots (50-msg.size(),'.'); root_print (msg + dots + "\n"); - run("dz", "UNUSED"); + run("dz", "midpoints"); root_print (msg + dots + " PASS!\n"); }; diff --git a/components/eamxx/src/diagnostics/vapor_flux.cpp b/components/eamxx/src/diagnostics/vapor_flux.cpp index ce88c144301..b6577bf1205 100644 --- a/components/eamxx/src/diagnostics/vapor_flux.cpp +++ b/components/eamxx/src/diagnostics/vapor_flux.cpp @@ -26,11 +26,6 @@ VaporFluxDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) } } -std::string VaporFluxDiagnostic::name() const -{ - return m_component==0 ? "ZonalVapFlux" : "MeridionalVapFlux"; -} - void VaporFluxDiagnostic::set_grids(const std::shared_ptr grids_manager) { using namespace ekat::units; @@ -51,7 +46,8 @@ void VaporFluxDiagnostic::set_grids(const std::shared_ptr gr add_field("horiz_winds", vector3d, m/s, grid_name); // Construct and allocate the diagnostic field - FieldIdentifier fid (name(), scalar2d, kg/m/s, grid_name); + std::string dname = m_component==0 ? "ZonalVapFlux" : "MeridionalVapFlux"; + FieldIdentifier fid (dname, scalar2d, kg/m/s, grid_name); m_diagnostic_output = Field(fid); m_diagnostic_output.allocate_view(); } diff --git a/components/eamxx/src/diagnostics/vapor_flux.hpp b/components/eamxx/src/diagnostics/vapor_flux.hpp index 3d82fd882f8..5bacd78a9b7 100644 --- a/components/eamxx/src/diagnostics/vapor_flux.hpp +++ b/components/eamxx/src/diagnostics/vapor_flux.hpp @@ -17,7 +17,7 @@ class VaporFluxDiagnostic : public AtmosphereDiagnostic VaporFluxDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); // The name of the diagnostic - std::string name () const; + std::string name () const override { return "VaporFlux"; } // Set the grid void set_grids (const std::shared_ptr grids_manager); diff --git a/components/eamxx/src/diagnostics/vertical_layer.cpp b/components/eamxx/src/diagnostics/vertical_layer.cpp index 32d870da03c..f9ff526dfcd 100644 --- a/components/eamxx/src/diagnostics/vertical_layer.cpp +++ b/components/eamxx/src/diagnostics/vertical_layer.cpp @@ -13,25 +13,27 @@ VerticalLayerDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& para : AtmosphereDiagnostic(comm,params) { m_diag_name = params.get("diag_name"); - std::vector supported = { - "z_int", - "z_mid", - "geopotential_int", - "geopotential_mid", - "height_int", - "height_mid", - "dz" - }; + std::vector supported = {"z","geopotential","height","dz"}; EKAT_REQUIRE_MSG(ekat::contains(supported,m_diag_name), "[VerticalLayerDiagnostic] Error! Invalid diag_name.\n" " - diag_name : " + m_diag_name + "\n" " - valid names: " + ekat::join(supported,", ") + "\n"); - m_is_interface_layout = m_diag_name.find("_int") != std::string::npos; + auto vert_pos = params.get("vert_location"); + EKAT_REQUIRE_MSG (vert_pos=="mid" || vert_pos=="int" || + vert_pos=="midpoints" || vert_pos=="interfaces", + "[VerticalLayerDiagnostic] Error! Invalid 'vert_location'.\n" + " - input value: " + vert_pos + "\n" + " - valid names: mid, midpoints, int, interfaces\n"); + m_is_interface_layout = vert_pos=="int" || vert_pos=="interfaces"; + + m_geopotential = m_diag_name=="geopotential"; + m_from_sea_level = m_diag_name=="z" or m_geopotential; - m_geopotential = m_diag_name.substr(0,12)=="geopotential"; - m_from_sea_level = m_diag_name[0]=='z' or m_geopotential; + if (m_diag_name!="dz") { + m_diag_name += m_is_interface_layout ? "_int" : "_mid"; + } } // ======================================================================================== void VerticalLayerDiagnostic:: @@ -88,7 +90,7 @@ initialize_impl (const RunType /*run_type*/) const auto VLEV = m_is_interface_layout ? ILEV : LEV; const auto nlevs = m_is_interface_layout ? m_num_levs+1 : m_num_levs; FieldLayout diag_layout ({COL,VLEV},{m_num_cols,nlevs}); - FieldIdentifier fid (name(), diag_layout, m_geopotential ? m2/s2 : m, grid_name); + FieldIdentifier fid (m_diag_name, diag_layout, m_geopotential ? m2/s2 : m, grid_name); m_diagnostic_output = Field(fid); auto& diag_fap = m_diagnostic_output.get_header().get_alloc_properties(); diff --git a/components/eamxx/src/diagnostics/vertical_layer.hpp b/components/eamxx/src/diagnostics/vertical_layer.hpp index 805fd70028f..a440bd6a8ee 100644 --- a/components/eamxx/src/diagnostics/vertical_layer.hpp +++ b/components/eamxx/src/diagnostics/vertical_layer.hpp @@ -24,7 +24,7 @@ class VerticalLayerDiagnostic : public AtmosphereDiagnostic VerticalLayerDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); // The name of the diagnostic. - std::string name () const { return m_diag_name; } + std::string name () const { return "VerticalLayer"; } // Set the grid void set_grids (const std::shared_ptr grids_manager); diff --git a/components/eamxx/src/diagnostics/water_path.cpp b/components/eamxx/src/diagnostics/water_path.cpp index 15fa5cdef38..bca771e6dab 100644 --- a/components/eamxx/src/diagnostics/water_path.cpp +++ b/components/eamxx/src/diagnostics/water_path.cpp @@ -32,11 +32,6 @@ WaterPathDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) } } -std::string WaterPathDiagnostic::name() const -{ - return m_kind + "WaterPath"; -} - void WaterPathDiagnostic:: set_grids(const std::shared_ptr grids_manager) { @@ -57,7 +52,7 @@ set_grids(const std::shared_ptr grids_manager) add_field(m_qname, scalar3d, kg/kg, grid_name); // Construct and allocate the diagnostic field - FieldIdentifier fid (name(), scalar2d, kg/m2, grid_name); + FieldIdentifier fid (m_kind + "WaterPath", scalar2d, kg/m2, grid_name); m_diagnostic_output = Field(fid); m_diagnostic_output.allocate_view(); } diff --git a/components/eamxx/src/diagnostics/water_path.hpp b/components/eamxx/src/diagnostics/water_path.hpp index 0b9515e0b42..722a51a3133 100644 --- a/components/eamxx/src/diagnostics/water_path.hpp +++ b/components/eamxx/src/diagnostics/water_path.hpp @@ -17,7 +17,7 @@ class WaterPathDiagnostic : public AtmosphereDiagnostic WaterPathDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); // The name of the diagnostic - std::string name () const; + std::string name () const override { return "WaterPath"; } // Set the grid void set_grids (const std::shared_ptr grids_manager); diff --git a/components/eamxx/src/dynamics/homme/CMakeLists.txt b/components/eamxx/src/dynamics/homme/CMakeLists.txt index b6a69a3605f..ffad3a4111f 100644 --- a/components/eamxx/src/dynamics/homme/CMakeLists.txt +++ b/components/eamxx/src/dynamics/homme/CMakeLists.txt @@ -23,7 +23,8 @@ set(BUILD_HOMME_PREQX_KOKKOS OFF CACHE BOOL "") set(BUILD_HOMME_PESE OFF CACHE BOOL "") set(BUILD_HOMME_SWIM OFF CACHE BOOL "") set(BUILD_HOMME_PRIM OFF CACHE BOOL "") -set(HOMME_ENABLE_COMPOSE ON CACHE BOOL "") +#set(HOMME_ENABLE_COMPOSE ON CACHE BOOL "") +set(HOMME_ENABLE_COMPOSE OFF CACHE BOOL "") set(BUILD_HOMME_TOOL OFF CACHE BOOL "") if (NOT Kokkos_ENABLE_SERIAL) @@ -146,7 +147,6 @@ macro (CreateDynamicsLib HOMME_TARGET NP PLEV QSIZE) ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_process_interface.cpp ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_fv_phys.cpp ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_rayleigh_friction.cpp - ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_iop.cpp ${SCREAM_DYNAMICS_SRC_DIR}/physics_dynamics_remapper.cpp ${SCREAM_DYNAMICS_SRC_DIR}/homme_grids_manager.cpp ${SCREAM_DYNAMICS_SRC_DIR}/interface/homme_context_mod.F90 diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp deleted file mode 100644 index 9c04a3f1ba6..00000000000 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ /dev/null @@ -1,610 +0,0 @@ -#include "eamxx_homme_process_interface.hpp" - -// EAMxx includes -#include "dynamics/homme/homme_dimensions.hpp" -#include "dynamics/homme/homme_dynamics_helpers.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/iop/intensive_observation_period.hpp" -#include "share/util/scream_column_ops.hpp" - -// Homme includes -#include "Context.hpp" -#include "ColumnOps.hpp" -#include "HommexxEnums.hpp" -#include "HybridVCoord.hpp" -#include "SimulationParams.hpp" -#include "Types.hpp" - -// SCREAM includes -#include "share/util/scream_common_physics_functions.hpp" - -// EKAT includes -#include "ekat/ekat_workspace.hpp" -#include "ekat/kokkos/ekat_kokkos_types.hpp" - -namespace scream { - -// Compute effects of large scale subsidence on T, q, u, and v. -KOKKOS_FUNCTION -void HommeDynamics:: -advance_iop_subsidence(const KT::MemberType& team, - const int nlevs, - const Real dt, - const Real ps, - const view_1d& pmid, - const view_1d& pint, - const view_1d& pdel, - const view_1d& omega, - const Workspace& workspace, - const view_1d& u, - const view_1d& v, - const view_1d& T, - const view_2d& Q) -{ - using ColOps = ColumnOps; - using C = physics::Constants; - constexpr Real Rair = C::Rair; - constexpr Real Cpair = C::Cpair; - - const auto n_q_tracers = Q.extent_int(0); - const auto nlev_packs = ekat::npack(nlevs); - - // Get some temporary views from WS - uview_1d omega_int, delta_u, delta_v, delta_T, tmp; - workspace.take_many_contiguous_unsafe<4>({"omega_int", "delta_u", "delta_v", "delta_T"}, - {&omega_int, &delta_u, &delta_v, &delta_T}); - const auto delta_Q_slot = workspace.take_macro_block("delta_Q", n_q_tracers); - uview_2d delta_Q(delta_Q_slot.data(), n_q_tracers, nlev_packs); - - auto s_pmid = ekat::scalarize(pmid); - auto s_omega = ekat::scalarize(omega); - auto s_delta_u = ekat::scalarize(delta_u); - auto s_delta_v = ekat::scalarize(delta_v); - auto s_delta_T = ekat::scalarize(delta_T); - auto s_delta_Q = ekat::scalarize(delta_Q); - auto s_omega_int = ekat::scalarize(omega_int); - - // Compute omega on the interface grid by using a weighted average in pressure - const int pack_begin = 1/Pack::n, pack_end = (nlevs-1)/Pack::n; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, pack_begin, pack_end+1), [&] (const int k){ - auto range_pack = ekat::range(k*Pack::n); - range_pack.set(range_pack<1, 1); - Pack pmid_k, pmid_km1, omega_k, omega_km1; - ekat::index_and_shift<-1>(s_pmid, range_pack, pmid_k, pmid_km1); - ekat::index_and_shift<-1>(s_omega, range_pack, omega_k, omega_km1); - - const auto weight = (pint(k) - pmid_km1)/(pmid_k - pmid_km1); - omega_int(k).set(range_pack>=1 and range_pack<=nlevs-1, - weight*omega_k + (1-weight)*omega_km1); - }); - omega_int(0)[0] = 0; - omega_int(nlevs/Pack::n)[nlevs%Pack::n] = 0; - - // Compute delta views for u, v, T, and Q (e.g., u(k+1) - u(k), k=0,...,nlevs-2) - ColOps::compute_midpoint_delta(team, nlevs-1, u, delta_u); - ColOps::compute_midpoint_delta(team, nlevs-1, v, delta_v); - ColOps::compute_midpoint_delta(team, nlevs-1, T, delta_T); - for (int iq=0; iq(k*Pack::n); - const auto at_top = range_pack==0; - const auto not_at_top = not at_top; - const auto at_bot = range_pack==nlevs-1; - const auto not_at_bot = not at_bot; - const bool any_at_top = at_top.any(); - const bool any_at_bot = at_bot.any(); - - // Get delta(k-1) packs. The range pack should not - // contain index 0 (so that we don't attempt to access - // k=-1 index) or index > nlevs-2 (since delta_* views - // are size nlevs-1). - auto range_pack_for_m1_shift = range_pack; - range_pack_for_m1_shift.set(range_pack<1, 1); - range_pack_for_m1_shift.set(range_pack>nlevs-2, nlevs-2); - Pack delta_u_k, delta_u_km1, - delta_v_k, delta_v_km1, - delta_T_k, delta_T_km1; - ekat::index_and_shift<-1>(s_delta_u, range_pack_for_m1_shift, delta_u_k, delta_u_km1); - ekat::index_and_shift<-1>(s_delta_v, range_pack_for_m1_shift, delta_v_k, delta_v_km1); - ekat::index_and_shift<-1>(s_delta_T, range_pack_for_m1_shift, delta_T_k, delta_T_km1); - - // At the top and bottom of the model, set the end points for - // delta_*_k and delta_*_km1 to be the first and last entries - // of delta_*, respectively. - if (any_at_top) { - delta_u_k.set(at_top, s_delta_u(0)); - delta_v_k.set(at_top, s_delta_v(0)); - delta_T_k.set(at_top, s_delta_T(0)); - } - if (any_at_bot) { - delta_u_km1.set(at_bot, s_delta_u(nlevs-2)); - delta_v_km1.set(at_bot, s_delta_v(nlevs-2)); - delta_T_km1.set(at_bot, s_delta_T(nlevs-2)); - } - - // Get omega_int(k+1) pack. The range pack should not - // contain index > nlevs-1 (since omega_int is size nlevs+1). - auto range_pack_for_p1_shift = range_pack; - range_pack_for_p1_shift.set(range_pack>nlevs-1, nlevs-1); - Pack omega_int_k, omega_int_kp1; - ekat::index_and_shift<1>(s_omega_int, range_pack, omega_int_k, omega_int_kp1); - - const auto fac = (dt/2)/pdel(k); - - // Update u - u(k).update(not_at_bot, fac*omega_int_kp1*delta_u_k, -1, 1); - u(k).update(not_at_top, fac*omega_int_k*delta_u_km1, -1, 1); - - // Update v - v(k).update(not_at_bot, fac*omega_int_kp1*delta_v_k, -1, 1); - v(k).update(not_at_top, fac*omega_int_k*delta_v_km1, -1, 1); - - // Before updating T, first scale using thermal - // expansion term due to LS vertical advection - T(k) *= 1 + (dt*Rair/Cpair)*omega(k)/pmid(k); - - // Update T - T(k).update(not_at_bot, fac*omega_int_kp1*delta_T_k, -1, 1); - T(k).update(not_at_top, fac*omega_int_k*delta_T_km1, -1, 1); - - // Update Q - Pack delta_tracer_k, delta_tracer_km1; - for (int iq=0; iq(s_delta_tracer, range_pack_for_m1_shift, delta_tracer_k, delta_tracer_km1); - if (any_at_top) delta_tracer_k.set(at_top, s_delta_tracer(0)); - if (any_at_bot) delta_tracer_km1.set(at_bot, s_delta_tracer(nlevs-2)); - - Q(iq, k).update(not_at_bot, fac*omega_int_kp1*delta_tracer_k, -1, 1); - Q(iq, k).update(not_at_top, fac*omega_int_k*delta_tracer_km1, -1, 1); - } - }); - - // Release WS views - workspace.release_macro_block(delta_Q_slot, n_q_tracers); - workspace.release_many_contiguous<4>({&omega_int, &delta_u, &delta_v, &delta_T}); -} - -// Apply large scale forcing for temperature and water vapor as provided by the IOP file -KOKKOS_FUNCTION -void HommeDynamics:: -advance_iop_forcing(const KT::MemberType& team, - const int nlevs, - const Real dt, - const view_1d& divT, - const view_1d& divq, - const view_1d& T, - const view_1d& qv) -{ - const auto nlev_packs = ekat::npack(nlevs); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const int k) { - T(k).update(divT(k), dt, 1.0); - qv(k).update(divq(k), dt, 1.0); - }); -} - -// Provide coriolis forcing to u and v winds, using large scale winds specified in IOP forcing file. -KOKKOS_FUNCTION -void HommeDynamics:: -iop_apply_coriolis(const KT::MemberType& team, - const int nlevs, - const Real dt, - const Real lat, - const view_1d& u_ls, - const view_1d& v_ls, - const view_1d& u, - const view_1d& v) -{ - using C = physics::Constants; - constexpr Real pi = C::Pi; - constexpr Real earth_rotation = C::omega; - - // Compute coriolis force - const auto fcor = 2*earth_rotation*std::sin(lat*pi/180); - - const auto nlev_packs = ekat::npack(nlevs); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const int k) { - const auto u_cor = v(k) - v_ls(k); - const auto v_cor = u(k) - u_ls(k); - u(k).update(u_cor, dt*fcor, 1.0); - v(k).update(v_cor, -dt*fcor, 1.0); - }); -} - -void HommeDynamics:: -apply_iop_forcing(const Real dt) -{ - using ESU = ekat::ExeSpaceUtils; - using PF = PhysicsFunctions; - using ColOps = ColumnOps; - - // Homme objects - const auto& c = Homme::Context::singleton(); - const auto& hvcoord = c.get(); - const auto& params = c.get(); - - // Dimensions - constexpr int NGP = HOMMEXX_NP; - constexpr int NLEV = HOMMEXX_NUM_LEV; - constexpr int NLEVI = HOMMEXX_NUM_LEV_P; - const auto nelem = m_dyn_grid->get_num_local_dofs()/(NGP*NGP); - const auto total_levels = m_dyn_grid->get_num_vertical_levels(); - const auto qsize = params.qsize; - - // Sanity checks since we will be switching between ekat::Pack - // and Homme::Scalar view types - EKAT_ASSERT_MSG(NLEV == ekat::npack(total_levels), - "Error! Dimension for vectorized Homme levels does not match level dimension " - "of the packed views used here. Check that Pack typedef is using a pack size " - "consistent with Homme's vector size.\n"); - EKAT_ASSERT_MSG(NLEVI == ekat::npack(total_levels+1), - "Error! Dimension for vectorized Homme levels does not match level dimension " - "of the packed views used here. Check that Pack typedef is using a pack size " - "consistent with Homme's vector size.\n"); - - // Hybrid coord values - const auto ps0 = hvcoord.ps0; - const auto hyam = m_dyn_grid->get_geometry_data("hyam").get_view(); - const auto hybm = m_dyn_grid->get_geometry_data("hybm").get_view(); - const auto hyai = m_dyn_grid->get_geometry_data("hyai").get_view(); - const auto hybi = m_dyn_grid->get_geometry_data("hybi").get_view(); - - // Homme element states - auto ps_dyn = get_internal_field("ps_dyn").get_view(); - auto dp3d_dyn = get_internal_field("dp3d_dyn").get_view(); - auto vtheta_dp_dyn = get_internal_field("vtheta_dp_dyn").get_view(); - auto phi_int_dyn = get_internal_field("phi_int_dyn").get_view(); - auto v_dyn = get_internal_field("v_dyn").get_view(); - auto Q_dyn = m_helper_fields.at("Q_dyn").get_view(); - auto Qdp_dyn = get_internal_field("Qdp_dyn").get_view(); - - // Load data from IOP files, if necessary - m_iop->read_iop_file_data(timestamp()); - - // Define local IOP param values - const auto iop_dosubsidence = m_iop->get_params().get("iop_dosubsidence"); - const auto iop_coriolis = m_iop->get_params().get("iop_coriolis"); - const auto iop_nudge_tq = m_iop->get_params().get("iop_nudge_tq"); - const auto iop_nudge_uv = m_iop->get_params().get("iop_nudge_uv"); - const auto use_large_scale_wind = m_iop->get_params().get("use_large_scale_wind"); - const auto use_3d_forcing = m_iop->get_params().get("use_3d_forcing"); - const auto lat = m_iop->get_params().get("target_latitude"); - const auto iop_nudge_tscale = m_iop->get_params().get("iop_nudge_tscale"); - const auto iop_nudge_tq_low = m_iop->get_params().get("iop_nudge_tq_low"); - const auto iop_nudge_tq_high = m_iop->get_params().get("iop_nudge_tq_high"); - - // Define local IOP field views - const Real ps_iop = m_iop->get_iop_field("Ps").get_view()(); - view_1d omega, divT, divq, u_ls, v_ls, qv_iop, t_iop, u_iop, v_iop; - divT = use_3d_forcing ? m_iop->get_iop_field("divT3d").get_view() - : m_iop->get_iop_field("divT").get_view(); - divq = use_3d_forcing ? m_iop->get_iop_field("divq3d").get_view() - : m_iop->get_iop_field("divq").get_view(); - if (iop_dosubsidence) { - omega = m_iop->get_iop_field("omega").get_view(); - } - if (iop_coriolis) { - u_ls = m_iop->get_iop_field("u_ls").get_view(); - v_ls = m_iop->get_iop_field("v_ls").get_view(); - } - if (iop_nudge_tq) { - qv_iop = m_iop->get_iop_field("q").get_view(); - t_iop = m_iop->get_iop_field("T").get_view(); - } - if (iop_nudge_uv) { - u_iop = use_large_scale_wind ? m_iop->get_iop_field("u_ls").get_view() - : m_iop->get_iop_field("u").get_view(); - v_iop = use_large_scale_wind ? m_iop->get_iop_field("v_ls").get_view() - : m_iop->get_iop_field("v").get_view(); - } - - // Team policy and workspace manager for eamxx - const auto policy_iop = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); - - // TODO: Create a memory buffer for this class - // and add the below WSM and views - WorkspaceMgr iop_wsm(NLEVI, 7+qsize, policy_iop); - view_Nd - temperature("temperature", nelem, NGP, NGP, NLEV); - - // Lambda for computing temperature - auto compute_temperature = [&] () { - Kokkos::parallel_for("compute_temperature_for_iop", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { - const int ie = team.league_rank()/(NGP*NGP); - const int igp = (team.league_rank()/NGP)%NGP; - const int jgp = team.league_rank()%NGP; - - // Get temp views from workspace - auto ws = iop_wsm.get_workspace(team); - uview_1d pmid; - ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); - - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - }); - team.team_barrier(); - - // Compute temperature from virtual potential temperature - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { - auto T_val = vtheta_dp_i(k); - T_val /= dp3d_i(k); - T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); - temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); - }); - - // Release WS views - ws.release_many_contiguous<1>({&pmid}); - }); - }; - - // Preprocess some homme states to get temperature - compute_temperature(); - Kokkos::fence(); - - // Apply IOP forcing - Kokkos::parallel_for("apply_iop_forcing", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { - const int ie = team.league_rank()/(NGP*NGP); - const int igp = (team.league_rank()/NGP)%NGP; - const int jgp = team.league_rank()%NGP; - - // Get temp views from workspace - auto ws = iop_wsm.get_workspace(team); - uview_1d pmid, pint, pdel; - ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, - {&pmid, &pint, &pdel}); - - auto ps_i = ps_dyn(ie, igp, jgp); - auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); - auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - auto s_pint = ekat::scalarize(pint); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { - s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; - if (k < total_levels) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - } - }); - team.team_barrier(); - ColOps::compute_midpoint_delta(team, total_levels, pint, pdel); - team.team_barrier(); - - if (iop_dosubsidence) { - // Compute subsidence due to large-scale forcing - advance_iop_subsidence(team, total_levels, dt, ps_i, pmid, pint, pdel, omega, ws, u_i, v_i, temperature_i, Q_i); - } - - // Update T and qv according to large scale forcing as specified in IOP file. - advance_iop_forcing(team, total_levels, dt, divT, divq, temperature_i, qv_i); - - if (iop_coriolis) { - // Apply coriolis forcing to u and v winds - iop_apply_coriolis(team, total_levels, dt, lat, u_ls, v_ls, u_i, v_i); - } - - // Release WS views - ws.release_many_contiguous<3>({&pmid, &pint, &pdel}); - }); - Kokkos::fence(); - - // Postprocess homme states Qdp and vtheta_dp - Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { - const int ie = team.league_rank()/(NGP*NGP); - const int igp = (team.league_rank()/NGP)%NGP; - const int jgp = team.league_rank()%NGP; - - // Get temp views from workspace - auto ws = iop_wsm.get_workspace(team); - uview_1d pmid, pint, pdel; - ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, - {&pmid, &pint, &pdel}); - - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - auto s_pint = ekat::scalarize(pint); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { - s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; - if (k < total_levels) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - } - }); - - team.team_barrier(); - - // Compute Qdp from updated Q - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV*qsize), [&] (const int k) { - const int ilev = k/qsize; - const int q = k%qsize; - - Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); - // For BFB on restarts, Q needs to be updated after we compute Qdp - Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); - }); - team.team_barrier(); - - // Convert updated temperature back to psuedo density virtual potential temperature - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { - const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); - vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); - }); - - // Release WS views - ws.release_many_contiguous<3>({&pmid, &pint, &pdel}); - }); - - if (iop_nudge_tq or iop_nudge_uv) { - // Nudge the domain based on the domain mean - // and observed quantities of T, Q, u, and - - if (iop_nudge_tq) { - // Compute temperature - compute_temperature(); - Kokkos::fence(); - } - - // Compute domain mean of qv, temperature, u, and v - - // TODO: add to local mem buffer - view_1d qv_mean, t_mean, u_mean, v_mean; - if (iop_nudge_tq) { - qv_mean = view_1d("u_mean", NLEV), - t_mean = view_1d("v_mean", NLEV); - } - if (iop_nudge_uv){ - u_mean = view_1d("u_mean", NLEV), - v_mean = view_1d("v_mean", NLEV); - } - - const auto qv_mean_h = Kokkos::create_mirror_view(qv_mean); - const auto t_mean_h = Kokkos::create_mirror_view(t_mean); - const auto u_mean_h = Kokkos::create_mirror_view(u_mean); - const auto v_mean_h = Kokkos::create_mirror_view(v_mean); - - for (int k=0; kget_num_global_dofs(); - t_mean_k /= m_dyn_grid->get_num_global_dofs(); - } - if (iop_nudge_uv){ - Real& u_mean_k = u_mean_h(k/Pack::n)[k%Pack::n]; - Real& v_mean_k = v_mean_h(k/Pack::n)[k%Pack::n]; - Kokkos::parallel_reduce("compute_domain_means_uv", - nelem*NGP*NGP, - KOKKOS_LAMBDA (const int idx, Real& u_sum, Real& v_sum) { - const int ie = idx/(NGP*NGP); - const int igp = (idx/NGP)%NGP; - const int jgp = idx%NGP; - - u_sum += v_dyn(ie, 0, igp, jgp, k/Pack::n)[k%Pack::n]; - v_sum += v_dyn(ie, 1, igp, jgp, k/Pack::n)[k%Pack::n]; - }, - u_mean_k, - v_mean_k); - - m_comm.all_reduce(&u_mean_k, 1, MPI_SUM); - m_comm.all_reduce(&v_mean_k, 1, MPI_SUM); - - u_mean_k /= m_dyn_grid->get_num_global_dofs(); - v_mean_k /= m_dyn_grid->get_num_global_dofs(); - } - } - Kokkos::deep_copy(qv_mean, qv_mean_h); - Kokkos::deep_copy(t_mean, t_mean_h); - Kokkos::deep_copy(u_mean, u_mean_h); - Kokkos::deep_copy(v_mean, v_mean_h); - - // Apply relaxation - const auto rtau = std::max(dt, iop_nudge_tscale); - Kokkos::parallel_for("apply_domain_relaxation", - policy_iop, - KOKKOS_LAMBDA (const KT::MemberType& team) { - - const int ie = team.league_rank()/(NGP*NGP); - const int igp = (team.league_rank()/NGP)%NGP; - const int jgp = team.league_rank()%NGP; - - // Get temp views from workspace - auto ws = iop_wsm.get_workspace(team); - uview_1d pmid; - ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); - - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); - auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); - - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - }); - team.team_barrier(); - - if (iop_nudge_tq or iop_nudge_uv) { - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&](const int& k) { - if (iop_nudge_tq) { - // Restrict nudging of T and qv to certain levels if requested by user - // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high - // is in units of [hPa], thus convert iop_nudge_tq_low/high - Mask nudge_level(false); - int max_size = hyam.size(); - for (int lev=k*Pack::n, p = 0; p < Pack::n && lev < max_size; ++lev, ++p) { - const auto pressure_from_iop = hyam(lev)*ps0 + hybm(lev)*ps_iop; - nudge_level.set(p, pressure_from_iop <= iop_nudge_tq_low*100 - and - pressure_from_iop >= iop_nudge_tq_high*100); - } - - qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); - temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); - - // Convert updated temperature back to virtual potential temperature - const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); - vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); - } - if (iop_nudge_uv) { - u_i(k).update(u_mean(k) - u_iop(k), -dt/rtau, 1.0); - v_i(k).update(v_mean(k) - v_iop(k), -dt/rtau, 1.0); - } - }); - } - - // Release WS views - ws.release_many_contiguous<1>({&pmid}); - }); - } -} - -} // namespace scream diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp index 8b7495ffd73..a30418c18ee 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp @@ -426,11 +426,6 @@ void HommeDynamics::initialize_impl (const RunType run_type) if (run_type==RunType::Initial) { initialize_homme_state (); } else { - if (m_iop) { - // We need to reload IOP data after restarting - m_iop->read_iop_file_data(timestamp()); - } - restart_homme_state (); } @@ -669,10 +664,6 @@ void HommeDynamics::homme_post_process (const double dt) { get_internal_field("w_int_dyn").get_header().get_alloc_properties().reset_subview_idx(tl.n0); } - if (m_iop) { - apply_iop_forcing(dt); - } - if (fv_phys_active()) { fv_phys_post_process(); // Apply Rayleigh friction to update temperature and horiz_winds @@ -820,7 +811,7 @@ void HommeDynamics::init_homme_views () { std::stringstream msg; msg << "\n************** HOMMEXX SimulationParams **********************\n\n"; msg << " time_step_type: " << Homme::etoi(params.time_step_type) << "\n"; - msg << " moisture: " << (params.moisture==Homme::MoistDry::DRY ? "dry" : "moist") << "\n"; + msg << " moisture: " << (params.use_moisture ? "moist" : "dry") << "\n"; msg << " remap_alg: " << Homme::etoi(params.remap_alg) << "\n"; msg << " test case: " << Homme::etoi(params.test_case) << "\n"; msg << " ftype: " << Homme::etoi(params.ftype) << "\n"; diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp index 93dff0cd72e..b3e01f8aa30 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp @@ -109,44 +109,6 @@ class HommeDynamics : public AtmosphereProcess void rayleigh_friction_init (); void rayleigh_friction_apply (const Real dt) const; - // IOP functions - void apply_iop_forcing(const Real dt); - - KOKKOS_FUNCTION - static void advance_iop_subsidence(const KT::MemberType& team, - const int nlevs, - const Real dt, - const Real ps, - const view_1d& pmid, - const view_1d& pint, - const view_1d& pdel, - const view_1d& omega, - const Workspace& workspace, - const view_1d& u, - const view_1d& v, - const view_1d& T, - const view_2d& Q); - - KOKKOS_FUNCTION - static void advance_iop_forcing(const KT::MemberType& team, - const int nlevs, - const Real dt, - const view_1d& divT, - const view_1d& divq, - const view_1d& T, - const view_1d& qv); - - - KOKKOS_FUNCTION - static void iop_apply_coriolis(const KT::MemberType& team, - const int nlevs, - const Real dt, - const Real lat, - const view_1d& u_ls, - const view_1d& v_ls, - const view_1d& u, - const view_1d& v); - public: // Fast boolean function returning whether Physics PGN is being used. bool fv_phys_active() const; diff --git a/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp b/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp index df5de6827f6..7376e9af9dd 100644 --- a/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp +++ b/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp @@ -188,7 +188,7 @@ void HommeGridsManager::build_dynamics_grid () { initialize_vertical_coordinates(dyn_grid); dyn_grid->m_short_name = "dyn"; - add_grid(dyn_grid); + add_nonconst_grid(dyn_grid); } void HommeGridsManager:: @@ -307,7 +307,7 @@ build_physics_grid (const ci_string& type, const ci_string& rebalance) { } phys_grid->m_short_name = type; - add_grid(phys_grid); + add_nonconst_grid(phys_grid); } void HommeGridsManager:: diff --git a/components/eamxx/src/dynamics/homme/interface/homme_context_mod.F90 b/components/eamxx/src/dynamics/homme/interface/homme_context_mod.F90 index 5e9f6bd2f9d..377f3c9ed2c 100644 --- a/components/eamxx/src/dynamics/homme/interface/homme_context_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/homme_context_mod.F90 @@ -168,37 +168,37 @@ end subroutine init_parallel_f90 function is_parallel_inited_f90 () result(inited) bind(c) logical (kind=c_bool) :: inited - inited = is_parallel_inited + inited = LOGICAL(is_parallel_inited,kind=c_bool) end function is_parallel_inited_f90 function is_params_inited_f90 () result(inited) bind(c) logical (kind=c_bool) :: inited - inited = is_params_inited + inited = LOGICAL(is_params_inited,kind=c_bool) end function is_params_inited_f90 function is_geometry_inited_f90 () result(inited) bind(c) logical (kind=c_bool) :: inited - inited = is_geometry_inited + inited = LOGICAL(is_geometry_inited,kind=c_bool) end function is_geometry_inited_f90 function is_data_structures_inited_f90 () result(inited) bind(c) logical (kind=c_bool) :: inited - inited = is_data_structures_inited + inited = LOGICAL(is_data_structures_inited,kind=c_bool) end function is_data_structures_inited_f90 function is_model_inited_f90 () result(inited) bind(c) logical (kind=c_bool) :: inited - inited = is_model_inited + inited = LOGICAL(is_model_inited,kind=c_bool) end function is_model_inited_f90 function is_hommexx_functors_inited_f90 () result(inited) bind(c) logical (kind=c_bool) :: inited - inited = is_hommexx_functors_inited + inited = LOGICAL(is_hommexx_functors_inited,kind=c_bool) end function is_hommexx_functors_inited_f90 end module homme_context_mod diff --git a/components/eamxx/src/dynamics/homme/interface/homme_driver_mod.F90 b/components/eamxx/src/dynamics/homme/interface/homme_driver_mod.F90 index eefcd65e8d7..aa6e537baa4 100644 --- a/components/eamxx/src/dynamics/homme/interface/homme_driver_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/homme_driver_mod.F90 @@ -192,7 +192,7 @@ subroutine prim_init_model_f90 () bind(c) elem, hybrid, hvcoord, deriv, tl ! Local variable - logical(kind=c_bool), parameter :: allocate_buffer = .false. + logical, parameter :: allocate_buffer = 0 if (.not. is_data_structures_inited) then call abortmp ("Error! 'prim_init_data_structures_f90' has not been called yet.\n") diff --git a/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp b/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp index 65158fdc390..a1e401dcd17 100644 --- a/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp +++ b/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp @@ -426,6 +426,10 @@ do_remap_fwd() const int team_size = std::min(256, std::min(128*m_num_phys_cols,32*(concurrency/this->m_num_fields+31)/32)); #endif +#ifdef KOKKOS_ENABLE_SYCL + const int team_size = 4; +#endif + //should exclude above cases of CUDA and HIP #ifndef EAMXX_ENABLE_GPU const int team_size = (concurrencym_num_fields ? 1 : concurrency/this->m_num_fields); diff --git a/components/eamxx/src/mct_coupling/CMakeLists.txt b/components/eamxx/src/mct_coupling/CMakeLists.txt index 308cd177623..39f864e728a 100644 --- a/components/eamxx/src/mct_coupling/CMakeLists.txt +++ b/components/eamxx/src/mct_coupling/CMakeLists.txt @@ -38,6 +38,7 @@ set (SCREAM_LIBS eamxx_cosp cld_fraction spa + iop_forcing nudging diagnostics tms diff --git a/components/eamxx/src/physics/CMakeLists.txt b/components/eamxx/src/physics/CMakeLists.txt index e0e89e60f80..f9beda35a20 100644 --- a/components/eamxx/src/physics/CMakeLists.txt +++ b/components/eamxx/src/physics/CMakeLists.txt @@ -8,8 +8,10 @@ add_subdirectory(p3) if (SCREAM_DOUBLE_PRECISION) add_subdirectory(rrtmgp) add_subdirectory(cosp) + add_subdirectory(tms) + add_subdirectory(iop_forcing) else() - message(STATUS "WARNING: RRTMGP and COSP only supported for double precision builds; skipping") + message(STATUS "WARNING: RRTMGP, COSP, TMS, and IOPForcing only supported for double precision builds; skipping") endif() add_subdirectory(shoc) add_subdirectory(cld_fraction) @@ -21,8 +23,4 @@ add_subdirectory(nudging) if (SCREAM_ENABLE_MAM) add_subdirectory(mam) endif() -if (SCREAM_DOUBLE_PRECISION) - add_subdirectory(tms) -else() - message(STATUS "WARNING: TMS only supported for double precision builds; skipping") -endif() + diff --git a/components/eamxx/src/physics/iop_forcing/CMakeLists.txt b/components/eamxx/src/physics/iop_forcing/CMakeLists.txt new file mode 100644 index 00000000000..093ceac73c9 --- /dev/null +++ b/components/eamxx/src/physics/iop_forcing/CMakeLists.txt @@ -0,0 +1,5 @@ +add_library(iop_forcing eamxx_iop_forcing_process_interface.cpp) +target_compile_definitions(iop_forcing PUBLIC EAMXX_HAS_IOP_FORCING) +target_link_libraries(iop_forcing physics_share scream_share) + +target_link_libraries(eamxx_physics INTERFACE iop_forcing) diff --git a/components/eamxx/src/physics/iop_forcing/eamxx_iop_forcing_process_interface.cpp b/components/eamxx/src/physics/iop_forcing/eamxx_iop_forcing_process_interface.cpp new file mode 100644 index 00000000000..c9a3714e1dc --- /dev/null +++ b/components/eamxx/src/physics/iop_forcing/eamxx_iop_forcing_process_interface.cpp @@ -0,0 +1,520 @@ +#include "physics/iop_forcing/eamxx_iop_forcing_process_interface.hpp" + +#include "share/field/field_utils.hpp" +#include "share/property_checks/field_within_interval_check.hpp" + +namespace scream +{ +// ========================================================================================= +void IOPForcing::set_grids(const std::shared_ptr grids_manager) +{ + using namespace ekat::units; + + m_grid = grids_manager->get_grid("Physics"); + const auto& grid_name = m_grid->name(); + + m_num_cols = m_grid->get_num_local_dofs(); // Number of columns on this rank + m_num_levs = m_grid->get_num_vertical_levels(); // Number of levels per column + + // Define the different field layouts that will be used for this process + FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); + FieldLayout scalar3d_mid = m_grid->get_3d_scalar_layout(true); + FieldLayout vector3d_mid = m_grid->get_3d_vector_layout(true,2); + + constexpr int pack_size = Pack::n; + + add_field("ps", scalar2d, Pa, grid_name); + + add_field("horiz_winds", vector3d_mid, m/s, grid_name, pack_size); + add_field("T_mid", scalar3d_mid, K, grid_name, pack_size); + + add_tracer("qv", m_grid, kg/kg, pack_size); + add_group("tracers", grid_name, pack_size, Bundling::Required); + + // Sanity check that iop data manager is setup by driver + EKAT_REQUIRE_MSG(m_iop_data_manager, + "Error! IOPDataManager not setup by driver, but IOPForcing" + "being used as an ATM process.\n"); + + // Create helper fields for finding horizontal means + auto level_only_scalar_layout = scalar3d_mid.clone().strip_dim(0); + auto level_only_vector_layout = vector3d_mid.clone().strip_dim(0); + const auto iop_nudge_tq = m_iop_data_manager->get_params().get("iop_nudge_tq"); + const auto iop_nudge_uv = m_iop_data_manager->get_params().get("iop_nudge_uv"); + if (iop_nudge_tq or iop_nudge_uv) { + create_helper_field("horiz_mean_weights", scalar2d, grid_name, pack_size); + } + if (iop_nudge_tq) { + create_helper_field("qv_mean", level_only_scalar_layout, grid_name, pack_size); + create_helper_field("t_mean", level_only_scalar_layout, grid_name, pack_size); + } + if (iop_nudge_uv) { + create_helper_field("horiz_winds_mean", level_only_vector_layout, grid_name, pack_size); + } +} +// ========================================================================================= +void IOPForcing:: +set_computed_group_impl (const FieldGroup& group) +{ + EKAT_REQUIRE_MSG(group.m_info->size() >= 1, + "Error! IOPForcing requires at least qv as tracer input.\n"); + + const auto& name = group.m_info->m_group_name; + + EKAT_REQUIRE_MSG(name=="tracers", + "Error! IOPForcing was not expecting a field group called '" << name << "\n"); + + EKAT_REQUIRE_MSG(group.m_info->m_bundled, + "Error! IOPForcing expects bundled fields for tracers.\n"); + + m_num_tracers = group.m_info->size(); +} +// ========================================================================================= +size_t IOPForcing::requested_buffer_size_in_bytes() const +{ + // Number of bytes needed by the WorkspaceManager passed to shoc_main + const int nlevi_packs = ekat::npack(m_num_levs+1); + const auto policy = ESU::get_default_team_policy(m_num_cols, nlevi_packs); + const size_t wsm_bytes = WorkspaceMgr::get_total_bytes_needed(nlevi_packs, 7+m_num_tracers, policy); + + return wsm_bytes; +} +// ========================================================================================= +void IOPForcing::init_buffers(const ATMBufferManager &buffer_manager) +{ + EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Buffers size not sufficient.\n"); + + const int nlevi_packs = ekat::npack(m_num_levs+1); + Pack* mem = reinterpret_cast(buffer_manager.get_memory()); + + // WSM data + m_buffer.wsm_data = mem; + + const auto policy = ESU::get_default_team_policy(m_num_cols, nlevi_packs); + const size_t wsm_npacks = WorkspaceMgr::get_total_bytes_needed(nlevi_packs, 7+m_num_tracers, policy)/sizeof(Pack); + mem += wsm_npacks; + + size_t used_mem = (reinterpret_cast(mem) - buffer_manager.get_memory())*sizeof(Real); + EKAT_REQUIRE_MSG(used_mem==requested_buffer_size_in_bytes(), "Error! Used memory != requested memory for IOPForcing.\n"); +} +// ========================================================================================= +void IOPForcing::create_helper_field (const std::string& name, + const FieldLayout& layout, + const std::string& grid_name, + const int ps) +{ + using namespace ekat::units; + FieldIdentifier id(name,layout,Units::nondimensional(),grid_name); + + // Create the field. Init with NaN's, so we spot instances of uninited memory usage + Field f(id); + f.get_header().get_alloc_properties().request_allocation(ps); + f.allocate_view(); + f.deep_copy(ekat::ScalarTraits::invalid()); + + m_helper_fields[name] = f; +} +// ========================================================================================= +void IOPForcing::initialize_impl (const RunType run_type) +{ + // Set field property checks for the fields in this process + using Interval = FieldWithinIntervalCheck; + add_postcondition_check(get_field_out("T_mid"),m_grid,100.0,500.0,false); + add_postcondition_check(get_field_out("horiz_winds"),m_grid,-400.0,400.0,false); + // For qv, ensure it doesn't get negative, by allowing repair of any neg value. + // TODO: use a repairable lb that clips only "small" negative values + add_postcondition_check(get_field_out("qv"),m_grid,0,0.2,true); + + // Setup WSM for internal local variables + const auto nlevi_packs = ekat::npack(m_num_levs+1); + const auto policy = ESU::get_default_team_policy(m_num_cols, nlevi_packs); + m_workspace_mgr.setup(m_buffer.wsm_data, nlevi_packs, 7+m_num_tracers, policy); + + // Compute field for horizontal contraction weights (1/num_global_dofs) + const auto iop_nudge_tq = m_iop_data_manager->get_params().get("iop_nudge_tq"); + const auto iop_nudge_uv = m_iop_data_manager->get_params().get("iop_nudge_uv"); + const Real one_over_num_dofs = 1.0/m_grid->get_num_global_dofs(); + if (iop_nudge_tq or iop_nudge_uv) m_helper_fields.at("horiz_mean_weights").deep_copy(one_over_num_dofs); +} +// ========================================================================================= +KOKKOS_FUNCTION +void IOPForcing:: +advance_iop_subsidence(const MemberType& team, + const int nlevs, + const Real dt, + const Real ps, + const view_1d& ref_p_mid, + const view_1d& ref_p_int, + const view_1d& ref_p_del, + const view_1d& omega, + const Workspace& workspace, + const view_1d& u, + const view_1d& v, + const view_1d& T, + const view_2d& Q) +{ + constexpr Real Rair = C::Rair; + constexpr Real Cpair = C::Cpair; + + const auto n_q_tracers = Q.extent_int(0); + const auto nlev_packs = ekat::npack(nlevs); + + // Get some temporary views from WS + uview_1d omega_int, delta_u, delta_v, delta_T, tmp; + workspace.take_many_contiguous_unsafe<4>({"omega_int", "delta_u", "delta_v", "delta_T"}, + {&omega_int, &delta_u, &delta_v, &delta_T}); + const auto delta_Q_slot = workspace.take_macro_block("delta_Q", n_q_tracers); + uview_2d delta_Q(delta_Q_slot.data(), n_q_tracers, nlev_packs); + + auto s_ref_p_mid = ekat::scalarize(ref_p_mid); + auto s_omega = ekat::scalarize(omega); + auto s_delta_u = ekat::scalarize(delta_u); + auto s_delta_v = ekat::scalarize(delta_v); + auto s_delta_T = ekat::scalarize(delta_T); + auto s_delta_Q = ekat::scalarize(delta_Q); + auto s_omega_int = ekat::scalarize(omega_int); + + // Compute omega on the interface grid by using a weighted average in pressure + const int pack_begin = 1/Pack::n, pack_end = (nlevs-1)/Pack::n; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, pack_begin, pack_end+1), [&] (const int k){ + auto range_pack = ekat::range(k*Pack::n); + range_pack.set(range_pack<1, 1); + Pack ref_p_mid_k, ref_p_mid_km1, omega_k, omega_km1; + ekat::index_and_shift<-1>(s_ref_p_mid, range_pack, ref_p_mid_k, ref_p_mid_km1); + ekat::index_and_shift<-1>(s_omega, range_pack, omega_k, omega_km1); + + const auto weight = (ref_p_int(k) - ref_p_mid_km1)/(ref_p_mid_k - ref_p_mid_km1); + omega_int(k).set(range_pack>=1 and range_pack<=nlevs-1, + weight*omega_k + (1-weight)*omega_km1); + }); + omega_int(0)[0] = 0; + omega_int(nlevs/Pack::n)[nlevs%Pack::n] = 0; + + // Compute delta views for u, v, T, and Q (e.g., u(k+1) - u(k), k=0,...,nlevs-2) + ColOps::compute_midpoint_delta(team, nlevs-1, u, delta_u); + ColOps::compute_midpoint_delta(team, nlevs-1, v, delta_v); + ColOps::compute_midpoint_delta(team, nlevs-1, T, delta_T); + for (int iq=0; iq(k*Pack::n); + const auto at_top = range_pack==0; + const auto not_at_top = not at_top; + const auto at_bot = range_pack==nlevs-1; + const auto not_at_bot = not at_bot; + const bool any_at_top = at_top.any(); + const bool any_at_bot = at_bot.any(); + + // Get delta(k-1) packs. The range pack should not + // contain index 0 (so that we don't attempt to access + // k=-1 index) or index > nlevs-2 (since delta_* views + // are size nlevs-1). + auto range_pack_for_m1_shift = range_pack; + range_pack_for_m1_shift.set(range_pack<1, 1); + range_pack_for_m1_shift.set(range_pack>nlevs-2, nlevs-2); + Pack delta_u_k, delta_u_km1, + delta_v_k, delta_v_km1, + delta_T_k, delta_T_km1; + ekat::index_and_shift<-1>(s_delta_u, range_pack_for_m1_shift, delta_u_k, delta_u_km1); + ekat::index_and_shift<-1>(s_delta_v, range_pack_for_m1_shift, delta_v_k, delta_v_km1); + ekat::index_and_shift<-1>(s_delta_T, range_pack_for_m1_shift, delta_T_k, delta_T_km1); + + // At the top and bottom of the model, set the end points for + // delta_*_k and delta_*_km1 to be the first and last entries + // of delta_*, respectively. + if (any_at_top) { + delta_u_k.set(at_top, s_delta_u(0)); + delta_v_k.set(at_top, s_delta_v(0)); + delta_T_k.set(at_top, s_delta_T(0)); + } + if (any_at_bot) { + delta_u_km1.set(at_bot, s_delta_u(nlevs-2)); + delta_v_km1.set(at_bot, s_delta_v(nlevs-2)); + delta_T_km1.set(at_bot, s_delta_T(nlevs-2)); + } + + // Get omega_int(k+1) pack. The range pack should not + // contain index > nlevs-1 (since omega_int is size nlevs+1). + auto range_pack_for_p1_shift = range_pack; + range_pack_for_p1_shift.set(range_pack>nlevs-1, nlevs-1); + Pack omega_int_k, omega_int_kp1; + ekat::index_and_shift<1>(s_omega_int, range_pack, omega_int_k, omega_int_kp1); + + const auto fac = (dt/2)/ref_p_del(k); + + // Update u + u(k).update(not_at_bot, fac*omega_int_kp1*delta_u_k, -1, 1); + u(k).update(not_at_top, fac*omega_int_k*delta_u_km1, -1, 1); + + // Update v + v(k).update(not_at_bot, fac*omega_int_kp1*delta_v_k, -1, 1); + v(k).update(not_at_top, fac*omega_int_k*delta_v_km1, -1, 1); + + // Before updating T, first scale using thermal + // expansion term due to LS vertical advection + T(k) *= 1 + (dt*Rair/Cpair)*omega(k)/ref_p_mid(k); + + // Update T + T(k).update(not_at_bot, fac*omega_int_kp1*delta_T_k, -1, 1); + T(k).update(not_at_top, fac*omega_int_k*delta_T_km1, -1, 1); + + // Update Q + Pack delta_tracer_k, delta_tracer_km1; + for (int iq=0; iq(s_delta_tracer, range_pack_for_m1_shift, delta_tracer_k, delta_tracer_km1); + if (any_at_top) delta_tracer_k.set(at_top, s_delta_tracer(0)); + if (any_at_bot) delta_tracer_km1.set(at_bot, s_delta_tracer(nlevs-2)); + + Q(iq, k).update(not_at_bot, fac*omega_int_kp1*delta_tracer_k, -1, 1); + Q(iq, k).update(not_at_top, fac*omega_int_k*delta_tracer_km1, -1, 1); + } + }); + + // Release WS views + workspace.release_macro_block(delta_Q_slot, n_q_tracers); + workspace.release_many_contiguous<4>({&omega_int, &delta_u, &delta_v, &delta_T}); +} +// ========================================================================================= +KOKKOS_FUNCTION +void IOPForcing:: +advance_iop_forcing(const MemberType& team, + const int nlevs, + const Real dt, + const view_1d& divT, + const view_1d& divq, + const view_1d& T, + const view_1d& qv) +{ + const auto nlev_packs = ekat::npack(nlevs); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const int k) { + T(k).update(divT(k), dt, 1.0); + qv(k).update(divq(k), dt, 1.0); + }); +} +// ========================================================================================= +KOKKOS_FUNCTION +void IOPForcing:: +iop_apply_coriolis(const MemberType& team, + const int nlevs, + const Real dt, + const Real lat, + const view_1d& u_ls, + const view_1d& v_ls, + const view_1d& u, + const view_1d& v) +{ + constexpr Real pi = C::Pi; + constexpr Real earth_rotation = C::omega; + + // Compute coriolis force + const auto fcor = 2*earth_rotation*std::sin(lat*pi/180); + + const auto nlev_packs = ekat::npack(nlevs); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const int k) { + const auto u_cor = v(k) - v_ls(k); + const auto v_cor = u(k) - u_ls(k); + u(k).update(u_cor, dt*fcor, 1.0); + v(k).update(v_cor, -dt*fcor, 1.0); + }); +} +// ========================================================================================= +void IOPForcing::run_impl (const double dt) +{ + // Pack dimensions + const auto nlev_packs = ekat::npack(m_num_levs); + + // Hybrid coord values + const auto ps0 = C::P0; + const auto hyam = m_grid->get_geometry_data("hyam").get_view(); + const auto hybm = m_grid->get_geometry_data("hybm").get_view(); + const auto hyai = m_grid->get_geometry_data("hyai").get_view(); + const auto hybi = m_grid->get_geometry_data("hybi").get_view(); + + // Get FM fields + const auto ps = get_field_in("ps").get_view(); + const auto horiz_winds = get_field_out("horiz_winds").get_view(); + const auto T_mid = get_field_out("T_mid").get_view(); + const auto qv = get_field_out("qv").get_view(); + const auto Q = get_group_out("tracers").m_bundle->get_view(); + + // Load data from IOP files, if necessary + m_iop_data_manager->read_iop_file_data(timestamp()); + + // Define local IOP param values + const auto iop_dosubsidence = m_iop_data_manager->get_params().get("iop_dosubsidence"); + const auto iop_coriolis = m_iop_data_manager->get_params().get("iop_coriolis"); + const auto iop_nudge_tq = m_iop_data_manager->get_params().get("iop_nudge_tq"); + const auto iop_nudge_uv = m_iop_data_manager->get_params().get("iop_nudge_uv"); + const auto use_large_scale_wind = m_iop_data_manager->get_params().get("use_large_scale_wind"); + const auto use_3d_forcing = m_iop_data_manager->get_params().get("use_3d_forcing"); + const auto target_lat = m_iop_data_manager->get_params().get("target_latitude"); + const auto iop_nudge_tscale = m_iop_data_manager->get_params().get("iop_nudge_tscale"); + const auto iop_nudge_tq_low = m_iop_data_manager->get_params().get("iop_nudge_tq_low"); + const auto iop_nudge_tq_high = m_iop_data_manager->get_params().get("iop_nudge_tq_high"); + + // Define local IOP field views + const Real ps_iop = m_iop_data_manager->get_iop_field("Ps").get_view()(); + view_1d omega, divT, divq, u_ls, v_ls, qv_iop, t_iop, u_iop, v_iop; + divT = use_3d_forcing ? m_iop_data_manager->get_iop_field("divT3d").get_view() + : m_iop_data_manager->get_iop_field("divT").get_view(); + divq = use_3d_forcing ? m_iop_data_manager->get_iop_field("divq3d").get_view() + : m_iop_data_manager->get_iop_field("divq").get_view(); + if (iop_dosubsidence) { + omega = m_iop_data_manager->get_iop_field("omega").get_view(); + } + if (iop_coriolis) { + u_ls = m_iop_data_manager->get_iop_field("u_ls").get_view(); + v_ls = m_iop_data_manager->get_iop_field("v_ls").get_view(); + } + if (iop_nudge_tq) { + qv_iop = m_iop_data_manager->get_iop_field("q").get_view(); + t_iop = m_iop_data_manager->get_iop_field("T").get_view(); + } + if (iop_nudge_uv) { + u_iop = use_large_scale_wind ? m_iop_data_manager->get_iop_field("u_ls").get_view() + : m_iop_data_manager->get_iop_field("u").get_view(); + v_iop = use_large_scale_wind ? m_iop_data_manager->get_iop_field("v_ls").get_view() + : m_iop_data_manager->get_iop_field("v").get_view(); + } + + // Team policy and workspace manager for eamxx + const auto policy_iop = ESU::get_default_team_policy(m_num_cols, nlev_packs); + + // Reset internal WSM variables. + m_workspace_mgr.reset_internals(); + + // Avoid implicit capture of this + auto wsm = m_workspace_mgr; + auto num_levs = m_num_levs; + + // Apply IOP forcing + Kokkos::parallel_for("apply_iop_forcing", policy_iop, KOKKOS_LAMBDA (const MemberType& team) { + const int icol = team.league_rank(); + + auto ps_i = ps(icol); + auto u_i = Kokkos::subview(horiz_winds, icol, 0, Kokkos::ALL()); + auto v_i = Kokkos::subview(horiz_winds, icol, 1, Kokkos::ALL()); + auto T_mid_i = ekat::subview(T_mid, icol); + auto qv_i = ekat::subview(qv, icol); + auto Q_i = Kokkos::subview(Q, icol, Kokkos::ALL(), Kokkos::ALL()); + + auto ws = wsm.get_workspace(team); + uview_1d ref_p_mid, ref_p_int, ref_p_del; + ws.take_many_contiguous_unsafe<3>({"ref_p_mid", "ref_p_int", "ref_p_del"}, + {&ref_p_mid, &ref_p_int, &ref_p_del}); + + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_ref_p_mid = ekat::scalarize(ref_p_mid); + auto s_ref_p_int = ekat::scalarize(ref_p_int); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, num_levs+1), [&](const int& k) { + s_ref_p_int(k) = hyai(k)*ps0 + hybi(k)*ps_i; + if (k < num_levs) { + s_ref_p_mid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + } + }); + team.team_barrier(); + ColOps::compute_midpoint_delta(team, num_levs, ref_p_int, ref_p_del); + team.team_barrier(); + + if (iop_dosubsidence) { + // Compute subsidence due to large-scale forcing + advance_iop_subsidence(team, num_levs, dt, ps_i, ref_p_mid, ref_p_int, ref_p_del, omega, ws, u_i, v_i, T_mid_i, Q_i); + } + + // Update T and qv according to large scale forcing as specified in IOP file. + advance_iop_forcing(team, num_levs, dt, divT, divq, T_mid_i, qv_i); + + if (iop_coriolis) { + // Apply coriolis forcing to u and v winds + iop_apply_coriolis(team, num_levs, dt, target_lat, u_ls, v_ls, u_i, v_i); + } + + // Release WS views + ws.release_many_contiguous<3>({&ref_p_mid, &ref_p_int, &ref_p_del}); + }); + + // Nudge the domain based on the domain mean + // and observed quantities of T, Q, u, and v + if (iop_nudge_tq or iop_nudge_uv) { + // Compute domain mean of qv, T_mid, u, and v + view_1d qv_mean, t_mean; + view_2d horiz_winds_mean; + if (iop_nudge_tq){ + horiz_contraction(m_helper_fields.at("qv_mean"), get_field_out("qv"), + m_helper_fields.at("horiz_mean_weights"), &m_comm); + qv_mean = m_helper_fields.at("qv_mean").get_view(); + + horiz_contraction(m_helper_fields.at("t_mean"), get_field_out("T_mid"), + m_helper_fields.at("horiz_mean_weights"), &m_comm); + t_mean = m_helper_fields.at("t_mean").get_view(); + } + if (iop_nudge_uv){ + horiz_contraction(m_helper_fields.at("horiz_winds_mean"), get_field_out("horiz_winds"), + m_helper_fields.at("horiz_mean_weights"), &m_comm); + horiz_winds_mean = m_helper_fields.at("horiz_winds_mean").get_view(); + } + + // Apply relaxation + const auto rtau = std::max(dt, iop_nudge_tscale); + Kokkos::parallel_for("apply_domain_relaxation", + policy_iop, + KOKKOS_LAMBDA (const MemberType& team) { + const int icol = team.league_rank(); + + auto ps_i = ps(icol); + auto u_i = Kokkos::subview(horiz_winds, icol, 0, Kokkos::ALL()); + auto v_i = Kokkos::subview(horiz_winds, icol, 1, Kokkos::ALL()); + auto T_mid_i = ekat::subview(T_mid, icol); + auto qv_i = ekat::subview(qv, icol); + + auto ws = wsm.get_workspace(team); + uview_1d ref_p_mid; + ws.take_many_contiguous_unsafe<1>({"ref_p_mid"},{&ref_p_mid}); + + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_ref_p_mid = ekat::scalarize(ref_p_mid); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, num_levs), [&](const int& k) { + s_ref_p_mid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + }); + team.team_barrier(); + + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&](const int& k) { + if (iop_nudge_tq) { + // Restrict nudging of T and qv to certain levels if requested by user + // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high + // is in units of [hPa], thus convert iop_nudge_tq_low/high + Mask nudge_level(false); + int max_size = hyam.size(); + for (int lev=k*Pack::n, p = 0; p < Pack::n && lev < max_size; ++lev, ++p) { + const auto pressure_from_iop = hyam(lev)*ps0 + hybm(lev)*ps_iop; + nudge_level.set(p, pressure_from_iop <= iop_nudge_tq_low*100 + and + pressure_from_iop >= iop_nudge_tq_high*100); + } + + qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); + T_mid_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); + } + if (iop_nudge_uv) { + u_i(k).update(horiz_winds_mean(0, k) - u_iop(k), -dt/rtau, 1.0); + v_i(k).update(horiz_winds_mean(1, k) - v_iop(k), -dt/rtau, 1.0); + } + }); + + // Release WS views + ws.release_many_contiguous<1>({&ref_p_mid}); + }); + } +} +// ========================================================================================= +} // namespace scream diff --git a/components/eamxx/src/physics/iop_forcing/eamxx_iop_forcing_process_interface.hpp b/components/eamxx/src/physics/iop_forcing/eamxx_iop_forcing_process_interface.hpp new file mode 100644 index 00000000000..7cec311a231 --- /dev/null +++ b/components/eamxx/src/physics/iop_forcing/eamxx_iop_forcing_process_interface.hpp @@ -0,0 +1,158 @@ +#ifndef SCREAM_IOP_FORCING_HPP +#define SCREAM_IOP_FORCING_HPP + +#include "ekat/ekat_parameter_list.hpp" +#include "ekat/ekat_workspace.hpp" + +#include "share/atm_process/atmosphere_process.hpp" +#include "share/atm_process/ATMBufferManager.hpp" +#include "share/util/scream_column_ops.hpp" + +#include "physics/share/physics_constants.hpp" + +#include + +namespace scream +{ +/* + * The class responsible for running EAMxx with an intensive + * observation period (IOP). + * + * The AD should store exactly ONE instance of this class stored + * in its list of subcomponents (the AD should make sure of this). + * + * Currently the only use case is the doubly + * periodic model (DP-SCREAM). + */ + +class IOPForcing : public scream::AtmosphereProcess +{ + // Typedefs for process + using KT = ekat::KokkosTypes; + using ESU = ekat::ExeSpaceUtils; + using Pack = ekat::Pack; + using IntPack = ekat::Pack; + using Mask = ekat::Mask; + using WorkspaceMgr = ekat::WorkspaceManager; + using Workspace = WorkspaceMgr::Workspace; + + using MemberType = KT::MemberType; + template + using view_1d = KT::view_1d; + template + using view_2d = KT::view_2d; + template + using uview_1d = ekat::Unmanaged>; + template + using uview_2d = ekat::Unmanaged>; + + using ColOps = ColumnOps; + using C = physics::Constants; + + + +public: + + // Constructors + IOPForcing (const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereProcess(comm, params) {} + + // The type of subcomponent + AtmosphereProcessType type () const { return AtmosphereProcessType::Physics; } + + // The name of the subcomponent + std::string name () const { return "iop"; } + + // Set the grid + void set_grids (const std::shared_ptr grids_manager); + +#ifndef KOKKOS_ENABLE_CUDA + // Cuda requires methods enclosing __device__ lambda's to be public +protected: +#endif + + void initialize_impl (const RunType run_type); + + // Compute effects of large scale subsidence on T, q, u, and v. + KOKKOS_FUNCTION + static void advance_iop_subsidence(const KT::MemberType& team, + const int nlevs, + const Real dt, + const Real ps, + const view_1d& pmid, + const view_1d& pint, + const view_1d& pdel, + const view_1d& omega, + const Workspace& workspace, + const view_1d& u, + const view_1d& v, + const view_1d& T, + const view_2d& Q); + + // Apply large scale forcing for temperature and water vapor as provided by the IOP file + KOKKOS_FUNCTION + static void advance_iop_forcing(const KT::MemberType& team, + const int nlevs, + const Real dt, + const view_1d& divT, + const view_1d& divq, + const view_1d& T, + const view_1d& qv); + + // Provide coriolis forcing to u and v winds, using large scale winds specified in IOP forcing file. + KOKKOS_FUNCTION + static void iop_apply_coriolis(const KT::MemberType& team, + const int nlevs, + const Real dt, + const Real lat, + const view_1d& u_ls, + const view_1d& v_ls, + const view_1d& u, + const view_1d& v); + + void run_impl (const double dt); + +protected: + + void finalize_impl () {} + + // Creates an helper field, not to be shared with the AD's FieldManager + void create_helper_field (const std::string& name, + const FieldLayout& layout, + const std::string& grid_name, + const int ps = 1); + + void set_computed_group_impl (const FieldGroup& group); + + // Computes total number of bytes needed for local variables + size_t requested_buffer_size_in_bytes() const; + + // Set local variables using memory provided by + // the ATMBufferManager + void init_buffers(const ATMBufferManager &buffer_manager); + + // Keep track of field dimensions and other scalar values + // needed in IOP + Int m_num_cols; + Int m_num_levs; + Int m_num_tracers; + + struct Buffer { + Pack* wsm_data; + }; + + // Some helper fields. + std::map m_helper_fields; + + // Struct which contains local variables + Buffer m_buffer; + + // WSM for internal local variables + WorkspaceMgr m_workspace_mgr; + + std::shared_ptr m_grid; +}; // class IOPForcing + +} // namespace scream + +#endif // SCREAM_IOP_FORCING_HPP diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index 8bb52c918d2..9dbe97a4ae7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -406,6 +406,10 @@ void call_function_dropmixnuc( } }); team.team_barrier(); + // HACK: dropmixnuc() requires the parameter enable_aero_vertical_mix, + // so we define it here until we have a better idea of where it + // might come from + const bool enable_aero_vertical_mix = true; mam4::ndrop::dropmixnuc( team, dt, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), ekat::subview(p_int, icol), ekat::subview(pdel, icol), @@ -417,6 +421,7 @@ void call_function_dropmixnuc( spechygro, lmassptr_amode, num2vol_ratio_min_nmodes, num2vol_ratio_max_nmodes, numptr_amode, nspec_amode, exp45logsig, alogsig, aten, mam_idx, mam_cnst_idx, + enable_aero_vertical_mix, ekat::subview(qcld, icol), // out ekat::subview(wsub, icol), // in ekat::subview(cloud_frac_prev, icol), // in diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 4a674fb094d..1c0053a87b9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -1,10 +1,10 @@ +#include #include - // impl namespace for some driver level functions for microphysics -#include "readfiles/photo_table_utils.cpp" -#include "readfiles/find_season_index_utils.hpp" #include "physics/rrtmgp/shr_orb_mod_c2f.hpp" +#include "readfiles/find_season_index_utils.hpp" +#include "readfiles/photo_table_utils.cpp" namespace scream { @@ -36,7 +36,7 @@ MAMMicrophysics::MAMMicrophysics(const ekat::Comm &comm, config_.linoz.o3_lbl = m_params.get("mam4_o3_lbl"); config_.linoz.o3_tau = m_params.get("mam4_o3_tau"); config_.linoz.o3_sfc = m_params.get("mam4_o3_sfc"); - config_.linoz.psc_T = m_params.get("mam4_psc_T"); + config_.linoz.psc_T = m_params.get("mam4_psc_T"); } AtmosphereProcessType MAMMicrophysics::type() const { @@ -70,6 +70,9 @@ void MAMMicrophysics::set_grids( const FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); const FieldLayout scalar3d_int = grid_->get_3d_scalar_layout(false); + // For U and V components of wind + const FieldLayout vector3d = grid_->get_3d_vector_layout(true, 2); + using namespace ekat::units; constexpr auto q_unit = kg / kg; // units of mass mixing ratios of tracers constexpr auto n_unit = 1 / kg; // units of number mixing ratios of tracers @@ -81,19 +84,20 @@ void MAMMicrophysics::set_grids( // ----------- Atmospheric quantities ------------- // Specific humidity [kg/kg](Require only for building DS) - add_tracer("qv", grid_, kg/kg); // specific humidity + add_tracer("qv", grid_, kg / kg); // specific humidity // Cloud liquid mass mixing ratio [kg/kg](Require only for building DS) - add_tracer("qc", grid_, kg/kg); // cloud liquid wet mixing ratio + add_tracer("qc", grid_, kg / kg); // cloud liquid wet mixing ratio // Cloud ice mass mixing ratio [kg/kg](Require only for building DS) - add_tracer("qi", grid_, kg/kg); // ice wet mixing ratio + add_tracer("qi", grid_, kg / kg); // ice wet mixing ratio // Cloud liquid number mixing ratio [1/kg](Require only for building DS) - add_tracer("nc", grid_, n_unit); // cloud liquid wet number mixing ratio + add_tracer("nc", grid_, + n_unit); // cloud liquid wet number mixing ratio // Cloud ice number mixing ratio [1/kg](Require only for building DS) - add_tracer("ni", grid_, n_unit); // ice number mixing ratio + add_tracer("ni", grid_, n_unit); // ice number mixing ratio // Temperature[K] at midpoints add_field("T_mid", scalar3d_mid, K, grid_name); @@ -120,11 +124,30 @@ void MAMMicrophysics::set_grids( // Surface geopotential [m2/s2] add_field("phis", scalar2d, m2 / s2, grid_name); + // Surface pressure [Pa] + add_field("ps", scalar2d, Pa, grid_name); + + // U and V components of the wind[m/s] + add_field("horiz_winds", vector3d, m / s, grid_name); + //----------- Variables from microphysics scheme ------------- constexpr auto nondim = ekat::units::Units::nondimensional(); // Total cloud fraction [fraction] add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name); + // Evaporation from stratiform rain [kg/kg/s] + add_field("nevapr", scalar3d_mid, kg / kg / s, grid_name); + + // Stratiform rain production rate [kg/kg/s] + add_field("precip_total_tend", scalar3d_mid, kg / kg / s, + grid_name); + + // precipitation liquid mass [kg/m2] + add_field("precip_liq_surf_mass", scalar3d_mid, kg / m2, grid_name); + + // precipitation ice mass [kg/m2] + add_field("precip_ice_surf_mass", scalar3d_mid, kg / m2, grid_name); + //----------- Variables from other mam4xx processes ------------ // Number of modes constexpr int nmodes = mam4::AeroConfig::num_modes(); @@ -142,18 +165,26 @@ void MAMMicrophysics::set_grids( // Wet density of interstitial aerosol [kg/m3] add_field("wetdens", scalar3d_mid_nmodes, kg / m3, grid_name); - //----------- Variables from coupler (land component)--------- + // For fractional land use + const FieldLayout vector2d_class = + grid_->get_2d_vector_layout(mam4::mo_drydep::n_land_type, "class"); + + // Fractional land use [fraction] + add_field("fraction_landuse", vector2d_class, nondim, grid_name); + + //----------- Variables from the coupler --------- // surface albedo shortwave, direct add_field("sfc_alb_dir_vis", scalar2d, nondim, grid_name); - //----------- Variables from microphysics scheme ------------- + // Surface temperature[K] + add_field("surf_radiative_T", scalar2d, K, grid_name); - // Evaporation from stratiform rain [kg/kg/s] - add_field("nevapr", scalar3d_mid, kg / kg / s, grid_name); + // snow depth land [m] + add_field("snow_depth_land", scalar2d, m, grid_name); - // Stratiform rain production rate [kg/kg/s] - add_field("precip_total_tend", scalar3d_mid, kg / kg / s, - grid_name); + //----------- Variables from the RRTMGP radiation --------- + // Downwelling solar flux at the surface [w/m2] + add_field("SW_flux_dn", scalar3d_int, W / m2, grid_name); // --------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process @@ -170,7 +201,7 @@ void MAMMicrophysics::set_grids( mam_coupling::int_aero_mmr_field_name(m, a); if(strlen(int_mmr_field_name) > 0) { - add_tracer(int_mmr_field_name, grid_, kg/kg); + add_tracer(int_mmr_field_name, grid_, kg / kg); } } // for loop species } // for loop nmodes interstitial @@ -192,8 +223,16 @@ void MAMMicrophysics::set_grids( // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_tracer(gas_mmr_field_name, grid_, kg/kg); + add_tracer(gas_mmr_field_name, grid_, kg / kg); } + //----------- Updated variables from other mam4xx processes ------------ + // layout for Constituent fluxes + FieldLayout scalar2d_pcnst = + grid_->get_2d_vector_layout(mam4::pcnst, "num_phys_constituents"); + + // Constituent fluxes of species in [kg/m2/s] + add_field("constituent_fluxes", scalar2d_pcnst, kg / m2 / s, + grid_name); // Creating a Linoz reader and setting Linoz parameters involves reading data // from a file and configuring the necessary parameters for the Linoz model. @@ -272,8 +311,10 @@ void MAMMicrophysics::set_grids( const auto file_name = m_params.get(item_name); elevated_emis_file_name_[var_name] = file_name; } - elevated_emis_var_names_["so2"] = {"BB", "ENE_ELEV", "IND_ELEV", "contvolc"}; - elevated_emis_var_names_["so4_a1"] = {"BB", "ENE_ELEV", "IND_ELEV", "contvolc"}; + elevated_emis_var_names_["so2"] = {"BB", "ENE_ELEV", "IND_ELEV", + "contvolc"}; + elevated_emis_var_names_["so4_a1"] = {"BB", "ENE_ELEV", "IND_ELEV", + "contvolc"}; elevated_emis_var_names_["so4_a2"] = {"contvolc"}; elevated_emis_var_names_["pom_a4"] = {"BB"}; elevated_emis_var_names_["bc_a4"] = {"BB"}; @@ -285,8 +326,8 @@ void MAMMicrophysics::set_grids( // FIXME: why the sectors in this files are num_a1; // I guess this should be num_a4? Is this a bug in the orginal nc files? elevated_emis_var_names_["num_a4"] = {"num_a1_BC_ELEV_BB", - "num_a1_POM_ELEV_BB"}; - elevated_emis_var_names_["soag"] = {"SOAbb_src", "SOAbg_src", "SOAff_src"}; + "num_a1_POM_ELEV_BB"}; + elevated_emis_var_names_["soag"] = {"SOAbb_src", "SOAbg_src", "SOAff_src"}; int elevated_emiss_cyclical_ymd = m_params.get("elevated_emiss_ymd"); @@ -300,9 +341,8 @@ void MAMMicrophysics::set_grids( auto hor_rem = scream::mam_coupling::create_horiz_remapper( grid_, file_name, extfrc_map_file, var_names, data_tracer); - auto file_reader = - scream::mam_coupling::create_tracer_data_reader(hor_rem, file_name, - data_tracer.file_type); + auto file_reader = scream::mam_coupling::create_tracer_data_reader( + hor_rem, file_name, data_tracer.file_type); ElevatedEmissionsHorizInterp_.push_back(hor_rem); ElevatedEmissionsDataReader_.push_back(file_reader); elevated_emis_data_.push_back(data_tracer); @@ -317,8 +357,9 @@ void MAMMicrophysics::set_grids( forcings_[i].nsectors = nvars; // I am assuming the order of species in extfrc_lst_. // Indexing in mam4xx is fortran. - forcings_[i].frc_ndx = i + 1; - const auto io_grid_emis = ElevatedEmissionsHorizInterp_[i]->get_tgt_grid(); + forcings_[i].frc_ndx = i + 1; + const auto io_grid_emis = + ElevatedEmissionsHorizInterp_[i]->get_tgt_grid(); const int num_cols_io_emis = io_grid_emis->get_num_local_dofs(); // Number of columns on this rank const int num_levs_io_emis = @@ -344,11 +385,11 @@ void MAMMicrophysics::set_grids( } // Tracer external forcing data { - const std::string season_wes_file = m_params.get("mam4_season_wes_file"); - const auto& clat = col_latitudes_; - mam_coupling::find_season_index_reader(season_wes_file, - clat, - index_season_lai_); + const std::string season_wes_file = + m_params.get("mam4_season_wes_file"); + const auto &clat = col_latitudes_; + mam_coupling::find_season_index_reader(season_wes_file, clat, + index_season_lai_); } } // set_grids @@ -523,7 +564,7 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { work_photo_table_ = view_2d("work_photo_table", ncol_, photo_table_len); const int sethet_work_len = mam4::mo_sethet::get_total_work_len_sethet(); work_set_het_ = view_2d("work_set_het_array", ncol_, sethet_work_len); - cmfdqr_ = view_1d("cmfdqr_", nlev_); + cmfdqr_ = view_1d("cmfdqr_", nlev_); // here's where we store per-column photolysis rates photo_rates_ = view_3d("photo_rates", ncol_, nlev_, mam4::mo_photo::phtcnt); @@ -541,8 +582,8 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { for(int i = 0; i < static_cast(extfrc_lst_.size()); ++i) { scream::mam_coupling::update_tracer_data_from_file( - ElevatedEmissionsDataReader_[i], curr_month, *ElevatedEmissionsHorizInterp_[i], - elevated_emis_data_[i]); + ElevatedEmissionsDataReader_[i], curr_month, + *ElevatedEmissionsHorizInterp_[i], elevated_emis_data_[i]); } invariants_ = view_3d("invarians", ncol_, nlev_, mam4::gas_chemistry::nfs); @@ -568,22 +609,25 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMMicrophysics::run_impl(const double dt) { + const int ncol = ncol_; + const int nlev = nlev_; const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol, nlev); const auto policy = - ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); + ekat::ExeSpaceUtils::get_default_team_policy(ncol, nlev); // preprocess input -- needs a scan for the calculation of atm height Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - //----------- Variables from microphysics scheme ------------- + //----------- Variables from microphysics scheme ------------- // Evaporation from stratiform rain [kg/kg/s] - const auto& nevapr = get_field_in("nevapr").get_view(); + const auto &nevapr = get_field_in("nevapr").get_view(); // Stratiform rain production rate [kg/kg/s] - const auto& prain = get_field_in("precip_total_tend").get_view(); + const auto &prain = + get_field_in("precip_total_tend").get_view(); const auto wet_geometric_mean_diameter_i = get_field_in("dgnumwet").get_view(); @@ -591,6 +635,46 @@ void MAMMicrophysics::run_impl(const double dt) { get_field_in("dgnum").get_view(); const auto wetdens = get_field_in("wetdens").get_view(); + // U wind component [m/s] + const const_view_2d u_wind = + get_field_in("horiz_winds").get_component(0).get_view(); + + // V wind component [m/s] + const const_view_2d v_wind = + get_field_in("horiz_winds").get_component(1).get_view(); + + // Liquid precip [kg/m2] + const const_view_2d precip_liq_surf_mass = + get_field_in("precip_liq_surf_mass").get_view(); + + // Ice precip [kg/m2] + const const_view_2d precip_ice_surf_mass = + get_field_in("precip_ice_surf_mass").get_view(); + + // Fractional land use [fraction] + const const_view_2d fraction_landuse = + get_field_in("fraction_landuse").get_view(); + + // Downwelling solar flux at the surface [w/m2] + const const_view_2d sw_flux_dn = + get_field_in("SW_flux_dn").get_view(); + + // Constituent fluxes of gas and aerosol species + view_2d constituent_fluxes = + get_field_out("constituent_fluxes").get_view(); + + // Surface temperature [K] + const const_view_1d sfc_temperature = + get_field_in("surf_radiative_T").get_view(); + + // Surface pressure [Pa] + const const_view_1d sfc_pressure = + get_field_in("ps").get_view(); + + // Snow depth on land [m] + const const_view_1d snow_depth_land = + get_field_in("snow_depth_land").get_view(); + // climatology data for linear stratospheric chemistry // ozone (climatology) [vmr] auto linoz_o3_clim = buffer_.scratch[0]; @@ -648,14 +732,15 @@ void MAMMicrophysics::run_impl(const double dt) { Kokkos::fence(); elevated_emiss_time_state_.t_now = ts.frac_of_year_in_days(); - int i = 0; + int i = 0; for(const auto &var_name : extfrc_lst_) { const auto file_name = elevated_emis_file_name_[var_name]; const auto var_names = elevated_emis_var_names_[var_name]; const int nsectors = int(var_names.size()); view_2d elevated_emis_output[nsectors]; for(int isp = 0; isp < nsectors; ++isp) { - elevated_emis_output[isp] = elevated_emis_output_[isp + forcings_[i].offset]; + elevated_emis_output[isp] = + elevated_emis_output_[isp + forcings_[i].offset]; } scream::mam_coupling::advance_tracer_data( ElevatedEmissionsDataReader_[i], *ElevatedEmissionsHorizInterp_[i], ts, @@ -722,7 +807,7 @@ void MAMMicrophysics::run_impl(const double dt) { // then deep copied to a device view. // Now use solar declination to calculate zenith angle for all points - for(int i = 0; i < ncol_; i++) { + for(int i = 0; i < ncol; i++) { Real lat = col_latitudes_host(i) * M_PI / 180.0; // Convert lat/lon to radians Real lon = col_longitudes_host(i) * M_PI / 180.0; @@ -735,10 +820,10 @@ void MAMMicrophysics::run_impl(const double dt) { const auto zenith_angle = acos_cosine_zenith_; constexpr int gas_pcnst = mam_coupling::gas_pcnst(); - const auto& elevated_emis_output = elevated_emis_output_; - const auto& extfrc = extfrc_; - const auto& forcings = forcings_; - constexpr int extcnt = mam4::gas_chemistry::extcnt; + const auto &elevated_emis_output = elevated_emis_output_; + const auto &extfrc = extfrc_; + const auto &forcings = forcings_; + constexpr int extcnt = mam4::gas_chemistry::extcnt; const int offset_aerosol = mam4::utils::gasses_start_ind(); Real adv_mass_kg_per_moles[gas_pcnst]; @@ -753,11 +838,18 @@ void MAMMicrophysics::run_impl(const double dt) { clsmap_4[i] = mam4::gas_chemistry::clsmap_4[i]; permute_4[i] = mam4::gas_chemistry::permute_4[i]; } - const auto& cmfdqr = cmfdqr_; - const auto& work_set_het =work_set_het_; + const auto &cmfdqr = cmfdqr_; + const auto &work_set_het = work_set_het_; + const mam4::seq_drydep::Data drydep_data = + mam4::seq_drydep::set_gas_drydep_data(); + const auto qv = wet_atm_.qv; + const int month = timestamp().get_month(); // 1-based + const int surface_lev = nlev - 1; // Surface level + // loop over atmosphere columns and compute aerosol microphyscs Kokkos::parallel_for( - policy, KOKKOS_LAMBDA(const ThreadTeam &team) { + "MAMMicrophysics::run_impl", policy, + KOKKOS_LAMBDA(const ThreadTeam &team) { const int icol = team.league_rank(); // column index const Real col_lat = col_latitudes(icol); // column latitude (degrees?) @@ -820,28 +912,81 @@ void MAMMicrophysics::run_impl(const double dt) { ekat::subview(linoz_dPmL_dO3col, icol); const auto linoz_cariolle_pscs_icol = ekat::subview(linoz_cariolle_pscs, icol); - const auto nevapr_icol = ekat::subview(nevapr, icol); - const auto prain_icol = ekat::subview(prain, icol); + const auto nevapr_icol = ekat::subview(nevapr, icol); + const auto prain_icol = ekat::subview(prain, icol); const auto work_set_het_icol = ekat::subview(work_set_het, icol); - // Note: All variables are inputs, except for progs, which is an - // input/output variable. + + // Surface temperature + const Real sfc_air_temp = atm.temperature(surface_lev); + + // Surface specific humidity + const Real sfc_spec_hum = atm.vapor_mixing_ratio(surface_lev); + + // Surface potential temperature + //(FIXME: We followed Fortran, compare it with MAM4xx's potential temp + // func) + const Real sfc_potential_temp = sfc_air_temp * (1.0 + sfc_spec_hum); + + // Surface pressure at 10m (Followed the fortran code) + const Real pressure_10m = dry_atm.p_mid(icol, surface_lev); + + // Wind speed at the surface + const Real wind_speed = + haero::sqrt(u_wind(icol, surface_lev) * u_wind(icol, surface_lev) + + v_wind(icol, surface_lev) * v_wind(icol, surface_lev)); + + // Total rain at the surface + const Real rain = precip_liq_surf_mass(icol, surface_lev) + + precip_ice_surf_mass(icol, surface_lev); + + // Snow depth on land [m] + const Real snow_height = snow_depth_land(icol); + + // Downwelling solar flux at the surface (value at interface) [w/m2] + const Real solar_flux = sw_flux_dn(icol, surface_lev + 1); + + Real fraction_landuse_icol[mam4::mo_drydep::n_land_type]; + for(int i = 0; i < mam4::mo_drydep::n_land_type; ++i) { + fraction_landuse_icol[i] = fraction_landuse(icol, i); + } + + // ????? FIXME: We should get its value after the rebase + const int col_index_season[mam4::mo_drydep::n_land_type] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9}; + // These output values need to be put somewhere: + Real dvel[gas_pcnst] = {}; // deposition velocity [1/cm/s] + Real dflx[gas_pcnst] = {}; // deposition flux [1/cm^2/s] + + // Output: values are dvel, dvlx + // Input/Output: progs::stateq, progs::qqcw mam4::microphysics::perform_atmospheric_chemistry_and_microphysics( - team, dt, rlats, cnst_offline_icol, forcings_in, atm, progs, - photo_table, chlorine_loading, config.setsox, config.amicphys, - config.linoz.psc_T, zenith_angle(icol), d_sfc_alb_dir_vis(icol), - o3_col_dens_i, photo_rates_icol, extfrc_icol, invariants_icol, - work_photo_table_icol, linoz_o3_clim_icol, linoz_t_clim_icol, - linoz_o3col_clim_icol, linoz_PmL_clim_icol, linoz_dPmL_dO3_icol, - linoz_dPmL_dT_icol, linoz_dPmL_dO3col_icol, - linoz_cariolle_pscs_icol, eccf, adv_mass_kg_per_moles, clsmap_4, - permute_4, offset_aerosol, - config.linoz.o3_sfc, config.linoz.o3_tau, config.linoz.o3_lbl, - dry_diameter_icol, wet_diameter_icol, wetdens_icol, - dry_atm.phis(icol), - cmfdqr, - prain_icol, - nevapr_icol, - work_set_het_icol); + team, dt, rlats, month, sfc_temperature(icol), sfc_air_temp, + sfc_potential_temp, sfc_pressure(icol), pressure_10m, sfc_spec_hum, + wind_speed, rain, snow_height, solar_flux, cnst_offline_icol, + forcings_in, atm, photo_table, chlorine_loading, config.setsox, + config.amicphys, config.linoz.psc_T, zenith_angle(icol), + d_sfc_alb_dir_vis(icol), o3_col_dens_i, photo_rates_icol, + extfrc_icol, invariants_icol, work_photo_table_icol, + linoz_o3_clim_icol, linoz_t_clim_icol, linoz_o3col_clim_icol, + linoz_PmL_clim_icol, linoz_dPmL_dO3_icol, linoz_dPmL_dT_icol, + linoz_dPmL_dO3col_icol, linoz_cariolle_pscs_icol, eccf, + adv_mass_kg_per_moles, fraction_landuse_icol, + + col_index_season, // FIXME: Get it after Changes sync with E3SM + + clsmap_4, permute_4, offset_aerosol, config.linoz.o3_sfc, + config.linoz.o3_tau, config.linoz.o3_lbl, dry_diameter_icol, + wet_diameter_icol, wetdens_icol, dry_atm.phis(icol), cmfdqr, + prain_icol, nevapr_icol, work_set_het_icol, drydep_data, dvel, dflx, + progs); + + // Update constituent fluxes with gas drydep fluxes (dflx) + // FIXME: Possible units mismatch (dflx is in kg/cm2/s but + // constituent_fluxes is kg/m2/s) (Following mimics Fortran code + // behavior but we should look into it) + for(int ispc = offset_aerosol; ispc < mam4::pcnst; ++ispc) { + constituent_fluxes(icol, ispc) = dflx[ispc - offset_aerosol]; + } }); // parallel_for for the column loop Kokkos::fence(); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp index 6ff846d0d0c..532e3042301 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp @@ -22,6 +22,7 @@ class MAMMicrophysics final : public scream::AtmosphereProcess { using view_2d = typename KT::template view_2d; using view_3d = typename KT::template view_3d; using const_view_1d = typename KT::template view_1d; + using const_view_2d = typename KT::template view_2d; using view_1d_host = typename KT::view_1d::HostMirror; diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp index 863da8021dd..0295a23cb4d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -539,12 +539,19 @@ void MAMWetscav::run_impl(const double dt) { auto wetdens_icol = ekat::subview(wetdens, icol); const auto prain_icol = ekat::subview(prain, icol); + Real scavimptblnum[mam4::aero_model::nimptblgrow_total] + [mam4::AeroConfig::num_modes()]; + Real scavimptblvol[mam4::aero_model::nimptblgrow_total] + [mam4::AeroConfig::num_modes()]; + + mam4::wetdep::init_scavimptbl(scavimptblvol, scavimptblnum); + mam4::wetdep::aero_model_wetdep( team, atm, progs, tends, dt, // inputs cldt_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, evapcsh_icol, dp_frac_icol, sh_frac_icol, icwmrdp_col, icwmrsh_icol, nevapr_icol, - dlf_icol, prain_icol, + dlf_icol, prain_icol, scavimptblnum, scavimptblvol, // outputs wet_diameter_icol, dry_diameter_icol, qaerwat_icol, wetdens_icol, aerdepwetis_icol, aerdepwetcw_icol, work_icol); diff --git a/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp b/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp index 389445fa024..6758dccdef5 100644 --- a/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp +++ b/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp @@ -154,10 +154,8 @@ void marineOrganicsFunctions::update_marine_organics_timestate( if(month != time_state.current_month) { // Update the marineOrganics time state information time_state.current_month = month; - time_state.t_beg_month = - util::TimeStamp({ts.get_year(), month + 1, 1}, {0, 0, 0}) - .frac_of_year_in_days(); - time_state.days_this_month = util::days_in_month(ts.get_year(), month + 1); + time_state.t_beg_month = ts.curr_month_beg().frac_of_year_in_days(); + time_state.days_this_month = ts.days_in_curr_month(); // Copy end'data into beg'data, and read in the new // end @@ -293,4 +291,4 @@ void marineOrganicsFunctions::init_marine_organics_file_read( } // namespace marine_organics } // namespace scream -#endif // MARINE_ORGANICS_IMPL_HPP \ No newline at end of file +#endif // MARINE_ORGANICS_IMPL_HPP diff --git a/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp b/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp index c605d3a0b9f..6e45750354d 100644 --- a/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp +++ b/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp @@ -508,10 +508,8 @@ inline void update_tracer_timestate( // Update the tracer external forcing time state information time_state.current_month = month; - time_state.t_beg_month = - util::TimeStamp({ts.get_year(), month + 1, 1}, {0, 0, 0}) - .frac_of_year_in_days(); - time_state.days_this_month = util::days_in_month(ts.get_year(), month + 1); + time_state.t_beg_month = ts.curr_month_beg().frac_of_year_in_days(); + time_state.days_this_month = ts.days_in_curr_month(); // Copy spa_end'data into spa_beg'data, and read in the new spa_end for(int ivar = 0; ivar < nvars; ++ivar) { diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index b8ebfdbe501..e1c471edf4b 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -227,10 +227,8 @@ void srfEmissFunctions::update_srfEmiss_timestate( if(month != time_state.current_month) { // Update the srfEmiss time state information time_state.current_month = month; - time_state.t_beg_month = - util::TimeStamp({ts.get_year(), month + 1, 1}, {0, 0, 0}) - .frac_of_year_in_days(); - time_state.days_this_month = util::days_in_month(ts.get_year(), month + 1); + time_state.t_beg_month = ts.curr_month_beg().frac_of_year_in_days(); + time_state.days_this_month = ts.days_in_curr_month(); // Copy srfEmiss_end'data into srfEmiss_beg'data, and read in the new // srfEmiss_end diff --git a/components/eamxx/src/physics/p3/CMakeLists.txt b/components/eamxx/src/physics/p3/CMakeLists.txt index 58a026e1601..6c39a9fcf1a 100644 --- a/components/eamxx/src/physics/p3/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/CMakeLists.txt @@ -1,6 +1,4 @@ set(P3_SRCS - p3_iso_c.f90 - ${SCREAM_BASE_DIR}/../eam/src/physics/p3/scream/micro_p3.F90 eamxx_p3_process_interface.cpp eamxx_p3_run.cpp ) @@ -82,11 +80,7 @@ endif() target_compile_definitions(p3 PUBLIC EAMXX_HAS_P3) foreach (P3_LIB IN LISTS P3_LIBS) - set_target_properties(${P3_LIB} PROPERTIES - Fortran_MODULE_DIRECTORY ${P3_LIB}/modules - ) target_include_directories(${P3_LIB} PUBLIC - ${CMAKE_CURRENT_BINARY_DIR}/${P3_LIB}/modules ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/impl ${SCREAM_BASE_DIR}/../eam/src/physics/cam @@ -102,10 +96,10 @@ else() endif() set (P3_TABLES scream/tables/p3_lookup_table_1.dat-v4.1.1 - scream/tables/mu_r_table_vals.dat${PRECISION_SUFFIX} - scream/tables/revap_table_vals.dat${PRECISION_SUFFIX} - scream/tables/vm_table_vals.dat${PRECISION_SUFFIX} - scream/tables/vn_table_vals.dat${PRECISION_SUFFIX} + scream/tables/mu_r_table_vals_v2.dat${PRECISION_SUFFIX} + scream/tables/revap_table_vals_v2.dat${PRECISION_SUFFIX} + scream/tables/vm_table_vals_v2.dat${PRECISION_SUFFIX} + scream/tables/vn_table_vals_v2.dat${PRECISION_SUFFIX} ) include (ScreamUtils) @@ -113,10 +107,6 @@ foreach (file IN ITEMS ${P3_TABLES}) GetInputFile(${file}) endforeach() -# This executable can be used to re-generate tables in ${SCREAM_DATA_DIR} -add_executable(p3_tables_setup EXCLUDE_FROM_ALL p3_tables_setup.cpp) -target_link_libraries(p3_tables_setup p3) - if (NOT SCREAM_LIB_ONLY) add_subdirectory(tests) endif() diff --git a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp index f6771d6bf17..15437fd567a 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp @@ -241,8 +241,8 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) add_postcondition_check(get_field_out("eff_radius_qr"),m_grid,0.0,5.0e3,false); // Initialize p3 - P3F::p3_init(/* write_tables = */ false, - this->get_comm().am_i_root()); + lookup_tables = P3F::p3_init(/* write_tables = */ false, + this->get_comm().am_i_root()); // Initialize all of the structures that are passed to p3_main in run_impl. // Note: Some variables in the structures are not stored in the field manager. For these @@ -280,7 +280,7 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) p3_preproc.set_variables(m_num_cols,nk_pack,pmid,pmid_dry,pseudo_density,pseudo_density_dry, T_atm,cld_frac_t, qv, qc, nc, qr, nr, qi, qm, ni, bm, qv_prev, - inv_exner, th_atm, cld_frac_l, cld_frac_i, cld_frac_r, dz); + inv_exner, th_atm, cld_frac_l, cld_frac_i, cld_frac_r, dz, runtime_options); // --Prognostic State Variables: prog_state.qc = p3_preproc.qc; prog_state.nc = p3_preproc.nc; @@ -411,12 +411,6 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) p3_postproc.set_mass_and_energy_fluxes(vapor_flux, water_flux, ice_flux, heat_flux); } - // Load tables - P3F::init_kokkos_ice_lookup_tables(lookup_tables.ice_table_vals, lookup_tables.collect_table_vals); - P3F::init_kokkos_tables(lookup_tables.vn_table_vals, lookup_tables.vm_table_vals, - lookup_tables.revap_table_vals, lookup_tables.mu_r_table_vals, - lookup_tables.dnu_table_vals); - // Setup WSM for internal local variables const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(m_num_cols, nk_pack); workspace_mgr.setup(m_buffer.wsm_data, nk_pack_p1, 52, policy); diff --git a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp index 7aedd97b5a8..c1323bdc8e8 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp @@ -105,23 +105,26 @@ class P3Microphysics : public AtmosphereProcess th_atm(icol,ipack) = PF::calculate_theta_from_T(T_atm_pack,pmid_pack); // Cloud fraction // Set minimum cloud fraction - avoids division by zero - cld_frac_l(icol,ipack) = ekat::max(cld_frac_t_pack,mincld); - cld_frac_i(icol,ipack) = ekat::max(cld_frac_t_pack,mincld); - cld_frac_r(icol,ipack) = ekat::max(cld_frac_t_pack,mincld); + // Alternatively set fraction to 1 everywhere to disable subgrid effects + cld_frac_l(icol,ipack) = runtime_opts.set_cld_frac_l_to_one ? 1 : ekat::max(cld_frac_t_pack,mincld); + cld_frac_i(icol,ipack) = runtime_opts.set_cld_frac_i_to_one ? 1 : ekat::max(cld_frac_t_pack,mincld); + cld_frac_r(icol,ipack) = runtime_opts.set_cld_frac_r_to_one ? 1 : ekat::max(cld_frac_t_pack,mincld); // update rain cloud fraction given neighboring levels using max-overlap approach. - for (int ivec=0;iveccld_frac_r(icol,ipack)[ivec] ? - cld_frac_t(icol,ipack_m1)[ivec_m1] : - cld_frac_r(icol,ipack)[ivec]; + if ( !runtime_opts.set_cld_frac_r_to_one ) { + for (int ivec=0;iveccld_frac_r(icol,ipack)[ivec] ? + cld_frac_t(icol,ipack_m1)[ivec_m1] : + cld_frac_r(icol,ipack)[ivec]; + } } } // @@ -152,6 +155,8 @@ class P3Microphysics : public AtmosphereProcess view_2d cld_frac_i; view_2d cld_frac_r; view_2d dz; + // Add runtime_options as a member variable + P3F::P3Runtime runtime_opts; // Assigning local variables void set_variables(const int ncol, const int npack, const view_2d_const& pmid_, const view_2d_const& pmid_dry_, @@ -161,7 +166,8 @@ class P3Microphysics : public AtmosphereProcess const view_2d& nc_, const view_2d& qr_, const view_2d& nr_, const view_2d& qi_, const view_2d& qm_, const view_2d& ni_, const view_2d& bm_, const view_2d& qv_prev_, const view_2d& inv_exner_, const view_2d& th_atm_, const view_2d& cld_frac_l_, - const view_2d& cld_frac_i_, const view_2d& cld_frac_r_, const view_2d& dz_ + const view_2d& cld_frac_i_, const view_2d& cld_frac_r_, const view_2d& dz_, + const P3F::P3Runtime& runtime_options ) { m_ncol = ncol; @@ -190,6 +196,7 @@ class P3Microphysics : public AtmosphereProcess cld_frac_i = cld_frac_i_; cld_frac_r = cld_frac_r_; dz = dz_; + runtime_opts = runtime_options; } // set_variables }; // p3_preamble /* --------------------------------------------------------------------------------------------*/ @@ -259,7 +266,7 @@ class P3Microphysics : public AtmosphereProcess ice_flux(icol) = precip_ice_surf_flux(icol); heat_flux(icol) = 0.0; } - } // operator + } // operator() // Local variables int m_ncol, m_npack; double m_dt; diff --git a/components/eamxx/src/physics/p3/impl/p3_init_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_init_impl.hpp index 9b4b999bce0..dff58ba3301 100644 --- a/components/eamxx/src/physics/p3/impl/p3_init_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_init_impl.hpp @@ -3,35 +3,344 @@ #include "p3_functions.hpp" // for ETI only but harmless for GPU -extern "C" { - void micro_p3_utils_init_c(scream::Real Cpair, scream::Real Rair, scream::Real RH2O, scream::Real RHO_H2O, - scream::Real MWH2O, scream::Real MWdry, scream::Real gravit, scream::Real LatVap, scream::Real LatIce, - scream::Real CpLiq, scream::Real Tmelt, scream::Real Pi, bool masterproc); - void p3_init_c(const char** lookup_file_dir, int* info, const bool& write_tables); -} +#include "ekat/util/ekat_file_utils.hpp" + +#include namespace scream { namespace p3 { +namespace { + +template +void read_ice_lookup_tables(const bool masterproc, const char* p3_lookup_base, const char* p3_version, IceT& ice_table_vals, CollT& collect_table_vals, int densize, int rimsize, int isize, int rcollsize) +{ + using DeviceIcetable = typename IceT::non_const_type; + using DeviceColtable = typename CollT::non_const_type; + + const auto ice_table_vals_d = DeviceIcetable("ice_table_vals"); + const auto collect_table_vals_d = DeviceColtable("collect_table_vals"); + + const auto ice_table_vals_h = Kokkos::create_mirror_view(ice_table_vals_d); + const auto collect_table_vals_h = Kokkos::create_mirror_view(collect_table_vals_d); + + // + // read in ice microphysics table into host views. We always read these as doubles. + // + + std::string filename = std::string(p3_lookup_base) + std::string(p3_version); + + if (masterproc) { + std::cout << "Reading ice lookup tables in file: " << filename << std::endl; + } + + std::ifstream in(filename); + + // read header + std::string version, version_val; + in >> version >> version_val; + EKAT_REQUIRE_MSG(version == "VERSION", "Bad " << filename << ", expected VERSION X.Y.Z header"); + EKAT_REQUIRE_MSG(version_val == p3_version, "Bad " << filename << ", expected version " << p3_version << ", but got " << version_val); + + // read tables + double dum_s; int dum_i; // dum_s needs to be double to stream correctly + for (int jj = 0; jj < densize; ++jj) { + for (int ii = 0; ii < rimsize; ++ii) { + for (int i = 0; i < isize; ++i) { + in >> dum_i >> dum_i; + int j_idx = 0; + for (int j = 0; j < 15; ++j) { + in >> dum_s; + if (j > 1 && j != 10) { + ice_table_vals_h(jj, ii, i, j_idx++) = dum_s; + } + } + } + + for (int i = 0; i < isize; ++i) { + for (int j = 0; j < rcollsize; ++j) { + in >> dum_i >> dum_i; + int k_idx = 0; + for (int k = 0; k < 6; ++k) { + in >> dum_s; + if (k == 3 || k == 4) { + collect_table_vals_h(jj, ii, i, j, k_idx++) = std::log10(dum_s); + } + } + } + } + } + } + + // deep copy to device + Kokkos::deep_copy(ice_table_vals_d, ice_table_vals_h); + Kokkos::deep_copy(collect_table_vals_d, collect_table_vals_h); + ice_table_vals = ice_table_vals_d; + collect_table_vals = collect_table_vals_d; +} + +template +void compute_tables(const bool masterproc, MuRT& mu_r_table_vals, VNT& vn_table_vals, VMT& vm_table_vals, RevapT& revap_table_vals) +{ + using c = scream::physics::Constants; + + int ii,jj,kk; + S lamr,mu_r,dm,dum1,dum2,dum3,dum4,dum5,dd,amg,vt,dia; + + using MuRT_NC = typename MuRT::non_const_type; + using VNT_NC = typename VNT::non_const_type; + using VMT_NC = typename VMT::non_const_type; + using RevapT_NC = typename RevapT::non_const_type; + + MuRT_NC mu_r_table_vals_nc("mu_r_table_vals"); + VNT_NC vn_table_vals_nc("vn_table_vals"); + VMT_NC vm_table_vals_nc("vm_table_vals"); + RevapT_NC revap_table_vals_nc("revap_table_vals"); + + // Get host views + auto mu_r_table_vals_h = Kokkos::create_mirror_view(mu_r_table_vals_nc); + auto revap_table_vals_h = Kokkos::create_mirror_view(revap_table_vals_nc); + auto vn_table_vals_h = Kokkos::create_mirror_view(vn_table_vals_nc); + auto vm_table_vals_h = Kokkos::create_mirror_view(vm_table_vals_nc); + + if (masterproc) { + std::cout << "Recomputing lookup (non-ice) tables" << std::endl; + } + + // ------------------------------------------------------------------------------------------ + + // Generate lookup table for rain shape parameter mu_r + // this is very fast so it can be generated at the start of each run + // make a 150x1 1D lookup table, this is done in parameter + // space of a scaled mean size proportional qr/Nr -- initlamr + + // write(iulog,*) ' Generating rain lookup-table ...' + + // AaronDonahue: Switching to table ver 4 means switching to a constand mu_r, + // so this section is commented out. + Kokkos::deep_copy(mu_r_table_vals_h, 1); // mu_r_constant =1. In other places, this is runtime_options.constant_mu_rain + + static constexpr S thrd = 1./3; + static constexpr S small = 1.e-30; + + //....................................................................... + // Generate lookup table for rain fallspeed and ventilation parameters + // the lookup table is two dimensional as a function of number-weighted mean size + // proportional to qr/Nr and shape parameter mu_r + for (ii = 1; ii <= 10; ++ii) { + mu_r = 1; // mu_r_constant = 1 + + // loop over number-weighted mean size + for (jj = 1; jj <= 300; ++jj) { + if (jj <= 20) { + dm = (jj*10 - 5)*1.e-6; // mean size [m] + } + else { + dm = ((jj-20)*30 + 195)*1.e-6; // mean size [m] + } + + lamr = (mu_r + 1)/dm; + + // do numerical integration over PSD + + dum1 = 0; // numerator, number-weighted fallspeed + dum2 = 0; // denominator, number-weighted fallspeed + dum3 = 0; // numerator, mass-weighted fallspeed + dum4 = 0; // denominator, mass-weighted fallspeed + dum5 = 0; // term for ventilation factor in evap + dd = 2; + + // loop over PSD to numerically integrate number and mass-weighted mean fallspeeds + for (kk = 1; kk <= 10000; ++kk) { + + dia = (kk*dd - dd*0.5)*1.e-6; // size bin [m] + amg = c::PIOV6*997 * std::pow(dia, 3); // mass [kg] + amg = amg*1000; // convert [kg] to [g] + + // get fallspeed as a function of size [m s-1] + if (dia*1.e+6 <= 134.43) { + vt = 4.5795e+3 * std::pow(amg, 2*thrd); + } + else if (dia*1.e+6 < 1511.64) { + vt = 4.962e+1 * std::pow(amg, thrd); + } + else if (dia*1.e+6 < 3477.84) { + vt = 1.732e+1 * std::pow(amg, c::SXTH); + } + else { + vt = 9.17; + } + + // note: factor of 4.*mu_r is non-answer changing and only needed to + // prevent underflow/overflow errors, same with 3.*mu_r for dum5 + dum1 += vt * std::pow(10, mu_r*std::log10(dia) + 4*mu_r) * std::exp(-lamr*dia) * dd * 1.e-6; + dum2 += std::pow(10, mu_r*std::log10(dia) + 4*mu_r) * std::exp(-lamr*dia) * dd * 1.e-6; + dum3 += vt * std::pow(10, (mu_r+3)*std::log10(dia) + 4*mu_r) * std::exp(-lamr*dia) * dd * 1.e-6; + dum4 += std::pow(10, (mu_r+3)*std::log10(dia) + 4*mu_r) * std::exp(-lamr*dia) * dd * 1.e-6; + dum5 += std::pow(vt*dia, 0.5) * std::pow(10, (mu_r+1)*std::log10(dia) + 3*mu_r) * std::exp(-lamr*dia) * dd * 1.e-6; + } + + dum2 = std::max(dum2, small); // to prevent divide-by-zero below + dum4 = std::max(dum4, small); // to prevent divide-by-zero below + dum5 = std::max(dum5, small); // to prevent log10-of-zero below + + vn_table_vals_h(jj-1,ii-1) = dum1/dum2; + vm_table_vals_h(jj-1,ii-1) = dum3/dum4; + revap_table_vals_h(jj-1,ii-1) = std::pow(10, std::log10(dum5) + (mu_r+1)*std::log10(lamr) - (3*mu_r)); + } + } + + Kokkos::deep_copy(mu_r_table_vals_nc, mu_r_table_vals_h); + Kokkos::deep_copy(revap_table_vals_nc, revap_table_vals_h); + Kokkos::deep_copy(vn_table_vals_nc, vn_table_vals_h); + Kokkos::deep_copy(vm_table_vals_nc, vm_table_vals_h); + + mu_r_table_vals = mu_r_table_vals_nc; + vn_table_vals = vn_table_vals_nc; + vm_table_vals = vm_table_vals_nc; + revap_table_vals = revap_table_vals_nc; +} + +template +static void action(const ekat::FILEPtr& fid, S* data, const size_t size) +{ + if constexpr (IsRead) { + ekat::read(data, size, fid); + } + else { + ekat::write(data, size, fid); + } +} + +template +void io_impl(const bool masterproc, const char* dir, MuRT& mu_r_table_vals, VNT& vn_table_vals, VMT& vm_table_vals, RevapT& revap_table_vals) +{ + if (masterproc) { + std::cout << (IsRead ? "Reading" : "Writing") << " lookup (non-ice) tables in dir " << dir << std::endl; + } + + std::string extension = +#ifdef SCREAM_DOUBLE_PRECISION + "8" +#else + "4" +#endif + ; + + const char* rw_flag = IsRead ? "r" : "w"; + + // Get host views + auto mu_r_table_vals_h = Kokkos::create_mirror_view(mu_r_table_vals); + auto revap_table_vals_h = Kokkos::create_mirror_view(revap_table_vals); + auto vn_table_vals_h = Kokkos::create_mirror_view(vn_table_vals); + auto vm_table_vals_h = Kokkos::create_mirror_view(vm_table_vals); + + // Add v2 because these tables are not identical to v1 due to roundoff differences + // caused by doing the math in C++ instead of f90. + std::string mu_r_filename = std::string(dir) + "/mu_r_table_vals_v2.dat" + extension; + std::string revap_filename = std::string(dir) + "/revap_table_vals_v2.dat" + extension; + std::string vn_filename = std::string(dir) + "/vn_table_vals_v2.dat" + extension; + std::string vm_filename = std::string(dir) + "/vm_table_vals_v2.dat" + extension; + + ekat::FILEPtr mu_r_file(fopen(mu_r_filename.c_str(), rw_flag)); + ekat::FILEPtr revap_file(fopen(revap_filename.c_str(), rw_flag)); + ekat::FILEPtr vn_file(fopen(vn_filename.c_str(), rw_flag)); + ekat::FILEPtr vm_file(fopen(vm_filename.c_str(), rw_flag)); + + // Read files + action(mu_r_file, mu_r_table_vals_h.data(), mu_r_table_vals.size()); + action(revap_file, revap_table_vals_h.data(), revap_table_vals.size()); + action(vn_file, vn_table_vals_h.data(), vn_table_vals.size()); + action(vm_file, vm_table_vals_h.data(), vm_table_vals.size()); + + // Copy back to device + if constexpr (IsRead) { + Kokkos::deep_copy(mu_r_table_vals, mu_r_table_vals_h); + Kokkos::deep_copy(revap_table_vals, revap_table_vals_h); + Kokkos::deep_copy(vn_table_vals, vn_table_vals_h); + Kokkos::deep_copy(vm_table_vals, vm_table_vals_h); + } +} + +template +void read_computed_tables(const bool masterproc, const char* dir, MuRT& mu_r_table_vals, VNT& vn_table_vals, VMT& vm_table_vals, RevapT& revap_table_vals) +{ + using MuRT_NC = typename MuRT::non_const_type; + using VNT_NC = typename VNT::non_const_type; + using VMT_NC = typename VMT::non_const_type; + using RevapT_NC = typename RevapT::non_const_type; + + MuRT_NC mu_r_table_vals_nc("mu_r_table_vals"); + VNT_NC vn_table_vals_nc("vn_table_vals"); + VMT_NC vm_table_vals_nc("vm_table_vals"); + RevapT_NC revap_table_vals_nc("revap_table_vals"); + + io_impl(masterproc, dir, mu_r_table_vals_nc, vn_table_vals_nc, vm_table_vals_nc, revap_table_vals_nc); + + mu_r_table_vals = mu_r_table_vals_nc; + vn_table_vals = vn_table_vals_nc; + vm_table_vals = vm_table_vals_nc; + revap_table_vals = revap_table_vals_nc; +} + +template +void write_computed_tables(const bool masterproc, const char* dir, const MuRT& mu_r_table_vals, const VNT& vn_table_vals, const VMT& vm_table_vals, const RevapT& revap_table_vals) +{ + io_impl(masterproc, dir, mu_r_table_vals, vn_table_vals, vm_table_vals, revap_table_vals); +} + +template +void compute_dnu(DnuT& dnu_table_vals) +{ + typename DnuT::non_const_type dnu_table_vals_non_const("dnu_table_vals"); + const auto dnu_table_h = Kokkos::create_mirror_view(dnu_table_vals_non_const); + dnu_table_h(0) = 0.000; + dnu_table_h(1) = -0.557; + dnu_table_h(2) = -0.430; + dnu_table_h(3) = -0.307; + dnu_table_h(4) = -0.186; + dnu_table_h(5) = -0.067; + dnu_table_h(6) = -0.050; + dnu_table_h(7) = -0.167; + dnu_table_h(8) = -0.282; + dnu_table_h(9) = -0.397; + dnu_table_h(10) = -0.512; + dnu_table_h(11) = -0.626; + dnu_table_h(12) = -0.739; + dnu_table_h(13) = -0.853; + dnu_table_h(14) = -0.966; + dnu_table_h(15) = -0.966; + Kokkos::deep_copy(dnu_table_vals_non_const, dnu_table_h); + dnu_table_vals = DnuT(dnu_table_vals_non_const); +} + +} + /* * Implementation of p3 init. Clients should NOT #include * this file, #include p3_functions.hpp instead. */ template -void Functions +typename Functions::P3LookupTables Functions ::p3_init (const bool write_tables, const bool masterproc) { - static bool is_init = false; - if (!is_init) { - using c = scream::physics::Constants; - micro_p3_utils_init_c(c::Cpair, c::Rair, c::RH2O, c::RHO_H2O, - c::MWH2O, c::MWdry, c::gravit, c::LatVap, c::LatIce, - c::CpLiq, c::Tmelt, c::Pi, masterproc); - static const char* dir = SCREAM_DATA_DIR "/tables"; - Int info; - p3_init_c(&dir, &info, write_tables); - EKAT_REQUIRE_MSG(info == 0, "p3_init_c returned info " << info); - is_init = true; + P3LookupTables lookup_tables; // This struct could be our global singleton + auto version = P3C::p3_version; + auto p3_lookup_base = P3C::p3_lookup_base; + static const char* dir = SCREAM_DATA_DIR "/tables"; + // p3_init_a (reads ice_table, collect_table) + read_ice_lookup_tables(masterproc, p3_lookup_base, version, lookup_tables.ice_table_vals, lookup_tables.collect_table_vals, P3C::densize, P3C::rimsize, P3C::isize, P3C::rcollsize); + if (write_tables) { + //p3_init_b (computes tables mu_r_table, revap_table, vn_table, vm_table) + compute_tables(masterproc, lookup_tables.mu_r_table_vals, lookup_tables.vn_table_vals, lookup_tables.vm_table_vals, lookup_tables.revap_table_vals); + write_computed_tables(masterproc, dir, lookup_tables.mu_r_table_vals, lookup_tables.vn_table_vals, lookup_tables.vm_table_vals, lookup_tables.revap_table_vals); } + else { + read_computed_tables(masterproc, dir, lookup_tables.mu_r_table_vals, lookup_tables.vn_table_vals, lookup_tables.vm_table_vals, lookup_tables.revap_table_vals); + } + // dnu is always computed/hardcoded + compute_dnu(lookup_tables.dnu_table_vals); + + return lookup_tables; } } // namespace p3 diff --git a/components/eamxx/src/physics/p3/impl/p3_table3_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_table3_impl.hpp index d099cd7c731..dbd7ea5726a 100644 --- a/components/eamxx/src/physics/p3/impl/p3_table3_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_table3_impl.hpp @@ -84,67 +84,15 @@ ::apply_table (const view_2d_table& table, template void Functions -::init_kokkos_tables (view_2d_table& vn_table_vals, view_2d_table& vm_table_vals, - view_2d_table& revap_table_vals, view_1d_table& mu_r_table_vals, - view_dnu_table& dnu) { - // initialize on host - - using DeviceTable1 = typename view_1d_table::non_const_type; - using DeviceTable2 = typename view_2d_table::non_const_type; - using DeviceDnuTable = typename view_dnu_table::non_const_type; - - const auto vn_table_vals_d = DeviceTable2("vn_table_vals"); - const auto vm_table_vals_d = DeviceTable2("vm_table_vals"); - const auto revap_table_vals_d = DeviceTable2("revap_table_vals"); - const auto mu_r_table_vals_d = DeviceTable1("mu_r_table_vals"); - const auto dnu_table_d = DeviceDnuTable("dnu"); - const auto vn_table_vals_h = Kokkos::create_mirror_view(vn_table_vals_d); - const auto vm_table_vals_h = Kokkos::create_mirror_view(vm_table_vals_d); - const auto revap_table_vals_h = Kokkos::create_mirror_view(revap_table_vals_d); - const auto mu_table_h = Kokkos::create_mirror_view(mu_r_table_vals_d); - const auto dnu_table_h = Kokkos::create_mirror_view(dnu_table_d); - - // Need 2d-tables with fortran-style layout - using P3F = Functions; - using LHostTable2 = typename P3F::KT::template lview; - LHostTable2 vn_table_vals_lh("vn_table_vals_lh"), vm_table_vals_lh("vm_table_vals_lh"), revap_table_vals_lh("revap_table_vals_lh"); - init_tables_from_f90_c(vn_table_vals_lh.data(), vm_table_vals_lh.data(), revap_table_vals_lh.data(), mu_table_h.data()); - for (int i = 0; i < C::VTABLE_DIM0; ++i) { - for (int j = 0; j < C::VTABLE_DIM1; ++j) { - vn_table_vals_h(i, j) = vn_table_vals_lh(i, j); - vm_table_vals_h(i, j) = vm_table_vals_lh(i, j); - revap_table_vals_h(i, j) = revap_table_vals_lh(i, j); - } - } - - dnu_table_h(0) = 0.000; - dnu_table_h(1) = -0.557; - dnu_table_h(2) = -0.430; - dnu_table_h(3) = -0.307; - dnu_table_h(4) = -0.186; - dnu_table_h(5) = -0.067; - dnu_table_h(6) = -0.050; - dnu_table_h(7) = -0.167; - dnu_table_h(8) = -0.282; - dnu_table_h(9) = -0.397; - dnu_table_h(10) = -0.512; - dnu_table_h(11) = -0.626; - dnu_table_h(12) = -0.739; - dnu_table_h(13) = -0.853; - dnu_table_h(14) = -0.966; - dnu_table_h(15) = -0.966; - - // deep copy to device - Kokkos::deep_copy(vn_table_vals_d, vn_table_vals_h); - Kokkos::deep_copy(vm_table_vals_d, vm_table_vals_h); - Kokkos::deep_copy(revap_table_vals_d, revap_table_vals_h); - Kokkos::deep_copy(mu_r_table_vals_d, mu_table_h); - Kokkos::deep_copy(dnu_table_d, dnu_table_h); - vn_table_vals = vn_table_vals_d; - vm_table_vals = vm_table_vals_d; - revap_table_vals = revap_table_vals_d; - mu_r_table_vals = mu_r_table_vals_d; - dnu = dnu_table_d; +::get_global_tables (view_2d_table& vn_table_vals, view_2d_table& vm_table_vals, + view_2d_table& revap_table_vals, view_1d_table& mu_r_table_vals, + view_dnu_table& dnu) { + auto tables = p3_init(); + vn_table_vals = tables.vn_table_vals; + vm_table_vals = tables.vm_table_vals; + revap_table_vals = tables.revap_table_vals; + mu_r_table_vals = tables.mu_r_table_vals; + dnu = tables.dnu_table_vals; } } // namespace p3 diff --git a/components/eamxx/src/physics/p3/impl/p3_table_ice_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_table_ice_impl.hpp index 4c2a43603da..9b28a4988bc 100644 --- a/components/eamxx/src/physics/p3/impl/p3_table_ice_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_table_ice_impl.hpp @@ -15,66 +15,10 @@ namespace p3 { template void Functions -::init_kokkos_ice_lookup_tables(view_ice_table& ice_table_vals, view_collect_table& collect_table_vals) { - - using DeviceIcetable = typename view_ice_table::non_const_type; - using DeviceColtable = typename view_collect_table::non_const_type; - - const auto ice_table_vals_d = DeviceIcetable("ice_table_vals"); - const auto collect_table_vals_d = DeviceColtable("collect_table_vals"); - - const auto ice_table_vals_h = Kokkos::create_mirror_view(ice_table_vals_d); - const auto collect_table_vals_h = Kokkos::create_mirror_view(collect_table_vals_d); - - // - // read in ice microphysics table into host views - // - - std::string filename = std::string(P3C::p3_lookup_base) + std::string(P3C::p3_version); - - std::ifstream in(filename); - - // read header - std::string version, version_val; - in >> version >> version_val; - EKAT_REQUIRE_MSG(version == "VERSION", "Bad " << filename << ", expected VERSION X.Y.Z header"); - EKAT_REQUIRE_MSG(version_val == P3C::p3_version, "Bad " << filename << ", expected version " << P3C::p3_version << ", but got " << version_val); - - // read tables - double dum_s; int dum_i; // dum_s needs to be double to stream correctly - for (int jj = 0; jj < P3C::densize; ++jj) { - for (int ii = 0; ii < P3C::rimsize; ++ii) { - for (int i = 0; i < P3C::isize; ++i) { - in >> dum_i >> dum_i; - int j_idx = 0; - for (int j = 0; j < 15; ++j) { - in >> dum_s; - if (j > 1 && j != 10) { - ice_table_vals_h(jj, ii, i, j_idx++) = dum_s; - } - } - } - - for (int i = 0; i < P3C::isize; ++i) { - for (int j = 0; j < P3C::rcollsize; ++j) { - in >> dum_i >> dum_i; - int k_idx = 0; - for (int k = 0; k < 6; ++k) { - in >> dum_s; - if (k == 3 || k == 4) { - collect_table_vals_h(jj, ii, i, j, k_idx++) = std::log10(dum_s); - } - } - } - } - } - } - - // deep copy to device - Kokkos::deep_copy(ice_table_vals_d, ice_table_vals_h); - Kokkos::deep_copy(collect_table_vals_d, collect_table_vals_h); - ice_table_vals = ice_table_vals_d; - collect_table_vals = collect_table_vals_d; +::get_global_ice_lookup_tables(view_ice_table& ice_table_vals, view_collect_table& collect_table_vals) { + auto tables = p3_init(); + ice_table_vals = tables.ice_table_vals; + collect_table_vals = tables.collect_table_vals; } template diff --git a/components/eamxx/src/physics/p3/p3_functions.hpp b/components/eamxx/src/physics/p3/p3_functions.hpp index eef5a8ec73e..f91c08da93c 100644 --- a/components/eamxx/src/physics/p3/p3_functions.hpp +++ b/components/eamxx/src/physics/p3/p3_functions.hpp @@ -131,6 +131,9 @@ struct Functions Scalar deposition_nucleation_exponent = 0.304; Scalar ice_sedimentation_factor = 1.0; bool do_ice_production = true; + bool set_cld_frac_l_to_one = false; + bool set_cld_frac_i_to_one = false; + bool set_cld_frac_r_to_one = false; void load_runtime_options_from_file(ekat::ParameterList& params) { max_total_ni = params.get("max_total_ni", max_total_ni); @@ -153,6 +156,9 @@ struct Functions deposition_nucleation_exponent = params.get("deposition_nucleation_exponent", deposition_nucleation_exponent); ice_sedimentation_factor = params.get("ice_sedimentation_factor", ice_sedimentation_factor); do_ice_production = params.get("do_ice_production", do_ice_production); + set_cld_frac_l_to_one = params.get("set_cld_frac_l_to_one", set_cld_frac_l_to_one); + set_cld_frac_i_to_one = params.get("set_cld_frac_i_to_one", set_cld_frac_i_to_one); + set_cld_frac_r_to_one = params.get("set_cld_frac_r_to_one", set_cld_frac_r_to_one); } }; @@ -349,16 +355,16 @@ struct Functions // --------- Functions --------- // - // Call from host to initialize the static table entries. - static void init_kokkos_tables( + // Call to get global tables + static void get_global_tables( view_2d_table& vn_table_vals, view_2d_table& vm_table_vals, view_2d_table& revap_table_vals, view_1d_table& mu_r_table_vals, view_dnu_table& dnu); - static void init_kokkos_ice_lookup_tables( + static void get_global_ice_lookup_tables( view_ice_table& ice_table_vals, view_collect_table& collect_table_vals); - static void p3_init(const bool write_tables = false, - const bool masterproc = false); + static P3LookupTables p3_init(const bool write_tables = false, + const bool masterproc = false); // Map (mu_r, lamr) to Table3 data. KOKKOS_FUNCTION @@ -1417,13 +1423,6 @@ struct Functions template constexpr ScalarT Functions::P3C::lookup_table_1a_dum1_c; -extern "C" { -// decl of fortran function for loading tables from fortran p3. This will -// continue to be a bit awkward until we have fully ported all of p3. -void init_tables_from_f90_c(Real* vn_table_vals_data, Real* vm_table_vals_data, - Real* revap_table_vals_data, Real* mu_table_data); -} - } // namespace p3 } // namespace scream diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 deleted file mode 100644 index 71c846b7167..00000000000 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ /dev/null @@ -1,158 +0,0 @@ -module p3_iso_c - use iso_c_binding - implicit none - -#include "scream_config.f" -#ifdef SCREAM_DOUBLE_PRECISION -# define c_real c_double -#else -# define c_real c_float -#endif - -! -! This file contains bridges from scream c++ to micro_p3 fortran. -! - -contains - subroutine init_tables_from_f90_c(vn_table_vals_c, vm_table_vals_c, revap_table_vals_c, mu_table_c) bind(C) - use micro_p3, only: p3_get_tables - - real(kind=c_real), intent(inout), dimension(300,10) :: vn_table_vals_c, vm_table_vals_c, revap_table_vals_c - real(kind=c_real), intent(inout), dimension(150) :: mu_table_c - - real(kind=c_real), dimension(150), target :: mu_table_f - real(kind=c_real), dimension(300,10), target :: vn_table_vals_f, vm_table_vals_f, revap_table_vals_f - - call p3_get_tables(mu_table_f, revap_table_vals_f, vn_table_vals_f, vm_table_vals_f) - vn_table_vals_c(:,:) = vn_table_vals_f(:,:) - vm_table_vals_c(:,:) = vm_table_vals_f(:,:) - revap_table_vals_c(:,:) = revap_table_vals_f(:,:) - mu_table_c(:) = mu_table_f(:) - - end subroutine init_tables_from_f90_c - - subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) - use ekat_array_io_mod, only: array_io_file_exists -#ifdef SCREAM_DOUBLE_PRECISION - use ekat_array_io_mod, only: array_io_read=>array_io_read_double, array_io_write=>array_io_write_double -#else - use ekat_array_io_mod, only: array_io_read=>array_io_read_float, array_io_write=>array_io_write_float -#endif - use micro_p3, only: p3_init_a, p3_init_b, p3_set_tables, p3_get_tables - - type(c_ptr), intent(in) :: lookup_file_dir_c - integer(kind=c_int), intent(out) :: info - logical(kind=c_bool), intent(in) :: write_tables - - real(kind=c_real), dimension(150), target :: mu_r_table_vals - real(kind=c_real), dimension(300,10), target :: vn_table_vals, vm_table_vals, revap_table_vals - - character(len=256), pointer :: lookup_file_dir - character(kind=c_char, len=512) :: mu_r_filename, revap_filename, vn_filename, vm_filename - integer :: len - logical :: ok - character(len=16) :: p3_version="4.1.1" ! TODO: Change to be dependent on table version and path specified in p3_functions.hpp - - call c_f_pointer(lookup_file_dir_c, lookup_file_dir) - len = index(lookup_file_dir, C_NULL_CHAR) - 1 - call p3_init_a(lookup_file_dir(1:len),p3_version) - - info = 0 - ok = .false. - -#ifdef SCREAM_DOUBLE_PRECISION - mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat8'//C_NULL_CHAR - revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat8'//C_NULL_CHAR - vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat8'//C_NULL_CHAR - vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat8'//C_NULL_CHAR -#else - mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat4'//C_NULL_CHAR - revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat4'//C_NULL_CHAR - vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat4'//C_NULL_CHAR - vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat4'//C_NULL_CHAR -#endif - - if (write_tables) then - call p3_init_b() - call p3_get_tables(mu_r_table_vals, revap_table_vals, vn_table_vals, vm_table_vals) - ok = array_io_write(mu_r_filename, c_loc(mu_r_table_vals), size(mu_r_table_vals)) .and. & - array_io_write(revap_filename, c_loc(revap_table_vals), size(revap_table_vals)) .and. & - array_io_write(vn_filename, c_loc(vn_table_vals), size(vn_table_vals)) .and. & - array_io_write(vm_filename, c_loc(vm_table_vals), size(vm_table_vals)) - if (.not. ok) then - print *, 'p3_iso_c::p3_init: Error when writing table files.' - info = -1 - end if - else - ! Check table files exist - ok = array_io_file_exists(mu_r_filename) .and. & - array_io_file_exists(revap_filename) .and. & - array_io_file_exists(vn_filename) .and. & - array_io_file_exists(vm_filename) - if (.not. ok) then - print *, 'p3_iso_c::p3_init: One or more table files does not exist' - info = -2 - return - endif - - ! Read files - if (.not. array_io_read(mu_r_filename, c_loc(mu_r_table_vals), size(mu_r_table_vals))) then - print *, "p3_iso_c::p3_init: error reading mu_r table from file "//mu_r_filename - info = -3 - return - elseif (.not. array_io_read(revap_filename, c_loc(revap_table_vals), size(revap_table_vals))) then - print *, "p3_iso_c::p3_init: error reading revap table from file "//revap_filename - info = -4 - return - - elseif (.not. array_io_read(vn_filename, c_loc(vn_table_vals), size(vn_table_vals))) then - print *, "p3_iso_c::p3_init: error reading vn table from file "//vn_filename - info = -5 - return - elseif (.not. array_io_read(vm_filename, c_loc(vm_table_vals), size(vm_table_vals))) then - print *, "p3_iso_c::p3_init: error reading vm table from file "//vm_filename - info = -6 - return - endif - - call p3_set_tables(mu_r_table_vals, revap_table_vals, vn_table_vals, vm_table_vals) - end if - - end subroutine p3_init_c - - subroutine micro_p3_utils_init_c(Cpair, Rair, RH2O, RHO_H2O, & - MWH2O, MWdry, gravit, LatVap, LatIce, & - CpLiq, Tmelt, Pi, masterproc) bind(C) - - use micro_p3_utils, only: micro_p3_utils_init - use iso_fortran_env, only: OUTPUT_UNIT - real(kind=c_real), value, intent(in) :: Cpair - real(kind=c_real), value, intent(in) :: Rair - real(kind=c_real), value, intent(in) :: RH2O - real(kind=c_real), value, intent(in) :: RHO_H2O - real(kind=c_real), value, intent(in) :: MWH2O - real(kind=c_real), value, intent(in) :: MWdry - real(kind=c_real), value, intent(in) :: gravit - real(kind=c_real), value, intent(in) :: LatVap - real(kind=c_real), value, intent(in) :: LatIce - real(kind=c_real), value, intent(in) :: CpLiq - real(kind=c_real), value, intent(in) :: Tmelt - real(kind=c_real), value, intent(in) :: Pi - logical(kind=c_bool), value, intent(in) :: masterproc - - call micro_p3_utils_init(Cpair,Rair,RH2O,RHO_H2O,MWH2O,MWdry,gravit,LatVap,LatIce, & - CpLiq,Tmelt,Pi,OUTPUT_UNIT,masterproc) - end subroutine micro_p3_utils_init_c - - subroutine p3_init_a_c(ice_table_vals_c, collect_table_vals_c) bind(C) - use micro_p3, only: ice_table_vals, collect_table_vals - use micro_p3_utils, only: densize,rimsize,isize,ice_table_size,rcollsize,collect_table_size - - real(kind=c_real), intent(out), dimension(densize,rimsize,isize,ice_table_size) :: ice_table_vals_c - real(kind=c_real), intent(out), dimension(densize,rimsize,isize,rcollsize,collect_table_size) :: collect_table_vals_c - - ice_table_vals_c(:,:,:,:) = ice_table_vals(:,:,:,:) - collect_table_vals_c(:,:,:,:,:) = collect_table_vals(:,:,:,:,:) - end subroutine p3_init_a_c - -end module p3_iso_c diff --git a/components/eamxx/src/physics/p3/p3_tables_setup.cpp b/components/eamxx/src/physics/p3/p3_tables_setup.cpp deleted file mode 100644 index ec3f5ccbb43..00000000000 --- a/components/eamxx/src/physics/p3/p3_tables_setup.cpp +++ /dev/null @@ -1,8 +0,0 @@ -// This is a tiny program that calls p3_init() to generate tables used by p3 - -#include "physics/p3/p3_data.hpp" - -int main(int /* argc */, char** /* argv */) { - scream::p3::p3_init(/* write_tables = */ true); - return 0; -} diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index d705b30cc4e..b8d6c7d2b7a 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -89,7 +89,7 @@ if (NOT SCREAM_P3_SMALL_KERNELS AND NOT SCREAM_ONLY_GENERATE_BASELINES) CreateUnitTest(p3_sk_tests "p3_main_unit_tests.cpp" LIBS p3_sk p3_test_infra EXE_ARGS "--args ${BASELINE_FILE_ARG}" - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} + THREADS ${P3_THREADS} LABELS "p3_sk;physics;baseline_cmp") endif() @@ -101,6 +101,10 @@ CreateUnitTest(p3_run_and_cmp "p3_run_and_cmp.cpp" EXE_ARGS "${BASELINE_FILE_ARG}" LABELS "p3;physics;baseline_gen;baseline_cmp") +# This executable can be used to re-generate tables in ${SCREAM_DATA_DIR} +add_executable(p3_tables_setup EXCLUDE_FROM_ALL p3_tables_setup.cpp) +target_link_libraries(p3_tables_setup p3) + # Make sure that a diff from baselines triggers a failed test (in debug only) if (SCREAM_ENABLE_BASELINE_TESTS) CreateUnitTest(p3_run_and_cmp_fail "p3_run_and_cmp.cpp" diff --git a/components/eamxx/src/physics/p3/tests/infra/p3_data.hpp b/components/eamxx/src/physics/p3/tests/infra/p3_data.hpp index df5b25e311a..55e74300215 100644 --- a/components/eamxx/src/physics/p3/tests/infra/p3_data.hpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_data.hpp @@ -16,9 +16,9 @@ struct P3Data { using KT = KokkosTypes; using Scalar = Real; - using Array1 = typename KT::template lview; - using Array2 = typename KT::template lview; - using Array3 = typename KT::template lview; + using Array1 = typename KT::template view_1d; + using Array2 = typename KT::template view_2d; + using Array3 = typename KT::template view_3d; bool do_predict_nc; bool do_prescribed_CCN; diff --git a/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.cpp b/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.cpp index 2b758a513ec..d4faa4c7052 100644 --- a/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.cpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.cpp @@ -31,7 +31,6 @@ int test_p3_init () { using P3F = Functions; P3F::p3_init(); - P3GlobalForFortran::deinit(); return 0; } @@ -43,7 +42,6 @@ int test_p3_ic () { d->dt = 300.0; P3F::p3_init(); p3_main_wrap(*d); - P3GlobalForFortran::deinit(); return 0; } diff --git a/components/eamxx/src/physics/p3/tests/infra/p3_test_data.cpp b/components/eamxx/src/physics/p3/tests/infra/p3_test_data.cpp index 23a9998f43c..2dff7183ec6 100644 --- a/components/eamxx/src/physics/p3/tests/infra/p3_test_data.cpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_test_data.cpp @@ -12,23 +12,9 @@ using scream::Real; using scream::Int; -extern "C" { - -void p3_init_a_c(Real* ice_table_vals, Real* collect_table_vals); - -} // extern "C" : end _c decls - namespace scream { namespace p3 { -void p3_init_a(P3InitAP3Data& d) -{ - using P3F = Functions; - - P3F::p3_init(); // need to initialize p3 first so that tables are loaded - p3_init_a_c(d.ice_table_vals.data(), d.collect_table_vals.data()); -} - void BackToCellAverageData::randomize(std::mt19937_64& engine) { // Populate the struct with numbers between 0 and 1. @@ -329,24 +315,6 @@ void PreventLiqSupersaturationData::randomize(std::mt19937_64& engine) /////////////////////////////////////////////////////////////////////////////// -std::shared_ptr P3GlobalForFortran::s_views; - -const P3GlobalForFortran::Views& P3GlobalForFortran::get() -{ - if (!P3GlobalForFortran::s_views) { - P3GlobalForFortran::s_views = std::make_shared(); - P3F::init_kokkos_ice_lookup_tables(s_views->m_ice_table_vals, s_views->m_collect_table_vals); - P3F::init_kokkos_tables(s_views->m_vn_table_vals, s_views->m_vm_table_vals, - s_views->m_revap_table_vals, s_views->m_mu_r_table_vals, s_views->m_dnu); - } - return *P3GlobalForFortran::s_views; -} - -void P3GlobalForFortran::deinit() -{ - P3GlobalForFortran::s_views = nullptr; -} - // // _host function definitions // @@ -581,7 +549,7 @@ void cloud_sedimentation_host( const Int nk_pack = ekat::npack(nk); // Set up views - const auto dnu = P3GlobalForFortran::dnu(); + const auto dnu = P3F::p3_init().dnu_table_vals; std::vector temp_d(CloudSedData::NUM_ARRAYS); @@ -672,7 +640,7 @@ void ice_sedimentation_host( ni_tend_d (temp_d[14]); // Call core function from kernel - auto ice_table_vals = P3GlobalForFortran::ice_table_vals(); + auto ice_table_vals = P3F::p3_init().ice_table_vals; auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, nk_pack); ekat::WorkspaceManager wsm(rho_d.extent(0), 6, policy); Real my_precip_ice_surf = 0; @@ -745,8 +713,9 @@ void rain_sedimentation_host( precip_liq_flux_d(temp_d[13]); // Call core function from kernel - auto vn_table_vals = P3GlobalForFortran::vn_table_vals(); - auto vm_table_vals = P3GlobalForFortran::vm_table_vals(); + auto tables = P3F::p3_init(); + auto vn_table_vals = tables.vn_table_vals; + auto vm_table_vals = tables.vm_table_vals; auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, nk_pack); ekat::WorkspaceManager wsm(rho_d.extent(0), 4, policy); Real my_precip_liq_surf = 0; @@ -1085,10 +1054,11 @@ void p3_main_part2_host( t_prev_d (temp_d[current_index++]); // Call core function from kernel - const auto dnu = P3GlobalForFortran::dnu(); - const auto ice_table_vals = P3GlobalForFortran::ice_table_vals(); - const auto collect_table_vals = P3GlobalForFortran::collect_table_vals(); - const auto revap_table_vals = P3GlobalForFortran::revap_table_vals(); + auto tables = P3F::p3_init(); + const auto dnu = tables.dnu_table_vals; + const auto ice_table_vals = tables.ice_table_vals; + const auto collect_table_vals = tables.collect_table_vals; + const auto revap_table_vals = tables.revap_table_vals; bview_1d bools_d("bools", 1); auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, nk_pack); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { @@ -1208,8 +1178,9 @@ void p3_main_part3_host( diag_eff_radius_qr_d (temp_d[current_index++]); // Call core function from kernel - const auto dnu = P3GlobalForFortran::dnu(); - const auto ice_table_vals = P3GlobalForFortran::ice_table_vals(); + auto tables = P3F::p3_init(); + const auto dnu = tables.dnu_table_vals; + const auto ice_table_vals = tables.ice_table_vals; auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, nk_pack); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { @@ -1257,12 +1228,6 @@ Int p3_main_host( using sview_1d = typename P3F::view_1d; using sview_2d = typename P3F::view_2d; - using view_1d_table = typename P3F::view_1d_table; - using view_2d_table = typename P3F::view_2d_table; - using view_ice_table = typename P3F::view_ice_table; - using view_collect_table = typename P3F::view_collect_table; - using view_dnu_table = typename P3F::view_dnu_table; - EKAT_REQUIRE_MSG(its == 1, "its must be 1, got " << its); EKAT_REQUIRE_MSG(kts == 1, "kts must be 1, got " << kts); @@ -1299,7 +1264,7 @@ Int p3_main_host( } } - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d); int counter = 0; view_2d @@ -1403,16 +1368,7 @@ Int p3_main_host( #endif // load tables - view_1d_table mu_r_table_vals; - view_2d_table vn_table_vals, vm_table_vals, revap_table_vals; - view_ice_table ice_table_vals; - view_collect_table collect_table_vals; - view_dnu_table dnu_table_vals; - P3F::init_kokkos_ice_lookup_tables(ice_table_vals, collect_table_vals); - P3F::init_kokkos_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu_table_vals); - - P3F::P3LookupTables lookup_tables{mu_r_table_vals, vn_table_vals, vm_table_vals, revap_table_vals, - ice_table_vals, collect_table_vals, dnu_table_vals}; + auto lookup_tables = P3F::p3_init(); P3F::P3Runtime runtime_options{740.0e3}; // Create local workspace @@ -1452,7 +1408,7 @@ Int p3_main_host( rho_qi, qv2qi_depos_tend, liq_ice_exchange, vap_liq_exchange, vap_ice_exchange, precip_liq_flux, precip_ice_flux, precip_liq_surf, precip_ice_surf }, - dim1_sizes_out, dim2_sizes_out, inout_views, true); + dim1_sizes_out, dim2_sizes_out, inout_views); return elapsed_microsec; } diff --git a/components/eamxx/src/physics/p3/tests/infra/p3_test_data.hpp b/components/eamxx/src/physics/p3/tests/infra/p3_test_data.hpp index a1bbe864f4c..775120e080d 100644 --- a/components/eamxx/src/physics/p3/tests/infra/p3_test_data.hpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_test_data.hpp @@ -15,71 +15,6 @@ namespace p3 { /////////////////////////////////////////////////////////////////////////////// -struct P3InitAP3Data -{ - // Must use Host as device, f90 code might not be able to use Device memory - using P3F = Functions; - using P3C = typename P3F::P3C; - - using view_ice_table = typename P3F::KT::template lview; - using view_collect_table = typename P3F::KT::template lview; - - // Need to be LayoutLeft to be fortran compatible - view_ice_table ice_table_vals; - view_collect_table collect_table_vals; - - P3InitAP3Data() : - ice_table_vals("P3InitAP3Data::ice_table_vals"), - collect_table_vals("P3InitAP3Data::collect_table_vals") - {} -}; - -/////////////////////////////////////////////////////////////////////////////// - -// Singleton for holding the same global data that are maintained in -// micro_p3, but for use in C++. -struct P3GlobalForFortran -{ - using P3F = Functions; - - using view_1d_table = typename P3F::view_1d_table; - using view_2d_table = typename P3F::view_2d_table; - using view_ice_table = typename P3F::view_ice_table; - using view_collect_table = typename P3F::view_collect_table; - using view_dnu_table = typename P3F::view_dnu_table; - using P3Runtime = P3F::P3Runtime; - - // All kokkos views must be destructed before Kokkos::finalize - static void deinit(); - - static const view_1d_table& mu_r_table_vals() { return get().m_mu_r_table_vals; } - static const view_2d_table& vn_table_vals() { return get().m_vn_table_vals; } - static const view_2d_table& vm_table_vals() { return get().m_vm_table_vals; } - static const view_2d_table& revap_table_vals() { return get().m_revap_table_vals; } - static const view_ice_table& ice_table_vals() { return get().m_ice_table_vals; } - static const view_collect_table& collect_table_vals() { return get().m_collect_table_vals; } - static const view_dnu_table& dnu() { return get().m_dnu; } - - P3GlobalForFortran() = delete; - ~P3GlobalForFortran() = delete; - P3GlobalForFortran(const P3GlobalForFortran&) = delete; - P3GlobalForFortran& operator=(const P3GlobalForFortran&) = delete; - - private: - struct Views { - view_1d_table m_mu_r_table_vals; - view_2d_table m_vn_table_vals, m_vm_table_vals, m_revap_table_vals; - view_ice_table m_ice_table_vals; - view_collect_table m_collect_table_vals; - view_dnu_table m_dnu; - }; - - static const Views& get(); - static std::shared_ptr s_views; -}; - -/////////////////////////////////////////////////////////////////////////////// - /** * Structs for holding data related to specific P3 calls; these are used for * the BFB unit tests. @@ -882,8 +817,6 @@ struct PreventLiqSupersaturationData { PTD_RW_SCALARS_ONLY(2, qi2qv_sublim_tend, qr2qv_evap_tend); }; -void p3_init_a(P3InitAP3Data& d); - /** * Convenience functions for calling p3 routines from the host with scalar data. * These function will pack your data, sync it to device, call the p3 function, diff --git a/components/eamxx/src/physics/p3/tests/infra/p3_unit_tests_common.hpp b/components/eamxx/src/physics/p3/tests/infra/p3_unit_tests_common.hpp index 9dd7dee95b3..232963ca780 100644 --- a/components/eamxx/src/physics/p3/tests/infra/p3_unit_tests_common.hpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_unit_tests_common.hpp @@ -83,7 +83,7 @@ struct UnitWrap { m_baseline_action(NONE), m_fid() { - Functions::p3_init(); // many tests will need fortran table data + Functions::p3_init(); // many tests will need table data auto& ts = ekat::TestSession::get(); if (ts.flags["c"]) { m_baseline_action = COMPARE; @@ -108,10 +108,7 @@ struct UnitWrap { } } - ~Base() - { - scream::p3::P3GlobalForFortran::deinit(); - } + ~Base() {} std::mt19937_64 get_engine() { diff --git a/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp index 2752746b792..cf302d78166 100644 --- a/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp @@ -31,10 +31,7 @@ struct UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale : public UnitWrap:: auto engine = Base::get_engine(); // Read in tables - view_2d_table vn_table_vals, vm_table_vals, revap_table_vals; - view_1d_table mu_r_table_vals; - view_dnu_table dnu; - Functions::init_kokkos_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu); + auto revap_table_vals = Functions::p3_init().revap_table_vals; using KTH = KokkosTypes; diff --git a/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp index 268607645f0..9eb783b0eb6 100644 --- a/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp @@ -29,7 +29,7 @@ struct UnitWrap::UnitTest::TestDsd2 : public UnitWrap::UnitTest::Base { // Read in tables view_2d_table vn_table_vals; view_2d_table vm_table_vals; view_2d_table revap_table_vals; view_1d_table mu_r_table_vals; view_dnu_table dnu; - Functions::init_kokkos_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu); + Functions::get_global_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu); // Load some lookup inputs, need at least one per pack value GetCloudDsd2Data gcdd[max_pack_size] = { diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp index 2561114b519..5d36ee79d54 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp @@ -31,7 +31,7 @@ struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest:: view_2d_table vm_table_vals; view_2d_table revap_table_vals; view_1d_table mu_r_table_vals; view_dnu_table dnu; - Functions::init_kokkos_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu); + Functions::get_global_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu); // Load some lookup inputs, need at least one per pack value IceCldliqCollectionData cldliq[max_pack_size] = { diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp index ea3fdaccf7e..0c75d81ab6e 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp @@ -24,43 +24,6 @@ namespace unit_test { template struct UnitWrap::UnitTest::TestTableIce : public UnitWrap::UnitTest::Base { - void test_read_lookup_tables_bfb() - { - // Read in ice tables - view_ice_table ice_table_vals; - view_collect_table collect_table_vals; - Functions::init_kokkos_ice_lookup_tables(ice_table_vals, collect_table_vals); - - // Get data from fortran - P3InitAP3Data d; - p3_init_a(d); - - // Copy device data to host - const auto ice_table_vals_host = Kokkos::create_mirror_view(ice_table_vals); - const auto collect_table_vals_host = Kokkos::create_mirror_view(collect_table_vals); - Kokkos::deep_copy(ice_table_vals_host, ice_table_vals); - Kokkos::deep_copy(collect_table_vals_host, collect_table_vals); - - // Compare (on host) - for (size_t i = 0; i < ice_table_vals_host.extent(0); ++i) { - for (size_t j = 0; j < ice_table_vals_host.extent(1); ++j) { - for (size_t k = 0; k < ice_table_vals_host.extent(2); ++k) { - - for (size_t l = 0; l < ice_table_vals_host.extent(3); ++l) { - REQUIRE(ice_table_vals_host(i, j, k, l) == d.ice_table_vals(i, j, k, l)); - } - - for (size_t l = 0; l < collect_table_vals_host.extent(3); ++l) { - for (size_t m = 0; m < collect_table_vals_host.extent(4); ++m) { - REQUIRE(collect_table_vals_host(i, j, k, l, m) == d.collect_table_vals(i, j, k, l, m)); - } - } - - } - } - } - } - template void init_table_linear_dimension(View& table, int linear_dimension) { @@ -103,7 +66,7 @@ struct UnitWrap::UnitTest::TestTableIce : public UnitWrap::UnitTest::Base // Read in ice tables view_ice_table ice_table_vals; view_collect_table collect_table_vals; - Functions::init_kokkos_ice_lookup_tables(ice_table_vals, collect_table_vals); + Functions::get_global_ice_lookup_tables(ice_table_vals, collect_table_vals); constexpr Scalar qsmall = C::QSMALL; @@ -373,7 +336,6 @@ TEST_CASE("p3_ice_tables", "[p3_functions]") using T = scream::p3::unit_test::UnitWrap::UnitTest::TestTableIce; T t; - t.test_read_lookup_tables_bfb(); t.run_phys(); t.run_bfb(); } diff --git a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp index 4fb471c3cf5..c60efd2b882 100644 --- a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp @@ -445,7 +445,6 @@ void run_bfb_p3_main() // Get data from cxx for (auto& d : isds_cxx) { - d.template transpose(); p3_main_host( d.qc, d.nc, d.qr, d.nr, d.th_atm, d.qv, d.dt, d.qi, d.qm, d.ni, d.bm, d.pres, d.dz, d.nc_nuceat_tend, d.nccn_prescribed, d.ni_activated, d.inv_qc_relvar, d.it, d.precip_liq_surf, @@ -453,7 +452,6 @@ void run_bfb_p3_main() d.rho_qi, d.do_predict_nc, d.do_prescribed_CCN, d.dpres, d.inv_exner, d.qv2qi_depos_tend, d.precip_liq_flux, d.precip_ice_flux, d.cld_frac_r, d.cld_frac_l, d.cld_frac_i, d.liq_ice_exchange, d.vap_liq_exchange, d.vap_ice_exchange, d.qv_prev, d.t_prev); - d.template transpose(); } if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp index cd4d1955aed..ac34856eea6 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp @@ -42,7 +42,7 @@ void run_bfb_rain_vel() // Read in tables view_2d_table vn_table_vals; view_2d_table vm_table_vals; view_2d_table revap_table_vals; view_1d_table mu_r_table_vals; view_dnu_table dnu; - Functions::init_kokkos_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu); + Functions::get_global_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu); // Load some lookup inputs, need at least one per pack value ComputeRainFallVelocityData crfv_baseline[max_pack_size] = { diff --git a/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp b/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp index ff069032013..f596b0910eb 100644 --- a/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp @@ -260,18 +260,18 @@ int main (int argc, char** argv) { std::string predict_nc = "both"; std::string prescribed_ccn = "both"; std::string baseline_fn; - for (int i = 1; i < argc-1; ++i) { + for (int i = 1; i < argc; ++i) { if (ekat::argv_matches(argv[i], "-g", "--generate")) { generate = true; no_baseline = false; } if (ekat::argv_matches(argv[i], "-c", "--compare")) { no_baseline = false; } - if (ekat::argv_matches(argv[i], "-t", "--tol")) { + if (ekat::argv_matches(argv[i], "-b", "--baseline-file")) { expect_another_arg(i, argc); ++i; - tol = std::atof(argv[i]); + baseline_fn = argv[i]; } - if (ekat::argv_matches(argv[i], "-b", "--baseline-file")) { + if (ekat::argv_matches(argv[i], "-t", "--tol")) { expect_another_arg(i, argc); ++i; - baseline_fn = argv[i]; + tol = std::atof(argv[i]); } if (ekat::argv_matches(argv[i], "-s", "--steps")) { expect_another_arg(i, argc); @@ -301,9 +301,6 @@ int main (int argc, char** argv) { expect_another_arg(i, argc); ++i; repeat = std::atoi(argv[i]); - if (repeat > 0) { - generate = true; - } } if (ekat::argv_matches(argv[i], "-pn", "--predict-nc")) { expect_another_arg(i, argc); @@ -338,7 +335,6 @@ int main (int argc, char** argv) { printf("Comparing with %s at tol %1.1e\n", baseline_fn.c_str(), tol); nerr += bln.run_and_cmp(baseline_fn, tol, no_baseline); } - P3GlobalForFortran::deinit(); } scream::finalize_scream_session(); diff --git a/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp index c58ac68b4a6..809c98558f5 100644 --- a/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp @@ -92,9 +92,9 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling : public UnitWrap::Un Spack c_scaling = Functions::subgrid_variance_scaling(relvars,1.0); if ( std::abs(c_scaling[0] - 1) > tol ){ - printf("subgrid_variance_scaling should be 1 for expon=1, but is %e. " + Kokkos::printf("subgrid_variance_scaling should be 1 for expon=1, but is %e. " "Diff = %e, Tol = %e\n",c_scaling[0],c_scaling[0]-1, tol); - errors++;} + errors++;} } //----------------------------------------------------------------- @@ -110,7 +110,7 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling : public UnitWrap::Un Real fact = std::tgamma(5.0); //factorial(n) = gamma(n+1) if ( std::abs(c_scaling[0] - fact) > tol ){ - printf("subgrid_variance_scaling should be factorial(expon) when relvar=1. " + Kokkos::printf("subgrid_variance_scaling should be factorial(expon) when relvar=1. " "For expon=4, should be %f but is=%f\n Diff = %e, Tol = %e\n", fact,c_scaling[0], c_scaling[0] - fact, tol); errors++;} @@ -143,7 +143,7 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling : public UnitWrap::Un const Real max_tol = tol*cond_num; if ( std::abs(targ - c_scaling[0]) > max_tol * targ ){ - printf("When expon=3, subgrid_variance_scaling doesn't match analytic expectation. " + Kokkos::printf("When expon=3, subgrid_variance_scaling doesn't match analytic expectation. " "Val = %e, expected = %e, rel diff = %e, tol = %e\n", c_scaling[0],targ, (targ-c_scaling[0]), max_tol*targ ); errors++; diff --git a/components/eamxx/src/physics/p3/tests/p3_table3_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_table3_unit_tests.cpp index 57197fea6c8..001a875c7e8 100644 --- a/components/eamxx/src/physics/p3/tests/p3_table3_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_table3_unit_tests.cpp @@ -76,7 +76,7 @@ struct UnitWrap::UnitTest::TestTable3 : public UnitWrap::UnitTest::Base { view_1d_table mu_r_table_vals; view_2d_table vn_table_vals, vm_table_vals, revap_table_vals; view_dnu_table dnu; - Functions::init_kokkos_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu); + Functions::get_global_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu); // Estimate two maximum slope magnitudes for two meshes, the second 10x // refined w.r.t. the first. diff --git a/components/eamxx/src/physics/p3/tests/p3_tables_setup.cpp b/components/eamxx/src/physics/p3/tests/p3_tables_setup.cpp new file mode 100644 index 00000000000..ad9b299a4d9 --- /dev/null +++ b/components/eamxx/src/physics/p3/tests/p3_tables_setup.cpp @@ -0,0 +1,14 @@ +// This is a tiny program that calls p3_init() to generate tables used by p3 + +#include "physics/p3/p3_functions.hpp" +#include "share/scream_session.hpp" + +int main(int argc, char** argv) { + using P3F = scream::p3::Functions; + + scream::initialize_scream_session(argc, argv); + P3F::p3_init(/* write_tables = */ true, /* masterproc */ true); + scream::finalize_scream_session(); + + return 0; +} diff --git a/components/eamxx/src/physics/register_physics.hpp b/components/eamxx/src/physics/register_physics.hpp index 99956bb75f5..dc1ce5745d3 100644 --- a/components/eamxx/src/physics/register_physics.hpp +++ b/components/eamxx/src/physics/register_physics.hpp @@ -41,6 +41,9 @@ #ifdef EAMXX_HAS_ML_CORRECTION #include "physics/ml_correction/eamxx_ml_correction_process_interface.hpp" #endif +#ifdef EAMXX_HAS_IOP_FORCING +#include "physics/iop_forcing/eamxx_iop_forcing_process_interface.hpp" +#endif namespace scream { @@ -82,6 +85,9 @@ inline void register_physics () { #ifdef EAMXX_HAS_ML_CORRECTION proc_factory.register_product("MLCorrection",&create_atmosphere_process); #endif +#ifdef EAMXX_HAS_IOP_FORCING + proc_factory.register_product("iop_forcing",&create_atmosphere_process); +#endif // If no physics was enabled, silence compile warning about unused var (void) proc_factory; diff --git a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt index a8e47384552..676f4558d23 100644 --- a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt @@ -72,6 +72,15 @@ else () string (REPLACE " " ";" YAKL_HIP_FLAGS_LIST ${YAKL_HIP_FLAGS}) endif() + ####### SYCL here + if (SYCL_BUILD) + set(YAKL_ARCH "SYCL") + set(YAKL_SYCL_FLAGS " -fp-model precise -DYAKL_ARCH_SYCL -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64") + string (REPLACE " " ";" YAKL_SYCL_FLAGS_LIST ${YAKL_SYCL_FLAGS}) + endif() + + + set (YAKL_SOURCE_DIR ${SCREAM_BASE_DIR}/../../externals/YAKL) add_subdirectory(${YAKL_SOURCE_DIR} ${CMAKE_BINARY_DIR}/externals/YAKL) @@ -168,12 +177,17 @@ yakl_process_target(scream_rrtmgp_yakl) # NOTE: cannot use 'PUBLIC' in target_link_libraries, # since yakl_process_target already used it # with the "plain" signature + +#<<<<<<< HEAD +#find_library(NETCDF_C netcdf HINTS $ENV{NETCDF_C_PATH}/lib) +#target_link_libraries(scream_rrtmgp_yakl ${NETCDF_C} rrtmgp scream_share) +#======= find_library(NETCDF_C netcdf HINTS ${NetCDF_C_PATH}/lib) target_link_libraries(scream_rrtmgp_yakl ${NETCDF_C} rrtmgp scream_share Kokkos::kokkos) target_include_directories(scream_rrtmgp_yakl PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories(scream_rrtmgp_yakl SYSTEM PUBLIC - ${NetCDF_C_PATH}/include + ${NETCDF_C_PATH}/include ${EAM_RRTMGP_DIR}/external) ################################## diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 0709bb1a37f..0121741963d 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -99,16 +99,16 @@ void RRTMGPRadiation::set_grids(const std::shared_ptr grids_ m_ncol = m_grid->get_num_local_dofs(); m_nlay = m_grid->get_num_vertical_levels(); - if (m_iop) { + if (m_iop_data_manager) { // For IOP runs, we need to use the lat/lon from the // IOP files instead of the geometry data. Deep copy // to device and sync to host since both will be used. m_lat = m_grid->get_geometry_data("lat").clone(); - m_lat.deep_copy(m_iop->get_params().get("target_latitude")); + m_lat.deep_copy(m_iop_data_manager->get_params().get("target_latitude")); m_lat.sync_to_host(); m_lon = m_grid->get_geometry_data("lon").clone(); - m_lon.deep_copy(m_iop->get_params().get("target_longitude")); + m_lon.deep_copy(m_iop_data_manager->get_params().get("target_longitude")); m_lon.sync_to_host(); } else { m_lat = m_grid->get_geometry_data("lat"); diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp index 466467919fc..a8864ca9934 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp @@ -46,9 +46,9 @@ template ; -using real1dk = typename interface_t::view_t; -using real2dk = typename interface_t::view_t; -using real3dk = typename interface_t::view_t; +using real1dk = typename interface_t::template view_t; +using real2dk = typename interface_t::template view_t; +using real3dk = typename interface_t::template view_t; using MDRP = typename conv::MDRP; static bool all_close(real2dk &arr1, real2dk &arr2, double tolerance) diff --git a/components/eamxx/src/physics/rrtmgp/tests/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/tests/CMakeLists.txt index c6fcfae76f8..ee8e4a383c4 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/tests/CMakeLists.txt @@ -1,14 +1,12 @@ if (SCREAM_ONLY_GENERATE_BASELINES) - # Build baseline code - add_executable(generate_baseline generate_baseline.cpp) - target_link_libraries(generate_baseline PUBLIC scream_rrtmgp rrtmgp_test_utils) - - # Generate allsky baseline with the usual cmake custom command-target pair pattern + # Generate allsky baseline # Note: these "baselines" are not to compare scream with a previous version, but # rather to compare scream::rrtmgp with raw rrtmgp. - CreateUnitTestFromExec( - rrtmgp-allsky-baseline generate_baseline + CreateUnitTest ( + rrtmgp-allsky-baseline generate_baseline.cpp + LIBS scream_rrtmgp rrtmgp_test_utils LABELS baseline_gen rrtmgp + EXCLUDE_MAIN_CPP EXE_ARGS "${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc ${SCREAM_BASELINES_DIR}/data/rrtmgp-allsky-baseline.nc" ) diff --git a/components/eamxx/src/physics/share/CMakeLists.txt b/components/eamxx/src/physics/share/CMakeLists.txt index d3c64a4a0b9..44ce0aba57b 100644 --- a/components/eamxx/src/physics/share/CMakeLists.txt +++ b/components/eamxx/src/physics/share/CMakeLists.txt @@ -3,11 +3,6 @@ set(PHYSICS_SHARE_SRCS physics_share.cpp physics_test_data.cpp scream_trcmix.cpp - ${SCREAM_BASE_DIR}/../eam/src/physics/cam/physics_utils.F90 - ${SCREAM_BASE_DIR}/../eam/src/physics/cam/scream_abortutils.F90 - ${SCREAM_BASE_DIR}/../eam/src/physics/cam/wv_sat_scream.F90 - ${SCREAM_BASE_DIR}/../eam/src/physics/p3/scream/micro_p3_utils.F90 - ${SCREAM_BASE_DIR}/../eam/src/physics/cam/debug_info.F90 ) # Add ETI source files if not on CUDA/HIP diff --git a/components/eamxx/src/physics/share/physics_share_f2c.F90 b/components/eamxx/src/physics/share/physics_share_f2c.F90 index 1bd3a7d21d7..885b026eac7 100644 --- a/components/eamxx/src/physics/share/physics_share_f2c.F90 +++ b/components/eamxx/src/physics/share/physics_share_f2c.F90 @@ -101,7 +101,7 @@ function scream_expm1(input) bind(C) ! return real(kind=c_real) :: scream_expm1 end function scream_expm1 - + function scream_tanh(input) bind(C) use iso_c_binding diff --git a/components/eamxx/src/physics/shoc/CMakeLists.txt b/components/eamxx/src/physics/shoc/CMakeLists.txt index 4796b30abef..3686c3cce7f 100644 --- a/components/eamxx/src/physics/shoc/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/CMakeLists.txt @@ -1,19 +1,7 @@ set(SHOC_SRCS - shoc_f90.cpp - shoc_ic_cases.cpp - shoc_iso_c.f90 - shoc_iso_f.f90 - ${SCREAM_BASE_DIR}/../eam/src/physics/cam/shoc.F90 eamxx_shoc_process_interface.cpp ) -if (NOT SCREAM_LIB_ONLY) - list(APPEND SHOC_SRCS - shoc_functions_f90.cpp - shoc_main_wrap.cpp - ) # Add f90 bridges needed for testing -endif() - set(SHOC_HEADERS shoc.hpp eamxx_shoc_process_interface.hpp diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index ac620e19add..1bbb17f8480 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -449,7 +449,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) const auto ncols = m_num_cols; view_1d cell_length("cell_length", ncols); if (m_grid->has_geometry_data("dx_short")) { - // We must be running with IntensiveObservationPeriod on, with a planar geometry + // In this case IOP is running with a planar geometry auto dx = m_grid->get_geometry_data("dx_short").get_view()(); Kokkos::deep_copy(cell_length, dx*1000); // convert km -> m } else { diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp index cd004adbd2d..19d897c63d9 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp @@ -153,10 +153,10 @@ void Functions::shoc_assumed_pdf( const Smask is_nan_Tl1_1 = isnan(Tl1_1) && active_entries; const Smask is_nan_Tl1_2 = isnan(Tl1_2) && active_entries; if (is_nan_Tl1_1.any() || is_nan_Tl1_2.any()) { - printf("WARNING: NaN Detected in Tl1_1 or Tl1_2!\n"); + Kokkos::printf("WARNING: NaN Detected in Tl1_1 or Tl1_2!\n"); for (int i=0; i::shoc_assumed_pdf( n_mask++; } } - printf("WARNING: Tl1_1 has %d values <= allowable value. Resetting to minimum value.\n",n_mask); + Kokkos::printf("WARNING: Tl1_1 has %d values <= allowable value. Resetting to minimum value.\n",n_mask); } if( is_small_Tl1_2.any() ) { Tl1_2.set(is_small_Tl1_2,Tl_min); @@ -196,7 +196,7 @@ void Functions::shoc_assumed_pdf( n_mask++; } } - printf("WARNING: Tl1_2 has %d values <= allowable value. Resetting to minimum value.\n",n_mask); + Kokkos::printf("WARNING: Tl1_2 has %d values <= allowable value. Resetting to minimum value.\n",n_mask); } // Compute qs and beta diff --git a/components/eamxx/src/physics/shoc/impl/shoc_pblintd_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_pblintd_impl.hpp index 34104e9a8fe..9c4cef44d1c 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_pblintd_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_pblintd_impl.hpp @@ -78,7 +78,16 @@ void Functions::pblintd( // Initialize bool check = true; + // The loop below fixes valgrind uninitialized mem errs. As in other + // places in eamxx, we use SCREAM_SHORT_TESTS as a proxy for knowing + // mem checking is on. +#if !defined(NDEBUG) || defined(SCREAM_SHORT_TESTS) + for (size_t i=0; i1 - set (TEST_SUFFIX _omp${SCREAM_TEST_MAX_THREADS}) - endif() - set_tests_properties (shoc_run_and_cmp_f90${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;baseline_cmp") - set_tests_properties (shoc_run_and_cmp_cxx${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;cxx baseline_cmp") +# If small kernels are ON, we don't need a separate executable to test them. +# Also, we never want to generate baselines with this separate executable +if (NOT SCREAM_SHOC_SMALL_KERNELS AND NOT SCREAM_ONLY_GENERATE_BASELINES) + CreateUnitTest(shoc_sk_tests "shoc_main_tests.cpp" + LIBS shoc_sk shoc_test_infra + THREADS ${SHOC_THREADS} + EXE_ARGS "--args ${BASELINE_FILE_ARG}" + LABELS "shoc;physics;baseline_cmp" + ) endif() + +CreateUnitTest(shoc_run_and_cmp "shoc_run_and_cmp.cpp" + LIBS shoc shoc_test_infra + EXCLUDE_MAIN_CPP + THREADS ${SCREAM_TEST_MAX_THREADS} + EXE_ARGS "${BASELINE_FILE_ARG}" + LABELS "shoc;physics;baseline_gen;baseline_cmp") diff --git a/components/eamxx/src/physics/shoc/tests/infra/CMakeLists.txt b/components/eamxx/src/physics/shoc/tests/infra/CMakeLists.txt new file mode 100644 index 00000000000..0c0c75ca6aa --- /dev/null +++ b/components/eamxx/src/physics/shoc/tests/infra/CMakeLists.txt @@ -0,0 +1,10 @@ +set(INFRA_SRCS + shoc_data.cpp + shoc_ic_cases.cpp + shoc_main_wrap.cpp + shoc_test_data.cpp +) + +add_library(shoc_test_infra ${INFRA_SRCS}) +target_link_libraries(shoc_test_infra shoc) +target_include_directories(shoc_test_infra PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/components/eamxx/src/physics/shoc/shoc_f90.cpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_data.cpp similarity index 87% rename from components/eamxx/src/physics/shoc/shoc_f90.cpp rename to components/eamxx/src/physics/shoc/tests/infra/shoc_data.cpp index bc14a9110d5..553a88f121b 100644 --- a/components/eamxx/src/physics/shoc/shoc_f90.cpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_data.cpp @@ -1,4 +1,4 @@ -#include "shoc_f90.hpp" +#include "shoc_data.hpp" #include "physics_constants.hpp" #include "shoc_ic_cases.hpp" @@ -6,11 +6,6 @@ using scream::Real; using scream::Int; -extern "C" { - void shoc_init_c(int nlev, Real gravit, Real rair, Real rh2o, Real cpair, - Real zvir, Real latvap, Real latice, Real karman, Real p0); - void shoc_use_cxx_c(bool use_cxx); -} namespace scream { namespace shoc { @@ -112,19 +107,6 @@ FortranDataIterator::getfield (Int i) const { return fields_[i]; } -void shoc_init(Int nlev, bool use_fortran, bool force_reinit) { - static bool is_init = false; - if (!is_init || force_reinit) { - using Scalar = Real; - using C = scream::physics::Constants; - - shoc_init_c((int)nlev, C::gravit, C::Rair, C::RH2O, C::Cpair, C::ZVIR, - C::LatVap, C::LatIce, C::Karman, C::P0); - is_init = true; - } - shoc_use_cxx_c(!use_fortran); -} - int test_FortranData () { Int shcol = 1; Int nlev = 128, num_tracers = 1; @@ -132,11 +114,5 @@ int test_FortranData () { return 0; } -int test_shoc_init (bool use_fortran) { - Int nz = 160; - shoc_init(nz, use_fortran); - return 0; -} - } // namespace shoc } // namespace scream diff --git a/components/eamxx/src/physics/shoc/shoc_f90.hpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_data.hpp similarity index 85% rename from components/eamxx/src/physics/shoc/shoc_f90.hpp rename to components/eamxx/src/physics/shoc/tests/infra/shoc_data.hpp index 22d6ff70e42..453ddbcda85 100644 --- a/components/eamxx/src/physics/shoc/shoc_f90.hpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_data.hpp @@ -16,9 +16,9 @@ struct FortranData { using KT = KokkosTypes; using Scalar = Real; - using Array1 = typename KT::template lview; - using Array2 = typename KT::template lview; - using Array3 = typename KT::template lview; + using Array1 = typename KT::template view_1d; + using Array2 = typename KT::template view_2d; + using Array3 = typename KT::template view_3d; Int shcol, nlev, nlevi, num_qtracers, nadv; @@ -67,9 +67,6 @@ struct FortranDataIterator { void init(const FortranData::Ptr& d); }; -// Initialize SHOC with the given number of levels. -void shoc_init(Int nlev, bool use_fortran=false, bool force_reinit=false); - // We will likely want to remove these checks in the future, as we're not tied // to the exact implementation or arithmetic in SHOC. For now, these checks are // here to establish that the initial regression-testing code gives results that @@ -77,7 +74,6 @@ void shoc_init(Int nlev, bool use_fortran=false, bool force_reinit=false); Int check_against_python(const FortranData& d); int test_FortranData(); -int test_shoc_init(bool use_fortran); } // namespace shoc } // namespace scream diff --git a/components/eamxx/src/physics/shoc/shoc_ic_cases.cpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_ic_cases.cpp similarity index 97% rename from components/eamxx/src/physics/shoc/shoc_ic_cases.cpp rename to components/eamxx/src/physics/shoc/tests/infra/shoc_ic_cases.cpp index 6af7b1ed7c2..da651ef05ba 100644 --- a/components/eamxx/src/physics/shoc/shoc_ic_cases.cpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_ic_cases.cpp @@ -16,11 +16,10 @@ namespace { // top and then flips everything, so we do the same. //------------------------------------------------------------------------ -using KT = KokkosTypes; using Scalar = Real; -using Array1 = typename KT::template lview; -using Array2 = typename KT::template lview; -using Array3 = typename KT::template lview; +using Array1 = typename FortranData::Array1; +using Array2 = typename FortranData::Array2; +using Array3 = typename FortranData::Array3; // Flip all vertical data in the given array. void flip_vertically(Array2& array) diff --git a/components/eamxx/src/physics/shoc/shoc_ic_cases.hpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_ic_cases.hpp similarity index 93% rename from components/eamxx/src/physics/shoc/shoc_ic_cases.hpp rename to components/eamxx/src/physics/shoc/tests/infra/shoc_ic_cases.hpp index 42c71c2536e..26c06a3cf12 100644 --- a/components/eamxx/src/physics/shoc/shoc_ic_cases.hpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_ic_cases.hpp @@ -1,7 +1,7 @@ #ifndef INCLUDE_SCREAM_SHOC_IC_CASES_HPP #define INCLUDE_SCREAM_SHOC_IC_CASES_HPP -#include "shoc_f90.hpp" +#include "shoc_data.hpp" namespace scream { namespace shoc { diff --git a/components/eamxx/src/physics/shoc/shoc_main_wrap.cpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_main_wrap.cpp similarity index 66% rename from components/eamxx/src/physics/shoc/shoc_main_wrap.cpp rename to components/eamxx/src/physics/shoc/tests/infra/shoc_main_wrap.cpp index 5edfdcd6d28..0266b1e6b4b 100644 --- a/components/eamxx/src/physics/shoc/shoc_main_wrap.cpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_main_wrap.cpp @@ -1,6 +1,6 @@ #include "shoc_main_wrap.hpp" -#include "shoc_f90.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_data.hpp" +#include "shoc_test_data.hpp" #include "physics_constants.hpp" #include "shoc_ic_cases.hpp" @@ -8,72 +8,36 @@ using scream::Real; using scream::Int; -extern "C" { - Int shoc_main_c(int shcol, int nlev, int nlevi, Real dtime, int nadv, - Real* host_dx, Real* host_dy, Real* thv, Real* zt_grid, - Real* zi_grid, Real* pres, Real* presi, Real* pdel, - Real* wthl_sfc, Real* wqw_sfc, Real* uw_sfc, Real* vw_sfc, - Real* wtracer_sfc, int num_qtracers, Real* w_field, - Real* inv_exner, Real* phis, Real* host_dse, Real* tke, - Real* thetal, Real* qw, Real* u_wind, Real* v_wind, - Real* qtracers, Real* wthv_sec, Real* tkh, Real* tk, - Real* shoc_ql, Real* shoc_cldfrac, Real* pblh, - Real* shoc_mix, Real* isotropy, Real* w_sec, Real* thl_sec, - Real* qw_sec, Real* qwthl_sec, Real* wthl_sec, Real* wqw_sec, - Real* wtke_sec, Real* uw_sec, Real* vw_sec, Real* w3, - Real* wqls_sec, Real* brunt, Real* shoc_ql2, Real* elapsed_s); -} namespace scream { namespace shoc { -Int shoc_main(FortranData& d, bool use_fortran) { +Int shoc_main(FortranData& d) { EKAT_REQUIRE_MSG(d.dtime > 0, "Invalid dtime"); EKAT_REQUIRE_MSG(d.nadv > 0, "Invalid nadv"); - if (use_fortran) { - Real elapsed_s; - shoc_main_c((int)d.shcol, (int)d.nlev, (int)d.nlevi, d.dtime, (int)d.nadv, - d.host_dx.data(), d.host_dy.data(), d.thv.data(), - d.zt_grid.data(), d.zi_grid.data(), d.pres.data(), d.presi.data(), - d.pdel.data(), d.wthl_sfc.data(), d.wqw_sfc.data(), d.uw_sfc.data(), - d.vw_sfc.data(), d.wtracer_sfc.data(), (int)d.num_qtracers, - d.w_field.data(), d.inv_exner.data(), d.phis.data(), d.host_dse.data(), - d.tke.data(), d.thetal.data(), d.qw.data(), d.u_wind.data(), - d.v_wind.data(), d.qtracers.data(), d.wthv_sec.data(), d.tkh.data(), - d.tk.data(), d.shoc_ql.data(), d.shoc_cldfrac.data(), d.pblh.data(), - d.shoc_mix.data(), d.isotropy.data(), d.w_sec.data(), - d.thl_sec.data(), d.qw_sec.data(), d.qwthl_sec.data(), - d.wthl_sec.data(), d.wqw_sec.data(), d.wtke_sec.data(), - d.uw_sec.data(), d.vw_sec.data(), d.w3.data(), d.wqls_sec.data(), - d.brunt.data(), d.shoc_ql2.data(), &elapsed_s); - return static_cast(elapsed_s * 1000000); - } else { - const int npbl = d.nlev; - return shoc_main_f((int)d.shcol, (int)d.nlev, (int)d.nlevi, d.dtime, (int)d.nadv, - npbl, d.host_dx.data(), d.host_dy.data(), - d.thv.data(), d.zt_grid.data(), d.zi_grid.data(), d.pres.data(), - d.presi.data(), d.pdel.data(), d.wthl_sfc.data(), - d.wqw_sfc.data(), d.uw_sfc.data(), d.vw_sfc.data(), - d.wtracer_sfc.data(), (int)d.num_qtracers, - d.w_field.data(), d.inv_exner.data(), d.phis.data(), d.host_dse.data(), - d.tke.data(), d.thetal.data(), d.qw.data(), - d.u_wind.data(), d.v_wind.data(), d.qtracers.data(), d.wthv_sec.data(), - d.tkh.data(), d.tk.data(), d.shoc_ql.data(), - d.shoc_cldfrac.data(), d.pblh.data(), d.shoc_mix.data(), d.isotropy.data(), - d.w_sec.data(), d.thl_sec.data(), - d.qw_sec.data(), d.qwthl_sec.data(), d.wthl_sec.data(), d.wqw_sec.data(), - d.wtke_sec.data(), d.uw_sec.data(), - d.vw_sec.data(), d.w3.data(), d.wqls_sec.data(), d.brunt.data(), - d.shoc_ql2.data()); - } + const int npbl = d.nlev; + return shoc_main_host((int)d.shcol, (int)d.nlev, (int)d.nlevi, d.dtime, (int)d.nadv, + npbl, d.host_dx.data(), d.host_dy.data(), + d.thv.data(), d.zt_grid.data(), d.zi_grid.data(), d.pres.data(), + d.presi.data(), d.pdel.data(), d.wthl_sfc.data(), + d.wqw_sfc.data(), d.uw_sfc.data(), d.vw_sfc.data(), + d.wtracer_sfc.data(), (int)d.num_qtracers, + d.w_field.data(), d.inv_exner.data(), d.phis.data(), d.host_dse.data(), + d.tke.data(), d.thetal.data(), d.qw.data(), + d.u_wind.data(), d.v_wind.data(), d.qtracers.data(), d.wthv_sec.data(), + d.tkh.data(), d.tk.data(), d.shoc_ql.data(), + d.shoc_cldfrac.data(), d.pblh.data(), d.shoc_mix.data(), d.isotropy.data(), + d.w_sec.data(), d.thl_sec.data(), + d.qw_sec.data(), d.qwthl_sec.data(), d.wthl_sec.data(), d.wqw_sec.data(), + d.wtke_sec.data(), d.uw_sec.data(), + d.vw_sec.data(), d.w3.data(), d.wqls_sec.data(), d.brunt.data(), + d.shoc_ql2.data()); } namespace { -using KT = KokkosTypes; -using Scalar = Real; -using Array2 = typename KT::template lview; -using Array3 = typename KT::template lview; +using Array2 = typename FortranData::Array2; +using Array3 = typename FortranData::Array3; // Returns a string representation of the given 2D array. std::string array_as_string(const Array2& array) @@ -252,9 +216,8 @@ void gen_plot_script(const std::vector >& data, } // end anonymous namespace -int test_shoc_ic (bool use_fortran, bool gen_plot_scripts) { +int test_shoc_ic (bool gen_plot_scripts) { Int nz = 160; - shoc_init(nz, use_fortran); // Here we: // 1. Initialize a standard case with settings identical to // scream-doc/Ñ•hoc_port/shocintr.py's example_run_case method @@ -272,7 +235,7 @@ int test_shoc_ic (bool use_fortran, bool gen_plot_scripts) { // 3. Run 100 steps, each of size dtime = 10 (as in that method) d->nadv = 100; d->dtime = 10; - shoc_main(*d,use_fortran); + shoc_main(*d); // 4. Generate a Python script that plots the results. { diff --git a/components/eamxx/src/physics/shoc/shoc_main_wrap.hpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_main_wrap.hpp similarity index 80% rename from components/eamxx/src/physics/shoc/shoc_main_wrap.hpp rename to components/eamxx/src/physics/shoc/tests/infra/shoc_main_wrap.hpp index 7ecfa1221a3..f11cf14992d 100644 --- a/components/eamxx/src/physics/shoc/shoc_main_wrap.hpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_main_wrap.hpp @@ -12,13 +12,12 @@ namespace shoc { struct FortranData; // Run SHOC subroutines, populating inout and out fields of d. -ekat::Int shoc_main(FortranData& d, bool use_fortran); - +ekat::Int shoc_main(FortranData& d); // Test SHOC by running initial conditions for a number of steps and comparing // against reference data. If gen_plot_scripts is true, Python scripts are // emitted that plot initial and final conditions. -int test_shoc_ic(bool use_fortran, bool gen_plot_scripts = false); +int test_shoc_ic(bool gen_plot_scripts = false); } // namespace shoc } // namespace scream diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_test_data.cpp similarity index 70% rename from components/eamxx/src/physics/shoc/shoc_functions_f90.cpp rename to components/eamxx/src/physics/shoc/tests/infra/shoc_test_data.cpp index e7468062d3c..2773c7ad220 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_test_data.cpp @@ -1,6 +1,6 @@ -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" -#include "shoc_f90.hpp" +#include "shoc_data.hpp" #include "ekat/ekat_assert.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" @@ -14,834 +14,305 @@ using scream::Real; using scream::Int; -// -// A C interface to SHOC fortran calls. The stubs below will link to fortran definitions in shoc_iso_c.f90 -// - -extern "C" { - -// Special shoc_init function for shoc_main_bfb test -void shoc_init_for_main_bfb_c(int nlev, Real gravit, Real rair, Real rh2o, Real cpair, - Real zvir, Real latvap, Real latice, Real karman, Real p0, - Real* pref_mid, int nbot_shoc, int ntop_shoc); -void shoc_use_cxx_c(bool use_cxx); - - -void shoc_grid_c(int shcol, int nlev, int nlevi, Real *zt_grid, Real *zi_grid, - Real *pdel, Real *dz_zt, Real *dzi_zi, Real *rho_zt); - -void shoc_diag_obklen_c(Int shcol, Real *uw_sfc, Real *vw_sfc, Real *wthl_sfc, - Real *wqw_sfc, Real *thl_sfc, Real *cldliq_sfc, - Real *qv_sfc, Real *ustar, Real *kbfs, Real *obklen); - -void update_host_dse_c(Int shcol, Int nlev, Real *thlm, Real *shoc_ql, - Real *inv_exner, Real *zt_grid, Real *phis, Real *host_dse); - -void shoc_energy_fixer_c(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, - Real *zt_grid, Real *zi_grid, Real *se_b, Real *ke_b, - Real *wv_b, Real *wl_b, Real *se_a, Real *ke_a, - Real *wv_a, Real *wl_a, Real *wthl_sfc, Real *wqw_sfc, - Real *rho_zt, Real *tke, Real *pint, - Real *host_dse); - -void shoc_energy_integrals_c(Int shcol, Int nlev, Real *host_dse, Real *pdel, - Real *rtm, Real *rcm, Real *u_wind, Real *v_wind, - Real *se_int, Real *ke_int, Real *wv_int, Real *wl_int); - -void shoc_energy_total_fixer_c(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, - Real *zt_grid, Real *zi_grid, - Real *se_b, Real *ke_b, Real *wv_b, Real *wl_b, - Real *se_a, Real *ke_a, Real *wv_a, Real *wl_a, - Real *wthl_sfc, Real *wqw_sfc, Real *rho_zt, Real *pint, - Real *te_a, Real *te_b); - -void shoc_energy_threshold_fixer_c(Int shcol, Int nlev, Int nlevi, - Real *pint, Real *tke, Real *te_a, Real *te_b, - Real *se_dis, Int *shoctop); - -void shoc_energy_dse_fixer_c(Int shcol, Int nlev, - Real *se_dis, Int *shoctop, - Real *host_dse); - -void calc_shoc_varorcovar_c(Int shcol, Int nlev, Int nlevi, Real tunefac, - Real *isotropy_zi, Real *tkh_zi, Real *dz_zi, - Real *invar1, Real *invar2, Real *varorcovar); - -void compute_tmpi_c(Int nlevi, Int shcol, Real dtime, Real *rho_zi, - Real *dz_zi, Real *tmpi); - -void dp_inverse_c(Int nlev, Int shcol, Real *rho_zt, Real *dz_zt, Real *rdp_zt); - -void sfc_fluxes_c(Int shcol, Int num_tracer, Real dtime, Real *rho_zi_sfc, - Real *rdp_zt_sfc, Real *wthl_sfc, Real *wqw_sfc, Real *wtracer_sfc, - Real *wtke_sfc, Real *thetal, Real *qw, Real *tke, Real *tracer); - -void impli_srf_stress_term_c(Int shcol, Real *rho_zi_sfc, Real *uw_sfc, - Real *vw_sfc, Real *u_wind_sfc, Real *v_wind_sfc, - Real *ksrf); - -void tke_srf_flux_term_c(Int shcol, Real *uw_sfc, Real *vw_sfc, - Real *wtke_sfc); - -void check_tke_c(Int shcol, Int nlev, Real *tke); - -void shoc_tke_c(Int shcol, Int nlev, Int nlevi, Real dtime, Real *wthv_sec, - Real *shoc_mix, Real *dz_zi, Real *dz_zt, Real *pres, - Real* tabs, Real *u_wind, Real *v_wind, Real *brunt, - Real *zt_grid, Real *zi_grid, Real *pblh, Real *tke, - Real *tk, Real *tkh, Real *isotropy); - -void integ_column_stability_c(Int nlev, Int shcol, Real *dz_zt, Real *pres, - Real *brunt, Real *brunt_int); - -void compute_shr_prod_c(Int nlevi, Int nlev, Int shcol, Real *dz_zi, - Real *u_wind, Real *v_wind, Real *sterm); - -void isotropic_ts_c(Int nlev, Int shcol, Real *brunt_int, Real *tke, - Real *a_diss, Real *brunt, Real *isotropy); - -void adv_sgs_tke_c(Int nlev, Int shcol, Real dtime, Real *shoc_mix, - Real *wthv_sec, Real *sterm_zt, Real *tk, - Real *tke, Real *a_diss); - -void eddy_diffusivities_c(Int nlev, Int shcol, Real *pblh, - Real *zt_grid, Real *tabs, Real *shoc_mix, Real *sterm_zt, - Real *isotropy, Real *tke, Real *tkh, Real *tk); - -void calc_shoc_vertflux_c(Int shcol, Int nlev, Int nlevi, Real *tkh_zi, - Real *dz_zi, Real *invar, Real *vertflux); - -void shoc_length_c(Int shcol, Int nlev, Int nlevi, Real *host_dx, - Real *host_dy, Real *zt_grid, - Real *zi_grid, Real *dz_zt, Real *tke, - Real *thv, Real *brunt, Real *shoc_mix); - -void compute_brunt_shoc_length_c(Int nlev, Int nlevi, Int shcol ,Real *dz_zt, - Real *thv, Real *thv_zi, Real *brunt); - -void compute_l_inf_shoc_length_c(Int nlev, Int shcol, Real *zt_grid, Real *dz_zt, - Real *tke, Real *l_inf); - -void compute_shoc_mix_shoc_length_c(Int nlev, Int shcol, Real *tke, Real* brunt, - Real *zt_grid, Real *l_inf, Real *shoc_mix); - -void check_length_scale_shoc_length_c(Int nlev, Int shcol, Real *host_dx, - Real *host_dy, Real *shoc_mix); - -void clipping_diag_third_shoc_moments_c(Int nlevi, Int shcol, Real *w_sec_zi, - Real *w3); - -void fterms_input_for_diag_third_shoc_moment_c(Real dz_zi, Real dz_zt, Real dz_zt_kc, - Real isotropy_zi, Real brunt_zi, Real thetal_zi, - Real *thedz, Real *thedz2, Real *iso, - Real *isosqrd, Real *buoy_sgs2, Real *bet2); -void f0_to_f5_diag_third_shoc_moment_c(Real thedz, Real thedz2, Real bet2, Real iso, - Real isosqrd, Real wthl_sec, Real wthl_sec_kc, - Real wthl_sec_kb, Real thl_sec_kc, - Real thl_sec_kb, Real w_sec, Real w_sec_kc, Real w_sec_zi, - Real tke, Real tke_kc, Real *f0, Real *f1, - Real *f2, Real *f3, Real *f4, Real *f5); - -void omega_terms_diag_third_shoc_moment_c(Real buoy_sgs2, Real f3, Real f4, - Real *omega0, Real *omega1, Real *omega2); - -void x_y_terms_diag_third_shoc_moment_c(Real buoy_sgs2, Real f0, Real f1, Real f2, - Real *x0, Real *y0, Real *x1, Real *y1); - -void aa_terms_diag_third_shoc_moment_c(Real omega0, Real omega1, Real omega2, - Real x0, Real x1, Real y0, Real y1, - Real *aa0, Real *aa1); - -void w3_diag_third_shoc_moment_c(Real aa0, Real aa1, Real x0, - Real x1, Real f5, Real *w3); -void shoc_diag_second_moments_srf_c(Int shcol, Real* wthl_sfc, Real* uw_sfc, Real* vw_sfc, - Real* ustar2, Real* wstar); - -void diag_third_shoc_moments_c(Int shoc, Int nlev, Int nlevi, Real *w_sec, - Real *thl_sec, - Real *wthl_sec, Real *isotropy, Real *brunt, - Real *thetal, Real *tke, - Real *dz_zt, Real *dz_zi, Real *zt_grid, - Real *zi_grid, Real *w3); - -void compute_diag_third_shoc_moment_c(Int shcol, Int nlev, Int nlevi, Real *w_sec, - Real *thl_sec, Real *wthl_sec, Real *tke, - Real *dz_zt, Real *dz_zi, Real *isotropy_zi, - Real *brunt_zi, Real *w_sec_zi, Real *thetal_zi, - Real *w3); - -void linear_interp_c(Real* x1, Real* x2, Real* y1, Real* y2, Int km1, Int km2, Int ncol, Real minthresh); - -void shoc_assumed_pdf_c(Int shcol, Int nlev, Int nlevi, Real *thetal, Real *qw, - Real *w_first, Real *thl_sec, Real *qw_sec, Real *wthl_sec, - Real *w_sec, Real *wqw_sec, Real *qwthl_sec, Real *w3, - Real *pres, Real *zt_grid, Real *zi_grid, - Real *shoc_cldfrac, Real *shoc_ql, Real *wqls, - Real *wthv_sec, Real *shoc_ql2); - -void shoc_assumed_pdf_tilde_to_real_c(Real w_first, Real sqrtw2, Real* w1); - -void shoc_assumed_pdf_vv_parameters_c(Real w_first, Real w_sec, Real w3var, - Real *Skew_w, Real *w1_1, Real *w1_2, - Real *w2_1, Real *w2_2, Real *a); - -void shoc_assumed_pdf_thl_parameters_c(Real wthlsec, Real sqrtw2, Real sqrtthl, - Real thlsec, Real thl_first, Real w1_1, - Real w1_2, Real Skew_w, Real a, bool dothetal_skew, - Real *thl1_1, Real *thl1_2, Real *thl2_1, - Real *thl2_2, Real *sqrtthl2_1, - Real *sqrtthl2_2); - -void shoc_assumed_pdf_qw_parameters_c(Real wqwsec, Real sqrtw2, Real Skew_w, - Real sqrtqt, Real qw_sec, Real w1_1, - Real w1_2, Real qw_first, Real a, - Real *qw1_1, Real *qw1_2, Real *qw2_1, - Real *qw2_2, Real *sqrtqw2_1, - Real *sqrtqw2_2); - -void shoc_assumed_pdf_inplume_correlations_c(Real sqrtqw2_1, Real sqrtthl2_1, - Real a, Real sqrtqw2_2, Real sqrtthl2_2, - Real qwthlsec, Real qw1_1, Real qw_first, - Real thl1_1, Real thl_first, Real qw1_2, - Real thl1_2, Real *r_qwthl_1); - -void shoc_assumed_pdf_compute_temperature_c(Real thl1, Real basepres, - Real pval, Real *Tl1); - -void shoc_assumed_pdf_compute_qs_c(Real Tl1_1, Real Tl1_2, Real pval, - Real *qs1, Real *beta1, Real *qs2, Real *beta2); - -void shoc_assumed_pdf_compute_s_c(Real qw1, Real qs1, Real beta, Real pval, Real thl2, - Real qw2,Real sqrtthl2, Real sqrtqw2, Real r_qwthl, - Real *s, Real *std_s, Real *qn, Real *C); - -void shoc_assumed_pdf_compute_sgs_liquid_c(Real a, Real ql1, Real ql2, Real *shoc_ql); - -void shoc_assumed_pdf_compute_cloud_liquid_variance_c(Real a, Real s1, Real ql1, - Real C1, Real std_s1, Real s2, Real ql2, Real C2, - Real std_s2, Real shoc_ql, Real *shoc_ql2); - -void shoc_assumed_pdf_compute_liquid_water_flux_c(Real a, Real w1_1, Real w_first, - Real ql1, Real w1_2, Real ql2, Real *wqls); - -void shoc_assumed_pdf_compute_buoyancy_flux_c(Real wthlsec, Real epsterm, Real wqwsec, - Real pval, Real wqls, Real *wthv_sec); - -void shoc_diag_second_moments_ubycond_c(Int shcol, Real* thl_sec, Real* qw_sec, - Real* wthl_sec, Real* wqw_sec, Real* qwthl_sec, - Real* uw_sec, Real* vw_sec, Real* wtke_sec); - -void shoc_pblintd_init_pot_c(Int shcol, Int nlev, Real* thl, Real* ql, Real* q, Real* thv); - -void diag_second_moments_lbycond_c(Int shcol, Real *wthl_sfc, Real *wqw_sfc, Real *uw_sfc, - Real *vw_sfc, Real *ustar2, Real *wstar, Real *wthl_sec, - Real *wqw_sec, Real *uw_sec, Real *vw_sec, Real *wtke_sec, - Real *thl_sec, Real *qw_sec, Real *qwthl_sec); - -void diag_second_moments_c(Int shcol, Int nlev, Int nlevi, Real *thetal, Real *qw, - Real *u_wind, Real *v_wind, Real *tke, Real *isotropy, - Real *tkh, Real *tk, Real *dz_zi, Real *zt_grid, Real *zi_grid, - Real *shoc_mix, Real *thl_sec, Real *qw_sec, Real *wthl_sec, - Real *wqw_sec, Real *qwthl_sec, Real *uw_sec, Real *vw_sec, - Real *wtke_sec, Real *w_sec); - -void diag_second_shoc_moments_c(Int shcol, Int nlev, Int nlevi, Real *thetal, - Real *qw, Real *u_wind, Real *v_wind, Real *tke, - Real *isotropy, Real *tkh, Real *tk, Real *dz_zi, - Real *zt_grid, Real *zi_grid, Real *shoc_mix, - Real *wthl_sfc, Real *wqw_sfc, Real *uw_sfc, - Real *vw_sfc, Real *thl_sec, Real *qw_sec, - Real *wthl_sec, Real *wqw_sec, Real *qwthl_sec, - Real *uw_sec, Real *vw_sec, Real *wtke_sec, Real *w_sec); - -void shoc_pblintd_cldcheck_c(Int shcol, Int nlev, Int nlevi, Real* zi, Real* cldn, Real* pblh); - -void compute_shoc_vapor_c(Int shcol, Int nlev, Real* qw, Real* ql, Real* qv); - -void update_prognostics_implicit_c(Int shcol, Int nlev, Int nlevi, Int num_tracer, Real dtime, - Real* dz_zt, Real* dz_zi, Real* rho_zt, Real* zt_grid, Real* zi_grid, - Real* tk, Real* tkh, Real* uw_sfc, Real* vw_sfc, Real* wthl_sfc, - Real* wqw_sfc, Real* wtracer_sfc, Real* thetal, Real* qw, Real* tracer, - Real* tke, Real* u_wind, Real* v_wind); - -void shoc_main_c(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Real* host_dx, Real* host_dy, - Real* thv, Real* zt_grid, Real* zi_grid, Real* pres, Real* presi, Real* pdel, - Real* wthl_sfc, Real* wqw_sfc, Real* uw_sfc, Real* vw_sfc, Real* wtracer_sfc, - Int num_qtracers, Real* w_field, Real* inv_exner, Real* phis, Real* host_dse, Real* tke, - Real* thetal, Real* qw, Real* u_wind, Real* v_wind, Real* qtracers, Real* wthv_sec, - Real* tkh, Real* tk, Real* shoc_ql, Real* shoc_cldfrac, Real* pblh, Real* shoc_mix, - Real* isotropy, Real* w_sec, Real* thl_sec, Real* qw_sec, Real* qwthl_sec, Real* wthl_sec, - Real* wqw_sec, Real* wtke_sec, Real* uw_sec, Real* vw_sec, Real* w3, Real* wqls_sec, - Real* brunt, Real* shoc_ql2, Real* elapsed_s); - -void pblintd_height_c(Int shcol, Int nlev, Int npbl_in, Real* z, Real* u, Real* v, Real* ustar, Real* thv, Real* thv_ref, Real* pblh, Real* rino, bool* check); - -void vd_shoc_decomp_c(Int shcol, Int nlev, Int nlevi, Real* kv_term, Real* tmpi, Real* rdp_zt, Real dtime, - Real* flux, Real* du, Real* dl, Real* d); - -void vd_shoc_solve_c(Int shcol, Int nlev, Real* du, Real* dl, Real* d, Real* var); -void pblintd_surf_temp_c(Int shcol, Int nlev, Int nlevi, Real* z, Real* ustar, Real* obklen, Real* kbfs, Real* thv, Real* tlv, Real* pblh, bool* check, Real* rino); -void pblintd_check_pblh_c(Int shcol, Int nlev, Int nlevi, Real* z, Real* ustar, bool* check, Real* pblh); -void pblintd_c(Int shcol, Int nlev, Int nlevi, Int npbl_in, Real* z, Real* zi, Real* thl, Real* ql, Real* q, Real* u, Real* v, Real* ustar, Real* obklen, Real* kbfs, Real* cldn, Real* pblh); - -void compute_shoc_temperature_c(Int shcol, Int nlev, Real* thetal, Real*ql, Real* inv_exner, Real* tabs); - -} // extern "C" : end _c decls - namespace scream { namespace shoc { // -// Glue functions to call fortran from from C++ with the Data struct +// Glue functions to call from host with the Data struct // -// In all C++ -> Fortran bridge functions you should see shoc_init(nlev, true). -// We are provisionally following P3 here in case SHOC uses global data. The -// 'true' argument is to set shoc to use its fortran implementations instead of -// calling back to C++. We want this behavior since it doesn't make much sense -// for C++ to bridge over to fortran only to have fortran bridge back to C++. -// Anyone who wants the C++ implementation should call it directly. We need -// need to be aware of data layout since f90 is different from cxx. All these -// functions will expect incoming data to be C layout. They will transpose to f90 -// before calling fortran and then back to C before returning. +// We are provisionally following P3 here in case SHOC uses global data. // void shoc_grid(ShocGridData& d) { - shoc_init(d.nlev, true); - d.transpose(); - shoc_grid_c(d.shcol, d.nlev, d.nlevi, d.zt_grid, d.zi_grid, d.pdel, d.dz_zt, d.dz_zi, d.rho_zt); - d.transpose(); + shoc_grid_host(d.shcol, d.nlev, d.nlevi, d.zt_grid, d.zi_grid, d.pdel, d.dz_zt, d.dz_zi, d.rho_zt); } void shoc_diag_obklen(ShocDiagObklenData& d) { - shoc_init(1, true); // single level function - shoc_diag_obklen_c(d.shcol, d.uw_sfc, d.vw_sfc, d.wthl_sfc, d.wqw_sfc, d.thl_sfc, d.cldliq_sfc, d.qv_sfc, d.ustar, d.kbfs, d.obklen); + shoc_diag_obklen_host(d.shcol, d.uw_sfc, d.vw_sfc, d.wthl_sfc, d.wqw_sfc, d.thl_sfc, d.cldliq_sfc, d.qv_sfc, d.ustar, d.kbfs, d.obklen); } void update_host_dse(UpdateHostDseData& d) { - shoc_init(d.nlev, true); - d.transpose(); - update_host_dse_c(d.shcol, d.nlev, d.thlm, d.shoc_ql, d.inv_exner, d.zt_grid, d.phis, d.host_dse); - d.transpose(); + update_host_dse_host(d.shcol, d.nlev, d.thlm, d.shoc_ql, d.inv_exner, d.zt_grid, d.phis, d.host_dse); } void shoc_energy_fixer(ShocEnergyFixerData& d) { - shoc_init(d.nlev, true); - d.transpose(); - shoc_energy_fixer_c(d.shcol, d.nlev, d.nlevi, d.dtime, d.nadv, d.zt_grid, d.zi_grid, d.se_b, d.ke_b, d.wv_b, d.wl_b, d.se_a, d.ke_a, d.wv_a, d.wl_a, d.wthl_sfc, d.wqw_sfc, d.rho_zt, d.tke, d.pint, d.host_dse); - d.transpose(); + shoc_energy_fixer_host(d.shcol, d.nlev, d.nlevi, d.dtime, d.nadv, d.zt_grid, d.zi_grid, d.se_b, d.ke_b, d.wv_b, d.wl_b, d.se_a, d.ke_a, d.wv_a, d.wl_a, d.wthl_sfc, d.wqw_sfc, d.rho_zt, d.tke, d.pint, d.host_dse); } void shoc_energy_integrals(ShocEnergyIntegralsData& d) { - shoc_init(d.nlev, true); - d.transpose(); - shoc_energy_integrals_c(d.shcol, d.nlev, d.host_dse, d.pdel, d.rtm, d.rcm, d.u_wind, d.v_wind, d.se_int, d.ke_int, d.wv_int, d.wl_int); - d.transpose(); -} - -void shoc_energy_total_fixer(ShocEnergyTotalFixerData& d) -{ - shoc_init(d.nlev, true); - d.transpose(); - shoc_energy_total_fixer_c(d.shcol, d.nlev, d.nlevi, d.dtime, d.nadv, - d.zt_grid, d.zi_grid, d.se_b, d.ke_b, d.wv_b, - d.wl_b, d.se_a, d.ke_a, d.wv_a, d.wl_a, d.wthl_sfc, - d.wqw_sfc, d.rho_zt, d.pint, d.te_a, d.te_b); - d.transpose(); -} - -void shoc_energy_threshold_fixer(ShocEnergyThresholdFixerData& d) -{ - shoc_init(d.nlev, true); - d.transpose(); - shoc_energy_threshold_fixer_c(d.shcol, d.nlev, d.nlevi, d.pint, d.tke, d.te_a, d.te_b, d.se_dis, d.shoctop); - d.transpose(); -} - -void shoc_energy_dse_fixer(ShocEnergyDseFixerData& d) -{ - shoc_init(d.nlev, true); - d.transpose(); - shoc_energy_dse_fixer_c(d.shcol, d.nlev, d.se_dis, d.shoctop, d.host_dse); - d.transpose(); + shoc_energy_integrals_host(d.shcol, d.nlev, d.host_dse, d.pdel, d.rtm, d.rcm, d.u_wind, d.v_wind, d.se_int, d.ke_int, d.wv_int, d.wl_int); } void calc_shoc_vertflux(CalcShocVertfluxData& d) { - shoc_init(d.nlev, true); - d.transpose(); - calc_shoc_vertflux_c(d.shcol, d.nlev, d.nlevi, d.tkh_zi, d.dz_zi, d.invar, d.vertflux); - d.transpose(); + calc_shoc_vertflux_host(d.shcol, d.nlev, d.nlevi, d.tkh_zi, d.dz_zi, d.invar, d.vertflux); } void calc_shoc_varorcovar(CalcShocVarorcovarData& d) { - shoc_init(d.nlev, true); - d.transpose(); - calc_shoc_varorcovar_c(d.shcol, d.nlev, d.nlevi, d.tunefac, d.isotropy_zi, d.tkh_zi, d.dz_zi, d.invar1, d.invar2, d.varorcovar); - d.transpose(); + calc_shoc_varorcovar_host(d.shcol, d.nlev, d.nlevi, d.tunefac, d.isotropy_zi, d.tkh_zi, d.dz_zi, d.invar1, d.invar2, d.varorcovar); } void compute_tmpi(ComputeTmpiData& d) { - shoc_init(d.nlevi - 1, true); // nlev = nlevi - 1 - d.transpose(); - compute_tmpi_c(d.nlevi, d.shcol, d.dtime, d.rho_zi, d.dz_zi, d.tmpi); - d.transpose(); + compute_tmpi_host(d.nlevi, d.shcol, d.dtime, d.rho_zi, d.dz_zi, d.tmpi); } void dp_inverse(DpInverseData& d) { - shoc_init(d.nlev, true); - d.transpose(); - dp_inverse_c(d.nlev, d.shcol, d.rho_zt, d.dz_zt, d.rdp_zt); - d.transpose(); -} - -void sfc_fluxes(SfcFluxesData& d) -{ - shoc_init(1, true); // single layer function - d.transpose(); - sfc_fluxes_c(d.shcol, d.num_tracer, d.dtime, d.rho_zi_sfc, d.rdp_zt_sfc, d.wthl_sfc, d.wqw_sfc, d.wtke_sfc, d.wtracer_sfc, d.thetal, d.qw, d.tke, d.wtracer); - d.transpose(); -} - -void impli_srf_stress_term(ImpliSrfStressTermData& d) -{ - shoc_init(1, true); // single layer function - impli_srf_stress_term_c(d.shcol, d.rho_zi_sfc, d.uw_sfc, d.vw_sfc, d.u_wind_sfc, d.v_wind_sfc, d.ksrf); -} - -void tke_srf_flux_term(TkeSrfFluxTermData& d) -{ - shoc_init(1, true); // single layer function - tke_srf_flux_term_c(d.shcol, d.uw_sfc, d.vw_sfc, d.wtke_sfc); + dp_inverse_host(d.nlev, d.shcol, d.rho_zt, d.dz_zt, d.rdp_zt); } void integ_column_stability(IntegColumnStabilityData& d) { - shoc_init(d.nlev, true); - d.transpose(); - integ_column_stability_c(d.nlev, d.shcol, d.dz_zt, d.pres, d.brunt, d.brunt_int); - d.transpose(); + integ_column_stability_host(d.nlev, d.shcol, d.dz_zt, d.pres, d.brunt, d.brunt_int); } void check_tke(CheckTkeData& d) { - shoc_init(d.nlev, true); - d.transpose(); - check_tke_c(d.shcol, d.nlev, d.tke); - d.transpose(); + check_tke_host(d.shcol, d.nlev, d.tke); } void shoc_tke(ShocTkeData& d) { - shoc_init(d.nlev, true); - d.transpose(); - shoc_tke_c(d.shcol, d.nlev, d.nlevi, d.dtime, d.wthv_sec, d.shoc_mix, d.dz_zi, d.dz_zt, d.pres, d.tabs, d.u_wind, d.v_wind, d.brunt, d.zt_grid, d.zi_grid, d.pblh, d.tke, d.tk, d.tkh, d.isotropy); - d.transpose(); + shoc_tke_host(d.shcol, d.nlev, d.nlevi, d.dtime, d.wthv_sec, d.shoc_mix, d.dz_zi, d.dz_zt, d.pres, d.tabs, d.u_wind, d.v_wind, d.brunt, d.zt_grid, d.zi_grid, d.pblh, d.tke, d.tk, d.tkh, d.isotropy); } void compute_shr_prod(ComputeShrProdData& d) { - shoc_init(d.nlev, true); - d.transpose(); - compute_shr_prod_c(d.nlevi, d.nlev, d.shcol, d.dz_zi, d.u_wind, d.v_wind, d.sterm); - d.transpose(); + compute_shr_prod_host(d.nlevi, d.nlev, d.shcol, d.dz_zi, d.u_wind, d.v_wind, d.sterm); } void isotropic_ts(IsotropicTsData& d) { - shoc_init(d.nlev, true); - d.transpose(); - isotropic_ts_c(d.nlev, d.shcol, d.brunt_int, d.tke, d.a_diss, d.brunt, d.isotropy); - d.transpose(); + isotropic_ts_host(d.nlev, d.shcol, d.brunt_int, d.tke, d.a_diss, d.brunt, d.isotropy); } void adv_sgs_tke(AdvSgsTkeData& d) { - shoc_init(d.nlev, true); - d.transpose(); - adv_sgs_tke_c(d.nlev, d.shcol, d.dtime, d.shoc_mix, d.wthv_sec, d.sterm_zt, d.tk, d.tke, d.a_diss); - d.transpose(); + adv_sgs_tke_host(d.nlev, d.shcol, d.dtime, d.shoc_mix, d.wthv_sec, d.sterm_zt, d.tk, d.tke, d.a_diss); } void eddy_diffusivities(EddyDiffusivitiesData& d) { - shoc_init(d.nlev, true); - d.transpose(); - eddy_diffusivities_c(d.nlev, d.shcol, d.pblh, d.zt_grid, d.tabs, d.shoc_mix, d.sterm_zt, d.isotropy, d.tke, d.tkh, d.tk); - d.transpose(); + eddy_diffusivities_host(d.nlev, d.shcol, d.pblh, d.zt_grid, d.tabs, d.shoc_mix, d.sterm_zt, d.isotropy, d.tke, d.tkh, d.tk); } void shoc_length(ShocLengthData& d) { - shoc_init(d.nlev, true); - d.transpose(); - shoc_length_c(d.shcol, d.nlev, d.nlevi, d.host_dx, d.host_dy, d.zt_grid, d.zi_grid, d.dz_zt, d.tke, d.thv, d.brunt, d.shoc_mix); - d.transpose(); + shoc_length_host(d.shcol, d.nlev, d.nlevi, d.host_dx, d.host_dy, d.zt_grid, d.zi_grid, d.dz_zt, d.tke, d.thv, d.brunt, d.shoc_mix); } void compute_brunt_shoc_length(ComputeBruntShocLengthData& d) { - shoc_init(d.nlev, true); - d.transpose(); - compute_brunt_shoc_length_c(d.nlev, d.nlevi, d.shcol, d.dz_zt, d.thv, d.thv_zi, d.brunt); - d.transpose(); + compute_brunt_shoc_length_host(d.nlev, d.nlevi, d.shcol, d.dz_zt, d.thv, d.thv_zi, d.brunt); } void compute_l_inf_shoc_length(ComputeLInfShocLengthData& d) { - shoc_init(d.nlev, true); - d.transpose(); - compute_l_inf_shoc_length_c(d.nlev, d.shcol, d.zt_grid, d.dz_zt, d.tke, d.l_inf); - d.transpose(); + compute_l_inf_shoc_length_host(d.nlev, d.shcol, d.zt_grid, d.dz_zt, d.tke, d.l_inf); } void compute_shoc_mix_shoc_length(ComputeShocMixShocLengthData& d) { - shoc_init(d.nlev, true); - d.transpose(); - compute_shoc_mix_shoc_length_c(d.nlev, d.shcol, d.tke, d.brunt, d.zt_grid, d.l_inf, d.shoc_mix); - d.transpose(); + compute_shoc_mix_shoc_length_host(d.nlev, d.shcol, d.tke, d.brunt, d.zt_grid, d.l_inf, d.shoc_mix); } void check_length_scale_shoc_length(CheckLengthScaleShocLengthData& d) { - shoc_init(d.nlev, true); - d.transpose(); - check_length_scale_shoc_length_c(d.nlev, d.shcol, d.host_dx, d.host_dy, d.shoc_mix); - d.transpose(); -} - -void fterms_input_for_diag_third_shoc_moment(FtermsInputForDiagThirdShocMomentData& d) -{ - shoc_init(1, true); // single level function - fterms_input_for_diag_third_shoc_moment_c(d.dz_zi, d.dz_zt, d.dz_zt_kc, d.isotropy_zi, d.brunt_zi, d.thetal_zi, &d.thedz, &d.thedz2, &d.iso, &d.isosqrd, &d.buoy_sgs2, &d.bet2); -} - -void aa_terms_diag_third_shoc_moment(AaTermsDiagThirdShocMomentData& d) -{ - shoc_init(1, true); // single level function - aa_terms_diag_third_shoc_moment_c(d.omega0, d.omega1, d.omega2, d.x0, d.x1, d.y0, d.y1, &d.aa0, &d.aa1); -} - -void f0_to_f5_diag_third_shoc_moment(F0ToF5DiagThirdShocMomentData& d) -{ - shoc_init(1, true); // single level function - f0_to_f5_diag_third_shoc_moment_c(d.thedz, d.thedz2, d.bet2, d.iso, d.isosqrd, d.wthl_sec, d.wthl_sec_kc, d.wthl_sec_kb, d.thl_sec_kc, d.thl_sec_kb, d.w_sec, d.w_sec_kc, d.w_sec_zi, d.tke, d.tke_kc, &d.f0, &d.f1, &d.f2, &d.f3, &d.f4, &d.f5); -} - -void omega_terms_diag_third_shoc_moment(OmegaTermsDiagThirdShocMomentData& d) -{ - shoc_init(1, true); // single level function - omega_terms_diag_third_shoc_moment_c(d.buoy_sgs2, d.f3, d.f4, &d.omega0, &d.omega1, &d.omega2); -} - -void x_y_terms_diag_third_shoc_moment(XYTermsDiagThirdShocMomentData& d) -{ - shoc_init(1, true); // single level function - x_y_terms_diag_third_shoc_moment_c(d.buoy_sgs2, d.f0, d.f1, d.f2, &d.x0, &d.y0, &d.x1, &d.y1); -} - -void w3_diag_third_shoc_moment(W3DiagThirdShocMomentData& d) -{ - shoc_init(1, true); // single level function - w3_diag_third_shoc_moment_c(d.aa0, d.aa1, d.x0, d.x1, d.f5, &d.w3); + check_length_scale_shoc_length_host(d.nlev, d.shcol, d.host_dx, d.host_dy, d.shoc_mix); } void clipping_diag_third_shoc_moments(ClippingDiagThirdShocMomentsData& d) { - shoc_init(d.nlevi - 1, true); // nlev = nlevi - 1 - d.transpose(); - clipping_diag_third_shoc_moments_c(d.nlevi, d.shcol, d.w_sec_zi, d.w3); - d.transpose(); + clipping_diag_third_shoc_moments_host(d.nlevi, d.shcol, d.w_sec_zi, d.w3); } void diag_second_moments_srf(DiagSecondMomentsSrfData& d) { - shoc_init(1, true); // single level function - shoc_diag_second_moments_srf_c(d.shcol, d.wthl_sfc, d.uw_sfc, d.vw_sfc, d.ustar2, d.wstar); + shoc_diag_second_moments_srf_host(d.shcol, d.wthl_sfc, d.uw_sfc, d.vw_sfc, d.ustar2, d.wstar); } void linear_interp(LinearInterpData& d) { - shoc_init(d.km1, true); - d.transpose(); - linear_interp_c(d.x1, d.x2, d.y1, d.y2, d.km1, d.km2, d.ncol, d.minthresh); - d.transpose(); + linear_interp_host(d.x1, d.x2, d.y1, d.y2, d.km1, d.km2, d.ncol, d.minthresh); } void diag_third_shoc_moments(DiagThirdShocMomentsData& d) { - shoc_init(d.nlev, true); - d.transpose(); - diag_third_shoc_moments_c(d.shcol, d.nlev, d.nlevi, d.w_sec, d.thl_sec, d.wthl_sec, d.isotropy, d.brunt, d.thetal, d.tke, d.dz_zt, d.dz_zi, d.zt_grid, d.zi_grid, d.w3); - d.transpose(); + diag_third_shoc_moments_host(d.shcol, d.nlev, d.nlevi, d.w_sec, d.thl_sec, d.wthl_sec, d.isotropy, d.brunt, d.thetal, d.tke, d.dz_zt, d.dz_zi, d.zt_grid, d.zi_grid, d.w3); } void compute_diag_third_shoc_moment(ComputeDiagThirdShocMomentData& d) { - shoc_init(d.nlev, true); - d.transpose(); - compute_diag_third_shoc_moment_c(d.shcol, d.nlev, d.nlevi, d.w_sec, d.thl_sec, d.wthl_sec, d.tke, d.dz_zt, d.dz_zi, d.isotropy_zi, d.brunt_zi, d.w_sec_zi, d.thetal_zi, d.w3); - d.transpose(); + compute_diag_third_shoc_moment_host(d.shcol, d.nlev, d.nlevi, d.w_sec, d.thl_sec, d.wthl_sec, d.tke, d.dz_zt, d.dz_zi, d.isotropy_zi, d.brunt_zi, d.w_sec_zi, d.thetal_zi, d.w3); } void shoc_assumed_pdf(ShocAssumedPdfData& d) { - shoc_init(d.nlev, true); - d.transpose(); - shoc_assumed_pdf_c(d.shcol, d.nlev, d.nlevi, d.thetal, d.qw, d.w_field, d.thl_sec, d.qw_sec, d.wthl_sec, d.w_sec, d.wqw_sec, d.qwthl_sec, d.w3, d.pres, d.zt_grid, d.zi_grid, d.shoc_cldfrac, d.shoc_ql, d.wqls, d.wthv_sec, d.shoc_ql2); - d.transpose(); + shoc_assumed_pdf_host(d.shcol, d.nlev, d.nlevi, d.thetal, d.qw, d.w_field, d.thl_sec, d.qw_sec, d.wthl_sec, d.w_sec, d.wqw_sec, d.qwthl_sec, d.w3, d.pres, d.zt_grid, d.zi_grid, d.shoc_cldfrac, d.shoc_ql, d.wqls, d.wthv_sec, d.shoc_ql2); } void shoc_assumed_pdf_tilde_to_real(ShocAssumedPdfTildeToRealData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_tilde_to_real_c(d.w_first, d.sqrtw2, &d.w1); + shoc_assumed_pdf_tilde_to_real_host(d.w_first, d.sqrtw2, &d.w1); } void shoc_assumed_pdf_vv_parameters(ShocAssumedPdfVvParametersData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_vv_parameters_c(d.w_first, d.w_sec, d.w3var, &d.skew_w, &d.w1_1, &d.w1_2, &d.w2_1, &d.w2_2, &d.a); + shoc_assumed_pdf_vv_parameters_host(d.w_first, d.w_sec, d.w3var, d.w_tol_sqd, &d.skew_w, &d.w1_1, &d.w1_2, &d.w2_1, &d.w2_2, &d.a); } void shoc_assumed_pdf_thl_parameters(ShocAssumedPdfThlParametersData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_thl_parameters_c(d.wthlsec, d.sqrtw2, d.sqrtthl, d.thlsec, d.thl_first, d.w1_1, d.w1_2, d.skew_w, d.a, d.dothetal_skew, &d.thl1_1, &d.thl1_2, &d.thl2_1, &d.thl2_2, &d.sqrtthl2_1, &d.sqrtthl2_2); + shoc_assumed_pdf_thl_parameters_host(d.wthlsec, d.sqrtw2, d.sqrtthl, d.thlsec, d.thl_first, d.w1_1, d.w1_2, d.skew_w, d.a, d.thl_tol, d.w_thresh, &d.thl1_1, &d.thl1_2, &d.thl2_1, &d.thl2_2, &d.sqrtthl2_1, &d.sqrtthl2_2); } void shoc_assumed_pdf_qw_parameters(ShocAssumedPdfQwParametersData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_qw_parameters_c(d.wqwsec, d.sqrtw2, d.skew_w, d.sqrtqt, d.qwsec, d.w1_2, d.w1_1, d.qw_first, d.a, &d.qw1_1, &d.qw1_2, &d.qw2_1, &d.qw2_2, &d.sqrtqw2_1, &d.sqrtqw2_2); + shoc_assumed_pdf_qw_parameters_host(d.wqwsec, d.sqrtw2, d.skew_w, d.sqrtqt, d.qwsec, d.w1_2, d.w1_1, d.qw_first, d.a, d.rt_tol, d.w_thresh, &d.qw1_1, &d.qw1_2, &d.qw2_1, &d.qw2_2, &d.sqrtqw2_1, &d.sqrtqw2_2); } void shoc_assumed_pdf_inplume_correlations(ShocAssumedPdfInplumeCorrelationsData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_inplume_correlations_c(d.sqrtqw2_1, d.sqrtthl2_1, d.a, d.sqrtqw2_2, d.sqrtthl2_2, d.qwthlsec, d.qw1_1, d.qw_first, d.thl1_1, d.thl_first, d.qw1_2, d.thl1_2, &d.r_qwthl_1); + shoc_assumed_pdf_inplume_correlations_host(d.sqrtqw2_1, d.sqrtthl2_1, d.a, d.sqrtqw2_2, d.sqrtthl2_2, d.qwthlsec, d.qw1_1, d.qw_first, d.thl1_1, d.thl_first, d.qw1_2, d.thl1_2, &d.r_qwthl_1); } void shoc_assumed_pdf_compute_temperature(ShocAssumedPdfComputeTemperatureData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_compute_temperature_c(d.thl1, d.basepres, d.pval, &d.tl1); + shoc_assumed_pdf_compute_temperature_host(d.thl1, d.pval, &d.tl1); } void shoc_assumed_pdf_compute_qs(ShocAssumedPdfComputeQsData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_compute_qs_c(d.tl1_1, d.tl1_2, d.pval, &d.qs1, &d.beta1, &d.qs2, &d.beta2); + shoc_assumed_pdf_compute_qs_host(d.tl1_1, d.tl1_2, d.pval, &d.qs1, &d.beta1, &d.qs2, &d.beta2); } void shoc_assumed_pdf_compute_s(ShocAssumedPdfComputeSData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_compute_s_c(d.qw1, d.qs1, d.beta, d.pval, d.thl2, d.qw2, d.sqrtthl2, d.sqrtqw2, d.r_qwthl, &d.s, &d.std_s, &d.qn, &d.c); + shoc_assumed_pdf_compute_s_host(d.qw1, d.qs1, d.beta, d.pval, d.thl2, d.qw2, d.sqrtthl2, d.sqrtqw2, d.r_qwthl, &d.s, &d.std_s, &d.qn, &d.c); } void shoc_assumed_pdf_compute_sgs_liquid(ShocAssumedPdfComputeSgsLiquidData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_compute_sgs_liquid_c(d.a, d.ql1, d.ql2, &d.shoc_ql); + shoc_assumed_pdf_compute_sgs_liquid_host(d.a, d.ql1, d.ql2, &d.shoc_ql); } void shoc_assumed_pdf_compute_cloud_liquid_variance(ShocAssumedPdfComputeCloudLiquidVarianceData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_compute_cloud_liquid_variance_c(d.a, d.s1, d.ql1, d.c1, d.std_s1, d.s2, d.ql2, d.c2, d.std_s2, d.shoc_ql, &d.shoc_ql2); + shoc_assumed_pdf_compute_cloud_liquid_variance_host(d.a, d.s1, d.ql1, d.c1, d.std_s1, d.s2, d.ql2, d.c2, d.std_s2, d.shoc_ql, &d.shoc_ql2); } void shoc_assumed_pdf_compute_liquid_water_flux(ShocAssumedPdfComputeLiquidWaterFluxData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_compute_liquid_water_flux_c(d.a, d.w1_1, d.w_first, d.ql1, d.w1_2, d.ql2, &d.wqls); + shoc_assumed_pdf_compute_liquid_water_flux_host(d.a, d.w1_1, d.w_first, d.ql1, d.w1_2, d.ql2, &d.wqls); } void shoc_assumed_pdf_compute_buoyancy_flux(ShocAssumedPdfComputeBuoyancyFluxData& d) { - shoc_init(1, true); // single level function - shoc_assumed_pdf_compute_buoyancy_flux_c(d.wthlsec, d.epsterm, d.wqwsec, d.pval, d.wqls, &d.wthv_sec); + shoc_assumed_pdf_compute_buoyancy_flux_host(d.wthlsec, d.wqwsec, d.pval, d.wqls, &d.wthv_sec); } void diag_second_moments_ubycond(DiagSecondMomentsUbycondData& d) { - shoc_init(1, true); // single level function - shoc_diag_second_moments_ubycond_c(d.shcol, d.thl_sec, d.qw_sec, d.wthl_sec, d.wqw_sec, d.qwthl_sec, d.uw_sec, d.vw_sec, d.wtke_sec); + shoc_diag_second_moments_ubycond_host(d.shcol, d.thl_sec, d.qw_sec, d.wthl_sec, d.wqw_sec, d.qwthl_sec, d.uw_sec, d.vw_sec, d.wtke_sec); } void pblintd_init_pot(PblintdInitPotData& d) { - shoc_init(d.nlev, true, true); - d.transpose(); - shoc_pblintd_init_pot_c(d.shcol, d.nlev, d.thl, d.ql, d.q, d.thv); - d.transpose(); + shoc_pblintd_init_pot_host(d.shcol, d.nlev, d.thl, d.ql, d.q, d.thv); } void pblintd_cldcheck(PblintdCldcheckData& d) { - shoc_init(d.nlev, true, true); - d.transpose(); - shoc_pblintd_cldcheck_c(d.shcol, d.nlev, d.nlevi, d.zi, d.cldn, d.pblh); - d.transpose(); + shoc_pblintd_cldcheck_host(d.shcol, d.nlev, d.nlevi, d.zi, d.cldn, d.pblh); } void diag_second_moments_lbycond(DiagSecondMomentsLbycondData& d) { - shoc_init(1, true); // single level function - diag_second_moments_lbycond_c(d.shcol, d.wthl_sfc, d.wqw_sfc, d.uw_sfc, d.vw_sfc, d.ustar2, d.wstar, d.wthl_sec, d.wqw_sec, d.uw_sec, d.vw_sec, d.wtke_sec, d.thl_sec, d.qw_sec, d.qwthl_sec); + diag_second_moments_lbycond_host(d.shcol, d.wthl_sfc, d.wqw_sfc, d.uw_sfc, d.vw_sfc, d.ustar2, d.wstar, d.wthl_sec, d.wqw_sec, d.uw_sec, d.vw_sec, d.wtke_sec, d.thl_sec, d.qw_sec, d.qwthl_sec); } void diag_second_moments(DiagSecondMomentsData& d) { - shoc_init(d.nlev, true); - d.transpose(); - diag_second_moments_c(d.shcol, d.nlev, d.nlevi, d.thetal, d.qw, d.u_wind, d.v_wind, d.tke, d.isotropy, d.tkh, d.tk, - d.dz_zi, d.zt_grid, d.zi_grid, d.shoc_mix, d.thl_sec, d.qw_sec, d.wthl_sec, d.wqw_sec, d.qwthl_sec, d.uw_sec, - d.vw_sec, d.wtke_sec, d.w_sec); - d.transpose(); + diag_second_moments_host(d.shcol, d.nlev, d.nlevi, d.thetal, d.qw, d.u_wind, d.v_wind, d.tke, d.isotropy, d.tkh, d.tk, + d.dz_zi, d.zt_grid, d.zi_grid, d.shoc_mix, d.thl_sec, d.qw_sec, d.wthl_sec, d.wqw_sec, d.qwthl_sec, d.uw_sec, + d.vw_sec, d.wtke_sec, d.w_sec); } void diag_second_shoc_moments(DiagSecondShocMomentsData& d) { - shoc_init(d.nlev, true); - d.transpose(); - diag_second_shoc_moments_c(d.shcol, d.nlev, d.nlevi, d.thetal, d.qw, d.u_wind, d.v_wind, d.tke, d.isotropy, d.tkh, - d.tk, d.dz_zi, d.zt_grid, d.zi_grid, d.shoc_mix, d.wthl_sfc, d.wqw_sfc, d.uw_sfc, d.vw_sfc, d.thl_sec, d.qw_sec, - d.wthl_sec, d.wqw_sec, d.qwthl_sec, d.uw_sec, d.vw_sec, d.wtke_sec, d.w_sec); - d.transpose(); + diag_second_shoc_moments_host(d.shcol, d.nlev, d.nlevi, d.thetal, d.qw, d.u_wind, d.v_wind, d.tke, d.isotropy, d.tkh, + d.tk, d.dz_zi, d.zt_grid, d.zi_grid, d.shoc_mix, d.wthl_sfc, d.wqw_sfc, d.uw_sfc, d.vw_sfc, d.thl_sec, d.qw_sec, + d.wthl_sec, d.wqw_sec, d.qwthl_sec, d.uw_sec, d.vw_sec, d.wtke_sec, d.w_sec); } void compute_shoc_vapor(ComputeShocVaporData& d) { - shoc_init(d.nlev, true); - d.transpose(); - compute_shoc_vapor_c(d.shcol, d.nlev, d.qw, d.ql, d.qv); - d.transpose(); + compute_shoc_vapor_host(d.shcol, d.nlev, d.qw, d.ql, d.qv); } void update_prognostics_implicit(UpdatePrognosticsImplicitData& d) { - shoc_init(d.nlev, true); - d.transpose(); - update_prognostics_implicit_c(d.shcol, d.nlev, d.nlevi, d.num_tracer, d.dtime, - d.dz_zt, d.dz_zi, d.rho_zt, d.zt_grid, d.zi_grid, - d.tk, d.tkh, d.uw_sfc, d.vw_sfc, d.wthl_sfc, d.wqw_sfc, - d.wtracer_sfc, d.thetal, d.qw, d.tracer, d.tke, d.u_wind, d.v_wind); - d.transpose(); + update_prognostics_implicit_host(d.shcol, d.nlev, d.nlevi, d.num_tracer, d.dtime, + d.dz_zt, d.dz_zi, d.rho_zt, d.zt_grid, d.zi_grid, + d.tk, d.tkh, d.uw_sfc, d.vw_sfc, d.wthl_sfc, d.wqw_sfc, + d.wtracer_sfc, d.thetal, d.qw, d.tracer, d.tke, d.u_wind, d.v_wind); } void shoc_main(ShocMainData& d) { - shoc_init(d.nlev, true, true); - d.transpose(); - shoc_main_c(d.shcol, d.nlev, d.nlevi, d.dtime, d.nadv, d.host_dx, d.host_dy, d.thv, d.zt_grid, d.zi_grid, + const int npbl = shoc_init_host(d.nlev, d.pref_mid, d.nbot_shoc, d.ntop_shoc); + d.elapsed_s = shoc_main_host(d.shcol, d.nlev, d.nlevi, d.dtime, d.nadv, npbl, d.host_dx, d.host_dy, d.thv, d.zt_grid, d.zi_grid, d.pres, d.presi, d.pdel, d.wthl_sfc, d.wqw_sfc, d.uw_sfc, d.vw_sfc, d.wtracer_sfc, d.num_qtracers, d.w_field, d.inv_exner, d.phis, d.host_dse, d.tke, d.thetal, d.qw, d.u_wind, d.v_wind, d.qtracers, d.wthv_sec, d.tkh, d.tk, d.shoc_ql, d.shoc_cldfrac, d.pblh, d.shoc_mix, d.isotropy, d.w_sec, d.thl_sec, d.qw_sec, d.qwthl_sec, d.wthl_sec, d.wqw_sec, - d.wtke_sec, d.uw_sec, d.vw_sec, d.w3, d.wqls_sec, d.brunt, d.shoc_ql2, &d.elapsed_s); - d.transpose(); -} - -void shoc_main_with_init(ShocMainData& d) -{ - using C = scream::physics::Constants; - - d.transpose(); - shoc_init_for_main_bfb_c(d.nlev, C::gravit, C::Rair, C::RH2O, C::Cpair, C::ZVIR, C::LatVap, C::LatIce, C::Karman, C::P0, - d.pref_mid, d.nbot_shoc, d.ntop_shoc+1); - shoc_use_cxx_c(false); - - - shoc_main_c(d.shcol, d.nlev, d.nlevi, d.dtime, d.nadv, d.host_dx, d.host_dy, d.thv, d.zt_grid, d.zi_grid, - d.pres, d.presi, d.pdel, d.wthl_sfc, d.wqw_sfc, d.uw_sfc, d.vw_sfc, d.wtracer_sfc, d.num_qtracers, - d.w_field, d.inv_exner, d.phis, d.host_dse, d.tke, d.thetal, d.qw, d.u_wind, d.v_wind, d.qtracers, - d.wthv_sec, d.tkh, d.tk, d.shoc_ql, d.shoc_cldfrac, d.pblh, d.shoc_mix, d.isotropy, d.w_sec, - d.thl_sec, d.qw_sec, d.qwthl_sec, d.wthl_sec, d.wqw_sec, d.wtke_sec, d.uw_sec, d.vw_sec, d.w3, - d.wqls_sec, d.brunt, d.shoc_ql2, &d.elapsed_s); - d.transpose(); + d.wtke_sec, d.uw_sec, d.vw_sec, d.w3, d.wqls_sec, d.brunt, d.shoc_ql2); } void pblintd_height(PblintdHeightData& d) { - shoc_init(d.nlev, true, true); - d.transpose(); - pblintd_height_c(d.shcol, d.nlev, d.npbl, d.z, d.u, d.v, d.ustar, d.thv, d.thv_ref, d.pblh, d.rino, d.check); - d.transpose(); + pblintd_height_host(d.shcol, d.nlev, d.npbl, d.z, d.u, d.v, d.ustar, d.thv, d.thv_ref, d.pblh, d.rino, d.check); } void vd_shoc_decomp_and_solve(VdShocDecompandSolveData& d) { - shoc_init(d.nlev, true); - d.transpose(); - // Call decomp subroutine - vd_shoc_decomp_c(d.shcol, d.nlev, d.nlevi, d.kv_term, d.tmpi, d.rdp_zt, d.dtime, d.flux, d.du, d.dl, d.d); - // Call solver for each problem. The `var` array represents 3d - // data with an entry per (shcol, nlev, n_rhs). Fortran requires - // 2d data (shcol, nlev) for each rhs. - const Int size = d.shcol*d.nlev; - for (Int n=0; n(); + vd_shoc_decomp_and_solve_host(d.shcol, d.nlev, d.nlevi, d.n_rhs, d.dtime, d.kv_term, d.tmpi, d.rdp_zt, d.flux, d.var); } void pblintd_surf_temp(PblintdSurfTempData& d) { - shoc_init(d.nlev, true, true); - d.transpose(); - pblintd_surf_temp_c(d.shcol, d.nlev, d.nlevi, d.z, d.ustar, d.obklen, d.kbfs, d.thv, d.tlv, d.pblh, d.check, d.rino); - d.transpose(); + pblintd_surf_temp_host(d.shcol, d.nlev, d.nlevi, d.z, d.ustar, d.obklen, d.kbfs, d.thv, d.tlv, d.pblh, d.check, d.rino); } void pblintd_check_pblh(PblintdCheckPblhData& d) { - shoc_init(d.nlev, true, true); - d.transpose(); - pblintd_check_pblh_c(d.shcol, d.nlev, d.nlevi, d.z, d.ustar, d.check, d.pblh); - d.transpose(); + pblintd_check_pblh_host(d.shcol, d.nlev, d.nlevi, d.nlev/*npbl*/, d.z, d.ustar, d.check, d.pblh); } void pblintd(PblintdData& d) { - shoc_init(d.nlev, true, true); - d.transpose(); - pblintd_c(d.shcol, d.nlev, d.nlevi, d.npbl, d.z, d.zi, d.thl, d.ql, d.q, d.u, d.v, d.ustar, d.obklen, d.kbfs, d.cldn, d.pblh); - d.transpose(); + pblintd_host(d.shcol, d.nlev, d.nlevi, d.npbl, d.z, d.zi, d.thl, d.ql, d.q, d.u, d.v, d.ustar, d.obklen, d.kbfs, d.cldn, d.pblh); } void compute_shoc_temperature(ComputeShocTempData& d) { - shoc_init(d.nlev, true, true); - d.transpose(); - compute_shoc_temperature_c(d.shcol, d.nlev, d.thetal, d.ql, d.inv_exner, d.tabs); - d.transpose(); + compute_shoc_temperature_host(d.shcol, d.nlev, d.thetal, d.ql, d.inv_exner, d.tabs); } // end _c impls // -// _f function definitions. These expect data in C layout +// _host function definitions. These expect data in C layout // -void calc_shoc_varorcovar_f(Int shcol, Int nlev, Int nlevi, Real tunefac, +void calc_shoc_varorcovar_host(Int shcol, Int nlev, Int nlevi, Real tunefac, Real *isotropy_zi, Real *tkh_zi, Real *dz_zi, Real *invar1, Real *invar2, Real *varorcovar) { @@ -861,7 +332,7 @@ void calc_shoc_varorcovar_f(Int shcol, Int nlev, Int nlevi, Real tunefac, std::vector ptr_array = {isotropy_zi, tkh_zi, dz_zi, invar1, invar2, varorcovar}; // Sync to device - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d); view_2d isotropy_zi_d (temp_d[0]), @@ -889,10 +360,10 @@ void calc_shoc_varorcovar_f(Int shcol, Int nlev, Int nlevi, Real tunefac, // Sync back to host std::vector inout_views = {varorcovar_d}; - ekat::device_to_host({varorcovar}, shcol, nlevi, inout_views, true); + ekat::device_to_host({varorcovar}, shcol, nlevi, inout_views); } -void calc_shoc_vertflux_f(Int shcol, Int nlev, Int nlevi, Real *tkh_zi, +void calc_shoc_vertflux_host(Int shcol, Int nlev, Int nlevi, Real *tkh_zi, Real *dz_zi, Real *invar, Real *vertflux) { using SHF = Functions; @@ -911,7 +382,7 @@ void calc_shoc_vertflux_f(Int shcol, Int nlev, Int nlevi, Real *tkh_zi, std::vector ptr_array = {tkh_zi, dz_zi, invar, vertflux}; // Sync to device - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d); view_2d tkh_zi_d (temp_d[0]), @@ -934,10 +405,10 @@ void calc_shoc_vertflux_f(Int shcol, Int nlev, Int nlevi, Real *tkh_zi, // Sync back to host std::vector inout_views = {vertflux_d}; - ekat::device_to_host({vertflux}, shcol, nlevi, inout_views, true); + ekat::device_to_host({vertflux}, shcol, nlevi, inout_views); } -void shoc_diag_second_moments_srf_f(Int shcol, Real* wthl_sfc, Real* uw_sfc, Real* vw_sfc, Real* ustar2, Real* wstar) +void shoc_diag_second_moments_srf_host(Int shcol, Real* wthl_sfc, Real* uw_sfc, Real* vw_sfc, Real* ustar2, Real* wstar) { using SHOC = Functions; using Scalar = typename SHOC::Scalar; @@ -975,7 +446,7 @@ void shoc_diag_second_moments_srf_f(Int shcol, Real* wthl_sfc, Real* uw_sfc, Rea ScreamDeepCopy::copy_to_host({ustar2, wstar}, shcol, inout_views); } -void shoc_diag_second_moments_ubycond_f(Int shcol, Real* thl_sec, Real* qw_sec, Real* wthl_sec, Real* wqw_sec, Real* qwthl_sec, Real* uw_sec, Real* vw_sec, +void shoc_diag_second_moments_ubycond_host(Int shcol, Real* thl_sec, Real* qw_sec, Real* wthl_sec, Real* wqw_sec, Real* qwthl_sec, Real* uw_sec, Real* vw_sec, Real* wtke_sec) { using SHOC = Functions; @@ -1020,7 +491,7 @@ void shoc_diag_second_moments_ubycond_f(Int shcol, Real* thl_sec, Real* qw_sec, ScreamDeepCopy::copy_to_host({thl_sec, qw_sec, qwthl_sec, wthl_sec, wqw_sec, uw_sec, vw_sec, wtke_sec}, shcol, host_views); } -void update_host_dse_f(Int shcol, Int nlev, Real* thlm, Real* shoc_ql, Real* inv_exner, Real* zt_grid, +void update_host_dse_host(Int shcol, Int nlev, Real* thlm, Real* shoc_ql, Real* inv_exner, Real* zt_grid, Real* phis, Real* host_dse) { using SHF = Functions; @@ -1039,7 +510,7 @@ void update_host_dse_f(Int shcol, Int nlev, Real* thlm, Real* shoc_ql, Real* inv // Sync to device ScreamDeepCopy::copy_to_device({phis}, shcol, temp_1d_d); - ekat::host_to_device(ptr_array, shcol, nlev, temp_2d_d, true); + ekat::host_to_device(ptr_array, shcol, nlev, temp_2d_d); view_1d phis_d(temp_1d_d[0]); @@ -1067,10 +538,10 @@ void update_host_dse_f(Int shcol, Int nlev, Real* thlm, Real* shoc_ql, Real* inv // Sync back to host std::vector inout_views = {host_dse_d}; - ekat::device_to_host({host_dse}, shcol, nlev, inout_views, true); + ekat::device_to_host({host_dse}, shcol, nlev, inout_views); } -void compute_diag_third_shoc_moment_f(Int shcol, Int nlev, Int nlevi, Real* w_sec, +void compute_diag_third_shoc_moment_host(Int shcol, Int nlev, Int nlevi, Real* w_sec, Real* thl_sec, Real* wthl_sec, Real* tke, Real* dz_zt, Real* dz_zi, Real* isotropy_zi, Real* brunt_zi, Real* w_sec_zi, Real* thetal_zi, @@ -1100,7 +571,7 @@ void compute_diag_third_shoc_moment_f(Int shcol, Int nlev, Int nlevi, Real* w_se w3}; // Sync to device - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d); view_2d w_sec_d (temp_d[0]), @@ -1141,10 +612,10 @@ void compute_diag_third_shoc_moment_f(Int shcol, Int nlev, Int nlevi, Real* w_se // Sync back to host std::vector inout_views = {w3_d}; - ekat::device_to_host({w3}, shcol, nlevi, inout_views, true); + ekat::device_to_host({w3}, shcol, nlevi, inout_views); } -void shoc_pblintd_init_pot_f(Int shcol, Int nlev, Real *thl, Real* ql, Real* q, +void shoc_pblintd_init_pot_host(Int shcol, Int nlev, Real *thl, Real* ql, Real* q, Real *thv) { using SHOC = Functions; @@ -1157,7 +628,7 @@ void shoc_pblintd_init_pot_f(Int shcol, Int nlev, Real *thl, Real* ql, Real* q, static constexpr Int num_arrays = 3; std::vector temp_d(num_arrays); - ekat::host_to_device({thl, ql, q}, shcol, nlev, temp_d, true); + ekat::host_to_device({thl, ql, q}, shcol, nlev, temp_d); view_2d thl_d(temp_d[0]), ql_d (temp_d[1]), @@ -1179,10 +650,10 @@ void shoc_pblintd_init_pot_f(Int shcol, Int nlev, Real *thl, Real* ql, Real* q, }); std::vector inout_views = {thv_d}; - ekat::device_to_host({thv}, shcol, nlev, inout_views, true); + ekat::device_to_host({thv}, shcol, nlev, inout_views); } -void compute_shoc_mix_shoc_length_f(Int nlev, Int shcol, Real* tke, Real* brunt, +void compute_shoc_mix_shoc_length_host(Int nlev, Int shcol, Real* tke, Real* brunt, Real* zt_grid, Real* l_inf, Real* shoc_mix) { using SHF = Functions; @@ -1201,7 +672,7 @@ void compute_shoc_mix_shoc_length_f(Int nlev, Int shcol, Real* tke, Real* brunt, // Sync to device ScreamDeepCopy::copy_to_device({l_inf}, shcol, temp_1d_d); - ekat::host_to_device(ptr_array, shcol, nlev, temp_2d_d, true); + ekat::host_to_device(ptr_array, shcol, nlev, temp_2d_d); view_1d l_inf_d (temp_1d_d[0]); @@ -1231,10 +702,10 @@ void compute_shoc_mix_shoc_length_f(Int nlev, Int shcol, Real* tke, Real* brunt, // Sync back to host std::vector inout_views = {shoc_mix_d}; - ekat::device_to_host({shoc_mix}, shcol, nlev, inout_views, true); + ekat::device_to_host({shoc_mix}, shcol, nlev, inout_views); } -void check_tke_f(Int shcol, Int nlev, Real* tke) +void check_tke_host(Int shcol, Int nlev, Real* tke) { using SHOC = Functions; using Spack = typename SHOC::Spack; @@ -1246,7 +717,7 @@ void check_tke_f(Int shcol, Int nlev, Real* tke) std::vector temp_2d_d(1); // Sync to device - ekat::host_to_device({tke}, shcol, nlev, temp_2d_d, true); + ekat::host_to_device({tke}, shcol, nlev, temp_2d_d); view_2d tke_d(temp_2d_d[0]); @@ -1263,10 +734,10 @@ void check_tke_f(Int shcol, Int nlev, Real* tke) // Sync back to host std::vector inout_views = {tke_d}; - ekat::device_to_host({tke}, shcol, nlev, inout_views, true); + ekat::device_to_host({tke}, shcol, nlev, inout_views); } -void linear_interp_f(Real* x1, Real* x2, Real* y1, Real* y2, Int km1, Int km2, Int ncol, Real minthresh) +void linear_interp_host(Real* x1, Real* x2, Real* y1, Real* y2, Int km1, Int km2, Int ncol, Real minthresh) { using SHF = Functions; @@ -1282,7 +753,7 @@ void linear_interp_f(Real* x1, Real* x2, Real* y1, Real* y2, Int km1, Int km2, I std::vector ptr_array = {x1, x2, y1}; // Sync to device - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d); view_2d x1_d(temp_2d_d[0]), @@ -1305,10 +776,10 @@ void linear_interp_f(Real* x1, Real* x2, Real* y1, Real* y2, Int km1, Int km2, I // Sync back to host std::vector inout_views = {y2_d}; - ekat::device_to_host({y2}, ncol, km2, inout_views, true); + ekat::device_to_host({y2}, ncol, km2, inout_views); } -void clipping_diag_third_shoc_moments_f(Int nlevi, Int shcol, Real *w_sec_zi, +void clipping_diag_third_shoc_moments_host(Int nlevi, Int shcol, Real *w_sec_zi, Real *w3) { using SHF = Functions; @@ -1321,7 +792,7 @@ void clipping_diag_third_shoc_moments_f(Int nlevi, Int shcol, Real *w_sec_zi, // Sync to device std::vector temp_d(2); - ekat::host_to_device({w_sec_zi, w3}, shcol, nlevi, temp_d, true); + ekat::host_to_device({w_sec_zi, w3}, shcol, nlevi, temp_d); view_2d w_sec_zi_d(temp_d[0]), @@ -1340,10 +811,10 @@ void clipping_diag_third_shoc_moments_f(Int nlevi, Int shcol, Real *w_sec_zi, // Sync back to host std::vector inout_views = {w3_d}; - ekat::device_to_host({w3}, shcol, nlevi, inout_views, true); + ekat::device_to_host({w3}, shcol, nlevi, inout_views); } -void shoc_energy_integrals_f(Int shcol, Int nlev, Real *host_dse, Real *pdel, +void shoc_energy_integrals_host(Int shcol, Int nlev, Real *host_dse, Real *pdel, Real *rtm, Real *rcm, Real *u_wind, Real *v_wind, Real *se_int, Real *ke_int, Real *wv_int, Real *wl_int) { @@ -1361,7 +832,7 @@ void shoc_energy_integrals_f(Int shcol, Int nlev, Real *host_dse, Real *pdel, std::vector ptr_array = {host_dse, pdel, rtm, rcm, u_wind, v_wind}; // Sync to device - ekat::host_to_device(ptr_array, shcol, nlev, temp_d, true); + ekat::host_to_device(ptr_array, shcol, nlev, temp_d); // inputs view_2d @@ -1410,7 +881,7 @@ void shoc_energy_integrals_f(Int shcol, Int nlev, Real *host_dse, Real *pdel, ScreamDeepCopy::copy_to_host({se_int,ke_int,wv_int,wl_int}, shcol, inout_views); } -void diag_second_moments_lbycond_f(Int shcol, Real* wthl_sfc, Real* wqw_sfc, Real* uw_sfc, Real* vw_sfc, Real* ustar2, Real* wstar, +void diag_second_moments_lbycond_host(Int shcol, Real* wthl_sfc, Real* wqw_sfc, Real* uw_sfc, Real* vw_sfc, Real* ustar2, Real* wstar, Real* wthl_sec, Real* wqw_sec, Real* uw_sec, Real* vw_sec, Real* wtke_sec, Real* thl_sec, Real* qw_sec, Real* qwthl_sec) { using SHOC = Functions; @@ -1473,7 +944,7 @@ void diag_second_moments_lbycond_f(Int shcol, Real* wthl_sfc, Real* wqw_sfc, Rea ScreamDeepCopy::copy_to_host({wthl_sec, wqw_sec, uw_sec, vw_sec, wtke_sec, thl_sec, qw_sec, qwthl_sec}, shcol, host_views); } -void diag_second_moments_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* u_wind, Real* v_wind, +void diag_second_moments_host(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* u_wind, Real* v_wind, Real* tke, Real* isotropy, Real* tkh, Real* tk, Real* dz_zi, Real* zt_grid, Real* zi_grid, Real* shoc_mix, Real* thl_sec, Real* qw_sec, Real* wthl_sec, Real* wqw_sec, Real* qwthl_sec, Real* uw_sec, Real* vw_sec, Real* wtke_sec, Real* w_sec) @@ -1493,7 +964,7 @@ void diag_second_moments_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* q std::vector ptr_array = {thetal, qw, u_wind, v_wind, tke, isotropy, tkh, tk, zt_grid, shoc_mix, thl_sec, qw_sec, wthl_sec, wqw_sec, qwthl_sec, uw_sec, vw_sec, wtke_sec, dz_zi, zi_grid}; - ekat::host_to_device(ptr_array, dim1_array, dim2_array, temp_2d, true); + ekat::host_to_device(ptr_array, dim1_array, dim2_array, temp_2d); view_2d thetal_2d (temp_2d[0]), @@ -1571,10 +1042,10 @@ void diag_second_moments_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* q std::vector dim1(9, shcol); std::vector dim2 = {nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlev }; std::vector host_views = {thl_sec_2d, qw_sec_2d, wthl_sec_2d, wqw_sec_2d, qwthl_sec_2d, uw_sec_2d, vw_sec_2d, wtke_sec_2d, w_sec_2d}; - ekat::device_to_host({thl_sec, qw_sec, wthl_sec, wqw_sec, qwthl_sec, uw_sec, vw_sec, wtke_sec, w_sec}, dim1, dim2, host_views, true); + ekat::device_to_host({thl_sec, qw_sec, wthl_sec, wqw_sec, qwthl_sec, uw_sec, vw_sec, wtke_sec, w_sec}, dim1, dim2, host_views); } -void diag_second_shoc_moments_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* u_wind, Real* v_wind, Real* tke, +void diag_second_shoc_moments_host(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* u_wind, Real* v_wind, Real* tke, Real* isotropy, Real* tkh, Real* tk, Real* dz_zi, Real* zt_grid, Real* zi_grid, Real* shoc_mix, Real* wthl_sfc, Real* wqw_sfc, Real* uw_sfc, Real* vw_sfc, Real* thl_sec, Real* qw_sec, Real* wthl_sec, Real* wqw_sec, Real* qwthl_sec, Real* uw_sec, Real* vw_sec, Real* wtke_sec, Real* w_sec) @@ -1603,7 +1074,7 @@ void diag_second_shoc_moments_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Re std::vector ptr_array = {thetal, qw, u_wind, v_wind, tke, isotropy, tkh, tk, zt_grid, shoc_mix, thl_sec, qw_sec, wthl_sec, wqw_sec, qwthl_sec, uw_sec, vw_sec, wtke_sec, dz_zi, zi_grid}; - ekat::host_to_device(ptr_array, dim1_array, dim2_array, temp_2d, true); + ekat::host_to_device(ptr_array, dim1_array, dim2_array, temp_2d); view_2d thetal_2d (temp_2d[0]), @@ -1689,10 +1160,10 @@ void diag_second_shoc_moments_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Re std::vector dim1(9, shcol); std::vector dim2 = {nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlev }; std::vector host_2d_views = {thl_sec_2d, qw_sec_2d, wthl_sec_2d, wqw_sec_2d, qwthl_sec_2d, uw_sec_2d, vw_sec_2d, wtke_sec_2d, w_sec_2d}; - ekat::device_to_host({thl_sec, qw_sec, wthl_sec, wqw_sec, qwthl_sec, uw_sec, vw_sec, wtke_sec, w_sec}, dim1, dim2, host_2d_views, true); + ekat::device_to_host({thl_sec, qw_sec, wthl_sec, wqw_sec, qwthl_sec, uw_sec, vw_sec, wtke_sec, w_sec}, dim1, dim2, host_2d_views); } -void compute_brunt_shoc_length_f(Int nlev, Int nlevi, Int shcol, Real* dz_zt, Real* thv, Real* thv_zi, Real* brunt) +void compute_brunt_shoc_length_host(Int nlev, Int nlevi, Int shcol, Real* dz_zt, Real* thv, Real* thv_zi, Real* brunt) { using SHF = Functions; @@ -1708,7 +1179,7 @@ void compute_brunt_shoc_length_f(Int nlev, Int nlevi, Int shcol, Real* dz_zt, Re std::vector ptr_array = {dz_zt, thv, thv_zi, brunt}; // Sync to device - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d); view_2d dz_zt_d (temp_d[0]), @@ -1731,10 +1202,10 @@ void compute_brunt_shoc_length_f(Int nlev, Int nlevi, Int shcol, Real* dz_zt, Re // Sync back to host std::vector inout_views = {brunt_d}; - ekat::device_to_host({brunt}, shcol, nlev, inout_views, true); + ekat::device_to_host({brunt}, shcol, nlev, inout_views); } -void compute_l_inf_shoc_length_f(Int nlev, Int shcol, Real *zt_grid, Real *dz_zt, +void compute_l_inf_shoc_length_host(Int nlev, Int shcol, Real *zt_grid, Real *dz_zt, Real *tke, Real *l_inf) { using SHF = Functions; @@ -1749,7 +1220,7 @@ void compute_l_inf_shoc_length_f(Int nlev, Int shcol, Real *zt_grid, Real *dz_zt // Sync to device std::vector temp_d(3); - ekat::host_to_device({zt_grid, dz_zt, tke}, shcol, nlev, temp_d, true); + ekat::host_to_device({zt_grid, dz_zt, tke}, shcol, nlev, temp_d); // inputs view_2d @@ -1782,7 +1253,7 @@ void compute_l_inf_shoc_length_f(Int nlev, Int shcol, Real *zt_grid, Real *dz_zt ScreamDeepCopy::copy_to_host({l_inf}, shcol, inout_views); } -void check_length_scale_shoc_length_f(Int nlev, Int shcol, Real* host_dx, Real* host_dy, Real* shoc_mix) +void check_length_scale_shoc_length_host(Int nlev, Int shcol, Real* host_dx, Real* host_dy, Real* shoc_mix) { using SHF = Functions; @@ -1798,7 +1269,7 @@ void check_length_scale_shoc_length_f(Int nlev, Int shcol, Real* host_dx, Real* std::vector temp_1d_d(2); std::vector temp_2d_d(1); ScreamDeepCopy::copy_to_device({host_dx,host_dy}, shcol, temp_1d_d); - ekat::host_to_device({shoc_mix}, shcol, nlev, temp_2d_d, true); + ekat::host_to_device({shoc_mix}, shcol, nlev, temp_2d_d); view_1d host_dx_d(temp_1d_d[0]), @@ -1821,10 +1292,10 @@ void check_length_scale_shoc_length_f(Int nlev, Int shcol, Real* host_dx, Real* // Sync back to host std::vector inout_views = {shoc_mix_d}; - ekat::device_to_host({shoc_mix}, shcol, nlev, inout_views, true); + ekat::device_to_host({shoc_mix}, shcol, nlev, inout_views); } -void shoc_diag_obklen_f(Int shcol, Real* uw_sfc, Real* vw_sfc, Real* wthl_sfc, Real* wqw_sfc, Real* thl_sfc, +void shoc_diag_obklen_host(Int shcol, Real* uw_sfc, Real* vw_sfc, Real* wthl_sfc, Real* wqw_sfc, Real* thl_sfc, Real* cldliq_sfc, Real* qv_sfc, Real* ustar, Real* kbfs, Real* obklen) { using SHF = Functions; @@ -1880,7 +1351,7 @@ void shoc_diag_obklen_f(Int shcol, Real* uw_sfc, Real* vw_sfc, Real* wthl_sfc, R ScreamDeepCopy::copy_to_host({ustar, kbfs, obklen}, shcol, inout_views); } -void shoc_pblintd_cldcheck_f(Int shcol, Int nlev, Int nlevi, Real* zi, Real* cldn, Real* pblh) { +void shoc_pblintd_cldcheck_host(Int shcol, Int nlev, Int nlevi, Real* zi, Real* cldn, Real* pblh) { using SHOC = Functions; using Spack = typename SHOC::Spack; using Scalar = typename SHOC::Scalar; @@ -1891,7 +1362,7 @@ void shoc_pblintd_cldcheck_f(Int shcol, Int nlev, Int nlevi, Real* zi, Real* cld std::vector dim2 = {nlevi, nlev}; std::vector cldcheck_2d(2); - ekat::host_to_device({zi, cldn}, dim1, dim2, cldcheck_2d, true); + ekat::host_to_device({zi, cldn}, dim1, dim2, cldcheck_2d); view_2d zi_2d (cldcheck_2d[0]), @@ -1919,7 +1390,7 @@ void shoc_pblintd_cldcheck_f(Int shcol, Int nlev, Int nlevi, Real* zi, Real* cld ScreamDeepCopy::copy_to_host({pblh}, shcol, inout_views); } -void shoc_length_f(Int shcol, Int nlev, Int nlevi, Real* host_dx, Real* host_dy, +void shoc_length_host(Int shcol, Int nlev, Int nlevi, Real* host_dx, Real* host_dy, Real* zt_grid, Real* zi_grid, Real*dz_zt, Real* tke, Real* thv, Real*brunt, Real* shoc_mix) { @@ -1942,7 +1413,7 @@ void shoc_length_f(Int shcol, Int nlev, Int nlevi, Real* host_dx, Real* host_dy, thv, brunt, shoc_mix}; // Sync to device ScreamDeepCopy::copy_to_device({host_dx, host_dy}, shcol, temp_1d_d); - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d); // inputs view_1d @@ -1991,10 +1462,10 @@ void shoc_length_f(Int shcol, Int nlev, Int nlevi, Real* host_dx, Real* host_dy, // Sync back to host std::vector inout_views = {brunt_d,shoc_mix_d}; - ekat::device_to_host({brunt,shoc_mix}, shcol, nlev, inout_views, true); + ekat::device_to_host({brunt,shoc_mix}, shcol, nlev, inout_views); } -void shoc_energy_fixer_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Real* zt_grid, +void shoc_energy_fixer_host(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Real* zt_grid, Real* zi_grid, Real* se_b, Real* ke_b, Real* wv_b, Real* wl_b, Real* se_a, Real* ke_a, Real* wv_a, Real* wl_a, Real* wthl_sfc, Real* wqw_sfc, Real* rho_zt, Real* tke, Real* pint, @@ -2022,7 +1493,7 @@ void shoc_energy_fixer_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, R // Sync to device ScreamDeepCopy::copy_to_device(ptr_array_1d, shcol, temp_1d_d); - ekat::host_to_device(ptr_array_2d, dim1_sizes, dim2_sizes, temp_2d_d, true); + ekat::host_to_device(ptr_array_2d, dim1_sizes, dim2_sizes, temp_2d_d); view_1d se_b_d(temp_1d_d[0]), @@ -2082,10 +1553,10 @@ void shoc_energy_fixer_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, R // Sync back to host std::vector inout_views = {host_dse_d}; - ekat::device_to_host({host_dse}, shcol, nlev, inout_views, true); + ekat::device_to_host({host_dse}, shcol, nlev, inout_views); } -void compute_shoc_vapor_f(Int shcol, Int nlev, Real* qw, Real* ql, Real* qv) +void compute_shoc_vapor_host(Int shcol, Int nlev, Real* qw, Real* ql, Real* qv) { using SHF = Functions; @@ -2099,7 +1570,7 @@ void compute_shoc_vapor_f(Int shcol, Int nlev, Real* qw, Real* ql, Real* qv) // Sync to device std::vector temp_d(num_arrays); - ekat::host_to_device( {qw, ql, qv}, shcol, nlev, temp_d, true); + ekat::host_to_device( {qw, ql, qv}, shcol, nlev, temp_d); // Inputs/Outputs view_2d @@ -2121,10 +1592,10 @@ void compute_shoc_vapor_f(Int shcol, Int nlev, Real* qw, Real* ql, Real* qv) // Sync back to host std::vector inout_views = {qv_d}; - ekat::device_to_host({qv}, shcol, nlev, inout_views, true); + ekat::device_to_host({qv}, shcol, nlev, inout_views); } -void update_prognostics_implicit_f(Int shcol, Int nlev, Int nlevi, Int num_tracer, Real dtime, +void update_prognostics_implicit_host(Int shcol, Int nlev, Int nlevi, Int num_tracer, Real dtime, Real* dz_zt, Real* dz_zi, Real* rho_zt, Real* zt_grid, Real* zi_grid, Real* tk, Real* tkh, Real* uw_sfc, Real* vw_sfc, Real* wthl_sfc, Real* wqw_sfc, Real* wtracer_sfc, Real* thetal, Real* qw, Real* tracer, @@ -2157,8 +1628,8 @@ void update_prognostics_implicit_f(Int shcol, Int nlev, Int nlevi, Int num_trace // Sync to device ScreamDeepCopy::copy_to_device({uw_sfc, vw_sfc, wthl_sfc, wqw_sfc}, shcol, temp_1d_d); - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d, true); - ekat::host_to_device({tracer}, shcol, nlev, num_tracer, temp_3d_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d); + ekat::host_to_device({tracer}, shcol, nlev, num_tracer, temp_3d_d); view_1d uw_sfc_d(temp_1d_d[0]), @@ -2256,13 +1727,13 @@ void update_prognostics_implicit_f(Int shcol, Int nlev, Int nlevi, Int num_trace // Sync back to host std::vector inout_views_2d = {thetal_d, qw_d, u_wind_d, v_wind_d, tke_d}; - ekat::device_to_host({thetal, qw, u_wind, v_wind, tke}, shcol, nlev, inout_views_2d, true); + ekat::device_to_host({thetal, qw, u_wind, v_wind, tke}, shcol, nlev, inout_views_2d); std::vector inout_views = {qtracers_f90_d}; - ekat::device_to_host({tracer}, shcol, nlev, num_tracer, inout_views, true); + ekat::device_to_host({tracer}, shcol, nlev, num_tracer, inout_views); } -void diag_third_shoc_moments_f(Int shcol, Int nlev, Int nlevi, Real* w_sec, Real* thl_sec, +void diag_third_shoc_moments_host(Int shcol, Int nlev, Int nlevi, Real* w_sec, Real* thl_sec, Real* wthl_sec, Real* isotropy, Real* brunt, Real* thetal, Real* tke, Real* dz_zt, Real* dz_zi, Real* zt_grid, Real* zi_grid, Real* w3) @@ -2285,7 +1756,7 @@ void diag_third_shoc_moments_f(Int shcol, Int nlev, Int nlevi, Real* w_sec, Real dz_zi, zt_grid, zi_grid, w3}; // Sync to device - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d); view_2d wsec_d(temp_d[0]), @@ -2338,10 +1809,10 @@ void diag_third_shoc_moments_f(Int shcol, Int nlev, Int nlevi, Real* w_sec, Real // Sync back to host std::vector inout_views = {w3_d}; - ekat::device_to_host({w3}, shcol, nlevi, inout_views, true); + ekat::device_to_host({w3}, shcol, nlevi, inout_views); } -void adv_sgs_tke_f(Int nlev, Int shcol, Real dtime, Real* shoc_mix, Real* wthv_sec, +void adv_sgs_tke_host(Int nlev, Int shcol, Real dtime, Real* shoc_mix, Real* wthv_sec, Real* sterm_zt, Real* tk, Real* tke, Real* a_diss) { using SHF = Functions; @@ -2358,7 +1829,7 @@ void adv_sgs_tke_f(Int nlev, Int shcol, Real dtime, Real* shoc_mix, Real* wthv_s std::vector ptr_array = {shoc_mix, wthv_sec, sterm_zt, tk, tke, a_diss}; // Sync to device - ekat::host_to_device(ptr_array, shcol, nlev, temp_d, true); + ekat::host_to_device(ptr_array, shcol, nlev, temp_d); view_2d //input @@ -2389,10 +1860,10 @@ void adv_sgs_tke_f(Int nlev, Int shcol, Real dtime, Real* shoc_mix, Real* wthv_s // Sync back to host std::vector inout_views = {tke_d, a_diss_d}; - ekat::device_to_host({tke, a_diss}, shcol, nlev, inout_views, true); + ekat::device_to_host({tke, a_diss}, shcol, nlev, inout_views); } -void shoc_assumed_pdf_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* w_field, +void shoc_assumed_pdf_host(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* w_field, Real* thl_sec, Real* qw_sec, Real* wthl_sec, Real* w_sec, Real* wqw_sec, Real* qwthl_sec, Real* w3, Real* pres, Real* zt_grid, Real* zi_grid, Real* shoc_cldfrac, Real* shoc_ql, Real* wqls, Real* wthv_sec, Real* shoc_ql2) @@ -2416,7 +1887,7 @@ void shoc_assumed_pdf_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, wqw_sec, qwthl_sec, w3, w_field, pres, zt_grid, zi_grid, shoc_cldfrac, shoc_ql, wqls, wthv_sec, shoc_ql2}; // Sync to device - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d); // Inputs/Outputs view_2d @@ -2477,9 +1948,9 @@ void shoc_assumed_pdf_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, // Sync back to host std::vector out_views = {shoc_cldfrac_d, shoc_ql_d, wqls_d, wthv_sec_d, shoc_ql2_d}; - ekat::device_to_host({shoc_cldfrac, shoc_ql, wqls, wthv_sec, shoc_ql2}, shcol, nlev, out_views, true); + ekat::device_to_host({shoc_cldfrac, shoc_ql, wqls, wthv_sec, shoc_ql2}, shcol, nlev, out_views); } -void compute_shr_prod_f(Int nlevi, Int nlev, Int shcol, Real* dz_zi, Real* u_wind, Real* v_wind, Real* sterm) +void compute_shr_prod_host(Int nlevi, Int nlev, Int shcol, Real* dz_zi, Real* u_wind, Real* v_wind, Real* sterm) { using SHF = Functions; @@ -2497,7 +1968,7 @@ void compute_shr_prod_f(Int nlevi, Int nlev, Int shcol, Real* dz_zi, Real* u_win std::vector ptr_array = {dz_zi, u_wind, v_wind, sterm}; // Sync to device - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_d); view_2d //input @@ -2525,10 +1996,10 @@ void compute_shr_prod_f(Int nlevi, Int nlev, Int shcol, Real* dz_zi, Real* u_win // Sync back to host std::vector inout_views = {sterm_d}; - ekat::device_to_host({sterm}, shcol, nlevi, inout_views, true); + ekat::device_to_host({sterm}, shcol, nlevi, inout_views); } -void compute_tmpi_f(Int nlevi, Int shcol, Real dtime, Real *rho_zi, Real *dz_zi, Real *tmpi) +void compute_tmpi_host(Int nlevi, Int shcol, Real dtime, Real *rho_zi, Real *dz_zi, Real *tmpi) { using SHF = Functions; @@ -2542,7 +2013,7 @@ void compute_tmpi_f(Int nlevi, Int shcol, Real dtime, Real *rho_zi, Real *dz_zi, // Sync to device std::vector temp_d(num_arrays); - ekat::host_to_device({rho_zi, dz_zi, tmpi}, shcol, nlevi, temp_d, true); + ekat::host_to_device({rho_zi, dz_zi, tmpi}, shcol, nlevi, temp_d); // Inputs/Outputs view_2d @@ -2564,10 +2035,10 @@ void compute_tmpi_f(Int nlevi, Int shcol, Real dtime, Real *rho_zi, Real *dz_zi, // Sync back to host std::vector inout_views = {tmpi_d}; - ekat::device_to_host({tmpi}, shcol, nlevi, inout_views, true); + ekat::device_to_host({tmpi}, shcol, nlevi, inout_views); } -void integ_column_stability_f(Int nlev, Int shcol, Real *dz_zt, +void integ_column_stability_host(Int nlev, Int shcol, Real *dz_zt, Real *pres, Real* brunt, Real *brunt_int) { using SHF = Functions; @@ -2584,7 +2055,7 @@ void integ_column_stability_f(Int nlev, Int shcol, Real *dz_zt, // Sync to device std::vector temp_d(num_arrays); - ekat::host_to_device({dz_zt, pres, brunt}, shcol, nlev, temp_d, true); + ekat::host_to_device({dz_zt, pres, brunt}, shcol, nlev, temp_d); // Inputs view_2d @@ -2619,7 +2090,7 @@ void integ_column_stability_f(Int nlev, Int shcol, Real *dz_zt, ScreamDeepCopy::copy_to_host({brunt_int}, shcol, inout_views); } -void isotropic_ts_f(Int nlev, Int shcol, Real* brunt_int, Real* tke, +void isotropic_ts_host(Int nlev, Int shcol, Real* brunt_int, Real* tke, Real* a_diss, Real* brunt, Real* isotropy) { using SHF = Functions; @@ -2640,7 +2111,7 @@ void isotropic_ts_f(Int nlev, Int shcol, Real* brunt_int, Real* tke, // Sync to device ScreamDeepCopy::copy_to_device({brunt_int}, shcol, temp_1d); - ekat::host_to_device(ptr_array, shcol, nlev, temp_2d, true); + ekat::host_to_device(ptr_array, shcol, nlev, temp_2d); //inputs view_1d brunt_int_d(temp_1d[0]); @@ -2668,8 +2139,8 @@ void isotropic_ts_f(Int nlev, Int shcol, Real* brunt_int, Real* tke, const auto isotropy_s = ekat::subview(isotropy_d, i); //output // Hard code these runtime options for F90 - const Real lambda_low = 0.001; - const Real lambda_high = 0.04; + const Real lambda_low = 0.001; + const Real lambda_high = 0.08; const Real lambda_slope = 2.65; const Real lambda_thresh = 0.02; SHF::isotropic_ts(team, nlev, lambda_low, lambda_high, lambda_slope, lambda_thresh, @@ -2678,11 +2149,11 @@ void isotropic_ts_f(Int nlev, Int shcol, Real* brunt_int, Real* tke, // Sync back to host std::vector inout_views = {isotropy_d}; - ekat::device_to_host({isotropy}, shcol, nlev, inout_views, true); + ekat::device_to_host({isotropy}, shcol, nlev, inout_views); } -void dp_inverse_f(Int nlev, Int shcol, Real *rho_zt, Real *dz_zt, Real *rdp_zt) +void dp_inverse_host(Int nlev, Int shcol, Real *rho_zt, Real *dz_zt, Real *rdp_zt) { using SHF = Functions; @@ -2696,7 +2167,7 @@ void dp_inverse_f(Int nlev, Int shcol, Real *rho_zt, Real *dz_zt, Real *rdp_zt) // Sync to device std::vector temp_d(num_arrays); - ekat::host_to_device({rho_zt, dz_zt, rdp_zt}, shcol, nlev, temp_d, true); + ekat::host_to_device({rho_zt, dz_zt, rdp_zt}, shcol, nlev, temp_d); // Inputs/Outputs view_2d @@ -2718,10 +2189,10 @@ void dp_inverse_f(Int nlev, Int shcol, Real *rho_zt, Real *dz_zt, Real *rdp_zt) // Sync back to host std::vector inout_views = {rdp_zt_d}; - ekat::device_to_host({rdp_zt}, shcol, nlev, inout_views, true); + ekat::device_to_host({rdp_zt}, shcol, nlev, inout_views); } -int shoc_init_f(Int nlev, Real *pref_mid, Int nbot_shoc, Int ntop_shoc) +int shoc_init_host(Int nlev, Real *pref_mid, Int nbot_shoc, Int ntop_shoc) { using SHF = Functions; using Spack = typename SHF::Spack; @@ -2735,7 +2206,7 @@ int shoc_init_f(Int nlev, Real *pref_mid, Int nbot_shoc, Int ntop_shoc) return SHF::shoc_init(nbot_shoc,ntop_shoc,pref_mid_d); } -Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, Real* host_dx, Real* host_dy, Real* thv, Real* zt_grid, +Int shoc_main_host(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, Real* host_dx, Real* host_dy, Real* thv, Real* zt_grid, Real* zi_grid, Real* pres, Real* presi, Real* pdel, Real* wthl_sfc, Real* wqw_sfc, Real* uw_sfc, Real* vw_sfc, Real* wtracer_sfc, Int num_qtracers, Real* w_field, Real* inv_exner, Real* phis, Real* host_dse, Real* tke, Real* thetal, Real* qw, Real* u_wind, Real* v_wind, Real* qtracers, Real* wthv_sec, Real* tkh, Real* tk, @@ -2783,14 +2254,14 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, std::vector ptr_array_2d = {zt_grid, zi_grid, pres, presi, pdel, thv, w_field, wtracer_sfc, inv_exner, host_dse, tke, thetal, qw, u_wind, v_wind, - wthv_sec, tk, shoc_cldfrac, shoc_ql, shoc_ql2, + wthv_sec, tk, shoc_cldfrac, shoc_ql, shoc_ql2, tkh, shoc_mix, w_sec, thl_sec, qw_sec, qwthl_sec, wthl_sec, wqw_sec, wtke_sec, uw_sec, vw_sec, w3, wqls_sec, brunt, isotropy}; ScreamDeepCopy::copy_to_device(ptr_array_1d, shcol, temp_1d_d); - ekat::host_to_device(ptr_array_2d, dim1_2d_sizes, dim2_2d_sizes, temp_2d_d, true); - ekat::host_to_device({qtracers}, shcol, nlev, num_qtracers, temp_3d_d, true); + ekat::host_to_device(ptr_array_2d, dim1_2d_sizes, dim2_2d_sizes, temp_2d_d); + ekat::host_to_device({qtracers}, shcol, nlev, num_qtracers, temp_3d_d); Int index_counter = 0; view_1d @@ -2972,16 +2443,16 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, qw_sec_d, qwthl_sec_d, wthl_sec_d, wqw_sec_d, wtke_sec_d, uw_sec_d, vw_sec_d, w3_d, wqls_sec_d, brunt_d, isotropy_d}; - ekat::device_to_host(ptr_array_2d_out, dim1_2d_out, dim2_2d_out, out_views_2d, true); + ekat::device_to_host(ptr_array_2d_out, dim1_2d_out, dim2_2d_out, out_views_2d); // 3d std::vector out_views_3d = {qtracers_f90_d}; - ekat::device_to_host({qtracers}, shcol, nlev, num_qtracers, out_views_3d, true); + ekat::device_to_host({qtracers}, shcol, nlev, num_qtracers, out_views_3d); return elapsed_microsec; } -void pblintd_height_f(Int shcol, Int nlev, Int npbl, Real* z, Real* u, Real* v, Real* ustar, Real* thv, Real* thv_ref, Real* pblh, Real* rino, bool* check) +void pblintd_height_host(Int shcol, Int nlev, Int npbl, Real* z, Real* u, Real* v, Real* ustar, Real* thv, Real* thv_ref, Real* pblh, Real* rino, bool* check) { using SHOC = Functions; using Spack = typename SHOC::Spack; @@ -2993,7 +2464,7 @@ void pblintd_height_f(Int shcol, Int nlev, Int npbl, Real* z, Real* u, Real* v, using MemberType = typename SHOC::MemberType; std::vector views_2d(5); - ekat::host_to_device({z, u, v, thv, rino}, shcol, nlev, views_2d, true); + ekat::host_to_device({z, u, v, thv, rino}, shcol, nlev, views_2d); view_2d z_2d (views_2d[0]), u_2d (views_2d[1]), @@ -3034,14 +2505,13 @@ void pblintd_height_f(Int shcol, Int nlev, Int npbl, Real* z, Real* u, Real* v, ScreamDeepCopy::copy_to_host({pblh}, shcol, out_1d_views); std::vector out_2d_views = {rino_2d}; - ekat::device_to_host({rino}, shcol, nlev, out_2d_views, true); + ekat::device_to_host({rino}, shcol, nlev, out_2d_views); std::vector out_bool_1d_views = {check_1d}; ScreamDeepCopy::copy_to_host({check}, shcol, out_bool_1d_views); } -void vd_shoc_decomp_and_solve_f(Int shcol, Int nlev, Int nlevi, Int num_rhs, Real* kv_term, Real* tmpi, Real* rdp_zt, Real dtime, - Real* flux, Real* var) +void vd_shoc_decomp_and_solve_host(Int shcol, Int nlev, Int nlevi, Int num_rhs, Real dtime, Real* kv_term, Real* tmpi, Real* rdp_zt, Real* flux, Real* var) { using SHF = Functions; @@ -3069,8 +2539,8 @@ void vd_shoc_decomp_and_solve_f(Int shcol, Int nlev, Int nlevi, Int num_rhs, Rea // Sync to device ScreamDeepCopy::copy_to_device({flux}, shcol, temp_1d_d); - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d, true); - ekat::host_to_device({var}, shcol, nlev, num_rhs, temp_3d_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d); + ekat::host_to_device({var}, shcol, nlev, num_rhs, temp_3d_d); view_1d flux_d(temp_1d_d[0]); @@ -3109,10 +2579,10 @@ void vd_shoc_decomp_and_solve_f(Int shcol, Int nlev, Int nlevi, Int num_rhs, Rea // Sync back to host std::vector inout_views = {var_d}; - ekat::device_to_host({var}, shcol, nlev, num_rhs, inout_views, true); + ekat::device_to_host({var}, shcol, nlev, num_rhs, inout_views); } -void shoc_grid_f(Int shcol, Int nlev, Int nlevi, Real* zt_grid, Real* zi_grid, Real* pdel, Real* dz_zt, Real* dz_zi, Real* rho_zt) +void shoc_grid_host(Int shcol, Int nlev, Int nlevi, Real* zt_grid, Real* zi_grid, Real* pdel, Real* dz_zt, Real* dz_zi, Real* rho_zt) { using SHF = Functions; @@ -3131,7 +2601,7 @@ void shoc_grid_f(Int shcol, Int nlev, Int nlevi, Real* zt_grid, Real* zi_grid, R dz_zt, dz_zi, rho_zt}; // Sync to device - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d); view_2d zt_grid_d(temp_2d_d[0]), @@ -3158,10 +2628,10 @@ void shoc_grid_f(Int shcol, Int nlev, Int nlevi, Real* zt_grid, Real* zi_grid, R // Sync back to host std::vector inout_views = {dz_zt_d, dz_zi_d, rho_zt_d}; - ekat::device_to_host({dz_zt, dz_zi, rho_zt}, {shcol, shcol, shcol}, {nlev, nlevi, nlev}, inout_views, true); + ekat::device_to_host({dz_zt, dz_zi, rho_zt}, {shcol, shcol, shcol}, {nlev, nlevi, nlev}, inout_views); } -void eddy_diffusivities_f(Int nlev, Int shcol, Real* pblh, Real* zt_grid, Real* tabs, Real* shoc_mix, Real* sterm_zt, +void eddy_diffusivities_host(Int nlev, Int shcol, Real* pblh, Real* zt_grid, Real* tabs, Real* shoc_mix, Real* sterm_zt, Real* isotropy, Real* tke, Real* tkh, Real* tk) { using SHF = Functions; @@ -3185,7 +2655,7 @@ void eddy_diffusivities_f(Int nlev, Int shcol, Real* pblh, Real* zt_grid, Real* // Sync to device ScreamDeepCopy::copy_to_device({pblh}, shcol, temp_1d_d); - ekat::host_to_device(ptr_array, shcol, nlev, temp_2d_d, true); + ekat::host_to_device(ptr_array, shcol, nlev, temp_2d_d); view_1d pblh_d(temp_1d_d[0]); @@ -3223,10 +2693,10 @@ void eddy_diffusivities_f(Int nlev, Int shcol, Real* pblh, Real* zt_grid, Real* // Sync back to host std::vector inout_views = {tkh_d, tk_d}; - ekat::device_to_host({tkh, tk}, shcol, nlev, inout_views, true); + ekat::device_to_host({tkh, tk}, shcol, nlev, inout_views); } -void pblintd_surf_temp_f(Int shcol, Int nlev, Int nlevi, Real* z, Real* ustar, Real* obklen, Real* kbfs, Real* thv, Real* tlv, Real* pblh, bool* check, Real* rino) +void pblintd_surf_temp_host(Int shcol, Int nlev, Int nlevi, Real* z, Real* ustar, Real* obklen, Real* kbfs, Real* thv, Real* tlv, Real* pblh, bool* check, Real* rino) { using SHOC = Functions; using Spack = typename SHOC::Spack; @@ -3236,7 +2706,7 @@ void pblintd_surf_temp_f(Int shcol, Int nlev, Int nlevi, Real* z, Real* ustar, R using view_2d = typename SHOC::view_2d; std::vector views_2d(3); - ekat::host_to_device({z, thv, rino}, shcol, nlev, views_2d, true); + ekat::host_to_device({z, thv, rino}, shcol, nlev, views_2d); view_2d z_2d (views_2d[0]), thv_2d (views_2d[1]), rino_2d(views_2d[2]); @@ -3277,13 +2747,13 @@ void pblintd_surf_temp_f(Int shcol, Int nlev, Int nlevi, Real* z, Real* ustar, R ScreamDeepCopy::copy_to_host({pblh, tlv}, shcol, out_1d_views); std::vector out_2d_views = {rino_2d}; - ekat::device_to_host({rino}, shcol, nlev, out_2d_views, true); + ekat::device_to_host({rino}, shcol, nlev, out_2d_views); std::vector out_bool_1d_views = {check_1d}; ScreamDeepCopy::copy_to_host({check}, shcol, out_bool_1d_views); } -void pblintd_check_pblh_f(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* ustar, bool* check, Real* pblh) +void pblintd_check_pblh_host(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* ustar, bool* check, Real* pblh) { using SHOC = Functions; using Spack = typename SHOC::Spack; @@ -3293,7 +2763,7 @@ void pblintd_check_pblh_f(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Rea using view_2d = typename SHOC::view_2d; std::vector views_2d(1); - ekat::host_to_device({z}, shcol, nlev, views_2d, true); + ekat::host_to_device({z}, shcol, nlev, views_2d); view_2d z_2d (views_2d[0]); std::vector views_1d(2); @@ -3319,7 +2789,7 @@ void pblintd_check_pblh_f(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Rea ScreamDeepCopy::copy_to_host({pblh}, shcol, out_1d_views); } -void pblintd_f(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* zi, Real* thl, Real* ql, Real* q, Real* u, Real* v, Real* ustar, Real* obklen, Real* kbfs, Real* cldn, Real* pblh) +void pblintd_host(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* zi, Real* thl, Real* ql, Real* q, Real* u, Real* v, Real* ustar, Real* obklen, Real* kbfs, Real* cldn, Real* pblh) { using SHF = Functions; @@ -3345,7 +2815,7 @@ void pblintd_f(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* zi, Real // Sync to device ScreamDeepCopy::copy_to_device({ustar, obklen, kbfs}, shcol, temp_1d_d); - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d); view_1d ustar_d(temp_1d_d[0]), @@ -3399,7 +2869,7 @@ void pblintd_f(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* zi, Real ScreamDeepCopy::copy_to_host({pblh}, shcol, out_views); } -void shoc_tke_f(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real* shoc_mix, Real* dz_zi, Real* dz_zt, Real* pres, +void shoc_tke_host(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real* shoc_mix, Real* dz_zi, Real* dz_zt, Real* pres, Real* tabs, Real* u_wind, Real* v_wind, Real* brunt, Real* zt_grid, Real* zi_grid, Real* pblh, Real* tke, Real* tk, Real* tkh, Real* isotropy) { @@ -3427,7 +2897,7 @@ void shoc_tke_f(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real // Sync to device ScreamDeepCopy::copy_to_device({pblh}, shcol, temp_1d_d); - ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d, true); + ekat::host_to_device(ptr_array, dim1_sizes, dim2_sizes, temp_2d_d); view_1d pblh_d(temp_1d_d[0]); @@ -3479,8 +2949,8 @@ void shoc_tke_f(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real const auto isotropy_s = ekat::subview(isotropy_d, i); // Hardcode for F90 testing - const Real lambda_low = 0.001; - const Real lambda_high = 0.04; + const Real lambda_low = 0.001; + const Real lambda_high = 0.08; const Real lambda_slope = 2.65; const Real lambda_thresh = 0.02; const Real Ckh = 0.1; @@ -3496,10 +2966,10 @@ void shoc_tke_f(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real // Sync back to host std::vector inout_views = {tke_d, tk_d, tkh_d, isotropy_d}; - ekat::device_to_host({tke, tk, tkh, isotropy}, shcol, nlev, inout_views, true); + ekat::device_to_host({tke, tk, tkh, isotropy}, shcol, nlev, inout_views); } -void compute_shoc_temperature_f(Int shcol, Int nlev, Real *thetal, Real *ql, Real *inv_exner, Real* tabs) +void compute_shoc_temperature_host(Int shcol, Int nlev, Real *thetal, Real *ql, Real *inv_exner, Real* tabs) { using SHF = Functions; @@ -3513,7 +2983,7 @@ void compute_shoc_temperature_f(Int shcol, Int nlev, Real *thetal, Real *ql, Rea // Sync to device std::vector temp_d(num_arrays); - ekat::host_to_device({thetal, ql, inv_exner, tabs}, shcol, nlev, temp_d, true); + ekat::host_to_device({thetal, ql, inv_exner, tabs}, shcol, nlev, temp_d); // Inputs/Outputs view_2d @@ -3537,7 +3007,280 @@ void compute_shoc_temperature_f(Int shcol, Int nlev, Real *thetal, Real *ql, Rea // Sync back to host std::vector out_views = {tabs_d}; - ekat::device_to_host({tabs}, shcol, nlev, out_views, true); + ekat::device_to_host({tabs}, shcol, nlev, out_views); +} + +void shoc_assumed_pdf_tilde_to_real_host(Real w_first, Real sqrtw2, Real* w1) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 1); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Real local_w1(*w1); + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack sqrtw2_(sqrtw2), w1_(local_w1), w_first_(w_first); + SHF::shoc_assumed_pdf_tilde_to_real(w_first_, sqrtw2_, w1_); + t_d(0) = w1_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *w1 = t_h(0); +} + +void shoc_assumed_pdf_vv_parameters_host(Real w_first, Real w_sec, Real w3var, Real w_tol_sqd, Real* skew_w, Real* w1_1, Real* w1_2, Real* w2_1, Real* w2_2, Real* a) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 6); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack w3var_(w3var), w_first_(w_first), w_sec_(w_sec), a_, skew_w_, w1_1_, w1_2_, w2_1_, w2_2_; + SHF::shoc_assumed_pdf_vv_parameters(w_first_, w_sec_, w3var_, w_tol_sqd, skew_w_, w1_1_, w1_2_, w2_1_, w2_2_, a_); + t_d(0) = a_[0]; + t_d(1) = skew_w_[0]; + t_d(2) = w1_1_[0]; + t_d(3) = w1_2_[0]; + t_d(4) = w2_1_[0]; + t_d(5) = w2_2_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *a = t_h(0); + *skew_w = t_h(1); + *w1_1 = t_h(2); + *w1_2 = t_h(3); + *w2_1 = t_h(4); + *w2_2 = t_h(5); +} + +void shoc_assumed_pdf_thl_parameters_host(Real wthlsec, Real sqrtw2, Real sqrtthl, Real thlsec, Real thl_first, Real w1_1, Real w1_2, Real skew_w, Real a, Real thl_tol, Real w_thresh, Real* thl1_1, Real* thl1_2, Real* thl2_1, Real* thl2_2, Real* sqrtthl2_1, Real* sqrtthl2_2) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 6); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack a_(a), skew_w_(skew_w), sqrtthl_(sqrtthl), sqrtw2_(sqrtw2), thl_first_(thl_first), thlsec_(thlsec), w1_1_(w1_1), w1_2_(w1_2), wthlsec_(wthlsec), sqrtthl2_1_, sqrtthl2_2_, thl1_1_, thl1_2_, thl2_1_, thl2_2_; + SHF::shoc_assumed_pdf_thl_parameters(wthlsec_, sqrtw2_, sqrtthl_, thlsec_, thl_first_, w1_1_, w1_2_, skew_w_, a_, thl_tol, w_thresh, thl1_1_, thl1_2_, thl2_1_, thl2_2_, sqrtthl2_1_, sqrtthl2_2_); + t_d(0) = sqrtthl2_1_[0]; + t_d(1) = sqrtthl2_2_[0]; + t_d(2) = thl1_1_[0]; + t_d(3) = thl1_2_[0]; + t_d(4) = thl2_1_[0]; + t_d(5) = thl2_2_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *sqrtthl2_1 = t_h(0); + *sqrtthl2_2 = t_h(1); + *thl1_1 = t_h(2); + *thl1_2 = t_h(3); + *thl2_1 = t_h(4); + *thl2_2 = t_h(5); +} + +void shoc_assumed_pdf_qw_parameters_host(Real wqwsec, Real sqrtw2, Real skew_w, Real sqrtqt, Real qwsec, Real w1_2, Real w1_1, Real qw_first, Real a, Real rt_tol, Real w_thresh, Real* qw1_1, Real* qw1_2, Real* qw2_1, Real* qw2_2, Real* sqrtqw2_1, Real* sqrtqw2_2) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 6); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack a_(a), qw_first_(qw_first), qwsec_(qwsec), skew_w_(skew_w), sqrtqt_(sqrtqt), sqrtw2_(sqrtw2), w1_1_(w1_1), w1_2_(w1_2), wqwsec_(wqwsec), qw1_1_, qw1_2_, qw2_1_, qw2_2_, sqrtqw2_1_, sqrtqw2_2_; + SHF::shoc_assumed_pdf_qw_parameters(wqwsec_, sqrtw2_, skew_w_, sqrtqt_, qwsec_, w1_2_, w1_1_, qw_first_, a_, rt_tol, w_thresh, qw1_1_, qw1_2_, qw2_1_, qw2_2_, sqrtqw2_1_, sqrtqw2_2_); + t_d(0) = qw1_1_[0]; + t_d(1) = qw1_2_[0]; + t_d(2) = qw2_1_[0]; + t_d(3) = qw2_2_[0]; + t_d(4) = sqrtqw2_1_[0]; + t_d(5) = sqrtqw2_2_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *qw1_1 = t_h(0); + *qw1_2 = t_h(1); + *qw2_1 = t_h(2); + *qw2_2 = t_h(3); + *sqrtqw2_1 = t_h(4); + *sqrtqw2_2 = t_h(5); +} + +void shoc_assumed_pdf_inplume_correlations_host(Real sqrtqw2_1, Real sqrtthl2_1, Real a, Real sqrtqw2_2, Real sqrtthl2_2, Real qwthlsec, Real qw1_1, Real qw_first, Real thl1_1, Real thl_first, Real qw1_2, Real thl1_2, Real* r_qwthl_1) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 1); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack a_(a), qw1_1_(qw1_1), qw1_2_(qw1_2), qw_first_(qw_first), qwthlsec_(qwthlsec), sqrtqw2_1_(sqrtqw2_1), sqrtqw2_2_(sqrtqw2_2), sqrtthl2_1_(sqrtthl2_1), sqrtthl2_2_(sqrtthl2_2), thl1_1_(thl1_1), thl1_2_(thl1_2), thl_first_(thl_first), r_qwthl_1_; + SHF::shoc_assumed_pdf_inplume_correlations(sqrtqw2_1_, sqrtthl2_1_, a_, sqrtqw2_2_, sqrtthl2_2_, qwthlsec_, qw1_1_, qw_first_, thl1_1_, thl_first_, qw1_2_, thl1_2_, r_qwthl_1_); + t_d(0) = r_qwthl_1_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *r_qwthl_1 = t_h(0); +} + +void shoc_assumed_pdf_compute_temperature_host(Real thl1, Real pval, Real* tl1) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 1); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack pval_(pval), thl1_(thl1), tl1_; + SHF::shoc_assumed_pdf_compute_temperature(thl1_, pval_, tl1_); + t_d(0) = tl1_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *tl1 = t_h(0); +} + +void shoc_assumed_pdf_compute_qs_host(Real tl1_1, Real tl1_2, Real pval, Real* qs1, Real* beta1, Real* qs2, Real* beta2) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using Smask = typename SHF::Smask; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 4); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack pval_(pval), tl1_1_(tl1_1), tl1_2_(tl1_2), beta1_, beta2_, qs1_, qs2_; + Smask active_entries(true); + SHF::shoc_assumed_pdf_compute_qs(tl1_1_, tl1_2_, pval_, active_entries, qs1_, beta1_, qs2_, beta2_); + t_d(0) = beta1_[0]; + t_d(1) = beta2_[0]; + t_d(2) = qs1_[0]; + t_d(3) = qs2_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *beta1 = t_h(0); + *beta2 = t_h(1); + *qs1 = t_h(2); + *qs2 = t_h(3); +} + +void shoc_assumed_pdf_compute_s_host(Real qw1, Real qs1, Real beta, Real pval, Real thl2, Real qw2, Real sqrtthl2, Real sqrtqw2, Real r_qwthl, Real* s, Real* std_s, Real* qn, Real* c) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 4); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack beta_(beta), pval_(pval), qs1_(qs1), qw1_(qw1), qw2_(qw2), r_qwthl_(r_qwthl), sqrtqw2_(sqrtqw2), sqrtthl2_(sqrtthl2), thl2_(thl2), c_, qn_, s_, std_s_; + SHF::shoc_assumed_pdf_compute_s(qw1_, qs1_, beta_, pval_, thl2_, qw2_, sqrtthl2_, sqrtqw2_, r_qwthl_, s_, std_s_, qn_, c_); + t_d(0) = c_[0]; + t_d(1) = qn_[0]; + t_d(2) = s_[0]; + t_d(3) = std_s_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *c = t_h(0); + *qn = t_h(1); + *s = t_h(2); + *std_s = t_h(3); +} + +void shoc_assumed_pdf_compute_sgs_liquid_host(Real a, Real ql1, Real ql2, Real* shoc_ql) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 1); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack a_(a), ql1_(ql1), ql2_(ql2), shoc_ql_; + SHF::shoc_assumed_pdf_compute_sgs_liquid(a_, ql1_, ql2_, shoc_ql_); + t_d(0) = shoc_ql_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *shoc_ql = t_h(0); +} + +void shoc_assumed_pdf_compute_cloud_liquid_variance_host(Real a, Real s1, Real ql1, Real c1, Real std_s1, Real s2, Real ql2, Real c2, Real std_s2, Real shoc_ql, Real* shoc_ql2) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 1); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack a_(a), c1_(c1), c2_(c2), ql1_(ql1), ql2_(ql2), s1_(s1), s2_(s2), shoc_ql_(shoc_ql), std_s1_(std_s1), std_s2_(std_s2), shoc_ql2_; + SHF::shoc_assumed_pdf_compute_cloud_liquid_variance(a_, s1_, ql1_, c1_, std_s1_, s2_, ql2_, c2_, std_s2_, shoc_ql_, shoc_ql2_); + t_d(0) = shoc_ql2_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *shoc_ql2 = t_h(0); +} + +void shoc_assumed_pdf_compute_liquid_water_flux_host(Real a, Real w1_1, Real w_first, Real ql1, Real w1_2, Real ql2, Real* wqls) +{ + using SHF = Functions; + + using Spack = typename SHF::Spack; + using view_1d = typename SHF::view_1d; + + view_1d t_d("t_d", 1); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack a_(a), ql1_(ql1), ql2_(ql2), w1_1_(w1_1), w1_2_(w1_2), w_first_(w_first), wqls_; + SHF::shoc_assumed_pdf_compute_liquid_water_flux(a_, w1_1_, w_first_, ql1_, w1_2_, ql2_, wqls_); + t_d(0) = wqls_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *wqls = t_h(0); +} + +void shoc_assumed_pdf_compute_buoyancy_flux_host(Real wthlsec, Real wqwsec, Real pval, Real wqls, Real* wthv_sec) +{ + using PF = Functions; + + using Spack = typename PF::Spack; + using view_1d = typename PF::view_1d; + + view_1d t_d("t_d", 1); + const auto t_h = Kokkos::create_mirror_view(t_d); + + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Spack pval_(pval), wqls_(wqls), wqwsec_(wqwsec), wthlsec_(wthlsec), wthv_sec_; + PF::shoc_assumed_pdf_compute_buoyancy_flux(wthlsec_, wqwsec_, pval_, wqls_, wthv_sec_); + t_d(0) = wthv_sec_[0]; + }); + Kokkos::deep_copy(t_h, t_d); + *wthv_sec = t_h(0); } } // namespace shoc diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.hpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_test_data.hpp similarity index 80% rename from components/eamxx/src/physics/shoc/shoc_functions_f90.hpp rename to components/eamxx/src/physics/shoc/tests/infra/shoc_test_data.hpp index bc61efb168e..2eb4236b4ed 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.hpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_test_data.hpp @@ -137,51 +137,6 @@ struct ShocEnergyIntegralsData : public PhysicsTestData { PTD_STD_DEF(ShocEnergyIntegralsData, 2, shcol, nlev); }; -struct ShocEnergyTotalFixerData : public ShocTestGridDataBase { - // Inputs - Int shcol, nlev, nlevi, nadv; - Real dtime; - Real *se_b, *ke_b, *wv_b, *wl_b, *se_a, *ke_a, *wv_a, *wl_a, *wthl_sfc, *wqw_sfc, *rho_zt, *pint; - - // Outputs - Real *te_a, *te_b; - - ShocEnergyTotalFixerData(Int shcol_, Int nlev_, Int nlevi_, Real dtime_, Int nadv_) : - ShocTestGridDataBase({{ shcol_, nlev_ }, { shcol_, nlevi_ }, { shcol_ }}, {{ &zt_grid, &rho_zt }, { &zi_grid, &pint }, { &se_b, &ke_b, &wv_b, &wl_b, &se_a, &ke_a, &wv_a, &wl_a, &wthl_sfc, &wqw_sfc, &te_a, &te_b }}), shcol(shcol_), nlev(nlev_), nlevi(nlevi_), nadv(nadv_), dtime(dtime_) {} - - PTD_STD_DEF(ShocEnergyTotalFixerData, 5, shcol, nlev, nlevi, dtime, nadv); -}; - -struct ShocEnergyThresholdFixerData : public PhysicsTestData { - // Inputs - Int shcol, nlev, nlevi; - Real *pint, *tke, *te_a, *te_b; - - // Outputs - Real *se_dis; - Int *shoctop; - - ShocEnergyThresholdFixerData(Int shcol_, Int nlev_, Int nlevi_) : - PhysicsTestData({{ shcol_, nlevi_ }, { shcol_, nlev_ }, { shcol_ }, { shcol_ }}, {{ &pint }, { &tke }, { &te_a, &te_b, &se_dis }}, {{ &shoctop }}), shcol(shcol_), nlev(nlev_), nlevi(nlevi_) {} - - PTD_STD_DEF(ShocEnergyThresholdFixerData, 3, shcol, nlev, nlevi); -}; - -struct ShocEnergyDseFixerData : public PhysicsTestData { - // Inputs - Int shcol, nlev; - Real *se_dis; - Int *shoctop; - - // Inputs/Outputs - Real *host_dse; - - ShocEnergyDseFixerData(Int shcol_, Int nlev_) : - PhysicsTestData({{ shcol_ }, { shcol_, nlev_ }, { shcol_ }}, {{ &se_dis }, { &host_dse }}, {{ &shoctop }}), shcol(shcol_), nlev(nlev_) {} - - PTD_STD_DEF(ShocEnergyDseFixerData, 2, shcol, nlev); -}; - struct CalcShocVertfluxData : public PhysicsTestData { // Inputs Int shcol, nlev, nlevi; @@ -240,49 +195,6 @@ struct DpInverseData : public PhysicsTestData { PTD_STD_DEF(DpInverseData, 2, shcol, nlev); }; -struct SfcFluxesData : public PhysicsTestData { - // Inputs - Int shcol, num_tracer; - Real dtime; - Real *rho_zi_sfc, *rdp_zt_sfc, *wthl_sfc, *wqw_sfc, *wtke_sfc, *wtracer_sfc; - - // Inputs/Outputs - Real *thetal, *qw, *tke, *wtracer; - - SfcFluxesData(Int shcol_, Int num_tracer_, Real dtime_) : - PhysicsTestData({{ shcol_ }, { shcol_, num_tracer_ }}, {{ &rho_zi_sfc, &rdp_zt_sfc, &wthl_sfc, &wqw_sfc, &wtke_sfc, &thetal, &qw, &tke }, { &wtracer_sfc, &wtracer }}), shcol(shcol_), num_tracer(num_tracer_), dtime(dtime_) {} - - PTD_STD_DEF(SfcFluxesData, 3, shcol, num_tracer, dtime); -}; - -struct ImpliSrfStressTermData : public PhysicsTestData { - // Inputs - Int shcol; - Real *rho_zi_sfc, *uw_sfc, *vw_sfc, *u_wind_sfc, *v_wind_sfc; - - // Outputs - Real *ksrf; - - ImpliSrfStressTermData(Int shcol_) : - PhysicsTestData({{ shcol_ }}, {{ &rho_zi_sfc, &uw_sfc, &vw_sfc, &u_wind_sfc, &v_wind_sfc, &ksrf }}), shcol(shcol_) {} - - PTD_STD_DEF(ImpliSrfStressTermData, 1, shcol); -}; - -struct TkeSrfFluxTermData : public PhysicsTestData { - // Inputs - Int shcol; - Real *uw_sfc, *vw_sfc; - - // Outputs - Real *wtke_sfc; - - TkeSrfFluxTermData(Int shcol_) : - PhysicsTestData({{ shcol_ }}, {{ &uw_sfc, &vw_sfc, &wtke_sfc }}), shcol(shcol_) {} - - PTD_STD_DEF(TkeSrfFluxTermData, 1, shcol); -}; - struct IntegColumnStabilityData : public PhysicsTestData { // Inputs Int shcol, nlev; @@ -476,54 +388,6 @@ struct CheckLengthScaleShocLengthData : public PhysicsTestData { PTD_STD_DEF(CheckLengthScaleShocLengthData, 2, shcol, nlev); }; -struct FtermsInputForDiagThirdShocMomentData { - // Inputs - Real dz_zi, dz_zt, dz_zt_kc, isotropy_zi, brunt_zi, thetal_zi; - - // Outputs - Real thedz, thedz2, iso, isosqrd, buoy_sgs2, bet2; -}; - -struct AaTermsDiagThirdShocMomentData { - // Inputs - Real omega0, omega1, omega2, x0, x1, y0, y1; - - // Outputs - Real aa0, aa1; -}; - -struct F0ToF5DiagThirdShocMomentData { - // Inputs - Real thedz, thedz2, bet2, iso, isosqrd, wthl_sec, wthl_sec_kc, wthl_sec_kb, thl_sec_kc, thl_sec_kb, w_sec, w_sec_kc, w_sec_zi, tke, tke_kc; - - // Outputs - Real f0, f1, f2, f3, f4, f5; -}; - -struct OmegaTermsDiagThirdShocMomentData { - // Inputs - Real buoy_sgs2, f3, f4; - - // Outputs - Real omega0, omega1, omega2; -}; - -struct XYTermsDiagThirdShocMomentData { - // Inputs - Real buoy_sgs2, f0, f1, f2; - - // Outputs - Real x0, y0, x1, y1; -}; - -struct W3DiagThirdShocMomentData { - // Inputs - Real aa0, aa1, x0, x1, f5; - - // Outputs - Real w3; -}; - struct ClippingDiagThirdShocMomentsData : public PhysicsTestData { // Inputs Int shcol, nlevi; @@ -619,7 +483,7 @@ struct ShocAssumedPdfTildeToRealData { struct ShocAssumedPdfVvParametersData { // Inputs - Real w_first, w_sec, w3var; + Real w_first, w_sec, w3var, w_tol_sqd; // Outputs Real skew_w, w1_1, w1_2, w2_1, w2_2, a; @@ -627,8 +491,7 @@ struct ShocAssumedPdfVvParametersData { struct ShocAssumedPdfThlParametersData { // Inputs - Real wthlsec, sqrtw2, sqrtthl, thlsec, thl_first, w1_1, w1_2, skew_w, a; - bool dothetal_skew; + Real wthlsec, sqrtw2, sqrtthl, thlsec, thl_first, w1_1, w1_2, skew_w, a, thl_tol, w_thresh; // Outputs Real thl1_1, thl1_2, thl2_1, thl2_2, sqrtthl2_1, sqrtthl2_2; @@ -636,7 +499,7 @@ struct ShocAssumedPdfThlParametersData { struct ShocAssumedPdfQwParametersData { // Inputs - Real wqwsec, sqrtw2, skew_w, sqrtqt, qwsec, w1_2, w1_1, qw_first, a; + Real wqwsec, sqrtw2, skew_w, sqrtqt, qwsec, w1_2, w1_1, qw_first, a, rt_tol, w_thresh; // Outputs Real qw1_1, qw1_2, qw2_1, qw2_2, sqrtqw2_1, sqrtqw2_2; @@ -652,7 +515,7 @@ struct ShocAssumedPdfInplumeCorrelationsData { struct ShocAssumedPdfComputeTemperatureData { // Inputs - Real thl1, basepres, pval; + Real thl1, pval; // Outputs Real tl1; @@ -700,7 +563,7 @@ struct ShocAssumedPdfComputeLiquidWaterFluxData { struct ShocAssumedPdfComputeBuoyancyFluxData { // Inputs - Real wthlsec, epsterm, wqwsec, pval, wqls; + Real wthlsec, wqwsec, pval, wqls; // Outputs Real wthv_sec; @@ -997,12 +860,11 @@ struct VdShocDecompandSolveData : public PhysicsTestData { Real *kv_term, *tmpi, *rdp_zt, *flux; // Inputs/Outputs - Real *du, *dl, *d; - Real *var, *rhs; + Real *var; VdShocDecompandSolveData(Int shcol_, Int nlev_, Int nlevi_, Real dtime_, Int n_rhs_) : PhysicsTestData({{shcol_}, {shcol_, nlev_}, {shcol_, nlevi_}, {shcol_, nlev_, n_rhs_}}, - {{&flux}, {&rdp_zt, &du, &dl, &d, &rhs}, {&kv_term, &tmpi}, {&var } }, {}), + {{&flux}, {&rdp_zt}, {&kv_term, &tmpi}, {&var } }, {}), shcol(shcol_), nlev(nlev_), nlevi(nlevi_), n_rhs(n_rhs_), dtime(dtime_) {} PTD_STD_DEF(VdShocDecompandSolveData, 5, shcol, nlev, nlevi, dtime, n_rhs); @@ -1069,23 +931,17 @@ struct ComputeShocTempData : public PhysicsTestData { PTD_STD_DEF(ComputeShocTempData, 2, shcol, nlev); }; -// Glue functions to call fortran from from C++ with the Data struct +// Glue functions to call from host with the Data struct void shoc_grid (ShocGridData& d); void shoc_diag_obklen (ShocDiagObklenData& d); void update_host_dse (UpdateHostDseData& d); void shoc_energy_fixer (ShocEnergyFixerData& d); void shoc_energy_integrals (ShocEnergyIntegralsData& d); -void shoc_energy_total_fixer (ShocEnergyTotalFixerData& d); -void shoc_energy_threshold_fixer (ShocEnergyThresholdFixerData& d); -void shoc_energy_dse_fixer (ShocEnergyDseFixerData& d); void calc_shoc_vertflux (CalcShocVertfluxData& d); void calc_shoc_varorcovar (CalcShocVarorcovarData& d); void compute_tmpi (ComputeTmpiData& d); void dp_inverse (DpInverseData& d); -void sfc_fluxes (SfcFluxesData& d); -void impli_srf_stress_term (ImpliSrfStressTermData& d); -void tke_srf_flux_term (TkeSrfFluxTermData& d); void integ_column_stability (IntegColumnStabilityData& d); void check_tke (CheckTkeData& d); void shoc_tke (ShocTkeData& d); @@ -1098,12 +954,6 @@ void compute_brunt_shoc_length (ComputeBruntShocLengthData& void compute_l_inf_shoc_length (ComputeLInfShocLengthData& d); void compute_shoc_mix_shoc_length (ComputeShocMixShocLengthData& d); void check_length_scale_shoc_length (CheckLengthScaleShocLengthData& d); -void fterms_input_for_diag_third_shoc_moment (FtermsInputForDiagThirdShocMomentData& d); -void aa_terms_diag_third_shoc_moment (AaTermsDiagThirdShocMomentData& d); -void f0_to_f5_diag_third_shoc_moment (F0ToF5DiagThirdShocMomentData& d); -void omega_terms_diag_third_shoc_moment (OmegaTermsDiagThirdShocMomentData& d); -void x_y_terms_diag_third_shoc_moment (XYTermsDiagThirdShocMomentData& d); -void w3_diag_third_shoc_moment (W3DiagThirdShocMomentData& d); void clipping_diag_third_shoc_moments (ClippingDiagThirdShocMomentsData& d); void diag_second_moments_srf (DiagSecondMomentsSrfData& d); void linear_interp (LinearInterpData& d); @@ -1131,94 +981,94 @@ void diag_second_shoc_moments (DiagSecondShocMomentsData& void compute_shoc_vapor (ComputeShocVaporData& d); void update_prognostics_implicit (UpdatePrognosticsImplicitData& d); void shoc_main (ShocMainData& d); -void shoc_main_with_init (ShocMainData& d); void pblintd_height (PblintdHeightData& d); void vd_shoc_decomp_and_solve (VdShocDecompandSolveData& d); void pblintd_surf_temp(PblintdSurfTempData& d); void pblintd_check_pblh(PblintdCheckPblhData& d); void pblintd(PblintdData& d); void compute_shoc_temperature(ComputeShocTempData& d); -extern "C" { // _f function decls -void calc_shoc_varorcovar_f(Int shcol, Int nlev, Int nlevi, Real tunefac, +// Call from host + +void calc_shoc_varorcovar_host(Int shcol, Int nlev, Int nlevi, Real tunefac, Real *isotropy_zi, Real *tkh_zi, Real *dz_zi, Real *invar1, Real *invar2, Real *varorcovar); -void calc_shoc_vertflux_f(Int shcol, Int nlev, Int nlevi, Real *tkh_zi, +void calc_shoc_vertflux_host(Int shcol, Int nlev, Int nlevi, Real *tkh_zi, Real *dz_zi, Real *invar, Real *vertflux); -void shoc_diag_second_moments_srf_f(Int shcol, Real* wthl, Real* uw, Real* vw, +void shoc_diag_second_moments_srf_host(Int shcol, Real* wthl, Real* uw, Real* vw, Real* ustar2, Real* wstar); -void shoc_diag_second_moments_ubycond_f(Int shcol, Real* thl, Real* qw, Real* wthl, +void shoc_diag_second_moments_ubycond_host(Int shcol, Real* thl, Real* qw, Real* wthl, Real* wqw, Real* qwthl, Real* uw, Real* vw, Real* wtke); -void update_host_dse_f(Int shcol, Int nlev, Real* thlm, Real* shoc_ql, Real* inv_exner, Real* zt_grid, +void update_host_dse_host(Int shcol, Int nlev, Real* thlm, Real* shoc_ql, Real* inv_exner, Real* zt_grid, Real* phis, Real* host_dse); -void compute_diag_third_shoc_moment_f(Int shcol, Int nlev, Int nlevi, Real* w_sec, +void compute_diag_third_shoc_moment_host(Int shcol, Int nlev, Int nlevi, Real* w_sec, Real* thl_sec, Real* wthl_sec, Real* tke, Real* dz_zt, Real* dz_zi, Real* isotropy_zi, Real* brunt_zi, Real* w_sec_zi, Real* thetal_zi, Real* w3); -void shoc_pblintd_init_pot_f(Int shcol, Int nlev, Real* thl, Real* ql, Real* q, Real* thv); -void compute_shoc_mix_shoc_length_f(Int nlev, Int shcol, Real* tke, Real* brunt, +void shoc_pblintd_init_pot_host(Int shcol, Int nlev, Real* thl, Real* ql, Real* q, Real* thv); +void compute_shoc_mix_shoc_length_host(Int nlev, Int shcol, Real* tke, Real* brunt, Real* zt_grid, Real* l_inf, Real* shoc_mix); -void check_tke_f(Int shcol, Int nlev, Real* tke); -void linear_interp_f(Real* x1, Real* x2, Real* y1, Real* y2, Int km1, Int km2, Int ncol, Real minthresh); -void clipping_diag_third_shoc_moments_f(Int nlevi, Int shcol, Real *w_sec_zi, +void check_tke_host(Int shcol, Int nlev, Real* tke); +void linear_interp_host(Real* x1, Real* x2, Real* y1, Real* y2, Int km1, Int km2, Int ncol, Real minthresh); +void clipping_diag_third_shoc_moments_host(Int nlevi, Int shcol, Real *w_sec_zi, Real *w3); -void shoc_energy_integrals_f(Int shcol, Int nlev, Real *host_dse, Real *pdel, +void shoc_energy_integrals_host(Int shcol, Int nlev, Real *host_dse, Real *pdel, Real *rtm, Real *rcm, Real *u_wind, Real *v_wind, Real *se_int, Real *ke_int, Real *wv_int, Real *wl_int); -void compute_brunt_shoc_length_f(Int nlev, Int nlevi, Int shcol, Real* dz_zt, Real* thv, +void compute_brunt_shoc_length_host(Int nlev, Int nlevi, Int shcol, Real* dz_zt, Real* thv, Real* thv_zi, Real* brunt); -void compute_l_inf_shoc_length_f(Int nlev, Int shcol, Real *zt_grid, Real *dz_zt, +void compute_l_inf_shoc_length_host(Int nlev, Int shcol, Real *zt_grid, Real *dz_zt, Real *tke, Real *l_inf); -void check_length_scale_shoc_length_f(Int nlev, Int shcol, Real* host_dx, Real* host_dy, +void check_length_scale_shoc_length_host(Int nlev, Int shcol, Real* host_dx, Real* host_dy, Real* shoc_mix); -void diag_second_moments_lbycond_f(Int shcol, Real* wthl_sfc, Real* wqw_sfc, Real* uw_sfc, Real* vw_sfc, Real* ustar2, Real* wstar, +void diag_second_moments_lbycond_host(Int shcol, Real* wthl_sfc, Real* wqw_sfc, Real* uw_sfc, Real* vw_sfc, Real* ustar2, Real* wstar, Real* wthl_sec, Real* wqw_sec, Real* uw_sec, Real* vw_sec, Real* wtke_sec, Real* thl_sec, Real* qw_sec, Real* qwthl_sec); -void diag_second_moments_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* u_wind, Real* v_wind, Real* tke, Real* isotropy, +void diag_second_moments_host(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* u_wind, Real* v_wind, Real* tke, Real* isotropy, Real* tkh, Real* tk, Real* dz_zi, Real* zt_grid, Real* zi_grid, Real* shoc_mix, Real* thl_sec, Real* qw_sec, Real* wthl_sec, Real* wqw_sec, Real* qwthl_sec, Real* uw_sec, Real* vw_sec, Real* wtke_sec, Real* w_sec); -void diag_second_shoc_moments_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* u_wind, Real* v_wind, Real* tke, +void diag_second_shoc_moments_host(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* u_wind, Real* v_wind, Real* tke, Real* isotropy, Real* tkh, Real* tk, Real* dz_zi, Real* zt_grid, Real* zi_grid, Real* shoc_mix, Real* wthl_sfc, Real* wqw_sfc, Real* uw_sfc, Real* vw_sfc, Real* thl_sec, Real* qw_sec, Real* wthl_sec, Real* wqw_sec, Real* qwthl_sec, Real* uw_sec, Real* vw_sec, Real* wtke_sec, Real* w_sec); -void shoc_diag_obklen_f(Int shcol, Real* uw_sfc, Real* vw_sfc, Real* wthl_sfc, Real* wqw_sfc, +void shoc_diag_obklen_host(Int shcol, Real* uw_sfc, Real* vw_sfc, Real* wthl_sfc, Real* wqw_sfc, Real* thl_sfc, Real* cldliq_sfc, Real* qv_sfc, Real* ustar, Real* kbfs, Real* obklen); -void shoc_pblintd_cldcheck_f(Int shcol, Int nlev, Int nlevi, Real* zi, Real* cldn, Real* pblh); -void compute_shr_prod_f(Int nlevi, Int nlev, Int shcol, Real* dz_zi, Real* u_wind, Real* v_wind, Real* sterm); -void shoc_length_f(Int shcol, Int nlev, Int nlevi, Real* host_dx, Real* host_dy, +void shoc_pblintd_cldcheck_host(Int shcol, Int nlev, Int nlevi, Real* zi, Real* cldn, Real* pblh); +void compute_shr_prod_host(Int nlevi, Int nlev, Int shcol, Real* dz_zi, Real* u_wind, Real* v_wind, Real* sterm); +void shoc_length_host(Int shcol, Int nlev, Int nlevi, Real* host_dx, Real* host_dy, Real* zt_grid, Real* zi_grid, Real*dz_zt, Real* tke, Real* thv, Real*brunt, Real* shoc_mix); -void shoc_energy_fixer_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Real* zt_grid, +void shoc_energy_fixer_host(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Real* zt_grid, Real* zi_grid, Real* se_b, Real* ke_b, Real* wv_b, Real* wl_b, Real* se_a, Real* ke_a, Real* wv_a, Real* wl_a, Real* wthl_sfc, Real* wqw_sfc, Real* rho_zt, Real* tke, Real* pint, Real* host_dse); -void compute_shoc_vapor_f(Int shcol, Int nlev, Real* qw, Real* ql, Real* qv); -void update_prognostics_implicit_f(Int shcol, Int nlev, Int nlevi, Int num_tracer, Real dtime, +void compute_shoc_vapor_host(Int shcol, Int nlev, Real* qw, Real* ql, Real* qv); +void update_prognostics_implicit_host(Int shcol, Int nlev, Int nlevi, Int num_tracer, Real dtime, Real* dz_zt, Real* dz_zi, Real* rho_zt, Real* zt_grid, Real* zi_grid, Real* tk, Real* tkh, Real* uw_sfc, Real* vw_sfc, Real* wthl_sfc, Real* wqw_sfc, Real* wtracer_sfc, Real* thetal, Real* qw, Real* tracer, Real* tke, Real* u_wind, Real* v_wind); -void diag_third_shoc_moments_f(Int shcol, Int nlev, Int nlevi, Real* w_sec, Real* thl_sec, +void diag_third_shoc_moments_host(Int shcol, Int nlev, Int nlevi, Real* w_sec, Real* thl_sec, Real* wthl_sec, Real* isotropy, Real* brunt, Real* thetal, Real* tke, Real* dz_zt, Real* dz_zi, Real* zt_grid, Real* zi_grid, Real* w3); -void adv_sgs_tke_f(Int nlev, Int shcol, Real dtime, Real* shoc_mix, Real* wthv_sec, Real* sterm_zt, +void adv_sgs_tke_host(Int nlev, Int shcol, Real dtime, Real* shoc_mix, Real* wthv_sec, Real* sterm_zt, Real* tk, Real* tke, Real* a_diss); -void shoc_assumed_pdf_f(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* w_field, +void shoc_assumed_pdf_host(Int shcol, Int nlev, Int nlevi, Real* thetal, Real* qw, Real* w_field, Real* thl_sec, Real* qw_sec, Real* wthl_sec, Real* w_sec, Real* wqw_sec, Real* qwthl_sec, Real* w3, Real* pres, Real* zt_grid, Real* zi_grid, Real* shoc_cldfrac, Real* shoc_ql, Real* wqls, Real* wthv_sec, Real* shoc_ql2); -void compute_tmpi_f(Int nlevi, Int shcol, Real dtime, Real *rho_zi, Real *dz_zi, Real *tmpi); -void integ_column_stability_f(Int nlev, Int shcol, Real *dz_zt, +void compute_tmpi_host(Int nlevi, Int shcol, Real dtime, Real *rho_zi, Real *dz_zi, Real *tmpi); +void integ_column_stability_host(Int nlev, Int shcol, Real *dz_zt, Real *pres, Real *brunt, Real *brunt_int); -void isotropic_ts_f(Int nlev, Int shcol, Real* brunt_int, Real* tke, +void isotropic_ts_host(Int nlev, Int shcol, Real* brunt_int, Real* tke, Real* a_diss, Real* brunt, Real* isotropy); -void dp_inverse_f(Int nlev, Int shcol, Real *rho_zt, Real *dz_zt, Real *rdp_zt); +void dp_inverse_host(Int nlev, Int shcol, Real *rho_zt, Real *dz_zt, Real *rdp_zt); -int shoc_init_f(Int nlev, Real* pref_mid, Int nbot_shoc, Int ntop_shoc); -Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, Real* host_dx, Real* host_dy, Real* thv, +int shoc_init_host(Int nlev, Real* pref_mid, Int nbot_shoc, Int ntop_shoc); +Int shoc_main_host(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, Real* host_dx, Real* host_dy, Real* thv, Real* zt_grid, Real* zi_grid, Real* pres, Real* presi, Real* pdel, Real* wthl_sfc, Real* wqw_sfc, Real* uw_sfc, Real* vw_sfc, Real* wtracer_sfc, Int num_qtracers, Real* w_field, Real* inv_exner, Real* phis, Real* host_dse, Real* tke, Real* thetal, Real* qw, Real* u_wind, Real* v_wind, @@ -1227,24 +1077,51 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, Real* wthl_sec, Real* wqw_sec, Real* wtke_sec, Real* uw_sec, Real* vw_sec, Real* w3, Real* wqls_sec, Real* brunt, Real* shoc_ql2); -void pblintd_height_f(Int shcol, Int nlev, Int npbl, Real* z, Real* u, Real* v, Real* ustar, Real* thv, Real* thv_ref, Real* pblh, Real* rino, bool* check); +void pblintd_height_host(Int shcol, Int nlev, Int npbl, Real* z, Real* u, Real* v, Real* ustar, Real* thv, Real* thv_ref, Real* pblh, Real* rino, bool* check); -void vd_shoc_decomp_and_solve_f(Int shcol, Int nlev, Int nlevi, Int num_rhs, Real* kv_term, Real* tmpi, Real* rdp_zt, Real dtime, - Real* flux, Real* var); +void vd_shoc_decomp_and_solve_host(Int shcol, Int nlev, Int nlevi, Int num_rhs, Real dtime, Real* kv_term, Real* tmpi, Real* rdp_zt, Real* flux, Real* var); -void pblintd_surf_temp_f(Int shcol, Int nlev, Int nlevi, Real* z, Real* ustar, Real* obklen, Real* kbfs, Real* thv, Real* tlv, Real* pblh, bool* check, Real* rino); -void pblintd_check_pblh_f(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* ustar, bool* check, Real* pblh); +void pblintd_surf_temp_host(Int shcol, Int nlev, Int nlevi, Real* z, Real* ustar, Real* obklen, Real* kbfs, Real* thv, Real* tlv, Real* pblh, bool* check, Real* rino); -void pblintd_f(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* zi, Real* thl, Real* ql, Real* q, Real* u, Real* v, Real* ustar, Real* obklen, Real* kbfs, Real* cldn, Real* pblh); -void shoc_grid_f(Int shcol, Int nlev, Int nlevi, Real* zt_grid, Real* zi_grid, Real* pdel, Real* dz_zt, Real* dz_zi, Real* rho_zt); -void eddy_diffusivities_f(Int nlev, Int shcol, Real* pblh, Real* zt_grid, Real* tabs, Real* shoc_mix, Real* sterm_zt, Real* isotropy, +void pblintd_check_pblh_host(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* ustar, bool* check, Real* pblh); + +void pblintd_host(Int shcol, Int nlev, Int nlevi, Int npbl, Real* z, Real* zi, Real* thl, Real* ql, Real* q, Real* u, Real* v, Real* ustar, Real* obklen, Real* kbfs, Real* cldn, Real* pblh); +void shoc_grid_host(Int shcol, Int nlev, Int nlevi, Real* zt_grid, Real* zi_grid, Real* pdel, Real* dz_zt, Real* dz_zi, Real* rho_zt); +void eddy_diffusivities_host(Int nlev, Int shcol, Real* pblh, Real* zt_grid, Real* tabs, Real* shoc_mix, Real* sterm_zt, Real* isotropy, Real* tke, Real* tkh, Real* tk); -void shoc_tke_f(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real* shoc_mix, Real* dz_zi, Real* dz_zt, Real* pres, +void shoc_tke_host(Int shcol, Int nlev, Int nlevi, Real dtime, Real* wthv_sec, Real* shoc_mix, Real* dz_zi, Real* dz_zt, Real* pres, Real* u_wind, Real* v_wind, Real* brunt, Real* obklen, Real* zt_grid, Real* zi_grid, Real* pblh, Real* tke, Real* tk, Real* tkh, Real* isotropy); -void compute_shoc_temperature_f(Int shcol, Int nlev, Real* thetal, Real* ql, Real* inv_exner, Real* tabs); -} // end _f function decls +void compute_shoc_temperature_host(Int shcol, Int nlev, Real* thetal, Real* ql, Real* inv_exner, Real* tabs); + +void shoc_energy_total_fixer_host(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Real* zt_grid, Real* zi_grid, Real* se_b, Real* ke_b, Real* wv_b, Real* wl_b, Real* se_a, Real* ke_a, Real* wv_a, Real* wl_a, Real* wthl_sfc, Real* wqw_sfc, Real* rho_zt, Real* pint, Real* te_a, Real* te_b); + +void shoc_assumed_pdf_tilde_to_real_host(Real w_first, Real sqrtw2, Real* w1); + +void shoc_assumed_pdf_vv_parameters_host(Real w_first, Real w_sec, Real w3var, Real w_tol_sqd, Real* skew_w, Real* w1_1, Real* w1_2, Real* w2_1, Real* w2_2, Real* a); + +void shoc_assumed_pdf_thl_parameters_host(Real wthlsec, Real sqrtw2, Real sqrtthl, Real thlsec, Real thl_first, Real w1_1, Real w1_2, Real skew_w, Real a, Real thl_tol, Real w_thresh, Real* thl1_1, Real* thl1_2, Real* thl2_1, Real* thl2_2, Real* sqrtthl2_1, Real* sqrtthl2_2); + +void shoc_assumed_pdf_qw_parameters_host(Real wqwsec, Real sqrtw2, Real skew_w, Real sqrtqt, Real qwsec, Real w1_2, Real w1_1, Real qw_first, Real a, Real rt_tol, Real w_thresh, Real* qw1_1, Real* qw1_2, Real* qw2_1, Real* qw2_2, Real* sqrtqw2_1, Real* sqrtqw2_2); + +void shoc_assumed_pdf_inplume_correlations_host(Real sqrtqw2_1, Real sqrtthl2_1, Real a, Real sqrtqw2_2, Real sqrtthl2_2, Real qwthlsec, Real qw1_1, Real qw_first, Real thl1_1, Real thl_first, Real qw1_2, Real thl1_2, Real* r_qwthl_1); + +void shoc_assumed_pdf_compute_temperature_host(Real thl1, Real pval, Real* tl1); + +void shoc_assumed_pdf_compute_qs_host(Real tl1_1, Real tl1_2, Real pval, Real* qs1, Real* beta1, Real* qs2, Real* beta2); + +void shoc_assumed_pdf_compute_s_host(Real qw1, Real qs1, Real beta, Real pval, Real thl2, Real qw2, Real sqrtthl2, Real sqrtqw2, Real r_qwthl, Real* s, Real* std_s, Real* qn, Real* c); + +void shoc_assumed_pdf_compute_sgs_liquid_host(Real a, Real ql1, Real ql2, Real* shoc_ql); + +void shoc_assumed_pdf_compute_cloud_liquid_variance_host(Real a, Real s1, Real ql1, Real c1, Real std_s1, Real s2, Real ql2, Real c2, Real std_s2, Real shoc_ql, Real* shoc_ql2); + +void shoc_assumed_pdf_compute_liquid_water_flux_host(Real a, Real w1_1, Real w_first, Real ql1, Real w1_2, Real ql2, Real* wqls); + +void shoc_assumed_pdf_compute_buoyancy_flux_host(Real wthlsec, Real wqwsec, Real pval, Real wqls, Real* wthv_sec); + +// end _host function decls } // namespace shoc } // namespace scream diff --git a/components/eamxx/src/physics/shoc/tests/shoc_unit_tests_common.hpp b/components/eamxx/src/physics/shoc/tests/infra/shoc_unit_tests_common.hpp similarity index 66% rename from components/eamxx/src/physics/shoc/tests/shoc_unit_tests_common.hpp rename to components/eamxx/src/physics/shoc/tests/infra/shoc_unit_tests_common.hpp index 0d74ebaa5fe..f7326c06df9 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_unit_tests_common.hpp +++ b/components/eamxx/src/physics/shoc/tests/infra/shoc_unit_tests_common.hpp @@ -4,6 +4,9 @@ #include "shoc_functions.hpp" #include "share/scream_types.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "share/util/scream_setup_random_test.hpp" +#include "ekat/util/ekat_file_utils.hpp" +#include "ekat/util/ekat_test_utils.hpp" namespace scream { namespace shoc { @@ -21,6 +24,12 @@ namespace unit_test { struct UnitWrap { + enum BASELINE_ACTION { + NONE, + COMPARE, + GENERATE + }; + template struct UnitTest : public KokkosTypes { @@ -48,8 +57,68 @@ struct UnitWrap { using Smask = typename Functions::Smask; using C = typename Functions::C; - static constexpr Int max_pack_size = 16; - static constexpr Int num_test_itrs = max_pack_size / Spack::n; + struct Base { + std::string m_baseline_path; + std::string m_test_name; + BASELINE_ACTION m_baseline_action; + ekat::FILEPtr m_fid; + + Base() : + m_baseline_path(""), + m_test_name(Catch::getResultCapture().getCurrentTestName()), + m_baseline_action(NONE), + m_fid() + { + //Functions::shoc_init(); // just in case there is ever global shoc data + auto& ts = ekat::TestSession::get(); + if (ts.flags["c"]) { + m_baseline_action = COMPARE; + } + else if (ts.flags["g"]) { + m_baseline_action = GENERATE; + } + else if (ts.flags["n"]) { + m_baseline_action = NONE; + } + m_baseline_path = ts.params["b"]; + + + EKAT_REQUIRE_MSG( !(m_baseline_action != NONE && m_baseline_path == ""), + "SHOC unit test flags problem: baseline actions were requested but no baseline path was provided"); + + std::string baseline_name = m_baseline_path + "/" + m_test_name; + if (m_baseline_action == COMPARE) { + m_fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + } + else if (m_baseline_action == GENERATE) { + m_fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + } + } + + ~Base() + { + } + + std::mt19937_64 get_engine() + { + if (m_baseline_action != COMPARE) { + // We can use any seed + int seed; + auto engine = setup_random_test(nullptr, &seed); + if (m_baseline_action == GENERATE) { + // Write the seed + ekat::write(&seed, 1, m_fid); + } + return engine; + } + else { + // Read the seed + int seed; + ekat::read(&seed, 1, m_fid); + return setup_random_test(seed); + } + } + }; // Put struct decls here struct TestCalcShocVertflux; diff --git a/components/eamxx/src/physics/shoc/tests/shoc_aa_diag_third_moms_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_aa_diag_third_moms_tests.cpp deleted file mode 100644 index d6d7d522d66..00000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_aa_diag_third_moms_tests.cpp +++ /dev/null @@ -1,110 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestAAdiagThirdMoms { - - static void run_property() - { - - // Tests for the SHOC function: - // aa_terms_diag_third_shoc_moment - - // Run test two times. One where 0 parameters stay the same - // and the other where 1 parameters vary. Verify that the aa0 - // term remains constant between the two tests. - - // omega0 term - constexpr static Real omega0 = 7.54e-2; - // omega1 term - constexpr static Real omega1 = 5.37e-3; - // omega2 term - constexpr static Real omega2 = 0.54; - // x0 term - constexpr static Real x0 = -4.31; - // y0 term - constexpr static Real y0 = 22.72; - // x1 term - constexpr static Real x1_test1a = 41.05; - // y1 term - constexpr static Real y1_test1a = 375.69; - - // Initialize data structure for bridging to F90 - AaTermsDiagThirdShocMomentData SDS; - - // Load up the data - SDS.omega0 = omega0; - SDS.x0 = x0; - SDS.y0 = y0; - SDS.omega1 = omega1; - SDS.x1 = x1_test1a; - SDS.y1 = y1_test1a; - SDS.omega2 = omega2; - - // Call the fortran implementation - aa_terms_diag_third_shoc_moment(SDS); - - // Save the output - Real aa0_test1a = SDS.aa0; - Real aa1_test1a = SDS.aa1; - - // Load up data where only the 1 terms have varies - SDS.x1 = 0.3*x1_test1a; - SDS.y1 = 0.3*y1_test1a; - - // Call the fortran implementation - aa_terms_diag_third_shoc_moment(SDS); - - // Check the result - REQUIRE(SDS.aa0 == aa0_test1a); - REQUIRE(SDS.aa1 < aa1_test1a); - - } - - static void run_bfb() - { - // TODO - } - -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace{ - -TEST_CASE("shoc_aa_diag_third_moms_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestAAdiagThirdMoms; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_aa_diag_third_moms_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestAAdiagThirdMoms; - - TestStruct::run_bfb(); - -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp index e947946fa20..93926020d82 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp @@ -1,504 +1,481 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" -#include "share/util/scream_setup_random_test.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestShocAssumedPdf { - - static void run_property() - { - static constexpr Int shcol = 2; - static constexpr Int nlev = 5; - static constexpr auto nlevi = nlev + 1; - - // Tests for the top level subroutine - // shoc_assumed_pdf - - // Tests will start simple, and gradually add complexity to test - // the physics. - // NOTE: for this test we want exactly two columns. - - // TEST ONE - // No SGS variability test. Given inputs where there is a saturated - // profile but NO SGS variability in the scalar fluxes or variances - // (i.e. all second and third moment terms are zero), then verify that - // that cloud fraction is either 1 or 0 and that the SGS variability - // outputs are also zero everywhere. - - // Define input data - - // Note that the moisture and height profiles below represent that - // of the BOMEX case, but the temperatures are much colder, to encourage - // there to be points with ample cloud produced for this test. - - // Liquid water potential temperature [K] - static constexpr Real thetal[nlev] = {303, 300, 298, 298, 300}; - // Total water mixing ratio [kg/kg] - static constexpr Real qw[nlev] = {0.003, 0.004, 0.011, 0.016, 0.017}; - // Pressure [Pa] - static constexpr Real pres[nlev] = {70000, 80000, 85000, 90000, 100000}; - // Define the heights on the zt grid [m] - static constexpr Real zi_grid[nlevi] = {3000, 2000, 1500, 1000, 500, 0}; - - // All variances will be given zero or minimum threshold inputs - - // Define some reasonable bounds for output - static constexpr Real wqls_bound = 0.1; - static constexpr Real wthv_sec_bound = 10; - static constexpr Real shoc_ql2_bound = 0.1; - static constexpr Real shoc_ql_bound = 0.1; - - Real zt_grid[nlev]; - // Compute heights on midpoint grid - for(Int n = 0; n < nlev; ++n) { - zt_grid[n] = 0.5*(zi_grid[n]+zi_grid[n+1]); - } - - // Initialize data structure for bridging to F90 - ShocAssumedPdfData SDS(shcol, nlev, nlevi); - - // Load input data - for(Int s = 0; s < shcol; ++s) { - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - - SDS.thetal[offset] = thetal[n]; - SDS.qw[offset] = qw[n]; - SDS.pres[offset] = pres[n]; - SDS.zt_grid[offset] = zt_grid[n]; - SDS.w_field[offset] = 0; - SDS.w_sec[offset] = 0.004; - } - - for(Int n = 0; n < nlevi; ++n) { - const auto offset = n + s * nlevi; - - SDS.thl_sec[offset] = 0; - SDS.qw_sec[offset] = 0; - SDS.wthl_sec[offset] = 0; - SDS.wqw_sec[offset] = 0; - SDS.qwthl_sec[offset] = 0; - SDS.w3[offset] = 0; - SDS.zi_grid[offset] = zi_grid[n]; - } - } - - // Check that the inputs make sense - for(Int s = 0; s < shcol; ++s) { - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - - REQUIRE(SDS.qw[offset] > 0); - REQUIRE(SDS.thetal[offset] > 0); - REQUIRE(SDS.pres[offset] > 0); - REQUIRE(SDS.zt_grid[offset] > 0); - } - - for(Int n = 0; n < nlevi; ++n) { - const auto offset = n + s * nlev; - - REQUIRE(SDS.zi_grid[offset] >= 0); - } - } - - // Check that zt increase updward and pres decrease upward - for(Int s = 0; s < shcol; ++s) { - for(Int n = 0; n < nlev - 1; ++n) { - const auto offset = n + s * nlev; - REQUIRE(SDS.zt_grid[offset + 1] - SDS.zt_grid[offset] < 0); - REQUIRE(SDS.pres[offset + 1] - SDS.pres[offset] > 0); - } - - // Check that zi increase upward - for(Int n = 0; n < nlevi - 1; ++n) { - const auto offset = n + s * nlevi; - REQUIRE(SDS.zi_grid[offset + 1] - SDS.zi_grid[offset] < 0); - } - - } - - // Test that the inputs are reasonable. - REQUIRE(SDS.nlevi - SDS.nlev == 1); - // For this test we want exactly two columns - REQUIRE(SDS.shcol == 2); - - // Call the C++ implementation. - SDS.transpose(); - // expects data in fortran layout - shoc_assumed_pdf_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.w_field, - SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.w_sec, SDS.wqw_sec, - SDS.qwthl_sec, SDS.w3, SDS.pres, SDS.zt_grid, SDS.zi_grid, - SDS.shoc_cldfrac, SDS.shoc_ql, SDS.wqls, SDS.wthv_sec, SDS.shoc_ql2); - SDS.transpose(); - - // Verify the result - // Make sure cloud fraction is either 1 or 0 and all - // SGS terms are zero. - - for(Int s = 0; s < shcol; ++s) { - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - - REQUIRE( (SDS.shoc_cldfrac[offset] == 0 || SDS.shoc_cldfrac[offset] == 1) ); - REQUIRE(SDS.wqls[offset] == 0); - REQUIRE(SDS.wthv_sec[offset] == 0); - REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); // Computation is not exactly BFB with 0 - REQUIRE(SDS.shoc_ql[offset] >= 0); - } - } - - // TEST TWO - // Add in Scalar fluxes. This should give us a nonzero - // buoyancy flux everywhere but other SGS terms should remain zero - - // We will assume turbulence with a uniform profile - - // Flux of liquid water [K m/s] - static constexpr Real wthl_sec = -0.03; - // Flux of total water [m/s kg/kg] - static constexpr Real wqw_sec = 0.0002; - - for(Int s = 0; s < shcol; ++s) { - for(Int n = 0; n < nlevi; ++n) { - const auto offset = n + s * nlevi; - - SDS.wthl_sec[offset] = wthl_sec; - SDS.wqw_sec[offset] = wqw_sec; - } - } - - // Call the C++ implementation. - SDS.transpose(); - // expects data in fortran layout - shoc_assumed_pdf_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.w_field, - SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.w_sec, SDS.wqw_sec, - SDS.qwthl_sec, SDS.w3, SDS.pres, SDS.zt_grid, SDS.zi_grid, - SDS.shoc_cldfrac, SDS.shoc_ql, SDS.wqls, SDS.wthv_sec, SDS.shoc_ql2); - SDS.transpose(); - - // Verify the result - // Make sure cloud fraction is either 1 or 0 and all - // SGS terms are zero, EXCEPT wthv_sec. - - for(Int s = 0; s < shcol; ++s) { - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - - REQUIRE( (SDS.shoc_cldfrac[offset] == 0 || SDS.shoc_cldfrac[offset] == 1) ); - REQUIRE(SDS.wqls[offset] == 0); - REQUIRE(SDS.wthv_sec[offset] != 0); - REQUIRE(std::abs(SDS.wthv_sec[offset] < wthv_sec_bound)); - REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); // Computation is not exactly BFB with 0 - REQUIRE(SDS.shoc_ql[offset] >= 0); - REQUIRE(SDS.shoc_ql[offset] < shoc_ql_bound); - } - } - - // TEST THREE and FOUR - // Add in Scalar variances, and POSITIVE vertical velocity skewness test. - - // Add strong scalar variances as such that will produce cloud at every level. - - // For the first column feed in zero vertical velocity skewness. - // For the second column feed in large veriticle velocity skewss. - // Verify that for points where cloud fraction was < 0.5 in the first column, - // that cloud fraction then vice versa for points with cloud fraction > 0.5. - - // Thetal variance [K^2] - static constexpr Real thl_sec = 2; - // total water variance [kg^2/kg^2] - static constexpr Real qw_sec = 0.0002; - // Temperature and total water covariance [K kg/kg] - static constexpr Real qwthl_sec = 1e-5; - // Vertical velocity variance [m2/s2] - static constexpr Real w_sec = 0.2; - - // Load input data - for(Int s = 0; s < shcol; ++s) { - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - - SDS.w_sec[offset] = w_sec; - } - - for(Int n = 0; n < nlevi; ++n) { - const auto offset = n + s * nlevi; - - SDS.thl_sec[offset] = thl_sec; - SDS.qw_sec[offset] = qw_sec; - SDS.qwthl_sec[offset] = qwthl_sec; - SDS.w3[offset] = s*1.0; - } - } - - // Call the C++ implementation. - SDS.transpose(); - // expects data in fortran layout - shoc_assumed_pdf_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.w_field, - SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.w_sec, SDS.wqw_sec, - SDS.qwthl_sec, SDS.w3, SDS.pres, SDS.zt_grid, SDS.zi_grid, - SDS.shoc_cldfrac, SDS.shoc_ql, SDS.wqls, SDS.wthv_sec, SDS.shoc_ql2); - SDS.transpose(); - - // Check the result - - // With such a turbulence and scalar profile, this should have - // encouraged cloud everywhere. Verify that this is true. - - // Also verify that output lies within some reasonable bounds. - - // Then verify vertical velocity skewness info - - for(Int s = 0; s < shcol; ++s) { - for(Int n = 0; n < nlevi; ++n) { - const auto offset = n + s * nlevi; - const auto offsets = n + (s+1) * nlevi; - if (s < shcol-1){ - // Verify input w3 is greater in subsequent columns - REQUIRE(SDS.w3[offsets] > SDS.w3[offset]); - } - } - - // Verify output falls within reasonable bounds. For this positive - // vertical velocity skewness test and with the give inputs there - // should be cloud everywhere and all flux terms should be positive. - - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - const auto offsets = n + (s+1) * nlev; - - REQUIRE( (SDS.shoc_cldfrac[offset] > 0 || SDS.shoc_cldfrac[offset] < 1) ); - REQUIRE(SDS.wqls[offset] > 0); - REQUIRE(SDS.wthv_sec[offset] > 0); - REQUIRE(SDS.shoc_ql2[offset] > 0); - REQUIRE(SDS.shoc_ql[offset] > 0); - - REQUIRE(SDS.wqls[offset] < 0.1); - REQUIRE(SDS.wthv_sec[offset] < wthv_sec_bound); - REQUIRE(SDS.shoc_ql2[offset] < shoc_ql2_bound); - REQUIRE(SDS.shoc_ql[offset] < shoc_ql_bound); - - // Now verify that the relationships in a strongly positive vertical - // velocity flux regime hold true, relative to a symmetric vertical - // velocity regime. - if (s < shcol-1){ - - if (SDS.shoc_cldfrac[offset] < 0.5){ - REQUIRE(SDS.shoc_cldfrac[offsets] < SDS.shoc_cldfrac[offset]); - } - else if (SDS.shoc_cldfrac[offset] > 0.5){ - REQUIRE(SDS.shoc_cldfrac[offsets] > SDS.shoc_cldfrac[offset]); - } - - // In addition, in a positive skewness environment, the following - // should also be true - - // Grid mean liquid water decreased - REQUIRE(SDS.shoc_ql[offsets] < SDS.shoc_ql[offset]); - // liquid water flux increased - REQUIRE(SDS.wqls[offsets] > SDS.wqls[offset]); - // buoyancy flux increased - REQUIRE(SDS.wthv_sec[offsets] > SDS.wthv_sec[offset]); - // liquid water variance increased - REQUIRE(SDS.shoc_ql2[offsets] > SDS.shoc_ql2[offset]); - } - } - } - - // TEST FIVE - // Negative vertical velocity skewness. - - // Using same input as the above test, feed one column with zero skeweness - // and another test with negative vertical velocity skewness and verify - // result is physical. - - // Load input data - for(Int s = 0; s < shcol; ++s) { - for(Int n = 0; n < nlevi; ++n) { - const auto offset = n + s * nlevi; - - SDS.w3[offset] = s*-1.0; - } - } - - // Call the C++ implementation. - SDS.transpose(); - // expects data in fortran layout - shoc_assumed_pdf_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.w_field, - SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.w_sec, SDS.wqw_sec, - SDS.qwthl_sec, SDS.w3, SDS.pres, SDS.zt_grid, SDS.zi_grid, - SDS.shoc_cldfrac, SDS.shoc_ql, SDS.wqls, SDS.wthv_sec, SDS.shoc_ql2); - SDS.transpose(); - - // Check the result - - // Verify that output lies within some reasonable bounds. - - // Then verify vertical velocity skewness info - - for(Int s = 0; s < shcol; ++s) { - for(Int n = 0; n < nlevi; ++n) { - const auto offset = n + s * nlevi; - const auto offsets = n + (s+1) * nlevi; - if (s < shcol-1){ - // Verify input w3 is greater in subsequent columns - REQUIRE(SDS.w3[offsets] < SDS.w3[offset]); - } - } - - // Verify output falls within reasonable bounds - // For this negative vertical velocity test some variables - // will be expected to be less than zero. - - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - const auto offsets = n + (s+1) * nlev; - - REQUIRE( (SDS.shoc_cldfrac[offset] >= 0 || SDS.shoc_cldfrac[offset] < 1) ); - REQUIRE(SDS.shoc_ql2[offset] > 0); - REQUIRE(SDS.shoc_ql[offset] >= 0); - - REQUIRE(std::abs(SDS.wqls[offset] ) < wqls_bound); - REQUIRE(std::abs(SDS.wthv_sec[offset]) < wthv_sec_bound); - REQUIRE(SDS.shoc_ql2[offset] < shoc_ql2_bound); - REQUIRE(SDS.shoc_ql[offset] < shoc_ql_bound); - - // Now verify that the relationships in a strongly negative vertical - // velocity flux regime hold true, relative to a symmetric vertical - // velocity regime. - if (s < shcol-1){ - - if (SDS.shoc_cldfrac[offset] < 0.5){ - REQUIRE(SDS.shoc_cldfrac[offsets] < SDS.shoc_cldfrac[offset]); - } - else if (SDS.shoc_cldfrac[offset] > 0.5){ - REQUIRE(SDS.shoc_cldfrac[offsets] > SDS.shoc_cldfrac[offset]); - } - - // In addition, in a positive skewness environment, the following - // should also be true - - // Grid mean liquid water decreased - REQUIRE(SDS.shoc_ql[offsets] < SDS.shoc_ql[offset]); - // if cloud present, verify liquid water and buoyancy flux is negative - if (SDS.shoc_ql[offsets] > 0){ - REQUIRE(SDS.wqls[offsets] < SDS.wqls[offset]); - REQUIRE(SDS.wqls[offsets] < 0); - - REQUIRE(SDS.wthv_sec[offsets] < SDS.wthv_sec[offset]); - REQUIRE(SDS.wthv_sec[offsets] < 0); - } - - // liquid water variance increased - REQUIRE(SDS.shoc_ql2[offsets] > SDS.shoc_ql2[offset]); - } - } - } - - } - - static void run_bfb() - { - auto engine = setup_random_test(); - - ShocAssumedPdfData SDS_f90[] = { - // shcol, nlev, nlevi - ShocAssumedPdfData(10, 71, 72), - ShocAssumedPdfData(10, 12, 13), - ShocAssumedPdfData(7, 16, 17), - ShocAssumedPdfData(2, 7, 8), - }; - - // Generate random input data - for (auto& d : SDS_f90) { - d.randomize(engine, { - {d.thetal, {500, 700}}, - {d.zi_grid, {0, 100}}, - }); - } - - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - ShocAssumedPdfData SDS_cxx[] = { - ShocAssumedPdfData(SDS_f90[0]), - ShocAssumedPdfData(SDS_f90[1]), - ShocAssumedPdfData(SDS_f90[2]), - ShocAssumedPdfData(SDS_f90[3]), - }; - - // Assume all data is in C layout - - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - shoc_assumed_pdf(d); - } - - // Get data from cxx - for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - shoc_assumed_pdf_f(d.shcol, d.nlev, d.nlevi, d.thetal, d.qw, d.w_field, - d.thl_sec, d.qw_sec, d.wthl_sec, d.w_sec, d.wqw_sec, - d.qwthl_sec, d.w3, d.pres, d.zt_grid, d.zi_grid, - d.shoc_cldfrac, d.shoc_ql, d.wqls, d.wthv_sec, d.shoc_ql2); - d.transpose(); - } - - // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(ShocAssumedPdfData); - for (Int i = 0; i < num_runs; ++i) { - ShocAssumedPdfData& d_f90 = SDS_f90[i]; - ShocAssumedPdfData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.wqls); ++k) { - REQUIRE(d_f90.shoc_cldfrac[k] == d_cxx.shoc_cldfrac[k]); - REQUIRE(d_f90.shoc_ql[k] == d_cxx.shoc_ql[k]); - REQUIRE(d_f90.wqls[k] == d_cxx.wqls[k]); - REQUIRE(d_f90.wthv_sec[k] == d_cxx.wthv_sec[k]); - REQUIRE(d_f90.shoc_ql2[k] == d_cxx.shoc_ql2[k]); - } - } - } - } -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace { - -TEST_CASE("shoc_assumed_pdf_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocAssumedPdf; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_assumed_pdf_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocAssumedPdf; - - TestStruct::run_bfb(); -} - -} // namespace +#include "catch2/catch.hpp" + +#include "shoc_unit_tests_common.hpp" +#include "shoc_functions.hpp" +#include "shoc_test_data.hpp" +#include "physics/share/physics_constants.hpp" +#include "share/scream_types.hpp" +#include "share/util/scream_setup_random_test.hpp" + +#include "ekat/ekat_pack.hpp" +#include "ekat/util/ekat_arch.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" + +#include +#include +#include +#include + +namespace scream { +namespace shoc { +namespace unit_test { + +template +struct UnitWrap::UnitTest::TestShocAssumedPdf : public UnitWrap::UnitTest::Base { + + void run_property() + { + static constexpr Int shcol = 2; + static constexpr Int nlev = 5; + static constexpr auto nlevi = nlev + 1; + + // Tests for the top level subroutine + // shoc_assumed_pdf + + // Tests will start simple, and gradually add complexity to test + // the physics. + // NOTE: for this test we want exactly two columns. + + // TEST ONE + // No SGS variability test. Given inputs where there is a saturated + // profile but NO SGS variability in the scalar fluxes or variances + // (i.e. all second and third moment terms are zero), then verify that + // that cloud fraction is either 1 or 0 and that the SGS variability + // outputs are also zero everywhere. + + // Define input data + + // Note that the moisture and height profiles below represent that + // of the BOMEX case, but the temperatures are much colder, to encourage + // there to be points with ample cloud produced for this test. + + // Liquid water potential temperature [K] + static constexpr Real thetal[nlev] = {303, 300, 298, 298, 300}; + // Total water mixing ratio [kg/kg] + static constexpr Real qw[nlev] = {0.003, 0.004, 0.011, 0.016, 0.017}; + // Pressure [Pa] + static constexpr Real pres[nlev] = {70000, 80000, 85000, 90000, 100000}; + // Define the heights on the zt grid [m] + static constexpr Real zi_grid[nlevi] = {3000, 2000, 1500, 1000, 500, 0}; + + // All variances will be given zero or minimum threshold inputs + + // Define some reasonable bounds for output + static constexpr Real wqls_bound = 0.1; + static constexpr Real wthv_sec_bound = 10; + static constexpr Real shoc_ql2_bound = 0.1; + static constexpr Real shoc_ql_bound = 0.1; + + Real zt_grid[nlev]; + // Compute heights on midpoint grid + for(Int n = 0; n < nlev; ++n) { + zt_grid[n] = 0.5*(zi_grid[n]+zi_grid[n+1]); + } + + // Initialize data structure for bridging to F90 + ShocAssumedPdfData SDS(shcol, nlev, nlevi); + + // Load input data + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + + SDS.thetal[offset] = thetal[n]; + SDS.qw[offset] = qw[n]; + SDS.pres[offset] = pres[n]; + SDS.zt_grid[offset] = zt_grid[n]; + SDS.w_field[offset] = 0; + SDS.w_sec[offset] = 0.004; + } + + for(Int n = 0; n < nlevi; ++n) { + const auto offset = n + s * nlevi; + + SDS.thl_sec[offset] = 0; + SDS.qw_sec[offset] = 0; + SDS.wthl_sec[offset] = 0; + SDS.wqw_sec[offset] = 0; + SDS.qwthl_sec[offset] = 0; + SDS.w3[offset] = 0; + SDS.zi_grid[offset] = zi_grid[n]; + } + } + + // Check that the inputs make sense + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + + REQUIRE(SDS.qw[offset] > 0); + REQUIRE(SDS.thetal[offset] > 0); + REQUIRE(SDS.pres[offset] > 0); + REQUIRE(SDS.zt_grid[offset] > 0); + } + + for(Int n = 0; n < nlevi; ++n) { + const auto offset = n + s * nlev; + + REQUIRE(SDS.zi_grid[offset] >= 0); + } + } + + // Check that zt increase updward and pres decrease upward + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev - 1; ++n) { + const auto offset = n + s * nlev; + REQUIRE(SDS.zt_grid[offset + 1] - SDS.zt_grid[offset] < 0); + REQUIRE(SDS.pres[offset + 1] - SDS.pres[offset] > 0); + } + + // Check that zi increase upward + for(Int n = 0; n < nlevi - 1; ++n) { + const auto offset = n + s * nlevi; + REQUIRE(SDS.zi_grid[offset + 1] - SDS.zi_grid[offset] < 0); + } + + } + + // Test that the inputs are reasonable. + REQUIRE(SDS.nlevi - SDS.nlev == 1); + // For this test we want exactly two columns + REQUIRE(SDS.shcol == 2); + + // Call the C++ implementation. + shoc_assumed_pdf(SDS); + + // Verify the result + // Make sure cloud fraction is either 1 or 0 and all + // SGS terms are zero. + + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + + REQUIRE( (SDS.shoc_cldfrac[offset] == 0 || SDS.shoc_cldfrac[offset] == 1) ); + REQUIRE(SDS.wqls[offset] == 0); + REQUIRE(SDS.wthv_sec[offset] == 0); + REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); // Computation is not exactly BFB with 0 + REQUIRE(SDS.shoc_ql[offset] >= 0); + } + } + + // TEST TWO + // Add in Scalar fluxes. This should give us a nonzero + // buoyancy flux everywhere but other SGS terms should remain zero + + // We will assume turbulence with a uniform profile + + // Flux of liquid water [K m/s] + static constexpr Real wthl_sec = -0.03; + // Flux of total water [m/s kg/kg] + static constexpr Real wqw_sec = 0.0002; + + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlevi; ++n) { + const auto offset = n + s * nlevi; + + SDS.wthl_sec[offset] = wthl_sec; + SDS.wqw_sec[offset] = wqw_sec; + } + } + + // Call the C++ implementation. + shoc_assumed_pdf(SDS); + + // Verify the result + // Make sure cloud fraction is either 1 or 0 and all + // SGS terms are zero, EXCEPT wthv_sec. + + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + + REQUIRE( (SDS.shoc_cldfrac[offset] == 0 || SDS.shoc_cldfrac[offset] == 1) ); + REQUIRE(SDS.wqls[offset] == 0); + REQUIRE(SDS.wthv_sec[offset] != 0); + REQUIRE(std::abs(SDS.wthv_sec[offset] < wthv_sec_bound)); + REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); // Computation is not exactly BFB with 0 + REQUIRE(SDS.shoc_ql[offset] >= 0); + REQUIRE(SDS.shoc_ql[offset] < shoc_ql_bound); + } + } + + // TEST THREE and FOUR + // Add in Scalar variances, and POSITIVE vertical velocity skewness test. + + // Add strong scalar variances as such that will produce cloud at every level. + + // For the first column feed in zero vertical velocity skewness. + // For the second column feed in large veriticle velocity skewss. + // Verify that for points where cloud fraction was < 0.5 in the first column, + // that cloud fraction then vice versa for points with cloud fraction > 0.5. + + // Thetal variance [K^2] + static constexpr Real thl_sec = 2; + // total water variance [kg^2/kg^2] + static constexpr Real qw_sec = 0.0002; + // Temperature and total water covariance [K kg/kg] + static constexpr Real qwthl_sec = 1e-5; + // Vertical velocity variance [m2/s2] + static constexpr Real w_sec = 0.2; + + // Load input data + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + + SDS.w_sec[offset] = w_sec; + } + + for(Int n = 0; n < nlevi; ++n) { + const auto offset = n + s * nlevi; + + SDS.thl_sec[offset] = thl_sec; + SDS.qw_sec[offset] = qw_sec; + SDS.qwthl_sec[offset] = qwthl_sec; + SDS.w3[offset] = s*1.0; + } + } + + // Call the C++ implementation. + shoc_assumed_pdf(SDS); + + // Check the result + + // With such a turbulence and scalar profile, this should have + // encouraged cloud everywhere. Verify that this is true. + + // Also verify that output lies within some reasonable bounds. + + // Then verify vertical velocity skewness info + + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlevi; ++n) { + const auto offset = n + s * nlevi; + const auto offsets = n + (s+1) * nlevi; + if (s < shcol-1){ + // Verify input w3 is greater in subsequent columns + REQUIRE(SDS.w3[offsets] > SDS.w3[offset]); + } + } + + // Verify output falls within reasonable bounds. For this positive + // vertical velocity skewness test and with the give inputs there + // should be cloud everywhere and all flux terms should be positive. + + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + const auto offsets = n + (s+1) * nlev; + + REQUIRE( (SDS.shoc_cldfrac[offset] > 0 || SDS.shoc_cldfrac[offset] < 1) ); + REQUIRE(SDS.wqls[offset] > 0); + REQUIRE(SDS.wthv_sec[offset] > 0); + REQUIRE(SDS.shoc_ql2[offset] > 0); + REQUIRE(SDS.shoc_ql[offset] > 0); + + REQUIRE(SDS.wqls[offset] < 0.1); + REQUIRE(SDS.wthv_sec[offset] < wthv_sec_bound); + REQUIRE(SDS.shoc_ql2[offset] < shoc_ql2_bound); + REQUIRE(SDS.shoc_ql[offset] < shoc_ql_bound); + + // Now verify that the relationships in a strongly positive vertical + // velocity flux regime hold true, relative to a symmetric vertical + // velocity regime. + if (s < shcol-1){ + + if (SDS.shoc_cldfrac[offset] < 0.5){ + REQUIRE(SDS.shoc_cldfrac[offsets] < SDS.shoc_cldfrac[offset]); + } + else if (SDS.shoc_cldfrac[offset] > 0.5){ + REQUIRE(SDS.shoc_cldfrac[offsets] > SDS.shoc_cldfrac[offset]); + } + + // In addition, in a positive skewness environment, the following + // should also be true + + // Grid mean liquid water decreased + REQUIRE(SDS.shoc_ql[offsets] < SDS.shoc_ql[offset]); + // liquid water flux increased + REQUIRE(SDS.wqls[offsets] > SDS.wqls[offset]); + // buoyancy flux increased + REQUIRE(SDS.wthv_sec[offsets] > SDS.wthv_sec[offset]); + // liquid water variance increased + REQUIRE(SDS.shoc_ql2[offsets] > SDS.shoc_ql2[offset]); + } + } + } + + // TEST FIVE + // Negative vertical velocity skewness. + + // Using same input as the above test, feed one column with zero skeweness + // and another test with negative vertical velocity skewness and verify + // result is physical. + + // Load input data + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlevi; ++n) { + const auto offset = n + s * nlevi; + + SDS.w3[offset] = s*-1.0; + } + } + + // Call the C++ implementation. + shoc_assumed_pdf(SDS); + + // Check the result + + // Verify that output lies within some reasonable bounds. + + // Then verify vertical velocity skewness info + + for(Int s = 0; s < shcol; ++s) { + for(Int n = 0; n < nlevi; ++n) { + const auto offset = n + s * nlevi; + const auto offsets = n + (s+1) * nlevi; + if (s < shcol-1){ + // Verify input w3 is greater in subsequent columns + REQUIRE(SDS.w3[offsets] < SDS.w3[offset]); + } + } + + // Verify output falls within reasonable bounds + // For this negative vertical velocity test some variables + // will be expected to be less than zero. + + for(Int n = 0; n < nlev; ++n) { + const auto offset = n + s * nlev; + const auto offsets = n + (s+1) * nlev; + + REQUIRE( (SDS.shoc_cldfrac[offset] >= 0 || SDS.shoc_cldfrac[offset] < 1) ); + REQUIRE(SDS.shoc_ql2[offset] > 0); + REQUIRE(SDS.shoc_ql[offset] >= 0); + + REQUIRE(std::abs(SDS.wqls[offset] ) < wqls_bound); + REQUIRE(std::abs(SDS.wthv_sec[offset]) < wthv_sec_bound); + REQUIRE(SDS.shoc_ql2[offset] < shoc_ql2_bound); + REQUIRE(SDS.shoc_ql[offset] < shoc_ql_bound); + + // Now verify that the relationships in a strongly negative vertical + // velocity flux regime hold true, relative to a symmetric vertical + // velocity regime. + if (s < shcol-1){ + + if (SDS.shoc_cldfrac[offset] < 0.5){ + REQUIRE(SDS.shoc_cldfrac[offsets] < SDS.shoc_cldfrac[offset]); + } + else if (SDS.shoc_cldfrac[offset] > 0.5){ + REQUIRE(SDS.shoc_cldfrac[offsets] > SDS.shoc_cldfrac[offset]); + } + + // In addition, in a positive skewness environment, the following + // should also be true + + // Grid mean liquid water decreased + REQUIRE(SDS.shoc_ql[offsets] < SDS.shoc_ql[offset]); + // if cloud present, verify liquid water and buoyancy flux is negative + if (SDS.shoc_ql[offsets] > 0){ + REQUIRE(SDS.wqls[offsets] < SDS.wqls[offset]); + REQUIRE(SDS.wqls[offsets] < 0); + + REQUIRE(SDS.wthv_sec[offsets] < SDS.wthv_sec[offset]); + REQUIRE(SDS.wthv_sec[offsets] < 0); + } + + // liquid water variance increased + REQUIRE(SDS.shoc_ql2[offsets] > SDS.shoc_ql2[offset]); + } + } + } + + } + + void run_bfb() + { + auto engine = Base::get_engine(); + + ShocAssumedPdfData SDS_baseline[] = { + // shcol, nlev, nlevi + ShocAssumedPdfData(10, 71, 72), + ShocAssumedPdfData(10, 12, 13), + ShocAssumedPdfData(7, 16, 17), + ShocAssumedPdfData(2, 7, 8), + }; + + // Generate random input data + for (auto& d : SDS_baseline) { + d.randomize(engine, { + {d.thetal, {500, 700}}, + {d.zi_grid, {0, 100}}, + }); + } + + // Create copies of data for use by cxx. Needs to happen before reads so that + // inout data is in original state + ShocAssumedPdfData SDS_cxx[] = { + ShocAssumedPdfData(SDS_baseline[0]), + ShocAssumedPdfData(SDS_baseline[1]), + ShocAssumedPdfData(SDS_baseline[2]), + ShocAssumedPdfData(SDS_baseline[3]), + }; + + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(ShocAssumedPdfData); + + // Assume all data is in C layout + + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } + } + + // Get data from cxx + for (auto& d : SDS_cxx) { + shoc_assumed_pdf(d); + } + + // Verify BFB results, all data should be in C layout + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + for (Int i = 0; i < num_runs; ++i) { + ShocAssumedPdfData& d_baseline = SDS_baseline[i]; + ShocAssumedPdfData& d_cxx = SDS_cxx[i]; + for (Int k = 0; k < d_baseline.total(d_baseline.wqls); ++k) { + REQUIRE(d_baseline.shoc_cldfrac[k] == d_cxx.shoc_cldfrac[k]); + REQUIRE(d_baseline.shoc_ql[k] == d_cxx.shoc_ql[k]); + REQUIRE(d_baseline.wqls[k] == d_cxx.wqls[k]); + REQUIRE(d_baseline.wthv_sec[k] == d_cxx.wthv_sec[k]); + REQUIRE(d_baseline.shoc_ql2[k] == d_cxx.shoc_ql2[k]); + } + } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } + } + } +}; + +} // namespace unit_test +} // namespace shoc +} // namespace scream + +namespace { + +TEST_CASE("shoc_assumed_pdf_property", "shoc") +{ + using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocAssumedPdf; + + TestStruct().run_property(); +} + +TEST_CASE("shoc_assumed_pdf_bfb", "shoc") +{ + using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocAssumedPdf; + + TestStruct().run_bfb(); +} + +} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_brunt_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_brunt_length_tests.cpp index 3018d0e5aa6..21112e0d0a2 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_brunt_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_brunt_length_tests.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -22,9 +22,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestCompBruntShocLength { +struct UnitWrap::UnitTest::TestCompBruntShocLength : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 5; @@ -91,10 +91,7 @@ struct UnitWrap::UnitTest::TestCompBruntShocLength { } // Call the C++ implementation. - SDS.transpose(); - // expects data in fortran layout - compute_brunt_shoc_length_f(SDS.nlev,SDS.nlevi,SDS.shcol,SDS.dz_zt,SDS.thv,SDS.thv_zi,SDS.brunt); - SDS.transpose(); + compute_brunt_shoc_length(SDS); // Check the results for(Int s = 0; s < shcol; ++s) { @@ -124,11 +121,11 @@ struct UnitWrap::UnitTest::TestCompBruntShocLength { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ComputeBruntShocLengthData SDS_f90[] = { + ComputeBruntShocLengthData SDS_baseline[] = { // shcol, nlev, nlevi ComputeBruntShocLengthData(10, 71, 72), ComputeBruntShocLengthData(10, 12, 13), @@ -137,45 +134,49 @@ struct UnitWrap::UnitTest::TestCompBruntShocLength { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ComputeBruntShocLengthData SDS_cxx[] = { - ComputeBruntShocLengthData(SDS_f90[0]), - ComputeBruntShocLengthData(SDS_f90[1]), - ComputeBruntShocLengthData(SDS_f90[2]), - ComputeBruntShocLengthData(SDS_f90[3]), + ComputeBruntShocLengthData(SDS_baseline[0]), + ComputeBruntShocLengthData(SDS_baseline[1]), + ComputeBruntShocLengthData(SDS_baseline[2]), + ComputeBruntShocLengthData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(ComputeBruntShocLengthData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - compute_brunt_shoc_length(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - compute_brunt_shoc_length_f(d.nlev,d.nlevi,d.shcol,d.dz_zt,d.thv,d.thv_zi,d.brunt); - d.transpose(); + compute_brunt_shoc_length(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(ComputeBruntShocLengthData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ComputeBruntShocLengthData& d_f90 = SDS_f90[i]; + ComputeBruntShocLengthData& d_baseline = SDS_baseline[i]; ComputeBruntShocLengthData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.brunt); ++k) { - REQUIRE(d_f90.brunt[k] == d_cxx.brunt[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.brunt); ++k) { + REQUIRE(d_baseline.brunt[k] == d_cxx.brunt[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -190,14 +191,14 @@ TEST_CASE("shoc_brunt_length_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestCompBruntShocLength; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_brunt_length_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestCompBruntShocLength; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_check_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_check_length_tests.cpp index bd023e2ed79..404176bfe3a 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_check_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_check_length_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestCheckShocLength { +struct UnitWrap::UnitTest::TestCheckShocLength : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Real maxlen = scream::shoc::Constants::maxlen; static constexpr Real minlen = scream::shoc::Constants::minlen; @@ -76,10 +76,7 @@ struct UnitWrap::UnitTest::TestCheckShocLength { } // Call the C++ implementation. - SDS.transpose(); - // expects data in fortran layout - check_length_scale_shoc_length_f(SDS.nlev,SDS.shcol,SDS.host_dx,SDS.host_dy,SDS.shoc_mix); - SDS.transpose(); + check_length_scale_shoc_length(SDS); // Check the results for(Int s = 0; s < shcol; ++s) { @@ -94,11 +91,11 @@ struct UnitWrap::UnitTest::TestCheckShocLength { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - CheckLengthScaleShocLengthData SDS_f90[] = { + CheckLengthScaleShocLengthData SDS_baseline[] = { // shcol, nlev CheckLengthScaleShocLengthData(10, 71), CheckLengthScaleShocLengthData(10, 12), @@ -107,45 +104,49 @@ struct UnitWrap::UnitTest::TestCheckShocLength { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state CheckLengthScaleShocLengthData SDS_cxx[] = { - CheckLengthScaleShocLengthData(SDS_f90[0]), - CheckLengthScaleShocLengthData(SDS_f90[1]), - CheckLengthScaleShocLengthData(SDS_f90[2]), - CheckLengthScaleShocLengthData(SDS_f90[3]), + CheckLengthScaleShocLengthData(SDS_baseline[0]), + CheckLengthScaleShocLengthData(SDS_baseline[1]), + CheckLengthScaleShocLengthData(SDS_baseline[2]), + CheckLengthScaleShocLengthData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(CheckLengthScaleShocLengthData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - check_length_scale_shoc_length(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - check_length_scale_shoc_length_f(d.nlev,d.shcol,d.host_dx,d.host_dy,d.shoc_mix); - d.transpose(); + check_length_scale_shoc_length(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(CheckLengthScaleShocLengthData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - CheckLengthScaleShocLengthData& d_f90 = SDS_f90[i]; + CheckLengthScaleShocLengthData& d_baseline = SDS_baseline[i]; CheckLengthScaleShocLengthData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.shoc_mix); ++k) { - REQUIRE(d_f90.shoc_mix[k] == d_cxx.shoc_mix[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.shoc_mix); ++k) { + REQUIRE(d_baseline.shoc_mix[k] == d_cxx.shoc_mix[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -160,14 +161,14 @@ TEST_CASE("shoc_check_length_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestCheckShocLength; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_check_length_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestCheckShocLength; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_check_tke_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_check_tke_tests.cpp index 75581e2283c..5aa189cfb5d 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_check_tke_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_check_tke_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocCheckTke { +struct UnitWrap::UnitTest::TestShocCheckTke : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Real mintke = scream::shoc::Constants::mintke; static constexpr Int shcol = 2; @@ -53,10 +53,7 @@ struct UnitWrap::UnitTest::TestShocCheckTke { REQUIRE((SDS.shcol > 0 && SDS.nlev > 0)); // Call the C++ implementation. - SDS.transpose(); - // expects data in fortran layout - check_tke_f(SDS.nlev, SDS.shcol, SDS.tke); - SDS.transpose(); + check_tke(SDS); // Check the result against the input values for(Int s = 0; s < shcol; ++s) { @@ -76,11 +73,11 @@ struct UnitWrap::UnitTest::TestShocCheckTke { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - CheckTkeData SDS_f90[] = { + CheckTkeData SDS_baseline[] = { // shcol, nlev CheckTkeData(10, 71), CheckTkeData(10, 12), @@ -89,45 +86,49 @@ struct UnitWrap::UnitTest::TestShocCheckTke { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state CheckTkeData SDS_cxx[] = { - CheckTkeData(SDS_f90[0]), - CheckTkeData(SDS_f90[1]), - CheckTkeData(SDS_f90[2]), - CheckTkeData(SDS_f90[3]), + CheckTkeData(SDS_baseline[0]), + CheckTkeData(SDS_baseline[1]), + CheckTkeData(SDS_baseline[2]), + CheckTkeData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(CheckTkeData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - check_tke(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - check_tke_f(d.nlev, d.shcol, d.tke); - d.transpose(); + check_tke(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(CheckTkeData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - CheckTkeData& d_f90 = SDS_f90[i]; + CheckTkeData& d_baseline = SDS_baseline[i]; CheckTkeData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.tke); ++k) { - REQUIRE(d_f90.tke[k] == d_cxx.tke[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.tke); ++k) { + REQUIRE(d_baseline.tke[k] == d_cxx.tke[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } @@ -143,14 +144,14 @@ TEST_CASE("shoc_check_tke_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocCheckTke; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_check_tke_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocCheckTke; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } diff --git a/components/eamxx/src/physics/shoc/tests/shoc_clip_third_moms_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_clip_third_moms_tests.cpp index 28bef1aae25..4f254e65cc5 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_clip_third_moms_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_clip_third_moms_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestClipThirdMoms { +struct UnitWrap::UnitTest::TestClipThirdMoms : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlevi = 5; @@ -83,10 +83,7 @@ struct UnitWrap::UnitTest::TestClipThirdMoms { } // Call the C++ implementation. - SDS.transpose(); - // expects data in fortran layout - clipping_diag_third_shoc_moments_f(SDS.nlevi,SDS.shcol,SDS.w_sec_zi,SDS.w3); - SDS.transpose(); + clipping_diag_third_shoc_moments(SDS); // Check the result // For large values of w3, verify that the result has been reduced @@ -103,11 +100,11 @@ struct UnitWrap::UnitTest::TestClipThirdMoms { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ClippingDiagThirdShocMomentsData SDS_f90[] = { + ClippingDiagThirdShocMomentsData SDS_baseline[] = { // shcol, nlevi ClippingDiagThirdShocMomentsData(10, 72), ClippingDiagThirdShocMomentsData(10, 13), @@ -116,45 +113,49 @@ struct UnitWrap::UnitTest::TestClipThirdMoms { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ClippingDiagThirdShocMomentsData SDS_cxx[] = { - ClippingDiagThirdShocMomentsData(SDS_f90[0]), - ClippingDiagThirdShocMomentsData(SDS_f90[1]), - ClippingDiagThirdShocMomentsData(SDS_f90[2]), - ClippingDiagThirdShocMomentsData(SDS_f90[3]), + ClippingDiagThirdShocMomentsData(SDS_baseline[0]), + ClippingDiagThirdShocMomentsData(SDS_baseline[1]), + ClippingDiagThirdShocMomentsData(SDS_baseline[2]), + ClippingDiagThirdShocMomentsData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(ClippingDiagThirdShocMomentsData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - clipping_diag_third_shoc_moments(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - clipping_diag_third_shoc_moments_f(d.nlevi,d.shcol,d.w_sec_zi,d.w3); - d.transpose(); + clipping_diag_third_shoc_moments(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(ClippingDiagThirdShocMomentsData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ClippingDiagThirdShocMomentsData& d_f90 = SDS_f90[i]; + ClippingDiagThirdShocMomentsData& d_baseline = SDS_baseline[i]; ClippingDiagThirdShocMomentsData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.w3); ++k) { - REQUIRE(d_f90.w3[k] == d_cxx.w3[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.w3); ++k) { + REQUIRE(d_baseline.w3[k] == d_cxx.w3[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -169,14 +170,14 @@ TEST_CASE("shoc_clip_third_moms_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestClipThirdMoms; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_clip_third_moms_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestClipThirdMoms; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_compute_diag_third_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_compute_diag_third_tests.cpp index d63bf34bb50..fc52c64550b 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_compute_diag_third_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_compute_diag_third_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocCompDiagThird { +struct UnitWrap::UnitTest::TestShocCompDiagThird : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 5; @@ -146,14 +146,7 @@ struct UnitWrap::UnitTest::TestShocCompDiagThird { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - compute_diag_third_shoc_moment_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.w_sec,SDS.thl_sec, - SDS.wthl_sec,SDS.tke,SDS.dz_zt, - SDS.dz_zi,SDS.isotropy_zi, - SDS.brunt_zi,SDS.w_sec_zi,SDS.thetal_zi, - SDS.w3); - SDS.transpose(); + compute_diag_third_shoc_moment(SDS); // Check the result @@ -193,11 +186,11 @@ struct UnitWrap::UnitTest::TestShocCompDiagThird { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ComputeDiagThirdShocMomentData SDS_f90[] = { + ComputeDiagThirdShocMomentData SDS_baseline[] = { // shcol, nlev, nlevi ComputeDiagThirdShocMomentData(10, 71, 72), ComputeDiagThirdShocMomentData(10, 12, 13), @@ -206,49 +199,49 @@ struct UnitWrap::UnitTest::TestShocCompDiagThird { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ComputeDiagThirdShocMomentData SDS_cxx[] = { - ComputeDiagThirdShocMomentData(SDS_f90[0]), - ComputeDiagThirdShocMomentData(SDS_f90[1]), - ComputeDiagThirdShocMomentData(SDS_f90[2]), - ComputeDiagThirdShocMomentData(SDS_f90[3]), + ComputeDiagThirdShocMomentData(SDS_baseline[0]), + ComputeDiagThirdShocMomentData(SDS_baseline[1]), + ComputeDiagThirdShocMomentData(SDS_baseline[2]), + ComputeDiagThirdShocMomentData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(ComputeDiagThirdShocMomentData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - compute_diag_third_shoc_moment(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - compute_diag_third_shoc_moment_f(d.shcol,d.nlev,d.nlevi,d.w_sec,d.thl_sec, - d.wthl_sec,d.tke,d.dz_zt, - d.dz_zi,d.isotropy_zi, - d.brunt_zi,d.w_sec_zi,d.thetal_zi, - d.w3); - d.transpose(); + compute_diag_third_shoc_moment(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(ComputeDiagThirdShocMomentData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ComputeDiagThirdShocMomentData& d_f90 = SDS_f90[i]; + ComputeDiagThirdShocMomentData& d_baseline = SDS_baseline[i]; ComputeDiagThirdShocMomentData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.w3); ++k) { - REQUIRE(d_f90.w3[k] == d_cxx.w3[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.w3); ++k) { + REQUIRE(d_baseline.w3[k] == d_cxx.w3[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -263,14 +256,14 @@ TEST_CASE("shoc_comp_diag_third_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocCompDiagThird; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_comp_diag_third_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocCompDiagThird; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp index 27452cdcf70..0566d12d932 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestComputeShocTemp { +struct UnitWrap::UnitTest::TestComputeShocTemp : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 1; static constexpr Int nlev = 3; @@ -68,9 +68,7 @@ struct UnitWrap::UnitTest::TestComputeShocTemp { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - compute_shoc_temperature_f(SDS.shcol, SDS.nlev, SDS.thetal, SDS.ql, SDS.inv_exner, SDS.tabs); - SDS.transpose(); // go back to C layout + compute_shoc_temperature(SDS); // Require that absolute temperature is equal to thetal for(Int s = 0; s < shcol; ++s) { @@ -116,9 +114,7 @@ struct UnitWrap::UnitTest::TestComputeShocTemp { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - compute_shoc_temperature_f(SDS.shcol, SDS.nlev, SDS.thetal, SDS.ql, SDS.inv_exner, SDS.tabs); - SDS.transpose(); // go back to C layout + compute_shoc_temperature(SDS); // Require that absolute temperature is greather than thetal for(Int s = 0; s < shcol; ++s) { @@ -177,9 +173,7 @@ struct UnitWrap::UnitTest::TestComputeShocTemp { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - compute_shoc_temperature_f(SDS.shcol, SDS.nlev, SDS.thetal, SDS.ql, SDS.inv_exner, SDS.tabs); - SDS.transpose(); // go back to C layout + compute_shoc_temperature(SDS); // Require that absolute temperature be less than thetal for(Int s = 0; s < shcol; ++s) { @@ -202,11 +196,11 @@ struct UnitWrap::UnitTest::TestComputeShocTemp { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ComputeShocTempData f90_data[] = { + ComputeShocTempData baseline_data[] = { // shcol, nlev ComputeShocTempData(10, 71), ComputeShocTempData(10, 12), @@ -215,44 +209,48 @@ struct UnitWrap::UnitTest::TestComputeShocTemp { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ComputeShocTempData cxx_data[] = { - ComputeShocTempData(f90_data[0]), - ComputeShocTempData(f90_data[1]), - ComputeShocTempData(f90_data[2]), - ComputeShocTempData(f90_data[3]), + ComputeShocTempData(baseline_data[0]), + ComputeShocTempData(baseline_data[1]), + ComputeShocTempData(baseline_data[2]), + ComputeShocTempData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - compute_shoc_temperature(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - compute_shoc_temperature_f(d.shcol, d.nlev, d.thetal, d.ql, d.inv_exner, d.tabs); - d.transpose(); // go back to C layout + compute_shoc_temperature(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(ComputeShocTempData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(ComputeShocTempData); for (Int i = 0; i < num_runs; ++i) { - ComputeShocTempData& d_f90 = f90_data[i]; + ComputeShocTempData& d_baseline = baseline_data[i]; ComputeShocTempData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.tabs); ++k) { - REQUIRE(d_f90.tabs[k] == d_cxx.tabs[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.tabs); ++k) { + REQUIRE(d_baseline.tabs[k] == d_cxx.tabs[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } }; @@ -267,14 +265,14 @@ TEST_CASE("shoc_compute_shoc_temperature_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestComputeShocTemp; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_compute_shoc_temperature_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestComputeShocTemp; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_vapor_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_vapor_tests.cpp index afd184e1823..1e0d4134204 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_vapor_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_vapor_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,9 +14,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestComputeShocVapor { +struct UnitWrap::UnitTest::TestComputeShocVapor : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 5; @@ -63,9 +63,7 @@ struct UnitWrap::UnitTest::TestComputeShocVapor { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - compute_shoc_vapor_f(SDS.shcol, SDS.nlev, SDS.qw, SDS.ql, SDS.qv); - SDS.transpose(); // go back to C layout + compute_shoc_vapor(SDS); // Verify the result for(Int s = 0; s < shcol; ++s) { @@ -88,11 +86,11 @@ struct UnitWrap::UnitTest::TestComputeShocVapor { } // run_property - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ComputeShocVaporData f90_data[] = { + ComputeShocVaporData baseline_data[] = { // shcol, nlev ComputeShocVaporData(10, 71), ComputeShocVaporData(10, 12), @@ -101,44 +99,48 @@ struct UnitWrap::UnitTest::TestComputeShocVapor { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ComputeShocVaporData cxx_data[] = { - ComputeShocVaporData(f90_data[0]), - ComputeShocVaporData(f90_data[1]), - ComputeShocVaporData(f90_data[2]), - ComputeShocVaporData(f90_data[3]), + ComputeShocVaporData(baseline_data[0]), + ComputeShocVaporData(baseline_data[1]), + ComputeShocVaporData(baseline_data[2]), + ComputeShocVaporData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - compute_shoc_vapor(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - compute_shoc_vapor_f(d.shcol, d.nlev, d.qw, d.ql, d.qv); - d.transpose(); // go back to C layout + compute_shoc_vapor(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(ComputeShocVaporData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(ComputeShocVaporData); for (Int i = 0; i < num_runs; ++i) { - ComputeShocVaporData& d_f90 = f90_data[i]; + ComputeShocVaporData& d_baseline = baseline_data[i]; ComputeShocVaporData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.qv); ++k) { - REQUIRE(d_f90.qv[k] == d_cxx.qv[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.qv); ++k) { + REQUIRE(d_baseline.qv[k] == d_cxx.qv[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb }; @@ -153,14 +155,14 @@ TEST_CASE("compute_shoc_vapor_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestComputeShocVapor; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("compute_shoc_vapor_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestComputeShocVapor; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_obklen_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_obklen_tests.cpp index 930faf31d6b..47e4f72ed99 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_obklen_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_obklen_tests.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "physics/share/physics_constants.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocDiagObklen { +struct UnitWrap::UnitTest::TestShocDiagObklen : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 5; @@ -84,12 +84,7 @@ struct UnitWrap::UnitTest::TestShocDiagObklen { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - shoc_diag_obklen_f(SDS.shcol, SDS.uw_sfc, SDS.vw_sfc, SDS.wthl_sfc, SDS.wqw_sfc, - SDS.thl_sfc, SDS.cldliq_sfc, SDS.qv_sfc, SDS.ustar, SDS.kbfs, - SDS.obklen); - SDS.transpose(); + shoc_diag_obklen(SDS); // Check the result @@ -154,12 +149,7 @@ struct UnitWrap::UnitTest::TestShocDiagObklen { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - shoc_diag_obklen_f(SDS.shcol, SDS.uw_sfc, SDS.vw_sfc, SDS.wthl_sfc, SDS.wqw_sfc, - SDS.thl_sfc, SDS.cldliq_sfc, SDS.qv_sfc, SDS.ustar, SDS.kbfs, - SDS.obklen); - SDS.transpose(); + shoc_diag_obklen(SDS); // Verify that DIMENSIONLESS obukhov length decreases as columns // increases due to the increasing surface fluxes @@ -172,11 +162,11 @@ struct UnitWrap::UnitTest::TestShocDiagObklen { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ShocDiagObklenData SDS_f90[] = { + ShocDiagObklenData SDS_baseline[] = { // shcol ShocDiagObklenData(12), ShocDiagObklenData(10), @@ -185,49 +175,51 @@ struct UnitWrap::UnitTest::TestShocDiagObklen { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ShocDiagObklenData SDS_cxx[] = { - ShocDiagObklenData(SDS_f90[0]), - ShocDiagObklenData(SDS_f90[1]), - ShocDiagObklenData(SDS_f90[2]), - ShocDiagObklenData(SDS_f90[3]) + ShocDiagObklenData(SDS_baseline[0]), + ShocDiagObklenData(SDS_baseline[1]), + ShocDiagObklenData(SDS_baseline[2]), + ShocDiagObklenData(SDS_baseline[3]) }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(ShocDiagObklenData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - shoc_diag_obklen(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - shoc_diag_obklen_f(d.shcol, d.uw_sfc, d.vw_sfc, d.wthl_sfc, d.wqw_sfc, - d.thl_sfc, d.cldliq_sfc, d.qv_sfc, d.ustar, d.kbfs, - d.obklen); - d.transpose(); + shoc_diag_obklen(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(ShocDiagObklenData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ShocDiagObklenData& d_f90 = SDS_f90[i]; + ShocDiagObklenData& d_baseline = SDS_baseline[i]; ShocDiagObklenData& d_cxx = SDS_cxx[i]; - for (Int s = 0; s < d_f90.shcol; ++s) { - REQUIRE(d_f90.ustar[s] == d_cxx.ustar[s]); - REQUIRE(d_f90.kbfs[s] == d_cxx.kbfs[s]); - REQUIRE(d_f90.obklen[s] == d_cxx.obklen[s]); + for (Int s = 0; s < d_baseline.shcol; ++s) { + REQUIRE(d_baseline.ustar[s] == d_cxx.ustar[s]); + REQUIRE(d_baseline.kbfs[s] == d_cxx.kbfs[s]); + REQUIRE(d_baseline.obklen[s] == d_cxx.obklen[s]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -242,14 +234,14 @@ TEST_CASE("shoc_diag_obklen_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocDiagObklen; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_diag_obklen_length_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocDiagObklen; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_srf_test.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_srf_test.cpp index 25f56139d77..95abfefab98 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_srf_test.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_srf_test.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -22,9 +22,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestSecondMomSrf { +struct UnitWrap::UnitTest::TestSecondMomSrf : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { // Property test for the SHOC subroutine: // diag_second_moments_srf @@ -52,7 +52,7 @@ struct UnitWrap::UnitTest::TestSecondMomSrf { } // Call the C++ implementation - shoc_diag_second_moments_srf_f(SDS.shcol, SDS.wthl_sfc, SDS.uw_sfc, SDS.vw_sfc, SDS.ustar2, SDS.wstar); + diag_second_moments_srf(SDS); // Verify the output for (Int s = 0; s < shcol; ++s){ @@ -82,12 +82,12 @@ struct UnitWrap::UnitTest::TestSecondMomSrf { } - static void run_bfb() + void run_bfb() { #if 0 - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - SHOCSecondMomentSrfData mom_srf_data_f90[] = { + SHOCSecondMomentSrfData mom_srf_data_baseline[] = { // shcol SHOCSecondMomentSrfData(36), SHOCSecondMomentSrfData(72), @@ -95,35 +95,40 @@ struct UnitWrap::UnitTest::TestSecondMomSrf { SHOCSecondMomentSrfData(256), }; - for (auto& d : mom_srf_data_f90) { + for (auto& d : mom_srf_data_baseline) { d.randomize(engine, { {d.wthl, {-1, 1}} }); } SHOCSecondMomentSrfData mom_srf_data_cxx[] = { - SHOCSecondMomentSrfData(mom_srf_data_f90[0]), - SHOCSecondMomentSrfData(mom_srf_data_f90[1]), - SHOCSecondMomentSrfData(mom_srf_data_f90[2]), - SHOCSecondMomentSrfData(mom_srf_data_f90[3]), + SHOCSecondMomentSrfData(mom_srf_data_baseline[0]), + SHOCSecondMomentSrfData(mom_srf_data_baseline[1]), + SHOCSecondMomentSrfData(mom_srf_data_baseline[2]), + SHOCSecondMomentSrfData(mom_srf_data_baseline[3]), }; - for (auto& d : mom_srf_data_f90) { + for (auto& d : mom_srf_data_baseline) { // expects data in C layout shoc_diag_second_moments_srf(d); } for (auto& d : mom_srf_data_cxx) { - shoc_diag_second_moments_srf_f(d.shcol, d.wthl_sfc, d.uw_sfc, d.vw_sfc, d.ustar2, d.wstar); + shoc_diag_second_moments_srf(d); } - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(mom_srf_data_f90) / sizeof(SHOCSecondMomentSrfData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(mom_srf_data_baseline) / sizeof(SHOCSecondMomentSrfData); for (Int i = 0; i < num_runs; ++i) { Int shcol = mom_srf_data_cxx[i].shcol; for (Int k = 0; k < shcol; ++k) { - REQUIRE(mom_srf_data_f90[i].ustar2[k] == mom_srf_data_cxx[i].ustar2[k]); - REQUIRE(mom_srf_data_f90[i].wstar[k] == mom_srf_data_cxx[i].wstar[k]); + REQUIRE(mom_srf_data_baseline[i].ustar2[k] == mom_srf_data_cxx[i].ustar2[k]); + REQUIRE(mom_srf_data_baseline[i].wstar[k] == mom_srf_data_cxx[i].wstar[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + cxx_data[i].write(Base::m_fid); + } } #endif } @@ -139,13 +144,13 @@ namespace { TEST_CASE("shoc_second_moments_srf_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestSecondMomSrf; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_second_moments_srf_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestSecondMomSrf; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_ubycond_test.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_ubycond_test.cpp index d4dbd3e7b71..06f8ccdd45a 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_ubycond_test.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_ubycond_test.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -23,9 +23,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestSecondMomUbycond { +struct UnitWrap::UnitTest::TestSecondMomUbycond : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { // Property test for SHOC subroutine: // diag_second_moments_ubycond @@ -43,8 +43,7 @@ struct UnitWrap::UnitTest::TestSecondMomUbycond { REQUIRE(shcol > 0); // Call the C++ implementation - shoc_diag_second_moments_ubycond_f(SDS.shcol, SDS.thl_sec, SDS.qw_sec, SDS.qwthl_sec, SDS.wthl_sec, - SDS.wqw_sec, SDS.uw_sec, SDS.vw_sec, SDS.wtke_sec); + diag_second_moments_ubycond(SDS); // Verify the result // all output should be zero. @@ -61,9 +60,9 @@ struct UnitWrap::UnitTest::TestSecondMomUbycond { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); DiagSecondMomentsUbycondData uby_fortran[] = { // shcol @@ -79,7 +78,7 @@ struct UnitWrap::UnitTest::TestSecondMomUbycond { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state DiagSecondMomentsUbycondData uby_cxx[num_runs] = { DiagSecondMomentsUbycondData(uby_fortran[0]), @@ -88,16 +87,18 @@ struct UnitWrap::UnitTest::TestSecondMomUbycond { DiagSecondMomentsUbycondData(uby_fortran[3]), }; - // Get data from fortran - for (auto& d : uby_fortran) { - diag_second_moments_ubycond(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : uby_fortran) { + d.read(Base::m_fid); + } } for (auto& d : uby_cxx) { - shoc_diag_second_moments_ubycond_f(d.shcol, d.thl_sec, d.qw_sec, d.qwthl_sec, d.wthl_sec, d.wqw_sec, d.uw_sec, d.vw_sec, d.wtke_sec); + diag_second_moments_ubycond(d); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { const Int shcol = uby_cxx[i].shcol; for (Int k = 0; k < shcol; ++k) { @@ -111,6 +112,11 @@ struct UnitWrap::UnitTest::TestSecondMomUbycond { REQUIRE(uby_fortran[i].wtke_sec[k] == uby_cxx[i].wtke_sec[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + uby_cxx[i].write(Base::m_fid); + } } } @@ -126,14 +132,14 @@ TEST_CASE("second_mom_uby_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestSecondMomUbycond; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("second_mom_uby_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestSecondMomUbycond; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_lbycond_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_lbycond_tests.cpp index 1f1f3df8e5c..9680984a446 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_lbycond_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_lbycond_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,9 +14,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestDiagSecondMomentsLbycond { +struct UnitWrap::UnitTest::TestDiagSecondMomentsLbycond : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { // Property tests for the SHOC function // diag_second_moments_lbycond @@ -76,10 +76,7 @@ struct UnitWrap::UnitTest::TestDiagSecondMomentsLbycond { } // Call the C++ implementation - diag_second_moments_lbycond_f(SDS.shcol, SDS.wthl_sfc, SDS.wqw_sfc, SDS.uw_sfc, - SDS.vw_sfc, SDS.ustar2, SDS.wstar, SDS.wthl_sec, - SDS.wqw_sec, SDS.uw_sec, SDS.vw_sec, SDS.wtke_sec, - SDS.thl_sec, SDS.qw_sec, SDS.qwthl_sec); + diag_second_moments_lbycond(SDS); // Verify output is as expected for (Int s = 0; s < shcol; ++s){ @@ -118,11 +115,11 @@ struct UnitWrap::UnitTest::TestDiagSecondMomentsLbycond { } // run_property - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - DiagSecondMomentsLbycondData f90_data[] = { + DiagSecondMomentsLbycondData baseline_data[] = { DiagSecondMomentsLbycondData(120), DiagSecondMomentsLbycondData(120), DiagSecondMomentsLbycondData(120), @@ -130,50 +127,55 @@ struct UnitWrap::UnitTest::TestDiagSecondMomentsLbycond { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state DiagSecondMomentsLbycondData cxx_data[] = { - DiagSecondMomentsLbycondData(f90_data[0]), - DiagSecondMomentsLbycondData(f90_data[1]), - DiagSecondMomentsLbycondData(f90_data[2]), - DiagSecondMomentsLbycondData(f90_data[3]), + DiagSecondMomentsLbycondData(baseline_data[0]), + DiagSecondMomentsLbycondData(baseline_data[1]), + DiagSecondMomentsLbycondData(baseline_data[2]), + DiagSecondMomentsLbycondData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - diag_second_moments_lbycond(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - diag_second_moments_lbycond_f(d.shcol, d.wthl_sfc, d.wqw_sfc, d.uw_sfc, d.vw_sfc, d.ustar2, d.wstar, - d.wthl_sec, d.wqw_sec, d.uw_sec, d.vw_sec, d.wtke_sec, d.thl_sec, d.qw_sec, d.qwthl_sec); + diag_second_moments_lbycond(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(DiagSecondMomentsLbycondData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(DiagSecondMomentsLbycondData); for (Int i = 0; i < num_runs; ++i) { - DiagSecondMomentsLbycondData& d_f90 = f90_data[i]; + DiagSecondMomentsLbycondData& d_baseline = baseline_data[i]; DiagSecondMomentsLbycondData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.shcol; ++k) { - REQUIRE(d_f90.wthl_sec[k] == d_cxx.wthl_sec[k]); - REQUIRE(d_f90.wqw_sec[k] == d_cxx.wqw_sec[k]); - REQUIRE(d_f90.uw_sec[k] == d_cxx.uw_sec[k]); - REQUIRE(d_f90.vw_sec[k] == d_cxx.vw_sec[k]); - REQUIRE(d_f90.wtke_sec[k] == d_cxx.wtke_sec[k]); - REQUIRE(d_f90.thl_sec[k] == d_cxx.thl_sec[k]); - REQUIRE(d_f90.qw_sec[k] == d_cxx.qw_sec[k]); - REQUIRE(d_f90.qwthl_sec[k] == d_cxx.qwthl_sec[k]); + for (Int k = 0; k < d_baseline.shcol; ++k) { + REQUIRE(d_baseline.wthl_sec[k] == d_cxx.wthl_sec[k]); + REQUIRE(d_baseline.wqw_sec[k] == d_cxx.wqw_sec[k]); + REQUIRE(d_baseline.uw_sec[k] == d_cxx.uw_sec[k]); + REQUIRE(d_baseline.vw_sec[k] == d_cxx.vw_sec[k]); + REQUIRE(d_baseline.wtke_sec[k] == d_cxx.wtke_sec[k]); + REQUIRE(d_baseline.thl_sec[k] == d_cxx.thl_sec[k]); + REQUIRE(d_baseline.qw_sec[k] == d_cxx.qw_sec[k]); + REQUIRE(d_baseline.qwthl_sec[k] == d_cxx.qwthl_sec[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb @@ -189,14 +191,14 @@ TEST_CASE("diag_second_moments_lbycond_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestDiagSecondMomentsLbycond; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("diag_second_moments_lbycond_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestDiagSecondMomentsLbycond; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_tests.cpp index f83bfa313f9..bd5dc1a77b1 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,9 +14,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestDiagSecondMoments { +struct UnitWrap::UnitTest::TestDiagSecondMoments : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 5; @@ -163,12 +163,7 @@ struct UnitWrap::UnitTest::TestDiagSecondMoments { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - diag_second_moments_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.u_wind, SDS.v_wind, - SDS.tke, SDS.isotropy, SDS.tkh, SDS.tk, SDS.dz_zi, SDS.zt_grid, SDS.zi_grid, SDS.shoc_mix, - SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.wqw_sec, SDS.qwthl_sec, SDS.uw_sec, - SDS.vw_sec, SDS.wtke_sec, SDS.w_sec); - SDS.transpose(); // go back to C layout + diag_second_moments(SDS); // Verify output makes sense for(Int s = 0; s < shcol; ++s) { @@ -254,11 +249,11 @@ struct UnitWrap::UnitTest::TestDiagSecondMoments { } // run_property - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - DiagSecondMomentsData f90_data[] = { + DiagSecondMomentsData baseline_data[] = { DiagSecondMomentsData(36, 72, 73), DiagSecondMomentsData(72, 72, 73), DiagSecondMomentsData(128, 72, 73), @@ -266,57 +261,58 @@ struct UnitWrap::UnitTest::TestDiagSecondMoments { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state DiagSecondMomentsData cxx_data[] = { - DiagSecondMomentsData(f90_data[0]), - DiagSecondMomentsData(f90_data[1]), - DiagSecondMomentsData(f90_data[2]), - DiagSecondMomentsData(f90_data[3]), + DiagSecondMomentsData(baseline_data[0]), + DiagSecondMomentsData(baseline_data[1]), + DiagSecondMomentsData(baseline_data[2]), + DiagSecondMomentsData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - diag_second_moments(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - diag_second_moments_f(d.shcol, d.nlev, d.nlevi, d.thetal, d.qw, d.u_wind, d.v_wind, - d.tke, d.isotropy, d.tkh, d.tk, d.dz_zi, d.zt_grid, d.zi_grid, d.shoc_mix, - d.thl_sec, d.qw_sec, d.wthl_sec, d.wqw_sec, d.qwthl_sec, d.uw_sec, - d.vw_sec, d.wtke_sec, d.w_sec); - d.transpose(); // go back to C layout + diag_second_moments(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(DiagSecondMomentsData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(DiagSecondMomentsData); for (Int i = 0; i < num_runs; ++i) { - DiagSecondMomentsData& d_f90 = f90_data[i]; + DiagSecondMomentsData& d_baseline = baseline_data[i]; DiagSecondMomentsData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.w_sec); ++k) { - REQUIRE(d_f90.w_sec[k] == d_cxx.w_sec[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.w_sec); ++k) { + REQUIRE(d_baseline.w_sec[k] == d_cxx.w_sec[k]); } - for (Int k = 0; k < d_f90.total(d_f90.thl_sec); ++k) { - REQUIRE(d_f90.thl_sec[k] == d_cxx.thl_sec[k]); - REQUIRE(d_f90.qw_sec[k] == d_cxx.qw_sec[k]); - REQUIRE(d_f90.wthl_sec[k] == d_cxx.wthl_sec[k]); - REQUIRE(d_f90.wqw_sec[k] == d_cxx.wqw_sec[k]); - REQUIRE(d_f90.qwthl_sec[k] == d_cxx.qwthl_sec[k]); - REQUIRE(d_f90.uw_sec[k] == d_cxx.uw_sec[k]); - REQUIRE(d_f90.vw_sec[k] == d_cxx.vw_sec[k]); - REQUIRE(d_f90.wtke_sec[k] == d_cxx.wtke_sec[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.thl_sec); ++k) { + REQUIRE(d_baseline.thl_sec[k] == d_cxx.thl_sec[k]); + REQUIRE(d_baseline.qw_sec[k] == d_cxx.qw_sec[k]); + REQUIRE(d_baseline.wthl_sec[k] == d_cxx.wthl_sec[k]); + REQUIRE(d_baseline.wqw_sec[k] == d_cxx.wqw_sec[k]); + REQUIRE(d_baseline.qwthl_sec[k] == d_cxx.qwthl_sec[k]); + REQUIRE(d_baseline.uw_sec[k] == d_cxx.uw_sec[k]); + REQUIRE(d_baseline.vw_sec[k] == d_cxx.vw_sec[k]); + REQUIRE(d_baseline.wtke_sec[k] == d_cxx.wtke_sec[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb @@ -332,14 +328,14 @@ TEST_CASE("diag_second_moments_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestDiagSecondMoments; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("diag_second_moments_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestDiagSecondMoments; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_shoc_moments_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_shoc_moments_tests.cpp index 0385335c808..45964c6b1d5 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_shoc_moments_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_shoc_moments_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,9 +14,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestDiagSecondShocMoments { +struct UnitWrap::UnitTest::TestDiagSecondShocMoments : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; @@ -161,12 +161,7 @@ struct UnitWrap::UnitTest::TestDiagSecondShocMoments { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - diag_second_shoc_moments_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.u_wind, SDS.v_wind, SDS.tke, SDS.isotropy, - SDS.tkh, SDS.tk, SDS.dz_zi, SDS.zt_grid, SDS.zi_grid, SDS.shoc_mix, SDS.wthl_sfc, SDS.wqw_sfc, - SDS.uw_sfc, SDS.vw_sfc, SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.wqw_sec, SDS.qwthl_sec, - SDS.uw_sec, SDS.vw_sec, SDS.wtke_sec, SDS.w_sec); - SDS.transpose(); // go back to C layout + diag_second_shoc_moments(SDS); // Verify output makes sense for(Int s = 0; s < shcol; ++s) { @@ -267,11 +262,11 @@ struct UnitWrap::UnitTest::TestDiagSecondShocMoments { } // run_property - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - DiagSecondShocMomentsData f90_data[] = { + DiagSecondShocMomentsData baseline_data[] = { DiagSecondShocMomentsData(36, 72, 73), DiagSecondShocMomentsData(72, 72, 73), DiagSecondShocMomentsData(128, 72, 73), @@ -279,56 +274,58 @@ struct UnitWrap::UnitTest::TestDiagSecondShocMoments { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state DiagSecondShocMomentsData cxx_data[] = { - DiagSecondShocMomentsData(f90_data[0]), - DiagSecondShocMomentsData(f90_data[1]), - DiagSecondShocMomentsData(f90_data[2]), - DiagSecondShocMomentsData(f90_data[3]), + DiagSecondShocMomentsData(baseline_data[0]), + DiagSecondShocMomentsData(baseline_data[1]), + DiagSecondShocMomentsData(baseline_data[2]), + DiagSecondShocMomentsData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - diag_second_shoc_moments(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - diag_second_shoc_moments_f(d.shcol, d.nlev, d.nlevi, d.thetal, d.qw, d.u_wind, d.v_wind, d.tke, d.isotropy, - d.tkh, d.tk, d.dz_zi, d.zt_grid, d.zi_grid, d.shoc_mix, d.wthl_sfc, d.wqw_sfc, d.uw_sfc, d.vw_sfc, d.thl_sec, - d.qw_sec, d.wthl_sec, d.wqw_sec, d.qwthl_sec, d.uw_sec, d.vw_sec, d.wtke_sec, d.w_sec); - d.transpose(); // go back to C layout + diag_second_shoc_moments(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(DiagSecondShocMomentsData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(DiagSecondShocMomentsData); for (Int i = 0; i < num_runs; ++i) { - DiagSecondShocMomentsData& d_f90 = f90_data[i]; + DiagSecondShocMomentsData& d_baseline = baseline_data[i]; DiagSecondShocMomentsData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.w_sec); ++k) { - REQUIRE(d_f90.w_sec[k] == d_cxx.w_sec[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.w_sec); ++k) { + REQUIRE(d_baseline.w_sec[k] == d_cxx.w_sec[k]); } - for (Int k = 0; k < d_f90.total(d_f90.thl_sec); ++k) { - REQUIRE(d_f90.thl_sec[k] == d_cxx.thl_sec[k]); - REQUIRE(d_f90.qw_sec[k] == d_cxx.qw_sec[k]); - REQUIRE(d_f90.wthl_sec[k] == d_cxx.wthl_sec[k]); - REQUIRE(d_f90.wqw_sec[k] == d_cxx.wqw_sec[k]); - REQUIRE(d_f90.qwthl_sec[k] == d_cxx.qwthl_sec[k]); - REQUIRE(d_f90.uw_sec[k] == d_cxx.uw_sec[k]); - REQUIRE(d_f90.vw_sec[k] == d_cxx.vw_sec[k]); - REQUIRE(d_f90.wtke_sec[k] == d_cxx.wtke_sec[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.thl_sec); ++k) { + REQUIRE(d_baseline.thl_sec[k] == d_cxx.thl_sec[k]); + REQUIRE(d_baseline.qw_sec[k] == d_cxx.qw_sec[k]); + REQUIRE(d_baseline.wthl_sec[k] == d_cxx.wthl_sec[k]); + REQUIRE(d_baseline.wqw_sec[k] == d_cxx.wqw_sec[k]); + REQUIRE(d_baseline.qwthl_sec[k] == d_cxx.qwthl_sec[k]); + REQUIRE(d_baseline.uw_sec[k] == d_cxx.uw_sec[k]); + REQUIRE(d_baseline.vw_sec[k] == d_cxx.vw_sec[k]); + REQUIRE(d_baseline.wtke_sec[k] == d_cxx.wtke_sec[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb @@ -344,14 +341,14 @@ TEST_CASE("diag_second_shoc_moments_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestDiagSecondShocMoments; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("diag_second_shoc_moments_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestDiagSecondShocMoments; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_third_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_third_tests.cpp index 82b4d88c0f7..3f1bfd557f0 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_third_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_third_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocDiagThird { +struct UnitWrap::UnitTest::TestShocDiagThird : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 5; @@ -143,13 +143,7 @@ struct UnitWrap::UnitTest::TestShocDiagThird { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - diag_third_shoc_moments_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.w_sec,SDS.thl_sec, - SDS.wthl_sec,SDS.isotropy,SDS.brunt,SDS.thetal, - SDS.tke,SDS.dz_zt,SDS.dz_zi,SDS.zt_grid,SDS.zi_grid, - SDS.w3); - SDS.transpose(); + diag_third_shoc_moments(SDS); // Check to make sure there is at least one // positive w3 value for convective boundary layer @@ -193,13 +187,7 @@ struct UnitWrap::UnitTest::TestShocDiagThird { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - diag_third_shoc_moments_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.w_sec,SDS.thl_sec, - SDS.wthl_sec,SDS.isotropy,SDS.brunt,SDS.thetal, - SDS.tke,SDS.dz_zt,SDS.dz_zi,SDS.zt_grid,SDS.zi_grid, - SDS.w3); - SDS.transpose(); + diag_third_shoc_moments(SDS); // Verify that new result is greater or equal in magnitude // that the result from test one @@ -214,11 +202,11 @@ struct UnitWrap::UnitTest::TestShocDiagThird { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - DiagThirdShocMomentsData SDS_f90[] = { + DiagThirdShocMomentsData SDS_baseline[] = { // shcol, nlev, nlevi DiagThirdShocMomentsData(10, 71, 72), DiagThirdShocMomentsData(10, 12, 13), @@ -227,48 +215,49 @@ struct UnitWrap::UnitTest::TestShocDiagThird { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine, {{d.thetal, {300, 301}}}); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state DiagThirdShocMomentsData SDS_cxx[] = { - DiagThirdShocMomentsData(SDS_f90[0]), - DiagThirdShocMomentsData(SDS_f90[1]), - DiagThirdShocMomentsData(SDS_f90[2]), - DiagThirdShocMomentsData(SDS_f90[3]), + DiagThirdShocMomentsData(SDS_baseline[0]), + DiagThirdShocMomentsData(SDS_baseline[1]), + DiagThirdShocMomentsData(SDS_baseline[2]), + DiagThirdShocMomentsData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(DiagThirdShocMomentsData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - diag_third_shoc_moments(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - diag_third_shoc_moments_f(d.shcol,d.nlev,d.nlevi,d.w_sec,d.thl_sec, - d.wthl_sec,d.isotropy,d.brunt,d.thetal, - d.tke,d.dz_zt,d.dz_zi,d.zt_grid,d.zi_grid, - d.w3); - d.transpose(); + diag_third_shoc_moments(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(DiagThirdShocMomentsData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - DiagThirdShocMomentsData& d_f90 = SDS_f90[i]; + DiagThirdShocMomentsData& d_baseline = SDS_baseline[i]; DiagThirdShocMomentsData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.w3); ++k) { - REQUIRE(d_f90.w3[k] == d_cxx.w3[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.w3); ++k) { + REQUIRE(d_baseline.w3[k] == d_cxx.w3[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -283,14 +272,14 @@ TEST_CASE("shoc_diag_third_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocDiagThird; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_diag_third_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocDiagThird; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp index 0a96e5adc62..3d33d55640c 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -22,9 +22,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocEddyDiff { +struct UnitWrap::UnitTest::TestShocEddyDiff : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 1; @@ -98,10 +98,7 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - eddy_diffusivities_f(SDS.nlev, SDS.shcol, SDS.pblh, SDS.zt_grid, SDS.tabs, SDS.shoc_mix, - SDS.sterm_zt, SDS.isotropy, SDS.tke, SDS.tkh, SDS.tk); - SDS.transpose(); // go back to C layout + eddy_diffusivities(SDS); // Check to make sure the answers in the columns are different for(Int s = 0; s < shcol-1; ++s) { @@ -168,10 +165,7 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - eddy_diffusivities_f(SDS.nlev, SDS.shcol, SDS.pblh, SDS.zt_grid, SDS.tabs, SDS.shoc_mix, - SDS.sterm_zt, SDS.isotropy, SDS.tke, SDS.tkh, SDS.tk); - SDS.transpose(); // go back to C layout + eddy_diffusivities(SDS); // Check to make sure the answers in the columns are larger // when the length scale and shear term are larger @@ -241,10 +235,7 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - eddy_diffusivities_f(SDS.nlev, SDS.shcol, SDS.pblh, SDS.zt_grid, SDS.tabs, SDS.shoc_mix, - SDS.sterm_zt, SDS.isotropy, SDS.tke, SDS.tkh, SDS.tk); - SDS.transpose(); // go back to C layout + eddy_diffusivities(SDS); // Check to make sure the diffusivities are smaller // in the columns where isotropy and tke are smaller @@ -263,11 +254,11 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - EddyDiffusivitiesData f90_data[] = { + EddyDiffusivitiesData baseline_data[] = { EddyDiffusivitiesData(10, 71), EddyDiffusivitiesData(10, 12), EddyDiffusivitiesData(7, 16), @@ -275,46 +266,50 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { }; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state EddyDiffusivitiesData cxx_data[] = { - EddyDiffusivitiesData(f90_data[0]), - EddyDiffusivitiesData(f90_data[1]), - EddyDiffusivitiesData(f90_data[2]), - EddyDiffusivitiesData(f90_data[3]), + EddyDiffusivitiesData(baseline_data[0]), + EddyDiffusivitiesData(baseline_data[1]), + EddyDiffusivitiesData(baseline_data[2]), + EddyDiffusivitiesData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - eddy_diffusivities(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - eddy_diffusivities_f(d.nlev, d.shcol, d.pblh, d.zt_grid, d.tabs, d.shoc_mix, d.sterm_zt, d.isotropy, d.tke, d.tkh, d.tk); - d.transpose(); // go back to C layout + eddy_diffusivities(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(EddyDiffusivitiesData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(EddyDiffusivitiesData); for (Int i = 0; i < num_runs; ++i) { - EddyDiffusivitiesData& d_f90 = f90_data[i]; + EddyDiffusivitiesData& d_baseline = baseline_data[i]; EddyDiffusivitiesData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.tkh); ++k) { - REQUIRE(d_f90.tkh[k] == d_cxx.tkh[k]); - REQUIRE(d_f90.tk[k] == d_cxx.tk[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.tkh); ++k) { + REQUIRE(d_baseline.tkh[k] == d_cxx.tkh[k]); + REQUIRE(d_baseline.tk[k] == d_cxx.tk[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb }; @@ -329,14 +324,14 @@ TEST_CASE("shoc_tke_eddy_diffusivities_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocEddyDiff; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_tke_eddy_diffusivities_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocEddyDiff; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_dse_fixer_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_dse_fixer_tests.cpp deleted file mode 100644 index 9b91a7ef0b2..00000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_dse_fixer_tests.cpp +++ /dev/null @@ -1,137 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" - -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestShocEnergyDseFixer { - - static void run_property() - { - static constexpr Int shcol = 6; - static constexpr Int nlev = 5; - - // Tests for the SHOC function - // shoc_energy_dse_fixer - - // TEST - // For columns that are identical EXCEPT for the shoctop indicee, - // verify that given a positive value of energy imbalance that - // columns with a higher SHOC top were subject to more energy removal. - - // Host model dry static energy [J kg-1] - static constexpr Real host_dse_input[nlev] = {350e3, 325e3, 315e3, 310e3, 300e3}; - - // Energy disbalance. For this test we assume all columns have - // the same disbalance magnitude. - static constexpr Real se_dis = 0.1; - - // level indicee of SHOC top layer - static constexpr Int shoctop[shcol] = {5, 3, 1, 2, 4, 4}; - - // Initialize data structure for bridging to F90 - ShocEnergyDseFixerData SDS(shcol, nlev); - - // Test that the inputs are reasonable. - // for this test we need exactly six columns - REQUIRE( (SDS.shcol == 6 && SDS.nlev == nlev) ); - - // Fill in test data on zt_grid. - for(Int s = 0; s < shcol; ++s) { - SDS.shoctop[s] = shoctop[s]; - SDS.se_dis[s] = se_dis; - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - - SDS.host_dse[offset] = host_dse_input[n]; - } - } - - // Check that the inputs make sense - - for(Int s = 0; s < shcol; ++s) { - // For this test we WANT se_dis > 0 - REQUIRE(SDS.se_dis[s] > 0.0); - REQUIRE(SDS.shoctop[s] >= 1); - REQUIRE(SDS.shoctop[s] <= nlev); - for (Int n = 0; n < nlev; ++n){ - const auto offset = n + s * nlev; - - REQUIRE(SDS.host_dse[offset] > 0.0); - } - } - - // Call the fortran implementation - shoc_energy_dse_fixer(SDS); - - // Check the results - Real temp_sum[shcol]; - for(Int s = 0; s < shcol; ++s) { - temp_sum[s] = 0.0; - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - temp_sum[s] += SDS.host_dse[offset]; - } - } - - // Verify that as shoctop values get lower that the - // summation of temperatures also gets lower. This is proportionally - // to the amount of energy we expect to be removed from a column. - for (Int s = 0; s < shcol-1; ++s) { - if (shoctop[s] < shoctop[s+1]){ - REQUIRE(temp_sum[s] < temp_sum[s+1]); - } - else if (shoctop[s] > shoctop[s+1]){ - REQUIRE(temp_sum[s] > temp_sum[s+1]); - } - else{ - REQUIRE(temp_sum[s] == temp_sum[s+1]); - } - } - - } - - static void run_bfb() - { - // TODO - } -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace { - -TEST_CASE("shoc_energy_dse_fixer_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocEnergyDseFixer; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_energy_dse_fixer_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocEnergyDseFixer; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_fixer_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_fixer_tests.cpp index dfb37104f01..89deaf7ad4b 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_fixer_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_energy_fixer_tests.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "shoc_constants.hpp" #include "share/scream_types.hpp" @@ -23,9 +23,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocEnergyFixer { +struct UnitWrap::UnitTest::TestShocEnergyFixer : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Real gravit = scream::physics::Constants::gravit; static constexpr Real Cpair = scream::physics::Constants::Cpair; @@ -158,14 +158,7 @@ struct UnitWrap::UnitTest::TestShocEnergyFixer { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - shoc_energy_fixer_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.nadv, - SDS.zt_grid, SDS.zi_grid, SDS.se_b, SDS.ke_b, SDS.wv_b, - SDS.wl_b, SDS.se_a, SDS.ke_a, SDS.wv_a, SDS.wl_a, SDS.wthl_sfc, - SDS.wqw_sfc, SDS.rho_zt, SDS.tke, SDS.pint, - SDS.host_dse); - SDS.transpose(); + shoc_energy_fixer(SDS); // Check test // Verify that the dry static energy has not changed if surface @@ -239,14 +232,7 @@ struct UnitWrap::UnitTest::TestShocEnergyFixer { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - shoc_energy_fixer_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.nadv, - SDS.zt_grid, SDS.zi_grid, SDS.se_b, SDS.ke_b, SDS.wv_b, - SDS.wl_b, SDS.se_a, SDS.ke_a, SDS.wv_a, SDS.wl_a, SDS.wthl_sfc, - SDS.wqw_sfc, SDS.rho_zt, SDS.tke, SDS.pint, - SDS.host_dse); - SDS.transpose(); + shoc_energy_fixer(SDS); // Verify the result for(Int s = 0; s < shcol; ++s) { @@ -275,11 +261,11 @@ struct UnitWrap::UnitTest::TestShocEnergyFixer { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ShocEnergyFixerData SDS_f90[] = { + ShocEnergyFixerData SDS_baseline[] = { // shcol, nlev, nlevi, dtime, nadv ShocEnergyFixerData(10, 71, 72, 300, 2), ShocEnergyFixerData(10, 12, 13, 100, 10), @@ -288,49 +274,49 @@ struct UnitWrap::UnitTest::TestShocEnergyFixer { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ShocEnergyFixerData SDS_cxx[] = { - ShocEnergyFixerData(SDS_f90[0]), - ShocEnergyFixerData(SDS_f90[1]), - ShocEnergyFixerData(SDS_f90[2]), - ShocEnergyFixerData(SDS_f90[3]), + ShocEnergyFixerData(SDS_baseline[0]), + ShocEnergyFixerData(SDS_baseline[1]), + ShocEnergyFixerData(SDS_baseline[2]), + ShocEnergyFixerData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(ShocEnergyFixerData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - shoc_energy_fixer(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - shoc_energy_fixer_f(d.shcol, d.nlev, d.nlevi, d.dtime, d.nadv, - d.zt_grid, d.zi_grid, d.se_b, d.ke_b, d.wv_b, - d.wl_b, d.se_a, d.ke_a, d.wv_a, d.wl_a, d.wthl_sfc, - d.wqw_sfc, d.rho_zt, d.tke, d.pint, - d.host_dse); - d.transpose(); + shoc_energy_fixer(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(ShocEnergyFixerData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ShocEnergyFixerData& d_f90 = SDS_f90[i]; + ShocEnergyFixerData& d_baseline = SDS_baseline[i]; ShocEnergyFixerData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.host_dse); ++k) { - REQUIRE(d_f90.host_dse[k] == d_cxx.host_dse[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.host_dse); ++k) { + REQUIRE(d_baseline.host_dse[k] == d_cxx.host_dse[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -345,14 +331,14 @@ TEST_CASE("shoc_energy_fixer_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocEnergyFixer; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_energy_fixer_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocEnergyFixer; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_integral_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_integral_tests.cpp index 8142da9371d..e26ec03139f 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_integral_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_energy_integral_tests.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -22,9 +22,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocEnergyInt { +struct UnitWrap::UnitTest::TestShocEnergyInt : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 5; @@ -108,12 +108,7 @@ struct UnitWrap::UnitTest::TestShocEnergyInt { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - shoc_energy_integrals_f(SDS.shcol, SDS.nlev, SDS.host_dse, SDS.pdel, - SDS.rtm, SDS.rcm, SDS.u_wind, SDS.v_wind, - SDS.se_int, SDS.ke_int, SDS.wv_int, SDS.wl_int); - SDS.transpose(); + shoc_energy_integrals(SDS); // Check test for(Int s = 0; s < shcol; ++s) { @@ -132,11 +127,11 @@ struct UnitWrap::UnitTest::TestShocEnergyInt { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ShocEnergyIntegralsData SDS_f90[] = { + ShocEnergyIntegralsData SDS_baseline[] = { // shcol, nlev ShocEnergyIntegralsData(10, 71), ShocEnergyIntegralsData(10, 12), @@ -145,50 +140,52 @@ struct UnitWrap::UnitTest::TestShocEnergyInt { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ShocEnergyIntegralsData SDS_cxx[] = { - ShocEnergyIntegralsData(SDS_f90[0]), - ShocEnergyIntegralsData(SDS_f90[1]), - ShocEnergyIntegralsData(SDS_f90[2]), - ShocEnergyIntegralsData(SDS_f90[3]), + ShocEnergyIntegralsData(SDS_baseline[0]), + ShocEnergyIntegralsData(SDS_baseline[1]), + ShocEnergyIntegralsData(SDS_baseline[2]), + ShocEnergyIntegralsData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(ShocEnergyIntegralsData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - shoc_energy_integrals(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - shoc_energy_integrals_f(d.shcol, d.nlev, d.host_dse, d.pdel, - d.rtm, d.rcm, d.u_wind, d.v_wind, - d.se_int, d.ke_int, d.wv_int, d.wl_int); - d.transpose(); + shoc_energy_integrals(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(ShocEnergyIntegralsData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ShocEnergyIntegralsData& d_f90 = SDS_f90[i]; + ShocEnergyIntegralsData& d_baseline = SDS_baseline[i]; ShocEnergyIntegralsData& d_cxx = SDS_cxx[i]; - for (Int c = 0; c < d_f90.shcol; ++c) { - REQUIRE(d_f90.se_int[c] == d_cxx.se_int[c]); - REQUIRE(d_f90.ke_int[c] == d_cxx.ke_int[c]); - REQUIRE(d_f90.wv_int[c] == d_cxx.wv_int[c]); - REQUIRE(d_f90.wl_int[c] == d_cxx.wl_int[c]); + for (Int c = 0; c < d_baseline.shcol; ++c) { + REQUIRE(d_baseline.se_int[c] == d_cxx.se_int[c]); + REQUIRE(d_baseline.ke_int[c] == d_cxx.ke_int[c]); + REQUIRE(d_baseline.wv_int[c] == d_cxx.wv_int[c]); + REQUIRE(d_baseline.wl_int[c] == d_cxx.wl_int[c]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -203,14 +200,14 @@ TEST_CASE("shoc_energy_integrals_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocEnergyInt; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_energy_integrals_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocEnergyInt; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_threshold_fixer_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_threshold_fixer_tests.cpp deleted file mode 100644 index 19743c39189..00000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_threshold_fixer_tests.cpp +++ /dev/null @@ -1,139 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" - -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestShocEnergyThreshFixer { - - static void run_property() - { - static constexpr Real mintke = scream::shoc::Constants::mintke; - static constexpr Int shcol = 2; - static constexpr Int nlev = 5; - static constexpr auto nlevi = nlev + 1; - - // Tests for the SHOC function - // shoc_energy_threshold_fixer - - // TEST ONE - // Set up a reasonable profile verify results are as expected - - // Host model TKE [m2/s2] - Real tke_input[nlev] = {mintke, mintke, 0.01, 0.4, 0.5}; - // Pressure at interface [Pa] - Real pint[nlevi] = {500e2, 600e2, 700e2, 800e2, 900e2, 1000e2}; - - // Integrated total energy after SHOC. - static constexpr Real te_a = 100; - // Integrated total energy before SHOC - static constexpr Real te_b = 110; - - // convert pressure to Pa - for(Int n = 0; n < nlevi; ++n) { - pint[n] = pint[n]; - } - - // Initialize data structure for bridging to F90 - ShocEnergyThresholdFixerData SDS(shcol, nlev, nlevi); - - // Test that the inputs are reasonable. - REQUIRE( (SDS.shcol == shcol && SDS.nlev == nlev && SDS.nlevi == nlevi) ); - REQUIRE(SDS.shcol > 1); - REQUIRE(nlev+1 == nlevi); - - // Fill in test data on zt_grid. - for(Int s = 0; s < shcol; ++s) { - SDS.te_a[s] = te_a; - SDS.te_b[s] = te_b; - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - - SDS.tke[offset] = tke_input[n]; - } - - for(Int n = 0; n < nlevi; ++n) { - const auto offset = n + s * nlevi; - - SDS.pint[offset] = pint[n]; - } - } - - // Check that the inputs make sense - - for(Int s = 0; s < shcol; ++s) { - for (Int n = 0; n < nlev; ++n){ - const auto offset = n + s * nlev; - - REQUIRE(SDS.tke[offset] >= mintke); - } - } - - // Call the fortran implementation - shoc_energy_threshold_fixer(SDS); - - // Verify the result - for(Int s = 0; s < shcol; ++s) { - // Make sure value of shoctop is within reasonable range - REQUIRE(SDS.shoctop[s] < nlev); - REQUIRE(SDS.shoctop[s] > 1); - - // Verify that shoctop represents what we want it to - // Make sure that thickness that bounds shoctop is positive - const auto offset_stopi = (SDS.shoctop[s]-1) + s * nlevi; - const auto offset_bot = (nlevi-1) + s * nlevi; - REQUIRE(SDS.pint[offset_bot] - SDS.pint[offset_stopi] > 0.0); - - if (SDS.shoctop[s] < nlev){ - const auto offset_stop = (SDS.shoctop[s]-1) + s * nlev; - REQUIRE(SDS.tke[offset_stop] == mintke); - REQUIRE(SDS.tke[offset_stop+1] > mintke); - } - } - - } - - static void run_bfb() - { - // TODO - } -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace { - -TEST_CASE("shoc_energy_threshold_fixer_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocEnergyThreshFixer; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_energy_threshold_fixer_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocEnergyThreshFixer; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_total_fixer_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_total_fixer_tests.cpp deleted file mode 100644 index ea59b217f77..00000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_total_fixer_tests.cpp +++ /dev/null @@ -1,167 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" - -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestShocTotEnergyFixer { - - static void run_property() - { - static constexpr Int shcol = 2; - static constexpr Int nlev = 5; - static constexpr auto nlevi = nlev + 1; - - // Tests for the SHOC function - // shoc_energy_total_fixer - - // FIRST TEST - // Surface flux test. Have two columns, one with zero surface fluxes - // and the other with positive surface fluxes. Verify the column - // with surface fluxes has greater total energy "before". - - // Timestep [s] - static constexpr Real dtime = 300; - // Number of macmic steps - static constexpr Int nadv = 2; - // Air density [km/m3] - static constexpr Real rho_zt[nlev] = {0.4, 0.6, 0.7, 0.9, 1.0}; - // Interface heights [m] - static constexpr Real zi_grid[nlevi] = {11000, 7500, 5000, 3000, 1500, 0}; - // Define integrated static energy, kinetic energy, water vapor, - // and liquid water respectively - static constexpr Real se = 200; - static constexpr Real ke = 150; - static constexpr Real wv = 0.5; - static constexpr Real wl = 0.1; - // Define surface sensible heat flux [K m/s] - static constexpr Real wthl_sfc = 0.5; - // Define surface total water flux [kg/kg m/s] - static constexpr Real wqw_sfc = 0.01; - // Pressure at interface [Pa] - static constexpr Real pint[nlevi] = {50000, 60000, 70000, 80000, 90000, 100000}; - - // Initialize data structure for bridging to F90 - ShocEnergyTotalFixerData SDS(shcol, nlev, nlevi, dtime, nadv); - - // Test that the inputs are reasonable. - // for this test we need exactly two columns - REQUIRE( (SDS.shcol == shcol && SDS.nlev == nlev && SDS.nlevi == nlevi && SDS.dtime == dtime && SDS.nadv == nadv) ); - REQUIRE(shcol == 2); - REQUIRE(nlevi == nlev+1); - - for(Int s = 0; s < shcol; ++s) { - // Set before and after integrals equal - SDS.se_a[s] = se; - SDS.se_b[s] = se; - SDS.ke_a[s] = ke; - SDS.ke_b[s] = ke; - SDS.wv_a[s] = wv; - SDS.wv_b[s] = wv; - SDS.wl_a[s] = wl; - SDS.wl_b[s] = wl; - - // Make first column be zero for the surface fluxes - SDS.wthl_sfc[s] = s*wthl_sfc; - SDS.wqw_sfc[s] = s*wqw_sfc; - - // Fill in test data on zt_grid. - for(Int n = 0; n < nlev; ++n) { - const auto offset = n + s * nlev; - - // For zt grid, set as midpoint of zi grid - SDS.zt_grid[offset] = 0.5*(zi_grid[n]+zi_grid[n+1]); - SDS.rho_zt[offset] = rho_zt[n]; - } - // Fill in test data on zi_grid. - for(Int n = 0; n < nlevi; ++n) { - const auto offset = n + s * nlevi; - - SDS.zi_grid[offset] = zi_grid[n]; - SDS.pint[offset] = pint[n]; - } - } - - // Check that the inputs make sense - - for(Int s = 0; s < shcol; ++s) { - for (Int n = 0; n < nlev; ++n){ - const auto offset = n + s * nlev; - - REQUIRE(SDS.zt_grid[offset] >= 0); - REQUIRE(SDS.rho_zt[offset] > 0); - - // Check that heights increase upward - if (n > nlev-1){ - REQUIRE(SDS.zt_grid[offset + 1] - SDS.zt_grid[offset] < 0); - } - } - for (Int n = 0; n < nlevi; ++n){ - const auto offset = n + s * nlevi; - - REQUIRE(SDS.zi_grid[offset] >= 0); - - // Check that heights increase upward - if (n > nlevi-1){ - REQUIRE(SDS.zi_grid[offset + 1] - SDS.zi_grid[offset] < 0); - } - } - } - - // Call the fortran implementation - shoc_energy_total_fixer(SDS); - - // Check test - - // For first column verify that total energies are the same - REQUIRE(SDS.te_a[0] == SDS.te_b[0]); - - // Verify that second column "before" energy is greater than - // the first column, since here we have active surface fluxes - REQUIRE(SDS.te_b[1] > SDS.te_b[0]); - } - - static void run_bfb() - { - // TODO - } -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace { - -TEST_CASE("shoc_energy_total_fixer_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocTotEnergyFixer; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_energy_total_fixer_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocTotEnergyFixer; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_update_dse_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_update_dse_tests.cpp index a4e01eadb85..639d48fc70d 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_update_dse_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_energy_update_dse_tests.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -22,9 +22,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocUpdateDse { +struct UnitWrap::UnitTest::TestShocUpdateDse : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 5; @@ -109,11 +109,7 @@ struct UnitWrap::UnitTest::TestShocUpdateDse { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - update_host_dse_f(SDS.shcol,SDS.nlev,SDS.thlm,SDS.shoc_ql,SDS.inv_exner,SDS.zt_grid, - SDS.phis,SDS.host_dse); - SDS.transpose(); + update_host_dse(SDS); // Check test for(Int s = 0; s < shcol; ++s) { @@ -140,11 +136,11 @@ struct UnitWrap::UnitTest::TestShocUpdateDse { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - UpdateHostDseData SDS_f90[] = { + UpdateHostDseData SDS_baseline[] = { // shcol, nlev UpdateHostDseData(10, 71), UpdateHostDseData(10, 12), @@ -153,46 +149,49 @@ struct UnitWrap::UnitTest::TestShocUpdateDse { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state UpdateHostDseData SDS_cxx[] = { - UpdateHostDseData(SDS_f90[0]), - UpdateHostDseData(SDS_f90[1]), - UpdateHostDseData(SDS_f90[2]), - UpdateHostDseData(SDS_f90[3]), + UpdateHostDseData(SDS_baseline[0]), + UpdateHostDseData(SDS_baseline[1]), + UpdateHostDseData(SDS_baseline[2]), + UpdateHostDseData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(UpdateHostDseData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - update_host_dse(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - update_host_dse_f(d.shcol,d.nlev,d.thlm,d.shoc_ql,d.inv_exner,d.zt_grid, - d.phis,d.host_dse); - d.transpose(); + update_host_dse(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(UpdateHostDseData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - UpdateHostDseData& d_f90 = SDS_f90[i]; + UpdateHostDseData& d_baseline = SDS_baseline[i]; UpdateHostDseData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.host_dse); ++k) { - REQUIRE(d_f90.host_dse[k] == d_cxx.host_dse[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.host_dse); ++k) { + REQUIRE(d_baseline.host_dse[k] == d_cxx.host_dse[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -207,14 +206,14 @@ TEST_CASE("shoc_energy_host_dse_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocUpdateDse; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_energy_host_dse_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocUpdateDse; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_fterm_diag_third_moms_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_fterm_diag_third_moms_tests.cpp deleted file mode 100644 index 4fd800ca4ad..00000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_fterm_diag_third_moms_tests.cpp +++ /dev/null @@ -1,175 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestFtermdiagThirdMoms { - - static void run_property() - { - - // Tests for the SHOC function: - // f0_to_f5_diag_third_shoc_moment - - // TEST ONE - // Zero test. Given no gradients, verify that relevant - // terms are zero. - - // 1/grid spacing [m-1] - constexpr static Real thedz = 0.1; - // 1/grid spacing for two grids [m-1] - constexpr static Real thedz2 = 0.05; - // bet2 term (ggr/thetal) - constexpr static Real bet2 = 0.0327; - // return to isotropy timescale [s] - constexpr static Real iso = 1000; - // liquid water flux [K m/s] - constexpr static Real wthl_sec_zero = 0.01; - // thetal variance [K^2] - constexpr static Real thl_sec_zero = 2; - // vertical velocity variance [m2/s2] - constexpr static Real w_sec_zero = 0.4; - // TKE [m2/s2] - constexpr static Real tke_zero = 0.5; - - // Initialize data structure for bridging to F90 - F0ToF5DiagThirdShocMomentData SDS; - - // Fill in data - SDS.thedz = thedz; - SDS.thedz2 = thedz2; - SDS.bet2 = bet2; - SDS.iso = iso; - SDS.isosqrd = iso*iso; - // for the following moments, feed each level - // the same value for this test - SDS.wthl_sec = wthl_sec_zero; - SDS.wthl_sec_kc = wthl_sec_zero; - SDS.wthl_sec_kb = wthl_sec_zero; - SDS.thl_sec_kc = thl_sec_zero; - SDS.thl_sec_kb = thl_sec_zero; - SDS.w_sec = w_sec_zero; - SDS.w_sec_kc = w_sec_zero; - SDS.w_sec_zi = w_sec_zero; - SDS.tke = tke_zero; - SDS.tke_kc = tke_zero; - - // Be sure inputs are as we expect - REQUIRE(SDS.thedz > 0); - REQUIRE(SDS.thedz2 > 0); - REQUIRE(SDS.wthl_sec_kc == SDS.wthl_sec_kb); - REQUIRE(SDS.thl_sec_kc == SDS.thl_sec_kb); - REQUIRE(SDS.w_sec_kc == SDS.w_sec); - REQUIRE(SDS.tke_kc == SDS.tke); - - // Call the fortran implementation - f0_to_f5_diag_third_shoc_moment(SDS); - - // Check result, make sure all outputs are zero - REQUIRE(SDS.f0 == 0); - REQUIRE(SDS.f1 == 0); - REQUIRE(SDS.f2 == 0); - REQUIRE(SDS.f3 == 0); - REQUIRE(SDS.f4 == 0); - REQUIRE(SDS.f5 == 0); - - // TEST TWO - // Positive gradient test. Feed the function values of the second - // moments with positive gradients. All fterms should have positive values - - // liquid water flux [K m/s] - constexpr static Real wthl_sec = 0.01; - // liquid water flux [K m/s] above - constexpr static Real wthl_sec_kc = 0.02; - // liquid water flux [K m/s] below - constexpr static Real wthl_sec_kb = 0; - // thetal variance [K^2] above - constexpr static Real thl_sec_kc = 2.5; - // thetal variance [K^2] - constexpr static Real thl_sec_kb = 1.7; - // vertical velocity variance [m2/s2] - constexpr static Real w_sec = 0.4; - // vertical velocity variance [m2/s2] above - constexpr static Real w_sec_kc = 0.5; - // TKE [m2/s2] - constexpr static Real tke = 0.5; - // TKE [m2/s2] above - constexpr static Real tke_kc = 0.55; - - // Feed in data - SDS.wthl_sec = wthl_sec; - SDS.wthl_sec_kc = wthl_sec_kc; - SDS.wthl_sec_kb = wthl_sec_kb; - SDS.thl_sec_kc = thl_sec_kc; - SDS.thl_sec_kb = thl_sec_kb; - SDS.w_sec = w_sec; - SDS.w_sec_kc = w_sec_kc; - SDS.w_sec_zi = w_sec; - SDS.tke = tke; - SDS.tke_kc = tke_kc; - - // Verify input is what we want for this test - REQUIRE(wthl_sec > 0); - REQUIRE(wthl_sec_kc > wthl_sec_kb); - REQUIRE(thl_sec_kc > thl_sec_kb); - REQUIRE(w_sec_kc > w_sec); - REQUIRE(tke_kc > tke); - - // Call the fortran implementation - f0_to_f5_diag_third_shoc_moment(SDS); - - // Check result, make sure all outputs are greater than zero - REQUIRE(SDS.f0 > 0); - REQUIRE(SDS.f1 > 0); - REQUIRE(SDS.f2 > 0); - REQUIRE(SDS.f3 > 0); - REQUIRE(SDS.f4 > 0); - REQUIRE(SDS.f5 > 0); - - } - - static void run_bfb() - { - // TODO - } - -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace{ - -TEST_CASE("shoc_fterm_diag_third_moms_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestFtermdiagThirdMoms; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_fterm_diag_third_moms_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestFtermdiagThirdMoms; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_fterm_input_third_moms_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_fterm_input_third_moms_tests.cpp deleted file mode 100644 index 8d7774f1aba..00000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_fterm_input_third_moms_tests.cpp +++ /dev/null @@ -1,112 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestFtermInputThirdMoms { - - static void run_property() - { - - // Tests for the SHOC function: - // fterms_input_for_diag_third_shoc_moment - - // TEST - // Given inputs, verify that output is reasonable - - // grid spacing on interface grid [m] - constexpr static Real dz_zi = 100; - // grid spacing on midpoint grid [m] - constexpr static Real dz_zt = 80; - // grid spacing on adjacent midpoint grid [m] - constexpr static Real dz_zt_kc = 120; - // Return to isotropic timescale [s] - constexpr static Real isotropy_zi = 1000; - // Brunt vaisalla frequency [s] - constexpr static Real brunt_zi = -0.05; - // Potential temperature on interface grid [K] - constexpr static Real thetal_zi = 300; - - // Initialize data structure for bridging to F90 - FtermsInputForDiagThirdShocMomentData SDS; - - SDS.dz_zi = dz_zi; - SDS.dz_zt = dz_zt; - SDS.dz_zt_kc = dz_zt_kc; - SDS.isotropy_zi = isotropy_zi; - SDS.brunt_zi = brunt_zi; - SDS.thetal_zi = thetal_zi; - - // Check that input is physical - REQUIRE(SDS.dz_zi > 0); - REQUIRE(SDS.dz_zt > 0); - REQUIRE(SDS.dz_zt_kc > 0); - REQUIRE(SDS.isotropy_zi > 0); - REQUIRE(SDS.thetal_zi > 0); - - // Call the fortran implementation - fterms_input_for_diag_third_shoc_moment(SDS); - - // Verify the result - - // Check that thedz2 is smaller than thedz. - REQUIRE(SDS.thedz2 < SDS.thedz); - - // Check that bet2 is smaller than thetal - REQUIRE(SDS.bet2 < SDS.thetal_zi); - - // Be sure that iso and isosqrd relationships hold - if (SDS.isotropy_zi > 1){ - REQUIRE(SDS.isosqrd > SDS.iso); - } - else{ - REQUIRE(SDS.isosqrd < SDS.iso); - } - - } - - static void run_bfb() - { - // TODO - } - -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace{ - -TEST_CASE("shoc_fterm_input_third_moms_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestFtermInputThirdMoms; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_fterm_input_third_moms_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestFtermInputThirdMoms; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_grid_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_grid_tests.cpp index 6a79eeca02e..f9bd348dc47 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_grid_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_grid_tests.cpp @@ -4,7 +4,7 @@ #include "physics/share/physics_constants.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -23,9 +23,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocGrid { +struct UnitWrap::UnitTest::TestShocGrid : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Real gravit = scream::physics::Constants::gravit; static constexpr Int shcol = 2; @@ -80,9 +80,7 @@ struct UnitWrap::UnitTest::TestShocGrid { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - shoc_grid_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.zt_grid, SDS.zi_grid, SDS.pdel, SDS.dz_zt, SDS.dz_zi, SDS.rho_zt); - SDS.transpose(); // go back to C layout + shoc_grid(SDS); // First check that dz is correct for(Int s = 0; s < shcol; ++s) { @@ -129,61 +127,66 @@ struct UnitWrap::UnitTest::TestShocGrid { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ShocGridData f90_data[] = { + ShocGridData baseline_data[] = { ShocGridData(10, 71, 72), ShocGridData(10, 12, 13), ShocGridData(7, 16, 17), ShocGridData(2, 7, 8), }; + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(ShocGridData); + // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ShocGridData cxx_data[] = { - ShocGridData(f90_data[0]), - ShocGridData(f90_data[1]), - ShocGridData(f90_data[2]), - ShocGridData(f90_data[3]), + ShocGridData(baseline_data[0]), + ShocGridData(baseline_data[1]), + ShocGridData(baseline_data[2]), + ShocGridData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - shoc_grid(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - shoc_grid_f(d.shcol, d.nlev, d.nlevi, d.zt_grid, d.zi_grid, d.pdel, d.dz_zt, d.dz_zi, d.rho_zt); - d.transpose(); // go back to C layout + shoc_grid(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(ShocGridData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ShocGridData& d_f90 = f90_data[i]; + ShocGridData& d_baseline = baseline_data[i]; ShocGridData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.dz_zt); ++k) { - REQUIRE(d_f90.dz_zt[k] == d_cxx.dz_zt[k]); - REQUIRE(d_f90.rho_zt[k] == d_cxx.rho_zt[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.dz_zt); ++k) { + REQUIRE(d_baseline.dz_zt[k] == d_cxx.dz_zt[k]); + REQUIRE(d_baseline.rho_zt[k] == d_cxx.rho_zt[k]); } - for (Int k = 0; k < d_f90.total(d_f90.dz_zi); ++k) { - REQUIRE(d_f90.dz_zi[k] == d_cxx.dz_zi[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.dz_zi); ++k) { + REQUIRE(d_baseline.dz_zi[k] == d_cxx.dz_zi[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + cxx_data[i].write(Base::m_fid); + } } } // run_bfb }; @@ -198,14 +201,14 @@ TEST_CASE("shoc_grid_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocGrid; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_grid_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocGrid; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_impli_comp_tmpi_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_impli_comp_tmpi_tests.cpp index 53280dab17a..1616bff130d 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_impli_comp_tmpi_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_impli_comp_tmpi_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestImpCompTmpi { +struct UnitWrap::UnitTest::TestImpCompTmpi : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlevi = 6; @@ -87,9 +87,7 @@ struct UnitWrap::UnitTest::TestImpCompTmpi { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - compute_tmpi_f(SDS.nlevi, SDS.shcol, SDS.dtime, SDS.rho_zi, SDS.dz_zi, SDS.tmpi); - SDS.transpose(); // go back to C layout + compute_tmpi(SDS); // Verify result for(Int s = 0; s < shcol; ++s) { @@ -118,11 +116,11 @@ struct UnitWrap::UnitTest::TestImpCompTmpi { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ComputeTmpiData f90_data[] = { + ComputeTmpiData baseline_data[] = { // shcol, nlevi, dtime ComputeTmpiData(10, 72, 1), ComputeTmpiData(10, 13, 10), @@ -131,44 +129,49 @@ struct UnitWrap::UnitTest::TestImpCompTmpi { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ComputeTmpiData cxx_data[] = { - ComputeTmpiData(f90_data[0]), - ComputeTmpiData(f90_data[1]), - ComputeTmpiData(f90_data[2]), - ComputeTmpiData(f90_data[3]), + ComputeTmpiData(baseline_data[0]), + ComputeTmpiData(baseline_data[1]), + ComputeTmpiData(baseline_data[2]), + ComputeTmpiData(baseline_data[3]), }; + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(ComputeTmpiData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - compute_tmpi(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - compute_tmpi_f(d.nlevi, d.shcol, d.dtime, d.rho_zi, d.dz_zi, d.tmpi); - d.transpose(); // go back to C layout + compute_tmpi(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(ComputeTmpiData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ComputeTmpiData& d_f90 = f90_data[i]; + ComputeTmpiData& d_baseline = baseline_data[i]; ComputeTmpiData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.tmpi); ++k) { - REQUIRE(d_f90.tmpi[k] == d_cxx.tmpi[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.tmpi); ++k) { + REQUIRE(d_baseline.tmpi[k] == d_cxx.tmpi[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + cxx_data[i].write(Base::m_fid); + } } } }; @@ -183,14 +186,14 @@ TEST_CASE("shoc_imp_comp_tmpi_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestImpCompTmpi; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_imp_comp_tmpi_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestImpCompTmpi; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_impli_dp_inverse_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_impli_dp_inverse_tests.cpp index aad9c026686..b5feb285f55 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_impli_dp_inverse_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_impli_dp_inverse_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestImpDpInverse { +struct UnitWrap::UnitTest::TestImpDpInverse : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 5; @@ -74,9 +74,7 @@ struct UnitWrap::UnitTest::TestImpDpInverse { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - dp_inverse_f(SDS.nlev, SDS.shcol, SDS.rho_zt, SDS.dz_zt, SDS.rdp_zt); - SDS.transpose(); // go back to C layout + dp_inverse(SDS); // Verify result for(Int s = 0; s < shcol; ++s) { @@ -98,11 +96,11 @@ struct UnitWrap::UnitTest::TestImpDpInverse { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - DpInverseData f90_data[] = { + DpInverseData baseline_data[] = { // shcol, nlev DpInverseData(10, 71), DpInverseData(10, 12), @@ -111,44 +109,49 @@ struct UnitWrap::UnitTest::TestImpDpInverse { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state DpInverseData cxx_data[] = { - DpInverseData(f90_data[0]), - DpInverseData(f90_data[1]), - DpInverseData(f90_data[2]), - DpInverseData(f90_data[3]), + DpInverseData(baseline_data[0]), + DpInverseData(baseline_data[1]), + DpInverseData(baseline_data[2]), + DpInverseData(baseline_data[3]), }; + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(DpInverseData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - dp_inverse(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - dp_inverse_f(d.nlev, d.shcol, d.rho_zt, d.dz_zt, d.rdp_zt); - d.transpose(); // go back to C layout + dp_inverse(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(DpInverseData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - DpInverseData& d_f90 = f90_data[i]; + DpInverseData& d_baseline = baseline_data[i]; DpInverseData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.rdp_zt); ++k) { - REQUIRE(d_f90.rdp_zt[k] == d_cxx.rdp_zt[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.rdp_zt); ++k) { + REQUIRE(d_baseline.rdp_zt[k] == d_cxx.rdp_zt[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + cxx_data[i].write(Base::m_fid); + } } } }; @@ -163,14 +166,14 @@ TEST_CASE("shoc_imp_dp_inverse_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestImpDpInverse; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_imp_dp_inverse_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestImpDpInverse; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_impli_sfc_fluxes_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_impli_sfc_fluxes_tests.cpp deleted file mode 100644 index 8f8f25cdd4a..00000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_impli_sfc_fluxes_tests.cpp +++ /dev/null @@ -1,202 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestImpSfcFluxes { - - static void run_property() - { - static constexpr Int shcol = 5; - static constexpr Int num_tracer = 10; - - // Tests for the SHOC subroutine - // sfc_fluxes - - // TEST - // Feed in several columns worth of data and make sure - // the output is consistent. - - // Surface density on the zi grid [kg/m3] - static constexpr Real rho_zi_sfc[shcol] = {1.2, 1.0, 0.9, 1.1, 1.15}; - // Rdp value on zt grid [ms^2/kg], same for all columns - static constexpr Real rdp_zt_sfc = 8.5e-3; - // heat flux at surface [K m/s] - static constexpr Real wthl_sfc[shcol] = {0.03, -0.03, 0.1, 0, -0.1}; - // moisture flux at surface [kg/kg m/s] - static constexpr Real wqw_sfc[shcol] = {2e-5, 1e-6, 0, -2e-5, 1e-4}; - // TKE flux at the surface [m3/s3] - static constexpr Real wtke_sfc[shcol] = {4e-2, 1e-3, -2e-3, 0, -1e-3}; - - // Supply input values - // liquid water potential temperature [K] - static constexpr Real thetal_in = 300; - // total water mixing ratio [kg/kg] - static constexpr Real qw_in = 0.015; - // turbulent kinetic energy [m2/s2] - static constexpr Real tke_in = 0.4; - - // time step [s] - static constexpr Real dtime = 300; - - // Input for tracer (no units) - Real tracer_in[num_tracer]; - - // Feed tracer random data from 1 to 1000 - for(Int t = 0; t < num_tracer; ++t) { - tracer_in[t] = rand()% 1000 + 1; - } - - // Initialize data structure for bridging to F90 - SfcFluxesData SDS(shcol, num_tracer, dtime); - - // Test that the inputs are reasonable. - REQUIRE(SDS.shcol == shcol); - REQUIRE(SDS.num_tracer == num_tracer); - REQUIRE(shcol > 1); - - // Fill in test data, column only - for(Int s = 0; s < shcol; ++s) { - SDS.rho_zi_sfc[s] = rho_zi_sfc[s]; - SDS.rdp_zt_sfc[s] = rdp_zt_sfc; - SDS.wthl_sfc[s] = wthl_sfc[s]; - SDS.wqw_sfc[s] = wqw_sfc[s]; - SDS.wtke_sfc[s] = wtke_sfc[s]; - - SDS.thetal[s] = thetal_in; - SDS.qw[s] = qw_in; - SDS.tke[s] = tke_in; - - for (Int t = 0; t < num_tracer; ++t){ - const auto offset = t + s * num_tracer; - SDS.wtracer[offset] = tracer_in[t]; - // Feed tracer flux random data from -100 to 100 - // note this is different for every point - SDS.wtracer_sfc[offset] = rand()% 200 + (-100); - } - - } - - // Check that the inputs make sense - for(Int s = 0; s < shcol; ++s) { - REQUIRE( (SDS.thetal[s] > 150 && SDS.thetal[s] < 350) ); - REQUIRE( (SDS.qw[s] > 0.0001 && SDS.qw[s] < 0.05) ); - REQUIRE( (SDS.tke[s] > 0 && SDS.tke[s] < 10) ); - REQUIRE( (SDS.rdp_zt_sfc[s] > 0 && SDS.rdp_zt_sfc[s] < 1) ); - REQUIRE( (SDS.rho_zi_sfc[s] > 0 && SDS.rho_zi_sfc[s] < 2) ); - REQUIRE(std::abs(SDS.wthl_sfc[s]) < 1); - REQUIRE(std::abs(SDS.wqw_sfc[s]) < 1e-3); - REQUIRE(std::abs(SDS.wtke_sfc[s]) < 0.1); - REQUIRE(SDS.dtime > 0); - } - - // Call the fortran implementation - sfc_fluxes(SDS); - - // Verify that output is reasonable - for(Int s = 0; s < shcol; ++s) { - - // Verify output falls within reasonable bounds - REQUIRE( (SDS.thetal[s] > 150 && SDS.thetal[s] < 350) ); - REQUIRE( (SDS.qw[s] > 0.0001 && SDS.qw[s] < 0.05) ); - REQUIRE( (SDS.tke[s] > 0 && SDS.tke[s] < 10) ); - - // Based on surface flux input, make sure that - // temperature, moisture, and tke all have output - // that is expected with respect to the input - - // Check temperature - if (wthl_sfc[s] > 0){ - REQUIRE(SDS.thetal[s] > thetal_in); - } - else if (wthl_sfc[s] < 0){ - REQUIRE(SDS.thetal[s] < thetal_in); - } - else{ - REQUIRE(SDS.thetal[s] == thetal_in); - } - - // Check moisture - if (wqw_sfc[s] > 0){ - REQUIRE(SDS.qw[s] > qw_in); - } - else if (wqw_sfc[s] < 0){ - REQUIRE(SDS.qw[s] < qw_in); - } - else{ - REQUIRE(SDS.qw[s] == qw_in); - } - - // Check TKE - if (wtke_sfc[s] > 0){ - REQUIRE(SDS.tke[s] > tke_in); - } - else if (wtke_sfc[s] < 0){ - REQUIRE(SDS.tke[s] < tke_in); - } - else{ - REQUIRE(SDS.tke[s] == tke_in); - } - - // Check tracer - for (Int t = 0; t < num_tracer; ++t){ - const auto offset = t + s * num_tracer; - if (SDS.wtracer_sfc[offset] > 0){ - REQUIRE(SDS.wtracer[offset] > tracer_in[t]); - } - else if (SDS.wtracer_sfc[offset] < 0){ - REQUIRE(SDS.wtracer[offset] < tracer_in[t]); - } - else{ - REQUIRE(SDS.wtracer[offset] == tracer_in[t]); - } - } - - } - - } - - static void run_bfb() - { - // TODO - } -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace { - -TEST_CASE("shoc_imp_sfc_fluxes_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestImpSfcFluxes; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_imp_sfc_fluxes_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestImpSfcFluxes; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_impli_srf_stress_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_impli_srf_stress_tests.cpp deleted file mode 100644 index 97f6d2a0be3..00000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_impli_srf_stress_tests.cpp +++ /dev/null @@ -1,131 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestImpSfcStress { - - static void run_property() - { - static constexpr Int shcol = 5; - - // Tests for the SHOC subroutine - // impli_srf_stress_term - - // TEST ONE - // Feed in several columns worth of data and make sure - // the output is consistent. - - // Surface density on the zi grid [kg/m3] - static constexpr Real rho_zi_sfc[shcol] = {1.2, 1.0, 0.9, 1.1, 1.15}; - // Surface moment flux, zonal direction [m3/s3] - static constexpr Real uw_sfc[shcol] = {0.03, -0.03, 0.1, 0, -0.1}; - // Surface moment flux, meridional direction [m3/s3] - static constexpr Real vw_sfc[shcol] = {-0.01, -0.01, 0.3, 0, -0.3}; - // Surface wind, zonal direction [m/s] - static constexpr Real u_wind_sfc[shcol] = {5, -5, 0, 2, -10}; - // Surface wind, meridional direction [m/s] - static constexpr Real v_wind_sfc[shcol] = {-10, 2, 20, 0, 1}; - - // Initialize data structure for bridging to F90 - ImpliSrfStressTermData SDS(shcol); - - // Test that the inputs are reasonable. - REQUIRE(SDS.shcol == shcol); - REQUIRE(shcol > 1); - - // Fill in test data, column only - for(Int s = 0; s < shcol; ++s) { - SDS.rho_zi_sfc[s] = rho_zi_sfc[s]; - SDS.uw_sfc[s] = uw_sfc[s]; - SDS.vw_sfc[s] = vw_sfc[s]; - SDS.u_wind_sfc[s] = u_wind_sfc[s]; - SDS.v_wind_sfc[s] = v_wind_sfc[s]; - } - - // Call the fortran implementation - impli_srf_stress_term(SDS); - - // Verify that output is reasonable - for(Int s = 0; s < shcol; ++s) { - // term should be greater than zero and less than one given - // reasonable input values - REQUIRE( (SDS.ksrf[s] > 0 && SDS.ksrf[s] < 1) ); - } - - // TEST TWO - // Given inputs that are identical but the absolute value of - // the surface fluxes are INCREASING, verify ksrf value is larger. - // Can recycle input from surface fluxes from last test. - - // Fill in test data, column only - for(Int s = 0; s < shcol; ++s) { - SDS.rho_zi_sfc[s] = 1.2; // density [kg/m3] - SDS.u_wind_sfc[s] = 5; // zonal wind [m/s] - SDS.v_wind_sfc[s] = -10; // meridional wind [m/s] - } - - // Call the fortran implementation - impli_srf_stress_term(SDS); - - Real stress1, stress2; - // Verify that output is as expected and reasonable - for(Int s = 0; s < shcol; ++s) { - // term should be greater than zero and less than one given - // reasonable input values - REQUIRE( (SDS.ksrf[s] > 0 && SDS.ksrf[s] < 1) ); - if (s < shcol-1){ - stress1 = uw_sfc[s]*uw_sfc[s] + uw_sfc[s]*uw_sfc[s]; - stress2 = uw_sfc[s+1]*uw_sfc[s+1] + uw_sfc[s+1]*uw_sfc[s+1]; - if (stress1 > stress2){ - REQUIRE(SDS.ksrf[s] > SDS.ksrf[s+1]); - } - } - } - - } - - static void run_bfb() - { - // TODO - } -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace { - -TEST_CASE("shoc_imp_sfc_stress_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestImpSfcStress; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_imp_sfc_stress_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestImpSfcStress; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_impli_srf_tke_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_impli_srf_tke_tests.cpp deleted file mode 100644 index bb19e21e30c..00000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_impli_srf_tke_tests.cpp +++ /dev/null @@ -1,101 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestImpTkeSfcStress { - - static void run_property() - { - static constexpr Int shcol = 5; - - // Tests for the SHOC subroutine - // impli_srf_stress_term - - // TEST ONE - // Feed in several columns worth of data and make sure - // the output is consistent. Make sure that columns with higher - // surface stress result in greater surface tke flux. - - // Surface moment flux, zonal direction [m3/s3] - static constexpr Real uw_sfc[shcol] = {0.03, -0.03, 0.1, 0, -0.1}; - // Surface moment flux, meridional direction [m3/s3] - static constexpr Real vw_sfc[shcol] = {-0.01, -0.01, 0.3, 0, -0.3}; - - // Initialize data structure for bridging to F90 - TkeSrfFluxTermData SDS(shcol); - - // Test that the inputs are reasonable. - REQUIRE(SDS.shcol == shcol); - REQUIRE(shcol > 1); - - // Fill in test data, column only - for(Int s = 0; s < shcol; ++s) { - SDS.uw_sfc[s] = uw_sfc[s]; - SDS.vw_sfc[s] = vw_sfc[s]; - } - - // Call the fortran implementation - tke_srf_flux_term(SDS); - - Real stress1, stress2; - // Verify that output is as expected and reasonable - for(Int s = 0; s < shcol; ++s) { - // term should be greater than zero and less than one given - // reasonable input values - REQUIRE(SDS.wtke_sfc[s] > 0); - if (s < shcol-1){ - stress1 = uw_sfc[s]*uw_sfc[s] + uw_sfc[s]*uw_sfc[s]; - stress2 = uw_sfc[s+1]*uw_sfc[s+1] + uw_sfc[s+1]*uw_sfc[s+1]; - if (stress1 > stress2){ - REQUIRE(SDS.wtke_sfc[s] > SDS.wtke_sfc[s+1]); - } - } - } - - } - - static void run_bfb() - { - // TODO - } -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace { - -TEST_CASE("shoc_imp_tkesfc_stress_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestImpTkeSfcStress; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_imp_tkesfc_stress_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestImpTkeSfcStress; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_l_inf_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_l_inf_length_tests.cpp index 84faaebe4b9..db769d17010 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_l_inf_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_l_inf_length_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestLInfShocLength { +struct UnitWrap::UnitTest::TestLInfShocLength : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 3; static constexpr Int nlev = 5; @@ -89,10 +89,7 @@ struct UnitWrap::UnitTest::TestLInfShocLength { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - compute_l_inf_shoc_length_f(SDS.nlev,SDS.shcol,SDS.zt_grid,SDS.dz_zt,SDS.tke,SDS.l_inf); - SDS.transpose(); + compute_l_inf_shoc_length(SDS); // Check the results // Make sure result is bounded correctly @@ -108,11 +105,11 @@ struct UnitWrap::UnitTest::TestLInfShocLength { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ComputeLInfShocLengthData SDS_f90[] = { + ComputeLInfShocLengthData SDS_baseline[] = { // shcol, nlev ComputeLInfShocLengthData(10, 71), ComputeLInfShocLengthData(10, 12), @@ -121,45 +118,49 @@ struct UnitWrap::UnitTest::TestLInfShocLength { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ComputeLInfShocLengthData SDS_cxx[] = { - ComputeLInfShocLengthData(SDS_f90[0]), - ComputeLInfShocLengthData(SDS_f90[1]), - ComputeLInfShocLengthData(SDS_f90[2]), - ComputeLInfShocLengthData(SDS_f90[3]), + ComputeLInfShocLengthData(SDS_baseline[0]), + ComputeLInfShocLengthData(SDS_baseline[1]), + ComputeLInfShocLengthData(SDS_baseline[2]), + ComputeLInfShocLengthData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(ComputeLInfShocLengthData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - compute_l_inf_shoc_length(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - compute_l_inf_shoc_length_f(d.nlev,d.shcol,d.zt_grid,d.dz_zt,d.tke,d.l_inf); - d.transpose(); + compute_l_inf_shoc_length(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(ComputeLInfShocLengthData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ComputeLInfShocLengthData& d_f90 = SDS_f90[i]; + ComputeLInfShocLengthData& d_baseline = SDS_baseline[i]; ComputeLInfShocLengthData& d_cxx = SDS_cxx[i]; - for (Int c = 0; c < d_f90.shcol; ++c) { - REQUIRE(d_f90.l_inf[c] == d_cxx.l_inf[c]); + for (Int c = 0; c < d_baseline.shcol; ++c) { + REQUIRE(d_baseline.l_inf[c] == d_cxx.l_inf[c]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -174,14 +175,14 @@ TEST_CASE("shoc_l_inf_length_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestLInfShocLength; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_l_inf_length_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestLInfShocLength; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_length_tests.cpp index 0f890db9ce1..b492f6a8e4b 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_length_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocLength { +struct UnitWrap::UnitTest::TestShocLength : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Real minlen = scream::shoc::Constants::minlen; static constexpr Real maxlen = scream::shoc::Constants::maxlen; @@ -121,12 +121,7 @@ struct UnitWrap::UnitTest::TestShocLength { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - shoc_length_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.host_dx,SDS.host_dy, - SDS.zt_grid,SDS.zi_grid,SDS.dz_zt,SDS.tke, - SDS.thv,SDS.brunt,SDS.shoc_mix); - SDS.transpose(); + shoc_length(SDS); // Verify output for(Int s = 0; s < shcol; ++s) { @@ -175,12 +170,7 @@ struct UnitWrap::UnitTest::TestShocLength { } // call C++ implentation - SDS.transpose(); - // expects data in fortran layout - shoc_length_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.host_dx,SDS.host_dy, - SDS.zt_grid,SDS.zi_grid,SDS.dz_zt,SDS.tke, - SDS.thv,SDS.brunt,SDS.shoc_mix); - SDS.transpose(); + shoc_length(SDS); // Verify output for(Int s = 0; s < shcol; ++s) { @@ -196,11 +186,11 @@ struct UnitWrap::UnitTest::TestShocLength { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ShocLengthData SDS_f90[] = { + ShocLengthData SDS_baseline[] = { // shcol, nlev, nlevi ShocLengthData(12, 71, 72), ShocLengthData(10, 12, 13), @@ -209,48 +199,50 @@ struct UnitWrap::UnitTest::TestShocLength { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ShocLengthData SDS_cxx[] = { - ShocLengthData(SDS_f90[0]), - ShocLengthData(SDS_f90[1]), - ShocLengthData(SDS_f90[2]), - ShocLengthData(SDS_f90[3]), + ShocLengthData(SDS_baseline[0]), + ShocLengthData(SDS_baseline[1]), + ShocLengthData(SDS_baseline[2]), + ShocLengthData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(ShocLengthData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - shoc_length(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - shoc_length_f(d.shcol,d.nlev,d.nlevi,d.host_dx,d.host_dy, - d.zt_grid,d.zi_grid,d.dz_zt,d.tke, - d.thv,d.brunt,d.shoc_mix); - d.transpose(); + shoc_length(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(ShocLengthData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ShocLengthData& d_f90 = SDS_f90[i]; + ShocLengthData& d_baseline = SDS_baseline[i]; ShocLengthData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.brunt); ++k) { - REQUIRE(d_f90.brunt[k] == d_cxx.brunt[k]); - REQUIRE(d_f90.shoc_mix[k] == d_cxx.shoc_mix[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.brunt); ++k) { + REQUIRE(d_baseline.brunt[k] == d_cxx.brunt[k]); + REQUIRE(d_baseline.shoc_mix[k] == d_cxx.shoc_mix[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -265,14 +257,14 @@ TEST_CASE("shoc_length_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocLength; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_length_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocLength; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_linear_interp_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_linear_interp_tests.cpp index 0a9c666688b..6be406bda09 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_linear_interp_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_linear_interp_tests.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -22,9 +22,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocLinearInt { +struct UnitWrap::UnitTest::TestShocLinearInt : public UnitWrap::UnitTest::Base { - static void run_property_fixed() + void run_property_fixed() { static constexpr Int shcol = 2; static constexpr Int km1 = 5; @@ -100,9 +100,7 @@ struct UnitWrap::UnitTest::TestShocLinearInt { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - linear_interp_f(SDS.x1, SDS.x2, SDS.y1, SDS.y2, SDS.km1, SDS.km2, SDS.ncol, SDS.minthresh); - SDS.transpose(); // go back to C layout + linear_interp(SDS); // First check that all output temperatures are greater than zero @@ -194,9 +192,7 @@ struct UnitWrap::UnitTest::TestShocLinearInt { } // Call the C++ implementation - SDS2.transpose(); // _f expects data in fortran layout - linear_interp_f(SDS2.x1, SDS2.x2, SDS2.y1, SDS2.y2, SDS2.km1, SDS2.km2, SDS2.ncol, SDS2.minthresh); - SDS2.transpose(); // go back to C layout + linear_interp(SDS2); // Check the result, make sure output is bounded correctly @@ -223,7 +219,7 @@ struct UnitWrap::UnitTest::TestShocLinearInt { } } - static void run_property_random(bool km1_bigger) + void run_property_random(bool km1_bigger) { std::default_random_engine generator; std::pair km1_range = {13, 25}; @@ -281,9 +277,7 @@ struct UnitWrap::UnitTest::TestShocLinearInt { } // Call the C++ implementation - d.transpose(); // _f expects data in fortran layout - linear_interp_f(d.x1, d.x2, d.y1, d.y2, d.km1, d.km2, d.ncol, d.minthresh); - d.transpose(); // go back to C layout + linear_interp(d); // The combination of single-precision and randomness generating points // close together can result in larger error margins. @@ -333,18 +327,18 @@ struct UnitWrap::UnitTest::TestShocLinearInt { } } - static void run_property() + void run_property() { run_property_fixed(); run_property_random(true); run_property_random(false); } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - LinearInterpData f90_data[] = { + LinearInterpData baseline_data[] = { // shcol, nlev(km1), nlevi(km2), minthresh LinearInterpData(10, 72, 71, 1e-15), LinearInterpData(10, 71, 72, 1e-15), @@ -355,46 +349,50 @@ struct UnitWrap::UnitTest::TestShocLinearInt { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state LinearInterpData cxx_data[] = { - LinearInterpData(f90_data[0]), - LinearInterpData(f90_data[1]), - LinearInterpData(f90_data[2]), - LinearInterpData(f90_data[3]), - LinearInterpData(f90_data[4]), - LinearInterpData(f90_data[5]), + LinearInterpData(baseline_data[0]), + LinearInterpData(baseline_data[1]), + LinearInterpData(baseline_data[2]), + LinearInterpData(baseline_data[3]), + LinearInterpData(baseline_data[4]), + LinearInterpData(baseline_data[5]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - linear_interp(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - linear_interp_f(d.x1, d.x2, d.y1, d.y2, d.km1, d.km2, d.ncol, d.minthresh); - d.transpose(); // go back to C layout + linear_interp(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(LinearInterpData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(LinearInterpData); for (Int i = 0; i < num_runs; ++i) { - LinearInterpData& d_f90 = f90_data[i]; + LinearInterpData& d_baseline = baseline_data[i]; LinearInterpData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.y2); ++k) { - REQUIRE(d_f90.y2[k] == d_cxx.y2[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.y2); ++k) { + REQUIRE(d_baseline.y2[k] == d_cxx.y2[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb @@ -410,14 +408,14 @@ TEST_CASE("shoc_linear_interp_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocLinearInt; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_linear_interp_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocLinearInt; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_main_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_main_tests.cpp index 5c3eb8f9860..b007da41322 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_main_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_main_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,9 +14,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocMain { +struct UnitWrap::UnitTest::TestShocMain : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Real mintke = scream::shoc::Constants::mintke; static constexpr Real minlen = scream::shoc::Constants::minlen; @@ -259,18 +259,7 @@ struct UnitWrap::UnitTest::TestShocMain { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - const int npbl = shoc_init_f(SDS.nlev, SDS.pref_mid, SDS.nbot_shoc, SDS.ntop_shoc); - - shoc_main_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.nadv, npbl, SDS.host_dx, SDS.host_dy, - SDS.thv, SDS.zt_grid, SDS.zi_grid, SDS.pres, SDS.presi, SDS.pdel, SDS.wthl_sfc, - SDS.wqw_sfc, SDS.uw_sfc, SDS.vw_sfc, SDS.wtracer_sfc, SDS.num_qtracers, - SDS.w_field, SDS.inv_exner, SDS.phis, SDS.host_dse, SDS.tke, SDS.thetal, SDS.qw, - SDS.u_wind, SDS.v_wind, SDS.qtracers, SDS.wthv_sec, SDS.tkh, SDS.tk, SDS.shoc_ql, - SDS.shoc_cldfrac, SDS.pblh, SDS.shoc_mix, SDS.isotropy, SDS.w_sec, SDS.thl_sec, - SDS.qw_sec, SDS.qwthl_sec, SDS.wthl_sec, SDS.wqw_sec, SDS.wtke_sec, SDS.uw_sec, - SDS.vw_sec, SDS.w3, SDS.wqls_sec, SDS.brunt, SDS.shoc_ql2); - SDS.transpose(); // go back to C layout + shoc_main(SDS); // Make sure output falls within reasonable bounds for(Int s = 0; s < shcol; ++s) { @@ -339,11 +328,11 @@ struct UnitWrap::UnitTest::TestShocMain { } // run_property - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ShocMainData f90_data[] = { + ShocMainData baseline_data[] = { // shcol, nlev, nlevi, num_qtracers, dtime, nadv, nbot_shoc, ntop_shoc(C++ indexing) ShocMainData(12, 72, 73, 5, 300, 15, 72, 0), ShocMainData(8, 12, 13, 3, 300, 10, 8, 3), @@ -352,7 +341,7 @@ struct UnitWrap::UnitTest::TestShocMain { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine, { {d.presi, {700e2,1000e2}}, @@ -374,113 +363,108 @@ struct UnitWrap::UnitTest::TestShocMain { }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ShocMainData cxx_data[] = { - ShocMainData(f90_data[0]), - ShocMainData(f90_data[1]), - ShocMainData(f90_data[2]), - ShocMainData(f90_data[3]) + ShocMainData(baseline_data[0]), + ShocMainData(baseline_data[1]), + ShocMainData(baseline_data[2]), + ShocMainData(baseline_data[3]) }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - shoc_main_with_init(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - const int npbl = shoc_init_f(d.nlev, d.pref_mid, d.nbot_shoc, d.ntop_shoc); - - shoc_main_f(d.shcol, d.nlev, d.nlevi, d.dtime, d.nadv, npbl, d.host_dx, d.host_dy, - d.thv, d.zt_grid, d.zi_grid, d.pres, d.presi, d.pdel, d.wthl_sfc, - d.wqw_sfc, d.uw_sfc, d.vw_sfc, d.wtracer_sfc, d.num_qtracers, - d.w_field, d.inv_exner, d.phis, d.host_dse, d.tke, d.thetal, d.qw, - d.u_wind, d.v_wind, d.qtracers, d.wthv_sec, d.tkh, d.tk, d.shoc_ql, - d.shoc_cldfrac, d.pblh, d.shoc_mix, d.isotropy, d.w_sec, d.thl_sec, - d.qw_sec, d.qwthl_sec, d.wthl_sec, d.wqw_sec, d.wtke_sec, d.uw_sec, - d.vw_sec, d.w3, d.wqls_sec, d.brunt, d.shoc_ql2); - d.transpose(); // go back to C layout + shoc_main(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(ShocMainData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(ShocMainData); for (Int i = 0; i < num_runs; ++i) { - ShocMainData& d_f90 = f90_data[i]; + ShocMainData& d_baseline = baseline_data[i]; ShocMainData& d_cxx = cxx_data[i]; - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.host_dse)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.tke)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.thetal)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.qw)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.u_wind)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.v_wind)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.wthv_sec)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.tkh)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.tk)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.shoc_ql)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.shoc_cldfrac)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.shoc_mix)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.isotropy)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.w_sec)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.wqls_sec)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.brunt)); - REQUIRE(d_f90.total(d_f90.host_dse) == d_cxx.total(d_cxx.shoc_ql2)); - for (Int k = 0; k < d_f90.total(d_f90.host_dse); ++k) { - REQUIRE(d_f90.host_dse[k] == d_cxx.host_dse[k]); - REQUIRE(d_f90.tke[k] == d_cxx.tke[k]); - REQUIRE(d_f90.thetal[k] == d_cxx.thetal[k]); - REQUIRE(d_f90.qw[k] == d_cxx.qw[k]); - REQUIRE(d_f90.u_wind[k] == d_cxx.u_wind[k]); - REQUIRE(d_f90.v_wind[k] == d_cxx.v_wind[k]); - REQUIRE(d_f90.wthv_sec[k] == d_cxx.wthv_sec[k]); - REQUIRE(d_f90.tk[k] == d_cxx.tk[k]); - REQUIRE(d_f90.shoc_ql[k] == d_cxx.shoc_ql[k]); - REQUIRE(d_f90.shoc_cldfrac[k] == d_cxx.shoc_cldfrac[k]); - REQUIRE(d_f90.shoc_mix[k] == d_cxx.shoc_mix[k]); - REQUIRE(d_f90.isotropy[k] == d_cxx.isotropy[k]); - REQUIRE(d_f90.w_sec[k] == d_cxx.w_sec[k]); - REQUIRE(d_f90.wqls_sec[k] == d_cxx.wqls_sec[k]); - REQUIRE(d_f90.brunt[k] == d_cxx.brunt[k]); - REQUIRE(d_f90.shoc_ql2[k] == d_cxx.shoc_ql2[k]); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.host_dse)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.tke)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.thetal)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.qw)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.u_wind)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.v_wind)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.wthv_sec)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.tkh)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.tk)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.shoc_ql)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.shoc_cldfrac)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.shoc_mix)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.isotropy)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.w_sec)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.wqls_sec)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.brunt)); + REQUIRE(d_baseline.total(d_baseline.host_dse) == d_cxx.total(d_cxx.shoc_ql2)); + for (Int k = 0; k < d_baseline.total(d_baseline.host_dse); ++k) { + REQUIRE(d_baseline.host_dse[k] == d_cxx.host_dse[k]); + REQUIRE(d_baseline.tke[k] == d_cxx.tke[k]); + REQUIRE(d_baseline.thetal[k] == d_cxx.thetal[k]); + REQUIRE(d_baseline.qw[k] == d_cxx.qw[k]); + REQUIRE(d_baseline.u_wind[k] == d_cxx.u_wind[k]); + REQUIRE(d_baseline.v_wind[k] == d_cxx.v_wind[k]); + REQUIRE(d_baseline.wthv_sec[k] == d_cxx.wthv_sec[k]); + REQUIRE(d_baseline.tk[k] == d_cxx.tk[k]); + REQUIRE(d_baseline.shoc_ql[k] == d_cxx.shoc_ql[k]); + REQUIRE(d_baseline.shoc_cldfrac[k] == d_cxx.shoc_cldfrac[k]); + REQUIRE(d_baseline.shoc_mix[k] == d_cxx.shoc_mix[k]); + REQUIRE(d_baseline.isotropy[k] == d_cxx.isotropy[k]); + REQUIRE(d_baseline.w_sec[k] == d_cxx.w_sec[k]); + REQUIRE(d_baseline.wqls_sec[k] == d_cxx.wqls_sec[k]); + REQUIRE(d_baseline.brunt[k] == d_cxx.brunt[k]); + REQUIRE(d_baseline.shoc_ql2[k] == d_cxx.shoc_ql2[k]); } - REQUIRE(d_f90.total(d_f90.qtracers) == d_cxx.total(d_cxx.qtracers)); - for (Int k = 0; k < d_f90.total(d_f90.qtracers); ++k) { - REQUIRE(d_f90.qtracers[k] == d_cxx.qtracers[k]); + REQUIRE(d_baseline.total(d_baseline.qtracers) == d_cxx.total(d_cxx.qtracers)); + for (Int k = 0; k < d_baseline.total(d_baseline.qtracers); ++k) { + REQUIRE(d_baseline.qtracers[k] == d_cxx.qtracers[k]); } - REQUIRE(d_f90.total(d_f90.pblh) == d_cxx.total(d_cxx.pblh)); - for (Int k = 0; k < d_f90.total(d_f90.pblh); ++k) { - REQUIRE(d_f90.pblh[k] == d_cxx.pblh[k]); + REQUIRE(d_baseline.total(d_baseline.pblh) == d_cxx.total(d_cxx.pblh)); + for (Int k = 0; k < d_baseline.total(d_baseline.pblh); ++k) { + REQUIRE(d_baseline.pblh[k] == d_cxx.pblh[k]); } - REQUIRE(d_f90.total(d_f90.thl_sec) == d_cxx.total(d_cxx.thl_sec)); - REQUIRE(d_f90.total(d_f90.thl_sec) == d_cxx.total(d_cxx.qw_sec)); - REQUIRE(d_f90.total(d_f90.thl_sec) == d_cxx.total(d_cxx.qwthl_sec)); - REQUIRE(d_f90.total(d_f90.thl_sec) == d_cxx.total(d_cxx.wthl_sec)); - REQUIRE(d_f90.total(d_f90.thl_sec) == d_cxx.total(d_cxx.wqw_sec)); - REQUIRE(d_f90.total(d_f90.thl_sec) == d_cxx.total(d_cxx.wtke_sec)); - REQUIRE(d_f90.total(d_f90.thl_sec) == d_cxx.total(d_cxx.uw_sec)); - REQUIRE(d_f90.total(d_f90.thl_sec) == d_cxx.total(d_cxx.vw_sec)); - REQUIRE(d_f90.total(d_f90.thl_sec) == d_cxx.total(d_cxx.w3)); - for (Int k = 0; k < d_f90.total(d_f90.thl_sec); ++k) { - REQUIRE(d_f90.thl_sec[k] == d_cxx.thl_sec[k]); - REQUIRE(d_f90.qw_sec[k] == d_cxx.qw_sec[k]); - REQUIRE(d_f90.qwthl_sec[k] == d_cxx.qwthl_sec[k]); - REQUIRE(d_f90.wthl_sec[k] == d_cxx.wthl_sec[k]); - REQUIRE(d_f90.wqw_sec[k] == d_cxx.wqw_sec[k]); - REQUIRE(d_f90.wtke_sec[k] == d_cxx.wtke_sec[k]); - REQUIRE(d_f90.uw_sec[k] == d_cxx.uw_sec[k]); - REQUIRE(d_f90.vw_sec[k] == d_cxx.vw_sec[k]); - REQUIRE(d_f90.w3[k] == d_cxx.w3[k]); + REQUIRE(d_baseline.total(d_baseline.thl_sec) == d_cxx.total(d_cxx.thl_sec)); + REQUIRE(d_baseline.total(d_baseline.thl_sec) == d_cxx.total(d_cxx.qw_sec)); + REQUIRE(d_baseline.total(d_baseline.thl_sec) == d_cxx.total(d_cxx.qwthl_sec)); + REQUIRE(d_baseline.total(d_baseline.thl_sec) == d_cxx.total(d_cxx.wthl_sec)); + REQUIRE(d_baseline.total(d_baseline.thl_sec) == d_cxx.total(d_cxx.wqw_sec)); + REQUIRE(d_baseline.total(d_baseline.thl_sec) == d_cxx.total(d_cxx.wtke_sec)); + REQUIRE(d_baseline.total(d_baseline.thl_sec) == d_cxx.total(d_cxx.uw_sec)); + REQUIRE(d_baseline.total(d_baseline.thl_sec) == d_cxx.total(d_cxx.vw_sec)); + REQUIRE(d_baseline.total(d_baseline.thl_sec) == d_cxx.total(d_cxx.w3)); + for (Int k = 0; k < d_baseline.total(d_baseline.thl_sec); ++k) { + REQUIRE(d_baseline.thl_sec[k] == d_cxx.thl_sec[k]); + REQUIRE(d_baseline.qw_sec[k] == d_cxx.qw_sec[k]); + REQUIRE(d_baseline.qwthl_sec[k] == d_cxx.qwthl_sec[k]); + REQUIRE(d_baseline.wthl_sec[k] == d_cxx.wthl_sec[k]); + REQUIRE(d_baseline.wqw_sec[k] == d_cxx.wqw_sec[k]); + REQUIRE(d_baseline.wtke_sec[k] == d_cxx.wtke_sec[k]); + REQUIRE(d_baseline.uw_sec[k] == d_cxx.uw_sec[k]); + REQUIRE(d_baseline.vw_sec[k] == d_cxx.vw_sec[k]); + REQUIRE(d_baseline.w3[k] == d_cxx.w3[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb }; @@ -495,14 +479,14 @@ TEST_CASE("shoc_main_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocMain; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_main_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocMain; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_mix_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_mix_length_tests.cpp index 4d6b5dfdce1..88336e52ea4 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_mix_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_mix_length_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestCompShocMixLength { +struct UnitWrap::UnitTest::TestCompShocMixLength : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 3; static constexpr Int nlev = 5; @@ -91,13 +91,7 @@ struct UnitWrap::UnitTest::TestCompShocMixLength { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - compute_shoc_mix_shoc_length_f(SDS.nlev, SDS.shcol, - SDS.tke, SDS.brunt, - SDS.zt_grid, - SDS.l_inf, SDS.shoc_mix); - SDS.transpose(); + compute_shoc_mix_shoc_length(SDS); // Check the results for(Int s = 0; s < shcol; ++s) { @@ -120,11 +114,11 @@ struct UnitWrap::UnitTest::TestCompShocMixLength { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ComputeShocMixShocLengthData SDS_f90[] = { + ComputeShocMixShocLengthData SDS_baseline[] = { // shcol, nlev ComputeShocMixShocLengthData(10, 71), ComputeShocMixShocLengthData(10, 12), @@ -133,48 +127,49 @@ struct UnitWrap::UnitTest::TestCompShocMixLength { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ComputeShocMixShocLengthData SDS_cxx[] = { - ComputeShocMixShocLengthData(SDS_f90[0]), - ComputeShocMixShocLengthData(SDS_f90[1]), - ComputeShocMixShocLengthData(SDS_f90[2]), - ComputeShocMixShocLengthData(SDS_f90[3]), + ComputeShocMixShocLengthData(SDS_baseline[0]), + ComputeShocMixShocLengthData(SDS_baseline[1]), + ComputeShocMixShocLengthData(SDS_baseline[2]), + ComputeShocMixShocLengthData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(ComputeShocMixShocLengthData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - compute_shoc_mix_shoc_length(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - compute_shoc_mix_shoc_length_f(d.nlev, d.shcol, - d.tke, d.brunt, - d.zt_grid, - d.l_inf, d.shoc_mix); - d.transpose(); + compute_shoc_mix_shoc_length(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(ComputeShocMixShocLengthData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - ComputeShocMixShocLengthData& d_f90 = SDS_f90[i]; + ComputeShocMixShocLengthData& d_baseline = SDS_baseline[i]; ComputeShocMixShocLengthData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.shoc_mix); ++k) { - REQUIRE(d_f90.shoc_mix[k] == d_cxx.shoc_mix[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.shoc_mix); ++k) { + REQUIRE(d_baseline.shoc_mix[k] == d_cxx.shoc_mix[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -189,14 +184,14 @@ TEST_CASE("shoc_mix_length_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestCompShocMixLength; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_mix_length_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestCompShocMixLength; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_omega_diag_third_moms_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_omega_diag_third_moms_tests.cpp deleted file mode 100644 index e7fd4a507ae..00000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_omega_diag_third_moms_tests.cpp +++ /dev/null @@ -1,106 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestOmegadiagThirdMoms { - - static void run_property() - { - - // Tests for the SHOC function: - // omega_terms_diag_third_shoc_moment - - // TEST ONE - // Call the function twice, with two different sets of terms. - // In the second test the fterms should be larger. Verify - // that the omega0 and omega1 terms are the same, but that the - // omega2 term has increased. - - // buoyancy term (isotropy squared * brunt vaisalla frequency) - constexpr static Real buoy_sgs2 = -100; - // f3 term - constexpr static Real f3_test1a = 14; - // f4 term - constexpr static Real f4_test1a = 5; - - // Initialize data structure for bridging to F90 - OmegaTermsDiagThirdShocMomentData SDS; - - // Load up the data - SDS.buoy_sgs2 = buoy_sgs2; - SDS.f3 = f3_test1a; - SDS.f4 = f4_test1a; - - // Call the fortran implementation - omega_terms_diag_third_shoc_moment(SDS); - - // Save test results - Real omega0_test1a = SDS.omega0; - Real omega1_test1a = SDS.omega1; - Real omega2_test1a = SDS.omega2; - - // Now load up data for second part of test - // Feed in LARGER values - SDS.f3 = 2*f3_test1a; - SDS.f4 = 2*f4_test1a; - - // Call the fortran implementation - omega_terms_diag_third_shoc_moment(SDS); - - // Now check the result - - // omega0 and omega1 should NOT have changed - REQUIRE(SDS.omega0 == omega0_test1a); - REQUIRE(SDS.omega1 == omega1_test1a); - - // omega2 should have increased - REQUIRE(SDS.omega2 > omega2_test1a); - - } - - static void run_bfb() - { - // TODO - } - -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace{ - -TEST_CASE("shoc_omega_diag_third_moms_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestOmegadiagThirdMoms; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_omega_diag_third_moms_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestOmegadiagThirdMoms; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_check_pblh_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_check_pblh_tests.cpp index 2d5f01c8ff2..02489c97fd3 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_check_pblh_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_check_pblh_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,9 +14,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestPblintdCheckPblh { +struct UnitWrap::UnitTest::TestPblintdCheckPblh : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr auto ustar_min = scream::shoc::Constants::ustar_min; static constexpr Int shcol = 5; @@ -68,9 +68,7 @@ struct UnitWrap::UnitTest::TestPblintdCheckPblh { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - pblintd_check_pblh_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.nlev, SDS.z, SDS.ustar, SDS.check, SDS.pblh); - SDS.transpose(); // go back to C layout + pblintd_check_pblh(SDS); // Check the result // Check that PBL height is greater than zero. This is an @@ -80,11 +78,11 @@ struct UnitWrap::UnitTest::TestPblintdCheckPblh { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - PblintdCheckPblhData f90_data[] = { + PblintdCheckPblhData baseline_data[] = { PblintdCheckPblhData(36, 72, 73), PblintdCheckPblhData(72, 72, 73), PblintdCheckPblhData(128, 72, 73), @@ -92,46 +90,50 @@ struct UnitWrap::UnitTest::TestPblintdCheckPblh { }; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine, { {d.check, {1, 1}} }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state PblintdCheckPblhData cxx_data[] = { - PblintdCheckPblhData(f90_data[0]), - PblintdCheckPblhData(f90_data[1]), - PblintdCheckPblhData(f90_data[2]), - PblintdCheckPblhData(f90_data[3]), + PblintdCheckPblhData(baseline_data[0]), + PblintdCheckPblhData(baseline_data[1]), + PblintdCheckPblhData(baseline_data[2]), + PblintdCheckPblhData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - pblintd_check_pblh(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - pblintd_check_pblh_f(d.shcol, d.nlev, d.nlevi, d.nlev, d.z, d.ustar, d.check, d.pblh); - d.transpose(); // go back to C layout + pblintd_check_pblh(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(PblintdCheckPblhData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(PblintdCheckPblhData); for (Int i = 0; i < num_runs; ++i) { - PblintdCheckPblhData& d_f90 = f90_data[i]; + PblintdCheckPblhData& d_baseline = baseline_data[i]; PblintdCheckPblhData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.pblh); ++k) { - REQUIRE(d_f90.total(d_f90.pblh) == d_cxx.total(d_cxx.pblh)); - REQUIRE(d_f90.pblh[k] == d_cxx.pblh[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.pblh); ++k) { + REQUIRE(d_baseline.total(d_baseline.pblh) == d_cxx.total(d_cxx.pblh)); + REQUIRE(d_baseline.pblh[k] == d_cxx.pblh[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb @@ -147,14 +149,14 @@ TEST_CASE("pblintd_check_pblh_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintdCheckPblh; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("pblintd_check_pblh_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintdCheckPblh; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_cldcheck_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_cldcheck_tests.cpp index df65050acac..a0beea7e4ea 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_cldcheck_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_cldcheck_tests.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -22,9 +22,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestPblintdCldCheck { +struct UnitWrap::UnitTest::TestPblintdCldCheck : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 5; static constexpr Int nlev = 3; @@ -80,8 +80,7 @@ struct UnitWrap::UnitTest::TestPblintdCldCheck { } // Call the C++ implementation - SDS.transpose(); - shoc_pblintd_cldcheck_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.zi, SDS.cldn, SDS.pblh); + pblintd_cldcheck(SDS); // Check the result for(Int s = 0; s < shcol; ++s) { @@ -94,11 +93,11 @@ struct UnitWrap::UnitTest::TestPblintdCldCheck { } // run_property - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - PblintdCldcheckData cldcheck_data_f90[] = { + PblintdCldcheckData cldcheck_data_baseline[] = { // shcol, nlev, nlevi PblintdCldcheckData(36, 128, 129), PblintdCldcheckData(72, 128, 129), @@ -106,35 +105,41 @@ struct UnitWrap::UnitTest::TestPblintdCldCheck { PblintdCldcheckData(256, 128, 129), }; - for (auto& d : cldcheck_data_f90) { + for (auto& d : cldcheck_data_baseline) { d.randomize(engine); } PblintdCldcheckData cldcheck_data_cxx[] = { - PblintdCldcheckData(cldcheck_data_f90[0]), - PblintdCldcheckData(cldcheck_data_f90[1]), - PblintdCldcheckData(cldcheck_data_f90[2]), - PblintdCldcheckData(cldcheck_data_f90[3]), + PblintdCldcheckData(cldcheck_data_baseline[0]), + PblintdCldcheckData(cldcheck_data_baseline[1]), + PblintdCldcheckData(cldcheck_data_baseline[2]), + PblintdCldcheckData(cldcheck_data_baseline[3]), }; - for (auto& d : cldcheck_data_f90) { - // expects data in C layout - pblintd_cldcheck(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : cldcheck_data_baseline) { + d.read(Base::m_fid); + } } for (auto& d : cldcheck_data_cxx) { - d.transpose(); - shoc_pblintd_cldcheck_f(d.shcol, d.nlev, d.nlevi, d.zi, d.cldn, d.pblh); + pblintd_cldcheck(d); } - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(cldcheck_data_f90) / sizeof(PblintdCldcheckData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(cldcheck_data_baseline) / sizeof(PblintdCldcheckData); for (Int i = 0; i < num_runs; ++i) { const Int shcol = cldcheck_data_cxx[i].shcol; for (Int k = 0; k < shcol; ++k) { - REQUIRE(cldcheck_data_f90[i].pblh[k] == cldcheck_data_cxx[i].pblh[k]); + REQUIRE(cldcheck_data_baseline[i].pblh[k] == cldcheck_data_cxx[i].pblh[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cldcheck_data_cxx) { + d.write(Base::m_fid); + } } } // run_bfb @@ -149,14 +154,14 @@ namespace { TEST_CASE("shoc_pblintd_cldcheck_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintdCldCheck; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_pblintd_cldcheck_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintdCldCheck; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_height_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_height_tests.cpp index 9c33c7de68b..1a44d74782f 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_height_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_height_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,9 +14,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestPblintdHeight { +struct UnitWrap::UnitTest::TestPblintdHeight : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr auto ustar_min = scream::shoc::Constants::ustar_min; static const auto approx_zero = Approx(0.0).margin(1e-16); @@ -86,10 +86,7 @@ struct UnitWrap::UnitTest::TestPblintdHeight { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - pblintd_height_f(SDS.shcol, SDS.nlev, SDS.npbl, SDS.z, SDS.u, SDS.v, SDS.ustar, - SDS.thv, SDS.thv_ref, SDS.pblh, SDS.rino, SDS.check); - SDS.transpose(); // go back to C layout + pblintd_height(SDS); // Check the result for(Int s = 0; s < shcol; ++s) { @@ -123,10 +120,7 @@ struct UnitWrap::UnitTest::TestPblintdHeight { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - pblintd_height_f(SDS.shcol, SDS.nlev, SDS.npbl, SDS.z, SDS.u, SDS.v, SDS.ustar, - SDS.thv, SDS.thv_ref, SDS.pblh, SDS.rino, SDS.check); - SDS.transpose(); // go back to C layout + pblintd_height(SDS); // Check the result for(Int s = 0; s < shcol; ++s) { @@ -165,10 +159,7 @@ struct UnitWrap::UnitTest::TestPblintdHeight { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - pblintd_height_f(SDS.shcol, SDS.nlev, SDS.npbl, SDS.z, SDS.u, SDS.v, SDS.ustar, - SDS.thv, SDS.thv_ref, SDS.pblh, SDS.rino, SDS.check); - SDS.transpose(); // go back to C layout + pblintd_height(SDS); // Check that PBLH is zero (not modified) everywhere for(Int s = 0; s < shcol; ++s) { @@ -177,13 +168,13 @@ struct UnitWrap::UnitTest::TestPblintdHeight { } // run_property - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); Int npbl_rand = rand()%72 + 1; - PblintdHeightData f90_data[] = { + PblintdHeightData baseline_data[] = { PblintdHeightData(10, 72, 1), PblintdHeightData(10, 72, 72), PblintdHeightData(10, 72, npbl_rand), @@ -193,49 +184,53 @@ struct UnitWrap::UnitTest::TestPblintdHeight { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state PblintdHeightData cxx_data[] = { - PblintdHeightData(f90_data[0]), - PblintdHeightData(f90_data[1]), - PblintdHeightData(f90_data[2]), - PblintdHeightData(f90_data[3]), - PblintdHeightData(f90_data[4]), - PblintdHeightData(f90_data[5]), + PblintdHeightData(baseline_data[0]), + PblintdHeightData(baseline_data[1]), + PblintdHeightData(baseline_data[2]), + PblintdHeightData(baseline_data[3]), + PblintdHeightData(baseline_data[4]), + PblintdHeightData(baseline_data[5]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - pblintd_height(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - pblintd_height_f(d.shcol, d.nlev, d.npbl, d.z, d.u, d.v, d.ustar, d.thv, d.thv_ref, d.pblh, d.rino, d.check); - d.transpose(); // go back to C layout + pblintd_height(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(PblintdHeightData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(PblintdHeightData); for (Int i = 0; i < num_runs; ++i) { - PblintdHeightData& d_f90 = f90_data[i]; + PblintdHeightData& d_baseline = baseline_data[i]; PblintdHeightData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.pblh); ++k) { - REQUIRE(d_f90.total(d_f90.pblh) == d_cxx.total(d_cxx.pblh)); - REQUIRE(d_f90.pblh[k] == d_cxx.pblh[k]); - REQUIRE(d_f90.total(d_f90.pblh) == d_cxx.total(d_cxx.check)); - REQUIRE(d_f90.check[k] == d_cxx.check[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.pblh); ++k) { + REQUIRE(d_baseline.total(d_baseline.pblh) == d_cxx.total(d_cxx.pblh)); + REQUIRE(d_baseline.pblh[k] == d_cxx.pblh[k]); + REQUIRE(d_baseline.total(d_baseline.pblh) == d_cxx.total(d_cxx.check)); + REQUIRE(d_baseline.check[k] == d_cxx.check[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb @@ -251,14 +246,14 @@ TEST_CASE("pblintd_height_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintdHeight; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("pblintd_height_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintdHeight; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_init_pot_test.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_init_pot_test.cpp index b01c39eed95..9f7c2d65142 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_init_pot_test.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_init_pot_test.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -22,9 +22,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestPblintdInitPot { +struct UnitWrap::UnitTest::TestPblintdInitPot : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 1; @@ -77,7 +77,7 @@ struct UnitWrap::UnitTest::TestPblintdInitPot { } // call the C++ implementation - shoc_pblintd_init_pot_f(SDS.shcol, SDS.nlev, SDS.thl, SDS.ql, SDS.q, SDS.thv); + pblintd_init_pot(SDS); // Check the result. // Verify that virtual potential temperature is idential @@ -126,7 +126,7 @@ struct UnitWrap::UnitTest::TestPblintdInitPot { } // Call the C++ implementation - shoc_pblintd_init_pot_f(SDS.shcol, SDS.nlev, SDS.thl, SDS.ql, SDS.q, SDS.thv); + pblintd_init_pot(SDS); // Check test // Verify that column with condensate loading @@ -149,11 +149,11 @@ struct UnitWrap::UnitTest::TestPblintdInitPot { } // run_property - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - PblintdInitPotData pblintd_init_pot_data_f90[] = { + PblintdInitPotData pblintd_init_pot_data_baseline[] = { // shcol, nlev PblintdInitPotData(36, 72), PblintdInitPotData(72, 72), @@ -161,37 +161,44 @@ struct UnitWrap::UnitTest::TestPblintdInitPot { PblintdInitPotData(256, 72), }; - for (auto& d : pblintd_init_pot_data_f90) { + for (auto& d : pblintd_init_pot_data_baseline) { d.randomize(engine); } PblintdInitPotData pblintd_init_pot_data_cxx[] = { - PblintdInitPotData(pblintd_init_pot_data_f90[0]), - PblintdInitPotData(pblintd_init_pot_data_f90[1]), - PblintdInitPotData(pblintd_init_pot_data_f90[2]), - PblintdInitPotData(pblintd_init_pot_data_f90[3]), + PblintdInitPotData(pblintd_init_pot_data_baseline[0]), + PblintdInitPotData(pblintd_init_pot_data_baseline[1]), + PblintdInitPotData(pblintd_init_pot_data_baseline[2]), + PblintdInitPotData(pblintd_init_pot_data_baseline[3]), }; - for (auto& d : pblintd_init_pot_data_f90) { - // expects data in C layout - pblintd_init_pot(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : pblintd_init_pot_data_baseline) { + d.read(Base::m_fid); + } } for (auto& d : pblintd_init_pot_data_cxx) { - shoc_pblintd_init_pot_f(d.shcol, d.nlev, d.thl, d.ql, d.q, d.thv); + pblintd_init_pot(d); } - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(pblintd_init_pot_data_f90) / sizeof(PblintdInitPotData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(pblintd_init_pot_data_baseline) / sizeof(PblintdInitPotData); for (Int i = 0; i < num_runs; ++i) { Int shcol = pblintd_init_pot_data_cxx[i].shcol; Int nlev = pblintd_init_pot_data_cxx[i].nlev; for (Int j = 0; j < shcol; ++j ) { for (Int k = 0; k < nlev; ++k) { - REQUIRE(pblintd_init_pot_data_f90[i].thv[j*k] == pblintd_init_pot_data_cxx[i].thv[j*k]); + REQUIRE(pblintd_init_pot_data_baseline[i].thv[j*k] == pblintd_init_pot_data_cxx[i].thv[j*k]); } } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : pblintd_init_pot_data_cxx) { + d.write(Base::m_fid); + } } } @@ -207,14 +214,14 @@ TEST_CASE("shoc_pblintd_init_pot_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintdInitPot; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_pblintd_init_pot_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintdInitPot; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_surf_temp_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_surf_temp_tests.cpp index ccf3d110fc8..2e876996a6a 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_surf_temp_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_surf_temp_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,9 +14,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestPblintdSurfTemp { +struct UnitWrap::UnitTest::TestPblintdSurfTemp : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr auto ustar_min = scream::shoc::Constants::ustar_min; static constexpr Int shcol = 4; @@ -84,10 +84,7 @@ struct UnitWrap::UnitTest::TestPblintdSurfTemp { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - pblintd_surf_temp_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.z, SDS.ustar, SDS.obklen, - SDS.kbfs, SDS.thv, SDS.tlv, SDS.pblh, SDS.check, SDS.rino); - SDS.transpose(); // go back to C layout + pblintd_surf_temp(SDS); // Check the result for(Int s = 0; s < shcol; ++s) { @@ -112,11 +109,11 @@ struct UnitWrap::UnitTest::TestPblintdSurfTemp { } // run_property - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - PblintdSurfTempData f90_data[] = { + PblintdSurfTempData baseline_data[] = { PblintdSurfTempData(6, 7, 8), PblintdSurfTempData(64, 72, 73), PblintdSurfTempData(128, 72, 73), @@ -124,54 +121,58 @@ struct UnitWrap::UnitTest::TestPblintdSurfTemp { }; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine, { {d.obklen, {100., 200.}} }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state PblintdSurfTempData cxx_data[] = { - PblintdSurfTempData(f90_data[0]), - PblintdSurfTempData(f90_data[1]), - PblintdSurfTempData(f90_data[2]), - PblintdSurfTempData(f90_data[3]), + PblintdSurfTempData(baseline_data[0]), + PblintdSurfTempData(baseline_data[1]), + PblintdSurfTempData(baseline_data[2]), + PblintdSurfTempData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - pblintd_surf_temp(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - pblintd_surf_temp_f(d.shcol, d.nlev, d.nlevi, d.z, d.ustar, d.obklen, d.kbfs, d.thv, d.tlv, d.pblh, d.check, d.rino); - d.transpose(); // go back to C layout + pblintd_surf_temp(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(PblintdSurfTempData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(PblintdSurfTempData); for (Int i = 0; i < num_runs; ++i) { - PblintdSurfTempData& d_f90 = f90_data[i]; + PblintdSurfTempData& d_baseline = baseline_data[i]; PblintdSurfTempData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.tlv); ++k) { - REQUIRE(d_f90.total(d_f90.tlv) == d_cxx.total(d_cxx.tlv)); - REQUIRE(d_f90.tlv[k] == d_cxx.tlv[k]); - REQUIRE(d_f90.total(d_f90.tlv) == d_cxx.total(d_cxx.pblh)); - REQUIRE(d_f90.pblh[k] == d_cxx.pblh[k]); - REQUIRE(d_f90.total(d_f90.tlv) == d_cxx.total(d_cxx.check)); - REQUIRE(d_f90.check[k] == d_cxx.check[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.tlv); ++k) { + REQUIRE(d_baseline.total(d_baseline.tlv) == d_cxx.total(d_cxx.tlv)); + REQUIRE(d_baseline.tlv[k] == d_cxx.tlv[k]); + REQUIRE(d_baseline.total(d_baseline.tlv) == d_cxx.total(d_cxx.pblh)); + REQUIRE(d_baseline.pblh[k] == d_cxx.pblh[k]); + REQUIRE(d_baseline.total(d_baseline.tlv) == d_cxx.total(d_cxx.check)); + REQUIRE(d_baseline.check[k] == d_cxx.check[k]); } - for (Int k = 0; k < d_f90.total(d_f90.rino); ++k) { - REQUIRE(d_f90.total(d_f90.rino) == d_cxx.total(d_cxx.rino)); - REQUIRE(d_f90.rino[k] == d_cxx.rino[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.rino); ++k) { + REQUIRE(d_baseline.total(d_baseline.rino) == d_cxx.total(d_cxx.rino)); + REQUIRE(d_baseline.rino[k] == d_cxx.rino[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb @@ -187,14 +188,14 @@ TEST_CASE("pblintd_surf_temp_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintdSurfTemp; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("pblintd_surf_temp_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintdSurfTemp; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_tests.cpp index b5cd117228b..dedc534fbd1 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,9 +14,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestPblintd { +struct UnitWrap::UnitTest::TestPblintd : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr auto ustar_min = scream::shoc::Constants::ustar_min; static constexpr Int shcol = 5; @@ -122,11 +122,7 @@ struct UnitWrap::UnitTest::TestPblintd { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - pblintd_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.npbl, SDS.z, SDS.zi, - SDS.thl, SDS.ql, SDS.q, SDS.u, SDS.v, SDS.ustar, SDS.obklen, - SDS.kbfs, SDS.cldn, SDS.pblh); - SDS.transpose(); // go back to C layout + pblintd(SDS); // Make sure PBL height is reasonable // Should be larger than second lowest zi level and lower @@ -140,13 +136,13 @@ struct UnitWrap::UnitTest::TestPblintd { } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); Int npbl_rand = rand()%71 + 1; - PblintdData f90_data[] = { + PblintdData baseline_data[] = { PblintdData(10, 71, 72, 71), PblintdData(10, 71, 72, 1), PblintdData(10, 71, 72, npbl_rand), @@ -156,47 +152,51 @@ struct UnitWrap::UnitTest::TestPblintd { }; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state PblintdData cxx_data[] = { - PblintdData(f90_data[0]), - PblintdData(f90_data[1]), - PblintdData(f90_data[2]), - PblintdData(f90_data[3]), - PblintdData(f90_data[4]), - PblintdData(f90_data[5]), + PblintdData(baseline_data[0]), + PblintdData(baseline_data[1]), + PblintdData(baseline_data[2]), + PblintdData(baseline_data[3]), + PblintdData(baseline_data[4]), + PblintdData(baseline_data[5]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - pblintd(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - pblintd_f(d.shcol, d.nlev, d.nlevi, d.npbl, d.z, d.zi, d.thl, d.ql, d.q, d.u, d.v, d.ustar, d.obklen, d.kbfs, d.cldn, d.pblh); - d.transpose(); // go back to C layout + pblintd(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(PblintdData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(PblintdData); for (Int i = 0; i < num_runs; ++i) { - PblintdData& d_f90 = f90_data[i]; + PblintdData& d_baseline = baseline_data[i]; PblintdData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.pblh); ++k) { - REQUIRE(d_f90.pblh[k] == d_cxx.pblh[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.pblh); ++k) { + REQUIRE(d_baseline.pblh[k] == d_cxx.pblh[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb @@ -212,14 +212,14 @@ TEST_CASE("pblintd_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintd; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("pblintd_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestPblintd; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_buoyflux_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_buoyflux_tests.cpp index 261cc24dc29..53c91a605c7 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_buoyflux_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_buoyflux_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" @@ -24,7 +24,6 @@ struct UnitWrap::UnitTest::TestShocPdfCompBuoyFlux { static void run_property() { - static constexpr Real epsterm = scream::physics::Constants::ep_2; // Property tests for the SHOC function // shoc_assumed_pdf_compute_buoyancy_flux @@ -53,7 +52,6 @@ struct UnitWrap::UnitTest::TestShocPdfCompBuoyFlux { SDS.wqwsec = wqwsec_dry; SDS.wqls = wqls_dry; SDS.pval = pval; - SDS.epsterm = epsterm; // Call the fortran implementation shoc_assumed_pdf_compute_buoyancy_flux(SDS); diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_cloudvar_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_cloudvar_tests.cpp index 79594e068a1..7fdd57ff005 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_cloudvar_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_cloudvar_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_liqflux_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_liqflux_tests.cpp index f1dd2e9bc0e..bee4f5db740 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_liqflux_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_liqflux_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_qs_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_qs_tests.cpp index 6e430a02f06..4a3e0cf8a6f 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_qs_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_qs_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_s_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_s_tests.cpp index 0f40ccb63a0..ffdddaa1928 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_s_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_s_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_sgsliq_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_sgsliq_tests.cpp index 3964cf6e7ed..af5e87dc418 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_sgsliq_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_compute_sgsliq_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_computetemp_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_computetemp_tests.cpp index ce4d27dc15a..f9044517b6a 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_computetemp_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_computetemp_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" @@ -37,7 +37,7 @@ struct UnitWrap::UnitTest::TestShocPdfComputeTemp { // Input liquid water potential temperature [K] static constexpr Real thl1 = 305; // Input basepressure [Pa] - static constexpr Real basepres = 100000; + static constexpr Real basepres = C::P0; // Input value of pval [Pa] Real pval = 110000; @@ -55,7 +55,6 @@ struct UnitWrap::UnitTest::TestShocPdfComputeTemp { // Fill in data SDS.thl1 = thl1; - SDS.basepres = basepres; SDS.pval = pval; Int num_tests = SDS.pval/abs(presincr); @@ -64,7 +63,7 @@ struct UnitWrap::UnitTest::TestShocPdfComputeTemp { REQUIRE(presincr < 0); // Make sure our starting pressure is greater than // basepres just so we test a range - REQUIRE(SDS.pval > SDS.basepres); + REQUIRE(SDS.pval > basepres); for (Int s = 0; s < num_tests; ++s){ @@ -81,11 +80,11 @@ struct UnitWrap::UnitTest::TestShocPdfComputeTemp { // If pressure is greater than basepressure then // make sure that temperature is greater than thetal - if (SDS.pval > SDS.basepres){ + if (SDS.pval > basepres){ REQUIRE(SDS.tl1 > SDS.thl1); } // otherwise temperature should be less than thetal - else if(SDS.pval < SDS.basepres){ + else if(SDS.pval < basepres){ REQUIRE(SDS.tl1 < SDS.thl1); } // otherwise if they are equal the temperatures diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_inplume_corr_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_inplume_corr_tests.cpp index d1d17fcebef..a9ffdf242f3 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_inplume_corr_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_inplume_corr_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_qw_parameters_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_qw_parameters_tests.cpp index 5e7b0b8d72a..0aabe3da33b 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_qw_parameters_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_qw_parameters_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" @@ -75,6 +75,8 @@ struct UnitWrap::UnitTest::TestShocQwParameters { SDS.w1_2 = w1_2_test1; SDS.skew_w = Skew_w_test1; SDS.a = a_test1; + SDS.rt_tol = 0; + SDS.w_thresh = 0; // Verify input is physical REQUIRE(SDS.sqrtw2 >= 0); diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_thl_parameters_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_thl_parameters_tests.cpp index f00c7b2c292..96fdbbc2dfc 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_thl_parameters_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_thl_parameters_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" @@ -50,8 +50,6 @@ struct UnitWrap::UnitTest::TestShocThlParameters { static constexpr Real Skew_w_test1 = 3; // Define fraction of first gaussian static constexpr Real a_test1 = 0.2; - // Define logical - static constexpr bool dothetal_skew = false; // Define reasonable bounds checking for output static constexpr Real thl_bound_low = 200; // [K] @@ -74,7 +72,8 @@ struct UnitWrap::UnitTest::TestShocThlParameters { SDS.w1_2 = w1_2_test1; SDS.skew_w = Skew_w_test1; SDS.a = a_test1; - SDS.dothetal_skew = dothetal_skew; + SDS.thl_tol = 0; + SDS.w_thresh = 0; // Verify input is physical REQUIRE(SDS.sqrtw2 >= 0); diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_tildetoreal_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_tildetoreal_tests.cpp index 3c0f84a6222..a0728f2e358 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_tildetoreal_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_tildetoreal_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pdf_vv_parameters_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pdf_vv_parameters_tests.cpp index 2f0abe43e76..f923e1b47e2 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pdf_vv_parameters_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pdf_vv_parameters_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" @@ -46,6 +46,7 @@ struct UnitWrap::UnitTest::TestShocVVParameters { SDS.w_first = w_first_sym; SDS.w_sec = w_sec_sym; SDS.w3var = w3var_sym; + SDS.w_tol_sqd = 0; // Verify input is physical REQUIRE(SDS.w_sec >= 0); diff --git a/components/eamxx/src/physics/shoc/tests/shoc_run_and_cmp.cpp b/components/eamxx/src/physics/shoc/tests/shoc_run_and_cmp.cpp index 94a58b4de9c..8c2313082e5 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_run_and_cmp.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_run_and_cmp.cpp @@ -1,5 +1,5 @@ #include "shoc_main_wrap.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "shoc_ic_cases.hpp" #include "share/scream_types.hpp" @@ -61,7 +61,7 @@ struct Baseline { params_.push_back({ic::Factory::standard, repeat, nsteps, ncol, nlev, num_qtracers, nadv, dt}); } - Int generate_baseline (const std::string& filename, bool use_fortran) { + Int generate_baseline (const std::string& filename) { auto fid = ekat::FILEPtr(fopen(filename.c_str(), "w")); EKAT_REQUIRE_MSG( fid, "generate_baseline can't write " << filename); Int nerr = 0; @@ -74,20 +74,17 @@ struct Baseline { // Run reference shoc on this set of parameters. const auto d = ic::Factory::create(ps.ic, ps.ncol, ps.nlev, ps.num_qtracers); set_params(ps, *d); - shoc_init(ps.nlev, use_fortran); if (ps.repeat > 0 && r == -1) { std::cout << "Running SHOC with ni=" << d->shcol << ", nk=" << d->nlev << ", dt=" << d->dtime << ", ts=" << ps.nsteps; - if (!use_fortran) { - std::cout << ", small_packn=" << SCREAM_SMALL_PACK_SIZE; - } + std::cout << ", small_packn=" << SCREAM_SMALL_PACK_SIZE; std::cout << std::endl; } for (int it = 0; it < ps.nsteps; ++it) { - Int current_microsec = shoc_main(*d, use_fortran); + Int current_microsec = shoc_main(*d); if (r != -1 && ps.repeat > 0) { // do not count the "cold" run duration += current_microsec; @@ -107,30 +104,42 @@ struct Baseline { return nerr; } - Int run_and_cmp (const std::string& filename, const double& tol, bool use_fortran) { - auto fid = ekat::FILEPtr(fopen(filename.c_str(), "r")); - EKAT_REQUIRE_MSG( fid, "generate_baseline can't read " << filename); + Int run_and_cmp (const std::string& filename, const double& tol, bool no_baseline) { + ekat::FILEPtr fid; + if (!no_baseline) { + fid = ekat::FILEPtr(fopen(filename.c_str(), "r")); + EKAT_REQUIRE_MSG( fid, "generate_baseline can't read " << filename); + } Int nerr = 0, ne; int case_num = 0; for (auto ps : params_) { case_num++; - // Read the reference impl's data from the baseline file. - const auto d_ref = ic::Factory::create(ps.ic, ps.ncol, ps.nlev, ps.num_qtracers); - set_params(ps, *d_ref); - // Now run a sequence of other impls. This includes the reference - // implementation b/c it's likely we'll want to change it as we go. - { + if (no_baseline) { const auto d = ic::Factory::create(ps.ic, ps.ncol, ps.nlev, ps.num_qtracers); set_params(ps, *d); - shoc_init(ps.nlev, use_fortran); - for (int it = 0; it < ps.nsteps; it++) { - std::cout << "--- checking case # " << case_num << ", timestep # = " << (it+1)*ps.nadv - << " ---\n" << std::flush; - read(fid, d_ref); - shoc_main(*d,use_fortran); - ne = compare(tol, d_ref, d); - if (ne) std::cout << "Ref impl failed.\n"; - nerr += ne; + for (int it=0; it Path to directory containing baselines.\n" " -t Tolerance for relative error.\n" " -s Number of timesteps. Default=10.\n" " -dt Length of timestep. Default=150.\n" " -i Number of columns(ncol). Default=8.\n" " -k Number of vertical levels. Default=72.\n" " -q Number of q tracers. Default=3.\n" - " -n Number of SHOC loops per timestep. Default=15.\n" + " -l Number of SHOC loops per timestep. Default=15.\n" " -r Number of repetitions, implies timing run (generate + no I/O). Default=0.\n"; return 1; } - bool generate = false, use_fortran = false; + bool generate = false, no_baseline = true; scream::Real tol = SCREAM_BFB_TESTING ? 0 : std::numeric_limits::infinity(); Int nsteps = 10; Int dt = 150; @@ -220,9 +231,9 @@ int main (int argc, char** argv) { Int repeat = 0; std::string baseline_fn; std::string device; - for (int i = 1; i < argc-1; ++i) { - if (ekat::argv_matches(argv[i], "-g", "--generate")) generate = true; - if (ekat::argv_matches(argv[i], "-f", "--fortran")) use_fortran = true; + for (int i = 1; i < argc; ++i) { + if (ekat::argv_matches(argv[i], "-g", "--generate")) { generate = true; no_baseline = false; } + if (ekat::argv_matches(argv[i], "-c", "--compare")) { no_baseline = false; } if (ekat::argv_matches(argv[i], "-b", "--baseline-file")) { expect_another_arg(i, argc); ++i; @@ -258,7 +269,7 @@ int main (int argc, char** argv) { ++i; num_qtracers = std::atoi(argv[i]); } - if (ekat::argv_matches(argv[i], "-n", "--nadv")) { + if (ekat::argv_matches(argv[i], "-l", "--nadv")) { expect_another_arg(i, argc); ++i; nadv = std::atoi(argv[i]); @@ -277,18 +288,22 @@ int main (int argc, char** argv) { } } - // Decorate baseline name with precision. - baseline_fn += std::to_string(sizeof(scream::Real)); + // Compute full baseline file name with precision. + baseline_fn += "/shoc_run_and_cmp.baseline" + std::to_string(sizeof(scream::Real)); scream::initialize_scream_session(argc, argv); { Baseline bln(nsteps, static_cast(dt), ncol, nlev, num_qtracers, nadv, repeat); if (generate) { std::cout << "Generating to " << baseline_fn << "\n"; - nerr += bln.generate_baseline(baseline_fn, use_fortran); - } else { + nerr += bln.generate_baseline(baseline_fn); + } else if (no_baseline) { + printf("Running with no baseline actions\n"); + nerr += bln.run_and_cmp(baseline_fn, tol, no_baseline); + } + else { printf("Comparing with %s at tol %1.1e\n", baseline_fn.c_str(), tol); - nerr += bln.run_and_cmp(baseline_fn, tol, use_fortran); + nerr += bln.run_and_cmp(baseline_fn, tol, no_baseline); } } scream::finalize_scream_session(); diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tests.cpp index 1872cf1b93a..1884c0e3458 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tests.cpp @@ -26,43 +26,8 @@ TEST_CASE("FortranDataIterator", "shoc") { REQUIRE(static_cast(f.size) == d->shcol); } -TEST_CASE("shoc_init_f", "shoc") { - int nerr = scream::shoc::test_shoc_init(true); - REQUIRE(nerr == 0); -} - -// This helper returns true if we've been asked to generate Python -// plotting scripts, false otherwise. -bool generating_plot_scripts() { - bool gen_plot_scripts = false; - auto& ts = ekat::TestSession::get(); - auto iter = ts.params.find("gen_plot_scripts"); - if (iter != ts.params.end()) { - // Here's val, passed as gen_plot_scripts=val - std::string val = iter->second; - // Low-case the thing. Isn't C++ a friendly language?? - std::transform(val.begin(), val.end(), val.begin(), - [](unsigned char c){ return std::tolower(c); }); - - // Now decide if the value is true or not. Use CMake sensibilities. - gen_plot_scripts = ((val == "1") or (val == "true") or - (val == "yes") or (val == "on")); - } - return gen_plot_scripts; -} - -TEST_CASE("shoc_ic_f", "shoc") { - int nerr = scream::shoc::test_shoc_ic(true, generating_plot_scripts()); - REQUIRE(nerr == 0); -} - -TEST_CASE("shoc_init_c", "shoc") { - int nerr = scream::shoc::test_shoc_init(false); - REQUIRE(nerr == 0); -} - TEST_CASE("shoc_ic_c", "shoc") { - int nerr = scream::shoc::test_shoc_ic(false); + int nerr = scream::shoc::test_shoc_ic(); REQUIRE(nerr == 0); } diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp index ea2c95470d8..165bf302e4c 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp @@ -1,7 +1,7 @@ #include "catch2/catch.hpp" #include "shoc_unit_tests_common.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "shoc_functions.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" @@ -22,9 +22,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocAdvSgsTke { +struct UnitWrap::UnitTest::TestShocAdvSgsTke : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Real mintke = scream::shoc::Constants::mintke; static constexpr Real maxtke = scream::shoc::Constants::maxtke; @@ -95,9 +95,7 @@ struct UnitWrap::UnitTest::TestShocAdvSgsTke { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - adv_sgs_tke_f(SDS.nlev, SDS.shcol, SDS.dtime, SDS.shoc_mix, SDS.wthv_sec, SDS.sterm_zt, SDS.tk, SDS.tke, SDS.a_diss); - SDS.transpose(); // go back to C layout + adv_sgs_tke(SDS); // Check to make sure that there has been // TKE growth @@ -162,9 +160,7 @@ struct UnitWrap::UnitTest::TestShocAdvSgsTke { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - adv_sgs_tke_f(SDS.nlev, SDS.shcol, SDS.dtime, SDS.shoc_mix, SDS.wthv_sec, SDS.sterm_zt, SDS.tk, SDS.tke, SDS.a_diss); - SDS.transpose(); // go back to C layout + adv_sgs_tke(SDS); // Check to make sure that the column with // the smallest length scale has larger @@ -194,11 +190,11 @@ struct UnitWrap::UnitTest::TestShocAdvSgsTke { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - AdvSgsTkeData f90_data[] = { + AdvSgsTkeData baseline_data[] = { // shcol, nlev AdvSgsTkeData(10, 71, 72), AdvSgsTkeData(10, 12, 13), @@ -207,45 +203,49 @@ struct UnitWrap::UnitTest::TestShocAdvSgsTke { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state AdvSgsTkeData cxx_data[] = { - AdvSgsTkeData(f90_data[0]), - AdvSgsTkeData(f90_data[1]), - AdvSgsTkeData(f90_data[2]), - AdvSgsTkeData(f90_data[3]), + AdvSgsTkeData(baseline_data[0]), + AdvSgsTkeData(baseline_data[1]), + AdvSgsTkeData(baseline_data[2]), + AdvSgsTkeData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - adv_sgs_tke(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - adv_sgs_tke_f(d.nlev, d.shcol, d.dtime, d.shoc_mix, d.wthv_sec, d.sterm_zt, d.tk, d.tke, d.a_diss); - d.transpose(); // go back to C layout + adv_sgs_tke(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(AdvSgsTkeData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(AdvSgsTkeData); for (Int i = 0; i < num_runs; ++i) { - AdvSgsTkeData& d_f90 = f90_data[i]; + AdvSgsTkeData& d_baseline = baseline_data[i]; AdvSgsTkeData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.tke); ++k) { - REQUIRE(d_f90.tke[k] == d_cxx.tke[k]); - REQUIRE(d_f90.a_diss[k] == d_cxx.a_diss[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.tke); ++k) { + REQUIRE(d_baseline.tke[k] == d_cxx.tke[k]); + REQUIRE(d_baseline.a_diss[k] == d_cxx.a_diss[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } }//run_bfb }; @@ -260,14 +260,14 @@ TEST_CASE("shoc_tke_adv_sgs_tke_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocAdvSgsTke; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_tke_adv_sgs_tke_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocAdvSgsTke; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_column_stab_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_column_stab_tests.cpp index eb68bbc11df..d234273f4c0 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_column_stab_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_column_stab_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocIntColStab { +struct UnitWrap::UnitTest::TestShocIntColStab : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 5; @@ -76,9 +76,7 @@ struct UnitWrap::UnitTest::TestShocIntColStab { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - integ_column_stability_f(SDS.nlev, SDS.shcol, SDS.dz_zt, SDS.pres, SDS.brunt, SDS.brunt_int); - SDS.transpose(); // go back to C layout + integ_column_stability(SDS); // Check test // Verify that output is zero @@ -110,9 +108,7 @@ struct UnitWrap::UnitTest::TestShocIntColStab { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - integ_column_stability_f(SDS.nlev, SDS.shcol, SDS.dz_zt, SDS.pres, SDS.brunt, SDS.brunt_int); - SDS.transpose(); // go back to C layout + integ_column_stability(SDS); // Check test // Verify that output is negative @@ -121,12 +117,12 @@ struct UnitWrap::UnitTest::TestShocIntColStab { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - //declare data for the f90 function call - IntegColumnStabilityData f90_data[] = { + //declare data for the baseline function call + IntegColumnStabilityData baseline_data[] = { IntegColumnStabilityData(10, 71), IntegColumnStabilityData(10, 12), IntegColumnStabilityData(7, 16), @@ -134,44 +130,48 @@ struct UnitWrap::UnitTest::TestShocIntColStab { }; //Generate random data - for (auto &d : f90_data) { + for (auto &d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data (if any) is in original state IntegColumnStabilityData cxx_data[] = { - IntegColumnStabilityData(f90_data[0]), - IntegColumnStabilityData(f90_data[1]), - IntegColumnStabilityData(f90_data[2]), - IntegColumnStabilityData(f90_data[3]), + IntegColumnStabilityData(baseline_data[0]), + IntegColumnStabilityData(baseline_data[1]), + IntegColumnStabilityData(baseline_data[2]), + IntegColumnStabilityData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto &d : f90_data) { - // expects data in C layout - integ_column_stability(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto &d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto &d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - integ_column_stability_f(d.nlev, d.shcol, d.dz_zt, d.pres, d.brunt, d.brunt_int); - d.transpose(); // go back to C layout + integ_column_stability(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(IntegColumnStabilityData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(IntegColumnStabilityData); for (Int i = 0; i < num_runs; ++i) { - IntegColumnStabilityData& d_f90 = f90_data[i]; + IntegColumnStabilityData& d_baseline = baseline_data[i]; IntegColumnStabilityData& d_cxx = cxx_data[i]; - for (Int c = 0; c < d_f90.shcol; ++c) { - REQUIRE(d_f90.brunt_int[c] == d_cxx.brunt_int[c]); + for (Int c = 0; c < d_baseline.shcol; ++c) { + REQUIRE(d_baseline.brunt_int[c] == d_cxx.brunt_int[c]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto &d : cxx_data) { + d.write(Base::m_fid); + } } } //run_bfb }; @@ -186,14 +186,14 @@ TEST_CASE("shoc_tke_column_stab_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocIntColStab; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_tke_column_stab_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocIntColStab; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_isotropic_ts_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_isotropic_ts_tests.cpp index c90128857cb..b7fba522d8f 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_isotropic_ts_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_isotropic_ts_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocIsotropicTs { +struct UnitWrap::UnitTest::TestShocIsotropicTs : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Real maxiso = scream::shoc::Constants::maxiso; static constexpr Int shcol = 2; @@ -82,9 +82,7 @@ struct UnitWrap::UnitTest::TestShocIsotropicTs { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - isotropic_ts_f(SDS.nlev, SDS.shcol, SDS.brunt_int, SDS.tke, SDS.a_diss, SDS.brunt, SDS.isotropy); - SDS.transpose(); // go back to C layout + isotropic_ts(SDS); // Check that output falls within reasonable bounds for(Int s = 0; s < shcol; ++s) { @@ -148,9 +146,7 @@ struct UnitWrap::UnitTest::TestShocIsotropicTs { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - isotropic_ts_f(SDS.nlev, SDS.shcol, SDS.brunt_int, SDS.tke, SDS.a_diss, SDS.brunt, SDS.isotropy); - SDS.transpose(); // go back to C layout + isotropic_ts(SDS); // Check that output falls within reasonable bounds for(Int s = 0; s < shcol; ++s) { @@ -178,11 +174,11 @@ struct UnitWrap::UnitTest::TestShocIsotropicTs { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - IsotropicTsData f90_data[] = { + IsotropicTsData baseline_data[] = { // shcol, nlev IsotropicTsData(10, 71), IsotropicTsData(10, 12), @@ -191,44 +187,48 @@ struct UnitWrap::UnitTest::TestShocIsotropicTs { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state IsotropicTsData cxx_data[] = { - IsotropicTsData(f90_data[0]), - IsotropicTsData(f90_data[1]), - IsotropicTsData(f90_data[2]), - IsotropicTsData(f90_data[3]), + IsotropicTsData(baseline_data[0]), + IsotropicTsData(baseline_data[1]), + IsotropicTsData(baseline_data[2]), + IsotropicTsData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - isotropic_ts(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - isotropic_ts_f(d.nlev, d.shcol, d.brunt_int, d.tke, d.a_diss, d.brunt, d.isotropy); - d.transpose(); // go back to C layout + isotropic_ts(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(IsotropicTsData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(IsotropicTsData); for (Int i = 0; i < num_runs; ++i) { - IsotropicTsData& d_f90 = f90_data[i]; + IsotropicTsData& d_baseline = baseline_data[i]; IsotropicTsData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.isotropy); ++k) { - REQUIRE(d_f90.isotropy[k] == d_cxx.isotropy[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.isotropy); ++k) { + REQUIRE(d_baseline.isotropy[k] == d_cxx.isotropy[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } }//run_bfb @@ -244,14 +244,14 @@ TEST_CASE("shoc_tke_isotropic_ts_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocIsotropicTs; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_tke_isotropic_ts_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocIsotropicTs; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_shr_prod_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_shr_prod_tests.cpp index 0672ce3b3ae..00b60a5efdb 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_shr_prod_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_shr_prod_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocShearProd { +struct UnitWrap::UnitTest::TestShocShearProd : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 5; @@ -94,9 +94,7 @@ struct UnitWrap::UnitTest::TestShocShearProd { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - compute_shr_prod_f(SDS.nlevi, SDS.nlev, SDS.shcol, SDS.dz_zi, SDS.u_wind, SDS.v_wind, SDS.sterm); - SDS.transpose(); // go back to C layout + compute_shr_prod(SDS); // Check test for(Int s = 0; s < shcol; ++s) { @@ -145,9 +143,7 @@ struct UnitWrap::UnitTest::TestShocShearProd { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - compute_shr_prod_f(SDS.nlevi, SDS.nlev, SDS.shcol, SDS.dz_zi, SDS.u_wind, SDS.v_wind, SDS.sterm); - SDS.transpose(); // go back to C layout + compute_shr_prod(SDS); // Check test // Verify that shear term is zero everywhere @@ -159,11 +155,11 @@ struct UnitWrap::UnitTest::TestShocShearProd { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ComputeShrProdData f90_data[] = { + ComputeShrProdData baseline_data[] = { // shcol, nlev ComputeShrProdData(10, 71, 72), ComputeShrProdData(10, 12, 13), @@ -172,44 +168,48 @@ struct UnitWrap::UnitTest::TestShocShearProd { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ComputeShrProdData cxx_data[] = { - ComputeShrProdData(f90_data[0]), - ComputeShrProdData(f90_data[1]), - ComputeShrProdData(f90_data[2]), - ComputeShrProdData(f90_data[3]), + ComputeShrProdData(baseline_data[0]), + ComputeShrProdData(baseline_data[1]), + ComputeShrProdData(baseline_data[2]), + ComputeShrProdData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - compute_shr_prod(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - compute_shr_prod_f(d.nlevi, d.nlev, d.shcol, d.dz_zi, d.u_wind, d.v_wind, d.sterm); - d.transpose(); // go back to C layout + compute_shr_prod(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(ComputeShrProdData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(ComputeShrProdData); for (Int i = 0; i < num_runs; ++i) { - ComputeShrProdData& d_f90 = f90_data[i]; + ComputeShrProdData& d_baseline = baseline_data[i]; ComputeShrProdData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.sterm); ++k) { - REQUIRE(d_f90.sterm[k] == d_cxx.sterm[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.sterm); ++k) { + REQUIRE(d_baseline.sterm[k] == d_cxx.sterm[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } //run_bfb }; @@ -224,14 +224,14 @@ TEST_CASE("shoc_tke_shr_prod_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocShearProd; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_tke_shr_prod_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocShearProd; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp index f09ecc79c7d..5273607dd70 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp @@ -2,7 +2,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocTke { +struct UnitWrap::UnitTest::TestShocTke : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Real mintke = scream::shoc::Constants::mintke; static constexpr Real maxtke = scream::shoc::Constants::maxtke; @@ -154,11 +154,7 @@ struct UnitWrap::UnitTest::TestShocTke { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - shoc_tke_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.wthv_sec, SDS.shoc_mix, SDS.dz_zi, SDS.dz_zt, - SDS.pres, SDS.tabs, SDS.u_wind, SDS.v_wind, SDS.brunt, SDS.zt_grid, SDS.zi_grid, SDS.pblh, - SDS.tke, SDS.tk, SDS.tkh, SDS.isotropy); - SDS.transpose(); // go back to C layout + shoc_tke(SDS); // Check test // Make sure that TKE has increased everwhere relative @@ -231,11 +227,7 @@ struct UnitWrap::UnitTest::TestShocTke { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - shoc_tke_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.wthv_sec, SDS.shoc_mix, SDS.dz_zi, SDS.dz_zt, - SDS.pres, SDS.tabs, SDS.u_wind, SDS.v_wind, SDS.brunt, SDS.zt_grid, SDS.zi_grid, SDS.pblh, - SDS.tke, SDS.tk, SDS.tkh, SDS.isotropy); - SDS.transpose(); // go back to C layout + shoc_tke(SDS); // Check the result @@ -254,11 +246,11 @@ struct UnitWrap::UnitTest::TestShocTke { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - ShocTkeData f90_data[] = { + ShocTkeData baseline_data[] = { ShocTkeData(10, 71, 72, 300), ShocTkeData(10, 12, 13, 100), ShocTkeData(7, 16, 17, 50), @@ -266,54 +258,56 @@ struct UnitWrap::UnitTest::TestShocTke { }; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state ShocTkeData cxx_data[] = { - ShocTkeData(f90_data[0]), - ShocTkeData(f90_data[1]), - ShocTkeData(f90_data[2]), - ShocTkeData(f90_data[3]), + ShocTkeData(baseline_data[0]), + ShocTkeData(baseline_data[1]), + ShocTkeData(baseline_data[2]), + ShocTkeData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - shoc_tke(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - shoc_tke_f(d.shcol, d.nlev, d.nlevi, d.dtime, d.wthv_sec, d.shoc_mix, d.dz_zi, d.dz_zt, - d.pres, d.tabs, d.u_wind, d.v_wind, d.brunt, d.zt_grid, d.zi_grid, d.pblh, - d.tke, d.tk, d.tkh, d.isotropy); - d.transpose(); // go back to C layout + shoc_tke(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(ShocTkeData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(ShocTkeData); for (Int i = 0; i < num_runs; ++i) { - ShocTkeData& d_f90 = f90_data[i]; + ShocTkeData& d_baseline = baseline_data[i]; ShocTkeData& d_cxx = cxx_data[i]; - REQUIRE(d_f90.total(d_f90.tke) == d_cxx.total(d_cxx.tke)); - REQUIRE(d_f90.total(d_f90.tke) == d_cxx.total(d_cxx.tk)); - REQUIRE(d_f90.total(d_f90.tke) == d_cxx.total(d_cxx.tkh)); - REQUIRE(d_f90.total(d_f90.tke) == d_cxx.total(d_cxx.isotropy)); - for (Int k = 0; k < d_f90.total(d_f90.tke); ++k) { - REQUIRE(d_f90.tke[k] == d_cxx.tke[k]); - REQUIRE(d_f90.tk[k] == d_cxx.tk[k]); - REQUIRE(d_f90.tkh[k] == d_cxx.tkh[k]); - REQUIRE(d_f90.isotropy[k] == d_cxx.isotropy[k]); + REQUIRE(d_baseline.total(d_baseline.tke) == d_cxx.total(d_cxx.tke)); + REQUIRE(d_baseline.total(d_baseline.tke) == d_cxx.total(d_cxx.tk)); + REQUIRE(d_baseline.total(d_baseline.tke) == d_cxx.total(d_cxx.tkh)); + REQUIRE(d_baseline.total(d_baseline.tke) == d_cxx.total(d_cxx.isotropy)); + for (Int k = 0; k < d_baseline.total(d_baseline.tke); ++k) { + REQUIRE(d_baseline.tke[k] == d_cxx.tke[k]); + REQUIRE(d_baseline.tk[k] == d_cxx.tk[k]); + REQUIRE(d_baseline.tkh[k] == d_cxx.tkh[k]); + REQUIRE(d_baseline.isotropy[k] == d_cxx.isotropy[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb }; @@ -328,14 +322,14 @@ TEST_CASE("shoc_tke_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocTke; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_tke_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocTke; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } diff --git a/components/eamxx/src/physics/shoc/tests/shoc_unit_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_unit_tests.cpp index 71fe03f3cbb..f564c74fab4 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_unit_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_unit_tests.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/scream_types.hpp" diff --git a/components/eamxx/src/physics/shoc/tests/shoc_update_prognostics_implicit_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_update_prognostics_implicit_tests.cpp index 83cfb8d5707..175be842928 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_update_prognostics_implicit_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_update_prognostics_implicit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,9 +14,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestUpdatePrognosticsImplicit { +struct UnitWrap::UnitTest::TestUpdatePrognosticsImplicit : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 5; static constexpr Int nlev = 5; @@ -259,17 +259,10 @@ struct UnitWrap::UnitTest::TestUpdatePrognosticsImplicit { } // Call the C++ implementation - SDS.transpose(); // _f expects data in fortran layout - update_prognostics_implicit_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.num_tracer, SDS.dtime, - SDS.dz_zt, SDS.dz_zi, SDS.rho_zt, SDS.zt_grid, SDS.zi_grid, - SDS.tk, SDS.tkh, SDS.uw_sfc, SDS.vw_sfc, SDS.wthl_sfc, SDS.wqw_sfc, - SDS.wtracer_sfc, SDS.thetal, SDS.qw, SDS.tracer, SDS.tke, SDS.u_wind, SDS.v_wind); - SDS.transpose(); // go back to C layout + update_prognostics_implicit(SDS); // Call linear interp to get rho value at surface for checking - SDSL.transpose(); // _f expects data in fortran layout - linear_interp_f(SDSL.x1, SDSL.x2, SDSL.y1, SDSL.y2, SDSL.km1, SDSL.km2, SDSL.ncol, SDSL.minthresh); - SDSL.transpose(); // go back to C layout + linear_interp(SDSL); // Check the result @@ -341,11 +334,11 @@ struct UnitWrap::UnitTest::TestUpdatePrognosticsImplicit { } // run_property - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - UpdatePrognosticsImplicitData f90_data[] = { + UpdatePrognosticsImplicitData baseline_data[] = { UpdatePrognosticsImplicitData(10, 71, 72, 19, .5), UpdatePrognosticsImplicitData(10, 12, 13, 7, .25), UpdatePrognosticsImplicitData(7, 16, 17, 2, .1), @@ -353,62 +346,63 @@ struct UnitWrap::UnitTest::TestUpdatePrognosticsImplicit { }; // Generate random input data - for (auto& d : f90_data) { + for (auto& d : baseline_data) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state UpdatePrognosticsImplicitData cxx_data[] = { - UpdatePrognosticsImplicitData(f90_data[0]), - UpdatePrognosticsImplicitData(f90_data[1]), - UpdatePrognosticsImplicitData(f90_data[2]), - UpdatePrognosticsImplicitData(f90_data[3]), + UpdatePrognosticsImplicitData(baseline_data[0]), + UpdatePrognosticsImplicitData(baseline_data[1]), + UpdatePrognosticsImplicitData(baseline_data[2]), + UpdatePrognosticsImplicitData(baseline_data[3]), }; // Assume all data is in C layout - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - update_prognostics_implicit(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - update_prognostics_implicit_f(d.shcol, d.nlev, d.nlevi, d.num_tracer, d.dtime, - d.dz_zt, d.dz_zi, d.rho_zt, d.zt_grid, d.zi_grid, - d.tk, d.tkh, d.uw_sfc, d.vw_sfc, d.wthl_sfc, d.wqw_sfc, - d.wtracer_sfc, d.thetal, d.qw, d.tracer, d.tke, d.u_wind, d.v_wind); - d.transpose(); // go back to C layout + update_prognostics_implicit(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(f90_data) / sizeof(UpdatePrognosticsImplicitData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(UpdatePrognosticsImplicitData); for (Int i = 0; i < num_runs; ++i) { - UpdatePrognosticsImplicitData& d_f90 = f90_data[i]; + UpdatePrognosticsImplicitData& d_baseline = baseline_data[i]; UpdatePrognosticsImplicitData& d_cxx = cxx_data[i]; - REQUIRE(d_f90.total(d_f90.thetal) == d_cxx.total(d_cxx.thetal)); - REQUIRE(d_f90.total(d_f90.qw) == d_cxx.total(d_cxx.qw)); - REQUIRE(d_f90.total(d_f90.tke) == d_cxx.total(d_cxx.tke)); - REQUIRE(d_f90.total(d_f90.u_wind) == d_cxx.total(d_cxx.u_wind)); - REQUIRE(d_f90.total(d_f90.v_wind) == d_cxx.total(d_cxx.v_wind)); - for (Int k = 0; k < d_f90.total(d_f90.thetal); ++k) { - REQUIRE(d_f90.thetal[k] == d_cxx.thetal[k]); - REQUIRE(d_f90.qw[k] == d_cxx.qw[k]); - REQUIRE(d_f90.tke[k] == d_cxx.tke[k]); - REQUIRE(d_f90.u_wind[k] == d_cxx.u_wind[k]); - REQUIRE(d_f90.v_wind[k] == d_cxx.v_wind[k]); + REQUIRE(d_baseline.total(d_baseline.thetal) == d_cxx.total(d_cxx.thetal)); + REQUIRE(d_baseline.total(d_baseline.qw) == d_cxx.total(d_cxx.qw)); + REQUIRE(d_baseline.total(d_baseline.tke) == d_cxx.total(d_cxx.tke)); + REQUIRE(d_baseline.total(d_baseline.u_wind) == d_cxx.total(d_cxx.u_wind)); + REQUIRE(d_baseline.total(d_baseline.v_wind) == d_cxx.total(d_cxx.v_wind)); + for (Int k = 0; k < d_baseline.total(d_baseline.thetal); ++k) { + REQUIRE(d_baseline.thetal[k] == d_cxx.thetal[k]); + REQUIRE(d_baseline.qw[k] == d_cxx.qw[k]); + REQUIRE(d_baseline.tke[k] == d_cxx.tke[k]); + REQUIRE(d_baseline.u_wind[k] == d_cxx.u_wind[k]); + REQUIRE(d_baseline.v_wind[k] == d_cxx.v_wind[k]); } - REQUIRE(d_f90.total(d_f90.tracer) == d_cxx.total(d_cxx.tracer)); - for (Int k = 0; k < d_f90.total(d_f90.tracer); ++k) { - REQUIRE(d_f90.tracer[k] == d_cxx.tracer[k]); + REQUIRE(d_baseline.total(d_baseline.tracer) == d_cxx.total(d_cxx.tracer)); + for (Int k = 0; k < d_baseline.total(d_baseline.tracer); ++k) { + REQUIRE(d_baseline.tracer[k] == d_cxx.tracer[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (auto& d : cxx_data) { + d.write(Base::m_fid); + } } } // run_bfb @@ -424,14 +418,14 @@ TEST_CASE("update_prognostics_implicit_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestUpdatePrognosticsImplicit; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("update_prognostics_implicit_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestUpdatePrognosticsImplicit; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_varorcovar_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_varorcovar_tests.cpp index 4a743009f24..35d60abfa5f 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_varorcovar_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_varorcovar_tests.cpp @@ -4,7 +4,7 @@ #include "physics/share/physics_constants.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -23,9 +23,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestShocVarorCovar { +struct UnitWrap::UnitTest::TestShocVarorCovar : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 4; @@ -116,13 +116,7 @@ struct UnitWrap::UnitTest::TestShocVarorCovar { } // Call the C++ implementation for variance - SDS.transpose(); - // expects data in fortran layout - calc_shoc_varorcovar_f(SDS.shcol, SDS.nlev, SDS.nlevi, - SDS.tunefac, SDS.isotropy_zi, - SDS.tkh_zi, SDS.dz_zi, - SDS.invar1, SDS.invar2, SDS.varorcovar); - SDS.transpose(); + calc_shoc_varorcovar(SDS); // Check the results for(Int s = 0; s < shcol; ++s) { @@ -179,13 +173,7 @@ struct UnitWrap::UnitTest::TestShocVarorCovar { } // Call the C++ implementation for covariance - SDS.transpose(); - // expects data in fortran layout - calc_shoc_varorcovar_f(SDS.shcol, SDS.nlev, SDS.nlevi, - SDS.tunefac, SDS.isotropy_zi, - SDS.tkh_zi, SDS.dz_zi, - SDS.invar1, SDS.invar2, SDS.varorcovar); - SDS.transpose(); + calc_shoc_varorcovar(SDS); // Check the results for(Int s = 0; s < shcol; ++s) { @@ -263,13 +251,7 @@ struct UnitWrap::UnitTest::TestShocVarorCovar { } // Call the C++ implementation for variance - SDS.transpose(); - // expects data in fortran layout - calc_shoc_varorcovar_f(SDS.shcol, SDS.nlev, SDS.nlevi, - SDS.tunefac, SDS.isotropy_zi, - SDS.tkh_zi, SDS.dz_zi, - SDS.invar1, SDS.invar2, SDS.varorcovar); - SDS.transpose(); + calc_shoc_varorcovar(SDS); // Check the results for(Int s = 0; s < shcol; ++s) { @@ -284,11 +266,11 @@ struct UnitWrap::UnitTest::TestShocVarorCovar { } } -static void run_bfb() +void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - CalcShocVarorcovarData SDS_f90[] = { + CalcShocVarorcovarData SDS_baseline[] = { // shcol, nlev, nlevi, tunefac CalcShocVarorcovarData(10, 71, 72, 1), CalcShocVarorcovarData(10, 12, 13, 1), @@ -297,48 +279,49 @@ static void run_bfb() }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state CalcShocVarorcovarData SDS_cxx[] = { - CalcShocVarorcovarData(SDS_f90[0]), - CalcShocVarorcovarData(SDS_f90[1]), - CalcShocVarorcovarData(SDS_f90[2]), - CalcShocVarorcovarData(SDS_f90[3]), + CalcShocVarorcovarData(SDS_baseline[0]), + CalcShocVarorcovarData(SDS_baseline[1]), + CalcShocVarorcovarData(SDS_baseline[2]), + CalcShocVarorcovarData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(CalcShocVarorcovarData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - calc_shoc_varorcovar(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - calc_shoc_varorcovar_f(d.shcol, d.nlev, d.nlevi, - d.tunefac, d.isotropy_zi, - d.tkh_zi, d.dz_zi, - d.invar1, d.invar2, d.varorcovar); - d.transpose(); + calc_shoc_varorcovar(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(CalcShocVarorcovarData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - CalcShocVarorcovarData& d_f90 = SDS_f90[i]; + CalcShocVarorcovarData& d_baseline = SDS_baseline[i]; CalcShocVarorcovarData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.varorcovar); ++k) { - REQUIRE(d_f90.varorcovar[k] == d_cxx.varorcovar[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.varorcovar); ++k) { + REQUIRE(d_baseline.varorcovar[k] == d_cxx.varorcovar[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } }; @@ -353,14 +336,14 @@ TEST_CASE("shoc_varorcovar_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocVarorCovar; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_varorcovar_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestShocVarorCovar; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_vd_shoc_decomp_and_solve_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_vd_shoc_decomp_and_solve_tests.cpp index 3bd213cf784..27326a0df16 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_vd_shoc_decomp_and_solve_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_vd_shoc_decomp_and_solve_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/util/scream_setup_random_test.hpp" #include "shoc_unit_tests_common.hpp" @@ -14,13 +14,13 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestVdShocDecompandSolve { +struct UnitWrap::UnitTest::TestVdShocDecompandSolve : public UnitWrap::UnitTest::Base { - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - VdShocDecompandSolveData f90_data[] = { + VdShocDecompandSolveData baseline_data[] = { // shcol, nlev, nlevi, dtime, n_rhs VdShocDecompandSolveData(10, 71, 72, 5, 19), VdShocDecompandSolveData(10, 12, 13, 2.5, 7), @@ -28,54 +28,54 @@ struct UnitWrap::UnitTest::TestVdShocDecompandSolve { VdShocDecompandSolveData(2, 7, 8, 1, 1) }; - static constexpr Int num_runs = sizeof(f90_data) / sizeof(VdShocDecompandSolveData); + static constexpr Int num_runs = sizeof(baseline_data) / sizeof(VdShocDecompandSolveData); // Generate random input data. Diagonals in solver data will be overwritten // after results of decomp routine. for (Int i = 0; i < num_runs; ++i) { - VdShocDecompandSolveData& d_f90 = f90_data[i]; - d_f90.randomize(engine); + VdShocDecompandSolveData& d_baseline = baseline_data[i]; + d_baseline.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state VdShocDecompandSolveData cxx_data[] = { - VdShocDecompandSolveData(f90_data[0]), - VdShocDecompandSolveData(f90_data[1]), - VdShocDecompandSolveData(f90_data[2]), - VdShocDecompandSolveData(f90_data[3]) + VdShocDecompandSolveData(baseline_data[0]), + VdShocDecompandSolveData(baseline_data[1]), + VdShocDecompandSolveData(baseline_data[2]), + VdShocDecompandSolveData(baseline_data[3]) }; // Assume all data is in C layout - // Get data from fortran. - for (Int i = 0; i < num_runs; ++i) { - VdShocDecompandSolveData& d_f90 = f90_data[i]; - // expects data in C layout - vd_shoc_decomp_and_solve(d_f90); + // Read baseline data. + if (this->m_baseline_action == COMPARE) { + for (auto& d: baseline_data) { + d.read(Base::m_fid); + } } // Get data from cxx for (Int i = 0; i < num_runs; ++i) { VdShocDecompandSolveData& d_cxx = cxx_data[i]; - - d_cxx.transpose(); // _f expects data in fortran layout - vd_shoc_decomp_and_solve_f(d_cxx.shcol, d_cxx.nlev, d_cxx.nlevi, d_cxx.n_rhs, - d_cxx.kv_term, d_cxx.tmpi, d_cxx.rdp_zt, - d_cxx.dtime, d_cxx.flux, d_cxx.var); - d_cxx.transpose(); // go back to C layout + vd_shoc_decomp_and_solve(d_cxx); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - VdShocDecompandSolveData& d_f90 = f90_data[i]; + VdShocDecompandSolveData& d_baseline = baseline_data[i]; VdShocDecompandSolveData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.var); ++k) { - REQUIRE(d_f90.total(d_f90.var) == d_cxx.total(d_cxx.var)); - REQUIRE(d_f90.var[k] == d_cxx.var[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.var); ++k) { + REQUIRE(d_baseline.total(d_baseline.var) == d_cxx.total(d_cxx.var)); + REQUIRE(d_baseline.var[k] == d_cxx.var[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + cxx_data[i].write(Base::m_fid); + } } } // run_bfb @@ -91,7 +91,7 @@ TEST_CASE("vd_shoc_solve_bfb", "[shoc]") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestVdShocDecompandSolve; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_vertflux_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_vertflux_tests.cpp index 7ac7c7c5316..895040f9b11 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_vertflux_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_vertflux_tests.cpp @@ -3,7 +3,7 @@ #include "shoc_unit_tests_common.hpp" #include "physics/share/physics_constants.hpp" #include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" +#include "shoc_test_data.hpp" #include "share/scream_types.hpp" #include "share/util/scream_setup_random_test.hpp" @@ -21,9 +21,9 @@ namespace shoc { namespace unit_test { template -struct UnitWrap::UnitTest::TestCalcShocVertflux { +struct UnitWrap::UnitTest::TestCalcShocVertflux : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() { static constexpr Int shcol = 2; static constexpr Int nlev = 4; @@ -96,10 +96,7 @@ struct UnitWrap::UnitTest::TestCalcShocVertflux { } // Call the C++ implementation - SDS.transpose(); - // expects data in fortran layout - calc_shoc_vertflux_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.tkh_zi, SDS.dz_zi, SDS.invar, SDS.vertflux); - SDS.transpose(); + calc_shoc_vertflux(SDS); // Check the results for(Int s = 0; s < shcol; ++s) { @@ -133,11 +130,11 @@ struct UnitWrap::UnitTest::TestCalcShocVertflux { } } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - CalcShocVertfluxData SDS_f90[] = { + CalcShocVertfluxData SDS_baseline[] = { // shcol, nlev, nlevi CalcShocVertfluxData(10, 71, 72), CalcShocVertfluxData(10, 12, 13), @@ -146,45 +143,49 @@ struct UnitWrap::UnitTest::TestCalcShocVertflux { }; // Generate random input data - for (auto& d : SDS_f90) { + for (auto& d : SDS_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state CalcShocVertfluxData SDS_cxx[] = { - CalcShocVertfluxData(SDS_f90[0]), - CalcShocVertfluxData(SDS_f90[1]), - CalcShocVertfluxData(SDS_f90[2]), - CalcShocVertfluxData(SDS_f90[3]), + CalcShocVertfluxData(SDS_baseline[0]), + CalcShocVertfluxData(SDS_baseline[1]), + CalcShocVertfluxData(SDS_baseline[2]), + CalcShocVertfluxData(SDS_baseline[3]), }; + static constexpr Int num_runs = sizeof(SDS_baseline) / sizeof(CalcShocVertfluxData); + // Assume all data is in C layout - // Get data from fortran - for (auto& d : SDS_f90) { - // expects data in C layout - calc_shoc_vertflux(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : SDS_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : SDS_cxx) { - d.transpose(); - // expects data in fortran layout - calc_shoc_vertflux_f(d.shcol, d.nlev, d.nlevi, d.tkh_zi, d.dz_zi, d.invar, d.vertflux); - d.transpose(); + calc_shoc_vertflux(d); } // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - static constexpr Int num_runs = sizeof(SDS_f90) / sizeof(CalcShocVertfluxData); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - CalcShocVertfluxData& d_f90 = SDS_f90[i]; + CalcShocVertfluxData& d_baseline = SDS_baseline[i]; CalcShocVertfluxData& d_cxx = SDS_cxx[i]; - for (Int k = 0; k < d_f90.total(d_f90.vertflux); ++k) { - REQUIRE(d_f90.vertflux[k] == d_cxx.vertflux[k]); + for (Int k = 0; k < d_baseline.total(d_baseline.vertflux); ++k) { + REQUIRE(d_baseline.vertflux[k] == d_cxx.vertflux[k]); } } + } // SCREAM_BFB_TESTING + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + SDS_cxx[i].write(Base::m_fid); + } } } @@ -200,14 +201,14 @@ TEST_CASE("shoc_vertflux_property", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestCalcShocVertflux; - TestStruct::run_property(); + TestStruct().run_property(); } TEST_CASE("shoc_vertflux_bfb", "shoc") { using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestCalcShocVertflux; - TestStruct::run_bfb(); + TestStruct().run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_w3_diag_third_moms_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_w3_diag_third_moms_tests.cpp deleted file mode 100644 index 39e9b63ba51..00000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_w3_diag_third_moms_tests.cpp +++ /dev/null @@ -1,122 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestW3diagThirdMoms { - - static void run_property() - { - - // Tests for the SHOC function: - // w3_diag_third_shoc_moment - - // TEST ONE - // Do series of tests to make sure output is as expected - - // aa0 term - constexpr static Real aa0_test1 = -0.2; - // aa1 term - constexpr static Real aa1_test1 = 5.65; - // x0 term - constexpr static Real x0_test1 = -4.31; - // y0 term - constexpr static Real x1_test1 = 41.05; - // f5 term - constexpr static Real f5_test1 = 4; - - // Initialize data structure for bridging to F90 - W3DiagThirdShocMomentData SDS; - - // Load up the data - SDS.aa0 = aa0_test1; - SDS.aa1 = aa1_test1; - SDS.x0 = x0_test1; - SDS.x1 = x1_test1; - SDS.f5 = f5_test1; - - // Call the fortran implementation - w3_diag_third_shoc_moment(SDS); - - // Verify result is negative - REQUIRE(SDS.w3 < 0); - - // TEST TWO - // Modify parameters to decrease w3 - // decrease this term - constexpr static Real aa1_test2 = 2.65; - - SDS.aa1 = aa1_test2; - - // Call the fortran implementation - w3_diag_third_shoc_moment(SDS); - - // Verify result has decreased - REQUIRE(SDS.w3 < SDS.aa1); - - // TEST THREE - // Modify parameters to get positive result - // x0 term - constexpr static Real x0_test3 = -4.31; - // y0 term - constexpr static Real x1_test3 = -41.05; - // f5 term - constexpr static Real f5_test3 = -4; - - SDS.x0 = x0_test3; - SDS.x1 = x1_test3; - SDS.f5 = f5_test3; - - // Call the fortran implementation - w3_diag_third_shoc_moment(SDS); - - // Verify the result is positive - REQUIRE(SDS.w3 > 0); - - } - - static void run_bfb() - { - // TODO - } - -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace{ - -TEST_CASE("shoc_w3_diag_third_moms_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestW3diagThirdMoms; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_w3_diag_third_moms_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestW3diagThirdMoms; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/shoc/tests/shoc_xy_diag_third_moms_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_xy_diag_third_moms_tests.cpp deleted file mode 100644 index ad6e332e731..00000000000 --- a/components/eamxx/src/physics/shoc/tests/shoc_xy_diag_third_moms_tests.cpp +++ /dev/null @@ -1,112 +0,0 @@ -#include "catch2/catch.hpp" - -#include "shoc_unit_tests_common.hpp" -#include "shoc_functions.hpp" -#include "shoc_functions_f90.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/util/ekat_arch.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace shoc { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestXYdiagThirdMoms { - - static void run_property() - { - - // Tests for the SHOC function: - // x_y_terms_diag_third_shoc_moment - - // TEST ONE - // Call the function twice, with two different sets of terms. - // In the second test the fterms should be larger. Verify - // that the x0 and y0 terms are the same, but that the - // x1 and y1 terms have increased. - - // buoyancy term (isotropy squared * brunt vaisalla frequency) - constexpr static Real buoy_sgs2 = -100; - // f0 term - constexpr static Real f0_test1a = 13000; - // f3 term - constexpr static Real f1_test1a = 8500; - // f4 term - constexpr static Real f2_test1a = 30; - - // Initialize data structure for bridging to F90 - XYTermsDiagThirdShocMomentData SDS; - - // Load up the data - SDS.buoy_sgs2 = buoy_sgs2; - SDS.f0 = f0_test1a; - SDS.f1 = f1_test1a; - SDS.f2 = f2_test1a; - - // Call the fortran implementation - x_y_terms_diag_third_shoc_moment(SDS); - - // Save test results - Real x0_test1a = SDS.x0; - Real y0_test1a = SDS.y0; - Real x1_test1a = SDS.x1; - Real y1_test1a = SDS.y1; - - // Now load up data for second part of test - // Feed in LARGER values - SDS.f0 = 1.2*f0_test1a; - SDS.f1 = 1.2*f1_test1a; - SDS.f2 = 1.2*f2_test1a; - - // Call the fortran implementation - x_y_terms_diag_third_shoc_moment(SDS); - - // Now check the result - - // x0 and y0 terms should NOT have changed - REQUIRE(SDS.x0 == x0_test1a); - REQUIRE(SDS.y0 == y0_test1a); - - // x1 and y1 terms should have increased - REQUIRE(SDS.x1 > x1_test1a); - REQUIRE(SDS.y1 > y1_test1a); - - } - - static void run_bfb() - { - // TODO - } - -}; - -} // namespace unit_test -} // namespace shoc -} // namespace scream - -namespace{ - -TEST_CASE("shoc_xy_diag_third_moms_property", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestXYdiagThirdMoms; - - TestStruct::run_property(); -} - -TEST_CASE("shoc_xy_diag_third_moms_bfb", "shoc") -{ - using TestStruct = scream::shoc::unit_test::UnitWrap::UnitTest::TestXYdiagThirdMoms; - - TestStruct::run_bfb(); -} - -} // namespace diff --git a/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp index 61c61b878c2..c27af18f85b 100644 --- a/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp +++ b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp @@ -64,11 +64,11 @@ void SPA::set_grids(const std::shared_ptr grids_manager) // where a single column of data corresponding to the closest lat/lon pair to // the IOP lat/lon parameters is read from file, and that column data is mapped // to all columns of the IdentityRemapper source fields. - EKAT_REQUIRE_MSG(spa_map_file == "" or spa_map_file == "None" or not m_iop, + EKAT_REQUIRE_MSG(spa_map_file == "" or spa_map_file == "None" or not m_iop_data_manager, "Error! Cannot define spa_remap_file for cases with an Intensive Observation Period defined. " "The IOP class defines it's own remap from file data -> model data.\n"); - SPAHorizInterp = SPAFunc::create_horiz_remapper (m_grid,spa_data_file,spa_map_file, m_iop!=nullptr); + SPAHorizInterp = SPAFunc::create_horiz_remapper (m_grid,spa_data_file,spa_map_file, m_iop_data_manager!=nullptr); // Grab a sw and lw field from the horiz interp, and check sw/lw dim against what we hardcoded in this class auto nswbands_data = SPAHorizInterp->get_src_field(4).get_header().get_identifier().get_layout().dim("swband"); @@ -128,8 +128,8 @@ void SPA::set_grids(const std::shared_ptr grids_manager) // AtmosphereInput object (for reading into standard // grids) or a SpaFunctions::IOPReader (for reading into // an IOP grid). - if (m_iop) { - SPAIOPDataReader = SPAFunc::create_spa_data_reader(m_iop,SPAHorizInterp,spa_data_file); + if (m_iop_data_manager) { + SPAIOPDataReader = SPAFunc::create_spa_data_reader(m_iop_data_manager,SPAHorizInterp,spa_data_file); } else { SPADataReader = SPAFunc::create_spa_data_reader(SPAHorizInterp,spa_data_file); } diff --git a/components/eamxx/src/physics/spa/spa_functions.hpp b/components/eamxx/src/physics/spa/spa_functions.hpp index 6bdba702e70..444b2d9992e 100644 --- a/components/eamxx/src/physics/spa/spa_functions.hpp +++ b/components/eamxx/src/physics/spa/spa_functions.hpp @@ -4,7 +4,7 @@ #include "share/grid/abstract_grid.hpp" #include "share/grid/remap/abstract_remapper.hpp" #include "share/io/scorpio_input.hpp" -#include "share/iop/intensive_observation_period.hpp" +#include "share/atm_process/IOPDataManager.hpp" #include "share/util/scream_time_stamp.hpp" #include "share/scream_types.hpp" @@ -30,7 +30,7 @@ struct SPAFunctions using gid_type = AbstractGrid::gid_type; - using iop_ptr_type = std::shared_ptr; + using iop_data_ptr_type = std::shared_ptr; template using view_1d = typename KT::template view_1d; @@ -128,11 +128,11 @@ struct SPAFunctions }; // SPAInput struct IOPReader { - IOPReader (iop_ptr_type& iop_, + IOPReader (iop_data_ptr_type& iop_, const std::string file_name_, const std::vector& io_fields_, const std::shared_ptr& io_grid_) - : iop(iop_), file_name(file_name_) + : iop_data_manager(iop_), file_name(file_name_) { field_mgr = std::make_shared(io_grid_); for (auto& f : io_fields_) { @@ -141,14 +141,14 @@ struct SPAFunctions } // Set IO info for this grid and file in IOP object - iop->setup_io_info(file_name, io_grid_); + iop_data_manager->setup_io_info(file_name, io_grid_); } void read_variables(const int time_index, const util::TimeStamp& ts) { - iop->read_fields_from_file_for_iop(file_name, field_names, ts, field_mgr, time_index); + iop_data_manager->read_fields_from_file_for_iop(file_name, field_names, ts, field_mgr, time_index); } - iop_ptr_type iop; + iop_data_ptr_type iop_data_manager; std::string file_name; std::vector field_names; std::shared_ptr field_mgr; @@ -175,7 +175,7 @@ struct SPAFunctions static std::shared_ptr create_spa_data_reader ( - iop_ptr_type& iop, + iop_data_ptr_type& iop_data_manager, const std::shared_ptr& horiz_remapper, const std::string& spa_data_file); diff --git a/components/eamxx/src/physics/spa/spa_functions_impl.hpp b/components/eamxx/src/physics/spa/spa_functions_impl.hpp index 287f69a9889..24195b9da1a 100644 --- a/components/eamxx/src/physics/spa/spa_functions_impl.hpp +++ b/components/eamxx/src/physics/spa/spa_functions_impl.hpp @@ -162,7 +162,7 @@ template std::shared_ptr::IOPReader> SPAFunctions:: create_spa_data_reader ( - iop_ptr_type& iop, + iop_data_ptr_type& iop_data_manager, const std::shared_ptr& horiz_remapper, const std::string& spa_data_file) { @@ -171,7 +171,7 @@ create_spa_data_reader ( io_fields.push_back(horiz_remapper->get_src_field(i)); } const auto io_grid = horiz_remapper->get_src_grid(); - return std::make_shared(iop, spa_data_file, io_fields, io_grid); + return std::make_shared(iop_data_manager, spa_data_file, io_fields, io_grid); } /*-----------------------------------------------------------------*/ @@ -521,14 +521,14 @@ ::update_spa_data_from_file( // Set the first/last entries of the spa data, so that linear interp // can extrapolate if the p_tgt is outside the p_src bounds Kokkos::single(Kokkos::PerTeam(team),[&]{ - spa_data_ccn3(icol,0) = 0; + spa_data_ccn3(icol,0) = ccn3(icol,0); for (int isw=0; isw("iop_file"); scorpio::release_file(iop_file); } -void IntensiveObservationPeriod:: +void IOPDataManager:: initialize_iop_file(const util::TimeStamp& run_t0, int model_nlevs) { @@ -310,7 +310,7 @@ initialize_iop_file(const util::TimeStamp& run_t0, m_helper_fields.insert({"model_pressure", model_pressure}); } -void IntensiveObservationPeriod:: +void IOPDataManager:: setup_io_info(const std::string& file_name, const grid_ptr& grid) { @@ -397,7 +397,7 @@ setup_io_info(const std::string& file_name, } } -void IntensiveObservationPeriod:: +void IOPDataManager:: read_fields_from_file_for_iop (const std::string& file_name, const vos& field_names_nc, const vos& field_names_eamxx, @@ -501,7 +501,7 @@ read_fields_from_file_for_iop (const std::string& file_name, } } -void IntensiveObservationPeriod:: +void IOPDataManager:: read_iop_file_data (const util::TimeStamp& current_ts) { // Query to see if we need to load data from IOP file. @@ -749,7 +749,7 @@ read_iop_file_data (const util::TimeStamp& current_ts) m_time_info.time_idx_of_current_data = iop_file_time_idx; } -void IntensiveObservationPeriod:: +void IOPDataManager:: set_fields_from_iop_data(const field_mgr_ptr field_mgr) { if (m_params.get("zero_non_iop_tracers") && field_mgr->has_group("tracers")) { @@ -858,7 +858,7 @@ set_fields_from_iop_data(const field_mgr_ptr field_mgr) }); } -void IntensiveObservationPeriod:: +void IOPDataManager:: correct_temperature_and_water_vapor(const field_mgr_ptr field_mgr) { // Find the first valid level index for t_iop, i.e., first non-zero entry diff --git a/components/eamxx/src/share/iop/intensive_observation_period.hpp b/components/eamxx/src/share/atm_process/IOPDataManager.hpp similarity index 92% rename from components/eamxx/src/share/iop/intensive_observation_period.hpp rename to components/eamxx/src/share/atm_process/IOPDataManager.hpp index 860eb082c42..0ca5e2ef6d1 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.hpp +++ b/components/eamxx/src/share/atm_process/IOPDataManager.hpp @@ -14,11 +14,9 @@ namespace scream { namespace control { /* - * Class which provides functionality for running EAMxx with an intensive - * observation period (IOP). Currently the only use case is the doubly - * periodic model (DP-SCREAM). + * Class which data for an intensive observation period (IOP). */ -class IntensiveObservationPeriod +class IOPDataManager { using vos = std::vector; using field_mgr_ptr = std::shared_ptr; @@ -47,15 +45,15 @@ class IntensiveObservationPeriod // - run_t0: Initial timestamp for the simulation // - model_nlevs: Number of vertical levels in the simulation. Needed since // the iop file contains a (potentially) different number of levels - IntensiveObservationPeriod(const ekat::Comm& comm, - const ekat::ParameterList& params, - const util::TimeStamp& run_t0, - const int model_nlevs, - const Field& hyam, - const Field& hybm); + IOPDataManager(const ekat::Comm& comm, + const ekat::ParameterList& params, + const util::TimeStamp& run_t0, + const int model_nlevs, + const Field& hyam, + const Field& hybm); - // Default destructor - ~IntensiveObservationPeriod(); + // Destructor + ~IOPDataManager(); // Read data from IOP file and store internally. void read_iop_file_data(const util::TimeStamp& current_ts); @@ -197,7 +195,7 @@ class IntensiveObservationPeriod std::map m_iop_file_varnames; std::map m_iop_field_surface_varnames; std::map m_iop_field_type; -}; // class IntensiveObservationPeriod +}; // class IOPDataManager } // namespace control } // namespace scream diff --git a/components/eamxx/src/share/atm_process/atmosphere_process.cpp b/components/eamxx/src/share/atm_process/atmosphere_process.cpp index ab6d34310ce..6a85097539a 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process.cpp @@ -896,6 +896,8 @@ get_field_in_impl(const std::string& field_name, const std::string& grid_name) c " field name: " + field_name + "\n" " grid name: " + grid_name + "\n"); } + static Field f; + return f; } Field& AtmosphereProcess:: @@ -917,6 +919,8 @@ get_field_in_impl(const std::string& field_name) const { " atm proc name: " + this->name() + "\n" " field name: " + field_name + "\n"); } + static Field f; + return f; } Field& AtmosphereProcess:: @@ -932,6 +936,8 @@ get_field_out_impl(const std::string& field_name, const std::string& grid_name) " field name: " + field_name + "\n" " grid name: " + grid_name + "\n"); } + static Field f; + return f; } Field& AtmosphereProcess:: @@ -953,6 +959,8 @@ get_field_out_impl(const std::string& field_name) const { " atm proc name: " + this->name() + "\n" " field name: " + field_name + "\n"); } + static Field f; + return f; } FieldGroup& AtmosphereProcess:: @@ -968,6 +976,8 @@ get_group_in_impl(const std::string& group_name, const std::string& grid_name) c " group name: " + group_name + "\n" " grid name: " + grid_name + "\n"); } + static FieldGroup g(""); + return g; } FieldGroup& AtmosphereProcess:: @@ -989,6 +999,8 @@ get_group_in_impl(const std::string& group_name) const { " atm proc name: " + this->name() + "\n" " group name: " + group_name + "\n"); } + static FieldGroup g(""); + return g; } FieldGroup& AtmosphereProcess:: @@ -1004,6 +1016,8 @@ get_group_out_impl(const std::string& group_name, const std::string& grid_name) " group name: " + group_name + "\n" " grid name: " + grid_name + "\n"); } + static FieldGroup g(""); + return g; } FieldGroup& AtmosphereProcess:: @@ -1025,6 +1039,8 @@ get_group_out_impl(const std::string& group_name) const { " atm proc name: " + this->name() + "\n" " group name: " + group_name + "\n"); } + static FieldGroup g(""); + return g; } Field& AtmosphereProcess:: @@ -1040,6 +1056,8 @@ get_internal_field_impl(const std::string& field_name, const std::string& grid_n " field name: " + field_name + "\n" " grid name: " + grid_name + "\n"); } + static Field f; + return f; } Field& AtmosphereProcess:: @@ -1061,6 +1079,8 @@ get_internal_field_impl(const std::string& field_name) const { " atm proc name: " + this->name() + "\n" " field name: " + field_name + "\n"); } + static Field f; + return f; } void AtmosphereProcess diff --git a/components/eamxx/src/share/atm_process/atmosphere_process.hpp b/components/eamxx/src/share/atm_process/atmosphere_process.hpp index 43fc35a3318..016321ea506 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process.hpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process.hpp @@ -1,10 +1,10 @@ #ifndef SCREAM_ATMOSPHERE_PROCESS_HPP #define SCREAM_ATMOSPHERE_PROCESS_HPP -#include "share/iop/intensive_observation_period.hpp" #include "share/atm_process/atmosphere_process_utils.hpp" #include "share/atm_process/ATMBufferManager.hpp" #include "share/atm_process/SCDataManager.hpp" +#include "share/atm_process/IOPDataManager.hpp" #include "share/field/field_identifier.hpp" #include "share/field/field_manager.hpp" #include "share/property_checks/property_check.hpp" @@ -83,7 +83,7 @@ class AtmosphereProcess : public ekat::enable_shared_from_this; - using iop_ptr = std::shared_ptr; + using iop_data_ptr = std::shared_ptr; // Base constructor to set MPI communicator and params AtmosphereProcess (const ekat::Comm& comm, const ekat::ParameterList& params); @@ -280,8 +280,8 @@ class AtmosphereProcess : public ekat::enable_shared_from_this get_logger () const { @@ -597,7 +597,7 @@ class AtmosphereProcess : public ekat::enable_shared_from_this\n" - << " " << html_fix(n.name) << ""; - if (verbosity>1) { + << " " << html_fix(n.name) + << "\n"; + + int sz_comp = n.computed.size(), sz_req = n.required.size(), + sz_grcomp = n.gr_computed.size(), sz_grreq = n.gr_required.size(); + int nfield = sz_comp + sz_req + sz_grcomp + sz_grreq; + if (verbosity > 1 && nfield > 0) { // FieldIntentifier prints bare min with verb 0. // DAG starts printing fids with verb 2, so fid verb is verb-2; int fid_verb = verbosity-2; - ofile << "
\n"; + ofile << "
\n"; + + if (sz_comp > 0) { + // Computed fields + if (n.id == id_begin) { + ofile << " " + << "Atm input fields from previous time step:\n"; + } else if (n.id == id_IC) { + ofile << " " + << "Initial Fields:\n"; + } else if (n.id != id_end) { + ofile << " " + << "Computed Fields:\n"; + } - // Computed fields - if (n.name=="Begin of atm time step") { - ofile << " Atm input fields from previous time step:\n"; - } else if (n.name!="End of atm time step"){ - ofile << " Computed Fields:\n"; - } - for (const auto& fid : n.computed) { - std::string fc = " "; - ofile << " " << fc << html_fix(print_fid(m_fids[fid],fid_verb)) << "\n"; + for (const auto& fid : n.computed) { + std::string fc = " "; + ofile << " " << fc + << html_fix(print_fid(m_fids[fid_out], fid_verb)) + << "\n"; + } } - // Required fields - if (n.name=="End of atm time step") { - ofile << " Atm output fields for next time step:\n"; - } else if (n.name!="Begin of atm time step") { - ofile << " Required Fields:\n"; - } - for (const auto& fid : n.required) { - std::string fc = " "; - ofile << " " << fc << html_fix(print_fid(m_fids[fid],fid_verb)); - if (ekat::contains(m_unmet_deps.at(n.id),fid)) { - ofile << " *** MISSING ***"; + if (sz_req > 0) { + // Required fields + if (n.id == id_end) { + ofile << " " + << "Atm output fields for next time step:\n"; + } else if (n.id != id_begin && n.id != id_IC) { + ofile << " " + << "Required Fields:\n"; + } + for (const auto& fid : n.required) { + std::string fc = " "; + ofile << " " << fc << html_fix(print_fid(m_fids[fid],fid_verb)); + if (ekat::contains(unmet, fid)) { + ofile << " *** MISSING ***"; + } else if (ekat::contains(unmet, -fid)) { + ofile << " (Init. Cond.)"; + has_IC_field = true; + } + ofile << "\n"; } - ofile << "\n"; } // Computed groups - if (n.gr_computed.size()>0) { - if (n.name=="Begin of atm time step") { - ofile << " Atm Input groups:\n"; - } else if (n.name!="End of atm time step"){ - ofile << " Computed Groups:\n"; + if (sz_grcomp > 0) { + if (n.id == id_begin) { + ofile << " Atm Input groups:\n"; + } else if (n.id != id_end){ + ofile << " Computed Groups:\n"; } for (const auto& gr_fid : n.gr_computed) { std::string fc = " "; ofile << " " << fc << html_fix(print_fid(m_fids[gr_fid],fid_verb)); ofile << "\n"; @@ -242,10 +293,8 @@ void AtmProcDAG::write_dag (const std::string& fname, const int verbosity) const size_t i = 0; for (const auto& fn : members_names) { const auto f = members.at(fn); - const auto& mfid = f->get_header().get_identifier(); - const auto mfid_id = get_fid_index(mfid); std::string mfc = ""; if (len>0) { ofile << ","; @@ -269,18 +318,18 @@ void AtmProcDAG::write_dag (const std::string& fname, const int verbosity) const } // Required groups - if (n.gr_required.size()>0) { + if (sz_grreq > 0) { if (n.name=="End of atm time step") { - ofile << " Atm Output Groups:\n"; + ofile << " Atm Output Groups:\n"; } else if (n.name!="Begin of atm time step") { - ofile << " Required Groups:\n"; + ofile << " Required Groups:\n"; } for (const auto& gr_fid : n.gr_required) { std::string fc = " "; ofile << " " << fc << html_fix(print_fid(m_fids[gr_fid],fid_verb)); - if (ekat::contains(m_unmet_deps.at(n.id),gr_fid)) { + if (ekat::contains(unmet, gr_fid)) { ofile << " *** MISSING ***"; } ofile << "\n"; @@ -327,10 +376,45 @@ void AtmProcDAG::write_dag (const std::string& fname, const int verbosity) const // Write all outgoing edges for (const auto c : n.children) { - ofile << n.id << "->" << c << "\n"; + ofile << n.id << "->" << c << "[penwidth=4];\n"; } } + if (!m_IC_processed && m_has_unmet_deps) { + int this_node_id = m_nodes.size() + 1; + ofile << this_node_id << " [\n" + << " shape=box\n" + << " color=\"#605d57\"\n" + << " fontcolor=\"#034a4a\"\n" + << " penwidth=8\n" + << " fontsize=40\n" + << " style=filled\n" + << " fillcolor=\"#999999\"\n" + << " align=\"center\"\n" + << " label=<NOTE: " + "Fields marked missing may be
provided by " + "the as-yet-unprocessed
initial condition>\n" + << "];\n"; + } + + if (m_IC_processed && has_IC_field) { + int this_node_id = m_nodes.size() + 1; + ofile << this_node_id << " [\n" + << " shape=box\n" + << " color=\"#605d57\"\n" + << " fontcolor=\"#031576\"\n" + << " penwidth=8\n" + << " fontsize=40\n" + << " style=filled\n" + << " fillcolor=\"#cccccc\"\n" + << " align=\"center\"\n" + << " label=<NOTE: Fields denoted " + "with green text " + "
indicate the field was provided by the " + "
initial conditions and never updated>\n" + << "];\n"; + } + // Close the file ofile << "}"; ofile.close(); @@ -341,6 +425,7 @@ void AtmProcDAG::cleanup () { m_fid_to_last_provider.clear(); m_unmet_deps.clear(); m_has_unmet_deps = false; + m_IC_processed = false; } void AtmProcDAG:: @@ -364,10 +449,9 @@ add_nodes (const group_type& atm_procs) add_nodes(*group); } else { // Create a node for the process - // Node& node = m_nodes[proc->name()]; int id = m_nodes.size(); m_nodes.push_back(Node()); - Node& node = m_nodes.back();; + Node& node = m_nodes.back(); node.id = id; node.name = proc->name(); m_unmet_deps[id].clear(); // Ensures an entry for this id is in the map @@ -507,6 +591,69 @@ void AtmProcDAG::add_edges () { } } +void AtmProcDAG::process_initial_conditions(const grid_field_map &ic_inited) { + // First, add the fields that were determined to come from the previous time + // step => IC for t = 0 + + // Create a node for the ICs by copying the begin_node. Recall that so far + // m_nodes contains [, 'beg-of-step', 'end-of-step'] + // WARNING: do NOT get a ref to beg-of-step node, since calls to m_nodes.push_back + // may resize the vector, invalidating the reference. + auto begin_node = m_nodes[m_nodes.size()-2]; + auto& ic_node = m_nodes.emplace_back(begin_node); + + // now set/clear the basic data for the ic_node + int id = m_nodes.size(); + ic_node.id = id; + ic_node.name = "Initial Conditions"; + m_unmet_deps[id].clear(); + ic_node.children.clear(); + // now add the begin_node as a child of the ic_node + ic_node.children.push_back(begin_node.id); + // return if there's nothing to process in the ic_inited vector + if (ic_inited.size() == 0) { + return; + } + std::set to_be_marked; + for (auto &node : m_nodes) { + if (m_unmet_deps.at(node.id).empty()) { + continue; + } else { + // NOTE: node_unmet_fields is a std::set + auto &node_unmet_fields = m_unmet_deps.at(node.id); + // add the current node as a child of the IC node + ic_node.children.push_back(node.id); + for (auto &um_fid : node_unmet_fields) { + for (auto &it1 : ic_inited) { + const auto &grid_name = it1.first; + // if this unmet-dependency field's name is in the ic_inited map for + // the provided grid_name key, we record the field id in to_be_marked + // (because changing it messes up the iterator) + if (ekat::contains(ic_inited.at(grid_name), m_fids[um_fid].name())) { + to_be_marked.insert(um_fid); + // add the fid of the formerly unmet dep to the initial condition + // node's computed list + ic_node.computed.insert(um_fid); + } else { + continue; + } + } + } + if (to_be_marked.empty()) { + continue; + } else { + // change the previously unmet dependency's field id to be negative, + // indicating that it is now met and provided by the initial condition + for (auto &fid : to_be_marked) { + node_unmet_fields.erase(fid); + node_unmet_fields.insert(-fid); + } + } + } + } + m_IC_processed = true; +} + int AtmProcDAG::add_fid (const FieldIdentifier& fid) { auto it = ekat::find(m_fids,fid); if (it==m_fids.end()) { diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_dag.hpp b/components/eamxx/src/share/atm_process/atmosphere_process_dag.hpp index 1a88381ecc8..52a0f9d8cd9 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_dag.hpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_dag.hpp @@ -16,6 +16,11 @@ class AtmProcDAG { void create_dag (const group_type& atm_procs); + using grid_field_map = std::map>; + void process_initial_conditions(const grid_field_map &ic_inited); + + void init_atm_proc_nodes(const group_type& atm_procs); + void add_surface_coupling (const std::set& imports, const std::set& exports); @@ -66,6 +71,7 @@ class AtmProcDAG { // Map a node id to a set of unmet field dependencies std::map> m_unmet_deps; bool m_has_unmet_deps; + bool m_IC_processed; // The nodes in the atm DAG std::vector m_nodes; diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_group.hpp b/components/eamxx/src/share/atm_process/atmosphere_process_group.hpp index 6af2b2434b3..43b5f576984 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_group.hpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_group.hpp @@ -106,9 +106,9 @@ class AtmosphereProcessGroup : public AtmosphereProcess void add_additional_data_fields_to_property_checks (const Field& data_field); // Loop through all proceeses in group and set IOP object - void set_iop(const iop_ptr& iop) { + void set_iop_data_manager(const iop_data_ptr& iop_data_manager) { for (auto& atm_proc : m_atm_processes) { - atm_proc->set_iop(iop); + atm_proc->set_iop_data_manager(iop_data_manager); } } diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index a338199f2ef..3fd15aa03ac 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -349,6 +349,7 @@ class Field { get_subview_1 (const get_view_type,HD>&, const int) const { EKAT_ERROR_MSG ("Error! Cannot subview a rank2 view along the second " "dimension without losing LayoutRight.\n"); + return get_view_type,HD>(); } template diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index 3d37853f478..e1d8c182d0e 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -966,6 +966,7 @@ auto Field::get_ND_view () const "MaxRank = 6.\n" "This should never be called at run time.\n" "Please contact developer if this functionality is required\n"); + return get_view_type,HD>(); } } // namespace scream diff --git a/components/eamxx/src/share/field/field_utils.hpp b/components/eamxx/src/share/field/field_utils.hpp index 8f977a7caa1..c6acbfc4079 100644 --- a/components/eamxx/src/share/field/field_utils.hpp +++ b/components/eamxx/src/share/field/field_utils.hpp @@ -11,16 +11,18 @@ inline bool views_are_equal(const Field& f1, const Field& f2, const ekat::Comm* EKAT_REQUIRE_MSG (f1.data_type()==f2.data_type(), "Error! Views have different data types.\n"); + bool ret = false; switch (f1.data_type()) { case DataType::IntType: - return impl::views_are_equal(f1,f2,comm); + ret = impl::views_are_equal(f1,f2,comm); break; case DataType::FloatType: - return impl::views_are_equal(f1,f2,comm); + ret = impl::views_are_equal(f1,f2,comm); break; case DataType::DoubleType: - return impl::views_are_equal(f1,f2,comm); + ret = impl::views_are_equal(f1,f2,comm); break; default: EKAT_ERROR_MSG ("Error! Unrecognized field data type.\n"); } + return ret; } template @@ -111,6 +113,162 @@ void perturb (const Field& f, impl::perturb(f, engine, pdf, base_seed, level_mask, dof_gids); } +// Utility to compute the contraction of a field along its column dimension. +// This is equivalent to f_out = einsum('i,i...k->...k', weight, f_in). +// The impl is such that: +// - f_out, f_in, and weight must be provided and allocated +// - The first dimension is for the columns (COL) +// - There can be only up to 3 dimensions of f_in +template +void horiz_contraction(const Field &f_out, const Field &f_in, + const Field &weight, const ekat::Comm *comm = nullptr) { + using namespace ShortFieldTagsNames; + + const auto &l_out = f_out.get_header().get_identifier().get_layout(); + + const auto &l_in = f_in.get_header().get_identifier().get_layout(); + + const auto &l_w = weight.get_header().get_identifier().get_layout(); + + // Sanity checks before handing off to the implementation + EKAT_REQUIRE_MSG(l_w.rank() == 1, + "Error! The weight field must be rank-1.\n" + "The input has rank " + << l_w.rank() << ".\n"); + EKAT_REQUIRE_MSG(l_w.tags() == std::vector({COL}), + "Error! The weight field must have a column dimension.\n" + "The input f1 layout is " + << l_w.tags() << ".\n"); + EKAT_REQUIRE_MSG(l_in.rank() <= 3, + "Error! The input field must be at most rank-3.\n" + "The input f_in rank is " + << l_in.rank() << ".\n"); + EKAT_REQUIRE_MSG(l_in.tags()[0] == COL, + "Error! The input field must have a column dimension.\n" + "The input f_in layout is " + << l_in.to_string() << ".\n"); + EKAT_REQUIRE_MSG( + l_w.dim(0) == l_in.dim(0), + "Error! input and weight fields must have the same dimension along " + "which we are taking the reducing the field.\n" + "The weight field has dimension " + << l_w.dim(0) + << " while " + "the input field has dimension " + << l_in.dim(0) << ".\n"); + EKAT_REQUIRE_MSG( + l_in.dim(0) > 0, + "Error! The input field must have a non-zero column dimension.\n" + "The input f_in layout is " + << l_in.to_string() << ".\n"); + EKAT_REQUIRE_MSG( + l_out == l_in.clone().strip_dim(0), + "Error! The output field must have the same layout as the input field " + "without the column dimension.\n" + "The input f_in layout is " + << l_in.to_string() << " and the output f_out layout is " + << l_out.to_string() << ".\n"); + EKAT_REQUIRE_MSG( + f_out.is_allocated() && f_in.is_allocated() && weight.is_allocated(), + "Error! All fields must be allocated."); + EKAT_REQUIRE_MSG(f_out.data_type() == f_in.data_type(), + "Error! In/out Fields have matching data types."); + EKAT_REQUIRE_MSG( + f_out.data_type() == weight.data_type(), + "Error! Weight field must have the same data type as input fields."); + + // All good, call the implementation + impl::horiz_contraction(f_out, f_in, weight, comm); +} + +// Utility to compute the contraction of a field along its level dimension. +// This is equivalent to f_out = einsum('...k->...', weight, f_in). +// The impl is such that: +// - f_out, f_in, and weight must be provided and allocated +// - The last dimension is for the levels (LEV/ILEV) +// - There can be only up to 3 dimensions of f_in +// - Weight is assumed to be (in order of checking/impl): +// - rank-1, with only LEV/ILEV dimension +// - rank-2, with only COL and LEV/ILEV dimensions +template +void vert_contraction(const Field &f_out, const Field &f_in, + const Field &weight, const ekat::Comm *comm = nullptr) { + using namespace ShortFieldTagsNames; + + const auto &l_out = f_out.get_header().get_identifier().get_layout(); + + const auto &l_in = f_in.get_header().get_identifier().get_layout(); + + const auto &l_w = weight.get_header().get_identifier().get_layout(); + + // Sanity checks before handing off to the implementation + EKAT_REQUIRE_MSG( + l_w.rank() == 1 or l_w.rank() == 2, + "Error! The weight field must be at least rank-1 and at most rank-2.\n" + "The weight field has rank " + << l_w.rank() << ".\n"); + EKAT_REQUIRE_MSG( + l_w.tags().back() == LEV or l_w.tags().back() == ILEV, + "Error! The weight field must have LEV as its last dimension.\n" + "The weight field layout is " + << l_w.to_string() << ".\n"); + EKAT_REQUIRE_MSG(l_in.rank() <= 3, + "Error! The input field must be at most rank-3.\n" + "The input field rank is " + << l_in.rank() << ".\n"); + EKAT_REQUIRE_MSG(l_in.rank() >= l_w.rank(), + "Error! The input field must have at least as many " + "dimensions as the weight field.\n" + "The input field rank is " + << l_in.rank() << " and the weight field rank is " + << l_w.rank() << ".\n"); + EKAT_REQUIRE_MSG(l_in.tags().back() == LEV or l_in.tags().back() == ILEV, + "Error! The input field must have a level dimension.\n" + "The input field layout is " + << l_in.to_string() << ".\n"); + EKAT_REQUIRE_MSG( + l_in.dims().back() == l_w.dims().back(), + "Error! input and weight fields must have the same dimension along " + "which we are taking the reducing the field (last dimensions).\n" + "The weight field has last dimension " + << l_w.dims().back() + << " while " + "the input field has last dimension " + << l_in.dims().back() << ".\n"); + EKAT_REQUIRE_MSG( + l_in.dims().back() > 0, + "Error! The input field must have a non-zero level dimension.\n" + "The input field layout is " + << l_in.to_string() << ".\n"); + if(l_w.rank() == 2) { + EKAT_REQUIRE_MSG(l_w.congruent(l_in.clone().strip_dim(CMP, false)), + "Error! Incompatible layouts\n" + " field in: " + + l_in.to_string() + + "\n" + " weight: " + + l_w.to_string() + "\n"); + } + EKAT_REQUIRE_MSG( + l_out == l_in.clone().strip_dim(l_in.rank() - 1), + "Error! The output field must have the same layout as the input field " + "without the level dimension.\n" + "The input field layout is " + << l_in.to_string() << " and the output field layout is " + << l_out.to_string() << ".\n"); + EKAT_REQUIRE_MSG( + f_out.is_allocated() && f_in.is_allocated() && weight.is_allocated(), + "Error! All fields must be allocated."); + EKAT_REQUIRE_MSG(f_out.data_type() == f_in.data_type(), + "Error! In/out Fields have matching data types."); + EKAT_REQUIRE_MSG( + f_out.data_type() == weight.data_type(), + "Error! Weight field must have the same data type as input field."); + + // All good, call the implementation + impl::vert_contraction(f_out, f_in, weight, comm); +} + template ST frobenius_norm(const Field& f, const ekat::Comm* comm = nullptr) { diff --git a/components/eamxx/src/share/field/field_utils_impl.hpp b/components/eamxx/src/share/field/field_utils_impl.hpp index eaedb1f62dd..94584cbd043 100644 --- a/components/eamxx/src/share/field/field_utils_impl.hpp +++ b/components/eamxx/src/share/field/field_utils_impl.hpp @@ -5,6 +5,8 @@ #include "ekat/mpi/ekat_comm.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" + #include #include @@ -293,6 +295,165 @@ void perturb (const Field& f, } } +template +void horiz_contraction(const Field &f_out, const Field &f_in, + const Field &weight, const ekat::Comm *comm) { + using KT = ekat::KokkosTypes; + using RangePolicy = Kokkos::RangePolicy; + using TeamPolicy = Kokkos::TeamPolicy; + using TeamMember = typename TeamPolicy::member_type; + using ESU = ekat::ExeSpaceUtils; + + auto l_out = f_out.get_header().get_identifier().get_layout(); + auto l_in = f_in.get_header().get_identifier().get_layout(); + + auto v_w = weight.get_view(); + + const int ncols = l_in.dim(0); + + switch(l_in.rank()) { + case 1: { + auto v_in = f_in.get_view(); + auto v_out = f_out.get_view(); + Kokkos::parallel_reduce( + f_out.name(), RangePolicy(0, ncols), + KOKKOS_LAMBDA(const int i, ST &ls) { ls += v_w(i) * v_in(i); }, + v_out); + } break; + case 2: { + auto v_in = f_in.get_view(); + auto v_out = f_out.get_view(); + const int d1 = l_in.dim(1); + auto p = ESU::get_default_team_policy(d1, ncols); + Kokkos::parallel_for( + f_out.name(), p, KOKKOS_LAMBDA(const TeamMember &tm) { + const int j = tm.league_rank(); + Kokkos::parallel_reduce( + Kokkos::TeamVectorRange(tm, ncols), + [&](int i, ST &ac) { ac += v_w(i) * v_in(i, j); }, v_out(j)); + }); + } break; + case 3: { + auto v_in = f_in.get_view(); + auto v_out = f_out.get_view(); + const int d1 = l_in.dim(1); + const int d2 = l_in.dim(2); + auto p = ESU::get_default_team_policy(d1 * d2, ncols); + Kokkos::parallel_for( + f_out.name(), p, KOKKOS_LAMBDA(const TeamMember &tm) { + const int idx = tm.league_rank(); + const int j = idx / d2; + const int k = idx % d2; + Kokkos::parallel_reduce( + Kokkos::TeamVectorRange(tm, ncols), + [&](int i, ST &ac) { ac += v_w(i) * v_in(i, j, k); }, + v_out(j, k)); + }); + } break; + default: + EKAT_ERROR_MSG("Error! Unsupported field rank.\n"); + } + + if(comm) { + // TODO: use device-side MPI calls + // TODO: the dev ptr causes problems; revisit this later + // TODO: doing cuda-aware MPI allreduce would be ~10% faster + Kokkos::fence(); + f_out.sync_to_host(); + comm->all_reduce(f_out.template get_internal_view_data(), + l_out.size(), MPI_SUM); + f_out.sync_to_dev(); + } +} + +template +void vert_contraction(const Field &f_out, const Field &f_in, + const Field &weight, const ekat::Comm *comm) { + using KT = ekat::KokkosTypes; + using RangePolicy = Kokkos::RangePolicy; + using TeamPolicy = Kokkos::TeamPolicy; + using TeamMember = typename TeamPolicy::member_type; + using ESU = ekat::ExeSpaceUtils; + + auto l_out = f_out.get_header().get_identifier().get_layout(); + auto l_in = f_in.get_header().get_identifier().get_layout(); + auto l_w = weight.get_header().get_identifier().get_layout(); + + const int nlevs = l_in.dim(l_in.rank() - 1); + + // To avoid duplicating code for the 1d and 2d weight cases, + // we use a view to access the weight ahead of time + typename Field::get_view_type w1d; + typename Field::get_view_type w2d; + auto w_is_1d = l_w.rank() == 1; + if(w_is_1d) { + w1d = weight.get_view(); + } else { + w2d = weight.get_view(); + } + + switch(l_in.rank()) { + case 1: { + auto v_w = weight.get_view(); + auto v_in = f_in.get_view(); + auto v_out = f_out.get_view(); + Kokkos::parallel_reduce( + f_out.name(), RangePolicy(0, nlevs), + KOKKOS_LAMBDA(const int i, ST &ls) { ls += v_w(i) * v_in(i); }, + v_out); + } break; + case 2: { + auto v_in = f_in.get_view(); + auto v_out = f_out.get_view(); + const int d0 = l_in.dim(0); + auto p = ESU::get_default_team_policy(d0, nlevs); + Kokkos::parallel_for( + f_out.name(), p, KOKKOS_LAMBDA(const TeamMember &tm) { + const int i = tm.league_rank(); + Kokkos::parallel_reduce( + Kokkos::TeamVectorRange(tm, nlevs), + [&](int j, ST &ac) { + ac += w_is_1d ? w1d(j) * v_in(i, j) : w2d(i, j) * v_in(i, j); + }, + v_out(i)); + }); + } break; + case 3: { + auto v_in = f_in.get_view(); + auto v_out = f_out.get_view(); + const int d0 = l_in.dim(0); + const int d1 = l_in.dim(1); + auto p = ESU::get_default_team_policy(d0 * d1, nlevs); + Kokkos::parallel_for( + f_out.name(), p, KOKKOS_LAMBDA(const TeamMember &tm) { + const int idx = tm.league_rank(); + const int i = idx / d1; + const int j = idx % d1; + Kokkos::parallel_reduce( + Kokkos::TeamVectorRange(tm, nlevs), + [&](int k, ST &ac) { + ac += w_is_1d ? w1d(k) * v_in(i, j, k) + : w2d(i, k) * v_in(i, j, k); + }, + v_out(i, j)); + }); + } break; + default: + EKAT_ERROR_MSG("Error! Unsupported field rank in vert_contraction.\n"); + } + + if(comm) { + // TODO: use device-side MPI calls + // TODO: the dev ptr causes problems; revisit this later + // TODO: doing cuda-aware MPI allreduce would be ~10% faster + Kokkos::fence(); + f_out.sync_to_host(); + comm->all_reduce(f_out.template get_internal_view_data(), + l_out.size(), MPI_SUM); + f_out.sync_to_dev(); + } +} + template ST frobenius_norm(const Field& f, const ekat::Comm* comm) { diff --git a/components/eamxx/src/share/grid/grids_manager.cpp b/components/eamxx/src/share/grid/grids_manager.cpp index 77300aa51c9..45494947aca 100644 --- a/components/eamxx/src/share/grid/grids_manager.cpp +++ b/components/eamxx/src/share/grid/grids_manager.cpp @@ -7,7 +7,24 @@ auto GridsManager:: get_grid(const std::string& name) const -> grid_ptr_type { - auto g = get_grid_nonconst(name); + EKAT_REQUIRE_MSG (has_grid(name), + "Error! Grids manager '" + this->name() + "' does not provide grid '" + name + "'.\n" + " Avaialble grids are: " + print_available_grids() + "\n"); + + grid_ptr_type g; + for (const auto& it : m_grids) { + if (it.second->name()==name or + ekat::contains(it.second->aliases(),name)) { + g = it.second; + break; + } + } + + EKAT_REQUIRE_MSG (g!=nullptr, + "Something went wrong while looking up a grid.\n" + " - grids manager: " + this->name() + "\n" + " - grid name : " + name + "\n"); + return g; } @@ -51,7 +68,14 @@ create_remapper (const grid_ptr_type& from_grid, } void GridsManager:: -add_grid (nonconstgrid_ptr_type grid) +add_nonconst_grid (nonconstgrid_ptr_type grid) +{ + add_grid(grid); + m_nonconst_grids[grid->name()] = grid; +} + +void GridsManager:: +add_grid (grid_ptr_type grid) { const auto& name = grid->name(); EKAT_REQUIRE_MSG (not has_grid(name), @@ -59,7 +83,6 @@ add_grid (nonconstgrid_ptr_type grid) " - grids manager: " + this->name() + "\n" " - grid name : " + name + "\n"); - m_nonconst_grids[name] = grid; m_grids[name] = grid; } @@ -86,7 +109,6 @@ get_grid_nonconst (const std::string& name) const " - grid name : " + name + "\n"); return g; - } void GridsManager:: diff --git a/components/eamxx/src/share/grid/grids_manager.hpp b/components/eamxx/src/share/grid/grids_manager.hpp index 4a0e25bc233..e70bb9678f0 100644 --- a/components/eamxx/src/share/grid/grids_manager.hpp +++ b/components/eamxx/src/share/grid/grids_manager.hpp @@ -56,7 +56,8 @@ class GridsManager protected: - void add_grid (nonconstgrid_ptr_type grid); + void add_nonconst_grid (nonconstgrid_ptr_type grid); + void add_grid (grid_ptr_type grid); void alias_grid (const std::string& grid_name, const std::string& grid_alias); virtual remapper_ptr_type diff --git a/components/eamxx/src/share/grid/library_grids_manager.hpp b/components/eamxx/src/share/grid/library_grids_manager.hpp new file mode 100644 index 00000000000..3e25ea60381 --- /dev/null +++ b/components/eamxx/src/share/grid/library_grids_manager.hpp @@ -0,0 +1,50 @@ +#ifndef EAMXX_LIBRARY_GRIDS_MANAGER_HPP +#define EAMXX_LIBRARY_GRIDS_MANAGER_HPP + +#include "share/grid/grids_manager.hpp" + +namespace scream { + +// This class is meant to be used within scopes that need a grids manager +// object, but they only have pre-built grids. The user can then simply +// create a LibraryGridsManager, and add the pre-existing grids. Afterwards, +// it's business as usual with GridsManager's interfaces. + +class LibraryGridsManager : public GridsManager +{ +public: + template + explicit LibraryGridsManager(Pointers&&... ptrs) { + add_grids(std::forward(ptrs)...); + } + + virtual ~LibraryGridsManager () = default; + + std::string name () const { return "Library grids_manager"; } + + void build_grids () override {} + + void add_grids () {} + + template + void add_grids (grid_ptr_type p, Pointers&&... ptrs) { + add_grid(p); + add_grids(std::forward(ptrs)...); + } + +protected: + remapper_ptr_type + do_create_remapper (const grid_ptr_type from_grid, + const grid_ptr_type to_grid) const + { + EKAT_ERROR_MSG ( + "Error! LibraryGridsManager is not capable of creating remappers.\n" + " - from_grid: " + from_grid->name() + "\n" + " - to_grid: " + to_grid->name() + "\n"); + return nullptr; + } +}; + +} // namespace scream + +#endif // EAMXX_LIBRARY_GRIDS_MANAGER_HPP diff --git a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp index b083dfcfa48..4d505803e66 100644 --- a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp +++ b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp @@ -106,7 +106,7 @@ build_se_grid (const std::string& name, ekat::ParameterList& params) se_grid->m_short_name = "se"; add_geo_data(se_grid); - add_grid(se_grid); + add_nonconst_grid(se_grid); } void MeshFreeGridsManager:: @@ -132,7 +132,7 @@ build_point_grid (const std::string& name, ekat::ParameterList& params) add_geo_data(pt_grid); pt_grid->m_short_name = "pt"; - add_grid(pt_grid); + add_nonconst_grid(pt_grid); } void MeshFreeGridsManager:: diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp index 65da6f2447a..0dc73499ce6 100644 --- a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp @@ -183,7 +183,7 @@ do_register_field (const identifier_type& src, const identifier_type& tgt) { constexpr auto COL = ShortFieldTagsNames::COL; EKAT_REQUIRE_MSG (src.get_layout().has_tag(COL), - "Error! Cannot register a field without COL tag in RefiningRemapperP2P.\n" + "Error! Cannot register a field without COL tag in HorizInterpRemapperBase.\n" " - field name: " + src.name() + "\n" " - field layout: " + src.get_layout().to_string() + "\n"); m_src_fields.push_back(field_type(src)); @@ -194,11 +194,11 @@ void HorizInterpRemapperBase:: do_bind_field (const int ifield, const field_type& src, const field_type& tgt) { EKAT_REQUIRE_MSG (src.data_type()==DataType::RealType, - "Error! RefiningRemapperRMA only allows fields with RealType data.\n" + "Error! HorizInterpRemapperBase only allows fields with RealType data.\n" " - src field name: " + src.name() + "\n" " - src field type: " + e2str(src.data_type()) + "\n"); EKAT_REQUIRE_MSG (tgt.data_type()==DataType::RealType, - "Error! RefiningRemapperRMA only allows fields with RealType data.\n" + "Error! HorizInterpRemapperBase only allows fields with RealType data.\n" " - tgt field name: " + tgt.name() + "\n" " - tgt field type: " + e2str(tgt.data_type()) + "\n"); @@ -352,7 +352,7 @@ local_mat_vec (const Field& x, const Field& y) const } default: { - EKAT_ERROR_MSG("Error::refining_remapper::local_mat_vec doesn't support fields of rank 4 or greater"); + EKAT_ERROR_MSG("[HorizInterpRemapperBase::local_mat_vec] Error! Fields of rank 4 or greater are not supported.\n"); } } } diff --git a/components/eamxx/src/share/grid/remap/refining_remapper_p2p.cpp b/components/eamxx/src/share/grid/remap/refining_remapper_p2p.cpp index f2b9871053e..5f9d91595ec 100644 --- a/components/eamxx/src/share/grid/remap/refining_remapper_p2p.cpp +++ b/components/eamxx/src/share/grid/remap/refining_remapper_p2p.cpp @@ -52,7 +52,7 @@ void RefiningRemapperP2P::do_remap_fwd () for (int i=0; i; + constexpr auto COL = ShortFieldTagsNames::COL; + auto export_pids = m_imp_exp->export_pids(); auto export_lids = m_imp_exp->export_lids(); auto ncols_send = m_imp_exp->num_exports_per_pid(); @@ -174,6 +178,10 @@ void RefiningRemapperP2P::pack_and_send () for (int ifield=0; ifieldtgt later + continue; + } const auto f_col_sizes_scan_sum = m_fields_col_sizes_scan_sum[ifield]; switch (fl.rank()) { case 1: @@ -308,6 +316,8 @@ void RefiningRemapperP2P::recv_and_unpack () using TeamMember = typename KT::MemberType; using ESU = ekat::ExeSpaceUtils; + constexpr auto COL = ShortFieldTagsNames::COL; + auto import_pids = m_imp_exp->import_pids(); auto import_lids = m_imp_exp->import_lids(); auto ncols_recv = m_imp_exp->num_imports_per_pid(); @@ -318,6 +328,10 @@ void RefiningRemapperP2P::recv_and_unpack () for (int ifield=0; ifieldtgt later + continue; + } const auto f_col_sizes_scan_sum = m_fields_col_sizes_scan_sum[ifield]; switch (fl.rank()) { case 1: diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp index 9b524cec5e4..cb55080da38 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp @@ -17,175 +17,229 @@ namespace scream { +std::shared_ptr VerticalRemapper:: -VerticalRemapper (const grid_ptr_type& src_grid, - const std::string& map_file, - const Field& pmid_src, - const Field& pint_src) - : VerticalRemapper(src_grid,map_file,pmid_src,pint_src,constants::DefaultFillValue::value) -{ - // Nothing to do here -} - -VerticalRemapper:: -VerticalRemapper (const grid_ptr_type& src_grid, - const std::string& map_file, - const Field& pmid_src, - const Field& pint_src, - const Real mask_val) - : AbstractRemapper() - , m_comm (src_grid->get_comm()) - , m_mask_val(mask_val) +create_tgt_grid (const grid_ptr_type& src_grid, + const std::string& map_file) { - using namespace ShortFieldTagsNames; - - // Sanity checks - EKAT_REQUIRE_MSG (src_grid->type()==GridType::Point, - "Error! VerticalRemapper only works on PointGrid grids.\n" - " - src grid name: " + src_grid->name() + "\n" - " - src_grid_type: " + e2str(src_grid->type()) + "\n"); - EKAT_REQUIRE_MSG (src_grid->is_unique(), - "Error! VerticalRemapper requires a unique source grid.\n"); - - // This is a vertical remapper. We only go in one direction - m_bwd_allowed = false; - - // Create tgt_grid that is a clone of the src grid but with - // the correct number of levels. Note that when vertically - // remapping the target field will be defined on the same DOFs - // as the source field, but will have a different number of - // vertical levels. + // Create tgt_grid as a clone of src_grid with different nlevs scorpio::register_file(map_file,scorpio::FileMode::Read); auto nlevs_tgt = scorpio::get_dimlen(map_file,"lev"); auto tgt_grid = src_grid->clone("vertical_remap_tgt_grid",true); tgt_grid->reset_num_vertical_lev(nlevs_tgt); - this->set_grids(src_grid,tgt_grid); - - // Set the LEV and ILEV vertical profiles for interpolation from - set_source_pressure_fields(pmid_src,pint_src); // Gather the pressure level data for vertical remapping - set_pressure_levels(map_file); + auto layout = tgt_grid->get_vertical_layout(true); + Field p_tgt(FieldIdentifier("p_levs",layout,ekat::units::Pa,tgt_grid->name())); + p_tgt.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); + p_tgt.allocate_view(); + scorpio::read_var(map_file,"p_levs",p_tgt.get_view().data()); + p_tgt.sync_to_dev(); // Add tgt pressure levels to the tgt grid - tgt_grid->set_geometry_data(m_tgt_pressure); + tgt_grid->set_geometry_data(p_tgt); scorpio::release_file(map_file); + + return tgt_grid; +} + +VerticalRemapper:: +VerticalRemapper (const grid_ptr_type& src_grid, + const std::string& map_file) + : VerticalRemapper(src_grid,create_tgt_grid(src_grid,map_file)) +{ + set_target_pressure (m_tgt_grid->get_geometry_data("p_levs"),Both); +} + +VerticalRemapper:: +VerticalRemapper (const grid_ptr_type& src_grid, + const grid_ptr_type& tgt_grid) +{ + // We only go in one direction for simplicity, since we need to setup some + // infrsatructures, and we don't want to setup 2x as many "just in case". + // If you need to remap bwd, just create another remapper with src/tgt grids swapped. + m_bwd_allowed = false; + + EKAT_REQUIRE_MSG (src_grid->get_2d_scalar_layout().congruent(tgt_grid->get_2d_scalar_layout()), + "Error! Source and target grid can only differ for their number of level.\n"); + + this->set_grids (src_grid,tgt_grid); } FieldLayout VerticalRemapper:: create_src_layout (const FieldLayout& tgt_layout) const { - using namespace ShortFieldTagsNames; - EKAT_REQUIRE_MSG (is_valid_tgt_layout(tgt_layout), "[VerticalRemapper] Error! Input target layout is not valid for this remapper.\n" " - input layout: " + tgt_layout.to_string()); - return create_layout(tgt_layout,m_src_grid); + EKAT_REQUIRE_MSG (not m_tgt_mid_same_as_int, + "[VerticalRemapper::create_src_layout] Error! Cannot deduce source layout.\n" + " The target layout does not distinguish between LEV and ILEV.\n"); + + const auto& mid_layout = m_src_pmid.get_header().get_identifier().get_layout(); + const auto& int_layout = m_src_pint.get_header().get_identifier().get_layout(); + return create_layout(tgt_layout,m_src_grid,mid_layout.congruent(int_layout)); } FieldLayout VerticalRemapper:: create_tgt_layout (const FieldLayout& src_layout) const { - using namespace ShortFieldTagsNames; - EKAT_REQUIRE_MSG (is_valid_src_layout(src_layout), - "[VerticalRemapper] Error! Input source layout is not valid for this remapper.\n" + "[VerticalRemapper::create_tgt_layout] Error! Input source layout is not valid for this remapper.\n" " - input layout: " + src_layout.to_string()); - return create_layout(src_layout,m_tgt_grid); + EKAT_REQUIRE_MSG (not m_src_mid_same_as_int, + "[VerticalRemapper::create_tgt_layout] Error! Cannot deduce target layout.\n" + " The source layout does not distinguish between LEV and ILEV.\n"); + + const auto& mid_layout = m_tgt_pmid.get_header().get_identifier().get_layout(); + const auto& int_layout = m_tgt_pint.get_header().get_identifier().get_layout(); + return create_layout(src_layout,m_tgt_grid,mid_layout.congruent(int_layout)); } FieldLayout VerticalRemapper:: -create_layout (const FieldLayout& fl_in, - const grid_ptr_type& grid_out) const +create_layout (const FieldLayout& from_layout, + const std::shared_ptr& to_grid, + const bool int_same_as_mid) const { - // NOTE: for the vert remapper, it doesn't really make sense to distinguish - // between midpoints and interfaces: we're simply asking for a quantity - // at a given set of pressure levels. So we choose to have fl_out - // to *always* have LEV as vertical tag. - auto fl_out = FieldLayout::invalid(); - switch (fl_in.type()) { + using namespace ShortFieldTagsNames; + + auto to_layout = FieldLayout::invalid(); + bool midpoints; + std::string vdim_name; + switch (from_layout.type()) { case LayoutType::Scalar0D: [[ fallthrough ]]; case LayoutType::Vector0D: [[ fallthrough ]]; case LayoutType::Scalar2D: [[ fallthrough ]]; case LayoutType::Vector2D: [[ fallthrough ]]; case LayoutType::Tensor2D: // These layouts do not have vertical dim tags, so no change - fl_out = fl_in; + to_layout = from_layout; break; case LayoutType::Scalar1D: - fl_out = grid_out->get_vertical_layout(true); + midpoints = int_same_as_mid || from_layout.tags().back()==LEV; + to_layout = to_grid->get_vertical_layout(midpoints); break; case LayoutType::Scalar3D: - fl_out = grid_out->get_3d_scalar_layout(true); + midpoints = int_same_as_mid || from_layout.tags().back()==LEV; + to_layout = to_grid->get_3d_scalar_layout(midpoints); break; case LayoutType::Vector3D: - fl_out = grid_out->get_3d_vector_layout(true,fl_in.get_vector_dim()); + vdim_name = from_layout.name(from_layout.get_vector_component_idx()); + midpoints = int_same_as_mid || from_layout.tags().back()==LEV; + to_layout = to_grid->get_3d_vector_layout(midpoints,from_layout.get_vector_dim(),vdim_name); break; default: // NOTE: this also include Tensor3D. We don't really have any atm proc // that needs to handle a tensor3d quantity, so no need to add it EKAT_ERROR_MSG ( "[VerticalRemapper] Error! Layout not supported by VerticalRemapper.\n" - " - input layout: " + fl_in.to_string() + "\n"); + " - input layout: " + from_layout.to_string() + "\n"); } - return fl_out; + return to_layout; } void VerticalRemapper:: -set_pressure_levels(const std::string& map_file) +set_extrapolation_type (const ExtrapType etype, const TopBot where) { - // Ensure each map file gets a different decomp name - static std::map file2idx; - if (file2idx.find(map_file)==file2idx.end()) { - file2idx[map_file] = file2idx.size(); + if (where & Top) { + m_etype_top = etype; + } + if (where & Bot) { + m_etype_bot = etype; } +} - using namespace ShortFieldTagsNames; - auto layout = m_tgt_grid->get_vertical_layout(true); - FieldIdentifier fid("p_levs",layout,ekat::units::Pa,m_tgt_grid->name()); - m_tgt_pressure = Field(fid); - // Just in case input fields are packed - m_tgt_pressure.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); - m_tgt_pressure.allocate_view(); +void VerticalRemapper:: +set_mask_value (const Real mask_val) +{ + EKAT_REQUIRE_MSG (not ekat::is_invalid(mask_val), + "[VerticalRemapper::set_mask_value] Error! Input mask value must be a valid number.\n"); - auto remap_pres_data = m_tgt_pressure.get_view().data(); - scorpio::read_var(map_file,"p_levs",remap_pres_data); + m_mask_val = mask_val; +} + +void VerticalRemapper:: +set_source_pressure (const Field& p, const ProfileType ptype) +{ + set_pressure (p, "source", ptype); +} - m_tgt_pressure.sync_to_dev(); +void VerticalRemapper:: +set_target_pressure (const Field& p, const ProfileType ptype) +{ + set_pressure (p, "target", ptype); } void VerticalRemapper:: -set_source_pressure_fields(const Field& pmid, const Field& pint) +set_pressure (const Field& p, const std::string& src_or_tgt, const ProfileType ptype) { using namespace ShortFieldTagsNames; + using PackT = ekat::Pack; + + bool src = src_or_tgt=="source"; + + std::string msg_prefix = "[VerticalRemapper::set_" + src_or_tgt + "_pressure] "; - EKAT_REQUIRE_MSG(pmid.is_allocated(), - "Error! Source midpoint pressure field is not yet allocated.\n" - " - field name: " + pmid.name() + "\n"); - - EKAT_REQUIRE_MSG(pint.is_allocated(), - "Error! Source interface pressure field is not yet allocated.\n" - " - field name: " + pint.name() + "\n"); - - const auto& pmid_layout = pmid.get_header().get_identifier().get_layout(); - const auto& pint_layout = pint.get_header().get_identifier().get_layout(); - EKAT_REQUIRE_MSG(pmid_layout.congruent(m_src_grid->get_3d_scalar_layout(true)), - "Error! Source midpoint pressure field has the wrong layout.\n" - " - field name: " + pmid.name() + "\n" - " - field layout: " + pmid_layout.to_string() + "\n" - " - expected layout: " + m_src_grid->get_3d_scalar_layout(true).to_string() + "\n"); - EKAT_REQUIRE_MSG(pint_layout.congruent(m_src_grid->get_3d_scalar_layout(false)), - "Error! Source interface pressure field has the wrong layout.\n" - " - field name: " + pint.name() + "\n" - " - field layout: " + pint_layout.to_string() + "\n" - " - expected layout: " + m_src_grid->get_3d_scalar_layout(false).to_string() + "\n"); - - m_src_pmid = pmid; - m_src_pint = pint; + EKAT_REQUIRE_MSG(p.is_allocated(), + msg_prefix + "Field is not yet allocated.\n" + " - field name: " + p.name() + "\n"); + + EKAT_REQUIRE_MSG(p.get_header().get_alloc_properties().is_compatible(), + msg_prefix + "Field not compatible with default pack size.\n" + " - pack size: " + std::to_string(SCREAM_PACK_SIZE) + "\n"); + + const int nlevs = src ? m_src_grid->get_num_vertical_levels() + : m_tgt_grid->get_num_vertical_levels(); + const auto& p_layout = p.get_header().get_identifier().get_layout(); + const auto vtag = p_layout.tags().back(); + const auto vdim = p_layout.dims().back(); + + FieldTag expected_tag; + int expected_dim; + switch (ptype) { + case Midpoints: + expected_tag = LEV; + expected_dim = nlevs; + if (src) { + m_src_pmid = p; + } else { + m_tgt_pmid = p; + } + break; + case Interfaces: + expected_tag = ILEV; + expected_dim = nlevs+1; + if (src) { + m_src_pint = p; + } else { + m_tgt_pint = p; + } + break; + case Both: + expected_tag = LEV; + expected_dim = nlevs; + if (src) { + m_src_pint = p; + m_src_pmid = p; + m_src_mid_same_as_int = true; + } else { + m_tgt_pint = p; + m_tgt_pmid = p; + m_tgt_mid_same_as_int = true; + } + break; + default: + EKAT_ERROR_MSG ("[VerticalRemapper::set_source_pressure] Error! Unrecognized value for 'ptype'.\n"); + } + EKAT_REQUIRE_MSG (vtag==expected_tag and vdim==expected_dim, + msg_prefix + "Invalid pressure layout.\n" + " - layout: " + p_layout.to_string() + "\n" + " - expected last layout tag: " + e2str(expected_tag) + "\n" + " - expected last layout dim: " + std::to_string(expected_dim) + "\n"); } void VerticalRemapper:: @@ -198,7 +252,7 @@ do_register_field (const identifier_type& src, const identifier_type& tgt) // could have src with ILEV and tgt with LEV) auto src_layout = src.get_layout().clone(); auto tgt_layout = tgt.get_layout().clone(); - EKAT_REQUIRE_MSG(src_layout.strip_dims({ILEV,LEV}).congruent(tgt_layout.strip_dims({LEV})), + EKAT_REQUIRE_MSG(src_layout.strip_dims({ILEV,LEV}).congruent(tgt_layout.strip_dims({LEV,ILEV})), "[VerticalRemapper] Error! Once vertical level tag is stripped, src/tgt layouts are incompatible.\n" " - src field name: " + src.name() + "\n" " - tgt field name: " + tgt.name() + "\n" @@ -224,61 +278,66 @@ do_bind_field (const int ifield, const field_type& src, const field_type& tgt) auto& f_tgt = m_tgt_fields[ifield]; // Nonconst, since we need to set extra data in the header if (src_layout.has_tag(LEV) or src_layout.has_tag(ILEV)) { // Determine if this field can be handled with packs, and whether it's at midpoints + // NOTE: we don't know if mid==int on src or tgt. If it is, we use the other to determine mid-vs-int // Add mask tracking to the target field. The mask tracks location of tgt pressure levs that are outside the // bounds of the src pressure field, and hence cannot be recovered by interpolation auto& ft = m_field2type[src.name()]; - ft.midpoints = src.get_header().get_identifier().get_layout().has_tag(LEV); + ft.midpoints = m_src_mid_same_as_int + ? tgt.get_header().get_identifier().get_layout().has_tag(LEV) + : src.get_header().get_identifier().get_layout().has_tag(LEV); ft.packed = src.get_header().get_alloc_properties().is_compatible() and tgt.get_header().get_alloc_properties().is_compatible(); - // NOTE: for now we assume that masking is determined only by the COL,LEV location in space - // and that fields with multiple components will have the same masking for each component - // at a specific COL,LEV - src_layout.strip_dims({CMP}); + if (m_etype_top==Mask or m_etype_bot==Mask) { + // NOTE: for now we assume that masking is determined only by the COL,LEV location in space + // and that fields with multiple components will have the same masking for each component + // at a specific COL,LEV + src_layout.strip_dims({CMP}); - // I this mask has already been created, retrieve it, otherwise create it - const auto mask_name = m_tgt_grid->name() + "_" + ekat::join(src_layout.names(),"_") + "_mask"; - Field tgt_mask; - if (m_field2type.count(mask_name)==0) { - auto nondim = ekat::units::Units::nondimensional(); - // Create this src/tgt mask fields, and assign them to these src/tgt fields extra data + // I this mask has already been created, retrieve it, otherwise create it + const auto mask_name = m_tgt_grid->name() + "_" + ekat::join(src_layout.names(),"_") + "_mask"; + Field tgt_mask; + if (m_field2type.count(mask_name)==0) { + auto nondim = ekat::units::Units::nondimensional(); + // Create this src/tgt mask fields, and assign them to these src/tgt fields extra data - FieldIdentifier src_mask_fid (mask_name, src_layout, nondim, m_src_grid->name() ); - FieldIdentifier tgt_mask_fid = create_tgt_fid(src_mask_fid); + FieldIdentifier src_mask_fid (mask_name, src_layout, nondim, m_src_grid->name() ); + FieldIdentifier tgt_mask_fid = create_tgt_fid(src_mask_fid); - Field src_mask (src_mask_fid); - src_mask.allocate_view(); + Field src_mask (src_mask_fid); + src_mask.allocate_view(); - tgt_mask = Field (tgt_mask_fid); - tgt_mask.allocate_view(); + tgt_mask = Field (tgt_mask_fid); + tgt_mask.allocate_view(); - // Initialize the src mask values to 1.0 - src_mask.deep_copy(1.0); + // Initialize the src mask values to 1.0 + src_mask.deep_copy(1.0); - m_src_masks.push_back(src_mask); - m_tgt_masks.push_back(tgt_mask); + m_src_masks.push_back(src_mask); + m_tgt_masks.push_back(tgt_mask); - auto& mt = m_field2type[src_mask_fid.name()]; - mt.packed = false; - mt.midpoints = src_layout.has_tag(LEV); - } else { - for (size_t i=0; i void VerticalRemapper:: setup_lin_interp (const ekat::LinInterp& lin_interp, - const Field& p_src) const + const Field& p_src, const Field& p_tgt) const { using LI_t = ekat::LinInterp; using ESU = ekat::ExeSpaceUtils; using PackT = ekat::Pack; - auto p_src_v = p_src.get_view(); - auto p_tgt_v = m_tgt_pressure.get_view(); + using view2d = typename KokkosTypes::view; + using view1d = typename KokkosTypes::view; + + auto src1d = p_src.rank()==1; + auto tgt1d = p_tgt.rank()==1; + + view2d p_src2d_v, p_tgt2d_v; + view1d p_src1d_v, p_tgt1d_v; + if (src1d) { + p_src1d_v = p_src.get_view(); + } else { + p_src2d_v = p_src.get_view(); + } + if (tgt1d) { + p_tgt1d_v = p_tgt.get_view(); + } else { + p_tgt2d_v = p_tgt.get_view(); + } auto lambda = KOKKOS_LAMBDA(typename LI_t::MemberType const& team) { const int icol = team.league_rank(); - lin_interp.setup(team,ekat::subview(p_src_v,icol), - p_tgt_v); + // Extract subviews if src/tgt were not 1d to start with + auto x_src = p_src1d_v; + if (not src1d) + x_src = ekat::subview(p_src2d_v,icol); + auto x_tgt = p_tgt1d_v; + if (not tgt1d) + x_tgt = ekat::subview(p_tgt2d_v,icol); + + lin_interp.setup(team,x_src,x_tgt); }; - const int ncols = m_src_grid->get_num_local_dofs(); const int nlevs_tgt = m_tgt_grid->get_num_vertical_levels(); const int npacks_tgt = ekat::PackInfo::num_packs(nlevs_tgt); @@ -465,8 +550,7 @@ template void VerticalRemapper:: apply_vertical_interpolation(const ekat::LinInterp& lin_interp, const Field& f_src, const Field& f_tgt, - const Field& p_src, - const Real mask_val) const + const Field& p_src, const Field& p_tgt) const { // Note: if Packsize==1, we grab packs of size 1, which are for sure // compatible with the allocation @@ -474,44 +558,51 @@ apply_vertical_interpolation(const ekat::LinInterp& lin_interp, using PackT = ekat::Pack; using ESU = ekat::ExeSpaceUtils; - auto p_src_v = p_src.get_view(); - auto x_tgt = m_tgt_pressure.get_view(); - const auto& f_src_l = f_src.get_header().get_identifier().get_layout(); + using view2d = typename KokkosTypes::view; + using view1d = typename KokkosTypes::view; + + auto src1d = p_src.rank()==1; + auto tgt1d = p_tgt.rank()==1; + + view2d p_src2d_v, p_tgt2d_v; + view1d p_src1d_v, p_tgt1d_v; + if (src1d) { + p_src1d_v = p_src.get_view(); + } else { + p_src2d_v = p_src.get_view(); + } + if (tgt1d) { + p_tgt1d_v = p_tgt.get_view(); + } else { + p_tgt2d_v = p_tgt.get_view(); + } + + const auto& f_tgt_l = f_tgt.get_header().get_identifier().get_layout(); const int ncols = m_src_grid->get_num_local_dofs(); - const int nlevs_tgt = m_tgt_grid->get_num_vertical_levels(); - const int nlevs_src = f_src_l.dims().back(); + const int nlevs_tgt = f_tgt_l.dims().back(); const int npacks_tgt = ekat::PackInfo::num_packs(nlevs_tgt); - const int last_src_pack_idx = ekat::PackInfo::last_pack_idx(nlevs_src); - const int last_src_pack_end = ekat::PackInfo::last_vec_end(nlevs_src); - switch(f_src.rank()) { case 2: { auto f_src_v = f_src.get_view(); auto f_tgt_v = f_tgt.get_view< PackT**>(); auto policy = ESU::get_default_team_policy(ncols,npacks_tgt); - auto lambda = KOKKOS_LAMBDA(typename LI_t::MemberType const& team) { - - // Interpolate + auto lambda = KOKKOS_LAMBDA(typename LI_t::MemberType const& team) + { const int icol = team.league_rank(); - auto x_src = ekat::subview(p_src_v,icol); + + // Extract subviews if src/tgt pressures were not 1d to start with + auto x_src = p_src1d_v; + auto x_tgt = p_tgt1d_v; + if (not src1d) + x_src = ekat::subview(p_src2d_v,icol); + if (not tgt1d) + x_tgt = ekat::subview(p_tgt2d_v,icol); + auto y_src = ekat::subview(f_src_v,icol); auto y_tgt = ekat::subview(f_tgt_v,icol); lin_interp.lin_interp(team,x_src,x_tgt,y_src,y_tgt,icol); - team.team_barrier(); - - // If x_tgt is extrapolated, set to mask_val - auto x_min = x_src[0][0]; - auto x_max = x_src[last_src_pack_idx][last_src_pack_end-1]; - auto set_mask = [&](const int ipack) { - auto in_range = ekat::range(ipack*Packsize) < nlevs_tgt; - auto oob = (x_tgt[ipack]x_max) and in_range; - if (oob.any()) { - y_tgt[ipack].set(oob,mask_val); - } - }; - Kokkos::parallel_for (Kokkos::TeamThreadRange(team,npacks_tgt), set_mask); }; Kokkos::parallel_for("VerticalRemapper::apply_vertical_interpolation",policy,lambda); break; @@ -520,31 +611,25 @@ apply_vertical_interpolation(const ekat::LinInterp& lin_interp, { auto f_src_v = f_src.get_view(); auto f_tgt_v = f_tgt.get_view< PackT***>(); - const auto& layout = f_src.get_header().get_identifier().get_layout(); - const int ncomps = layout.get_vector_dim(); + const int ncomps = f_tgt_l.get_vector_dim(); auto policy = ESU::get_default_team_policy(ncols*ncomps,npacks_tgt); auto lambda = KOKKOS_LAMBDA(typename LI_t::MemberType const& team) { - // Interpolate const int icol = team.league_rank() / ncomps; const int icmp = team.league_rank() % ncomps; - auto x_src = ekat::subview(p_src_v,icol); + + // Extract subviews if src/tgt pressures were not 1d to start with + auto x_src = p_src1d_v; + auto x_tgt = p_tgt1d_v; + if (not src1d) + x_src = ekat::subview(p_src2d_v,icol); + if (not tgt1d) + x_tgt = ekat::subview(p_tgt2d_v,icol); + auto y_src = ekat::subview(f_src_v,icol,icmp); auto y_tgt = ekat::subview(f_tgt_v,icol,icmp); lin_interp.lin_interp(team,x_src,x_tgt,y_src,y_tgt,icol); - team.team_barrier(); - - // If x_tgt is extrapolated, set to mask_val - auto x_min = x_src[0][0]; - auto x_max = x_src[last_src_pack_idx][last_src_pack_end-1]; - auto set_mask = [&](const int ipack) { - auto oob = x_tgt[ipack]x_max; - if (oob.any()) { - y_tgt[ipack].set(oob,mask_val); - } - }; - Kokkos::parallel_for (Kokkos::TeamThreadRange(team,npacks_tgt), set_mask); }; Kokkos::parallel_for("VerticalRemapper::apply_vertical_interpolation",policy,lambda); break; @@ -557,4 +642,151 @@ apply_vertical_interpolation(const ekat::LinInterp& lin_interp, } } +void VerticalRemapper:: +extrapolate (const Field& f_src, + const Field& f_tgt, + const Field& p_src, + const Field& p_tgt, + const Real mask_val) const +{ + using ESU = ekat::ExeSpaceUtils; + + using view2d = typename KokkosTypes::view; + using view1d = typename KokkosTypes::view; + + auto src1d = p_src.rank()==1; + auto tgt1d = p_tgt.rank()==1; + + view2d p_src2d_v, p_tgt2d_v; + view1d p_src1d_v, p_tgt1d_v; + if (src1d) { + p_src1d_v = p_src.get_view(); + } else { + p_src2d_v = p_src.get_view(); + } + if (tgt1d) { + p_tgt1d_v = p_tgt.get_view(); + } else { + p_tgt2d_v = p_tgt.get_view(); + } + + const auto& f_tgt_l = f_tgt.get_header().get_identifier().get_layout(); + const auto& f_src_l = f_src.get_header().get_identifier().get_layout(); + const int ncols = m_src_grid->get_num_local_dofs(); + const int nlevs_tgt = f_tgt_l.dims().back(); + const int nlevs_src = f_src_l.dims().back(); + + auto etop = m_etype_top; + auto ebot = m_etype_bot; + auto mid = nlevs_tgt / 2; + switch(f_src.rank()) { + case 2: + { + auto f_src_v = f_src.get_view(); + auto f_tgt_v = f_tgt.get_view< Real**>(); + auto policy = ESU::get_default_team_policy(ncols,nlevs_tgt); + + using MemberType = typename decltype(policy)::member_type; + auto lambda = KOKKOS_LAMBDA(const MemberType& team) + { + const int icol = team.league_rank(); + + // Extract subviews if src/tgt pressures were not 1d to start with + auto x_src = p_src1d_v; + auto x_tgt = p_tgt1d_v; + if (not src1d) + x_src = ekat::subview(p_src2d_v,icol); + if (not tgt1d) + x_tgt = ekat::subview(p_tgt2d_v,icol); + + auto y_src = ekat::subview(f_src_v,icol); + auto y_tgt = ekat::subview(f_tgt_v,icol); + + auto x_min = x_src[0]; + auto x_max = x_src[nlevs_src-1]; + auto extrapolate = [&](const int ilev) { + if (ilev>=mid) { + // Near surface + if (x_tgt[ilev]>x_max) { + if (ebot==P0) { + y_tgt[ilev] = y_src[nlevs_src-1]; + } else { + y_tgt[ilev] = mask_val; + } + } + } else { + // Near top + if (x_tgt[ilev](); + auto f_tgt_v = f_tgt.get_view< Real***>(); + const int ncomps = f_tgt_l.get_vector_dim(); + auto policy = ESU::get_default_team_policy(ncols*ncomps,nlevs_tgt); + + using MemberType = typename decltype(policy)::member_type; + auto lambda = KOKKOS_LAMBDA(const MemberType& team) + { + const int icol = team.league_rank() / ncomps; + const int icmp = team.league_rank() % ncomps; + + // Extract subviews if src/tgt pressures were not 1d to start with + auto x_src = p_src1d_v; + auto x_tgt = p_tgt1d_v; + if (not src1d) + x_src = ekat::subview(p_src2d_v,icol); + if (not tgt1d) + x_tgt = ekat::subview(p_tgt2d_v,icol); + + auto y_src = ekat::subview(f_src_v,icol,icmp); + auto y_tgt = ekat::subview(f_tgt_v,icol,icmp); + auto x_min = x_src[0]; + auto x_max = x_src[nlevs_src-1]; + auto extrapolate = [&](const int ilev) { + if (ilev>=mid) { + // Near surface + if (x_tgt[ilev]>x_max) { + if (ebot==P0) { + y_tgt[ilev] = y_src[nlevs_src-1]; + } else { + y_tgt[ilev] = mask_val; + } + } + } else { + // Near top + if (x_tgt[ilev] + create_tgt_grid (const grid_ptr_type& src_grid, + const std::string& map_file); + protected: - FieldLayout create_layout (const FieldLayout& fl_in, - const grid_ptr_type& grid_out) const; + void set_pressure (const Field& p, const std::string& src_or_tgt, const ProfileType ptype); + FieldLayout create_layout (const FieldLayout& from_layout, + const std::shared_ptr& to_grid, + const bool int_same_as_mid) const; const identifier_type& do_get_src_field_id (const int ifield) const override { return m_src_fields[ifield].get_header().get_identifier(); @@ -89,25 +134,23 @@ class VerticalRemapper : public AbstractRemapper EKAT_ERROR_MSG ("VerticalRemapper only supports fwd remapping.\n"); } - void set_pressure_levels (const std::string& map_file); - void do_print(); - #ifdef KOKKOS_ENABLE_CUDA public: #endif template void apply_vertical_interpolation (const ekat::LinInterp& lin_interp, const Field& f_src, const Field& f_tgt, - const Field& p_src, - const Real mask_value) const; + const Field& p_src, const Field& p_tgt) const; + void extrapolate (const Field& f_src, const Field& f_tgt, + const Field& p_src, const Field& p_tgt, + const Real mask_val) const; template void setup_lin_interp (const ekat::LinInterp& lin_interp, - const Field& p_src) const; + const Field& p_src, const Field& p_tgt) const; protected: - void set_source_pressure_fields(const Field& pmid, const Field& pint); void create_lin_interp (); using KT = KokkosTypes; @@ -122,14 +165,22 @@ class VerticalRemapper : public AbstractRemapper // Source and target fields std::vector m_src_fields; std::vector m_tgt_fields; - std::vector m_tgt_masks; std::vector m_src_masks; + std::vector m_tgt_masks; // Vertical profile fields, both for source and target - Real m_mask_val; - Field m_tgt_pressure; - Field m_src_pmid; // Src vertical profile for LEV layouts - Field m_src_pint; // Src vertical profile for ILEV layouts + Field m_src_pmid; + Field m_src_pint; + Field m_tgt_pmid; + Field m_tgt_pint; + + bool m_src_mid_same_as_int = false; + bool m_tgt_mid_same_as_int = false; + + // Extrapolation settings at top/bottom. Default to P0 extrapolation + ExtrapType m_etype_top = P0; + ExtrapType m_etype_bot = P0; + Real m_mask_val = std::numeric_limits::quiet_NaN(); // We need to remap mid/int fields separately, and we want to use packs if possible, // so we need to divide input fields into 4 separate categories diff --git a/components/eamxx/src/share/io/CMakeLists.txt b/components/eamxx/src/share/io/CMakeLists.txt index 6c24d3bc51d..4908e4ae0c9 100644 --- a/components/eamxx/src/share/io/CMakeLists.txt +++ b/components/eamxx/src/share/io/CMakeLists.txt @@ -59,7 +59,7 @@ add_library(scream_io scream_io_utils.cpp ) -target_link_libraries(scream_io PUBLIC scream_share scream_scorpio_interface) +target_link_libraries(scream_io PUBLIC scream_share scream_scorpio_interface diagnostics) if (NOT SCREAM_LIB_ONLY) add_subdirectory(tests) diff --git a/components/eamxx/src/share/io/scorpio_input.cpp b/components/eamxx/src/share/io/scorpio_input.cpp index e8a510130e0..f81bb09f8a3 100644 --- a/components/eamxx/src/share/io/scorpio_input.cpp +++ b/components/eamxx/src/share/io/scorpio_input.cpp @@ -46,6 +46,14 @@ AtmosphereInput (const std::string& filename, init(params,fm); } +AtmosphereInput:: +AtmosphereInput (const std::vector& fields_names, + const std::shared_ptr& grid) +{ + set_grid(grid); + m_fields_names = fields_names; +} + AtmosphereInput:: ~AtmosphereInput () { @@ -73,10 +81,10 @@ init (const ekat::ParameterList& params, // Sets the internal field mgr, and possibly sets up the remapper set_field_manager(field_mgr); + m_inited_with_fields = true; + // Init scorpio internal structures init_scorpio_structures (); - - m_inited_with_fields = true; } void AtmosphereInput:: @@ -113,10 +121,10 @@ init (const ekat::ParameterList& params, " layout = " + it.first); } + m_inited_with_views = true; + // Init scorpio internal structures init_scorpio_structures (); - - m_inited_with_views = true; } /* ---------------------------------------------------------- */ @@ -173,6 +181,28 @@ set_field_manager (const std::shared_ptr& field_mgr) } } +void AtmosphereInput:: +set_fields (const std::vector& fields) { + auto fm = std::make_shared(m_io_grid); + m_fields_names.clear(); + for (const auto& f : fields) { + fm->add_field(f); + m_fields_names.push_back(f.name()); + } + set_field_manager(fm); + m_inited_with_fields = true; +} + +void AtmosphereInput:: +reset_filename (const std::string& filename) +{ + if (m_filename!="") { + scorpio::release_file(m_filename); + } + m_params.set("Filename",filename); + m_filename = filename; + init_scorpio_structures(); +} /* ---------------------------------------------------------- */ void AtmosphereInput:: @@ -220,6 +250,7 @@ void AtmosphereInput::read_variables (const int time_index) // Read the data auto v1d = m_host_views_1d.at(name); + scorpio::read_var(m_filename,name,v1d.data(),time_index); // If we have a field manager, make sure the data is correctly @@ -350,6 +381,9 @@ void AtmosphereInput::finalize() /* ---------------------------------------------------------- */ void AtmosphereInput::init_scorpio_structures() { + EKAT_REQUIRE_MSG (m_inited_with_views or m_inited_with_fields, + "Error! Cannot init scorpio structures until fields/views have been set.\n"); + std::string iotype_str = m_params.get("iotype", "default"); auto iotype = scorpio::str2iotype(iotype_str); diff --git a/components/eamxx/src/share/io/scorpio_input.hpp b/components/eamxx/src/share/io/scorpio_input.hpp index 2c0a7e76c3e..7a53f1e8063 100644 --- a/components/eamxx/src/share/io/scorpio_input.hpp +++ b/components/eamxx/src/share/io/scorpio_input.hpp @@ -61,6 +61,10 @@ class AtmosphereInput const std::shared_ptr& grid, const std::vector& fields, const bool skip_grid_checks = false); + // This constructor only sets the minimal info, deferring initialization + // to when set_field_manager/reset_fields and reset_filename are called + AtmosphereInput (const std::vector& fields_names, + const std::shared_ptr& grid); // Due to resource acquisition (in scorpio), avoid copies AtmosphereInput (const AtmosphereInput&) = delete; @@ -98,9 +102,11 @@ class AtmosphereInput // Getters std::string get_filename() { return m_filename; } // Simple getter to query the filename for this stream. - // Expose the ability to set field manager for cases like time_interpolation where we swap fields - // between field managers to avoid deep_copy. + // Expose the ability to set/reset fields/field_manager for cases like data interpolation, + // where we swap pointers but all the scorpio data structures are unchanged. void set_field_manager (const std::shared_ptr& field_mgr); + void set_fields (const std::vector& fields); + void reset_filename (const std::string& filename); // Option to add a logger void set_logger(const std::shared_ptr& atm_logger) { diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index e8f322b85f3..10b2ee23d86 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -6,6 +6,8 @@ #include "share/util/scream_timing.hpp" #include "share/field/field_utils.hpp" +#include "diagnostics/register_diagnostics.hpp" + #include "ekat/util/ekat_units.hpp" #include "ekat/util/ekat_string_utils.hpp" #include "ekat/std_meta/ekat_std_utils.hpp" @@ -227,9 +229,13 @@ AtmosphereOutput (const ekat::Comm& comm, const ekat::ParameterList& params, if (use_vertical_remap_from_file) { // We build a remapper, to remap fields from the fm grid to the io grid auto vert_remap_file = params.get("vertical_remap_file"); - auto f_lev = get_field("p_mid","sim"); - auto f_ilev = get_field("p_int","sim"); - m_vert_remapper = std::make_shared(io_grid,vert_remap_file,f_lev,f_ilev,m_fill_value); + auto p_mid = get_field("p_mid","sim"); + auto p_int = get_field("p_int","sim"); + auto vert_remapper = std::make_shared(io_grid,vert_remap_file); + vert_remapper->set_source_pressure (p_mid,p_int); + vert_remapper->set_mask_value(m_fill_value); + vert_remapper->set_extrapolation_type(VerticalRemapper::Mask); // both Top AND Bot + m_vert_remapper = vert_remapper; io_grid = m_vert_remapper->get_tgt_grid(); set_grid(io_grid); @@ -1104,7 +1110,7 @@ AtmosphereOutput::get_var_dof_offsets(const FieldLayout& layout) // Precompute this *before* the early return, since it involves collectives. // If one rank owns zero cols, and returns prematurely, the others will be left waiting. - AbstractGrid::gid_type min_gid; + AbstractGrid::gid_type min_gid = -1; if (layout.has_tag(COL) or layout.has_tag(EL)) { min_gid = m_io_grid->get_global_min_dof_gid(); } @@ -1308,6 +1314,8 @@ get_field(const std::string& name, const std::string& mode) const } else { EKAT_ERROR_MSG ("ERROR::AtmosphereOutput::get_field Field " + name + " not found in " + mode + " field manager or diagnostics list."); } + static Field f; + return f; } /* ---------------------------------------------------------- */ void AtmosphereOutput::set_diagnostics() @@ -1330,137 +1338,31 @@ void AtmosphereOutput::set_diagnostics() /* ---------------------------------------------------------- */ std::shared_ptr -AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { - auto& diag_factory = AtmosphereDiagnosticFactory::instance(); +AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) +{ + // We need scream scope resolution, since this->create_diagnostic is hiding it + auto diag = scream::create_diagnostic(diag_field_name,get_field_manager("sim")->get_grid()); - // Construct a diagnostic by this name - ekat::ParameterList params; - std::string diag_name; + // Some diags need some extra setup or trigger extra behaviors std::string diag_avg_cnt_name = ""; - - if (diag_field_name.find("_at_")!=std::string::npos) { - // The diagnostic must be one of - // - ${field_name}_at_lev_${N} <- interface fields still use "_lev_" - // - ${field_name}_at_model_bot - // - ${field_name}_at_model_top - // - ${field_name}_at_${M}X - // where M/N are numbers (N integer), X=Pa, hPa, mb, or m - auto tokens = ekat::split(diag_field_name,"_at_"); - EKAT_REQUIRE_MSG (tokens.size()==2, - "Error! Unexpected diagnostic name: " + diag_field_name + "\n"); - - const auto& fname = tokens.front(); - params.set("field_name",fname); - params.set("grid_name",get_field_manager("sim")->get_grid()->name()); - - params.set("vertical_location", tokens[1]); + auto& params = diag->get_params(); + if (diag->name()=="FieldAtPressureLevel") { params.set("mask_value",m_fill_value); - - // Conventions on notation (N=any integer): - // FieldAtLevel : var_at_lev_N, var_at_model_top, var_at_model_bot - // FieldAtPressureLevel: var_at_Nx, with x=mb,Pa,hPa - // FieldAtHeight : var_at_Nm_above_Y (Y=sealevel or surface) - if (tokens[1].find_first_of("0123456789.")==0) { - auto units_start = tokens[1].find_first_not_of("0123456789."); - auto units = tokens[1].substr(units_start); - if (units.find("_above_") != std::string::npos) { - // The field is at a height above a specific reference. - // Currently we only support FieldAtHeight above "sealevel" or "surface" - auto subtokens = ekat::split(units,"_above_"); - params.set("surface_reference",subtokens[1]); - units = subtokens[0]; - // Need to reset the vertical location to strip the "_above_" part of the string. - params.set("vertical_location", tokens[1].substr(0,units_start)+subtokens[0]); - // If the slice is "above_sealevel" then we need to track the avg cnt uniquely. - // Note, "above_surface" is expected to never have masking and can thus use - // the typical 2d layout avg cnt. - if (subtokens[1]=="sealevel") { - diag_avg_cnt_name = "_" + tokens[1]; // Set avg_cnt tracking for this specific slice - // If we have 2D slices we need to be tracking the average count, - // if m_avg_type is not Instant - m_track_avg_cnt = m_track_avg_cnt || m_avg_type!=OutputAvgType::Instant; - } - } - if (units=="m") { - diag_name = "FieldAtHeight"; - EKAT_REQUIRE_MSG(params.isParameter("surface_reference"),"Error! Output field request for " + diag_field_name + " is missing a surface reference." - " Please add either '_above_sealevel' or '_above_surface' to the field name"); - } else if (units=="mb" or units=="Pa" or units=="hPa") { - diag_name = "FieldAtPressureLevel"; - diag_avg_cnt_name = "_" + tokens[1]; // Set avg_cnt tracking for this specific slice - // If we have 2D slices we need to be tracking the average count, - // if m_avg_type is not Instant - m_track_avg_cnt = m_track_avg_cnt || m_avg_type!=OutputAvgType::Instant; - } else { - EKAT_ERROR_MSG ("Error! Invalid units x for 'field_at_Nx' diagnostic.\n"); - } - } else { - diag_name = "FieldAtLevel"; + diag_avg_cnt_name = "_" + + params.get("pressure_value") + + params.get("pressure_units"); + m_track_avg_cnt = m_track_avg_cnt || m_avg_type!=OutputAvgType::Instant; + } else if (diag->name()=="FieldAtHeight") { + if (params.get("surface_reference")=="sealevel") { + diag_avg_cnt_name = "_" + + params.get("height_value") + + params.get("height_units") + "_above_sealevel"; + m_track_avg_cnt = m_track_avg_cnt || m_avg_type!=OutputAvgType::Instant; } - } else if (diag_field_name=="precip_liq_surf_mass_flux" or - diag_field_name=="precip_ice_surf_mass_flux" or - diag_field_name=="precip_total_surf_mass_flux") { - diag_name = "precip_surf_mass_flux"; - // split will return [X, ''], with X being whatever is before '_surf_mass_flux' - auto type = ekat::split(diag_field_name.substr(7),"_surf_mass_flux").front(); - params.set("precip_type",type); - } else if (diag_field_name=="IceWaterPath" or - diag_field_name=="LiqWaterPath" or - diag_field_name=="RainWaterPath" or - diag_field_name=="RimeWaterPath" or - diag_field_name=="VapWaterPath") { - diag_name = "WaterPath"; - // split will return the list [X, ''], with X being whatever is before 'WaterPath' - params.set("Water Kind",ekat::split(diag_field_name,"WaterPath").front()); - } else if (diag_field_name=="IceNumberPath" or - diag_field_name=="LiqNumberPath" or - diag_field_name=="RainNumberPath") { - diag_name = "NumberPath"; - // split will return the list [X, ''], with X being whatever is before 'NumberPath' - params.set("Number Kind",ekat::split(diag_field_name,"NumberPath").front()); - } else if (diag_field_name=="AeroComCldTop" or - diag_field_name=="AeroComCldBot") { - diag_name = "AeroComCld"; - // split will return the list ['', X], with X being whatever is after 'AeroComCld' - params.set("AeroComCld Kind",ekat::split(diag_field_name,"AeroComCld").back()); - } else if (diag_field_name=="MeridionalVapFlux" or - diag_field_name=="ZonalVapFlux") { - diag_name = "VaporFlux"; - // split will return the list [X, ''], with X being whatever is before 'VapFlux' - params.set("Wind Component",ekat::split(diag_field_name,"VapFlux").front()); - } else if (diag_field_name.find("_atm_backtend")!=std::string::npos) { - diag_name = "AtmBackTendDiag"; - // Set the grid_name - params.set("grid_name",get_field_manager("sim")->get_grid()->name()); - // split will return [X, ''], with X being whatever is before '_atm_tend' - params.set("Tendency Name",ekat::split(diag_field_name,"_atm_backtend").front()); - } else if (diag_field_name=="PotentialTemperature" or - diag_field_name=="LiqPotentialTemperature") { - diag_name = "PotentialTemperature"; - if (diag_field_name == "LiqPotentialTemperature") { - params.set("Temperature Kind", "Liq"); - } else { - params.set("Temperature Kind", "Tot"); - } - } else { - diag_name = diag_field_name; - } - - // These fields are special case of VerticalLayer diagnostic. - // The diagnostics requires the name to be given as param value. - if (diag_name == "z_int" or diag_name == "z_mid" or - diag_name == "geopotential_int" or diag_name == "geopotential_mid" or - diag_name == "height_int" or diag_name == "height_mid" or - diag_name == "dz") { - params.set("diag_name", diag_name); } - // Create the diagnostic - auto diag = diag_factory.create(diag_name,m_comm,params); - diag->set_grids(m_grids_manager); - // Ensure there's an entry in the map for this diag, so .at(diag_name) always works - auto& deps = m_diag_depends_on_diags[diag->name()]; + auto& deps = m_diag_depends_on_diags[diag_field_name]; // Initialize the diagnostic const auto sim_field_mgr = get_field_manager("sim"); @@ -1471,12 +1373,13 @@ AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { if (m_diagnostics.count(fname)==0) { m_diagnostics[fname] = create_diagnostic(fname); } - auto dep = m_diagnostics.at(fname); deps.push_back(fname); } diag->set_required_field (get_field(fname,"sim")); } + diag->initialize(util::TimeStamp(),RunType::Initial); + // If specified, set avg_cnt tracking for this diagnostic. if (m_track_avg_cnt) { const auto diag_field = diag->get_diagnostic(); diff --git a/components/eamxx/src/share/io/scorpio_output.hpp b/components/eamxx/src/share/io/scorpio_output.hpp index 696e08c9982..b1dd6b36cf3 100644 --- a/components/eamxx/src/share/io/scorpio_output.hpp +++ b/components/eamxx/src/share/io/scorpio_output.hpp @@ -6,7 +6,8 @@ #include "share/field/field_manager.hpp" #include "share/grid/abstract_grid.hpp" #include "share/grid/grids_manager.hpp" -#include "share/util//scream_time_stamp.hpp" +#include "share/util/scream_time_stamp.hpp" +#include "share/util/scream_utils.hpp" #include "share/atm_process/atmosphere_diagnostic.hpp" #include "ekat/ekat_parameter_list.hpp" diff --git a/components/eamxx/src/share/io/scream_io_control.hpp b/components/eamxx/src/share/io/scream_io_control.hpp index b24eaeea727..07e60b4f24d 100644 --- a/components/eamxx/src/share/io/scream_io_control.hpp +++ b/components/eamxx/src/share/io/scream_io_control.hpp @@ -74,38 +74,39 @@ struct IOControl { void compute_next_write_ts () { EKAT_REQUIRE_MSG (last_write_ts.is_valid(), "Error! Cannot compute next_write_ts, since last_write_ts was never set.\n"); + next_write_ts = last_write_ts; if (frequency_units=="nsteps") { // This avoids having an invalid/wrong date/time in StorageSpecs::snapshot_fits // if storage type is NumSnaps - next_write_ts = last_write_ts + dt*frequency; + next_write_ts += dt*frequency; next_write_ts.set_num_steps(last_write_ts.get_num_steps()+frequency); } else if (frequency_units=="nsecs") { - next_write_ts = last_write_ts; next_write_ts += frequency; } else if (frequency_units=="nmins") { - next_write_ts = last_write_ts; next_write_ts += frequency*60; } else if (frequency_units=="nhours") { - next_write_ts = last_write_ts; next_write_ts += frequency*3600; } else if (frequency_units=="ndays") { - next_write_ts = last_write_ts; next_write_ts += frequency*86400; - } else if (frequency_units=="nmonths" or frequency_units=="nyears") { + } else if (frequency_units=="nmonths") { auto date = last_write_ts.get_date(); - if (frequency_units=="nmonths") { - int temp = date[1] + frequency - 1; - date[1] = temp % 12 + 1; - date[0] += temp / 12; - } else { - date[0] += frequency; - } - - // Fix day, in case we moved to a month/year where current days. E.g., if last_write - // was on Mar 31st, and units='nmonths', next write is on Apr 30th. HOWEVER, this - // means we will *always* write on the 30th of each month after then, since we have - // no memory of the fact that we were writing on the 31st before. - date[2] = std::min(date[2],util::days_in_month(date[0],date[1])); + int temp = date[1] + frequency - 1; + date[1] = temp % 12 + 1; + date[0] += temp / 12; + + // NOTE: we MAY have moved to an invalid date. E.g., if last_write + // was on Mar 31st, and units='nmonths', date now points to Apr 31st. + // We fix this by adjusting the date to the last day of the month. + // HOWEVER, this means we will *always* write on the 30th of each month after then, + // since we have no memory of the fact that we were writing on the 31st before. + auto month_beg = util::TimeStamp({date[0],date[1],1},{0,0,0}); + auto last_day = month_beg.days_in_curr_month(); + date[2] = std::min(date[2],last_day); + + next_write_ts = util::TimeStamp(date,last_write_ts.get_time()); + } else if (frequency_units=="nyears") { + auto date = last_write_ts.get_date(); + date[0] += frequency; next_write_ts = util::TimeStamp(date,last_write_ts.get_time()); } else { EKAT_ERROR_MSG ("Error! Unrecognized/unsupported frequency unit '" + frequency_units + "'\n"); diff --git a/components/eamxx/src/share/io/scream_io_file_specs.hpp b/components/eamxx/src/share/io/scream_io_file_specs.hpp index 4b0054b372a..ae5a00ff55b 100644 --- a/components/eamxx/src/share/io/scream_io_file_specs.hpp +++ b/components/eamxx/src/share/io/scream_io_file_specs.hpp @@ -56,6 +56,7 @@ struct StorageSpecs { default: EKAT_ERROR_MSG ("Error! Unrecognized/unsupported file storage type.\n"); } + return false; } void update_storage (const util::TimeStamp& t) { diff --git a/components/eamxx/src/share/io/scream_io_utils.cpp b/components/eamxx/src/share/io/scream_io_utils.cpp index 9318728d657..cacb153b929 100644 --- a/components/eamxx/src/share/io/scream_io_utils.cpp +++ b/components/eamxx/src/share/io/scream_io_utils.cpp @@ -1,6 +1,7 @@ #include "share/io/scream_io_utils.hpp" #include "share/io/scream_scorpio_interface.hpp" +#include "share/grid/library_grids_manager.hpp" #include "share/util/scream_utils.hpp" #include "share/scream_config.hpp" @@ -117,4 +118,99 @@ util::TimeStamp read_timestamp (const std::string& filename, return ts; } +std::shared_ptr +create_diagnostic (const std::string& diag_field_name, + const std::shared_ptr& grid) +{ + // Note: use grouping (the (..) syntax), so you can later query the content + // of each group in the matches output var! + // Note: use raw string syntax R"()" to avoid having to escape the \ character + // Note: the number for field_at_p/h can match positive integer/floating-point numbers + std::regex field_at_l (R"(([A-Za-z0-9_]+)_at_(lev_(\d+)|model_(top|bot))$)"); + std::regex field_at_p (R"(([A-Za-z0-9_]+)_at_(\d+(\.\d+)?)(hPa|mb|Pa)$)"); + std::regex field_at_h (R"(([A-Za-z0-9_]+)_at_(\d+(\.\d+)?)(m)_above_(sealevel|surface)$)"); + std::regex surf_mass_flux ("precip_(liq|ice|total)_surf_mass_flux$"); + std::regex water_path ("(Ice|Liq|Rain|Rime|Vap)WaterPath$"); + std::regex number_path ("(Ice|Liq|Rain)NumberPath$"); + std::regex aerocom_cld ("AeroComCld(Top|Bot)$"); + std::regex vap_flux ("(Meridional|Zonal)VapFlux$"); + std::regex backtend ("([A-Za-z0-9_]+)_atm_backtend$"); + std::regex pot_temp ("(Liq)?PotentialTemperature$"); + std::regex vert_layer ("(z|geopotential|height)_(mid|int)$"); + std::regex horiz_avg ("([A-Za-z0-9_]+)_horiz_avg$"); + + std::string diag_name; + std::smatch matches; + ekat::ParameterList params(diag_field_name); + + if (std::regex_search(diag_field_name,matches,field_at_l)) { + params.set("field_name",matches[1].str()); + params.set("grid_name",grid->name()); + params.set("vertical_location", matches[2].str()); + diag_name = "FieldAtLevel"; + } else if (std::regex_search(diag_field_name,matches,field_at_p)) { + params.set("field_name",matches[1].str()); + params.set("grid_name",grid->name()); + params.set("pressure_value",matches[2].str()); + params.set("pressure_units", matches[4].str()); + diag_name = "FieldAtPressureLevel"; + } else if (std::regex_search(diag_field_name,matches,field_at_h)) { + params.set("field_name",matches[1].str()); + params.set("grid_name",grid->name()); + params.set("height_value",matches[2].str()); + params.set("height_units",matches[4].str()); + params.set("surface_reference", matches[5].str()); + diag_name = "FieldAtHeight"; + } else if (std::regex_search(diag_field_name,matches,surf_mass_flux)) { + diag_name = "precip_surf_mass_flux"; + params.set("precip_type",matches[1].str()); + } else if (std::regex_search(diag_field_name,matches,water_path)) { + diag_name = "WaterPath"; + params.set("Water Kind",matches[1].str()); + } else if (std::regex_search(diag_field_name,matches,number_path)) { + diag_name = "NumberPath"; + params.set("Number Kind",matches[1].str()); + } else if (std::regex_search(diag_field_name,matches,aerocom_cld)) { + diag_name = "AeroComCld"; + params.set("AeroComCld Kind",matches[1].str()); + } else if (std::regex_search(diag_field_name,matches,vap_flux)) { + diag_name = "VaporFlux"; + params.set("Wind Component",matches[1].str()); + } else if (std::regex_search(diag_field_name,matches,backtend)) { + diag_name = "AtmBackTendDiag"; + // Set the grid_name + params.set("grid_name",grid->name()); + params.set("Tendency Name",matches[1].str()); + } else if (std::regex_search(diag_field_name,matches,pot_temp)) { + diag_name = "PotentialTemperature"; + params.set("Temperature Kind", matches[1].str()!="" ? matches[1].str() : std::string("Tot")); + } else if (std::regex_search(diag_field_name,matches,vert_layer)) { + diag_name = "VerticalLayer"; + params.set("diag_name",matches[1].str()); + params.set("vert_location",matches[2].str()); + } else if (diag_field_name=="dz") { + diag_name = "VerticalLayer"; + params.set("diag_name","dz"); + params.set("vert_location","mid"); + } + else if (std::regex_search(diag_field_name,matches,horiz_avg)) { + diag_name = "HorizAvgDiag"; + // Set the grid_name + params.set("grid_name",grid->name()); + params.set("field_name",matches[1].str()); + } + else + { + // No existing special regex matches, so we assume that the diag field name IS the diag name. + diag_name = diag_field_name; + } + + auto comm = grid->get_comm(); + auto diag = AtmosphereDiagnosticFactory::instance().create(diag_name,comm,params); + auto gm = std::make_shared(grid); + diag->set_grids(gm); + + return diag; +} + } // namespace scream diff --git a/components/eamxx/src/share/io/scream_io_utils.hpp b/components/eamxx/src/share/io/scream_io_utils.hpp index 9eda82e2bbd..0bea91dc3fa 100644 --- a/components/eamxx/src/share/io/scream_io_utils.hpp +++ b/components/eamxx/src/share/io/scream_io_utils.hpp @@ -3,11 +3,14 @@ #include "scream_io_control.hpp" #include "share/util/scream_time_stamp.hpp" +#include "share/atm_process/atmosphere_diagnostic.hpp" +#include "share/grid/abstract_grid.hpp" #include #include #include +#include namespace scream { @@ -72,111 +75,6 @@ std::string find_filename_in_rpointer ( const OutputAvgType avg_type = OutputAvgType::Instant, const IOControl& control = {}); -struct DefaultMetadata { - - std::string get_longname (const std::string& name) { - if (name_2_longname.count(name)>0) { - return name_2_longname.at(name); - } else { - // TODO: Do we want to print a Warning message? I'm not sure if its needed. - return name; - } - } - - std::string get_standardname (const std::string& name) { - if (name_2_standardname.count(name)>0) { - return name_2_standardname.at(name); - } else { - // TODO: Do we want to print a Warning message? I'm not sure if its needed. - return name; - } - } - - // Create map of longnames, can be added to as developers see fit. - std::map name_2_longname = { - {"lev","hybrid level at midpoints (1000*(A+B))"}, - {"ilev","hybrid level at interfaces (1000*(A+B))"}, - {"hyai","hybrid A coefficient at layer interfaces"}, - {"hybi","hybrid B coefficient at layer interfaces"}, - {"hyam","hybrid A coefficient at layer midpoints"}, - {"hybm","hybrid B coefficient at layer midpoints"} - }; - - // Create map of longnames, can be added to as developers see fit. - std::map name_2_standardname = { - {"p_mid" , "air_pressure"}, - {"p_mid_at_cldtop" , "air_pressure_at_cloud_top"}, - {"T_2m" , "air_temperature"}, - {"T_mid" , "air_temperature"}, - {"T_mid_at_cldtop" , "air_temperature_at_cloud_top"}, - {"aero_g_sw" , "asymmetry_factor_of_ambient_aerosol_particles"}, - {"pbl_height" , "atmosphere_boundary_layer_thickness"}, - {"precip_liq_surf_mass" , "atmosphere_mass_content_of_liquid_precipitation"}, - {"cldlow" , "low_type_cloud_area_fraction"}, - {"cldmed" , "medium_type_cloud_area_fraction"}, - {"cldhgh" , "high_type_cloud_area_fraction"}, - {"cldtot" , "cloud_area_fraction"}, - {"cldfrac_tot_at_cldtop" , "cloud_area_fraction"}, - {"cldfrac_tot" , "cloud_area_fraction_in_atmosphere_layer"}, - {"cldfrac_tot_for_analysis" , "cloud_area_fraction_in_atmosphere_layer"}, - {"cldfrac_rad" , "cloud_area_fraction_in_atmosphere_layer"}, - {"qi" , "cloud_ice_mixing_ratio"}, - {"qc" , "cloud_liquid_water_mixing_ratio"}, - {"U" , "eastward_wind"}, - {"eff_radius_qi" , "effective_radius_of_cloud_ice_particles"}, - {"eff_radius_qc" , "effective_radius_of_cloud_liquid_water_particles"}, - {"eff_radius_qc_at_cldtop" , "effective_radius_of_cloud_liquid_water_particles_at_liquid_water_cloud_top"}, - {"eff_radius_qr" , "effective_radius_of_cloud_rain_particles"}, - {"qv" , "humidity_mixing_ratio"}, - {"cldfrac_ice_at_cldtop" , "ice_cloud_area_fraction"}, - {"cldfrac_ice" , "ice_cloud_area_fraction_in_atmosphere_layer"}, - {"omega" , "lagrangian_tendency_of_air_pressure"}, - {"landfrac" , "land_area_fraction"}, - {"latitude" , "latitude"}, - {"cldfrac_liq_at_cldtop" , "liquid_water_cloud_area_fraction"}, - {"cldfrac_liq" , "liquid_water_cloud_area_fraction_in_atmosphere_layer"}, - {"longitude" , "longitude"}, - {"rainfrac" , "mass_fraction_of_liquid_precipitation_in_air"}, - {"V" , "northward_wind"}, - {"nc" , "number_concentration_of_cloud_liquid_water_particles_in_air"}, - {"cdnc_at_cldtop" , "number_concentration_of_cloud_liquid_water_particles_in_air_at_liquid_water_cloud_top"}, - {"ni" , "number_concentration_of_ice_crystals_in_air"}, - {"aero_tau_sw" , "optical_thickness_of_atmosphere_layer_due_to_ambient_aerosol_particles"}, - {"aero_tau_lw" , "optical_thickness_of_atmosphere_layer_due_to_ambient_aerosol_particles"}, - {"aero_ssa_sw" , "single_scattering_albedo_in_air_due_to_ambient_aerosol_particles"}, - {"sunlit" , "sunlit_binary_mask"}, - {"ps" , "surface_air_pressure"}, - {"LW_flux_dn_at_model_bot" , "surface_downwelling_longwave_flux_in_air"}, - {"SW_flux_dn_at_model_bot" , "surface_downwelling_shortwave_flux_in_air"}, - {"SW_clrsky_flux_dn_at_model_bot" , "surface_downwelling_shortwave_flux_in_air_assuming_clear_sky"}, - {"phis" , "surface_geopotential"}, - {"surf_radiative_T" , "surface_temperature"}, - {"surf_sens_flux" , "surface_upward_sensible_heat_flux"}, - {"SW_flux_dn_at_model_top" , "toa_incoming_shortwave_flux"}, - {"LW_flux_up_at_model_top" , "toa_outgoing_longwave_flux"}, - {"LW_clrsky_flux_up_at_model_top" , "toa_outgoing_longwave_flux_assuming_clear_sky"}, - {"surf_evap" , "water_evapotranspiration_flux"}, - {"AtmosphereDensity" , "air_density"}, - {"PotentialTemperature" , "air_potential_temperature"}, - {"SeaLevelPressure" , "air_pressure_at_mean_sea_level"}, - {"IceWaterPath" , "atmosphere_mass_content_of_cloud_ice"}, - {"LiqWaterPath" , "atmosphere_mass_content_of_cloud_liquid_water"}, - {"VapWaterPath" , "atmosphere_mass_content_of_water_vapor"}, - {"AerosolOpticalDepth550nm" , "atmosphere_optical_thickness_due_to_ambient_aerosol_particles"}, - {"Exner" , "dimensionless_exner_function"}, - {"z_mid" , "geopotential_height"}, - {"geopotential_mid" , "geopotential_height"}, - {"RelativeHumidity" , "relative_humidity"}, - {"surface_upward_latent_heat_flux" , "surface_upward_latent_heat_flux"}, - {"LongwaveCloudForcing" , "toa_longwave_cloud_radiative_effect"}, - {"ShortwaveCloudForcing" , "toa_shortwave_cloud_radiative_effect"}, - {"VirtualTemperature" , "virtual_temperature"}, - {"VaporFlux" , "water_evapotranspiration_flux"}, - {"wind_speed" , "wind_speed"} - }; - -}; - // Shortcut to write/read to/from YYYYMMDD/HHMMSS attributes in the NC file void write_timestamp (const std::string& filename, const std::string& ts_name, const util::TimeStamp& ts, const bool write_nsteps = false); @@ -184,5 +82,11 @@ util::TimeStamp read_timestamp (const std::string& filename, const std::string& ts_name, const bool read_nsteps = false); +// Create a diagnostic from a string representation of it. +// E.g., create the diag to compute fieldX_at_500hPa. +std::shared_ptr +create_diagnostic (const std::string& diag_name, + const std::shared_ptr& grid); + } // namespace scream #endif // SCREAM_IO_UTILS_HPP diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 8d2f64994dd..c19740dbcc1 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -141,6 +141,7 @@ int nctype (const std::string& type) { } else { EKAT_ERROR_MSG ("Error! Unrecognized/unsupported data type '" + type + "'.\n"); } + return -1; } template diff --git a/components/eamxx/src/share/io/tests/CMakeLists.txt b/components/eamxx/src/share/io/tests/CMakeLists.txt index 8c819f7896c..89d92075723 100644 --- a/components/eamxx/src/share/io/tests/CMakeLists.txt +++ b/components/eamxx/src/share/io/tests/CMakeLists.txt @@ -17,6 +17,12 @@ CreateUnitTest(io_utils "io_utils.cpp" PROPERTIES RESOURCE_LOCK rpointer_file ) +# Test creation of diagnostic from diag_field_name +CreateUnitTest(create_diag "create_diag.cpp" + LIBS diagnostics scream_io + LABELS io diagnostics +) + ## Test basic output (no packs, no diags, all avg types, all freq units) CreateUnitTest(io_basic "io_basic.cpp" LIBS scream_io LABELS io diff --git a/components/eamxx/src/share/io/tests/create_diag.cpp b/components/eamxx/src/share/io/tests/create_diag.cpp new file mode 100644 index 00000000000..bc509175971 --- /dev/null +++ b/components/eamxx/src/share/io/tests/create_diag.cpp @@ -0,0 +1,166 @@ +#include "catch2/catch.hpp" + +#include "diagnostics/register_diagnostics.hpp" + +#include "share/io/scream_io_utils.hpp" +#include "share/grid/point_grid.hpp" + +namespace scream { + +TEST_CASE("create_diag") +{ + ekat::Comm comm(MPI_COMM_WORLD); + + register_diagnostics(); + + // Create a grid + const int ncols = 3*comm.size(); + const int nlevs = 10; + auto grid = create_point_grid("Physics",ncols,nlevs,comm); + + SECTION ("field_at") { + // FieldAtLevel + auto d1 = create_diagnostic("BlaH_123_at_model_top",grid); + REQUIRE (std::dynamic_pointer_cast(d1)!=nullptr); + auto d2 = create_diagnostic("BlaH_123_at_model_bot",grid); + REQUIRE (std::dynamic_pointer_cast(d2)!=nullptr); + auto d3 = create_diagnostic("BlaH_123_at_lev_10",grid); + REQUIRE (std::dynamic_pointer_cast(d3)!=nullptr); + + REQUIRE_THROWS(create_diagnostic("BlaH_123_at_modeltop",grid)); // misspelled + + // FieldAtPressureLevel + auto d4 = create_diagnostic("BlaH_123_at_10mb",grid); + REQUIRE (std::dynamic_pointer_cast(d4)!=nullptr); + auto d5 = create_diagnostic("BlaH_123_at_10hPa",grid); + REQUIRE (std::dynamic_pointer_cast(d5)!=nullptr); + auto d6 = create_diagnostic("BlaH_123_at_10Pa",grid); + REQUIRE (std::dynamic_pointer_cast(d6)!=nullptr); + + REQUIRE_THROWS(create_diagnostic("BlaH_123_at_400KPa",grid)); // invalid units + + // FieldAtHeight + auto d7 = create_diagnostic("BlaH_123_at_10m_above_sealevel",grid); + REQUIRE (std::dynamic_pointer_cast(d7)!=nullptr); + auto d8 = create_diagnostic("BlaH_123_at_10m_above_surface",grid); + REQUIRE (std::dynamic_pointer_cast(d8)!=nullptr); + + REQUIRE_THROWS(create_diagnostic("BlaH_123_at_10.5m",grid)); // missing _above_X + REQUIRE_THROWS(create_diagnostic("BlaH_123_at_1km_above_sealevel",grid)); // invalid units + REQUIRE_THROWS(create_diagnostic("BlaH_123_at_1m_above_the_surface",grid)); // invalid reference + } + + SECTION ("precip_mass_flux") { + auto d1 = create_diagnostic("precip_liq_surf_mass_flux",grid); + REQUIRE (std::dynamic_pointer_cast(d1)!=nullptr); + REQUIRE (d1->get_params().get("precip_type")=="liq"); + + auto d2 = create_diagnostic("precip_ice_surf_mass_flux",grid); + REQUIRE (std::dynamic_pointer_cast(d2)!=nullptr); + REQUIRE (d2->get_params().get("precip_type")=="ice"); + + auto d3 = create_diagnostic("precip_total_surf_mass_flux",grid); + REQUIRE (std::dynamic_pointer_cast(d3)!=nullptr); + REQUIRE (d3->get_params().get("precip_type")=="total"); + } + + SECTION ("water_and_number_path") { + auto d1 = create_diagnostic("LiqWaterPath",grid); + REQUIRE (std::dynamic_pointer_cast(d1)!=nullptr); + REQUIRE (d1->get_params().get("Water Kind")=="Liq"); + + auto d2 = create_diagnostic("IceWaterPath",grid); + REQUIRE (std::dynamic_pointer_cast(d2)!=nullptr); + REQUIRE (d2->get_params().get("Water Kind")=="Ice"); + + auto d3 = create_diagnostic("RainWaterPath",grid); + REQUIRE (std::dynamic_pointer_cast(d3)!=nullptr); + REQUIRE (d3->get_params().get("Water Kind")=="Rain"); + + auto d4 = create_diagnostic("RimeWaterPath",grid); + REQUIRE (std::dynamic_pointer_cast(d4)!=nullptr); + REQUIRE (d4->get_params().get("Water Kind")=="Rime"); + + auto d5 = create_diagnostic("VapWaterPath",grid); + REQUIRE (std::dynamic_pointer_cast(d5)!=nullptr); + REQUIRE (d5->get_params().get("Water Kind")=="Vap"); + + auto d6 = create_diagnostic("LiqNumberPath",grid); + REQUIRE (std::dynamic_pointer_cast(d6)!=nullptr); + REQUIRE (d6->get_params().get("Number Kind")=="Liq"); + + auto d7 = create_diagnostic("IceNumberPath",grid); + REQUIRE (std::dynamic_pointer_cast(d7)!=nullptr); + REQUIRE (d7->get_params().get("Number Kind")=="Ice"); + + auto d8 = create_diagnostic("RainNumberPath",grid); + REQUIRE (std::dynamic_pointer_cast(d8)!=nullptr); + REQUIRE (d8->get_params().get("Number Kind")=="Rain"); + } + + SECTION ("aerocom_cld") { + auto d1 = create_diagnostic("AeroComCldTop",grid); + REQUIRE (std::dynamic_pointer_cast(d1)!=nullptr); + REQUIRE (d1->get_params().get("AeroComCld Kind")=="Top"); + + auto d2 = create_diagnostic("AeroComCldBot",grid); + REQUIRE (std::dynamic_pointer_cast(d2)!=nullptr); + REQUIRE (d2->get_params().get("AeroComCld Kind")=="Bot"); + } + + SECTION ("vapor_flux") { + auto d1 = create_diagnostic("MeridionalVapFlux",grid); + REQUIRE (std::dynamic_pointer_cast(d1)!=nullptr); + REQUIRE (d1->get_params().get("Wind Component")=="Meridional"); + + auto d2 = create_diagnostic("ZonalVapFlux",grid); + REQUIRE (std::dynamic_pointer_cast(d2)!=nullptr); + REQUIRE (d2->get_params().get("Wind Component")=="Zonal"); + } + + SECTION ("atm_tend") { + auto d1 = create_diagnostic("BlaH_123_atm_backtend",grid); + REQUIRE (std::dynamic_pointer_cast(d1)!=nullptr); + REQUIRE (d1->get_params().get("Tendency Name")=="BlaH_123"); + } + + SECTION ("pot_temp") { + auto d1 = create_diagnostic("LiqPotentialTemperature",grid); + REQUIRE (std::dynamic_pointer_cast(d1)!=nullptr); + REQUIRE (d1->get_params().get("Temperature Kind")=="Liq"); + + auto d2 = create_diagnostic("PotentialTemperature",grid); + REQUIRE (std::dynamic_pointer_cast(d2)!=nullptr); + REQUIRE (d2->get_params().get("Temperature Kind")=="Tot"); + } + + SECTION ("vert_layer") { + auto d1 = create_diagnostic("z_mid",grid); + REQUIRE (std::dynamic_pointer_cast(d1)!=nullptr); + REQUIRE (d1->get_params().get("vert_location")=="mid"); + auto d2 = create_diagnostic("z_int",grid); + REQUIRE (std::dynamic_pointer_cast(d2)!=nullptr); + REQUIRE (d2->get_params().get("vert_location")=="int"); + + auto d3 = create_diagnostic("height_mid",grid); + REQUIRE (std::dynamic_pointer_cast(d3)!=nullptr); + REQUIRE (d3->get_params().get("vert_location")=="mid"); + auto d4 = create_diagnostic("height_int",grid); + REQUIRE (std::dynamic_pointer_cast(d4)!=nullptr); + REQUIRE (d4->get_params().get("vert_location")=="int"); + + auto d5 = create_diagnostic("geopotential_mid",grid); + REQUIRE (std::dynamic_pointer_cast(d5)!=nullptr); + REQUIRE (d5->get_params().get("vert_location")=="mid"); + auto d6 = create_diagnostic("geopotential_int",grid); + REQUIRE (std::dynamic_pointer_cast(d6)!=nullptr); + REQUIRE (d6->get_params().get("vert_location")=="int"); + + auto d7 = create_diagnostic("dz",grid); + REQUIRE (std::dynamic_pointer_cast(d7)!=nullptr); + REQUIRE (d7->get_params().get("vert_location")=="mid"); + } + +} + +} // namespace scream diff --git a/components/eamxx/src/share/io/tests/io_se_grid.cpp b/components/eamxx/src/share/io/tests/io_se_grid.cpp index 01afb5369f1..0e6d28b53d6 100644 --- a/components/eamxx/src/share/io/tests/io_se_grid.cpp +++ b/components/eamxx/src/share/io/tests/io_se_grid.cpp @@ -152,8 +152,9 @@ get_test_fm(const std::shared_ptr& grid, // field_2 is not partitioned, so let's sync it across ranks auto f2 = fm->get_field("field_2"); - auto v2 = f2.get_view(); + auto v2 = f2.get_view(); comm.all_reduce(v2.data(),nlevs,MPI_MAX); + f2.sync_to_dev(); return fm; } diff --git a/components/eamxx/src/share/property_checks/field_nan_check.cpp b/components/eamxx/src/share/property_checks/field_nan_check.cpp index eef52adfbf7..5b709ff916d 100644 --- a/components/eamxx/src/share/property_checks/field_nan_check.cpp +++ b/components/eamxx/src/share/property_checks/field_nan_check.cpp @@ -186,6 +186,7 @@ PropertyCheck::ResultAndMsg FieldNaNCheck::check() const { "Internal error in FieldNaNCheck: unsupported field data type.\n" "You should not have reached this line. Please, contact developers.\n"); } + return ResultAndMsg{}; } } // namespace scream diff --git a/components/eamxx/src/share/property_checks/field_within_interval_check.cpp b/components/eamxx/src/share/property_checks/field_within_interval_check.cpp index bd436bb192a..eee5f96855d 100644 --- a/components/eamxx/src/share/property_checks/field_within_interval_check.cpp +++ b/components/eamxx/src/share/property_checks/field_within_interval_check.cpp @@ -332,6 +332,7 @@ PropertyCheck::ResultAndMsg FieldWithinIntervalCheck::check() const { "Internal error in FieldWithinIntervalCheck: unsupported field data type.\n" "You should not have reached this line. Please, contact developers.\n"); } + return ResultAndMsg{}; } template diff --git a/components/eamxx/src/share/scream_types.hpp b/components/eamxx/src/share/scream_types.hpp index 941ae81ac8a..cbe26aae75f 100644 --- a/components/eamxx/src/share/scream_types.hpp +++ b/components/eamxx/src/share/scream_types.hpp @@ -39,7 +39,8 @@ enum class RunType { // We cannot expect BFB results between f90 and cxx if optimizations are on. // Same goes for cuda-memcheck because it makes the bfb math layer prohibitively -// expensive and so must be turned off. +// expensive and so must be turned off. SCREAM_SHORT_TESTS is a proxy +// for mem checking. #if defined (NDEBUG) || defined (SCREAM_SHORT_TESTS) static constexpr bool SCREAM_BFB_TESTING = false; #else diff --git a/components/eamxx/src/share/tests/CMakeLists.txt b/components/eamxx/src/share/tests/CMakeLists.txt index 3b4b60cb283..c64659575cb 100644 --- a/components/eamxx/src/share/tests/CMakeLists.txt +++ b/components/eamxx/src/share/tests/CMakeLists.txt @@ -51,10 +51,16 @@ if (NOT SCREAM_ONLY_GENERATE_BASELINES) LIBS scream_io MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS}) - # Test vertical remap - CreateUnitTest(time_interpolation "eamxx_time_interpolation_tests.cpp" + # Generate data for data interpolation test + CreateUnitTest(data_interpolation_setup "data_interpolation_setup.cpp" LIBS scream_io - MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS}) + FIXTURES_SETUP data_interpolation_setup) + + # Test data interpolation + CreateUnitTest(data_interpolation "data_interpolation_tests.cpp" + LIBS scream_io + MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} + FIXTURES_REQUIRED data_interpolation_setup) # Test common physics functions CreateUnitTest(common_physics "common_physics_functions_tests.cpp") diff --git a/components/eamxx/src/share/tests/data_interpolation_setup.cpp b/components/eamxx/src/share/tests/data_interpolation_setup.cpp new file mode 100644 index 00000000000..3ce20351312 --- /dev/null +++ b/components/eamxx/src/share/tests/data_interpolation_setup.cpp @@ -0,0 +1,170 @@ +#include + +#include "data_interpolation_tests.hpp" + +#include "share/io/scream_io_utils.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include "share/grid/point_grid.hpp" + +namespace scream { + +TEST_CASE ("data_interpolation_setup") +{ + // NOTE: ensure these match what's used in data_interpolation_tests.cpp + constexpr int ngcols = data_ngcols; + constexpr int nlevs = data_nlevs; + + auto t_ref = get_t_ref(); + + // Init test session + ekat::Comm comm(MPI_COMM_WORLD); + scorpio::init_subsystem(comm); + + // We use raw scorpio calls without decomp, so ensure we're in serial case + EKAT_REQUIRE_MSG (comm.size()==1, + "Error! You should run the data_interpolation_setup test with ONE rank.\n"); + + // Create grid + std::shared_ptr grid = create_point_grid("pg",ngcols,nlevs,comm); + + // Create and setup two files, so we can test both YearlyPeriodic and LinearHistory + std::vector files = { + "data_interpolation_0", + "data_interpolation_1" + }; + + for (auto int_same_as_mid : {true, false}) { + auto suffix = int_same_as_mid ? "_no_ilev.nc" : ".nc"; + for (const std::string& fname : files) { + scorpio::register_file(fname+suffix,scorpio::Write); + + scorpio::define_dim (fname+suffix,"ncol",ngcols); + scorpio::define_dim (fname+suffix,"lev",nlevs); + scorpio::define_dim (fname+suffix,"dim2",ncmps); + scorpio::define_time(fname+suffix,"days since " + t_ref.to_string()); + if (not int_same_as_mid) { + scorpio::define_dim (fname+suffix,"ilev",nlevs+1); + } + + std::string ilev = int_same_as_mid ? "lev" : "ilev"; + + scorpio::define_var(fname+suffix,"s2d", {"ncol"}, "real", true); + scorpio::define_var(fname+suffix,"s2d", {"ncol"}, "real", true); + scorpio::define_var(fname+suffix,"v2d", {"ncol","dim2"}, "real", true); + scorpio::define_var(fname+suffix,"s3d_m",{"ncol","lev"}, "real", true); + scorpio::define_var(fname+suffix,"v3d_m",{"ncol","dim2","lev"},"real", true); + scorpio::define_var(fname+suffix,"s3d_i",{"ncol",ilev}, "real", true); + scorpio::define_var(fname+suffix,"v3d_i",{"ncol","dim2",ilev}, "real", true); + + // We keep p1d and p3d NOT time-dep + scorpio::define_var(fname+suffix,"p1d", {"lev"},"real", false); + scorpio::define_var(fname+suffix,"p3d", {"ncol","lev"},"real", false); + + scorpio::enddef(fname+suffix); + } + + // Fields and some helper fields (for later) + // NOTE: if we save a pressure field, there is not distinction + // between interfaces and midpoints in the file + // NOTE: do not pad, so that we can grab pointers and pass them to scorpio + auto base_fields = create_fields(grid,true, int_same_as_mid,false); + auto fields = create_fields(grid,false,int_same_as_mid,false); + auto ones = create_fields(grid,false,int_same_as_mid,false); + for (const auto& f : ones) { + f.deep_copy(1); + } + // Loop over time, and add 30 to the value for the first 6 months, + // and subtract 30 for the last 6 months. This guarantees that the data + // is indeed periodic. We'll write at the 15th of each month + // Generate three files: + // - one to be used for yearly-periodic interp + // - two to be used for linear-hystory interp + util::TimeStamp time = get_first_slice_time (); + + // We keep p1d and p3d NOT time-dep, so we write outside the loop + auto p1d = base_fields.back(); + auto p3d = base_fields[2].alias("p3d"); + p1d.sync_to_host(); + p3d.sync_to_host(); + for (const std::string& fname : files) { + scorpio::write_var(fname+suffix,p1d.name(),p1d.get_internal_view_data()); + scorpio::write_var(fname+suffix,p3d.name(),p3d.get_internal_view_data()); + } + + int nfields = fields.size() - 1; // Don't handle p1d, since it's done above + for (int mm=0; mm<12; ++mm) { + std::string file_name = "data_interpolation_" + std::to_string(mm/6) + suffix; + + // We start the files with July + int mm_index = mm+6; + scorpio::update_time(file_name,time.days_from(t_ref)); + for (int i=0; i()); + } + time += 86400*time.days_in_curr_month(); + } + + for (const std::string& fname : files) { + write_timestamp(fname+suffix,"reference_time_stamp",t_ref); + scorpio::release_file(fname+suffix); + } + } + + // Now write a map file for horiz remap, that splits each dof interval in two + const int ngdofs_src = data_ngcols; + const int ngdofs_tgt = fine_ngcols; + + // Existing dofs are "copied", added dofs are averaged from neighbors + const int nnz = ngdofs_src + 2*(ngdofs_src-1); + + std::string filename = map_file_name; + scorpio::register_file(filename, scorpio::FileMode::Write); + + scorpio::define_dim(filename, "n_a", ngdofs_src); + scorpio::define_dim(filename, "n_b", ngdofs_tgt); + scorpio::define_dim(filename, "n_s", nnz); + + scorpio::define_var(filename, "col", {"n_s"}, "int"); + scorpio::define_var(filename, "row", {"n_s"}, "int"); + scorpio::define_var(filename, "S", {"n_s"}, "double"); + + scorpio::enddef(filename); + + std::vector col(nnz), row(nnz); + std::vector S(nnz); + for (int i=0,nnz=0; i + +#include "data_interpolation_tests.hpp" + +#include "share/io/scream_scorpio_interface.hpp" +#include "share/util/eamxx_data_interpolation.hpp" +#include "share/grid/point_grid.hpp" +#include "share/field/field_utils.hpp" +#include "share/scream_config.hpp" + +namespace scream { + +// Give ourselves some room for roundoff errors, since our manual +// evaluation may be different (in finite prec) from the one in the class. +constexpr auto tol = std::numeric_limits::epsilon()*10; + +constexpr auto P1D = DataInterpolation::Static1D; +constexpr auto P3D = DataInterpolation::Dynamic3D; + +using strvec_t = std::vector; +using namespace ShortFieldTagsNames; + +util::TimeStamp reset_year (const util::TimeStamp& t_in, int yy) +{ + auto date = t_in.get_date(); + auto time = t_in.get_time(); + date[0] = yy; + return util::TimeStamp(date,time); +} + +std::shared_ptr +create_interp (const std::shared_ptr& grid, + const std::vector& fields) +{ + return std::make_shared(grid,fields); +} + +void root_print (const ekat::Comm& comm, + const std::string& msg) +{ + if (comm.am_i_root()) { + printf("%s",msg.c_str()); + } +} + +// Run the data interpolation to the input grid, and check against expected values +void run_tests (const std::shared_ptr& grid, + const strvec_t& input_files, util::TimeStamp t_beg, + const util::TimeLine timeline, + const DataInterpolation::VRemapType vr_type = DataInterpolation::None) +{ + auto t_end = t_beg + t_beg.days_in_curr_month()*spd; + auto t0 = t_beg + (t_end-t_beg)/2; + + auto ncols = grid->get_num_local_dofs(); + auto nlevs = grid->get_num_vertical_levels(); + + auto vcoarse_grid = grid->clone("vcoarse",true); + vcoarse_grid->reset_num_vertical_lev(data_nlevs); + + // These are the fields we will compute + auto fields = create_fields(grid,false,false); + fields.pop_back(); // We don't interpolate p1d... + + std::string map_file = grid->get_num_global_dofs()==data_ngcols ? "" : map_file_name; + + // These are used to check the answer + auto base = create_fields(grid,true); + auto ones = create_fields(grid,false); + auto diff = create_fields(grid,false); + auto expected = create_fields(grid,false); + for (auto& f : ones) { + f.deep_copy(1); + } + + std::string data_pname = vr_type==P1D ? "p1d" : "p3d"; // if vr_type==None, it's not used anyways + auto model_pmid = base[2].clone("pmid"); + auto model_pint = base[4].clone("pint"); + if (vr_type==P1D) { + // It's complicated to test the static profile, since we'd have to really do + // a manual interpolation. But setting all model pressure equal to the 1st col + // of the pressure makes things doable + auto comm = grid->get_comm(); + for (const Field& p : {model_pmid, model_pint}) { + auto col_0 = p.subfield(0,0); + auto len = col_0.get_header().get_identifier().get_layout().size(); + comm.broadcast(col_0.get_internal_view_data(),len,0); + col_0.sync_to_dev(); + + for (int icol=0; icol expected_vcoarse, base_vcoarse, ones_vcoarse; + if (vr_type==P1D or vr_type==P3D) { + // If we do remap, there is some P0 extrapolation, + // for which we need to know the data at the top/bot + // NOTE: the data has NO ilev coord in this case + expected_vcoarse = create_fields(vcoarse_grid,false,true); + base_vcoarse = create_fields(vcoarse_grid,true,true); + ones_vcoarse = create_fields(vcoarse_grid,false,true); + for (auto& f : ones_vcoarse) { + f.deep_copy(1); + } + } + + int nfields = fields.size(); + auto interp = create_interp(grid,fields); + interp->setup_time_database(input_files,util::TimeLine::YearlyPeriodic); + interp->setup_remappers (map_file,vr_type,data_pname,model_pmid,model_pint); + interp->init_data_interval(t0); + + // We jump ahead by 2 months, but the shift interval logic cannot keep up with + // a dt that long, so we should get an error due to the interpolation param being + // outside the [0,1] interval. + REQUIRE_THROWS (interp->run(t0+60*spd)); + + // Loop for two year at a 20 day increment + int dt = 20*spd; + for (auto time = t0+dt; time.days_from(t0)<365; time+=dt) { + if (t_end1) { + std::cout << "TEST ERROR:\n" + << " t beg: " << t_beg.to_string() << "\n" + << " t end: " << t_end.to_string() << "\n" + << " time : " << time.to_string() << "\n" + << " t-beg: " << time_from_beg.length << "\n" + << " days in mm_beg: " << t_beg.days_in_curr_month() << "\n" + << " alpha: " << alpha << "\n" + << " delta_beg: " << delta_data[mm_beg] << "\n" + << " delta_end: " << delta_data[mm_end] << "\n" + << " delta: " << delta << "\n"; + } + // Compute expected difference from base value + interp->run(time); + for (int i=0; i(); + auto e_vcoarse = expected_vcoarse[i].get_view(); + + for (int icol=0; icol(); + auto e_vcoarse = expected_vcoarse[i].get_view(); + + for (int icol=0; icol(expected[i])); + REQUIRE (frobenius_norm(diff[i])setup_time_database(files,util::TimeLine::Linear)); // Input file not readable + + interp->setup_time_database({"./data_interpolation_0.nc"},util::TimeLine::Linear); + util::TimeStamp t0 ({2000,1,1},{0,0,0}); + REQUIRE_THROWS (interp->init_data_interval(t0)); // linear timeline, but t0init_data_interval(t1)); // linear timeline, but t0>last_slice + + scorpio::finalize_subsystem(); +} + +TEST_CASE ("interpolation") +{ + ekat::Comm comm(MPI_COMM_WORLD); + + // Regardless of how EAMxx is configured, ignore leap years for this test + set_use_leap_year(false); + + scorpio::init_subsystem(comm); + + auto data_grid = create_point_grid("pg",data_ngcols,data_nlevs,comm); + auto hfine_grid = create_point_grid("pg_h",fine_ngcols,data_nlevs,comm); + auto vfine_grid = create_point_grid("pg_v",data_ngcols,fine_nlevs,comm); + auto hvfine_grid = create_point_grid("pg_hv",fine_ngcols,fine_nlevs,comm); + + SECTION ("periodic") { + // We assume we start with t0 sometime between the first and second input slice. + auto t_beg = reset_year(get_last_slice_time(),2019); + auto timeline = util::TimeLine::YearlyPeriodic; + strvec_t files = {"data_interpolation_0.nc","data_interpolation_1.nc"}; + strvec_t files_no_ilev = {"data_interpolation_0_no_ilev.nc","data_interpolation_1_no_ilev.nc"}; + + SECTION ("no-horiz") { + SECTION ("no-vert") { + root_print(comm," timeline=PERIODIC, horiz_remap=NO, vert_remap=NO ..........\n"); + run_tests (data_grid,files,t_beg,timeline); + root_print(comm," timeline=PERIODIC, horiz_remap=NO, vert_remap=NO .......... PASS\n"); + } + SECTION ("p1d-vert") { + root_print(comm," timeline=PERIODIC, horiz_remap=NO, vert_remap=p1d .........\n"); + run_tests (vfine_grid,files_no_ilev,t_beg,timeline,P1D); + root_print(comm," timeline=PERIODIC, horiz_remap=NO, vert_remap=p1d ......... PASS\n"); + } + SECTION ("p3d-vert") { + root_print(comm," timeline=PERIODIC, horiz_remap=NO, vert_remap=p3d .........\n"); + run_tests (vfine_grid,files_no_ilev,t_beg,timeline,P3D); + root_print(comm," timeline=PERIODIC, horiz_remap=NO, vert_remap=p3d ......... PASS\n"); + } + } + + SECTION ("yes-horiz") { + SECTION ("no-vert") { + root_print(comm," timeline=PERIODIC, horiz_remap=YES, vert_remap=NO ..........\n"); + run_tests (hfine_grid,files,t_beg,timeline); + root_print(comm," timeline=PERIODIC, horiz_remap=YES, vert_remap=NO .......... PASS\n"); + } + SECTION ("p1d-vert") { + root_print(comm," timeline=PERIODIC, horiz_remap=YES, vert_remap=p1d .........\n"); + run_tests (hvfine_grid,files_no_ilev,t_beg,timeline,P1D); + root_print(comm," timeline=PERIODIC, horiz_remap=YES, vert_remap=p1d ......... PASS\n"); + } + SECTION ("p3d-vert") { + root_print(comm," timeline=PERIODIC, horiz_remap=YES, vert_remap=p3d .........\n"); + run_tests (hvfine_grid,files_no_ilev,t_beg,timeline,P3D); + root_print(comm," timeline=PERIODIC, horiz_remap=YES, vert_remap=p3d ......... PASS\n"); + } + } + } + + SECTION ("linear") { + // We assume we start with t0 sometime between the first and second input slice. + auto t_beg = get_first_slice_time(); + auto timeline = util::TimeLine::Linear; + strvec_t files = {"data_interpolation_0.nc","data_interpolation_1.nc"}; + strvec_t files_no_ilev = {"data_interpolation_0_no_ilev.nc","data_interpolation_1_no_ilev.nc"}; + + SECTION ("no-horiz") { + SECTION ("no-vert") { + root_print(comm," timeline=LINEAR, horiz_remap=NO, vert_remap=NO ..........\n"); + run_tests (data_grid,files,t_beg,timeline); + root_print(comm," timeline=LINEAR, horiz_remap=NO, vert_remap=NO .......... PASS\n"); + } + SECTION ("p1d-vert") { + root_print(comm," timeline=LINEAR, horiz_remap=NO, vert_remap=p1d .........\n"); + run_tests (vfine_grid,files_no_ilev,t_beg,timeline,P1D); + root_print(comm," timeline=LINEAR, horiz_remap=NO, vert_remap=p1d ......... PASS\n"); + } + SECTION ("p3d-vert") { + root_print(comm," timeline=LINEAR, horiz_remap=NO, vert_remap=p3d .........\n"); + run_tests (vfine_grid,files_no_ilev,t_beg,timeline,P3D); + root_print(comm," timeline=LINEAR, horiz_remap=NO, vert_remap=p3d ......... PASS\n"); + } + } + + SECTION ("yes-horiz") { + SECTION ("no-vert") { + root_print(comm," timeline=LINEAR, horiz_remap=YES, vert_remap=NO ..........\n"); + run_tests (hfine_grid,files,t_beg,timeline); + root_print(comm," timeline=LINEAR, horiz_remap=YES, vert_remap=NO .......... PASS\n"); + } + SECTION ("p1d-vert") { + root_print(comm," timeline=LINEAR, horiz_remap=YES, vert_remap=p1d .........\n"); + run_tests (hvfine_grid,files_no_ilev,t_beg,timeline,P1D); + root_print(comm," timeline=LINEAR, horiz_remap=YES, vert_remap=p1d ......... PASS\n"); + } + SECTION ("p3d-vert") { + root_print(comm," timeline=LINEAR, horiz_remap=YES, vert_remap=p3d .........\n"); + run_tests (hvfine_grid,files_no_ilev,t_beg,timeline,P3D); + root_print(comm," timeline=LINEAR, horiz_remap=YES, vert_remap=p3d ......... PASS\n"); + } + } + } + + scorpio::finalize_subsystem(); +} + +} // anonymous namespace diff --git a/components/eamxx/src/share/tests/data_interpolation_tests.hpp b/components/eamxx/src/share/tests/data_interpolation_tests.hpp new file mode 100644 index 00000000000..fc2c784c8f1 --- /dev/null +++ b/components/eamxx/src/share/tests/data_interpolation_tests.hpp @@ -0,0 +1,159 @@ +#ifndef EAMXX_DATA_INTERPOLATION_TESTS_HPP +#define EAMXX_DATA_INTERPOLATION_TESTS_HPP + +#include "share/grid/abstract_grid.hpp" +#include "share/field/field.hpp" +#include "share/util/scream_universal_constants.hpp" + +#include + +namespace scream +{ + +constexpr int ncmps = 2; +constexpr auto spd = constants::seconds_per_day; +constexpr int data_ngcols = 12; +constexpr int fine_ngcols = 2*data_ngcols-1; // stick one dof between each data dofs +constexpr int data_nlevs = 32; +constexpr int fine_nlevs = 64; +const std::string map_file_name = "map_file_for_data_interp.nc"; + +// At each month in the input data, we are adding a delta to the "base" value of the fields. +constexpr double delta_data[12] = {0, 30, 60, 90, 120, 150, 180, 150, 120, 90, 60, 30}; + +inline util::TimeStamp get_t_ref () { + return util::TimeStamp ({2010,1,1},{0,0,0}); +} + +// Slices are at midnight between 15th and 16th of each month +// First slice is Jul 15th +inline util::TimeStamp get_first_slice_time () { + // 15 days after the reference time + auto t = get_t_ref() + spd*15; // Mid Jan + for (int mm=0; mm<6; ++mm) { + t += spd*t.days_in_curr_month(); + } + return t; +} + +inline util::TimeStamp get_last_slice_time () { + // 11 months after the 1st slice + auto t = get_first_slice_time(); + for (int mm=0; mm<11; ++mm) { + t += spd*t.days_in_curr_month(); + } + return t; +} + +std::vector +create_fields (const std::shared_ptr& grid, + const bool init_values, + const bool int_same_as_mid = false, + const bool pad_for_packing = true) +{ + constexpr auto m = ekat::units::m; + const auto& gn = grid->name(); + + int ncols = grid->get_num_local_dofs(); + int nlevs = grid->get_num_vertical_levels(); + + // Create fields + + auto layout_s2d = grid->get_2d_scalar_layout(); + auto layout_v2d = grid->get_2d_vector_layout(ncmps); + auto layout_s3d_m = grid->get_3d_scalar_layout(true); + auto layout_v3d_m = grid->get_3d_vector_layout(true,ncmps); + auto layout_s3d_i = grid->get_3d_scalar_layout(int_same_as_mid); + auto layout_v3d_i = grid->get_3d_vector_layout(int_same_as_mid,ncmps); + + Field s2d (FieldIdentifier("s2d", layout_s2d, m, gn)); + Field v2d (FieldIdentifier("v2d", layout_v2d, m, gn)); + Field s3d_m(FieldIdentifier("s3d_m", layout_s3d_m, m, gn)); + Field v3d_m(FieldIdentifier("v3d_m", layout_v3d_m, m, gn)); + Field s3d_i(FieldIdentifier("s3d_i", layout_s3d_i, m, gn)); + Field v3d_i(FieldIdentifier("v3d_i", layout_v3d_i, m, gn)); + + if (pad_for_packing) { + s3d_m.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); + v3d_m.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); + s3d_i.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); + v3d_i.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); + } + + s2d.allocate_view(); + v2d.allocate_view(); + s3d_m.allocate_view(); + v3d_m.allocate_view(); + s3d_i.allocate_view(); + v3d_i.allocate_view(); + + if (init_values) { + // We set horiz/vert values based on the position of the dof, assuming that + // - we have a 1d horiz grid + // - leftmost h dof gets a value of 0, rightmost a value of 1.0 + // - bottom interface gets a value of 0, top interface get a value of 1.0 + // - midpoints are the avg of interfaces + // - we do h_value*v_value + icmp + int ngcols = grid->get_num_global_dofs(); + int nh_intervals = ngcols - 1; + double h_value, v_value; + double h_max = 1.0; + double v_max = 1.0; + double dh = h_max / nh_intervals; + double dv = v_max / nlevs; + auto gids = grid->get_dofs_gids().get_view(); + for (int icol=0; icol()(icol,icmp,ilev) = h_value*(v_value + dv/2) + icmp; + if (int_same_as_mid) { + v3d_i.get_view()(icol,icmp,ilev) = h_value*(v_value + dv/2) + icmp; + } else { + v3d_i.get_view()(icol,icmp,ilev) = h_value*(v_value) + icmp; + } + } + s3d_m.get_view()(icol,ilev) = h_value*(v_value + dv/2); + if (int_same_as_mid) { + s3d_i.get_view()(icol,ilev) = h_value*(v_value + dv/2); + } else { + s3d_i.get_view()(icol,ilev) = h_value*(v_value); + } + } + // Last interface (if mid!=int), where v_value=1 + if (not int_same_as_mid) { + s3d_i.get_view()(icol,nlevs) = h_value; + for (int icmp=0; icmp()(icol,icmp,nlevs) = h_value + icmp; + } + } + + // 2D quantities + for (int icmp=0; icmp()(icol,icmp) = h_value + icmp; + } + s2d.get_view()(icol) = h_value; + } + + s2d.sync_to_dev(); + v2d.sync_to_dev(); + s3d_m.sync_to_dev(); + v3d_m.sync_to_dev(); + s3d_i.sync_to_dev(); + v3d_i.sync_to_dev(); + } + + auto p1d = s3d_m.subfield(0,0).clone("p1d"); + auto comm = grid->get_comm(); + comm.broadcast(p1d.get_internal_view_data(),nlevs,0); + p1d.sync_to_dev(); + + return {s2d, v2d, s3d_m, v3d_m, s3d_i, v3d_i, p1d}; +} + +} // namespace scream + +#endif // EAMXX_DATA_INTERPOLATION_TESTS_HPP diff --git a/components/eamxx/src/share/tests/field_utils.cpp b/components/eamxx/src/share/tests/field_utils.cpp index f444ec75d52..4ba0a8294e4 100644 --- a/components/eamxx/src/share/tests/field_utils.cpp +++ b/components/eamxx/src/share/tests/field_utils.cpp @@ -126,6 +126,288 @@ TEST_CASE("utils") { REQUIRE(field_sum(f1,&comm)==gsum); } + SECTION("horiz_contraction") { + using RPDF = std::uniform_real_distribution; + auto engine = setup_random_test(); + RPDF pdf(0, 1); + + int dim0 = 3; + int dim1 = 9; + int dim2 = 2; + + // Set a weight field + FieldIdentifier f00("f", {{COL}, {dim0}}, m / s, "g"); + Field field00(f00); + field00.allocate_view(); + field00.sync_to_host(); + auto v00 = field00.get_strided_view(); + for(int i = 0; i < dim0; ++i) { + v00(i) = (i + 1) / sp(6); + } + field00.sync_to_dev(); + + // Create (random) sample fields + FieldIdentifier fsc("f", {{}, {}}, m / s, "g"); // scalar + FieldIdentifier f10("f", {{COL, CMP}, {dim0, dim1}}, m / s, "g"); + FieldIdentifier f11("f", {{COL, LEV}, {dim0, dim2}}, m / s, "g"); + FieldIdentifier f20("f", {{COL, CMP, LEV}, {dim0, dim1, dim2}}, m / s, "g"); + Field fieldsc(fsc); + Field field10(f10); + Field field11(f11); + Field field20(f20); + fieldsc.allocate_view(); + field10.allocate_view(); + field11.allocate_view(); + field20.allocate_view(); + randomize(fieldsc, engine, pdf); + randomize(field10, engine, pdf); + randomize(field11, engine, pdf); + randomize(field20, engine, pdf); + + FieldIdentifier F_x("fx", {{COL}, {dim0}}, m / s, "g"); + FieldIdentifier F_y("fy", {{LEV}, {dim2}}, m / s, "g"); + FieldIdentifier F_z("fz", {{CMP}, {dim1}}, m / s, "g"); + FieldIdentifier F_w("fyz", {{CMP, LEV}, {dim1, dim2}}, m / s, "g"); + + Field field_x(F_x); + Field field_y(F_y); + Field field_z(F_z); + Field field_w(F_w); + + // Test invalid inputs + REQUIRE_THROWS(horiz_contraction(fieldsc, field_x, + field00)); // x not allocated yet + + field_x.allocate_view(); + field_y.allocate_view(); + field_z.allocate_view(); + field_w.allocate_view(); + + REQUIRE_THROWS(horiz_contraction(fieldsc, field_y, + field_x)); // unmatching layout + REQUIRE_THROWS(horiz_contraction(field_z, field11, + field11)); // wrong weight layout + + Field result; + + // Ensure a scalar case works + result = fieldsc.clone(); + horiz_contraction(result, field00, field00); + result.sync_to_host(); + auto v = result.get_view(); + REQUIRE(v() == (1 / sp(36) + 4 / sp(36) + 9 / sp(36))); + + // Test higher-order cases + result = field_z.clone(); + horiz_contraction(result, field10, field00); + REQUIRE(result.get_header().get_identifier().get_layout().tags() == + std::vector({CMP})); + REQUIRE(result.get_header().get_identifier().get_layout().dim(0) == dim1); + + result = field_y.clone(); + horiz_contraction(result, field11, field00); + REQUIRE(result.get_header().get_identifier().get_layout().tags() == + std::vector({LEV})); + REQUIRE(result.get_header().get_identifier().get_layout().dim(0) == dim2); + + result = field_w.clone(); + horiz_contraction(result, field20, field00); + REQUIRE(result.get_header().get_identifier().get_layout().tags() == + std::vector({CMP, LEV})); + REQUIRE(result.get_header().get_identifier().get_layout().dim(0) == dim1); + REQUIRE(result.get_header().get_identifier().get_layout().dim(1) == dim2); + + // Check a 3D case + field20.sync_to_host(); + auto manual_result = result.clone(); + manual_result.deep_copy(0); + manual_result.sync_to_host(); + auto v2 = field20.get_strided_view(); + auto mr = manual_result.get_strided_view(); + for(int i = 0; i < dim0; ++i) { + for(int j = 0; j < dim1; ++j) { + for(int k = 0; k < dim2; ++k) { + mr(j, k) += v00(i) * v2(i, j, k); + } + } + } + field20.sync_to_dev(); + manual_result.sync_to_dev(); + REQUIRE(views_are_equal(result, manual_result)); + } + + SECTION("vert_contraction") { + std::vector lev_tags = {LEV, ILEV}; + // iterate over lev_tags + for(auto lev_tag : lev_tags) { + using RPDF = std::uniform_real_distribution; + auto engine = setup_random_test(); + RPDF pdf(0, 1); + + int dim0 = 4; + int dim1 = 9; + // Note that parallel reduction is happening over dim2 (LEV/ILEV) + // If it is more than 3, the order of ops will lead to non-bfb results + int dim2 = lev_tag == LEV ? 3 : 3; + + // Set a weight field + FieldIdentifier f00("f", {{lev_tag}, {dim2}}, m / s, "g"); + Field field00(f00); + field00.allocate_view(); + field00.sync_to_host(); + auto v00 = field00.get_strided_view(); + for(int i = 0; i < dim2; ++i) { + // denominator is the sum of the first dim2 integers (analytically known) + v00(i) = sp(i + 1) / sp(dim2*(dim2+1)/2); + } + field00.sync_to_dev(); + + // Create (random) sample fields + FieldIdentifier fsc("f", {{}, {}}, m / s, "g"); // scalar + FieldIdentifier f10("f", {{COL, lev_tag}, {dim0, dim2}}, m / s, "g"); + FieldIdentifier f11("f", {{CMP, lev_tag}, {dim1, dim2}}, m / s, "g"); + FieldIdentifier f20("f", {{COL, CMP, lev_tag}, {dim0, dim1, dim2}}, m / s, + "g"); + Field fieldsc(fsc); + Field field10(f10); + Field field11(f11); + Field field20(f20); + fieldsc.allocate_view(); + field10.allocate_view(); + field11.allocate_view(); + field20.allocate_view(); + randomize(fieldsc, engine, pdf); + randomize(field10, engine, pdf); + randomize(field11, engine, pdf); + randomize(field20, engine, pdf); + + FieldIdentifier F_x("fx", {{COL}, {dim0}}, m / s, "g"); + FieldIdentifier F_y("fy", {{CMP}, {dim1}}, m / s, "g"); + FieldIdentifier F_z("fz", {{COL, CMP}, {dim0, dim1}}, m / s, "g"); + + Field field_x(F_x); + Field field_y(F_y); + Field field_z(F_z); + + // Test invalid inputs + REQUIRE_THROWS(vert_contraction(fieldsc, field_x, + field00)); // x not allocated yet + + field_x.allocate_view(); + field_y.allocate_view(); + field_z.allocate_view(); + + REQUIRE_THROWS(vert_contraction(fieldsc, field_y, + field_x)); // unmatching layout + REQUIRE_THROWS(vert_contraction(field_z, field11, + field11)); // wrong weight layout + + Field result; + + // Add test for invalid rank-2 weight field layout + FieldIdentifier bad_w("bad_w", {{CMP, lev_tag}, {dim1, dim2}}, m / s, "g"); + Field bad_weight(bad_w); + bad_weight.allocate_view(); + REQUIRE_THROWS(vert_contraction(result, field20, bad_weight)); + + // Add test for mismatched weight field dimensions + FieldIdentifier wrong_size_w("wrong_w", {{COL, lev_tag}, {dim0 + 1, dim2}}, + m / s, "g"); + Field wrong_weight(wrong_size_w); + wrong_weight.allocate_view(); + REQUIRE_THROWS(vert_contraction(result, field20, wrong_weight)); + + // Ensure a scalar case works + result = fieldsc.clone(); + vert_contraction(result, field00, field00); + result.sync_to_host(); + auto v = result.get_view(); + // numerator is the sum of the suqares of the first dim2 integers (analytically known) + // denominator is the sum of the first dim2 integers squared (analytically known) + REQUIRE(v() == sp(dim2*(dim2+1)*(2*dim2+1)/6) / sp((dim2*(dim2+1)/2)*(dim2*(dim2+1)/2))); + + // Test higher-order cases + result = field_x.clone(); + vert_contraction(result, field10, field00); + REQUIRE(result.get_header().get_identifier().get_layout().tags() == + std::vector({COL})); + REQUIRE(result.get_header().get_identifier().get_layout().dim(0) == dim0); + + // Check a 2D case with 1D weight + field10.sync_to_host(); + auto manual_result = result.clone(); + manual_result.deep_copy(0); + manual_result.sync_to_host(); + auto v1 = field10.get_strided_view(); + auto mr = manual_result.get_strided_view(); + for(int i = 0; i < dim0; ++i) { + for(int j = 0; j < dim2; ++j) { + mr(i) += v00(j) * v1(i, j); + } + } + field11.sync_to_dev(); + manual_result.sync_to_dev(); + REQUIRE(views_are_equal(result, manual_result)); + + result = field_y.clone(); + vert_contraction(result, field11, field00); + REQUIRE(result.get_header().get_identifier().get_layout().tags() == + std::vector({CMP})); + REQUIRE(result.get_header().get_identifier().get_layout().dim(0) == dim1); + + result = field_z.clone(); + vert_contraction(result, field20, field00); + REQUIRE(result.get_header().get_identifier().get_layout().tags() == + std::vector({COL, CMP})); + REQUIRE(result.get_header().get_identifier().get_layout().dim(0) == dim0); + REQUIRE(result.get_header().get_identifier().get_layout().dim(1) == dim1); + + // Check a 3D case with 1D weight + field20.sync_to_host(); + result.sync_to_host(); + manual_result = result.clone(); + manual_result.deep_copy(0); + manual_result.sync_to_host(); + auto v2 = field20.get_strided_view(); + auto mr2 = manual_result.get_strided_view(); + for(int i = 0; i < dim0; ++i) { + for(int j = 0; j < dim1; ++j) { + for(int k = 0; k < dim2; ++k) { + mr2(i, j) += v00(k) * v2(i, j, k); + } + } + } + field20.sync_to_dev(); + manual_result.sync_to_dev(); + REQUIRE(views_are_equal(result, manual_result)); + + // Check a 3D case with 2D weight + result = field_z.clone(); + vert_contraction(result, field20, field10); + REQUIRE(result.get_header().get_identifier().get_layout().tags() == + std::vector({COL, CMP})); + REQUIRE(result.get_header().get_identifier().get_layout().dim(0) == dim0); + REQUIRE(result.get_header().get_identifier().get_layout().dim(1) == dim1); + + field20.sync_to_host(); + result.sync_to_host(); + manual_result = result.clone(); + manual_result.deep_copy(0); + manual_result.sync_to_host(); + auto mr3 = manual_result.get_strided_view(); + for(int i = 0; i < dim0; ++i) { + for(int j = 0; j < dim1; ++j) { + for(int k = 0; k < dim2; ++k) { + mr3(i, j) += v1(i, k) * v2(i, j, k); + } + } + } + field20.sync_to_dev(); + manual_result.sync_to_dev(); + REQUIRE(views_are_equal(result, manual_result)); + } + } + SECTION ("frobenius") { auto v1 = f1.get_strided_view(); diff --git a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp index 34351c65db8..b4f1627a16e 100644 --- a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp @@ -4,37 +4,46 @@ #include "share/grid/remap/coarsening_remapper.hpp" #include "share/grid/point_grid.hpp" #include "share/io/scream_scorpio_interface.hpp" +#include "share/util/scream_timing.hpp" +#include "share/field/field_utils.hpp" namespace scream { -template -typename ViewT::HostMirror -cmvc (const ViewT& v) { - auto vh = Kokkos::create_mirror_view(v); - Kokkos::deep_copy(vh,v); - return vh; -} +constexpr int vec_dim = 3; +constexpr auto P0 = VerticalRemapper::P0; +constexpr auto Mask = VerticalRemapper::Mask; +constexpr auto Top = VerticalRemapper::Top; +constexpr auto Bot = VerticalRemapper::Bot; +constexpr auto TopBot = VerticalRemapper::TopAndBot; +constexpr Real mask_val = -99999.0; -void print (const std::string& msg, const ekat::Comm& comm) { +template +void print (const std::string& fmt, const ekat::Comm& comm, Args&&... args) { + if (comm.am_i_root()) { + printf(fmt.c_str(),std::forward(args)...); + } +} +// Overload for when there are no additional arguments +void print(const std::string& fmt, const ekat::Comm& comm) { if (comm.am_i_root()) { - printf("%s",msg.c_str()); + printf(fmt.c_str()); } } // Helper function to create a grid given the number of dof's and a comm group. std::shared_ptr -build_src_grid(const ekat::Comm& comm, const int nldofs_src, const int nlevs_src) +build_grid(const ekat::Comm& comm, const int nldofs, const int nlevs) { using gid_type = AbstractGrid::gid_type; - auto src_grid = std::make_shared("src",nldofs_src,nlevs_src,comm); + auto grid = std::make_shared("src",nldofs,nlevs,comm); - auto src_dofs = src_grid->get_dofs_gids(); - auto src_dofs_h = src_dofs.get_view(); - std::iota(src_dofs_h.data(),src_dofs_h.data()+nldofs_src,nldofs_src*comm.rank()); - src_dofs.sync_to_dev(); + auto dofs = grid->get_dofs_gids(); + auto dofs_h = dofs.get_view(); + std::iota(dofs_h.data(),dofs_h.data()+nldofs,nldofs*comm.rank()); + dofs.sync_to_dev(); - return src_grid; + return grid; } // Helper function to create fields @@ -64,7 +73,133 @@ Real data_func(const int col, const int vec, const Real pres) { // - pres, the current pressure // Should ensure that the interpolated values match exactly, since vertical interp is also a linear interpolator. // Note, we don't use the level, because here the vertical interpolation is over pressure, so it represents the level. - return col*pres + vec*100.0; + return (col+1)*pres + vec*100.0; +} + +void compute_field (const Field& f, const Field& p) +{ + Field::view_host_t p1d; + Field::view_host_t p2d; + bool rank1 = p.rank()==1; + const auto& l = f.get_header().get_identifier().get_layout(); + const int ncols = l.dims().front(); + const int nlevs = l.dims().back(); + if (rank1) { + p1d = p.get_view(); + } else { + p2d = p.get_view(); + } + + // Grab correct pressure (1d or 2d) + auto pval = [&](int i, int k) { + if (rank1) return p1d(k); + else return p2d(i,k); + }; + + switch (l.type()) { + case LayoutType::Scalar2D: + { + const auto v = f.get_view(); + for (int i=0; i(); + for (int i=0; i(); + for (int i=0; i(); + for (int i=0; i p1d_src,p1d_tgt; + Field::view_host_t p2d_src,p2d_tgt; + if (p_src.rank()==1) { + p1d_src = p_src.get_view(); + } else { + p2d_src = p_src.get_view(); + } + if (p_tgt.rank()==1) { + p1d_tgt = p_tgt.get_view(); + } else { + p2d_tgt = p_tgt.get_view(); + } + + auto pval = [&](auto p1d, auto p2d, int i, int k, int rank) { + if (rank==1) return p1d(k); + else return p2d(i,k); + }; + + const auto& l = f.get_header().get_identifier().get_layout(); + const int ncols = l.dims().front(); + const int nlevs = l.dims().back(); + const int nlevs_src = p_src.get_header().get_identifier().get_layout().dims().back(); + // print_field_hyperslab(p_src); + switch (l.type()) { + case LayoutType::Scalar2D: break; + case LayoutType::Vector2D: break; + case LayoutType::Scalar3D: + { + const auto v = f.get_view(); + for (int i=0; ipmax) { + v(i,j) = etype_bot==Mask ? mask_val : data_func(i,0,pmax); + } else if (p(); + for (int i=0; ipmax) { + v(i,j,k) = etype_bot==Mask ? mask_val : data_func(i,j,pmax); + } else if (p dofs_p(nlevs_tgt); std::iota(dofs_p.begin(),dofs_p.end(),0); std::vector p_tgt; - for (int ii=0; ii creating map file ... done!\n",comm); // -------------------------------------- // - // Build src grid and remapper // + // Build src grid // // -------------------------------------- // - print (" -> creating grid and remapper ...\n",comm); + print (" -> creating src grid ...\n",comm); + auto src_grid = build_grid(comm, nldofs, nlevs_src); + print (" -> creating src grid ... done!\n",comm); - const Real mask_val = -99999.0; + // -------------------------------------- // + // Retrieve tgt grid // + // -------------------------------------- // - auto src_grid = build_src_grid(comm, nldofs_src, nlevs_src); - - // We need the source pressure level fields for both p_mid and p_int - auto pmid_src = create_field("p_mid", src_grid, false, false, true, SCREAM_PACK_SIZE); - auto pint_src = create_field("p_int", src_grid, false, false, false, SCREAM_PACK_SIZE); - // Set the source pressures - { - // By adding 1 to the pbot_tgt and subtrating 1 from ptop_tgt we ensure some masking, which - // we also want to check. - const Real ptop_src = ptop_tgt-1; - const Real pbot_src = pbot_tgt+1; - const Real dp_src = (pbot_src-ptop_src)/(nlevs_src-1); - auto pmid_v = pmid_src.get_view(); - auto pint_v = pint_src.get_view(); - for (int ii=0; ii(src_grid,filename,pmid_src,pint_src,mask_val); - print (" -> creating grid and remapper ... done!\n",comm); + print (" -> retreiving tgt grid ...\n",comm); + auto tgt_grid = VerticalRemapper::create_tgt_grid (src_grid,filename); + print (" -> retreiving tgt grid ... done!\n",comm); // -------------------------------------- // - // Create src/tgt grid fields // + // Check tgt grid // // -------------------------------------- // - print (" -> creating fields ...\n",comm); - constexpr int vec_dim = 3; + print (" -> checking tgt grid ...\n",comm); + REQUIRE (tgt_grid->get_num_local_dofs()==src_grid->get_num_local_dofs()); + REQUIRE (tgt_grid->get_num_vertical_levels()==nlevs_tgt); + REQUIRE (tgt_grid->has_geometry_data("p_levs")); + auto p_levs = tgt_grid->get_geometry_data("p_levs"); + auto p_levs_v = p_levs.get_view(); + for (int k=0; kget_tgt_grid(); - // Check that the target grid made by the remapper has the same number of columns as the source grid. - // Also check that the number of levels matches the expectation. - REQUIRE(tgt_grid->get_num_vertical_levels()==nlevs_tgt); - REQUIRE(tgt_grid->get_num_global_dofs()==src_grid->get_num_global_dofs()); - - auto src_s2d = create_field("s2d", src_grid,true,false); - auto src_v2d = create_field("v2d", src_grid,true,true); - // For now we can only support PS = SCREAM_PACK_SIZE because the source and target pressure levels will assume that packsize. - // If we use a smaller packsize we throw an error in the property check step of the vertical interpolation scheme. - // TODO: Fix that. - auto src_s3d_m = create_field("s3d_m",src_grid,false,false,true, 1); - auto src_s3d_i = create_field("s3d_i",src_grid,false,false,false,SCREAM_PACK_SIZE); - auto src_v3d_m = create_field("v3d_m",src_grid,false,true ,true, 1); - auto src_v3d_i = create_field("v3d_i",src_grid,false,true ,false,SCREAM_PACK_SIZE); - - auto tgt_s2d = create_field("s2d", tgt_grid,true,false); - auto tgt_v2d = create_field("v2d", tgt_grid,true,true); - auto tgt_s3d_m = create_field("s3d_m",tgt_grid,false,false,true, 1); - auto tgt_s3d_i = create_field("s3d_i",tgt_grid,false,false,true, SCREAM_PACK_SIZE); - auto tgt_v3d_m = create_field("v3d_m",tgt_grid,false,true ,true, 1); - auto tgt_v3d_i = create_field("v3d_i",tgt_grid,false,true ,true, SCREAM_PACK_SIZE); - - std::vector src_f = {src_s2d,src_v2d,src_s3d_m,src_s3d_i,src_v3d_m,src_v3d_i}; - std::vector tgt_f = {tgt_s2d,tgt_v2d,tgt_s3d_m,tgt_s3d_i,tgt_v3d_m,tgt_v3d_i}; - - print (" -> creating fields ... done!\n",comm); + print (" -> checking tgt grid ... done!\n",comm); - // -------------------------------------- // - // Register fields in the remapper // - // -------------------------------------- // + // Clean up scorpio stuff + scorpio::finalize_subsystem(); - print (" -> registering fields ...\n",comm); - remap->registration_begins(); - remap->register_field(src_s2d, tgt_s2d); - remap->register_field(src_v2d, tgt_v2d); - remap->register_field(src_s3d_m,tgt_s3d_m); - remap->register_field(src_s3d_i,tgt_s3d_i); - remap->register_field(src_v3d_m,tgt_v3d_m); - remap->register_field(src_v3d_i,tgt_v3d_i); - remap->registration_ends(); - print (" -> registering fields ... done!\n",comm); + print ("Testing retrieval of tgt grid from map file ...\n",comm); +} +TEST_CASE ("vertical_remapper") { // -------------------------------------- // - // Check remapper internals // + // Init MPI and PIO // // -------------------------------------- // - print (" -> Checking remapper internal state ...\n",comm); + ekat::Comm comm(MPI_COMM_WORLD); + print ("Testing vertical remapper ...\n",comm); - // Check the pressure levels read from map file + scorpio::init_subsystem(comm); // -------------------------------------- // - // Generate data for src fields // + // Set grid/map sizes // // -------------------------------------- // - print (" -> generate src fields data ...\n",comm); - using namespace ShortFieldTagsNames; - // Generate data in a deterministic way, so that when we check results, - // we know a priori what the input data that generated the tgt field's - // values was, even if that data was off rank. - auto pmid_v = pmid_src.get_view(); - auto pint_v = pint_src.get_view(); - auto src_gids = remap->get_src_grid()->get_dofs_gids().get_view(); - for (const auto& f : src_f) { - const auto& l = f.get_header().get_identifier().get_layout(); - switch (l.type()) { - case LayoutType::Scalar2D: - { - const auto v_src = f.get_view(); - for (int i=0; i creating src grid ...\n",comm); + auto src_grid = build_grid(comm, nldofs, nlevs_src); + print (" nlevs src: %d\n",comm,nlevs_src); + print (" -> creating src grid ...done!\n",comm); + + // Tgt grid must have same 2d layout as src grid + REQUIRE_THROWS (std::make_shared(src_grid,build_grid(comm,nldofs+1,nlevs_src))); + + // Helper lambda, to create p_int profile. If it is a 3d field, make same profile on each col + auto create_pint = [&](const auto& grid, const bool one_d, const Real ptop, const Real pbot) { + auto layout = one_d ? grid->get_vertical_layout(false) + : grid->get_3d_scalar_layout(false); + FieldIdentifier fid("p_int",layout,ekat::units::Pa,grid->name()); + Field pint (fid); + pint.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); + pint.allocate_view(); + + int nlevs = grid->get_num_vertical_levels(); + const Real dp = (pbot-ptop)/nlevs; + + if (one_d) { + auto pv = pint.get_view(); + pv(nlevs) = pbot; + for (int k=nlevs; k>0; --k) { + pv(k-1) = pv(k) - dp; + } + } else { + auto pv = pint.get_view(); + for (int i=0; i0; --k) { + pv(i,k-1) = pv(i,k) - dp; } - } break; - case LayoutType::Vector2D: - { - const auto v_src = f.get_view(); - for (int i=0; i(); - for (int i=0; i(); - for (int i=0; i generate src fields data ... done!\n",comm); - - // No bwd remap - REQUIRE_THROWS(remap->remap(false)); - - for (int irun=0; irun<5; ++irun) { - print (" -> run remap ...\n",comm); - remap->remap(true); - print (" -> run remap ... done!\n",comm); - - // -------------------------------------- // - // Check remapped fields // - // -------------------------------------- // - - print (" -> check tgt fields ...\n",comm); - const auto tgt_gids = tgt_grid->get_dofs_gids().get_view(); - const int ntgt_gids = tgt_gids.size(); - for (size_t ifield=0; ifield Checking field with source layout " + ls +" " + dots + "\n",comm); - - f.sync_to_host(); - - switch (lsrc.type()) { - case LayoutType::Scalar2D: - { - // This is a flat array w/ no LEV tag so the interpolated value for source and target should match. - const auto v_src = fsrc.get_view(); - const auto v_tgt = f.get_view(); - for (int i=0; i(); - const auto v_src = fsrc.get_view(); - for (int i=0; i(); - for (int i=0; ip_v(i,nlevs_p-1) || p_tgt[j](); - for (int i=0; ip_v(i,nlevs_p-1) || p_tgt[k](); + auto pmid_v = pmid.get_view< Real*,Host>(); + for (int k=0; k(); + auto pmid_v = pmid.get_view< Real**,Host>(); + for (int i=0; i Checking field with source layout " + ls + " " + dots + " OK!\n",comm); + // Test tgt grid with 2x and 0.5x as many levels as src grid + for (int nlevs_tgt : {nlevs_src/2, 2*nlevs_src}) { + for (bool src_1d : {true, false}) { + for (bool tgt_1d : {true, false}) { + for (auto etype_top : {P0, Mask}) { + for (auto etype_bot : {P0, Mask}) { + print ("************************************************\n",comm); + print (" nlevs tgt: %d\n",comm,nlevs_tgt); + print (" src pressure is 1d: %s\n",comm,src_1d ? "true" : "false"); + print (" tgt pressure is 1d: %s\n",comm,tgt_1d ? "true" : "false"); + print (" extrap type at top: %s\n",comm,etype_top==P0 ? "p0" : "masked"); + print (" extrap type at bot: %s\n",comm,etype_bot==P0 ? "p0" : "masked"); + print ("************************************************\n",comm); + + print (" -> creating tgt grid ...\n",comm); + auto tgt_grid = src_grid->clone("tgt",true); + tgt_grid->reset_num_vertical_lev(nlevs_tgt); + print (" -> creating tgt grid ...done!\n",comm); + + print (" -> creating src/tgt pressure fields ...\n",comm); + auto pint_src = create_pint(src_grid, src_1d, ptop_src, pbot_src); + auto pmid_src = create_pmid(pint_src); + + // Make ptop_tgtpsurf_src, so we do have extrapolation + const Real ptop_tgt = 10; + const Real pbot_tgt = 1020; + auto pint_tgt = create_pint(tgt_grid, tgt_1d, ptop_tgt, pbot_tgt); + auto pmid_tgt = create_pmid(pint_tgt); + print (" -> creating src/tgt pressure fields ... done!\n",comm); + + print (" -> creating fields ... done!\n",comm); + auto src_s2d = create_field("s2d", src_grid,true,false); + auto src_v2d = create_field("v2d", src_grid,true,true); + auto src_s3d_m = create_field("s3d_m",src_grid,false,false,true, 1); + auto src_s3d_i = create_field("s3d_i",src_grid,false,false,false,SCREAM_PACK_SIZE); + auto src_v3d_m = create_field("v3d_m",src_grid,false,true ,true, 1); + auto src_v3d_i = create_field("v3d_i",src_grid,false,true ,false,SCREAM_PACK_SIZE); + + auto tgt_s2d = create_field("s2d", tgt_grid,true,false); + auto tgt_v2d = create_field("v2d", tgt_grid,true,true); + auto tgt_s3d_m = create_field("s3d_m",tgt_grid,false,false,true, 1); + auto tgt_s3d_i = create_field("s3d_i",tgt_grid,false,false,true, SCREAM_PACK_SIZE); + auto tgt_v3d_m = create_field("v3d_m",tgt_grid,false,true ,true, 1); + auto tgt_v3d_i = create_field("v3d_i",tgt_grid,false,true ,true, SCREAM_PACK_SIZE); + + auto expected_s2d = tgt_s2d.clone(); + auto expected_v2d = tgt_v2d.clone(); + auto expected_s3d_m = tgt_s3d_m.clone(); + auto expected_s3d_i = tgt_s3d_i.clone(); + auto expected_v3d_m = tgt_v3d_m.clone(); + auto expected_v3d_i = tgt_v3d_i.clone(); + print (" -> creating fields ... done!\n",comm); + + // -------------------------------------- // + // Register fields in the remapper // + // -------------------------------------- // + + print (" -> creating and initializing remapper ...\n",comm); + auto remap = std::make_shared(src_grid,tgt_grid); + remap->set_source_pressure (pmid_src, pint_src); + remap->set_target_pressure (pmid_tgt, pint_tgt); + remap->set_extrapolation_type(etype_top,Top); + remap->set_extrapolation_type(etype_bot,Bot); + REQUIRE_THROWS (remap->set_mask_value(std::numeric_limits::quiet_NaN())); + remap->set_mask_value(mask_val); // Only needed if top and/or bot use etype=Mask + + remap->registration_begins(); + remap->register_field(src_s2d, tgt_s2d); + remap->register_field(src_v2d, tgt_v2d); + remap->register_field(src_s3d_m,tgt_s3d_m); + remap->register_field(src_s3d_i,tgt_s3d_i); + remap->register_field(src_v3d_m,tgt_v3d_m); + remap->register_field(src_v3d_i,tgt_v3d_i); + remap->registration_ends(); + print (" -> creating and initializing remapper ... done!\n",comm); + + // -------------------------------------- // + // Generate data for src fields // + // -------------------------------------- // + + print (" -> generate fields data ...\n",comm); + compute_field(src_s2d, pmid_src); + compute_field(src_v2d, pmid_src); + compute_field(src_s3d_m,pmid_src); + compute_field(src_s3d_i,pint_src); + compute_field(src_v3d_m,pmid_src); + compute_field(src_v3d_i,pint_src); + + // Pre-compute what we expect the tgt fields to be + compute_field(expected_s2d, pmid_tgt); + compute_field(expected_v2d, pmid_tgt); + compute_field(expected_s3d_m,pmid_tgt); + compute_field(expected_s3d_i,pint_tgt); + compute_field(expected_v3d_m,pmid_tgt); + compute_field(expected_v3d_i,pint_tgt); + + extrapolate(pmid_src,pmid_tgt,expected_s2d, etype_top,etype_bot); + extrapolate(pmid_src,pmid_tgt,expected_v2d, etype_top,etype_bot); + extrapolate(pmid_src,pmid_tgt,expected_s3d_m,etype_top,etype_bot); + extrapolate(pint_src,pint_tgt,expected_s3d_i,etype_top,etype_bot); + extrapolate(pmid_src,pmid_tgt,expected_v3d_m,etype_top,etype_bot); + extrapolate(pint_src,pint_tgt,expected_v3d_i,etype_top,etype_bot); + print (" -> generate fields data ... done!\n",comm); + + // -------------------------------------- // + // Perform remap // + // -------------------------------------- // + + // No bwd remap + REQUIRE_THROWS(remap->remap(false)); + + print (" -> run remap ...\n",comm); + remap->remap(true); + print (" -> run remap ... done!\n",comm); + + // -------------------------------------- // + // Check remapped fields // + // -------------------------------------- // + + using namespace Catch::Matchers; + Real tol = 10*std::numeric_limits::epsilon(); + + print (" -> check tgt fields ...\n",comm); + { + auto diff = tgt_s2d.clone("diff"); + auto ex_norm = frobenius_norm(expected_s2d); + diff.update(expected_s2d,1/ex_norm,-1/ex_norm); + REQUIRE (frobenius_norm(diff)(expected_v2d); + diff.update(expected_v2d,1/ex_norm,-1/ex_norm); + REQUIRE (frobenius_norm(diff)(expected_s3d_m); + diff.update(expected_s3d_m,1/ex_norm,-1/ex_norm); + REQUIRE (frobenius_norm(diff)(expected_s3d_i); + diff.update(expected_s3d_i,1/ex_norm,-1/ex_norm); + REQUIRE (frobenius_norm(diff)(expected_v3d_m); + diff.update(expected_v3d_m,1 / ex_norm,-1 / ex_norm); + REQUIRE (frobenius_norm(diff)(expected_v3d_i); + diff.update(expected_v3d_i,1 / ex_norm,-1 / ex_norm); + REQUIRE (frobenius_norm(diff) check tgt fields ... done!\n",comm); + } + } + } } - print ("check tgt fields ... done!\n",comm); } // Clean up scorpio stuff scorpio::finalize_subsystem(); + + print ("Testing vertical remapper ... done!\n",comm); } } // namespace scream diff --git a/components/eamxx/src/share/util/eamxx_data_interpolation.cpp b/components/eamxx/src/share/util/eamxx_data_interpolation.cpp new file mode 100644 index 00000000000..92fec65151b --- /dev/null +++ b/components/eamxx/src/share/util/eamxx_data_interpolation.cpp @@ -0,0 +1,455 @@ +#include "share/util/eamxx_data_interpolation.hpp" + +#include "share/grid/remap/identity_remapper.hpp" +#include "share/grid/remap/vertical_remapper.hpp" +#include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include "share/io/scream_io_utils.hpp" +#include "share/util/scream_universal_constants.hpp" + +#include +#include +#include +#include + +namespace scream{ + +DataInterpolation:: +DataInterpolation (const std::shared_ptr& model_grid, + const std::vector& fields) + : m_model_grid (model_grid) + , m_fields (fields) +{ + EKAT_REQUIRE_MSG (model_grid!=nullptr, + "[DataInterpolation] Error! Invalid grid pointer.\n"); + + m_nfields = m_fields.size(); + m_comm = model_grid->get_comm(); +} + +void DataInterpolation::run (const util::TimeStamp& ts) +{ + EKAT_REQUIRE_MSG (m_data_initialized, + "[DataInterpolation] Error! You must call 'init_data_interval' before calling 'run'.\n"); + + // If we went past the current interval end, we need to update the end state + if (not m_data_interval.contains(ts)) { + shift_data_interval (); + } + + // Perform the time interpolation: f_out = f_beg*alpha + f_end*(1-alpha), + // where alpha = (ts-t_beg) / (t_end-t_beg). + // NOTE: pay attention to time strategy, since for YearlyPeriodic you may + // have t_beg>t_end + util::TimeInterval beg_to_ts (m_data_interval.beg,ts,m_data_interval.timeline); + double alpha = beg_to_ts.length / m_data_interval.length; + EKAT_REQUIRE_MSG (alpha>=0 and alpha<=1, + "[DataInterpolation] Error! Input timestamp is outside the current data time interval.\n" + " data interval beg ; " + m_data_interval.beg.to_string() + "\n" + " data interval end ; " + m_data_interval.end.to_string() + "\n" + " input timestamp ; " + ts.to_string() + "\n" + " interval length : " + std::to_string(m_data_interval.length) + "\n" + " interpolation coeff: " + std::to_string(alpha) + "\n"); + + for (int i=0; iget_tgt_field(i); + const auto& end = m_horiz_remapper_end->get_tgt_field(i); + auto out = m_vert_remapper->get_src_field(i); + + out.deep_copy(beg); + out.update(end,alpha,1-alpha); + } + // For Dynamic3D profile we also need to compute the source pressure profile + // NOTE: this can't be done in the loop above, since src_p is not a "remapped" + // field in the vertical remapper (also, we need to use ad different ptr) + if (m_vr_type==Dynamic3D) { + // The pressure field is THE LAST registered in the horiz remappers + const auto p_beg = m_horiz_remapper_beg->get_tgt_field(m_nfields); + const auto p_end = m_horiz_remapper_end->get_tgt_field(m_nfields); + + auto p = m_vremap->get_source_pressure(true); // mid or int doesn't matter + p.deep_copy(p_beg); + p.update(p_end,alpha,1-alpha); + } + + m_vert_remapper->remap(true); +} + +void DataInterpolation::shift_data_interval () +{ + m_curr_interval_idx.first = m_curr_interval_idx.second; + m_curr_interval_idx.second = m_time_database.get_next_idx(m_curr_interval_idx.first); + + m_data_interval.advance(m_time_database.slices[m_curr_interval_idx.second].time); + std::swap (m_horiz_remapper_beg,m_horiz_remapper_end); + update_end_fields (); +} + +void DataInterpolation:: +update_end_fields () +{ + // First, set the correct fields in the reader + std::vector fields; + for (int i=0; iget_src_field(i)); + } + + if (m_vr_type==Dynamic3D) { + // We also need to read the src pressure profile + fields.push_back(m_horiz_remapper_end->get_src_field(m_nfields)); + } + m_reader->set_fields(fields); + + // If we're also changing the file, must (re)init the scorpio structures + const auto& slice = m_time_database.slices[m_curr_interval_idx.second]; + if (m_reader->get_filename()!=slice.filename) { + m_reader->reset_filename(slice.filename); + } + + // Read and interpolate fields + m_reader->read_variables(slice.time_idx); + m_horiz_remapper_end->remap(true); +} + +void DataInterpolation:: +init_data_interval (const util::TimeStamp& t0) +{ + EKAT_REQUIRE_MSG (m_remappers_created, + "[DataInterpolation] Error! Cannot call 'init_data_interval' until after remappers creation.\n"); + + // Create a bare reader. Fields and filename are set inside the update_end_fields call + strvec_t fnames; + for (auto f : m_fields) { + fnames.push_back(f.name()); + } + + m_reader = std::make_shared(fnames,m_horiz_remapper_beg->get_src_grid()); + + // Loop over all stored time slices to find an interval that contains t0 + auto t0_interval = m_time_database.find_interval(t0); + const auto& t_beg = m_time_database.slices[t0_interval].time; + + // We need to read in the beg/end fields for the initial interval. However, our generic + // framework can only load the end slice (since that's what we need at runtime). + // So, load end state for t=t_beg, then call shift_data_interval + // NOTE: don't compute length now, since beg time point is invalid (we don't need length yet). + m_data_interval = util::TimeInterval (util::TimeStamp(),t_beg,m_time_database.timeline,false); + m_curr_interval_idx.second = t0_interval; + update_end_fields (); + shift_data_interval (); + + m_data_initialized = true; +} + +void DataInterpolation:: +setup_time_database (const strvec_t& input_files, + const util::TimeLine timeline) +{ + // Log the final list of files, so the user know if something went wrong (e.g. a bad regex) + if (m_dbg_output and m_comm.am_i_root()) { + std::cout << "Setting up DataInerpolation object. List of input files:\n"; + for (const auto& fname : input_files) { + std::cout << " - " << fname << "\n"; + } + } + + // Make sure there are no repetitions + auto num_unique_files = std::unordered_set(input_files.begin(),input_files.end()).size(); + EKAT_REQUIRE_MSG (num_unique_files==input_files.size(), + "[DataInterpolation] Error! The input files list contains duplicates.\n" + " - input_files:\n " + ekat::join(input_files,"\n ") + "\n"); + + // We perform a bunch of checks on the input files + namespace fs = std::filesystem; + + auto file_readable = [] (const std::string& fileName) { + std::ifstream file(fileName); + return file.good(); // Check if the file can be opened + }; + + // Read what time stamps we have in each file + auto ts2str = [](const util::TimeStamp& t) { return t.to_string(); }; + std::vector> times; + for (const auto& fname : input_files) { + EKAT_REQUIRE_MSG (file_readable(input_files.back()), + "Error! One of the input files is not readable.\n" + " - file : " + input_files.back() + "\n"); + + scorpio::register_file(fname,scorpio::Read); + + auto file_times = scorpio::get_all_times(fname); + EKAT_REQUIRE_MSG (file_times.size()>0, + "[DataInterpolation] Error! Input file contains no time variable.\n" + " - file name: " + fname + "\n"); + + auto t_ref = read_timestamp (fname,"reference_time_stamp"); + + times.emplace_back(); + for (const auto& t : file_times) { + times.back().push_back(t_ref + t*constants::seconds_per_day); + } + scorpio::release_file(fname); + + // Ensure time slices are sorted (it would make code messy otherwise) + EKAT_REQUIRE_MSG (std::is_sorted(times.back().begin(),times.back().end()), + "[DataInterpolation] Error! One of the input files has time slices not sorted.\n" + " - file name : " + fname + "\n" + " - time stamps: " + ekat::join(times.back(),ts2str,", ") + "\n"); + } + + // Sort the files based on start date + auto fileCmp = [](const std::vector& times1, + const std::vector& times2) + { + return times1.front() < times2.front(); + }; + std::sort(times.begin(),times.end(),fileCmp); + + // Setup the time database + m_time_database.timeline = timeline; + m_time_database.files = input_files; + + int nfiles = input_files.size(); + for (int i=0; i0) { + // Ensure files don't overlap (it would be a mess) + const auto& prev = times[i-1]; + const auto& next = times[i]; + EKAT_REQUIRE_MSG (prev.back() < next.front(), + "[DataInterpolation] Error! The input files contain overlapping time slices.\n" + " - file1 name : " + input_files[i-1] + "\n" + " - file2 name : " + input_files[i] + "\n" + " - file1 times: " + ekat::join(prev,ts2str,", ") + "\n" + " - file2 times: " + ekat::join(next,ts2str,", ") + "\n"); + } + } + + // To avoid trouble in our logic of handling time stamps relationshipc, + // we must ensure we have 2+ time slices overall + EKAT_REQUIRE_MSG (m_time_database.size()>=2, + "[DataInterpolation] Error! Input file(s) only contain 1 time slice overall.\n"); + + m_time_db_created = true; +} + +void DataInterpolation:: +setup_remappers (const std::string& hremap_filename, + const VRemapType vr_type, + const std::string& data_pname, + const Field& model_pmid, + const Field& model_pint) +{ + setup_remappers(hremap_filename, + vr_type,"P0","P0", + -1, // Unused, since we do P0 extrapolation at top/bot + data_pname,model_pmid,model_pint); +} + +void DataInterpolation:: +setup_remappers (const std::string& hremap_filename, + const VRemapType vr_type, + const std::string& extrap_type_top, + const std::string& extrap_type_bot, + const Real mask_value, + const std::string& data_pname, + const Field& model_pmid, + const Field& model_pint) +{ + EKAT_REQUIRE_MSG (m_time_db_created, + "[DataInterpolation] Error! Cannot create remappers before time database.\n"); + + using IDR = IdentityRemapper; + constexpr auto SAT = IDR::SrcAliasTgt; + + // Whether horiz remap happens or not, the tgt grid of hremap is the same + // as the model grid, but with the same nubmer of levels as in the input files + auto grid_after_hremap = m_model_grid->clone("after_hremap",true); + int nlevs_data = get_input_files_dimlen ("lev"); + grid_after_hremap->reset_num_vertical_lev(nlevs_data); + + if (hremap_filename!="") { + m_horiz_remapper_beg = std::make_shared(grid_after_hremap,hremap_filename); + m_horiz_remapper_end = std::make_shared(grid_after_hremap,hremap_filename); + } else { + // If there's NO hremap, then ncols from the data must match the model grid (nlev can differ) + int ncols = get_input_files_dimlen ("ncol"); + EKAT_REQUIRE_MSG (ncols==m_model_grid->get_num_global_dofs(), + "Error! No horiz remap was requested, but the 'ncol' dim from file does not match with the model grid one.\n" + " - model grid num global cols: " + std::to_string(m_model_grid->get_num_global_dofs()) + "\n" + " - input data num global cols: " + std::to_string(ncols) + "\n"); + + m_horiz_remapper_beg = std::make_shared(grid_after_hremap,SAT); + m_horiz_remapper_end = std::make_shared(grid_after_hremap,SAT); + } + + if (vr_type!=None) { + auto s2et = [](const std::string& s) { + if (s=="P0") { + return VerticalRemapper::P0; + } else if (s=="Mask") { + return VerticalRemapper::Mask; + } else { + EKAT_ERROR_MSG ( + "Error! Invalid/unsupported extrapolation type.\n" + " - input value : " + s + "\n" + " - valid values: P0, Mask\n"); + return static_cast(-1); + } + }; + + m_vert_remapper = m_vremap = std::make_shared(grid_after_hremap,m_model_grid); + m_vremap->set_extrapolation_type(s2et(extrap_type_top),VerticalRemapper::Top); + m_vremap->set_extrapolation_type(s2et(extrap_type_bot),VerticalRemapper::Bot); + m_vremap->set_mask_value(mask_value); + } else { + // If no vert remap is requested, model_grid and grid_after_hremap MUST have same nlevs + int model_nlevs = m_model_grid->get_num_vertical_levels(); + EKAT_REQUIRE_MSG (model_nlevs==nlevs_data, + "Error! No vertical remap was requested, but the 'lev' dim from file does not match the model grid one.\n" + " - model grid num vert levels: " + std::to_string(model_nlevs) + "\n" + " - input data num vert levels: " + std::to_string(nlevs_data) + "\n"); + m_vert_remapper = std::make_shared(grid_after_hremap,SAT); + } + + // Setup vertical pressure profiles (which can add 1 extra field to hremap) + // This MUST be done before registering in vremap, since register_field_from_tgt + // REQUIRES to have source pressure profiles set BEFORE. + Field data_p; + if (vr_type==Dynamic3D) { + // We also need to load and remap the pressure from the input files + auto hr_tgt_grid = m_horiz_remapper_beg->get_tgt_grid(); + auto p_layout = hr_tgt_grid->get_3d_scalar_layout(true); + data_p = Field (FieldIdentifier(data_pname,p_layout,ekat::units::Pa,hr_tgt_grid->name())); + data_p.allocate_view(); + + m_vremap->set_source_pressure (data_p,VerticalRemapper::Both); + m_vremap->set_target_pressure (model_pmid,model_pint); + } else if (vr_type==Static1D) { + auto hr_tgt_grid = m_horiz_remapper_beg->get_tgt_grid(); + auto p_layout = hr_tgt_grid->get_vertical_layout(true); + data_p = Field (FieldIdentifier(data_pname,p_layout,ekat::units::Pa,hr_tgt_grid->name())); + data_p.allocate_view(); + + // Use raw scorpio to read this var, since it's not decomposed. Use any file, since it's static + auto filename = m_time_database.files.front(); + scorpio::register_file(filename,scorpio::Read); + scorpio::read_var(filename,data_pname,data_p.get_internal_view_data()); + scorpio::release_file(filename); + data_p.sync_to_dev(); + + m_vremap->set_source_pressure (data_p,VerticalRemapper::Both); + m_vremap->set_target_pressure (model_pmid,model_pint); + } + m_vr_type = vr_type; + + // Register fields in the remappers. Vertical first, since we only have model-grid fields + m_vert_remapper->registration_begins(); + for (int i=0; iregister_field_from_tgt(m_fields[i]); + } + m_vert_remapper->registration_ends(); + + m_horiz_remapper_beg->registration_begins(); + m_horiz_remapper_end->registration_begins(); + for (int i=0; iget_src_field(i); + m_horiz_remapper_beg->register_field_from_tgt(f.clone()); + m_horiz_remapper_end->register_field_from_tgt(f.clone()); + } + if (vr_type==Dynamic3D) { + m_horiz_remapper_beg->register_field_from_tgt(data_p.clone()); + m_horiz_remapper_end->register_field_from_tgt(data_p.clone()); + } + m_horiz_remapper_beg->registration_ends(); + m_horiz_remapper_end->registration_ends(); + + m_remappers_created = true; +} + +int DataInterpolation::TimeDatabase:: +get_next_idx (int prev) const +{ + int next = prev+1; + if (next >= size()) { + EKAT_REQUIRE_MSG (timeline==util::TimeLine::YearlyPeriodic, + "[TimeDatabase::get_next_idx] Error! Requesting slice that is past the database end.\n"); + next = next % size(); + } + return next; +} + +int DataInterpolation::TimeDatabase:: +find_interval (const util::TimeStamp& t) const +{ + EKAT_REQUIRE_MSG (size()>1, + "[TimeDatabase::find_interval] Error! The database has not been initialized yet.\n"); + + auto contains = [&](int beg, int end, const util::TimeStamp& t) { + const auto& t_beg = slices[beg].time; + const auto& t_end = slices[end].time; + util::TimeInterval t_int (t_beg,t_end,timeline); + return t_int.contains(t); + }; + int beg=0; + int end=1; + while (end; + enum VRemapType { + None, + Static1D, + Dynamic3D + }; + + // Constructor(s) & Destructor + DataInterpolation (const std::shared_ptr& model_grid, + const std::vector& fields); + + ~DataInterpolation () = default; + + void toggle_debug_output (bool enable_dbg_output) { m_dbg_output = enable_dbg_output; } + + void setup_time_database (const strvec_t& input_files, const util::TimeLine timeline); + + void setup_remappers (const std::string& hremap_filename, + const VRemapType vr_type, + const std::string& data_pname, + const Field& model_pmid, + const Field& model_pint); + + void setup_remappers (const std::string& hremap_filename, + const VRemapType vr_type, + const std::string& extrap_type_top, + const std::string& extrap_type_bot, + const Real mask_value, + const std::string& data_pname, + const Field& model_pmid, + const Field& model_pint); + + void init_data_interval (const util::TimeStamp& t0); + + void run (const util::TimeStamp& ts); + +protected: + + void shift_data_interval (); + void update_end_fields (); + + int get_input_files_dimlen (const std::string& dimname) const; + + // ----------- Internal data types ---------- // + + struct DataSlice { + util::TimeStamp time; + std::string filename; + int time_idx; // slice index within the input file + }; + + struct TimeDatabase { + strvec_t files; + std::vector slices; + util::TimeLine timeline; + + int size () const { return slices.size(); } + int get_next_idx (int prev_idx) const; + + // Find interval containing t + int find_interval (const util::TimeStamp& t) const; + }; + + // --------------- Internal data ------------- // + + std::shared_ptr m_reader; + + std::shared_ptr m_model_grid; + + std::vector m_fields; + + // Use two horiz remappers, so we only set them up once (it may be costly) + std::shared_ptr m_horiz_remapper_beg; + std::shared_ptr m_horiz_remapper_end; + std::shared_ptr m_vert_remapper; + std::shared_ptr m_vremap; + + VRemapType m_vr_type; + int m_nfields; + + util::TimeInterval m_data_interval; + std::pair m_curr_interval_idx; + + TimeDatabase m_time_database; + + ekat::Comm m_comm; + ekat::ParameterList m_params; + + bool m_time_db_created = false; + bool m_remappers_created = false; + bool m_data_initialized = false; + + bool m_dbg_output = false; +}; + +} // namespace scream + +#endif // EAMXX_DATA_INTERPOLATION_HPP diff --git a/components/eamxx/src/share/util/scream_data_type.hpp b/components/eamxx/src/share/util/scream_data_type.hpp index 31ce6b9f8ef..8c45cd734b6 100644 --- a/components/eamxx/src/share/util/scream_data_type.hpp +++ b/components/eamxx/src/share/util/scream_data_type.hpp @@ -52,6 +52,7 @@ inline std::string e2str (const DataType data_type) { default: EKAT_ERROR_MSG("Error! Unsupported DataType value.\n"); } + return ""; } inline int get_type_size (const DataType data_type) { @@ -62,6 +63,7 @@ inline int get_type_size (const DataType data_type) { default: EKAT_ERROR_MSG("Error! Unsupported DataType value.\n"); } + return -1; } } // namespace scream diff --git a/components/eamxx/src/share/util/scream_time_stamp.cpp b/components/eamxx/src/share/util/scream_time_stamp.cpp index d2ef8243e25..3bd8d092964 100644 --- a/components/eamxx/src/share/util/scream_time_stamp.cpp +++ b/components/eamxx/src/share/util/scream_time_stamp.cpp @@ -16,15 +16,6 @@ namespace scream { namespace util { -int days_in_month (const int yy, const int mm) { - EKAT_REQUIRE_MSG (mm>=1 && mm<=12, - "Error! Month out of bounds. Did you call `days_in_month` with yy and mm swapped?\n"); - constexpr int nonleap_days [12] = {31,28,31,30,31,30,31,31,30,31,30,31}; - constexpr int leap_days [12] = {31,29,31,30,31,30,31,31,30,31,30,31}; - auto& arr = is_leap_year(yy) ? leap_days : nonleap_days; - return arr[mm-1]; -} - bool is_leap_year (const int yy) { if (use_leap_year()) { if (yy%4==0) { @@ -41,6 +32,15 @@ bool is_leap_year (const int yy) { return false; } +int days_in_month (const int yy, const int mm) { + EKAT_REQUIRE_MSG (mm>=1 && mm<=12, + "Error! Month out of bounds. Did you call `days_in_month` with yy and mm swapped?\n"); + constexpr int nonleap_days [12] = {31,28,31,30,31,30,31,31,30,31,30,31}; + constexpr int leap_days [12] = {31,29,31,30,31,30,31,31,30,31,30,31}; + auto& arr = is_leap_year(yy) ? leap_days : nonleap_days; + return arr[mm-1]; +} + TimeStamp::TimeStamp() : m_date (3,std::numeric_limits::lowest()) , m_time (3,std::numeric_limits::lowest()) @@ -135,6 +135,23 @@ double TimeStamp::frac_of_year_in_days () const { return doy; } +int TimeStamp::days_in_curr_month () const +{ + return days_in_month(m_date[0],m_date[1]); +} + +int TimeStamp::days_in_curr_year () const +{ + return is_leap_year(m_date[0]) ? 366 : 365; +} + +TimeStamp TimeStamp::curr_month_beg () const +{ + auto date = m_date; + date[2] = 1; + return TimeStamp (date,{0,0,0}); +} + TimeStamp& TimeStamp::operator+=(const double seconds) { // Sanity checks // Note: (x-int(x)) only works for x small enough that can be stored in an int, diff --git a/components/eamxx/src/share/util/scream_time_stamp.hpp b/components/eamxx/src/share/util/scream_time_stamp.hpp index 259686eacf6..1fd1c04e421 100644 --- a/components/eamxx/src/share/util/scream_time_stamp.hpp +++ b/components/eamxx/src/share/util/scream_time_stamp.hpp @@ -45,6 +45,11 @@ class TimeStamp { std::string get_time_string () const; double frac_of_year_in_days () const; + int days_in_curr_month () const; + int days_in_curr_year () const; + + TimeStamp curr_month_beg () const; + // === Update method(s) === // // Set the counter for the number of steps. @@ -78,13 +83,89 @@ std::int64_t operator- (const TimeStamp& ts1, const TimeStamp& ts2); // Rewind time by given number of seconds TimeStamp operator- (const TimeStamp& ts, const int dt); -// Time-related free-functions -int days_in_month (const int year, const int month); -bool is_leap_year (const int year); - // If input string is not of the format YYYY-MM-DD-XXXXX, returns an invalid time stamp TimeStamp str_to_time_stamp (const std::string& s); +// An enum describing two ways to look at timestamps: +// - Linear: treat them as part of a 1d line +// - YearlyPeriodic: treat them as part of a yearly periodic orbit +// This is used in the TimeInterval class below to correctly handle time stamps differences +enum class TimeLine { + YearlyPeriodic, + Linear +}; + +/* + * Small struct to deal with time intervals + * + * The struct simply contains timestamps for [begin,end] interval, + * and allows two things: compute the interval length (in days), and check if + * a timestamp lies within the interval. + * + * When the TimeLine arg to the ctor is YearlyPeriodic, the year part of beg/end + * time points is ignored. In this case, the length of the time interval is bound + * to be in the interval [0,365] (in non-leap years) + */ +struct TimeInterval { + + TimeStamp beg; + TimeStamp end; + TimeLine timeline = TimeLine::Linear; + double length = -1; // the interval length + + TimeInterval () = default; + TimeInterval (const util::TimeStamp& b, const util::TimeStamp& e, TimeLine tl, bool do_compute_length = true) + : beg (b), end (e), timeline (tl) + { + if (do_compute_length) + compute_length (); + } + + bool contains (const util::TimeStamp& t) const { + if (timeline==TimeLine::Linear) { + // Compare the full time stamps + return beg<=t and t<=end; + } else { + // Compare the fraction of year for beg/end and t. + // Pay extra attention to the case where new year's eve + // is in [bec,end] + auto t_frac = t.frac_of_year_in_days(); + auto end_frac = end.frac_of_year_in_days(); + auto beg_frac = beg.frac_of_year_in_days(); + bool across_nye = beg.get_month()>end.get_month(); + if (not across_nye) { + return beg_frac<=t_frac and t_frac<=end_frac; + } else { + // We are either PAST beg or BEFORE end (but not both) + return t_frac>=beg_frac or t_frac<=end_frac; + } + } + } + + void compute_length () { + if (timeline==TimeLine::Linear) { + length = end.days_from(beg); + } else { + bool across_nye = beg.get_month()>end.get_month(); + auto frac_beg = beg.frac_of_year_in_days(); + auto frac_end = end.frac_of_year_in_days(); + if (across_nye) { + double year = end.days_in_curr_year(); + length = frac_end + (year - frac_beg); + } else { + length = frac_end - frac_beg; + } + } + } + + // Advance the interval, so that it now starts from the old end + void advance(const TimeStamp& new_end) { + beg = end; + end = new_end; + compute_length(); + } +}; + } // namespace util } // namespace scream diff --git a/components/eamxx/src/share/util/scream_utils.hpp b/components/eamxx/src/share/util/scream_utils.hpp index dbb315fc4b9..9577b5597bf 100644 --- a/components/eamxx/src/share/util/scream_utils.hpp +++ b/components/eamxx/src/share/util/scream_utils.hpp @@ -368,6 +368,112 @@ constexpr int eamxx_vis_swband_idx() { return 10; } +struct DefaultMetadata { + + std::string get_longname (const std::string& name) { + if (name_2_longname.count(name)>0) { + return name_2_longname.at(name); + } else { + // TODO: Do we want to print a Warning message? I'm not sure if its needed. + return name; + } + } + + std::string get_standardname (const std::string& name) { + if (name_2_standardname.count(name)>0) { + return name_2_standardname.at(name); + } else { + // TODO: Do we want to print a Warning message? I'm not sure if its needed. + return name; + } + } + + // Create map of longnames, can be added to as developers see fit. + std::map name_2_longname = { + {"lev","hybrid level at midpoints (1000*(A+B))"}, + {"ilev","hybrid level at interfaces (1000*(A+B))"}, + {"hyai","hybrid A coefficient at layer interfaces"}, + {"hybi","hybrid B coefficient at layer interfaces"}, + {"hyam","hybrid A coefficient at layer midpoints"}, + {"hybm","hybrid B coefficient at layer midpoints"} + }; + + // Create map of longnames, can be added to as developers see fit. + std::map name_2_standardname = { + {"p_mid" , "air_pressure"}, + {"p_mid_at_cldtop" , "air_pressure_at_cloud_top"}, + {"T_2m" , "air_temperature"}, + {"T_mid" , "air_temperature"}, + {"T_mid_at_cldtop" , "air_temperature_at_cloud_top"}, + {"aero_g_sw" , "asymmetry_factor_of_ambient_aerosol_particles"}, + {"pbl_height" , "atmosphere_boundary_layer_thickness"}, + {"precip_liq_surf_mass" , "atmosphere_mass_content_of_liquid_precipitation"}, + {"cldlow" , "low_type_cloud_area_fraction"}, + {"cldmed" , "medium_type_cloud_area_fraction"}, + {"cldhgh" , "high_type_cloud_area_fraction"}, + {"cldtot" , "cloud_area_fraction"}, + {"cldfrac_tot_at_cldtop" , "cloud_area_fraction"}, + {"cldfrac_tot" , "cloud_area_fraction_in_atmosphere_layer"}, + {"cldfrac_tot_for_analysis" , "cloud_area_fraction_in_atmosphere_layer"}, + {"cldfrac_rad" , "cloud_area_fraction_in_atmosphere_layer"}, + {"qi" , "cloud_ice_mixing_ratio"}, + {"qc" , "cloud_liquid_water_mixing_ratio"}, + {"U" , "eastward_wind"}, + {"eff_radius_qi" , "effective_radius_of_cloud_ice_particles"}, + {"eff_radius_qc" , "effective_radius_of_cloud_liquid_water_particles"}, + {"eff_radius_qc_at_cldtop" , "effective_radius_of_cloud_liquid_water_particles_at_liquid_water_cloud_top"}, + {"eff_radius_qr" , "effective_radius_of_cloud_rain_particles"}, + {"qv" , "humidity_mixing_ratio"}, + {"cldfrac_ice_at_cldtop" , "ice_cloud_area_fraction"}, + {"cldfrac_ice" , "ice_cloud_area_fraction_in_atmosphere_layer"}, + {"omega" , "lagrangian_tendency_of_air_pressure"}, + {"landfrac" , "land_area_fraction"}, + {"latitude" , "latitude"}, + {"cldfrac_liq_at_cldtop" , "liquid_water_cloud_area_fraction"}, + {"cldfrac_liq" , "liquid_water_cloud_area_fraction_in_atmosphere_layer"}, + {"longitude" , "longitude"}, + {"rainfrac" , "mass_fraction_of_liquid_precipitation_in_air"}, + {"V" , "northward_wind"}, + {"nc" , "number_concentration_of_cloud_liquid_water_particles_in_air"}, + {"cdnc_at_cldtop" , "number_concentration_of_cloud_liquid_water_particles_in_air_at_liquid_water_cloud_top"}, + {"ni" , "number_concentration_of_ice_crystals_in_air"}, + {"aero_tau_sw" , "optical_thickness_of_atmosphere_layer_due_to_ambient_aerosol_particles"}, + {"aero_tau_lw" , "optical_thickness_of_atmosphere_layer_due_to_ambient_aerosol_particles"}, + {"aero_ssa_sw" , "single_scattering_albedo_in_air_due_to_ambient_aerosol_particles"}, + {"sunlit" , "sunlit_binary_mask"}, + {"ps" , "surface_air_pressure"}, + {"LW_flux_dn_at_model_bot" , "surface_downwelling_longwave_flux_in_air"}, + {"SW_flux_dn_at_model_bot" , "surface_downwelling_shortwave_flux_in_air"}, + {"SW_clrsky_flux_dn_at_model_bot" , "surface_downwelling_shortwave_flux_in_air_assuming_clear_sky"}, + {"phis" , "surface_geopotential"}, + {"surf_radiative_T" , "surface_temperature"}, + {"surf_sens_flux" , "surface_upward_sensible_heat_flux"}, + {"SW_flux_dn_at_model_top" , "toa_incoming_shortwave_flux"}, + {"LW_flux_up_at_model_top" , "toa_outgoing_longwave_flux"}, + {"LW_clrsky_flux_up_at_model_top" , "toa_outgoing_longwave_flux_assuming_clear_sky"}, + {"surf_evap" , "water_evapotranspiration_flux"}, + {"AtmosphereDensity" , "air_density"}, + {"PotentialTemperature" , "air_potential_temperature"}, + {"SeaLevelPressure" , "air_pressure_at_mean_sea_level"}, + {"IceWaterPath" , "atmosphere_mass_content_of_cloud_ice"}, + {"LiqWaterPath" , "atmosphere_mass_content_of_cloud_liquid_water"}, + {"VapWaterPath" , "atmosphere_mass_content_of_water_vapor"}, + {"AerosolOpticalDepth550nm" , "atmosphere_optical_thickness_due_to_ambient_aerosol_particles"}, + {"Exner" , "dimensionless_exner_function"}, + {"z_mid" , "geopotential_height"}, + {"geopotential_mid" , "geopotential_height"}, + {"RelativeHumidity" , "relative_humidity"}, + {"surface_upward_latent_heat_flux" , "surface_upward_latent_heat_flux"}, + {"LongwaveCloudForcing" , "toa_longwave_cloud_radiative_effect"}, + {"ShortwaveCloudForcing" , "toa_shortwave_cloud_radiative_effect"}, + {"VirtualTemperature" , "virtual_temperature"}, + {"VaporFlux" , "water_evapotranspiration_flux"}, + {"wind_speed" , "wind_speed"} + }; + +}; + + } // namespace scream #endif // SCREAM_UTILS_HPP diff --git a/components/eamxx/tests/generic/fail_check/valg_fail.cpp b/components/eamxx/tests/generic/fail_check/valg_fail.cpp index ba0ba6f59b3..5bbd694dad3 100644 --- a/components/eamxx/tests/generic/fail_check/valg_fail.cpp +++ b/components/eamxx/tests/generic/fail_check/valg_fail.cpp @@ -6,15 +6,18 @@ namespace scream { TEST_CASE("force_valgrind_err") { - bool uninit; + bool* uninit = new bool[1]; int i = 0; - if (uninit) { + if (uninit[0]) { ++i; } else { i += 4; } - REQUIRE(i < 10); + if (i<4) { + printf("less than four\n"); + } + delete uninit; } } // empty namespace diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/input.yaml index 316b43ffdf7..15b8f8572b8 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/input.yaml @@ -41,7 +41,7 @@ atmosphere_processes: shoc: enable_column_conservation_checks: true lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/input.yaml index 8e463b8be24..c9f0bdaa8b8 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/input.yaml @@ -46,7 +46,7 @@ atmosphere_processes: max_total_ni: 740.0e3 shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/input.yaml index d2f51e9e1cc..d8dc6182ae4 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/input.yaml @@ -37,7 +37,7 @@ atmosphere_processes: shoc: check_flux_state_consistency: true lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/input.yaml index b92889cdcaa..3fd2b41cd31 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/input.yaml @@ -37,7 +37,7 @@ atmosphere_processes: max_total_ni: 740.0e3 shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/CMakeLists.txt index f56d8beefa6..db5c8585943 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/CMakeLists.txt @@ -11,7 +11,7 @@ CreateDynamicsLib("theta-l_kokkos" 4 72 10) set (TEST_LABELS "dynamics;driver;tms;shoc;cld;spa;p3;rrtmgp;physics;dp") CreateUnitTest(homme_shoc_cld_spa_p3_rrtmgp_pg2_dp "homme_shoc_cld_spa_p3_rrtmgp_pg2_dp.cpp" LABELS ${TEST_LABELS} - LIBS cld_fraction tms shoc spa p3 scream_rrtmgp ${dynLibName} scream_control diagnostics + LIBS cld_fraction tms shoc spa iop_forcing p3 scream_rrtmgp ${dynLibName} scream_control diagnostics MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/input.yaml index 0950f6bfdbc..fdd46617bd5 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/input.yaml @@ -40,7 +40,7 @@ atmosphere_processes: homme: Moisture: moist physics: - atm_procs_list: [mac_aero_mic,rrtmgp] + atm_procs_list: [iop_forcing,mac_aero_mic,rrtmgp] schedule_type: Sequential Type: Group mac_aero_mic: @@ -55,7 +55,7 @@ atmosphere_processes: shoc: check_flux_state_consistency: true lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/input.yaml index 3dbd99682fd..adae9b0688a 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/input.yaml @@ -62,7 +62,7 @@ atmosphere_processes: shoc: enable_column_conservation_checks: true lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/input.yaml index e6b94236488..21791e3391e 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/input.yaml @@ -43,7 +43,7 @@ atmosphere_processes: shoc: enable_column_conservation_checks: true lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml index 6778e18ad8d..a95d2677fe4 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml @@ -49,7 +49,7 @@ atmosphere_processes: shoc: check_flux_state_consistency: true lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_baseline.yaml b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_baseline.yaml index c308625a6e1..b5d749bfeff 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_baseline.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_baseline.yaml @@ -41,7 +41,7 @@ atmosphere_processes: max_total_ni: 740.0e3 shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_initial.yaml b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_initial.yaml index d5339747fdb..4ab3bdc369d 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_initial.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_initial.yaml @@ -41,7 +41,7 @@ atmosphere_processes: do_prescribed_ccn: false shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_restarted.yaml b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_restarted.yaml index f7d1075e334..345424605a7 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_restarted.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_restarted.yaml @@ -28,7 +28,7 @@ atmosphere_processes: do_prescribed_ccn: false shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/input.yaml b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/input.yaml index 4e9f9f79adb..b23046cf2d1 100644 --- a/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/input.yaml @@ -16,7 +16,7 @@ atmosphere_processes: max_total_ni: 740.0e3 shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml index 9c0ed4d9798..1cc9eb6eb4d 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml @@ -21,7 +21,7 @@ atmosphere_processes: do_prescribed_ccn: false shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml index 20d6f142c52..9483f3790bf 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml @@ -24,7 +24,7 @@ atmosphere_processes: do_prescribed_ccn: false shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml index 7ec317eda92..2a571ba96f1 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml @@ -24,7 +24,7 @@ atmosphere_processes: do_prescribed_ccn: false shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml index 804939a2205..d970765d2bd 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml @@ -21,7 +21,7 @@ atmosphere_processes: do_prescribed_ccn: false shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml index ae997bb3da5..c70f1b8085a 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml @@ -21,7 +21,7 @@ atmosphere_processes: top_level_mam4xx: 6 shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml index 24f610f041c..3f598140309 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml @@ -18,7 +18,7 @@ atmosphere_processes: number_of_subcycles: ${MAC_MIC_SUBCYCLES} shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/input.yaml index fc0a1b6bc8a..529a075b10d 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/input.yaml @@ -21,7 +21,7 @@ atmosphere_processes: do_prescribed_ccn: false shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/input.yaml index 542625798f5..a98cbe30e85 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/input.yaml @@ -22,7 +22,7 @@ atmosphere_processes: max_total_ni: 740.0e3 shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml index 6a4c0003376..12acd670ad9 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml @@ -23,7 +23,7 @@ atmosphere_processes: source_pressure_file: vertical_remap.nc ## Only used in the case of STATIC_1D_VERTICAL_PROFILE shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging_glob_novert.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging_glob_novert.yaml index 35d75015e2a..68e8841586d 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging_glob_novert.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging_glob_novert.yaml @@ -24,7 +24,7 @@ atmosphere_processes: source_pressure_file: vertical_remap.nc ## Only used in the case of STATIC_1D_VERTICAL_PROFILE shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_source_data.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_source_data.yaml index c0c3056312e..c6ec0e361c6 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_source_data.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_source_data.yaml @@ -15,7 +15,7 @@ atmosphere_processes: max_total_ni: 720.0e3 shoc: lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/eamxx/tests/single-process/CMakeLists.txt b/components/eamxx/tests/single-process/CMakeLists.txt index 2d0ae749c49..074a30c924f 100644 --- a/components/eamxx/tests/single-process/CMakeLists.txt +++ b/components/eamxx/tests/single-process/CMakeLists.txt @@ -22,10 +22,13 @@ if (SCREAM_ENABLE_MAM) add_subdirectory(mam/optics) add_subdirectory(mam/aci) add_subdirectory(mam/drydep) - add_subdirectory(mam/wet_scav) add_subdirectory(mam/emissions) add_subdirectory(mam/constituent_fluxes) - add_subdirectory(mam/aero_microphys) + add_subdirectory(mam/wet_scav) + + # Currently this test produces non-deterministic output. + # Commenting it so the other PRs can test normally + # add_subdirectory(mam/aero_microphys) endif() if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) add_subdirectory(zm) diff --git a/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml b/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml index 7dd28a87f47..48070ee3b0d 100644 --- a/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml +++ b/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml @@ -60,6 +60,15 @@ initial_conditions: wetdens: [1038.67760516297, 1046.20002003441, 1031.74623165457, 1086.79731859184] nevapr: 0.0 precip_total_tend: 0.0 + surf_radiative_T: 288.0 + ps: 105000.0 + horiz_winds: [-0.24988988196194634E+000, -0.23959782871450760E+000] + precip_liq_surf_mass: 0.1 + precip_ice_surf_mass: 0.1 + snow_depth_land: 0.01 + fraction_landuse: 0.0 + SW_flux_dn: 500.0 + constituent_fluxes: 0.0 # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] diff --git a/components/eamxx/tests/single-process/shoc/input.yaml b/components/eamxx/tests/single-process/shoc/input.yaml index befb43d98a0..179aa09c467 100644 --- a/components/eamxx/tests/single-process/shoc/input.yaml +++ b/components/eamxx/tests/single-process/shoc/input.yaml @@ -14,7 +14,7 @@ atmosphere_processes: number_of_subcycles: ${NUM_SUBCYCLES} compute_tendencies: [all] lambda_low: 0.001 - lambda_high: 0.04 + lambda_high: 0.08 lambda_slope: 2.65 lambda_thresh: 0.02 thl2tune: 1.0 diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index c76271b7baf..87a6da722af 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -314,7 +314,7 @@ Allowed values are: 5 : use gross domestic production and population datasets to simulate anthropogenic fire supression - Set FATES harvesting mode by setting fates_harvest_mode diff --git a/components/elm/src/biogeophys/HydrologyDrainageMod.F90 b/components/elm/src/biogeophys/HydrologyDrainageMod.F90 index 850b7eedac5..fbd4d8e2b6f 100755 --- a/components/elm/src/biogeophys/HydrologyDrainageMod.F90 +++ b/components/elm/src/biogeophys/HydrologyDrainageMod.F90 @@ -47,7 +47,7 @@ subroutine HydrologyDrainage(bounds, & ! ! !USES: !$acc routine seq - use landunit_varcon , only : istice, istwet, istsoil, istice_mec, istcrop + use landunit_varcon , only : istice, istwet, istsoil, istice_mec, istcrop, istice use column_varcon , only : icol_roof, icol_road_imperv, icol_road_perv, icol_sunwall, icol_shadewall use elm_varcon , only : denh2o, denice, secspday use elm_varctl , only : glc_snow_persistence_max_days, use_vichydro, use_betr @@ -120,7 +120,9 @@ subroutine HydrologyDrainage(bounds, & qflx_runoff_r => col_wf%qflx_runoff_r , & ! Output: [real(r8) (:) ] Rural total runoff (qflx_drain+qflx_surf+qflx_qrgwl) (mm H2O /s) qflx_snwcp_ice => col_wf%qflx_snwcp_ice , & ! Output: [real(r8) (:) ] excess snowfall due to snow capping (mm H2O /s) [+]` qflx_glcice => col_wf%qflx_glcice , & ! Output: [real(r8) (:) ] flux of new glacier ice (mm H2O /s) - qflx_glcice_frz => col_wf%qflx_glcice_frz & ! Output: [real(r8) (:) ] ice growth (positive definite) (mm H2O/s) + qflx_glcice_frz => col_wf%qflx_glcice_frz , & ! Output: [real(r8) (:) ] ice growth (positive definite) (mm H2O/s) + qflx_glcice_diag => col_wf%qflx_glcice_diag , & ! Output: [real(r8) (:) ] flux of new glacier ice (mm H2O/s) - diagnostic, no MECs or GLC + qflx_glcice_frz_diag => col_wf%qflx_glcice_frz_diag & ! Output: [real(r8) (:) ] ice growth (positive definite) (mm H2O/s)) - diagnostic, no MECs or GLC ) ! Determine time step and step size @@ -215,6 +217,12 @@ subroutine HydrologyDrainage(bounds, & do c = bounds%begc,bounds%endc qflx_glcice_frz(c) = 0._r8 + qflx_glcice_frz_diag(c) = 0._r8 + + if (lun_pp%itype(l)==istice .and. qflx_snwcp_ice(c) > 0.0_r8) then + qflx_glcice_frz_diag(c) = qflx_snwcp_ice(c) + qflx_glcice_diag(c) = qflx_glcice_diag(c) + qflx_glcice_frz_diag(c) + endif end do do fc = 1,num_do_smb_c c = filter_do_smb_c(fc) @@ -222,10 +230,10 @@ subroutine HydrologyDrainage(bounds, & g = col_pp%gridcell(c) ! In the following, we convert glc_snow_persistence_max_days to r8 to avoid overflow if ( (snow_persistence(c) >= (real(glc_snow_persistence_max_days, r8) * secspday)) & - .or. lun_pp%itype(l) == istice_mec) then - qflx_glcice_frz(c) = qflx_snwcp_ice(c) - qflx_glcice(c) = qflx_glcice(c) + qflx_glcice_frz(c) - if (glc_dyn_runoff_routing(g)) qflx_snwcp_ice(c) = 0._r8 + .or. lun_pp%itype(l) == istice_mec ) then + qflx_glcice_frz(c) = qflx_snwcp_ice(c) + qflx_glcice(c) = qflx_glcice(c) + qflx_glcice_frz(c) + if (glc_dyn_runoff_routing(g)) qflx_snwcp_ice(c) = 0._r8 end if end do diff --git a/components/elm/src/biogeophys/SnowHydrologyMod.F90 b/components/elm/src/biogeophys/SnowHydrologyMod.F90 index f9270289d05..49503da8ad8 100644 --- a/components/elm/src/biogeophys/SnowHydrologyMod.F90 +++ b/components/elm/src/biogeophys/SnowHydrologyMod.F90 @@ -670,7 +670,7 @@ subroutine SnowCompaction(bounds, num_snowc, filter_snowc, & if (bi > dm) ddz1 = ddz1*exp(-46.0e-3_r8*(bi-dm)) else ddz1_fresh = (-grav * (burden(c) + wx/2._r8)) / & - (0.007_r8 * bi**(4.75_r8 + td/40._r8)) + (0.007_r8 * min(max(bi,dm),denice)**(4.75_r8 + min(td,0._r8)/40._r8)) snw_ssa = 3.e6_r8 / (denice * snw_rds(c,j)) if (snw_ssa < 50._r8) then ddz1_fresh = ddz1_fresh * exp(-46.e-2_r8 * (50._r8 - snw_ssa)) diff --git a/components/elm/src/biogeophys/SoilTemperatureMod.F90 b/components/elm/src/biogeophys/SoilTemperatureMod.F90 index d4a1074bf4a..451b3cbccf2 100644 --- a/components/elm/src/biogeophys/SoilTemperatureMod.F90 +++ b/components/elm/src/biogeophys/SoilTemperatureMod.F90 @@ -1316,7 +1316,7 @@ subroutine Phasechange_beta (bounds, num_nolakec, filter_nolakec, dhsdT, & use elm_varctl , only : iulog use elm_varcon , only : tfrz, hfus, grav use column_varcon , only : icol_roof, icol_sunwall, icol_shadewall, icol_road_perv - use landunit_varcon , only : istsoil, istcrop, istice_mec + use landunit_varcon , only : istsoil, istcrop, istice_mec,istice ! ! !ARGUMENTS: type(bounds_type) , intent(in) :: bounds @@ -1369,6 +1369,8 @@ subroutine Phasechange_beta (bounds, num_nolakec, filter_nolakec, dhsdT, & qflx_snofrz_col => col_wf%qflx_snofrz , & ! Output: [real(r8) (:) ] column-integrated snow freezing rate (positive definite) [kg m-2 s-1] qflx_glcice => col_wf%qflx_glcice , & ! Output: [real(r8) (:) ] flux of new glacier ice (mm H2O/s) [+ = ice grows] qflx_glcice_melt => col_wf%qflx_glcice_melt , & ! Output: [real(r8) (:) ] ice melt (positive definite) (mm H2O/s) + qflx_glcice_diag => col_wf%qflx_glcice_diag , & ! Output: [real(r8) (:) ] flux of new glacier ice (mm H2O/s) [+ = ice grows] + qflx_glcice_melt_diag => col_wf%qflx_glcice_melt_diag , & ! Output: [real(r8) (:) ] ice melt (positive definite) (mm H2O/s) qflx_snomelt => col_wf%qflx_snomelt , & ! Output: [real(r8) (:) ] snow melt (mm H2O /s) eflx_snomelt => col_ef%eflx_snomelt , & ! Output: [real(r8) (:) ] snow melt heat flux (W/m**2) @@ -1393,6 +1395,7 @@ subroutine Phasechange_beta (bounds, num_nolakec, filter_nolakec, dhsdT, & qflx_snofrz_lyr(c,-nlevsno+1:0) = 0._r8 qflx_snofrz_col(c) = 0._r8 qflx_glcice_melt(c) = 0._r8 + qflx_glcice_melt_diag(c) = 0._r8 qflx_snow_melt(c) = 0._r8 end do @@ -1643,8 +1646,8 @@ subroutine Phasechange_beta (bounds, num_nolakec, filter_nolakec, dhsdT, & ! as computed in HydrologyDrainageMod.F90. l = col_pp%landunit(c) - if (lun_pp%itype(l)==istice_mec) then + if ( lun_pp%itype(l)==istice_mec) then if (j>=1 .and. h2osoi_liq(c,j) > 0._r8) then ! ice layer with meltwater ! melting corresponds to a negative ice flux qflx_glcice_melt(c) = qflx_glcice_melt(c) + h2osoi_liq(c,j)/dtime @@ -1656,6 +1659,16 @@ subroutine Phasechange_beta (bounds, num_nolakec, filter_nolakec, dhsdT, & endif ! liquid water is present endif ! istice_mec + ! for diagnostic QICE SMB output only - + ! these are to calculate SMB even without MECs + if ( lun_pp%itype(l)==istice) then + if (j>=1 .and. h2osoi_liq(c,j) > 0._r8) then ! ice layer with meltwater + ! melting corresponds to a negative ice flux + qflx_glcice_melt_diag(c) = qflx_glcice_melt_diag(c) + h2osoi_liq(c,j)/dtime + qflx_glcice_diag(c) = qflx_glcice_diag(c) - h2osoi_liq(c,j)/dtime + endif ! liquid water is present + endif ! istice_mec + end do ! end of column-loop enddo ! end of level-loop diff --git a/components/elm/src/cpl/lnd_comp_mct.F90 b/components/elm/src/cpl/lnd_comp_mct.F90 index e533d5bcbc2..8d3ae5e2997 100644 --- a/components/elm/src/cpl/lnd_comp_mct.F90 +++ b/components/elm/src/cpl/lnd_comp_mct.F90 @@ -18,7 +18,6 @@ module lnd_comp_mct #ifdef HAVE_MOAB use seq_comm_mct, only: mlnid! id of moab land app - use seq_comm_mct, only: mb_land_mesh! true if land is full mesh (on the river mesh) use seq_comm_mct, only: num_moab_exports #ifdef MOABCOMP use seq_comm_mct , only: seq_comm_compare_mb_mct @@ -51,7 +50,6 @@ module lnd_comp_mct integer :: mpicom_lnd_moab ! used also for mpi-reducing the difference between moab tags and mct avs integer :: rank2 - logical :: samegrid_al ! #endif !--------------------------------------------------------------------------- @@ -314,13 +312,6 @@ subroutine lnd_init_mct( EClock, cdata_l, x2l_l, l2x_l, NLFilename ) call lnd_domain_mct( bounds, lsz, gsMap_lnd, dom_l ) #ifdef HAVE_MOAB -! find out samegrid_al or not; from infodata - samegrid_al = .true. - call seq_infodata_GetData(infodata , & - atm_gnam=atm_gnam , & - lnd_gnam=lnd_gnam ) - if (trim(atm_gnam) /= trim(lnd_gnam)) samegrid_al = .false. - mb_land_mesh = .not. samegrid_al ! global variable, saved in seq_comm call init_moab_land(bounds, LNDID) #endif call mct_aVect_init(x2l_l, rList=seq_flds_x2l_fields, lsize=lsz) @@ -547,8 +538,7 @@ subroutine lnd_run_mct(EClock, cdata_l, x2l_l, l2x_l) ! loop over all fields in seq_flds_x2l_fields call mct_list_init(temp_list ,seq_flds_x2l_fields) size_list=mct_list_nitem (temp_list) - ent_type = 0 ! entity type is vertex for land, usually (bigrid case) - if (mb_land_mesh) ent_type = 1 + ent_type = 0 ! entity type is vertex for land, always if (rank2 .eq. 0) print *, num_moab_exports, trim(seq_flds_x2l_fields), ' lnd import check' do index_list = 1, size_list call mct_list_get(mctOStr,index_list,temp_list) @@ -846,7 +836,7 @@ subroutine init_moab_land(bounds, LNDID) use spmdmod , only: masterproc use iMOAB , only: iMOAB_CreateVertices, iMOAB_WriteMesh, iMOAB_RegisterApplication, & iMOAB_DefineTagStorage, iMOAB_SetIntTagStorage, iMOAB_SetDoubleTagStorage, & - iMOAB_ResolveSharedEntities, iMOAB_CreateElements, iMOAB_UpdateMeshInfo + iMOAB_ResolveSharedEntities, iMOAB_UpdateMeshInfo type(bounds_type) , intent(in) :: bounds integer , intent(in) :: LNDID ! id of the land app @@ -893,200 +883,97 @@ subroutine init_moab_land(bounds, LNDID) vgids(n) = ldecomp%gdc2glo(bounds%begg+n-1) ! local to global ! end do gsize = ldomain%ni * ldomain%nj ! size of the total grid - ! if ldomain%nv > 3 , create mesh - - ! Case where land and river share mesh (tri-grid) - if (ldomain%nv .ge. 3 .and. .not.samegrid_al) then - ! number of vertices is nv * lsz ! - allocate(moab_vert_coords(lsz*dims*ldomain%nv)) - ! loop over ldomain - allocate(moabconn(ldomain%nv * lsz)) - do n = bounds%begg, bounds%endg - i = (n - bounds%begg) * ldomain%nv - do iv = 1, ldomain%nv - lonv = ldomain%mblonv(n, iv) * SHR_CONST_PI/180. - latv = ldomain%mblatv(n, iv) * SHR_CONST_PI/180. - - i = i + 1 ! iv-th vertex of cell n; i starts at 1 - moab_vert_coords(3*i-2)=COS(latv)*COS(lonv) - moab_vert_coords(3*i-1)=COS(latv)*SIN(lonv) - moab_vert_coords(3*i )=SIN(latv) - moabconn(i) = i - enddo - enddo - ierr = iMOAB_CreateVertices(mlnid, lsz * 3 * ldomain%nv, dims, moab_vert_coords) - if (ierr > 0 ) & - call endrun('Error: fail to create MOAB vertices in land model') - - mbtype = 2 ! triangle - if (ldomain%nv .eq. 4) mbtype = 3 ! quad - if (ldomain%nv .gt. 4) mbtype = 4 ! polygon - block_ID = 100 !some value - ierr = iMOAB_CreateElements( mlnid, lsz, mbtype, ldomain%nv, moabconn, block_ID ); - - - ! define some useful tags on cells - tagtype = 0 ! dense, integer - numco = 1 - tagname='GLOBAL_ID'//C_NULL_CHAR - ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) - if (ierr > 0 ) & - call endrun('Error: fail to retrieve GLOBAL_ID tag ') - - ent_type = 1 ! element type - ierr = iMOAB_SetIntTagStorage ( mlnid, tagname, lsz , ent_type, vgids) - if (ierr > 0 ) & - call endrun('Error: fail to set GLOBAL_ID tag ') - - ! use moab_vert_coords as a data holder for a frac tag and area tag that we will create - ! on the vertices; do not allocate other data array - ! Define and Set Fraction - tagname='frac'//C_NULL_CHAR - tagtype = 1 ! dense, double - ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) - if (ierr > 0 ) & - call endrun('Error: fail to create frac tag ') - - do i = 1, lsz - n = i-1 + bounds%begg - moab_vert_coords(i) = ldomain%frac(n) - enddo - ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsz , ent_type, moab_vert_coords) - if (ierr > 0 ) & - call endrun('Error: fail to set frac tag ') - - ! Define and Set area - tagname='area'//C_NULL_CHAR - ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) - if (ierr > 0 ) & - call endrun('Error: fail to create area tag ') - do i = 1, lsz - n = i-1 + bounds%begg - moab_vert_coords(i) = ldomain%area(n)/(re*re) ! use the same doubles for second tag :) - enddo - - ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsz , ent_type, moab_vert_coords ) - if (ierr > 0 ) & - call endrun('Error: fail to set area tag ') - - ! Define aream - tagname='aream'//C_NULL_CHAR - ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) - if (ierr > 0 ) & - call endrun('Error: fail to create aream tag ') - - deallocate(moabconn) - deallocate(vgids) - - - ! Now do the verticies - allocate(vgids(lsz*ldomain%nv)) ! - do n = 1, lsz - do i=1,ldomain%nv - vgids( (n-1)*ldomain%nv+i ) = (ldecomp%gdc2glo(bounds%begg+n-1)-1)*ldomain%nv+i ! local to global ! - end do - end do - ent_type = 0 ! vertices now - tagname = 'GLOBAL_ID'//C_NULL_CHAR - ierr = iMOAB_SetIntTagStorage ( mlnid, tagname, lsz , ent_type, vgids ) - if (ierr > 0 ) & - call endrun('Error: fail to set global ID tag on vertices in land mesh ') - ierr = iMOAB_UpdateMeshInfo( mlnid ) - if (ierr > 0 ) & - call endrun('Error: fail to update mesh info ') - - ! Case where land and atmosphere share mesh - else ! old point cloud mesh - allocate(moab_vert_coords(lsz*dims)) - do i = 1, lsz - n = i-1 + bounds%begg - lonv = ldomain%lonc(n) *SHR_CONST_PI/180. - latv = ldomain%latc(n) *SHR_CONST_PI/180. - moab_vert_coords(3*i-2)=COS(latv)*COS(lonv) - moab_vert_coords(3*i-1)=COS(latv)*SIN(lonv) - moab_vert_coords(3*i )=SIN(latv) - enddo - ierr = iMOAB_CreateVertices(mlnid, lsz*3, dims, moab_vert_coords) - if (ierr > 0 ) & - call endrun('Error: fail to create MOAB vertices in land model') - - tagtype = 0 ! dense, integer - numco = 1 - tagname='GLOBAL_ID'//C_NULL_CHAR - ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) - if (ierr > 0 ) & - call endrun('Error: fail to retrieve GLOBAL_ID tag ') - - ent_type = 0 ! vertex type - ierr = iMOAB_SetIntTagStorage ( mlnid, tagname, lsz , ent_type, vgids) - if (ierr > 0 ) & - call endrun('Error: fail to set GLOBAL_ID tag ') - - ierr = iMOAB_ResolveSharedEntities( mlnid, lsz, vgids ); - if (ierr > 0 ) & - call endrun('Error: fail to resolve shared entities') - - !there are no shared entities, but we will set a special partition tag, in order to see the - ! partitions ; it will be visible with a Pseudocolor plot in VisIt - tagname='partition'//C_NULL_CHAR - ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) - if (ierr > 0 ) & - call endrun('Error: fail to create new partition tag ') - - vgids = iam - ierr = iMOAB_SetIntTagStorage ( mlnid, tagname, lsz , ent_type, vgids) - if (ierr > 0 ) & - call endrun('Error: fail to set partition tag ') - - ! use moab_vert_coords as a data holder for a frac tag and area tag that we will create - ! on the vertices; do not allocate other data array - tagname='frac'//C_NULL_CHAR - tagtype = 1 ! dense, double - ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) - if (ierr > 0 ) & - call endrun('Error: fail to create frac tag ') - - do i = 1, lsz - n = i-1 + bounds%begg - moab_vert_coords(i) = ldomain%frac(n) - enddo - ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsz , ent_type, moab_vert_coords) - if (ierr > 0 ) & - call endrun('Error: fail to set frac tag ') - - tagname='area'//C_NULL_CHAR - ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) - if (ierr > 0 ) & - call endrun('Error: fail to create area tag ') - do i = 1, lsz - n = i-1 + bounds%begg - moab_vert_coords(i) = ldomain%area(n)/(re*re) ! use the same doubles for second tag :) - enddo - - ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsz , ent_type, moab_vert_coords ) - if (ierr > 0 ) & - call endrun('Error: fail to set area tag ') - - ! aream needed in cime_init for now. - tagname='aream'//C_NULL_CHAR - ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) - if (ierr > 0 ) & - call endrun('Error: fail to create aream tag ') - ! ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsz , ent_type, moab_vert_coords ) - ! if (ierr > 0 ) & - ! call endrun('Error: fail to set aream tag ') - ierr = iMOAB_UpdateMeshInfo( mlnid ) - if (ierr > 0 ) & - call endrun('Error: fail to update mesh info ') - endif - ! add more domain fields that are missing from domain fields: lat, lon, mask, hgt - tagname = 'lat:lon:mask:hgt'//C_NULL_CHAR - tagtype = 1 ! dense, double + + allocate(moab_vert_coords(lsz*dims)) + do i = 1, lsz + n = i-1 + bounds%begg + lonv = ldomain%lonc(n) *SHR_CONST_PI/180. + latv = ldomain%latc(n) *SHR_CONST_PI/180. + moab_vert_coords(3*i-2)=COS(latv)*COS(lonv) + moab_vert_coords(3*i-1)=COS(latv)*SIN(lonv) + moab_vert_coords(3*i )=SIN(latv) + enddo + ierr = iMOAB_CreateVertices(mlnid, lsz*3, dims, moab_vert_coords) + if (ierr > 0 ) & + call endrun('Error: fail to create MOAB vertices in land model') + + tagtype = 0 ! dense, integer numco = 1 + tagname='GLOBAL_ID'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) & + call endrun('Error: fail to retrieve GLOBAL_ID tag ') + + ent_type = 0 ! vertex type + ierr = iMOAB_SetIntTagStorage ( mlnid, tagname, lsz , ent_type, vgids) + if (ierr > 0 ) & + call endrun('Error: fail to set GLOBAL_ID tag ') + + ierr = iMOAB_ResolveSharedEntities( mlnid, lsz, vgids ); + if (ierr > 0 ) & + call endrun('Error: fail to resolve shared entities') + + !there are no shared entities, but we will set a special partition tag, in order to see the + ! partitions ; it will be visible with a Pseudocolor plot in VisIt + tagname='partition'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) & + call endrun('Error: fail to create new partition tag ') + + vgids = iam + ierr = iMOAB_SetIntTagStorage ( mlnid, tagname, lsz , ent_type, vgids) + if (ierr > 0 ) & + call endrun('Error: fail to set partition tag ') + + ! use moab_vert_coords as a data holder for a frac tag and area tag that we will create + ! on the vertices; do not allocate other data array + tagname='frac'//C_NULL_CHAR + tagtype = 1 ! dense, double + ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) & + call endrun('Error: fail to create frac tag ') + + do i = 1, lsz + n = i-1 + bounds%begg + moab_vert_coords(i) = ldomain%frac(n) + enddo + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsz , ent_type, moab_vert_coords) + if (ierr > 0 ) & + call endrun('Error: fail to set frac tag ') + + tagname='area'//C_NULL_CHAR ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) if (ierr > 0 ) & - call endrun('Error: fail to create lat:lon:mask:hgt tags ') + call endrun('Error: fail to create area tag ') + do i = 1, lsz + n = i-1 + bounds%begg + moab_vert_coords(i) = ldomain%area(n)/(re*re) ! use the same doubles for second tag :) + enddo + + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsz , ent_type, moab_vert_coords ) + if (ierr > 0 ) & + call endrun('Error: fail to set area tag ') + + ! aream needed in cime_init for now. + tagname='aream'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) & + call endrun('Error: fail to create aream tag ') + ! ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsz , ent_type, moab_vert_coords ) + ! if (ierr > 0 ) & + ! call endrun('Error: fail to set aream tag ') + ierr = iMOAB_UpdateMeshInfo( mlnid ) + if (ierr > 0 ) & + call endrun('Error: fail to update mesh info ') + + ! add more domain fields that are missing from domain fields: lat, lon, mask, hgt + tagname = 'lat:lon:mask:hgt'//C_NULL_CHAR + tagtype = 1 ! dense, double + numco = 1 + ierr = iMOAB_DefineTagStorage(mlnid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) & + call endrun('Error: fail to create lat:lon:mask:hgt tags ') ! moab_vert_coords is big enough in both case to hold enough data for us: lat, lon, mask do i = 1, lsz @@ -1098,9 +985,7 @@ subroutine init_moab_land(bounds, LNDID) tagname = 'lat:lon:mask'//C_NULL_CHAR ent_type = 0 ! point cloud usually - if (ldomain%nv .ge. 3 .and. .not.samegrid_al) then - ent_type = 1 ! cell in tri-grid case - endif + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsz*3 , ent_type, moab_vert_coords) if (ierr > 0 ) & call endrun('Error: fail to set lat lon mask tag ') @@ -1255,11 +1140,7 @@ subroutine lnd_export_moab(EClock, bounds, lnd2atm_vars, lnd2glc_vars) end do tagname=trim(seq_flds_l2x_fields)//C_NULL_CHAR - if (samegrid_al) then - ent_type = 0 ! vertices, cells only if samegrid_al false - else - ent_type = 1 - endif + ent_type = 0 ! vertices only, from now on ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, totalmbls , ent_type, l2x_lm(1,1) ) if (ierr > 0 ) & call shr_sys_abort( sub//' Error: fail to set moab l2x '// trim(seq_flds_l2x_fields) ) @@ -1452,11 +1333,7 @@ subroutine lnd_import_moab(EClock, bounds, atm2lnd_vars, glc2lnd_vars) call endrun('Error: fail to write the moab lnd mesh before import ') #endif tagname=trim(seq_flds_x2l_fields)//C_NULL_CHAR - if (samegrid_al) then - ent_type = 0 ! vertices, cells only if samegrid_al false - else - ent_type = 1 - endif + ent_type = 0 ! vertices ierr = iMOAB_GetDoubleTagStorage ( mlnid, tagname, totalmblsimp , ent_type, x2l_lm(1,1) ) if ( ierr > 0) then call endrun('Error: fail to get seq_flds_x2l_fields for land moab instance on component') diff --git a/components/elm/src/data_types/ColumnDataType.F90 b/components/elm/src/data_types/ColumnDataType.F90 index d0a4a10cd3d..ba278d1fc46 100644 --- a/components/elm/src/data_types/ColumnDataType.F90 +++ b/components/elm/src/data_types/ColumnDataType.F90 @@ -502,6 +502,9 @@ module ColumnDataType real(r8), pointer :: qflx_glcice (:) => null() ! net flux of new glacial ice (growth - melt) (mm H2O/s), passed to GLC real(r8), pointer :: qflx_glcice_frz (:) => null() ! ice growth (positive definite) (mm H2O/s) real(r8), pointer :: qflx_glcice_melt (:) => null() ! ice melt (positive definite) (mm H2O/s) + real(r8), pointer :: qflx_glcice_diag (:) => null() ! net flux of new glacial ice (growth - melt) (mm H2O/s), passed to GLC + real(r8), pointer :: qflx_glcice_frz_diag (:) => null() ! ice growth (positive definite) (mm H2O/s) + real(r8), pointer :: qflx_glcice_melt_diag(:) => null() ! ice melt (positive definite) (mm H2O/s) real(r8), pointer :: qflx_drain_vr (:,:) => null() ! liquid water lost as drainage (m /time step) real(r8), pointer :: qflx_h2osfc2topsoi (:) => null() ! liquid water coming from surface standing water top soil (mm H2O/s) real(r8), pointer :: qflx_snow2topsoi (:) => null() ! liquid water coming from residual snow to topsoil (mm H2O/s) @@ -5725,6 +5728,9 @@ subroutine col_wf_init(this, begc, endc) allocate(this%qflx_glcice (begc:endc)) ; this%qflx_glcice (:) = spval allocate(this%qflx_glcice_frz (begc:endc)) ; this%qflx_glcice_frz (:) = spval allocate(this%qflx_glcice_melt (begc:endc)) ; this%qflx_glcice_melt (:) = spval + allocate(this%qflx_glcice_diag (begc:endc)) ; this%qflx_glcice_diag (:) = spval + allocate(this%qflx_glcice_frz_diag (begc:endc)) ; this%qflx_glcice_frz_diag (:) = spval + allocate(this%qflx_glcice_melt_diag (begc:endc)) ; this%qflx_glcice_melt_diag(:) = spval allocate(this%qflx_drain_vr (begc:endc,1:nlevgrnd)) ; this%qflx_drain_vr (:,:) = spval allocate(this%qflx_h2osfc2topsoi (begc:endc)) ; this%qflx_h2osfc2topsoi (:) = spval allocate(this%qflx_snow2topsoi (begc:endc)) ; this%qflx_snow2topsoi (:) = spval @@ -5842,23 +5848,39 @@ subroutine col_wf_init(this, begc, endc) call hist_addfld1d (fname='QSNOFRZ', units='kg/m2/s', & avgflag='A', long_name='column-integrated snow freezing rate', & ptr_col=this%qflx_snofrz, set_lake=spval, c2l_scale_type='urbanf', default='inactive') - + if (create_glacier_mec_landunit) then - this%qflx_glcice(begc:endc) = spval - call hist_addfld1d (fname='QICE', units='mm/s', & - avgflag='A', long_name='ice growth/melt', & - ptr_col=this%qflx_glcice, l2g_scale_type='ice') - - this%qflx_glcice_frz(begc:endc) = spval - call hist_addfld1d (fname='QICE_FRZ', units='mm/s', & - avgflag='A', long_name='ice growth', & - ptr_col=this%qflx_glcice_frz, l2g_scale_type='ice') - - this%qflx_glcice_melt(begc:endc) = spval - call hist_addfld1d (fname='QICE_MELT', units='mm/s', & - avgflag='A', long_name='ice melt', & - ptr_col=this%qflx_glcice_melt, l2g_scale_type='ice') - endif + this%qflx_glcice(begc:endc) = spval + call hist_addfld1d (fname='QICE', units='mm/s', & + avgflag='A', long_name='ice growth/melt (with active GLC/MECs)', & + ptr_col=this%qflx_glcice, l2g_scale_type='ice') + + this%qflx_glcice_frz(begc:endc) = spval + call hist_addfld1d (fname='QICE_FRZ', units='mm/s', & + avgflag='A', long_name='ice growth (with active GLC/MECs)', & + ptr_col=this%qflx_glcice_frz, l2g_scale_type='ice') + + this%qflx_glcice_melt(begc:endc) = spval + call hist_addfld1d (fname='QICE_MELT', units='mm/s', & + avgflag='A', long_name='ice melt (with active GLC/MECs)', & + ptr_col=this%qflx_glcice_melt, l2g_scale_type='ice') + else + this%qflx_glcice_diag(begc:endc) = spval + call hist_addfld1d (fname='QICE', units='mm/s', & + avgflag='A', long_name='diagnostic ice growth/melt (no active GLC/MECs)', & + ptr_col=this%qflx_glcice_diag, l2g_scale_type='ice') + + this%qflx_glcice_frz_diag(begc:endc) = spval + call hist_addfld1d (fname='QICE_FRZ', units='mm/s', & + avgflag='A', long_name='diagnostic ice growth (no active GLC/MECs)', & + ptr_col=this%qflx_glcice_frz_diag, l2g_scale_type='ice') + + this%qflx_glcice_melt_diag(begc:endc) = spval + call hist_addfld1d (fname='QICE_MELT', units='mm/s', & + avgflag='A', long_name='diagnostic ice melt (no active GLC/MECs)', & + ptr_col=this%qflx_glcice_melt_diag, l2g_scale_type='ice') + end if + ! As defined here, snow_sources - snow_sinks will equal the change in h2osno at any ! given time step but only if there is at least one snow layer (for all landunits diff --git a/components/elm/src/external_models/fates b/components/elm/src/external_models/fates index 825579d0b40..e3e7d2cd86a 160000 --- a/components/elm/src/external_models/fates +++ b/components/elm/src/external_models/fates @@ -1 +1 @@ -Subproject commit 825579d0b406fe99344591b5ed8356e5c7aeebec +Subproject commit e3e7d2cd86a66f8ca0e8f6dc4a823246a2bdb95b diff --git a/components/elm/src/external_models/sbetr b/components/elm/src/external_models/sbetr index 66260f4991d..08d8a8184a6 160000 --- a/components/elm/src/external_models/sbetr +++ b/components/elm/src/external_models/sbetr @@ -1 +1 @@ -Subproject commit 66260f4991d61439d4cba92eb633590b09f97920 +Subproject commit 08d8a8184a605a23d4dce4f91a33d8f2bba29ae9 diff --git a/components/elm/src/main/elm_driver.F90 b/components/elm/src/main/elm_driver.F90 index 51f08bf235c..18ecae88e37 100644 --- a/components/elm/src/main/elm_driver.F90 +++ b/components/elm/src/main/elm_driver.F90 @@ -1604,6 +1604,8 @@ subroutine elm_drv_init(bounds, & qflx_glcice => col_wf%qflx_glcice , & ! Output: [real(r8) (:) ] flux of new glacier ice (mm H2O/s) [+ = ice grows] + qflx_glcice_diag => col_wf%qflx_glcice_diag , & ! Output: [real(r8) (:) ] flux of new glacier ice (mm H2O/s) [+ = ice grows] + eflx_bot => col_ef%eflx_bot , & ! Output: [real(r8) (:) ] heat flux from beneath soil/ice column (W/m**2) cisun_z => photosyns_vars%cisun_z_patch , & ! Output: [real(r8) (:) ] intracellular sunlit leaf CO2 (Pa) @@ -1637,6 +1639,7 @@ subroutine elm_drv_init(bounds, & ! Initialize qflx_glcice everywhere, to zero. qflx_glcice(c) = 0._r8 + qflx_glcice_diag(c) = 0._r8 end do diff --git a/components/elm/src/main/elm_varctl.F90 b/components/elm/src/main/elm_varctl.F90 index fb9ca0dbd24..69a3209e52f 100644 --- a/components/elm/src/main/elm_varctl.F90 +++ b/components/elm/src/main/elm_varctl.F90 @@ -221,7 +221,7 @@ module elm_varctl logical, public :: use_fates = .false. ! true => use ED integer, public :: fates_spitfire_mode = 0 ! 0 for no fire; 1 for constant ignitions - character(len=13), public :: fates_harvest_mode = '' ! five different harvest modes; see namelist_definitions + character(len=256), public :: fates_harvest_mode = '' ! five different harvest modes; see namelist_definitions logical, public :: use_fates_fixed_biogeog = .false. ! true => use fixed biogeography mode logical, public :: use_fates_planthydro = .false. ! true => turn on fates hydro logical, public :: use_fates_cohort_age_tracking = .false. ! true => turn on cohort age tracking diff --git a/components/elm/src/main/elmfates_interfaceMod.F90 b/components/elm/src/main/elmfates_interfaceMod.F90 index a20444d6758..e7134a580fd 100644 --- a/components/elm/src/main/elmfates_interfaceMod.F90 +++ b/components/elm/src/main/elmfates_interfaceMod.F90 @@ -3517,7 +3517,7 @@ subroutine hlm_bounds_to_fates_bounds(hlm, fates) use FatesInterfaceTypesMod, only : nlevage_fates => nlevage use FatesInterfaceTypesMod, only : nlevheight_fates => nlevheight use FatesInterfaceTypesMod, only : nlevdamage_fates => nlevdamage - use FatesLitterMod, only : nfsc_fates => nfsc + use FatesFuelClassesMod, only : nfc_fates => num_fuel_classes use FatesLitterMod, only : ncwd_fates => ncwd use EDParamsMod, only : nlevleaf_fates => nlevleaf use EDParamsMod, only : nclmax_fates => nclmax @@ -3555,7 +3555,7 @@ subroutine hlm_bounds_to_fates_bounds(hlm, fates) fates%sizeage_class_end = nlevsclass_fates * nlevage_fates fates%fuel_begin = 1 - fates%fuel_end = nfsc_fates + fates%fuel_end = nfc_fates fates%cdpf_begin = 1 fates%cdpf_end = nlevdamage_fates * numpft_fates * nlevsclass_fates @@ -3606,7 +3606,7 @@ subroutine hlm_bounds_to_fates_bounds(hlm, fates) fates%coage_class_end = nlevcoage fates%agefuel_begin = 1 - fates%agefuel_end = nlevage_fates * nfsc_fates + fates%agefuel_end = nlevage_fates * nfc_fates fates%landuse_begin = 1 fates%landuse_end = n_landuse_cats diff --git a/components/elm/src/main/histFileMod.F90 b/components/elm/src/main/histFileMod.F90 index 1b86c3a1618..8de3f27c30e 100644 --- a/components/elm/src/main/histFileMod.F90 +++ b/components/elm/src/main/histFileMod.F90 @@ -28,7 +28,7 @@ module histFileMod use FatesInterfaceTypesMod , only : nlevheight_fates => nlevheight use FatesInterfaceTypesMod , only : nlevdamage_fates => nlevdamage use FatesInterfaceTypesMod , only : nlevcoage - use FatesLitterMod , only : nfsc_fates => nfsc + use FatesFuelClassesMod , only : nfc_fates => num_fuel_classes use FatesConstantsMod , only : n_landuse_cats use FatesLitterMod , only : ncwd_fates => ncwd use FatesInterfaceTypesMod , only : numpft_fates => numpft @@ -1933,7 +1933,7 @@ subroutine htape_create (t, histrest) call ncd_defdim(lnfid, 'fates_levcacls',nlevcoage, dimid) call ncd_defdim(lnfid, 'fates_levpft', numpft_fates, dimid) call ncd_defdim(lnfid, 'fates_levage', nlevage_fates, dimid) - call ncd_defdim(lnfid, 'fates_levfuel', nfsc_fates, dimid) + call ncd_defdim(lnfid, 'fates_levfuel', nfc_fates, dimid) call ncd_defdim(lnfid, 'fates_levcwdsc', ncwd_fates, dimid) call ncd_defdim(lnfid, 'fates_levscpf', nlevsclass_fates*numpft_fates, dimid) call ncd_defdim(lnfid, 'fates_levcapf', nlevcoage*numpft_fates, dimid) @@ -1951,7 +1951,7 @@ subroutine htape_create (t, histrest) call ncd_defdim(lnfid, 'fates_levelpft', nelements_fates * numpft_fates, dimid) call ncd_defdim(lnfid, 'fates_levelcwd', nelements_fates * ncwd_fates, dimid) call ncd_defdim(lnfid, 'fates_levelage', nelements_fates * nlevage_fates, dimid) - call ncd_defdim(lnfid, 'fates_levagefuel', nlevage_fates * nfsc_fates, dimid) + call ncd_defdim(lnfid, 'fates_levagefuel', nlevage_fates * nfc_fates, dimid) call ncd_defdim(lnfid, 'fates_levlanduse', n_landuse_cats, dimid) call ncd_defdim(lnfid, 'fates_levlulu', n_landuse_cats * n_landuse_cats, dimid) end if @@ -4796,7 +4796,7 @@ subroutine hist_addfld2d (fname, type2d, units, avgflag, long_name, type1d_out, case ('fates_levelage') num2d = nelements_fates*nlevage_fates case ('fates_levagefuel') - num2d = nlevage_fates*nfsc_fates + num2d = nlevage_fates*nfc_fates case('cft') if (cft_size > 0) then num2d = cft_size @@ -4842,7 +4842,7 @@ subroutine hist_addfld2d (fname, type2d, units, avgflag, long_name, type1d_out, case ('fates_levage') num2d = nlevage_fates case ('fates_levfuel') - num2d = nfsc_fates + num2d = nfc_fates case ('fates_levcwdsc') num2d = ncwd_fates case ('fates_levscpf') diff --git a/components/elm/src/main/surfrdMod.F90 b/components/elm/src/main/surfrdMod.F90 index a8146e5a0f1..97a4fae62d4 100755 --- a/components/elm/src/main/surfrdMod.F90 +++ b/components/elm/src/main/surfrdMod.F90 @@ -20,11 +20,6 @@ module surfrdMod use ncdio_pio , only : ncd_io, check_var, ncd_inqfdims, check_dim, ncd_inqdid, ncd_inqdlen use pio -#ifdef HAVE_MOAB - use mct_mod , only : mct_gsMap - use decompMod , only : get_elmlevel_gsmap - ! use spmdMod , only : iam ! rank on the land communicator -#endif use spmdMod use topounit_varcon , only : max_topounits, has_topounit @@ -184,11 +179,6 @@ subroutine surfrd_get_grid(begg, endg, ldomain, filename, glcfilename) ! pflotran:beg----------------------------- integer :: j, np, nv -#ifdef HAVE_MOAB - type(mct_gsMap), pointer :: gsMap - integer :: i, iv , iseg, ig, local ! ni, nj, nv, nseg, global ig - -#endif ! pflotran:end----------------------------- character(len=32) :: subname = 'surfrd_get_grid' ! subroutine name @@ -258,59 +248,6 @@ subroutine surfrd_get_grid(begg, endg, ldomain, filename, glcfilename) end if ! pflotran:end----------------------------------------------- - -#ifdef HAVE_MOAB - ! read xv and yv for MOAB to learn mesh verticies - if (ldomain%nv>=3 ) then - call get_elmlevel_gsmap (grlnd, gsMap) - allocate(rdata3d(nv,ni,nj)) ! transpose from c, as this is fortran - vname = 'xv' - ! this should be improved in a distributed read, that does not use full grid ni * nj * nv 720*360*4*8 ~ 8Mb - call ncd_io(ncid=ncid, varname=trim(vname), data=rdata3d, flag='read', readvar=readvar) - if (.not. readvar) call endrun( msg=trim(subname)//' ERROR: xv NOT on file'//errMsg(__FILE__, __LINE__)) - ! fill up the ldomain%mblonv(begg:endg, 1:nv) array - local = begg - do iseg = 1, gsMap%ngseg - if (gsMap%pe_loc(iseg) .eq. iam) then - do ig = gsMap%start(iseg), gsMap%start(iseg) + gsMap%length(iseg) - 1 - j = (ig-1)/ni + 1 - i = ig - ni*(j-1) - do iv = 1, nv - if (local .le. endg) then - ldomain%mblonv(local, iv ) = rdata3d(iv, i, j) - else - write (iulog, *), 'OVERFLOW', iseg, gsMap%pe_loc(iseg), gsMap%start(iseg), gsMap%length(iseg), local - endif - enddo - local = local + 1 - enddo - endif - enddo - ! repeat for mblatv - vname = 'yv' - call ncd_io(ncid=ncid, varname=trim(vname), data=rdata3d, flag='read', readvar=readvar) - if (.not. readvar) call endrun( msg=trim(subname)//' ERROR: yv NOT on file'//errMsg(__FILE__, __LINE__)) - ! fill up the ldomain%lonv(begg:endg, 1:nv) array - local = begg - do iseg = 1, gsMap%ngseg - if (gsMap%pe_loc(iseg) .eq. iam) then - do ig = gsMap%start(iseg), gsMap%start(iseg) + gsMap%length(iseg) - 1 - j = (ig-1)/ni + 1 - i = ig - ni*(j-1) - do iv = 1, nv - if (local .le. endg) then - ldomain%mblatv(local, iv ) = rdata3d(iv, i, j) - endif - enddo - local = local + 1 - enddo - endif - enddo - ! deallocate what is not needed anymore (for half degree land model, ~8Mb) - deallocate(rdata3d) - - end if -#endif else call ncd_io(ncid=ncid, varname= 'AREA', flag='read', data=ldomain%area, & dim1name=grlnd, readvar=readvar) diff --git a/components/elm/src/utils/domainMod.F90 b/components/elm/src/utils/domainMod.F90 index 5ef3ae611cf..2c7771179d2 100755 --- a/components/elm/src/utils/domainMod.F90 +++ b/components/elm/src/utils/domainMod.F90 @@ -52,10 +52,6 @@ module domainMod integer :: nv ! number of vertices real(r8),pointer :: latv(:,:) ! latitude of grid cell's vertices (deg) real(r8),pointer :: lonv(:,:) ! longitude of grid cell's vertices (deg) -#ifdef HAVE_MOAB - real(r8),pointer :: mblatv(:,:) ! latitude of grid cell's vertices (deg) for MOAB - real(r8),pointer :: mblonv(:,:) ! longitude of grid cell's vertices (deg) for MOAB -#endif real(r8) :: lon0 ! the origin lon/lat (Most western/southern corner, if not globally covered grids; OR -180W(360E)/-90N) real(r8) :: lat0 ! the origin lon/lat (Most western/southern corner, if not globally covered grids; OR -180W(360E)/-90N) @@ -154,22 +150,6 @@ subroutine domain_init(domain,isgrid2d,ni,nj,nbeg,nend,elmlevel) endif end if ! pflotran:end----------------------------------------------------- -#ifdef HAVE_MOAB - if (domain%nv > 0 .and. domain%nv /= huge(1)) then - if(.not.associated(domain%mblonv)) then - allocate(domain%mblonv(nb:ne, 1:domain%nv), stat=ier) - if (ier /= 0) & - call shr_sys_abort('domain_init ERROR: allocate mblonv ') - domain%mblonv = nan - endif - if(.not.associated(domain%mblatv)) then - allocate(domain%mblatv(nb:ne, 1:domain%nv)) - if (ier /= 0) & - call shr_sys_abort('domain_init ERROR: allocate mblatv ') - domain%mblatv = nan - endif - end if -#endif if (present(elmlevel)) then domain%elmlevel = elmlevel @@ -265,23 +245,6 @@ subroutine domain_clean(domain) endif endif ! pflotran:beg----------------------------------------------------- -#ifdef HAVE_MOAB - if (domain%nv > 0 .and. domain%nv /= huge(1)) then - if (associated(domain%mblonv)) then - deallocate(domain%mblonv, stat=ier) - if (ier /= 0) & - call shr_sys_abort('domain_clean ERROR: deallocate mblonv ') - nullify(domain%mblonv) - endif - - if (associated(domain%mblatv)) then - deallocate(domain%mblatv, stat=ier) - if (ier /= 0) & - call shr_sys_abort('domain_clean ERROR: deallocate mblatv ') - nullify(domain%mblatv) - endif - endif -#endif else if (masterproc) then diff --git a/components/homme/CMakeLists.txt b/components/homme/CMakeLists.txt index ec291d8bd26..4bbfb3d73ac 100644 --- a/components/homme/CMakeLists.txt +++ b/components/homme/CMakeLists.txt @@ -321,7 +321,10 @@ IF (HOMME_USE_KOKKOS) IF (CUDA_BUILD OR HIP_BUILD OR SYCL_BUILD) SET (DEFAULT_VECTOR_SIZE 1) SET (HOMMEXX_ENABLE_GPU TRUE) - SET (HOMMEXX_ENABLE_GPU_F90 TRUE) + SET (HOMMEXX_ENABLE_GPU_F90 TRUE) + IF (SYCL_BUILD) + SET (DISABLE_TIMERS_IN_FIRST_STEP TRUE) + ENDIF() ELSE () SET (DEFAULT_VECTOR_SIZE 8) ENDIF() diff --git a/components/homme/cmake/machineFiles/perlmutter-gnu.cmake b/components/homme/cmake/machineFiles/perlmutter-gnu.cmake index a9ad558677a..a27f83900c1 100644 --- a/components/homme/cmake/machineFiles/perlmutter-gnu.cmake +++ b/components/homme/cmake/machineFiles/perlmutter-gnu.cmake @@ -13,7 +13,7 @@ SET(HDF5_DIR $ENV{CRAY_HDF5_PARALLEL_PREFIX} CACHE FILEPATH "") SET (NetCDF_C_PATH $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") SET (NetCDF_Fortran_PATH $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") -SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") +SET(BUILD_HOMME_WITHOUT_PIOLIBRARY FALSE CACHE BOOL "") SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") @@ -31,6 +31,7 @@ SET(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "") SET(Kokkos_ENABLE_CUDA ON CACHE BOOL "") SET(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "") SET(Kokkos_ARCH_AMPERE80 ON CACHE BOOL "") +SET(Kokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC OFF CACHE BOOL "") #SET(Kokkos_ARCH_ZEN2 ON CACHE BOOL "") # works, and perf same if both AMPERE80 and ZEN2 are on #SET(Kokkos_ENABLE_CUDA_UVM ON CACHE BOOL "") SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "") @@ -42,7 +43,10 @@ SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "") SET(CMAKE_C_COMPILER "cc" CACHE STRING "") SET(CMAKE_Fortran_COMPILER "ftn" CACHE STRING "") SET(CMAKE_CXX_COMPILER "CC" CACHE STRING "") -# Note: need to set MPICH_CXX env variable and perhaps NVCC_WRAPPER_DEFAULT_COMPILER +# Note: No longer need to set MPICH_CXX env variable and perhaps +# NVCC_WRAPPER_DEFAULT_COMPILER. Ignore the warning about nvcc_wrapper during +# configuration. +SET(CUDA_BUILD TRUE CACHE STRING "") SET(CXXLIB_SUPPORTED_CACHE FALSE CACHE BOOL "") diff --git a/components/homme/src/preqx/prim_advection_mod.F90 b/components/homme/src/preqx/prim_advection_mod.F90 index 8e7fd8b0cfb..ecbc745c730 100644 --- a/components/homme/src/preqx/prim_advection_mod.F90 +++ b/components/homme/src/preqx/prim_advection_mod.F90 @@ -6,14 +6,13 @@ module prim_advection_mod use dimensions_mod, only : nlev, qsize, nelemd use kinds, only : real_kind - use parallel_mod, only : parallel_t + use parallel_mod, only : parallel_t, abortmp use derivative_mod, only : derivative_t use element_mod, only : element_t use hybvcoord_mod, only : hvcoord_t use time_mod, only : TimeLevel_t use hybrid_mod, only : hybrid_t use control_mod, only : transport_alg - use sl_advection, only : prim_advec_tracers_remap_ALE, sl_init1 use prim_advection_base, only: prim_advec_init1_rk2, prim_advec_tracers_remap_rk2,& prim_advec_init2 @@ -35,12 +34,20 @@ subroutine Prim_Advec_Init1(par, elem) type (element_t) :: elem(:) call prim_advec_init1_rk2(par, elem) - call sl_init1(par,elem) - end subroutine Prim_Advec_Init1 + subroutine Prim_Advec_Tracers_observe_velocity(elem, tl, n, nets, nete) + type (element_t) , intent(inout) :: elem(:) + type (TimeLevel_t) , intent(in ) :: tl + integer , intent(in ) :: n + integer , intent(in ) :: nets + integer , intent(in ) :: nete + + ! Do nothing. Only SL transport uses this routine, and it's not supported in + ! preqx. + end subroutine Prim_Advec_Tracers_observe_velocity - subroutine Prim_Advec_Tracers_remap( elem , deriv , hvcoord , hybrid , dt , tl , nets , nete ) + subroutine Prim_Advec_Tracers_remap( elem , deriv , hvcoord , hybrid , dt , tl , nets , nete ) implicit none type (element_t) , intent(inout) :: elem(:) type (derivative_t) , intent(in ) :: deriv @@ -54,8 +61,8 @@ subroutine Prim_Advec_Tracers_remap( elem , deriv , hvcoord , hybrid , dt , tl if (transport_alg == 0) then call Prim_Advec_Tracers_remap_rk2( elem , deriv , hvcoord , hybrid , dt , tl , nets , nete ) - else - call Prim_Advec_Tracers_remap_ALE( elem , deriv , hvcoord , hybrid , dt , tl , nets , nete ) + else + call abortmp('Semi-Lagrangian transport is not supported in preqx.') end if end subroutine Prim_Advec_Tracers_remap diff --git a/components/homme/src/preqx_acc/prim_advection_mod.F90 b/components/homme/src/preqx_acc/prim_advection_mod.F90 index dbf055eb920..694c8440530 100644 --- a/components/homme/src/preqx_acc/prim_advection_mod.F90 +++ b/components/homme/src/preqx_acc/prim_advection_mod.F90 @@ -41,6 +41,7 @@ module prim_advection_mod logical, private :: first_time = .true. public :: Prim_Advec_Tracers_remap + public :: Prim_Advec_Tracers_observe_velocity public :: prim_advec_init1 public :: prim_advec_init2 @@ -302,6 +303,17 @@ subroutine prim_advec_init2(elem,hvcoord,hybrid) !$omp barrier end subroutine prim_advec_init2 + subroutine Prim_Advec_Tracers_observe_velocity(elem, tl, n, nets, nete) + type (element_t) , intent(inout) :: elem(:) + type (TimeLevel_t) , intent(in ) :: tl + integer , intent(in ) :: n + integer , intent(in ) :: nets + integer , intent(in ) :: nete + + ! Do nothing. Only SL transport uses this routine, and it's not supported in + ! preqx. + end subroutine Prim_Advec_Tracers_observe_velocity + subroutine advance_hypervis_scalar( elem , hvcoord , hybrid , deriv , nt , nt_qdp , nets , nete , dt2 ) ! hyperviscsoity operator for foward-in-time scheme ! take one timestep of: diff --git a/components/homme/src/preqx_kokkos/CMakeLists.txt b/components/homme/src/preqx_kokkos/CMakeLists.txt index dff42bb97c6..53691c9f2da 100644 --- a/components/homme/src/preqx_kokkos/CMakeLists.txt +++ b/components/homme/src/preqx_kokkos/CMakeLists.txt @@ -115,7 +115,7 @@ MACRO(PREQX_KOKKOS_SETUP) ${TEST_SRC_DIR}/dcmip12_wrapper.F90 ${TEST_SRC_DIR}/dcmip16_wrapper.F90 ${TEST_SRC_DIR}/dcmip2012_test1_2_3.F90 - ${TEST_SRC_DIR}/dcmip2012_test1_conv.F90 + ${TEST_SRC_DIR}/dcmip2012_test1_conv_mod.F90 ${TEST_SRC_DIR}/dcmip2012_test4.F90 ${TEST_SRC_DIR}/dcmip2012_test5.F90 ${TEST_SRC_DIR}/dcmip2016-baroclinic.F90 diff --git a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp index aa21f5c16f2..b433a48c2ab 100644 --- a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp +++ b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp @@ -91,7 +91,6 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, params.hypervis_scaling = hypervis_scaling; params.disable_diagnostics = disable_diagnostics; params.use_moisture = use_moisture; - params.moisture = params.use_moisture ? MoistDry::MOIST : MoistDry::DRY; //todo-repo-unification params.use_cpstar = use_cpstar; params.transport_alg = transport_alg; // SphereOperators parameters; preqx supports only the sphere. diff --git a/components/homme/src/preqx_kokkos/prim_advection_mod.F90 b/components/homme/src/preqx_kokkos/prim_advection_mod.F90 index b3d5595b874..07895e67e8e 100644 --- a/components/homme/src/preqx_kokkos/prim_advection_mod.F90 +++ b/components/homme/src/preqx_kokkos/prim_advection_mod.F90 @@ -39,6 +39,16 @@ subroutine Prim_Advec_Init1(par, elem) end subroutine Prim_Advec_Init1 + subroutine Prim_Advec_Tracers_observe_velocity(elem, tl, n, nets, nete) + type (element_t) , intent(inout) :: elem(:) + type (TimeLevel_t) , intent(in ) :: tl + integer , intent(in ) :: n + integer , intent(in ) :: nets + integer , intent(in ) :: nete + + ! Do nothing. Only SL transport uses this routine, and it's not supported in + ! preqx. + end subroutine Prim_Advec_Tracers_observe_velocity subroutine Prim_Advec_Tracers_remap( elem , deriv , hvcoord , hybrid , dt , tl , nets , nete ) implicit none diff --git a/components/homme/src/prim_main.F90 b/components/homme/src/prim_main.F90 index d6901151d36..fcd8a847302 100644 --- a/components/homme/src/prim_main.F90 +++ b/components/homme/src/prim_main.F90 @@ -68,6 +68,7 @@ end subroutine finalize_kokkos_f90 character (len=20) :: numtrac_char logical :: dir_e ! boolean existence of directory where output netcdf goes + logical :: call_enablef ! ===================================================== ! Begin executable code set distributed memory world... @@ -228,7 +229,20 @@ end subroutine finalize_kokkos_f90 if(par%masterproc) print *,"Entering main timestepping loop" call t_startf('prim_main_loop') + call_enablef = .false. do while(tl%nstep < nEndStep) +#ifdef DISABLE_TIMERS_IN_FIRST_STEP + ! Certain compilers, e.g., for Intel GPU, do just-in-time compilation. Turn + ! off timers in the first step to avoid counting that cost. + if (tl%nstep == 0) then + call t_disablef() + call_enablef = .true. + elseif (call_enablef) then + call t_enablef() + call_enablef = .false. + end if +#endif + #if (defined HORIZ_OPENMP) !$OMP PARALLEL NUM_THREADS(hthreads), DEFAULT(SHARED), PRIVATE(ithr,nets,nete,hybrid) call omp_set_num_threads(vthreads) diff --git a/components/homme/src/share/compose/CMakeLists.txt b/components/homme/src/share/compose/CMakeLists.txt index a052dcc3032..6bec15c08a0 100644 --- a/components/homme/src/share/compose/CMakeLists.txt +++ b/components/homme/src/share/compose/CMakeLists.txt @@ -30,12 +30,14 @@ add_library (${COMPOSE_LIBRARY} compose_slmm_islmpi_pack.cpp compose_slmm_islmpi_q.cpp compose_slmm_islmpi_qextrema.cpp + compose_slmm_islmpi_interpolate.cpp compose_slmm_islmpi_step.cpp compose_cedr_sl_run_global.cpp compose_cedr_sl_run_local.cpp compose_cedr_sl_run_check.cpp compose_cedr_qlt.cpp compose_cedr_caas.cpp + compose_slmm_islmpi_calc_trajectory.cpp cedr_util.cpp cedr_mpi.cpp cedr_local.cpp diff --git a/components/homme/src/share/compose/compose_cedr.cpp b/components/homme/src/share/compose/compose_cedr.cpp index 0aeaebf9487..9a8648d86da 100644 --- a/components/homme/src/share/compose/compose_cedr.cpp +++ b/components/homme/src/share/compose/compose_cedr.cpp @@ -419,12 +419,12 @@ struct TreeReducer : }; template -CDR::CDR (Int cdr_alg_, Int ngblcell_, Int nlclcell_, Int nlev_, Int qsize_, - bool use_sgi, bool independent_time_steps, const bool hard_zero_, - const Int* gid_data, const Int* rank_data, +CDR::CDR (Int cdr_alg_, Int ngblcell_, Int nlclcell_, Int nlev_, Int np_, + Int qsize_, bool use_sgi, bool independent_time_steps, + const bool hard_zero_, const Int* gid_data, const Int* rank_data, const cedr::mpi::Parallel::Ptr& p_, Int fcomm) : alg(Alg::convert(cdr_alg_)), - ncell(ngblcell_), nlclcell(nlclcell_), nlev(nlev_), qsize(qsize_), + ncell(ngblcell_), nlclcell(nlclcell_), nlev(nlev_), np(np_), qsize(qsize_), nsublev(Alg::is_suplev(alg) ? nsublev_per_suplev : 1), nsuplev((nlev + nsublev - 1) / nsublev), threed(independent_time_steps), @@ -444,8 +444,9 @@ CDR::CDR (Int cdr_alg_, Int ngblcell_, Int nlclcell_, Int nlev_, Int qsize_, cdr = std::make_shared(p, nleaf, tree, options, threed ? nsuplev : 0); tree = nullptr; } else if (Alg::is_caas(alg)) { - const Int n_accum_in_place = n_id_in_suplev*(cdr_over_super_levels ? - nsuplev : 1); + const Int n_accum_in_place = (n_id_in_suplev* + (Alg::is_point(alg) ? np*np : 1)* + (cdr_over_super_levels ? nsuplev : 1)); typename CAAST::UserAllReducer::Ptr reducer; //todo Measure perf on CPU and GPU of TreeReducer vs // ReproSumReducer. For now, I'll continue to use ReproSumReducer. @@ -458,7 +459,8 @@ CDR::CDR (Int cdr_alg_, Int ngblcell_, Int nlclcell_, Int nlev_, Int qsize_, } else { reducer = std::make_shared >(fcomm, n_accum_in_place); } - const auto caas = std::make_shared(p, nlclcell*n_accum_in_place, reducer); + const auto caas = std::make_shared(p, nlclcell*n_accum_in_place, + reducer); cdr = caas; } else { cedr_throw_if(true, "Invalid semi_lagrange_cdr_alg " << alg); @@ -502,11 +504,13 @@ void set_ie2gci (CDR& q, const Int ie, const Int gci) { q.ie2gci_h[ie] = gci template void init_ie2lci (CDR& q) { + const Int n_in_elem = Alg::is_point(q.alg) ? q.np*q.np : 1; const Int n_id_in_suplev = q.caas_in_suplev ? 1 : q.nsublev; const Int nleaf = n_id_in_suplev* q.ie2gci.size()* - (q.cdr_over_super_levels ? q.nsuplev : 1); + (q.cdr_over_super_levels ? q.nsuplev : 1)* + n_in_elem; q.ie2lci = typename CDR::Idxs("ie2lci", nleaf); q.ie2lci_h = Kokkos::create_mirror_view(q.ie2lci); if (Alg::is_qlt(q.alg)) { @@ -516,9 +520,9 @@ void init_ie2lci (CDR& q) { for (size_t ie = 0; ie < q.ie2gci_h.size(); ++ie) for (Int spli = 0; spli < q.nsuplev; ++spli) for (Int sbli = 0; sbli < n_id_in_suplev; ++sbli) - // local indexing is fastest over the whole column + // Local indexing is fastest over the whole column ... q.ie2lci_h[nlevwrem*ie + n_id_in_suplev*spli + sbli] = - // but global indexing is organized according to the tree + // ... but global indexing is organized according to the tree. qlt->gci2lci(n_id_in_suplev*(q.ncell*spli + q.ie2gci_h[ie]) + sbli); } else { for (size_t ie = 0; ie < q.ie2gci_h.size(); ++ie) @@ -531,16 +535,18 @@ void init_ie2lci (CDR& q) { const auto nlevwrem = q.nsuplev*n_id_in_suplev; for (size_t ie = 0; ie < q.ie2gci_h.size(); ++ie) for (Int spli = 0; spli < q.nsuplev; ++spli) - for (Int sbli = 0; sbli < n_id_in_suplev; ++sbli) { - const Int id = nlevwrem*ie + n_id_in_suplev*spli + sbli; - q.ie2lci_h[id] = id; - } + for (Int sbli = 0; sbli < n_id_in_suplev; ++sbli) + for (Int k = 0; k < n_in_elem; ++k) { + const Int id = nlevwrem*(n_in_elem*ie + k) + n_id_in_suplev*spli + sbli; + q.ie2lci_h[id] = id; + } } else { for (size_t ie = 0; ie < q.ie2gci_h.size(); ++ie) - for (Int sbli = 0; sbli < n_id_in_suplev; ++sbli) { - const Int id = n_id_in_suplev*ie + sbli; - q.ie2lci_h[id] = id; - } + for (Int sbli = 0; sbli < n_id_in_suplev; ++sbli) + for (Int k = 0; k < n_in_elem; ++k) { + const Int id = n_id_in_suplev*(n_in_elem*ie + k) + sbli; + q.ie2lci_h[id] = id; + } } } Kokkos::deep_copy(q.ie2lci, q.ie2lci_h); @@ -625,12 +631,12 @@ extern "C" void cedr_init_impl (const homme::Int fcomm, const homme::Int cdr_alg, const bool use_sgi, const homme::Int* gid_data, const homme::Int* rank_data, const homme::Int gbl_ncell, const homme::Int lcl_ncell, - const homme::Int nlev, const homme::Int qsize, + const homme::Int nlev, const homme::Int np, const homme::Int qsize, const bool independent_time_steps, const bool hard_zero, const homme::Int, const homme::Int) { const auto p = cedr::mpi::make_parallel(MPI_Comm_f2c(fcomm)); g_cdr = std::make_shared >( - cdr_alg, gbl_ncell, lcl_ncell, nlev, qsize, use_sgi, + cdr_alg, gbl_ncell, lcl_ncell, nlev, np, qsize, use_sgi, independent_time_steps, hard_zero, gid_data, rank_data, p, fcomm); } @@ -650,17 +656,7 @@ extern "C" void cedr_set_bufs (homme::Real* sendbuf, homme::Real* recvbuf, extern "C" void cedr_set_null_bufs () { cedr_set_bufs(nullptr, nullptr, 0, 0); } extern "C" void cedr_unittest (const homme::Int fcomm, homme::Int* nerrp) { -#if 0 - auto p = cedr::mpi::make_parallel(MPI_Comm_f2c(fcomm)); - cedr_assert(g_cdr); - cedr_assert(g_cdr->tree); - if (homme::CDR::Alg::is_qlt(g_cdr->alg)) - *nerrp = cedr::qlt::test::test_qlt(p, g_cdr->tree, g_cdr->nsublev*g_cdr->ncell, - 1, false, false, true, false); - else - *nerrp = cedr::caas::test::unittest(p); -#endif - *nerrp += compose::test::cedr_unittest(); + *nerrp = compose::test::cedr_unittest(); } extern "C" void cedr_set_ie2gci (const homme::Int ie, const homme::Int gci) { @@ -715,7 +711,6 @@ extern "C" void cedr_sl_run_global (homme::Real* minq, const homme::Real* maxq, cedr_assert(g_cdr); cedr_assert(g_sl); { homme::Timer timer("h2d"); - //if (g_cdr->p->amroot() && s_h2d) printf("cedr_h2d\n"); homme::cedr_h2d(*g_sl->ta, s_h2d); } homme::sl::run_global(*g_cdr, *g_sl, minq, maxq, nets-1, nete-1); } @@ -730,7 +725,6 @@ extern "C" void cedr_sl_run_local (homme::Real* minq, const homme::Real* maxq, homme::sl::run_local(*g_cdr, *g_sl, minq, maxq, nets-1, nete-1, use_ir, limiter_option); { homme::Timer timer("d2h"); - //if (g_cdr->p->amroot() && s_d2h) printf("cedr_d2h\n"); homme::cedr_d2h(*g_sl->ta, s_d2h); } } diff --git a/components/homme/src/share/compose/compose_cedr_cdr.hpp b/components/homme/src/share/compose/compose_cedr_cdr.hpp index 1c15b364fdf..fd8e92bc703 100644 --- a/components/homme/src/share/compose/compose_cedr_cdr.hpp +++ b/components/homme/src/share/compose/compose_cedr_cdr.hpp @@ -8,7 +8,8 @@ namespace homme { struct Alg { - enum Enum { qlt, qlt_super_level, qlt_super_level_local_caas, caas, caas_super_level }; + enum Enum { qlt, qlt_super_level, qlt_super_level_local_caas, caas, + caas_super_level }; static Enum convert (int cdr_alg) { switch (cdr_alg) { case 2: return qlt; @@ -27,6 +28,9 @@ struct Alg { static bool is_caas (Enum e) { return e == caas || e == caas_super_level; } + static bool is_point (Enum e) { + return false; + } static bool is_suplev (Enum e) { return (e == qlt_super_level || e == caas_super_level || e == qlt_super_level_local_caas); @@ -55,7 +59,7 @@ struct CDR { enum { nsublev_per_suplev = 8 }; const Alg::Enum alg; - const Int ncell, nlclcell, nlev, qsize, nsublev, nsuplev; + const Int ncell, nlclcell, nlev, np, qsize, nsublev, nsuplev; const bool threed, cdr_over_super_levels, caas_in_suplev, hard_zero; const cedr::mpi::Parallel::Ptr p; cedr::tree::Node::Ptr tree; // Don't need this except for unit testing. @@ -67,9 +71,10 @@ struct CDR { BoolsH nonneg_h; bool run; // for debugging, it can be useful not to run the CEDR. - CDR(Int cdr_alg_, Int ngblcell_, Int nlclcell_, Int nlev_, Int qsize_, bool use_sgi, - bool independent_time_steps, const bool hard_zero_, const Int* gid_data, - const Int* rank_data, const cedr::mpi::Parallel::Ptr& p_, Int fcomm); + CDR(Int cdr_alg_, Int ngblcell_, Int nlclcell_, Int nlev_, Int np_, Int qsize_, + bool use_sgi, bool independent_time_steps, const bool hard_zero_, + const Int* gid_data, const Int* rank_data, const cedr::mpi::Parallel::Ptr& p_, + Int fcomm); CDR(const CDR&) = delete; CDR& operator=(const CDR&) = delete; diff --git a/components/homme/src/share/compose/compose_homme.cpp b/components/homme/src/share/compose/compose_homme.cpp index 4399e0b5cf9..938403d096f 100644 --- a/components/homme/src/share/compose/compose_homme.cpp +++ b/components/homme/src/share/compose/compose_homme.cpp @@ -9,7 +9,6 @@ TracerArrays::TracerArrays (Int nelemd_, Int nlev_, Int np_, Int qsize_, Int pdp(nelemd, np2, nlev), pdp3d(nelemd, np2, nlev, -1, 3), pqdp(nelemd, np2, nlev, qsized, 2), pq(nelemd, np2, nlev, qsized), #if defined COMPOSE_PORT - dep_points("dep_points", nelemd, nlev, np2), q_min("q_min", nelemd, qsize, np2, nlev), q_max("q_max", nelemd, qsize, np2, nlev) #else @@ -31,46 +30,109 @@ void TracerArrays::alloc_if_not () { #endif template -void sl_h2d (TracerArrays& ta, bool transfer, Cartesian3D* dep_points) { +void sl_traj_h2d (TracerArrays& ta, Real* dep_points, Real* vnode, + Real* vdep, Int ndim) { #if defined COMPOSE_PORT +# if defined COMPOSE_HORIZ_OPENMP +# pragma omp master + { +# endif ko::fence(); ta.alloc_if_not(); const Int nelemd = ta.nelemd, qsize = ta.qsize, np2 = ta.np2, nlev = ta.nlev; - const DepPointsH cart_h(reinterpret_cast(dep_points), nelemd, nlev, np2); - const auto dep_points_h = ko::create_mirror_view(ta.dep_points); - for (Int ie = 0; ie < nelemd; ++ie) - for (Int lev = 0; lev < nlev; ++lev) - for (Int k = 0; k < np2; ++k) - for (Int d = 0; d < 3; ++d) - dep_points_h(ie,lev,k,d) = cart_h(ie,lev,k,d); - ko::deep_copy(ta.dep_points, dep_points_h); - if ( ! transfer) return; - const auto qdp_m = ko::create_mirror_view(ta.qdp); - const auto dp_m = ko::create_mirror_view(ta.dp); + const DepPointsH cart_h(dep_points, nelemd, nlev, np2, ndim); + ko::deep_copy(ta.dep_points, cart_h); + if (vnode) { + const DepPointsH h(vnode, nelemd, nlev, np2, ndim); + ko::deep_copy(ta.vnode, h); + } + if (vdep) { + const DepPointsH h(vdep, nelemd, nlev, np2, ndim); + ko::deep_copy(ta.vdep, h); + } +# ifdef COMPOSE_HORIZ_OPENMP + } +# pragma omp barrier +# endif +#endif +} + +template +void sl_traj_d2h (const TracerArrays& ta, Real* dep_points, Real* vnode, + Real* vdep, Int ndim) { +#if defined COMPOSE_PORT +# if defined COMPOSE_HORIZ_OPENMP +# pragma omp master + { +# endif + ko::fence(); const auto q_m = ko::create_mirror_view(ta.q); - for (Int ie = 0; ie < nelemd; ++ie) - for (Int iq = 0; iq < qsize; ++iq) + const Int nelemd = ta.nelemd, np2 = ta.np2, nlev = ta.nlev; + const DepPointsH dep_points_h(dep_points, nelemd, nlev, np2, ndim); + ko::deep_copy(dep_points_h, ta.dep_points); + if (vnode) { + const DepPointsH h(vnode, nelemd, nlev, np2, ndim); + ko::deep_copy(h, ta.vnode); + } + if (vdep) { + const DepPointsH h(vdep, nelemd, nlev, np2, ndim); + ko::deep_copy(h, ta.vdep); + } +# ifdef COMPOSE_HORIZ_OPENMP + } +# pragma omp barrier +# endif +#endif +} + +template +void sl_h2d (TracerArrays& ta, bool transfer, Real* dep_points, Int ndim) { +#if defined COMPOSE_PORT +# if defined COMPOSE_HORIZ_OPENMP +# pragma omp master + { +# endif + ko::fence(); + ta.alloc_if_not(); + const Int nelemd = ta.nelemd, qsize = ta.qsize, np2 = ta.np2, nlev = ta.nlev; + const DepPointsH cart_h(dep_points, nelemd, nlev, np2, ndim); + ko::deep_copy(ta.dep_points, cart_h); + if (transfer) { + const auto qdp_m = ko::create_mirror_view(ta.qdp); + const auto dp_m = ko::create_mirror_view(ta.dp); + const auto q_m = ko::create_mirror_view(ta.q); + for (Int ie = 0; ie < nelemd; ++ie) + for (Int iq = 0; iq < qsize; ++iq) + for (Int k = 0; k < np2; ++k) + for (Int lev = 0; lev < nlev; ++lev) { + for (Int qtl = 0; qtl < 2; ++qtl) + qdp_m(ie,qtl,iq,k,lev) = ta.pqdp(ie,qtl,iq,k,lev); + q_m(ie,iq,k,lev) = ta.pq(ie,iq,k,lev); + } + for (Int ie = 0; ie < nelemd; ++ie) for (Int k = 0; k < np2; ++k) - for (Int lev = 0; lev < nlev; ++lev) { - for (Int qtl = 0; qtl < 2; ++qtl) - qdp_m(ie,qtl,iq,k,lev) = ta.pqdp(ie,qtl,iq,k,lev); - q_m(ie,iq,k,lev) = ta.pq(ie,iq,k,lev); - } - for (Int ie = 0; ie < nelemd; ++ie) - for (Int k = 0; k < np2; ++k) - for (Int lev = 0; lev < nlev; ++lev) - dp_m(ie,k,lev) = ta.pdp(ie,k,lev); - ko::deep_copy(ta.qdp, qdp_m); - ko::deep_copy(ta.dp, dp_m); - ko::deep_copy(ta.q, q_m); + for (Int lev = 0; lev < nlev; ++lev) + dp_m(ie,k,lev) = ta.pdp(ie,k,lev); + ko::deep_copy(ta.qdp, qdp_m); + ko::deep_copy(ta.dp, dp_m); + ko::deep_copy(ta.q, q_m); + } +# ifdef COMPOSE_HORIZ_OPENMP + } +# pragma omp barrier +# endif #endif } template -void sl_d2h (const TracerArrays& ta, bool transfer, Cartesian3D* dep_points, +void sl_d2h (const TracerArrays& ta, bool transfer, Real* dep_points, Int ndim, Real* minq, Real* maxq) { #if defined COMPOSE_PORT if ( ! transfer) return; +# if defined COMPOSE_HORIZ_OPENMP +# pragma omp master + { +# endif ko::fence(); const auto q_m = ko::create_mirror_view(ta.q); const Int nelemd = ta.nelemd, qsize = ta.qsize, np2 = ta.np2, nlev = ta.nlev; @@ -80,13 +142,17 @@ void sl_d2h (const TracerArrays& ta, bool transfer, Cartesian3D* dep_points, for (Int k = 0; k < np2; ++k) for (Int lev = 0; lev < nlev; ++lev) ta.pq(ie,iq,k,lev) = q_m(ie,iq,k,lev); - const DepPointsH dep_points_h(reinterpret_cast(dep_points), nelemd, nlev, np2); + const DepPointsH dep_points_h(dep_points, nelemd, nlev, np2, ndim); const QExtremaH q_min_h(minq, nelemd, qsize, np2, nlev), q_max_h(maxq, nelemd, qsize, np2, nlev); ko::deep_copy(dep_points_h, ta.dep_points); ko::deep_copy(q_min_h, ta.q_min); ko::deep_copy(q_max_h, ta.q_max); +# ifdef COMPOSE_HORIZ_OPENMP + } +# pragma omp barrier +# endif #endif } @@ -94,6 +160,10 @@ template void cedr_h2d (const TracerArrays& ta, bool transfer) { #if defined COMPOSE_PORT if ( ! transfer) return; +# if defined COMPOSE_HORIZ_OPENMP +# pragma omp master + { +# endif ko::fence(); const auto dp3d_m = ko::create_mirror_view(ta.dp3d); const auto q_m = ko::create_mirror_view(ta.q); @@ -114,6 +184,10 @@ void cedr_h2d (const TracerArrays& ta, bool transfer) { ko::deep_copy(ta.dp3d, dp3d_m); ko::deep_copy(ta.q, q_m); ko::deep_copy(ta.spheremp, spheremp_m); +# ifdef COMPOSE_HORIZ_OPENMP + } +# pragma omp barrier +# endif #endif } @@ -121,6 +195,10 @@ template void cedr_d2h (const TracerArrays& ta, bool transfer) { #if defined COMPOSE_PORT if ( ! transfer) return; +# if defined COMPOSE_HORIZ_OPENMP +# pragma omp master + { +# endif ko::fence(); const auto q_m = ko::create_mirror_view(ta.q); const auto qdp_m = ko::create_mirror_view(ta.qdp); @@ -135,6 +213,10 @@ void cedr_d2h (const TracerArrays& ta, bool transfer) { ta.pqdp(ie,n1_qdp,iq,k,lev) = qdp_m(ie,n1_qdp,iq,k,lev); ta.pq(ie,iq,k,lev) = q_m(ie,iq,k,lev); } +# ifdef COMPOSE_HORIZ_OPENMP + } +# pragma omp barrier +# endif #endif } @@ -161,10 +243,14 @@ void delete_tracer_arrays () { } template struct TracerArrays; +template void sl_traj_h2d(TracerArrays& ta, + Real*, Real*, Real*, Int ndim); +template void sl_traj_d2h(const TracerArrays& ta, + Real*, Real*, Real*, Int ndim); template void sl_h2d(TracerArrays& ta, bool transfer, - Cartesian3D* dep_points); + Real* dep_points, Int ndim); template void sl_d2h(const TracerArrays& ta, bool transfer, - Cartesian3D* dep_points, Real* minq, Real* maxq); + Real* dep_points, Int ndim, Real* minq, Real* maxq); template void cedr_h2d(const TracerArrays& ta, bool transfer); template void cedr_d2h(const TracerArrays& ta, bool transfer); diff --git a/components/homme/src/share/compose/compose_homme.hpp b/components/homme/src/share/compose/compose_homme.hpp index a3b40a204a9..4f12b44fccf 100644 --- a/components/homme/src/share/compose/compose_homme.hpp +++ b/components/homme/src/share/compose/compose_homme.hpp @@ -22,7 +22,7 @@ template using FA4 = ko::View using FA5 = ko::View; template using DepPoints = - ko::View; + ko::View; template using QExtrema = ko::View; @@ -140,26 +140,26 @@ struct HommeFormatArray { COMPOSE_FORCEINLINE_FUNCTION T& operator() (const Int& ie, const Int& i) const { static_assert(rank == 2, "rank 2 array"); - assert(i >= 0); - assert(ie_data_ptr[ie]); // These routines are not used on the GPU, but they can be called from // KOKKOS_FUNCTIONs on CPU in GPU builds. Avoid nvcc warnings as follows: #if defined __CUDA_ARCH__ || defined __HIP_DEVICE_COMPILE__ return unused(); #else + assert(i >= 0); + assert(ie_data_ptr[ie]); return *(ie_data_ptr[ie] + i); #endif } COMPOSE_FORCEINLINE_FUNCTION T& operator() (const Int& ie, const Int& k, const Int& lev) const { static_assert(rank == 3, "rank 3 array"); +#if defined __CUDA_ARCH__ || defined __HIP_DEVICE_COMPILE__ + return unused(); +#else assert(k >= 0); assert(lev >= 0); assert(ie_data_ptr[ie]); check(ie, k, lev); -#if defined __CUDA_ARCH__ || defined __HIP_DEVICE_COMPILE__ - return unused(); -#else return *(ie_data_ptr[ie] + lev*np2 + k); #endif } @@ -167,14 +167,14 @@ struct HommeFormatArray { T& operator() (const Int& ie, const Int& q_or_timelev, const Int& k, const Int& lev) const { static_assert(rank == 4, "rank 4 array"); +#if defined __CUDA_ARCH__ || defined __HIP_DEVICE_COMPILE__ + return unused(); +#else assert(q_or_timelev >= 0); assert(k >= 0); assert(lev >= 0); assert(ie_data_ptr[ie]); check(ie, k, lev, q_or_timelev); -#if defined __CUDA_ARCH__ || defined __HIP_DEVICE_COMPILE__ - return unused(); -#else return *(ie_data_ptr[ie] + (q_or_timelev*nlev + lev)*np2 + k); #endif } @@ -182,15 +182,15 @@ struct HommeFormatArray { T& operator() (const Int& ie, const Int& timelev, const Int& q, const Int& k, const Int& lev) const { static_assert(rank == 5, "rank 4 array"); +#if defined __CUDA_ARCH__ || defined __HIP_DEVICE_COMPILE__ + return unused(); +#else assert(timelev >= 0); assert(q >= 0); assert(k >= 0); assert(lev >= 0); assert(ie_data_ptr[ie]); check(ie, k, lev, q, timelev); -#if defined __CUDA_ARCH__ || defined __HIP_DEVICE_COMPILE__ - return unused(); -#else return *(ie_data_ptr[ie] + ((timelev*qsized + q)*nlev + lev)*np2 + k); #endif } @@ -255,7 +255,7 @@ struct TracerArrays { View dp3d; // elem%state%dp3d or the sl3d equivalent View qdp; // elem%state%Qdp(:,:,:,:,:) View q; // elem%state%Q - DepPoints dep_points; + DepPoints dep_points, vnode, vdep; QExtrema q_min, q_max; void alloc_if_not(); #else @@ -287,10 +287,18 @@ subview_ie (const Int ie, const TracerView& s) { return TracerView(&s(ie,0,0,0,0), s.extent(1), s.extent(2), s.extent(3), s.extent(4)); } template -void sl_h2d(TracerArrays& ta, bool transfer, Cartesian3D* dep_points); +void sl_traj_h2d(TracerArrays& ta, Real* dep_points, Real* vnode, Real* vdep, + Int ndim); + +template +void sl_traj_d2h(const TracerArrays& ta, Real* dep_points, Real* vnode, + Real* vdep, Int ndim); + +template +void sl_h2d(TracerArrays& ta, bool transfer, Real* dep_points, Int ndim); template -void sl_d2h(const TracerArrays& ta, bool transfer, Cartesian3D* dep_points, +void sl_d2h(const TracerArrays& ta, bool transfer, Real* dep_points, Int ndim, Real* minq, Real* maxq); template diff --git a/components/homme/src/share/compose/compose_hommexx.cpp b/components/homme/src/share/compose/compose_hommexx.cpp index fdc6cdf4bcf..0b88e6024a1 100644 --- a/components/homme/src/share/compose/compose_hommexx.cpp +++ b/components/homme/src/share/compose/compose_hommexx.cpp @@ -21,10 +21,13 @@ template using View = typename TracerArrays::View; #endif -void set_views (const SetView& spheremp, - const SetView& dp, const SetView& dp3d, - const SetView& qdp, const SetView& q, - const SetView& dep_points) { +void set_views (const SetView& spheremp, + const SetView& dp, const SetView5& dp3d, + const SetView& qdp, const SetView5& q, + const SetView5& dep_points, const SetView5& vnode, + const SetView5& vdep, const Int ndim) { + static_assert(std::is_same::value, + "Hommexx and Compose real types must be the same."); #ifdef COMPOSE_PORT auto& ta = *get_tracer_arrays(); const auto nel = spheremp.extent_int(0); @@ -35,13 +38,29 @@ void set_views (const SetView& spheremp, ta.dp3d = View(dp3d.data(), nel, dp3d.extent_int(1), np2, nlev); ta.qdp = View(qdp.data(), nel, qdp.extent_int(1), qdp.extent_int(2), np2, nlev); ta.q = View(q.data(), nel, q.extent_int(1), np2, nlev); - ta.dep_points = View(dep_points.data(), nel, dep_points.extent_int(1), np2); + ta.dep_points = View(dep_points.data(), nel, dep_points.extent_int(1), np2, ndim); + if (vnode.data()) + ta.vnode = View(vnode.data(), nel, vnode.extent_int(1), np2, ndim); + if (vdep.data()) + ta.vdep = View(vdep.data(), nel, vdep .extent_int(1), np2, ndim); #else slmm_throw_if(true, "Running a Hommexx code path with the non-Hommexx build" " is not supported.\n"); #endif } +void set_hvcoord (const HommexxReal etai_beg, const HommexxReal etai_end, + const HommexxReal* etam) { + auto& cm = *get_isl_mpi_singleton(); + islmpi::set_hvcoord(cm, etai_beg, etai_end, etam); +} + +void calc_v_departure (const int step, const HommexxReal dtsub) { + auto& cm = *get_isl_mpi_singleton(); + islmpi::calc_v_departure<>(cm, 0, cm.nelemd - 1, step, dtsub, + nullptr, nullptr, nullptr); +} + void advect (const int np1, const int n0_qdp, const int np1_qdp) { auto& cm = *get_isl_mpi_singleton(); cm.tracer_arrays->np1 = np1; diff --git a/components/homme/src/share/compose/compose_hommexx.hpp b/components/homme/src/share/compose/compose_hommexx.hpp index 6cd8f510e8e..b2e33905540 100644 --- a/components/homme/src/share/compose/compose_hommexx.hpp +++ b/components/homme/src/share/compose/compose_hommexx.hpp @@ -6,13 +6,22 @@ namespace homme { namespace compose { +typedef double HommexxReal; + template using SetView = Kokkos::View; -void set_views(const SetView& spheremp, - const SetView& dp, const SetView& dp3d, - const SetView& qdp, const SetView& q, - const SetView& dep_points); +typedef SetView SetView5; + +void set_views(const SetView& spheremp, + const SetView& dp, const SetView5& dp3d, + const SetView& qdp, const SetView5& q, + const SetView5& dep_points, const SetView5& vnode, + const SetView5& vdep, const int trajectory_ndim); + +void set_hvcoord(const HommexxReal etai_beg, const HommexxReal etai_end, + const HommexxReal* etam); +void calc_v_departure(const int step, const HommexxReal dtsub); void advect(const int np1, const int n0_qdp, const int np1_qdp); diff --git a/components/homme/src/share/compose/compose_slmm.cpp b/components/homme/src/share/compose/compose_slmm.cpp index 27ab1e1a7d0..ec570e21580 100644 --- a/components/homme/src/share/compose/compose_slmm.cpp +++ b/components/homme/src/share/compose/compose_slmm.cpp @@ -98,11 +98,12 @@ init (const typename IslMpi::Advecter::ConstPtr& advecter, const mpi::Parallel::Ptr& p, Int np, Int nlev, Int qsize, Int qsized, Int nelemd, const Int* nbr_id_rank, const Int* nirptr, - Int halo) { - slmm_throw_if(halo < 1 || halo > 2, "halo must be 1 (default) or 2."); + Int halo, Int traj_3d, Int traj_nsubstep) { + slmm_throw_if(halo < 1, "halo must be 1 (default) or larger."); auto tracer_arrays = homme::init_tracer_arrays(nelemd, nlev, np, qsize, qsized); auto cm = std::make_shared >(p, advecter, tracer_arrays, np, nlev, - qsize, qsized, nelemd, halo); + qsize, qsized, nelemd, halo, traj_3d, + traj_nsubstep); setup_comm_pattern(*cm, nbr_id_rank, nirptr); return cm; } @@ -111,13 +112,43 @@ init (const typename IslMpi::Advecter::ConstPtr& advecter, // already has a ref to the const'ed one. template void finalize_init_phase (IslMpi& cm, typename IslMpi::Advecter& advecter) { - if (cm.halo == 2) + if (cm.halo > 1) extend_halo::extend_local_meshes(*cm.p, cm.ed_h, advecter); advecter.fill_nearest_points_if_needed(); advecter.sync_to_device(); sync_to_device(cm); } +template +void set_hvcoord (IslMpi& cm, const Real etai_beg, const Real etai_end, + const Real* etam) { + if (cm.etam.size() > 0) return; +#if defined COMPOSE_HORIZ_OPENMP +# pragma omp barrier +# pragma omp master +#endif + { + slmm_assert(cm.nlev > 0); + cm.etai_beg = etai_beg; + cm.etai_end = etai_end; + cm.etam = typename IslMpi::template ArrayD("etam", cm.nlev); + const auto h = ko::create_mirror_view(cm.etam); + for (int k = 0; k < cm.nlev; ++k) { + h(k) = etam[k]; + slmm_assert(k == 0 || h(k) > h(k-1)); + slmm_assert(h(k) > 0 && h(k) < 1); + } + ko::deep_copy(cm.etam, h); + } +#if defined COMPOSE_HORIZ_OPENMP +# pragma omp barrier +#endif +} + +template void set_hvcoord( + IslMpi& cm, const Real etai_beg, const Real etai_end, + const Real* etam); + // Set pointers to HOMME data arrays. template void set_elem_data (IslMpi& cm, const Int ie, Real* qdp, const Int n0_qdp, @@ -297,8 +328,9 @@ void slmm_init_impl ( homme::Int nelemd, homme::Int cubed_sphere_map, homme::Int geometry, const homme::Int* lid2gid, const homme::Int* lid2facenum, const homme::Int* nbr_id_rank, const homme::Int* nirptr, - homme::Int sl_nearest_point_lev, homme::Int, homme::Int, homme::Int, - homme::Int) + homme::Int sl_halo, homme::Int sl_traj_3d, homme::Int sl_traj_nsubstep, + homme::Int sl_nearest_point_lev, + homme::Int, homme::Int, homme::Int, homme::Int) { amb::dev_init_threads(); homme::slmm_init(np, nelem, nelemd, transport_alg, cubed_sphere_map, @@ -308,7 +340,7 @@ void slmm_init_impl ( const auto p = homme::mpi::make_parallel(MPI_Comm_f2c(fcomm)); homme::g_csl_mpi = homme::islmpi::init( homme::g_advecter, p, np, nlev, qsize, qsized, nelemd, - nbr_id_rank, nirptr, 2 /* halo */); + nbr_id_rank, nirptr, sl_halo, sl_traj_3d, sl_traj_nsubstep); amb::dev_fin_threads(); } @@ -374,6 +406,42 @@ void slmm_check_ref2sphere (homme::Int ie, homme::Cartesian3D* p) { amb::dev_fin_threads(); } +void slmm_set_hvcoord (const homme::Real etai_beg, const homme::Real etai_end, + const homme::Real* etam) { + amb::dev_init_threads(); + slmm_assert(homme::g_csl_mpi); + homme::islmpi::set_hvcoord(*homme::g_csl_mpi, etai_beg, etai_end, etam); + amb::dev_fin_threads(); +} + +void slmm_calc_v_departure ( + homme::Int nets, homme::Int nete, homme::Int step, homme::Real dtsub, + homme::Real* dep_points, homme::Int dep_points_ndim, homme::Real* vnode, + homme::Real* vdep, homme::Int* info) +{ + amb::dev_init_threads(); + check_threading(); + slmm_assert(homme::g_csl_mpi); + slmm_assert(homme::g_csl_mpi->sendsz.empty()); // alloc_mpi_buffers was called + auto& cm = *homme::g_csl_mpi; + slmm_assert(cm.dep_points_ndim == dep_points_ndim); + { + slmm::Timer timer("h2d"); + homme::sl_traj_h2d(*cm.tracer_arrays, dep_points, vnode, vdep, + cm.dep_points_ndim); + } + homme::islmpi::calc_v_departure(cm, nets - 1, nete - 1, step - 1, + dtsub, dep_points, vnode, vdep); + *info = 0; + { + slmm::Timer timer("d2h"); + homme::sl_traj_d2h(*cm.tracer_arrays, dep_points, vnode, vdep, + cm.dep_points_ndim); + } + amb::dev_fin_threads(); +} + +// Request extra data to be transferred for analysis. static bool s_h2d, s_d2h; void slmm_csl_set_elem_data ( @@ -389,34 +457,35 @@ void slmm_csl_set_elem_data ( amb::dev_fin_threads(); } -void slmm_csl ( - homme::Int nets, homme::Int nete, homme::Cartesian3D* dep_points, - homme::Real* minq, homme::Real* maxq, homme::Int* info) -{ +void slmm_csl (homme::Int nets, homme::Int nete, homme::Real* dep_points, + homme::Int dep_points_ndim, homme::Real* minq, homme::Real* maxq, + homme::Int* info) { amb::dev_init_threads(); check_threading(); slmm_assert(homme::g_csl_mpi); slmm_assert(homme::g_csl_mpi->sendsz.empty()); // alloc_mpi_buffers was called - { slmm::Timer timer("h2d"); - //if (homme::g_csl_mpi->p->amroot() && s_h2d) printf("sl_h2d\n"); - homme::sl_h2d(*homme::g_csl_mpi->tracer_arrays, s_h2d, dep_points); } + auto& cm = *homme::g_csl_mpi; + slmm_assert(cm.dep_points_ndim == dep_points_ndim); + { + slmm::Timer timer("h2d"); + homme::sl_h2d(*cm.tracer_arrays, s_h2d, dep_points, cm.dep_points_ndim); + } *info = 0; -#if 0 -#pragma message "RM TRY-CATCH WHILE DEV'ING" +#if 1 try { - homme::islmpi::step(*homme::g_csl_mpi, nets - 1, nete - 1, - reinterpret_cast(dep_points), minq, maxq); + homme::islmpi::step(cm, nets - 1, nete - 1, dep_points, minq, maxq); } catch (const std::exception& e) { std::cerr << e.what(); *info = -1; } #else - homme::islmpi::step(*homme::g_csl_mpi, nets - 1, nete - 1, - reinterpret_cast(dep_points), minq, maxq); + homme::islmpi::step(cm, nets - 1, nete - 1, dep_points, minq, maxq); #endif - { slmm::Timer timer("d2h"); - //if (homme::g_csl_mpi->p->amroot() && s_d2h) printf("sl_d2h\n"); - homme::sl_d2h(*homme::g_csl_mpi->tracer_arrays, s_d2h, dep_points, minq, maxq); } + { + slmm::Timer timer("d2h"); + homme::sl_d2h(*cm.tracer_arrays, s_d2h, dep_points, cm.dep_points_ndim, + minq, maxq); + } amb::dev_fin_threads(); } diff --git a/components/homme/src/share/compose/compose_slmm_departure_point.hpp b/components/homme/src/share/compose/compose_slmm_departure_point.hpp index de912c54216..c226787cfc8 100644 --- a/components/homme/src/share/compose/compose_slmm_departure_point.hpp +++ b/components/homme/src/share/compose/compose_slmm_departure_point.hpp @@ -1,6 +1,7 @@ #ifndef INCLUDE_COMPOSE_SLMM_DEPARTURE_POINT_HPP #define INCLUDE_COMPOSE_SLMM_DEPARTURE_POINT_HPP +#include "compose.hpp" #include "compose_slmm.hpp" namespace slmm { diff --git a/components/homme/src/share/compose/compose_slmm_islmpi.cpp b/components/homme/src/share/compose/compose_slmm_islmpi.cpp index 822780dd967..635e020877c 100644 --- a/components/homme/src/share/compose/compose_slmm_islmpi.cpp +++ b/components/homme/src/share/compose/compose_slmm_islmpi.cpp @@ -92,20 +92,23 @@ typedef std::vector GidRankPairs; typedef std::map Gid2Nbrs; typedef std::vector IntBuf; typedef std::vector RealBuf; +typedef std::map Gid2Count; template -GidRankPairs all_nbrs_but_me (const typename IslMpi::ElemDataH& ed) { +GidRankPairs all_1halo_nbrs_but_me (const typename IslMpi::ElemDataH& ed) { GidRankPairs gs; - gs.reserve(ed.nbrs.size() - 1); - for (const auto& n : ed.nbrs) + gs.reserve(ed.nin1halo - 1); + for (Int i = 0; i < ed.nin1halo; ++i) { + const auto& n = ed.nbrs(i); if (&n != ed.me) gs.push_back(GidRankPair(n.gid, n.rank)); + } return gs; } template void fill_gid2nbrs (const mpi::Parallel& p, const typename IslMpi::ElemDataListH& eds, - Gid2Nbrs& gid2nbrs) { + Gid2Nbrs& gid2nbrs, Gid2Count& gid2ninprevhalo) { static const Int tag = 6; const Rank my_rank = p.rank(); const Int n_owned = eds.size(); @@ -113,7 +116,7 @@ void fill_gid2nbrs (const mpi::Parallel& p, const typename IslMpi::ElemDataL // Fill in the ones we know. for (const auto& ed : eds) { slmm_assert(ed.me->rank == my_rank); - gid2nbrs[ed.me->gid] = all_nbrs_but_me(ed); + gid2nbrs[ed.me->gid] = all_1halo_nbrs_but_me(ed); } std::vector ranks; @@ -126,13 +129,18 @@ void fill_gid2nbrs (const mpi::Parallel& p, const typename IslMpi::ElemDataL std::map needgid2rank; { std::set unique_ranks; - for (const auto& item : gid2nbrs) - for (const auto& n : item.second) - if (n.rank != my_rank) { - slmm_assert(gid2nbrs.find(n.gid) == gid2nbrs.end()); - needgid2rank.insert(std::make_pair(n.gid, n.rank)); - unique_ranks.insert(n.rank); - } + for (const auto& ed : eds) { + // We only need information for GIDs in the current outermost halo. + const auto& it = gid2ninprevhalo.find(ed.me->gid); + const Int i0 = it == gid2ninprevhalo.end() ? 0 : it->second; + for (Int i = i0; i < ed.nbrs.size(); ++i) { + const auto& n = ed.nbrs(i); + if (n.rank == my_rank) continue; + slmm_assert(gid2nbrs.find(n.gid) == gid2nbrs.end()); + needgid2rank.insert(std::make_pair(n.gid, n.rank)); + unique_ranks.insert(n.rank); + } + } nrank = unique_ranks.size(); ranks.insert(ranks.begin(), unique_ranks.begin(), unique_ranks.end()); Int i = 0; @@ -171,8 +179,9 @@ void fill_gid2nbrs (const mpi::Parallel& p, const typename IslMpi::ElemDataL std::vector nbr_send_reqs(nrank), nbr_recv_reqs(nrank); for (Int i = 0; i < nrank; ++i) { auto& r = nbr_recvs[i]; - // 20 is from dimensions_mod::set_mesh_dimensions; factor of 2 is to get - // (gid,rank); 1 is for size datum. + // 20 is from dimensions_mod::set_mesh_dimensions, the maximum size of the + // 1-halo minus the 0-halo; factor of 2 is to get (gid,rank); 1 is for the + // size datum. r.resize((20*2 + 1)*(req_sends[i].size() - 1)); mpi::irecv(p, r.data(), r.size(), ranks[i], tag, &nbr_recv_reqs[i]); } @@ -215,17 +224,22 @@ void fill_gid2nbrs (const mpi::Parallel& p, const typename IslMpi::ElemDataL } template -void extend_nbrs (const Gid2Nbrs& gid2nbrs, typename IslMpi::ElemDataListH& eds) { +void extend_nbrs (const Gid2Nbrs& gid2nbrs, typename IslMpi::ElemDataListH& eds, + Gid2Count& gid2ninprevhalo) { for (auto& ed : eds) { - // Get all <=2-halo neighbors. - std::set new_nbrs; - for (const auto& n : ed.nbrs) { - if (&n == ed.me) continue; - const auto& it = gid2nbrs.find(n.gid); - slmm_assert(it != gid2nbrs.end()); - const auto& gid_nbrs = it->second; - for (const auto& gn : gid_nbrs) - new_nbrs.insert(gn); + // Get all <=(n+1)-halo neighbors, where we already have <=n-halo neighbors. + std::set new_nbrs; { + const auto& it = gid2ninprevhalo.find(ed.me->gid); + const Int i0 = it == gid2ninprevhalo.end() ? 0 : it->second; + for (Int i = i0; i < ed.nbrs.size(); ++i) { + const auto& n = ed.nbrs(i); + if (&n == ed.me) continue; + const auto& it = gid2nbrs.find(n.gid); + slmm_assert(it != gid2nbrs.end()); + const auto& gid_nbrs = it->second; + for (const auto& gn : gid_nbrs) + new_nbrs.insert(gn); + } } // Remove the already known ones. for (const auto& n : ed.nbrs) @@ -238,8 +252,9 @@ void extend_nbrs (const Gid2Nbrs& gid2nbrs, typename IslMpi::ElemDataListH& break; } slmm_assert(me >= 0); - // Append the, now only new, 2-halo ones. + // Append the, now only new, (n+1)-halo ones. Int i = ed.nbrs.size(); + gid2ninprevhalo[ed.me->gid] = i; ed.nbrs.reset_capacity(i + new_nbrs.size(), true); ed.me = &ed.nbrs(me); for (const auto& n : new_nbrs) { @@ -250,14 +265,22 @@ void extend_nbrs (const Gid2Nbrs& gid2nbrs, typename IslMpi::ElemDataListH& en.lid_on_rank = -1; en.lid_on_rank_idx = -1; } +#ifndef NDEBUG + { + std::set ugid; + for (Int i = 0; i < ed.nbrs.size(); ++i) ugid.insert(ed.nbrs(i).gid); + slmm_assert(ugid.size() == size_t(ed.nbrs.size())); + } +#endif } } template -void collect_gid_rank (const mpi::Parallel& p, typename IslMpi::ElemDataListH& eds) { +void collect_gid_rank (const mpi::Parallel& p, typename IslMpi::ElemDataListH& eds, + Gid2Count& gid2ninprevhalo) { Gid2Nbrs gid2nbrs; - fill_gid2nbrs(p, eds, gid2nbrs); - extend_nbrs(gid2nbrs, eds); + fill_gid2nbrs(p, eds, gid2nbrs, gid2ninprevhalo); + extend_nbrs(gid2nbrs, eds, gid2ninprevhalo); } template @@ -478,7 +501,11 @@ void collect_gid_rank (IslMpi& cm, const Int* nbr_id_rank, const Int* nirptr } slmm_assert(ed.me); } - if (cm.halo == 2) extend_halo::collect_gid_rank(*cm.p, cm.ed_h); + if (cm.halo > 1) { + extend_halo::Gid2Count gid2ninprevhalo; + for (int halo = 2; halo <= cm.halo; ++halo) + extend_halo::collect_gid_rank(*cm.p, cm.ed_h, gid2ninprevhalo); + } #ifdef COMPOSE_PORT cm.own_dep_mask = typename IslMpi::DepMask("own_dep_mask", cm.nelemd, cm.nlev, cm.np2); @@ -682,17 +709,20 @@ void size_mpi_buffers (IslMpi& cm, const Rank2Gids& rank2rmtgids, const Int sor = sizeof(Real), soi = sizeof(Int), sosi = sor; static_assert(sizeof(Real) >= sizeof(Int), "For buffer packing, we require sizeof(Real) >= sizeof(Int)"); + const bool calc_trajectory = cm.traj_nsubstep > 0; + const Int ndim = calc_trajectory ? cm.dep_points_ndim : 3; + const Int qsize = calc_trajectory ? std::max(cm.dep_points_ndim, cm.qsize) : cm.qsize; const auto xbufcnt = [&] (const std::set& rmtgids, const std::set& owngids, const bool include_bulk = true) -> Int { - return (sosi + (2*soi + (2*soi)*cm.nlev)*rmtgids.size() + // meta data - (include_bulk ? 1 : 0)*owngids.size()*cm.nlev*cm.np2*3*sor); // bulk data + return (sosi + (2*soi + (2*soi)*cm.nlev)*rmtgids.size() + // meta data + (include_bulk ? 1 : 0)*owngids.size()*cm.nlev*cm.np2*ndim*sor); // bulk data }; const auto qbufcnt = [&] (const std::set& rmtgids, const std::set& owngids) -> Int { return ((rmtgids.size()*2 + // min/max q owngids.size()*cm.np2)* // q - cm.qsize*cm.nlev*sor); + qsize*cm.nlev*sor); }; const auto bytes2real = [&] (const Int& bytes) { return (bytes + sor - 1)/sor; diff --git a/components/homme/src/share/compose/compose_slmm_islmpi.hpp b/components/homme/src/share/compose/compose_slmm_islmpi.hpp index ef30c826acf..1e60d602dea 100644 --- a/components/homme/src/share/compose/compose_slmm_islmpi.hpp +++ b/components/homme/src/share/compose/compose_slmm_islmpi.hpp @@ -11,7 +11,9 @@ #include // AMB 2017/06-2020/05 Initial for E3SMv2 -// AMB 2020/05-? Performance-portable impl +// AMB 2020/05-2021/01 Performance-portable impl +// AMB 2021/04 Support doubly-periodic planar mode +// AMB 2024/04-2025/01 Enhanced trajectory method namespace homme { namespace mpi { //todo Share with cedr. @@ -20,14 +22,14 @@ class Parallel { MPI_Comm comm_; public: typedef std::shared_ptr Ptr; - Parallel(MPI_Comm comm) : comm_(comm) {} + Parallel (MPI_Comm comm) : comm_(comm) {} MPI_Comm comm () const { return comm_; } - Int size() const { + Int size () const { int sz = 0; MPI_Comm_size(comm_, &sz); return sz; } - Int rank() const { + Int rank () const { int pid = 0; MPI_Comm_rank(comm_, &pid); return pid; @@ -85,6 +87,12 @@ int irecv (const Parallel& p, T* buf, int count, int src, int tag, Request* ireq int waitany(int count, Request* reqs, int* index, MPI_Status* stats = nullptr); int waitall(int count, Request* reqs, MPI_Status* stats = nullptr); int wait(Request* req, MPI_Status* stat = nullptr); + +template +int all_reduce (const Parallel& p, const T* sendbuf, T* rcvbuf, int count, MPI_Op op) { + MPI_Datatype dt = get_type(); + return MPI_Allreduce(const_cast(sendbuf), rcvbuf, count, dt, op, p.comm()); +} } // namespace mpi namespace islmpi { @@ -251,6 +259,40 @@ void deep_copy (FixedCapList& d, const FixedCapList& s) { #endif } +template +struct FixedCapListHostOnly { + FixedCapListHostOnly (const Int cap = 0) { + slmm_assert_high(cap >= 0); + reset_capacity(cap); + } + + void reset_capacity (const Int cap, const bool also_size = false) { + slmm_assert(cap >= 0); + d_.resize(cap); + n_ = also_size ? cap : 0; + } + + Int capacity () const { return d_.size(); } + Int size () const { return n_; } + Int n () const { return n_; } + + void clear () { n_ = 0; } + + void inc () { ++n_; slmm_kernel_assert_high(n_ <= static_cast(d_.size())); } + void inc (const Int& dn) { n_ += dn; slmm_kernel_assert_high(n_ <= static_cast(d_.size())); } + + T& operator() (const Int& i) { slmm_kernel_assert_high(i >= 0 && i < n_); return d_[i]; } + + T* data () { return d_.data(); } + T& back () { slmm_kernel_assert_high(n_ > 0); return d_[n_-1]; } + T* begin () { return d_.data(); } + T* end () { return d_.data() + n_; } + +private: + std::vector d_; + Int n_; +}; + template struct BufferLayoutArray; template @@ -512,6 +554,11 @@ struct IslMpi { const mpi::Parallel::Ptr p; const typename Advecter::ConstPtr advecter; const Int np, np2, nlev, qsize, qsized, nelemd, halo; + const bool traj_3d; + const Int traj_nsubstep, dep_points_ndim; + + Real etai_beg, etai_end; + ArrayD etam; ElemDataListH ed_h; // this rank's owned cells, indexed by LID ElemDataListD ed_d; @@ -526,7 +573,7 @@ struct IslMpi { BufferLayoutArray bla; // MPI comm data. - FixedCapList sendreq, recvreq; + FixedCapListHostOnly sendreq, recvreq; FixedCapList recvreq_ri; ListOfLists sendbuf, recvbuf; #ifdef COMPOSE_MPI_ON_HOST @@ -559,11 +606,14 @@ struct IslMpi { Int own_dep_list_len; IslMpi (const mpi::Parallel::Ptr& ip, const typename Advecter::ConstPtr& advecter, - const typename TracerArrays::Ptr& tracer_arrays_, - Int inp, Int inlev, Int iqsize, Int iqsized, Int inelemd, Int ihalo) + const typename TracerArrays::Ptr& itracer_arrays, + Int inp, Int inlev, Int iqsize, Int iqsized, Int inelemd, Int ihalo, + Int itraj_3d, Int itraj_nsubstep) : p(ip), advecter(advecter), np(inp), np2(np*np), nlev(inlev), qsize(iqsize), qsized(iqsized), nelemd(inelemd), - halo(ihalo), tracer_arrays(tracer_arrays_) + halo(ihalo), traj_3d(itraj_3d), traj_nsubstep(itraj_nsubstep), + dep_points_ndim(traj_3d && traj_nsubstep > 0 ? 4 : 3), + tracer_arrays(itracer_arrays) {} IslMpi(const IslMpi&) = delete; @@ -635,16 +685,17 @@ void wait_on_send (IslMpi& cm, const bool skip_if_empty = false); template void recv(IslMpi& cm, const bool skip_if_empty = false); -const int nreal_per_2int = (2*sizeof(Int) + sizeof(Real) - 1) / sizeof(Real); - template -void pack_dep_points_sendbuf_pass1(IslMpi& cm); +void pack_dep_points_sendbuf_pass1(IslMpi& cm, const bool trajectory = false); template -void pack_dep_points_sendbuf_pass2(IslMpi& cm, const DepPoints& dep_points); +void pack_dep_points_sendbuf_pass2(IslMpi& cm, const DepPoints& dep_points, + const bool trajectory = false); template void calc_q_extrema(IslMpi& cm, const Int& nets, const Int& nete); +template +void calc_rmt_q_pass1(IslMpi& cm, const bool trajectory = false); template void calc_rmt_q(IslMpi& cm); template @@ -681,8 +732,17 @@ void copy_q(IslMpi& cm, const Int& nets, template void step( IslMpi& cm, const Int nets, const Int nete, - Real* dep_points_r, // dep_points(1:3, 1:np, 1:np) - Real* q_min_r, Real* q_max_r); // q_{min,max}(1:np, 1:np, lev, 1:qsize, ie-nets+1) + Real* dep_points_r, + Real* q_min_r, Real* q_max_r); + +template +void set_hvcoord(IslMpi& cm, const Real etai_beg, const Real etai_end, + const Real* etam); + +template +void calc_v_departure( + IslMpi& cm, const Int nets, const Int nete, const Int step, const Real dtsub, + Real* dep_points_r, const Real* vnode, Real* vdep); } // namespace islmpi } // namespace homme diff --git a/components/homme/src/share/compose/compose_slmm_islmpi_buf.hpp b/components/homme/src/share/compose/compose_slmm_islmpi_buf.hpp new file mode 100644 index 00000000000..9cf8b0fe61f --- /dev/null +++ b/components/homme/src/share/compose/compose_slmm_islmpi_buf.hpp @@ -0,0 +1,53 @@ +#ifndef INCLUDE_COMPOSE_SLMM_ISLMPI_BUF_HPP +#define INCLUDE_COMPOSE_SLMM_ISLMPI_BUF_HPP + +namespace homme { +namespace islmpi { + +const int nreal_per_2int = (2*sizeof(Int) + sizeof(Real) - 1) / sizeof(Real); + +template SLMM_KIF +Int setbuf (Buffer& buf, const Int& os, const Int& i1, const Int& i2) { + Int* const b = reinterpret_cast(&buf(os)); + b[0] = i1; + b[1] = i2; + return nreal_per_2int; +} + +template SLMM_KIF +Int setbuf (Buffer& buf, const Int& os, const Int& i1, const short& i2, const short& i3) { + static_assert(sizeof(Int) >= 2*sizeof(short), "Need >= 2 shorts per Int"); + Int* const b = reinterpret_cast(&buf(os)); + b[0] = i1; + short* const b2 = reinterpret_cast(b+1); + b2[0] = i2; + b2[1] = i3; + return nreal_per_2int; +} + +template SLMM_KIF +Int setbuf (Buffer& buf, const Int& os, const Int& i1, const Int& i2, + const bool final) { + if (final) setbuf(buf, os, i1, i2); + return nreal_per_2int; +} + +template SLMM_KIF +Int setbuf (Buffer& buf, const Int& os, const Int& i1, const short& i2, const short& i3, + const bool final) { + if (final) setbuf(buf, os, i1, i2, i3); + return nreal_per_2int; +} + +template SLMM_KIF +Int getbuf (Buffer& buf, const Int& os, Int& i1, Int& i2) { + const Int* const b = reinterpret_cast(&buf(os)); + i1 = b[0]; + i2 = b[1]; + return nreal_per_2int; +} + +} // namespace islmpi +} // namespace homme + +#endif diff --git a/components/homme/src/share/compose/compose_slmm_islmpi_calc_trajectory.cpp b/components/homme/src/share/compose/compose_slmm_islmpi_calc_trajectory.cpp new file mode 100644 index 00000000000..273a7e80e9c --- /dev/null +++ b/components/homme/src/share/compose/compose_slmm_islmpi_calc_trajectory.cpp @@ -0,0 +1,333 @@ +#include "compose_slmm_islmpi.hpp" +#include "compose_slmm_islmpi_interpolate.hpp" +#include "compose_slmm_islmpi_buf.hpp" + +namespace homme { +namespace islmpi { + +template using CA4 = ko::View; + +template SLMM_KF void +interpolate_vertical (const Int nlev, const Real etai_beg, const Real etai_end, + const EtamT& etam, const VnodeT& vnode, + const Int src_lid, const Int lev, const Real eta_dep, + const Real rx[np], const Real ry[np], Real* const v_tgt) { + slmm_kernel_assert(eta_dep > etai_beg && eta_dep < etai_end); + + // Search for the eta midpoint values that support the departure point's eta + // value. + Int lev_dep = lev; + if (eta_dep != etam(lev)) { + if (eta_dep < etam(lev)) { + for (lev_dep = lev-1; lev_dep >= 0; --lev_dep) + if (eta_dep >= etam(lev_dep)) + break; + } else { + for (lev_dep = lev; lev_dep < nlev-1; ++lev_dep) + if (eta_dep < etam(lev_dep+1)) + break; + } + } + slmm_kernel_assert(lev_dep >= -1 && lev_dep < nlev); + slmm_kernel_assert(lev_dep == -1 || eta_dep >= etam(lev_dep)); + Real a; + bool bdy = false; + if (lev_dep == -1) { + lev_dep = 0; + a = 0; + bdy = true; + } else if (lev_dep == nlev-1) { + a = 0; + bdy = true; + } else { + a = ((eta_dep - etam(lev_dep)) / + (etam(lev_dep+1) - etam(lev_dep))); + } + // Linear interp coefficients. + const Real alpha[] = {1-a, a}; + + for (int d = 0; d < 4; ++d) + v_tgt[d] = 0; + for (int i = 0; i < 2; ++i) { + if (alpha[i] == 0) continue; + for (int d = 0; d < 4; ++d) { + Real vel_nodes[np*np]; + for (int k = 0; k < np*np; ++k) + vel_nodes[k] = vnode(src_lid,lev_dep+i,k,d); + v_tgt[d] += alpha[i]*calc_q_tgt(rx, ry, vel_nodes); + } + } + // Treat eta_dot specially since eta_dot goes to 0 at the boundaries. + if (bdy) { + slmm_kernel_assert(etam(0) > etai_beg); + slmm_kernel_assert(etam(nlev-1) < etai_end); + if (lev_dep == 0) + v_tgt[3] *= (eta_dep - etai_beg)/(etam(0) - etai_beg); + else + v_tgt[3] *= (etai_end - eta_dep)/(etai_end - etam(nlev-1)); + } +} + +template +void calc_v (const IslMpi& cm, const VnodeT& vnode, + const Int src_lid, const Int lev, + const Real* const dep_point, Real* const v_tgt) { + // Horizontal interpolation. + Real rx[np], ry[np]; { + Real ref_coord[2]; + const auto& m = cm.advecter->local_mesh(src_lid); + cm.advecter->s2r().calc_sphere_to_ref(src_lid, m, dep_point, + ref_coord[0], ref_coord[1]); + interpolate(cm.advecter->alg(), ref_coord, rx, ry); + } + + if (not cm.traj_3d) { + for (int d = 0; d < cm.dep_points_ndim; ++d) { + Real vel_nodes[np*np]; + for (int k = 0; k < np*np; ++k) + vel_nodes[k] = vnode(src_lid,lev,k,d); + v_tgt[d] = calc_q_tgt(rx, ry, vel_nodes); + } + return; + } + + // Vertical Interpolation. + slmm_kernel_assert(cm.dep_points_ndim == 4); + interpolate_vertical(cm.nlev, cm.etai_beg, cm.etai_end, cm.etam, vnode, + src_lid, lev, dep_point[3], rx, ry, v_tgt); +} + +template +struct CalcVData { + typedef slmm::Advecter Adv; + const typename Adv::LocalMeshesD local_meshes; + const typename Adv::Alg::Enum interp_alg; + const slmm::SphereToRef s2r; + const bool traj_3d; + const int dep_points_ndim; + const int nlev; + const Real etai_beg, etai_end; + const typename IslMpi::template ArrayD etam; + + CalcVData (const IslMpi& cm) + : local_meshes(cm.advecter->local_meshes()), + interp_alg(cm.advecter->alg()), + s2r(cm.advecter->s2r()), + traj_3d(cm.traj_3d), + dep_points_ndim(cm.dep_points_ndim), + nlev(cm.nlev), + etai_beg(cm.etai_beg), etai_end(cm.etai_end), + etam(cm.etam) + {} +}; + +template SLMM_KF +void calc_v (const CalcVData& cvd, const VnodeT& vnode, + const Int src_lid, const Int lev, + const Real* const dep_point, Real* const v_tgt) { + // Horizontal interpolation. + Real rx[np], ry[np]; { + Real ref_coord[2]; + const auto& m = cvd.local_meshes(src_lid); + cvd.s2r.calc_sphere_to_ref(src_lid, m, dep_point, + ref_coord[0], ref_coord[1]); + interpolate(cvd.interp_alg, ref_coord, rx, ry); + } + + if (not cvd.traj_3d) { + for (int d = 0; d < cvd.dep_points_ndim; ++d) { + Real vel_nodes[np*np]; + for (int k = 0; k < np*np; ++k) + vel_nodes[k] = vnode(src_lid,lev,k,d); + v_tgt[d] = calc_q_tgt(rx, ry, vel_nodes); + } + return; + } + + // Vertical Interpolation. + slmm_kernel_assert(cvd.dep_points_ndim == 4); + interpolate_vertical(cvd.nlev, cvd.etai_beg, cvd.etai_end, cvd.etam, vnode, + src_lid, lev, dep_point[3], rx, ry, v_tgt); +} + +template +void traj_calc_rmt_next_step (IslMpi& cm, const VnodeT& vnode) { + calc_rmt_q_pass1(cm, true); + const auto ndim = cm.dep_points_ndim; + const auto& rmt_xs = cm.rmt_xs; + const auto& sendbuf = cm.sendbuf; + const auto& recvbuf = cm.recvbuf; + CalcVData cvd(cm); +#ifdef COMPOSE_PORT + ko::parallel_for(ko::RangePolicy(0, cm.nrmt_xs), + COMPOSE_LAMBDA (const Int it) +#else +# ifdef COMPOSE_HORIZ_OPENMP +# pragma omp for +# endif + for (Int it = 0; it < cm.nrmt_xs; ++it) +#endif + { + const Int + ri = rmt_xs(5*it), lid = rmt_xs(5*it + 1), lev = rmt_xs(5*it + 2), + xos = rmt_xs(5*it + 3), vos = ndim*rmt_xs(5*it + 4); + const auto&& xs = recvbuf(ri); + auto&& v = sendbuf(ri); + calc_v(cvd, vnode, lid, lev, &xs(xos), &v(vos)); + } +#ifdef COMPOSE_PORT + ); +#endif +} + +template +void traj_calc_own_next_step (IslMpi& cm, const DepPoints& dep_points, + const VnodeT& vnode, const VdepT& vdep) { + const auto ndim = cm.dep_points_ndim; +#ifdef COMPOSE_PORT + const auto& ed_d = cm.ed_d; + const auto& own_dep_list = cm.own_dep_list; + CalcVData cvd(cm); + const auto f = COMPOSE_LAMBDA (const Int& it) { + const Int tci = own_dep_list(it,0); + const Int tgt_lev = own_dep_list(it,1); + const Int tgt_k = own_dep_list(it,2); + const auto& ed = ed_d(tci); + const Int slid = ed.nbrs(ed.src(tgt_lev, tgt_k)).lid_on_rank; + Real v_tgt[4]; + calc_v(cvd, vnode, slid, tgt_lev, &dep_points(tci,tgt_lev,tgt_k,0), v_tgt); + for (int d = 0; d < ndim; ++d) + vdep(tci,tgt_lev,tgt_k,d) = v_tgt[d]; + }; + ko::parallel_for( + ko::RangePolicy(0, cm.own_dep_list_len), f); +#else + const int tid = get_tid(); + for (Int tci = 0; tci < cm.nelemd; ++tci) { + auto& ed = cm.ed_d(tci); + const Int ned = ed.own.n(); +#ifdef COMPOSE_HORIZ_OPENMP +# pragma omp for +#endif + for (Int idx = 0; idx < ned; ++idx) { + const auto& e = ed.own(idx); + const Int slid = ed.nbrs(ed.src(e.lev, e.k)).lid_on_rank; + Real v_tgt[4]; + calc_v(cm, vnode, slid, e.lev, &dep_points(tci,e.lev,e.k,0), v_tgt); + for (int d = 0; d < ndim; ++d) + vdep(tci,e.lev,e.k,d) = v_tgt[d]; + } + } +#endif +} + +template +void traj_copy_next_step (IslMpi& cm, const VdepT& vdep) { + const auto myrank = cm.p->rank(); + const auto ndim = cm.dep_points_ndim; +#ifdef COMPOSE_PORT + const auto& mylid_with_comm = cm.mylid_with_comm_d; + const auto& ed_d = cm.ed_d; + const auto& recvbufs = cm.recvbuf; + const Int nlid = cm.mylid_with_comm_h.size(); + const Int nlev = cm.nlev, np2 = cm.np2; + const auto f = COMPOSE_LAMBDA (const Int& it) { + const Int tci = mylid_with_comm(it/(np2*nlev)); + const Int rmt_id = it % (np2*nlev); + auto& ed = ed_d(tci); + if (rmt_id >= ed.rmt.size()) return; + const auto& e = ed.rmt(rmt_id); + slmm_kernel_assert(ed.nbrs(ed.src(e.lev, e.k)).rank != myrank); + const Int ri = ed.nbrs(ed.src(e.lev, e.k)).rank_idx; + const auto&& recvbuf = recvbufs(ri); + for (int d = 0; d < ndim; ++d) + vdep(tci,e.lev,e.k,d) = recvbuf(e.q_ptr + d); + }; + ko::parallel_for(ko::RangePolicy(0, nlid*np2*nlev), f); +#else + const int tid = get_tid(); + for (Int ptr = cm.mylid_with_comm_tid_ptr_h(tid), + end = cm.mylid_with_comm_tid_ptr_h(tid+1); + ptr < end; ++ptr) { + const Int tci = cm.mylid_with_comm_d(ptr); + auto& ed = cm.ed_d(tci); + for (const auto& e: ed.rmt) { + slmm_assert(ed.nbrs(ed.src(e.lev, e.k)).rank != myrank); + const Int ri = ed.nbrs(ed.src(e.lev, e.k)).rank_idx; + const auto&& recvbuf = cm.recvbuf(ri); + for (int d = 0; d < ndim; ++d) + vdep(tci,e.lev,e.k,d) = recvbuf(e.q_ptr + d); + } + } +#endif +} + +// vnode and vdep are indexed as (ie,lev,k,dim), On entry, vnode contains nodal +// velocity data. These data are used to provide updates at departure points for +// both own and remote departure points, writing to vdep. dim = 0:2 is for the +// 3D Cartesian representation of the horizontal velocity; dim = 3 is for +// eta_dot. +template void +calc_v_departure (IslMpi& cm, const Int nets, const Int nete, + const Int step, const Real dtsub, + Real* dep_points_r, const Real* vnode_r, Real* vdep_r) +{ + const int np = 4; + + slmm_assert(cm.np == np); + slmm_assert((cm.traj_3d and cm.dep_points_ndim == 4) or + (not cm.traj_3d and cm.dep_points_ndim == 3)); +#ifdef COMPOSE_PORT + slmm_assert(nets == 0 && nete+1 == cm.nelemd); +#endif + + // If step = 0, the departure points are at the nodes and no interpolation is + // needed. calc_v_departure should not have been called; rather, the calling + // routine should use vnode instead of vdep in subsequent calculations. + slmm_assert(step > 0); + + const auto ndim = cm.dep_points_ndim; + +#ifdef COMPOSE_PORT + const auto& vnode = cm.tracer_arrays->vnode; + const auto& vdep = cm.tracer_arrays->vdep; +#else + CA4 vnode(vnode_r, cm.nelemd, cm.nlev, cm.np2, ndim); + CA4< Real> vdep (vdep_r , cm.nelemd, cm.nlev, cm.np2, ndim); +#endif + slmm_assert(vnode.extent_int(3) == ndim); + slmm_assert(vdep .extent_int(3) == ndim); + +#ifdef COMPOSE_PORT + const auto& dep_points = cm.tracer_arrays->dep_points; +#else + DepPointsH dep_points(dep_points_r, cm.nelemd, cm.nlev, cm.np2, ndim); +#endif + slmm_assert(dep_points.extent_int(3) == ndim); + + // See comments in homme::islmpi::step for details. Each substep follows + // essentially the same pattern. + if (cm.mylid_with_comm_tid_ptr_h.capacity() == 0) + init_mylid_with_comm_threaded(cm, nets, nete); + setup_irecv(cm); + analyze_dep_points(cm, nets, nete, dep_points); + pack_dep_points_sendbuf_pass1(cm, true /* trajectory */); + pack_dep_points_sendbuf_pass2(cm, dep_points, true /* trajectory */); + isend(cm); + recv_and_wait_on_send(cm); + traj_calc_rmt_next_step(cm, vnode); + Kokkos::fence(); + isend(cm, true /* want_req */, true /* skip_if_empty */); + setup_irecv(cm, true /* skip_if_empty */); + traj_calc_own_next_step(cm, dep_points, vnode, vdep); + recv(cm, true /* skip_if_empty */); + traj_copy_next_step(cm, vdep); + wait_on_send(cm, true /* skip_if_empty */); +} + +template void calc_v_departure( + IslMpi&, const Int, const Int, const Int, const Real, + Real*, const Real*, Real*); + +} // namespace islmpi +} // namespace homme diff --git a/components/homme/src/share/compose/compose_slmm_islmpi_interpolate.cpp b/components/homme/src/share/compose/compose_slmm_islmpi_interpolate.cpp new file mode 100644 index 00000000000..3076fb89e78 --- /dev/null +++ b/components/homme/src/share/compose/compose_slmm_islmpi_interpolate.cpp @@ -0,0 +1,67 @@ +#include "compose_slmm_islmpi_interpolate.hpp" + +namespace slmm { + +static Int test_gll () { + Int nerr = 0; + const Real tol = 1e2*std::numeric_limits::epsilon(); + GLL gll; + const Real* x, * wt; + for (Int np = 2; np <= 4; ++np) { + for (Int monotone_type = 0; monotone_type <= 1; ++monotone_type) { + const Basis b(np, monotone_type); + gll.get_coef(b, x, wt); + Real sum = 0; + for (Int i = 0; i < b.np; ++i) + sum += wt[i]; + if (std::abs(2 - sum) > tol) { + std::cerr << "test_gll " << np << ", " << monotone_type + << ": 2 - sum = " << 2 - sum << "\n"; + ++nerr; + } + for (Int j = 0; j < b.np; ++j) { + Real gj[GLL::np_max]; + gll.eval(b, x[j], gj); + for (Int i = 0; i < b.np; ++i) { + if (j == i) continue; + if (std::abs(gj[i]) > tol) { + std::cerr << "test_gll " << np << ", " << monotone_type << ": gj[" + << i << "] = " << gj[i] << "\n"; + ++nerr; + } + } + } + } + } + for (Int np = 2; np <= 4; ++np) { + const Basis b(np, 0); + Real a[] = {-0.9, -0.7, -0.3, 0.1, 0.2, 0.4, 0.6, 0.8}; + const Real delta = std::sqrt(std::numeric_limits::epsilon()); + for (size_t ia = 0; ia < sizeof(a)/sizeof(Real); ++ia) { + Real gj[GLL::np_max], gjp[GLL::np_max], gjm[GLL::np_max]; + gll.eval_derivative(b, a[ia], gj); + gll.eval(b, a[ia] + delta, gjp); + gll.eval(b, a[ia] - delta, gjm); + for (Int i = 0; i < b.np; ++i) { + const Real fd = (gjp[i] - gjm[i])/(2*delta); + if (std::abs(fd - gj[i]) >= delta*std::abs(gjp[i])) + ++nerr; + } + } + } + return nerr; +} + +} // namespace slmm + +namespace compose { +namespace test { + +int interpolate_unittest () { + int nerr = 0; + nerr += slmm::test_gll(); + return nerr; +} + +} // namespace test +} // namespace compose diff --git a/components/homme/src/share/compose/compose_slmm_islmpi_interpolate.hpp b/components/homme/src/share/compose/compose_slmm_islmpi_interpolate.hpp new file mode 100644 index 00000000000..29536e9ab9a --- /dev/null +++ b/components/homme/src/share/compose/compose_slmm_islmpi_interpolate.hpp @@ -0,0 +1,142 @@ +#ifndef INCLUDE_COMPOSE_SLMM_ISLMPI_INTERPOLATE_HPP +#define INCLUDE_COMPOSE_SLMM_ISLMPI_INTERPOLATE_HPP + +#include "compose.hpp" +#include "compose_slmm.hpp" +#include "compose_slmm_islmpi.hpp" + +namespace slmm { + +static constexpr Real sqrt5 = 2.23606797749978969641; // std::sqrt(5.0); +static constexpr Real oosqrt5 = 1.0 / sqrt5; + +SLMM_KIF void gll_np4_eval (const Real x, Real y[4]) { + static constexpr Real oo8 = 1.0/8.0; + const Real x2 = x*x; + y[0] = (1.0 - x)*(5.0*x2 - 1.0)*oo8; + y[1] = -sqrt5*oo8*(sqrt5 - 5.0*x)*(x2 - 1.0); + y[2] = -sqrt5*oo8*(sqrt5 + 5.0*x)*(x2 - 1.0); + y[3] = (1.0 + x)*(5.0*x2 - 1.0)*oo8; +} + +// Linear interp in each region. +SLMM_KIF void gll_np4_subgrid_eval_impl (const Real& x, Real y[4]) { + if (x < -oosqrt5) { + const Real alpha = (x + 1)/(1 - oosqrt5); + y[0] = 1 - alpha; + y[1] = alpha; + y[2] = 0; + y[3] = 0; + } else { + const Real alpha = (x + oosqrt5)/(2*oosqrt5); + y[0] = 0; + y[1] = 1 - alpha; + y[2] = alpha; + y[3] = 0; + } +} + +SLMM_KIF void gll_np4_subgrid_eval (const Real& x, Real y[4]) { + if (x > 0) { + gll_np4_subgrid_eval_impl(-x, y); + ko::swap(y[0], y[3]); + ko::swap(y[1], y[2]); + return; + } + gll_np4_subgrid_eval_impl(x, y); +} + +// Quadratic interpolant across nodes 1,2,3 -- i.e., excluding node 0 -- of the +// np=4 reference element. +SLMM_KIF void outer_eval (const Real& x, Real v[4]) { + static constexpr Real + xbar = (2*oosqrt5) / (1 + oosqrt5), + ooxbar = 1 / xbar, + ybar = 1 / (xbar - 1); + const Real xn = (x + oosqrt5) / (1 + oosqrt5); + v[0] = 0; + v[1] = 1 + ybar*xn*((1 - ooxbar)*xn + ooxbar - xbar); + v[2] = ybar*ooxbar*xn*(xn - 1); + v[3] = ybar*xn*(xbar - xn); +} + +// In the middle region, use the standard GLL np=4 interpolant; in the two outer +// regions, use an order-reduced interpolant that stabilizes the method. +SLMM_KIF void gll_np4_subgrid_exp_eval (const Real& x, Real y[4]) { + static constexpr Real + alpha = 0.5527864045000416708, + v = 0.427*(1 + alpha), + x2 = 0.4472135954999579277, + x3 = 1 - x2, + det = x2*x3*(x2 - x3), + y2 = alpha, + y3 = v, + c1 = (x3*y2 - x2*y3)/det, + c2 = (-x3*x3*y2 + x2*x2*y3)/det; + if (x < -oosqrt5 || x > oosqrt5) { + if (x < -oosqrt5) { + outer_eval(-x, y); + ko::swap(y[0], y[3]); + ko::swap(y[1], y[2]); + } else + outer_eval(x, y); + Real y4[4]; + gll_np4_eval(x, y4); + const Real x0 = 1 - std::abs(x); + const Real a = (c1*x0 + c2)*x0; + for (int i = 0; i < 4; ++i) + y[i] = a*y[i] + (1 - a)*y4[i]; + } else + gll_np4_eval(x, y); +} + +} // namespace slmm + +namespace homme { +namespace islmpi { + +template +SLMM_KIF void interpolate (const typename IslMpi::Advecter::Alg::Enum& alg, + const Real ref_coord[2], Real rx[4], Real ry[4]) { + typedef typename IslMpi::Advecter::Alg Alg; + switch (alg) { + case Alg::csl_gll: + slmm::gll_np4_eval(ref_coord[0], rx); + slmm::gll_np4_eval(ref_coord[1], ry); + break; + case Alg::csl_gll_subgrid: + slmm::gll_np4_subgrid_eval(ref_coord[0], rx); + slmm::gll_np4_subgrid_eval(ref_coord[1], ry); + break; + case Alg::csl_gll_exp: + slmm::gll_np4_subgrid_exp_eval(ref_coord[0], rx); + slmm::gll_np4_subgrid_exp_eval(ref_coord[1], ry); + break; + default: + slmm_kernel_assert(0); + } +} + +SLMM_KIF Real calc_q_tgt (const Real rx[4], const Real ry[4], const Real qs[16]) { + return (ry[0]*(rx[0]*qs[ 0] + rx[1]*qs[ 1] + rx[2]*qs[ 2] + rx[3]*qs[ 3]) + + ry[1]*(rx[0]*qs[ 4] + rx[1]*qs[ 5] + rx[2]*qs[ 6] + rx[3]*qs[ 7]) + + ry[2]*(rx[0]*qs[ 8] + rx[1]*qs[ 9] + rx[2]*qs[10] + rx[3]*qs[11]) + + ry[3]*(rx[0]*qs[12] + rx[1]*qs[13] + rx[2]*qs[14] + rx[3]*qs[15])); +} + +SLMM_KIF Real calc_q_tgt (const Real rx[4], const Real ry[4], const Real qdp[16], + const Real dp[16]) { + return (ry[0]*(rx[0]*(qdp[ 0]/dp[ 0]) + rx[1]*(qdp[ 1]/dp[ 1]) + + rx[2]*(qdp[ 2]/dp[ 2]) + rx[3]*(qdp[ 3]/dp[ 3])) + + ry[1]*(rx[0]*(qdp[ 4]/dp[ 4]) + rx[1]*(qdp[ 5]/dp[ 5]) + + rx[2]*(qdp[ 6]/dp[ 6]) + rx[3]*(qdp[ 7]/dp[ 7])) + + ry[2]*(rx[0]*(qdp[ 8]/dp[ 8]) + rx[1]*(qdp[ 9]/dp[ 9]) + + rx[2]*(qdp[10]/dp[10]) + rx[3]*(qdp[11]/dp[11])) + + ry[3]*(rx[0]*(qdp[12]/dp[12]) + rx[1]*(qdp[13]/dp[13]) + + rx[2]*(qdp[14]/dp[14]) + rx[3]*(qdp[15]/dp[15]))); +} + +} // namespace islmpi +} // namespace homme + +#endif diff --git a/components/homme/src/share/compose/compose_slmm_islmpi_pack.cpp b/components/homme/src/share/compose/compose_slmm_islmpi_pack.cpp index 4fe325728df..213226b6c3b 100644 --- a/components/homme/src/share/compose/compose_slmm_islmpi_pack.cpp +++ b/components/homme/src/share/compose/compose_slmm_islmpi_pack.cpp @@ -1,46 +1,14 @@ #include "compose_slmm_islmpi.hpp" +#include "compose_slmm_islmpi_buf.hpp" namespace homme { namespace islmpi { -template SLMM_KIF -Int setbuf (Buffer& buf, const Int& os, const Int& i1, const Int& i2) { - Int* const b = reinterpret_cast(&buf(os)); - b[0] = i1; - b[1] = i2; - return nreal_per_2int; -} - -template SLMM_KIF -Int setbuf (Buffer& buf, const Int& os, const Int& i1, const short& i2, const short& i3) { - static_assert(sizeof(Int) >= 2*sizeof(short), "Need >= 2 shorts per Int"); - Int* const b = reinterpret_cast(&buf(os)); - b[0] = i1; - short* const b2 = reinterpret_cast(b+1); - b2[0] = i2; - b2[1] = i3; - return nreal_per_2int; -} - -template SLMM_KIF -Int setbuf (Buffer& buf, const Int& os, const Int& i1, const Int& i2, - const bool final) { - if (final) setbuf(buf, os, i1, i2); - return nreal_per_2int; -} - -template SLMM_KIF -Int setbuf (Buffer& buf, const Int& os, const Int& i1, const short& i2, const short& i3, - const bool final) { - if (final) setbuf(buf, os, i1, i2, i3); - return nreal_per_2int; -} - #ifdef COMPOSE_PORT /* GPU metadata are arranged differently than described below. The scheme is the following: - (#x-in-rank int - x-bulk-data-offset i + (x-bulk-data-offset int + #x-in-rank i (lid-on-rank i only packed if #x in lid > 0 lev short #x) s @@ -56,7 +24,7 @@ struct Accum { }; template -void pack_dep_points_sendbuf_pass1_scan (IslMpi& cm) { +void pack_dep_points_sendbuf_pass1_scan (IslMpi& cm, const bool trajectory) { ko::fence(); deep_copy(cm.nx_in_rank_h, cm.nx_in_rank); const auto& sendbufs = cm.sendbuf; @@ -68,6 +36,7 @@ void pack_dep_points_sendbuf_pass1_scan (IslMpi& cm) { const auto& blas = cm.bla; const auto nlev = cm.nlev; const Int nrmtrank = static_cast(cm.ranks.size()) - 1; + const Int ndim = trajectory ? cm.dep_points_ndim : 3; for (Int ri = 0; ri < nrmtrank; ++ri) { const Int lid_on_rank_n = cm.lid_on_rank_h(ri).n(); const auto f = COMPOSE_LAMBDA (const int idx, Accum& a, const bool fin) { @@ -97,10 +66,10 @@ void pack_dep_points_sendbuf_pass1_scan (IslMpi& cm) { if (nx > 0) { const auto dos = setbuf(sendbuf, a.mos, lid_on_rank(lidi), lev, nx, fin); a.mos += dos; - a.sendcount += dos + 3*nx; + a.sendcount += dos + ndim*nx; if (fin) t.xptr = a.xos; - a.xos += 3*nx; - a.qos += 2 + nx; + a.xos += ndim*nx; + a.qos += trajectory ? nx : 2 + nx; } }; Accum a; @@ -121,8 +90,8 @@ void pack_dep_points_sendbuf_pass1_scan (IslMpi& cm) { /* Pack the departure points (x). We use two passes. We also set up the q metadata. Two passes let us do some efficient tricks that are not available with one pass. Departure point and q messages are formatted as follows: - xs: (#x-in-rank int <- - x-bulk-data-offset i | + xs: (x-bulk-data-offset int <- + #x-in-rank i | (lid-on-rank i only packed if #x in lid > 0 | #x-in-lid i > 0 |- meta data (lev i only packed if #x in (lid,lev) > 0 | @@ -135,7 +104,7 @@ void pack_dep_points_sendbuf_pass1_scan (IslMpi& cm) { *#x) *#lev *#lid *#rank */ template -void pack_dep_points_sendbuf_pass1_noscan (IslMpi& cm) { +void pack_dep_points_sendbuf_pass1_noscan (IslMpi& cm, const bool trajectory) { #ifdef COMPOSE_PORT ko::fence(); deep_copy(cm.nx_in_rank_h, cm.nx_in_rank); @@ -143,6 +112,7 @@ void pack_dep_points_sendbuf_pass1_noscan (IslMpi& cm) { deep_copy(cm.bla_h, cm.bla); #endif const Int nrmtrank = static_cast(cm.ranks.size()) - 1; + const Int ndim = trajectory ? cm.dep_points_ndim : 3; #ifdef COMPOSE_HORIZ_OPENMP # pragma omp for #endif @@ -179,10 +149,10 @@ void pack_dep_points_sendbuf_pass1_noscan (IslMpi& cm) { slmm_assert_high(nx > 0); const auto dos = setbuf(sendbuf, mos, lev, nx); mos += dos; - sendcount += dos + 3*nx; + sendcount += dos + ndim*nx; t.xptr = xos; - xos += 3*nx; - qos += 2 + nx; + xos += ndim*nx; + qos += trajectory ? nx : 2 + nx; nx_in_lid -= nx; } slmm_assert(nx_in_lid == 0); @@ -210,17 +180,18 @@ void pack_dep_points_sendbuf_pass1_noscan (IslMpi& cm) { } template -void pack_dep_points_sendbuf_pass1 (IslMpi& cm) { +void pack_dep_points_sendbuf_pass1 (IslMpi& cm, const bool trajectory) { #if defined COMPOSE_PORT && ! defined COMPOSE_PACK_NOSCAN if (ko::OnGpu::value) - pack_dep_points_sendbuf_pass1_scan(cm); + pack_dep_points_sendbuf_pass1_scan(cm, trajectory); else #endif - pack_dep_points_sendbuf_pass1_noscan(cm); + pack_dep_points_sendbuf_pass1_noscan(cm, trajectory); } template -void pack_dep_points_sendbuf_pass2 (IslMpi& cm, const DepPoints& dep_points) { +void pack_dep_points_sendbuf_pass2 (IslMpi& cm, const DepPoints& dep_points, + const bool trajectory) { const auto myrank = cm.p->rank(); #ifdef COMPOSE_PORT const Int start = 0, end = cm.mylid_with_comm_h.n(); @@ -242,6 +213,7 @@ void pack_dep_points_sendbuf_pass2 (IslMpi& cm, const DepPoints& dep_poi } { ConstExceptGnu Int np2 = cm.np2, nlev = cm.nlev, qsize = cm.qsize; + ConstExceptGnu Int ndim = trajectory ? cm.dep_points_ndim : 3; const auto& ed_d = cm.ed_d; const auto& mylid_with_comm_d = cm.mylid_with_comm_d; const auto& sendbuf = cm.sendbuf; @@ -279,17 +251,21 @@ void pack_dep_points_sendbuf_pass2 (IslMpi& cm, const DepPoints& dep_poi ++t.cnt; #endif qptr = t.qptr; - xptr = x_bulkdata_offset(ri) + t.xptr + 3*cnt; + xptr = x_bulkdata_offset(ri) + t.xptr + ndim*cnt; } #ifdef COMPOSE_HORIZ_OPENMP if (horiz_openmp) omp_unset_lock(lock); #endif slmm_kernel_assert_high(xptr > 0); - for (Int i = 0; i < 3; ++i) + for (Int i = 0; i < ndim; ++i) sb(xptr + i) = dep_points(tci,lev,k,i); auto& item = ed.rmt.atomic_inc_and_return_next(); - item.q_extrema_ptr = qsize * qptr; - item.q_ptr = item.q_extrema_ptr + qsize*(2 + cnt); + if (trajectory) { + item.q_extrema_ptr = item.q_ptr = ndim*(qptr + cnt); + } else { + item.q_extrema_ptr = qsize * qptr; + item.q_ptr = item.q_extrema_ptr + qsize*(2 + cnt); + } item.lev = lev; item.k = k; }; @@ -300,9 +276,11 @@ void pack_dep_points_sendbuf_pass2 (IslMpi& cm, const DepPoints& dep_poi } } -template void pack_dep_points_sendbuf_pass1(IslMpi& cm); -template void pack_dep_points_sendbuf_pass2(IslMpi& cm, - const DepPoints& dep_points); +template void pack_dep_points_sendbuf_pass1( + IslMpi& cm, const bool trajectory); +template void pack_dep_points_sendbuf_pass2( + IslMpi& cm, const DepPoints& dep_points, + const bool trajectory); } // namespace islmpi } // namespace homme diff --git a/components/homme/src/share/compose/compose_slmm_islmpi_q.cpp b/components/homme/src/share/compose/compose_slmm_islmpi_q.cpp index c6633a0f5ae..aa848de4138 100644 --- a/components/homme/src/share/compose/compose_slmm_islmpi_q.cpp +++ b/components/homme/src/share/compose/compose_slmm_islmpi_q.cpp @@ -1,198 +1,10 @@ #include "compose_slmm_islmpi.hpp" - -namespace slmm { -static Int test_gll () { - Int nerr = 0; - const Real tol = 1e2*std::numeric_limits::epsilon(); - GLL gll; - const Real* x, * wt; - for (Int np = 2; np <= 4; ++np) { - for (Int monotone_type = 0; monotone_type <= 1; ++monotone_type) { - const Basis b(np, monotone_type); - gll.get_coef(b, x, wt); - Real sum = 0; - for (Int i = 0; i < b.np; ++i) - sum += wt[i]; - if (std::abs(2 - sum) > tol) { - std::cerr << "test_gll " << np << ", " << monotone_type - << ": 2 - sum = " << 2 - sum << "\n"; - ++nerr; - } - for (Int j = 0; j < b.np; ++j) { - Real gj[GLL::np_max]; - gll.eval(b, x[j], gj); - for (Int i = 0; i < b.np; ++i) { - if (j == i) continue; - if (std::abs(gj[i]) > tol) { - std::cerr << "test_gll " << np << ", " << monotone_type << ": gj[" - << i << "] = " << gj[i] << "\n"; - ++nerr; - } - } - } - } - } - for (Int np = 2; np <= 4; ++np) { - const Basis b(np, 0); - Real a[] = {-0.9, -0.7, -0.3, 0.1, 0.2, 0.4, 0.6, 0.8}; - const Real delta = std::sqrt(std::numeric_limits::epsilon()); - for (size_t ia = 0; ia < sizeof(a)/sizeof(Real); ++ia) { - Real gj[GLL::np_max], gjp[GLL::np_max], gjm[GLL::np_max]; - gll.eval_derivative(b, a[ia], gj); - gll.eval(b, a[ia] + delta, gjp); - gll.eval(b, a[ia] - delta, gjm); - for (Int i = 0; i < b.np; ++i) { - const Real fd = (gjp[i] - gjm[i])/(2*delta); - if (std::abs(fd - gj[i]) >= delta*std::abs(gjp[i])) - ++nerr; - } - } - } - return nerr; -} - -int unittest () { - int nerr = 0; - nerr += test_gll(); - return nerr; -} - -static constexpr Real sqrt5 = 2.23606797749978969641; // std::sqrt(5.0); -static constexpr Real oosqrt5 = 1.0 / sqrt5; - -SLMM_KF void gll_np4_eval (const Real x, Real y[4]) { - static constexpr Real oo8 = 1.0/8.0; - const Real x2 = x*x; - y[0] = (1.0 - x)*(5.0*x2 - 1.0)*oo8; - y[1] = -sqrt5*oo8*(sqrt5 - 5.0*x)*(x2 - 1.0); - y[2] = -sqrt5*oo8*(sqrt5 + 5.0*x)*(x2 - 1.0); - y[3] = (1.0 + x)*(5.0*x2 - 1.0)*oo8; -} - -// Linear interp in each region. -SLMM_KF void gll_np4_subgrid_eval_impl (const Real& x, Real y[4]) { - if (x < -oosqrt5) { - const Real alpha = (x + 1)/(1 - oosqrt5); - y[0] = 1 - alpha; - y[1] = alpha; - y[2] = 0; - y[3] = 0; - } else { - const Real alpha = (x + oosqrt5)/(2*oosqrt5); - y[0] = 0; - y[1] = 1 - alpha; - y[2] = alpha; - y[3] = 0; - } -} - -SLMM_KF void gll_np4_subgrid_eval (const Real& x, Real y[4]) { - if (x > 0) { - gll_np4_subgrid_eval_impl(-x, y); - ko::swap(y[0], y[3]); - ko::swap(y[1], y[2]); - return; - } - gll_np4_subgrid_eval_impl(x, y); -} - -// Quadratic interpolant across nodes 1,2,3 -- i.e., excluding node 0 -- of the -// np=4 reference element. -SLMM_KF void outer_eval (const Real& x, Real v[4]) { - static constexpr Real - xbar = (2*oosqrt5) / (1 + oosqrt5), - ooxbar = 1 / xbar, - ybar = 1 / (xbar - 1); - const Real xn = (x + oosqrt5) / (1 + oosqrt5); - v[0] = 0; - v[1] = 1 + ybar*xn*((1 - ooxbar)*xn + ooxbar - xbar); - v[2] = ybar*ooxbar*xn*(xn - 1); - v[3] = ybar*xn*(xbar - xn); -} - -// In the middle region, use the standard GLL np=4 interpolant; in the two outer -// regions, use an order-reduced interpolant that stabilizes the method. -SLMM_KF void gll_np4_subgrid_exp_eval (const Real& x, Real y[4]) { - static constexpr Real - alpha = 0.5527864045000416708, - v = 0.427*(1 + alpha), - x2 = 0.4472135954999579277, - x3 = 1 - x2, - det = x2*x3*(x2 - x3), - y2 = alpha, - y3 = v, - c1 = (x3*y2 - x2*y3)/det, - c2 = (-x3*x3*y2 + x2*x2*y3)/det; - if (x < -oosqrt5 || x > oosqrt5) { - if (x < -oosqrt5) { - outer_eval(-x, y); - ko::swap(y[0], y[3]); - ko::swap(y[1], y[2]); - } else - outer_eval(x, y); - Real y4[4]; - gll_np4_eval(x, y4); - const Real x0 = 1 - std::abs(x); - const Real a = (c1*x0 + c2)*x0; - for (int i = 0; i < 4; ++i) - y[i] = a*y[i] + (1 - a)*y4[i]; - } else - gll_np4_eval(x, y); -} -} // namespace slmm +#include "compose_slmm_islmpi_interpolate.hpp" +#include "compose_slmm_islmpi_buf.hpp" namespace homme { namespace islmpi { -template -SLMM_KIF void interpolate (const typename IslMpi::Advecter::Alg::Enum& alg, - const Real ref_coord[2], Real rx[4], Real ry[4]) { - typedef typename IslMpi::Advecter::Alg Alg; - switch (alg) { - case Alg::csl_gll: - slmm::gll_np4_eval(ref_coord[0], rx); - slmm::gll_np4_eval(ref_coord[1], ry); - break; - case Alg::csl_gll_subgrid: - slmm::gll_np4_subgrid_eval(ref_coord[0], rx); - slmm::gll_np4_subgrid_eval(ref_coord[1], ry); - break; - case Alg::csl_gll_exp: - slmm::gll_np4_subgrid_exp_eval(ref_coord[0], rx); - slmm::gll_np4_subgrid_exp_eval(ref_coord[1], ry); - break; - default: - slmm_kernel_assert(0); - } -} - -SLMM_KIF Real calc_q_tgt (const Real rx[4], const Real ry[4], const Real qs[16]) { - return (ry[0]*(rx[0]*qs[ 0] + rx[1]*qs[ 1] + rx[2]*qs[ 2] + rx[3]*qs[ 3]) + - ry[1]*(rx[0]*qs[ 4] + rx[1]*qs[ 5] + rx[2]*qs[ 6] + rx[3]*qs[ 7]) + - ry[2]*(rx[0]*qs[ 8] + rx[1]*qs[ 9] + rx[2]*qs[10] + rx[3]*qs[11]) + - ry[3]*(rx[0]*qs[12] + rx[1]*qs[13] + rx[2]*qs[14] + rx[3]*qs[15])); -} - -SLMM_KIF Real calc_q_tgt (const Real rx[4], const Real ry[4], const Real qdp[16], - const Real dp[16]) { - return (ry[0]*(rx[0]*(qdp[ 0]/dp[ 0]) + rx[1]*(qdp[ 1]/dp[ 1]) + - rx[2]*(qdp[ 2]/dp[ 2]) + rx[3]*(qdp[ 3]/dp[ 3])) + - ry[1]*(rx[0]*(qdp[ 4]/dp[ 4]) + rx[1]*(qdp[ 5]/dp[ 5]) + - rx[2]*(qdp[ 6]/dp[ 6]) + rx[3]*(qdp[ 7]/dp[ 7])) + - ry[2]*(rx[0]*(qdp[ 8]/dp[ 8]) + rx[1]*(qdp[ 9]/dp[ 9]) + - rx[2]*(qdp[10]/dp[10]) + rx[3]*(qdp[11]/dp[11])) + - ry[3]*(rx[0]*(qdp[12]/dp[12]) + rx[1]*(qdp[13]/dp[13]) + - rx[2]*(qdp[14]/dp[14]) + rx[3]*(qdp[15]/dp[15]))); -} - -template SLMM_KIF -Int getbuf (Buffer& buf, const Int& os, Int& i1, Int& i2) { - const Int* const b = reinterpret_cast(&buf(os)); - i1 = b[0]; - i2 = b[1]; - return nreal_per_2int; -} - #ifndef COMPOSE_PORT // Homme computational pattern. @@ -267,7 +79,7 @@ void calc_own_q (IslMpi& cm, const Int& nets, const Int& nete, auto& ed = cm.ed_d(tci); const FA3 q_tgt(ed.q, cm.np2, cm.nlev, cm.qsize); const Int ned = ed.own.n(); -#ifdef HORIZ_OPENMP +#ifdef COMPOSE_HORIZ_OPENMP # pragma omp for #endif for (Int idx = 0; idx < ned; ++idx) { @@ -317,7 +129,7 @@ template void calc_rmt_q_pass2 (IslMpi& cm) { const Int qsize = cm.qsize; -#ifdef HORIZ_OPENMP +#ifdef COMPOSE_HORIZ_OPENMP # pragma omp for #endif for (Int it = 0; it < cm.nrmt_qs_extrema; ++it) { @@ -331,7 +143,7 @@ void calc_rmt_q_pass2 (IslMpi& cm) { qs(qos + 2*iq + i) = ed.q_extrema(iq, lev, i); } -#ifdef HORIZ_OPENMP +#ifdef COMPOSE_HORIZ_OPENMP # pragma omp for #endif for (Int it = 0; it < cm.nrmt_xs; ++it) { @@ -466,12 +278,13 @@ struct Accum { } }; -template -void calc_rmt_q_pass1_scan (IslMpi& cm) { +template +void calc_rmt_q_pass1_scan (IslMpi& cm, const bool trajectory) { const auto& recvbuf = cm.recvbuf; const auto& rmt_xs = cm.rmt_xs; const auto& rmt_qs_extrema = cm.rmt_qs_extrema; const Int nrmtrank = static_cast(cm.ranks.size()) - 1; + const Int ndim = trajectory ? cm.dep_points_ndim : 3; Int cnt = 0, qcnt = 0; for (Int ri = 0; ri < nrmtrank; ++ri) { const auto get_xos = COMPOSE_LAMBDA (const Int, Int& xos) { @@ -492,7 +305,7 @@ void calc_rmt_q_pass1_scan (IslMpi& cm) { short lev, nx; getbuf(xs, (idx + 1)*nreal_per_2int, lid, lev, nx); slmm_kernel_assert(nx > 0); - if (fin) { + if (fin && ! trajectory) { const auto qcnt_tot = qcnt + a.qcnt; rmt_qs_extrema(4*qcnt_tot + 0) = ri; rmt_qs_extrema(4*qcnt_tot + 1) = lid; @@ -500,7 +313,8 @@ void calc_rmt_q_pass1_scan (IslMpi& cm) { rmt_qs_extrema(4*qcnt_tot + 3) = a.qos; } a.qcnt += 1; - a.qos += 2; + if ( ! trajectory) + a.qos += 2; if (fin) { for (Int xi = 0; xi < nx; ++xi) { const auto cnt_tot = cnt + a.cnt; @@ -510,23 +324,23 @@ void calc_rmt_q_pass1_scan (IslMpi& cm) { rmt_xs(5*cnt_tot + 3) = xos + a.xos; rmt_xs(5*cnt_tot + 4) = a.qos; a.cnt += 1; - a.xos += 3; + a.xos += ndim; a.qos += 1; } } else { a.cnt += nx; - a.xos += 3*nx; - a.qos += nx; + a.xos += ndim*nx; + a.qos += nx; } }; Accum a; ko::parallel_scan(ko::RangePolicy(0, xos/nreal_per_2int - 1), f, a); - cm.sendcount_h(ri) = cm.qsize*a.qos; + cm.sendcount_h(ri) = (trajectory ? ndim : cm.qsize)*a.qos; cnt += a.cnt; qcnt += a.qcnt; } cm.nrmt_xs = cnt; - cm.nrmt_qs_extrema = qcnt; + cm.nrmt_qs_extrema = trajectory ? 0 : qcnt; } template @@ -593,13 +407,20 @@ void calc_rmt_q_pass2 (IslMpi& cm) { #endif // COMPOSE_PORT -template -void calc_rmt_q_pass1_noscan (IslMpi& cm) { +template +void calc_rmt_q_pass1_noscan (IslMpi& cm, const bool trajectory) { const Int nrmtrank = static_cast(cm.ranks.size()) - 1; + const Int ndim = trajectory ? cm.dep_points_ndim : 3; #ifdef COMPOSE_PORT_SEPARATE_VIEWS +#ifdef COMPOSE_HORIZ_OPENMP +# pragma omp for +#endif for (Int ri = 0; ri < nrmtrank; ++ri) ko::deep_copy(ko::View(cm.recvbuf_meta_h(ri).data(), 1), ko::View(cm.recvbuf.get_h(ri).data(), 1)); +#ifdef COMPOSE_HORIZ_OPENMP +# pragma omp for +#endif for (Int ri = 0; ri < nrmtrank; ++ri) { const auto&& xs = cm.recvbuf_meta_h(ri); Int n, unused; @@ -610,71 +431,79 @@ void calc_rmt_q_pass1_noscan (IslMpi& cm) { ko::View(cm.recvbuf.get_h(ri).data(), n)); } #endif - Int cnt = 0, qcnt = 0; - for (Int ri = 0; ri < nrmtrank; ++ri) { - const auto&& xs = cm.recvbuf_meta_h(ri); - Int mos = 0, qos = 0, nx_in_rank, xos; - mos += getbuf(xs, mos, xos, nx_in_rank); - if (nx_in_rank == 0) { - cm.sendcount_h(ri) = 0; - continue; - } - // The upper bound is to prevent an inf loop if the msg is corrupted. - for (Int lidi = 0; lidi < cm.nelemd; ++lidi) { - Int lid, nx_in_lid; - mos += getbuf(xs, mos, lid, nx_in_lid); - for (Int levi = 0; levi < cm.nlev; ++levi) { // same re: inf loop - Int lev, nx; - mos += getbuf(xs, mos, lev, nx); - slmm_assert(nx > 0); - { - cm.rmt_qs_extrema_h(4*qcnt + 0) = ri; - cm.rmt_qs_extrema_h(4*qcnt + 1) = lid; - cm.rmt_qs_extrema_h(4*qcnt + 2) = lev; - cm.rmt_qs_extrema_h(4*qcnt + 3) = qos; - ++qcnt; - qos += 2; - } - for (Int xi = 0; xi < nx; ++xi) { - cm.rmt_xs_h(5*cnt + 0) = ri; - cm.rmt_xs_h(5*cnt + 1) = lid; - cm.rmt_xs_h(5*cnt + 2) = lev; - cm.rmt_xs_h(5*cnt + 3) = xos; - cm.rmt_xs_h(5*cnt + 4) = qos; - ++cnt; - xos += 3; - ++qos; +#ifdef COMPOSE_HORIZ_OPENMP +# pragma omp master +#endif + { + Int cnt = 0, qcnt = 0; + for (Int ri = 0; ri < nrmtrank; ++ri) { + const auto&& xs = cm.recvbuf_meta_h(ri); + Int mos = 0, qos = 0, nx_in_rank, xos; + mos += getbuf(xs, mos, xos, nx_in_rank); + if (nx_in_rank == 0) { + cm.sendcount_h(ri) = 0; + continue; + } + // The upper bound is to prevent an inf loop if the msg is corrupted. + for (Int lidi = 0; lidi < cm.nelemd; ++lidi) { + Int lid, nx_in_lid; + mos += getbuf(xs, mos, lid, nx_in_lid); + for (Int levi = 0; levi < cm.nlev; ++levi) { // same re: inf loop + Int lev, nx; + mos += getbuf(xs, mos, lev, nx); + slmm_assert(nx > 0); + if ( ! trajectory) { + cm.rmt_qs_extrema_h(4*qcnt + 0) = ri; + cm.rmt_qs_extrema_h(4*qcnt + 1) = lid; + cm.rmt_qs_extrema_h(4*qcnt + 2) = lev; + cm.rmt_qs_extrema_h(4*qcnt + 3) = qos; + ++qcnt; + qos += 2; + } + for (Int xi = 0; xi < nx; ++xi) { + cm.rmt_xs_h(5*cnt + 0) = ri; + cm.rmt_xs_h(5*cnt + 1) = lid; + cm.rmt_xs_h(5*cnt + 2) = lev; + cm.rmt_xs_h(5*cnt + 3) = xos; + cm.rmt_xs_h(5*cnt + 4) = qos; + ++cnt; + xos += ndim; + ++qos; + } + nx_in_lid -= nx; + nx_in_rank -= nx; + if (nx_in_lid == 0) break; } - nx_in_lid -= nx; - nx_in_rank -= nx; - if (nx_in_lid == 0) break; + slmm_assert(nx_in_lid == 0); + if (nx_in_rank == 0) break; } - slmm_assert(nx_in_lid == 0); - if (nx_in_rank == 0) break; + slmm_assert(nx_in_rank == 0); + cm.sendcount_h(ri) = (trajectory ? ndim : cm.qsize)*qos; } - slmm_assert(nx_in_rank == 0); - cm.sendcount_h(ri) = cm.qsize*qos; + cm.nrmt_xs = cnt; + cm.nrmt_qs_extrema = trajectory ? 0 : qcnt; + deep_copy(cm.rmt_xs, cm.rmt_xs_h); + deep_copy(cm.rmt_qs_extrema, cm.rmt_qs_extrema_h); } - cm.nrmt_xs = cnt; - cm.nrmt_qs_extrema = qcnt; - deep_copy(cm.rmt_xs, cm.rmt_xs_h); - deep_copy(cm.rmt_qs_extrema, cm.rmt_qs_extrema_h); +#ifdef COMPOSE_HORIZ_OPENMP +# pragma omp barrier +#endif } -template -void calc_rmt_q_pass1 (IslMpi& cm) { +template +void calc_rmt_q_pass1 (IslMpi& cm, const bool trajectory) { #if defined COMPOSE_PORT && ! defined COMPOSE_PACK_NOSCAN if (ko::OnGpu::value) - calc_rmt_q_pass1_scan(cm); + calc_rmt_q_pass1_scan(cm, trajectory); else #endif - calc_rmt_q_pass1_noscan(cm); + calc_rmt_q_pass1_noscan(cm, trajectory); } template void calc_rmt_q (IslMpi& cm) { { slmm::Timer t("09_rmt_q_pass1"); - calc_rmt_q_pass1(cm); } + calc_rmt_q_pass1(cm); } { slmm::Timer t("09_rmt_q_pass2"); calc_rmt_q_pass2(cm); } } @@ -697,6 +526,8 @@ void calc_rmt_q (IslMpi& cm) { } } +template void calc_rmt_q_pass1(IslMpi& cm, + const bool trajectory); template void calc_rmt_q(IslMpi& cm); template void calc_own_q(IslMpi& cm, const Int& nets, const Int& nete, diff --git a/components/homme/src/share/compose/compose_slmm_islmpi_step.cpp b/components/homme/src/share/compose/compose_slmm_islmpi_step.cpp index 1ae91c3e538..3c707c8919f 100644 --- a/components/homme/src/share/compose/compose_slmm_islmpi_step.cpp +++ b/components/homme/src/share/compose/compose_slmm_islmpi_step.cpp @@ -24,11 +24,13 @@ void step ( const auto& q_min = cm.tracer_arrays->q_min; const auto& q_max = cm.tracer_arrays->q_max; #else - const DepPointsH dep_points(dep_points_r, cm.nelemd, cm.nlev, cm.np2); + const DepPointsH dep_points(dep_points_r, cm.nelemd, cm.nlev, cm.np2, + cm.dep_points_ndim); const QExtremaH q_min(q_min_r, cm.nelemd, cm.qsize, cm.nlev, cm.np2), q_max(q_max_r, cm.nelemd, cm.qsize, cm.nlev, cm.np2); #endif + slmm_assert(dep_points.extent_int(3) == cm.dep_points_ndim); // Partition my elements that communicate with remotes among threads, if I // haven't done that yet. diff --git a/components/homme/src/share/compose/compose_test.cpp b/components/homme/src/share/compose/compose_test.cpp index 4b29f3ae7c3..944114d7975 100644 --- a/components/homme/src/share/compose/compose_test.cpp +++ b/components/homme/src/share/compose/compose_test.cpp @@ -327,7 +327,7 @@ struct StandaloneTracersTester { #endif } }; - + static StandaloneTracersTester::Ptr g_stt; } // namespace test } // namespace compose diff --git a/components/homme/src/share/compose/compose_test.hpp b/components/homme/src/share/compose/compose_test.hpp index 9a3e7d4b350..ff4d18cdfd9 100644 --- a/components/homme/src/share/compose/compose_test.hpp +++ b/components/homme/src/share/compose/compose_test.hpp @@ -13,6 +13,7 @@ namespace test { int slmm_unittest(); int cedr_unittest(); int cedr_unittest(MPI_Comm comm); +int interpolate_unittest(); typedef double Real; typedef int Int; diff --git a/components/homme/src/share/compose_mod.F90 b/components/homme/src/share/compose_mod.F90 index 8b134def384..6085eaf1238 100644 --- a/components/homme/src/share/compose_mod.F90 +++ b/components/homme/src/share/compose_mod.F90 @@ -25,22 +25,24 @@ subroutine cedr_unittest(comm, nerr) bind(c) end subroutine cedr_unittest subroutine cedr_init_impl(comm, cdr_alg, use_sgi, gid_data, rank_data, & - ncell, nlclcell, nlev, qsize, independent_time_steps, hard_zero, & + ncell, nlclcell, nlev, np, qsize, independent_time_steps, hard_zero, & gid_data_sz, rank_data_sz) bind(c) use iso_c_binding, only: c_int, c_bool - integer(kind=c_int), value, intent(in) :: comm, cdr_alg, ncell, nlclcell, nlev, & + integer(kind=c_int), value, intent(in) :: comm, cdr_alg, ncell, nlclcell, nlev, np, & qsize, gid_data_sz, rank_data_sz logical(kind=c_bool), value, intent(in) :: use_sgi, independent_time_steps, hard_zero integer(kind=c_int), intent(in) :: gid_data(gid_data_sz), rank_data(rank_data_sz) end subroutine cedr_init_impl subroutine slmm_init_impl(comm, transport_alg, np, nlev, qsize, qsize_d, & - nelem, nelemd, cubed_sphere_map, geometry, lid2gid, lid2facenum, nbr_id_rank, nirptr, & - sl_nearest_point_lev, lid2gid_sz, lid2facenum_sz, nbr_id_rank_sz, nirptr_sz) bind(c) + nelem, nelemd, cubed_sphere_map, geometry, lid2gid, lid2facenum, & + nbr_id_rank, nirptr, sl_halo, sl_traj_3d, sl_traj_nsubstep, sl_nearest_point_lev, & + lid2gid_sz, lid2facenum_sz, nbr_id_rank_sz, nirptr_sz) bind(c) use iso_c_binding, only: c_int - integer(kind=c_int), value, intent(in) :: comm, transport_alg, np, nlev, qsize, qsize_d, & - nelem, nelemd, cubed_sphere_map, geometry, sl_nearest_point_lev, lid2gid_sz, & - lid2facenum_sz, nbr_id_rank_sz, nirptr_sz + integer(kind=c_int), value, intent(in) :: comm, transport_alg, np, nlev, qsize, & + qsize_d, nelem, nelemd, cubed_sphere_map, geometry, sl_halo, sl_traj_3d, & + sl_traj_nsubstep, sl_nearest_point_lev, lid2gid_sz, lid2facenum_sz, & + nbr_id_rank_sz, nirptr_sz integer(kind=c_int), intent(in) :: lid2gid(lid2gid_sz), lid2facenum(lid2facenum_sz), & nbr_id_rank(nbr_id_rank_sz), nirptr(nirptr_sz) end subroutine slmm_init_impl @@ -186,7 +188,28 @@ subroutine slmm_check_ref2sphere(ie, sphere_cart_coord) bind(c) type(cartesian3D_t), intent(in) :: sphere_cart_coord end subroutine slmm_check_ref2sphere - subroutine slmm_csl_set_elem_data(ie, metdet, qdp, n0_qdp, dp, q, nelem_in_patch, h2d, d2h) bind(c) + subroutine slmm_set_hvcoord(etai_beg, etai_end, etam) bind(c) + use iso_c_binding, only: c_double + use dimensions_mod, only : nlev + real(kind=c_double), value, intent(in) :: etai_beg, etai_end + real(kind=c_double), intent(in) :: etam(nlev) + end subroutine slmm_set_hvcoord + + subroutine slmm_calc_v_departure(nets, nete, step, dtsub, dep_points, & + dep_points_ndim, vnode, vdep, info) bind(c) + use iso_c_binding, only: c_int, c_double + use dimensions_mod, only : np, nlev, nelemd, qsize + use coordinate_systems_mod, only : cartesian3D_t + integer(kind=c_int), value, intent(in) :: nets, nete, step, dep_points_ndim + real(kind=c_double), value, intent(in) :: dtsub + real(kind=c_double), intent(inout) :: dep_points(dep_points_ndim,np,np,nlev,nelemd) + real(kind=c_double), intent(in) :: vnode(dep_points_ndim,np,np,nlev,nelemd) + real(kind=c_double), intent(out) :: vdep(dep_points_ndim,np,np,nlev,nelemd) + integer(kind=c_int), intent(out) :: info + end subroutine slmm_calc_v_departure + + subroutine slmm_csl_set_elem_data(ie, metdet, qdp, n0_qdp, dp, q, nelem_in_patch, & + h2d, d2h) bind(c) use iso_c_binding, only: c_int, c_double, c_bool use dimensions_mod, only : nlev, np, qsize real(kind=c_double), intent(in) :: metdet(np,np), qdp(np,np,nlev,qsize,2), & @@ -195,17 +218,17 @@ subroutine slmm_csl_set_elem_data(ie, metdet, qdp, n0_qdp, dp, q, nelem_in_patch logical(kind=c_bool), value, intent(in) :: h2d, d2h end subroutine slmm_csl_set_elem_data - subroutine slmm_csl(nets, nete, dep_points, minq, maxq, info) bind(c) + subroutine slmm_csl(nets, nete, dep_points, dep_points_ndim, minq, maxq, info) bind(c) use iso_c_binding, only: c_int, c_double use dimensions_mod, only : np, nlev, nelemd, qsize use coordinate_systems_mod, only : cartesian3D_t - integer(kind=c_int), value, intent(in) :: nets, nete + integer(kind=c_int), value, intent(in) :: nets, nete, dep_points_ndim ! dep_points is const in principle, but if lev <= ! semi_lagrange_nearest_point_lev, a departure point may be altered if ! the winds take it outside of the comm halo. - type(cartesian3D_t), intent(inout) :: dep_points(np,np,nlev,nelemd) + real(kind=c_double), intent(inout) :: dep_points(dep_points_ndim,np,np,nlev,nelemd) real(kind=c_double), intent(in) :: & - minq(np,np,nlev,qsize,nets:nete), maxq(np,np,nlev,qsize,nets:nete) + minq(np,np,nlev,qsize,nelemd), maxq(np,np,nlev,qsize,nelemd) integer(kind=c_int), intent(out) :: info end subroutine slmm_csl @@ -238,6 +261,7 @@ subroutine compose_init(par, elem, GridVertex, init_kokkos) use element_mod, only: element_t use gridgraph_mod, only: GridVertex_t use control_mod, only: semi_lagrange_cdr_alg, transport_alg, cubed_sphere_map, & + semi_lagrange_halo, semi_lagrange_trajectory_nsubstep, & semi_lagrange_nearest_point_lev, dt_remap_factor, dt_tracer_factor, geometry use physical_constants, only: Sx, Sy, Lx, Ly use scalable_grid_init_mod, only: sgi_is_initialized, sgi_get_rank2sfc, & @@ -254,7 +278,7 @@ subroutine compose_init(par, elem, GridVertex, init_kokkos) ! These are for non-scalable grid initialization, still used for RRM. sc2gci(:), sc2rank(:) ! space curve index -> (GID, rank) integer :: lid2gid(nelemd), lid2facenum(nelemd) - integer :: i, j, k, sfc, gid, igv, sc, geometry_type + integer :: i, j, k, sfc, gid, igv, sc, geometry_type, sl_traj_3d ! To map SFC index to IDs and ranks logical(kind=c_bool) :: use_sgi, owned, independent_time_steps, hard_zero integer, allocatable :: owned_ids(:) @@ -273,6 +297,16 @@ subroutine compose_init(par, elem, GridVertex, init_kokkos) hard_zero = .true. independent_time_steps = dt_remap_factor < dt_tracer_factor + + if (semi_lagrange_halo < 1) then + ! For test problems, the relationship between dt_tracer_factor and halo + ! may not be clear. But for real problems, the advective CFL implies that + ! a parcel can cross a cell in three time steps. Since this is closely + ! related to the dynamics' tstep, dt_tracer_factor is meaningful, + ! implying: + semi_lagrange_halo = (dt_tracer_factor + 2) / 3 + if (semi_lagrange_halo < 1) semi_lagrange_halo = 1 + end if geometry_type = 0 ! sphere if (trim(geometry) == "plane") then @@ -316,12 +350,12 @@ subroutine compose_init(par, elem, GridVertex, init_kokkos) if (use_sgi) then if (.not. allocated(owned_ids)) allocate(owned_ids(1)) call cedr_init_impl(par%comm, semi_lagrange_cdr_alg, & - use_sgi, owned_ids, rank2sfc, nelem, nelemd, nlev, qsize, & + use_sgi, owned_ids, rank2sfc, nelem, nelemd, nlev, np, qsize, & independent_time_steps, hard_zero, size(owned_ids), size(rank2sfc)) else if (.not. allocated(sc2gci)) allocate(sc2gci(1), sc2rank(1)) call cedr_init_impl(par%comm, semi_lagrange_cdr_alg, & - use_sgi, sc2gci, sc2rank, nelem, nelemd, nlev, qsize, & + use_sgi, sc2gci, sc2rank, nelem, nelemd, nlev, np, qsize, & independent_time_steps, hard_zero, size(sc2gci), size(sc2rank)) end if if (allocated(sc2gci)) deallocate(sc2gci, sc2rank) @@ -360,9 +394,12 @@ subroutine compose_init(par, elem, GridVertex, init_kokkos) end do end do nirptr(nelemd+1) = k - 1 + sl_traj_3d = 0 + if (independent_time_steps) sl_traj_3d = 1 call slmm_init_impl(par%comm, transport_alg, np, nlev, qsize, qsize_d, & nelem, nelemd, cubed_sphere_map, geometry_type, lid2gid, lid2facenum, & - nbr_id_rank, nirptr, semi_lagrange_nearest_point_lev, & + nbr_id_rank, nirptr, semi_lagrange_halo, sl_traj_3d, & + semi_lagrange_trajectory_nsubstep, semi_lagrange_nearest_point_lev, & size(lid2gid), size(lid2facenum), size(nbr_id_rank), size(nirptr)) if (geometry_type == 1) call slmm_init_plane(Sx, Sy, Lx, Ly) deallocate(nbr_id_rank, nirptr) diff --git a/components/homme/src/share/compose_test_mod.F90 b/components/homme/src/share/compose_test_mod.F90 index 8421e41e5fa..8ba3b7f44d7 100644 --- a/components/homme/src/share/compose_test_mod.F90 +++ b/components/homme/src/share/compose_test_mod.F90 @@ -117,7 +117,7 @@ subroutine compose_test(par, hvcoord, dom_mt, elem, eval) ! 1. Unit tests. call compose_unittest() - call sl_unittest(par) + call sl_unittest(par, hvcoord) nerr = 0 call cedr_unittest(par%comm, nerr) if (nerr /= 0) print *, 'cedr_unittest returned', nerr @@ -240,7 +240,7 @@ subroutine compose_stt(hybrid, dom_mt, nets, nete, hvcoord, deriv, elem, eval) ! nsteps = nint(6*ne*(15.d0/qsplit)) nsteps = nmax if (hybrid%par%masterproc .and. hybrid%masterthread) then - print *, 'COMPOSE> nsteps', nsteps + print '(a,i6,a,i5)', 'COMPOSE> nsteps ', nsteps, ' ne ', ne end if dt = twelve_days / nsteps call t_barrierf('compose_stt_step_start_barrier', hybrid%par%comm) diff --git a/components/homme/src/share/control_mod.F90 b/components/homme/src/share/control_mod.F90 index 0e9494f5a6c..0e0276fbf2f 100644 --- a/components/homme/src/share/control_mod.F90 +++ b/components/homme/src/share/control_mod.F90 @@ -26,6 +26,8 @@ module control_mod ! 3 CAAS ! 20 QLT with superlevels ! 30 CAAS with superlevels + ! 4* reserved for debugging + ! 5 CAAS-point integer, public :: semi_lagrange_cdr_alg = 3 ! If true, check mass conservation and shape preservation. The second ! implicitly checks tracer consistency. @@ -39,6 +41,10 @@ module control_mod ! halo available to it if the actual point is outside the halo. This is done ! in levels <= this parameter. integer, public :: semi_lagrange_nearest_point_lev = 256 + integer, public :: semi_lagrange_halo = -1 + integer, public :: semi_lagrange_trajectory_nsubstep = 0 + integer, public :: semi_lagrange_trajectory_nvelocity = -1 + integer, public :: semi_lagrange_diagnostics = 0 ! flag used by preqx, theta-l and theta-c models ! should be renamed to "hydrostatic_mode" diff --git a/components/homme/src/share/cxx/ComposeTransport.cpp b/components/homme/src/share/cxx/ComposeTransport.cpp index 760a3dd06fa..8d734909328 100644 --- a/components/homme/src/share/cxx/ComposeTransport.cpp +++ b/components/homme/src/share/cxx/ComposeTransport.cpp @@ -69,9 +69,13 @@ std::vector > ComposeTransport::run_unit_tests () { assert(is_setup); std::vector > fails; - int nerr; - nerr = m_compose_impl->run_trajectory_unit_tests(); - if (nerr) fails.push_back(std::make_pair("run_trajectory_unit_tests", nerr)); + int ne, nerr = 0; + ne = m_compose_impl->run_trajectory_unit_tests(); + if (ne) fails.push_back(std::make_pair("run_trajectory_unit_tests", nerr)); + nerr += ne; + ne = m_compose_impl->run_enhanced_trajectory_unit_tests(); + if (ne) fails.push_back(std::make_pair("run_enhanced_trajectory_unit_tests", nerr)); + nerr += ne; return fails; } diff --git a/components/homme/src/share/cxx/ComposeTransportImpl.hpp b/components/homme/src/share/cxx/ComposeTransportImpl.hpp index 09bd43d9d53..b536b8ba65a 100644 --- a/components/homme/src/share/cxx/ComposeTransportImpl.hpp +++ b/components/homme/src/share/cxx/ComposeTransportImpl.hpp @@ -41,7 +41,6 @@ struct ComposeTransportImpl { enum : int { max_num_lev_pack = NUM_LEV_P }; enum : int { max_num_lev_aligned = max_num_lev_pack*packn }; enum : int { num_phys_lev = NUM_PHYSICAL_LEV }; - enum : int { num_work = 12 }; static_assert(max_num_lev_aligned >= 3, "We use wrk(0:2,:) and so need max_num_lev_aligned >= 3"); @@ -49,21 +48,56 @@ struct ComposeTransportImpl { using TeamPolicy = Kokkos::TeamPolicy; using MT = typename TeamPolicy::member_type; - using Buf1 = ExecViewUnmanaged; + // For the enhanced trajectory, we need one extra level beyond the usual. + using Buf1Alloc = ExecViewUnmanaged; + using Buf1o = ExecViewUnmanaged; + using Buf1e = Buf1Alloc; + using Buf2 = ExecViewUnmanaged; - using DeparturePoints = ExecViewManaged; + using DeparturePoints = ExecViewManaged; + + typedef ExecViewUnmanaged SNlev; + typedef ExecViewUnmanaged RNlev; + typedef ExecViewUnmanaged SNlevp; + typedef ExecViewUnmanaged RNlevp; + typedef ExecViewUnmanaged S2Nlev; + typedef ExecViewUnmanaged R2Nlev; + typedef ExecViewUnmanaged S2Nlevp; + typedef typename ViewConst::type CSNlev; + typedef typename ViewConst::type CRNlev; + typedef typename ViewConst::type CSNlevp; + typedef typename ViewConst::type CRNlevp; + typedef typename ViewConst::type CS2Nlev; + typedef typename ViewConst::type CR2Nlev; + + using DpSlot = ExecViewUnmanaged< Scalar** [NP][NP][NUM_LEV]>; + using VSlot = ExecViewUnmanaged< Scalar**[2][NP][NP][NUM_LEV]>; + using CDpSlot = ExecViewUnmanaged; + using CVSlot = ExecViewUnmanaged; + struct VelocityRecord; struct Data { int nelemd, qsize, limiter_option, cdr_check, hv_q, hv_subcycle_q; int geometry_type; // 0: sphere, 1: plane - Real nu_q, hv_scaling, dp_tol; + int trajectory_nsubstep; // 0: original alg, >= 1: enhanced + Real nu_q, hv_scaling, dp_tol, deta_tol; bool independent_time_steps; - Buf1 buf1[3]; - Buf2 buf2[2]; + // buf1o and buf1e point to the same memory, sized to the larger of the + // two. They are used in different parts of the code. + static constexpr int n_buf1 = 4, n_buf2 = 4; + Buf1o buf1o[n_buf1]; + Buf1e buf1e[n_buf1]; + Buf2 buf2[n_buf2]; + + ExecView hydetai; // diff(etai) + ExecView hydetam_ref; + + DeparturePoints dep_pts, vnode, vdep; // (ie,lev,i,j,d) - DeparturePoints dep_pts; + std::shared_ptr vrec; Data () : nelemd(-1), qsize(-1), limiter_option(9), cdr_check(0), hv_q(0), @@ -99,6 +133,7 @@ struct ComposeTransportImpl { } void set_dp_tol(); + void setup_enhanced_trajectory(); void reset(const SimulationParams& params); int requested_buffer_size() const; void init_buffers(const FunctorsBuffersManager& fbm); @@ -108,6 +143,7 @@ struct ComposeTransportImpl { void remap_q(const TimeLevel& tl); void calc_trajectory(const int np1, const Real dt); + void calc_enhanced_trajectory(const int np1, const Real dt); void remap_v(const ExecViewUnmanaged& dp3d, const int np1, const ExecViewUnmanaged& dp, const ExecViewUnmanaged& v); @@ -115,6 +151,7 @@ struct ComposeTransportImpl { void advance_hypervis_scalar(const Real dt); int run_trajectory_unit_tests(); + int run_enhanced_trajectory_unit_tests(); ComposeTransport::TestDepView::HostMirror test_trajectory(Real t0, Real t1, const bool independent_time_steps); @@ -122,8 +159,8 @@ struct ComposeTransportImpl { // avoid non-bfb-ness in, e.g., trig functions. void test_2d(const bool bfb, const int nstep, std::vector& eval); - template KOKKOS_INLINE_FUNCTION - static void loop_ijk (const KernelVariables& kv, const Fn& h) { + template KOKKOS_INLINE_FUNCTION + static void loop_ijk (const int KLIM, const KernelVariables& kv, const Fn& h) { using Kokkos::parallel_for; if (OnGpu::value) { const auto ttr = Kokkos::TeamThreadRange(kv.team, NP*NP); @@ -150,6 +187,11 @@ struct ComposeTransportImpl { } } + template KOKKOS_INLINE_FUNCTION + static void loop_ijk (const KernelVariables& kv, const Fn& h) { + loop_ijk(KLIM, kv, h); + } + template KOKKOS_INLINE_FUNCTION static void loop_ij (const KernelVariables& kv, const Fn& h) { if (OnGpu::value) { @@ -250,10 +292,113 @@ struct ComposeTransportImpl { return h; } + static KOKKOS_INLINE_FUNCTION + Real* pack2real (Scalar* pack) { return &(*pack)[0]; } + static KOKKOS_INLINE_FUNCTION + const Real* pack2real (const Scalar* pack) { return &(*pack)[0]; } template static KOKKOS_INLINE_FUNCTION - Real* pack2real (const View& v) { return &(*v.data())[0]; } + Real* pack2real (const View& v) { return pack2real(v.data()); } template static KOKKOS_INLINE_FUNCTION - const Real* cpack2real (const View& v) { return &(*v.data())[0]; } + const Real* cpack2real (const View& v) { return pack2real(v.data()); } + + KOKKOS_FUNCTION + static void ugradv_sphere ( + const SphereOperators& sphere_ops, const KernelVariables& kv, + const typename ViewConst >::type& vec_sphere2cart, + // velocity, latlon + const typename ViewConst >::type& u, + const typename ViewConst >::type& v, + const ExecViewUnmanaged& v_cart, + const ExecViewUnmanaged& ugradv_cart, + // [u dot grad] v, latlon + const ExecViewUnmanaged& ugradv) + { + for (int d_cart = 0; d_cart < 3; ++d_cart) { + const auto f1 = [&] (const int i, const int j, const int k) { + v_cart(i,j,k) = (vec_sphere2cart(0,d_cart,i,j) * v(0,i,j,k) + + vec_sphere2cart(1,d_cart,i,j) * v(1,i,j,k)); + }; + loop_ijk(kv, f1); + kv.team_barrier(); + + sphere_ops.gradient_sphere(kv, v_cart, ugradv_cart); + + const auto f2 = [&] (const int i, const int j, const int k) { + if (d_cart == 0) ugradv(0,i,j,k) = ugradv(1,i,j,k) = 0; + for (int d_latlon = 0; d_latlon < 2; ++d_latlon) + ugradv(d_latlon,i,j,k) += + vec_sphere2cart(d_latlon,d_cart,i,j)* + (u(0,i,j,k) * ugradv_cart(0,i,j,k) + u(1,i,j,k) * ugradv_cart(1,i,j,k)); + }; + loop_ijk(kv, f2); + } + } + + // Form a 3rd-degree Lagrange polynomial over (x(k-1:k+1), y(k-1:k+1)) and set + // yi(k) to its derivative at x(k). yps(:,:,0) is not written. + template + KOKKOS_FUNCTION static Real approx_derivative ( + const Real& xkm1, const Real& xk, const Real& xkp1, + const Real& ykm1, const Real& yk, const Real& ykp1) + { + return (ykm1*(( 1 /(xkm1 - xk ))*((xk - xkp1)/(xkm1 - xkp1))) + + yk *(( 1 /(xk - xkm1))*((xk - xkp1)/(xk - xkp1)) + + ((xk - xkm1)/(xk - xkm1))*( 1 /(xk - xkp1))) + + ykp1*(((xk - xkm1)/(xkp1 - xkm1))*( 1 /(xkp1 - xk )))); + } + + KOKKOS_INLINE_FUNCTION static void approx_derivative ( + const KernelVariables& kv, const CSNlevp& xs, const CSNlevp& ys, + const SNlev& yps) // yps(:,:,0) is undefined + { + CRNlevp x(cpack2real(xs)); + CRNlevp y(cpack2real(ys)); + RNlev yp(pack2real(yps)); + const auto f = [&] (const int i, const int j, const int k) { + if (k == 0) return; + const auto& xkm1 = x(i,j,k-1); + const auto& xk = x(i,j,k ); // also the interpolation point + const auto& xkp1 = x(i,j,k+1); + yp(i,j,k) = approx_derivative(x(i,j,k-1), x(i,j,k), x(i,j,k+1), + y(i,j,k-1), y(i,j,k), y(i,j,k+1)); + }; + loop_ijk(kv, f); + } + + template + KOKKOS_FUNCTION static void calc_eta_dot_dpdn ( + const KernelVariables& kv, + const HyBiPackT& hybrid_bi, // const Scalar[NUM_LEV_P] + // divergence_sphere of (v dp) at midpoints, scalar + const DivDpScalT& divdps, + // eta_dot_dpdn at interfaces, pack and scalar views of same data + const EddPackT& edd, const EddScalT& edds) + { + const auto ttr = Kokkos::TeamThreadRange(kv.team, NP*NP); + const auto tvr = Kokkos::ThreadVectorRange(kv.team, NUM_LEV); + const auto f = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + const auto r = [&] (const int k, Real& dps, const bool final) { + assert(k != 0 || dps == 0); + if (final) edds(i,j,k) = dps; + dps += divdps(i,j,k); + }; + Dispatch<>::parallel_scan(kv.team, num_phys_lev, r); + const int kend = num_phys_lev - 1; + const Real dps = edds(i,j,kend) + divdps(i,j,kend); + assert(hybrid_bi(0)[0] == 0); + const auto s = [&] (const int kp) { + edd(i,j,kp) = hybrid_bi(kp)*dps - edd(i,j,kp); + if (kp == 0) edd(i,j,kp)[0] = 0; + }; + Kokkos::parallel_for(tvr, s); + assert(edds(i,j,0) == 0); + const int bottom = num_phys_lev; + edds(i,j,bottom) = 0; // benign write race + }; + Kokkos::parallel_for(ttr, f); + } }; } // namespace Homme diff --git a/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectory.cpp b/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectory.cpp new file mode 100644 index 00000000000..62bd190bc30 --- /dev/null +++ b/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectory.cpp @@ -0,0 +1,523 @@ +/******************************************************************************** + * HOMMEXX 1.0: Copyright of Sandia Corporation + * This software is released under the BSD license + * See the file 'COPYRIGHT' in the HOMMEXX/src/share/cxx directory + *******************************************************************************/ + +#include "Config.hpp" +#ifdef HOMME_ENABLE_COMPOSE + +/* This is the second trajectory method for semi-Lagrangian transport; the first + is in ComposeTransportImplTrajectory.cpp. + + Usage + + To use it, add the following setting to the Homme namelist: + semi_lagrange_trajectory_nsubstep = N ! where N > 0 + A value of 0 (default) means Homme will use the original method. + + Another option makes the method use more than two velocity snapshots per time + step: + semi_lagrange_trajectory_nvelocity = N ! where N > 2 + A value <= 2 other than -1 means Homme will use the standard two velocity + snapshots (time step end points). -1 (default) triggers an internal + calculation based on nsubstep. This option has no effect if nsubstep=0 + (default). + + Summary + + This method provides multiple benefits over the original method, depending on + configuration: + * Supports (much) longer time steps than the original method. + * Maximizes flexibility in specifying the various atmosphere time steps. + * Greater accuracy than the original method for time steps the original + method can handle. + * Extreme accuracy in hypothetical niche applications. + + Method overview + + Recall that semi-Lagrangian tracer transport has six phases: + 1. At time step n+1, for each GLL point on the Eulerian grid, compute a + trajectory backward in time to the departure point at time n. This step is + independent of number of tracers. + 2. Simultaneously reconstruct vertically Lagrangian levels at time + n+1. This step is independent of number of tracers. + 3. In each level, for each level-midpoint departure point, interpolate tracer + mixing ratios at time n to the point. These mixing ratios are then the new + ratios at time n+1 on the Eulerian grid. + 4. Optionally apply hyperviscosity. + 5. Apply the Communication-Efficient Density Reconstructor (CEDR). + 6. Vertically remap tracers at time n+1 from the reconstructed vertically + Lagrangian levels to the vertically Eulerian grid. + + Trajectory methods implement phases 1 and 2. + + The key capability of this enhanced trajectory method (ETM) is to be able to + take multiple substeps when computing the departure points. The original + method cannot substep. Each substep has second-order accuracy, so the overall + method is always second-order accurate. But as the number of substeps + increases, so does accuracy. + + A second capability of the ETM is to use more velocity snapshots than just + the tracer time step end-point snapshots. For example, if there are two + substeps, the method can use three velocity snapshots: beginning, middle, + end. The first substep uses (middle, end), and the second uses (beginning, + middle). (This might be the opposite of what you expected, but recall that + the trajectory is computed backward in time.) + + At a software level, a third capability is to use an arbitrarily large + element halo when computing departure points. The original method is limited + to two halo layers. Extra halo layers do not increase the cost of search for + a fixed time step because of the ordering of elements in the layer. + + Speedup comes from the fact that taking a longer time step means phases 3-6, + the most expensive phases, run less often. Phases 1 and 2 also run less often + but take more time per run, summing to about the same cost over a fixed time + duration T as the original method. + + Algorithm outline + + This method works principally in the eta coordinate. eta is constant in a + level on the Eulerian grid. + Let time t1 > t0 and consider a trajectory substep from t1 to t0. + Terminology: An arrival point is the time-t1 point of a trajectory. A + departure point is the time-t0 point. + For each interface node at times t0 and t1, compute eta_dot dp/deta + (calc_eta_dot_dpdn). + For each midpoint node at times t0 and t1, compute + eta_dot = eta_dot dp/deta/(A_eta p0 + B_eta ps) + (calc_etadotmid_from_etadotdpdnint). + Use eta_dot, the horizontal velocity data, and the update formula described + in the comment for calc_nodal_velocities to compute the velocity term in the + update formula at the Eulerian vertical-midpoint nodes. Call the result V + (calc_vel_horiz_formula_node_ref_mid, calc_eta_dot_formula_node_ref_mid). + In general, the trajectory arrival point at t1 is not on the grid, but it + is in the first substep. If it is not on the grid, interpolate V to the + arrival points to produce V_dep (calc_v_departure). A detail here is we + should actually think of the original velocity data as being interpolated, + and then V_dep is computed from the interpolated data. But the formula to + compute V is linear in these data, so we can defer interpolation to the end + and do it just once. + Integrate the trajectory backward from t1 at the arrival point to t0 at the + departure point using V_dep (update_dep_points). + The algorithm up to this point can be substepped, running multiple times to + go backward from t1 to t0 in multiple steps. + After substepping is finished, there is one final part. + So far we have computed departure points corresponding to Eulerian-grid + arrival points. But now we need to account for the fact that the levels are + vertically Lagrangian ("floating"). The arrival points we actually need are + those on the floating levels at time t1 corresponding to Eulerian levels at + time t0. This is implemented in + describe interp_departure_points_to_floating_level_midpoints. + We use the following notation: yi = I[y(x)](xi) is an interpolant + constructed from y(x) and evaluated at xi. + On input, we have departure-point level-midpoint eta, eta_dep_mid, and the + corresponding horizontal position, p_dep_mid. We also know eta level + interfaces and midpoints on the reference grid, eta_ref_int and eta_ref_mid, + and the top and bottom boundary values of eta, eta(0) and eta(1). + First, reconstruct Lagrangian levels at t1 on the arrival column: + eta_arr_int = I[eta_ref_mid([eta(0),eta_dep_mid,eta(1)])](eta_ref_int). + Second, compute the Lagrangian level midpoints at t1 on the arrival column: + eta_arr_mid = I[eta_ref_mid([eta(0),eta_dep_mid,eta(1)])](eta_ref_mid). + Third, compute the departure horizontal points corresponding to the arrival + Lagrangian level midpoints: + p_dep_mid(eta_arr_mid) = I[p_dep_mid(eta_ref_mid)](eta_arr_mid). + These calcualtions provide us with the final results: p_dep_mid is used in + phase 3, and eta_arr_int is used in phase 6. + */ + +#include "ComposeTransportImplEnhancedTrajectoryImpl.hpp" + +namespace Homme { +namespace { + +// Set dep_points_all to level-midpoint arrival points. +void init_dep_points (const CTI& c, const cti::DeparturePoints& dep_pts) { + const auto independent_time_steps = c.m_data.independent_time_steps; + const auto& sphere_cart = c.m_geometry.m_sphere_cart; + const CRNV hyetam(cti::cpack2real(c.m_hvcoord.etam)); + assert(not independent_time_steps or dep_pts.extent_int(4) == 4); + const auto f = KOKKOS_LAMBDA (const int idx) { + int ie, lev, i, j; + cti::idx_ie_physlev_ij(idx, ie, lev, i, j); + for (int d = 0; d < 3; ++d) + dep_pts(ie,lev,i,j,d) = sphere_cart(ie,i,j,d); + if (independent_time_steps) + dep_pts(ie,lev,i,j,3) = hyetam(lev); + }; + c.launch_ie_physlev_ij(f); +} + +void update_dep_points ( + const CTI& c, const Real dtsub, const cti::DeparturePoints& vdep, + const cti::DeparturePoints& dep_pts) +{ + const auto independent_time_steps = c.m_data.independent_time_steps; + const auto is_sphere = c.m_data.geometry_type == 0; + const auto scale_factor = c.m_geometry.m_scale_factor; + const auto f = KOKKOS_LAMBDA (const int idx) { + int ie, lev, i, j; + cti::idx_ie_physlev_ij(idx, ie, lev, i, j); + // Update horizontal position. + Real p[3]; + for (int d = 0; d < 3; ++d) + p[d] = dep_pts(ie,lev,i,j,d) - dtsub*vdep(ie,lev,i,j,d)/scale_factor; + if (is_sphere) { + const auto norm = std::sqrt(square(p[0]) + square(p[1]) + square(p[2])); + for (int d = 0; d < 3; ++d) + p[d] /= norm; + } + for (int d = 0; d < 3; ++d) + dep_pts(ie,lev,i,j,d) = p[d]; + if (independent_time_steps) { + // Update vertical position. + dep_pts(ie,lev,i,j,3) -= dtsub*vdep(ie,lev,i,j,3); + } + }; + c.launch_ie_physlev_ij(f); +} + +/* Evaluate a formula to provide an estimate of nodal velocities that are use to + create a 2nd-order update to the trajectory. The fundamental formula for the + update in position p from arrival point p1 to departure point p0 is + p0 = p1 - dt/2 (v(p1,t0) + v(p1,t1) - dt v(p1,t1) grad v(p1,t0)). + Here we compute the velocity estimate at the nodes: + 1/2 (v(p1,t0) + v(p1,t1) - dt v(p1,t1) grad v(p1,t0)). +*/ +void calc_nodal_velocities ( + const CTI& c, const Real dtsub, const Real halpha[2], + const cti::CVSlot& v1, const cti::CDpSlot& dp1, const int idx1, + const cti::CVSlot& v2, const cti::CDpSlot& dp2, const int idx2, + const cti::DeparturePoints& vnode) +{ + using Kokkos::ALL; + const auto& d = c.m_data; + const auto& h = c.m_hvcoord; + const auto& sphere_ops = c.m_sphere_ops; + const auto& vec_sph2cart = c.m_geometry.m_vec_sph2cart; + const bool independent_time_steps = d.independent_time_steps; + const auto ps0 = h.ps0; + const auto hyai0 = h.hybrid_ai0; + const auto& hybi = h.hybrid_bi_packed; + const auto& hydai = h.hybrid_ai_delta; + const auto& hydbi = h.hybrid_bi_delta; + const auto& hyetam = h.etam; + const auto& hyetai = h.etai; + const auto& hydetai = d.hydetai; + const auto& buf1a = d.buf1o[0]; const auto& buf1b = d.buf1o[1]; + const auto& buf1c = d.buf1o[2]; const auto& buf1d = d.buf1o[3]; + const auto& buf2a = d.buf2 [0]; const auto& buf2b = d.buf2 [1]; + const auto& buf2c = d.buf2 [2]; const auto& buf2d = d.buf2 [3]; + const auto alpha0 = halpha[0], alpha1 = halpha[1]; + const auto f = KOKKOS_LAMBDA (const cti::MT& team) { + KernelVariables kv(team); + const int ie = kv.ie; + const auto wrk1 = Homme::subview(buf1a, kv.team_idx); + const auto wrk2 = Homme::subview(buf1b, kv.team_idx); + const auto vwrk1 = Homme::subview(buf2a, kv.team_idx); + const auto vwrk2 = Homme::subview(buf2b, kv.team_idx); + const auto v1_ie = Homme::subview(v1, ie, idx1); + const auto v2_ie = Homme::subview(v2, ie, idx2); + const Real alpha[] = {alpha0, alpha1}; + CSelNlevp eta_dot[] = {Homme::subview(buf1c, kv.team_idx), + Homme::subview(buf1d, kv.team_idx)}; + { + SelNlevp eta_dot[] = {Homme::subview(buf1c, kv.team_idx), + Homme::subview(buf1d, kv.team_idx)}; + if (independent_time_steps) { + const auto dp1_ie = Homme::subview(dp1, ie, idx1); + const auto dp2_ie = Homme::subview(dp2, ie, idx2); + calc_eta_dot_ref_mid(kv, sphere_ops, + ps0, hyai0, hybi, hydai, hydbi, hydetai, + alpha, v1_ie, dp1_ie, v2_ie, dp2_ie, + wrk1, wrk2, vwrk1, + eta_dot); + } else { + for (int t = 0; t < 2; ++t) { + const auto& ed = eta_dot[t]; + const auto f = [&] (const int i, const int j, const int k) { + ed(i,j,k) = 0; + }; + cti::loop_ijk(kv, f); + } + } + } + // Collect the horizontal nodal velocities. v1,2 are on Eulerian levels. v1 + // is from time t1 < t2. + auto* vm1 = Homme::subview(buf2c, kv.team_idx).data(); + auto* vm2 = Homme::subview(buf2d, kv.team_idx).data(); + CS2elNlev vsph[] = {CS2elNlev(vm1), CS2elNlev(vm2)}; + { + S2elNlev vsph[] = {S2elNlev(vm1), S2elNlev(vm2)}; + for (int t = 0; t < 2; ++t) { + const auto& v = vsph[t]; + for (int d = 0; d < 2; ++d) { + const auto f = [&] (const int i, const int j, const int k) { + v(d,i,j,k) = ((1 - alpha[t])*v1_ie(d,i,j,k) + + /**/ alpha[t] *v2_ie(d,i,j,k)); + }; + cti::loop_ijk(kv, f); + } + } + } + kv.team_barrier(); + // Given the vertical and horizontal nodal velocities at time endpoints, + // evaluate the velocity estimate formula, providing the final horizontal + // and vertical velocity estimates at midpoint nodes. + const auto vnode_ie = Kokkos::subview(vnode, ie, ALL,ALL,ALL,ALL); + const auto vec_sph2cart_ie = Homme::subview(vec_sph2cart, ie); + calc_vel_horiz_formula_node_ref_mid(kv, sphere_ops, + hyetam, vec_sph2cart_ie, + dtsub, vsph, eta_dot, + wrk1, vwrk1, vwrk2, + vnode_ie); + if (independent_time_steps) { + kv.team_barrier(); + calc_eta_dot_formula_node_ref_mid(kv, sphere_ops, + hyetai, hyetam, + dtsub, vsph, eta_dot, + wrk1, vwrk1, + vnode_ie); + } + }; + Kokkos::parallel_for(c.m_tp_ne, f); +} + +// Determine the departure points corresponding to the vertically Lagrangian +// grid's arrival midpoints, where the floating levels are those that evolve +// over the course of the full tracer time step. Also compute divdp, which holds +// the floating levels' dp values for later use in vertical remap. +void interp_departure_points_to_floating_level_midpoints (const CTI& c, const int np1) { + using Kokkos::ALL; + const int nlev = NUM_PHYSICAL_LEV, nlevp = nlev+1; + const auto is_sphere = c.m_data.geometry_type == 0; + const auto& d = c.m_data; + const auto& h = c.m_hvcoord; + const auto ps0 = h.ps0; + const auto hyai0 = h.hybrid_ai0; + const auto& hybi = h.hybrid_bi; + const auto& hyetai = h.etai; + const CRNV hyetam(cti::cpack2real(h.etam)); + const auto& detam_ref = d.hydetam_ref; + const auto deta_tol = d.deta_tol; + const auto& dep_pts = d.dep_pts; + const auto& dp3d = c.m_state.m_dp3d; + const auto& buf1a = d.buf1e[0]; const auto& buf1b = d.buf1e[1]; + const auto& buf1c = d.buf1e[2]; const auto& buf1d = d.buf1e[3]; + const auto& buf2a = d.buf2[0]; + const auto f = KOKKOS_LAMBDA (const cti::MT& team) { + KernelVariables kv(team); + const int ie = kv.ie; + const auto wrk1 = Homme::subview(buf1a, kv.team_idx); + const auto wrk2 = Homme::subview(buf1b, kv.team_idx); + const auto wrk3 = Homme::subview(buf1c, kv.team_idx); + const auto wrk4 = Homme::subview(buf1d, kv.team_idx); + const auto vwrk = Homme::subview(buf2a, kv.team_idx); + // Reconstruct Lagrangian levels at t1 on arrival column: + // eta_arr_int = I[eta_ref_mid([eta(0),eta_dep_mid,eta(1)])](eta_ref_int) + const auto etam = p2rel(wrk3.data(), nlev); + const auto f = [&] (const int i, const int j, const int k) { + etam(i,j,k) = dep_pts(ie,k,i,j,3); + }; + cti::loop_ijk(kv, f); + kv.team_barrier(); + limit_etam(kv, nlev, + hyetai, detam_ref, deta_tol, + p2rel(wrk1.data(), nlevp), p2rel(wrk2.data(), nlevp), + etam); + kv.team_barrier(); + { + // Compute eta_arr_int. + const auto etai_arr = p2rel(wrk4.data(), nlevp); + eta_interp_eta(kv, nlev, + hyetai, + etam, hyetam, + p2rel(wrk1.data(), nlev+2), RnV(cti::pack2real(wrk2), nlev+2), + nlevp-2, hyetai, etai_arr, 1); + const auto f = [&] (const int i, const int j) { + etai_arr(i,j,0) = hyetai(0); + etai_arr(i,j,nlev) = hyetai(nlev); + }; + c.loop_ij(kv, f); + // Compute divdp. + const ExecViewUnmanaged ps(cti::pack2real(vwrk)); + calc_ps(kv, nlev, + ps0, hyai0, + Homme::subview(dp3d, ie, np1), + ps); + kv.team_barrier(); + eta_to_dp(kv, nlev, + ps0, hybi, hyetai, + ps, etai_arr, + p2rel(wrk2.data(), nlev+1), + RelnV(cti::pack2real(Homme::subview(c.m_derived.m_divdp, ie)), + NP, NP, NUM_LEV*VECTOR_SIZE)); + kv.team_barrier(); + } + // Compute Lagrangian level midpoints at t1 on arrival column: + // eta_arr_mid = I[eta_ref_mid([eta(0),eta_dep_mid,eta(1)])](eta_ref_mid) + const auto etam_arr = p2rel(wrk4.data(), nlev); + eta_interp_eta(kv, nlev, + hyetai, + etam, hyetam, + p2rel(wrk1.data(), nlev+2), RnV(cti::pack2real(wrk2), nlev+2), + nlev, hyetam, etam_arr); + kv.team_barrier(); + // Compute departure horizontal points corresponding to arrival + // Lagrangian level midpoints: + // p_dep_mid(eta_arr_mid) = I[p_dep_mid(eta_ref_mid)](eta_arr_mid) + { + const RelnV dpts_in(cti::pack2real(vwrk), NP, NP, nlev); + const RelnV dpts_out(dpts_in.data() + NP*NP*nlev, NP, NP, nlev); + for (int d = 0; d < 3; ++d) { + const auto f = [&] (const int i, const int j, const int k) { + dpts_in(i,j,k) = dep_pts(ie,k,i,j,d); + }; + c.loop_ijk(kv, f); + kv.team_barrier(); + eta_interp_horiz(kv, nlev, + hyetai, + hyetam, dpts_in, + RnV(cti::pack2real(wrk2), nlev+2), p2rel(wrk1.data(), nlev+2), + etam_arr, dpts_out); + kv.team_barrier(); + const auto g = [&] (const int i, const int j, const int k) { + dep_pts(ie,k,i,j,d) = dpts_out(i,j,k); + }; + c.loop_ijk(kv, g); + kv.team_barrier(); + } + if (is_sphere) { + // Normalize. + const auto h = [&] (const int i, const int j, const int k) { + Real norm = 0; + for (int d = 0; d < 3; ++d) norm += square(dep_pts(ie,k,i,j,d)); + norm = std::sqrt(norm); + for (int d = 0; d < 3; ++d) dep_pts(ie,k,i,j,d) /= norm; + }; + c.loop_ijk(kv, h); + } + } + }; + Kokkos::parallel_for(c.m_tp_ne, f); +} + +void dss_vnode (const CTI& c, const cti::DeparturePoints& vnode) { + const int ndim = c.m_data.independent_time_steps ? 4 : 3; + const auto& spheremp = c.m_geometry.m_spheremp; + const auto& rspheremp = c.m_geometry.m_rspheremp; + const auto& vp = c.m_tracers.qtens_biharmonic; + const ExecViewUnmanaged + v(cti::pack2real(vp), vp.extent_int(0), vp.extent_int(1)); + const auto f = KOKKOS_LAMBDA (const int idx) { + int ie, lev, i, j; + cti::idx_ie_physlev_ij(idx, ie, lev, i, j); + for (int d = 0; d < ndim; ++d) + v(ie,d,i,j,lev) = vnode(ie,lev,i,j,d)*spheremp(ie,i,j)*rspheremp(ie,i,j); + }; + c.launch_ie_physlev_ij(f); + Kokkos::fence(); + const auto be = c.m_v_dss_be[c.m_data.independent_time_steps ? 1 : 0]; + be->exchange(); + Kokkos::fence(); + const auto g = KOKKOS_LAMBDA (const int idx) { + int ie, lev, i, j; + cti::idx_ie_physlev_ij(idx, ie, lev, i, j); + for (int d = 0; d < ndim; ++d) + vnode(ie,lev,i,j,d) = v(ie,d,i,j,lev); + }; + c.launch_ie_physlev_ij(g); +} + +} // namespace anon + +// For limit_etam. +void ComposeTransportImpl::setup_enhanced_trajectory () { + const auto etai = cmvdc(m_hvcoord.etai); + const Real deta_ave = (etai(num_phys_lev) - etai(0)) / num_phys_lev; + m_data.deta_tol = 10*std::numeric_limits::epsilon()*deta_ave; + + // diff(etai) + m_data.hydetai = decltype(m_data.hydetai)("hydetai"); + const auto detai_pack = Kokkos::create_mirror_view(m_data.hydetai); + HostViewUnmanaged detai(pack2real(detai_pack)); + for (int k = 0; k < num_phys_lev; ++k) + detai(k) = etai(k+1) - etai(k); + Kokkos::deep_copy(m_data.hydetai, detai_pack); + + const auto etamp = cmvdc(m_hvcoord.etam); + HostViewUnmanaged etam(pack2real(etamp)); + + // hydetam_ref. + m_data.hydetam_ref = decltype(m_data.hydetam_ref)("hydetam_ref"); + const auto m = Kokkos::create_mirror_view(m_data.hydetam_ref); + const int nlev = num_phys_lev; + m(0) = etam(0) - etai(0); + for (int k = 1; k < nlev; ++k) m(k) = etam(k) - etam(k-1); + m(nlev) = etai(nlev) - etam(nlev-1); + Kokkos::deep_copy(m_data.hydetam_ref, m); + + // etam + homme::compose::set_hvcoord(etai(0), etai(num_phys_lev), etam.data()); +} + +void ComposeTransportImpl::calc_enhanced_trajectory (const int np1, const Real dt) { + GPTLstart("compose_calc_enhanced_trajectory"); + + const auto& dep_pts = m_data.dep_pts; + const auto& vnode = m_data.vnode; + const auto& vdep = m_data.vdep; + + init_dep_points(*this, dep_pts); + + const int nelemd = m_data.nelemd; + const Real dtsub = dt / m_data.trajectory_nsubstep; + const int nsubstep = m_data.trajectory_nsubstep; + for (int step = 0; step < nsubstep; ++step) { + { + Kokkos::fence(); + GPTLstart("compose_vnode"); + const Real alpha[] = {Real(nsubstep-step-1)/nsubstep, + Real(nsubstep-step )/nsubstep}; + const CVSlot v1(m_derived.m_vstar.data(), nelemd, 1); + const CDpSlot dp1(m_derived.m_dp.data(), nelemd, 1); + const auto& v2 = m_state.m_v; + const auto& dp2 = m_state.m_dp3d; + calc_nodal_velocities(*this, dtsub, alpha, + v1, dp1, 0, v2, dp2, np1, + vnode); + Kokkos::fence(); + GPTLstop("compose_vnode"); + } + + GPTLstart("compose_v_bexchv"); + dss_vnode(*this, vnode); + Kokkos::fence(); + GPTLstop("compose_v_bexchv"); + + if (step == 0) { + update_dep_points(*this, dtsub, vnode, dep_pts); + } else { + GPTLstart("compose_vdep"); + homme::compose::calc_v_departure(step, dtsub); + Kokkos::fence(); + GPTLstop("compose_vdep"); + + update_dep_points(*this, dtsub, vdep, dep_pts); + } + } + Kokkos::fence(); + + if (m_data.independent_time_steps) { + GPTLstart("compose_floating_dep_pts"); + interp_departure_points_to_floating_level_midpoints(*this, np1); + Kokkos::fence(); + GPTLstop("compose_floating_dep_pts"); + } + + GPTLstop("compose_calc_enhanced_trajectory"); +} + +} // namespace Homme + +#endif // HOMME_ENABLE_COMPOSE diff --git a/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectoryImpl.hpp b/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectoryImpl.hpp new file mode 100644 index 00000000000..1d9ee6da4ad --- /dev/null +++ b/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectoryImpl.hpp @@ -0,0 +1,720 @@ +/******************************************************************************** + * HOMMEXX 1.0: Copyright of Sandia Corporation + * This software is released under the BSD license + * See the file 'COPYRIGHT' in the HOMMEXX/src/share/cxx directory + *******************************************************************************/ + +#include "Config.hpp" +#ifdef HOMME_ENABLE_COMPOSE + +#ifndef HOMMEXX_COMPOSE_TRANSPORT_IMPL_ENHANCED_TRAJECTORY_IMPL_HPP +#define HOMMEXX_COMPOSE_TRANSPORT_IMPL_ENHANCED_TRAJECTORY_IMPL_HPP + +#include "ComposeTransportImpl.hpp" + +#include "compose_hommexx.hpp" + +namespace Homme { +namespace { // anon + +using cti = ComposeTransportImpl; +using CTI = ComposeTransportImpl; +using CSelNlev = cti::CSNlev; +using CRelNlev = cti::CRNlev; +using CSelNlevp = cti::CSNlevp; +using CRelNlevp = cti::CRNlevp; +using CS2elNlev = cti::CS2Nlev; +using CR2elNlev = cti::CR2Nlev; +using SelNlev = cti::SNlev; +using RelNlev = cti::RNlev; +using SelNlevp = cti::SNlevp; +using RelNlevp = cti::RNlevp; +using S2elNlev = cti::S2Nlev; +using R2elNlev = cti::R2Nlev; +using S2elNlevp = cti::S2Nlevp; + +using RelV = ExecViewUnmanaged; +using CRelV = typename ViewConst::type; + +template using SelNV = ExecViewUnmanaged; +template using CSelNV = typename ViewConst>::type; + +template using RelNV = ExecViewUnmanaged; +template using CRelNV = typename ViewConst>::type; + +template using RNV = ExecViewUnmanaged; +template using CRNV = typename ViewConst>::type; +using RNlevp = RNV; +using CRNlevp = CRNV; + +using RnV = ExecViewUnmanaged; +using CRnV = ExecViewUnmanaged; +using SnV = ExecViewUnmanaged; +using CSnV = ExecViewUnmanaged; + +template using SNV = ExecViewUnmanaged; +template using CSNV = typename ViewConst>::type; + +using RelnV = ExecViewUnmanaged; +using CRelnV = ExecViewUnmanaged; +using SelnV = ExecViewUnmanaged; +using CSelnV = ExecViewUnmanaged; + +// Helper functions to move between various array data structures and assert +// things about them. + +KOKKOS_INLINE_FUNCTION +static int calc_npack (const int nscal) { + return (nscal + cti::packn - 1) / VECTOR_SIZE; +} + +KOKKOS_INLINE_FUNCTION +static int calc_nscal (const int npack) { + return npack * VECTOR_SIZE; +} + +KOKKOS_INLINE_FUNCTION +RnV getcol (const RelnV& a, const int i, const int j) { + return Kokkos::subview(a,i,j,Kokkos::ALL); +} + +KOKKOS_INLINE_FUNCTION +CRnV getcolc (const CRelnV& a, const int i, const int j) { + return Kokkos::subview(a,i,j,Kokkos::ALL); +} + +KOKKOS_INLINE_FUNCTION +RelnV elp2r (const SelnV& p) { + return RelnV(cti::pack2real(p), NP, NP, calc_nscal(p.extent_int(2))); +} + +KOKKOS_INLINE_FUNCTION +CRelnV elp2r (const CSelnV& p) { + return CRelnV(cti::cpack2real(p), NP, NP, calc_nscal(p.extent_int(2))); +} + +KOKKOS_INLINE_FUNCTION +RelnV p2rel (Scalar* data, const int nlev) { + return RelnV(cti::pack2real(data), NP, NP, nlev); +} + +KOKKOS_INLINE_FUNCTION +void assert_eln (const CRelnV& a, const int nlev) { + assert(a.extent_int(0) >= NP); + assert(a.extent_int(1) >= NP); + assert(a.extent_int(2) >= nlev); +} + +KOKKOS_INLINE_FUNCTION +void assert_eln (const CSelnV& a, const int nlev) { + assert(a.extent_int(0) >= NP); + assert(a.extent_int(1) >= NP); + assert(calc_nscal(a.extent_int(2)) >= nlev); +} + +// Find the support for the linear interpolant. +// For sorted ascending x[0:n] and x in [x[0], x[n-1]] with hint xi_idx, +// return i such that x[i] <= xi <= x[i+1]. +// This function is meant for the case that x_idx is very close to the +// support. If that isn't true, then this method is inefficient; binary search +// should be used instead. +template +KOKKOS_FUNCTION static +int find_support (const int n, const ConstRealArray& x, const int x_idx, + const Real xi) { + assert(xi >= x[0] and xi <= x[n-1]); + // Handle the most common case. + if (x_idx < n-1 and xi >= x[x_idx ] and xi <= x[x_idx+1]) return x_idx; + if (x_idx > 0 and xi >= x[x_idx-1] and xi <= x[x_idx ]) return x_idx-1; + // Move on to less common ones. + const int max_step = max(x_idx, n-1 - x_idx); + for (int step = 1; step <= max_step; ++step) { + if (x_idx < n-1-step and xi >= x[x_idx+step ] and xi <= x[x_idx+step+1]) + return x_idx+step; + if (x_idx > step and xi >= x[x_idx-step-1] and xi <= x[x_idx-step ]) + return x_idx-step-1; + } + assert(false); + return -1; +} + +// Linear interpolation core formula. +template +KOKKOS_FUNCTION Real +linterp (const int n, const XT& x, const YT& y, const int x_idx, const Real xi) { + const auto isup = find_support(n, x, x_idx, xi); + const Real a = (xi - x[isup])/(x[isup+1] - x[isup]); + return (1-a)*y[isup] + a*y[isup+1]; +} + +// Linear interpolation at the lowest level of team ||ism. +// Range provides this ||ism over index 0 <= k < ni. +// Interpolate y(x) to yi(xi). +// x_idx_offset is added to k in the call to find_support. +// Arrays should all have rank 1. +// Notation: yi = I[y(x)](xi) is an interpolant constructed from y(x) and +// evaluated at xi. +template +KOKKOS_FUNCTION void +linterp (const Range& range, + const int n , const XT& x , const YT& y, + const int ni, const XIT& xi, const YIT& yi, + const int x_idx_offset = 0, const char* const caller = nullptr) { +#ifndef NDEBUG + if (xi[0] < x[0] or xi[ni-1] > x[n-1]) { + if (caller) + printf("linterp: xi out of bounds: %s %1.15e %1.15e %1.15e %1.15e\n", + caller ? caller : "NONE", x[0], xi[0], xi[ni-1], x[n-1]); + assert(false); + } +#endif + assert(range.start == 0); + assert(range.end == ni); + const auto f = [&] (const int k) { + yi[k] = linterp(n, x, y, k + x_idx_offset, xi[k]); + }; + Kokkos::parallel_for(range, f); +} + +// Compute Lagrangian level midpoints at t1 on arrival column: +// eta_arr_mid = I[eta_ref_mid([eta(0),eta_dep_mid,eta(1)])](eta_ref_mid). +KOKKOS_FUNCTION void +eta_interp_eta (const KernelVariables& kv, const int nlev, + const CRnV& hy_etai, const CRelnV& x, const CRnV& y, + const RelnV& xwrk, const RnV& ywrk, + // Use xi(i_os:), yi(i,j,i_os:). + const int ni, const CRnV& xi, const RelnV& yi, const int i_os = 0) { + const auto& xbdy = xwrk; + const auto& ybdy = ywrk; + assert(hy_etai.extent_int(0) >= nlev+1); + assert_eln(x, nlev); + assert(y.extent_int(0) >= nlev); + assert_eln(xbdy, nlev+2); + assert(ybdy.extent_int(0) >= nlev+2); + assert(xi.extent_int(0) >= i_os + ni); + assert_eln(yi, i_os + ni); + const auto ttr = Kokkos::TeamThreadRange(kv.team, NP*NP); + const auto tvr_ni = Kokkos::ThreadVectorRange(kv.team, ni); + const auto tvr_nlevp2 = Kokkos::ThreadVectorRange(kv.team, nlev+2); + const auto f_y = [&] (const int k) { + ybdy(k) = (k == 0 ? hy_etai(0) : + k == nlev+1 ? hy_etai(nlev) : + /**/ y(k-1)); + }; + Kokkos::parallel_for(Kokkos::TeamVectorRange(kv.team, nlev+2), f_y); + kv.team_barrier(); + const auto f_x = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + const auto g = [&] (const int k) { + xbdy(i,j,k) = (k == 0 ? hy_etai(0) : + k == nlev+1 ? hy_etai(nlev) : + /**/ x(i,j,k-1)); + }; + Kokkos::parallel_for(tvr_nlevp2, g); + }; + Kokkos::parallel_for(ttr, f_x); + kv.team_barrier(); + const auto f_linterp = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + linterp(tvr_ni, + nlev+2, getcolc(xbdy,i,j), ybdy, + ni, xi.data() + i_os, getcol(yi,i,j).data() + i_os, + 1, "eta_interp_eta"); + }; + Kokkos::parallel_for(ttr, f_linterp); +} + +// Compute departure horizontal points corresponding to arrival Lagrangian level +// midpoints: +// p_dep_mid(eta_arr_mid) = I[p_dep_mid(eta_ref_mid)](eta_arr_mid) +KOKKOS_FUNCTION void +eta_interp_horiz (const KernelVariables& kv, const int nlev, + const CRnV& hy_etai, const CRnV& x, const CRelnV& y, + const RnV& xwrk, const RelnV& ywrk, + const CRelnV& xi, const RelnV& yi) { + const auto& xbdy = xwrk; + const auto& ybdy = ywrk; + assert(hy_etai.extent_int(0) >= nlev+1); + assert(x.extent_int(0) >= nlev); + assert_eln(y, nlev); + assert(xbdy.extent_int(0) >= nlev+2); + assert_eln(ybdy, nlev+2); + assert_eln(xi, nlev); + assert_eln(yi, nlev); + const auto ttr = Kokkos::TeamThreadRange(kv.team, NP*NP); + const auto tvr_nlev = Kokkos::ThreadVectorRange(kv.team, nlev); + const auto tvr_nlevp2 = Kokkos::ThreadVectorRange(kv.team, nlev+2); + const auto f_x = [&] (const int k) { + xbdy(k) = (k == 0 ? hy_etai(0) : + k == nlev+1 ? hy_etai(nlev) : + /**/ x(k-1)); + }; + Kokkos::parallel_for(Kokkos::TeamVectorRange(kv.team, nlev+2), f_x); + kv.team_barrier(); + const auto f_y = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + const auto g = [&] (const int k) { + // Constant interp outside of the etam support. + ybdy(i,j,k) = (k == 0 ? y(i,j,0) : + k == nlev+1 ? y(i,j,nlev-1) : + /**/ y(i,j,k-1)); + }; + Kokkos::parallel_for(tvr_nlevp2, g); + }; + Kokkos::parallel_for(ttr, f_y); + kv.team_barrier(); + const auto f_linterp = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + linterp(tvr_nlev, + nlev+2, xbdy, getcolc(ybdy,i,j), + nlev, getcolc(xi,i,j), getcol(yi,i,j), + 1, "eta_interp_horiz"); + }; + Kokkos::parallel_for(ttr, f_linterp); +} + +/* Compute level pressure thickness given eta at interfaces using the following + approximation: + e = A(e) + B(e) + p(e) = A(e) p0 + B(e) ps + = e p0 + B(e) (ps - p0) + a= e p0 + I[Bi(eref)](e) (ps - p0). + Then dp = diff(p). +*/ +KOKKOS_FUNCTION void +eta_to_dp (const KernelVariables& kv, const int nlev, + const Real hy_ps0, const CRnV& hy_bi, const CRnV& hy_etai, + const CRelV& ps, const CRelnV& etai, const RelnV& wrk, + const RelnV& dp) { + const int nlevp = nlev + 1; + assert(hy_bi.extent_int(0) >= nlevp); + assert(hy_etai.extent_int(0) >= nlevp); + assert_eln(etai, nlevp); + assert_eln(wrk, nlevp); + assert_eln(dp, nlev); + const auto& bi = wrk; + const auto ttr = Kokkos::TeamThreadRange(kv.team, NP*NP); + const auto tvr_linterp = Kokkos::ThreadVectorRange(kv.team, nlevp); + const auto f_linterp = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + linterp(tvr_linterp, + nlevp, hy_etai, hy_bi, + nlevp, getcolc(etai,i,j), getcol(bi,i,j), + 0, "eta_to_dp"); + }; + Kokkos::parallel_for(ttr, f_linterp); + kv.team_barrier(); + const auto tvr = Kokkos::ThreadVectorRange(kv.team, nlev); + const auto f = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + const auto dps = ps(i,j) - hy_ps0; + const auto g = [&] (const int k) { + dp(i,j,k) = ((etai(i,j,k+1) - etai(i,j,k))*hy_ps0 + + (bi(i,j,k+1) - bi(i,j,k))*dps); + }; + Kokkos::parallel_for(tvr, g); + }; + Kokkos::parallel_for(ttr, f); +} + +/* Limit eta levels so their thicknesses, deta, are bounded below by 'low'. + + This method pulls mass only from intervals k that are larger than their + reference value (deta(k) > deta_ref(k)), and only down to their reference + value. This concentrates changes to intervals that, by having a lot more mass + than usual, drive other levels negative, leaving all the other intervals + unchanged. + + This selective use of mass provides enough to fulfill the needed mass. + Inputs: + m (deta): input mass + r (deta_ref): level mass reference. + Preconditions: + (1) 0 <= low <= min r(i) + (2) 1 = sum r(i) = sum(m(i)). + Rewrite (2) as + 1 = sum_{m(i) >= r(i)} m(i) + sum_{m(i) < r(i)} m(i) + and, thus, + 0 = sum_{m(i) >= r(i)} (m(i) - r(i)) + sum_{m(i) < r(i)} (m(i) - r(i)). + Then + sum_{m(i) >= r(i)} (m(i) - r(i)) (available mass to redistribute) + = -sum_{m(i) < r(i)} (m(i) - r(i)) + >= -sum_{m(i) < lo } (m(i) - r(i)) + >= -sum_{m(i) < lo } (m(i) - lo ) (mass to fill in). + Thus, if the preconditions hold, then there's enough mass to redistribute. + */ +template +KOKKOS_FUNCTION void +deta_caas (const KernelVariables& kv, const Range& tvr_nlevp, + const CRnV& deta_ref, const Real low, const RnV& w, + const RnV& deta) { + const auto g1 = [&] (const int k, Kokkos::Real2& sums) { + Real wk; + if (deta(k) < low) { + sums.v[0] += deta(k) - low; + deta(k) = low; + wk = 0; + } else { + wk = (deta(k) > deta_ref(k) ? + deta(k) - deta_ref(k) : + 0); + } + sums.v[1] += wk; + w(k) = wk; + }; + Kokkos::Real2 sums; + Dispatch<>::parallel_reduce(kv.team, tvr_nlevp, g1, sums); + const Real wneeded = sums.v[0]; + if (wneeded == 0) return; + // Remove what is needed from the donors. + const Real wavail = sums.v[1]; + const auto g2 = [&] (const int k) { + deta(k) += wneeded*(w(k)/wavail); + }; + Kokkos::parallel_for(tvr_nlevp, g2); +} + +// Wrapper to above. +KOKKOS_FUNCTION void +deta_caas (const KernelVariables& kv, const int nlevp, const CRnV& deta_ref, + const Real low, const RelnV& wrk, const RelnV& deta) { + assert(deta_ref.extent_int(0) >= nlevp); + assert_eln(wrk, nlevp); + assert_eln(deta, nlevp); + const auto ttr = Kokkos::TeamThreadRange(kv.team, NP*NP); + const auto tvr = Kokkos::ThreadVectorRange(kv.team, nlevp); + const auto f = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + deta_caas(kv, tvr, deta_ref, low, getcol(wrk,i,j), getcol(deta,i,j)); + }; + Kokkos::parallel_for(ttr, f); +} + +// Wrapper to deta_caas. On input and output, eta contains the midpoint eta +// values. On output, deta_caas has been applied, if necessary, to +// diff(eta(i,j,:)). +KOKKOS_FUNCTION void +limit_etam (const KernelVariables& kv, const int nlev, const CRnV& hy_etai, + const CRnV& deta_ref, const Real deta_tol, const RelnV& wrk1, + const RelnV& wrk2, const RelnV& eta) { + assert(hy_etai.extent_int(0) >= nlev+1); + assert(deta_ref.extent_int(0) >= nlev+1); + const auto deta = wrk2; + assert_eln(wrk1, nlev+1); + assert_eln(deta, nlev+1); + assert_eln(eta , nlev ); + const auto ttr = Kokkos::TeamThreadRange(kv.team, NP*NP); + const auto tvr = Kokkos::ThreadVectorRange(kv.team, nlev+1); + // eta -> deta; limit deta if needed. + const auto f1 = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + const auto etaij = getcolc( eta,i,j); + const auto detaij = getcol(deta,i,j); + const auto g1 = [&] (const int k, int& nbad) { + const auto d = (k == 0 ? etaij(0) - hy_etai(0) : + k == nlev ? hy_etai(nlev) - etaij(nlev-1) : + /**/ etaij(k) - etaij(k-1)); + const bool ok = d >= deta_tol; + if (not ok) ++nbad; + detaij(k) = d; + }; + int nbad = 0; + Dispatch<>::parallel_reduce(kv.team, tvr, g1, nbad); + if (nbad == 0) { + // Signal this column is fine. + Kokkos::single(Kokkos::PerThread(kv.team), [&] () { detaij(0) = -1; }); + return; + }; + deta_caas(kv, tvr, deta_ref, deta_tol, getcol(wrk1,i,j), detaij); + }; + Kokkos::parallel_for(ttr, f1); + kv.team_barrier(); + // deta -> eta; ignore columns where limiting wasn't needed. + const auto f2 = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + const auto etaij = getcol( eta,i,j); + const auto detaij = getcol(deta,i,j); + if (detaij(0) == -1) return; + const auto g = [&] (const int k, Real& accum, const bool final) { + assert(k != 0 or accum == 0); + const Real d = k == 0 ? hy_etai(0) + detaij(0) : detaij(k); + accum += d; + if (final) etaij(k) = accum; + }; + Dispatch<>::parallel_scan(kv.team, nlev, g); + }; + Kokkos::parallel_for(ttr, f2); +} + +// Compute surface pressure ps = ai(0) ps0 + sum dp. +KOKKOS_FUNCTION void calc_ps ( + const KernelVariables& kv, const int nlev, + const Real& ps0, const Real& hyai0, + const CSelnV& dp, + const ExecViewUnmanaged& ps) +{ + assert_eln(dp, nlev); + const auto ttr = Kokkos::TeamThreadRange(kv.team, NP*NP); + const auto tvr_snlev = Kokkos::ThreadVectorRange(kv.team, nlev); + const CRelnV dps = elp2r(dp); + const auto f1 = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + const auto g = [&] (int k, Real& sum) { sum += dps(i,j,k); }; + Real sum; + Dispatch<>::parallel_reduce(kv.team, tvr_snlev, g, sum); + Kokkos::single(Kokkos::PerThread(kv.team), + [&] { ps(i,j) = hyai0*ps0 + sum; }); + }; + Kokkos::parallel_for(ttr, f1); +} + +// Compute the surface pressure ps[i] at time point i corresponding to +// dp[i] = (1-alpha[i]) dp1 + alpha[i] dp2. +KOKKOS_FUNCTION void calc_ps ( + const KernelVariables& kv, const int nlev, + const Real& ps0, const Real& hyai0, + const Real alpha[2], const CSelnV& dp1, const CSelnV& dp2, + const ExecViewUnmanaged& ps) +{ + assert_eln(dp1, nlev); + assert_eln(dp2, nlev); + const auto ttr = Kokkos::TeamThreadRange(kv.team, NP*NP); + const auto tvr_snlev = Kokkos::ThreadVectorRange(kv.team, nlev); + const CRelnV dps[] = {elp2r(dp1), elp2r(dp2)}; + const auto f1 = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + for (int t = 0; t < 2; ++t) { + const auto& dp = dps[t]; + const auto g = [&] (int k, Real& sum) { sum += dp(i,j,k); }; + Real sum; + Dispatch<>::parallel_reduce(kv.team, tvr_snlev, g, sum); + Kokkos::single(Kokkos::PerThread(kv.team), [&] { ps(t,i,j) = sum; }); + } + }; + Kokkos::parallel_for(ttr, f1); + kv.team_barrier(); + const auto f2 = [&] (const int idx) { + const int i = idx / NP, j = idx % NP; + const auto g = [&] () { + Real vals[2]; + for (int t = 0; t < 2; ++t) + vals[t] = (hyai0*ps0 + + (1 - alpha[t])*ps(0,i,j) + + /**/ alpha[t] *ps(1,i,j)); + for (int t = 0; t < 2; ++t) + ps(t,i,j) = vals[t]; + }; + Kokkos::single(Kokkos::PerThread(kv.team), g); + }; + Kokkos::parallel_for(ttr, f2); +} + +// Transform eta_dot_dpdn at interfaces to eta_dot at midpoints using the +// formula +// eta_dot = eta_dot_dpdn/(A_eta p0 + B_eta ps). +// a= eta_dot_dpdn diff(eta)/(diff(A) p0 + diff(B) ps). +KOKKOS_FUNCTION void calc_etadotmid_from_etadotdpdnint ( + const KernelVariables& kv, const int nlev, + const Real& ps0, const CSnV& hydai, const CSnV& hydbi, + const CSnV& hydetai, const CRelV& ps, const SelnV& wrk, + // in: eta_dot_dpdn at interfaces + // out: eta_dot at midpoints, final slot unused + const SelnV& ed) +{ + assert(calc_nscal(hydai.extent_int(0)) >= nlev); + assert(calc_nscal(hydbi.extent_int(0)) >= nlev); + assert(calc_nscal(hydetai.extent_int(0)) >= nlev); + assert_eln(wrk, nlev+1); + assert_eln(ed, nlev+1); + const auto& edd_mid = wrk; + { + const CRelnV edd(elp2r(ed)); + const RelnV tmp(elp2r(wrk)); + const auto f = [&] (const int i, const int j, const int k) { + tmp(i,j,k) = (edd(i,j,k) + edd(i,j,k+1))/2; + }; + cti::loop_ijk(nlev, kv, f); + } + kv.team_barrier(); + { + const auto f = [&] (const int i, const int j, const int kp) { + ed(i,j,kp) = (edd_mid(i,j,kp) + * hydetai(kp) + / (hydai(kp)*ps0 + hydbi(kp)*ps(i,j))); + }; + cti::loop_ijk(calc_npack(nlev), kv, f); + } +} + +// Compute eta_dot at midpoint nodes at the start and end of the substep. +KOKKOS_FUNCTION void calc_eta_dot_ref_mid ( + const KernelVariables& kv, const SphereOperators& sphere_ops, + const Real& ps0, const Real& hyai0, const CSNV& hybi, + const CSNV& hydai, const CSNV& hydbi, // delta ai, bi + const CSNV& hydetai, // delta etai + const Real alpha[2], + const CS2elNlev& v1, const CSelNlev& dp1, const CS2elNlev& v2, const CSelNlev& dp2, + const SelNlevp& wrk1, const SelNlevp& wrk2, const S2elNlevp& vwrk1, + // Holds interface levels as intermediate data but is midpoint data on output, + // with final slot unused. + const SelNlevp eta_dot[2]) +{ + using Kokkos::ALL; + const int nlev = NUM_PHYSICAL_LEV; + const SelNlev divdp(wrk1.data()); + const S2elNlev vdp(vwrk1.data()); + const ExecViewUnmanaged ps(cti::pack2real(wrk2)); + // Calc surface pressure for use at the end. + calc_ps(kv, nlev, + ps0, hyai0, + alpha, dp1, dp2, + ps); + kv.team_barrier(); + for (int t = 0; t < 2; ++t) { + // Compute divdp. + const auto f = [&] (const int i, const int j, const int kp) { + for (int d = 0; d < 2; ++d) + vdp(d,i,j,kp) = ((1 - alpha[t])*v1(d,i,j,kp)*dp1(i,j,kp) + + /**/ alpha[t] *v2(d,i,j,kp)*dp2(i,j,kp)); + }; + cti::loop_ijk(kv, f); + kv.team_barrier(); + sphere_ops.divergence_sphere(kv, vdp, divdp); + kv.team_barrier(); + // Compute eta_dot_dpdn at interface nodes. + const auto& edd = eta_dot[t]; + const RelNlevp edds(cti::pack2real(edd)); + const RelNlev divdps(cti::pack2real(wrk1)); + cti::calc_eta_dot_dpdn(kv, + hybi, + divdps, edd, + edds); + kv.team_barrier(); + calc_etadotmid_from_etadotdpdnint(kv, nlev, + ps0, hydai, hydbi, hydetai, + Kokkos::subview(ps,t,ALL,ALL), + wrk1, + edd); + // No team_barrier: wrk1 is protected in second iteration. + } +} + +// Given the vertical and horizontal nodal velocities at time endpoints, +// evaluate the velocity estimate formula, providing the final horizontal +// velocity estimates at midpoint nodes. +KOKKOS_FUNCTION void calc_vel_horiz_formula_node_ref_mid ( + const KernelVariables& kv, const SphereOperators& sphere_ops, + const CSNV& hyetam, const ExecViewUnmanaged& vec_sph2cart, + // Velocities are at midpoints. Final eta_dot entry is ignored. + const Real dtsub, const CS2elNlev vsph[2], const CSelNlevp eta_dot[2], + const SelNlevp& wrk1, const S2elNlevp& vwrk1, const S2elNlevp& vwrk2, + const ExecViewUnmanaged& vnode) +{ + using Kokkos::ALL; + const S2elNlev vfsph(vwrk1.data()), vw2(vwrk2.data()); + const SelNlev w1(wrk1.data()); + const R2elNlev vfsphs(cti::pack2real(vfsph)); + const auto& vsph1 = vsph[0]; + const auto& vsph2 = vsph[1]; + { // Horizontal terms. + cti::ugradv_sphere(sphere_ops, kv, vec_sph2cart, vsph2, vsph1, w1, vw2, vfsph); + for (int d = 0; d < 2; ++d) { + const auto f = [&] (const int i, const int j, const int k) { + vfsph(d,i,j,k) = vsph1(d,i,j,k) + vsph2(d,i,j,k) - dtsub*vfsph(d,i,j,k); + }; + cti::loop_ijk(kv, f); + } + } + kv.team_barrier(); + { // Vertical terms. + const CRNV etams(cti::cpack2real(hyetam)); + const CR2elNlev vsph1s(cti::cpack2real(vsph1)); + const CRelNlevp eds(cti::cpack2real(eta_dot[1])); + for (int d = 0; d < 2; ++d) { + const auto f = [&] (const int i, const int j, const int k) { + Real deriv; + if (k == 0 or k+1 == NUM_PHYSICAL_LEV) { + const int k1 = k == 0 ? 0 : NUM_PHYSICAL_LEV-2; + const int k2 = k == 0 ? 1 : NUM_PHYSICAL_LEV-1; + deriv = ((vsph1s(d,i,j,k2) - vsph1s(d,i,j,k1)) / + (etams(k2) - etams(k1))); + } else { + deriv = cti::approx_derivative( + etams(k-1), etams(k), etams(k+1), + vsph1s(d,i,j,k-1), vsph1s(d,i,j,k), vsph1s(d,i,j,k+1)); + } + vfsphs(d,i,j,k) = (vfsphs(d,i,j,k) - dtsub*eds(i,j,k)*deriv)/2; + }; + cti::loop_ijk(kv, f); + } + } + { // Transform to Cartesian. + for (int d = 0; d < 3; ++d) { + const auto f = [&] (const int i, const int j, const int k) { + vnode(k,i,j,d) = (vec_sph2cart(0,d,i,j)*vfsphs(0,i,j,k) + + vec_sph2cart(1,d,i,j)*vfsphs(1,i,j,k)); + }; + cti::loop_ijk(kv, f); + } + } +} + +// Given the vertical and horizontal nodal velocities at time endpoints, +// evaluate the velocity estimate formula, providing the final vertical velocity +// estimates at midpoint nodes. +KOKKOS_FUNCTION void calc_eta_dot_formula_node_ref_mid ( + const KernelVariables& kv, const SphereOperators& sphere_ops, + const CRNV& hyetai, const CSNV& hyetam, + // Velocities are at midpoints. Final eta_dot entry is ignored. + const Real dtsub, const CS2elNlev vsph[2], const CSelNlevp eta_dot[2], + const SelNlevp& wrk1, const S2elNlevp& vwrk1, + const ExecViewUnmanaged& vnode) +{ + const SelNlev ed1_vderiv(wrk1.data()); + { + const CRNV etams(cti::cpack2real(hyetam)); + const CRelNlevp ed1s(cti::cpack2real(eta_dot[0])); + const RelNlev ed1_vderiv_s(cti::pack2real(ed1_vderiv)); + const auto f = [&] (const int i, const int j, const int k) { + Real deriv; + if (k == 0 or k+1 == NUM_PHYSICAL_LEV) { + deriv = cti::approx_derivative( + k == 0 ? hyetai(0) : etams(k-1), + etams(k), + k+1 == NUM_PHYSICAL_LEV ? hyetai(NUM_PHYSICAL_LEV) : etams(k+1), + k == 0 ? 0 : ed1s(i,j,k-1), + ed1s(i,j,k), + k+1 == NUM_PHYSICAL_LEV ? 0 : ed1s(i,j,k+1)); + } else { + deriv = cti::approx_derivative( + etams(k-1), etams(k), etams(k+1), + ed1s(i,j,k-1), ed1s(i,j,k), ed1s(i,j,k+1)); + } + ed1_vderiv_s(i,j,k) = deriv; + }; + cti::loop_ijk(kv, f); + } + kv.team_barrier(); + const S2elNlev ed1_hderiv(vwrk1.data()); + sphere_ops.gradient_sphere(kv, eta_dot[0], ed1_hderiv, NUM_LEV); + { + const auto& vsph2 = vsph[1]; + const auto& ed1 = eta_dot[0]; + const auto& ed2 = eta_dot[1]; + const auto f = [&] (const int i, const int j, const int k) { + const auto v = (ed1(i,j,k) + ed2(i,j,k) + - dtsub*( vsph2(0,i,j,k)*ed1_hderiv(0,i,j,k) + + vsph2(1,i,j,k)*ed1_hderiv(1,i,j,k) + + ed2( i,j,k)*ed1_vderiv( i,j,k)))/2; + for (int s = 0; s < VECTOR_SIZE; ++s) + vnode(VECTOR_SIZE*k+s, i,j,3) = v[s]; + }; + cti::loop_ijk(kv, f); + } +} + +} // namespace anon +} // namespace Homme + +#endif +#endif diff --git a/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectoryTests.cpp b/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectoryTests.cpp new file mode 100644 index 00000000000..da8fbab4d91 --- /dev/null +++ b/components/homme/src/share/cxx/ComposeTransportImplEnhancedTrajectoryTests.cpp @@ -0,0 +1,839 @@ +/******************************************************************************** + * HOMMEXX 1.0: Copyright of Sandia Corporation + * This software is released under the BSD license + * See the file 'COPYRIGHT' in the HOMMEXX/src/share/cxx directory + *******************************************************************************/ + +#include "Config.hpp" +#ifdef HOMME_ENABLE_COMPOSE + +#include "ComposeTransportImplEnhancedTrajectoryImpl.hpp" + +#include + +namespace Homme { +namespace { // anon + +Kokkos::TeamPolicy +get_test_team_policy (const int nelem, const int nlev, const int ncol=NP*NP) { + ThreadPreferences tp; + tp.max_threads_usable = ncol; + tp.max_vectors_usable = nlev; + tp.prefer_threads = true; + tp.prefer_larger_team = true; + return Homme::get_default_team_policy(nelem, tp); +} + +struct TestData { + std::mt19937_64 engine; + static const Real eps; + const ComposeTransportImpl& cti; + + TestData (const CTI& cti_, const int seed = 0) + : cti(cti_), engine(seed == 0 ? std::random_device()() : seed) + {} + + Real urand (const Real lo = 0, const Real hi = 1) { + std::uniform_real_distribution urb(lo, hi); + return urb(engine); + } +}; + +// Data to deal with views of packs easily in tests. +struct ColData { + int npack; + ExecView d; + ExecView::HostMirror h; + ExecView::HostMirror r; + + ColData (const std::string& name, const int nlev) { + npack = calc_npack(nlev); + d = decltype(d)(name, npack); + h = Kokkos::create_mirror_view(d); + r = decltype(r)(cti::pack2real(h), calc_nscal(npack)); + } + + void h2d () { Kokkos::deep_copy(d, h); } +}; + +struct ElData { + int npack; + ExecView d; + ExecView::HostMirror h; + ExecView::HostMirror r; + + ElData (const std::string& name, const int nlev) { + npack = calc_npack(nlev); + d = decltype(d)(name, NP, NP, npack); + h = Kokkos::create_mirror_view(d); + r = decltype(r)(cti::pack2real(h), NP, NP, calc_nscal(npack)); + } + + void d2h () { Kokkos::deep_copy(h, d); } + void h2d () { Kokkos::deep_copy(d, h); } +}; + +const Real TestData::eps = std::numeric_limits::epsilon(); + +int test_find_support (TestData&) { + int ne = 0; + const int n = 97; + std::vector x(n); + for (int i = 0; i < n; ++i) x[i] = -11.7 + (i*i)/n; + const int ntest = 10000; + for (int i = 0; i < ntest; ++i) { + const Real xi = x[0] + (Real(i)/ntest)*(x[n-1] - x[0]); + for (int x_idx : {0, 1, n/3, n/2, n-2, n-1}) { + const int sup = find_support(n, x.data(), x_idx, xi); + if (sup > n-2) ++ne; + else if (xi < x[sup] or xi > x[sup+1]) ++ne; + } + } + return ne; +} + +void todev (const std::vector& h, const RnV& d) { + assert(h.size() <= d.size()); + const auto m = Kokkos::create_mirror_view(d); + for (size_t i = 0; i < h.size(); ++i) m(i) = h[i]; + Kokkos::deep_copy(d, m); +} + +void fillcols (const int n, const Real* const h, const RelnV::HostMirror& a) { + assert(n <= a.extent_int(2)); + for (int i = 0; i < a.extent_int(0); ++i) + for (int j = 0; j < a.extent_int(1); ++j) + for (size_t k = 0; k < n; ++k) + a(i,j,k) = h[k]; +} + +void todev (const int n, const Real* const h, const RelnV& d) { + const auto m = Kokkos::create_mirror_view(d); + fillcols(n, h, m) ; + Kokkos::deep_copy(d, m); +} + +void todev (const std::vector& h, const RelnV& d) { + todev(h.size(), h.data(), d); +} + +void tohost (const ExecView& d, std::vector& h) { + assert(h.size() <= d.size()); + const auto m = Kokkos::create_mirror_view(d); + Kokkos::deep_copy(m, d); + for (size_t i = 0; i < h.size(); ++i) h[i] = m(i); +} + +void run_linterp (const std::vector& x, const std::vector& y, + std::vector& xi, std::vector& yi) { + const auto n = x.size(), ni = xi.size(); + assert(y.size() == n); assert(yi.size() == ni); + // input -> device (test different sizes >= n) + ExecView xv("xv", n), yv("yv", n+1), xiv("xiv", ni+2), yiv("yiv", ni+3); + todev(x, xv); + todev(y, yv); + todev(xi, xiv); + // call linterp + const auto f = KOKKOS_LAMBDA(const cti::MT& team) { + const auto range = Kokkos::TeamVectorRange(team, ni); + linterp(range, n, xv, yv, ni, xiv, yiv, 0, "unittest"); + }; + Homme::ThreadPreferences tp; + tp.max_threads_usable = 1; + tp.max_vectors_usable = ni; + tp.prefer_threads = false; + tp.prefer_larger_team = true; + const auto policy = get_test_team_policy(1, n); + Kokkos::parallel_for(policy, f); + Kokkos::fence(); + // output -> host + tohost(yiv, yi); +} + +void make_random_sorted (TestData& td, const int n, const Real xlo, const Real xhi, + std::vector& x) { + assert(n >= 2); + x.resize(n); + x[0] = xlo; + for (int i = 1; i < n-1; ++i) x[i] = td.urand(xlo, xhi); + x[n-1] = xhi; + std::sort(x.begin(), x.end()); +} + +int test_linterp (TestData& td) { + int nerr = 0; + { // xi == x => yi == y. + int ne = 0; + const int n = 30; + std::vector x(n), y(n), xi(n), yi(n); + make_random_sorted(td, n, -0.1, 1.2, x); + make_random_sorted(td, n, -3, -1, y); + for (int i = 0; i < n; ++i) xi[i] = x[i]; + run_linterp(x, y, xi, yi); + for (int i = 0; i < n; ++i) + if (yi[i] != y[i]) + ++ne; + nerr += ne; + } + { // Reconstruct a linear function exactly. + int ne = 0; + const int n = 56, ni = n-3; + const Real xlo = -1.2, xhi = 3.1; + const auto f = [&] (const Real x) { return -0.7 + 1.3*x; }; + std::vector x(n), y(n), xi(ni), yi(ni); + for (int trial = 0; trial < 4; ++trial) { + make_random_sorted(td, n, xlo, xhi, x); + make_random_sorted(td, ni, + xlo + (trial == 1 or trial == 3 ? 0.1 : 0), + xhi + (trial == 2 or trial == 3 ? -0.5 : 0), + xi); + for (int i = 0; i < n; ++i) y[i] = f(x[i]); + run_linterp(x, y, xi, yi); + for (int i = 0; i < ni; ++i) + if (std::abs(yi[i] - f(xi[i])) > 100*td.eps) + ++ne; + } + nerr += ne; + } + return nerr; +} + +int make_random_deta (TestData& td, const Real deta_tol, const int nlev, + Real* const deta) { + int nerr = 0; + Real sum = 0; + for (int k = 0; k < nlev; ++k) { + deta[k] = td.urand(0, 1) + 0.1; + sum += deta[k]; + } + for (int k = 0; k < nlev; ++k) { + deta[k] /= sum; + if (deta[k] < deta_tol) ++nerr; + } + return nerr; +} + +int make_random_deta (TestData& td, const Real deta_tol, const RnV& deta) { + int nerr = 0; + const int nlev = deta.extent_int(0); + const auto m = Kokkos::create_mirror_view(deta); + nerr = make_random_deta(td, deta_tol, nlev, &m(0)); + Kokkos::deep_copy(deta, m); + return nerr; +} + +int make_random_deta (TestData& td, const Real deta_tol, const RelnV& deta) { + int nerr = 0; + const int nlev = deta.extent_int(2); + const auto m = Kokkos::create_mirror_view(deta); + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) + nerr += make_random_deta(td, deta_tol, nlev, &m(i,j,0)); + Kokkos::deep_copy(deta, m); + return nerr; +} + +int test_deta_caas (TestData& td) { + int nerr = 0; + const Real tol = 100*td.eps; + + for (const int nlev : {15, 128, 161}) { + const Real deta_tol = 10*td.eps/nlev; + const auto err = [&] (const char* lbl) { + ++nerr; + printf("test_deta_caa nlev %d: %s\n", nlev, lbl); + }; + + // nlev+1 deltas: deta = diff([0, etam, 1]) + ExecView deta_ref("deta_ref", nlev+1); + ExecView deta("deta",NP,NP,nlev+1), wrk("wrk",NP,NP,nlev+1); + nerr += make_random_deta(td, deta_tol, deta_ref); + + const auto policy = get_test_team_policy(1, nlev); + const auto run = [&] (const RelnV& deta) { + const auto f = KOKKOS_LAMBDA(const cti::MT& team) { + KernelVariables kv(team); + deta_caas(kv, nlev+1, deta_ref, deta_tol, wrk, deta); + }; + Kokkos::parallel_for(policy, f); + Kokkos::fence(); + }; + + { // Test that if all is OK, the input is not altered. + nerr += make_random_deta(td, deta_tol, deta); + ExecView::HostMirror copy("copy",NP,NP,nlev+1); + Kokkos::deep_copy(copy, deta); + run(deta); + const auto m = cti::cmvdc(deta); + bool diff = false; + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) + for (int k = 0; k <= nlev; ++k) + if (m(i,j,k) != copy(i,j,k)) + diff = true; + if (diff) err("input not altered"); + } + + { // Modify one etam and test that only adjacent intervals change beyond eps. + // nlev midpoints + ExecView etam_ref("etam_ref",nlev); + const auto her = Kokkos::create_mirror_view(etam_ref); + const auto hder = cti::cmvdc(deta_ref); + { + her(0) = hder(0); + for (int k = 1; k < nlev; ++k) + her(k) = her(k-1) + hder(k); + Kokkos::deep_copy(etam_ref, her); + } + std::vector etam(nlev); + const auto hde = Kokkos::create_mirror_view(deta); + const auto get_idx = [&] (const int i, const int j) { + const int idx = static_cast(0.15*nlev); + return std::max(1, std::min(nlev-2, idx+NP*i+j)); + }; + for (int trial = 0; trial < 2; ++trial) { + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) { + for (int k = 0; k < nlev; ++k) etam[k] = her(k); + // Perturb one level. + const int idx = get_idx(i,j); + etam[idx] += trial == 0 ? 1.1 : -13.1; + hde(i,j,0) = etam[0]; + for (int k = 1; k < nlev; ++k) hde(i,j,k) = etam[k] - etam[k-1]; + hde(i,j,nlev) = 1 - etam[nlev-1]; + // Make sure we have a meaningful test. + Real minval = 1; + for (int k = 0; k <= nlev; ++k) minval = std::min(minval, hde(i,j,k)); + if (minval >= deta_tol) err("meaningful test"); + } + Kokkos::deep_copy(deta, hde); + run(deta); + Kokkos::deep_copy(hde, deta); + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) { + const int idx = get_idx(i,j); + // Min val should be deta_tol. + Real minval = 1; + for (int k = 0; k <= nlev; ++k) minval = std::min(minval, hde(i,j,k)); + if (minval != deta_tol) err("min val"); + // Sum of levels should be 1. + Real sum = 0; + for (int k = 0; k <= nlev; ++k) sum += hde(i,j,k); + if (std::abs(sum - 1) > tol) err("sum 1"); + // Only two deltas should be affected. + Real maxdiff = 0; + for (int k = 0; k <= nlev; ++k) { + const auto diff = std::abs(hde(i,j,k) - hder(k)); + if (k == idx or k == idx+1) { + if (diff <= deta_tol) err("2 deltas a"); + } else { + maxdiff = std::max(maxdiff, diff); + } + } + if (maxdiff > tol) err("2 deltas b"); + } + } + } + + { // Test generally (and highly) perturbed levels. + const auto hde = Kokkos::create_mirror_view(deta); + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) { + Real sum = 0; + for (int k = 0; k <= nlev; ++k) { + hde(i,j,k) = td.urand(-0.5, 0.5); + sum += hde(i,j,k); + } + // Make the column sum to 0.2 for safety in the next step. + const Real colsum = 0.2; + for (int k = 0; k <= nlev; ++k) hde(i,j,k) += (colsum - sum)/(nlev+1); + for (int k = 0; k <= nlev; ++k) hde(i,j,k) /= colsum; + sum = 0; + for (int k = 0; k <= nlev; ++k) sum += hde(i,j,k); + if (std::abs(sum - 1) > 10*tol) err("general sum 1"); + } + Kokkos::deep_copy(deta, hde); + run(deta); + Kokkos::deep_copy(hde, deta); + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) { + Real sum = 0, minval = 1; + for (int k = 0; k <= nlev; ++k) sum += hde(i,j,k); + for (int k = 0; k <= nlev; ++k) minval = std::min(minval, hde(i,j,k)); + if (std::abs(sum - 1) > 1e3*td.eps) ++nerr; + if (minval != deta_tol) err("general minval"); + } + } + } + + return nerr; +} + +struct HybridLevels { + Real ps0, a_eta, b_eta; + std::vector ai, dai, bi, dbi, am, bm, etai, detai, etam, detam; +}; + +// Follow DCMIP2012 3D tracer transport specification for a, b, eta. +void fill (HybridLevels& h, const int n) { + h.ai.resize(n+1); h.bi.resize(n+1); + h.am.resize(n ); h.bm.resize(n ); + h.etai.resize(n+1); h.etam.resize(n); + + const auto Rd = PhysicalConstants::Rgas; + const auto T0 = 300; // K + const auto p0 = PhysicalConstants::p0; + const auto g = PhysicalConstants::g; + const Real ztop = 12e3; // m + + h.ps0 = p0; + + const auto calc_pressure = [&] (const Real z) { + return p0*std::exp(-g*z/(Rd*T0)); + }; + + const Real eta_top = calc_pressure(ztop)/p0; + assert(eta_top > 0); + for (int i = 0; i <= n; ++i) { + const auto z = (Real(n - i)/n)*ztop; + h.etai[i] = calc_pressure(z)/p0; + h.bi[i] = i == 0 ? 0 : (h.etai[i] - eta_top)/(1 - eta_top); + h.ai[i] = h.etai[i] - h.bi[i]; + assert(i == 0 or h.etai[i] > h.etai[i-1]); + } + assert(h.bi [0] == 0); // Real(n - i)/n is exactly 1, so exact = holds + assert(h.bi [n] == 1); // exp(0) is exactly 0, so exact = holds + assert(h.etai[n] == 1); // same + // b = (eta - eta_top)/(1 - eta_top) => b_eta = 1/(1 - eta_top) + // a = eta - b => a_eta = 1 - b_eta = -eta_top/(1 - eta_top) + // p_eta = a_eta p0 + b_eta ps + h.b_eta = 1/(1 - eta_top); + h.a_eta = 1 - h.b_eta; + + const auto tomid = [&] (const std::vector& in, std::vector& mi) { + for (int i = 0; i < n; ++i) mi[i] = (in[i] + in[i+1])/2; + }; + tomid(h.ai, h.am); + tomid(h.bi, h.bm); + tomid(h.etai, h.etam); + + const auto diff = [&] (const std::vector& ai, std::vector& dai) { + dai.resize(n); + for (int i = 0; i < n; ++i) dai[i] = ai[i+1] - ai[i]; + }; + diff(h.ai, h.dai); + diff(h.bi, h.dbi); + diff(h.etai, h.detai); + + h.detam.resize(n+1); + h.detam[0] = h.etam[0] - h.etai[0]; + for (int i = 1; i < n; ++i) h.detam[i] = h.etam[i] - h.etam[i-1]; + h.detam[n] = h.etai[n] - h.etam[n-1]; +} + +int test_limit_etam (TestData& td) { + int nerr = 0; + const Real tol = 100*td.eps; + + for (const int nlev : {143, 128, 81}) { + const Real deta_tol = 1e5*td.eps/nlev; + + ExecView hy_etai("hy_etai",nlev+1), detam("detam",nlev+1); + ExecView wrk1("wrk1",NP,NP,nlev+1), wrk2("wrk2",NP,NP,nlev+1); + ExecView etam("etam",NP,NP,nlev); + + HybridLevels h; + fill(h, nlev); + todev(h.etai, hy_etai); + todev(h.detam, detam); + + const auto he = Kokkos::create_mirror_view(etam); + + const auto policy = get_test_team_policy(1, nlev); + const auto run = [&] () { + Kokkos::deep_copy(etam, he); + const auto f = KOKKOS_LAMBDA(const cti::MT& team) { + KernelVariables kv(team); + limit_etam(kv, nlev, hy_etai, detam, deta_tol, wrk1, wrk2, etam); + }; + Kokkos::parallel_for(policy, f); + Kokkos::fence(); + Kokkos::deep_copy(he, etam); + }; + + fillcols(h.etam.size(), h.etam.data(), he); + // Col 0 should be untouched. Cols 1 and 2 should have very specific changes. + const int col1_idx = static_cast(0.25*nlev); + he(0,1,col1_idx) += 0.3; + const int col2_idx = static_cast(0.8*nlev); + he(0,2,col2_idx) -= 5.3; + // The rest of the columns get wild changes. + for (int idx = 3; idx < NP*NP; ++idx) { + const int i = idx / NP, j = idx % NP; + for (int k = 0; k < nlev; ++k) + he(i,j,k) += td.urand(-1, 1)*(h.etai[k+1] - h.etai[k]); + } + run(); + bool ok = true; + for (int k = 0; k < nlev; ++k) + if (he(0,0,k) != h.etam[k]) ok = false; + for (int k = 0; k < nlev; ++k) { + if (k == col1_idx) continue; + if (std::abs(he(0,1,k) - h.etam[k]) > tol) ok = false; + } + for (int k = 0; k < nlev; ++k) { + if (k == col2_idx) continue; + if (std::abs(he(0,2,k) - h.etam[k]) > tol) ok = false; + } + Real mingap = 1; + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) { + mingap = std::min(mingap, he(i,j,0) - h.etai[0]); + for (int k = 1; k < nlev; ++k) + mingap = std::min(mingap, he(i,j,k) - he(i,j,k-1)); + mingap = std::min(mingap, h.etai[nlev] - he(i,j,nlev-1)); + } + // Test minimum level delta, with room for numerical error. + if (mingap < 0.8*deta_tol) ok = false; + if (not ok) ++nerr; + } + + return nerr; +} + +int test_eta_interp (TestData& td) { + int nerr = 0; + const Real tol = 100*td.eps; + + for (const int nlev : {15, 128, 161}) { + HybridLevels h; + fill(h, nlev); + + ExecView hy_etai("hy_etai",nlev+1); + ExecView x("x",NP,NP,nlev), y("y",NP,NP,nlev); + ExecView xi("xi",NP,NP,nlev+1), yi("yi",NP,NP,nlev+1); + ExecView xwrk("xwrk",NP,NP,nlev+2), ywrk("ywrk",NP,NP,nlev+2); + + todev(h.etai, hy_etai); + + const auto xh = Kokkos::create_mirror_view(x ); + const auto yh = Kokkos::create_mirror_view(y ); + const auto xih = Kokkos::create_mirror_view(xi); + const auto yih = Kokkos::create_mirror_view(yi); + + const auto policy = get_test_team_policy(1, nlev); + const auto run_eta = [&] (const int ni) { + Kokkos::deep_copy(x, xh); Kokkos::deep_copy(y, yh); + Kokkos::deep_copy(xi, xih); + const auto f = KOKKOS_LAMBDA(const cti::MT& team) { + KernelVariables kv(team); + eta_interp_eta(kv, nlev, hy_etai, + x, getcolc(y,0,0), + xwrk, getcol(ywrk,0,0), + ni, getcolc(xi,0,0), yi); + }; + Kokkos::parallel_for(policy, f); + Kokkos::fence(); + Kokkos::deep_copy(yih, yi); + }; + const auto run_horiz = [&] () { + Kokkos::deep_copy(x, xh); Kokkos::deep_copy(y, yh); + Kokkos::deep_copy(xi, xih); + const auto f = KOKKOS_LAMBDA(const cti::MT& team) { + KernelVariables kv(team); + eta_interp_horiz(kv, nlev, hy_etai, + getcolc(x,0,0), y, + getcol(xwrk,0,0), ywrk, + xi, yi); + }; + Kokkos::parallel_for(policy, f); + Kokkos::fence(); + Kokkos::deep_copy(yih, yi); + }; + + std::vector v; + const Real d = 1e-6, vlo = h.etai[0]+d, vhi = h.etai[nlev]-d; + + for (const int ni : {int(0.7*nlev), nlev-1, nlev, nlev+1}) { + make_random_sorted(td, nlev, vlo, vhi, v); + fillcols(nlev, v.data(), xh); + fillcols(nlev, v.data(), yh); + make_random_sorted(td, ni, vlo, vhi, v); + fillcols(ni, v.data(), xih); + run_eta(ni); + bool ok = true; + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) + for (int k = 0; k < ni; ++k) + if (std::abs(yih(i,j,k) - xih(i,j,k)) > tol) + ok = false; + if (not ok) ++nerr; + } + + { // Test exact interp of line in the interior, const interp near the bdys. + make_random_sorted(td, nlev, vlo+0.05, vhi-0.1, v); + fillcols(nlev, v.data(), xh); + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) { + for (int k = 0; k < nlev; ++k) + yh(i,j,k) = i*xh(0,0,k) - j; + make_random_sorted(td, nlev, vlo, vhi, v); + for (int k = 0; k < nlev; ++k) + xih(i,j,k) = v[k]; + } + run_horiz(); + bool ok = true; + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) + for (int k = 0; k < nlev; ++k) { + if (xih(i,j,k) < xh(0,0,0)) { + if (std::abs(yih(i,j,k) - yih(i,j,0)) > tol) + ok = false; + } else if (xih(i,j,k) > xh(0,0,nlev-1)) { + if (std::abs(yih(i,j,k) - yih(i,j,nlev-1)) > tol) + ok = false; + } else { + if (std::abs(yih(i,j,k) - (i*xih(i,j,k) - j)) > tol) + ok = false; + } + } + if (not ok) ++nerr; + } + } + + return nerr; +} + +int test_eta_to_dp (TestData& td) { + int nerr = 0; + const Real tol = 100*td.eps; + + for (const int nlev : {143, 128, 81}) { + const auto err = [&] (const char* lbl) { + ++nerr; + printf("test_eta_to_dp nlev %d: %s\n", nlev, lbl); + }; + + HybridLevels h; + fill(h, nlev); + + ExecView hy_bi("hy_bi",nlev+1), hy_etai("hy_etai",nlev+1); + ExecView etai("etai",NP,NP,nlev+1), wrk("wrk",NP,NP,nlev+1); + ExecView dp("dp",NP,NP,nlev); + ExecView ps("ps"); + const Real hy_ps0 = h.ps0; + + todev(h.bi, hy_bi); + todev(h.etai, hy_etai); + + const auto psm = Kokkos::create_mirror_view(ps); + HostView dp1("dp1",NP,NP,nlev); + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) + psm(i,j) = (1 + 0.1*td.urand(-1, 1))*h.ps0; + Kokkos::deep_copy(ps, psm); + + const auto policy = get_test_team_policy(1, nlev); + const auto run = [&] () { + const auto f = KOKKOS_LAMBDA(const cti::MT& team) { + KernelVariables kv(team); + eta_to_dp(kv, nlev, hy_ps0, hy_bi, hy_etai, ps, etai, wrk, dp); + }; + Kokkos::parallel_for(policy, f); + Kokkos::fence(); + }; + + { // Test that for etai_ref we get the same as the usual formula. + todev(h.etai, etai); + HostView dp1("dp1",NP,NP,nlev); + Real dp1_max = 0; + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) + for (int k = 0; k < nlev; ++k) { + dp1(i,j,k) = ((h.ai[k+1] - h.ai[k])*h.ps0 + + (h.bi[k+1] - h.bi[k])*psm(i,j)); + dp1_max = std::max(dp1_max, std::abs(dp1(i,j,k))); + } + run(); + const auto dph = cti::cmvdc(dp); + Real err_max = 0; + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) + for (int k = 0; k < nlev; ++k) + err_max = std::max(err_max, std::abs(dph(i,j,k) - dp1(i,j,k))); + if (err_max > tol*dp1_max) err("t1"); + } + + { // Test that sum(dp) = ps for random input etai. + std::vector etai_r; + make_random_sorted(td, nlev+1, h.etai[0], h.etai[nlev], etai_r); + todev(etai_r, etai); + run(); + const auto dph1 = cti::cmvdc(dp); + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) { + Real ps = h.ai[0]*h.ps0; + for (int k = 0; k < nlev; ++k) + ps += dph1(i,j,k); + if (std::abs(ps - psm(i,j)) > tol*psm(i,j)) err("t2"); + } + // Test that values on input don't affect solution. + Kokkos::deep_copy(wrk, 0); + Kokkos::deep_copy(dp, 0); + run(); + const auto dph2 = cti::cmvdc(dp); + bool alleq = true; + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) + for (int k = 0; k < nlev; ++k) + if (dph2(i,j,k) != dph1(i,j,k)) + alleq = false; + if (not alleq) err("t3"); + } + } + + return nerr; +} + +int test_calc_ps (TestData& td) { + int nerr = 0; + const Real tol = 100*td.eps; + + for (const int nlev : {15, 128, 161}) { + HybridLevels h; + fill(h, nlev); + const auto ps0 = h.ps0, hyai0 = h.ai[0]; + + ElData dp1("dp1", nlev), dp2("dp2", nlev); + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) + for (int k = 0; k < nlev; ++k) { + dp1.r(i,j,k) = td.urand(0, 1000); + dp2.r(i,j,k) = td.urand(0, 1000); + } + dp1.h2d(); + dp2.h2d(); + + const Real alpha[] = {td.urand(0,1), td.urand(0,1)}; + + ExecView ps("ps"); + ExecView ps2("ps2"); + const auto policy = get_test_team_policy(1, nlev); + const auto f = KOKKOS_LAMBDA(const cti::MT& team) { + KernelVariables kv(team); + calc_ps(kv, nlev, ps0, hyai0, alpha, dp1.d, dp2.d, ps2); + calc_ps(kv, nlev, ps0, hyai0, dp1.d, ps); + }; + Kokkos::parallel_for(policy, f); + Kokkos::fence(); + + const auto ps_h = cti::cmvdc(ps); + const auto ps2_h = cti::cmvdc(ps2); + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) { + { + Real ps = h.ai[0]*h.ps0; + for (int k = 0; k < nlev; ++k) + ps += dp1.r(i,j,k); + if (std::abs(ps_h(i,j) - ps) > tol*ps) ++nerr; + } + for (int t = 0; t < 2; ++t) { + Real ps = h.ai[0]*h.ps0; + for (int k = 0; k < nlev; ++k) + ps += (1 - alpha[t])*dp1.r(i,j,k) + alpha[t]*dp2.r(i,j,k); + if (std::abs(ps2_h(t,i,j) - ps) > tol*ps) ++nerr; + } + } + } + + return nerr; +} + +int test_calc_etadotmid_from_etadotdpdnint (TestData& td) { + int nerr = 0; + const Real tol = 100*td.eps; + + for (const int nlev : {143, 128, 81}) { + HybridLevels h; + fill(h, nlev); + + // Test function: + // eta_dot_dpdn(eta) = c eta + d. + // Then + // eta_dot = eta_dot_dpdn(eta)/dpdn(eta) + // = (c eta + d)/(a_eta p0 + b_eta ps). + // Since a_eta, b_eta are constants independent of eta in this test, eta_dot + // is then also a linear function of eta. Thus, we can test for exact + // agreement with the true solution. + + ColData hydai("hydai",nlev), hydbi("hydbi",nlev), hydetai("hydetai",nlev); + ElData wrk("wrk",nlev+1), ed("ed",nlev+1); + ExecView ps("ps"); + const Real ps0 = h.ps0; + + const auto ps_m = Kokkos::create_mirror_view(ps); + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) { + ps_m(i,j) = td.urand(0.5, 1.2)*ps0; + for (int k = 0; k < nlev; ++k) { + hydai.r[k] = h.dai[k]; + hydbi.r[k] = h.dbi[k]; + hydetai.r[k] = h.detai[k]; + } + for (int k = 0; k <= nlev; ++k) + ed.r(i,j,k) = (i-j)*h.etai[k] + 0.3; + } + Kokkos::deep_copy(ps, ps_m); + hydai.h2d(); hydbi.h2d(); hydetai.h2d(); + ed.h2d(); + + const auto policy = get_test_team_policy(1, nlev); + const auto f = KOKKOS_LAMBDA(const cti::MT& team) { + KernelVariables kv(team); + calc_etadotmid_from_etadotdpdnint( + kv, nlev, ps0, hydai.d, hydbi.d, hydetai.d, ps, wrk.d, ed.d); + }; + Kokkos::parallel_for(policy, f); + Kokkos::fence(); + ed.d2h(); + + for (int i = 0; i < NP; ++i) + for (int j = 0; j < NP; ++j) { + const auto den = h.a_eta*h.ps0 + h.b_eta*ps_m(i,j); + for (int k = 0; k < nlev; ++k) { + const auto ed_true = ((i-j)*h.etam[k] + 0.3)/den; + if (std::abs(ed.r(i,j,k) - ed_true) > tol*(10/den)) ++nerr; + } + } + } + + return nerr; +} + +} // namespace anon + +#define comunittest(f) do { \ + ne = f(td); \ + if (ne) printf(#f " ne %d\n", ne); \ + nerr += ne; \ + } while (0) + +int ComposeTransportImpl::run_enhanced_trajectory_unit_tests () { + int nerr = 0, ne; + TestData td(*this); + comunittest(test_find_support); + comunittest(test_linterp); + comunittest(test_eta_interp); + comunittest(test_eta_to_dp); + comunittest(test_deta_caas); + comunittest(test_limit_etam); + comunittest(test_calc_ps); + comunittest(test_calc_etadotmid_from_etadotdpdnint); + return nerr; +} + +#undef comunittest + +} // namespace Homme + +#endif diff --git a/components/homme/src/share/cxx/ComposeTransportImplGeneral.cpp b/components/homme/src/share/cxx/ComposeTransportImplGeneral.cpp index ad62ca3904f..bbffbeb8e48 100644 --- a/components/homme/src/share/cxx/ComposeTransportImplGeneral.cpp +++ b/components/homme/src/share/cxx/ComposeTransportImplGeneral.cpp @@ -12,7 +12,8 @@ extern "C" void sl_get_params(double* nu_q, double* hv_scaling, int* hv_q, int* hv_subcycle_q, - int* limiter_option, int* cdr_check, int* geometry_type); + int* limiter_option, int* cdr_check, int* geometry_type, + int* trajectory_nsubstep); namespace Homme { @@ -46,6 +47,8 @@ void ComposeTransportImpl::setup () { m_sphere_ops = Context::singleton().get(); set_dp_tol(); + setup_enhanced_trajectory(); + nslot = calc_nslot(m_geometry.num_elems()); } @@ -53,6 +56,11 @@ void ComposeTransportImpl::reset (const SimulationParams& params) { const auto num_elems = Context::singleton().get().get_num_local_elements(); const bool independent_time_steps = params.dt_tracer_factor > params.dt_remap_factor; + + sl_get_params(&m_data.nu_q, &m_data.hv_scaling, &m_data.hv_q, &m_data.hv_subcycle_q, + &m_data.limiter_option, &m_data.cdr_check, &m_data.geometry_type, + &m_data.trajectory_nsubstep); + if (independent_time_steps != m_data.independent_time_steps || m_data.nelemd != num_elems || m_data.qsize != params.qsize) { const auto& g = m_geometry; @@ -61,7 +69,14 @@ void ComposeTransportImpl::reset (const SimulationParams& params) { const auto& d = m_derived; const auto nel = num_elems; const auto nlev = NUM_LEV*packn; - m_data.dep_pts = DeparturePoints("dep_pts", nel); + const int ndim = (m_data.trajectory_nsubstep == 0 ? + 3 : + (independent_time_steps ? 4 : 3)); + m_data.dep_pts = DeparturePoints("dep_pts", nel, num_phys_lev, np, np, ndim); + if (m_data.trajectory_nsubstep > 0) + m_data.vnode = DeparturePoints("vnode", nel, num_phys_lev, np, np, ndim); + if (m_data.trajectory_nsubstep > 1) + m_data.vdep = DeparturePoints("vdep" , nel, num_phys_lev, np, np, ndim); homme::compose::set_views( g.m_spheremp, homme::compose::SetView (reinterpret_cast(d.m_dp.data()), @@ -76,8 +91,9 @@ void ComposeTransportImpl::reset (const SimulationParams& params) { nel, t.qdp.extent_int(1), t.qdp.extent_int(2), np, np, nlev), homme::compose::SetView (reinterpret_cast(t.Q.data()), nel, t.Q.extent_int(1), np, np, nlev), - m_data.dep_pts); + m_data.dep_pts, m_data.vnode, m_data.vdep, ndim); } + m_data.independent_time_steps = independent_time_steps; if (m_data.nelemd == num_elems && m_data.qsize == params.qsize) return; @@ -86,8 +102,6 @@ void ComposeTransportImpl::reset (const SimulationParams& params) { "SL transport requires qsize > 0; if qsize == 0, use Eulerian."); m_data.nelemd = num_elems; - sl_get_params(&m_data.nu_q, &m_data.hv_scaling, &m_data.hv_q, &m_data.hv_subcycle_q, - &m_data.limiter_option, &m_data.cdr_check, &m_data.geometry_type); Errors::runtime_check(m_data.hv_q >= 0 && m_data.hv_q <= m_data.qsize, "semi_lagrange_hv_q should be in [0, qsize]."); Errors::runtime_check(m_data.hv_subcycle_q >= 0, @@ -113,17 +127,18 @@ void ComposeTransportImpl::reset (const SimulationParams& params) { int ComposeTransportImpl::requested_buffer_size () const { // FunctorsBuffersManager wants the size in terms of sizeof(Real). - return (3*Buf1::shmem_size(nslot) + - 2*Buf2::shmem_size(nslot))/sizeof(Real); + return (m_data.n_buf1*Buf1Alloc::shmem_size(nslot) + + m_data.n_buf2*Buf2::shmem_size(nslot))/sizeof(Real); } void ComposeTransportImpl::init_buffers (const FunctorsBuffersManager& fbm) { Scalar* mem = reinterpret_cast(fbm.get_memory()); - for (int i = 0; i < 3; ++i) { - m_data.buf1[i] = Buf1(mem, nslot); - mem += Buf1::shmem_size(nslot)/sizeof(Scalar); + for (int i = 0; i < m_data.n_buf1; ++i) { + m_data.buf1o[i] = Buf1o(mem, nslot); + m_data.buf1e[i] = Buf1e(mem, nslot); // use the same memory + mem += Buf1Alloc::shmem_size(nslot)/sizeof(Scalar); } - for (int i = 0; i < 2; ++i) { + for (int i = 0; i < m_data.n_buf2; ++i) { m_data.buf2[i] = Buf2(mem, nslot); mem += Buf2::shmem_size(nslot)/sizeof(Scalar); } @@ -148,16 +163,29 @@ void ComposeTransportImpl::init_boundary_exchanges () { be->registration_completed(); } - for (int i = 0; i < 2; ++i) { - m_v_dss_be[i] = std::make_shared(); - auto be = m_v_dss_be[i]; - be->set_label(std::string("ComposeTransport-v-DSS-" + std::to_string(i))); - be->set_diagnostics_level(sp.internal_diagnostics_level); - be->set_buffers_manager(bm_exchange); - be->set_num_fields(0, 0, 2 + (i ? 1 : 0)); - be->register_field(m_derived.m_vstar, 2, 0); - if (i) be->register_field(m_derived.m_divdp); - be->registration_completed(); + if (m_data.trajectory_nsubstep == 0) { + for (int i = 0; i < 2; ++i) { + m_v_dss_be[i] = std::make_shared(); + auto be = m_v_dss_be[i]; + be->set_label(std::string("ComposeTransport-v-DSS-" + std::to_string(i))); + be->set_diagnostics_level(sp.internal_diagnostics_level); + be->set_buffers_manager(bm_exchange); + be->set_num_fields(0, 0, 2+i); + be->register_field(m_derived.m_vstar, 2, 0); + if (i) be->register_field(m_derived.m_divdp); + be->registration_completed(); + } + } else { + for (int i = 0; i < 2; ++i) { + m_v_dss_be[i] = std::make_shared(); + auto be = m_v_dss_be[i]; + be->set_label(std::string("ComposeTransport-v-DSS-" + std::to_string(i))); + be->set_diagnostics_level(sp.internal_diagnostics_level); + be->set_buffers_manager(bm_exchange); + be->set_num_fields(0, 0, 3+i); + be->register_field(m_tracers.qtens_biharmonic, 3+i, 0); + be->registration_completed(); + } } // For optional HV applied to q. @@ -181,10 +209,14 @@ void ComposeTransportImpl::init_boundary_exchanges () { void ComposeTransportImpl::run (const TimeLevel& tl, const Real dt) { GPTLstart("compose_transport"); - calc_trajectory(tl.np1, dt); + if (m_data.trajectory_nsubstep == 0) + calc_trajectory(tl.np1, dt); + else + calc_enhanced_trajectory(tl.np1, dt); GPTLstart("compose_isl"); homme::compose::advect(tl.np1, tl.n0_qdp, tl.np1_qdp); + Kokkos::fence(); GPTLstop("compose_isl"); if (m_data.hv_q > 0 && m_data.nu_q > 0) { diff --git a/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp b/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp index aa91c752a7c..197ec7528a4 100644 --- a/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp +++ b/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp @@ -14,75 +14,19 @@ namespace Homme { using cti = ComposeTransportImpl; - -KOKKOS_FUNCTION -static void ugradv_sphere ( - const SphereOperators& sphere_ops, const KernelVariables& kv, - const typename ViewConst >::type& vec_sphere2cart, - // velocity, latlon - const typename ViewConst >::type& u, - const typename ViewConst >::type& v, - const ExecViewUnmanaged& v_cart, - const ExecViewUnmanaged& ugradv_cart, - // [u dot grad] v, latlon - const ExecViewUnmanaged& ugradv) -{ - for (int d_cart = 0; d_cart < 3; ++d_cart) { - const auto f1 = [&] (const int i, const int j, const int k) { - v_cart(i,j,k) = (vec_sphere2cart(0,d_cart,i,j) * v(0,i,j,k) + - vec_sphere2cart(1,d_cart,i,j) * v(1,i,j,k)); - }; - cti::loop_ijk(kv, f1); - kv.team_barrier(); - - sphere_ops.gradient_sphere(kv, v_cart, ugradv_cart); - - const auto f2 = [&] (const int i, const int j, const int k) { - if (d_cart == 0) ugradv(0,i,j,k) = ugradv(1,i,j,k) = 0; - for (int d_latlon = 0; d_latlon < 2; ++d_latlon) - ugradv(d_latlon,i,j,k) += - vec_sphere2cart(d_latlon,d_cart,i,j)* - (u(0,i,j,k) * ugradv_cart(0,i,j,k) + u(1,i,j,k) * ugradv_cart(1,i,j,k)); - }; - cti::loop_ijk(kv, f2); - } -} - -typedef typename ViewConst >::type CSNlev; -typedef typename ViewConst >::type CRNlev; -typedef typename ViewConst >::type CSNlevp; -typedef typename ViewConst >::type CRNlevp; -typedef typename ViewConst >::type CS2Nlev; -typedef ExecViewUnmanaged SNlev; -typedef ExecViewUnmanaged RNlev; -typedef ExecViewUnmanaged SNlevp; -typedef ExecViewUnmanaged RNlevp; -typedef ExecViewUnmanaged S2Nlev; -typedef ExecViewUnmanaged R2Nlev; -typedef ExecViewUnmanaged S2Nlevp; - -/* Form a 3rd-degree Lagrange polynomial over (x(k-1:k+1), y(k-1:k+1)) and set - yi(k) to its derivative at x(k). yps(:,:,0) is not written. - */ -KOKKOS_FUNCTION static void approx_derivative ( - const KernelVariables& kv, const CSNlevp& xs, const CSNlevp& ys, - const SNlev& yps) // yps(:,:,0) is undefined -{ - CRNlevp x(cti::cpack2real(xs)); - CRNlevp y(cti::cpack2real(ys)); - RNlev yp(cti::pack2real(yps)); - const auto f = [&] (const int i, const int j, const int k) { - if (k == 0) return; - const auto& xkm1 = x(i,j,k-1); - const auto& xk = x(i,j,k ); // also the interpolation point - const auto& xkp1 = x(i,j,k+1); - yp(i,j,k) = (y(i,j,k-1)*(( 1 /(xkm1 - xk ))*((xk - xkp1)/(xkm1 - xkp1))) + - y(i,j,k )*(( 1 /(xk - xkm1))*((xk - xkp1)/(xk - xkp1)) + - ((xk - xkm1)/(xk - xkm1))*( 1 /(xk - xkp1))) + - y(i,j,k+1)*(((xk - xkm1)/(xkp1 - xkm1))*( 1 /(xkp1 - xk )))); - }; - cti::loop_ijk(kv, f); -} +using CSNlev = cti::CSNlev; +using CRNlev = cti::CRNlev; +using CSNlevp = cti::CSNlevp; +using CRNlevp = cti::CRNlevp; +using CS2Nlev = cti::CS2Nlev; +using CR2Nlev = cti::CR2Nlev; +using SNlev = cti::SNlev; +using RNlev = cti::RNlev; +using SNlevp = cti::SNlevp; +using RNlevp = cti::RNlevp; +using S2Nlev = cti::S2Nlev; +using R2Nlev = cti::R2Nlev; +using S2Nlevp = cti::S2Nlevp; // Pad by an amount ~ smallest level to keep the computed dp > 0. void ComposeTransportImpl::set_dp_tol () { @@ -234,31 +178,11 @@ KOKKOS_FUNCTION static void calc_vertically_lagrangian_levels ( }; cti::loop_ijk(kv, f); } - + kv.team_barrier(); sphere_ops.divergence_sphere(kv, vdp, divdp); - + kv.team_barrier(); RNlevp edds(cti::pack2real(edd)), divdps(cti::pack2real(divdp)); - const auto f = [&] (const int idx) { - const int i = idx / NP, j = idx % NP; - const auto r = [&] (const int k, Real& dps, const bool final) { - assert(k != 0 || dps == 0); - if (final) edds(i,j,k) = dps; - dps += divdps(i,j,k); - }; - Dispatch<>::parallel_scan(kv.team, cti::num_phys_lev, r); - const int kend = cti::num_phys_lev - 1; - const Real dps = edds(i,j,kend) + divdps(i,j,kend); - assert(hybrid_bi(0)[0] == 0); - const auto s = [&] (const int kp) { - edd(i,j,kp) = hybrid_bi(kp)*dps - edd(i,j,kp); - if (kp == 0) edd(i,j,kp)[0] = 0; - }; - parallel_for(tvr, s); - assert(edds(i,j,0) == 0); - const int bottom = cti::num_phys_lev; - edds(i,j,bottom) = 0; - }; - parallel_for(ttr, f); + cti::calc_eta_dot_dpdn(kv, hybrid_bi, divdps, edd, edds); } // Use p0 as the reference coordinate system. p0 differs from p1 by B(eta) @@ -278,11 +202,11 @@ KOKKOS_FUNCTION static void calc_vertically_lagrangian_levels ( // Gradient of eta_dot_dpdn = p_eta deta/dt at final time w.r.t. p at initial // time. const auto& ptp0 = dprecon; - approx_derivative(kv, pref, *eta_dot_dpdn[1], ptp0); + cti::approx_derivative(kv, pref, *eta_dot_dpdn[1], ptp0); { const auto& edd = *eta_dot_dpdn[0]; - const R2Nlev vstars(cti::pack2real(vstar)); + const CR2Nlev vstars(cti::cpack2real(vstar)); const auto f_v = [&] (const int i, const int j, const int kp) { // Horizontal velocity at initial time. Scalar v[2]; @@ -353,9 +277,9 @@ void ComposeTransportImpl::calc_trajectory (const int np1, const Real dt) { const auto m_vstar = m_derived.m_vstar; const auto tu_ne = m_tu_ne; { // Calculate midpoint velocity. - const auto buf1a = m_data.buf1[0]; const auto buf1b = m_data.buf1[1]; - const auto buf1c = m_data.buf1[2]; const auto buf2a = m_data.buf2[0]; - const auto buf2b = m_data.buf2[1]; + const auto buf1a = m_data.buf1o[0]; const auto buf1b = m_data.buf1o[1]; + const auto buf1c = m_data.buf1o[2]; + const auto buf2a = m_data.buf2[0]; const auto buf2b = m_data.buf2[1]; const auto m_spheremp = geo.m_spheremp; const auto m_rspheremp = geo.m_rspheremp; const auto m_v = m_state.m_v; @@ -457,7 +381,7 @@ void ComposeTransportImpl::calc_trajectory (const int np1, const Real dt) { const auto vstar = Homme::subview(m_vstar, ie); const auto vec_sphere2cart = Homme::subview(m_vec_sph2cart, ie); const auto sphere_cart = Homme::subview(m_sphere_cart, ie); - const auto dep_pts = Homme::subview(m_dep_pts, ie); + const auto dep_pts = m_dep_pts; const auto f = [&] (const int i, const int j, const int k) { // dp = p1 - dt v/scale_factor Scalar dp[3]; @@ -476,7 +400,7 @@ void ComposeTransportImpl::calc_trajectory (const int np1, const Real dt) { // No vec call for sqrt. const auto r = is_sphere ? std::sqrt(r2[s]) : 1; for (int d = 0; d < 3; ++d) - dep_pts(oss,i,j,d) = dp[d][s]/r; + dep_pts(ie,oss,i,j,d) = dp[d][s]/r; } }; cti::loop_ijk(kv, f); @@ -511,7 +435,7 @@ static int test_approx_derivative () { { // Run approx_derivative. const auto f = KOKKOS_LAMBDA (const cti::MT& team) { KernelVariables kv(team); - approx_derivative(kv, xp, yp, yip); + cti::approx_derivative(kv, xp, yp, yip); }; Kokkos::fence(); Kokkos::parallel_for(policy, f); diff --git a/components/homme/src/share/cxx/ExecSpaceDefs.cpp b/components/homme/src/share/cxx/ExecSpaceDefs.cpp index 0b7a3ab34f8..dbe82d8758b 100644 --- a/components/homme/src/share/cxx/ExecSpaceDefs.cpp +++ b/components/homme/src/share/cxx/ExecSpaceDefs.cpp @@ -14,7 +14,7 @@ #include "utilities/MathUtils.hpp" #ifdef KOKKOS_ENABLE_CUDA -# include +#include #endif #ifdef KOKKOS_ENABLE_HIP @@ -32,56 +32,46 @@ namespace Homme { // own. As a side benefit, we'll end up running on GPU platforms optimally // without having to specify --kokkos-ndevices on the command line. void initialize_kokkos () { - // This is in fact const char*, but Kokkos::initialize requires char*. - std::vector args; - - // This is the only way to get the round-robin rank assignment Kokkos + // Count up our devices. + // This is the only way to get the round-robin rank assignment Kokkos // provides, as that algorithm is hardcoded in Kokkos::initialize(int& narg, // char* arg[]). Once the behavior is exposed in the InitArguments version of // initialize, we can remove this string code. // If for some reason we're running on a GPU platform, have Cuda enabled, // but are using a different execution space, this initialization is still // OK. The rank gets a GPU assigned and simply will ignore it. -#ifdef KOKKOS_ENABLE_CUDA - int nd; + int nd = 1; +#ifdef HOMMEXX_ENABLE_GPU +# if defined KOKKOS_ENABLE_CUDA const auto ret = cudaGetDeviceCount(&nd); - if (ret != cudaSuccess) { - // It isn't a big deal if we can't get the device count. - nd = 1; - } -#elif defined(KOKKOS_ENABLE_HIP) - int nd; + const bool ok = (ret == cudaSuccess); +# elif defined KOKKOS_ENABLE_HIP const auto ret = hipGetDeviceCount(&nd); - if (ret != hipSuccess) { + const bool ok = (ret == hipSuccess); +# elif defined KOKKOS_ENABLE_SYCL + nd = 0; + auto gpu_devs = sycl::device::get_devices(sycl::info::device_type::gpu); + for (auto &dev : gpu_devs) { + if (dev.get_info() > 0) { + auto subDevs = dev.create_sub_devices(sycl::info::partition_affinity_domain::numa); + nd += subDevs.size(); + } else { + nd++; + } + } + const bool ok = true; +# endif + if (not ok) { // It isn't a big deal if we can't get the device count. nd = 1; } -#elif defined(KOKKOS_ENABLE_SYCL) +#endif // HOMMEXX_ENABLE_GPU -//https://developer.codeplay.com/products/computecpp/ce/2.11.0/guides/sycl-for-cuda-developers/migrating-from-cuda-to-sycl - -//to make it build - int nd = 1; - -#endif - - -#ifdef HOMMEXX_ENABLE_GPU - std::stringstream ss; - ss << "--kokkos-num-devices=" << nd; - const auto key = ss.str(); - std::vector str(key.size()+1); - std::copy(key.begin(), key.end(), str.begin()); - str.back() = 0; - args.push_back(const_cast(str.data())); -#endif - - - const char* silence = "--kokkos-disable-warnings"; - args.push_back(const_cast(silence)); - - int narg = args.size(); - Kokkos::initialize(narg, args.data()); + auto const settings = Kokkos::InitializationSettings() + .set_map_device_id_by("mpi_rank") + .set_num_devices(nd) + .set_disable_warnings(true); + Kokkos::initialize(settings); } ThreadPreferences::ThreadPreferences () @@ -214,7 +204,7 @@ team_num_threads_vectors (const int num_parallel_iterations, #endif min_num_warps = std::min(min_num_warps, max_num_warps); - + return Parallel::team_num_threads_vectors_for_gpu( num_warps_device, num_threads_warp, min_num_warps, max_num_warps, diff --git a/components/homme/src/share/cxx/GllFvRemapImpl.hpp b/components/homme/src/share/cxx/GllFvRemapImpl.hpp index 11738b2bf45..38899fcf499 100644 --- a/components/homme/src/share/cxx/GllFvRemapImpl.hpp +++ b/components/homme/src/share/cxx/GllFvRemapImpl.hpp @@ -39,7 +39,6 @@ struct GllFvRemapImpl { enum : int { num_lev_aligned = num_lev_pack*packn }; enum : int { num_levp_aligned = max_num_lev_pack*packn }; enum : int { num_phys_lev = NUM_PHYSICAL_LEV }; - enum : int { num_work = 12 }; typedef GllFvRemap::Phys1T Phys1T; typedef GllFvRemap::Phys2T Phys2T; diff --git a/components/homme/src/share/cxx/SimulationParams.hpp b/components/homme/src/share/cxx/SimulationParams.hpp index 9247f54a352..4f36962b16c 100644 --- a/components/homme/src/share/cxx/SimulationParams.hpp +++ b/components/homme/src/share/cxx/SimulationParams.hpp @@ -24,7 +24,6 @@ struct SimulationParams TimeStepType time_step_type; bool use_moisture; - MoistDry moisture; //todo-repo-unification RemapAlg remap_alg; TestCase test_case; ForcingAlg ftype = ForcingAlg::FORCING_OFF; diff --git a/components/homme/src/share/cxx/Tracers.cpp b/components/homme/src/share/cxx/Tracers.cpp index cac9b3b15d7..7d338229d51 100644 --- a/components/homme/src/share/cxx/Tracers.cpp +++ b/components/homme/src/share/cxx/Tracers.cpp @@ -32,7 +32,10 @@ void Tracers::init(const int num_elems, const int num_tracers) nt = num_tracers; qdp = decltype(qdp)("tracers mass", num_elems); - qtens_biharmonic = decltype(qtens_biharmonic)("qtens(_biharmonic)", num_elems); + // Also used in ComposeTransportImplEnhancedTrajectory for communication, + // where 4 slots are needed. + qtens_biharmonic = decltype(qtens_biharmonic)( + "qtens(_biharmonic)", num_elems, std::max(4, num_tracers)); qlim = decltype(qlim)("qlim", num_elems); Q = decltype(Q)("tracers concentration", num_elems,num_tracers); diff --git a/components/homme/src/share/cxx/Tracers.hpp b/components/homme/src/share/cxx/Tracers.hpp index 78f93c06a70..592350894ab 100644 --- a/components/homme/src/share/cxx/Tracers.hpp +++ b/components/homme/src/share/cxx/Tracers.hpp @@ -35,8 +35,8 @@ struct Tracers { bool inited () const { return m_inited; } ExecViewManaged qdp; - ExecViewManaged qtens_biharmonic; // Also doubles as just qtens. - ExecViewManaged qlim; + ExecViewManaged qtens_biharmonic; // Also doubles as just qtens. + ExecViewManaged qlim; ExecViewManaged Q; ExecViewManaged fq; diff --git a/components/homme/src/share/cxx/prim_step.cpp b/components/homme/src/share/cxx/prim_step.cpp index 9e6afdec0d1..c5c75e54a35 100644 --- a/components/homme/src/share/cxx/prim_step.cpp +++ b/components/homme/src/share/cxx/prim_step.cpp @@ -183,7 +183,9 @@ void prim_step_flexible (const Real dt, const bool compute_diagnostics) { Errors::err_not_implemented); } else { // Remap dynamics variables but not tracers. + GPTLstart("tl-sc vertical_remap"); vertical_remap(dt_remap); + GPTLstop("tl-sc vertical_remap"); } } } diff --git a/components/homme/src/share/cxx/utilities/SubviewUtils.hpp b/components/homme/src/share/cxx/utilities/SubviewUtils.hpp index 76171a1432c..c78517a3b82 100644 --- a/components/homme/src/share/cxx/utilities/SubviewUtils.hpp +++ b/components/homme/src/share/cxx/utilities/SubviewUtils.hpp @@ -352,6 +352,21 @@ subview(ViewType +KOKKOS_INLINE_FUNCTION ViewUnmanaged +subview(ViewType v_in, + int ie, int idim0) { + assert(v_in.data() != nullptr); + assert(ie < v_in.extent_int(0)); + assert(ie >= 0); + assert(idim0 < v_in.extent_int(1)); + assert(idim0 >= 0); + return ViewUnmanaged( + &v_in.impl_map().reference(ie, idim0, 0, 0, 0, 0)); +} + // Force a subview to be const template KOKKOS_INLINE_FUNCTION diff --git a/components/homme/src/share/cxx/vector/vector_pragmas.hpp b/components/homme/src/share/cxx/vector/vector_pragmas.hpp index b23788a8ccf..fc58b819a29 100644 --- a/components/homme/src/share/cxx/vector/vector_pragmas.hpp +++ b/components/homme/src/share/cxx/vector/vector_pragmas.hpp @@ -7,11 +7,18 @@ #ifndef HOMMEXX_VECTOR_PRAGMAS_HPP #define HOMMEXX_VECTOR_PRAGMAS_HPP +#include "Config.hpp" + #if defined(__INTEL_COMPILER) || defined(__INTEL_CLANG_COMPILER) || defined(__INTEL_LLVM_COMPILER) #define VECTOR_IVDEP_LOOP _Pragma("ivdep") #define ALWAYS_VECTORIZE_LOOP _Pragma("vector always") #define VECTOR_SIMD_LOOP _Pragma("omp simd") +#if HOMMEXX_VECTOR_SIZE == 1 +# define VECTOR_SIMD_LOOP +#else +# define VECTOR_SIMD_LOOP _Pragma("omp simd") +#endif #elif defined(__GNUG__) && !defined(__NVCC__) #if(__GNUG__ == 4 && __GNUC_MINOR__ >= 9) || __GNUG__ > 4 diff --git a/components/homme/src/share/namelist_mod.F90 b/components/homme/src/share/namelist_mod.F90 index 1d47090182b..8f3eff70e3e 100644 --- a/components/homme/src/share/namelist_mod.F90 +++ b/components/homme/src/share/namelist_mod.F90 @@ -46,6 +46,10 @@ module namelist_mod semi_lagrange_cdr_check, & semi_lagrange_hv_q, & semi_lagrange_nearest_point_lev, & + semi_lagrange_halo, & + semi_lagrange_trajectory_nsubstep, & + semi_lagrange_trajectory_nvelocity, & + semi_lagrange_diagnostics, & tstep_type, & cubed_sphere_map, & qsplit, & @@ -272,6 +276,11 @@ subroutine readnl(par) semi_lagrange_cdr_check, & semi_lagrange_hv_q, & semi_lagrange_nearest_point_lev, & + semi_lagrange_halo, & + semi_lagrange_trajectory_nsubstep, & + semi_lagrange_trajectory_nvelocity, & + semi_lagrange_diagnostics, & + semi_lagrange_hv_q, & tstep_type, & cubed_sphere_map, & qsplit, & @@ -426,6 +435,13 @@ subroutine readnl(par) se_ftype = ftype ! MNL: For non-CAM runs, ftype=0 in control_mod nsplit = 1 pertlim = 0.0_real_kind +#else + se_partmethod = SFCURVE + se_ne = 0 + se_ne_x = 0 + se_ne_y = 0 + se_lx = 0 + se_ly = 0 #endif sub_case = 1 numnodes = -1 @@ -446,6 +462,10 @@ subroutine readnl(par) semi_lagrange_cdr_check = .false. semi_lagrange_hv_q = 1 semi_lagrange_nearest_point_lev = 256 + semi_lagrange_halo = 2 + semi_lagrange_trajectory_nsubstep = 0 + semi_lagrange_trajectory_nvelocity = -1 + semi_lagrange_diagnostics = 0 disable_diagnostics = .false. se_fv_phys_remap_alg = 1 internal_diagnostics_level = 0 @@ -856,6 +876,10 @@ subroutine readnl(par) call MPI_bcast(semi_lagrange_cdr_check ,1,MPIlogical_t,par%root,par%comm,ierr) call MPI_bcast(semi_lagrange_hv_q ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(semi_lagrange_nearest_point_lev ,1,MPIinteger_t,par%root,par%comm,ierr) + call MPI_bcast(semi_lagrange_halo ,1,MPIinteger_t,par%root,par%comm,ierr) + call MPI_bcast(semi_lagrange_trajectory_nsubstep ,1,MPIinteger_t,par%root,par%comm,ierr) + call MPI_bcast(semi_lagrange_trajectory_nvelocity ,1,MPIinteger_t,par%root,par%comm,ierr) + call MPI_bcast(semi_lagrange_diagnostics ,1,MPIinteger_t,par%root,par%comm,ierr) call MPI_bcast(tstep_type,1,MPIinteger_t ,par%root,par%comm,ierr) call MPI_bcast(cubed_sphere_map,1,MPIinteger_t ,par%root,par%comm,ierr) call MPI_bcast(qsplit,1,MPIinteger_t ,par%root,par%comm,ierr) @@ -1172,6 +1196,10 @@ subroutine readnl(par) write(iulog,*)"readnl: semi_lagrange_cdr_check = ",semi_lagrange_cdr_check write(iulog,*)"readnl: semi_lagrange_hv_q = ",semi_lagrange_hv_q write(iulog,*)"readnl: semi_lagrange_nearest_point_lev = ",semi_lagrange_nearest_point_lev + write(iulog,*)"readnl: semi_lagrange_halo = ",semi_lagrange_halo + write(iulog,*)"readnl: semi_lagrange_trajectory_nsubstep = ",semi_lagrange_trajectory_nsubstep + write(iulog,*)"readnl: semi_lagrange_trajectory_nvelocity = ",semi_lagrange_trajectory_nvelocity + write(iulog,*)"readnl: semi_lagrange_diagnostics = ",semi_lagrange_diagnostics write(iulog,*)"readnl: tstep_type = ",tstep_type write(iulog,*)"readnl: theta_advect_form = ",theta_advect_form write(iulog,*)"readnl: vtheta_thresh = ",vtheta_thresh diff --git a/components/homme/src/share/prim_driver_base.F90 b/components/homme/src/share/prim_driver_base.F90 index df83e1cc386..9e036347320 100644 --- a/components/homme/src/share/prim_driver_base.F90 +++ b/components/homme/src/share/prim_driver_base.F90 @@ -46,6 +46,8 @@ module prim_driver_base public :: applyCAMforcing_tracers + public :: set_tracer_transport_derived_values + ! Service variables used to partition the mesh. ! Note: GridEdge and MeshVertex are public, cause kokkos targets need to access them type (GridVertex_t), pointer :: GridVertex(:) @@ -1316,7 +1318,7 @@ subroutine prim_step_flexible(hybrid, elem, nets, nete, dt, tl, hvcoord, compute use hybvcoord_mod, only: hvcoord_t use parallel_mod, only: abortmp use prim_advance_mod, only: prim_advance_exp, applycamforcing_dynamics - use prim_advection_mod, only: prim_advec_tracers_remap + use prim_advection_mod, only: prim_advec_tracers_observe_velocity, prim_advec_tracers_remap use reduction_mod, only: parallelmax use time_mod, only: TimeLevel_t, timelevel_update, timelevel_qdp use prim_state_mod, only: prim_printstate @@ -1334,7 +1336,7 @@ subroutine prim_step_flexible(hybrid, elem, nets, nete, dt, tl, hvcoord, compute real(kind=real_kind) :: dt_q, dt_remap, dp(np,np,nlev) integer :: ie, q, k, n, n0_qdp, np1_qdp - logical :: compute_diagnostics_it, apply_forcing + logical :: compute_diagnostics_it, apply_forcing, observe dt_q = dt*dt_tracer_factor if (dt_remap_factor == 0) then @@ -1392,11 +1394,13 @@ subroutine prim_step_flexible(hybrid, elem, nets, nete, dt, tl, hvcoord, compute call prim_advance_exp(elem, deriv1, hvcoord, hybrid, dt, tl, nets, nete, & compute_diagnostics_it) + observe = .false. if (dt_remap_factor == 0) then ! Set np1_qdp to -1. Since dt_remap == 0, the only part of ! vertical_remap that is active is the updates to ! ps_v(:,:,np1) and dp3d(:,:,:,np1). call vertical_remap(hybrid, elem, hvcoord, dt_remap, tl%np1, -1, nets, nete) + observe = .true. else if (modulo(n, dt_remap_factor) == 0) then if (compute_diagnostics) call run_diagnostics(elem,hvcoord,tl,4,.false.,nets,nete) @@ -1417,8 +1421,10 @@ subroutine prim_step_flexible(hybrid, elem, nets, nete, dt, tl, hvcoord, compute ! not tracers. call vertical_remap(hybrid, elem, hvcoord, dt_remap, tl%np1, -1, nets, nete) end if + observe = .true. end if end if + if (observe) call Prim_Advec_Tracers_observe_velocity(elem, tl, n, nets, nete) ! defer final timelevel update until after Q update. enddo call t_stopf("prim_step_dyn") diff --git a/components/homme/src/share/reduction_mod.F90 b/components/homme/src/share/reduction_mod.F90 index 83232478ef3..72e0e89231f 100644 --- a/components/homme/src/share/reduction_mod.F90 +++ b/components/homme/src/share/reduction_mod.F90 @@ -25,7 +25,7 @@ module reduction_mod integer :: ctr end type ReductionBuffer_ordered_1d_t - public :: ParallelMin,ParallelMax + public :: ParallelMin,ParallelMax,ParallelSum !type (ReductionBuffer_ordered_1d_t), public :: red_sum type (ReductionBuffer_int_1d_t), public :: red_max_int @@ -51,6 +51,9 @@ module reduction_mod module procedure ParallelMax0d module procedure ParallelMax0d_int end interface + interface ParallelSum + module procedure ParallelSum0d_int + end interface interface pmax_mt module procedure pmax_mt_int_1d @@ -61,6 +64,10 @@ module reduction_mod module procedure pmin_mt_r_1d end interface + interface psum_mt + module procedure psum_mt_int_1d + end interface + interface InitReductionBuffer module procedure InitReductionBuffer_int_1d module procedure InitReductionBuffer_r_1d @@ -179,7 +186,19 @@ function ParallelMax0d_int(data,hybrid) result(pmax) end function ParallelMax0d_int + !**************************************************************** + function ParallelSum0d_int(data,hybrid) result(psum) + use hybrid_mod, only : hybrid_t + implicit none + integer , intent(in) :: data + type (hybrid_t), intent(in) :: hybrid + integer :: psum + integer :: tmp(1) + tmp(1)=data + call psum_mt(red_sum_int,tmp,1,hybrid) + psum = red_sum_int%buf(1) + end function ParallelSum0d_int !**************************************************************** subroutine InitReductionBuffer_int_1d(red,len) @@ -486,7 +505,53 @@ subroutine pmin_mt_r_1d(red,redp,len,hybrid) !$OMP BARRIER end subroutine pmin_mt_r_1d + ! ======================================= + ! psum_mt: + ! + ! thread safe, parallel reduce sum of a + ! one dimensional INTEGER reduction vector + ! ======================================= + subroutine psum_mt_int_1d(red,redp,len,hybrid) + use hybrid_mod, only : hybrid_t +#ifdef _MPI + use parallel_mod, only: mpi_sum, mpiinteger_t +#endif + use parallel_mod, only: abortmp + + type (ReductionBuffer_int_1d_t) :: red ! shared memory reduction buffer struct + integer, intent(in) :: len ! buffer length + integer, intent(inout) :: redp(len) ! thread private vector of partial sum + type (hybrid_t), intent(in) :: hybrid ! parallel handle + + ! Local variables + integer ierr, k + + if (len>red%len) call abortmp('ERROR: threadsafe reduction buffer too small') + + !$OMP BARRIER + ! the first and fastest thread performs initializing copy + !$OMP SINGLE + red%buf(1:len) = redp(1:len) + red%ctr = hybrid%ithr + !$OMP END SINGLE + !$OMP CRITICAL (CRITMAXINT) + if (hybrid%ithr /= red%ctr) then + do k=1,len + red%buf(k) = red%buf(k) + redp(k) + enddo + end if + !$OMP END CRITICAL (CRITMAXINT) +#ifdef _MPI + !$OMP BARRIER + if (hybrid%ithr==0) then + call MPI_Allreduce(red%buf(1),redp,len,MPIinteger_t, & + MPI_SUM,hybrid%par%comm,ierr) + red%buf(1:len)=redp(1:len) + end if +#endif + !$OMP BARRIER + end subroutine psum_mt_int_1d ! ======================================= subroutine ElementSum_1d(res,variable,type,hybrid) diff --git a/components/homme/src/share/sl_advection.F90 b/components/homme/src/share/sl_advection.F90 index 3f6544953b6..63a476d0325 100644 --- a/components/homme/src/share/sl_advection.F90 +++ b/components/homme/src/share/sl_advection.F90 @@ -5,12 +5,12 @@ module sl_advection use kinds, only : real_kind, int_kind use dimensions_mod, only : nlev, nlevp, np, qsize, qsize_d - use derivative_mod, only : derivative_t, gradient_sphere, divergence_sphere + use derivative_mod, only : derivative_t, gradient_sphere, divergence_sphere, ugradv_sphere use element_mod, only : element_t use hybvcoord_mod, only : hvcoord_t use time_mod, only : TimeLevel_t, TimeLevel_Qdp - use control_mod, only : integration, test_case, hypervis_order, transport_alg, limiter_option,& - vert_remap_q_alg + use control_mod, only : integration, test_case, hypervis_order, transport_alg, & + & limiter_option, vert_remap_q_alg, semi_lagrange_diagnostics use edge_mod, only : edgevpack_nlyr, edgevunpack_nlyr, edge_g use edgetype_mod, only : EdgeDescriptor_t, EdgeBuffer_t use hybrid_mod, only : hybrid_t @@ -26,18 +26,20 @@ module sl_advection private + ! Constants real(real_kind), parameter :: zero = 0.0_real_kind, fourth = 0.25_real_kind, & half = 0.5_real_kind, one = 1.0_real_kind, two = 2.0_real_kind, & eps = epsilon(1.0_real_kind) - type (cartesian3D_t), allocatable :: dep_points_all(:,:,:,:) ! (np,np,nlev,nelemd) - real(kind=real_kind), dimension(:,:,:,:,:), allocatable :: minq, maxq ! (np,np,nlev,qsize,nelemd) - logical :: is_sphere + ! Configuration. + logical :: is_sphere, enhanced_trajectory + integer :: dep_points_ndim - ! For use in make_positive. - real(kind=real_kind) :: dp_tol + ! For use in make_positive. Set at initialization to a function of hvcoord%dp0. + real(kind=real_kind) :: dp_tol, deta_tol - public :: prim_advec_tracers_remap_ALE, sl_init1, sl_vertically_remap_tracers, sl_unittest + public :: prim_advec_tracers_observe_velocity_ALE, prim_advec_tracers_remap_ALE, & + & sl_init1, sl_vertically_remap_tracers, sl_unittest ! For testing public :: calc_trajectory, dep_points_all, sphere2cart @@ -45,8 +47,36 @@ module sl_advection ! For C++ public :: sl_get_params + ! Barrier for performance analysis. Should be false in production runs. logical, parameter :: barrier = .false. + ! Bounds for shape preservation. + real(kind=real_kind), dimension(:,:,:,:,:), allocatable :: minq, maxq ! (np,np,nlev,qsize,nelemd) + + ! Trajectory velocity data. + real(kind=real_kind), dimension(:,:,:,:,:), allocatable :: vnode, vdep ! (ndim,np,np,nlev,nelemd) + real(kind=real_kind), allocatable :: dep_points_all(:,:,:,:,:) ! (ndim,np,np,nlev,nelemd) + + type :: velocity_record_t + integer :: nvel + ! Times to which velocity slots correspond, in reference time [0,dtf]. + real(kind=real_kind), allocatable :: t_vel(:) ! 1:nvel + ! For n = 1:dtf, obs_slots(n,:) = [slot1, slot2], -1 if unused. These are + ! the slots to which velocity sample n contributes. obs_slots(dtf,:) is + ! always -1. + integer, allocatable :: obs_slots(:,:) + ! obs_wts(n,:) = [wt1, wt2], 0 if unused. + real(kind=real_kind), allocatable :: obs_wts(:,:) + ! Substep end point n in 0:nsub uses velocity slots run_step(n), + ! run_step(n)-1. + integer, allocatable :: run_step(:) + ! Store state%v and state%dp3d at t_vel points. + real(kind=real_kind), allocatable :: vel(:,:,:,:,:,:) ! (np,np,2,nlev,nss,nelemd) + real(kind=real_kind), allocatable :: dp (:,:, :,:,:) ! (np,np, nlev,nss,nelemd) + end type velocity_record_t + + type(velocity_record_t) :: vrec + contains !=================================================================================================! @@ -87,7 +117,9 @@ end subroutine sphere2cart subroutine sl_init1(par, elem) use interpolate_mod, only : interpolate_tracers_init use control_mod, only : transport_alg, semi_lagrange_cdr_alg, cubed_sphere_map, & - nu_q, semi_lagrange_hv_q, semi_lagrange_cdr_check, geometry + nu_q, semi_lagrange_hv_q, semi_lagrange_cdr_check, semi_lagrange_trajectory_nsubstep, & + semi_lagrange_trajectory_nvelocity, geometry, dt_remap_factor, dt_tracer_factor, & + semi_lagrange_halo use element_state, only : timelevels use coordinate_systems_mod, only : cartesian3D_t use perf_mod, only: t_startf, t_stopf @@ -102,14 +134,13 @@ subroutine sl_init1(par, elem) #ifdef HOMME_ENABLE_COMPOSE call t_startf('sl_init1') if (transport_alg > 0) then - call sl_parse_transport_alg(transport_alg, slmm, cisl, qos, sl_test, independent_time_steps) - if (par%masterproc .and. nu_q > 0 .and. semi_lagrange_hv_q > 0) & - write(iulog,*) 'COMPOSE> use HV; nu_q, all:', nu_q, semi_lagrange_hv_q + call sl_parse_transport_alg(transport_alg, slmm, cisl, qos, sl_test, & + independent_time_steps) is_sphere = trim(geometry) /= 'plane' + enhanced_trajectory = semi_lagrange_trajectory_nsubstep > 0 + dep_points_ndim = 3 + if (enhanced_trajectory .and. independent_time_steps) dep_points_ndim = 4 nslots = nlev*qsize - ! Technically a memory leak, but the array persists for the entire - ! run, so not a big deal for now. - allocate(dep_points_all(np,np,nlev,size(elem))) do ie = 1, size(elem) ! Provide a point inside the target element. call sphere2cart(elem(ie)%spherep(2,2), pinside) @@ -130,21 +161,45 @@ subroutine sl_init1(par, elem) need_conservation = 1 call cedr_sl_init(np, nlev, qsize, qsize_d, timelevels, need_conservation) end if - allocate(minq(np,np,nlev,qsize,size(elem)), maxq(np,np,nlev,qsize,size(elem))) + allocate(minq(np,np,nlev,qsize,size(elem)), maxq(np,np,nlev,qsize,size(elem)), & + & dep_points_all(dep_points_ndim,np,np,nlev,size(elem))) + if (enhanced_trajectory) then + allocate(vnode(dep_points_ndim,np,np,nlev,size(elem)), & + & vdep (dep_points_ndim,np,np,nlev,size(elem))) + end if + call init_velocity_record(size(elem), dt_tracer_factor, dt_remap_factor, & + semi_lagrange_trajectory_nsubstep, semi_lagrange_trajectory_nvelocity, & + vrec, i) dp_tol = -one + deta_tol = -one + if (par%masterproc) then + if (nu_q > 0 .and. semi_lagrange_hv_q > 0) then + write(iulog,'(a,es13.4,i3)') 'COMPOSE> use HV; nu_q, hv_q:', & + nu_q, semi_lagrange_hv_q + end if + if (enhanced_trajectory) then + write(iulog,'(a,i3,i3,i3)') & + 'COMPOSE> dt_tracer_factor, dt_remap_factor, halo:', & + dt_tracer_factor, dt_remap_factor, semi_lagrange_halo + write(iulog,'(a,i3,i3)') & + 'COMPOSE> use enhanced trajectory; nsub, nvel:', & + semi_lagrange_trajectory_nsubstep, vrec%nvel + end if + end if endif call t_stopf('sl_init1') #endif end subroutine sl_init1 subroutine sl_get_params(nu_q_out, hv_scaling, hv_q, hv_subcycle_q, limiter_option_out, & - cdr_check, geometry_type) bind(c) + cdr_check, geometry_type, trajectory_nsubstep) bind(c) use control_mod, only: semi_lagrange_hv_q, hypervis_subcycle_q, semi_lagrange_cdr_check, & - nu_q, hypervis_scaling, limiter_option, geometry + nu_q, hypervis_scaling, limiter_option, geometry, semi_lagrange_trajectory_nsubstep use iso_c_binding, only: c_int, c_double real(c_double), intent(out) :: nu_q_out, hv_scaling - integer(c_int), intent(out) :: hv_q, hv_subcycle_q, limiter_option_out, cdr_check, geometry_type + integer(c_int), intent(out) :: hv_q, hv_subcycle_q, limiter_option_out, cdr_check, & + geometry_type, trajectory_nsubstep nu_q_out = nu_q hv_scaling = hypervis_scaling @@ -155,21 +210,143 @@ subroutine sl_get_params(nu_q_out, hv_scaling, hv_q, hv_subcycle_q, limiter_opti if (semi_lagrange_cdr_check) cdr_check = 1 geometry_type = 0 ! sphere if (trim(geometry) == "plane") geometry_type = 1 - + trajectory_nsubstep = semi_lagrange_trajectory_nsubstep end subroutine sl_get_params + subroutine init_velocity_record(nelemd, dtf, drf_param, nsub, nvel_param, v, error) + integer, intent(in) :: nelemd, dtf, drf_param, nsub, nvel_param + type (velocity_record_t), intent(out) :: v + integer, intent(out) :: error + + real(kind=real_kind) :: t_avail(0:dtf), time + integer :: nvel, drf, navail, n, i, iav + + error = 0 + drf = drf_param + if (drf == 0) drf = 1 ! drf = 0 if vertically Eulerian + nvel = nvel_param + if (nvel == -1) nvel = 2 + ((nsub-1) / 2) + nvel = min(nvel, nsub+1) + navail = dtf/drf + 1 + nvel = min(nvel, navail) + + ! nsub <= 1: No substepping. + ! nvel <= 2: Save velocity only at endpoints, as always occurs. + if (nsub <= 1 .or. nvel <= 2) then + v%nvel = 2 + return + end if + + v%nvel = nvel + allocate(v%t_vel(nvel), v%obs_slots(dtf,2), v%obs_wts(dtf,2), v%run_step(0:nsub), & + & v%vel(np,np,2,nlev,2:nvel-1,nelemd), v%dp(np,np,nlev,2:nvel-1,nelemd)) + + ! Times at which velocity data are available. + t_avail(0) = 0 + i = 1 + do n = 1, dtf + if (modulo(n, drf) == 0) then + t_avail(i) = n + i = i + 1 + end if + end do + if (i /= navail) error = 1 + + ! Times to which we associate velocity data. + do n = 1, nvel + if (modulo((n-1)*dtf, nvel-1) == 0) then + ! Cast integer values at end of calculation. + v%t_vel(n) = ((n-1)*dtf)/(nvel-1) + else + v%t_vel(n) = real((n-1)*dtf, real_kind)/(nvel-1) + end if + end do + + ! Build the tables mapping n in 1:dtf to velocity slots to accumulate into. + do n = 1, dtf-1 + v%obs_slots(n,:) = -1 + v%obs_wts(n,:) = 0 + if (modulo(n, drf) /= 0) cycle + time = n + do i = 1, navail-1 + if (time == t_avail(i)) exit + end do + iav = i + if (iav > navail-1) error = 2 + do i = 2, nvel-1 + if (t_avail(iav-1) < v%t_vel(i) .and. time > v%t_vel(i)) then + v%obs_slots(n,1) = i + v%obs_wts(n,1) = (v%t_vel(i) - t_avail(iav-1))/(t_avail(iav) - t_avail(iav-1)) + end if + if (time <= v%t_vel(i) .and. t_avail(iav+1) > v%t_vel(i)) then + v%obs_slots(n,2) = i + v%obs_wts(n,2) = (t_avail(iav+1) - v%t_vel(i))/(t_avail(iav+1) - t_avail(iav)) + end if + end do + end do + v%obs_slots(dtf,:) = -1 + v%obs_wts(dtf,:) = 0 + + ! Build table mapping n to interval to use. The trajectories go backward in + ! time, and this table reflects that. + v%run_step(0) = nvel + v%run_step(nsub) = 2 + do n = 1, nsub-1 + time = real((nsub-n)*dtf, real_kind)/nsub + do i = 1, nvel-1 + if (v%t_vel(i) <= time .and. time <= v%t_vel(i+1)) exit + end do + if (i > nvel-1) error = 4 + v%run_step(n) = i+1 + end do + end subroutine init_velocity_record + + subroutine prim_advec_tracers_observe_velocity_ALE(elem, tl, n, nets, nete) + use control_mod, only: dt_remap_factor + + type (element_t) , intent(inout) :: elem(:) + type (TimeLevel_t) , intent(in ) :: tl + integer , intent(in ) :: n ! step in 1:dt_tracer_factor + integer , intent(in ) :: nets + integer , intent(in ) :: nete + + integer :: nstore, islot, slot, k, ie + + if (vrec%nvel == 2) return + + if (n == dt_remap_factor .or. (dt_remap_factor == 0 .and. n == 1)) then + ! First observation of the tracer time step: zero accumulated quantities. + do ie = nets, nete + do slot = 2, vrec%nvel-1 + vrec%vel(:,:,:,:,slot,ie) = 0 + vrec%dp (:,:, :,slot,ie) = 0 + end do + end do + end if + + do islot = 1, 2 + slot = vrec%obs_slots(n,islot) + if (slot == -1) cycle + do ie = nets, nete + do k = 1, nlev + vrec%vel(:,:,:,k,slot,ie) = vrec%vel(:,:,:,k,slot,ie) + & + vrec%obs_wts(n,islot) * elem(ie)%state%v(:,:,:,k,tl%np1) + vrec%dp (:,:, k,slot,ie) = vrec%dp (:,:, k,slot,ie) + & + vrec%obs_wts(n,islot) * elem(ie)%state%dp3d(:,:,k,tl%np1) + end do + end do + end do + end subroutine prim_advec_tracers_observe_velocity_ALE + subroutine prim_advec_tracers_remap_ALE(elem, deriv, hvcoord, hybrid, dt, tl, nets, nete) use coordinate_systems_mod, only : cartesian3D_t, cartesian2D_t use dimensions_mod, only : max_neigh_edges use interpolate_mod, only : interpolate_tracers, minmax_tracers use control_mod, only : dt_tracer_factor, nu_q, transport_alg, semi_lagrange_hv_q, & - semi_lagrange_cdr_alg, semi_lagrange_cdr_check + semi_lagrange_cdr_alg, semi_lagrange_cdr_check, semi_lagrange_trajectory_nsubstep ! For DCMIP16 supercell test case. use control_mod, only : dcmip16_mu_q use prim_advection_base, only : advance_physical_vis -#ifdef HOMME_ENABLE_COMPOSE - use compose_mod, only : compose_h2d, compose_d2h -#endif use iso_c_binding, only : c_bool implicit none @@ -182,8 +359,6 @@ subroutine prim_advec_tracers_remap_ALE(elem, deriv, hvcoord, hybrid, dt, tl, ne integer , intent(in ) :: nets integer , intent(in ) :: nete - type(cartesian3D_t) :: dep_points (np,np) - integer :: i,j,k,l,n,q,ie,n0_qdp,np1_qdp integer :: scalar_q_bounds, info logical :: slmm, cisl, qos, sl_test, independent_time_steps @@ -194,19 +369,18 @@ subroutine prim_advec_tracers_remap_ALE(elem, deriv, hvcoord, hybrid, dt, tl, ne call t_startf('Prim_Advec_Tracers_remap_ALE') call sl_parse_transport_alg(transport_alg, slmm, cisl, qos, sl_test, independent_time_steps) - ! Until I get the DSS onto GPU, always need to h<->d. - !h2d = hybrid%par%nprocs > 1 .or. semi_lagrange_cdr_check .or. & (semi_lagrange_hv_q > 0 .and. nu_q > 0) h2d = .true. -#ifdef HOMME_ENABLE_COMPOSE d2h = compose_d2h .or. h2d h2d = compose_h2d .or. h2d -#else - d2h = h2d -#endif call TimeLevel_Qdp(tl, dt_tracer_factor, n0_qdp, np1_qdp) - call calc_trajectory(elem, deriv, hvcoord, hybrid, dt, tl, & - independent_time_steps, nets, nete) + if (enhanced_trajectory) then + call calc_enhanced_trajectory(elem, deriv, hvcoord, hybrid, dt, tl, nets, nete, & + semi_lagrange_trajectory_nsubstep, independent_time_steps) + else + call calc_trajectory(elem, deriv, hvcoord, hybrid, dt, tl, & + independent_time_steps, nets, nete) + end if call t_startf('SLMM_csl') !todo Here and in the set-pointer loop for CEDR, do just in the first call. @@ -218,14 +392,14 @@ subroutine prim_advec_tracers_remap_ALE(elem, deriv, hvcoord, hybrid, dt, tl, ne h2d, d2h) end do ! edge_g buffers are shared by SLMM, CEDR, other places in HOMME, and - ! dp_coupling in EAM. Thus, we must take care to protected threaded - ! access. In the following, "No barrier needed" comments justify why a - ! barrier isn't needed. + ! dp_coupling in EAM. Thus, we must take care to protect threaded access. In + ! the following, "No barrier needed" comments justify why a barrier isn't + ! needed. ! No barrier needed: ale_rkdss has a horiz thread barrier at the end. - call slmm_csl(nets, nete, dep_points_all, minq, maxq, info) + call slmm_csl(nets, nete, dep_points_all, dep_points_ndim, minq, maxq, info) ! No barrier needed: slmm_csl has a horiz thread barrier at the end. if (info /= 0) then - call write_velocity_data(elem, nets, nete, hybrid, deriv, dt, tl) + call write_velocity_data(elem, nets, nete, hybrid, dt, tl) call abortmp('slmm_csl returned -1; see output above for more information.') end if if (barrier) call perf_barrier(hybrid) @@ -261,7 +435,7 @@ subroutine prim_advec_tracers_remap_ALE(elem, deriv, hvcoord, hybrid, dt, tl, ne call t_stopf('CEDR') call t_startf('CEDR_local') call cedr_sl_run_local(minq, maxq, nets, nete, scalar_q_bounds, limiter_option) - ! Barrier needed to protect edge_g buffers use in CEDR. + ! Barrier needed to protect edge_g buffers used in CEDR. #if (defined HORIZ_OPENMP) !$omp barrier #endif @@ -350,7 +524,7 @@ subroutine calc_trajectory(elem, deriv, hvcoord, hybrid, dt, tl, & !$omp parallel do private(k) #endif do k = 1, nlev - call ALE_departure_from_gll(dep_points_all(:,:,k,ie), & + call ALE_departure_from_gll(dep_points_all(:,:,:,k,ie), dep_points_ndim, & elem(ie)%derived%vstar(:,:,:,k), elem(ie), dt, normalize=is_sphere) end do end do @@ -361,7 +535,7 @@ end subroutine calc_trajectory !SUBROUTINE ALE_RKDSS ! AUTHOR: CHRISTOPH ERATH, MARK TAYLOR, 06. December 2012 ! - ! DESCRIPTION: ! create a runge kutta taylor serios mixture to calculate the departure grid + ! DESCRIPTION: ! create a runge kutta taylor series mixture to calculate the departure grid ! ! CALLS: ! INPUT: @@ -372,7 +546,6 @@ end subroutine calc_trajectory ! this will calculate the velocity at time t+1/2 along the trajectory s(t) given the velocities ! at the GLL points at time t and t+1 using a second order time accurate formulation. subroutine ALE_RKdss(elem, nets, nete, hy, deriv, dt, tl, independent_time_steps) - use derivative_mod, only : derivative_t, ugradv_sphere use edgetype_mod, only : EdgeBuffer_t use bndry_mod, only : bndry_exchangev use kinds, only : real_kind @@ -397,22 +570,20 @@ subroutine ALE_RKdss(elem, nets, nete, hy, deriv, dt, tl, independent_time_steps ! RK-SSP 2 stage 2nd order: ! x*(t+1) = x(t) + U(x(t),t) dt - ! x(t+1) = x(t) + 1/2 ( U(x*(t+1),t+1) + U(x(t),t) ) dt + ! x(t+1) = x(t) + 1/2 ( U(x*(t+1),t+1) + U(x(t),t) ) dt ! apply taylor series: - ! U(x*(t+1),t+1) = U(x(t),t+1) + (x*(t+1)-x(t)) gradU(x(t),t+1) - ! - ! x(t+1) = x(t) + 1/2 ( U(x(t),t+1) + (x*(t+1)-x(t)) gradU(x(t),t+1) + U(x(t),t) ) dt - ! (x(t+1) - x(t)) / dt = 1/2 ( U(x(t),t+1) + (x*(t+1)-x(t)) gradU(x(t),t+1) + U(x(t),t) ) - ! (x(t+1) - x(t)) / dt = 1/2 ( U(x(t),t+1) + U(x(t),t) + (x*(t+1)-x(t)) gradU(x(t),t+1) ) - ! (x(t+1) - x(t)) / dt = 1/2 ( U(x(t),t+1) + U(x(t),t) + U(x(t),t) dt gradU(x(t),t+1) ) + ! U(x*(t+1),t+1) = U(x(t),t+1) + (x*(t+1)-x(t)) gradU(x(t),t+1) ! + ! x(t+1) = x(t) + 1/2 ( U(x(t),t+1) + (x*(t+1)-x(t)) gradU(x(t),t+1) + U(x(t),t) ) dt + ! (x(t+1) - x(t)) / dt = 1/2 ( U(x(t),t+1) + (x*(t+1)-x(t)) gradU(x(t),t+1) + U(x(t),t) ) + ! = 1/2 ( U(x(t),t+1) + U(x(t),t) + (x*(t+1)-x(t)) gradU(x(t),t+1) ) + ! = 1/2 ( U(x(t),t+1) + U(x(t),t) + U(x(t),t) dt gradU(x(t),t+1) ) ! - ! (x(t+1)-x(t))/dt = 1/2(U(x(t),t+1) + U(x(t),t) + dt U(x(t),t) gradU(x(t),t+1)) + ! => (x(t+1)-x(t))/dt = 1/2 (U(x(t),t+1) + U(x(t),t) + dt U(x(t),t) gradU(x(t),t+1)) ! ! suppose dt = -ts (we go backward) - ! (x(t-ts)-x(t))/-ts = 1/2( U(x(t),t-ts)+U(x(t),t)) - ts 1/2 U(x(t),t) gradU(x(t),t-ts) - ! - ! x(t-ts) = x(t)) -ts * [ 1/2( U(x(t),t-ts)+U(x(t),t)) - ts 1/2 U(x(t),t) gradU(x(t),t-ts) ] + ! (x(t-ts)-x(t))/-ts = 1/2 (U(x(t),t-ts)+U(x(t),t)) - ts 1/2 U(x(t),t) gradU(x(t),t-ts) + ! x(t-ts) = x(t)) - ts * [1/2 (U(x(t),t-ts)+U(x(t),t)) - ts 1/2 U(x(t),t) gradU(x(t),t-ts)] nlyr = 2*nlev if (independent_time_steps) nlyr = nlyr + nlev @@ -451,7 +622,7 @@ subroutine ALE_RKdss(elem, nets, nete, hy, deriv, dt, tl, independent_time_steps do ie = nets,nete call edgeVunpack_nlyr(edge_g,elem(ie)%desc,elem(ie)%derived%vstar,2*nlev,0,nlyr) if (independent_time_steps) then - call edgeVunpack_nlyr(edge_g,elem(ie)%desc,elem(ie)%derived%divdp,nlevp,2*nlev,nlyr) + call edgeVunpack_nlyr(edge_g,elem(ie)%desc,elem(ie)%derived%divdp,nlev,2*nlev,nlyr) end if end do @@ -460,8 +631,7 @@ subroutine ALE_RKdss(elem, nets, nete, hy, deriv, dt, tl, independent_time_steps #endif end subroutine ALE_RKdss - subroutine write_velocity_data(elem, nets, nete, hy, deriv, dt, tl) - use derivative_mod, only : derivative_t, ugradv_sphere + subroutine write_velocity_data(elem, nets, nete, hy, dt, tl) use edgetype_mod, only : EdgeBuffer_t use bndry_mod, only : bndry_exchangev use kinds, only : real_kind @@ -474,7 +644,6 @@ subroutine write_velocity_data(elem, nets, nete, hy, deriv, dt, tl) integer , intent(in) :: nets integer , intent(in) :: nete type (hybrid_t) , intent(in) :: hy - type (derivative_t) , intent(in) :: deriv real (kind=real_kind), intent(in) :: dt type (TimeLevel_t) , intent(in) :: tl @@ -507,7 +676,7 @@ end subroutine write_velocity_data ! ! OUTPUT: !-----------------------------------------------------------------------------------! - subroutine ALE_departure_from_gll(acart, vstar, elem, dt, normalize) + subroutine ALE_departure_from_gll(acart, ndim, vstar, elem, dt, normalize) use physical_constants, only : scale_factor use coordinate_systems_mod, only : spherical_polar_t, cartesian3D_t use time_mod, only : timelevel_t @@ -517,14 +686,15 @@ subroutine ALE_departure_from_gll(acart, vstar, elem, dt, normalize) implicit none - type (cartesian3D_t) ,intent(out) :: acart(np,np) + integer ,intent(in) :: ndim + real (kind=real_kind) ,intent(out) :: acart(ndim,np,np) real (kind=real_kind) ,intent(in) :: vstar(np,np,2) type (element_t) ,intent(in) :: elem real (kind=real_kind) ,intent(in) :: dt - logical, intent(in) :: normalize - - integer :: i,j + logical ,intent(in) :: normalize + integer :: i,j, d + type (cartesian3D_t) :: c3d real (kind=real_kind) :: uxyz (np,np,3), norm ! convert velocity from lat/lon to cartesian 3D @@ -539,16 +709,14 @@ subroutine ALE_departure_from_gll(acart, vstar, elem, dt, normalize) ! crude, 1st order accurate approximation. to be improved do i=1,np do j=1,np - call sphere2cart(elem%spherep(i,j), acart(i,j)) - acart(i,j)%x = acart(i,j)%x - dt*uxyz(i,j,1)/scale_factor - acart(i,j)%y = acart(i,j)%y - dt*uxyz(i,j,2)/scale_factor - acart(i,j)%z = acart(i,j)%z - dt*uxyz(i,j,3)/scale_factor + call sphere2cart(elem%spherep(i,j), c3d) + acart(1,i,j) = c3d%x - dt*uxyz(i,j,1)/scale_factor + acart(2,i,j) = c3d%y - dt*uxyz(i,j,2)/scale_factor + acart(3,i,j) = c3d%z - dt*uxyz(i,j,3)/scale_factor if (normalize) then - norm = sqrt(acart(i,j)%x*acart(i,j)%x + acart(i,j)%y*acart(i,j)%y + & - acart(i,j)%z*acart(i,j)%z) - acart(i,j)%x = acart(i,j)%x / norm - acart(i,j)%y = acart(i,j)%y / norm - acart(i,j)%z = acart(i,j)%z / norm + norm = sqrt(acart(1,i,j)*acart(1,i,j) + acart(2,i,j)*acart(2,i,j) + & + acart(3,i,j)*acart(3,i,j)) + acart(1:3,i,j) = acart(1:3,i,j)/norm end if enddo enddo @@ -801,7 +969,8 @@ subroutine calc_vertically_lagrangian_levels( & ! x1 - x0 = dt u(p0,t0) + O(dt^2) ! z1 - z0 = dt w(p0,t0) + O(dt^2) ! z1 = z0 + dt/2 (w(p0,t0) + w(p0,t1) + - ! dt (w_x(p0,t1) u(p0,t0) + w_z(p0,t1) w(p0,t0))) + O(dt^3) (*) + ! dt (w_x(p0,t1) u(p0,t0) + w_z(p0,t1) w(p0,t0))) + ! + O(dt^3). (*) ! Now we compute z(x0,t1). First, we need ! x0 - x1 = -dt u(p0,t0) + O(dt^2) ! and @@ -826,7 +995,7 @@ subroutine calc_vertically_lagrangian_levels( & ! - dt^2 w_x(p0,t1) u(p0,t0) + O(dt^3) ! = z0 + dt/2 (w(p0,t0) + w(p0,t1) + ! dt (-w_x(p0,t1) u(p0,t0) + w_z(p0,t1) w(p0,t0))) - ! + O(dt^3) + ! + O(dt^3). ! This is locally accurate to O(dt^3) and so globally 2nd-order ! accurate. Notably, compared with (*), this formula differs only in a ! sign. Note also that a straightforward first-order accurate formula is @@ -884,8 +1053,7 @@ subroutine calc_vertically_lagrangian_levels( & end do ! Use p0 as the reference coordinate system. p0 differs from p1 by B(eta) - ! (ps1 - ps0); dp3d already accounts for this term - ! w.r.t. derived%dp. Recall + ! (ps1 - ps0); dp3d already accounts for this term w.r.t. derived%dp. Recall ! eta_dot_dpdn = p_eta eta_dot = (A_eta p0 + B_eta ps) deta/dt, ! except that in the code eta_dot_dpdn is actually dp deta/dt rather than ! dp/deta deta/dt. eta_dot_dpdn is the motion of a pressure level excluding @@ -1158,18 +1326,1045 @@ function test_reconstruct_and_limit_dp() result(nerr) end do end function test_reconstruct_and_limit_dp - subroutine sl_unittest(par) + subroutine calc_enhanced_trajectory(elem, deriv, hvcoord, hybrid, dt, tl, & + nets, nete, nsubstep, independent_time_steps) + ! Top-level routine for new enhanced trajectory method. This new method + ! permits multiple substeps, optionally using more reference-grid velocity + ! snapshots. + + use reduction_mod, only: ParallelSum use kinds, only: iulog + + type (element_t), intent(inout) :: elem(:) + type (derivative_t), intent(in) :: deriv + type (hvcoord_t), intent(in) :: hvcoord + type (hybrid_t), intent(in) :: hybrid + real(real_kind), intent(in) :: dt + type (TimeLevel_t), intent(in) :: tl + integer, intent(in) :: nets, nete, nsubstep + logical, intent(in) :: independent_time_steps - type (parallel_t), intent(in) :: par +#ifdef HOMME_ENABLE_COMPOSE + integer :: step, ie, info, limiter_active_count + real(real_kind) :: alpha(2), dtsub + + call t_startf('SLMM_trajectory') + + call slmm_set_hvcoord(hvcoord%etai(1), hvcoord%etai(nlevp), hvcoord%etam) + + ! Set dep_points_all to level-midpoint arrival points. + call init_dep_points_all(elem, hvcoord, nets, nete, independent_time_steps) + + limiter_active_count = 0 + dtsub = dt / nsubstep + do step = 1, nsubstep + ! Fill vnode. + if (vrec%nvel == 2) then + alpha(1) = real(nsubstep - step , real_kind)/nsubstep + alpha(2) = real(nsubstep - step + 1, real_kind)/nsubstep + do ie = nets, nete + call calc_nodal_velocities(elem(ie), deriv, hvcoord, tl, & + independent_time_steps, dtsub, alpha, & + elem(ie)%derived%vstar, elem(ie)%derived%dp, & + elem(ie)%state%v(:,:,:,:,tl%np1), elem(ie)%state%dp3d(:,:,:,tl%np1), & + vnode(:,:,:,:,ie)) + end do + else + call calc_nodal_velocities_using_vrec(elem, deriv, hvcoord, tl, & + independent_time_steps, dtsub, nsubstep, step, nets, nete) + end if + + call dss_vnode(elem, nets, nete, hybrid, vnode) + + if (step == 1) then + call update_dep_points_all(independent_time_steps, dtsub, nets, nete, vnode) + else + ! Fill vdep. + call slmm_calc_v_departure(nets, nete, step, dtsub, dep_points_all, & + & dep_points_ndim, vnode, vdep, info) + + ! Using vdep, update dep_points_all to departure points. + call update_dep_points_all(independent_time_steps, dtsub, nets, nete, vdep) + end if + end do + + if (independent_time_steps) then + call interp_departure_points_to_floating_level_midpoints( & + elem, nets, nete, tl, hvcoord, dep_points_all, limiter_active_count) + ! Not needed in practice. Corner cases will be cleaned up by dss_Qdp. + !call dss_divdp(elem, nets, nete, hybrid) + if (iand(semi_lagrange_diagnostics, 1) /= 0) then + limiter_active_count = ParallelSum(limiter_active_count, hybrid) + if (limiter_active_count > 0 .and. hybrid%masterthread) then + write(iulog, '(a,i11)') 'COMPOSE> limiter_active_count', & + limiter_active_count + end if + end if + end if + + call t_stopf('SLMM_trajectory') +#endif + end subroutine calc_enhanced_trajectory + + subroutine init_dep_points_all(elem, hvcoord, nets, nete, independent_time_steps) + ! Initialize dep_points_all to the Eulerian coordinates. + + type (element_t), intent(inout) :: elem(:) + type (hvcoord_t), intent(in) :: hvcoord + integer, intent(in) :: nets, nete + logical, intent(in) :: independent_time_steps + + type (cartesian3D_t) :: c3d + integer :: ie, i, j, k + + do ie = nets, nete + do j = 1, np + do i = 1, np + call sphere2cart(elem(ie)%spherep(i,j), c3d) + dep_points_all(1,i,j,1,ie) = c3d%x + dep_points_all(2,i,j,1,ie) = c3d%y + dep_points_all(3,i,j,1,ie) = c3d%z + do k = 2, nlev + dep_points_all(1:3,i,j,k,ie) = dep_points_all(1:3,i,j,1,ie) + end do + if (independent_time_steps) then + do k = 1, nlev + dep_points_all(4,i,j,k,ie) = hvcoord%etam(k) + end do + end if + end do + end do + end do + end subroutine init_dep_points_all + + subroutine calc_nodal_velocities_using_vrec(elem, deriv, hvcoord, tl, & + independent_time_steps, dtsub, nsubstep, step, nets, nete) + + ! Wrapper to calc_nodal_velocities that orchestrates the use of the various + ! sources of velocity data. + + type (element_t), intent(in) :: elem(:) + type (derivative_t), intent(in) :: deriv + type (hvcoord_t), intent(in) :: hvcoord + type (TimeLevel_t), intent(in) :: tl + logical, intent(in) :: independent_time_steps + real(real_kind), intent(in) :: dtsub + integer, intent(in) :: nsubstep, step, nets, nete + + integer :: ie, i, k, os + real(real_kind) :: time, alpha(2), vs(np,np,2,nlev,3), dps(np,np,nlev,3) + + do ie = nets, nete + do i = 1, 2 + k = nsubstep - step + (i-1) + time = (k*vrec%t_vel(vrec%nvel))/nsubstep + os = i-1 + k = vrec%run_step(step+1-i) + if (k == 2) then + vs(:,:,:,:,os+1) = elem(ie)%derived%vstar + dps(:,:,:,os+1) = elem(ie)%derived%dp + else + vs(:,:,:,:,os+1) = vrec%vel(:,:,:,:,k-1,ie) + dps(:,:,:,os+1) = vrec%dp(:,:,:,k-1,ie) + end if + if (k == vrec%nvel) then + vs(:,:,:,:,os+2) = elem(ie)%state%v(:,:,:,:,tl%np1) + dps(:,:,:,os+2) = elem(ie)%state%dp3d(:,:,:,tl%np1) + else + vs(:,:,:,:,os+2) = vrec%vel(:,:,:,:,k,ie) + dps(:,:,:,os+2) = vrec%dp(:,:,:,k,ie) + end if + alpha(1) = (vrec%t_vel(k) - time)/(vrec%t_vel(k) - vrec%t_vel(k-1)) + alpha(2) = 1 - alpha(1) + vs(:,:,:,:,os+1) = alpha(1)*vs(:,:,:,:,os+1) + alpha(2)*vs(:,:,:,:,os+2) + dps(:,:,:, os+1) = alpha(1)*dps(:,:,:, os+1) + alpha(2)*dps(:,:,:, os+2) + end do + alpha(1) = 0 + alpha(2) = 1 + call calc_nodal_velocities(elem(ie), deriv, hvcoord, tl, & + independent_time_steps, dtsub, alpha, & + vs(:,:,:,:,1), dps(:,:,:,1), vs(:,:,:,:,2), dps(:,:,:,2), & + vnode(:,:,:,:,ie)) + end do + end subroutine calc_nodal_velocities_using_vrec + + subroutine calc_nodal_velocities(elem, deriv, hvcoord, tl, & + independent_time_steps, dtsub, alpha, v1, dp1, v2, dp2, vnode) + ! Evaluate a formula to provide an estimate of nodal velocities that + ! are use to create a 2nd-order update to the trajectory. The + ! fundamental formula for the update in position p from arrival point + ! p1 to departure point p0 is + ! p0 = p1 - dt/2 (v(p1,t0) + v(p1,t1) - dt v(p1,t1) grad v(p1,t0)). + ! Here we compute the velocity estimate at the nodes: + ! 1/2 (v(p1,t0) + v(p1,t1) - dt v(p1,t1) grad v(p1,t0)). + + type (element_t), intent(in) :: elem + type (derivative_t), intent(in) :: deriv + type (hvcoord_t), intent(in) :: hvcoord + type (TimeLevel_t), intent(in) :: tl + logical, intent(in) :: independent_time_steps + real(real_kind), intent(in) :: dtsub, alpha(2) + real(real_kind), dimension(np,np,2,nlev), intent(in) :: v1, v2 + real(real_kind), dimension(np,np,nlev), intent(in) :: dp1, dp2 + real(real_kind), intent(out) :: vnode(:,:,:,:) + + real(real_kind) :: vsph(np,np,2,nlev,2), eta_dot(np,np,nlevp,2) + integer :: t + + if (independent_time_steps) then + call calc_eta_dot_ref_mid(elem, deriv, tl, hvcoord, alpha, & + & v1, dp1, v2, dp2, eta_dot) + else + eta_dot = zero + end if + + ! Collect the horizontal nodal velocities. v1,2 are on Eulerian levels. v1 + ! is from time t1 < t2. + do t = 1, 2 + vsph(:,:,:,:,t) = (1 - alpha(t))*v1 + alpha(t)*v2 + end do + + ! Given the vertical and horizontal nodal velocities at time + ! endpoints, evaluate the velocity estimate formula, providing the + ! final horizontal and vertical velocity estimates at midpoint nodes. + call calc_vel_horiz_formula_node_ref_mid( & + & elem, deriv, hvcoord, dtsub, vsph, eta_dot, vnode) + if (independent_time_steps) then + call calc_eta_dot_formula_node_ref_mid( & + elem, deriv, hvcoord, dtsub, vsph, eta_dot, vnode) + end if + end subroutine calc_nodal_velocities + + subroutine calc_eta_dot_ref_mid(elem, deriv, tl, hvcoord, alpha, v1, dp1, v2, dp2, eta_dot) + ! Compute eta_dot at midpoint nodes at the start and end of the substep. + + type (element_t), intent(in) :: elem + type (derivative_t), intent(in) :: deriv + type (TimeLevel_t), intent(in) :: tl + type (hvcoord_t), intent(in) :: hvcoord + real(real_kind), intent(in) :: alpha(2) + real(real_kind), dimension(np,np,2,nlev), intent(in) :: v1, v2 + real(real_kind), dimension(np,np,nlev), intent(in) :: dp1, dp2 + real(real_kind), intent(out) :: eta_dot(np,np,nlevp,2) + + real(real_kind) :: vdp(np,np,2), w1(np,np) + integer :: t, k, d + + do t = 1,2 + ! eta_dot_dpdn at interface nodes. + eta_dot(:,:,1,t) = zero + do k = 1,nlev + do d = 1,2 + vdp(:,:,d) = (1 - alpha(t))*v1(:,:,d,k)*dp1(:,:,k) + & + & alpha(t) *v2(:,:,d,k)*dp2(:,:,k) + end do + w1 = divergence_sphere(vdp, deriv, elem) + eta_dot(:,:,k+1,t) = eta_dot(:,:,k,t) + w1 + end do + w1 = eta_dot(:,:,nlevp,t) + eta_dot(:,:,nlevp,t) = zero + do k = 2,nlev + eta_dot(:,:,k,t) = hvcoord%hybi(k)*w1 - eta_dot(:,:,k,t) + end do + ! Transform eta_dot_dpdn at interfaces to eta_dot at midpoints using the + ! formula + ! eta_dot = eta_dot_dpdn/(A_eta p0 + B_eta ps) + ! a= eta_dot_dpdn diff(eta)/(diff(A) p0 + diff(B) ps). + ! Compute ps. + w1 = hvcoord%hyai(1)*hvcoord%ps0 + & + & (1 - alpha(t))*sum(dp1, 3) + & + & alpha(t) *sum(dp2, 3) + do k = 1,nlev + eta_dot(:,:,k,t) = half*(eta_dot(:,:,k,t) + eta_dot(:,:,k+1,t)) & + & * (hvcoord%etai(k+1) - hvcoord%etai(k)) & + & / ( (hvcoord%hyai(k+1) - hvcoord%hyai(k))*hvcoord%ps0 & + & + (hvcoord%hybi(k+1) - hvcoord%hybi(k))*w1) + end do + end do + end subroutine calc_eta_dot_ref_mid + + subroutine calc_vel_horiz_formula_node_ref_mid( & + elem, deriv, hvcoord, dtsub, vsph, eta_dot, vnode) + + type (element_t), intent(in) :: elem + type (derivative_t), intent(in) :: deriv + type (hvcoord_t), intent(in) :: hvcoord + real(real_kind), intent(in) :: dtsub, vsph(np,np,2,nlev,2), eta_dot(np,np,nlevp,2) + real(real_kind), intent(inout) :: vnode(:,:,:,:) + + integer, parameter :: t0 = 1, t1 = 2 + + real(real_kind) :: vfsph(np,np,2), w1(np,np), w2(np,np), w3(np,np,3) + integer :: k, d, i, k1, k2 + + do k = 1, nlev + ! Horizontal terms. + vfsph = ugradv_sphere(vsph(:,:,:,k,t1), vsph(:,:,:,k,t0), deriv, elem) + vfsph = vsph(:,:,:,k,t0) + vsph(:,:,:,k,t1) - dtsub*vfsph + ! Vertical term. + do d = 1, 2 ! horiz vel dims + if (k == 1 .or. k == nlev) then + if (k == 1) then + k1 = 1; k2 = 2 + else + k1 = nlev-1; k2 = nlev + end if + w1 = (vsph(:,:,d,k2,t0) - vsph(:,:,d,k1,t0)) / & + (hvcoord%etam(k2) - hvcoord%etam(k1)) + else + do i = 1, 3 + w3(:,:,i) = hvcoord%etam(k-2+i) ! interp support + end do + w2 = hvcoord%etam(k) ! derivative at this eta value + call eval_lagrange_poly_derivative(3, w3, vsph(:,:,d,k-1:k+1,t0), w2, w1) + end if + vfsph(:,:,d) = vfsph(:,:,d) - dtsub*eta_dot(:,:,k,t1)*w1 + end do + ! Finish the formula. + vfsph = half*vfsph + ! Transform to Cartesian. + do d = 1, 3 + vnode(d,:,:,k) = sum(elem%vec_sphere2cart(:,:,d,:)*vfsph, 3) + end do + end do + end subroutine calc_vel_horiz_formula_node_ref_mid + + subroutine calc_eta_dot_formula_node_ref_mid( & + elem, deriv, hvcoord, dtsub, vsph, eta_dot, vnode) + type (element_t), intent(in) :: elem + type (derivative_t), intent(in) :: deriv + type (hvcoord_t), intent(in) :: hvcoord + real(real_kind), intent(in) :: dtsub, vsph(np,np,2,nlev,2), eta_dot(np,np,nlevp,2) + real(real_kind), intent(inout) :: vnode(:,:,:,:) + + integer, parameter :: t0 = 1, t1 = 2 + + real(real_kind) :: vfsph(np,np,2), w1(np,np), w2(np,np), w3(np,np,3), w4(np,np,3) + integer :: k, d, i, k1, k2 + + do k = 1, nlev + w2 = hvcoord%etam(k) + if (k == 1 .or. k == nlev) then + if (k == 1) then + w3(:,:,1) = hvcoord%etai(1) + w4(:,:,1) = zero + do i = 1, 2 + w3(:,:,i+1) = hvcoord%etam(i) + w4(:,:,i+1) = eta_dot(:,:,i,t0) + end do + else + do i = 1, 2 + w3(:,:,i) = hvcoord%etam(nlev-2+i) + w4(:,:,i) = eta_dot(:,:,nlev-2+i,t0) + end do + w3(:,:,3) = hvcoord%etai(nlevp) + w4(:,:,3) = zero + end if + call eval_lagrange_poly_derivative(3, w3, w4, w2, w1) + else + k1 = k-1 + k2 = k+1 + do i = 1, 3 + w3(:,:,i) = hvcoord%etam(k1-1+i) + end do + call eval_lagrange_poly_derivative(k2-k1+1, w3, eta_dot(:,:,k1:k2,t0), w2, w1) + end if + w3(:,:,1:2) = gradient_sphere(eta_dot(:,:,k,t0), deriv, elem%Dinv) + vnode(4,:,:,k) = & + half*(eta_dot(:,:,k,t0) + eta_dot(:,:,k,t1) & + & - dtsub*(vsph(:,:,1,k,t1)*w3(:,:,1) + vsph(:,:,2,k,t1)*w3(:,:,2) & + & + eta_dot(:,:,k,t1)*w1)) + end do + end subroutine calc_eta_dot_formula_node_ref_mid + + subroutine update_dep_points_all(independent_time_steps, dtsub, nets, nete, vdep) + ! Determine the departure points corresponding to the reference grid's + ! arrival midpoints. Reads and writes dep_points_all. Reads vdep. + + use physical_constants, only: scale_factor + + logical, intent(in) :: independent_time_steps + real(real_kind), intent(in) :: dtsub + integer, intent(in) :: nets, nete + real(real_kind), intent(in) :: vdep(:,:,:,:,:) + + real(real_kind) :: norm, p(3) + integer :: ie, k, j, i + + do ie = nets, nete + do k = 1, nlev + do j = 1, np + do i = 1, np + ! Update horizontal position. + p = dep_points_all(1:3,i,j,k,ie) + p = p - dtsub*vdep(1:3,i,j,k,ie)/scale_factor + if (is_sphere) then + norm = sqrt(p(1)*p(1) + p(2)*p(2) + p(3)*p(3)) + p = p/norm + end if + dep_points_all(1:3,i,j,k,ie) = p + if (independent_time_steps) then + ! Update vertical position. + dep_points_all(4,i,j,k,ie) = dep_points_all(4,i,j,k,ie) - & + & dtsub*vdep(4,i,j,k,ie) + end if + end do + end do + end do + end do + end subroutine update_dep_points_all + + subroutine interp_departure_points_to_floating_level_midpoints( & + elem, nets, nete, tl, hvcoord, dep_points_all, limcnt) + ! Determine the departure points corresponding to the vertically Lagragnian + ! grid's arrival midpoints, where the floating levels are those that evolve + ! over the course of the full tracer time step. Also compute + ! elem%derived%divdp, which holds the floating levels' dp values for later + ! use in vertical remap. + + type (element_t), intent(inout) :: elem(:) + integer, intent(in) :: nets, nete + type (hvcoord_t), intent(in) :: hvcoord + type (TimeLevel_t), intent(in) :: tl + real(real_kind), intent(inout) :: dep_points_all(:,:,:,:,:) + integer, intent(inout) :: limcnt + + real(real_kind) :: deta_ref(nlevp), w1(np,np), v1(np,np,nlev), & + & v2(np,np,nlevp), p(3) + integer :: ie, i, j, k, d + + call set_deta_tol(hvcoord) + + deta_ref(1) = hvcoord%etam(1) - hvcoord%etai(1) + do k = 2, nlev + deta_ref(k) = hvcoord%etam(k) - hvcoord%etam(k-1) + end do + deta_ref(nlevp) = hvcoord%etai(nlevp) - hvcoord%etam(nlev) + + do ie = nets, nete + ! Surface pressure. + w1 = hvcoord%hyai(1)*hvcoord%ps0 + sum(elem(ie)%state%dp3d(:,:,:,tl%np1), 3) + + ! Reconstruct Lagrangian levels at t1 on arrival column: + ! eta_arr_int = I[eta_ref_mid([0,eta_dep_mid,1])](eta_ref_int) + call limit_etam(hvcoord, deta_ref, dep_points_all(4,:,:,:,ie), v1, limcnt) + v2(:,:,1) = hvcoord%etai(1) + v2(:,:,nlevp) = hvcoord%etai(nlevp) + call eta_interp_eta(hvcoord, v1, hvcoord%etam, & + & nlevp-2, hvcoord%etai(2:nlev), v2(:,:,2:nlev)) + call eta_to_dp(hvcoord, w1, v2, elem(ie)%derived%divdp) + + ! Compute Lagrangian level midpoints at t1 on arrival column: + ! eta_arr_mid = I[eta_ref_mid([0,eta_dep_mid,1])](eta_ref_mid) + call eta_interp_eta(hvcoord, v1, hvcoord%etam, & + & nlev, hvcoord%etam, v2(:,:,1:nlev)) + dep_points_all(4,:,:,:,ie) = v2(:,:,1:nlev) + + ! Compute departure horizontal points corresponding to arrival + ! Lagrangian level midpoints: + ! p_dep_mid(eta_arr_mid) = I[p_dep_mid(eta_ref_mid)](eta_arr_mid) + do d = 1, 3 + v1 = dep_points_all(d,:,:,:,ie) + call eta_interp_horiz(hvcoord, hvcoord%etam, v1, & + & v2(:,:,1:nlev), dep_points_all(d,:,:,:,ie)) + end do + if (is_sphere) then + ! Normalize p = (x,y,z). + do k = 1, nlev + do j = 1, np + do i = 1, np + p = dep_points_all(1:3,i,j,k,ie) + p = p/sqrt(p(1)*p(1) + p(2)*p(2) + p(3)*p(3)) + dep_points_all(1:3,i,j,k,ie) = p + end do + end do + end do + end if + end do + end subroutine interp_departure_points_to_floating_level_midpoints + + subroutine set_deta_tol(hvcoord) + type (hvcoord_t), intent(in) :: hvcoord + + real(real_kind) :: deta_ave + integer :: k + + if (deta_tol >= 0) return + + ! Benign write race condition. A thread might see eta_tol < 0 and set it + ! here even as another thread does the same. But because there is no read + ! and only one value to write, the redundant writes don't matter. + + deta_ave = (hvcoord%etai(nlev+1) - hvcoord%etai(1)) / nlev + deta_tol = 10_real_kind*eps*deta_ave + end subroutine set_deta_tol + + subroutine limit_etam(hvcoord, deta_ref, eta, eta_lim, cnt) + type (hvcoord_t), intent(in) :: hvcoord + real(real_kind), intent(in) :: deta_ref(nlevp), eta(np,np,nlev) + real(real_kind), intent(out) :: eta_lim(np,np,nlev) + integer, intent(inout) :: cnt + + real(real_kind) :: deta(nlevp) + integer :: i, j, k + logical :: ok + + do j = 1, np + do i = 1, np + ! Check nonmonotonicity in eta. + ok = eta(i,j,1) - hvcoord%etai(1) >= deta_tol + if (ok) then + do k = 2, nlev + if (eta(i,j,k) - eta(i,j,k-1) < deta_tol) then + ok = .false. + exit + end if + end do + if (ok) then + ok = hvcoord%etai(nlevp) - eta(i,j,nlev) >= deta_tol + end if + end if + ! eta is monotonically increasing, so don't need to do anything + ! further. + if (ok) then + eta_lim(i,j,:) = eta(i,j,:) + cycle + end if + + deta(1) = eta(i,j,1) - hvcoord%etai(1) + do k = 2, nlev + deta(k) = eta(i,j,k) - eta(i,j,k-1) + end do + deta(nlevp) = hvcoord%etai(nlevp) - eta(i,j,nlev) + ! [0, etam(1)] and [etam(nlev),1] are half levels, but deta_tol is so + ! small there's no reason not to use it as a lower bound for these. + cnt = cnt + 1 + call deta_caas(nlevp, deta_ref, deta_tol, deta) + eta_lim(i,j,1) = hvcoord%etai(1) + deta(1) + do k = 2, nlev + eta_lim(i,j,k) = eta_lim(i,j,k-1) + deta(k) + end do + end do + end do + end subroutine limit_etam + + subroutine deta_caas(nlp, deta_ref, lo, deta) + integer, intent(in) :: nlp + real(real_kind), intent(in) :: deta_ref(nlp), lo + real(real_kind), intent(inout) :: deta(nlp) + + real(real_kind) :: nerr, w(nlp) + integer :: k + + nerr = zero + do k = 1, nlp + if (deta(k) < lo) then + nerr = nerr + (deta(k) - lo) + deta(k) = lo + w(k) = zero + else + if (deta(k) > deta_ref(k)) then + ! Only pull mass from intervals that are larger than their + ! reference value. This concentrates changes to intervals that, by + ! having a lot more mass than usual, drive other levels negative. + w(k) = deta(k) - deta_ref(k) + else + w(k) = zero + end if + end if + end do + if (nerr /= zero) deta = deta + nerr*(w/sum(w)) + end subroutine deta_caas + + subroutine linterp(n, x, y, ni, xi, yi, caller) + ! Linear interpolant: yi = I[y(x)](xi). + ! x and xi must be in ascending order. + ! xi(1) must be >= x(1) and xi(ni) must be <= x(n). + + use kinds, only: iulog + + integer, intent(in) :: n, ni + real(real_kind), intent(in) :: x(n), y(n), xi(ni) + real(real_kind), intent(out) :: yi(ni) + character(len=*), intent(in) :: caller + + integer :: k, ki + real(real_kind) :: a + + if (xi(1) < x(1) .or. xi(ni) > x(n)) then + write(iulog,*) 'x', x + write(iulog,*) 'xi', xi + call abortmp('sl_vertically_remap_tracers> linterp xi out of bounds: ' & + // trim(caller)) + end if + + k = 2 + do ki = 1, ni + do while (x(k) < xi(ki)) + k = k + 1 + end do + a = (xi(ki) - x(k-1))/(x(k) - x(k-1)) + yi(ki) = (1 - a)*y(k-1) + a*y(k) + end do + end subroutine linterp + + subroutine eta_interp_eta(hvcoord, x, y, ni, xi, yi) + type (hvcoord_t), intent(in) :: hvcoord + real(real_kind), intent(in) :: x(np,np,nlev), y(nlev) + integer, intent(in) :: ni + real(real_kind), intent(in) :: xi(ni) + real(real_kind), intent(out) :: yi(np,np,ni) + + real(real_kind) :: x01(nlev+2), y01(nlev+2) + integer :: i, j + + x01(1) = hvcoord%etai(1) + x01(nlev+2) = hvcoord%etai(nlevp) + y01(1) = hvcoord%etai(1) + y01(2:nlev+1) = y + y01(nlev+2) = hvcoord%etai(nlevp) + do j = 1, np + do i = 1, np + x01(2:nlev+1) = x(i,j,:) + call linterp(nlev+2, x01, y01, & + & ni, xi, yi(i,j,:), & + & 'eta_interp_eta') + end do + end do + end subroutine eta_interp_eta + + subroutine eta_interp_horiz(hvcoord, x, y, xi, yi) + type (hvcoord_t), intent(in) :: hvcoord + real(real_kind), intent(in) :: x(nlev), y(np,np,nlev), xi(np,np,nlev) + real(real_kind), intent(out) :: yi(np,np,nlev) + + real(real_kind) :: xbdy(nlev+2), ybdy(nlev+2) + integer :: i, j + + xbdy(1) = hvcoord%etai(1) + xbdy(2:nlev+1) = x + xbdy(nlev+2) = hvcoord%etai(nlevp) + do j = 1, np + do i = 1, np + ! Do constant interp outside of the etam support. + ybdy(1) = y(i,j,1) + ybdy(2:nlev+1) = y(i,j,:) + ybdy(nlev+2) = y(i,j,nlev) + call linterp(nlev+2, xbdy, ybdy, & + & nlev, xi(i,j,:), yi(i,j,:), & + & 'eta_interp_horiz') + end do + end do + end subroutine eta_interp_horiz + + subroutine eta_to_dp(hvcoord, ps, etai, dp) + ! e = A(e) + B(e) + ! p(e) = A(e) p0 + B(e) ps + ! = e p0 + B(e) (ps - p0) + ! a= e p0 + I[Bi(eref)](e) (ps - p0) + + type (hvcoord_t), intent(in) :: hvcoord + real(real_kind), intent(in) :: ps(np,np), etai(np,np,nlevp) + real(real_kind), intent(out) :: dp(np,np,nlev) + + real(real_kind) :: Bi(nlevp), dps + integer :: i, j, k + + do j = 1, np + do i = 1, np + call linterp(nlevp, hvcoord%etai, hvcoord%hybi, & + & nlevp, etai(i,j,:), Bi, & + & 'eta_to_dp') + dps = ps(i,j) - hvcoord%ps0 + do k = 1, nlev + dp(i,j,k) = (etai(i,j,k+1) - etai(i,j,k))*hvcoord%ps0 + & + & (Bi(k+1) - Bi(k))*dps + end do + end do + end do + end subroutine eta_to_dp + + subroutine dss_vnode(elem, nets, nete, hybrid, vnode) + type (element_t), intent(in) :: elem(:) + type (hybrid_t), intent(in) :: hybrid + integer, intent(in) :: nets, nete + real(real_kind) :: vnode(:,:,:,:,:) + + integer :: nd, nlyr, ie, k, d + + nd = size(vnode, 1) + nlyr = nd*nlev + + do ie = nets, nete + do k = 1, nlev + do d = 1, nd + vnode(d,:,:,k,ie) = vnode(d,:,:,k,ie)* & + & elem(ie)%spheremp*elem(ie)%rspheremp + end do + end do + do d = 1, nd + call edgeVpack_nlyr(edge_g, elem(ie)%desc, vnode(d,:,:,:,ie), & + & nlev, nlev*(d-1), nlyr) + end do + end do + + call t_startf('SLMM_bexchV') + call bndry_exchangeV(hybrid, edge_g) + call t_stopf('SLMM_bexchV') + + do ie = nets, nete + do d = 1, nd + call edgeVunpack_nlyr(edge_g, elem(ie)%desc, vnode(d,:,:,:,ie), & + & nlev, nlev*(d-1), nlyr) + end do + end do + +#if (defined HORIZ_OPENMP) + !$omp barrier +#endif + end subroutine dss_vnode + + subroutine dss_divdp(elem, nets, nete, hybrid) + type (element_t), intent(inout) :: elem(:) + type (hybrid_t), intent(in) :: hybrid + integer, intent(in) :: nets, nete + + integer :: ie, k + + do ie = nets, nete + do k = 1, nlev + elem(ie)%derived%divdp(:,:,k) = elem(ie)%derived%divdp(:,:,k)* & + & elem(ie)%spheremp*elem(ie)%rspheremp + end do + call edgeVpack_nlyr(edge_g, elem(ie)%desc, elem(ie)%derived%divdp, & + & nlev, 0, nlev) + end do + + call t_startf('SLMM_bexchV') + call bndry_exchangeV(hybrid, edge_g) + call t_stopf('SLMM_bexchV') + + do ie = nets, nete + call edgeVunpack_nlyr(edge_g, elem(ie)%desc, elem(ie)%derived%divdp, & + & nlev, 0, nlev) + end do + +#if (defined HORIZ_OPENMP) + !$omp barrier +#endif + end subroutine dss_divdp + + function assert(b, msg) result(nerr) + use kinds, only: iulog + + logical, intent(in) :: b + character(*), optional, intent(in) :: msg + + character(len=128) :: s integer :: nerr nerr = 0 - nerr = nerr + test_lagrange() - nerr = nerr + test_reconstruct_and_limit_dp() + if (b) return - if (nerr > 0 .and. par%masterproc) write(iulog,'(a,i3)') 'sl_unittest FAIL', nerr + s = '' + if (present(msg)) s = msg + write(iulog,'(a,a)') 'COMPOSE> sl_advection ASSERT: ', trim(s) + nerr = 1 + end function assert + + function test_linterp() result (nerr) + integer, parameter :: n = 128, ni = 111 + + real(real_kind) :: x(n), y(n), xi(ni), yi(ni), yin(n), a + integer :: k, nerr + + call random_number(x) + do k = 2, n + x(k) = x(k) + x(k-1) + end do + y = 3*x + + do k = 1, ni + a = real(k, real_kind)/(ni+1) + xi(k) = (1 - a)*x(1) + a*x(n) + end do + + call linterp(n, x, y, ni, xi, yi, 'test_linterp 1') + nerr = assert(maxval(abs( yi - 3*xi)) < 100*eps*x(n), 'linterp 1') + + call linterp(n, x, y, n, x, yin, 'test_linterp 2') + nerr = nerr + assert(maxval(abs(yin - y)) < 10*eps, 'linterp 2') + end function test_linterp + + function test_eta_to_dp(hvcoord) result(nerr) + type (hvcoord_t), intent(in) :: hvcoord + + real(real_kind) :: ps(np,np), etai(np,np,nlevp), dp1(np,np,nlev), & + & dp2(np,np,nlev) + integer :: nerr, i, j, k + + nerr = 0 + + call random_number(ps) + ps = (one + 0.2*(ps - 0.5))*hvcoord%ps0 + + do k = 1, nlev + dp1(:,:,k) = (hvcoord%hyai(k+1) - hvcoord%hyai(k))*hvcoord%ps0 + & + & (hvcoord%hybi(k+1) - hvcoord%hybi(k))*ps + end do + + ! Test that for etai_ref we get the same as the usual formula. + do j = 1, np + do i = 1, np + etai(i,j,:) = hvcoord%etai + end do + end do + call eta_to_dp(hvcoord, ps, etai, dp2) + nerr = nerr + assert(maxval(abs(dp2-dp1)) < 100*eps*maxval(dp1), 'eta_to_dp 1') + end function test_eta_to_dp + + function test_deta_caas() result(nerr) + integer, parameter :: nl = 128, nlp = nl+1 + + real(real_kind) :: deta_ref(nlp), etam_ref(nl), deta_tol, etam(nl), & + & deta(nlp) + integer :: i, k, nerr + + nerr = 0 + + call random_number(deta_ref) + deta_ref = deta_ref + 0.1 + deta_ref = deta_ref/sum(deta_ref) + + deta_tol = 10_real_kind*eps*sum(deta_ref)/size(deta_ref) + nerr = nerr + assert(deta_tol < minval(deta_ref), 'deta_caas 1') + + ! Test: Input not touched. + deta = deta_ref + call deta_caas(nlp, deta_ref, deta_tol, deta) + nerr = nerr + assert(maxval(abs(deta-deta_ref)) == zero, 'deta_caas 2') + + etam_ref(1) = deta_ref(1) + do k = 2, nl + etam_ref(k) = etam_ref(k-1) + deta_ref(k) + end do + + ! Test: Modify one etam and only adjacent intervals change beyond eps. + do i = 1, 2 + etam = etam_ref + if (i == 1) then + etam(11) = etam(11) + 1.1 + else + etam(11) = etam(11) - 13.1 + end if + deta(1) = etam(1) + do k = 2, nl + deta(k) = etam(k) - etam(k-1) + end do + deta(nlp) = one - etam(nl) + nerr = nerr + assert(minval(deta) < deta_tol, 'deta_caas 3') + call deta_caas(nlp, deta_ref, deta_tol, deta) + nerr = nerr + assert(minval(deta) == deta_tol, 'deta_caas 4') + nerr = nerr + assert(abs(sum(deta) - one) < 100*eps, 'deta_caas 5') + deta = abs(deta - deta_ref) + nerr = nerr + assert(maxval(deta(:10)) < 100*eps, 'deta_caas 6') + nerr = nerr + assert(maxval(deta(13:)) < 100*eps, 'deta_caas 7') + end do + + ! Test: Completely messed up levels. + call random_number(deta) + deta = deta - 0.5_real_kind + if (sum(deta) < 0.1) deta = deta + (0.1 - sum(deta))/nlp + deta = deta/sum(deta) + call deta_caas(nlp, deta_ref, deta_tol, deta) + nerr = nerr + assert(minval(deta) == deta_tol, 'deta_caas 8') + nerr = nerr + assert(abs(sum(deta) - one) < 1000*eps, 'deta_caas 9') + end function test_deta_caas + + function test_init_velocity_record() result(error) + integer :: dtf, drf, nsub, nvel, e, error + type (velocity_record_t) :: v + + error = 0 + dtf = 6 + drf = 2 + nsub = 3 + nvel = 4 + call init_velocity_record(1, dtf, drf, nsub, nvel, v, e) + call test(dtf, drf, nsub, nvel, v, e) + if (e > 0) error = 1 + call cleanup(v) + nvel = 3 + call init_velocity_record(1, dtf, drf, nsub, nvel, v, e) + call test(dtf, drf, nsub, nvel, v, e) + if (e > 0) error = 1 + call cleanup(v) + drf = 3 + nvel = 6 + call init_velocity_record(1, dtf, drf, nsub, nvel, v, e) + call test(dtf, drf, nsub, nvel, v, e) + if (e > 0) error = 1 + call cleanup(v) + drf = 1 + nsub = 5 + call init_velocity_record(1, dtf, drf, nsub, nvel, v, e) + call test(dtf, drf, nsub, nvel, v, e) + if (e > 0) error = 1 + call cleanup(v) + dtf = 12 + drf = 2 + nsub = 3 + nvel = -1 + call init_velocity_record(1, dtf, drf, nsub, nvel, v, e) + call test(dtf, drf, nsub, nvel, v, e) + if (e > 0) error = 1 + call cleanup(v) + nsub = 5 + nvel = 5 + call init_velocity_record(1, dtf, drf, nsub, nvel, v, e) + call test(dtf, drf, nsub, nvel, v, e) + if (e > 0) error = 1 + call cleanup(v) + dtf = 27 + drf = 3 + nsub = 51 + nvel = 99 + call init_velocity_record(1, dtf, drf, nsub, nvel, v, e) + call test(dtf, drf, nsub, nvel, v, e) + if (e > 0) error = 1 + call cleanup(v) + + contains + subroutine test(dtf, drf, nsub, nvel, v, error) + integer, intent(in) :: dtf, drf, nsub, nvel + integer, intent(inout) :: error + type (velocity_record_t), intent(in) :: v + + real(kind=real_kind) :: endslots(2), ys(dtf), a, x, y, y0, y1, & + & xsup(2), ysup(2) + integer :: n, e, i, k + + e = 0 + + if (modulo(dtf, drf) /= 0) then + print *, 'TESTING ERROR: dtf % drf == 0 is required:', dtf, drf + end if + + ! Check that t_vel is monotonically increasing. + do n = 2, v%nvel + if (v%t_vel(n) <= v%t_vel(n-1)) e = 1 + end do + + ! Check that obs_slots does not reference end points. This should not + ! happen b/c nvel <= navail and observations are uniformly spaced. + do n = 1, dtf + do i = 1, 2 + if (v%obs_slots(n,i) == 0 .or. v%obs_slots(n,i) == dtf) e = 11 + end do + end do + + ! Check that weights sum to 1. + ys = 0 + do n = 1, dtf + do i = 1, 2 + if (v%obs_slots(n,i) > 0) then + ys(v%obs_slots(n,i)) = ys(v%obs_slots(n,i)) + v%obs_wts(n,i) + end if + end do + end do + do i = 2, v%nvel-1 + if (abs(ys(i) - 1) > 1e3*eps) e = 12 + end do + + ! Test for exact interp of an affine function. + ! Observe data forward in time. + endslots(1) = tfn(0.d0) + endslots(2) = tfn(real(dtf, real_kind)) + ys(:) = 0 + do n = 1, dtf + if (modulo(n, drf) /= 0) cycle + y = tfn(real(n, real_kind)) + do i = 1, 2 + if (v%obs_slots(n,i) == -1) cycle + ys(v%obs_slots(n,i)) = ys(v%obs_slots(n,i)) + v%obs_wts(n,i)*y + end do + end do + ! Use the data backward in time. + do n = 1, nsub + ! Each segment orders the data forward in time. Thus, data are always + ! ordered forward in time but used backward. + do i = 1, 2 + k = nsub - n + (i-1) + xsup(i) = (k*v%t_vel(v%nvel))/nsub + k = v%run_step(n+1-i) + if (k == 2) then + y0 = endslots(1) + else + y0 = ys(k-1) + end if + if (k == v%nvel) then + y1 = endslots(2) + else + y1 = ys(k) + end if + ysup(i) = ((v%t_vel(k) - xsup(i))*y0 + (xsup(i) - v%t_vel(k-1))*y1) / & + & (v%t_vel(k) - v%t_vel(k-1)) + end do + do i = 0, 10 + a = real(i, real_kind)/10 + x = (1-a)*xsup(1) + a*xsup(2) + y = (1-a)*ysup(1) + a*ysup(2) + if (abs(y - tfn(x)) > 1e3*eps) e = 2 + end do + end do + + if (error > 0 .or. e > 0) then + print *, 'ERROR', error, e + print '(a,i3,a,i3,a,i3,a,i3,a,i3)', 'dtf',dtf,' drf',drf,' nsub',nsub, & + ' nvel',nvel,' v%nvel',v%nvel + print '(a,99es11.3)', ' t_vel', (v%t_vel(n),n=1,v%nvel) + do n = 1, dtf-1 + print '(3i3,2f5.2)', n, v%obs_slots(n,:), v%obs_wts(n,:) + end do + do n = 0, nsub + print '(i3,i3)', n, v%run_step(n) + end do + error = 1 + end if + end subroutine test + + function tfn(x) result(y) + real(kind=real_kind), intent(in) :: x + real(kind=real_kind) :: y + + y = 7.1*x - 11.5 + end function tfn + + subroutine cleanup(v) + type (velocity_record_t), intent(inout) :: v + deallocate(v%t_vel, v%obs_slots, v%obs_wts, v%run_step, v%vel, v%dp) + end subroutine cleanup + end function test_init_velocity_record + + subroutine sl_unittest(par, hvcoord) + use kinds, only: iulog + + type (parallel_t), intent(in) :: par + type (hvcoord_t), intent(in) :: hvcoord + + integer :: n(6) + + n(1) = test_lagrange() + n(2) = test_reconstruct_and_limit_dp() + n(3) = test_deta_caas() + n(4) = test_linterp() + n(5) = test_eta_to_dp(hvcoord) + n(6) = test_init_velocity_record() + + if (sum(n) > 0 .and. par%masterproc) then + write(iulog,'(a,6i2)') 'COMPOSE> sl_unittest FAIL ', n + end if end subroutine sl_unittest end module sl_advection diff --git a/components/homme/src/test_mod.F90 b/components/homme/src/test_mod.F90 index 361465218e5..0f5220282cc 100644 --- a/components/homme/src/test_mod.F90 +++ b/components/homme/src/test_mod.F90 @@ -21,7 +21,7 @@ module test_mod use baroclinic_inst_mod, only: binst_init_state, jw_baroclinic use dcmip12_wrapper, only: dcmip2012_test1_1, dcmip2012_test1_2, dcmip2012_test1_3,& dcmip2012_test2_0, dcmip2012_test2_x, dcmip2012_test3, & - dcmip2012_test4_init, mtest_init, dcmip2012_test1_1_conv + dcmip2012_test4_init, mtest_init, dcmip2012_test1_conv use dcmip16_wrapper, only: dcmip2016_test1, dcmip2016_test2, dcmip2016_test3, & dcmip2016_test1_forcing, dcmip2016_test2_forcing, dcmip2016_test3_forcing, & dcmip2016_pg_init, dcmip2016_test1_pg, dcmip2016_test1_pg_forcing, dcmip2016_init @@ -68,7 +68,8 @@ subroutine set_test_initial_conditions(elem, deriv, hybrid, hvcoord, tl, nets, n case('asp_tracer'); case('baroclinic'); case('dcmip2012_test1_1'); - case('dcmip2012_test1_1_conv'); + case('dcmip2012_test1_3a_conv', 'dcmip2012_test1_3b_conv', 'dcmip2012_test1_3c_conv', & + 'dcmip2012_test1_3d_conv', 'dcmip2012_test1_3e_conv', 'dcmip2012_test1_3f_conv') case('dcmip2012_test1_2'); case('dcmip2012_test1_3'); case('dcmip2012_test2_0'); @@ -118,9 +119,10 @@ subroutine set_test_initial_conditions(elem, deriv, hybrid, hvcoord, tl, nets, n case('asp_tracer'); call asp_tracer (elem,hybrid,hvcoord,nets,nete) case('baroclinic'); call binst_init_state (elem,hybrid, nets, nete, hvcoord) case('dcmip2012_test1_1'); call dcmip2012_test1_1(elem,hybrid,hvcoord,nets,nete,0.0d0,1,timelevels) - case('dcmip2012_test1_1_conv') + case('dcmip2012_test1_3a_conv', 'dcmip2012_test1_3b_conv', 'dcmip2012_test1_3c_conv', & + 'dcmip2012_test1_3d_conv', 'dcmip2012_test1_3e_conv', 'dcmip2012_test1_3f_conv') midpoint_eta_dot_dpdn = .true. - call dcmip2012_test1_1_conv(elem,hybrid,hvcoord,nets,nete,0.0d0,1,timelevels) + call dcmip2012_test1_conv(test_case,elem,hybrid,hvcoord,deriv,nets,nete,0.0d0,1,timelevels) case('dcmip2012_test1_2'); call dcmip2012_test1_2(elem,hybrid,hvcoord,nets,nete,0.0d0,1,timelevels) case('dcmip2012_test1_3'); call dcmip2012_test1_3(elem,hybrid,hvcoord,nets,nete,0.0d0,1,timelevels,deriv) case('dcmip2012_test2_0'); call dcmip2012_test2_0(elem,hybrid,hvcoord,nets,nete) @@ -197,7 +199,9 @@ subroutine set_test_prescribed_wind(elem, deriv, hybrid, hvcoord, dt, tl, nets, ! set prescribed quantities at timelevel np1 select case(test_case) case('dcmip2012_test1_1'); call dcmip2012_test1_1(elem,hybrid,hvcoord,nets,nete,time,np1,np1) - case('dcmip2012_test1_1_conv'); call dcmip2012_test1_1_conv(elem,hybrid,hvcoord,nets,nete,time,np1,np1) + case('dcmip2012_test1_3a_conv', 'dcmip2012_test1_3b_conv', 'dcmip2012_test1_3c_conv', & + 'dcmip2012_test1_3d_conv', 'dcmip2012_test1_3e_conv', 'dcmip2012_test1_3f_conv') + call dcmip2012_test1_conv(test_case,elem,hybrid,hvcoord,deriv,nets,nete,time,np1,np1) case('dcmip2012_test1_2'); call dcmip2012_test1_2(elem,hybrid,hvcoord,nets,nete,time,np1,np1) case('dcmip2012_test1_3'); call dcmip2012_test1_3(elem,hybrid,hvcoord,nets,nete,time,np1,np1,deriv) endselect @@ -358,7 +362,9 @@ subroutine print_test_results(elem, tl, hvcoord, par) type(parallel_t), intent(in) :: par select case(test_case) - case('dcmip2012_test1_1_conv'); call dcmip2012_print_test1_conv_results(elem, tl, hvcoord, par, 1) + case('dcmip2012_test1_3a_conv', 'dcmip2012_test1_3b_conv', 'dcmip2012_test1_3c_conv', & + 'dcmip2012_test1_3d_conv', 'dcmip2012_test1_3e_conv', 'dcmip2012_test1_3f_conv') + call dcmip2012_print_test1_conv_results(test_case, elem, tl, hvcoord, par, 1) end select end subroutine print_test_results diff --git a/components/homme/src/test_src/dcmip12_wrapper.F90 b/components/homme/src/test_src/dcmip12_wrapper.F90 index bd4da6eff78..2325137f391 100644 --- a/components/homme/src/test_src/dcmip12_wrapper.F90 +++ b/components/homme/src/test_src/dcmip12_wrapper.F90 @@ -12,7 +12,7 @@ module dcmip12_wrapper use control_mod, only: test_case, dcmip4_moist, dcmip4_X, vanalytic use dcmip2012_test1_2_3, only: test1_advection_deformation, test1_advection_hadley, test1_advection_orography, & test2_steady_state_mountain, test2_schaer_mountain,test3_gravity_wave -use dcmip2012_test1_conv, only: test1_conv_advection_deformation +use dcmip2012_test1_conv_mod, only: test1_conv_advection, test1_conv_print_results use dcmip2012_test4, only: test4_baroclinic_wave use mtests, only: mtest_state use derivative_mod, only: derivative_t, gradient_sphere @@ -116,82 +116,6 @@ subroutine dcmip2012_test1_1(elem,hybrid,hvcoord,nets,nete,time,n0,n1) end subroutine -!_____________________________________________________________________ -subroutine dcmip2012_test1_1_conv(elem,hybrid,hvcoord,nets,nete,time,n0,n1) - - ! 3d deformational flow - - ! Use physical constants consistent with HOMME - use physical_constants, only: Rd => Rgas, p0 - - type(element_t), intent(inout), target :: elem(:) ! element array - type(hybrid_t), intent(in) :: hybrid ! hybrid parallel structure - type(hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coordinates - integer, intent(in) :: nets,nete ! start, end element index - real(rl), intent(in) :: time ! current time - integer, intent(in) :: n0,n1 ! time level indices - - logical :: initialized = .false. - - integer, parameter :: zcoords = 0 ! we are not using z coords - logical, parameter :: use_eta = .true. ! we are using hybrid eta coords - real(rl), parameter :: & - T0 = 300.d0, & ! temperature (K) - ztop = 12000.d0, & ! model top (m) - H = Rd * T0 / g ! scale height - - integer :: i,j,k,ie ! loop indices - real(rl):: lon,lat ! pointwise coordiantes - real(rl):: p,z,phis,u,v,w,T,phis_ps,ps,rho,q(4),dp,eta_dot,dp_dn ! pointwise field values - - ! set analytic vertical coordinates at t=0 - if(.not. initialized) then - !$omp barrier - !$omp master - if (hybrid%masterthread) write(iulog,*) 'initializing dcmip2012 test 1-1: 3d deformational flow' - call get_evenly_spaced_p(zi,zm,0.0_rl,ztop,H) ! get evenly spaced p levels - hvcoord%etai = exp(-zi/H) ! set eta levels from z - call set_hybrid_coefficients(hvcoord,hybrid, hvcoord%etai(1),1.0_rl)! set hybrid A and B from eta levels - call set_layer_locations(hvcoord, .true., hybrid%masterthread) - initialized = .true. - !$omp end master - !$omp barrier - endif - - ! set prescribed state at level midpoints - do ie = nets,nete; do k=1,nlev; do j=1,np; do i=1,np - lon = elem(ie)%spherep(i,j)%lon; lat = elem(ie)%spherep(i,j)%lat - z = H * log(1.0d0/hvcoord%etam(k)) - p = p0 * hvcoord%etam(k) - call test1_conv_advection_deformation(time,lon,lat,p,z,zcoords,u,v,w,T,phis,ps,rho,q(1),q(2),q(3),q(4)) - - dp = pressure_thickness(ps,k,hvcoord) - call set_state(u,v,w,T,ps,phis,p,dp,zm(k),g, i,j,k,elem(ie),n0,n1) - if(time==0) call set_tracers(q,qsize,dp,i,j,k,lat,lon,elem(ie)) - - enddo; enddo; enddo; enddo - - ! set prescribed state at level interfaces - do ie = nets,nete; do k=1,nlevp; do j=1,np; do i=1,np - lon = elem(ie)%spherep(i,j)%lon; lat = elem(ie)%spherep(i,j)%lat - z = H * log(1.0d0/hvcoord%etai(k)) - p = p0 * hvcoord%etai(k) - call test1_conv_advection_deformation(time,lon,lat,p,z,zcoords,u,v,w,T,phis,ps,rho,q(1),q(2),q(3),q(4)) - call set_state_i(u,v,w,T,ps,phis,p,zi(k),g, i,j,k,elem(ie),n0,n1) - - ! get vertical derivative of p at point i,j,k - dp_dn = ddn_hyai(k)*p0 + ddn_hybi(k)*ps - - ! get vertical eta velocity at point i,j,k - eta_dot = -g*rho*w/p0 - - ! store vertical mass flux - elem(ie)%derived%eta_dot_dpdn_prescribed(i,j,k) = eta_dot * dp_dn - - enddo; enddo; enddo; enddo - -end subroutine - !_____________________________________________________________________ subroutine dcmip2012_test1_2(elem,hybrid,hvcoord,nets,nete,time,n0,n1) @@ -333,6 +257,105 @@ subroutine dcmip2012_test1_3(elem,hybrid,hvcoord,nets,nete,time,n0,n1,deriv) end subroutine +!_____________________________________________________________________ +subroutine dcmip2012_test1_conv(test_case,elem,hybrid,hvcoord,deriv,nets,nete,time,n0,n1) + + ! 3D tracer transport tests, modified to permit good convergence testing. + + ! Use physical constants consistent with HOMME + use physical_constants, only: Rd => Rgas, p0 + + character(len=*), intent(in) :: test_case + type(element_t), intent(inout), target :: elem(:) ! element array + type(hybrid_t), intent(in) :: hybrid ! hybrid parallel structure + type(hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coordinates + type (derivative_t),intent(in) :: deriv + integer, intent(in) :: nets,nete ! start, end element index + real(rl), intent(in) :: time ! current time + integer, intent(in) :: n0,n1 ! time level indices + + logical :: initialized = .false. + + real(rl), parameter :: & + T0 = 300.d0, & ! temperature (K) + ztop = 12000.d0, & ! model top (m) + H = Rd * T0 / g ! scale height + + integer :: i,j,k,ie ! loop indices + real(rl):: lon,lat,hyai,hyam,hybi,hybm ! pointwise coordiantes + real(rl):: p,z,phis,u,v,w,T,phis_ps,ps,rho,q(5),dp,eta_dot,dp_dn ! pointwise field values + logical :: use_w + real(rl):: grad_p(np,np,2),p_i(np,np),u_i(np,np),v_i(np,np) + + ! set analytic vertical coordinates at t=0 + if (.not. initialized) then + !$omp barrier + !$omp master + if (hybrid%masterthread) then + write(iulog,*) 'initializing dcmip2012 test 3(a-e): & + &modified 3d deformational flows for convergence testing' + end if + call get_evenly_spaced_z(zi,zm,0.0_rl,ztop) ! get evenly spaced z levels + hvcoord%etai = exp(-zi/H) ! set eta levels from z + call set_hybrid_coefficients(hvcoord,hybrid,hvcoord%etai(1),1.0_rl)! set hybrid A and B from eta levels + call set_layer_locations(hvcoord, .true., hybrid%masterthread) + initialized = .true. + !$omp end master + !$omp barrier + endif + + ! set prescribed state at level midpoints + do ie = nets,nete; do k=1,nlev; do j=1,np; do i=1,np + hyam = hvcoord%hyam(k); hybm = hvcoord%hybm(k) + lon = elem(ie)%spherep(i,j)%lon; lat = elem(ie)%spherep(i,j)%lat + z = H * log(1.0d0/hvcoord%etam(k)) + p = p0 * hvcoord%etam(k) + call test1_conv_advection(test_case,time,lon,lat,hyam,hybm,p,z,u,v,w,use_w, & + & T,phis,ps,rho,q) + dp = pressure_thickness(ps,k,hvcoord) + call set_state(u,v,w,T,ps,phis,p,dp,zm(k),g, i,j,k,elem(ie),n0,n1) + if (time==0) call set_tracers(q,qsize,dp,i,j,k,lat,lon,elem(ie)) + enddo; enddo; enddo; enddo + + ! set prescribed state at level interfaces + do ie = nets,nete + do k = 1,nlevp + do j = 1,np + do i = 1,np + hyai = hvcoord%hyai(k); hybi = hvcoord%hybi(k) + lon = elem(ie)%spherep(i,j)%lon; lat = elem(ie)%spherep(i,j)%lat + z = H * log(1.0d0/hvcoord%etai(k)) + p = p0 * hvcoord%etai(k) + call test1_conv_advection(test_case,time,lon,lat,hyai,hybi,p,z,u,v,w,use_w, & + & T,phis,ps,rho,q) + call set_state_i(u,v,w,T,ps,phis,p,zi(k),g,i,j,k,elem(ie),n0,n1) + if (use_w) then + ! get vertical derivative of p at point i,j,k + dp_dn = ddn_hyai(k)*p0 + ddn_hybi(k)*ps + ! get vertical eta velocity at point i,j,k + eta_dot = -g*rho*w/p0 + ! store vertical mass flux + elem(ie)%derived%eta_dot_dpdn_prescribed(i,j,k) = eta_dot * dp_dn + else + p_i(i,j) = p + u_i(i,j) = u + v_i(i,j) = v + end if + enddo + enddo + if (.not. use_w) then + ! get vertical mass flux + grad_p = gradient_sphere(p_i,deriv,elem(ie)%Dinv) + elem(ie)%derived%eta_dot_dpdn_prescribed(:,:,k) = -u_i*grad_p(:,:,1) - v_i*grad_p(:,:,2) + end if + enddo + if (.not. use_w) then + elem(ie)%derived%eta_dot_dpdn_prescribed(:,:,1) = 0 + elem(ie)%derived%eta_dot_dpdn_prescribed(:,:,nlevp) = 0 + end if + enddo +end subroutine dcmip2012_test1_conv + !_____________________________________________________________________ subroutine dcmip2012_test2_0(elem,hybrid,hvcoord,nets,nete) @@ -814,67 +837,18 @@ subroutine set_tracers(q,nq, dp,i,j,k,lat,lon,elem) end subroutine -subroutine dcmip2012_print_test1_conv_results(elem, tl, hvcoord, par, subnum) +subroutine dcmip2012_print_test1_conv_results(test_case, elem, tl, hvcoord, par, subnum) use time_mod, only: timelevel_t use parallel_mod, only: parallel_t - use dimensions_mod, only: nelemd, nlev, qsize - use parallel_mod, only: global_shared_buf, global_shared_sum - use global_norms_mod, only: wrap_repro_sum - use physical_constants, only: Rd => Rgas, p0 + character(len=*), intent(in) :: test_case type(element_t), intent(in) :: elem(:) type(timelevel_t), intent(in) :: tl type(hvcoord_t), intent(in) :: hvcoord type(parallel_t), intent(in) :: par integer, intent(in) :: subnum - integer, parameter :: zcoords = 0 - real(rl), parameter :: & - T0 = 300.d0, & ! temperature (K) - ztop = 12000.d0, & ! model top (m) - H = Rd * T0 / g ! scale height - - real(rl) :: q(np,np,4), lon, lat, z, p, phis, u, v, w, T, phis_ps, ps, rho, time, & - a, b, reldif - integer :: ie, k, iq, i, j - - ! Set time to 0 to get the initial conditions. - time = 0._rl - - do ie = 1,nelemd - global_shared_buf(ie,:2*qsize) = 0._rl - do k = 1,nlev - z = H * log(1.0d0/hvcoord%etam(k)) - p = p0 * hvcoord%etam(k) - do j = 1,np - do i = 1,np - lon = elem(ie)%spherep(i,j)%lon - lat = elem(ie)%spherep(i,j)%lat - select case(subnum) - case (1) - call test1_conv_advection_deformation( & - time,lon,lat,p,z,zcoords,u,v,w,T,phis,ps,rho, & - q(i,j,1),q(i,j,2),q(i,j,3),q(i,j,4)) - end select - end do - end do - do iq = 1,qsize - global_shared_buf(ie,2*iq-1) = global_shared_buf(ie,2*iq-1) + & - sum(elem(ie)%spheremp*(elem(ie)%state%Q(:,:,k,iq) - q(:,:,iq))**2) - global_shared_buf(ie,2*iq) = global_shared_buf(ie,2*iq) + & - sum(elem(ie)%spheremp*q(:,:,iq)**2) - end do - end do - end do - call wrap_repro_sum(nvars=2*qsize, comm=par%comm) - if (par%masterproc) then - do iq = 1,qsize - a = global_shared_sum(2*iq-1) - b = global_shared_sum(2*iq) - reldif = sqrt(a/b) - print '(a,i2,es24.16)', 'test1_conv> Q', iq, reldif - end do - end if + call test1_conv_print_results(test_case, elem, tl, hvcoord, par, subnum) end subroutine dcmip2012_print_test1_conv_results end module dcmip12_wrapper diff --git a/components/homme/src/test_src/dcmip2012_test1_conv.F90 b/components/homme/src/test_src/dcmip2012_test1_conv.F90 deleted file mode 100644 index 274dfcacb58..00000000000 --- a/components/homme/src/test_src/dcmip2012_test1_conv.F90 +++ /dev/null @@ -1,175 +0,0 @@ -module dcmip2012_test1_conv - implicit none - -contains - -SUBROUTINE test1_conv_advection_deformation (time,lon,lat,p,z,zcoords,u,v,w,t,phis,ps,rho,q1,q2,q3,q4) -!----------------------------------------------------------------------- -! input/output params parameters at given location -!----------------------------------------------------------------------- - - ! Use physical constants consistent with HOMME - use physical_constants, only: a=>rearth0, Rd => Rgas, g, cp, pi=>dd_pi, p0 - - real(8), intent(in) :: time ! simulation time (s) - real(8), intent(in) :: lon ! Longitude (radians) - real(8), intent(in) :: lat ! Latitude (radians) - real(8), intent(in) :: z ! Height (m) - real(8), intent(inout) :: p ! Pressure (Pa) - integer, intent(in) :: zcoords ! 0 or 1 see below - real(8), intent(out) :: u ! Zonal wind (m s^-1) - real(8), intent(out) :: v ! Meridional wind (m s^-1) - real(8), intent(out) :: w ! Vertical Velocity (m s^-1) - real(8), intent(out) :: T ! Temperature (K) - real(8), intent(out) :: phis ! Surface Geopotential (m^2 s^-2) - real(8), intent(out) :: ps ! Surface Pressure (Pa) - real(8), intent(out) :: rho ! density (kg m^-3) - real(8), intent(out) :: q1 ! Tracer q1 (kg/kg) - real(8), intent(out) :: q2 ! Tracer q2 (kg/kg) - real(8), intent(out) :: q3 ! Tracer q3 (kg/kg) - real(8), intent(out) :: q4 ! Tracer q4 (kg/kg) - - ! if zcoords = 1, then we use z and output p - ! if zcoords = 0, then we use p - -!----------------------------------------------------------------------- -! test case parameters -!----------------------------------------------------------------------- - real(8), parameter :: & - tau = 12.d0 * 86400.d0, & ! period of motion 12 days - u0 = (2.d0*pi*a)/tau, & ! 2 pi a / 12 days - k0 = (10.d0*a)/tau, & ! velocity magnitude - omega0 = (2*23000.d0*pi)/tau, & ! velocity magnitude - T0 = 300.d0, & ! temperature - H = Rd * T0 / g, & ! scale height - RR = 1.d0/2.d0, & ! horizontal half width divided by 'a' - ZZ = 1000.d0, & ! vertical half width - z0 = 5000.d0, & ! center point in z - lambda0 = 5.d0*pi/6.d0, & ! center point in longitudes - lambda1 = 7.d0*pi/6.d0, & ! center point in longitudes - phi0 = 0.d0, & ! center point in latitudes - phi1 = 0.d0, & - ztop = 12000.d0 - - real(8) :: height ! The height of the model levels - real(8) :: ptop ! model top in p - real(8) :: sin_tmp, cos_tmp, sin_tmp2, cos_tmp2 ! Calculate great circle distances - real(8) :: d1, d2, r, r2, d3, d4 ! For tracer calculations - real(8) :: s, bs, s_p ! Shape function, and parameter - real(8) :: lonp ! Translational longitude, depends on time - real(8) :: ud ! Divergent part of u - real(8) :: x,y,zeta,tmp - - !--------------------------------------------------------------------- - ! HEIGHT AND PRESSURE - !--------------------------------------------------------------------- - - ! height and pressure are aligned (p = p0 exp(-z/H)) - if (zcoords .eq. 1) then - height = z - p = p0 * exp(-z/H) - else - height = H * log(p0/p) - endif - - ! model top in p - ptop = p0*exp(-ztop/H) - - !--------------------------------------------------------------------- - ! THE VELOCITIES ARE TIME DEPENDENT AND THEREFORE MUST BE UPDATED - ! IN THE DYNAMICAL CORE - !--------------------------------------------------------------------- - - ! shape function - bs = 1.0d0 - s = 1.0 + exp((ptop-p0)/(bs*ptop)) - exp((p-p0)/(bs*ptop)) - exp((ptop-p)/(bs*ptop)) - s_p = (-exp((p-p0)/(bs*ptop)) + exp((ptop-p)/(bs*ptop)))/(bs*ptop) - - ! translational longitude - lonp = lon - 2.d0*pi*time/tau - - ! zonal velocity - ud = (omega0*a) * cos(lonp) * (cos(lat)**2.0) * cos(pi*time/tau) * s_p - - u = k0*sin(lonp)*sin(lonp)*sin(2.d0*lat)*cos(pi*time/tau) + u0*cos(lat) + ud - - ! meridional velocity - v = k0*sin(2.d0*lonp)*cos(lat)*cos(pi*time/tau) - - ! vertical velocity - can be changed to vertical pressure velocity by - ! omega = -(g*p)/(Rd*T0)*w - - w = -((Rd*T0)/(g*p))*omega0*sin(lonp)*cos(lat)*cos(pi*time/tau)*s - - !----------------------------------------------------------------------- - ! TEMPERATURE IS CONSTANT 300 K - !----------------------------------------------------------------------- - t = T0 - - !----------------------------------------------------------------------- - ! PHIS (surface geopotential) - !----------------------------------------------------------------------- - phis = 0.d0 - - !----------------------------------------------------------------------- - ! PS (surface pressure) - !----------------------------------------------------------------------- - ps = p0 - - !----------------------------------------------------------------------- - ! RHO (density) - !----------------------------------------------------------------------- - rho = p/(Rd*t) - - !----------------------------------------------------------------------- - ! initialize Q, set to zero - !----------------------------------------------------------------------- - ! q = 0.d0 - - !----------------------------------------------------------------------- - ! initialize tracers - !----------------------------------------------------------------------- - ! tracer 1 - a C^inf tracer field for order of accuracy analysis - - x = cos(lat)*cos(lon) - y = cos(lat)*sin(lon) - zeta = sin(lat) - q1 = 0.3d0*(1.1 + sin(0.25d0*pi*x)*sin(0.3d0*pi*y)*sin(0.25d0*pi*zeta)*sin(pi*(p-ptop)/(p0-ptop))) - - ! tracer 2 - correlated with 1 - q2 = 0.9d0 - 0.8d0*q1**2 - - ! tracer 3 - slotted ellipse - - sin_tmp = sin(lat) * sin(phi0) - cos_tmp = cos(lat) * cos(phi0) - sin_tmp2 = sin(lat) * sin(phi1) - cos_tmp2 = cos(lat) * cos(phi1) - - ! great circle distance without 'a' - r = ACOS (sin_tmp + cos_tmp*cos(lon-lambda0)) - r2 = ACOS (sin_tmp2 + cos_tmp2*cos(lon-lambda1)) - d1 = min( 1.d0, (r/RR)**2 + ((height-z0)/ZZ)**2 ) - d2 = min( 1.d0, (r2/RR)**2 + ((height-z0)/ZZ)**2 ) - - ! make the ellipse - if (d1 .le. RR) then - q3 = 1.d0 - elseif (d2 .le. RR) then - q3 = 1.d0 - else - q3 = 0.1d0 - endif - - ! put in the slot - if (height .gt. z0 .and. abs(lat) .lt. 0.125d0) then - q3 = 0.1d0 - endif - - ! tracer 4: q4 is chosen so that, in combination with the other three tracer - ! fields with weight (3/10), the sum is equal to one - q4 = 1.d0 - 0.3d0*(q1+q2+q3) - -END SUBROUTINE test1_conv_advection_deformation - -end module dcmip2012_test1_conv diff --git a/components/homme/src/test_src/dcmip2012_test1_conv_mod.F90 b/components/homme/src/test_src/dcmip2012_test1_conv_mod.F90 new file mode 100644 index 00000000000..b8805e4486e --- /dev/null +++ b/components/homme/src/test_src/dcmip2012_test1_conv_mod.F90 @@ -0,0 +1,548 @@ +module dcmip2012_test1_conv_mod + + ! Based on DCMIP 2012 tests 1-1,2,3. + + use parallel_mod, only: abortmp + ! Use physical constants consistent with HOMME + use physical_constants, only: a => rearth0, Rd => Rgas, g, cp, pi => dd_pi, p0 + + implicit none + private + + integer, parameter :: rt = 8 + + real(rt), parameter :: & + tau = 12.d0 * 86400.d0, & ! period of motion 12 days + T0 = 300.d0, & ! temperature (K) + ztop = 12000.d0, & ! model top (m) + H = Rd * T0 / g ! scale height + + ! For tracers. + real(rt), parameter :: qlon1 = 5.d0*(pi/6.d0), qlat1 = 0, & + & qlon2 = -qlon1, qlat2 = 0, & + & qsc_min = 0.1d0 + + real(rt), parameter :: zero = 0.d0, one = 1.d0, two = 2.d0 + + public :: test1_conv_advection, test1_conv_print_results + +contains + + subroutine get_nondiv2d_uv(time, lon, lat, u, v) + ! Classic 2D nondivergent flow field. + + real(rt), intent(in ) :: time, lon, lat + real(rt), intent(out) :: u, v + + real(rt), parameter :: & + u0 = (2.d0*pi*a)/tau, & ! 2 pi a / 12 days + k0 = (10.d0*a)/tau ! velocity magnitude + + real(rt) :: lonp + + ! translational longitude + lonp = lon - 2.d0*pi*time/tau + ! zonal velocity + u = k0*sin(lonp)*sin(lonp)*sin(2.d0*lat)*cos(pi*time/tau) + u0*cos(lat) + ! meridional velocity + v = k0*sin(2.d0*lonp)*cos(lat)*cos(pi*time/tau) + end subroutine get_nondiv2d_uv + + subroutine get_nondiv3d_uv(bs, pbot, ptop, zbot, ztop, ztaper, time, lon, lat, p, z, u, v, w) + real(rt), intent(in ) :: bs, pbot, ptop, zbot, ztop, ztaper, time, lon, lat, p, z + real(rt), intent(out) :: u, v, w + + real(rt), parameter :: omega0 = (2*23000.d0*pi)/tau + + real(rt) :: s, s_p, lonp, ud, c, arg + + ! This is essentially the test 1-1 flow. The key difference in this flow + ! removes the factor of 2 in ud and w cos-time factors. The 2 in the + ! original code makes trajectories not return to their initial points. + + ! Shape function in p. + if (p >= pbot .or. p <= ptop) then + s = 0 + s_p = 0 + else + c = 0.3d0 + arg = pi*(p - ptop)/(pbot - ptop) + s = c*sin(arg)**3 + s_p = (3*c*pi/(pbot - ptop))*sin(arg)**2*cos(arg) + end if + ! Translational longitude. + lonp = lon - 2.d0*pi*time/tau + ! Nondivergent 2D flow. + call get_nondiv2d_uv(time, lon, lat, u, v) + ! Taper the 2D nondiv (u,v) flow in the z direction. This does not induce + ! any w, and the 2D field remains nondivergent at each z. + u = u*ztaper + v = v*ztaper + ! Divergent flow. + ud = (omega0*a)*cos(lonp)*(cos(lat)**2.0)*cos(pi*time/tau)*s_p + u = u + ud + w = -((Rd*T0)/(g*p))*omega0*sin(lonp)*cos(lat)*cos(pi*time/tau)*s + end subroutine get_nondiv3d_uv + + function get_2d_cinf_tracer(lon, lat) result(q) + real(rt), intent(in) :: lon, lat + + real(rt) :: q + + real(rt) :: x, y, zeta + + x = cos(lat)*cos(lon) + y = cos(lat)*sin(lon) + zeta = sin(lat) + q = 1.5d0*(1 + sin(pi*x)*sin(pi*y)*sin(pi*zeta)) + end function get_2d_cinf_tracer + + subroutine ll2xyz(lon, lat, x, y, z) + ! Unit sphere. + + real(rt), intent(in) :: lon, lat + real(rt), intent(out) :: x, y, z + + real(rt) :: sinl, cosl + + sinl = sin(lat) + cosl = cos(lat) + x = cos(lon)*cosl + y = sin(lon)*cosl + z = sinl + end subroutine ll2xyz + + function great_circle_dist(lon1, lat1, lon2, lat2) result(d) + ! Unit sphere. + + real(rt), intent(in) :: lon1, lat1, lon2, lat2 + real(rt) :: d + + real(rt) xA, yA, zA, xB, yB, zB, cp1, cp2, cp3, cpnorm, dotprod + + call ll2xyz(lon1, lat1, xA, yA, zA) + call ll2xyz(lon2, lat2, xB, yB, zB) + cp1 = yA*zB - yB*zA + cp2 = xB*zA - xA*zB + cp3 = xA*yB - xB*yA + cpnorm = sqrt(cp1*cp1 + cp2*cp2 + cp3*cp3) + dotprod = xA*xB + yA*yB + zA*zB + d = atan2(cpnorm, dotprod) + end function great_circle_dist + + function q_gh(x, y, z, xi, yi, zi) result(q) + real(rt), intent(in) :: x, y, z, xi, yi, zi + real(rt) :: q + + real(rt), parameter :: h_max = 0.95d0, b = 5.d0 + real(rt) :: r2 + + r2 = (x - xi)**2 + (y - yi)**2 + (z - zi)**2 + q = h_max*exp(-b*r2) + end function q_gh + + function q_cb(r, ri) result(q) + real(rt), intent(in) :: r, ri + real(rt) :: q + + real(rt), parameter :: h_max = one + + q = 0.5d0*h_max*(1 + cos(pi*ri/r)) + end function q_cb + + function q_sc(clon_in, clat, lon, lat, up_slot) result(q) + real(rt), intent(in) :: clon_in, clat, lon, lat + logical, intent(in) :: up_slot + real(rt) :: q + + real(rt), parameter :: b = qsc_min, c = one, r = 0.5d0, & + & lon_thr = r/6.d0, lat_thr = 5*(r/12.d0) + + real(rt) :: clon, ri + + clon = clon_in + if (clon < zero) clon = clon + two*pi + + ri = great_circle_dist(lon, lat, clon, clat) + q = b + if (ri <= r) then + if (abs(lon - clon) >= lon_thr) then + q = c + return + else + if (up_slot) then + if (lat - clat < -lat_thr) then + q = c + return + end if + else + if (lat - clat > lat_thr) then + q = c + return + end if + end if + end if + end if + end function q_sc + + function get_2d_gaussian_hills(lon, lat) result(q) + real(rt), intent(in) :: lon, lat + real(rt) :: q + + real(rt) :: x1, y1, z1, x2, y2, z2, x, y, z + + call ll2xyz(qlon1, qlat1, x1, y1, z1) + call ll2xyz(qlon2, qlat2, x2, y2, z2) + call ll2xyz(lon, lat, x, y, z) + q = q_gh(x, y, z, x1, y1, z1) + q_gh(x, y, z, x2, y2, z2) + end function get_2d_gaussian_hills + + function get_2d_cosine_bells(lon, lat) result(q) + real(rt), intent(in) :: lon, lat + real(rt) :: q + + real(rt), parameter :: r = 0.5d0, b = 0.1d0, c = 0.9d0 + real(rt) :: h, ri + + h = 0 + ri = great_circle_dist(lon, lat, qlon1, qlat1) + if (ri < r) then + h = q_cb(r, ri) + else + ri = great_circle_dist(lon, lat, qlon2, qlat2) + if (ri < r) h = q_cb(r, ri) + end if + q = b + c*h + end function get_2d_cosine_bells + + function get_2d_correlated_cosine_bells(lon, lat) result(q) + real(rt), intent(in) :: lon, lat + real(rt) :: q + + real(rt), parameter :: a = -0.8d0, b = 0.9d0 + + q = get_2d_cosine_bells(lon, lat) + q = a*q + b + end function get_2d_correlated_cosine_bells + + function get_2d_slotted_cylinders(lon, lat) result(q) + real(rt), intent(in) :: lon, lat + real(rt) :: q + + q = q_sc(qlon1, qlat1, lon, lat, .true.) + if (q < 0.5d0) q = q_sc(qlon2, qlat2, lon, lat, .false.) + end function get_2d_slotted_cylinders + + subroutine test1_conv_advection_orography( & + test_minor,time,lon,lat,p,z,zcoords,cfv,hybrid_eta,hya,hyb,u,v,w,t,phis,ps,rho,q1,q2,q3,q4) + + character(len=1), intent(in) :: test_minor ! a, b, c, d, or e + real(rt), intent(in) :: time ! simulation time (s) + real(rt), intent(in) :: lon ! Longitude (radians) + real(rt), intent(in) :: lat ! Latitude (radians) + real(rt), intent(in) :: hya ! A coefficient for hybrid-eta coordinate + real(rt), intent(in) :: hyb ! B coefficient for hybrid-eta coordinate + + logical, intent(in) :: hybrid_eta ! flag to indicate whether the hybrid sigma-p (eta) coordinate is used + + real(rt), intent(out) :: p ! Pressure (Pa) + real(rt), intent(out) :: z ! Height (m) + + integer , intent(in) :: zcoords ! 0 or 1 see below + integer , intent(in) :: cfv ! 0, 1 or 2 see below + real(rt), intent(out) :: u ! Zonal wind (m s^-1) + real(rt), intent(out) :: v ! Meridional wind (m s^-1) + real(rt), intent(out) :: w ! Vertical Velocity (m s^-1) + real(rt), intent(out) :: t ! Temperature (K) + real(rt), intent(out) :: phis ! Surface Geopotential (m^2 s^-2) + real(rt), intent(out) :: ps ! Surface Pressure (Pa) + real(rt), intent(out) :: rho ! density (kg m^-3) + real(rt), intent(out) :: q1 ! Tracer q1 (kg/kg) + real(rt), intent(out) :: q2 ! Tracer q2 (kg/kg) + real(rt), intent(out) :: q3 ! Tracer q3 (kg/kg) + real(rt), intent(out) :: q4 ! Tracer q4 (kg/kg) + + real(rt), parameter :: & + u0 = 2.d0*pi*a/tau, & ! Velocity Magnitude (m/s) + alpha = pi/6.d0, & ! rotation angle (radians), 30 degrees + lambdam = 3.d0*pi/2.d0, & ! mountain longitude center point (radians) + phim = zero, & ! mountain latitude center point (radians) + h0 = 2000.d0, & ! peak height of the mountain range (m) + Rm = 3.d0*pi/4.d0, & ! mountain radius (radians) + ztop_t = 2000.d0, & ! transition layer + zbot_q = ztop_t + 500.d0, & ! bottom of tracers; below, all q = 0 + lon_offset = 0.5d0*pi, & ! longitudinal translation of std 2d test flow and qs + ! For Hadley-like flow. Multiply w and tracer vertical extent by (ztop + ! - ztop_t)/ztop to compensate for smaller domain. + tau_h = 86400.d0, & ! period of motion 1 day (in s) + z1_h = ztop_t + 1000.d0, & ! position of lower tracer bound (m) + z2_h = z1_h + 6000.d0, & ! position of upper tracer bound (m) + z0_h = 0.5d0*(z1_h+z2_h), & ! midpoint (m) + u0_h = 250.d0, & ! Zonal velocity magnitude (m/s) + ! w0_h is the main parameter to modify to make the test easier (smaller + ! w0_h) or harder (larger). + w0_h = 0.05d0, & ! Vertical velocity magnitude (m/s) + ! For 3D deformational flow. + bs_a = 1.0d0 ! shape function smoothness + + real(rt) :: r, height, zs, zetam, ztaper, rho0, z_q_shape, ptop, ptop_t, & + & c0, fl, fl_lat, gz, gz_z, fz, fz_z, delta, lambdam_t, u_topo_fac, & + & u0_topo, tau_topo + logical :: ps_timedep + + if (cfv /= 0) call abortmp('test1_conv_advection_orography does not support cfv != 0') + if (.not. hybrid_eta) call abortmp('test1_conv_advection_orography does not support !hybrid_eta') + if (zcoords /= 0) call abortmp('test1_conv_advection_orography does not support zcoords != 0') + + ! Mountain oscillation half-width (radians). + zetam = pi/14.d0 + ! Smooth mountains for very less resource-intensive convergence testing. + if (test_minor == 'c') zetam = pi/2.d0 + ! Smoother than default but still fairly rough. + if (test_minor == 'd' .or. test_minor == 'f') zetam = pi/6.d0 + + ps_timedep = test_minor == 'e' .or. test_minor == 'f' + lambdam_t = lambdam + if (ps_timedep) then + ! Move the topography to make ps depend on time. + u0_topo = u0 + tau_topo = tau + if (test_minor == 'e') then + u0_topo = u0_h + tau_topo = tau_h + end if + u_topo_fac = -u0_topo/two + lambdam_t = lambdam_t + & + & sin(pi*time/tau_topo)*(tau_topo/pi)*u_topo_fac & ! integral of u at lat = 0 + & /a ! to radians + end if + r = great_circle_dist(lambdam_t, phim, lon, lat) + if (r .lt. Rm) then + zs = (h0/2.d0)*(one+cos(pi*r/Rm))*cos(pi*r/zetam)**2.d0 + else + zs = zero + endif + if (test_minor == 'a') zs = zero + zs = -zs ! holes instead of mountains + phis = g*zs + ps = p0 * exp(-zs/H) + + p = hya*p0 + hyb*ps + height = H * log(p0/p) + z = height + + T = T0 + + rho = p/(Rd*T) + rho0 = p0/(Rd*T) + + if (z <= 0) then + ztaper = 0 + elseif (z >= ztop_t) then + ztaper = 1 + else + ztaper = (1 + cos(pi*(1 + z/ztop_t)))/2 + end if + + w = zero + + select case(test_minor) + case('z') ! currently unused + ! Solid body rotation + ! Zonal Velocity + u = u0*(cos(lat)*cos(alpha)+sin(lat)*cos(lon)*sin(alpha)) + ! Meridional Velocity + v = -u0*(sin(lon)*sin(alpha)) + u = u*ztaper + v = v*ztaper + case('b') + ! 2D nondiv flow in each layer. + call get_nondiv2d_uv(time, lon + lon_offset, lat, u, v) + u = u*ztaper + v = v*ztaper + case('a', 'c', 'd', 'f') + ! 3D nondiv flow. + ptop_t = p0*exp(-ztop_t/H) + ptop = p0*exp(-ztop/H) + call get_nondiv3d_uv(bs_a, ptop_t, ptop, ztop_t, ztop, ztaper, & + & time, lon + lon_offset, lat, p, z, u, v, w) + case('e') + ! Similar to Hadley-like flow but with more smoothness in derivatives. + u = u0_h*cos(lat)*cos(pi*time/tau_h)*ztaper + fl = cos(lat)**2 + fl_lat = -2*cos(lat)*sin(lat) + if (z <= 0) then + fz = 0 + fz_z = 0 + else + gz = pi*z/ztop + gz_z = pi/ztop + fz = -sin(gz)**3 + fz_z = -3*sin(gz)**2*cos(gz)*gz_z + end if + c0 = w0_h*(rho0/rho)*cos(pi*time/tau_h) + w = c0*(cos(lat)*fl_lat - 2*sin(lat)*fl)*fz + v = -a*c0*(cos(lat)*fl )*fz_z + case default + call abortmp('test1_conv_advection_orography: invalid case') + end select + + if (ps_timedep) then + ! Low-level solid-body rotational wind for consistency with the moving ps + ! field. + u = u + cos(pi*time/tau_topo)*u_topo_fac*(1 - ztaper)*cos(lat) + end if + + if (time > 0) then + q1 = 0; q2 = 0; q3 = 0; q4 = 0 + return + end if + + z_q_shape = 0.5d0*(1 - cos(2*pi*(z - zbot_q)/(ztop - zbot_q))) + if (z < zbot_q .or. z > ztop) z_q_shape = zero + + select case(test_minor) + case('e') + if (height < z2_h .and. height > z1_h) then + q1 = 0.5d0 * (one + cos(2.d0*pi*(z-z0_h)/(z2_h-z1_h))) + else + q1 = zero + end if + q2 = q1 * get_2d_cinf_tracer(lon, lat) + q3 = q1 * get_2d_gaussian_hills(lon - lon_offset, lat) + q4 = q1 * get_2d_cosine_bells(lon - lon_offset, lat) + + case default + q1 = z_q_shape * get_2d_gaussian_hills(lon - lon_offset, lat) + q2 = z_q_shape * get_2d_cosine_bells(lon - lon_offset, lat) + q4 = z_q_shape * get_2d_correlated_cosine_bells(lon - lon_offset, lat) + ! Tracer discontinuous in 3D. + q3 = qsc_min + delta = z2_h - z1_h + if ( (z >= z1_h .and. z <= z1_h + 0.25d0*delta) .or. & + (z >= z1_h + 0.4d0 *delta .and. z <= z2_h - 0.4d0 *delta) .or. & + (z <= z2_h .and. z >= z2_h - 0.25d0*delta)) then + q3 = get_2d_slotted_cylinders(lon - lon_offset, lat) + end if + end select + end subroutine test1_conv_advection_orography + + subroutine test1_conv_advection(test_case,time,lon,lat,hya,hyb,p,z,u,v,w,use_w,t,phis,ps,rho,q) + character(len=*), intent(in) :: test_case ! dcmip2012_test1_{3a-f}_conv + real(rt), intent(in) :: time ! simulation time (s) + real(rt), intent(in) :: lon, lat ! Longitude, latitude (radians) + real(rt), intent(in) :: hya, hyb ! Hybrid a, b coefficients + real(rt), intent(inout) :: z ! Height (m) + real(rt), intent(inout) :: p ! Pressure (Pa) + real(rt), intent(out) :: u ! Zonal wind (m s^-1) + real(rt), intent(out) :: v ! Meridional wind (m s^-1) + real(rt), intent(out) :: w ! Vertical Velocity (m s^-1) + logical , intent(out) :: use_w ! Should caller use w or instead div(u,v)? + real(rt), intent(out) :: T ! Temperature (K) + real(rt), intent(out) :: phis ! Surface Geopotential (m^2 s^-2) + real(rt), intent(out) :: ps ! Surface Pressure (Pa) + real(rt), intent(out) :: rho ! density (kg m^-3) + real(rt), intent(out) :: q(5) ! Tracer q1 (kg/kg) + + integer, parameter :: cfv = 0, zcoords = 0 + logical, parameter :: use_eta = .true. + + character(len=1) :: test_major, test_minor + + test_major = test_case(17:17) + if (test_major == '3') test_minor = test_case(18:18) + + use_w = .false. + select case(test_major) + case('3') + call test1_conv_advection_orography( & + test_minor,time,lon,lat,p,z,zcoords,cfv,use_eta,hya,hyb,u,v,w,t,phis,ps,rho, & + q(1),q(2),q(3),q(4)) + end select + end subroutine test1_conv_advection + + subroutine test1_conv_print_results(test_case, elem, tl, hvcoord, par, subnum) + use element_mod, only: element_t + use time_mod, only: timelevel_t + use hybvcoord_mod, only: hvcoord_t + use parallel_mod, only: parallel_t, pmax_1d + use dimensions_mod, only: nelemd, nlev, qsize, np + use parallel_mod, only: global_shared_buf, global_shared_sum + use global_norms_mod, only: wrap_repro_sum + use physical_constants, only: Rd => Rgas, p0 + + character(len=*), intent(in) :: test_case + type(element_t), intent(in) :: elem(:) + type(timelevel_t), intent(in) :: tl + type(hvcoord_t), intent(in) :: hvcoord + type(parallel_t), intent(in) :: par + integer, intent(in) :: subnum + + real(rt) :: q(np,np,5), lon, lat, z, p, phis, u, v, w, T, phis_ps, ps, rho, time, & + hya, hyb, a, b, reldif, linf_num(qsize), linf_den(qsize) + integer :: ie, k, iq, i, j + logical :: use_w + + ! Set time to 0 to get the initial conditions. + time = 0._rt + + linf_num = 0 + linf_den = 0 + do ie = 1,nelemd + global_shared_buf(ie,:2*qsize) = 0._rt + do k = 1,nlev + ! test1_conv_advection_orography uses these: + hya = hvcoord%hyam(k) + hyb = hvcoord%hybm(k) + ! test1_advection_deformation uses these, in which ps = p0: + p = p0 * hvcoord%etam(k) + z = H * log(1.0d0/hvcoord%etam(k)) + + ! Normwise relative errors. We weight the horizontal direction by + ! sphereme but do not weight the vertical direction; each vertical + ! level in a column has equal weight. + + do j = 1,np + do i = 1,np + lon = elem(ie)%spherep(i,j)%lon + lat = elem(ie)%spherep(i,j)%lat + select case(subnum) + case (1) + call test1_conv_advection( & + test_case,time,lon,lat,hya,hyb,p,z,u,v,w,use_w,T,phis,ps,rho,q(i,j,:)) + end select + end do + end do + + do iq = 1,qsize + global_shared_buf(ie,2*iq-1) = global_shared_buf(ie,2*iq-1) + & + sum(elem(ie)%spheremp*(elem(ie)%state%Q(:,:,k,iq) - q(:,:,iq))**2) + global_shared_buf(ie,2*iq) = global_shared_buf(ie,2*iq) + & + sum(elem(ie)%spheremp*q(:,:,iq)**2) + linf_num(iq) = max(linf_num(iq), & + maxval(abs(elem(ie)%state%Q(:,:,k,iq) - q(:,:,iq)))) + linf_den(iq) = max(linf_den(iq), & + maxval(abs(q(:,:,iq)))) + end do + end do + end do + + call wrap_repro_sum(nvars=2*qsize, comm=par%comm) + do iq = 1, qsize + linf_num(iq) = pmax_1d(linf_num(iq:iq), par) + linf_den(iq) = pmax_1d(linf_den(iq:iq), par) + end do + + if (par%masterproc) then + print '(a)', 'test1_conv> l2 linf' + do iq = 1,qsize + a = global_shared_sum(2*iq-1) + b = global_shared_sum(2*iq) + reldif = sqrt(a/b) + print '(a,i2,es24.16,es24.16)', 'test1_conv> Q', & + iq, reldif, linf_num(iq)/linf_den(iq) + end do + end if + end subroutine test1_conv_print_results + +end module dcmip2012_test1_conv_mod diff --git a/components/homme/src/theta-l/share/prim_advection_mod.F90 b/components/homme/src/theta-l/share/prim_advection_mod.F90 index f5941519adb..e8c6b68f0f9 100644 --- a/components/homme/src/theta-l/share/prim_advection_mod.F90 +++ b/components/homme/src/theta-l/share/prim_advection_mod.F90 @@ -13,7 +13,8 @@ module prim_advection_mod use time_mod, only : TimeLevel_t use hybrid_mod, only : hybrid_t use control_mod, only : transport_alg - use sl_advection, only : prim_advec_tracers_remap_ALE, sl_init1 + use sl_advection, only : prim_advec_tracers_observe_velocity_ale, & + prim_advec_tracers_remap_ALE, sl_init1 use prim_advection_base, only: prim_advec_init2, prim_advec_init1_rk2, & prim_advec_tracers_remap_rk2 @@ -40,6 +41,16 @@ subroutine Prim_Advec_Init1(par, elem) end subroutine Prim_Advec_Init1 + subroutine Prim_Advec_Tracers_observe_velocity(elem, tl, n, nets, nete) + type (element_t) , intent(inout) :: elem(:) + type (TimeLevel_t) , intent(in ) :: tl + integer , intent(in ) :: n ! step in 1:dt_tracer_factor + integer , intent(in ) :: nets + integer , intent(in ) :: nete + + if (transport_alg /= 0) call Prim_Advec_Tracers_observe_velocity_ALE(elem, tl, n, nets, nete) + end subroutine Prim_Advec_Tracers_observe_velocity + subroutine Prim_Advec_Tracers_remap( elem , deriv , hvcoord , hybrid , dt , tl , nets , nete ) implicit none type (element_t) , intent(inout) :: elem(:) diff --git a/components/homme/src/theta-l_kokkos/CMakeLists.txt b/components/homme/src/theta-l_kokkos/CMakeLists.txt index a585090ea76..191e3821cdd 100644 --- a/components/homme/src/theta-l_kokkos/CMakeLists.txt +++ b/components/homme/src/theta-l_kokkos/CMakeLists.txt @@ -119,7 +119,7 @@ MACRO(THETAL_KOKKOS_SETUP) ${TEST_SRC_DIR}/baroclinic_inst_mod.F90 ${TEST_SRC_DIR}/dcmip12_wrapper.F90 ${TEST_SRC_DIR}/dcmip16_wrapper.F90 - ${TEST_SRC_DIR}/dcmip2012_test1_conv.F90 + ${TEST_SRC_DIR}/dcmip2012_test1_conv_mod.F90 ${TEST_SRC_DIR}/dcmip2012_test1_2_3.F90 ${TEST_SRC_DIR}/dcmip2012_test4.F90 ${TEST_SRC_DIR}/dcmip2016-baroclinic.F90 @@ -161,6 +161,8 @@ MACRO(THETAL_KOKKOS_SETUP) ${SRC_SHARE_DIR}/cxx/ComposeTransport.cpp ${SRC_SHARE_DIR}/cxx/ComposeTransportImplGeneral.cpp ${SRC_SHARE_DIR}/cxx/ComposeTransportImplTrajectory.cpp + ${SRC_SHARE_DIR}/cxx/ComposeTransportImplEnhancedTrajectory.cpp + ${SRC_SHARE_DIR}/cxx/ComposeTransportImplEnhancedTrajectoryTests.cpp ${SRC_SHARE_DIR}/cxx/ComposeTransportImplVerticalRemap.cpp ${SRC_SHARE_DIR}/cxx/ComposeTransportImplHypervis.cpp ${SRC_SHARE_DIR}/cxx/ComposeTransportImplTest2D.cpp diff --git a/components/homme/src/theta-l_kokkos/config.h.cmake.in b/components/homme/src/theta-l_kokkos/config.h.cmake.in index 7e378c1b795..5215d7bc641 100644 --- a/components/homme/src/theta-l_kokkos/config.h.cmake.in +++ b/components/homme/src/theta-l_kokkos/config.h.cmake.in @@ -77,3 +77,7 @@ /* Detect whether COMPOSE passive tracer transport is enabled */ #cmakedefine HOMME_ENABLE_COMPOSE + +/* For just-in-time compilation (e.g., SYCL compilers), disable timers at the */ +/* first prim_run level when nstep == 1. */ +#cmakedefine DISABLE_TIMERS_IN_FIRST_STEP diff --git a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp index e96b838f096..40c4ae64dc9 100644 --- a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp @@ -114,7 +114,6 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, params.hypervis_scaling = hypervis_scaling; params.disable_diagnostics = (bool)disable_diagnostics; params.use_moisture = (bool)use_moisture; - params.moisture = params.use_moisture ? MoistDry::MOIST : MoistDry::DRY; //todo-repo-unification params.use_cpstar = (bool)use_cpstar; params.transport_alg = transport_alg; params.theta_hydrostatic_mode = (bool)theta_hydrostatic_mode; diff --git a/components/homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp b/components/homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp index f0c0c1c728a..8f7fc13d315 100644 --- a/components/homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp @@ -26,6 +26,9 @@ void ttype7_imex_timestep (const TimeLevel& tl, const Real dt, const Real eta_av void ttype9_imex_timestep (const TimeLevel& tl, const Real dt, const Real eta_ave_w); void ttype10_imex_timestep(const TimeLevel& tl, const Real dt, const Real eta_ave_w); +// Prescribed-wind F90-C++ bridge. Test inputs are all implemented in F90. +extern "C" void set_prescribed_wind_f_bridge(int n0, int np1, int nstep, Real dt); + // -------------- IMPLEMENTATIONS -------------- // void prim_advance_exp (TimeLevel& tl, const Real dt, const bool compute_diagnostics) @@ -72,10 +75,12 @@ void prim_advance_exp (TimeLevel& tl, const Real dt, const bool compute_diagnost } #if !defined(CAM) && !defined(SCREAM) - // If prescribed wind, the dynamics was set explicitly in - // prim_driver_mod::prim_run_subcycle; skip time-integration. - if (params.prescribed_wind) + // If prescribed wind, set the dynamics explicitly and skip time-integration. + if (params.prescribed_wind) { + set_prescribed_wind_f_bridge(tl.n0, tl.np1, tl.nstep, dt); + GPTLstop("tl-ae prim_advance_exp"); return; + } #endif switch (params.time_step_type) { diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index e7cc245a2bd..4e49c5c06ab 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -11,7 +11,11 @@ module prim_driver_mod use prim_driver_base, only : deriv1, smooth_topo_datasets use prim_cxx_driver_base, only : prim_init1, prim_finalize use physical_constants, only : scale_factor, laplacian_rigid_factor - + use hybrid_mod, only : hybrid_t + use hybvcoord_mod, only : hvcoord_t + use derivative_mod, only : derivative_t + use time_mod, only : timelevel_t + implicit none public :: prim_init2 @@ -23,10 +27,19 @@ module prim_driver_mod public :: prim_init_ref_states_views public :: prim_init_diags_views + type, private :: PrescribedWind_t + type (element_t), pointer :: elem(:) + type (hybrid_t) :: hybrid + type (hvcoord_t) :: hvcoord + type (derivative_t) :: deriv + integer :: nets, nete + end type PrescribedWind_t + + type (PrescribedWind_t), private :: prescribed_wind_args + contains subroutine prim_init2(elem, hybrid, nets, nete, tl, hvcoord) - use hybrid_mod, only : hybrid_t use hybvcoord_mod, only : hvcoord_t use time_mod, only : timelevel_t use prim_driver_base, only : deriv1, prim_init2_base => prim_init2 @@ -46,10 +59,6 @@ subroutine prim_init2(elem, hybrid, nets, nete, tl, hvcoord) ! Call the base version of prim_init2 call prim_init2_base(elem,hybrid,nets,nete,tl,hvcoord) - if (prescribed_wind == 1) then - call init_standalone_test(elem,deriv1,hybrid,hvcoord,tl,nets,nete) - end if - ! Init the c data structures call prim_create_c_data_structures(tl,hvcoord,elem(1)%mp) @@ -61,6 +70,10 @@ subroutine prim_init2(elem, hybrid, nets, nete, tl, hvcoord) ! Initialize dp3d from ps_v call initialize_dp3d_from_ps_c () + + if (prescribed_wind == 1) then + call init_standalone_test(elem,deriv1,hybrid,hvcoord,tl,nets,nete) + end if end subroutine prim_init2 subroutine prim_create_c_data_structures (tl, hvcoord, mp) @@ -353,14 +366,12 @@ subroutine prim_init_elements_views (elem) end subroutine prim_init_elements_views subroutine prim_init_kokkos_functors (allocate_buffer) - !todo-repo-unification Remove the use of c_bool here. It's used in purely - ! F90 code. - use iso_c_binding, only : c_int, c_bool + use iso_c_binding, only : c_int use theta_f2c_mod, only : init_functors_c, init_boundary_exchanges_c ! ! Optional Input ! - logical(kind=c_bool), intent(in), optional :: allocate_buffer ! Whether functor memory buffer should be allocated internally + logical, intent(in), optional :: allocate_buffer ! Whether functor memory buffer should be allocated internally integer(kind=c_int) :: ab ! Initialize the C++ functors in the C++ context ! If no argument allocate_buffer is present, @@ -461,8 +472,8 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv elem_derived_FPHI, elem_derived_FQ) call t_stopf('push_to_cxx') end if - if (prescribed_wind == 1) then ! standalone Homme - call set_prescribed_wind_f(elem,deriv1,hybrid,hvcoord,dt,tl,nets,nete) + if (prescribed_wind == 1) then + call init_prescribed_wind_subcycle(elem,nets,nete,tl) end if call prim_run_subcycle_c(dt,nstep_c,nm1_c,n0_c,np1_c,nextOutputStep,nsplit_iteration) @@ -616,7 +627,7 @@ subroutine init_standalone_test(elem,deriv,hybrid,hvcoord,tl,nets,nete) use element_mod, only : element_t use derivative_mod, only : derivative_t #if !defined(CAM) && !defined(SCREAM) - use test_mod, only : set_prescribed_wind + use test_mod, only : set_test_initial_conditions #endif type (element_t), intent(inout), target :: elem(:) @@ -628,11 +639,18 @@ subroutine init_standalone_test(elem,deriv,hybrid,hvcoord,tl,nets,nete) integer , intent(in) :: nete #if !defined(CAM) && !defined(SCREAM) - real(kind=real_kind) :: dt, eta_ave_w - - dt = 0 ! value unused in initialization - eta_ave_w = 0 ! same - call set_prescribed_wind(elem,deriv,hybrid,hvcoord,dt,tl,nets,nete,eta_ave_w) + ! Already called in prim_driver_base::prim_init2: + ! call set_test_initial_conditions(elem,deriv,hybrid,hvcoord,tl,nets,nete) + ! Also already taken care of: + ! call push_test_state_to_c_wrapper() + + ! Save arguments for the C++-F90 bridge for prescribed winds. + prescribed_wind_args%elem => elem + prescribed_wind_args%hybrid = hybrid + prescribed_wind_args%hvcoord = hvcoord + prescribed_wind_args%deriv = deriv + prescribed_wind_args%nets = nets + prescribed_wind_args%nete = nete #endif end subroutine init_standalone_test @@ -657,8 +675,77 @@ subroutine compute_test_forcing_f(elem,hybrid,hvcoord,nt,ntQ,dt,nets,nete,tl) #endif end subroutine compute_test_forcing_f + subroutine push_test_state_to_c_wrapper() +#if !defined(CAM) && !defined(SCREAM) + use iso_c_binding, only : c_ptr, c_loc + use perf_mod, only : t_startf, t_stopf + use theta_f2c_mod, only : push_test_state_to_c + use element_state, only : elem_state_v, elem_state_w_i, elem_state_vtheta_dp, & + elem_state_phinh_i, elem_state_dp3d, elem_state_ps_v, & + elem_derived_eta_dot_dpdn, elem_derived_vn0 + + type (c_ptr) :: elem_state_v_ptr, elem_state_w_i_ptr, elem_state_vtheta_dp_ptr, elem_state_phinh_i_ptr + type (c_ptr) :: elem_state_dp3d_ptr, elem_state_Qdp_ptr, elem_state_Q_ptr, elem_state_ps_v_ptr + type (c_ptr) :: elem_derived_eta_dot_dpdn_ptr, elem_derived_vn0_ptr + + call t_startf('push_to_cxx') + elem_state_v_ptr = c_loc(elem_state_v) + elem_state_w_i_ptr = c_loc(elem_state_w_i) + elem_state_vtheta_dp_ptr = c_loc(elem_state_vtheta_dp) + elem_state_phinh_i_ptr = c_loc(elem_state_phinh_i) + elem_state_dp3d_ptr = c_loc(elem_state_dp3d) + elem_state_ps_v_ptr = c_loc(elem_state_ps_v) + elem_derived_vn0_ptr = c_loc(elem_derived_vn0) + elem_derived_eta_dot_dpdn_ptr = c_loc(elem_derived_eta_dot_dpdn) + call push_test_state_to_c(elem_state_ps_v_ptr, elem_state_dp3d_ptr, & + elem_state_vtheta_dp_ptr, elem_state_phinh_i_ptr, elem_state_v_ptr, & + elem_state_w_i_ptr, elem_derived_eta_dot_dpdn_ptr, elem_derived_vn0_ptr) + call t_stopf('push_to_cxx') +#endif + end subroutine push_test_state_to_c_wrapper + + subroutine init_prescribed_wind_subcycle(elem, nets, nete, tl) + ! Set the derived values used in tracer transport on the F90 side even + ! though most of the work is done on the C++ side. This is needed because + ! set_prescribed_wind accumulates certain derived quantities during + ! prim_advance_exp that get repeatedly copied from F90 to C++. Here we + ! initialize values for accumulation. + ! In summary: Call this before entering the prim_run_subcycle loop. + + use prim_driver_base, only: set_tracer_transport_derived_values + + type (element_t), intent(inout) :: elem(:) + integer, intent(in) :: nets, nete + type (timelevel_t) :: tl + + call set_tracer_transport_derived_values(elem, nets, nete, tl) + end subroutine init_prescribed_wind_subcycle + + subroutine set_prescribed_wind_f_bridge(n0, np1, nstep, dt) bind(c) + ! This routine is called from the C++ prim_advance_exp implementation inside + ! the prim_run_subcycle loop. + + use iso_c_binding, only: c_int, c_double + + integer(c_int), value, intent(in) :: n0, np1, nstep + real(c_double), value, intent(in) :: dt + + type (TimeLevel_t) :: tl + + ! Only these fields need to be valid. + tl%n0 = n0+1 + tl%np1 = np1+1 + tl%nstep = nstep + + call set_prescribed_wind_f(prescribed_wind_args%elem, prescribed_wind_args%deriv, & + prescribed_wind_args%hybrid, prescribed_wind_args%hvcoord, dt, tl, & + prescribed_wind_args%nets, prescribed_wind_args%nete) + end subroutine set_prescribed_wind_f_bridge + subroutine set_prescribed_wind_f(elem,deriv,hybrid,hvcoord,dt,tl,nets,nete) - use iso_c_binding, only : c_ptr, c_loc + ! Here we finally can compute the prescribed wind in F90 and then push the + ! data to C++. + use hybrid_mod, only : hybrid_t use hybvcoord_mod, only : hvcoord_t use time_mod, only : timelevel_t @@ -666,12 +753,7 @@ subroutine set_prescribed_wind_f(elem,deriv,hybrid,hvcoord,dt,tl,nets,nete) use derivative_mod, only : derivative_t #if !defined(CAM) && !defined(SCREAM) use control_mod, only : qsplit - use perf_mod, only : t_startf, t_stopf - use theta_f2c_mod, only : push_test_state_to_c use test_mod, only : set_prescribed_wind - use element_state, only : elem_state_v, elem_state_w_i, elem_state_vtheta_dp, & - elem_state_phinh_i, elem_state_dp3d, elem_state_ps_v, & - elem_derived_eta_dot_dpdn, elem_derived_vn0 #endif type (element_t), intent(inout), target :: elem(:) @@ -685,39 +767,17 @@ subroutine set_prescribed_wind_f(elem,deriv,hybrid,hvcoord,dt,tl,nets,nete) #if !defined(CAM) && !defined(SCREAM) type (hvcoord_t) :: hv - type (c_ptr) :: elem_state_v_ptr, elem_state_w_i_ptr, elem_state_vtheta_dp_ptr, elem_state_phinh_i_ptr - type (c_ptr) :: elem_state_dp3d_ptr, elem_state_Qdp_ptr, elem_state_Q_ptr, elem_state_ps_v_ptr - type (c_ptr) :: elem_derived_eta_dot_dpdn_ptr, elem_derived_vn0_ptr real(kind=real_kind) :: eta_ave_w ! We need to set up an hvcoord_t that can be passed as intent(inout), even ! though at this point, it won't be changed in the set_prescribed_wind call. - hv%ps0 = hvcoord%ps0 - hv%hyai = hvcoord%hyai - hv%hyam = hvcoord%hyam - hv%hybi = hvcoord%hybi - hv%hybm = hvcoord%hybm - hv%etam = hvcoord%etam - hv%etai = hvcoord%etai - hv%dp0 = hvcoord%dp0 + hv = hvcoord eta_ave_w = 1d0/qsplit call set_prescribed_wind(elem,deriv,hybrid,hv,dt,tl,nets,nete,eta_ave_w) - call t_startf('push_to_cxx') - elem_state_v_ptr = c_loc(elem_state_v) - elem_state_w_i_ptr = c_loc(elem_state_w_i) - elem_state_vtheta_dp_ptr = c_loc(elem_state_vtheta_dp) - elem_state_phinh_i_ptr = c_loc(elem_state_phinh_i) - elem_state_dp3d_ptr = c_loc(elem_state_dp3d) - elem_state_ps_v_ptr = c_loc(elem_state_ps_v) - elem_derived_vn0_ptr = c_loc(elem_derived_vn0) - elem_derived_eta_dot_dpdn_ptr = c_loc(elem_derived_eta_dot_dpdn) - call push_test_state_to_c(elem_state_ps_v_ptr, elem_state_dp3d_ptr, & - elem_state_vtheta_dp_ptr, elem_state_phinh_i_ptr, elem_state_v_ptr, & - elem_state_w_i_ptr, elem_derived_eta_dot_dpdn_ptr, elem_derived_vn0_ptr) - call t_stopf('push_to_cxx') + call push_test_state_to_c_wrapper() #endif end subroutine set_prescribed_wind_f diff --git a/components/homme/test/reg_test/namelists/thetah-sl-test11conv-r0t1-cdr30-rrm.nl b/components/homme/test/reg_test/namelists/thetah-sl-test11conv-r0t1-cdr30-rrm.nl index 564ea8be17b..5b7e987bd0b 100644 --- a/components/homme/test/reg_test/namelists/thetah-sl-test11conv-r0t1-cdr30-rrm.nl +++ b/components/homme/test/reg_test/namelists/thetah-sl-test11conv-r0t1-cdr30-rrm.nl @@ -2,7 +2,7 @@ nthreads = -1 ! use OMP_NUM_THREADS partmethod = 4 ! mesh parition method: 4 = space filling curve topology = "cube" ! mesh type: cubed sphere - test_case = "dcmip2012_test1_1_conv" ! test identifier + test_case = "dcmip2012_test1_3a_conv" ! test identifier prescribed_wind = 1 mesh_file = 'mountain_10_x2.g' qsize = 4 ! num tracer fields diff --git a/components/homme/test/reg_test/namelists/thetah-sl-test11conv-r1t2-cdr20.nl b/components/homme/test/reg_test/namelists/thetah-sl-test11conv-r1t2-cdr20.nl index 484d3612dd0..c6a7bfb2b1c 100644 --- a/components/homme/test/reg_test/namelists/thetah-sl-test11conv-r1t2-cdr20.nl +++ b/components/homme/test/reg_test/namelists/thetah-sl-test11conv-r1t2-cdr20.nl @@ -2,7 +2,7 @@ nthreads = -1 ! use OMP_NUM_THREADS partmethod = 4 ! mesh parition method: 4 = space filling curve topology = "cube" ! mesh type: cubed sphere - test_case = "dcmip2012_test1_1_conv" ! test identifier + test_case = "dcmip2012_test1_3a_conv" ! test identifier prescribed_wind = 1 ne = 5 ! number of elements per cube face qsize = 4 ! num tracer fields diff --git a/components/homme/test/reg_test/namelists/thetah-sl-testconv-3e.nl b/components/homme/test/reg_test/namelists/thetah-sl-testconv-3e.nl new file mode 100644 index 00000000000..d140d689ac0 --- /dev/null +++ b/components/homme/test/reg_test/namelists/thetah-sl-testconv-3e.nl @@ -0,0 +1,62 @@ +&ctl_nl + nthreads = -1 + partmethod = 4 + topology = "cube" + test_case = 'dcmip2012_test1_3e_conv' + prescribed_wind = 1 + qsize = 4 + ndays = 1 + statefreq = 240 + restartfreq = -1 + runtype = 0 + ne = 20 + integration = 'explicit' + tstep_type = 1 + smooth = 0 + nu = 1.585e13 ! nu values are irrelevant + nu_s = 1.585e13 + nu_p = 42 ! to satisfy kokkos exe + se_ftype = -1 + limiter_option = 9 + hypervis_order = 2 + hypervis_subcycle = 1 + moisture = 'dry' + theta_hydrostatic_mode = .true. + dcmip16_prec_type = 1 + dcmip16_pbl_type = -1 + transport_alg = 12 + semi_lagrange_cdr_alg = 3 + semi_lagrange_cdr_check = .false. + semi_lagrange_hv_q = 0 + semi_lagrange_nearest_point_lev = 0 + semi_lagrange_halo = 2 + dt_remap_factor = 0 + dt_tracer_factor = 4 + tstep = 200.0 + semi_lagrange_trajectory_nsubstep = 2 + semi_lagrange_trajectory_nvelocity = 3 + semi_lagrange_diagnostics = 1 + hypervis_subcycle_q = 0 + limiter_option = 9 + vert_remap_q_alg = 10 +/ +&vert_nl + vanalytic = 1 + vtop = 0.2549944 +/ +&analysis_nl + output_dir = "./movies/" + output_timeunits = 2, ! 1=days, 2=hours, 0=timesteps + output_frequency = 2, + output_varnames1 = 'ps','Q','u','v' + interp_type = 0 + output_type = 'netcdf' + num_io_procs = 16 + interp_nlon = 180 + interp_nlat = 91 + interp_gridtype = 2 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/test/reg_test/run_tests/test-list.cmake b/components/homme/test/reg_test/run_tests/test-list.cmake index dbb0fc4f98e..884a785d843 100644 --- a/components/homme/test/reg_test/run_tests/test-list.cmake +++ b/components/homme/test/reg_test/run_tests/test-list.cmake @@ -47,7 +47,7 @@ IF (HOMME_ENABLE_COMPOSE) thetah-sl-test11conv-r1t2-cdr20.cmake thetah-sl-test11conv-r0t1-cdr30-rrm.cmake thetah-sl-dcmip16_test1pg2.cmake - ) + thetah-sl-testconv-3e.cmake) ENDIF() SET(HOMME_RUN_TESTS_DIR ${HOMME_SOURCE_DIR}/test/reg_test/run_tests) @@ -92,7 +92,9 @@ ENDIF() IF (BUILD_HOMME_THETA_KOKKOS) # Various one-off tests. IF (HOMME_ENABLE_COMPOSE) - LIST(APPEND HOMME_TESTS thetah-sl-test11conv-r0t1-cdr30-rrm-kokkos.cmake) + LIST(APPEND HOMME_TESTS + thetah-sl-test11conv-r0t1-cdr30-rrm-kokkos.cmake + thetah-sl-testconv-3e-kokkos.cmake) IF (HOMMEXX_BFB_TESTING) LIST(APPEND HOMME_ONEOFF_CVF_TESTS thetah-sl-test11conv-r0t1-cdr30-rrm) diff --git a/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r0t1-cdr30-rrm-kokkos.cmake b/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r0t1-cdr30-rrm-kokkos.cmake index 0bf617bcdb7..fb7e41a3d04 100644 --- a/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r0t1-cdr30-rrm-kokkos.cmake +++ b/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r0t1-cdr30-rrm-kokkos.cmake @@ -11,4 +11,4 @@ SET(MESH_FILES ${HOMME_ROOT}/test/mesh_files/mountain_10_x2.g) # compare all of these files against baselines: SET(NC_OUTPUT_FILES - dcmip2012_test1_1_conv1.nc) + dcmip2012_test1_3a_conv1.nc) diff --git a/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r0t1-cdr30-rrm.cmake b/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r0t1-cdr30-rrm.cmake index 62387e9474b..688b946ed59 100644 --- a/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r0t1-cdr30-rrm.cmake +++ b/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r0t1-cdr30-rrm.cmake @@ -11,4 +11,4 @@ SET(MESH_FILES ${HOMME_ROOT}/test/mesh_files/mountain_10_x2.g) # compare all of these files against baselines: SET(NC_OUTPUT_FILES - dcmip2012_test1_1_conv1.nc) + dcmip2012_test1_3a_conv1.nc) diff --git a/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r1t2-cdr20.cmake b/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r1t2-cdr20.cmake index 8dacd43d872..ca0245833b6 100644 --- a/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r1t2-cdr20.cmake +++ b/components/homme/test/reg_test/run_tests/thetah-sl-test11conv-r1t2-cdr20.cmake @@ -9,4 +9,4 @@ SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/thetah-sl-test11conv-r1 # compare all of these files against baselines: SET(NC_OUTPUT_FILES - dcmip2012_test1_1_conv1.nc) + dcmip2012_test1_3a_conv1.nc) diff --git a/components/homme/test/reg_test/run_tests/thetah-sl-testconv-3e-kokkos.cmake b/components/homme/test/reg_test/run_tests/thetah-sl-testconv-3e-kokkos.cmake new file mode 100644 index 00000000000..4d87d00d655 --- /dev/null +++ b/components/homme/test/reg_test/run_tests/thetah-sl-testconv-3e-kokkos.cmake @@ -0,0 +1,11 @@ +# The name of this test (should be the basename of this file) +SET(TEST_NAME thetah-sl-testconv-3e-kokkos) +# The specifically compiled executable that this test uses +SET(EXEC_NAME theta-l-nlev30-kokkos) + +SET(NUM_CPUS 16) + +SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/thetah-sl-testconv-3e.nl) + +# compare all of these files against baselines: +SET(NC_OUTPUT_FILES dcmip2012_test1_3e_conv1.nc) diff --git a/components/homme/test/reg_test/run_tests/thetah-sl-testconv-3e.cmake b/components/homme/test/reg_test/run_tests/thetah-sl-testconv-3e.cmake new file mode 100644 index 00000000000..6eaff42f94d --- /dev/null +++ b/components/homme/test/reg_test/run_tests/thetah-sl-testconv-3e.cmake @@ -0,0 +1,11 @@ +# The name of this test (should be the basename of this file) +SET(TEST_NAME thetah-sl-testconv-3e) +# The specifically compiled executable that this test uses +SET(EXEC_NAME theta-l-nlev30) + +SET(NUM_CPUS 16) + +SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/thetah-sl-testconv-3e.nl) + +# compare all of these files against baselines: +SET(NC_OUTPUT_FILES dcmip2012_test1_3e_conv1.nc) diff --git a/components/homme/test/unit_tests/tester.cpp b/components/homme/test/unit_tests/tester.cpp index 826257930e8..0acce65a41d 100644 --- a/components/homme/test/unit_tests/tester.cpp +++ b/components/homme/test/unit_tests/tester.cpp @@ -25,13 +25,13 @@ int main(int argc, char **argv) { Homme::initialize_hommexx_session(); // Filter arguments so catch2 doesn't try to interpret hommexx-specific ones. - hommexx_catch2_argc = argc; + hommexx_catch2_argc = 0; hommexx_catch2_argv = argv; for (int i = 1; i < argc; ++i) { if (std::string(argv[i]) == "hommexx") { - argc = i; - hommexx_catch2_argc -= i + 1; + hommexx_catch2_argc = argc - (i + 1); hommexx_catch2_argv = argv + i + 1; + argc = i; break; } } diff --git a/components/homme/test_execs/CMakeLists.txt b/components/homme/test_execs/CMakeLists.txt index a007a5532b6..12a55d64025 100644 --- a/components/homme/test_execs/CMakeLists.txt +++ b/components/homme/test_execs/CMakeLists.txt @@ -195,9 +195,6 @@ IF(${BUILD_HOMME_PREQX}) IF(${BUILD_HOMME_PREQX_ACC}) ADD_SUBDIRECTORY(baroCam-acc) ENDIF() - IF(${HOMME_ENABLE_COMPOSE}) - ADD_SUBDIRECTORY(stt) - ENDIF() ENDIF() # Add the test exec subdirs for the prim executable @@ -222,6 +219,8 @@ IF(${BUILD_HOMME_THETA}) # ADD_SUBDIRECTORY(theta-l-nlev200-native) # ADD_SUBDIRECTORY(theta-l-nlev256-native) # ADD_SUBDIRECTORY(theta-l-nlev300-native) + # Special test for -DHOMME_WITHOUT_PIOLIBRARY. + ADD_SUBDIRECTORY(stt) ENDIF() IF (${BUILD_HOMME_THETA_KOKKOS}) diff --git a/components/homme/test_execs/stt/CMakeLists.txt b/components/homme/test_execs/stt/CMakeLists.txt index c0268957730..93b083ca1c5 100644 --- a/components/homme/test_execs/stt/CMakeLists.txt +++ b/components/homme/test_execs/stt/CMakeLists.txt @@ -1,7 +1,7 @@ -preqx_setup() +thetal_setup() ADD_DEFINITIONS(-DHOMME_WITHOUT_PIOLIBRARY) # Set the variables for this test executable # NP NC PLEV USE_PIO WITH_ENERGY QSIZE_D -createTestExec(stt preqx 4 4 3 FALSE TRUE 5) +createTestExec(stt theta-l 4 4 3 FALSE TRUE 5) diff --git a/components/homme/test_execs/thetal_kokkos_ut/compose_interface.F90 b/components/homme/test_execs/thetal_kokkos_ut/compose_interface.F90 index 4296b6609a0..a34d5411e95 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/compose_interface.F90 +++ b/components/homme/test_execs/thetal_kokkos_ut/compose_interface.F90 @@ -8,7 +8,7 @@ module compose_interface contains subroutine init_compose_f90(ne, hyai, hybi, hyam, hybm, ps0, dvv, mp, qsize_in, hv_q, & - lim, cdr_check, is_sphere) bind(c) + lim, cdr_check, is_sphere, nearest_point, halo, traj_nsubstep) bind(c) use hybvcoord_mod, only: set_layer_locations use thetal_test_interface, only: init_f90 use theta_f2c_mod, only: init_elements_c @@ -16,7 +16,8 @@ subroutine init_compose_f90(ne, hyai, hybi, hyam, hybm, ps0, dvv, mp, qsize_in, use control_mod, only: transport_alg, semi_lagrange_cdr_alg, semi_lagrange_cdr_check, & semi_lagrange_hv_q, limiter_option, nu_q, hypervis_subcycle_q, hypervis_order, & vert_remap_q_alg, qsplit, rsplit, dt_remap_factor, dt_tracer_factor, & - theta_hydrostatic_mode + theta_hydrostatic_mode, semi_lagrange_nearest_point_lev, semi_lagrange_halo, & + semi_lagrange_trajectory_nsubstep use geometry_interface_mod, only: GridVertex use bndry_mod, only: sort_neighbor_buffer_mapping use reduction_mod, only: initreductionbuffer, red_sum, red_min, red_max @@ -25,17 +26,17 @@ subroutine init_compose_f90(ne, hyai, hybi, hyam, hybm, ps0, dvv, mp, qsize_in, use sl_advection, only: sl_init1 real (real_kind), intent(in) :: hyai(nlevp), hybi(nlevp), hyam(nlev), hybm(nlev) - integer (c_int), value, intent(in) :: ne, qsize_in, hv_q, lim + integer (c_int), value, intent(in) :: ne, qsize_in, hv_q, lim, halo, traj_nsubstep real (real_kind), value, intent(in) :: ps0 real (real_kind), intent(out) :: dvv(np,np), mp(np,np) - logical (c_bool), value, intent(in) :: cdr_check, is_sphere + logical (c_bool), value, intent(in) :: cdr_check, is_sphere, nearest_point integer :: ie, edgesz if (.not. is_sphere) print *, "NOT IMPL'ED YET" transport_alg = 12 - semi_lagrange_cdr_alg = 30 + semi_lagrange_cdr_alg = 3 semi_lagrange_cdr_check = cdr_check qsize = qsize_in limiter_option = lim @@ -45,6 +46,10 @@ subroutine init_compose_f90(ne, hyai, hybi, hyam, hybm, ps0, dvv, mp, qsize_in, dt_tracer_factor = -1 dt_remap_factor = -1 theta_hydrostatic_mode = .true. + semi_lagrange_nearest_point_lev = -1 + if (nearest_point) semi_lagrange_nearest_point_lev = 100000 + semi_lagrange_halo = halo + semi_lagrange_trajectory_nsubstep = traj_nsubstep hypervis_order = 2 semi_lagrange_hv_q = hv_q @@ -148,7 +153,7 @@ subroutine run_compose_standalone_test_f90(nmax_out, eval) bind(c) use thread_mod, only: hthreads, vthreads use dimensions_mod, only: nlev, qsize - integer(c_int), intent(out) :: nmax_out + integer(c_int), intent(inout) :: nmax_out real(c_double), intent(out) :: eval((nlev+1)*qsize) type (domain1d_t), pointer :: dom_mt(:) @@ -162,8 +167,12 @@ subroutine run_compose_standalone_test_f90(nmax_out, eval) bind(c) dom_mt(0)%start = 1 dom_mt(0)%end = nelemd transport_alg = 19 - nmax = 7*ne - nmax_out = nmax + if (nmax_out <= 1) then + nmax = 7*ne + nmax_out = nmax + else + nmax = nmax_out + end if statefreq = 2*ne call compose_test(par, hvcoord, dom_mt, elem, buf) do i = 1,size(buf) @@ -190,7 +199,7 @@ subroutine run_trajectory_f90(t0, t1, independent_time_steps, dep, dprecon) bind type (timelevel_t) :: tl type (hybrid_t) :: hybrid real(real_kind) :: dt - integer :: ie, i, j, k, testno, geometry_type + integer :: ie, i, j, k, d, testno, geometry_type logical :: its call timelevel_init_default(tl) @@ -216,9 +225,9 @@ subroutine run_trajectory_f90(t0, t1, independent_time_steps, dep, dprecon) bind do k = 1,nlev do j = 1,np do i = 1,np - dep(1,i,j,k,ie) = dep_points_all(i,j,k,ie)%x - dep(2,i,j,k,ie) = dep_points_all(i,j,k,ie)%y - dep(3,i,j,k,ie) = dep_points_all(i,j,k,ie)%z + do d = 1, 3 + dep(d,i,j,k,ie) = dep_points_all(d,i,j,k,ie) + end do dprecon(i,j,k,ie) = elem(ie)%derived%divdp(i,j,k) end do end do diff --git a/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp index 0c0d06cff67..85a30056766 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp @@ -37,7 +37,8 @@ extern char** hommexx_catch2_argv; extern "C" { void init_compose_f90(int ne, const Real* hyai, const Real* hybi, const Real* hyam, const Real* hybm, Real ps0, Real* dvv, Real* mp, int qsize, - int hv_q, int limiter_option, bool cdr_check, bool is_sphere); + int hv_q, int limiter_option, bool cdr_check, bool is_sphere, + bool nearest_point, int halo, int traj_nsubstep); void init_geometry_f90(); void cleanup_compose_f90(); void run_compose_standalone_test_f90(int* nmax, Real* eval); @@ -99,8 +100,8 @@ void fill (Random& r, const V& a, } struct Session { - int ne, hv_q; - bool cdr_check, is_sphere; + int ne, hv_q, nmax, halo, traj_nsubstep; + bool cdr_check, is_sphere, run_only_advection_test, nearest_point; HybridVCoord h; Random r; std::shared_ptr e; @@ -113,6 +114,11 @@ struct Session { const auto seed = r.gen_seed(); printf("seed %u\n", seed); + nlev = NUM_PHYSICAL_LEV; + assert(nlev > 0); + np = NP; + assert(np == 4); + assert(QSIZE_D >= 4); parse_command_line(); assert(is_sphere); // planar isn't available in Hxx yet @@ -141,11 +147,12 @@ struct Session { const auto hybi = cmvdc(h.hybrid_bi); const auto hyam = cmvdc(h.hybrid_am); const auto hybm = cmvdc(h.hybrid_bm); + auto& ref_FE = c.create(); std::vector dvv(NP*NP), mp(NP*NP); init_compose_f90(ne, hyai.data(), hybi.data(), &hyam(0)[0], &hybm(0)[0], h.ps0, dvv.data(), mp.data(), qsize, hv_q, p.limiter_option, cdr_check, - is_sphere); + is_sphere, nearest_point, halo, traj_nsubstep); ref_FE.init_mass(mp.data()); ref_FE.init_deriv(dvv.data()); @@ -168,11 +175,6 @@ struct Session { ct.init_buffers(fbm); ct.init_boundary_exchanges(); - nlev = NUM_PHYSICAL_LEV; - assert(nlev > 0); - np = NP; - assert(np == 4); - c.create(); } @@ -203,37 +205,87 @@ struct Session { // compose_ut hommexx -ne NE -qsize QSIZE -hvq HV_Q -cdrcheck void parse_command_line () { - const bool am_root = get_comm().root(); ne = 2; qsize = QSIZE_D; hv_q = 1; cdr_check = false; is_sphere = true; + run_only_advection_test = false; + nmax = -1; + halo = 2; + traj_nsubstep = 0; + nearest_point = true; + + const bool am_root = get_comm().root(); bool ok = true; int i; for (i = 0; i < hommexx_catch2_argc; ++i) { const std::string tok(hommexx_catch2_argv[i]); if (tok == "-ne") { - if (i+1 == hommexx_catch2_argc) { ok = false; break; } + if (i+1 == hommexx_catch2_argc) ok = false; ne = std::atoi(hommexx_catch2_argv[++i]); + if (ne < 2) { + printf("ne must be >= 2\n"); + ok = false; + } } else if (tok == "-qsize") { - if (i+1 == hommexx_catch2_argc) { ok = false; break; } + if (i+1 == hommexx_catch2_argc) ok = false; qsize = std::atoi(hommexx_catch2_argv[++i]); + if (qsize > QSIZE_D || qsize < 1) { + printf("qsize must be >= 1 and <= QSIZE_D\n"); + ok = false; + } } else if (tok == "-hvq") { - if (i+1 == hommexx_catch2_argc) { ok = false; break; } + if (i+1 == hommexx_catch2_argc) ok = false; hv_q = std::atoi(hommexx_catch2_argv[++i]); } else if (tok == "-cdrcheck") { cdr_check = true; } else if (tok == "-planar") { is_sphere = false; + } else if (tok == "-convergence") { + // When running this as a convergence-test driver, don't run any tests + // except the prescribed-flow one. + run_only_advection_test = true; + } else if (tok == "-nmax") { + if (i+1 == hommexx_catch2_argc) ok = false; + nmax = std::atoi(hommexx_catch2_argv[++i]); + if (nmax < 1) { + printf("nmax must be >= 1\n"); + ok = false; + } + } else if (tok == "-halo") { + if (i+1 == hommexx_catch2_argc) ok = false; + halo = std::atoi(hommexx_catch2_argv[++i]); + if (halo < 1) { + printf("halo must be >= 1"); + ok = false; + } + } else if (tok == "-trajnsubstep") { + if (i+1 == hommexx_catch2_argc) ok = false; + traj_nsubstep = std::atoi(hommexx_catch2_argv[++i]); + if (traj_nsubstep < 0) { + printf("traj_nsubstep must be >= 0\n"); + ok = false; + } + } else if (tok == "-nonearest") { + nearest_point = false; + } else { + printf("unrecognized token %s\n", tok.c_str()); + ok = false; } + if ( ! ok) break; } - ne = std::max(2, std::min(128, ne)); + + ne = std::max(2, ne); qsize = std::max(1, std::min(QSIZE_D, qsize)); hv_q = std::max(0, std::min(qsize, hv_q)); - if ( ! ok && am_root) + + if ( ! ok && am_root) { printf("compose_ut> Failed to parse command line, starting with: %s\n", hommexx_catch2_argv[i]); + Homme::Errors::runtime_abort("compose_ut invalid command line"); + } + if (am_root) { const int bfb = #ifdef HOMMEXX_BFB_TESTING @@ -241,8 +293,10 @@ struct Session { #else 0; #endif - printf("compose_ut> bfb %d ne %d qsize %d hv_q %d cdr_check %d\n", - bfb, ne, qsize, hv_q, cdr_check ? 1 : 0); + printf("compose_ut> sphere %d bfb %d ne %d qsize %d hv_q %d cdr_check %d " + "halo %d traj_nsubstep %d nearest %d\n", + int(is_sphere), bfb, ne, qsize, hv_q, cdr_check ? 1 : 0, halo, + traj_nsubstep, int(nearest_point)); } } }; @@ -337,10 +391,19 @@ TEST_CASE ("compose_transport_testing") { static constexpr Real tol = std::numeric_limits::epsilon(); auto& s = Session::singleton(); try { + do { // breakable + + if (s.run_only_advection_test) { + int nmax = s.nmax; + std::vector eval_f((s.nlev+1)*s.qsize); + run_compose_standalone_test_f90(&nmax, eval_f.data()); + break; + } // unit tests REQUIRE(compose::test::slmm_unittest() == 0); REQUIRE(compose::test::cedr_unittest() == 0); + REQUIRE(compose::test::interpolate_unittest() == 0); REQUIRE(compose::test::cedr_unittest(s.get_comm().mpi_comm()) == 0); auto& ct = Context::singleton().get(); @@ -349,33 +412,35 @@ TEST_CASE ("compose_transport_testing") { REQUIRE(fails.empty()); // trajectory BFB - for (const bool independent_time_steps : {false, true}) { - printf("independent_time_steps %d\n", independent_time_steps); - const Real twelve_days = 3600 * 24 * 12; - const Real t0 = 0.13*twelve_days; - const Real t1 = independent_time_steps ? t0 + 1800 : 0.22*twelve_days; - CA5d depf("depf", s.nelemd, s.nlev, s.np, s.np, 3); - CA4d dpreconf("dpreconf", s.nelemd, s.nlev, s.np, s.np); - run_trajectory_f90(t0, t1, independent_time_steps, depf.data(), - dpreconf.data()); - const auto depc = ct.test_trajectory(t0, t1, independent_time_steps); - REQUIRE(depc.extent_int(0) == s.nelemd); - REQUIRE(depc.extent_int(2) == s.np); - REQUIRE(depc.extent_int(4) == 3); - if (independent_time_steps) { - const auto dpreconc = cmvdc(RNlev(pack2real(s.e->m_derived.m_divdp), s.nelemd)); + if (s.traj_nsubstep == 0) { + for (const bool independent_time_steps : {false, true}) { + printf("independent_time_steps %d\n", independent_time_steps); + const Real twelve_days = 3600 * 24 * 12; + const Real t0 = 0.13*twelve_days; + const Real t1 = independent_time_steps ? t0 + 1800 : 0.22*twelve_days; + CA5d depf("depf", s.nelemd, s.nlev, s.np, s.np, 3); + CA4d dpreconf("dpreconf", s.nelemd, s.nlev, s.np, s.np); + run_trajectory_f90(t0, t1, independent_time_steps, depf.data(), + dpreconf.data()); + const auto depc = ct.test_trajectory(t0, t1, independent_time_steps); + REQUIRE(depc.extent_int(0) == s.nelemd); + REQUIRE(depc.extent_int(2) == s.np); + REQUIRE(depc.extent_int(4) == 3); + if (independent_time_steps) { + const auto dpreconc = cmvdc(RNlev(pack2real(s.e->m_derived.m_divdp), s.nelemd)); + for (int ie = 0; ie < s.nelemd; ++ie) + for (int lev = 0; lev < s.nlev; ++lev) + for (int i = 0; i < s.np; ++i) + for (int j = 0; j < s.np; ++j) + REQUIRE(equal(dpreconf(ie,lev,i,j), dpreconc(ie,i,j,lev), 100*tol)); + } for (int ie = 0; ie < s.nelemd; ++ie) for (int lev = 0; lev < s.nlev; ++lev) for (int i = 0; i < s.np; ++i) for (int j = 0; j < s.np; ++j) - REQUIRE(equal(dpreconf(ie,lev,i,j), dpreconc(ie,i,j,lev), 10*tol)); + for (int d = 0; d < 3; ++d) + REQUIRE(equal(depf(ie,lev,i,j,d), depc(ie,lev,i,j,d), 100*tol)); } - for (int ie = 0; ie < s.nelemd; ++ie) - for (int lev = 0; lev < s.nlev; ++lev) - for (int i = 0; i < s.np; ++i) - for (int j = 0; j < s.np; ++j) - for (int d = 0; d < 3; ++d) - REQUIRE(equal(depf(ie,lev,i,j,d), depc(ie,lev,i,j,d), 10*tol)); } { // q vertical remap @@ -386,7 +451,7 @@ TEST_CASE ("compose_transport_testing") { } { // 2D SL BFB - int nmax; + int nmax = s.nmax; std::vector eval_f((s.nlev+1)*s.qsize), eval_c(eval_f.size()); run_compose_standalone_test_f90(&nmax, eval_f.data()); for (const bool bfb : {false, true}) { @@ -397,9 +462,10 @@ TEST_CASE ("compose_transport_testing") { if (s.get_comm().root()) { const Real f = bfb ? 0 : 1; const int n = s.nlev*s.qsize; - // When not a BFB build, still expect l2 error to be the same to a few digits. - for (int i = 0; i < n; ++i) REQUIRE(almost_equal(eval_f[i], eval_c[i], f*1e-3)); - // Mass conservation error should be within a factor of 10 of each other. + // When not a BFB build, still expect l2 error to be the same to several digits. + for (int i = 0; i < n; ++i) REQUIRE(almost_equal(eval_f[i], eval_c[i], f*1e5*tol)); + // Mass conservation error should be within a factor of 10 of each + // other. for (int i = n; i < n + s.qsize; ++i) REQUIRE(almost_equal(eval_f[i], eval_c[i], f*10)); // And mass conservation itself should be small. for (int i = n; i < n + s.qsize; ++i) REQUIRE(std::abs(eval_f[i]) <= 20*tol); @@ -409,6 +475,7 @@ TEST_CASE ("compose_transport_testing") { } } + } while (false); // do } catch (...) {} Session::delete_singleton(); } diff --git a/components/mpas-albany-landice/bld/build-namelist b/components/mpas-albany-landice/bld/build-namelist index f08d94a0e27..1f230c7543e 100755 --- a/components/mpas-albany-landice/bld/build-namelist +++ b/components/mpas-albany-landice/bld/build-namelist @@ -559,6 +559,10 @@ add_default($nl, 'config_alpha_subglacial_discharge'); add_default($nl, 'config_subglacial_discharge_coefficient'); add_default($nl, 'config_subglacial_discharge_intercept'); add_default($nl, 'config_uniform_face_melt_rate'); +add_default($nl, 'config_ocean_data_extrapolation'); +add_default($nl, 'config_ocean_data_extrap_ncells_extra'); +add_default($nl, 'config_invalid_value_TF'); +add_default($nl, 'config_weight_value_cell'); ####################################### # Namelist group: physical_parameters # diff --git a/components/mpas-albany-landice/bld/build-namelist-section b/components/mpas-albany-landice/bld/build-namelist-section index 735dca93c19..14c0732bc2c 100644 --- a/components/mpas-albany-landice/bld/build-namelist-section +++ b/components/mpas-albany-landice/bld/build-namelist-section @@ -121,6 +121,10 @@ add_default($nl, 'config_alpha_subglacial_discharge'); add_default($nl, 'config_subglacial_discharge_coefficient'); add_default($nl, 'config_subglacial_discharge_intercept'); add_default($nl, 'config_uniform_face_melt_rate'); +add_default($nl, 'config_ocean_data_extrapolation'); +add_default($nl, 'config_ocean_data_extrap_ncells_extra'); +add_default($nl, 'config_invalid_value_TF'); +add_default($nl, 'config_weight_value_cell'); ####################################### # Namelist group: physical_parameters # diff --git a/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml b/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml index 3e0b5625835..b437bcdfc83 100644 --- a/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml +++ b/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml @@ -98,7 +98,7 @@ 1.0 0.0 0.25 -'none' +'ismip6' .false. 1.18 0.0 @@ -106,6 +106,10 @@ 3.0e-4 0.15 0.0 +.false. +10 +1.0e36 +0.9 910.0 diff --git a/components/mpas-albany-landice/bld/namelist_files/namelist_definition_mali.xml b/components/mpas-albany-landice/bld/namelist_files/namelist_definition_mali.xml index c75c1e163aa..3ea4ef5e478 100644 --- a/components/mpas-albany-landice/bld/namelist_files/namelist_definition_mali.xml +++ b/components/mpas-albany-landice/bld/namelist_files/namelist_definition_mali.xml @@ -795,6 +795,38 @@ Valid values: any non-negative value Default: Defined in namelist_defaults.xml + +If true, extrapolate ocean data (temperature, salinity, thermal forcing) from external source into underneath the ice draft. + +Valid values: .true. or .false. +Default: Defined in namelist_defaults.xml + + + +number of extra cells for over-extrapolation into grounded ice + +Valid values: any non-negative value +Default: Defined in namelist_defaults.xml + + + +value assigned to indicate invalid thermal forcing value when config_ocean_data_extrapolation is set to true + +Valid values: Any real value +Default: Defined in namelist_defaults.xml + + + +weight used to smooth horizontal extrapolation of ocean data field. Value close to 1 implies more weight of the current cell value versus the averaged value of the neighbouring cells around the cell + +Valid values: any real value between 0 and 1 +Default: Defined in namelist_defaults.xml + + diff --git a/components/mpas-albany-landice/cime_config/buildnml b/components/mpas-albany-landice/cime_config/buildnml index 3b38ed39c43..d0fd5551e81 100755 --- a/components/mpas-albany-landice/cime_config/buildnml +++ b/components/mpas-albany-landice/cime_config/buildnml @@ -88,7 +88,7 @@ def buildnml(case, caseroot, compname): decomp_date += '051920' decomp_prefix += 'mpasli.graph.info.' elif glc_grid == 'mpas.gis1to10kmR2': - grid_date += '20230202' + grid_date += '20240513' grid_prefix += 'gis_1to10km_r02' decomp_date += '020223' decomp_prefix += 'mpasli.graph.info.' @@ -247,6 +247,9 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') diff --git a/components/mpas-albany-landice/driver/glc_comp_mct.F b/components/mpas-albany-landice/driver/glc_comp_mct.F index a3eb5d17924..57a940552c7 100644 --- a/components/mpas-albany-landice/driver/glc_comp_mct.F +++ b/components/mpas-albany-landice/driver/glc_comp_mct.F @@ -80,6 +80,8 @@ module glc_comp_mct integer :: glcLogUnit ! unit number for glc log + logical :: ocn_c2_glctf ! .true. => ocn to glc thermal forcing coupling on + ! MPAS Datatypes !type (dm_info), pointer :: dminfo type (core_type), pointer :: corelist => null() @@ -526,6 +528,9 @@ end subroutine xml_stream_get_attributes ! Determine coupling type (not currently needed by MALI) call seq_infodata_GetData(infodata, cpl_seq_option=cpl_seq_option) + ! Determine if ocn to glc thermal forcing coupling is on + call seq_infodata_GetData(infodata, ocn_c2_glctf=ocn_c2_glctf) + ! Initialize the MALI core ierr = domain % core % core_init(domain, timeStamp) if ( ierr /= 0 ) then @@ -1394,7 +1399,8 @@ subroutine glc_import_mct(x2g_g, errorCode) floatingBasalMassBal,& surfaceTemperature,& basalOceanHeatflx,& - OceanDensity + OceanDensity, & + ismip6_2dThermalForcing errorCode = 0 @@ -1412,6 +1418,7 @@ subroutine glc_import_mct(x2g_g, errorCode) call mpas_pool_get_array(geometryPool, 'sfcMassBal', sfcMassBal) call mpas_pool_get_array(geometryPool, 'floatingBasalMassBal',floatingBasalMassBal) call mpas_pool_get_array(thermalPool, 'surfaceTemperature',surfaceTemperature) + call mpas_pool_get_array(geometryPool, 'ismip6_2dThermalForcing', ismip6_2dThermalForcing) ! call mpas_pool_get_array(thermalPool, 'basalOceanHeatflx',basalOceanHeatflx) !call mpas_pool_get_array(geometryPool, 'OceanDensity',OceanDensity) @@ -1419,6 +1426,8 @@ subroutine glc_import_mct(x2g_g, errorCode) n = n + 1 sfcMassBal(i) = x2g_g % rAttr(index_x2g_Flgl_qice, n) floatingBasalMassBal(i) = x2g_g % rAttr(index_x2g_Fogx_qiceli, n) + if (ocn_c2_glctf) & + ismip6_2dThermalForcing(i) = x2g_g % rAttr(index_x2g_So_tf2d, n) ! surfaceTemperature(i) = x2g_g % rAttr(index_x2g_Sl_tsrf, n) !JW basalOceanHeatflx(i) = x2g_g % rAttr(index_x2g_Fogo_qiceh, n) ! basalOceanHeatflx(i) = x2g_g % rAttr(index_x2g_Fogx_qicehi, n) @@ -1490,10 +1499,12 @@ subroutine glc_export_mct(g2x_g, errorCode) do i = 1, nCellsSolve n = n + 1 - !call route_ice_runoff(0.0_RKIND, & !Recuperate runoff routing switch code (originally in glc_route_ice_runoff module in earlier code), and attach to ice calving flux once present... + ! Recuperate runoff routing switch code (originally in glc_route_ice_runoff module in earlier code), + ! and attach to ice calving flux once present... + !call route_ice_runoff(0.0_RKIND, & ! rofi_to_ocn=Fogg_rofi, & ! rofi_to_ice=Figg_rofi) - g2x_g % rAttr(index_g2x_Fogg_rofi,n)=0.0 !...and remove these placeholders + g2x_g % rAttr(index_g2x_Fogg_rofi,n)=0.0!...and remove these placeholders g2x_g % rAttr(index_g2x_Figg_rofi,n)=0.0 !...and remove these placeholders g2x_g % rAttr(index_g2x_Fogg_rofl,n) = 0.0 !Attach to subglacial liquid flux once present diff --git a/components/mpas-albany-landice/driver/glc_cpl_indices.F b/components/mpas-albany-landice/driver/glc_cpl_indices.F index 123185225f8..459b534a7e0 100644 --- a/components/mpas-albany-landice/driver/glc_cpl_indices.F +++ b/components/mpas-albany-landice/driver/glc_cpl_indices.F @@ -22,6 +22,7 @@ module glc_cpl_indices integer, public :: index_x2g_So_htv = 0 !Ice shelf ocean heat transfer velocity integer, public :: index_x2g_So_stv = 0 !Ice shelf ocean salinity transfer velocity integer, public :: index_x2g_So_rhoeff = 0 !Ocean effective pressure + integer, public :: index_x2g_So_tf2d = 0 !Ocean thermal forcing at predefined critical depth integer, public :: index_x2g_Fogx_qiceli = 0 !Subshelf mass flux integer, public :: index_x2g_Fogx_qicehi = 0 !Subshelf heat flux for the ice sheet @@ -70,6 +71,7 @@ subroutine glc_cpl_indices_set( ) index_x2g_Fogx_qiceli = mct_avect_indexra(x2g,'Fogx_qiceli',perrwith='quiet') index_x2g_Fogx_qicehi = mct_avect_indexra(x2g,'Fogx_qicehi',perrwith='quiet') index_x2g_So_rhoeff = mct_avect_indexra(x2g,'So_rhoeff',perrwith='quiet') + index_x2g_So_tf2d = mct_avect_indexra(x2g,'So_tf2d',perrwith='quiet') !Following block of x2g/g2x vectors are used internally within coupler for subshelf melt flux !calculations (and so do not have directly-related export-side arrays) diff --git a/components/mpas-albany-landice/src/Registry.xml b/components/mpas-albany-landice/src/Registry.xml index 014d8a96591..1b0730fe4e2 100644 --- a/components/mpas-albany-landice/src/Registry.xml +++ b/components/mpas-albany-landice/src/Registry.xml @@ -453,7 +453,22 @@ descrption="Apply a uniform linear submarine melt rate at all grounded marine margins. config_mass_bal_grounded must be set to 'uniform'." possible_values="any non-negative value" /> - + + + + + + @@ -797,6 +813,8 @@ + + @@ -900,6 +919,8 @@ + + @@ -1386,6 +1407,9 @@ is the value of that variable from the *previous* time level! + @@ -1437,6 +1461,14 @@ is the value of that variable from the *previous* time level! description="temporary copy of cellMask" persistence="scratch" /> + + @@ -1647,6 +1679,34 @@ is the value of that variable from the *previous* time level! /> + + + + + + + + + + + + @@ -1731,7 +1791,6 @@ is the value of that variable from the *previous* time level! description="grow mask for flood fill" persistence="scratch" /> - diff --git a/components/mpas-albany-landice/src/Registry_subglacial_hydro.xml b/components/mpas-albany-landice/src/Registry_subglacial_hydro.xml index 8b3f07d4750..ac756b6d33f 100644 --- a/components/mpas-albany-landice/src/Registry_subglacial_hydro.xml +++ b/components/mpas-albany-landice/src/Registry_subglacial_hydro.xml @@ -178,8 +178,10 @@ + + description="Effective pressure used in basal friction calculation. If subglacial hydrology model is active, this will be effectivePressureSGH averaged over the subglacial hydrology model timestepping subcycles. If subglacial hydrology model is inactive, this will come from a file or a parameterization."/> thermalCellMaskField % array @@ -366,12 +376,14 @@ subroutine li_advection_thickness_tracers(& ! given the old thickness, compute the thickness in each layer call li_calculate_layerThickness(meshPool, thickness, layerThickness) - ! update masks - call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) - err = ior(err, err_tmp) - - ! save old copycellMask for determining cells changing from grounded to floating and vice versa + ! Save copies of masks because we need to preserve mask + ! states prior to advection for accurate time integration. + ! A mask update is necessary to calculate grounding line flux, + ! after which we will reset the masks to their previous states. cellMaskTemporaryField % array(:) = cellMask(:) + edgeMaskTemporaryField % array(:) = edgeMask(:) + vertexMaskTemporaryField % array(:) = vertexMask(:) + layerThicknessEdgeFlux(:,:) = 0.0_RKIND !----------------------------------------------------------------- @@ -540,10 +552,10 @@ subroutine li_advection_thickness_tracers(& dynamicThickening = (sum(layerThickness, 1) - thickness) / dt * scyr ! units of m/yr - ! Update the thickness and cellMask before applying the mass balance. - ! The update is needed because the SMB and BMB depend on whether ice is present. + ! Update the thickness before applying the mass balance, but + ! do not update masks because mass balance acts on geometry + ! before advection took place. thickness = sum(layerThickness, 1) - call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) @@ -627,6 +639,9 @@ subroutine li_advection_thickness_tracers(& enddo endif + ! We need an updated set of masks to calculate fluxAcrossGroundingLine, + ! but we will reset this to the previous state below for accuracy of the + ! time integration scheme. call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) err = ior(err, err_tmp) @@ -667,12 +682,17 @@ subroutine li_advection_thickness_tracers(& endif enddo ! edges + ! Reset masks to state before advection and mass balance for + ! accuracy of time integration scheme. + cellMask(:) = cellMaskTemporaryField % array(:) + edgeMask(:) = edgeMaskTemporaryField % array(:) + vertexMask(:) = vertexMaskTemporaryField % array(:) ! Remap tracers to the standard vertical sigma coordinate ! Note: If tracers are not being advected, then this subroutine simply restores the ! layer thickness to sigma coordinate values. - call vertical_remap(thickness, cellMask, meshPool, layerThickness, advectedTracers, err_tmp) + call vertical_remap(thickness, meshPool, layerThickness, advectedTracers, err_tmp) err = ior(err, err_tmp) if (config_print_thickness_advection_info) then @@ -717,13 +737,7 @@ subroutine li_advection_thickness_tracers(& ! Deallocate arrays for fct if ( (trim(config_thickness_advection) .eq. 'fct') .or. & (trim(config_tracer_advection) .eq. 'fct') ) then - deallocate( nAdvCellsForEdge, & - advCellsForEdge, & - advCoefs, & - advCoefs3rd, & - advMaskHighOrder, & - advMask2ndOrder, & - tend) + deallocate(tend) endif ! clean up @@ -733,6 +747,8 @@ subroutine li_advection_thickness_tracers(& call mpas_deallocate_scratch_field(basalTracersField, .true.) call mpas_deallocate_scratch_field(surfaceTracersField, .true.) call mpas_deallocate_scratch_field(cellMaskTemporaryField, .true.) + call mpas_deallocate_scratch_field(edgeMaskTemporaryField, .true.) + call mpas_deallocate_scratch_field(vertexMaskTemporaryField, .true.) call mpas_deallocate_scratch_field(thermalCellMaskField, .true.) ! === error check @@ -1093,7 +1109,6 @@ subroutine tracer_setup(& nTracers) use li_thermal, only: li_temperature_to_enthalpy_kelvin - use li_tracer_advection_fct_shared use li_tracer_advection_fct !----------------------------------------------------------------- ! @@ -1168,11 +1183,9 @@ subroutine tracer_setup(& real (kind=RKIND), pointer :: & config_ice_density ! ice density - integer :: iCell, iTracer, k, err, err1, err2 + integer :: iCell, iTracer, k, err err = 0 - err1 = 0 - err2 = 0 ! get dimensions call mpas_pool_get_dimension(meshPool, 'nCells', nCells) @@ -1301,14 +1314,12 @@ subroutine tracer_setup(& ! May need to increase maxTracers in the Registry. if ( (trim(config_tracer_advection) == 'fct') .or. & (trim(config_thickness_advection) == 'fct') ) then - call li_tracer_advection_fct_shared_init(geometryPool, err1) - call li_tracer_advection_fct_init(err2) + call li_tracer_advection_fct_init(err) - if (err1 /= 0 .or. err2 /= 0) then - err = 1 + if (err /= 0) then call mpas_log_write( & 'Error encountered during fct tracer advection init', & - MPAS_LOG_ERR, masterOnly=.true.) + MPAS_LOG_ERR) endif endif @@ -1950,7 +1961,7 @@ end subroutine li_layer_normal_velocity !> OpenMP over either blocks or cells. ! !----------------------------------------------------------------------- - subroutine vertical_remap(thickness, cellMask, meshPool, layerThickness, tracers, err) + subroutine vertical_remap(thickness, meshPool, layerThickness, tracers, err) !----------------------------------------------------------------- ! @@ -1964,9 +1975,6 @@ subroutine vertical_remap(thickness, cellMask, meshPool, layerThickness, tracers real(kind=RKIND), dimension(:), intent(in) :: & thickness !< Input: ice thickness - integer, dimension(:), intent(in) :: & - cellMask !< Input: mask for cells (needed for determining presence/absence of ice) - !----------------------------------------------------------------- ! ! input/output variables diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F index a460c3224a0..1637c7d697d 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F @@ -3828,9 +3828,8 @@ subroutine li_finalize_damage_after_advection(domain, err) call mpas_pool_get_array(geometryPool, 'damageNye', damageNye) call mpas_pool_get_array(velocityPool, 'stiffnessFactor', stiffnessFactor) - ! make sure masks are up to date. May not be necessary, but safer to do anyway. - call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) - err = ior(err, err_tmp) + ! Note: In order to preserve accuracy of time integration, + ! we do not update masks before finalizing damage. if (config_preserve_damage) then do iCell = 1, nCells diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_core_interface.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_core_interface.F index 8a278675da3..f9f0185499c 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_core_interface.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_core_interface.F @@ -108,6 +108,7 @@ function li_setup_packages(configPool, packagePool, iocontext) result(ierr) logical, pointer :: config_SGH logical, pointer :: config_adaptive_timestep_include_DCFL logical, pointer :: config_write_albany_ascii_mesh + logical, pointer :: config_ocean_data_extrapolation logical, pointer :: higherOrderVelocityActive logical, pointer :: SIAvelocityActive @@ -116,6 +117,7 @@ function li_setup_packages(configPool, packagePool, iocontext) result(ierr) logical, pointer :: ismip6ShelfMeltActive logical, pointer :: ismip6GroundedFaceMeltActive logical, pointer :: thermalActive + logical, pointer :: extrapOceanDataActive ierr = 0 @@ -126,6 +128,7 @@ function li_setup_packages(configPool, packagePool, iocontext) result(ierr) call mpas_pool_get_config(configPool, 'config_basal_mass_bal_float', config_basal_mass_bal_float) call mpas_pool_get_config(configPool, 'config_front_mass_bal_grounded', config_front_mass_bal_grounded) call mpas_pool_get_config(configPool, 'config_use_3d_thermal_forcing_for_face_melt', config_use_3d_thermal_forcing_for_face_melt) + call mpas_pool_get_config(configPool, 'config_ocean_data_extrapolation', config_ocean_data_extrapolation) call mpas_pool_get_config(configPool, 'config_thermal_solver', config_thermal_solver) call mpas_pool_get_package(packagePool, 'SIAvelocityActive', SIAvelocityActive) @@ -135,6 +138,7 @@ function li_setup_packages(configPool, packagePool, iocontext) result(ierr) call mpas_pool_get_package(packagePool, 'ismip6ShelfMeltActive', ismip6ShelfMeltActive) call mpas_pool_get_package(packagePool, 'ismip6GroundedFaceMeltActive', ismip6GroundedFaceMeltActive) call mpas_pool_get_package(packagePool, 'thermalActive', thermalActive) + call mpas_pool_get_package(packagePool, 'extrapOceanDataActive', extrapOceanDataActive) if (trim(config_velocity_solver) == 'sia') then SIAvelocityActive = .true. @@ -157,13 +161,21 @@ function li_setup_packages(configPool, packagePool, iocontext) result(ierr) "'config_write_albany_ascii_mesh' is set to .true.") endif + if (config_ocean_data_extrapolation) then + extrapOceanDataActive = .true. + call mpas_log_write("The 'extrapOceanDataActive' package and assocated variables have been enabled because " // & + "'config_ocean_data_extrapolation' is set to .true.") + endif + if ( (trim(config_basal_mass_bal_float) == 'ismip6') .or. & ((trim(config_front_mass_bal_grounded) == 'ismip6') .and. & - (config_use_3d_thermal_forcing_for_face_melt)) ) then + (config_use_3d_thermal_forcing_for_face_melt)) .or. & + (config_ocean_data_extrapolation) ) then ismip6ShelfMeltActive = .true. call mpas_log_write("The 'ismip6Melt' package and assocated variables have been enabled because " // & "'config_basal_mass_bal_float' is set to 'ismip6' or 'config_front_mass_bal_grounded' is set to 'ismip6' // & - and 'config_use_3d_thermal_forcing_for_face_melt' is set to .true.") + and 'config_use_3d_thermal_forcing_for_face_melt' is set to .true. " // & + "or 'config_ocean_data_extrapolation' is set to .true.") endif if ((trim(config_front_mass_bal_grounded) == 'ismip6') .or. (trim(config_calving) == 'ismip6_retreat')) then diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_iceshelf_melt.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_iceshelf_melt.F index 5a92ecba377..daa1d5cca87 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_iceshelf_melt.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_iceshelf_melt.F @@ -1162,6 +1162,8 @@ end subroutine calc_iceshelf_draft_info subroutine iceshelf_melt_ismip6(domain, err) + use li_constants, only: oceanFreezingTempDepthDependence + !----------------------------------------------------------------- ! input/output variables !----------------------------------------------------------------- @@ -1182,6 +1184,7 @@ subroutine iceshelf_melt_ismip6(domain, err) integer :: ksup, kk, kinf integer, pointer :: nCellsSolve real(kind=RKIND), pointer :: rhoi, rhosw + real (kind=RKIND), pointer :: invalid_value_TF real(kind=RKIND) :: cste type (block_type), pointer :: block type (mpas_pool_type), pointer :: geometryPool, meshPool @@ -1206,6 +1209,7 @@ subroutine iceshelf_melt_ismip6(domain, err) call mpas_pool_get_config(liConfigs, 'config_ice_density', rhoi) call mpas_pool_get_config(liConfigs, 'config_ocean_density', rhosw) + call mpas_pool_get_config(liConfigs, 'config_invalid_value_TF', invalid_value_TF) cste = (rhosw*cp_seawater/(rhoi*latent_heat_ice))**2 ! in K^(-2) allocate(mean_TF(maxNumBasins)) @@ -1247,28 +1251,65 @@ subroutine iceshelf_melt_ismip6(domain, err) mean_TF(:) = 0.d0 IS_area(:) = 0.d0 + ! Check zOcean for valid values + if (minval(zOcean) >= 0.0_RKIND) then + call mpas_log_write("Invalid value for zOcean. It should have negative values but min value >= 0.0 was found", & + MPAS_LOG_ERR) + err = ior(err, 1) + endif + if (maxval(zOcean) > 0.0_RKIND) then + call mpas_log_write("Invalid value for zOcean. It should have non-positive values but max value greater than 0.0 was found", & + MPAS_LOG_ERR) + err = ior(err, 1) + endif + do iCell = 1, nCellsSolve if ( li_mask_is_floating_ice(cellMask(iCell)) ) then ! 1 - Linear interpolation of the thermal forcing on the ice draft depth : - ksup=1 - do kk=2,nOceanLayers-1 + ksup=0 + do kk=1,nOceanLayers if ( zOcean(kk) >= lowerSurface(iCell) ) ksup = kk enddo kinf = ksup + 1 - if ((zOcean(ksup)-zOcean(kinf)) == 0) then - call mpas_log_write("iceshelf_melt_ismip6: Invalid value for zOcean. " // & - "ksup=$i kinf=$i zOcean(ksup)=$r zOcean(kinf)=$r indexToCellID=$i lowerSurface=$r", MPAS_LOG_ERR, & - intArgs=(/ksup, kinf, indexToCellID(iCell)/), & - realArgs=(/zOcean(ksup), zOcean(kinf), lowerSurface(iCell) /) ) - err = ior(err, 1) - endif + !call mpas_log_write("kinf=$i, zOcean(kinf)=$r, TFocean=$r",realArgs=(/zOcean(kinf),TFocean(kinf,iCell)/), & ! intArgs=(/kinf/)) !call mpas_log_write("ksup=$i, zOcean(ksup)=$r, TFocean=$r",realArgs=(/zOcean(ksup),TFocean(ksup,iCell)/), & ! intArgs=(/ksup/)) - TFdraft(iCell) = ( (zOcean(ksup)-lowerSurface(iCell)) * TFocean(kinf, iCell) & - + (lowerSurface(iCell)-zOcean(kinf)) * TFocean(ksup, iCell) ) / (zOcean(ksup)-zOcean(kinf)) + + ! check if any invalid TFocean value is used for calculating TF at the draft + if (ksup >= 1) then + if (TFocean(ksup, iCell) == invalid_value_TF) then + call mpas_log_write("iceshelf_melt_ismip6: Invalid value for TFocean. " // & + "ksup=$i TFocean(ksup, iCell)=$r indexToCellID=$i", MPAS_LOG_ERR, & + intArgs=(/ksup, indexToCellID(iCell)/), & + realArgs=(/TFocean(ksup, iCell) /) ) + err = ior(err, 1) + endif + endif + if (kinf <= nOceanLayers) then + if (TFocean(kinf, iCell) == invalid_value_TF) then + call mpas_log_write("iceshelf_melt_ismip6: Invalid value for TFocean. " // & + "kinf=$i TFocean(kinf,iCell)=$r indexToCellID=$i", MPAS_LOG_ERR, & + intArgs=(/kinf, indexToCellID(iCell)/), & + realArgs=(/TFocean(kinf, iCell) /) ) + err = ior(err, 1) + endif + endif + + if (ksup == 0) then + ! For depths shallower than shallowest layer center, use shallowest layer + TFdraft(iCell) = TFocean(1, iCell) + elseif (ksup == nOceanLayers) then + ! for depths below the deepest layer center, use deepest layer corrected for Tfreeze + TFdraft(iCell) = TFocean(nOceanLayers, iCell) - & + (zOcean(nOceanLayers) - lowerSurface(iCell)) * oceanFreezingTempDepthDependence + else + ! for depths between the first and last layer centers, linearly interpolate + TFdraft(iCell) = ( (zOcean(ksup)-lowerSurface(iCell)) * TFocean(kinf, iCell) & + + (lowerSurface(iCell)-zOcean(kinf)) * TFocean(ksup, iCell) ) / (zOcean(ksup)-zOcean(kinf)) + endif ! 2 - Mean Thermal forcing in individual basins (NB: fortran norm while basins start at zero): mean_TF(basinNumber(iCell)+1) = mean_TF(basinNumber(iCell)+1) + areaCell(iCell) * TFdraft(iCell) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_ocean_extrap.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_ocean_extrap.F new file mode 100644 index 00000000000..de501d62960 --- /dev/null +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_ocean_extrap.F @@ -0,0 +1,601 @@ +! Copyright (c) 2013-2018, Los Alamos National Security, LLC (LANS) +! and the University Corporation for Atmospheric Research (UCAR). +! +! Unless noted otherwise source code is licensed under the BSD license. +! Additional copyright and license information can be found in the LICENSE file +! distributed with this code, or at http://mpas-dev.github.com/license.html +! + +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! li_ocean_extrap +! +!> \MPAS land-ice ocean-data extrapolation driver +!> \author Holly Han +!> \date January 2022 +!> \details +!> This module contains the routines for extrapolating +!> ocean data (e.g., temperature, salinity, thermal forcing) +!> into ice draft +! +!----------------------------------------------------------------------- + +module li_ocean_extrap + + use mpas_derived_types + use mpas_pool_routines + use mpas_dmpar + use mpas_log + use li_mask + use li_setup + + implicit none + private + + !-------------------------------------------------------------------- + ! Public parameters + !-------------------------------------------------------------------- + + !-------------------------------------------------------------------- + ! Public member functions + !-------------------------------------------------------------------- + + public :: li_ocean_extrap_solve + + !-------------------------------------------------------------------- + ! Private module variables + !-------------------------------------------------------------------- + +!*********************************************************************** + +contains + +!*********************************************************************** +! +! routine li_ocean_extrap_solve +! +!> \brief Initializes ocean extrapolation scheme +!> \author Holly Han +!> \date 12 Jan 2023 +!> \details +!> This routine performs horizontal and vertical extrapolation +!> of ocean data (e.g., temperature, salinity, thermal forcing) +! +!----------------------------------------------------------------------- + + subroutine li_ocean_extrap_solve(domain, err) + use li_calving, only: li_flood_fill + + !----------------------------------------------------------------- + ! input variables + !----------------------------------------------------------------- + + !----------------------------------------------------------------- + ! input/output variables + !----------------------------------------------------------------- + type (domain_type), intent(inout) :: domain !< Input/Output: domain object + + !----------------------------------------------------------------- + ! output variables + !----------------------------------------------------------------- + integer, intent(out) :: err !< Output: error flag + + !----------------------------------------------------------------- + ! local variables + !----------------------------------------------------------------- + logical, pointer :: config_ocean_data_extrapolation + real(kind=RKIND), pointer :: config_sea_level, invalid_value_TF + type (block_type), pointer :: block + type (mpas_pool_type), pointer :: scratchPool, geometryPool, meshPool, extrapOceanDataPool + real (kind=RKIND) :: layerTop + real (kind=RKIND), dimension(:,:), pointer :: TFocean, TFoceanOld + real (kind=RKIND), dimension(:,:), pointer :: ismip6shelfMelt_3dThermalForcing, ismip6shelfMelt_zBndsOcean + real (kind=RKIND), dimension(:), pointer :: ismip6shelfMelt_zOcean + real (kind=RKIND), dimension(:), pointer :: thickness, bedTopography + integer, dimension(:,:), pointer :: orig3dOceanMask + integer, dimension(:,:), pointer :: validOceanMask, availOceanMask !masks to pass to flood-fill routine + integer, dimension(:), pointer :: seedOceanMaskHoriz, growOceanMaskHoriz + integer, dimension(:), allocatable :: seedOceanMaskHorizOld + integer, pointer :: nCells, nCellsSolve, nISMIP6OceanLayers, nCellsExtra + integer, dimension(:), pointer :: cellMask, nEdgesOnCell + integer, dimension(:,:), pointer :: cellsOnCell + integer :: iCell, jCell, iLayer, iNeighbor, iter, err_tmp + integer :: GlobalLoopCount, newMaskCountGlobal + type (field1dInteger), pointer :: seedMaskField + type (field1dInteger), pointer :: growMaskField + integer, dimension(:), pointer :: connectedMarineMask, growMask !masks to pass to flood-fill routine + + ! No init is needed. + err = 0 + err_tmp = 0 + + call mpas_pool_get_config(liConfigs, 'config_ocean_data_extrapolation', config_ocean_data_extrapolation) + + if ( config_ocean_data_extrapolation ) then + ! call the extrapolation scheme + call mpas_log_write('ocean data will be extrapolated into the MALI ice draft') + block => domain % blocklist + + ! initialize the ocean data and mask fields + call mpas_pool_get_config(liConfigs, 'config_sea_level', config_sea_level) + call mpas_pool_get_config(liConfigs, 'config_ocean_data_extrap_ncells_extra', nCellsExtra) + call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) + call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) + call mpas_pool_get_subpool(block % structs, 'extrapOceanData', extrapOceanDataPool) + call mpas_pool_get_dimension(meshPool, 'nISMIP6OceanLayers', nISMIP6OceanLayers) + call mpas_pool_get_dimension(meshPool, 'nCells', nCells) + call mpas_pool_get_dimension(meshPool, 'nCellsSolve', nCellsSolve) + call mpas_pool_get_array(meshPool, 'cellsOnCell', cellsOnCell) + call mpas_pool_get_array(meshPool, 'nEdgesOnCell', nEdgesOnCell) + call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) + call mpas_pool_get_array(geometryPool, 'thickness', thickness) + call mpas_pool_get_array(geometryPool, 'bedTopography', bedTopography) + call mpas_pool_get_array(geometryPool, 'ismip6shelfMelt_3dThermalForcing', ismip6shelfMelt_3dThermalForcing) + call mpas_pool_get_array(extrapOceanDataPool, 'orig3dOceanMask', orig3dOceanMask) + call mpas_pool_get_array(extrapOceanDataPool, 'validOceanMask', validOceanMask) + call mpas_pool_get_array(extrapOceanDataPool, 'availOceanMask', availOceanMask) + call mpas_pool_get_array(extrapOceanDataPool, 'seedOceanMaskHoriz', seedOceanMaskHoriz) + call mpas_pool_get_array(extrapOceanDataPool, 'growOceanMaskHoriz', growOceanMaskHoriz) + call mpas_pool_get_array(extrapOceanDataPool, 'TFoceanOld', TFoceanOld) + call mpas_pool_get_array(extrapOceanDataPool, 'TFocean', TFocean) + call mpas_pool_get_config(liConfigs, 'config_invalid_value_TF', invalid_value_TF) + + ! create a 2D mask based on the open ocean + floating ice + grounded ice mask to be used for defining the location of the n-extra cells into the grounded ice + allocate(seedOceanMaskHorizOld(nCells+1)) + seedOceanMaskHorizOld(:) = 0 + seedOceanMaskHoriz(:) = 0 + growOceanMaskHoriz(:) = 0 + ! define seedOceanMaskHoriz and growOceanMaskHoriz for horizontal floodfill + do iCell = 1, nCellsSolve + if ( .not. li_mask_is_grounded_ice(cellMask(iCell)) .and. bedTopography(iCell) < config_sea_level ) then + seedOceanMaskHoriz(iCell) = 1 + endif + if ( bedTopography(iCell) < config_sea_level ) then + growOceanMaskHoriz(iCell) = 1 + endif + enddo + + ! Start horizontal floodfill to define locations of n-extra cells + seedOceanMaskHorizOld(:) = seedOceanMaskHoriz(:) + ! go through the loop to get nCells extra into grounded ice + do iter = 1, nCellsExtra + do iCell = 1, nCellsSolve + if ( growOceanMaskHoriz(iCell) == 1 .and. seedOceanMaskHorizOld(iCell) == 0 ) then + do iNeighbor = 1, nEdgesOnCell(iCell) + jCell = cellsOnCell(iNeighbor, iCell) + if ( seedOceanMaskHorizOld(jCell) == 1 ) then + seedOceanMaskHoriz(iCell) = 1 + endif + enddo + endif + enddo + seedOceanMaskHorizOld(:) = seedOceanMaskHoriz(:) + ! Update halos + ! Note: halo updates could be skipped for a number of iterations equal to the + ! number of halo rows as a possible performance improvement in the future. + ! (And the loop above would need to be changed to from nCellsSolve to nCells) + call mpas_timer_start("halo updates") + call mpas_dmpar_field_halo_exch(domain, 'growOceanMaskHoriz') + call mpas_dmpar_field_halo_exch(domain, 'seedOceanMaskHoriz') + call mpas_timer_stop("halo updates") + enddo + deallocate(seedOceanMaskHorizOld) + + ! Calculate mask of connected ocean + call mpas_pool_get_subpool(domain % blocklist % structs, 'scratch', scratchPool) + call mpas_pool_get_field(scratchPool, 'seedMask', seedMaskField) + call mpas_allocate_scratch_field(seedMaskField, single_block_in = .true.) + connectedMarineMask => seedMaskField % array + connectedMarineMask(:) = 0 + call mpas_pool_get_field(scratchPool, 'growMask', growMaskField) + call mpas_allocate_scratch_field(growMaskField, single_block_in = .true.) + growMask => growMaskField % array + growMask(:) = 0 + + do iCell = 1, nCells + ! seedMask = open ocean cells in contact with the domain boundary + if ((bedTopography(iCell) < config_sea_level) .and. (thickness(iCell) == 0.0_RKIND)) then + do iNeighbor = 1, nEdgesOnCell(iCell) + if (cellsOnCell(iNeighbor, iCell) == nCells + 1) then + connectedMarineMask(iCell) = 1 + exit ! no need to keep checking neighbors + endif + enddo + endif + ! growMask - all marine cells + if (bedTopography(iCell) < config_sea_level) then + growMask(iCell) = 1 + endif + enddo + ! now create mask of all marine locations connected to open ocean - to be used below to screen out lakes + call li_flood_fill(connectedMarineMask, growMask, domain) + + ! make it a 3D mask based on the topography (loop through nISMIP6OceanLayers) + call mpas_pool_get_dimension(meshPool, 'nISMIP6OceanLayers', nISMIP6OceanLayers) + call mpas_pool_get_array(geometryPool, 'ismip6shelfMelt_zOcean', ismip6shelfMelt_zOcean) + call mpas_pool_get_array(geometryPool, 'ismip6shelfMelt_zBndsOcean', ismip6shelfMelt_zBndsOcean) + ! check for valid data + do iLayer = 1, nISMIP6OceanLayers + if (ismip6shelfMelt_zOcean(iLayer) >= 0.0_RKIND) then + call mpas_log_write("ismip6shelfMelt_zOcean has invalid value of $r in layer $i", MPAS_LOG_ERR, & + realArgs=(/ismip6shelfMelt_zOcean(iLayer)/), intArgs=(/iLayer/)) + call mpas_log_write("ismip6shelfMelt_zOcean must have negative values because they represent " // & + "depths below sea level.", MPAS_LOG_ERR) + err = ior(err, 1) + endif + if ((ismip6shelfMelt_zBndsOcean(1,iLayer) > 0.0_RKIND) .or. & + (ismip6shelfMelt_zBndsOcean(1,iLayer) < ismip6shelfMelt_zOcean(iLayer))) then + call mpas_log_write("ismip6shelfMelt_zBndsOcean(1,:) has invalid value of $r in layer $i", MPAS_LOG_ERR, & + realArgs=(/ismip6shelfMelt_zBndsOcean(1,iLayer)/), intArgs=(/iLayer/)) + call mpas_log_write("ismip6shelfMelt_zBndsOcean(1,:) must be less than or equal to zero " // & + "because it represents the upper bound of an ocean layer", MPAS_LOG_ERR) + err = ior(err, 1) + endif + if ((ismip6shelfMelt_zBndsOcean(2,iLayer) >= 0.0_RKIND) .or. & + (ismip6shelfMelt_zBndsOcean(2,iLayer) > ismip6shelfMelt_zOcean(iLayer))) then + call mpas_log_write("ismip6shelfMelt_zBndsOcean(2,:) has invalid value of $r in layer $i", MPAS_LOG_ERR, & + realArgs=(/ismip6shelfMelt_zBndsOcean(2,iLayer)/), intArgs=(/iLayer/)) + call mpas_log_write("ismip6shelfMelt_zBndsOcean(2,:) must be less than zero " // & + "because it represents the lower bound of an ocean layer", MPAS_LOG_ERR) + err = ior(err, 1) + endif + enddo + availOceanMask(:,:) = 0 + do iCell = 1, nCells + do iLayer = 1, nISMIP6OceanLayers + layerTop = ismip6shelfMelt_zBndsOcean(1, iLayer) + if ( (seedOceanMaskHoriz(iCell) == 1) .and. (connectedMarineMask(iCell) == 1)) then + if (bedTopography(iCell) < layerTop) then + availOceanMask(iLayer,iCell) = 1 + else + ! keep the first layer below the seafloor in the region to be filled + ! this ensures linear interpolation from above and below the seafloor is possible + availOceanMask(iLayer,iCell) = 1 + exit ! stop looping over levels after we've included the first level below the seafloor + endif + endif + enddo + enddo + + ! Make a copy of original mask to use for extending the mask during extrapolation + validOceanMask(:,:) = orig3dOceanMask(:,:) + + ! initialize the TF field + TFocean(:,:) = ismip6shelfMelt_3dThermalForcing(:,:) * validOceanMask(:,:) + ! initialize the invalid data locations with fill value + do iCell = 1, nCellsSolve + do iLayer = 1, nISMIP6OceanLayers + if ((connectedMarineMask(iCell) == 0) .and. (bedTopography(iCell) < config_sea_level)) then + ! Don't assign invalid value to lakes/inland seas disconnected from global ocean + ! Let them retain the existing value: + ! This will take on the valid ocean data value where it exists or + ! zero where valid ocean data does not exist + elseif (validOceanMask(iLayer,iCell) == 0) then + ! everywhere else where valid ocean data does not exist, insert invalid value outside of validOceanMask + TFocean(iLayer,iCell) = invalid_value_TF + endif + enddo + enddo + + ! deallocate scratch fields used for flood fill + call mpas_deallocate_scratch_field(seedMaskField, single_block_in=.true.) + call mpas_deallocate_scratch_field(growMaskField, single_block_in=.true.) + + ! flood-fill the valid ocean mask and TF field through + ! horizontal and vertial extrapolation + ! get initial 3D valid data based on the original ISMIP6 field + newMaskCountGlobal = 1 + GlobalLoopCount = 0 + do while (newMaskCountGlobal > 0) + newMaskCountGlobal = 0 + GlobalLoopCount = GlobalLoopCount + 1 + if (GlobalLoopCount > 20) then + ! There will only be an additional time through this loop for each sill behind a previous sill + ! so it should not need to alternate very many times + call mpas_log_write("Ocean extrapolation has alternated between horizontal and vertical " // & + "extrapolation more than $i times. Aborting", MPAS_LOG_ERR, intArgs=(/GlobalLoopCount/)) + err = ior(err, 1) + endif + ! call the horizontal extrapolation routine + call mpas_timer_start("horizontal scheme") + call horizontal_extrapolation(domain, availOceanMask, validOceanMask, orig3dOceanMask, TFocean, err_tmp) + err = ior(err, err_tmp) + call mpas_timer_stop("horizontal scheme") + ! call the vertical extrapolation routine + call mpas_timer_start("vertical scheme") + call vertical_extrapolation(domain, availOceanMask, validOceanMask, newMaskCountGlobal, TFocean, err_tmp) + err = ior(err, err_tmp) + call mpas_timer_stop("vertical scheme") + + if (err > 0) then + call mpas_log_write("Ocean extrapolation main iteration loop has encountered an error", MPAS_LOG_ERR) + return + endif + enddo + + ! Reassign extrapolated TF back to primary TF field + ismip6shelfMelt_3dThermalForcing(:,:) = TFocean(:,:) + endif + !-------------------------------------------------------------------- + + end subroutine li_ocean_extrap_solve +!----------------------------------------------------------------------- + + +!*********************************************************************** +!*********************************************************************** +! Private subroutines: +!*********************************************************************** +!*********************************************************************** + + + +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! ! routine horizontal_extrapolation +! +!> \brief Extrapolate validOceanMask horizontally +!> \author Holly Kyeore Han +!> \date November 2023 +!> \details +!> This routine extrapolates takes the initialized availOceanMask +!> and validOceanMask and extrapolates validOceanMask in horizontal +!> direction until the local new mask count stops updating. +!> The output of the routine is an updated validOceanMask field and +!> newMaskCountLocal. + +!----------------------------------------------------------------------- + + subroutine horizontal_extrapolation(domain, availOceanMask, validOceanMask, validOceanMaskOrig, TFocean, err) + + !----------------------------------------------------------------- + ! input variables + !----------------------------------------------------------------- + integer, dimension(:,:), pointer, intent(in) :: availOceanMask, validOceanMaskOrig + + !----------------------------------------------------------------- + ! input/output variables + !----------------------------------------------------------------- + type (domain_type), intent(inout) :: domain !< Input/Output: domain object + integer, dimension(:,:), pointer, intent(inout) :: validOceanMask + integer, intent(inout) :: err !< Output: error flag + real (kind=RKIND), dimension(:,:), pointer, intent(inout) :: TFocean + + !----------------------------------------------------------------- + ! output variables + !----------------------------------------------------------------- + + !----------------------------------------------------------------- + ! local variables + !----------------------------------------------------------------- + type (block_type), pointer :: block + type (mpas_pool_type), pointer :: scratchPool, geometryPool, meshPool, extrapOceanDataPool + real (kind=RKIND) :: layerTop, TFsum, areaSum, weightCellLocal + real (kind=RKIND), pointer :: weightCell + integer, dimension(:,:), allocatable :: validOceanMaskOld + real (kind=RKIND), dimension(:,:), pointer :: ismip6shelfMelt_3dThermalForcing + real (kind=RKIND), dimension(:,:), pointer :: TFoceanOld + real (kind=RKIND), dimension(:), pointer :: thickness, bedTopography, areaCell + integer, pointer :: nCells, nCellsSolve, nISMIP6OceanLayers, nCellsExtra + integer, dimension(:), pointer :: cellMask, nEdgesOnCell + integer, dimension(:), pointer :: indexToCellID + integer, dimension(:,:), pointer :: cellsOnCell + integer :: iCell, jCell, iLayer, iNeighbor, iter + integer :: localLoopCount + integer :: nValidNeighb, newValidCount, newMaskCountLocalAccum, newMaskCountGlobal + integer :: newMaskCountTotal + + err = 0 + + ! initialize the ocean data and mask fields + block => domain % blocklist + call mpas_pool_get_config(liConfigs, 'config_ocean_data_extrap_ncells_extra', nCellsExtra) + call mpas_pool_get_config(liConfigs,'config_weight_value_cell', weightCell) + call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) + call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) + call mpas_pool_get_subpool(block % structs, 'extrapOceanData', extrapOceanDataPool) + call mpas_pool_get_dimension(meshPool, 'nISMIP6OceanLayers', nISMIP6OceanLayers) + call mpas_pool_get_dimension(meshPool, 'nCells', nCells) + call mpas_pool_get_dimension(meshPool, 'nCellsSolve', nCellsSolve) + call mpas_pool_get_array(meshPool, 'cellsOnCell', cellsOnCell) + call mpas_pool_get_array(meshPool, 'areaCell', areaCell) + call mpas_pool_get_array(meshPool, 'nEdgesOnCell', nEdgesOnCell) + call mpas_pool_get_array(meshPool, 'indexToCellID', indexToCellID) + call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) + call mpas_pool_get_array(geometryPool, 'thickness', thickness) + call mpas_pool_get_array(geometryPool, 'bedTopography', bedTopography) + call mpas_pool_get_array(extrapOceanDataPool, 'TFoceanOld', TFoceanOld) + + !TFoceanOld(:,:) = 1.0 ! for debugging, set TF to ones to make it easy to verify the horizonal/vertical averaging + ! perform horizontal extrapolation until the validOceanMask is unchanged + allocate(validOceanMaskOld(nISMIP6OceanLayers,nCells+1)) + validOceanMaskOld(:,:) = validOceanMask(:,:) + TFoceanOld(:,:) = TFocean(:,:) + + ! initialize the local loop and count for validOceanMask + localLoopCount = 0 + newMaskCountTotal = 0 + newMaskCountGlobal = 1 + call mpas_log_write('Weight given to the cell with valid data from extrapolation: $r', realArgs=(/weightCell/)) + do while ( newMaskCountGlobal > 0 ) + localLoopCount = localLoopCount + 1 + newMaskCountLocalAccum = 0 + do iCell = 1, nCellsSolve + do iLayer = 1, nISMIP6OceanLayers + if ( (availOceanMask(iLayer,iCell) == 1) .and. (validOceanMaskOrig(iLayer,iCell) == 0) ) then + TFsum = 0.0 + areaSum = 0.0 + newValidCount = 0 + nValidNeighb = 0 + do iNeighbor = 1, nEdgesOnCell(iCell) + jCell = cellsOnCell(iNeighbor, iCell) + if ( validOceanMaskOld(iLayer,jCell) == 1 ) then + if ( TFoceanOld(iLayer,jCell) > 1.0e6_RKIND) then + ! raise error if an invalid ocean data value is used + call mpas_log_write("ocean data value used for extrapolation is invalid " // & + "in horizontal_extrapolation: cell id=$i, iLayer=$i, TF=$r", & + MPAS_LOG_ERR, intArgs=(/indexToCellID(jCell), iLayer/), realArgs=(/TFoceanOld(iLayer,jCell)/)) + err = ior(err,1) + else + TFsum = TFsum + (TFoceanOld(iLayer,jCell) * areaCell(jCell)) + endif + areaSum = areaSum + areaCell(jCell) + nValidNeighb = nValidNeighb + 1 + endif + enddo + if ( validOceanMaskOld(iLayer,iCell) == 0 .and. nValidNeighb > 0 ) then + ! if current cell is not valid, set its weight to zero + weightCellLocal = 0.0_RKIND + validOceanMask(iLayer,iCell) = 1 + newValidCount = 1 + else + weightCellLocal = weightCell + endif + ! perform area-weighted averaging of the thermal forcing field + if ( nValidNeighb == 0 ) then + TFocean(iLayer,iCell) = TFoceanOld(iLayer,iCell) + else + TFocean(iLayer,iCell) = ( weightCellLocal * TFoceanOld(iLayer,iCell) * areaCell(iCell) + & + & ((1.0_RKIND - weightCellLocal) * (TFsum / nValidNeighb)) ) / & + & ( weightCellLocal * areaCell(iCell) + & + & (1.0_RKIND - weightCellLocal) * (areaSum / nValidNeighb) ) + endif + ! Accumulate cells added locally until we do the next global reduce + newMaskCountLocalAccum = newMaskCountLocalAccum + newValidCount + endif + enddo + enddo + + ! update halo for validOceanMask and ocean data + call mpas_timer_start("halo updates") + call mpas_dmpar_field_halo_exch(domain, 'validOceanMask') + call mpas_dmpar_field_halo_exch(domain, 'TFocean') + call mpas_timer_stop("halo updates") + + validOceanMaskOld(:,:) = validOceanMask(:,:) + TFoceanOld(:,:) = TFocean(:,:) + + ! update count of cells added to mask globally + call mpas_dmpar_sum_int(domain % dminfo, newMaskCountLocalAccum, newMaskCountGlobal) + newMaskCountTotal = newMaskCountTotal + newMaskCountGlobal + !call mpas_log_write('Horizontal extrap: Added total $i new cells to validOceanMask', intArgs=(/newMaskCountGlobal/)) + enddo + call mpas_log_write('Horizontal extrapolation done after $i iterations. Added total of $i cells across all processors', & + intArgs=(/localLoopCount, newMaskCountTotal/)) + deallocate(validOceanMaskOld) + + end subroutine horizontal_extrapolation +!----------------------------------------------------------------------- + + +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! ! routine vertical_extrapolation +! +!> \brief Extrapolate validOceanMask vertically +!> \author Holly Kyeore Han +!> \date November 2023 +!> \details +!> This routine extrapolates the horizontally extrapolated +!> validOceanMask through the vertical layers of the ocean. +!> The vertical extrapolation is completed once local new mask count +!> stops updating. The output of the routine is an updated +!> validOceanMask, thermal forcing fields and newMaskCountLocal. + +!----------------------------------------------------------------------- + + subroutine vertical_extrapolation(domain, availOceanMask, validOceanMask, newMaskCountGlobal, TFocean, err) + + use li_constants, only: oceanFreezingTempDepthDependence + + !----------------------------------------------------------------- + ! input variables + !----------------------------------------------------------------- + integer, dimension(:,:), pointer, intent(in) :: availOceanMask + + !----------------------------------------------------------------- + ! input/output variables + !----------------------------------------------------------------- + type (domain_type), intent(inout) :: domain !< Input/Output: domain object + integer, dimension(:,:), pointer, intent(inout) :: validOceanMask + integer, intent(inout) :: err !< Output: error flag + real (kind=RKIND), dimension(:,:), pointer, intent(inout) :: TFocean + + !----------------------------------------------------------------- + ! output variables + !----------------------------------------------------------------- + integer, intent(out) :: newMaskCountGlobal + + !----------------------------------------------------------------- + ! local variables + !----------------------------------------------------------------- + type (block_type), pointer :: block + type (mpas_pool_type), pointer :: scratchPool, geometryPool, meshPool, extrapOceanDataPool + real (kind=RKIND) :: layerTop, TFsum, areaSum + real (kind=RKIND), dimension(:), pointer :: ismip6shelfMelt_zOcean + real (kind=RKIND), dimension(:), pointer :: thickness, bedTopography, areaCell + integer, pointer :: nCells, nCellsSolve, nISMIP6OceanLayers + integer, dimension(:), pointer :: cellMask, nEdgesOnCell + integer, dimension(:), pointer :: indexToCellID + integer, dimension(:,:), pointer :: cellsOnCell + integer :: iCell, jCell, iLayer, iNeighbor, iter + integer :: localLoopCount, newMaskCountLocalAccum + + err = 0 + + ! initialize the ocean data and mask fields + block => domain % blocklist + call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) + call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) + call mpas_pool_get_subpool(block % structs, 'extrapOceanData', extrapOceanDataPool) + call mpas_pool_get_dimension(meshPool, 'nISMIP6OceanLayers', nISMIP6OceanLayers) + call mpas_pool_get_dimension(meshPool, 'nCells', nCells) + call mpas_pool_get_dimension(meshPool, 'nCellsSolve', nCellsSolve) + call mpas_pool_get_array(meshPool, 'cellsOnCell', cellsOnCell) + call mpas_pool_get_array(meshPool, 'areaCell', areaCell) + call mpas_pool_get_array(meshPool, 'nEdgesOnCell', nEdgesOnCell) + call mpas_pool_get_array(meshPool, 'indexToCellID', indexToCellID) + call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) + call mpas_pool_get_array(geometryPool, 'thickness', thickness) + call mpas_pool_get_array(geometryPool, 'bedTopography', bedTopography) + call mpas_pool_get_array(geometryPool, 'ismip6shelfMelt_zOcean', ismip6shelfMelt_zOcean) + + ! initialize the local loop and count for validOceanMask + newMaskCountGlobal = 0 + newMaskCountLocalAccum = 0 + do iCell = 1, nCellsSolve + do iLayer = 2, nISMIP6OceanLayers + if ( (availOceanMask(iLayer,iCell) == 1) .and. (validOceanMask(iLayer,iCell) == 0) ) then + if ( validOceanMask(iLayer-1,iCell) == 1 ) then + if (TFocean(iLayer-1,iCell) > 1.0e6_RKIND) then + ! raise error if an invalid ocean data value is used + call mpas_log_write("ocean data value used for extrapolation is invalid " // & + "in vertical_extrapolation: cell id=$i, iLayer=$i, TF=$r", & + MPAS_LOG_ERR, intArgs=(/indexToCellID(iCell), iLayer-1/), realArgs=(/TFocean(iLayer-1,iCell)/)) + err = ior(err,1) + else + TFocean(iLayer,iCell) = TFocean(iLayer-1,iCell) - & + (ismip6shelfMelt_zOcean(iLayer-1) - ismip6shelfMelt_zOcean(iLayer)) * & + oceanFreezingTempDepthDependence + endif + validOceanMask(iLayer,iCell) = 1 + newMaskCountLocalAccum = newMaskCountLocalAccum + 1 + endif + endif + enddo + enddo + + ! update halo for validOceanMask and ocean data + call mpas_timer_start("halo updates") + call mpas_dmpar_field_halo_exch(domain, 'validOceanMask') + call mpas_dmpar_field_halo_exch(domain, 'TFocean') + call mpas_timer_stop("halo updates") + ! update count of cells added to mask globally + call mpas_dmpar_sum_int(domain % dminfo, newMaskCountLocalAccum, newMaskCountGlobal) + call mpas_log_write('Vertical extrap: Added total $i new cells to the validOceanMask', intArgs=(/newMaskCountGlobal/)) + + end subroutine vertical_extrapolation +!----------------------------------------------------------------------- +end module li_ocean_extrap + +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F index b722459761e..bf9616b4339 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F @@ -59,6 +59,9 @@ module li_subglacial_hydro ! Minimum gradMagPhiBaseEdge and gradMagPhiEdge allowed before all dependent variables are zeroed out real(kind=RKIND), parameter :: SMALL_GRADPHI = 1.0e-6_RKIND + !Minimum outflowing hydropotential slope applied at grounding line + real(kind=RKIND), parameter :: MIN_PHISLOPE_GL = 1e-10_RKIND + !*********************************************************************** contains @@ -107,6 +110,7 @@ subroutine li_SGH_init(domain, err) real (kind=RKIND), dimension(:), pointer :: thickness real (kind=RKIND), dimension(:), pointer :: bedTopography real (kind=RKIND), dimension(:), pointer :: iceThicknessHydro + real (kind=RKIND), dimension(:), pointer :: hydropotential integer, dimension(:), pointer :: cellMask real (kind=RKIND), pointer :: tillMax real (kind=RKIND), pointer :: rhoi, rhoo @@ -199,18 +203,25 @@ subroutine li_SGH_init(domain, err) call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) call mpas_pool_get_array(hydroPool, 'waterPressure', waterPressure) + call mpas_pool_get_array(hydroPool, 'hydropotential', hydropotential) call mpas_pool_get_array(hydroPool, 'iceThicknessHydro', iceThicknessHydro) - - waterPressure = max(0.0_RKIND, waterPressure) - waterPressure = min(waterPressure, rhoi * gravity * iceThicknessHydro) - ! set pressure correctly under floating ice and open ocean call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) call mpas_pool_get_array(geometryPool, 'bedTopography', bedTopography) - where ( (li_mask_is_floating_ice(cellMask)) .or. & - ( (.not. li_mask_is_ice(cellMask)) .and. (bedTopography < config_sea_level) ) ) - waterPressure = rhoo * gravity * (config_sea_level - bedTopography) + + waterPressure = max(0.0_RKIND, waterPressure) + where (li_mask_is_grounded_ice(cellMask)) + waterPressure = min(waterPressure, rhoi * gravity * iceThicknessHydro) end where - + + ! set pressure and hydropotential correctly on ice-free land and in ocean + where ((.not. (li_mask_is_grounded_ice(cellMask))) .and. (bedTopography > config_sea_level)) + waterPressure = 0.0_RKIND + hydropotential = rho_water * gravity * bedTopography + elsewhere ((.not. (li_mask_is_grounded_ice(cellMask))) .and. (bedTopography <= config_sea_level)) + waterPressure = gravity * rhoo * (config_sea_level - bedTopography) + hydropotential = 0.0_RKIND + end where + ! Initialize diagnostic pressure variables call calc_pressure_diag_vars(block, err_tmp) err = ior(err, err_tmp) @@ -303,6 +314,8 @@ subroutine li_SGH_solve(domain, err) real (kind=RKIND), dimension(:), pointer :: waterVelocity real (kind=RKIND), dimension(:), pointer :: waterVelocityCellX real (kind=RKIND), dimension(:), pointer :: waterVelocityCellY + real (kind=RKIND), dimension(:), pointer :: effectivePressureSGH + real (kind=RKIND), dimension(:), pointer :: effectivePressure real (kind=RKIND), dimension(:), allocatable :: cellJunk integer, dimension(:), pointer :: nEdgesOnCell integer, dimension(:,:), pointer :: edgesOnCell @@ -317,6 +330,7 @@ subroutine li_SGH_solve(domain, err) integer, pointer :: nCells integer :: iCell, iEdge, iEdgeOnCell real (kind=RKIND) :: timeLeft ! subcycling time remaining until master dt is reached + real (kind=RKIND) :: deltatSGHaccumulated integer :: numSubCycles ! number of subcycles integer :: err_tmp @@ -370,6 +384,19 @@ subroutine li_SGH_solve(domain, err) call mpas_pool_get_array(meshPool, 'deltat', masterDeltat) timeLeft = masterDeltat ! in seconds numSubCycles = 0 + + block => domain % blocklist + do while (associated(block)) + + call mpas_pool_get_subpool(block % structs, 'hydro', hydroPool) + call mpas_pool_get_array(hydroPool, 'effectivePressure', effectivePressure) + effectivePressure = 0.0_RKIND + deltatSGHaccumulated = 0.0_RKIND + + block => block % next + end do + + ! ============= ! ============= ! ============= @@ -669,7 +696,22 @@ subroutine li_SGH_solve(domain, err) call mpas_dmpar_field_halo_exch(domain, 'waterPressureSmooth') call mpas_timer_stop("halo updates") + !Average effectivePressureSGH over coupling interval for use in dynamics model + block => domain % blocklist + do while (associated(block)) + + call mpas_pool_get_subpool(block % structs, 'hydro', hydroPool) + call mpas_pool_get_array(hydroPool, 'effectivePressureSGH', effectivePressureSGH) + call mpas_pool_get_array(hydroPool, 'effectivePressure', effectivePressure) + call mpas_pool_get_array(hydroPool, 'deltatSGH', deltatSGH) + + effectivePressure = (effectivePressure * deltatSGHaccumulated + effectivePressureSGH * deltatSGH) / (deltatSGHaccumulated + deltatSGH) + + block => block % next + end do + deltatSGHaccumulated = deltatSGHaccumulated + deltatSGH + ! ============= ! ============= ! ============= @@ -821,6 +863,7 @@ subroutine calc_edge_quantities(block, err) real (kind=RKIND), dimension(:), pointer :: waterFluxDiffu real (kind=RKIND), dimension(:), pointer :: waterPressureSmooth integer, dimension(:), pointer :: hydroMarineMarginMask + integer, dimension(:), pointer :: hydroTerrestrialMarginMask integer, dimension(:), pointer :: waterFluxMask integer, dimension(:,:), pointer :: edgeSignOnCell integer, dimension(:), pointer :: cellMask @@ -897,6 +940,7 @@ subroutine calc_edge_quantities(block, err) call mpas_pool_get_array(hydroPool, 'waterFluxDiffu', waterFluxDiffu) call mpas_pool_get_array(hydroPool, 'waterFluxMask', waterFluxMask) call mpas_pool_get_array(hydroPool, 'hydroMarineMarginMask', hydroMarineMarginMask) + call mpas_pool_get_array(hydroPool, 'hydroTerrestrialMarginMask', hydroTerrestrialMarginMask) call mpas_pool_get_array(geometryPool, 'edgeMask', edgeMask) call mpas_pool_get_array(hydroPool, 'waterPressureSmooth', waterPressureSmooth) @@ -920,53 +964,58 @@ subroutine calc_edge_quantities(block, err) waterPressureSlopeNormal(iEdge) = (waterPressureSmooth(cell2) - waterPressureSmooth(cell1)) / dcEdge(iEdge) end do - ! At boundaries of hydro domain, disallow inflow. Allow outflow if hydropotential gradient requires it. + ! At terrestrial margin, ignore the downslope bed topography gradient. Including it can lead to unrealistically large + ! hydropotential gradients and unstable channel growth. + ! The hydropotential at the terrestrial margin should be determined by the geometry + ! at the edge of the cell in a 1-sided sense. For hydropotentialBase = rho*g*Zb + Pw, this means hydropotentialBaseSlopeNormal + ! at the terrestrial margin is equal to Pw/dcEdge. + ! This one-sided implementation also creates outflowing conditions at terrestrial boundary do iEdge = 1, nEdges - if ( (li_mask_is_margin(edgeMask(iEdge)) .and. li_mask_is_grounded_ice(edgeMask(iEdge))) .or. & - (hydroMarineMarginMask(iEdge)==1)) then + if (hydroTerrestrialMarginMask(iEdge) == 1) then cell1 = cellsOnEdge(1, iEdge) cell2 = cellsOnEdge(2, iEdge) - if (li_mask_is_grounded_ice(cellMask(cell1))) then ! cell2 is the cell outside the hydro domain - hydropotentialBaseSlopeNormal(iEdge) = min(0.0_RKIND, hydropotentialBaseSlopeNormal(iEdge)) - hydropotentialSlopeNormal(iEdge) = min(0.0_RKIND, hydropotentialSlopeNormal(iEdge)) - else ! cell1 is the cell outside the hydro domain - hydropotentialBaseSlopeNormal(iEdge) = max(0.0_RKIND, hydropotentialBaseSlopeNormal(iEdge)) - hydropotentialSlopeNormal(iEdge) = max(0.0_RKIND, hydropotentialSlopeNormal(iEdge)) + if (li_mask_is_grounded_ice(cellMask(cell1))) then ! cell2 is the icefree cell - replace phi there with cell1 Phig + + hydropotentialBaseSlopeNormal(iEdge) = - waterPressure(cell1) / dcEdge(iEdge) + hydropotentialSlopeNormal(iEdge) = - (rho_water * gravity * waterThickness(cell1) + waterPressure(cell1)) / dcEdge(iEdge) + + else ! cell1 is the icefree cell - replace phi there with cell2 Phig + + hydropotentialBaseSlopeNormal(iEdge) = waterPressure(cell2) / dcEdge(iEdge) + hydropotentialSlopeNormal(iEdge) = (rho_water * gravity * waterThickness(cell2) + waterPressure(cell2)) / dcEdge(iEdge) + endif ! which cell is icefree endif ! if edge of grounded ice end do - ! At terrestrial margin, ignore the downslope bed topography gradient. Including it can lead to unrealistically large - ! hydropotential gradients and unstable channel growth. - ! We also want to do this at marine margins because otherwise the offshore topography can create a barrier to flow, - ! but that is unrealistic. - ! So for all boundaries of the hydro system where outflow is occuring, - ! the hydropotential at the margin should be determined by the geometry - ! at the edge of the cell in a 1-sided sense. + ! Disallow inflow from the marine margin by imposing a minimum outflowing hydropotential gradient at the grounding line. do iEdge = 1, nEdges - if ( (li_mask_is_margin(edgeMask(iEdge)) .and. li_mask_is_grounded_ice(edgeMask(iEdge))) .or. & - (hydroMarineMarginMask(iEdge)==1)) then + if ( hydroMarineMarginMask(iEdge) == 1) then cell1 = cellsOnEdge(1, iEdge) cell2 = cellsOnEdge(2, iEdge) - if (li_mask_is_grounded_ice(cellMask(cell1))) then ! cell2 is the icefree cell - replace phi there with cell1 Phig - hydropotentialBaseSlopeNormal(iEdge) = (rho_water * gravity * bedTopography(cell1) + & - max(rhoo * gravity * (config_sea_level - bedTopography(cell1)), 0.0_RKIND) - & - hydropotentialBase(cell1)) / dcEdge(iEdge) - hydropotentialSlopeNormal(iEdge) = (rho_water * gravity * bedTopography(cell1) + & - max(rhoo * gravity * (config_sea_level - bedTopography(cell1)), 0.0_RKIND) - & - hydropotential(cell1)) / dcEdge(iEdge) - else ! cell1 is the icefree cell - replace phi there with cell2 Phig - hydropotentialBaseSlopeNormal(iEdge) = (hydropotentialBase(cell2) - & - ( rho_water * gravity * bedTopography(cell2) + & - max(rhoo * gravity * (config_sea_level - bedTopography(cell2)), 0.0_RKIND) ) ) / dcEdge(iEdge) - hydropotentialSlopeNormal(iEdge) = (hydropotential(cell2) - & - ( rho_water * gravity * bedTopography(cell2) + & - max(rhoo * gravity * (config_sea_level - bedTopography(cell2)), 0.0_RKIND) ) ) / dcEdge(iEdge) + if (li_mask_is_grounded_ice(cellMask(cell1))) then ! cell2 is the cell outside the hydro domain + + if (hydropotentialBaseSlopeNormal(iEdge) > -MIN_PHISLOPE_GL) then + hydropotentialBaseSlopeNormal(iEdge) = -MIN_PHISLOPE_GL + endif + if (hydropotentialSlopeNormal(iEdge) > -MIN_PHISLOPE_GL) then + hydropotentialSlopeNormal(iEdge) = -MIN_PHISLOPE_GL + endif + + else ! cell1 is the cell outside the hydro domain + + if (hydropotentialBaseSlopeNormal(iEdge) < MIN_PHISLOPE_GL) then + hydropotentialBaseSlopeNormal(iEdge) = MIN_PHISLOPE_GL + endif + if (hydropotentialSlopeNormal(iEdge) < MIN_PHISLOPE_GL) then + hydropotentialSlopeNormal(iEdge) = MIN_PHISLOPE_GL + endif + endif ! which cell is icefree endif ! if edge of grounded ice end do - ! zero gradients at edges that are marked as no flux. These should be applied at boundaries of the mesh. + ! zero gradients along zero flux boundaries do iEdge = 1, nEdges if (waterFluxMask(iEdge) == 2) then hydropotentialBaseSlopeNormal(iEdge) = 0.0_RKIND @@ -1516,7 +1565,7 @@ subroutine calc_pressure(block, err) real (kind=RKIND), dimension(:), pointer :: waterPressureOld real (kind=RKIND), dimension(:), pointer :: waterPressureTendency real (kind=RKIND), dimension(:), pointer :: waterThickness - real (kind=RKIND), dimension(:), pointer :: effectivePressure + real (kind=RKIND), dimension(:), pointer :: effectivePressureSGH real (kind=RKIND), dimension(:), pointer :: zeroOrderSum real (kind=RKIND), dimension(:), pointer :: closingRate real (kind=RKIND), dimension(:), pointer :: openingRate @@ -1575,7 +1624,7 @@ subroutine calc_pressure(block, err) call mpas_pool_get_array(meshPool, 'nEdgesOnCell', nEdgesOnCell) call mpas_pool_get_array(meshPool, 'edgesOnCell', edgesOnCell) - call mpas_pool_get_array(hydroPool, 'effectivePressure', effectivePressure) + call mpas_pool_get_array(hydroPool, 'effectivePressureSGH', effectivePressureSGH) call mpas_pool_get_array(hydroPool, 'waterPressure', waterPressure) call mpas_pool_get_array(hydroPool, 'waterPressureOld', waterPressureOld) call mpas_pool_get_array(hydroPool, 'waterPressureTendency', waterPressureTendency) @@ -1605,8 +1654,8 @@ subroutine calc_pressure(block, err) openingRate = max(0.0_RKIND, openingRate) closingRate(:) = creepCoeff * flowParamA(nVertLevels, :) * & - effectivePressure(:)**3 * waterThickness(:) -! closingRate(:) = waterThickness(:) * effectivePressure(:) / 1.0e13_RKIND + effectivePressureSGH(:)**3 * waterThickness(:) +! closingRate(:) = waterThickness(:) * effectivePressureSGH(:) / 1.0e13_RKIND ! ! Hewitt 2011 creep closure form. Denominator is ice viscosity zeroOrderSum = closingRate - openingRate + (basalMeltInput + externalWaterInput) / rho_water - & @@ -1617,22 +1666,23 @@ subroutine calc_pressure(block, err) select case (trim(config_SGH_pressure_calc)) case ('cavity') - where (li_mask_is_floating_ice(cellMask)) - waterPressure = rhoi * gravity * iceThicknessHydro - elsewhere (.not. li_mask_is_ice(cellMask)) - waterPressure = 0.0_RKIND - elsewhere + where (li_mask_is_grounded_ice(cellMask)) waterPressure = (zeroOrderSum - divergence - divergenceChannel - channelAreaChangeCell) * & - rho_water * gravity * deltatSGH / porosity + waterPressureOld + rho_water * gravity * deltatSGH / porosity + waterPressureOld + elsewhere ((.not. (li_mask_is_grounded_ice(cellMask))) .and. (bedTopography > config_sea_level)) + waterPressure = 0.0_RKIND + elsewhere ! should evaluate to ((.not. (li_mask_is_grounded_ice(cellMask))) .and. (bedTopography <= config_sea_level)) + waterPressure = gravity * rhoo * (config_sea_level - bedTopography) end where case ('overburden') - where (li_mask_is_floating_ice(cellMask)) + + where (li_mask_is_grounded_ice(cellMask)) waterPressure = rhoi * gravity * iceThicknessHydro - elsewhere (.not. li_mask_is_ice(cellMask)) + elsewhere ((.not. (li_mask_is_grounded_ice(cellMask))) .and. (bedTopography > config_sea_level)) waterPressure = 0.0_RKIND - elsewhere - waterPressure = rhoi * gravity * iceThicknessHydro + elsewhere ! should evaluate to ((.not. (li_mask_is_grounded_ice(cellMask))) .and. (bedTopography <= config_sea_level)) + waterPressure = gravity * rhoo * (config_sea_level - bedTopography) end where case default @@ -1641,27 +1691,23 @@ subroutine calc_pressure(block, err) end select waterPressure = max(0.0_RKIND, waterPressure) - waterPressure = min(waterPressure, rhoi * gravity * iceThicknessHydro) + where (li_mask_is_grounded_ice(cellMask)) + waterPressure = min(waterPressure, rhoi * gravity * iceThicknessHydro) + end where do iCell = 1, nCells - if ( li_mask_is_floating_ice(cellMask(iCell)) .or. & - ((.not. li_mask_is_ice(cellMask(iCell))) .and. (bedTopography(iCell) < config_sea_level) ) ) then - ! set pressure correctly under floating ice and open ocean - waterPressure(iCell) = rhoo * gravity * (config_sea_level - bedTopography(iCell)) - else - onMarineMargin = .false. - do iEdge = 1, nEdgesOnCell(iCell) - if (hydroMarineMarginMask(edgesOnCell(iEdge, iCell)) == 1) then - onMarineMargin = .true. - exit - endif - enddo - if (onMarineMargin) then - ! At marine margin, don't let water pressure fall below ocean pressure - ! TODO: Not sure if this should include the water layer thickness term. Leaving it off. - if (waterPressure(iCell) < rhoo * gravity * (config_sea_level - bedTopography(iCell))) then - waterPressure(iCell) = rhoo * gravity * (config_sea_level - bedTopography(iCell)) - endif + onMarineMargin = .false. + do iEdge = 1, nEdgesOnCell(iCell) + if (hydroMarineMarginMask(edgesOnCell(iEdge, iCell)) == 1) then + onMarineMargin = .true. + exit + endif + enddo + if (onMarineMargin) then + ! At marine margin, don't let water pressure fall below ocean pressure + ! TODO: Not sure if this should include the water layer thickness term. Leaving it off. + if (waterPressure(iCell) < rho_water * gravity * (config_sea_level - bedTopography(iCell))) then + waterPressure(iCell) = rho_water * gravity * (config_sea_level - bedTopography(iCell)) endif endif enddo @@ -1712,7 +1758,7 @@ subroutine calc_pressure_diag_vars(block, err) real (kind=RKIND), dimension(:), pointer :: hydropotentialBase real (kind=RKIND), dimension(:), pointer :: waterThickness real (kind=RKIND), dimension(:), pointer :: hydropotential - real (kind=RKIND), dimension(:), pointer :: effectivePressure + real (kind=RKIND), dimension(:), pointer :: effectivePressureSGH real (kind=RKIND), dimension(:), pointer :: iceThicknessHydro integer, dimension(:), pointer :: cellMask real (kind=RKIND), pointer :: config_sea_level @@ -1727,7 +1773,7 @@ subroutine calc_pressure_diag_vars(block, err) call mpas_pool_get_config(liConfigs, 'config_sea_level', config_sea_level) call mpas_pool_get_config(liConfigs, 'config_ocean_density', rhoo) - call mpas_pool_get_array(hydroPool, 'effectivePressure', effectivePressure) + call mpas_pool_get_array(hydroPool, 'effectivePressureSGH', effectivePressureSGH) call mpas_pool_get_array(hydroPool, 'waterPressure', waterPressure) call mpas_pool_get_array(geometryPool, 'bedTopography', bedTopography) call mpas_pool_get_array(hydroPool, 'hydropotentialBase', hydropotentialBase) @@ -1736,17 +1782,18 @@ subroutine calc_pressure_diag_vars(block, err) call mpas_pool_get_array(hydroPool, 'iceThicknessHydro', iceThicknessHydro) call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) - effectivePressure = rhoi * gravity * iceThicknessHydro - waterPressure + effectivePressureSGH = rhoi * gravity * iceThicknessHydro - waterPressure ! < this should evalute to 0 for floating ice if Pw set correctly there. - where (.not. li_mask_is_grounded_ice(cellmask)) - effectivePressure = 0.0_RKIND ! zero effective pressure where no ice to avoid confusion + where (.not. (li_mask_is_grounded_ice(cellMask))) + effectivePressureSGH = 0.0_RKIND ! zero effective pressure where no ice to avoid confusion end where - + hydropotentialBase = rho_water * gravity * bedTopography + waterPressure - ! This is still correct under ice shelves/open ocean because waterPressure has been set appropriately there already. - ! Note this leads to a nonuniform hydropotential at sea level that is a function of the ocean depth. - ! That is what we want because we use this as a boundary condition on the subglacial system, - ! and we want the subglacial system to feel the pressure of the ocean column at its edge. + where ((.not. li_mask_is_grounded_ice(cellMask)) .and. (bedTopography <= config_sea_level)) + hydropotentialBase = 0.0_RKIND !zero hydropotential in ocean + elsewhere ((.not. li_mask_is_grounded_ice(cellMask)) .and. (bedTopography > config_sea_level)) + hydropotentialBase = rho_water * gravity * bedTopography ! for completeness, but won't matter with one-side slope calculations on terrestrial boundaries + end where ! hydropotential with water thickness hydropotential = hydropotentialBase + rho_water * gravity * waterThickness @@ -1812,8 +1859,9 @@ subroutine update_channel(block, err) real (kind=RKIND), dimension(:), pointer :: channelChangeRate real (kind=RKIND), dimension(:), pointer :: flowParamAChannel real (kind=RKIND), dimension(:), pointer :: channelEffectivePressure - real (kind=RKIND), dimension(:), pointer :: effectivePressure + real (kind=RKIND), dimension(:), pointer :: effectivePressureSGH real (kind=RKIND), dimension(:), pointer :: channelDiffusivity + real (kind=RKIND), dimension(:), pointer :: waterThicknessEdgeUpwind integer, dimension(:), pointer :: waterFluxMask integer, dimension(:), pointer :: hydroMarineMarginMask integer, dimension(:), pointer :: edgeMask @@ -1855,10 +1903,11 @@ subroutine update_channel(block, err) call mpas_pool_get_array(hydroPool, 'channelEffectivePressure', channelEffectivePressure) call mpas_pool_get_array(meshPool, 'cellsOnEdge', cellsOnEdge) call mpas_pool_get_array(velocityPool, 'flowParamA', flowParamA) - call mpas_pool_get_array(hydroPool, 'effectivePressure', effectivePressure) + call mpas_pool_get_array(hydroPool, 'effectivePressureSGH', effectivePressureSGH) call mpas_pool_get_array(hydroPool, 'waterFluxMask', waterFluxMask) call mpas_pool_get_array(hydroPool, 'hydroMarineMarginMask', hydroMarineMarginMask) call mpas_pool_get_array(hydroPool, 'channelDiffusivity', channelDiffusivity) + call mpas_pool_get_array(hydroPool, 'waterThicknessEdgeUpwind', waterThicknessEdgeUpwind) call mpas_pool_get_array(geometryPool, 'edgeMask', edgeMask) ! Calculate terms needed for opening (melt) rate @@ -1881,9 +1930,8 @@ subroutine update_channel(block, err) channelDischarge = 0.0_RKIND end where - ! Disable channels from forming if there is no sheet flux - ! TODO: Make a function of sheet dissipation threshold? - where (abs(waterFlux) <= 1e-10_RKIND) + ! Similar to waterFlux, shut off channelDischarge if no water upstream. Prevents self-sustaining channels in the absence of distributed water + where (waterThicknessEdgeUpwind == 0.0_RKIND) channelArea = 0.0_RKIND channelDischarge = 0.0_RKIND end where @@ -1919,7 +1967,7 @@ subroutine update_channel(block, err) ! Not sure if these ought to be upwind average, but using centered flowParamAChannel(iEdge) = 0.5_RKIND * (flowParamA(nVertLevels, cell1) + flowParamA(nVertLevels, cell2) ) - channelEffectivePressure(iEdge) = 0.5_RKIND * (effectivePressure(cell1) + effectivePressure(cell2)) + channelEffectivePressure(iEdge) = 0.5_RKIND * (effectivePressureSGH(cell1) + effectivePressureSGH(cell2)) end do channelClosingRate(:) = creep_coeff * channelArea(:) * flowParamAChannel(:) * channelEffectivePressure(:)**3 diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_time_integration_fe_rk.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_time_integration_fe_rk.F index e4c58cc42b1..e6051a3ec75 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_time_integration_fe_rk.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_time_integration_fe_rk.F @@ -39,6 +39,7 @@ module li_time_integration_fe_rk use li_constants use li_mesh use li_mask + use li_tracer_advection_fct_shared implicit none private @@ -87,6 +88,7 @@ subroutine li_time_integrator_forwardeuler_rungekutta(domain, err) use li_bedtopo use li_mask use li_advection, only: li_grounded_to_floating + use li_ocean_extrap !----------------------------------------------------------------- ! input variables @@ -114,7 +116,8 @@ subroutine li_time_integrator_forwardeuler_rungekutta(domain, err) logical, pointer :: config_calculate_damage logical, pointer :: config_finalize_damage_after_advection logical, pointer :: config_update_velocity_before_calving - character (len=StrKIND), pointer :: config_thickness_advection + character (len=StrKIND), pointer :: config_thickness_advection, & + config_tracer_advection character (len=StrKIND), pointer :: config_thermal_solver character (len=StrKIND), pointer :: config_time_integration integer, pointer :: config_rk_order, config_rk3_stages @@ -166,9 +169,10 @@ subroutine li_time_integrator_forwardeuler_rungekutta(domain, err) call mpas_pool_get_config(liConfigs, 'config_restore_calving_front', config_restore_calving_front) call mpas_pool_get_config(liConfigs, 'config_restore_calving_front_prevent_retreat', config_restore_calving_front_prevent_retreat) - call mpas_pool_get_config(liConfigs, 'config_calculate_damage',config_calculate_damage) + call mpas_pool_get_config(liConfigs, 'config_calculate_damage', config_calculate_damage) call mpas_pool_get_config(liConfigs, 'config_finalize_damage_after_advection', config_finalize_damage_after_advection) call mpas_pool_get_config(liConfigs, 'config_thickness_advection', config_thickness_advection) + call mpas_pool_get_config(liConfigs, 'config_tracer_advection', config_tracer_advection) call mpas_pool_get_config(liConfigs, 'config_thermal_solver', config_thermal_solver) call mpas_pool_get_config(liConfigs, 'config_rk_order', config_rk_order) call mpas_pool_get_config(liConfigs, 'config_rk3_stages', config_rk3_stages) @@ -241,14 +245,18 @@ subroutine li_time_integrator_forwardeuler_rungekutta(domain, err) call mpas_timer_stop("advancing clock") !TODO: Determine whether grounded melting should in fact be called first +! === Ocean forcing extrapolation into ice-shelf cavities =========== + call mpas_timer_start("ocean forcing extrapolation") + call li_ocean_extrap_solve(domain, err_tmp) + err = ior(err, err_tmp) + call mpas_timer_stop("ocean forcing extrapolation") + ! === Face melting for grounded ice =========== call mpas_timer_start("face melting for grounded ice") call li_face_melt_grounded_ice(domain, err_tmp) err = ior(err, err_tmp) call mpas_timer_stop("face melting for grounded ice") -! *** TODO: Should basal melt rate calculation and column physics go inside RK loop? *** - ! === Basal melting for floating ice =========== call mpas_timer_start("basal melting for floating ice") call li_basal_melt_floating_ice(domain, err_tmp) @@ -281,6 +289,17 @@ subroutine li_time_integrator_forwardeuler_rungekutta(domain, err) enddo deltatFull = deltat ! Save deltat in order to reset it at end of RK loop + if ( (trim(config_tracer_advection) == 'fct') .or. & + (trim(config_thickness_advection) == 'fct') ) then + call li_tracer_advection_fct_shared_init(geometryPool, err_tmp) + if (err_tmp /= 0) then + err = 1 + call mpas_log_write( & + 'Error encountered during fct tracer advection shared init', & + MPAS_LOG_ERR) + endif + endif + ! Set RK weights based on desired time integration method. Note ! that rkSubstepWeights are used to update at each sub-step, and ! are thus offset from the typical writing of the coefficients @@ -350,7 +369,13 @@ subroutine li_time_integrator_forwardeuler_rungekutta(domain, err) // ' is not supported with config_rk_order = $i', & intArgs=(/config_rk_order/), messageType=MPAS_LOG_ERR) return - endif + endif + + ! Calculate masks prior to RK loop, but do not update masks within the loop + ! to preserve the accuracy of time integration. + call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) + err = ior(err, err_tmp) + ! *** Start RK loop *** do rkStage = 1, nRKstages call mpas_log_write('beginning rk stage $i of $i', & @@ -387,9 +412,8 @@ subroutine li_time_integrator_forwardeuler_rungekutta(domain, err) layerThickness(:,:) = rkSSPweights(rkStage) * layerThicknessPrev(:,:) + & (1.0_RKIND - rkSSPweights(rkStage)) * layerThickness(:,:) thickness = sum(layerThickness, 1) - ! Calculate masks after updating thickness - call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) - err = ior(err, err_tmp) + ! Do not calculate masks after updating thickness! We need to keep masks + ! constant for now to preserve accuracy of time integration if (trim(config_thermal_solver) .ne. 'none') then do iCell = 1, nCells @@ -521,6 +545,17 @@ subroutine li_time_integrator_forwardeuler_rungekutta(domain, err) fluxAcrossGroundingLine(:) = fluxAcrossGroundingLineAccum(:) fluxAcrossGroundingLineOnCells(:) = fluxAcrossGroundingLineOnCellsAccum(:) + ! Deallocate arrays for fct + if ( (trim(config_thickness_advection) .eq. 'fct') .or. & + (trim(config_tracer_advection) .eq. 'fct') ) then + deallocate( nAdvCellsForEdge, & + advCellsForEdge, & + advCoefs, & + advCoefs3rd, & + advMaskHighOrder, & + advMask2ndOrder) + endif + ! Reset time step to full length after RK loop deltat = deltatFull @@ -1010,6 +1045,11 @@ subroutine advection_solver(domain, err) real (kind=RKIND), dimension(:), pointer :: thicknessNew real (kind=RKIND), dimension(:), pointer :: thickness real (kind=RKIND), dimension(:,:), pointer :: layerThickness + real (kind=RKIND), dimension(:,:), pointer :: normalVelocity + real (kind=RKIND), dimension(:,:), pointer :: layerNormalVelocity + + integer, dimension(:), pointer :: edgeMask + real (kind=RKIND) :: allowableDtACFL real (kind=RKIND), dimension(:,:), pointer :: temperature real (kind=RKIND), dimension(:,:), pointer :: waterFrac @@ -1020,6 +1060,7 @@ subroutine advection_solver(domain, err) character (len=StrKIND), pointer :: config_thickness_advection logical, pointer :: config_restore_thickness_after_advection character (len=StrKIND), pointer :: config_tracer_advection + character (len=StrKIND), pointer :: config_time_integration logical, pointer :: config_print_thickness_advection_info @@ -1033,6 +1074,7 @@ subroutine advection_solver(domain, err) call mpas_pool_get_config(liConfigs, 'config_thickness_advection', config_thickness_advection) call mpas_pool_get_config(liConfigs, 'config_tracer_advection', config_tracer_advection) + call mpas_pool_get_config(liConfigs, 'config_time_integration', config_time_integration) call mpas_pool_get_config(liConfigs, 'config_print_thickness_advection_info', config_print_thickness_advection_info) call mpas_pool_get_config(liConfigs, 'config_restore_thickness_after_advection', config_restore_thickness_after_advection) @@ -1059,7 +1101,20 @@ subroutine advection_solver(domain, err) call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) call mpas_pool_get_subpool(block % structs, 'velocity', velocityPool) call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) - + if (trim(config_time_integration) == "runge_kutta") then + call mpas_pool_get_array(velocityPool, 'normalVelocity', normalVelocity) + call mpas_pool_get_array(geometryPool, 'edgeMask', edgeMask) + call mpas_pool_get_array(velocityPool, 'layerNormalVelocity', layerNormalVelocity) + + call li_layer_normal_velocity( & + meshPool, & + normalVelocity, & + edgeMask, & + layerNormalVelocity, & + allowableDtACFL, & + err_tmp) + err = ior(err,err_tmp) + endif call calculate_layerThicknessEdge(meshPool, geometryPool, velocityPool, err_tmp) err = ior(err,err_tmp) diff --git a/components/mpas-framework/src/framework/mpas_forcing.F b/components/mpas-framework/src/framework/mpas_forcing.F index 950b103604c..95de2ca6184 100644 --- a/components/mpas-framework/src/framework/mpas_forcing.F +++ b/components/mpas-framework/src/framework/mpas_forcing.F @@ -35,7 +35,8 @@ module mpas_forcing mpas_forcing_init_field_data, & mpas_forcing_get_forcing, & mpas_forcing_get_forcing_time, & - mpas_forcing_write_restart_times + mpas_forcing_write_restart_times, & + mpas_advance_forcing_clock contains @@ -1193,7 +1194,7 @@ subroutine mpas_forcing_get_forcing(&!{{{ if (trim(forcingGroup % forcingGroupName) == trim(forcingGroupName)) then ! advance the forcing time - call advance_forcing_clock(forcingGroup, dt) + call mpas_advance_forcing_clock(forcingGroup, dt) ! cycle the forcing clock if (forcingGroup % forcingCycleUse) then @@ -1230,7 +1231,7 @@ end subroutine mpas_forcing_get_forcing!}}} !||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| ! -! advance_forcing_clock +! mpas_advance_forcing_clock ! !> \brief set the forcing clock !> \author Adrian K. Turner, LANL @@ -1241,7 +1242,7 @@ end subroutine mpas_forcing_get_forcing!}}} ! !----------------------------------------------------------------------- - subroutine advance_forcing_clock(&!{{{ + subroutine mpas_advance_forcing_clock(&!{{{ forcingGroup, & dt) @@ -1256,9 +1257,10 @@ subroutine advance_forcing_clock(&!{{{ ! increment clock with timestep call mpas_set_timeInterval(timeStep, dt=dt) + !call mpas_log_write('time step from mpas_set_timeInterval: $i $i $i', intArgs=(/int(timeStep%ti%basetime%S), int(timeStep%ti%basetime%Sn), int(timeStep%ti%basetime%Sd) /)) call mpas_advance_clock(forcingGroup % forcingClock, timeStep) - end subroutine advance_forcing_clock!}}} + end subroutine mpas_advance_forcing_clock!}}} !||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| ! @@ -1597,8 +1599,10 @@ subroutine get_interpolants_linear(interpolants, forcingStream, currentTime)!{{{ call mpas_get_timeInterval(diff1, forcingStream % forcingTimes(1), dt=diffr1) call mpas_get_timeInterval(diff2, currentTime, dt=diffr2) + !call mpas_log_write('diffr2 $r, diffr, $r', realArgs=(/ diffr2, diffr /)) interpolants(1) = diffr2 / diffr interpolants(2) = 1.0_RKIND - interpolants(1) !diffr1 / diffr + end subroutine get_interpolants_linear!}}} @@ -1612,7 +1616,7 @@ end subroutine get_interpolants_linear!}}} !> \details !> Given the current time and forcing times calculate the correct !> interpolants with piecewise constant interpolation -! + !----------------------------------------------------------------------- subroutine get_interpolants_constant(interpolants, forcingStream, currentTime)!{{{ diff --git a/components/mpas-framework/src/framework/mpas_timekeeping.F b/components/mpas-framework/src/framework/mpas_timekeeping.F index 37a24d77358..57d9651335a 100644 --- a/components/mpas-framework/src/framework/mpas_timekeeping.F +++ b/components/mpas-framework/src/framework/mpas_timekeeping.F @@ -1427,6 +1427,8 @@ subroutine mpas_set_timeInterval(interval, YY, MM, DD, H, M, S, S_n, S_d, S_i8, timeString_ = timeString end if + !call mpas_log_write('timeString_ '//trim(timeString_)) + numerator = 0 denominator = 1 @@ -1519,6 +1521,12 @@ subroutine mpas_set_timeInterval(interval, YY, MM, DD, H, M, S, S_n, S_d, S_i8, return end if + if (sec < 0) then + numerator = -numerator + end if + + !call mpas_log_write('sec $i, num $i, denom $i', intArgs=(/int(sec),numerator,denominator/)) + call ESMF_TimeIntervalSet(interval % ti, YY=year, MM=month, D=day, H=hour, M=min, S_i8=sec, Sn=numerator, Sd=denominator, rc=ierr) else diff --git a/components/mpas-ocean/bld/build-namelist b/components/mpas-ocean/bld/build-namelist index e38ba077228..f2050753b5a 100755 --- a/components/mpas-ocean/bld/build-namelist +++ b/components/mpas-ocean/bld/build-namelist @@ -740,6 +740,10 @@ if (($OCN_ISMF ne 'none') && ($OCN_FORCING ne 'active_atm')) { } else { add_default($nl, 'config_remove_ais_river_runoff', 'val'=>".false."); } +add_default($nl, 'config_scale_dismf_by_removed_ice_runoff'); +add_default($nl, 'config_ais_ice_runoff_history_days'); +add_default($nl, 'config_glc_thermal_forcing_coupling_mode'); +add_default($nl, 'config_2d_thermal_forcing_depth'); ###################################### # Namelist group: shortwaveRadiation # @@ -789,7 +793,6 @@ add_default($nl, 'config_self_attraction_and_loading_beta'); add_default($nl, 'config_use_frazil_ice_formation'); add_default($nl, 'config_frazil_in_open_ocean'); -add_default($nl, 'config_frazil_under_land_ice'); add_default($nl, 'config_frazil_heat_of_fusion'); add_default($nl, 'config_frazil_ice_density'); add_default($nl, 'config_frazil_fractional_thickness_limit'); @@ -799,18 +802,32 @@ add_default($nl, 'config_frazil_sea_ice_reference_salinity'); add_default($nl, 'config_frazil_maximum_freezing_temperature'); add_default($nl, 'config_frazil_use_surface_pressure'); +if ($OCN_ISMF eq 'coupled') { + add_default($nl, 'config_frazil_under_land_ice', 'val'=>".true."); +} elsif ($OCN_ISMF eq 'internal') { + add_default($nl, 'config_frazil_under_land_ice', 'val'=>".true."); +} elsif ($OCN_ISMF eq 'data') { + add_default($nl, 'config_frazil_under_land_ice', 'val'=>".false."); +} else { + add_default($nl, 'config_frazil_under_land_ice'); +} + ################################### # Namelist group: land_ice_fluxes # ################################### if ($OCN_ISMF eq 'coupled') { add_default($nl, 'config_land_ice_flux_mode', 'val'=>"coupled"); + add_default($nl, 'config_frazil_under_land_ice', 'val'=>".true."); } elsif ($OCN_ISMF eq 'internal') { add_default($nl, 'config_land_ice_flux_mode', 'val'=>"standalone"); + add_default($nl, 'config_frazil_under_land_ice', 'val'=>".true."); } elsif ($OCN_ISMF eq 'data') { add_default($nl, 'config_land_ice_flux_mode', 'val'=>"data"); + add_default($nl, 'config_frazil_under_land_ice', 'val'=>".false."); } else { add_default($nl, 'config_land_ice_flux_mode'); + add_default($nl, 'config_frazil_under_land_ice'); } if ($OCN_TIDAL_MIXING eq 'true') { add_default($nl, 'config_land_ice_flux_tidal_Jourdain_alpha', 'val'=>"0.777"); diff --git a/components/mpas-ocean/bld/build-namelist-section b/components/mpas-ocean/bld/build-namelist-section index c5dad5d935a..74c699d878b 100644 --- a/components/mpas-ocean/bld/build-namelist-section +++ b/components/mpas-ocean/bld/build-namelist-section @@ -239,6 +239,10 @@ add_default($nl, 'config_sgr_salinity_prescribed'); add_default($nl, 'config_remove_ais_river_runoff'); add_default($nl, 'config_remove_ais_ice_runoff'); +add_default($nl, 'config_scale_dismf_by_removed_ice_runoff'); +add_default($nl, 'config_ais_ice_runoff_history_days'); +add_default($nl, 'config_glc_thermal_forcing_coupling_mode'); +add_default($nl, 'config_2d_thermal_forcing_depth'); ###################################### # Namelist group: shortwaveRadiation # diff --git a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml index f1ab05d2207..73d4eeecd52 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_defaults_mpaso.xml @@ -364,6 +364,10 @@ .false. .false. +.false. +731 +'off' +300.0 'jerlov' @@ -401,7 +405,7 @@ .true. .true. -.true. +.false. 3.337e5 1000.0 0.1 @@ -1138,6 +1142,7 @@ .false. +.true. .true. '0000-00-00_01:00:00' 'eddyProductVariablesOutput' @@ -1251,10 +1256,13 @@ .false. +.true. 'dt' 'conservationCheckOutput' .false. +.true. .false. +.true. .true. 'conservationCheckRestart' diff --git a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml index bea1e98d9de..7a5e8bf1dda 100644 --- a/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml +++ b/components/mpas-ocean/bld/namelist_files/namelist_definition_mpaso.xml @@ -1271,6 +1271,38 @@ Valid values: .true. or .false. Default: Defined in namelist_defaults.xml + +Whether to scale data ice-shelf melt fluxes by the running mean of removed ice runoff. + +Valid values: .true. or .false. +Default: Defined in namelist_defaults.xml + + + +The number of days over for which the history of removed AIS runoff is stored. The default is 731 days (2 years + 1 day). + +Valid values: Any positive integer +Default: Defined in namelist_defaults.xml + + + +If and how MPAS-Ocean sends thermal forcing to GLC (MALI) in E3SM. This is used for ocean coupling with a melt parameterization for grounded marine ice-cliffs in MALI. This is primarily relevant to the Greenland Ice Sheet, but also relevant to the Antarctic Ice Sheet. 'none' means no coupling of thermal forcing. '2d' means thermal forcing at a prescribed depth is passed to GLC. That depth is controlled by 'config_2d_thermal_forcing_depth', and the resulting thermal forcing field is calculated in the field 'avgThermalForcingAtCritDepth'. + +Valid values: 'off', '2d' +Default: Defined in namelist_defaults.xml + + + +Depth at which to pass 2d thermal forcing to the coupler for use in the GLC component. Note that mapping files for this field must be created with a mask to exclude ocean grid cells shallower than this value and thus must be regenerated if this value is changed. + +Valid values: any non-negative value +Default: Defined in namelist_defaults.xml + + diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index d45d94344fe..778d268d570 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -376,7 +376,7 @@ def buildnml(case, caseroot, compname): ic_prefix = 'mpaso.IcoswISC30E3r5.20231120+MARBL_ICfromOMIP_64levels' eco_forcing_file = 'ecoForcingSurfaceMonthly.IcoswISC30E3r5.20231215.nc' if ocn_ismf == 'data': - data_ismf_file = 'prescribed_ismf_paolo2023.IcoswISC30E3r5.20240227.nc' + data_ismf_file = 'prescribed_ismf_paolo2023.IcoswISC30E3r5.20240805.nc' if ocn_tidal_mixing == 'true': u_tidal_rms_file = 'velocityTidalRMS_CATS2008.IcoswISC30E3r5.20231120.nc' if ocn_sgr == 'data': @@ -417,7 +417,7 @@ def buildnml(case, caseroot, compname): ic_date = '20240829' ic_prefix = 'mpaso.SOwISC12to30E3r3.rstFromG-chrysalis' if ocn_ismf == 'data': - data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20240829.nc' + data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20241017.nc' #-------------------------------------------------------------------- @@ -1818,6 +1818,7 @@ def buildnml(case, caseroot, compname): lines.append(' packages="dataLandIceFluxesPKG">') lines.append(' ') lines.append(' ') + lines.append(' ') lines.append('') lines.append('') diff --git a/components/mpas-ocean/cime_config/config_component.xml b/components/mpas-ocean/cime_config/config_component.xml index 83b4501ee5e..ee55bf415d9 100644 --- a/components/mpas-ocean/cime_config/config_component.xml +++ b/components/mpas-ocean/cime_config/config_component.xml @@ -107,6 +107,8 @@ constant none + none + none constant constant diagnostic diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/ocn_glc_tf_coupling/README b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/ocn_glc_tf_coupling/README new file mode 100644 index 00000000000..bd801c44f8a --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/ocn_glc_tf_coupling/README @@ -0,0 +1,12 @@ +This testdef is used to test a stealth feature that enables coupling between +OCN and GLC for Greenland, which passes ocean thermal forcing from OCN to GLC +and uses that in a parameterization for marine melting of grounded vertical +cliffs. + +It changes one mpaso namelist variable, + config_glc_thermal_forcing_coupling_mode +from its default value to '2d'. +This tests the ocn/glc TF coupling. + +It also specified that DATM forcing should be restricted to 1958. +This allows JRA1p5 forcing to be used without a large input data requirement. diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/ocn_glc_tf_coupling/shell_commands b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/ocn_glc_tf_coupling/shell_commands new file mode 100644 index 00000000000..1d43ad8c5ba --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/ocn_glc_tf_coupling/shell_commands @@ -0,0 +1,4 @@ +./xmlchange DATM_CLMNCEP_YR_START=1958 +./xmlchange DATM_CLMNCEP_YR_END=1958 +./xmlchange DROF_STRM_YR_START=1958 +./xmlchange DROF_STRM_YR_END=1958 diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/ocn_glc_tf_coupling/user_nl_mpaso b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/ocn_glc_tf_coupling/user_nl_mpaso new file mode 100644 index 00000000000..0e137862083 --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/ocn_glc_tf_coupling/user_nl_mpaso @@ -0,0 +1 @@ +config_glc_thermal_forcing_coupling_mode = '2d' diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/scaled_dib_dismf/README b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/scaled_dib_dismf/README new file mode 100644 index 00000000000..f9f97d27de1 --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/scaled_dib_dismf/README @@ -0,0 +1,8 @@ +This testdef is used to test a stealth feature in mpaso and mpassi introduced +by PR #6696. It simply changes sets the following namelist option in mpaso +to true + config_scale_dismf_by_removed_ice_runoff +and the following namelist option in mpassi to true + config_scale_dib_by_removed_ice_runoff +These flags only makes sense to use in a B-case runs with data icebergs (DIB) +and data ice-shelf melt fluxes (DISMF). diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/scaled_dib_dismf/user_nl_mpaso b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/scaled_dib_dismf/user_nl_mpaso new file mode 100644 index 00000000000..902d4418f5a --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/scaled_dib_dismf/user_nl_mpaso @@ -0,0 +1 @@ + config_scale_dismf_by_removed_ice_runoff = .true. diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/scaled_dib_dismf/user_nl_mpassi b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/scaled_dib_dismf/user_nl_mpassi new file mode 100644 index 00000000000..8512001af93 --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/scaled_dib_dismf/user_nl_mpassi @@ -0,0 +1 @@ + config_scale_dib_by_removed_ice_runoff = .true. diff --git a/components/mpas-ocean/driver/mpaso_cpl_indices.F b/components/mpas-ocean/driver/mpaso_cpl_indices.F index f099cf8ea46..9802d526bc5 100644 --- a/components/mpas-ocean/driver/mpaso_cpl_indices.F +++ b/components/mpas-ocean/driver/mpaso_cpl_indices.F @@ -37,6 +37,7 @@ module mpaso_cpl_indices integer :: index_o2x_So_htv !ocean heat-transfer velocity integer :: index_o2x_So_stv !ocean salt-transfer velocity integer :: index_o2x_So_rhoeff !ocean effective density + integer :: index_o2x_So_tf2d !ocean thermal forcing at predefined critical depth ! ocn -> drv (BGC) @@ -208,6 +209,7 @@ subroutine mpaso_cpl_indices_set( ) index_o2x_So_htv = mct_avect_indexra(o2x,'So_htv') index_o2x_So_stv = mct_avect_indexra(o2x,'So_stv') index_o2x_So_rhoeff = mct_avect_indexra(o2x,'So_rhoeff') + index_o2x_So_tf2d = mct_avect_indexra(o2x,'So_tf2d',perrWith='quiet') index_o2x_So_algae1 = mct_avect_indexra(o2x,'So_algae1',perrWith='quiet') index_o2x_So_algae2 = mct_avect_indexra(o2x,'So_algae2',perrWith='quiet') diff --git a/components/mpas-ocean/driver/ocn_comp_mct.F b/components/mpas-ocean/driver/ocn_comp_mct.F index d1b140563bb..31dc99904eb 100644 --- a/components/mpas-ocean/driver/ocn_comp_mct.F +++ b/components/mpas-ocean/driver/ocn_comp_mct.F @@ -69,6 +69,7 @@ module ocn_comp_mct use ocn_config use ocn_submesoscale_eddies use ocn_eddy_parameterization_helpers + use ocn_scaled_dismf ! ! !PUBLIC MEMBER FUNCTIONS: implicit none @@ -108,6 +109,8 @@ module ocn_comp_mct integer :: nsend, nrecv + logical :: ocn_c2_glctf ! .true. => ocn to glc thermal forcing coupling on + character(len=StrKIND) :: runtype, coupleTimeStamp type(seq_infodata_type), pointer :: infodata @@ -214,6 +217,7 @@ subroutine ocn_init_mct( EClock, cdata_o, x2o_o, o2x_o, NLFilename )!{{{ character(len=StrKIND) :: iotype logical :: streamsExists integer :: mesh_iotype + logical :: ocn_c2_glcshelf logical, pointer :: tempLogicalConfig character(len=StrKIND), pointer :: tempCharConfig @@ -222,11 +226,15 @@ subroutine ocn_init_mct( EClock, cdata_o, x2o_o, o2x_o, NLFilename )!{{{ logical, pointer :: config_use_CFCTracers logical, pointer :: config_use_activeTracers_surface_restoring logical, pointer :: config_use_surface_salinity_monthly_restoring + logical, pointer :: config_scale_dismf_by_removed_ice_runoff character (len=StrKIND), pointer :: config_land_ice_flux_mode + character (len=StrKIND), pointer :: config_glc_thermal_forcing_coupling_mode ! ssh coupling interval initialization integer, pointer :: index_avgZonalSSHGradient, index_avgMeridionalSSHGradient real (kind=RKIND), dimension(:,:), pointer :: avgSSHGradient + real (kind=RKIND), pointer :: & + runningMeanRemovedIceRunoff ! the area integrated, running mean of removed ice runoff from the ocean #ifdef HAVE_MOAB character*100 outfile, wopts @@ -304,6 +312,9 @@ end subroutine xml_stream_get_attributes ! Determine coupling type call seq_infodata_GetData(infodata, cpl_seq_option=cpl_seq_option) + ! Determine if ocn to glc thermal forcing coupling is on + call seq_infodata_GetData(infodata, ocn_c2_glctf=ocn_c2_glctf) + !----------------------------------------------------------------------- ! ! initialize the model run @@ -815,6 +826,9 @@ end subroutine xml_stream_get_attributes call ocn_time_average_coupled_accumulate(statePool, forcingPool, 1) block_ptr => block_ptr % next end do + + ! initialize scaled data ice-shelf melt fluxes based on remove ice runoff + call ocn_init_scaled_dismf(domain) end if !----------------------------------------------------------------------- @@ -860,6 +874,9 @@ end subroutine xml_stream_get_attributes call ocn_time_average_coupled_accumulate(statePool, forcingPool, 1) block_ptr => block_ptr % next end do + + ! initialize scaled data ice-shelf melt fluxes based on remove ice runoff + call ocn_init_scaled_dismf(domain) end if call t_stopf ('mpaso_mct_init') @@ -869,14 +886,35 @@ end subroutine xml_stream_get_attributes trim(config_land_ice_flux_mode) == 'pressure_only' .or. & trim(config_land_ice_flux_mode) == 'data' .or. & trim(config_land_ice_flux_mode) == 'standalone' ) then - call seq_infodata_PutData( infodata, ocn_prognostic=.true., ocnrof_prognostic=.true., & - ocn_c2_glcshelf=.false.) + ocn_c2_glcshelf = .false. else if ( trim(config_land_ice_flux_mode) .eq. 'coupled' ) then - call seq_infodata_PutData( infodata, ocn_prognostic=.true., ocnrof_prognostic=.true., & - ocn_c2_glcshelf=.true.) + ocn_c2_glcshelf = .true. else call mpas_log_write('ERROR: unknown land_ice_flux_mode: ' // trim(config_land_ice_flux_mode), MPAS_LOG_CRIT) end if + call seq_infodata_PutData(infodata, ocn_prognostic=.true., ocnrof_prognostic=.true., & + ocn_c2_glcshelf=ocn_c2_glcshelf) + + call mpas_pool_get_config(domain % configs, 'config_scale_dismf_by_removed_ice_runoff', & + config_scale_dismf_by_removed_ice_runoff) + if (config_scale_dismf_by_removed_ice_runoff) then + ! independent of space so should be no need to loop over blocks + block_ptr => domain % blocklist + call mpas_pool_get_subpool(block_ptr % structs, 'forcing', forcingPool) + call MPAS_pool_get_array(forcingPool, "runningMeanRemovedIceRunoff", & + runningMeanRemovedIceRunoff) + call seq_infodata_PutData(infodata, rmean_rmv_ice_runoff=runningMeanRemovedIceRunoff) + end if + + call mpas_pool_get_config(domain % configs, 'config_glc_thermal_forcing_coupling_mode', config_glc_thermal_forcing_coupling_mode) + if ( trim(config_glc_thermal_forcing_coupling_mode) == 'off' ) then + call seq_infodata_PutData(infodata, ocn_c2_glctf=.false.) + else if ( trim(config_glc_thermal_forcing_coupling_mode) == '2d' ) then + call seq_infodata_PutData(infodata, ocn_c2_glctf=.true.) + else + call mpas_log_write('ERROR: unknown config_glc_thermal_forcing_coupling_mode: ' // & + trim(config_glc_thermal_forcing_coupling_mode), MPAS_LOG_CRIT) + end if !----------------------------------------------------------------------- ! @@ -884,13 +922,7 @@ end subroutine xml_stream_get_attributes ! !----------------------------------------------------------------------- - call ocn_import_mct(x2o_o, errorCode) - if (errorCode /= 0) then - call mpas_log_write('Error in ocn_import_mct', MPAS_LOG_CRIT) - endif - #ifdef HAVE_MOAB - #ifdef MOABCOMP ! loop over all fields in seq_flds_x2o_fields call mct_list_init(temp_list ,seq_flds_x2o_fields) @@ -906,6 +938,14 @@ end subroutine xml_stream_get_attributes enddo call mct_list_clean(temp_list) #endif +#endif + + call ocn_import_mct(x2o_o, errorCode) + if (errorCode /= 0) then + call mpas_log_write('Error in ocn_import_mct', MPAS_LOG_CRIT) + endif + +#ifdef HAVE_MOAB call ocn_import_moab(Eclock, errorCode) if (errorCode /= 0) then @@ -987,12 +1027,15 @@ subroutine ocn_run_mct( EClock, cdata_o, x2o_o, o2x_o)!{{{ logical, pointer :: config_use_CFCTracers logical, pointer :: config_use_activeTracers_surface_restoring logical, pointer :: config_use_surface_salinity_monthly_restoring + logical, pointer :: config_scale_dismf_by_removed_ice_runoff character (len=StrKIND), pointer :: config_restart_timestamp_name character (len=StrKIND), pointer :: config_sw_absorption_type ! Added for coupling interval initialization integer, pointer :: index_avgZonalSSHGradient, index_avgMeridionalSSHGradient real (kind=RKIND), dimension(:,:), pointer :: avgSSHGradient + real (kind=RKIND), pointer :: & + runningMeanRemovedIceRunoff ! the area integrated, running mean of removed ice runoff from the ocean #ifdef HAVE_MOAB #ifdef MOABCOMP @@ -1029,12 +1072,8 @@ subroutine ocn_run_mct( EClock, cdata_o, x2o_o, o2x_o)!{{{ call mpas_get_timeInterval(timeStep, dt=dt) call mpas_reset_clock_alarm(domain_ptr % clock, coupleAlarmID, ierr=ierr) - ! Import state from coupler - call ocn_import_mct(x2o_o, ierr) - ! Import state from moab coupler -#ifdef HAVE_MOAB - +#ifdef HAVE_MOAB #ifdef MOABCOMP ! loop over all fields in seq_flds_x2o_fields call mct_list_init(temp_list ,seq_flds_x2o_fields) @@ -1048,8 +1087,14 @@ subroutine ocn_run_mct( EClock, cdata_o, x2o_o, o2x_o)!{{{ call seq_comm_compare_mb_mct(modelStr, mpicom_moab, x2o_o, mct_field, MPOID, tagname, ent_type, difference) enddo call mct_list_clean(temp_list) +#endif #endif + ! Import state from coupler + call ocn_import_mct(x2o_o, ierr) + ! Import state from moab coupler +#ifdef HAVE_MOAB + call ocn_import_moab(Eclock, ierr) if (ierr /= 0) then call mpas_log_write('Error in ocn_import_moab', MPAS_LOG_CRIT) @@ -1193,6 +1238,9 @@ subroutine ocn_run_mct( EClock, cdata_o, x2o_o, o2x_o)!{{{ block_ptr => block_ptr % next end do + ! update scaled data ice-shelf melt fluxes based on remove ice runoff + call ocn_update_scaled_dismf(domain) + if (debugOn) call mpas_log_write(' Computing analysis members') call ocn_analysis_compute(domain_ptr, ierr) if (iam==0.and.debugOn) then @@ -1290,6 +1338,17 @@ subroutine ocn_run_mct( EClock, cdata_o, x2o_o, o2x_o)!{{{ #endif call check_clocks_sync(domain % clock, Eclock, ierr) + call mpas_pool_get_config(domain % configs, 'config_scale_dismf_by_removed_ice_runoff', & + config_scale_dismf_by_removed_ice_runoff) + if (config_scale_dismf_by_removed_ice_runoff) then + ! independent of space so should be no need to loop over blocks + block_ptr => domain % blocklist + call mpas_pool_get_subpool(block_ptr % structs, 'forcing', forcingPool) + call MPAS_pool_get_array(forcingPool, "runningMeanRemovedIceRunoff", & + runningMeanRemovedIceRunoff) + call seq_infodata_PutData(infodata, rmean_rmv_ice_runoff=runningMeanRemovedIceRunoff) + end if + ! Reset I/O logs call shr_file_setLogUnit (shrlogunit) call shr_file_setLogLevel(shrloglev) @@ -2138,7 +2197,7 @@ subroutine ocn_import_mct(x2o_o, errorCode)!{{{ call shr_sys_abort ('Error: incoming rofi_F is negative') end if if (config_remove_ais_ice_runoff) then - if (latCell(i) < -1.04719666667_RKIND) then ! 60S in radians + if (latCell(i) < -0.99483767345_RKIND) then ! 57S in radians removedIceRunoffFlux(i) = iceRunoffFlux(i) iceRunoffFlux(i) = 0.0_RKIND removedIceRunoffFluxThisProc = removedIceRunoffFluxThisProc + removedIceRunoffFlux(i) @@ -2689,7 +2748,8 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ avgRemovedRiverRunoffFlux, & avgRemovedIceRunoffFlux, & avgLandIceHeatFlux, & - avgRemovedIceRunoffHeatFlux + avgRemovedIceRunoffHeatFlux, & + avgThermalForcingAtCritDepth real (kind=RKIND), dimension(:,:), pointer :: avgTracersSurfaceValue, avgSurfaceVelocity, & avgSSHGradient, avgOceanSurfacePhytoC, & @@ -2708,6 +2768,7 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ config_use_MacroMoleculesTracers_sea_ice_coupling character (len=StrKIND), pointer :: config_land_ice_flux_mode + character (len=StrKIND), pointer :: config_glc_thermal_forcing_coupling_mode logical :: keepFrazil @@ -2718,6 +2779,8 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ call mpas_pool_get_config(domain % configs, 'config_land_ice_flux_mode', config_land_ice_flux_mode) call mpas_pool_get_config(domain % configs, 'config_remove_ais_river_runoff', config_remove_ais_river_runoff) call mpas_pool_get_config(domain % configs, 'config_remove_ais_ice_runoff', config_remove_ais_ice_runoff) + call mpas_pool_get_config(domain % configs, 'config_glc_thermal_forcing_coupling_mode', & + config_glc_thermal_forcing_coupling_mode) call mpas_pool_get_config(domain % configs, 'config_use_DMSTracers', config_use_DMSTracers) call mpas_pool_get_config(domain % configs, 'config_use_MacroMoleculesTracers', config_use_MacroMoleculesTracers) call mpas_pool_get_config(domain % configs, 'config_use_ecosysTracers_sea_ice_coupling', & @@ -2772,6 +2835,9 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoffFlux', avgRemovedIceRunoffFlux) call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoffHeatFlux', avgRemovedIceRunoffHeatFlux) endif + if (trim(config_glc_thermal_forcing_coupling_mode) == '2d') then + call mpas_pool_get_array(forcingPool, 'avgThermalForcingAtCritDepth', avgThermalForcingAtCritDepth) + endif ! BGC fields if (config_use_ecosysTracers) then @@ -2933,6 +2999,10 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ o2x_o % rAttr(index_o2x_So_stv, n) = landIceTracerTransferVelocities(indexSaltTrans,i) o2x_o % rAttr(index_o2x_So_rhoeff, n) = 0.0_RKIND endif + if (trim(config_glc_thermal_forcing_coupling_mode) == '2d' .and. ocn_c2_glctf) then + o2x_o % rAttr(index_o2x_So_tf2d, n) = avgThermalForcingAtCritDepth(i) + endif + !Fyke: test !write(stderrUnit,*) 'n=',n @@ -3146,15 +3216,19 @@ end subroutine datetime!}}} #ifdef HAVE_MOAB -! import method from moab -! copied from ocn_import_mct, will replace x2o_o AV with x2o_om array read locally - subroutine ocn_import_moab( Eclock, errorCode)!{{{ + +!*********************************************************************** +!BOP +! !IROUTINE: ocn_import_moab +! !INTERFACE: + + subroutine ocn_import_moab(Eclock, errorCode)!{{{ ! !DESCRIPTION: !----------------------------------------------------------------------- -! This routine receives message from cpl7 driver +! This routine receives message from moab driver ! -! The following fields are always received from the coupler: +! The following fields are always received from the driver: ! ! o taux -- zonal wind stress (taux) (W/m2 ) ! o tauy -- meridonal wind stress (tauy) (W/m2 ) @@ -3171,11 +3245,22 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ ! o ifrac -- ice fraction (%) ! o rofl -- river runoff flux (kg/m2/s) ! o rofi -- ice runoff flux (kg/m2/s) +! o rofDIN -- DIN runoff flux (kg/m2/s) +! o rofDIP -- DIP runoff flux (kg/m2/s) +! o rofDON -- DON runoff flux (kg/m2/s) +! o rofDOP -- DOP runoff flux (kg/m2/s) +! o rofDOC -- DOC runoff flux (kg/m2/s) +! o rofPP -- PP runoff flux (kg/m2/s) +! o rofDSi -- DSi runoff flux (kg/m2/s) +! o rofPOC -- POC runoff flux (kg/m2/s) +! o rofPN -- PN runoff flux (kg/m2/s) +! o rofDIC -- DIC runoff flux (kg/m2/s) +! o rofFe -- Fe runoff flux (kg/m2/s) ! ! The following fields are sometimes received from the coupler, ! depending on model options: ! -! o pbot -- bottom atm pressure (Pa) +! o pslv -- atmospheric pressure at sea level (Pa) ! o duu10n -- 10m wind speed squared (m^2/s^2) ! o co2prog-- bottom atm level prognostic co2 ! o co2diag-- bottom atm level diagnostic co2 @@ -3184,28 +3269,15 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ ! ! !REVISION HISTORY: ! same as module - + use iMOAB, only : iMOAB_GetDoubleTagStorage, iMOAB_WriteMesh ! !INPUT/OUTPUT PARAMETERS: + type(ESMF_Clock), intent(inout) :: EClock ! type(mct_aVect) , intent(inout) :: x2o_o ! instead, we will get x2o_om from MPOID ! !OUTPUT PARAMETERS: - use iMOAB, only : iMOAB_GetDoubleTagStorage, iMOAB_WriteMesh - !EOP - !BOC - !----------------------------------------------------------------------- - ! - ! local variables - !----------------------------------------------------------------------- - ! - ! local variables - ! - !----------------------------------------------------------------------- - integer :: ent_type, ierr - character(CXX) :: tagname - type(ESMF_Clock), intent(inout) :: EClock integer, intent(out) :: & errorCode ! returned error code @@ -3221,6 +3293,9 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ label, & message + integer :: ent_type, ierr + character(CXX) :: tagname + integer :: & i,n @@ -3232,6 +3307,7 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ config_use_DMSTracers_sea_ice_coupling, & config_use_MacroMoleculesTracers, & config_use_MacroMoleculesTracers_sea_ice_coupling, & + config_use_CFCTracers, & config_remove_ais_river_runoff, & config_remove_ais_ice_runoff, & config_cvmix_kpp_use_theory_wave @@ -3250,7 +3326,8 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ ecosysAuxiliary, & ecosysSeaIceCoupling, & DMSSeaIceCoupling, & - MacroMoleculesSeaIceCoupling + MacroMoleculesSeaIceCoupling, & + CFCAuxiliary integer, pointer :: nCellsSolve @@ -3278,10 +3355,22 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ iceFluxFeParticulateField, & iceFluxFeDissolvedField, & iceFluxDustField, & + riverFluxNO3Field, & + riverFluxPO4Field, & + riverFluxSiO3Field, & + riverFluxDOCField, & + riverFluxDONField, & + riverFluxDOPField, & + riverFluxDICField, & + riverFluxALKField, & + riverFluxFeField, & landIceFreshwaterFluxField, & landIceHeatFluxField, & landIceFractionField, & - windSpeed10mField + windSpeed10mField, & + significantWaveHeightField, & + peakWaveFrequencyField, & + peakWaveDirectionField !landIcePressureField type (field2DReal), pointer :: iceFluxPhytoCField, & @@ -3289,6 +3378,9 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ type (field2DReal), pointer :: landIceInterfaceTracersField + type (field2DReal), pointer :: stokesDriftZonalWavenumberField, & + stokesDriftMeridionalWavenumberField + real (kind=RKIND), dimension(:), pointer :: windStressZonal, windStressMeridional, & latentHeatFlux, sensibleHeatFlux, & longWaveHeatFluxUp, & @@ -3302,6 +3394,7 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ atmosphericPressure, iceFraction, & seaIcePressure, windSpeedSquared10m, & atmosphericCO2, atmosphericCO2_ALT_CO2, & + windSpeedSquared10mCFC, & iceFluxDIC, & iceFluxDON, & iceFluxNO3, & @@ -3313,26 +3406,44 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ iceFluxFeParticulate, & iceFluxFeDissolved, & iceFluxDust, & + riverFluxNO3, & + riverFluxPO4, & + riverFluxSiO3, & + riverFluxDOC, & + riverFluxDON, & + riverFluxDOP, & + riverFluxDIC, & + riverFluxALK, & + riverFluxFe, & landIceFreshwaterFlux, & landIceHeatFlux, & landIceFraction, & - windSpeed10m + areaCell, & + windSpeed10m, & + significantWaveHeight, & + peakWaveFrequency, & + peakWaveDirection !landIcePressure real (kind=RKIND), dimension(:), pointer :: latCell real (kind=RKIND), dimension(:,:), pointer :: iceFluxPhytoC, & - iceFluxDOC + iceFluxDOC, & + stokesDriftZonalWavenumber, & + stokesDriftMeridionalWavenumber real (kind=RKIND) :: removedRiverRunoffFluxThisProc, removedIceRunoffFluxThisProc real (kind=RKIND) :: removedRiverRunoffFluxReduced, removedIceRunoffFluxReduced real (kind=RKIND), dimension(:,:), pointer :: landIceInterfaceTracers + real (kind=RKIND) :: riverFactor + !----------------------------------------------------------------------- ! ! zero out padded cells ! +!----------------------------------------------------------------------- !----------------------------------------------------------------------- integer :: cur_ocn_stepno #ifdef MOABDEBUG @@ -3349,11 +3460,10 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ write(ocnLogUnit,*) 'Fail to write ocean state ' endif #endif + errorCode = 0 ! get moab tags from MPOID - - ent_type = 1 ! cells ! get all tags in one method tagname = trim(seq_flds_x2o_fields)//C_NULL_CHAR @@ -3361,7 +3471,6 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ if ( ierr /= 0 ) then write(ocnLogUnit,*) 'Fail to get MOAB fields ' endif - !----------------------------------------------------------------------- ! ! unpack and distribute wind stress, then convert to correct units @@ -3380,6 +3489,7 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ config_use_DMSTracers_sea_ice_coupling) call mpas_pool_get_config(domain % configs, 'config_use_MacroMoleculesTracers_sea_ice_coupling', & config_use_MacroMoleculesTracers_sea_ice_coupling) + call mpas_pool_get_config(domain % configs, 'config_use_CFCTracers', config_use_CFCTracers) call mpas_pool_get_config(domain % configs, 'config_remove_ais_river_runoff', config_remove_ais_river_runoff) call mpas_pool_get_config(domain % configs, 'config_remove_ais_ice_runoff', config_remove_ais_ice_runoff) call mpas_pool_get_config(domain % configs, 'config_cvmix_kpp_use_theory_wave', config_cvmix_kpp_use_theory_wave) @@ -3418,6 +3528,11 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ call mpas_pool_get_field(forcingPool, 'iceRunoffFlux', iceRunoffFluxField) call mpas_pool_get_field(forcingPool, 'removedRiverRunoffFlux', removedRiverRunoffFluxField) call mpas_pool_get_field(forcingPool, 'removedIceRunoffFlux', removedIceRunoffFluxField) + call mpas_pool_get_field(forcingPool, 'stokesDriftZonalWavenumber', stokesDriftZonalWavenumberField) + call mpas_pool_get_field(forcingPool, 'stokesDriftMeridionalWavenumber', stokesDriftMeridionalWavenumberField) + call mpas_pool_get_field(forcingPool, 'significantWaveHeight', significantWaveHeightField) + call mpas_pool_get_field(forcingPool, 'peakWaveFrequency', peakWaveFrequencyField) + call mpas_pool_get_field(forcingPool, 'peakWaveDirection', peakWaveDirectionField) call mpas_pool_get_field(forcingPool, 'landIceFreshwaterFlux', landIceFreshwaterFluxField) call mpas_pool_get_field(forcingPool, 'landIceHeatFlux', landIceHeatFluxField) @@ -3459,9 +3574,15 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ landIceInterfaceTracers => landIceInterfaceTracersField % array landIceFraction => landIceFractionField % array windSpeed10m => windSpeed10mField % array + stokesDriftZonalWavenumber => stokesDriftZonalWavenumberField % array + stokesDriftMeridionalWavenumber => stokesDriftMeridionalWavenumberField % array + significantWaveHeight => significantWaveHeightField % array + peakWaveFrequency => peakWaveFrequencyField % array + peakWaveDirection => peakWaveDirectionField % array !landIcePressure => landIcePressureField % array call mpas_pool_get_array(meshPool, 'latCell', latCell) + call mpas_pool_get_array(meshPool, 'areaCell', areaCell) ! BGC fields if (config_use_ecosysTracers) then @@ -3474,6 +3595,27 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ call mpas_pool_get_field(ecosysAuxiliary, 'atmosphericCO2_ALT_CO2', atmosphericCO2_ALT_CO2Field) atmosphericCO2_ALT_CO2 => atmosphericCO2_ALT_CO2Field % array + if (config_use_ecosysTracers_river_inputs_from_coupler) then + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxNO3' , riverFluxNO3Field) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxPO4' , riverFluxPO4Field) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxDON' , riverFluxDONField) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxDOP' , riverFluxDOPField) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxSiO3', riverFluxSiO3Field) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxDOC' , riverFluxDOCField) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxDIC' , riverFluxDICField) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxALK' , riverFluxALKField) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxFe' , riverFluxFeField) + riverFluxNO3 => riverFluxNO3Field % array + riverFluxPO4 => riverFluxPO4Field % array + riverFluxDON => riverFluxDONField % array + riverFluxDOP => riverFluxDOPField % array + riverFluxSiO3 => riverFluxSiO3Field % array + riverFluxDOC => riverFluxDOCField % array + riverFluxDIC => riverFluxDICField % array + riverFluxALK => riverFluxALKField % array + riverFluxFe => riverFluxFeField % array + endif + call mpas_pool_get_config(domain % configs, 'config_ecosys_atm_co2_option', & config_ecosys_atm_co2_option) call mpas_pool_get_config(domain % configs, 'config_ecosys_atm_alt_co2_option', & @@ -3519,6 +3661,13 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ iceFluxDMSP => iceFluxDMSPField % array endif + ! CFC fields + if (config_use_CFCTracers) then + call mpas_pool_get_subpool(forcingPool, 'CFCAuxiliary', CFCAuxiliary) + call mpas_pool_get_field(CFCAuxiliary, 'windSpeedSquared10mCFC', windSpeedSquared10mField) + windSpeedSquared10mCFC => windSpeedSquared10mField % array + endif + if (config_remove_ais_river_runoff) then ! Initialize this field removedRiverRunoffFlux(:) = 0.0_RKIND @@ -3533,8 +3682,7 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ ! Initialize this field windSpeed10m(:) = 0.0_RKIND endif - -! replace 'x2o_o % rAttr(' to 'x2o_om(n, ' and ', n)' with ')' +! ! replace 'x2o_o % rAttr(' to 'x2o_om(n, ' and ', n)' with ')' do i = 1, nCellsSolve n = n + 1 if ( windStressZonalField % isActive ) then @@ -3593,7 +3741,7 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ call shr_sys_abort ('Error: incoming rofi_F is negative') end if if (config_remove_ais_ice_runoff) then - if (latCell(i) < -1.04719666667_RKIND) then ! 60S in radians + if (latCell(i) < -0.99483767345_RKIND) then ! 57S in radians removedIceRunoffFlux(i) = iceRunoffFlux(i) iceRunoffFlux(i) = 0.0_RKIND removedIceRunoffFluxThisProc = removedIceRunoffFluxThisProc + removedIceRunoffFlux(i) @@ -3618,6 +3766,32 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ iceFraction(i) = x2o_om(n, index_x2o_Si_ifrac) end if + if ( stokesDriftZonalWavenumberField % isActive ) then + stokesDriftZonalWavenumber(1,i) = x2o_om(n, index_x2o_Sw_ustokes_wavenumber_1) + stokesDriftZonalWavenumber(2,i) = x2o_om(n, index_x2o_Sw_ustokes_wavenumber_2) + stokesDriftZonalWavenumber(3,i) = x2o_om(n, index_x2o_Sw_ustokes_wavenumber_3) + stokesDriftZonalWavenumber(4,i) = x2o_om(n, index_x2o_Sw_ustokes_wavenumber_4) + stokesDriftZonalWavenumber(5,i) = x2o_om(n, index_x2o_Sw_ustokes_wavenumber_5) + stokesDriftZonalWavenumber(6,i) = x2o_om(n, index_x2o_Sw_ustokes_wavenumber_6) + end if + if ( stokesDriftMeridionalWavenumberField % isActive ) then + stokesDriftMeridionalWavenumber(1,i) = x2o_om(n, index_x2o_Sw_vstokes_wavenumber_1) + stokesDriftMeridionalWavenumber(2,i) = x2o_om(n, index_x2o_Sw_vstokes_wavenumber_2) + stokesDriftMeridionalWavenumber(3,i) = x2o_om(n, index_x2o_Sw_vstokes_wavenumber_3) + stokesDriftMeridionalWavenumber(4,i) = x2o_om(n, index_x2o_Sw_vstokes_wavenumber_4) + stokesDriftMeridionalWavenumber(5,i) = x2o_om(n, index_x2o_Sw_vstokes_wavenumber_5) + stokesDriftMeridionalWavenumber(6,i) = x2o_om(n, index_x2o_Sw_vstokes_wavenumber_6) + end if + if ( significantWaveHeightField % isActive ) then + significantWaveHeight(i) = x2o_om(n, index_x2o_Sw_Hs) + end if + if ( peakWaveFrequencyField % isActive ) then + peakWaveFrequency(i) = x2o_om(n, index_x2o_Sw_Fp) + end if + if ( peakWaveDirectionField % isActive ) then + peakWaveDirection(i) = x2o_om(n, index_x2o_Sw_Dp) + end if + if (config_cvmix_kpp_use_theory_wave) then if ( windSpeed10mField% isActive ) then windSpeed10m(i) = sqrt( x2o_om(n, index_x2o_So_duu10n)) @@ -3675,12 +3849,36 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ else if ( config_ecosys_atm_alt_co2_option == 'bdrc') then atmosphericCO2_ALT_CO2(i) = config_ecosys_atm_co2_constant_value else if ( config_ecosys_atm_alt_co2_option == 'bdrd') then - atmosphericCO2_ALT_CO2(i) = x2o_om(n, index_x2o_Sa_co2diag) + atmosphericCO2_ALT_CO2(i) = x2o_om(n, index_x2o_Sa_co2diag) else atmosphericCO2_ALT_CO2(i) = config_ecosys_atm_co2_constant_value end if end if + if (config_use_ecosysTracers_river_inputs_from_coupler) then + riverFluxNO3(i) = x2o_om(n, index_x2o_Foxx_rofDIN) + riverFluxPO4(i) = x2o_om(n, index_x2o_Foxx_rofDIP) + riverFluxDON(i) = x2o_om(n, index_x2o_Foxx_rofDON) + riverFluxDOP(i) = x2o_om(n, index_x2o_Foxx_rofDOP) + riverFluxSiO3(i) = x2o_om(n, index_x2o_Foxx_rofDSi) + riverFluxDOC(i) = x2o_om(n, index_x2o_Foxx_rofDOC) + riverFluxDIC(i) = x2o_om(n, index_x2o_Foxx_rofDIC) + riverFluxFe(i) = x2o_om(n, index_x2o_Foxx_rofFe ) + +! convert from kgNutrient/(m2-s) to mmol/m3 m/s + riverFactor = 1.e6_RKIND + riverFluxNO3(i) = riverFluxNO3(i)*riverFactor/14.007_RKIND + riverFluxPO4(i) = riverFluxPO4(i)*riverFactor/30.974_RKIND + riverFluxDON(i) = riverFluxDON(i)*riverFactor/14.007_RKIND + riverFluxDOP(i) = riverFluxDOP(i)*riverFactor/30.974_RKIND + riverFluxSiO3(i) = riverFluxSiO3(i)*riverFactor/28.085_RKIND + riverFluxDOC(i) = riverFluxDOC(i)*riverFactor/12.001_RKIND + riverFluxDIC(i) = riverFluxDIC(i)*riverFactor/12.001_RKIND + riverFluxFe(i) = riverFluxFe(i)*riverFactor/55.845_RKIND + + riverFluxALK(i) = riverFluxDIC(i) + endif + if (config_use_ecosysTracers_sea_ice_coupling) then if ( iceFluxPhytoCField % isActive ) then iceFluxPhytoC(1,i) = x2o_om(n, index_x2o_Fioi_algae1) @@ -3714,6 +3912,7 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ if ( iceFluxDOCField % isActive ) then iceFluxDOC(1,i) = x2o_om(n, index_x2o_Fioi_doc1) iceFluxDOC(2,i) = x2o_om(n, index_x2o_Fioi_doc2) + iceFluxDOC(3,i) = x2o_om(n, index_x2o_Fioi_doc3) endif if ( iceFluxDONField % isActive ) then iceFluxDON(i) = x2o_om(n, index_x2o_Fioi_don1) @@ -3730,6 +3929,13 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ endif endif + ! CFC fields + if (config_use_CFCTracers) then + if ( windSpeedSquared10mField % isActive ) then + windSpeedSquared10mCFC(i) = x2o_om(n, index_x2o_So_duu10n) + end if + end if + end do block_ptr => block_ptr % next @@ -3757,6 +3963,11 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ call mpas_pool_get_field(forcingPool, 'atmosphericPressure', atmosphericPressureField) call mpas_pool_get_field(forcingPool, 'seaIcePressure', seaIcePressureField) call mpas_pool_get_field(forcingPool, 'iceFraction', iceFractionField) + call mpas_pool_get_field(forcingPool, 'stokesDriftZonalWavenumber', stokesDriftZonalWavenumberField) + call mpas_pool_get_field(forcingPool, 'stokesDriftMeridionalWavenumber', stokesDriftMeridionalWavenumberField) + call mpas_pool_get_field(forcingPool, 'significantWaveHeight', significantWaveHeightField) + call mpas_pool_get_field(forcingPool, 'peakWaveFrequency', peakWaveFrequencyField) + call mpas_pool_get_field(forcingPool, 'peakWaveDirection', peakWaveDirectionField) call mpas_pool_get_field(forcingPool, 'landIceFreshwaterFlux', landIceFreshwaterFluxField) call mpas_pool_get_field(forcingPool, 'landIceHeatFlux', landIceHeatFluxField) @@ -3777,6 +3988,18 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ call mpas_pool_get_field(ecosysAuxiliary, 'atmosphericCO2', atmosphericCO2Field) call mpas_pool_get_field(ecosysAuxiliary, 'atmosphericCO2_ALT_CO2', atmosphericCO2_ALT_CO2Field) + if (config_use_ecosysTracers_river_inputs_from_coupler) then + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxNO3' , riverFluxNO3Field) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxPO4' , riverFluxPO4Field) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxDON' , riverFluxDONField) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxDOP' , riverFluxDOPField) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxSiO3', riverFluxSiO3Field) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxDOC' , riverFluxDOCField) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxDIC' , riverFluxDICField) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxALK' , riverFluxALKField) + call mpas_pool_get_field(ecosysAuxiliary, 'riverFluxFe' , riverFluxFeField) + endif + if (config_use_ecosysTracers_sea_ice_coupling) then call mpas_pool_get_subpool(forcingPool, 'ecosysSeaIceCoupling', ecosysSeaIceCoupling) @@ -3800,6 +4023,12 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ call mpas_pool_get_field(DMSSeaIceCoupling, 'iceFluxDMSP', iceFluxDMSPField) endif + ! CFC fields + if (config_use_CFCTracers) then + call mpas_pool_get_subpool(forcingPool, 'CFCAuxiliary', CFCAuxiliary) + call mpas_pool_get_field(CFCAuxiliary, 'windSpeedSquared10mCFC', windSpeedSquared10mField) + endif + if ( windStressMeridionalField % isActive ) then call mpas_dmpar_exch_halo_field(windStressMeridionalField) end if @@ -3860,6 +4089,21 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ if ( iceFractionField % isActive ) then call mpas_dmpar_exch_halo_field(iceFractionField) end if + if ( stokesDriftZonalWavenumberField % isActive ) then + call mpas_dmpar_exch_halo_field(stokesDriftZonalWavenumberField) + end if + if ( stokesDriftMeridionalWavenumberField % isActive ) then + call mpas_dmpar_exch_halo_field(stokesDriftMeridionalWavenumberField) + end if + if ( significantWaveHeightField % isActive ) then + call mpas_dmpar_exch_halo_field(significantWaveHeightField) + end if + if ( peakWaveFrequencyField % isActive ) then + call mpas_dmpar_exch_halo_field(peakWaveFrequencyField) + end if + if ( peakWaveDirectionField % isActive ) then + call mpas_dmpar_exch_halo_field(peakWaveDirectionField) + end if if ( landIceFreshwaterFluxField % isActive ) then call mpas_dmpar_exch_halo_field(landIceFreshwaterFluxField) @@ -3894,32 +4138,62 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ call mpas_dmpar_exch_halo_field(atmosphericCO2_ALT_CO2Field) end if - if (config_use_ecosysTracers_sea_ice_coupling) then - if ( iceFluxPhytoCField % isActive ) then - call mpas_dmpar_exch_halo_field(iceFluxPhytoCField) - endif - if ( iceFluxDICField % isActive ) then - call mpas_dmpar_exch_halo_field(iceFluxDICField) - endif - if ( iceFluxNO3Field % isActive ) then - call mpas_dmpar_exch_halo_field(iceFluxNO3Field) - endif - if ( iceFluxSiO3Field % isActive ) then - call mpas_dmpar_exch_halo_field(iceFluxSiO3Field) - endif - if ( iceFluxNH4Field % isActive ) then - call mpas_dmpar_exch_halo_field(iceFluxNH4Field) - endif - if ( iceFluxDOCrField % isActive ) then - call mpas_dmpar_exch_halo_field(iceFluxDOCrField) - endif - if ( iceFluxFeParticulateField % isActive ) then - call mpas_dmpar_exch_halo_field(iceFluxFeParticulateField) - endif - if ( iceFluxFeDissolvedField % isActive ) then - call mpas_dmpar_exch_halo_field(iceFluxFeDissolvedField) - endif - if ( iceFluxDustField % isActive ) then + if (config_use_ecosysTracers_river_inputs_from_coupler) then + if ( riverFluxNO3Field % isActive ) then + call mpas_dmpar_exch_halo_field(riverFluxNO3Field) + end if + if ( riverFluxPO4Field % isActive ) then + call mpas_dmpar_exch_halo_field(riverFluxPO4Field) + end if + if ( riverFluxDONField % isActive ) then + call mpas_dmpar_exch_halo_field(riverFluxDONField) + end if + if ( riverFluxDOPField % isActive ) then + call mpas_dmpar_exch_halo_field(riverFluxDOPField) + end if + if ( riverFluxSiO3Field % isActive ) then + call mpas_dmpar_exch_halo_field(riverFluxSiO3Field) + end if + if ( riverFluxDOCField % isActive ) then + call mpas_dmpar_exch_halo_field(riverFluxDOCField) + end if + if ( riverFluxDICField % isActive ) then + call mpas_dmpar_exch_halo_field(riverFluxDICField) + end if + if ( riverFluxALKField % isActive ) then + call mpas_dmpar_exch_halo_field(riverFluxALKField) + end if + if ( riverFluxFeField % isActive ) then + call mpas_dmpar_exch_halo_field(riverFluxFeField) + end if + endif + + if (config_use_ecosysTracers_sea_ice_coupling) then + if ( iceFluxPhytoCField % isActive ) then + call mpas_dmpar_exch_halo_field(iceFluxPhytoCField) + endif + if ( iceFluxDICField % isActive ) then + call mpas_dmpar_exch_halo_field(iceFluxDICField) + endif + if ( iceFluxNO3Field % isActive ) then + call mpas_dmpar_exch_halo_field(iceFluxNO3Field) + endif + if ( iceFluxSiO3Field % isActive ) then + call mpas_dmpar_exch_halo_field(iceFluxSiO3Field) + endif + if ( iceFluxNH4Field % isActive ) then + call mpas_dmpar_exch_halo_field(iceFluxNH4Field) + endif + if ( iceFluxDOCrField % isActive ) then + call mpas_dmpar_exch_halo_field(iceFluxDOCrField) + endif + if ( iceFluxFeParticulateField % isActive ) then + call mpas_dmpar_exch_halo_field(iceFluxFeParticulateField) + endif + if ( iceFluxFeDissolvedField % isActive ) then + call mpas_dmpar_exch_halo_field(iceFluxFeDissolvedField) + endif + if ( iceFluxDustField % isActive ) then call mpas_dmpar_exch_halo_field(iceFluxDustField) endif if ( iceFluxDOCField % isActive ) then @@ -3939,302 +4213,348 @@ subroutine ocn_import_moab( Eclock, errorCode)!{{{ endif endif + ! CFC fields + if (config_use_CFCTracers .and. .not. config_use_ecosysTracers) then + if ( windSpeedSquared10mField % isActive ) then + call mpas_dmpar_exch_halo_field(windSpeedSquared10mField) + end if + endif + !----------------------------------------------------------------------- !EOC end subroutine ocn_import_moab!}}} +!*********************************************************************** +!BOP +! !IROUTINE: ocn_export_moab +! !INTERFACE: - subroutine ocn_export_moab(EClock) !{{{ - - ! !DESCRIPTION: - ! This routine calls the routines necessary to send mpas ocean fields to MOAB coupler - ! - use iMOAB, only : iMOAB_SetDoubleTagStorage, iMOAB_WriteMesh - !EOP - !BOC - type(ESMF_Clock) , intent(inout) :: EClock ! Input synchronization clock from driver - ! - ! local variables - ! - !----------------------------------------------------------------------- - integer :: ent_type, ierr, cur_ocn_stepno - character(len=100) :: outfile, wopts, localmeshfile, lnum - character(CXX) :: tagname - - integer :: i, n - integer, pointer :: nCellsSolve, index_temperatureSurfaceValue, index_salinitySurfaceValue, & - index_avgZonalSurfaceVelocity, index_avgMeridionalSurfaceVelocity, & - index_avgZonalSSHGradient, index_avgMeridionalSSHGradient + subroutine ocn_export_moab(EClock) !{{{ - type (block_type), pointer :: block_ptr +! !DESCRIPTION: +! This routine calls the routines necessary to send MPASO fields to +! the MOAB driver +! +! !REVISION HISTORY: +! same as module + use iMOAB, only : iMOAB_SetDoubleTagStorage, iMOAB_WriteMesh +! !INPUT/OUTPUT PARAMETERS: - type (mpas_pool_type), pointer :: meshPool, & - forcingPool, & - statePool, & - tracersPool, & - ecosysAuxiliary, & - ecosysSeaIceCoupling, & - DMSSeaIceCoupling, & - MacroMoleculesSeaIceCoupling - - integer, dimension(:), pointer :: landIceMask - - real (kind=RKIND), dimension(:), pointer :: seaIceEnergy, accumulatedFrazilIceMass, frazilSurfacePressure, & - avgTotalFreshWaterTemperatureFlux, & - avgCO2_gas_flux, DMSFlux, surfaceUpwardCO2Flux, & - avgOceanSurfaceDIC, & - avgOceanSurfaceDON, & - avgOceanSurfaceNO3, & - avgOceanSurfaceSiO3, & - avgOceanSurfaceNH4, & - avgOceanSurfaceDMS, & - avgOceanSurfaceDMSP, & - avgOceanSurfaceDOCr, & - avgOceanSurfaceDOCSemiLabile, & - avgOceanSurfaceFeParticulate, & - avgOceanSurfaceFeDissolved, & - ssh, & - avgLandIceFreshwaterFlux, & - avgRemovedRiverRunoffFlux, & - avgRemovedIceRunoffFlux, & - avgLandIceHeatFlux, & - avgRemovedIceRunoffHeatFlux - - real (kind=RKIND), dimension(:,:), pointer :: avgTracersSurfaceValue, avgSurfaceVelocity, & - avgSSHGradient, avgOceanSurfacePhytoC, & - avgOceanSurfaceDOC, layerThickness - - real (kind=RKIND) :: surfaceFreezingTemp - - logical, pointer :: frazilIceActive, & - config_remove_ais_river_runoff, & - config_remove_ais_ice_runoff, & - config_use_ecosysTracers, & - config_use_DMSTracers, & - config_use_MacroMoleculesTracers, & - config_use_ecosysTracers_sea_ice_coupling, & - config_use_DMSTracers_sea_ice_coupling, & - config_use_MacroMoleculesTracers_sea_ice_coupling - - character (len=StrKIND), pointer :: config_land_ice_flux_mode - - logical :: keepFrazil - - - ! get configure options - call mpas_pool_get_package(domain % packages, 'frazilIceActive', frazilIceActive) - call mpas_pool_get_config(domain % configs, 'config_use_ecosysTracers', config_use_ecosysTracers) - call mpas_pool_get_config(domain % configs, 'config_land_ice_flux_mode', config_land_ice_flux_mode) - call mpas_pool_get_config(domain % configs, 'config_remove_ais_river_runoff', config_remove_ais_river_runoff) - call mpas_pool_get_config(domain % configs, 'config_remove_ais_ice_runoff', config_remove_ais_ice_runoff) - call mpas_pool_get_config(domain % configs, 'config_use_DMSTracers', config_use_DMSTracers) - call mpas_pool_get_config(domain % configs, 'config_use_MacroMoleculesTracers', config_use_MacroMoleculesTracers) - call mpas_pool_get_config(domain % configs, 'config_use_ecosysTracers_sea_ice_coupling', & - config_use_ecosysTracers_sea_ice_coupling) - call mpas_pool_get_config(domain % configs, 'config_use_DMSTracers_sea_ice_coupling', & - config_use_DMSTracers_sea_ice_coupling) - call mpas_pool_get_config(domain % configs, 'config_use_MacroMoleculesTracers_sea_ice_coupling', & - config_use_MacroMoleculesTracers_sea_ice_coupling) - - n = 0 - block_ptr => domain % blocklist - do while(associated(block_ptr)) - call mpas_pool_get_subpool(block_ptr % structs, 'mesh', meshPool) - call mpas_pool_get_subpool(block_ptr % structs, 'forcing', forcingPool) - call mpas_pool_get_subpool(block_ptr % structs, 'state', statePool) + type(ESMF_Clock) , intent(inout) :: EClock ! Input synchronization clock from driver - call mpas_pool_get_subpool(statePool, 'tracers', tracersPool) +! !OUTPUT PARAMETERS: - call mpas_pool_get_dimension(meshPool, 'nCellsSolve', nCellsSolve) +!EOP +!BOC +!----------------------------------------------------------------------- +! +! local variables +! +!----------------------------------------------------------------------- + integer :: ent_type, ierr, cur_ocn_stepno + character(len=100) :: outfile, wopts, localmeshfile, lnum + character(CXX) :: tagname - call mpas_pool_get_dimension(forcingPool, 'index_avgTemperatureSurfaceValue', index_temperatureSurfaceValue) - call mpas_pool_get_dimension(forcingPool, 'index_avgSalinitySurfaceValue', index_salinitySurfaceValue) - call mpas_pool_get_dimension(forcingPool, 'index_avgSurfaceVelocityZonal', index_avgZonalSurfaceVelocity) - call mpas_pool_get_dimension(forcingPool, 'index_avgSurfaceVelocityMeridional', index_avgMeridionalSurfaceVelocity) - call mpas_pool_get_dimension(forcingPool, 'index_avgSSHGradientZonal', index_avgZonalSSHGradient) - call mpas_pool_get_dimension(forcingPool, 'index_avgSSHGradientMeridional', index_avgMeridionalSSHGradient) + integer :: i, n + integer, pointer :: nCellsSolve, index_temperatureSurfaceValue, index_salinitySurfaceValue, & + index_avgZonalSurfaceVelocity, index_avgMeridionalSurfaceVelocity, & + index_avgZonalSSHGradient, index_avgMeridionalSSHGradient + type (block_type), pointer :: block_ptr - call mpas_pool_get_array(statePool, 'ssh', ssh, 1) - call mpas_pool_get_array(statePool, 'layerThickness', layerThickness, 1) + type (mpas_pool_type), pointer :: meshPool, & + forcingPool, & + statePool, & + tracersPool, & + ecosysAuxiliary, & + ecosysSeaIceCoupling, & + DMSSeaIceCoupling, & + MacroMoleculesSeaIceCoupling - call mpas_pool_get_array(forcingPool, 'landIceMask', landIceMask) - call mpas_pool_get_array(forcingPool, 'avgTracersSurfaceValue', avgTracersSurfaceValue) - call mpas_pool_get_array(forcingPool, 'avgSurfaceVelocity', avgSurfaceVelocity) - call mpas_pool_get_array(forcingPool, 'avgSSHGradient', avgSSHGradient) - call mpas_pool_get_array(forcingPool, 'avgTotalFreshWaterTemperatureFlux', avgTotalFreshWaterTemperatureFlux) - if ( frazilIceActive ) then - call mpas_pool_get_array(forcingPool, 'seaIceEnergy', seaIceEnergy) - call mpas_pool_get_array(forcingPool, 'frazilSurfacePressure', frazilSurfacePressure) - call mpas_pool_get_array(statePool, 'accumulatedFrazilIceMass', accumulatedFrazilIceMass, 1) - end if + integer, dimension(:), pointer :: landIceMask - if (trim(config_land_ice_flux_mode) == 'standalone' .or. trim(config_land_ice_flux_mode) == 'data') then - call mpas_pool_get_array(forcingPool, 'avgLandIceFreshwaterFlux', avgLandIceFreshwaterFlux) - call mpas_pool_get_array(forcingPool, 'avgLandIceHeatFlux', avgLandIceHeatFlux) - endif - if (config_remove_ais_river_runoff) then - call mpas_pool_get_array(forcingPool, 'avgRemovedRiverRunoffFlux', avgRemovedRiverRunoffFlux) - endif - if (config_remove_ais_ice_runoff) then - call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoffFlux', avgRemovedIceRunoffFlux) - call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoffHeatFlux', avgRemovedIceRunoffHeatFlux) - endif + real (kind=RKIND), dimension(:), pointer :: seaIceEnergy, accumulatedFrazilIceMass, frazilSurfacePressure, & + avgTotalFreshWaterTemperatureFlux, & + avgCO2_gas_flux, DMSFlux, surfaceUpwardCO2Flux, & + avgOceanSurfaceDIC, & + avgOceanSurfaceDON, & + avgOceanSurfaceNO3, & + avgOceanSurfaceSiO3, & + avgOceanSurfaceNH4, & + avgOceanSurfaceDMS, & + avgOceanSurfaceDMSP, & + avgOceanSurfaceDOCr, & + avgOceanSurfaceDOCSemiLabile, & + avgOceanSurfaceFeParticulate, & + avgOceanSurfaceFeDissolved, & + ssh, & + avgLandIceFreshwaterFlux, & + avgRemovedRiverRunoffFlux, & + avgRemovedIceRunoffFlux, & + avgLandIceHeatFlux, & + avgRemovedIceRunoffHeatFlux - ! BGC fields - if (config_use_ecosysTracers) then + real (kind=RKIND), dimension(:,:), pointer :: avgTracersSurfaceValue, avgSurfaceVelocity, & + avgSSHGradient, avgOceanSurfacePhytoC, & + avgOceanSurfaceDOC, layerThickness - call mpas_pool_get_subpool(forcingPool, 'ecosysAuxiliary', ecosysAuxiliary) - call mpas_pool_get_array(ecosysAuxiliary, 'avgCO2_gas_flux', avgCO2_gas_flux) + real (kind=RKIND) :: surfaceFreezingTemp - end if + logical, pointer :: frazilIceActive, & + config_remove_ais_river_runoff, & + config_remove_ais_ice_runoff, & + config_use_ecosysTracers, & + config_use_DMSTracers, & + config_use_MacroMoleculesTracers, & + config_use_ecosysTracers_sea_ice_coupling, & + config_use_DMSTracers_sea_ice_coupling, & + config_use_MacroMoleculesTracers_sea_ice_coupling - if (config_use_ecosysTracers .and. config_use_ecosysTracers_sea_ice_coupling) then - call mpas_pool_get_subpool(forcingPool, 'ecosysSeaIceCoupling', ecosysSeaIceCoupling) + character (len=StrKIND), pointer :: config_land_ice_flux_mode - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfacePhytoC', avgOceanSurfacePhytoC) - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDIC', avgOceanSurfaceDIC) - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceNO3', avgOceanSurfaceNO3) - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceSiO3', avgOceanSurfaceSiO3) - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceNH4', avgOceanSurfaceNH4) - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCr', avgOceanSurfaceDOCr) - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCSemiLabile', avgOceanSurfaceDOCSemiLabile) - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceFeParticulate', avgOceanSurfaceFeParticulate) - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceFeDissolved', avgOceanSurfaceFeDissolved) - endif - if (config_use_DMSTracers .and. config_use_DMSTracers_sea_ice_coupling) then - call mpas_pool_get_subpool(forcingPool, 'DMSSeaIceCoupling', DMSSeaIceCoupling) + logical :: keepFrazil - call mpas_pool_get_array(DMSSeaIceCoupling, 'avgOceanSurfaceDMS', avgOceanSurfaceDMS) - call mpas_pool_get_array(DMSSeaIceCoupling, 'avgOceanSurfaceDMSP', avgOceanSurfaceDMSP) - endif - if (config_use_MacroMoleculesTracers .and. config_use_MacroMoleculesTracers_sea_ice_coupling) then - call mpas_pool_get_subpool(forcingPool, 'MacroMoleculesSeaIceCoupling', MacroMoleculesSeaIceCoupling) - call mpas_pool_get_array(MacroMoleculesSeaIceCoupling, 'avgOceanSurfaceDOC', avgOceanSurfaceDOC) - call mpas_pool_get_array(MacroMoleculesSeaIceCoupling, 'avgOceanSurfaceDON', avgOceanSurfaceDON) - endif - ! call mpas_pool_get_array(forcingPool, 'CO2Flux', CO2Flux) - ! call mpas_pool_get_array(forcingPool, 'DMSFlux', DMSFlux) - ! call mpas_pool_get_array(forcingPool, 'surfaceUpwardCO2Flux', surfaceUpwardCO2Flux) + ! get configure options + call mpas_pool_get_package(domain % packages, 'frazilIceActive', frazilIceActive) + call mpas_pool_get_config(domain % configs, 'config_use_ecosysTracers', config_use_ecosysTracers) + call mpas_pool_get_config(domain % configs, 'config_land_ice_flux_mode', config_land_ice_flux_mode) + call mpas_pool_get_config(domain % configs, 'config_remove_ais_river_runoff', config_remove_ais_river_runoff) + call mpas_pool_get_config(domain % configs, 'config_remove_ais_ice_runoff', config_remove_ais_ice_runoff) + call mpas_pool_get_config(domain % configs, 'config_use_DMSTracers', config_use_DMSTracers) + call mpas_pool_get_config(domain % configs, 'config_use_MacroMoleculesTracers', config_use_MacroMoleculesTracers) + call mpas_pool_get_config(domain % configs, 'config_use_ecosysTracers_sea_ice_coupling', & + config_use_ecosysTracers_sea_ice_coupling) + call mpas_pool_get_config(domain % configs, 'config_use_DMSTracers_sea_ice_coupling', & + config_use_DMSTracers_sea_ice_coupling) + call mpas_pool_get_config(domain % configs, 'config_use_MacroMoleculesTracers_sea_ice_coupling', & + config_use_MacroMoleculesTracers_sea_ice_coupling) + + n = 0 + block_ptr => domain % blocklist + do while(associated(block_ptr)) + call mpas_pool_get_subpool(block_ptr % structs, 'mesh', meshPool) + call mpas_pool_get_subpool(block_ptr % structs, 'forcing', forcingPool) + call mpas_pool_get_subpool(block_ptr % structs, 'state', statePool) - do i = 1, nCellsSolve - n = n + 1 + call mpas_pool_get_subpool(statePool, 'tracers', tracersPool) - o2x_om(n, index_o2x_So_t) = avgTracersSurfaceValue(index_temperatureSurfaceValue, i) - o2x_om(n, index_o2x_So_s) = avgTracersSurfaceValue(index_salinitySurfaceValue, i) - o2x_om(n, index_o2x_So_u) = avgSurfaceVelocity(index_avgZonalSurfaceVelocity, i) - o2x_om(n, index_o2x_So_v) = avgSurfaceVelocity(index_avgMeridionalSurfaceVelocity, i) + call mpas_pool_get_dimension(meshPool, 'nCellsSolve', nCellsSolve) - o2x_om(n, index_o2x_So_ssh) = ssh(i) - o2x_om(n, index_o2x_So_dhdx) = avgSSHGradient(index_avgZonalSSHGradient, i) - o2x_om(n, index_o2x_So_dhdy) = avgSSHGradient(index_avgMeridionalSSHGradient, i) + call mpas_pool_get_dimension(forcingPool, 'index_avgTemperatureSurfaceValue', index_temperatureSurfaceValue) + call mpas_pool_get_dimension(forcingPool, 'index_avgSalinitySurfaceValue', index_salinitySurfaceValue) + call mpas_pool_get_dimension(forcingPool, 'index_avgSurfaceVelocityZonal', index_avgZonalSurfaceVelocity) + call mpas_pool_get_dimension(forcingPool, 'index_avgSurfaceVelocityMeridional', index_avgMeridionalSurfaceVelocity) + call mpas_pool_get_dimension(forcingPool, 'index_avgSSHGradientZonal', index_avgZonalSSHGradient) + call mpas_pool_get_dimension(forcingPool, 'index_avgSSHGradientMeridional', index_avgMeridionalSSHGradient) - o2x_om(n, index_o2x_Faoo_h2otemp) = avgTotalFreshWaterTemperatureFlux(i) * rho_sw * cp_sw + call mpas_pool_get_array(statePool, 'ssh', ssh, 1) + call mpas_pool_get_array(statePool, 'layerThickness', layerThickness, 1) - if (trim(config_land_ice_flux_mode) == 'standalone' .or. trim(config_land_ice_flux_mode) == 'data') then - o2x_om(n, index_o2x_Foxo_ismw) = avgLandIceFreshwaterFlux(i) - o2x_om(n, index_o2x_Foxo_ismh) = avgLandIceHeatFlux(i) - endif - if (config_remove_ais_river_runoff) then - o2x_om(n, index_o2x_Foxo_rrofl) = avgRemovedRiverRunoffFlux(i) - endif - if (config_remove_ais_ice_runoff) then - o2x_om(n, index_o2x_Foxo_rrofi) = avgRemovedIceRunoffFlux(i) - o2x_om(n, index_o2x_Foxo_rrofih) = avgRemovedIceRunoffHeatFlux(i) - endif + call mpas_pool_get_array(forcingPool, 'landIceMask', landIceMask) + call mpas_pool_get_array(forcingPool, 'avgTracersSurfaceValue', avgTracersSurfaceValue) + call mpas_pool_get_array(forcingPool, 'avgSurfaceVelocity', avgSurfaceVelocity) + call mpas_pool_get_array(forcingPool, 'avgSSHGradient', avgSSHGradient) + call mpas_pool_get_array(forcingPool, 'avgTotalFreshWaterTemperatureFlux', avgTotalFreshWaterTemperatureFlux) - if ( frazilIceActive ) then - ! negative when frazil ice can be melted - keepFrazil = .true. - if ( associated(landIceMask) ) then - if ( landIceMask(i) == 1 ) then - keepFrazil = .false. - end if - end if + if ( frazilIceActive ) then + call mpas_pool_get_array(forcingPool, 'seaIceEnergy', seaIceEnergy) + call mpas_pool_get_array(forcingPool, 'frazilSurfacePressure', frazilSurfacePressure) + call mpas_pool_get_array(statePool, 'accumulatedFrazilIceMass', accumulatedFrazilIceMass, 1) + end if - if ( keepFrazil ) then + ! Cryo fields + if (trim(config_land_ice_flux_mode) == 'standalone' .or. trim(config_land_ice_flux_mode) == 'data') then + call mpas_pool_get_array(forcingPool, 'avgLandIceFreshwaterFlux', avgLandIceFreshwaterFlux) + call mpas_pool_get_array(forcingPool, 'avgLandIceHeatFlux', avgLandIceHeatFlux) + endif + if (config_remove_ais_river_runoff) then + call mpas_pool_get_array(forcingPool, 'avgRemovedRiverRunoffFlux', avgRemovedRiverRunoffFlux) + endif + if (config_remove_ais_ice_runoff) then + call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoffFlux', avgRemovedIceRunoffFlux) + call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoffHeatFlux', avgRemovedIceRunoffHeatFlux) + endif - ! Calculate energy associated with frazil mass transfer to sea ice if frazil has accumulated - if ( accumulatedFrazilIceMass(i) > 0.0_RKIND ) then + ! BGC fields + if (config_use_ecosysTracers) then - seaIceEnergy(i) = accumulatedFrazilIceMass(i) * config_frazil_heat_of_fusion + call mpas_pool_get_subpool(forcingPool, 'ecosysAuxiliary', ecosysAuxiliary) + call mpas_pool_get_array(ecosysAuxiliary, 'avgCO2_gas_flux', avgCO2_gas_flux) - ! Otherwise calculate the melt potential where avgTracersSurfaceValue represents only the - ! top layer of the ocean - else + end if - surfaceFreezingTemp = ocn_freezing_temperature(salinity=avgTracersSurfaceValue(index_salinitySurfaceValue, i), & - pressure=0.0_RKIND, inLandIceCavity=.false.) + if (config_use_ecosysTracers .and. config_use_ecosysTracers_sea_ice_coupling) then + call mpas_pool_get_subpool(forcingPool, 'ecosysSeaIceCoupling', ecosysSeaIceCoupling) - seaIceEnergy(i) = min(rho_sw*cp_sw*layerThickness(1, i)*( surfaceFreezingTemp + T0_Kelvin & - - avgTracersSurfaceValue(index_temperatureSurfaceValue, i) ), 0.0_RKIND ) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfacePhytoC', avgOceanSurfacePhytoC) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDIC', avgOceanSurfaceDIC) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceNO3', avgOceanSurfaceNO3) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceSiO3', avgOceanSurfaceSiO3) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceNH4', avgOceanSurfaceNH4) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCr', avgOceanSurfaceDOCr) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCSemiLabile', avgOceanSurfaceDOCSemiLabile) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceFeParticulate', avgOceanSurfaceFeParticulate) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceFeDissolved', avgOceanSurfaceFeDissolved) + endif + if (config_use_DMSTracers .and. config_use_DMSTracers_sea_ice_coupling) then + call mpas_pool_get_subpool(forcingPool, 'DMSSeaIceCoupling', DMSSeaIceCoupling) - end if + call mpas_pool_get_array(DMSSeaIceCoupling, 'avgOceanSurfaceDMS', avgOceanSurfaceDMS) + call mpas_pool_get_array(DMSSeaIceCoupling, 'avgOceanSurfaceDMSP', avgOceanSurfaceDMSP) + endif + if (config_use_MacroMoleculesTracers .and. config_use_MacroMoleculesTracers_sea_ice_coupling) then + call mpas_pool_get_subpool(forcingPool, 'MacroMoleculesSeaIceCoupling', MacroMoleculesSeaIceCoupling) - o2x_om(n, index_o2x_Fioo_q) = seaIceEnergy(i) / ocn_cpl_dt - o2x_om(n, index_o2x_Fioo_frazil) = accumulatedFrazilIceMass(i) / ocn_cpl_dt + call mpas_pool_get_array(MacroMoleculesSeaIceCoupling, 'avgOceanSurfaceDOC', avgOceanSurfaceDOC) + call mpas_pool_get_array(MacroMoleculesSeaIceCoupling, 'avgOceanSurfaceDON', avgOceanSurfaceDON) + endif +! call mpas_pool_get_array(forcingPool, 'CO2Flux', CO2Flux) +! call mpas_pool_get_array(forcingPool, 'DMSFlux', DMSFlux) +! call mpas_pool_get_array(forcingPool, 'surfaceUpwardCO2Flux', surfaceUpwardCO2Flux) - else +! replace 'o2x_o % rAttr(' with 'o2x_om(n, ' and ', n)' with ')' + do i = 1, nCellsSolve + n = n + 1 - o2x_om(n, index_o2x_Fioo_q) = 0.0_RKIND - o2x_om(n, index_o2x_Fioo_frazil) = 0.0_RKIND - if (trim(config_land_ice_flux_mode) == 'standalone' .or. trim(config_land_ice_flux_mode) == 'data') then - o2x_om(n, index_o2x_Foxo_q_li) = accumulatedFrazilIceMass(i) * config_frazil_heat_of_fusion / ocn_cpl_dt - o2x_om(n, index_o2x_Foxo_frazil_li) = accumulatedFrazilIceMass(i) / ocn_cpl_dt - endif - end if + o2x_om(n, index_o2x_So_t) = avgTracersSurfaceValue(index_temperatureSurfaceValue, i) + o2x_om(n, index_o2x_So_s) = avgTracersSurfaceValue(index_salinitySurfaceValue, i) + o2x_om(n, index_o2x_So_u) = avgSurfaceVelocity(index_avgZonalSurfaceVelocity, i) + o2x_om(n, index_o2x_So_v) = avgSurfaceVelocity(index_avgMeridionalSurfaceVelocity, i) - ! Reset SeaIce Energy and Accumulated Frazil Ice - seaIceEnergy(i) = 0.0_RKIND - accumulatedFrazilIceMass(i) = 0.0_RKIND - frazilSurfacePressure(i) = 0.0_RKIND - end if + o2x_om(n, index_o2x_So_ssh) = ssh(i) + o2x_om(n, index_o2x_So_dhdx) = avgSSHGradient(index_avgZonalSSHGradient, i) + o2x_om(n, index_o2x_So_dhdy) = avgSSHGradient(index_avgMeridionalSSHGradient, i) - ! BGC fields - if (config_use_ecosysTracers) then - ! convert from mmolC/m2/s to kg CO2/m2/s - o2x_om(n, index_o2x_Faoo_fco2_ocn) = avgCO2_gas_flux(i)*44.e-6_RKIND - endif - if (config_use_ecosysTracers .and. config_use_ecosysTracers_sea_ice_coupling) then - o2x_om(n, index_o2x_So_algae1) = max(0.0_RKIND,avgOceanSurfacePhytoC(1,i)) - o2x_om(n, index_o2x_So_algae2) = max(0.0_RKIND,avgOceanSurfacePhytoC(2,i)) - o2x_om(n, index_o2x_So_algae3) = max(0.0_RKIND,avgOceanSurfacePhytoC(3,i)) - o2x_om(n, index_o2x_So_dic1) = max(0.0_RKIND,avgOceanSurfaceDIC(i)) - o2x_om(n, index_o2x_So_doc1) = max(0.0_RKIND,avgOceanSurfaceDOCSemiLabile(i)) - o2x_om(n, index_o2x_So_doc2) = max(0.0_RKIND,avgOceanSurfaceDOCSemiLabile(i)) - o2x_om(n, index_o2x_So_doc3) = max(0.0_RKIND,avgOceanSurfaceDOCSemiLabile(i)) - o2x_om(n, index_o2x_So_don1) = 0.0_RKIND - o2x_om(n, index_o2x_So_no3) = max(0.0_RKIND,avgOceanSurfaceNO3(i)) - o2x_om(n, index_o2x_So_sio3) = max(0.0_RKIND,avgOceanSurfaceSiO3(i)) - o2x_om(n, index_o2x_So_nh4) = max(0.0_RKIND,avgOceanSurfaceNH4(i)) - o2x_om(n, index_o2x_So_docr) = max(0.0_RKIND,avgOceanSurfaceDOCr(i)) - o2x_om(n, index_o2x_So_fep1) = max(0.0_RKIND,avgOceanSurfaceFeParticulate(i)) - o2x_om(n, index_o2x_So_fed1) = max(0.0_RKIND,avgOceanSurfaceFeDissolved(i)) - endif - if (config_use_DMSTracers .and. config_use_DMSTracers_sea_ice_coupling) then - o2x_om(n, index_o2x_So_dms) = max(0.0_RKIND,avgOceanSurfaceDMS(i)) - o2x_om(n, index_o2x_So_dmsp) = max(0.0_RKIND,avgOceanSurfaceDMSP(i)) - endif - if (config_use_MacroMoleculesTracers .and. config_use_MacroMoleculesTracers_sea_ice_coupling) then - o2x_om(n, index_o2x_So_doc1) = max(0.0_RKIND,avgOceanSurfaceDOC(1,i)) - o2x_om(n, index_o2x_So_doc2) = max(0.0_RKIND,avgOceanSurfaceDOC(2,i)) - o2x_om(n, index_o2x_So_don1) = max(0.0_RKIND,avgOceanSurfaceDON(i)) - endif + o2x_om(n, index_o2x_Faoo_h2otemp) = avgTotalFreshWaterTemperatureFlux(i) * rho_sw * cp_sw - if ( trim(config_land_ice_flux_mode) .eq. 'standalone' .or. & - trim(config_land_ice_flux_mode) .eq. 'coupled' ) then - o2x_om(n, index_o2x_So_blt) = landIceBoundaryLayerTracers(indexBLT,i) - o2x_om(n, index_o2x_So_bls) = landIceBoundaryLayerTracers(indexBLS,i) - o2x_om(n, index_o2x_So_htv) = landIceTracerTransferVelocities(indexHeatTrans,i) - o2x_om(n, index_o2x_So_stv) = landIceTracerTransferVelocities(indexSaltTrans,i) - o2x_om(n, index_o2x_So_rhoeff) = 0.0_RKIND - endif - end do + ! Cryo fields + if (trim(config_land_ice_flux_mode) == 'standalone' .or. trim(config_land_ice_flux_mode) == 'data') then + o2x_om(n, index_o2x_Foxo_ismw) = avgLandIceFreshwaterFlux(i) + o2x_om(n, index_o2x_Foxo_ismh) = avgLandIceHeatFlux(i) + endif + if (config_remove_ais_river_runoff) then + o2x_om(n, index_o2x_Foxo_rrofl) = avgRemovedRiverRunoffFlux(i) + endif + if (config_remove_ais_ice_runoff) then + o2x_om(n, index_o2x_Foxo_rrofi) = avgRemovedIceRunoffFlux(i) + o2x_om(n, index_o2x_Foxo_rrofih) = avgRemovedIceRunoffHeatFlux(i) + endif - block_ptr => block_ptr % next - end do + if ( frazilIceActive ) then + ! negative when frazil ice can be melted + keepFrazil = .true. + if ( associated(landIceMask) ) then + if ( landIceMask(i) == 1 ) then + keepFrazil = .false. + end if + end if + + if ( keepFrazil ) then + + ! Calculate energy associated with frazil mass transfer to sea ice if frazil has accumulated + if ( accumulatedFrazilIceMass(i) > 0.0_RKIND ) then + + seaIceEnergy(i) = accumulatedFrazilIceMass(i) * config_frazil_heat_of_fusion + + ! Otherwise calculate the melt potential where avgTracersSurfaceValue represents only the + ! top layer of the ocean + else + + surfaceFreezingTemp = ocn_freezing_temperature(salinity=avgTracersSurfaceValue(index_salinitySurfaceValue, i), & + pressure=0.0_RKIND, inLandIceCavity=.false.) + + seaIceEnergy(i) = min(rho_sw*cp_sw*layerThickness(1, i)*( surfaceFreezingTemp + T0_Kelvin & + - avgTracersSurfaceValue(index_temperatureSurfaceValue, i) ), 0.0_RKIND ) + + end if + + o2x_om(n, index_o2x_Fioo_q) = seaIceEnergy(i) / ocn_cpl_dt + o2x_om(n, index_o2x_Fioo_frazil) = accumulatedFrazilIceMass(i) / ocn_cpl_dt + + else + + o2x_om(n, index_o2x_Fioo_q) = 0.0_RKIND + o2x_om(n, index_o2x_Fioo_frazil) = 0.0_RKIND + if (trim(config_land_ice_flux_mode) == 'standalone' .or. trim(config_land_ice_flux_mode) == 'data') then + o2x_om(n, index_o2x_Foxo_q_li) = accumulatedFrazilIceMass(i) * config_frazil_heat_of_fusion / ocn_cpl_dt + o2x_om(n, index_o2x_Foxo_frazil_li) = accumulatedFrazilIceMass(i) / ocn_cpl_dt + endif + + end if + + ! Reset SeaIce Energy and Accumulated Frazil Ice + seaIceEnergy(i) = 0.0_RKIND + accumulatedFrazilIceMass(i) = 0.0_RKIND + frazilSurfacePressure(i) = 0.0_RKIND + end if + + ! BGC fields + if (config_use_ecosysTracers .and. index_o2x_Faoo_fco2_ocn /= 0) then + ! convert from mmolC/m2/s to kg CO2/m2/s + o2x_om(n, index_o2x_Faoo_fco2_ocn) = avgCO2_gas_flux(i)*44.e-6_RKIND + endif + if (config_use_ecosysTracers .and. config_use_ecosysTracers_sea_ice_coupling) then + o2x_om(n, index_o2x_So_algae1) = max(0.0_RKIND,avgOceanSurfacePhytoC(1,i)) + o2x_om(n, index_o2x_So_algae2) = max(0.0_RKIND,avgOceanSurfacePhytoC(2,i)) + o2x_om(n, index_o2x_So_algae3) = max(0.0_RKIND,avgOceanSurfacePhytoC(3,i)) + o2x_om(n, index_o2x_So_dic1) = max(0.0_RKIND,avgOceanSurfaceDIC(i)) + o2x_om(n, index_o2x_So_doc1) = max(0.0_RKIND,avgOceanSurfaceDOCSemiLabile(i)) + o2x_om(n, index_o2x_So_doc2) = max(0.0_RKIND,avgOceanSurfaceDOCSemiLabile(i)) + o2x_om(n, index_o2x_So_doc3) = max(0.0_RKIND,avgOceanSurfaceDOCSemiLabile(i)) + o2x_om(n, index_o2x_So_don1) = 0.0_RKIND + o2x_om(n, index_o2x_So_no3) = max(0.0_RKIND,avgOceanSurfaceNO3(i)) + o2x_om(n, index_o2x_So_sio3) = max(0.0_RKIND,avgOceanSurfaceSiO3(i)) + o2x_om(n, index_o2x_So_nh4) = max(0.0_RKIND,avgOceanSurfaceNH4(i)) + o2x_om(n, index_o2x_So_docr) = max(0.0_RKIND,avgOceanSurfaceDOCr(i)) + o2x_om(n, index_o2x_So_fep1) = max(0.0_RKIND,avgOceanSurfaceFeParticulate(i)) + o2x_om(n, index_o2x_So_fed1) = max(0.0_RKIND,avgOceanSurfaceFeDissolved(i)) + endif + if (config_use_DMSTracers .and. config_use_DMSTracers_sea_ice_coupling) then + o2x_om(n, index_o2x_So_dms) = max(0.0_RKIND,avgOceanSurfaceDMS(i)) + o2x_om(n, index_o2x_So_dmsp) = max(0.0_RKIND,avgOceanSurfaceDMSP(i)) + endif + if (config_use_MacroMoleculesTracers .and. config_use_MacroMoleculesTracers_sea_ice_coupling) then + o2x_om(n, index_o2x_So_doc1) = max(0.0_RKIND,avgOceanSurfaceDOC(1,i)) + o2x_om(n, index_o2x_So_doc2) = max(0.0_RKIND,avgOceanSurfaceDOC(2,i)) + o2x_om(n, index_o2x_So_doc3) = max(0.0_RKIND,avgOceanSurfaceDOC(3,i)) + o2x_om(n, index_o2x_So_don1) = 0.0_RKIND + endif +! o2x_om(n, index_o2x_Faoo_fco2_ocn) = CO2Flux(i) +! o2x_om(n, index_o2x_Faoo_fdms_ocn) = DMSFlux(i) +! o2x_om(n, index_o2x_Faoo_fco2_ocn) = surfaceUpwardCO2Flux(i) + +!JW o2x_om(n, index_o2x_So_blt) = landIceBoundaryLayerTemperature(i) +!JW o2x_om(n, index_o2x_So_bls) = landIceBoundaryLayerSalinity(i) +!JW o2x_om(n, index_o2x_So_htv) = landIceHeatTransferVelocity(i) +!JW o2x_om(n, index_o2x_So_stv) = landIceSaltTransferVelocity(i) + + if ( trim(config_land_ice_flux_mode) .eq. 'standalone' .or. & + trim(config_land_ice_flux_mode) .eq. 'coupled' ) then + o2x_om(n, index_o2x_So_blt) = landIceBoundaryLayerTracers(indexBLT,i) + o2x_om(n, index_o2x_So_bls) = landIceBoundaryLayerTracers(indexBLS,i) + o2x_om(n, index_o2x_So_htv) = landIceTracerTransferVelocities(indexHeatTrans,i) + o2x_om(n, index_o2x_So_stv) = landIceTracerTransferVelocities(indexSaltTrans,i) + o2x_om(n, index_o2x_So_rhoeff) = 0.0_RKIND + endif + + !Fyke: test + !write(stderrUnit,*) 'n=',n + !write(stderrUnit,*) 'o2x_om(n, index_o2x_So_blt)=',o2x_om(n, index_o2x_So_blt) + !write(stderrUnit,*) 'o2x_om(n, index_o2x_So_bls)=',o2x_om(n, index_o2x_So_bls) + !write(stderrUnit,*) 'o2x_om(n, index_o2x_So_htv)=',o2x_om(n, index_o2x_So_htv) + !write(stderrUnit,*) 'o2x_om(n, index_o2x_So_stv)=',o2x_om(n, index_o2x_So_stv) + !write(stderrUnit,*) 'o2x_om(n, index_o2x_So_rhoeff)=',o2x_om(n, index_o2x_So_rhoeff) + !o2x_om(n, index_o2x_So_blt) = 0._r8 + !o2x_om(n, index_o2x_So_bls) = 34.5_r8 + !o2x_om(n, index_o2x_So_htv) = 1.e-4_r8 + !o2x_om(n, index_o2x_So_stv) = 3.e-6_r8 + !o2x_om(n, index_o2x_So_rhoeff) = 1000._r8*9.81_r8*918._r8 !lithostatic pressure of 1km of ice + + end do + block_ptr => block_ptr % next + end do ent_type = 1 ! cells ! set all tags in one method tagname = trim(seq_flds_o2x_fields)//C_NULL_CHAR @@ -4251,9 +4571,12 @@ subroutine ocn_export_moab(EClock) !{{{ wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR ierr = iMOAB_WriteMesh(MPOID, outfile, wopts) #endif - end subroutine ocn_export_moab!}}} -#endif +!----------------------------------------------------------------------- +!EOC + + end subroutine ocn_export_moab!}}} +#endif end module ocn_comp_mct !||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 2a21e22f40d..87100a22a86 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -113,6 +113,9 @@ + @@ -807,6 +810,22 @@ description="If true, solid runoff from the Antarctic Ice Sheet (below 60S latitude) coming from the coupled is zeroed in the coupler import routines. To be used with data iceberg fluxes coming from the sea ice model." possible_values=".true. or .false." /> + + + + - @@ -1717,6 +1736,7 @@ + @@ -2008,6 +2028,7 @@ + @@ -2043,6 +2064,11 @@ + + + + + + + + + + + @@ -4010,6 +4060,9 @@ description="The time-averaged effective ocean density within ice shelves based on Archimedes' principle." packages="landIceCouplingPKG" /> + + + + domain % blocklist + call mpas_pool_get_subpool(block % structs, 'forcing', forcingPool) + call mpas_pool_get_array(globalStatsAMPool, 'gsRunningMeanRemovedIceRunoff', gsRunningMeanRemovedIceRunoff) + call mpas_pool_get_array(forcingPool, 'runningMeanRemovedIceRunoff', runningMeanRemovedIceRunoff) + gsRunningMeanRemovedIceRunoff = runningMeanRemovedIceRunoff + end if + ! calculate fresh water conservation check quantities absoluteFreshWaterConservation = totalVolumeChange - netFreshwaterInput if (abs(totalVolumeChange) < 1e-12_RKIND) then diff --git a/components/mpas-ocean/src/driver/mpas_ocn_core_interface.F b/components/mpas-ocean/src/driver/mpas_ocn_core_interface.F index 6d542fb0c3f..66c2349e151 100644 --- a/components/mpas-ocean/src/driver/mpas_ocn_core_interface.F +++ b/components/mpas-ocean/src/driver/mpas_ocn_core_interface.F @@ -140,6 +140,7 @@ function ocn_setup_packages(configPool, packagePool, iocontext) result(ierr)!{{{ logical, pointer :: verticalRemapPKGActive logical, pointer :: activeWavePKGActive logical, pointer :: subgridWetDryPKGActive + logical, pointer :: scaledDISMFPKGActive type (mpas_pool_iterator_type) :: pkgItr logical, pointer :: packageActive @@ -175,6 +176,7 @@ function ocn_setup_packages(configPool, packagePool, iocontext) result(ierr)!{{{ logical, pointer :: config_use_gotm logical, pointer :: config_use_active_wave logical, pointer :: config_use_subgrid_wetting_drying + logical, pointer :: config_scale_dismf_by_removed_ice_runoff character (len=StrKIND), pointer :: config_time_integrator character (len=StrKIND), pointer :: config_ocean_run_mode @@ -449,6 +451,17 @@ function ocn_setup_packages(configPool, packagePool, iocontext) result(ierr)!{{{ subgridWetDryPKGActive = .true. end if + ! + ! test for scaling data ice-shelf melt fluxes by the running mean of removed ice runoff + ! + call mpas_pool_get_package(packagePool, 'scaledDISMFPKGActive', scaledDISMFPKGActive) + call mpas_pool_get_config(configPool, & + 'config_scale_dismf_by_removed_ice_runoff', & + config_scale_dismf_by_removed_ice_runoff) + if (config_scale_dismf_by_removed_ice_runoff) then + scaledDISMFPKGActive = .true. + end if + ! ! call into analysis member driver to set analysis member packages ! diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F index d5ba6b08eb6..b7ec0bed057 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F @@ -26,6 +26,7 @@ module ocn_forward_mode use mpas_stream_manager use mpas_timekeeping use mpas_dmpar + use mpas_forcing use mpas_timer use mpas_log use mpas_decomp @@ -94,6 +95,7 @@ module ocn_forward_mode use ocn_forcing use ocn_time_varying_forcing + use ocn_framework_forcing use ocn_constants use ocn_config @@ -664,7 +666,12 @@ function ocn_forward_mode_run(domain) result(ierr)!{{{ ! initialize time-varying forcing call ocn_time_varying_forcing_init(domain) - call ocn_time_varying_forcing_get(domain % streamManager, domain, domain % clock) + + ! if not using RK4, calculate time varying forcing terms once per + ! time-step as opposed at each RK substage as implemented in RK4 + if (timeIntegratorChoice /= timeIntRK4) then + call ocn_time_varying_forcing_get(domain % streamManager, domain, domain % clock) + endif ! During integration, time level 1 stores the model state at the beginning of the ! time step, and time level 2 stores the state advanced dt in time by timestep(...) @@ -834,7 +841,17 @@ function ocn_forward_mode_run(domain) result(ierr)!{{{ endif ! read in next time level data required for time-varying forcing - call ocn_time_varying_forcing_get(domain % streamManager, domain, domain % clock) + if (timeIntegratorChoice /= timeIntRK4) then + ! if not using RK4, calculate time varying forcing terms once per + ! time-step as opposed at each RK substage as implemented in RK4 + call ocn_time_varying_forcing_get(domain % streamManager, domain, domain % clock) + else + if (config_use_time_varying_atmospheric_forcing .or. & + config_use_time_varying_land_ice_forcing) then + ! increment forcing clock to next time-step + call mpas_advance_forcing_clock(forcingGroupHead, dt) + endif + endif ! Validate that the state is OK to run with for the next timestep. call ocn_validate_state(domain, timeLevel=1) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration.F index 68edc933535..89ab1294514 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration.F @@ -48,6 +48,19 @@ module ocn_time_integration ! !-------------------------------------------------------------------- + ! Enum for selecting different time integrators + integer, public :: timeIntegratorChoice + + integer, public, parameter :: & + timeIntUnknown = 0, &! unknown or undefined + timeIntSplitExplicit = 1, &! split-explicit + timeIntUnsplitExplicit = 2, &! unsplit-explicit + timeIntSemiImplicit = 3, &! Semi-implicit + timeIntRK4 = 4, &! 4th-order Runge-Kutta + timeIntLTS = 5, &! local time-stepping + timeIntFBLTS = 6, &! forward-backward lts + timeIntSplitExplicitAB2 = 7 ! split-explicit AB2 baroclinic + !-------------------------------------------------------------------- ! ! Public member functions @@ -63,18 +76,6 @@ module ocn_time_integration ! !-------------------------------------------------------------------- - ! Enum for selecting different time integrators - integer :: timeIntegratorChoice - - integer, parameter :: & - timeIntUnknown = 0, &! unknown or undefined - timeIntSplitExplicit = 1, &! split-explicit - timeIntUnsplitExplicit = 2, &! unsplit-explicit - timeIntSemiImplicit = 3, &! Semi-implicit - timeIntRK4 = 4, &! 4th-order Runge-Kutta - timeIntLTS = 5, &! local time-stepping - timeIntFBLTS = 6, &! forward-backward lts - timeIntSplitExplicitAB2 = 7 ! split-explicit AB2 baroclinic !*********************************************************************** diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_rk4.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_rk4.F index 5fbb56e6ec1..9c4cda353df 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_rk4.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_rk4.F @@ -43,6 +43,7 @@ module ocn_time_integration_rk4 use ocn_effective_density_in_land_ice use ocn_surface_land_ice_fluxes use ocn_transport_tests + use ocn_time_varying_forcing use ocn_subgrid @@ -118,7 +119,7 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ type (mpas_pool_type), pointer :: nextProvisPool, prevProvisPool - real (kind=RKIND), dimension(4) :: rk_weights, rk_substep_weights + real (kind=RKIND), dimension(4) :: rk_weights, rk_substep_weights, forcingTimeIncrementRK4 real (kind=RKIND) :: coef real (kind=RKIND), dimension(:,:), pointer :: & @@ -186,6 +187,7 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ ! Tidal boundary condition logical, pointer :: config_use_tidal_forcing character (len=StrKIND), pointer :: config_tidal_forcing_type + real (kind=RKIND), pointer :: forcingTimeIncrement real (kind=RKIND), dimension(:), pointer :: tidalInputMask, tidalBCValue real (kind=RKIND), dimension(:,:), pointer :: restingThickness real (kind=RKIND) :: totalDepth @@ -233,6 +235,7 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ call mpas_pool_get_subpool(statePool, 'tracers', tracersPool) call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) call mpas_pool_get_subpool(block % structs, 'diagnostics', diagnosticsPool) + call mpas_pool_get_subpool(block % structs, 'forcing', forcingPool) call mpas_pool_get_subpool(block % structs, 'provis_state', provisStatePool) @@ -260,6 +263,9 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ call mpas_pool_get_array(meshPool, 'subgridSshCellTableRange', & subgridSshCellTableRange) + call mpas_pool_get_array(forcingPool, 'forcingTimeIncrement', forcingTimeIncrement) + + forcingTimeIncrement = 0.0_RKIND ! Lower k-loop limit of 1 rather than minLevel* needed in *New = *Cur ! assignments below are needed to maintain bit-for-bit results @@ -343,10 +349,10 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ ! The coefficients of k_j are b_j = (1/6, 1/3, 1/3, 1/6) and are ! initialized here as delta t * b_j: - rk_weights(1) = dt/6. - rk_weights(2) = dt/3. - rk_weights(3) = dt/3. - rk_weights(4) = dt/6. + rk_weights(1) = dt/6.0_RKIND + rk_weights(2) = dt/3.0_RKIND + rk_weights(3) = dt/3.0_RKIND + rk_weights(4) = dt/6.0_RKIND ! The a_j coefficients of h in the computation of k_j are typically written (0, 1/2, 1/2, 1). ! However, in the algorithm below we pre-compute the state for the tendency one iteration early. @@ -355,11 +361,19 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ ! That is why the coefficients of h are one index early in the following, i.e. ! a = (1/2, 1/2, 1) - rk_substep_weights(1) = dt/2. - rk_substep_weights(2) = dt/2. + rk_substep_weights(1) = dt/2.0_RKIND + rk_substep_weights(2) = dt/2.0_RKIND rk_substep_weights(3) = dt rk_substep_weights(4) = dt ! a_4 only used for ALE step, otherwise it is skipped. + ! these are time increments to evaluate the tidal forcing at the + ! intermediate time-steps as required by RK4 + + forcingTimeIncrementRK4(1) = 0.0_RKIND + forcingTimeIncrementRK4(2) = dt/2.0_RKIND + forcingTimeIncrementRK4(3) = dt/2.0_RKIND + forcingTimeIncrementRK4(4) = dt + call mpas_timer_start("RK4-main loop") !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -481,8 +495,17 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{ block => domain % blocklist do while (associated(block)) - call ocn_time_integrator_rk4_compute_vel_tends(domain, block, dt, rk_substep_weights(rk_step), domain % dminfo, err ) + call mpas_pool_get_subpool(block % structs, 'forcing', forcingPool) + call mpas_pool_get_array(forcingPool, 'forcingTimeIncrement', forcingTimeIncrement) + forcingTimeIncrement = forcingTimeIncrementRK4(rk_step) + block => block % next + end do + + call ocn_time_varying_forcing_get(domain % streamManager, domain, domain % clock) + block => domain % blocklist + do while (associated(block)) + call ocn_time_integrator_rk4_compute_vel_tends(domain, block, dt, rk_substep_weights(rk_step), domain % dminfo, err ) call ocn_time_integrator_rk4_compute_thick_tends( block, dt, rk_substep_weights(rk_step), err ) block => block % next end do diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F index 8e542d5ac61..77bf1331d3d 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F @@ -380,6 +380,8 @@ subroutine ocn_time_integrator_si(domain, dt)!{{{ real (kind=RKIND), dimension(:,:,:), pointer :: activeTracersNew + real (kind=RKIND), pointer :: forcingTimeIncrement + ! Remap variables real (kind=RKIND), dimension(:,:), pointer :: & layerThicknessLagNew @@ -507,6 +509,11 @@ subroutine ocn_time_integrator_si(domain, dt)!{{{ call mpas_pool_get_array(tracersPool, 'activeTracers', activeTracersNew, 2) + call mpas_pool_get_array(forcingPool, 'forcingTimeIncrement', & + forcingTimeIncrement) + + forcingTimeIncrement = 0.0_RKIND + allocate(bottomDepthEdge(nEdgesAll+1)) if (config_transport_tests_flow_id > 0) then diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F index 2b35a4efae4..5c1069f4e3b 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F @@ -280,6 +280,8 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ real (kind=RKIND), dimension(:,:,:), pointer :: activeTracersNew + real (kind=RKIND), pointer :: forcingTimeIncrement + ! Remap variables real (kind=RKIND), dimension(:,:), pointer :: & layerThicknessLagNew @@ -398,6 +400,11 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ allocate(baroclinicThickness(nEdgesAll+1)) + call mpas_pool_get_array(forcingPool, 'forcingTimeIncrement', & + forcingTimeIncrement) + + forcingTimeIncrement = 0.0_RKIND + if (config_transport_tests_flow_id > 0) then ! This is a transport test. Write advection velocity from prescribed ! flow field. diff --git a/components/mpas-ocean/src/ocean.cmake b/components/mpas-ocean/src/ocean.cmake index 8866a8cea3d..f2d5303fdd7 100644 --- a/components/mpas-ocean/src/ocean.cmake +++ b/components/mpas-ocean/src/ocean.cmake @@ -114,6 +114,7 @@ list(APPEND RAW_SOURCES core_ocean/shared/mpas_ocn_stokes_drift.F core_ocean/shared/mpas_ocn_manufactured_solution.F core_ocean/shared/mpas_ocn_subgrid.F + core_ocean/shared/mpas_ocn_scaled_dismf.F ) set(OCEAN_DRIVER diff --git a/components/mpas-ocean/src/shared/Makefile b/components/mpas-ocean/src/shared/Makefile index d378b68624b..27c3db10fee 100644 --- a/components/mpas-ocean/src/shared/Makefile +++ b/components/mpas-ocean/src/shared/Makefile @@ -157,7 +157,7 @@ mpas_ocn_tracer_short_wave_absorption_variable.o: mpas_ocn_constants.o mpas_ocn_ mpas_ocn_tracer_short_wave_absorption_jerlov.o: mpas_ocn_constants.o mpas_ocn_config.o -mpas_ocn_vmix.o: mpas_ocn_vmix_cvmix.o mpas_ocn_vmix_coefs_redi.o mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_diagnostics_variables.o mpas_ocn_vmix_gotm.o +mpas_ocn_vmix.o: mpas_ocn_vmix_cvmix.o mpas_ocn_vmix_coefs_redi.o mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_diagnostics_variables.o mpas_ocn_vmix_gotm.o mpas_ocn_diagnostics.o mpas_ocn_vmix_cvmix.o: mpas_ocn_constants.o mpas_ocn_config.o mpas_ocn_diagnostics_variables.o mpas_ocn_mesh.o mpas_ocn_stokes_drift.o diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index 0334667f2f1..691bfe70b58 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -64,7 +64,7 @@ module ocn_diagnostics ocn_fuperp, & ocn_filter_btr_mode_tend_vel, & ocn_reconstruct_eddy_vectors, & - ocn_compute_kpp_input_fields, & + ocn_compute_mixing_input_fields, & ocn_validate_state, & ocn_build_log_filename, & ocn_diagnostics_init @@ -3319,12 +3319,12 @@ end subroutine ocn_filter_btr_mode_tend_vel!}}} !*********************************************************************** ! -! routine ocn_compute_KPP_input_fields +! routine ocn_compute_mixing_input_fields ! !> \brief -!> Compute fields necessary to drive the CVMix KPP module -!> \author Todd Ringler -!> \date 20 August 2013 +!> Compute fields necessary to drive the CVMix KPP and gotm modules +!> \author Todd Ringler, Luke Van Roekel +!> \date 11 July 2024 !> \details !> CVMix/KPP requires the following fields as intent(in): !> surfaceBuoyancyForcing @@ -3333,7 +3333,7 @@ end subroutine ocn_filter_btr_mode_tend_vel!}}} ! !----------------------------------------------------------------------- - subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & + subroutine ocn_compute_mixing_input_fields(statePool, forcingPool, & meshPool, timeLevelIn)!{{{ !----------------------------------------------------------------- @@ -3385,8 +3385,7 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & evapTemperatureFlux, & icebergTemperatureFlux, & seaIceTemperatureFlux, & - surfaceStress, & - surfaceStressMagnitude + sfcStressMag real (kind=RKIND), dimension(:,:), pointer :: & layerThickness, &! layer thickness @@ -3419,7 +3418,7 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & !----------------------------------------------------------------- ! Begin code - call mpas_timer_start('KPP input fields') + call mpas_timer_start('Mixing input fields') if (present(timeLevelIn)) then timeLevel = timeLevelIn @@ -3469,18 +3468,8 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & surfaceThicknessFluxSubglacialRunoff) call mpas_pool_get_array(forcingPool, 'penetrativeTemperatureFlux', & penetrativeTemperatureFlux) - call mpas_pool_get_array(forcingPool, 'surfaceStress', & - surfaceStress) call mpas_pool_get_array(forcingPool, 'surfaceStressMagnitude', & - surfaceStressMagnitude) - call mpas_pool_get_array(forcingPool, 'rainTemperatureFlux', & - rainTemperatureFlux) - call mpas_pool_get_array(forcingPool, 'evapTemperatureFlux', & - evapTemperatureFlux) - call mpas_pool_get_array(forcingPool, 'seaIceTemperatureFlux', & - seaIceTemperatureFlux) - call mpas_pool_get_array(forcingPool, 'icebergTemperatureFlux', & - icebergTemperatureFlux) + sfcStressMag) ! allocate scratch space displaced density computation ncells = nCellsAll @@ -3594,17 +3583,8 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & activeTracersSurfaceFlux(indexTempFlux,iCell) & + penetrativeTemperatureFlux(iCell) & - penetrativeTemperatureFluxOBL(iCell) & - - fracAbsorbed* (rainTemperatureFlux(iCell) + & - evapTemperatureFlux(iCell) + & - seaIceTemperatureFlux(iCell) + & - icebergTemperatureFlux(iCell)) & - - fracAbsorbedRunoff* & - activeTracersSurfaceFluxRunoff(indexTempFlux,iCell) - if (trim(config_subglacial_runoff_mode) == 'data') then - nonLocalSurfaceTracerFlux(indexTempFlux, iCell) = nonLocalSurfaceTracerFlux(indexTempFlux, iCell) & - - fracAbsorbedSubglacialRunoff* & - activeTracersSurfaceFluxSubglacialRunoff(indexTempFlux,iCell) - end if + - fracAbsorbed*surfaceThicknessFlux(iCell) * & + activeTracers(indexTempFlux,kmin,iCell) nonLocalSurfaceTracerFlux(indexSaltFlux,iCell) = & activeTracersSurfaceFlux(indexSaltFlux,iCell) & @@ -3630,21 +3610,8 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & surfaceBuoyancyForcing(iCell) = surfaceBuoyancyForcing(iCell)* & gravity - ! compute magnitude of surface stress - sumSurfaceStressSquared = 0.0_RKIND - do i = 1, nEdgesOnCell(iCell) - iEdge = edgesOnCell(i, iCell) - sumSurfaceStressSquared = sumSurfaceStressSquared & - + edgeAreaFractionOfCell(i,iCell)* & - surfaceStress(iEdge)**2 - enddo - - ! NOTE that the factor of 2 is from averaging dot products - ! to cell centers on a C-grid - surfaceStressMagnitude(iCell) = & - sqrt(2.0_RKIND * sumSurfaceStressSquared) surfaceFrictionVelocity(iCell) = & - sqrt(surfaceStressMagnitude(iCell) / rho_sw) + sqrt(sfcStressMag(iCell) / rho_sw) enddo !$omp end do @@ -3655,11 +3622,11 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & salineContractionCoeff, & densitySurfaceDisplaced) - call mpas_timer_stop('KPP input fields') + call mpas_timer_stop('Mixing input fields') !------------------------------------------------------------------- - end subroutine ocn_compute_KPP_input_fields!}}} + end subroutine ocn_compute_mixing_input_fields!}}} !*********************************************************************** ! diff --git a/components/mpas-ocean/src/shared/mpas_ocn_manufactured_solution.F b/components/mpas-ocean/src/shared/mpas_ocn_manufactured_solution.F index 2631ba1737b..6dd1e1d6e31 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_manufactured_solution.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_manufactured_solution.F @@ -58,6 +58,7 @@ module ocn_manufactured_solution real (kind=RKIND) :: kx, ky real (kind=RKIND) :: ang_freq real (kind=RKIND) :: eta0 + real (kind=RKIND) :: viscDel2, viscDel4 real (kind=RKIND) :: H0 !*********************************************************************** @@ -76,7 +77,13 @@ module ocn_manufactured_solution ! !----------------------------------------------------------------------- - subroutine ocn_manufactured_solution_tend_thick(tend, err)!{{{ + subroutine ocn_manufactured_solution_tend_thick(forcingPool, tend, err)!{{{ + + !----------------------------------------------------------------- + ! input variables + !----------------------------------------------------------------- + + type (mpas_pool_type), intent(in) :: forcingPool !----------------------------------------------------------------- ! input/output variables @@ -97,6 +104,7 @@ subroutine ocn_manufactured_solution_tend_thick(tend, err)!{{{ integer :: iCell, kmin, kmax, k real (kind=RKIND) :: phase, time + real (kind=RKIND), pointer :: forcingTimeIncrement ! End preamble !------------- @@ -104,7 +112,9 @@ subroutine ocn_manufactured_solution_tend_thick(tend, err)!{{{ if (.not. config_use_manufactured_solution) return - time = daysSinceStartOfSim*86400.0_RKIND + call mpas_pool_get_array(forcingPool, 'forcingTimeIncrement', forcingTimeIncrement) + + time = daysSinceStartOfSim*86400.0_RKIND + forcingTimeIncrement do iCell = 1,nCellsOwned @@ -137,7 +147,13 @@ end subroutine ocn_manufactured_solution_tend_thick!}}} !> This routine computes the velocity tendency for the manufactured solution ! !----------------------------------------------------------------------- - subroutine ocn_manufactured_solution_tend_vel(tend, err)!{{{ + subroutine ocn_manufactured_solution_tend_vel(forcingPool, tend, err)!{{{ + + !----------------------------------------------------------------- + ! input variables + !----------------------------------------------------------------- + + type (mpas_pool_type), intent(in) :: forcingPool !----------------------------------------------------------------- ! input/output variables @@ -158,6 +174,7 @@ subroutine ocn_manufactured_solution_tend_vel(tend, err)!{{{ integer :: iEdge, kmin, kmax, k real (kind=RKIND) :: phase, u, v, time + real (kind=RKIND), pointer :: forcingTimeIncrement ! End preamble !----------------------------------------------------------------- @@ -165,7 +182,9 @@ subroutine ocn_manufactured_solution_tend_vel(tend, err)!{{{ if (.not. config_use_manufactured_solution) return - time = daysSinceStartOfSim*86400.0_RKIND + call mpas_pool_get_array(forcingPool, 'forcingTimeIncrement', forcingTimeIncrement) + + time = daysSinceStartOfSim*86400.0_RKIND + forcingTimeIncrement do iEdge = 1, nEdgesOwned @@ -183,6 +202,17 @@ subroutine ocn_manufactured_solution_tend_vel(tend, err)!{{{ + ang_freq*sin(phase) & - 0.5_RKIND*eta0*(kx + ky)*sin(2.0_RKIND*(phase))) + if (.not. config_disable_vel_hmix) then + if (config_use_mom_del2) then + u = u + viscDel2 * eta0 * (kx**2 + ky**2) * cos(phase) + v = v + viscDel2 * eta0 * (kx**2 + ky**2) * cos(phase) + endif + if (config_use_mom_del4) then + u = u - viscDel4 * eta0 * ((kx**4 + ky**4 + kx**2 * ky**2) * cos(phase)) + v = v - viscDel4 * eta0 * ((kx**4 + ky**4 + kx**2 * ky**2) * cos(phase)) + endif + endif + tend(k,iEdge) = tend(k,iEdge) + u*cos(angleEdge(iEdge)) + v*sin(angleEdge(iEdge)) enddo @@ -217,6 +247,9 @@ subroutine ocn_manufactured_solution_init(domain, err)!{{{ if (.not. config_use_manufactured_solution) return + viscDel2 = config_mom_del2 + viscDel4 = config_mom_del4 + kx = 2.0_RKIND*pi / config_manufactured_solution_wavelength_x ky = 2.0_RKIND*pi / config_manufactured_solution_wavelength_y diff --git a/components/mpas-ocean/src/shared/mpas_ocn_scaled_dismf.F b/components/mpas-ocean/src/shared/mpas_ocn_scaled_dismf.F new file mode 100644 index 00000000000..bfdb39e49ad --- /dev/null +++ b/components/mpas-ocean/src/shared/mpas_ocn_scaled_dismf.F @@ -0,0 +1,309 @@ +! Copyright (c) 2013, Los Alamos National Security, LLC (LANS) +! and the University Corporation for Atmospheric Research (UCAR). +! +! Unless noted otherwise source code is licensed under the BSD license. +! Additional copyright and license information can be found in the LICENSE file +! distributed with this code, or at http://mpas-dev.github.io/license.html +! +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! ocn_scaled_dismf +!> \brief MPAS ocean scale data ice-shelf melt fluxes +!> \author Xylar Asay-Davis +!> \date July 2024 +!> \details +!> This module contains routines for scaling data ice-shelf melt fluxes by +!> the running mean of the removed ice runoff +!> Design document located at: +!> https://acme-climate.atlassian.net/wiki/spaces/PSC/pages/4210098268/Design+Document+Data+iceberg+and+ice-shelf+melt+flux+patterns+for+E3SM+runs +! +!----------------------------------------------------------------------- + +module ocn_scaled_dismf + + use mpas_kind_types + use mpas_derived_types + use mpas_global_sum_mod + use mpas_timekeeping + use mpas_timer + + use ocn_config + use ocn_mesh + + use shr_const_mod + + implicit none + private + save + + !-------------------------------------------------------------------- + ! + ! Public parameters + ! + !-------------------------------------------------------------------- + + !-------------------------------------------------------------------- + ! + ! Public member functions + ! + !-------------------------------------------------------------------- + + public :: ocn_init_scaled_dismf, & + ocn_update_scaled_dismf + + !-------------------------------------------------------------------- + ! + ! Private module variables + ! + !-------------------------------------------------------------------- + + logical :: scaledDISMFOn + character (len=*), parameter :: alarmID = 'scaledDISMFUpdateAlarm' + +!*********************************************************************** + +contains + + +!*********************************************************************** +! +! routine ocn_init_scaled_dismf +! +!> \brief Initialize scaling of data ice-shelf melt fluxes +!> \author Xylar Asay-Davis +!> \date July 2024 +!> \details +!> Set alarms needed to compute daily and running means of removed ice runoff +! +!----------------------------------------------------------------------- + + subroutine ocn_init_scaled_dismf(domain)!{{{ + + type (domain_type), intent(inout) :: domain + + ! Alarm variables + type (MPAS_Time_Type) :: alarmTime + type (MPAS_TimeInterval_type) :: alarmTimeStep + + integer :: err_tmp + + if (config_scale_dismf_by_removed_ice_runoff) then + scaledDISMFOn = .true. + else + scaledDISMFOn = .false. + return + endif + + if (.not. config_remove_ais_ice_runoff) then + call mpas_log_write('config_scale_dismf_by_removed_ice_runoff = .true. requires config_remove_ais_ice_runoff = .true.', & + MPAS_LOG_CRIT) + endif + + if (trim(config_land_ice_flux_mode) /= 'data') then + call mpas_log_write('config_scale_dismf_by_removed_ice_runoff = .true. requires config_land_ice_flux_mode = "data"', & + MPAS_LOG_CRIT) + endif + + ! Setup Alarm for updating of scaled DISMF + alarmTime = mpas_get_clock_time(domain % clock, & + MPAS_START_TIME, & + ierr=err_tmp) + call mpas_set_timeInterval(alarmTimeStep, & + timeString='0000-00-01_00:00:00', & + ierr=err_tmp) + call mpas_add_clock_alarm(domain % clock, alarmID, alarmTime + alarmTimeStep, & + alarmTimeInterval=alarmTimeStep, ierr=err_tmp) + + end subroutine ocn_init_scaled_dismf!}}} + + +!*********************************************************************** +! +! routine ocn_update_scaled_dismf +! +!> \brief Update scaled data ice-shelf melt fluxes +!> \author Xylar Asay-Davis +!> \date August 2024 +!> \details +!> Accumulate daily mean of removed runoff. If we are at the end of a day, +!> update the running mean of removed ice runoff, and use it to scale +!> the ice shelf melt flux based on the pattern from a data file +! +!----------------------------------------------------------------------- + + subroutine ocn_update_scaled_dismf(domain)!{{{ + + type (domain_type), intent(inout) :: domain + + integer :: err_tmp + + if (.not. scaledDISMFOn) then + return + end if + + ! since the current clock time is for the end of the accumulation + ! intereval, first, accumulate the daily mean + call accumulate_mean_removed_ice_runoff(domain) + + ! then, compute update the history and running mean if we are at the + ! end of a day + if(mpas_is_alarm_ringing(domain % clock, alarmID, ierr=err_tmp)) then +#ifdef MPAS_DEBUG + call mpas_log_write(' Computing Scaled DISMF') +#endif + call update_scaled_dismf(domain) + call mpas_reset_clock_alarm(domain % clock, alarmID, ierr=err_tmp) + endif + + end subroutine ocn_update_scaled_dismf!}}} + + +!*********************************************************************** +! +! routine accumulate_mean_removed_ice_runoff +! +!> \brief Accumulate mean removed ice runoff +!> \author Xylar Asay-Davis +!> \date August 2024 +!> \details +!> Accumulate current removed ice runoff into the daily mean value +! +!----------------------------------------------------------------------- + + subroutine accumulate_mean_removed_ice_runoff(domain)!{{{ + + type (domain_type), intent(inout) :: domain + + type (block_type), pointer :: block_ptr + type (mpas_pool_type), pointer :: forcingPool + + real (kind=RKIND), dimension(:), pointer :: removedIceRunoffFlux + real (kind=RKIND), pointer :: avgRemovedIceRunoff + integer, pointer :: nCellsSolve, nAccumulated + + real (kind=RKIND) :: totalFlux + real (kind=RKIND), dimension(:), allocatable :: localArrayForReproSum + + integer :: iCell + + block_ptr => domain % blocklist + + call mpas_pool_get_subpool(block_ptr % structs, 'forcing', forcingPool) + + ! independent of blocks + call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoff', avgRemovedIceRunoff) + call mpas_pool_get_dimension(forcingPool, 'nCellsSolve', nCellsSolve) + call mpas_pool_get_array(forcingPool, 'nAccumulatedRemovedIceRunoff', nAccumulated) + + do while(associated(block_ptr)) + call mpas_pool_get_subpool(block_ptr % structs, 'forcing', forcingPool) + + call mpas_pool_get_array(forcingPool, 'removedIceRunoffFlux', removedIceRunoffFlux) + call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoff', avgRemovedIceRunoff) + + call mpas_pool_get_dimension(forcingPool, 'nCellsSolve', nCellsSolve) + + call mpas_pool_get_array(forcingPool, 'nAccumulatedRemovedIceRunoff', nAccumulated) + + + allocate(localArrayForReproSum(nCellsSolve)) + localArrayForReproSum(:) = 0.0_RKIND + + !$omp parallel + !$omp do schedule(runtime) + do iCell=1,nCellsSolve + localArrayForReproSum(iCell) = removedIceRunoffFlux(iCell) * areaCell(iCell) + enddo + !$omp end do + !$omp end parallel + + block_ptr => block_ptr % next + end do ! block_ptr + + totalFlux = mpas_global_sum(localArrayForReproSum, domain % dminfo % comm) + + deallocate (localArrayForReproSum) + + avgRemovedIceRunoff = (avgRemovedIceRunoff * nAccumulated + totalFlux) & + / ( nAccumulated + 1 ) + + nAccumulated = nAccumulated + 1 + + end subroutine accumulate_mean_removed_ice_runoff!}}} + + + +!*********************************************************************** +! +! routine update_scaled_dismf +! +!> \brief Update scaled data ice-shelf melt fluxes +!> \author Xylar Asay-Davis +!> \date August 2024 +!> \details +!> Update the running mean of removed ice runoff, and use it to scale +!> the ice shelf melt flux based on the pattern from a data file +! +!----------------------------------------------------------------------- + + subroutine update_scaled_dismf(domain)!{{{ + + type (domain_type), intent(inout) :: domain + + type (block_type), pointer :: block_ptr + type (mpas_pool_type), pointer :: forcingPool + + real (kind=RKIND), pointer :: avgRemovedIceRunoff, runningMeanRemovedIceRunoff + real (kind=RKIND), dimension(:), pointer :: totalRemovedIceRunoffHistory + integer, pointer :: nValidHistory, nAccumulated + + real (kind=RKIND), dimension(:), allocatable :: tmpHistory + + real (kind=RKIND) :: previousTotal, timeInterval + integer :: nHistory + + block_ptr => domain % blocklist + call mpas_pool_get_subpool(block_ptr % structs, 'forcing', forcingPool) + + call mpas_pool_get_array(forcingPool, 'nAccumulatedRemovedIceRunoff', nAccumulated) + call mpas_pool_get_array(forcingPool, 'nValidTotalRemovedIceRunoffHistory', nValidHistory) + call mpas_pool_get_array(forcingPool, 'avgRemovedIceRunoff', avgRemovedIceRunoff) + call mpas_pool_get_array(forcingPool, 'totalRemovedIceRunoffHistory', totalRemovedIceRunoffHistory) + call mpas_pool_get_array(forcingPool, 'runningMeanRemovedIceRunoff', runningMeanRemovedIceRunoff) + + nHistory = config_ais_ice_runoff_history_days + + previousTotal = totalRemovedIceRunoffHistory(nValidHistory) + + if (nValidHistory == 0) then + ! we keep zero as the first entry in totalRemovedIceRunoffHistory + nValidHistory = 1 + end if + + if (nValidHistory == nHistory) then + ! we need to shift the history, since it's full + allocate(tmpHistory(nHistory)) + tmpHistory(:) = totalRemovedIceRunoffHistory(:) + totalRemovedIceRunoffHistory(1:nHistory - 1) = tmpHistory(2:nHistory) + else + ! the history isn't full yet, so we just add to the end + nValidHistory = nValidHistory + 1 + end if + + ! add the new total, the previous total plus the new daily average + totalRemovedIceRunoffHistory(nValidHistory) = previousTotal + SHR_CONST_CDAY * avgRemovedIceRunoff + + timeInterval = SHR_CONST_CDAY * (nValidHistory - 1) + ! the running mean is the difference between the newest and oldest + ! totals divided by the time between them + runningMeanRemovedIceRunoff = & + (totalRemovedIceRunoffHistory(nValidHistory) - totalRemovedIceRunoffHistory(1)) & + / timeInterval + + ! reset daily averaging of the removed runoff + nAccumulated = 0 + avgRemovedIceRunoff = 0.0_RKIND + + end subroutine update_scaled_dismf!}}} + +end module ocn_scaled_dismf \ No newline at end of file diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F index fe489029a13..db5df3dfbe0 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F @@ -278,6 +278,12 @@ subroutine ocn_surface_land_ice_fluxes_thick(forcingPool, surfaceThicknessFlux, real (kind=RKIND), dimension(:), pointer :: landIceFreshwaterFlux, & dataLandIceFreshwaterFlux + real (kind=RKIND), pointer :: runningMeanRemovedIceRunoff, & + areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux + + + real (kind=RKIND) :: scaling + err = 0 if (.not.landIceFluxesOn) return @@ -287,6 +293,16 @@ subroutine ocn_surface_land_ice_fluxes_thick(forcingPool, surfaceThicknessFlux, call mpas_pool_get_array(forcingPool, 'landIceFreshwaterFlux', landIceFreshwaterFlux) if ( landIceDataOn ) then + + if (config_scale_dismf_by_removed_ice_runoff) then + call mpas_pool_get_array(forcingPool, 'runningMeanRemovedIceRunoff', runningMeanRemovedIceRunoff) + call mpas_pool_get_array(forcingPool, 'areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux', & + areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux) + scaling = runningMeanRemovedIceRunoff / areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux + else + scaling = 1.0_RKIND + end if + call mpas_pool_get_array(forcingPool, 'dataLandIceFreshwaterFlux', dataLandIceFreshwaterFlux) #ifdef MPAS_OPENACC !$acc enter data copyin(landIceFreshwaterFlux) @@ -297,7 +313,7 @@ subroutine ocn_surface_land_ice_fluxes_thick(forcingPool, surfaceThicknessFlux, !$omp do schedule(runtime) #endif do iCell = 1, nCellsAll - landIceFreshwaterFlux(iCell) = dataLandIceFreshwaterFlux(iCell) + landIceFreshwaterFlux(iCell) = scaling * dataLandIceFreshwaterFlux(iCell) end do #ifndef MPAS_OPENACC !$omp end do @@ -378,6 +394,12 @@ subroutine ocn_surface_land_ice_fluxes_active_tracers(meshPool, forcingPool, tra real (kind=RKIND), dimension(:), pointer :: landIceHeatFlux, & dataLandIceHeatFlux + real (kind=RKIND), pointer :: runningMeanRemovedIceRunoff, & + areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux + + + real (kind=RKIND) :: scaling + err = 0 if (.not.landIceFluxesOn) return @@ -387,11 +409,21 @@ subroutine ocn_surface_land_ice_fluxes_active_tracers(meshPool, forcingPool, tra call mpas_pool_get_array(forcingPool, 'landIceHeatFlux', landIceHeatFlux) if ( landIceDataOn ) then + + if (config_scale_dismf_by_removed_ice_runoff) then + call mpas_pool_get_array(forcingPool, 'runningMeanRemovedIceRunoff', runningMeanRemovedIceRunoff) + call mpas_pool_get_array(forcingPool, 'areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux', & + areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux) + scaling = runningMeanRemovedIceRunoff / areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux + else + scaling = 1.0_RKIND + end if + call mpas_pool_get_array(forcingPool, 'dataLandIceHeatFlux', dataLandIceHeatFlux) !$omp parallel !$omp do schedule(runtime) do iCell = 1, nCells - landIceHeatFlux(iCell) = dataLandIceHeatFlux(iCell) + landIceHeatFlux(iCell) = scaling * dataLandIceHeatFlux(iCell) end do !$omp end do !$omp end parallel diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index 182fca7dbb9..0d049d1eb42 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -264,7 +264,8 @@ subroutine ocn_tend_thick(tendPool, forcingPool)!{{{ tendThick, err) ! Compute thickness tendency to manufactured solution - call ocn_manufactured_solution_tend_thick(tendThick, err) + call ocn_manufactured_solution_tend_thick(forcingPool, & + tendThick, err) #ifdef MPAS_OPENACC !$acc exit data copyout(tendThick, surfaceThicknessFlux, surfaceThicknessFluxRunoff, & @@ -493,7 +494,8 @@ subroutine ocn_tend_vel(domain, tendPool, statePool, forcingPool, & layerThickEdgeMean, tendVel, err) ! Compute tendency term for manufactured solution - call ocn_manufactured_solution_tend_vel(tendVel, err) + call ocn_manufactured_solution_tend_vel(forcingPool, & + tendVel, err) ! vertical mixing treated implicitly in a later routine ! adjust total velocity tendency based on wetting/drying @@ -1317,12 +1319,9 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & ! ! Compute tracer tendency due to non-local flux computed in KPP ! - if (config_use_cvmix_kpp .or. config_use_gotm) then - call ocn_compute_KPP_input_fields(statePool, forcingPool,& - meshPool, timeLevel) - - if (.not. config_cvmix_kpp_nonlocal_with_implicit_mix) then + if (config_use_cvmix_kpp) then call mpas_timer_start("non-local flux from KPP") + if (.not. config_cvmix_kpp_nonlocal_with_implicit_mix) then if (computeBudgets) then !$omp parallel !$omp do schedule(runtime) private(k,n) @@ -1365,8 +1364,8 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, & !$omp end do !$omp end parallel endif ! compute budgets - call mpas_timer_stop("non-local flux from KPP") end if ! not non-local with implicit mix + call mpas_timer_stop("non-local flux from KPP") end if ! KPP ! Compute tracer tendency due to production/destruction of diff --git a/components/mpas-ocean/src/shared/mpas_ocn_time_average_coupled.F b/components/mpas-ocean/src/shared/mpas_ocn_time_average_coupled.F index c8ce0b8dc15..0b74b7c56b8 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_time_average_coupled.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_time_average_coupled.F @@ -25,6 +25,8 @@ module ocn_time_average_coupled use mpas_pool_routines use ocn_constants use ocn_config + use ocn_equation_of_state + use ocn_mesh use ocn_tracer_ecosys use ocn_tracer_DMS use ocn_tracer_MacroMolecules @@ -58,6 +60,8 @@ subroutine ocn_time_average_coupled_init(forcingPool)!{{{ avgRemovedRiverRunoffFlux, avgRemovedIceRunoffFlux, & avgLandIceHeatFlux, avgRemovedIceRunoffHeatFlux + real (kind=RKIND), dimension(:), pointer :: avgThermalForcingAtCritDepth + integer :: iCell integer, pointer :: nAccumulatedCoupled, nCells @@ -160,6 +164,18 @@ subroutine ocn_time_average_coupled_init(forcingPool)!{{{ !$omp end parallel end if + if (trim(config_glc_thermal_forcing_coupling_mode) == '2d') then + call mpas_pool_get_array(forcingPool, 'avgThermalForcingAtCritDepth', avgThermalForcingAtCritDepth) + + !$omp parallel + !$omp do schedule(runtime) + do iCell = 1, nCells + avgThermalForcingAtCritDepth(iCell) = 0.0_RKIND + end do + !$omp end do + !$omp end parallel + endif + ! set up BGC coupling fields if necessary if (config_use_ecosysTracers) then @@ -257,7 +273,7 @@ subroutine ocn_time_average_coupled_accumulate(statePool, forcingPool, timeLevel real (kind=RKIND), dimension(:,:), pointer :: avgSSHGradient integer :: iCell integer, pointer :: index_temperaturePtr, index_SSHzonalPtr, & - index_SSHmeridionalPtr, nAccumulatedCoupled, nCells + index_SSHmeridionalPtr, nAccumulatedCoupled, nCells, nVertLevels integer :: index_temperature, index_SSHzonal, index_SSHmeridional real (kind=RKIND), dimension(:,:), pointer :: & avgLandIceBoundaryLayerTracers, avgLandIceTracerTransferVelocities @@ -267,10 +283,16 @@ subroutine ocn_time_average_coupled_accumulate(statePool, forcingPool, timeLevel landIceHeatFlux, avgLandIceHeatFlux, & removedRiverRunoffFlux, avgRemovedRiverRunoffFlux, & removedIceRunoffFlux, avgRemovedIceRunoffFlux, & - avgRemovedIceRunoffHeatFlux + avgRemovedIceRunoffHeatFlux, & + avgThermalForcingAtCritDepth type (mpas_pool_type), pointer :: tracersPool + real (kind=RKIND), dimension(:,:,:), pointer :: activeTracers + integer, pointer :: indexTemperature, indexSalinity + integer :: iLevelCritDepth, iLevel + real (kind=RKIND) :: freezingTemp + real (kind=RKIND), dimension(:,:,:), pointer :: & ecosysTracers, & DMSTracers, & @@ -306,6 +328,7 @@ subroutine ocn_time_average_coupled_accumulate(statePool, forcingPool, timeLevel call mpas_pool_get_array(forcingPool, 'avgTotalFreshWaterTemperatureFlux', avgTotalFreshWaterTemperatureFlux) call mpas_pool_get_dimension(forcingPool, 'nCells', nCells) + call mpas_pool_get_dimension(forcingPool, 'nVertLevels', nVertLevels) call mpas_pool_get_dimension(forcingPool, & 'index_avgTemperatureSurfaceValue', & index_temperaturePtr) @@ -349,6 +372,7 @@ subroutine ocn_time_average_coupled_accumulate(statePool, forcingPool, timeLevel call mpas_pool_get_array(forcingPool, 'avgLandIceTracerTransferVelocities', avgLandIceTracerTransferVelocities) call mpas_pool_get_array(forcingPool, 'avgEffectiveDensityInLandIce', avgEffectiveDensityInLandIce) + !$omp parallel !$omp do schedule(runtime) do iCell = 1, nCells @@ -416,6 +440,41 @@ subroutine ocn_time_average_coupled_accumulate(statePool, forcingPool, timeLevel end if + if (trim(config_glc_thermal_forcing_coupling_mode) == '2d') then + call mpas_pool_get_array(forcingPool, 'avgThermalForcingAtCritDepth', avgThermalForcingAtCritDepth) + call mpas_pool_get_subpool(statePool, 'tracers', tracersPool) + call mpas_pool_get_array(tracersPool, 'activeTracers', activeTracers, 2) + call mpas_pool_get_dimension(tracersPool, 'index_temperature', indexTemperature) + call mpas_pool_get_dimension(tracersPool, 'index_salinity', indexSalinity) + + + ! find vertical level that is just above the critical depth reference level + ! this does not account for depression due to ice shelf cavities or sea ice + iLevelCritDepth = nVertLevels ! default to deepest layer if we don't find the desired depth + do iLevel = 1, nVertLevels + if(refBottomDepth(iLevel) > config_2d_thermal_forcing_depth) then + iLevelCritDepth = iLevel + exit + end if + end do + !$omp parallel + !$omp do schedule(runtime) + ! calculate thermal forcing at identified level for each cell + do iCell = 1, nCells + ! ignore cells that are too shallow + if (iLevelCritDepth <= maxLevelCell(iCell)) then + ! this uses the level shallower than the reference level. could interpolate instead + ! note: assuming no LandIce cavity, but we may want to support that + freezingTemp = ocn_freezing_temperature(salinity=activeTracers(indexSalinity, iLevelCritDepth, iCell), & + pressure=pressure(iLevelCritDepth, iCell), inLandIceCavity=.false.) + avgThermalForcingAtCritDepth(iCell) = ( avgThermalForcingAtCritDepth(iCell) * nAccumulatedCoupled & + + activeTracers(indexTemperature, iLevelCritDepth, iCell) - freezingTemp ) / ( nAccumulatedCoupled + 1) + end if + end do + !$omp end do + !$omp end parallel + endif + ! accumulate BGC coupling fields if necessary if (config_use_ecosysTracers) then diff --git a/components/mpas-ocean/src/shared/mpas_ocn_time_varying_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_time_varying_forcing.F index 8e64618a27a..2afa3064050 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_time_varying_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_time_varying_forcing.F @@ -258,13 +258,21 @@ subroutine atmospheric_forcing(streamManager, domain, simulationClock)!{{{ character(len=StrKIND) :: timeStamp - real (kind=RKIND) :: dtSim + real (kind=RKIND) :: dtSim, dtSimReverse + + real (kind=RKIND), pointer :: forcingTimeIncrement + + type (mpas_pool_type), pointer :: forcingPool integer :: err + call mpas_pool_get_subpool(domain % blocklist % structs, 'forcing', forcingPool) + call mpas_pool_get_array(forcingPool, 'forcingTimeIncrement', forcingTimeIncrement) + ! convert config_dt to real call mpas_set_timeInterval(timeStepESMF, timeString=config_dt,ierr=err) - call mpas_get_timeInterval(timeStepESMF, dt=dtSim) + dtSim = forcingTimeIncrement + dtSimReverse = -dtSim ! use the forcing layer to get data call MPAS_forcing_get_forcing(& @@ -277,8 +285,18 @@ subroutine atmospheric_forcing(streamManager, domain, simulationClock)!{{{ forcingGroupHead, & ! forcingGroupHead "ocn_atmospheric_forcing", & ! forcingGroupName currentForcingTime) ! forcingTime - !call mpas_get_time(curr_time=currentForcingTime, dateTimeString=timeStamp, ierr=err) - !call mpas_log_write('Forcing time ' // trim(timeStamp)) + call mpas_get_time(curr_time=currentForcingTime, dateTimeString=timeStamp, ierr=err) + !call mpas_log_write('Forcing time for atmospheric forcing' // trim(timeStamp)) + + call mpas_advance_forcing_clock(forcingGroupHead, dtSimReverse) + + call MPAS_forcing_get_forcing_time(& + forcingGroupHead, & ! forcingGroupHead + "ocn_atmospheric_forcing", & ! forcingGroupName + currentForcingTime) ! forcingTime + + call mpas_get_time(curr_time=currentForcingTime, dateTimeString=timeStamp, ierr=err) + !call mpas_log_write('Forcing time reversed for atmospheric forcing' // trim(timeStamp)) ! perform post forcing block => domain % blocklist @@ -332,7 +350,10 @@ subroutine post_atmospheric_forcing(block)!{{{ windStressCoefficient, & rhoAir, & ramp, & - windStressCoefficientLimit + windStressCoefficientLimit, & + t + + real(kind=RKIND), pointer :: forcingTimeIncrement integer, pointer :: & nCells @@ -354,12 +375,14 @@ subroutine post_atmospheric_forcing(block)!{{{ call MPAS_pool_get_array(timeVaryingForcingPool, "atmosPressure", atmosPressure) call MPAS_pool_get_array(forcingPool, "atmosphericPressure", atmosphericPressure) + call mpas_pool_get_array(forcingPool, 'forcingTimeIncrement', forcingTimeIncrement) rhoAir = 1.225_RKIND windStressCoefficientLimit = 0.0035_RKIND if (daysSinceStartOfSim >= config_time_varying_atmospheric_forcing_ramp_delay) then - ramp = tanh((2.0_RKIND*(daysSinceStartOfSim-config_time_varying_atmospheric_forcing_ramp_delay)) & + t = (daysSinceStartOfSim*86400_RKIND + forcingTimeIncrement)/86400.0_RKIND + ramp = tanh((2.0_RKIND*(t-config_time_varying_atmospheric_forcing_ramp_delay)) & /config_time_varying_atmospheric_forcing_ramp) else ramp = 0.0_RKIND @@ -483,11 +506,13 @@ subroutine land_ice_forcing(streamManager, domain, simulationClock)!{{{ type (block_type), pointer :: block -! character(len=StrKIND), pointer :: config_dt + character(len=StrKIND) :: timeStamp type (MPAS_timeInterval_type) :: timeStepESMF - real (kind=RKIND) :: dtSim + real (kind=RKIND) :: dtSim, dtSimReverse + + real (kind=RKIND), pointer :: forcingTimeIncrement integer :: err @@ -512,14 +537,41 @@ subroutine land_ice_forcing(streamManager, domain, simulationClock)!{{{ integer :: & iCell + call mpas_pool_get_subpool(domain % blocklist % structs, 'forcing', forcingPool) + call mpas_pool_get_array(forcingPool, 'forcingTimeIncrement', forcingTimeIncrement) + ! convert config_dt to real call mpas_set_timeInterval(timeStepESMF, timeString=config_dt,ierr=err) call mpas_get_timeInterval(timeStepESMF, dt=dtSim) + dtSim = forcingTimeIncrement + dtSimReverse = -dtSim + ! use the forcing layer to get data - call mpas_forcing_get_forcing(forcingGroupHead, "ocn_land_ice_forcing", streamManager, dtSim) + call MPAS_forcing_get_forcing(& + forcingGroupHead, & ! forcingGroupHead + "ocn_land_ice_forcing", & ! forcingGroupName + streamManager, & ! streamManager + dtSim) ! dt + + call MPAS_forcing_get_forcing_time(& + forcingGroupHead, & ! forcingGroupHead + "ocn_land_ice_forcing", & ! forcingGroupName + currentForcingTime) ! forcingTime + + call mpas_get_time(curr_time=currentForcingTime, dateTimeString=timeStamp, ierr=err) + call mpas_log_write('Forcing time for land ice forcing' // trim(timeStamp)) + + + call mpas_advance_forcing_clock(forcingGroupHead, dtSimReverse) + + call MPAS_forcing_get_forcing_time(& + forcingGroupHead, & ! forcingGroupHead + "ocn_land_ice_forcing", & ! forcingGroupName + currentForcingTime) ! forcingTime - call mpas_forcing_get_forcing_time(forcingGroupHead, "ocn_land_ice_forcing", currentForcingTime) + call mpas_get_time(curr_time=currentForcingTime, dateTimeString=timeStamp, ierr=err) + call mpas_log_write('Forcing time reversed for land ice forcing' // trim(timeStamp)) block => domain % blocklist do while (associated(block)) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F b/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F index daafa816fab..bd3d3298246 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F @@ -80,6 +80,8 @@ module ocn_vel_tidal_potential tidalConstituentAstronomical, &! tidalConstituentNodalPhase ! + real (kind=RKIND), pointer :: forcingTimeIncrement + real (kind=RKIND), dimension(:,:), pointer :: & latitudeFunction @@ -273,8 +275,8 @@ subroutine ocn_compute_tidal_potential_forcing(err)!{{{ err = 0 if (tidalPotentialOff) return - ramp = tanh((2.0_RKIND*daysSinceStartOfSim)/tidalPotRamp) - t = daysSinceStartOfSim*86400.0_RKIND + t = daysSinceStartOfSim*86400.0_RKIND + forcingTimeIncrement + ramp = tanh((2.0_RKIND*t/86400.0_RKIND)/tidalPotRamp) do iCell = 1, nCellsAll tidalPotEta(iCell) = 0.0_RKIND @@ -416,6 +418,9 @@ subroutine ocn_vel_tidal_potential_init(domain,err)!{{{ call mpas_pool_get_array(forcingPool, & 'tidalPotentialLatitudeFunction', & latitudeFunction) + call mpas_pool_get_array(forcingPool, & + 'forcingTimeIncrement', & + forcingTimeIncrement) call mpas_set_time(refTime, & dateTimeString=config_tidal_potential_reference_time) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vmix.F b/components/mpas-ocean/src/shared/mpas_ocn_vmix.F index 68ef9d82a6b..cabdf86ba11 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vmix.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vmix.F @@ -27,6 +27,7 @@ module ocn_vmix use mpas_timer use ocn_mesh + use ocn_diagnostics use mpas_constants use ocn_constants use ocn_config @@ -202,6 +203,11 @@ subroutine ocn_vmix_coefs(meshPool, statePool, forcingPool, scratchPool, err, ti !$omp end parallel #endif + if(config_use_cvmix_kpp .or. config_use_gotm) then + call ocn_compute_mixing_input_fields(statePool, forcingPool,& + meshPool, timeLevel) + end if + #ifdef MPAS_OPENACC !$acc update host(vertViscTopOfEdge, vertDiffTopOfCell) #endif diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F b/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F index c059fd5e9ca..08042ae1294 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F @@ -64,7 +64,7 @@ module ocn_vmix_cvmix type(cvmix_shear_params_type) :: cvmix_shear_params type(cvmix_tidal_params_type) :: cvmix_tidal_params - logical :: cvmixOn, cvmixConvectionOn, cvmixKPPOn + logical :: lnonzero_surf_nonlocal, cvmixOn, cvmixConvectionOn, cvmixKPPOn real (kind=RKIND) :: backgroundVisc, backgroundDiff integer :: cvmixBackgroundChoice ! user choice of cvmix background scheme @@ -1061,19 +1061,22 @@ subroutine ocn_vmix_cvmix_init(domain,err)!{{{ ! initialize KPP boundary layer scheme ! if (config_use_cvmix_kpp) then - if(config_cvmix_kpp_matching.eq."MatchBoth") then - call mpas_log_write( & - "Use of option MatchBoth is discouraged, use SimpleShapes instead", & - MPAS_LOG_WARN) - elseif(.not. config_cvmix_kpp_matching.eq."SimpleShapes") then + if(.not. config_cvmix_kpp_matching.eq."SimpleShapes" .and. & + .not. config_cvmix_kpp_matching.eq."MatchBoth" .and. & + .not. config_cvmix_kpp_matching.eq."ParabolicNonLocal") then call mpas_log_write( & "Unknown value for config_cvmix_kpp_matching., supported values are:" // & - " SimpleShapes or MatchBoth", & + " SimpleShapes or MatchBoth or ParabolicNonLocal", & MPAS_LOG_CRIT) err = 1 return endif + lnonzero_surf_nonlocal = .false. + if(config_cvmix_kpp_matching .eq."ParabolicNonLocal") then + lnonzero_surf_nonlocal = .true. + end if + if (trim(config_cvmix_kpp_langmuir_mixing_opt) .ne. "NONE" .and. & trim(config_cvmix_kpp_langmuir_mixing_opt) .ne. "LWF16" .and. & trim(config_cvmix_kpp_langmuir_mixing_opt) .ne. "RWHGK16") then @@ -1100,13 +1103,15 @@ subroutine ocn_vmix_cvmix_init(domain,err)!{{{ call cvmix_init_kpp ( & ri_crit = config_cvmix_kpp_criticalBulkRichardsonNumber, & interp_type = config_cvmix_kpp_interpolationOMLType, & - interp_type2 = config_cvmix_kpp_interpolationOMLType, & + interp_type2 = 'LMD94', & lEkman = config_cvmix_kpp_EkmanOBL, & lMonOb = config_cvmix_kpp_MonObOBL, & MatchTechnique = config_cvmix_kpp_matching, & surf_layer_ext = config_cvmix_kpp_surface_layer_extent, & langmuir_mixing_str = config_cvmix_kpp_langmuir_mixing_opt, & langmuir_entrainment_str = config_cvmix_kpp_langmuir_entrainment_opt, & + lnoDGat1 = .true., & + lnonzero_surf_nonlocal = lnonzero_surf_nonlocal, & lenhanced_diff = config_cvmix_kpp_use_enhanced_diff) endif diff --git a/components/mpas-seaice/bld/build-namelist b/components/mpas-seaice/bld/build-namelist index 3d7ce2c619f..87831cc94b0 100755 --- a/components/mpas-seaice/bld/build-namelist +++ b/components/mpas-seaice/bld/build-namelist @@ -946,6 +946,7 @@ if ($iceberg_mode eq 'data') { } else { add_default($nl, 'config_use_data_icebergs', 'val'=>"false"); } +add_default($nl, 'config_scale_dib_by_removed_ice_runoff'); add_default($nl, 'config_salt_flux_coupling_type'); add_default($nl, 'config_ice_ocean_drag_coefficient'); diff --git a/components/mpas-seaice/bld/build-namelist-section b/components/mpas-seaice/bld/build-namelist-section index d2866ee4c81..90dd6ce39f2 100644 --- a/components/mpas-seaice/bld/build-namelist-section +++ b/components/mpas-seaice/bld/build-namelist-section @@ -451,6 +451,7 @@ add_default($nl, 'config_ocean_heat_transfer_type'); add_default($nl, 'config_sea_freezing_temperature_type'); add_default($nl, 'config_ocean_surface_type'); add_default($nl, 'config_use_data_icebergs'); +add_default($nl, 'config_scale_dib_by_removed_ice_runoff'); add_default($nl, 'config_salt_flux_coupling_type'); add_default($nl, 'config_ice_ocean_drag_coefficient'); diff --git a/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml b/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml index 69294640ff6..8128c268c6e 100644 --- a/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml +++ b/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml @@ -456,6 +456,7 @@ 'mushy' 'free' false +false 'constant' 0.00536 diff --git a/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml b/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml index ba6ae4be4bb..d19f3f4eaad 100644 --- a/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml +++ b/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml @@ -2686,6 +2686,14 @@ Valid values: true or false Default: Defined in namelist_defaults.xml + +Whether to scale data iceberg fluxes by the running mean of removed ice runoff + +Valid values: true or false +Default: Defined in namelist_defaults.xml + + Type of salt flux to ocean method diff --git a/components/mpas-seaice/cime_config/buildnml b/components/mpas-seaice/cime_config/buildnml index 29b017f6d8b..5df71d69b1a 100755 --- a/components/mpas-seaice/cime_config/buildnml +++ b/components/mpas-seaice/cime_config/buildnml @@ -300,7 +300,7 @@ def buildnml(case, caseroot, compname): grid_prefix = 'mpassi.IcoswISC30E3r5' decomp_date = '20231120' decomp_prefix = 'partitions/mpas-seaice.graph.info.' - data_iceberg_file = 'Iceberg_Climatology_Merino.IcoswISC30E3r5.20231120.nc' + data_iceberg_file = 'Iceberg_Climatology_Merino.IcoswISC30E3r5.20240805.nc' dust_iron_file = 'ecoForcingSurfaceMonthly+WetDryDustSolFrac.IcoswISC30E3r5.20240511.nc' if ice_ic_mode == 'spunup': if iceberg_mode == 'data': @@ -335,7 +335,7 @@ def buildnml(case, caseroot, compname): grid_prefix = 'mpassi.SOwISC12to30E3r3' decomp_date = '20240829' decomp_prefix = 'partitions/mpas-seaice.graph.info.' - data_iceberg_file = 'Iceberg_Climatology_Merino.SOwISC12to30E3r3.20240829.nc' + data_iceberg_file = 'Iceberg_Climatology_Merino.SOwISC12to30E3r3.20241017.nc' if ice_ic_mode == 'spunup': grid_date = '20240829' grid_prefix = 'mpassi.SOwISC12to30E3r3.rstFromG-chrysalis' @@ -668,6 +668,7 @@ def buildnml(case, caseroot, compname): lines.append(' input_interval="none" >') lines.append('') lines.append(' ') + lines.append(' ') lines.append(' ') lines.append('') lines.append('') diff --git a/components/mpas-seaice/driver/ice_comp_mct.F b/components/mpas-seaice/driver/ice_comp_mct.F index 3c579aee77a..f9d79bab96f 100644 --- a/components/mpas-seaice/driver/ice_comp_mct.F +++ b/components/mpas-seaice/driver/ice_comp_mct.F @@ -194,7 +194,9 @@ subroutine ice_init_mct( EClock, cdata_i, x2i_i, i2x_i, NLFilename )!{{{ type (MPAS_TimeInterval_Type) :: alarmTimeStep type (block_type), pointer :: block - type (MPAS_Pool_Type), pointer :: shortwave + type (MPAS_Pool_Type), pointer :: & + shortwave, & + berg_forcing logical :: exists logical :: verbose_taskmap_output ! true then use verbose task-to-node mapping format @@ -241,15 +243,18 @@ subroutine ice_init_mct( EClock, cdata_i, x2i_i, i2x_i, NLFilename )!{{{ integer :: size_list, index_list type(mct_string) :: mctOStr ! character(CXX) :: mct_field, modelStr -#endif +#endif #endif - logical, pointer :: tempLogicalConfig + logical, pointer :: & + tempLogicalConfig, & + config_scale_dib_by_removed_ice_runoff character(len=StrKIND), pointer :: tempCharConfig real (kind=RKIND), pointer :: tempRealConfig real(kind=RKIND), pointer :: & - dayOfNextShortwaveCalculation ! needed for CESM like coupled simulations + dayOfNextShortwaveCalculation, & ! needed for CESM like coupled simulations + runningMeanRemovedIceRunoff ! the area integrated, running mean of removed ice runoff from the ocean interface subroutine xml_stream_parser(xmlname, mgr_p, comm, ierr) bind(c) @@ -505,6 +510,9 @@ end subroutine xml_stream_get_attributes end if + call MPAS_pool_get_config(domain % configs, "config_scale_dib_by_removed_ice_runoff", & + config_scale_dib_by_removed_ice_runoff) + ! Setup MPASSI simulation clock ierr = domain % core % setup_clock(domain % clock, domain % configs) if ( ierr /= 0 ) then @@ -636,6 +644,15 @@ end subroutine xml_stream_get_attributes ! Determine coupling type call seq_infodata_GetData(infodata, cpl_seq_option=cpl_seq_option) + if (config_scale_dib_by_removed_ice_runoff) then + ! independent of space so should be no need to loop over blocks + block => domain % blocklist + call MPAS_pool_get_subpool(block % structs, "berg_forcing", berg_forcing) + call MPAS_pool_get_array(berg_forcing, "runningMeanRemovedIceRunoff", & + runningMeanRemovedIceRunoff) + call seq_infodata_GetData(infodata, rmean_rmv_ice_runoff=runningMeanRemovedIceRunoff ) + end if + ! Determine time of next atmospheric shortwave calculation block => domain % blocklist do while (associated(block)) @@ -730,7 +747,7 @@ end subroutine xml_stream_get_attributes allocate (x2i_im(lsize, nrecv) ) i2x_im = 0._r8 x2i_im = 0._r8 - ! define tags according to the seq_flds_i2x_fields + ! define tags according to the seq_flds_i2x_fields ! also zero them out tagtype = 1 ! dense, double numco = 1 ! one value per cell / entity @@ -763,7 +780,7 @@ end subroutine xml_stream_get_attributes endif #endif - + !----------------------------------------------------------------------- ! @@ -827,10 +844,10 @@ end subroutine xml_stream_get_attributes else if (trim(tempCharConfig) == "column_package") then call seaice_column_coupling_prep(domain) endif ! config_column_physics_type - + call MPAS_pool_get_config(domain % configs, "config_use_floe_size_distribution", tempLogicalConfig) if (tempLogicalConfig) then - call mpas_log_write('FloeSizeDistribution coming online soon. Turn FSD off for now.', MPAS_LOG_CRIT) + call mpas_log_write('FloeSizeDistribution coming online soon. Turn FSD off for now.', MPAS_LOG_CRIT) endif !----------------------------------------------------------------------- ! @@ -854,7 +871,7 @@ end subroutine xml_stream_get_attributes ! ! get intial state from driver ! - call ice_import_mct(x2i_i, errorCode) + call ice_import_mct(x2i_i, errorCode) if (errorCode /= 0) then call mpas_log_write('Error in ice_import_mct', MPAS_LOG_CRIT) endif @@ -862,7 +879,7 @@ end subroutine xml_stream_get_attributes #ifdef HAVE_MOAB #ifdef MOABCOMP - mpicom_moab = mpicom_i ! save it for run method + mpicom_moab = mpicom_i ! save it for run method ! loop over all fields in seq_flds_x2i_fields call mct_list_init(temp_list ,seq_flds_x2i_fields) size_list=mct_list_nitem (temp_list) @@ -879,8 +896,8 @@ end subroutine xml_stream_get_attributes #endif call ice_import_moab(Eclock) -#endif - +#endif + currTime = mpas_get_clock_time(domain % clock, MPAS_NOW, ierr) call mpas_get_time(curr_time=currTime, dateTimeString=timeStamp, ierr=ierr) @@ -1151,7 +1168,8 @@ subroutine ice_run_mct( EClock, cdata_i, x2i_i, i2x_i)!{{{ ! Variable related to MPASSI type (block_type), pointer :: block type (MPAS_Pool_type), pointer :: & - shortwave + shortwave, & + berg_forcing real (kind=RKIND) :: current_wallclock_time type (MPAS_Time_Type) :: currTime @@ -1159,13 +1177,16 @@ subroutine ice_run_mct( EClock, cdata_i, x2i_i, i2x_i)!{{{ type (MPAS_timeInterval_type) :: timeStep integer :: ierr, streamDirection, iam logical :: streamActive, debugOn - logical, pointer :: config_write_output_on_startup + logical, pointer :: & + config_write_output_on_startup, & + config_scale_dib_by_removed_ice_runoff logical, save :: first=.true. character (len=StrKIND), pointer :: & config_restart_timestamp_name, & config_column_physics_type real(kind=RKIND), pointer :: & - dayOfNextShortwaveCalculation ! needed for CESM like coupled simulations + dayOfNextShortwaveCalculation, & ! needed for CESM like coupled simulations + runningMeanRemovedIceRunoff ! the area integrated, running mean of removed ice runoff from the ocean #ifdef MOABCOMP real(r8) :: difference @@ -1173,7 +1194,7 @@ subroutine ice_run_mct( EClock, cdata_i, x2i_i, i2x_i)!{{{ integer :: size_list, index_list, ent_type type(mct_string) :: mctOStr ! character(CXX) :: mct_field, modelStr, tagname -#endif +#endif iam = domain % dminfo % my_proc_id @@ -1187,8 +1208,10 @@ subroutine ice_run_mct( EClock, cdata_i, x2i_i, i2x_i)!{{{ mpas_log_info => domain % logInfo if (debugOn) call mpas_log_write("=== Beginning ice_run_mct ===") - call mpas_pool_get_config(domain % configs, 'config_restart_timestamp_name', config_restart_timestamp_name) + call MPAS_pool_get_config(domain % configs, 'config_restart_timestamp_name', config_restart_timestamp_name) call MPAS_pool_get_config(domain % configs, "config_column_physics_type", config_column_physics_type) + call MPAS_pool_get_config(domain % configs, "config_scale_dib_by_removed_ice_runoff", & + config_scale_dib_by_removed_ice_runoff) ! Setup log information. call shr_file_getLogUnit (shrlogunit) @@ -1225,6 +1248,15 @@ subroutine ice_run_mct( EClock, cdata_i, x2i_i, i2x_i)!{{{ ! Post coupling calls block => domain % blocklist + + if (config_scale_dib_by_removed_ice_runoff) then + ! independent of space so should be no need to loop over blocks + call MPAS_pool_get_subpool(block % structs, "berg_forcing", berg_forcing) + call MPAS_pool_get_array(berg_forcing, "runningMeanRemovedIceRunoff", & + runningMeanRemovedIceRunoff) + call seq_infodata_GetData(infodata, rmean_rmv_ice_runoff=runningMeanRemovedIceRunoff ) + end if + do while (associated(block)) ! Determine time of next atmospheric shortwave calculation @@ -2998,8 +3030,8 @@ subroutine frazil_mass(freezingPotential, frazilMassFlux, seaSurfaceSalinity) qi0new, & vi0new - call MPAS_pool_get_config(domain % configs, "config_thermodynamics_type", config_thermodynamics_type) - call MPAS_pool_get_config(domain % configs, "config_column_physics_type", config_column_physics_type) + call MPAS_pool_get_config(domain % configs, "config_thermodynamics_type", config_thermodynamics_type) + call MPAS_pool_get_config(domain % configs, "config_column_physics_type", config_column_physics_type) if (freezingPotential > 0.0_RKIND) then @@ -3479,15 +3511,15 @@ subroutine ice_import_moab(Eclock)!{{{ ! o dhdx -- ocn surface slope, zonal ! o dhdy -- ocn surface slope, meridional ! o lwdn -- downward lw heat flux -! o rain -- prec: liquid -! o snow -- prec: frozen +! o rain -- prec: liquid +! o snow -- prec: frozen ! o swndr -- sw: nir direct downward ! o swvdr -- sw: vis direct downward ! o swndf -- sw: nir diffuse downward ! o swvdf -- sw: vis diffuse downward ! o swnet -- sw: net ! o q -- ocn frazil heat flux(+) / melt potential(-) -! o frazil -- ocn frazil mass flux +! o frazil -- ocn frazil mass flux ! o bcphidry -- Black Carbon hydrophilic dry deposition flux ! o bcphodry -- Black Carbon hydrophobic dry deposition flux ! o bcphiwet -- Black Carbon hydrophilic wet deposition flux @@ -3502,10 +3534,10 @@ subroutine ice_import_moab(Eclock)!{{{ ! o dstdry2 -- Size 2 dust -- dry deposition flux ! o dstdry3 -- Size 3 dust -- dry deposition flux ! o dstdry4 -- Size 4 dust -- dry deposition flux -! +! ! The following fields are sometimes received from the coupler, ! depending on model options: -! +! ! o algae1 -- ! o algae2 -- ! o algae3 -- @@ -3530,7 +3562,7 @@ subroutine ice_import_moab(Eclock)!{{{ ! o zaer4 -- ! o zaer5 -- ! o zaer6 -- -! +! !----------------------------------------------------------------------- ! ! !REVISION HISTORY: @@ -3550,7 +3582,7 @@ subroutine ice_import_moab(Eclock)!{{{ character (len=StrKIND) :: & label, & message - + integer :: & i,n @@ -3687,10 +3719,10 @@ subroutine ice_import_moab(Eclock)!{{{ if (ierr > 0 ) then write(iceLogUnit,*) 'Fail to write ice state ' endif -#endif -!----------------------------------------------------------------------- +#endif +!----------------------------------------------------------------------- ! -! zero out padded cells +! zero out padded cells ! !----------------------------------------------------------------------- diff --git a/components/mpas-seaice/src/Registry.xml b/components/mpas-seaice/src/Registry.xml index 78830161c62..26559e4aa49 100644 --- a/components/mpas-seaice/src/Registry.xml +++ b/components/mpas-seaice/src/Registry.xml @@ -1879,6 +1879,10 @@ description="Use data iceberg meltwater forcing" possible_values="true or false" /> + + + + @@ -4956,11 +4963,26 @@ - + + + diff --git a/components/mpas-seaice/src/model_forward/mpas_seaice_core_interface.F b/components/mpas-seaice/src/model_forward/mpas_seaice_core_interface.F index 4b128d0c534..6240ea9f705 100644 --- a/components/mpas-seaice/src/model_forward/mpas_seaice_core_interface.F +++ b/components/mpas-seaice/src/model_forward/mpas_seaice_core_interface.F @@ -8,6 +8,7 @@ module seaice_core_interface use seaice_analysis_driver use mpas_log, only: mpas_log_write + implicit none public contains @@ -546,10 +547,10 @@ end subroutine setup_packages_column_physics!}}} ! routine setup_packages_bergs ! !> \brief Setup icebergs package - !> \author Darin Comeau - !> \date 19 May 2017 - !> \details This routine is intended to set the icebergs package PkgBergs - !> as active/deactive based on the namelist option config_use_bergs. + !> \author Darin Comeau, Xylar Asay-Davis + !> \date August 2024 + !> \details This routine is intended to set the icebergs packages PkgBergs + !> and pkgScaledDIB as active/deactive based on the namelist options. ! !----------------------------------------------------------------------- @@ -561,10 +562,13 @@ subroutine setup_packages_bergs(configPool, packagePool, ierr)!{{{ ! icebergs package logical, pointer :: & - config_use_data_icebergs + config_use_data_icebergs, & + config_scale_dib_by_removed_ice_runoff + logical, pointer :: & - pkgBergsActive + pkgBergsActive, & + pkgScaledDIBActive ierr = 0 @@ -576,6 +580,10 @@ subroutine setup_packages_bergs(configPool, packagePool, ierr)!{{{ call MPAS_pool_get_package(packagePool, "pkgBergsActive", pkgBergsActive) pkgBergsActive = config_use_data_icebergs + call MPAS_pool_get_config(configPool, "config_scale_dib_by_removed_ice_runoff", config_scale_dib_by_removed_ice_runoff) + call MPAS_pool_get_package(packagePool, "pkgScaledDIBActive", pkgScaledDIBActive) + pkgScaledDIBActive = config_scale_dib_by_removed_ice_runoff + end subroutine setup_packages_bergs!}}} !*********************************************************************** diff --git a/components/mpas-seaice/src/shared/mpas_seaice_forcing.F b/components/mpas-seaice/src/shared/mpas_seaice_forcing.F index ca0c4c5a962..86818975700 100644 --- a/components/mpas-seaice/src/shared/mpas_seaice_forcing.F +++ b/components/mpas-seaice/src/shared/mpas_seaice_forcing.F @@ -3318,6 +3318,9 @@ subroutine get_data_iceberg_fluxes(domain) berg_forcing, & berg_fluxes + logical, pointer :: & + config_scale_dib_by_removed_ice_runoff + integer, pointer :: & nCellsSolve @@ -3326,14 +3329,36 @@ subroutine get_data_iceberg_fluxes(domain) bergFreshwaterFlux, & ! iceberg freshwater flux for ocean (kg/m^2/s) bergLatentHeatFlux ! iceberg latent heat flux for ocean (J/m^2/s) + real(kind=RKIND), pointer :: & + areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux, & ! area integrated, annual mean freshwater flux from icebegs and ice shelves (kg/s) + runningMeanRemovedIceRunoff ! the area integrated, running mean of removed ice runoff (kg/s) + integer :: & iCell + real(kind=RKIND) :: & + scaling + ! dc including as parameters here so as not to create new namelist options real(kind=RKIND), parameter :: & specificHeatFreshIce = 2106.0_RKIND, & ! specific heat of fresh ice J * kg^-1 * K^-1 bergTemperature = -4.0_RKIND ! iceberg temperature, assumed constant + + call MPAS_pool_get_config(domain % configs, "config_scale_dib_by_removed_ice_runoff", & + config_scale_dib_by_removed_ice_runoff) + + if (config_scale_dib_by_removed_ice_runoff) then + block => domain % blocklist + call MPAS_pool_get_subpool(block % structs, "berg_forcing", berg_forcing) + call mpas_pool_get_array(berg_forcing, 'runningMeanRemovedIceRunoff', runningMeanRemovedIceRunoff) + call mpas_pool_get_array(berg_forcing, 'areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux', & + areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux) + scaling = runningMeanRemovedIceRunoff / areaIntegAnnMeanDataIcebergIceShelfFreshwaterFlux + else + scaling = 1.0_RKIND + end if + block => domain % blocklist do while (associated(block)) @@ -3349,8 +3374,8 @@ subroutine get_data_iceberg_fluxes(domain) do iCell = 1, nCellsSolve - bergFreshwaterFlux(iCell) = bergFreshwaterFluxData(iCell) - bergLatentHeatFlux(iCell) = -bergFreshwaterFluxData(iCell) * & + bergFreshwaterFlux(iCell) = scaling * bergFreshwaterFluxData(iCell) + bergLatentHeatFlux(iCell) = -scaling * bergFreshwaterFluxData(iCell) * & (seaiceLatentHeatMelting - specificHeatFreshIce*bergTemperature) enddo diff --git a/docs/dev-guide/adding-grid-support/adding-grid-support-step-by-step-guide/add-grid-config.md b/docs/dev-guide/adding-grid-support/adding-grid-support-step-by-step-guide/add-grid-config.md index f46a84afa76..bc45fadc964 100644 --- a/docs/dev-guide/adding-grid-support/adding-grid-support-step-by-step-guide/add-grid-config.md +++ b/docs/dev-guide/adding-grid-support/adding-grid-support-step-by-step-guide/add-grid-config.md @@ -1,3 +1,207 @@ # Add New Grid Configuration to E3SM +In addition to generating input data to support a new grid, several modifications to XML files are required before E3SM can run with the grid. +However, the specific changes will depend on how the grid will be used. The intended model configuration for the new grid will change which files need to be modified. For instance, a grid intended for aquaplanet experiments does not require as many changes as a historical AMIP-style run. + +The guidelines here are meant to outline various possible changes the user should consider when adding support for a new grid for the land and/or atmosphere. +This document cannot be exhaustive, and it is important that the user understands the changes they are making. It is often useful to use a pre-existing grid configuration as a template. +Note that the guidelines here are only relevant for "horizontal" grids in the atmosphere and/or land. +Additional considerations are needed to support a new vertical grid in the atmosphere, which is a topic not currently covered here. + +When setting up a new grid for the atmosphere and/or land model, you will need to edit some or all of these files: + +- `cime_config/config_grids.xml` +- `components/eam/bld/config_files/horiz_grid.xml` +- `components/eam/bld/namelist_files/namelist_defaults_eam.xml` +- `components/eam/bld/namelist_files/namelist_definition.xml` +- `components/elm/bld/namelist_files/namelist_definition.xml` +- `components/elm/bld/namelist_files/namelist_defaults.xml` + +## Mono-Grid vs Bi-Grid vs Tri-Grid + +The mono-bi-tri grid options in E3SM can be confusing, but it is important to understand what these terms mean when adding a new grid to E3SM. At the surface these terms mean that the whole model is either using a single grid for all componennt models, or a combination of 2 or 3 grids shared among the component models. Note that mono-grid and bi-grid terms often ignore that the river model needs to be on its own regular lat-lon grid. + +In practice, "bi" and "tri" grids are most commonly used and the main difference between them comes down to whether the land surface model shares a grid with the atmosphere or not. The component coupler is responsible for facilitating communication between component models, primarily through fluxes, and so mapping files are needed to support a combination of different grids. E3SMv3 uses a tri-grid configuration for production simulations. + +## Grid Naming Conventions + +### Atmosphere + +The atmosphere grid name should always indicate the base "ne" value and whether the physgrid is being used, usually by adding ".pg2" at the end. For a regionally refined mesh (RRM) the grid name should always start with `ne0` followed a descriptive string that includes the region being refined and the degree of refinement. + +**Example**: `ne0np4_northamerica_30x4v1.pg2` + +Note that this example differs from how the North American grid is currently named as `ne0np4_northamericax4v1.pg2`, which indicates a `4x` refinement, but does not indicate the base resolution, which is useful to know. The more informative grid name `ne0np4_northamerica_30x4v1.pg2` makes it clear that unrefined regions are consistent with `ne30pg2`. + +### River (or Land in tri-grid) + +For a rectilinear lat-lon grid used by the land and/or river models the grid name should start with "r" and typically use spacing less than one degree, so they indicate the nominal grid spacing, starting with "0" and omitting the decimal. + +**Examples**: `r05` is 0.5 degree spacing and `r0125` is 1/8 or 0.125 degree spacing. + +### Grid Aliases + +Grid aliases are short strings used to represent the complete set of grids used in the model configuration. +For a mono-grid the convention is that the grid alias is the base mesh written twice to indicate that both atmosphere/land and ocean/sea-ice models are on the same grid. A mono-grid is typically only used for idealized simulations such as aqua planet and RCE, but can also be used for F-compsets if the CICE sea-ice model is used in place of the MPAS sea-ice model (MPASSI). + +**Example**: `ne30pg2_ne30pg2` + +Bi-grid options should indicate two different grids used for atmosphere/land and ocean/sea-ice models. + +**Example**: `ne30pg2_IcoswISC30E3r5` + +Tri-grid options should indicate three different grids used for atmosphere, land, and ocean/sea-ice models, with the land grid appearing in the middle. + +**Example**: `ne30pg2_r05_IcoswISC30E3r5` + +### Grid longnames + +For any combination of grids, the full grid definition has a long form representation that spells out the grid in more detail. + +**Example**: + +```shell + alias: ne4pg2_ne4pg2 + + longname: a%ne4np4.pg2_l%ne4np4.pg2_oi%ne4np4.pg2_r%r05_g%null_w%null_z%null_m%oQU240 + non-default grids are: atm:ne4np4.pg2 lnd:ne4np4.pg2 ocnice:ne4np4.pg2 rof:r05 glc:null wav:null + mask is: oQU240 +``` + +## Defining a New Atmosphere Grid for EAM + +When defining a new atmosphere grid, information needs to be provided on how the grid is constructed. + +To define a new atmosphere grid a line must be added to `components/eam/bld/config_files/horiz_grid.xml` that indicates the number of elements and physics columns. In the lines below for `ne30np4` (without the physgrid) and `ne30pg2` (with the physgrid) you can see the value of `ne` is the same (number of elements along a cube edge), but the number of physics columns is different. + +```xml + + +``` + +An explanation of how to calculate the number of physics columns can be found here: [Atmosphere Grid Overview](../../../EAM/tech-guide/atmosphere-grid-overview.md). + +For a grid with regional refinement, follow the conventions of other grids in this file. There is no formula to calculate the number of columns for RRM grids, but the value can be obtained from the grid files used for mapping. + +```xml + +``` + +## Defining a New Land Grid for ELM + +If you are creating a new grid that will be used by the land model the grid name needs to be added to the list `valid_values` associated with the `res` entry in the file `components/elm/bld/namelist_files/namelist_definition.xml` that holds the definition of namelist variables used by the land model. + +```xml + +Horizontal resolutions +Note: 0.1x0.1, 0.5x0.5, 5x5min, 10x10min, 3x3min and 0.33x0.33 are only used for ELM tools + +``` + +Simply add the name of your new grid to the list of `valid_values`. + +## Using New Grids in Default Namelists + +Each new grid will likely need various new default parameter values to be specified. These parameters can be set for individual simulations by editing the `user_nl_*` files in the case directory, but for these to become defaults any time the grid is used then new defaults need to be specified. + +The lists below show namelist parameters that may need to be specified for a new grid. The creator of a new grid is responsible for understanding these parameters and deciding when new defaults are appropriate. + +### Atmosphere Namelist Parameters + +- `drydep_srf_file` - Data file for surface aerosol deposition +- `bnd_topo` - Surface topography (smoothed for target grid) +- `mesh_file` - HOMME np4 mesh file (exodus format) +- `se_tstep` - HOMME time step [seconds] +- `dt_remap_factor` - HOMME vertical remap factor +- `dt_tracer_factor` - HOMME tracer advection factor +- `hypervis_subcycle_q` - HOMME tracer hyperviscosity factor + +### Land Namelist Parameters + +- `fsurdat` - Surface data file +- `finidat` - Land model initial condition file +- `flanduse_timeseries` - Time-evolving land-use data file + +## Defining a new grid for CIME + +The CIME Case Control system will configure a case according to the component set and grid alias you specify with the `--res` argument. +As part of that configuration, CIME needs to know +how to translate the grid alias and set the paths for domain and mapping files used by the grid so the model can find them at runtime. + +### Adding a New Grid Alias + +Grid aliases are defined in `cime_config/config_grids.xml`. Below is an example grid alias for the `ne30pg2_r05_IcoswISC30E3r5` grid used in E3SMv3 production simulations. + +```xml + + ne30np4.pg2 + r05 + IcoswISC30E3r5 + r05 + null + null + IcoswISC30E3r5 + +``` + +Add a similar block for your new grid. Aliases must be unique within `config_grids.xml` + +### Domain Files + +Domain files are needed for each grid and are specified in the `` section of `cime_config/config_grids.xml`. The default domain files are grouped by the atmosphere grid. The section for the typical `ne30pg2` grid looks as follows: + +```xml + + 21600 + 1 + ... + $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_IcoswISC30E3r5.231121.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_IcoswISC30E3r5.231121.nc + ... + ne30np4.pg2 is Spectral Elem 1-deg grid w/ 2x2 FV physics grid per element: + +``` + +Notice the ellipses `...` are used here to omit all entries that are not relevant to the `ne30pg2_r05_IcoswISC30E3r5` grid. Also, note that all of these paths are relative to the input data path set as `DIN_LOC_ROOT` which has a default for each machine. See [Generating Domain Files](../../../generate_domain_files/index.md) for information about creating domain files. + +### Coupler Mapping Files + +The mapping files used by the component coupler to communicate fluxes between the component models must be specified in the `` section of `cime_config/config_grids.xml`. These are organized for specific pairs of grids, such that tri-grids will require multiple sections. The entries relevant for `ne30pg2_r05_IcoswISC30E3r5` are shown below. + +```xml + + cpl/gridmaps/ne30pg2/map_ne30pg2_to_IcoswISC30E3r5_traave.20231121.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_IcoswISC30E3r5_trbilin.20231121.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_IcoswISC30E3r5-nomask_trbilin.20231121.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_ne30pg2_traave.20231121.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_ne30pg2_traave.20231121.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_IcoswISC30E3r5_trfvnp2.20231121.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_IcoswISC30E3r5_trfvnp2.20231121.nc + +``` + +```xml + + cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_traave.20231130.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_trfvnp2.230516.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_trbilin.20231130.nc + cpl/gridmaps/ne30pg2/map_r05_to_ne30pg2_traave.20231130.nc + cpl/gridmaps/ne30pg2/map_r05_to_ne30pg2_traave.20231130.nc + +``` + +```xml + + cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_traave.20231130.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_trfvnp2.230516.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_r05_trbilin.20231130.nc + +``` + +Note that all of these paths are relative to the input data path set as `DIN_LOC_ROOT` which has a default for each machine. Mapping files can be created with +the [ncremap](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/754286611/Regridding+E3SM+Data+with+ncremap) utility in NCO + Back to step-by-step guide for [Adding Support for New Grids](../adding-grid-support-step-by-step-guide.md) diff --git a/docs/index.md b/docs/index.md index 4d979cf0afe..6a30bcf5545 100644 --- a/docs/index.md +++ b/docs/index.md @@ -24,6 +24,7 @@ research problems and Department of Energy mission needs while efficiently using - [MPAS-Ocean](./MPAS-Ocean/index.md) - [MPAS-seaice](./MPAS-seaice/index.md) - [Omega](https://docs.e3sm.org/Omega/omega/) — not yet supported. +- [Data Models](./Data-Models/index.md) ## Tools diff --git a/docs/refs/eam.bib b/docs/refs/eam.bib index ff4415a519b..84bdd2499ce 100644 --- a/docs/refs/eam.bib +++ b/docs/refs/eam.bib @@ -1035,3 +1035,128 @@ @article{neale_description_2012 journal = {UNKNOWN}, year = {2012}, } + +@article{xie_an_2020, + title = {An Orographic-Drag Parametrization Scheme Including Orographic Anisotropy for All Flow Directions}, + url = {https://agupubs.onlinelibrary.wiley.com/doi/full/10.1029/2019MS001921/}, + doi = {10.1029/2019MS001921}, + language = {en}, + urldate = {2024-11-19}, + author = {J., Xie and M.,Zhang and Z., Xie and H., Liu and Z., Chai and J., He and H. Zhang}, + journal = {Journal of Advances in Modeling Earth Systems}, + year = {2020}, +} + +@article{beljaars_a_2020, + title = {A new parametrization of turbulent orographic form drag}, + url = {https://rmets.onlinelibrary.wiley.com/doi/abs/10.1256/qj.03.73/}, + doi = {10.1256/qj.03.73}, + language = {en}, + urldate = {2024-11-19}, + author = {A. Beljaars, A. Brown, N. Wood}, + journal = {Quarterly Journal of the Royal Meterological Society}, + year = {2004}, +} + +@article{tsiringakis_small_2020, + title = {Small-scale orographic gravity wave drag in stable boundary layers and its impact on synoptic systems and near-surface meteorology}, + url = {https://rmets.onlinelibrary.wiley.com/doi/abs/10.1002/qj.3021}, + doi = {10.1002/qj.3021}, + language = {en}, + urldate = {2024-11-19}, + author = {A. Tsiringakis, G.J. Steeneveld, A.A.M. Holtslag}, + journal = {Quarterly Journal of the Royal Meterological Society}, + year = {2017}, +} + +@article{mcfarlane_the_1987, + title = {The Effect of Orographically Excited Gravity Wave Drag on the General Circulation of the Lower Stratosphere and Troposphere}, + url = {https://journals.ametsoc.org/view/journals/atsc/44/14/1520-0469_1987_044_1775_teooeg_2_0_co_2.xml}, + doi = {10.1175/1520-0469(1987)044<1775:TEOOEG>2.0.CO;2}, + language = {en}, + urldate = {2024-11-19}, + author = {N.A. McFarlane}, + journal = {Journal of the Atmospheric Sciences}, + year = {1987}, +} + +@article{richter_the_2010, + title = {The Effect of Orographically Excited Gravity Wave Drag on the General Circulation of the Lower Stratosphere and Troposphere}, + url = {https://journals.ametsoc.org/view/journals/atsc/67/1/2009jas3112.1.xml?tab_body=pdf}, + doi = {10.1175/2009JAS3112.1}, + language = {en}, + urldate = {2024-11-19}, + author = {J.H. Richter, F. Sassi, R.R. Garcia}, + journal = {Journal of the Atmospheric Sciences}, + year = {2010}, +} + +@article{holtslag_preface_2006, + title = {Preface:GEWEXatmospheric boundary-layer study (GABLS) on stable boundary layers}, + url = {https://link.springer.com/article/10.1007/s10546-005-9008-6}, + doi = {10.1007/s10546-005-9008-6}, + language = {en}, + urldate = {2024-11-19}, + author = {A.A Holtslag}, + journal = {Boundary-Layer Meterology}, + year = {2006}, +} + +@article{blackburn_APE_context_2013, + title = {Context and {Aims} of the {Aqua}-{Planet} {Experiment}}, + volume = {91A}, + issn = {2186-9057}, + doi = {10.2151/jmsj.2013-A01}, + journal = {Journal of the Meteorological Society of Japan. Ser. II}, + author = {BLACKBURN, Michael and HOSKINS, Brian J.}, + month = oct, + year = {2013}, + keywords = {precipitation, circulation models, comparison of atmospheric general, gcms, intertropical convergence zone, itcz, modelling hierarchy, tropical circulation}, + pages = {1--15}, +} + +@article{wing_rcemip1_2018, + title = {Radiative–convective equilibrium model intercomparison project}, + volume = {11}, + issn = {1991-959X}, + doi = {10.5194/gmd-11-793-2018}, + language = {English}, + number = {2}, + journal = {Geoscientific Model Development}, + author = {Wing, Allison A. and Reed, Kevin A. and Satoh, Masaki and Stevens, Bjorn and Bony, Sandrine and Ohno, Tomoki}, + month = mar, + year = {2018}, + note = {Publisher: Copernicus GmbH}, + pages = {793--813}, + file = {Full Text PDF:/Users/hannah6/Zotero/storage/KHKQVU33/Wing et al. - 2018 - Radiative–convective equilibrium model intercomparison project.pdf:application/pdf}, +} + +@article{wing_rcemip2_2024, + title = {{RCEMIP}-{II}: mock-{Walker} simulations as phase {II} of the radiative–convective equilibrium model intercomparison project}, + volume = {17}, + issn = {1991-959X}, + shorttitle = {{RCEMIP}-{II}}, + doi = {10.5194/gmd-17-6195-2024}, + language = {English}, + number = {16}, + journal = {Geoscientific Model Development}, + author = {Wing, Allison A. and Silvers, Levi G. and Reed, Kevin A.}, + month = aug, + year = {2024}, + note = {Publisher: Copernicus GmbH}, + pages = {6195--6225}, + file = {Full Text PDF:/Users/hannah6/Zotero/storage/IY2SP66J/Wing et al. - 2024 - RCEMIP-II mock-Walker simulations as phase II of the radiative–convective equilibrium model interco.pdf:application/pdf}, +} + +@article { Zarzycki_TC-ocn-cpl_2016, + author = "Colin M. Zarzycki", + title = "Tropical Cyclone Intensity Errors Associated with Lack of Two-Way Ocean Coupling in High-Resolution Global Simulations", + journal = "Journal of Climate", + year = "2016", + publisher = "American Meteorological Society", + address = "Boston MA, USA", + volume = "29", + number = "23", + doi = "10.1175/JCLI-D-16-0273.1", + pages = "8589 - 8610", +} diff --git a/driver-mct/cime_config/buildnml b/driver-mct/cime_config/buildnml index 4938e9da0b1..c65a6047555 100755 --- a/driver-mct/cime_config/buildnml +++ b/driver-mct/cime_config/buildnml @@ -41,6 +41,7 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): config['CPL_EPBAL'] = case.get_value('CPL_EPBAL') config['FLDS_WISO'] = case.get_value('FLDS_WISO') config['FLDS_POLAR'] = case.get_value('FLDS_POLAR') + config['FLDS_TF'] = case.get_value('FLDS_TF') config['BUDGETS'] = case.get_value('BUDGETS') config['MACH'] = case.get_value('MACH') config['MPILIB'] = case.get_value('MPILIB') diff --git a/driver-mct/cime_config/config_component.xml b/driver-mct/cime_config/config_component.xml index cdfca40befa..666474353c7 100644 --- a/driver-mct/cime_config/config_component.xml +++ b/driver-mct/cime_config/config_component.xml @@ -1898,40 +1898,58 @@ glc2ocn runoff mapping file decomp type for ice runoff - + char idmap_ignore run_domain env_run.xml - ocn2glc flux mapping file - the default value idmap_ignore, if set, will be ignored by buildnml and + ocn2glc shelf flux mapping file - the default value idmap_ignore, if set, will be ignored by buildnml and will generate a runtime error if in fact a file is required for the given compset - + char X,Y Y run_domain env_run.xml - ocn2glc flux mapping file decomp type + ocn2glc shelf flux mapping file decomp type - + char idmap_ignore run_domain env_run.xml - ocn2glc state mapping file - the default value idmap_ignore, if set, will be ignored by buildnml and + ocn2glc shelf state mapping file - the default value idmap_ignore, if set, will be ignored by buildnml and will generate a runtime error if in fact a file is required for the given compset - + char X,Y Y run_domain env_run.xml - ocn2glc state mapping file decomp type + ocn2glc shelf state mapping file decomp type + + + + char + idmap_ignore + run_domain + env_run.xml + ocn2glc state mapping file for thermal forcing - the default value idmap_ignore, if set, will be ignored by buildnml and + will generate a runtime error if in fact a file is required for the given compset + + + + char + X,Y + Y + run_domain + env_run.xml + ocn2glc thermal forcing state mapping file decomp type diff --git a/driver-mct/cime_config/config_component_e3sm.xml b/driver-mct/cime_config/config_component_e3sm.xml index 8c793f144ef..c16a493f045 100755 --- a/driver-mct/cime_config/config_component_e3sm.xml +++ b/driver-mct/cime_config/config_component_e3sm.xml @@ -185,6 +185,18 @@ Turn on the passing of polar fields through the coupler + + logical + TRUE,FALSE + FALSE + + TRUE + + run_flags + env_run.xml + Turn on the passing of ocean thermal forcing fields through the coupler + + char minus1p8,linear_salt,mushy diff --git a/driver-mct/cime_config/namelist_definition_drv.xml b/driver-mct/cime_config/namelist_definition_drv.xml index 982acabe64d..a0cfc87e526 100644 --- a/driver-mct/cime_config/namelist_definition_drv.xml +++ b/driver-mct/cime_config/namelist_definition_drv.xml @@ -149,6 +149,18 @@ + + logical + seq_flds + seq_cplflds_inparm + + If set to .true. thermal forcing fields will be passed from the ocean to the coupler. + + + $FLDS_TF + + + logical seq_flds @@ -4531,20 +4543,50 @@ - + + char + mapping + abs + seq_maps + + ocn to glc shelf mapping file for fluxes + + + $OCN2GLC_SHELF_FMAPNAME + + + + + char + mapping + seq_maps + + The type of mapping desired, either "source" or "destination" mapping. + X is associated with rearrangement of the source grid to the + destination grid and then local mapping. Y is associated with mapping + on the source grid and then rearrangement and sum to the destination + grid. + + + $OCN2GLC_SHELF_FMAPTYPE + X + + + + char mapping abs seq_maps - ocn to glc flux mapping file for fluxes + ocn to glc shelf mapping file for states - $OCN2GLC_FMAPNAME + $OCN2GLC_SHELF_SMAPNAME - + char mapping seq_maps @@ -4556,25 +4598,25 @@ grid. - $OCN2GLC_FMAPTYPE + $OCN2GLC_SHELF_SMAPTYPE X - + char mapping abs seq_maps - ocn to glc state mapping file for states + ocn to glc state mapping file for thermal forcing state fields - $OCN2GLC_SMAPNAME + $OCN2GLC_TF_SMAPNAME - + char mapping seq_maps @@ -4586,7 +4628,7 @@ grid. - $OCN2GLC_SMAPTYPE + $OCN2GLC_TF_SMAPTYPE X diff --git a/driver-mct/main/cime_comp_mod.F90 b/driver-mct/main/cime_comp_mod.F90 index 7cf442b9413..4614474bac3 100644 --- a/driver-mct/main/cime_comp_mod.F90 +++ b/driver-mct/main/cime_comp_mod.F90 @@ -145,7 +145,8 @@ module cime_comp_mod ! diagnostic routines use seq_diag_mct, only : seq_diag_zero_mct , seq_diag_avect_mct, seq_diag_lnd_mct use seq_diag_mct, only : seq_diag_rof_mct , seq_diag_ocn_mct , seq_diag_atm_mct - use seq_diag_mct, only : seq_diag_ice_mct , seq_diag_accum_mct, seq_diag_print_mct + use seq_diag_mct, only : seq_diag_ice_mct , seq_diag_glc_mct + use seq_diag_mct, only : seq_diag_accum_mct, seq_diag_print_mct use seq_diagBGC_mct, only : seq_diagBGC_zero_mct , seq_diagBGC_avect_mct, seq_diagBGC_lnd_mct use seq_diagBGC_mct, only : seq_diagBGC_rof_mct , seq_diagBGC_ocn_mct , seq_diagBGC_atm_mct use seq_diagBGC_mct, only : seq_diagBGC_ice_mct , seq_diagBGC_accum_mct @@ -434,6 +435,7 @@ module cime_comp_mod logical :: lnd_c2_glc ! .true. => lnd to glc coupling on logical :: ocn_c2_atm ! .true. => ocn to atm coupling on logical :: ocn_c2_ice ! .true. => ocn to ice coupling on + logical :: ocn_c2_glctf ! .true. => ocn to glc thermal forcing coupling on logical :: ocn_c2_glcshelf ! .true. => ocn to glc ice shelf coupling on logical :: ocn_c2_wav ! .true. => ocn to wav coupling on logical :: ocn_c2_rof ! .true. => ocn to rof coupling on @@ -1674,6 +1676,7 @@ subroutine cime_init() ocn_prognostic=ocn_prognostic, & ocnrof_prognostic=ocnrof_prognostic, & ocn_c2_glcshelf=ocn_c2_glcshelf, & + ocn_c2_glctf=ocn_c2_glctf, & glc_prognostic=glc_prognostic, & rof_prognostic=rof_prognostic, & rofocn_prognostic=rofocn_prognostic, & @@ -1873,8 +1876,9 @@ subroutine cime_init() write(logunit,F0L)'lnd_c2_rof = ',lnd_c2_rof write(logunit,F0L)'lnd_c2_glc = ',lnd_c2_glc write(logunit,F0L)'ocn_c2_atm = ',ocn_c2_atm - write(logunit,F0L)'ocn_c2_ice = ',ocn_c2_ice write(logunit,F0L)'ocn_c2_glcshelf = ',ocn_c2_glcshelf + write(logunit,F0L)'ocn_c2_glctf = ',ocn_c2_glctf + write(logunit,F0L)'ocn_c2_ice = ',ocn_c2_ice write(logunit,F0L)'ocn_c2_wav = ',ocn_c2_wav write(logunit,F0L)'ocn_c2_rof = ',ocn_c2_rof write(logunit,F0L)'ice_c2_atm = ',ice_c2_atm @@ -1959,7 +1963,7 @@ subroutine cime_init() endif if ((ocn_c2_glcshelf .and. .not. glcshelf_c2_ocn) .or. (glcshelf_c2_ocn .and. .not. ocn_c2_glcshelf)) then ! Current logic will not allow this to be true, but future changes could make it so, which may be nonsensical - call shr_sys_abort(subname//' ERROR: if glc_c2_ocn must also have ocn_c2_glc and vice versa. '//& + call shr_sys_abort(subname//' ERROR: if glcshelf_c2_ocn must also have ocn_c2_glcshelf and vice versa. '//& 'Boundary layer fluxes calculated in coupler require input from both components.') endif if (rofice_present .and. .not.rof_present) then @@ -2028,7 +2032,7 @@ subroutine cime_init() call prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) - call prep_glc_init(infodata, lnd_c2_glc, ocn_c2_glcshelf) + call prep_glc_init(infodata, lnd_c2_glc, ocn_c2_glctf, ocn_c2_glcshelf) call prep_wav_init(infodata, atm_c2_wav, ocn_c2_wav, ice_c2_wav) @@ -3052,8 +3056,12 @@ subroutine cime_run() !---------------------------------------------------------- !| GLC SETUP-SEND !---------------------------------------------------------- - if (glc_present .and. glcrun_alarm) then - call cime_run_glc_setup_send(lnd2glc_averaged_now, prep_glc_accum_avg_called) + if (glc_present) then + if (glcrun_alarm) then + call cime_run_glc_setup_send(lnd2glc_averaged_now, prep_glc_accum_avg_called) + else + call prep_glc_zero_fields() + endif endif ! ------------------------------------------------------------------------ @@ -3100,6 +3108,7 @@ subroutine cime_run() endif endif + !---------------------------------------------------------- !| Budget with old fractions !---------------------------------------------------------- @@ -4216,12 +4225,16 @@ subroutine cime_run_ocnglc_coupling() if (glc_present) then + ! create o2x_gx for either ocn-glc coupling or ocn-glc shelf coupling + if (ocn_c2_glctf .or. (ocn_c2_glcshelf .and. glcshelf_c2_ocn)) then + call prep_glc_calc_o2x_gx(ocn_c2_glctf, ocn_c2_glcshelf, timer='CPL:glcprep_ocn2glc') !remap ocean fields to o2x_g at ocean couping interval + endif + + ! if ice-shelf coupling is on, now proceed to handle those calculations here in the coupler if (ocn_c2_glcshelf .and. glcshelf_c2_ocn) then ! the boundary flux calculations done in the coupler require inputs from both GLC and OCN, ! so they will only be valid if both OCN->GLC and GLC->OCN - call prep_glc_calc_o2x_gx(timer='CPL:glcprep_ocn2glc') !remap ocean fields to o2x_g at ocean couping interval - call prep_glc_calculate_subshelf_boundary_fluxes ! this is actual boundary layer flux calculation !this outputs !x2g_g/g2x_g, where latter is going @@ -4348,7 +4361,7 @@ subroutine cime_run_glc_setup_send(lnd2glc_averaged_now, prep_glc_accum_avg_call if (drv_threading) call seq_comm_setnthreads(nthreads_CPLID) ! NOTE - only create appropriate input to glc if the avg_alarm is on - if (lnd_c2_glc .or. ocn_c2_glcshelf) then + if (lnd_c2_glc .or. ocn_c2_glctf .or. ocn_c2_glcshelf) then if (glcrun_avg_alarm) then call prep_glc_accum_avg(timer='CPL:glcprep_avg', & lnd2glc_averaged_now=lnd2glc_averaged_now) @@ -4361,6 +4374,13 @@ subroutine cime_run_glc_setup_send(lnd2glc_averaged_now, prep_glc_accum_avg_call call prep_glc_mrg_lnd(infodata, fractions_gx, timer_mrg='CPL:glcprep_mrgx2g') endif + if (ocn_c2_glctf) then + ! note: o2x_gx is handled in prep_glc_calc_o2x_gx, which is called + ! from cime_run_ocnglc_coupling in this module + call prep_glc_mrg_ocn(infodata, fractions_gx, timer_mrg='CPL:glcprep_mrgocnx2g') + endif + + call component_diag(infodata, glc, flow='x2c', comment='send glc', & info_debug=info_debug, timer_diag='CPL:glcprep_diagav') @@ -4749,6 +4769,9 @@ subroutine cime_run_calc_budgets1(in_cplrun) if (ice_present) then call seq_diag_ice_mct(ice(ens1), fractions_ix(ens1), infodata, do_x2i=.true.) endif + if (glc_present) then + call seq_diag_glc_mct(glc(ens1), fractions_gx(ens1), infodata, do_x2g=.true.) + endif if (do_bgc_budgets) then if (rof_present) then call seq_diagBGC_rof_mct(rof(ens1), fractions_rx(ens1), infodata) @@ -4788,6 +4811,9 @@ subroutine cime_run_calc_budgets2(in_cplrun) if (ice_present) then call seq_diag_ice_mct(ice(ens1), fractions_ix(ens1), infodata, do_i2x=.true.) endif + if (glc_present) then + call seq_diag_glc_mct(glc(ens1), fractions_gx(ens1), infodata, do_g2x=.true.) + endif if (do_bgc_budgets) then if (atm_present) then call seq_diagBGC_atm_mct(atm(ens1), fractions_ax(ens1), infodata, do_a2x=.true., do_x2a=.true.) @@ -5595,3 +5621,4 @@ function copy_and_trim_rpointer_file(src, dst) result(out) end function copy_and_trim_rpointer_file end module cime_comp_mod + diff --git a/driver-mct/main/prep_glc_mod.F90 b/driver-mct/main/prep_glc_mod.F90 index 07aeb9890bd..890edc82295 100644 --- a/driver-mct/main/prep_glc_mod.F90 +++ b/driver-mct/main/prep_glc_mod.F90 @@ -31,6 +31,7 @@ module prep_glc_mod public :: prep_glc_init public :: prep_glc_mrg_lnd + public :: prep_glc_mrg_ocn public :: prep_glc_accum_lnd public :: prep_glc_accum_ocn @@ -45,6 +46,7 @@ module prep_glc_mod public :: prep_glc_get_l2gacc_lx public :: prep_glc_get_l2gacc_lx_one_instance public :: prep_glc_get_l2gacc_lx_cnt + public :: prep_glc_get_l2gacc_lx_cnt_avg public :: prep_glc_get_o2x_gx public :: prep_glc_get_x2gacc_gx @@ -53,8 +55,8 @@ module prep_glc_mod public :: prep_glc_get_mapper_Sl2g public :: prep_glc_get_mapper_Fl2g - public :: prep_glc_get_mapper_So2g - public :: prep_glc_get_mapper_Fo2g + public :: prep_glc_get_mapper_So2g_shelf + public :: prep_glc_get_mapper_Fo2g_shelf public :: prep_glc_calculate_subshelf_boundary_fluxes @@ -76,8 +78,9 @@ module prep_glc_mod ! mappers type(seq_map), pointer :: mapper_Sl2g type(seq_map), pointer :: mapper_Fl2g - type(seq_map), pointer :: mapper_So2g - type(seq_map), pointer :: mapper_Fo2g + type(seq_map), pointer :: mapper_So2g_shelf + type(seq_map), pointer :: mapper_Fo2g_shelf + type(seq_map), pointer :: mapper_So2g_tf type(seq_map), pointer :: mapper_Fg2l ! attribute vectors @@ -91,6 +94,7 @@ module prep_glc_mod type(mct_aVect), pointer :: l2gacc_lx(:) ! Lnd export, lnd grid, cpl pes - allocated in driver integer , target :: l2gacc_lx_cnt ! l2gacc_lx: number of time samples accumulated + integer , target :: l2gacc_lx_cnt_avg ! l2gacc_lx: number of time samples averaged ! other module variables integer :: mpicom_CPLID ! MPI cpl communicator @@ -135,7 +139,7 @@ module prep_glc_mod !================================================================================================ - subroutine prep_glc_init(infodata, lnd_c2_glc, ocn_c2_glcshelf) + subroutine prep_glc_init(infodata, lnd_c2_glc, ocn_c2_glctf, ocn_c2_glcshelf) !--------------------------------------------------------------- ! Description @@ -144,7 +148,8 @@ subroutine prep_glc_init(infodata, lnd_c2_glc, ocn_c2_glcshelf) ! Arguments type (seq_infodata_type) , intent(inout) :: infodata logical , intent(in) :: lnd_c2_glc ! .true. => lnd to glc coupling on - logical , intent(in) :: ocn_c2_glcshelf ! .true. => ocn to glc coupling on + logical , intent(in) :: ocn_c2_glctf ! .true. => ocn to glc thermal forcing coupling on + logical , intent(in) :: ocn_c2_glcshelf ! .true. => ocn to glc shelf coupling on ! ! Local Variables integer :: eli, egi, eoi @@ -178,8 +183,9 @@ subroutine prep_glc_init(infodata, lnd_c2_glc, ocn_c2_glcshelf) allocate(mapper_Sl2g) allocate(mapper_Fl2g) - allocate(mapper_So2g) - allocate(mapper_Fo2g) + allocate(mapper_So2g_shelf) + allocate(mapper_So2g_tf) + allocate(mapper_Fo2g_shelf) allocate(mapper_Fg2l) smb_renormalize = prep_glc_do_renormalize_smb(infodata) @@ -195,6 +201,7 @@ subroutine prep_glc_init(infodata, lnd_c2_glc, ocn_c2_glcshelf) call mct_aVect_zero(l2gacc_lx(eli)) end do l2gacc_lx_cnt = 0 + l2gacc_lx_cnt_avg = 0 end if if (glc_present .and. lnd_c2_glc) then @@ -249,8 +256,8 @@ subroutine prep_glc_init(infodata, lnd_c2_glc, ocn_c2_glcshelf) end if - if (glc_present .and. ocn_c2_glcshelf) then - + ! setup needed for either kind of ocn2glc coupling + if (glc_present .and. (ocn_c2_glctf .or. ocn_c2_glcshelf)) then call seq_comm_getData(CPLID, & mpicom=mpicom_CPLID, iamroot=iamroot_CPLID) @@ -275,21 +282,35 @@ subroutine prep_glc_init(infodata, lnd_c2_glc, ocn_c2_glcshelf) x2gacc_gx_cnt = 0 samegrid_go = .true. if (trim(ocn_gnam) /= trim(glc_gnam)) samegrid_go = .false. + end if + + ! setup needed for ocn2glc TF coupling + if (glc_present .and. ocn_c2_glctf) then if (iamroot_CPLID) then write(logunit,*) ' ' - write(logunit,F00) 'Initializing mapper_So2g' + write(logunit,F00) 'Initializing mapper_So2g_tf' end if - call seq_map_init_rcfile(mapper_So2g, ocn(1), glc(1), & - 'seq_maps.rc','ocn2glc_smapname:','ocn2glc_smaptype:',samegrid_go, & - 'mapper_So2g initialization',esmf_map_flag) + call seq_map_init_rcfile(mapper_So2g_tf, ocn(1), glc(1), & + 'seq_maps.rc','ocn2glc_tf_smapname:','ocn2glc_tf_smaptype:',samegrid_go, & + 'mapper_So2g_tf initialization',esmf_map_flag) + end if + + ! setup needed for ocn2glcshelf coupling + if (glc_present .and. ocn_c2_glcshelf) then if (iamroot_CPLID) then write(logunit,*) ' ' - write(logunit,F00) 'Initializing mapper_Fo2g' + write(logunit,F00) 'Initializing mapper_So2g_shelf' end if - call seq_map_init_rcfile(mapper_Fo2g, ocn(1), glc(1), & - 'seq_maps.rc','ocn2glc_fmapname:','ocn2glc_fmaptype:',samegrid_go, & - 'mapper_Fo2g initialization',esmf_map_flag) - + call seq_map_init_rcfile(mapper_So2g_shelf, ocn(1), glc(1), & + 'seq_maps.rc','ocn2glc_shelf_smapname:','ocn2glc_shelf_smaptype:',samegrid_go, & + 'mapper_So2g_shelf initialization',esmf_map_flag) + if (iamroot_CPLID) then + write(logunit,*) ' ' + write(logunit,F00) 'Initializing mapper_Fo2g_shelf' + end if + call seq_map_init_rcfile(mapper_Fo2g_shelf, ocn(1), glc(1), & + 'seq_maps.rc','ocn2glc_shelf_fmapname:','ocn2glc_shelf_fmaptype:',samegrid_go, & + 'mapper_Fo2g_shelf initialization',esmf_map_flag) !Initialize module-level arrays associated with compute_melt_fluxes allocate(oceanTemperature(lsize_g)) allocate(oceanSalinity(lsize_g)) @@ -307,10 +328,9 @@ subroutine prep_glc_init(infodata, lnd_c2_glc, ocn_c2_glcshelf) ! TODO: Can we allocate these only while used or are we worried about performance hit? ! TODO: add deallocates! - call shr_sys_flush(logunit) - end if + call shr_sys_flush(logunit) end subroutine prep_glc_init @@ -502,6 +522,7 @@ subroutine prep_glc_accum_avg(timer, lnd2glc_averaged_now) call mct_avect_avg(l2gacc_lx(eli), l2gacc_lx_cnt) end do end if + l2gacc_lx_cnt_avg = l2gacc_lx_cnt l2gacc_lx_cnt = 0 ! Accumulation for OCN @@ -521,6 +542,154 @@ subroutine prep_glc_accum_avg(timer, lnd2glc_averaged_now) end subroutine prep_glc_accum_avg + !================================================================================================ + + subroutine prep_glc_mrg_ocn(infodata, fractions_gx, timer_mrg) + + !--------------------------------------------------------------- + ! Description + ! Merge glc inputs + ! + ! Arguments + type(seq_infodata_type) , intent(in) :: infodata + type(mct_aVect) , intent(in) :: fractions_gx(:) + character(len=*) , intent(in) :: timer_mrg + ! + ! Local Variables + integer :: egi, eoi, efi + type(mct_avect), pointer :: x2g_gx + character(*), parameter :: subname = '(prep_glc_mrg_ocn)' + !--------------------------------------------------------------- + + call t_drvstartf (trim(timer_mrg),barrier=mpicom_CPLID) + do egi = 1,num_inst_glc + ! Use fortran mod to address ensembles in merge + eoi = mod((egi-1),num_inst_ocn) + 1 + efi = mod((egi-1),num_inst_frc) + 1 + + x2g_gx => component_get_x2c_cx(glc(egi)) + call prep_glc_merge_ocn_forcing(o2x_gx(eoi), fractions_gx(efi), x2g_gx) + enddo + call t_drvstopf (trim(timer_mrg)) + + end subroutine prep_glc_mrg_ocn + + !================================================================================================ + + subroutine prep_glc_merge_ocn_forcing( o2x_g, fractions_g, x2g_g ) + + !----------------------------------------------------------------------- + ! Description + ! "Merge" ocean forcing for glc input. + ! + ! State fields are copied directly, meaning that averages are taken just over the + ! ocean-covered portion of the glc domain. + ! + ! Flux fields are downweighted by landfrac, which effectively sends a 0 flux from the + ! non-ocean-covered portion of the glc domain. + ! + ! Arguments + type(mct_aVect), intent(inout) :: o2x_g ! input + type(mct_aVect), intent(in) :: fractions_g + type(mct_aVect), intent(inout) :: x2g_g ! output + !----------------------------------------------------------------------- + + integer :: num_flux_fields + integer :: num_state_fields + integer :: nflds + integer :: i,n + integer :: mrgstr_index + integer :: index_o2x + integer :: index_x2g + integer :: index_ofrac + integer :: lsize + logical :: iamroot + logical, save :: first_time = .true. + character(CL),allocatable :: mrgstr(:) ! temporary string + character(CL) :: field ! string converted to char + character(*), parameter :: subname = '(prep_glc_merge_ocn_forcing) ' + + !----------------------------------------------------------------------- + + call seq_comm_getdata(CPLID, iamroot=iamroot) + lsize = mct_aVect_lsize(x2g_g) + + !num_flux_fields = shr_string_listGetNum(trim(seq_flds_x2g_fluxes_from_ocn)) + num_flux_fields = 0 + num_state_fields = shr_string_listGetNum(trim(seq_flds_x2g_tf_states_from_ocn)) + + if (first_time) then + nflds = num_flux_fields + num_state_fields + allocate(mrgstr(nflds)) + end if + + mrgstr_index = 1 + + do i = 1, num_state_fields + call seq_flds_getField(field, i, seq_flds_x2g_tf_states_from_ocn) + index_o2x = mct_aVect_indexRA(o2x_g, trim(field)) + index_x2g = mct_aVect_indexRA(x2g_g, trim(field)) + + if (first_time) then + mrgstr(mrgstr_index) = subname//'x2g%'//trim(field)//' =' // & + ' = o2x%'//trim(field) + end if + + do n = 1, lsize + x2g_g%rAttr(index_x2g,n) = o2x_g%rAttr(index_o2x,n) + end do + + mrgstr_index = mrgstr_index + 1 + enddo + + !index_lfrac = mct_aVect_indexRA(fractions_g,"lfrac") + !do i = 1, num_flux_fields + + ! call seq_flds_getField(field, i, seq_flds_x2g_fluxes_from_lnd) + ! index_l2x = mct_aVect_indexRA(l2x_g, trim(field)) + ! index_x2g = mct_aVect_indexRA(x2g_g, trim(field)) + + ! if (trim(field) == qice_fieldname) then + + ! if (first_time) then + ! mrgstr(mrgstr_index) = subname//'x2g%'//trim(field)//' =' // & + ! ' = l2x%'//trim(field) + ! end if + + ! ! treat qice as if it were a state variable, with a simple copy. + ! do n = 1, lsize + ! x2g_g%rAttr(index_x2g,n) = l2x_g%rAttr(index_l2x,n) + ! end do + + ! else + ! write(logunit,*) subname,' ERROR: Flux fields other than ', & + ! qice_fieldname, ' currently are not handled in lnd2glc remapping.' + ! write(logunit,*) '(Attempt to handle flux field <', trim(field), '>.)' + ! write(logunit,*) 'Substantial thought is needed to determine how to remap other fluxes' + ! write(logunit,*) 'in a smooth, conservative manner.' + ! call shr_sys_abort(subname//& + ! ' ERROR: Flux fields other than qice currently are not handled in lnd2glc remapping.') + ! endif ! qice_fieldname + + ! mrgstr_index = mrgstr_index + 1 + + !end do + + if (first_time) then + if (iamroot) then + write(logunit,'(A)') subname//' Summary:' + do i = 1,nflds + write(logunit,'(A)') trim(mrgstr(i)) + enddo + endif + deallocate(mrgstr) + endif + + first_time = .false. + + end subroutine prep_glc_merge_ocn_forcing + + !================================================================================================ subroutine prep_glc_mrg_lnd(infodata, fractions_gx, timer_mrg) @@ -604,7 +773,7 @@ subroutine prep_glc_merge_lnd_forcing( l2x_g, fractions_g, x2g_g ) mrgstr_index = 1 do i = 1, num_state_fields - call seq_flds_getField(field, i, seq_flds_x2g_states) + call seq_flds_getField(field, i, seq_flds_x2g_states_from_lnd) index_l2x = mct_aVect_indexRA(l2x_g, trim(field)) index_x2g = mct_aVect_indexRA(x2g_g, trim(field)) @@ -668,13 +837,15 @@ subroutine prep_glc_merge_lnd_forcing( l2x_g, fractions_g, x2g_g ) end subroutine prep_glc_merge_lnd_forcing - subroutine prep_glc_calc_o2x_gx(timer) + subroutine prep_glc_calc_o2x_gx(ocn_c2_glctf, ocn_c2_glcshelf, timer) !--------------------------------------------------------------- ! Description ! Create o2x_gx ! Arguments character(len=*), intent(in) :: timer + logical, intent(in) :: ocn_c2_glctf + logical, intent(in) :: ocn_c2_glcshelf character(*), parameter :: subname = '(prep_glc_calc_o2x_gx)' ! Local Variables @@ -684,8 +855,14 @@ subroutine prep_glc_calc_o2x_gx(timer) call t_drvstartf (trim(timer),barrier=mpicom_CPLID) do eoi = 1,num_inst_ocn o2x_ox => component_get_c2x_cx(ocn(eoi)) - call seq_map_map(mapper_So2g, o2x_ox, o2x_gx(eoi), & - fldlist=seq_flds_x2g_states_from_ocn,norm=.true.) + if (ocn_c2_glctf) then + call seq_map_map(mapper_So2g_tf, o2x_ox, o2x_gx(eoi), & + fldlist=seq_flds_x2g_tf_states_from_ocn,norm=.true.) + end if + if (ocn_c2_glcshelf) then + call seq_map_map(mapper_So2g_shelf, o2x_ox, o2x_gx(eoi), & + fldlist=seq_flds_x2g_shelf_states_from_ocn,norm=.true.) + end if enddo call t_drvstopf (trim(timer)) @@ -850,8 +1027,8 @@ subroutine prep_glc_calculate_subshelf_boundary_fluxes !Done here instead of in glc-frequency mapping so it happens within ocean coupling interval. ! Also could map o2x_ox->o2x_gx(1) but using x2g_gx as destination allows us to see ! these fields on the GLC grid of the coupler history file, which helps with debugging. - call seq_map_map(mapper_So2g, o2x_ox, x2g_gx, & - fldlist=seq_flds_x2g_states_from_ocn,norm=.true.) + call seq_map_map(mapper_So2g_shelf, o2x_ox, x2g_gx, & + fldlist=seq_flds_x2g_shelf_states_from_ocn,norm=.true.) ! inputs to melt flux calculation index_x2g_So_blt = mct_avect_indexra(x2g_gx,'So_blt',perrwith='quiet') @@ -950,6 +1127,7 @@ subroutine prep_glc_zero_fields() type(mct_avect), pointer :: x2g_gx !--------------------------------------------------------------- + do egi = 1,num_inst_glc x2g_gx => component_get_x2c_cx(glc(egi)) call mct_aVect_zero(x2g_gx) @@ -1195,8 +1373,9 @@ subroutine prep_glc_renormalize_smb(eli, fractions_lx, g2x_gx, mapper_Fg2l, area aream_l(:) = dom_l%data%rAttr(km,:) ! Export land fractions from fractions_lx to a local array + ! Note that for E3SM we are using lfrin instead of lfrac allocate(lfrac(lsize_l)) - call mct_aVect_exportRattr(fractions_lx, "lfrac", lfrac) + call mct_aVect_exportRattr(fractions_lx, "lfrin", lfrac) ! Map Sg_icemask from the glc grid to the land grid. ! This may not be necessary, if Sg_icemask_l has already been mapped from Sg_icemask_g. @@ -1379,6 +1558,8 @@ subroutine prep_glc_renormalize_smb(eli, fractions_lx, g2x_gx, mapper_Fg2l, area endif if (iamroot) then + write(logunit,*) 'global_accum_on_land_grid = ', global_accum_on_land_grid + write(logunit,*) 'global_accum_on_glc_grid = ', global_accum_on_glc_grid write(logunit,*) 'accum_renorm_factor = ', accum_renorm_factor write(logunit,*) 'ablat_renorm_factor = ', ablat_renorm_factor endif @@ -1424,6 +1605,11 @@ function prep_glc_get_l2gacc_lx_cnt() prep_glc_get_l2gacc_lx_cnt => l2gacc_lx_cnt end function prep_glc_get_l2gacc_lx_cnt + function prep_glc_get_l2gacc_lx_cnt_avg() + integer, pointer :: prep_glc_get_l2gacc_lx_cnt_avg + prep_glc_get_l2gacc_lx_cnt_avg => l2gacc_lx_cnt_avg + end function prep_glc_get_l2gacc_lx_cnt_avg + function prep_glc_get_o2x_gx() type(mct_aVect), pointer :: prep_glc_get_o2x_gx(:) prep_glc_get_o2x_gx => o2x_gx(:) @@ -1449,15 +1635,15 @@ function prep_glc_get_mapper_Fl2g() prep_glc_get_mapper_Fl2g => mapper_Fl2g end function prep_glc_get_mapper_Fl2g - function prep_glc_get_mapper_So2g() - type(seq_map), pointer :: prep_glc_get_mapper_So2g - prep_glc_get_mapper_So2g=> mapper_So2g - end function prep_glc_get_mapper_So2g + function prep_glc_get_mapper_So2g_shelf() + type(seq_map), pointer :: prep_glc_get_mapper_So2g_shelf + prep_glc_get_mapper_So2g_shelf=> mapper_So2g_shelf + end function prep_glc_get_mapper_So2g_shelf - function prep_glc_get_mapper_Fo2g() - type(seq_map), pointer :: prep_glc_get_mapper_Fo2g - prep_glc_get_mapper_Fo2g=> mapper_Fo2g - end function prep_glc_get_mapper_Fo2g + function prep_glc_get_mapper_Fo2g_shelf() + type(seq_map), pointer :: prep_glc_get_mapper_Fo2g_shelf + prep_glc_get_mapper_Fo2g_shelf=> mapper_Fo2g_shelf + end function prep_glc_get_mapper_Fo2g_shelf !*********************************************************************** ! diff --git a/driver-mct/main/seq_diag_mct.F90 b/driver-mct/main/seq_diag_mct.F90 index 8534008f9be..be9c3a1426c 100644 --- a/driver-mct/main/seq_diag_mct.F90 +++ b/driver-mct/main/seq_diag_mct.F90 @@ -48,6 +48,9 @@ module seq_diag_mct use shr_reprosum_mod, only : shr_reprosum_calc use seq_diagBGC_mct, only : seq_diagBGC_preprint_mct, seq_diagBGC_print_mct + use prep_glc_mod, only : prep_glc_get_l2gacc_lx_cnt_avg + use glc_elevclass_mod, only: glc_get_num_elevation_classes + implicit none save private @@ -140,42 +143,45 @@ module seq_diag_mct integer(in),parameter :: f_hsen =10 ! heat : sensible integer(in),parameter :: f_hpolar =11 ! heat : AIS imbalance integer(in),parameter :: f_hh2ot =12 ! heat : water temperature - integer(in),parameter :: f_wfrz =13 ! water: freezing - integer(in),parameter :: f_wmelt =14 ! water: melting - integer(in),parameter :: f_wrain =15 ! water: precip, liquid - integer(in),parameter :: f_wsnow =16 ! water: precip, frozen - integer(in),parameter :: f_wpolar =17 ! water: AIS imbalance - integer(in),parameter :: f_wevap =18 ! water: evaporation - integer(in),parameter :: f_wroff =19 ! water: runoff/flood - integer(in),parameter :: f_wioff =20 ! water: frozen runoff - integer(in),parameter :: f_wirrig =21 ! water: irrigation - integer(in),parameter :: f_wfrz_16O =22 ! water: freezing - integer(in),parameter :: f_wmelt_16O =23 ! water: melting - integer(in),parameter :: f_wrain_16O =24 ! water: precip, liquid - integer(in),parameter :: f_wsnow_16O =25 ! water: precip, frozen - integer(in),parameter :: f_wevap_16O =26 ! water: evaporation - integer(in),parameter :: f_wroff_16O =27 ! water: runoff/flood - integer(in),parameter :: f_wioff_16O =28 ! water: frozen runoff - integer(in),parameter :: f_wfrz_18O =29 ! water: freezing - integer(in),parameter :: f_wmelt_18O =30 ! water: melting - integer(in),parameter :: f_wrain_18O =31 ! water: precip, liquid - integer(in),parameter :: f_wsnow_18O =32 ! water: precip, frozen - integer(in),parameter :: f_wevap_18O =33 ! water: evaporation - integer(in),parameter :: f_wroff_18O =34 ! water: runoff/flood - integer(in),parameter :: f_wioff_18O =35 ! water: frozen runoff - integer(in),parameter :: f_wfrz_HDO =36 ! water: freezing - integer(in),parameter :: f_wmelt_HDO =37 ! water: melting - integer(in),parameter :: f_wrain_HDO =38 ! water: precip, liquid - integer(in),parameter :: f_wsnow_HDO =39 ! water: precip, frozen - integer(in),parameter :: f_wevap_HDO =40 ! water: evaporation - integer(in),parameter :: f_wroff_HDO =41 ! water: runoff/flood - integer(in),parameter :: f_wioff_HDO =42 ! water: frozen runoff + integer(in),parameter :: f_hgsmb =13 ! heat : Greenland ice sheet surface mass balance + integer(in),parameter :: f_wfrz =14 ! water: freezing + integer(in),parameter :: f_wmelt =15 ! water: melting + integer(in),parameter :: f_wrain =16 ! water: precip, liquid + integer(in),parameter :: f_wsnow =17 ! water: precip, frozen + integer(in),parameter :: f_wpolar =18 ! water: AIS imbalance + integer(in),parameter :: f_wgsmb =19 ! water: Greenland ice sheet surface mass balance + integer(in),parameter :: f_wevap =20 ! water: evaporation + integer(in),parameter :: f_wroff =21 ! water: runoff/flood + integer(in),parameter :: f_wioff =22 ! water: frozen runoff + integer(in),parameter :: f_wirrig =23 ! water: irrigation + integer(in),parameter :: f_wfrz_16O =24 ! water: freezing + integer(in),parameter :: f_wmelt_16O =25 ! water: melting + integer(in),parameter :: f_wrain_16O =26 ! water: precip, liquid + integer(in),parameter :: f_wsnow_16O =27 ! water: precip, frozen + integer(in),parameter :: f_wevap_16O =28 ! water: evaporation + integer(in),parameter :: f_wroff_16O =29 ! water: runoff/flood + integer(in),parameter :: f_wioff_16O =30 ! water: frozen runoff + integer(in),parameter :: f_wfrz_18O =31 ! water: freezing + integer(in),parameter :: f_wmelt_18O =32 ! water: melting + integer(in),parameter :: f_wrain_18O =33 ! water: precip, liquid + integer(in),parameter :: f_wsnow_18O =34 ! water: precip, frozen + integer(in),parameter :: f_wevap_18O =35 ! water: evaporation + integer(in),parameter :: f_wroff_18O =36 ! water: runoff/flood + integer(in),parameter :: f_wioff_18O =37 ! water: frozen runoff + integer(in),parameter :: f_wfrz_HDO =38 ! water: freezing + integer(in),parameter :: f_wmelt_HDO =39 ! water: melting + integer(in),parameter :: f_wrain_HDO =40 ! water: precip, liquid + integer(in),parameter :: f_wsnow_HDO =41 ! water: precip, frozen + integer(in),parameter :: f_wevap_HDO =42 ! water: evaporation + integer(in),parameter :: f_wroff_HDO =43 ! water: runoff/flood + integer(in),parameter :: f_wioff_HDO =44 ! water: frozen runoff integer(in),parameter :: f_size = f_wioff_HDO ! Total array size of all elements integer(in),parameter :: f_a = f_area ! 1st index for area integer(in),parameter :: f_a_end = f_area ! last index for area integer(in),parameter :: f_h = f_hfrz ! 1st index for heat - integer(in),parameter :: f_h_end = f_hh2ot ! Last index for heat + !integer(in),parameter :: f_h_end = f_hh2ot ! Last index for heat + integer(in),parameter :: f_h_end = f_hgsmb ! Last index for heat integer(in),parameter :: f_w = f_wfrz ! 1st index for water integer(in),parameter :: f_w_end = f_wirrig ! Last index for water integer(in),parameter :: f_16O = f_wfrz_16O ! 1st index for 16O water isotope @@ -189,8 +195,10 @@ module seq_diag_mct (/' area',' hfreeze',' hmelt',' hnetsw',' hlwdn', & ' hlwup',' hlatvap',' hlatfus',' hiroff',' hsen', & - ' hpolar',' hh2otemp',' wfreeze',' wmelt',' wrain', & - ' wsnow',' wpolar',' wevap',' wrunoff',' wfrzrof', & +! ' hpolar',' hh2otemp',' wfreeze',' wmelt',' wrain', & + ' hpolar',' hh2otemp',' hgsmb',' wfreeze',' wmelt',' wrain', & +! ' wsnow',' wpolar',' wevap',' wrunoff',' wfrzrof', & + ' wsnow',' wpolar',' wgsmb',' wevap',' wrunoff',' wfrzrof', & ' wirrig', & ' wfreeze_16O',' wmelt_16O',' wrain_16O',' wsnow_16O', & ' wevap_16O',' wrunoff_16O',' wfrzrof_16O', & @@ -262,6 +270,9 @@ module seq_diag_mct integer :: index_l2x_Flrl_irrig integer :: index_l2x_Flrl_wslake + integer :: index_x2l_Sg_icemask + integer, allocatable :: index_l2x_Flgl_qice(:) + integer, allocatable :: index_x2l_Sg_ice_covered(:) integer :: index_x2l_Faxa_lwdn integer :: index_x2l_Faxa_rainc @@ -338,6 +349,9 @@ module seq_diag_mct integer :: index_g2x_Fogg_rofi integer :: index_g2x_Figg_rofi + integer :: index_x2g_Flgl_qice + integer :: index_g2x_Sg_icemask + integer :: index_x2o_Foxx_rofl_16O integer :: index_x2o_Foxx_rofi_16O integer :: index_x2o_Foxx_rofl_18O @@ -434,6 +448,9 @@ module seq_diag_mct integer :: index_x2i_Faxa_snow_18O integer :: index_x2i_Faxa_snow_HDO + integer :: glc_nec + integer :: l2gacc_lx_cnt_avg + !=============================================================================== contains !=============================================================================== @@ -866,7 +883,7 @@ subroutine seq_diag_lnd_mct( lnd, frac_l, infodata, do_l2x, do_x2l) type(mct_aVect), pointer :: l2x_l ! model to drv bundle type(mct_aVect), pointer :: x2l_l ! drv to model bundle type(mct_ggrid), pointer :: dom_l - integer(in) :: n,ic,nf,ip ! generic index + integer(in) :: n,ic,nf,ip ! generic index integer(in) :: kArea ! index of area field in aVect integer(in) :: kl ! fraction indices integer(in) :: lSize ! size of aVect @@ -874,6 +891,13 @@ subroutine seq_diag_lnd_mct( lnd, frac_l, infodata, do_l2x, do_x2l) logical,save :: first_time = .true. logical,save :: flds_wiso_lnd = .false. + real(r8) :: l2x_Flgl_qice_col_sum ! for summing fluxes over no. of elev. classes + real(r8) :: effective_area + + character(len=64) :: name + character(len= 2) :: cnum + integer(in) :: num + !----- formats ----- character(*),parameter :: subName = '(seq_diag_lnd_mct) ' @@ -894,6 +918,15 @@ subroutine seq_diag_lnd_mct( lnd, frac_l, infodata, do_l2x, do_x2l) kArea = mct_aVect_indexRA(dom_l%data,afldname) kl = mct_aVect_indexRA(frac_l,lfrinname) + ! get number of elevation classes and allocate relevant sets of indices + glc_nec = glc_get_num_elevation_classes() + if (glc_nec.ge.1) then + if (first_time) then + allocate(index_l2x_Flgl_qice(0:glc_nec)) + allocate(index_x2l_Sg_ice_covered(0:glc_nec)) + end if + end if + if (present(do_l2x)) then if (first_time) then index_l2x_Fall_swnet = mct_aVect_indexRA(l2x_l,'Fall_swnet') @@ -909,6 +942,17 @@ subroutine seq_diag_lnd_mct( lnd, frac_l, infodata, do_l2x, do_x2l) index_l2x_Flrl_irrig = mct_aVect_indexRA(l2x_l,'Flrl_irrig', perrWith='quiet') index_l2x_Flrl_wslake = mct_aVect_indexRA(l2x_l,'Flrl_wslake') + if (glc_nec.ge.1) then + index_x2l_Sg_icemask = mct_avect_indexRA(x2l_l,'Sg_icemask') + do num=0,glc_nec + write(cnum,'(i2.2)') num + name = 'Flgl_qice' // cnum + index_l2x_Flgl_qice(num) = mct_avect_indexRA(l2x_l,trim(name)) + name = 'Sg_ice_covered' // cnum + index_x2l_Sg_ice_covered(num) = mct_avect_indexRA(x2l_l,trim(name)) + end do + end if + index_l2x_Fall_evap_16O = mct_aVect_indexRA(l2x_l,'Fall_evap_16O',perrWith='quiet') if ( index_l2x_Fall_evap_16O /= 0 ) flds_wiso_lnd = .true. if ( flds_wiso_lnd )then @@ -942,7 +986,19 @@ subroutine seq_diag_lnd_mct( lnd, frac_l, infodata, do_l2x, do_x2l) if (index_l2x_Flrl_irrig /= 0) then nf = f_wroff ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - ca_l*l2x_l%rAttr(index_l2x_Flrl_irrig,n) end if - nf = f_wioff ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - ca_l*l2x_l%rAttr(index_l2x_Flrl_rofi,n) + nf = f_wioff ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - ca_l*l2x_l%rAttr(index_l2x_Flrl_rofi,n) ! contribution from land ice calving currently zero + + l2x_Flgl_qice_col_sum = 0.0d0 + if (glc_nec.ge.1) then + effective_area = min(frac_l%rAttr(kl,n),x2l_l%rAttr(index_x2l_Sg_icemask,n)) * dom_l%data%rAttr(kArea,n) + do num=0,glc_nec + ! sums the contributions from fluxes in each set of elevation classes + ! RHS product is flux times fraction of area in specific elevation class times land cell area + l2x_Flgl_qice_col_sum = l2x_Flgl_qice_col_sum + l2x_l%rAttr(index_l2x_Flgl_qice(num),n) * & + x2l_l%rAttr(index_x2l_Sg_ice_covered(num),n) * effective_area + end do + end if + nf = f_wgsmb ; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - l2x_Flgl_qice_col_sum if ( flds_wiso_lnd )then nf = f_wevap_16O; @@ -976,7 +1032,14 @@ subroutine seq_diag_lnd_mct( lnd, frac_l, infodata, do_l2x, do_x2l) ca_l*l2x_l%rAttr(index_l2x_Flrl_rofi_HDO,n) end if end do - budg_dataL(f_hioff,ic,ip) = -budg_dataL(f_wioff,ic,ip)*shr_const_latice + + budg_dataL(f_hioff,ic,ip) = -budg_dataL(f_wioff,ic,ip)*shr_const_latice ! contribution from land ice calving currently zero + budg_dataL(f_hgsmb,ic,ip) = budg_dataL(f_wgsmb,ic,ip)*shr_const_latice + + ! Nneeded? Not sure if / when these should be deallocated + !deallocate(index_l2x_Flgl_qice(0:glc_nec)) + !deallocate(index_x2l_Sg_ice_covered(0:glc_nec)) + end if if (present(do_x2l)) then @@ -1252,15 +1315,18 @@ end subroutine seq_diag_rof_mct ! Compute global glc input/output flux diagnostics ! ! !REVISION HISTORY: - ! 2008-jul-10 - T. Craig - update + ! 2008-Jul-10 - T. Craig - update + ! 2024-Dec-06 - S. Price, J. Wolfe - update ! ! !INTERFACE: ------------------------------------------------------------------ - subroutine seq_diag_glc_mct( glc, frac_g, infodata) + subroutine seq_diag_glc_mct( glc, frac_g, infodata, do_x2g, do_g2x ) type(component_type) , intent(in) :: glc ! component type for instance1 - type(mct_aVect) , intent(in) :: frac_g ! frac bundle + type(mct_aVect) , intent(in) :: frac_g ! frac bundle (may not be used / needed here) type(seq_infodata_type) , intent(in) :: infodata + logical , intent(in), optional :: do_x2g + logical , intent(in), optional :: do_g2x !EOP @@ -1269,11 +1335,13 @@ subroutine seq_diag_glc_mct( glc, frac_g, infodata) type(mct_aVect), pointer :: x2g_g type(mct_ggrid), pointer :: dom_g integer(in) :: n,ic,nf,ip ! generic index - integer(in) :: kArea ! index of area field in aVect - integer(in) :: lSize ! size of aVect - real(r8) :: ca_g ! area of a grid cell + integer(in) :: kArea ! index of area field in aVect + integer(in) :: lSize ! size of aVect + real(r8) :: ca_g ! area of a grid cell logical,save :: first_time = .true. + integer,save :: smb_vector_length,calving_vector_length + !----- formats ----- character(*),parameter :: subName = '(seq_diag_glc_mct) ' @@ -1289,23 +1357,61 @@ subroutine seq_diag_glc_mct( glc, frac_g, infodata) g2x_g => component_get_c2x_cx(glc) x2g_g => component_get_x2c_cx(glc) - if (first_time) then - index_g2x_Fogg_rofl = mct_aVect_indexRA(g2x_g,'Fogg_rofl') - index_g2x_Fogg_rofi = mct_aVect_indexRA(g2x_g,'Fogg_rofi') - index_g2x_Figg_rofi = mct_aVect_indexRA(g2x_g,'Figg_rofi') - end if - ip = p_inst - ic = c_glc_gs - kArea = mct_aVect_indexRA(dom_g%data,afldname) - lSize = mct_avect_lSize(g2x_g) - do n=1,lSize - ca_g = dom_g%data%rAttr(kArea,n) - nf = f_wroff; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - ca_g*g2x_g%rAttr(index_g2x_Fogg_rofl,n) - nf = f_wioff; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - ca_g*g2x_g%rAttr(index_g2x_Fogg_rofi,n) & - - ca_g*g2x_g%rAttr(index_g2x_Figg_rofi,n) - end do - budg_dataL(f_hioff,ic,ip) = -budg_dataL(f_wioff,ic,ip)*shr_const_latice + + if( present(do_g2x))then ! do fields from glc to coupler (g2x_) + + if (first_time) then + + calving_vector_length = 0 + + index_g2x_Fogg_rofl = mct_aVect_indexRA(g2x_g,'Fogg_rofl') + index_g2x_Fogg_rofi = mct_aVect_indexRA(g2x_g,'Fogg_rofi') + index_g2x_Figg_rofi = mct_aVect_indexRA(g2x_g,'Figg_rofi') + + end if + + ic = c_glc_gr + kArea = mct_aVect_indexRA(dom_g%data,afldname) + lSize = mct_avect_lSize(g2x_g) + + do n=1,lSize + ca_g = dom_g%data%rAttr(kArea,n) + nf = f_wroff; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - ca_g*g2x_g%rAttr(index_g2x_Fogg_rofl,n) + nf = f_wioff; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) - ca_g*g2x_g%rAttr(index_g2x_Fogg_rofi,n) & + - ca_g*g2x_g%rAttr(index_g2x_Figg_rofi,n) + end do + + budg_dataL(f_hioff,ic,ip) = -budg_dataL(f_wioff,ic,ip)*shr_const_latice + + endif ! end do fields from glc to coupler (g2x_) + + if( present(do_x2g))then ! do fields from coupler to glc (x2g_) + + if (first_time) then + + index_x2g_Flgl_qice = mct_aVect_indexRA(x2g_g,'Flgl_qice') + index_g2x_Sg_icemask = mct_avect_indexRA(g2x_g,'Sg_icemask') + + end if + + l2gacc_lx_cnt_avg = prep_glc_get_l2gacc_lx_cnt_avg() ! counter for how many times SMB flux accumulation has occured + ic = c_glc_gs + kArea = mct_aVect_indexRA(dom_g%data,afldname) + lSize = mct_avect_lSize(x2g_g) + + do n=1,lSize + ca_g = dom_g%data%rAttr(kArea,n)*g2x_g%rAttr(index_g2x_Sg_icemask,n) + nf = f_wgsmb; budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) + ca_g*x2g_g%rAttr(index_x2g_Flgl_qice,n) + end do + + budg_dataL(nf,ic,ip) = budg_dataL(nf,ic,ip) * l2gacc_lx_cnt_avg + + budg_dataL(f_hgsmb,ic,ip) = budg_dataL(f_wgsmb,ic,ip)*shr_const_latice + + smb_vector_length = smb_vector_length +lSize + + end if ! end do fields from coupler to glc (x2g_) first_time = .false. diff --git a/driver-mct/main/seq_rest_mod.F90 b/driver-mct/main/seq_rest_mod.F90 index 0ad62de966f..53660278331 100644 --- a/driver-mct/main/seq_rest_mod.F90 +++ b/driver-mct/main/seq_rest_mod.F90 @@ -519,6 +519,8 @@ subroutine seq_rest_write(EClock_d, seq_SyncClock, infodata, & call seq_io_write(rest_file,rvar,'seq_infodata_precip_fact',whead=whead,wdata=wdata) call seq_infodata_GetData(infodata,case_name=cvar) call seq_io_write(rest_file,trim(cvar),'seq_infodata_case_name',whead=whead,wdata=wdata) + call seq_infodata_GetData(infodata,rmean_rmv_ice_runoff=rvar) + call seq_io_write(rest_file,rvar,'seq_infodata_rmean_rmv_ice_runoff',whead=whead,wdata=wdata) call seq_timemgr_EClockGetData( EClock_d, start_ymd=ivar) call seq_io_write(rest_file,ivar,'seq_timemgr_start_ymd',whead=whead,wdata=wdata) diff --git a/driver-mct/shr/seq_flds_mod.F90 b/driver-mct/shr/seq_flds_mod.F90 index dbfba0889d0..d0f46ed36e2 100644 --- a/driver-mct/shr/seq_flds_mod.F90 +++ b/driver-mct/shr/seq_flds_mod.F90 @@ -212,7 +212,8 @@ module seq_flds_mod character(CXX) :: seq_flds_g2o_ice_fluxes character(CXX) :: seq_flds_x2g_states character(CXX) :: seq_flds_x2g_states_from_lnd - character(CXX) :: seq_flds_x2g_states_from_ocn + character(CXX) :: seq_flds_x2g_shelf_states_from_ocn + character(CXX) :: seq_flds_x2g_tf_states_from_ocn character(CXX) :: seq_flds_x2g_fluxes character(CXX) :: seq_flds_x2g_fluxes_from_lnd @@ -347,7 +348,8 @@ subroutine seq_flds_set(nmlfile, ID, infodata) character(CXX) :: g2o_ice_fluxes = '' character(CXX) :: x2g_states = '' character(CXX) :: x2g_states_from_lnd = '' - character(CXX) :: x2g_states_from_ocn = '' + character(CXX) :: x2g_shelf_states_from_ocn = '' + character(CXX) :: x2g_tf_states_from_ocn = '' character(CXX) :: x2g_fluxes = '' character(CXX) :: x2g_fluxes_from_lnd = '' character(CXX) :: xao_albedo = '' @@ -380,11 +382,12 @@ subroutine seq_flds_set(nmlfile, ID, infodata) logical :: flds_bgc_oi logical :: flds_wiso logical :: flds_polar + logical :: flds_tf integer :: glc_nec namelist /seq_cplflds_inparm/ & - flds_co2a, flds_co2b, flds_co2c, flds_co2_dmsa, flds_wiso, flds_polar, glc_nec, & - ice_ncat, seq_flds_i2o_per_cat, flds_bgc_oi, & + flds_co2a, flds_co2b, flds_co2c, flds_co2_dmsa, flds_wiso, flds_polar, flds_tf, & + glc_nec, ice_ncat, seq_flds_i2o_per_cat, flds_bgc_oi, & nan_check_component_fields, rof_heat, atm_flux_method, atm_gustiness, & rof2ocn_nutrients, lnd_rof_two_way, ocn_rof_two_way, rof_sed @@ -418,6 +421,7 @@ subroutine seq_flds_set(nmlfile, ID, infodata) flds_bgc_oi = .false. flds_wiso = .false. flds_polar = .false. + flds_tf = .false. glc_nec = 0 ice_ncat = 1 seq_flds_i2o_per_cat = .false. @@ -452,6 +456,7 @@ subroutine seq_flds_set(nmlfile, ID, infodata) call shr_mpi_bcast(flds_bgc_oi , mpicom) call shr_mpi_bcast(flds_wiso , mpicom) call shr_mpi_bcast(flds_polar , mpicom) + call shr_mpi_bcast(flds_tf , mpicom) call shr_mpi_bcast(glc_nec , mpicom) call shr_mpi_bcast(ice_ncat , mpicom) call shr_mpi_bcast(seq_flds_i2o_per_cat, mpicom) @@ -2938,7 +2943,7 @@ subroutine seq_flds_set(nmlfile, ID, infodata) name = 'So_blt' call seq_flds_add(o2x_states,trim(name)) call seq_flds_add(x2g_states,trim(name)) - call seq_flds_add(x2g_states_from_ocn,trim(name)) + call seq_flds_add(x2g_shelf_states_from_ocn,trim(name)) longname = 'Ice shelf boundary layer ocean temperature' stdname = 'Ice_shelf_boundary_layer_ocean_temperature' units = 'C' @@ -2948,7 +2953,7 @@ subroutine seq_flds_set(nmlfile, ID, infodata) name = 'So_bls' call seq_flds_add(o2x_states,trim(name)) call seq_flds_add(x2g_states,trim(name)) - call seq_flds_add(x2g_states_from_ocn,trim(name)) + call seq_flds_add(x2g_shelf_states_from_ocn,trim(name)) longname = 'Ice shelf boundary layer ocean salinity' stdname = 'Ice_shelf_boundary_layer_ocean_salinity' units = 'psu' @@ -2958,7 +2963,7 @@ subroutine seq_flds_set(nmlfile, ID, infodata) name = 'So_htv' call seq_flds_add(o2x_states,trim(name)) call seq_flds_add(x2g_states,trim(name)) - call seq_flds_add(x2g_states_from_ocn,trim(name)) + call seq_flds_add(x2g_shelf_states_from_ocn,trim(name)) longname = 'Ice shelf ocean heat transfer velocity' stdname = 'Ice_shelf_ocean_heat_transfer_velocity' units = 'm/s' @@ -2968,7 +2973,7 @@ subroutine seq_flds_set(nmlfile, ID, infodata) name = 'So_stv' call seq_flds_add(o2x_states,trim(name)) call seq_flds_add(x2g_states,trim(name)) - call seq_flds_add(x2g_states_from_ocn,trim(name)) + call seq_flds_add(x2g_shelf_states_from_ocn,trim(name)) longname = 'Ice shelf ocean salinity transfer velocity' stdname = 'Ice_shelf_ocean_salinity_transfer_velocity' units = 'm/s' @@ -2978,13 +2983,27 @@ subroutine seq_flds_set(nmlfile, ID, infodata) name = 'So_rhoeff' call seq_flds_add(o2x_states,trim(name)) call seq_flds_add(x2g_states,trim(name)) - call seq_flds_add(x2g_states_from_ocn,trim(name)) + call seq_flds_add(x2g_shelf_states_from_ocn,trim(name)) longname = 'Ocean effective pressure' stdname = 'Ocean_effective_pressure' units = 'Pa' attname = 'So_rhoeff' call metadata_set(attname, longname, stdname, units) + if (flds_tf) then + + name = 'So_tf2d' + call seq_flds_add(o2x_states,trim(name)) + call seq_flds_add(x2g_states,trim(name)) + call seq_flds_add(x2g_tf_states_from_ocn,trim(name)) + longname = 'ocean thermal forcing at predefined critical depth' + stdname = 'ocean_thermal_forcing_at_critical_depth' + units = 'C' + attname = name + call metadata_set(attname, longname, stdname, units) + + end if + name = 'Fogx_qicelo' call seq_flds_add(g2x_fluxes,trim(name)) call seq_flds_add(x2o_fluxes,trim(name)) @@ -3936,7 +3955,8 @@ subroutine seq_flds_set(nmlfile, ID, infodata) seq_flds_g2x_states_to_lnd = trim(g2x_states_to_lnd) seq_flds_x2g_states = trim(x2g_states) seq_flds_x2g_states_from_lnd = trim(x2g_states_from_lnd) - seq_flds_x2g_states_from_ocn = trim(x2g_states_from_ocn) + seq_flds_x2g_shelf_states_from_ocn = trim(x2g_shelf_states_from_ocn) + seq_flds_x2g_tf_states_from_ocn = trim(x2g_tf_states_from_ocn) seq_flds_xao_states = trim(xao_states) seq_flds_xao_albedo = trim(xao_albedo) seq_flds_xao_diurnl = trim(xao_diurnl) @@ -4003,7 +4023,8 @@ subroutine seq_flds_set(nmlfile, ID, infodata) write(logunit,*) subname//': seq_flds_x2g_states= ',trim(seq_flds_x2g_states) write(logunit,*) subname//': seq_flds_x2g_states_from_lnd= ',trim(seq_flds_x2g_states_from_lnd) write(logunit,*) subname//': seq_flds_l2x_states_to_glc= ',trim(seq_flds_l2x_states_to_glc) - write(logunit,*) subname//': seq_flds_x2g_states_from_ocn= ',trim(seq_flds_x2g_states_from_ocn) + write(logunit,*) subname//': seq_flds_x2g_shelf_states_from_ocn= ',trim(seq_flds_x2g_shelf_states_from_ocn) + write(logunit,*) subname//': seq_flds_x2g_tf_states_from_ocn= ',trim(seq_flds_x2g_tf_states_from_ocn) write(logunit,*) subname//': seq_flds_x2g_fluxes= ',trim(seq_flds_x2g_fluxes) write(logunit,*) subname//': seq_flds_x2g_fluxes_from_lnd= ',trim(seq_flds_x2g_fluxes_from_lnd) write(logunit,*) subname//': seq_flds_l2x_fluxes_to_glc= ',trim(seq_flds_l2x_fluxes_to_glc) diff --git a/driver-mct/shr/seq_infodata_mod.F90 b/driver-mct/shr/seq_infodata_mod.F90 index fcc6a21eef1..be6810b613e 100644 --- a/driver-mct/shr/seq_infodata_mod.F90 +++ b/driver-mct/shr/seq_infodata_mod.F90 @@ -203,6 +203,7 @@ MODULE seq_infodata_mod logical :: ocn_prognostic ! does component model need input data from driver logical :: ocnrof_prognostic ! does component need rof data logical :: ocn_c2_glcshelf ! will ocn component send data for ice shelf fluxes in driver + logical :: ocn_c2_glctf ! will ocn component send data for thermal forcing in driver logical :: ice_present ! does component model exist logical :: ice_prognostic ! does component model need input data from driver logical :: iceberg_prognostic ! does the ice model support icebergs @@ -250,7 +251,8 @@ MODULE seq_infodata_mod integer(SHR_KIND_IN) :: iac_phase ! iac phase logical :: atm_aero ! atmosphere aerosols logical :: glc_g2lupdate ! update glc2lnd fields in lnd model - real(shr_kind_r8) :: max_cplstep_time ! abort if cplstep time exceeds this value + real(SHR_KIND_R8) :: max_cplstep_time ! abort if cplstep time exceeds this value + real(SHR_KIND_R8) :: rmean_rmv_ice_runoff ! running mean of removed Antarctic ice runoff !--- set from restart file --- character(SHR_KIND_CL) :: rest_case_name ! Short case identification !--- set by driver and may be time varying @@ -761,10 +763,11 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid, cpl_tag) infodata%atm_prognostic = .false. infodata%lnd_prognostic = .false. infodata%rof_prognostic = .false. - infodata%rofocn_prognostic = .false. + infodata%rofocn_prognostic = .false. infodata%ocn_prognostic = .false. infodata%ocnrof_prognostic = .false. infodata%ocn_c2_glcshelf = .false. + infodata%ocn_c2_glctf = .false. infodata%ice_prognostic = .false. infodata%glc_prognostic = .false. ! It's safest to assume glc_coupled_fluxes = .true. if it's not set elsewhere, @@ -808,6 +811,7 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid, cpl_tag) infodata%atm_aero = .false. infodata%glc_g2lupdate = .false. infodata%glc_valid_input = .true. + infodata%rmean_rmv_ice_runoff = -1.0_SHR_KIND_R8 infodata%max_cplstep_time = max_cplstep_time infodata%model_doi_url = model_doi_url @@ -907,11 +911,13 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid, cpl_tag) call seq_io_read(infodata%restart_file,pioid,infodata%nextsw_cday ,'seq_infodata_nextsw_cday') call seq_io_read(infodata%restart_file,pioid,infodata%precip_fact ,'seq_infodata_precip_fact') call seq_io_read(infodata%restart_file,pioid,infodata%rest_case_name,'seq_infodata_case_name') + call seq_io_read(infodata%restart_file,pioid,infodata%rmean_rmv_ice_runoff,'seq_infodata_rmean_rmv_ice_runoff') endif !--- Send from CPLID ROOT to GLOBALID ROOT, use bcast as surrogate call shr_mpi_bcast(infodata%nextsw_cday,mpicom,pebcast=seq_comm_gloroot(CPLID)) call shr_mpi_bcast(infodata%precip_fact,mpicom,pebcast=seq_comm_gloroot(CPLID)) call shr_mpi_bcast(infodata%rest_case_name,mpicom,pebcast=seq_comm_gloroot(CPLID)) + call shr_mpi_bcast(infodata%rmean_rmv_ice_runoff,mpicom,pebcast=seq_comm_gloroot(CPLID)) endif if (seq_comm_iamroot(ID)) then @@ -1004,7 +1010,8 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ atm_present, atm_prognostic, & lnd_present, lnd_prognostic, & rof_present, rof_prognostic, rofocn_prognostic, & - ocn_present, ocn_prognostic, ocnrof_prognostic, ocn_c2_glcshelf, & + ocn_present, ocn_prognostic, ocnrof_prognostic, & + ocn_c2_glcshelf, ocn_c2_glctf, & ice_present, ice_prognostic, & glc_present, glc_prognostic, & iac_present, iac_prognostic, & @@ -1041,7 +1048,8 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ reprosum_use_ddpdd, reprosum_allow_infnan, & reprosum_diffmax, reprosum_recompute, & mct_usealltoall, mct_usevector, max_cplstep_time, model_doi_url, & - glc_valid_input, nlmaps_verbosity, nlmaps_exclude_fields) + glc_valid_input, nlmaps_verbosity, nlmaps_exclude_fields, & + rmean_rmv_ice_runoff) implicit none @@ -1179,6 +1187,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ logical, optional, intent(OUT) :: ocn_prognostic logical, optional, intent(OUT) :: ocnrof_prognostic logical, optional, intent(OUT) :: ocn_c2_glcshelf + logical, optional, intent(OUT) :: ocn_c2_glctf logical, optional, intent(OUT) :: ice_present logical, optional, intent(OUT) :: ice_prognostic logical, optional, intent(OUT) :: iceberg_prognostic @@ -1228,6 +1237,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ real(shr_kind_r8), optional, intent(out) :: max_cplstep_time character(SHR_KIND_CL), optional, intent(OUT) :: model_doi_url logical, optional, intent(OUT) :: glc_valid_input + real(SHR_KIND_R8), optional, intent(out) :: rmean_rmv_ice_runoff !----- local ----- character(len=*), parameter :: subname = '(seq_infodata_GetData_explicit) ' @@ -1365,6 +1375,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ if ( present(ocn_prognostic) ) ocn_prognostic = infodata%ocn_prognostic if ( present(ocnrof_prognostic) ) ocnrof_prognostic = infodata%ocnrof_prognostic if ( present(ocn_c2_glcshelf) ) ocn_c2_glcshelf = infodata%ocn_c2_glcshelf + if ( present(ocn_c2_glctf) ) ocn_c2_glctf = infodata%ocn_c2_glctf if ( present(ice_present) ) ice_present = infodata%ice_present if ( present(ice_prognostic) ) ice_prognostic = infodata%ice_prognostic if ( present(iceberg_prognostic)) iceberg_prognostic = infodata%iceberg_prognostic @@ -1427,6 +1438,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ if ( present(model_doi_url) ) model_doi_url = infodata%model_doi_url if ( present(glc_valid_input)) glc_valid_input = infodata%glc_valid_input + if ( present(rmean_rmv_ice_runoff) ) rmean_rmv_ice_runoff = infodata%rmean_rmv_ice_runoff END SUBROUTINE seq_infodata_GetData_explicit @@ -1557,7 +1569,8 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ atm_present, atm_prognostic, & lnd_present, lnd_prognostic, & rof_present, rof_prognostic, rofocn_prognostic, & - ocn_present, ocn_prognostic, ocnrof_prognostic, ocn_c2_glcshelf, & + ocn_present, ocn_prognostic, ocnrof_prognostic, & + ocn_c2_glcshelf, ocn_c2_glctf, & ice_present, ice_prognostic, & glc_present, glc_prognostic, & glc_coupled_fluxes, & @@ -1595,7 +1608,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ reprosum_use_ddpdd, reprosum_allow_infnan, & reprosum_diffmax, reprosum_recompute, & mct_usealltoall, mct_usevector, glc_valid_input, & - nlmaps_verbosity, nlmaps_exclude_fields) + nlmaps_verbosity, nlmaps_exclude_fields, rmean_rmv_ice_runoff) implicit none @@ -1732,6 +1745,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ logical, optional, intent(IN) :: ocn_prognostic logical, optional, intent(IN) :: ocnrof_prognostic logical, optional, intent(IN) :: ocn_c2_glcshelf + logical, optional, intent(IN) :: ocn_c2_glctf logical, optional, intent(IN) :: ice_present logical, optional, intent(IN) :: ice_prognostic logical, optional, intent(IN) :: iceberg_prognostic @@ -1778,6 +1792,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ logical, optional, intent(IN) :: atm_aero ! atm aerosols logical, optional, intent(IN) :: glc_g2lupdate ! update glc2lnd fields in lnd model logical, optional, intent(IN) :: glc_valid_input + real(SHR_KIND_R8), optional, intent(IN) :: rmean_rmv_ice_runoff ! running mean of removed Antarctic ice runoff !EOP @@ -1917,6 +1932,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ if ( present(ocn_prognostic) ) infodata%ocn_prognostic = ocn_prognostic if ( present(ocnrof_prognostic)) infodata%ocnrof_prognostic = ocnrof_prognostic if ( present(ocn_c2_glcshelf)) infodata%ocn_c2_glcshelf = ocn_c2_glcshelf + if ( present(ocn_c2_glctf)) infodata%ocn_c2_glctf = ocn_c2_glctf if ( present(ice_present) ) infodata%ice_present = ice_present if ( present(ice_prognostic) ) infodata%ice_prognostic = ice_prognostic if ( present(iceberg_prognostic)) infodata%iceberg_prognostic = iceberg_prognostic @@ -1963,6 +1979,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ if ( present(atm_aero) ) infodata%atm_aero = atm_aero if ( present(glc_g2lupdate) ) infodata%glc_g2lupdate = glc_g2lupdate if ( present(glc_valid_input) ) infodata%glc_valid_input = glc_valid_input + if ( present(rmean_rmv_ice_runoff) ) infodata%rmean_rmv_ice_runoff = rmean_rmv_ice_runoff END SUBROUTINE seq_infodata_PutData_explicit @@ -2229,6 +2246,7 @@ subroutine seq_infodata_bcast(infodata,mpicom) call shr_mpi_bcast(infodata%ocn_prognostic, mpicom) call shr_mpi_bcast(infodata%ocnrof_prognostic, mpicom) call shr_mpi_bcast(infodata%ocn_c2_glcshelf, mpicom) + call shr_mpi_bcast(infodata%ocn_c2_glctf, mpicom) call shr_mpi_bcast(infodata%ice_present, mpicom) call shr_mpi_bcast(infodata%ice_prognostic, mpicom) call shr_mpi_bcast(infodata%iceberg_prognostic, mpicom) @@ -2277,6 +2295,7 @@ subroutine seq_infodata_bcast(infodata,mpicom) call shr_mpi_bcast(infodata%glc_valid_input, mpicom) call shr_mpi_bcast(infodata%model_doi_url, mpicom) call shr_mpi_bcast(infodata%constant_zenith_deg, mpicom) + call shr_mpi_bcast(infodata%rmean_rmv_ice_runoff, mpicom) end subroutine seq_infodata_bcast @@ -2515,6 +2534,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) call shr_mpi_bcast(infodata%ocn_prognostic, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ocnrof_prognostic, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ocn_c2_glcshelf, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%ocn_c2_glctf, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ocn_nx, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ocn_ny, mpicom, pebcast=cmppe) ! dead_comps is true if it's ever set to true @@ -2591,6 +2611,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) call shr_mpi_bcast(infodata%ocn_prognostic, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%ocnrof_prognostic, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%ocn_c2_glcshelf, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%ocn_c2_glctf, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%ice_present, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%ice_prognostic, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%iceberg_prognostic, mpicom, pebcast=cplpe) @@ -2617,6 +2638,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) if (ocn2cplr) then call shr_mpi_bcast(infodata%precip_fact, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%rmean_rmv_ice_runoff, mpicom, pebcast=cmppe) endif if (cpl2r) then @@ -2624,6 +2646,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) call shr_mpi_bcast(infodata%precip_fact, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%glc_g2lupdate, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%glc_valid_input, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%rmean_rmv_ice_runoff, mpicom, pebcast=cplpe) endif end subroutine seq_infodata_Exchange @@ -2948,6 +2971,7 @@ SUBROUTINE seq_infodata_print( infodata ) write(logunit,F0L) subname,'ocn_prognostic = ', infodata%ocn_prognostic write(logunit,F0L) subname,'ocnrof_prognostic = ', infodata%ocnrof_prognostic write(logunit,F0L) subname,'ocn_c2_glcshelf = ', infodata%ocn_c2_glcshelf + write(logunit,F0L) subname,'ocn_c2_glctf = ', infodata%ocn_c2_glctf write(logunit,F0L) subname,'ice_present = ', infodata%ice_present write(logunit,F0L) subname,'ice_prognostic = ', infodata%ice_prognostic write(logunit,F0L) subname,'iceberg_prognostic = ', infodata%iceberg_prognostic @@ -2995,6 +3019,7 @@ SUBROUTINE seq_infodata_print( infodata ) write(logunit,F0S) subname,'iac_phase = ', infodata%iac_phase write(logunit,F0L) subname,'glc_g2lupdate = ', infodata%glc_g2lupdate + write(logunit,F0R) subname,'rmean_rmv_ice_runoff = ', infodata%rmean_rmv_ice_runoff ! endif END SUBROUTINE seq_infodata_print diff --git a/driver-moab/cime_config/config_component.xml b/driver-moab/cime_config/config_component.xml index ac9e46a476e..a32fb9bd529 100644 --- a/driver-moab/cime_config/config_component.xml +++ b/driver-moab/cime_config/config_component.xml @@ -1375,6 +1375,14 @@ atm2ocn flux mapping file + + char + idmap_ignore + run_domain + env_run.xml + atm2ice flux mapping file + + char idmap diff --git a/driver-moab/cime_config/config_component_e3sm.xml b/driver-moab/cime_config/config_component_e3sm.xml index 044cc1e9f9d..c16a493f045 100644 --- a/driver-moab/cime_config/config_component_e3sm.xml +++ b/driver-moab/cime_config/config_component_e3sm.xml @@ -185,6 +185,18 @@ Turn on the passing of polar fields through the coupler + + logical + TRUE,FALSE + FALSE + + TRUE + + run_flags + env_run.xml + Turn on the passing of ocean thermal forcing fields through the coupler + + char minus1p8,linear_salt,mushy @@ -245,6 +257,7 @@ CESM1_MOD CESM1_MOD RASM_OPTION1 + RASM_OPTION2 run_coupling env_run.xml @@ -275,7 +288,7 @@ none CO2C CO2A - CO2A + CO2A CO2A CO2A CO2A @@ -383,6 +396,10 @@ 24 48 48 + 180 + 360 + 720 + 1440 48 48 96 @@ -407,6 +424,7 @@ 144 432 864 + 864 144 96 48 @@ -446,6 +464,7 @@ 1 $ATM_NCPL 48 + $ATM_NCPL $ATM_NCPL 12 96 @@ -747,8 +766,13 @@ 312.821 388.717 388.717 - 0.000001 - 0.000001 + 284.317 + 284.317 + 284.317 + 284.317 + 284.317 + 284.317 + 0.000001 284.317 284.317 284.317 @@ -851,6 +875,7 @@ 1 2 + 2 shr_dust_nl env_run.xml diff --git a/driver-moab/main/component_mod.F90 b/driver-moab/main/component_mod.F90 index 3e8ba9042a5..c7a8f51b99e 100644 --- a/driver-moab/main/component_mod.F90 +++ b/driver-moab/main/component_mod.F90 @@ -471,6 +471,8 @@ subroutine component_init_aream(infodata, rof_c2_ocn, samegrid_ao, samegrid_al, ! character(1024) :: domain_file ! file containing domain info (set my input) use seq_comm_mct, only: mboxid ! iMOAB id for MPAS ocean migrated mesh to coupler pes use seq_comm_mct, only: mbaxid ! iMOAB id for atm migrated mesh to coupler pes + use seq_comm_mct, only: mbrxid ! iMOAB id for rof migrated mesh to coupler pes + use seq_comm_mct, only: mb_rof_aream_computed #endif ! ! Arguments @@ -527,6 +529,9 @@ subroutine component_init_aream(infodata, rof_c2_ocn, samegrid_ao, samegrid_al, dom_s%data%rAttr(km,:) = dom_s%data%rAttr(ka,:) #ifdef HAVE_MOAB + ! TODO should actually compute aream from mesh model + ! we do a lot of unnecessary gymnastics, and very inefficient, because we have a + ! different distribution compared to mct source grid atm tagtype = 1 ! dense, double tagname='aream'//C_NULL_CHAR nloc = mct_avect_lsize(dom_s%data) @@ -542,6 +547,8 @@ subroutine component_init_aream(infodata, rof_c2_ocn, samegrid_ao, samegrid_al, write(logunit,*) subname,' error in setting the aream tag on atm ' call shr_sys_abort(subname//' ERROR in setting aream tag on atm ') endif + deallocate(gids) + deallocate(data1) ! project now aream on ocean (from atm) #endif call seq_map_map(mapper_Fa2o, av_s=dom_s%data, av_d=dom_d%data, fldlist='aream') @@ -597,7 +604,40 @@ subroutine component_init_aream(infodata, rof_c2_ocn, samegrid_ao, samegrid_al, gsmap_s=gsmap_s, av_s=dom_s%data, avfld_s='aream', filefld_s='area_a', & string='rof2ocn ice aream initialization') call t_stopf('CPL:seq_map_readdata-rof2ocn_ice') - + ! this should be more efficient if we just compute aream on coupler side, from actual mesh that we have + ! we need to expose that method in iMOAB, which is local + ! what we do here, we get aream from the domain dom_rx, we just filled it above, with readdata + if(.not. mb_rof_aream_computed) then + + ! we do a lot of unnecessary gymnastics, and very inefficient, because we have a + ! different distribution compared to mct source grid atm + tagtype = 1 ! dense, double + tagname='aream'//C_NULL_CHAR + nloc = mct_avect_lsize(dom_s%data) + allocate(data1(nloc)) + km = mct_aVect_indexRa(dom_s%data, "aream" ) + data1 = dom_s%data%rAttr(km,:) + ent_type = 1 ! element dense double tags + allocate(gids(nloc)) + gids = dom_s%data%iAttr(mct_aVect_indexIA(dom_s%data,"GlobGridNum"),:) + ! ! now set data on the coupler side too + ierr = iMOAB_SetDoubleTagStorageWithGid ( mbrxid, tagname, nloc, ent_type, & + data1, gids) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in setting the aream tag on rof ' + call shr_sys_abort(subname//' ERROR in setting aream tag on rof ') + endif + deallocate(gids) + deallocate(data1) +#ifdef MOABDEBUG + ierr = iMOAB_WriteMesh(mbrxid, trim('recRofWithAream.h5m'//C_NULL_CHAR), & + trim(';PARALLEL=WRITE_PART'//C_NULL_CHAR)) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in writing rof mesh coupler ' + call shr_sys_abort(subname//' ERROR in writing rof mesh coupler ') + endif +#endif + endif endif end if diff --git a/driver-moab/main/cplcomp_exchange_mod.F90 b/driver-moab/main/cplcomp_exchange_mod.F90 index 76737ad3141..259f6953cff 100644 --- a/driver-moab/main/cplcomp_exchange_mod.F90 +++ b/driver-moab/main/cplcomp_exchange_mod.F90 @@ -24,7 +24,6 @@ module cplcomp_exchange_mod use seq_comm_mct, only : mhpgid ! iMOAB app id for atm pgx grid, on atm pes use seq_comm_mct, only : atm_pg_active ! flag if PG mesh instanced use seq_comm_mct, only : mlnid , mblxid ! iMOAB app id for land , on land pes and coupler pes - use seq_comm_mct, only : mb_land_mesh ! if true mesh for land use seq_comm_mct, only : mphaid ! iMOAB app id for phys atm; comp atm is 5, phys 5+200 use seq_comm_mct, only : MPSIID, mbixid ! sea-ice on comp pes and on coupler pes use seq_comm_mct, only : mrofid, mbrxid ! iMOAB id of moab rof app on comp pes and on coupler too @@ -1520,16 +1519,12 @@ subroutine cplcomp_moab_Init(infodata,comp) ! we are now on joint pes, compute comm graph between lnd and coupler model typeA = 2 ! point cloud on component PEs, land - if (mb_land_mesh) then - typeA = 3 - endif typeB = 3 ! full mesh on coupler pes, we just read it if (mlnid >= 0) then ierr = iMOAB_GetMeshInfo ( mlnid, nvert, nvise, nbl, nsurf, nvisBC ) comp%mbApCCid = mlnid ! phys atm comp%mbGridType = typeA - 2 ! 0 or 1, pc or cells comp%mblsize = nvert(1) ! vertices - if (mb_land_mesh) comp%mblsize = nvise(1) ! cells endif ierr = iMOAB_ComputeCommGraph( mlnid, mblxid, mpicom_join, mpigrp_old, mpigrp_cplid, & typeA, typeB, id_old, id_join) diff --git a/driver-moab/main/prep_lnd_mod.F90 b/driver-moab/main/prep_lnd_mod.F90 index 7bdb12376a3..097782f330d 100644 --- a/driver-moab/main/prep_lnd_mod.F90 +++ b/driver-moab/main/prep_lnd_mod.F90 @@ -18,6 +18,7 @@ module prep_lnd_mod use seq_comm_mct, only: mbrxid ! iMOAB id of moab rof on coupler pes (FV now) use seq_comm_mct, only: mbintxal ! iMOAB id for intx mesh between atm and lnd use seq_comm_mct, only: mbintxrl ! iMOAB id for intx mesh between river and land + use seq_comm_mct, only: mb_rof_aream_computed ! signal use seq_comm_mct, only: mbaxid ! iMOAB id for atm migrated mesh to coupler pes use seq_comm_mct, only: atm_pg_active ! whether the atm uses FV mesh or not ; made true if fv_nphys > 0 @@ -327,6 +328,9 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln fNoBubble, monotonicity, volumetric, fInverseDistanceMap, & noConserve, validate, & trim(dofnameS), trim(dofnameT) ) + + ! signal that the aream for rof has been computed + mb_rof_aream_computed = .true. if (ierr .ne. 0) then write(logunit,*) subname,' error in computing rl weights ' call shr_sys_abort(subname//' ERROR in computing rl weights ') diff --git a/driver-moab/main/prep_ocn_mod.F90 b/driver-moab/main/prep_ocn_mod.F90 index b13cce62f95..84c5631bfec 100644 --- a/driver-moab/main/prep_ocn_mod.F90 +++ b/driver-moab/main/prep_ocn_mod.F90 @@ -203,7 +203,7 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc use iMOAB, only: iMOAB_ComputeMeshIntersectionOnSphere, iMOAB_RegisterApplication, & iMOAB_WriteMesh, iMOAB_DefineTagStorage, iMOAB_ComputeCommGraph, iMOAB_ComputeScalarProjectionWeights, & iMOAB_MigrateMapMesh, iMOAB_WriteLocalMesh, iMOAB_GetMeshInfo, iMOAB_SetDoubleTagStorage, & - iMOAB_WriteMappingWeightsToFile + iMOAB_WriteMappingWeightsToFile, iMOAB_SetGhostLayers !--------------------------------------------------------------- ! Description ! Initialize module attribute vectors and all other non-mapping @@ -269,6 +269,7 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc integer ent_type ! for setting tags integer noflds ! used for number of fields in allocating moab accumulated array x2oacc_om real (kind=R8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 + integer nghlay ! used to set the number of ghost layers, needed for bilinear map !--------------------------------------------------------------- @@ -426,6 +427,13 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc ! next, let us compute the ATM and OCN data transfer if (.not. samegrid_ao) then ! not a data OCN model + ! for bilinear maps, we need to have a layer of ghosts on source + nghlay = 1 ! number of ghost layers + ierr = iMOAB_SetGhostLayers( mbaxid, nghlay ) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in setting the number of layers' + call shr_sys_abort(subname//' error in setting the number of layers') + endif ! first compute the overlap mesh between mbaxid (ATM) and mboxid (OCN) on coupler PEs ierr = iMOAB_ComputeMeshIntersectionOnSphere (mbaxid, mboxid, mbintxao) if (ierr .ne. 0) then @@ -1043,6 +1051,7 @@ subroutine prep_ocn_accum_avg_moab() use iMOAB, only : iMOAB_SetDoubleTagStorage, iMOAB_WriteMesh ! Local Variables integer :: ent_type, ierr + integer noflds, lsize ! used for restart case only? character(CXX) :: tagname character(*), parameter :: subname = '(prep_ocn_accum_avg_moab)' #ifdef MOABDEBUG @@ -1056,7 +1065,18 @@ subroutine prep_ocn_accum_avg_moab() x2oacc_om = 1./x2oacc_om_cnt * x2oacc_om end if + if (.not. allocated(x2o_om)) then + ! we could come here in the restart case; not sure why only for + ! the case ERS_Vmoab_T62_oQU120.CMPASO-NYF + lsize = size(x2oacc_om, 1) + noflds = size(x2oacc_om, 2) + allocate (x2o_om(lsize, noflds)) + arrSize_x2o_om = noflds * lsize + + endif + ! ***NOTE***THE FOLLOWING ACTUALLY MODIFIES x2o_om + x2o_om = x2oacc_om !call mct_avect_copy(x2oacc_ox(eoi), x2o_ox) ! modify the tags @@ -1339,9 +1359,11 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) !nwflds = mct_aVect_nRattr(w2x_o) nxflds = mct_aVect_nRattr(xao_o) - !ngflds = mct_aVect_nRattr(g2x_o) - allocate(x2o_om (lsize, noflds)) - arrSize_x2o_om = lsize * noflds ! this willbe used to set/get x2o_om tags + if (.not. allocated(x2o_om)) then + !ngflds = mct_aVect_nRattr(g2x_o) + allocate(x2o_om (lsize, noflds)) + arrSize_x2o_om = lsize * noflds ! this willbe used to set/get x2o_om tags + endif allocate(a2x_om (lsize, naflds)) allocate(i2x_om (lsize, niflds)) allocate(r2x_om (lsize, nrflds)) diff --git a/driver-moab/main/prep_rof_mod.F90 b/driver-moab/main/prep_rof_mod.F90 index 83a8e331e91..2ab02f7bdea 100644 --- a/driver-moab/main/prep_rof_mod.F90 +++ b/driver-moab/main/prep_rof_mod.F90 @@ -30,6 +30,7 @@ module prep_rof_mod use component_type_mod, only: ocn ! used for context for projection towards ocean from rof ! use prep_lnd_mod, only: prep_lnd_get_mapper_Fr2l use map_lnd2rof_irrig_mod, only: map_lnd2rof_irrig + use seq_comm_mct, only: mb_rof_aream_computed ! signal use iso_c_binding #ifdef MOABCOMP @@ -417,6 +418,7 @@ subroutine prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) fNoBubble, monotonicity, volumetric, fInverseDistanceMap, & noConserve, validate, & trim(dofnameS), trim(dofnameT) ) + mb_rof_aream_computed = .true. ! signal if (ierr .ne. 0) then write(logunit,*) subname,' error in computing lr weights ' call shr_sys_abort(subname//' ERROR in computing lr weights ') diff --git a/driver-moab/main/seq_rest_mod.F90 b/driver-moab/main/seq_rest_mod.F90 index ab40eab408a..dafd4d05668 100644 --- a/driver-moab/main/seq_rest_mod.F90 +++ b/driver-moab/main/seq_rest_mod.F90 @@ -97,7 +97,7 @@ module seq_rest_mod public :: seq_rest_mb_write ! read cpl7_moab restart data #ifdef MOABDEBUG - public :: write_moab_state ! debug, write files + public :: write_moab_state ! debug, write files #endif ! !PUBLIC DATA MEMBERS: @@ -367,7 +367,7 @@ subroutine seq_rest_mb_read(rest_file, infodata, samegrid_al) use seq_comm_mct, only: mbaxid, mbixid, mboxid, mblxid, mbrxid, mbofxid ! coupler side instances use iMOAB, only: iMOAB_GetGlobalInfo use seq_comm_mct , only: num_moab_exports ! it is used only as a counter for moab h5m files - + implicit none character(*) , intent(in) :: rest_file ! restart file path/name @@ -379,7 +379,7 @@ subroutine seq_rest_mb_read(rest_file, infodata, samegrid_al) real(r8),allocatable :: ns(:) ! for reshaping diag data for restart file character(CXX) :: moab_rest_file - character(CXX) :: tagname + character(CXX) :: tagname integer (in), pointer :: o2racc_om_cnt ! replacement, moab version for o2racc_ox_cnt integer (in), pointer :: x2oacc_om_cnt ! replacement, moab version for x2oacc_ox_cnt @@ -392,7 +392,7 @@ subroutine seq_rest_mb_read(rest_file, infodata, samegrid_al) real(r8), dimension(:,:), pointer :: p_l2racc_lm character(len=*), parameter :: subname = "(seq_rest_mb_read) " - + !------------------------------------------------------------------------------- ! !------------------------------------------------------------------------------- @@ -517,7 +517,7 @@ subroutine seq_rest_mb_read(rest_file, infodata, samegrid_al) call seq_io_read(moab_rest_file, mboxid, 'fractions_ox', & 'afrac:ifrac:ofrac:ifrad:ofrad') ! fraclist_o = 'afrac:ifrac:ofrac:ifrad:ofrad' call seq_io_read(moab_rest_file, mboxid, 'o2x_ox', & - trim(seq_flds_o2x_fields)) + trim(seq_flds_o2x_fields)) tagname = trim(seq_flds_x2o_fields) x2oacc_om_cnt => prep_ocn_get_x2oacc_om_cnt() p_x2oacc_om => prep_ocn_get_x2oacc_om() @@ -525,7 +525,7 @@ subroutine seq_rest_mb_read(rest_file, infodata, samegrid_al) call seq_io_read (moab_rest_file, mboxid, 'x2oacc_ox', & trim(tagname), & matrix=p_x2oacc_om) - call seq_io_read(moab_rest_file, x2oacc_om_cnt, 'x2oacc_ox_cnt') + call seq_io_read(moab_rest_file, x2oacc_om_cnt, 'x2oacc_ox_cnt') ! tagname = trim(seq_flds_xao_fields)//C_NULL_CHAR ! arrsize = nxflds * lsize ! allocate (xao_om (lsize, nxflds)) ! ierr = iMOAB_GetDoubleTagStorage ( mbofxid, tagname, arrsize , ent_type, xao_om) @@ -548,7 +548,7 @@ subroutine seq_rest_mb_read(rest_file, infodata, samegrid_al) call seq_io_read(moab_rest_file, mbixid, 'fractions_ix', & 'afrac:ifrac:ofrac') ! fraclist_i = 'afrac:ifrac:ofrac' call seq_io_read(moab_rest_file, mbixid, 'i2x_ix', & - trim(seq_flds_i2x_fields) ) + trim(seq_flds_i2x_fields) ) ! gsmap => component_get_gsmap_cx(ice(1)) ! call seq_io_read(rest_file, gsmap, fractions_ix, 'fractions_ix') ! call seq_io_read(rest_file, ice, 'c2x', 'i2x_ix') @@ -557,7 +557,7 @@ subroutine seq_rest_mb_read(rest_file, infodata, samegrid_al) call seq_io_read(moab_rest_file, mbrxid, 'fractions_rx', & 'lfrac:lfrin:rfrac') ! fraclist_r = 'lfrac:lfrin:rfrac' call seq_io_read(moab_rest_file, mbrxid, 'r2x_rx', & - trim(seq_flds_r2x_fields) ) + trim(seq_flds_r2x_fields) ) ! gsmap => component_get_gsmap_cx(rof(1)) ! call seq_io_read(rest_file, gsmap, fractions_rx, 'fractions_rx') ! call seq_io_read(rest_file, rof, 'c2x', 'r2x_rx') @@ -799,6 +799,8 @@ subroutine seq_rest_write(EClock_d, seq_SyncClock, infodata, & call seq_io_write(rest_file,rvar,'seq_infodata_precip_fact',whead=whead,wdata=wdata) call seq_infodata_GetData(infodata,case_name=cvar) call seq_io_write(rest_file,trim(cvar),'seq_infodata_case_name',whead=whead,wdata=wdata) + call seq_infodata_GetData(infodata,rmean_rmv_ice_runoff=rvar) + call seq_io_write(rest_file,rvar,'seq_infodata_rmean_rmv_ice_runoff',whead=whead,wdata=wdata) call seq_timemgr_EClockGetData( EClock_d, start_ymd=ivar) call seq_io_write(rest_file,ivar,'seq_timemgr_start_ymd',whead=whead,wdata=wdata) @@ -1120,6 +1122,8 @@ subroutine seq_rest_mb_write(EClock_d, seq_SyncClock, infodata, & call seq_io_write(rest_file,rvar,'seq_infodata_precip_fact',whead=whead,wdata=wdata) call seq_infodata_GetData(infodata,case_name=cvar) call seq_io_write(rest_file,trim(cvar),'seq_infodata_case_name',whead=whead,wdata=wdata) + call seq_infodata_GetData(infodata,rmean_rmv_ice_runoff=rvar) + call seq_io_write(rest_file,rvar,'seq_infodata_rmean_rmv_ice_runoff',whead=whead,wdata=wdata) call seq_timemgr_EClockGetData( EClock_d, start_ymd=ivar) call seq_io_write(rest_file,ivar,'seq_timemgr_start_ymd',whead=whead,wdata=wdata) @@ -1169,15 +1173,15 @@ subroutine seq_rest_mb_write(EClock_d, seq_SyncClock, infodata, & ! nx for land will be from global nb atmosphere ierr = iMOAB_GetGlobalInfo(mbaxid, dummy, nx_lnd) ! max id for land will come from atm call seq_io_write(rest_file, mblxid, 'fractions_lx', & - 'afrac:lfrac:lfrin', & ! seq_frac_mod: character(*),parameter :: fraclist_l = 'afrac:lfrac:lfrin' - whead=whead, wdata=wdata, nx=nx_lnd) + 'afrac:lfrac:lfrin', & ! seq_frac_mod: character(*),parameter :: fraclist_l = 'afrac:lfrac:lfrin' + whead=whead, wdata=wdata, nx=nx_lnd) else call seq_io_write(rest_file, mblxid, 'fractions_lx', & - 'afrac:lfrac:lfrin', & ! seq_frac_mod: character(*),parameter :: fraclist_l = 'afrac:lfrac:lfrin' + 'afrac:lfrac:lfrin', & ! seq_frac_mod: character(*),parameter :: fraclist_l = 'afrac:lfrac:lfrin' whead=whead, wdata=wdata) endif ! call seq_io_write(rest_file, mblxid, 'fractions_lx', & - ! 'afrac:lfrac:lfrin', & ! seq_frac_mod: character(*),parameter :: fraclist_l = 'afrac:lfrac:lfrin' + ! 'afrac:lfrac:lfrin', & ! seq_frac_mod: character(*),parameter :: fraclist_l = 'afrac:lfrac:lfrin' ! whead=whead, wdata=wdata) ! gsmap => component_get_gsmap_cx(lnd(1)) ! call seq_io_write(rest_file, gsmap, fractions_lx, 'fractions_lx', & @@ -1192,7 +1196,7 @@ subroutine seq_rest_mb_write(EClock_d, seq_SyncClock, infodata, & ierr = iMOAB_GetGlobalInfo(mbaxid, dummy, nx_lnd) ! max id for land will come from atm call seq_io_write(rest_file, mblxid, 'l2racc_lx', & trim(tagname), & - whead=whead, wdata=wdata, matrix = p_l2racc_lm, nx=nx_lnd) + whead=whead, wdata=wdata, matrix = p_l2racc_lm, nx=nx_lnd) else call seq_io_write(rest_file, mblxid, 'l2racc_lx', & trim(tagname), & @@ -1247,11 +1251,11 @@ subroutine seq_rest_mb_write(EClock_d, seq_SyncClock, infodata, & if (ocn_present) then ! gsmap => component_get_gsmap_cx(ocn(1)) ! x2oacc_ox => prep_ocn_get_x2oacc_ox() - + call seq_io_write(rest_file, mboxid, 'fractions_ox', & 'afrac:ifrac:ofrac:ifrad:ofrad', & ! fraclist_o = 'afrac:ifrac:ofrac:ifrad:ofrad' whead=whead, wdata=wdata) - + call seq_io_write(rest_file, mboxid, 'o2x_ox', & trim(seq_flds_o2x_fields), & whead=whead, wdata=wdata) @@ -1293,7 +1297,7 @@ subroutine seq_rest_mb_write(EClock_d, seq_SyncClock, infodata, & whead=whead, wdata=wdata) call seq_io_write(rest_file, mbixid, 'i2x_ix', & trim(seq_flds_i2x_fields), & - whead=whead, wdata=wdata) + whead=whead, wdata=wdata) ! gsmap => component_get_gsmap_cx(ice(1)) ! call seq_io_write(rest_file, gsmap, fractions_ix, 'fractions_ix', & ! whead=whead, wdata=wdata) @@ -1348,10 +1352,10 @@ end subroutine seq_rest_mb_write !=============================================================================== #ifdef MOABDEBUG - subroutine write_moab_state ( before_reading ) ! debug, write files + subroutine write_moab_state ( before_reading ) ! debug, write files use seq_comm_mct, only: mbaxid, mbixid, mboxid, mblxid, mbrxid, mbofxid ! coupler side instances use seq_comm_mct, only: num_moab_exports - use iso_c_binding + use iso_c_binding use iMOAB, only: iMOAB_WriteMesh implicit none @@ -1414,7 +1418,7 @@ subroutine write_moab_state ( before_reading ) ! debug, write files endif endif - end subroutine write_moab_state + end subroutine write_moab_state #endif end module seq_rest_mod diff --git a/driver-moab/shr/seq_comm_mct.F90 b/driver-moab/shr/seq_comm_mct.F90 index 44d29e91ad2..9c85df84d2b 100644 --- a/driver-moab/shr/seq_comm_mct.F90 +++ b/driver-moab/shr/seq_comm_mct.F90 @@ -245,9 +245,9 @@ module seq_comm_mct integer, public :: mbintxar ! iMOAB id for intx mesh between atm and river integer, public :: mbintxlr ! iMOAB id for intx mesh between land and river integer, public :: mbintxrl ! iMOAB id for intx mesh between river and land - logical, public :: mb_land_mesh = .false. ! whether the land uses full FV mesh or not ; made true if domain mesh is read on comp land - + integer, public :: num_moab_exports ! iMOAB id for atm phys grid, on atm pes + logical, public :: mb_rof_aream_computed = .false. ! whether the aream for rof has been set or not !======================================================================= contains diff --git a/driver-moab/shr/seq_infodata_mod.F90 b/driver-moab/shr/seq_infodata_mod.F90 index 512adb853b6..d9e92f368ad 100644 --- a/driver-moab/shr/seq_infodata_mod.F90 +++ b/driver-moab/shr/seq_infodata_mod.F90 @@ -198,6 +198,7 @@ MODULE seq_infodata_mod logical :: ocn_prognostic ! does component model need input data from driver logical :: ocnrof_prognostic ! does component need rof data logical :: ocn_c2_glcshelf ! will ocn component send data for ice shelf fluxes in driver + logical :: ocn_c2_glctf ! will ocn component send data for thermal forcing in driver logical :: ice_present ! does component model exist logical :: ice_prognostic ! does component model need input data from driver logical :: iceberg_prognostic ! does the ice model support icebergs @@ -251,7 +252,8 @@ MODULE seq_infodata_mod integer(SHR_KIND_IN) :: iac_phase ! iac phase logical :: atm_aero ! atmosphere aerosols logical :: glc_g2lupdate ! update glc2lnd fields in lnd model - real(shr_kind_r8) :: max_cplstep_time ! abort if cplstep time exceeds this value + real(SHR_KIND_R8) :: max_cplstep_time ! abort if cplstep time exceeds this value + real(SHR_KIND_R8) :: rmean_rmv_ice_runoff ! running mean of removed Antarctic ice runoff !--- set from restart file --- character(SHR_KIND_CL) :: rest_case_name ! Short case identification !--- set by driver and may be time varying @@ -759,10 +761,11 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid, cpl_tag) infodata%atm_prognostic = .false. infodata%lnd_prognostic = .false. infodata%rof_prognostic = .false. - infodata%rofocn_prognostic = .false. + infodata%rofocn_prognostic = .false. infodata%ocn_prognostic = .false. infodata%ocnrof_prognostic = .false. infodata%ocn_c2_glcshelf = .false. + infodata%ocn_c2_glctf = .false. infodata%ice_prognostic = .false. infodata%glc_prognostic = .false. ! It's safest to assume glc_coupled_fluxes = .true. if it's not set elsewhere, @@ -795,8 +798,8 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid, cpl_tag) infodata%lnd_domain = 'none' infodata%rof_mesh = 'none' infodata%rof_domain = 'none' - infodata%ocn_domain = 'none' ! will be used for ocean data models only; will be used as a signal - infodata%ice_domain = 'none' ! will be used for ice data models only; will be used as a signal + infodata%ocn_domain = 'none' ! will be used for ocean data models only; will be used as a signal + infodata%ice_domain = 'none' ! will be used for ice data models only; will be used as a signal infodata%atm_mesh = 'none' ! will be used for atmosphere data models only; will be used as a signal ! not sure if it exists always actually @@ -813,6 +816,7 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid, cpl_tag) infodata%atm_aero = .false. infodata%glc_g2lupdate = .false. infodata%glc_valid_input = .true. + infodata%rmean_rmv_ice_runoff = -1.0_SHR_KIND_R8 infodata%max_cplstep_time = max_cplstep_time infodata%model_doi_url = model_doi_url @@ -912,11 +916,13 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid, cpl_tag) call seq_io_read(infodata%restart_file,pioid,infodata%nextsw_cday ,'seq_infodata_nextsw_cday') call seq_io_read(infodata%restart_file,pioid,infodata%precip_fact ,'seq_infodata_precip_fact') call seq_io_read(infodata%restart_file,pioid,infodata%rest_case_name,'seq_infodata_case_name') + call seq_io_read(infodata%restart_file,pioid,infodata%rmean_rmv_ice_runoff ,'seq_infodata_rmean_rmv_ice_runoff') endif !--- Send from CPLID ROOT to GLOBALID ROOT, use bcast as surrogate call shr_mpi_bcast(infodata%nextsw_cday,mpicom,pebcast=seq_comm_gloroot(CPLID)) call shr_mpi_bcast(infodata%precip_fact,mpicom,pebcast=seq_comm_gloroot(CPLID)) call shr_mpi_bcast(infodata%rest_case_name,mpicom,pebcast=seq_comm_gloroot(CPLID)) + call shr_mpi_bcast(infodata%rmean_rmv_ice_runoff,mpicom,pebcast=seq_comm_gloroot(CPLID)) endif if (seq_comm_iamroot(ID)) then @@ -1009,7 +1015,8 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ atm_present, atm_prognostic, & lnd_present, lnd_prognostic, & rof_present, rof_prognostic, rofocn_prognostic, & - ocn_present, ocn_prognostic, ocnrof_prognostic, ocn_c2_glcshelf, & + ocn_present, ocn_prognostic, ocnrof_prognostic, & + ocn_c2_glcshelf, ocn_c2_glctf, & ice_present, ice_prognostic, & glc_present, glc_prognostic, & iac_present, iac_prognostic, & @@ -1047,7 +1054,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ reprosum_use_ddpdd, reprosum_allow_infnan, & reprosum_diffmax, reprosum_recompute, & mct_usealltoall, mct_usevector, max_cplstep_time, model_doi_url, & - glc_valid_input, nlmaps_verbosity) + glc_valid_input, nlmaps_verbosity, rmean_rmv_ice_runoff) implicit none @@ -1183,6 +1190,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ logical, optional, intent(OUT) :: ocn_prognostic logical, optional, intent(OUT) :: ocnrof_prognostic logical, optional, intent(OUT) :: ocn_c2_glcshelf + logical, optional, intent(OUT) :: ocn_c2_glctf logical, optional, intent(OUT) :: ice_present logical, optional, intent(OUT) :: ice_prognostic logical, optional, intent(OUT) :: iceberg_prognostic @@ -1238,6 +1246,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ real(shr_kind_r8), optional, intent(out) :: max_cplstep_time character(SHR_KIND_CL), optional, intent(OUT) :: model_doi_url logical, optional, intent(OUT) :: glc_valid_input + real(SHR_KIND_R8), optional, intent(OUT) :: rmean_rmv_ice_runoff ! running mean of removed Antarctic ice runoff !----- local ----- character(len=*), parameter :: subname = '(seq_infodata_GetData_explicit) ' @@ -1374,6 +1383,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ if ( present(ocn_prognostic) ) ocn_prognostic = infodata%ocn_prognostic if ( present(ocnrof_prognostic) ) ocnrof_prognostic = infodata%ocnrof_prognostic if ( present(ocn_c2_glcshelf) ) ocn_c2_glcshelf = infodata%ocn_c2_glcshelf + if ( present(ocn_c2_glctf) ) ocn_c2_glctf = infodata%ocn_c2_glctf if ( present(ice_present) ) ice_present = infodata%ice_present if ( present(ice_prognostic) ) ice_prognostic = infodata%ice_prognostic if ( present(iceberg_prognostic)) iceberg_prognostic = infodata%iceberg_prognostic @@ -1442,6 +1452,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ if ( present(model_doi_url) ) model_doi_url = infodata%model_doi_url if ( present(glc_valid_input)) glc_valid_input = infodata%glc_valid_input + if ( present(rmean_rmv_ice_runoff) ) rmean_rmv_ice_runoff = infodata%rmean_rmv_ice_runoff END SUBROUTINE seq_infodata_GetData_explicit @@ -1572,7 +1583,8 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ atm_present, atm_prognostic, & lnd_present, lnd_prognostic, & rof_present, rof_prognostic, rofocn_prognostic, & - ocn_present, ocn_prognostic, ocnrof_prognostic, ocn_c2_glcshelf, & + ocn_present, ocn_prognostic, ocnrof_prognostic, & + ocn_c2_glcshelf, ocn_c2_glctf, & ice_present, ice_prognostic, & glc_present, glc_prognostic, & glc_coupled_fluxes, & @@ -1610,7 +1622,8 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ eps_aarea, eps_omask, eps_ogrid, eps_oarea, & reprosum_use_ddpdd, reprosum_allow_infnan, & reprosum_diffmax, reprosum_recompute, & - mct_usealltoall, mct_usevector, glc_valid_input, nlmaps_verbosity) + mct_usealltoall, mct_usevector, glc_valid_input, nlmaps_verbosity, & + rmean_rmv_ice_runoff) implicit none @@ -1746,6 +1759,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ logical, optional, intent(IN) :: ocn_prognostic logical, optional, intent(IN) :: ocnrof_prognostic logical, optional, intent(IN) :: ocn_c2_glcshelf + logical, optional, intent(IN) :: ocn_c2_glctf logical, optional, intent(IN) :: ice_present logical, optional, intent(IN) :: ice_prognostic logical, optional, intent(IN) :: iceberg_prognostic @@ -1798,6 +1812,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ logical, optional, intent(IN) :: atm_aero ! atm aerosols logical, optional, intent(IN) :: glc_g2lupdate ! update glc2lnd fields in lnd model logical, optional, intent(IN) :: glc_valid_input + real(SHR_KIND_R8), optional, intent(IN) :: rmean_rmv_ice_runoff ! running mean of removed Antarctic ice runoff !EOP @@ -1936,6 +1951,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ if ( present(ocn_prognostic) ) infodata%ocn_prognostic = ocn_prognostic if ( present(ocnrof_prognostic)) infodata%ocnrof_prognostic = ocnrof_prognostic if ( present(ocn_c2_glcshelf)) infodata%ocn_c2_glcshelf = ocn_c2_glcshelf + if ( present(ocn_c2_glctf)) infodata%ocn_c2_glctf = ocn_c2_glctf if ( present(ice_present) ) infodata%ice_present = ice_present if ( present(ice_prognostic) ) infodata%ice_prognostic = ice_prognostic if ( present(iceberg_prognostic)) infodata%iceberg_prognostic = iceberg_prognostic @@ -1988,6 +2004,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ if ( present(atm_aero) ) infodata%atm_aero = atm_aero if ( present(glc_g2lupdate) ) infodata%glc_g2lupdate = glc_g2lupdate if ( present(glc_valid_input) ) infodata%glc_valid_input = glc_valid_input + if ( present(rmean_rmv_ice_runoff) ) infodata%rmean_rmv_ice_runoff = rmean_rmv_ice_runoff END SUBROUTINE seq_infodata_PutData_explicit @@ -2249,6 +2266,7 @@ subroutine seq_infodata_bcast(infodata,mpicom) call shr_mpi_bcast(infodata%ocn_prognostic, mpicom) call shr_mpi_bcast(infodata%ocnrof_prognostic, mpicom) call shr_mpi_bcast(infodata%ocn_c2_glcshelf, mpicom) + call shr_mpi_bcast(infodata%ocn_c2_glctf, mpicom) call shr_mpi_bcast(infodata%ice_present, mpicom) call shr_mpi_bcast(infodata%ice_prognostic, mpicom) call shr_mpi_bcast(infodata%iceberg_prognostic, mpicom) @@ -2302,6 +2320,7 @@ subroutine seq_infodata_bcast(infodata,mpicom) call shr_mpi_bcast(infodata%glc_valid_input, mpicom) call shr_mpi_bcast(infodata%model_doi_url, mpicom) call shr_mpi_bcast(infodata%constant_zenith_deg, mpicom) + call shr_mpi_bcast(infodata%rmean_rmv_ice_runoff, mpicom) end subroutine seq_infodata_bcast @@ -2544,6 +2563,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) call shr_mpi_bcast(infodata%ocn_prognostic, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ocnrof_prognostic, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ocn_c2_glcshelf, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%ocn_c2_glctf, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ocn_nx, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ocn_ny, mpicom, pebcast=cmppe) call shr_mpi_bcast(infodata%ocn_domain, mpicom, pebcast=cmppe) @@ -2622,6 +2642,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) call shr_mpi_bcast(infodata%ocn_prognostic, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%ocnrof_prognostic, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%ocn_c2_glcshelf, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%ocn_c2_glctf, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%ice_present, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%ice_prognostic, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%iceberg_prognostic, mpicom, pebcast=cplpe) @@ -2648,6 +2669,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) if (ocn2cplr) then call shr_mpi_bcast(infodata%precip_fact, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%rmean_rmv_ice_runoff, mpicom, pebcast=cmppe) endif if (cpl2r) then @@ -2655,6 +2677,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) call shr_mpi_bcast(infodata%precip_fact, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%glc_g2lupdate, mpicom, pebcast=cplpe) call shr_mpi_bcast(infodata%glc_valid_input, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%rmean_rmv_ice_runoff, mpicom, pebcast=cplpe) endif end subroutine seq_infodata_Exchange @@ -2972,6 +2995,7 @@ SUBROUTINE seq_infodata_print( infodata ) write(logunit,F0L) subname,'ocn_prognostic = ', infodata%ocn_prognostic write(logunit,F0L) subname,'ocnrof_prognostic = ', infodata%ocnrof_prognostic write(logunit,F0L) subname,'ocn_c2_glcshelf = ', infodata%ocn_c2_glcshelf + write(logunit,F0L) subname,'ocn_c2_glctf = ', infodata%ocn_c2_glctf write(logunit,F0L) subname,'ice_present = ', infodata%ice_present write(logunit,F0L) subname,'ice_prognostic = ', infodata%ice_prognostic write(logunit,F0L) subname,'iceberg_prognostic = ', infodata%iceberg_prognostic @@ -3025,6 +3049,7 @@ SUBROUTINE seq_infodata_print( infodata ) write(logunit,F0S) subname,'iac_phase = ', infodata%iac_phase write(logunit,F0L) subname,'glc_g2lupdate = ', infodata%glc_g2lupdate + write(logunit,F0R) subname,'rmean_rmv_ice_runoff = ', infodata%rmean_rmv_ice_runoff ! endif END SUBROUTINE seq_infodata_print diff --git a/externals/ekat b/externals/ekat index 1d441b22df3..17adc61faae 160000 --- a/externals/ekat +++ b/externals/ekat @@ -1 +1 @@ -Subproject commit 1d441b22df3e4f8f8b3ea96099b0e848eb74afd7 +Subproject commit 17adc61faae0ebb6de19fe389596e3dd3622d2d2 diff --git a/externals/mam4xx b/externals/mam4xx index fdbb0816c5c..f3f215185ca 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit fdbb0816c5c0c541265ec17f544908da935d8af6 +Subproject commit f3f215185ca59cd765704e763c499b484791fc62 diff --git a/share/pacer/Pacer.cpp b/share/pacer/Pacer.cpp new file mode 100644 index 00000000000..ac1ad329b54 --- /dev/null +++ b/share/pacer/Pacer.cpp @@ -0,0 +1,211 @@ +//===-- Pacer.cpp - Timers for E3SM --*- C++ -*-===// +// +// \file +// \brief Timer infrastructure for E3SM C++ components +// +// +////===------------------------------------------===// + +#include "Pacer.h" +#include +#include +#include + +#define PACER_CHECK_INIT() {\ + if (!IsInitialized) { \ + std::cerr << "[ERROR] Pacer: Not initialized." << std::endl; \ + return false; \ + } \ +} + +#define PACER_CHECK_ERROR(x) {\ + if ( (x) != 0 ) { \ + std::cerr << "[ERROR] Pacer: Failure calling GPTL function: " << #x << std::endl; \ + return false; \ + } \ +} + +/// Helper function to check if GPTL is initialized +/// Function declaration is missing in gptl.h +/// Hence the declaration here +extern "C" { + extern int GPTLis_initialized(void); +} + +/// Check if Pacer is initialized +/// Returns true if initialized +inline bool Pacer::isInitialized(void){ + if (!IsInitialized) { + std::cerr << "[ERROR] Pacer: Not initialized." << std::endl; + return false; + } + return true; +} + +/// Initialize Pacer timing +/// InComm: overall MPI communicator used by application. +/// InMode: Pacer standalone (default) or within CIME +bool Pacer::initialize(MPI_Comm InComm, PacerModeType InMode /* = PACER_STANDALONE */) { + + int errCode; + + PacerMode = InMode; + + // Check if already initialized and return + if (IsInitialized) + return true; + + // Duplicate comm and get MPI rank for both standalone and integrated modes + if (MPI_Comm_dup(InComm, &InternalComm) != MPI_SUCCESS) + std::cerr << "Pacer: Error duplicating MPI communicator" << std::endl; + MPI_Comm_rank(InternalComm, &MyRank); + + if (PacerMode == PACER_STANDALONE ) { + // GPTL set default options + PACER_CHECK_ERROR(GPTLsetoption(GPTLdepthlimit, 20)); + PACER_CHECK_ERROR(GPTLsetoption(GPTLdopr_quotes, 1)); + PACER_CHECK_ERROR(GPTLsetoption(GPTLprofile_ovhd, 1)); + // GPTL default is set to 52 + // Presently setting to 64 + PACER_CHECK_ERROR(GPTLsetoption(GPTLmaxthreads, 64)); + + PACER_CHECK_ERROR(GPTLsetutr(GPTLmpiwtime)); + + errCode = GPTLinitialize(); + + if (errCode) { + std::cerr << "[ERROR] Pacer: Unable to initialize GPTL." << std::endl; + return false; + } + else + IsInitialized = true; + } + else if (PacerMode == PACER_INTEGRATED) { + // GPTL is assumed to be initialized by E3SM driver in coupled modeling context + + // Check if GPTL is initialized + errCode = GPTLis_initialized(); + + if (errCode == true) { + IsInitialized = true; + } + else { + IsInitialized = false; + std::cerr << "[ERROR] Pacer: GPTL is not initialized in PACER_INTEGRATED mode." << std::endl; + return false; + } + } + + return true; +} + +/// Start the time named TimerName +bool Pacer::start(const std::string &TimerName) +{ + PACER_CHECK_INIT(); + + PACER_CHECK_ERROR(GPTLstart(TimerName.c_str())); + + auto it = OpenTimers.find(TimerName); + if (it != OpenTimers.end() ) + OpenTimers[TimerName]++; + else + OpenTimers[TimerName] = 1; + return true; +} + +/// Stop the time named TimerName +/// Issues warning if timer hasn't been started yet +bool Pacer::stop(const std::string &TimerName) +{ + PACER_CHECK_INIT(); + + auto it = OpenTimers.find(TimerName); + + if (it != OpenTimers.end() ) { + PACER_CHECK_ERROR(GPTLstop(TimerName.c_str())); + + if ( OpenTimers[TimerName] == 1 ) + OpenTimers.erase(TimerName); + else + OpenTimers[TimerName]--; + } + else { + std::cerr << "[WARNING] Pacer: Trying to stop timer: \"" + << TimerName << "\" before starting it." << std::endl; + + return false; + } + + return true; +} + +/// Sets named prefix for all subsequent timers +bool Pacer::setPrefix(const std::string &Prefix) +{ + PACER_CHECK_INIT(); + + PACER_CHECK_ERROR(GPTLprefix_set(Prefix.c_str())); + + return true; +} + +/// Unsets prefix for all subsequent timers +bool Pacer::unsetPrefix() +{ + PACER_CHECK_INIT(); + + PACER_CHECK_ERROR(GPTLprefix_unset()); + + return true; +} + +/// Prints timing statistics and global summary files +/// Output Files: TimerFilePrefix.timing. +/// TimerFilePrefix.summary +/// PrintAllRanks: flag to control if per rank timing files are printed +bool Pacer::print(const std::string &TimerFilePrefix, bool PrintAllRanks /*= = false */) +{ + PACER_CHECK_INIT(); + + std::string TimerFileName = TimerFilePrefix + ".timing." + std::to_string(MyRank); + std::string SummaryFileName = TimerFilePrefix + ".summary"; + + PACER_CHECK_ERROR(GPTLpr_summary_file(InternalComm, SummaryFileName.c_str())); + + if ( PrintAllRanks == false ) { + if (MyRank == 0) { + PACER_CHECK_ERROR(GPTLpr_file(TimerFileName.c_str())); + } + } + else + PACER_CHECK_ERROR(GPTLpr_file(TimerFileName.c_str())); + + return true; +} + +/// Cleans up Pacer +/// Issues warning if any timers are still open +bool Pacer::finalize() +{ + PACER_CHECK_INIT(); + + if ( PacerMode == PACER_STANDALONE ) + PACER_CHECK_ERROR(GPTLfinalize()); + + if ( (MyRank == 0) && ( OpenTimers.size() > 0) ){ + std::cerr << "[WARNING] Pacer: Following " << OpenTimers.size() << " timer(s) is/are still open." << std::endl; + for (auto i = OpenTimers.begin(); i != OpenTimers.end(); i++) + std::cerr << '\t' << i->first << std::endl; + } + OpenTimers.clear(); + + // Clear Pacer state and free communicator + IsInitialized = false; + MPI_Comm_free(&InternalComm); + + return true; +} + + +//===----------------------------------------------------------------------------===// diff --git a/share/pacer/Pacer.h b/share/pacer/Pacer.h new file mode 100644 index 00000000000..3c64f001fa8 --- /dev/null +++ b/share/pacer/Pacer.h @@ -0,0 +1,75 @@ +//===-- Pacer.h - Pacer timing interface -------------*- C++ +//-*-===// +// +// \file +// \brief Provides timer functionality for E3SM +// +// The Pacer class provides an interface to timers for +// E3SM components. +// +//===--------------------------------------------------===// +#ifndef PACER_H +#define PACER_H + +#include +#include +#include +#include + +namespace Pacer { + + /// Flag to determine if the timing infrastructure is initialized + static bool IsInitialized = false; + + /// MPI communicator used within Pacer + static MPI_Comm InternalComm; + + /// MPI rank of process + static int MyRank; + + /// hash table of open timers with count (for multiple parents) + static std::unordered_map OpenTimers; + + enum PacerModeType { PACER_STANDALONE, PACER_INTEGRATED }; + + /// Pacer Mode: standalone or within CIME + static PacerModeType PacerMode; + + /// Initialize Pacer timing + /// InComm: overall MPI communicator used by application. + /// InMode: Pacer Mode: standalone (default) or within CIME + bool initialize(MPI_Comm InComm, PacerModeType InMode = PACER_STANDALONE); + + /// Check if Pacer is initialized + /// Returns true if initialized + bool isInitialized(void); + + /// IntegratedMode: Pacer standalone (default:false) or within CIME (true) + // bool initialize(MPI_Comm InComm, bool IntegratedMode = false); + + /// Start the time named TimerName + bool start(const std::string &TimerName); + + /// Stop the time named TimerName + /// Issues warning if timer hasn't been started yet + bool stop(const std::string &TimerName); + + /// Sets named prefix for all subsequent timers + bool setPrefix(const std::string &Prefix); + + /// Unsets prefix for all subsequent timers + bool unsetPrefix(); + + /// Prints timing statistics and global summary files + /// Output Files: TimerFilePrefix.timing. + /// TimerFilePrefix.summary + /// PrintAllRanks: flag to control if per rank timing files are printed + bool print(const std::string &TimerFilePrefix, bool PrintAllRanks = false); + + /// Cleans up Pacer + /// Issues warning if any timers are still open + bool finalize(); + +}; + +#endif diff --git a/share/pacer/TestPacer.cpp b/share/pacer/TestPacer.cpp new file mode 100644 index 00000000000..96f29786232 --- /dev/null +++ b/share/pacer/TestPacer.cpp @@ -0,0 +1,64 @@ +////===-- TestPacer.cpp - Simple test for Pacer --*- C++ -*-===// +// +// \file +// \brief Simple example illustrating Pacer API usage. +// +// This test exercises basic timer functionality +// with the Pacer API. +// +// This test program should create two files: +// test.timing.0 and test.summary +// It is also expected to issue couple of warnings +// to illustrate likely scenarios where a timer is +// not started/stopped properly. +// +////===-----------------------------------------------------===// + +#include +#include +#include "Pacer.h" + +int main(int argc, char **argv){ + + int err; + int myrank; + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &myrank); + + Pacer::initialize(MPI_COMM_WORLD); + // Second argument is optional (default is Pacer::PACER_STANDALONE) + // Pacer::initialize(MPI_COMM_WORLD, Pacer::PACER_STANDALONE); + + Pacer::setPrefix("Omega:"); + + Pacer::start("run_loop"); + + float tmp = 1; + + for (int i = 1; i <= 10000; i++){ + tmp *= i; + } + + Pacer::stop("run_loop"); + + Pacer::unsetPrefix(); + + // illustrating situation where attempt to stop timer before starting + // will print a warning + Pacer::stop("final"); + + // as well as dangling timer which is never stopped explicitly + // will print a warning at the end during finalize + Pacer::start("final"); + + + // print: First argument is the prefix used for timing output file names + // Second argument (optional) controls if timing output should be from all ranks + // default is false, only rank 0 writes timing output + Pacer::print("test"); + // Pacer::print("test", true); + + Pacer::finalize(); +} + diff --git a/share/util/shr_scam_mod.F90 b/share/util/shr_scam_mod.F90 index dd82ab5e171..2fb92f769cc 100644 --- a/share/util/shr_scam_mod.F90 +++ b/share/util/shr_scam_mod.F90 @@ -647,11 +647,13 @@ subroutine shr_scam_checkSurface(scmlon, scmlat, scm_multcols, scm_nx, scm_ny, & real(r8) :: sst_constant_value character(len=CL) :: restfilm = 'unset' character(len=CL) :: restfils = 'unset' + real(r8) :: RSO_fixed_MLD + real(r8) :: RSO_relax_tau integer(IN) :: nfrac logical :: force_prognostic_true = .false. namelist /dom_inparm/ sstcyc, nrevsn, rest_pfile, bndtvs, focndomain namelist / docn_nml / decomp, sst_constant_value, force_prognostic_true, & - restfilm, restfils + restfilm, restfils, RSO_fixed_MLD, RSO_relax_tau !------------------------------------------------------------------------------- ! Notes: