Skip to content

Commit 59eae3f

Browse files
authored
Merge pull request #19 from JCSDA/feature/prepCRTMv3
Commit CRTMv3 configuration (backward compatible with 2.4)
2 parents b5a3386 + 92fe7d1 commit 59eae3f

28 files changed

Lines changed: 2018 additions & 363 deletions

CMakeLists.txt

Lines changed: 90 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,60 +1,117 @@
11
cmake_minimum_required(VERSION 3.10.2)
22
project(skbuild_pycrtm)
33
enable_language(Fortran)
4+
45
list( APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake )
56
find_package(F2PY REQUIRED)
67
find_package(PythonLibs REQUIRED)
78
find_package(Python3 REQUIRED COMPONENTS NumPy)
89
find_package( OpenMP COMPONENTS Fortran )
9-
find_package( NetCDF REQUIRED COMPONENTS Fortran )
10+
find_package( NetCDF REQUIRED COMPONENTS Fortran)
1011
include_directories(${PYTHON_INCLUDE_DIRS})
1112
include_directories(${_Python3_NumPy_INCLUDE_DIR})
12-
include_directories(${NetCDF_INCLUDE_DIRS})
13+
include_directories(NetCDF_F90_INCLUDE_DIRS)
1314
get_filename_component(CDF_LIB_PATH ${NetCDF_Fortran_LIBRARY} DIRECTORY)
14-
set(crtm_INCLUDE $ENV{CRTM_INSTALL}/module/crtm/${CMAKE_Fortran_COMPILER_ID}/${CMAKE_Fortran_COMPILER_VERSION}/)
15-
set(crtm_LIB $ENV{CRTM_INSTALL}/lib/)
15+
find_package(Python REQUIRED COMPONENTS Interpreter Development.Module NumPy)
16+
17+
if (EXISTS $ENV{CRTM_INSTALL}/module/crtm/${CMAKE_Fortran_COMPILER_ID}/${CMAKE_Fortran_COMPILER_VERSION}/)
18+
set(crtm_INCLUDE $ENV{CRTM_INSTALL}/module/crtm/${CMAKE_Fortran_COMPILER_ID}/${CMAKE_Fortran_COMPILER_VERSION}/)
19+
else()
20+
set(crtm_INCLUDE $ENV{CRTM_INSTALL}/include/)
21+
endif()
22+
23+
if (EXISTS $ENV{CRTM_INSTALL}/lib64/)
24+
set(crtm_LIB $ENV{CRTM_INSTALL}/lib64/)
25+
elseif(EXISTS $ENV{CRTM_INSTALL}/lib/)
26+
set(crtm_LIB $ENV{CRTM_INSTALL}/lib/)
27+
else()
28+
set(crtm_LIB $ENV{CRTM_INSTALL})
29+
endif()
1630

1731
if ("${CMAKE_Fortran_COMPILER_ID}" MATCHES "Intel")
1832
set(ompFlag "-liomp5")
33+
set(switch_fun "intelem")
34+
set(preproc "-fpp")
1935
else()
2036
set(ompFlag "-lgomp")
37+
set(switch_fun "gfortran")
38+
set(preproc "-cpp")
2139
endif()
22-
40+
set(f2py_module_name "pycrtm_")
2341
set(fortran_src_file "${CMAKE_CURRENT_SOURCE_DIR}/pycrtm.f90")
24-
set(f2py_module_name "pycrtm")
2542
set(generated_module_file ${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_name}${PYTHON_EXTENSION_MODULE_SUFFIX})
26-
2743
#_dumb is used to avoid circular dependency which breaks Ninja.
2844
add_custom_target(${f2py_module_name}_dumb ALL
2945
DEPENDS ${generated_module_file}
3046
)
31-
32-
add_custom_command(
33-
OUTPUT ${generated_module_file}
34-
COMMAND ${F2PY_EXECUTABLE}
35-
-m ${f2py_module_name}
36-
-h ${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_name}.pyf ${fortran_src_file}
37-
--overwrite-signature
38-
COMMAND ${F2PY_EXECUTABLE}
39-
-m ${f2py_module_name}
40-
-c
41-
${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_name}.pyf
42-
${fortran_src_file}
43-
-I${crtm_INCLUDE}
44-
-I${NetCDF_INCLUDE_DIRS}
45-
-L${CDF_LIB_PATH}
46-
-L${NetCDF_INCLUDE_DIRS}
47-
-L${crtm_LIB}
48-
-lcrtm
49-
-lnetcdff
50-
${ompFlag}
51-
only:
52-
wrap_forward
53-
wrap_k_matrix
54-
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
55-
)
47+
if (EXISTS $ENV{CRTM_INSTALL}/module/crtm/${CMAKE_Fortran_COMPILER_ID}/${CMAKE_Fortran_COMPILER_VERSION}/crtm_active_sensor.mod)
48+
set(preproc "${preproc} -DPYCRTM_ACTIVE")
49+
add_custom_command(
50+
OUTPUT ${generated_module_file}
51+
COMMAND python -m numpy.f2py
52+
-m ${f2py_module_name}
53+
-h ${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_name}.pyf ${fortran_src_file}
54+
COMMAND python -m numpy.f2py
55+
--fcompiler=${switch_fun}
56+
--f90flags=${preproc}
57+
-c
58+
${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_name}.pyf
59+
${fortran_src_file}
60+
-I${crtm_INCLUDE}
61+
-I${NETCDF_FORTRAN_INCLUDE_DIRS}
62+
-L${CDF_LIB_PATH}
63+
-L${crtm_LIB}
64+
-lcrtm
65+
-lnetcdff
66+
${ompFlag}
67+
# should work according to f2py documentation but doesn't. Have to set it through --f90flags as above.
68+
# -DPYCRTM_ACTIVE
69+
only:
70+
wrap_forward
71+
wrap_k_matrix
72+
wrap_forward_active
73+
wrap_k_matrix_active
74+
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
75+
)
76+
else()
77+
add_custom_command(
78+
OUTPUT ${generated_module_file}
79+
COMMAND python -m numpy.f2py
80+
-m ${f2py_module_name}
81+
-h ${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_name}.pyf ${fortran_src_file}
82+
COMMAND python -m numpy.f2py
83+
--fcompiler=${switch_fun}
84+
-c
85+
${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_name}.pyf
86+
${fortran_src_file}
87+
--f90flags=${preproc}
88+
-I${crtm_INCLUDE}
89+
-I${NETCDF_FORTRAN_INCLUDE_DIRS}
90+
-L${CDF_LIB_PATH}
91+
-L${crtm_LIB}
92+
-lcrtm
93+
-lnetcdff
94+
${ompFlag}
95+
only:
96+
wrap_forward
97+
wrap_k_matrix
98+
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
99+
)
100+
endif()
56101
#Shouldn't need this, but scikit-build won't grab the shared object otherwise.
57102
list(GET _Python3_INTERPRETER_PROPERTIES 6 version)
58103
set(produced_so ${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_name}.${version}.so)
59104
#Move to pycrtm directory
60-
install(FILES ${produced_so} DESTINATION pycrtm)
105+
if (EXISTS ${crtm_LIB}/libcrtm.so)
106+
set(libcrtm_so ${crtm_LIB}/libcrtm.so)
107+
endif()
108+
if (EXISTS ${crtm_LIB}/libcrtm.dylib)
109+
set(libcrtm_so ${crtm_LIB}/libcrtm.dylib)
110+
endif()
111+
112+
113+
#Move to pycrtm directory
114+
install(FILES ${produced_so} DESTINATION pycrtm_)
115+
install(FILES ${libcrtm_so} DESTINATION pycrtm_)
116+
#Move configuration into install
117+
install(FILES pycrtm_setup.txt DESTINATION pycrtm_)

README.md

Lines changed: 41 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
## Bryan M. Karpowicz, Ph.D. - USRA/GESTAR/NASA 610.1 Global Modeling and Assimilation Office, with Contributions from Patrick Stegmann, Dr.-Ing. - JCSDA
44

5-
This is a basic python interface to CRTM v2.4.0.
5+
This is a basic python interface to CRTM v2.4.x or CRTMv3.
66

77
The user interface is designed to be very similar to the python RTTOV interface. So, the user sets profiles, passes them to an object for a desired sensor, runs either the forward model/K-matrix, and pulls the brightness temperature/Jacobian/transmission/emissivity out of the object.
88

@@ -14,57 +14,47 @@ This `README` has 4 parts:
1414
3. Importing -- how to use this library in a project.
1515
4. Using the interface -- HOWTO/run through on how to use this interface
1616

17-
- Bryan Karpowicz -- March 20, 2021
17+
- Bryan Karpowicz -- May 7, 2025
1818
----------------------------------------------------------------------------------------
1919

2020
## 1. Installation:
21-
- For the novice that doesn't care about where or how this installs, look at the crtm-bundle and run kickstart_pyCRTM.sh Otherwise...
22-
- Dependencies CRTM, h5py, numpy and scikit-build (install those first, if you don't have them). Note crtm must be built with the static option (`ecbuild --static`)
23-
- Configuration
24-
First modify `setup.cfg` to point to the crtm install location (path underneath should contain `lib/libcrtm.a`).
21+
For a quicker install experience users may choose to install using the `make_it_so.sh` which will install conda along with the required packages in a miniconda environment `pycrtm`. There are four options `apple_silicon` for Macs with an M2/M3/M4/M? processor, `apple_intel` to install miniconda for Macs with an intel processor, `linux` for all other linux systems, `skip_install` which will skip installing miniconda and attempt to overwrite a `pycrtm` miniconda environment, `skip_create` which will use an existing miniconda pycrtm environment, and `skip_crtm_download` which will skip downloading CRTMv3 in addition to skipping the creation of a miniconda environment. Users are cautioned to look over the script to make sure it will not overwrite existing installs, or fill up your home directory, if space is limited. For example if your system does not have a python install and you a starting from scratch on an M4 Mac simply type:
2522
```
26-
[Setup]
27-
# Specify the location of the crtm install (ecbuild install ONLY)
28-
crtm_install = /discover/nobackup/bkarpowi/github/JCSDA_crtm/crtm-bundle/crtm/build
29-
# Download Coefficients
30-
# Controls whether coefficients are downloaded
31-
download = True
32-
#This will move the coefficients with the package install.
33-
coef_with_install = True
34-
[Coefficients]
35-
# Use to specify alternative coefficient file location where Little Endian Coefficient files are stored.
36-
# If user desires coefficients to be stored with the installed package, leave this alone.
37-
# If user selects coef_with_install = False, this must be specified.
38-
# set argument below (path) to the full path of the coefficients.
39-
path = /discover/nobackup/projects/gmao/obsdev/bkarpowi/tstCoef/
23+
./make_it_so.sh apple_silicon
4024
```
41-
In the example above the coefficients will be included with the pycrtm install. To change this, set `coef_with_install` and set `path` to the location where you would like crtm coefficients stored. If you already have a directory with coefficients, you can set `download` and `coef_with_install` to False, and set `path` to that location. The pycrtm configuration will then point to the location in `path`.
42-
43-
- Installation
44-
There are two recommended ways to install. The first, if the user has full write access to their python distribution, it may be installed globally using:
25+
The script will install miniconda3, CRTMv3, pyCRTM, and run the `test_atms.py` script to verify pyCRTM is working. If you already have miniconda on your machine you can simply run `skip_install` which will just install a pycrtm miniconda environment, CRTMv3, pyCRTM and run the `test_atms.py` script to verify pyCRTM has been installed and is functioning properly. Once installed a user may use the new `pycrtm` conda environment by typing:
4526
```
46-
python3 setup.py install
27+
conda activate pycrtm
4728
```
48-
This will take some time as it will download coefficients, move them around, compile the pycrtm module, and link against th crtm library.
4929

50-
The second, if the user doesn't have full write access to their python distribution is to first build a wheel, and install using pip:
51-
```
52-
python3 setup.py bdist_wheel
53-
```
54-
This will take some time as it will download coefficients, move them around, compile the pycrtm module, and link against the crtm library. Once the wheel has been built, it may be installed locally using pip:
30+
If that doesn't suit your taste, read on for a more step-by-step approach.
31+
32+
- Dependencies CRTM, h5py, numpy and scikit-build (install those first, if you don't have them).
33+
- Configuration
34+
First modify `setup.cfg` to point to the crtm install location (path underneath should contain one of the following: `lib/libcrtm.a`,`lib64/libcrtm.a`, `lib/libcrtm.so`, or `lib64/libcrtm.so`).
5535
```
56-
pip install dist/pyCRTM_JCSDA*.whl --target /discover/nobackup/projects/gmao/obsdev/bkarpowi/pythonModules/
36+
[Setup]
37+
# Specify the location of the crtm install
38+
crtm_install =/Users/karpob/pycrtm/ext/CRTMv3/build/
39+
link_from_source_to_path_used = True
40+
[Coefficients]
41+
# source specify coefficient directory will grab little endian binary coefficients and netcdf and link them to path_used
42+
source_path =/Users/karpob/pycrtm/ext/CRTMv3/build/test_data/
43+
# path used by pycrtm to read coefficients
44+
path_used =/Users/karpob/pycrtm/ext/crtm_coefficients
5745
```
58-
paired with appending `/discover/nobackup/projects/gmao/obsdev/bkarpowi/pythonModules/` to the `PYTHONPATH` environment variable in your .bashrc or .cshrc.
59-
60-
For Bash this is:
46+
Next, pycrtm must have a location where all desired coefficients are expanded in a flat directory. In the configuration above, the installer will create `path_used` and populate it with symbolic links to all available coefficients in `source_path.` If `link_from_source_to_path_used` is set to `False`, `source_path` will be ignored and it is assumed the user has placed coefficients in `path_used` and pyCRTM will search for coefficients in this directory.
47+
- Installation
48+
If the user has full write access to their python distribution, it may be installed globally using:
6149
```
62-
export PYTHONPATH="${PYTHONPATH}:/discover/nobackup/projects/gmao/obsdev/bkarpowi/pythonModules/"
50+
pip install .
6351
```
64-
For Tcsh/csh:
52+
Otherwise, the standard --user option is also available which will install under $HOME/.local/
6553
```
66-
setenv PYTHONPATH ${PYTHONPATH}:/discover/nobackup/projects/gmao/obsdev/bkarpowi/pythonModules
54+
pip install . --user
6755
```
56+
Optionally, you may supply "-v" for a more verbose output while it is installing. Either way, this will take some time as it will symlink coefficients downloaded when you install CRTM, compile the pycrtm module, and link against th crtm library.
57+
6858

6959
Compiler options are handled autmoatically through cmake. On HPC systems this means loading the right set of modules. For example, if you would like pycrtm compiled with intel, you would load the same intel modules you used to build crtm.
7060

@@ -91,6 +81,17 @@ The following scripts will run CRTM without aerosols or clouds:
9181
For those Jupyter notebook fans, there is even Jupyter notebook example simulating ATMS:
9282
* `$PWD/testCases/test_atms.ipynb`
9383

84+
Additonal More Advanced Examples:
85+
* `$PWD/testCases/test_atms_jacobian.py` Provides cloud jacobians (provide --plot command line argument to generate plot)
86+
* `$PWD/testCases/test_atms_subset_cloudnames.py` Provides example of using a channel subset, along with Cloud type names.
87+
* `$PWD/testCases/test_atms_subset.py` Provides exmaple using a channel subset.
88+
* `$PWD/testCases/test_cris_jacobian.py` Provides cloud/aerosol jacobians (provide --plot command line argument to generate plot)
89+
* `$PWD/testCases/test_cris_subset.py` Provides exmaple using a channel subset.
90+
91+
Active Sensor Examples (Available with CRTMv3.1.x)
92+
* `$PWD/testCases/test_cloudsat.py` tests forward model of active sensor (provide --plot for plot of reflectivity/attenuated reflectivity)
93+
* `$PWD/testCases/test_cloudsat_jacobian.py` tests cloud jacobians (provide --plot for cloud jacobian plot, --attenuated for attenuated reflectivity, otherwise jacobians of reflectivity are plotted.
94+
9495
## 3. Importing
9596

9697
```Python
@@ -108,7 +109,6 @@ Once initialized, the user will need to provide values for the desired profiles
108109

109110
```Python
110111
crtmOb = pyCRTM()
111-
crtmOb.coefficientPath = pathInfo['CRTM']['coeffs_dir']
112112
crtmOb.sensor_id = sensor_id
113113
crtmOb.nThreads = 4
114114
crtmOb.profiles = profiles
@@ -134,10 +134,10 @@ brightnessTemperature = crtmOb.Bt
134134
#Transmission (to compute weighting functions) ( nprofiles, nchan, nlayers)
135135
Tau = crtmOb.TauLevels
136136

137-
#Temperature, Water Vapo[u]r, and Ozone Jacobians ( npforfiles, nchan, nlayers)
137+
#Temperature, Water Vapo[u]r, and Ozone Jacobians ( nprofiles, nchan, nlayers)
138138
O3_Jacobian = crtmOb.O3K
139139
Water_Vapor_Jacobian = crtmOb.QK
140-
Temperature_Jacobian = crtm.TK
140+
Temperature_Jacobian = crtmOb.TK
141141

142142
#Emissivity (nprofiles, nchan)
143143
Emissivity = crtmOb.surfEmisRefl

0 commit comments

Comments
 (0)