Skip to content

Commit 0ee6036

Browse files
SpheMakhSpheMakh
authored
Merge pull request #452 from SpheMakh/podman
Podman
2 parents 84d9fa8 + ac8e01a commit 0ee6036

292 files changed

Lines changed: 4896 additions & 3948 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎CHANGES.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,7 @@
4343
- Add DDFacet cab!!
4444
- Fix --us-only option in 'stimela build'
4545
- Upgrade to kern-2 (wsclean is using kern-dev)
46+
47+
# 1.2.0
48+
- add support for podman
49+
- upgrade CASA to 5.6.0

‎Jenkinsfile.sh‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,16 @@ mkdir $TEST_OUTPUT_DIR
88
#Custom home for this run's temporary stuff
99
HOME=$WORKSPACE_ROOT
1010
export HOME
11+
SINGULARITY_STORAGE="${WORKSPACE}/../../../.singularity"
1112
ln -s $WORKSPACE/../../../.udocker ${WORKSPACE_ROOT}/.udocker
12-
ln -s $WORKSPACE/../../../.singularity ${WORKSPACE_ROOT}/.singularity
13+
ln -s ${SINGULARITY_STORAGE} ${WORKSPACE_ROOT}/.singularity
14+
15+
# setup podman image storage. Using .singularity volume
16+
POD_STORAGE="${WORKSPACE_ROOT}/.local/share/containers/storage"
17+
mkdir -p ${POD_STORAGE} ${SINGULARITY_STORAGE}/podman
18+
ln -s ${SINGULARITY_STORAGE}/podman ${POD_STORAGE}
19+
20+
1321
# Install Stimela into a virtual env
1422
virtualenv ${WORKSPACE_ROOT}/projects/pyenv
1523
. ${WORKSPACE_ROOT}/projects/pyenv/bin/activate
@@ -24,13 +32,14 @@ tar -xzvf $TEST_DATA_DIR/kat-7-small.ms.tar.gz -C $TEST_OUTPUT_DIR/msdir
2432
mkdir $TEST_OUTPUT_DIR/input
2533
cp -r $TEST_DATA_DIR/beams $TEST_OUTPUT_DIR/input/beams
2634

27-
which stimela
2835
stimela --version
2936
singularity -v
3037
docker -v
3138
export SINGULARITY_PULLFOLDER=${WORKSPACE_ROOT}/singularity_images
3239
mkdir $SINGULARITY_PULLFOLDER
33-
stimela pull -s
40+
stimela pull -s -cb simms
41+
stimela pull -cb tricolour
42+
#stimela pull -p -cb casa_listobs
3443
# fresh build
3544
stimela build -nc
3645

‎bin/stimela‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,11 @@ import os
44
import sys
55

66
import stimela
7+
import stimela.main as main
78

89
# Ensure that logging infrastructure exists
910

1011
if not os.path.exists(stimela.LOG_HOME):
1112
os.mkdir(stimela.LOG_HOME)
1213

13-
stimela.main([a for a in sys.argv[1:]])
14+
main.main([a for a in sys.argv[1:]])

‎examples/simulation_pipeline.py‎

Lines changed: 65 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,15 @@
33
import os
44

55
# Recipe I/O configuration
6-
INPUT = "input" # This folder must exist
6+
INPUT = "input" # This folder must exist
77
OUTPUT = "output"
88
MSDIR = "msdir"
99
PREFIX = "stimela-example" # Prefix for output images
1010
try:
1111
SINGULARTITY_IMAGE_DIR = os.environ["STIMELA_SINGULARTITY_IMAGE_DIR"]
1212
except KeyError:
1313
SINGULARTITY_IMAGE_DIR = None
14-
14+
1515

1616
# MS name
1717
MS = "meerkat_simulation_example.ms"
@@ -22,25 +22,25 @@
2222

2323
# Start stimela Recipe instance
2424
pipeline = stimela.Recipe("Simulation Example", # Recipe name
25-
ms_dir=MSDIR,
26-
singularity_image_dir=SINGULARTITY_IMAGE_DIR,
27-
log_dir = os.path.join(OUTPUT, "logs")
28-
)
25+
ms_dir=MSDIR,
26+
singularity_image_dir=SINGULARTITY_IMAGE_DIR,
27+
log_dir=os.path.join(OUTPUT, "logs")
28+
)
2929

30-
pipeline.JOB_TYPE = "udocker"
30+
pipeline.JOB_TYPE = "docker"
3131

32-
# 1: Make empty MS
33-
pipeline.add("cab/simms", # Executor image to start container from
32+
# 1: Make empty MS
33+
pipeline.add("cab/simms", # Executor image to start container from
3434
"simms_example", # Container name
35-
{ # Parameters to parse to executor container
36-
"msname" : MS,
37-
"telescope" : "meerkat", # Telescope name
38-
"direction" : "J2000,0deg,-30deg", # Phase tracking centre of observation
39-
"synthesis" : 0.128, # Synthesis time of observation
40-
"dtime" : 10, # Integration time in seconds
41-
"freq0" : "750MHz", # Start frequency of observation
42-
"dfreq" : "1MHz", # Channel width
43-
"nchan" : 1 # Number of channels
35+
{ # Parameters to parse to executor container
36+
"msname": MS,
37+
"telescope": "meerkat", # Telescope name
38+
"direction": "J2000,0deg,-30deg", # Phase tracking centre of observation
39+
"synthesis": 0.128, # Synthesis time of observation
40+
"dtime": 10, # Integration time in seconds
41+
"freq0": "750MHz", # Start frequency of observation
42+
"dfreq": "1MHz", # Channel width
43+
"nchan": 1 # Number of channels
4444
},
4545
input=INPUT, # Input folder
4646
output=OUTPUT, # Output folder
@@ -50,70 +50,70 @@
5050

5151

5252
# 2: Simulate visibilities into it
53-
pipeline.add("cab/simulator",
53+
pipeline.add("cab/simulator",
5454
"simulator_example",
5555
{
56-
"msname" : MS,
57-
"skymodel" : LSM, # Sky model to simulate into MS
58-
"addnoise" : True, # Add thermal noise to visibilities
59-
"column" : "CORRECTED_DATA", # Simulated data will be saved in this column
60-
"sefd" : 831, # Compute noise from this SEFD
61-
"recenter" : True, # Recentre sky model to phase tracking centre of MS
62-
"tile-size" : 64,
63-
"threads" : 4,
56+
"msname": MS,
57+
"skymodel": LSM, # Sky model to simulate into MS
58+
"addnoise": True, # Add thermal noise to visibilities
59+
"column": "CORRECTED_DATA", # Simulated data will be saved in this column
60+
"sefd": 831, # Compute noise from this SEFD
61+
# Recentre sky model to phase tracking centre of MS
62+
"recenter": True,
63+
"tile-size": 64,
64+
"threads": 4,
65+
},
66+
input=INPUT, output=OUTPUT,
67+
label="Simulating visibilities")
68+
69+
70+
pipeline.add('cab/casa_plotms',
71+
'plot_vis',
72+
{
73+
"vis": MS,
74+
"xaxis": 'uvdist',
75+
"yaxis": 'amp',
76+
"xdatacolumn": 'corrected',
77+
"ydatacolumn": 'corrected',
78+
"plotfile": PREFIX+'-amp_uvdist.png',
79+
"overwrite": True,
6480
},
65-
input=INPUT, output=OUTPUT,
66-
label="Simulating visibilities")
67-
68-
69-
pipeline.add('cab/casa_plotms',
70-
'plot_vis',
71-
{
72-
"vis" : MS,
73-
"xaxis" : 'uvdist',
74-
"yaxis" : 'amp',
75-
"xdatacolumn" : 'corrected',
76-
"ydatacolumn" : 'corrected',
77-
"plotfile" : PREFIX+'-amp_uvdist.png',
78-
"overwrite" : True,
79-
},
80-
input=INPUT,
81-
output=OUTPUT,
82-
label='plot_amp_uvdist:: Plot amplitude vs uv-distance')
83-
84-
## 3: Image
85-
# Make things a bit interesting by imaging with different weights
81+
input=INPUT,
82+
output=OUTPUT,
83+
label='plot_amp_uvdist:: Plot amplitude vs uv-distance')
84+
85+
# 3: Image
86+
# Make things a bit interesting by imaging with different weights
8687
# Briggs robust values to use for each image
8788
briggs_robust = 2, 0, -2
8889

8990
for i, robust in enumerate(briggs_robust):
9091

9192
pipeline.add("cab/wsclean",
92-
"imager_example_robust_{:d}".format(i),
93+
"imager_example_robust_{:d}".format(i),
9394
{
94-
"msname" : MS,
95-
"weight" : "briggs {:d}".format(i),
96-
"prefix" : "{:s}_robust-{:d}".format(PREFIX, robust),
97-
"npix" : 2048, # Image size in pixels
98-
"cellsize" : 2, # Size of each square pixel
99-
"clean_iterations" : 1000, # Perform 1000 iterarions of clean (Deconvolution)
95+
"msname": MS,
96+
"weight": "briggs {:d}".format(robust),
97+
"prefix": "{:s}_robust-{:d}".format(PREFIX, robust),
98+
"npix": 2048, # Image size in pixels
99+
"cellsize": 2, # Size of each square pixel
100+
# Perform 1000 iterarions of clean (Deconvolution)
101+
"clean_iterations": 1000,
100102
},
101-
input=INPUT,
102-
output=OUTPUT,
103-
label="Imaging MS, robust={:d}".format(robust),
103+
input=INPUT,
104+
output=OUTPUT,
105+
label="Imaging MS, robust={:d}".format(robust),
104106
cpus=2,
105107
memory_limit="2gb")
106108

107109
pipeline.add("cab/casa_rmtables", "delete_ms", {
108-
"tablenames" : MS + ":msfile",
109-
},
110-
input=INPUT,
111-
output=OUTPUT,
112-
label="Remove MS")
113-
110+
"tablenames": MS + ":msfile",
111+
},
112+
input=INPUT,
113+
output=OUTPUT,
114+
label="Remove MS")
114115

115116

116117
# Run recipe. The 'steps' added above will be executed in the sequence that they were adde. The 'steps' added above will be
117118
# executed in the sequence that they were added
118119
pipeline.run()
119-

‎setup.py‎

Lines changed: 31 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -5,38 +5,42 @@
55
from setuptools import setup
66

77

8-
requirements = ["pyyaml",
8+
requirements = ["pyyaml",
99
"nose>=1.3.7",
1010
"future",
11-
],
11+
],
1212

1313
if sys.version_info <= (3, 0):
1414
requirements += [
15-
"udocker",
16-
],
15+
"udocker",
16+
],
1717

1818
PACKAGE_NAME = "stimela"
19-
__version__ = "1.1.5"
19+
__version__ = "1.2.0"
2020

21-
setup(name = PACKAGE_NAME,
22-
version = __version__,
23-
description = "Dockerized radio interferometry scripting framework",
24-
author = "Sphesihle Makhathini",
25-
author_email = "sphemakh@gmail.com",
26-
url = "https://github.com/sphemakh/Stimela",
27-
packages = ["stimela","stimela_misc", "stimela/cargo","stimela/utils", "stimela/cargo/cab"],
28-
package_data = { "stimela/cargo" : [
29-
"cab/*/Dockerfile",
30-
"base/*/Dockerfile",
31-
"cab/*/src/*.py",
32-
"cab/*/src/*.sh",
33-
"cab/*/src/*.json",
34-
"cab/*/xvfb.init.d",
35-
"cab/*/parameters.json",
36-
"cab/*/src/tdlconf.profiles",
37-
"cab/singularity_run",
38-
]},
39-
install_requires = requirements,
40-
scripts = ["bin/" + i for i in os.listdir("bin")],
41-
classifiers = [],
42-
)
21+
setup(name=PACKAGE_NAME,
22+
version=__version__,
23+
description="Dockerized radio interferometry scripting framework",
24+
author="Sphesihle Makhathini",
25+
author_email="sphemakh@gmail.com",
26+
url="https://github.com/sphemakh/Stimela",
27+
packages=["stimela", "stimela_misc", "stimela/cargo",
28+
"stimela/utils", "stimela/cargo/cab"],
29+
include_package_data = True,
30+
package_data={"stimela/cargo": [
31+
"cab/*/Dockerfile",
32+
"base/*/Dockerfile",
33+
"cab/*/src/*.py",
34+
"cab/*/src/*.sh",
35+
"cab/*/src/*.json",
36+
"base/*/xvfb.init.d",
37+
"cab/*/xvfb.init.d",
38+
"cab/*/parameters.json",
39+
"cab/*/src/tdlconf.profiles",
40+
"cab/singularity_run",
41+
"cab/docker_run",
42+
]},
43+
install_requires=requirements,
44+
scripts=["bin/" + i for i in os.listdir("bin")],
45+
classifiers=[],
46+
)

0 commit comments

Comments
 (0)