Skip to content

Commit fc96a85

Browse files
committed
Merge branch 'devel' of github.com:I2PC/xmipp into ms_python_installer
2 parents 831775d + 3bdc457 commit fc96a85

31 files changed

+548
-267
lines changed

.github/workflows/release.yml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
name: Release new version
22
on:
3-
push:
4-
branches: [ master ]
53
workflow_dispatch:
64

75
jobs:
86
generate-new-release:
7+
environment:
8+
name: release-approval
9+
910
runs-on: ubuntu-latest
1011
steps:
1112

CHANGELOG.md

Lines changed: 41 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,52 @@
1-
## Release 3.24.X -
1+
## Release 4.0.0 - Apoferritin
2+
### Xmipp Programs
3+
- Programs updated
4+
- align volume and particles: Allow considering mirrors
5+
- miccCleaner: updated enviroment to tensorflow2
6+
7+
- Programs fixed
8+
- tranform geometry: Apply geometry program now considers the original geometry in the metadata when the parameter --shift_to is passed.
9+
- ‎angular_project_library: Fix indexing problem in angular_project_library
10+
- ctf_correct_phase fix and two constants for tomography were added
11+
- angular_project_library: Fix indexing bug
12+
13+
### Installation and user guide
14+
- Fixed issue with CUDA 12.5
15+
16+
### More Xmipp
17+
- Forced compiler GCC >= 9
18+
- getPSF added to the xmipp binding
19+
- replace "conda env export" to avoid deprecation issue.
20+
- Fixing read when encountering NaN-s and infinite values
21+
22+
23+
## Release 3.25.06.0 - Rhea
224
### Xmipp Programs
325
- New programs
426
- continuous_create_residuals: Make better residuals by continuous assignment updates to projections
27+
- tomo_detect_landmarks: to detect landmarks in tilt series (TS)
28+
- tomo_calculate_landmark_residuals: to calculate residuals in TS
29+
- tomo_detect_misalignment_residuals: to detect misalignment from residuals in TS
30+
- batch_tomo_misalignment_resid_statistics: Python script to calculate statistical metrics from a set of residuals
31+
- tomo_tiltseries_detect_misalignment_corr: (to legacy) for calculating relative shifts between tilt images, as a prealignment strategy
532

633
- Programs updated
7-
-
8-
34+
- Particle subtraction: now generates a single stack + includes noise power estimation
35+
- classify_pca: allows setting CUDA_DEVICE for NVIDIA drivers ≤ 470
36+
937
- Programs fixed
10-
-
11-
38+
- Fixed missing b in angular_continuous_assign2_gpu
39+
- CUDAFFT test fixed
40+
- Minor compatibility fixes for CUDA 12.6
41+
- DT_SChar added to micrographs
42+
1243
### Installation and user guide
13-
-
44+
- Fixed broken links in documentation
45+
- Enhanced data collection steps (context suggests improvements in reproducibility or logging)
1446

1547
### More Xmipp
48+
- DLTK environments: channels limited to conda-forge and pip. Updated deepEMhancer version and models, update all enviroments
49+
- Allowing to skip sorting in metadata_vec::split
1650

1751
## Release 3.24.12.0 - Poseidon
1852
### Xmipp Programs
@@ -23,7 +57,7 @@
2357
- angular_continuous_assign2_gpu: Make a continuous angular assignment with GPU
2458
- cuda_fourier_projection
2559
- predict_deep_center
26-
- cl2d_clustering.py
60+
- cl2d_clustering
2761

2862
- Programs updated
2963
- tomogram_reconstruction: averaging with gold standard, tigre with internal interpolation, enviroment of tigre updated

CMakeLists.txt

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,14 @@ if(NOT DEFINED CMAKE_CXX_STANDARD)
6565
set(CMAKE_CXX_STANDARD_REQUIRED ON)
6666
endif()
6767

68+
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
69+
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
70+
message(FATAL_ERROR "GCC version must be at least 9.0. Found ${CMAKE_CXX_COMPILER_VERSION}")
71+
endif()
72+
else()
73+
message(WARNING "You are not using GCC. Currenlty, we only officially support GCC.")
74+
endif()
75+
6876
# Find Python
6977
find_package(Python3 COMPONENTS Interpreter Development NumPy REQUIRED)
7078
file(APPEND ${XMIPP_VERSIONS_FILE} "Python3=${Python3_VERSION}\n")

README.md

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,7 @@ For developers, it becomes handy to install Xmipp in an external directory for e
2020

2121
The first step is to download Xmipp sources:
2222

23-
`git clone https://github.com/I2PC/xmipp.git`
24-
25-
`cd xmipp`
23+
`git clone https://github.com/I2PC/xmipp.git xmipp-bundle & cd xmipp-bundle`
2624

2725
If you want to checkout an scpecific branch use the following command. For repositories where the branch does not exist, devel is utilized.
2826

cmake/fetch_cifpp.cmake

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ function(fetch_cifpp)
3030

3131
FetchContent_Declare(
3232
libcifpp
33-
GIT_REPOSITORY https://github.com/MartinSalinas98/libcifpp.git
33+
GIT_REPOSITORY https://github.com/I2PC/libcifpp.git
3434
GIT_TAG ms_feature_ciflibrary
3535
)
3636
FetchContent_MakeAvailable(libcifpp)

src/xmipp/applications/scripts/deepRes_resolution/batch_deepRes_resolution.py

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,7 @@
2727

2828
from keras.models import load_model
2929
from keras.utils import Sequence
30-
import matplotlib.pyplot as plt
3130
import numpy as np
32-
import os
3331
import sys
3432
import argparse
3533
import xmippLib
@@ -116,11 +114,11 @@ def advance(self):
116114
if (self.x+self.y+self.z)%2==0:
117115
break
118116
ok=self.advancePos()
119-
return ok
117+
return ok
120118

121119

122120
def __getitem__(self,idx):
123-
count=0;
121+
count=0
124122
batchX = []
125123
ok = True
126124
while (count<maxSize and ok):
@@ -129,10 +127,12 @@ def __getitem__(self,idx):
129127
ok=self.advance()
130128
count+=1
131129
batchX=np.asarray(batchX).astype("float32")
132-
#print("count = ", count)
130+
#tf.print(f'count = {count}', output_stream=sys.stdout)
131+
133132
batchX = batchX.reshape(count, batchX.shape[1], batchX.shape[2], batchX.shape[3], 1)
134133

135-
#print("batchX.shape = ", batchX.shape)
134+
#tf.print(f'batchX.shape = {batchX.shape}', output_stream=sys.stdout)
135+
136136
return (batchX)
137137

138138

@@ -214,7 +214,7 @@ def produceOutput(fnVolInOrNumpy, fnMaskOrNumpy, model, sampling, Y, fnVolOut):
214214

215215
if fnVolOut is not None:
216216
Vxmipp = xmippLib.Image()
217-
Vxmipp.setData(V)
217+
Vxmipp.setData(V.astype(np.float32))
218218
Vxmipp.write(fnVolOut)
219219
return V
220220

@@ -226,8 +226,9 @@ def main(fnModel, fnVolIn, fnMask, sampling, fnVolOut):
226226
fnModel= XmippScript.getModel("deepRes", "model_w7.h5")
227227

228228
model = load_model(fnModel)
229-
manager = VolumeManager(fnVolIn, fnMask)
230-
Y = model.predict_generator(manager, manager.getNumberOfBlocks())
229+
#tf.print(f'load_model', output_stream=sys.stdout)
230+
Vmanager = VolumeManager(fnVolIn, fnMask)
231+
Y = model.predict_generator(Vmanager, steps=Vmanager.st)
231232

232233
if fnModel == XmippScript.getModel("deepRes", "model_w13.h5"):
233234
model = 1

src/xmipp/applications/scripts/deep_volume_postprocessing/deep_volume_postprocessing.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ def run(self):
9494
if self.checkParam('--checkpoint'):
9595
params += " --deepLearningModelPath %s "%os.path.expanduser(self.getParam("--checkpoint"))
9696
else:
97-
params += " --deepLearningModelPath %s "%XmippScript.getModel("deepEMhancer", "production_checkpoints/deepEMhancer_tightTarget.hd5")
97+
params += " --deepLearningModelPath %s "%XmippScript.getModel("deepEMhancer_v016", "production_checkpoints/deepEMhancer_tightTarget.hd5")
9898

9999
if self.checkParam('--sampling_rate'):
100100
params += " --samplingRate %f " % self.getDoubleParam('--sampling_rate')

src/xmipp/applications/scripts/volume_consensus/volume_consensus.py

Lines changed: 83 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,13 @@
2727
*
2828
**************************************************************************
2929
"""
30-
from os.path import splitext
3130
import math
31+
import os
32+
import itertools
3233
import numpy as np
3334
import pywt
3435
import pywt.data
36+
3537
from scipy.ndimage import zoom
3638
from xmipp_base import XmippScript
3739
import xmippLib
@@ -54,51 +56,105 @@ def run(self):
5456
outVolFn = self.getParam('-o')
5557
self.computeVolumeConsensus(inputFile, outVolFn)
5658

59+
def resize(self, image, dim):
60+
imageFt = np.fft.rfftn(image)
61+
resultFt = np.zeros(dim[:-1] + (dim[-1]//2+1,), dtype=imageFt.dtype)
62+
63+
copyExtent = np.minimum(image.shape, dim) // 2
64+
srcCornerStart = image.shape-copyExtent
65+
dstCornerStart = dim-copyExtent
66+
for corners in itertools.product(range(2), repeat=len(dim)-1):
67+
corners = np.array(corners + (0, ))
68+
srcStart = np.where(corners, srcCornerStart, 0)
69+
srcEnd = srcStart + copyExtent
70+
dstStart = np.where(corners, dstCornerStart, 0)
71+
dstEnd = dstStart + copyExtent
72+
srcSlices = [slice(s, e) for s, e in zip(srcStart, srcEnd)]
73+
dstSlices = [slice(s, e) for s, e in zip(dstStart, dstEnd)]
74+
resultFt[tuple(dstSlices)] = imageFt[tuple(srcSlices)]
75+
76+
return np.fft.irfftn(resultFt)
77+
5778
def computeVolumeConsensus(self, inputFile, outVolFn, wavelet='sym11'):
5879
outputWt = None
5980
outputMin = None
6081
xdim2 = None
6182
xdimOrig = None
83+
image = xmippLib.Image()
6284
with open(inputFile) as f:
6385
for line in f:
6486
fileName = line.split()[0]
6587
if fileName.endswith('.mrc'):
6688
fileName += ':mrc'
67-
V = xmippLib.Image(fileName)
68-
vol = V.getData()
89+
90+
image.read(fileName)
91+
volume = image.getData()
92+
6993
if xdimOrig is None:
70-
xdimOrig = vol.shape[0]
71-
xdim2 = 2**(math.ceil(math.log(xdimOrig, 2))) # Next power of 2
72-
ydimOrig = vol.shape[1]
73-
ydim2 = 2 ** (math.ceil(math.log(ydimOrig, 2))) # Next power of 2
74-
zdimOrig = vol.shape[2]
75-
zdim2 = 2 ** (math.ceil(math.log(zdimOrig, 2))) # Next power of 2
94+
xdimOrig = volume.shape[0]
95+
xdim2 = 2**(math.ceil(math.log2(xdimOrig))) # Next power of 2
96+
ydimOrig = volume.shape[1]
97+
ydim2 = 2**(math.ceil(math.log2(ydimOrig))) # Next power of 2
98+
zdimOrig = volume.shape[2]
99+
zdim2 = 2**(math.ceil(math.log2(zdimOrig))) # Next power of 2
100+
76101
if xdimOrig!=xdim2 or ydimOrig!=ydim2 or zdimOrig!=zdim2:
77-
vol = zoom(vol, (xdim2/xdimOrig,ydim2/ydimOrig,zdim2/zdimOrig))
78-
nlevel = pywt.swt_max_level(len(vol))
79-
wt = pywt.swtn(vol, wavelet, nlevel, 0)
102+
#volume = zoom(volume, (xdim2/xdimOrig,ydim2/ydimOrig,zdim2/zdimOrig))
103+
volume = self.resize(volume, (zdim2, ydim2, xdim2))
104+
105+
nlevel = pywt.dwtn_max_level(volume.shape, wavelet=wavelet)
106+
wt = pywt.wavedecn(
107+
data=volume,
108+
wavelet=wavelet,
109+
level=nlevel
110+
)
111+
80112
if outputWt == None:
81113
outputWt = wt
82-
outputMin = wt[0]['aaa']*0
114+
outputMin = np.zeros_like(volume)
83115
else:
84-
for level in range(0, nlevel):
116+
diff = np.abs(np.abs(wt[0]) - np.abs(outputWt[0]))
117+
diff = self.resize(diff, outputMin.shape)
118+
np.maximum(
119+
diff, outputMin,
120+
out=outputMin
121+
)
122+
outputWt[0] = np.where(
123+
np.abs(wt[0]) > np.abs(outputWt[0]),
124+
wt[0], outputWt[0]
125+
)
126+
127+
for level in range(1, nlevel+1):
85128
wtLevel = wt[level]
86129
outputWtLevel = outputWt[level]
87-
for key in wtLevel:
88-
outputWtLevel[key] = np.where(np.abs(outputWtLevel[key]) > np.abs(wtLevel[key]),
89-
outputWtLevel[key], wtLevel[key])
90-
diff = np.abs(np.abs(outputWtLevel[key]) - np.abs(wtLevel[key]))
91-
outputMin = np.where(outputMin > diff, outputMin, diff)
130+
for detail in wtLevel:
131+
wtLevelDetail = wtLevel[detail]
132+
outputWtLevelDetail = outputWtLevel[detail]
133+
134+
diff = np.abs(np.abs(wtLevelDetail) - np.abs(outputWtLevelDetail))
135+
diff = self.resize(diff, outputMin.shape)
136+
np.maximum(
137+
diff, outputMin,
138+
out=outputMin
139+
)
140+
141+
outputWtLevelDetail[...] = np.where(
142+
np.abs(wtLevelDetail) > np.abs(outputWtLevelDetail),
143+
wtLevelDetail, outputWtLevelDetail
144+
)
145+
146+
92147
f.close()
93-
consensus = pywt.iswtn(outputWt, wavelet)
148+
consensus = pywt.waverecn(outputWt, wavelet)
149+
if xdimOrig!=xdim2 or ydimOrig!=ydim2 or zdimOrig!=zdim2:
150+
consensus = self.resize(consensus, (zdimOrig, ydimOrig, xdimOrig))
151+
image.setData(consensus)
152+
image.write(outVolFn)
94153
if xdimOrig!=xdim2 or ydimOrig!=ydim2 or zdimOrig!=zdim2:
95-
consensus = zoom(consensus, (xdimOrig/xdim2,ydimOrig/ydim2,zdimOrig/zdim2))
96-
V = xmippLib.Image()
97-
V.setData(consensus)
98-
V.write(outVolFn)
99-
V.setData(outputMin)
100-
outVolFn2 = splitext(outVolFn)[0] + '_diff.mrc'
101-
V.write(outVolFn2)
154+
outputMin = self.resize(outputMin, (zdimOrig, ydimOrig, xdimOrig))
155+
image.setData(outputMin)
156+
outVolFn2 = os.path.splitext(outVolFn)[0] + '_diff.mrc'
157+
image.write(outVolFn2)
102158

103159

104160
if __name__=="__main__":
Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,21 @@
11
#Protocols ussing the enviroment:
2-
#protocol_screen_deeplearning
3-
#protocol_deep_denoising
2+
43
#protocol_resolution_deepres
54
#protocol_screen_deepConsensus
65

76
name: xmipp_DLTK_v0.3
87
channels:
9-
- anaconda
10-
- defaults
118
- conda-forge
129
dependencies:
1310
- python=3.7
14-
- scikit-image=0.14
15-
- tensorflow-gpu=1.15
16-
- keras=2.2
17-
- scikit-learn=0.22
18-
- libstdcxx-ng>=14.2.0 # To be compatible with scipion env
11+
- libstdcxx-ng # To be compatible with scipion env
12+
- cudnn=7.6
13+
- cudatoolkit==10.0.130
1914
- pip
2015
- pip:
21-
- numpy==1.21
16+
- tensorflow-gpu==1.15.0
17+
- numpy==1.19.5
2218
- h5py==2.10.0
19+
- keras==2.2.5
20+
- scikit-learn==0.22
21+
- protobuf==3.20.3
Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,21 @@
11
#Protocols ussing the enviroment:
2-
#protocol_screen_deeplearning
3-
#protocol_deep_denoising
2+
43
#protocol_resolution_deepres
54
#protocol_screen_deepConsensus
65

76
name: xmipp_DLTK_v0.3
87
channels:
9-
- anaconda
10-
- defaults
118
- conda-forge
12-
139
dependencies:
1410
- python=3.7
15-
- scikit-image=0.14
16-
- tensorflow=1.15
17-
- keras=2.2
18-
- scikit-learn=0.22
11+
- libstdcxx-ng # To be compatible with scipion env
12+
- cudnn=7.6
13+
- cudatoolkit==10.0.130
1914
- pip
20-
- libstdcxx-ng>=14.2.0 # To be compatible with scipion env
2115
- pip:
22-
- numpy==1.21
16+
- tensorflow==1.15.0
17+
- numpy==1.19.5
2318
- h5py==2.10.0
19+
- keras==2.2.5
20+
- scikit-learn==0.22
21+
- protobuf==3.20.3

0 commit comments

Comments
 (0)