Skip to content

Commit 0152800

Browse files
Critsium-xyclaude
andcommitted
Refactor: inject nlocal/nbands into the LCAO dense diagonalizers
Group D of the source_hsolver PARAM removal, and the largest single cluster: the six dense LCAO diagonalizers all sized their eigenvalue buffers from PARAM.globalv.nlocal and copied out PARAM.inp.nbands eigenvalues. Each of DiagoLapack, DiagoScalapack, DiagoElpa, DiagoElpaNative, DiagoCusolver and DiagoCusolverMP now takes (nlocal, nbands) through its constructor and stores them as members. DiagoElpaNative additionally takes use_gpu, replacing its PARAM.inp.device == "gpu" test. HSolverLCAO gains nbands and use_gpu (it already received nlocal in deepmodeling#7711) and supplies all eleven construction sites across hamiltSolvePsiK, parakSolve and parakSolve_cusolver. The three call contexts agree on both values: parakSolve distributes the same nlocal x nlocal matrix over a smaller pool grid, and parakSolve_cusolver gathers it per k-point, so a single pair covers all of them. nbands is injected rather than read from ParaV->get_nbands(). The two are equal for the LCAO path (LCAO_init_basis.cpp seeds ParaV from PARAM.inp.nbands), but module_lr sets paraMat_.nbands to nocc + nvirt, so sourcing it from ParaV would plant a trap for any future caller. The existing ParaV->get_nbands() reads inside parakSolve are left alone; they are not PARAM reads and are out of scope. In diago_scalapack.cpp, four call sites pass the address of the value to the Fortran routines. PARAM.globalv returns a const reference, so &PARAM.globalv.nlocal was already a const int*, matching pdsygvx_/pzhegvx_'s const int* n; &this->nlocal has the same type. Also resolves the two error-message strings in diago_lapack.cpp and diago_scalapack.cpp that embedded "PARAM.inp.nbands = ". These were deferred from deepmodeling#7706 because removing them needed exactly the nbands member added here. Two dead commented-out ELPA_Solver calls in diago_elpa.cpp are removed rather than updated; each duplicated the live statement two lines below it. The three LCAO diagonalizer tests supply their own nlocal/nbands, which they already read from the H/S input files, so their PARAM.sys.nlocal / PARAM.input.nbands writes and the now-dead `#define private public` include blocks are dropped. PARAM occurrences in source_hsolver production code: 60 -> 5. What remains is DiagoIterAssist's basis_type/calculation switches (group E) and one comment in simple_pexsi.cpp. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1 parent c5e066e commit 0152800

21 files changed

Lines changed: 137 additions & 113 deletions

source/source_esolver/esolver_ks_lcao.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -438,7 +438,9 @@ void ESolver_KS_LCAO<TK, TR>::hamilt2rho_single(UnitCell& ucell, int istep, int
438438
PARAM.inp.ks_solver,
439439
PARAM.globalv.kpar_lcao,
440440
PARAM.globalv.nlocal,
441-
PARAM.inp.nelec);
441+
PARAM.inp.nbands,
442+
PARAM.inp.nelec,
443+
PARAM.inp.device == "gpu");
442444
hsolver_lcao_obj.solve(static_cast<hamilt::Hamilt<TK>*>(this->p_hamilt), this->psi[0], this->pelec, *this->dmat.dm,
443445
this->chr, PARAM.inp.nspin, skip_charge);
444446
}

source/source_esolver/esolver_ks_lcao_tddft.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -333,7 +333,9 @@ void ESolver_KS_LCAO_TDDFT<TR, Device>::hamilt2rho_single(UnitCell& ucell,
333333
PARAM.inp.ks_solver,
334334
PARAM.globalv.kpar_lcao,
335335
PARAM.globalv.nlocal,
336-
PARAM.inp.nelec);
336+
PARAM.inp.nbands,
337+
PARAM.inp.nelec,
338+
PARAM.inp.device == "gpu");
337339
hsolver_lcao_obj.solve(static_cast<hamilt::Hamilt<std::complex<double>>*>(this->p_hamilt),
338340
this->psi[0],
339341
this->pelec,

source/source_hsolver/diago_cusolver.cpp

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
#include "source_base/module_external/scalapack_connector.h"
77
#include "source_base/tool_title.h"
88
#include "source_base/timer.h"
9-
#include "source_io/module_parameter/parameter.h"
109

1110
#include <memory>
1211
#include <type_traits>
@@ -22,7 +21,7 @@ template <typename T>
2221
int DiagoCusolver<T>::DecomposedState = 0;
2322

2423
template <typename T>
25-
DiagoCusolver<T>::DiagoCusolver()
24+
DiagoCusolver<T>::DiagoCusolver(const int nlocal_in, const int nbands_in) : nlocal(nlocal_in), nbands(nbands_in)
2625
{
2726
}
2827

@@ -42,13 +41,13 @@ void DiagoCusolver<T>::diag(
4241
ModuleBase::TITLE("DiagoCusolver", "diag");
4342
ModuleBase::timer::start("DiagoCusolver", "cusolver");
4443
// Allocate memory for eigenvalues
45-
std::vector<double> eigen(PARAM.globalv.nlocal, 0.0);
44+
std::vector<double> eigen(this->nlocal, 0.0);
4645
std::vector<T> eigenvectors(h_mat.row * h_mat.col);
4746
this->dc.Dngvd(h_mat.row, h_mat.col, h_mat.p, s_mat.p, eigen.data(), eigenvectors.data());
4847
const int size = psi.get_nbands() * psi.get_nbasis();
4948
BlasConnector::copy(size, eigenvectors.data(), 1, psi.get_pointer(), 1);
5049
const int inc = 1;
51-
BlasConnector::copy(PARAM.inp.nbands, eigen.data(), inc, eigenvalue_in, inc);
50+
BlasConnector::copy(this->nbands, eigen.data(), inc, eigenvalue_in, inc);
5251
ModuleBase::timer::end("DiagoCusolver", "cusolver");
5352
}
5453

source/source_hsolver/diago_cusolver.h

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,11 @@ class DiagoCusolver
1919

2020
public:
2121

22-
DiagoCusolver();
22+
/// @param nlocal_in global dimension of the NAO Hamiltonian
23+
/// @param nbands_in number of lowest eigenpairs to compute
24+
DiagoCusolver(const int nlocal_in, const int nbands_in);
2325
~DiagoCusolver();
24-
26+
2527
// Override the diag function for CUSOLVER diagonalization
2628
void diag(
2729
hamilt::MatrixBlock<T>& h_mat,
@@ -40,6 +42,9 @@ class DiagoCusolver
4042
// Function to check if ELPA handle needs to be created or reused in MPI settings
4143
bool ifElpaHandle(const bool& newIteration, const bool& ifNSCF) const;
4244
#endif
45+
46+
const int nlocal;
47+
const int nbands;
4348
};
4449

4550
} // namespace hsolver

source/source_hsolver/diago_cusolvermp.cpp

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
#ifdef __CUSOLVERMP
22

3-
#include "source_io/module_parameter/parameter.h"
43
#include "diago_cusolvermp.h"
54

65
#include "source_base/module_external/blas_connector.h"
@@ -18,7 +17,7 @@ void DiagoCusolverMP<T>::diag(hamilt::Hamilt<T>* phm_in, psi::Psi<T>& psi, Real*
1817
hamilt::MatrixBlock<T> h_mat, s_mat;
1918
phm_in->matrix(h_mat, s_mat);
2019

21-
std::vector<Real> eigen(PARAM.globalv.nlocal, 0.0);
20+
std::vector<Real> eigen(this->nlocal, 0.0);
2221
std::vector<T> eigenvectors(h_mat.row * h_mat.col);
2322

2423
MPI_Comm COMM_DIAG = MPI_COMM_WORLD; // use all processes
@@ -30,7 +29,7 @@ void DiagoCusolverMP<T>::diag(hamilt::Hamilt<T>* phm_in, psi::Psi<T>& psi, Real*
3029
ModuleBase::timer::end("DiagoCusolverMP", "Diag_CusolverMP_gvd");
3130
}
3231
const int inc = 1;
33-
BlasConnector::copy(PARAM.inp.nbands, eigen.data(), inc, eigenvalue_in, inc);
32+
BlasConnector::copy(this->nbands, eigen.data(), inc, eigenvalue_in, inc);
3433
const int size = psi.get_nbands() * psi.get_nbasis();
3534
BlasConnector::copy(size, eigenvectors.data(), inc, psi.get_pointer(), inc);
3635
}

source/source_hsolver/diago_cusolvermp.h

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,17 @@ class DiagoCusolverMP
1616
using Real = typename GetTypeReal<T>::type;
1717

1818
public:
19-
DiagoCusolverMP()
19+
/// @param nlocal_in global dimension of the NAO Hamiltonian
20+
/// @param nbands_in number of lowest eigenpairs to compute
21+
DiagoCusolverMP(const int nlocal_in, const int nbands_in) : nlocal(nlocal_in), nbands(nbands_in)
2022
{
2123
}
2224
// the diag function for CUSOLVERMP diagonalization
2325
void diag(hamilt::Hamilt<T>* phm_in, psi::Psi<T>& psi, Real* eigenvalue_in);
26+
27+
private:
28+
const int nlocal;
29+
const int nbands;
2430
};
2531
} // namespace hsolver
2632
#endif // __CUSOLVERMP

source/source_hsolver/diago_elpa.cpp

Lines changed: 12 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
#include "source_base/global_function.h"
33
#include "source_base/module_external/blas_connector.h"
44

5-
#include "source_io/module_parameter/parameter.h"
65
#include "module_genelpa/elpa_solver.h"
76
#include "source_base/module_external/blacs_connector.h"
87
#include "source_base/global_variable.h"
@@ -75,13 +74,13 @@ void DiagoElpa<std::complex<double>>::diag(
7574
matcd h_mat, s_mat;
7675
phm_in->matrix(h_mat, s_mat);
7776

78-
std::vector<double> eigen(PARAM.globalv.nlocal, 0.0);
77+
std::vector<double> eigen(this->nlocal, 0.0);
7978

8079
bool isReal = false;
8180
MPI_Comm COMM_DIAG = setmpicomm(); // set mpi_comm needed
8281
ELPA_Solver es((const bool)isReal,
8382
COMM_DIAG,
84-
(const int)PARAM.inp.nbands,
83+
(const int)this->nbands,
8584
(const int)h_mat.row,
8685
(const int)h_mat.col,
8786
(const int*)h_mat.desc);
@@ -97,7 +96,7 @@ void DiagoElpa<std::complex<double>>::diag(
9796
es.exit();
9897

9998
const int inc = 1;
100-
BlasConnector::copy(PARAM.inp.nbands, eigen.data(), inc, eigenvalue_in, inc);
99+
BlasConnector::copy(this->nbands, eigen.data(), inc, eigenvalue_in, inc);
101100
#else
102101
ModuleBase::WARNING_QUIT("DiagoElpa",
103102
"DiagoElpa only can be used with macro __MPI");
@@ -113,15 +112,13 @@ void DiagoElpa<double>::diag(hamilt::Hamilt<double>* phm_in,
113112
matd h_mat, s_mat;
114113
phm_in->matrix(h_mat, s_mat);
115114

116-
std::vector<double> eigen(PARAM.globalv.nlocal, 0.0);
115+
std::vector<double> eigen(this->nlocal, 0.0);
117116

118117
bool isReal = true;
119118
MPI_Comm COMM_DIAG = setmpicomm(); // set mpi_comm needed
120-
// ELPA_Solver es(isReal, COMM_DIAG, PARAM.inp.nbands, h_mat.row, h_mat.col,
121-
// h_mat.desc);
122119
ELPA_Solver es((const bool)isReal,
123120
COMM_DIAG,
124-
(const int)PARAM.inp.nbands,
121+
(const int)this->nbands,
125122
(const int)h_mat.row,
126123
(const int)h_mat.col,
127124
(const int*)h_mat.desc);
@@ -135,7 +132,7 @@ void DiagoElpa<double>::diag(hamilt::Hamilt<double>* phm_in,
135132
es.exit();
136133

137134
const int inc = 1;
138-
BlasConnector::copy(PARAM.inp.nbands, eigen.data(), inc, eigenvalue_in, inc);
135+
BlasConnector::copy(this->nbands, eigen.data(), inc, eigenvalue_in, inc);
139136
#else
140137
ModuleBase::WARNING_QUIT("DiagoElpa",
141138
"DiagoElpa only can be used with macro __MPI");
@@ -151,11 +148,11 @@ void DiagoElpa<std::complex<double>>::diag_pool(hamilt::MatrixBlock<std::complex
151148
Real* eigenvalue_in,
152149
MPI_Comm& comm)
153150
{
154-
std::vector<double> eigen(PARAM.globalv.nlocal, 0.0);
151+
std::vector<double> eigen(this->nlocal, 0.0);
155152
bool isReal = false;
156153
ELPA_Solver es((const bool)isReal,
157154
comm,
158-
(const int)PARAM.inp.nbands,
155+
(const int)this->nbands,
159156
(const int)h_mat.row,
160157
(const int)h_mat.col,
161158
(const int*)h_mat.desc);
@@ -170,7 +167,7 @@ void DiagoElpa<std::complex<double>>::diag_pool(hamilt::MatrixBlock<std::complex
170167
ModuleBase::timer::end("DiagoElpa", "elpa_solve");
171168
es.exit();
172169
const int inc = 1;
173-
BlasConnector::copy(PARAM.inp.nbands, eigen.data(), inc, eigenvalue_in, inc);
170+
BlasConnector::copy(this->nbands, eigen.data(), inc, eigenvalue_in, inc);
174171
}
175172

176173
template <>
@@ -180,14 +177,12 @@ void DiagoElpa<double>::diag_pool(hamilt::MatrixBlock<double>& h_mat,
180177
Real* eigenvalue_in,
181178
MPI_Comm& comm)
182179
{
183-
std::vector<double> eigen(PARAM.globalv.nlocal, 0.0);
180+
std::vector<double> eigen(this->nlocal, 0.0);
184181

185182
bool isReal = true;
186-
// ELPA_Solver es(isReal, COMM_DIAG, PARAM.inp.nbands, h_mat.row, h_mat.col,
187-
// h_mat.desc);
188183
ELPA_Solver es((const bool)isReal,
189184
comm,
190-
(const int)PARAM.inp.nbands,
185+
(const int)this->nbands,
191186
(const int)h_mat.row,
192187
(const int)h_mat.col,
193188
(const int*)h_mat.desc);
@@ -203,7 +198,7 @@ void DiagoElpa<double>::diag_pool(hamilt::MatrixBlock<double>& h_mat,
203198
const int inc = 1;
204199
ModuleBase::GlobalFunc::OUT(GlobalV::ofs_running,
205200
"K-S equation was solved by genelpa2");
206-
BlasConnector::copy(PARAM.inp.nbands, eigen.data(), inc, eigenvalue_in, inc);
201+
BlasConnector::copy(this->nbands, eigen.data(), inc, eigenvalue_in, inc);
207202
ModuleBase::GlobalFunc::OUT(GlobalV::ofs_running,
208203
"eigenvalues were copied to ekb");
209204
}

source/source_hsolver/diago_elpa.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@ class DiagoElpa
1515
using Real = typename GetTypeReal<T>::type;
1616

1717
public:
18+
/// @param nlocal_in global dimension of the NAO Hamiltonian
19+
/// @param nbands_in number of lowest eigenpairs to compute
20+
DiagoElpa(const int nlocal_in, const int nbands_in) : nlocal(nlocal_in), nbands(nbands_in) {};
21+
1822
void diag(hamilt::Hamilt<T>* phm_in, psi::Psi<T>& psi, Real* eigenvalue_in);
1923
#ifdef __MPI
2024
// diagnolization used in parallel-k case
@@ -30,6 +34,9 @@ class DiagoElpa
3034
bool ifElpaHandle(const bool& newIteration, const bool& ifNSCF) const;
3135
static int lastmpinum; // last using mpi;
3236
#endif
37+
38+
const int nlocal;
39+
const int nbands;
3340
};
3441

3542
template <typename T>

source/source_hsolver/diago_elpa_native.cpp

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
#include "source_base/module_external/blas_connector.h"
55
#include "source_base/module_external/blacs_connector.h"
66
#include "source_base/global_variable.h"
7-
#include "source_io/module_parameter/parameter.h"
87
#include "source_base/timer.h"
98
#include "source_base/tool_quit.h"
109
#include "source_hsolver/module_genelpa/elpa_new.h"
@@ -59,7 +58,7 @@ void DiagoElpaNative<T>::diag_pool(hamilt::MatrixBlock<T>& h_mat,
5958

6059
ModuleBase::timer::start("DiagoElpaNative", "elpa_solve");
6160

62-
int nev = PARAM.inp.nbands;
61+
int nev = this->nbands;
6362
int narows = h_mat.row;
6463
int nacols = h_mat.col;
6564

@@ -70,7 +69,7 @@ void DiagoElpaNative<T>::diag_pool(hamilt::MatrixBlock<T>& h_mat,
7069
int nprows, npcols, myprow, mypcol;
7170

7271
Cblacs_gridinfo(cblacs_ctxt, &nprows, &npcols, &myprow, &mypcol);
73-
std::vector<Real> eigen(PARAM.globalv.nlocal, 0.0);
72+
std::vector<Real> eigen(this->nlocal, 0.0);
7473
std::vector<T> eigenvectors(narows * nacols);
7574

7675
if (elpa_init(20210430) != ELPA_OK)
@@ -107,7 +106,7 @@ void DiagoElpaNative<T>::diag_pool(hamilt::MatrixBlock<T>& h_mat,
107106
#define ELPA_WITH_SYCL_GPU_VERSION 0
108107
*/
109108
#if ELPA_WITH_NVIDIA_GPU_VERSION
110-
if (PARAM.inp.device == "gpu")
109+
if (this->use_gpu)
111110
{
112111
elpa_set(handle, "nvidia-gpu", 1, &success);
113112
elpa_set(handle, "real_kernel", ELPA_2STAGE_REAL_NVIDIA_GPU, &success);
@@ -138,7 +137,7 @@ void DiagoElpaNative<T>::diag_pool(hamilt::MatrixBlock<T>& h_mat,
138137
}
139138

140139
const int inc = 1;
141-
BlasConnector::copy(PARAM.inp.nbands, eigen.data(), inc, eigenvalue_in, inc);
140+
BlasConnector::copy(this->nbands, eigen.data(), inc, eigenvalue_in, inc);
142141
const int size = psi.get_nbands() * psi.get_nbasis();
143142
BlasConnector::copy(size, eigenvectors.data(), inc, psi.get_pointer(), inc);
144143
}

source/source_hsolver/diago_elpa_native.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,12 @@ class DiagoElpaNative
1515
using Real = typename GetTypeReal<T>::type;
1616

1717
public:
18+
/// @param nlocal_in global dimension of the NAO Hamiltonian
19+
/// @param nbands_in number of lowest eigenpairs to compute
20+
/// @param use_gpu_in offload to the NVIDIA-GPU ELPA kernels when ELPA was built with GPU support
21+
DiagoElpaNative(const int nlocal_in, const int nbands_in, const bool use_gpu_in)
22+
: nlocal(nlocal_in), nbands(nbands_in), use_gpu(use_gpu_in) {};
23+
1824
void diag(hamilt::Hamilt<T>* phm_in, psi::Psi<T>& psi, Real* eigenvalue_in);
1925
#ifdef __MPI
2026
// diagnolization used in parallel-k case
@@ -27,6 +33,10 @@ class DiagoElpaNative
2733

2834
static int DecomposedState;
2935

36+
private:
37+
const int nlocal;
38+
const int nbands;
39+
const bool use_gpu;
3040
};
3141

3242
template <typename T>

0 commit comments

Comments
 (0)