@@ -206,9 +206,21 @@ Psi<T, Device>::Psi(const Psi& psi_in)
206206 const size_t total_size = static_cast <size_t >(this ->nk ) * this ->nbands * this ->nbasis ;
207207 const T* src_ptr = psi_in.get_pointer () - psi_in.get_psi_bias ();
208208
209- if (psi_in.get_storage_mode () == PsiStorageMode::PAGED_GPU )
209+ if (psi_in.get_storage_mode () == PsiStorageMode::PAGED_GPU )
210210 {
211- std::memcpy (this ->psi_cpu_ , psi_in.get_cpu_pointer (), sizeof (T) * total_size);
211+ // Source may have PAGED_GPU flag set but psi_cpu_ not yet allocated
212+ // (e.g. after set_storage_mode() in setup_psi_pw). In that case,
213+ // copy from the source's main buffer (psi) into our psi_cpu_.
214+ const T* cp_ptr = psi_in.get_cpu_pointer_safe ();
215+ if (cp_ptr != nullptr )
216+ {
217+ std::memcpy (this ->psi_cpu_ , cp_ptr, sizeof (T) * total_size);
218+ }
219+ else
220+ {
221+ // Fallback: source data is in psi buffer (ALL_GPU layout)
222+ std::memcpy (this ->psi_cpu_ , src_ptr, sizeof (T) * total_size);
223+ }
212224 }
213225 else if (std::is_same<Device, base_device::DEVICE_CPU >::value)
214226 {
@@ -255,10 +267,23 @@ Psi<T, Device>::Psi(const Psi<T_in, Device_in>& psi_in)
255267 {
256268 const size_t total_size = static_cast <size_t >(this ->nk ) * this ->nbands * this ->nbasis ;
257269
258- if (psi_in.get_storage_mode () == PsiStorageMode::PAGED_GPU )
270+ if (psi_in.get_storage_mode () == PsiStorageMode::PAGED_GPU )
259271 {
260- const T_in* src_ptr = psi_in.get_cpu_pointer ();
261- detail::TypeCopy<T, T_in, std::is_same<T, T_in>::value>::copy (this ->psi_cpu_ , src_ptr, total_size);
272+ // Source may have PAGED_GPU flag set but psi_cpu_ not yet allocated.
273+ const T_in* cp_ptr = psi_in.get_cpu_pointer_safe ();
274+ if (cp_ptr != nullptr )
275+ {
276+ detail::TypeCopy<T, T_in, std::is_same<T, T_in>::value>::copy (this ->psi_cpu_ , cp_ptr, total_size);
277+ }
278+ else
279+ {
280+ // Fallback: source data is in main psi buffer (ALL_GPU layout)
281+ auto * arr = (T*)malloc (sizeof (T) * total_size);
282+ base_device::memory::cast_memory_op<T, T_in, Device_in, Device_in>()(
283+ arr, psi_in.get_pointer () - psi_in.get_psi_bias (), total_size);
284+ std::memcpy (this ->psi_cpu_ , arr, sizeof (T) * total_size);
285+ free (arr);
286+ }
262287 }
263288 else
264289 {
@@ -317,13 +342,30 @@ Psi<T, Device>& Psi<T, Device>::operator=(const Psi<T, Device>& psi_in)
317342 this ->k_first = psi_in.get_k_first ();
318343 // this function will copy psi_in.psi to this->psi no matter the device types of each other.
319344
345+ this ->storage_mode_ = psi_in.get_storage_mode ();
320346 this ->resize (psi_in.get_nk (), psi_in.get_nbands (), psi_in.get_nbasis ());
321- base_device::memory::synchronize_memory_op<T, Device, Device>()(this ->psi ,
322- psi_in.psi ,
323- psi_in.size ());
324- this ->psi_bias = psi_in.get_psi_bias ();
347+
348+ if (this ->storage_mode_ == PsiStorageMode::PAGED_GPU )
349+ {
350+ // PAGED_GPU: copy the full CPU buffer, GPU buffer stays single-k
351+ if (psi_cpu_ != nullptr && psi_in.psi_cpu_ != nullptr )
352+ {
353+ const size_t total_size = static_cast <size_t >(this ->nk ) * this ->nbands * this ->nbasis ;
354+ std::memcpy (this ->psi_cpu_ , psi_in.psi_cpu_ , sizeof (T) * total_size);
355+ }
356+ this ->current_k_gpu_ = -1 ;
357+ this ->psi_bias = 0 ;
358+ this ->psi_current = this ->psi ;
359+ }
360+ else
361+ {
362+ base_device::memory::synchronize_memory_op<T, Device, Device>()(this ->psi ,
363+ psi_in.psi ,
364+ psi_in.size ());
365+ this ->psi_bias = psi_in.get_psi_bias ();
366+ this ->psi_current = this ->psi + psi_in.get_psi_bias ();
367+ }
325368 this ->current_nbasis = psi_in.get_current_nbas ();
326- this ->psi_current = this ->psi + psi_in.get_psi_bias ();
327369
328370 return *this ;
329371}
@@ -485,7 +527,17 @@ void Psi<T, Device>::fix_k(const int ik) const
485527 if (storage_mode_ == PsiStorageMode::PAGED_GPU )
486528 {
487529 this ->psi_bias = 0 ;
488- this ->psi_current = const_cast <T*>(this ->psi );
530+ if (this ->current_k_gpu_ == ik)
531+ {
532+ // This k-point is loaded on GPU by load_k_to_gpu
533+ this ->psi_current = const_cast <T*>(this ->psi );
534+ }
535+ else
536+ {
537+ // Read from CPU full buffer (psi_cpu_ has all k-points)
538+ this ->psi_current = const_cast <T*>(psi_cpu_
539+ + static_cast <size_t >(ik) * this ->nbands * this ->nbasis );
540+ }
489541 return ;
490542 }
491543
@@ -508,6 +560,13 @@ void Psi<T, Device>::fix_b(const int ib) const
508560 assert (ib >= 0 );
509561 this ->current_b = ib;
510562
563+ if (storage_mode_ == PsiStorageMode::PAGED_GPU )
564+ {
565+ this ->psi_bias = 0 ;
566+ this ->psi_current = const_cast <T*>(this ->psi );
567+ return ;
568+ }
569+
511570 if (!this ->k_first )
512571 {
513572 this ->current_k = 0 ;
@@ -532,6 +591,14 @@ void Psi<T, Device>::fix_kb(const int ik, const int ib) const
532591 assert (ik >= 0 && ib >= 0 );
533592 this ->current_k = ik;
534593 this ->current_b = ib;
594+
595+ if (storage_mode_ == PsiStorageMode::PAGED_GPU )
596+ {
597+ this ->psi_bias = 0 ;
598+ this ->psi_current = const_cast <T*>(this ->psi );
599+ return ;
600+ }
601+
535602 if (ik >= this ->nk || ib >= this ->nbands )
536603 { // fix to 0
537604 this ->psi_bias = 0 ;
@@ -549,6 +616,14 @@ T& Psi<T, Device>::operator()(const int ikb1, const int ikb2, const int ibasis)
549616{
550617 assert (ikb1 >= 0 && ikb2 >= 0 && ibasis >= 0 );
551618 assert (this ->k_first ? ikb1 < this ->nk && ikb2 < this ->nbands : ikb1 < this ->nbands && ikb2 < this ->nk );
619+
620+ if (storage_mode_ == PsiStorageMode::PAGED_GPU )
621+ {
622+ // PAGED_GPU: GPU buffer is single-k, read from CPU full buffer (k_first layout)
623+ assert (psi_cpu_ != nullptr );
624+ return const_cast <T*>(psi_cpu_)[(ikb1 * this ->nbands + ikb2) * this ->nbasis + ibasis];
625+ }
626+
552627 return this ->k_first ? this ->psi [(ikb1 * this ->nbands + ikb2) * this ->nbasis + ibasis]
553628 : this ->psi [(ikb1 * this ->nk + ikb2) * this ->nbasis + ibasis];
554629}
@@ -597,8 +672,21 @@ const int& Psi<T, Device>::get_ngk(const int ik_in) const
597672template <typename T, typename Device>
598673void Psi<T, Device>::zero_out()
599674{
600- // this->psi.assign(this->psi.size(), T(0));
601- set_memory_op ()(this ->psi , 0 , this ->size ());
675+ if (storage_mode_ == PsiStorageMode::PAGED_GPU )
676+ {
677+ // Zero full CPU buffer and single-k GPU buffer
678+ if (psi_cpu_ != nullptr )
679+ {
680+ const size_t total_size = static_cast <size_t >(this ->nk ) * this ->nbands * this ->nbasis ;
681+ std::memset (psi_cpu_, 0 , sizeof (T) * total_size);
682+ }
683+ const size_t k_size = static_cast <size_t >(this ->nbands ) * this ->nbasis ;
684+ set_memory_op ()(this ->psi , 0 , k_size);
685+ }
686+ else
687+ {
688+ set_memory_op ()(this ->psi , 0 , this ->size ());
689+ }
602690}
603691
604692template <typename T, typename Device>
0 commit comments