Skip to content

Commit 0a7ffbd

Browse files
szaman19szaman19
committed
Buildling and linking implementation
- Run with new pre-commit hook
1 parent a4f367d commit 0a7ffbd

File tree

8 files changed

+182
-181
lines changed

8 files changed

+182
-181
lines changed

‎include/lbann/layers/regularizers/distconv/distconv_layer_norm.hpp‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
#ifndef LBANN_LAYERSE_REGULARIZERS_DISTCONV_LAYER_NORM
2828
#define LBANN_LAYERSE_REGULARIZERS_DISTCONV_LAYER_NORM
2929

30-
#if LBANN_HAS_DISTCONV
30+
#ifdef LBANN_HAS_DISTCONV
3131

3232
namespace distconv {
3333
template <typename Backend, typename DataType>
@@ -39,17 +39,13 @@ class LayerNormalization
3939
using DCTensor = tensor::Tensor<DataType, LocaleMPI, Allocator>;
4040

4141
public:
42-
LayerNormalization(Backend& backend,
43-
Datatype epsilon,
44-
size_t max_mini_batch_size)
45-
: m_backend(backend),
46-
m_epsilon(epsilon),
47-
m_max_mini_batch_size(max_mini_batch_size)
42+
LayerNormalization(Backend& backend, DataType epsilon)
43+
: m_backend(backend), m_epsilon(epsilon)
4844
{}
4945

5046
template <typename Allocator>
5147
void calculate_forward_stats(const DCTensor<Allocator>& input,
52-
DC<Allocator>& statistics);
48+
DCTensor<Allocator>& statistics);
5349

5450
template <typename Allocator>
5551
void apply_normalization(const DCTensor<Allocator>& input,
@@ -74,10 +70,9 @@ class LayerNormalization
7470

7571
private:
7672
DataType m_epsilon;
77-
size_t m_max_mini_batch_size;
7873

7974
}; // class definition LayerNorm
8075
} // namespace distconv
8176

8277
#endif // LBANN_HAS_DISTONV
83-
#endif // LBANN_LAYERSE_REGULARIZERS_DISTCONV_LAYER_NORM
78+
#endif // LBANN_LAYERSE_REGULARIZERS_DISTCONV_LAYER_NORM

‎include/lbann/layers/regularizers/layer_norm.hpp‎

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -35,19 +35,20 @@
3535
#include <memory>
3636

3737
#ifdef LBANN_HAS_DISTCONV
38-
#include "lbann/utils/distconv.hpp"
3938
#include "lbann/layers/data_type_distconv_adapter.hpp"
4039
#include "lbann/layers/regularizers/distconv/distconv_layer_norm.hpp"
40+
#include "lbann/utils/distconv.hpp"
4141
#endif // LBANN_HAS_DISTCONV
4242

4343
namespace lbann {
4444

4545
#ifdef LBANN_HAS_DISTCONV
4646
namespace dc {
47-
using Shape = ::distconv::tensor::Shape;
48-
using Backend= ::distconv::BackendDNNLib;
47+
using Shape = ::distconv::tensor::Shape;
48+
using Backend = ::distconv::BackendDNNLib;
4949
template <typename TensorDataType>
50-
using LayerNormalization = ::distconv::LayerNormalization<Backend, TensorDataType>;
50+
using LayerNormalization =
51+
::distconv::LayerNormalization<Backend, TensorDataType>;
5152
} // namespace dc
5253

5354
template <typename TensorDataType, data_layout Layout, El::Device Device>
@@ -66,12 +67,10 @@ class layer_norm_distconv_adapter
6667

6768
void setup_distributions(tensor_overlap_constraints& constraints) override;
6869
void setup_layer(size_t workspace_capacity) override;
69-
void setup_fp_tensors() override;
70-
void setup_bp_tensors() override;
7170

7271
void fp_compute();
7372
void bp_compute();
74-
73+
7574
TensorDevType m_statistics;
7675
TensorDevType m_statistics_grad;
7776
std::unique_ptr<dc::LayerNormalization<TensorDataType>> m_layer_norm_operator;
@@ -167,14 +166,11 @@ class layer_norm_layer : public data_type_layer<TensorDataType>
167166
std::unique_ptr<AbsDistMatType> m_statistics_gradient;
168167
};
169168

169+
LBANN_DEFINE_LAYER_BUILDER(layer_norm);
170170

171-
#endif // LBANN_HAS_DISTCONV
172-
173-
LBANN_DEFINE_LAYER_BUILDER(layer_norm);
174-
175-
// =========================================================
176-
// Explicit template instantiation
177-
// =========================================================
171+
// =========================================================
172+
// Explicit template instantiation
173+
// =========================================================
178174

179175
#ifndef LBANN_LAYER_NORM_LAYER_INSTANTIATE
180176
#define PROTO_DEVICE(T, Device) \

‎include/lbann/layers/regularizers/layer_norm_impl.hpp‎

Lines changed: 43 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,11 @@
2929

3030
#include "lbann/layers/regularizers/layer_norm.hpp"
3131

32-
#ifdef LBANN_HAS_DISTONV
32+
#ifdef LBANN_HAS_DISTCONV
3333
#include "lbann/layers/data_type_distconv_adapter.hpp"
34-
#endif
35-
36-
namespace lbann{
34+
#endif // LBANN_HAS_DISTCONV
3735

36+
namespace lbann {
3837

3938
// =========================================================
4039
// Implementation
@@ -135,7 +134,6 @@ void layer_norm_layer<TensorDataType, Layout, Device>::setup_data(
135134
m_statistics_gradient.reset(AbsDistMatrixType::Instantiate(dist));
136135
}
137136

138-
139137
#ifdef LBANN_HAS_DISTCONV
140138

141139
// =============================================================
@@ -174,57 +172,53 @@ layer_norm_layer<TensorDataType, Layout, Device>::get_distconv_adapter()
174172
layer_norm_distconv_adapter<TensorDataType, Layout, Device>&>(
175173
static_cast<const layer_norm_layer<TensorDataType, Layout, Device>&>(*this)
176174
.get_distconv_adapter());
175+
}
177176

178177
// =============================================================
179178
// LayerNorm DistConv Adapter implementation
180179
// =============================================================
181180

182-
template <typename TensorDataType, data_layout Layout, El::Device Device>
183-
void layer_norm_distconv_adapter<TensorDataType, Layout, Device>::
184-
setup_distributions(tensor_overlap_constraints & constraints)
185-
{
186-
data_type_distconv_adapter<TensorDataType>::setup_distributions(
187-
constraints);
188-
// no overlap needed
189-
for (auto& d : this->m_prev_activations_dists) {
190-
d.clear_overlap();
191-
constraints.mark_updated(d);
192-
constraints.mark_invariant(d);
193-
}
194-
for (auto& d : this->m_activations_dists) {
195-
d.clear_overlap();
196-
constraints.mark_updated(d);
197-
constraints.mark_invariant(d);
198-
}
199-
for (auto& d : this->m_prev_error_signals_dists) {
200-
d.clear_overlap();
201-
constraints.mark_updated(d);
202-
constraints.mark_invariant(d);
203-
}
204-
for (auto& d : this->m_error_signals_dists) {
205-
d.clear_overlap();
206-
constraints.mark_updated(d);
207-
constraints.mark_invariant(d);
208-
}
181+
template <typename TensorDataType, data_layout Layout, El::Device Device>
182+
void layer_norm_distconv_adapter<TensorDataType, Layout, Device>::
183+
setup_distributions(tensor_overlap_constraints& constraints)
184+
{
185+
data_type_distconv_adapter<TensorDataType>::setup_distributions(constraints);
186+
// no overlap needed
187+
for (auto& d : this->m_prev_activations_dists) {
188+
d.clear_overlap();
189+
constraints.mark_updated(d);
190+
constraints.mark_invariant(d);
209191
}
210-
211-
template <typename TensorDataType, data_layout Layout, El::Device Device>
212-
void layer_norm_distconv_adapter<TensorDataType, Layout, Device>::setup_layer(
213-
size_t workspace_capacity)
214-
{
215-
data_type_distconv_adapter<TensorDataType>::setup_layer(workspace_capacity);
216-
auto& layer = dynamic_cast<
217-
channelwise_fully_connected_layer<TensorDataType, Layout, Device>&>(
218-
this->layer());
219-
const auto max_mini_batch_size =
220-
layer.get_model()->m_max_mini_batch_size_distconv;
221-
222-
m_layer_norm_operator =
223-
make_unique<dc::LayerNormalization<TensorDataType>>(dc::get_backend(),
224-
layer.m_epsilon,
225-
max_mini_batch_size);
192+
for (auto& d : this->m_activations_dists) {
193+
d.clear_overlap();
194+
constraints.mark_updated(d);
195+
constraints.mark_invariant(d);
196+
}
197+
for (auto& d : this->m_prev_error_signals_dists) {
198+
d.clear_overlap();
199+
constraints.mark_updated(d);
200+
constraints.mark_invariant(d);
226201
}
202+
for (auto& d : this->m_error_signals_dists) {
203+
d.clear_overlap();
204+
constraints.mark_updated(d);
205+
constraints.mark_invariant(d);
206+
}
207+
}
208+
209+
template <typename TensorDataType, data_layout Layout, El::Device Device>
210+
void layer_norm_distconv_adapter<TensorDataType, Layout, Device>::setup_layer(
211+
size_t workspace_capacity)
212+
{
213+
data_type_distconv_adapter<TensorDataType>::setup_layer(workspace_capacity);
214+
auto& layer = dynamic_cast<layer_norm_layer<TensorDataType, Layout, Device>&>(
215+
this->layer());
216+
217+
m_layer_norm_operator =
218+
make_unique<dc::LayerNormalization<TensorDataType>>(dc::get_backend(),
219+
layer.m_epsilon);
220+
}
227221

228-
#endif LBANN_HAS_DISTCONV
222+
#endif // LBANN_HAS_DISTCONV
229223
} // namespace lbann
230224
#endif // LBANN_LAYER_REGULARIZER_LAYER_NORM_IMPL_HPP_INCLUDED

‎include/lbann/layers/transform/distconv/distconv_gather.hpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,12 @@
2929
#include "distconv/base.hpp"
3030
#include "distconv/tensor/tensor.hpp"
3131
#include "distconv/tensor/tensor_mpi.hpp"
32-
#include "lbann/layers/transform/distconv/distconv_nvshmem_vector_addressing.hpp"
3332
#include "lbann/utils/distconv.hpp"
3433

3534
#if defined(LBANN_HAS_NVSHMEM) && defined(LBANN_HAS_DISTCONV)
3635

36+
#include "lbann/layers/transform/distconv/distconv_nvshmem_vector_addressing.hpp"
37+
3738
namespace distconv {
3839
template <typename Backend, typename DataType>
3940
class Gather

‎include/lbann/layers/transform/distconv/distconv_scatter.hpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,12 @@
2929
#include "distconv/base.hpp"
3030
#include "distconv/tensor/tensor.hpp"
3131
#include "distconv/tensor/tensor_mpi.hpp"
32-
#include "lbann/layers/transform/distconv/distconv_nvshmem_vector_addressing.hpp"
3332
#include "lbann/utils/distconv.hpp"
3433

3534
#if defined(LBANN_HAS_NVSHMEM) && defined(LBANN_HAS_DISTCONV)
3635

36+
#include "lbann/layers/transform/distconv/distconv_nvshmem_vector_addressing.hpp"
37+
3738
namespace distconv {
3839
template <typename Backend, typename DataType>
3940
class Scatter

0 commit comments

Comments
 (0)