diff --git a/bridge/include/bridge.h b/bridge/include/bridge.h index 7ac0e956..3437afa5 100644 --- a/bridge/include/bridge.h +++ b/bridge/include/bridge.h @@ -3,6 +3,16 @@ #include +#ifndef __cplusplus +#include +#endif + +#ifdef __cplusplus +#include +#else +#include +#endif + #ifdef __cplusplus extern "C" { #endif @@ -106,9 +116,17 @@ bridge_tensor_t max_pool2d( int dilation ); -proto_bridge_simple(relu); +void split_loop(int64_t idx, int64_t n); +void split_loop_filler(int64_t n,int64_t* ret); -proto_bridge_simple(relu6); + +// bridge_tensor_t conv2d( +// bridge_tensor_t input, +// bridge_tensor_t kernel, +// nil_scalar_tensor_t bias, +// nil_scalar_tensor_t stride, +// nil_scalar_tensor_t padding +// ); proto_bridge_simple(gelu); @@ -116,36 +134,63 @@ proto_bridge_simple(logsigmoid); proto_bridge_simple(mish); -proto_bridge_simple(selu); +proto_bridge_simple(relu); -proto_bridge_simple(silu); +proto_bridge_simple(relu6); -proto_bridge_simple(softmax); +proto_bridge_simple(selu); -proto_bridge_simple(softmin); +proto_bridge_simple(silu); proto_bridge_simple(softsign); proto_bridge_simple(tanhshrink); -void split_loop(int64_t idx, int64_t n); -void split_loop_filler(int64_t n,int64_t* ret); +bridge_tensor_t rrelu(bridge_tensor_t input, float lower, float upper, bool training); +bridge_tensor_t hardshrink(bridge_tensor_t input, float lambda); -// bridge_tensor_t conv2d( -// bridge_tensor_t input, -// bridge_tensor_t kernel, -// nil_scalar_tensor_t bias, -// nil_scalar_tensor_t stride, -// nil_scalar_tensor_t padding -// ); +bridge_tensor_t hardtanh(bridge_tensor_t input, float min_val, float max_val); + +bridge_tensor_t elu(bridge_tensor_t input, float alpha); + +bridge_tensor_t softplus(bridge_tensor_t input, float beta, float threhsold); + +bridge_tensor_t threshold(bridge_tensor_t input, float threshold, float value); + +bridge_tensor_t celu(bridge_tensor_t input, float alpha); + +bridge_tensor_t leaky_relu(bridge_tensor_t input, float negative_slope); + +bridge_tensor_t softshrink(bridge_tensor_t input, float lambda); + +#ifdef __cplusplus +bridge_tensor_t softmax(bridge_tensor_t input, std::int64_t dim); +#else +bridge_tensor_t softmax(bridge_tensor_t input, int64_t dim); +#endif + +#ifdef __cplusplus +bridge_tensor_t softmin(bridge_tensor_t input, std::int64_t dim); +#else +bridge_tensor_t softmin(bridge_tensor_t input, int64_t dim); +#endif + +bridge_tensor_t dropout(bridge_tensor_t input, double p, bool training); + +bridge_tensor_t alpha_dropout(bridge_tensor_t input, double p, bool training); + +bridge_tensor_t feature_alpha_dropout(bridge_tensor_t input, double p, bool training); + +bridge_tensor_t dropout2d(bridge_tensor_t input, double p, bool training); +bridge_tensor_t dropout3d(bridge_tensor_t input, double p, bool training); +#undef proto_bridge_simple #ifdef __cplusplus } #endif -#endif // BRIDGE_H -//hello +#endif // BRIDGE_H \ No newline at end of file diff --git a/bridge/lib/bridge.cpp b/bridge/lib/bridge.cpp index e94dd741..32f2fe0e 100644 --- a/bridge/lib/bridge.cpp +++ b/bridge/lib/bridge.cpp @@ -22,15 +22,16 @@ +namespace tnf = torch::nn::functional; + + #define def_bridge_simple(Name) \ extern "C" bridge_tensor_t Name(bridge_tensor_t input) { \ auto t_input = bridge_to_torch(input); \ - auto t_output = torch::Name(t_input); \ + auto t_output = torch::nn::functional::Name(t_input); \ return torch_to_bridge(t_output); \ } - - // Globals @@ -591,6 +592,250 @@ extern "C" void split_loop_filler(int64_t n,int64_t* ret) { // } // } -// cap.release(); -// cv::destroyAllWindows(); -// } \ No newline at end of file + +// Simple activation function defs + +def_bridge_simple(gelu); + +def_bridge_simple(logsigmoid); + +def_bridge_simple(mish); + +def_bridge_simple(relu); + +def_bridge_simple(relu6); + +def_bridge_simple(selu); + +def_bridge_simple(silu); + +def_bridge_simple(softsign); + +def_bridge_simple(tanhshrink); + + +// More complex activation functions with scary parameters + +extern "C" bridge_tensor_t rrelu( + bridge_tensor_t input, + float lower, + float upper, + bool training +) { + auto t_input = bridge_to_torch(input); + auto t_output = tnf::rrelu(t_input, + tnf::RReLUFuncOptions() + .lower(lower) + .upper(upper) + .training(training)); + + return torch_to_bridge(t_output); +} + + +extern "C" bridge_tensor_t hardshrink( + bridge_tensor_t input, + float lambda +) { + auto t_input = bridge_to_torch(input); + auto t_output = tnf::hardshrink(t_input, + tnf::HardshrinkFuncOptions() + .lambda(lambda)); + + return torch_to_bridge(t_output); +} + + +extern "C" bridge_tensor_t hardtanh( + bridge_tensor_t input, + float min_val, + float max_val +) { + auto t_input = bridge_to_torch(input); + auto t_output = tnf::hardtanh(t_input, + tnf::HardtanhFuncOptions() + .min_val(min_val) + .max_val(max_val)); + + return torch_to_bridge(t_output); +} + + +extern "C" bridge_tensor_t elu( + bridge_tensor_t input, + float alpha +) { + auto t_input = bridge_to_torch(input); + auto t_output = tnf::elu(t_input, + tnf::ELUFuncOptions() + .alpha(alpha)); + + return torch_to_bridge(t_output); +} + + +extern "C" bridge_tensor_t softplus( + bridge_tensor_t input, + float beta, + float threshold +) { + auto t_input = bridge_to_torch(input); + auto t_output = tnf::softplus(t_input, + tnf::SoftplusFuncOptions() + .beta(beta) + .threshold(threshold)); + + return torch_to_bridge(t_output) +} + + +extern "C" bridge_tensor_t threshold( + bridge_tensor_t input, + float threshold, + float value +) { + auto t_input = bridge_to_torch(input); + auto t_output = tnf::threshold(t_input, + tnf::ThresholdFuncOptions() + .threshold(threshold) + .value(value)); + + return torch_to_bridge(t_output); +} + + +extern "C" bridge_tensor_t celu( + bridge_tensor_t input, + float alpha +) { + auto t_input = bridge_to_torch(input); + auto t_output = tnf::celu(t_input, + tnf::CELUFuncOptions() + .alpha(alpha)); + + return t_output; +} + + +extern "C" bridge_tensor_t leaky_relu( + bridge_tensor_t input, + float negative_slope +) { + auto t_input = bridge_to_torch(input); + auto t_output = tnf::leaky_relu(t_input, + tnf::LeakyReLUFuncOptions() + .negative_slope(negative_slope)); + + return torch_to_bridge(t_output); +} + + +extern "C" bridge_tensor_t softshrink( + bridge_tensor_t input, + float lambda +) { + auto t_input = bridge_to_torch(input); + auto t_output = tnf::softshrink(t_input, + tnf::SoftshrinkFuncOptions(lambda)); + + return torch_to_bridge(t_output); +} + + +extern "C" bridge_tensor_t softmax( + bridge_tensor_t input, + std::int64_t dim +) { + auto t_input = bridge_to_torch(input); + auto t_output = tnf::softmax(t_input, + tnf::SoftmaxFuncOptions(dim)); + + return torch_to_bridge(t_output); +} + + +extern "C" bridge_tensor_t softmin( + bridge_tensor_t input, + std::int64_t dim +) { + auto t_input = bridge_to_torch(input); + auto t_output = tnf::softmin(t_input, + tnf::SoftminFuncOptions(dim)); + + return torch_to_bridge(t_output); +} + + +extern "C" bridge_tensor_t dropout( + bridge_tensor_t input, + double p, + bool training +) { + auto t_input = bridge_to_torch(input); + auto t_output = tnf::dropout( + tnf::DropoutFuncOptions() + .p(p) + .training(training)); + + return torch_to_bridge(t_output); +} + + +extern "C" bridge_tensor_t alpha_dropout( + bridge_tensor_t input, + double p, + bool training +) { + auto t_input = bridge_to_torch(input); + auto t_output = tnf::alpha_dropout( + tnf::AlphaDropoutFuncOptions() + .p(p) + .training(training)); + + return torch_to_bridge(t_output); +} + + +extern "C" bridge_tensor_t feature_alpha_dropout( + bridge_tensor_t input, + double p, + bool training +) { + auto t_input = bridge_to_torch(input); + auto t_output = tnf::feature_alpha_dropout( + tnf::FeatureAlphaDropoutFuncOptions() + .p(p) + .training(training)); + + return torch_to_bridge(t_output); +} + + +extern "C" bridge_tensor_t dropout2d( + bridge_tensor_t input, + double p, + bool training +) { + auto t_input = bridge_to_torch(input); + auto t_output = tnf::dropout2d( + tnf::Dropout2dFuncOptions() + .p(p) + .training(training)); + + return torch_to_bridge(t_output); +} + + +extern "C" bridge_tensor_t dropout3d( + bridge_tensor_t input, + double p, + bool training +) { + auto t_input = bridge_to_torch(input); + auto t_output = tnf::dropout3d( + tnf::Dropout3dFuncOptions() + .p(p) + .training(training)); + + return torch_to_bridge(t_output); +} diff --git a/lib/Bridge.chpl b/lib/Bridge.chpl index afd46edd..18be1fea 100644 --- a/lib/Bridge.chpl +++ b/lib/Bridge.chpl @@ -194,6 +194,114 @@ module Bridge { return result; } + extern proc gelu(in input: bridge_tensor_t): bridge_tensor_t; + + extern proc logsigmoid(in input: bridge_tensor_t): bridge_tensor_t; + + extern proc mish(in input: bridge_tensor_t): bridge_tensor_t; + + extern proc relu(in input: bridge_tensor_t): bridge_tensor_t; + + extern proc relu6(in input: bridge_tensor_t): bridge_tensor_t; + + extern proc selu(in input: bridge_tensor_t): bridge_tensor_t; + + extern proc silu(in input: bridge_tensor_t): bridge_tensor_t; + + extern proc softsign(in input: bridge_tensor_t): bridge_tensor_t; + + extern proc tanhshrink(in input: bridge_tensor_t): bridge_tensor_t; + + extern proc rrelu( + in input: bridge_tensor_t, + in lower: real(32), + in upper: real(32), + in training: bool + ) : bridge_tensor_t; + + extern proc hardshrink( + in input: bridge_tensor_t, + in alpha: real(32) + ) : bridge_tensor_t; + + extern proc hardtanh( + in input: bridge_tensor_t, + in minVal : real(32), + in maxVal : real(32) + ) : bridge_tensor_t; + + extern proc elu( + in input: bridge_tensor_t, + in alpha: real(32) + ) : bridge_tensor_t; + + extern proc softplus( + in input : bridge_tensor_t, + in beta : real(32), + in threshold : real(32) + ) : bridge_tensor_t; + + extern proc threshold( + in input : bridge_tensor_t, + in threshold : real(32), + in value : real(32) + ) : bridge_tensor_t; + + extern proc celu( + in input : bridge_tensor_t, + in alpha : real(32) + ) : bridge_tensor_t; + + extern "leaky_relu" proc leakyRelu( + in input : bridge_tensor_t, + in negativeSlope : real(32) + ) : bridge_tensor_t; + + extern proc softshrink( + in input : bridge_tensor_t, + in alpha : real(32) + ) : bridge_tensor_t; + + extern proc softmax( + in input : bridge tensor_t, + param dim : int(64) + ) : bridge_tensor_t; + + extern proc softmin( + in input : bridge_tensor_t, + param dim : int(64) + ) : bridge_tensor_t; + + extern proc dropout( + in input : bridge_tensor_t, + in p : real, + in training : bool + ) : bridge_tensor_t; + + extern "alpha_dropout" proc alphaDropout( + in input : bridge_tensor_t, + in p : real, + in training : bool + ) : bridge_tensor_t; + + extern "feature_alpha_dropout" proc featureAlphaDropout( + in input : bridge_tensor_t, + in p : real, + in training : bool + ) : bridge_tensor_t; + + extern proc dropout2d( + in input : bridge_tensor_t, + in p : real, + in training : bool + ) : bridge_tensor_t; + + extern proc dropout3d( + in input : bridge_tensor_t, + in p : real, + in training : bool + ) : bridge_tensor_t; + proc createBridgeTensorWithShape(const ref data: [] real(32),shape: ?rank*int): bridge_tensor_t { var result: bridge_tensor_t; result.data = c_ptrToConst(data) : c_ptr(real(32)); @@ -240,6 +348,4 @@ module Bridge { return shape; } } - - } \ No newline at end of file diff --git a/lib/NDArray.chpl b/lib/NDArray.chpl index f4fedfb9..431a53d2 100644 --- a/lib/NDArray.chpl +++ b/lib/NDArray.chpl @@ -269,6 +269,7 @@ proc ndarray.this(args: int...rank) { return data.this((...args)); } + proc ref ndarray.setData(const arr: [] eltType) where arr.rank == rank do if arr.domain == this.domain then @@ -276,6 +277,7 @@ proc ref ndarray.setData(const arr: [] eltType) else this = arr; + proc ref ndarray.reshapeDomain(const dom: domain(rank,int)) where isRegularDomain(dom) { _domain = dom; @@ -517,6 +519,7 @@ proc ndarray.sum(axes: int...?axesCount): ndarray(rank,eltType) { return acc; } + /* Yields the indices of all of the axes of the :record:`ndarray`, as a tuple. :returns: A tuple representing the indices of the axes of the :record:`ndarray` @@ -529,6 +532,7 @@ proc ndarray.nDimTuple(): rank * int { return tpl; } + proc ndarray.mean(): ndarray(rank,eltType) do return this.mean((...this.nDimTuple())); @@ -541,6 +545,8 @@ proc ndarray.mean(axes: int...?axesCount): ndarray(rank,eltType) { } return this.sum((...axes)) / denom; } + + proc ndarray.shrink(narg: 2*int ... rank,param exactBounds = false): ndarray(rank,eltType) { var newShape: rank * int; var sliceRanges: rank * range; @@ -564,6 +570,7 @@ proc ndarray.shrink(narg: 2*int ... rank,param exactBounds = false): ndarray(ran return shrunk; } + proc ndarray.pad(narg: 2*int ... rank,value: eltType = 0): ndarray(rank,eltType) { var newShape: rank * int; var sliceRanges: rank * range; @@ -672,9 +679,7 @@ proc ndarray.squeeze(param newRank: int): ndarray(newRank,eltType) where newRank } -/* Yields the minimum value from an :record:`ndarray`, as an :record:`ndarray`. - - :returns: The minimum value from the :record:`ndarray`. +/* :returns: The minimum value from the :record:`ndarray`. :rtype: ndarray(1, eltType) */ proc ndarray.min(): ndarray(1,eltType) { @@ -685,9 +690,7 @@ proc ndarray.min(): ndarray(1,eltType) { } -/* Yields the maximum value from an :record:`ndarray`, as an :record:`ndarray`. - - :returns: The maximum value from the :record:`ndarray`. +/* :returns: The maximum value from the :record:`ndarray`. :rtype: ndarray(1, eltType) */ proc ndarray.max(): ndarray(1,eltType) { @@ -802,9 +805,7 @@ proc ndarray.topk(k: int): ndarray(1, int) where rank == 1 { } -/* Retrieve the index of the largest element in a one-dimensional :record:`ndarray`. - - :returns: The index of the largest element in a one-dimensional :record:`ndarray`. +/* :returns: The index of the largest element in a one-dimensional :record:`ndarray`. If there are multiple indices in the array that hold the maximal element, this method will return the smallest such index. :rtype: int @@ -842,82 +843,48 @@ proc ndarray.argmax() where rank == 1 { :returns: A new :record:`ndarray` with every element run through the recitifed linear unit function. */ inline proc ndarray.relu() { - const ref thisData = data; - const dom = this.domain; - var rl = new ndarray(dom,eltType); - ref rlD = rl.data; - // @assertOnGpu - forall i in dom.every() { - const x = thisData[i]; - rlD[i] = Math.max(0,x); - } - return rl; + return Bridge.relu(this : Bridge.tensorHandle(eltType)) : ndarray(rank, eltType); } -/* Square every element of an :record:`ndarray`. - - :returns: The a new :record:`ndarray` with the same data as the input, - squared elementwise. - */ -inline proc ndarray.square() { - const ref thisData = data; - const dom = this.domain; - var rl = new ndarray(dom,eltType); - ref rlD = rl.data; - forall i in dom.every() { - const x = thisData[i]; - rlD[i] = x * x; - } - return rl; -} /* Computes the Gaussian error linear units function for each element. .. math:: - \mathrm{GELU}(x) = 0.5 * x * \mathrm{erf}(x * \frac{1}{\sqrt{2}}) + \mathrm{GELU}(x) = 0.5 \cdot x \cdot \mathrm{erf}(x \cdot \frac{1}{\sqrt{2}}) :returns: A new :record:`ndarray` where every element has been passed through ``GELU`` as defined above. */ inline proc ndarray.gelu() { - // Here because `Math.recipSqrt2` is unstable. - param recipSqrt2 = 0.70710678118654752440; - - const ref thisData = data; - const dom = this.domain; - var rl = new ndarray(dom,eltType); - ref rlD = rl.data; - forall i in dom.every() { - const x = thisData[i]; - rlD[i] = x * (0.5 * (1.0 + Math.erf(x * recipSqrt2))); - } - return rl; + return Bridge.gelu(this : Bridge.tensorHandle(eltType)) : ndarray(rank, eltType); } +/* Computes the Sigmoid linear unit function for each element. + + This function is also known as the swish function. + + .. math:: + + \mathrm{silu}(x) = \frac{x}{\sigma(x)}\mathrm{,\ where}\ \sigma(x)\ \mathrm{is\ the\ logistic\ sigmoid.} + + :returns: A new :record:`ndarray` where the ``silu`` has been computed for each element, as defined above. +*/ inline proc ndarray.silu() { - const ref thisData = data; - const dom = this.domain; - var rl = new ndarray(dom, eltType); - ref rld = rl.data; - forall i in dom.every() { - const x = thisData[i]; - rld[i] = x / (1 + Math.exp(-x)); - } - return rl; + return Bridge.silu(this : Bridge.tensorHandle(eltType)) : ndarray(rank, eltType); } +/* Computes the mish function for each element. + + .. math:: + + \mathrm{mish}(x) = x\tanh(\ln(1 + e^x)) + + :returns: A new :record:`ndarray` where ``mish`` has been computed for each element, as defined above. + */ inline proc ndarray.mish() { - const ref thisData = data; - const dom = this.domain; - var rl = new ndarray(dom, eltType); - ref rld = rl.data; - forall i in dom.every() { - const x = thisData[i]; - rld[i] = x * Math.tanh(Math.log(1 + Math.exp(x))); - } - return rl; + return Bridge.mish(this : Bridge.tensorHandle(eltType)) : ndarray(rank, eltType); } @@ -971,108 +938,101 @@ inline proc ndarray.tanh() { Clamps every element in the range :math:`[0, 6]`. - :returns: A new :record:`ndarray` where every element has been clamped to the range :math:`[0, 6]`. + :returns: A new :record:`ndarray` where every element has been clamped to the range :math:`[0, 6]`. */ inline proc ndarray.relu6() { - const ref thisData = data; - const dom = this.domain; - var rl = new ndarray(dom, eltType); - ref rld = rl.data; - forall i in dom.every() { - const x = thisData[i]; - rld[i] = Math.min(Math.max(0, x), 6); - } - return rl; + return Bridge.relu6(this : Bridge.tensorHandle(eltType)) : ndarray(rank, eltType); } +/* Computes the SELU function for each element. + + .. math:: + + \mathrm{SELU}(x) = s \cdot (\max(0, x) + \min(0, \alpha \cdot (e^x - 1))) + + where :math:`\alpha = 1.6732632423543772848170429916717` and :math:`s = 1.0507009873554804934193349852946`. + + :returns: A new :record:`ndarray` where every element has been run through SELU as defined above. + */ inline proc ndarray.selu() { - const ref thisData = data; - const dom = this.domain; - var rl = new ndarray(dom, eltType); - ref rld = rl.data; - const alpha: eltType = 1.6732632423543772848170429916717; - const scale: eltType = 1.0507009873554804934193349852946; - forall i in dom.every() { - const x = thisData[i]; - rld[i] = scale * (Math.max(0, x) + Math.min(0, alpha * (Math.exp(x) - 1))); - } - return rl; + return Bridge.selu(this : Bridge.tensorHandle(eltType)) : ndarray(rank, eltType); } +/* Computes LogSigmoid for each element. + + .. math:: + + \mathrm{LogSigmoid}(x_i) = \log(\frac{1}{1 + e^{-x_i}}) + + + :returns: A new :record:`ndarray` where every element has had LogSigmoid computed for it. + */ inline proc ndarray.logsigmoid() { - const ref thisData = data; - const dom = this.domain; - var rl = new ndarray(dom, eltType); - ref rld = rl.data; - // branching is ok here b/c still in CPU + return Bridge.logsigmoid(this : Bridge.tensorHandle(eltType)) : ndarray(rank, eltType); +} - use CTypes only c_sizeof, c_ptrTo; - use OS.POSIX only memcpy; - type intType = int(numBits(eltType)); // integer w/ same #ofbits as real - forall i in dom.every() { - const x = thisData[i]; // negative x - const threshold: eltType = 20.0; - - var func: eltType = -Math.log1p(Math.exp(-x)); // result after applying logsigmoid(x) = -softplus(-x) activation function - var lin: eltType = x; // linear function: - (-x) = x +/* Computes Tanhshrink for each element. + + .. math:: + + \mathrm{Tanhshrink}(x) = x - \mathrm{Tanh}(x) - var func_bits: intType; - var lin_bits: intType; - memcpy(c_ptrTo(func_bits), c_ptrTo(func), c_sizeof(eltType)); - memcpy(c_ptrTo(lin_bits), c_ptrTo(lin), c_sizeof(eltType)); + :returns: A new :record:`ndarray` where every element has had Tanhshrink applied to it. + */ +inline proc ndarray.tanhshrink() { + return Bridge.tanhshrink(this : Bridge.tensorHandle(eltType)) : ndarray(rank, eltType); +} - const condition: intType = (-x > threshold); - const mask: intType = 0 - condition; - var result_bits: intType = (mask & lin_bits) | (~mask & func_bits); +/* Computes Softsign for each element. - // rld[i] = output; - memcpy(c_ptrTo(rld[i]), c_ptrTo(result_bits), c_sizeof(eltType)); - } - return rl; -} + .. math:: -inline proc ndarray.tanhshrink() { - const ref thisData = data; - const dom = this.domain; - var rl = new ndarray(dom, eltType); - ref rld = rl.data; - forall i in dom.every() { - const x = thisData[i]; - rld[i] = x - Math.tanh(x); - } - return rl; -} + \mathrm{Softsign}(x) = \frac{x}{1 + \left|x\right|} + :returns: A new :record:`ndarray` where every element has had Softsign applied to it. + */ inline proc ndarray.softsign() { - const ref thisData = data; - const dom = this.domain; - var rl = new ndarray(dom, eltType); - ref rld = rl.data; - forall i in dom.every() { - const x = thisData[i]; - rld[i] = x / (1 + Math.abs(x)); - } - return rl; + return Bridge.softsign(this : Bridge.tensorHandle(eltType)) : ndarray(rank, eltType); } + +/* Computes the Randomized Leaky ReLU function for each element. + + .. math:: + + \mathrm{RReLU}(x) = \begin{cases} + x & \text{ if } x \geq 0 \\ + \alpha{x} & \text{ otherwise } + \end{cases} + + where :math:`\alpha` is randomly sampled from uniform distribution + :math:`\mathcal{U}(\mathrm{lower}, \mathrm{upper})` during training + while during evaluation :math:`\alpha` is fixed with + :math:`\alpha = \frac{\mathrm{lower} + \mathrm{upper}}{2}`. + + :arg lower: The lower bound of the uniform distribution :math:`\mathcal{U}`. Default: :math:`\frac{1}{8}` + :type lower: eltType + + :arg upper: The upper bound of the uniform distribution :math:`\mathcal{U}`. Default: :math:`\frac{1}{3}` + :type upper: eltType + + :arg training: Whether or not in training mode. Default: `false` + :type training: bool + + :returns: A new :record:`ndarray` where RReLU has been computed for each element. + :rtype: ndarray(rank, eltType) + */ inline proc ndarray.rrelu(lower: eltType=1.0/8.0, upper: eltType=1.0/3.0, training: bool=false) { - const ref thisData = data; - const dom = this.domain; - var rl = new ndarray(dom, eltType); - ref rld = rl.data; - var a: [dom] eltType = (lower + upper) / 2.0; - if training then - Random.fillRandom(a,min=lower,max=upper); - forall i in dom.every() { - const x = thisData[i]; - const alpha = a[i]; - rld[i] = Math.max(0, x) + alpha * Math.min(0,x); - } - return rl; + return Bridge.rrelu( + this : Bridge.tensorHandle(eltType), + lower : real(32), + upper : real(32), + training + ) : ndarray(rank, eltType); } inline proc ndarray.hardswish() { @@ -1102,160 +1062,202 @@ inline proc ndarray.hardsigmoid() { return rl; } -inline proc ndarray.hardShrink(alpha: eltType=0.5) { - const ref thisData = data; - const dom = this.domain; - var rl = new ndarray(dom, eltType); - ref rld = rl.data; - forall i in dom.every() { - const x = thisData[i]; - const floatMax = Types.max(eltType); - const xmap0 = Math.ceil(1.0 / floatMax * (x - alpha) * (x + alpha)); // 0 if x in [-l, l], 1 otherwise - rld[i] = x * xmap0; - } - return rl; + +/* Computes the HardShrink function for each element. + + .. math:: + + \mathrm{HardShrink}(x) = \begin{cases} + x & \text{ if } x > \alpha \\ + x & \text{ if } x < -\alpha \\ + 0 \text{ otherwise } + \end{cases} + + :arg alpha: :math:`\alpha` in the definition of HardShrink. Default: :math:`\frac{1}{2}` + :type alpha: eltType + + :returns: A new :record:`ndarray` where HardShrink has been computed for each element. + :rtype: ndarray(rank, eltType) + */ +inline proc ndarray.hardshrink(alpha: eltType=0.5) { + return Bridge.hardshrink( + this : Bridge.tensorHandle(eltType), + alpha : real(32) + ) : ndarray(rank, eltType); } -inline proc ndarray.hardTanh(minVal: eltType=-1.0, maxVal: eltType=1.0) { - const ref thisData = data; - const dom = this.domain; - var rl = new ndarray(dom, eltType); - ref rld = rl.data; - forall i in dom.every() { - const x = thisData[i]; - const floatMax: eltType = Types.max(eltType); - const xgmaxval: eltType = Math.ceil(1.0 / floatMax * (x - maxVal)); // x greater than maxVal: 1 if true, 0 otherwise - const xlminval: eltType = Math.ceil(1.0 / floatMax * (x - minVal)); // x less than minVal: 1 if true, o otherwise - rld[i] = Math.max(x, minVal) * (1 - xlminval) + Math.min(x, maxVal) * xgmaxval + x * xlminval * (1 - xgmaxval); - } - return rl; + +/* Computes the HardTanh function for each element. + + .. math:: + + \mathrm{HardTanh}(x) = \begin{cases} + \mathrm{maxVal} & \text{ if } x > \mathrm{maxVal} \\ + \mathrm{minVal} & \text{ if } x < \mathrm{minVal} \\ + x & \text{ otherwise } + \end{cases} + + :arg minVal: The minimum value. Default: `-1` + :type minVal: eltType + + :arg maxVal: The maximum value. Default: `1` + :type maxVal: eltType + + :returns: A new :record:`ndarray` where HardTanh has been applied to each element. + :rtype: ndarray(rank, eltType) + */ +inline proc ndarray.hardtanh(minVal: eltType=-1.0, maxVal: eltType=1.0) { + return Bridge.hardtanh( + this : Bridge.tensorHandle(eltType), + minVal : real(32), + maxVal : real(32) + ) : ndarray(rank, eltType); } + +/* Computes ELU for each element. + + .. math:: + + \mathrm{ELU}(x) = \begin{cases} + x & \text{ if } x > 0 \\ + \alpha \cdot (e^x - 1) & \text{ otherwise } + \end{cases} + + :arg alpha: The :math:`\alpha` value for ELU. Default: `1` + :type alpha: eltType + + :returns: A new :record:`ndarray` where ELU has been computed for each element. + :rtype: ndarray(rank, eltType) + */ inline proc ndarray.elu(alpha: eltType=1.0) { - const ref thisData = data; - const dom = this.domain; - var rl = new ndarray(dom, eltType); - ref rld = rl.data; - forall i in dom.every() { - const x = thisData[i]; - const floatMax: eltType = Types.max(eltType); - const xgz: eltType = Math.ceil(x / floatMax); // x greater than zero: 1 if true, 0 otherwise - rld[i] = x * xgz + alpha * (Math.exp(x) - 1) * (1 - xgz); - } - return rl; + return Bridge.elu( + this : Bridge.tensorHandle(eltType), + alpha : real(32) + ) : ndarray(rank, eltType); } -inline proc ndarray.softplus(beta: eltType=1.0, threshold: eltType=20.0) { - const ref thisData = data; - const dom = this.domain; - var rl = new ndarray(dom, eltType); - ref rld = rl.data; +/* Computes Softplus for each element. - // branching is ok here since we are still in the CPU + .. math:: - // beta = 0 should return inf according to PyTorch, but there is no real32 inf - if beta == 0 { - rld = Math.inf; - } + \mathrm{Softplus}(x) = \frac{1 + e^(\beta \cdot x)}{\beta} - // beta non-zero here - else { - use CTypes only c_sizeof, c_ptrTo; - use OS.POSIX only memcpy; + Reverts to the linear function when :math:`\beta{\mathrm{input}} > \mathrm{threshold}`. + + :arg beta: :math:`\beta` value for Softplus. Default: `1` + :type beta: eltType + + :arg threshold: The threshold over which to revert to the linear function. + :type threshold: eltType + + :returns: A new :record:`ndarray` where Softplus has been computed for each element. + :rtype: ndarray(rank, eltType) + */ +inline proc ndarray.softplus(beta: eltType=1.0, threshold: eltType=20.0) { + return Bridge.softplus( + this : Bridge.tensorHandle(eltType), + beta : real(32), + threshold : real(32) + ) : ndarray(rank, eltType); +} - type intType = int(numBits(eltType)); // integer w/ same #ofbits as real - forall i in dom.every() { - const x = thisData[i]; - - var func: eltType = Math.log1p(Math.exp(beta * x)) / beta; // result after applying softplus activation function - var lin: eltType = x; // linear function - var func_bits: intType; - var lin_bits: intType; - memcpy(c_ptrTo(func_bits), c_ptrTo(func), c_sizeof(eltType)); - memcpy(c_ptrTo(lin_bits), c_ptrTo(lin), c_sizeof(eltType)); +/* Thresholds each element of the input tensor. - const condition: intType = (x * beta > threshold); - const mask: intType = 0 - condition; + .. math:: - var result_bits: intType = (mask & lin_bits) | (~mask & func_bits); + \mathrm{Threshold}(x) = \begin{cases} + x & \text{ if } x > \mathrm{threshold} \\ + \mathrm{value} & \text{ otherwise } + \end{cases} - // rld[i] = output; - memcpy(c_ptrTo(rld[i]), c_ptrTo(result_bits), c_sizeof(eltType)); - } - } - return rl; -} + :arg threshold: The value to threshold at + :type threshold: eltType + + :arg value: The value to replace with + :type value: eltType + :returns: A new :record:`ndarray` where every element has been thresholded. + :rtype: ndarray(rank, eltType) + */ inline proc ndarray.threshold(threshold: eltType, value: eltType) { // PyTorch has no defaults for threshold - const ref thisData = data; - const dom = this.domain; - var rl = new ndarray(dom, eltType); - ref rld = rl.data; - forall i in dom.every() { - const x = thisData[i]; - const floatMax: eltType = Types.max(eltType); // maximum value a float_64 can take - const xgeqt: eltType = Math.ceil((x - threshold) / floatMax); // 1 if x >= threshold, 0 otherwise - rld[i] = x * xgeqt + value * (1 - xgeqt); - } - return rl; + return Bridge.threshold( + this : Bridge.tensorHandle(eltType), + threshold : real(32), + value : real(32) + ) : ndarray(rank, eltType); } -// inline proc ndarray.softplus(beta: eltType=1.0, threshold: eltType=20.0) { -// const ref thisData = data; -// const dom = this.domain; -// var rl = new ndarray(dom, eltType); -// ref rld = rl.data; -// forall i in dom.every() { -// const x = thisData[i]; -// const floatMax: eltType = Types.max(eltType); -// const xgbt: eltType = Math.ceil((x - threshold / beta) / floatMax); // x greater than beta * threshold: 1 if true, 0 otherwise -// rld[i] = x * xgbt + 1.0 / beta * Math.log(1 + Math.exp(beta * x)) * (1 - xgbt); -// } -// return rl; -// } +/* Compute CELU for each element. + + .. math:: + + \mathrm{CELU}(x) = \max(0, x) + \min(0, \alpha \cdot (e^\frac{x}{\alpha} - 1)) + + :arg alpha: The :math:`\alpha` value for CELU. Default: `1` + :type alpha: eltType + + :returns: A new :record:`ndarray` where CELU has been computed for each element. + :rtype: ndarray(rank, eltType) + */ inline proc ndarray.celu(alpha: eltType=1.0) { - const ref thisData = data; - const dom = this.domain; - var rl = new ndarray(dom, eltType); - ref rld = rl.data; - forall i in dom.every() { - const x = thisData[i]; - rld[i] = Math.max(0.0, x) + Math.min(0.0, alpha * (Math.exp(x / alpha) - 1.0)); - } - return rl; + return Bridge.celu( + this : Bridge.tensorHandle(eltType), + alpha : real(32) + ) : ndarray(rank, eltType); } -inline proc ndarray.leakyrelu(negativeSlope: eltType=Math.exp(-2)) { - const ref thisData = data; - const dom = this.domain; - var rl = new ndarray(dom, eltType); - ref rld = rl.data; - forall i in dom.every() { - const x = thisData[i]; - rld[i] = Math.max(0, x) + negativeSlope * Math.min(0, x); - } - return rl; + +/* Compute LeakyReLU for each element. + + .. math:: + + \mathrm{LeakyReLU}(x) = \max(0, x) + \mathrm{negativeSlope} \cdot \min(0, x) + + :arg negativeSlope: Controls the angle of the negative slope which is used for negative input values. Default: :math:`\frac{1}{100}` + :type negativeSlope: eltType + + :returns: A new :record:`ndarray` where LeakyReLU has been applied to each element. + :rtype: ndarray(rank, eltType) + */ +inline proc ndarray.leakyRelu(negativeSlope: eltType=0.01) { + return Bridge.leakyRelu( + this : Bridge.tensorHandle(eltType), + negativeSlope : real(32) + ) : ndarray(rank, eltType); } + +/* Applies the soft shrinkage function to each element. + + .. math:: + + \mathrm{SoftShrinkage}(x) = \begin{cases} + x - \alpha & \text{ if } x > \alpha \\ + x + \alpha & \text{ if } x < -\alpha \\ + 0 & \text{ otherwise } + \end{cases} + + :arg alpha: The :math:`alpha` value for the SoftShrinkage formulation. + :type alpha: eltType + + :returns: A new :record:`ndarray` where SoftShrinkage has been applied to each element. + :rtype: ndarray(rank, eltType) + */ inline proc ndarray.softshrink(alpha: eltType=0.5) { // l must be non-negative - if alpha < 0 then util.err("Argument to softshrink function must be non-negative"); - const ref thisData = data; - const dom = this.domain; - var rl = new ndarray(dom, eltType); - ref rld = rl.data; - const floatMax: eltType = Types.max(eltType); - forall i in dom.every() { - const x = thisData[i]; - const xgl = Math.ceil(1.0 / floatMax * (x - alpha)); // x greater than lambda: 1 if true, otherwise - const xlnl = 1 - Math.ceil(1.0 / floatMax * (x + alpha)); // x less than negative lambda, 1 if true, 0 otherwise - rld[i] = xgl * (x - alpha) + xlnl * (x + alpha); - } - return rl; + return Bridge.softshrink( + this : Bridge.tensorHandle(eltType), + alpha : real(32) + ) : ndarray(rank, eltType); } + +/* :returns: A one-dimensional array consisting of all the elements of `this`. + :rtype: [] eltType + */ proc ndarray.degenerateFlatten(): [] eltType { const myDom = this.domain; const mySize = myDom.size; @@ -1338,54 +1340,6 @@ operator :(it: _iteratorRecord, type t: ndarray(?rank,?eltType)) do return new ndarray(it); -// Need help implementtion these. -// operator =(ref lhs: ndarray(?rank,?eltType), rhs: _iteratorRecord) { -// var arr = rhs; -// lhs._domain = arr.domain; -// lhs.data = arr; -// } -// operator :(val: _iteratorRecord, type t: ndarray(?rank,?eltType)) { -// return new ndarray(val); -// } - - -// // This bunch is problematic. -// proc remote.init(other: ndarray(?rank,?eltType)) { -// this.init(ndarray(rank,eltType)); -// other.populateRemote(this); -// } - -// proc remote.init=(ref other: ndarray(?rank,?eltType)) { -// this.init(ndarray(rank,eltType)); -// other.populateRemote(this); -// } - - - -// operator =(ref lhs: remote(ndarray(?rank,?eltType)), rhs: ndarray(rank,eltType)) { -// rhs.populateRemote(lhs); -// } - -// operator :(val: ndarray(?rank,?eltType), type t: remote(ndarray(rank,eltType))) { -// return val.toRemote(); -// } - -// proc remote.init(ref other: remote(ndarray(?rank,?eltType))) { -// this.eltType = ndarray(rank,eltType); -// this.remoteResource = other.remoteResource; -// } - -// proc remote.init=(ref other: remote(ndarray(?rank,?eltType))) { -// this.eltType = ndarray(rank,eltType); -// this.remoteResource = other.remoteResource; -// } - -// operator =(ref lhs: remote(ndarray(?rank,?eltType)), rhs:remote( ndarray(rank,eltType))) { -// lhs.remoteResource = rhs.remoteResource; -// } - -// End problemetic bunch. - proc zipArr(a: ndarray(?rank,?eltType),b: ndarray(rank,eltType),f): ndarray(rank,eltType) { const dom = a.domain; var c: ndarray(rank,eltType) = new ndarray(a.domain,eltType); @@ -2443,14 +2397,6 @@ proc type ndarray.multiReader(path: string) throws { } proc type ndarray.loadFrom(filePath: string, param rank: int, type eltType = defaultEltType): ndarray(rank,eltType) throws { - // var arr = new ndarray(rank,eltType); - // var file = IO.open(filePath, IO.ioMode.r); - // var fr = file.reader(); - // arr.read(fr); - // return arr; - - // return ndarray.readInPlace(multiReader(filePath),rank,eltType); - var arr = new ndarray(rank,eltType); var fr = ndarray.multiReader(filePath); arr.read(fr); @@ -2474,82 +2420,6 @@ class _tensor_resource { } -// Some examples. -// const dom = util.emptyDomain(2); -// writeln(dom); - -// printType(ndarray(1)); -// var A: ndarray(1); -// A._domain = {0..#3}; -// // A.data = [1.0,2.0,3.0]; -// A = [1.0,2.0,3.0]; -// writeln(A); - -// use GpuDiagnostics; - -// startGpuDiagnostics(); -// startVerboseGpu(); - -// var device = here.gpus[0]; - -// on device { -// var arr = [1.0,2.0,3.0]; - -// var A: ndarray(1) = arr; -// writeln("A -> ",A); - -// var B: ndarray(1) = new ndarray(arr); -// // B = [1.0,2.0,3.0]; - -// writeln("B -> ",B.domain); - -// var dom = {0..#3,0..#5}; - -// var C: ndarray(2) = new ndarray(dom); -// writeln("C -> ",C); -// } - -// var A: ndarray(1) = [1.0,2.0,3.0]; -// writeln(A); - -// var B: ndarray(1) = new ndarray([1.0,2.0,3.0]); -// // B = [1.0,2.0,3.0]; - -// writeln(B.domain); - -// var C: ndarray(2) = new ndarray({0..#3,0..#5}); -// var c = 0; -// for i in C.domain { -// C.data[i] = c; -// c += 1; -// } -// writeln(C); - -// C = C.reshape({0..#5,0..#3}); -// writeln(C); - -// writeln(C[0,0]); - -// C[0,0] = 70.0; - -// var D = C[{0..1,0..1}]; - -// C[{0..1,0..1}]=2.0; - - -// writeln(D,D.type:string); -// writeln(C,C.type:string); - -// var E = C.slice(1,..); -// writeln(E); - - -// type T = (int,real); -// writeln(T:string,isTupleType(T)); - -// end examples. - - proc type ndarray.fullOuter(a: ndarray(?rankA,?eltType), b: ndarray(?rankB, eltType)): ndarray(rankA + rankB, eltType) { param rankC = rankA + rankB; @@ -2638,19 +2508,13 @@ proc drop(param count: int, param s: string) param do proc type ndarray.einsum(param subscripts: string,a: ndarray(?rankA,?eltType), b: ndarray(?rankB, eltType)) { - for param i in 0..ih",A,B); - // writeln(E); - // E[1,1] = 2; +proc ndarray.featureAlphaDropout(p : real = 0.5, training : bool = false) { + return Bridge.featureAlphaDropout( + this : Bridge.tensorHandle(eltType), + p, training + ) : ndarray(rank, eltType); +} - var A = ndarray.fromRanges(real, 0..<5,0..<3); - var B = ndarray.fromRanges(real, 0..<5,0..<3); +/* Randomly zero out entire channels (a channel is a 2D feature map). - writeln(A + B); + For example, the :math:`j`-th channel of the :math:`i`-th sample in the batched + input is a 2D tensor :math:`\mathrm{input}[i,j]` of the input tensor. + Each channel will be zeroed out independently on every forward call with probability `p` + using samples from a Bernoulli distribution. + :arg p: Probability of a channel to be zeroed. Default: `0.5` + :type p: real - // param r = 0..<3; - // writeln(r); -}*/ + :arg training: Apply dropout if `true`. Default: `false` + :type training: bool + :returns: A new :record:`ndarray` where dropout has been applied. + :rtype: ndarray(rank, eltType) + */ +proc ndarray.dropout2d(p : real = 0.5, training : bool = false) { + return Bridge.dropout2d( + this : Bridge.tensorHandle(eltType), + p, training + ) : ndarray(rank, eltType); } + + +/* Randomly zero out entire channels (a channel is a 3D feature map). + + For example, the :math:`j`-th channel of the :math:`i`-th sample in the batched + input is a 3D tensor :math:`\mathrm{input}[i,j]` of the input tensor. + Each channel will be zeroed out independently on every forward call with probability `p` + using samples from a Bernoulli distribution. + + :arg p: Probability of a channel to be zeroed. Default: `0.5` + :type p: real + + :arg training: Apply dropout if `true`. Default: `false` + :type training: bool + + :returns: A new :record:`ndarray` where dropout has been applied. + :rtype: ndarray(rank, eltType) + */ +proc ndarray.dropout3d(p : real = 0.5, training : bool = false) { + return Bridge.dropout3d( + this : Bridge.tensorHandle(eltType), + p, training + ) : ndarray(rank, eltType); +} + + +} \ No newline at end of file