|
| 1 | +/* |
| 2 | +* Simd Library (http://ermig1979.github.io/Simd). |
| 3 | +* |
| 4 | +* Copyright (c) 2011-2026 Yermalayeu Ihar. |
| 5 | +* |
| 6 | +* Permission is hereby granted, free of charge, to any person obtaining a copy |
| 7 | +* of this software and associated documentation files (the "Software"), to deal |
| 8 | +* in the Software without restriction, including without limitation the rights |
| 9 | +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| 10 | +* copies of the Software, and to permit persons to whom the Software is |
| 11 | +* furnished to do so, subject to the following conditions: |
| 12 | +* |
| 13 | +* The above copyright notice and this permission notice shall be included in |
| 14 | +* all copies or substantial portions of the Software. |
| 15 | +* |
| 16 | +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 17 | +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 18 | +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 19 | +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 20 | +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 21 | +* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 22 | +* SOFTWARE. |
| 23 | +*/ |
| 24 | +#include "Simd/SimdSynetConvolution32f.h" |
| 25 | +#include "Simd/SimdSynetConvolution32fCommon.h" |
| 26 | +#include "Simd/SimdSynet.h" |
| 27 | +#include "Simd/SimdSve2.h" |
| 28 | +#include "Simd/SimdNeon.h" |
| 29 | +#include "Simd/SimdGemm.h" |
| 30 | +#include "Simd/SimdCpu.h" |
| 31 | + |
| 32 | +namespace Simd |
| 33 | +{ |
| 34 | +#if defined(SIMD_SVE2_ENABLE) && defined(SIMD_SYNET_ENABLE) |
| 35 | + namespace Sve2 |
| 36 | + { |
| 37 | + SynetConvolution32fGemmNN::SynetConvolution32fGemmNN(const ConvParam & p) |
| 38 | + : Base::SynetConvolution32fGemmNN(p) |
| 39 | + { |
| 40 | + const size_t F = svcntw(); |
| 41 | + _index.Resize(F); |
| 42 | + for (size_t i = 0; i < F; ++i) |
| 43 | + _index[i] = int(i * p.strideX); |
| 44 | + _nose.Resize(p.kernelX); |
| 45 | + _tail.Resize(p.kernelX); |
| 46 | + _start.Resize(p.kernelX); |
| 47 | + for (size_t kx = 0; kx < p.kernelX; ++kx) |
| 48 | + { |
| 49 | + _nose[kx] = 0; |
| 50 | + _tail[kx] = int(p.dstW); |
| 51 | + ptrdiff_t sx = kx * p.dilationX - p.padX; |
| 52 | + for (size_t dx = 0; dx < p.dstW; ++dx) |
| 53 | + { |
| 54 | + if (sx < 0) |
| 55 | + _nose[kx]++; |
| 56 | + if (sx >= ptrdiff_t(p.srcW)) |
| 57 | + _tail[kx]--; |
| 58 | + sx += p.strideX; |
| 59 | + } |
| 60 | + _start[kx] = int(kx * p.dilationX - p.padX + _nose[kx] * p.strideX); |
| 61 | + } |
| 62 | + _gemm.Init(InitGemmFuncs(Sve2::Gemm32fNN, "Sve2")); |
| 63 | + _biasAndActivation = Neon::ConvolutionBiasAndActivation; |
| 64 | + } |
| 65 | + |
| 66 | + //------------------------------------------------------------------------------------------------- |
| 67 | + |
| 68 | + void SynetConvolution32fGemmNN::ImgToCol(const float * src, float * dst) |
| 69 | + { |
| 70 | + const ConvParam & p = _param; |
| 71 | + size_t srcSize = p.srcW * p.srcH; |
| 72 | + if (p.dilationX == 1 && p.dilationY == 1 && p.strideX == 2 && p.strideY == 2 && p.padX == 0 && p.padY == 0 && p.padW == 0 && p.padH == 0 && p.kernelX == 1 && p.kernelY == 1) |
| 73 | + { |
| 74 | + for (size_t c = 0; c < p.srcC; ++c) |
| 75 | + { |
| 76 | + for (size_t dy = 0; dy < p.dstH; ++dy) |
| 77 | + { |
| 78 | + const float * psrc = src + 2 * dy * p.srcW; |
| 79 | + for (size_t dx = 0, sx = 0; dx < p.dstW; ++dx, sx += 2) |
| 80 | + *(dst++) = psrc[sx]; |
| 81 | + } |
| 82 | + src += srcSize; |
| 83 | + } |
| 84 | + } |
| 85 | + else if (p.dilationX * p.dilationY * p.strideX * p.strideY != 1) |
| 86 | + { |
| 87 | + const size_t F = svcntw(); |
| 88 | + svbool_t all = svptrue_b32(); |
| 89 | + svuint32_t index = svld1_u32(all, (const uint32_t*)_index.data); |
| 90 | + for (size_t c = 0; c < p.srcC; ++c) |
| 91 | + { |
| 92 | + for (size_t ky = 0; ky < p.kernelY; ky++) |
| 93 | + { |
| 94 | + for (size_t kx = 0; kx < p.kernelX; kx++) |
| 95 | + { |
| 96 | + size_t noseDx = _nose[kx]; |
| 97 | + size_t tailDx = _tail[kx]; |
| 98 | + size_t bodyDx = AlignLo(tailDx - noseDx, F) + noseDx; |
| 99 | + size_t sx0 = _start[kx]; |
| 100 | + size_t sy = ky * p.dilationY - p.padY; |
| 101 | + for (size_t dy = 0; dy < p.dstH; ++dy) |
| 102 | + { |
| 103 | + if (sy < p.srcH) |
| 104 | + { |
| 105 | + size_t dx = 0, sx = sx0 + sy * p.srcW; |
| 106 | + for (; dx < noseDx; ++dx) |
| 107 | + *(dst++) = 0; |
| 108 | + for (; dx < bodyDx; dx += F, sx += p.strideX * F, dst += F) |
| 109 | + svst1_f32(all, dst, svld1_gather_u32index_f32(all, src + sx, index)); |
| 110 | + for (; dx < tailDx; ++dx, sx += p.strideX) |
| 111 | + *(dst++) = src[sx]; |
| 112 | + for (; dx < p.dstW; ++dx) |
| 113 | + *(dst++) = 0; |
| 114 | + } |
| 115 | + else |
| 116 | + { |
| 117 | + memset(dst, 0, p.dstW * sizeof(float)); |
| 118 | + dst += p.dstW; |
| 119 | + } |
| 120 | + sy += p.strideY; |
| 121 | + } |
| 122 | + } |
| 123 | + } |
| 124 | + src += srcSize; |
| 125 | + } |
| 126 | + } |
| 127 | + else |
| 128 | + { |
| 129 | + Base::SynetConvolution32fGemmNN::ImgToCol(src, dst); |
| 130 | + } |
| 131 | + } |
| 132 | + |
| 133 | + //------------------------------------------------------------------------------------------------- |
| 134 | + |
| 135 | + void SynetConvolution32fGemmNN::ImgToRow(const float * src, float * dst) |
| 136 | + { |
| 137 | + const ConvParam & p = _param; |
| 138 | + assert(p.trans); |
| 139 | + const size_t F = svcntw(); |
| 140 | + size_t size = p.srcC / p.group; |
| 141 | + for (size_t g = 0; g < p.group; ++g) |
| 142 | + { |
| 143 | + for (size_t dy = 0; dy < p.dstH; ++dy) |
| 144 | + { |
| 145 | + for (size_t dx = 0; dx < p.dstW; ++dx) |
| 146 | + { |
| 147 | + for (size_t ky = 0; ky < p.kernelY; ky++) |
| 148 | + { |
| 149 | + size_t sy = dy * p.strideY + ky * p.dilationY - p.padY; |
| 150 | + if (sy < p.srcH) |
| 151 | + { |
| 152 | + for (size_t kx = 0; kx < p.kernelX; kx++) |
| 153 | + { |
| 154 | + size_t sx = dx * p.strideX + kx * p.dilationX - p.padX; |
| 155 | + if (sx < p.srcW) |
| 156 | + { |
| 157 | + const float * s = src + (sy * p.srcW + sx) * p.srcC; |
| 158 | + for (size_t i = 0; i < size; i += F) |
| 159 | + { |
| 160 | + svbool_t mask = svwhilelt_b32(i, size); |
| 161 | + svst1_f32(mask, dst + i, svld1_f32(mask, s + i)); |
| 162 | + } |
| 163 | + dst += size; |
| 164 | + } |
| 165 | + else |
| 166 | + { |
| 167 | + for (size_t i = 0; i < size; i += F) |
| 168 | + { |
| 169 | + svbool_t mask = svwhilelt_b32(i, size); |
| 170 | + svst1_f32(mask, dst + i, svdup_n_f32(0.0f)); |
| 171 | + } |
| 172 | + dst += size; |
| 173 | + } |
| 174 | + } |
| 175 | + } |
| 176 | + else |
| 177 | + { |
| 178 | + size_t n = p.kernelX * size; |
| 179 | + for (size_t i = 0; i < n; i += F) |
| 180 | + { |
| 181 | + svbool_t mask = svwhilelt_b32(i, n); |
| 182 | + svst1_f32(mask, dst + i, svdup_n_f32(0.0f)); |
| 183 | + } |
| 184 | + dst += n; |
| 185 | + } |
| 186 | + } |
| 187 | + } |
| 188 | + } |
| 189 | + src += size; |
| 190 | + } |
| 191 | + } |
| 192 | + |
| 193 | + //------------------------------------------------------------------------------------------------- |
| 194 | + |
| 195 | + void * SynetConvolution32fInit(size_t batch, const SimdConvolutionParameters * conv) |
| 196 | + { |
| 197 | + ConvParam param(batch, conv, SimdSynetCompatibilityDefault); |
| 198 | + if (!param.Valid(SimdTensorData32f)) |
| 199 | + return NULL; |
| 200 | + if (Neon::SynetConvolution32fDepthwiseDotProduct::Preferable(param)) |
| 201 | + return new Neon::SynetConvolution32fDepthwiseDotProduct(param); |
| 202 | + else if (Neon::SynetConvolution32fWinograd::Preferable(param)) |
| 203 | + return new Neon::SynetConvolution32fWinograd(param); |
| 204 | + else if (Neon::SynetConvolution32fDirectNchw::Preferable(param)) |
| 205 | + return new Neon::SynetConvolution32fDirectNchw(param); |
| 206 | + else if (Neon::SynetConvolution32fGemmNT::Preferable(param)) |
| 207 | + return new Neon::SynetConvolution32fGemmNT(param); |
| 208 | + else if (Neon::SynetConvolution32fNhwcDirect::Preferable(param)) |
| 209 | + return new Neon::SynetConvolution32fNhwcDirect(param); |
| 210 | + else if (Neon::SynetConvolution32fNhwcDepthwise::Preferable(param)) |
| 211 | + return new Neon::SynetConvolution32fNhwcDepthwise(param); |
| 212 | + else if (Base::SynetConvolution32fNhwcGroupedBlock1x2::Preferable(param)) |
| 213 | + return new Base::SynetConvolution32fNhwcGroupedBlock1x2(param); |
| 214 | + else |
| 215 | + return new SynetConvolution32fGemmNN(param); |
| 216 | + } |
| 217 | + } |
| 218 | +#endif |
| 219 | +} |
0 commit comments