Skip to content

Commit aeed46f

Browse files
rsudermanclaude
rsuderman
and
claude
authored
Add SOFTPLUS_FWD and SWISH_FWD pointwise ops (#355)
Emit torch.aten.softplus with configurable beta and threshold attributes (defaulting to 1.0 and 20.0) and torch.aten.silu for swish. Adds sample and lit tests for both ops. Signed-off-by: Rob Suderman <rob.suderman@gmail.com> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent f6de190 commit aeed46f

6 files changed

Lines changed: 182 additions & 2 deletions

File tree

‎include/fusilli/attributes/pointwise_attributes.h‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,11 +64,11 @@ namespace fusilli {
6464
OP(SIGMOID_FWD) \
6565
OP(SIN) \
6666
/* OP(SOFTPLUS_BWD) */ \
67-
/* OP(SOFTPLUS_FWD) */ \
67+
OP(SOFTPLUS_FWD) \
6868
OP(SQRT) \
6969
OP(SUB) \
7070
/* OP(SWISH_BWD) */ \
71-
/* OP(SWISH_FWD) */ \
71+
OP(SWISH_FWD) \
7272
OP(TAN) \
7373
/* OP(TANH_BWD) */ \
7474
OP(TANH_FWD)
@@ -106,6 +106,16 @@ class PointwiseAttr : public AttributesCRTP<PointwiseAttr> {
106106
return *this;
107107
}
108108

109+
PointwiseAttr &setSoftplusBeta(float beta) {
110+
softplusBeta_ = beta;
111+
return *this;
112+
}
113+
114+
PointwiseAttr &setSoftplusThreshold(float threshold) {
115+
softplusThreshold_ = threshold;
116+
return *this;
117+
}
118+
109119
// Getters:
110120
FUSILLI_GENERIC_INPUT_TENSOR_GETTER(InputNames, IN_0)
111121
FUSILLI_GENERIC_INPUT_TENSOR_GETTER(InputNames, IN_1)
@@ -114,6 +124,8 @@ class PointwiseAttr : public AttributesCRTP<PointwiseAttr> {
114124

115125
Mode getMode() const { return mode_; }
116126
float getEluAlpha() const { return eluAlpha_; }
127+
float getSoftplusBeta() const { return softplusBeta_; }
128+
float getSoftplusThreshold() const { return softplusThreshold_; }
117129

118130
// Utilities for pointwise modes.
119131
static const std::unordered_map<Mode, std::string> kModeToStr;
@@ -123,6 +135,8 @@ class PointwiseAttr : public AttributesCRTP<PointwiseAttr> {
123135
private:
124136
Mode mode_ = Mode::NOT_SET;
125137
float eluAlpha_ = 1.0f;
138+
float softplusBeta_ = 1.0f;
139+
float softplusThreshold_ = 20.0f;
126140
};
127141

128142
#define FUSILLI_DECLARE_STRINGIFY_POINTWISE_MODE(mode) \
@@ -165,8 +179,10 @@ inline const std::unordered_map<PointwiseAttr::Mode, int>
165179
{PointwiseAttr::Mode::RSQRT, 1},
166180
{PointwiseAttr::Mode::SIGMOID_FWD, 1},
167181
{PointwiseAttr::Mode::SIN, 1},
182+
{PointwiseAttr::Mode::SOFTPLUS_FWD, 1},
168183
{PointwiseAttr::Mode::SQRT, 1},
169184
{PointwiseAttr::Mode::SUB, 2},
185+
{PointwiseAttr::Mode::SWISH_FWD, 1},
170186
{PointwiseAttr::Mode::TAN, 1},
171187
{PointwiseAttr::Mode::TANH_FWD, 1}};
172188

‎include/fusilli/support/asm_emitter.h‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1801,6 +1801,14 @@ inline std::string PointwiseNode::emitNodePreAsm() const {
18011801
{5}
18021802
)";
18031803

1804+
constexpr std::string_view kSoftplusSchema = R"(
1805+
{0}
1806+
%softplus_beta_{7} = torch.constant.float {8:e}
1807+
%softplus_threshold_{7} = torch.constant.float {9:e}
1808+
{1} = {6} {2}, %softplus_beta_{7}, %softplus_threshold_{7} : {3}, !torch.float, !torch.float -> {4}
1809+
{5}
1810+
)";
1811+
18041812
#define FUSILLI_DECLARE_UNARY_TORCH_EMITTER(PWOP, OPIR) \
18051813
FUSILLI_DECLARE_UNARY_POINTWISE_EMITTER(PWOP, kUnaryTorchSchema, OPIR)
18061814
#define FUSILLI_DECLARE_BINARY_TORCH_EMITTER(PWOP, OPIR) \
@@ -1839,6 +1847,20 @@ inline std::string PointwiseNode::emitNodePreAsm() const {
18391847
approx /* {8} */
18401848
);
18411849
}
1850+
case PointwiseAttr::Mode::SOFTPLUS_FWD: {
1851+
return std::format(kSoftplusSchema, permuteIN0, /* {0} */
1852+
getResultNamesAsm(), /* {1} */
1853+
getOperandNamesAsm(), /* {2} */
1854+
getOperandTypesAsm(), /* {3} */
1855+
getResultTypesAsm(), /* {4} */
1856+
permuteOUT0, /* {5} */
1857+
"torch.aten.softplus", /* {6} */
1858+
getName(), /* {7} */
1859+
pointwiseAttr.getSoftplusBeta(), /* {8} */
1860+
pointwiseAttr.getSoftplusThreshold() /* {9} */
1861+
);
1862+
}
1863+
FUSILLI_DECLARE_UNARY_TORCH_EMITTER(SWISH_FWD, torch.aten.silu)
18421864
FUSILLI_DECLARE_UNARY_POINTWISE_EMITTER(IDENTITY, kIdentitySchema,
18431865
torch.aten.clone)
18441866
FUSILLI_DECLARE_UNARY_TORCH_EMITTER(ERF, torch.aten.erf)

‎samples/pointwise/pointwise_unary_ops.cpp‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,9 @@ TEST_CASE("Pointwise unary ops", "[pointwise][graph]") {
7575
case PointwiseAttr::Mode::RSQRT:
7676
case PointwiseAttr::Mode::SIGMOID_FWD:
7777
case PointwiseAttr::Mode::SIN:
78+
case PointwiseAttr::Mode::SOFTPLUS_FWD:
7879
case PointwiseAttr::Mode::SQRT:
80+
case PointwiseAttr::Mode::SWISH_FWD:
7981
case PointwiseAttr::Mode::TAN:
8082
case PointwiseAttr::Mode::TANH_FWD:
8183
return true;
@@ -102,7 +104,9 @@ TEST_CASE("Pointwise unary ops", "[pointwise][graph]") {
102104
PointwiseAttr::Mode::RSQRT,
103105
PointwiseAttr::Mode::SIGMOID_FWD,
104106
PointwiseAttr::Mode::SIN,
107+
PointwiseAttr::Mode::SOFTPLUS_FWD,
105108
PointwiseAttr::Mode::SQRT,
109+
PointwiseAttr::Mode::SWISH_FWD,
106110
PointwiseAttr::Mode::TAN,
107111
PointwiseAttr::Mode::TANH_FWD);
108112
// clang-format on
@@ -248,11 +252,25 @@ TEST_CASE("Pointwise unary ops", "[pointwise][graph]") {
248252
y = std::sin(xD);
249253
break;
250254
}
255+
case PointwiseAttr::Mode::SOFTPLUS_FWD: {
256+
double xD = static_cast<double>(x);
257+
// SOFTPLUS(x) = log(1 + exp(x)); for x > threshold, result ~= x.
258+
// Matches torch.aten.softplus with beta=1.0 and threshold=20.0.
259+
constexpr double threshold = 20.0;
260+
y = xD > threshold ? xD : std::log1p(std::exp(xD));
261+
break;
262+
}
251263
case PointwiseAttr::Mode::SQRT: {
252264
double xD = static_cast<double>(x);
253265
y = std::sqrt(xD);
254266
break;
255267
}
268+
case PointwiseAttr::Mode::SWISH_FWD: {
269+
double xD = static_cast<double>(x);
270+
// SWISH(x) = SiLU(x) = x * sigmoid(x).
271+
y = xD / (1.0 + std::exp(-xD));
272+
break;
273+
}
256274
case PointwiseAttr::Mode::TAN: {
257275
double xD = static_cast<double>(x);
258276
y = std::tan(xD);

‎tests/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,9 @@ add_fusilli_lit_tests(
176176
lit/test_pointwise_asm_emitter_rsqrt.cpp
177177
lit/test_pointwise_asm_emitter_sigmoid.cpp
178178
lit/test_pointwise_asm_emitter_sin.cpp
179+
lit/test_pointwise_asm_emitter_softplus_fwd.cpp
179180
lit/test_pointwise_asm_emitter_sqrt.cpp
181+
lit/test_pointwise_asm_emitter_swish_fwd.cpp
180182
lit/test_pointwise_asm_emitter_tan.cpp
181183
lit/test_pointwise_asm_emitter_tanh.cpp
182184
lit/test_pointwise_asm_emitter_sub.cpp

‎tests/lit/test_pointwise_asm_emitter_softplus_fwd.cpp‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
// Copyright 2026 Advanced Micro Devices, Inc.
2+
//
3+
// Licensed under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
7+
// RUN: %{TEST_EXE} | iree-opt --verify-roundtrip
8+
// RUN: %{TEST_EXE} | FileCheck %s --check-prefix=TORCH-CHECK
9+
// RUN: %{TEST_EXE} stats | FileCheck %s --check-prefix=%{BACKEND}-STATS-CHECK
10+
11+
// clang-format off
12+
//
13+
// TORCH-CHECK: module @module {
14+
// TORCH-CHECK: func.func @main(%result_: !torch.tensor<[16,256,64,32],f32>, %arg0: !torch.vtensor<[16,256,64,32],f32>) attributes {torch.assume_strict_symbolic_shapes} {
15+
// TORCH-CHECK: %permute_IN_0_val_0_pointwise_softplus_fwd = torch.constant.int 0
16+
// TORCH-CHECK: %permute_IN_0_val_1_pointwise_softplus_fwd = torch.constant.int 1
17+
// TORCH-CHECK: %permute_IN_0_val_2_pointwise_softplus_fwd = torch.constant.int 2
18+
// TORCH-CHECK: %permute_IN_0_val_3_pointwise_softplus_fwd = torch.constant.int 3
19+
// TORCH-CHECK: %permute_IN_0_pointwise_softplus_fwd = torch.prim.ListConstruct %permute_IN_0_val_0_pointwise_softplus_fwd, %permute_IN_0_val_1_pointwise_softplus_fwd, %permute_IN_0_val_2_pointwise_softplus_fwd, %permute_IN_0_val_3_pointwise_softplus_fwd : (!torch.int, !torch.int, !torch.int, !torch.int) -> !torch.list<int>
20+
// TORCH-CHECK: %arg0_pointwise_softplus_fwd_perm = torch.aten.permute %arg0, %permute_IN_0_pointwise_softplus_fwd : !torch.vtensor<[16,256,64,32],f32>, !torch.list<int> -> !torch.vtensor<[16,256,64,32],f32>
21+
// TORCH-CHECK: %softplus_beta_pointwise_softplus_fwd = torch.constant.float 1.000000e+00
22+
// TORCH-CHECK: %softplus_threshold_pointwise_softplus_fwd = torch.constant.float 2.000000e+01
23+
// TORCH-CHECK: %result_pointwise_softplus_fwd_perm = torch.aten.softplus %arg0_pointwise_softplus_fwd_perm, %softplus_beta_pointwise_softplus_fwd, %softplus_threshold_pointwise_softplus_fwd : !torch.vtensor<[16,256,64,32],f32>, !torch.float, !torch.float -> !torch.vtensor<[16,256,64,32],f32>
24+
// TORCH-CHECK: %permute_OUT_0_val_0_pointwise_softplus_fwd = torch.constant.int 0
25+
// TORCH-CHECK: %permute_OUT_0_val_1_pointwise_softplus_fwd = torch.constant.int 1
26+
// TORCH-CHECK: %permute_OUT_0_val_2_pointwise_softplus_fwd = torch.constant.int 2
27+
// TORCH-CHECK: %permute_OUT_0_val_3_pointwise_softplus_fwd = torch.constant.int 3
28+
// TORCH-CHECK: %permute_OUT_0_pointwise_softplus_fwd = torch.prim.ListConstruct %permute_OUT_0_val_0_pointwise_softplus_fwd, %permute_OUT_0_val_1_pointwise_softplus_fwd, %permute_OUT_0_val_2_pointwise_softplus_fwd, %permute_OUT_0_val_3_pointwise_softplus_fwd : (!torch.int, !torch.int, !torch.int, !torch.int) -> !torch.list<int>
29+
// TORCH-CHECK: %result = torch.aten.permute %result_pointwise_softplus_fwd_perm, %permute_OUT_0_pointwise_softplus_fwd : !torch.vtensor<[16,256,64,32],f32>, !torch.list<int> -> !torch.vtensor<[16,256,64,32],f32>
30+
// TORCH-CHECK: torch.overwrite.tensor.contents %result overwrites %result_ : !torch.vtensor<[16,256,64,32],f32>, !torch.tensor<[16,256,64,32],f32>
31+
// TORCH-CHECK: return
32+
// TORCH-CHECK: }
33+
// TORCH-CHECK: }
34+
//
35+
// AMDGPU-STATS-CHECK: "transient-memory-size": 0
36+
// AMDGPU-STATS-CHECK: "dispatch-count": 1
37+
// CPU-STATS-CHECK: "transient-memory-size": 0
38+
// CPU-STATS-CHECK: "dispatch-count": 1
39+
//
40+
// clang-format on
41+
42+
#include <fusilli.h>
43+
44+
#include "pointwise_utils.h"
45+
46+
#include <iostream>
47+
#include <string>
48+
49+
using namespace fusilli;
50+
51+
int main(int argc, char **argv) {
52+
std::string mode = (argc > 1) ? argv[1] : "default";
53+
54+
auto status = testUnaryPointwiseAsmEmitter(
55+
"pointwise_asm_emitter_softplus_fwd", "pointwise_softplus_fwd", mode,
56+
PointwiseAttr::Mode::SOFTPLUS_FWD, {16, 256, 64, 32});
57+
if (isError(status)) {
58+
std::cerr << "Test failed: " << status << std::endl;
59+
return 1;
60+
}
61+
return 0;
62+
}

‎tests/lit/test_pointwise_asm_emitter_swish_fwd.cpp‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
// Copyright 2026 Advanced Micro Devices, Inc.
2+
//
3+
// Licensed under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
7+
// RUN: %{TEST_EXE} | iree-opt --verify-roundtrip
8+
// RUN: %{TEST_EXE} | FileCheck %s --check-prefix=TORCH-CHECK
9+
// RUN: %{TEST_EXE} stats | FileCheck %s --check-prefix=%{BACKEND}-STATS-CHECK
10+
11+
// clang-format off
12+
//
13+
// TORCH-CHECK: module @module {
14+
// TORCH-CHECK: func.func @main(%result_: !torch.tensor<[16,256,64,32],f32>, %arg0: !torch.vtensor<[16,256,64,32],f32>) attributes {torch.assume_strict_symbolic_shapes} {
15+
// TORCH-CHECK: %permute_IN_0_val_0_pointwise_swish_fwd = torch.constant.int 0
16+
// TORCH-CHECK: %permute_IN_0_val_1_pointwise_swish_fwd = torch.constant.int 1
17+
// TORCH-CHECK: %permute_IN_0_val_2_pointwise_swish_fwd = torch.constant.int 2
18+
// TORCH-CHECK: %permute_IN_0_val_3_pointwise_swish_fwd = torch.constant.int 3
19+
// TORCH-CHECK: %permute_IN_0_pointwise_swish_fwd = torch.prim.ListConstruct %permute_IN_0_val_0_pointwise_swish_fwd, %permute_IN_0_val_1_pointwise_swish_fwd, %permute_IN_0_val_2_pointwise_swish_fwd, %permute_IN_0_val_3_pointwise_swish_fwd : (!torch.int, !torch.int, !torch.int, !torch.int) -> !torch.list<int>
20+
// TORCH-CHECK: %arg0_pointwise_swish_fwd_perm = torch.aten.permute %arg0, %permute_IN_0_pointwise_swish_fwd : !torch.vtensor<[16,256,64,32],f32>, !torch.list<int> -> !torch.vtensor<[16,256,64,32],f32>
21+
// TORCH-CHECK: %result_pointwise_swish_fwd_perm = torch.aten.silu %arg0_pointwise_swish_fwd_perm : !torch.vtensor<[16,256,64,32],f32> -> !torch.vtensor<[16,256,64,32],f32>
22+
// TORCH-CHECK: %permute_OUT_0_val_0_pointwise_swish_fwd = torch.constant.int 0
23+
// TORCH-CHECK: %permute_OUT_0_val_1_pointwise_swish_fwd = torch.constant.int 1
24+
// TORCH-CHECK: %permute_OUT_0_val_2_pointwise_swish_fwd = torch.constant.int 2
25+
// TORCH-CHECK: %permute_OUT_0_val_3_pointwise_swish_fwd = torch.constant.int 3
26+
// TORCH-CHECK: %permute_OUT_0_pointwise_swish_fwd = torch.prim.ListConstruct %permute_OUT_0_val_0_pointwise_swish_fwd, %permute_OUT_0_val_1_pointwise_swish_fwd, %permute_OUT_0_val_2_pointwise_swish_fwd, %permute_OUT_0_val_3_pointwise_swish_fwd : (!torch.int, !torch.int, !torch.int, !torch.int) -> !torch.list<int>
27+
// TORCH-CHECK: %result = torch.aten.permute %result_pointwise_swish_fwd_perm, %permute_OUT_0_pointwise_swish_fwd : !torch.vtensor<[16,256,64,32],f32>, !torch.list<int> -> !torch.vtensor<[16,256,64,32],f32>
28+
// TORCH-CHECK: torch.overwrite.tensor.contents %result overwrites %result_ : !torch.vtensor<[16,256,64,32],f32>, !torch.tensor<[16,256,64,32],f32>
29+
// TORCH-CHECK: return
30+
// TORCH-CHECK: }
31+
// TORCH-CHECK: }
32+
//
33+
// AMDGPU-STATS-CHECK: "transient-memory-size": 0
34+
// AMDGPU-STATS-CHECK: "dispatch-count": 1
35+
// CPU-STATS-CHECK: "transient-memory-size": 0
36+
// CPU-STATS-CHECK: "dispatch-count": 1
37+
//
38+
// clang-format on
39+
40+
#include <fusilli.h>
41+
42+
#include "pointwise_utils.h"
43+
44+
#include <iostream>
45+
#include <string>
46+
47+
using namespace fusilli;
48+
49+
int main(int argc, char **argv) {
50+
std::string mode = (argc > 1) ? argv[1] : "default";
51+
52+
auto status = testUnaryPointwiseAsmEmitter(
53+
"pointwise_asm_emitter_swish_fwd", "pointwise_swish_fwd", mode,
54+
PointwiseAttr::Mode::SWISH_FWD, {16, 256, 64, 32});
55+
if (isError(status)) {
56+
std::cerr << "Test failed: " << status << std::endl;
57+
return 1;
58+
}
59+
return 0;
60+
}

0 commit comments

Comments
 (0)