fx2trt converters - change of prototype and addition of activation operation #1745

apbose · 2023-03-17T19:06:29Z

Description

This PR addresses the issues #1744 and #1713
The PR has the following changes-

Reorganization of the acc_ops_converter.py and aten_ops_converter.py for the activation operations
Moving the implementation of the operators from acc_ops_converter.py to operator.py and activation.py
Implementation of the binary operators, activation operations- relu, leaky_relu, tanh, hard_tanh, sigmoid, elu, selu'
Adding tests for the above

Fixes: #1744
Fixes: #1713

… converters to aten and nn

github-actions

Code conforms to C++ style guidelines

github-actions

Code conforms to C++ style guidelines

github-actions

There are some changes that do not conform to Python style guidelines:

--- py/torch_tensorrt/fx/converters/activation.py	2023-03-17 19:06:46.073116 +0000
+++ py/torch_tensorrt/fx/converters/activation.py	2023-03-17 19:07:01.727064 +0000
@@ -23,19 +23,20 @@
    TRTPlugin,
    TRTPluginFieldCollection,
    TRTTensor,
)

+
def add_activation_layer(
    network: TRTNetwork,
    input_val: TRTTensor,
    operation_type: trt.ActivationType,
    target: Target,
    name: str,
    alpha: Optional[Any] = None,
    beta: Optional[Any] = None,
-    dyn_range_fn: Optional[Callable[[float, float], Any]] = None
+    dyn_range_fn: Optional[Callable[[float, float], Any]] = None,
) -> TRTTensor:
    """
    Add a TensorRT Activation layer to `network`.

    Args:
@@ -49,11 +50,11 @@
        alpha (Optional[Any]): If not None, we will use it to set the alpha
            attribute of the created TensorRT activation layer.
        beta (Optional[Any]): If not None, we will use it to set the beta
            attribute of the created TensorRT activation layer.
        dyn_range_fn: Optional[Callable[Tuple[float, float]]]: A function which takes the dynamic range of a TensorRT Tensor and returns the output dynamic range
-        
+

    Returns:
        The output of TensorRT Activation layer.
    """
    if not isinstance(input_val, TRTTensor):
@@ -71,50 +72,59 @@
    if input_val.dynamic_range is not None:
        dyn_range = dyn_range_fn(input_val.dynamic_range)
        mark_as_int8_layer(layer, dyn_range)
    return layer.get_output(0)

+
def add_relu(network, target, kwargs, name):
    input_val = kwargs["input"]
    operation_type = trt.ActivationType.RELU
    return add_activation_layer(network, input_val, operation_type, target, name)
+

def add_leaky_relu(network, target, kwargs, name):
    input_val = kwargs["input"]
    negative_slope = kwargs["negative_slope"]
    operation_type = trt.ActivationType.LEAKY_RELU
    return add_activation_layer(
        network, input_val, operation_type, target, name, negative_slope
    )
+

def add_elu(network, target, kwargs, name):
    input_val = kwargs["input"]
    alpha = kwargs["alpha"]
    operation_type = trt.ActivationType.ELU
    return add_activation_layer(network, input_val, operation_type, target, name, alpha)

+
def add_selu(network, target, kwargs, name):
    input_val = kwargs["input"]
    operation_type = trt.ActivationType.SELU
    return add_activation_layer(network, input_val, operation_type, target, name)

+
def add_softsign(network, target, kwargs, name):
    input_val = kwargs["input"]
    operation_type = trt.ActivationType.SOFTSIGN
    return add_activation_layer(network, input_val, operation_type, target, name)

+
def add_tanh(network, target, kwargs, name):
    input_val = kwargs["input"]
    operation_type = trt.ActivationType.TANH
    return add_activation_layer(network, input_val, operation_type, target, name)
+

def add_gelu(network, target, kwargs, name):
    input_val = kwargs["input"]
    if "approximate" in kwargs.keys():
        approximate = kwargs["approximate"]
        if approximate != "none":
-            raise RuntimeError("GeLU converter currently doesn't support fast gelu compute")
+            raise RuntimeError(
+                "GeLU converter currently doesn't support fast gelu compute"
+            )
    if not isinstance(input_val, TRTTensor):
        raise RuntimeError(
            f"GELU received input {input_val} that is not part "
            "of the TensorRT region!"
        )
@@ -134,10 +144,11 @@
    plugin = get_trt_plugin(plugin_name, field_collection, plugin_version)

    layer = network.add_plugin_v2([input_val], plugin)
    set_layer_name(layer, target, name)
    return layer.get_output(0)
+

def add_hard_sigmoid(network, target, kwargs, name):
    input_val = kwargs["input"]

    if not isinstance(input_val, TRTTensor):
@@ -154,10 +165,11 @@
        name,
        alpha=1 / 6,
        beta=0.5,
    )

+
def add_sigmoid(network, target, kwargs, name):
    input_val = kwargs["input"]

    if not isinstance(input_val, TRTTensor):
        raise RuntimeError(
@@ -167,14 +179,16 @@

    return add_activation_layer(
        network, input_val, trt.ActivationType.SIGMOID, target, name
    )

+
def add_hard_tanh(network, target, kwargs, name):
    input_val = kwargs["input"]
    operation_type = trt.ActivationType.TANH
    return add_activation_layer(network, input_val, operation_type, target, name)
+

def add_sigmoid(network, target, kwargs, name):
    input_val = kwargs["input"]

    if not isinstance(input_val, TRTTensor):
@@ -190,6 +204,5 @@
        target,
        name,
        alpha=1 / 6,
        beta=0.5,
    )
-
--- py/torch_tensorrt/fx/converters/aten_ops_converters.py	2023-03-17 19:06:46.077116 +0000
+++ py/torch_tensorrt/fx/converters/aten_ops_converters.py	2023-03-17 19:07:02.103174 +0000
@@ -145,17 +145,13 @@
    }
    rounding_mode = kwargs.get("rounding_mode")
    if rounding_mode is None:
        return add_div(network, target, None, kwargs_new, name)
    elif rounding_mode == "floor":
-        return add_floor_div(
-            network, target, None, kwargs_new, name
-        )
+        return add_floor_div(network, target, None, kwargs_new, name)
    elif rounding_mode == "trunc":
-        return add_trunc_div(
-            network, target, None, kwargs_new, name
-        )
+        return add_trunc_div(network, target, None, kwargs_new, name)
    else:
        raise RuntimeError(
            f"Target {target} does not support rounding mode {rounding_mode}"
        )

@@ -299,11 +295,12 @@
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    kwargs_new = {
        "input": args[0],
    }
    return add_relu(network, target, kwargs_new, name)
-    
+
+
@tensorrt_converter(torch.ops.aten.sub.Tensor)
def aten_ops_sub(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
@@ -380,11 +377,11 @@
    kwargs_new = {
        "input": args[0],
        "sizes": args[1],
    }
    return add_expand(network, target, kwargs_new, name)
-     
+

@tensorrt_converter(operator.floordiv)
def aten_ops_operator_floordiv(
    network: TRTNetwork,
    target: Target,
@@ -492,17 +489,17 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    kwargs_new = {
-        "input" : args[0],
-        "dim" : args[1],
-        "start" : args[2],
-        "stop" : args[3],
-        "step" : args[4],
-    }
-    return add_slice(network, target. kwargs_new, name)
+        "input": args[0],
+        "dim": args[1],
+        "start": args[2],
+        "stop": args[3],
+        "step": args[4],
+    }
+    return add_slice(network, target.kwargs_new, name)


@tensorrt_converter(torch.ops.aten.select)
def aten_ops_select(
    network: TRTNetwork,
@@ -510,15 +507,15 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    kwargs_new = {
-        "input" : args[0],
-        "dim" : args[1],
-        "index" : args[2],
-    }
-    return add_select(network, target. kwargs_new, name)
+        "input": args[0],
+        "dim": args[1],
+        "index": args[2],
+    }
+    return add_select(network, target.kwargs_new, name)


@tensorrt_converter(torch.ops.aten.leaky_relu.default)
def aten_ops_leaky_relu(
    network: TRTNetwork,
@@ -572,10 +569,11 @@
    kwargs_new = {
        "input": args[0],
    }
    return add_gelu(network, target, kwargs_new, name)

+
@tensorrt_converter(torch.ops.aten.tanh.default)
def aten_ops_tanh(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
@@ -612,11 +610,5 @@
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    kwargs_new = {
        "input": args[0],
    }
    return add_sigmoid(network, target, kwargs_new, name)
-
-
-
-
-
-
--- py/torch_tensorrt/fx/converters/converter_utils.py	2023-03-17 19:06:46.077116 +0000
+++ py/torch_tensorrt/fx/converters/converter_utils.py	2023-03-17 19:07:02.338487 +0000
@@ -407,22 +407,20 @@
    select_layer = network.add_select(condition_val, input_shape, scale_res)
    set_layer_name(select_layer, target, f"{name}_select")
    return select_layer.get_output(0)


-
-
-
def squeeze_left(const: torch.Tensor):
    """
    Squeeze the size-1 dimensions on the left side of the shape tuple.
    PyTorch's `squeeze()` doesn't support passing multiple `dim`s at once, so
    we do it iteratively.
    """
    while len(const.shape) > 0 and const.shape[0] == 1:
        const = const.squeeze(dim=0)
    return const
+

def add_reduce_layer(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
@@ -638,10 +636,11 @@
        target,
        f"{name}_output",
    )

    return output
+

def dtype_uniform(
    network: TRTNetwork, target: Target, name: str, input: TRTTensor, other: TRTTensor
):
    table = {trt.bool: 0, trt.int32: 1, trt.float16: 2, trt.float32: 3}
--- py/torch_tensorrt/fx/converters/nn_ops_converters.py	2023-03-17 19:06:46.077116 +0000
+++ py/torch_tensorrt/fx/converters/nn_ops_converters.py	2023-03-17 19:07:02.413182 +0000
@@ -7,25 +7,27 @@
from ..converter_registry import tensorrt_converter

from .converter_utils import mark_as_int8_layer
import activation

+
@tensorrt_converter(torch.nn.functional.relu)
@tensorrt_converter(torch.nn.modules.activation.ReLU)
def relu(network, submod, args, kwargs, layer_name):
    # args/kwargs should have already been normalized to kwargs
    assert len(args) == 0
-    return activation.add_relu(network,"tensorrt", kwargs, layer_name)
+    return activation.add_relu(network, "tensorrt", kwargs, layer_name)
+

@tensorrt_converter(torch.nn.functional.leaky_relu)
@tensorrt_converter(torch.nn.modules.activation.leaky_relu)
def leaky_relu(network, submod, args, kwargs, layer_name):
    # args/kwargs should have already been normalized to kwargs
    assert len(args) == 0
-    return activation.add_leaky_relu(network,"tensorrt", kwargs, layer_name)
+    return activation.add_leaky_relu(network, "tensorrt", kwargs, layer_name)
+

@tensorrt_converter(torch.nn.modules.activation.Sigmoid)
def sigmoid(network, submod, args, kwargs, layer_name):
    # args/kwargs should have already been normalized to kwargs
    assert len(args) == 0
-    return activation.add_sigmoid(network,"tensorrt", kwargs, layer_name)
-
+    return activation.add_sigmoid(network, "tensorrt", kwargs, layer_name)
--- py/torch_tensorrt/fx/converters/operator.py	2023-03-17 19:06:46.077116 +0000
+++ py/torch_tensorrt/fx/converters/operator.py	2023-03-17 19:07:02.951127 +0000
@@ -33,10 +33,11 @@
    TRTPluginFieldCollection,
    TRTTensor,
)

_LOGGER: logging.Logger = logging.getLogger(__name__)
+

def get_python_op_from_trt_elementwise_op(
    trt_op: TRTElementWiseOp,
) -> Callable[[Any, Any], Any]:
    if trt_op == trt.ElementWiseOperation.SUM:
@@ -49,10 +50,11 @@
        return operator.truediv
    elif trt_op == trt.ElementWiseOperation.FLOOR_DIV:
        return operator.floordiv
    else:
        raise RuntimeError(f"{trt_op} is not supported yet!")
+

def add_binary_elementwise_layer(
    network: TRTNetwork,
    lhs_val: Union[int, float, TRTTensor, torch.Tensor],
    rhs_val: Union[int, float, TRTTensor, torch.Tensor],
@@ -157,10 +159,11 @@
    set_layer_name(layer, target, name)
    output = layer.get_output(0)
    output.name = output.name + "_" + target.__name__
    return output

+
def trunc_div(
    input: TRTTensor, other: TRTTensor, network: TRTNetwork, target: Target, name: str
) -> TRTTensor:
    """
    Perform trunc divide on Tensor, result of divide will be round toward zero.
@@ -212,10 +215,11 @@
        target,
        f"{name}_output",
    )

    return output
+

def add_tile(network, target, kwargs, name):
    input_t = kwargs["input"]
    input_val = get_trt_tensor(network, input_t, f"{name}_input")

@@ -312,10 +316,11 @@
        layer.set_input(1, starts_tensor)
        layer.set_input(2, slice_shapes_tensor)

    return layer.get_output(0)

+
def add_linear(network, target, kwargs, name):
    input_val = kwargs["input"]

    if not isinstance(input_val, TRTTensor):
        raise RuntimeError(
@@ -366,10 +371,11 @@
            target,
            f"{name}_add",
        )
    return res

+
def add_unary_layer(
    network: TRTNetwork,
    input_val: TRTTensor,
    operation_type: trt.UnaryOperation,
    target: Target,
@@ -397,10 +403,11 @@
    set_layer_name(layer, target, name)
    output = layer.get_output(0)
    output.name = output.name + "_" + target.__name__
    return layer.get_output(0)

+
def layer_norm(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
@@ -507,19 +514,21 @@
        trt.ElementWiseOperation.SUM,
        target,
        name,
    )

+
def add_add(network, target, kwargs, name):
    return add_binary_elementwise_layer(
        network,
        kwargs["input"],
        kwargs["other"],
        trt.ElementWiseOperation.SUM,
        target,
        name,
    )
+

def add_matmul(network, target, kwargs, name):
    input_val = get_trt_tensor(network, kwargs["input"], f"{name}_input")
    other_val = get_trt_tensor(network, kwargs["other"], f"{name}_other")

@@ -546,10 +555,11 @@
    layer = network.add_matrix_multiply(
        input_val, input_matrix_op, other_val, other_matrix_op
    )
    set_layer_name(layer, target, name)
    return layer.get_output(0)
+

def add_layer_norm(network, target, kwargs, name):
    input_val = kwargs["input"]

    if not isinstance(input_val, trt.tensorrt.ITensor):
@@ -589,10 +599,11 @@
        )
        return layer_norm(network, target, args, kwargs, name)
    layer = network.add_plugin_v2([input_val], plugin)
    layer.name = name
    return layer.get_output(0)
+

def add_cumsum(network, target, kwargs, name):
    input_val = kwargs["input"]
    dim = cast(int, kwargs["dim"])
    input_shape = input_val.shape  # type: ignore[union-attr]
@@ -661,79 +672,88 @@
    loop_output = loop.add_loop_output(current_sum, trt.LoopOutput.CONCATENATE, dim)
    set_layer_name(loop_output, target, f"{name}_loop_output")
    loop_output.set_input(1, trip_limit)
    return loop_output.get_output(0)

+
def add_maximum(network, target, kwargs, name):
    return add_binary_elementwise_layer(
        network,
        kwargs["input"],
        kwargs["other"],
        trt.ElementWiseOperation.MAX,
        target,
        name,
    )

+
def add_mul(network, target, kwargs, name):
    return add_binary_elementwise_layer(
        network,
        kwargs["input"],
        kwargs["other"],
        trt.ElementWiseOperation.PROD,
        target,
        name,
    )

+
def add_pow(network, target, kwargs, name):
    return add_binary_elementwise_layer(
        network,
        kwargs["input"],
        kwargs["other"],
        trt.ElementWiseOperation.POW,
        target,
        name,
    )

+
def add_floor_div(network, target, kwargs, name):
    return add_binary_elementwise_layer(
        network,
        kwargs["input"],
        kwargs["other"],
        trt.ElementWiseOperation.FLOOR_DIV,
        target,
        name,
    )

+
def add_div(network, target, kwargs, name):
    return add_binary_elementwise_layer(
        network,
        kwargs["input"],
        kwargs["other"],
        trt.ElementWiseOperation.DIV,
        target,
        name,
    )

+
def add_sub(network, target, kwargs, name):
    return add_binary_elementwise_layer(
        network,
        kwargs["input"],
        kwargs["other"],
        trt.ElementWiseOperation.SUB,
        target,
        name,
    )
+
+
def add_minimum(network, target, kwargs, name):
    return add_binary_elementwise_layer(
        network,
        kwargs["input"],
        kwargs["other"],
        trt.ElementWiseOperation.MIN,
        target,
        name,
    )

+
def add_logical_and(network, target, kwargs, name):
    if network.has_implicit_batch_dimension:
        raise RuntimeError(
            "The `ne` function should be called with explicit batch dimension."
        )
@@ -749,10 +769,11 @@
        network, input_t, other_t, trt.ElementWiseOperation.EQUAL, target, name
    )

    return add_unary_layer(network, eq_t, trt.UnaryOperation.NOT, target, name)

+
def add_ne(network, target, kwargs, name):
    if network.has_implicit_batch_dimension:
        raise RuntimeError(
            "The `ne` function should be called with explicit batch dimension."
        )
@@ -768,10 +789,11 @@
        network, input_t, other_t, trt.ElementWiseOperation.EQUAL, target, name
    )

    return add_unary_layer(network, eq_t, trt.UnaryOperation.NOT, target, name)

+
def add_eq(network, target, kwargs, name):
    if network.has_implicit_batch_dimension:
        raise RuntimeError(
            "The `eq` function should be called with explicit batch dimension."
        )
@@ -785,10 +807,11 @@
    input_t, other_t = dtype_uniform(network, target, name, input_t, other_t)
    return add_binary_elementwise_layer(
        network, input_t, other_t, trt.ElementWiseOperation.EQUAL, target, name
    )

+
def add_gt(network, target, kwargs, name):
    if network.has_implicit_batch_dimension:
        raise RuntimeError(
            "The `gt` function should be called with explicit batch dimension."
        )
@@ -802,10 +825,11 @@
    input_t, other_t = dtype_uniform(network, target, name, input_t, other_t)
    return add_binary_elementwise_layer(
        network, input_t, other_t, trt.ElementWiseOperation.GREATER, target, name
    )

+
def add_lt(network, target, kwargs, name):
    if network.has_implicit_batch_dimension:
        raise RuntimeError(
            "The `le` function should be called with explicit batch dimension."
        )
@@ -818,10 +842,11 @@

    input_t, other_t = dtype_uniform(network, target, name, input_t, other_t)
    return add_binary_elementwise_layer(
        network, input_t, other_t, trt.ElementWiseOperation.LESS, target, name
    )
+

def add_logical_or(network, target, kwargs, name):
    if network.has_implicit_batch_dimension:
        raise RuntimeError(
            "The `logical_or` function should be called with explicit batch dimension."
@@ -848,10 +873,11 @@

    return add_binary_elementwise_layer(
        network, input_t, other_t, trt.ElementWiseOperation.OR, target, name
    )

+
def add_logical_xor(network, target, kwargs, name):
    if network.has_implicit_batch_dimension:
        raise RuntimeError(
            "The `logical_xor` function should be called with explicit batch dimension."
        )
@@ -877,12 +903,13 @@

    return add_binary_elementwise_layer(
        network, input_t, other_t, trt.ElementWiseOperation.XOR, target, name
    )

+
def add_fmod(network, target, kwargs, name):
-     # NOTE: TRT doesnt currently implement fmod so we need multiple operations to perform it
+    # NOTE: TRT doesnt currently implement fmod so we need multiple operations to perform it
    trunc_div_value = trunc_div(
        kwargs["input"], kwargs["other"], network, target, name + "_trunc_div"
    )
    prod_value = add_binary_elementwise_layer(
        network,
@@ -900,12 +927,14 @@
        target,
        name + "_sub",
    )
    return sub_value

+
def add_trunc_div(network, target, kwargs, name):
    return trunc_div(kwargs["input"], kwargs["other"], network, target, name)
+

def add_expand(network, target, kwargs, name):
    input_t = kwargs["input"]
    shape = list(kwargs["sizes"])

@@ -927,10 +956,11 @@
    )  # stride == 1 if dimensions match, 0 otherwise
    layer = network.add_slice(input_val, start=start, shape=shape, stride=stride)
    set_layer_name(layer, target, name)
    return layer.get_output(0)

+
def add_slice(network, target, kwargs, name):
    input_val = kwargs["input"]

    if not isinstance(input_val, TRTTensor):
        raise RuntimeError(
@@ -974,9 +1004,5 @@
    )
    if dynamic_shape:
        layer.set_input(2, output_shape)
    set_layer_name(layer, target, name)
    return layer.get_output(0)
-
-
-        
-    
--- py/torch_tensorrt/fx/converters/acc_ops_converters.py	2023-03-17 19:06:46.073116 +0000
+++ py/torch_tensorrt/fx/converters/acc_ops_converters.py	2023-03-17 19:07:03.317452 +0000
@@ -678,11 +678,12 @@


@tensorrt_converter(acc_ops.layer_norm)
def acc_ops_layer_norm(network, target, args, kwargs, name):
    return add_layer_norm(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.softmax)
def acc_ops_softmax(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
@@ -729,11 +730,12 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_tile(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.sign)
def acc_ops_sign(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
@@ -758,50 +760,55 @@
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_relu(network, target, kwargs, name)

+
@tensorrt_converter(acc_ops.leaky_relu)
def acc_ops_leaky_relu(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_leaky_relu(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.elu)
def acc_ops_elu(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_elu(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.selu)
def acc_ops_selu(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_selu(network, target, kwargs, name)

+
@tensorrt_converter(acc_ops.softsign)
def acc_ops_softsign(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_softsign(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.sin)
def acc_ops_sin(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
@@ -872,10 +879,11 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_tanh(network, target, kwargs, name)
+

@tensorrt_converter(acc_ops.asin)
def acc_ops_asin(
    network: TRTNetwork,
    target: Target,
@@ -1189,21 +1197,23 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_maximum(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.minimum)
def acc_ops_minimum(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_minimum(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.dtype)
def acc_ops_dtype(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
@@ -1269,71 +1279,77 @@
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_logical_and(network, target, kwargs, name)

+
@tensorrt_converter(acc_ops.ne, no_implicit_batch_dim=True)
def acc_ops_ne(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_ne(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.eq, no_implicit_batch_dim=True)
def acc_ops_eq(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_eq(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.gt, no_implicit_batch_dim=True)
def acc_ops_gt(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_gt(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.lt, no_implicit_batch_dim=True)
def acc_ops_lt(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_lt(network, target, kwargs, name)
-    
+

@tensorrt_converter(acc_ops.logical_or, no_implicit_batch_dim=True)
def acc_ops_logical_or(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_logical_or(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.logical_xor, no_implicit_batch_dim=True)
def acc_ops_logical_xor(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_logical_xor(network, target, kwargs, name)
-    
+
+
# T113156424 Have some accuracy problems in hf_T5.
# [TRT] [W] Weights [name=isinf_1_inf_t]: Converted FP32 value in weights (either FP32 infinity or FP32 value outside FP16 range) to corresponding FP16 infinity. If this is not the desired behavior, please modify the weights or retrain with regularization to reduce the magnitude of the weights.
# @tensorrt_converter(acc_ops.isinf)
# def acc_ops_isinf(
#     network: TRTNetwork,
@@ -1422,10 +1438,11 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_fmod(network, target, kwargs, name)
+

# T113156424 embedding implemenatation is very limited and shows no usage in hf models due to the indices are int64.
# if we cast to int32, it will create accuracy issues. We'd better leave it to future implementation.
# @tensorrt_converter(acc_ops.embedding, no_implicit_batch_dim=True)
# def acc_ops_embedding(
@@ -1651,69 +1668,76 @@
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_add(network, target, kwargs, name)

+
@tensorrt_converter(acc_ops.sub)
def acc_ops_sub(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_sub(network, target, kwargs, name)

+
@tensorrt_converter(acc_ops.div)
def acc_ops_div(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_div(network, target, kwargs, name)

+
@tensorrt_converter(acc_ops.floor_div)
def acc_ops_floor_div(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_floor_div(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.trunc_div)
def acc_ops_trunc_div(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_trunc_div(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.mul)
def acc_ops_mul(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_mul(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.pow)
def acc_ops_pow(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_pow(network, target, kwargs, name)
+

@tensorrt_converter(acc_ops.unsqueeze)
def acc_ops_unsqueeze(
    network: TRTNetwork,
    target: Target,
@@ -1980,22 +2004,22 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_slice(network, target, kwargs, name)
-    
+

@tensorrt_converter(acc_ops.expand)
def acc_ops_expand_tensor(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_expand(network, target, kwargs, name)
-    
+

@tensorrt_converter(acc_ops.where)
def acc_ops_where(
    network: TRTNetwork,
    target: Target,
@@ -2213,11 +2237,12 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_linear(network, target, kwargs, name)
-    
+
+
def add_clamp(network, input, val, op, name):
    if not len(input.shape):
        # clamping scalar
        acc_ops_clamp_trt = get_trt_tensor(
            network,
@@ -2466,11 +2491,12 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_matmul(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.hardsigmoid)
def acc_ops_hard_sigmoid(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
@@ -2688,11 +2714,11 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_gelu(network, target, kwargs, name)
-    
+

@tensorrt_converter(acc_ops.chunk)
def acc_ops_chunk(
    network: TRTNetwork,
    target: Target,
@@ -2765,20 +2791,22 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_cumsum(network, target, kwargs, name)
-   
+
+
@tensorrt_converter(acc_ops.hardtanh)
def acc_ops_hardtanh(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_hardtanh(network, target, kwargs, name)
+

@tensorrt_converter(acc_ops.interpolate)
def acc_ops_interpolate(
    network: TRTNetwork,
    target: Target,
--- py/torch_tensorrt/fx/test/converters/aten_op/test_gelu_aten.py	2023-03-17 19:06:46.081116 +0000
+++ py/torch_tensorrt/fx/test/converters/aten_op/test_gelu_aten.py	2023-03-17 19:07:04.256285 +0000
@@ -1,20 +1,19 @@
import torch
import torch.nn as nn
from torch.testing._internal.common_utils import run_tests
from torch_tensorrt.fx.tools.common_fx2trt import DispatchTestCase, InputTensorSpec
+

class TestGeLUConverter(DispatchTestCase):
    def test_gelu(self):
        class TestModule(nn.Module):
            def forward(self, x):
                return nn.functional.gelu(x)

        inputs = [torch.randn(1, 10)]
-        self.run_test(
-            TestModule(), inputs, expected_ops={torch.ops.aten.gelu.default}
-        )
+        self.run_test(TestModule(), inputs, expected_ops={torch.ops.aten.gelu.default})

    def test_gelu_with_dynamic_shape(self):
        class TestModule(nn.Module):
            def forward(self, x):
                return nn.functional.gelu(x)
@@ -47,6 +46,6 @@
            TestModule(), input_specs, expected_ops={torch.ops.aten.gelu.default}
        )


if __name__ == "__main__":
-    run_tests()
\ No newline at end of file
+    run_tests()
--- py/torch_tensorrt/fx/test/converters/aten_op/test_leaky_relu_aten.py	2023-03-17 19:06:46.081116 +0000
+++ py/torch_tensorrt/fx/test/converters/aten_op/test_leaky_relu_aten.py	2023-03-17 19:07:04.270383 +0000
@@ -48,6 +48,6 @@
            TestModule(), input_specs, expected_ops={torch.ops.aten.leaky_relu.default}
        )


if __name__ == "__main__":
-    run_tests()
\ No newline at end of file
+    run_tests()
--- py/torch_tensorrt/fx/test/converters/aten_op/test_tanh_aten.py	2023-03-17 19:06:46.081116 +0000
+++ py/torch_tensorrt/fx/test/converters/aten_op/test_tanh_aten.py	2023-03-17 19:07:04.331639 +0000
@@ -1,20 +1,19 @@
import torch
import torch.nn as nn
from torch.testing._internal.common_utils import run_tests
from torch_tensorrt.fx.tools.common_fx2trt import DispatchTestCase, InputTensorSpec
+

class TestTanhConverter(DispatchTestCase):
    def test_tanh(self):
        class TestModule(nn.Module):
            def forward(self, x):
                return nn.functional.tanh(x)

        inputs = [torch.randn(1, 10)]
-        self.run_test(
-            TestModule(), inputs, expected_ops={torch.ops.aten.tanh.default}
-        )
+        self.run_test(TestModule(), inputs, expected_ops={torch.ops.aten.tanh.default})

    def test_tanh_with_dynamic_shape(self):
        class TestModule(nn.Module):
            def forward(self, x):
                return nn.functional.tanh(x)
@@ -47,6 +46,6 @@
            TestModule(), input_specs, expected_ops={torch.ops.aten.tanh.default}
        )


if __name__ == "__main__":
-    run_tests()
\ No newline at end of file
+    run_tests()

github-actions

There are some changes that do not conform to Python style guidelines:

--- py/torch_tensorrt/fx/converters/activation.py	2023-03-17 19:06:49.232797 +0000
+++ py/torch_tensorrt/fx/converters/activation.py	2023-03-17 19:07:03.740525 +0000
@@ -23,19 +23,20 @@
    TRTPlugin,
    TRTPluginFieldCollection,
    TRTTensor,
)

+
def add_activation_layer(
    network: TRTNetwork,
    input_val: TRTTensor,
    operation_type: trt.ActivationType,
    target: Target,
    name: str,
    alpha: Optional[Any] = None,
    beta: Optional[Any] = None,
-    dyn_range_fn: Optional[Callable[[float, float], Any]] = None
+    dyn_range_fn: Optional[Callable[[float, float], Any]] = None,
) -> TRTTensor:
    """
    Add a TensorRT Activation layer to `network`.

    Args:
@@ -49,11 +50,11 @@
        alpha (Optional[Any]): If not None, we will use it to set the alpha
            attribute of the created TensorRT activation layer.
        beta (Optional[Any]): If not None, we will use it to set the beta
            attribute of the created TensorRT activation layer.
        dyn_range_fn: Optional[Callable[Tuple[float, float]]]: A function which takes the dynamic range of a TensorRT Tensor and returns the output dynamic range
-        
+

    Returns:
        The output of TensorRT Activation layer.
    """
    if not isinstance(input_val, TRTTensor):
@@ -71,50 +72,59 @@
    if input_val.dynamic_range is not None:
        dyn_range = dyn_range_fn(input_val.dynamic_range)
        mark_as_int8_layer(layer, dyn_range)
    return layer.get_output(0)

+
def add_relu(network, target, kwargs, name):
    input_val = kwargs["input"]
    operation_type = trt.ActivationType.RELU
    return add_activation_layer(network, input_val, operation_type, target, name)
+

def add_leaky_relu(network, target, kwargs, name):
    input_val = kwargs["input"]
    negative_slope = kwargs["negative_slope"]
    operation_type = trt.ActivationType.LEAKY_RELU
    return add_activation_layer(
        network, input_val, operation_type, target, name, negative_slope
    )
+

def add_elu(network, target, kwargs, name):
    input_val = kwargs["input"]
    alpha = kwargs["alpha"]
    operation_type = trt.ActivationType.ELU
    return add_activation_layer(network, input_val, operation_type, target, name, alpha)

+
def add_selu(network, target, kwargs, name):
    input_val = kwargs["input"]
    operation_type = trt.ActivationType.SELU
    return add_activation_layer(network, input_val, operation_type, target, name)

+
def add_softsign(network, target, kwargs, name):
    input_val = kwargs["input"]
    operation_type = trt.ActivationType.SOFTSIGN
    return add_activation_layer(network, input_val, operation_type, target, name)

+
def add_tanh(network, target, kwargs, name):
    input_val = kwargs["input"]
    operation_type = trt.ActivationType.TANH
    return add_activation_layer(network, input_val, operation_type, target, name)
+

def add_gelu(network, target, kwargs, name):
    input_val = kwargs["input"]
    if "approximate" in kwargs.keys():
        approximate = kwargs["approximate"]
        if approximate != "none":
-            raise RuntimeError("GeLU converter currently doesn't support fast gelu compute")
+            raise RuntimeError(
+                "GeLU converter currently doesn't support fast gelu compute"
+            )
    if not isinstance(input_val, TRTTensor):
        raise RuntimeError(
            f"GELU received input {input_val} that is not part "
            "of the TensorRT region!"
        )
@@ -134,10 +144,11 @@
    plugin = get_trt_plugin(plugin_name, field_collection, plugin_version)

    layer = network.add_plugin_v2([input_val], plugin)
    set_layer_name(layer, target, name)
    return layer.get_output(0)
+

def add_hard_sigmoid(network, target, kwargs, name):
    input_val = kwargs["input"]

    if not isinstance(input_val, TRTTensor):
@@ -154,10 +165,11 @@
        name,
        alpha=1 / 6,
        beta=0.5,
    )

+
def add_sigmoid(network, target, kwargs, name):
    input_val = kwargs["input"]

    if not isinstance(input_val, TRTTensor):
        raise RuntimeError(
@@ -167,14 +179,16 @@

    return add_activation_layer(
        network, input_val, trt.ActivationType.SIGMOID, target, name
    )

+
def add_hard_tanh(network, target, kwargs, name):
    input_val = kwargs["input"]
    operation_type = trt.ActivationType.TANH
    return add_activation_layer(network, input_val, operation_type, target, name)
+

def add_sigmoid(network, target, kwargs, name):
    input_val = kwargs["input"]

    if not isinstance(input_val, TRTTensor):
@@ -190,6 +204,5 @@
        target,
        name,
        alpha=1 / 6,
        beta=0.5,
    )
-
--- py/torch_tensorrt/fx/converters/aten_ops_converters.py	2023-03-17 19:06:49.232797 +0000
+++ py/torch_tensorrt/fx/converters/aten_ops_converters.py	2023-03-17 19:07:04.122519 +0000
@@ -145,17 +145,13 @@
    }
    rounding_mode = kwargs.get("rounding_mode")
    if rounding_mode is None:
        return add_div(network, target, None, kwargs_new, name)
    elif rounding_mode == "floor":
-        return add_floor_div(
-            network, target, None, kwargs_new, name
-        )
+        return add_floor_div(network, target, None, kwargs_new, name)
    elif rounding_mode == "trunc":
-        return add_trunc_div(
-            network, target, None, kwargs_new, name
-        )
+        return add_trunc_div(network, target, None, kwargs_new, name)
    else:
        raise RuntimeError(
            f"Target {target} does not support rounding mode {rounding_mode}"
        )

@@ -299,11 +295,12 @@
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    kwargs_new = {
        "input": args[0],
    }
    return add_relu(network, target, kwargs_new, name)
-    
+
+
@tensorrt_converter(torch.ops.aten.sub.Tensor)
def aten_ops_sub(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
@@ -380,11 +377,11 @@
    kwargs_new = {
        "input": args[0],
        "sizes": args[1],
    }
    return add_expand(network, target, kwargs_new, name)
-     
+

@tensorrt_converter(operator.floordiv)
def aten_ops_operator_floordiv(
    network: TRTNetwork,
    target: Target,
@@ -492,17 +489,17 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    kwargs_new = {
-        "input" : args[0],
-        "dim" : args[1],
-        "start" : args[2],
-        "stop" : args[3],
-        "step" : args[4],
-    }
-    return add_slice(network, target. kwargs_new, name)
+        "input": args[0],
+        "dim": args[1],
+        "start": args[2],
+        "stop": args[3],
+        "step": args[4],
+    }
+    return add_slice(network, target.kwargs_new, name)


@tensorrt_converter(torch.ops.aten.select)
def aten_ops_select(
    network: TRTNetwork,
@@ -510,15 +507,15 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    kwargs_new = {
-        "input" : args[0],
-        "dim" : args[1],
-        "index" : args[2],
-    }
-    return add_select(network, target. kwargs_new, name)
+        "input": args[0],
+        "dim": args[1],
+        "index": args[2],
+    }
+    return add_select(network, target.kwargs_new, name)


@tensorrt_converter(torch.ops.aten.leaky_relu.default)
def aten_ops_leaky_relu(
    network: TRTNetwork,
@@ -572,10 +569,11 @@
    kwargs_new = {
        "input": args[0],
    }
    return add_gelu(network, target, kwargs_new, name)

+
@tensorrt_converter(torch.ops.aten.tanh.default)
def aten_ops_tanh(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
@@ -612,11 +610,5 @@
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    kwargs_new = {
        "input": args[0],
    }
    return add_sigmoid(network, target, kwargs_new, name)
-
-
-
-
-
-
--- py/torch_tensorrt/fx/converters/converter_utils.py	2023-03-17 19:06:49.232797 +0000
+++ py/torch_tensorrt/fx/converters/converter_utils.py	2023-03-17 19:07:04.359481 +0000
@@ -407,22 +407,20 @@
    select_layer = network.add_select(condition_val, input_shape, scale_res)
    set_layer_name(select_layer, target, f"{name}_select")
    return select_layer.get_output(0)


-
-
-
def squeeze_left(const: torch.Tensor):
    """
    Squeeze the size-1 dimensions on the left side of the shape tuple.
    PyTorch's `squeeze()` doesn't support passing multiple `dim`s at once, so
    we do it iteratively.
    """
    while len(const.shape) > 0 and const.shape[0] == 1:
        const = const.squeeze(dim=0)
    return const
+

def add_reduce_layer(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
@@ -638,10 +636,11 @@
        target,
        f"{name}_output",
    )

    return output
+

def dtype_uniform(
    network: TRTNetwork, target: Target, name: str, input: TRTTensor, other: TRTTensor
):
    table = {trt.bool: 0, trt.int32: 1, trt.float16: 2, trt.float32: 3}
--- py/torch_tensorrt/fx/converters/nn_ops_converters.py	2023-03-17 19:06:49.232797 +0000
+++ py/torch_tensorrt/fx/converters/nn_ops_converters.py	2023-03-17 19:07:04.449247 +0000
@@ -7,25 +7,27 @@
from ..converter_registry import tensorrt_converter

from .converter_utils import mark_as_int8_layer
import activation

+
@tensorrt_converter(torch.nn.functional.relu)
@tensorrt_converter(torch.nn.modules.activation.ReLU)
def relu(network, submod, args, kwargs, layer_name):
    # args/kwargs should have already been normalized to kwargs
    assert len(args) == 0
-    return activation.add_relu(network,"tensorrt", kwargs, layer_name)
+    return activation.add_relu(network, "tensorrt", kwargs, layer_name)
+

@tensorrt_converter(torch.nn.functional.leaky_relu)
@tensorrt_converter(torch.nn.modules.activation.leaky_relu)
def leaky_relu(network, submod, args, kwargs, layer_name):
    # args/kwargs should have already been normalized to kwargs
    assert len(args) == 0
-    return activation.add_leaky_relu(network,"tensorrt", kwargs, layer_name)
+    return activation.add_leaky_relu(network, "tensorrt", kwargs, layer_name)
+

@tensorrt_converter(torch.nn.modules.activation.Sigmoid)
def sigmoid(network, submod, args, kwargs, layer_name):
    # args/kwargs should have already been normalized to kwargs
    assert len(args) == 0
-    return activation.add_sigmoid(network,"tensorrt", kwargs, layer_name)
-
+    return activation.add_sigmoid(network, "tensorrt", kwargs, layer_name)
--- py/torch_tensorrt/fx/converters/operator.py	2023-03-17 19:06:49.232797 +0000
+++ py/torch_tensorrt/fx/converters/operator.py	2023-03-17 19:07:04.986548 +0000
@@ -33,10 +33,11 @@
    TRTPluginFieldCollection,
    TRTTensor,
)

_LOGGER: logging.Logger = logging.getLogger(__name__)
+

def get_python_op_from_trt_elementwise_op(
    trt_op: TRTElementWiseOp,
) -> Callable[[Any, Any], Any]:
    if trt_op == trt.ElementWiseOperation.SUM:
@@ -49,10 +50,11 @@
        return operator.truediv
    elif trt_op == trt.ElementWiseOperation.FLOOR_DIV:
        return operator.floordiv
    else:
        raise RuntimeError(f"{trt_op} is not supported yet!")
+

def add_binary_elementwise_layer(
    network: TRTNetwork,
    lhs_val: Union[int, float, TRTTensor, torch.Tensor],
    rhs_val: Union[int, float, TRTTensor, torch.Tensor],
@@ -157,10 +159,11 @@
    set_layer_name(layer, target, name)
    output = layer.get_output(0)
    output.name = output.name + "_" + target.__name__
    return output

+
def trunc_div(
    input: TRTTensor, other: TRTTensor, network: TRTNetwork, target: Target, name: str
) -> TRTTensor:
    """
    Perform trunc divide on Tensor, result of divide will be round toward zero.
@@ -212,10 +215,11 @@
        target,
        f"{name}_output",
    )

    return output
+

def add_tile(network, target, kwargs, name):
    input_t = kwargs["input"]
    input_val = get_trt_tensor(network, input_t, f"{name}_input")

@@ -312,10 +316,11 @@
        layer.set_input(1, starts_tensor)
        layer.set_input(2, slice_shapes_tensor)

    return layer.get_output(0)

+
def add_linear(network, target, kwargs, name):
    input_val = kwargs["input"]

    if not isinstance(input_val, TRTTensor):
        raise RuntimeError(
@@ -366,10 +371,11 @@
            target,
            f"{name}_add",
        )
    return res

+
def add_unary_layer(
    network: TRTNetwork,
    input_val: TRTTensor,
    operation_type: trt.UnaryOperation,
    target: Target,
@@ -397,10 +403,11 @@
    set_layer_name(layer, target, name)
    output = layer.get_output(0)
    output.name = output.name + "_" + target.__name__
    return layer.get_output(0)

+
def layer_norm(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
@@ -507,19 +514,21 @@
        trt.ElementWiseOperation.SUM,
        target,
        name,
    )

+
def add_add(network, target, kwargs, name):
    return add_binary_elementwise_layer(
        network,
        kwargs["input"],
        kwargs["other"],
        trt.ElementWiseOperation.SUM,
        target,
        name,
    )
+

def add_matmul(network, target, kwargs, name):
    input_val = get_trt_tensor(network, kwargs["input"], f"{name}_input")
    other_val = get_trt_tensor(network, kwargs["other"], f"{name}_other")

@@ -546,10 +555,11 @@
    layer = network.add_matrix_multiply(
        input_val, input_matrix_op, other_val, other_matrix_op
    )
    set_layer_name(layer, target, name)
    return layer.get_output(0)
+

def add_layer_norm(network, target, kwargs, name):
    input_val = kwargs["input"]

    if not isinstance(input_val, trt.tensorrt.ITensor):
@@ -589,10 +599,11 @@
        )
        return layer_norm(network, target, args, kwargs, name)
    layer = network.add_plugin_v2([input_val], plugin)
    layer.name = name
    return layer.get_output(0)
+

def add_cumsum(network, target, kwargs, name):
    input_val = kwargs["input"]
    dim = cast(int, kwargs["dim"])
    input_shape = input_val.shape  # type: ignore[union-attr]
@@ -661,79 +672,88 @@
    loop_output = loop.add_loop_output(current_sum, trt.LoopOutput.CONCATENATE, dim)
    set_layer_name(loop_output, target, f"{name}_loop_output")
    loop_output.set_input(1, trip_limit)
    return loop_output.get_output(0)

+
def add_maximum(network, target, kwargs, name):
    return add_binary_elementwise_layer(
        network,
        kwargs["input"],
        kwargs["other"],
        trt.ElementWiseOperation.MAX,
        target,
        name,
    )

+
def add_mul(network, target, kwargs, name):
    return add_binary_elementwise_layer(
        network,
        kwargs["input"],
        kwargs["other"],
        trt.ElementWiseOperation.PROD,
        target,
        name,
    )

+
def add_pow(network, target, kwargs, name):
    return add_binary_elementwise_layer(
        network,
        kwargs["input"],
        kwargs["other"],
        trt.ElementWiseOperation.POW,
        target,
        name,
    )

+
def add_floor_div(network, target, kwargs, name):
    return add_binary_elementwise_layer(
        network,
        kwargs["input"],
        kwargs["other"],
        trt.ElementWiseOperation.FLOOR_DIV,
        target,
        name,
    )

+
def add_div(network, target, kwargs, name):
    return add_binary_elementwise_layer(
        network,
        kwargs["input"],
        kwargs["other"],
        trt.ElementWiseOperation.DIV,
        target,
        name,
    )

+
def add_sub(network, target, kwargs, name):
    return add_binary_elementwise_layer(
        network,
        kwargs["input"],
        kwargs["other"],
        trt.ElementWiseOperation.SUB,
        target,
        name,
    )
+
+
def add_minimum(network, target, kwargs, name):
    return add_binary_elementwise_layer(
        network,
        kwargs["input"],
        kwargs["other"],
        trt.ElementWiseOperation.MIN,
        target,
        name,
    )

+
def add_logical_and(network, target, kwargs, name):
    if network.has_implicit_batch_dimension:
        raise RuntimeError(
            "The `ne` function should be called with explicit batch dimension."
        )
@@ -749,10 +769,11 @@
        network, input_t, other_t, trt.ElementWiseOperation.EQUAL, target, name
    )

    return add_unary_layer(network, eq_t, trt.UnaryOperation.NOT, target, name)

+
def add_ne(network, target, kwargs, name):
    if network.has_implicit_batch_dimension:
        raise RuntimeError(
            "The `ne` function should be called with explicit batch dimension."
        )
@@ -768,10 +789,11 @@
        network, input_t, other_t, trt.ElementWiseOperation.EQUAL, target, name
    )

    return add_unary_layer(network, eq_t, trt.UnaryOperation.NOT, target, name)

+
def add_eq(network, target, kwargs, name):
    if network.has_implicit_batch_dimension:
        raise RuntimeError(
            "The `eq` function should be called with explicit batch dimension."
        )
@@ -785,10 +807,11 @@
    input_t, other_t = dtype_uniform(network, target, name, input_t, other_t)
    return add_binary_elementwise_layer(
        network, input_t, other_t, trt.ElementWiseOperation.EQUAL, target, name
    )

+
def add_gt(network, target, kwargs, name):
    if network.has_implicit_batch_dimension:
        raise RuntimeError(
            "The `gt` function should be called with explicit batch dimension."
        )
@@ -802,10 +825,11 @@
    input_t, other_t = dtype_uniform(network, target, name, input_t, other_t)
    return add_binary_elementwise_layer(
        network, input_t, other_t, trt.ElementWiseOperation.GREATER, target, name
    )

+
def add_lt(network, target, kwargs, name):
    if network.has_implicit_batch_dimension:
        raise RuntimeError(
            "The `le` function should be called with explicit batch dimension."
        )
@@ -818,10 +842,11 @@

    input_t, other_t = dtype_uniform(network, target, name, input_t, other_t)
    return add_binary_elementwise_layer(
        network, input_t, other_t, trt.ElementWiseOperation.LESS, target, name
    )
+

def add_logical_or(network, target, kwargs, name):
    if network.has_implicit_batch_dimension:
        raise RuntimeError(
            "The `logical_or` function should be called with explicit batch dimension."
@@ -848,10 +873,11 @@

    return add_binary_elementwise_layer(
        network, input_t, other_t, trt.ElementWiseOperation.OR, target, name
    )

+
def add_logical_xor(network, target, kwargs, name):
    if network.has_implicit_batch_dimension:
        raise RuntimeError(
            "The `logical_xor` function should be called with explicit batch dimension."
        )
@@ -877,12 +903,13 @@

    return add_binary_elementwise_layer(
        network, input_t, other_t, trt.ElementWiseOperation.XOR, target, name
    )

+
def add_fmod(network, target, kwargs, name):
-     # NOTE: TRT doesnt currently implement fmod so we need multiple operations to perform it
+    # NOTE: TRT doesnt currently implement fmod so we need multiple operations to perform it
    trunc_div_value = trunc_div(
        kwargs["input"], kwargs["other"], network, target, name + "_trunc_div"
    )
    prod_value = add_binary_elementwise_layer(
        network,
@@ -900,12 +927,14 @@
        target,
        name + "_sub",
    )
    return sub_value

+
def add_trunc_div(network, target, kwargs, name):
    return trunc_div(kwargs["input"], kwargs["other"], network, target, name)
+

def add_expand(network, target, kwargs, name):
    input_t = kwargs["input"]
    shape = list(kwargs["sizes"])

@@ -927,10 +956,11 @@
    )  # stride == 1 if dimensions match, 0 otherwise
    layer = network.add_slice(input_val, start=start, shape=shape, stride=stride)
    set_layer_name(layer, target, name)
    return layer.get_output(0)

+
def add_slice(network, target, kwargs, name):
    input_val = kwargs["input"]

    if not isinstance(input_val, TRTTensor):
        raise RuntimeError(
@@ -974,9 +1004,5 @@
    )
    if dynamic_shape:
        layer.set_input(2, output_shape)
    set_layer_name(layer, target, name)
    return layer.get_output(0)
-
-
-        
-    
--- py/torch_tensorrt/fx/converters/acc_ops_converters.py	2023-03-17 19:06:49.232797 +0000
+++ py/torch_tensorrt/fx/converters/acc_ops_converters.py	2023-03-17 19:07:05.346527 +0000
@@ -678,11 +678,12 @@


@tensorrt_converter(acc_ops.layer_norm)
def acc_ops_layer_norm(network, target, args, kwargs, name):
    return add_layer_norm(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.softmax)
def acc_ops_softmax(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
@@ -729,11 +730,12 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_tile(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.sign)
def acc_ops_sign(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
@@ -758,50 +760,55 @@
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_relu(network, target, kwargs, name)

+
@tensorrt_converter(acc_ops.leaky_relu)
def acc_ops_leaky_relu(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_leaky_relu(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.elu)
def acc_ops_elu(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_elu(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.selu)
def acc_ops_selu(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_selu(network, target, kwargs, name)

+
@tensorrt_converter(acc_ops.softsign)
def acc_ops_softsign(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_softsign(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.sin)
def acc_ops_sin(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
@@ -872,10 +879,11 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_tanh(network, target, kwargs, name)
+

@tensorrt_converter(acc_ops.asin)
def acc_ops_asin(
    network: TRTNetwork,
    target: Target,
@@ -1189,21 +1197,23 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_maximum(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.minimum)
def acc_ops_minimum(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_minimum(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.dtype)
def acc_ops_dtype(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
@@ -1269,71 +1279,77 @@
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_logical_and(network, target, kwargs, name)

+
@tensorrt_converter(acc_ops.ne, no_implicit_batch_dim=True)
def acc_ops_ne(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_ne(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.eq, no_implicit_batch_dim=True)
def acc_ops_eq(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_eq(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.gt, no_implicit_batch_dim=True)
def acc_ops_gt(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_gt(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.lt, no_implicit_batch_dim=True)
def acc_ops_lt(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_lt(network, target, kwargs, name)
-    
+

@tensorrt_converter(acc_ops.logical_or, no_implicit_batch_dim=True)
def acc_ops_logical_or(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_logical_or(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.logical_xor, no_implicit_batch_dim=True)
def acc_ops_logical_xor(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_logical_xor(network, target, kwargs, name)
-    
+
+
# T113156424 Have some accuracy problems in hf_T5.
# [TRT] [W] Weights [name=isinf_1_inf_t]: Converted FP32 value in weights (either FP32 infinity or FP32 value outside FP16 range) to corresponding FP16 infinity. If this is not the desired behavior, please modify the weights or retrain with regularization to reduce the magnitude of the weights.
# @tensorrt_converter(acc_ops.isinf)
# def acc_ops_isinf(
#     network: TRTNetwork,
@@ -1422,10 +1438,11 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_fmod(network, target, kwargs, name)
+

# T113156424 embedding implemenatation is very limited and shows no usage in hf models due to the indices are int64.
# if we cast to int32, it will create accuracy issues. We'd better leave it to future implementation.
# @tensorrt_converter(acc_ops.embedding, no_implicit_batch_dim=True)
# def acc_ops_embedding(
@@ -1651,69 +1668,76 @@
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_add(network, target, kwargs, name)

+
@tensorrt_converter(acc_ops.sub)
def acc_ops_sub(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_sub(network, target, kwargs, name)

+
@tensorrt_converter(acc_ops.div)
def acc_ops_div(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_div(network, target, kwargs, name)

+
@tensorrt_converter(acc_ops.floor_div)
def acc_ops_floor_div(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_floor_div(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.trunc_div)
def acc_ops_trunc_div(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_trunc_div(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.mul)
def acc_ops_mul(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_mul(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.pow)
def acc_ops_pow(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_pow(network, target, kwargs, name)
+

@tensorrt_converter(acc_ops.unsqueeze)
def acc_ops_unsqueeze(
    network: TRTNetwork,
    target: Target,
@@ -1980,22 +2004,22 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_slice(network, target, kwargs, name)
-    
+

@tensorrt_converter(acc_ops.expand)
def acc_ops_expand_tensor(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_expand(network, target, kwargs, name)
-    
+

@tensorrt_converter(acc_ops.where)
def acc_ops_where(
    network: TRTNetwork,
    target: Target,
@@ -2213,11 +2237,12 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_linear(network, target, kwargs, name)
-    
+
+
def add_clamp(network, input, val, op, name):
    if not len(input.shape):
        # clamping scalar
        acc_ops_clamp_trt = get_trt_tensor(
            network,
@@ -2466,11 +2491,12 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_matmul(network, target, kwargs, name)
-    
+
+
@tensorrt_converter(acc_ops.hardsigmoid)
def acc_ops_hard_sigmoid(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
@@ -2688,11 +2714,11 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_gelu(network, target, kwargs, name)
-    
+

@tensorrt_converter(acc_ops.chunk)
def acc_ops_chunk(
    network: TRTNetwork,
    target: Target,
@@ -2765,20 +2791,22 @@
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_cumsum(network, target, kwargs, name)
-   
+
+
@tensorrt_converter(acc_ops.hardtanh)
def acc_ops_hardtanh(
    network: TRTNetwork,
    target: Target,
    args: Tuple[Argument, ...],
    kwargs: Dict[str, Argument],
    name: str,
) -> Union[TRTTensor, Sequence[TRTTensor]]:
    return add_hardtanh(network, target, kwargs, name)
+

@tensorrt_converter(acc_ops.interpolate)
def acc_ops_interpolate(
    network: TRTNetwork,
    target: Target,
--- py/torch_tensorrt/fx/test/converters/aten_op/test_gelu_aten.py	2023-03-17 19:06:49.236797 +0000
+++ py/torch_tensorrt/fx/test/converters/aten_op/test_gelu_aten.py	2023-03-17 19:07:06.249147 +0000
@@ -1,20 +1,19 @@
import torch
import torch.nn as nn
from torch.testing._internal.common_utils import run_tests
from torch_tensorrt.fx.tools.common_fx2trt import DispatchTestCase, InputTensorSpec
+

class TestGeLUConverter(DispatchTestCase):
    def test_gelu(self):
        class TestModule(nn.Module):
            def forward(self, x):
                return nn.functional.gelu(x)

        inputs = [torch.randn(1, 10)]
-        self.run_test(
-            TestModule(), inputs, expected_ops={torch.ops.aten.gelu.default}
-        )
+        self.run_test(TestModule(), inputs, expected_ops={torch.ops.aten.gelu.default})

    def test_gelu_with_dynamic_shape(self):
        class TestModule(nn.Module):
            def forward(self, x):
                return nn.functional.gelu(x)
@@ -47,6 +46,6 @@
            TestModule(), input_specs, expected_ops={torch.ops.aten.gelu.default}
        )


if __name__ == "__main__":
-    run_tests()
\ No newline at end of file
+    run_tests()
--- py/torch_tensorrt/fx/test/converters/aten_op/test_leaky_relu_aten.py	2023-03-17 19:06:49.236797 +0000
+++ py/torch_tensorrt/fx/test/converters/aten_op/test_leaky_relu_aten.py	2023-03-17 19:07:06.253917 +0000
@@ -48,6 +48,6 @@
            TestModule(), input_specs, expected_ops={torch.ops.aten.leaky_relu.default}
        )


if __name__ == "__main__":
-    run_tests()
\ No newline at end of file
+    run_tests()
--- py/torch_tensorrt/fx/test/converters/aten_op/test_tanh_aten.py	2023-03-17 19:06:49.236797 +0000
+++ py/torch_tensorrt/fx/test/converters/aten_op/test_tanh_aten.py	2023-03-17 19:07:06.317063 +0000
@@ -1,20 +1,19 @@
import torch
import torch.nn as nn
from torch.testing._internal.common_utils import run_tests
from torch_tensorrt.fx.tools.common_fx2trt import DispatchTestCase, InputTensorSpec
+

class TestTanhConverter(DispatchTestCase):
    def test_tanh(self):
        class TestModule(nn.Module):
            def forward(self, x):
                return nn.functional.tanh(x)

        inputs = [torch.randn(1, 10)]
-        self.run_test(
-            TestModule(), inputs, expected_ops={torch.ops.aten.tanh.default}
-        )
+        self.run_test(TestModule(), inputs, expected_ops={torch.ops.aten.tanh.default})

    def test_tanh_with_dynamic_shape(self):
        class TestModule(nn.Module):
            def forward(self, x):
                return nn.functional.tanh(x)
@@ -47,6 +46,6 @@
            TestModule(), input_specs, expected_ops={torch.ops.aten.tanh.default}
        )


if __name__ == "__main__":
-    run_tests()
\ No newline at end of file
+    run_tests()

gs-olive · 2023-03-17T20:36:50Z

py/torch_tensorrt/fx/converters/aten_ops_converters.py

-    return acc_ops_converters.acc_ops_add(network, target, None, kwargs_new, name)
+    return add_add(network, target, None, kwargs_new, name)


When using the PR on the following model:

class Sample(torch.nn.Module): def __init__(self): super(Sample, self).__init__() def forward(self, x): return torch.add(x, x)

I am encountering this error:

Error

Supported node types in the model: torch.ops.aten.add.Tensor: ((torch.float16, torch.float16), {}) Unsupported node types in the model: Got 1 acc subgraphs and 0 non-acc subgraphs Traceback (most recent call last): File "case_test.py", line 29, in <module> main() File "case_test.py", line 25, in main trt_model = torch_tensorrt.fx.compile(model, [input_], min_acc_module_size=1, is_aten=True) File "~/TensorRT/py/torch_tensorrt/fx/lower.py", line 86, in compile return lowerer(module, input) File "~/TensorRT/py/torch_tensorrt/fx/lower.py", line 316, in __call__ return do_lower(module, inputs) File "~/TensorRT/py/torch_tensorrt/fx/passes/pass_utils.py", line 118, in pass_with_validation processed_module = pass_(module, input, *args, **kwargs) File "~/TensorRT/py/torch_tensorrt/fx/lower.py", line 313, in do_lower lower_result = pm(module) File "/usr/local/lib/python3.8/dist-packages/torch/fx/passes/pass_manager.py", line 246, in __call__ out = _pass(out) File "/usr/local/lib/python3.8/dist-packages/torch/fx/passes/pass_manager.py", line 246, in __call__ out = _pass(out) File "~/TensorRT/py/torch_tensorrt/fx/passes/lower_pass_manager_builder.py", line 202, in lower_func lowered_module = self._lower_func( File "~/TensorRT/py/torch_tensorrt/fx/lower.py", line 178, in lower_pass interp_res: TRTInterpreterResult = interpreter(mod, input, module_name) File "~/TensorRT/py/torch_tensorrt/fx/lower.py", line 130, in __call__ interp_result: TRTInterpreterResult = interpreter.run( File "~/TensorRT/py/torch_tensorrt/fx/fx2trt.py", line 204, in run super().run() File "/usr/local/lib/python3.8/dist-packages/torch/fx/interpreter.py", line 137, in run self.env[node] = self.run_node(node) File "~/TensorRT/py/torch_tensorrt/fx/fx2trt.py", line 275, in run_node trt_node = super().run_node(n) File "/usr/local/lib/python3.8/dist-packages/torch/fx/interpreter.py", line 179, in run_node return getattr(self, n.op)(n.target, args, kwargs) File "~/TensorRT/py/torch_tensorrt/fx/fx2trt.py", line 328, in call_function return converter(self.network, target, args, kwargs, self._cur_node_name) File "~/TensorRT/py/torch_tensorrt/fx/converters/aten_ops_converters.py", line 45, in aten_ops_add return add_add(network, target, None, kwargs_new, name) TypeError: add_add() takes 4 positional arguments but 5 were given While executing %add_tensor : [#users=1] = call_function[target=torch.ops.aten.add.Tensor](args = (%arg0, %arg0), kwargs = {_itensor_to_tensor_meta: {<tensorrt.tensorrt.ITensor object at 0x7f06def57470>: None}}) Original traceback: File "case_test.py", line 17, in forward return torch.add(x, x)

github-actions

Code conforms to C++ style guidelines

github-actions

Code conforms to Python style guidelines

github-actions

Code conforms to C++ style guidelines

github-actions

Code conforms to Python style guidelines

github-actions

Code conforms to Python style guidelines

github-actions

Code conforms to C++ style guidelines

…onverters_transformer_encoder

…ed in lowering

…mer_encoder fx2trt converters transformer encoder

…lect fx2trt converters aten::slice,aten::select and aten::matmul

github-actions

Code conforms to C++ style guidelines

github-actions

Code conforms to Python style guidelines

github-actions

Code conforms to Python style guidelines

github-actions

Code conforms to C++ style guidelines

frank-wei · 2023-04-24T05:50:28Z

This PR is an alias of other PR like #1814 and #1770.
I think I have commented on the #1814 #1814 (comment)
The basic idea is that unit tests could not protect us 100% from the development bug. The fx2trt is used in prod and we are really careful about any big changes. Even if we merge here but once we import it, it has more chances to break internal tests and we have to revert it. So we really don't expect any potential refactoring change.
cc @wushirong for other comments.

github-actions

Code conforms to Python style guidelines

github-actions

Code conforms to C++ style guidelines

apbose added 6 commits March 10, 2023 10:39

fx2trt converters - reorg of the existing converters, addition of new…

840de6b

… converters to aten and nn

aten converter- matmul, tanh, gelu, slice select

3f3a925

Fixing matmul, select, tanh tests

8c8e897

Modifications to matmul and select tests

4f742a9

Fixing aten::select test

e8c2786

Removing matmul and select operator

038520d

apbose requested a review from narendasan March 17, 2023 19:06

facebook-github-bot added cla signed fx labels Mar 17, 2023

github-actions bot added component: api [Python] Issues re: Python API component: fx labels Mar 17, 2023

github-actions bot requested a review from frank-wei March 17, 2023 19:06

github-actions bot approved these changes Mar 17, 2023

View reviewed changes

github-actions bot requested changes Mar 17, 2023

View reviewed changes

This was referenced Mar 17, 2023

Prototype reorg of the FX converters to clean dependency chain #1683

Closed

Fx2trt converters #1658

Closed

apbose removed request for narendasan and frank-wei March 17, 2023 19:17

apbose self-assigned this Mar 17, 2023

gs-olive reviewed Mar 17, 2023

View reviewed changes

github-actions bot approved these changes Mar 20, 2023

View reviewed changes

fx2trt fixing add converter and python linting changes

e8a8e38

github-actions bot approved these changes Mar 21, 2023

View reviewed changes

binary operator changes in aten

d7c82ab

github-actions bot approved these changes Mar 21, 2023

View reviewed changes

matmul changes, bmm changes and adding broadcastable

09a52b9

apbose mentioned this pull request Apr 19, 2023

fx2trt converters aten::slice,aten::select and aten::matmul #1770

Merged

apbose and others added 14 commits April 19, 2023 08:43

Correcting pre-commit hooks

d1fd1d7

Resolving merge conflicts with bose_fx2trt_converters

39e4da6

Merge branch 'main' into bose_fx2trt_converters_transformer_encoder

d9545be

Merge branch 'bose_fx2trt_converters_slice_select' into bose_fx2trt_c…

89f2fcf

…onverters_transformer_encoder

Correcting rsqrt and rsub operator

ce7f122

python linting issues and removing chunk test

30c5fd6

Correcting acc squeeze test

7ab071d

test_reshape expected ops aten.reshape since aten.view has been remov…

36ac0cf

…ed in lowering

removing aten.view in lowering pass

eb851b1

layer_norm test

6b234e0

correcting linting error

95c1ada

correcting dynamic shape layer norm

1a1b809

Merge pull request #1814 from pytorch/bose_fx2trt_converters_transfor…

f721ed1

…mer_encoder fx2trt converters transformer encoder

Merge pull request #1770 from pytorch/bose_fx2trt_converters_slice_se…

ae2706d

…lect fx2trt converters aten::slice,aten::select and aten::matmul

github-actions bot approved these changes Apr 21, 2023

View reviewed changes

merging main

34268e7

frank-wei requested a review from wushirong April 24, 2023 05:34

github-actions bot approved these changes Apr 24, 2023

View reviewed changes

removing aten_tracer and lower_basic_pass_aten changes

01e5aa1

narendasan added priority: high release: v1.4 Tagged to be included in v1.4 labels Apr 24, 2023

github-actions bot requested a review from narendasan April 24, 2023 20:26

narendasan closed this May 22, 2023

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

fx2trt converters - change of prototype and addition of activation operation #1745

fx2trt converters - change of prototype and addition of activation operation #1745

apbose commented Mar 17, 2023 •

edited by narendasan

Loading

github-actions bot left a comment

github-actions bot left a comment

github-actions bot left a comment

github-actions bot left a comment

gs-olive Mar 17, 2023

github-actions bot left a comment

github-actions bot left a comment

github-actions bot left a comment

github-actions bot left a comment

github-actions bot left a comment

github-actions bot left a comment

github-actions bot left a comment

github-actions bot left a comment

github-actions bot left a comment

github-actions bot left a comment

frank-wei commented Apr 24, 2023

github-actions bot left a comment

github-actions bot left a comment

		return acc_ops_converters.acc_ops_add(network, target, None, kwargs_new, name)
		return add_add(network, target, None, kwargs_new, name)

fx2trt converters - change of prototype and addition of activation operation #1745

fx2trt converters - change of prototype and addition of activation operation #1745

Conversation

apbose commented Mar 17, 2023 • edited by narendasan Loading

Description

github-actions bot left a comment

Choose a reason for hiding this comment

github-actions bot left a comment

Choose a reason for hiding this comment

github-actions bot left a comment

Choose a reason for hiding this comment

github-actions bot left a comment

Choose a reason for hiding this comment

gs-olive Mar 17, 2023

Choose a reason for hiding this comment

github-actions bot left a comment

Choose a reason for hiding this comment

github-actions bot left a comment

Choose a reason for hiding this comment

github-actions bot left a comment

Choose a reason for hiding this comment

github-actions bot left a comment

Choose a reason for hiding this comment

github-actions bot left a comment

Choose a reason for hiding this comment

github-actions bot left a comment

Choose a reason for hiding this comment

github-actions bot left a comment

Choose a reason for hiding this comment

github-actions bot left a comment

Choose a reason for hiding this comment

github-actions bot left a comment

Choose a reason for hiding this comment

github-actions bot left a comment

Choose a reason for hiding this comment

frank-wei commented Apr 24, 2023

github-actions bot left a comment

Choose a reason for hiding this comment

github-actions bot left a comment

Choose a reason for hiding this comment

apbose commented Mar 17, 2023 •

edited by narendasan

Loading