[API Proposal]: Tensor Operators

[michaelgsharp]

Background and motivation

Tensors are often used to do multi-dimensional math in an efficient manner. We need to add these operators to our Tensor class to enable this behavior. The language team is working on adding 2 features to enable this behavior. First, the ability to restrict which types get certain operators, and second, the ability for users to implement their own compound operators, such as +=.

Note: this API is currently blocked waiting for 2 language features that are supposed to be implemented for .NET 10. All the following API are based on their current in progress work, but if anything major changes we will need to adjust accordingly.

API Proposal

namespace System.Numerics.Tensors;

public static partial class Tensor
{
    extension<TSelf, T>(TSelf)
        where TSelf: IReadOnlyTensor<TSelf, T>
        where T : IMultiplyOperators<T, T, T>
    {
        public static TSelf operator *(TSelf left, T right) { ... }
        public static TSelf operator *(TSelf left, TSelf right) { ... }
        public static TSelf operator *(T left, TSelf right) { ... }
    }

    extension<TSelf, T>(TSelf)
        where TSelf: IReadOnlyTensor<TSelf, T>
        where T : IAdditionOperators<T, T, T>
    {
        public static TSelf operator +(TSelf left, T right) { ... }
        public static TSelf operator +(TSelf left, TSelf right) { ... }
        public static TSelf operator +(T left, TSelf right) { ... }
    }

    extension<TSelf, T>(TSelf)
        where TSelf: IReadOnlyTensor<TSelf, T>
        where T : IModulusOperators<T, T, T>
    {
        public static TSelf operator %(TSelf left, T right) { ... }
        public static TSelf operator %(TSelf left, TSelf right) { ... }
        public static TSelf operator %(T left, TSelf right) { ... }
    }

    extension<TSelf, T>(TSelf)
        where TSelf: IReadOnlyTensor<TSelf, T>
        where T : IDivisionOperators<T, T, T>
    {
        public static TSelf operator /(TSelf left, T right) { ... }
        public static TSelf operator /(TSelf left, TSelf right) { ... }
        public static TSelf operator /(T left, TSelf right) { ... }
    }

    extension<TSelf, T>(TSelf)
        where TSelf: IReadOnlyTensor<TSelf, T>
        where T : ISubtractionOperators<T, T, T>
    {
        public static TSelf operator -(TSelf left, T right) { ... }
        public static TSelf operator -(TSelf left, TSelf right) { ... }
        public static TSelf operator -(T left, TSelf right) { ... }
    }

       // REVIEW: no == for now

    extension<TSelf, T>(TSelf)
        where TSelf: IReadOnlyTensor<TSelf, T>
        where T : IUnaryNegationOperators<T, T>
    {
        public static TSelf operator -(TSelf tensor) { ... }
    }

    extension<TSelf, T>(TSelf)
        where TSelf: IReadOnlyTensor<TSelf, T>
        where T : IUnaryPlusOperators<T, T>
    {
        public static TSelf operator +(TSelf tensor) { ... }
    }

    extension<TSelf, T>(TSelf)
        where TSelf: IReadOnlyTensor<TSelf, T>
        where T : IBitwiseOperators<T, T, T>
    {
        public static TSelf operator ~(TSelf tensor) { ... }

        public static TSelf operator &(TSelf left, T right) { ... }
        public static TSelf operator &(TSelf left, TSelf right) { ... }
        public static TSelf operator &(T left, TSelf right) { ... }

        public static TSelf operator |(TSelf left, T right) { ... }
        public static TSelf operator |(TSelf left, TSelf right) { ... }
        public static TSelf operator |(T left, TSelf right) { ... }

        public static TSelf operator ^(TSelf left, T right) { ... }
        public static TSelf operator ^(TSelf left, TSelf right) { ... }
        public static TSelf operator ^(T left, TSelf right) { ... }
    }

    extension<TSelf, T>(TSelf)
    where TSelf: IReadOnlyTensor<TSelf, T>
    where T : IShiftOperators<T, T, int>
    {
        public static TSelf operator <<(this TSelf tensor, int shiftAmount) { ... }
        public static TSelf operator >>(this TSelf tensor, int shiftAmount) { ... }
        public static TSelf operator >>>(this TSelf tensor, int shiftAmount) { ... }
    }

    extension<TSelf, T>(TSelf)
        where TSelf: ITensor<TSelf, T>, class
        where T : IMultiplyOperators<T, T, T>
    {
        // In place updates need no allocation
        public static void operator *=(this TSelf tensor, T other) { ... }
        public static void operator *=(this TSelf tensor, TSelf other) { ... }
    }

    extension<TSelf, T>(TSelf)
        where TSelf: IReadOnlyTensor<TSelf, T>, struct
        where T : IMultiplyOperators<T, T, T>
    {
        // In place updates need no allocation
        public static void operator *=(ref this TSelf tensor, T other) { ... }
        public static void operator *=(ref this TSelf tensor, TSelf other) { ... }
    }

    extension<TSelf, T>(TSelf)
        where TSelf: ITensor<TSelf, T>, class
        where T : IAdditionOperators<T, T, T>
    {
        public static void operator +=(this TSelf tensor, TSelf other) { ... }
        public static void operator +=(this TSelf tensor, T other) { ... }
    }

    extension<TSelf, T>(TSelf)
        where TSelf: ITensor<TSelf, T>, struct
        where T : IAdditionOperators<T, T, T>, IAdditiveIdentity<T, T>
    {
        public static void operator +=(ref this TSelf tensor, TSelf other) { ... }
        public static void operator +=(ref this TSelf tensor, T other) { ... }
    }

    extension<TSelf, T>(TSelf)
        where TSelf: ITensor<TSelf, T>, class
        where T : IIncrementOperators<T>
    {
        public static void operator ++(this TSelf tensor) { ... }
    }

    extension<TSelf, T>(TSelf)
        where TSelf: ITensor<TSelf, T>, class
        where T : IModulusOperators<T, T, T>
    {
        public static void operator %=(this TSelf tensor, T other) { ... }
        public static void operator %=(this TSelf tensor, TSelf other) { ... }
    }

    extension<TSelf, T>(TSelf)
        where TSelf: ITensor<TSelf, T>, struct
        where T : IModulusOperators<T, T, T>
    {
        public static void operator %=(ref this TSelf tensor, T other) { ... }
        public static void operator %=(ref this TSelf tensor, TSelf other) { ... }
    }

    extension<TSelf, T>(TSelf)
        where TSelf: ITensor<TSelf, T>, class
        where T : IDivisionOperators<T, T, T>
    {
        public static void operator /=(this TSelf tensor, T other) { ... }
        public static void operator /=(this TSelf tensor, TSelf other) { ... }
    }

    extension<TSelf, T>(TSelf)
        where TSelf: ITensor<TSelf, T>, struct
        where T : IDivisionOperators<T, T, T>
    {
        public static void operator /=(ref this TSelf tensor, T other) { ... }
        public static void operator /=(ref this TSelf tensor, TSelf other) { ... }
    }

    extension<TSelf, T>(TSelf)
        where TSelf: ITensor<TSelf, T>, class
        where T : ISubtractionOperators<T, T, T>
    {
        public static void operator -=(this TSelf tensor, T other) { ... }
        public static void operator -=(this TSelf tensor, TSelf other) { ... }
    }

    extension<TSelf, T>(TSelf)
        where TSelf: ITensor<TSelf, T>, struct
        where T : ISubtractionOperators<T, T, T>
    {
        public static void operator -=(ref this TSelf tensor, T other) { ... }
        public static void operator -=(ref this TSelf tensor, TSelf other) { ... }
    }

    extension<TSelf, T>(TSelf)
        where TSelf: ITensor<TSelf, T>, class
        where T : IDecrementOperator<T>
    {
        public static void operator --(this TSelf tensor) { ... }
    }

    extension<TSelf, T>(TSelf)
    where TSelf: ITensor<TSelf, T>, class
    where T : IShiftOperators<T, T, int>
    {
        public static void operator <<=(this TSelf tensor, int shiftAmount) { ... }
        public static void operator >>=(this TSelf tensor, int shiftAmount) { ... }
        public static void operator >>>=(this TSelf tensor, int shiftAmount) { ... }
    }

    extension<TSelf, T>(TSelf)
    where TSelf: ITensor<TSelf, T>, struct
    where T : IShiftOperators<T, T, int>
    {
        public static void operator <<=(ref this TSelf tensor, int shiftAmount) { ... }
        public static void operator >>=(ref this TSelf tensor, int shiftAmount) { ... }
        public static void operator >>>=(ref this TSelf tensor, int shiftAmount) { ... }
    }

    extension<TSelf, T>(TSelf)
    where TSelf: ITensor<TSelf, T>, class
    where T : IBitwiseOperators<T, T, T>
    {
        public static void operator &=(thisTSelf tensor, T other) { ... }
        public static void operator &=(this TSelf tensor, TSelf other) { ... }

        public static void operator |=(this TSelf tensor, T other) { ... }
        public static void operator |=(this TSelf tensor, TSelf other) { ... }

        public static void operator ^=(this TSelf tensor, T other) { ... }
        public static void operator ^=(this TSelf tensor, TSelf other) { ... }
    }

    extension<TSelf, T>(TSelf)
    where TSelf: ITensor<TSelf, T>, struct
    where T : IBitwiseOperators<T, T, T>
    {
        public static void operator &=(ref this TSelf tensor, T other) { ... }
        public static void operator &=(ref this TSelf tensor, TSelf other) { ... }

        public static void operator |=(ref this TSelf tensor, T other) { ... }
        public static void operator |=(ref this TSelf tensor, TSelf other) { ... }

        public static void operator ^=(ref this TSelf tensor, T other) { ... }
        public static void operator ^=(ref this TSelf tensor, TSelf other) { ... }
    }
}

API Usage

Tensor<int> tensor = Tensor.Create<int>([1, 2, 3, 4], [2, 2]);

// After this the values inside should be [3, 4, 5, 6]
tensor += 2;

// after this the values inside should [9, 12, 15, 18] 
tensor *= 3;

// Allocates a new tensor with values [10, 13, 16, 19]
Tensor<int> tensor2 = tensor + 1;

Alternative Designs

None. We needed this functionality and so the language team figured out the best way to incorporate it in C# and we are just adopting it.

Risks

Low risk because it's a preview type and its all new features. The main risk will be around the language team and any delays they may have.

[dotnet-policy-service]

Tagging subscribers to this area: @dotnet/area-system-numerics-tensors
See info in area-owners.md if you want to be subscribed.

[tannergooding]

@michaelgsharp, some notes:

• The extension operators would be defined on the Tensor static class, not on the interfaces
• They should likely be taking TSelf rather than IReadOnlyTensor<TSelf, T>, to avoid boxing for value types implementing the interfaces
• The compound assignment operators would be void returning and take this TSelf as the first parameter.
• We should potentially also consider if we need both this TSelf and ref this TSelf for the value type variants (which should be possible with the new extension feature)
• We do need the set of TSelf op TSelf, T op TSelf and TSelf op T where that's relevant (same places we exposed the set for the named methods like Add, Multiply, or Divide).

[tannergooding]

There's some other operator extensions that look to be missing as well. Ones like &, |, ^, ~, unary -, unary +, ++, --, <<, >>, and >>>.

[michaelgsharp]

I wasn't sure if the other operators really applied to the tensors. but I'll get them added in along with your other suggestions.

[bartonjs]

Video

• (TSelf, T) => (TTensor, TScalar)
• There are a lot of syntax errors because of evolution of the feature. Read through them to the intent.
• Otherwise, looks good as proposed.

namespace System.Numerics.Tensors;

public static partial class Tensor
{
    extension<TTensor, TScalar>(TTensor)
        where TTensor: IReadOnlyTensor<TTensor, TScalar>
        where TScalar : IMultiplyOperators<TScalar, TScalar, TScalar>
    {
        public static TTensor operator *(TTensor left, TScalar right) { ... }
        public static TTensor operator *(TTensor left, TTensor right) { ... }
        public static TTensor operator *(TScalar left, TTensor right) { ... }
    }

    extension<TTensor, TScalar>(TTensor)
        where TTensor: IReadOnlyTensor<TTensor, TScalar>
        where TScalar : IAdditionOperators<TScalar, TScalar, TScalar>
    {
        public static TTensor operator +(TTensor left, TScalar right) { ... }
        public static TTensor operator +(TTensor left, TTensor right) { ... }
        public static TTensor operator +(TScalar left, TTensor right) { ... }
    }

    extension<TTensor, TScalar>(TTensor)
        where TTensor: IReadOnlyTensor<TTensor, TScalar>
        where TScalar : IModulusOperators<TScalar, TScalar, TScalar>
    {
        public static TTensor operator %(TTensor left, TScalar right) { ... }
        public static TTensor operator %(TTensor left, TTensor right) { ... }
        public static TTensor operator %(TScalar left, TTensor right) { ... }
    }

    extension<TTensor, TScalar>(TTensor)
        where TTensor: IReadOnlyTensor<TTensor, TScalar>
        where TScalar : IDivisionOperators<TScalar, TScalar, TScalar>
    {
        public static TTensor operator /(TTensor left, TScalar right) { ... }
        public static TTensor operator /(TTensor left, TTensor right) { ... }
        public static TTensor operator /(TScalar left, TTensor right) { ... }
    }

    extension<TTensor, TScalar>(TTensor)
        where TTensor: IReadOnlyTensor<TTensor, TScalar>
        where TScalar : ISubtractionOperators<TScalar, TScalar, TScalar>
    {
        public static TTensor operator -(TTensor left, TScalar right) { ... }
        public static TTensor operator -(TTensor left, TTensor right) { ... }
        public static TTensor operator -(TScalar left, TTensor right) { ... }
    }

       // REVIEW: no == for now

    extension<TTensor, TScalar>(TTensor)
        where TTensor: IReadOnlyTensor<TTensor, TScalar>
        where TScalar : IUnaryNegationOperators<TScalar, TScalar>
    {
        public static TTensor operator -(TTensor tensor) { ... }
    }

    extension<TTensor, TScalar>(TTensor)
        where TTensor: IReadOnlyTensor<TTensor, TScalar>
        where TScalar : IUnaryPlusOperators<TScalar, TScalar>
    {
        public static TTensor operator +(TTensor tensor) { ... }
    }

    extension<TTensor, TScalar>(TTensor)
        where TTensor: IReadOnlyTensor<TTensor, TScalar>
        where TScalar : IBitwiseOperators<TScalar, TScalar, TScalar>
    {
        public static TTensor operator ~(TTensor tensor) { ... }

        public static TTensor operator &(TTensor left, TScalar right) { ... }
        public static TTensor operator &(TTensor left, TTensor right) { ... }
        public static TTensor operator &(TScalar left, TTensor right) { ... }

        public static TTensor operator |(TTensor left, TScalar right) { ... }
        public static TTensor operator |(TTensor left, TTensor right) { ... }
        public static TTensor operator |(TScalar left, TTensor right) { ... }

        public static TTensor operator ^(TTensor left, TScalar right) { ... }
        public static TTensor operator ^(TTensor left, TTensor right) { ... }
        public static TTensor operator ^(TScalar left, TTensor right) { ... }
    }

    extension<TTensor, TScalar>(TTensor tensor)
    where TTensor: IReadOnlyTensor<TTensor, TScalar>
    where TScalar : IShiftOperators<TScalar, TScalar, int>
    {
        public static TTensor operator <<(TTensor tensor, int shiftAmount) { ... }
        public static TTensor operator >>(TTensor tensor, int shiftAmount) { ... }
        public static TTensor operator >>>(TTensor tensor, int shiftAmount) { ... }
    }

    extension<TTensor, TScalar>(TTensor tensor)
        where TTensor: ITensor<TTensor, TScalar>, class
        where TScalar : IMultiplyOperators<TScalar, TScalar, TScalar>
    {
        // In place updates need no allocation
        public void operator *=(TScalar other) { ... }
        public void operator *=(TTensor other) { ... }
    }

    extension<TTensor, TScalar>(ref TTensor tensor)
        where TTensor: IReadOnlyTensor<TTensor, TScalar>, struct
        where TScalar : IMultiplyOperators<TScalar, TScalar, TScalar>
    {
        // In place updates need no allocation
        public void operator *=(TScalar other) { ... }
        public void operator *=(TTensor other) { ... }
    }

    extension<TTensor, TScalar>(TTensor)
        where TTensor: ITensor<TTensor, TScalar>, class
        where TScalar : IAdditionOperators<TScalar, TScalar, TScalar>
    {
        public static void operator +=(this TTensor tensor, TTensor other) { ... }
        public static void operator +=(this TTensor tensor, TScalar other) { ... }
    }

    extension<TTensor, TScalar>(TTensor)
        where TTensor: ITensor<TTensor, TScalar>, struct
        where TScalar : IAdditionOperators<TScalar, TScalar, TScalar>, IAdditiveIdentity<TScalar, TScalar>
    {
        public static void operator +=(ref this TTensor tensor, TTensor other) { ... }
        public static void operator +=(ref this TTensor tensor, TScalar other) { ... }
    }

    extension<TTensor, TScalar>(TTensor)
        where TTensor: ITensor<TTensor, TScalar>, class
        where TScalar : IIncrementOperators<TScalar>
    {
        public static void operator ++(this TTensor tensor) { ... }
    }

    extension<TTensor, TScalar>(TTensor)
        where TTensor: ITensor<TTensor, TScalar>, class
        where TScalar : IModulusOperators<TScalar, TScalar, TScalar>
    {
        public static void operator %=(this TTensor tensor, TScalar other) { ... }
        public static void operator %=(this TTensor tensor, TTensor other) { ... }
    }

    extension<TTensor, TScalar>(TTensor)
        where TTensor: ITensor<TTensor, TScalar>, struct
        where TScalar : IModulusOperators<TScalar, TScalar, TScalar>
    {
        public static void operator %=(ref this TTensor tensor, TScalar other) { ... }
        public static void operator %=(ref this TTensor tensor, TTensor other) { ... }
    }

    extension<TTensor, TScalar>(TTensor)
        where TTensor: ITensor<TTensor, TScalar>, class
        where TScalar : IDivisionOperators<TScalar, TScalar, TScalar>
    {
        public static void operator /=(this TTensor tensor, TScalar other) { ... }
        public static void operator /=(this TTensor tensor, TTensor other) { ... }
    }

    extension<TTensor, TScalar>(TTensor)
        where TTensor: ITensor<TTensor, TScalar>, struct
        where TScalar : IDivisionOperators<TScalar, TScalar, TScalar>
    {
        public static void operator /=(ref this TTensor tensor, TScalar other) { ... }
        public static void operator /=(ref this TTensor tensor, TTensor other) { ... }
    }

    extension<TTensor, TScalar>(TTensor)
        where TTensor: ITensor<TTensor, TScalar>, class
        where TScalar : ISubtractionOperators<TScalar, TScalar, TScalar>
    {
        public static void operator -=(this TTensor tensor, TScalar other) { ... }
        public static void operator -=(this TTensor tensor, TTensor other) { ... }
    }

    extension<TTensor, TScalar>(TTensor)
        where TTensor: ITensor<TTensor, TScalar>, struct
        where TScalar : ISubtractionOperators<TScalar, TScalar, TScalar>
    {
        public static void operator -=(ref this TTensor tensor, TScalar other) { ... }
        public static void operator -=(ref this TTensor tensor, TTensor other) { ... }
    }

    extension<TTensor, TScalar>(TTensor)
        where TTensor: ITensor<TTensor, TScalar>, class
        where TScalar : IDecrementOperator<TScalar>
    {
        public static void operator --(this TTensor tensor) { ... }
    }

    extension<TTensor, TScalar>(TTensor)
    where TTensor: ITensor<TTensor, TScalar>, class
    where TScalar : IShiftOperators<TScalar, TScalar, int>
    {
        public static void operator <<=(this TTensor tensor, int shiftAmount) { ... }
        public static void operator >>=(this TTensor tensor, int shiftAmount) { ... }
        public static void operator >>>=(this TTensor tensor, int shiftAmount) { ... }
    }

    extension<TTensor, TScalar>(TTensor)
    where TTensor: ITensor<TTensor, TScalar>, struct
    where TScalar : IShiftOperators<TScalar, TScalar, int>
    {
        public static void operator <<=(ref this TTensor tensor, int shiftAmount) { ... }
        public static void operator >>=(ref this TTensor tensor, int shiftAmount) { ... }
        public static void operator >>>=(ref this TTensor tensor, int shiftAmount) { ... }
    }

    extension<TTensor, TScalar>(TTensor)
    where TTensor: ITensor<TTensor, TScalar>, class
    where TScalar : IBitwiseOperators<TScalar, TScalar, TScalar>
    {
        public static void operator &=(thisTTensor tensor, TScalar other) { ... }
        public static void operator &=(this TTensor tensor, TTensor other) { ... }

        public static void operator |=(this TTensor tensor, TScalar other) { ... }
        public static void operator |=(this TTensor tensor, TTensor other) { ... }

        public static void operator ^=(this TTensor tensor, TScalar other) { ... }
        public static void operator ^=(this TTensor tensor, TTensor other) { ... }
    }

    extension<TTensor, TScalar>(TTensor)
    where TTensor: ITensor<TTensor, TScalar>, struct
    where TScalar : IBitwiseOperators<TScalar, TScalar, TScalar>
    {
        public static void operator &=(ref this TTensor tensor, TScalar other) { ... }
        public static void operator &=(ref this TTensor tensor, TTensor other) { ... }

        public static void operator |=(ref this TTensor tensor, TScalar other) { ... }
        public static void operator |=(ref this TTensor tensor, TTensor other) { ... }

        public static void operator ^=(ref this TTensor tensor, TScalar other) { ... }
        public static void operator ^=(ref this TTensor tensor, TTensor other) { ... }
    }
}