Experimental (#114)

JPietrzykTUD · alexKrauseTUD · web-flow · commit 732d2eb8b3f4 · 2025-02-28T22:37:54.000+01:00
* Fixed auto tagging for dockerimages

* Fixed typo.

* Added check + support for wget if curl does not exist on the target system in setup_tsl.sh

* Fixed regex error

* fixed left shift warnings

* Added better functor overload indication for HTML output

* added archid to possible cli params

* Added functionality to check whether a primitive is supported by the backend

* Added functionality to check whether a primitive is supported by the backend

* Fixed wrong bits set (integral all true) if number of lanes &lt; 8

---------

Co-authored-by: Alexander Krause &lt;alexander.krause@tu-dresden.de&gt;
diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -32,17 +32,17 @@ jobs:
       - name: Parse Target Specs
         id: parse-target-specs
         run: |
-          echo "x86_flags_matrix=$(jq '{include: .x86}' .github/workflows/target_specs.json -c)" >> $GITHUB_OUTPUT
+          echo "x86_flags_matrix=$(jq '{include: .x86}' target_specs.json -c)" >> $GITHUB_OUTPUT
           echo "x86_flags_compiler_matrix=$(jq '.x86 |= map([
             . + {compiler: "g++"}, 
             . + {compiler: "clang++"}
-          ]) | .x86 |= flatten | {include: .x86}' .github/workflows/target_specs.json -c)" >> $GITHUB_OUTPUT
-          echo "aarch64_flags_matrix=$(jq '{include: .aarch64}' .github/workflows/target_specs.json -c)" >> $GITHUB_OUTPUT
+          ]) | .x86 |= flatten | {include: .x86}' target_specs.json -c)" >> $GITHUB_OUTPUT
+          echo "aarch64_flags_matrix=$(jq '{include: .aarch64}' target_specs.json -c)" >> $GITHUB_OUTPUT
           echo "aarch64_flags_compiler_matrix=$(jq '.aarch64 |= map([
             . + {compiler: "aarch64-linux-gnu-g++"}, 
             . + {compiler: "clang++"}
-          ]) | .aarch64 |= flatten | {include: .aarch64}' .github/workflows/target_specs.json -c)" >> $GITHUB_OUTPUT
-          echo "target_matrix=$(jq -c '{include: ([keys[] as $k | .[$k][] | (. + { arch: $k })])}' .github/workflows/target_specs.json -c)" >> $GITHUB_OUTPUT
+          ]) | .aarch64 |= flatten | {include: .aarch64}' target_specs.json -c)" >> $GITHUB_OUTPUT
+          echo "target_matrix=$(jq -c '{include: ([keys[] as $k | .[$k][] | (. + { arch: $k })])}' target_specs.json -c)" >> $GITHUB_OUTPUT
   
   build-generation-image:
     name: Generation Image (build and push on demand)
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -29,7 +29,7 @@ jobs:
       - name: Parse Target Specs
         id: parse-target-specs
         run: |
-          echo "target_matrix=$(jq -c '{include: ([keys[] as $k | .[$k][] | (. + { arch: $k })])}' .github/workflows/target_specs.json -c)" >> $GITHUB_OUTPUT
+          echo "target_matrix=$(jq -c '{include: ([keys[] as $k | .[$k][] | (. + { arch: $k })])}' target_specs.json -c)" >> $GITHUB_OUTPUT
   
   build-generation-image:
     if: ${{ github.event_name != 'workflow_run' }}
@@ -142,7 +142,7 @@ jobs:
           python3 ${{ github.workspace }}/repository/.github/workflows/release/prepare_select_flavor.py \
             --install-sh ${{ github.workspace }}/release_tmp/select_flavor.sh \
             --folder-prefix "tsl" \
-            --targets-spec-file repository/.github/workflows/target_specs.json \
+            --targets-spec-file repository/target_specs.json \
             --tsl-folder-ph "$<< TslFolderArrayValues >>" \
             --default-flags-array-ph "$<< DefaultFlagsArrayValues >>" \
             --alt-flags-array-ph "$<< AlternativeFlagsArrayValues >>" \
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -29,6 +29,13 @@ else()
     set(DTD "DRAW_TEST_DEPENDENCIES")
   endif()
 
+  if (DEFINED ArchId)
+    message(STATUS "ArchId: ${ArchId}")
+  else()
+    message(STATUS "ArchId not defined. Defaulting to 0.")
+    set(ArchId "")
+  endif()
+
   create_tsl(
     ${WAW}
     ${UC}
@@ -37,6 +44,7 @@ else()
     TSLGENERATOR_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
     DESTINATION ${DESTINATION}
     TARGETS_FLAGS ${TARGETS_FLAGS}
+    TARGET_ARCHID ${ArchId}
     APPEND_TARGETS_FLAGS ${APPEND_TARGETS_FLAGS}
     PRIMITIVES_FILTER ${PRIMITIVES_FILTER}
     DATATYPES_FILTER ${DATATYPES_FILTER}
diff --git a/generator/core/tsl_config.py b/generator/core/tsl_config.py
@@ -1,5 +1,6 @@
 import argparse
 import copy
+import json
 import logging.config
 import pathlib
 import os
@@ -422,6 +423,7 @@ def parse_args(**kwargs) -> dict:
     parser.add_argument('--targets', default=None, nargs='*',
                         help='List of target flags which match the lscpu_flags from the extension/primitive files.',
                         dest='targets')
+    parser.add_argument('--archid', type=str, dest='target_archid', metavar="ArchId", help="Identifier of a target platform (e.g., 'skylake').")
     types_help = 'List of types which should be considered for generation.'
     if "known_types" in kwargs:
         types_help += f" Choose from the following list: [{', '.join(kwargs['known_types'])}]"
@@ -464,9 +466,29 @@ def parse_args(**kwargs) -> dict:
                 current_dict = current_dict[match]
             current_dict[matches[-1]] = value
 
+    targets_set = set()
+    if "target_archid" in args_dict:
+      with open("target_specs.json", 'r') as specs_file:
+        target_specs = json.load(specs_file)
+        found = False
+        for targets in target_specs.values():
+          for target in targets:
+            if target['name'] == args_dict["target_archid"]:
+              found = True
+              targets_set.update([flag.strip() for flag in target['flags'].split(" ")])
+              break
+        if not found:
+          print(f"Unknown target archid: {args_dict['target_archid']}")
+          exit(1)
+    
     if "targets" not in args_dict:
-        args_dict["targets"] = None
+        if len(targets_set) == 0:
+            args_dict["targets"] = None
+        else:
+            args_dict["targets"] = list(targets_set)
+    else:
+        targets_set.update(args_dict["targets"])
+        args_dict["targets"] = list(targets_set)
     # if "relevant_types" not in args_dict:
     #     args_dict["relevant_types"] = None
     return args_dict
-
diff --git a/generator/core/tsl_generator.py b/generator/core/tsl_generator.py
@@ -141,7 +141,7 @@ def generate(self, relevant_hardware_flags: List[str] = None, relevant_primitive
                 for primitive in pClass:
                     implregex_list.append( primitive.declaration.functor_name )
 
-            implregex_string = f'(?<!_)({"|".join(implregex_list)})(?!([a-zA-Z]|_|\[|\.))'
+            implregex_string = rf'(?<!_)({"|".join(implregex_list)})(?!([a-zA-Z]|_|\[|\.))'
 
             implregex = re.compile(implregex_string, re.IGNORECASE)
 
diff --git a/generator/static_files/core/utils/preprocessor.yaml b/generator/static_files/core/utils/preprocessor.yaml
@@ -136,5 +136,30 @@ implementations:
       };
       template <class... Functors>
       inline constexpr bool tsl_primitives_redeclared_v = tsl_primitives_redeclared<Functors...>::is_ambiguous::value;
-
+    - |
+      template <class Func, typename ArgsTuple, typename = std::void_t<>>
+      struct has_function : std::false_type {};
+      // Specialization: When Func<Ts...>(Args...) is valid
+      template <class Func, typename... Args>
+      struct has_function<Func, std::tuple<Args...>,
+          std::void_t<decltype(Func(std::declval<Args>()...))>> : std::true_type {};
+      template <typename... Args>
+      struct callable_by_types {
+          bool available;
+          template <typename Fun>
+          constexpr callable_by_types(Fun&&): available(std::is_invocable_v<Fun, Args...>) {}
+      };
+      struct callable_by_values {
+          bool available;
+          template <typename Fun, typename... Args>
+          constexpr callable_by_values(Fun&&, Args&&... args): available(has_function<Fun, std::tuple<decltype(args)...>>::value) {}
+      };
+      #define TSL_MAKE_CALLABLE(func) \
+          [](auto&&... args) -> decltype(func(std::forward<decltype(args)>(args)...)) { \
+              return func(std::forward<decltype(args)>(args)...); \
+          }
+      #define TSL_BACKEND_SUPPORTS_BY_TYPE(func, ...) \
+          tsl::callable_by_types<__VA_ARGS__>(TSL_MAKE_CALLABLE(func)).available
+      #define TSL_BACKEND_SUPPORTS_BY_VALUE(func, ...) \
+          tsl::callable_by_values(TSL_MAKE_CALLABLE(func), __VA_ARGS__).available
 ...
diff --git a/generator/static_files/core/utils/runtime.yaml b/generator/static_files/core/utils/runtime.yaml
@@ -13,7 +13,7 @@ implementations:
       class executor {
         public:
           template<typename T, int Par>
-          using simd_ext_t = typename ExecTarget::template simd_ext_t<T, Par>;
+          using simd_ext_by_par_t = typename ExecTarget::template simd_ext_by_par_t<T, Par>;
         private:
           ExecTarget target;
         public:
@@ -83,6 +83,17 @@ implementations:
             return ExecTarget::template available_parallelism<T>();
           }
 
+          template<TSLArithmetic T, int Par>
+          static constexpr bool parallelism_available() {
+            constexpr auto avail = available_parallelism<T>();
+            for (size_t i = 0; i < avail.size(); ++i) {
+              if (avail[i] == Par) {
+                return true;
+              }
+            }
+            return false;
+          }
+
           void wait() {
             target.wait();
           }
diff --git a/parseForPrimitiveTable.py b/parseForPrimitiveTable.py
@@ -13,8 +13,9 @@
 from generator.core.ctrl.tsl_lib import TSLLib
 
 class PrintablePrimitive:
-    def __init__(self, name: str, description: str, ctype_to_extension_dict: dict, vec_type: List[str], parameters: List[str], return_type: str) -> None:
+    def __init__(self, name: str, functor_name: str, description: str, ctype_to_extension_dict: dict, vec_type: List[str], parameters: List[str], return_type: str) -> None:
         self.name = name
+        self.functor_name = functor_name
         self.description = description
         self.ctype_to_extension_dict = ctype_to_extension_dict
         self.vec_type = vec_type
@@ -26,9 +27,14 @@ def __repr__(self) -> str:
         return f"{self.name}: {self.ctype_to_extension_dict}"
     
     def to_html(self, considered_types: list, considered_exts: list) -> str:
+        button_label = ""
+        if self.name != self.functor_name:
+          button_label = f"{self.name} <div style='font-size: small;'><i>overload</i> (calls tsl::functors::{self.functor_name})</div>"
+        else:
+           button_label = self.name
         call_signature = f"{self.name}&lt;{', '.join(self.vec_type)}&gt;({', '.join(self.parameters)}) -> {self.return_type}"
-        primitive_button = f"""<div class="primitiveContainer"><div class="primitive"><button id ="{self.name}_link" onclick="togglePrimitive(event, '{self.name}')">{self.name}</button></div>"""
-        primitive_table_start = f"""<div id="{self.name}" class="primitiveinfo"><p>Brief: <span class="description">{self.description}</span></p><p>Call signature: <br><code>{call_signature}</code></p><center><table border=1 cellpadding=10 cellspacing=0>"""
+        primitive_button = f"""<div class="primitiveContainer"><div class="primitive"><button id ="{self.functor_name}_link" onclick="togglePrimitive(event, '{self.functor_name}')">{button_label}</button></div>"""
+        primitive_table_start = f"""<div id="{self.functor_name}" class="primitiveinfo"><p>Brief: <span class="description">{self.description}</span></p><p>Call signature: <br><code>{call_signature}</code></p><center><table border=1 cellpadding=10 cellspacing=0>"""
         primitive_table_end = """</table></center></div><br/></div>"""
         
         top_left_corner = """<td style="border-top:0;border-left:0;"></td>"""
@@ -115,7 +121,8 @@ def create_primitive_index_html(tsl_lib: TSLLib) -> None:
     checkbox_html += add_checkbox(primitive_class.name)
     primitive_html += f"""<div class="primitiveCategory" id="{primitive_class.name}">"""
     for primitive in primitive_class:
-      name = primitive.declaration.functor_name
+      name = primitive.declaration.name
+      functor_name = primitive.declaration.functor_name
       brief_description = primitive.declaration.data["brief_description"]
       detailed_description = primitive.declaration.data["detailed_description"]
       ctype_ext_dict = copy.deepcopy(raw_primitive_dict)
@@ -127,7 +134,7 @@ def create_primitive_index_html(tsl_lib: TSLLib) -> None:
       vnames = [primitive.declaration.data["vector_name"]]
       if primitive.declaration.data["additional_simd_template_parameter"]['name'] != "":
         vnames.append(primitive.declaration.data["additional_simd_template_parameter"]['name'])
-      pP = PrintablePrimitive(name, brief_description, ctype_ext_dict, 
+      pP = PrintablePrimitive(name, functor_name,  brief_description, ctype_ext_dict, 
                               vnames,
                               [f"{html.escape(param['ctype'])} {param['name']}" for param in primitive.declaration.data["parameters"]],
                                html.escape(primitive.declaration.data['returns']['ctype']))
diff --git a/primitive_data/primitives/mask.yaml b/primitive_data/primitives/mask.yaml
@@ -798,42 +798,60 @@ definitions:
     implementation: "return ~0;"
   #INTEL - AVX2
   - target_extension: "avx2"
-    ctype: [ "uint8_t", "uint16_t", "uint32_t", "uint64_t", "int8_t", "int16_t", "int32_t", "int64_t", "float", "double" ]
+    ctype: [ "uint8_t", "uint16_t", "uint32_t", "int8_t", "int16_t", "int32_t", "float"]
     lscpu_flags: [ "avx2" ]
     includes: ["<type_traits>"]
     implementation: |
-      if constexpr(Vec::vector_element_count() < 8) {
-        return ((static_cast<typename Vec::imask_type>(1)<<Vec::vector_element_count()) - 1);
-      } else {
-        return ~0;
-      }
+      return static_cast<typename Vec::imask_type>(~0);
+  - target_extension: "avx2"
+    ctype: [ "uint64_t", "int64_t", "double" ]
+    lscpu_flags: [ "avx2" ]
+    includes: ["<type_traits>"]
+    implementation: |
+      return static_cast<typename Vec::imask_type>(0b1111);
   #INTEL - SSE
   - target_extension: "sse"
-    ctype: [ "uint8_t", "uint16_t", "uint32_t", "uint64_t", "int8_t", "int16_t", "int32_t", "int64_t", "float", "double" ]
+    ctype: [ "uint8_t", "uint16_t", "int8_t", "int16_t"]
     lscpu_flags: [ "sse" ]
     includes: ["<type_traits>"]
     implementation: |
-      if constexpr(Vec::vector_element_count() < 8) {
-        return ((static_cast<typename Vec::imask_type>(1)<<Vec::vector_element_count()) - 1);
-      } else {
-        return ~0;
-      }
+      return static_cast<typename Vec::imask_type>(~0);
+  - target_extension: "sse"
+    ctype: [ "uint32_t", "int32_t", "float" ]
+    lscpu_flags: [ "sse" ]
+    includes: ["<type_traits>"]
+    implementation: |
+      return static_cast<typename Vec::imask_type>(0b1111);
+  - target_extension: "sse"
+    ctype: [ "uint64_t", "int64_t", "double" ]
+    lscpu_flags: [ "sse" ]
+    includes: ["<type_traits>"]
+    implementation: |
+      return static_cast<typename Vec::imask_type>(0b11);
   #SCALAR
   - target_extension: "scalar"
     ctype: [ "uint8_t", "uint16_t", "uint32_t", "uint64_t", "int8_t", "int16_t", "int32_t", "int64_t", "float", "double" ]
     lscpu_flags: []
     implementation: return true;
   #ARM - NEON
   - target_extension: "neon"
-    ctype: [ "uint8_t", "uint16_t", "uint32_t", "uint64_t", "int8_t", "int16_t", "int32_t", "int64_t", "float", "double" ]
+    ctype: [ "uint8_t", "uint16_t", "int8_t", "int16_t"]
     lscpu_flags: [ "neon" ]
     includes: ["<type_traits>"]
     implementation: |
-      if constexpr(Vec::vector_element_count() < 8) {
-        return ((static_cast<typename Vec::imask_type>(1)<<Vec::vector_element_count()) - 1);
-      } else {
-        return ~0;
-      }
+      return static_cast<typename Vec::imask_type>(~0);
+  - target_extension: "neon"
+    ctype: [ "uint32_t", "int32_t", "float" ]
+    lscpu_flags: [ "neon" ]
+    includes: ["<type_traits>"]
+    implementation: |
+      return static_cast<typename Vec::imask_type>(0b1111);
+  - target_extension: "neon"
+    ctype: [ "uint64_t", "int64_t", "double" ]
+    lscpu_flags: [ "neon" ]
+    includes: ["<type_traits>"]
+    implementation: |
+      return static_cast<typename Vec::imask_type>(0b11);
 ...
 ---
 primitive_name: "integral_all_false"
@@ -957,19 +975,19 @@ definitions:
   - target_extension: "avx512"
     ctype: ["uint8_t", "uint16_t", "uint32_t", "uint64_t", "int8_t", "int16_t", "int32_t", "int64_t", "float", "double"]
     lscpu_flags: ["avx512f"]
-    implementation: "return (mask >> position) & ((1ULL << Vec::vector_element_count()) - 1);"
+    implementation: "return (mask >> position) & (static_cast<typename Vec::imask_type>(~0));"
   - target_extension: "avx2"
     ctype: ["uint8_t", "uint16_t", "uint32_t", "uint64_t", "int8_t", "int16_t", "int32_t", "int64_t", "float", "double"]
     lscpu_flags: ["avx"]
-    implementation: "return (mask >> position) & ((1ULL << Vec::vector_element_count()) - 1);"
+    implementation: "return (mask >> position) & (static_cast<typename Vec::imask_type>(~0));"
   - target_extension: "sse"
     ctype: ["uint8_t", "uint16_t", "uint32_t", "uint64_t", "int8_t", "int16_t", "int32_t", "int64_t", "float", "double"]
     lscpu_flags: ["sse"]
-    implementation: "return (mask >> position) & ((1ULL << Vec::vector_element_count()) - 1);"
+    implementation: "return (mask >> position) & (static_cast<typename Vec::imask_type>(~0));"
   - target_extension: "scalar"
     ctype: ["uint8_t", "uint16_t", "uint32_t", "uint64_t", "int8_t", "int16_t", "int32_t", "int64_t", "float", "double"]
     lscpu_flags: []
-    implementation: "return (mask >> position) & ((1ULL << Vec::vector_element_count()) - 1);"
+    implementation: "return (mask >> position) & (static_cast<typename Vec::imask_type>(~0));"
 ...
 #---
 #primitive_name: "mask_reduce"
diff --git a/supplementary/runtime/cpu/include/tslCPUrt.hpp b/supplementary/runtime/cpu/include/tslCPUrt.hpp
@@ -19,10 +19,14 @@ namespace tsl {
     }
     class cpu {
       public:
-        template<typename T, int Par>
-        using simd_ext_t = typename std::conditional_t<
-            (Par==1), scalar, typename details::simd_ext_helper_t<sizeof(T)*CHAR_BIT*Par>::extension_t
-          >;
+        template <typename T, int Par>
+        struct simd_ext_by_par_t {
+          using type = typename details::simd_ext_helper_t<sizeof(T) * CHAR_BIT * Par>::extension_t;
+        };
+        template <typename T>
+        struct simd_ext_by_par_t<T, 1> {
+          using type = scalar;
+        };
 
         using max_width_extension_t = {{ avail_extension_types_dict[avail_extension_types_dict.keys()|max] }};
         using min_width_extension_t = {{ avail_extension_types_dict[avail_extension_types_dict.keys()|min] }};
diff --git a/target_specs.json b/target_specs.json
diff --git a/tsl.cmake b/tsl.cmake
