Make ready for data tagging

.azure-pipelines/azure-pipelines.yml

@@ -71,6 +71,20 @@ stages:
             - test: 3
             - test: 4
             - test: extra
+  - stage: Sanity_datatagging
+    displayName: Sanity datatagging
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          nameFormat: Test {0}
+          testFormat: datatagging/sanity/{0}
+          targets:
+            - test: 1
+            - test: 2
+            - test: 3
+            - test: 4
+            - test: extra
   - stage: Sanity_2_18
     displayName: Sanity 2.18
     dependsOn: []
@@ -126,6 +140,21 @@ stages:
             - test: '3.11'
             - test: '3.12'
             - test: '3.13'
+  - stage: Units_datatagging
+    displayName: Units datatagging
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          nameFormat: Python {0}
+          testFormat: datatagging/units/{0}/1
+          targets:
+            - test: 3.8
+            - test: 3.9
+            - test: '3.10'
+            - test: '3.11'
+            - test: '3.12'
+            - test: '3.13'
   - stage: Units_2_18
     displayName: Units 2.18
     dependsOn: []
@@ -200,6 +229,26 @@ stages:
             - 1
             - 2
             - 3
+  - stage: Remote_datatagging
+    displayName: Remote datatagging
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          testFormat: datatagging/{0}
+          targets:
+            - name: macOS 15.3
+              test: macos/15.3
+            - name: RHEL 9.5
+              test: rhel/9.5
+            - name: FreeBSD 14.2
+              test: freebsd/14.2
+            - name: FreeBSD 13.5
+              test: freebsd/13.5
+          groups:
+            - 1
+            - 2
+            - 3
   - stage: Remote_2_18
     displayName: Remote 2.18
     dependsOn: []
@@ -280,6 +329,26 @@ stages:
             - 1
             - 2
             - 3
+  - stage: Docker_datatagging
+    displayName: Docker datatagging
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          testFormat: datatagging/linux/{0}
+          targets:
+            - name: Fedora 41
+              test: fedora41
+            - name: Alpine 3.21
+              test: alpine321
+            - name: Ubuntu 22.04
+              test: ubuntu2204
+            - name: Ubuntu 24.04
+              test: ubuntu2404
+          groups:
+            - 1
+            - 2
+            - 3
   - stage: Docker_2_18
     displayName: Docker 2.18
     dependsOn: []
@@ -409,19 +478,23 @@ stages:
   - stage: Summary
     condition: succeededOrFailed()
     dependsOn:
+      - Sanity_datatagging
       - Sanity_devel
       - Sanity_2_18
       - Sanity_2_17
       - Sanity_2_16
+      - Units_datatagging
       - Units_devel
       - Units_2_18
       - Units_2_17
       - Units_2_16
       - Remote_devel_extra_vms
+      - Remote_datatagging
       - Remote_devel
       - Remote_2_18
       - Remote_2_17
       - Remote_2_16
+      - Docker_datatagging
       - Docker_devel
       - Docker_2_18
       - Docker_2_17

.github/workflows/ansible-test.yml

@@ -76,7 +76,7 @@ jobs:
           pre-test-cmd: >-
             mkdir -p ../../ansible
             ;
-            git clone --depth=1 --single-branch https://github.com/ansible-collections/community.internal_test_tools.git ../../community/internal_test_tools
+            git clone --depth=1 --single-branch --branch ci-data-tagging https://github.com/felixfontein/community.internal_test_tools.git ../../community/internal_test_tools
           pull-request-change-detection: 'true'
           target-python-version: ${{ matrix.python }}
           testing-type: units
@@ -160,7 +160,7 @@ jobs:
             ;
             git clone --depth=1 --single-branch https://github.com/ansible-collections/community.docker.git ../../community/docker
             ;
-            git clone --depth=1 --single-branch https://github.com/ansible-collections/community.internal_test_tools.git ../../community/internal_test_tools
+            git clone --depth=1 --single-branch --branch ci-data-tagging https://github.com/felixfontein/community.internal_test_tools.git ../../community/internal_test_tools
           pull-request-change-detection: 'true'
           target: ${{ matrix.target }}
           target-python-version: ${{ matrix.python }}

changelogs/fragments/9833-data-tagging.yml

@@ -0,0 +1,8 @@
+bugfixes:
+  - "dependent look plugin - make compatible with ansible-core's Data Tagging feature (https://github.com/ansible-collections/community.general/pull/9833)."
+  - "reveal_ansible_type filter plugin and ansible_type test plugin - make compatible with ansible-core's Data Tagging feature (https://github.com/ansible-collections/community.general/pull/9833)."
+  - "diy callback plugin - make compatible with ansible-core's Data Tagging feature (https://github.com/ansible-collections/community.general/pull/9833)."
+known_issues:
+  - "reveal_ansible_type filter plugin and ansible_type test plugin - note that ansible-core's Data Tagging feature implements new aliases,
+     such as ``_AnsibleTaggedStr`` for ``str``, ``_AnsibleTaggedInt`` for ``int``, and ``_AnsibleTaggedFloat`` for ``float``
+     (https://github.com/ansible-collections/community.general/pull/9833)."

plugins/callback/diy.py

@@ -785,6 +785,12 @@
 from ansible.plugins.callback.default import CallbackModule as Default
 from ansible.module_utils.common.text.converters import to_text
 
+try:
+    from ansible.template import trust_as_template  # noqa: F401, pylint: disable=unused-import
+    SUPPORTS_DATA_TAGGING = True
+except ImportError:
+    SUPPORTS_DATA_TAGGING = False
+
 
 class DummyStdout(object):
     def flush(self):
@@ -838,7 +844,10 @@ def _get_output_specification(self, loader, variables):
         return _ret
 
     def _using_diy(self, spec):
-        return (spec['msg'] is not None) and (spec['msg'] != spec['vars']['omit'])
+        sentinel = object()
+        omit = spec['vars'].get('omit', sentinel)
+        # With Data Tagging, omit is sentinel
+        return (spec['msg'] is not None) and (spec['msg'] != omit or omit is sentinel)
 
     def _parent_has_callback(self):
         return hasattr(super(CallbackModule, self), sys._getframe(1).f_code.co_name)
@@ -894,7 +903,7 @@ def __deepcopy__(self, memo):
             )
         _ret.update(_all)
 
-        _ret.update(_ret.get(self.DIY_NS, {self.DIY_NS: CallbackDIYDict()}))
+        _ret.update(_ret.get(self.DIY_NS, {self.DIY_NS: {} if SUPPORTS_DATA_TAGGING else CallbackDIYDict()}))
 
         _ret[self.DIY_NS].update({'playbook': {}})
         _playbook_attributes = ['entries', 'file_name', 'basedir']

plugins/filter/reveal_ansible_type.py

@@ -23,29 +23,29 @@
 """
 
 EXAMPLES = r"""
-# Substitution converts str to AnsibleUnicode
-# -------------------------------------------
+# Substitution converts str to AnsibleUnicode or _AnsibleTaggedStr
+# ----------------------------------------------------------------
 
-# String. AnsibleUnicode.
+# String. AnsibleUnicode or _AnsibleTaggedStr.
 - data: "abc"
   result: '{{ data | community.general.reveal_ansible_type }}'
-# result => AnsibleUnicode
+# result => AnsibleUnicode (or _AnsibleTaggedStr)
 
-# String. AnsibleUnicode alias str.
-- alias: {"AnsibleUnicode": "str"}
+# String. AnsibleUnicode/_AnsibleTaggedStr alias str.
+- alias: {"AnsibleUnicode": "str", "_AnsibleTaggedStr": "str"}
   data: "abc"
   result: '{{ data | community.general.reveal_ansible_type(alias) }}'
 # result => str
 
-# List. All items are AnsibleUnicode.
+# List. All items are AnsibleUnicode/_AnsibleTaggedStr.
 - data: ["a", "b", "c"]
   result: '{{ data | community.general.reveal_ansible_type }}'
-# result => list[AnsibleUnicode]
+# result => list[AnsibleUnicode] or list[_AnsibleTaggedStr]
 
-# Dictionary. All keys are AnsibleUnicode. All values are AnsibleUnicode.
+# Dictionary. All keys and values are AnsibleUnicode/_AnsibleTaggedStr.
 - data: {"a": "foo", "b": "bar", "c": "baz"}
   result: '{{ data | community.general.reveal_ansible_type }}'
-# result => dict[AnsibleUnicode, AnsibleUnicode]
+# result => dict[AnsibleUnicode, AnsibleUnicode] or dict[_AnsibleTaggedStr, _AnsibleTaggedStr]
 
 # No substitution and no alias. Type of strings is str
 # ----------------------------------------------------
@@ -82,29 +82,43 @@
 - result: '{{ {"a": 1, "b": 2} | community.general.reveal_ansible_type }}'
 # result => dict[str, int]
 
-# Type of strings is AnsibleUnicode or str
-# ----------------------------------------
+# Type of strings is AnsibleUnicode, _AnsibleTaggedStr, or str
+# ------------------------------------------------------------
 
 # Dictionary. The keys are integers or strings. All values are strings.
-- alias: {"AnsibleUnicode": "str"}
+- alias:
+    AnsibleUnicode: str
+    _AnsibleTaggedStr: str
+    _AnsibleTaggedInt: int
   data: {1: 'a', 'b': 'b'}
   result: '{{ data | community.general.reveal_ansible_type(alias) }}'
 # result => dict[int|str, str]
 
 # Dictionary. All keys are integers. All values are keys.
-- alias: {"AnsibleUnicode": "str"}
+- alias:
+    AnsibleUnicode: str
+    _AnsibleTaggedStr: str
+    _AnsibleTaggedInt: int
   data: {1: 'a', 2: 'b'}
   result: '{{ data | community.general.reveal_ansible_type(alias) }}'
 # result => dict[int, str]
 
 # Dictionary. All keys are strings. Multiple types values.
-- alias: {"AnsibleUnicode": "str"}
+- alias:
+    AnsibleUnicode: str
+    _AnsibleTaggedStr: str
+    _AnsibleTaggedInt: int
+    _AnsibleTaggedFloat: float
   data: {'a': 1, 'b': 1.1, 'c': 'abc', 'd': true, 'e': ['x', 'y', 'z'], 'f': {'x': 1, 'y': 2}}
   result: '{{ data | community.general.reveal_ansible_type(alias) }}'
 # result => dict[str, bool|dict|float|int|list|str]
 
 # List. Multiple types items.
-- alias: {"AnsibleUnicode": "str"}
+- alias:
+    AnsibleUnicode: str
+    _AnsibleTaggedStr: str
+    _AnsibleTaggedInt: int
+    _AnsibleTaggedFloat: float
   data: [1, 2, 1.1, 'abc', true, ['x', 'y', 'z'], {'x': 1, 'y': 2}]
   result: '{{ data | community.general.reveal_ansible_type(alias) }}'
 # result => list[bool|dict|float|int|list|str]
@@ -122,6 +136,7 @@
 def reveal_ansible_type(data, alias=None):
     """Returns data type"""
 
+    # TODO: expose use_native_type parameter
     return _ansible_type(data, alias)
 
 

plugins/lookup/dependent.py

@@ -130,12 +130,24 @@
 
 from ansible_collections.community.general.plugins.module_utils.version import LooseVersion
 
+try:
+    from ansible.template import trust_as_template as _trust_as_template
+    HAS_DATATAGGING = True
+except ImportError:
+    HAS_DATATAGGING = False
+
 
 # Whether Templar has a cache, which can be controlled by Templar.template()'s cache option.
 # The cache was removed for ansible-core 2.14 (https://github.com/ansible/ansible/pull/78419)
 _TEMPLAR_HAS_TEMPLATE_CACHE = LooseVersion(ansible_version) < LooseVersion('2.14.0')
 
 
+def _make_safe(value):
+    if HAS_DATATAGGING and isinstance(value, str):
+        return _trust_as_template(value)
+    return value
+
+
 class LookupModule(LookupBase):
     def __evaluate(self, expression, templar, variables):
         """Evaluate expression with templar.
@@ -144,10 +156,13 @@ def __evaluate(self, expression, templar, variables):
         ``variables`` are the variables to use.
         """
         templar.available_variables = variables or {}
-        expression = "{0}{1}{2}".format("{{", expression, "}}")
+        quoted_expression = "{0}{1}{2}".format("{{", expression, "}}")
         if _TEMPLAR_HAS_TEMPLATE_CACHE:
-            return templar.template(expression, cache=False)
-        return templar.template(expression)
+            return templar.template(quoted_expression, cache=False)
+        if hasattr(templar, 'evaluate_expression'):
+            # This is available since the Data Tagging PR has been merged
+            return templar.evaluate_expression(_make_safe(expression))
+        return templar.template(quoted_expression)
 
     def __process(self, result, terms, index, current, templar, variables):
         """Fills ``result`` list with evaluated items.

plugins/plugin_utils/ansible_type.py

@@ -8,17 +8,31 @@
 from ansible.errors import AnsibleFilterError
 from ansible.module_utils.common._collections_compat import Mapping
 
+try:
+    # Introduced with Data Tagging (https://github.com/ansible/ansible/pull/84621):
+    from ansible.module_utils.datatag import native_type_name as _native_type_name
+except ImportError:
+    _native_type_name = None
 
-def _atype(data, alias):
+
+def _atype(data, alias, *, use_native_type: bool = False):
     """
     Returns the name of the type class.
     """
 
-    data_type = type(data).__name__
+    if use_native_type and _native_type_name:
+        data_type = _native_type_name(data)
+    else:
+        data_type = type(data).__name__
+    # The following types were introduced with Data Tagging (https://github.com/ansible/ansible/pull/84621):
+    if data_type == "_AnsibleLazyTemplateDict":
+        data_type = "dict"
+    elif data_type == "_AnsibleLazyTemplateList":
+        data_type = "list"
     return alias.get(data_type, data_type)
 
 
-def _ansible_type(data, alias):
+def _ansible_type(data, alias, *, use_native_type: bool = False):
     """
     Returns the Ansible data type.
     """
@@ -30,16 +44,16 @@ def _ansible_type(data, alias):
         msg = "The argument alias must be a dictionary. %s is %s"
         raise AnsibleFilterError(msg % (alias, type(alias)))
 
-    data_type = _atype(data, alias)
+    data_type = _atype(data, alias, use_native_type=use_native_type)
 
     if data_type == 'list' and len(data) > 0:
-        items = [_atype(i, alias) for i in data]
+        items = [_atype(i, alias, use_native_type=use_native_type) for i in data]
         items_type = '|'.join(sorted(set(items)))
         return ''.join((data_type, '[', items_type, ']'))
 
     if data_type == 'dict' and len(data) > 0:
-        keys = [_atype(i, alias) for i in data.keys()]
-        vals = [_atype(i, alias) for i in data.values()]
+        keys = [_atype(i, alias, use_native_type=use_native_type) for i in data.keys()]
+        vals = [_atype(i, alias, use_native_type=use_native_type) for i in data.values()]
         keys_type = '|'.join(sorted(set(keys)))
         vals_type = '|'.join(sorted(set(vals)))
         return ''.join((data_type, '[', keys_type, ', ', vals_type, ']'))