handle-large-output-results.yaml

# Output parameters and 'results' are passed between pods via annotations. Annotations have a size
# limit. If your output is larger than 256 kB, you should use an artifact instead of a parameter.
#
# If you're looping over an output parameter using withParam, and your parameter gets too large, you
# can use an output artifact combined with a 'count' and withSequence loop to work around the size
# limit. See (https://stackoverflow.com/q/62537126/684776) for further explanation.
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
  generateName: handle-large-output-results-
spec:
  entrypoint: handle-large-output-results
  templates:
    - name: handle-large-output-results
      steps:
        - - name: get-items
            template: get-items
        - - name: sequence-param
            template: echo
            arguments:
              parameters:
                - name: index
                  value: "{{item}}"
              artifacts:
                - name: items
                  from: "{{steps.get-items.outputs.artifacts.items}}"
            withSequence:
              count: "{{steps.get-items.outputs.parameters.count}}"
    - name: get-items
      container:
        image: alpine:latest
        command: ["/bin/sh", "-c"]
        # This small array is just for an example. Make sure the count matches the actual size of
        # your JSON array.
        args: ["echo '[\"a\", \"b\", \"c\"]' > /tmp/items && echo '3' > /tmp/count"]
      outputs:
        artifacts:
          - name: items
            path: /tmp/items
        parameters:
          - name: count
            valueFrom:
              path: /tmp/count
    - name: echo
      inputs:
        parameters:
          - name: index
        artifacts:
          - name: items
            path: /tmp/items
      container:
        image: stedolan/jq:latest
        command: [sh, -c]
        # jq is just one way to get the appropriate item.
        args: ["cat /tmp/items | jq '.[{{inputs.parameters.index}}]'"]