Update charts and configurations for OTel-Arrow v0.25 with OTel-Contrib v0.107 components

Author: jmacd

arrow/config/gateway-collector.yaml

@@ -8,7 +8,7 @@
 #   resources:
 #     requests:
 #       cpu: 2
-#       memory: 3Gi
+#       memory: 4Gi
 #     limits:
 #       cpu: 2
 #       memory: 8Gi
@@ -17,13 +17,15 @@
 #
 #   resources:
 #     requests:
-#       cpu: 8
-#       memory: 6Gi
+#       cpu: 32
+#       memory: 48Gi
 #     limits:
-#       cpu: 8
-#       memory: 24Gi
+#       cpu: 32
+#       memory: 64Gi
 #
 # In the larger configuration tested, we used `max_in_flight_size_mib: 256`.
+# In the smaller configuration, relatively more memory is required due to
+# garbage collection costs.
 
 receivers:
   # otelarrow is the OpenTelemetry Protocol with Apache Arrow receiver
@@ -60,20 +62,16 @@ processors:
     send_batch_size: 1000
     timeout: 1s
 
-    # For larger configurations, consider raising this parameter.
-    max_in_flight_size_mib: 128
+    # Use max_in_flight_size_mib=64 in a 2 cpu configuration;
+    # Use max_in_flight_size_mib=1024 in a 16 cpu configuration.
+    max_in_flight_size_mib: 64
 
 exporters:
   otelarrow:
     endpoint: "ingest.lightstep.com:443"
     headers:
       "lightstep-access-token": "${LS_TOKEN}"
 
-    arrow:
-      disabled: false
-      max_stream_lifetime: 2m
-      num_streams: 6
-
     # The pipeline will continue trying requests until they timeout.
     # Timeout and retry settings are independent.  If retry_on_failure
     # is also enabled, each (retried) request will also have this
@@ -86,11 +84,33 @@ exporters:
     retry_on_failure:
       enabled: false
 
-    # Do not enable the sending queue.  The concurrent batch processor
-    # is a better way to parallelize the export.
+    # Do not enable the sending queue.  The concurrent batch processor,
+    # configured here, a better way to parallelize the export.
     sending_queue:
       enabled: false
 
+    # OTel-Arrow notes: these settings are specific to OTel-Arrow
+    # To discard this configuration and restore the standard OTLP
+    # exporter, simply drop the arrow section below and replace
+    # "otelarrow" by "otlp" as the component name above and in
+    # service::pipelines::traces::exporters.
+    arrow:
+      # This prevents the OTel-Arrow exporter from falling back to
+      # standard OTLP in case of misconfiguration.
+      disable_downgrade: true
+      
+      # We recommend a small number of streams, since they consume
+      # substantial resources.  More than one stream is recommended
+      # to help balance load.
+      #
+      # Use num_streams=2 in a 2 cpu configuration;
+      # Use num_streams=12 in a 16 cpu configuration.
+      num_streams: 2
+
+      # A stream lifetime limit is required to avoid spurious
+      # disconnect error messages in the collector logs.
+      max_stream_lifetime: 4m
+
 service:
   pipelines:
     traces:

arrow/otelcolarrow-build.yaml

@@ -37,44 +37,42 @@ dist:
   #   description: My Organization telemetry collector
   #
   name: otelarrowcol
-  version: 0.24.0
+  version: 0.25.0
   description: ServiceNow Cloud Observability OpenTelemetry Protocol with Apache Arrow gateway collector
 
   # This indicates which version of the core collector components as
   # well as the builder version.  At the time of this writing, we
   # recommend using at least the current release.
-  otelcol_version: 0.103.0
+  otelcol_version: 0.107.0
 
 # We recommend building in both the OTLP exporter and the OTel-Arrow
 # exporter.
 exporters:
   # This is the core OpenTelemetry Protocol with Apache Arrow exporter,
   # recommended for exporting to ServiceNow Cloud Observability using
   # either OTel-Arrow or standard OTLP over gRPC.
-  - gomod: github.com/open-telemetry/otel-arrow/collector v0.24.0
-    import: github.com/open-telemetry/otel-arrow/collector/exporter/otelarrowexporter
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/otelarrowexporter v0.107.0
 
   # The following components may be useful for debugging.
 
   # As an alternate to the OTel-Arrow exporter, we recommend building
-  # with the core OTLP exporter as a fallback.  These components use
-  # compatible configuration.
-  - gomod: go.opentelemetry.io/collector/exporter/otlpexporter v0.103.0
+  # with the core OTLP exporter as a fallback.  This component and the
+  # OTel-Arrow exporter use compatible configuration.
+  - gomod: go.opentelemetry.io/collector/exporter/otlpexporter v0.107.0
 
   # In case OTLP/HTTP export is required, as opposed to the two
   # gRPC options above.
-  - gomod: go.opentelemetry.io/collector/exporter/otlphttpexporter v0.103.0
+  - gomod: go.opentelemetry.io/collector/exporter/otlphttpexporter v0.107.0
 
   # In case you want to record an OTLP telemetry session to a JSON
   # file, we recommend this additional utility.  See also the
   # obfuscation processor, listed below.  The OTel-Arrow provides
   # offline tools that can help explain poor compression performance
   # using inputs generated by this exporter, for example.
-  - gomod: github.com/open-telemetry/otel-arrow/collector v0.24.0
-    import: github.com/open-telemetry/otel-arrow/collector/exporter/fileexporter
+  - gomod: github.com/open-telemetry/otel-arrow/collector/exporter/fileexporter v0.25.0
 
   # The debug exporter, useful for printing telemetry to the console.
-  - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.103.0
+  - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.107.0
 
 receivers:
   # The OTel-Arrow receiver supports multiple protocols including OTel-Arrow, OTLP gRPC,
@@ -84,60 +82,57 @@ receivers:
   # This enables other OTel Collectors or experimental SDKs that support OTel-Arrow to
   # send to this collector, such as this one for the OTel-Go Trace SDK:
   # https://github.com/lightstep/otel-launcher-go/tree/main/lightstep/sdk/trace/exporters/otlp/otelcol
-  - gomod: github.com/open-telemetry/otel-arrow/collector v0.24.0
-    import: github.com/open-telemetry/otel-arrow/collector/receiver/otelarrowreceiver
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/otelarrowreceiver v0.107.0
 
   # To support receiving OTLP/HTTP.
-  - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.103.0
+  - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.107.0
 
   # You may wish to enable other receivers from the Collector-Contrib repository or
   # elsewhere.  Here are some that might be useful:
   #
-  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.103.0
-  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver v0.103.0
-  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver v0.103.0
-  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8sclusterreceiver v0.103.0
+  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.107.0
+  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver v0.107.0
+  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver v0.107.0
+  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8sclusterreceiver v0.107.0
 
 processors:
   # We recommend use of the OTel-Arrow concurrent batch processor.
-  - gomod: github.com/open-telemetry/otel-arrow/collector v0.24.0
-    import: github.com/open-telemetry/otel-arrow/collector/processor/concurrentbatchprocessor
+  - gomod: github.com/open-telemetry/otel-arrow/collector/processor/concurrentbatchprocessor v0.25.0
 
   # We recommend building with the follow processor for obfuscation, in case you want to
   # record telemetry sessions for offline analysis.
-  - gomod: github.com/open-telemetry/otel-arrow/collector v0.24.0
-    import: github.com/open-telemetry/otel-arrow/collector/processor/obfuscationprocessor
+  - gomod: github.com/open-telemetry/otel-arrow/collector/processor/obfuscationprocessor v0.25.0
 
   # We emphatically DO NOT recommend use of the memory limiter
   # processor or the core batch processor, i.e., do not build with
   # either of:
   # - go.opentelemetry.io/collector/processor/memorylimiterprocessor.
-  # - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.103.0
+  # - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.107.0
 
   # These Collector-Contrib components are referred to in the charts in this package.
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.103.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.103.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.103.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor v0.103.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/remotetapprocessor v0.103.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.103.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.107.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.107.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.107.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor v0.107.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/remotetapprocessor v0.107.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.107.0
 
   # ServiceNow does not at this time recommend the probabilistic sampler processor,
   # because it leads to undercounting of spans.  We will update this recommendation
   # when the sampler is fully supported.
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.103.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.107.0
 
   # You may be interested in other transform components in the
   # Collector-Contrib repository.  Here are some that might be useful:
   #
-  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.103.0
+  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.107.0
 
 extensions:
   # You may be interested in externsions from the Collector-Contrib repository.  None are
   # required.  Here are some that might be useful:
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.103.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension v0.103.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/opampextension v0.103.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.107.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension v0.107.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/opampextension v0.107.0
 
   # We do not support use of headersetterextension to apply the lightstep-access-token
   # header on a per-request basis.  Contact ServiceNow if you are interested in

charts/collector-k8s/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: otelcollector
 description: Deprecated chart for using the OpenTelemetry Collector to scape static or dynamic metric targets.
 type: application
-version: 0.3.2
-appVersion: 0.91.1
+version: 0.3.3
+appVersion: 0.106.1
 deprecated: true
 dependencies: []
 # cert manager must be manually installed because it has CRDs

charts/kube-otel-stack/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: kube-otel-stack
 description: Chart for sending Kubernetes metrics to Lightstep using the OpenTelemetry Operator.
 type: application
-version: 0.8.0
-appVersion: 0.92.0
+version: 0.9.0
+appVersion: 0.106.1
 dependencies:
   # cert manager must be manually installed because it has CRDs
   # https://github.com/kubernetes-sigs/security-profiles-operator/issues/1062