feat: directory support, map[string]yaml var type, migration (#1685)

* feat: add map[string]yaml var type

* feat: extend filesystem (#1699)

* fix: use `relative_path` for file entry info

* docs: type

* feat: extend filesystem to fully support directories

* feat: a specific dir for each filesystem entity (files, dirs)

* test: templating

* test: move test-only impls to test module

* test: templating

* test: proper yaml rendering

* docs: detail how this module works

* feat: SafePath -> PathBuf

* feat(rendering): generate writable file list

* feat: rendering filesystem entry list

* style: rename path field

* test: rendering expected file entries

* test(integration): fix filesystem ops

* style: trace level logs to paths written

* refactor: migrate host agent types to filesystem

* style: use capital letters

* docs: clarify

* chore: remove unused From impl

* chore: remove leftover

* feat: merge files and directories

* refactor: make writes more safe by hiding rendering function

The function `FileSystem::rendered` was public before, which could cause access to the internal `HashMap<PathBuf, String>`. If bound as mutable, it could be modified to write to unintended locations. Now the creation of `RenderedFileSystemEntries` is safer and doesn't provide visibility to the paths, only to write.

* test: fix rendered paths comparison

* fix: correctly handle yaml templating

Author: DavSanchez

agent-control/agent-type-registry/newrelic/com.newrelic.infrastructure-0.1.0.yaml

@@ -5,22 +5,19 @@ variables:
   on_host:
     config_agent:
       description: "Newrelic infra configuration"
-      type: file
+      type: yaml
       required: false
       default: ""
-      file_path: "newrelic-infra.yml"
     config_integrations:
       description: "map of YAML configs for the OHIs"
-      type: map[string]file
+      type: map[string]yaml
       required: false
-      default: { }
-      file_path: "integrations.d"
+      default: {}
     config_logging:
       description: "map of YAML config for logging"
-      type: map[string]file
+      type: map[string]yaml
       required: false
-      default: { }
-      file_path: "logging.d"
+      default: {}
     backoff_delay:
       description: "seconds until next retry if agent fails to start"
       type: string
@@ -48,28 +45,28 @@ variables:
         description: "newrelic-infrastructure chart values"
         type: yaml
         required: false
-        default: { }
+        default: {}
       nri-metadata-injection:
         description: "nri-metadata-injection chart values"
         type: yaml
         required: false
-        default: { }
+        default: {}
       kube-state-metrics:
         description: "kube-state-metrics chart values"
         type: yaml
         required: false
-        default: { }
+        default: {}
       nri-kube-events:
         description: "nri-kube-events chart values"
         type: yaml
         required: false
-        default: { }
+        default: {}
       # Global values must have their own key, even if there is only one sub-chart.
       global:
         description: "Global chart values"
         type: yaml
         required: false
-        default: { }
+        default: {}
     # HelmChart based sub agents should always have `chart_version`.
     chart_version:
       description: "nri-bundle chart version"
@@ -101,7 +98,7 @@ variables:
         type: yaml
         required: false
         default: null
-    
+
 deployment:
   on_host:
     enable_file_logging: ${nr-var:enable_file_logging}
@@ -116,14 +113,20 @@ deployment:
       path: /usr/bin/newrelic-infra
       args: --version
       regex: \d+\.\d+\.\d+
+    filesystem:
+      config:
+        newrelic-infra.yaml: |
+          ${nr-var:config_agent}
+      integrations.d: ${nr-var:config_integrations}
+      logging.d: ${nr-var:config_logging}
     executables:
       - id: newrelic-infra
         path: /usr/bin/newrelic-infra
         args: >-
-          --config=${nr-var:config_agent}
+          --config=${nr-sub:autogenerated_agent_dir}/config/newrelic-infra.yaml
         env:
-          NRIA_PLUGIN_DIR: "${nr-var:config_integrations}"
-          NRIA_LOGGING_CONFIGS_DIR: "${nr-var:config_logging}"
+          NRIA_PLUGIN_DIR: "${nr-sub:autogenerated_agent_dir}/integrations.d"
+          NRIA_LOGGING_CONFIGS_DIR: "${nr-sub:autogenerated_agent_dir}/logging.d"
           NRIA_STATUS_SERVER_ENABLED: true
           NRIA_STATUS_SERVER_PORT: "${nr-var:health_port}"
           NR_HOST_ID: "${nr-ac:host_id}"
@@ -216,7 +219,7 @@ deployment:
             newrelic-prometheus-agent:
               enabled: false
             newrelic-k8s-metrics-adapter:
-              enabled: false 
+              enabled: false
       # Flux [HelmRelease](https://fluxcd.io/flux/components/helmreleases/) that installs the nri-bundle chart.
       release:
         apiVersion: helm.toolkit.fluxcd.io/v2
@@ -240,7 +243,7 @@ deployment:
               chart: ${nr-var:chart_name}
               # This is where the chart version to be installed is defined. Whenever AC updates this Flux triggers a reconciliation.
               version: ${nr-var:chart_version}
-              # This value gets rendered even if not specified so, even 'ChartVersion' is the default, it is required to specify 
+              # This value gets rendered even if not specified so, even 'ChartVersion' is the default, it is required to specify
               # it to avoid the drift detection on AC supervisor, and trigger re-apply on each interval.
               reconcileStrategy: ChartVersion
               sourceRef:

agent-control/agent-type-registry/newrelic/io.opentelemetry.collector-0.1.0.yaml

@@ -5,10 +5,9 @@ variables:
   on_host:
     config:
       description: "Newrelic otel collector configuration"
-      type: file
+      type: yaml
       required: false
-      default: ""
-      file_path: "config.yaml"
+      default: {}
     backoff_delay:
       description: "seconds until next retry if agent fails to start"
       type: string
@@ -34,12 +33,12 @@ variables:
         description: "Newrelic otel collector chart values"
         type: yaml
         required: false
-        default: { }
+        default: {}
       global:
         description: "Global values for the chart"
         type: yaml
         required: false
-        default: { }
+        default: {}
     chart_version:
       description: "Newrelic otel collector chart version"
       type: string
@@ -81,12 +80,16 @@ deployment:
       path: /usr/bin/nrdot-collector-host
       args: -v
       regex: \d+\.\d+\.\d+
+    filesystem:
+      otel-config:
+        config.yaml: |
+          ${nr-var:config}
     executables:
       - # Important to note the binary name is nrdot-collector-host matching the new nrdot binary
         id: nrdot-collector-host
         path: /usr/bin/nrdot-collector-host
         args: >-
-          --config=${nr-var:config}
+          --config=${nr-sub:autogenerated_agent_dir}/otel-config/config.yaml
           --feature-gates=-pkg.translator.prometheus.NormalizeName
         env:
           # sets the otel-collector "env" source resource detector