Skip to content

Commit efeca8f

Browse files
jefchienjefchien
authored
Update file_storage to write state to subdirectory (#2184)
1 parent 11fef6f commit efeca8f

11 files changed

Lines changed: 44 additions & 47 deletions

File tree

‎translator/tocwconfig/sampleConfig/complete_linux_config.yaml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -184,15 +184,15 @@ extensions:
184184
entitystore:
185185
mode: ec2
186186
region: us-west-2
187-
file_storage/journald:
187+
file_storage/opentelemetry:
188188
compaction:
189189
check_interval: 5s
190-
directory: /var/lib/otelcol/file_storage
190+
directory: /opt/aws/amazon-cloudwatch-agent/logs/state/otel
191191
max_transaction_size: 65536
192192
rebound_needed_threshold_mib: 100
193193
rebound_trigger_threshold_mib: 10
194194
create_directory: true
195-
directory: /opt/aws/amazon-cloudwatch-agent/logs/state
195+
directory: /opt/aws/amazon-cloudwatch-agent/logs/state/otel
196196
directory_permissions: "0750"
197197
timeout: 1s
198198
processors:
@@ -452,7 +452,7 @@ receivers:
452452
max_elapsed_time: 5m0s
453453
max_interval: 30s
454454
start_at: end
455-
storage: file_storage/journald
455+
storage: file_storage/opentelemetry
456456
type: journald_input
457457
units:
458458
- systemd
@@ -467,7 +467,7 @@ receivers:
467467
max_elapsed_time: 5m0s
468468
max_interval: 30s
469469
start_at: end
470-
storage: file_storage/journald
470+
storage: file_storage/opentelemetry
471471
type: journald_input
472472
units:
473473
- nginx
@@ -558,7 +558,7 @@ service:
558558
- agenthealth/statuscode
559559
- agenthealth/logs
560560
- agenthealth/traces
561-
- file_storage/journald
561+
- file_storage/opentelemetry
562562
- entitystore
563563
pipelines:
564564
logs/emf_logs:

‎translator/tocwconfig/sampleConfig/journaldlogs_filters.yaml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -86,15 +86,15 @@ extensions:
8686
entitystore:
8787
mode: ec2
8888
region: us-west-2
89-
file_storage/journald:
89+
file_storage/opentelemetry:
9090
compaction:
9191
check_interval: 5s
92-
directory: /var/lib/otelcol/file_storage
92+
directory: /opt/aws/amazon-cloudwatch-agent/logs/state/otel
9393
max_transaction_size: 65536
9494
rebound_needed_threshold_mib: 100
9595
rebound_trigger_threshold_mib: 10
9696
create_directory: true
97-
directory: /opt/aws/amazon-cloudwatch-agent/logs/state
97+
directory: /opt/aws/amazon-cloudwatch-agent/logs/state/otel
9898
directory_permissions: "0750"
9999
timeout: 1s
100100
processors:
@@ -136,7 +136,7 @@ receivers:
136136
max_elapsed_time: 5m0s
137137
max_interval: 30s
138138
start_at: end
139-
storage: file_storage/journald
139+
storage: file_storage/opentelemetry
140140
type: journald_input
141141
journald/journald_1:
142142
id: journald_input
@@ -148,13 +148,13 @@ receivers:
148148
max_elapsed_time: 5m0s
149149
max_interval: 30s
150150
start_at: end
151-
storage: file_storage/journald
151+
storage: file_storage/opentelemetry
152152
type: journald_input
153153
service:
154154
extensions:
155155
- agenthealth/logs
156156
- agenthealth/statuscode
157-
- file_storage/journald
157+
- file_storage/opentelemetry
158158
- entitystore
159159
pipelines:
160160
logs/journald/0:

‎translator/tocwconfig/sampleConfig/journaldlogs_matches.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,15 +52,15 @@ extensions:
5252
entitystore:
5353
mode: ec2
5454
region: us-west-2
55-
file_storage/journald:
55+
file_storage/opentelemetry:
5656
compaction:
5757
check_interval: 5s
58-
directory: /var/lib/otelcol/file_storage
58+
directory: /opt/aws/amazon-cloudwatch-agent/logs/state/otel
5959
max_transaction_size: 65536
6060
rebound_needed_threshold_mib: 100
6161
rebound_trigger_threshold_mib: 10
6262
create_directory: true
63-
directory: /opt/aws/amazon-cloudwatch-agent/logs/state
63+
directory: /opt/aws/amazon-cloudwatch-agent/logs/state/otel
6464
directory_permissions: "0750"
6565
timeout: 1s
6666
processors:
@@ -83,13 +83,13 @@ receivers:
8383
max_elapsed_time: 5m0s
8484
max_interval: 30s
8585
start_at: end
86-
storage: file_storage/journald
86+
storage: file_storage/opentelemetry
8787
type: journald_input
8888
service:
8989
extensions:
9090
- agenthealth/logs
9191
- agenthealth/statuscode
92-
- file_storage/journald
92+
- file_storage/opentelemetry
9393
- entitystore
9494
pipelines:
9595
logs/journald/0:

‎translator/tocwconfig/sampleConfig/journaldlogs_priority.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,15 +52,15 @@ extensions:
5252
entitystore:
5353
mode: ec2
5454
region: us-west-2
55-
file_storage/journald:
55+
file_storage/opentelemetry:
5656
compaction:
5757
check_interval: 5s
58-
directory: /var/lib/otelcol/file_storage
58+
directory: /opt/aws/amazon-cloudwatch-agent/logs/state/otel
5959
max_transaction_size: 65536
6060
rebound_needed_threshold_mib: 100
6161
rebound_trigger_threshold_mib: 10
6262
create_directory: true
63-
directory: /opt/aws/amazon-cloudwatch-agent/logs/state
63+
directory: /opt/aws/amazon-cloudwatch-agent/logs/state/otel
6464
directory_permissions: "0750"
6565
timeout: 1s
6666
processors:
@@ -80,13 +80,13 @@ receivers:
8080
max_elapsed_time: 5m0s
8181
max_interval: 30s
8282
start_at: end
83-
storage: file_storage/journald
83+
storage: file_storage/opentelemetry
8484
type: journald_input
8585
service:
8686
extensions:
8787
- agenthealth/logs
8888
- agenthealth/statuscode
89-
- file_storage/journald
89+
- file_storage/opentelemetry
9090
- entitystore
9191
pipelines:
9292
logs/journald/0:

‎translator/tocwconfig/sampleConfig/journaldlogs_units.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,15 +52,15 @@ extensions:
5252
entitystore:
5353
mode: ec2
5454
region: us-west-2
55-
file_storage/journald:
55+
file_storage/opentelemetry:
5656
compaction:
5757
check_interval: 5s
58-
directory: /var/lib/otelcol/file_storage
58+
directory: /opt/aws/amazon-cloudwatch-agent/logs/state/otel
5959
max_transaction_size: 65536
6060
rebound_needed_threshold_mib: 100
6161
rebound_trigger_threshold_mib: 10
6262
create_directory: true
63-
directory: /opt/aws/amazon-cloudwatch-agent/logs/state
63+
directory: /opt/aws/amazon-cloudwatch-agent/logs/state/otel
6464
directory_permissions: "0750"
6565
timeout: 1s
6666
processors:
@@ -80,15 +80,15 @@ receivers:
8080
max_elapsed_time: 5m0s
8181
max_interval: 30s
8282
start_at: end
83-
storage: file_storage/journald
83+
storage: file_storage/opentelemetry
8484
type: journald_input
8585
units:
8686
- sshd
8787
service:
8888
extensions:
8989
- agenthealth/logs
9090
- agenthealth/statuscode
91-
- file_storage/journald
91+
- file_storage/opentelemetry
9292
- entitystore
9393
pipelines:
9494
logs/journald/0:

‎translator/tocwconfig/sampleConfig/journaldlogs_units_and_priority.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,15 +52,15 @@ extensions:
5252
entitystore:
5353
mode: ec2
5454
region: us-west-2
55-
file_storage/journald:
55+
file_storage/opentelemetry:
5656
compaction:
5757
check_interval: 5s
58-
directory: /var/lib/otelcol/file_storage
58+
directory: /opt/aws/amazon-cloudwatch-agent/logs/state/otel
5959
max_transaction_size: 65536
6060
rebound_needed_threshold_mib: 100
6161
rebound_trigger_threshold_mib: 10
6262
create_directory: true
63-
directory: /opt/aws/amazon-cloudwatch-agent/logs/state
63+
directory: /opt/aws/amazon-cloudwatch-agent/logs/state/otel
6464
directory_permissions: "0750"
6565
timeout: 1s
6666
processors:
@@ -80,7 +80,7 @@ receivers:
8080
max_elapsed_time: 5m0s
8181
max_interval: 30s
8282
start_at: end
83-
storage: file_storage/journald
83+
storage: file_storage/opentelemetry
8484
type: journald_input
8585
units:
8686
- sshd
@@ -89,7 +89,7 @@ service:
8989
extensions:
9090
- agenthealth/logs
9191
- agenthealth/statuscode
92-
- file_storage/journald
92+
- file_storage/opentelemetry
9393
- entitystore
9494
pipelines:
9595
logs/journald/0:

‎translator/translate/otel/extension/filestorage/translator.go‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,23 +11,18 @@ import (
1111
"go.opentelemetry.io/collector/confmap"
1212
"go.opentelemetry.io/collector/extension"
1313

14-
"github.com/aws/amazon-cloudwatch-agent/tool/paths"
14+
"github.com/aws/amazon-cloudwatch-agent/translator/translate/logs/util"
1515
"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common"
1616
)
1717

18-
const (
19-
name = "journald"
20-
)
21-
2218
type translator struct {
2319
factory extension.Factory
2420
}
2521

2622
var _ common.ComponentTranslator = (*translator)(nil)
2723

28-
// StorageComponentID returns the component.ID for the file_storage/journald extension.
29-
func StorageComponentID() component.ID {
30-
return component.NewIDWithName(filestorage.NewFactory().Type(), name)
24+
func ComponentID() component.ID {
25+
return component.NewIDWithName(filestorage.NewFactory().Type(), common.OpenTelemetryKey)
3126
}
3227

3328
func NewTranslator() common.ComponentTranslator {
@@ -37,12 +32,14 @@ func NewTranslator() common.ComponentTranslator {
3732
}
3833

3934
func (t *translator) ID() component.ID {
40-
return component.NewIDWithName(t.factory.Type(), name)
35+
return component.NewIDWithName(t.factory.Type(), common.OpenTelemetryKey)
4136
}
4237

4338
func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) {
4439
cfg := t.factory.CreateDefaultConfig().(*filestorage.Config)
45-
cfg.Directory = filepath.Join(paths.AgentDir, "logs", "state")
40+
dir := filepath.Join(util.GetFileStateFolder(), "otel")
41+
cfg.Directory = dir
42+
cfg.Compaction.Directory = dir
4643
cfg.CreateDirectory = true
4744
return cfg, nil
4845
}

‎translator/translate/otel/extension/filestorage/translator_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import (
1616

1717
func TestTranslator(t *testing.T) {
1818
tt := NewTranslator()
19-
assert.Equal(t, "file_storage/journald", tt.ID().String())
19+
assert.Equal(t, "file_storage/opentelemetry", tt.ID().String())
2020

2121
conf := confmap.NewFromStringMap(map[string]interface{}{})
2222
got, err := tt.Translate(conf)
@@ -25,6 +25,7 @@ func TestTranslator(t *testing.T) {
2525

2626
gotCfg, ok := got.(*filestorage.Config)
2727
require.True(t, ok)
28-
assert.Equal(t, "/opt/aws/amazon-cloudwatch-agent/logs/state", gotCfg.Directory)
28+
assert.Equal(t, "/opt/aws/amazon-cloudwatch-agent/logs/state/otel", gotCfg.Directory)
29+
assert.Equal(t, "/opt/aws/amazon-cloudwatch-agent/logs/state/otel", gotCfg.Compaction.Directory)
2930
assert.True(t, gotCfg.CreateDirectory)
3031
}

‎translator/translate/otel/pipeline/journald/translator.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,6 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators
200200
translators.Extensions.Set(agenthealth.NewTranslator(agenthealth.LogsName, []string{agenthealth.OperationPutLogEvents}))
201201
translators.Extensions.Set(agenthealth.NewTranslatorWithStatusCode(agenthealth.StatusCodeName, nil, true))
202202

203-
// Add file storage extension for journald cursor persistence
204203
translators.Extensions.Set(filestorage.NewTranslator())
205204

206205
return &translators, nil

‎translator/translate/otel/receiver/journald/translator.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) {
5757
copy(cfg.InputConfig.Matches, t.matches)
5858
}
5959

60-
storageID := filestorage.StorageComponentID()
60+
storageID := filestorage.ComponentID()
6161
cfg.StorageID = &storageID
6262

6363
return cfg, nil

0 commit comments

Comments
 (0)