Skip to content

Commit 50777fd

Browse files
nberleenberlee
committed
out_azure_blob: add path templating support comparable to s3
fixes #7515 Signed-off-by: Nico Berlee <[email protected]>
1 parent b07b11f commit 50777fd

File tree

11 files changed

+870
-125
lines changed

11 files changed

+870
-125
lines changed

‎plugins/out_azure_blob/azure_blob.c‎

Lines changed: 614 additions & 77 deletions
Large diffs are not rendered by default.

‎plugins/out_azure_blob/azure_blob.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include <fluent-bit/flb_upstream.h>
2525
#include <fluent-bit/flb_sds.h>
2626
#include <fluent-bit/flb_sqldb.h>
27+
#include <fluent-bit/flb_time.h>
2728

2829
/* Content-Type */
2930
#define AZURE_BLOB_CT "Content-Type"
@@ -62,6 +63,7 @@ struct flb_azure_blob {
6263
flb_sds_t shared_key;
6364
flb_sds_t endpoint;
6465
flb_sds_t path;
66+
int path_templating_enabled;
6567
flb_sds_t date_key;
6668
flb_sds_t auth_type;
6769
flb_sds_t sas_token;
@@ -166,4 +168,10 @@ struct flb_azure_blob {
166168
struct flb_config *config;
167169
};
168170

171+
int azb_resolve_path(struct flb_azure_blob *ctx,
172+
const char *tag,
173+
int tag_len,
174+
const struct flb_time *timestamp,
175+
flb_sds_t *out_path);
176+
169177
#endif

‎plugins/out_azure_blob/azure_blob_appendblob.c‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,24 +24,38 @@
2424
#include "azure_blob_conf.h"
2525
#include "azure_blob_uri.h"
2626

27-
flb_sds_t azb_append_blob_uri(struct flb_azure_blob *ctx, char *tag)
27+
flb_sds_t azb_append_blob_uri(struct flb_azure_blob *ctx,
28+
const char *path_prefix,
29+
const char *tag)
2830
{
2931
flb_sds_t uri;
32+
const char *effective_path;
3033

3134
uri = azb_uri_container(ctx);
3235
if (!uri) {
3336
return NULL;
3437
}
3538

36-
if (ctx->path) {
37-
flb_sds_printf(&uri, "/%s/%s?comp=appendblock", ctx->path, tag);
39+
effective_path = azb_effective_path(ctx, path_prefix);
40+
41+
if (effective_path && effective_path[0] != '\0') {
42+
if (flb_sds_printf(&uri, "/%s/%s?comp=appendblock", effective_path, tag) == -1) {
43+
flb_sds_destroy(uri);
44+
return NULL;
45+
}
3846
}
3947
else {
40-
flb_sds_printf(&uri, "/%s?comp=appendblock", tag);
48+
if (flb_sds_printf(&uri, "/%s?comp=appendblock", tag) == -1) {
49+
flb_sds_destroy(uri);
50+
return NULL;
51+
}
4152
}
4253

4354
if (ctx->atype == AZURE_BLOB_AUTH_SAS && ctx->sas_token) {
44-
flb_sds_printf(&uri, "&%s", ctx->sas_token);
55+
if (flb_sds_printf(&uri, "&%s", ctx->sas_token) == -1) {
56+
flb_sds_destroy(uri);
57+
return NULL;
58+
}
4559
}
4660

4761
return uri;

‎plugins/out_azure_blob/azure_blob_appendblob.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
#include <fluent-bit/flb_output_plugin.h>
2424
#include "azure_blob.h"
2525

26-
flb_sds_t azb_append_blob_uri(struct flb_azure_blob *ctx, char *tag);
26+
flb_sds_t azb_append_blob_uri(struct flb_azure_blob *ctx,
27+
const char *path_prefix,
28+
const char *tag);
2729

2830
#endif

‎plugins/out_azure_blob/azure_blob_blockblob.c‎

Lines changed: 43 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -31,18 +31,23 @@
3131
#include "azure_blob_uri.h"
3232
#include "azure_blob_http.h"
3333

34-
flb_sds_t azb_block_blob_blocklist_uri(struct flb_azure_blob *ctx, char *name)
34+
flb_sds_t azb_block_blob_blocklist_uri(struct flb_azure_blob *ctx,
35+
const char *path_prefix,
36+
const char *name)
3537
{
3638
flb_sds_t uri;
39+
const char *effective_path;
3740

3841
uri = azb_uri_container(ctx);
3942
if (!uri) {
4043
return NULL;
4144
}
4245

43-
if (ctx->path) {
46+
effective_path = azb_effective_path(ctx, path_prefix);
47+
48+
if (effective_path && effective_path[0] != '\0') {
4449
flb_sds_printf(&uri, "/%s/%s?comp=blocklist",
45-
ctx->path, name);
50+
effective_path, name);
4651
}
4752
else {
4853
flb_sds_printf(&uri, "/%s?comp=blocklist", name);
@@ -55,13 +60,18 @@ flb_sds_t azb_block_blob_blocklist_uri(struct flb_azure_blob *ctx, char *name)
5560
return uri;
5661
}
5762

58-
flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *name,
59-
char *blockid, uint64_t ms, char *random_str)
63+
flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx,
64+
const char *path_prefix,
65+
const char *name,
66+
const char *blockid,
67+
uint64_t ms,
68+
const char *random_str)
6069
{
6170
int len;
6271
flb_sds_t uri;
6372
char *ext;
6473
char *encoded_blockid;
74+
const char *effective_path;
6575

6676
len = strlen(blockid);
6777
encoded_blockid = azb_uri_encode(blockid, len);
@@ -82,14 +92,16 @@ flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *name,
8292
ext = "";
8393
}
8494

85-
if (ctx->path) {
95+
effective_path = azb_effective_path(ctx, path_prefix);
96+
97+
if (effective_path && effective_path[0] != '\0') {
8698
if (ms > 0) {
8799
flb_sds_printf(&uri, "/%s/%s.%s.%" PRIu64 "%s?blockid=%s&comp=block",
88-
ctx->path, name, random_str, ms, ext, encoded_blockid);
100+
effective_path, name, random_str, ms, ext, encoded_blockid);
89101
}
90102
else {
91103
flb_sds_printf(&uri, "/%s/%s.%s%s?blockid=%s&comp=block",
92-
ctx->path, name, random_str, ext, encoded_blockid);
104+
effective_path, name, random_str, ext, encoded_blockid);
93105
}
94106
}
95107
else {
@@ -112,10 +124,14 @@ flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *name,
112124
}
113125

114126
flb_sds_t azb_block_blob_uri_commit(struct flb_azure_blob *ctx,
115-
char *tag, uint64_t ms, char *str)
127+
const char *path_prefix,
128+
const char *tag,
129+
uint64_t ms,
130+
const char *str)
116131
{
117132
char *ext;
118133
flb_sds_t uri;
134+
const char *effective_path;
119135

120136
uri = azb_uri_container(ctx);
121137
if (!uri) {
@@ -129,9 +145,13 @@ flb_sds_t azb_block_blob_uri_commit(struct flb_azure_blob *ctx,
129145
ext = "";
130146
}
131147

132-
if (ctx->path) {
133-
flb_sds_printf(&uri, "/%s/%s.%s.%" PRIu64 "%s?comp=blocklist", ctx->path, tag, str,
134-
ms, ext);
148+
effective_path = azb_effective_path(ctx, path_prefix);
149+
150+
if (effective_path && effective_path[0] != '\0') {
151+
flb_sds_printf(&uri,
152+
"/%s/%s.%s.%" PRIu64 "%s?comp=blocklist",
153+
effective_path, tag, str,
154+
ms, ext);
135155
}
136156
else {
137157
flb_sds_printf(&uri, "/%s.%s.%" PRIu64 "%s?comp=blocklist", tag, str, ms, ext);
@@ -331,14 +351,19 @@ int azb_block_blob_put_block_list(struct flb_azure_blob *ctx, flb_sds_t uri, flb
331351
}
332352

333353
/* Commit a single block */
334-
int azb_block_blob_commit_block(struct flb_azure_blob *ctx, char *blockid, char *tag, uint64_t ms, char *str)
354+
int azb_block_blob_commit_block(struct flb_azure_blob *ctx,
355+
const char *path_prefix,
356+
const char *blockid,
357+
const char *tag,
358+
uint64_t ms,
359+
const char *str)
335360
{
336361
int ret;
337362
flb_sds_t uri = NULL;
338363
flb_sds_t payload;
339364

340365
/* Compose commit URI */
341-
uri = azb_block_blob_uri_commit(ctx, tag, ms, str);
366+
uri = azb_block_blob_uri_commit(ctx, path_prefix, tag, ms, str);
342367
if (!uri) {
343368
return FLB_ERROR;
344369
}
@@ -367,7 +392,9 @@ int azb_block_blob_commit_block(struct flb_azure_blob *ctx, char *blockid, char
367392
return ret;
368393
}
369394

370-
int azb_block_blob_commit_file_parts(struct flb_azure_blob *ctx, uint64_t file_id, cfl_sds_t path, cfl_sds_t part_ids)
395+
int azb_block_blob_commit_file_parts(struct flb_azure_blob *ctx, uint64_t file_id,
396+
cfl_sds_t path, cfl_sds_t part_ids,
397+
const char *path_prefix)
371398
{
372399
int ret;
373400
uint64_t id;
@@ -419,7 +446,7 @@ int azb_block_blob_commit_file_parts(struct flb_azure_blob *ctx, uint64_t file_i
419446
cfl_sds_cat_safe(&payload, "</BlockList>", 12);
420447
flb_utils_split_free(list);
421448

422-
uri = azb_block_blob_blocklist_uri(ctx, path);
449+
uri = azb_block_blob_blocklist_uri(ctx, path_prefix, path);
423450
if (!uri) {
424451
flb_sds_destroy(payload);
425452
return -1;

‎plugins/out_azure_blob/azure_blob_blockblob.h‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,26 @@
2323
#include <fluent-bit/flb_output_plugin.h>
2424
#include "azure_blob.h"
2525

26-
flb_sds_t azb_block_blob_blocklist_uri(struct flb_azure_blob *ctx, char *name);
27-
flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *tag, char *blockid,
28-
uint64_t ms, char *random_str);
26+
flb_sds_t azb_block_blob_blocklist_uri(struct flb_azure_blob *ctx,
27+
const char *path_prefix,
28+
const char *name);
29+
flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx,
30+
const char *path_prefix,
31+
const char *name,
32+
const char *blockid,
33+
uint64_t ms,
34+
const char *random_str);
2935
char *azb_block_blob_id_logs(uint64_t *ms);
3036
char *azb_block_blob_id_blob(struct flb_azure_blob *ctx, char *path, uint64_t part_id);
3137

32-
int azb_block_blob_commit_block(struct flb_azure_blob *ctx, char *blockid, char *tag, uint64_t ms, char *str);
38+
int azb_block_blob_commit_block(struct flb_azure_blob *ctx,
39+
const char *path_prefix,
40+
const char *blockid,
41+
const char *tag,
42+
uint64_t ms,
43+
const char *str);
3344
int azb_block_blob_commit_file_parts(struct flb_azure_blob *ctx, uint64_t file_id,
34-
cfl_sds_t path, cfl_sds_t part_ids);
45+
cfl_sds_t path, cfl_sds_t part_ids,
46+
const char *path_prefix);
3547

3648
#endif

‎plugins/out_azure_blob/azure_blob_conf.c‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -582,6 +582,22 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in
582582
}
583583
}
584584

585+
/* Sanitize path and mark templating enabled after remote overrides */
586+
if (ctx->path) {
587+
size_t path_len;
588+
589+
path_len = flb_sds_len(ctx->path);
590+
if (path_len > 0 && ctx->path[path_len - 1] == '/') {
591+
ctx->path[path_len - 1] = '\0';
592+
flb_sds_len_set(ctx->path, path_len - 1);
593+
path_len--;
594+
}
595+
596+
if (path_len > 0) {
597+
ctx->path_templating_enabled = FLB_TRUE;
598+
}
599+
}
600+
585601
if (!ctx->container_name) {
586602
flb_plg_error(ctx->ins, "'container_name' has not been set");
587603
return NULL;
@@ -755,13 +771,6 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in
755771
flb_sds_printf(&ctx->shared_key_prefix, "SharedKey %s:", ctx->account_name);
756772
}
757773

758-
/* Sanitize path: remove any ending slash */
759-
if (ctx->path) {
760-
if (ctx->path[flb_sds_len(ctx->path) - 1] == '/') {
761-
ctx->path[flb_sds_len(ctx->path) - 1] = '\0';
762-
}
763-
}
764-
765774
/* database file for blob signal handling */
766775
if (ctx->database_file) {
767776
ctx->db = azb_db_open(ctx, ctx->database_file);
@@ -805,6 +814,7 @@ void flb_azure_blob_conf_destroy(struct flb_azure_blob *ctx)
805814
flb_sds_destroy(ctx->path);
806815
ctx->path = NULL;
807816
}
817+
ctx->path_templating_enabled = FLB_FALSE;
808818

809819
if (ctx->decoded_sk) {
810820
flb_free(ctx->decoded_sk);

0 commit comments

Comments
 (0)