Skip to content

out_azure_blob: add path templating support comparable to s3 #11178

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nberlee wants to merge 2 commits into
base: master
Choose a base branch
from
Open

out_azure_blob: add path templating support comparable to s3 #11178

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e91b2e9
tests: add coverage for azure blob path templating
nberlee Dec 10, 2025
cf1ef14
out_azure_blob: add path templating support comparable to s3
nberlee Dec 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
793 changes: 687 additions & 106 deletions plugins/out_azure_blob/azure_blob.c
View file
Open in desktop

Large diffs are not rendered by default.

11 changes: 11 additions & 0 deletions plugins/out_azure_blob/azure_blob.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
#include <fluent-bit/flb_upstream.h>
#include <fluent-bit/flb_sds.h>
#include <fluent-bit/flb_sqldb.h>
#include <fluent-bit/flb_time.h>

/* Content-Type */
#define AZURE_BLOB_CT "Content-Type"
Expand Down Expand Up @@ -62,6 +63,7 @@ struct flb_azure_blob {
flb_sds_t shared_key;
flb_sds_t endpoint;
flb_sds_t path;
int path_templating_enabled;
flb_sds_t date_key;
flb_sds_t auth_type;
flb_sds_t sas_token;
Expand Down Expand Up @@ -166,4 +168,13 @@ struct flb_azure_blob {
struct flb_config *config;
};

int azb_resolve_path(struct flb_azure_blob *ctx,
const char *tag,
int tag_len,
const struct flb_time *timestamp,
flb_sds_t *out_path);

const char *azb_commit_prefix_with_fallback(struct flb_azure_blob *ctx,
const char *db_prefix);

#endif
24 changes: 19 additions & 5 deletions plugins/out_azure_blob/azure_blob_appendblob.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,24 +24,38 @@
#include "azure_blob_conf.h"
#include "azure_blob_uri.h"

flb_sds_t azb_append_blob_uri(struct flb_azure_blob *ctx, char *tag)
flb_sds_t azb_append_blob_uri(struct flb_azure_blob *ctx,
const char *path_prefix,
const char *tag)
{
flb_sds_t uri;
const char *effective_path;

uri = azb_uri_container(ctx);
if (!uri) {
return NULL;
}

if (ctx->path) {
flb_sds_printf(&uri, "/%s/%s?comp=appendblock", ctx->path, tag);
effective_path = azb_effective_path(ctx, path_prefix);

if (effective_path && effective_path[0] != '\0') {
if (flb_sds_printf(&uri, "/%s/%s?comp=appendblock", effective_path, tag) == NULL) {
flb_sds_destroy(uri);
return NULL;
}
}
else {
flb_sds_printf(&uri, "/%s?comp=appendblock", tag);
if (flb_sds_printf(&uri, "/%s?comp=appendblock", tag) == NULL) {
flb_sds_destroy(uri);
return NULL;
}
}

if (ctx->atype == AZURE_BLOB_AUTH_SAS && ctx->sas_token) {
flb_sds_printf(&uri, "&%s", ctx->sas_token);
if (flb_sds_printf(&uri, "&%s", ctx->sas_token) == NULL) {
flb_sds_destroy(uri);
return NULL;
}
}

return uri;
Expand Down
4 changes: 3 additions & 1 deletion plugins/out_azure_blob/azure_blob_appendblob.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
#include <fluent-bit/flb_output_plugin.h>
#include "azure_blob.h"

flb_sds_t azb_append_blob_uri(struct flb_azure_blob *ctx, char *tag);
flb_sds_t azb_append_blob_uri(struct flb_azure_blob *ctx,
const char *path_prefix,
const char *tag);

#endif
63 changes: 47 additions & 16 deletions plugins/out_azure_blob/azure_blob_blockblob.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,18 +31,23 @@
#include "azure_blob_uri.h"
#include "azure_blob_http.h"

flb_sds_t azb_block_blob_blocklist_uri(struct flb_azure_blob *ctx, char *name)
flb_sds_t azb_block_blob_blocklist_uri(struct flb_azure_blob *ctx,
const char *path_prefix,
const char *name)
{
flb_sds_t uri;
const char *effective_path;

uri = azb_uri_container(ctx);
if (!uri) {
return NULL;
}

if (ctx->path) {
effective_path = azb_effective_path(ctx, path_prefix);

if (effective_path && effective_path[0] != '\0') {
flb_sds_printf(&uri, "/%s/%s?comp=blocklist",
ctx->path, name);
effective_path, name);
}
else {
flb_sds_printf(&uri, "/%s?comp=blocklist", name);
Expand All @@ -55,13 +60,18 @@ flb_sds_t azb_block_blob_blocklist_uri(struct flb_azure_blob *ctx, char *name)
return uri;
}

flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *name,
char *blockid, uint64_t ms, char *random_str)
flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx,
const char *path_prefix,
const char *name,
const char *blockid,
uint64_t ms,
const char *random_str)
{
int len;
flb_sds_t uri;
char *ext;
char *encoded_blockid;
const char *effective_path;

len = strlen(blockid);
encoded_blockid = azb_uri_encode(blockid, len);
Expand All @@ -82,14 +92,16 @@ flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *name,
ext = "";
}

if (ctx->path) {
effective_path = azb_effective_path(ctx, path_prefix);

if (effective_path && effective_path[0] != '\0') {
if (ms > 0) {
flb_sds_printf(&uri, "/%s/%s.%s.%" PRIu64 "%s?blockid=%s&comp=block",
ctx->path, name, random_str, ms, ext, encoded_blockid);
effective_path, name, random_str, ms, ext, encoded_blockid);
}
else {
flb_sds_printf(&uri, "/%s/%s.%s%s?blockid=%s&comp=block",
ctx->path, name, random_str, ext, encoded_blockid);
effective_path, name, random_str, ext, encoded_blockid);
}
}
else {
Expand All @@ -112,10 +124,18 @@ flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *name,
}

flb_sds_t azb_block_blob_uri_commit(struct flb_azure_blob *ctx,
char *tag, uint64_t ms, char *str)
const char *path_prefix,
const char *tag,
uint64_t ms,
const char *str)
{
char *ext;
flb_sds_t uri;
const char *effective_path;

if (!ctx || !tag || !str) {
return NULL;
}

uri = azb_uri_container(ctx);
if (!uri) {
Expand All @@ -129,9 +149,13 @@ flb_sds_t azb_block_blob_uri_commit(struct flb_azure_blob *ctx,
ext = "";
}

if (ctx->path) {
flb_sds_printf(&uri, "/%s/%s.%s.%" PRIu64 "%s?comp=blocklist", ctx->path, tag, str,
ms, ext);
effective_path = azb_effective_path(ctx, path_prefix);

if (effective_path && effective_path[0] != '\0') {
flb_sds_printf(&uri,
"/%s/%s.%s.%" PRIu64 "%s?comp=blocklist",
effective_path, tag, str,
ms, ext);
}
else {
flb_sds_printf(&uri, "/%s.%s.%" PRIu64 "%s?comp=blocklist", tag, str, ms, ext);
Expand Down Expand Up @@ -331,14 +355,19 @@ int azb_block_blob_put_block_list(struct flb_azure_blob *ctx, flb_sds_t uri, flb
}

/* Commit a single block */
int azb_block_blob_commit_block(struct flb_azure_blob *ctx, char *blockid, char *tag, uint64_t ms, char *str)
int azb_block_blob_commit_block(struct flb_azure_blob *ctx,
const char *path_prefix,
const char *blockid,
const char *tag,
uint64_t ms,
const char *str)
{
int ret;
flb_sds_t uri = NULL;
flb_sds_t payload;

/* Compose commit URI */
uri = azb_block_blob_uri_commit(ctx, tag, ms, str);
uri = azb_block_blob_uri_commit(ctx, path_prefix, tag, ms, str);
if (!uri) {
return FLB_ERROR;
}
Expand Down Expand Up @@ -367,7 +396,9 @@ int azb_block_blob_commit_block(struct flb_azure_blob *ctx, char *blockid, char
return ret;
}

int azb_block_blob_commit_file_parts(struct flb_azure_blob *ctx, uint64_t file_id, cfl_sds_t path, cfl_sds_t part_ids)
int azb_block_blob_commit_file_parts(struct flb_azure_blob *ctx, uint64_t file_id,
cfl_sds_t path, cfl_sds_t part_ids,
const char *path_prefix)
{
int ret;
uint64_t id;
Expand Down Expand Up @@ -419,7 +450,7 @@ int azb_block_blob_commit_file_parts(struct flb_azure_blob *ctx, uint64_t file_i
cfl_sds_cat_safe(&payload, "</BlockList>", 12);
flb_utils_split_free(list);

uri = azb_block_blob_blocklist_uri(ctx, path);
uri = azb_block_blob_blocklist_uri(ctx, path_prefix, path);
if (!uri) {
flb_sds_destroy(payload);
return -1;
Expand Down
27 changes: 22 additions & 5 deletions plugins/out_azure_blob/azure_blob_blockblob.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,31 @@
#include <fluent-bit/flb_output_plugin.h>
#include "azure_blob.h"

flb_sds_t azb_block_blob_blocklist_uri(struct flb_azure_blob *ctx, char *name);
flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *tag, char *blockid,
uint64_t ms, char *random_str);
flb_sds_t azb_block_blob_blocklist_uri(struct flb_azure_blob *ctx,
const char *path_prefix,
const char *name);
flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx,
const char *path_prefix,
const char *name,
const char *blockid,
uint64_t ms,
const char *random_str);
flb_sds_t azb_block_blob_uri_commit(struct flb_azure_blob *ctx,
const char *path_prefix,
const char *tag,
uint64_t ms,
const char *str);
char *azb_block_blob_id_logs(uint64_t *ms);
char *azb_block_blob_id_blob(struct flb_azure_blob *ctx, char *path, uint64_t part_id);

int azb_block_blob_commit_block(struct flb_azure_blob *ctx, char *blockid, char *tag, uint64_t ms, char *str);
int azb_block_blob_commit_block(struct flb_azure_blob *ctx,
const char *path_prefix,
const char *blockid,
const char *tag,
uint64_t ms,
const char *str);
int azb_block_blob_commit_file_parts(struct flb_azure_blob *ctx, uint64_t file_id,
cfl_sds_t path, cfl_sds_t part_ids);
cfl_sds_t path, cfl_sds_t part_ids,
const char *path_prefix);

#endif
24 changes: 17 additions & 7 deletions plugins/out_azure_blob/azure_blob_conf.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -582,6 +582,22 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in
}
}

/* Sanitize path and mark templating enabled after remote overrides */
if (ctx->path) {
size_t path_len;

path_len = flb_sds_len(ctx->path);
if (path_len > 0 && ctx->path[path_len - 1] == '/') {
ctx->path[path_len - 1] = '\0';
flb_sds_len_set(ctx->path, path_len - 1);
path_len--;
}

if (path_len > 0) {
ctx->path_templating_enabled = FLB_TRUE;
}
}

if (!ctx->container_name) {
flb_plg_error(ctx->ins, "'container_name' has not been set");
return NULL;
Expand Down Expand Up @@ -755,13 +771,6 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in
flb_sds_printf(&ctx->shared_key_prefix, "SharedKey %s:", ctx->account_name);
}

/* Sanitize path: remove any ending slash */
if (ctx->path) {
if (ctx->path[flb_sds_len(ctx->path) - 1] == '/') {
ctx->path[flb_sds_len(ctx->path) - 1] = '\0';
}
}

/* database file for blob signal handling */
if (ctx->database_file) {
ctx->db = azb_db_open(ctx, ctx->database_file);
Expand Down Expand Up @@ -805,6 +814,7 @@ void flb_azure_blob_conf_destroy(struct flb_azure_blob *ctx)
flb_sds_destroy(ctx->path);
ctx->path = NULL;
}
ctx->path_templating_enabled = FLB_FALSE;

if (ctx->decoded_sk) {
flb_free(ctx->decoded_sk);
Expand Down
Loading