Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 48 additions & 7 deletions plugins/out_s3/s3.c
Original file line number Diff line number Diff line change
Expand Up @@ -812,6 +812,10 @@ static void s3_warn_other_buffer_mode(struct flb_s3 *ctx, char *root)
if (entry->d_name[0] == '.') {
continue;
}
/* $INDEX metadata is shared by both modes; it is not buffered data. */
if (strcmp(entry->d_name, "sequence") == 0) {
continue;
}
is_msgpack = strcmp(entry->d_name, "msgpack") == 0;
if ((ctx->arrow_schema != NULL) == is_msgpack) {
continue;
Expand Down Expand Up @@ -932,16 +936,44 @@ static int init_seq_index(void *context) {

ctx->key_fmt_has_seq_index = FLB_TRUE;

ctx->stream_metadata = flb_fstore_stream_create(ctx->fs, "sequence");
if (!ctx->stream_metadata) {
flb_plg_error(ctx->ins, "could not initialize metadata stream");
flb_fstore_destroy(ctx->fs);
ctx->fs = NULL;
return -1;
/*
* Keep the index under the mode-independent buffer root. With a schema the
* file store is rooted at <root>/msgpack; putting the index there would
* restart it at zero whenever parquet.schema_file is toggled, and with
* $INDEX keys (no random suffix) that can overwrite existing S3 objects.
*/
if (ctx->arrow_schema == NULL) {
ctx->stream_metadata = flb_fstore_stream_create(ctx->fs, "sequence");
if (!ctx->stream_metadata) {
flb_plg_error(ctx->ins, "could not initialize metadata stream");
flb_fstore_destroy(ctx->fs);
ctx->fs = NULL;
return -1;
}
ctx->metadata_dir = flb_sds_create(ctx->stream_metadata->path);
}
else {
tmp = "/sequence";
ctx->metadata_dir = flb_sds_create(ctx->buffer_root);
if (ctx->metadata_dir != NULL &&
flb_sds_cat_safe(&ctx->metadata_dir, tmp, strlen(tmp)) < 0) {
flb_sds_destroy(ctx->metadata_dir);
ctx->metadata_dir = NULL;
}
if (ctx->metadata_dir != NULL) {
#ifdef FLB_SYSTEM_WINDOWS
ret = mkdir(ctx->metadata_dir);
#else
ret = mkdir(ctx->metadata_dir, 0700);
#endif
if (ret < 0 && errno != EEXIST) {
flb_plg_error(ctx->ins, "Failed to create metadata directory");
return -1;
}
}
}

/* Construct directories and file path names */
ctx->metadata_dir = flb_sds_create(ctx->stream_metadata->path);
if (ctx->metadata_dir == NULL) {
flb_plg_error(ctx->ins, "Failed to create metadata path");
flb_errno();
Expand Down Expand Up @@ -1086,6 +1118,10 @@ static void s3_context_destroy(struct flb_s3 *ctx)
flb_sds_destroy(ctx->buffer_dir);
}

if (ctx->buffer_root) {
flb_sds_destroy(ctx->buffer_root);
}

if (ctx->metadata_dir) {
flb_sds_destroy(ctx->metadata_dir);
}
Expand Down Expand Up @@ -1344,6 +1380,11 @@ static int s3_init(struct flb_output_instance *ins,
}
s3_warn_other_buffer_mode(ctx, tmp_sds);
ctx->buffer_dir = tmp_sds;
ctx->buffer_root = flb_sds_create(tmp_sds);
if (ctx->buffer_root == NULL) {
flb_errno();
return -1;
}
if (ctx->arrow_schema != NULL) {
tmp_sds = flb_sds_cat(ctx->buffer_dir, "/msgpack", 8);
if (tmp_sds == NULL) {
Expand Down
1 change: 1 addition & 0 deletions plugins/out_s3/s3.h
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ struct flb_s3 {
flb_sds_t date_key;

flb_sds_t buffer_dir;
flb_sds_t buffer_root; /* bucket buffer path, same in every buffer mode */

char *store_dir;
struct flb_fstore *fs;
Expand Down
13 changes: 7 additions & 6 deletions src/aws/compression/arrow/compress.c
Original file line number Diff line number Diff line change
Expand Up @@ -268,11 +268,12 @@ static GArrowResizableBuffer* table_to_parquet_buffer(GArrowTable *table,
}
#endif

/* data is JSON lines when schema is NULL, otherwise Fluent Bit MessagePack events. */
static int compress_columnar(int columnar_format,
void *json, size_t size,
void **out_buf, size_t *out_size,
int compression_type,
struct flb_arrow_schema *schema)
void *data, size_t size,
void **out_buf, size_t *out_size,
int compression_type,
struct flb_arrow_schema *schema)
{
GArrowTable *table;
GArrowResizableBuffer *buffer;
Expand All @@ -290,10 +291,10 @@ static int compress_columnar(int columnar_format,
}

if (schema != NULL) {
table = flb_arrow_schema_table(schema, json, size);
table = flb_arrow_schema_table(schema, data, size);
}
else {
table = parse_json((uint8_t *) json, size);
table = parse_json((uint8_t *) data, size);
}
if (table == NULL) {
flb_error("[aws][compress] Failed to build Arrow Table");
Expand Down
38 changes: 38 additions & 0 deletions tests/integration/scenarios/out_s3/tests/test_out_s3_001.py
Original file line number Diff line number Diff line change
Expand Up @@ -883,3 +883,41 @@ def test_out_s3_schema_accepts_arrow_metadata(tmp_path, metadata):
assert table.to_pylist() == [{"x": True}]
assert not table.schema.metadata
assert not table.schema.field("x").metadata


# Schema-less restarts also reset $INDEX today (empty-stream cleanup at
# shutdown removes the sequence directory); that is tracked separately, so
# every sequence here starts with the schema enabled.
@pytest.mark.parametrize("modes", [(True, True), (True, False)])
def test_out_s3_schema_restart_preserves_seq_index(tmp_path, modes):
# $INDEX keys have no random suffix, so a sequence index that restarts at
# zero across schema restarts or a schema toggle would overwrite objects.
fields = [_arrow_field("x", "int", False, bitWidth=32, isSigned=True)]
config_file, config = _schema_config(tmp_path, "arrow", fields, {"x": 42})
output = config["pipeline"]["outputs"][0]
output["s3_key_format"] = "/$TAG/$INDEX"
schema_file = output.pop("parquet.schema_file")

keys = []
for use_schema in modes:
if use_schema:
output["parquet.schema_file"] = schema_file
else:
output.pop("parquet.schema_file", None)
config_file.write_text(yaml.safe_dump(config))
service = Service(str(config_file))
_start_or_skip_unsupported_columnar_format(service, "requires arrow-glib")
try:
request = service.wait_for_request()
log = Path(service.service.flb.log_file).read_text()
finally:
service.stop()
assert request["method"] == "PUT"
# The shared $INDEX metadata directory must not look like pending
# JSON-lines buffers when the schema is enabled.
if use_schema:
assert "JSON-lines buffer directory exists" not in log
keys.append(request["path"])

indexes = [int(key.rsplit("/", 1)[1]) for key in keys]
assert indexes == sorted(set(indexes)), f"S3 keys reused or out of order: {keys}"
Loading