|
| 1 | +// Copyright 2021-present StarRocks, Inc. All rights reserved. |
| 2 | +// |
| 3 | +// Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +// you may not use this file except in compliance with the License. |
| 5 | +// You may obtain a copy of the License at |
| 6 | +// |
| 7 | +// https://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +// |
| 9 | +// Unless required by applicable law or agreed to in writing, software |
| 10 | +// distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +// See the License for the specific language governing permissions and |
| 13 | +// limitations under the License. |
| 14 | + |
| 15 | +#include "connector/common/partition_chunk_writer.h" |
| 16 | + |
| 17 | +#include <algorithm> |
| 18 | + |
| 19 | +#include "base/failpoint/fail_point.h" |
| 20 | +#include "column/chunk.h" |
| 21 | +#include "formats/file_writer.h" |
| 22 | +#include "formats/io/async_flush_stream_poller.h" |
| 23 | + |
| 24 | +namespace starrocks::connector { |
| 25 | + |
| 26 | +DEFINE_FAIL_POINT(parquet_chunk_writer_init_failed); |
| 27 | + |
| 28 | +PartitionChunkWriter::PartitionChunkWriter(std::string partition, std::vector<int8_t> partition_field_null_list, |
| 29 | + const std::shared_ptr<PartitionChunkWriterContext>& ctx) |
| 30 | + : _partition(std::move(partition)), |
| 31 | + _partition_field_null_list(std::move(partition_field_null_list)), |
| 32 | + _file_writer_factory(ctx->file_writer_factory), |
| 33 | + _location_provider(ctx->location_provider), |
| 34 | + _max_file_size(ctx->max_file_size), |
| 35 | + _is_default_partition(ctx->is_default_partition) { |
| 36 | + _commit_extra_data.resize(_partition_field_null_list.size(), '0'); |
| 37 | + std::transform(_partition_field_null_list.begin(), _partition_field_null_list.end(), _commit_extra_data.begin(), |
| 38 | + [](int8_t b) { return b + '0'; }); |
| 39 | +} |
| 40 | + |
| 41 | +Status PartitionChunkWriter::create_file_writer_if_needed() { |
| 42 | + if (!_file_writer) { |
| 43 | + std::string path = _is_default_partition ? _location_provider->get() : _location_provider->get(_partition); |
| 44 | + ASSIGN_OR_RETURN(auto new_writer_and_stream, _file_writer_factory->create(path)); |
| 45 | + _file_writer = std::move(new_writer_and_stream.writer); |
| 46 | + _out_stream = std::move(new_writer_and_stream.stream); |
| 47 | + RETURN_IF_ERROR(_file_writer->init()); |
| 48 | + |
| 49 | + FAIL_POINT_TRIGGER_EXECUTE(parquet_chunk_writer_init_failed, |
| 50 | + { return Status::InternalError("Create file writer failed due to fail point"); }); |
| 51 | + _io_poller->enqueue(_out_stream); |
| 52 | + } |
| 53 | + return Status::OK(); |
| 54 | +} |
| 55 | + |
| 56 | +Status PartitionChunkWriter::commit_file() { |
| 57 | + if (!_file_writer) { |
| 58 | + return Status::OK(); |
| 59 | + } |
| 60 | + SCOPED_TIMER(_sink_profile ? _sink_profile->commit_file_timer : nullptr); |
| 61 | + auto file_result = _file_writer->close(); |
| 62 | + const auto io_status = file_result.io_status; |
| 63 | + const auto file_size = file_result.file_statistics.file_size; |
| 64 | + CommitResult result{.file_result = std::move(file_result)}; |
| 65 | + result.set_partition_null_fingerprint(_commit_extra_data); |
| 66 | + _commit_callback(result); |
| 67 | + _file_writer = nullptr; |
| 68 | + VLOG(3) << "commit to remote file, filename: " << _out_stream->filename() << ", size: " << file_size; |
| 69 | + _out_stream = nullptr; |
| 70 | + return io_status; |
| 71 | +} |
| 72 | + |
| 73 | +Status BufferPartitionChunkWriter::init() { |
| 74 | + RETURN_IF_ERROR(create_file_writer_if_needed()); |
| 75 | + return Status::OK(); |
| 76 | +} |
| 77 | + |
| 78 | +Status BufferPartitionChunkWriter::write(const ChunkPtr& chunk) { |
| 79 | + if (_file_writer && _file_writer->get_written_bytes() >= _max_file_size) { |
| 80 | + RETURN_IF_ERROR(commit_file()); |
| 81 | + } |
| 82 | + RETURN_IF_ERROR(create_file_writer_if_needed()); |
| 83 | + SCOPED_TIMER(_sink_profile ? _sink_profile->write_file_timer : nullptr); |
| 84 | + return _file_writer->write(chunk.get()); |
| 85 | +} |
| 86 | + |
| 87 | +Status BufferPartitionChunkWriter::flush() { |
| 88 | + return commit_file(); |
| 89 | +} |
| 90 | + |
| 91 | +Status BufferPartitionChunkWriter::wait_flush() { |
| 92 | + return Status::OK(); |
| 93 | +} |
| 94 | + |
| 95 | +Status BufferPartitionChunkWriter::finish() { |
| 96 | + return commit_file(); |
| 97 | +} |
| 98 | + |
| 99 | +} // namespace starrocks::connector |
0 commit comments