Merge branch 'main' of github.com:Takuka0311/ilogtail

Change-Id: Ia87ad7c6667c92a37504dc2322e9f4a3a2dab420
Co-developed-by: Cursor <noreply@cursor.com>

Author: Takuka0311

.devcontainer/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/loongcollector-community-edition/loongcollector-build-linux:2.1.9
+FROM sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/loongcollector-community-edition/loongcollector-build-linux:2.1.10
 
 ARG USERNAME=admin
 ARG USER_PASSWORD

.devcontainer/Dockerfile.lldb

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/loongcollector-community-edition/loongcollector-build-linux:2.1.9
+FROM sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/loongcollector-community-edition/loongcollector-build-linux:2.1.10
 
 ARG USERNAME=admin
 ARG USER_PASSWORD

.github/workflows/benchmark.yaml .github/workflows/release-edge.yaml.github/workflows/benchmark.yaml renamed to .github/workflows/release-edge.yaml

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: Benchmark
+name: Build Edge Image
 
 on:
   push:
@@ -27,25 +27,28 @@ on:
       - 'CHANGELOG.md'
 
 jobs:
-  Build-LoongCollector-Linux:
+  # 第一步：构建不同架构的安装包
+  build-linux:
     runs-on: ${{ matrix.runner }}
+    timeout-minutes: 60
     strategy:
       matrix:
         include:
           - arch: amd64
-            runner: [self-hosted, Linux, X64]
+            runner: ubuntu-latest
           - arch: arm64
-            runner: [self-hosted, Linux, ARM64]
-      fail-fast: false
-    permissions:
-      contents: write
-      packages: write
+            runner: ubuntu-24.04-arm
     steps:
       - name: Check out code
         uses: actions/checkout@v4
         with:
           submodules: false
-  
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.19.10
+
       - name: Set custom submodule URL and fetch
         run: |
           SUBMODULE_PATH="core/_thirdparty/coolbpf"
@@ -54,26 +57,115 @@ jobs:
           cd $SUBMODULE_PATH
           echo "Current commit: $(git rev-parse HEAD)"
 
-      - name: Build linux/${{ matrix.arch }} image
+      - name: Build Linux ${{ matrix.arch }} package
+        env:
+          VERSION: edge
         run: |
-          REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
-          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $REPO_OWNER --password-stdin
-          docker build --tag ghcr.io/$REPO_OWNER/loongcollector:edge-linux-${{ matrix.arch }} \
-              --file ./docker/Dockerfile_edge_linux .
-          docker push ghcr.io/$REPO_OWNER/loongcollector:edge-linux-${{ matrix.arch }}
+          make dist
+          ls -la dist/
 
-  Build-LoongCollector-Image:
+      - name: Upload Linux ${{ matrix.arch }} artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: loongcollector-edge.linux-${{ matrix.arch }}.tar.gz
+          path: dist/loongcollector-edge.linux-${{ matrix.arch }}.tar.gz
+          retention-days: 1
+
+  # 第二步：构建和上传 Docker 镜像
+  upload-images:
+    needs: build-linux
     runs-on: ubuntu-latest
-    needs: [ Build-LoongCollector-Linux ]
+    timeout-minutes: 60
+    permissions:
+      contents: write
+      packages: write
     steps:
-      - name: Create and push manifest
+      - name: Check out code
+        uses: actions/checkout@v4
+        with:
+          submodules: false
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Download Linux AMD64 artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: loongcollector-edge.linux-amd64.tar.gz
+          path: dist/
+
+      - name: Download Linux ARM64 artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: loongcollector-edge.linux-arm64.tar.gz
+          path: dist/
+
+      - name: List artifacts for Docker build
         run: |
+          echo "Artifacts available for Docker build:"
+          ls -la dist/ || echo "No artifacts found"
+
+      - name: Build and push Docker images
+        run: |
+          VERSION="edge"
           REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
-          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $REPO_OWNER --password-stdin
-          docker manifest create ghcr.io/$REPO_OWNER/loongcollector:edge \
-            --amend ghcr.io/$REPO_OWNER/loongcollector:edge-linux-amd64 \
-            --amend ghcr.io/$REPO_OWNER/loongcollector:edge-linux-arm64
-          docker manifest push ghcr.io/$REPO_OWNER/loongcollector:edge
+          
+          # 获取当前时间戳
+          TIMESTAMP=$(date +"%Y%m%d%H%M%S")
+          TIMESTAMPED_TAG="edge-$TIMESTAMP"
+          
+          echo "Building and pushing Docker images for version $VERSION and $TIMESTAMPED_TAG"
+          
+          # 检查是否有Linux构建产物
+          LINUX_AMD64_FILE="dist/loongcollector-$VERSION.linux-amd64.tar.gz"
+          LINUX_ARM64_FILE="dist/loongcollector-$VERSION.linux-arm64.tar.gz"
+          
+          if [ ! -f "$LINUX_AMD64_FILE" ] && [ ! -f "$LINUX_ARM64_FILE" ]; then
+            echo "Error: No Linux packages found for Docker build"
+            echo "Available files in dist/:"
+            ls -la dist/ || echo "dist/ directory is empty"
+            echo "Expected files:"
+            echo "- $LINUX_AMD64_FILE"
+            echo "- $LINUX_ARM64_FILE"
+            exit 1
+          fi
+          
+          # 确定要构建的平台
+          PLATFORMS=""
+          if [ -f "$LINUX_AMD64_FILE" ]; then
+            PLATFORMS="$PLATFORMS,linux/amd64"
+            echo "Found Linux AMD64 package: $LINUX_AMD64_FILE"
+          fi
+          if [ -f "$LINUX_ARM64_FILE" ]; then
+            PLATFORMS="$PLATFORMS,linux/arm64"
+            echo "Found Linux ARM64 package: $LINUX_ARM64_FILE"
+          fi
+          
+          # 移除开头的逗号
+          PLATFORMS=$(echo $PLATFORMS | sed 's/^,//')
+          
+          echo "Building Docker images for platforms: $PLATFORMS"
+          
+          # 构建并推送多架构镜像，同时打两个标签
+          docker buildx build --platform "$PLATFORMS" \
+            --file docker/Dockerfile_release \
+            --build-arg VERSION=$VERSION \
+            --build-arg HOST_OS=Linux \
+            --tag ghcr.io/$REPO_OWNER/loongcollector:$VERSION \
+            --tag ghcr.io/$REPO_OWNER/loongcollector:$TIMESTAMPED_TAG \
+            --push .
+          
+          echo "Docker images pushed successfully:"
+          echo "- ghcr.io/$REPO_OWNER/loongcollector:$VERSION"
+          echo "- ghcr.io/$REPO_OWNER/loongcollector:$TIMESTAMPED_TAG"
+
 
   # Benchmark:
   #   runs-on: ${{ matrix.runner }}
@@ -192,7 +284,7 @@ jobs:
   #       run: git push 'https://${{ github.repository_owner }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository_owner }}/LoongCollector.git' gh-pages:gh-pages
 
   actions-timeline:
-    needs: [Build-LoongCollector-Image]
+    needs: [upload-images]
     runs-on: ubuntu-latest
     permissions:
       actions: read

README.md

@@ -24,6 +24,14 @@
 
 LoongCollector is a high-performance, lightweight observability data collector designed for modern cloud-native environments. Born from Alibaba's 15-year journey handling exponential traffic growth and powering tens of millions of deployments, LoongCollector delivers **10x higher throughput** with **80% lower resource usage** compared to open-source alternatives.
 
+LoongCollector is a key component of LoongSuite(Alibaba's unified observability data collection suite). LoongSuite includes the following key components:
+
+* [LoongCollector](https://github.com/alibaba/loongcollector): universal node agent, which prodivdes log collection, prometheus metric collection, and network and security collection capabilities based on eBPF.
+* [LoongSuite Python Agent](https://github.com/alibaba/loongsuite-python-agent): a process agent providing instrumentaion for python applications.
+* [LoongSuite Go Agent](https://github.com/alibaba/loongsuite-go-agent): a process agent for golang with compile time instrumentation.
+* [LoongSuite Java Agent](https://github.com/alibaba/loongsuite-java-agent): a process agent for Java applications.
+* Other upcoming language agent.
+
 ### 🚀 Core Advantages
 
 - **⚡ High Performance**
@@ -173,9 +181,17 @@ We welcome contributions from the community! Here are some ways you can help:
 - **WeChat/DingTalk**: LoongCollector社区
 
 <div align="center">
-<img src="https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/images/chatgroup/chatgroup.png" alt="Community QR Code" width="300"/>
+<img src="https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/images/chatgroup/chatgroup.png" alt="Community QR Code" width="60%"/>
 </div>
 
+### Other LoongSuite components's Community contact information
+
+We are looking forward to your feedback and suggestions. You can scan the QR code below to engage with us.
+
+| LoongSuite Python SIG | LoongSuite Go SIG | LoongSuite Java SIG |
+|----|----|----|
+| <img src="https://github.com/alibaba/loongsuite-python-agent/blob/main/docs/_assets/img/loongsuite-python-sig-dingtalk.jpg" height="150"> | <img src="https://github.com/alibaba/loongsuite-python-agent/blob/main/docs/_assets/img/loongsuite-go-sig-dingtalk.png" height="150"> | <img src="https://github.com/alibaba/loongsuite-python-agent/blob/main/docs/_assets/img/loongsuite-java-sig-dingtalk.jpg" height="150"> |
+
 ## 📄 License
 
 LoongCollector is licensed under the [Apache 2.0 License](./LICENSE).

core/CMakeLists.txt

@@ -34,6 +34,7 @@ cmake_dependent_option(ENABLE_CORP_FEATURE "Enable corp feature" OFF "ENABLE_ENT
 option(WITHOUTGDB "Build Logtail without gdb")
 option(WITHSPL "Build Logtail and UT with SPL" ON)
 option(BUILD_LOGTAIL_UT "Build unit test for Logtail")
+option(BUILD_LOGTAIL_UT_WITH_BENCHMARK "Build unit test for benchmark" OFF)
 cmake_dependent_option(ENABLE_ADDRESS_SANITIZER "Enable address sanitizer" ON "CMAKE_BUILD_TYPE STREQUAL Debug;NOT ANDROID" OFF)
 set(PROVIDER_PATH ${CMAKE_CURRENT_SOURCE_DIR}/provider CACHE PATH "Path to the provider module") # external provider path can be set with -DPROVIDER_PATH
 set(UNITTEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/unittest CACHE PATH "Path to the unittest module") # external unittest path can be set with -DUNITTEST_PATH
@@ -60,8 +61,12 @@ endif ()
 
 if (NOT WITHSPL)
     add_definitions(-D__EXCLUDE_SPL__)
+else ()
+    add_definitions(-D__INCLUDE_SSE4_2__)
 endif()
 
+message(STATUS "loongcollector Cpp compile flags: ${CMAKE_CXX_FLAGS}")
+
 add_definitions(-DXXH_DISPATCH_AVX2=0)
 add_definitions(-DXXH_DISPATCH_AVX512=0)
 
@@ -84,9 +89,11 @@ if (UNIX)
     string(REPLACE "-O3" "" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
     string(REPLACE "-O3" "" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
     if (BUILD_LOGTAIL_UT)
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
-        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
-        SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage")
+        if (NOT BUILD_LOGTAIL_UT_WITH_BENCHMARK) 
+            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
+            SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
+            SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage")
+        endif ()
     endif ()
 elseif (MSVC)
     add_definitions(-DNOMINMAX)
@@ -151,7 +158,7 @@ if (LINUX)
     endif()
     set(SUB_DIRECTORIES_LIST ${SUB_DIRECTORIES_LIST} ebpf ebpf/type ebpf/type/table ebpf/util ebpf/util/sampler ebpf/protocol/http ebpf/protocol ebpf/plugin/file_security ebpf/plugin/network_observer ebpf/plugin/process_security ebpf/plugin/network_security ebpf/plugin ebpf/observer ebpf/security
         prometheus prometheus/labels prometheus/schedulers prometheus/async prometheus/component
-        host_monitor host_monitor/collector forward forward/loongsuite
+        host_monitor host_monitor/collector host_monitor/common forward forward/loongsuite
         )
 elseif(MSVC)
 endif ()

core/_thirdparty/coolbpf

@@ -1509,7 +1509,7 @@ void AppConfig::ReadFlagsFromMap(const std::unordered_map<std::string, std::stri
  *    - 记录无法转换的值
  */
 void AppConfig::RecurseParseJsonToFlags(const Json::Value& confJson, std::string prefix) {
-    const static unordered_set<string> sIgnoreKeySet = {"data_server_list", "data_servers"};
+    const static unordered_set<string> sIgnoreKeySet = {"data_server_list", "data_servers", "auth_config"};
     const static unordered_set<string> sForceKeySet = {"config_server_address_list", "config_servers"};
     for (auto name : confJson.getMemberNames()) {
         auto jsonvalue = confJson[name];

core/app_config/AppConfig.cpp

@@ -216,17 +216,29 @@ void Application::Start() { // GCOVR_EXCL_START
 
     // config provider
     {
-        // add local config dir
-        filesystem::path localConfigPath = filesystem::path(AppConfig::GetInstance()->GetLoongcollectorConfDir())
-            / GetContinuousPipelineConfigDir() / "local";
-        error_code ec;
-        filesystem::create_directories(localConfigPath, ec);
-        if (ec) {
+        // add local continuous config dir
+        filesystem::path localContinuousConfigPath
+            = filesystem::path(AppConfig::GetInstance()->GetLoongcollectorConfDir()) / GetContinuousPipelineConfigDir()
+            / "local";
+        error_code ec1;
+        filesystem::create_directories(localContinuousConfigPath, ec1);
+        if (ec1) {
             LOG_WARNING(sLogger,
                         ("failed to create dir for local continuous_pipeline_config",
-                         "manual creation may be required")("error code", ec.value())("error msg", ec.message()));
+                         "manual creation may be required")("error code", ec1.value())("error msg", ec1.message()));
         }
-        PipelineConfigWatcher::GetInstance()->AddSource(localConfigPath.string());
+        PipelineConfigWatcher::GetInstance()->AddSource(localContinuousConfigPath.string());
+        // add local onetime config dir
+        filesystem::path localOnetimeConfigPath = filesystem::path(AppConfig::GetInstance()->GetLoongcollectorConfDir())
+            / "onetime_pipeline_config" / "local";
+        error_code ec2;
+        filesystem::create_directories(localOnetimeConfigPath, ec2);
+        if (ec2) {
+            LOG_WARNING(sLogger,
+                        ("failed to create dir for local onetime_pipeline_config",
+                         "manual creation may be required")("error code", ec2.value())("error msg", ec2.message()));
+        }
+        PipelineConfigWatcher::GetInstance()->AddSource(localOnetimeConfigPath.string());
     }
 #ifdef __ENTERPRISE__
     EnterpriseConfigProvider::GetInstance()->Start();

core/application/Application.cpp

@@ -90,6 +90,7 @@ bool CollectionPipeline::Init(CollectionConfig&& config) {
     // for special treatment below
     const InputFile* inputFile = nullptr;
     const InputContainerStdio* inputContainerStdio = nullptr;
+    const InputStaticFile* inputStaticFile = nullptr;
     bool hasFlusherSLS = false;
 
     // to send alarm and init MetricsRecord before flusherSLS is built, a temporary object is made, which will be
@@ -122,6 +123,8 @@ bool CollectionPipeline::Init(CollectionConfig&& config) {
                 inputFile = static_cast<const InputFile*>(mInputs[0]->GetPlugin());
             } else if (pluginType == InputContainerStdio::sName) {
                 inputContainerStdio = static_cast<const InputContainerStdio*>(mInputs[0]->GetPlugin());
+            } else if (pluginType == InputStaticFile::sName) {
+                inputStaticFile = static_cast<const InputStaticFile*>(mInputs[0]->GetPlugin());
             }
         } else {
             AddPluginToGoPipeline(pluginType, detail, "inputs", mGoPipelineWithInput);
@@ -251,7 +254,8 @@ bool CollectionPipeline::Init(CollectionConfig&& config) {
     }
 
     // mandatory override global.DefaultLogQueueSize in Go pipeline when input_file and Go processing coexist.
-    if ((inputFile != nullptr || inputContainerStdio != nullptr) && IsFlushingThroughGoPipeline()) {
+    if ((inputFile != nullptr || inputContainerStdio != nullptr || inputStaticFile != nullptr)
+        && IsFlushingThroughGoPipeline()) {
         mGoPipelineWithoutInput["global"]["DefaultLogQueueSize"]
             = Json::Value(INT32_FLAG(default_plugin_log_queue_size));
     }

core/collection_pipeline/CollectionPipeline.cpp

@@ -34,6 +34,7 @@
 #include "monitor/MetricManager.h"
 #include "plugin/input/InputContainerStdio.h"
 #include "plugin/input/InputFile.h"
+#include "plugin/input/InputStaticFile.h"
 
 namespace logtail {