Merge branch 'master' into asv_s3_more

Author: grusev

.github/PULL_REQUEST_TEMPLATE.md

@@ -3,12 +3,6 @@
 
 #### What does this implement or fix?
 
-## Change Type (Required)
-- [ ] **Patch** (Bug fix or non-breaking improvement)
-- [ ] **Minor** (New feature, but backward compatible)
-- [ ] **Major** (Breaking changes)
-- [ ] **Cherry pick**
-
 #### Any other comments?
 
 #### Checklist

.github/workflows/analysis_workflow.yml

@@ -10,11 +10,12 @@ on:
         type: string
         default: latest
       suite_to_run:
-          description: Run LMDB suite or REAL storage
+          description: Run LMDB suite or REAL storage (or both - ALL)
           type: choice
           options:
             - 'LMDB'
             - 'REAL'
+            - 'ALL'
           default: 'LMDB'
       suite_overwrite:
         description: Specify regular expression for specific tests to be executed
@@ -63,7 +64,7 @@ jobs:
       run_all_benchmarks: ${{ inputs.run_all_benchmarks || false }}
       run_on_pr_head: ${{ github.event_name == 'pull_request_target' }}
       dev_image_tag: ${{ inputs.dev_image_tag || 'latest' }}
-      suite_to_run: ${{ inputs.suite_to_run || 'LMDB'}}
+      suite_to_run: ${{  github.event_name == 'schedule' && 'ALL' || inputs.suite_to_run || 'LMDB'}}
       suite_overwrite: ${{ inputs.suite_overwrite || ''}}
 
   publish_benchmark_results_to_gh_pages:  
@@ -87,7 +88,7 @@ jobs:
         run: |
           apt update
           apt install -y git
-          python -m pip install arcticdb[Testing] "protobuf<5"
+          python -m pip install arcticdb[Testing] "protobuf<6"
 
       - name: Setup softlink for SSL
         shell: bash -el {0}

.github/workflows/automated_release.calculate_next_version.yml

@@ -48,9 +48,9 @@ jobs:
         id: get_increment_type
         env:
           PULL_REQUESTS: ${{ steps.pull_requests.outputs.pull_requests }}
-          MAJOR_LABEL: major
-          MINOR_LABEL: minor
-          PATCH_LABEL: patch
+          MAJOR_LABEL: "api break"
+          MINOR_LABEL: "enhancement"
+          PATCH_LABEL: "bug"
         run: |
           if [ -z "$PULL_REQUESTS" ]; then
             echo "Error: No PRs found between branches" && exit 1

.github/workflows/automated_release.check_pr_labels.yml

@@ -0,0 +1,21 @@
+name: Check Pull Request Labels
+on:
+  pull_request:
+    types: [opened, labeled, unlabeled, synchronize]
+    branches: master
+jobs:
+  check_labels:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: mheap/github-action-required-labels@v5
+        with:
+          mode: exactly
+          count: 1
+          labels: |
+            patch
+            minor
+            major
+          add_comment: true

.github/workflows/automated_release.create_release_branch.yml

@@ -33,13 +33,16 @@ jobs:
   create-next-release-branch:
     needs: [calculate-next-version, get-master-sha]
     runs-on: ubuntu-latest
+    environment: TestPypi # For the branch restrictions token
+    permissions:
+      contents: write
     outputs:
       new_branch: ${{ needs.calculate-next-version.outputs.version }}
     steps:
     - name: Create branch ${{ needs.calculate-next-version.outputs.version }} from ${{ needs.get-master-sha.outputs.sha }}
       uses: peterjgrainger/[email protected]
       env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        GITHUB_TOKEN: ${{ secrets.TAGGING_TOKEN }}
       with:
         branch: ${{ needs.calculate-next-version.outputs.version }}
         sha: ${{ needs.get-master-sha.outputs.sha }}

.github/workflows/automated_release.parse_pr_template.yml

@@ -6,7 +6,7 @@ on:
       commit:             {required: true, type: string, description: commit hash that will be benchmarked}
       run_on_pr_head:     {required: false, default: false, type: boolean, description: Specifies if the benchmark should run on PR head branch}
       dev_image_tag:      {required: false, default: 'latest', type: string, description: Tag of the ArcticDB development image}
-      suite_to_run:       {required: true, type: string, default: 'LMDB', description: Default benchmark on 'LMDB' storage (or 'REAL' storage)}
+      suite_to_run:       {required: true, type: string, default: 'LMDB', description: Default benchmark on 'LMDB' storage (or 'REAL' storage ond 'ALL' for both)}
       suite_overwrite:       {required: false, type: string, default: '', description: User defined tests to run}
 jobs:
   start_ec2_runner:
@@ -74,12 +74,17 @@ jobs:
           pip install asv virtualenv
           python -m asv machine -v --yes --machine ArcticDB-Medium-Runner
 
-      - name: Confugure what suite or tests to execute
+      - name: Configure what suite or tests to execute
         shell: bash -el {0}
         run: | 
-            if [ "${{ github.event.inputs.suite_to_run }}" == "REAL" ]; then
+            if [ "${{ inputs.suite_to_run }}" == "REAL" ]; then
+              # ASV tests with real storage starts with 'real_' in the test name
               SUITE='^(real_).*'
+            elif [ "${{ inputs.suite_to_run }}" == "ALL" ]; then
+              # Select all tests
+              SUITE='^.*'
             else
+              # LMDB tests are other tests not starting with 'real_'
               SUITE='^(?!real_).*'
             fi
             echo "selection of suite SUITE=$SUITE"
@@ -110,7 +115,7 @@ jobs:
       - name: Add results to ArcticDB database
         shell: bash -el {0}
         run: |
-          pip install arcticdb[Testing]
+          pip install arcticdb[Testing] "protobuf<6"
           python build_tooling/transform_asv_results.py --mode save ${{ github.ref != 'refs/heads/master' && format('--arcticdb_library {0}_asv_results', github.ref_name) || ''}}
 
   stop-ec2-runner:

.github/workflows/benchmark_commits.yml

@@ -5,7 +5,7 @@ on:
       version: { type: string, required: false, description: "The version to build (used in git and pypi). git-tag='v{version}-docs'.  If not specified then use selected branch and wheel from the last successful build."}
       latest: { type: boolean, required: false, description: Alias this version as the 'latest' stable docs.  This should be set for the latest stable release.}
       deploy: { type: boolean, required: false, description: Push the built docs to the docs-pages branch on github.}
-      
+
 jobs:
   docs_build:
     runs-on: ubuntu-22.04
@@ -34,7 +34,7 @@ jobs:
           prefix = {"CPython": "cp", "PyPy": "pp"}[platform.python_implementation()]
           v = platform.python_version_tuple()
           print(f"PY_IMPL={prefix}{v[0]}{v[1]}")' | tee -a $GITHUB_ENV
-  
+
       - id: download-wheel-artifact
         name: Download wheel artifact from last successful build
         if: ${{!inputs.version}}
@@ -75,7 +75,7 @@ jobs:
           cd docs/mkdocs
           mike list
 
-      - name: Versioned mkDocs build 
+      - name: Versioned mkDocs build
         run: |
           set -x
           cd docs/mkdocs
@@ -91,16 +91,3 @@ jobs:
           set -x
           cd docs/mkdocs
           mike list
-
-      - name: Checkout docs-pages branch
-        run: git checkout docs-pages
-
-      - name: Deploy Docs to TestPypi
-        uses: cloudflare/[email protected]
-        with:
-          apiToken: ${{secrets.CLOUDFLARE_API_TOKEN}}
-          accountId: ${{vars.CLOUDFLARE_ACCOUNT_ID}}
-          projectName: ${{vars.CLOUDFLARE_PAGES_PROJECT}}
-          directory: .
-          branch: test
-

.github/workflows/docs_build.yml

@@ -14,7 +14,7 @@ jobs:
         with:
           status: ${{ github.event.workflow_run.conclusion }}
           notification_title: " ${{github.event.workflow_run.name}} - ${{github.event.workflow_run.conclusion}} on ${{github.event.workflow_run.head_branch}} - <${{github.server_url}}/${{github.repository}}/actions/runs/${{github.event.workflow_run.id}}|View Failure>"
-          message_format: ":fire: *${{github.event.workflow_run.name}}* ${{github.event.workflow_run.conclusion}} in <${{github.server_url}}/${{github.repository}}/${{github.event.workflow_run.head_branch}}|${{github.repository}}>"
+          message_format: ":fire: *${{github.event.workflow_run.name}}* ${{github.event.workflow_run.conclusion}} in ${{github.server_url}}/${{github.repository}}/actions/runs/${{github.event.workflow_run.id}}|${{github.repository}}>"
           footer: "Linked Repo <${{github.server_url}}/${{github.repository}}|${{github.repository}}> | <${{github.server_url}}/${{github.repository}}/actions/runs/${{github.event.workflow_run.id}}|View Failure>"
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/docs_publish.yml

@@ -48,7 +48,7 @@ jobs:
 
       - id: changelog
         name: Release Changelog Builder
-        uses: mikepenz/release-changelog-builder-action@v3.7.3
+        uses: mikepenz/release-changelog-builder-action@v5
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:

.github/workflows/failure_notification.yaml

@@ -50,3 +50,6 @@ docs/mkdocs/docs/notebooks/.ipynb_checkpoints/
 # Ignore asv results
 python/.asv/html/
 python/.asv/results/
+
+# Ignore azurite files
+__azurite_*

.github/workflows/publish.yml

@@ -1,12 +1,16 @@
 {
 "categories": [
+    {
+        "title": "## ⚠️ Breaking Changes",
+        "labels": ["api break", "major"]
+    },
     {
         "title": "## 🚀 Features",
-        "labels": ["enhancement"]
+        "labels": ["enhancement", "minor"]
     },
     {
         "title": "## 🐛 Fixes",
-        "labels": ["bug"]
+        "labels": ["bug", "patch"]
     }
 ],
 "pr_template": "- ${{TITLE}} (#${{NUMBER}})",

.gitignore

@@ -937,7 +937,7 @@ if(${TEST})
             processing/test/test_component_manager.cpp
             processing/test/test_expression.cpp
             processing/test/test_filter_and_project_sparse.cpp
-            processing/test/test_has_valid_type_promotion.cpp
+            processing/test/test_type_promotion.cpp
             processing/test/test_operation_dispatch.cpp
             processing/test/test_parallel_processing.cpp
             processing/test/test_resample.cpp

build_tooling/change_log.json

@@ -139,6 +139,7 @@ struct DeserializedSegmentData {
 };
 
 DeserializedSegmentData decode_header_and_fields(const uint8_t*& src, bool copy_data) {
+    util::check(src != nullptr, "Got null data ptr from segment");
     auto* fixed_hdr = reinterpret_cast<const FixedHeader*>(src);
     ARCTICDB_DEBUG(log::codec(), "Reading header: {} + {} = {}", FIXED_HEADER_SIZE, fixed_hdr->header_bytes, FIXED_HEADER_SIZE + fixed_hdr->header_bytes);
 
@@ -279,14 +280,41 @@ std::pair<uint8_t*, size_t> Segment::serialize_v1_header_in_place(size_t total_h
 
 std::tuple<uint8_t*, size_t, std::unique_ptr<Buffer>> Segment::serialize_v1_header_to_buffer(size_t hdr_size) {
     auto tmp = std::make_unique<Buffer>();
-    ARCTICDB_TRACE(log::storage(), "Header doesn't fit in internal buffer, needed {} bytes but had {}, writing to temp buffer at {:x}", hdr_size, buffer_.preamble_bytes(),uintptr_t(tmp->data()));
-    tmp->ensure(calculate_size());
+    auto bytes_to_copy = buffer().bytes();
+    auto offset = FIXED_HEADER_SIZE + hdr_size;
+
+    auto total_size = offset + bytes_to_copy;
+
+    // Verify we have enough space for everything
+    tmp->ensure(total_size);
+    util::check(tmp->available() >= total_size,
+        "Buffer available space {} is less than required size {}", 
+        tmp->available(), 
+        total_size);
+
+    // This is both a sanity check and a way to populate the segment with the correct size
+    auto calculated_size = calculate_size();
+    util::check(total_size == calculated_size, "Expected total size {} to be equal to calculated size {}", total_size, calculated_size);
+
     auto* dst = tmp->preamble();
-    write_proto_header(dst);
-    std::memcpy(dst + FIXED_HEADER_SIZE + hdr_size,
-                buffer().data(),
-                buffer().bytes());
-    return std::make_tuple(tmp->preamble(), calculate_size(), std::move(tmp));
+    util::check(dst != nullptr, "Expected dst to be non-null");
+
+    auto written_hdr_size = write_proto_header(dst);
+    util::check(written_hdr_size == hdr_size, "Expected written header size {} to be equal to expected header size {}", written_hdr_size, hdr_size);
+
+    auto *final_dst = dst + offset;
+
+    auto *src = buffer().data();
+    if (src != nullptr) {   
+        std::memcpy(final_dst,
+                    src,
+                    bytes_to_copy);
+    } else {
+        util::check(bytes_to_copy == 0, "Expected bytes_to_copy to be 0 when src is nullptr");
+        ARCTICDB_DEBUG(log::codec(), "src is nullptr, skipping memcpy");
+    }
+
+    return std::make_tuple(dst, total_size, std::move(tmp));
 }
 
 std::tuple<uint8_t*, size_t, std::unique_ptr<Buffer>> Segment::serialize_header_v1() {