diff --git a/.circleci/get_scm_version.py b/.github/scripts/get_scm_version.py similarity index 53% rename from .circleci/get_scm_version.py rename to .github/scripts/get_scm_version.py index c432167958..79dfee9e5d 100644 --- a/.circleci/get_scm_version.py +++ b/.github/scripts/get_scm_version.py @@ -1,4 +1,4 @@ from setuptools_scm import get_version -version = get_version(root='../', relative_to=__file__) +version = get_version(root='../../', relative_to=__file__) print(version.split('+')[0]) diff --git a/.github/workflows/private-repo-test.yaml b/.github/workflows/private-repo-test.yaml new file mode 100644 index 0000000000..48ab89f132 --- /dev/null +++ b/.github/workflows/private-repo-test.yaml @@ -0,0 +1,92 @@ +name: Private Repo Testing + +on: + pull_request: + branches: + - main + +concurrency: + group: 'private-test-${{ github.event.pull_request.number }}' + cancel-in-progress: true + +jobs: + trigger-private-test: + runs-on: ubuntu-latest + if: github.event.pull_request.head.repo.full_name == github.repository + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install uv + uses: astral-sh/setup-uv@v6 + - name: Set up Node.js for UI build + uses: actions/setup-node@v4 + with: + node-version: '20' + - name: Install pnpm + uses: pnpm/action-setup@v4 + with: + version: latest + - name: Install UI dependencies + run: pnpm install + - name: Build UI + run: pnpm --prefix web/client run build + - name: Install Python dependencies + run: | + python -m venv .venv + source .venv/bin/activate + pip install build twine setuptools_scm + - name: Generate development version + id: version + run: | + source .venv/bin/activate + # Generate a development version using existing script + DEV_VERSION=$(python .github/scripts/get_scm_version.py) + echo "version=$DEV_VERSION" >> $GITHUB_OUTPUT + echo "Generated development version: $DEV_VERSION" + - name: Build package + run: | + source .venv/bin/activate + python -m build + - name: Configure PyPI for private repository + run: | + cat > ~/.pypirc << EOF + [distutils] + index-servers = tobiko-private + [tobiko-private] + repository = ${{ secrets.TOBIKO_PRIVATE_PYPI_URL }} + username = _json_key_base64 + password = ${{ secrets.TOBIKO_PRIVATE_PYPI_KEY }} + EOF + - name: Publish to private PyPI + run: | + source .venv/bin/activate + python -m twine upload -r tobiko-private dist/* + - name: Get commit information + id: commit + run: | + echo "author=$(git log -1 --format='%an')" >> $GITHUB_OUTPUT + echo "hash=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + echo "message=$(git log -1 --format='%s')" >> $GITHUB_OUTPUT + - name: Trigger private repository workflow + uses: convictional/trigger-workflow-and-wait@v1.6.5 + with: + owner: ${{ secrets.PRIVATE_REPO_OWNER }} + repo: ${{ secrets.PRIVATE_REPO_NAME }} + github_token: ${{ secrets.PRIVATE_REPO_TOKEN }} + workflow_file_name: ${{ secrets.PRIVATE_WORKFLOW_FILE }} + client_payload: | + { + "package_version": "${{ steps.version.outputs.version }}", + "python_version": "3.12", + "author": "${{ steps.commit.outputs.author }}", + "hash": "${{ steps.commit.outputs.hash }}", + "message": "${{ steps.commit.outputs.message }}", + "pr_number": "${{ github.event.pull_request.number }}" + } diff --git a/sqlmesh/core/engine_adapter/base.py b/sqlmesh/core/engine_adapter/base.py index 33ad4c398a..d1ded978f9 100644 --- a/sqlmesh/core/engine_adapter/base.py +++ b/sqlmesh/core/engine_adapter/base.py @@ -1762,21 +1762,19 @@ def remove_managed_columns( # Historical Records that Do Not Change .with_( "static", - existing_rows_query.where(valid_to_col.is_(exp.Null()).not_()) - if truncate - else existing_rows_query.where( + existing_rows_query.where( exp.and_( valid_to_col.is_(exp.Null().not_()), valid_to_col < cleanup_ts, ), - ), + ) + if truncate + else existing_rows_query.where(valid_to_col.is_(exp.Null()).not_()), ) # Latest Records that can be updated .with_( "latest", - existing_rows_query.where(valid_to_col.is_(exp.Null())) - if truncate - else exp.select( + exp.select( *( to_time_column( exp.null(), time_data_type, self.dialect, nullable=True @@ -1796,7 +1794,9 @@ def remove_managed_columns( valid_to_col >= cleanup_ts, ), ) - ), + ) + if truncate + else existing_rows_query.where(valid_to_col.is_(exp.Null())), ) # Deleted records which can be used to determine `valid_from` for undeleted source records .with_(