From 026bf6b98f6edbec4b645f141827c8eb3548fa2d Mon Sep 17 00:00:00 2001 From: Raki Rahman Date: Thu, 14 May 2026 09:11:01 +0000 Subject: [PATCH 1/6] No prompt s for add-apt --- contrib/bootstrap-dev-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/bootstrap-dev-env.sh b/contrib/bootstrap-dev-env.sh index be1d9bad..8c3e6a4c 100755 --- a/contrib/bootstrap-dev-env.sh +++ b/contrib/bootstrap-dev-env.sh @@ -22,7 +22,7 @@ command -v gh &>/dev/null || PACKAGES="$PACKAGES gh" if ! [ -x "$(command -v docker)" ]; then echo "docker is not installed on your devbox, installing..." curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + sudo add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" sudo apt-get update -q sudo apt-get install -y apt-transport-https ca-certificates curl sudo apt-get install -y --allow-downgrades docker-ce="$DOCKER_VERSION" docker-ce-cli="$DOCKER_VERSION" containerd.io From faa4c8e2bf69e7cb7936baa1c5242b7c5f14c84a Mon Sep 17 00:00:00 2001 From: Raki Rahman Date: Thu, 14 May 2026 09:25:31 +0000 Subject: [PATCH 2/6] Script update --- contrib/bootstrap-dev-env.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/contrib/bootstrap-dev-env.sh b/contrib/bootstrap-dev-env.sh index 8c3e6a4c..a5b72842 100755 --- a/contrib/bootstrap-dev-env.sh +++ b/contrib/bootstrap-dev-env.sh @@ -40,10 +40,14 @@ if grep -q "$ACR_URL" ~/.docker/config.json 2>/dev/null; then if [ -n "$ACR_PASSWORD" ]; then docker_password="$ACR_PASSWORD" else - read -sp "Enter Docker Admin password for ${ACR_URL}: " docker_password + read -sp "If you are a Microsoft Employee and you plan on contributing to the Devcontainer image, please ping @mdrrahman and enter Docker Admin password for ${ACR_URL} - otherwise, leave blank and press [ENTER]: " docker_password echo fi - echo "$docker_password" | docker login "$ACR_URL" --username "$ACR_NAME" --password-stdin + if [ -z "$docker_password" ]; then + echo "You left the password empty, skipping docker login" + else + echo "$docker_password" | docker login "$ACR_URL" --username "$ACR_NAME" --password-stdin + fi fi export PATH=$(echo "$PATH" | tr ':' '\n' | grep -v "/mnt/c" | tr '\n' ':' | sed 's/:$//') From 7e424c2541daa7605d5a7a02d22b8e7b9cf37694 Mon Sep 17 00:00:00 2001 From: Raki Rahman Date: Thu, 14 May 2026 09:35:23 +0000 Subject: [PATCH 3/6] Improve contributing --- CONTRIBUTING.md | 153 ++++++++++++++++++++++------------------------ contrib/README.md | 86 -------------------------- 2 files changed, 73 insertions(+), 166 deletions(-) delete mode 100644 contrib/README.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a6a7c65d..2ae965ac 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,107 +1,100 @@ -# Contributing to `dbt-fabricspark` +# Contributing -1. [About this document](#about-this-document) -3. [Getting the code](#getting-the-code) -5. [Running `dbt-fabricspark` in development](#running-dbt-fabricspark-in-development) -6. [Testing](#testing) -7. [Updating Docs](#updating-docs) -7. [Submitting a Pull Request](#submitting-a-pull-request) +[![Deep-dive Walkthrough](https://rakirahman.blob.core.windows.net/public/images/Misc/dbt-fabricspark-contrib.png)](https://rakirahman.blob.core.windows.net/public/videos/dbt-fabricspark-local-development-deep-dive.mp4) -## About this document -This document is a guide intended for folks interested in contributing to `dbt-fabricspark`. Below, we document the process by which members of the community should create issues and submit pull requests (PRs) in this repository. It is not intended as a guide for using `dbt-fabricspark`, and it assumes a certain level of familiarity with Python concepts such as virtualenvs, `pip`, Python modules, and so on. This guide assumes you are using macOS or Linux and are comfortable with the command line. +If you're a windows user, we only use windows to get into WSL - everything is Linux from there. -For those wishing to contribute we highly suggest reading the dbt-core's [contribution guide](https://github.com/dbt-labs/dbt-core/blob/HEAD/CONTRIBUTING.md) if you haven't already. Almost all of the information there is applicable to contributing here, too! +Spark and Livy are notoriously hard to run on Windows. +Therefore, the development environment we support is Linux, using [VSCode Devcontainer](https://code.visualstudio.com/docs/devcontainers/containers). +This repo's CI runs the exact same devcontainer. -## Getting the code +Therefore, if the tests pass locally, they are highly likely to pass in CI as well. -You will need `git` in order to download and modify the `dbt-fabricspark` source code. You can find directions [here](https://github.com/git-guides/install-git) on how to install `git`. +## How to use, on a Linux machine -### External contributors +1. Windows pre-reqs -If you are not a member of the `Microsoft` GitHub organization, you can contribute to `dbt-fabricspark` by forking the `dbt-fabricspark` repository. For a detailed overview on forking, check out the [GitHub docs on forking](https://help.github.com/en/articles/fork-a-repo). In short, you will need to: + ```powershell + winget install -e --id Microsoft.VisualStudioCode + ``` -1. fork the `dbt-fabricspark` repository -2. clone your fork locally -3. check out a new branch for your proposed changes -4. push changes to your fork -5. open a pull request against `microsoft/dbt-fabricspark` from your forked repository +1. Get a fresh new WSL machine up: -### Microsoft Org contributors + ```powershell + $GIT_ROOT = git rev-parse --show-toplevel + & "$GIT_ROOT\contrib\bootstrap-dev-env.ps1" + ``` -If you are a member of the `Microsoft` GitHub organization, you will have push access to the `dbt-fabricspark` repo. Rather than forking `dbt-fabricspark` to make your changes, just clone the repository, check out a new branch, and push directly to that branch. +1. Clone the repo, and open VSCode in it: + ```bash + sudo mkdir -p /workspaces && sudo chmod 777 /workspaces && cd /workspaces -## Running `dbt-fabricspark` in development + read -p "Enter your name (e.g. 'FirstName LastName'): " user_name + read -p "Enter your email (e.g. 'your-alias@foo.com'): " user_email + read -p "Enter your git fork (e.g. 'https://github.com/microsoft/dbt-fabricspark.git'): " git_fork_url + read -p "Enter the existing branch to switch to: (e.g. 'main'): " branch_name + + git config --global user.name "$user_name" + git config --global user.email "$user_email" + git clone "$git_fork_url" -### Installation + cd /workspaces/dbt-fabricspark + git checkout "$branch_name" -First make sure that you set up your `virtualenv` as described in [Setting up an environment](https://github.com/dbt-labs/dbt-core/blob/HEAD/CONTRIBUTING.md#setting-up-an-environment). Ensure you have the uv installed with `pip install uv` or via curl. + code . + ``` -Next, if this is first time installation of `dbt-fabricspark` with latest dependencies: +1. Run the bootstrapper script, that installs all tools idempotently: -```sh -uv pip install -e . --group dev -``` + ```bash + GIT_ROOT=$(git rev-parse --show-toplevel) + chmod +x ${GIT_ROOT}/contrib/bootstrap-dev-env.sh && ${GIT_ROOT}/contrib/bootstrap-dev-env.sh + ``` -If have installed packages locally using uv pip install, make sure to run uv sync to update the uv.lock file from your environment. -```sh -uv pip install -``` +1. Launch the devcontainer: -When `dbt-fabricspark` is installed this way, any changes you make to the `dbt-fabricspark` source code will be reflected immediately in your next `dbt-fabricspark` run. + ```bash + cd /workspaces/dbt-fabricspark + HEX=$(printf '%s' "$(wslpath -w .)" | xxd -ps -c 256) + code --folder-uri "vscode-remote://dev-container+${HEX}/workspaces/dbt-fabricspark" + ``` -To confirm you have correct version of `dbt-core` installed please run `dbt --version` and `which dbt`. +1. Install recommended developer tooling (optional): + ```bash + curl -fsSL https://gh.io/copilot-install | bash + $HOME/.local/bin/copilot --yolo + ``` -## Testing +1. Login to github and ensure to authorize `Microsoft` if you're an employee (optional): -### Initial Setup + ```bash + gh auth login + ``` -`dbt-fabricspark` uses test credentials specified in a `test.env` file in the root of the repository. This `test.env` file is git-ignored, but please be _extra_ careful to never check in credentials or other sensitive information when developing. To create your `test.env` file, copy the provided example file, then supply your relevant credentials. +1. Create a `test.env` file with the two Fabric workspace IDs and display names — you need `Contributor` on both. The functional test suite uses **two** workspaces: -``` -cp test.env.example test.env -$EDITOR test.env -``` + - **Workspace 1 (`WORKSPACE_ID_1` / `WORKSPACE_NAME_1`)** — the *primary* workspace where dbt models are materialized during tests. + - **Workspace 2 (`WORKSPACE_ID_2` / `WORKSPACE_NAME_2`)** — the *secondary* workspace that hosts a seeded fixture lakehouse, used to verify cross-workspace 4-part naming end-to-end. -### Test commands -There are a few methods for running tests locally. + ```bash + GIT_ROOT=$(git rev-parse --show-toplevel) + read -p "Enter Fabric workspace 1 ID: " ws1_id + read -p "Enter Fabric workspace 1 display name: " ws1_name + read -p "Enter Fabric workspace 2 ID: " ws2_id + read -p "Enter Fabric workspace 2 display name: " ws2_name + { + echo "WORKSPACE_ID_1=${ws1_id}" + echo "WORKSPACE_NAME_1=${ws1_name}" + echo "WORKSPACE_ID_2=${ws2_id}" + echo "WORKSPACE_NAME_2=${ws2_name}" + } > "${GIT_ROOT}/test.env" + ``` -#### `tox` -`tox` takes care of managing Python virtualenvs and installing dependencies in order to run tests. You can also run tests in parallel, for example you can run unit tests for Python 3.8, Python 3.9, and `flake8` checks in parallel with `tox -p`. Also, you can run unit tests for specific python versions with `tox -e `. The configuration of these tests are located in `tox.ini`. +1. All build and tests should now run green: -#### `pytest` -Finally, you can also run a specific test or group of tests using `pytest` directly. With a Python virtualenv active and dev dependencies installed you can do things like: - -```sh -# run all functional tests -uv run pytest --profile az_cli tests/functional/ -# run specific functional tests -uv run pytest --profile az_cli tests/functional/adapter/basic/* -# run all unit tests in a file -uv run pytest tests/unit/test_adapter.py -# run a specific unit test -uv run pytest test/unit/test_adapter.py::TestSparkAdapter::test_profile_with_database -``` -## Updating Docs - -Many changes will require and update to the `dbt-fabricspark` docs here are some useful resources. - -- Docs are [here](https://docs.getdbt.com/). -- The docs repo for making changes is located [here]( https://github.com/dbt-labs/docs.getdbt.com). -- The changes made are likely to impact one or both of [Fabric Spark Profile](https://docs.getdbt.com/reference/warehouse-profiles/fabricspark-profile), or [Saprk Configs](https://docs.getdbt.com/reference/resource-configs/spark-configs). -- We ask every community member who makes a user-facing change to open an issue or PR regarding doc changes. - -## Adding CHANGELOG Entry - -Changelogs are managed manually for now. As you raise a PR, provide the changes made in your commits. - -## Submitting a Pull Request - -Microsoft provides a CI environment to test changes to the `dbt-fabricspark` adapter, and periodic checks against the development version of `dbt-core` through Github Actions. - -A `dbt-fabricspark` maintainer will review your PR. They may suggest code revision for style or clarity, or request that you add unit or functional test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code. - -Once all requests and answers have been answered the `dbt-fabricspark` maintainer can trigger CI testing. - -Once all tests are passing and your PR has been approved, a `dbt-fabricspark` maintainer will merge your changes into the active development branch. And that's it! Happy developing :tada: + ```bash + npx nx run dbt-fabricspark:build + npx nx run dbt-fabricspark:test --output-style=stream + ``` \ No newline at end of file diff --git a/contrib/README.md b/contrib/README.md deleted file mode 100644 index adc2434e..00000000 --- a/contrib/README.md +++ /dev/null @@ -1,86 +0,0 @@ -# Contributing - -[![Deep-dive Walkthrough](https://rakirahman.blob.core.windows.net/public/images/Misc/dbt-fabricspark-contrib.png)](https://rakirahman.blob.core.windows.net/public/videos/dbt-fabricspark-local-development-deep-dive.mp4) - - -## How to use, on a Linux machine - -1. Windows pre-reqs - - ```powershell - winget install -e --id Microsoft.VisualStudioCode - ``` - -1. Get a fresh new WSL machine up: - - ```powershell - $GIT_ROOT = git rev-parse --show-toplevel - & "$GIT_ROOT\contrib\bootstrap-dev-env.ps1" - ``` - -1. Clone the repo, and open VSCode in it: - - ```bash - sudo mkdir -p /workspaces && sudo chmod 777 /workspaces && cd /workspaces - - read -p "Enter your name (e.g. 'FirstName LastName'): " user_name - read -p "Enter your email (e.g. 'your-alias@foo.com'): " user_email - read -p "Enter your git fork (e.g. 'https://github.com/microsoft/dbt-fabricspark.git'): " git_fork_url - read -p "Enter the existing branch to switch to: (e.g. 'main'): " branch_name - - git config --global user.name "$user_name" - git config --global user.email "$user_email" - git clone "$git_fork_url" - - cd /workspaces/dbt-fabricspark - git checkout "$branch_name" - - code . - ``` - -1. Run the bootstrapper script, that installs all tools idempotently: - - ```bash - GIT_ROOT=$(git rev-parse --show-toplevel) - chmod +x ${GIT_ROOT}/contrib/bootstrap-dev-env.sh && ${GIT_ROOT}/contrib/bootstrap-dev-env.sh - ``` - -1. Launch the devcontainer: - - ```bash - cd /workspaces/dbt-fabricspark - HEX=$(printf '%s' "$(wslpath -w .)" | xxd -ps -c 256) - code --folder-uri "vscode-remote://dev-container+${HEX}/workspaces/dbt-fabricspark" - ``` - -1. Install recommended developer tooling (optional): - - ```bash - curl -fsSL https://gh.io/copilot-install | bash - $HOME/.local/bin/copilot -i /login - ``` - -1. Login to github and ensure to authorize `Microsoft` if you're an employee (optional): - - ```bash - gh auth login - ``` - -1. Create a `test.env` file with the two Fabric workspace IDs and display names — you need `Contributor` on both. The functional test suite uses **two** workspaces: - - - **Workspace 1 (`WORKSPACE_ID_1` / `WORKSPACE_NAME_1`)** — the *primary* workspace where dbt models are materialized during tests. - - **Workspace 2 (`WORKSPACE_ID_2` / `WORKSPACE_NAME_2`)** — the *secondary* workspace that hosts a seeded fixture lakehouse, used to verify cross-workspace 4-part naming end-to-end. - - ```bash - GIT_ROOT=$(git rev-parse --show-toplevel) - read -p "Enter Fabric workspace 1 ID: " ws1_id - read -p "Enter Fabric workspace 1 display name: " ws1_name - read -p "Enter Fabric workspace 2 ID: " ws2_id - read -p "Enter Fabric workspace 2 display name: " ws2_name - { - echo "WORKSPACE_ID_1=${ws1_id}" - echo "WORKSPACE_NAME_1=${ws1_name}" - echo "WORKSPACE_ID_2=${ws2_id}" - echo "WORKSPACE_NAME_2=${ws2_name}" - } > "${GIT_ROOT}/test.env" - ``` \ No newline at end of file From c5c0e4679173beadf7e1dadb9eb94f38d14aae3c Mon Sep 17 00:00:00 2001 From: Raki Rahman Date: Thu, 14 May 2026 09:47:04 +0000 Subject: [PATCH 4/6] Update PR template --- .github/pull_request_template.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index e3fe87d2..25681bb8 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -4,6 +4,9 @@ > [!TIP] > This repo uses [Conventional Commit conventions](https://www.conventionalcommits.org/en/v1.0.0/) - please try to rename your PR headline to match it. +> [!WARNING] +> Please ensure to read through this whole set of instructions, specially the `Test` section. + # Why this change is needed Describe what issue this change is trying to address. @@ -22,4 +25,25 @@ Describe how the change works. # Test -- What tests have been run? Please describe any verification steps you used. \ No newline at end of file +## Important: Non-Microsoft Employee contributors + +If you are not a Microsoft employee with a `foo@microsoft.com` email, you will not be able to run CI as it runs in the `@microsoft.com` Fabric Tenant where you do not have access. + +In order for your PR to be considered for review, you **must** attach a clear screenshot of the output of you running the following command successfully: + +```bash +npx nx run dbt-fabricspark:test --output-style=stream +``` + +Here's an example of a successful run: + +![A successful CI run locally](https://rakirahman.blob.core.windows.net/public/images/Misc/dbt-fabricspark-ci-run-success.png) + +> ⚠️ Delete the above image and attach your own screenshot + +To keep the quality of the repo high, if you **do not** attach a screenshot of successful local testing, your PR will be promptly closed. + +## Microsoft Employee contributors + +Your PR will be subjected to full regression suite via GitHub Action. +It's highly recommended to run the tests locally so your contributions are promptly merged rather than failing in CI. \ No newline at end of file From 7703cef5155b8c8f20942d40f2c0720e0e823f1b Mon Sep 17 00:00:00 2001 From: Raki Rahman Date: Thu, 14 May 2026 09:48:32 +0000 Subject: [PATCH 5/6] Proofread --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 25681bb8..fa0ffcd0 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,5 +1,5 @@ > [!NOTE] -> Thank you for making change! Please consider filling this template for your pull request to improve quality of checkin message. +> Thank you for making change! Please fill this template for your pull request to improve quality of check-in message. > [!TIP] > This repo uses [Conventional Commit conventions](https://www.conventionalcommits.org/en/v1.0.0/) - please try to rename your PR headline to match it. From ccba973cb9b2d55a79e888a908fcb47072741ffd Mon Sep 17 00:00:00 2001 From: Raki Rahman Date: Thu, 14 May 2026 09:49:17 +0000 Subject: [PATCH 6/6] Proofread --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 51df8b8a..9be18ba0 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,8 @@ The `dbt-fabricspark` package contains all of the code enabling dbt to work with - [Install dbt](https://docs.getdbt.com/docs/installation) - Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/) +> To contribute to this adapter codebase, see [CONTRIBUTING.md](CONTRIBUTING.md). + ### Installation ```bash