diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e3e2de9 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +*.sh text eol=lf +*.yml text eol=lf +*.bicep text eol=lf +*.tf text eol=lf diff --git a/.github/workflows/run-samples.yml b/.github/workflows/run-samples.yml new file mode 100644 index 0000000..31f37bb --- /dev/null +++ b/.github/workflows/run-samples.yml @@ -0,0 +1,126 @@ +name: Samples CI + +# Theory of Operation: +# This workflow automates the testing of Azure sample applications against the LocalStack Azure emulator. +# It follows the best practices from the localstack-pro repository: +# 1. Parallel Testing: Splits the sample suite into shards to reduce execution time. +# 2. Standardized Tooling: Uses a Makefile for environment setup and test orchestration. +# 3. Cloud Emulation: Configures the Azure CLI to target the LocalStack emulator. + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +on: + pull_request: + branches: [ main ] + workflow_dispatch: + +jobs: + scripts: + name: "Run Test Scripts (amd64) — Part ${{ matrix.shard }} of ${{ matrix.splits }}" + environment: AZURE + strategy: + fail-fast: false + matrix: + shard: [1, 2] + splits: [2] + runs-on: ubuntu-latest + + env: + IMAGE_NAME: localstack/localstack-azure-alpha + DEFAULT_TAG: latest + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Set up environment + run: echo "AZURE_CONFIG_DIR=${{ runner.temp }}/azure-cli" >> $GITHUB_ENV + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '9.0' + + - name: Install System Dependencies + # Essential tools for script execution, app packaging, and database connectivity. + # jq: for parsing JSON responses from Azure CLI. + # zip: for packaging function/web apps. + # unixodbc-dev & libsnappy-dev: required for Python database drivers (pyodbc, pymongo). + run: | + sudo apt-get update + sudo apt-get install -y jq zip unixodbc-dev libsnappy-dev + find . -name "*.sh" -exec chmod +x {} + + + - name: Install test dependencies + # Mirroring the localstack-pro approach: install all Python dependencies + # (including the localstack CLI) into a virtual environment to avoid system-level conflicts. + run: make install + + - name: Login to Docker Hub + # Mandatory login to Docker Hub to benefit from higher rate limits for authenticated pulls. + # This prevents '429 Too Many Requests' errors during the pull of large emulator images. + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_PULL_USERNAME }} + password: ${{ secrets.DOCKERHUB_PULL_TOKEN }} + + - name: Free up disk space + # Azure emulator images are large. Pruning unused Docker objects ensures enough + # disk space is available on the GitHub runner for image pulls and sidecar containers. + run: | + docker system prune -af --volumes + docker builder prune -af + + - name: Pull LocalStack Azure Image + # Explicitly pull the image before starting. This mirrors the "Build Docker Image" + # step in localstack-pro and ensures the pull logic is separated from the start logic. + run: docker pull ${{ env.IMAGE_NAME }}:${{ env.DEFAULT_TAG }} + + - name: Start LocalStack + # Run the emulator in detached mode using the virtual environment. + # We use 'python -m localstack.cli.main' to ensure the correct CLI version from the venv is used. + run: | + source .venv/bin/activate + python -m localstack.cli.main start -d + python -m localstack.cli.main wait -t 120 + env: + IMAGE_NAME: ${{ env.IMAGE_NAME }}:${{ env.DEFAULT_TAG }} + LOCALSTACK_AUTH_TOKEN: ${{ secrets.TEST_LOCALSTACK_AUTH_TOKEN }} + DOCKER_FLAGS: "-e MSSQL_ACCEPT_EULA=Y" + LS_LOG: "DEBUG" + DISABLE_EVENTS: "1" + ACTIVATE_PRO: "1" + DNS_ADDRESS: "0" + + - name: Install Azure Functions Core Tools + # Required for publishing function app samples to the emulator. + run: npm install -g azure-functions-core-tools@4 --unsafe-perm true + + - name: Install MSSQL ODBC and Tools + # Required for the 'web-app-sql-database' sample which uses 'sqlcmd' to + # initialize and verify the database schema in the local emulator. + run: | + curl https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc + curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list + sudo apt-get update + sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 mssql-tools18 + echo "/opt/mssql-tools18/bin" >> $GITHUB_PATH + + - name: Run Test Scripts + # Executes the sharded test suite. Each shard runs a subset of samples in parallel. + run: make test SHARD=${{ matrix.shard }} SPLITS=${{ matrix.splits }} + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.TEST_LOCALSTACK_AUTH_TOKEN }} + + - name: Get LocalStack Logs + # Captured on failure or success to provide a detailed audit trail of the emulator's activity. + if: always() + run: make logs diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..fac6445 --- /dev/null +++ b/Makefile @@ -0,0 +1,31 @@ +VENV_BIN = python3 -m venv +VENV_DIR ?= .venv +VENV_ACTIVATE = $(VENV_DIR)/bin/activate +VENV_RUN = . $(VENV_ACTIVATE) + +ifeq ($(OS),Windows_NT) + VENV_ACTIVATE = $(VENV_DIR)/Scripts/activate + VENV_RUN = $(VENV_DIR)/Scripts/activate +endif + +venv: $(VENV_ACTIVATE) + +$(VENV_ACTIVATE): pyproject.toml + test -d $(VENV_DIR) || $(VENV_BIN) $(VENV_DIR) + $(VENV_RUN); pip install --upgrade pip setuptools wheel + touch $(VENV_ACTIVATE) + +install: venv ## Install dependencies + $(VENV_RUN); pip install -r requirements-dev.txt + chmod +x run-samples.sh + +SHARD ?= 1 +SPLITS ?= 1 + +test: venv ## Run all samples + $(VENV_RUN); bash ./run-samples.sh $(SHARD) $(SPLITS) + +logs: venv ## Get LocalStack logs + $(VENV_RUN); localstack logs + +.PHONY: venv install test logs diff --git a/README.md b/README.md index 6d7e93b..edaf852 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,29 @@ Each sample project is organized by Azure service and includes: - Step-by-step deployment guides and tutorials. - Optionally, testing and validation scripts. +## Local Testing + +To validate all samples locally, you can run the same test suite used in the CI. This script will start LocalStack, configure the Azure CLI cloud profile, and execute the deployment and test scripts for each sample. + +```bash +cd localstack-azure-samples + +# Set your LOCALSTACK_AUTH_TOKEN +export LOCALSTACK_AUTH_TOKEN= + +# Or create a .env file: +# echo "LOCALSTACK_AUTH_TOKEN=" > .env + +./run-samples.sh +``` + +### Troubleshooting: Line Endings +If you encounter errors like `invalid option name` or `: command not found` when running on Linux/WSL, it's likely due to Windows-style line endings (CRLF). You can fix this by running: +```bash +find . -name "*.sh" -exec sed -i 's/\r$//' {} + +``` +Or by installing and using `dos2unix`. + ## Configuration Follow the comprehensive setup guide in [LocalStack for Azure Quick Start](./docs/LOCALSTACK.md) to configure your LocalStack for Azure development environment. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2e37f44 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = ['setuptools>=64', 'wheel'] +build-backend = "setuptools.build_meta" + +[project] +name = "localstack-azure-samples" +version = "0.1.0" +description = "Samples for LocalStack Azure" +requires-python = ">=3.10" +dependencies = [ + "flask", + "pyodbc", + "pymongo", + "azure-functions", + "azure-identity", + "azure-storage-blob", + "azure-mgmt-cosmosdb", + "azure-core", + "python-dotenv", + "localstack", + "azlocal", + "terraform-local", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0", + "pytest-xdist", + "pytest-timeout", + "pytest-rerunfailures", +] diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..b57188b --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,5 @@ +-r requirements-runtime.txt +pytest +pytest-xdist +pytest-timeout +pytest-rerunfailures diff --git a/requirements-runtime.txt b/requirements-runtime.txt new file mode 100644 index 0000000..573e0d0 --- /dev/null +++ b/requirements-runtime.txt @@ -0,0 +1,12 @@ +flask +pyodbc +pymongo +azure-functions +azure-identity +azure-storage-blob +azure-mgmt-cosmosdb +azure-core +python-dotenv +localstack +azlocal +terraform-local diff --git a/run-samples.sh b/run-samples.sh new file mode 100644 index 0000000..d38150e --- /dev/null +++ b/run-samples.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Helper script to run all sample tests locally, replicating the CI environment. +# Requirements: +# - Docker +# - Python 3.12+ +# - .NET 9.0+ +# - Node.js & npm +# - Azure CLI (az) +# - LocalStack CLI +# - azlocal & terraform-local (pip install azlocal terraform-local) +# - funclocal (pip install funclocal) +# - Azure Functions Core Tools (func) +# - jq & zip (sudo apt-get install jq zip) +# - MSSQL Tools (sqlcmd) +# - LOCALSTACK_AUTH_TOKEN environment variable + +# 0. Load environment variables from .env file if it exists +if [ -f .env ]; then + echo "Loading environment variables from .env file..." + # Use a subshell to avoid exporting everything if not needed, + # but here we actually want them in the environment. + set -a + source .env + set +a +fi + +# 1. Check for required tools +command -v localstack >/dev/null 2>&1 || { echo >&2 "localstack CLI is required but not installed. Aborting."; exit 1; } +command -v az >/dev/null 2>&1 || { echo >&2 "az CLI is required but not installed. Aborting."; exit 1; } +command -v azlocal >/dev/null 2>&1 || { echo >&2 "azlocal is required but not installed. Run 'pip install azlocal'. Aborting."; exit 1; } +command -v funclocal >/dev/null 2>&1 || { echo >&2 "funclocal is required but not installed. Run 'pip install azlocal'. Aborting."; exit 1; } +command -v tflocal >/dev/null 2>&1 || { echo >&2 "tflocal is required but not installed. Run 'pip install terraform-local'. Aborting."; exit 1; } +command -v func >/dev/null 2>&1 || { echo >&2 "Azure Functions Core Tools (func) is required but not installed. Aborting."; exit 1; } + +if [ -z "${LOCALSTACK_AUTH_TOKEN:-}" ]; then + echo "Error: LOCALSTACK_AUTH_TOKEN is not set. It is required for the Azure emulator." + exit 1 +fi + +# 1. Start LocalStack +if ! localstack status | grep -q "running"; then + echo "Starting LocalStack Azure emulator..." + IMAGE_NAME=localstack/localstack-azure-alpha localstack start -d + localstack wait -t 60 +else + echo "LocalStack is already running." +fi + +# 2. Configure Azure CLI for LocalStack +echo "Configuring Azure CLI for LocalStack..." +if [ -n "${AZURE_CONFIG_DIR:-}" ]; then + mkdir -p "$AZURE_CONFIG_DIR" +fi + +if command -v azlocal >/dev/null 2>&1; then + azlocal login || true + azlocal start_interception +else + az login --service-principal -u any-app -p any-pass --tenant any-tenant || true +fi + + +# 3. Define Samples +SAMPLES=( + "samples/function-app-front-door/python|bash scripts/deploy_all.sh --name-prefix testafd --use-localstack|" + "samples/function-app-managed-identity/python|bash scripts/user-managed-identity.sh|bash scripts/validate.sh && bash scripts/test.sh" + "samples/function-app-storage-http/dotnet|bash scripts/deploy.sh|bash scripts/validate.sh && bash scripts/call-http-triggers.sh" + "samples/web-app-cosmosdb-mongodb-api/python|bash scripts/deploy.sh|bash scripts/validate.sh && bash scripts/call-web-app.sh" + "samples/web-app-managed-identity/python|bash scripts/user-assigned.sh|bash scripts/validate.sh && bash scripts/call-web-app.sh" + "samples/web-app-sql-database/python|bash scripts/deploy.sh|bash scripts/validate.sh && bash scripts/get-web-app-url.sh" +) + +# 4. Calculate Shard +TOTAL=${#SAMPLES[@]} +SHARD=${1:-1} +SPLITS=${2:-1} + +COUNT=$(( TOTAL / SPLITS )) +START=$(( (SHARD - 1) * COUNT )) + +if [ "$SHARD" -eq "$SPLITS" ]; then + COUNT=$(( TOTAL - START )) +fi + +echo "Running samples shard $SHARD of $SPLITS (index $START, count $COUNT)" + +# 5. Run Samples +for (( i=START; i /dev/null + + echo "Deploying..." + eval "$deploy" + + if [ -n "$test" ]; then + echo "Testing..." + eval "$test" + fi + + popd > /dev/null + echo "Completed: $path" + + # Cleanup Docker resources after each test to free up disk space + echo "Cleaning up Docker resources..." + docker system prune -af --volumes || true + echo "" +done + +echo "All samples completed successfully!" diff --git a/samples/function-app-front-door/python/scripts/.last_deploy_all.env b/samples/function-app-front-door/python/scripts/.last_deploy_all.env new file mode 100644 index 0000000..69f3cff --- /dev/null +++ b/samples/function-app-front-door/python/scripts/.last_deploy_all.env @@ -0,0 +1,10 @@ +RESOURCE_GROUP="rg-testafd-30236" +PROFILE_NAME="afd-testafd-30236" +EP_BASIC="ep-testafd-basic-30236" +EP_MULTI="ep-testafd-multi-30236" +EP_SPEC="ep-testafd-spec-30236" +EP_RULES="ep-testafd-rules-30236" +EP_STATE="ep-testafd-state-30236" +FUNC_MAIN="fa-testafd-30236" +FUNC_A="fa-testafda-30236" +FUNC_B="fa-testafdb-30236" diff --git a/samples/function-app-front-door/python/scripts/deploy_all.sh b/samples/function-app-front-door/python/scripts/deploy_all.sh index 96b4799..27fa508 100644 --- a/samples/function-app-front-door/python/scripts/deploy_all.sh +++ b/samples/function-app-front-door/python/scripts/deploy_all.sh @@ -212,7 +212,7 @@ create_function_app() { STORAGE_KEY=$(az storage account keys list -g "$RESOURCE_GROUP" -n "$storageName" --query "[0].value" -o tsv) if [[ -z "$STORAGE_KEY" ]]; then echo "Failed to get storage key for $storageName" >&2; exit 1; fi local STORAGE_CONNECTION_STRING - STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=https;AccountName=$storageName;AccountKey=$STORAGE_KEY;BlobEndpoint=https://$storageName.blob.localhost.localstack.cloud:4566;QueueEndpoint=https://$storageName.queue.localhost.localstack.cloud:4566;TableEndpoint=https://$storageName.table.localhost.localstack.cloud:4566;FileEndpoint=https://$storageName.file.localhost.localstack.cloud:4566" + STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=$storageName;AccountKey=$STORAGE_KEY;BlobEndpoint=http://$storageName.blob.localhost.localstack.cloud:4566;QueueEndpoint=http://$storageName.queue.localhost.localstack.cloud:4566;TableEndpoint=http://$storageName.table.localhost.localstack.cloud:4566;FileEndpoint=http://$storageName.file.localhost.localstack.cloud:4566" az functionapp config appsettings set -g "$RESOURCE_GROUP" -n "$funcName" \ --settings AzureWebJobsStorage="$STORAGE_CONNECTION_STRING" WEBSITE_CONTENTAZUREFILECONNECTIONSTRING="$STORAGE_CONNECTION_STRING" SCM_RUN_FROM_PACKAGE= -o none fi @@ -228,7 +228,7 @@ publish_function_code() { echo "Error: Azure Functions Core Tools ('func') not found in PATH." >&2; exit 1 fi pushd "$FUNCTION_SRC" >/dev/null - funclocal azure functionapp publish "$funcName" --python --build local --verbose --debug + funclocal azure functionapp publish "$funcName" --python --build local #--verbose --debug popd >/dev/null else rm -f "$zipPath"; ( cd "$FUNCTION_SRC" && zip -rq "$zipPath" . ) diff --git a/samples/function-app-managed-identity/python/images/azure-storage-explorer.png:Zone.Identifier b/samples/function-app-managed-identity/python/images/azure-storage-explorer.png:Zone.Identifier deleted file mode 100644 index d6c1ec6..0000000 Binary files a/samples/function-app-managed-identity/python/images/azure-storage-explorer.png:Zone.Identifier and /dev/null differ diff --git a/samples/function-app-storage-http/dotnet/scripts/deploy.sh b/samples/function-app-storage-http/dotnet/scripts/deploy.sh index bdacc47..49e5b0c 100644 --- a/samples/function-app-storage-http/dotnet/scripts/deploy.sh +++ b/samples/function-app-storage-http/dotnet/scripts/deploy.sh @@ -150,6 +150,10 @@ fi # CD into the function app directory cd ../src/sample || exit -# Publish the function app echo "Publishing function app [$FUNCTION_APP_NAME]..." -$FUNC azure functionapp publish $FUNCTION_APP_NAME --dotnet-isolated --verbose --debug \ No newline at end of file +if [[ $ENVIRONMENT == "LocalStack" ]]; then + # Disable proxy for NuGet during build to avoid proxy interference + NO_PROXY="api.nuget.org,*.nuget.org" no_proxy="api.nuget.org,*.nuget.org" $FUNC azure functionapp publish $FUNCTION_APP_NAME --dotnet-isolated #--verbose --debug +else + $FUNC azure functionapp publish $FUNCTION_APP_NAME --dotnet-isolated #--verbose --debug +fi \ No newline at end of file diff --git a/samples/web-app-sql-database/python/scripts/deploy.sh b/samples/web-app-sql-database/python/scripts/deploy.sh index fc1c538..d38d0ae 100755 --- a/samples/web-app-sql-database/python/scripts/deploy.sh +++ b/samples/web-app-sql-database/python/scripts/deploy.sh @@ -144,12 +144,13 @@ fi # Create server-level login echo "Creating login [$DATABASE_USER_NAME] at server level..." sqlcmd -S "$SQL_SERVER_FQDN" \ - -d master \ - -U "$ADMIN_USER" \ - -P "$ADMIN_PASSWORD" \ - -Q "IF NOT EXISTS (SELECT * FROM sys.sql_logins WHERE name = '$DATABASE_USER_NAME') - CREATE LOGIN [$DATABASE_USER_NAME] WITH PASSWORD = '$DATABASE_USER_PASSWORD';" \ - -V 1 + -d master \ + -U "$ADMIN_USER" \ + -P "$ADMIN_PASSWORD" \ + -N -C \ + -Q "IF NOT EXISTS (SELECT * FROM sys.sql_logins WHERE name = '$DATABASE_USER_NAME') + CREATE LOGIN [$DATABASE_USER_NAME] WITH PASSWORD = '$DATABASE_USER_PASSWORD';" \ + -V 1 if [ $? -eq 0 ]; then echo "Login [$DATABASE_USER_NAME] created successfully" @@ -161,12 +162,13 @@ fi # Create database user echo "Creating user [$DATABASE_USER_NAME] in database [$SQL_DATABASE_NAME]..." sqlcmd -S "$SQL_SERVER_FQDN" \ - -d "$SQL_DATABASE_NAME" \ - -U "$ADMIN_USER" \ - -P "$ADMIN_PASSWORD" \ - -Q "IF NOT EXISTS (SELECT * FROM sys.database_principals WHERE name = '$DATABASE_USER_NAME') - CREATE USER [$DATABASE_USER_NAME] FOR LOGIN [$DATABASE_USER_NAME];" \ - -V 1 + -d "$SQL_DATABASE_NAME" \ + -U "$ADMIN_USER" \ + -P "$ADMIN_PASSWORD" \ + -N -C \ + -Q "IF NOT EXISTS (SELECT * FROM sys.database_principals WHERE name = '$DATABASE_USER_NAME') + CREATE USER [$DATABASE_USER_NAME] FOR LOGIN [$DATABASE_USER_NAME];" \ + -V 1 if [ $? -eq 0 ]; then echo "User [$DATABASE_USER_NAME] created successfully in database [$SQL_DATABASE_NAME]" @@ -181,6 +183,7 @@ sqlcmd -S "$SQL_SERVER_FQDN" \ -d "$SQL_DATABASE_NAME" \ -U "$ADMIN_USER" \ -P "$ADMIN_PASSWORD" \ + -N -C \ -Q "ALTER ROLE db_datareader ADD MEMBER [$DATABASE_USER_NAME]; ALTER ROLE db_datawriter ADD MEMBER [$DATABASE_USER_NAME]; ALTER ROLE db_ddladmin ADD MEMBER [$DATABASE_USER_NAME];" \ @@ -199,6 +202,7 @@ sqlcmd -S "$SQL_SERVER_FQDN" \ -d "$SQL_DATABASE_NAME" \ -U "$DATABASE_USER_NAME" \ -P "$DATABASE_USER_PASSWORD" \ + -N -C \ -Q "SELECT SYSTEM_USER AS CurrentUser, DB_NAME() AS CurrentDatabase, GETDATE() AS CurrentTime;" \ -V 1 @@ -215,6 +219,7 @@ sqlcmd -S "$SQL_SERVER_FQDN" \ -d "$SQL_DATABASE_NAME" \ -U "$DATABASE_USER_NAME" \ -P "$DATABASE_USER_PASSWORD" \ + -N -C \ -Q "IF NOT EXISTS (SELECT * FROM sys.tables WHERE name = 'Activities' AND schema_id = SCHEMA_ID('dbo')) CREATE TABLE dbo.Activities ( -- Primary Key: UNIQUEIDENTIFIER with a default of a new sequential GUID (best for indexing) @@ -244,6 +249,7 @@ sqlcmd -S "$SQL_SERVER_FQDN" \ -d "$SQL_DATABASE_NAME" \ -U "$DATABASE_USER_NAME" \ -P "$DATABASE_USER_PASSWORD" \ + -N -C \ -Q "INSERT INTO Activities (username, activity, timestamp) VALUES ('paolo', 'Go to Paris', GETDATE()), @@ -264,6 +270,7 @@ sqlcmd -S "$SQL_SERVER_FQDN" \ -d "$SQL_DATABASE_NAME" \ -U "$DATABASE_USER_NAME" \ -P "$DATABASE_USER_PASSWORD" \ + -N -C \ -Q "SELECT * FROM Activities;" \ -V 1