module 19 solution

Author: amalF

module-19-ci-cd/solution/.github/worklows/ci.yml

@@ -0,0 +1,101 @@
+name: CI/CD Pipeline
+
+on:
+  push:
+    branches: [main]
+
+env:
+  ECR_REPOSITORY: beans-api
+  TASK_DEFINITION: beans-api
+  CONTAINER_NAME: beans-api
+  ECS_SERVICE: beans-api-service
+  ECS_CLUSTER: beans-api-cluster
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install ruff
+      - run: ruff check main.py quality_check.py
+
+  quality-check:
+    runs-on: ubuntu-latest
+    needs: lint
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: pip-deepchecks-${{ hashFiles('requirements-quality.txt') }}
+      - uses: actions/cache@v4
+        with:
+          path: ~/.cache/huggingface
+          key: hf-beans-vit
+      - run: pip install -r requirements-quality.txt
+      - name: Run Deepchecks quality gate
+        env:
+          TOKENIZERS_PARALLELISM: "false"
+        run: python quality_check.py
+
+  build:
+    runs-on: ubuntu-latest
+    needs: [lint, quality-check]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Login to ECR
+        uses: aws-actions/amazon-ecr-login@v2
+      - name: Build and push image
+        env:
+          ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }}
+        run: |
+          IMAGE_TAG="${GITHUB_SHA:0:8}-$(date +%s)"
+          IMAGE_URI="$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG"
+          docker build -t $IMAGE_URI .
+          docker push $IMAGE_URI
+          docker tag $IMAGE_URI $ECR_REGISTRY/$ECR_REPOSITORY:latest
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest
+
+  deploy:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Download current task definition
+        run: |
+          aws ecs describe-task-definition \
+            --task-definition ${{ env.TASK_DEFINITION }} \
+            --query taskDefinition > task-definition.json
+      - name: Render ECS task definition
+        id: task-def
+        uses: aws-actions/amazon-ecs-render-task-definition@v1
+        with:
+          task-definition: task-definition.json
+          container-name: ${{ env.CONTAINER_NAME }}
+          image: ${{ secrets.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:latest
+      - name: Deploy to ECS
+        uses: aws-actions/amazon-ecs-deploy-task-definition@v2
+        with:
+          task-definition: ${{ steps.task-def.outputs.task-definition }}
+          service: ${{ env.ECS_SERVICE }}
+          cluster: ${{ env.ECS_CLUSTER }}
+          wait-for-service-stability: true

module-19-ci-cd/solution/Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+RUN python -m spacy download en_core_web_sm
+
+COPY main.py .
+
+EXPOSE 8000
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

module-19-ci-cd/solution/main.py

@@ -0,0 +1,31 @@
+import spacy
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+
+app = FastAPI()
+
+nlp = spacy.load("en_core_web_sm")
+
+
+class TextRequest(BaseModel):
+    text: str
+
+
+@app.post("/predict")
+async def predict(request: TextRequest):
+    if not request.text.strip():
+        raise HTTPException(status_code=400, detail="Text cannot be empty")
+
+    doc = nlp(request.text)
+
+    return {
+        "entities": [
+            {"word": ent.text, "entity_group": ent.label_, "start": ent.start_char, "end": ent.end_char}
+            for ent in doc.ents
+        ]
+    }
+
+
+@app.get("/health")
+def health():
+    return {"status": "healthy"}

module-19-ci-cd/solution/quality_check.py

@@ -0,0 +1,90 @@
+"""
+Deepchecks quality check for the beans ViT model.
+"""
+
+import sys
+
+import numpy as np
+from datasets import load_dataset
+from deepchecks.core import CheckFailure
+from deepchecks.vision import VisionData
+from deepchecks.vision.suites import train_test_validation
+from deepchecks.vision.vision_data import BatchOutputFormat
+from torch.utils.data import DataLoader
+from torch.utils.data import Dataset as TorchDataset
+from transformers import pipeline
+
+MODEL_NAME = "nateraw/vit-base-beans"
+
+
+class BeansDataset(TorchDataset):
+    def __init__(self, hf_dataset):
+        self.dataset = hf_dataset
+
+    def __len__(self):
+        return len(self.dataset)
+
+    def __getitem__(self, idx):
+        return self.dataset[idx]
+
+
+def make_collate_fn(pipe, label_names):
+    def collate_fn(examples):
+        images_pil = [ex["image"].convert("RGB") for ex in examples]
+        labels = [ex["labels"] for ex in examples]
+        predictions_raw = pipe(images_pil)
+        proba = []
+        for preds in predictions_raw:
+            scores = {p["label"]: p["score"] for p in preds}
+            proba.append([scores.get(lbl, 0.0) for lbl in label_names])
+        images_np = [np.array(img) for img in images_pil]
+        return BatchOutputFormat(images=images_np, labels=labels, predictions=proba)
+    return collate_fn
+
+
+def build_vision_data(hf_split, pipe, label_names):
+    dataset = BeansDataset(load_dataset("beans", split=hf_split))
+    label_map = {i: lbl for i, lbl in enumerate(label_names)}
+    loader = DataLoader(
+        dataset,
+        batch_size=8,
+        collate_fn=make_collate_fn(pipe, label_names),
+    )
+    return VisionData(batch_loader=loader, task_type="classification", label_map=label_map)
+
+
+def main():
+    print(f"Loading model: {MODEL_NAME}...")
+    pipe = pipeline("image-classification", model=MODEL_NAME)
+
+    dataset = load_dataset("beans", split="validation")
+    label_names = dataset.features["labels"].names
+
+    print("Building VisionData...")
+    train_data = build_vision_data("train", pipe, label_names)
+    test_data = build_vision_data("test", pipe, label_names)
+
+    print("Running Deepchecks suite...")
+    suite = train_test_validation()
+    result = suite.run(train_data, test_data, max_samples=500)
+
+    print(f"{len(result.results)} checks executed.")
+
+    # As an example, the model quality check will fail if label drift score exceeds 0.5.
+    for check_result in result.results:
+        if isinstance(check_result, CheckFailure):
+            continue
+        if check_result.check.name() == "Label Drift":
+            drift_score = check_result.value["Samples Per Class"]["Drift score"]
+            if drift_score > 0.5:
+                print(f"  [FAIL] Label Drift: score {drift_score:.3f} exceeds threshold 0.5")
+                sys.exit(1)
+            else:
+                print(f"  [PASS] Label Drift: score {drift_score:.3f} ≤ 0.5")
+            break
+
+    print("\nAll checks passed.")
+
+
+if __name__ == "__main__":
+    main()

module-19-ci-cd/solution/requirements-quality.txt

@@ -0,0 +1,6 @@
+deepchecks>=0.18.0
+transformers>=4.44.0
+torch>=2.0.0
+datasets>=2.18.0
+numpy>=1.24.0
+anywidget

module-19-ci-cd/solution/requirements.txt

@@ -0,0 +1,4 @@
+fastapi==0.110.0
+uvicorn==0.27.1
+spacy>=3.7.0
+python-multipart==0.0.9