build.yml

name: Deploy Scraper to Prefect

on:
  push:
    branches:
      - main

jobs:
  deploy:
    runs-on: ubuntu-latest
    services:
      elasticsearch:
        image: docker.elastic.co/elasticsearch/elasticsearch:8.17.3
        ports:
          - 9200:9200
        env:
          discovery.type: single-node
          ELASTIC_PASSWORD: ${{ secrets.ELASTIC_PASSWORD }}
          xpack.security.enabled: "false"
        options: >-
          --name es01
          --health-cmd="curl --silent --fail localhost:9200/_cluster/health || exit 1"
          --health-interval=10s
          --health-timeout=5s
          --health-retries=10

    steps:
      - uses: actions/checkout@v3
      
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.10'
      
      - name: Install dependencies
        run: |
          pip install --upgrade pip
          pip install -r requirements.txt

      - name: Wait for Elasticsearch to be ready
        run: |
          echo "Waiting for Elasticsearch..."
          until curl -s -u elastic:${{ secrets.ELASTIC_PASSWORD }} http://localhost:9200 >/dev/null; do
            sleep 5
          done
          echo "Elasticsearch is up!"

      - name: Log in to Docker Hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Build Docker image
        run: docker build -t ${{ secrets.DOCKER_USERNAME }}/scraping-image:latest .

      - name: Push Docker image
        run: docker push ${{ secrets.DOCKER_USERNAME }}/scraping-image:latest

      - name: Deploy flow to Prefect
        env:
          PREFECT_API_KEY: ${{ secrets.PREFECT_API_KEY }}
          PREFECT_API_URL: https://api.prefect.cloud/api/accounts/${{ secrets.PREFECT_ACCOUNT }}/workspaces/${{ secrets.PREFECT_WORKSPACE }}
        run: prefect deploy --prefect-file deployment.yaml

      - name: Elastic indexing
        run: |
         python elastic_script.py
         echo "Events indexed to Elasticsearch successfully."
      - name: Wait for indexing
        run: |
          echo "Waiting for events to be indexed..."
          sleep 100 # Adjust the sleep time as necessary
          
      - name: All indexed
        run: |
          echo "All events have been indexed to Elasticsearch."
          curl -X GET "localhost:9200/events_index/_search?pretty" -u elastic:${{ secrets.ELASTIC_PASSWORD }} -H 'Content-Type: application/json' -d'
          {
            "query": {
              "match_all": {}
            },
            "size": 10
          }'
      - name: Run FastAPI
        run: |
          uvicorn main:app --host 0.0.0.0 --port 8000 &
          sleep 5  # wait a few seconds for server to start

      - name: Test streaming endpoint
        run: |
          curl -v http://localhost:8000/events
          echo "Streaming endpoint tested successfully."