testing after secrets #2
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy Scraper to Prefect | |
| on: | |
| push: | |
| branches: | |
| - main | |
| jobs: | |
| deploy: | |
| runs-on: ubuntu-latest | |
| services: | |
| elasticsearch: | |
| image: docker.elastic.co/elasticsearch/elasticsearch:8.17.3 | |
| ports: | |
| - 9200:9200 | |
| env: | |
| discovery.type: single-node | |
| ELASTIC_PASSWORD: ${{ secrets.ELASTIC_PASSWORD }} | |
| xpack.security.enabled: "false" | |
| options: >- | |
| --name es01 | |
| --health-cmd="curl --silent --fail localhost:9200/_cluster/health || exit 1" | |
| --health-interval=10s | |
| --health-timeout=5s | |
| --health-retries=10 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Wait for Elasticsearch to be ready | |
| run: | | |
| echo "Waiting for Elasticsearch..." | |
| until curl -s -u elastic:${{ secrets.ELASTIC_PASSWORD }} http://localhost:9200 >/dev/null; do | |
| sleep 5 | |
| done | |
| echo "Elasticsearch is up!" | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Build Docker image | |
| run: docker build -t ${{ secrets.DOCKER_USERNAME }}/scraping-image:latest . | |
| - name: Push Docker image | |
| run: docker push ${{ secrets.DOCKER_USERNAME }}/scraping-image:latest | |
| - name: Deploy flow to Prefect | |
| env: | |
| PREFECT_API_KEY: ${{ secrets.PREFECT_API_KEY }} | |
| PREFECT_API_URL: https://api.prefect.cloud/api/accounts/${{ secrets.PREFECT_ACCOUNT }}/workspaces/${{ secrets.PREFECT_WORKSPACE }} | |
| run: prefect deploy --prefect-file deployment.yaml | |
| - name: Elastic indexing | |
| run: | | |
| python elastic_script.py | |
| echo "Events indexed to Elasticsearch successfully." | |
| - name: Wait for indexing | |
| run: | | |
| echo "Waiting for events to be indexed..." | |
| sleep 100 # Adjust the sleep time as necessary | |
| - name: All indexed | |
| run: | | |
| echo "All events have been indexed to Elasticsearch." | |
| curl -X GET "localhost:9200/events_index/_search?pretty" -u elastic:${{ secrets.ELASTIC_PASSWORD }} -H 'Content-Type: application/json' -d' | |
| { | |
| "query": { | |
| "match_all": {} | |
| }, | |
| "size": 10 | |
| }' | |
| - name: Run FastAPI | |
| run: | | |
| uvicorn main:app --host 0.0.0.0 --port 8000 & | |
| sleep 5 # wait a few seconds for server to start | |
| - name: Test streaming endpoint | |
| run: | | |
| curl -v http://localhost:8000/events | |
| echo "Streaming endpoint tested successfully." | |