1st commit

Author: Chaimaekit

.github/workflows/build.yml

@@ -0,0 +1,94 @@
+name: Deploy Scraper to Prefect
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    services:
+      elasticsearch:
+        image: docker.elastic.co/elasticsearch/elasticsearch:8.17.3
+        ports:
+          - 9200:9200
+        env:
+          discovery.type: single-node
+          ELASTIC_PASSWORD: ${{ secrets.ELASTIC_PASSWORD }}
+          xpack.security.enabled: "false"
+        options: >-
+          --name es01
+          --health-cmd="curl --silent --fail localhost:9200/_cluster/health || exit 1"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=10
+
+    steps:
+      - uses: actions/checkout@v3
+      
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Wait for Elasticsearch to be ready
+        run: |
+          echo "Waiting for Elasticsearch..."
+          until curl -s -u elastic:${{ secrets.ELASTIC_PASSWORD }} http://localhost:9200 >/dev/null; do
+            sleep 5
+          done
+          echo "Elasticsearch is up!"
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build Docker image
+        run: docker build -t ${{ secrets.DOCKER_USERNAME }}/scraping-image:latest .
+
+      - name: Push Docker image
+        run: docker push ${{ secrets.DOCKER_USERNAME }}/scraping-image:latest
+
+      - name: Deploy flow to Prefect
+        env:
+          PREFECT_API_KEY: ${{ secrets.PREFECT_API_KEY }}
+          PREFECT_API_URL: https://api.prefect.cloud/api/accounts/${{ secrets.PREFECT_ACCOUNT }}/workspaces/${{ secrets.PREFECT_WORKSPACE }}
+        run: prefect deploy --prefect-file deployment.yaml
+
+      - name: Elastic indexing
+        run: |
+         python elastic_script.py
+         echo "Events indexed to Elasticsearch successfully."
+      - name: Wait for indexing
+        run: |
+          echo "Waiting for events to be indexed..."
+          sleep 100 # Adjust the sleep time as necessary
+          
+      - name: All indexed
+        run: |
+          echo "All events have been indexed to Elasticsearch."
+          curl -X GET "localhost:9200/events_index/_search?pretty" -u elastic:${{ secrets.ELASTIC_PASSWORD }} -H 'Content-Type: application/json' -d'
+          {
+            "query": {
+              "match_all": {}
+            },
+            "size": 10
+          }'
+      - name: Run FastAPI
+        run: |
+          uvicorn main:app --host 0.0.0.0 --port 8000 &
+          sleep 5  # wait a few seconds for server to start
+
+      - name: Test streaming endpoint
+        run: |
+          curl -v http://localhost:8000/events
+          echo "Streaming endpoint tested successfully."
+        

.gitignore

@@ -0,0 +1,4 @@
+__pycache__/
+.venv
+.env
+text.json

Dockerfile

@@ -0,0 +1,18 @@
+# Use official Python 3.12 slim image
+FROM python:3.12-slim
+
+# Set working directory
+WORKDIR /app
+
+# Copy requirements and install dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy all source code
+COPY . .
+
+# Expose FastAPI port
+EXPOSE 8000
+
+# Run the FastAPI app with uvicorn
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"]

README.md

@@ -0,0 +1 @@
+# Events_Project

deployment.yaml

@@ -0,0 +1,16 @@
+version: 1
+pull:
+  - prefect.deployments.steps.git_clone:
+      repository: https://github.com/Chaimaekit/Events_Project.git
+      branch: main
+deployments:
+  - name: print-events
+    entrypoint: workflow.py:print_events
+    work_pool:
+      name: my-process-pool
+
+    infrastructure:
+      image: chaimaaeljerrar/scraping-image:latest
+    schedules:
+      - cron: "*/2 * * * *"
+        timezone: UTC

docker-compose.yml

@@ -0,0 +1,31 @@
+version: '3.9'
+
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.12.1
+    container_name: elasticsearch
+    environment:
+      - discovery.type=single-node
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - xpack.security.enabled=false
+    ports:
+      - "9200:9200"
+    networks:
+      - backend
+
+  fastapi:
+    build: .
+    container_name: fastapi_app
+    ports:
+      - "8000:8000"
+    depends_on:
+      - elasticsearch
+    networks:
+      - backend
+    environment:
+      - ELASTICSEARCH_HOST=http://elasticsearch:9200
+
+networks:
+  backend:
+    driver: bridge
+

elastic_script.py

@@ -0,0 +1,65 @@
+from elasticsearch import Elasticsearch
+from into_db.connection import get_db_events
+from scrape.casaevents import get_casa_events
+from scrape.eventbrit import get_event_brit
+from scrape.eventsma import get_events_ma
+from scrape.guichet import get_guichet
+from dotenv import load_dotenv
+import os
+import time
+
+
+
+load_dotenv()
+elastic_password = os.getenv("ELASTIC_PASSWORD")
+
+
+
+def indexing():
+    es = Elasticsearch("http://localhost:9200", basic_auth=("elastic", f"{elastic_password}"))
+    events = []
+    for _ in range(20):
+        if es.ping():
+            print("Connected to Elasticsearch.")
+            break
+        print("Waiting for Elasticsearch to be ready...")
+        time.sleep(1)
+    else:
+        raise ValueError("Elasticsearch not available after waiting 20 seconds.")
+
+
+    es.indices.create(index="events_index", ignore=400)
+    casa_events = get_casa_events()
+    event_brit = get_event_brit()
+    events_ma = get_events_ma()
+    guichet = get_guichet()
+    events.extend(casa_events)
+    events.extend(event_brit)
+    events.extend(events_ma)
+    events.extend(guichet)
+    for event in events:
+        doc = {
+            "name": event.get("name", ""),
+            "date": event.get("date", ""),
+            "location": event.get("city", "")+" "+event.get("place", ""),
+            "description": event.get("description", ""),
+            "url": event.get("url", "")
+        }
+        es.index(index="events_index", document=doc)
+    print("Indexing succeded !")
+
+def check_doc(index_name, event):
+    es = Elasticsearch("http://localhost:9200", basic_auth=("elastic", f"{elastic_password}"))
+    result = es.search(index=index_name, body={
+        "query": {
+            "match":{
+                "name":event
+            }
+        },
+        "size": 20 
+    })
+    if result:
+        return result["hits"]["hits"]
+    
+if __name__ == "__main__":
+    indexing()