Automatically upload crawlers and update environment variables(Activity) (#18)

* add github action

* add github action

* add github action

* add github action

* add github action

* add github action

* add github action

* add github action

* add github action

* add github action

* add github action

Author: MH-limarco

.github/workflows/daily_activity-run.yml

@@ -26,20 +26,19 @@ jobs:
         -   name: Run daily activity script
             run: python crawler_testing.py
 
-        -   name: Upload activity_result.json to remote server
-            run: |
-                curl -X POST ${{secrets.UPDATE_ACT_INFO_URL}} \
-                -H "Content-Type: application/json" \
-                --data-binary @crawler/save/activity/activity_result.json
-                
+        - name: Upload activity_result.json to remote server
+          run: |
+              curl -v -X POST "${{secrets.UPDATE_ACT_INFO_URL}}?file_type=json" \
+                -F "file=@crawler/save/activity/activity_result.json"
 
         -   name: Update act_info
             run: |
-                curl -X POST ${{secrets.DIFY_BASE_URL}} \
+                curl -X POST ${{secrets.DIFY_BASE_URL}}/workflows/run \
                 --header 'Authorization: Bearer ${{secrets.UPDATE_ACT_INFO_API_KEY}}' \
                 --header 'Content-Type: application/json' \
                 --data-raw '{
                     "inputs": {},
                     "response_mode": "blocking",
+                    "user": "github"
                 }'
 

crawler/requirements.txt

@@ -1,4 +1,7 @@
 selenium
 colorlog
 bs4
-requests
+requests
+PyYAML
+tqdm
+requests

crawler_testing.py

@@ -22,5 +22,6 @@ def diff_time(start, name=""):
     END_STR = "=END="
 
     start_time = timer()
-    activity_crawler(URL, PATH, END_STR, max_worker=0, headless=True)
+    
+    activity_crawler(URL, PATH, END_STR, max_worker=4, headless=True)
     diff_time(start_time, "Code")