fix: ibm change max retries limit (#469) #2645
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: End-to-End Tests | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: [ main, release/* ] | |
| merge_group: | |
| branches: [ main ] | |
| concurrency: | |
| group: "${{ github.workflow }}-${{ github.ref }}" | |
| cancel-in-progress: true | |
| jobs: | |
| check_untagged_tests: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: run check | |
| run: | | |
| make check-untagged-tests | |
| api_based_int_test: | |
| strategy: | |
| matrix: | |
| test: ["partitioners", "chunkers"] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: 'actions/checkout@v4' | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: ./.github/actions/setup-environment | |
| with: | |
| python-version: "3.10" | |
| - name: Run ${{ matrix.test }} integration test | |
| env: | |
| UNSTRUCTURED_API_KEY: ${{ secrets.UNS_PAID_API_KEY }} | |
| UNSTRUCTURED_API_URL: "https://api.unstructuredapp.io" | |
| run: | | |
| make integration-test-${{ matrix.test }} | |
| make parse-skipped-tests | |
| embedders_int_test: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: 'actions/checkout@v4' | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: ./.github/actions/setup-environment | |
| with: | |
| python-version: "3.10" | |
| - name: Run ${{ matrix.test }} integration test | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| MXBAI_API_KEY: ${{ secrets.MXBAI_API_KEY }} | |
| TOGETHERAI_API_KEY: ${{ secrets.TOGETHERAI_API_KEY }} | |
| VOYAGEAI_API_KEY: ${{ secrets.VOYAGEAI_API_KEY }} | |
| VERTEXAI_API_KEY: ${{ secrets.VERTEXAI_API_KEY }} | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} | |
| run: | | |
| make integration-test-embedders | |
| make parse-skipped-tests | |
| blob_storage_connectors_int_test: | |
| runs-on: ubuntu-latest-m | |
| needs: [ check_untagged_tests ] | |
| steps: | |
| - uses: 'actions/checkout@v4' | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: ./.github/actions/setup-environment | |
| with: | |
| python-version: "3.10" | |
| - name: Get full Python version | |
| id: full-python-version | |
| run: echo version=$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") >> $GITHUB_OUTPUT | |
| - name: Setup up docker | |
| run: | | |
| sudo make install-docker-compose | |
| docker compose version | |
| - name: Run Integration Tests | |
| env: | |
| # Databricks | |
| DATABRICKS_HOST: ${{secrets.DATABRICKS_HOST}} | |
| DATABRICKS_CATALOG: ${{secrets.DATABRICKS_CATALOG}} | |
| DATABRICKS_CLIENT_ID: ${{secrets.DATABRICKS_CLIENT_ID}} | |
| DATABRICKS_CLIENT_SECRET: ${{secrets.DATABRICKS_CLIENT_SECRET}} | |
| DATABRICKS_PAT: ${{ secrets.DATABRICKS_PAT }} | |
| # Onedrive | |
| MS_CLIENT_CRED: ${{ secrets.MS_CLIENT_CRED }} | |
| MS_CLIENT_ID: ${{ secrets.MS_CLIENT_ID }} | |
| MS_TENANT_ID: ${{ secrets.MS_TENANT_ID }} | |
| MS_USER_EMAIL: ${{ secrets.MS_USER_EMAIL }} | |
| MS_USER_PNAME: ${{ secrets.MS_USER_PNAME }} | |
| # S3 | |
| S3_INGEST_TEST_ACCESS_KEY: ${{ secrets.S3_INGEST_TEST_ACCESS_KEY }} | |
| S3_INGEST_TEST_SECRET_KEY: ${{ secrets.S3_INGEST_TEST_SECRET_KEY }} | |
| # Sharepoint | |
| SHAREPOINT_CLIENT_ID: ${{ secrets.SHAREPOINT_CLIENT_ID }} | |
| SHAREPOINT_CRED: ${{ secrets.SHAREPOINT_CRED }} | |
| run : | | |
| make integration-test-connectors-blob-storage | |
| make parse-skipped-tests | |
| sql_connectors_int_test: | |
| runs-on: ubuntu-latest-m | |
| needs: [ check_untagged_tests ] | |
| steps: | |
| - uses: 'actions/checkout@v4' | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: ./.github/actions/setup-environment | |
| with: | |
| python-version: "3.10" | |
| - name: Get full Python version | |
| id: full-python-version | |
| run: echo version=$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") >> $GITHUB_OUTPUT | |
| - name: Setup up docker | |
| run: | | |
| sudo make install-docker-compose | |
| docker compose version | |
| - name: Run Integration Tests | |
| env: | |
| # Motherduck | |
| # MOTHERDUCK_TOKEN: ${{ secrets.MOTHERDUCK_TOKEN }} | |
| # Snowflake | |
| # LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} | |
| # Delta Tables | |
| S3_INGEST_TEST_ACCESS_KEY: ${{ secrets.S3_INGEST_TEST_ACCESS_KEY }} | |
| S3_INGEST_TEST_SECRET_KEY: ${{ secrets.S3_INGEST_TEST_SECRET_KEY }} | |
| # Databricks Delta Tables | |
| DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_SERVER_HOSTNAME }} | |
| DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH }} | |
| DATABRICKS_ACCESS_TOKEN: ${{ secrets.DATABRICKS_ACCESS_TOKEN }} | |
| run : | | |
| make integration-test-connectors-sql | |
| make parse-skipped-tests | |
| nosql_connectors_int_test: | |
| runs-on: ubuntu-latest-m | |
| needs: [ check_untagged_tests ] | |
| steps: | |
| - uses: 'actions/checkout@v4' | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: ./.github/actions/setup-environment | |
| with: | |
| python-version: "3.10" | |
| - name: Get full Python version | |
| id: full-python-version | |
| run: echo version=$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") >> $GITHUB_OUTPUT | |
| - name: Setup up docker | |
| run: | | |
| sudo make install-docker-compose | |
| docker compose version | |
| - name: Run Integration Tests | |
| env: | |
| # MongoDB | |
| MONGODB_URI: ${{ secrets.MONGODB_URI }} | |
| MONGODB_DATABASE: ${{ secrets.MONGODB_DATABASE_NAME }} | |
| # Redis | |
| AZURE_REDIS_INGEST_TEST_PASSWORD: ${{ secrets.AZURE_REDIS_INGEST_TEST_PASSWORD }} | |
| # Vectara | |
| VECTARA_OAUTH_CLIENT_ID: ${{secrets.VECTARA_OAUTH_CLIENT_ID}} | |
| VECTARA_OAUTH_SECRET: ${{secrets.VECTARA_OAUTH_SECRET}} | |
| VECTARA_CUSTOMER_ID: ${{secrets.VECTARA_CUSTOMER_ID}} | |
| run : | | |
| make integration-test-connectors-nosql | |
| make parse-skipped-tests | |
| vector_db_connectors_int_test: | |
| runs-on: ubuntu-latest-m | |
| needs: [ check_untagged_tests ] | |
| steps: | |
| - uses: 'actions/checkout@v4' | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: ./.github/actions/setup-environment | |
| with: | |
| python-version: "3.10" | |
| - name: Get full Python version | |
| id: full-python-version | |
| run: echo version=$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") >> $GITHUB_OUTPUT | |
| - name: Setup up docker | |
| run: | | |
| sudo make install-docker-compose | |
| docker compose version | |
| - name: Run Integration Tests | |
| env: | |
| # AstraDB | |
| ASTRA_DB_APPLICATION_TOKEN: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN }} | |
| ASTRA_DB_API_ENDPOINT: ${{ secrets.ASTRA_DB_ENDPOINT }} | |
| # Azure AI Search | |
| AZURE_SEARCH_API_KEY: ${{ secrets.AZURE_SEARCH_API_KEY }} | |
| # LanceDB | |
| AZURE_DEST_CONNECTION_STR: ${{ secrets.AZURE_DEST_CONNECTION_STR }} | |
| S3_INGEST_TEST_ACCESS_KEY: ${{ secrets.S3_INGEST_TEST_ACCESS_KEY }} | |
| S3_INGEST_TEST_SECRET_KEY: ${{ secrets.S3_INGEST_TEST_SECRET_KEY }} | |
| GCP_INGEST_SERVICE_KEY: ${{ secrets.GCP_INGEST_SERVICE_KEY }} | |
| # Pinecone | |
| PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }} | |
| run : | | |
| make integration-test-connectors-vector-db | |
| make parse-skipped-tests | |
| uncategorized_connectors_int_test: | |
| runs-on: ubuntu-latest-m | |
| needs: [ check_untagged_tests ] | |
| steps: | |
| - uses: 'actions/checkout@v4' | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: ./.github/actions/setup-environment | |
| with: | |
| python-version: "3.10" | |
| - name: Get full Python version | |
| id: full-python-version | |
| run: echo version=$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") >> $GITHUB_OUTPUT | |
| - name: Setup up docker | |
| run: | | |
| sudo make install-docker-compose | |
| docker compose version | |
| - name: Run Integration Tests | |
| env: | |
| # Discord | |
| DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }} | |
| DISCORD_CHANNELS: ${{ secrets.DISCORD_CHANNELS }} | |
| # Kafka | |
| KAFKA_API_KEY: ${{ secrets.KAFKA_API_KEY }} | |
| KAFKA_SECRET: ${{ secrets.KAFKA_SECRET }} | |
| KAFKA_BOOTSTRAP_SERVER: ${{ secrets.KAFKA_BOOTSTRAP_SERVER }} | |
| # Confluence | |
| CONFLUENCE_USER_EMAIL: ${{secrets.CONFLUENCE_USER_EMAIL}} | |
| CONFLUENCE_API_TOKEN: ${{secrets.CONFLUENCE_API_TOKEN}} | |
| # Notion | |
| NOTION_API_KEY: ${{ secrets.NOTION_API_KEY }} | |
| # Jira | |
| JIRA_INGEST_API_TOKEN: ${{ secrets.JIRA_INGEST_API_TOKEN }} | |
| JIRA_INGEST_USER_EMAIL: ${{ secrets.JIRA_INGEST_USER_EMAIL }} | |
| # Zendesk | |
| ZENDESK_TOKEN: ${{ secrets.ZENDESK_TOKEN }} | |
| run : | | |
| make integration-test-connectors-uncategorized | |
| make parse-skipped-tests | |
| src_e2e_test: | |
| runs-on: ubuntu-latest-m | |
| steps: | |
| # actions/checkout MUST come before auth | |
| - uses: 'actions/checkout@v4' | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: ./.github/actions/setup-environment | |
| with: | |
| python-version: "3.10" | |
| - name: Get full Python version | |
| id: full-python-version | |
| run: echo version=$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") >> $GITHUB_OUTPUT | |
| - name: Test (end-to-end) | |
| env: | |
| AIRTABLE_PERSONAL_ACCESS_TOKEN: ${{ secrets.AIRTABLE_PERSONAL_ACCESS_TOKEN }} | |
| BOX_APP_CONFIG: ${{ secrets.BOX_APP_CONFIG }} | |
| DATABRICKS_HOST: ${{secrets.DATABRICKS_HOST}} | |
| DATABRICKS_CATALOG: ${{secrets.DATABRICKS_CATALOG}} | |
| DATABRICKS_CLIENT_ID: ${{secrets.DATABRICKS_CLIENT_ID}} | |
| DATABRICKS_CLIENT_SECRET: ${{secrets.DATABRICKS_CLIENT_SECRET}} | |
| DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }} | |
| DROPBOX_APP_KEY: ${{ secrets.DROPBOX_APP_KEY }} | |
| DROPBOX_APP_SECRET: ${{ secrets.DROPBOX_APP_SECRET }} | |
| DROPBOX_REFRESH_TOKEN: ${{ secrets.DROPBOX_REFRESH_TOKEN }} | |
| GCP_INGEST_SERVICE_KEY: ${{ secrets.GCP_INGEST_SERVICE_KEY }} | |
| GH_READ_ONLY_ACCESS_TOKEN: ${{ secrets.GH_READ_ONLY_ACCESS_TOKEN }} | |
| HUBSPOT_API_TOKEN: ${{ secrets.HUBSPOT_API_TOKEN }} | |
| JIRA_INGEST_API_TOKEN: ${{ secrets.JIRA_INGEST_API_TOKEN }} | |
| JIRA_INGEST_USER_EMAIL: ${{ secrets.JIRA_INGEST_USER_EMAIL }} | |
| MS_CLIENT_CRED: ${{ secrets.MS_CLIENT_CRED }} | |
| MS_CLIENT_ID: ${{ secrets.MS_CLIENT_ID }} | |
| MS_TENANT_ID: ${{ secrets.MS_TENANT_ID }} | |
| MS_USER_EMAIL: ${{ secrets.MS_USER_EMAIL }} | |
| MS_USER_PNAME: ${{ secrets.MS_USER_PNAME }} | |
| SALESFORCE_USERNAME: ${{secrets.SALESFORCE_USERNAME}} | |
| SALESFORCE_CONSUMER_KEY: ${{secrets.SALESFORCE_CONSUMER_KEY}} | |
| SALESFORCE_PRIVATE_KEY: ${{secrets.SALESFORCE_PRIVATE_KEY}} | |
| SHAREPOINT_CLIENT_ID: ${{secrets.SHAREPOINT_CLIENT_ID}} | |
| SHAREPOINT_CRED: ${{secrets.SHAREPOINT_CRED}} | |
| SHAREPOINT_SITE: ${{secrets.SHAREPOINT_SITE}} | |
| SHAREPOINT_PERMISSIONS_APP_ID: ${{secrets.SHAREPOINT_PERMISSIONS_APP_ID}} | |
| SHAREPOINT_PERMISSIONS_APP_CRED: ${{secrets.SHAREPOINT_PERMISSIONS_APP_CRED}} | |
| SHAREPOINT_PERMISSIONS_TENANT: ${{secrets.SHAREPOINT_PERMISSIONS_TENANT}} | |
| SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} | |
| UNS_PAID_API_KEY: ${{ secrets.UNS_PAID_API_KEY }} | |
| NOTION_API_KEY: ${{ secrets.NOTION_API_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AZURE_SEARCH_ENDPOINT: ${{ secrets.AZURE_SEARCH_ENDPOINT }} | |
| AZURE_SEARCH_API_KEY: ${{ secrets.AZURE_SEARCH_API_KEY }} | |
| PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }} | |
| ASTRA_DB_APPLICATION_TOKEN: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN }} | |
| ASTRA_DB_API_ENDPOINT: ${{ secrets.ASTRA_DB_ENDPOINT }} | |
| TABLE_OCR: "tesseract" | |
| OCR_AGENT: "unstructured.partition.utils.ocr_models.tesseract_ocr.OCRAgentTesseract" | |
| CI: "true" | |
| run: | | |
| sudo make install-docker-compose | |
| docker compose version | |
| pip freeze | |
| ./test_e2e/test-src.sh | |
| src_api_test: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # actions/checkout MUST come before auth | |
| - uses: 'actions/checkout@v4' | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: ./.github/actions/setup-environment | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Get full Python version | |
| id: full-python-version | |
| run: echo version=$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") >> $GITHUB_OUTPUT | |
| - name: Install limited dependencies | |
| run: | | |
| uv sync --frozen --extra remote | |
| - name: Run test against remote API | |
| env: | |
| UNS_PAID_API_KEY: ${{ secrets.UNS_PAID_API_KEY }} | |
| run: | | |
| ./test_e2e/src/against-api.sh | |
| dest_e2e_test: | |
| environment: ci | |
| runs-on: ubuntu-latest-m | |
| steps: | |
| # actions/checkout MUST come before auth | |
| - uses: 'actions/checkout@v4' | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: ./.github/actions/setup-environment | |
| with: | |
| python-version: "3.10" | |
| - name: Get full Python version | |
| id: full-python-version | |
| run: echo version=$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") >> $GITHUB_OUTPUT | |
| - name: Test (end-to-end) | |
| env: | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| S3_INGEST_TEST_ACCESS_KEY: ${{ secrets.S3_INGEST_TEST_ACCESS_KEY }} | |
| S3_INGEST_TEST_SECRET_KEY: ${{ secrets.S3_INGEST_TEST_SECRET_KEY }} | |
| AZURE_SEARCH_ENDPOINT: ${{ secrets.AZURE_SEARCH_ENDPOINT }} | |
| AZURE_SEARCH_API_KEY: ${{ secrets.AZURE_SEARCH_API_KEY }} | |
| AZURE_REDIS_INGEST_TEST_PASSWORD: ${{ secrets.AZURE_REDIS_INGEST_TEST_PASSWORD }} | |
| BOX_APP_CONFIG: ${{ secrets.BOX_APP_CONFIG }} | |
| DROPBOX_APP_KEY: ${{ secrets.DROPBOX_APP_KEY }} | |
| DROPBOX_APP_SECRET: ${{ secrets.DROPBOX_APP_SECRET }} | |
| DROPBOX_REFRESH_TOKEN: ${{ secrets.DROPBOX_REFRESH_TOKEN }} | |
| GCP_INGEST_SERVICE_KEY: ${{ secrets.GCP_INGEST_SERVICE_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| AZURE_DEST_CONNECTION_STR: ${{ secrets.AZURE_DEST_CONNECTION_STR }} | |
| PINECONE_API_KEY: ${{secrets.PINECONE_API_KEY}} | |
| VECTARA_OAUTH_CLIENT_ID: ${{secrets.VECTARA_OAUTH_CLIENT_ID}} | |
| VECTARA_OAUTH_SECRET: ${{secrets.VECTARA_OAUTH_SECRET}} | |
| VECTARA_CUSTOMER_ID: ${{secrets.VECTARA_CUSTOMER_ID}} | |
| ASTRA_DB_APPLICATION_TOKEN: ${{secrets.ASTRA_DB_APPLICATION_TOKEN}} | |
| ASTRA_DB_API_ENDPOINT: ${{secrets.ASTRA_DB_ENDPOINT}} | |
| CLARIFAI_API_KEY: ${{secrets.CLARIFAI_API_KEY}} | |
| SHAREPOINT_CLIENT_ID: ${{secrets.SHAREPOINT_CLIENT_ID}} | |
| SHAREPOINT_CRED: ${{secrets.SHAREPOINT_CRED}} | |
| KDBAI_BEARER_TOKEN: ${{ secrets.KDBAI_BEARER_TOKEN }} | |
| KDBAI_USERNAME: ${{ secrets.KDBAI_USERNAME }} | |
| KDBAI_CIPHER_KEY: ${{ secrets.KDBAI_CIPHER_KEY }} | |
| UNS_PAID_API_KEY: ${{ secrets.UNS_PAID_API_KEY }} | |
| TABLE_OCR: "tesseract" | |
| OCR_AGENT: "unstructured.partition.utils.ocr_models.tesseract_ocr.OCRAgentTesseract" | |
| CI: "true" | |
| run: | | |
| sudo make install-docker-compose | |
| docker compose version | |
| pip freeze | |
| ./test_e2e/test-dest.sh |