SAC-30337: Add dynamic schema workflow with discovery support (#1)

* SAC-30337: Add dynamic schema workflow with discovery support

* SAC-30337: Resolve PR code review comments

- Fix grammar: 'due permissions' -> 'due to permissions' in sync.py
- Fix update_currently_syncing to use state.pop() avoiding null state write
- Update type hint to Optional[str] on update_currently_syncing
- Use tap_stream_id consistently and add parent-stream fallback in child_map
- Pass pre-built child_map to write_schema to avoid O(n) catalog re-traversal
- Handle 204 No Content and empty bodies in _make_request to avoid JSON decode errors
- Guard int() cast in _get_retry_after with ValueError fallback for HTTP-date headers

* Remove parent-filter keys from automatic fields

* Update auth mechanism to basic-auth with fallback to OAuth

* Update required config keys

* Update logging

* SAC-30338: Add sync logic for the tap (#2)

* SAC-30338: Add sync logic for the tap along with unittests and spikes

* Update tests to use unittests

* Resolve Copilot comments

* Add discovery time stream probe and prune strategy

* Remove redundant files and code lines

* Integration tests (#3)

* Add Integration tests for the tap

* Update integration testsand yml file

* Resolve Copilot Comments

* Add retry on dynamic probe

Author: satyendra101

.circleci/config.yml

@@ -12,17 +12,12 @@ jobs:
             source /usr/local/share/virtualenvs/tap-sap-success-factors/bin/activate
             uv pip install -U setuptools
             uv pip install .[dev]
-      - run:
-          name: "JSON Validator"
-          command: |
-            source /usr/local/share/virtualenvs/tap-tester/bin/activate
-            stitch-validate-json tap_branch/schemas/*.json
       - run:
           name: "pylint"
           command: |
             source /usr/local/share/virtualenvs/tap-sap-success-factors/bin/activate
             uv pip install pylint
-            pylint tap_branch -d C,R,W
+            pylint tap_sap_success_factors -d C,R,W
       - add_ssh_keys
       - run:
           name: "Unit Tests"
@@ -40,7 +35,7 @@ jobs:
           command: |
             source /usr/local/share/virtualenvs/tap-tester/bin/activate
             uv pip install --upgrade awscli
-            aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/tap_tester_sandbox dev_env.sh
+            aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/vm dev_env.sh
             source dev_env.sh
             unset USE_STITCH_BACKEND
             run-test --tap=tap-sap-success-factors tests

README.md

@@ -34,12 +34,10 @@ Example config:
 
 ```json
 {
-  "api_endpoint": "https://<company>.successfactors.com",
-  "oauth_endpoint": "https://<company>.successfactors.com/oauth/token",
+  "api_server": "https://<company>.successfactors.com",
   "client_id": "...",
   "company_id": "...",
   "assertion": "...",
-  "grant_type": "...",
   "start_date": "2024-01-01T00:00:00Z",
   "request_timeout": 300
 }

sample_config.json

@@ -1,11 +1,8 @@
 {
     "client_id": "your_client_id",
     "user_id": "your_user_id",
-    "oauth_endpoint": "oauth_endpoint_url",
     "company_id": "your_company_id",
-    "api_endpoint": "api_endpoint_url",
-    "grant_type": "grant_type",
+    "api_server": "api_server_url",
     "start_date": "2024-01-01T00:00:00Z",
-    "lookback_window_days": 1,
     "assertion": "saml_assertion"
 }

spikes/generate_saml_assertion.py

@@ -0,0 +1,192 @@
+""" Module to generate a SAML assertion for SuccessFactors API authentication using the OAuth Bearer flow.
+    Following dependencies are required:
+        - lxml
+        - signxml
+        - cryptography (for signxml)
+        - python-dotenv (for loading environment variables from a .env file)
+    Environment variables expected in .env file:
+        - SF_CLIENT_ID: SuccessFactors API client_id
+        - SF_USERNAME: SuccessFactors username
+        - SF_TOKEN_URL: SuccessFactors token endpoint URL (e.g. https://api4.successfactors.com/sf/oauth/token)
+        - SF_PRIVATE_KEY_FILE: Path to the private key file (PEM format) used for signing the assertion (default: private_key.pem)
+        - SF_CERT_FILE: Path to the certificate file (PEM format) corresponding to the private key (default: certificate.pem)
+
+    Run the script:
+        - python generate_saml_assertion.py
+    The output will be a base64 encoded SAML assertion that can be used in the OAuth
+    Bearer flow to request an access token.
+
+    NOTE: This workflow will be accommodated in the SuccessFactors API client library,
+    so this script is primarily for demonstration and testing purposes.
+"""
+
+import base64
+import os
+import uuid
+from datetime import datetime, timedelta, timezone
+
+from dotenv import load_dotenv
+from lxml import etree
+from signxml import XMLSigner, methods
+
+# ==============================
+# CONFIGURATION
+# ==============================
+
+load_dotenv()
+
+CLIENT_ID = os.getenv("SF_CLIENT_ID")  # SuccessFactors API client_id
+USERNAME = os.getenv("SF_USERNAME")      # SuccessFactors username
+TOKEN_URL = os.getenv("SF_TOKEN_URL")    # SuccessFactors token endpoint URL (e.g. https://api4.successfactors.com/sf/oauth/token)
+
+PRIVATE_KEY_FILE = os.getenv("SF_PRIVATE_KEY_FILE", "private_key.pem")
+CERT_FILE = os.getenv("SF_CERT_FILE", "certificate.pem")
+
+
+def generate_saml_assertion():
+
+    now = datetime.now(timezone.utc)
+    issue_instant = now.strftime("%Y-%m-%dT%H:%M:%SZ")
+    not_on_or_after = (now + timedelta(minutes=30)).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    assertion_id = "_" + str(uuid.uuid4())
+
+    NSMAP = {
+        "saml2": "urn:oasis:names:tc:SAML:2.0:assertion",
+        "ds": "http://www.w3.org/2000/09/xmldsig#",
+        "xs": "http://www.w3.org/2001/XMLSchema",
+        "xsi": "http://www.w3.org/2001/XMLSchema-instance",
+    }
+
+    assertion = etree.Element(
+        "{urn:oasis:names:tc:SAML:2.0:assertion}Assertion",
+        nsmap=NSMAP,
+        ID=assertion_id,
+        Version="2.0",
+        IssueInstant=issue_instant,
+    )
+
+    # ----------------------
+    # Issuer (must be client_id)
+    # ----------------------
+    issuer = etree.SubElement(
+        assertion,
+        "{urn:oasis:names:tc:SAML:2.0:assertion}Issuer"
+    )
+    issuer.text = CLIENT_ID
+
+    # ----------------------
+    # Subject (userId mode)
+    # ----------------------
+    subject = etree.SubElement(
+        assertion,
+        "{urn:oasis:names:tc:SAML:2.0:assertion}Subject"
+    )
+
+    name_id = etree.SubElement(
+        subject,
+        "{urn:oasis:names:tc:SAML:2.0:assertion}NameID",
+        Format="urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified"
+    )
+    name_id.text = USERNAME
+
+    subject_confirmation = etree.SubElement(
+        subject,
+        "{urn:oasis:names:tc:SAML:2.0:assertion}SubjectConfirmation",
+        Method="urn:oasis:names:tc:SAML:2.0:cm:bearer"
+    )
+
+    etree.SubElement(
+        subject_confirmation,
+        "{urn:oasis:names:tc:SAML:2.0:assertion}SubjectConfirmationData",
+        NotOnOrAfter=not_on_or_after,
+        Recipient=TOKEN_URL,
+        InResponseTo=CLIENT_ID
+    )
+
+    # ----------------------
+    # Conditions
+    # ----------------------
+    conditions = etree.SubElement(
+        assertion,
+        "{urn:oasis:names:tc:SAML:2.0:assertion}Conditions",
+        NotBefore=issue_instant,
+        NotOnOrAfter=not_on_or_after,
+    )
+
+    audience_restriction = etree.SubElement(
+        conditions,
+        "{urn:oasis:names:tc:SAML:2.0:assertion}AudienceRestriction"
+    )
+
+    audience = etree.SubElement(
+        audience_restriction,
+        "{urn:oasis:names:tc:SAML:2.0:assertion}Audience"
+    )
+    audience.text = TOKEN_URL
+
+    # ----------------------
+    # Required AttributeStatement
+    # ----------------------
+    attribute_statement = etree.SubElement(
+        assertion,
+        "{urn:oasis:names:tc:SAML:2.0:assertion}AttributeStatement"
+    )
+
+    def add_attribute(name, value):
+        attr = etree.SubElement(
+            attribute_statement,
+            "{urn:oasis:names:tc:SAML:2.0:assertion}Attribute",
+            Name=name
+        )
+        attr_value = etree.SubElement(
+            attr,
+            "{urn:oasis:names:tc:SAML:2.0:assertion}AttributeValue"
+        )
+        attr_value.set(
+            "{http://www.w3.org/2001/XMLSchema-instance}type",
+            "xs:string"
+        )
+        attr_value.text = value
+
+    add_attribute("api_key", CLIENT_ID)
+    add_attribute("use_username", "false")
+    add_attribute("external_user", "false")
+
+    # ----------------------
+    # SIGN ASSERTION (SHA256)
+    # ----------------------
+    with open(PRIVATE_KEY_FILE, "rb") as key_file:
+        private_key = key_file.read()
+
+    with open(CERT_FILE, "rb") as cert_file:
+        cert = cert_file.read()
+
+    signer = XMLSigner(
+        method=methods.enveloped,
+        signature_algorithm="rsa-sha256",
+        digest_algorithm="sha256",
+        c14n_algorithm="http://www.w3.org/2001/10/xml-exc-c14n#"
+    )
+
+    signed_assertion = signer.sign(
+        assertion,
+        key=private_key,
+        cert=cert,
+        reference_uri=assertion_id
+    )
+
+    saml_xml = etree.tostring(
+        signed_assertion,
+        xml_declaration=True,
+        encoding="UTF-8"
+    )
+
+    return base64.b64encode(saml_xml).decode("utf-8")
+
+
+if __name__ == "__main__":
+    saml_assertion = generate_saml_assertion()
+
+    print("\nBase64 Encoded SAML Assertion:\n")
+    print(saml_assertion)

tap_sap_success_factors/__init__.py

@@ -0,0 +1,43 @@
+import json
+import sys
+
+import singer
+
+from tap_sap_success_factors.client import SAPSuccessFactorsClient
+from tap_sap_success_factors.discover import discover
+from tap_sap_success_factors.sync import sync
+
+LOGGER = singer.get_logger()
+
+REQUIRED_CONFIG_KEYS = ["client_id", "user_id", "company_id",
+                        "username", "password", "api_server", "start_date"]
+
+
+def do_discover(client: SAPSuccessFactorsClient):
+    """Discover and emit the catalog."""
+    LOGGER.info("Starting discover")
+    catalog = discover(client)
+    json.dump(catalog.to_dict(), sys.stdout, indent=2)
+    LOGGER.info("Finished discover")
+
+
+@singer.utils.handle_top_exception(LOGGER)
+def main():
+    """Run the tap entrypoint."""
+    parsed_args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS)
+    state = parsed_args.state or {}
+
+    with SAPSuccessFactorsClient(parsed_args.config) as client:
+        if parsed_args.discover:
+            do_discover(client)
+        elif parsed_args.catalog:
+            sync(
+                client=client,
+                config=parsed_args.config,
+                catalog=parsed_args.catalog,
+                state=state,
+            )
+
+
+if __name__ == "__main__":
+    main()

tap_sap_success_factors/auth.py

@@ -0,0 +1,50 @@
+import base64
+from typing import Dict, Optional
+
+from singer import get_logger
+
+LOGGER = get_logger()
+
+
+def build_basic_auth_header(config: Dict) -> Optional[str]:
+    """Return a ``Basic <base64>`` authorization header value when the config
+    contains ``username`` and ``password``.
+
+    The encoded token is ``base64(username:password)`` per RFC 7617.
+    Returns ``None`` when the required keys are absent so callers can fall
+    back to the OAuth / SAML bearer flow.
+    """
+    username = config.get("username")
+    password = config.get("password")
+    if username and password:
+        token = base64.b64encode(f"{username}:{password}".encode()).decode()
+        LOGGER.info("Using HTTP Basic authentication mechanism with provided username and password.")
+        return f"Basic {token}"
+    return None
+
+
+def build_token_request(config: Dict) -> Dict:
+    """Build OAuth token request payload.
+
+    Supported modes:
+    - static token in `access_token`
+    - SAML bearer flow with `saml_assertion`
+    """
+    if config.get("access_token"):
+        return {}
+
+    payload = {
+        "client_id": config.get("client_id"),
+        "company_id": config.get("company_id"),
+        "grant_type": "urn:ietf:params:oauth:grant-type:saml2-bearer",
+        "assertion": config.get("assertion") or config.get("saml_assertion"),
+    }
+
+    if config.get("refresh_token"):
+        payload = {
+            "client_id": config.get("client_id"),
+            "grant_type": "refresh_token",
+            "refresh_token": config.get("refresh_token"),
+        }
+
+    return {k: v for k, v in payload.items() if v is not None}