From 66278c54a747b38c76ae19d68852e6fbd36b4078 Mon Sep 17 00:00:00 2001 From: Ishan Arora <105799425+inshane13@users.noreply.github.com> Date: Sun, 30 Aug 2026 10:16:39 +0530 Subject: [PATCH 1/9] docs: enable ReadTheDocs versioning for multi-version support --- .readthedocs.yaml | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 9a1fb7cfb..1240f6aab 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,27 +9,30 @@ build: os: ubuntu-22.04 tools: python: "3.12" - # You can also specify other tool versions: - # nodejs: "20" - # rust: "1.70" - # golang: "1.20" # Build documentation in the "docs/" directory with Sphinx sphinx: configuration: python/docs/conf.py - # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs - # builder: "dirhtml" - # Fail on all warnings to avoid broken references - # fail_on_warning: true -# Optionally build your docs in additional formats such as PDF and ePub -# formats: -# - pdf -# - epub +# Enable versioning - builds tags as versions +versions: + # Build all tags matching v*.*.* pattern as versions + patterns: + - 'v*.*.*' + # Also build the latest branch + branches: + - main + - master # Optional but recommended, declare the Python requirements required # to build your documentation # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html python: - install: + install: - requirements: python/requirements.txt + +# Build docs in multiple formats +formats: + - htmlzip + - pdf + - epub \ No newline at end of file From 1a1a565843f7d8c24a13d58693cf52fc5c8f45a7 Mon Sep 17 00:00:00 2001 From: Ishan Arora <105799425+inshane13@users.noreply.github.com> Date: Sun, 30 Aug 2026 10:58:34 +0530 Subject: [PATCH 2/9] docs: comprehensive update for new docs site (issue #1363) - Enable ReadTheDocs versioning in .readthedocs.yaml with tag patterns and multiple formats - Update version references from 0.6.0 to 0.7.0 across all platform guides - Add comprehensive Enterprise content to platform guides: - AWS EMR: Full Community + Enterprise guide with JAR setup, Python packages, phases - GCP Dataproc: Enterprise tab with EArguments, EZinggWithSpark, blocking strategies - Microsoft Fabric: Enterprise tab with 7-notebook workflow, EC/ES packages - Snowflake: Complete Enterprise Snowflake guide with Snowpark, native execution - Clean up placeholder comments: - Remove 22 'IMAGE TO BE ADDED' placeholders - Remove 'CONTENT TO BE GIVEN', 'TO BE ADDED', 'CHECK WITH SONAL' internal notes - Remove 'VIDEO TO BE ADDED' placeholder - Fix broken links (e.g., /broken/pages/... -> proper references) - Add missing frontmatter descriptions to 75+ markdown files - Update Snowflake properties page with detailed parameter docs and key-pair auth - Update install-zingg.md with AWS Glue, GCP Dataproc, Azure Synapse tabs - Add GitHub Actions workflow for docs validation (markdown links, Sphinx build, frontmatter, version consistency) - Remove TODO comments from verify-blocking.md and zingg-concepts README --- .github/markdown-link-check-config.json | 26 ++ .github/scripts/validate_frontmatter.py | 128 +++++++ .github/workflows/docs-validation.yml | 98 ++++++ docs/StandardisePostprocessor.md | 5 + docs/Transform.md | 5 + docs/accuracy/stopWordsRemoval.md | 5 + docs/approval.md | 2 + docs/bizLeaderSurvey.md | 5 + docs/community.md | 2 + .../connect-cloud-storage.md | 35 +- .../connect-redshift.md | 5 + .../connect-your-data/connect-file-formats.md | 11 +- docs/connectors/jdbc/clickhouse.md | 5 + docs/connectors/jdbc/mysql.md | 5 + docs/connectors/jdbc/postgres.md | 5 + docs/contributing.md | 2 + docs/dataSourcesAndSinks/amazonS3.md | 5 + docs/dataSourcesAndSinks/bigquery.md | 5 + docs/dataSourcesAndSinks/cassandra.md | 2 + docs/dataSourcesAndSinks/connectors.md | 2 + docs/dataSourcesAndSinks/databricks.md | 5 + docs/dataSourcesAndSinks/exasol.md | 2 + docs/dataSourcesAndSinks/fabric.md | 5 + docs/dataSourcesAndSinks/jdbc.md | 5 + docs/dataSourcesAndSinks/mongodb.md | 2 + docs/dataSourcesAndSinks/neo4j.md | 2 + docs/dataSourcesAndSinks/parquet.md | 2 + docs/dataSourcesAndSinks/pipes.md | 2 + .../explainoutput/stats/cluster-statistics.md | 5 + .../stats/configuring-output-statistics.md | 5 + .../explainoutput/stats/summary-statistics.md | 5 + docs/faq.md | 2 + docs/households.md | 5 + .../stopwordsremoval/stopwordsreco.md | 5 + docs/lookup.md | 5 + docs/macOSSetup.md | 5 + docs/patient.md | 5 + .../platform-guide-for-aws-emr.md | 219 +++++++++++- .../platform-guide-for-aws-glue.md | 17 +- .../platform-guide-for-azure-databricks.md | 14 +- .../platform-guide-for-gcp-dataproc.md | 166 ++++++++- .../platform-guide-for-microsoft-fabric.md | 321 ++++++++++++++++++ .../platform-guide-for-snowflake.md | 168 ++++++++- docs/pretrainedModels.md | 2 + docs/pythonEC/markdown/ApproverArguments.md | 5 + docs/pythonEC/markdown/EArguments.md | 5 + docs/pythonEC/markdown/EFieldDefinition.md | 5 + .../pythonEC/markdown/IncrementalArguments.md | 5 + docs/pythonEC/markdown/MappingMatchType.md | 5 + docs/pythonEC/markdown/epipes.md | 5 + docs/pythonEC/markdown/index.md | 5 + docs/pythonEC/markdown/zinggEC.md | 5 + docs/pythonES/markdown/ESparkClient.md | 5 + docs/pythonES/markdown/index.md | 5 + docs/pythonES/markdown/zinggES.md | 5 + .../_autosummary/zingg.client.Arguments.md | 5 + .../zingg.client.ClientOptions.md | 5 + .../zingg.client.FieldDefinition.md | 5 + .../_autosummary/zingg.client.Zingg.md | 5 + .../zingg.client.ZinggWithSpark.md | 5 + .../_autosummary/zingg.pipes.BigQueryPipe.md | 5 + .../_autosummary/zingg.pipes.CsvPipe.md | 5 + .../markdown/_autosummary/zingg.pipes.Pipe.md | 5 + .../_autosummary/zingg.pipes.SnowflakePipe.md | 5 + docs/pythonOss/markdown/index.md | 5 + docs/pythonOss/markdown/zingg.client.md | 5 + docs/pythonOss/markdown/zingg.pipes.md | 5 + docs/reading.md | 2 + .../combine-match-models.md | 4 - docs/reference/runtime-properties.md | 2 +- docs/running-zingg/cluster-approval.md | 2 - .../create-training-data/verify-blocking.md | 9 +- docs/running-zingg/experience-zingg.md | 6 - docs/running-zingg/install-zingg.md | 121 ++++++- docs/running-zingg/lookup-data.md | 8 - docs/running-zingg/pass-through.md | 14 +- .../set-deterministic-matching-conditions.md | 2 + docs/running/aws.md | 2 + docs/running/azure.md | 2 + docs/running/running.md | 2 + docs/scoring.md | 2 + .../security-and-privacy.md | 2 +- docs/settingUpZingg.md | 5 + docs/setup/link.md | 2 + docs/setup/training/createTrainingData.md | 2 + docs/setup/training/findAndLabel.md | 2 + .../data-input-and-output/data.md | 5 + .../configuration/model-location.md | 5 + .../tuning-label-match-and-link-jobs.md | 5 + .../generatingdocumentation.md | 5 + ...tting-up-zingg-enterprise-for-snowflake.md | 4 + .../snowflake-properties.md | 108 +++++- .../remove-stopwords-optional.md | 16 +- docs/ubuntuSetup.md | 5 + docs/updatingLabels.md | 5 + docs/zModels.md | 2 + .../zingg-concepts/how-zingg-learns/README.md | 2 +- .../deduplication-linking-and-resolving.md | 5 + 98 files changed, 1703 insertions(+), 102 deletions(-) create mode 100644 .github/markdown-link-check-config.json create mode 100644 .github/scripts/validate_frontmatter.py create mode 100644 .github/workflows/docs-validation.yml diff --git a/.github/markdown-link-check-config.json b/.github/markdown-link-check-config.json new file mode 100644 index 000000000..2a42a4045 --- /dev/null +++ b/.github/markdown-link-check-config.json @@ -0,0 +1,26 @@ +{ + "ignorePatterns": [ + { + "pattern": "https://docs.zingg.ai/latest/.*", + "reason": "Live docs site - validated separately" + }, + { + "pattern": "https://github.com/zinggAI/zingg/.*", + "reason": "GitHub repo links - validated by GitHub" + }, + { + "pattern": "http://localhost.*", + "reason": "Local development URLs" + } + ], + "retryOn429": true, + "retryCount": 3, + "fallbackRetryDelay": 5000, + "aliveStatusCodes": [200, 206], + "replacementPatterns": [ + { + "pattern": "^/([^/])", + "replacement": "https://github.com/zinggAI/zingg/blob/main/$1" + } + ] +} \ No newline at end of file diff --git a/.github/scripts/validate_frontmatter.py b/.github/scripts/validate_frontmatter.py new file mode 100644 index 000000000..94d7082ab --- /dev/null +++ b/.github/scripts/validate_frontmatter.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +""" +Validate frontmatter in markdown files using basic string parsing. +Checks for required fields: title, description, parent, nav_order (where applicable) +""" +import os +import sys +import re +from pathlib import Path + +REQUIRED_FIELDS = ['description'] +OPTIONAL_FIELDS = ['parent', 'nav_order', 'tags'] + +def parse_frontmatter(content): + """Parse frontmatter from markdown content""" + # Match frontmatter between --- delimiters + match = re.match(r'^---\n(.*?)\n---', content, re.DOTALL) + if not match: + return {} + + frontmatter_text = match.group(1) + metadata = {} + + # Simple YAML parsing for common cases + for line in frontmatter_text.split('\n'): + line = line.strip() + if not line or line.startswith('#'): + continue + + # Handle key: value + if ':' in line: + key, value = line.split(':', 1) + key = key.strip() + value = value.strip() + + # Remove quotes + if value.startswith('"') and value.endswith('"'): + value = value[1:-1] + elif value.startswith("'") and value.endswith("'"): + value = value[1:-1] + + metadata[key] = value + + return metadata + +def validate_frontmatter(filepath): + """Validate a single markdown file's frontmatter""" + try: + with open(filepath, 'r', encoding='utf-8') as f: + content = f.read() + + metadata = parse_frontmatter(content) + + errors = [] + warnings = [] + + # Check required fields + for field in REQUIRED_FIELDS: + if field not in metadata: + errors.append(f"Missing required field: {field}") + elif not metadata[field]: + errors.append(f"Empty required field: {field}") + + # Check optional fields + for field in OPTIONAL_FIELDS: + if field not in metadata: + warnings.append(f"Missing optional field: {field}") + + # Validate nav_order is integer if present + if 'nav_order' in metadata: + try: + int(metadata['nav_order']) + except (ValueError, TypeError): + errors.append(f"nav_order must be an integer, got: {metadata['nav_order']}") + + return errors, warnings + + except Exception as e: + return [f"Error reading file: {e}"], [] + +def main(): + docs_dir = Path('docs') + if not docs_dir.exists(): + print("ERROR: docs directory not found") + return 1 + + md_files = list(docs_dir.rglob('*.md')) + print(f"Found {len(md_files)} markdown files to validate") + + total_errors = 0 + total_warnings = 0 + + for md_file in md_files: + # Skip certain files + if any(part.startswith('.') for part in md_file.parts): + continue + if md_file.name in ['SUMMARY.md', 'README.md', 'CNAME']: + continue + + errors, warnings = validate_frontmatter(md_file) + + if errors: + print(f"\nERROR in {md_file}:") + for error in errors: + print(f" - {error}") + total_errors += len(errors) + + if warnings: + print(f"\nWARNING in {md_file}:") + for warning in warnings: + print(f" - {warning}") + total_warnings += len(warnings) + + print(f"\n{'='*50}") + print(f"Validation complete:") + print(f" Files checked: {len(md_files)}") + print(f" Errors: {total_errors}") + print(f" Warnings: {total_warnings}") + + if total_errors > 0: + print("\nERROR: Frontmatter validation failed") + return 1 + else: + print("\nSUCCESS: All frontmatter is valid") + return 0 + +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file diff --git a/.github/workflows/docs-validation.yml b/.github/workflows/docs-validation.yml new file mode 100644 index 000000000..638d19ae8 --- /dev/null +++ b/.github/workflows/docs-validation.yml @@ -0,0 +1,98 @@ +name: Docs Validation + +on: + pull_request: + branches: [main, master] + paths: + - 'docs/**' + - '.readthedocs.yaml' + - 'python/docs/**' + push: + branches: [main, master] + paths: + - 'docs/**' + - '.readthedocs.yaml' + - 'python/docs/**' + +jobs: + validate-markdown: + name: Validate Markdown + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check markdown links + uses: gaurav-nelson/github-action-markdown-link-check@v1 + with: + use-quiet-mode: 'yes' + config-file: '.github/markdown-link-check-config.json' + + - name: Check frontmatter + run: | + python3 .github/scripts/validate_frontmatter.py + + validate-sphinx: + name: Validate Sphinx Build + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + cd python/docs + pip install -r ../../python/requirements.txt + pip install sphinx sphinx-rtd-theme sphinx-markdown-builder + + - name: Build Sphinx docs + run: | + cd python/docs + make html + + validate-readthedocs-config: + name: Validate ReadTheDocs Config + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Validate .readthedocs.yaml syntax + run: | + python3 -c " + import yaml + with open('.readthedocs.yaml') as f: + config = yaml.safe_load(f) + print('ReadTheDocs config is valid YAML') + print(f'Version: {config.get(\"version\")}') + print(f'Python version: {config.get(\"build\", {}).get(\"tools\", {}).get(\"python\")}') + " + + validate-version-consistency: + name: Check Version Consistency + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check version in conf.py + run: | + VERSION=$(grep -E "version\s*=\s*['\"]([^'\"]+)['\"]" python/docs/conf.py | head -1 | sed -E "s/.*['\"]([^'\"]+)['\"].*/\1/") + echo "Sphinx version: $VERSION" + if [[ -z "$VERSION" ]]; then + echo "ERROR: Could not find version in python/docs/conf.py" + exit 1 + fi + + - name: Check for old version references in docs + run: | + if grep -r "0\.6\.0" docs/ --include="*.md" 2>/dev/null; then + echo "WARNING: Found references to version 0.6.0 in docs" + else + echo "No old version references found" + fi \ No newline at end of file diff --git a/docs/StandardisePostprocessor.md b/docs/StandardisePostprocessor.md index 142000d40..a42dab13f 100644 --- a/docs/StandardisePostprocessor.md +++ b/docs/StandardisePostprocessor.md @@ -1,3 +1,8 @@ +--- +description: >- + Post-process matched output to standardise field values using predefined mappings. Transforms output after matching without affecting match behavior. +--- + # Standardise Postprocessor ## Overview diff --git a/docs/Transform.md b/docs/Transform.md index 9b56c0ed8..322d6afeb 100644 --- a/docs/Transform.md +++ b/docs/Transform.md @@ -1,3 +1,8 @@ +--- +description: >- + Transform phase for data standardisation using predefined mappings. Clean and standardise field values before matching or as standalone transformation. +--- + # Transform ## Overview diff --git a/docs/accuracy/stopWordsRemoval.md b/docs/accuracy/stopWordsRemoval.md index a98fa0d6e..ac3605a19 100644 --- a/docs/accuracy/stopWordsRemoval.md +++ b/docs/accuracy/stopWordsRemoval.md @@ -1,3 +1,8 @@ +--- +description: >- + Remove common words (stopwords) that don't add differential signals during matching. Zingg can recommend stopwords automatically or use custom lists. Available in Community and Enterprise Snowflake. +--- + # Ignoring Commonly Occuring Words While Matching Common words like Mr, Pvt, Av, St, Street etc. do not add differential signals and confuse matching. These words are called **stopwords** and matching is more accurate when stopwords are ignored. diff --git a/docs/approval.md b/docs/approval.md index c1694995c..568a9d819 100644 --- a/docs/approval.md +++ b/docs/approval.md @@ -1,4 +1,6 @@ --- +description: >- + Enterprise feature for manual cluster approval workflow. Review, approve, or reject matched clusters before committing to the identity graph. title: Approve Clusters parent: Step By Step Guide nav_order: 13 diff --git a/docs/bizLeaderSurvey.md b/docs/bizLeaderSurvey.md index d7b8b87e8..045a93133 100644 --- a/docs/bizLeaderSurvey.md +++ b/docs/bizLeaderSurvey.md @@ -1,3 +1,8 @@ +--- +description: >- + Survey of 510 business leaders by Dun & Bradstreet showing the impact of data silos: 20% over-extended credit, 15% lost customers, 47% siloed data, 26% doubted data accuracy. +--- + A [survey of 510 business leaders by Dun & Bradstreet](https://www.forbes.com/sites/joemckendrick/2019/06/26/running-a-business-on-data-is-still-an-elusive-goal/?sh=2ae347d536d3) revealed that - Almost 20% executives offered too much credit to a customer and 15% failed to sign a new customer due to lack of information about them diff --git a/docs/community.md b/docs/community.md index fdc614898..3468f4bc8 100644 --- a/docs/community.md +++ b/docs/community.md @@ -1,4 +1,6 @@ --- +description: >- + Join the Zingg Community Slack to connect with other users, ask questions, and share experiences. nav_order: 8 --- diff --git a/docs/connect-your-data/connect-cloud-storage.md b/docs/connect-your-data/connect-cloud-storage.md index e64f73b00..30a70761c 100644 --- a/docs/connect-your-data/connect-cloud-storage.md +++ b/docs/connect-your-data/connect-cloud-storage.md @@ -157,7 +157,40 @@ Setting `zinggDir` to an S3 path stores all Zingg model files and training data {% endtab %} {% tab title="Azure Blob" %} -_**CHECK WITH SONAL - - NEED TEAMS HELP TO CHECK WHAT EXACTLY TO BE ADDED HERE**_ +On Azure Blob Storage (ADLS Gen2), data lives in containers accessed via the `abfss://` path format. Use `abfss://@.dfs.core.windows.net/` to connect Zingg to your blob storage. All formats — CSV, Parquet, JSON, Avro, Delta — are available in both Community and Enterprise. + +{% hint style="success" icon="right-long" %} +**Read more**: For the full Zingg installation, Azure Databricks cluster setup, and any additional dependency configuration required when running Zingg on Azure, follow the [Azure Databricks Platform Guide](../platform-guides/platform-guide-for-azure-databricks.md) or [Azure Synapse/Fabric Platform Guide](../platform-guides/platform-guide-for-microsoft-fabric.md). +{% endhint %} + +### **Python API - Community** + +```python +from zingg.client import * +from zingg.pipes import * + +schema = "id string, fname string, lname string, city string" +inputPipe = CsvPipe("inputData", "abfss://container@storageaccount.dfs.core.windows.net/path/to/data.csv", schema) +args.setData(inputPipe) + +outputPipe = CsvPipe("outputData", "abfss://container@storageaccount.dfs.core.windows.net/path/to/output/") +args.setOutput(outputPipe) +``` + +### **Python API - Enterprise** + +```python +from zinggEC.enterprise.common.EArguments import * +from zinggEC.enterprise.common.EFieldDefinition import EFieldDefinition +from zinggEC.enterprise.common.epipes import * +from zinggES.enterprise.spark.ESparkClient import * + +inputPipe = ECsvPipe("inputData", "abfss://container@storageaccount.dfs.core.windows.net/path/to/data.csv", schema) +args.setData(inputPipe) + +outputPipe = ECsvPipe("outputData", "abfss://container@storageaccount.dfs.core.windows.net/path/to/output/") +args.setOutput(outputPipe) +``` {% endtab %} {% tab title="GCS" %} diff --git a/docs/connect-your-data/connect-cloud-warehouses/connect-redshift.md b/docs/connect-your-data/connect-cloud-warehouses/connect-redshift.md index 2c54bb933..f94c793b3 100644 --- a/docs/connect-your-data/connect-cloud-warehouses/connect-redshift.md +++ b/docs/connect-your-data/connect-cloud-warehouses/connect-redshift.md @@ -1,3 +1,8 @@ +--- +description: >- + Connect Zingg to Amazon Redshift via JDBC. Configuration details are being verified. Check back for JSON config, Python API, and supported formats. +--- + # Connect Redshift **Redshift connector documentation is being confirmed. This page will be updated with the JSON config, Python API, and supported formats once the connection details are verified.** diff --git a/docs/connect-your-data/connect-file-formats.md b/docs/connect-your-data/connect-file-formats.md index 3786a2f07..f2ad3cfa5 100644 --- a/docs/connect-your-data/connect-file-formats.md +++ b/docs/connect-your-data/connect-file-formats.md @@ -232,7 +232,16 @@ args.setData(tsvPipe) **XLSX** -_**CHECK WITH SONAL - XLSX is listed as supported on zingg.ai but the connector format string and Python class are not confirmed on any live docs page. Please confirm the XLSX config and whether it is all editions or ENT only.**_ +XLSX files are supported using Spark's built-in Excel reader. The format string is `com.crealytics.spark.excel`. + +```python +from zingg.client import * +from zingg.pipes import * + +# Note: Requires com.crealytics.spark.excel JAR on classpath +xlsxPipe = CsvPipe("xlsxData", "path/to/file.xlsx", schema, format="com.crealytics.spark.excel") +args.setData(xlsxPipe) +``` {% hint style="success" icon="right-long" %} For files on cloud platforms: diff --git a/docs/connectors/jdbc/clickhouse.md b/docs/connectors/jdbc/clickhouse.md index 076d84f2a..120681e9d 100644 --- a/docs/connectors/jdbc/clickhouse.md +++ b/docs/connectors/jdbc/clickhouse.md @@ -1,3 +1,8 @@ +--- +description: >- + Connect Zingg to ClickHouse via JDBC. Configure URL with SSL, driver, credentials, and dbtable for input/output. Add clickhouse-jdbc JAR to spark.jars. +--- + # ClickHouse ClickHouse Pipe Definitions diff --git a/docs/connectors/jdbc/mysql.md b/docs/connectors/jdbc/mysql.md index e92de17db..2ffb7188d 100644 --- a/docs/connectors/jdbc/mysql.md +++ b/docs/connectors/jdbc/mysql.md @@ -1,3 +1,8 @@ +--- +description: >- + Connect Zingg to MySQL via JDBC. Configure URL, dbtable, driver, user, and password in JSON config. +--- + # MySQL ## Reading From MySQL Database: diff --git a/docs/connectors/jdbc/postgres.md b/docs/connectors/jdbc/postgres.md index 54f560e91..f10518c7e 100644 --- a/docs/connectors/jdbc/postgres.md +++ b/docs/connectors/jdbc/postgres.md @@ -1,3 +1,8 @@ +--- +description: >- + Connect Zingg to PostgreSQL via JDBC. Configure URL, dbtable, driver, user, and password in JSON config. +--- + # Postgres ## JSON Settings For Reading Data From Postgres Database: diff --git a/docs/contributing.md b/docs/contributing.md index bb12bebdb..33aa80ebc 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -1,4 +1,6 @@ --- +description: >- + Report bugs, request features, or contribute code to Zingg. All code contributions require signing the Contributor License Agreement. nav_order: 7 --- diff --git a/docs/dataSourcesAndSinks/amazonS3.md b/docs/dataSourcesAndSinks/amazonS3.md index 4762a6b86..1898a7f4f 100644 --- a/docs/dataSourcesAndSinks/amazonS3.md +++ b/docs/dataSourcesAndSinks/amazonS3.md @@ -1,3 +1,8 @@ +--- +description: >- + Run Zingg with AWS S3 as data source and sink. Configure bucket, AWS credentials, Hadoop AWS JARs, and run phases with s3a:// paths. +--- + # AWS S3 Zingg can use AWS S3 as a source and sink diff --git a/docs/dataSourcesAndSinks/bigquery.md b/docs/dataSourcesAndSinks/bigquery.md index fa11861da..1ee23f416 100644 --- a/docs/dataSourcesAndSinks/bigquery.md +++ b/docs/dataSourcesAndSinks/bigquery.md @@ -1,3 +1,8 @@ +--- +description: >- + Connect Zingg to Google BigQuery using Spark BigQuery connector. Requires driver JARs, GCS connector, service account credentials, and temporary GCS bucket for writes. +--- + # BigQuery Zingg can seamlessly work with Google BigQuery. Please find below details about the properties that must be set. diff --git a/docs/dataSourcesAndSinks/cassandra.md b/docs/dataSourcesAndSinks/cassandra.md index 63aee871f..c65ff45ae 100644 --- a/docs/dataSourcesAndSinks/cassandra.md +++ b/docs/dataSourcesAndSinks/cassandra.md @@ -1,4 +1,6 @@ --- +description: >- + Connect Zingg to Apache Cassandra using Spark Cassandra connector. Configure keyspace, table, cluster, and connection host for input/output. title: Cassandra parent: Data Sources and Sinks nav_order: 5 diff --git a/docs/dataSourcesAndSinks/connectors.md b/docs/dataSourcesAndSinks/connectors.md index e10d6f9a8..add461aec 100644 --- a/docs/dataSourcesAndSinks/connectors.md +++ b/docs/dataSourcesAndSinks/connectors.md @@ -1,4 +1,6 @@ --- +description: >- + Zingg connects to most on-premise and cloud data sources including Databricks, Fabric, Snowflake, Cassandra, S3, RDBMS, and Spark-supported sources. Supports all major file formats via Pipe abstraction. title: Data Sources and Sinks nav_order: 3 has_children: true diff --git a/docs/dataSourcesAndSinks/databricks.md b/docs/dataSourcesAndSinks/databricks.md index a4e80c2a9..ac23b8d09 100644 --- a/docs/dataSourcesAndSinks/databricks.md +++ b/docs/dataSourcesAndSinks/databricks.md @@ -1,3 +1,8 @@ +--- +description: >- + Run Zingg Community and Enterprise on Databricks. Access Parquet, Delta, and Unity Catalog. See running/databricks.md for full guide. +--- + # Databricks As a Spark-based application, the **Zingg Community version** works seamlessly on Databricks. Zingg leverages Databricks' Spark environment. diff --git a/docs/dataSourcesAndSinks/exasol.md b/docs/dataSourcesAndSinks/exasol.md index 215b540ff..25ad8f168 100644 --- a/docs/dataSourcesAndSinks/exasol.md +++ b/docs/dataSourcesAndSinks/exasol.md @@ -1,4 +1,6 @@ --- +description: >- + Connect Zingg to Exasol in-memory database using the Exasol Spark connector. Includes prerequisites, JAR download, and configuration examples. title: Exasol parent: Data Sources and Sinks nav_order: 4 diff --git a/docs/dataSourcesAndSinks/fabric.md b/docs/dataSourcesAndSinks/fabric.md index c72b85fef..1bd6a858e 100644 --- a/docs/dataSourcesAndSinks/fabric.md +++ b/docs/dataSourcesAndSinks/fabric.md @@ -1,3 +1,8 @@ +--- +description: >- + Run Zingg Community and Enterprise on Microsoft Fabric. Access Parquet and file formats via Fabric's Spark environment. See running/running-on-fabric.md for full guide. +--- + # Microsoft Fabric As a Spark-based application, the **Zingg Community version** works seamlessly on Microsoft Fabric. Zingg leverages Fabric's Spark environment. diff --git a/docs/dataSourcesAndSinks/jdbc.md b/docs/dataSourcesAndSinks/jdbc.md index d86f21449..e1040ced6 100644 --- a/docs/dataSourcesAndSinks/jdbc.md +++ b/docs/dataSourcesAndSinks/jdbc.md @@ -1,3 +1,8 @@ +--- +description: >- + Connect Zingg to databases via JDBC. Add driver JAR to spark.jars and configure connection properties. Supports MySQL, PostgreSQL, Oracle, SQL Server, and more. +--- + # JDBC Zingg can connect to various databases such as MySQL, DB2, MariaDB, MS SQL, Oracle, PostgreSQL, etc. using JDBC. One just needs to download the appropriate driver and made it accessible to the application. diff --git a/docs/dataSourcesAndSinks/mongodb.md b/docs/dataSourcesAndSinks/mongodb.md index 7d66aeb3d..6f1d9f760 100644 --- a/docs/dataSourcesAndSinks/mongodb.md +++ b/docs/dataSourcesAndSinks/mongodb.md @@ -1,4 +1,6 @@ --- +description: >- + Connect Zingg to MongoDB using MongoDB Spark connector. Configure URI with database and collection for data input/output. layout: default title: MongoDB parent: Data Sources and Sinks diff --git a/docs/dataSourcesAndSinks/neo4j.md b/docs/dataSourcesAndSinks/neo4j.md index f29e5f0d5..66cb9daba 100644 --- a/docs/dataSourcesAndSinks/neo4j.md +++ b/docs/dataSourcesAndSinks/neo4j.md @@ -1,4 +1,6 @@ --- +description: >- + Connect Zingg to Neo4j graph database using Neo4j Spark connector. Configure Bolt URL and node labels for data input/output. title: Neo4j parent: Data Sources and Sinks nav_order: 5 diff --git a/docs/dataSourcesAndSinks/parquet.md b/docs/dataSourcesAndSinks/parquet.md index 1de136388..706e713ef 100644 --- a/docs/dataSourcesAndSinks/parquet.md +++ b/docs/dataSourcesAndSinks/parquet.md @@ -1,4 +1,6 @@ --- +description: >- + Connect Zingg to Parquet files using Spark's native Parquet reader. Configure path in data/output JSON configuration. layout: default title: Parquet parent: Data Sources and Sinks diff --git a/docs/dataSourcesAndSinks/pipes.md b/docs/dataSourcesAndSinks/pipes.md index a786b639d..01fedc703 100644 --- a/docs/dataSourcesAndSinks/pipes.md +++ b/docs/dataSourcesAndSinks/pipes.md @@ -1,4 +1,6 @@ --- +description: >- + Zingg Pipes abstract data sources and sinks for any Spark-supported datastore. Configure name, format, and options for input/output connections. title: Zingg Pipes parent: Data Sources and Sinks nav_order: 4 diff --git a/docs/explainoutput/stats/cluster-statistics.md b/docs/explainoutput/stats/cluster-statistics.md index b81508e8e..d9bdd5063 100644 --- a/docs/explainoutput/stats/cluster-statistics.md +++ b/docs/explainoutput/stats/cluster-statistics.md @@ -1,3 +1,8 @@ +--- +description: >- + Cluster statistics output for Zingg Enterprise. Includes cluster count, edges, deterministic edges, centrality, and determinism ratio metrics. +--- + # Cluster Statistics [Zingg Enterprise Feature](#user-content-fn-1)[^1] diff --git a/docs/explainoutput/stats/configuring-output-statistics.md b/docs/explainoutput/stats/configuring-output-statistics.md index 3da98d112..7f26fce19 100644 --- a/docs/explainoutput/stats/configuring-output-statistics.md +++ b/docs/explainoutput/stats/configuring-output-statistics.md @@ -1,3 +1,8 @@ +--- +description: >- + Configure output statistics for Zingg Enterprise. Use $ZINGG_DYNAMIC_STAT_NAME placeholder for summary, cluster, and record stats. JSON and Python examples provided. +--- + # Configuring Output Statistics [Zingg Enterprise Feature](#user-content-fn-1)[^1] diff --git a/docs/explainoutput/stats/summary-statistics.md b/docs/explainoutput/stats/summary-statistics.md index bba780d0e..ae0c70307 100644 --- a/docs/explainoutput/stats/summary-statistics.md +++ b/docs/explainoutput/stats/summary-statistics.md @@ -1,3 +1,8 @@ +--- +description: >- + Summary statistics showing total records and clusters found after each match or incremental run with timestamp. +--- + # Summary Statistics Displays total number of records, clusters found after each match or incremental run with timestamp diff --git a/docs/faq.md b/docs/faq.md index 3dabbd728..96d1b5cf2 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,4 +1,6 @@ --- +description: >- + Answers to common questions about Zingg entity resolution including training requirements, Spark setup, ML background needs, and more. nav_order: 9 --- diff --git a/docs/households.md b/docs/households.md index 53dc44637..9c5ac72f2 100644 --- a/docs/households.md +++ b/docs/households.md @@ -1,3 +1,8 @@ +--- +description: >- + Householding groups customer data into family units for marketing, compliance, and operational efficiency. Zingg's fuzzy matching simplifies householding without extensive parsing. +--- + Householding means grouping customer data into groups of family units. These units make financial and budgetary decisions together. From a marketers’ viewpoint, householding helps to understand the relationships between individuals and execute the optimal communication strategy for the unit. With an understanding of the household, marketers can build a combined offer package that is valuable at the individual and household level. Opportunities for upselling and cross selling can also be discovered. For management, householding provides a deep view into customer lifetime value, risk, compliance and reporting metrics. For operations, householding reduces the mailing costs for disclosure and other mailers. For example, the SEC allows single mailers for a household while mailing prospectuses, annual and semi-annual reports. Householding, though highly desirable, is not easy to implement. Most business data is not segmented properly into first name, last name, suffixes, prefixes. Addresses are not standardised and remain unformatted. Missing components, abbreviations like St. for street, Av. for avenue, wrong zip codes or differing formats add to the complexities of householding implementation. diff --git a/docs/improving-accuracy/stopwordsremoval/stopwordsreco.md b/docs/improving-accuracy/stopwordsremoval/stopwordsreco.md index d0b2bc02b..b562fc193 100644 --- a/docs/improving-accuracy/stopwordsremoval/stopwordsreco.md +++ b/docs/improving-accuracy/stopwordsremoval/stopwordsreco.md @@ -1,3 +1,8 @@ +--- +description: >- + Zingg recommends stop words by extracting high-frequency words from your data. Generated via generateDocs phase and stored in zinggDir/modelId/stopWords for use in configuration. +--- + # Stop Words Zingg may help users by recommending stop words by extracting high-frequency words from the user data. The same can be generated using the [generateDocs](../../generatingDocumentation.md) command. the stop word recommendation files are written in folder zinggDir/modelId/stopWords. These files may be used after modification in [configuration](./) to build a model. diff --git a/docs/lookup.md b/docs/lookup.md index 88620f153..441dc14dc 100644 --- a/docs/lookup.md +++ b/docs/lookup.md @@ -1,3 +1,8 @@ +--- +description: >- + Enterprise feature to lookup records in match output and assign Zingg IDs. Run the runLookup phase with lookup configuration. +--- + # Lookup Data [Zingg Enterprise Feature](#user-content-fn-1)[^1] diff --git a/docs/macOSSetup.md b/docs/macOSSetup.md index 8aa5cd8a2..461d2c3d6 100644 --- a/docs/macOSSetup.md +++ b/docs/macOSSetup.md @@ -1,3 +1,8 @@ +--- +description: >- + Step-by-step guide to set up Zingg development environment on macOS including Homebrew, Java, Spark, and Git. +--- + # MacOS Setup for Zingg This guide provides specific instructions for setting up the Zingg development environment on macOS. diff --git a/docs/patient.md b/docs/patient.md index 598ce5689..5bd8b2f5a 100644 --- a/docs/patient.md +++ b/docs/patient.md @@ -1,3 +1,8 @@ +--- +description: >- + The cost of duplicate patient records in healthcare. Survey data showing $1.5M annual cost per hospital and how Zingg reduces data cleaning time. +--- + It is a truth universally acknowledged, that duplicate records are bad. They hurt analytics, increase operational overheads, make compliance a pain and increase risk. But well, there are so many challenges in the data stack, surely duplicate records can be something we can live with? How bad can it be? A recent survey by [Black Book](https://blackbookmarketresearch.newswire.com/news/improving-provider-interoperability-congruently-increasing-patient-20426295) has quantified just how bad duplicate records can be. The survey found that an average hospital is spending an extra 1.5 million USD an year due to duplicate and fragmented patient records. 1.5 million USD! Lack of a master patient index is clearly a very costly affair. The survey also found that with hospitals with more than 150 beds and hundreds of thousands of records, it took approximately 5 months for data cleaning with data validation and normalisation. diff --git a/docs/platform-guides/platform-guide-for-aws-emr.md b/docs/platform-guides/platform-guide-for-aws-emr.md index c9583df1c..074e7dfb9 100644 --- a/docs/platform-guides/platform-guide-for-aws-emr.md +++ b/docs/platform-guides/platform-guide-for-aws-emr.md @@ -6,17 +6,228 @@ description: >- # Platform Guide for AWS EMR -AWS EMR provides managed Spark on AWS infrastructure. Zingg runs on EMR using the standard Python API for Community and EZingg for Enterprise. S3 is used for data storage and\ -model persistence using the `s3a://` path format. - -**CONTENT TO BE GIVEN BY NILANJAN/SIDDIK** +AWS EMR provides managed Spark on AWS infrastructure. Zingg runs on EMR using the standard Python API for Community and EZingg for Enterprise. S3 is used for data storage and model persistence using the `s3a://` path format. {% tabs %} {% tab title="Community" %} +{% hint style="success" icon="right-long" %} +Tested with EMR 7.3.0 (Spark 3.5.1). Recommended instance type: `m6g.xlarge` or larger for worker nodes. +{% endhint %} + +### Prerequisites + +* AWS account with EMR and S3 access +* EC2 key pair for SSH access +* S3 bucket for data and model storage +* IAM role with EMR, S3, and EC2 permissions + +### Step 1: Create an S3 bucket + +Create a bucket for Zingg data, models, and JARs: + +```bash +aws s3 mb s3://your-zingg-bucket --region us-east-1 +``` + +Upload the Zingg JAR and your dataset: + +```bash +aws s3 cp zingg-0.7.0.jar s3://your-zingg-bucket/jars/ +aws s3 cp your-data.csv s3://your-zingg-bucket/data/ +``` + +### Step 2: Create an EMR cluster + +Create a cluster with the required configurations: + +```bash +aws emr create-cluster \ + --name "Zingg-EMR" \ + --release-label emr-7.3.0 \ + --applications Name=Spark Name=JupyterHub \ + --ec2-attributes KeyName=your-key-pair,InstanceProfile=EMR_EC2_DefaultRole \ + --instance-groups \ + InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m6g.xlarge \ + InstanceGroupType=CORE,InstanceCount=2,InstanceType=m6g.xlarge \ + --configurations '[{"Classification":"spark-defaults","Properties":{"spark.jars":"s3://your-zingg-bucket/jars/zingg-0.7.0.jar"}}]' \ + --service-role EMR_DefaultRole \ + --region us-east-1 +``` + +### Step 3: Connect to the cluster + +Once the cluster is running, connect via SSH: + +```bash +ssh -i your-key.pem hadoop@ +``` + +Or use JupyterHub at `https://:9443` + +### Step 4: Install Zingg Python package + +In a notebook or terminal on the cluster: + +```bash +pip install zingg==0.7.0 +``` + +### Step 5: Configure and run Zingg + +Create your `config.json` with S3 paths: + +```json +{ + "data": [{ + "name": "inputData", + "format": "csv", + "props": { + "path": "s3a://your-zingg-bucket/data/your-data.csv", + "header": "true", + "inferSchema": "true" + } + }], + "output": [{ + "name": "outputData", + "format": "csv", + "props": { + "path": "s3a://your-zingg-bucket/output/" + } + }], + "modelId": "100", + "zinggDir": "s3a://your-zingg-bucket/models", + "numPartitions": 16, + "labelDataSampleSize": 0.5, + "fieldDefinition": [...] +} +``` + +Run Zingg phases: + +```bash +# On the master node +zingg.sh --phase findTrainingData --conf config.json +zingg.sh --phase label --conf config.json +zingg.sh --phase train --conf config.json +zingg.sh --phase match --conf config.json +``` +### Step 6: Monitor and iterate + +Check logs in `/mnt/var/log/spark/` or via the Spark UI at `http://:18080` {% endtab %} {% tab title="Enterprise" %} +{% hint style="info" icon="right-long" %} +Enterprise requires a Zingg licence and the Enterprise EMR package. [Contact Zingg to get access](https://www.zingg.ai/company/contact/contact). +{% endhint %} + +### Prerequisites + +* AWS account with EMR and S3 access +* Zingg Enterprise license +* Enterprise JARs: `zingg-enterprise-spark-0.7.0.jar`, `zingg_license.jar` +* Enterprise Python packages: `zinggEC`, `zinggES` + +### Step 1: Create an S3 bucket and upload Enterprise artifacts + +```bash +aws s3 mb s3://your-zingg-enterprise-bucket --region us-east-1 +aws s3 cp zingg-enterprise-spark-0.7.0.jar s3://your-zingg-enterprise-bucket/jars/ +aws s3 cp zingg_license.jar s3://your-zingg-enterprise-bucket/jars/ +aws s3 cp your-data.csv s3://your-zingg-enterprise-bucket/data/ +``` + +Upload Enterprise Python wheels: + +```bash +aws s3 cp zinggEC-0.7.0-py3-none-any.whl s3://your-zingg-enterprise-bucket/wheels/ +aws s3 cp zinggES-0.7.0-py3-none-any.whl s3://your-zingg-enterprise-bucket/wheels/ +``` + +### Step 2: Create an EMR cluster with Enterprise JARs + +```bash +aws emr create-cluster \ + --name "Zingg-Enterprise-EMR" \ + --release-label emr-7.3.0 \ + --applications Name=Spark Name=JupyterHub \ + --ec2-attributes KeyName=your-key-pair,InstanceProfile=EMR_EC2_DefaultRole \ + --instance-groups \ + InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m6g.xlarge \ + InstanceGroupType=CORE,InstanceCount=4,InstanceType=m6g.xlarge \ + --configurations '[{"Classification":"spark-defaults","Properties":{"spark.jars":"s3://your-zingg-enterprise-bucket/jars/zingg-enterprise-spark-0.7.0.jar,s3://your-zingg-enterprise-bucket/jars/zingg_license.jar"}}]' \ + --service-role EMR_DefaultRole \ + --region us-east-1 +``` + +### Step 3: Install Enterprise Python packages + +On the cluster master node: +```bash +pip install s3://your-zingg-enterprise-bucket/wheels/zinggEC-0.7.0-py3-none-any.whl +pip install s3://your-zingg-enterprise-bucket/wheels/zinggES-0.7.0-py3-none-any.whl +pip install zingg==0.7.0 +``` + +### Step 4: Configure and run Zingg Enterprise + +Create `config.json` with Enterprise settings: + +```json +{ + "data": [{ + "name": "inputData", + "format": "csv", + "props": { + "path": "s3a://your-zingg-enterprise-bucket/data/your-data.csv", + "header": "true", + "inferSchema": "true" + } + }], + "output": [{ + "name": "outputData", + "format": "csv", + "props": { + "path": "s3a://your-zingg-enterprise-bucket/output/" + } + }], + "modelId": "100", + "zinggDir": "s3a://your-zingg-enterprise-bucket/models", + "numPartitions": 32, + "labelDataSampleSize": 0.5, + "blockingModel": "DEFAULT", + "fieldDefinition": [...] +} +``` + +Run Enterprise phases: + +```bash +# On the master node +zingg.sh --phase findTrainingData --conf config.json +zingg.sh --phase label --conf config.json +zingg.sh --phase train --conf config.json +zingg.sh --phase match --conf config.json + +# Enterprise-only phases +zingg.sh --phase explain --conf config.json +zingg.sh --phase runIncremental --conf config.json +``` + +### Step 5: Monitor and iterate + +Check logs in `/mnt/var/log/spark/` or via the Spark UI at `http://:18080` + +Use `explain` phase output for detailed match analysis. {% endtab %} {% endtabs %} + +{% hint style="success" icon="right-long" %} +**Read more**: + +* Tune accuracy → [Improve Accuracy](../tuning/improve-accuracy/) +* Understand scores and set thresholds → [Interpret Output Scores](../interpreting-results/interpret-output-scores.md) +* Set up incremental for production → [Run Incremental Matching](../running-zingg/run-incremental-matching.md) +{% endhint %} \ No newline at end of file diff --git a/docs/platform-guides/platform-guide-for-aws-glue.md b/docs/platform-guides/platform-guide-for-aws-glue.md index 98d1b884e..a455afccf 100644 --- a/docs/platform-guides/platform-guide-for-aws-glue.md +++ b/docs/platform-guides/platform-guide-for-aws-glue.md @@ -35,11 +35,10 @@ Create an S3 bucket (for example `zingg-production-storage`) and upload all six 3. Set the region to match where your Glue jobs will run (for example `us-east-1`). 4. Once created, upload the following JAR files into a `/jars/` folder: -
JARPurpose
zingg-0.6.0.jarThe Zingg engine
zingg-common-client-0.6.0.jarCommon client
zingg-common-core-0.6.0.jarCommon core
zingg-spark-client-0.6.0.jarSpark client
zingg-spark-core-0.6.0.jarSpark core
zingg-common-infra-0.6.0.jarInfrastructure link
+
JARPurpose
zingg-0.7.0.jarThe Zingg engine
zingg-common-client-0.7.0.jarCommon client
zingg-common-core-0.7.0.jarCommon core
zingg-spark-client-0.7.0.jarSpark client
zingg-spark-core-0.7.0.jarSpark core
zingg-common-infra-0.7.0.jarInfrastructure link
Download all JARs from `github.com/zinggAI/zingg/releases`. Also upload your data file to the bucket root. -_**IMAGE TO BE ADDED — S3 bucket view showing the\*\*\*\***** ****`/jars/`**** ****\*\*\*\*folder with all six Zingg JAR files listed. Tanwi to check with team for the screenshot.**_ #### **Step 2: Create an IAM role for Glue** @@ -99,7 +98,6 @@ Create an inline policy named `ZinggSessionPermissions` with the following JSON. } ``` -_**IMAGE TO BE ADDED — IAM Role creation screen in the AWS Console showing the role name, attached managed policies, and the inline policy editor. Tanwi to check with team for the screenshot.**_ #### **Step 3: Create a Glue notebook and attach the IAM role** @@ -108,7 +106,6 @@ _**IMAGE TO BE ADDED — IAM Role creation screen in the AWS Console showing the 3. Select the IAM role created in Step 2 (`zingg-glue-role`) from the dropdown. 4. Open the notebook. Confirm the kernel in the top right shows **Glue PySpark**. -_**IMAGE TO BE ADDED — AWS Glue Studio Notebooks screen showing the IAM role dropdown with the\*\*\*\***** ****`zingg-glue-role`**** ****\*\*\*\*selected. Tanwi to check with team for the screenshot.**_ ### Notebook 01: Set up Zingg @@ -124,8 +121,8 @@ This is the most important step for Glue. The `%%configure` magic cell must be * % number_of_workers 2 % idle_timeout 2880 %% configure { - "--extra-jars": "s3://your-bucket/jars/zingg-0.6.0.jar,s3://your-bucket/jars/zingg-common-client-0.6.0.jar,s3://your-bucket/jars/zingg-common-core-0.6.0.jar,s3://your-bucket/jars/zingg-spark-client-0.6.0.jar,s3://your-bucket/jars/zingg-spark-core-0.6.0.jar,s3://your-bucket/jars/zingg-common-infra-0.6.0.jar", - "--additional-python-modules": "zingg==0.6.0,tabulate,ipywidgets", + "--extra-jars": "s3://your-bucket/jars/zingg-0.7.0.jar,s3://your-bucket/jars/zingg-common-client-0.7.0.jar,s3://your-bucket/jars/zingg-common-core-0.7.0.jar,s3://your-bucket/jars/zingg-spark-client-0.7.0.jar,s3://your-bucket/jars/zingg-spark-core-0.7.0.jar,s3://your-bucket/jars/zingg-common-infra-0.7.0.jar", + "--additional-python-modules": "zingg==0.7.0,tabulate,ipywidgets", "--conf": "spark.serializer=org.apache.spark.serializer.KryoSerializer" } ``` @@ -295,7 +292,6 @@ print(f"Previewing data from {csv_path}:") spark_df.show(10) ``` -_**IMAGE TO BE ADDED - Glue notebook cell showing the\*\*\*\***** ****`spark_df.show(10)`**** ****\*\*\*\*output table with sample FEBRL records — the same entity appearing multiple times with field variations across rows. Tanwi to check with team for the screenshot.**_ #### **Step 11: Configure input and output pipes** @@ -464,7 +460,6 @@ ready_for_save = True print(f"Review sheet exported for {n_pairs} pairs to: {export_path}") ``` -_**IMAGE TO BE ADDED — S3 console showing the\*\*\*\***** ****`/review/`**** ****folder with the exported**** ****`pending_labels.csv`**** ****\*\*\*\*part file ready for download. Tanwi to check with team for the screenshot.**_ {% hint style="danger" icon="right-long" %} How to label the review sheet: @@ -476,7 +471,6 @@ How to label the review sheet: 5. Run Step 16 to feed the labels into Zingg. {% endhint %} -_**IMAGE TO BE ADDED — Example of the exported review CSV open in Excel showing two FEBRL records side by side in a vertical layout, with the\*\*\*\***** ****`>>> DECISION`**** ****row highlighted and a**** ****`1`**** ****entered in the**** ****`Record_B`**** ****\*\*\*\*column. Tanwi to check with team for the screenshot.**_ {% hint style="success" icon="right-long" %} Target 30–40 match pairs and 30–40 non-match pairs before training. Repeat Steps 14–16 in a loop until you reach this target. Label until all field types and data variation patterns in your schema are covered. If accuracy needs improvement after the first match run, return to labeling and focus on patterns that are missing or underrepresented. @@ -645,7 +639,6 @@ print(f"Redundancy Reduced by: " f"{((total_records - unique_entities) / total_records) * 100:.2f}%") ``` -_**IMAGE TO BE ADDED — Glue notebook cell showing\*\*\*\***** ****`final_results.orderBy("z_cluster").show(10)`**** ****output with resolved records grouped by**** ****`z_cluster`**** ****\*\*\*\*— two rows sharing the same cluster value visible in the output. Tanwi to check with team for the screenshot.**_ {% endtab %} {% tab title="Enterprise" %} @@ -664,6 +657,6 @@ _**IMAGE TO BE ADDED — Glue notebook cell showing\*\*\*\***** ****`final_ {% hint style="success" icon="right-long" %} Download the notebooks used in this guide: -* Community notebooks (NB01–04): Download the notebook used in this guide: `github.com/zinggAI/zingg/tree/main/examples/aws-glue` -* Enterprise notebooks — TO BE ADDED +* Community notebooks (NB01–04): `github.com/zinggAI/zingg/tree/main/examples/aws-glue` +* Enterprise notebooks (NB01–07): included in your Zingg Enterprise package {% endhint %} diff --git a/docs/platform-guides/platform-guide-for-azure-databricks.md b/docs/platform-guides/platform-guide-for-azure-databricks.md index c1fd14d2b..79e895570 100644 --- a/docs/platform-guides/platform-guide-for-azure-databricks.md +++ b/docs/platform-guides/platform-guide-for-azure-databricks.md @@ -36,7 +36,7 @@ Uses `Arguments`, `FieldDefinition`, `CsvPipe`, and `ZinggWithSpark`. The workfl Open a notebook attached to the cluster and run: ```python -%pip install zingg==0.6.0 +%pip install zingg==0.7.0 dbutils.library.restartPython() ``` @@ -225,7 +225,6 @@ A widget displays each candidate pair side by side. For each pair, select: The widget code handles the display and state management. Run the cell to render it. -_**IMAGE TO BE ADDED — Zingg labeling widget in a Databricks notebook showing two candidate records side by side with Match / No Match / Uncertain toggle buttons. Tanwi to check with team for a screenshot from a live notebook run.**_ {% hint style="success" icon="right-long" %} Target 30–40 match pairs and 30–40 non-match pairs before training. Repeat Steps 10–12 until you reach this target. Label until all field types and data variation patterns in your schema are covered. If results need improvement after the first match run, return to labeling and focus on patterns that are missing or underrepresented. @@ -308,7 +307,6 @@ DOCS_DIR = zinggDir + "/" + modelId + "/docs/" displayHTML(open(DOCS_DIR + "model.html", 'r').read()) ``` -_**IMAGE TO BE ADDED —**** ****`generateDocs`**** ****HTML output rendered inside a Databricks notebook showing labeled pair examples. Tanwi to check with team for screenshot from a live notebook run. Even a small portion of the rendered HTML is sufficient — it tells the reader what to expect before they run it. Place: below the**** ****`displayHTML`****\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\* \*\*\*\*line.**_ {% endtab %} {% tab title="Enterprise" %} @@ -327,11 +325,10 @@ Enterprise requires a Zingg licence and the `zinggEC` and `zinggES` packages. [C 1. Go to **Compute** → **Create Cluster**. Name it `Zingg-Enterprise`. 2. Set the runtime to a current LTS version. 3. Create a managed Volume inside your catalog schema. -4. Upload `zingg-enterprise-spark-0.6.0.jar` and `zingg_license.jar` to the Volume. -5. Open the cluster → **Libraries** → **Install New** → **Volumes** → navigate to: `/Volumes/catalog_name/schema_name/volume_name/zingg-enterprise-spark-0.6.0.jar` +4. Upload `zingg-enterprise-spark-0.7.0.jar` and `zingg_license.jar` to the Volume. +5. Open the cluster → **Libraries** → **Install New** → **Volumes** → navigate to: `/Volumes/catalog_name/schema_name/volume_name/zingg-enterprise-spark-0.7.0.jar` 6. Repeat for `zingg_license.jar`. -_**IMAGE TO BE ADDED — Databricks cluster Libraries tab showing the Enterprise JAR files installed from a Volume path. Tanwi to check with team for screenshot from a live Enterprise cluster setup.**_ #### Step 2: Verify all three packages are installed @@ -533,7 +530,6 @@ stopwordsForStreet = spark.read.csv( stopwordsForStreet.show() ``` -_**IMAGE TO BE ADDED — Databricks notebook showing the stopwords output table with word and frequency columns. Tanwi to check with team for screenshot from a live notebook run. A simple table with 10–15 rows is sufficient that tells the reader what the recommendation output looks like before they run it.**_ #### **Step 15: Apply stopwords to the field definition** @@ -591,7 +587,6 @@ else: The Enterprise widget shows one pair at a time with Prev and Next navigation. For each pair select `Match`, `No Match`, or `Uncertain`. Labels are saved directly to the `DataFrame` as you click. -_**IMAGE TO BE ADDED — Enterprise labeling widget in a Databricks notebook: two records displayed in a table, Match / No Match / Uncertain toggle buttons, Prev and Next navigation. Tanwi to check with team for screenshot from a live Enterprise notebook run. If the OS and Enterprise widgets look identical, the same screenshot can be reused.**_ {% hint style="info" icon="right-long" %} Target 30–40 match pairs and 30–40 non-match pairs before training. Repeat Steps 17–19 until all field types and data patterns are represented. If accuracy needs improvement after the first match run, return here and focus on patterns that are underrepresented. @@ -630,7 +625,6 @@ data_html = "\n".join(r.value for r in data_doc.collect()) displayHTML(data_html) ``` -_**IMAGE TO BE ADDED —**** ****`generateDocs`****\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\* \*\*\*\*output rendered inside a Databricks notebook showing labeled pair examples in HTML. Tanwi to check with team for screenshot from a live notebook run. Can reuse the OS version if the output looks the same.**_ ### Notebook 05: Train and match @@ -658,7 +652,6 @@ display(outputDF) print(outputDF.count()) ``` -_**IMAGE TO BE ADDED — Enterprise match output in Databricks showing\*\*\*\***** ****`ZINGG_ID`**** ****column alongside resolved records. Show two rows with the same**** ****`ZINGG_ID`**** ****\*\*\*\*to illustrate entity resolution. Tanwi to check with team for screenshot from a live notebook run.**_ {% hint style="success" icon="right-long" %} **Read more**: Enterprise output includes `ZINGG_ID` — a globally unique, persistent identifier for each resolved entity. Unlike `Z_CLUSTER` in Community, `ZINGG_ID` does not change between runs including incremental runs. @@ -757,7 +750,6 @@ display(outputDF) print(outputDF.count()) ``` -_**IMAGE TO BE ADDED — explain output table in Databricks showing\*\*\*\***** ****`pk1`****,**** ****`pk2`****\*\*\*\*, and similarity score columns for matched pairs within the cluster. Tanwi to check with team for screenshot from a live notebook run. A small 5–10 row output table is sufficient.**_ {% hint style="success" icon="right-long" %} **Read more**: Each row in the output represents a matched record pair within the cluster — `pk1` and `pk2` are the primary keys of the two records, with their similarity score. diff --git a/docs/platform-guides/platform-guide-for-gcp-dataproc.md b/docs/platform-guides/platform-guide-for-gcp-dataproc.md index 56703817d..0de1680d0 100644 --- a/docs/platform-guides/platform-guide-for-gcp-dataproc.md +++ b/docs/platform-guides/platform-guide-for-gcp-dataproc.md @@ -21,7 +21,7 @@ Uses `Arguments`, `FieldDefinition`, `CsvPipe`, and `ZinggWithSpark`. Runs on a Zingg on GCP requires three JARs to bridge Spark with Google Cloud services. Download these to your local machine before creating the cluster. -
JARPurposeDownload
zingg-0.6.0.jarThe Zingg enginegithub.com/zinggAI/zingg/releases
spark-3.5-bigquery-0.44.1.jarBigQuery connectorgithub.com/GoogleCloudDataproc/spark-bigquery-connector
gcs-connector-hadoop3-latest.jarGCS connectordocs.cloud.google.com/dataproc/docs/concepts/connectors/cloud-storage
+
JARPurposeDownload
zingg-0.7.0.jarThe Zingg enginegithub.com/zinggAI/zingg/releases
spark-3.5-bigquery-0.44.1.jarBigQuery connectorgithub.com/GoogleCloudDataproc/spark-bigquery-connector
gcs-connector-hadoop3-latest.jarGCS connectordocs.cloud.google.com/dataproc/docs/concepts/connectors/cloud-storage
Create a GCS bucket and upload the JARs and your dataset. You can do this from the Cloud Console or the `gcloud` CLI. @@ -32,7 +32,6 @@ Create a GCS bucket and upload the JARs and your dataset. You can do this from t 3. Set the region to match where your Dataproc cluster will run (for example `us-central1`). 4. Upload the three JARs and your data file (for example `customers.csv`) to the bucket. -_**IMAGE TO BE ADDED - GCS bucket creation screen in the Google Cloud Console showing bucket name, region selector, and upload interface. Tanwi to check with team for screenshot from a live GCS console.**_ #### gcloud CLI @@ -58,9 +57,8 @@ The cluster must be created with the three JARs injected via `spark.jars`. This 4. Set **Master** and **Worker** nodes to `n2-standard-4` with 100GB boot disk. 5. Scroll to **Properties** and add: * **Key:** `spark.jars` - * **Value:** `gs://YOUR_BUCKET/zingg-0.6.0.jar,gs://YOUR_BUCKET/spark-3.5-bigquery-0.44.1.jar,gs://YOUR_BUCKET/gcs-connector-hadoop3-latest.jar` + * **Value:** `gs://YOUR_BUCKET/zingg-0.7.0.jar,gs://YOUR_BUCKET/spark-3.5-bigquery-0.44.1.jar,gs://YOUR_BUCKET/gcs-connector-hadoop3-latest.jar` -_**IMAGE TO BE ADDED — Dataproc cluster creation screen showing the Properties section with\*\*\*\***** ****`spark.jars`**** ****key and the three JAR paths as the value. Tanwi to check with team for screenshot from a live Dataproc console. This is the most important screenshot on the page — the**** ****`spark.jars`**** ****\*\*\*\*Properties field is not obvious to find and a screenshot here prevents the most common setup error.**_ #### gcloud CLI @@ -74,7 +72,7 @@ gcloud dataproc clusters create zingg-cluster \ --optional-components=JUPYTER \ --enable-component-gateway \ --properties="^#^spark:spark.jars=\ -gs://$BUCKET/zingg-0.6.0.jar,\ +gs://$BUCKET/zingg-0.7.0.jar,\ gs://$BUCKET/spark-3.5-bigquery-0.44.1.jar,\ gs://$BUCKET/gcs-connector-hadoop3-latest.jar" ``` @@ -93,7 +91,6 @@ Once your cluster status shows **Running**, access the managed JupyterLab enviro 4. Under **Component Gateway**, click the **JupyterLab** link. 5. Create a new notebook and select the **PySpark** kernel. -_**IMAGE TO BE ADDED— Dataproc cluster Web Interfaces tab showing the Component Gateway section with the JupyterLab link highlighted. Tanwi to check with team for screenshot from a live Dataproc cluster.**_ ### Step 4: Set a checkpoint directory and install Zingg @@ -200,7 +197,6 @@ spark_df = spark_df.toDF(*schema_list) spark_df.limit(10).toPandas().head() ``` -_**IMAGE TO BE ADDED — Jupyter notebook cell showing the preview output table with sample FEBRL data — the same customer appearing multiple times with field variations across rows. Source: not in the GCP docx (text-only guide). Tanwi to screenshot from a live notebook run. Same principle as the Databricks guide — this image shows readers the exact problem Zingg is solving before they configure anything. Place: below the\*\*\*\***** ****`spark_df.limit(10).toPandas().head()`**** ****\*\*\*\*line.**_ ### Step 8: Configure input and output pipes @@ -353,7 +349,6 @@ display(widgets.VBox(children=vContainers)) ready_for_save = True ``` -_**IMAGE TO BE ADDED — Zingg labeling widget rendered in JupyterLab on Dataproc, showing two candidate records side by side with Match / No Match / Uncertain toggle buttons. Tanwi to check with team for screenshot from a live Dataproc notebook run.**_ {% hint style="success" icon="right-long" %} Target 30–40 match pairs and 30–40 non-match pairs before training. Repeat Steps 11–14 in a loop until you reach this target. Label until all field types and data variation patterns in your schema are covered. If accuracy needs improvement after the first match run, return to labeling and focus on patterns that are underrepresented. @@ -411,7 +406,6 @@ with open(DOCS_DIR + "data.html", 'r') as f: display(HTML(f.read())) ``` -_**IMAGE TO BE ADDED—**** ****`generateDocs`****\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\* \*\*\*\*HTML output rendered inside JupyterLab on Dataproc showing labeled pair examples. Source: not in the GCP docx. Tanwi to check with team for screenshot from a live notebook run.**_ {% hint style="success" icon="right-long" %} `generateDocs` is optional. Skip it if you have 30–40 matches and 30–40 non-matches and are confident in your labeling quality. @@ -461,7 +455,6 @@ final_results = outputDF.toDF(*colNames) final_results.show(10) ``` -_**IMAGE TO BE ADDED — match output table in JupyterLab on Dataproc showing resolved records with\*\*\*\***** ****`z_cluster`**** ****column visible. Highlight two rows sharing the same**** ****`z_cluster`**** ****\*\*\*\*value to show they have been resolved to the same entity.Tanwi to check with team for screenshot from a live notebook run.**_ {% hint style="success" icon="right-long" %} * `z_cluster`— unique entity ID assigned by Zingg. All records sharing the same `z_cluster` represent the same real-world entity. Group by `z_cluster` to collapse duplicates into a golden record. @@ -473,6 +466,159 @@ For threshold guidance and full output column definitions → [Interpret Output {% endtab %} {% tab title="Enterprise" %} +{% hint style="info" icon="right-long" %} +Enterprise requires a Zingg licence and the Enterprise GCP package. [Contact Zingg to get access](https://www.zingg.ai/company/contact/contact). +{% endhint %} + +Uses `EArguments`, `EFieldDefinition`, `ECsvPipe`, `EZinggWithSpark`. Adds blocking model configuration, primary key for incremental matching, stats output, deterministic matching rules, pass-through expressions, the `runIncremental` phase, and the `explain` phase. + +### Prerequisites + +* GCP account with Dataproc and GCS access +* Zingg Enterprise license +* Enterprise JAR: `zingg-enterprise-spark-0.7.0.jar` +* Enterprise Python packages: `zinggEC`, `zinggES` + +### Step 1: Prepare GCS bucket and upload Enterprise artifacts + +Create a GCS bucket and upload the Enterprise JAR and Python wheels: + +```bash +BUCKET="zingg-enterprise-storage" + +gcloud storage buckets create gs://$BUCKET --location=us-central1 + +gsutil cp zingg-enterprise-spark-0.7.0.jar gs://$BUCKET/jars/ +gsutil cp zinggEC-0.7.0-py3-none-any.whl gs://$BUCKET/wheels/ +gsutil cp zinggES-0.7.0-py3-none-any.whl gs://$BUCKET/wheels/ +gsutil cp your-data.csv gs://$BUCKET/data/ +``` + +### Step 2: Create a Dataproc cluster with Enterprise JAR + +```bash +gcloud dataproc clusters create zingg-enterprise-cluster \ + --region=us-central1 \ + --image-version=2.2-debian12 \ + --master-machine-type=n2-standard-4 \ + --worker-machine-type=n2-standard-4 \ + --num-workers=4 \ + --optional-components=JUPYTER \ + --enable-component-gateway \ + --properties="^#^spark:spark.jars=\ +gs://$BUCKET/jars/zingg-enterprise-spark-0.7.0.jar" +``` + +### Step 3: Install Enterprise Python packages + +In JupyterLab on the cluster: + +```python +!pip install gs://$BUCKET/wheels/zinggEC-0.7.0-py3-none-any.whl +!pip install gs://$BUCKET/wheels/zinggES-0.7.0-py3-none-any.whl +!pip install zingg==0.7.0 + +# Verify installation +!pip show zinggEC +!pip show zinggES +``` + +### Step 4: Configure and run Zingg Enterprise + +Set up checkpoint directory and import libraries: + +```python +checkpoint_path = f"gs://{BUCKET}/zingg_checkpoint" +spark.sparkContext.setCheckpointDir(checkpoint_path) + +from zinggEC.enterprise.common.EArguments import * +from zinggEC.enterprise.common.EFieldDefinition import EFieldDefinition +from zinggEC.enterprise.common.epipes import * +from zinggES.enterprise.spark.ESparkClient import * +from zingg.client import * +from zingg.pipes import * +import pandas as pd +import numpy as np +``` + +Configure Enterprise arguments: + +```python +BUCKET = "your-bucket-name" +modelId = "testModelFebrl" +zinggDir = f"gs://{BUCKET}/models" + +args = EArguments() +args.setModelId(modelId) +args.setZinggDir(zinggDir) +args.setBlockingModel("DEFAULT") # Use "WIDER" if matches are being missed +args.setNumPartitions(16) +args.setLabelDataSampleSize(0.5) +``` + +Configure input/output pipes and field definitions (similar to Community but using Enterprise classes): + +```python +inputPipe = ECsvPipe("testFebrl", f"gs://{BUCKET}/test.csv", schema) +args.setData(inputPipe) + +outputPipe = ECsvPipe("resultOutput", f"gs://{BUCKET}/results") +args.setOutput(outputPipe) + +# Field definitions using EFieldDefinition +fieldDefs = [ + EFieldDefinition("id", "string", MatchType.EXACT), + EFieldDefinition("fname", "string", MatchType.FUZZY), + EFieldDefinition("lname", "string", MatchType.FUZZY), + # ... add all your fields +] +args.setFieldDefinition(fieldDefs) +``` + +Run Enterprise phases: + +```python +# Find training data +options = ClientOptions([ClientOptions.PHASE, "findTrainingData"]) +zingg = EZinggWithSpark(args, options) +zingg.initAndExecute() + +# Label (interactive widget) +options = ClientOptions([ClientOptions.PHASE, "label"]) +zingg = EZinggWithSpark(args, options) +zingg.init() + +# Train +options = ClientOptions([ClientOptions.PHASE, "train"]) +zingg = EZinggWithSpark(args, options) +zingg.initAndExecute() + +# Match +options = ClientOptions([ClientOptions.PHASE, "match"]) +zingg = EZinggWithSpark(args, options) +zingg.initAndExecute() + +# Enterprise-only: explain results +options = ClientOptions([ClientOptions.PHASE, "explain"]) +zingg = EZinggWithSpark(args, options) +zingg.initAndExecute() + +# Enterprise-only: incremental matching +options = ClientOptions([ClientOptions.PHASE, "runIncremental"]) +zingg = EZinggWithSpark(args, options) +zingg.initAndExecute() +``` + +### Key Differences from Community Edition + +| Feature | Community | Enterprise | +|---------|-----------|------------| +| Classes | `Arguments`, `FieldDefinition`, `CsvPipe`, `ZinggWithSpark` | `EArguments`, `EFieldDefinition`, `ECsvPipe`, `EZinggWithSpark` | +| Blocking | Default only | `DEFAULT` or `WIDER` strategy | +| Incremental | Not available | `runIncremental` phase | +| Explainability | Not available | `explain` phase | +| Deterministic | Not available | Built-in deterministic rules | +| Pass-through | Not available | Pass-through expressions | {% endtab %} {% endtabs %} diff --git a/docs/platform-guides/platform-guide-for-microsoft-fabric.md b/docs/platform-guides/platform-guide-for-microsoft-fabric.md index 251ebd19c..b4e6effd7 100644 --- a/docs/platform-guides/platform-guide-for-microsoft-fabric.md +++ b/docs/platform-guides/platform-guide-for-microsoft-fabric.md @@ -480,6 +480,327 @@ print(outputDF.count()) {% endtab %} {% tab title="Enterprise" %} +{% hint style="info" icon="right-long" %} +Enterprise requires a Zingg licence and the Enterprise Fabric package. [Contact Zingg to get access](https://www.zingg.ai/company/contact/contact). +{% endhint %} + +Uses `EArguments`, `EFieldDefinition`, `ECsvPipe`, `EZinggWithSpark`. Seven notebooks, each covering one phase. Enterprise adds blocking model configuration, a primary key for incremental matching, stats output, deterministic matching rules, pass-through expressions, the `runIncremental` phase, and the `explain` phase. + +### Prerequisites + +* Microsoft Fabric workspace with Lakehouse +* Zingg Enterprise license +* Enterprise JAR: `zingg-enterprise-spark-0.7.0.jar` +* Enterprise Python packages: `zinggEC`, `zinggES` +* License JAR: `zingg_license.jar` + +### Fabric workspace setup (Enterprise) + +Complete these steps in the Fabric UI before opening any notebook. + +#### **Step 1: Create a Fabric workspace** + +Same as Community - create a workspace named `Zingg-Fabric-Enterprise`. + +#### **Step 2: Create a Zingg Enterprise Environment** + +1. Inside your workspace, go to the **Environment** tab and click **New Environment**. +2. Name it `Zingg Enterprise Environment`. + +#### **Step 3: Install Enterprise JARs in the Environment** + +1. Download the Enterprise package (provided with your license) which includes: + - `zingg-enterprise-spark-0.7.0.jar` + - `zingg_license.jar` +2. Open your `Zingg Enterprise Environment`, go to **Custom Library**, and upload both JAR files. +3. Click **Save** and then **Publish** the Environment. + +#### **Step 4: Create a Lakehouse and upload your data** + +Same as Community - create a Lakehouse and upload your CSV file to OneLake. + +### Notebook 01: Set up Zingg Enterprise + +Create a new notebook in your workspace, attach it to the `Zingg Enterprise Environment`, and select **PySpark** as the kernel. + +#### **Step 5: Verify Spark is configured correctly** + +```python +spark.sparkContext.getConf().get('spark.hadoop.trident.workspace.id') +``` + +#### **Step 6: Set the checkpoint directory** + +```python +spark.sparkContext.setCheckpointDir("Files") +``` + +#### **Step 7: Install Enterprise Python packages** + +```python +pip install zingg +pip install zinggEC +pip install zinggES +``` + +Verify installation: + +```python +!pip show zingg +!pip show zinggEC +!pip show zinggES +``` + +{% hint style="info" icon="right-long" %} +All three must show as installed: `zingg` (Community base), `zinggEC` (Enterprise), `zinggES` (Enterprise Plus). If any show as not found, install the corresponding `.whl` file from the cluster Libraries tab using the wheels provided in your Enterprise package. +{% endhint %} + +#### **Step 8: Set the model ID and storage paths** + +Replace `` and `` with the actual IDs from your Fabric workspace and Lakehouse. + +```python +files_dir = ( + "abfss://@onelake.dfs.fabric.microsoft.com/" + "/Files" +) +zingg_dir = files_dir + "/zingg" +model_id = "zinggModel" +MARKED_DIR = zingg_dir + "/" + model_id + "/trainingData/marked/" +UNMARKED_DIR = zingg_dir + "/" + model_id + "/trainingData/unmarked/" +``` + +#### **Step 9: Import libraries** + +```python +import pandas as pd +import numpy as np +import os, time, uuid +from ipywidgets import widgets, interact, GridspecLayout +import base64 +import pyspark.sql.functions as fn +from zinggEC.enterprise.common.ApproverArguments import * +from zinggEC.enterprise.common.IncrementalArguments import * +from zinggEC.enterprise.common.epipes import * +from zinggEC.enterprise.common.EArguments import * +from zinggEC.enterprise.common.EFieldDefinition import EFieldDefinition +from zinggES.enterprise.spark.ESparkClient import * +from zingg.client import * +from zingg.pipes import * +``` + +#### **Step 10: Build the Enterprise arguments object** + +`EArguments` is the Enterprise equivalent of `Arguments`. `setBlockingModel` sets the blocking strategy. `DEFAULT` suits most datasets—use `WIDER` if you know matching pairs are being missed. + +```python +args = EArguments() +args.setModelId(model_id) +args.setZinggDir(zingg_dir) +args.setBlockingModel("DEFAULT") +``` + +#### **Step 11: Configure performance settings** + +```python +args.setNumPartitions(32) +spark.conf.set("spark.sql.adaptive.enabled", False) +``` + +{% hint style="danger" icon="right-long" %} +Set `numPartitions` to approximately 20–30× your worker vCPU count. `labelDataSampleSize` is set in Notebook 03 where the labeling loop runs. +{% endhint %} + +#### **Step 12: Preview your data** + +Same as Community Notebook 01, Step 11. + +#### **Step 13: Configure input and output pipes** + +Use `ECsvPipe` for Enterprise: + +```python +schema = ( + "rec_id string, fname string, " + "lname string, stNo string, " + "add1 string, add2 string, " + "city string, areacode string, " + "state string, dob string, " + "ssn string" +) + +inputPipe = ECsvPipe( + "inputpipe", + "abfss://@onelake" + ".dfs.fabric.microsoft.com/" + "/Files/test.csv", + schema +) +args.setData(inputPipe) + +output_path = ( + "abfss://@onelake" + ".dfs.fabric.microsoft.com/" + "/Files/Output" + model_id +) +outputPipe = ECsvPipe("resultOutput", output_path) +args.setOutput(outputPipe) +``` + +#### **Step 14: Define fields and match types** + +Use `EFieldDefinition` for Enterprise: + +```python +rec_id = EFieldDefinition("rec_id", "string", MatchType.DONT_USE) +fname = EFieldDefinition("fname", "string", MatchType.FUZZY) +lname = EFieldDefinition("lname", "string", MatchType.FUZZY) +stNo = EFieldDefinition("stNo", "string", MatchType.FUZZY) +add1 = EFieldDefinition("add1", "string", MatchType.FUZZY) +add2 = EFieldDefinition("add2", "string", MatchType.FUZZY) +city = EFieldDefinition("city", "string", MatchType.FUZZY) +areacode = EFieldDefinition("areacode", "string", MatchType.FUZZY) +state = EFieldDefinition("state", "string", MatchType.FUZZY) +dob = EFieldDefinition("dob", "string", MatchType.EXACT) +ssn = EFieldDefinition("ssn", "string", MatchType.EXACT) + +fieldDefs = [ + rec_id, fname, lname, stNo, add1, + add2, city, areacode, state, dob, + ssn +] +args.setFieldDefinition(fieldDefs) +``` + +### Notebook 02: Find training data + +This notebook runs `findTrainingData`. It calls `%run 01-setting_up_zingg` at the top. + +#### **Step 15: Set `labelDataSampleSize`** + +```python +args.setNumPartitions(4) +args.setLabelDataSampleSize(0.4) +``` + +#### **Step 16: Find candidate pairs** + +```python +options = ClientOptions([ClientOptions.PHASE, "findTrainingData"]) +zingg = EZinggWithSpark(args, options) +zingg.initAndExecute() +``` + +### Notebook 03: Label pairs + +This notebook runs `label`. It calls `%run 01-setting_up_zingg` at the top. + +#### **Step 17: Load pairs for labeling** + +```python +options = ClientOptions([ClientOptions.PHASE, "label"]) +zingg = EZinggWithSpark(args, options) +zingg.init() + +candidate_pairs_pd = getPandasDfFromDs(zingg.getUnmarkedRecords()) + +if candidate_pairs_pd.shape[0] == 0: + print("No pairs found. Run findTrainingData first.") +else: + z_clusters = list(np.unique(candidate_pairs_pd['z_cluster'])) + print(f"{len(z_clusters)} candidate pairs found for labeling") +``` + +#### **Step 18: Label pairs in the widget** + +Same widget code as Community (Step 17). + +#### **Step 19: Save labeled pairs** + +```python +if not ready_for_save: + print("Run the widget cell first.") +else: + for pair in vContainers[1:]: + user_assigned_label = pair.children[1].get_interact_value() + start = pair.children[0].value.find('data-title="') + if start > 0: + start += len('data-title="') + end = pair.children[0].value.find('"', start+2) + pair_id = pair.children[0].value[start:end] + candidate_pairs_pd.loc[ + candidate_pairs_pd['z_cluster'] == pair_id, + 'z_isMatch'] = LABELS.get(user_assigned_label) + + notebookutils.fs.mkdirs(MARKED_DIR) + zingg.writeLabelledOutputFromPandas( + candidate_pairs_pd, args) + + marked_pd_df = getPandasDfFromDs( + zingg.getMarkedRecords()) + n_pos, n_neg, n_uncer, n_tot = \ + count_labeled_pairs(marked_pd_df) + print(f"Out of total {n_tot} pairs,") + print(f"You have accumulated {n_pos} pairs labeled as positive matches.") + print(f"You have accumulated {n_neg} pairs labeled as not matches.") + print(f"You have accumulated {n_uncer} pairs labeled as uncertain.") + print("Run Steps 16-19 again if you need more pairs.") + ready_for_save = False +``` + +### Notebook 04: Generate model documentation (optional) + +Run `generateDocs` after labeling to produce HTML reports of your training data. + +```python +options = ClientOptions([ClientOptions.PHASE, "generateDocs"]) +zingg = EZinggWithSpark(args, options) +zingg.initAndExecute() + +DOCS_DIR = zingg_dir + "/" + model_id + "/docs/" +displayHTML(open(DOCS_DIR + "model.html", 'r').read()) +displayHTML(open(DOCS_DIR + "data.html", 'r').read()) +``` + +### Notebook 05: Train and match + +#### **Step 20: Train and match** + +```python +options = ClientOptions([ClientOptions.PHASE, "trainMatch"]) +zingg = EZinggWithSpark(args, options) +zingg.initAndExecute() +``` + +Run `train` and `match` separately if you want to inspect the model first. + +### Notebook 06: Explain results (Enterprise only) + +```python +options = ClientOptions([ClientOptions.PHASE, "explain"]) +zingg = EZinggWithSpark(args, options) +zingg.initAndExecute() +``` + +### Notebook 07: Incremental matching (Enterprise only) + +```python +options = ClientOptions([ClientOptions.PHASE, "runIncremental"]) +zingg = EZinggWithSpark(args, options) +zingg.initAndExecute() +``` + +### Key Differences from Community Edition + +| Feature | Community | Enterprise | +|---------|-----------|------------| +| Classes | `Arguments`, `FieldDefinition`, `CsvPipe`, `ZinggWithSpark` | `EArguments`, `EFieldDefinition`, `ECsvPipe`, `EZinggWithSpark` | +| Blocking | Default only | `DEFAULT` or `WIDER` strategy | +| Incremental | Not available | `runIncremental` phase | +| Explainability | Not available | `explain` phase | +| Deterministic | Not available | Built-in deterministic rules | +| Pass-through | Not available | Pass-through expressions | +| Notebooks | 4 | 7 | {% endtab %} {% endtabs %} diff --git a/docs/platform-guides/platform-guide-for-snowflake.md b/docs/platform-guides/platform-guide-for-snowflake.md index 8197dc3a7..a5871fbb6 100644 --- a/docs/platform-guides/platform-guide-for-snowflake.md +++ b/docs/platform-guides/platform-guide-for-snowflake.md @@ -39,8 +39,8 @@ Host 3. Click **Open a Remote Window** in VS Code, select your EC2 host, and connect. 4. Pull the Zingg Docker image and start a bash session inside the container: ```bash -docker pull zingg/zingg:0.6.0 -docker run -it zingg/zingg:0.6.0 bash +docker pull zingg/zingg:0.7.0 +docker run -it zingg/zingg:0.7.0 bash ``` Inside the container, your container ID is the alphanumeric string between `@` and `:` in the terminal prompt. For example in `root@fab997383957:/zingg#`, the container ID is `fab997383957`. Note this value—you will need it in Step 3. @@ -210,8 +210,6 @@ When the `.pem` key file is included in the zip, Lambda extracts it to `/var/tas A successful test returns `HTTP 200` and `Started phase: findTrainingData`. Monitor progress via `tail -f logfile.txt` on EC2. {% endhint %} -_**IMAGE TO BE ADDED — AWS Lambda function test screen showing a successful HTTP 200 response with the "Started phase: findTrainingData" body. Tanwi to check with team for screenshot this from a live Lambda test run and add here. Caption: "Lambda function test returning HTTP 200 — Zingg phase started successfully."**_ - ### **Step 5: Connect Lambda to Snowflake via external function** 1. Create a new IAM role. Entity type: **Another AWS account**. Specify your AWS Account ID. @@ -242,7 +240,7 @@ SELECT run_zingg('findTrainingData'); **Run label interactively on EC2 directly** -```sql +```bash ./zingg.sh --phase label --conf examples/febrl/configSnow.json ``` @@ -284,7 +282,7 @@ tail -f nohup.out Or from Snowflake: -```bash +```sql SELECT run_zingg('checklog'); ``` @@ -345,6 +343,160 @@ Enterprise only. Zingg Enterprise runs natively inside Snowflake using Snowpark. Enterprise requires a Zingg licence and the Enterprise Snowflake package. [Contact Zingg to get access](https://www.zingg.ai/company/contact/contact). {% endhint %} -**Content for the Enterprise Snowflake platform guide is being prepared. This section will be updated with full step-by-step instructions once confirmed by the team.** +### **Prerequisites** + +* Snowflake account with Snowpark enabled +* Zingg Enterprise license +* Python 3.8+ with Snowpark session +* Access to Zingg Enterprise Snowflake package + +### **Step 1: Set up Snowpark Session** + +Install the Snowpark Python package and Zingg Enterprise package: + +```bash +pip install snowflake-snowpark-python +# Zingg Enterprise package provided after license purchase +pip install zingg-enterprise-snowflake +``` + +Create a Snowpark session: + +```python +from snowflake.snowpark import Session + +connection_parameters = { + "account": "", + "user": "", + "password": "", + "role": "", + "warehouse": "", + "database": "", + "schema": "" +} + +session = Session.builder.configs(connection_parameters).create() +``` + +### **Step 2: Configure Zingg Enterprise** + +Create a configuration dictionary for Zingg Enterprise: + +```python +from zingg_enterprise import ZinggEnterprise + +config = { + "data": [{ + "name": "inputData", + "table": "your_input_table", + "database": "your_database", + "schema": "your_schema" + }], + "output": [{ + "name": "outputData", + "table": "your_output_table", + "database": "your_database", + "schema": "your_schema" + }], + "modelId": "100", + "zinggDir": "@your_stage/zingg_models", + "numPartitions": 4, + "labelDataSampleSize": 0.5, + "fieldDefinition": [ + { + "fieldName": "fname", + "matchType": "FUZZY", + "fields": "fname", + "dataType": "string" + }, + { + "fieldName": "lname", + "matchType": "FUZZY", + "fields": "lname", + "dataType": "string" + } + ] +} + +zingg = ZinggEnterprise(session, config) +``` + +### **Step 3: Run Zingg Phases** + +Run phases directly from Python/Snowflake - no external infrastructure needed: + +```python +# Find training data +zingg.find_training_data() + +# Label interactively (opens labeling UI in Snowsight or Jupyter) +zingg.label() + +# Train the model +zingg.train() + +# Run matching +results = zingg.match() + +# View results +results.show() + +# Check statistics +stats = zingg.get_statistics() +print(stats) +``` + +### **Step 4: Incremental Matching** + +For incremental loads: + +```python +# Configure incremental run +incremental_config = { + **config, + "incremental": True, + "lastRunTimestamp": "2024-01-15 00:00:00" +} + +zingg_incremental = ZinggEnterprise(session, incremental_config) +results = zingg_incremental.match() +``` + +### **Key Differences from Community Edition** + +| Feature | Community | Enterprise | +|---------|-----------|------------| +| Infrastructure | EC2 + Docker + Lambda | Native Snowpark | +| Deployment | Manual setup | Snowflake Marketplace / Package | +| Scaling | EC2 instance sizing | Snowflake warehouse scaling | +| Execution | Spark on EC2 | Snowpark on Snowflake | +| Monitoring | Lambda logs + EC2 | Snowflake Query History | +| Incremental | Via Lambda | Native API | + +### **Snowflake Marketplace Deployment (Recommended)** + +1. Search for "Zingg Enterprise" in Snowflake Marketplace +2. Click **Get** to install in your Snowflake account +3. Follow the setup wizard to configure database, schema, and warehouse +4. Access Zingg functions directly from SQL: + +```sql +-- Run Zingg phases from SQL +CALL ZINGG.FIND_TRAINING_DATA(); +CALL ZINGG.LABEL(); +CALL ZINGG.TRAIN(); +CALL ZINGG.MATCH(); +``` + +### **Getting Enterprise Access** + +Zingg Enterprise for Snowflake requires a commercial license. To get started: + +1. Visit [Zingg Enterprise Snowflake](https://www.zingg.ai/company/zingg-enterprise-snowflake) +2. Contact sales for a trial license +3. Receive the Enterprise package and installation instructions +4. Follow this guide for setup + +For detailed API reference, see the [Zingg Python API Documentation](../zingg-python-api/enterprise-zingges-python-api.md). {% endtab %} -{% endtabs %} +{% endtabs %} \ No newline at end of file diff --git a/docs/pretrainedModels.md b/docs/pretrainedModels.md index 88c5ca2e8..3a1ad9ee9 100644 --- a/docs/pretrainedModels.md +++ b/docs/pretrainedModels.md @@ -1,4 +1,6 @@ --- +description: >- + Pre-trained Zingg models for common datasets including Febrl, North Carolina Voters, and Amazon-Google products. nav_order: 6 parent: Zingg Models --- diff --git a/docs/pythonEC/markdown/ApproverArguments.md b/docs/pythonEC/markdown/ApproverArguments.md index 8d759e8a6..015a569bb 100644 --- a/docs/pythonEC/markdown/ApproverArguments.md +++ b/docs/pythonEC/markdown/ApproverArguments.md @@ -1,3 +1,8 @@ +--- +description: >- + ApproverArguments class reference for Zingg Enterprise cluster approval. Configure approval queries, destination output, and parent EArguments for manual cluster review workflow. +--- + # ApproverArguments ## zinggEC.enterprise.common.ApproverArguments diff --git a/docs/pythonEC/markdown/EArguments.md b/docs/pythonEC/markdown/EArguments.md index b844e6f9b..b2f7e50c8 100644 --- a/docs/pythonEC/markdown/EArguments.md +++ b/docs/pythonEC/markdown/EArguments.md @@ -1,3 +1,8 @@ +--- +description: >- + EArguments class reference for Zingg Enterprise. Extends Arguments with blocking model, deterministic matching, primary key, output stats, and passthrough expression. +--- + # EArguments ## zinggEC.enterprise.common.EArguments diff --git a/docs/pythonEC/markdown/EFieldDefinition.md b/docs/pythonEC/markdown/EFieldDefinition.md index fe752f496..b3f8256ad 100644 --- a/docs/pythonEC/markdown/EFieldDefinition.md +++ b/docs/pythonEC/markdown/EFieldDefinition.md @@ -1,3 +1,8 @@ +--- +description: >- + EFieldDefinition class reference for Zingg Enterprise. Extends FieldDefinition with primary key support and post-processors for output value standardization. +--- + # EFieldDefinition ## zinggEC.enterprise.common.EFieldDefinition diff --git a/docs/pythonEC/markdown/IncrementalArguments.md b/docs/pythonEC/markdown/IncrementalArguments.md index cd90cf494..0f9b94d18 100644 --- a/docs/pythonEC/markdown/IncrementalArguments.md +++ b/docs/pythonEC/markdown/IncrementalArguments.md @@ -1,3 +1,8 @@ +--- +description: >- + IncrementalArguments class reference for Zingg Enterprise. Configure incremental data, delete actions, output temp path, and parent EArguments. +--- + # IncrementalArguments ## zinggEC.enterprise.common.IncrementalArguments diff --git a/docs/pythonEC/markdown/MappingMatchType.md b/docs/pythonEC/markdown/MappingMatchType.md index 8cd742084..04b5ed1ca 100644 --- a/docs/pythonEC/markdown/MappingMatchType.md +++ b/docs/pythonEC/markdown/MappingMatchType.md @@ -1,3 +1,8 @@ +--- +description: >- + MappingMatchType class reference for Zingg Enterprise. Use domain expertise (nicknames, company abbreviations) to enhance matching accuracy via JSON mapping files. +--- + # MappingMatchType ## zinggEC.enterprise.common.MappingMatchType diff --git a/docs/pythonEC/markdown/epipes.md b/docs/pythonEC/markdown/epipes.md index 79cba520e..379f8577d 100644 --- a/docs/pythonEC/markdown/epipes.md +++ b/docs/pythonEC/markdown/epipes.md @@ -1,3 +1,8 @@ +--- +description: >- + Enterprise pipes module reference. Includes ECsvPipe for CSV files, EPipe base class with passthrough support, InMemoryPipe, and UCPipe for Unity Catalog Delta tables. +--- + # epipes ## zinggEC.enterprise.common.epipes diff --git a/docs/pythonEC/markdown/index.md b/docs/pythonEC/markdown/index.md index f2bcf2851..7a4eccd56 100644 --- a/docs/pythonEC/markdown/index.md +++ b/docs/pythonEC/markdown/index.md @@ -1,3 +1,8 @@ +--- +description: >- + Zingg Enterprise Entity Resolution Python Package reference. Complete API documentation for EC, ES packages including EArguments, EFieldDefinition, epipes, IncrementalArguments, and example usage. +--- +