Merge branch 'datahub-project:master' into master

Author: anshbansal

.github/workflows/docker-unified.yml

@@ -76,7 +76,7 @@ task unversionZip(type: Copy, dependsOn: [':datahub-web-react:distZip', distZip]
 
 docker {
   dependsOn(stageMainDist)
-  name "${docker_registry}/${docker_repo}:v${version}"
+  name "${docker_registry}/${docker_repo}:${versionTag}"
   dockerfile file("${rootProject.projectDir}/docker/${docker_dir}/Dockerfile")
   files "${buildDir}/stage"
   files fileTree(rootProject.projectDir) {

datahub-frontend/build.gradle

@@ -172,7 +172,7 @@ task runNoCode(type: Exec) {
 
 docker {
   dependsOn(bootJar)
-  name "${docker_registry}/${docker_repo}:v${version}"
+  name "${docker_registry}/${docker_repo}:${versionTag}"
   dockerfile file("${rootProject.projectDir}/docker/${docker_repo}/Dockerfile")
   files bootJar.outputs.files
   files fileTree(rootProject.projectDir) {

datahub-upgrade/build.gradle

@@ -257,6 +257,7 @@ quickstart_configs.each { taskName, config ->
                 // Only restart containers that had their modules rebuilt
                 if (containersToRestart) {
                     def cmd = ["docker compose -p datahub --profile ${config.profile}"] + ['-f', compose_base] + ['restart'] + containersToRestart
+                    println(cmd.join(" "))
                     commandLine 'bash', '-c', cmd.join(" ")
                 } else {
                     // If no containers need restart, make this a no-op

docker/build.gradle

@@ -17,8 +17,7 @@ ext {
 
 docker {
     dependsOn build
-    name "${docker_registry}/${docker_repo}:v${docker_version}"
-    //version "v${docker_version}"
+    name "${docker_registry}/${docker_repo}:${docker_version}"
     dockerfile file("${rootProject.projectDir}/docker/${docker_dir}/Dockerfile")
     files fileTree(rootProject.projectDir) {
         include '.dockerignore'

docker/datahub-ingestion-base/build.gradle

@@ -22,7 +22,7 @@ dependencies {
 
 docker {
     dependsOn 'build', ':docker:datahub-ingestion-base:docker', ':metadata-ingestion:codegen'
-    name "${docker_registry}/${docker_repo}:v${docker_version}"
+    name "${docker_registry}/${docker_repo}:${docker_version}"
     dockerfile file("${rootProject.projectDir}/docker/${docker_dir}/Dockerfile${docker_target == "slim" ? "-slim-only" : ""}")
     files fileTree(rootProject.projectDir) {
         include '.dockerignore'

docker/datahub-ingestion/build.gradle

@@ -12,7 +12,7 @@ ext {
 
 docker {
     dependsOn(build)
-    name "${docker_registry}/${docker_repo}:v${version}"
+    name "${docker_registry}/${docker_repo}:${versionTag}"
     dockerfile file("${rootProject.projectDir}/docker/${docker_dir}/Dockerfile")
     files fileTree(rootProject.projectDir) {
         include '.dockerignore'

docker/elasticsearch-setup/build.gradle

@@ -12,7 +12,7 @@ ext {
 
 docker {
     dependsOn(build)
-    name "${docker_registry}/${docker_repo}:v${version}"
+    name "${docker_registry}/${docker_repo}:${versionTag}"
     dockerfile file("${rootProject.projectDir}/docker/${docker_dir}/Dockerfile")
     files fileTree(rootProject.projectDir) {
         include '.dockerignore'

docker/kafka-setup/build.gradle

@@ -13,7 +13,7 @@ ext {
 
 docker {
     dependsOn build
-    name "${docker_registry}/${docker_repo}:v${version}"
+    name "${docker_registry}/${docker_repo}:${versionTag}"
     dockerfile file("${rootProject.projectDir}/docker/${docker_dir}/Dockerfile")
     files fileTree(rootProject.projectDir) {
         include '.dockerignore'

docker/mysql-setup/build.gradle

@@ -12,7 +12,7 @@ ext {
 }
 
 docker {
-    name "${docker_registry}/${docker_repo}:v${version}"
+    name "${docker_registry}/${docker_repo}:${versionTag}"
     dockerfile file("${rootProject.projectDir}/docker/${docker_dir}/Dockerfile")
     files fileTree(rootProject.projectDir) {
         include '.dockerignore'

docker/postgres-setup/build.gradle

@@ -367,6 +367,7 @@ module.exports = {
         },
         {
           "DataHub Cloud Release History": [
+            "docs/managed-datahub/release-notes/v_0_3_10",
             "docs/managed-datahub/release-notes/v_0_3_9",
             "docs/managed-datahub/release-notes/v_0_3_8",
             "docs/managed-datahub/release-notes/v_0_3_7",

docs-website/sidebars.js

@@ -0,0 +1,68 @@
+# v0.3.10
+---
+
+Release Availability Date
+---
+4-April-2025
+
+Recommended CLI/SDK
+---
+1.0.0.1
+
+Helm Chart
+---
+1.5.20
+
+## Known Issues
+
+- API Tracing: Some aspects (i.e. `siblings`) which can be generated by system hooks are reported as a trace error if the system aspect and ingestion are creating conflicting aspects.
+
+## Release Changelog
+---
+
+### v0.3.10
+
+- All changes in https://github.com/datahub-project/datahub/releases/tag/v1.0.0
+
+- Breaking Changes
+
+- Bug Fixes
+  - [Schema Tab] Render rich text in the V1 experience where you can click on links instead of having to click "View more"
+  - [Subscriptions] Fix edge case where we were not showing display names for users or groups properly in subscription summaries
+  - [Nav Bar] Fix the styling of the logo in the new nav bar header experience in Safari
+  - [Group Membership] Harden the UI to prevent errors when showing group memberships on a user profile page
+  - [Structured Properties] Allow users to search for Glossary Nodes when applying asset values to structured properties
+  - [Lineage] Hide a few unwanted and unneeded filters in the lineage list view from the UI
+  - [Lineage] Remove restriction on per-hop node limit, causing some results to be hidden.
+  - [Lineage] Fix an issue where indirect upstream lineage was incomplete for certain entity types like Charts. Both Impact Analysis & Lineage Graph. This was also causing Upstream Health sidebar section to underreport upstream assets with health problems. 
+  - [Queries Tab] Change the page size
+  - [Search Filters] Adding support for a "Has Siblings" filter (disabled by default, available by request). 
+
+- Product
+
+  - **Navigation**: Our new simplified nav bar is now switched on by default in the V2 UI
+  - **Custom Asset Actions**: Links create via the Documentation tab can now be elevated to display in page headers and search results. We've also added an `upsertLink` GraphQL endpoint for easier programmatic management of links.
+  - **Proposals**: Entity profile pages now display the new proposal types: Ownership, Domains, and Structured Properties.
+  - **Proposals**: Our redesigned proposals experience (Proposals 2.0) is now in beta (enabled by request)! Try out the improved interface for viewing proposals you can approve/decline as well as those that you've created.
+  - **Proposal Notifications**: You can now configure to be notified when you are assigned to review a proposal, or a proposal you raised is approved or denied from **Settings** > **My Notifications**.
+  - **Observe**: Introducing on-demand smart assertions in beta (enabled by request)! You can now create freshness, volume, and field metric anomaly detection checks via the UI for any table that's supported by Observe.
+  - **Observe**: Smart assertion tuning - Improve your anomaly monitors by adjusting training data lookback window, tightening or loosening the sensitivity, and adding data-point exclusion windows.
+  - **Observe**: Incidents V2 - Incidents have received a significant uplift! With capabilities to set assignees, severity, stage, and impacted assets. Also: search, group & filter incidents on the incidents tab. 
+  - **Remote Executor**: Introduces a new **Executors** tab in the **Manage Data Sources** page for improved management and observability of Remote Executor Pools
+  - **Remote Executors**: Create and provision new Executor Pools directly from the UI and optionally set them as default for future Ingestion Source configurations. Easily view the set of deployed executors, their status, configured Ingestion Sources, and current running tasks.
+  - **Compliance Forms**: Support deep linking to a specific Compliance Form completion experience inside the Compliance Center.
+  - **Search**: Support searching for Deprecated assets via a new Deprecated filter. Also, show deprecation in Lineage sidebar as an upstream health warning. 
+  
+- Platform
+  - **OpenAPI**: API Tracing improvements (Experimental)
+  - **GraphQL**: Sorting added to `scrollAcrossEntities`
+  - **Performance**: Improved access policy performance when using nested domains and containers
+  - **OpenAPI**: Operation endpoints created to fetch raw ES documents and view kafka lag on MCP and MCL topics
+  - **OpenAPI**: Additional validation for URNs and entity/aspect names when using async endpoints
+  - **OpenAPI**: Fixed timeseries aspect ingestion bug #12912
+  - **Restore Indices Job**: Support added for read-only replicas
+
+- Ingestion 
+  - **MLflow**: Support dataset ↔ run lineage with customizable platform mapping for datasets.
+  - **OpenAPI**: Ingestion can use OpenAPI instead of Rest.li, set environment variable `DATAHUB_REST_SINK_DEFAULT_ENDPOINT=OPENAPI`
+  - **API Tracing**: Ingestion can trace each async write by tracking the write to primary and search storage, set environment variable `DATAHUB_REST_TRACE_MODE=ENABLED`

docs/managed-datahub/release-notes/v_0_3_10.md

@@ -23,10 +23,14 @@ def detailedVersionString = "0.0.0-unknown-SNAPSHOT"
 def cliMajorVersion = "0.15.0" // base default cli major version
 def snapshotVersion = false
 def javaVersion = ""
+// tag for docker images. the prefix v is used in tag only if the tag is computed from a version. If a releaseVersion is supplied, use it as is.
+// This enables pr tags to be used without the v prefix. This variance was previouslyhandled in the CI steps build images without using gradle.
+def versionTag = "" 
 
 if (project.hasProperty("releaseVersion")) {
   version = releaseVersion
   detailedVersionString = releaseVersion
+  versionTag = releaseVersion
 } else {
   try {
       // apply this plugin in a try-catch block so that we can handle cases without .git directory
@@ -35,6 +39,7 @@ if (project.hasProperty("releaseVersion")) {
       detailedVersionString = gitVersion()
       version = details.lastTag
       version = version.startsWith("v")? version.substring(1): version
+      versionTag = "v" + version
       def suffix = details.isCleanTag? "": "-SNAPSHOT"
       snapshotVersion = ! details.isCleanTag
     }
@@ -78,14 +83,16 @@ if (snapshotVersion) {
       // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version
       javaVersion = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT'
     }
+    versionTag = "v" + version
 }
 
 // Note: No task, we want this executed during config phase, once for rootProject.
 def data = [
     fullVersion: detailedVersionString,
     cliMajorVersion: cliMajorVersion,
     version: version,
-    javaVersion: javaVersion
+    javaVersion: javaVersion,
+    versionTag: versionTag
 ]
 
 // Convert to JSON

gradle/versioning/versioning-global.gradle

@@ -4,9 +4,12 @@ import org.apache.tools.ant.filters.ReplaceTokens
 
 def detailedVersionString = "0.0.0-unknown-SNAPSHOT"
 def cliMajorVersion = "0.15.0" // base default cli major version
-
 def inputFile = file("${rootProject.buildDir}/version.json")
 
+ext {
+  versionTag = "v${detailedVersionString}"
+}
+
 task readJsonData {
   if (inputFile.exists()) {
     def jsonSlurper = new JsonSlurper()
@@ -15,6 +18,7 @@ task readJsonData {
     detailedVersionString = data.fullVersion
     cliMajorVersion = data.cliMajorVersion
     version = data.version
+    versionTag = data.versionTag
   } else {
     println "git.properties JSON file not found: ${inputFile.path}"
   }

gradle/versioning/versioning.gradle

@@ -47,7 +47,7 @@ bootJar {
 
 docker {
     dependsOn(bootJar)
-    name "${docker_registry}/${docker_repo}:v${version}"
+    name "${docker_registry}/${docker_repo}:${versionTag}"
     //version "v${version}"
     dockerfile file("${rootProject.projectDir}/docker/${docker_repo}/Dockerfile")
     files bootJar.outputs.files

metadata-jobs/mae-consumer-job/build.gradle

@@ -58,7 +58,7 @@ bootJar {
 
 docker {
     dependsOn(bootJar)
-    name "${docker_registry}/${docker_repo}:v${version}"
+    name "${docker_registry}/${docker_repo}:${versionTag}"
     //version "v${version}"
     dockerfile file("${rootProject.projectDir}/docker/${docker_repo}/Dockerfile")
     files bootJar.outputs.files

metadata-jobs/mce-consumer-job/build.gradle

@@ -86,7 +86,7 @@ bootRun {
 
 docker {
   dependsOn bootJar
-  name "${docker_registry}/${docker_repo}:v${version}"
+  name "${docker_registry}/${docker_repo}:${versionTag}"
   dockerfile file("${rootProject.projectDir}/docker/${docker_repo}/Dockerfile")
   files bootJar.outputs.files
   files fileTree(rootProject.projectDir) {
@@ -123,4 +123,4 @@ test {
                 "com.linkedin.gms.ServletConfig",
                 "com.linkedin.gms.GMSApplication"]
   }
-}
+}

metadata-service/war/build.gradle

@@ -49,6 +49,43 @@ def pytest_sessionfinish(session, exitstatus):
     send_message(exitstatus)
 
 
+def bin_pack_tasks(tasks, n_buckets):
+    """
+    Bin-pack tasks into n_buckets with roughly equal weights.
+
+    Parameters:
+    tasks (list): List of (task, weight) tuples. If only task is provided, weight defaults to 1.
+    n_buckets (int): Number of buckets to distribute tasks into.
+
+    Returns:
+    list: List of buckets, where each bucket is a list of tasks.
+    """
+    # Normalize the tasks to ensure they're all (task, weight) tuples
+    normalized_tasks = []
+    for task in tasks:
+        if isinstance(task, tuple) and len(task) == 2:
+            normalized_tasks.append(task)
+        else:
+            normalized_tasks.append((task, 1))
+
+    # Sort tasks by weight in descending order
+    sorted_tasks = sorted(normalized_tasks, key=lambda x: x[1], reverse=True)
+
+    # Initialize the buckets with zero weight
+    buckets: List = [[] for _ in range(n_buckets)]
+    bucket_weights: List[int] = [0] * n_buckets
+
+    # Assign each task to the bucket with the lowest current weight
+    for task, weight in sorted_tasks:
+        # Find the bucket with the minimum weight
+        min_bucket_idx = bucket_weights.index(min(bucket_weights))
+
+        # Add the task to this bucket
+        buckets[min_bucket_idx].append(task)
+        bucket_weights[min_bucket_idx] += weight
+
+    return buckets
+
 def get_batch_start_end(num_tests: int) -> Tuple[int, int]:
     batch_count_env = os.getenv("BATCH_COUNT", 1)
     batch_count = int(batch_count_env)
@@ -71,16 +108,15 @@ def get_batch_start_end(num_tests: int) -> Tuple[int, int]:
     batch_end = batch_start + batch_size
     # We must have exactly as many batches as specified by BATCH_COUNT.
     if (
-        num_tests - batch_end < batch_size
-    ):  # We must have exactly as many batches as specified by BATCH_COUNT, put the remaining in the last batch.
+            batch_number == batch_count - 1  # this is the last batch
+    ):  # If ths is last batch put any remaining tests in the last batch.
         batch_end = num_tests
 
     if batch_count > 0:
         print(f"Running tests for batch {batch_number} of {batch_count}")
 
     return batch_start, batch_end
 
-
 def pytest_collection_modifyitems(
     session: pytest.Session, config: pytest.Config, items: List[Item]
 ) -> None:

smoke-test/conftest.py

@@ -36,11 +36,13 @@ source ./set-cypress-creds.sh
 # set environment variables for the test
 source ./set-test-env-vars.sh
 
+echo "TEST_STRATEGY: $TEST_STRATEGY, BATCH_COUNT: $BATCH_COUNT, BATCH_NUMBER: $BATCH_NUMBER"
+
 # TEST_STRATEGY:
 #   if set to pytests, runs all pytests, skips cypress tests(though cypress test launch is via  a pytest).
 #   if set tp cypress, runs all cypress tests
 #   if blank, runs all.
-# When invoked via the github action, BATCH_COUNT and BATCH_NUM env vars are set to run a slice of those tests per
+# When invoked via the github action, BATCH_COUNT and BATCH_NUMBER env vars are set to run a slice of those tests per
 # worker for parallelism. docker-unified.yml generates a test matrix of pytests/cypress in batches. As number of tests
 # increase, the batch_count config (in docker-unified.yml) may need adjustment.
 if [[ "${TEST_STRATEGY}" == "pytests" ]]; then
@@ -51,7 +53,7 @@ elif [[ "${TEST_STRATEGY}" == "cypress" ]]; then
   # github workflow test matrix will invoke this in multiple runners for each batch.
   # Skipping the junit at the pytest level since cypress itself generates junits on a per-test basis. The pytest is a single test for all cypress
   # tests and isnt very helpful.
-  pytest -rP --durations=20 -vv --continue-on-collection-errors tests/cypress/integration_test.py
+  pytest -rP --durations=20 -vvs --continue-on-collection-errors tests/cypress/integration_test.py
 else
-  pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke-all.xml
+  pytest -rP --durations=20 -vvs --continue-on-collection-errors --junit-xml=junit.smoke-all.xml
 fi