Skip to content

Commit 92b19ab

Browse files
david-leifkercursoragent
david-leifker
and
cursoragent
committed
feat(gms): add entity graph cache for hierarchies and membership
Introduce a Hazelcast-backed entity graph cache with bundled domain, glossary, container, and membership graphs. Route hot GraphQL, VBAC, and search paths through cache-first client bindings with aspect and scroll fallbacks. Retain corp vs native group distinction in SessionActorIdentity so session-user membership fast paths label IsMemberOfGroup and IsMemberOfNativeGroup correctly. Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent c78c7d5 commit 92b19ab

209 files changed

Lines changed: 24547 additions & 1519 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,7 @@ project.ext.externalDependency = [
241241
'jettyClient': "org.eclipse.jetty:jetty-client:$jettyVersion",
242242
'jettyJmx': "org.eclipse.jetty:jetty-jmx:$jettyVersion",
243243
'jettison': 'org.codehaus.jettison:jettison:1.5.4',
244-
'jgrapht': 'org.jgrapht:jgrapht-core:1.5.1',
244+
'jgrapht': 'org.jgrapht:jgrapht-core:1.5.3',
245245
'jna': 'net.java.dev.jna:jna:5.12.1',
246246
'jsonPatch': 'jakarta.json:jakarta.json-api:2.1.3',
247247
'jsonPathImpl': 'org.eclipse.parsson:parsson:1.1.6',

‎datahub-graphql-core/gradle.lockfile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,7 @@ org.apache.velocity:velocity-engine-core:2.4=runtimeClasspath,testRuntimeClasspa
312312
org.apache.yetus:audience-annotations:0.12.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
313313
org.apache.zookeeper:zookeeper-jute:3.8.6=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
314314
org.apache.zookeeper:zookeeper:3.8.6=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
315+
org.apfloat:apfloat:1.14.0=runtimeClasspath,testRuntimeClasspath
315316
org.checkerframework:checker-qual:3.37.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
316317
org.codehaus.jackson:jackson-core-asl:1.8.8=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
317318
org.codehaus.mojo:animal-sniffer-annotations:1.27=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
@@ -331,6 +332,8 @@ org.jacoco:org.jacoco.report:0.8.12=jacocoAnt
331332
org.javassist:javassist:3.26.0-GA=runtimeClasspath,testRuntimeClasspath
332333
org.javatuples:javatuples:1.2=runtimeClasspath,testRuntimeClasspath
333334
org.jctools:jctools-core:4.0.6=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
335+
org.jgrapht:jgrapht-core:1.5.3=runtimeClasspath,testRuntimeClasspath
336+
org.jheaps:jheaps:0.14=runtimeClasspath,testRuntimeClasspath
334337
org.json:json:20231013=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
335338
org.jspecify:jspecify:1.0.0=compileClasspath,runtimeClasspath,spotless865458226,testCompileClasspath,testRuntimeClasspath
336339
org.knowm.xchart:xchart:3.2.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -978,7 +978,9 @@ private void configureContainerResolvers(final RuntimeWiring.Builder builder) {
978978
"Container",
979979
typeWiring ->
980980
typeWiring
981-
.dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient))
981+
.dataFetcher(
982+
"relationships",
983+
new EntityRelationshipsResultResolver(graphClient, entityService))
982984
.dataFetcher(
983985
"relatedDocuments",
984986
new com.linkedin.datahub.graphql.resolvers.knowledge.RelatedDocumentsResolver(
@@ -3109,10 +3111,12 @@ private void configureDomainResolvers(final RuntimeWiring.Builder builder) {
31093111
typeWiring ->
31103112
typeWiring
31113113
.dataFetcher("entities", new DomainEntitiesResolver(this.entityClient))
3112-
.dataFetcher("parentDomains", new ParentDomainsResolver(this.entityClient))
3114+
.dataFetcher("parentDomains", new ParentDomainsResolver())
31133115
.dataFetcher("privileges", new EntityPrivilegesResolver(entityClient))
31143116
.dataFetcher("aspects", new WeaklyTypedAspectsResolver())
3115-
.dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient))
3117+
.dataFetcher(
3118+
"relationships",
3119+
new EntityRelationshipsResultResolver(graphClient, entityService))
31163120
.dataFetcher(
31173121
"relatedDocuments",
31183122
new com.linkedin.datahub.graphql.resolvers.knowledge.RelatedDocumentsResolver(

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/container/ParentContainersResolver.java‎

Lines changed: 26 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,23 @@
11
package com.linkedin.datahub.graphql.resolvers.container;
22

33
import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.getQueryContext;
4-
import static com.linkedin.metadata.Constants.CONTAINER_ASPECT_NAME;
54

65
import com.linkedin.common.urn.Urn;
7-
import com.linkedin.data.DataMap;
6+
import com.linkedin.common.urn.UrnUtils;
87
import com.linkedin.datahub.graphql.QueryContext;
98
import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils;
10-
import com.linkedin.datahub.graphql.exception.DataHubGraphQLException;
119
import com.linkedin.datahub.graphql.generated.Container;
1210
import com.linkedin.datahub.graphql.generated.Entity;
1311
import com.linkedin.datahub.graphql.generated.ParentContainersResult;
1412
import com.linkedin.datahub.graphql.types.container.mappers.ContainerMapper;
1513
import com.linkedin.entity.EntityResponse;
1614
import com.linkedin.entity.client.EntityClient;
15+
import com.linkedin.metadata.graph.cache.client.BoundHierarchyAccess;
16+
import com.linkedin.metadata.graph.cache.client.HierarchyBindings;
1717
import graphql.schema.DataFetcher;
1818
import graphql.schema.DataFetchingEnvironment;
1919
import java.util.ArrayList;
20-
import java.util.Collections;
21-
import java.util.HashSet;
2220
import java.util.List;
23-
import java.util.Set;
2421
import java.util.concurrent.CompletableFuture;
2522

2623
public class ParentContainersResolver
@@ -32,67 +29,39 @@ public ParentContainersResolver(final EntityClient entityClient) {
3229
_entityClient = entityClient;
3330
}
3431

35-
private void aggregateParentContainers(
36-
List<Container> containers,
37-
String urn,
38-
QueryContext context,
39-
Set<String> visitedUrns,
40-
int depth) {
41-
if (depth >= context.getMaxParentDepth() || !visitedUrns.add(urn)) {
42-
return;
43-
}
44-
try {
45-
Urn entityUrn = new Urn(urn);
46-
EntityResponse entityResponse =
47-
_entityClient.getV2(
48-
context.getOperationContext(),
49-
entityUrn.getEntityType(),
50-
entityUrn,
51-
Collections.singleton(CONTAINER_ASPECT_NAME));
52-
53-
if (entityResponse != null
54-
&& entityResponse.getAspects().containsKey(CONTAINER_ASPECT_NAME)) {
55-
DataMap dataMap = entityResponse.getAspects().get(CONTAINER_ASPECT_NAME).getValue().data();
56-
com.linkedin.container.Container container = new com.linkedin.container.Container(dataMap);
57-
Urn containerUrn = container.getContainer();
58-
if (containerUrn == null || visitedUrns.contains(containerUrn.toString())) {
59-
return;
60-
}
61-
EntityResponse response =
62-
_entityClient.getV2(
63-
context.getOperationContext(), containerUrn.getEntityType(), containerUrn, null);
64-
if (response != null) {
65-
Container mappedContainer = ContainerMapper.map(context, response);
66-
containers.add(mappedContainer);
67-
aggregateParentContainers(
68-
containers, mappedContainer.getUrn(), context, visitedUrns, depth + 1);
69-
}
70-
}
71-
} catch (Exception e) {
72-
throw new RuntimeException("Failed to retrieve parent containers from GMS", e);
73-
}
74-
}
75-
7632
@Override
7733
public CompletableFuture<ParentContainersResult> get(DataFetchingEnvironment environment) {
7834

7935
final QueryContext context = getQueryContext(environment);
80-
final String urn = ((Entity) environment.getSource()).getUrn();
81-
final List<Container> containers = new ArrayList<>();
36+
final Urn urn = UrnUtils.getUrn(((Entity) environment.getSource()).getUrn());
37+
8238
return GraphQLConcurrencyUtils.supplyAsync(
8339
() -> {
8440
try {
85-
Set<String> visitedUrns = new HashSet<>();
86-
aggregateParentContainers(containers, urn, context, visitedUrns, 0);
87-
final ParentContainersResult result = new ParentContainersResult();
41+
List<Urn> parentUrns =
42+
BoundHierarchyAccess.orderedParents(
43+
context.getOperationContext(),
44+
HierarchyBindings.containerSpec(context.getOperationContext()),
45+
urn,
46+
context.getMaxParentDepth());
8847

89-
List<Container> viewable = new ArrayList<>(containers);
48+
List<Container> containers = new ArrayList<>();
49+
for (Urn parentUrn : parentUrns) {
50+
EntityResponse response =
51+
_entityClient.getV2(
52+
context.getOperationContext(), parentUrn.getEntityType(), parentUrn, null);
53+
if (response != null) {
54+
containers.add(ContainerMapper.map(context, response));
55+
}
56+
}
9057

91-
result.setCount(viewable.size());
92-
result.setContainers(viewable);
58+
final ParentContainersResult result = new ParentContainersResult();
59+
result.setCount(containers.size());
60+
result.setContainers(containers);
9361
return result;
94-
} catch (DataHubGraphQLException e) {
95-
throw new RuntimeException("Failed to load all containers", e);
62+
} catch (Exception e) {
63+
throw new RuntimeException(
64+
String.format("Failed to load parent containers for entity %s", urn), e);
9665
}
9766
},
9867
this.getClass().getSimpleName(),

‎datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolver.java‎

Lines changed: 11 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -10,31 +10,21 @@
1010
import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils;
1111
import com.linkedin.datahub.graphql.generated.Entity;
1212
import com.linkedin.datahub.graphql.generated.ParentDomainsResult;
13-
import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils;
14-
import com.linkedin.entity.client.EntityClient;
13+
import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper;
14+
import com.linkedin.metadata.graph.cache.client.BoundHierarchyAccess;
15+
import com.linkedin.metadata.graph.cache.client.HierarchyBindings;
1516
import graphql.schema.DataFetcher;
1617
import graphql.schema.DataFetchingEnvironment;
17-
import java.util.ArrayList;
18-
import java.util.HashSet;
1918
import java.util.List;
20-
import java.util.Set;
2119
import java.util.concurrent.CompletableFuture;
2220
import java.util.stream.Collectors;
2321

2422
public class ParentDomainsResolver implements DataFetcher<CompletableFuture<ParentDomainsResult>> {
2523

26-
private final EntityClient _entityClient;
27-
28-
public ParentDomainsResolver(final EntityClient entityClient) {
29-
_entityClient = entityClient;
30-
}
31-
3224
@Override
3325
public CompletableFuture<ParentDomainsResult> get(DataFetchingEnvironment environment) {
3426
final QueryContext context = getQueryContext(environment);
3527
final Urn urn = UrnUtils.getUrn(((Entity) environment.getSource()).getUrn());
36-
final List<Entity> parentDomains = new ArrayList<>();
37-
final Set<String> visitedParentUrns = new HashSet<>();
3828

3929
if (!DOMAIN_ENTITY_NAME.equals(urn.getEntityType())) {
4030
throw new IllegalArgumentException(
@@ -44,22 +34,16 @@ public CompletableFuture<ParentDomainsResult> get(DataFetchingEnvironment enviro
4434
return GraphQLConcurrencyUtils.supplyAsync(
4535
() -> {
4636
try {
47-
visitedParentUrns.add(urn.toString());
48-
Entity parentDomain = DomainUtils.getParentDomain(urn, context, _entityClient);
49-
int depth = 0;
50-
51-
while (parentDomain != null
52-
&& depth < context.getMaxParentDepth()
53-
&& visitedParentUrns.add(parentDomain.getUrn())) {
54-
parentDomains.add(parentDomain);
55-
depth++;
56-
parentDomain =
57-
DomainUtils.getParentDomain(
58-
Urn.createFromString(parentDomain.getUrn()), context, _entityClient);
59-
}
37+
List<Urn> parentUrns =
38+
BoundHierarchyAccess.orderedParents(
39+
context.getOperationContext(),
40+
HierarchyBindings.domainSpec(context.getOperationContext()),
41+
urn,
42+
context.getMaxParentDepth());
6043

6144
List<Entity> viewable =
62-
parentDomains.stream()
45+
parentUrns.stream()
46+
.map(parentUrn -> UrnToEntityMapper.map(context, parentUrn))
6347
.filter(
6448
e ->
6549
canViewRelationship(

0 commit comments

Comments
 (0)