Skip to content

Commit 891f393

Browse files
pedro93pedro93
authored
Merge branch 'datahub-project:master' into master
2 parents 9b0513a + 1e6db06 commit 891f393

File tree

7 files changed

+140
-33
lines changed

7 files changed

+140
-33
lines changed

‎.github/workflows/react-cloudflare-pages.yml

+1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ jobs:
3737
deployments: write
3838
timeout-minutes: 30
3939
needs: setup
40+
if: !github.event.pull_request.head.repo.fork
4041
steps:
4142
- name: Check out the repo
4243
uses: acryldata/sane-checkout-action@v3

‎datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/AbstractMCLStep.java

+1-3
Original file line numberDiff line numberDiff line change
@@ -113,9 +113,7 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
113113
List<Pair<Future<?>, SystemAspect>> futures;
114114
futures =
115115
EntityUtils.toSystemAspectFromEbeanAspects(
116-
opContext.getRetrieverContext(),
117-
batch.collect(Collectors.toList()),
118-
false)
116+
opContext.getRetrieverContext(), batch.collect(Collectors.toList()))
119117
.stream()
120118
.map(
121119
systemAspect -> {

‎datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/GenerateSchemaFieldsFromSchemaMetadataStep.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -175,8 +175,7 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
175175
ebeanAspectV2 ->
176176
EntityUtils.toSystemAspectFromEbeanAspects(
177177
opContext.getRetrieverContext(),
178-
Set.of(ebeanAspectV2),
179-
true)
178+
Set.of(ebeanAspectV2))
180179
.stream())
181180
.map(
182181
systemAspect ->

‎metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java

+6-11
Original file line numberDiff line numberDiff line change
@@ -342,8 +342,7 @@ public Map<Urn, List<RecordTemplate>> getLatestAspects(
342342
}
343343

344344
List<SystemAspect> systemAspects =
345-
EntityUtils.toSystemAspects(
346-
opContext.getRetrieverContext(), batchGetResults.values(), false);
345+
EntityUtils.toSystemAspects(opContext.getRetrieverContext(), batchGetResults.values());
347346

348347
systemAspects.stream()
349348
// for now, don't add the key aspect here we have already added it above
@@ -371,8 +370,7 @@ public Map<String, RecordTemplate> getLatestAspectsForUrn(
371370
Map<EntityAspectIdentifier, EntityAspect> batchGetResults =
372371
getLatestAspect(opContext, new HashSet<>(Arrays.asList(urn)), aspectNames, forUpdate);
373372

374-
return EntityUtils.toSystemAspects(
375-
opContext.getRetrieverContext(), batchGetResults.values(), false)
373+
return EntityUtils.toSystemAspects(opContext.getRetrieverContext(), batchGetResults.values())
376374
.stream()
377375
.map(
378376
systemAspect -> Pair.of(systemAspect.getAspectName(), systemAspect.getRecordTemplate()))
@@ -802,7 +800,7 @@ public ListResult<RecordTemplate> listLatestAspects(
802800
}
803801

804802
return new ListResult<>(
805-
EntityUtils.toSystemAspects(opContext.getRetrieverContext(), entityAspects, false).stream()
803+
EntityUtils.toSystemAspects(opContext.getRetrieverContext(), entityAspects).stream()
806804
.map(SystemAspect::getRecordTemplate)
807805
.collect(Collectors.toList()),
808806
aspectMetadataList.getMetadata(),
@@ -1675,9 +1673,7 @@ public List<RestoreIndicesResult> restoreIndices(
16751673
try {
16761674
List<SystemAspect> systemAspects =
16771675
EntityUtils.toSystemAspectFromEbeanAspects(
1678-
opContext.getRetrieverContext(),
1679-
batch.collect(Collectors.toList()),
1680-
false);
1676+
opContext.getRetrieverContext(), batch.collect(Collectors.toList()));
16811677

16821678
RestoreIndicesResult result =
16831679
restoreIndices(opContext, systemAspects, logger, args.createDefaultAspects());
@@ -1730,8 +1726,7 @@ public List<RestoreIndicesResult> restoreIndices(
17301726
List<SystemAspect> systemAspects =
17311727
EntityUtils.toSystemAspects(
17321728
opContext.getRetrieverContext(),
1733-
getLatestAspect(opContext, entityBatch.getValue(), aspectNames, false).values(),
1734-
false);
1729+
getLatestAspect(opContext, entityBatch.getValue(), aspectNames, false).values());
17351730
long timeSqlQueryMs = System.currentTimeMillis() - startTime;
17361731

17371732
RestoreIndicesResult result =
@@ -2839,7 +2834,7 @@ private Map<EntityAspectIdentifier, EnvelopedAspect> getEnvelopedAspects(
28392834
final Map<EntityAspectIdentifier, EntityAspect> dbEntries = aspectDao.batchGet(dbKeys, false);
28402835

28412836
List<SystemAspect> envelopedAspects =
2842-
EntityUtils.toSystemAspects(opContext.getRetrieverContext(), dbEntries.values(), false);
2837+
EntityUtils.toSystemAspects(opContext.getRetrieverContext(), dbEntries.values());
28432838

28442839
return envelopedAspects.stream()
28452840
.collect(

‎metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java

+8-17
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ public static Optional<SystemAspect> toSystemAspect(
170170
@Nullable EntityAspect entityAspect,
171171
boolean forUpdate) {
172172
return Optional.ofNullable(entityAspect)
173-
.map(aspect -> toSystemAspects(retrieverContext, List.of(aspect), forUpdate))
173+
.map(aspect -> toSystemAspects(retrieverContext, List.of(aspect)))
174174
.filter(systemAspects -> !systemAspects.isEmpty())
175175
.map(systemAspects -> systemAspects.get(0));
176176
}
@@ -186,15 +186,13 @@ public static Optional<SystemAspect> toSystemAspect(
186186
@Nonnull
187187
public static Map<String, Map<String, SystemAspect>> toSystemAspects(
188188
@Nonnull RetrieverContext retrieverContext,
189-
@Nonnull Map<String, Map<String, EntityAspect>> rawAspects,
190-
boolean forUpdate) {
189+
@Nonnull Map<String, Map<String, EntityAspect>> rawAspects) {
191190
List<SystemAspect> systemAspects =
192191
toSystemAspects(
193192
retrieverContext,
194193
rawAspects.values().stream()
195194
.flatMap(m -> m.values().stream())
196-
.collect(Collectors.toList()),
197-
forUpdate);
195+
.collect(Collectors.toList()));
198196

199197
// map the list into the desired shape
200198
return systemAspects.stream()
@@ -217,13 +215,10 @@ public static Map<String, Map<String, SystemAspect>> toSystemAspects(
217215

218216
@Nonnull
219217
public static List<SystemAspect> toSystemAspectFromEbeanAspects(
220-
@Nonnull RetrieverContext retrieverContext,
221-
@Nonnull Collection<EbeanAspectV2> rawAspects,
222-
boolean forUpdate) {
218+
@Nonnull RetrieverContext retrieverContext, @Nonnull Collection<EbeanAspectV2> rawAspects) {
223219
return toSystemAspects(
224220
retrieverContext,
225-
rawAspects.stream().map(EbeanAspectV2::toEntityAspect).collect(Collectors.toList()),
226-
forUpdate);
221+
rawAspects.stream().map(EbeanAspectV2::toEntityAspect).collect(Collectors.toList()));
227222
}
228223

229224
/**
@@ -232,14 +227,14 @@ public static List<SystemAspect> toSystemAspectFromEbeanAspects(
232227
* <p>This should be the 1 point that all conversions from database representations to java
233228
* objects happens since we need to enforce read mutations happen.
234229
*
230+
* @param retrieverContext retriever context to fetch with
235231
* @param rawAspects raw aspects to convert
236232
* @return map converted aspects
237233
*/
238234
@Nonnull
239235
public static List<SystemAspect> toSystemAspects(
240236
@Nonnull final RetrieverContext retrieverContext,
241-
@Nonnull Collection<EntityAspect> rawAspects,
242-
boolean forUpdate) {
237+
@Nonnull Collection<EntityAspect> rawAspects) {
243238
EntityRegistry entityRegistry = retrieverContext.getAspectRetriever().getEntityRegistry();
244239

245240
// Build
@@ -258,11 +253,7 @@ public static List<SystemAspect> toSystemAspects(
258253
aspectSpec != null,
259254
String.format("Aspect %s could not be found", raw.getAspect()));
260255

261-
if (forUpdate) {
262-
return EntityAspect.EntitySystemAspect.builder().forUpdate(raw, entityRegistry);
263-
} else {
264-
return EntityAspect.EntitySystemAspect.builder().forInsert(raw, entityRegistry);
265-
}
256+
return EntityAspect.EntitySystemAspect.builder().forUpdate(raw, entityRegistry);
266257
})
267258
.collect(Collectors.toList());
268259

‎smoke-test/tests/openapi/v3/aspects.json

+121
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
[
2+
{
3+
"request": {
4+
"url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2Cdataset1AspectV3%2CPROD%29",
5+
"description": "Remove test dataset 1",
6+
"method": "delete"
7+
}
8+
},
9+
{
10+
"request": {
11+
"url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2Cdataset1AspectV3%2CPROD%29/globaltags",
12+
"params": {
13+
"createIfNotExists": "false",
14+
"createEntityIfNotExists": "false",
15+
"async": "false"
16+
},
17+
"description": "Add 1 tag",
18+
"json": {
19+
"value": {
20+
"tags": [
21+
{
22+
"tag": "urn:li:tag:tag1"
23+
}
24+
]
25+
}
26+
}
27+
}
28+
},
29+
{
30+
"request": {
31+
"url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2Cdataset1AspectV3%2CPROD%29/globaltags",
32+
"params": {
33+
"createIfNotExists": "false",
34+
"createEntityIfNotExists": "false",
35+
"async": "false"
36+
},
37+
"description": "Add 2 tags",
38+
"json": {
39+
"value": {
40+
"tags": [
41+
{
42+
"tag": "urn:li:tag:tag1"
43+
},
44+
{
45+
"tag": "urn:li:tag:tag2"
46+
}
47+
]
48+
}
49+
}
50+
}
51+
},
52+
{
53+
"request": {
54+
"url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2Cdataset1AspectV3%2CPROD%29/globaltags",
55+
"params": {
56+
"version": "0"
57+
},
58+
"method": "get",
59+
"description": "Get version 0"
60+
},
61+
"response": {
62+
"json": {
63+
"value": {
64+
"tags": [
65+
{
66+
"tag": "urn:li:tag:tag1"
67+
},
68+
{
69+
"tag": "urn:li:tag:tag2"
70+
}
71+
]
72+
}
73+
}
74+
}
75+
},
76+
{
77+
"request": {
78+
"url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2Cdataset1AspectV3%2CPROD%29/globaltags",
79+
"params": {
80+
"version": "2"
81+
},
82+
"method": "get",
83+
"description": "Get version 2"
84+
},
85+
"response": {
86+
"json": {
87+
"value": {
88+
"tags": [
89+
{
90+
"tag": "urn:li:tag:tag1"
91+
},
92+
{
93+
"tag": "urn:li:tag:tag2"
94+
}
95+
]
96+
}
97+
}
98+
}
99+
},
100+
{
101+
"request": {
102+
"url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2Cdataset1AspectV3%2CPROD%29/globaltags",
103+
"params": {
104+
"version": "1"
105+
},
106+
"method": "get",
107+
"description": "Get version 1"
108+
},
109+
"response": {
110+
"json": {
111+
"value": {
112+
"tags": [
113+
{
114+
"tag": "urn:li:tag:tag1"
115+
}
116+
]
117+
}
118+
}
119+
}
120+
}
121+
]

‎smoke-test/tests/utilities/concurrent_openapi.py

+2
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,9 @@ def run_tests(auth_session, fixture_globs, num_workers=3):
107107
"""
108108
with concurrent.futures.ThreadPoolExecutor(max_workers=num_workers) as executor:
109109
futures = []
110+
logger.info(f"Processing openapi test glob: {fixture_globs}")
110111
for fixture_glob in fixture_globs:
112+
logger.info(f"Processing openapi test: {fixture_glob}")
111113
for test_fixture, test_data in load_tests(fixture_glob=fixture_glob):
112114
futures.append(
113115
executor.submit(

0 commit comments

Comments
 (0)