Skip to content

Commit 9ec26ee

Browse files
ikhoonikhoon
authored
Provide a way to view raw JSON content (line#1218)
Motivation: Exposing raw content has two benefits: - It makes it easier to compare the user's original data with the data stored in Central Dogma. - It simplifies supporting JSON5. When a JSON content is normalized via `JsonNode` before, we lose information present only in the original, such as JSON5 comments, which makes JSON5 support challenging. In this PR, the content pushed by the user or mirrored by a scheduler is stored in the Central Dogma repository in its original form and a new feature is added to return the raw content when `viewRaw` option is specified on files read or watch operations. Modifications: - Push side) - Added the `rawContent()` method to `Change`. - It stores the original value and is now part of the serialized output. - Refactored `Change` implementations to handle `rawContent()` easily. - Implemented a new serializer and deserializer for `Change` to correctly handle the new `rawContent` field. - If `rawContent` exits, `content` is excluded from the serialized output. - Migrated duplicate serializers and deserializers to use the new implementations. - `Command.push()` now returns `PushAsIsCommand` since normalization is no longer necessary. - `ExecutionContext` is added to provide additional information when executing `Command`. - Currently, it only indicates whether the `Command` is executed by a replay. - Refactor internal Git repository implementations to store the original content. - Read side) - Added the `rawContent()` method to `Entry`. - It stores the original content retrieved from Central Dogma Git repository. - Added `viewRaw` parameter to the `getFiles` REST API. - When `viewRaw` is true, `EntryDto` includes the original content. - `viewRaw` option is disabled by default in the client API for backward compatibility. - `viewRaw` option is always enabled in the web application so that users can easily compare the content with the upstream data. - Fixed front-end Typescript code to render the raw content in the viewer and store the edited data as-is. Result: Central Dogma now support storing and retrieving the original JSON content.
1 parent 5c4287a commit 9ec26ee

69 files changed

Lines changed: 1882 additions & 638 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎client/java-armeria-legacy/src/main/java/com/linecorp/centraldogma/client/armeria/legacy/LegacyCentralDogma.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -222,10 +222,11 @@ public CompletableFuture<Map<String, EntryType>> listFiles(String projectName, S
222222

223223
@Override
224224
public <T> CompletableFuture<Entry<T>> getFile(String projectName, String repositoryName,
225-
Revision revision, Query<T> query) {
225+
Revision revision, Query<T> query, boolean viewRaw) {
226226
requireNonNull(query, "query");
227227
return maybeNormalizeRevision(projectName, repositoryName, revision).thenCompose(normRev -> {
228228
final CompletableFuture<GetFileResult> future = run(callback -> {
229+
// viewRaw is not supported in LegacyCentralDogma.
229230
client.getFile(projectName, repositoryName,
230231
RevisionConverter.TO_DATA.convert(normRev),
231232
QueryConverter.TO_DATA.convert(query), callback);
@@ -279,11 +280,13 @@ private static <T> Entry<T> entryAsText(Query<T> query, Revision normRev, String
279280

280281
@Override
281282
public CompletableFuture<Map<String, Entry<?>>> getFiles(String projectName, String repositoryName,
282-
Revision revision, PathPattern pathPattern) {
283+
Revision revision, PathPattern pathPattern,
284+
boolean viewRaw) {
283285
requireNonNull(pathPattern, "pathPattern");
284286
return maybeNormalizeRevision(projectName, repositoryName, revision).thenCompose(normRev -> {
285287
final CompletableFuture<List<com.linecorp.centraldogma.internal.thrift.Entry>> future =
286288
run(callback -> {
289+
// viewRaw is not supported in LegacyCentralDogma.
287290
client.getFiles(projectName, repositoryName,
288291
RevisionConverter.TO_DATA.convert(normRev),
289292
pathPattern.patternString(), callback);
@@ -484,12 +487,14 @@ public CompletableFuture<Revision> watchRepository(String projectName, String re
484487
@Override
485488
public <T> CompletableFuture<Entry<T>> watchFile(String projectName, String repositoryName,
486489
Revision lastKnownRevision, Query<T> query,
487-
long timeoutMillis, boolean errorOnEntryNotFound) {
490+
long timeoutMillis, boolean errorOnEntryNotFound,
491+
boolean viewRaw) {
488492
checkArgument(!errorOnEntryNotFound, "errorOnEntryNotFound is not supported in LegacyCentralDogma.");
489493
validateProjectAndRepositoryName(projectName, repositoryName);
490494
requireNonNull(lastKnownRevision, "lastKnownRevision");
491495
requireNonNull(query, "query");
492496
final CompletableFuture<WatchFileResult> future = run(callback -> {
497+
// viewRaw is not supported in LegacyCentralDogma.
493498
client.watchFile(projectName, repositoryName,
494499
RevisionConverter.TO_DATA.convert(lastKnownRevision),
495500
QueryConverter.TO_DATA.convert(query),

‎client/java-armeria/src/main/java/com/linecorp/centraldogma/internal/client/armeria/ArmeriaCentralDogma.java‎

Lines changed: 77 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,12 @@
4444

4545
import javax.annotation.Nullable;
4646

47+
import org.slf4j.Logger;
48+
import org.slf4j.LoggerFactory;
49+
4750
import com.fasterxml.jackson.core.JsonParseException;
4851
import com.fasterxml.jackson.core.JsonProcessingException;
52+
import com.fasterxml.jackson.databind.JsonMappingException;
4953
import com.fasterxml.jackson.databind.JsonNode;
5054
import com.fasterxml.jackson.databind.node.ArrayNode;
5155
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
@@ -82,7 +86,6 @@
8286
import com.linecorp.centraldogma.common.CentralDogmaException;
8387
import com.linecorp.centraldogma.common.Change;
8488
import com.linecorp.centraldogma.common.ChangeConflictException;
85-
import com.linecorp.centraldogma.common.ChangeType;
8689
import com.linecorp.centraldogma.common.Commit;
8790
import com.linecorp.centraldogma.common.Entry;
8891
import com.linecorp.centraldogma.common.EntryNotFoundException;
@@ -120,6 +123,8 @@
120123

121124
public final class ArmeriaCentralDogma extends AbstractCentralDogma {
122125

126+
private static final Logger logger = LoggerFactory.getLogger(ArmeriaCentralDogma.class);
127+
123128
private static final MediaType JSON_PATCH_UTF8 = MediaType.JSON_PATCH.withCharset(StandardCharsets.UTF_8);
124129

125130
private static final byte[] UNREMOVE_PATCH = toBytes(JsonNodeFactory.instance.arrayNode(1).add(
@@ -462,7 +467,7 @@ private static Map<String, EntryType> listFiles(AggregatedHttpResponse res) {
462467

463468
@Override
464469
public <T> CompletableFuture<Entry<T>> getFile(String projectName, String repositoryName, Revision revision,
465-
Query<T> query) {
470+
Query<T> query, boolean viewRaw) {
466471
validateProjectAndRepositoryName(projectName, repositoryName);
467472
requireNonNull(revision, "revision");
468473
requireNonNull(query, "query");
@@ -472,29 +477,34 @@ public <T> CompletableFuture<Entry<T>> getFile(String projectName, String reposi
472477
final StringBuilder path = pathBuilder(projectName, repositoryName);
473478
path.append("/contents").append(query.path());
474479
path.append("?revision=").append(normRev.text());
480+
if (viewRaw) {
481+
path.append("&viewRaw=true");
482+
}
475483
appendJsonPaths(path, query.type(), query.expressions());
476484

477485
return client.execute(headers(HttpMethod.GET, path.toString()))
478486
.aggregate()
479-
.thenApply(res -> getFile(normRev, res, query));
487+
.thenApply(res -> getFile(normRev, res, query, viewRaw));
480488
});
481489
} catch (Exception e) {
482490
return exceptionallyCompletedFuture(e);
483491
}
484492
}
485493

486-
private static <T> Entry<T> getFile(Revision normRev, AggregatedHttpResponse res, Query<T> query) {
494+
private static <T> Entry<T> getFile(Revision normRev, AggregatedHttpResponse res, Query<T> query,
495+
boolean viewRaw) {
487496
if (res.status().code() == 200) {
488497
final JsonNode node = toJson(res, JsonNodeType.OBJECT);
489-
return toEntry(normRev, node, query.type());
498+
return toEntry(normRev, node, query.type(), viewRaw);
490499
}
491500

492501
return handleErrorResponse(res);
493502
}
494503

495504
@Override
496505
public CompletableFuture<Map<String, Entry<?>>> getFiles(String projectName, String repositoryName,
497-
Revision revision, PathPattern pathPattern) {
506+
Revision revision, PathPattern pathPattern,
507+
boolean viewRaw) {
498508
validateProjectAndRepositoryName(projectName, repositoryName);
499509
requireNonNull(revision, "revision");
500510
requireNonNull(pathPattern, "pathPattern");
@@ -506,27 +516,31 @@ public CompletableFuture<Map<String, Entry<?>>> getFiles(String projectName, Str
506516
.append(pathPattern.encoded())
507517
.append("?revision=")
508518
.append(normRev.major());
519+
if (viewRaw) {
520+
path.append("&viewRaw=true");
521+
}
509522

510523
return client.execute(headers(HttpMethod.GET, path.toString()))
511524
.aggregate()
512-
.thenApply(res -> getFiles(normRev, res));
525+
.thenApply(res -> getFiles(normRev, res, viewRaw));
513526
});
514527
} catch (Exception e) {
515528
return exceptionallyCompletedFuture(e);
516529
}
517530
}
518531

519-
private static Map<String, Entry<?>> getFiles(Revision normRev, AggregatedHttpResponse res) {
532+
private static Map<String, Entry<?>> getFiles(Revision normRev, AggregatedHttpResponse res,
533+
boolean viewRaw) {
520534
switch (res.status().code()) {
521535
case 200:
522536
final JsonNode node = toJson(res, null);
523537
final ImmutableMap.Builder<String, Entry<?>> builder = ImmutableMap.builder();
524538
if (node.isObject()) { // Single entry
525-
final Entry<?> entry = toEntry(normRev, node, QueryType.IDENTITY);
539+
final Entry<?> entry = toEntry(normRev, node, QueryType.IDENTITY, viewRaw);
526540
builder.put(entry.path(), entry);
527541
} else if (node.isArray()) { // Multiple entries
528542
node.forEach(e -> {
529-
final Entry<?> entry = toEntry(normRev, e, QueryType.IDENTITY);
543+
final Entry<?> entry = toEntry(normRev, e, QueryType.IDENTITY, viewRaw);
530544
builder.put(entry.path(), entry);
531545
});
532546
} else {
@@ -852,11 +866,17 @@ private static Revision watchRepository(AggregatedHttpResponse res, QueryType un
852866
@Override
853867
public <T> CompletableFuture<Entry<T>> watchFile(String projectName, String repositoryName,
854868
Revision lastKnownRevision, Query<T> query,
855-
long timeoutMillis, boolean errorOnEntryNotFound) {
869+
long timeoutMillis, boolean errorOnEntryNotFound,
870+
boolean viewRaw) {
856871
validateProjectAndRepositoryName(projectName, repositoryName);
857872
requireNonNull(lastKnownRevision, "lastKnownRevision");
858873
requireNonNull(query, "query");
859874
checkArgument(timeoutMillis > 0, "timeoutMillis: %s (expected: > 0)", timeoutMillis);
875+
if (viewRaw && query.type() == QueryType.JSON_PATH) {
876+
// JSON_PATH query cannot return raw content because the raw content is normalized
877+
// when applying JSON_PATH.
878+
throw new IllegalArgumentException("JSON_PATH query cannot be used with raw view");
879+
}
860880
try {
861881

862882
final StringBuilder path = pathBuilder(projectName, repositoryName);
@@ -869,21 +889,25 @@ public <T> CompletableFuture<Entry<T>> watchFile(String projectName, String repo
869889
// Remove the trailing '?' or '&'.
870890
path.setLength(path.length() - 1);
871891
}
892+
if (viewRaw) {
893+
// The query type can't be JSON_PATH here as checked above.
894+
path.append("?viewRaw=true");
895+
}
872896

873897
return watch(lastKnownRevision, timeoutMillis, path.toString(), query.type(),
874-
ArmeriaCentralDogma::watchFile, errorOnEntryNotFound);
898+
(res, queryType) -> watchFile(res, queryType, viewRaw), errorOnEntryNotFound);
875899
} catch (Exception e) {
876900
return exceptionallyCompletedFuture(e);
877901
}
878902
}
879903

880904
@Nullable
881-
private static <T> Entry<T> watchFile(AggregatedHttpResponse res, QueryType queryType) {
905+
private static <T> Entry<T> watchFile(AggregatedHttpResponse res, QueryType queryType, boolean viewRaw) {
882906
switch (res.status().code()) {
883907
case 200: // OK
884908
final JsonNode node = toJson(res, JsonNodeType.OBJECT);
885909
final Revision revision = new Revision(getField(node, "revision").asInt());
886-
return toEntry(revision, getField(node, "entry"), queryType);
910+
return toEntry(revision, getField(node, "entry"), queryType, viewRaw);
887911
case 304: // Not Modified
888912
return null;
889913
}
@@ -1000,20 +1024,7 @@ private static byte[] toBytes(JsonNode content) {
10001024
* Encodes a list of {@link Change}s into a JSON array.
10011025
*/
10021026
private static ArrayNode toJson(Iterable<? extends Change<?>> changes) {
1003-
final ArrayNode changesNode = JsonNodeFactory.instance.arrayNode();
1004-
changes.forEach(c -> {
1005-
final ObjectNode changeNode = JsonNodeFactory.instance.objectNode();
1006-
changeNode.put("path", c.path());
1007-
changeNode.put("type", c.type().name());
1008-
final Class<?> contentType = c.type().contentType();
1009-
if (contentType == JsonNode.class) {
1010-
changeNode.set("content", (JsonNode) c.content());
1011-
} else if (contentType == String.class) {
1012-
changeNode.put("content", (String) c.content());
1013-
}
1014-
changesNode.add(changeNode);
1015-
});
1016-
return changesNode;
1027+
return Jackson.valueToTree(changes);
10171028
}
10181029

10191030
/**
@@ -1048,33 +1059,44 @@ private static String toString(AggregatedHttpResponse res) {
10481059
return res.content(charset);
10491060
}
10501061

1051-
private static <T> Entry<T> toEntry(Revision revision, JsonNode node, QueryType queryType) {
1062+
private static <T> Entry<T> toEntry(Revision revision, JsonNode node, QueryType queryType,
1063+
boolean viewRaw) {
10521064
final String entryPath = getField(node, "path").asText();
10531065
final EntryType receivedEntryType = EntryType.valueOf(getField(node, "type").asText());
10541066
switch (queryType) {
10551067
case IDENTITY_TEXT:
1056-
return entryAsText(revision, node, entryPath);
1068+
return entryAsText(revision, node, entryPath, viewRaw);
10571069
case IDENTITY_JSON:
10581070
case JSON_PATH:
10591071
if (receivedEntryType != EntryType.JSON) {
10601072
throw new CentralDogmaException("invalid entry type. entry type: " + receivedEntryType +
10611073
" (expected: " + queryType + ')');
10621074
}
1063-
return entryAsJson(revision, node, entryPath);
1075+
return entryAsJson(revision, node, entryPath, viewRaw);
10641076
case IDENTITY:
10651077
switch (receivedEntryType) {
10661078
case JSON:
1067-
return entryAsJson(revision, node, entryPath);
1079+
return entryAsJson(revision, node, entryPath, viewRaw);
10681080
case TEXT:
1069-
return entryAsText(revision, node, entryPath);
1081+
return entryAsText(revision, node, entryPath, viewRaw);
10701082
case DIRECTORY:
10711083
return unsafeCast(Entry.ofDirectory(revision, entryPath));
10721084
}
10731085
}
10741086
throw new Error(); // Should never reach here.
10751087
}
10761088

1077-
private static <T> Entry<T> entryAsText(Revision revision, JsonNode node, String entryPath) {
1089+
private static <T> Entry<T> entryAsText(Revision revision, JsonNode node, String entryPath,
1090+
boolean viewRaw) {
1091+
if (viewRaw) {
1092+
final JsonNode rawContent = node.get("rawContent");
1093+
if (rawContent != null) {
1094+
return unsafeCast(Entry.ofText(revision, entryPath, rawContent.asText()));
1095+
}
1096+
logger.warn("The server does not support raw content. Using Entry#content() instead. path: {}",
1097+
entryPath);
1098+
// The server version may be old so fall back to normal content.
1099+
}
10781100
final JsonNode content = getField(node, "content");
10791101
final String content0;
10801102
if (content.isContainerNode()) {
@@ -1085,7 +1107,22 @@ private static <T> Entry<T> entryAsText(Revision revision, JsonNode node, String
10851107
return unsafeCast(Entry.ofText(revision, entryPath, content0));
10861108
}
10871109

1088-
private static <T> Entry<T> entryAsJson(Revision revision, JsonNode node, String entryPath) {
1110+
private static <T> Entry<T> entryAsJson(Revision revision, JsonNode node, String entryPath,
1111+
boolean viewRaw) {
1112+
if (viewRaw) {
1113+
final JsonNode rawContent = node.get("rawContent");
1114+
if (rawContent != null) {
1115+
try {
1116+
return unsafeCast(Entry.ofJson(revision, entryPath, rawContent.asText()));
1117+
} catch (JsonParseException e) {
1118+
// Should never reach here as the raw JSON text was already validated by the server.
1119+
throw new IllegalStateException(e);
1120+
}
1121+
}
1122+
logger.warn("The server does not support raw content. Using Entry#content() instead. path: {}",
1123+
entryPath);
1124+
// The server version may be old so fall back to normal content.
1125+
}
10891126
return unsafeCast(Entry.ofJson(revision, entryPath, getField(node, "content")));
10901127
}
10911128

@@ -1104,24 +1141,12 @@ private static Commit toCommit(JsonNode node) {
11041141
}
11051142

11061143
private static <T> Change<T> toChange(JsonNode node) {
1107-
final String actualPath = getField(node, "path").asText();
1108-
final ChangeType type = ChangeType.valueOf(getField(node, "type").asText());
1109-
switch (type) {
1110-
case UPSERT_JSON:
1111-
return unsafeCast(Change.ofJsonUpsert(actualPath, getField(node, "content")));
1112-
case UPSERT_TEXT:
1113-
return unsafeCast(Change.ofTextUpsert(actualPath, getField(node, "content").asText()));
1114-
case REMOVE:
1115-
return unsafeCast(Change.ofRemoval(actualPath));
1116-
case RENAME:
1117-
return unsafeCast(Change.ofRename(actualPath, getField(node, "content").asText()));
1118-
case APPLY_JSON_PATCH:
1119-
return unsafeCast(Change.ofJsonPatch(actualPath, getField(node, "content")));
1120-
case APPLY_TEXT_PATCH:
1121-
return unsafeCast(Change.ofTextPatch(actualPath, getField(node, "content").asText()));
1144+
try {
1145+
//noinspection unchecked
1146+
return Jackson.treeToValue(node, Change.class);
1147+
} catch (JsonParseException | JsonMappingException e) {
1148+
throw new IllegalStateException("fail to parse a JSON node into Change.", e);
11221149
}
1123-
1124-
throw new Error(); // Never reaches here.
11251150
}
11261151

11271152
private static Set<String> handleNameList(AggregatedHttpResponse res) {

‎client/java/src/main/java/com/linecorp/centraldogma/client/AbstractFileRequest.java‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Copyright 2025 LY Corporation
3+
*
4+
* LY Corporation licenses this file to you under the Apache License,
5+
* version 2.0 (the "License"); you may not use this file except in compliance
6+
* with the License. You may obtain a copy of the License at:
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations
14+
* under the License.
15+
*/
16+
17+
package com.linecorp.centraldogma.client;
18+
19+
import com.linecorp.centraldogma.common.QueryType;
20+
21+
class AbstractFileRequest<SELF extends AbstractFileRequest<SELF>> {
22+
23+
private boolean viewRaw;
24+
25+
/**
26+
* Sets whether to view the raw content of the file.
27+
* Default is {@code false} which means the content may be normalized.
28+
*
29+
* <p>Note that {@link QueryType#JSON_PATH} query cannot be used with raw view.
30+
*/
31+
public SELF viewRaw(boolean viewRaw) {
32+
this.viewRaw = viewRaw;
33+
return self();
34+
}
35+
36+
/**
37+
* Returns whether to view the raw content of the file.
38+
*/
39+
boolean viewRaw() {
40+
return viewRaw;
41+
}
42+
43+
@SuppressWarnings("unchecked")
44+
private SELF self() {
45+
return (SELF) this;
46+
}
47+
}

0 commit comments

Comments
 (0)