diff --git a/client/java-armeria-legacy/src/main/java/com/linecorp/centraldogma/client/armeria/legacy/LegacyCentralDogma.java b/client/java-armeria-legacy/src/main/java/com/linecorp/centraldogma/client/armeria/legacy/LegacyCentralDogma.java index 37b824142..7affe242d 100644 --- a/client/java-armeria-legacy/src/main/java/com/linecorp/centraldogma/client/armeria/legacy/LegacyCentralDogma.java +++ b/client/java-armeria-legacy/src/main/java/com/linecorp/centraldogma/client/armeria/legacy/LegacyCentralDogma.java @@ -220,6 +220,13 @@ public CompletableFuture> listFiles(String projectName, S e -> EntryConverter.convertEntryType(e.getType())))); } + @Override + public CompletableFuture>> listFiles(String projectName, String repositoryName, + Revision revision, PathPattern pathPattern, + int includeLastFileRevision) { + throw new UnsupportedOperationException("Use ArmeriaCentralDogma instead."); + } + @Override public CompletableFuture> getFile(String projectName, String repositoryName, Revision revision, Query query) { @@ -293,6 +300,13 @@ public CompletableFuture>> getFiles(String projectName, Str }); } + @Override + public CompletableFuture>> getFiles(String projectName, String repositoryName, + Revision revision, PathPattern pathPattern, + int includeLastFileRevision) { + throw new UnsupportedOperationException("Use ArmeriaCentralDogma instead."); + } + @Override public CompletableFuture> mergeFiles(String projectName, String repositoryName, Revision revision, MergeQuery mergeQuery) { diff --git a/client/java-armeria/src/main/java/com/linecorp/centraldogma/internal/client/armeria/ArmeriaCentralDogma.java b/client/java-armeria/src/main/java/com/linecorp/centraldogma/internal/client/armeria/ArmeriaCentralDogma.java index 1409cf2f7..e823b3d10 100644 --- a/client/java-armeria/src/main/java/com/linecorp/centraldogma/internal/client/armeria/ArmeriaCentralDogma.java +++ b/client/java-armeria/src/main/java/com/linecorp/centraldogma/internal/client/armeria/ArmeriaCentralDogma.java @@ -412,29 +412,68 @@ private static Revision normalizeRevision(AggregatedHttpResponse res) { @Override public CompletableFuture> listFiles(String projectName, String repositoryName, Revision revision, PathPattern pathPattern) { + return listFiles0(projectName, repositoryName, revision, pathPattern, 1, + ArmeriaCentralDogma::listFiles); + } + + @Override + public CompletableFuture>> listFiles(String projectName, String repositoryName, + Revision revision, PathPattern pathPattern, + int includeLastFileRevision) { + return listFiles0(projectName, repositoryName, revision, pathPattern, includeLastFileRevision, + ArmeriaCentralDogma::listFilesWithRevision); + } + + private static Map listFiles(AggregatedHttpResponse res) { + switch (res.status().code()) { + case 200: + final ImmutableMap.Builder builder = ImmutableMap.builder(); + final JsonNode node = toJson(res, JsonNodeType.ARRAY); + node.forEach(e -> builder.put( + getField(e, "path").asText(), + EntryType.valueOf(getField(e, "type").asText()))); + return builder.build(); + case 204: + return ImmutableMap.of(); + } + + return handleErrorResponse(res); + } + + private CompletableFuture listFiles0(String projectName, String repositoryName, + Revision revision, PathPattern pathPattern, + int includeLastFileRevision, + Function responseConverter) { validateProjectAndRepositoryName(projectName, repositoryName); requireNonNull(revision, "revision"); requireNonNull(pathPattern, "pathPattern"); try { final StringBuilder path = pathBuilder(projectName, repositoryName); path.append("/list").append(pathPattern.encoded()).append("?revision=").append(revision.major()); + if (includeLastFileRevision > 1) { + path.append("&includeLastFileRevision=").append(includeLastFileRevision); + } return client.execute(headers(HttpMethod.GET, path.toString())) .aggregate() - .thenApply(ArmeriaCentralDogma::listFiles); + .thenApply(responseConverter); } catch (Exception e) { return exceptionallyCompletedFuture(e); } } - private static Map listFiles(AggregatedHttpResponse res) { + private static Map> listFilesWithRevision(AggregatedHttpResponse res) { switch (res.status().code()) { case 200: - final ImmutableMap.Builder builder = ImmutableMap.builder(); + final ImmutableMap.Builder> builder = ImmutableMap.builder(); final JsonNode node = toJson(res, JsonNodeType.ARRAY); - node.forEach(e -> builder.put( - getField(e, "path").asText(), - EntryType.valueOf(getField(e, "type").asText()))); + for (JsonNode e : node) { + final Revision revision = new Revision(getField(e, "revision").asInt()); + final String path = getField(e, "path").asText(); + final EntryType type = EntryType.valueOf(getField(e, "type").asText()); + final Entry entry = Entry.ofNullable(revision, path, type, null); + builder.put(path, entry); + } return builder.build(); case 204: return ImmutableMap.of(); @@ -459,17 +498,17 @@ public CompletableFuture> getFile(String projectName, String reposi return client.execute(headers(HttpMethod.GET, path.toString())) .aggregate() - .thenApply(res -> getFile(normRev, res, query)); + .thenApply(res -> getFile(res, query)); }); } catch (Exception e) { return exceptionallyCompletedFuture(e); } } - private static Entry getFile(Revision normRev, AggregatedHttpResponse res, Query query) { + private static Entry getFile(AggregatedHttpResponse res, Query query) { if (res.status().code() == 200) { final JsonNode node = toJson(res, JsonNodeType.OBJECT); - return toEntry(normRev, node, query.type()); + return toEntry(node, query.type()); } return handleErrorResponse(res); @@ -477,7 +516,8 @@ private static Entry getFile(Revision normRev, AggregatedHttpResponse res @Override public CompletableFuture>> getFiles(String projectName, String repositoryName, - Revision revision, PathPattern pathPattern) { + Revision revision, PathPattern pathPattern, + int includeLastFileRevision) { validateProjectAndRepositoryName(projectName, repositoryName); requireNonNull(revision, "revision"); requireNonNull(pathPattern, "pathPattern"); @@ -489,27 +529,30 @@ public CompletableFuture>> getFiles(String projectName, Str .append(pathPattern.encoded()) .append("?revision=") .append(normRev.major()); + if (includeLastFileRevision > 1) { + path.append("&includeLastFileRevision=").append(includeLastFileRevision); + } return client.execute(headers(HttpMethod.GET, path.toString())) .aggregate() - .thenApply(res -> getFiles(normRev, res)); + .thenApply(res -> getFiles(res)); }); } catch (Exception e) { return exceptionallyCompletedFuture(e); } } - private static Map> getFiles(Revision normRev, AggregatedHttpResponse res) { + private static Map> getFiles(AggregatedHttpResponse res) { switch (res.status().code()) { case 200: final JsonNode node = toJson(res, null); final ImmutableMap.Builder> builder = ImmutableMap.builder(); if (node.isObject()) { // Single entry - final Entry entry = toEntry(normRev, node, QueryType.IDENTITY); + final Entry entry = toEntry(node, QueryType.IDENTITY); builder.put(entry.path(), entry); } else if (node.isArray()) { // Multiple entries node.forEach(e -> { - final Entry entry = toEntry(normRev, e, QueryType.IDENTITY); + final Entry entry = toEntry(e, QueryType.IDENTITY); builder.put(entry.path(), entry); }); } else { @@ -865,8 +908,7 @@ private static Entry watchFile(AggregatedHttpResponse res, QueryType quer switch (res.status().code()) { case 200: // OK final JsonNode node = toJson(res, JsonNodeType.OBJECT); - final Revision revision = new Revision(getField(node, "revision").asInt()); - return toEntry(revision, getField(node, "entry"), queryType); + return toEntry(getField(node, "entry"), queryType); case 304: // Not Modified return null; } @@ -1031,9 +1073,11 @@ private static String toString(AggregatedHttpResponse res) { return res.content(charset); } - private static Entry toEntry(Revision revision, JsonNode node, QueryType queryType) { + private static Entry toEntry(JsonNode node, QueryType queryType) { final String entryPath = getField(node, "path").asText(); final EntryType receivedEntryType = EntryType.valueOf(getField(node, "type").asText()); + final Revision revision = new Revision(getField(node, "revision").asInt()); + switch (queryType) { case IDENTITY_TEXT: return entryAsText(revision, node, entryPath); diff --git a/client/java/src/main/java/com/linecorp/centraldogma/client/CentralDogma.java b/client/java/src/main/java/com/linecorp/centraldogma/client/CentralDogma.java index ab5c72940..7463cff28 100644 --- a/client/java/src/main/java/com/linecorp/centraldogma/client/CentralDogma.java +++ b/client/java/src/main/java/com/linecorp/centraldogma/client/CentralDogma.java @@ -159,7 +159,26 @@ default CompletableFuture> listFiles(String projectName, * @return a {@link Map} of file path and type pairs */ CompletableFuture> listFiles(String projectName, String repositoryName, - Revision revision, PathPattern pathPattern); + Revision revision, PathPattern pathPattern); + + /** + * Retrieves the list of the files matched by the given {@link PathPattern}. + * This method is equivalent to calling: + * 
{@code
+     * CentralDogma dogma = ...
+     * // Find the last file revision from the last 100 revisions.
+     * int includeLastFileRevision = 100;
+     * dogma.forRepo(projectName, repositoryName)
+     *      .files(pathPattern)
+     *      .includeLastFileRevision(includeLastFileRevision)
+     *      .list(revision);
+     * }
+ * + * @return a {@link Map} of file path and {@link Entry} pairs + */ + CompletableFuture>> listFiles(String projectName, String repositoryName, + Revision revision, PathPattern pathPattern, + int includeLastFileRevision); /** * Retrieves the file at the specified revision and path. This method is a shortcut of @@ -220,8 +239,30 @@ default CompletableFuture>> getFiles(String projectName, St * * @return a {@link Map} of file path and {@link Entry} pairs */ + default CompletableFuture>> getFiles(String projectName, String repositoryName, + Revision revision, PathPattern pathPattern) { + return getFiles(projectName, repositoryName, revision, pathPattern, 1); + } + + /** + * Retrieves the files matched by the {@link PathPattern}. + * This method is equivalent to calling: + * 
{@code
+     * CentralDogma dogma = ...
+     * // Find the last file revision from the last 100 revisions.
+     * // If the last file revision is not found, the `Revision.INIT` revision is returned instead.
+     * int includeLastFileRevision = 100;
+     * dogma.forRepo(projectName, repositoryName)
+     *      .file(pathPattern)
+     *      .includeLastFileRevision(includeLastFileRevision)
+     *      .get(revision);
+     * }
+ * + * @return a {@link Map} of file path and {@link Entry} pairs + */ CompletableFuture>> getFiles(String projectName, String repositoryName, - Revision revision, PathPattern pathPattern); + Revision revision, PathPattern pathPattern, + int includeLastFileRevision); /** * Retrieves the merged entry of the specified {@link MergeSource}s at the specified revision. diff --git a/client/java/src/main/java/com/linecorp/centraldogma/client/FilesRequest.java b/client/java/src/main/java/com/linecorp/centraldogma/client/FilesRequest.java index 9269757ef..0edaacca7 100644 --- a/client/java/src/main/java/com/linecorp/centraldogma/client/FilesRequest.java +++ b/client/java/src/main/java/com/linecorp/centraldogma/client/FilesRequest.java @@ -15,6 +15,7 @@ */ package com.linecorp.centraldogma.client; +import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import java.util.Map; @@ -40,6 +41,25 @@ public final class FilesRequest { this.pathPattern = pathPattern; } + /** + * Includes the last revision of the file in the result. This option is disabled by default. + * If the last file revision is not found from the specified range of revisions, {@link Revision#INIT} is + * returned. + * + *

Note that this operation may be slow because it needs to search for the last file revisions from the + * revision history. So please use it carefully. + * + * @param includeLastFileRevision the maximum number of revisions to search for the last file revision. + * If the value is equal to 1, the head revision is + * included instead. + */ + public FilesWithRevisionRequest includeLastFileRevision(int includeLastFileRevision) { + checkArgument(includeLastFileRevision >= 1 && includeLastFileRevision <= 1000, + "includeLastFileRevision: %s (expected: 1 .. 1000)", + includeLastFileRevision); + return new FilesWithRevisionRequest(centralDogmaRepo, pathPattern, includeLastFileRevision); + } + /** * Retrieves the list of the files matched by the given path pattern at the {@link Revision#HEAD}. * diff --git a/client/java/src/main/java/com/linecorp/centraldogma/client/FilesWithRevisionRequest.java b/client/java/src/main/java/com/linecorp/centraldogma/client/FilesWithRevisionRequest.java new file mode 100644 index 000000000..cc3af599b --- /dev/null +++ b/client/java/src/main/java/com/linecorp/centraldogma/client/FilesWithRevisionRequest.java @@ -0,0 +1,89 @@ +/* + * Copyright 2025 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.client; + +import static java.util.Objects.requireNonNull; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import com.linecorp.centraldogma.common.Entry; +import com.linecorp.centraldogma.common.PathPattern; +import com.linecorp.centraldogma.common.Revision; + +/** + * Prepares to send a {@link CentralDogma#getFiles(String, String, Revision, PathPattern, int)} or + * {@link CentralDogma#listFiles(String, String, Revision, PathPattern, int)} request to the + * Central Dogma repository. + */ +public final class FilesWithRevisionRequest { + + private final CentralDogmaRepository centralDogmaRepo; + private final PathPattern pathPattern; + private final int includeLastFileRevision; + + FilesWithRevisionRequest(CentralDogmaRepository centralDogmaRepo, PathPattern pathPattern, + int includeLastFileRevision) { + this.centralDogmaRepo = centralDogmaRepo; + this.pathPattern = pathPattern; + this.includeLastFileRevision = includeLastFileRevision; + } + + /** + * Retrieves the list of the files matched by the given path pattern at the {@link Revision#HEAD}. + * Note that the returned {@link Entry} does not contain the content of the file. + * + * @return a {@link Map} of file path and {@link Entry} pairs. + */ + public CompletableFuture>> list() { + return list(Revision.HEAD); + } + + /** + * Retrieves the list of the files matched by the given path pattern at the {@link Revision}. + * Note that the returned {@link Entry} does not contain the content of the file. + * + * @return a {@link Map} of file path and {@link Entry} pairs + */ + public CompletableFuture>> list(Revision revision) { + requireNonNull(revision, "revision"); + return centralDogmaRepo.centralDogma().listFiles(centralDogmaRepo.projectName(), + centralDogmaRepo.repositoryName(), + revision, pathPattern, includeLastFileRevision); + } + + /** + * Retrieves the files matched by the path pattern at the {@link Revision#HEAD}. + * + * @return a {@link Map} of file path and {@link Entry} pairs + */ + public CompletableFuture>> get() { + return get(Revision.HEAD); + } + + /** + * Retrieves the files matched by the path pattern at the {@link Revision}. + * + * @return a {@link Map} of file path and {@link Entry} pairs + */ + public CompletableFuture>> get(Revision revision) { + requireNonNull(revision, "revision"); + return centralDogmaRepo.centralDogma().getFiles(centralDogmaRepo.projectName(), + centralDogmaRepo.repositoryName(), + revision, pathPattern, includeLastFileRevision); + } +} diff --git a/client/java/src/main/java/com/linecorp/centraldogma/internal/client/ReplicationLagTolerantCentralDogma.java b/client/java/src/main/java/com/linecorp/centraldogma/internal/client/ReplicationLagTolerantCentralDogma.java index 57c8d752a..a2b8021b2 100644 --- a/client/java/src/main/java/com/linecorp/centraldogma/internal/client/ReplicationLagTolerantCentralDogma.java +++ b/client/java/src/main/java/com/linecorp/centraldogma/internal/client/ReplicationLagTolerantCentralDogma.java @@ -238,6 +238,27 @@ public String toString() { }); } + @Override + public CompletableFuture>> listFiles( + String projectName, String repositoryName, Revision revision, PathPattern pathPattern, + int includeLastFileRevision) { + return normalizeRevisionAndExecuteWithRetries( + projectName, repositoryName, revision, + new Function>>>() { + @Override + public CompletableFuture>> apply(Revision normRev) { + return delegate.listFiles(projectName, repositoryName, normRev, pathPattern, + includeLastFileRevision); + } + + @Override + public String toString() { + return "listFiles(" + projectName + ", " + repositoryName + ", " + + revision + ", " + pathPattern + ", " + includeLastFileRevision + ')'; + } + }); + } + @Override public CompletableFuture> getFile( String projectName, String repositoryName, Revision revision, Query query) { @@ -276,6 +297,27 @@ public String toString() { }); } + @Override + public CompletableFuture>> getFiles( + String projectName, String repositoryName, Revision revision, PathPattern pathPattern, + int includeLastFileRevision) { + return normalizeRevisionAndExecuteWithRetries( + projectName, repositoryName, revision, + new Function>>>() { + @Override + public CompletableFuture>> apply(Revision normRev) { + return delegate.getFiles(projectName, repositoryName, normRev, pathPattern, + includeLastFileRevision); + } + + @Override + public String toString() { + return "getFiles(" + projectName + ", " + repositoryName + ", " + + revision + ", " + pathPattern + ", " + includeLastFileRevision + ')'; + } + }); + } + @Override public CompletableFuture> mergeFiles( String projectName, String repositoryName, Revision revision, diff --git a/common/src/main/java/com/linecorp/centraldogma/common/Entry.java b/common/src/main/java/com/linecorp/centraldogma/common/Entry.java index 42bb08e9e..00c1e0c12 100644 --- a/common/src/main/java/com/linecorp/centraldogma/common/Entry.java +++ b/common/src/main/java/com/linecorp/centraldogma/common/Entry.java @@ -44,7 +44,7 @@ public final class Entry implements ContentHolder { * @param path the path of the directory */ public static Entry ofDirectory(Revision revision, String path) { - return new Entry<>(revision, path, EntryType.DIRECTORY, null); + return new Entry<>(revision, path, EntryType.DIRECTORY, null, true); } /** @@ -55,7 +55,7 @@ public static Entry ofDirectory(Revision revision, String path) { * @param content the content of the JSON file */ public static Entry ofJson(Revision revision, String path, JsonNode content) { - return new Entry<>(revision, path, EntryType.JSON, content); + return new Entry<>(revision, path, EntryType.JSON, content, false); } /** @@ -80,7 +80,7 @@ public static Entry ofJson(Revision revision, String path, String cont * @param content the content of the text file */ public static Entry ofText(Revision revision, String path, String content) { - return new Entry<>(revision, path, EntryType.TEXT, content); + return new Entry<>(revision, path, EntryType.TEXT, content, false); } /** @@ -93,7 +93,20 @@ public static Entry ofText(Revision revision, String path, String conten * @param the content type. {@link JsonNode} if JSON. {@link String} if text. */ public static Entry of(Revision revision, String path, EntryType type, @Nullable T content) { - return new Entry<>(revision, path, type, content); + return new Entry<>(revision, path, type, content, false); + } + + /** + * Returns a newly-created {@link Entry} whose content is nullable. + * + * @param revision the revision of the {@link Entry} + * @param path the path of the {@link Entry} + * @param content the content of the {@link Entry} + * @param type the type of the {@link Entry} + * @param the content type. {@link JsonNode} if JSON. {@link String} if text. + */ + public static Entry ofNullable(Revision revision, String path, EntryType type, @Nullable T content) { + return new Entry<>(revision, path, type, content, true); } private final Revision revision; @@ -114,7 +127,8 @@ public static Entry of(Revision revision, String path, EntryType type, @N * @param type the type of given {@code content} * @param content an object of given type {@code T} */ - private Entry(Revision revision, String path, EntryType type, @Nullable T content) { + private Entry(Revision revision, String path, EntryType type, @Nullable T content, + boolean allowNullContent) { requireNonNull(revision, "revision"); checkArgument(!revision.isRelative(), "revision: %s (expected: absolute revision)", revision); this.revision = revision; @@ -127,9 +141,13 @@ private Entry(Revision revision, String path, EntryType type, @Nullable T conten checkArgument(content == null, "content: %s (expected: null)", content); this.content = null; } else { - @SuppressWarnings("unchecked") - final T castContent = (T) entryContentType.cast(requireNonNull(content, "content")); - this.content = castContent; + if (content == null && allowNullContent) { + this.content = null; + } else { + @SuppressWarnings("unchecked") + final T castContent = (T) entryContentType.cast(requireNonNull(content, "content")); + this.content = castContent; + } } } @@ -140,6 +158,14 @@ public Revision revision() { return revision; } + /** + * Returns a new {@link Entry} with the specified {@link Revision}. + */ + public Entry withRevision(Revision revision) { + requireNonNull(revision, "revision"); + return new Entry<>(revision, path, type, content, true); + } + /** * Returns the path of this {@link Entry}. */ @@ -148,7 +174,7 @@ public String path() { } /** - * Returns if this {@link Entry} has content, which is always {@code true} if it's not a directory. + * Returns if this {@link Entry} has content. */ public boolean hasContent() { return content != null; diff --git a/common/src/main/java/com/linecorp/centraldogma/internal/api/v1/WatchResultDto.java b/common/src/main/java/com/linecorp/centraldogma/internal/api/v1/WatchResultDto.java index 575861015..9f6b88bf7 100644 --- a/common/src/main/java/com/linecorp/centraldogma/internal/api/v1/WatchResultDto.java +++ b/common/src/main/java/com/linecorp/centraldogma/internal/api/v1/WatchResultDto.java @@ -30,6 +30,7 @@ @JsonInclude(Include.NON_EMPTY) public class WatchResultDto { + // TODO(ikhoon): Remove the revision field because it is already included in the entry field. private final Revision revision; private final EntryDto entry; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1.java index ad1dcd9e4..cc18138be 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1.java @@ -43,6 +43,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Streams; @@ -123,19 +124,27 @@ public ContentServiceV1(CommandExecutor executor, WatchService watchService, Met public CompletableFuture>> listFiles(ServiceRequestContext ctx, @Param String path, @Param @Default("-1") String revision, + @Param @Default("1") int includeLastFileRevision, Repository repository) { final String normalizedPath = normalizePath(path); final Revision normalizedRev = repository.normalizeNow(new Revision(revision)); increaseCounterIfOldRevisionUsed(ctx, repository, normalizedRev); final CompletableFuture>> future = new CompletableFuture<>(); - listFiles(repository, normalizedPath, normalizedRev, false, future); + listFiles(repository, normalizedPath, normalizedRev, false, includeLastFileRevision, future); return future; } private static void listFiles(Repository repository, String pathPattern, Revision normalizedRev, - boolean withContent, CompletableFuture>> result) { - final Map, ?> options = withContent ? FindOptions.FIND_ALL_WITH_CONTENT - : FindOptions.FIND_ALL_WITHOUT_CONTENT; + boolean withContent, int includeLastFileRevision, + CompletableFuture>> result) { + final Map, ?> options; + if (includeLastFileRevision <= 1) { + options = withContent ? FindOptions.FIND_ALL_WITH_CONTENT + : FindOptions.FIND_ALL_WITHOUT_CONTENT; + } else { + options = ImmutableMap.of(FindOption.FETCH_LAST_FILE_REVISION, includeLastFileRevision, + FindOption.FETCH_CONTENT, withContent); + } repository.find(normalizedRev, pathPattern, options).handle((entries, thrown) -> { if (thrown != null) { @@ -147,10 +156,11 @@ private static void listFiles(Repository repository, String pathPattern, Revisio // This is called once at most, because the pathPattern is not a valid file path anymore. if (isValidFilePath(pathPattern) && entries.size() == 1 && entries.values().iterator().next().type() == DIRECTORY) { - listFiles(repository, pathPattern + "/*", normalizedRev, withContent, result); + listFiles(repository, pathPattern + "/*", normalizedRev, withContent, includeLastFileRevision, + result); } else { result.complete(entries.values().stream() - .map(entry -> convert(repository, normalizedRev, entry, withContent)) + .map(entry -> convert(repository, entry, withContent)) .collect(toImmutableList())); } return null; @@ -249,7 +259,7 @@ public CompletableFuture>> preview( * jsonpath={jsonpath} * *

Returns the entry of files in the path. This is same with - * {@link #listFiles(ServiceRequestContext, String, String, Repository)} except that containing + * {@link #listFiles(ServiceRequestContext, String, String, int, Repository)} except that containing * the content of the files. * Note that if the {@link HttpHeaderNames#IF_NONE_MATCH} in which has a revision is sent with, * this will await for the time specified in {@link HttpHeaderNames#PREFER}. @@ -261,6 +271,7 @@ public CompletableFuture>> preview( public CompletableFuture getFiles( ServiceRequestContext ctx, @Param String path, @Param @Default("-1") String revision, + @Param @Default("1") int includeLastFileRevision, Repository repository, @RequestConverter(WatchRequestConverter.class) @Nullable WatchRequest watchRequest, @RequestConverter(QueryRequestConverter.class) @Nullable Query query) { @@ -284,14 +295,13 @@ public CompletableFuture getFiles( final Revision normalizedRev = repository.normalizeNow(new Revision(revision)); if (query != null) { // get a file - return repository.get(normalizedRev, query) - .handle(returnOrThrow((Entry result) -> convert(repository, normalizedRev, - result, true))); + return repository.get(normalizedRev, query, includeLastFileRevision) + .handle(returnOrThrow((Entry result) -> convert(repository, result, true))); } // get files final CompletableFuture>> future = new CompletableFuture<>(); - listFiles(repository, normalizedPath, normalizedRev, true, future); + listFiles(repository, normalizedPath, normalizedRev, true, includeLastFileRevision, future); return future; } @@ -306,9 +316,8 @@ private CompletableFuture watchFile(ServiceRequestContext ctx, } return future.thenApply(entry -> { - final Revision revision = entry.revision(); - final EntryDto entryDto = convert(repository, revision, entry, true); - return (Object) new WatchResultDto(revision, entryDto); + final EntryDto entryDto = convert(repository, entry, true); + return (Object) new WatchResultDto(entry.revision(), entryDto); }).exceptionally(ContentServiceV1::handleWatchFailure); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/DtoConverter.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/DtoConverter.java index c3805a233..248a4cf2e 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/DtoConverter.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/DtoConverter.java @@ -59,13 +59,12 @@ public static RepositoryDto convert(Repository repository) { repository.creationTimeMillis()); } - public static EntryDto convert(Repository repository, Revision revision, - Entry entry, boolean withContent) { + public static EntryDto convert(Repository repository, Entry entry, boolean withContent) { requireNonNull(entry, "entry"); if (withContent && entry.hasContent()) { - return convert(repository, revision, entry.path(), entry.type(), entry.content()); + return convert(repository, entry.revision(), entry.path(), entry.type(), entry.content()); } - return convert(repository, revision, entry.path(), entry.type()); + return convert(repository, entry.revision(), entry.path(), entry.type()); } private static EntryDto convert(Repository repository, Revision revision, diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableQueryCall.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableQueryCall.java index 5ca784817..bd5f885ce 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableQueryCall.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableQueryCall.java @@ -37,14 +37,16 @@ final class CacheableQueryCall extends AbstractCacheableCall> { final Revision revision; final Query query; + final int includeLastFileRevision; final int hashCode; - CacheableQueryCall(Repository repo, Revision revision, Query query) { + CacheableQueryCall(Repository repo, Revision revision, Query query, int includeLastFileRevision) { super(repo); this.revision = requireNonNull(revision, "revision"); this.query = requireNonNull(query, "query"); + this.includeLastFileRevision = includeLastFileRevision; - hashCode = Objects.hash(revision, query) * 31 + System.identityHashCode(repo); + hashCode = Objects.hash(revision, query, includeLastFileRevision) * 31 + System.identityHashCode(repo); assert !revision.isRelative(); } @@ -59,13 +61,17 @@ public int weigh(Entry value) { if (value != null && value.hasContent()) { weight += value.contentAsText().length(); } + if (includeLastFileRevision > 0) { + weight += includeLastFileRevision; + } return weight; } @Override public CompletableFuture> execute() { logger.debug("Cache miss: {}", this); - return repo().getOrNull(revision, query).thenApply(e -> firstNonNull(e, EMPTY)); + return repo().getOrNull(revision, query, includeLastFileRevision) + .thenApply(e -> firstNonNull(e, EMPTY)); } @Override @@ -81,12 +87,14 @@ public boolean equals(Object o) { final CacheableQueryCall that = (CacheableQueryCall) o; return revision.equals(that.revision) && - query.equals(that.query); + query.equals(that.query) && + includeLastFileRevision == that.includeLastFileRevision; } @Override protected void toString(ToStringHelper helper) { helper.add("revision", revision) - .add("query", query); + .add("query", query) + .add("includeLastFileRevision", includeLastFileRevision); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java index 4b88c3a4c..ea0f99e1d 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java @@ -19,6 +19,8 @@ import static com.google.common.base.MoreObjects.toStringHelper; import static com.linecorp.centraldogma.internal.Util.unsafeCast; import static com.linecorp.centraldogma.server.internal.api.HttpApiUtil.throwUnsafelyIfNonNull; +import static com.linecorp.centraldogma.server.storage.repository.FindOption.FETCH_LAST_FILE_REVISION; +import static com.linecorp.centraldogma.server.storage.repository.FindOption.MAX_ENTRIES; import static com.linecorp.centraldogma.server.storage.repository.FindOptions.FIND_ONE_WITH_CONTENT; import static java.util.Objects.requireNonNull; @@ -28,6 +30,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; +import com.google.common.collect.ImmutableMap; + import com.linecorp.armeria.common.CommonPools; import com.linecorp.armeria.common.RequestContext; import com.linecorp.armeria.common.util.Exceptions; @@ -82,7 +86,8 @@ public Author author() { } @Override - public CompletableFuture> getOrNull(Revision revision, Query query) { + public CompletableFuture> getOrNull(Revision revision, Query query, + int includeLastFileRevision) { requireNonNull(revision, "revision"); requireNonNull(query, "query"); @@ -92,13 +97,20 @@ public CompletableFuture> getOrNull(Revision revision, Query que // If the query is an IDENTITY type, call find() so that the caches are reused in one place when // calls getOrNull(), find() and mergeFiles(). final String path = query.path(); + final Map, ?> options; + if (includeLastFileRevision <= 1) { + options = FIND_ONE_WITH_CONTENT; + } else { + options = ImmutableMap.of(MAX_ENTRIES, 1, FETCH_LAST_FILE_REVISION, includeLastFileRevision); + } final CompletableFuture> future = - find(revision, path, FIND_ONE_WITH_CONTENT).thenApply(findResult -> findResult.get(path)); + find(revision, path, options).thenApply( + findResult -> findResult.get(path)); return unsafeCast(future); } final CompletableFuture future = - cache.get(new CacheableQueryCall(repo, normalizedRevision, query)) + cache.get(new CacheableQueryCall(repo, normalizedRevision, query, includeLastFileRevision)) .handleAsync((result, cause) -> { throwUnsafelyIfNonNull(cause); return result != CacheableQueryCall.EMPTY ? result : null; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java index 5b6fb9401..59b05f3e1 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java @@ -31,10 +31,13 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executor; @@ -45,6 +48,7 @@ import java.util.function.Function; import java.util.function.Supplier; import java.util.regex.Pattern; +import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -70,6 +74,7 @@ import org.eclipse.jgit.treewalk.CanonicalTreeParser; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.AndTreeFilter; +import org.eclipse.jgit.treewalk.filter.PathFilterGroup; import org.eclipse.jgit.treewalk.filter.TreeFilter; import org.eclipse.jgit.util.SystemReader; import org.slf4j.Logger; @@ -460,6 +465,7 @@ private Map> blockingFind( final Revision normRevision = normalizeNow(revision); final boolean fetchContent = FindOption.FETCH_CONTENT.get(options); final int maxEntries = FindOption.MAX_ENTRIES.get(options); + final int maxCommitsForFileRevision = FindOption.FETCH_LAST_FILE_REVISION.get(options); readLock(); try (ObjectReader reader = jGitRepository.newObjectReader(); @@ -535,6 +541,28 @@ private Map> blockingFind( result.put(path, entry); } + if (maxCommitsForFileRevision > 1 && !result.isEmpty()) { + // Fetch the last revisions of the files if the FETCH_LAST_FILE_REVISION option is enabled. + final List files = result.values().stream() + .filter(entry -> entry.type().type() != Void.class) + // Remove the heading `/` + .map(entry -> entry.path().substring(1)) + .collect(Collectors.toList()); + + final Map lastRevisions = + blockingFindLastFileRevisions(normRevision, + normRevision.backward(maxCommitsForFileRevision - 1), + files); + + for (Map.Entry fileAndRevision : lastRevisions.entrySet()) { + final String filePath = '/' + fileAndRevision.getKey(); + final Revision fileRevision = fileAndRevision.getValue(); + if (fileRevision.equals(normRevision)) { + continue; + } + result.computeIfPresent(filePath, (path, entry) -> entry.withRevision(fileRevision)); + } + } return Util.unsafeCast(result); } catch (CentralDogmaException | IllegalArgumentException e) { throw e; @@ -547,6 +575,90 @@ private Map> blockingFind( } } + // TODO(ikhoon): Design a storage format to store the revision information of each file and read them + // without retrieving the history. + private Map blockingFindLastFileRevisions(Revision from, Revision to, + List paths) { + final RevisionRange range = normalizeNow(from, to); + final RevisionRange descendingRange = range.toDescending(); + final Set remainingFiles = new HashSet<>(paths); + final Map lastRevisionMap = new HashMap<>(); + + // At this point, we are sure: from.major >= to.major and read lock is acquired. + try (ObjectReader reader = jGitRepository.newObjectReader(); + TreeWalk treeWalk = new TreeWalk(reader); + RevWalk revWalk = newRevWalk(reader)) { + final ObjectIdOwnerMap revWalkInternalMap = + (ObjectIdOwnerMap) revWalkObjectsField.get(revWalk); + + final ObjectId fromCommitId = commitIdDatabase.get(descendingRange.from()); + final ObjectId toCommitId = commitIdDatabase.get(descendingRange.to()); + + revWalk.markStart(revWalk.parseCommit(fromCommitId)); + revWalk.setRetainBody(false); + // Ensure subdirectories are traversed. + treeWalk.setRecursive(true); + + int numProcessedCommits = 0; + boolean needsToUpdateFilter = true; + for (RevCommit revCommit : revWalk) { + if (revCommit.getParentCount() == 0) { + break; // Reached the root commit + } + + numProcessedCommits++; + + treeWalk.reset(); + treeWalk.addTree(revCommit.getTree()); + treeWalk.addTree(revCommit.getParent(0).getTree()); + treeWalk.setRecursive(true); + if (needsToUpdateFilter) { + // Dynamically update the filter to exclude already found files + treeWalk.setFilter(PathFilterGroup.createFromStrings(remainingFiles)); + needsToUpdateFilter = false; + } + + while (treeWalk.next()) { + final String filePath = treeWalk.getPathString(); + + // `treeWalk.idEqual(0, 1)` compares the Object IDs of the current entry in the + // first tree (nthA = 0) and the second tree (nthB = 1). + // - If the IDs are different, it indicates that the file’s content has changed between + // the two commits. + // - If the IDs are the same, the file’s content has not changed. + if (!treeWalk.idEqual(0, 1)) { + lastRevisionMap.put(filePath, from.backward(numProcessedCommits - 1)); + // Remove to avoid redundant checks + remainingFiles.remove(filePath); + needsToUpdateFilter = true; + } + } + + if (revCommit.getId().equals(toCommitId)) { + break; + } + + // Clear the internal lookup table of RevWalk to reduce the memory usage. + // This is safe because we have linear history and traverse in one direction. + if (numProcessedCommits % 16 == 0) { + revWalkInternalMap.clear(); + } + } + if (!remainingFiles.isEmpty()) { + // If there are remaining files, it means that the files have not been changed + // in the range. So we set the initial revision to them to indicate that they are unchanged. + remainingFiles.forEach(file -> lastRevisionMap.put(file, Revision.INIT)); + } + return lastRevisionMap; + } catch (CentralDogmaException e) { + throw e; + } catch (Exception e) { + throw new StorageException( + "failed to retrieve the last revisions: " + parent.name() + '/' + name + + " (" + paths + ", " + from + ".." + to + ')', e); + } + } + @Override public CompletableFuture> history( Revision from, Revision to, String pathPattern, int maxCommits) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/FindOption.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/FindOption.java index e28379241..6165964d1 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/FindOption.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/FindOption.java @@ -20,8 +20,6 @@ import javax.annotation.Nullable; -import com.linecorp.centraldogma.internal.Util; - /** * An option which is specified when retrieving one or more files. * @@ -45,14 +43,24 @@ boolean isValid(Integer value) { } }; + /** + * The maximum number of revisions to search for the last modified file revision. The default value is + * {@code 1} which means the latest revision is returned instead retrieving the old revisions. + */ + public static final FindOption FETCH_LAST_FILE_REVISION = + new FindOption("FETCH_LAST_REVISION", 1) { + @Override + boolean isValid(Integer value) { + return value >= 1; + } + }; + private final String name; private final T defaultValue; - private final String fullName; FindOption(String name, T defaultValue) { this.name = name; this.defaultValue = defaultValue; - fullName = Util.simpleTypeName(FindOption.class) + '.' + name; } /** diff --git a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java index 244b1d427..00eef72de 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java @@ -157,7 +157,21 @@ default CompletableFuture> get(Revision revision, String path) { * @see #getOrNull(Revision, Query) */ default CompletableFuture> get(Revision revision, Query query) { - return getOrNull(revision, query).thenApply(res -> { + return get(revision, query, -1); + } + + /** + * Performs the specified {@link Query}. If {@code includeLastFileRevision} is greater than 1, it attempts + * to find the last file revision from the last {@code includeLastFileRevision} commits. + * + * + * @throws EntryNotFoundException if there's no entry at the path specified in the {@link Query} + * + * @see #getOrNull(Revision, Query) + */ + default CompletableFuture> get(Revision revision, Query query, + int includeLastFileRevision) { + return getOrNull(revision, query, includeLastFileRevision).thenApply(res -> { if (res == null) { throw new EntryNotFoundException(revision, query.path()); } @@ -177,7 +191,29 @@ default CompletableFuture> get(Revision revision, Query query) { default CompletableFuture> getOrNull(Revision revision, String path) { validateFilePath(path, "path"); - return find(revision, path, FIND_ONE_WITH_CONTENT).thenApply(findResult -> findResult.get(path)); + return getOrNull(revision, path, -1); + } + + /** + * Retrieves an {@link Entry} at the specified {@code path}. If {@code includeLastFileRevision} is greater + * than 1, it attempts to find the last file revision from the last {@code includeLastFileRevision} + * commits. + * + * @return the {@link Entry} at the specified {@code path} if exists. + * The specified {@code other} if there's no such {@link Entry}. + * + * @see #get(Revision, String) + */ + default CompletableFuture> getOrNull(Revision revision, String path, int includeLastFileRevision) { + validateFilePath(path, "path"); + final Map, ?> options; + if (includeLastFileRevision > 1) { + options = ImmutableMap.of(FindOption.FETCH_LAST_FILE_REVISION, includeLastFileRevision, + FindOption.FETCH_CONTENT, true); + } else { + options = FIND_ONE_WITH_CONTENT; + } + return find(revision, path, options).thenApply(findResult -> findResult.get(path)); } /** @@ -189,10 +225,24 @@ default CompletableFuture> getOrNull(Revision revision, String path) { * @see #get(Revision, Query) */ default CompletableFuture> getOrNull(Revision revision, Query query) { + return getOrNull(revision, query, -1); + } + + /** + * Performs the specified {@link Query}. If {@code includeLastFileRevision} is greater than 1, it attempts + * to find the last file revision from the last {@code includeLastFileRevision} commits. + * + * @return the {@link Entry} on a successful query. + * The specified {@code other} on a failure due to missing entry. + * + * @see #get(Revision, Query) + */ + default CompletableFuture> getOrNull(Revision revision, Query query, + int includeLastFileRevision) { requireNonNull(query, "query"); requireNonNull(revision, "revision"); - return getOrNull(revision, query.path()).thenApply(result -> { + return getOrNull(revision, query.path(), includeLastFileRevision).thenApply(result -> { if (result == null) { return null; } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/LastFileRevisionTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/LastFileRevisionTest.java new file mode 100644 index 000000000..838a308d6 --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/LastFileRevisionTest.java @@ -0,0 +1,157 @@ +/* + * Copyright 2025 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.internal.api; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Map; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.linecorp.centraldogma.client.CentralDogma; +import com.linecorp.centraldogma.client.CentralDogmaRepository; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.Entry; +import com.linecorp.centraldogma.common.PathPattern; +import com.linecorp.centraldogma.common.PushResult; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.testing.junit.CentralDogmaExtension; + +class LastFileRevisionTest { + + @RegisterExtension + final CentralDogmaExtension dogma = new CentralDogmaExtension() { + @Override + protected void scaffold(CentralDogma client) { + client.createProject("foo").join(); + client.createRepository("foo", "bar").join(); + } + + @Override + protected boolean runForEachTest() { + return true; + } + }; + + @Test + void shouldFillFileRevisions() { + final CentralDogmaRepository repo = dogma.client().forRepo("foo", "bar"); + final PushResult resultA = repo.commit("add a file", Change.ofTextUpsert("/a.txt", "aaa")) + .push().join(); + final PushResult resultB = repo.commit("add a file", Change.ofTextUpsert("/b.txt", "bbb")) + .push().join(); + assertThat(resultB.revision().major()).isEqualTo(3); + final PushResult resultC = repo.commit("add a file", Change.ofTextUpsert("/a/c.txt", "ccc")).push() + .join(); + final PushResult resultD = repo.commit("add a file", Change.ofTextUpsert("/a/c/d.txt", "ddd")).push() + .join(); + assertThat(resultD.revision().major()).isEqualTo(5); + final PushResult resultE = repo.commit("add a file", Change.ofTextUpsert("/a/c/d/e.txt", "eee")).push() + .join(); + final PushResult resultB1 = repo.commit("update a file", Change.ofTextUpsert("/b.txt", "bbbb")) + .push().join(); + final PushResult resultD1 = repo.commit("update a file", Change.ofTextUpsert("/a/c/d.txt", "dddd")) + .push().join(); + + final Map> listFilesWithLastRevision = repo.file(PathPattern.all()) + .includeLastFileRevision(10) + .list().join(); + Entry entryA = listFilesWithLastRevision.get("/a.txt"); + assertThat(entryA.revision()).isEqualTo(resultA.revision()); + assertThat(entryA.hasContent()).isFalse(); + + Entry entryB = listFilesWithLastRevision.get("/b.txt"); + assertThat(entryB.revision()).isEqualTo(resultB1.revision()); + assertThat(entryB.hasContent()).isFalse(); + + Entry entryC = listFilesWithLastRevision.get("/a/c.txt"); + assertThat(entryC.revision()).isEqualTo(resultC.revision()); + assertThat(entryC.hasContent()).isFalse(); + + Entry entryD = listFilesWithLastRevision.get("/a/c/d.txt"); + assertThat(entryD.revision()).isEqualTo(resultD1.revision()); + assertThat(entryD.hasContent()).isFalse(); + + Entry entryE = listFilesWithLastRevision.get("/a/c/d/e.txt"); + assertThat(entryE.revision()).isEqualTo(resultE.revision()); + assertThat(entryE.hasContent()).isFalse(); + + final Map> getFilesWithLastRevision = repo.file(PathPattern.all()) + .includeLastFileRevision(10) + .get().join(); + + entryA = getFilesWithLastRevision.get("/a.txt"); + assertThat(entryA.revision()).isEqualTo(resultA.revision()); + assertThat(entryA.contentAsText().trim()).isEqualTo("aaa"); + + entryB = getFilesWithLastRevision.get("/b.txt"); + assertThat(entryB.revision()).isEqualTo(resultB1.revision()); + assertThat(entryB.contentAsText().trim()).isEqualTo("bbbb"); + + entryC = getFilesWithLastRevision.get("/a/c.txt"); + assertThat(entryC.revision()).isEqualTo(resultC.revision()); + assertThat(entryC.contentAsText().trim()).isEqualTo("ccc"); + + entryD = getFilesWithLastRevision.get("/a/c/d.txt"); + assertThat(entryD.revision()).isEqualTo(resultD1.revision()); + assertThat(entryD.contentAsText().trim()).isEqualTo("dddd"); + + entryE = getFilesWithLastRevision.get("/a/c/d/e.txt"); + assertThat(entryE.revision()).isEqualTo(resultE.revision()); + assertThat(entryE.contentAsText().trim()).isEqualTo("eee"); + + final Map> content = repo.file(PathPattern.all()).get().join(); + content.forEach((path, entry) -> { + assertThat(entry.revision()).isEqualTo(resultD1.revision()); + }); + } + + @Test + void shouldFillInitRevisionOnMissing() { + final CentralDogmaRepository repo = dogma.client().forRepo("foo", "bar"); + final PushResult resultA = repo.commit("add a file", Change.ofTextUpsert("/a.txt", "aaa")) + .push().join(); + final PushResult resultB = repo.commit("add a file", Change.ofTextUpsert("/b.txt", "bbb")) + .push().join(); + final PushResult resultB1 = repo.commit("update a file", Change.ofTextUpsert("/b.txt", "bbbb")) + .push().join(); + + final Map> getFilesWithLastRevision = repo.file(PathPattern.all()) + .includeLastFileRevision(2) + .get().join(); + Entry entryA = getFilesWithLastRevision.get("/a.txt"); + assertThat(entryA.revision()).isEqualTo(Revision.INIT); + assertThat(entryA.contentAsText().trim()).isEqualTo("aaa"); + + Entry entryB = getFilesWithLastRevision.get("/b.txt"); + assertThat(entryB.revision()).isEqualTo(resultB1.revision()); + assertThat(entryB.contentAsText().trim()).isEqualTo("bbbb"); + + final Map> listFilesWithLastRevision = repo.file(PathPattern.all()) + .includeLastFileRevision(2) + .list().join(); + + entryA = listFilesWithLastRevision.get("/a.txt"); + assertThat(entryA.revision()).isEqualTo(Revision.INIT); + assertThat(entryA.hasContent()).isFalse(); + + entryB = listFilesWithLastRevision.get("/b.txt"); + assertThat(entryB.revision()).isEqualTo(resultB1.revision()); + assertThat(entryB.hasContent()).isFalse(); + } +} diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepositoryTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepositoryTest.java index 45f4951ff..5a3c6d3ae 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepositoryTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepositoryTest.java @@ -25,6 +25,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -60,6 +61,7 @@ import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.repository.DiffResultType; +import com.linecorp.centraldogma.server.storage.repository.FindOption; import com.linecorp.centraldogma.server.storage.repository.Repository; class CachingRepositoryTest { @@ -103,9 +105,9 @@ void jsonPathQuery() throws JsonParseException { doReturn(new Revision(10)).when(delegateRepo).normalizeNow(HEAD); // Uncached - when(delegateRepo.getOrNull(any(), any(Query.class))).thenReturn(completedFuture(queryResult)); + when(delegateRepo.getOrNull(any(), any(Query.class), eq(-1))).thenReturn(completedFuture(queryResult)); assertThat(repo.get(HEAD, query).join()).isEqualTo(queryResult); - verify(delegateRepo).getOrNull(new Revision(10), query); + verify(delegateRepo).getOrNull(new Revision(10), query, -1); verifyNoMoreInteractions(delegateRepo); // Cached @@ -174,9 +176,9 @@ void jsonPathQueryMissingEntry() { doReturn(new Revision(10)).when(delegateRepo).normalizeNow(HEAD); // Uncached - when(delegateRepo.getOrNull(any(), any(Query.class))).thenReturn(completedFuture(null)); + when(delegateRepo.getOrNull(any(), any(Query.class), eq(-1))).thenReturn(completedFuture(null)); assertThat(repo.getOrNull(HEAD, query).join()).isNull(); - verify(delegateRepo).getOrNull(new Revision(10), query); + verify(delegateRepo).getOrNull(new Revision(10), query, -1); verifyNoMoreInteractions(delegateRepo); // Cached @@ -384,6 +386,45 @@ void watchSlowPath() { verifyNoMoreInteractions(delegateRepo); } + @Test + void getWithIncludeLastFileRevision() { + final Repository repo = setMockNames(newCachingRepo()); + final Query query = Query.ofText("/baz.txt"); + + final Entry result = Entry.ofText(new Revision(10), "/baz.txt", "qux"); + final Entry resultWithFileRevision = Entry.ofText(new Revision(8), "/baz.txt", "qux"); + + final Map> entries = ImmutableMap.of("/baz.txt", result); + final Map> entriesWithFileRevision = + ImmutableMap.of("/baz.txt", resultWithFileRevision); + + doReturn(new Revision(10)).when(delegateRepo).normalizeNow(new Revision(10)); + doReturn(new Revision(10)).when(delegateRepo).normalizeNow(HEAD); + + // Uncached + when(delegateRepo.find(any(), any(), eq(FIND_ONE_WITH_CONTENT))).thenReturn(completedFuture(entries)); + when(delegateRepo.find(any(), any(), + eq(ImmutableMap.of(FindOption.MAX_ENTRIES, 1, + FindOption.FETCH_LAST_FILE_REVISION, 3)))) + .thenReturn(completedFuture(entriesWithFileRevision)); + assertThat(repo.get(HEAD, query).join()).isEqualTo(result); + assertThat(repo.get(HEAD, query, 3).join()).isEqualTo(resultWithFileRevision); + verify(delegateRepo).find(new Revision(10), "/baz.txt", FIND_ONE_WITH_CONTENT); + verify(delegateRepo).find(new Revision(10), "/baz.txt", + ImmutableMap.of(FindOption.MAX_ENTRIES, 1, + FindOption.FETCH_LAST_FILE_REVISION, 3)); + verifyNoMoreInteractions(delegateRepo); + + // Cached + clearInvocations(delegateRepo); + assertThat(repo.get(HEAD, query).join()).isEqualTo(result); + assertThat(repo.get(new Revision(10), query).join()).isEqualTo(result); + assertThat(repo.get(HEAD, query, 3).join()).isEqualTo(resultWithFileRevision); + assertThat(repo.get(new Revision(10), query, 3).join()).isEqualTo(resultWithFileRevision); + verify(delegateRepo, never()).find(any(), any(), any()); + verifyNoMoreInteractions(delegateRepo); + } + private CachingRepository newCachingRepo() { final CachingRepository cachingRepo = new CachingRepository( delegateRepo, new RepositoryCache("maximumSize=1000", NoopMeterRegistry.get()));