diff --git a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java index f66dce3..f1e075b 100644 --- a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java +++ b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java @@ -26,6 +26,7 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.inject.Inject; +import com.google.inject.Singleton; import com.google.inject.name.Named; import com.mastfrog.acteur.errors.ResponseException; import com.mastfrog.acteur.headers.Headers; @@ -75,6 +76,7 @@ * * @author Tim Boudreau */ +@Singleton public class DownloaderV2A { private final HttpClient client; @@ -87,7 +89,7 @@ public class DownloaderV2A { static final AtomicLong counter = new AtomicLong(); private final TempFiles tempFiles; private final String runId; - private final String userAgent; + // private final String userAgent; private final ExecutorService pool; @Inject @@ -104,7 +106,7 @@ public DownloaderV2A(HttpClient client, Config config, FileFinder finder, this.control = control; this.runId = runId; this.tempFiles = tempFiles; - userAgent = "tmpx-" + ver.version; + // userAgent = "tmpx-" + ver.version; } String nextDownloadId() { @@ -115,13 +117,15 @@ public boolean isFailedPath(Path path) { return failedURLs.getIfPresent(path) != null; } - CompletableFuture download(Path path, RequestID rid, DownloadReceiver recv) throws URISyntaxException { - CompletableFuture tf = download(path, rid); + CompletableFuture download(Path path, RequestID rid, String userAgent, DownloadReceiver recv) throws URISyntaxException { + CompletableFuture tf = download(path, rid, userAgent); Logs requestLog = logger.child("download", rid); tf.whenComplete((file, thrown) -> { if (thrown != null) { - failedURLs.put(path, path); // System.out.println("DO FAIL FOR " + thrown + " " + path); + if (!(thrown instanceof CancellationException)) { + failedURLs.put(path, path); + } recv.failed(GONE, thrown.getMessage()); } else if (file != null) { HttpResponseStatus status = file.info().map(info -> { @@ -137,7 +141,7 @@ CompletableFuture download(Path path, RequestID rid, DownloadReceiver return tf; } - public CompletableFuture download(Path path, RequestID rid) throws URISyntaxException { + public CompletableFuture download(Path path, RequestID rid, String userAgent) throws URISyntaxException { Collection urls = config.withPath(path); List> futures = new ArrayList<>(urls.size()); Int remainder = Int.createAtomic(); @@ -216,8 +220,15 @@ public CompletableFuture download(Path path, RequestID rid) throws URI lr.add("cancelled"); if (remaining == 0) { // System.out.println("REMAINING 0 COMPLETE 1 " + path); + // result.completeExceptionally(new CancellationException("No result " + path)); + // if timeout，cache No result url result.completeExceptionally(new ResponseException(GONE, "No result " + path)); } + } else if (thrown instanceof ResponseException) { + lr.add("faild"); + if (remaining == 0) { + result.completeExceptionally(thrown); + } } else if (thrown != null) { if (remaining == 0) { // System.out.println("REMAINING 0 COMPLETE 2 " + path); @@ -235,11 +246,16 @@ public CompletableFuture download(Path path, RequestID rid) throws URI result.completeExceptionally(ex); } // } - }); + }).orTimeout(20, TimeUnit.MINUTES); // fut.whenComplete(onComplete); BH bh = new BH(dlId, u, fut, perUrl); futures.add(fut); - client.sendAsync(req, bh); + client.sendAsync(req, bh).orTimeout(15, TimeUnit.MINUTES).exceptionally((e) -> { + if(!fut.isCancelled()){ + fut.cancel(true); + } + return null; + }); } return result; } @@ -276,7 +292,9 @@ public BodySubscriber apply(HttpResponse.ResponseInfo info) { logs.warn("request-failed") .add("status", info.statusCode()).close(); // .add("headers", info.headers().map()).close(); - result.cancel(true); + result.completeExceptionally(new ResponseException(HttpResponseStatus.valueOf(info.statusCode()), + HttpResponseStatus.valueOf(info.statusCode()).reasonPhrase())); + // result.cancel(true); return NO_OP; } else { logs.info("potential-success") diff --git a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/FileFinder.java b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/FileFinder.java index bd1452a..00887c9 100644 --- a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/FileFinder.java +++ b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/FileFinder.java @@ -24,6 +24,7 @@ package com.mastfrog.tinymavenproxy; import com.google.inject.Inject; +import com.google.inject.Singleton; import com.google.inject.name.Named; import com.mastfrog.acteur.server.ServerModule; import static com.mastfrog.tinymavenproxy.GetActeur.isGzipCacheFile; @@ -48,6 +49,7 @@ * * @author Tim Boudreau */ +@Singleton public class FileFinder { private final Config config; diff --git a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/GetActeur.java b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/GetActeur.java index 3aa2155..6d564a1 100644 --- a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/GetActeur.java +++ b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/GetActeur.java @@ -179,7 +179,8 @@ public class GetActeur extends Acteur { Path pth = path.elideEmptyElements(); def.defer((Resumer res) -> { config.debugLog(" defer and download ", pth); - CompletableFuture l = dl.download(pth, id, new DownloadReceiverImpl(res, config)); + CompletableFuture l = dl.download(pth, id, req.header(Headers.USER_AGENT).toString(), + new DownloadReceiverImpl(res, config)); req.channel().closeFuture().addListener(cl -> { l.cancel(false); }); diff --git a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/TinyMavenProxy.java b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/TinyMavenProxy.java index 6214481..f49a183 100644 --- a/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/TinyMavenProxy.java +++ b/tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/TinyMavenProxy.java @@ -104,11 +104,12 @@ static SettingsBuilder defaultSettings() { .add("cors.enabled", false) .add("download-tmp", System.getProperty("java.io.tmpdir")) .add(HTTP_COMPRESSION, "false") - .add(SETTINGS_KEY_DOWNLOAD_THREADS, "24") + .add(SETTINGS_KEY_DOWNLOAD_THREADS, 24) .add(SETTINGS_KEY_ASYNC_LOGGING, false) .add(LoggingModule.SETTINGS_KEY_LOG_TO_CONSOLE, true) - .add(WORKER_THREADS, "6") - .add(EVENT_THREADS, "3") + .add(WORKER_THREADS, 6) + .add(EVENT_THREADS, 3) + .add(ServerModule.BACKGROUND_THREADS, 4) .add(ServerModule.SETTINGS_KEY_SOCKET_WRITE_SPIN_COUNT, 32) .add(SETTINGS_KEY_LOG_LEVEL, "trace") // .add(MAX_CONTENT_LENGTH, "128") // we don't accept PUTs, no need for a big buffer