Skip to content

Commit 5857ce2

Browse files
author
Dmytro Ukhlov
committed
JENKINS-75675: Refactor classloader logic in order to reduce memory consumption
1 parent 86a6fd8 commit 5857ce2

File tree

6 files changed

+239
-98
lines changed

6 files changed

+239
-98
lines changed

‎core/src/main/java/hudson/ClassicPluginStrategy.java‎

Lines changed: 55 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
import hudson.Plugin.DummyImpl;
3232
import hudson.PluginWrapper.Dependency;
3333
import hudson.model.Hudson;
34+
import hudson.util.AbstractCachingClassLoader;
35+
import hudson.util.CompoundEnumeration;
3436
import hudson.util.CyclicGraphDetector;
3537
import hudson.util.CyclicGraphDetector.CycleDetectedException;
3638
import hudson.util.IOUtils;
@@ -53,6 +55,7 @@
5355
import java.util.List;
5456
import java.util.Set;
5557
import java.util.UUID;
58+
import java.util.concurrent.atomic.AtomicReference;
5659
import java.util.jar.Attributes;
5760
import java.util.jar.JarFile;
5861
import java.util.jar.Manifest;
@@ -559,7 +562,7 @@ private static void unzipExceptClasses(File archive, File destDir, Project prj)
559562
/**
560563
* Used to load classes from dependency plugins.
561564
*/
562-
static final class DependencyClassLoader extends ClassLoader {
565+
static final class DependencyClassLoader extends AbstractCachingClassLoader {
563566
/**
564567
* This classloader is created for this plugin. Useful during debugging.
565568
*/
@@ -572,30 +575,26 @@ static final class DependencyClassLoader extends ClassLoader {
572575
/**
573576
* Topologically sorted list of transitive dependencies. Lazily initialized via double-checked locking.
574577
*/
575-
private volatile List<PluginWrapper> transitiveDependencies;
576-
577-
static {
578-
registerAsParallelCapable();
579-
}
578+
private final AtomicReference<List<PluginWrapper>> transitiveDependencies = new AtomicReference<>();
580579

581580
DependencyClassLoader(ClassLoader parent, File archive, List<Dependency> dependencies, PluginManager pluginManager) {
582-
super("dependency ClassLoader for " + archive.getPath(), parent);
581+
super("dependency ClassLoader for " + archive.getPath(), parent, 10);
583582
this._for = archive;
584583
this.dependencies = List.copyOf(dependencies);
585584
this.pluginManager = pluginManager;
586585
}
587586

588587
private void updateTransitiveDependencies() {
589588
// This will be recalculated at the next time.
590-
transitiveDependencies = null;
589+
transitiveDependencies.set(null);
591590
}
592591

593592
private List<PluginWrapper> getTransitiveDependencies() {
594-
List<PluginWrapper> localTransitiveDependencies = transitiveDependencies;
595-
if (localTransitiveDependencies == null) {
596-
synchronized (this) {
597-
localTransitiveDependencies = transitiveDependencies;
598-
if (localTransitiveDependencies == null) {
593+
return transitiveDependencies.accumulateAndGet(null, (old, ignored) -> {
594+
if (old != null) {
595+
return old;
596+
}
597+
599598
CyclicGraphDetector<PluginWrapper> cgd = new CyclicGraphDetector<>() {
600599
@Override
601600
protected List<PluginWrapper> getEdges(PluginWrapper pw) {
@@ -619,15 +618,18 @@ protected List<PluginWrapper> getEdges(PluginWrapper pw) {
619618
throw new AssertionError(e); // such error should have been reported earlier
620619
}
621620

622-
transitiveDependencies = localTransitiveDependencies = cgd.getSorted();
623-
}
624-
}
625-
}
626-
return localTransitiveDependencies;
621+
return cgd.getSorted();
622+
});
627623
}
628624

629625
@Override
630-
protected Class<?> findClass(String name) throws ClassNotFoundException {
626+
public Class<?> doLoadClass(String name) {
627+
try {
628+
return getParent().loadClass(name);
629+
} catch (ClassNotFoundException ignored) {
630+
// OK, try next
631+
}
632+
631633
if (PluginManager.FAST_LOOKUP) {
632634
for (PluginWrapper pw : getTransitiveDependencies()) {
633635
try {
@@ -649,49 +651,65 @@ protected Class<?> findClass(String name) throws ClassNotFoundException {
649651
}
650652
}
651653

652-
throw new ClassNotFoundException(name);
654+
return null;
653655
}
654656

655657
@Override
656658
@SuppressFBWarnings(value = "DMI_COLLECTION_OF_URLS",
657659
justification = "Should not produce network overheads since the URL is local. JENKINS-53793 is a follow-up")
658-
protected Enumeration<URL> findResources(String name) throws IOException {
659-
HashSet<URL> result = new HashSet<>();
660+
public Enumeration<URL> getResources(String name) throws IOException {
661+
ArrayList<Enumeration<? extends URL>> enumerations = new ArrayList<>();
662+
663+
Enumeration<URL> result;
660664

661665
if (PluginManager.FAST_LOOKUP) {
662-
for (PluginWrapper pw : getTransitiveDependencies()) {
663-
Enumeration<URL> urls = ClassLoaderReflectionToolkit._findResources(pw.classLoader, name);
664-
while (urls != null && urls.hasMoreElements())
665-
result.add(urls.nextElement());
666-
}
666+
enumerations.add(getParent().getResources(name));
667+
668+
for (PluginWrapper pw : getTransitiveDependencies()) {
669+
enumerations.add(ClassLoaderReflectionToolkit._findResources(pw.classLoader, name));
670+
}
671+
result = new CompoundEnumeration<>(enumerations);
667672
} else {
668673
for (Dependency dep : dependencies) {
669674
PluginWrapper p = pluginManager.getPlugin(dep.shortName);
670675
if (p != null) {
671-
Enumeration<URL> urls = p.classLoader.getResources(name);
672-
while (urls != null && urls.hasMoreElements())
673-
result.add(urls.nextElement());
676+
enumerations.add(p.classLoader.getResources(name));
674677
}
675678
}
679+
result = new CompoundEnumeration<>(enumerations);
680+
Set<URL> resultSet = new HashSet<>();
681+
while (result.hasMoreElements()) {
682+
resultSet.add(result.nextElement());
683+
}
684+
685+
result = Collections.enumeration(resultSet);
676686
}
677687

678-
return Collections.enumeration(result);
688+
return result;
679689
}
680690

681691
@Override
682-
protected URL findResource(String name) {
692+
public URL getResource(String name) {
693+
URL url = getParent().getResource(name);
694+
if (url != null) {
695+
return url;
696+
}
697+
683698
if (PluginManager.FAST_LOOKUP) {
684-
for (PluginWrapper pw : getTransitiveDependencies()) {
685-
URL url = ClassLoaderReflectionToolkit._findResource(pw.classLoader, name);
686-
if (url != null) return url;
699+
for (PluginWrapper pw : getTransitiveDependencies()) {
700+
url = ClassLoaderReflectionToolkit._findResource(pw.classLoader, name);
701+
if (url != null) {
702+
return url;
687703
}
704+
}
688705
} else {
689706
for (Dependency dep : dependencies) {
690707
PluginWrapper p = pluginManager.getPlugin(dep.shortName);
691708
if (p != null) {
692-
URL url = p.classLoader.getResource(name);
693-
if (url != null)
709+
url = p.classLoader.getResource(name);
710+
if (url != null) {
694711
return url;
712+
}
695713
}
696714
}
697715
}

‎core/src/main/java/hudson/PluginFirstClassLoader2.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,6 @@
2121
*/
2222
@Restricted(NoExternalUse.class)
2323
public class PluginFirstClassLoader2 extends URLClassLoader2 {
24-
static {
25-
registerAsParallelCapable();
26-
}
27-
2824

2925
public PluginFirstClassLoader2(String name, @NonNull URL[] urls, @NonNull ClassLoader parent) {
3026
super(name, Objects.requireNonNull(urls), Objects.requireNonNull(parent));

‎core/src/main/java/hudson/PluginManager.java‎

Lines changed: 79 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@
5858
import hudson.security.ACLContext;
5959
import hudson.security.Permission;
6060
import hudson.security.PermissionScope;
61+
import hudson.util.AbstractCachingClassLoader;
62+
import hudson.util.CompoundEnumeration;
6163
import hudson.util.CyclicGraphDetector;
6264
import hudson.util.CyclicGraphDetector.CycleDetectedException;
6365
import hudson.util.FormValidation;
@@ -106,13 +108,11 @@
106108
import java.util.Locale;
107109
import java.util.Map;
108110
import java.util.Objects;
109-
import java.util.Optional;
110111
import java.util.ServiceLoader;
111112
import java.util.Set;
112113
import java.util.TreeMap;
113114
import java.util.UUID;
114115
import java.util.concurrent.ConcurrentHashMap;
115-
import java.util.concurrent.ConcurrentMap;
116116
import java.util.concurrent.CopyOnWriteArrayList;
117117
import java.util.concurrent.Future;
118118
import java.util.function.Supplier;
@@ -2396,76 +2396,111 @@ public <T> T of(String key, Class<T> type, Supplier<T> func) {
23962396
/**
23972397
* {@link ClassLoader} that can see all plugins.
23982398
*/
2399-
public static final class UberClassLoader extends ClassLoader {
2399+
public static final class UberClassLoader extends AbstractCachingClassLoader {
2400+
// 100k entries is a reasonable default. ~20MB of memory.
2401+
private static final int MISSED_CLASS_CACHE_SIZE = Integer.parseInt(
2402+
System.getProperty(UberClassLoader.class.getName() + ".MISSED_CLASS_CACHE_SIZE", "100000"));
2403+
24002404
private final List<PluginWrapper> activePlugins;
24012405

24022406
/** Cache of loaded, or known to be unloadable, classes. */
2403-
private final ConcurrentMap<String, Optional<Class<?>>> loaded = new ConcurrentHashMap<>();
2404-
2405-
static {
2406-
registerAsParallelCapable();
2407-
}
2408-
24092407
public UberClassLoader(List<PluginWrapper> activePlugins) {
2410-
super("UberClassLoader", PluginManager.class.getClassLoader());
2408+
super("UberClassLoader", PluginManager.class.getClassLoader(), MISSED_CLASS_CACHE_SIZE);
24112409
this.activePlugins = activePlugins;
24122410
}
24132411

24142412
@Override
2415-
protected Class<?> findClass(String name) throws ClassNotFoundException {
2416-
if (name.startsWith("SimpleTemplateScript")) { // cf. groovy.text.SimpleTemplateEngine
2417-
throw new ClassNotFoundException("ignoring " + name);
2413+
protected boolean isClassKnownMissed(String name) {
2414+
if (name.startsWith("SimpleTemplateScript") ||
2415+
name.startsWith("groovy.tmp.")) { // cf. groovy.text.SimpleTemplateEngine
2416+
return true;
24182417
}
2419-
return loaded.computeIfAbsent(name, this::computeValue).orElseThrow(() -> new ClassNotFoundException(name));
2418+
2419+
return super.isClassKnownMissed(name);
24202420
}
24212421

2422-
private Optional<Class<?>> computeValue(String name) {
2423-
for (PluginWrapper p : activePlugins) {
2424-
try {
2425-
if (FAST_LOOKUP) {
2426-
return Optional.of(ClassLoaderReflectionToolkit.loadClass(p.classLoader, name));
2427-
} else {
2428-
return Optional.of(p.classLoader.loadClass(name));
2422+
@Override
2423+
protected Class<?> doLoadClass(String name) {
2424+
try {
2425+
return getParent().loadClass(name);
2426+
} catch (ClassNotFoundException e) {
2427+
// Not found. Try the next class loader.
2428+
}
2429+
2430+
if (FAST_LOOKUP) {
2431+
for (PluginWrapper p : activePlugins) {
2432+
try {
2433+
return ClassLoaderReflectionToolkit.loadClass(p.classLoader, name);
2434+
} catch (ClassNotFoundException e) {
2435+
// Not found. Try the next class loader.
2436+
}
2437+
}
2438+
} else {
2439+
for (PluginWrapper p : activePlugins) {
2440+
try {
2441+
return p.classLoader.loadClass(name);
2442+
} catch (ClassNotFoundException e) {
2443+
// Not found. Try the next class loader.
24292444
}
2430-
} catch (ClassNotFoundException e) {
2431-
// Not found. Try the next class loader.
24322445
}
24332446
}
2447+
24342448
// Not found in any of the class loaders. Delegate.
2435-
return Optional.empty();
2449+
return null;
24362450
}
24372451

24382452
@Override
2439-
protected URL findResource(String name) {
2440-
for (PluginWrapper p : activePlugins) {
2441-
URL url;
2442-
if (FAST_LOOKUP) {
2443-
url = ClassLoaderReflectionToolkit._findResource(p.classLoader, name);
2444-
} else {
2445-
url = p.classLoader.getResource(name);
2453+
public URL getResource(String name) {
2454+
URL url = getParent().getResource(name);
2455+
if (url != null) {
2456+
return url;
2457+
}
2458+
2459+
if (PluginManager.FAST_LOOKUP) {
2460+
for (PluginWrapper pw : activePlugins) {
2461+
url = ClassLoaderReflectionToolkit._findResource(pw.classLoader, name);
2462+
if (url != null) {
2463+
return url;
2464+
}
24462465
}
2447-
if (url != null) {
2448-
return url;
2466+
} else {
2467+
for (PluginWrapper pw : activePlugins) {
2468+
url = pw.classLoader.getResource(name);
2469+
if (url != null) {
2470+
return url;
2471+
}
24492472
}
24502473
}
2474+
24512475
return null;
24522476
}
24532477

24542478
@Override
2455-
protected Enumeration<URL> findResources(String name) throws IOException {
2456-
List<URL> resources = new ArrayList<>();
2457-
for (PluginWrapper p : activePlugins) {
2458-
if (FAST_LOOKUP) {
2459-
resources.addAll(Collections.list(ClassLoaderReflectionToolkit._findResources(p.classLoader, name)));
2460-
} else {
2461-
resources.addAll(Collections.list(p.classLoader.getResources(name)));
2479+
public Enumeration<URL> getResources(String name) throws IOException {
2480+
ArrayList<Enumeration<? extends URL>> enumerations = new ArrayList<>();
2481+
Enumeration<URL> result;
2482+
2483+
if (PluginManager.FAST_LOOKUP) {
2484+
enumerations.add(getParent().getResources(name));
2485+
2486+
for (PluginWrapper pw : activePlugins) {
2487+
enumerations.add(ClassLoaderReflectionToolkit._findResources(pw.classLoader, name));
2488+
}
2489+
result = new CompoundEnumeration<>(enumerations);
2490+
} else {
2491+
for (PluginWrapper pw : activePlugins) {
2492+
enumerations.add(pw.classLoader.getResources(name));
2493+
}
2494+
result = new CompoundEnumeration<>(enumerations);
2495+
Set<URL> resultSet = new HashSet<>();
2496+
while (result.hasMoreElements()) {
2497+
resultSet.add(result.nextElement());
24622498
}
2499+
2500+
result = Collections.enumeration(resultSet);
24632501
}
2464-
return Collections.enumeration(resources);
2465-
}
24662502

2467-
void clearCacheMisses() {
2468-
loaded.values().removeIf(Optional::isEmpty);
2503+
return result;
24692504
}
24702505

24712506
@Override

0 commit comments

Comments
 (0)