Skip to content

Commit 25e29b6

Browse files
author
Dmytro Ukhlov
committed
JENKINS-75675: Refactor classloader logic in order to reduce memory consumption
1 parent 86a6fd8 commit 25e29b6

6 files changed

Lines changed: 209 additions & 85 deletions

File tree

‎core/src/main/java/hudson/ClassicPluginStrategy.java‎

Lines changed: 46 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
import hudson.Plugin.DummyImpl;
3232
import hudson.PluginWrapper.Dependency;
3333
import hudson.model.Hudson;
34+
import hudson.util.AbstractCachingClassLoader;
35+
import hudson.util.CompoundEnumeration;
3436
import hudson.util.CyclicGraphDetector;
3537
import hudson.util.CyclicGraphDetector.CycleDetectedException;
3638
import hudson.util.IOUtils;
@@ -559,7 +561,7 @@ private static void unzipExceptClasses(File archive, File destDir, Project prj)
559561
/**
560562
* Used to load classes from dependency plugins.
561563
*/
562-
static final class DependencyClassLoader extends ClassLoader {
564+
static final class DependencyClassLoader extends AbstractCachingClassLoader {
563565
/**
564566
* This classloader is created for this plugin. Useful during debugging.
565567
*/
@@ -574,12 +576,8 @@ static final class DependencyClassLoader extends ClassLoader {
574576
*/
575577
private volatile List<PluginWrapper> transitiveDependencies;
576578

577-
static {
578-
registerAsParallelCapable();
579-
}
580-
581579
DependencyClassLoader(ClassLoader parent, File archive, List<Dependency> dependencies, PluginManager pluginManager) {
582-
super("dependency ClassLoader for " + archive.getPath(), parent);
580+
super("dependency ClassLoader for " + archive.getPath(), parent, 10);
583581
this._for = archive;
584582
this.dependencies = List.copyOf(dependencies);
585583
this.pluginManager = pluginManager;
@@ -627,7 +625,14 @@ protected List<PluginWrapper> getEdges(PluginWrapper pw) {
627625
}
628626

629627
@Override
630-
protected Class<?> findClass(String name) throws ClassNotFoundException {
628+
protected Class<?> doLoadClass(String name) {
629+
630+
try {
631+
return getParent().loadClass(name);
632+
} catch (ClassNotFoundException ignored) {
633+
// OK, try next
634+
}
635+
631636
if (PluginManager.FAST_LOOKUP) {
632637
for (PluginWrapper pw : getTransitiveDependencies()) {
633638
try {
@@ -649,49 +654,65 @@ protected Class<?> findClass(String name) throws ClassNotFoundException {
649654
}
650655
}
651656

652-
throw new ClassNotFoundException(name);
657+
return null;
653658
}
654659

655660
@Override
656661
@SuppressFBWarnings(value = "DMI_COLLECTION_OF_URLS",
657662
justification = "Should not produce network overheads since the URL is local. JENKINS-53793 is a follow-up")
658-
protected Enumeration<URL> findResources(String name) throws IOException {
659-
HashSet<URL> result = new HashSet<>();
663+
public Enumeration<URL> getResources(String name) throws IOException {
664+
ArrayList<Enumeration<? extends URL>> enumerations = new ArrayList<>();
665+
666+
Enumeration<URL> result;
660667

661668
if (PluginManager.FAST_LOOKUP) {
662-
for (PluginWrapper pw : getTransitiveDependencies()) {
663-
Enumeration<URL> urls = ClassLoaderReflectionToolkit._findResources(pw.classLoader, name);
664-
while (urls != null && urls.hasMoreElements())
665-
result.add(urls.nextElement());
666-
}
669+
enumerations.add(getParent().getResources(name));
670+
671+
for (PluginWrapper pw : getTransitiveDependencies()) {
672+
enumerations.add(ClassLoaderReflectionToolkit._findResources(pw.classLoader, name));
673+
}
674+
result = new CompoundEnumeration<>(enumerations);
667675
} else {
668676
for (Dependency dep : dependencies) {
669677
PluginWrapper p = pluginManager.getPlugin(dep.shortName);
670678
if (p != null) {
671-
Enumeration<URL> urls = p.classLoader.getResources(name);
672-
while (urls != null && urls.hasMoreElements())
673-
result.add(urls.nextElement());
679+
enumerations.add(p.classLoader.getResources(name));
674680
}
675681
}
682+
result = new CompoundEnumeration<>(enumerations);
683+
Set<URL> resultSet = new HashSet<>();
684+
while (result.hasMoreElements()) {
685+
resultSet.add(result.nextElement());
686+
}
687+
688+
result = Collections.enumeration(resultSet);
676689
}
677690

678-
return Collections.enumeration(result);
691+
return result;
679692
}
680693

681694
@Override
682-
protected URL findResource(String name) {
695+
public URL getResource(String name) {
696+
URL url = getParent().getResource(name);
697+
if (url != null) {
698+
return url;
699+
}
700+
683701
if (PluginManager.FAST_LOOKUP) {
684-
for (PluginWrapper pw : getTransitiveDependencies()) {
685-
URL url = ClassLoaderReflectionToolkit._findResource(pw.classLoader, name);
686-
if (url != null) return url;
702+
for (PluginWrapper pw : getTransitiveDependencies()) {
703+
url = ClassLoaderReflectionToolkit._findResource(pw.classLoader, name);
704+
if (url != null) {
705+
return url;
687706
}
707+
}
688708
} else {
689709
for (Dependency dep : dependencies) {
690710
PluginWrapper p = pluginManager.getPlugin(dep.shortName);
691711
if (p != null) {
692-
URL url = p.classLoader.getResource(name);
693-
if (url != null)
712+
url = p.classLoader.getResource(name);
713+
if (url != null) {
694714
return url;
715+
}
695716
}
696717
}
697718
}

‎core/src/main/java/hudson/PluginFirstClassLoader2.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,6 @@
2121
*/
2222
@Restricted(NoExternalUse.class)
2323
public class PluginFirstClassLoader2 extends URLClassLoader2 {
24-
static {
25-
registerAsParallelCapable();
26-
}
27-
2824

2925
public PluginFirstClassLoader2(String name, @NonNull URL[] urls, @NonNull ClassLoader parent) {
3026
super(name, Objects.requireNonNull(urls), Objects.requireNonNull(parent));

‎core/src/main/java/hudson/PluginManager.java‎

Lines changed: 79 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@
5858
import hudson.security.ACLContext;
5959
import hudson.security.Permission;
6060
import hudson.security.PermissionScope;
61+
import hudson.util.AbstractCachingClassLoader;
62+
import hudson.util.CompoundEnumeration;
6163
import hudson.util.CyclicGraphDetector;
6264
import hudson.util.CyclicGraphDetector.CycleDetectedException;
6365
import hudson.util.FormValidation;
@@ -106,13 +108,11 @@
106108
import java.util.Locale;
107109
import java.util.Map;
108110
import java.util.Objects;
109-
import java.util.Optional;
110111
import java.util.ServiceLoader;
111112
import java.util.Set;
112113
import java.util.TreeMap;
113114
import java.util.UUID;
114115
import java.util.concurrent.ConcurrentHashMap;
115-
import java.util.concurrent.ConcurrentMap;
116116
import java.util.concurrent.CopyOnWriteArrayList;
117117
import java.util.concurrent.Future;
118118
import java.util.function.Supplier;
@@ -2396,76 +2396,111 @@ public <T> T of(String key, Class<T> type, Supplier<T> func) {
23962396
/**
23972397
* {@link ClassLoader} that can see all plugins.
23982398
*/
2399-
public static final class UberClassLoader extends ClassLoader {
2399+
public static final class UberClassLoader extends AbstractCachingClassLoader {
2400+
// 100k entries is a reasonable default. ~20MB of memory.
2401+
private static final int MISSED_CLASS_CACHE_SIZE = Integer.parseInt(
2402+
System.getProperty(UberClassLoader.class.getName() + ".MISSED_CLASS_CACHE_SIZE", "100000"));
2403+
24002404
private final List<PluginWrapper> activePlugins;
24012405

24022406
/** Cache of loaded, or known to be unloadable, classes. */
2403-
private final ConcurrentMap<String, Optional<Class<?>>> loaded = new ConcurrentHashMap<>();
2404-
2405-
static {
2406-
registerAsParallelCapable();
2407-
}
2408-
24092407
public UberClassLoader(List<PluginWrapper> activePlugins) {
2410-
super("UberClassLoader", PluginManager.class.getClassLoader());
2408+
super("UberClassLoader", PluginManager.class.getClassLoader(), MISSED_CLASS_CACHE_SIZE);
24112409
this.activePlugins = activePlugins;
24122410
}
24132411

24142412
@Override
2415-
protected Class<?> findClass(String name) throws ClassNotFoundException {
2416-
if (name.startsWith("SimpleTemplateScript")) { // cf. groovy.text.SimpleTemplateEngine
2417-
throw new ClassNotFoundException("ignoring " + name);
2413+
protected boolean isClassKnownAsMissed(String name) {
2414+
if (name.startsWith("SimpleTemplateScript") ||
2415+
name.startsWith("groovy.tmp.")) { // cf. groovy.text.SimpleTemplateEngine
2416+
return true;
24182417
}
2419-
return loaded.computeIfAbsent(name, this::computeValue).orElseThrow(() -> new ClassNotFoundException(name));
2418+
2419+
return super.isClassKnownAsMissed(name);
24202420
}
24212421

2422-
private Optional<Class<?>> computeValue(String name) {
2423-
for (PluginWrapper p : activePlugins) {
2424-
try {
2425-
if (FAST_LOOKUP) {
2426-
return Optional.of(ClassLoaderReflectionToolkit.loadClass(p.classLoader, name));
2427-
} else {
2428-
return Optional.of(p.classLoader.loadClass(name));
2422+
@Override
2423+
protected Class<?> doLoadClass(String name) {
2424+
try {
2425+
return getParent().loadClass(name);
2426+
} catch (ClassNotFoundException e) {
2427+
// Not found. Try the next class loader.
2428+
}
2429+
2430+
if (FAST_LOOKUP) {
2431+
for (PluginWrapper p : activePlugins) {
2432+
try {
2433+
return ClassLoaderReflectionToolkit.loadClass(p.classLoader, name);
2434+
} catch (ClassNotFoundException e) {
2435+
// Not found. Try the next class loader.
2436+
}
2437+
}
2438+
} else {
2439+
for (PluginWrapper p : activePlugins) {
2440+
try {
2441+
return p.classLoader.loadClass(name);
2442+
} catch (ClassNotFoundException e) {
2443+
// Not found. Try the next class loader.
24292444
}
2430-
} catch (ClassNotFoundException e) {
2431-
// Not found. Try the next class loader.
24322445
}
24332446
}
2447+
24342448
// Not found in any of the class loaders. Delegate.
2435-
return Optional.empty();
2449+
return null;
24362450
}
24372451

24382452
@Override
2439-
protected URL findResource(String name) {
2440-
for (PluginWrapper p : activePlugins) {
2441-
URL url;
2442-
if (FAST_LOOKUP) {
2443-
url = ClassLoaderReflectionToolkit._findResource(p.classLoader, name);
2444-
} else {
2445-
url = p.classLoader.getResource(name);
2453+
public URL getResource(String name) {
2454+
URL url = getParent().getResource(name);
2455+
if (url != null) {
2456+
return url;
2457+
}
2458+
2459+
if (PluginManager.FAST_LOOKUP) {
2460+
for (PluginWrapper pw : activePlugins) {
2461+
url = ClassLoaderReflectionToolkit._findResource(pw.classLoader, name);
2462+
if (url != null) {
2463+
return url;
2464+
}
24462465
}
2447-
if (url != null) {
2448-
return url;
2466+
} else {
2467+
for (PluginWrapper pw : activePlugins) {
2468+
url = pw.classLoader.getResource(name);
2469+
if (url != null) {
2470+
return url;
2471+
}
24492472
}
24502473
}
2474+
24512475
return null;
24522476
}
24532477

24542478
@Override
2455-
protected Enumeration<URL> findResources(String name) throws IOException {
2456-
List<URL> resources = new ArrayList<>();
2457-
for (PluginWrapper p : activePlugins) {
2458-
if (FAST_LOOKUP) {
2459-
resources.addAll(Collections.list(ClassLoaderReflectionToolkit._findResources(p.classLoader, name)));
2460-
} else {
2461-
resources.addAll(Collections.list(p.classLoader.getResources(name)));
2479+
public Enumeration<URL> getResources(String name) throws IOException {
2480+
ArrayList<Enumeration<? extends URL>> enumerations = new ArrayList<>();
2481+
Enumeration<URL> result;
2482+
2483+
if (PluginManager.FAST_LOOKUP) {
2484+
enumerations.add(getParent().getResources(name));
2485+
2486+
for (PluginWrapper pw : activePlugins) {
2487+
enumerations.add(ClassLoaderReflectionToolkit._findResources(pw.classLoader, name));
2488+
}
2489+
result = new CompoundEnumeration<>(enumerations);
2490+
} else {
2491+
for (PluginWrapper pw : activePlugins) {
2492+
enumerations.add(pw.classLoader.getResources(name));
2493+
}
2494+
result = new CompoundEnumeration<>(enumerations);
2495+
Set<URL> resultSet = new HashSet<>();
2496+
while (result.hasMoreElements()) {
2497+
resultSet.add(result.nextElement());
24622498
}
2499+
2500+
result = Collections.enumeration(resultSet);
24632501
}
2464-
return Collections.enumeration(resources);
2465-
}
24662502

2467-
void clearCacheMisses() {
2468-
loaded.values().removeIf(Optional::isEmpty);
2503+
return result;
24692504
}
24702505

24712506
@Override

‎core/src/main/java/hudson/util/AbstractCachingClassLoader.java‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
package hudson.util;
2+
3+
import java.util.Collections;
4+
import java.util.LinkedHashMap;
5+
import java.util.Map;
6+
import java.util.concurrent.ConcurrentHashMap;
7+
8+
/**
9+
* {@link ClassLoader} that caches loadClass invocation results
10+
*
11+
* <p>
12+
* This class is used for constructing a facade class loader which use a lot of other class sources.
13+
* It helps to avoid looking up using all the class sources each time
14+
*
15+
* @author Dmytro Ukhlov
16+
*/
17+
public abstract class AbstractCachingClassLoader extends ClassLoader {
18+
private final ConcurrentHashMap<String, Class<?>> loadedClassMapping = new ConcurrentHashMap<>();
19+
20+
private final int missedClassCacheSize;
21+
22+
private final Map<String, String> missedClassMapping = Collections.synchronizedMap(new LinkedHashMap<>() {
23+
protected boolean removeEldestEntry(Map.Entry<String, String> eldest) {
24+
return size() > missedClassCacheSize;
25+
}
26+
});
27+
28+
/**
29+
* Initialize base implementation, bypass parameters to java.lang.ClassLoader base implementation
30+
*
31+
* @param name class loader name; or {@code null} if not named
32+
* @param parent the parent class loader
33+
*
34+
* @param missedClassCacheSize maximum number of missed classes to be remembered
35+
*
36+
*/
37+
protected AbstractCachingClassLoader(String name, ClassLoader parent, int missedClassCacheSize) {
38+
super(name, parent);
39+
this.missedClassCacheSize = missedClassCacheSize;
40+
}
41+
42+
public final void clearCacheMisses() {
43+
missedClassMapping.clear();
44+
}
45+
46+
@Override
47+
protected final Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
48+
if (isClassKnownAsMissed(name)) {
49+
throw new ClassNotFoundException(name);
50+
}
51+
52+
Class<?> clazz = loadedClassMapping.computeIfAbsent(name, key -> {
53+
if (getResource(name.replace('.', '/') + ".class") == null) {
54+
return null;
55+
}
56+
return doLoadClass(name);
57+
});
58+
59+
if (clazz == null) {
60+
missedClassMapping.put(name, name);
61+
throw new ClassNotFoundException(name);
62+
}
63+
64+
if (resolve) {
65+
resolveClass(clazz);
66+
}
67+
68+
return clazz;
69+
}
70+
71+
protected boolean isClassKnownAsMissed(String name) {
72+
return missedClassMapping.containsKey(name);
73+
}
74+
75+
protected abstract Class<?> doLoadClass(String name);
76+
}

0 commit comments

Comments
 (0)