Skip to content

Commit b9045b6

Browse files
author
Dmytro Ukhlov
committed
JENKINS-75675: Refactor classloader logic in order to reduce memory consumption
1 parent 86a6fd8 commit b9045b6

6 files changed

Lines changed: 145 additions & 55 deletions

File tree

‎core/src/main/java/hudson/ClassicPluginStrategy.java‎

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import hudson.Plugin.DummyImpl;
3232
import hudson.PluginWrapper.Dependency;
3333
import hudson.model.Hudson;
34+
import hudson.util.AbstractCachingClassLoader;
3435
import hudson.util.CyclicGraphDetector;
3536
import hudson.util.CyclicGraphDetector.CycleDetectedException;
3637
import hudson.util.IOUtils;
@@ -559,7 +560,7 @@ private static void unzipExceptClasses(File archive, File destDir, Project prj)
559560
/**
560561
* Used to load classes from dependency plugins.
561562
*/
562-
static final class DependencyClassLoader extends ClassLoader {
563+
static final class DependencyClassLoader extends AbstractCachingClassLoader {
563564
/**
564565
* This classloader is created for this plugin. Useful during debugging.
565566
*/
@@ -574,12 +575,8 @@ static final class DependencyClassLoader extends ClassLoader {
574575
*/
575576
private volatile List<PluginWrapper> transitiveDependencies;
576577

577-
static {
578-
registerAsParallelCapable();
579-
}
580-
581578
DependencyClassLoader(ClassLoader parent, File archive, List<Dependency> dependencies, PluginManager pluginManager) {
582-
super("dependency ClassLoader for " + archive.getPath(), parent);
579+
super("dependency ClassLoader for " + archive.getPath(), parent, 10);
583580
this._for = archive;
584581
this.dependencies = List.copyOf(dependencies);
585582
this.pluginManager = pluginManager;
@@ -627,7 +624,14 @@ protected List<PluginWrapper> getEdges(PluginWrapper pw) {
627624
}
628625

629626
@Override
630-
protected Class<?> findClass(String name) throws ClassNotFoundException {
627+
protected Class<?> doLoadClass(String name) {
628+
629+
try {
630+
return getParent().loadClass(name);
631+
} catch (ClassNotFoundException ignored) {
632+
// OK, try next
633+
}
634+
631635
if (PluginManager.FAST_LOOKUP) {
632636
for (PluginWrapper pw : getTransitiveDependencies()) {
633637
try {
@@ -649,26 +653,31 @@ protected Class<?> findClass(String name) throws ClassNotFoundException {
649653
}
650654
}
651655

652-
throw new ClassNotFoundException(name);
656+
return null;
653657
}
654658

655659
@Override
656660
@SuppressFBWarnings(value = "DMI_COLLECTION_OF_URLS",
657661
justification = "Should not produce network overheads since the URL is local. JENKINS-53793 is a follow-up")
658-
protected Enumeration<URL> findResources(String name) throws IOException {
662+
public Enumeration<URL> getResources(String name) throws IOException {
659663
HashSet<URL> result = new HashSet<>();
660664

665+
Enumeration<URL> urls = getParent().getResources(name);
666+
while (urls.hasMoreElements()) {
667+
result.add(urls.nextElement());
668+
}
669+
661670
if (PluginManager.FAST_LOOKUP) {
662671
for (PluginWrapper pw : getTransitiveDependencies()) {
663-
Enumeration<URL> urls = ClassLoaderReflectionToolkit._findResources(pw.classLoader, name);
672+
urls = ClassLoaderReflectionToolkit._findResources(pw.classLoader, name);
664673
while (urls != null && urls.hasMoreElements())
665674
result.add(urls.nextElement());
666675
}
667676
} else {
668677
for (Dependency dep : dependencies) {
669678
PluginWrapper p = pluginManager.getPlugin(dep.shortName);
670679
if (p != null) {
671-
Enumeration<URL> urls = p.classLoader.getResources(name);
680+
urls = p.classLoader.getResources(name);
672681
while (urls != null && urls.hasMoreElements())
673682
result.add(urls.nextElement());
674683
}
@@ -679,17 +688,22 @@ protected Enumeration<URL> findResources(String name) throws IOException {
679688
}
680689

681690
@Override
682-
protected URL findResource(String name) {
691+
public URL getResource(String name) {
692+
URL url = getParent().getResource(name);
693+
if (url != null) {
694+
return url;
695+
}
696+
683697
if (PluginManager.FAST_LOOKUP) {
684698
for (PluginWrapper pw : getTransitiveDependencies()) {
685-
URL url = ClassLoaderReflectionToolkit._findResource(pw.classLoader, name);
699+
url = ClassLoaderReflectionToolkit._findResource(pw.classLoader, name);
686700
if (url != null) return url;
687701
}
688702
} else {
689703
for (Dependency dep : dependencies) {
690704
PluginWrapper p = pluginManager.getPlugin(dep.shortName);
691705
if (p != null) {
692-
URL url = p.classLoader.getResource(name);
706+
url = p.classLoader.getResource(name);
693707
if (url != null)
694708
return url;
695709
}

‎core/src/main/java/hudson/PluginFirstClassLoader2.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,6 @@
2121
*/
2222
@Restricted(NoExternalUse.class)
2323
public class PluginFirstClassLoader2 extends URLClassLoader2 {
24-
static {
25-
registerAsParallelCapable();
26-
}
27-
2824

2925
public PluginFirstClassLoader2(String name, @NonNull URL[] urls, @NonNull ClassLoader parent) {
3026
super(name, Objects.requireNonNull(urls), Objects.requireNonNull(parent));

‎core/src/main/java/hudson/PluginManager.java‎

Lines changed: 33 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
import hudson.security.ACLContext;
5959
import hudson.security.Permission;
6060
import hudson.security.PermissionScope;
61+
import hudson.util.AbstractCachingClassLoader;
6162
import hudson.util.CyclicGraphDetector;
6263
import hudson.util.CyclicGraphDetector.CycleDetectedException;
6364
import hudson.util.FormValidation;
@@ -106,13 +107,11 @@
106107
import java.util.Locale;
107108
import java.util.Map;
108109
import java.util.Objects;
109-
import java.util.Optional;
110110
import java.util.ServiceLoader;
111111
import java.util.Set;
112112
import java.util.TreeMap;
113113
import java.util.UUID;
114114
import java.util.concurrent.ConcurrentHashMap;
115-
import java.util.concurrent.ConcurrentMap;
116115
import java.util.concurrent.CopyOnWriteArrayList;
117116
import java.util.concurrent.Future;
118117
import java.util.function.Supplier;
@@ -2396,49 +2395,60 @@ public <T> T of(String key, Class<T> type, Supplier<T> func) {
23962395
/**
23972396
* {@link ClassLoader} that can see all plugins.
23982397
*/
2399-
public static final class UberClassLoader extends ClassLoader {
2398+
public static final class UberClassLoader extends AbstractCachingClassLoader {
2399+
// 100k entries is a reasonable default. ~20MB of memory.
2400+
private static final int MISSED_CLASS_CACHE_SIZE = Integer.parseInt(
2401+
System.getProperty(UberClassLoader.class.getName() + ".MISSED_CLASS_CACHE_SIZE", "100000"));
2402+
24002403
private final List<PluginWrapper> activePlugins;
24012404

24022405
/** Cache of loaded, or known to be unloadable, classes. */
2403-
private final ConcurrentMap<String, Optional<Class<?>>> loaded = new ConcurrentHashMap<>();
2404-
2405-
static {
2406-
registerAsParallelCapable();
2407-
}
2408-
24092406
public UberClassLoader(List<PluginWrapper> activePlugins) {
2410-
super("UberClassLoader", PluginManager.class.getClassLoader());
2407+
super("UberClassLoader", PluginManager.class.getClassLoader(), MISSED_CLASS_CACHE_SIZE);
24112408
this.activePlugins = activePlugins;
24122409
}
24132410

24142411
@Override
2415-
protected Class<?> findClass(String name) throws ClassNotFoundException {
2416-
if (name.startsWith("SimpleTemplateScript")) { // cf. groovy.text.SimpleTemplateEngine
2417-
throw new ClassNotFoundException("ignoring " + name);
2412+
protected boolean isClassKnownAsMissed(String name) {
2413+
if (name.startsWith("SimpleTemplateScript") || // cf. groovy.text.SimpleTemplateEngine
2414+
name.startsWith("groovy.tmp.")) {
2415+
return true;
24182416
}
2419-
return loaded.computeIfAbsent(name, this::computeValue).orElseThrow(() -> new ClassNotFoundException(name));
2417+
2418+
return super.isClassKnownAsMissed(name);
24202419
}
24212420

2422-
private Optional<Class<?>> computeValue(String name) {
2421+
@Override
2422+
protected Class<?> doLoadClass(String name) {
2423+
try {
2424+
return getParent().loadClass(name);
2425+
} catch (ClassNotFoundException e) {
2426+
// Not found. Try the next class loader.
2427+
}
2428+
24232429
for (PluginWrapper p : activePlugins) {
24242430
try {
24252431
if (FAST_LOOKUP) {
2426-
return Optional.of(ClassLoaderReflectionToolkit.loadClass(p.classLoader, name));
2432+
return ClassLoaderReflectionToolkit.loadClass(p.classLoader, name);
24272433
} else {
2428-
return Optional.of(p.classLoader.loadClass(name));
2434+
return p.classLoader.loadClass(name);
24292435
}
24302436
} catch (ClassNotFoundException e) {
24312437
// Not found. Try the next class loader.
24322438
}
24332439
}
24342440
// Not found in any of the class loaders. Delegate.
2435-
return Optional.empty();
2441+
return null;
24362442
}
24372443

24382444
@Override
2439-
protected URL findResource(String name) {
2445+
public URL getResource(String name) {
2446+
URL url = getParent().getResource(name);
2447+
if (url != null) {
2448+
return url;
2449+
}
2450+
24402451
for (PluginWrapper p : activePlugins) {
2441-
URL url;
24422452
if (FAST_LOOKUP) {
24432453
url = ClassLoaderReflectionToolkit._findResource(p.classLoader, name);
24442454
} else {
@@ -2452,8 +2462,10 @@ protected URL findResource(String name) {
24522462
}
24532463

24542464
@Override
2455-
protected Enumeration<URL> findResources(String name) throws IOException {
2465+
public Enumeration<URL> getResources(String name) throws IOException {
24562466
List<URL> resources = new ArrayList<>();
2467+
resources.addAll(Collections.list(getParent().getResources(name)));
2468+
24572469
for (PluginWrapper p : activePlugins) {
24582470
if (FAST_LOOKUP) {
24592471
resources.addAll(Collections.list(ClassLoaderReflectionToolkit._findResources(p.classLoader, name)));
@@ -2464,10 +2476,6 @@ protected Enumeration<URL> findResources(String name) throws IOException {
24642476
return Collections.enumeration(resources);
24652477
}
24662478

2467-
void clearCacheMisses() {
2468-
loaded.values().removeIf(Optional::isEmpty);
2469-
}
2470-
24712479
@Override
24722480
public String toString() {
24732481
// only for debugging purpose

‎core/src/main/java/hudson/util/AbstractCachingClassLoader.java‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
package hudson.util;
2+
3+
import java.util.Collections;
4+
import java.util.LinkedHashMap;
5+
import java.util.Map;
6+
import java.util.concurrent.ConcurrentHashMap;
7+
8+
/**
9+
* {@link ClassLoader} that caches loadClass invocation results
10+
*
11+
* <p>
12+
* This class is used for constructing a facade class loader which use a lot of other class sources.
13+
* It helps to avoid looking up using all the class sources each time
14+
*
15+
* @author Dmytro Ukhlov
16+
*/
17+
public abstract class AbstractCachingClassLoader extends ClassLoader {
18+
private final ConcurrentHashMap<String, Class<?>> loadedClassMapping = new ConcurrentHashMap<>();
19+
20+
private final int missedClassCacheSize;
21+
22+
private final Map<String, String> missedClassMapping = Collections.synchronizedMap(new LinkedHashMap<>() {
23+
protected boolean removeEldestEntry(Map.Entry<String, String> eldest) {
24+
return size() > missedClassCacheSize;
25+
}
26+
});
27+
28+
/**
29+
* Initialize base implementation, bypass parameters to java.lang.ClassLoader base implementation
30+
*
31+
* @param name class loader name; or {@code null} if not named
32+
* @param parent the parent class loader
33+
*
34+
* @param missedClassCacheSize maximum number of missed classes to be remembered
35+
*
36+
*/
37+
protected AbstractCachingClassLoader(String name, ClassLoader parent, int missedClassCacheSize) {
38+
super(name, parent);
39+
this.missedClassCacheSize = missedClassCacheSize;
40+
}
41+
42+
public final void clearCacheMisses() {
43+
missedClassMapping.clear();
44+
}
45+
46+
@Override
47+
protected final Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
48+
if (isClassKnownAsMissed(name)) {
49+
throw new ClassNotFoundException(name);
50+
}
51+
52+
Class<?> clazz = loadedClassMapping.computeIfAbsent(name, key -> {
53+
if (getResource(name.replace('.', '/') + ".class") == null) {
54+
return null;
55+
}
56+
return doLoadClass(name);
57+
});
58+
59+
if (clazz == null) {
60+
missedClassMapping.put(name, name);
61+
throw new ClassNotFoundException(name);
62+
}
63+
64+
if (resolve) {
65+
resolveClass(clazz);
66+
}
67+
68+
return clazz;
69+
}
70+
71+
protected boolean isClassKnownAsMissed(String name) {
72+
return missedClassMapping.containsKey(name);
73+
}
74+
75+
protected abstract Class<?> doLoadClass(String name);
76+
}

‎core/src/main/java/hudson/util/MaskingClassLoader.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,6 @@ public class MaskingClassLoader extends ClassLoader {
5050

5151
private final List<String> masksResources;
5252

53-
static {
54-
registerAsParallelCapable();
55-
}
56-
5753
public MaskingClassLoader(ClassLoader parent, String... masks) {
5854
this(parent, Arrays.asList(masks));
5955
}
@@ -76,21 +72,26 @@ protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundE
7672
throw new ClassNotFoundException();
7773
}
7874

79-
return super.loadClass(name, resolve);
75+
Class<?> clazz = getParent().loadClass(name);
76+
77+
if (resolve) {
78+
resolveClass(clazz);
79+
}
80+
return clazz;
8081
}
8182

8283
@Override
8384
public URL getResource(String name) {
8485
if (isMasked(name)) return null;
8586

86-
return super.getResource(name);
87+
return getParent().getResource(name);
8788
}
8889

8990
@Override
9091
public Enumeration<URL> getResources(String name) throws IOException {
9192
if (isMasked(name)) return Collections.emptyEnumeration();
9293

93-
return super.getResources(name);
94+
return getParent().getResources(name);
9495
}
9596

9697
private boolean isMasked(String name) {

‎core/src/main/java/jenkins/util/URLClassLoader2.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,6 @@
1212
*/
1313
@Restricted(NoExternalUse.class)
1414
public class URLClassLoader2 extends URLClassLoader implements JenkinsClassLoader {
15-
16-
static {
17-
registerAsParallelCapable();
18-
}
19-
2015
/**
2116
* @deprecated use {@link URLClassLoader2#URLClassLoader2(String, URL[])}
2217
*/

0 commit comments

Comments
 (0)