Skip to content

Commit 0432761

Browse files
huangzhaobo99huangzhaobo99
authored
HDFS-17657. The balancer service supports httpserver. (#7242) Contribtued by Zhaobo Huang.
Reviewed-by: Tao Li <[email protected]> Signed-off-by: Shilun Fan <[email protected]>
1 parent 0d72896 commit 0432761

File tree

19 files changed

+702
-8
lines changed

19 files changed

+702
-8
lines changed

‎hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3HttpServer.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ void start() throws IOException {
5252
NfsConfigKeys.NFS_HTTPS_ADDRESS_DEFAULT);
5353
InetSocketAddress httpsAddr = NetUtils.createSocketAddr(httpsAddrString);
5454

55-
HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf,
55+
HttpServer2.Builder builder = DFSUtil.getHttpServerTemplate(conf,
5656
httpAddr, httpsAddr, "nfs3",
5757
NfsConfigKeys.DFS_NFS_KERBEROS_PRINCIPAL_KEY,
5858
NfsConfigKeys.DFS_NFS_KEYTAB_FILE_KEY);

‎hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ protected void serviceInit(Configuration configuration) throws Exception {
8282
protected void serviceStart() throws Exception {
8383
// Build and start server
8484
String webApp = "router";
85-
HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(
85+
HttpServer2.Builder builder = DFSUtil.getHttpServerTemplate(
8686
this.conf, this.httpAddress, this.httpsAddress, webApp,
8787
RBFConfigKeys.DFS_ROUTER_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY,
8888
RBFConfigKeys.DFS_ROUTER_KEYTAB_FILE_KEY);

‎hadoop-hdfs-project/hadoop-hdfs/pom.xml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -283,6 +283,9 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
283283
<copy file="${basedir}/src/main/webapps/proto-web.xml"
284284
tofile="${project.build.directory}/webapps/journal/WEB-INF/web.xml"
285285
filtering="true"/>
286+
<copy file="${basedir}/src/main/webapps/proto-web.xml"
287+
tofile="${project.build.directory}/webapps/balancer/WEB-INF/web.xml"
288+
filtering="true"/>
286289
<copy file="${basedir}/src/main/webapps/proto-web.xml"
287290
tofile="${project.build.directory}/webapps/nfs3/WEB-INF/web.xml"
288291
filtering="true"/>
@@ -428,6 +431,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
428431
<exclude>src/main/webapps/hdfs/robots.txt</exclude>
429432
<exclude>src/main/webapps/journal/robots.txt</exclude>
430433
<exclude>src/main/webapps/secondary/robots.txt</exclude>
434+
<exclude>src/main/webapps/balancer/robots.txt</exclude>
431435
<exclude>src/contrib/**</exclude>
432436
<exclude>src/site/resources/images/*</exclude>
433437
<exclude>src/main/webapps/static/bootstrap-3.4.1/**</exclude>

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -811,6 +811,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
811811
public static final String DFS_BALANCER_ADDRESS_DEFAULT= "0.0.0.0:0";
812812
public static final String DFS_BALANCER_KEYTAB_FILE_KEY = "dfs.balancer.keytab.file";
813813
public static final String DFS_BALANCER_KERBEROS_PRINCIPAL_KEY = "dfs.balancer.kerberos.principal";
814+
public static final String DFS_BALANCER_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY =
815+
"dfs.balancer.kerberos.internal.spnego.principal";
814816
public static final String DFS_BALANCER_BLOCK_MOVE_TIMEOUT = "dfs.balancer.block-move.timeout";
815817
public static final int DFS_BALANCER_BLOCK_MOVE_TIMEOUT_DEFAULT = 0;
816818
public static final String DFS_BALANCER_MAX_NO_MOVE_INTERVAL_KEY = "dfs.balancer.max-no-move-interval";
@@ -821,6 +823,19 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
821823
public static final long DFS_BALANCER_SERVICE_INTERVAL_DEFAULT = TimeUnit.MINUTES.toMillis(5); //5 mins
822824
public static final String DFS_BALANCER_SERVICE_RETRIES_ON_EXCEPTION = "dfs.balancer.service.retries.on.exception";
823825
public static final int DFS_BALANCER_SERVICE_RETRIES_ON_EXCEPTION_DEFAULT = 5;
826+
public static final String DFS_BALANCER_HTTPSERVER_ENABLED_KEY =
827+
"dfs.balancer.httpserver.enabled";
828+
public static final Boolean DFS_BALANCER_HTTPSERVER_ENABLED_DEFAULT = false;
829+
public static final String DFS_BALANCER_HTTP_ADDRESS_KEY = "dfs.balancer.http-address";
830+
public static final int DFS_BALANCER_HTTP_PORT_DEFAULT = 8590;
831+
public static final String DFS_BALANCER_HTTP_BIND_HOST_KEY = "dfs.balancer.http-bind-host";
832+
public static final String DFS_BALANCER_HTTP_ADDRESS_DEFAULT =
833+
"0.0.0.0:" + DFS_BALANCER_HTTP_PORT_DEFAULT;
834+
public static final String DFS_BALANCER_HTTPS_ADDRESS_KEY = "dfs.balancer.https-address";
835+
public static final int DFS_BALANCER_HTTPS_PORT_DEFAULT = 8591;
836+
public static final String DFS_BALANCER_HTTPS_BIND_HOST_KEY = "dfs.balancer.https-bind-host";
837+
public static final String DFS_BALANCER_HTTPS_ADDRESS_DEFAULT =
838+
"0.0.0.0:" + DFS_BALANCER_HTTPS_PORT_DEFAULT;
824839

825840
public static final String DFS_MOVER_MOVEDWINWIDTH_KEY = "dfs.mover.movedWinWidth";
826841
public static final long DFS_MOVER_MOVEDWINWIDTH_DEFAULT = 5400*1000L;

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1736,11 +1736,11 @@ public static Configuration loadSslConfiguration(Configuration conf) {
17361736
}
17371737

17381738
/**
1739-
* Return a HttpServer.Builder that the journalnode / namenode / secondary
1739+
* Return a HttpServer.Builder that the journalnode / namenode / secondary / router / balancer
17401740
* namenode can use to initialize their HTTP / HTTPS server.
17411741
*
17421742
*/
1743-
public static HttpServer2.Builder httpServerTemplateForNNAndJN(
1743+
public static HttpServer2.Builder getHttpServerTemplate(
17441744
Configuration conf, final InetSocketAddress httpAddr,
17451745
final InetSocketAddress httpsAddr, String name, String spnegoUserNameKey,
17461746
String spnegoKeytabFileKey) throws IOException {

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ void start() throws IOException {
7373
}
7474
}
7575

76-
HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf,
76+
HttpServer2.Builder builder = DFSUtil.getHttpServerTemplate(conf,
7777
httpAddr, httpsAddr, "journal",
7878
DFSConfigKeys.DFS_JOURNALNODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY,
7979
DFSConfigKeys.DFS_JOURNALNODE_KEYTAB_FILE_KEY);

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java

Lines changed: 54 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@
3838
import java.util.concurrent.TimeUnit;
3939
import java.util.concurrent.atomic.AtomicInteger;
4040

41+
import javax.management.ObjectName;
42+
4143
import org.apache.commons.lang3.builder.ToStringBuilder;
4244
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
4345
import org.apache.hadoop.metrics2.source.JvmMetrics;
@@ -66,6 +68,7 @@
6668
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
6769
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
6870
import org.apache.hadoop.io.IOUtils;
71+
import org.apache.hadoop.metrics2.util.MBeans;
6972
import org.apache.hadoop.net.NetUtils;
7073
import org.apache.hadoop.security.SecurityUtil;
7174
import org.apache.hadoop.security.UserGroupInformation;
@@ -76,6 +79,7 @@
7679
import org.apache.hadoop.util.ToolRunner;
7780

7881
import org.apache.hadoop.util.Preconditions;
82+
import org.apache.hadoop.util.VersionInfo;
7983

8084
/** <p>The balancer is a tool that balances disk space usage on an HDFS cluster
8185
* when some datanodes become full or when new empty nodes join the cluster.
@@ -180,7 +184,7 @@
180184
*/
181185

182186
@InterfaceAudience.Private
183-
public class Balancer {
187+
public class Balancer implements BalancerMXBean {
184188
static final Logger LOG = LoggerFactory.getLogger(Balancer.class);
185189

186190
static final Path BALANCER_ID_PATH = new Path("/system/balancer.id");
@@ -241,6 +245,7 @@ public class Balancer {
241245
private final boolean sortTopNodes;
242246
private final int limitOverUtilizedNum;
243247
private final BalancerMetrics metrics;
248+
private ObjectName balancerInfoBeanName;
244249

245250
// all data node lists
246251
private final Collection<Source> overUtilized = new LinkedList<Source>();
@@ -377,6 +382,8 @@ static int getFailedTimesSinceLastSuccessfulBalance() {
377382
DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
378383
DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
379384
this.metrics = BalancerMetrics.create(this);
385+
386+
registerBalancerMXBean();
380387
}
381388

382389
private static long getCapacity(DatanodeStorageReport report, StorageType t) {
@@ -680,6 +687,13 @@ private boolean matchStorageGroups(StorageGroup left, StorageGroup right,
680687
left.getDatanodeInfo(), right.getDatanodeInfo());
681688
}
682689

690+
/**
691+
* Register BalancerMXBean.
692+
*/
693+
private void registerBalancerMXBean() {
694+
balancerInfoBeanName = MBeans.register("Balancer", "BalancerInfo", this);
695+
}
696+
683697
/* reset all fields in a balancer preparing for the next iteration */
684698
void resetData(Configuration conf) {
685699
this.overUtilized.clear();
@@ -689,12 +703,32 @@ void resetData(Configuration conf) {
689703
this.policy.reset();
690704
this.dispatcher.reset(conf);
691705
DefaultMetricsSystem.removeSourceName(metrics.getName());
706+
if (balancerInfoBeanName != null) {
707+
MBeans.unregister(balancerInfoBeanName);
708+
balancerInfoBeanName = null;
709+
}
692710
}
693711

694712
NameNodeConnector getNnc() {
695713
return nnc;
696714
}
697715

716+
@Override
717+
public String getVersion() {
718+
return VersionInfo.getVersion() + ", r" + VersionInfo.getRevision();
719+
}
720+
721+
@Override
722+
public String getSoftwareVersion() {
723+
return VersionInfo.getVersion();
724+
}
725+
726+
@Override
727+
public String getCompileInfo() {
728+
return VersionInfo.getDate() + " by " + VersionInfo.getUser() + " from "
729+
+ VersionInfo.getBranch();
730+
}
731+
698732
static class Result {
699733
private final ExitStatus exitStatus;
700734
private final long bytesLeftToMove;
@@ -860,6 +894,7 @@ static private int doBalance(Collection<URI> namenodes,
860894
+ " NameNode");
861895

862896
List<NameNodeConnector> connectors = Collections.emptyList();
897+
BalancerHttpServer balancerHttpServer = startBalancerHttpServer(conf);
863898
try {
864899
connectors = NameNodeConnector.newNameNodeConnectors(namenodes, nsIds,
865900
Balancer.class.getSimpleName(), BALANCER_ID_PATH, conf,
@@ -872,6 +907,9 @@ static private int doBalance(Collection<URI> namenodes,
872907
if (p.getBlockPools().size() == 0
873908
|| p.getBlockPools().contains(nnc.getBlockpoolID())) {
874909
final Balancer b = new Balancer(nnc, p, conf);
910+
if (balancerHttpServer != null) {
911+
balancerHttpServer.setBalancerAttribute(b);
912+
}
875913
final Result r = b.runOneIteration();
876914
r.print(iteration, nnc, System.out);
877915

@@ -898,6 +936,9 @@ static private int doBalance(Collection<URI> namenodes,
898936
for(NameNodeConnector nnc : connectors) {
899937
IOUtils.cleanupWithLogger(LOG, nnc);
900938
}
939+
if (balancerHttpServer != null) {
940+
balancerHttpServer.stop();
941+
}
901942
}
902943
return ExitStatus.SUCCESS.getExitCode();
903944
}
@@ -969,6 +1010,18 @@ static void stop() {
9691010
serviceRunning = false;
9701011
}
9711012

1013+
private static BalancerHttpServer startBalancerHttpServer(Configuration conf) throws IOException {
1014+
boolean httpServerEnabled = conf.getBoolean(DFSConfigKeys.DFS_BALANCER_HTTPSERVER_ENABLED_KEY,
1015+
DFSConfigKeys.DFS_BALANCER_HTTPSERVER_ENABLED_DEFAULT);
1016+
if (httpServerEnabled) {
1017+
BalancerHttpServer balancerHttpServer = new BalancerHttpServer(conf);
1018+
balancerHttpServer.start();
1019+
return balancerHttpServer;
1020+
} else {
1021+
return null;
1022+
}
1023+
}
1024+
9721025
private static void checkKeytabAndInit(Configuration conf)
9731026
throws IOException {
9741027
if (conf.getBoolean(DFSConfigKeys.DFS_BALANCER_KEYTAB_ENABLED_KEY,

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/BalancerHttpServer.java

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.hdfs.server.balancer;
19+
20+
import java.io.IOException;
21+
import java.net.InetSocketAddress;
22+
23+
import org.apache.hadoop.conf.Configuration;
24+
import org.apache.hadoop.hdfs.DFSConfigKeys;
25+
import org.apache.hadoop.hdfs.DFSUtil;
26+
import org.apache.hadoop.hdfs.server.common.JspHelper;
27+
import org.apache.hadoop.http.HttpConfig;
28+
import org.apache.hadoop.http.HttpServer2;
29+
import org.apache.hadoop.net.NetUtils;
30+
31+
public class BalancerHttpServer {
32+
33+
private static final String BALANCER_ATTRIBUTE_KEY = "current.balancer";
34+
35+
private final Configuration conf;
36+
private InetSocketAddress httpAddress;
37+
private InetSocketAddress httpsAddress;
38+
private HttpServer2 httpServer;
39+
40+
public BalancerHttpServer(Configuration conf) {
41+
this.conf = conf;
42+
}
43+
44+
public void start() throws IOException {
45+
String webApp = "balancer";
46+
// Get HTTP address
47+
httpAddress = conf.getSocketAddr(DFSConfigKeys.DFS_BALANCER_HTTP_BIND_HOST_KEY,
48+
DFSConfigKeys.DFS_BALANCER_HTTP_ADDRESS_KEY,
49+
DFSConfigKeys.DFS_BALANCER_HTTP_ADDRESS_DEFAULT,
50+
DFSConfigKeys.DFS_BALANCER_HTTP_PORT_DEFAULT);
51+
52+
// Get HTTPs address
53+
httpsAddress = conf.getSocketAddr(DFSConfigKeys.DFS_BALANCER_HTTPS_BIND_HOST_KEY,
54+
DFSConfigKeys.DFS_BALANCER_HTTPS_ADDRESS_KEY,
55+
DFSConfigKeys.DFS_BALANCER_HTTPS_ADDRESS_DEFAULT,
56+
DFSConfigKeys.DFS_BALANCER_HTTPS_PORT_DEFAULT);
57+
58+
HttpServer2.Builder builder =
59+
DFSUtil.getHttpServerTemplate(conf, httpAddress, httpsAddress, webApp,
60+
DFSConfigKeys.DFS_BALANCER_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY,
61+
DFSConfigKeys.DFS_BALANCER_KEYTAB_FILE_KEY);
62+
63+
final boolean xFrameEnabled = conf.getBoolean(DFSConfigKeys.DFS_XFRAME_OPTION_ENABLED,
64+
DFSConfigKeys.DFS_XFRAME_OPTION_ENABLED_DEFAULT);
65+
66+
final String xFrameOptionValue = conf.getTrimmed(DFSConfigKeys.DFS_XFRAME_OPTION_VALUE,
67+
DFSConfigKeys.DFS_XFRAME_OPTION_VALUE_DEFAULT);
68+
69+
builder.configureXFrame(xFrameEnabled).setXFrameOption(xFrameOptionValue);
70+
71+
httpServer = builder.build();
72+
httpServer.setAttribute(JspHelper.CURRENT_CONF, conf);
73+
httpServer.start();
74+
75+
HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf);
76+
int connIdx = 0;
77+
if (policy.isHttpEnabled()) {
78+
httpAddress = httpServer.getConnectorAddress(connIdx++);
79+
if (httpAddress != null) {
80+
conf.set(DFSConfigKeys.DFS_BALANCER_HTTP_ADDRESS_KEY,
81+
NetUtils.getHostPortString(httpAddress));
82+
}
83+
}
84+
if (policy.isHttpsEnabled()) {
85+
httpsAddress = httpServer.getConnectorAddress(connIdx);
86+
if (httpsAddress != null) {
87+
conf.set(DFSConfigKeys.DFS_BALANCER_HTTPS_ADDRESS_KEY,
88+
NetUtils.getHostPortString(httpsAddress));
89+
}
90+
}
91+
}
92+
93+
public void setBalancerAttribute(Balancer balancer) {
94+
httpServer.setAttribute(BALANCER_ATTRIBUTE_KEY, balancer);
95+
}
96+
97+
public void stop() throws IOException {
98+
if (httpServer != null) {
99+
try {
100+
httpServer.stop();
101+
} catch (Exception e) {
102+
throw new IOException(e);
103+
}
104+
}
105+
}
106+
107+
public InetSocketAddress getHttpAddress() {
108+
return httpAddress;
109+
}
110+
111+
public InetSocketAddress getHttpsAddress() {
112+
return httpsAddress;
113+
}
114+
}

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/BalancerMXBean.java

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.hdfs.server.balancer;
19+
20+
public interface BalancerMXBean {
21+
22+
/**
23+
* Gets the version of Hadoop.
24+
*
25+
* @return the version of Hadoop
26+
*/
27+
String getVersion();
28+
29+
/**
30+
* Get the version of software running on the Balancer.
31+
*
32+
* @return a string representing the version.
33+
*/
34+
String getSoftwareVersion();
35+
36+
/**
37+
* Get the compilation information which contains date, user and branch.
38+
*
39+
* @return the compilation information, as a JSON string.
40+
*/
41+
String getCompileInfo();
42+
43+
}

0 commit comments

Comments
 (0)