Skip to content

Commit c87caa5

Browse files
adoroszlaiadoroszlai
authored
HDDS-12226. TestSecureOzoneRpcClient tests not run due to UnknownHostException (apache#7827)
1 parent 1513c34 commit c87caa5

File tree

10 files changed

+136
-104
lines changed

10 files changed

+136
-104
lines changed

‎hadoop-hdds/test-utils/src/main/java/org/apache/ozone/test/GenericTestUtils.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
import org.apache.log4j.Logger;
5151
import org.junit.jupiter.api.Assertions;
5252
import org.mockito.Mockito;
53+
import org.slf4j.LoggerFactory;
5354

5455
/**
5556
* Provides some very generic helpers which might be used across the tests.
@@ -179,6 +180,18 @@ public static void setLogLevel(org.slf4j.Logger logger,
179180
setLogLevel(toLog4j(logger), Level.toLevel(level.toString()));
180181
}
181182

183+
public static void withLogDisabled(Class<?> clazz, Runnable task) {
184+
org.slf4j.Logger logger = LoggerFactory.getLogger(clazz);
185+
final Logger log4j = toLog4j(logger);
186+
final Level level = log4j.getLevel();
187+
setLogLevel(log4j, Level.OFF);
188+
try {
189+
task.run();
190+
} finally {
191+
setLogLevel(log4j, level);
192+
}
193+
}
194+
182195
public static <T> T mockFieldReflection(Object object, String fieldName)
183196
throws NoSuchFieldException, IllegalAccessException {
184197
Field field = object.getClass().getDeclaredField(fieldName);

‎hadoop-ozone/dev-support/checks/_mvn_unit_report.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,15 @@ if [[ "${CHECK:-unit}" == "integration" ]]; then
4545
cat ${leaks} >> "${tempfile}"
4646
fi
4747

48+
cluster=${REPORT_DIR}/cluster-startup-errors.txt
49+
if [[ "${CHECK:-unit}" == "integration" ]]; then
50+
find hadoop-ozone/integration-test -not -path '*/iteration*' -name '*-output.txt' -print0 \
51+
| xargs -n1 -0 "grep" -l -E "Unable to build MiniOzoneCluster" \
52+
| awk -F/ '{sub("-output.txt",""); print $NF}' \
53+
> "${cluster}"
54+
cat ${cluster} >> "${tempfile}"
55+
fi
56+
4857
#Copy heap dump and dump leftovers
4958
find "." -not -path '*/iteration*' \
5059
\( -name "*.hprof" \
@@ -114,6 +123,12 @@ if [[ -s "${leaks}" ]]; then
114123
fi
115124
rm -f "${leaks}"
116125

126+
if [[ -s "${cluster}" ]]; then
127+
printf "# Cluster Startup Errors\n\n" >> "$SUMMARY_FILE"
128+
cat "${cluster}" | sed 's/^/ * /' >> "$SUMMARY_FILE"
129+
fi
130+
rm -f "${cluster}"
131+
117132
if [[ -s "${crashes}" ]]; then
118133
printf "# Crashed Tests\n\n" >> "$SUMMARY_FILE"
119134
cat "${crashes}" | sed 's/^/ * /' >> "$SUMMARY_FILE"

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestWatchForCommit.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -347,7 +347,6 @@ public void test2WayCommitForTimeoutException(RaftProtos.ReplicationLevel watchT
347347
assertEquals(2, ratisClient.getCommitInfoMap().size());
348348
String output = logCapturer.getOutput();
349349
assertThat(output).contains("ALL_COMMITTED way commit failed");
350-
assertThat(output).contains("TimeoutException");
351350
assertThat(output).contains("Committed by majority");
352351
} else {
353352
assertEquals(3, ratisClient.getCommitInfoMap().size());

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -662,6 +662,8 @@ public MiniOzoneCluster build() throws IOException {
662662
stopSCM(scm);
663663
removeConfiguration();
664664

665+
LOG.warn("Unable to build MiniOzoneCluster", ex);
666+
665667
if (ex instanceof IOException) {
666668
throw (IOException) ex;
667669
}

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,13 +108,15 @@
108108
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
109109
import org.apache.ozone.test.GenericTestUtils;
110110
import org.apache.ozone.test.tag.Flaky;
111+
import org.apache.ozone.test.tag.Unhealthy;
111112
import org.junit.jupiter.api.AfterAll;
112113
import org.junit.jupiter.api.BeforeAll;
113114
import org.junit.jupiter.api.Test;
114115
import org.junit.jupiter.api.io.TempDir;
115116
import org.junit.jupiter.params.ParameterizedTest;
116117
import org.junit.jupiter.params.provider.EnumSource;
117118

119+
@Unhealthy("HDDS-11879")
118120
class TestOzoneAtRestEncryption {
119121

120122
private static MiniOzoneCluster cluster = null;

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java‎

Lines changed: 65 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,6 @@
6666
import org.apache.hadoop.ozone.client.SecretKeyTestClient;
6767
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
6868
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
69-
import org.apache.hadoop.ozone.om.OMConfigKeys;
7069
import org.apache.hadoop.ozone.om.OMMetadataManager;
7170
import org.apache.hadoop.ozone.om.OMMetrics;
7271
import org.apache.hadoop.ozone.om.OzoneManager;
@@ -100,7 +99,7 @@ class TestSecureOzoneRpcClient extends OzoneRpcClientTests {
10099

101100
@TempDir
102101
private static File testDir;
103-
private static String keyProviderUri = "kms://http@kms:9600/kms";
102+
private static String keyProviderUri = "kms://http@localhost:9600/kms";
104103

105104
@BeforeAll
106105
public static void init() throws Exception {
@@ -115,11 +114,6 @@ public static void init() throws Exception {
115114
OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS_NATIVE);
116115
CertificateClientTestImpl certificateClientTest =
117116
new CertificateClientTestImpl(conf);
118-
// These tests manually insert keys into RocksDB. This is easier to do
119-
// with object store layout so keys with path IDs do not need to be
120-
// constructed.
121-
conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
122-
OMConfigKeys.OZONE_BUCKET_LAYOUT_OBJECT_STORE);
123117
conf.setBoolean(OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED, true);
124118
conf.setBoolean("ozone.client.hbase.enhancements.allowed", true);
125119
conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true);
@@ -236,26 +230,27 @@ public void testFileRecovery(boolean forceRecovery) throws Exception {
236230
// force recovery file
237231
System.setProperty(FORCE_LEASE_RECOVERY_ENV, String.valueOf(forceRecovery));
238232
conf.setBoolean(String.format("fs.%s.impl.disable.cache", OZONE_OFS_URI_SCHEME), true);
239-
RootedOzoneFileSystem fs = (RootedOzoneFileSystem) FileSystem.get(conf);
240-
OzoneOutputStream out = null;
241-
try {
242-
out = bucket.createKey(keyName, value.getBytes(UTF_8).length, ReplicationType.RATIS,
243-
ReplicationFactor.THREE, new HashMap<>());
244-
out.write(value.getBytes(UTF_8));
245-
out.hsync();
246-
247-
if (forceRecovery) {
248-
fs.recoverLease(file);
249-
} else {
250-
assertThrows(OMException.class, () -> fs.recoverLease(file));
251-
}
252-
} finally {
253-
if (out != null) {
233+
try (RootedOzoneFileSystem fs = (RootedOzoneFileSystem) FileSystem.get(conf)) {
234+
OzoneOutputStream out = null;
235+
try {
236+
out = bucket.createKey(keyName, value.getBytes(UTF_8).length, ReplicationType.RATIS,
237+
ReplicationFactor.THREE, new HashMap<>());
238+
out.write(value.getBytes(UTF_8));
239+
out.hsync();
240+
254241
if (forceRecovery) {
255-
// close failure because the key is already committed
256-
assertThrows(OMException.class, out::close);
242+
fs.recoverLease(file);
257243
} else {
258-
out.close();
244+
assertThrows(OMException.class, () -> fs.recoverLease(file));
245+
}
246+
} finally {
247+
if (out != null) {
248+
if (forceRecovery) {
249+
// close failure because the key is already committed
250+
assertThrows(OMException.class, out::close);
251+
} else {
252+
out.close();
253+
}
259254
}
260255
}
261256
}
@@ -288,51 +283,52 @@ public void testPreallocateFileRecovery(long dataSize) throws Exception {
288283
final String rootPath = String.format("%s://%s/",
289284
OZONE_OFS_URI_SCHEME, conf.get(OZONE_OM_ADDRESS_KEY));
290285
conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
291-
RootedOzoneFileSystem fs = (RootedOzoneFileSystem) FileSystem.get(conf);
292-
OzoneOutputStream out = null;
293-
long totalBlock = 10;
294-
long usedBlock = (dataSize - 1) / fs.getDefaultBlockSize() + 1;
295-
long fileSize = fs.getDefaultBlockSize() * totalBlock;
296-
OMMetrics metrics = getCluster().getOzoneManager().getMetrics();
297-
long committedBytes = metrics.getDataCommittedBytes();
298-
try {
299-
out = bucket.createKey(keyName, fileSize, ReplicationType.RATIS,
300-
ReplicationFactor.THREE, new HashMap<>());
301-
// init used quota check
302-
bucket = volume.getBucket(bucketName);
303-
assertEquals(0, bucket.getUsedNamespace());
304-
assertEquals(0, bucket.getUsedBytes());
305-
306-
out.write(data);
307-
out.hsync();
308-
fs.recoverLease(file);
309-
310-
// check file length
311-
FileStatus fileStatus = fs.getFileStatus(file);
312-
assertEquals(dataSize, fileStatus.getLen());
313-
// check committed bytes
314-
assertEquals(committedBytes + dataSize,
315-
getCluster().getOzoneManager().getMetrics().getDataCommittedBytes());
316-
// check used quota
317-
bucket = volume.getBucket(bucketName);
318-
assertEquals(1, bucket.getUsedNamespace());
319-
assertEquals(dataSize * ReplicationFactor.THREE.getValue(), bucket.getUsedBytes());
320-
321-
// check unused pre-allocated blocks are reclaimed
322-
Table<String, RepeatedOmKeyInfo> deletedTable =
323-
getCluster().getOzoneManager().getMetadataManager().getDeletedTable();
324-
try (TableIterator<String, ? extends Table.KeyValue<String, RepeatedOmKeyInfo>>
325-
keyIter = deletedTable.iterator()) {
326-
while (keyIter.hasNext()) {
327-
Table.KeyValue<String, RepeatedOmKeyInfo> kv = keyIter.next();
328-
OmKeyInfo key = kv.getValue().getOmKeyInfoList().get(0);
329-
assertEquals(totalBlock - usedBlock, key.getKeyLocationVersions().get(0).getLocationListCount());
286+
try (RootedOzoneFileSystem fs = (RootedOzoneFileSystem) FileSystem.get(conf)) {
287+
OzoneOutputStream out = null;
288+
long totalBlock = 10;
289+
long usedBlock = (dataSize - 1) / fs.getDefaultBlockSize() + 1;
290+
long fileSize = fs.getDefaultBlockSize() * totalBlock;
291+
OMMetrics metrics = getCluster().getOzoneManager().getMetrics();
292+
long committedBytes = metrics.getDataCommittedBytes();
293+
try {
294+
out = bucket.createKey(keyName, fileSize, ReplicationType.RATIS,
295+
ReplicationFactor.THREE, new HashMap<>());
296+
// init used quota check
297+
bucket = volume.getBucket(bucketName);
298+
assertEquals(0, bucket.getUsedNamespace());
299+
assertEquals(0, bucket.getUsedBytes());
300+
301+
out.write(data);
302+
out.hsync();
303+
fs.recoverLease(file);
304+
305+
// check file length
306+
FileStatus fileStatus = fs.getFileStatus(file);
307+
assertEquals(dataSize, fileStatus.getLen());
308+
// check committed bytes
309+
assertEquals(committedBytes + dataSize,
310+
getCluster().getOzoneManager().getMetrics().getDataCommittedBytes());
311+
// check used quota
312+
bucket = volume.getBucket(bucketName);
313+
assertEquals(1, bucket.getUsedNamespace());
314+
assertEquals(dataSize * ReplicationFactor.THREE.getValue(), bucket.getUsedBytes());
315+
316+
// check unused pre-allocated blocks are reclaimed
317+
Table<String, RepeatedOmKeyInfo> deletedTable =
318+
getCluster().getOzoneManager().getMetadataManager().getDeletedTable();
319+
try (TableIterator<String, ? extends Table.KeyValue<String, RepeatedOmKeyInfo>>
320+
keyIter = deletedTable.iterator()) {
321+
while (keyIter.hasNext()) {
322+
Table.KeyValue<String, RepeatedOmKeyInfo> kv = keyIter.next();
323+
OmKeyInfo key = kv.getValue().getOmKeyInfoList().get(0);
324+
assertEquals(totalBlock - usedBlock, key.getKeyLocationVersions().get(0).getLocationListCount());
325+
}
326+
}
327+
} finally {
328+
if (out != null) {
329+
// close failure because the key is already committed
330+
assertThrows(OMException.class, out::close);
330331
}
331-
}
332-
} finally {
333-
if (out != null) {
334-
// close failure because the key is already committed
335-
assertThrows(OMException.class, out::close);
336332
}
337333
}
338334
}

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmStartupSlvLessThanMlv.java‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,12 @@
2727
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
2828
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
2929
import org.apache.hadoop.ozone.MiniOzoneCluster;
30+
import org.apache.hadoop.ozone.MiniOzoneClusterImpl;
3031
import org.apache.hadoop.ozone.om.exceptions.OMException;
3132
import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
3233
import org.apache.hadoop.ozone.upgrade.LayoutFeature;
3334
import org.apache.hadoop.ozone.upgrade.UpgradeTestUtils;
35+
import org.apache.ozone.test.GenericTestUtils;
3436
import org.junit.jupiter.api.Test;
3537
import org.junit.jupiter.api.io.TempDir;
3638

@@ -66,11 +68,13 @@ public void testStartupSlvLessThanMlv() throws Exception {
6668

6769
MiniOzoneCluster.Builder clusterBuilder = MiniOzoneCluster.newBuilder(conf);
6870

69-
OMException omException = assertThrows(OMException.class,
70-
clusterBuilder::build);
71-
String expectedMessage = String.format("Cannot initialize " +
72-
"VersionManager. Metadata layout version (%s) > software layout" +
73-
" version (%s)", mlv, largestSlv);
74-
assertEquals(expectedMessage, omException.getMessage());
71+
GenericTestUtils.withLogDisabled(MiniOzoneClusterImpl.class, () -> {
72+
OMException omException = assertThrows(OMException.class,
73+
clusterBuilder::build);
74+
String expectedMessage = String.format("Cannot initialize " +
75+
"VersionManager. Metadata layout version (%s) > software layout" +
76+
" version (%s)", mlv, mlv - 1);
77+
assertEquals(expectedMessage, omException.getMessage());
78+
});
7579
}
7680
}

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerConfiguration.java‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,13 @@
3535
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
3636
import org.apache.hadoop.net.NetUtils;
3737
import org.apache.hadoop.ozone.MiniOzoneCluster;
38+
import org.apache.hadoop.ozone.MiniOzoneClusterImpl;
3839
import org.apache.hadoop.ozone.OzoneConsts;
3940
import org.apache.hadoop.ozone.OzoneIllegalArgumentException;
4041
import org.apache.hadoop.ozone.ha.ConfUtils;
4142
import org.apache.hadoop.ozone.om.helpers.OMNodeDetails;
4243
import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer;
44+
import org.apache.ozone.test.GenericTestUtils;
4345
import org.apache.ratis.protocol.RaftPeer;
4446
import org.apache.ratis.util.LifeCycle;
4547
import org.junit.jupiter.api.AfterEach;
@@ -342,34 +344,35 @@ public void testWrongConfiguration() {
342344
conf.set(omNode2RpcAddrKey, "125.0.0.2:9862");
343345
conf.set(omNode3RpcAddrKey, "124.0.0.124:9862");
344346

345-
OzoneIllegalArgumentException exception = assertThrows(OzoneIllegalArgumentException.class, this::startCluster);
346-
assertThat(exception).hasMessage(
347-
"Configuration has no " + OZONE_OM_ADDRESS_KEY + " address that matches local node's address.");
347+
GenericTestUtils.withLogDisabled(MiniOzoneClusterImpl.class, () -> {
348+
Exception exception = assertThrows(OzoneIllegalArgumentException.class, this::startCluster);
349+
assertThat(exception).hasMessage(
350+
"Configuration has no " + OZONE_OM_ADDRESS_KEY + " address that matches local node's address.");
351+
});
348352
}
349353

350354
/**
351355
* A configuration with an empty node list while service ID is configured.
352356
* Cluster should fail to start during config check.
353-
* @throws Exception
354357
*/
355358
@Test
356-
public void testNoOMNodes() throws Exception {
359+
public void testNoOMNodes() {
357360
String omServiceId = "service1";
358361
conf.set(OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY, omServiceId);
359362
// Deliberately skip OZONE_OM_NODES_KEY and OZONE_OM_ADDRESS_KEY config
360-
OzoneIllegalArgumentException e =
361-
assertThrows(OzoneIllegalArgumentException.class, () -> startCluster());
362-
// Expect error message
363-
assertTrue(e.getMessage().contains("List of OM Node ID's should be specified"));
363+
GenericTestUtils.withLogDisabled(MiniOzoneClusterImpl.class, () -> {
364+
Exception e = assertThrows(OzoneIllegalArgumentException.class, this::startCluster);
365+
// Expect error message
366+
assertThat(e).hasMessageContaining("List of OM Node ID's should be specified");
367+
});
364368
}
365369

366370
/**
367371
* A configuration with no OM addresses while service ID is configured.
368372
* Cluster should fail to start during config check.
369-
* @throws Exception
370373
*/
371374
@Test
372-
public void testNoOMAddrs() throws Exception {
375+
public void testNoOMAddrs() {
373376
String omServiceId = "service1";
374377

375378
String omNode1Id = "omNode1";
@@ -382,9 +385,11 @@ public void testNoOMAddrs() throws Exception {
382385
conf.set(OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY, omServiceId);
383386
conf.set(omNodesKey, omNodesKeyValue);
384387
// Deliberately skip OZONE_OM_ADDRESS_KEY config
385-
OzoneIllegalArgumentException e = assertThrows(OzoneIllegalArgumentException.class, () -> startCluster());
386-
// Expect error message
387-
assertTrue(e.getMessage().contains("OM RPC Address should be set for all node"));
388+
GenericTestUtils.withLogDisabled(MiniOzoneClusterImpl.class, () -> {
389+
Exception e = assertThrows(OzoneIllegalArgumentException.class, this::startCluster);
390+
// Expect error message
391+
assertThat(e).hasMessageContaining("OM RPC Address should be set for all node");
392+
});
388393
}
389394

390395
/**

0 commit comments

Comments
 (0)