diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java index 19729a4bde8..90f2470d94a 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java @@ -41,6 +41,7 @@ import org.apache.bookkeeper.proto.BookieAddressResolver; import org.apache.bookkeeper.stats.StatsLogger; import org.apache.bookkeeper.util.BookKeeperConstants; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -360,8 +361,12 @@ public PlacementResult> newEnsemble(int ensembleSize, int writeQu effectiveMinRegionsForDurability, minNumRacksPerWriteQuorum); TopologyAwareEnsemblePlacementPolicy nextPolicy = perRegionPlacement.get( availableRegions.iterator().next()); - return nextPolicy.newEnsemble(ensembleSize, writeQuorumSize, writeQuorumSize, + + PlacementResult> placementResult = nextPolicy.newEnsemble(ensembleSize, writeQuorumSize, writeQuorumSize, comprehensiveExclusionBookiesSet, ensemble, ensemble); + return PlacementResult.of(placementResult.getResult(), + isEnsembleAdheringToPlacementPolicy( + placementResult.getResult(), writeQuorumSize, ackQuorumSize)); } int remainingEnsemble = ensembleSize; @@ -644,12 +649,73 @@ public final DistributionSchedule.WriteSet reorderReadLACSequence( @Override public PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy(List ensembleList, int writeQuorumSize, int ackQuorumSize) { - /** - * TODO: have to implement actual logic for this method for - * RegionAwareEnsemblePlacementPolicy. For now return true value. - * - * - https://github.com/apache/bookkeeper/issues/1898 - */ - return PlacementPolicyAdherence.MEETS_STRICT; + if (CollectionUtils.isEmpty(ensembleList)) { + return PlacementPolicyAdherence.FAIL; + } + + int effectiveMinRegionsForDurability = disableDurabilityFeature.isAvailable() ? 1 : minRegionsForDurability; + PlacementPolicyAdherence placementPolicyAdherence = PlacementPolicyAdherence.MEETS_STRICT; + + int ensembleSize = ensembleList.size(); + // region -> bookie set + Map> regionsInQuorum = new HashMap<>(); + BookieId bookie; + for (int i = 0; i < ensembleList.size(); i++) { + regionsInQuorum.clear(); + for (int j = 0; j < writeQuorumSize; j++) { + bookie = ensembleList.get((i + j) % ensembleSize); + BookieNode bookieNode = knownBookies.get(bookie); + if (bookieNode != null) { + String region = getLocalRegion(bookieNode); + if (regionsInQuorum.containsKey(region)) { + regionsInQuorum.get(region).add(bookie); + } else { + Set bookieSet = new HashSet<>(); + bookieSet.add(bookie); + regionsInQuorum.put(region, bookieSet); + } + } else if (LOG.isDebugEnabled()) { + LOG.debug("bookie {} is not in the list of knownBookies", bookie); + } + } + + if (regionsInQuorum.size() < effectiveMinRegionsForDurability) { + if (LOG.isDebugEnabled()) { + LOG.debug("For ensemble {}, write set starting at {} are from {} regions, " + + "less than effectiveMinRegionsForDurability: {}.", + ensembleList, i, regionsInQuorum.size(), effectiveMinRegionsForDurability); + } + return PlacementPolicyAdherence.FAIL; + } + + if (regionsInQuorum.size() < writeQuorumSize) { + // not enough regions for each writeQuorum + if (LOG.isDebugEnabled()) { + LOG.debug("For ensemble: {}, write set starting at {} are from {} regions, " + + "less than writeQuorumSize {}.", + ensembleList, i, regionsInQuorum.size(), writeQuorumSize); + } + + // try to check whether each region matches RackawareEnsemblePlacementPolicy MEETS_STRICT + for (String region : regionsInQuorum.keySet()) { + Set bookieIds = regionsInQuorum.get(region); + + TopologyAwareEnsemblePlacementPolicy policyWithinRegion = perRegionPlacement.get(region); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = policyWithinRegion + .isEnsembleAdheringToPlacementPolicy(new ArrayList<>(bookieIds), bookieIds.size(), 1); + if (isEnsembleAdheringToPlacementPolicy == PlacementPolicyAdherence.FAIL) { + if (LOG.isDebugEnabled()) { + LOG.debug("For ensemble {}, write set starting at {} are all from one region, " + + "fall back to RackawareEnsemblePlacementPolicy and fail.", ensembleList, i); + } + return PlacementPolicyAdherence.FAIL; + } + } + + placementPolicyAdherence = PlacementPolicyAdherence.MEETS_SOFT; + } + } + + return placementPolicyAdherence; } } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java index 9d8e36a350d..774f1764747 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java @@ -47,6 +47,7 @@ import java.util.concurrent.TimeUnit; import junit.framework.TestCase; import org.apache.bookkeeper.client.BKException.BKNotEnoughBookiesException; +import org.apache.bookkeeper.client.EnsemblePlacementPolicy.PlacementPolicyAdherence; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.feature.FeatureProvider; import org.apache.bookkeeper.feature.SettableFeature; @@ -603,12 +604,22 @@ public void testNewEnsembleWithSingleRegion() throws Exception { addrs.add(addr4.toBookieId()); repp.onClusterChanged(addrs, new HashSet()); try { - List ensemble = repp.newEnsemble(3, 2, 2, null, - new HashSet()).getResult(); + EnsemblePlacementPolicy.PlacementResult> ensembleResponse = + repp.newEnsemble(3, 2, 2, null, new HashSet()); + List ensemble = ensembleResponse.getResult(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); assertEquals(0, getNumCoveredRegionsInWriteQuorum(ensemble, 2)); - List ensemble2 = repp.newEnsemble(4, 2, 2, null, - new HashSet()).getResult(); + assertEquals(PlacementPolicyAdherence.FAIL, isEnsembleAdheringToPlacementPolicy); + assertEquals(PlacementPolicyAdherence.FAIL, + repp.isEnsembleAdheringToPlacementPolicy(ensemble, 2, 2)); + EnsemblePlacementPolicy.PlacementResult> ensembleResponse2 = + repp.newEnsemble(4, 2, 2, null, + new HashSet()); + List ensemble2 = ensembleResponse2.getResult(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy2 = ensembleResponse2.getAdheringToPolicy(); assertEquals(0, getNumCoveredRegionsInWriteQuorum(ensemble2, 2)); + assertEquals(PlacementPolicyAdherence.FAIL, isEnsembleAdheringToPlacementPolicy2); + assertEquals(PlacementPolicyAdherence.FAIL, repp.isEnsembleAdheringToPlacementPolicy(ensemble2, 2, 2)); } catch (BKNotEnoughBookiesException bnebe) { fail("Should not get not enough bookies exception even there is only one rack."); } @@ -637,19 +648,30 @@ public void testNewEnsembleWithMultipleRegions() throws Exception { addrs.add(addr4.toBookieId()); repp.onClusterChanged(addrs, new HashSet()); try { - List ensemble = repp.newEnsemble(3, 2, 2, null, - new HashSet()).getResult(); + EnsemblePlacementPolicy.PlacementResult> ensembleResponse = + repp.newEnsemble(3, 2, 2, null, new HashSet()); + List ensemble = ensembleResponse.getResult(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); + int numCovered = getNumCoveredRegionsInWriteQuorum(ensemble, 2); assertTrue(numCovered >= 1); assertTrue(numCovered < 3); + assertEquals(PlacementPolicyAdherence.FAIL, isEnsembleAdheringToPlacementPolicy); + assertEquals(PlacementPolicyAdherence.FAIL, repp.isEnsembleAdheringToPlacementPolicy(ensemble, 2, 2)); } catch (BKNotEnoughBookiesException bnebe) { fail("Should not get not enough bookies exception even there is only one rack."); } try { - List ensemble2 = repp.newEnsemble(4, 2, 2, null, - new HashSet()).getResult(); + EnsemblePlacementPolicy.PlacementResult> ensembleResponse2 = + repp.newEnsemble(4, 2, 2, null, + new HashSet()); + List ensemble2 = ensembleResponse2.getResult(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy2 = ensembleResponse2.getAdheringToPolicy(); + int numCovered = getNumCoveredRegionsInWriteQuorum(ensemble2, 2); assertTrue(numCovered >= 1 && numCovered < 3); + assertEquals(PlacementPolicyAdherence.FAIL, isEnsembleAdheringToPlacementPolicy2); + assertEquals(PlacementPolicyAdherence.FAIL, repp.isEnsembleAdheringToPlacementPolicy(ensemble2, 2, 2)); } catch (BKNotEnoughBookiesException bnebe) { fail("Should not get not enough bookies exception even there is only one rack."); } @@ -696,9 +718,14 @@ public void testNewEnsembleWithPickDifferentRack() throws Exception { repp.newEnsemble(ensembleSize, writeQuorumSize, ackQuorumSize, null, excludeBookies); List ensemble = ensembleResponse.getResult(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); if (ensemble.contains(addr1.toBookieId()) && ensemble.contains(addr2.toBookieId())) { fail("addr1 and addr2 is same rack."); } + // 4 writeQuorum but only 2 regions, each region matches RackawareEnsemblePlacementPolicy MEETS_STRICT + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, isEnsembleAdheringToPlacementPolicy); + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, + repp.isEnsembleAdheringToPlacementPolicy(ensemble, writeQuorumSize, ackQuorumSize)); } //addr4 shutdown. @@ -709,7 +736,12 @@ public void testNewEnsembleWithPickDifferentRack() throws Exception { repp.newEnsemble(ensembleSize, writeQuorumSize, ackQuorumSize, null, excludeBookies); List ensemble = ensembleResponse.getResult(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); assertTrue(ensemble.contains(addr1.toBookieId()) && ensemble.contains(addr2.toBookieId())); + // 4 writeQuorum but only 2 regions, each region matches RackawareEnsemblePlacementPolicy MEETS_STRICT + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, isEnsembleAdheringToPlacementPolicy); + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, + repp.isEnsembleAdheringToPlacementPolicy(ensemble, writeQuorumSize, ackQuorumSize)); } } @@ -744,12 +776,24 @@ public void testNewEnsembleWithEnoughRegions() throws Exception { addrs.add(addr8.toBookieId()); repp.onClusterChanged(addrs, new HashSet()); try { - List ensemble1 = repp.newEnsemble(3, 2, 2, null, - new HashSet()).getResult(); + EnsemblePlacementPolicy.PlacementResult> ensembleResponse1 = + repp.newEnsemble(3, 2, 2, null, new HashSet()); + List ensemble1 = ensembleResponse1.getResult(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy1 = ensembleResponse1.getAdheringToPolicy(); assertEquals(3, getNumCoveredRegionsInWriteQuorum(ensemble1, 2)); - List ensemble2 = repp.newEnsemble(4, 2, 2, null, - new HashSet()).getResult(); + assertEquals(PlacementPolicyAdherence.MEETS_STRICT, isEnsembleAdheringToPlacementPolicy1); + assertEquals(PlacementPolicyAdherence.MEETS_STRICT, + repp.isEnsembleAdheringToPlacementPolicy(ensemble1, 2, 2)); + + EnsemblePlacementPolicy.PlacementResult> ensembleResponse2 = + repp.newEnsemble(4, 2, 2, null, + new HashSet()); + List ensemble2 = ensembleResponse2.getResult(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy2 = ensembleResponse2.getAdheringToPolicy(); assertEquals(4, getNumCoveredRegionsInWriteQuorum(ensemble2, 2)); + assertEquals(PlacementPolicyAdherence.MEETS_STRICT, isEnsembleAdheringToPlacementPolicy2); + assertEquals(PlacementPolicyAdherence.MEETS_STRICT, + repp.isEnsembleAdheringToPlacementPolicy(ensemble2, 2, 2)); } catch (BKNotEnoughBookiesException bnebe) { fail("Should not get not enough bookies exception even there is only one rack."); } @@ -857,27 +901,50 @@ public void testNewEnsembleWithThreeRegions() throws Exception { addrs.add(addr10.toBookieId()); repp.onClusterChanged(addrs, new HashSet()); try { - List ensemble = repp.newEnsemble(6, 6, 4, null, - new HashSet()).getResult(); + EnsemblePlacementPolicy.PlacementResult> ensembleResponse = + repp.newEnsemble(6, 6, 4, null, new HashSet()); + List ensemble = ensembleResponse.getResult(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); assert(ensemble.contains(addr4.toBookieId())); assert(ensemble.contains(addr8.toBookieId())); assert(ensemble.size() == 6); assertEquals(3, getNumRegionsInEnsemble(ensemble)); - ensemble = repp.newEnsemble(7, 7, 4, null, new HashSet()).getResult(); + // 6 writeQuorum but only 3 regions, each region matches RackawareEnsemblePlacementPolicy MEETS_STRICT + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, isEnsembleAdheringToPlacementPolicy); + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, repp.isEnsembleAdheringToPlacementPolicy(ensemble, 6, 4)); + + ensembleResponse = repp.newEnsemble(7, 7, 4, null, new HashSet()); + ensemble = ensembleResponse.getResult(); + isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); assert(ensemble.contains(addr4.toBookieId())); assert(ensemble.contains(addr8.toBookieId())); assert(ensemble.size() == 7); assertEquals(3, getNumRegionsInEnsemble(ensemble)); - ensemble = repp.newEnsemble(8, 8, 5, null, new HashSet()).getResult(); + // 7 writeQuorum but only 3 regions, each region matches RackawareEnsemblePlacementPolicy MEETS_STRICT + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, isEnsembleAdheringToPlacementPolicy); + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, repp.isEnsembleAdheringToPlacementPolicy(ensemble, 7, 4)); + + ensembleResponse = repp.newEnsemble(8, 8, 5, null, new HashSet()); + ensemble = ensembleResponse.getResult(); + isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); assert(ensemble.contains(addr4.toBookieId())); assert(ensemble.contains(addr8.toBookieId())); assert(ensemble.size() == 8); assertEquals(3, getNumRegionsInEnsemble(ensemble)); - ensemble = repp.newEnsemble(9, 9, 5, null, new HashSet()).getResult(); + // 8 writeQuorum but only 3 regions, each region matches RackawareEnsemblePlacementPolicy MEETS_STRICT + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, isEnsembleAdheringToPlacementPolicy); + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, repp.isEnsembleAdheringToPlacementPolicy(ensemble, 8, 5)); + + ensembleResponse = repp.newEnsemble(9, 9, 5, null, new HashSet()); + ensemble = ensembleResponse.getResult(); + isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); assert(ensemble.contains(addr4.toBookieId())); assert(ensemble.contains(addr8.toBookieId())); assert(ensemble.size() == 9); assertEquals(3, getNumRegionsInEnsemble(ensemble)); + // 9 writeQuorum but only 3 regions, each region matches RackawareEnsemblePlacementPolicy MEETS_STRICT + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, isEnsembleAdheringToPlacementPolicy); + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, repp.isEnsembleAdheringToPlacementPolicy(ensemble, 9, 5)); } catch (BKNotEnoughBookiesException bnebe) { fail("Should not get not enough bookies exception even there is only one rack."); } @@ -1024,10 +1091,15 @@ public void testNewEnsembleWithFiveRegions() throws Exception { repp.onClusterChanged(addrs, new HashSet()); try { - List ensemble = repp.newEnsemble(10, 10, 10, null, - new HashSet()).getResult(); + EnsemblePlacementPolicy.PlacementResult> ensembleResponse = + repp.newEnsemble(10, 10, 10, null, new HashSet()); + List ensemble = ensembleResponse.getResult(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); assert(ensemble.size() == 10); assertEquals(5, getNumRegionsInEnsemble(ensemble)); + // 10 writeQuorum but only 5 regions, each region matches RackawareEnsemblePlacementPolicy MEETS_STRICT + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, isEnsembleAdheringToPlacementPolicy); + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, repp.isEnsembleAdheringToPlacementPolicy(ensemble, 10, 10)); } catch (BKNotEnoughBookiesException bnebe) { LOG.error("BKNotEnoughBookiesException", bnebe); fail("Should not get not enough bookies exception even there is only one rack."); @@ -1036,11 +1108,29 @@ public void testNewEnsembleWithFiveRegions() throws Exception { try { Set excludedAddrs = new HashSet(); excludedAddrs.add(addr10.toBookieId()); - List ensemble = repp.newEnsemble(10, 10, 10, null, - excludedAddrs).getResult(); + EnsemblePlacementPolicy.PlacementResult> ensembleResponse = + repp.newEnsemble(10, 10, 10, null, excludedAddrs); + List ensemble = ensembleResponse.getResult(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); assert(ensemble.contains(addr11.toBookieId()) && ensemble.contains(addr12.toBookieId())); assert(ensemble.size() == 10); assertEquals(5, getNumRegionsInEnsemble(ensemble)); + // 10 writeQuorum but only 5 regions, each region matches RackawareEnsemblePlacementPolicy MEETS_STRICT + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, isEnsembleAdheringToPlacementPolicy); + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, repp.isEnsembleAdheringToPlacementPolicy(ensemble, 10, 10)); + } catch (BKNotEnoughBookiesException bnebe) { + fail("Should not get not enough bookies exception even there is only one rack."); + } + + try { + EnsemblePlacementPolicy.PlacementResult> ensembleResponse = + repp.newEnsemble(5, 5, 5, null, new HashSet()); + List ensemble = ensembleResponse.getResult(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); + assert(ensemble.size() == 5); + assertEquals(5, getNumRegionsInEnsemble(ensemble)); + assertEquals(PlacementPolicyAdherence.MEETS_STRICT, isEnsembleAdheringToPlacementPolicy); + assertEquals(PlacementPolicyAdherence.MEETS_STRICT, repp.isEnsembleAdheringToPlacementPolicy(ensemble, 5, 5)); } catch (BKNotEnoughBookiesException bnebe) { fail("Should not get not enough bookies exception even there is only one rack."); } @@ -1130,9 +1220,16 @@ public void testEnsembleWithThreeRegionsReplaceInternal(int minDurability, boole List ensemble; try { - ensemble = repp.newEnsemble(6, 6, ackQuorum, null, new HashSet()).getResult(); + EnsemblePlacementPolicy.PlacementResult> ensembleResponse = + repp.newEnsemble(6, 6, ackQuorum, null, new HashSet()); + ensemble = ensembleResponse.getResult(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); assert(ensemble.size() == 6); assertEquals(3, getNumRegionsInEnsemble(ensemble)); + // 6 writeQuorum but only 3 regions, each region matches RackawareEnsemblePlacementPolicy MEETS_STRICT + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, isEnsembleAdheringToPlacementPolicy); + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, + repp.isEnsembleAdheringToPlacementPolicy(ensemble, 6, ackQuorum)); } catch (BKNotEnoughBookiesException bnebe) { LOG.error("BKNotEnoughBookiesException", bnebe); fail("Should not get not enough bookies exception even there is only one rack."); @@ -1263,8 +1360,15 @@ public void testEnsembleDurabilityDisabledInternal(int minDurability, boolean di List ensemble; try { - ensemble = repp.newEnsemble(6, 6, 4, null, new HashSet()).getResult(); + EnsemblePlacementPolicy.PlacementResult> ensembleResponse = + repp.newEnsemble(6, 6, 4, null, new HashSet()); + ensemble = ensembleResponse.getResult(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); assert(ensemble.size() == 6); + // 6 writeQuorum but only 1 region with 9 racks, which matches RackawareEnsemblePlacementPolicy MEETS_STRICT + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, isEnsembleAdheringToPlacementPolicy); + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, + repp.isEnsembleAdheringToPlacementPolicy(ensemble, 6, 4)); } catch (BKNotEnoughBookiesException bnebe) { LOG.error("BKNotEnoughBookiesException", bnebe); fail("Should not get not enough bookies exception even there is only one rack."); @@ -1388,9 +1492,15 @@ public void testBasicReorderReadLACSequenceWithLocalRegion() throws Exception { private void basicReorderReadSequenceWithLocalRegionTest(String myRegion, boolean isReadLAC) throws Exception { prepareNetworkTopologyForReorderTests(myRegion); - List ensemble = repp.newEnsemble(9, 9, 5, null, - new HashSet()).getResult(); + EnsemblePlacementPolicy.PlacementResult> ensembleResponse = + repp.newEnsemble(9, 9, 5, null, new HashSet()); + List ensemble = ensembleResponse.getResult(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); + assertEquals(9, getNumCoveredRegionsInWriteQuorum(ensemble, 9)); + // 9 nodes, 3 regions and 3 racks for each region + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, isEnsembleAdheringToPlacementPolicy); + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, repp.isEnsembleAdheringToPlacementPolicy(ensemble, 9, 5)); DistributionSchedule ds = new RoundRobinDistributionSchedule(9, 9, 9); @@ -1449,9 +1559,15 @@ public void testBasicReorderReadLACSequenceWithRemoteRegion() throws Exception { private void basicReorderReadSequenceWithRemoteRegionTest(String myRegion, boolean isReadLAC) throws Exception { prepareNetworkTopologyForReorderTests(myRegion); - List ensemble = repp.newEnsemble(9, 9, 5, null, - new HashSet()).getResult(); + EnsemblePlacementPolicy.PlacementResult> ensembleResponse = + repp.newEnsemble(9, 9, 5, null, new HashSet()); + List ensemble = ensembleResponse.getResult(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); + assertEquals(9, getNumCoveredRegionsInWriteQuorum(ensemble, 9)); + // 9 nodes, 3 regions and 3 racks for each region + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, isEnsembleAdheringToPlacementPolicy); + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, repp.isEnsembleAdheringToPlacementPolicy(ensemble, 9, 5)); DistributionSchedule ds = new RoundRobinDistributionSchedule(9, 9, 9); @@ -1522,9 +1638,15 @@ private void reorderReadSequenceWithUnavailableOrReadOnlyBookiesTest(boolean isR prepareNetworkTopologyForReorderTests(myRegion); - List ensemble = repp.newEnsemble(9, 9, 5, null, - new HashSet()).getResult(); + EnsemblePlacementPolicy.PlacementResult> ensembleResponse = + repp.newEnsemble(9, 9, 5, null, new HashSet()); + List ensemble = ensembleResponse.getResult(); + PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy = ensembleResponse.getAdheringToPolicy(); + assertEquals(9, getNumCoveredRegionsInWriteQuorum(ensemble, 9)); + // 9 nodes, 3 regions and 3 racks for each region + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, isEnsembleAdheringToPlacementPolicy); + assertEquals(PlacementPolicyAdherence.MEETS_SOFT, repp.isEnsembleAdheringToPlacementPolicy(ensemble, 9, 5)); DistributionSchedule ds = new RoundRobinDistributionSchedule(9, 9, 9);