diff --git a/core/src/main/java/org/apache/accumulo/core/Constants.java b/core/src/main/java/org/apache/accumulo/core/Constants.java index b8fc0840047..ed99d852123 100644 --- a/core/src/main/java/org/apache/accumulo/core/Constants.java +++ b/core/src/main/java/org/apache/accumulo/core/Constants.java @@ -26,8 +26,9 @@ public class Constants { public static final String VERSION_DIR = "version"; public static final String APPNAME = "org.apache.accumulo"; - // important directories - public static final String INSTANCE_ID_DIR = "instance_id"; + public static final String INSTANCE_DIR = "instance"; + public static final String INSTANCE_ID_PREFIX = "id_"; + public static final String INSTANCE_NAME_PREFIX = "name_"; public static final String TABLE_DIR = "tables"; public static final String RECOVERY_DIR = "recovery"; public static final String WAL_DIR = "wal"; diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientInfo.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientInfo.java index d569fdc2dba..a60689264b2 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientInfo.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientInfo.java @@ -27,6 +27,7 @@ import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.data.InstanceId; +import org.apache.accumulo.core.dataImpl.InstanceInfo; import org.apache.accumulo.core.zookeeper.ZooSession; import org.apache.hadoop.conf.Configuration; @@ -37,6 +38,11 @@ */ public interface ClientInfo { + /** + * @return Accumulo instance info + */ + InstanceInfo getInstanceInfo(); + /** * @return Accumulo instance name */ diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientInfoImpl.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientInfoImpl.java index fd05a848dc2..434e2b9c820 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientInfoImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientInfoImpl.java @@ -36,6 +36,7 @@ import org.apache.accumulo.core.conf.ClientProperty; import org.apache.accumulo.core.conf.ConfigurationTypeHelper; import org.apache.accumulo.core.data.InstanceId; +import org.apache.accumulo.core.dataImpl.InstanceInfo; import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.core.zookeeper.ZooSession; import org.apache.hadoop.conf.Configuration; @@ -51,6 +52,7 @@ public class ClientInfoImpl implements ClientInfo { // suppliers for lazily loading private final Supplier tokenSupplier; private final Supplier hadoopConf; + private final Supplier instanceInfo; private final Supplier instanceId; private final BiFunction zooSessionForName; @@ -62,6 +64,7 @@ public ClientInfoImpl(Properties properties, Optional token this.hadoopConf = memoize(Configuration::new); this.zooSessionForName = (name, rootPath) -> new ZooSession(name, getZooKeepers() + rootPath, getZooKeepersSessionTimeOut(), null); + this.instanceInfo = memoize(() -> new InstanceInfo(getInstanceName(), getInstanceId())); this.instanceId = memoize(() -> { try (var zk = getZooKeeperSupplier(getClass().getSimpleName() + ".getInstanceId()", "").get()) { @@ -70,6 +73,11 @@ public ClientInfoImpl(Properties properties, Optional token }); } + @Override + public InstanceInfo getInstanceInfo() { + return instanceInfo.get(); + } + @Override public String getInstanceName() { return getString(ClientProperty.INSTANCE_NAME); diff --git a/core/src/main/java/org/apache/accumulo/core/dataImpl/InstanceInfo.java b/core/src/main/java/org/apache/accumulo/core/dataImpl/InstanceInfo.java new file mode 100644 index 00000000000..c6ae9e213cd --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/dataImpl/InstanceInfo.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.dataImpl; + +import static java.util.Objects.requireNonNull; + +import java.util.Objects; + +import org.apache.accumulo.core.data.InstanceId; + +public class InstanceInfo { + + private final String name; + private final InstanceId id; + + public InstanceInfo(String name, InstanceId id) { + this.name = requireNonNull(name); + this.id = requireNonNull(id); + } + + public String getInstanceName() { + return name; + } + + public InstanceId getInstanceId() { + return id; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof InstanceInfo) { + var o = (InstanceInfo) obj; + return Objects.equals(getInstanceName(), o.getInstanceName()) + && Objects.equals(getInstanceId(), o.getInstanceId()); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(getInstanceName(), getInstanceId()); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "[name=" + getInstanceName() + ";id=" + getInstanceId() + + "]"; + } + +} diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java b/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java index 5077705505f..93cb46c6f86 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java +++ b/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java @@ -49,9 +49,9 @@ import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.conf.SiteConfiguration; import org.apache.accumulo.core.crypto.CryptoFactoryLoader; -import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.data.NamespaceId; import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.dataImpl.InstanceInfo; import org.apache.accumulo.core.lock.ServiceLock; import org.apache.accumulo.core.metadata.schema.Ample; import org.apache.accumulo.core.metrics.MetricsInfo; @@ -137,9 +137,8 @@ private ServerContext(ServerInfo info) { /** * Used during initialization to set the instance name and ID. */ - public static ServerContext initialize(SiteConfiguration siteConfig, String instanceName, - InstanceId instanceID) { - return new ServerContext(ServerInfo.initialize(siteConfig, instanceName, instanceID)); + public static ServerContext initialize(SiteConfiguration siteConfig, InstanceInfo instanceInfo) { + return new ServerContext(ServerInfo.initialize(siteConfig, instanceInfo)); } /** diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerDirs.java b/server/base/src/main/java/org/apache/accumulo/server/ServerDirs.java index 062198ba4b8..d42e8e5435a 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/ServerDirs.java +++ b/server/base/src/main/java/org/apache/accumulo/server/ServerDirs.java @@ -34,7 +34,7 @@ import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.data.InstanceId; +import org.apache.accumulo.core.dataImpl.InstanceInfo; import org.apache.accumulo.core.volume.Volume; import org.apache.accumulo.core.volume.VolumeConfiguration; import org.apache.accumulo.server.fs.VolumeManager; @@ -77,17 +77,17 @@ public Set checkBaseUris(Configuration hadoopConf, Set configure // all base dirs must have same instance id and data version, any dirs that have neither should // be ignored String firstDir = null; - InstanceId firstIid = null; + InstanceInfo firstInfo = null; Integer firstVersion = null; // preserve order from configuration (to match user expectations a bit when volumes get sent to // user-implemented VolumeChoosers) LinkedHashSet baseDirsList = new LinkedHashSet<>(); for (String baseDir : configuredBaseDirs) { - Path path = new Path(baseDir, Constants.INSTANCE_ID_DIR); - InstanceId currentIid; + Path path = new Path(baseDir, Constants.INSTANCE_DIR); + InstanceInfo currentInstanceInfo; int currentVersion; try { - currentIid = VolumeManager.getInstanceIDFromHdfs(path, hadoopConf); + currentInstanceInfo = VolumeManager.getInstanceInfoFromHdfs(path, hadoopConf); Path vpath = new Path(baseDir, Constants.VERSION_DIR); currentVersion = getAccumuloPersistentVersion(vpath.getFileSystem(hadoopConf), vpath); } catch (Exception e) { @@ -98,14 +98,14 @@ public Set checkBaseUris(Configuration hadoopConf, Set configure } } - if (firstIid == null) { - firstIid = currentIid; + if (firstInfo == null) { + firstInfo = currentInstanceInfo; firstDir = baseDir; firstVersion = currentVersion; - } else if (!currentIid.equals(firstIid)) { + } else if (!currentInstanceInfo.equals(firstInfo)) { throw new IllegalArgumentException("Configuration " + Property.INSTANCE_VOLUMES.getKey() - + " contains paths that have different instance ids " + baseDir + " has " + currentIid - + " and " + firstDir + " has " + firstIid); + + " contains paths that have different instance ids " + baseDir + " has " + + currentInstanceInfo + " and " + firstDir + " has " + firstInfo); } else if (currentVersion != firstVersion) { throw new IllegalArgumentException("Configuration " + Property.INSTANCE_VOLUMES.getKey() + " contains paths that have different versions " + baseDir + " has " + currentVersion @@ -238,8 +238,8 @@ public int getAccumuloPersistentVersion(FileSystem fs, Path path) { } } - public Path getInstanceIdLocation(Volume v) { + public Path getInstanceInfoLocation(Volume v) { // all base dirs should have the same instance id, so can choose any one - return v.prefixChild(Constants.INSTANCE_ID_DIR); + return v.prefixChild(Constants.INSTANCE_DIR); } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerInfo.java b/server/base/src/main/java/org/apache/accumulo/server/ServerInfo.java index 617e26b8def..6825b670f4b 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/ServerInfo.java +++ b/server/base/src/main/java/org/apache/accumulo/server/ServerInfo.java @@ -37,7 +37,7 @@ import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.conf.SiteConfiguration; import org.apache.accumulo.core.data.InstanceId; -import org.apache.accumulo.core.fate.zookeeper.ZooUtil; +import org.apache.accumulo.core.dataImpl.InstanceInfo; import org.apache.accumulo.core.zookeeper.ZooSession; import org.apache.accumulo.server.fs.VolumeManager; import org.apache.accumulo.server.fs.VolumeManagerImpl; @@ -54,38 +54,28 @@ public class ServerInfo implements ClientInfo { // set things up using the config file, the instanceId from HDFS, and ZK for the instanceName static ServerInfo fromServerConfig(SiteConfiguration siteConfig) { - final Function instanceNameFromZk = si -> { - try (var zk = - si.getZooKeeperSupplier(ServerInfo.class.getSimpleName() + ".getInstanceName()", "") - .get()) { - return ZooUtil.getInstanceName(zk, si.getInstanceId()); - } - }; final Function instanceIdFromHdfs = si -> VolumeManager.getInstanceIDFromHdfs( - si.getServerDirs().getInstanceIdLocation(si.getVolumeManager().getFirst()), + InstanceInfo> instanceInfoFromHdfs = si -> VolumeManager.getInstanceInfoFromHdfs( + si.getServerDirs().getInstanceInfoLocation(si.getVolumeManager().getFirst()), si.getHadoopConf()); return new ServerInfo(siteConfig, GET_ZK_HOSTS_FROM_CONFIG, GET_ZK_TIMEOUT_FROM_CONFIG, - instanceNameFromZk, instanceIdFromHdfs); + instanceInfoFromHdfs); } // set things up using a provided instanceName and InstanceId to initialize the system, but still // have a ServerContext that is functional without bootstrapping issues, so long as you don't call // functions from it that require an instance to have already been initialized - static ServerInfo initialize(SiteConfiguration siteConfig, String instanceName, - InstanceId instanceId) { - requireNonNull(instanceName); - requireNonNull(instanceId); + static ServerInfo initialize(SiteConfiguration siteConfig, InstanceInfo instanceInfo) { + requireNonNull(instanceInfo); return new ServerInfo(siteConfig, GET_ZK_HOSTS_FROM_CONFIG, GET_ZK_TIMEOUT_FROM_CONFIG, - si -> instanceName, si -> instanceId); + si -> instanceInfo); } // set things up using the config file, and the client config for a server-side CLI utility static ServerInfo fromServerAndClientConfig(SiteConfiguration siteConfig, ClientInfo info) { // ClientInfo.getInstanceId looks up the ID in ZK using the provided instance name return new ServerInfo(siteConfig, si -> info.getZooKeepers(), - si -> info.getZooKeepersSessionTimeOut(), si -> info.getInstanceName(), - si -> info.getInstanceId()); + si -> info.getZooKeepersSessionTimeOut(), si -> info.getInstanceInfo()); } static ServerInfo forTesting(SiteConfiguration siteConfig, String instanceName, String zooKeepers, @@ -106,8 +96,7 @@ static ServerInfo forTesting(SiteConfiguration siteConfig, String instanceName, private final Supplier serverDirs; private final Supplier zooKeepers; private final Supplier zooKeepersSessionTimeOut; // can't memoize IntSupplier - private final Supplier instanceId; - private final Supplier instanceName; + private final Supplier instanceInfo; private final Supplier credentials; private final BiFunction zooSessionForName; @@ -118,13 +107,12 @@ static ServerInfo forTesting(SiteConfiguration siteConfig, String instanceName, // another, but because things are lazily loaded, it is okay if one depends on another in one // direction only private ServerInfo(SiteConfiguration siteConfig, Function zkHostsFunction, - ToIntFunction zkTimeoutFunction, Function instanceNameFunction, - Function instanceIdFunction) { + ToIntFunction zkTimeoutFunction, + Function instanceInfoFunction) { this.siteConfig = requireNonNull(siteConfig); requireNonNull(zkHostsFunction); requireNonNull(zkTimeoutFunction); - requireNonNull(instanceNameFunction); - requireNonNull(instanceIdFunction); + requireNonNull(instanceInfoFunction); this.hadoopConf = memoize(Configuration::new); this.volumeManager = memoize(() -> { @@ -144,8 +132,7 @@ private ServerInfo(SiteConfiguration siteConfig, Function zkH // from here on, set up the suppliers based on what was passed in, to support different cases this.zooKeepers = memoize(() -> zkHostsFunction.apply(this)); this.zooKeepersSessionTimeOut = memoize(() -> zkTimeoutFunction.applyAsInt(this)); - this.instanceId = memoize(() -> instanceIdFunction.apply(this)); - this.instanceName = memoize(() -> instanceNameFunction.apply(this)); + this.instanceInfo = memoize(() -> instanceInfoFunction.apply(this)); } public SiteConfiguration getSiteConfiguration() { @@ -156,9 +143,19 @@ public VolumeManager getVolumeManager() { return volumeManager.get(); } + @Override + public InstanceInfo getInstanceInfo() { + return instanceInfo.get(); + } + + @Override + public String getInstanceName() { + return getInstanceInfo().getInstanceName(); + } + @Override public InstanceId getInstanceId() { - return instanceId.get(); + return getInstanceInfo().getInstanceId(); } @Override @@ -203,11 +200,6 @@ public Properties getClientProperties() { return properties; } - @Override - public String getInstanceName() { - return instanceName.get(); - } - public Credentials getCredentials() { return credentials.get(); } diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java index ca16e4872b2..58937daea63 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java +++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java @@ -28,6 +28,7 @@ import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.data.InstanceId; +import org.apache.accumulo.core.dataImpl.InstanceInfo; import org.apache.accumulo.core.fate.FateId; import org.apache.accumulo.core.volume.Volume; import org.apache.accumulo.core.volume.VolumeConfiguration; @@ -207,28 +208,51 @@ default Volume getFirst() { Logger log = LoggerFactory.getLogger(VolumeManager.class); - static InstanceId getInstanceIDFromHdfs(Path instanceDirectory, Configuration hadoopConf) { + static InstanceInfo getInstanceInfoFromHdfs(Path instanceDirectory, Configuration hadoopConf) { try { - FileSystem fs = - VolumeConfiguration.fileSystemForPath(instanceDirectory.toString(), hadoopConf); + log.debug("Trying to read instance info from {}", instanceDirectory); + var fs = VolumeConfiguration.fileSystemForPath(instanceDirectory.toString(), hadoopConf); FileStatus[] files = null; try { files = fs.listStatus(instanceDirectory); } catch (FileNotFoundException ex) { // ignored } - log.debug("Trying to read instance id from {}", instanceDirectory); - if (files == null || files.length == 0) { - log.error("unable to obtain instance id at {}", instanceDirectory); - throw new IllegalStateException( - "Accumulo not initialized, there is no instance id at " + instanceDirectory); - } else if (files.length != 1) { - log.error("multiple potential instances in {}", instanceDirectory); + InstanceId instanceId = null; + String instanceName = null; + if (files != null) { + for (FileStatus file : files) { + String fileName = file.getPath().getName(); + if (fileName.startsWith(Constants.INSTANCE_ID_PREFIX)) { + if (instanceId == null) { + instanceId = InstanceId.of(fileName.substring(Constants.INSTANCE_ID_PREFIX.length())); + } else { + log.error("multiple potential instances in {}", instanceDirectory); + throw new IllegalStateException( + "Accumulo found multiple instance ids in " + instanceDirectory); + } + } else if (fileName.startsWith(Constants.INSTANCE_NAME_PREFIX)) { + if (instanceName == null) { + instanceName = fileName.substring(Constants.INSTANCE_NAME_PREFIX.length()); + } else { + log.error("multiple potential instances in {}", instanceDirectory); + throw new IllegalStateException( + "Accumulo found multiple instance names in " + instanceDirectory); + } + } else { + log.error("found unexpected file {} in {}", fileName, instanceDirectory); + throw new IllegalStateException( + "Accumulo found unexpected file " + fileName + " in " + instanceDirectory); + } + } + } + if (instanceName == null || instanceId == null) { + log.error("unable to obtain instance id ({}) or name ({}) at {}", instanceId, instanceName, + instanceDirectory); throw new IllegalStateException( - "Accumulo found multiple possible instance ids in " + instanceDirectory); - } else { - return InstanceId.of(files[0].getPath().getName()); + "Accumulo not initialized, instance info is missing at " + instanceDirectory); } + return new InstanceInfo(instanceName, instanceId); } catch (IOException e) { log.error("Problem reading instance id out of hdfs at " + instanceDirectory, e); throw new UncheckedIOException( diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java index 8d7a8b4bfc6..36525f3b0d5 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java +++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java @@ -39,6 +39,7 @@ import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.conf.SiteConfiguration; import org.apache.accumulo.core.data.InstanceId; +import org.apache.accumulo.core.dataImpl.InstanceInfo; import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; import org.apache.accumulo.core.file.FileOperations; import org.apache.accumulo.core.metadata.AccumuloTable; @@ -154,13 +155,13 @@ private boolean doInit(ZooReaderWriter zoo, Opts opts, VolumeManager fs, return false; } - InstanceId instanceId = InstanceId.of(UUID.randomUUID()); + var instanceInfo = new InstanceInfo(instanceName, InstanceId.of(UUID.randomUUID())); ZooKeeperInitializer zki = new ZooKeeperInitializer(); - zki.initializeConfig(instanceId, zoo); - zki.initInstanceNameAndId(zoo, instanceId, opts.clearInstanceName, instanceNamePath); + zki.initializeConfig(instanceInfo.getInstanceId(), zoo); + zki.initInstanceNameAndId(zoo, instanceInfo.getInstanceId(), opts.clearInstanceName, + instanceNamePath); - try (ServerContext context = - ServerContext.initialize(initConfig.getSiteConf(), instanceName, instanceId)) { + try (var context = ServerContext.initialize(initConfig.getSiteConf(), instanceInfo)) { var chooserEnv = new VolumeChooserEnvironmentImpl(Scope.INIT, AccumuloTable.ROOT.tableId(), null, context); String rootTabletDirName = RootTable.ROOT_TABLET_DIR_NAME; @@ -170,7 +171,7 @@ private boolean doInit(ZooReaderWriter zoo, Opts opts, VolumeManager fs, + rootTabletDirName + SEPARATOR + "00000_00000." + ext).toString(); zki.initialize(context, rootTabletDirName, rootTabletFileUri); - if (!createDirs(fs, instanceId, initConfig.getVolumeUris())) { + if (!createDirs(fs, instanceInfo, initConfig.getVolumeUris())) { throw new IOException("Problem creating directories on " + fs.getVolumes()); } var fileSystemInitializer = new FileSystemInitializer(initConfig); @@ -247,13 +248,14 @@ private static boolean zookeeperAvailable(ZooReaderWriter zoo) { } /** - * Create the version directory and the instance id path and file. The method tries to create the - * directories and instance id file for all base directories provided unless an IOException is - * thrown. If an IOException occurs, this method won't retry and will return false. + * Create the version directory and the instance path and id and name files. The method tries to + * create the directories and instances files for all base directories provided unless an + * IOException is thrown. If an IOException occurs, this method won't retry and will return false. * * @return false if an IOException occurred, true otherwise. */ - private static boolean createDirs(VolumeManager fs, InstanceId instanceId, Set baseDirs) { + private static boolean createDirs(VolumeManager fs, InstanceInfo instanceInfo, + Set baseDirs) { boolean success; try { @@ -273,7 +275,7 @@ private static boolean createDirs(VolumeManager fs, InstanceId instanceId, Set init(File newFile, List uuids, List dataVer accumuloPaths.add(accumuloPath); if (uuids.get(i) != null) { - fs.mkdirs(new Path(accumuloPath + "/" + Constants.INSTANCE_ID_DIR)); - fs.createNewFile( - new Path(accumuloPath + "/" + Constants.INSTANCE_ID_DIR + "/" + uuids.get(i))); + fs.mkdirs(new Path(accumuloPath + "/" + Constants.INSTANCE_DIR)); + fs.createNewFile(new Path(accumuloPath + "/" + Constants.INSTANCE_DIR + "/" + + Constants.INSTANCE_ID_PREFIX + uuids.get(i))); + fs.createNewFile(new Path(accumuloPath + "/" + Constants.INSTANCE_DIR + "/" + + Constants.INSTANCE_NAME_PREFIX + instanceName)); } if (dataVersions.get(i) != null) { diff --git a/server/base/src/test/java/org/apache/accumulo/server/init/InitializeTest.java b/server/base/src/test/java/org/apache/accumulo/server/init/InitializeTest.java index c0b4817a2f9..27286e17636 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/init/InitializeTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/init/InitializeTest.java @@ -113,7 +113,7 @@ public void testCheckInit_FSException() throws Exception { public void testCheckInit_OK() throws Exception { expect(zk.exists("/", null)).andReturn(new Stat()); // check for volumes initialized calls exists twice for each volume - // once for instance_id, and once for version + // once for instance dir, and once for version dir expect(fs.exists(anyObject(Path.class))).andReturn(false).times(4); replay(sconf, zk, fs); Initialize.checkInit(zrw, fs, initConfig); diff --git a/server/base/src/test/java/org/apache/accumulo/server/security/SystemCredentialsTest.java b/server/base/src/test/java/org/apache/accumulo/server/security/SystemCredentialsTest.java index ecd76424ebb..b61fa5bf0a8 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/security/SystemCredentialsTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/security/SystemCredentialsTest.java @@ -21,50 +21,21 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.io.File; -import java.io.IOException; import java.util.UUID; -import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.clientImpl.Credentials; import org.apache.accumulo.core.conf.SiteConfiguration; import org.apache.accumulo.core.data.InstanceId; -import org.apache.accumulo.server.AccumuloDataVersion; import org.apache.accumulo.server.security.SystemCredentials.SystemToken; import org.apache.commons.codec.digest.Crypt; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; - public class SystemCredentialsTest { private static SiteConfiguration siteConfig = SiteConfiguration.empty().build(); private InstanceId instanceId = InstanceId.of(UUID.nameUUIDFromBytes(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0})); - @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "input not from a user") - @BeforeAll - public static void setUp() throws IOException { - File testInstanceId = - new File(new File(new File(new File("target"), "instanceTest"), Constants.INSTANCE_ID_DIR), - UUID.fromString("00000000-0000-0000-0000-000000000000").toString()); - if (!testInstanceId.exists()) { - assertTrue( - testInstanceId.getParentFile().mkdirs() || testInstanceId.getParentFile().isDirectory()); - assertTrue(testInstanceId.createNewFile()); - } - - File testInstanceVersion = - new File(new File(new File(new File("target"), "instanceTest"), Constants.VERSION_DIR), - AccumuloDataVersion.get() + ""); - if (!testInstanceVersion.exists()) { - assertTrue(testInstanceVersion.getParentFile().mkdirs() - || testInstanceVersion.getParentFile().isDirectory()); - assertTrue(testInstanceVersion.createNewFile()); - } - } - @Test public void testWireVersion() { // sanity check to make sure it's a positive number diff --git a/test/src/main/java/org/apache/accumulo/test/VolumeIT.java b/test/src/main/java/org/apache/accumulo/test/VolumeIT.java index e6336d6d350..0c94d37276a 100644 --- a/test/src/main/java/org/apache/accumulo/test/VolumeIT.java +++ b/test/src/main/java/org/apache/accumulo/test/VolumeIT.java @@ -29,6 +29,8 @@ import java.util.Map.Entry; import java.util.SortedSet; import java.util.TreeSet; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Accumulo; @@ -39,10 +41,10 @@ import org.apache.accumulo.core.client.admin.NewTableConfiguration; import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.dataImpl.InstanceInfo; import org.apache.accumulo.core.metadata.AccumuloTable; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.StoredTabletFile; @@ -112,7 +114,7 @@ public void testAddVolumes() throws Exception { try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) { String[] tableNames = getUniqueNames(2); - InstanceId uuid = verifyAndShutdownCluster(client, tableNames[0]); + InstanceInfo instanceInfo = verifyAndShutdownCluster(client, tableNames[0]); updateConfig(config -> config.setProperty(Property.INSTANCE_VOLUMES.getKey(), v1 + "," + v2 + "," + v3)); @@ -120,7 +122,7 @@ public void testAddVolumes() throws Exception { // initialize volume assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").getProcess().waitFor()); - checkVolumesInitialized(Arrays.asList(v1, v2, v3), uuid); + checkVolumesInitialized(Arrays.asList(v1, v2, v3), instanceInfo); // start cluster and verify that new volume is used cluster.start(); @@ -130,15 +132,17 @@ public void testAddVolumes() throws Exception { } // grab uuid before shutting down cluster - private InstanceId verifyAndShutdownCluster(AccumuloClient c, String tableName) throws Exception { - InstanceId uuid = c.instanceOperations().getInstanceId(); + private InstanceInfo verifyAndShutdownCluster(AccumuloClient c, String tableName) + throws Exception { + var instanceInfo = new InstanceInfo(((ClientContext) c).getInstanceName(), + c.instanceOperations().getInstanceId()); verifyVolumesUsed(c, tableName, false, v1, v2); assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor()); cluster.stop(); - return uuid; + return instanceInfo; } @Test @@ -148,14 +152,14 @@ public void testNonConfiguredVolumes() throws Exception { try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) { - InstanceId uuid = verifyAndShutdownCluster(client, tableNames[0]); + InstanceInfo instanceInfo = verifyAndShutdownCluster(client, tableNames[0]); updateConfig(config -> config.setProperty(Property.INSTANCE_VOLUMES.getKey(), v2 + "," + v3)); // initialize volume assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").getProcess().waitFor()); - checkVolumesInitialized(Arrays.asList(v1, v2, v3), uuid); + checkVolumesInitialized(Arrays.asList(v1, v2, v3), instanceInfo); // start cluster and verify that new volume is used cluster.start(); @@ -169,13 +173,17 @@ public void testNonConfiguredVolumes() throws Exception { } // check that all volumes are initialized - private void checkVolumesInitialized(List volumes, InstanceId uuid) throws Exception { + private void checkVolumesInitialized(List volumes, InstanceInfo instanceInfo) + throws Exception { for (Path volumePath : volumes) { FileSystem fs = volumePath.getFileSystem(cluster.getServerContext().getHadoopConf()); - Path vp = new Path(volumePath, Constants.INSTANCE_ID_DIR); - FileStatus[] iids = fs.listStatus(vp); - assertEquals(1, iids.length); - assertEquals(uuid.canonical(), iids[0].getPath().getName()); + Path vp = new Path(volumePath, Constants.INSTANCE_DIR); + var list = Stream.of(fs.listStatus(vp)).map(FileStatus::getPath).map(Path::getName) + .collect(Collectors.toList()); + assertEquals(2, list.size()); + assertTrue( + list.contains(Constants.INSTANCE_ID_PREFIX + instanceInfo.getInstanceId().canonical())); + assertTrue(list.contains(Constants.INSTANCE_NAME_PREFIX + instanceInfo.getInstanceName())); } }