From f965118361d58afa762881749a8227295739aefc Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 25 Apr 2024 00:16:09 +0530 Subject: [PATCH 1/8] util: cleanup old mac address code Cleanup old mac address handling code to use JDK11 lib instead of hacks. Also really strange to see some basic string parsing code was written by hand, replaced with Long.parseValue(str, 16) to convert hex string to long. Signed-off-by: Rohit Yadav --- .../java/com/cloud/utils/net/MacAddress.java | 225 +++--------------- .../com/cloud/utils/net/MacAddressTest.java | 14 +- 2 files changed, 34 insertions(+), 205 deletions(-) diff --git a/utils/src/main/java/com/cloud/utils/net/MacAddress.java b/utils/src/main/java/com/cloud/utils/net/MacAddress.java index d7ac9e39e7fe..0c0191a6a6ce 100644 --- a/utils/src/main/java/com/cloud/utils/net/MacAddress.java +++ b/utils/src/main/java/com/cloud/utils/net/MacAddress.java @@ -19,25 +19,18 @@ package com.cloud.utils.net; -import static com.cloud.utils.AutoCloseableUtil.closeAutoCloseable; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.SocketException; import java.net.UnknownHostException; +import java.util.Enumeration; import java.util.Formatter; -import org.apache.log4j.Logger; - /** * This class retrieves the (first) MAC address for the machine is it is loaded on and stores it statically for retrieval. * It can also be used for formatting MAC addresses. - * copied fnd addpeted rom the public domain utility from John Burkard. **/ public class MacAddress { - private static final Logger s_logger = Logger.getLogger(MacAddress.class); private long _addr = 0; protected MacAddress() { @@ -79,209 +72,45 @@ public String toString() { static { String macAddress = null; - Process p = null; - BufferedReader in = null; - try { - String osname = System.getProperty("os.name"); - - if (osname.startsWith("Windows")) { - p = Runtime.getRuntime().exec(new String[] {"ipconfig", "/all"}, null); - } else if (osname.startsWith("Solaris") || osname.startsWith("SunOS")) { - // Solaris code must appear before the generic code - String hostName = MacAddress.getFirstLineOfCommand(new String[] {"uname", "-n"}); - if (hostName != null) { - p = Runtime.getRuntime().exec(new String[] {"/usr/sbin/arp", hostName}, null); - } - } else if (new File("/usr/sbin/lanscan").exists()) { - p = Runtime.getRuntime().exec(new String[] {"/usr/sbin/lanscan"}, null); - } else if (new File("/sbin/ifconfig").exists()) { - p = Runtime.getRuntime().exec(new String[] {"/sbin/ifconfig", "-a"}, null); - } - - if (p != null) { - in = new BufferedReader(new InputStreamReader(p.getInputStream()), 128); - String l = null; - while ((l = in.readLine()) != null) { - macAddress = MacAddress.parse(l); - if (macAddress != null) { - short parsedShortMacAddress = MacAddress.parseShort(macAddress); - if (parsedShortMacAddress != 0xff && parsedShortMacAddress != 0x00) - break; - } - macAddress = null; - } - } - - } catch (SecurityException ex) { - s_logger.info("[ignored] security exception in static initializer of MacAddress", ex); - } catch (IOException ex) { - s_logger.info("[ignored] io exception in static initializer of MacAddress"); - } finally { - if (p != null) { - closeAutoCloseable(in, "closing init process input stream"); - closeAutoCloseable(p.getErrorStream(), "closing init process error output stream"); - closeAutoCloseable(p.getOutputStream(), "closing init process std output stream"); - p.destroy(); - } + Enumeration networkInterfaces = NetworkInterface.getNetworkInterfaces(); + while (networkInterfaces.hasMoreElements()) { + NetworkInterface network = networkInterfaces.nextElement(); + final byte [] mac = network.getHardwareAddress(); + if (mac != null && !network.isVirtual() && + !network.getName().startsWith("br-") && + !network.getName().startsWith("veth") && + !network.getName().startsWith("vnet")) { + StringBuilder macAddressBuilder = new StringBuilder(); + for (byte b : mac) { + macAddressBuilder.append(String.format("%02X", b)); + } + macAddress = macAddressBuilder.toString(); + } + } + } catch (SocketException ignore) { } - long clockSeqAndNode = 0; + long macAddressInLong = 0; if (macAddress != null) { - if (macAddress.indexOf(':') != -1) { - clockSeqAndNode |= MacAddress.parseLong(macAddress); - } else if (macAddress.startsWith("0x")) { - clockSeqAndNode |= MacAddress.parseLong(macAddress.substring(2)); - } + macAddressInLong = Long.parseLong(macAddress, 16); } else { try { byte[] local = InetAddress.getLocalHost().getAddress(); - clockSeqAndNode |= (local[0] << 24) & 0xFF000000L; - clockSeqAndNode |= (local[1] << 16) & 0xFF0000; - clockSeqAndNode |= (local[2] << 8) & 0xFF00; - clockSeqAndNode |= local[3] & 0xFF; + macAddressInLong |= (local[0] << 24) & 0xFF000000L; + macAddressInLong |= (local[1] << 16) & 0xFF0000; + macAddressInLong |= (local[2] << 8) & 0xFF00; + macAddressInLong |= local[3] & 0xFF; } catch (UnknownHostException ex) { - clockSeqAndNode |= (long)(Math.random() * 0x7FFFFFFF); + macAddressInLong |= (long)(Math.random() * 0x7FFFFFFF); } } - s_address = new MacAddress(clockSeqAndNode); + s_address = new MacAddress(macAddressInLong); } public static MacAddress getMacAddress() { return s_address; } - - private static String getFirstLineOfCommand(String[] commands) throws IOException { - - Process p = null; - BufferedReader reader = null; - - try { - p = Runtime.getRuntime().exec(commands); - reader = new BufferedReader(new InputStreamReader(p.getInputStream()), 128); - - return reader.readLine(); - } finally { - if (p != null) { - closeAutoCloseable(reader, "closing process input stream"); - closeAutoCloseable(p.getErrorStream(), "closing process error output stream"); - closeAutoCloseable(p.getOutputStream(), "closing process std output stream"); - p.destroy(); - } - } - - } - - /** - * The MAC address parser attempts to find the following patterns: - *
    - *
  • .{1,2}:.{1,2}:.{1,2}:.{1,2}:.{1,2}:.{1,2}
    • - *
  • .{1,2}-.{1,2}-.{1,2}-.{1,2}-.{1,2}-.{1,2}
    • - *
- * - * This is copied from the author below. The author encouraged copying - * it. - * - */ - static String parse(String in) { - - // lanscan - - int hexStart = in.indexOf("0x"); - if (hexStart != -1) { - int hexEnd = in.indexOf(' ', hexStart); - if (hexEnd != -1) { - return in.substring(hexStart, hexEnd); - } - } - - int octets = 0; - int lastIndex, old, end; - - if (in.indexOf('-') > -1) { - in = in.replace('-', ':'); - } - - lastIndex = in.lastIndexOf(':'); - - if (lastIndex > in.length() - 2) - return null; - - end = Math.min(in.length(), lastIndex + 3); - - ++octets; - old = lastIndex; - while (octets != 5 && lastIndex != -1 && lastIndex > 1) { - lastIndex = in.lastIndexOf(':', --lastIndex); - if (old - lastIndex == 3 || old - lastIndex == 2) { - ++octets; - old = lastIndex; - } - } - - if (octets == 5 && lastIndex > 1) { - return in.substring(lastIndex - 2, end).trim(); - } - return null; - } - - /** - * Parses a long from a hex encoded number. This method will skip - * all characters that are not 0-9 and a-f (the String is lower cased first). - * Returns 0 if the String does not contain any interesting characters. - * - * @param s the String to extract a long from, may not be null - * @return a long - * @throws NullPointerException if the String is null - */ - private static long parseLong(String s) throws NullPointerException { - s = s.toLowerCase(); - long out = 0; - byte shifts = 0; - char c; - for (int i = 0; i < s.length() && shifts < 16; i++) { - c = s.charAt(i); - if ((c > 47) && (c < 58)) { - out <<= 4; - ++shifts; - out |= c - 48; - } else if ((c > 96) && (c < 103)) { - ++shifts; - out <<= 4; - out |= c - 87; - } - } - return out; - } - - /** - * Parses a short from a hex encoded number. This method will skip - * all characters that are not 0-9 and a-f (the String is lower cased first). - * Returns 0 if the String does not contain any interesting characters. - * - * @param s the String to extract a short from, may not be null - * @return a short - * @throws NullPointerException if the String is null - */ - private static short parseShort(String s) throws NullPointerException { - s = s.toLowerCase(); - short out = 0; - byte shifts = 0; - char c; - for (int i = 0; i < s.length() && shifts < 4; i++) { - c = s.charAt(i); - if ((c > 47) && (c < 58)) { - out <<= 4; - ++shifts; - out |= c - 48; - } else if ((c > 96) && (c < 103)) { - ++shifts; - out <<= 4; - out |= c - 87; - } - } - return out; - } } diff --git a/utils/src/test/java/com/cloud/utils/net/MacAddressTest.java b/utils/src/test/java/com/cloud/utils/net/MacAddressTest.java index 42b03b70daff..ff33ed03f004 100644 --- a/utils/src/test/java/com/cloud/utils/net/MacAddressTest.java +++ b/utils/src/test/java/com/cloud/utils/net/MacAddressTest.java @@ -41,14 +41,14 @@ public final void testMacAddressLong() throws Exception { public final void testMacAddressToLong() throws Exception { // TODO this test should fail this address is beyond the acceptable range for macaddresses MacAddress mac = new MacAddress(Long.MAX_VALUE); - assertEquals(Long.MAX_VALUE,mac.toLong()); + assertEquals(Long.MAX_VALUE, mac.toLong()); System.out.println(mac.toString()); } - // TODO public final void testToLong() throws Exception { - // TODO public final void testToByteArray() throws Exception { - // TODO public final void testToStringString() throws Exception { - // TODO public final void testToString() throws Exception { - // TODO public final void testGetMacAddress() throws Exception { - // TODO public final void testParse() throws Exception { + @Test + public final void testSpecificMacAddress() throws Exception { + // Test specific mac address 76:3F:76:EB:02:81 + MacAddress mac = new MacAddress(130014950130305L); + assertEquals("76:3f:76:eb:02:81", mac.toString()); + } } From 9830bbe6614800461dc84c1a6a8f85e86b1216b3 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Wed, 24 Apr 2024 17:21:31 +0530 Subject: [PATCH 2/8] framework/db: introduce a new MySQL table based distributed lock This introduces a MySQL innodb table based distributed lock which can be used by one or more management server and its threads. This removes usage of MySQL server provided locking functions (GET_LOCK, RELEASE_LOCK) which are not replicated or supported currently by any MySQL clustering solutions. This would be the first main step in having CloudStack to work with a MySQL clustering solution such as InnoDB cluster, Percona Xtradb cluster, MariaDB galera cluster. There may be other changes required which can be found in due course if this feature works at scale. Signed-off-by: Rohit Yadav --- .../cloud/upgrade/DatabaseUpgradeChecker.java | 35 +++++++++++ .../main/java/com/cloud/utils/db/DbUtil.java | 59 ++++++++++--------- .../test/java/com/cloud/utils/DbUtilTest.java | 23 +++----- setup/db/create-schema.sql | 1 + 4 files changed, 76 insertions(+), 42 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java index 614e6058aa67..b394db7ae17e 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java @@ -35,6 +35,7 @@ import com.cloud.utils.FileUtil; import org.apache.cloudstack.utils.CloudStackVersion; +import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; @@ -398,6 +399,7 @@ protected void executeViewScripts() { @Override public void check() { + initDistributedLock(); GlobalLock lock = GlobalLock.getInternLock("DatabaseUpgrade"); try { s_logger.info("Grabbing lock to check for database upgrade."); @@ -441,6 +443,39 @@ public void check() { } } + private void initDistributedLock() { + s_logger.info("Setting up distributed lock table if not created."); + TransactionLegacy txn = TransactionLegacy.open("initDistributedLock"); + txn.start(); + String errorMessage = "Unable to get the database connections"; + try { + Connection conn = txn.getConnection(); + errorMessage = "Unable to create distributed_lock table in the 'cloud' database "; + String sql = "CREATE TABLE IF NOT EXISTS `cloud`.`distributed_lock` (" + + " `name` varchar(1024) NOT NULL," + + " `thread` varchar(1024) NOT NULL," + + " `ms_id` bigint NOT NULL, `pid` int NOT NULL," + + " `created` datetime DEFAULT NULL," + + " PRIMARY KEY (`name`)," + + " UNIQUE KEY `name` (`name`)" + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8"; + try (PreparedStatement pstmt = conn.prepareStatement(sql)) { + pstmt.execute(); + } + try (PreparedStatement pstmt = conn.prepareStatement("DELETE FROM cloud.distributed_lock WHERE ms_id=?")) { + pstmt.setLong(1, ManagementServerNode.getManagementServerId()); + pstmt.execute(); + } + txn.commit(); + } catch (CloudRuntimeException | SQLException e) { + s_logger.error(e.getMessage()); + errorMessage = String.format("%s due to %s.", errorMessage, e.getMessage()); + throw new CloudRuntimeException(errorMessage, e); + } finally { + txn.close(); + } + } + private void initializeDatabaseEncryptors() { TransactionLegacy txn = TransactionLegacy.open("initializeDatabaseEncryptors"); txn.start(); diff --git a/framework/db/src/main/java/com/cloud/utils/db/DbUtil.java b/framework/db/src/main/java/com/cloud/utils/db/DbUtil.java index 68424bc5dd9e..93ed1720a5b3 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/DbUtil.java +++ b/framework/db/src/main/java/com/cloud/utils/db/DbUtil.java @@ -41,6 +41,7 @@ import javax.persistence.Table; import javax.persistence.Transient; +import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.log4j.Logger; import static com.cloud.utils.AutoCloseableUtil.closeAutoCloseable; @@ -198,28 +199,36 @@ public static final String getTableName(Class clazz) { public static boolean getGlobalLock(String name, int timeoutSeconds) { Connection conn = getConnectionForGlobalLocks(name, true); if (conn == null) { - LOGGER.error("Unable to acquire DB connection for global lock system"); + LOGGER.error("Unable to acquire DB connection for distributed lock: " + name); return false; } - try (PreparedStatement pstmt = conn.prepareStatement("SELECT COALESCE(GET_LOCK(?, ?),0)");) { - pstmt.setString(1, name); - pstmt.setInt(2, timeoutSeconds); - - try (ResultSet rs = pstmt.executeQuery();) { - if (rs != null && rs.first()) { - if (rs.getInt(1) > 0) { - return true; - } else { - if (LOGGER.isDebugEnabled()) - LOGGER.debug("GET_LOCK() timed out on lock : " + name); - } + int remainingTime = timeoutSeconds; + while (remainingTime > 0) { + try (PreparedStatement pstmt = conn.prepareStatement( + "INSERT INTO cloud.distributed_lock (name, thread, ms_id, pid, created) " + + "VALUES (?, ?, ?, ?, now()) ON DUPLICATE KEY UPDATE name=name")) { + pstmt.setString(1, name); + pstmt.setString(2, Thread.currentThread().getName()); + pstmt.setLong(3, ManagementServerNode.getManagementServerId()); + pstmt.setLong(4, ProcessHandle.current().pid()); + if (pstmt.executeUpdate() > 0) { + return true; } + } catch (SQLException e) { + LOGGER.error("Inserting to cloud.distributed_lock query threw exception ", e); + } catch (Throwable e) { + LOGGER.error("Inserting to cloud.distributed_lock query threw exception ", e); + } + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Waiting, cloud.distributed_lock already has the lock: " + name); + } + remainingTime = remainingTime - 1; + try { + Thread.sleep(1000L); + } catch (InterruptedException ignore) { } - } catch (SQLException e) { - LOGGER.error("GET_LOCK() throws exception ", e); - } catch (Throwable e) { - LOGGER.error("GET_LOCK() throws exception ", e); } removeConnectionForGlobalLocks(name); @@ -234,24 +243,20 @@ public static Class getEntityBeanType(GenericDao dao) { public static boolean releaseGlobalLock(String name) { try (Connection conn = getConnectionForGlobalLocks(name, false);) { if (conn == null) { - LOGGER.error("Unable to acquire DB connection for global lock system"); + LOGGER.error("Unable to acquire DB connection for distributed lock: " + name); assert (false); return false; } - try (PreparedStatement pstmt = conn.prepareStatement("SELECT COALESCE(RELEASE_LOCK(?), 0)");) { + try (PreparedStatement pstmt = conn.prepareStatement("DELETE FROM cloud.distributed_lock WHERE name=?")) { pstmt.setString(1, name); - try (ResultSet rs = pstmt.executeQuery();) { - if (rs != null && rs.first()) { - return rs.getInt(1) > 0; - } - LOGGER.error("releaseGlobalLock:RELEASE_LOCK() returns unexpected result"); + if (pstmt.executeUpdate() > 0) { + return true; } + LOGGER.warn("releaseGlobalLock: failed to remove cloud.distributed_lock lock which does not exist: " + name); } - } catch (SQLException e) { - LOGGER.error("RELEASE_LOCK() throws exception ", e); } catch (Throwable e) { - LOGGER.error("RELEASE_LOCK() throws exception ", e); + LOGGER.error("Removing cloud.distributed_lock lock threw exception ", e); } return false; } diff --git a/framework/db/src/test/java/com/cloud/utils/DbUtilTest.java b/framework/db/src/test/java/com/cloud/utils/DbUtilTest.java index a2153fee506c..892b214038de 100644 --- a/framework/db/src/test/java/com/cloud/utils/DbUtilTest.java +++ b/framework/db/src/test/java/com/cloud/utils/DbUtilTest.java @@ -151,28 +151,24 @@ public void cleanup() throws SecurityException, NoSuchFieldException, IllegalArg public void getGlobalLock() throws SQLException { Mockito.when(dataSource.getConnection()).thenReturn(connection); Mockito.when(connection.prepareStatement(Matchers.anyString())).thenReturn(preparedStatement); - Mockito.when(preparedStatement.executeQuery()).thenReturn(resultSet); - Mockito.when(resultSet.first()).thenReturn(true); - Mockito.when(resultSet.getInt(1)).thenReturn(1); + Mockito.when(preparedStatement.executeUpdate()).thenReturn(1); Assert.assertTrue(DbUtil.getGlobalLock("TEST", 600)); Mockito.verify(connection).prepareStatement(Matchers.anyString()); Mockito.verify(preparedStatement).close(); - Mockito.verify(resultSet).close(); } @Test public void getGlobalLockTimeout() throws SQLException { Mockito.when(dataSource.getConnection()).thenReturn(connection); Mockito.when(connection.prepareStatement(Matchers.anyString())).thenReturn(preparedStatement); - Mockito.when(preparedStatement.executeQuery()).thenReturn(resultSet); - Mockito.when(resultSet.first()).thenReturn(true); - Mockito.when(resultSet.getInt(1)).thenReturn(0); - Assert.assertFalse(DbUtil.getGlobalLock("TEST", 600)); + Mockito.when(preparedStatement.executeUpdate()).thenReturn(0); - Mockito.verify(connection).prepareStatement(Matchers.anyString()); - Mockito.verify(preparedStatement).close(); - Mockito.verify(resultSet).close(); + final int tries = 2; + Assert.assertFalse(DbUtil.getGlobalLock("TEST", tries)); + + Mockito.verify(connection, Mockito.times(tries)).prepareStatement(Matchers.anyString()); + Mockito.verify(preparedStatement, Mockito.times(tries)).close(); Mockito.verify(connection).close(); // if any error happens, the connection map must be cleared @@ -238,13 +234,10 @@ void releaseGlobalLockNotexisting() throws SQLException { @Test public void releaseGlobalLock() throws SQLException { Mockito.when(connection.prepareStatement(Matchers.anyString())).thenReturn(preparedStatement); - Mockito.when(preparedStatement.executeQuery()).thenReturn(resultSet); - Mockito.when(resultSet.first()).thenReturn(true); - Mockito.when(resultSet.getInt(1)).thenReturn(1); + Mockito.when(preparedStatement.executeUpdate()).thenReturn(1); connectionMap.put("testLock", connection); Assert.assertTrue(DbUtil.releaseGlobalLock("testLock")); - Mockito.verify(resultSet).close(); Mockito.verify(preparedStatement).close(); Mockito.verify(connection).close(); Assert.assertFalse(connectionMap.containsKey("testLock")); diff --git a/setup/db/create-schema.sql b/setup/db/create-schema.sql index 3f14fccd0103..9b31bbe6a037 100755 --- a/setup/db/create-schema.sql +++ b/setup/db/create-schema.sql @@ -195,6 +195,7 @@ DROP TABLE IF EXISTS `cloud`.`image_data_store`; DROP TABLE IF EXISTS `cloud`.`vm_compute_tags`; DROP TABLE IF EXISTS `cloud`.`vm_root_disk_tags`; DROP TABLE IF EXISTS `cloud`.`vm_network_map`; +DROP TABLE IF EXISTS `cloud`.`distributed_lock`; CREATE TABLE `cloud`.`version` ( `id` bigint unsigned NOT NULL UNIQUE AUTO_INCREMENT COMMENT 'id', From 5b8c24edec11469d2a7b129c4917c1b82589409b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 26 Dec 2024 12:32:32 +0530 Subject: [PATCH 3/8] test fix Signed-off-by: Abhishek Kumar --- .../db/src/test/java/com/cloud/utils/DbUtilTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/framework/db/src/test/java/com/cloud/utils/DbUtilTest.java b/framework/db/src/test/java/com/cloud/utils/DbUtilTest.java index 0237bc3cd813..ce9f1269044d 100644 --- a/framework/db/src/test/java/com/cloud/utils/DbUtilTest.java +++ b/framework/db/src/test/java/com/cloud/utils/DbUtilTest.java @@ -150,7 +150,7 @@ public void cleanup() throws SecurityException, NoSuchFieldException, IllegalArg @Test public void getGlobalLock() throws SQLException { Mockito.when(dataSource.getConnection()).thenReturn(connection); - Mockito.when(connection.prepareStatement(Matchers.anyString())).thenReturn(preparedStatement); + Mockito.when(connection.prepareStatement(Mockito.anyString())).thenReturn(preparedStatement); Mockito.when(preparedStatement.executeUpdate()).thenReturn(1); Assert.assertTrue(DbUtil.getGlobalLock("TEST", 600)); @@ -163,13 +163,13 @@ public void getGlobalLock() throws SQLException { public void getGlobalLockTimeout() throws SQLException { Mockito.when(dataSource.getConnection()).thenReturn(connection); - Mockito.when(connection.prepareStatement(Matchers.anyString())).thenReturn(preparedStatement); + Mockito.when(connection.prepareStatement(Mockito.anyString())).thenReturn(preparedStatement); Mockito.when(preparedStatement.executeUpdate()).thenReturn(0); final int tries = 2; Assert.assertFalse(DbUtil.getGlobalLock("TEST", tries)); - Mockito.verify(connection, Mockito.times(tries)).prepareStatement(Matchers.anyString()); + Mockito.verify(connection, Mockito.times(tries)).prepareStatement(Mockito.anyString()); Mockito.verify(preparedStatement, Mockito.times(tries)).close(); Mockito.verify(connection).close(); @@ -235,7 +235,7 @@ void releaseGlobalLockNotexisting() throws SQLException { @Test public void releaseGlobalLock() throws SQLException { - Mockito.when(connection.prepareStatement(Matchers.anyString())).thenReturn(preparedStatement); + Mockito.when(connection.prepareStatement(Mockito.anyString())).thenReturn(preparedStatement); Mockito.when(preparedStatement.executeUpdate()).thenReturn(1); connectionMap.put("testLock", connection); From b9e28465f729ca269f26a284889f76d8b86a18aa Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 26 Dec 2024 12:57:06 +0530 Subject: [PATCH 4/8] fix build Signed-off-by: Abhishek Kumar --- .../main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java index bfd9a14a59fa..f56b696ee3f9 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java @@ -489,7 +489,7 @@ public void check() { } private void initDistributedLock() { - s_logger.info("Setting up distributed lock table if not created."); + LOGGER.info("Setting up distributed lock table if not created."); TransactionLegacy txn = TransactionLegacy.open("initDistributedLock"); txn.start(); String errorMessage = "Unable to get the database connections"; @@ -513,7 +513,7 @@ private void initDistributedLock() { } txn.commit(); } catch (CloudRuntimeException | SQLException e) { - s_logger.error(e.getMessage()); + LOGGER.error(e.getMessage()); errorMessage = String.format("%s due to %s.", errorMessage, e.getMessage()); throw new CloudRuntimeException(errorMessage, e); } finally { From 3e39c50e1ec7ea03cf544065f9069c1ac0a4c5c1 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Tue, 4 Mar 2025 09:42:40 +0530 Subject: [PATCH 5/8] Update engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java Co-authored-by: Wei Zhou --- .../main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java index f56b696ee3f9..9713e7cf2301 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java @@ -499,7 +499,8 @@ private void initDistributedLock() { String sql = "CREATE TABLE IF NOT EXISTS `cloud`.`distributed_lock` (" + " `name` varchar(1024) NOT NULL," + " `thread` varchar(1024) NOT NULL," + - " `ms_id` bigint NOT NULL, `pid` int NOT NULL," + + " `ms_id` bigint NOT NULL," + + " `pid` int NOT NULL," + " `created` datetime DEFAULT NULL," + " PRIMARY KEY (`name`)," + " UNIQUE KEY `name` (`name`)" + From b3c5bd4089a7e40d6901d2a8aa7c8b1dd782314c Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Tue, 4 Mar 2025 09:45:47 +0530 Subject: [PATCH 6/8] Update engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java Co-authored-by: Wei Zhou --- .../src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java | 1 + 1 file changed, 1 insertion(+) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java index 9713e7cf2301..f3feb156891b 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java @@ -508,6 +508,7 @@ private void initDistributedLock() { try (PreparedStatement pstmt = conn.prepareStatement(sql)) { pstmt.execute(); } + LOGGER.info("Deleting existing distributed locks with ms_id = " + ManagementServerNode.getManagementServerId()); try (PreparedStatement pstmt = conn.prepareStatement("DELETE FROM cloud.distributed_lock WHERE ms_id=?")) { pstmt.setLong(1, ManagementServerNode.getManagementServerId()); pstmt.execute(); From d15c67251daf9da53aa0c4179959408437589165 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Wed, 5 Mar 2025 10:37:05 +0530 Subject: [PATCH 7/8] REMOVE-ME: added dummy global lock bg-task for checking feature This is only for testing purposes to see things in logs Signed-off-by: Rohit Yadav --- .../apache/cloudstack/ca/CAManagerImpl.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java b/server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java index 22f8939e7eb7..120ab742b2d2 100644 --- a/server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java @@ -40,6 +40,7 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; +import com.cloud.utils.db.GlobalLock; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.admin.ca.IssueCertificateCmd; @@ -295,6 +296,42 @@ public char[] getKeyStorePassphrase() { /////////////// CA Manager Setup /////////////////// //////////////////////////////////////////////////// + public static final class DummyBGTask extends ManagedContextRunnable implements BackgroundPollTask { + + public DummyBGTask() { + } + + @Override + protected void runInContext() { + final GlobalLock lock = GlobalLock.getInternLock("DummyBGTask"); + try { + logger.info("DummyBGTask: Grabbing lock to check for DB HA at " + DateTime.now(DateTimeZone.UTC)); + if (lock.lock(5)) { + try { + logger.info("DummyBGTask: Lock acquired, now sleeping for 5 seconds"); + try { + Thread.sleep(5 * 1000L); + } catch (InterruptedException ignore) { + } + } finally { + logger.info("DummyBGTask: Unlocking now"); + lock.unlock(); + } + } else { + logger.info("DummyBGTask: Could not get lock"); + } + logger.info("DummyBGTask: One round of lock-grab for DB HA over at " + DateTime.now(DateTimeZone.UTC)); + } finally { + lock.releaseRef(); + } + } + + @Override + public Long getDelay() { + return 2000L; + } + } + public static final class CABackgroundTask extends ManagedContextRunnable implements BackgroundPollTask { private CAManager caManager; private HostDao hostDao; @@ -405,6 +442,7 @@ public boolean start() { @Override public boolean configure(final String name, final Map params) throws ConfigurationException { + backgroundPollManager.submitTask(new DummyBGTask()); backgroundPollManager.submitTask(new CABackgroundTask(this, hostDao)); return true; } From 4b0d0fe3475a8bbb4a8f4ef25af4a7582aa1bca4 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 6 Mar 2025 16:41:38 +0530 Subject: [PATCH 8/8] REMOVE-ME: bump timeout & sleep Signed-off-by: Rohit Yadav --- .../java/org/apache/cloudstack/ca/CAManagerImpl.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java b/server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java index 120ab742b2d2..d494d0b22c7f 100644 --- a/server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java @@ -305,12 +305,12 @@ public DummyBGTask() { protected void runInContext() { final GlobalLock lock = GlobalLock.getInternLock("DummyBGTask"); try { - logger.info("DummyBGTask: Grabbing lock to check for DB HA at " + DateTime.now(DateTimeZone.UTC)); - if (lock.lock(5)) { + logger.info("DummyBGTask: Trying to get lock at " + DateTime.now(DateTimeZone.UTC)); + if (lock.lock(50)) { try { logger.info("DummyBGTask: Lock acquired, now sleeping for 5 seconds"); try { - Thread.sleep(5 * 1000L); + Thread.sleep(10 * 1000L); } catch (InterruptedException ignore) { } } finally { @@ -318,9 +318,9 @@ protected void runInContext() { lock.unlock(); } } else { - logger.info("DummyBGTask: Could not get lock"); + logger.info("DummyBGTask: Could not get lock, retrying..."); } - logger.info("DummyBGTask: One round of lock-grab for DB HA over at " + DateTime.now(DateTimeZone.UTC)); + logger.info("DummyBGTask: One round of lock-grab over at " + DateTime.now(DateTimeZone.UTC)); } finally { lock.releaseRef(); } @@ -328,7 +328,7 @@ protected void runInContext() { @Override public Long getDelay() { - return 2000L; + return 5000L; } }