Adding Count Resource Domain

.github/workflows/ci.yml

@@ -187,7 +187,8 @@ jobs:
                   component/test_vpc_offerings
                   component/test_vpc_routers
                   component/test_vpn_users",
-                "component/test_vpc_network_lbrules" ]
+                "component/test_vpc_network_lbrules",
+                "component/test_router_resources"]
 
     steps:
       - uses: actions/checkout@v3

api/src/main/java/com/cloud/user/ResourceLimitService.java

@@ -23,6 +23,7 @@
 import com.cloud.configuration.ResourceLimit;
 import com.cloud.domain.Domain;
 import com.cloud.exception.ResourceAllocationException;
+import com.cloud.offering.ServiceOffering;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.user.ResourceReservation;
 
@@ -210,4 +211,10 @@ public interface ResourceLimitService {
      */
     ResourceReservation getReservation(Account account, Boolean displayResource, ResourceType type, Long delta) throws ResourceAllocationException;
 
+    /**
+     * Returns the service offering by the given configuration.
+     *
+     * @return the service offering found or null if not found
+     */
+    ServiceOffering getServiceOfferingByConfig();
 }

engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java

@@ -22,6 +22,7 @@
 import java.util.List;
 import java.util.Map;
 
+import com.cloud.utils.exception.CloudRuntimeException;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.framework.config.ConfigKey;
 
@@ -83,10 +84,21 @@ public interface VirtualMachineManager extends Manager {
     ConfigKey<Boolean> AllowExposeDomainInMetadata = new ConfigKey<>("Advanced", Boolean.class, "metadata.allow.expose.domain",
             "false", "If set to true, it allows the VM's domain to be seen in metadata.", true, ConfigKey.Scope.Domain);
 
+    ConfigKey<Boolean> ResourceCountRouters = new ConfigKey<>("Advanced", Boolean.class, "resource.count.routers",
+            "false","Count the CPU and memory resource count of virtual routers towards domain resource calculation",
+            true, ConfigKey.Scope.Domain);
+
+    ConfigKey<String> ResourceCountRoutersType = new ConfigKey<>("Advanced", String.class, "resource.count.routers.type", "all",
+            "Possible values are all and delta. If value is all then entire VR cpu and ram are counted else diff " +
+                    "between current VR offering and default VR offering is considered", true, ConfigKey.Scope.Domain);
+
     interface Topics {
         String VM_POWER_STATE = "vm.powerstate";
     }
 
+    static final String COUNT_ALL_VR_RESOURCES = "all";
+    static final String COUNT_DELTA_VR_RESOURCES = "delta";
+
     /**
      * Allocates a new virtual machine instance in the CloudStack DB.  This
      * orchestrates the creation of all virtual resources needed in CloudStack
@@ -97,8 +109,7 @@ interface Topics {
      *        define this VM but it must be unique for all of CloudStack.
      * @param template The template this VM is based on.
      * @param serviceOffering The service offering that specifies the offering this VM should provide.
-     * @param defaultNetwork The default network for the VM.
-     * @param rootDiskOffering For created VMs not based on templates, root disk offering specifies the root disk.
+     * @param rootDiskOfferingInfo For created VMs not based on templates, root disk offering specifies the root disk.
      * @param dataDiskOfferings Data disks to attach to the VM.
      * @param auxiliaryNetworks additional networks to attach the VMs to.
      * @param plan How to deploy the VM.
@@ -173,9 +184,9 @@ void advanceReboot(String vmUuid, Map<VirtualMachineProfile.Param, Object> param
     void checkIfCanUpgrade(VirtualMachine vmInstance, ServiceOffering newServiceOffering);
 
     /**
-     * @param vmId
-     * @param serviceOfferingId
-     * @return
+     * @param vmId the vm id
+     * @param newServiceOffering the new service offering
+     * @return true if the vm db was upgraded, false otherwise
      */
     boolean upgradeVmDb(long vmId, ServiceOffering newServiceOffering, ServiceOffering currentServiceOffering);
 
@@ -219,9 +230,8 @@ NicProfile addVmToNetwork(VirtualMachine vm, Network network, NicProfile request
     NicTO toNicTO(NicProfile nic, HypervisorType hypervisorType);
 
     /**
-     * @param profile
-     * @param hvGuru
-     * @return
+     * @param profile the vm profile
+     * @return the vmTO
      */
     VirtualMachineTO toVmTO(VirtualMachineProfile profile);
 
@@ -285,4 +295,7 @@ static String getHypervisorHostname(String name) {
 
     HashMap<Long, List<? extends VmNetworkStats>> getVmNetworkStatistics(long hostId, String hostName, Map<Long, ? extends VirtualMachine> vmMap);
 
+    void incrementVrResourceCount(ServiceOffering offering, Account owner, boolean isDeployOrDestroy) throws CloudRuntimeException;
+
+    void decrementVrResourceCount(ServiceOffering offering, Account owner, boolean isDeployOrDestroy) throws CloudRuntimeException;
 }

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -27,7 +27,6 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -47,6 +46,7 @@
 import javax.naming.ConfigurationException;
 import javax.persistence.EntityExistsException;
 
+import com.cloud.exception.ResourceAllocationException;
 import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
 import org.apache.cloudstack.annotation.AnnotationService;
 import org.apache.cloudstack.annotation.dao.AnnotationDao;
@@ -1095,7 +1095,13 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
         // check resource count if ResourceCountRunningVMsonly.value() = true
         final Account owner = _entityMgr.findById(Account.class, vm.getAccountId());
         if (VirtualMachine.Type.User.equals(vm.type) && ResourceCountRunningVMsonly.value()) {
-            resourceCountIncrement(owner.getAccountId(),new Long(offering.getCpu()), new Long(offering.getRamSize()));
+            resourceCountIncrement(owner.getAccountId(),Long.valueOf(offering.getCpu()), Long.valueOf(offering.getRamSize()));
+        }
+
+        // Increment the VR resource count if the global setting is set to true
+        // and resource.count.running.vms is also true
+        if (VirtualMachine.Type.DomainRouter.equals(vm.type) && ResourceCountRouters.valueIn(owner.getDomainId())) {
+            incrementVrResourceCount(offering, owner, false);
         }
 
         boolean canRetry = true;
@@ -1402,8 +1408,11 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
         } finally {
             if (startedVm == null) {
                 if (VirtualMachine.Type.User.equals(vm.type) && ResourceCountRunningVMsonly.value()) {
-                    resourceCountDecrement(owner.getAccountId(),new Long(offering.getCpu()), new Long(offering.getRamSize()));
+                    resourceCountDecrement(owner.getAccountId(),Long.valueOf(offering.getCpu()), Long.valueOf(offering.getRamSize()));
                 }
+
+                updateVrCountResourceBy(vm.type, owner.getDomainId(), offering, owner, true);
+
                 if (canRetry) {
                     try {
                         changeState(vm, Event.OperationFailed, null, work, Step.Done);
@@ -1455,15 +1464,131 @@ private void logBootModeParameters(Map<VirtualMachineProfile.Param, Object> para
         }
     }
 
+    /**
+     * Counts VR resources for the domain if global setting is true.
+     * If the value is "all", counts all VR resources, otherwise get the difference between
+     * current VR offering and default VR offering.
+     *
+     * @param offering VR service offering
+     * @param defaultRouterOffering default VR service offering
+     * @param owner account
+     * @return a Pair of CPU and RAM
+     */
+    public Pair<Long, Long> resolveCpuAndMemoryCount(ServiceOffering offering, ServiceOffering defaultRouterOffering, Account owner) {
+        Integer cpuCount = 0;
+        Integer memoryCount = 0;
+        if (COUNT_ALL_VR_RESOURCES.equalsIgnoreCase(ResourceCountRoutersType.valueIn(owner.getDomainId()))) {
+            cpuCount = offering.getCpu();
+            memoryCount = offering.getRamSize();
+        } else if (COUNT_DELTA_VR_RESOURCES.equalsIgnoreCase(ResourceCountRoutersType.valueIn(owner.getDomainId()))) {
+            // Default offering value can be greater than current offering value
+            if (offering.getCpu() >= defaultRouterOffering.getCpu()) {
+                cpuCount = offering.getCpu() - defaultRouterOffering.getCpu();
+            }
+            if (offering.getRamSize() >= defaultRouterOffering.getRamSize()) {
+                memoryCount = offering.getRamSize() - defaultRouterOffering.getRamSize();
+            }
+        }
+
+        return Pair.of(cpuCount.longValue(), memoryCount.longValue());
+    }
+
+    private void validateResourceCount(Pair<Long, Long> cpuMemoryCount, Account owner) {
+        final Long cpuCount = cpuMemoryCount.first();
+        final Long memoryCount = cpuMemoryCount.second();
+        try {
+            if (cpuCount > 0) {
+                _resourceLimitMgr.checkResourceLimit(owner, ResourceType.cpu, cpuCount);
+            }
+            if (memoryCount > 0) {
+                _resourceLimitMgr.checkResourceLimit(owner, ResourceType.memory, memoryCount);
+            }
+        } catch (ResourceAllocationException ex) {
+            throw new CloudRuntimeException(String.format("Unable to deploy/start routers due to %s.", ex.getMessage()));
+        }
+    }
+
+    /**
+     * Checks if resource count can be allocated to the account/domain.
+     *
+     * @param cpuMemoryCount a Pair of cpu and ram
+     * @param owner the account
+     */
+    public void calculateResourceCount(Pair<Long, Long> cpuMemoryCount, Account owner, boolean isIncrement) {
+        validateResourceCount(cpuMemoryCount, owner);
+        final Long cpuCount = cpuMemoryCount.first();
+        final Long memoryCount = cpuMemoryCount.second();
+
+        // Increment the resource count
+        if (s_logger.isDebugEnabled()) {
+            if (isIncrement) {
+                s_logger.debug(String.format("Incrementing the CPU count with value %s and RAM value with %s.", cpuCount, memoryCount));
+            } else {
+                s_logger.debug(String.format("Decrementing CPU resource count with value %s and memory resource with value %s.",cpuCount, memoryCount));
+            }
+        }
+
+        if(isIncrement) {
+            _resourceLimitMgr.incrementResourceCount(owner.getAccountId(), ResourceType.cpu, cpuCount);
+            _resourceLimitMgr.incrementResourceCount(owner.getAccountId(), ResourceType.memory, memoryCount);
+        } else {
+            _resourceLimitMgr.decrementResourceCount(owner.getAccountId(), ResourceType.cpu, cpuCount);
+            _resourceLimitMgr.decrementResourceCount(owner.getAccountId(), ResourceType.memory, memoryCount);
+        }
+    }
+
+    /**
+     * Increments the VR resource count.
+     * If the global setting resource.count.router is true then the VR
+     * resource count will be considered as well.
+     * If the global setting resource.count.router.type is "all" then
+     * the total VR resource count will be considered, otherwise the difference between
+     * the current VR service offering and the default offering will
+     * be considered.
+     * During router deployment/destroy, we increment the resource
+     * count only if resource.count.running.vms is false, otherwise
+     * we increment it during VR start/stop. Same applies for
+     * decrementing resource count.
+     *
+     * @param offering VR service offering
+     * @param owner account
+     * @param isDeployOrDestroy true if router is being deployed/destroyed
+     */
+    @Override
+    public void incrementVrResourceCount(ServiceOffering offering, Account owner, boolean isDeployOrDestroy) {
+        if (isDeployOrDestroy == Boolean.TRUE.equals(ResourceCountRunningVMsonly.value())) {
+            return;
+        } //
+
+        final ServiceOffering defaultRouterOffering = _resourceLimitMgr.getServiceOfferingByConfig();
+        final Pair<Long, Long> cpuMemoryCount = resolveCpuAndMemoryCount(offering, defaultRouterOffering, owner);
+        calculateResourceCount(cpuMemoryCount, owner, true);
+    }
+
+    /**
+     * Decrements the VR resource count.
+     *
+     * @param offering the service offering of the VR
+     * @param owner the account of the VR
+     * @param isDeployOrDestroy true if the VR is being deployed or destroyed, false if the VR is being started or stopped
+     */
+    @Override
+    public void decrementVrResourceCount(ServiceOffering offering, Account owner, boolean isDeployOrDestroy) {
+        if (isDeployOrDestroy == Boolean.TRUE.equals(ResourceCountRunningVMsonly.value())) {
+            return;
+        }
+
+        final ServiceOffering defaultRouterOffering = _resourceLimitMgr.getServiceOfferingByConfig();
+        final Pair<Long, Long> cpuMemoryCount = resolveCpuAndMemoryCount(offering, defaultRouterOffering, owner);
+        calculateResourceCount(cpuMemoryCount, owner, false);
+    }
+
     private void resetVmNicsDeviceId(Long vmId) {
         final List<NicVO> nics = _nicsDao.listByVmId(vmId);
-        Collections.sort(nics, new Comparator<NicVO>() {
-            @Override
-            public int compare(NicVO nic1, NicVO nic2) {
-                Long nicDevId1 = Long.valueOf(nic1.getDeviceId());
-                Long nicDevId2 = Long.valueOf(nic2.getDeviceId());
-                return nicDevId1.compareTo(nicDevId2);
-            }
+        nics.sort((nic1, nic2) -> {
+            Long nicDevId1 = (long) nic1.getDeviceId();
+            Long nicDevId2 = (long) nic2.getDeviceId();
+            return nicDevId1.compareTo(nicDevId2);
         });
         int deviceId = 0;
         for (final NicVO nic : nics) {
@@ -2151,10 +2276,13 @@ private void advanceStop(final VMInstanceVO vm, final boolean cleanUpEvenIfUnabl
 
             boolean result = stateTransitTo(vm, Event.OperationSucceeded, null);
             if (result) {
+                ServiceOfferingVO offering = _offeringDao.findById(vm.getId(), vm.getServiceOfferingId());
                 if (VirtualMachine.Type.User.equals(vm.type) && ResourceCountRunningVMsonly.value()) {
-                    ServiceOfferingVO offering = _offeringDao.findById(vm.getId(), vm.getServiceOfferingId());
-                    resourceCountDecrement(vm.getAccountId(),new Long(offering.getCpu()), new Long(offering.getRamSize()));
+                    resourceCountDecrement(vm.getAccountId(),Long.valueOf(offering.getCpu()), Long.valueOf(offering.getRamSize()));
                 }
+
+                final Account owner = _entityMgr.findById(Account.class, vm.getAccountId());
+                updateVrCountResourceBy(vm.type, vm.getDomainId(), offering, owner, true);
             } else {
                 throw new CloudRuntimeException("unable to stop " + vm);
             }
@@ -2165,6 +2293,16 @@ private void advanceStop(final VMInstanceVO vm, final boolean cleanUpEvenIfUnabl
         }
     }
 
+    private void updateVrCountResourceBy(VirtualMachine.Type type, long domainId, ServiceOffering offering, Account owner, boolean decrement) {
+        if (VirtualMachine.Type.DomainRouter.equals(type) && Boolean.TRUE.equals(ResourceCountRouters.valueIn(domainId))) {
+            if (decrement) {
+                decrementVrResourceCount(offering, owner, true);
+            } else {
+                incrementVrResourceCount(offering, owner, true);
+            }
+        }
+    }
+
     private void setStateMachine() {
         _stateMachine = VirtualMachine.State.getStateMachine();
     }
@@ -3536,13 +3674,10 @@ protected VirtualMachineTO getVmTO(Long vmId) {
         final VMInstanceVO vm = _vmDao.findById(vmId);
         final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm);
         final List<NicVO> nics = _nicsDao.listByVmId(profile.getId());
-        Collections.sort(nics, new Comparator<NicVO>() {
-            @Override
-            public int compare(NicVO nic1, NicVO nic2) {
-                Long nicId1 = Long.valueOf(nic1.getDeviceId());
-                Long nicId2 = Long.valueOf(nic2.getDeviceId());
-                return nicId1.compareTo(nicId2);
-            }
+        nics.sort((nic1, nic2) -> {
+            Long nicDevId1 = (long) nic1.getDeviceId();
+            Long nicDevId2 = (long) nic2.getDeviceId();
+            return nicDevId1.compareTo(nicDevId2);
         });
 
         for (final NicVO nic : nics) {
@@ -4672,7 +4807,7 @@ public ConfigKey<?>[] getConfigKeys() {
                 VmOpLockStateRetry, VmOpWaitInterval, ExecuteInSequence, VmJobCheckInterval, VmJobTimeout, VmJobStateReportInterval,
                 VmConfigDriveLabel, VmConfigDriveOnPrimaryPool, VmConfigDriveForceHostCacheUse, VmConfigDriveUseHostCacheOnUnsupportedPool,
                 HaVmRestartHostUp, ResourceCountRunningVMsonly, AllowExposeHypervisorHostname, AllowExposeHypervisorHostnameAccountLevel, SystemVmRootDiskSize,
-                AllowExposeDomainInMetadata
+                AllowExposeDomainInMetadata, ResourceCountRouters, ResourceCountRoutersType
         };
     }