UserData as first class resource (#6202)

This PR introduces a new feature to make userdata as a first class resource much like existing SSH keys.

Detailed feature specification document:
https://cwiki.apache.org/confluence/display/CLOUDSTACK/Userdata+as+a+first+class+resource

Author: harikrishna-patnala

api/src/main/java/com/cloud/event/EventTypes.java

@@ -101,6 +101,8 @@ public class EventTypes {
     public static final String EVENT_VM_DYNAMIC_SCALE = "VM.DYNAMIC.SCALE";
     public static final String EVENT_VM_RESETPASSWORD = "VM.RESETPASSWORD";
     public static final String EVENT_VM_RESETSSHKEY = "VM.RESETSSHKEY";
+
+    public static final String EVENT_VM_RESETUSERDATA = "VM.RESETUSERDATA";
     public static final String EVENT_VM_MIGRATE = "VM.MIGRATE";
     public static final String EVENT_VM_MOVE = "VM.MOVE";
     public static final String EVENT_VM_RESTORE = "VM.RESTORE";
@@ -235,6 +237,9 @@ public class EventTypes {
     //registering SSH keypair events
     public static final String EVENT_REGISTER_SSH_KEYPAIR = "REGISTER.SSH.KEYPAIR";
 
+    //registering userdata events
+    public static final String EVENT_REGISTER_USER_DATA = "REGISTER.USER.DATA";
+
     //register for user API and secret keys
     public static final String EVENT_REGISTER_FOR_SECRET_API_KEY = "REGISTER.USER.KEY";
 

api/src/main/java/com/cloud/network/NetworkModel.java

@@ -18,6 +18,7 @@
 package com.cloud.network;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -85,6 +86,9 @@ public interface NetworkModel {
             .put(HYPERVISOR_HOST_NAME_FILE, HYPERVISOR_HOST_NAME_FILE)
             .build();
 
+    List<String> metadataFileNames = new ArrayList<>(Arrays.asList(SERVICE_OFFERING_FILE, AVAILABILITY_ZONE_FILE, LOCAL_HOSTNAME_FILE, LOCAL_IPV4_FILE, PUBLIC_HOSTNAME_FILE, PUBLIC_IPV4_FILE,
+            INSTANCE_ID_FILE, VM_ID_FILE, PUBLIC_KEYS_FILE, CLOUD_IDENTIFIER_FILE, HYPERVISOR_HOST_NAME_FILE));
+
     static final ConfigKey<Integer> MACIdentifier = new ConfigKey<Integer>("Advanced",Integer.class, "mac.identifier", "0",
             "This value will be used while generating the mac addresses for isolated and shared networks. The hexadecimal equivalent value will be present at the 2nd octet of the mac address. Default value is null which means this feature is disabled.Its scope is global.", true, ConfigKey.Scope.Global);
 
@@ -325,7 +329,7 @@ public interface NetworkModel {
 
     boolean getNetworkEgressDefaultPolicy(Long networkId);
 
-    List<String[]> generateVmData(String userData, String serviceOffering, long datacenterId,
+    List<String[]> generateVmData(String userData, String userDataDetails, String serviceOffering, long datacenterId,
                                   String vmName, String vmHostName, long vmId, String vmUuid, String guestIpAddress, String publicKey, String password, Boolean isWindows, String hostname);
 
     String getValidNetworkCidr(Network guestNetwork);

api/src/main/java/com/cloud/server/ManagementService.java

@@ -20,6 +20,7 @@
 import java.util.List;
 import java.util.Map;
 
+import com.cloud.user.UserData;
 import org.apache.cloudstack.api.command.admin.cluster.ListClustersCmd;
 import org.apache.cloudstack.api.command.admin.config.ListCfgsByCmd;
 import org.apache.cloudstack.api.command.admin.config.UpdateHypervisorCapabilitiesCmd;
@@ -56,6 +57,9 @@
 import org.apache.cloudstack.api.command.user.ssh.DeleteSSHKeyPairCmd;
 import org.apache.cloudstack.api.command.user.ssh.ListSSHKeyPairsCmd;
 import org.apache.cloudstack.api.command.user.ssh.RegisterSSHKeyPairCmd;
+import org.apache.cloudstack.api.command.user.userdata.DeleteUserDataCmd;
+import org.apache.cloudstack.api.command.user.userdata.ListUserDataCmd;
+import org.apache.cloudstack.api.command.user.userdata.RegisterUserDataCmd;
 import org.apache.cloudstack.api.command.user.vm.GetVMPasswordCmd;
 import org.apache.cloudstack.api.command.user.vmgroup.UpdateVMGroupCmd;
 import org.apache.cloudstack.config.Configuration;
@@ -333,6 +337,33 @@ public interface ManagementService {
      */
     String generateRandomPassword();
 
+    /**
+     * Search registered userdatas for the logged in user.
+     *
+     * @param cmd
+     *            The api command class.
+     * @return The list of userdatas found.
+     */
+    Pair<List<? extends UserData>, Integer> listUserDatas(ListUserDataCmd cmd);
+
+    /**
+     * Registers a userdata.
+     *
+     * @param cmd
+     *            The api command class.
+     * @return A VO with the registered userdata.
+     */
+    UserData registerUserData(RegisterUserDataCmd cmd);
+
+    /**
+     * Deletes a userdata.
+     *
+     * @param cmd
+     *            The api command class.
+     * @return True on success. False otherwise.
+     */
+    boolean deleteUserData(DeleteUserDataCmd cmd);
+
     /**
      * Search registered key pairs for the logged in user.
      *

api/src/main/java/com/cloud/template/TemplateApiService.java

@@ -40,6 +40,7 @@
 import com.cloud.exception.StorageUnavailableException;
 import com.cloud.user.Account;
 import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.cloudstack.api.command.user.userdata.LinkUserDataToTemplateCmd;
 import org.apache.cloudstack.api.response.GetUploadParamsResponse;
 
 public interface TemplateApiService {
@@ -56,6 +57,8 @@ public interface TemplateApiService {
 
     VirtualMachineTemplate prepareTemplate(long templateId, long zoneId, Long storageId);
 
+
+
     boolean detachIso(long vmId, boolean forced);
 
     boolean attachIso(long isoId, long vmId, boolean forced);
@@ -106,4 +109,6 @@ public interface TemplateApiService {
     VirtualMachineTemplate updateTemplate(UpdateIsoCmd cmd);
 
     VirtualMachineTemplate updateTemplate(UpdateTemplateCmd cmd);
+
+    VirtualMachineTemplate linkUserDataToTemplate(LinkUserDataToTemplateCmd cmd);
 }

api/src/main/java/com/cloud/template/VirtualMachineTemplate.java

@@ -19,6 +19,7 @@
 import java.util.Date;
 import java.util.Map;
 
+import com.cloud.user.UserData;
 import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.api.Identity;
 import org.apache.cloudstack.api.InternalIdentity;
@@ -142,4 +143,9 @@ public enum TemplateFilter {
     Date getUpdated();
 
     boolean isDeployAsIs();
+
+    Long getUserDataId();
+
+    UserData.UserDataOverridePolicy getUserDataOverridePolicy();
+
 }

api/src/main/java/com/cloud/user/UserData.java

@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.user;
+
+import org.apache.cloudstack.acl.ControlledEntity;
+import org.apache.cloudstack.api.Identity;
+import org.apache.cloudstack.api.InternalIdentity;
+
+public interface UserData extends ControlledEntity, InternalIdentity, Identity {
+
+    public enum UserDataOverridePolicy {
+        ALLOWOVERRIDE, APPEND, DENYOVERRIDE
+    }
+
+    String getUserData();
+
+    String getParams();
+}

api/src/main/java/com/cloud/uservm/UserVm.java

@@ -35,6 +35,14 @@ public interface UserVm extends VirtualMachine, ControlledEntity {
 
     void setUserData(String userData);
 
+    void setUserDataId(Long userDataId);
+
+    Long getUserDataId();
+
+    void setUserDataDetails(String userDataDetails);
+
+    String getUserDataDetails();
+
     String getDetail(String name);
 
     void setAccountId(long accountId);

api/src/main/java/com/cloud/vm/UserVmService.java

@@ -29,6 +29,7 @@
 import org.apache.cloudstack.api.command.user.vm.RemoveNicFromVMCmd;
 import org.apache.cloudstack.api.command.user.vm.ResetVMPasswordCmd;
 import org.apache.cloudstack.api.command.user.vm.ResetVMSSHKeyCmd;
+import org.apache.cloudstack.api.command.user.vm.ResetVMUserDataCmd;
 import org.apache.cloudstack.api.command.user.vm.RestoreVMCmd;
 import org.apache.cloudstack.api.command.user.vm.ScaleVMCmd;
 import org.apache.cloudstack.api.command.user.vm.StartVMCmd;
@@ -103,6 +104,8 @@ public interface UserVmService {
      */
     UserVm resetVMSSHKey(ResetVMSSHKeyCmd cmd) throws ResourceUnavailableException, InsufficientCapacityException;
 
+    UserVm resetVMUserData(ResetVMUserDataCmd cmd) throws ResourceUnavailableException, InsufficientCapacityException;
+
     UserVm startVirtualMachine(StartVMCmd cmd) throws StorageUnavailableException, ExecutionException, ConcurrentOperationException, ResourceUnavailableException,
         InsufficientCapacityException, ResourceAllocationException;
 
@@ -146,6 +149,12 @@ UserVm startVirtualMachine(StartVMCmd cmd) throws StorageUnavailableException, E
      *
      *
      *
+     * @param sshKeyPair
+     *            - name of the ssh key pair used to login to the virtual
+     *            machine
+     * @param cpuSpeed
+     * @param memory
+     * @param cpuNumber
      * @param zone
      *            - availability zone for the virtual machine
      * @param serviceOffering
@@ -181,29 +190,25 @@ UserVm startVirtualMachine(StartVMCmd cmd) throws StorageUnavailableException, E
      *            base64 encoded before adding it to the request. Currently only
      *            HTTP GET is supported. Using HTTP GET (via querystring), you
      *            can send up to 2KB of data after base64 encoding
-     * @param sshKeyPair
-     *            - name of the ssh key pair used to login to the virtual
-     *            machine
+     * @param userDataId
+     * @param userDataDetails
      * @param requestedIps
      *            TODO
      * @param defaultIp
      *            TODO
      * @param displayVm
      *            - Boolean flag whether to the display the vm to the end user or not
      * @param affinityGroupIdList
-     * @param cpuSpeed
-     * @param memory
-     * @param cpuNumber
      * @param customId
      * @param dhcpOptionMap
      *           - Maps the dhcp option code and the dhcp value to the network uuid
-     * @return UserVm object if successful.
      * @param dataDiskTemplateToDiskOfferingMap
      *            - Datadisk template to Disk offering Map
      *             an optional parameter that creates additional data disks for the virtual machine
      *             For each of the templates in the map, a data disk will be created from the corresponding
      *             disk offering obtained from the map
      *
+     * @return UserVm object if successful.
      * @throws InsufficientCapacityException
      *             if there is insufficient capacity to deploy the VM.
      * @throws ConcurrentOperationException
@@ -214,11 +219,11 @@ UserVm startVirtualMachine(StartVMCmd cmd) throws StorageUnavailableException, E
      *             available.
      */
     UserVm createBasicSecurityGroupVirtualMachine(DataCenter zone, ServiceOffering serviceOffering, VirtualMachineTemplate template, List<Long> securityGroupIdList,
-        Account owner, String hostName, String displayName, Long diskOfferingId, Long diskSize, String group, HypervisorType hypervisor, HTTPMethod httpmethod,
-        String userData, List<String> sshKeyPairs, Map<Long, IpAddresses> requestedIps, IpAddresses defaultIp, Boolean displayVm, String keyboard,
-        List<Long> affinityGroupIdList, Map<String, String> customParameter, String customId, Map<String, Map<Integer, String>> dhcpOptionMap,
-        Map<Long, DiskOffering> dataDiskTemplateToDiskOfferingMap,
-        Map<String, String> userVmOVFProperties, boolean dynamicScalingEnabled, Long overrideDiskOfferingId) throws InsufficientCapacityException,
+                                                  Account owner, String hostName, String displayName, Long diskOfferingId, Long diskSize, String group, HypervisorType hypervisor, HTTPMethod httpmethod,
+                                                  String userData, Long userDataId, String userDataDetails, List<String> sshKeyPairs, Map<Long, IpAddresses> requestedIps, IpAddresses defaultIp, Boolean displayVm, String keyboard,
+                                                  List<Long> affinityGroupIdList, Map<String, String> customParameter, String customId, Map<String, Map<Integer, String>> dhcpOptionMap,
+                                                  Map<Long, DiskOffering> dataDiskTemplateToDiskOfferingMap,
+                                                  Map<String, String> userVmOVFProperties, boolean dynamicScalingEnabled, Long overrideDiskOfferingId) throws InsufficientCapacityException,
         ConcurrentOperationException, ResourceUnavailableException, StorageUnavailableException, ResourceAllocationException;
 
     /**
@@ -227,6 +232,7 @@ UserVm createBasicSecurityGroupVirtualMachine(DataCenter zone, ServiceOffering s
      *
      *
      *
+     * @param type
      * @param zone
      *            - availability zone for the virtual machine
      * @param serviceOffering
@@ -264,6 +270,8 @@ UserVm createBasicSecurityGroupVirtualMachine(DataCenter zone, ServiceOffering s
      *            base64 encoded before adding it to the request. Currently only
      *            HTTP GET is supported. Using HTTP GET (via querystring), you
      *            can send up to 2KB of data after base64 encoding
+     * @param userDataId
+     * @param userDataDetails
      * @param requestedIps
      *            TODO
      * @param defaultIps
@@ -279,7 +287,6 @@ UserVm createBasicSecurityGroupVirtualMachine(DataCenter zone, ServiceOffering s
      *             an optional parameter that creates additional data disks for the virtual machine
      *             For each of the templates in the map, a data disk will be created from the corresponding
      *             disk offering obtained from the map
-     * @param type
      * @return UserVm object if successful.
      *
      * @throws InsufficientCapacityException
@@ -292,17 +299,23 @@ UserVm createBasicSecurityGroupVirtualMachine(DataCenter zone, ServiceOffering s
      *             available.
      */
     UserVm createAdvancedSecurityGroupVirtualMachine(DataCenter zone, ServiceOffering serviceOffering, VirtualMachineTemplate template, List<Long> networkIdList,
-        List<Long> securityGroupIdList, Account owner, String hostName, String displayName, Long diskOfferingId, Long diskSize, String group, HypervisorType hypervisor,
-        HTTPMethod httpmethod, String userData, List<String> sshKeyPairs, Map<Long, IpAddresses> requestedIps, IpAddresses defaultIps, Boolean displayVm, String keyboard,
-        List<Long> affinityGroupIdList, Map<String, String> customParameters, String customId, Map<String, Map<Integer, String>> dhcpOptionMap,
-        Map<Long, DiskOffering> dataDiskTemplateToDiskOfferingMap, Map<String, String> userVmOVFProperties, boolean dynamicScalingEnabled, Long overrideDiskOfferingId, String vmType) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException, StorageUnavailableException, ResourceAllocationException;
+                                                     List<Long> securityGroupIdList, Account owner, String hostName, String displayName, Long diskOfferingId, Long diskSize, String group, HypervisorType hypervisor,
+                                                     HTTPMethod httpmethod, String userData, Long userDataId, String userDataDetails, List<String> sshKeyPairs, Map<Long, IpAddresses> requestedIps, IpAddresses defaultIps, Boolean displayVm, String keyboard,
+                                                     List<Long> affinityGroupIdList, Map<String, String> customParameters, String customId, Map<String, Map<Integer, String>> dhcpOptionMap,
+                                                     Map<Long, DiskOffering> dataDiskTemplateToDiskOfferingMap, Map<String, String> userVmOVFProperties, boolean dynamicScalingEnabled, Long overrideDiskOfferingId, String vmType) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException, StorageUnavailableException, ResourceAllocationException;
 
     /**
      * Creates a User VM in Advanced Zone (Security Group feature is disabled)
      * in the database and returns the VM to the caller.
      *
      *
      *
+     * @param sshKeyPair
+     *            - name of the ssh key pair used to login to the virtual
+     *            machine
+     * @param cpuSpeed
+     * @param memory
+     * @param cpuNumber
      * @param zone
      *            - availability zone for the virtual machine
      * @param serviceOffering
@@ -337,19 +350,15 @@ UserVm createAdvancedSecurityGroupVirtualMachine(DataCenter zone, ServiceOfferin
      *            base64 encoded before adding it to the request. Currently only
      *            HTTP GET is supported. Using HTTP GET (via querystring), you
      *            can send up to 2KB of data after base64 encoding
-     * @param sshKeyPair
-     *            - name of the ssh key pair used to login to the virtual
-     *            machine
+     * @param userDataId
+     * @param userDataDetails
      * @param requestedIps
      *            TODO
      * @param defaultIps
      *            TODO
      * @param displayVm
      *            - Boolean flag whether to the display the vm to the end user or not
      * @param affinityGroupIdList
-     * @param cpuSpeed
-     * @param memory
-     * @param cpuNumber
      * @param customId
      * @param dhcpOptionMap
      *             - Map that maps the DhcpOption code and their value on the Network uuid
@@ -370,10 +379,10 @@ UserVm createAdvancedSecurityGroupVirtualMachine(DataCenter zone, ServiceOfferin
      *             available.
      */
     UserVm createAdvancedVirtualMachine(DataCenter zone, ServiceOffering serviceOffering, VirtualMachineTemplate template, List<Long> networkIdList, Account owner,
-        String hostName, String displayName, Long diskOfferingId, Long diskSize, String group, HypervisorType hypervisor, HTTPMethod httpmethod, String userData,
-        List<String> sshKeyPairs, Map<Long, IpAddresses> requestedIps, IpAddresses defaultIps, Boolean displayVm, String keyboard, List<Long> affinityGroupIdList,
-        Map<String, String> customParameters, String customId, Map<String, Map<Integer, String>> dhcpOptionMap, Map<Long, DiskOffering> dataDiskTemplateToDiskOfferingMap,
-        Map<String, String> templateOvfPropertiesMap, boolean dynamicScalingEnabled, String vmType, Long overrideDiskOfferingId)
+                                        String hostName, String displayName, Long diskOfferingId, Long diskSize, String group, HypervisorType hypervisor, HTTPMethod httpmethod, String userData,
+                                        Long userDataId, String userDataDetails, List<String> sshKeyPairs, Map<Long, IpAddresses> requestedIps, IpAddresses defaultIps, Boolean displayVm, String keyboard, List<Long> affinityGroupIdList,
+                                        Map<String, String> customParameters, String customId, Map<String, Map<Integer, String>> dhcpOptionMap, Map<Long, DiskOffering> dataDiskTemplateToDiskOfferingMap,
+                                        Map<String, String> templateOvfPropertiesMap, boolean dynamicScalingEnabled, String vmType, Long overrideDiskOfferingId)
 
         throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException, StorageUnavailableException, ResourceAllocationException;
 

api/src/main/java/org/apache/cloudstack/annotation/AnnotationService.java

@@ -39,7 +39,7 @@ public interface AnnotationService {
 
     enum EntityType {
         VM(true), VOLUME(true), SNAPSHOT(true),
-        VM_SNAPSHOT(true), INSTANCE_GROUP(true), SSH_KEYPAIR(true),
+        VM_SNAPSHOT(true), INSTANCE_GROUP(true), SSH_KEYPAIR(true), USER_DATA(true),
         NETWORK(true), VPC(true), PUBLIC_IP_ADDRESS(true), VPN_CUSTOMER_GATEWAY(true),
         TEMPLATE(true), ISO(true), KUBERNETES_CLUSTER(true),
         SERVICE_OFFERING(false), DISK_OFFERING(false), NETWORK_OFFERING(false),