Tank API V2 Clients Release (#245)

* feat: Add V2 clients

* fix: clean up + switching to webclient exception handling

* fix: better exception handling

* feat: [SRE-26521] upload/download for scripts/datafiles

fix: supports both normal and gzip file uploads

fix: updating get specific script to script JSON description, download handles content

fix: updated max file size and added exception handling for file exceptions

fix: set debug to false

fix: group all exception handlers

fix: updating prefix

fix: unneeded file

fix: reverting file change

Revert "fix: reverting file change"

This reverts commit 0ba90f6.

fix: revert file

fix: reverting file as well

fix: detailed error msg for incorrect request

* Revert "feat: [SRE-26521] upload/download for scripts/datafiles"

This reverts commit 66d2ca4.

* feat: Updated clients, new routes, status response from agent/job ops, and testing agent client with APIHarness

* fix: listing only user-facing endpoints

* fix: small updates to  deps + docs for client changes

* fix: temp fix for tools while testing agent

* fix: removing unneeded files

* fix: temp fix for client tools

* fix: missing file

* moved last backend V1 deps over with models + tested locally

* updating file documentation

* reverting debug flag

* moving other misc backend changes to this branch

* moving over one more backend change

* updated in UI ref branch

* cleaning up, removing old files

* testing base agent client

* small clean up

* dep injection fix

* need to shorten V2 url due to dep inj fix

* missing dependency

* unneeded files

* reverting cloud models until move away from Java EE CDI

* updating vmtracker as well

* TankAPI fix

* moving vmmanger + models outside of rest api

* update pom for logging

* vmTracker models moved to api

* updating import ref to cloud

* reverting defaultcontroller

* revert ui for now

* reverting module deletion, saving for V1 removal

* reverting local files

* updating package names

* update UI refs to /v2 path

* banner fixed to stay closed each session

* updated tools for changes

* banner logic for SSOlogin

* exclude to reenable agent logging

* commenting back httpclient tests

* should fix agent logging

* reverting files

* adding to child pom

* logback vs log4j2 mismatch - probable cause for log drop between modules

* attempt to force spring to use log4j

* correct file path for log4j2

* revert after testing logging fix

* legacy spring startup, removing from web/WEB-INF

* re-adding bridge deps

* fixed versions

* logging config issue fix: including maven-shade-plugin fixes issue locally

* updating gitignore

* logs showing locally, just needed to exclude extra servlet fragment

* switching scope out for absolute-ordering, logging to file locally

* moving from tank-api to agent module

* exclude Log4j2plugins.dat w/ maven shaded plugin instead, avoiding dep issues

* updating fix for other modules, tested locally

* uncommenting tests now that builds passing

* updated not needed, pending deletion

* filter endpoint fix - moving script models out to api module

* Revert "filter endpoint fix - moving script models out to api module"

This reverts commit 6db2e74.

* new filter fix: reverting all script models to v1 until external scripts updated

* reduced war file size by 80M, still needs more dep exclusions from spring

---------

Co-authored-by: zkofiro <[email protected]>

agent/agent_standalone/pom.xml

@@ -16,8 +16,46 @@
   <dependencies>
     <dependency>
       <groupId>${project.groupId}</groupId>
-      <artifactId>agent-client</artifactId>
+      <artifactId>tank-api</artifactId>
       <version>${project.version}</version>
+      <exclusions>
+        <exclusion>
+            <groupId>org.springdoc</groupId>
+            <artifactId>springdoc-openapi-ui</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>${project.groupId}</groupId>
+          <artifactId>script-engine</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>${project.groupId}</groupId>
+          <artifactId>data-access</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>${project.groupId}</groupId>
+          <artifactId>tank-vm-manager</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>${project.groupId}</groupId>
+          <artifactId>tank-script-processor</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>${project.groupId}</groupId>
+          <artifactId>api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.projectlombok</groupId>
+          <artifactId>lombok</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.logging.log4j</groupId>
+          <artifactId>log4j-core</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>jakarta.xml.bind</groupId>
+          <artifactId>jakarta.xml.bind-api</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>

agent/agent_standalone/src/main/java/com/intuit/tank/standalone/agent/StandaloneAgentStartup.java

@@ -28,7 +28,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import com.intuit.tank.AgentServiceClient;
+import com.intuit.tank.rest.mvc.rest.clients.AgentClient;
 import com.intuit.tank.harness.HostInfo;
 import com.intuit.tank.vm.agent.messages.AgentAvailability;
 import com.intuit.tank.vm.agent.messages.AgentAvailabilityStatus;
@@ -37,23 +37,23 @@
 public class StandaloneAgentStartup implements Runnable {
 
     private static Logger LOG = LogManager.getLogger(StandaloneAgentStartup.class);
-    public static final String SERVICE_RELATIVE_PATH = "/rest/v1/agent-service";
+    public static final String SERVICE_RELATIVE_PATH = "/v2/agent";
     private static String API_HARNESS_COMMAND = "./startAgent.sh";
     public static final String METHOD_SETTINGS = "/settings";
-    public static final String METHOD_SUPPORT = "/supportFiles";
+    public static final String METHOD_SUPPORT = "/support-files";
     private static final long PING_TIME = 1000 * 60 * 5;// five minutes
 
     private String controllerBase;
     private AgentAvailability currentAvailability;
-    private AgentServiceClient agentClient;
+    private AgentClient agentClient;
     private String instanceId;
     private String hostname;
     private int capacity = 4000;
 
     @Override
     public void run() {
         CommandListener.startHttpServer(CommandListener.PORT, this);
-        agentClient = new AgentServiceClient(controllerBase);
+        agentClient = new AgentClient(controllerBase);
 
         if (hostname != null) {
             instanceId = hostname;
@@ -152,7 +152,7 @@ private void sendAvailability() {
                 currentAvailability.getInstanceUrl(), currentAvailability.getCapacity(),
                 currentAvailability.getAvailabilityStatus());
         LOG.info("Sending availaability: " + ToStringBuilder.reflectionToString(availability));
-        agentClient.standaloneAgentAvailable(availability);
+        agentClient.setStandaloneAgentAvailability(availability);
     }
 
     public static void main(String[] args) {

agent/agent_standalone_pkg/src/main/assembly/assembly.xml

@@ -16,6 +16,11 @@
       <useProjectArtifact>true</useProjectArtifact>
       <unpack>true</unpack>
       <scope>runtime</scope>
+      <unpackOptions>
+        <excludes>
+          <exclude>**/Log4j2Plugins.dat</exclude>
+        </excludes>
+      </unpackOptions>
     </dependencySet>
   </dependencySets>
 </assembly>

agent/agent_startup/src/main/java/com/intuit/tank/agent/AgentStartup.java

@@ -34,10 +34,10 @@
 
 public class AgentStartup implements Runnable {
     private static final Logger logger = LogManager.getLogger(AgentStartup.class);
-    private static final String SERVICE_RELATIVE_PATH = "/rest/v1/agent-service";
+    private static final String SERVICE_RELATIVE_PATH = "/v2/agent";
     private static final String METHOD_SETTINGS = "/settings";
     private static final String API_HARNESS_COMMAND = "./startAgent.sh";
-    private static final String METHOD_SUPPORT = "/supportFiles";
+    private static final String METHOD_SUPPORT = "/support-files";
     private static final int[] FIBONACCI = new int[] { 1, 1, 2, 3, 5, 8, 13 };
 
     private final String controllerBaseUrl;
@@ -120,5 +120,8 @@ private static void usage() {
         System.out.println("Tank Test Startup Usage:");
         System.out.println("java -cp agent-startup-pkg-1.0-all.jar com/intuit/tank/agent/AgentStartup <options>");
         System.out.println("-controller=<controller_base_url>:  The url of the controller to get test info from");
+        System.out.println("Service Path: " + SERVICE_RELATIVE_PATH);
+        System.out.println("Settings Method: " + METHOD_SETTINGS);
+        System.out.println("Support Files Method: " + METHOD_SUPPORT);
     }
 }

agent/apiharness/pom.xml

@@ -27,6 +27,50 @@
       <version>${project.version}</version>
     </dependency>
 
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>tank-api</artifactId>
+      <version>${project.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.springdoc</groupId>
+          <artifactId>springdoc-openapi-ui</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>${project.groupId}</groupId>
+          <artifactId>script-engine</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>${project.groupId}</groupId>
+          <artifactId>data-access</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>${project.groupId}</groupId>
+          <artifactId>tank-vm-manager</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>${project.groupId}</groupId>
+          <artifactId>tank-script-processor</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>${project.groupId}</groupId>
+          <artifactId>api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.projectlombok</groupId>
+          <artifactId>lombok</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.logging.log4j</groupId>
+          <artifactId>log4j-core</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>jakarta.xml.bind</groupId>
+          <artifactId>jakarta.xml.bind-api</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>agent-client</artifactId>
@@ -81,5 +125,4 @@
       <version>${project.version}</version>
     </dependency>
   </dependencies>
-
 </project>

agent/apiharness/src/main/java/com/intuit/tank/harness/APIMonitor.java

@@ -18,10 +18,10 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import com.intuit.tank.CloudServiceClient;
-import com.intuit.tank.api.model.v1.cloud.CloudVmStatus;
-import com.intuit.tank.api.model.v1.cloud.VMStatus;
-import com.intuit.tank.api.model.v1.cloud.ValidationStatus;
+import com.intuit.tank.rest.mvc.rest.clients.AgentClient;
+import com.intuit.tank.vm.vmManager.models.CloudVmStatus;
+import com.intuit.tank.vm.vmManager.models.VMStatus;
+import com.intuit.tank.vm.vmManager.models.ValidationStatus;
 import com.intuit.tank.harness.logging.LogUtil;
 import com.intuit.tank.reporting.api.TPSInfoContainer;
 import com.intuit.tank.vm.agent.messages.WatsAgentStatusResponse;
@@ -36,7 +36,7 @@ public class APIMonitor implements Runnable {
     private static final int MIN_REPORT_TIME = 15000;
     private static Logger LOG = LogManager.getLogger(APIMonitor.class);
     private static boolean doMonitor = true;
-    private static CloudServiceClient client;
+    private static AgentClient client;
     private static CloudVmStatus status;
     private long reportInterval = APIMonitor.MIN_REPORT_TIME;
     private boolean isLocal;
@@ -45,7 +45,7 @@ public APIMonitor(Boolean isLocal, CloudVmStatus vmStatus) {
         this.isLocal = isLocal;
         status = vmStatus;
         try {
-            client = new CloudServiceClient(APITestHarness.getInstance().getTankConfig().getControllerBase());
+            client = new AgentClient(APITestHarness.getInstance().getTankConfig().getControllerBase());
             reportInterval = Math.max(APITestHarness.getInstance().getTankConfig().getAgentConfig()
                     .getStatusReportIntervalMilis(reportInterval), MIN_REPORT_TIME);
         } catch (Exception e) {
@@ -65,15 +65,15 @@ public void run() {
                     newStatus.setTotalTps(tpsInfo.getTotalTps());
                     sendTps(tpsInfo);
                 }
-                if (!isLocal) client.setVmStatus(newStatus.getInstanceId(), newStatus);
+                if (!isLocal) client.setInstanceStatus(newStatus.getInstanceId(), newStatus);
                 APITestHarness.getInstance().checkAgentThreads();
                 Thread.sleep(reportInterval);
             } catch (Exception t) {
                 LOG.error(LogUtil.getLogMessage("Unable to send status metrics | " + t.getMessage()), t);
             }
         }
         CloudVmStatus newStatus = createStatus(APITestHarness.getInstance().getStatus());
-        client.setVmStatus(newStatus.getInstanceId(), newStatus);
+        client.setInstanceStatus(newStatus.getInstanceId(), newStatus);
     }
 
     private void sendTps(final TPSInfoContainer tpsInfo) {
@@ -136,7 +136,7 @@ public synchronized static void setJobStatus(JobStatus jobStatus) {
                         stats.getMaxVirtualUsers(),
                         stats.getCurrentNumberUsers(), status.getStartTime(), endTime);
                 status.setUserDetails(APITestHarness.getInstance().getUserTracker().getSnapshot());
-                client.setVmStatus(status.getInstanceId(), status);
+                client.setInstanceStatus(status.getInstanceId(), status);
             } catch (Exception e) {
                 LOG.error("Error sending status to controller: " + e.toString(), e);
             }

agent/apiharness/src/main/java/com/intuit/tank/harness/APITestHarness.java

@@ -41,10 +41,10 @@
 import org.apache.logging.log4j.core.config.LoggerConfig;
 import org.apache.logging.log4j.message.ObjectMessage;
 
-import com.intuit.tank.AgentServiceClient;
-import com.intuit.tank.api.model.v1.cloud.CloudVmStatus;
-import com.intuit.tank.api.model.v1.cloud.VMStatus;
-import com.intuit.tank.api.model.v1.cloud.ValidationStatus;
+import com.intuit.tank.rest.mvc.rest.clients.AgentClient;
+import com.intuit.tank.vm.vmManager.models.CloudVmStatus;
+import com.intuit.tank.vm.vmManager.models.VMStatus;
+import com.intuit.tank.vm.vmManager.models.ValidationStatus;
 import com.intuit.tank.harness.data.HDTestPlan;
 import com.intuit.tank.harness.data.HDWorkload;
 import com.intuit.tank.harness.logging.LogUtil;
@@ -252,7 +252,7 @@ private void startHttp(String baseUrl) {
         if (baseUrl == null) {
             baseUrl = AmazonUtil.getControllerBaseUrl();
         }
-        AgentServiceClient client = new AgentServiceClient(baseUrl);
+        AgentClient client = new AgentClient(baseUrl);
         String instanceUrl = null;
         int retryCount = 0;
         while (instanceUrl == null) {

agent/apiharness/src/main/java/com/intuit/tank/harness/UserTracker.java

@@ -19,7 +19,7 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
-import com.intuit.tank.api.model.v1.cloud.UserDetail;
+import com.intuit.tank.vm.vmManager.models.UserDetail;
 
 /**
  * 

agent/apiharness/src/test/java/com/intuit/tank/harness/APIMonitorTest.java

@@ -1,8 +1,8 @@
 package com.intuit.tank.harness;
 
-import com.intuit.tank.api.model.v1.cloud.CloudVmStatus;
-import com.intuit.tank.api.model.v1.cloud.VMStatus;
-import com.intuit.tank.api.model.v1.cloud.ValidationStatus;
+import com.intuit.tank.vm.vmManager.models.CloudVmStatus;
+import com.intuit.tank.vm.vmManager.models.VMStatus;
+import com.intuit.tank.vm.vmManager.models.ValidationStatus;
 import com.intuit.tank.vm.api.enumerated.JobStatus;
 import com.intuit.tank.vm.api.enumerated.VMImageType;
 import com.intuit.tank.vm.api.enumerated.VMRegion;

agent/apiharness/src/test/java/com/intuit/tank/harness/UserTrackerTest.java

@@ -18,7 +18,7 @@
 
 import java.util.List;
 
-import com.intuit.tank.api.model.v1.cloud.UserDetail;
+import com.intuit.tank.vm.vmManager.models.UserDetail;
 import com.intuit.tank.test.TestGroups;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;

agent/apiharness_pkg/src/main/assembly/assembly.xml

@@ -16,6 +16,11 @@
       <useProjectArtifact>true</useProjectArtifact>
       <unpack>true</unpack>
       <scope>runtime</scope>
+      <unpackOptions>
+        <excludes>
+          <exclude>**/Log4j2Plugins.dat</exclude>
+        </excludes>
+      </unpackOptions>
     </dependencySet>
   </dependencySets>
 </assembly>

api/src/main/java/com/intuit/tank/vm/api/service/v1/project/ProjectServiceUrlBuilder.java

@@ -39,7 +39,7 @@ private ProjectServiceUrlBuilder() {
      */
     public static String getScriptXmlUrl(String jobId) {
         String baseUrl = new TankConfig().getControllerBase();
-        return baseUrl + "/rest/v1/project-service/script/" + jobId;
+        return baseUrl + "/v2/jobs/script/" + jobId;
     }
 
 }

rest/api/cloud/src/main/java/com/intuit/tank/api/cloud/VMTracker.java → api/src/main/java/com/intuit/tank/vm/vmManager/VMTracker.java

@@ -1,7 +1,7 @@
 /**
  * Copyright 2011 Intuit Inc. All Rights Reserved
  */
-package com.intuit.tank.api.cloud;
+package com.intuit.tank.vm.vmManager;
 
 /*
  * #%L
@@ -20,9 +20,9 @@
 
 import javax.annotation.Nonnull;
 
-import com.intuit.tank.api.model.v1.cloud.CloudVmStatus;
-import com.intuit.tank.api.model.v1.cloud.CloudVmStatusContainer;
-import com.intuit.tank.api.model.v1.cloud.ProjectStatusContainer;
+import com.intuit.tank.vm.vmManager.models.CloudVmStatus;
+import com.intuit.tank.vm.vmManager.models.CloudVmStatusContainer;
+import com.intuit.tank.vm.vmManager.models.ProjectStatusContainer;
 import com.intuit.tank.vm.event.JobEvent;
 
 /**