[SOAR-18202] [Salesforce] Ensure date.now includes microseconds (#2960)

Author: joneill-r7

plugins/salesforce/.CHECKSUM

@@ -1,7 +1,7 @@
 {
-	"spec": "e182e26e61f7d3375dc3a9bc3df8fc11",
-	"manifest": "4d555de9a1d8b4ead1868f7d8ae7c1b5",
-	"setup": "8d0731798d9f79b7c98d821e8abf0001",
+	"spec": "63b7270d95683b98e315808e4df20354",
+	"manifest": "391ed2bce80fc53ee24774278259d26e",
+	"setup": "295d03a5efdf6658a6a10babe80a9a06",
 	"schemas": [
 		{
 			"identifier": "advanced_search/schema.py",

plugins/salesforce/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 rapid7/insightconnect-python-3-plugin:6.1.4
+FROM --platform=linux/amd64 rapid7/insightconnect-python-3-plugin:6.2.0
 
 LABEL organization=rapid7
 LABEL sdk=python

plugins/salesforce/bin/komand_salesforce

@@ -6,7 +6,7 @@ from sys import argv
 
 Name = "Salesforce"
 Vendor = "rapid7"
-Version = "2.1.11"
+Version = "2.1.12"
 Description = "[Salesforce](https://www.salesforce.com) is a CRM solution that brings together all customer information in a single, integrated platform that enables building a customer-centered business from marketing right through to sales, customer service and business analysis. The Salesforce plugin allows you to search, update, and manage salesforce records. This plugin utilizes the [Salesforce API](https://developer.salesforce.com/docs/atlas.en-us.216.0.api_rest.meta/api_rest/intro_what_is_rest_api.htm)"
 
 

plugins/salesforce/help.md

@@ -529,10 +529,11 @@ Example output:
 
 ## Troubleshooting
 
-*There is no troubleshooting for this plugin.*
+*This plugin does not contain a troubleshooting.*
 
 # Version History
 
+* 2.1.12 - Task Monitor Users: ensure datetime includes microseconds | Bump SDK to 6.2.0
 * 2.1.11 - Task Monitor Users: Return 500 for retry your request error | Bump SDK to 6.1.4
 * 2.1.10 - Set Monitor Users task output length | Fix to remove whitespace from connection inputs
 * 2.1.9 - SDK Bump to 6.1.0 | Task Connection test added

plugins/salesforce/komand_salesforce/tasks/monitor_users/task.py

@@ -10,6 +10,11 @@
 DEFAULT_CUTOFF_HOURS = 24 * 7
 INITIAL_LOOKBACK = 24
 
+exp_frmt = "%Y-%m-%d %H:%M:%S.%f%z"  # the format the state should be in
+old_frmt = "%Y-%m-%dT%H:%M:%S.%f%z"  # an old backwards compatible state
+bugged_frmt = "%Y-%m-%d %H:%M:%S%z"  # str value without the microseconds - SOAR-18202
+SUPPORTED_STR_TYPES = [exp_frmt, old_frmt, bugged_frmt]
+
 
 class MonitorUsers(insightconnect_plugin_runtime.Task):
     USER_LOGIN_QUERY = "SELECT LoginTime, UserId, LoginType, LoginUrl, SourceIp, Status, Application, Browser FROM LoginHistory WHERE LoginTime >= {start_timestamp} AND LoginTime < {end_timestamp}"
@@ -63,15 +68,17 @@ def run(self, params={}, state={}, custom_config={}):  # noqa: C901
 
                 # we only check Salesforce for new users every 24 hours / first run
                 get_users = True
-                state[self.NEXT_USER_COLLECTION_TIMESTAMP] = str(now + timedelta(hours=24))
+                state[self.NEXT_USER_COLLECTION_TIMESTAMP] = self.convert_dt_to_string(now + timedelta(hours=24))
 
                 # we check for any user profile updates every task execution
-                state[self.LAST_USER_UPDATE_COLLECTION_TIMESTAMP] = str(user_update_last_collection)
+                state[self.LAST_USER_UPDATE_COLLECTION_TIMESTAMP] = self.convert_dt_to_string(
+                    user_update_last_collection
+                )
 
                 # we only check for login data every hour
                 get_user_login_history = True
-                state[self.NEXT_USER_LOGIN_COLLECTION_TIMESTAMP] = str(now + timedelta(hours=1))
-                state[self.LAST_USER_LOGIN_COLLECTION_TIMESTAMP] = str(user_login_end_timestamp)
+                state[self.NEXT_USER_LOGIN_COLLECTION_TIMESTAMP] = self.convert_dt_to_string(now + timedelta(hours=1))
+                state[self.LAST_USER_LOGIN_COLLECTION_TIMESTAMP] = self.convert_dt_to_string(user_login_end_timestamp)
             elif users_next_page_id or user_login_next_page_id or updated_users_next_page_id:
                 self.logger.info("Getting next page of results...")
                 if users_next_page_id:
@@ -82,9 +89,11 @@ def run(self, params={}, state={}, custom_config={}):  # noqa: C901
             else:
                 self.logger.info("Subsequent run")
 
-                is_valid_state, key = self._is_valid_state(state)
-                if not is_valid_state:
-                    self.logger.info(f"Bad request error occurred. Invalid timestamp format for {key}")
+                valid_state, key = self._make_valid_state(state)
+                if not valid_state:
+                    self.logger.info(
+                        f"Bad request error occurred. Invalid timestamp format for {key}. Got value {state[key]}"
+                    )
                     return (
                         [],
                         state,
@@ -100,29 +109,34 @@ def run(self, params={}, state={}, custom_config={}):  # noqa: C901
                     state, cut_off_time, self.LAST_USER_UPDATE_COLLECTION_TIMESTAMP
                 )
                 # move the end time stamp to now
-                state[self.LAST_USER_UPDATE_COLLECTION_TIMESTAMP] = str(user_update_last_collection)
+                state[self.LAST_USER_UPDATE_COLLECTION_TIMESTAMP] = self.convert_dt_to_string(
+                    user_update_last_collection
+                )
 
                 # this allows us to poll for new users every 24 hours
                 next_user_collection_timestamp = state.get(self.NEXT_USER_COLLECTION_TIMESTAMP)
                 if next_user_collection_timestamp and self.compare_timestamp(
                     now, self.convert_to_datetime(next_user_collection_timestamp)
                 ):
                     get_users = True
-                    state[self.NEXT_USER_COLLECTION_TIMESTAMP] = str(now + timedelta(hours=24))  # poll again in 24 hrs
+                    # poll again in 24 hrs
+                    state[self.NEXT_USER_COLLECTION_TIMESTAMP] = self.convert_dt_to_string(now + timedelta(hours=24))
 
                 # this allows us to poll for user login data every hour
                 next_user_login_collection_timestamp = state.get(self.NEXT_USER_LOGIN_COLLECTION_TIMESTAMP)
                 if next_user_login_collection_timestamp and self.compare_timestamp(
                     now, self.convert_to_datetime(next_user_login_collection_timestamp)
                 ):
                     get_user_login_history = True
-                    state[self.NEXT_USER_LOGIN_COLLECTION_TIMESTAMP] = str(
+                    state[self.NEXT_USER_LOGIN_COLLECTION_TIMESTAMP] = self.convert_dt_to_string(
                         now + timedelta(hours=1)
                     )  # poll again in 1 hr
                     user_login_start_timestamp = self._get_recent_timestamp(
                         state, cut_off_time, self.LAST_USER_LOGIN_COLLECTION_TIMESTAMP
                     )
-                    state[self.LAST_USER_LOGIN_COLLECTION_TIMESTAMP] = str(user_login_end_timestamp)
+                    state[self.LAST_USER_LOGIN_COLLECTION_TIMESTAMP] = self.convert_dt_to_string(
+                        user_login_end_timestamp
+                    )
 
             try:
                 records = []
@@ -134,8 +148,8 @@ def run(self, params={}, state={}, custom_config={}):  # noqa: C901
                     self.logger.info(msg)
                     response = self.connection.api.query(
                         self.UPDATED_USERS_QUERY.format(
-                            start_timestamp=user_update_start_timestamp.isoformat(),
-                            end_timestamp=user_update_last_collection.isoformat(),
+                            start_timestamp=user_update_start_timestamp.isoformat(timespec="microseconds"),
+                            end_timestamp=user_update_last_collection.isoformat(timespec="microseconds"),
                         ),
                         updated_users_next_page_id,
                     )
@@ -169,8 +183,8 @@ def run(self, params={}, state={}, custom_config={}):  # noqa: C901
                     self.logger.info(msg)
                     response = self.connection.api.query(
                         self.USER_LOGIN_QUERY.format(
-                            start_timestamp=user_login_start_timestamp.isoformat(),
-                            end_timestamp=user_login_end_timestamp.isoformat(),
+                            start_timestamp=user_login_start_timestamp.isoformat(timespec="microseconds"),
+                            end_timestamp=user_login_end_timestamp.isoformat(timespec="microseconds"),
                         ),
                         user_login_next_page_id,
                     )
@@ -197,16 +211,21 @@ def run(self, params={}, state={}, custom_config={}):  # noqa: C901
             self.connection.api.unset_token()
             return [], state, False, 500, PluginException(preset=PluginException.Preset.UNKNOWN, data=error)
 
-    def _is_valid_state(self, state: dict) -> Tuple[bool, str]:
+    def _make_valid_state(self, state: dict) -> Tuple[bool, str]:
+        # it looks like we used to store the timestamp in the state with the `T` delimiter and then swapped when we
+        # started to do str(datetime) which does not have this delimiter but then swap it back and forward throughout
+        # the task logic. Allow the state to have any and a time without the microseconds also.
+        last_attempt = len(SUPPORTED_STR_TYPES) - 1
         for key, value in state.items():
-            try:
-                self.convert_to_datetime(value)
-            except ValueError:
+            for attempt_x, str_format in enumerate(SUPPORTED_STR_TYPES):
                 try:
-                    state[key] = str(self.convert_to_datetime_from_old_format(value))
+                    dt_value = datetime.strptime(value, str_format)
+                    state[key] = self.convert_dt_to_string(dt_value)
+                    break  # we're happy with this state value now move to the next one
                 except ValueError:
-                    state[key] = str(self.get_current_time())
-                    return False, key
+                    if attempt_x != last_attempt:
+                        continue  # try the next type
+                return False, key
         return True, ""
 
     def _get_recent_timestamp(self, state: dict, fallback_timestamp: datetime, key: str) -> datetime:
@@ -331,5 +350,7 @@ def add_data_type_field(records: list, field_value: str) -> list:
             record["DataType"] = field_value
         return records
 
-    def convert_to_datetime_from_old_format(self, timestamp: str) -> datetime:
-        return datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%f%z")
+    @staticmethod
+    def convert_dt_to_string(timestamp: datetime) -> str:
+        # Force microseconds to keep microseconds in the string but remove the `T`
+        return timestamp.isoformat(timespec="microseconds").replace("T", " ")

plugins/salesforce/plugin.spec.yaml

@@ -4,7 +4,7 @@ products: [insightconnect]
 name: salesforce
 title: Salesforce
 description: "[Salesforce](https://www.salesforce.com) is a CRM solution that brings together all customer information in a single, integrated platform that enables building a customer-centered business from marketing right through to sales, customer service and business analysis. The Salesforce plugin allows you to search, update, and manage salesforce records. This plugin utilizes the [Salesforce API](https://developer.salesforce.com/docs/atlas.en-us.216.0.api_rest.meta/api_rest/intro_what_is_rest_api.htm)"
-version: 2.1.11
+version: 2.1.12
 connection_version: 2
 vendor: rapid7
 support: community
@@ -13,7 +13,7 @@ status: []
 supported_versions: ["Salesforce API v58 2023-06-30"]
 sdk:
   type: full
-  version: 6.1.4
+  version: 6.2.0
   user: nobody
 resources:
   source_url: https://github.com/rapid7/insightconnect-plugins/tree/master/plugins/salesforce
@@ -37,6 +37,7 @@ references:
   - "[Connecting your app to the API](https://developer.salesforce.com/docs/atlas.en-us.216.0.api_rest.meta/api_rest/quickstart.htm)"
   - "[SOQL](https://developer.salesforce.com/docs/atlas.en-us.216.0.soql_sosl.meta/soql_sosl/sforce_api_calls_soql.htm)"
 version_history:
+  - "2.1.12 - Task Monitor Users: ensure datetime includes microseconds | Bump SDK to 6.2.0"
   - "2.1.11 - Task Monitor Users: Return 500 for retry your request error | Bump SDK to 6.1.4"
   - "2.1.10 - Set Monitor Users task output length | Fix to remove whitespace from connection inputs"
   - "2.1.9 - SDK Bump to 6.1.0 | Task Connection test added"

plugins/salesforce/setup.py

@@ -3,7 +3,7 @@
 
 
 setup(name="salesforce-rapid7-plugin",
-      version="2.1.11",
+      version="2.1.12",
       description="[Salesforce](https://www.salesforce.com) is a CRM solution that brings together all customer information in a single, integrated platform that enables building a customer-centered business from marketing right through to sales, customer service and business analysis. The Salesforce plugin allows you to search, update, and manage salesforce records. This plugin utilizes the [Salesforce API](https://developer.salesforce.com/docs/atlas.en-us.216.0.api_rest.meta/api_rest/intro_what_is_rest_api.htm)",
       author="rapid7",
       author_email="",

plugins/salesforce/unit_test/expected/monitor_users_bad_request.json.exp

@@ -1,6 +1,6 @@
 {
   "state": {
-    "last_user_update_collection_timestamp": "2023-07-20 16:21:15.340262+00:00",
+    "last_user_update_collection_timestamp": "invalid",
     "next_user_collection_timestamp": "2023-07-20 16:21:15.340262+00:00",
     "next_user_login_collection_timestamp": "2023-07-20 16:21:15.340262+00:00",
     "last_user_login_collection_timestamp": "2023-07-20 15:21:15.340262+00:00"