Agent status timeout configurable (#1306)

* Agent status timeout configurable

* Using k8s yamls from checked-out repo instead of the hard-coded release branch

Author: alesnovak-s1

.circleci/smoketest_k8s.sh

@@ -70,15 +70,15 @@ if [[ "$delete_existing_objects" == "delete_existing_k8s_objs" ]]; then
     kubectl delete daemonset scalyr-agent-2 || true
     kubectl delete configmap scalyr-config || true
     kubectl delete secret scalyr-api-key || true
-    kubectl delete -f https://raw.githubusercontent.com/scalyr/scalyr-agent-2/release/k8s/scalyr-service-account.yaml || true
+    kubectl delete -f ./k8s/scalyr-service-account.yaml || true
     echo "::endgroup::"
 fi
 
 echo ""
 echo "::group::Creating k8s objects"
 echo "=================================================="
 # Create service account
-kubectl create -f https://raw.githubusercontent.com/scalyr/scalyr-agent-2/release/k8s/scalyr-service-account.yaml
+kubectl create -f ./k8s/scalyr-service-account.yaml
 
 # Define api key
 kubectl create secret generic scalyr-api-key --from-literal=scalyr-api-key=${SCALYR_API_KEY}

scalyr_agent/agent_main.py

@@ -926,7 +926,7 @@ def __detailed_status(
                 return 2
 
         # We wait for five seconds at most to get the status.
-        deadline = time.time() + 5
+        deadline = time.time() + self.__config.agent_status_timeout
 
         last_debug_stat_time = 0
         # Now loop until we see it show up.
@@ -980,9 +980,9 @@ def __detailed_status(
                     debug_stats_str = ""
 
                 print(
-                    "Failed to get status within 5 seconds.  Giving up.  The agent process is "
+                    "Failed to get status within %d seconds.  Giving up.  The agent process is "
                     "possibly stuck.  See %s for more details.\n%s"
-                    % (agent_log, debug_stats_str),
+                    % (self.__config.agent_status_timeout, agent_log, debug_stats_str),
                     file=sys.stderr,
                 )
                 return 1
@@ -2329,8 +2329,8 @@ def __report_status_to_file(self):
 
         log.log(
             scalyr_logging.DEBUG_LEVEL_4,
-            'Wrote agent status data in "%s" format to %s'
-            % (status_format, final_file_path),
+            'Wrote agent status data in "%s" format to %s in %.2f seconds'
+            % (status_format, final_file_path, time.time() - start_ts),
         )
 
         return final_file_path

scalyr_agent/configuration.py

@@ -1498,6 +1498,11 @@ def api_key(self):
         """Returns the configuration value for 'api_key'."""
         return self.__get_config().get_string("api_key")
 
+    @property
+    def agent_status_timeout(self):
+        """Returns the configuration value for 'agent_status_timeout'."""
+        return self.__get_config().get_int("agent_status_timeout")
+
     @property
     def scalyr_server(self):
         """Returns the configuration value for 'scalyr_server'."""
@@ -3604,6 +3609,14 @@ def __verify_main_config_and_apply_defaults(
             apply_defaults,
         )
 
+        self.__verify_or_set_optional_int(
+            config,
+            "agent_status_timeout",
+            30,
+            description,
+            apply_defaults,
+        )
+
     def __verify_compression_type(self, compression_type):
         """
         Verify that the library for the specified compression type (algorithm) is available.