Feature/sts assume role

README.md

@@ -20,6 +20,7 @@ sudo ./setup.py
 ### Plugin specific configuration
 The default location of the configuration file used by collectd-cloudwatch plugin is: `/opt/collectd-plugins/cloudwatch/config/plugin.conf`.  The parameters in this file are optional when plugin is executed on EC2 instance. This file allows modification of the following parameters:
  * __credentials_path__ - Used to point to AWS account configuration file
+ * __arn_role__ - Used when you want to explicitly assume a role using AWS STS (Security Token Service).
  * __region__ - Manual override for [region](http://docs.aws.amazon.com/general/latest/gr/rande.html#cw_region)  used to publish metrics
  * __host__ - Manual override for EC2 Instance ID and Host information propagated by collectd
  * __proxy_server_name__ - Manual override for proxy server name, used by plugin to connect aws cloudwatch at *.amazonaws.com.
@@ -36,6 +37,7 @@ The default location of the configuration file used by collectd-cloudwatch plugi
 #### Example configuration file
 ```
 credentials_path = "/home/user/.aws/credentials"
+arn_role = "arn:aws:test:eu-west-1:1234567890:role/to_assume"
 region = "us-west-1"
 host = "Server1"
 proxy_server_name = "http://myproxyserver.com"
@@ -82,6 +84,12 @@ aws_access_key = valid_access_key
 aws_secret_key = valid_secret_key
 ```
 
+### Assuming role using AWS STS(Security toke service)
+By setting __arn_role__ in configuration you can explicitly assume a role. It's helpful when you want to send metrics from an account to another account.
+Suppose you want send a metric from an EC2 instance in `account_A` to cloudwatch in `account_B`. Then, You should define a role in the `account_A` with
+proper policies accessing cloudwatch of the `account A` and then create another role in the `account B` with a policy which grants assuming role already
+defined in the `account_A`. Finally, attached the later role one to your EC2 IAM role. For more information visit [aws sts assume role](http://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html) documentation.
+
 ### Whitelist configuration
 The CloudWatch collectd plugin allows users to select metrics to be published. This is done by adding metric names or regular expressions written in [python regex syntax](https://docs.python.org/2/library/re.html#regular-expression-syntax) to the whitelist config file. The default location of this configuration is: `/opt/collectd-plugins/cloudwatch/config/whitelist.conf`.
 

src/cloudwatch/config/plugin.conf

@@ -1,6 +1,10 @@
 # The path to the AWS credentials file. This value has to be provided if plugin is used outside of EC2 instances 
 #credentials_path = "/home/user/.aws/credentials"
 
+# The arn of role used by sts to assuming and gets temporary credentials.
+# SEE http://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html#API_AssumeRole_Examples for more information.
+#arn_role = 
+
 # The target region which will be used to publish metric data
 # For list of valid regions visit: http://docs.aws.amazon.com/general/latest/gr/rande.html#cw_region
 #region = "us-west-1"

src/cloudwatch/modules/awscredentials.py

@@ -1,3 +1,7 @@
+from datetime import datetime
+
+AWS_CREDENTIALS_TIMEFORMAT = '%Y-%m-%dT%H:%M:%SZ'
+
 class AWSCredentials(object):
     """
     The AWSCredentials object encapsulates the credentials used for signing put requests.
@@ -7,9 +11,24 @@ class AWSCredentials(object):
     secret_key -- the AWS secret key (default None)
     token -- the temporary security token obtained through a call to 
              AWS Security Token Service when using IAM Role (default None)
+    expire_at -- The date string in ISO 8601 standard format(YYYYMMDDThhmmssZ) 
+             on which the current credentials expire (default None, means never)
     """
-
-    def __init__(self, access_key=None, secret_key=None, token=None):
+
+    def __init__(self, access_key=None, secret_key=None, token=None, expire_at=None):
+
         self.access_key = access_key
         self.secret_key = secret_key
         self.token = token
+
+        if expire_at:
+            self.expire_at = datetime.strptime(expire_at, AWS_CREDENTIALS_TIMEFORMAT)
+        else:
+            self.expire_at = None
+
+    def is_expired(self):
+        """ True if credentials has been expired """
+        now = datetime.utcnow()
+        return self.expire_at and self.expire_at < now
+
+

src/cloudwatch/modules/client/assumerolereqbuilder.py

@@ -0,0 +1,42 @@
+from baserequestbuilder import BaseRequestBuilder
+
+class AssumeRoleReqBuilder(BaseRequestBuilder):
+    """
+    The request builder is responsible for building the AssumeRole requests using HTTP GET. 
+
+    Keyword arguments:
+    credentials -- The AWSCredentials object containing access and secret keys
+    region -- The region to which the data should be published
+    arn_role -- The arn_role to assume
+    """
+    _SERVICE = "sts"
+    _ACTION = "AssumeRole"
+    _API_VERSION = "2011-06-15"
+
+    def __init__(self, credentials, region):
+        super(self.__class__, self).__init__(credentials, region, self._SERVICE, self._ACTION, self._API_VERSION)
+
+    def create_signed_request(self, request_map):
+        """ Creates a ready to send request with metrics from the metric list passed as parameter """
+        self._init_timestamps()
+        canonical_querystring = self._create_canonical_querystring(request_map)
+        signature = self.signer.create_request_signature(canonical_querystring, self._get_credential_scope(),
+                                            self.aws_timestamp, self.datestamp, self._get_canonical_headers(),
+                                            self._get_signed_headers(), self.payload)
+        canonical_querystring += '&X-Amz-Signature=' + signature
+        return canonical_querystring
+
+    def _create_canonical_querystring(self, request_map):
+        """ 
+        Creates a canonical querystring as defined in the official AWS API documentation: 
+        http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html 
+        """
+        return self.querystring_builder.build_querystring_from_map(request_map, self._get_request_map())
+
+    def _get_host(self):
+        """ Returns the endpoint's hostname derived from the region """
+        if self.region == "localhost":
+            return "localhost"
+        elif self.region.startswith("cn-"):
+            return "sts." + self.region + ".amazonaws.com.cn"
+        return "sts." + self.region + ".amazonaws.com"

src/cloudwatch/modules/client/stsassumeroleclient.py

@@ -0,0 +1,124 @@
+import re
+import os
+
+from ..plugininfo import PLUGIN_NAME, PLUGIN_VERSION
+from assumerolereqbuilder import AssumeRoleReqBuilder
+from ..logger.logger import get_logger
+from requests.adapters import HTTPAdapter
+from requests.sessions import Session
+from requests import RequestException
+from tempfile import gettempdir
+import xml.etree.ElementTree as ET
+from ..awscredentials import AWSCredentials
+
+class StsAssumRoleClient(object):
+    """
+    This is a simple HTTPClient wrapper which supports assumeRole operation on sts endpoints. 
+
+    Keyword arguments:
+    region -- the region used for request signing.
+    endpoint -- the endpoint used for publishing metric data
+    credentials -- the AWSCredentials object containing access_key, secret_key or 
+                IAM Role token used for request signing
+    connection_timeout -- the amount of time in seconds to wait for extablishing server connection
+    response_timeout -- the amount of time in seconds to wait for the server response 
+    """
+
+    _LOGGER = get_logger(__name__)
+    _DEFAULT_CONNECTION_TIMEOUT = 1
+    _DEFAULT_RESPONSE_TIMEOUT = 3
+    _TOTAL_RETRIES = 1
+    _LOG_FILE_MAX_SIZE = 10*1024*1024
+
+
+    def __init__(self, credentials, endpoint='', region='', proxy_server_name='', proxy_server_port='', debug=False, connection_timeout=_DEFAULT_CONNECTION_TIMEOUT, response_timeout=_DEFAULT_RESPONSE_TIMEOUT):
+        self.assumerole_req_builder = AssumeRoleReqBuilder(credentials, region)
+        self._validate_and_set_endpoint(endpoint)
+        self.timeout = (connection_timeout, response_timeout)
+        self.proxy_server_name = proxy_server_name
+        self.proxy_server_port = proxy_server_port
+        self.debug = debug
+        self._prepare_session()
+
+    def get_credentials(self, arn_role, role_session_name, duration_seconds):
+        """
+        Requests a temporary keys by assumming give arn_role. Returns None in case of error.
+        """
+        request_map = {}
+        request_map["RoleSessionName"] = role_session_name
+        request_map["RoleArn"] = arn_role
+        request_map["DurationSeconds"] = duration_seconds
+        request = self.assumerole_req_builder.create_signed_request(request_map)
+
+        try:
+            xml_content = self._run_request(request).content
+            xmldoc = ET.fromstring(xml_content)
+            ns={'sts': 'https://sts.amazonaws.com/doc/2011-06-15/'}
+            cred_xml = xmldoc.find('sts:AssumeRoleResult/sts:Credentials',ns)
+            cred = {}
+            cred["session_token"] = cred_xml.find('sts:SessionToken', ns).text.strip()
+            cred["secret_access_key"] = cred_xml.find('sts:SecretAccessKey', ns).text.strip()
+            cred["access_key_id"] = cred_xml.find('sts:AccessKeyId', ns).text.strip()
+            cred["expiration"] = cred_xml.find('sts:Expiration', ns).text.strip()
+
+            if not cred["session_token"] or not cred['secret_access_key'] or not cred["access_key_id"] or not cred["expiration"]:
+                raise ValueError("Incomplete credentials retrieved.")
+        except RequestException as e:
+            self._LOGGER.warning("Could not assume '" + arn_role + "' using the following endpoint: '" + self.endpoint +"'. [Exception: " + str(e) + "]")
+            self._LOGGER.warning("Request details: '" + request + "'")
+            raise ValueError(e)
+        except Exception as e:
+            raise ValueError(e)
+
+        return AWSCredentials(cred['access_key_id'], cred['secret_access_key'], cred["session_token"], cred["expiration"])
+
+    def _prepare_session(self):
+        self.session = Session()
+        if self.proxy_server_name is not None:
+            proxy_server = self.proxy_server_name
+            self._LOGGER.info("Using proxy server: " + proxy_server)
+            if self.proxy_server_port is not None:
+                proxy_server = proxy_server + ":" + self.proxy_server_port
+                self._LOGGER.info("Using proxy server port: " + self.proxy_server_port)
+            proxies = {'https': proxy_server}
+            self.session.proxies.update(proxies)
+        else:
+            self._LOGGER.info("No proxy server is in use")
+        self.session.mount("http://", HTTPAdapter(max_retries=self._TOTAL_RETRIES))
+        self.session.mount("https://", HTTPAdapter(max_retries=self._TOTAL_RETRIES))
+
+    def _validate_and_set_endpoint(self, endpoint):
+        pattern = re.compile("http[s]?://*/")
+        if pattern.match(endpoint) or "localhost" in endpoint: 
+            self.endpoint = endpoint    
+        else:
+            msg = "Provided endpoint '" + endpoint + "' is not a valid URL."
+            self._LOGGER.error(msg)
+            raise StsAssumRoleClient.InvalidEndpointException(msg)
+
+    def _get_custom_headers(self):
+        """ Returns dictionary of HTTP headers to be attached to each request """
+        return {"User-Agent": self._get_user_agent_header()}
+
+    def _get_user_agent_header(self):
+        """ Returns the plugin name and version used as User-Agent information """
+        return PLUGIN_NAME + "/" + str(PLUGIN_VERSION)
+
+    def _run_request(self, request):
+        """
+        Executes HTTP GET request with timeout using the endpoint defined upon client creation.
+        """
+        if self.debug:
+            file_path = gettempdir() + "/collectd_plugin_request_trace_log"
+            if os.path.isfile(file_path) and os.path.getsize(file_path) > self._LOG_FILE_MAX_SIZE:
+                os.remove(file_path)
+            with open(file_path, "a") as logfile:
+                logfile.write("curl -i -v -connect-timeout 1 -m 3 -w %{http_code}:%{http_connect}:%{content_type}:%{time_namelookup}:%{time_redirect}:%{time_pretransfer}:%{time_connect}:%{time_starttransfer}:%{time_total}:%{speed_download} -A \"collectd/1.0\" \'" + self.endpoint + "?" + request + "\'")
+                logfile.write("\n\n")
+
+        result = self.session.get(self.endpoint + "?" + request, headers=self._get_custom_headers(), timeout=self.timeout)
+        result.raise_for_status()
+        return result
+
+    class InvalidEndpointException(Exception):
+        pass

src/cloudwatch/modules/configuration/confighelper.py

@@ -5,6 +5,8 @@
 from credentialsreader import CredentialsReader
 from whitelist import Whitelist, WhitelistConfigReader
 from ..client.ec2getclient import EC2GetClient
+from ..client.stsassumeroleclient import StsAssumRoleClient
+from ..plugininfo import PLUGIN_NAME, PLUGIN_VERSION
 import traceback
 
 class ConfigHelper(object):
@@ -35,9 +37,11 @@ def __init__(self, config_path=_DEFAULT_CONFIG_PATH, metadata_server=_METADATA_S
         self._config_path = config_path
         self._metadata_server = metadata_server
         self._use_iam_role_credentials = False
+        self._arn_role = ''
         self.region = ''
         self.endpoint = ''
         self.ec2_endpoint = ''
+        self.sts_endpoint = ''
         self.host = ''
         self.asg_name = 'NONE'
         self.proxy_server_name = ''
@@ -58,11 +62,19 @@ def credentials(self):
         Returns credentials. If IAM role is used, credentials will be updated.
         Otherwise old credentials are returned.
         """
-        if self._use_iam_role_credentials:
+        if self._use_iam_role_credentials and self._credentials.is_expired():
             try:
                 self._credentials = self._get_credentials_from_iam_role()
             except:
                 self._LOGGER.warning("Could not retrieve credentials using IAM Role. Using old credentials instead.")
+        elif self._arn_role and self._credentials and self._credentials.is_expired():
+            try:
+                # First use iam role to query sts temporary credentials
+                self._credentials = self._get_credentials_from_iam_role()
+                self._credentials = self._get_credentials_by_sts_assuming_role()
+            except:
+                self._LOGGER.warning("Could not retrieve credentials assuming IAM Role. Using old credentials instead.")
+
         return self._credentials
 
     @credentials.setter
@@ -77,13 +89,16 @@ def _load_configuration(self):
         self._load_credentials()
         self._load_region()
         self._load_hostname()
+        self._load_arn_role()
         self._load_proxy_server_name()
         self._load_proxy_server_port()
         self.enable_high_resolution_metrics = self.config_reader.enable_high_resolution_metrics
         self._load_flush_interval_in_seconds()
         self._set_endpoint()
         self._set_ec2_endpoint()
+        self._set_sts_endpoint()
         self._load_autoscaling_group()
+        self._overwrite_credentials_by_assuming_role()
         self.debug = self.config_reader.debug
         self.pass_through = self.config_reader.pass_through
         self.push_asg = self.config_reader.push_asg
@@ -107,6 +122,25 @@ def _load_credentials(self):
             self._use_iam_role_credentials = True
             self.credentials = self._get_credentials_from_iam_role()
 
+    def _overwrite_credentials_by_assuming_role(self):
+        """
+        Tries to load and overwrite credentials with new credentials got by assuming role if 
+        any arn_role was provided.
+        """
+        if self._arn_role and self.credentials:
+            try:
+                self.credentials = self._get_credentials_by_sts_assuming_role()
+                self._use_iam_role_credentials = False
+            except Exception as e:
+                self._LOGGER.error("Failed to set credentials by assuming role. Continue by iam role credentials. Cause: " + str(e))
+
+    def _get_credentials_by_sts_assuming_role(self):
+        #Don't use credentials getter method, otherwise it runs in an infinit loop
+        stsAssumRoleClient = StsAssumRoleClient(self._credentials, self.sts_endpoint, self.region, self.proxy_server_name, self.proxy_server_port, self.debug)
+        duration_seconds = 3600
+        role_session_name = PLUGIN_NAME + "_v" + str(PLUGIN_VERSION)
+        return stsAssumRoleClient.get_credentials(self._arn_role, role_session_name, duration_seconds)
+
     def _get_credentials_from_iam_role(self):
         """ Queries IAM Role metadata for latest credentials """
         return self.metadata_reader.get_iam_role_credentials(self.metadata_reader.get_iam_role_name())
@@ -137,7 +171,14 @@ def _load_hostname(self):
             except Exception as e:
                 ConfigHelper._LOGGER.warning("Cannot retrieve Instance ID from the local metadata server. Cause: " + str(e) +  
                     " Using host information provided by Collectd.")
-
+
+    def _load_arn_role(self):
+        """
+        Loads arn_role from plugin configuration file.
+        """
+        if self.config_reader.arn_role:
+            self._arn_role = self.config_reader.arn_role
+
     def _set_ec2_endpoint(self):
         """ Creates endpoint from region information """
         if self.region is "localhost":
@@ -146,6 +187,15 @@ def _set_ec2_endpoint(self):
             self.ec2_endpoint = "https://ec2." + self.region + ".amazonaws.com.cn/"
         else:
             self.ec2_endpoint = "https://ec2." + self.region + ".amazonaws.com/"
+
+    def _set_sts_endpoint(self):
+        """ Creates endpoint from region information """
+        if self.region is "localhost":
+            self.sts_endpoint = "http://" + self.region + "/"
+        elif self.region.startswith("cn-"):
+            self.sts_endpoint = "https://sts." + self.region + ".amazonaws.com.cn/"
+        else:
+            self.sts_endpoint = "https://sts." + self.region + ".amazonaws.com/"
 
     def _load_proxy_server_name(self):
         """

src/cloudwatch/modules/configuration/configreader.py

@@ -27,6 +27,7 @@ class ConfigReader(object):
     _PASS_THROUGH_DEFAULT_VALUE = False
     _PUSH_ASG_DEFAULT_VALUE = False
     _PUSH_CONSTANT_DEFAULT_VALUE = False
+    ARN_ROLE_CONFIG_KEY = "arn_role"
     REGION_CONFIG_KEY = "region"
     HOST_CONFIG_KEY = "host"
     CREDENTIALS_PATH_KEY = "credentials_path"
@@ -43,6 +44,7 @@ class ConfigReader(object):
     def __init__(self, config_path):
         self.config_path = config_path
         self.credentials_path = ""
+        self.arn_role = ''
         self.region = ''
         self.host = ''
         self.pass_through = self._PASS_THROUGH_DEFAULT_VALUE
@@ -67,6 +69,7 @@ def _parse_config_file(self):
         in format ['key=value', 'key2=value2'] 
         """
         self.credentials_path = self.reader_utils.get_string(self.CREDENTIALS_PATH_KEY)
+        self.arn_role = self.reader_utils.get_string(self.ARN_ROLE_CONFIG_KEY)
         self.host = self.reader_utils.get_string(self.HOST_CONFIG_KEY)
         self.region = self.reader_utils.get_string(self.REGION_CONFIG_KEY)
         self.proxy_server_name = self.reader_utils.get_string(self.PROXY_SERVER_NAME_KEY)

src/cloudwatch/modules/configuration/metadatareader.py

@@ -43,8 +43,8 @@ def get_iam_role_credentials(self, role_name):
         """ Get the IAMRoleCredentials object with values from IAM metadata """
         try:
             iam_data = loads(self._get_metadata(self._IAM_ROLE_CREDENTIAL_REQUEST + role_name))
-            if iam_data['AccessKeyId'] and iam_data['SecretAccessKey'] and iam_data['Token']:
-                return AWSCredentials(iam_data['AccessKeyId'], iam_data['SecretAccessKey'], iam_data['Token'])
+            if iam_data['AccessKeyId'] and iam_data['SecretAccessKey'] and iam_data['Token'] and iam_data['Expiration']:
+                return AWSCredentials(iam_data['AccessKeyId'], iam_data['SecretAccessKey'], iam_data['Token'], iam_data['Expiration'])
             else:
                 raise ValueError("Incomplete credentials retrieved.")
         except Exception as e: