Merge pull request #192 from chanchiem/hostname

Downstream Http Calls should use hostname rather than full URL as subsegment name

Author: chanchiem

aws_xray_sdk/ext/aiohttp/client.py

@@ -8,7 +8,7 @@
 from aws_xray_sdk.core import xray_recorder
 from aws_xray_sdk.core.models import http
 from aws_xray_sdk.core.utils import stacktrace
-from aws_xray_sdk.ext.util import inject_trace_header, strip_url
+from aws_xray_sdk.ext.util import inject_trace_header, strip_url, get_hostname
 
 # All aiohttp calls will entail outgoing HTTP requests, only in some ad-hoc
 # exceptions the namespace will be flip back to local.
@@ -22,7 +22,7 @@
 
 
 async def begin_subsegment(session, trace_config_ctx, params):
-    name = trace_config_ctx.name if trace_config_ctx.name else strip_url(str(params.url))
+    name = trace_config_ctx.name if trace_config_ctx.name else get_hostname(str(params.url))
     subsegment = xray_recorder.begin_subsegment(name, REMOTE_NAMESPACE)
 
     # No-op if subsegment is `None` due to `LOG_ERROR`.
@@ -31,7 +31,7 @@ async def begin_subsegment(session, trace_config_ctx, params):
     else:
         trace_config_ctx.give_up = False
         subsegment.put_http_meta(http.METHOD, params.method)
-        subsegment.put_http_meta(http.URL, params.url.human_repr())
+        subsegment.put_http_meta(http.URL, strip_url(params.url.human_repr()))
         inject_trace_header(params.headers, subsegment)
 
 

aws_xray_sdk/ext/httplib/patch.py

@@ -8,7 +8,7 @@
 from aws_xray_sdk.core.models import http
 from aws_xray_sdk.core.exceptions.exceptions import SegmentNotFoundException
 from aws_xray_sdk.core.patcher import _PATCHED_MODULES
-from aws_xray_sdk.ext.util import inject_trace_header, strip_url, unwrap
+from aws_xray_sdk.ext.util import inject_trace_header, strip_url, unwrap, get_hostname
 
 if sys.version_info >= (3, 0, 0):
     PY2 = False
@@ -33,7 +33,7 @@ def http_response_processor(wrapped, instance, args, kwargs, return_value,
         return
 
     subsegment.put_http_meta(http.METHOD, xray_data.method)
-    subsegment.put_http_meta(http.URL, xray_data.url)
+    subsegment.put_http_meta(http.URL, strip_url(xray_data.url))
 
     if return_value:
         subsegment.put_http_meta(http.STATUS, return_value.status)
@@ -57,7 +57,7 @@ def _xray_traced_http_getresponse(wrapped, instance, args, kwargs):
 
     return xray_recorder.record_subsegment(
         wrapped, instance, args, kwargs,
-        name=strip_url(xray_data.url),
+        name=get_hostname(xray_data.url),
         namespace='remote',
         meta_processor=http_response_processor,
     )
@@ -71,7 +71,7 @@ def http_send_request_processor(wrapped, instance, args, kwargs, return_value,
 
     # we don't delete the attr as we can have multiple reads
     subsegment.put_http_meta(http.METHOD, xray_data.method)
-    subsegment.put_http_meta(http.URL, xray_data.url)
+    subsegment.put_http_meta(http.URL, strip_url(xray_data.url))
 
     if exception:
         subsegment.add_exception(exception, stack)
@@ -111,7 +111,7 @@ def decompose_args(method, url, body, headers, encode_chunked=False):
         # we add a segment here in case connect fails
         return xray_recorder.record_subsegment(
             wrapped, instance, args, kwargs,
-            name=strip_url(xray_data.url),
+            name=get_hostname(xray_data.url),
             namespace='remote',
             meta_processor=http_send_request_processor
         )
@@ -127,7 +127,7 @@ def http_read_processor(wrapped, instance, args, kwargs, return_value,
 
     # we don't delete the attr as we can have multiple reads
     subsegment.put_http_meta(http.METHOD, xray_data.method)
-    subsegment.put_http_meta(http.URL, xray_data.url)
+    subsegment.put_http_meta(http.URL, strip_url(xray_data.url))
     subsegment.put_http_meta(http.STATUS, instance.status)
 
     if exception:
@@ -141,7 +141,7 @@ def _xray_traced_http_client_read(wrapped, instance, args, kwargs):
 
     return xray_recorder.record_subsegment(
         wrapped, instance, args, kwargs,
-        name=strip_url(xray_data.url),
+        name=get_hostname(xray_data.url),
         namespace='remote',
         meta_processor=http_read_processor
     )

aws_xray_sdk/ext/requests/patch.py

@@ -2,7 +2,7 @@
 
 from aws_xray_sdk.core import xray_recorder
 from aws_xray_sdk.core.models import http
-from aws_xray_sdk.ext.util import inject_trace_header, strip_url
+from aws_xray_sdk.ext.util import inject_trace_header, strip_url, get_hostname
 
 
 def patch():
@@ -26,7 +26,7 @@ def _xray_traced_requests(wrapped, instance, args, kwargs):
 
     return xray_recorder.record_subsegment(
         wrapped, instance, args, kwargs,
-        name=strip_url(url),
+        name=get_hostname(url),
         namespace='remote',
         meta_processor=requests_processor,
     )
@@ -48,7 +48,7 @@ def requests_processor(wrapped, instance, args, kwargs,
     url = kwargs.get('url') or args[1]
 
     subsegment.put_http_meta(http.METHOD, method)
-    subsegment.put_http_meta(http.URL, url)
+    subsegment.put_http_meta(http.URL, strip_url(url))
 
     if return_value is not None:
         subsegment.put_http_meta(http.STATUS, return_value.status_code)

aws_xray_sdk/ext/util.py

@@ -4,10 +4,17 @@
 from aws_xray_sdk.core.models import http
 
 import wrapt
+import sys
+
+if sys.version_info.major >= 3:  # Python 3 and above
+    from urllib.parse import urlparse
+else:  # Python 2 and below
+    from urlparse import urlparse
 
 
 first_cap_re = re.compile('(.)([A-Z][a-z]+)')
 all_cap_re = re.compile('([a-z0-9])([A-Z])')
+UNKNOWN_HOSTNAME = "UNKNOWN HOST"
 
 
 def inject_trace_header(headers, entity):
@@ -118,6 +125,16 @@ def strip_url(url):
     return url.partition('?')[0] if url else url
 
 
+def get_hostname(url):
+    if url is None:
+        return UNKNOWN_HOSTNAME
+    url_parse = urlparse(url)
+    hostname = url_parse.hostname
+    if hostname is None:
+        return UNKNOWN_HOSTNAME
+    return hostname if hostname else url  # If hostname is none, we return the regular URL; indication of malformed url
+
+
 def unwrap(obj, attr):
     """
     Will unwrap a `wrapt` attribute

tests/ext/aiohttp/test_client.py

@@ -4,7 +4,7 @@
 from aws_xray_sdk.core import xray_recorder
 from aws_xray_sdk.core.async_context import AsyncContext
 from aws_xray_sdk.core.exceptions.exceptions import SegmentNotFoundException
-from aws_xray_sdk.ext.util import strip_url
+from aws_xray_sdk.ext.util import strip_url, get_hostname
 from aws_xray_sdk.ext.aiohttp.client import aws_xray_trace_config
 from aws_xray_sdk.ext.aiohttp.client import REMOTE_NAMESPACE, LOCAL_NAMESPACE
 
@@ -34,11 +34,11 @@ async def test_ok(loop, recorder):
             pass
 
     subsegment = xray_recorder.current_segment().subsegments[0]
-    assert subsegment.name == strip_url(url)
+    assert subsegment.name == get_hostname(url)
     assert subsegment.namespace == REMOTE_NAMESPACE
 
     http_meta = subsegment.http
-    assert http_meta['request']['url'] == url
+    assert http_meta['request']['url'] == strip_url(url)
     assert http_meta['request']['method'] == 'GET'
     assert http_meta['response']['status'] == status_code
 
@@ -66,11 +66,11 @@ async def test_error(loop, recorder):
             pass
 
     subsegment = xray_recorder.current_segment().subsegments[0]
-    assert subsegment.name == url
+    assert subsegment.name == get_hostname(url)
     assert subsegment.error
 
     http_meta = subsegment.http
-    assert http_meta['request']['url'] == url
+    assert http_meta['request']['url'] == strip_url(url)
     assert http_meta['request']['method'] == 'POST'
     assert http_meta['response']['status'] == status_code
 
@@ -85,12 +85,12 @@ async def test_throttle(loop, recorder):
             pass
 
     subsegment = xray_recorder.current_segment().subsegments[0]
-    assert subsegment.name == url
+    assert subsegment.name == get_hostname(url)
     assert subsegment.error
     assert subsegment.throttle
 
     http_meta = subsegment.http
-    assert http_meta['request']['url'] == url
+    assert http_meta['request']['url'] == strip_url(url)
     assert http_meta['request']['method'] == 'HEAD'
     assert http_meta['response']['status'] == status_code
 
@@ -105,11 +105,11 @@ async def test_fault(loop, recorder):
             pass
 
     subsegment = xray_recorder.current_segment().subsegments[0]
-    assert subsegment.name == url
+    assert subsegment.name == get_hostname(url)
     assert subsegment.fault
 
     http_meta = subsegment.http
-    assert http_meta['request']['url'] == url
+    assert http_meta['request']['url'] == strip_url(url)
     assert http_meta['request']['method'] == 'PUT'
     assert http_meta['response']['status'] == status_code
 

tests/ext/httplib/test_httplib.py

@@ -4,7 +4,7 @@
 from aws_xray_sdk.core import patch
 from aws_xray_sdk.core import xray_recorder
 from aws_xray_sdk.core.context import Context
-from aws_xray_sdk.ext.util import strip_url
+from aws_xray_sdk.ext.util import strip_url, get_hostname
 
 if sys.version_info >= (3, 0, 0):
     import http.client as httplib
@@ -57,10 +57,10 @@ def test_ok():
     url = 'https://{}/status/{}?foo=bar&baz=foo'.format(BASE_URL, status_code)
     _do_req(url)
     subsegment = xray_recorder.current_segment().subsegments[1]
-    assert subsegment.name == strip_url(url)
+    assert subsegment.name == get_hostname(url)
 
     http_meta = subsegment.http
-    assert http_meta['request']['url'] == url
+    assert http_meta['request']['url'] == strip_url(url)
     assert http_meta['request']['method'].upper() == 'GET'
     assert http_meta['response']['status'] == status_code
 
@@ -70,11 +70,11 @@ def test_error():
     url = 'https://{}/status/{}'.format(BASE_URL, status_code)
     _do_req(url, 'POST')
     subsegment = xray_recorder.current_segment().subsegments[1]
-    assert subsegment.name == url
+    assert subsegment.name == get_hostname(url)
     assert subsegment.error
 
     http_meta = subsegment.http
-    assert http_meta['request']['url'] == url
+    assert http_meta['request']['url'] == strip_url(url)
     assert http_meta['request']['method'].upper() == 'POST'
     assert http_meta['response']['status'] == status_code
 
@@ -84,12 +84,12 @@ def test_throttle():
     url = 'https://{}/status/{}'.format(BASE_URL, status_code)
     _do_req(url, 'HEAD')
     subsegment = xray_recorder.current_segment().subsegments[1]
-    assert subsegment.name == url
+    assert subsegment.name == get_hostname(url)
     assert subsegment.error
     assert subsegment.throttle
 
     http_meta = subsegment.http
-    assert http_meta['request']['url'] == url
+    assert http_meta['request']['url'] == strip_url(url)
     assert http_meta['request']['method'].upper() == 'HEAD'
     assert http_meta['response']['status'] == status_code
 
@@ -99,11 +99,11 @@ def test_fault():
     url = 'https://{}/status/{}'.format(BASE_URL, status_code)
     _do_req(url, 'PUT')
     subsegment = xray_recorder.current_segment().subsegments[1]
-    assert subsegment.name == url
+    assert subsegment.name == get_hostname(url)
     assert subsegment.fault
 
     http_meta = subsegment.http
-    assert http_meta['request']['url'] == url
+    assert http_meta['request']['url'] == strip_url(url)
     assert http_meta['request']['method'].upper() == 'PUT'
     assert http_meta['response']['status'] == status_code
 
@@ -126,7 +126,7 @@ def test_correct_identify_http():
     url = 'http://{}/status/{}?foo=bar&baz=foo'.format(BASE_URL, status_code)
     _do_req(url, use_https=False)
     subsegment = xray_recorder.current_segment().subsegments[0]
-    assert subsegment.name == strip_url(url)
+    assert subsegment.name == get_hostname(url)
 
     http_meta = subsegment.http
     assert http_meta['request']['url'].split(":")[0] == 'http'
@@ -137,7 +137,7 @@ def test_correct_identify_https():
     url = 'https://{}/status/{}?foo=bar&baz=foo'.format(BASE_URL, status_code)
     _do_req(url, use_https=True)
     subsegment = xray_recorder.current_segment().subsegments[0]
-    assert subsegment.name == strip_url(url)
+    assert subsegment.name == get_hostname(url)
 
     https_meta = subsegment.http
     assert https_meta['request']['url'].split(":")[0] == 'https'