Skip to content

Commit a248688

Browse files
hmstepanekhmstepanek
authored
Instrument RedisCluster (#809)
* Add instrumentation for RedisCluster * Add tests for redis cluster
1 parent 56ea815 commit a248688

File tree

8 files changed

+337
-1
lines changed

8 files changed

+337
-1
lines changed

‎.github/workflows/tests.yml

+100
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ jobs:
4646
- postgres
4747
- rabbitmq
4848
- redis
49+
- rediscluster
4950
- solr
5051

5152
steps:
@@ -384,6 +385,105 @@ jobs:
384385
path: ./**/.coverage.*
385386
retention-days: 1
386387

388+
rediscluster:
389+
env:
390+
TOTAL_GROUPS: 1
391+
392+
strategy:
393+
fail-fast: false
394+
matrix:
395+
group-number: [1]
396+
397+
runs-on: ubuntu-20.04
398+
container:
399+
image: ghcr.io/newrelic/newrelic-python-agent-ci:latest
400+
options: >-
401+
--add-host=host.docker.internal:host-gateway
402+
timeout-minutes: 30
403+
404+
services:
405+
redis1:
406+
image: hmstepanek/redis-cluster-node:1.0.0
407+
ports:
408+
- 6379:6379
409+
- 16379:16379
410+
options: >-
411+
--add-host=host.docker.internal:host-gateway
412+
413+
redis2:
414+
image: hmstepanek/redis-cluster-node:1.0.0
415+
ports:
416+
- 6380:6379
417+
- 16380:16379
418+
options: >-
419+
--add-host=host.docker.internal:host-gateway
420+
421+
redis3:
422+
image: hmstepanek/redis-cluster-node:1.0.0
423+
ports:
424+
- 6381:6379
425+
- 16381:16379
426+
options: >-
427+
--add-host=host.docker.internal:host-gateway
428+
429+
redis4:
430+
image: hmstepanek/redis-cluster-node:1.0.0
431+
ports:
432+
- 6382:6379
433+
- 16382:16379
434+
options: >-
435+
--add-host=host.docker.internal:host-gateway
436+
437+
redis5:
438+
image: hmstepanek/redis-cluster-node:1.0.0
439+
ports:
440+
- 6383:6379
441+
- 16383:16379
442+
options: >-
443+
--add-host=host.docker.internal:host-gateway
444+
445+
redis6:
446+
image: hmstepanek/redis-cluster-node:1.0.0
447+
ports:
448+
- 6384:6379
449+
- 16384:16379
450+
options: >-
451+
--add-host=host.docker.internal:host-gateway
452+
453+
cluster-setup:
454+
image: hmstepanek/redis-cluster:1.0.0
455+
options: >-
456+
--add-host=host.docker.internal:host-gateway
457+
458+
steps:
459+
- uses: actions/checkout@v3
460+
461+
- name: Fetch git tags
462+
run: |
463+
git config --global --add safe.directory "$GITHUB_WORKSPACE"
464+
git fetch --tags origin
465+
466+
- name: Get Environments
467+
id: get-envs
468+
run: |
469+
echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT
470+
env:
471+
GROUP_NUMBER: ${{ matrix.group-number }}
472+
473+
- name: Test
474+
run: |
475+
tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto
476+
env:
477+
TOX_PARALLEL_NO_SPINNER: 1
478+
PY_COLORS: 0
479+
480+
- name: Upload Coverage Artifacts
481+
uses: actions/upload-artifact@v3
482+
with:
483+
name: coverage-${{ github.job }}-${{ strategy.job-index }}
484+
path: ./**/.coverage.*
485+
retention-days: 1
486+
387487
redis:
388488
env:
389489
TOTAL_GROUPS: 2

‎newrelic/config.py

+4
Original file line numberDiff line numberDiff line change
@@ -2849,6 +2849,10 @@ def _process_module_builtin_defaults():
28492849
)
28502850
_process_module_definition("redis.client", "newrelic.hooks.datastore_redis", "instrument_redis_client")
28512851

2852+
_process_module_definition(
2853+
"redis.commands.cluster", "newrelic.hooks.datastore_redis", "instrument_redis_commands_cluster"
2854+
)
2855+
28522856
_process_module_definition(
28532857
"redis.commands.core", "newrelic.hooks.datastore_redis", "instrument_redis_commands_core"
28542858
)

‎newrelic/hooks/datastore_redis.py

+4
Original file line numberDiff line numberDiff line change
@@ -597,6 +597,10 @@ def instrument_redis_commands_bf_commands(module):
597597
_instrument_redis_commands_module(module, "TOPKCommands")
598598

599599

600+
def instrument_redis_commands_cluster(module):
601+
_instrument_redis_commands_module(module, "RedisClusterCommands")
602+
603+
600604
def _instrument_redis_commands_module(module, class_name):
601605
for name in _redis_client_methods:
602606
if hasattr(module, class_name):

‎setup.cfg

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@ license_files =
55

66
[flake8]
77
max-line-length = 120
8-
extend-ignore = E122,E126,E127,E128,E203,E501,E722,F841,W504
8+
extend-ignore = E122,E126,E127,E128,E203,E501,E722,F841,W504,E731

‎tests/datastore_rediscluster/conftest.py

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Copyright 2010 New Relic, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611
16+
collector_agent_registration_fixture,
17+
collector_available_fixture,
18+
)
19+
20+
_default_settings = {
21+
"transaction_tracer.explain_threshold": 0.0,
22+
"transaction_tracer.transaction_threshold": 0.0,
23+
"transaction_tracer.stack_trace_threshold": 0.0,
24+
"debug.log_data_collector_payloads": True,
25+
"debug.record_transaction_failure": True,
26+
}
27+
28+
collector_agent_registration = collector_agent_registration_fixture(
29+
app_name="Python Agent Test (datastore_redis)",
30+
default_settings=_default_settings,
31+
linked_applications=["Python Agent Test (datastore)"],
32+
)

‎tests/datastore_rediscluster/test_uninstrumented_rediscluster_methods.py

+168
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
# Copyright 2010 New Relic, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import redis
16+
from testing_support.db_settings import redis_cluster_settings
17+
18+
DB_CLUSTER_SETTINGS = redis_cluster_settings()[0]
19+
20+
# Set socket_timeout to 5s for fast fail, otherwise the default is to wait forever.
21+
client = redis.RedisCluster(host=DB_CLUSTER_SETTINGS["host"], port=DB_CLUSTER_SETTINGS["port"], socket_timeout=5)
22+
23+
IGNORED_METHODS = {
24+
"MODULE_CALLBACKS",
25+
"MODULE_VERSION",
26+
"NAME",
27+
"add_edge",
28+
"add_node",
29+
"append_bucket_size",
30+
"append_capacity",
31+
"append_error",
32+
"append_expansion",
33+
"append_items_and_increments",
34+
"append_items",
35+
"append_max_iterations",
36+
"append_no_create",
37+
"append_no_scale",
38+
"append_values_and_weights",
39+
"append_weights",
40+
"batch_indexer",
41+
"BatchIndexer",
42+
"bulk",
43+
"call_procedure",
44+
"client_tracking_off",
45+
"client_tracking_on",
46+
"client",
47+
"close",
48+
"commandmixin",
49+
"connection_pool",
50+
"connection",
51+
"debug_segfault",
52+
"edges",
53+
"execute_command",
54+
"flush",
55+
"from_url",
56+
"get_connection_kwargs",
57+
"get_encoder",
58+
"get_label",
59+
"get_params_args",
60+
"get_property",
61+
"get_relation",
62+
"get_retry",
63+
"hscan_iter",
64+
"index_name",
65+
"labels",
66+
"list_keys",
67+
"load_document",
68+
"load_external_module",
69+
"lock",
70+
"name",
71+
"nodes",
72+
"parse_response",
73+
"pipeline",
74+
"property_keys",
75+
"register_script",
76+
"relationship_types",
77+
"response_callbacks",
78+
"RESPONSE_CALLBACKS",
79+
"sentinel",
80+
"set_file",
81+
"set_path",
82+
"set_response_callback",
83+
"set_retry",
84+
"transaction",
85+
"version",
86+
"ALL_NODES",
87+
"CLUSTER_COMMANDS_RESPONSE_CALLBACKS",
88+
"COMMAND_FLAGS",
89+
"DEFAULT_NODE",
90+
"ERRORS_ALLOW_RETRY",
91+
"NODE_FLAGS",
92+
"PRIMARIES",
93+
"RANDOM",
94+
"REPLICAS",
95+
"RESULT_CALLBACKS",
96+
"RedisClusterRequestTTL",
97+
"SEARCH_COMMANDS",
98+
"client_no_touch",
99+
"cluster_addslotsrange",
100+
"cluster_bumpepoch",
101+
"cluster_delslotsrange",
102+
"cluster_error_retry_attempts",
103+
"cluster_flushslots",
104+
"cluster_links",
105+
"cluster_myid",
106+
"cluster_myshardid",
107+
"cluster_replicas",
108+
"cluster_response_callbacks",
109+
"cluster_setslot_stable",
110+
"cluster_shards",
111+
"command_flags",
112+
"commands_parser",
113+
"determine_slot",
114+
"disconnect_connection_pools",
115+
"encoder",
116+
"get_default_node",
117+
"get_node",
118+
"get_node_from_key",
119+
"get_nodes",
120+
"get_primaries",
121+
"get_random_node",
122+
"get_redis_connection",
123+
"get_replicas",
124+
"keyslot",
125+
"mget_nonatomic",
126+
"monitor",
127+
"mset_nonatomic",
128+
"node_flags",
129+
"nodes_manager",
130+
"on_connect",
131+
"pubsub",
132+
"read_from_replicas",
133+
"reinitialize_counter",
134+
"reinitialize_steps",
135+
"replace_default_node",
136+
"result_callbacks",
137+
"set_default_node",
138+
"user_on_connect_func",
139+
}
140+
141+
REDIS_MODULES = {
142+
"bf",
143+
"cf",
144+
"cms",
145+
"ft",
146+
"graph",
147+
"json",
148+
"tdigest",
149+
"topk",
150+
"ts",
151+
}
152+
153+
IGNORED_METHODS |= REDIS_MODULES
154+
155+
156+
def test_uninstrumented_methods():
157+
methods = {m for m in dir(client) if not m[0] == "_"}
158+
is_wrapped = lambda m: hasattr(getattr(client, m), "__wrapped__")
159+
uninstrumented = {m for m in methods - IGNORED_METHODS if not is_wrapped(m)}
160+
161+
for module in REDIS_MODULES:
162+
if hasattr(client, module):
163+
module_client = getattr(client, module)()
164+
module_methods = {m for m in dir(module_client) if not m[0] == "_"}
165+
is_wrapped = lambda m: hasattr(getattr(module_client, m), "__wrapped__")
166+
uninstrumented |= {m for m in module_methods - IGNORED_METHODS if not is_wrapped(m)}
167+
168+
assert not uninstrumented, "Uninstrumented methods: %s" % sorted(uninstrumented)

‎tests/testing_support/db_settings.py

+25
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,31 @@ def redis_settings():
121121
return settings
122122

123123

124+
def redis_cluster_settings():
125+
"""Return a list of dict of settings for connecting to redis cluster.
126+
127+
Will return the correct settings, depending on which of the environments it
128+
is running in. It attempts to set variables in the following order, where
129+
later environments override earlier ones.
130+
131+
1. Local
132+
2. Github Actions
133+
"""
134+
135+
host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost"
136+
instances = 1
137+
base_port = 6379
138+
139+
settings = [
140+
{
141+
"host": host,
142+
"port": base_port + instance_num,
143+
}
144+
for instance_num in range(instances)
145+
]
146+
return settings
147+
148+
124149
def memcached_settings():
125150
"""Return a list of dict of settings for connecting to memcached.
126151

‎tox.ini

+3
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ envlist =
9696
solr-datastore_pysolr-{py27,py37,py38,py39,py310,py311,pypy27,pypy38},
9797
redis-datastore_redis-{py27,py37,py38,pypy27,pypy38}-redis03,
9898
redis-datastore_redis-{py37,py38,py39,py310,py311,pypy38}-redis{0400,latest},
99+
rediscluster-datastore_rediscluster-{py37,py311,pypy38}-redis{latest},
99100
redis-datastore_aioredis-{py37,py38,py39,py310,pypy38}-aioredislatest,
100101
redis-datastore_aioredis-{py37,py38,py39,py310,py311,pypy38}-redislatest,
101102
redis-datastore_aredis-{py37,py38,py39,pypy38}-aredislatest,
@@ -254,6 +255,7 @@ deps =
254255
datastore_pymysql: PyMySQL<0.11
255256
datastore_pysolr: pysolr<4.0
256257
datastore_redis-redislatest: redis
258+
datastore_rediscluster-redislatest: redis
257259
datastore_redis-redis0400: redis<4.1
258260
datastore_redis-redis03: redis<4.0
259261
datastore_redis-{py27,pypy27}: rb
@@ -457,6 +459,7 @@ changedir =
457459
datastore_pymysql: tests/datastore_pymysql
458460
datastore_pysolr: tests/datastore_pysolr
459461
datastore_redis: tests/datastore_redis
462+
datastore_rediscluster: tests/datastore_rediscluster
460463
datastore_aioredis: tests/datastore_aioredis
461464
datastore_aredis: tests/datastore_aredis
462465
datastore_sqlite: tests/datastore_sqlite

0 commit comments

Comments
 (0)