Skip to content

Commit 7fc9f37

Browse files
the-infinitythe-infinity
committed
OpenTelemetry logging
1 parent 0f26c09 commit 7fc9f37

18 files changed

+536
-24
lines changed

‎.pre-commit-config.yaml

+22-6
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,24 @@
11
repos:
2-
- repo: local
2+
- repo: https://github.com/pre-commit/pre-commit-hooks
3+
rev: v5.0.0
34
hooks:
4-
- id: format-and-lint
5-
name: Check formatting and linting
6-
entry: mobidata-bw-proxy-proxy:latest ruff ./app
7-
language: docker_image
8-
types: [ python ]
5+
- id: trailing-whitespace
6+
- id: end-of-file-fixer
7+
- id: check-yaml
8+
- id: check-added-large-files
9+
10+
- repo: https://github.com/astral-sh/ruff-pre-commit
11+
rev: v0.9.10
12+
hooks:
13+
- id: ruff-format
14+
- id: ruff
15+
args: [ --fix, --exit-non-zero-on-fix ]
16+
17+
- repo: https://github.com/pre-commit/mirrors-mypy
18+
rev: v1.15.0
19+
hooks:
20+
- id: mypy
21+
exclude: ^(tests|dev)/
22+
additional_dependencies: [
23+
'types-PyYAML',
24+
]

‎addons.py

+13-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,19 @@
11
"""
22
MobiData BW Proxy
33
Copyright (c) 2023, binary butterfly GmbH
4-
All rights reserved.
4+
5+
Licensed under the EUPL, Version 1.2 or – as soon they will be approved by
6+
the European Commission - subsequent versions of the EUPL (the "Licence");
7+
You may not use this work except in compliance with the Licence.
8+
You may obtain a copy of the Licence at:
9+
10+
https://joinup.ec.europa.eu/software/page/eupl
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the Licence is distributed on an "AS IS" basis,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the Licence for the specific language governing permissions and
16+
limitations under the Licence.
517
"""
618

719
from app import App

‎app/__init__.py

+13-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,19 @@
11
"""
22
MobiData BW Proxy
33
Copyright (c) 2023, binary butterfly GmbH
4-
All rights reserved.
4+
5+
Licensed under the EUPL, Version 1.2 or – as soon they will be approved by
6+
the European Commission - subsequent versions of the EUPL (the "Licence");
7+
You may not use this work except in compliance with the Licence.
8+
You may obtain a copy of the Licence at:
9+
10+
https://joinup.ec.europa.eu/software/page/eupl
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the Licence is distributed on an "AS IS" basis,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the Licence for the specific language governing permissions and
16+
limitations under the Licence.
517
"""
618

719
from .app import App

‎app/app.py

+63-8
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,55 @@
11
"""
22
MobiData BW Proxy
33
Copyright (c) 2023, binary butterfly GmbH
4-
All rights reserved.
4+
5+
Licensed under the EUPL, Version 1.2 or – as soon they will be approved by
6+
the European Commission - subsequent versions of the EUPL (the "Licence");
7+
You may not use this work except in compliance with the Licence.
8+
You may obtain a copy of the Licence at:
9+
10+
https://joinup.ec.europa.eu/software/page/eupl
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the Licence is distributed on an "AS IS" basis,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the Licence for the specific language governing permissions and
16+
limitations under the Licence.
517
"""
618

719
import json
820
import logging
21+
import traceback
922
from importlib import import_module
1023
from inspect import isclass
1124
from json import JSONDecodeError
25+
from logging.config import dictConfig
1226
from pathlib import Path
1327
from pkgutil import iter_modules
1428
from typing import Dict, List
1529

16-
from mitmproxy.http import HTTPFlow
30+
from mitmproxy.http import HTTPFlow, Response
1731

1832
from app.base_converter import BaseConverter
1933
from app.config_helper import ConfigHelper
34+
from app.utils import ContextHelper
35+
from app.utils.context_helper import context_helper
36+
from app.utils.default_json_encoder import DefaultJSONEncoder
2037

21-
logger = logging.getLogger('converters.requests')
38+
logger = logging.getLogger(__name__)
2239

2340

2441
class App:
42+
config_helper: ConfigHelper
43+
context_helper: ContextHelper
44+
2545
json_converters: Dict[str, List[BaseConverter]]
2646

2747
def __init__(self):
2848
self.config_helper = ConfigHelper()
49+
self.context_helper = context_helper
50+
51+
# configure logging
52+
dictConfig(self.config_helper.get('LOGGING'))
2953

3054
self.json_converters = {}
3155

@@ -48,29 +72,60 @@ def __init__(self):
4872
self.json_converters[hostname].append(obj)
4973

5074
def request(self, flow: HTTPFlow):
75+
self.context_helper.initialize_context()
76+
5177
if flow.request.host in self.config_helper.get('HTTP_TO_HTTPS_HOSTS', []):
5278
flow.request.scheme = 'https'
5379
flow.request.port = 443
5480

81+
self.context_helper.set_attribute('url.host', flow.request.host)
82+
self.context_helper.set_attribute('url.scheme', flow.request.scheme)
83+
self.context_helper.set_attribute('url.port', flow.request.port)
84+
self.context_helper.set_attribute('url.path', flow.request.path)
85+
5586
def response(self, flow: HTTPFlow):
5687
# Log requests
57-
logger.info(f'{flow.request.method} {flow.request.url}: HTTP {"-" if flow.response is None else flow.response.status_code}')
88+
logger.debug(f'{flow.request.method} {flow.request.url}: HTTP {"-" if flow.response is None else flow.response.status_code}')
5889

5990
# if there is no converter for the requested host, don't do anything
6091
if flow.request.host not in self.json_converters:
92+
logger.warning('No JSON converter for request.')
6193
return
6294

6395
# try to load the response. If there is any error, return.
64-
if not flow.response or not flow.response.text:
96+
if not flow.response:
97+
logger.warning('No response for request.')
6598
return
99+
100+
response: Response = flow.response
101+
102+
if not response.text:
103+
logger.warning('Empty response for request.')
104+
return
105+
66106
try:
67-
json_data = json.loads(flow.response.text)
107+
json_data = json.loads(response.text)
68108
except (JSONDecodeError, TypeError):
109+
logger.warning(f'Invalid JSON in request: {response.text}.')
69110
return
70111

71112
# iterate all converters and apply them
72113
for json_converter in self.json_converters[flow.request.host]:
73-
json_data = json_converter.convert(data=json_data, path=flow.request.path)
114+
self.context_helper.set_attribute('converter', json_converter.__class__.__name__)
115+
try:
116+
json_data = json_converter.convert(data=json_data, path=flow.request.path)
117+
except Exception as e:
118+
logger.error(
119+
f'Converter {json_converter.__class__.__name__} threw an exception {e.__class__.__name__}: {e}',
120+
extra={
121+
'attributes': {
122+
# This needs to be json_data, as json_data is maybe already transformed
123+
'data': json.dumps(json_data),
124+
'traceback': traceback.format_exc(),
125+
},
126+
},
127+
)
128+
return
74129

75130
# set the returning json content
76-
flow.response.text = json.dumps(json_data)
131+
flow.response.text = json.dumps(json_data, cls=DefaultJSONEncoder)

‎app/base_converter.py

+13-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,19 @@
11
"""
22
MobiData BW Proxy
33
Copyright (c) 2023, binary butterfly GmbH
4-
All rights reserved.
4+
5+
Licensed under the EUPL, Version 1.2 or – as soon they will be approved by
6+
the European Commission - subsequent versions of the EUPL (the "Licence");
7+
You may not use this work except in compliance with the Licence.
8+
You may obtain a copy of the Licence at:
9+
10+
https://joinup.ec.europa.eu/software/page/eupl
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the Licence is distributed on an "AS IS" basis,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the Licence for the specific language governing permissions and
16+
limitations under the Licence.
517
"""
618

719
from abc import ABC, abstractmethod

‎app/config_helper.py

+14-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,19 @@
11
"""
22
MobiData BW Proxy
33
Copyright (c) 2023, binary butterfly GmbH
4-
All rights reserved.
4+
5+
Licensed under the EUPL, Version 1.2 or – as soon they will be approved by
6+
the European Commission - subsequent versions of the EUPL (the "Licence");
7+
You may not use this work except in compliance with the Licence.
8+
You may obtain a copy of the Licence at:
9+
10+
https://joinup.ec.europa.eu/software/page/eupl
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the Licence is distributed on an "AS IS" basis,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the Licence for the specific language governing permissions and
16+
limitations under the Licence.
517
"""
618

719
from pathlib import Path
@@ -22,5 +34,5 @@ def __init__(self):
2234
else:
2335
self._config = {}
2436

25-
def get(self, key: str, default: Any) -> Any:
37+
def get(self, key: str, default: Any = None) -> Any:
2638
return self._config.get(key, default)

‎app/converters/donkey_free_bike_status_fix.py

+14-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,19 @@
11
"""
22
MobiData BW Proxy
33
Copyright (c) 2023, systect Holger Bruch
4-
All rights reserved.
4+
5+
Licensed under the EUPL, Version 1.2 or – as soon they will be approved by
6+
the European Commission - subsequent versions of the EUPL (the "Licence");
7+
You may not use this work except in compliance with the Licence.
8+
You may obtain a copy of the Licence at:
9+
10+
https://joinup.ec.europa.eu/software/page/eupl
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the Licence is distributed on an "AS IS" basis,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the Licence for the specific language governing permissions and
16+
limitations under the Licence.
517
"""
618

719
import logging
@@ -10,7 +22,7 @@
1022

1123
from app.base_converter import BaseConverter
1224

13-
logger = logging.getLogger('converters.DonkeyFreeBikeStatusConverter')
25+
logger = logging.getLogger(__name__)
1426

1527

1628
class DonkeyFreeBikeStatusConverter(BaseConverter):

‎app/utils/__init__.py

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
"""
2+
MobiData BW Proxy
3+
Copyright (c) 2023, binary butterfly GmbH
4+
5+
Licensed under the EUPL, Version 1.2 or – as soon they will be approved by
6+
the European Commission - subsequent versions of the EUPL (the "Licence");
7+
You may not use this work except in compliance with the Licence.
8+
You may obtain a copy of the Licence at:
9+
10+
https://joinup.ec.europa.eu/software/page/eupl
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the Licence is distributed on an "AS IS" basis,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the Licence for the specific language governing permissions and
16+
limitations under the Licence.
17+
"""
18+
19+
from .context_helper import Context, ContextHelper
20+
from .gbfs_util import update_stations_availability_status

‎app/utils/context_helper.py

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
"""
2+
MobiData BW Proxy
3+
Copyright (c) 2025, binary butterfly GmbH
4+
5+
Licensed under the EUPL, Version 1.2 or – as soon they will be approved by
6+
the European Commission - subsequent versions of the EUPL (the "Licence");
7+
You may not use this work except in compliance with the Licence.
8+
You may obtain a copy of the Licence at:
9+
10+
https://joinup.ec.europa.eu/software/page/eupl
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the Licence is distributed on an "AS IS" basis,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the Licence for the specific language governing permissions and
16+
limitations under the Licence.
17+
"""
18+
19+
import secrets
20+
from dataclasses import dataclass, field
21+
22+
23+
@dataclass
24+
class Context:
25+
trace_id: str
26+
span_id: str
27+
attributes: dict[str, str | int | float] = field(default_factory=dict)
28+
29+
30+
class InitializationRequiredException(Exception): ...
31+
32+
33+
class ContextHelper:
34+
_current_context: Context | None
35+
36+
def initialize_context(self, trace_id: str | None = None, span_id: str | None = None):
37+
if trace_id is None:
38+
trace_id = secrets.token_hex(16)
39+
if span_id is None:
40+
span_id = secrets.token_hex(8)
41+
self._current_context = Context(trace_id=trace_id, span_id=span_id)
42+
43+
def set_attribute(self, key: str, value: str | int | float):
44+
if self._current_context is None:
45+
raise InitializationRequiredException()
46+
47+
self._current_context.attributes[key] = value
48+
49+
def get_current_attributes(self) -> dict[str, str | int | float]:
50+
if self._current_context is None:
51+
raise InitializationRequiredException()
52+
53+
return self._current_context.attributes
54+
55+
def get_current_trace_id(self) -> str:
56+
if self._current_context is None:
57+
raise InitializationRequiredException()
58+
59+
return self._current_context.trace_id
60+
61+
def get_current_span_id(self) -> str:
62+
if self._current_context is None:
63+
raise InitializationRequiredException()
64+
65+
return self._current_context.span_id
66+
67+
68+
# The ContextHelper is initialized globally for logging system access
69+
context_helper = ContextHelper()

0 commit comments

Comments
 (0)