Skip to content

Commit 6a3ab22

Browse files
cmallwitzcmallwitz
committed
Added BID/ASK/BIDSIZE/ASKSIZE/EXPIRY_DATE fields for FT/YAHOO and minor ccy fix
1 parent 46e6a68 commit 6a3ab22

10 files changed

Lines changed: 244 additions & 125 deletions

‎README.md‎

Lines changed: 41 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -68,38 +68,43 @@ Depending on your system's language and default LibreOffice settings, you maybe
6868

6969
You can either specify numbers or names (lower or upper case) - not all bits are available from all sources and they are not necessarily consistent across sources either.
7070

71-
|Name|Code|YAHOO|FT|GOOGLE|YAHOO (historic)|
72-
| :--- | :--- | :---: | :---: | :---: | :---: |
73-
|PREV_CLOSE|5|Yes|Yes|Yes||
74-
|OPEN|6|Yes|Yes|Yes|Yes|
75-
|CHANGE|7|Yes|Yes|Yes||
76-
|LAST_PRICE_DATE|8|Yes|Yes|Yes||
77-
|LAST_PRICE_TIME|10|Yes|Yes|Yes||
78-
|CHANGE_IN_PERCENT|11|Yes|Yes|Yes||
79-
|LOW|14|Yes|Yes|Yes|Yes|
80-
|HIGH|16|Yes|Yes|Yes|Yes|
81-
|LAST_PRICE|21|Yes|Yes|Yes||
82-
|HIGH_52_WEEK|24|Yes|Yes|Yes||
83-
|LOW_52_WEEK|26|Yes|Yes|Yes||
84-
|MARKET_CAP|27|Yes|Yes|Yes||
85-
|VOLUME|35|Yes|Yes|No|Yes|
86-
|AVG_DAILY_VOL_3MONTH|39|Yes|Yes|No||
87-
|BETA|67|Yes|Yes|No||
88-
|EPS|68|Yes|Yes|No||
89-
|PE_RATIO|69|Yes|Yes|No||
90-
|DIV|70|Yes|Yes|No||
91-
|DIV_YIELD|71|Yes|Yes|No||
92-
|EX_DIV_DATE|72|Yes|Yes|No||
93-
|PAYOUT_RATIO|73|Yes|No|No||
94-
|CLOSE|90|No|No|No|Yes|
95-
|ADJ_CLOSE|91|No|No|No|Yes|
96-
|SECTOR|98|Yes|Yes|No||
97-
|INDUSTRY|99|Yes|Yes|No||
98-
|TICKER|101|Yes|Yes|Yes||
99-
|EXCHANGE|102|Yes|No|Yes||
100-
|CURRENCY|103|Yes|Yes|Yes||
101-
|NAME|104|Yes|Yes|Yes||
102-
|TIMEZONE|105|Yes|Yes|Yes||
71+
|Name|Code|YAHOO|FT|GOOGLE|YAHOO (historic)|Notes|
72+
| :--- | :--- | :---: | :---: | :---: | :---: | :---: |
73+
|PREV_CLOSE|5|Yes|Yes|Yes|||
74+
|OPEN|6|Yes|Yes|Yes|Yes||
75+
|CHANGE|7|Yes|Yes|Yes|||
76+
|LAST_PRICE_DATE|8|Yes|Yes|Yes| |yyyy-mm-dd|
77+
|LAST_PRICE_TIME|10|Yes|Yes|Yes|||
78+
|CHANGE_IN_PERCENT|11|Yes|Yes|Yes|||
79+
|LOW|14|Yes|Yes|Yes|Yes||
80+
|HIGH|16|Yes|Yes|Yes|Yes||
81+
|LAST_PRICE|21|Yes|Yes|Yes|||
82+
|BID|22|Yes|Yes|No|||
83+
|ASK|25|Yes|Yes|No|||
84+
|HIGH_52_WEEK|24|Yes|Yes|Yes|||
85+
|LOW_52_WEEK|26|Yes|Yes|Yes|||
86+
|MARKET_CAP|27|Yes|Yes|Yes|||
87+
|BIDSIZE|30|Yes|Yes|No|||
88+
|ASKSIZE|31|Yes|Yes|No|||
89+
|VOLUME|35|Yes|Yes|No|Yes||
90+
|AVG_DAILY_VOL_3MONTH|39|Yes|Yes|No|||
91+
|BETA|67|Yes|Yes|No|||
92+
|EPS|68|Yes|Yes|No|||
93+
|PE_RATIO|69|Yes|Yes|No|||
94+
|DIV|70|Yes|Yes|No|||
95+
|DIV_YIELD|71|Yes|Yes|No|||
96+
|EX_DIV_DATE|72|Yes|Yes|No| |yyyy-mm-dd|
97+
|PAYOUT_RATIO|73|Yes|No|No|||
98+
|EXPIRY_DATE|74|Yes (on options)|No|No| |yyyy-mm-dd|
99+
|CLOSE|90|No|No|No|Yes||
100+
|ADJ_CLOSE|91|No|No|No|Yes||
101+
|SECTOR|98|Yes|Yes|No|||
102+
|INDUSTRY|99|Yes|Yes|No|||
103+
|TICKER|101|Yes|Yes|Yes|||
104+
|EXCHANGE|102|Yes|No|Yes|||
105+
|CURRENCY|103|Yes|Yes|Yes|||
106+
|NAME|104|Yes|Yes|Yes|||
107+
|TIMEZONE|105|Yes|Yes|Yes|||
103108

104109
### Dealing with missing data:
105110

@@ -112,6 +117,10 @@ your favorite web browser (or other tools) to check if the page actually contain
112117
If it does, the file trace.log has a record of all calls to the extension with the value returned to LibreOffice.
113118
Otherwise, the file extension.log in the same location might have more details about errors or exceptions.
114119

120+
In general, web scraping can't be compared to using a stable API - the websites might have issues - from a technical or
121+
data perspective. I have found especially on the weekend it can sometimes be "flaky" and closing/reopening LibreCalc can
122+
refresh things.
123+
115124
### Build:
116125

117126
You will need the LibreOffice SDK installed.

‎examples.ods‎

2.88 KB
Binary file not shown.

‎src/baseclient.py‎

Lines changed: 64 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -142,9 +142,12 @@ def urlopen(self, url, redirect=True, data=None, headers={}, cookies=[], **kwarg
142142
self.response = self.request('POST' if data else 'GET', url, data, headers, cookies, **kwargs)
143143
text = self.response.read()
144144

145-
# Allow two redirects: used by Yahoo for some cookie based consent
145+
# Allow redirects - used by Yahoo for some cookie based consent
146+
redirect_count = 3
146147

147-
if 300 <= self.response.status < 400:
148+
while 300 <= self.response.status < 400 and redirect_count >= 0:
149+
150+
redirect_count -= 1
148151
location = self.response.getheader('Location')
149152

150153
if location and redirect:
@@ -156,21 +159,6 @@ def urlopen(self, url, redirect=True, data=None, headers={}, cookies=[], **kwarg
156159
self.response = self.request('POST' if data else 'GET', location, data, headers, cookies, **kwargs)
157160
text = self.response.read()
158161

159-
if 300 <= self.response.status < 400:
160-
location = self.response.getheader('Location')
161-
162-
if location and redirect:
163-
164-
if location.startswith('/'):
165-
scheme, _, host, path = url.split('/', 3)
166-
location = '{}//{}{}'.format(scheme, host, location)
167-
168-
self.response = self.request('POST' if data else 'GET', location, data, headers, cookies,
169-
**kwargs)
170-
text = self.response.read()
171-
else:
172-
raise RedirectException(location)
173-
174162
else:
175163
raise RedirectException(location)
176164

@@ -187,6 +175,47 @@ def urlopen(self, url, redirect=True, data=None, headers={}, cookies=[], **kwarg
187175

188176
return text
189177

178+
def get_ticker(self):
179+
180+
tick = {}
181+
182+
tick[Datacode.ADJ_CLOSE] = None
183+
tick[Datacode.ASKSIZE] = None
184+
tick[Datacode.ASK] = None
185+
tick[Datacode.AVG_DAILY_VOL_3MONTH] = None
186+
tick[Datacode.BETA] = None
187+
tick[Datacode.BIDSIZE] = None
188+
tick[Datacode.BID] = None
189+
tick[Datacode.CHANGE] = None
190+
tick[Datacode.CHANGE_IN_PERCENT] = None
191+
tick[Datacode.CURRENCY] = None
192+
tick[Datacode.DIV] = None
193+
tick[Datacode.DIV_YIELD] = None
194+
tick[Datacode.EPS] = None
195+
tick[Datacode.EXCHANGE] = None
196+
tick[Datacode.EXPIRY_DATE] = None
197+
tick[Datacode.EX_DIV_DATE] = None
198+
tick[Datacode.HIGH] = None
199+
tick[Datacode.HIGH_52_WEEK] = None
200+
tick[Datacode.INDUSTRY] = None
201+
tick[Datacode.LAST_PRICE] = None
202+
tick[Datacode.LAST_PRICE_DATE] = None
203+
tick[Datacode.LAST_PRICE_TIME] = None
204+
tick[Datacode.LOW] = None
205+
tick[Datacode.LOW_52_WEEK] = None
206+
tick[Datacode.MARKET_CAP] = None
207+
tick[Datacode.NAME] = None
208+
tick[Datacode.OPEN] = None
209+
tick[Datacode.PAYOUT_RATIO] = None
210+
tick[Datacode.PE_RATIO] = None
211+
tick[Datacode.PREV_CLOSE] = None
212+
tick[Datacode.SECTOR] = None
213+
tick[Datacode.TICKER] = None
214+
tick[Datacode.TIMEZONE] = None
215+
tick[Datacode.VOLUME] = None
216+
217+
return tick
218+
190219
def _return_value(self, data: dict, datacode: int):
191220

192221
"""
@@ -234,6 +263,18 @@ def _return_value(self, data: dict, datacode: int):
234263
elif datacode == Datacode.LAST_PRICE.value and Datacode.LAST_PRICE in data:
235264
return data[Datacode.LAST_PRICE]
236265

266+
elif datacode == Datacode.BID.value and Datacode.BID in data:
267+
return data[Datacode.BID]
268+
269+
elif datacode == Datacode.ASK.value and Datacode.ASK in data:
270+
return data[Datacode.ASK]
271+
272+
elif datacode == Datacode.BIDSIZE.value and Datacode.BIDSIZE in data:
273+
return data[Datacode.BIDSIZE]
274+
275+
elif datacode == Datacode.ASKSIZE.value and Datacode.ASKSIZE in data:
276+
return data[Datacode.ASKSIZE]
277+
237278
elif datacode == Datacode.LOW_52_WEEK.value and Datacode.LOW_52_WEEK in data:
238279
return data[Datacode.LOW_52_WEEK]
239280

@@ -273,6 +314,12 @@ def _return_value(self, data: dict, datacode: int):
273314
elif datacode == Datacode.PAYOUT_RATIO.value and Datacode.PAYOUT_RATIO in data:
274315
return data[Datacode.PAYOUT_RATIO]
275316

317+
elif datacode == Datacode.EXPIRY_DATE.value and Datacode.EXPIRY_DATE in data:
318+
if data[Datacode.EXPIRY_DATE]:
319+
return data[Datacode.EXPIRY_DATE].isoformat()
320+
else:
321+
return data[Datacode.EXPIRY_DATE]
322+
276323
elif datacode == Datacode.CLOSE.value and Datacode.CLOSE in data:
277324
return data[Datacode.CLOSE]
278325

‎src/datacode.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,11 @@ class Datacode(Enum):
2424

2525
LAST_PRICE = 21
2626

27+
BID = 22
28+
ASK = 25
29+
BIDSIZE = 30
30+
ASKSIZE = 31
31+
2732
HIGH_52_WEEK = 24
2833
LOW_52_WEEK = 26
2934
MARKET_CAP = 27
@@ -38,6 +43,7 @@ class Datacode(Enum):
3843
DIV_YIELD = 71
3944
EX_DIV_DATE = 72
4045
PAYOUT_RATIO = 73
46+
EXPIRY_DATE = 74
4147

4248
CLOSE = 90
4349
ADJ_CLOSE = 91

‎src/financials_ft.py‎

Lines changed: 20 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,9 @@ def __init__(self, ctx):
5252
def getRealtime(self, ticker: str, datacode: int):
5353

5454
"""
55-
Retrieve data for ticker from Alpha Vantage and cache it for further lookups
55+
Retrieve data for ticker from Financial Times and cache it for further lookups
5656
57-
:param ticker: the ticker symbol e.g. VOD.LON
57+
:param ticker: the ticker symbol e.g. VOD:LSE
5858
:param datacode: the requested datacode
5959
:return:
6060
"""
@@ -71,7 +71,7 @@ def getRealtime(self, ticker: str, datacode: int):
7171
del self.realtime[ticker]
7272

7373
if ticker not in self.realtime:
74-
self.realtime[ticker] = {}
74+
self.realtime[ticker] = self.get_ticker()
7575

7676
tick = self.realtime[ticker]
7777

@@ -88,40 +88,11 @@ def getRealtime(self, ticker: str, datacode: int):
8888
try:
8989
with open(os.path.join(self.basedir, f'ft-{ticker}.html'), "w", encoding="utf-8") as text_file:
9090
print(f"<!-- '{self.last_url}' -->\r\n\r\n{text}", file=text_file)
91-
except BaseException as e:
92-
logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
91+
except BaseException:
92+
logger.exception("BaseException ticker=%s datacode=%s %s", ticker, datacode)
9393

9494
tick[Datacode.TIMESTAMP] = time.time()
9595

96-
tick[Datacode.NAME] = None
97-
tick[Datacode.TICKER] = None
98-
tick[Datacode.CURRENCY] = None
99-
tick[Datacode.LAST_PRICE] = None
100-
tick[Datacode.CHANGE] = None
101-
tick[Datacode.CHANGE_IN_PERCENT] = None
102-
tick[Datacode.VOLUME] = None
103-
tick[Datacode.LOW_52_WEEK] = None
104-
tick[Datacode.HIGH_52_WEEK] = None
105-
tick[Datacode.LAST_PRICE_DATE] = None
106-
tick[Datacode.LAST_PRICE_TIME] = None
107-
tick[Datacode.TIMEZONE] = None
108-
109-
tick[Datacode.OPEN] = None
110-
tick[Datacode.HIGH] = None
111-
tick[Datacode.LOW] = None
112-
tick[Datacode.PREV_CLOSE] = None
113-
tick[Datacode.MARKET_CAP] = None
114-
115-
tick[Datacode.EXCHANGE] = None
116-
tick[Datacode.AVG_DAILY_VOL_3MONTH] = None
117-
tick[Datacode.BETA] = None
118-
tick[Datacode.EPS] = None
119-
tick[Datacode.PE_RATIO] = None
120-
tick[Datacode.DIV] = None
121-
tick[Datacode.DIV_YIELD] = None
122-
tick[Datacode.EX_DIV_DATE] = None
123-
tick[Datacode.PAYOUT_RATIO] = None
124-
12596
try:
12697
r = '<h1 class="mod-tearsheet-overview__header__name mod-tearsheet-overview__header__name--large">(.*?)</h1>'
12798
match = re.compile(r, flags=re.DOTALL).search(text)
@@ -148,7 +119,7 @@ def getRealtime(self, ticker: str, datacode: int):
148119
tick[Datacode.INDUSTRY] = self.save_wrapper(
149120
lambda: html.unescape(match.group(2)).strip())
150121

151-
r = r'<span [^>]*>Price \(([A-Z]+|--)\)</span><span [^>]*>([0-9,\.]+)</span>'
122+
r = r'<span [^>]*>Price \(([A-Za-z]{3}|--)\)</span><span [^>]*>([0-9,\.]+)</span>'
152123
match = re.compile(r, flags=re.DOTALL).search(text, start)
153124
if match:
154125
start = match.span(0)[1]
@@ -205,7 +176,7 @@ def getRealtime(self, ticker: str, datacode: int):
205176
if len(time_bits) >= 4:
206177
tick[Datacode.TIMEZONE] = time_bits[-1]
207178

208-
except BaseException as e:
179+
except BaseException:
209180
pass
210181

211182
# second attempt at 52 week range
@@ -247,6 +218,18 @@ def getRealtime(self, ticker: str, datacode: int):
247218
tick[Datacode.LOW] = self.save_wrapper(
248219
lambda: float(html.unescape(match.group(1)).replace(',', '').strip()))
249220

221+
r = r'<th>\s*Bid\s*</th><td>([0-9,\.]+)</td>'
222+
match = re.compile(r, flags=re.DOTALL).search(text, start)
223+
if match:
224+
tick[Datacode.BID] = self.save_wrapper(
225+
lambda: float(html.unescape(match.group(1)).replace(',', '').strip()))
226+
227+
r = r'<th>\s*Offer\s*</th><td>([0-9,\.]+)</td>'
228+
match = re.compile(r, flags=re.DOTALL).search(text, start)
229+
if match:
230+
tick[Datacode.ASK] = self.save_wrapper(
231+
lambda: float(html.unescape(match.group(1)).replace(',', '').strip()))
232+
250233
r = r'<th>\s*Previous close\s*</th><td>\s*([0-9,\.]+)\s*</td>'
251234
match = re.compile(r, flags=re.DOTALL).search(text, start)
252235
if match:
@@ -300,7 +283,7 @@ def getRealtime(self, ticker: str, datacode: int):
300283
value = html.unescape(match.group(1)).strip()
301284
dt = dateutil.parser.parse(value, yearfirst=True, dayfirst=False, tzinfos=whois_timezone_info)
302285
tick[Datacode.EX_DIV_DATE] = dt.date()
303-
except BaseException as e:
286+
except BaseException:
304287
pass
305288

306289
except BaseException as e:

‎src/financials_google.py‎

Lines changed: 1 addition & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -102,34 +102,12 @@ def getRealtime(self, ticker: str, datacode: int):
102102
logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
103103

104104
if ticker not in self.realtime:
105-
self.realtime[ticker] = {}
105+
self.realtime[ticker] = self.get_ticker()
106106

107107
tick = self.realtime[ticker]
108108

109109
tick[Datacode.TIMESTAMP] = time.time()
110110

111-
tick[Datacode.NAME] = None
112-
tick[Datacode.TICKER] = None
113-
tick[Datacode.CURRENCY] = None
114-
tick[Datacode.LAST_PRICE] = None
115-
tick[Datacode.CHANGE] = None
116-
tick[Datacode.CHANGE_IN_PERCENT] = None
117-
tick[Datacode.VOLUME] = None
118-
tick[Datacode.LOW_52_WEEK] = None
119-
tick[Datacode.HIGH_52_WEEK] = None
120-
tick[Datacode.LAST_PRICE_DATE] = None
121-
tick[Datacode.LAST_PRICE_TIME] = None
122-
tick[Datacode.TIMEZONE] = None
123-
124-
tick[Datacode.OPEN] = None
125-
tick[Datacode.HIGH] = None
126-
tick[Datacode.LOW] = None
127-
tick[Datacode.PREV_CLOSE] = None
128-
tick[Datacode.MARKET_CAP] = None
129-
130-
tick[Datacode.EXCHANGE] = None
131-
tick[Datacode.AVG_DAILY_VOL_3MONTH] = None
132-
133111
try:
134112
r = '<span[^>]+role="heading"[^>]+>(.*?)</span>'
135113
pattern = re.compile(r)

0 commit comments

Comments
 (0)