Skip to content

Commit ddf944b

Browse files
committed
add privatbank statement fetching api
1 parent cf8c3fa commit ddf944b

File tree

14 files changed

+483
-110
lines changed

14 files changed

+483
-110
lines changed

.gitignore

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,4 +149,6 @@ api-src/
149149
account.json
150150
venv_old
151151
_/
152-
_maxsivkov
152+
_maxsivkov
153+
archive
154+
autocli

README.md

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -57,12 +57,14 @@ Export statements from Bank Statements (usually XLS file) into set of json files
5757
```
5858
В результате в папке, на которую указывает `accounts_folder` должен появиться каталог для данных ФОПа, чтото типа такого:
5959
![содержимое accounts_folder](doc/accounts-folder-content.png "содержимое accounts_folder")
60-
- Зайти на Приват24 для бизнеса, раздел счета и выписки - выбираем все счета - Выписка - указываем период по кварталу:
61-
__I__ : 01.01.2020 - 31.03.2020
62-
__II__ : 01.04.2020 - 31.06.2020
63-
__III__ : 01.07.2020 - 30.09.2020
64-
__IV__ : 01.10.2020 - 31.12.2020
65-
и экспорт в xls файл. Файл нужно поместить в подпапку `data_in` которая нахордится в папке ФОПа
60+
- Получить банковские выписки за нужный период
61+
* [Автоматоматически](doc/how-to-fetch-statements.md)
62+
* Зайти на Приват24 для бизнеса, раздел счета и выписки - выбираем все счета - Выписка - указываем период по кварталу:
63+
__I__ : 01.01.2020 - 31.03.2020
64+
__II__ : 01.04.2020 - 31.06.2020
65+
__III__ : 01.07.2020 - 30.09.2020
66+
__IV__ : 01.10.2020 - 31.12.2020
67+
и экспорт в xls файл. Файл нужно поместить в подпапку `data_in` которая нахордится в папке ФОПа
6668
- Для создания json файлов выполнить
6769
```
6870
python main.py process
@@ -90,4 +92,8 @@ Export statements from Bank Statements (usually XLS file) into set of json files
9092

9193
**process** - обработка входных файлов (xls или xlsx), формирование файлов с операциями в json формате для последующей передачи
9294

93-
**push** - загрузка файлов с операциями в json формате через апи в таскер
95+
**push** - загрузка файлов с операциями в json формате через апи в taxer
96+
97+
**[fetch-statements](doc/how-to-fetch-statements.md)** - получение выписок из банка
98+
99+
**[balance](doc/how-to-balance.md)** - получение баланса по счетам (из банка и из taxer)

action.py

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
from typing import List, Dict, Tuple
2+
from datadir import DataDir, UserAccounts, UserAccount
3+
from confuse import Configuration
4+
from logging import Logger
5+
from stmt_driver_xlsx import XlsxDriver
6+
from utils import list_files, get_config_int
7+
from decimal import Decimal
8+
import os, datetime, re, ujson
9+
from bank_api import PbApi
10+
11+
def action_user(d:DataDir, config:Configuration, logger:Logger, f):
12+
for userid in d.Users(): f(d, config, logger, userid)
13+
14+
def action_init(d:DataDir, config:Configuration, logger:Logger, userid:int):
15+
d.UpdateAll()
16+
17+
def action_process(d:DataDir, config:Configuration, logger:Logger, userid:int):
18+
if not os.path.exists(d.user_config_path(userid)):
19+
raise ValueError(f'config file for user {userid} not found')
20+
user_data = DataDir(config)
21+
user_data.add_config(d.user_config_path(userid))
22+
23+
acc:UserAccounts = d.UserAccounts(userid)
24+
25+
# try to convert xls to xlsx
26+
for datafile in d.UserXLSDataFiles(userid):
27+
try:
28+
import pyexcel as p
29+
30+
xlsx_file = datafile + 'x'
31+
logger.info(f'Trying to convert {datafile} to {xlsx_file}')
32+
if os.path.exists(xlsx_file):
33+
logger.info(f'IGNORED (Already exists) {xlsx_file}')
34+
else:
35+
p.save_book_as(file_name=datafile,
36+
dest_file_name=xlsx_file)
37+
logger.info(f'Converted')
38+
except ImportError as e:
39+
logger.error('Error during xls to xlsx conversion', 'Error', exc_info=e)
40+
41+
datafiles : List[str] = d.UserDataFiles(userid)
42+
if len(datafiles) == 0:
43+
logger.error('No xlsx files')
44+
import_tag : str = f'IMP-{datetime.date.today().strftime("%Y-%m-%d")}'
45+
for datafile in d.UserDataFiles(userid):
46+
#datafile_parts:str = os.path.splitext(os.path.basename(datafile))
47+
outputfn = os.path.join(user_data.user_folder_output(userid), os.path.basename(datafile))
48+
49+
stmt_data = XlsxDriver(filepath=datafile, output_file_path=outputfn, first_row=get_config_int(user_data.config, 'datafile.first_row'))
50+
stmt_data.read()
51+
52+
from execution_context import ExecutionContext
53+
54+
ec = ExecutionContext(stmt_data, acc, list_files('rules', r'.*\.yaml'), d.UserTaxNumber(userid), d.user_folder_output(userid))
55+
(total_rows, rows_processed) = ec.execute(import_tag)
56+
stmt_data.close()
57+
logger.info(f'DONE. Processed {total_rows} rows, prepared {rows_processed} statements with tag {import_tag}')
58+
59+
60+
def action_push(d:DataDir, config:Configuration, logger:Logger, userid:int):
61+
if not os.path.exists(d.user_config_path(userid)):
62+
raise ValueError(f'config file for user {userid} not found')
63+
user_data = DataDir(config)
64+
user_data.add_config(d.user_config_path(userid))
65+
output_folder = d.user_folder_output(userid)
66+
if not os.path.exists(output_folder):
67+
return
68+
# raise ValueError(f'folder does not exist {output_folder}')
69+
processed_files = 0
70+
for f in sorted(os.listdir(output_folder)):
71+
m = re.match(f'(?P<index>[0-9]+)-(?P<operation>\w+)-(?P<operation_id>\w+).json', f)
72+
if not m: continue
73+
index = m.group('index')
74+
operation = m.group('operation')
75+
operation_id = m.group('operation_id')
76+
print(f'{index} : processing {operation} / {operation_id}')
77+
with open(os.path.join(output_folder, f), encoding='utf8') as json_file:
78+
data = ujson.load(json_file)
79+
user_data.ProcessOperation(operation, userid, data)
80+
processed_files = processed_files + 1
81+
print(f'DONE. Processed {processed_files} statements')
82+
83+
def action_fetch_statements(d:DataDir, config:Configuration, logger:Logger, userid:int):
84+
if not os.path.exists(d.user_config_path(userid)):
85+
raise ValueError(f'config file for user {userid} not found')
86+
user_data = DataDir(config)
87+
user_data.add_config(d.user_config_path(userid))
88+
accounts = user_data.UserAccounts(userid)
89+
bankApi:PbApi = PbApi(user_data)
90+
logger.info(f'getting statements for period {bankApi.first_date} - {bankApi.last_date}')
91+
bankApi.fetch_statements()
92+
num_entries = bankApi.store(os.path.join(user_data.user_folder_input(userid), f'{bankApi.period_str}.xlsx'), get_config_int(user_data.config, 'datafile.first_row'))
93+
logger.info(f'{num_entries} statements stored in {bankApi.period_str}.xlsx')
94+
95+
def action_balance(d:DataDir, config:Configuration, logger:Logger, userid:int):
96+
if not os.path.exists(d.user_config_path(userid)):
97+
raise ValueError(f'config file for user {userid} not found')
98+
user_data = DataDir(config)
99+
user_data.add_config(d.user_config_path(userid))
100+
accounts = user_data.UserAccounts(userid)
101+
bankApi:PbApi = PbApi(user_data)
102+
balance = {}
103+
for b in bankApi.balance():
104+
amount = Decimal(b['balanceIn'])
105+
balance[b["acc"]] = {'bank_balance': Decimal(b['balanceIn']), 'currency': b["currency"]}
106+
107+
userAccounts = user_data.GetUserAccounts(userid)
108+
for a in user_data.GetUserAccounts(userid):
109+
_d = {'taxer_balance': Decimal(a.balance), 'currency': a.currency}
110+
num = a.num if a.num else a.title
111+
if num in balance:
112+
balance[num].update(_d)
113+
else:
114+
balance[num] = _d
115+
logger.info(f'{"Номер счета": <30} {"Банк": >15} {"Taxer": >15} {"Валюта": <3}')
116+
for acc in balance:
117+
b = balance[acc]
118+
bank_balance = b['bank_balance'] if 'bank_balance' in b else Decimal(0)
119+
taxer_balance = b['taxer_balance'] if 'taxer_balance' in b else Decimal(0)
120+
currency = b["currency"]
121+
if bank_balance.is_zero() and taxer_balance.is_zero(): continue
122+
logger.info(f'{acc: <30} {bank_balance: >15.2f} {taxer_balance: >15.2f} {currency: <3}')

bank_api.py

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
from datadir import DataDir
2+
from typing import List
3+
import requests, re, datetime, dateutil
4+
from itertools import chain
5+
from functools import reduce
6+
from operator import add
7+
8+
from dateutil.relativedelta import relativedelta
9+
from dateutil.parser import parse
10+
from decimal import Decimal
11+
12+
from openpyxl import Workbook
13+
14+
class ApiBase(object):
15+
QUARTER_FIRST_DATE = ['01-01', '04-01', '07-01', '10-01'] # MM.DD
16+
17+
def __init__(self, d:DataDir):
18+
self.quarter_first_date=[]
19+
self._period_str=None
20+
self.period = self._period(d.get_config_v('statements.period.value'), d.get_config_v('statements.period.start'), d.get_config_v('statements.period.end'))
21+
self.ignore_accounts = d.get_config_v('statements.ignore_accounts')
22+
self.accounts:List[str] = []
23+
self._statements = {}
24+
25+
def _period(self, value, start, end):
26+
first_date = None
27+
last_date = None
28+
29+
if start and end:
30+
(st_year, st_month, st_day) = self._parse_date(start)
31+
(end_year, end_month, end_day) = self._parse_date(end)
32+
if st_year:
33+
first_date = datetime.date(st_year, st_month, st_day)
34+
if end_year:
35+
last_date = datetime.date(end_year, end_month, end_day)
36+
if not first_date:
37+
first_date = dateutil.parser.parse(start, fuzzy=True)
38+
if not last_date:
39+
last_date = dateutil.parser.parse(end, fuzzy=True)
40+
self._period_str = f'{first_date} -- {last_date}'
41+
elif value:
42+
if value.lower() == 'prev':
43+
first_date, last_date = self.prev_quarter(datetime.date.today())
44+
self._period_str=f'{first_date} -- {last_date}'
45+
else:
46+
(year, quarter, _) = self._parse_date(value)
47+
self._period_str = f'{year}-{quarter.lower()}'
48+
if not quarter.isdigit():
49+
quarter = quarter.lower()
50+
if quarter == 'i': quarter = 1
51+
if quarter == 'ii': quarter = 2
52+
if quarter == 'iii': quarter = 3
53+
if quarter == 'iv': quarter = 4
54+
else: quarter = int(quarter)
55+
quarter_start = f'{year}-{ApiBase.QUARTER_FIRST_DATE[quarter-1]}'
56+
first_date = dateutil.parser.parse(quarter_start)
57+
last_date = first_date + relativedelta(months=3, days=-1)
58+
return first_date, last_date
59+
60+
@property
61+
def first_date(self): return self.period[0]
62+
63+
@property
64+
def last_date(self): return self.period[1]
65+
66+
@property
67+
def period_str(self): return self._period_str
68+
69+
def prev_quarter(self, ref):
70+
first_month_of_quarter = ((ref.month - 1) // 3) * 3 + 1
71+
last_date = ref.replace(month=first_month_of_quarter, day=1) - relativedelta(days=1)
72+
first_date = last_date - relativedelta(months=3, days=-1)
73+
return first_date, last_date
74+
75+
def _parse_date(self, q):
76+
if not q: return q
77+
exps = [r'(?P<day>\d{1,2}).(?P<month>\d{1,2}).(?P<year>\d{4})', #20.1.2020, 20-1-2020
78+
r'(?P<year>\d{4}).(?P<month>\d{1,2}).(?P<day>\d{1,2})', # 2020.1.20, 2020-1-20
79+
r'(?P<year>\d{4}).(?P<quarter>\d)', # 2020.4, 2020-4
80+
r'(?P<year>\d{4}).(?P<quarter>[iIvV]{1,2})', # 2020.iv, 2020-iv
81+
r'(?P<quarter>\d).(?P<year>\d{4})', # 4.2020, 4-2020
82+
r'(?P<quarter>[iIvV]{1,2}).(?P<year>\d{4})', # iv.2020, iv-2020
83+
]
84+
for r in exps:
85+
m = re.match(r, q)
86+
if m:
87+
values = m.groupdict()
88+
return (int(values['year']), values['quarter'], None) if 'quarter' in values else (int(values['year']), int(values['month']), int(values['day']) if "day" in values else None)
89+
return (None, None, None)
90+
91+
def ignored(self, account):
92+
for ap in self.ignore_accounts:
93+
if re.match(ap, account) is not None:
94+
return True
95+
return False
96+
97+
class PbApi(ApiBase):
98+
def __init__(self, d:DataDir):
99+
ApiBase.__init__(self, d)
100+
101+
self.pb_id:str = d.get_config_str('statements.pb_id')
102+
self.pb_token:str = d.get_config_str('statements.pb_token')
103+
104+
def fetch_statements(self):
105+
self.accounts:List[str] = []
106+
107+
params = {'startDate': self.first_date.strftime('%d-%m-%Y'), 'endDate': self.last_date.strftime('%d-%m-%Y')}
108+
headers = {'id': self.pb_id
109+
, 'token': self.pb_token
110+
, 'Content-type': 'application/json;charset=utf8'}
111+
r = requests.get("https://acp.privatbank.ua/api/proxy/transactions", params=params, headers=headers)
112+
r.raise_for_status()
113+
stmts = r.json()['StatementsResponse']['statements']
114+
self.accounts = list(chain.from_iterable(stmts))
115+
for account_item in stmts:
116+
self._statements.update(account_item.items())
117+
118+
def balance(self):
119+
120+
params = {'startDate': self.first_date.strftime('%d-%m-%Y'), 'endDate': self.last_date.strftime('%d-%m-%Y')}
121+
headers = {'id': self.pb_id
122+
, 'token': self.pb_token
123+
, 'Content-type': 'application/json;charset=utf8'}
124+
r = requests.get("https://acp.privatbank.ua/api/statements/balance/final", headers=headers)
125+
r.raise_for_status()
126+
return r.json()['balances']
127+
128+
129+
def statements(self, accountNo:str): return list(chain.from_iterable([x.values() for x in self._statements[accountNo]]))
130+
131+
def datetime_pb(self, s) -> datetime : return datetime.datetime.strptime(s, '%d.%m.%Y %H:%M:%S')
132+
133+
def store(self, wb_filename, start_row:int):
134+
all_statements = []
135+
for accountNo in self.accounts:
136+
if self.ignored(accountNo): continue
137+
all_statements.extend(self.statements(accountNo))
138+
all_statements.sort(key=lambda x: self.datetime_pb(x['DATE_TIME_DAT_OD_TIM_P']))
139+
wb = Workbook()
140+
ws = wb.active
141+
142+
ws.cell(row=2, column=1, value=f'Виписка по декількох рахунках з {self.first_date} по {self.last_date}')
143+
144+
#ws.title = f'{self.first_date}-{self.last_date}'
145+
ws.cell(row=4, column=1, value='№')
146+
ws.cell(row=4, column=2, value='Дата проводки')
147+
ws.cell(row=4, column=3, value='Час проводки')
148+
ws.cell(row=4, column=4, value='Сума')
149+
ws.cell(row=4, column=5, value='Валюта')
150+
ws.cell(row=4, column=6, value='Призначення платежу')
151+
ws.cell(row=4, column=7, value='ЄДРПОУ')
152+
ws.cell(row=4, column=8, value='Назва контрагента')
153+
ws.cell(row=4, column=9, value='Рахунок контрагента')
154+
ws.cell(row=4, column=10, value='МФО контрагента')
155+
ws.cell(row=4, column=11, value='Ваш МФО')
156+
ws.cell(row=4, column=12, value='Ваш ЄДРПОУ')
157+
ws.cell(row=4, column=13, value='Ваш рахунок')
158+
ws.cell(row=4, column=14, value='Назва вашого рахунку')
159+
ws.cell(row=4, column=15, value='Референс')
160+
161+
162+
row = start_row
163+
for s in all_statements:
164+
ws.cell(row=row, column=1, value=s['BPL_NUM_DOC'])
165+
date = self.datetime_pb(s['DATE_TIME_DAT_OD_TIM_P'])
166+
amount=Decimal(s['BPL_SUM'])
167+
ws.cell(row=row, column=2, value=date.strftime('%d.%m.%Y'))
168+
ws.cell(row=row, column=3, value=date.strftime('%H:%M:%S'))
169+
ws.cell(row=row, column=4, value=-amount if s['TRANTYPE'] == 'D' else amount)
170+
ws.cell(row=row, column=5, value=s['BPL_CCY'])
171+
ws.cell(row=row, column=6, value=s['BPL_OSND'])
172+
ws.cell(row=row, column=7, value=s['AUT_CNTR_CRF'])
173+
ws.cell(row=row, column=8, value=s['AUT_CNTR_NAM'])
174+
ws.cell(row=row, column=9, value=s['AUT_CNTR_ACC'])
175+
ws.cell(row=row, column=10, value=s['AUT_CNTR_MFO'])
176+
ws.cell(row=row, column=11, value=s['AUT_MY_MFO'])
177+
ws.cell(row=row, column=12, value=s['AUT_MY_CRF'])
178+
ws.cell(row=row, column=13, value=s['AUT_MY_ACC'])
179+
ws.cell(row=row, column=14, value=s['AUT_MY_NAM'])
180+
ws.cell(row=row, column=15, value=s['BPL_REF'])
181+
row=row+1
182+
wb.save(wb_filename)
183+
return len(all_statements)

cmdline_arguments.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,15 @@ def parser() -> argparse.ArgumentParser:
1212
default=os.getenv('BSTMT_OUTPUT_DIR', DataDir.default_output_folder))
1313
parser.add_argument("-r", "--first_row", nargs='?', help="first row",
1414
default=os.getenv('BSTMT_FIRST_ROW', DataDir.default_first_row))
15+
16+
parser.add_argument("-p", "--period", nargs='?', help="period",
17+
default=os.getenv('BSTMT_PERIOD', None), dest='statements.period.value')
18+
parser.add_argument("-s", "--period-start", nargs='?', help="period start",
19+
default=os.getenv('BSTMT_PERIOD_START', None), dest='statements.period.start')
20+
parser.add_argument("-e", "--period-end", nargs='?', help="period end",
21+
default=os.getenv('BSTMT_PERIOD_END', None), dest='statements.period.end')
22+
1523
parser.add_argument('action', help='Action: init - initialize; process - process input files, prepare json output files; push - upload json files to taxer-api, test')
24+
25+
#parser.add_argument('args', nargs=argparse.REMAINDER)
1626
return parser

config.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ def configuration() -> confuse.Configuration:
55
conf = confuse.Configuration('taxer-statements')
66
if os.path.exists('config.yaml'):
77
conf.set_file('config.yaml')
8+
#args, unknown_args = parser().parse_known_args()
89
args = parser().parse_args()
9-
conf.set_args(args)
10+
conf.set_args(args, dots=True)
1011
return conf

config.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,11 @@
1+
datafile:
2+
first_row: 6
13
accounts_folder: data
4+
input_folder: data_in
5+
output_folder: data_out
6+
statements:
7+
ignore_accounts: ['UA..{6}.000002603.*']
8+
period:
9+
value: prev
10+
start:
11+
end:

0 commit comments

Comments
 (0)