From 75edc09da177b5eacdf9d8eb935f0187e593c639 Mon Sep 17 00:00:00 2001 From: "jani.heikkinen" Date: Thu, 22 Feb 2018 17:34:09 +0200 Subject: [PATCH 01/14] added b2share --- src/eudat/accounting/b2share/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 src/eudat/accounting/b2share/__init__.py diff --git a/src/eudat/accounting/b2share/__init__.py b/src/eudat/accounting/b2share/__init__.py new file mode 100644 index 0000000..a227e9c --- /dev/null +++ b/src/eudat/accounting/b2share/__init__.py @@ -0,0 +1,7 @@ +import pkg_resources + +try: + __version__ = pkg_resources.get_distribution(u'eudat.accounting.b2share').version +except: + #LOG.warning("Could not get the package version from pkg_resources") + __version__ = 'unknown' \ No newline at end of file From 7f73bd81c85afad5a7c44355f1830614a0af9d3e Mon Sep 17 00:00:00 2001 From: "jani.heikkinen" Date: Thu, 22 Feb 2018 17:45:46 +0200 Subject: [PATCH 02/14] changed version, added B2SHAREcollector entrypoint --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 06dd2fc..0b921ba 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages import os, sys -version = '1.0.2.dev0' +version = '1.0.2.dev0-b2share' this_directory = os.path.abspath(os.path.dirname(__file__)) @@ -56,6 +56,7 @@ def read(*names): 'console_scripts': [ 'addRecord=eudat.accounting.client.__main__:main', 'iRODScollector=eudat.accounting.client.iRODScollector:main', + 'B2SHAREcollector=eudat.accounting.b2share.b2share_collector:main' ] }, tests_require=dev_require, From b06575bb47946be045e35847d722961d704bc3cd Mon Sep 17 00:00:00 2001 From: "jani.heikkinen" Date: Thu, 22 Feb 2018 17:57:22 +0200 Subject: [PATCH 03/14] eudat.accounting.b2share_collector --- .../accounting/b2share/b2share_collector.py | 199 ++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 src/eudat/accounting/b2share/b2share_collector.py diff --git a/src/eudat/accounting/b2share/b2share_collector.py b/src/eudat/accounting/b2share/b2share_collector.py new file mode 100644 index 0000000..4c1ad9c --- /dev/null +++ b/src/eudat/accounting/b2share/b2share_collector.py @@ -0,0 +1,199 @@ +# -*- coding: utf-8 -*- +""" +=============================== +eudat.accounting.b2share_collector +=============================== +""" + +import json +import argparse +import logging +import logging.handlers +import sys + +try: + from ConfigParser import SafeConfigParser +except ImportError: + # Python 3 + from configparser import SafeConfigParser + +from eudat.accounting.client import __version__, LOG, utils +from eudat.accounting.client.__main__ import Application as ApplicationBase + +from eudat.accounting.b2share.b2share_accounting import B2SHAREAccounting + + +################################################################################ +# Configuration Class # +################################################################################ + + +class Configuration(object): + """ + Get configuration parameters from configuration file + """ + + def __init__(self, file, logger, fileparser): + self.file = file + self.logger = logger + self.fileparser = fileparser + + def parseConf(self): + """Parse configuration file""" + + print('Configuration file: %s \n' % self.file) + + self.logfile = self.fileparser.get('Logging', 'log_file') + self.base_url = self.fileparser.get('Report', 'base_url') + self.domain = self.fileparser.get('Report', 'domain') + self.account = self.fileparser.get('Report', 'account') + self.user = self.fileparser.get('Report', 'user') + self.password = self.fileparser.get('Report', 'password') + self.service_uuid = self.fileparser.get('Report', 'service_uuid') + self.db_user = self.fileparser.get('Database', 'user') + self.db_host = self.fileparser.get('Database', 'host') + self.db_name == self.fileparser.get('Database', 'name') + self.db_password == self.fileparser.get('Database', 'password') + + # create a file handler + handler = logging.handlers.RotatingFileHandler(self.logfile, \ + maxBytes=10000000, \ + backupCount=9) + handler.setLevel(logging.INFO) + + # create a logging format + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s\ + - %(message)s', "%Y-%m-%d %H:%M:%S") + handler.setFormatter(formatter) + + # add the handlers to the logger + self.logger.addHandler(handler) + + +################################################################################ +# EUDAT accounting Class # +################################################################################ + + +class EUDATAccounting(object): + """ + Class implementing the computation of statistics about resource consumption. + """ + + def __init__(self, conf, logger): + """ + Initialize object with configuration parameters. + """ + self.conf = conf + self.logger = logger + self.b2share_accounting = B2SHAREAccounting(conf) + + def _toAccountingRecord(self, stats): + """ + Cast to format of an eudat accounting record + """ + return { + 'account': self.conf.account, + 'number': stats[0], + 'value': stats[1], + } + + def reportStatistics(self, args): + """ + Report statistical data on resource consumption to remote server + """ + data = self.b2share_accounting.report(args) + + acctRecords = [] + acctRecords.append(self._toAccountingRecord(data)) + # adding the data to the args so other command line args + # resp their defaults are available as well + args.account = acctRecords[0]['account'] + args.value = acctRecords[0]['value'] + args.number = acctRecords[0]['number'] + pretty_data = json.dumps(acctRecords, indent=4) + self.logger.info('Data: ' + pretty_data) + + credentials = utils.getCredentials(self.conf) + self.logger.info("Credentials found") + self.logger.debug("Credentials: " + str(credentials)) + url = utils.getUrl(self.conf) + self.logger.info("URL to call: " + url) + data = utils.getData(args) + self.logger.info("Data as query string: " + data) + + if args.test: + print("Test: Would send the following data: " \ + + data) + return None + + response = utils.call(credentials, url, data) + + self.logger.info('Data sent. Status code: ' \ + + str(response.status_code)) + if args.verbose: + print("\nData sent. Status code: " \ + + str(response.status_code)) + print("Key of generated accounting record: " \ + + response.text) + + +def main(argv=sys.argv): + logging.basicConfig(filename='.accounting.log', + level=logging.INFO, + format='%(asctime)s - %(name)s \ + - %(levelname)s - %(message)s') + exit_code = 1 + try: + app = Application(argv) + app.run() + exit_code = 0 + except KeyboardInterrupt: + exit_code = 0 + except Exception as exc: + LOG.exception(exc) + sys.exit(exit_code) + + +class Application(ApplicationBase): + """ + The main Application class of the B2SHARE collector + + :param argv: The command line as a list as ``sys.argv`` + """ + + def __init__(self, argv): + ap = argparse.ArgumentParser() + ap.add_argument('--version', action='version', version=__version__) + + ap.add_argument('-c', '--configpath', default='./b2sharecollector.cfg', + help='path to configuration file. ' \ + 'Default: "./b2sharecollector.cfg" (in the current working directory)') + + utils.addCommonArguments(ap) + + self.args = ap.parse_args(args=argv[1:]) + # sneak in some default values that the utility functions expect + self.args.unit = 'byte' + self.args.service = '(default)' # XXX TODO: should this come from the config? + self.args.object_type = 'registered object' + """Arguments of your app""" + + def run(self): + LOG.info("B2SHAREcollector called with: " + str(self.args)) + print("B2SHAREcollector called with: %s" % str(self.args)) + + fileparser = SafeConfigParser() + fileparser.read(self.args.configpath) + + logger = logging.getLogger('StorageAccounting') + logger.setLevel(logging.INFO) + + configuration = Configuration(self.args.configpath, + logger, fileparser) + configuration.parseConf() + + eurep = EUDATAccounting(configuration, logger) + logger.info("Accounting starting ...") + eurep.reportStatistics(self.args) + logger.info("Accounting finished") From 41719e014810ba34bf98221a6cb45bd8e4b7db5c Mon Sep 17 00:00:00 2001 From: "jani.heikkinen" Date: Thu, 22 Feb 2018 18:01:03 +0200 Subject: [PATCH 04/14] added psycopg2 install requirement --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 0b921ba..4190fa2 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,7 @@ def read(*names): # 3rd party 'setuptools', 'requests', + 'psycopg2', ], entry_points={ 'console_scripts': [ From aee279c3775e03c241a78c755327cb33a81d3e3b Mon Sep 17 00:00:00 2001 From: "jani.heikkinen" Date: Thu, 22 Feb 2018 18:01:47 +0200 Subject: [PATCH 05/14] added B2SHAREAccounting --- .../accounting/b2share/b2share_accounting.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 src/eudat/accounting/b2share/b2share_accounting.py diff --git a/src/eudat/accounting/b2share/b2share_accounting.py b/src/eudat/accounting/b2share/b2share_accounting.py new file mode 100644 index 0000000..0de3913 --- /dev/null +++ b/src/eudat/accounting/b2share/b2share_accounting.py @@ -0,0 +1,64 @@ +# Copyright (c) 2018 CSC - IT Center for Science Ltd. + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import psycopg2 +import os + + +class B2SHAREAccounting(object): + + def __init__(self, conf): + self.account = conf.account + self.dbname = os.environ.get('B2SHARE_POSTGRESQL_DBNAME', conf.db_name) + self.user = os.environ.get('B2SHARE_POSTGRESQL_USER', conf.db_user) + self.password = os.environ.get('B2SHARE_POSTGRESQL_PASSWORD', conf.db_password) + self.host = os.environ.get('B2SHARE_POSTGRESQL_HOST', conf.db_host) + self.db_connect_info = 'host={0} dbname={1} user={2} password={3}'.format(self.host, self.dbname, + self.user, self.password) + + def report(self, args): + db_conn = None + accounting = {} + + try: + db_conn = psycopg2.connect(self.db_connect_info) + + cur = db_conn.cursor() + cur.execute("select id,json from records_metadata;") + + for record_id, record in cur: + if record is None: + continue + + community = record['community'] + if community not in accounting: + accounting[community] = 1 + else: + accounting[community] += 1 + cur.close() + + except psycopg2.Error, e: + print "Error", e + + finally: + if db_conn: + db_conn.close() + + return accounting[self.account], 0 From 1c36752afaa0af3a1d72aa1bd59f815a47dffe47 Mon Sep 17 00:00:00 2001 From: "jani.heikkinen" Date: Thu, 22 Feb 2018 18:09:13 +0200 Subject: [PATCH 06/14] added template configuration --- b2sharecollector.ini | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 b2sharecollector.ini diff --git a/b2sharecollector.ini b/b2sharecollector.ini new file mode 100644 index 0000000..553cff1 --- /dev/null +++ b/b2sharecollector.ini @@ -0,0 +1,32 @@ +# +# template of a configuration file for EUDAT's b2sharecollector +# + +# section containing the logging options +[Logging] +log_file=eudatacct.log + +# section containing the properties to access the accounting server +# to get statistical data and report them +[Report] +# base URL of the accounting server to be used +base_url=https://accounting.eudat.eu +# domain: either eudat or test or demo +domain=eudat +# uid of the corresponding registered storage resource on DPMT +# (same as storage_space_uuid on RCT) +account= +# username of the provider on the accouniting server +# owning the account specified above +# contact dp-admin@mpcdf.mpg.de if you need one +user= +# if you have an access token from RCT already reuse that here +password= +service_uuid= + +# section contains database settings +[Database] +name= +user= +password= +host= \ No newline at end of file From e808dc25aae3e196884ab5acbd032be57ea0fd67 Mon Sep 17 00:00:00 2001 From: "jani.heikkinen" Date: Thu, 22 Feb 2018 18:18:37 +0200 Subject: [PATCH 07/14] pep fix --- src/eudat/accounting/b2share/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/eudat/accounting/b2share/__init__.py b/src/eudat/accounting/b2share/__init__.py index a227e9c..521240a 100644 --- a/src/eudat/accounting/b2share/__init__.py +++ b/src/eudat/accounting/b2share/__init__.py @@ -3,5 +3,5 @@ try: __version__ = pkg_resources.get_distribution(u'eudat.accounting.b2share').version except: - #LOG.warning("Could not get the package version from pkg_resources") - __version__ = 'unknown' \ No newline at end of file + # LOG.warning("Could not get the package version from pkg_resources") + __version__ = 'unknown' From 8e265286d8ff2ac54821c37bf03388832a1f6e4a Mon Sep 17 00:00:00 2001 From: "jani.heikkinen" Date: Fri, 23 Feb 2018 16:22:16 +0200 Subject: [PATCH 08/14] added dict key check --- src/eudat/accounting/b2share/b2share_accounting.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/eudat/accounting/b2share/b2share_accounting.py b/src/eudat/accounting/b2share/b2share_accounting.py index 0de3913..d32f171 100644 --- a/src/eudat/accounting/b2share/b2share_accounting.py +++ b/src/eudat/accounting/b2share/b2share_accounting.py @@ -61,4 +61,7 @@ def report(self, args): if db_conn: db_conn.close() - return accounting[self.account], 0 + if self.account in accounting: + return accounting[self.account], 0 + else: + return 0, 0 From 348076d86aab838d9397559b1562163851e1342f Mon Sep 17 00:00:00 2001 From: "jani.heikkinen" Date: Fri, 23 Feb 2018 16:22:41 +0200 Subject: [PATCH 09/14] fixed assignment --- src/eudat/accounting/b2share/b2share_collector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/eudat/accounting/b2share/b2share_collector.py b/src/eudat/accounting/b2share/b2share_collector.py index 4c1ad9c..7e6f926 100644 --- a/src/eudat/accounting/b2share/b2share_collector.py +++ b/src/eudat/accounting/b2share/b2share_collector.py @@ -52,8 +52,8 @@ def parseConf(self): self.service_uuid = self.fileparser.get('Report', 'service_uuid') self.db_user = self.fileparser.get('Database', 'user') self.db_host = self.fileparser.get('Database', 'host') - self.db_name == self.fileparser.get('Database', 'name') - self.db_password == self.fileparser.get('Database', 'password') + self.db_name = self.fileparser.get('Database', 'name') + self.db_password = self.fileparser.get('Database', 'password') # create a file handler handler = logging.handlers.RotatingFileHandler(self.logfile, \ From db51073340522a83dea7801b0e45c9c181d5dfe9 Mon Sep 17 00:00:00 2001 From: "jani.heikkinen" Date: Mon, 26 Feb 2018 16:50:18 +0200 Subject: [PATCH 10/14] change to rest api --- b2sharecollector.ini | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/b2sharecollector.ini b/b2sharecollector.ini index 553cff1..6ecd41d 100644 --- a/b2sharecollector.ini +++ b/b2sharecollector.ini @@ -25,8 +25,6 @@ password= service_uuid= # section contains database settings -[Database] -name= -user= -password= -host= \ No newline at end of file +[B2SHARE] +url=https://b2share.eudat.eu +community= \ No newline at end of file From fd00a6bccfa5534c8c393cef0428d2b06c29f57d Mon Sep 17 00:00:00 2001 From: "jani.heikkinen" Date: Mon, 26 Feb 2018 16:50:40 +0200 Subject: [PATCH 11/14] change to rest api --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 4190fa2..0b921ba 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,6 @@ def read(*names): # 3rd party 'setuptools', 'requests', - 'psycopg2', ], entry_points={ 'console_scripts': [ From 7e1c3bb6ae14248a679d1e2a06a81baeffd4cdcd Mon Sep 17 00:00:00 2001 From: "jani.heikkinen" Date: Mon, 26 Feb 2018 16:51:24 +0200 Subject: [PATCH 12/14] change to rest api --- src/eudat/accounting/b2share/b2share_collector.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/eudat/accounting/b2share/b2share_collector.py b/src/eudat/accounting/b2share/b2share_collector.py index 7e6f926..4d138a2 100644 --- a/src/eudat/accounting/b2share/b2share_collector.py +++ b/src/eudat/accounting/b2share/b2share_collector.py @@ -50,10 +50,8 @@ def parseConf(self): self.user = self.fileparser.get('Report', 'user') self.password = self.fileparser.get('Report', 'password') self.service_uuid = self.fileparser.get('Report', 'service_uuid') - self.db_user = self.fileparser.get('Database', 'user') - self.db_host = self.fileparser.get('Database', 'host') - self.db_name = self.fileparser.get('Database', 'name') - self.db_password = self.fileparser.get('Database', 'password') + self.b2share_community = self.fileparser.get('B2SHARE', 'community') + self.b2share_url = self.fileparser.get('B2SHARE', 'url') # create a file handler handler = logging.handlers.RotatingFileHandler(self.logfile, \ @@ -86,7 +84,7 @@ def __init__(self, conf, logger): """ self.conf = conf self.logger = logger - self.b2share_accounting = B2SHAREAccounting(conf) + self.b2share_accounting = B2SHAREAccounting(conf, logger) def _toAccountingRecord(self, stats): """ From 55d30a62d2b229f1f64b8ab84365dd7efbd861e1 Mon Sep 17 00:00:00 2001 From: "jani.heikkinen" Date: Mon, 26 Feb 2018 16:51:50 +0200 Subject: [PATCH 13/14] change to rest api --- .../accounting/b2share/b2share_accounting.py | 49 ++++--------------- 1 file changed, 10 insertions(+), 39 deletions(-) diff --git a/src/eudat/accounting/b2share/b2share_accounting.py b/src/eudat/accounting/b2share/b2share_accounting.py index d32f171..93f3a61 100644 --- a/src/eudat/accounting/b2share/b2share_accounting.py +++ b/src/eudat/accounting/b2share/b2share_accounting.py @@ -18,50 +18,21 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -import psycopg2 -import os +import requests class B2SHAREAccounting(object): - def __init__(self, conf): - self.account = conf.account - self.dbname = os.environ.get('B2SHARE_POSTGRESQL_DBNAME', conf.db_name) - self.user = os.environ.get('B2SHARE_POSTGRESQL_USER', conf.db_user) - self.password = os.environ.get('B2SHARE_POSTGRESQL_PASSWORD', conf.db_password) - self.host = os.environ.get('B2SHARE_POSTGRESQL_HOST', conf.db_host) - self.db_connect_info = 'host={0} dbname={1} user={2} password={3}'.format(self.host, self.dbname, - self.user, self.password) + def __init__(self, conf, logger): + self.logger = logger + self.url = conf.b2share_url + self.community = conf.b2share_community def report(self, args): - db_conn = None - accounting = {} - try: - db_conn = psycopg2.connect(self.db_connect_info) + url = self.url + "/api/records/?q=community:" + self.community + r = requests.get(url, verify=True) + if r.status_code != requests.codes.ok: + self.logger.warning('get community records status code: %d', r.status_code) - cur = db_conn.cursor() - cur.execute("select id,json from records_metadata;") - - for record_id, record in cur: - if record is None: - continue - - community = record['community'] - if community not in accounting: - accounting[community] = 1 - else: - accounting[community] += 1 - cur.close() - - except psycopg2.Error, e: - print "Error", e - - finally: - if db_conn: - db_conn.close() - - if self.account in accounting: - return accounting[self.account], 0 - else: - return 0, 0 + return (r.json()['hits']['total'], 0) if (r.status_code == requests.codes.ok) else (0, 0) From 7bf3894879e66242905927765c97971ee54ee3da Mon Sep 17 00:00:00 2001 From: "jani.heikkinen" Date: Thu, 19 Apr 2018 13:58:39 +0300 Subject: [PATCH 14/14] added used storage data amount reporting --- .../accounting/b2share/b2share_accounting.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/eudat/accounting/b2share/b2share_accounting.py b/src/eudat/accounting/b2share/b2share_accounting.py index 93f3a61..b49686f 100644 --- a/src/eudat/accounting/b2share/b2share_accounting.py +++ b/src/eudat/accounting/b2share/b2share_accounting.py @@ -31,8 +31,18 @@ def __init__(self, conf, logger): def report(self, args): url = self.url + "/api/records/?q=community:" + self.community - r = requests.get(url, verify=True) + try: + r = requests.get(url, verify=True) + except requests.exceptions.RequestException as e: + self.logger.error('get community records request failed:' + str(e)) + return 0, 0 if r.status_code != requests.codes.ok: - self.logger.warning('get community records status code: %d', r.status_code) + self.logger.warn('get community records status code:' + r.status_code) - return (r.json()['hits']['total'], 0) if (r.status_code == requests.codes.ok) else (0, 0) + total_amount = 0 + for record in r.json()['hits']['hits']: + if 'files' in record: + for record_file in record['files']: + total_amount += record_file['size'] + + return (r.json()['hits']['total'], total_amount) if (r.status_code == requests.codes.ok) else (0, 0)