Used storage amount

b2sharecollector.ini

@@ -0,0 +1,30 @@
+#
+# template of a configuration file for EUDAT's b2sharecollector
+#
+
+# section containing the logging options
+[Logging]
+log_file=eudatacct.log
+
+# section containing the properties to access the accounting server
+# to get statistical data and report them
+[Report]
+# base URL of the accounting server to be used
+base_url=https://accounting.eudat.eu
+# domain: either eudat or test or demo
+domain=eudat
+# uid of the corresponding registered storage resource on DPMT 
+# (same as storage_space_uuid on RCT)
+account=<insert uid here>
+# username of the provider on the accouniting server
+# owning the account specified above
+# contact dp-admin@mpcdf.mpg.de if you need one
+user=<username of provider>
+# if you have an access token from RCT already reuse that here
+password=<password or access token>
+service_uuid=<unsuported at the moment>
+
+# section contains database settings
+[B2SHARE]
+url=https://b2share.eudat.eu
+community=

setup.py

@@ -10,7 +10,7 @@
 from setuptools import setup, find_packages
 import os, sys
 
-version = '1.0.2.dev0'
+version = '1.0.2.dev0-b2share'
 
 this_directory = os.path.abspath(os.path.dirname(__file__))
 
@@ -56,6 +56,7 @@ def read(*names):
           'console_scripts': [
               'addRecord=eudat.accounting.client.__main__:main',
               'iRODScollector=eudat.accounting.client.iRODScollector:main',
+              'B2SHAREcollector=eudat.accounting.b2share.b2share_collector:main'
           ]
           },
       tests_require=dev_require,

src/eudat/accounting/b2share/__init__.py

@@ -0,0 +1,7 @@
+import pkg_resources
+
+try:
+    __version__ = pkg_resources.get_distribution(u'eudat.accounting.b2share').version
+except:
+    # LOG.warning("Could not get the package version from pkg_resources")
+    __version__ = 'unknown'

src/eudat/accounting/b2share/b2share_accounting.py

@@ -0,0 +1,48 @@
+# Copyright (c) 2018 CSC - IT Center for Science Ltd.
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import requests
+
+
+class B2SHAREAccounting(object):
+
+    def __init__(self, conf, logger):
+        self.logger = logger
+        self.url = conf.b2share_url
+        self.community = conf.b2share_community
+
+    def report(self, args):
+
+        url = self.url + "/api/records/?q=community:" + self.community
+        try:
+            r = requests.get(url, verify=True)
+        except requests.exceptions.RequestException as e:
+            self.logger.error('get community records request failed:' + str(e))
+            return 0, 0
+        if r.status_code != requests.codes.ok:
+            self.logger.warn('get community records status code:' + r.status_code)
+
+        total_amount = 0
+        for record in r.json()['hits']['hits']:
+            if 'files' in record:
+                for record_file in record['files']:
+                    total_amount += record_file['size']
+
+        return (r.json()['hits']['total'], total_amount) if (r.status_code == requests.codes.ok) else (0, 0)

src/eudat/accounting/b2share/b2share_collector.py

@@ -0,0 +1,197 @@
+# -*- coding: utf-8 -*-
+"""
+===============================
+eudat.accounting.b2share_collector
+===============================
+"""
+
+import json
+import argparse
+import logging
+import logging.handlers
+import sys
+
+try:
+    from ConfigParser import SafeConfigParser
+except ImportError:
+    # Python 3
+    from configparser import SafeConfigParser
+
+from eudat.accounting.client import __version__, LOG, utils
+from eudat.accounting.client.__main__ import Application as ApplicationBase
+
+from eudat.accounting.b2share.b2share_accounting import B2SHAREAccounting
+
+
+################################################################################
+# Configuration Class #
+################################################################################
+
+
+class Configuration(object):
+    """
+    Get configuration parameters from configuration file
+    """
+
+    def __init__(self, file, logger, fileparser):
+        self.file = file
+        self.logger = logger
+        self.fileparser = fileparser
+
+    def parseConf(self):
+        """Parse configuration file"""
+
+        print('Configuration file: %s \n' % self.file)
+
+        self.logfile = self.fileparser.get('Logging', 'log_file')
+        self.base_url = self.fileparser.get('Report', 'base_url')
+        self.domain = self.fileparser.get('Report', 'domain')
+        self.account = self.fileparser.get('Report', 'account')
+        self.user = self.fileparser.get('Report', 'user')
+        self.password = self.fileparser.get('Report', 'password')
+        self.service_uuid = self.fileparser.get('Report', 'service_uuid')
+        self.b2share_community = self.fileparser.get('B2SHARE', 'community')
+        self.b2share_url = self.fileparser.get('B2SHARE', 'url')
+
+        # create a file handler
+        handler = logging.handlers.RotatingFileHandler(self.logfile, \
+                                                       maxBytes=10000000, \
+                                                       backupCount=9)
+        handler.setLevel(logging.INFO)
+
+        # create a logging format
+        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s\
+                                            - %(message)s', "%Y-%m-%d %H:%M:%S")
+        handler.setFormatter(formatter)
+
+        # add the handlers to the logger
+        self.logger.addHandler(handler)
+
+
+################################################################################
+# EUDAT accounting Class #
+################################################################################
+
+
+class EUDATAccounting(object):
+    """
+    Class implementing the computation of statistics about resource consumption.
+    """
+
+    def __init__(self, conf, logger):
+        """
+        Initialize object with configuration parameters.
+        """
+        self.conf = conf
+        self.logger = logger
+        self.b2share_accounting = B2SHAREAccounting(conf, logger)
+
+    def _toAccountingRecord(self, stats):
+        """
+        Cast to format of an eudat accounting record
+        """
+        return {
+            'account': self.conf.account,
+            'number': stats[0],
+            'value': stats[1],
+        }
+
+    def reportStatistics(self, args):
+        """
+        Report statistical data on resource consumption to remote server
+        """
+        data = self.b2share_accounting.report(args)
+
+        acctRecords = []
+        acctRecords.append(self._toAccountingRecord(data))
+        # adding the data to the args so other command line args
+        # resp their defaults are available as well
+        args.account = acctRecords[0]['account']
+        args.value = acctRecords[0]['value']
+        args.number = acctRecords[0]['number']
+        pretty_data = json.dumps(acctRecords, indent=4)
+        self.logger.info('Data: ' + pretty_data)
+
+        credentials = utils.getCredentials(self.conf)
+        self.logger.info("Credentials found")
+        self.logger.debug("Credentials: " + str(credentials))
+        url = utils.getUrl(self.conf)
+        self.logger.info("URL to call: " + url)
+        data = utils.getData(args)
+        self.logger.info("Data as query string: " + data)
+
+        if args.test:
+            print("Test: Would send the following data: " \
+                + data)
+            return None
+
+        response = utils.call(credentials, url, data)
+
+        self.logger.info('Data sent. Status code: ' \
+                         + str(response.status_code))
+        if args.verbose:
+            print("\nData sent. Status code: " \
+                + str(response.status_code))
+            print("Key of generated accounting record: " \
+                + response.text)
+
+
+def main(argv=sys.argv):
+    logging.basicConfig(filename='.accounting.log',
+                        level=logging.INFO,
+                        format='%(asctime)s - %(name)s \
+                        - %(levelname)s - %(message)s')
+    exit_code = 1
+    try:
+        app = Application(argv)
+        app.run()
+        exit_code = 0
+    except KeyboardInterrupt:
+        exit_code = 0
+    except Exception as exc:
+        LOG.exception(exc)
+    sys.exit(exit_code)
+
+
+class Application(ApplicationBase):
+    """
+    The main Application class of the B2SHARE collector
+
+    :param argv: The command line as a list as ``sys.argv``
+    """
+
+    def __init__(self, argv):
+        ap = argparse.ArgumentParser()
+        ap.add_argument('--version', action='version', version=__version__)
+
+        ap.add_argument('-c', '--configpath', default='./b2sharecollector.cfg',
+                        help='path to configuration file. ' \
+                             'Default: "./b2sharecollector.cfg" (in the current working directory)')
+
+        utils.addCommonArguments(ap)
+
+        self.args = ap.parse_args(args=argv[1:])
+        # sneak in some default values that the utility functions expect
+        self.args.unit = 'byte'
+        self.args.service = '(default)'  # XXX TODO: should this come from the config?
+        self.args.object_type = 'registered object'
+        """Arguments of your app"""
+
+    def run(self):
+        LOG.info("B2SHAREcollector called with: " + str(self.args))
+        print("B2SHAREcollector called with: %s" % str(self.args))
+
+        fileparser = SafeConfigParser()
+        fileparser.read(self.args.configpath)
+
+        logger = logging.getLogger('StorageAccounting')
+        logger.setLevel(logging.INFO)
+
+        configuration = Configuration(self.args.configpath,
+                                      logger, fileparser)
+        configuration.parseConf()
+
+        eurep = EUDATAccounting(configuration, logger)
+        logger.info("Accounting starting ...")
+        eurep.reportStatistics(self.args)
+        logger.info("Accounting finished")