Move reading of new resource files to reverence

Author: DarkFenX

miner/__init__.py

@@ -21,8 +21,8 @@
 from .bulkdata import BulkdataMiner
 from .cached_calls import CachedCallsMiner
 from .metadata import MetadataMiner
+from .rescache import ResourcePickleMiner
 from .sqlite import SqliteMiner
-from .stuff import StuffedPickleMiner
 from .traits import TraitMiner
 from .abstract_miner import ContainerNameError
 
@@ -31,7 +31,7 @@
     'BulkdataMiner',
     'CachedCallsMiner',
     'MetadataMiner',
+    'ResourcePickleMiner',
     'SqliteMiner',
-    'StuffedPickleMiner',
     'TraitMiner'
 )

miner/stuff/__init__.py miner/rescache/__init__.pyminer/stuff/__init__.py renamed to miner/rescache/__init__.py

@@ -18,4 +18,4 @@
 #===============================================================================
 
 
-from .stuffed_pickle import StuffedPickleMiner
+from .respickle import ResourcePickleMiner

miner/stuff/unstuff.py miner/rescache/resbrowser.pyminer/stuff/unstuff.py renamed to miner/rescache/resbrowser.py

@@ -21,26 +21,27 @@
 from itertools import chain
 
 
-class Unstuffer(object):
+class ResourceBrowser(object):
     """
-    Class, responsible for browsing virtual filesystem of
-    .stuff files and read data from there.
+    Class, responsible for browsing/retrieval of resources.
     """
 
     def __init__(self, rvr):
-        self._efs = rvr.rot.efs
+        self._rvr = rvr
 
     def get_filelist(self):
         """
-        Aggregate filepaths from all .stuff files and return
+        Aggregate filepaths from all resource files and return
         them in the form of single list.
         """
-        resfilepaths = chain(*(stuff.files for stuff in self._efs.stuff))
+        resfilepaths = chain(
+            self._rvr.rot.res._nameMap.keys(),
+            *(stuff.files for stuff in self._rvr.rot.efs.stuff)
+        )
         return sorted(resfilepaths)
 
     def get_file(self, resfilepath):
         """
-        Return file contents for requested resource located
-        on .stuff filesystem.
+        Return file contents for requested resource.
         """
-        return self._efs.open(resfilepath).read()
+        return self._rvr.readstuff(resfilepath)

miner/stuff/stuffed_pickle.py miner/rescache/respickle.pyminer/stuff/stuffed_pickle.py renamed to miner/rescache/respickle.py

@@ -22,17 +22,17 @@
 
 from miner.abstract_miner import AbstractMiner
 from util import CachedProperty
-from .unstuff import Unstuffer
+from .resbrowser import ResourceBrowser
 
 
-class StuffedPickleMiner(AbstractMiner):
+class ResourcePickleMiner(AbstractMiner):
     """
-    Class, which attempts to get data from stuffed
+    Class, which attempts to get data from resource
     pickles (this is not guaranteed to succeed).
     """
 
     def __init__(self, rvr):
-        self._unstuffer = Unstuffer(rvr)
+        self._resbrowser = ResourceBrowser(rvr)
 
     def contname_iter(self):
         for resolved_name in sorted(self._resolved_source_map):
@@ -44,7 +44,7 @@ def get_data(self, resolved_name, **kwargs):
         except KeyError:
             self._container_not_found(resolved_name)
         else:
-            resfiledata = self._unstuffer.get_file(resfilepath)
+            resfiledata = self._resbrowser.get_file(resfilepath)
             data = pickle.loads(resfiledata)
             return data
 
@@ -58,7 +58,7 @@ def _resolved_source_map(self):
         pickle_ext = '.pickle'
         # Format: {safe path: [source paths]}
         safe_source_map = {}
-        for source_path in self._unstuffer.get_filelist():
+        for source_path in self._resbrowser.get_filelist():
             # We also strip .pickle extension from names
             if not source_path.endswith(pickle_ext):
                 continue

run.py

@@ -48,6 +48,7 @@
 
     parser = argparse.ArgumentParser(description='This script pulls data out of EVE client and writes it in JSON format')
     parser.add_argument('-e', '--eve', help='path to eve folder', required=True)
+    parser.add_argument('-r', '--res', help='path to eve shared cache folder', required=True)
     parser.add_argument('-c', '--cache', help='path to eve cache folder', required=True)
     parser.add_argument('-s', '--server', default='tranquility', help='server which was specified in EVE shortcut, defaults to tranquility')
     languages = ('de', 'en-us', 'es', 'fr', 'it', 'ja', 'ru', 'zh', 'multi')
@@ -58,22 +59,23 @@
 
     # Expand home directory
     path_eve = os.path.expanduser(args.eve)
+    path_res = os.path.expanduser(args.res)
     path_cache = os.path.expanduser(args.cache)
     path_json = os.path.expanduser(args.json)
 
     rvr_language = args.translate if args.translate != 'multi' else 'en-us'
-    rvr = reverence.blue.EVE(path_eve, cachepath=path_cache, server=args.server, languageID=rvr_language)
+    rvr = reverence.blue.EVE(path_eve, respath=path_res, cachepath=path_cache, server=args.server, languageID=rvr_language)
 
-    spickle_miner = StuffedPickleMiner(rvr)
-    trans = Translator(spickle_miner)
+    pickle_miner = ResourcePickleMiner(rvr)
+    trans = Translator(pickle_miner)
     bulkdata_miner = BulkdataMiner(rvr, trans)
     miners = (
         MetadataMiner(path_eve),
         bulkdata_miner,
         TraitMiner(bulkdata_miner, trans),
         SqliteMiner(path_eve, trans),
         CachedCallsMiner(rvr, trans),
-        spickle_miner
+        pickle_miner
     )
 
     writers = (

translator.py

@@ -29,8 +29,8 @@ class Translator(object):
     Class responsible for text localization.
     """
 
-    def __init__(self, spickle_miner):
-        self._spminer = spickle_miner
+    def __init__(self, pickle_miner):
+        self._pickle_miner = pickle_miner
         # Format: {language code: {message ID: message text}}
         self._loaded_langs = {}
         # Container for data we fetch from shared language data
@@ -237,7 +237,7 @@ def _print_current_stats(self, stats):
     # Related to loading language data
 
     def _load_pickle(self, name):
-        return self._spminer.get_data(name)
+        return self._pickle_miner.get_data(name)
 
     def _load_lang_data(self, language):
         """
@@ -246,8 +246,8 @@ def _load_lang_data(self, language):
         """
         msg_map_phb = {}
         try:
-            lang_data_old = self._load_pickle('res/localization/localization_{}'.format(language))
-            lang_data_fsd = self._load_pickle('res/localizationfsd/localization_fsd_{}'.format(language))
+            lang_data_old = self._load_pickle('res:/localization/localization_{}'.format(language))
+            lang_data_fsd = self._load_pickle('res:/localizationfsd/localization_fsd_{}'.format(language))
         except ContainerNameError:
             msg = u'data for language "{}" cannot be loaded'.format(language)
             raise LanguageNotAvailable(msg)
@@ -345,8 +345,8 @@ def _load_shared_data(self):
         """
         languages = set()
         lbl_map_phb = {}
-        main_old = self._load_pickle('res/localization/localization_main')
-        main_fsd = self._load_pickle('res/localizationfsd/localization_fsd_main')
+        main_old = self._load_pickle('res:/localization/localization_main')
+        main_fsd = self._load_pickle('res:/localizationfsd/localization_fsd_main')
         # Load list of languages
         languages.update(main_old['languages'].keys())
         languages.update(main_fsd['languages'])