Revert "Attempt #1 to adapt to new resource storage scheme"

This reverts commit 90c4d69.

Author: DarkFenX

miner/__init__.py

@@ -21,8 +21,8 @@
 from .bulkdata import BulkdataMiner
 from .cached_calls import CachedCallsMiner
 from .metadata import MetadataMiner
-from .rescache import ResourcePickleMiner
 from .sqlite import SqliteMiner
+from .stuff import StuffedPickleMiner
 from .traits import TraitMiner
 from .abstract_miner import ContainerNameError
 
@@ -31,7 +31,7 @@
     'BulkdataMiner',
     'CachedCallsMiner',
     'MetadataMiner',
-    'ResourcePickleMiner',
     'SqliteMiner',
+    'StuffedPickleMiner',
     'TraitMiner'
 )

miner/rescache/__init__.py miner/stuff/__init__.pyminer/rescache/__init__.py renamed to miner/stuff/__init__.py

@@ -18,4 +18,4 @@
 #===============================================================================
 
 
-from .respickle import ResourcePickleMiner
+from .stuffed_pickle import StuffedPickleMiner

miner/rescache/respickle.py miner/stuff/stuffed_pickle.pyminer/rescache/respickle.py renamed to miner/stuff/stuffed_pickle.py

@@ -22,17 +22,17 @@
 
 from miner.abstract_miner import AbstractMiner
 from util import CachedProperty
-from .resbrowser import ResourceBrowser
+from .unstuff import Unstuffer
 
 
-class ResourcePickleMiner(AbstractMiner):
+class StuffedPickleMiner(AbstractMiner):
     """
-    Class, which attempts to get data from resource
+    Class, which attempts to get data from stuffed
     pickles (this is not guaranteed to succeed).
     """
 
-    def __init__(self, path_eve, path_resources):
-        self._resbrowser = ResourceBrowser(path_eve, path_resources)
+    def __init__(self, rvr):
+        self._unstuffer = Unstuffer(rvr)
 
     def contname_iter(self):
         for resolved_name in sorted(self._resolved_source_map):
@@ -44,7 +44,7 @@ def get_data(self, resolved_name, **kwargs):
         except KeyError:
             self._container_not_found(resolved_name)
         else:
-            resfiledata = self._resbrowser.get_file(resfilepath)
+            resfiledata = self._unstuffer.get_file(resfilepath)
             data = pickle.loads(resfiledata)
             return data
 
@@ -58,7 +58,7 @@ def _resolved_source_map(self):
         pickle_ext = '.pickle'
         # Format: {safe path: [source paths]}
         safe_source_map = {}
-        for source_path in self._resbrowser.get_filelist():
+        for source_path in self._unstuffer.get_filelist():
             # We also strip .pickle extension from names
             if not source_path.endswith(pickle_ext):
                 continue

miner/rescache/resbrowser.py miner/stuff/unstuff.pyminer/rescache/resbrowser.py renamed to miner/stuff/unstuff.py

@@ -18,40 +18,29 @@
 #===============================================================================
 
 
-import csv
-import os.path
+from itertools import chain
 
 
-class ResourceBrowser(object):
+class Unstuffer(object):
     """
-    Class, responsible for browsing/retrieval of resources.
+    Class, responsible for browsing virtual filesystem of
+    .stuff files and read data from there.
     """
 
-    def __init__(self, path_eve, path_resources):
-        # Get resource file list
-        # Format:
-        # {embedFS file name: full path to file}
-        self._resfiles = {}
-        path_index = os.path.join(path_eve, 'resfileindex.txt')
-        with open(path_index) as csvfile:
-            index_reader = csv.reader(csvfile)
-            for row in index_reader:
-                embedfs_path = row[0]
-                resource_path = os.path.join(path_resources, 'ResFiles', row[1])
-                self._resfiles[embedfs_path] = resource_path
+    def __init__(self, rvr):
+        self._efs = rvr.rot.efs
 
     def get_filelist(self):
         """
-        Aggregate filepaths from all resource files and return
+        Aggregate filepaths from all .stuff files and return
         them in the form of single list.
         """
-        return sorted(self._resfiles)
+        resfilepaths = chain(*(stuff.files for stuff in self._efs.stuff))
+        return sorted(resfilepaths)
 
-    def get_file(self, embedfs_path):
+    def get_file(self, resfilepath):
         """
-        Return file contents for requested resource.
+        Return file contents for requested resource located
+        on .stuff filesystem.
         """
-        resource_path = self._resfiles[embedfs_path]
-        with open(resource_path) as f:
-            data = f.read()
-        return data
+        return self._efs.open(resfilepath).read()

run.py

@@ -48,7 +48,6 @@
 
     parser = argparse.ArgumentParser(description='This script pulls data out of EVE client and writes it in JSON format')
     parser.add_argument('-e', '--eve', help='path to eve folder', required=True)
-    parser.add_argument('-r', '--res', help='path to eve shared cache folder', required=True)
     parser.add_argument('-c', '--cache', help='path to eve cache folder', required=True)
     parser.add_argument('-s', '--server', default='tranquility', help='server which was specified in EVE shortcut, defaults to tranquility')
     languages = ('de', 'en-us', 'es', 'fr', 'it', 'ja', 'ru', 'zh', 'multi')
@@ -59,23 +58,22 @@
 
     # Expand home directory
     path_eve = os.path.expanduser(args.eve)
-    path_res = os.path.expanduser(args.res)
     path_cache = os.path.expanduser(args.cache)
     path_json = os.path.expanduser(args.json)
 
     rvr_language = args.translate if args.translate != 'multi' else 'en-us'
     rvr = reverence.blue.EVE(path_eve, cachepath=path_cache, server=args.server, languageID=rvr_language)
 
-    pickle_miner = ResourcePickleMiner(path_eve, path_res)
-    trans = Translator(pickle_miner)
+    spickle_miner = StuffedPickleMiner(rvr)
+    trans = Translator(spickle_miner)
     bulkdata_miner = BulkdataMiner(rvr, trans)
     miners = (
         MetadataMiner(path_eve),
         bulkdata_miner,
         TraitMiner(bulkdata_miner, trans),
         SqliteMiner(path_eve, trans),
         CachedCallsMiner(rvr, trans),
-        pickle_miner
+        spickle_miner
     )
 
     writers = (

translator.py

@@ -29,8 +29,8 @@ class Translator(object):
     Class responsible for text localization.
     """
 
-    def __init__(self, pickle_miner):
-        self._pickle_miner = pickle_miner
+    def __init__(self, spickle_miner):
+        self._spminer = spickle_miner
         # Format: {language code: {message ID: message text}}
         self._loaded_langs = {}
         # Container for data we fetch from shared language data
@@ -237,7 +237,7 @@ def _print_current_stats(self, stats):
     # Related to loading language data
 
     def _load_pickle(self, name):
-        return self._pickle_miner.get_data(name)
+        return self._spminer.get_data(name)
 
     def _load_lang_data(self, language):
         """
@@ -246,8 +246,8 @@ def _load_lang_data(self, language):
         """
         msg_map_phb = {}
         try:
-            lang_data_old = self._load_pickle('res:/localization/localization_{}'.format(language))
-            lang_data_fsd = self._load_pickle('res:/localizationfsd/localization_fsd_{}'.format(language))
+            lang_data_old = self._load_pickle('res/localization/localization_{}'.format(language))
+            lang_data_fsd = self._load_pickle('res/localizationfsd/localization_fsd_{}'.format(language))
         except ContainerNameError:
             msg = u'data for language "{}" cannot be loaded'.format(language)
             raise LanguageNotAvailable(msg)
@@ -345,8 +345,8 @@ def _load_shared_data(self):
         """
         languages = set()
         lbl_map_phb = {}
-        main_old = self._load_pickle('res:/localization/localization_main')
-        main_fsd = self._load_pickle('res:/localizationfsd/localization_fsd_main')
+        main_old = self._load_pickle('res/localization/localization_main')
+        main_fsd = self._load_pickle('res/localizationfsd/localization_fsd_main')
         # Load list of languages
         languages.update(main_old['languages'].keys())
         languages.update(main_fsd['languages'])