Skip to content

Commit 33d0e97

Browse files
Sandip117Sandip117
authored
Merge pull request #22 from FNNDSC/px-push-rewrite
Px push rewrite
2 parents d479159 + acb8105 commit 33d0e97

File tree

4 files changed

+330
-37
lines changed

4 files changed

+330
-37
lines changed

‎pypx/pfstorage.py‎

Lines changed: 281 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,14 +58,18 @@ def __init__(self, arg, *args, **kwargs):
5858
"""
5959

6060
proto = 'https' if arg['str_swiftPort'] == '443' else 'http'
61+
if arg['str_storeBaseLocation']:
62+
cont_name = arg['str_storeBaseLocation']
63+
else:
64+
cont_name = "users"
6165
self.state_create(
6266
{
6367
"swift": {
6468
"auth_url": "%s://%s:%s/auth/v1.0" % \
6569
(proto, arg['str_swiftIP'], arg['str_swiftPort']),
6670
"username": arg['str_swiftLogin'],
6771
"key": "testing",
68-
"container_name": "users",
72+
"container_name": cont_name,
6973
"auto_create_container": True,
7074
"file_storage": "swift.storage.SwiftStorage"
7175
}
@@ -783,4 +787,279 @@ def run(self, opt={}) -> dict:
783787
self.log(str_msg, comms = 'error')
784788
self.log(json.dumps(d_actionResult, indent = 4), comms = 'tx')
785789

786-
return d_actionResult
790+
return d_actionResult
791+
792+
793+
class fileStorage(PfStorage):
794+
def __init__(self, arg, *args, **kwargs):
795+
"""
796+
Core initialization and logic in the base class
797+
"""
798+
# Check if an upstream object exists, and if so
799+
# merge those args with the current namespace:
800+
if 'upstream' in arg.keys():
801+
d_argCopy = arg.copy()
802+
# "merge" these 'arg's with upstream.
803+
arg.update(arg['reportData']['args'])
804+
# Since this might overwrite some args specific to this
805+
# app, we update again to the copy.
806+
arg.update(d_argCopy)
807+
808+
PfStorage.__init__(self, arg, *args, **kwargs)
809+
810+
def objPull(self, *args, **kwargs):
811+
pass
812+
813+
def objPut_process(self, *args, **kwargs) -> dict:
814+
"""
815+
Process the 'objPut' directive.
816+
817+
DICOM handling
818+
--------------
819+
820+
A special behaviour is available for DICOM files, triggered by passing
821+
a kwarg of 'DICOMsubstr = <X>'. In this case, DICOM files (as identi-
822+
fied by containing the substring pattern within the filename) will be
823+
read for tag information used to generate the fully qualified storage
824+
path.
825+
826+
This fully qualified storage path will be substituted into the
827+
'toLocation = <someswiftpath>' by replacing the special tag
828+
'%pack' in the <someswiftpath>.
829+
830+
NOTE:
831+
* Typically a list of files all constitute the same DICOM SERIES
832+
and as such, only one of file in the list needs to be processed for
833+
packing tags.
834+
* If the 'do' 'objPut' directive contains a true value for the field
835+
'packEachDICOM', then each DICOM will be explicitly examined and
836+
packed individually.
837+
838+
"""
839+
840+
def toLocation_updateWithDICOMtags(str_DICOMfilename) -> dict:
841+
"""
842+
Read the str_DICOMfilename, determine the pack path,
843+
and update the 'toLocation' if necessary.
844+
845+
Return the original and modified 'toLocation' and status flag.
846+
"""
847+
b_pack = False
848+
d_DICOMread = self.packer.DICOMfile_read(file=str_DICOMfilename)
849+
d_path = self.packer.packPath_resolve(d_DICOMread)
850+
self.obj[str_DICOMfilename] = d_DICOMread
851+
str_origTo = d_args['toLocation']
852+
if '%pack' in d_args['toLocation']:
853+
b_pack = True
854+
d_args['toLocation'] = \
855+
d_args['toLocation'].replace('%pack', d_path['packDir'])
856+
return {
857+
'pack': b_pack,
858+
'originalLocation': str_origTo,
859+
'path': d_path,
860+
'toLocation': d_args['toLocation']
861+
}
862+
863+
def files_putSingly() -> dict:
864+
"""
865+
Handle a single file put, return and update d_ret
866+
"""
867+
nonlocal d_ret
868+
nonlocal b_singleShot
869+
d_pack: dict = {}
870+
b_singleShot = True
871+
d_ret = {
872+
'status': False,
873+
'localFileList': [],
874+
'objectFileList': []
875+
}
876+
self.obj = {}
877+
# pudb.set_trace()
878+
for f in d_fileList['l_fileFS']:
879+
d_pack = toLocation_updateWithDICOMtags(f)
880+
d_args['file'] = f
881+
d_args['remoteFile'] = d_pack['path']['imageFile']
882+
d_put = self.objPut(**d_args)
883+
d_ret[f] = d_put
884+
d_args['toLocation'] = d_pack['originalLocation']
885+
d_ret['status'] = d_put['status']
886+
if d_ret['status']:
887+
d_ret['localFileList'].append(d_put['localFileList'][0])
888+
d_ret['objectFileList'].append(d_put['objectFileList'][0])
889+
else:
890+
break
891+
return d_ret
892+
893+
d_ret: dict = {
894+
'status': False,
895+
'msg': "No 'arg' JSON directive found in request"
896+
}
897+
898+
d_msg: dict = {}
899+
d_args: dict = {}
900+
str_localPath: str = ""
901+
str_DICOMsubstr: str = ""
902+
b_singleShot: bool = False
903+
904+
for k, v in kwargs.items():
905+
if k == 'request': d_msg = v
906+
# pudb.set_trace()
907+
if 'args' in d_msg:
908+
d_args = d_msg['args']
909+
if 'localpath' in d_args:
910+
str_localPath = d_args['localpath']
911+
if 'DICOMsubstr' in d_args:
912+
d_fileList = self.filesFind(
913+
root=str_localPath,
914+
fileSubStr=d_args['DICOMsubstr']
915+
)
916+
if 'packEachDICOM' in d_args:
917+
if d_args['packEachDICOM']: files_putSingly()
918+
if len(d_fileList['l_fileFS']) and not b_singleShot:
919+
toLocation_updateWithDICOMtags(d_fileList['l_fileFS'][0])
920+
else:
921+
d_fileList = self.filesFind(
922+
root=str_localPath
923+
)
924+
if d_fileList['status'] and not b_singleShot:
925+
d_args['fileList'] = d_fileList['l_fileFS']
926+
d_ret = self.objPut(**d_args)
927+
elif not d_fileList['status']:
928+
d_ret['msg'] = 'No valid file list generated'
929+
return d_ret
930+
931+
def objPut(self, *args, **kwargs) -> dict:
932+
"""
933+
Put an object (or list of objects) into swift storage.
934+
935+
This method also "maps" tree locations in the local storage
936+
to new locations in the object storage. For example, assume
937+
a list of local locations starting with:
938+
939+
/home/user/project/data/ ...
940+
941+
and we want to pack everything in the 'data' dir to
942+
object storage, at location '/storage'. In this case, the
943+
pattern of kwargs specifying this would be:
944+
945+
fileList = ['/home/user/project/data/file1',
946+
'/home/user/project/data/dir1/file_d1',
947+
'/home/user/project/data/dir2/file_d2'],
948+
toLocation = '/storage',
949+
mapLocationOver = '/home/user/project/data'
950+
951+
will replace, for each file in <fileList>, the <mapLocationOver> with
952+
<inLocation>, resulting in a new list
953+
954+
'/storage/file1',
955+
'/storage/dir1/file_d1',
956+
'/storage/dir2/file_d2'
957+
958+
"""
959+
b_status: bool = True
960+
l_localfile: list = [] # Name on the local file system
961+
l_remotefileName: list = [] # A replacement for the remote filename
962+
l_objectfile: list = [] # Name in the object storage
963+
str_swiftLocation: str = ''
964+
str_mapLocationOver: str = ''
965+
str_localfilename: str = ''
966+
str_storagefilename: str = ''
967+
str_swiftLocation: str = ""
968+
str_remoteFile: str = ""
969+
dst: str = ""
970+
d_ret: dict = {
971+
'status': b_status,
972+
'localFileList': [],
973+
'objectFileList': [],
974+
'localpath': ''
975+
}
976+
977+
for k, v in kwargs.items():
978+
if k == 'file': l_localfile.append(v)
979+
if k == 'remoteFile': l_remotefileName.append(v)
980+
if k == 'remoteFileList': l_remotefileName = v
981+
if k == 'fileList': l_localfile = v
982+
if k == 'toLocation': str_swiftLocation = v
983+
if k == 'mapLocationOver': str_mapLocationOver = v
984+
985+
if len(str_mapLocationOver):
986+
# replace the local file path with object store path
987+
l_objectfile = [w.replace(str_mapLocationOver, str_swiftLocation) \
988+
for w in l_localfile]
989+
else:
990+
# Prepend the swiftlocation to each element in the localfile list:
991+
l_objectfile = [str_swiftLocation + '{0}'.format(i) for i in l_localfile]
992+
993+
# Check and possibly change the actual file *names* to put into swift storage
994+
# (the default is to use the same name as the local file -- however in the
995+
# case of DICOM files, the actual final file name might also change)
996+
if len(l_remotefileName):
997+
l_objectfile = [l.replace(os.path.basename(l), f) for l, f in
998+
zip(l_objectfile, l_remotefileName)]
999+
1000+
d_ret['localpath'] = os.path.dirname(l_localfile[0])
1001+
d_conn = self.state('/swift/container_name')
1002+
if d_conn:
1003+
for str_localfilename, str_storagefilename in zip(l_localfile, l_objectfile):
1004+
try:
1005+
d_ret['status'] = True and d_ret['status']
1006+
dst = Path(d_conn)/str_storagefilename
1007+
dst.parent.mkdir(parents=True, exist_ok=True)
1008+
shutil.copyfile(str_localfilename, str(dst))
1009+
except Exception as e:
1010+
d_ret['error'] = '%s' % e
1011+
d_ret['status'] = False
1012+
d_ret['localFileList'].append(str_localfilename)
1013+
d_ret['objectFileList'].append(str(dst))
1014+
return d_ret
1015+
1016+
def connect(self, *args, **kwargs):
1017+
pass
1018+
1019+
def ls(self, *args, **kwargs):
1020+
pass
1021+
1022+
def ls_process(self, *args, **kwargs):
1023+
pass
1024+
1025+
def objExists(self, *args, **kwargs):
1026+
pass
1027+
1028+
def run(self, opt={}) -> dict:
1029+
"""
1030+
Perform the storage operation
1031+
"""
1032+
d_actionResult : dict = {
1033+
'status' : False,
1034+
'msg' : ''
1035+
}
1036+
try:
1037+
# First see if the "do" directive is a CLI
1038+
# flag captured in the self.arg structure
1039+
d_do : dict = json.loads(self.arg['do'])
1040+
except:
1041+
# Else, assume that the d_do is the passed opt
1042+
d_do = opt
1043+
if 'action' in d_do:
1044+
self.log("verb: %s detected." % d_do['action'],
1045+
comms = 'status')
1046+
str_method = '%s_process' % d_do['action']
1047+
self.log("method to call: %s(request = d_msg) " % str_method,
1048+
comms = 'status')
1049+
try:
1050+
# pudb.set_trace()
1051+
method = getattr(self, str_method)
1052+
d_actionResult = method(request = d_do)
1053+
except Exception as ex:
1054+
str_msg = "Class '{}' does not implement method '{}':{}".format(
1055+
self.__class__.__name__,
1056+
str_method,
1057+
str(ex))
1058+
d_actionResult = {
1059+
'status': False,
1060+
'msg': str_msg
1061+
}
1062+
self.log(str_msg, comms = 'error')
1063+
self.log(json.dumps(d_actionResult, indent = 4), comms = 'tx')
1064+
1065+
return d_actionResult

‎pypx/push.py‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
import sys
1212
from datetime import date, datetime
1313

14-
from .pfstorage import swiftStorage
14+
from .pfstorage import swiftStorage, fileStorage
1515

1616
# PYPX modules
1717
import pypx.smdb
@@ -120,9 +120,9 @@ def parser_setup(str_desc):
120120
action = 'store_true',
121121
default = False)
122122
parser.add_argument(
123-
'--swiftBaseLocation',
123+
'--storeBaseLocation',
124124
action = 'store',
125-
dest = 'str_swiftBaseLocation',
125+
dest = 'str_storeBaseLocation',
126126
type = str,
127127
default = '',
128128
help = 'swift base location to push files')
@@ -228,8 +228,9 @@ class Push(Base):
228228
"""
229229
``px-push`` is the primary vehicle for transmitting a DICOM file
230230
to a remote location. The remote location can be either another
231-
PACS node (in which case the PACS related args are used), or
232-
swift storage (in which the swift related args are used). In the
231+
PACS node (in which case the PACS related args are used), a
232+
swift storage (in which the swift related args are used), or a
233+
file system (in which store base related args are used). In the
233234
case of swift storage, and if CUBE related args are used, then
234235
this module will also register the files that have been pushed
235236
to the CUBE instance.
@@ -244,12 +245,15 @@ def serviceKey_process(self) -> dict:
244245
d_swiftInfo : dict = {}
245246
d_swiftInfo['status'] = False
246247
if len(self.arg['swift']):
247-
d_swiftInfo = self.smdb.service_keyAccess('swift')
248+
d_swiftInfo = self.smdb.service_keyAccess('storage')
248249
if d_swiftInfo['status']:
249-
self.arg['str_swiftIP'] = d_swiftInfo['swift'][self.arg['swift']]['ip']
250-
self.arg['str_swiftPort'] = d_swiftInfo['swift'][self.arg['swift']]['port']
251-
self.arg['str_swiftLogin'] = d_swiftInfo['swift'][self.arg['swift']]['login']
252-
250+
storageType = d_swiftInfo['storage'][self.arg['swift']]['storagetype']
251+
if storageType == "swift":
252+
self.arg['str_swiftIP'] = d_swiftInfo['storage'][self.arg['swift']]['ip']
253+
self.arg['str_swiftPort'] = d_swiftInfo['storage'][self.arg['swift']]['port']
254+
self.arg['str_swiftLogin'] = d_swiftInfo['storage'][self.arg['swift']]['login']
255+
elif storageType == "fs":
256+
self.arg['str_storeBaseLocation'] = d_swiftInfo['storage'][self.arg['swift']]['storepath']
253257
return d_swiftInfo
254258

255259
def __init__(self, arg):
@@ -329,8 +333,10 @@ def path_pushToSwift(self):
329333
'mapLocationOver' : self.arg['str_xcrdir']
330334
}
331335
}
332-
333-
store = swiftStorage(self.arg)
336+
if self.arg['str_storeBaseLocation']:
337+
store = fileStorage(self.arg)
338+
else:
339+
store = swiftStorage(self.arg)
334340
d_storeDo = store.run(d_do)
335341

336342
# Record in the smdb an entry for each series
@@ -344,11 +350,10 @@ def path_pushToSwift(self):
344350
self.smdb.seriesData('push', 'timestamp', now.strftime("%Y-%m-%d, %H:%M:%S"))
345351
if len(self.arg['swift']):
346352
self.smdb.seriesData('push', 'swift',
347-
self.smdb.service_keyAccess('swift')['swift'][self.arg['swift']])
353+
self.smdb.service_keyAccess('storage')['storage'][self.arg['swift']])
348354
return d_storeDo
349355

350356
def run(self, opt={}) -> dict:
351-
352357
d_push : dict = {}
353358

354359
if self.pushToSwift_true():

0 commit comments

Comments
 (0)