diff --git a/FormatLog.py b/FormatLog.py index 04c0f23..c685764 100644 --- a/FormatLog.py +++ b/FormatLog.py @@ -1,9 +1,9 @@ import inspect import gc from datetime import datetime -#decorator functions +#decorator functions def format_arguments(func): - """ formate many args into one string, then calls decorated function with string formatted together + """ format many args into one string, then calls decorated function with string formatted together I.E. func("I love ",johnny.fullname,"because he is",100,"years old",**kwargs) might look like func("I love Jonnathan because he is 100 years old",**kwargs) """ @@ -28,7 +28,7 @@ def format_args_and_call(self,*args,**kwargs): def get_context_wrapper(function_to_call): """ - passes in the context in which the wrapped function was called + passes in the context in which the wrapped function was called """ from functools import wraps @wraps(function_to_call) @@ -47,27 +47,27 @@ def context_call(self,*args,**kwargs): funcs.append(func) if len(funcs) > 1: break - + args.append(str(funcs[0])[10:-16] if funcs and len(funcs) == 1 else "") return function_to_call(self,*args,**kwargs) return context_call class FormatLogger(): """ - singleton, in order to make everything log to the same files in the right order with out passing in the log class + singleton, in order to make everything log to the same files in the right order with out passing in the log class """ _instance=None def __new__(cls): if cls._instance is None: cls._instance = object.__new__(cls) - FormatLogger._instance.logfile = "/dev/null" - FormatLogger._instance.failure_file = "/dev/null" + FormatLogger._instance.logfile = "/dev/null" + FormatLogger._instance.failure_file = "/dev/null" FormatLogger._instance.proccess_status = "/dev/null" - FormatLogger._instance.truncate = False - FormatLogger._instance.files = [] - FormatLogger._instance.prints = 0 - FormatLogger._instance.num_success = 0 - FormatLogger._instance.num_fail = 0 + FormatLogger._instance.truncate = False + FormatLogger._instance.files = [] + FormatLogger._instance.prints = 0 + FormatLogger._instance.num_success = 0 + FormatLogger._instance.num_fail = 0 return cls._instance def __init__(self): self.truncate = self._instance.truncate @@ -117,7 +117,7 @@ def write(self,desc,level = 1): if n == level: break write_line_to_file(file,desc) - + @format_arguments @get_context_wrapper def status(self,desc,cont): @@ -143,18 +143,18 @@ def warning(self,desc,cont,context = False): print(string) for file in [self.logfile,self.proccess_status]: write_line_to_file(file,string) - + @format_arguments def error(self,desc): if self.prints <=2: print(desc) for file in [self.proccess_status]: write_line_to_file(file,desc) - + @format_arguments @get_context_wrapper def critical(self,desc,cont):# pylint: disable=E1120 - string = "\n-- CRITIICAL FAILURE in {} --:{}\n".format(cont if cont else "Main Scope?",desc) + string = "\n-- CRITICAL FAILURE in {} --:{}\n".format(cont if cont else "Main Scope?",desc) print(string) for file in self.files: write_line_to_file(file,string) @@ -165,7 +165,7 @@ def success(self,desc): print("SUCCESS: " + desc) for file in [self.logfile,self.proccess_status]: write_line_to_file(file,"SUCCESS: " + desc) - + @format_arguments def failure(self,desc): self.num_fail += 1 @@ -175,7 +175,7 @@ def failure(self,desc): write_line_to_file(file,"FAILURE: " + desc) def close(self): - suc = "Succeded on {} out of {} total".format(self.num_success,self.num_fail+self.num_success) + suc = "Succeeded on {} out of {} total".format(self.num_success,self.num_fail+self.num_success) if self.prints <=3: print(suc) write_line_to_file(self.failure_file,"Failed {} out of {} total".format(self.num_fail,self.num_fail+self.num_success)) @@ -211,6 +211,3 @@ def get_context(): return "- Main Scope" elif len(stack) > 3: return stack[3].code_context[0] - - - diff --git a/README.md b/README.md index 88439b4..591ab26 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Requires Python >= 3.5 virtualenv -p python3 ENV source ENV/bin/activate -3. install needed python libraries +3. install needed python libraries pip install -r requirements.txt @@ -29,10 +29,10 @@ Requires Python >= 3.5 To run batch-loader: `python batch_loader.py ` - OR if instead of haveing the column `files` you have the column `fulltext_url` of the related resource + OR if instead of having the column `files` you have the column `fulltext_url` of the related resource `python batch_loader.py --url` see example.csv and url_example.csv - finally it can also be run on json files, using the same elements as the csv. + finally it can also be run on json files, using the same elements as the csv. `python batch_loader.py --json` there are more options as well such as what collections to ingest to if your rake task can handle that, whether or not to generate tiffs, and print level. diff --git a/batch_loader.py b/batch_loader.py index c6e49c9..3e34c72 100644 --- a/batch_loader.py +++ b/batch_loader.py @@ -99,7 +99,7 @@ def run_ingest_process(self): logger.failure("%s was not ingested" % (upload_id) ) self.failed.append(row) if logger.num_success == 0 and logger.num_fail >= 5: - print("Warning: Ingest Failed frist 5 in a row!") + print("Warning: Ingest Failed first 5 in a row!") logger.status('End of',upload_id,'\n') except KeyboardInterrupt as yikes_stop_error: @@ -172,7 +172,7 @@ def __init__(self,*args,**kwargs): self.repeating_field_names = None # fields that will be a list self.current = 0 # the index of the current work we going to ingest self.base_filepath = None # where the csv is stored, used for non url ingests - self.raw_download_dir = None # temperary dircectory to download work related files + self.raw_download_dir = None # temporary directory to download work related files self.field_names = None # original field names given in the csv def __iter__(self): @@ -378,7 +378,7 @@ class IngestFactory(): def create_controller(cls,args,config): if args.json:#Json ingest_controller = JsonIngestController() - else:#csv, defualt + else:#csv, default ingest_controller = CsvIngestController() ingest_controller.init(args.file,config.ingest_command,config.ingest_path,config.ingest_depositor,config.auth_enable,config.auth_user,config.auth_pass,args.worktype) diff --git a/get_file.py b/get_file.py index 5a8bc83..b3e74a1 100644 --- a/get_file.py +++ b/get_file.py @@ -26,11 +26,11 @@ def create_tiff_imagemagick(file): tiff = os.path.splitext(file)[0] + '.tiff' return_code = subprocess.run(['convert',file,tiff], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) # if return_code != 0: - # raise Exception("non zero return code for image magick convert, if you are on windows this doesnt work.\ncommand:convert {} {}".format(file,tiff)) + # raise Exception("non zero return code for image magick convert, if you are on windows this does not work.\ncommand:convert {} {}".format(file,tiff)) if os.path.exists(tiff): return tiff logger.error('Could not create TIFF') - raise Exception("image magick convert failed to produce tiff, if you are on windows this doesnt work use magick convert instead.\n\t command: convert {} {}".format(file,tiff)) + raise Exception("image magick convert failed to produce tiff, if you are on windows this does not work use magick convert instead.\n\t command: convert {} {}".format(file,tiff)) def create_dir_for(files): """ @@ -43,7 +43,7 @@ def create_dir_for(files): tmpdir = tempfile.mkdtemp(dir=parentdir) for path in files: file_name = os.path.basename(path) - os.rename(path,os.path.join(tmpdir,file_name)) # move the file into the temporay dir basically mv(source=path,dest=tmpdir) + os.rename(path,os.path.join(tmpdir,file_name)) # move the file into the temporary dir basically mv(source=path,dest=tmpdir) return tmpdir def get_file_name_from_url(url): @@ -83,7 +83,7 @@ def mv(path,new_path,args = None): return subprocess.run(['sudo','mv',path,new_path]+args, stdout=subprocess.PIPE) def download_file(url, dwnld_dir=None, auth_enable=False, auth_user=None, auth_pass=None): - """ if the given url is valid and we have access to the file attached to it. this funciton + """ if the given url is valid and we have access to the file attached to it. this function will download said file to the directory given or just put it in the current dir. args: url: the url @@ -168,7 +168,7 @@ def download_file(url, dwnld_dir=None, auth_enable=False, auth_user=None, auth_p print('done downloading %s' % (local_filename),"file size:",file_size) if file_size == 0: - logger.error("file size is 0, file must not have downlaoded correctly") + logger.error("file size is 0, file must not have downloaded correctly") raise UrlException('Failed to downlaod') return os.path.abspath(local_filename) except PermissionError as e: @@ -178,7 +178,7 @@ def download_file(url, dwnld_dir=None, auth_enable=False, auth_user=None, auth_p if grant_access(dwnld_dir).returncode == 0: print('success') return download_file(url,dwnld_dir) - logger.error("could not aquire permission to download to target dir") + logger.error("could not acquire permission to download to target dir") raise text = ''