diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..92e88b5 --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +ignore = E121,E123,E126,E226,E24,E704,W503,W504 +max-line-length = 100 +exclude = tests/* +max-complexity = 10 \ No newline at end of file diff --git a/.gitignore b/.gitignore index f69f409..af0fbcd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,14 @@ +# python ._* *.pyc -#test* -.vscode* \ No newline at end of file +*.egg-info + +# test +.vscode* + +# pip +build +dist + +# git +.git \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..27a89e6 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +global-include *.pp3 +global-include *.yaml \ No newline at end of file diff --git a/README.md b/README.md index 97a9a70..c74a880 100644 --- a/README.md +++ b/README.md @@ -97,19 +97,27 @@ On OSX, [Homebrew](https://brew.sh) can easily be used to install the needed dep Pip can be used to install the python dependencies needed: -`pip install PyYAML` - +`pip install debayer` ### Linux On Fedora Linux, most dependencies can be installed with the package manager. `dnf install python rawtherapee pip openimageio dcraw exiftool` -`pip install --user PyYAML` +`pip install debayer` For other distributions I'm assuming it would be pretty straightforward as well... +### Windows +On Windows you will have to download and install [Python](https://www.python.org/downloads). Make sure to you check the option to "Add Python to PATH". + +Install [ImageMagick](https://imagemagick.org/script/download.php#windows) and make sure the "Add application to your system path" option is checked. + +You will also have to install [RawTherapee](https://rawtherapee.com) and [ExifTool](https://exiftool.org). Make sure to edit your Windows environment variables to point to the folders holding these binaries. + +Debayer can be then installed using PyPi. +`pip install debayer` ## The MIT License Copyright 2019 Jedediah Smith diff --git a/debayer b/debayer/__init__.py old mode 100755 new mode 100644 similarity index 75% rename from debayer rename to debayer/__init__.py index 36e10b6..199f8cf --- a/debayer +++ b/debayer/__init__.py @@ -3,12 +3,17 @@ from __future__ import print_function from __future__ import division -import os, sys, argparse, re +import os +import sys +import argparse +import re import tempfile import shutil -import time, datetime +import time +import datetime import logging -import subprocess, shlex +import subprocess +import shlex import pyseq import distutils.spawn import yaml @@ -16,7 +21,7 @@ # Get config yaml file dir_path = os.path.dirname(os.path.realpath(__file__)) -CONFIG = os.path.join(dir_path, "debayer-config.yaml") +CONFIG = os.path.join(dir_path, "config.yaml") # Set up logger # %(levelname)s %(asctime)s logging.basicConfig( @@ -52,6 +57,7 @@ class Debayer(): default_format_resize: raw_formats: ''' + def __init__(self): # Create tmp directory on local disk @@ -65,7 +71,10 @@ def __init__(self): # Remove temp directory if self.tmp_dir: - shutil.rmtree(self.tmp_dir) + try: + shutil.rmtree(self.tmp_dir) + except WindowsError as error: + log.error(error) def validate(self): ''' @@ -96,45 +105,45 @@ def validate(self): if exe.startswith(os.path.sep): self.rawtherapee_cli = location else: - self.rawtherapee_cli = distutils.spawn.find_executable(location) - if self.rawtherapee_cli: - if not os.path.exists(self.rawtherapee_cli): - log.error("Error: rawtherapee-cli does not exist: \t{0}".format(self.rawtherapee_cli)) - self.rawtherapee_cli = None - return None + self.rawtherapee_cli = distutils.spawn.find_executable( + location) + if not self.rawtherapee_cli or not os.path.exists(self.rawtherapee_cli): + msg = 'rawtherapee-cli executable count not be found in "{}".' + log.error(msg.format(self.rawtherapee_cli)) + self.rawtherapee_cli = None + return None elif exe == "dcraw": if exe.startswith(os.path.sep): self.dcraw = location else: self.dcraw = distutils.spawn.find_executable(location) - if self.dcraw: - if not os.path.exists(self.dcraw): - log.error("Error: dcraw does not exist: \t{0}".format(self.oiiotool)) - self.dcraw = None + if not self.dcraw or not os.path.exists(self.dcraw): + msg = 'dcraw executable count not be found in "{}".' + log.error(msg.format(self.dcraw)) + self.dcraw = None elif exe == "oiiotool": if exe.startswith(os.path.sep): self.oiiotool = location else: self.oiiotool = distutils.spawn.find_executable(location) - if self.oiiotool: - if not os.path.exists(self.oiiotool): - log.error("Error: oiiotool does not exist: \t{0}".format(self.oiiotool)) - self.oiiotool = None - return None + if not self.oiiotool or not os.path.exists(self.oiiotool): + msg = 'oiiotool executable count not be found in "{}".' + log.error(msg.format(self.oiiotool)) + self.oiiotool = None + return None - elif exe == "exiftool": + elif exe == 'exiftool': if exe.startswith(os.path.sep): self.exiftool = location else: self.exiftool = distutils.spawn.find_executable(location) - if self.exiftool: - if not os.path.exists(self.exiftool): - # If no exiftool, we'll just skip tif metadata copying - log.error("Error: exiftool does not exist. No metadata will be copied to tif files") - self.exiftool = None - + if not self.exiftool or not os.path.exists(self.exiftool): + # If no exiftool, we'll just skip tif metadata copying. + msg = 'exiftool executable count not be found in "{}". No metadata will be copied to tif files.' + log.error(msg.format(self.exiftool)) + self.exiftool = None self.profile = self.config.get('rt_default_profile') possible_output_formats = self.config.get('possible_output_formats') @@ -146,13 +155,16 @@ def validate(self): self.colorspaces_out = self.config.get('colorspaces_out') self.default_autoexposure = self.config.get('autoexpose') self.autoexposure_target = self.config.get('autoexposure_target') - self.autoexposure_center_percentage = self.config.get('autoexposure_center_percentage') - self.ocioconfig = self.config.get('default_ocioconfig') + self.autoexposure_center_percentage = self.config.get( + 'autoexposure_center_percentage') + self.ocioconfig = self.config.get('default_ocioconfig') or '' self.resize_filter = self.config.get('resize_filter') self.default_format_resize = self.config.get('default_format_resize') self.raw_formats = self.config.get('raw_formats') - self.raw_formats = self.raw_formats + [r.upper() for r in self.raw_formats] + self.raw_formats = self.raw_formats + \ + [r.upper() for r in self.raw_formats] + log.debug("Processing raw formats:\n" + " ".join(self.raw_formats)) # Get commandline arguments @@ -238,9 +250,8 @@ def validate(self): "Can be comma separated list.", required=False) - # Show help if no args. - if len(sys.argv)==1: + if len(sys.argv) == 1: parser.print_help() return None @@ -262,7 +273,8 @@ def validate(self): if "." in input_path: input_path_ext = input_path.split('.')[-1] if input_path_ext in self.raw_formats: - self.image_sequences.append((os.path.dirname(input_path), pyseq.Sequence([input_path]))) + self.image_sequences.append( + (os.path.dirname(input_path), pyseq.Sequence([input_path]))) if args.output: self.dst = args.output self.dst = os.path.expanduser(self.dst) @@ -290,17 +302,20 @@ def validate(self): self.aberration = args.aberration if self.aberration: # Copy pp3 profile to temp dir - tmp_profile = os.path.join(self.tmp_dir, os.path.basename(self.profile)) - log.debug("Chromatic Aberration specified: Creating temp profile: {0}".format(tmp_profile)) + tmp_profile = os.path.join( + self.tmp_dir, os.path.basename(self.profile)) + log.debug("Chromatic Aberration specified: Creating temp profile: {0}".format( + tmp_profile)) shutil.copyfile(self.profile, tmp_profile) - sed_proc = subprocess.Popen(shlex.split("sed -i -e 's/CA=false/CA=true/' {0}".format(tmp_profile))) + cmd = ['sed', '-i', '-e', "'s/CA=false/CA=true/'", tmp_profile] + log.debug(' '.join(cmd)) + sed_proc = subprocess.Popen(cmd) result, error = sed_proc.communicate() if not error: self.profile = tmp_profile self.resize = args.resize - self.autoexpose = args.autoexpose self.autoexpose_each = args.autoexpose_each @@ -318,14 +333,12 @@ def validate(self): elif self.default_autoexposure == "ae": self.autoexpose_each = True - - - # Ignore string self.search_exclude = args.search_exclude if self.search_exclude: if "," in self.search_exclude: - self.search_exclude_list = [i.strip() for i in self.search_exclude.rsplit(',')] + self.search_exclude_list = [ + i.strip() for i in self.search_exclude.rsplit(',')] else: self.search_exclude_list = [self.search_exclude.strip()] else: @@ -335,7 +348,8 @@ def validate(self): self.search_include = args.search_include if self.search_include: if "," in self.search_include: - self.search_include_list = [i.strip() for i in self.search_include.rsplit(',')] + self.search_include_list = [ + i.strip() for i in self.search_include.rsplit(',')] else: self.search_include_list = [self.search_include.strip()] else: @@ -356,7 +370,7 @@ def validate(self): if output_formats_string in possible_output_formats: self.output_formats.append(output_formats_string) if not self.output_formats: - log.error("Error: Invalid output format specified.") + log.error('Error: Invalid output format specified') return None # OCIO Config Setup @@ -365,17 +379,17 @@ def validate(self): self.ocioconfig = os.path.realpath(self.ocioconfig) self.ocioconfig = os.path.expanduser(self.ocioconfig) if not os.path.isfile(self.ocioconfig): - log.error("Error: Specified OCIO Config does not exist.") - self.ocioconfig = None + log.error('Error: Specified OCIO Config does not exist') + self.ocioconfig = '' if not self.ocioconfig: env_ocio = os.getenv("OCIO") if env_ocio: self.ocioconfig = env_ocio if not os.path.exists(self.ocioconfig): - log.warning("OCIO Config does not exist: \n\t{0}\n\tNo OCIO color transform will be applied".format( + log.warning('OCIO Config does not exist: \n\t{0}\n\tNo OCIO color transform will be applied'.format( self.ocioconfig)) self.ocioconfig = None - log.debug("OCIO Config: {0}".format(self.ocioconfig)) + log.debug('OCIO Config: {0}'.format(self.ocioconfig)) # OCIO Colorspace for output image if args.colorspaces_out: @@ -391,20 +405,21 @@ def validate(self): else: # If no image format specified, assume exr self.colorspaces_out['exr'] = args.colorspaces_out - log.debug("Output Color Transform: {0}".format(self.colorspaces_out)) + log.debug("Output Color Transform: {0}".format( + self.colorspaces_out)) # Find source raw images for input_dir in input_dirs: self.gather_images(input_dir) if self.image_sequences: - log.debug("Found these image sequences:\n" + "\n".join([img[1].path() for img in self.image_sequences])) + log.debug("Found these image sequences:\n" + + "\n".join([img[1].path() for img in self.image_sequences])) return True else: - log.error("Found no image sequences... Please try again.") + log.error('Found no image sequences... Please try again') return False - def process(self): ''' Convert every raw image in image_sequences into the output format(s). @@ -413,7 +428,8 @@ def process(self): # Loop through all found image sequences to process for root_dir, sequence in self.image_sequences: if len(sequence) > 2: - log.info("Debayering image sequence:\n\t{0}".format(os.path.join(sequence.dirname, sequence.format('%h%p%t %r')))) + log.info("Debayering image sequence:\n\t{0}".format( + os.path.join(sequence.dirname, sequence.format('%h%p%t %r')))) # Create destination directories if they don't exist dst_dir = self.dstconv(sequence.dirname, root_dir) @@ -425,17 +441,21 @@ def process(self): middle_frame = sequence[int(sequence.length() / 2)] if not self.autoexpose_each: # temp debayer for auto exposure - output_image_path = self.dstconv(middle_frame.path, root_dir) + output_image_path = self.dstconv( + middle_frame.path, root_dir) output_image_path = os.path.splitext(output_image_path)[0] - tmp_output_image = self.debayer_image(middle_frame.path, output_image_path) + tmp_output_image = self.debayer_image( + middle_frame.path, output_image_path) if tmp_output_image: - self.autoexposure = self.calc_autoexposure(tmp_output_image) + self.autoexposure = self.calc_autoexposure( + tmp_output_image) log.info("Auto exposure:\n\t{0} calculated from middle frame: {1}".format( self.autoexposure, os.path.basename(middle_frame)) ) os.remove(tmp_output_image) else: - log.error("Error: Could not calc autoexposure. Setting to 1.0") + log.error( + "Error: Could not calc autoexposure. Setting to 1.0") self.autoexposure = 1.0 else: self.autoexposure = 1.0 @@ -449,17 +469,19 @@ def process(self): log.info("\nsrc: \t{0}".format(image.path)) # Debayer image to temp tif file - tmp_output_image = self.debayer_image(image.path, output_image_path) + tmp_output_image = self.debayer_image( + image.path, output_image_path) if tmp_output_image: for output_format in self.output_formats: - output_image = self.convert_image(tmp_output_image, output_image_path, output_format=output_format) + output_image = self.convert_image( + tmp_output_image, output_image_path, output_format=output_format) log.info("dst:\t{0}".format(output_image)) - elapsed_time = datetime.timedelta(seconds = time.time() - start_time) + elapsed_time = datetime.timedelta( + seconds=time.time() - start_time) log.info("time: \t{0}".format(elapsed_time)) # Clean up file os.remove(tmp_output_image) - def dstconv(self, src_path, toplevel_dir, dst_path=None): ''' Convert src_path to dstpath, reconstructing path under rootpath in dstpath. @@ -471,7 +493,6 @@ def dstconv(self, src_path, toplevel_dir, dst_path=None): dst_path = self.dst return dst_path + src_path.split(toplevel_dir)[-1] - def gather_images(self, source_dir): ''' Gather all image sequences found in source dir and add to self.image_sequences. @@ -482,11 +503,11 @@ def gather_images(self, source_dir): if generator: for path, parentdir, sequences in generator: for sequence in sequences: - file_extension = os.path.splitext(sequence.path())[1][1:].strip() + file_extension = os.path.splitext( + sequence.path())[1][1:].strip() if file_extension in self.raw_formats: self.image_sequences.append((source_dir, sequence)) - def debayer_image(self, input_image, output_image_path, retry=0): ''' Debayer the given input image into a scene-linear float tiff file @@ -498,9 +519,11 @@ def debayer_image(self, input_image, output_image_path, retry=0): # Skip existing output image files unless self.overwrite is enabled if not self.overwrite: for image_format in self.output_formats: - final_output_image = output_image_path + ".{0}".format(image_format) + final_output_image = output_image_path + \ + ".{0}".format(image_format) if os.path.exists(final_output_image) and self.get_size(final_output_image)[0] > 0.01: - log.warning("skip existing:\t{0}".format(final_output_image)) + log.warning("skip existing:\t{0}".format( + final_output_image)) return None # Skip files in ignore_list if self.search_exclude_list: @@ -515,7 +538,8 @@ def debayer_image(self, input_image, output_image_path, retry=0): log.warning("exluded:\t{0}".format(input_image)) return None - tmp_output_image = os.path.join(self.tmp_dir, os.path.basename(output_image_path)) + ".tif" + tmp_output_image = os.path.join( + self.tmp_dir, os.path.basename(output_image_path)) + ".tif" log.debug("tmp:\t{0}".format(tmp_output_image)) @@ -526,11 +550,13 @@ def debayer_image(self, input_image, output_image_path, retry=0): # dcraw tutorial: http://www.guillermoluijk.com/tutorial/dcraw/index_en.htm # rawpy might be another option https://letmaik.github.io/rawpy/api/rawpy.Params.html#rawpy.Params - dcraw_cmd = "dcraw -v -T -4 -o 6 -q 3 -w -H 0 -W -c '{0}'".format(input_image) + dcraw_cmd = ['dcraw', '-v', '-T', '-4', '-o', '6', + '-q', '3', '-w', '-H', '0', '-W', '-c', input_image] with open(tmp_output_image, "w") as output_file: + log.debug(' '.join(dcraw_cmd)) dcraw_proc = subprocess.Popen( - shlex.split(dcraw_cmd), + dcraw_cmd, stdout=output_file ) result, error = dcraw_proc.communicate() @@ -538,21 +564,20 @@ def debayer_image(self, input_image, output_image_path, retry=0): log.error("Error Processing Result: " + error) if not os.path.isfile(tmp_output_image): if retry < 3: - self.debayer_image(input_image, output_image_path, retry=retry+1) + self.debayer_image( + input_image, output_image_path, retry=retry+1) else: self.copy_metadata(input_image, tmp_output_image) return tmp_output_image elif self.debayer_engine == "rt": - rtcli_cmd = "{0} -o {1} -p {2} -b16f -Y -q -f -t -c {3}".format( - self.rawtherapee_cli, - tmp_output_image, - self.profile, - input_image - ) - log.debug(rtcli_cmd) + rtcli_cmd = [ + self.rawtherapee_cli, '-o', tmp_output_image, '-p', + self.profile, '-b16f', '-Y', '-q', '-f', '-t', '-c', input_image, + ] + log.debug(' '.join(rtcli_cmd)) rtcli_proc = subprocess.Popen( - shlex.split(rtcli_cmd), + rtcli_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE ) @@ -560,20 +585,23 @@ def debayer_image(self, input_image, output_image_path, retry=0): if error: log.error("Error Processing Result: " + error) if not os.path.isfile(tmp_output_image): - log.warning("Output file did not generate:\n\t{0}".format(tmp_output_image)) + log.warning( + "Output file did not generate:\n\t{0}".format(tmp_output_image)) if retry < 1: - self.debayer_image(input_image, output_image_path, retry=retry+1) + self.debayer_image( + input_image, output_image_path, retry=retry+1) else: output_size, output_size_str = self.get_size(tmp_output_image) log.debug("Output size is: {0}".format(output_size_str)) if output_size > 0.01: return tmp_output_image else: - log.warning("File size of output is less than 4kb, processing again...") + log.warning( + "File size of output is less than 4kb, processing again...") os.remove(tmp_output_image) if retry < 1: - self.debayer_image(input_image, output_image_path, retry=retry+1) - + self.debayer_image( + input_image, output_image_path, retry=retry+1) def convert_image(self, src_img, dst_img, output_format="exr"): ''' Convert src_img to specified format at dst_img @@ -585,59 +613,68 @@ def convert_image(self, src_img, dst_img, output_format="exr"): # calculate new autoexposure for each frame if self.autoexpose_each if self.autoexpose_each: self.autoexposure = self.calc_autoexposure(src_img) - log.info("autoexposure {0} for {1}".format(self.autoexposure, os.path.basename(src_img))) + log.info("autoexposure {0} for {1}".format( + self.autoexposure, os.path.basename(src_img))) dst_img = dst_img + ".{0}".format(output_format) - - oiiotool_cmd = "{0} -v {1}".format(self.oiiotool, src_img) + oiiotool_cmd = [self.oiiotool, '-v', src_img] # Setup resize string - resize_string = " --rangecompress --resize" + resize_list = ['--rangecompress'] if self.resize_filter: - resize_string +=":filter={0}".format(self.resize_filter) + resize_list.append('--resize:filter={}'.format(self.resize_filter)) + else: + resize_list.append('--resize') current_format_resize = self.default_format_resize[output_format] if current_format_resize: resize = current_format_resize else: resize = self.resize - resize_string += " {0} --rangeexpand".format(resize) + resize_list += [resize, '--rangeexpand'] if resize: - oiiotool_cmd += resize_string + oiiotool_cmd += resize_list # Specify datatype if output_format in self.datatypes: if self.datatypes[output_format] in ["uint8", "sint8", "uint10", "uint12", "uint16", "sint16", "uint32", "sint32", "half", "float", "double"]: - oiiotool_cmd += " -d {0}".format(self.datatypes[output_format]) + oiiotool_cmd += ['-d', self.datatypes[output_format]] if self.autoexposure != 1.0: - oiiotool_cmd += " --mulc {0},{0},{0},1.0".format(self.autoexposure) + exp = str(self.autoexposure) + exp = [exp, exp, exp, '1.0'] + oiiotool_cmd += ['--mulc', ','.join(exp)] if self.exposure: - oiiotool_cmd += " --mulc {0},{0},{0},1.0".format(self.exposure) + exp = str(self.exposure) + exp = [exp, exp, exp, '1.0'] + oiiotool_cmd += ['--mulc', ','.join(exp)] # Validate OCIO stuff if self.colorspaces_out: if output_format in self.colorspaces_out: if self.ocioconfig: - oiiotool_cmd += " --colorconfig \"{0}\" --colorconvert \"{1}\" \"{2}\"".format( + oiiotool_cmd += [ + '--colorconfig', self.ocioconfig, + '--colorconvert', self.colorspace_in, - self.colorspaces_out[output_format] - ) - + self.colorspaces_out[output_format], + ] # Output format compression options if self.compression: if output_format in self.compression: - oiiotool_cmd += " {0}".format(self.compression[output_format]) + oiiotool_cmd += self.compression[output_format] # Output image - oiiotool_cmd += " -o {0}".format(dst_img) - log.debug(oiiotool_cmd) - oiiotool_proc = subprocess.Popen(shlex.split(oiiotool_cmd), - stdin=subprocess.PIPE, - stdout=subprocess.PIPE) + oiiotool_cmd += ['-o', dst_img] + log.debug(' '.join(oiiotool_cmd)) + oiiotool_proc = subprocess.Popen( + oiiotool_cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE + ) result, error = oiiotool_proc.communicate() if os.path.isfile(dst_img): @@ -646,10 +683,10 @@ def convert_image(self, src_img, dst_img, output_format="exr"): self.copy_metadata(src_img, dst_img) return dst_img else: - log.error("Output {0} file did not generate!".format(output_format)) + log.error( + "Output {0} file did not generate!".format(output_format)) return None - def calc_autoexposure(self, imgpath): ''' Calculate an autoexposure value based on an average pixel sample. @@ -658,7 +695,8 @@ def calc_autoexposure(self, imgpath): try: import OpenImageIO as oiio except ImportError: - log.error("Error: Missing dependencies. Please install OpenImageIO to enable autoexposure. Using default exposure of 1.0") + log.error( + "Missing dependencies. Please install OpenImageIO to enable autoexposure. Using default exposure of 1.0") return 1.0 # Create ImageBuf from ImageInput imgbuf = oiio.ImageBuf(imgpath) @@ -666,14 +704,15 @@ def calc_autoexposure(self, imgpath): # Construct ROI from self.autoexposure_center_percentage center = (int(imgspec.width/2), int(imgspec.height/2)) - size = (int(imgspec.width * self.autoexposure_center_percentage), int(imgspec.height * self.autoexposure_center_percentage)) + size = (int(imgspec.width * self.autoexposure_center_percentage), + int(imgspec.height * self.autoexposure_center_percentage)) - box_roi = oiio.ROI(int(center[0]-size[0]/2), int(center[0]+size[0]/2), int(center[1]-size[1]/2), int(center[1]+size[1]/2)) + box_roi = oiio.ROI(int(center[0]-size[0]/2), int(center[0]+size[0]/2), + int(center[1]-size[1]/2), int(center[1]+size[1]/2)) pixels = imgbuf.get_pixels(format=oiio.FLOAT, roi=box_roi) img_average = pixels.mean() return self.autoexposure_target / img_average - def copy_metadata(self, src, dst): '''Uses exiftool to copy all metadata from src image path to dst image path Args: @@ -681,9 +720,11 @@ def copy_metadata(self, src, dst): dst {str} -- image path of destination ''' if self.exiftool: - exiftool_cmd = "{0} -overwrite_original -tagsFromFile {1} {2}".format(self.exiftool, src, dst) - log.debug("Copying exif metadata from raw: \t{0}".format(exiftool_cmd)) - exiftool_proc = subprocess.Popen(shlex.split(exiftool_cmd)) + exiftool_cmd = [self.exiftool, + '-overwrite_original', '-tagsFromFile', src, dst] + log.debug('Copying exif metadata from raw') + log.debug(' '.join(exiftool_cmd)) + exiftool_proc = subprocess.Popen(exiftool_cmd) result, error = exiftool_proc.communicate() if error: log.warning("Copying metadata failed: {0}".format(error)) @@ -691,19 +732,18 @@ def copy_metadata(self, src, dst): else: return True else: - log.error("Error: exiftool not found. Will skip setting the metadata.") + log.error("Error: exiftool not found. Will skip setting the metadata") return None - def get_size(self, filepath, suffix='B'): ''' Return filesize of filepath ''' num = os.path.getsize(filepath) - for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: + for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: if abs(num) < 1024.0: return num, "%3.1f%s%s" % (num, unit, suffix) num /= 1024.0 return num, "%.1f%s%s" % (num, 'Yi', suffix) -if __name__=="__main__": - debayer = Debayer() \ No newline at end of file +if __name__ == "__main__": + Debayer() diff --git a/debayer-config.yaml b/debayer/config.yaml similarity index 95% rename from debayer-config.yaml rename to debayer/config.yaml index 210693b..7c16ddb 100644 --- a/debayer-config.yaml +++ b/debayer/config.yaml @@ -48,9 +48,9 @@ default_output_formats: [exr] # Tiff compression: "none", "lzw", "zip" (deflate), "ccittrle" # Jpeg compression: --quality --attrib "jpeg:subsampling" "4:2:2" --resize 50% compression: - exr: --compression dwaa:45 - tif: --compression zip - jpg: --quality 90 --attrib "jpeg:subsampling" "4:2:2" + exr: ["--compression", "dwaa:45"] + tif: ["--compression", "zip"] + jpg: ["--quality", "90", "--attrib", "jpeg:subsampling", "4:2:2"] # oiiotool resize argument: e.g. 1920x1080, 1920x0 (resizes height to match input AR), 50%, 25% # TODO: Only active if -r not specified on commandline for now diff --git a/lin_ap0.pp3 b/debayer/lin_ap0.pp3 similarity index 100% rename from lin_ap0.pp3 rename to debayer/lin_ap0.pp3 diff --git a/pyseq.py b/pyseq.py deleted file mode 100755 index d4e1d90..0000000 --- a/pyseq.py +++ /dev/null @@ -1,1189 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# --------------------------------------------------------------------------------------------- -# Copyright (c) 2011-2017, Ryan Galloway (ryan@rsgalloway.com) -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# - Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# - Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# - Neither the name of the software nor the names of its contributors -# may be used to endorse or promote products derived from this software -# without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# ----------------------------------------------------------------------------- - -"""PySeq is a python module that finds groups of items that follow a naming -convention containing a numerical sequence index, e.g. :: - - fileA.001.png, fileA.002.png, fileA.003.png... - -and serializes them into a compressed sequence string representing the entire -sequence, e.g. :: - - fileA.1-3.png - -It should work regardless of where the numerical sequence index is embedded -in the name. - -Docs and latest version available for download at - - http://github.com/rsgalloway/pyseq -""" - -import os -import re -import logging -import warnings -import functools -from glob import glob -from glob import iglob -from datetime import datetime - -__version__ = "0.5.1" - -# default serialization format string -global_format = '%4l %h%p%t %R' -default_format = '%h%r%t' - -# use strict padding on sequences (pad length must match) -# https://github.com/rsgalloway/pyseq/issues/41 -strict_pad = True - -# regex for matching numerical characters -digits_re = re.compile(r'\d+') - -# regex for matching format directives -format_re = re.compile(r'%(?P\d+)?(?P\w+)') - -# character to join explicit frame ranges on -range_join = os.environ.get('PYSEQ_RANGE_SEP', ', ') - -__all__ = [ - 'SequenceError', 'FormatError', 'Item', 'Sequence', 'diff', 'uncompress', - 'getSequences', 'get_sequences', 'walk' -] - -# logging handlers -log = logging.getLogger('pyseq') -log.addHandler(logging.StreamHandler()) -log.setLevel(int(os.environ.get('PYSEQ_LOG_LEVEL', logging.INFO))) - -# show deprecationWarnings in 2.7+ -warnings.simplefilter('always', DeprecationWarning) - -# python 3 strings -try: - unicode = unicode -except NameError: - str = str - unicode = str - bytes = bytes - basestring = (str,bytes) -else: - str = str - unicode = unicode - bytes = str - basestring = basestring - - -def _natural_key(x): - """ Splits a string into characters and digits. This helps in sorting file - names in a 'natural' way. - """ - return [int(c) if c.isdigit() else c.lower() for c in re.split("(\d+)", x)] - - -def _ext_key(x): - """ Similar to '_natural_key' except this one uses the file extension at - the head of split string. This fixes issues with files that are named - similar but with different file extensions: - This example: - file.001.jpg - file.001.tiff - file.002.jpg - file.002.tiff - Would get properly sorted into: - file.001.jpg - file.002.jpg - file.001.tiff - file.002.tiff - """ - name, ext = os.path.splitext(x) - return [ext] + _natural_key(name) - - -def natural_sort(items): - return sorted(items, key=_natural_key) - - -class SequenceError(Exception): - """Special exception for Sequence errors - """ - pass - - -class FormatError(Exception): - """Special exception for Sequence format errors - """ - pass - - -def deprecated(func): - """Deprecation warning decorator - """ - def inner(*args, **kwargs): - warnings.warn("Call to deprecated method {}".format(func.__name__), - category=DeprecationWarning, stacklevel=2) - return func(*args, **kwargs) - inner.__name__ = func.__name__ - inner.__doc__ = func.__doc__ - inner.__dict__.update(func.__dict__) - return inner - - -class Item(str): - """Sequence member file class - - :param item: Path to file. - """ - - def __init__(self, item): - super(Item, self).__init__() - log.debug('adding %s', item) - self.item = item - self.__path = getattr(item, 'path', None) - if self.__path is None: - self.__path = os.path.abspath(str(item)) - self.__dirname, self.__filename = os.path.split(self.__path) - self.__digits = digits_re.findall(self.name) - self.__parts = digits_re.split(self.name) - self.__stat = None - - # modified by self.is_sibling() - self.frame = None - self.head = self.name - self.tail = '' - self.pad = None - - def __eq__(self, other): - return self.path == other.path - - def __ne__(self, other): - return self.path != other.path - - def __lt__(self, other): - return self.frame < other.frame - - def __gt__(self, other): - return self.frame > other.frame - - def __ge__(self, other): - return self.frame >= other.frame - - def __le__(self, other): - return self.frame <= other.frame - - def __str__(self): - return str(self.name) - - def __repr__(self): - return '' % self.name - - def __getattr__(self, key): - return getattr(self.item, key) - - @property - def path(self): - """Item absolute path, if a filesystem item. - """ - return self.__path - - @property - def name(self): - """Item base name attribute - """ - return self.__filename - - @property - def dirname(self): - """"Item directory name, if a filesystem item." - """ - return self.__dirname - - @property - def digits(self): - """Numerical components of item name. - """ - return self.__digits - - @property - def parts(self): - """Non-numerical components of item name - """ - return self.__parts - - @property - def exists(self): - """Returns True if this item exists on disk - """ - return os.path.isfile(self.__path) - - @property - def size(self): - """Returns the size of the Item, reported by os.stat - """ - return self.stat.st_size - - @property - def mtime(self): - """Returns the modification time of the Item - """ - return self.stat.st_mtime - - @property - def stat(self): - """ Returns the os.stat object for this file. - """ - if self.__stat is None: - self.__stat = os.stat(self.__path) - return self.__stat - - @deprecated - def isSibling(self, item): - """Deprecated: use is_sibling instead - """ - return self.is_sibling(item) - - def is_sibling(self, item): - """Determines if this and item are part of the same sequence. - - :param item: An :class:`.Item` instance. - - :return: True if this and item are sequential siblings. - """ - if not isinstance(item, Item): - item = Item(item) - - d = diff(self, item) - is_sibling = (len(d) == 1) and (self.parts == item.parts) - - # I do not understand why we are updating information - # while this is a predicate method - if is_sibling: - frame = d[0]['frames'][0] - self.frame = int(frame) - self.pad = len(frame) - self.head = self.name[:d[0]['start']] - self.tail = self.name[d[0]['end']:] - frame = d[0]['frames'][1] - item.frame = int(frame) - item.pad = len(frame) - item.head = item.name[:d[0]['start']] - item.tail = item.name[d[0]['end']:] - - return is_sibling - - -class Sequence(list): - """Extends list class with methods that handle item sequentialness. - - For example: - - >>> s = Sequence(['file.0001.jpg', 'file.0002.jpg', 'file.0003.jpg']) - >>> print(s) - file.1-3.jpg - >>> s.append('file.0006.jpg') - >>> print(s.format('%4l %h%p%t %R')) - 4 file.%04d.jpg 1-3 6 - >>> s.includes('file.0009.jpg') - True - >>> s.includes('file.0009.pic') - False - >>> s.contains('file.0006.jpg') - False - >>> print(s.format('%h%p%t %r (%R)')) - file.%04d.jpg 1-6 (1-3 6) - """ - - def __init__(self, items): - """ - Create a new Sequence class object. - - :param: items: Sequential list of items. - - :return: pyseq.Sequence class instance. - """ - # otherwise Sequence consumes the list - items = items[::] - super(Sequence, self).__init__([Item(items.pop(0))]) - self.__missing = [] - self.__dirty = False - self.__frames = None - - while items: - f = Item(items.pop(0)) - try: - self.append(f) - log.debug('+Item belongs to sequence.') - except SequenceError: - log.debug('-Item does not belong to sequence.') - continue - except KeyboardInterrupt: - log.info("Stopping.") - break - - def __attrs__(self): - """Replaces format directives with callables to get their values.""" - return { - 'l': self.length, - 's': self.start, - 'e': self.end, - 'f': self.frames, - 'm': self.missing, - 'M': functools.partial(self._get_framerange, self.missing(), missing=True), - 'd': lambda *x: self.size, - 'D': self.directory, - 'p': self._get_padding, - 'r': functools.partial(self._get_framerange, self.frames(), missing=False), - 'R': functools.partial(self._get_framerange, self.frames(), missing=True), - 'h': self.head, - 't': self.tail - } - - def __str__(self): - return self.format(default_format) - - def __repr__(self): - return '' % str(self) - - def __getattr__(self, key): - return getattr(self[0], key) - - def __contains__(self, item): - super(Sequence, self).__contains__(Item(item)) - - def __setitem__(self, index, item): - """ Used to set a particular element in the sequence - """ - if type(item) is not Item: - item = Item(item) - if self.includes(item): - super(Sequence, self).__setitem__(index, item) - self.__frames = None - self.__missing = None - else: - raise SequenceError("Item is not a member of sequence.") - - def __setslice__(self, start, end, item): - if isinstance(item, basestring): - item = Sequence([item]) - if isinstance(item, list) is False: - raise TypeError("Invalid type to add to sequence") - for i in item: - if self.includes(i) is False: - raise SequenceError("Item (%s) is not a member of sequence." - % i) - super(Sequence, self).__setslice__(start, end, item) - self.__frames = None - self.__missing = None - - def __add__(self, item): - """ return a new sequence with the item appended. Accepts an Item, - a string, or a list. - """ - if isinstance(item, basestring): - item = Sequence([item]) - if isinstance(item, list) is False: - raise TypeError("Invalid type to add to sequence") - ns = Sequence(self[::]) - ns.extend(item) - return ns - - def __iadd__(self, item): - if isinstance(item, basestring) or type(item) is Item: - item = [item] - if isinstance(item, list) is False: - raise TypeError("Invalid type to add to sequence") - self.extend(item) - return self - - def format(self, fmt=global_format): - """Format the stdout string. - - The following directives can be embedded in the format string. - Format directives support padding, for example: "%04l". - - +-----------+--------------------------------------+ - | Directive | Meaning | - +===========+======================================+ - | ``%s`` | sequence start | - +-----------+--------------------------------------+ - | ``%e`` | sequence end | - +-----------+--------------------------------------+ - | ``%l`` | sequence length | - +-----------+--------------------------------------+ - | ``%f`` | list of found files | - +-----------+--------------------------------------+ - | ``%m`` | list of missing files | - +-----------+--------------------------------------+ - | ``%M`` | explicit missingfiles [11-14,19-21] | - +-----------+--------------------------------------+ - | ``%p`` | padding, e.g. %06d | - +-----------+--------------------------------------+ - | ``%r`` | implied range, start-end | - +-----------+--------------------------------------+ - | ``%R`` | explicit broken range, [1-10, 15-20] | - +-----------+--------------------------------------+ - | ``%d`` | disk usage | - +-----------+--------------------------------------+ - | ``%D`` | parent directory | - +-----------+--------------------------------------+ - | ``%h`` | string preceding sequence number | - +-----------+--------------------------------------+ - | ``%t`` | string after the sequence number | - +-----------+--------------------------------------+ - - :param fmt: Format string. Default is '%4l %h%p%t %R'. - - :return: Formatted string. - """ - format_char_types = { - 's': 'i', - 'e': 'i', - 'l': 'i', - 'f': 's', - 'm': 's', - 'M': 's', - 'p': 's', - 'r': 's', - 'R': 's', - 'd': 's', - 'D': 's', - 'h': 's', - 't': 's' - } - - atts = self.__attrs__() - for m in format_re.finditer(fmt): - var = m.group('var') - pad = m.group('pad') - try: - fmt_char = format_char_types[var] - except KeyError as err: - raise FormatError("Bad directive: %%%s" % var) - _old = '%s%s' % (pad or '', var) - _new = '(%s)%s%s' % (var, pad or '', fmt_char) - fmt = fmt.replace(_old, _new) - val = atts[var] - # only execute the callable once, just in case - if callable(val): - val = atts[var]() - atts[var] = val - - return fmt % atts - - @property - def mtime(self): - """Returns the latest mtime of all items - """ - maxDate = list() - for i in self: - maxDate.append(i.mtime) - return max(maxDate) - - @property - def size(self): - """Returns the size all items (divide by 1024*1024 for MBs) - """ - tempSize = list() - for i in self: - tempSize.append(i.size) - return sum(tempSize) - - def directory(self): - return self[0].dirname + os.sep - - def length(self): - """:return: The length of the sequence.""" - return len(self) - - def frames(self): - """:return: List of files in sequence.""" - if not hasattr(self, '__frames') or not self.__frames or self.__dirty: - self.__frames = self._get_frames() - self.__frames.sort() - return self.__frames - - def start(self): - """:return: First index number in sequence - """ - try: - return self.frames()[0] - except IndexError: - return 0 - - def end(self): - """:return: Last index number in sequence - """ - try: - return self.frames()[-1] - except IndexError: - return 0 - - def missing(self): - """:return: List of missing files.""" - if not hasattr(self, '__missing') or not self.__missing: - self.__missing = self._get_missing() - return self.__missing - - def head(self): - """:return: String before the sequence index number.""" - return self[0].head - - def tail(self): - """:return: String after the sequence index number.""" - return self[0].tail - - def path(self): - """:return: Absolute path to sequence.""" - _dirname = str(os.path.dirname(os.path.abspath(self[0].path))) - return os.path.join(_dirname, str(self)) - - def includes(self, item): - """Checks if the item can be contained in this sequence that is if it - is a sibling of any of the items in the list - - For example: - - >>> s = Sequence(['fileA.0001.jpg', 'fileA.0002.jpg']) - >>> print(s) - fileA.1-2.jpg - >>> s.includes('fileA.0003.jpg') - True - >>> s.includes('fileB.0003.jpg') - False - """ - if len(self) > 0: - if not isinstance(item, Item): - item = Item(item) - if self[-1] != item: - return self[-1].is_sibling(item) - elif self[0] != item: - return self[0].is_sibling(item) - else: - # it should be the only item in the list - if self[0] == item: - return True - - return True - - def contains(self, item): - """Checks for sequence membership. Calls Item.is_sibling() and returns - True if item is part of the sequence. - - For example: - - >>> s = Sequence(['fileA.0001.jpg', 'fileA.0002.jpg']) - >>> print(s) - fileA.1-2.jpg - >>> s.contains('fileA.0003.jpg') - False - >>> s.contains('fileB.0003.jpg') - False - - :param item: pyseq.Item class object. - - :return: True if item is a sequence member. - """ - if len(self) > 0: - if not isinstance(item, Item): - item = Item(item) - return self.includes(item)\ - and self.end() >= item.frame >= self.start() - - return False - - def append(self, item): - """Adds another member to the sequence. - - :param item: pyseq.Item object. - - :exc:`SequenceError` raised if item is not a sequence member. - """ - if type(item) is not Item: - item = Item(item) - - if self.includes(item): - super(Sequence, self).append(item) - self.__frames = None - self.__missing = None - else: - raise SequenceError('Item is not a member of this sequence') - - def insert(self, index, item): - """ Add another member to the sequence at the given index. - :param item: pyseq.Item object. - :exc: `SequenceError` raised if item is not a sequence member. - """ - if type(item) is not Item: - item = Item(item) - - if self.includes(item): - super(Sequence, self).insert(index, item) - self.__frames = None - self.__missing = None - else: - raise SequenceError("Item is not a member of this sequence.") - - def extend(self, items): - """ Add members to the sequence. - :param items: list of pyseq.Item objects. - :exc: `SequenceError` raised if any items are not a sequence - member. - """ - for item in items: - if type(item) is not Item: - item = Item(item) - - if self.includes(item): - super(Sequence, self).append(item) - self.__frames = None - self.__missing = None - else: - raise SequenceError("Item (%s) is not a member of this " - "sequence." % item) - - def reIndex(self, offset, padding=None): - """Renames and reindexes the items in the sequence, e.g. :: - - >>> seq.reIndex(offset=100) - - will add a 100 frame offset to each Item in `seq`, and rename - the files on disk. - - :param offset: the frame offset to apply to each item - :param padding: change the padding - """ - if not padding: - padding = self.format("%p") - - if offset > 0: - gen = ((image, frame) for (image, frame) in zip(reversed(self), - reversed(self.frames()))) - else: - gen = ((image, frame) for (image, frame) in zip(self, self.frames())) - - for image, frame in gen: - oldName = image.path - newFrame = padding % (frame + offset) - newFileName = "%s%s%s" % (self.format("%h"), newFrame, - self.format("%t")) - newName = os.path.join(image.dirname, newFileName) - - try: - import shutil - shutil.move(oldName, newName) - except Exception as err: - log.error(err) - else: - log.debug('renaming %s %s' % (oldName, newName)) - self.__dirty = True - image.frame = int(newFrame) - - else: - self.frames() - - def _get_padding(self): - """:return: padding string, e.g. %07d""" - try: - pad = self[0].pad - if pad is None: - return "" - if pad < 2: - return '%d' - return '%%%02dd' % pad - except IndexError: - return '' - - def _get_framerange(self, frames, missing=True): - """Returns frame range string, e.g. [1-500]. - - :param frames: list of ints like [1,4,8,12,15]. - :param missing: Expand sequence to exclude missing sequence indices. - - :return: formatted frame range string. - """ - frange = [] - start = '' - end = '' - if not missing: - if frames: - return '%s-%s' % (self.start(), self.end()) - else: - return '' - - if not frames: - return '' - - for i in range(0, len(frames)): - frame = frames[i] - if i != 0 and frame != frames[i - 1] + 1: - if start != end: - frange.append('%s-%s' % (str(start), str(end))) - elif start == end: - frange.append(str(start)) - start = end = frame - continue - if start is '' or int(start) > frame: - start = frame - if end is '' or int(end) < frame: - end = frame - if start == end: - frange.append(str(start)) - else: - frange.append('%s-%s' % (str(start), str(end))) - return "[%s]" % range_join.join(frange) - - def _get_frames(self): - """finds the sequence indexes from item names - """ - return [f.frame for f in self if f.frame is not None] - - def _get_missing(self): - """Looks for missing sequence indexes in sequence - - .. todo:: change this to: - r = range(frames[0], frames[-1] + 1) - return sorted(list(set(frames).symmetric_difference(r))) - """ - missing = [] - frames = self.frames() - if len(frames) == 0: - return missing - - r = range(frames[0], frames[-1] + 1) - return sorted(list(set(frames).symmetric_difference(r))) - - -def diff(f1, f2): - """Examines diffs between f1 and f2 and deduces numerical sequence number. - - For example :: - - >>> diff('file01_0040.rgb', 'file01_0041.rgb') - [{'frames': ('0040', '0041'), 'start': 7, 'end': 11}] - - >>> diff('file3.03.rgb', 'file4.03.rgb') - [{'frames': ('3', '4'), 'start': 4, 'end': 5}] - - :param f1: pyseq.Item object. - :param f2: pyseq.Item object, for comparison. - - :return: Dictionary with keys: frames, start, end. - """ - log.debug('diff: %s %s' % (f1, f2)) - if not type(f1) == Item: - f1 = Item(f1) - if not type(f2) == Item: - f2 = Item(f2) - - l1 = [m for m in digits_re.finditer(f1.name)] - l2 = [m for m in digits_re.finditer(f2.name)] - - d = [] - if len(l1) == len(l2): - for i in range(0, len(l1)): - m1 = l1.pop(0) - m2 = l2.pop(0) - if (m1.start() == m2.start()) and (m1.group() != m2.group()): - if strict_pad is True and (len(m1.group()) != len(m2.group())): - continue - d.append({ - 'start': m1.start(), - 'end': m1.end(), - 'frames': (m1.group(), m2.group()) - }) - - log.debug(d) - return d - - -def uncompress(seq_string, fmt=global_format): - """Basic uncompression or deserialization of a compressed sequence string. - - For example: :: - - >>> seq = uncompress('./tests/files/012_vb_110_v001.%04d.png 1-10', fmt='%h%p%t %r') - >>> print(seq) - 012_vb_110_v001.1-10.png - >>> len(seq) - 10 - >>> seq2 = uncompress('./tests/files/a.%03d.tga [1-3, 10, 12-14]', fmt='%h%p%t %R') - >>> print(seq2) - a.1-14.tga - >>> len(seq2) - 7 - >>> seq3 = uncompress('a.%03d.tga 1-14 ([1-3, 10, 12-14])', fmt='%h%p%t %r (%R)') - >>> print(seq3) - a.1-14.tga - >>> len(seq3) - 7 - - See unit tests for more examples. - - :param seq_string: Compressed sequence string. - :param fmt: Format of sequence string. - - :return: :class:`.Sequence` instance. - """ - dirname = os.path.dirname(seq_string) - # remove directory - if "%D" in fmt: - fmt = fmt.replace("%D", "") - name = os.path.basename(seq_string) - log.debug('uncompress: %s' % name) - - # map of directives to regex - remap = { - 's': '\d+', - 'e': '\d+', - 'l': '\d+', - 'h': '(\S+)?', - 't': '(\S+)?', - 'r': '\d+-\d+', - 'R': '\[[\d\s?\-%s?]+\]' % re.escape(range_join), - 'p': '%\d+d', - 'm': '\[.*\]', - 'f': '\[.*\]', - } - - log.debug('fmt in: %s' % fmt) - - # escape any re chars in format - fmt = re.escape(fmt) - - # replace \% with % back again - fmt = fmt.replace('\\%', '%') - - log.debug('fmt escaped: %s' % fmt) - - for m in format_re.finditer(fmt): - _old = '%%%s%s' % (m.group('pad') or '', m.group('var')) - _new = '(?P<%s>%s)' % ( - m.group('var'), - remap.get(m.group('var'), '\w+') - ) - fmt = fmt.replace(_old, _new) - - log.debug('fmt: %s' % fmt) - - regex = re.compile(fmt) - match = regex.match(name) - - log.debug("match: %s" % match.groupdict() if match else "") - - frames = [] - missing = [] - s = None - e = None - - if not match: - log.debug('No matches.') - return - - try: - pad = match.group('p') - - except IndexError: - pad = "%d" - - try: - R = match.group('R') - R = R[1:-1] - number_groups = R.split(range_join) - pad_len = 0 - for number_group in number_groups: - if '-' in number_group: - splits = number_group.split('-') - pad_len = max(pad_len, len(splits[0]), len(splits[1])) - start = int(splits[0]) - end = int(splits[1]) - frames.extend(range(start, end + 1)) - - else: - end = int(number_group) - pad_len = max(pad_len, len(number_group)) - frames.append(end) - if pad == "%d" and pad_len != 0: - pad = "%0" + str(pad_len) + "d" - - except IndexError: - try: - r = match.group('r') - s, e = r.split('-') - frames = range(int(s), int(e) + 1) - - except IndexError: - s = match.group('s') - e = match.group('e') - - try: - frames = eval(match.group('f')) - - except IndexError: - pass - - try: - missing = eval(match.group('m')) - - except IndexError: - pass - - items = [] - if missing: - for i in range(int(s), int(e) + 1): - if i in missing: - continue - f = pad % i - name = '%s%s%s' % ( - match.groupdict().get('h', ''), f, - match.groupdict().get('t', '') - ) - items.append(Item(os.path.join(dirname, name))) - - else: - for i in frames: - f = pad % i - name = '%s%s%s' % ( - match.groupdict().get('h', ''), f, - match.groupdict().get('t', '') - ) - items.append(Item(os.path.join(dirname, name))) - - seqs = get_sequences(items) - if seqs: - return seqs[0] - return seqs - - -@deprecated -def getSequences(source): - """Deprecated: use get_sequences instead - """ - return get_sequences(source) - - -def get_sequences(source): - """Returns a list of Sequence objects given a directory or list that contain - sequential members. - - Get sequences in a directory: - - >>> seqs = get_sequences('./tests/files/') - >>> for s in seqs: print(s) - ... - 012_vb_110_v001.1-10.png - 012_vb_110_v002.1-10.png - a.1-14.tga - alpha.txt - bnc01_TinkSO_tx_0_ty_0.101-105.tif - bnc01_TinkSO_tx_0_ty_1.101-105.tif - bnc01_TinkSO_tx_1_ty_0.101-105.tif - bnc01_TinkSO_tx_1_ty_1.101-105.tif - file.1-2.tif - file.info.03.rgb - file01_40-43.rgb - file02_44-47.rgb - file1-4.03.rgb - file_02.tif - z1_001_v1.1-4.png - z1_002_v1.1-4.png - z1_002_v2.1-4.png - - Get sequences from a list of file names: - - >>> seqs = get_sequences(['fileA.1.rgb', 'fileA.2.rgb', 'fileB.1.rgb']) - >>> for s in seqs: print(s) - ... - fileA.1-2.rgb - fileB.1.rgb - - Get sequences from a list of objects, preserving object attrs: - - >>> seqs = get_sequences(repo.files()) - >>> seqs[0].date - datetime.datetime(2011, 3, 21, 17, 31, 24) - - :param source: Can be directory path, list of strings, or sortable list of objects. - - :return: List of pyseq.Sequence class objects. - """ - start = datetime.now() - - # list for storing sequences to be returned later - seqs = [] - - if isinstance(source, list): - items = sorted(source, key=lambda x: str(x)) - - elif isinstance(source, basestring): - if os.path.isdir(source): - items = sorted(glob(os.path.join(source, '*'))) - else: - items = sorted(glob(source)) - - else: - raise TypeError('Unsupported format for source argument') - - log.debug('Found %s files' % len(items)) - - # organize the items into sequences - while items: - item = Item(items.pop(0)) - found = False - for seq in seqs[::-1]: - if seq.includes(item): - seq.append(item) - found = True - break - if not found: - seq = Sequence([item]) - seqs.append(seq) - - log.debug('time: %s' % (datetime.now() - start)) - - return list(seqs) - - -def iget_sequences(source): - """ Generator version of get_sequences. Creates Sequences from a various - source files. A notable difference is the sort order of iget_sequences - versus get_sequences. iget_sequences uses an adaption of natural sorting - that starts with the file extension. Because of this, Sequences are - returned ordered by their file extension. - - Get sequences in a directory: - - >>> seqs = iget_sequences('./tests/files/') - >>> for s in seqs: print(s) - ... - file01.1-4.j2k - fileA.1-3.jpg - 012_vb_110_v001.1-10.png - 012_vb_110_v002.1-10.png - fileA.1-3.png - z1_001_v1.1-4.png - z1_002_v1.1-4.png - z1_002_v2.1-4.png - file1.03.rgb - file01_40-43.rgb - file2.03.rgb - file02_44-47.rgb - file3-4.03.rgb - file.info.03.rgb - a.1-14.tga - bnc01_TinkSO_tx_0_ty_0.101-105.tif - bnc01_TinkSO_tx_0_ty_1.101-105.tif - bnc01_TinkSO_tx_1_ty_0.101-105.tif - bnc01_TinkSO_tx_1_ty_1.101-105.tif - file.1-2.tif - file_02.tif - alpha.txt - - Get sequences from a list of file names: - - >>> seqs = iget_sequences(['fileA.1.rgb', 'fileA.2.rgb', 'fileB.1.rgb']) - >>> for s in seqs: print(s) - ... - fileA.1-2.rgb - fileB.1.rgb - - - :param source: Can be directory path, list of strings, or sortable list of objects. - - :return: List of pyseq.Sequence class objects. - """ - start = datetime.now() - if isinstance(source, list): - items = source - elif isinstance(source, str): - if os.path.isdir(source): - join = os.path.join - items = [join(source, x) for x in os.listdir(source)] - else: - items = iglob(source) - else: - raise TypeError("Unsupported format for source argument") - - items = sorted(items, key=_ext_key) - log.debug("Found %d files", len(items)) - - seq = None - while items: - item = Item(items.pop(0)) - if seq is None: - seq = Sequence([item]) - elif seq.includes(item): - seq.append(item) - else: - yield seq - seq = Sequence([item]) - - if seq is not None: - yield seq - log.debug("time: %s", datetime.now() - start) - - -def walk(source, level=-1, topdown=True, onerror=None, followlinks=False, hidden=False): - """Generator that traverses a directory structure starting at - source looking for sequences. - - :param source: valid folder path to traverse - :param level: int, if < 0 traverse entire structure otherwise - traverse to given depth - :param topdown: walk from the top down - :param onerror: callable to handle os.listdir errors - :param followlinks: whether to follow links - :param hidden: include hidden files and dirs - """ - start = datetime.now() - assert isinstance(source, basestring) is True - assert os.path.exists(source) is True - source = os.path.abspath(source) - - for root, dirs, files in os.walk(source, topdown, onerror, followlinks): - - if not hidden: - files = [f for f in files if not f[0] == '.'] - dirs[:] = [d for d in dirs if not d[0] == '.'] - - files = [os.path.join(root, f) for f in files] - - if topdown is True: - parts = root.replace(source, "").split(os.sep) - while "" in parts: - parts.remove("") - if len(parts) == level - 1: - del dirs[:] - - yield root, dirs, get_sequences(files) - - log.debug('time: %s' % (datetime.now() - start)) \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..1ca452d --- /dev/null +++ b/setup.py @@ -0,0 +1,46 @@ +import os + +from codecs import open +from setuptools import setup + +name = 'debayer' +dirname = os.path.dirname(os.path.abspath(__file__)) + +# Get the long description from the README file +with open(os.path.join(dirname, 'README.md'), encoding='utf-8') as f: + long_description = f.read() + +setup( + name=name, + version='0.1.0', + description=r'Command line tool to process camera raw image formats into scene-linear exr and other formats.', + long_description=long_description, + url='https://github.com/jedypod/{}.git'.format(name), + download_url='https://github.com/jedypod/{}/archive/master.zip'.format(name), + license='Proprietary', + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Operating System :: OS Independent', + 'License :: Other/Proprietary License', + 'Private :: Do Not Upload', + ], + packages=[name], + keywords=name, + include_package_data=True, + author='Jedediah Smith', + install_requires=[ + 'pyseq==0.5.1', + 'PyYAML', + ], + entry_points={ + 'console_scripts': [ + 'debayer = debayer:Debayer', + ], + } +)