diff --git a/mrrs/nodes/colmap/Meshroom2ColmapSfmConvertions.py b/mrrs/nodes/colmap/Meshroom2ColmapSfmConvertions.py index 491ab49..59b14d9 100644 --- a/mrrs/nodes/colmap/Meshroom2ColmapSfmConvertions.py +++ b/mrrs/nodes/colmap/Meshroom2ColmapSfmConvertions.py @@ -90,7 +90,7 @@ def processChunk(self, chunk): new_images_path = [os.path.join( images_output_folder, basename) for basename in images_basename] #get if we must resize - do_resize = chunk.node.maxImageSize.value == 0 or image_sizes[0][0]>chunk.node.maxImageSize.value + do_resize = (chunk.node.maxImageSize.value != 0) and (image_sizes[0][0]>chunk.node.maxImageSize.value) #modify .sfm with new sizes and filepath if do_resize: @@ -134,4 +134,4 @@ def processChunk(self, chunk): os.rename(os.path.join(chunk.node.output.value, 'sparse2'), os.path.join(chunk.node.output.value, 'sparse')) #os.rmdir(os.path.join(chunk.node.output.value,'dense')) - \ No newline at end of file + diff --git a/mrrs/nodes/inpainting3d/FilterMasks.py b/mrrs/nodes/inpainting3d/FilterMasks.py new file mode 100755 index 0000000..dfdf4a9 --- /dev/null +++ b/mrrs/nodes/inpainting3d/FilterMasks.py @@ -0,0 +1,300 @@ + +__version__ = "3.0" + +import os + +from meshroom.core import desc +import cv2 +import numpy as np + +SHAPES = {'rect':cv2.MORPH_RECT, 'ellipse':cv2.MORPH_ELLIPSE, 'cross':cv2.MORPH_CROSS} + +#TODO: add params + +def _temporal_filtering(frames, masks, window_size=3, use_of=True, metric=np.median): + """ + Ensure temporal consistency of segmentation masks across video frames using a sliding window and median filtering. + """ + import numpy as np + import cv2 + + if len(frames) != len(masks): + raise ValueError("The number of frames and masks must be the same.") + + n_frames = len(frames) + consistent_masks = [] + + for i in range(n_frames): + start_idx = max(0, i - window_size // 2) + end_idx = min(n_frames, i + window_size // 2 + 1) + + window_masks = [] + for j in range(start_idx, end_idx): + if use_of: + # OF calculation from j to i + prev_gray = cv2.cvtColor(frames[j], cv2.COLOR_BGR2GRAY) + curr_gray = cv2.cvtColor(frames[i], cv2.COLOR_BGR2GRAY) + flow = cv2.calcOpticalFlowFarneback(prev_gray, curr_gray, None, + pyr_scale=0.5, levels=3, winsize=15, + iterations=3, poly_n=5, poly_sigma=1.2, flags=0) + # warp the masks j to i + h, w, _ = masks[j].shape + grid_x, grid_y = np.meshgrid(np.arange(w), np.arange(h)) + map_x = grid_x + flow[..., 0] + map_y = grid_y + flow[..., 1] + warped_mask = cv2.remap(masks[j].astype(np.float32), map_x.astype(np.float32), + map_y.astype(np.float32), interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0) + # threshold needed because of interpolation + warped_mask = (warped_mask > 0.5).astype(np.uint8) + window_masks.append(warped_mask) + else: + window_masks.append(masks[j]) + + # median of the masks in the window = final mask + median_mask = metric(window_masks, axis=0).astype(np.uint8) + consistent_masks.append(median_mask) + + return consistent_masks + +def temporal_filtering_of(frames, masks, window_size=3, dummy=None): + """ + Ensure temporal consistency of segmentation masks across video frames using a sliding window and median filtering. + """ + return _temporal_filtering(frames, masks, window_size=3, use_of=True) + +def temporal_filtering_nof(frames, masks, window_size=3, dummy=None): + """ + Ensure temporal consistency of segmentation masks across video frames using a sliding window and median filtering. + """ + return _temporal_filtering(frames, masks, window_size=3, use_of=False) + +def temporal_filtering_acc(frames, masks, window_size=3, dummy=None): + """ + Ensure temporal consistency of segmentation masks across video frames using a sliding window and accumulative filtering. + """ + return _temporal_filtering(frames, masks, window_size=3, use_of=True, metric=np.min) + + +def guided_filtering(frames, masks, kernel_size=5, dummy=None, epsilon=0.01): + """ + Apply guided filtering to a list of masks and their corresponding RGB frames using OpenCV's guidedFilter. + """ + import cv2 + import numpy as np + + filtered_masks = [] + + for mask, frame in zip(masks, frames): + # guided filter + filtered = cv2.ximgproc.guidedFilter( + guide=frame, + src=mask.astype(np.float32), + radius=radius, + eps=epsilon + ) + filtered_masks.append(filtered) + + return filtered_masks + + +def alpha_mate(frames, masks, base_kernel_size=5, dummy=None): + """ + Compute the alpah mate using trimat compused of the difference if erosion + dilatation + """ + import cv2 + import numpy as np + + filtered_masks = [] + + for mask, frame in zip(masks, frames): + # guided filter + filtered = cv2.ximgproc.FastBilateralSolverFilter( + guide=frame, + src=mask.astype(np.float32), + radius=radius, + eps=epsilon + ) + filtered_masks.append(filtered) + + return filtered_masks + + +def dilation(frames, masks, kernel_size=5, shape='rect', iterations=1): + """ + Apply dilation to a list of masks. + """ + import cv2 + import numpy as np + kernel = cv2.getStructuringElement(SHAPES[shape], (kernel_size, kernel_size)) + dilated_masks = [cv2.dilate(mask, kernel, iterations=iterations) for mask in masks] + return dilated_masks + +def erosion(frames, masks, kernel_size=5, shape='rect', iterations=1): + """ + Apply erosion to a list of masks. + """ + import cv2 + import numpy as np + + kernel = cv2.getStructuringElement(SHAPES[shape], (kernel_size, kernel_size)) + dilated_masks = [cv2.erode(mask, kernel, iterations=iterations) for mask in masks] + return dilated_masks + +def opening(frames, masks, kernel_size=5, shape='rect', iterations=1): + """ + Apply opening to a list of masks. + """ + import cv2 + import numpy as np + + kernel = cv2.getStructuringElement(SHAPES[shape], (kernel_size, kernel_size)) + dilated_masks = [cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel) for mask in masks] + return dilated_masks + +def closing(frames, masks, kernel_size=5, shape='rect', iterations=1): + """ + Apply closing to a list of masks. + """ + import cv2 + import numpy as np + + kernel = cv2.getStructuringElement(SHAPES[shape], (kernel_size, kernel_size)) + dilated_masks = [cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel) for mask in masks] + return dilated_masks + +def blur(frames, masks, kernel_size=5, shape=None): + ''' + Blur a list of masks. + ''' + import cv2 + import numpy as np + blurred_masks = [cv2.blur(mask, (kernel_size, kernel_size)) for mask in masks] + return blurred_masks + +def invertion(frames, masks, dummy, dummy2): + ''' + Invert a list of masks. + ''' + return [1-mask for mask in masks] + +def segmentation_refinement(frames, masks, window_size=3, dummy=None): + """ + """ + pass + +class FilterMasks(desc.Node): + + category = 'Inpainting 3D' + documentation = '''''' + + inputs = [ + desc.File( + name='inputSfM', + label='SfMData', + description='SfMData file.', + value='', + ), + + desc.File( + name="maskFolder", + label="Mask Folder", + description="maskFolder", + value="", + ), + + desc.ChoiceParam( + name='filterType', + label='Filter Type', + description=''' ''', + value='segmentation_refinement', + values=['guided_filtering', 'temporal_filtering_nof', 'temporal_filtering_of', 'temporal_filtering_acc', 'dilation', 'erosion', 'closing', 'opening', 'invertion', 'blur', 'segmentation_refinement'], + exclusive=True, + ), + + desc.ChoiceParam( + name='kernelShape', + label='Filter shape', + description='For spatial filtering, the shape of the kernel', + value='rect', + values=['rect', 'ellipse', 'cross'], + exclusive=True, + ), + + desc.IntParam( + name="kernelSize", + label="Filter size", + description="For temporal foltering, the size of the temporal window, for the spatial filtering, the size of the kernel", + value=3, + range=(3, 100000, 1) + ), + + desc.ChoiceParam( + name='verboseLevel', + label='Verbose Level', + description='''verbosity level (fatal, error, warning, info, debug, trace).''', + value='info', + values=['fatal', 'error', 'warning', 'info', 'debug', 'trace'], + exclusive=True, + ), + + ] + + outputs = [ + desc.File( + name='outputFolder', + label='outputFolder', + description='outputFolder', + value=desc.Node.internalFolder, + group='', + ), + desc.File( + name="masks", + label="Masks", + description="Generated segmentation masks.", + semantic="image", + value=lambda attr: desc.Node.internalFolder + ".jpg", + group="", + visible=False + ), + ] + + def processChunk(self, chunk): + """ + """ + + import json + import numpy as np + import cv2 + from mrrs.core.ios import open_image, save_image + + chunk.logManager.start(chunk.node.verboseLevel.value) + if chunk.node.inputSfM.value == '': + raise RuntimeError("No inputSfM specified") + if chunk.node.maskFolder.value == '': + raise RuntimeError("No maskFolder specified") + chunk.logger.info("Loading masks") + sfm_data=json.load(open(chunk.node.inputSfM.value,"r")) + #temporal sort + sfm_data["views"]=sorted(sfm_data["views"], key=lambda v:int(v["frameId"])) + images=[] + masks=[] + for view in sfm_data["views"]: + # image_basename = view["viewId"] + image_basename = os.path.basename(view["path"])[:-4] + mask_file = os.path.join(chunk.node.maskFolder.value, image_basename+".exr") + if not os.path.exists(mask_file): + mask_file = os.path.join(chunk.node.maskFolder.value, image_basename+".jpg") + images.append(open_image(view["path"], auto_rotate=True)) + masks.append(open_image(mask_file)/255.0) + + chunk.logger.info("Applying filter") + filter_function = eval(chunk.node.filterType.value) + filtered_masks = filter_function(images, masks, chunk.node.kernelSize.value, chunk.node.kernelShape.value) + + chunk.logger.info("Saving masks") + for view, mask in zip(sfm_data["views"], filtered_masks): + # image_basename = view["viewId"] + image_basename = os.path.basename(view["path"])[:-4] + save_image(chunk.node.outputFolder.value+"/"+image_basename+".jpg",(255*mask).astype(np.uint8)) + chunk.logManager.end() + diff --git a/mrrs/nodes/inpainting3d/FilterSfm.py b/mrrs/nodes/inpainting3d/FilterSfm.py new file mode 100755 index 0000000..fdf0abc --- /dev/null +++ b/mrrs/nodes/inpainting3d/FilterSfm.py @@ -0,0 +1,63 @@ +__version__ = "1.0" + +from meshroom.core import desc +from meshroom.core.utils import VERBOSE_LEVEL + +import json +import os + + +class FilterSfm(desc.Node): + #size = desc.DynamicNodeSize('inputFiles') + + category = 'Inpainting 3D' + documentation = ''' +TODO +''' + + inputs = [ + desc.File( + name="sfmData", + label="SfmData", + description="TODO", + value="", + exposed=True, + ), + desc.IntParam( + name="intrinsicId", + label="Intrinsic ID", + description="TODO", + value=0, + exposed=True, + ) + ] + + outputs = [ + desc.File( + name="filteredSfmData", + label="Filtered Sfm Data", + description="TODO", + value=os.path.join(desc.Node.internalFolder, 'sfm.sfm'), + ), + ] + + def processChunk(self, chunk): + try: + #chunk.logManager.start(chunk.node.verboseLevel.value) + + sfmData = json.load(open(chunk.node.sfmData.value, 'r')) + intrinsicID = chunk.node.intrinsicId.value + + views = [] + for view in sfmData["views"]: + if int(view["intrinsicId"]) == intrinsicID: + views.append(view) + + output = sfmData + output["views"] = views + + with open(chunk.node.filteredSfmData.value, 'w') as f: + json.dump(output, f, indent=4) + + finally: + chunk.logManager.end() diff --git a/mrrs/nodes/inpainting3d/GaussianSplattingOptim.py b/mrrs/nodes/inpainting3d/GaussianSplattingOptim.py new file mode 100755 index 0000000..c6ef344 --- /dev/null +++ b/mrrs/nodes/inpainting3d/GaussianSplattingOptim.py @@ -0,0 +1,136 @@ +__version__ = "1.0" + +from meshroom.core import desc +import os.path + +currentDir = os.path.dirname(os.path.abspath(__file__)) + +class GaussianSplattingOptim(desc.CommandLineNode): + + commandLine = 'rez env {rezEnvNameValue} -- gaussianSplattingOptim default --data_factor {dataFactorValue} --test_every 0 --max_steps {n_stepsValue} --data_dir {sfmValue} --result_dir {cache}/{nodeType}/{uid} --disable_viewer --eval_steps' + gpu = desc.Level.INTENSIVE + cpu = desc.Level.NORMAL + ram = desc.Level.INTENSIVE + + # size = desc.DynamicNodeSize('sfm') + # parallelization = desc.Parallelization(blockSize=40) + # commandLineRange = '--rangeStart {rangeStart} --rangeSize {rangeBlockSize}' + + category = 'Inpainting 3D' + documentation = ''' + This node creates and optimizes a gaussian splatting model based on sfm data and images. +''' + + def buildCommandLine(self, chunk): + cmdLine = super(GaussianSplattingOptim, self).buildCommandLine(chunk) # ou juste super().buildCommandLine(chunk) + node = chunk.node + if node.use_masks.value: + cmdLine = cmdLine + " --use_masks" + return cmdLine + " --save_steps " + (str(node.n_steps.value) if not node.custom_ckpts.value else " ".join([str(e.value) for e in node.save_steps.value])) + + inputs = [ + desc.File( + name="rezEnvName", + label="Rez package name", + description="Name (with path if necessary) of the rez package into which the computation should be executed.", + value="gsplat-maskSupport", + invalidate=False, + group="", + advanced=True, + exposed=False, + ), + desc.File( + name="sfm", + label="SfmDataFolder", + description="SfMData with the views, poses and intrinsics to use (in JSON format). Downscaled versions of the views should also be provided.", + value="", + group="", + ), + desc.IntParam( + name="n_steps", + label="Number of Steps", + description="The number of steps performed by the optimization.", + value=30000, + group="", + ), + desc.IntParam( + name="dataFactor", + label="Data Factor", + description="The height of the pyramid of subimages.", + value=4, + group="", + ), + desc.BoolParam( + name="custom_ckpts", + label="Use custom checkpoints", + description="Set to True to provide custom checkpoints; default behavior is to save the model once, at the end of the optimization.", + value=False, + group="", + ), + desc.ListAttribute( + desc.IntParam( + name="save_step", + label="Saving step", + description="Step at which the model will be saved as checkpoint.", + value=30000, + group="", + ), + name="save_steps", + label="Saving steps", + description="All the steps at which the model will be saved as a check point.", + group="", + # TODO add enabled = + ), + desc.BoolParam( + name="use_masks", + label="Use Masks", + description="Whether segmentation masks will be used to remove certain parts of the image from the optimization, if available.", + value=True, + group="", + ), + # desc.BoolParam( + # name='use_masks', + # label="_", + # exposed=False, + # enabled = lambda node: node.useMasks.value, + # ), + desc.ListAttribute( + elementDesc=desc.IntParam( + name="input", + label="Input", + description="", + value=0, + #invalidate=False, + ), + name="stalling", + label="Stalling", + description="This input will not be processed. Use it to stall the node until some preliminary computation is done.", + group="", + exposed=False, + ) + ] + + # def onCustom_ckptsChanged(self, node): + # node.save_steps_hidden.value = node.custom_ckpts.value + + outputs = [ + desc.File( + name="output", + label="Output", + description="Output folder.", + value=desc.Node.internalFolder, + ), + desc.File( + name="model", + label="Model", + description="Optimized gaussian splatting model", + value = lambda attr: desc.Node.internalFolder + f"/ckpts/ckpt_{attr.node.n_steps.value-1}_rank0.pt", + group="", + ), + # desc.File( + # name="save_steps_hidden", + # label="Hidden", + # description="truc", + # value=lambda node: str(node.n_steps.value) if not node.custom_ckpts.value else " ".join([e.value for e in node.save_steps.value]) + # ), + ] diff --git a/mrrs/nodes/inpainting3d/GaussianSplattingRender.py b/mrrs/nodes/inpainting3d/GaussianSplattingRender.py new file mode 100755 index 0000000..91d616c --- /dev/null +++ b/mrrs/nodes/inpainting3d/GaussianSplattingRender.py @@ -0,0 +1,96 @@ +__version__ = "1.0" + +from meshroom.core import desc +import os.path + +currentDir = os.path.dirname(os.path.abspath(__file__)) + +class GaussianSplattingRender(desc.CommandLineNode): + + commandLine = 'rez env {rezEnvNameValue} -- gaussianSplattingOptim default --ckpt {modelValue} --data_factor {resolutionFactorValue} --test_every 1 --data_dir {camerasValue} --result_dir {cache}/{nodeType}/{uid} --disable_viewer' + + gpu = desc.Level.INTENSIVE + cpu = desc.Level.NORMAL + ram = desc.Level.INTENSIVE + + # size = desc.DynamicNodeSize('cameras') + # parallelization = desc.Parallelization(blockSize=40) + # commandLineRange = '--rangeStart {rangeStart} --rangeSize {rangeBlockSize}' + + category = 'Inpainting 3D' + documentation = ''' + This node computes the rasterization of a given gaussian splatting model from given viewpoints. +''' + + inputs = [ + desc.File( + name="rezEnvName", + label="Rez package name", + description="Name (with path if necessary) of the rez package into which the computation should be executed.", + value="gsplat-maskSupport", + invalidate=False, + group="", + advanced=True, + exposed=False, + ), + desc.File( + name="cameras", + label="Cameras", + description="SfMData with the views, poses and intrinsics to use (in JSON format).", + value="", + ), + desc.File( + name="model", + label="Model", + description="Gaussian splats (.ckpt) to render.", + value="", + group="", + ), + desc.IntParam( + name="resolutionFactor", + label="Subsampling factor", + description="How low in the resolution pyramid should the rendering be done. Ex: a value of 4 -> 16x less pixels", + value=1, + group="", + exposed=False, + ), + desc.ListAttribute( + elementDesc=desc.IntParam( + name="input", + label="Input", + description="", + value=0, + invalidate=False, + ), + name="stalling", + label="Stalling", + description="This input will not be processed. Use it to stall the node until some preliminary computation is done.", + group="", + exposed=False, + ), + ] + + outputs = [ + desc.File( + name="output", + label="Output", + description="Output folder.", + value=desc.Node.internalFolder, + ), + desc.File( + name="frames", + label="Frames", + description="Frames rendered using gaussian splatting.", + semantic="sequence", #"image", # use "image" when logic, and "sequence" when *.jpg logic + # semantic="sequence", + value=os.path.join(desc.Node.internalFolder,'renders', '*.jpg'), #/renders/_.JPG', + group="", + ), + desc.File( + name="render_folder", + label="Render Folder", + description="Output folder.", + value=os.path.join(desc.Node.internalFolder, 'renders') + ), + + ] diff --git a/mrrs/nodes/inpainting3d/ImageCompo.py b/mrrs/nodes/inpainting3d/ImageCompo.py new file mode 100755 index 0000000..58ed04c --- /dev/null +++ b/mrrs/nodes/inpainting3d/ImageCompo.py @@ -0,0 +1,79 @@ +__version__ = "1.0" + +from meshroom.core import desc +from meshroom.core.utils import VERBOSE_LEVEL + + +import OpenImageIO as oiio +import os + + +class ImageCompo(desc.Node): + #size = desc.DynamicNodeSize('inputFiles') + + category = 'Inpainting 3D' + documentation = ''' +This node allows to perform basic alpha compositing. +''' + + inputs = [ + desc.File( + name="folderA", + label="Folder A", + description="Foreground images folder.", + value="", + ), + desc.File( + name="folderB", + label="Folder B", + description="Background images folder.", + value="", + ), + desc.File( + name="folderMask", + label="Alpha masks folder", + description="The alpha masks used to combine the images.", + value="", + ), + ] + + outputs = [ + desc.File( + name="combinedImageFolder", + label="Combined images folder", + description="The output images are mixes between input images.", + value=desc.Node.internalFolder, + ), + ] + + def processChunk(self, chunk): + try: + + im1_names = [e for e in os.listdir(chunk.node.folderA.value) if len(e)>=5 and e[-4:]==".jpg"] + im2_names = [e for e in os.listdir(chunk.node.folderB.value) if len(e)>=5 and e[-4:]==".jpg"] + masks_names = [e for e in os.listdir(chunk.node.folderMask.value) if len(e)>=5 and e[-4:]==".jpg"] + + assert len(im1_names)==len(im2_names), f"Number of images is different in two folders: {len(im1_names)} VS {len(im2_names)}" + assert len(im1_names)==len(masks_names), f"Number of images is different from number of masks: {len(im1_names)} VS {len(masks_names)}" + + for a,b in zip(im1_names, im2_names): + im1 = oiio.ImageBuf(os.path.join(chunk.node.folderA.value, a)).get_pixels() + im2 = oiio.ImageBuf(os.path.join(chunk.node.folderB.value, b)).get_pixels() + c = [e for e in masks_names if a in e or e in a] + #print(masks_names) + #print(a) + #print(c) + assert len(c)==1, f"error, len(c)={len(c)}" + c = c[0] + mask = oiio.ImageBuf(os.path.join(chunk.node.folderMask.value, c)).get_pixels() + + res = im2*(1-mask) + im1*mask + + out = oiio.ImageOutput.create(os.path.join(chunk.node.combinedImageFolder.value, a)) + spec = oiio.ImageSpec(res.shape[1], res.shape[0], res.shape[2], oiio.UINT8) + out.open(os.path.join(chunk.node.combinedImageFolder.value, a), spec) + out.write_image(res) + out.close() + + finally: + chunk.logManager.end() diff --git a/mrrs/nodes/inpainting3d/InterpolatePoses.py b/mrrs/nodes/inpainting3d/InterpolatePoses.py new file mode 100755 index 0000000..6776ab3 --- /dev/null +++ b/mrrs/nodes/inpainting3d/InterpolatePoses.py @@ -0,0 +1,124 @@ +__version__ = "3.0" + +import os +import json + +from meshroom.core import desc + +class InterpolatePoses(desc.Node): + + category = 'Inpainting 3D' + documentation = '''''' + + inputs = [ + desc.File( + name='inputSfM', + label='SfMData', + description='SfMData file.', + value='', + ), + + desc.ChoiceParam( + name='verboseLevel', + label='Verbose Level', + description='''Verbosity level (fatal, error, warning, info, debug, trace).''', + value='info', + values=['fatal', 'error', 'warning', 'info', 'debug', 'trace'], + exclusive=True, + ), + + ] + + outputs = [ + desc.File( + name='outputSfMData', + label='Output', + description='Output SfM data.', + value=os.path.join(desc.Node.internalFolder, "sfm.sfm"), + ) + ] + + def processChunk(self, chunk): + """ + Computes the different transforms + """ + + import numpy as np + import cv2 + from mrrs.core.ios import sfm_data_from_matrices, matrices_from_sfm_data + + chunk.logManager.start(chunk.node.verboseLevel.value) + if chunk.node.inputSfM.value == '': + raise RuntimeError("No inputSfM specified") + + #load .sfm data + sfm_data=json.load(open(chunk.node.inputSfM.value,"r")) + #sort by frameId + sfm_data["views"]=sorted(sfm_data["views"], key=lambda v:int(v["frameId"])) + #load poses into matrices + extrinsics, intrinsics, views_id, poses_ids, intrinsics_ids, pixel_sizes_all_cams, images_size = matrices_from_sfm_data(sfm_data, True) + sensor_width = pixel_sizes_all_cams[0]*images_size[0,0] + + # + print("Saving") + #intrinsics in pixels for export + for i in range(len(intrinsics)): + if intrinsics[i] is not None: + intrinsics[i]/= pixel_sizes_all_cams[i] + intrinsics[i][2,2]=1 + sfm_data_out = sfm_data_from_matrices(extrinsics, intrinsics, poses_ids, intrinsics_ids, images_size, + sfm_data = sfm_data, sensor_width = sensor_width) + with open(os.path.join(chunk.node.outputSfMData.value), 'w') as f: + json.dump(sfm_data_out, f, indent=4) + + + # #tmp + # extrinsics[4]=None + + # # trasnsform into so3 + # print("transforming") + # rot_so3=[] + # missing=0 + # for e in extrinsics: + # rotVec = None + # if e is not None: + # rotVec,_=cv2.Rodrigues(e[0:3,0:3]) + # else: + # missing+=1 + # rot_so3.append(rotVec) + + # print("%d poses missing"%missing) + + # #interpolation + # print("Intepolation") + # extrinsics_intrp = extrinsics.copy() + # prev_valid = -1 + # for i,e in enumerate(extrinsics): + # if e is None and prev_valid!=-1: + # next_valid = [j for j in range(i, len(extrinsics)) if extrinsics[j] is not None][0] + # print("interpolatin view %d from %d and %d"%(i,prev_valid,next_valid)) + # print(sfm_data["views"][i]) + # print(sfm_data["views"][prev_valid]) + # print(sfm_data["views"][next_valid]) + # w = (i-prev_valid)/(next_valid-prev_valid) + # r_interp, _ = cv2.Rodrigues((1-w)*rot_so3[prev_valid]+w*rot_so3[next_valid]) + # t_interp = (1-w)*extrinsics[prev_valid][:,3]+w*extrinsics[next_valid][:,3] + # e_interp=np.concatenate([r_interp, np.expand_dims(t_interp, axis=1) ], axis=1) + # extrinsics_intrp[i] = e_interp + # else: + # prev_valid = i + + + # #write result + # print("Saving") + # #intrinsics in pixels for export + # for i in range(len(intrinsics)): + # if intrinsics[i] is not None: + # intrinsics[i]/= pixel_sizes_all_cams[i] + # intrinsics[i][2,2]=1 + # sfm_data_out = sfm_data_from_matrices(extrinsics_intrp, intrinsics, poses_ids, intrinsics_ids, images_size, + # sfm_data = sfm_data, sensor_width = sensor_width) + # with open(os.path.join(chunk.node.outputSfMData.value), 'w') as f: + # json.dump(sfm_data_out, f, indent=4) + chunk.logManager.end() + diff --git a/mrrs/nodes/inpainting3d/PrepareFolderForGSplat.py b/mrrs/nodes/inpainting3d/PrepareFolderForGSplat.py new file mode 100755 index 0000000..d062663 --- /dev/null +++ b/mrrs/nodes/inpainting3d/PrepareFolderForGSplat.py @@ -0,0 +1,107 @@ + +__version__ = "3.0" + +import os + +from meshroom.core import desc + +class PrepareFolderForGS(desc.Node): + + category = 'Inpainting 3D' + documentation = '''''' + + inputs = [ + desc.File( + name='colmapFolder', + label='colmapFolder', + description='', + value='', + ), + + desc.File( + name="maskFolder", + label="Mask Folder", + description="maskFolder", + value="", + ), + + desc.ChoiceParam( + name='scale', + label='scale', + description=''' ''', + value=['1', '2', '4', '8'], + values=['1', '2', '4', '8', '16', '32', '64', '128', '256'], + exclusive=False, + ), + + desc.ChoiceParam( + name='verboseLevel', + label='Verbose Level', + description='''verbosity level (fatal, error, warning, info, debug, trace).''', + value='info', + values=['fatal', 'error', 'warning', 'info', 'debug', 'trace'], + exclusive=True, + ), + + ] + + outputs = [ + desc.File( + name='outputFolder', + label='outputFolder', + description='outputFolder', + value=desc.Node.internalFolder, + group='', + ), + desc.File( + name='outputImageFolder', + label='outputImageFolder', + description='outputImageFolder', + value=os.path.join(desc.Node.internalFolder, "images"), + group='', + ) + ] + + def processChunk(self, chunk): + """ + image_ + mask_ + in output folder + """ + + import json + import numpy as np + from shutil import copytree + import cv2 + from mrrs.core.ios import open_image, save_image + + chunk.logManager.start(chunk.node.verboseLevel.value) + + #copy colmap folder + copytree(chunk.node.colmapFolder.value, chunk.node.outputFolder.value, dirs_exist_ok=True) + + #resize images and images + def save_downsampled(inputfolder, name): + chunk.logger.info("Loading...") + image_files = [f + for f in os.listdir(inputfolder) if f.endswith(".exr") or f.endswith(".jpg")] + images = [] + for image_file in image_files: + image_file=os.path.join(inputfolder,image_file) + if not os.path.exists(image_file): + chunk.logger.info("Issue with "+image_file+", skipping") + images.append(open_image(image_file)/255.0) + #for each scale, save in mask_ + for scale in chunk.node.scale.value: + chunk.logger.info("Resizing at "+scale) + images_resized = [cv2.resize(m, (0,0), fx=1/float(scale), fy=1/float(scale)) for m in images] + chunk.logger.info("Saving images") + dirname = name+"_"+scale if int(scale)!=1 else name + os.makedirs(os.path.join(chunk.node.outputFolder.value, dirname), exist_ok=True) + for image_file, image in zip(image_files, images_resized): + save_image(os.path.join(chunk.node.outputFolder.value, dirname, image_file[:-4]+".jpg"),(255*image).astype(np.uint8)) + + save_downsampled(chunk.node.maskFolder.value, "masks") + save_downsampled(os.path.join(chunk.node.colmapFolder.value, "images"), "images") + chunk.logManager.end() + diff --git a/mrrs/nodes/inpainting3d/__init__.py b/mrrs/nodes/inpainting3d/__init__.py new file mode 100755 index 0000000..e69de29