From 8ed3b2e6d49ba129e743667643f1b2f4f487b739 Mon Sep 17 00:00:00 2001 From: Thomas Barlow Date: Wed, 22 Jun 2022 02:20:47 +0100 Subject: [PATCH 1/2] Fix error on Blender 2.80 and older where bpy.ops.object.material_slot_remove_unused does not exist Fix running combine_materials on all meshes for every mesh Fix add_principled_shader making duplicate mmd materials be considered non-duplicate Replace EDIT mode selection and assignment of materials with OBJECT mode foreach_get/set for performance Add material hashes for 2.80+ materials that don't use nodes Fix ordering and remove unnecessary setting of active_material_index in common.clean_material_names Fix multiple cases of code not checking if a material_slot is empty --- tools/armature.py | 44 ++++-- tools/common.py | 14 +- tools/importer.py | 19 +-- tools/material.py | 379 ++++++++++++++++++++++------------------------ 4 files changed, 224 insertions(+), 232 deletions(-) diff --git a/tools/armature.py b/tools/armature.py index 61d86631..103ebd58 100644 --- a/tools/armature.py +++ b/tools/armature.py @@ -424,20 +424,24 @@ def execute(self, context): Common.sort_shape_keys(mesh.name, shapekey_order) - - # Clean material names. Combining mats would do this too - Common.clean_material_names(mesh) + if not context.scene.combine_mats: + # Clean material names. Combining mats would do this too + Common.clean_material_names(mesh) # If all materials are transparent, make them visible. Also set transparency always to Z-Transparency if version_2_79_or_older(): all_transparent = True for mat_slot in mesh.material_slots: - mat_slot.material.transparency_method = 'Z_TRANSPARENCY' - if mat_slot.material.alpha > 0: - all_transparent = False + mat = mat_slot.material + if mat: + mat.transparency_method = 'Z_TRANSPARENCY' + if mat.alpha > 0: + all_transparent = False if all_transparent: for mat_slot in mesh.material_slots: - mat_slot.material.alpha = 1 + mat = mat_slot.material + if mat: + mat.alpha = 1 else: if context.scene.fix_materials: # Make materials exportable in Blender 2.80 and remove glossy mmd shader look @@ -445,20 +449,15 @@ def execute(self, context): if mmd_tools_installed: Common.fix_mmd_shader(mesh) Common.fix_vrm_shader(mesh) - Common.add_principled_shader(mesh) for mat_slot in mesh.material_slots: # Fix transparency per polygon and general garbage look in blender. Asthetic purposes to fix user complaints. - mat_slot.material.shadow_method = "HASHED" - mat_slot.material.blend_method = "HASHED" + mat = mat_slot.material + mat.shadow_method = "HASHED" + mat.blend_method = "HASHED" - # Remove empty shape keys and then save the shape key order + # Remove empty shape keys and then save the shape key order Common.clean_shapekeys(mesh) Common.save_shapekey_order(mesh.name) - # Combines same materials - if context.scene.combine_mats: - bpy.ops.cats_material.combine_mats() - - # Reorders vrc shape keys to the correct order Common.sort_shape_keys(mesh.name) @@ -478,6 +477,19 @@ def execute(self, context): uv.data[vert].uv.y = 0 fixed_uv_coords += 1 + # Combines same materials + # combine_mats runs on all meshes in Common.get_meshes_objects() and gathers material hashes of all the + # materials of those meshes before combining, so it must be run after we have fixed all the materials. + if context.scene.combine_mats: + bpy.ops.cats_material.combine_mats() + + # Adding principled shader to materials must happen after same materials have been combined, otherwise + # mmd_shader materials that are considered duplicates will have their same diffuse and ambient colors baked + # to different images, making the materials no longer be considered duplicates. + if not version_2_79_or_older() and context.scene.fix_materials: + for mesh in meshes: + Common.add_principled_shader(mesh) + # Translate bones and unhide them all to_translate = [] for bone in armature.data.bones: diff --git a/tools/common.py b/tools/common.py index 4f939f35..3adac05c 100644 --- a/tools/common.py +++ b/tools/common.py @@ -1769,13 +1769,13 @@ def get_bone_orientations(armature): def clean_material_names(mesh): - for j, mat in enumerate(mesh.material_slots): - if mat.name.endswith('.001'): - mesh.active_material_index = j - mesh.active_material.name = mat.name[:-4] - if mat.name.endswith(('. 001', ' .001')): - mesh.active_material_index = j - mesh.active_material.name = mat.name[:-5] + for mat_slot in mesh.material_slots: + mat = mat_slot.material + if mat: + if mat.name.endswith(('. 001', ' .001')): + mat.name = mat.name[:-5] + elif mat.name.endswith('.001'): + mat.name = mat.name[:-4] def mix_weights(mesh, vg_from, vg_to, mix_strength=1.0, mix_mode='ADD', delete_old_vg=True): diff --git a/tools/importer.py b/tools/importer.py index 2270ddf7..644b2a3d 100644 --- a/tools/importer.py +++ b/tools/importer.py @@ -796,15 +796,16 @@ def execute(self, context): objname = obj.name if bpy.data.objects[objname].type == "MESH": print("lowercasing material name for gmod for object "+objname) - for material in bpy.data.objects[objname].material_slots: - mat = material.material - sanitized_material_name = "" - for i in mat.name.lower(): - if i.isalnum() or i == "_": - sanitized_material_name += i - else: - sanitized_material_name += "_" - mat.name = sanitized_material_name + for mat_slot in bpy.data.objects[objname].material_slots: + mat = mat_slot.material + if mat: + sanitized_material_name = "" + for i in mat.name.lower(): + if i.isalnum() or i == "_": + sanitized_material_name += i + else: + sanitized_material_name += "_" + mat.name = sanitized_material_name print("zeroing transforms and then scaling to gmod scale, then applying transforms.") diff --git a/tools/material.py b/tools/material.py index 77e968e7..93391edd 100644 --- a/tools/material.py +++ b/tools/material.py @@ -29,6 +29,7 @@ import os import bpy +import numpy as np from . import common as Common from .register import register_wrap @@ -63,9 +64,11 @@ def execute(self, context): for mesh in Common.get_meshes_objects(): for mat_slot in mesh.material_slots: - for i, tex_slot in enumerate(mat_slot.material.texture_slots): - if i > 0 and tex_slot: - mat_slot.material.use_textures[i] = False + mat = mat_slot.material + if mat: + for i, tex_slot in enumerate(mat.texture_slots): + if i > 0 and tex_slot: + mat.use_textures[i] = False saved_data.load() @@ -98,9 +101,11 @@ def execute(self, context): for mesh in Common.get_meshes_objects(): for mat_slot in mesh.material_slots: - for i, tex_slot in enumerate(mat_slot.material.texture_slots): - if i > 0 and tex_slot: - tex_slot.texture = None + mat = mat_slot.material + if mat: + for i, tex_slot in enumerate(mat.texture_slots): + if i > 0 and tex_slot: + tex_slot.texture = None saved_data.load() @@ -133,15 +138,16 @@ def execute(self, context): for mesh in Common.get_meshes_objects(): for mat_slot in mesh.material_slots: + mat = mat_slot.material + if mat: + mat.transparency_method = 'Z_TRANSPARENCY' + mat.alpha = 1 - mat_slot.material.transparency_method = 'Z_TRANSPARENCY' - mat_slot.material.alpha = 1 - - for tex_slot in mat_slot.material.texture_slots: - if tex_slot: - tex_slot.use_map_alpha = True - tex_slot.use_map_color_diffuse = True - tex_slot.blend_type = 'MULTIPLY' + for tex_slot in mat.texture_slots: + if tex_slot: + tex_slot.use_map_alpha = True + tex_slot.use_map_color_diffuse = True + tex_slot.blend_type = 'MULTIPLY' saved_data.load() @@ -156,211 +162,184 @@ class CombineMaterialsButton(bpy.types.Operator): bl_description = t('CombineMaterialsButton.desc') bl_options = {'REGISTER', 'UNDO', 'INTERNAL'} - combined_tex = {} - @classmethod def poll(cls, context): if Common.get_armature() is None: return False return len(Common.get_meshes_objects(check=False)) > 0 - def assignmatslots(self, ob, matlist): - scn = bpy.context.scene - ob_active = Common.get_active() - Common.set_active(ob) - - for s in ob.material_slots: - bpy.ops.object.material_slot_remove() - - i = 0 - for m in matlist: - mat = bpy.data.materials[m] - ob.data.materials.append(mat) - i += 1 - - Common.set_active(ob_active) - - def cleanmatslots(self): - objs = bpy.context.selected_editable_objects - - for ob in objs: - if ob.type == 'MESH': - Common.set_active(ob) - bpy.ops.object.material_slot_remove_unused() - - #Why do all below when you can just do what is above? - @989onan - - -# mats = ob.material_slots.keys() - -# usedMatIndex = [] -# faceMats = [] -# me = ob.data -# for f in me.polygons: -# faceindex = f.material_index -# if faceindex >= len(mats): -# continue - -# currentfacemat = mats[faceindex] -# faceMats.append(currentfacemat) - -# found = False -# for m in usedMatIndex: -# if m == faceindex: -# found = True - -# if found == False: -# usedMatIndex.append(faceindex) - -# ml = [] -# mnames = [] -# for u in usedMatIndex: -# ml.append(mats[u]) -# mnames.append(mats[u]) - -# self.assignmatslots(ob, ml) - -# i = 0 -# for f in me.polygons: -# if i >= len(faceMats): -# continue -# matindex = mnames.index(faceMats[i]) -# f.material_index = matindex -# i += 1 - - # Iterates over each material slot and hashes combined image filepaths and material settings - # Then uses this hash as the dict keys and material data as values - def generate_combined_tex(self): - self.combined_tex = {} - for ob in Common.get_meshes_objects(): - for index, mat_slot in enumerate(ob.material_slots): - hash_this = '' - - if Common.version_2_79_or_older(): - if mat_slot.material: - for tex_index, mtex_slot in enumerate(mat_slot.material.texture_slots): - if mtex_slot: - if mat_slot.material.use_textures[tex_index]: - if hasattr(mtex_slot.texture, 'image') and bpy.data.materials[mat_slot.name].use_textures[tex_index] and mtex_slot.texture.image: - hash_this += mtex_slot.texture.image.filepath # Filepaths makes the hash unique - hash_this += str(mat_slot.material.alpha) # Alpha setting on material makes the hash unique - hash_this += str(mat_slot.material.diffuse_color) # Diffuse color makes the hash unique - # hash_this += str(mat_slot.material.specular_color) # Specular color makes the hash unique # Specular Color is no used by Unity - - # print('---------------------------------------------------') - # print(mat_slot.name, hash_this) - - # Now create or add to the dict key that has this hash value - if hash_this not in self.combined_tex: - self.combined_tex[hash_this] = [] - self.combined_tex[hash_this].append({'mat': mat_slot.name, 'index': index}) - - else: - hash_this = '' - ignore_nodes = ['Material Output', 'mmd_tex_uv', 'Cats Export Shader'] - - if mat_slot.material and mat_slot.material.node_tree: - # print('MAT: ', mat_slot.material.name) - nodes = mat_slot.material.node_tree.nodes - for node in nodes: - - # Skip certain known nodes - if node.name in ignore_nodes or node.label in ignore_nodes: - continue - - # Add images to hash and skip toon and shpere textures - if node.type == 'TEX_IMAGE': - image = node.image - if 'toon' in node.name or 'sphere' in node.name: - nodes.remove(node) - continue - if not image: - nodes.remove(node) - continue - # print(' ', node.name) - # print(' ', image.name) - hash_this += node.name + image.name - continue - # Skip nodes with no input - if not node.inputs: + @staticmethod + def combine_exact_duplicate_mats(ob, unique_sorted_mat_indices): + mat_names = ob.material_slots.keys() + + # Find duplicate materials and get the first index of the material slot with that same material + mat_first_occurrence = {} + # Note that empty material slots use '' as the material name + for i, mat_name in enumerate(mat_names): + if mat_name not in mat_first_occurrence: + # This is the first time we've seen this material, add it to the first occurrence dict with the current + # index + mat_first_occurrence[mat_name] = i + else: + # We've seen this material already, find its occurrences (if any) in the unique mat indices array and + # set it to the index of the first occurrence of this material + unique_sorted_mat_indices[unique_sorted_mat_indices == i] = mat_first_occurrence[mat_name] + + return unique_sorted_mat_indices + + @staticmethod + def remove_unused_mat_slots(ob, used_mat_indices): + # Remove unused material slots + # material_slot_remove_unused was added in 2.81 + # Unfortunately, material_slot_remove_unused completely ignores context overrides as of Blender 2.91, instead + # getting the object(s) to operate on directly from the context's view_layer, otherwise we would use it in + # Blender 2.91 and newer too. + if (2, 81) <= bpy.app.version < (2, 91): + context_override = {'active_object': ob} + bpy.ops.object.material_slot_remove_unused(context_override) + else: + # Context override so that we don't need to set the object as the active object to run the operator on it + context_override = {'object': ob} + # Convert to a set to remove any duplicates and for quick checking of whether a material index is used + used_mat_indices = set(used_mat_indices) + # Iterate through the material slots, removing any which are not used + # We iterate in reverse order, so that removing a material slot doesn't change the indices of any material + # slots we are yet to iterate + for i in reversed(range(len(ob.material_slots))): + if i not in used_mat_indices: + ob.active_material_index = i + bpy.ops.object.material_slot_remove(context_override) + + @staticmethod + def generate_mat_hash(mat): + hash_this = '' + if mat: + if Common.version_2_79_or_older(): + for tex_index, mtex_slot in enumerate(mat.texture_slots): + if mtex_slot: + if mat.use_textures[tex_index]: + if hasattr(mtex_slot.texture, 'image') and mat.use_textures[tex_index] and mtex_slot.texture.image: + hash_this += mtex_slot.texture.image.filepath # Filepaths makes the hash unique + hash_this += str(mat.alpha) # Alpha setting on material makes the hash unique + hash_this += str(mat.diffuse_color) # Diffuse color makes the hash unique + # hash_this += str(mat.specular_color) # Specular color makes the hash unique # Specular Color is no used by Unity + + return hash_this + else: + ignore_nodes = {'Material Output', 'mmd_tex_uv', 'Cats Export Shader'} + if mat.use_nodes and mat.node_tree: + # print('MAT: ', mat.name) + nodes = mat.node_tree.nodes + for node in nodes: + + # Skip certain known nodes + if node.name in ignore_nodes or node.label in ignore_nodes: + continue + + # Add images to hash and skip toon and shpere textures + if node.type == 'TEX_IMAGE': + image = node.image + if 'toon' in node.name or 'sphere' in node.name: + nodes.remove(node) continue - - # On MMD models only add diffuse and transparency to the hash - if node.name == 'mmd_shader': - # print(' ', node.name) - # print(' ', node.inputs['Diffuse Color'].default_value[:]) - # print(' ', node.inputs['Alpha'].default_value) - hash_this += node.name\ - + str(node.inputs['Diffuse Color'].default_value[:])\ - + str(node.inputs['Alpha'].default_value) + if not image: + nodes.remove(node) continue - - # Add all other nodes to the hash # print(' ', node.name) - hash_this += node.name - for input, value in node.inputs.items(): - if hasattr(value, 'default_value'): - try: - # print(' ', input, value.default_value[:]) - hash_this += str(value.default_value[:]) - except TypeError: - # print(' ', input, value.default_value) - hash_this += str(value.default_value) - else: - # print(' ', input, 'name:', value.name) - hash_this += value.name - - # Now create or add to the dict key that has this hash value - if hash_this not in self.combined_tex: - self.combined_tex[hash_this] = [] - self.combined_tex[hash_this].append({'mat': mat_slot.name, 'index': index}) - - # for key, value in self.combined_tex.items(): - # print(key) - # for mat in value: - # print(mat) + # print(' ', image.name) + hash_this += node.name + image.name + continue + # Skip nodes with no input + if not node.inputs: + continue + + # On MMD models only add diffuse and transparency to the hash + if node.name == 'mmd_shader': + # print(' ', node.name) + # print(' ', node.inputs['Diffuse Color'].default_value[:]) + # print(' ', node.inputs['Alpha'].default_value) + hash_this += node.name \ + + str(node.inputs['Diffuse Color'].default_value[:]) \ + + str(node.inputs['Alpha'].default_value) + continue + + # Add all other nodes to the hash + # print(' ', node.name) + hash_this += node.name + for input, value in node.inputs.items(): + if hasattr(value, 'default_value'): + try: + # print(' ', input, value.default_value[:]) + hash_this += str(value.default_value[:]) + except TypeError: + # print(' ', input, value.default_value) + hash_this += str(value.default_value) + else: + # print(' ', input, 'name:', value.name) + hash_this += value.name + else: + # Materials almost always use nodes, but on the off chance that a material doesn't, create the hash + # based on the non-node properties + hash_this += str(mat.diffuse_color[:]) + hash_this += str(mat.metallic) + hash_this += str(mat.roughness) + hash_this += str(mat.specular_intensity) + + return hash_this def execute(self, context): print('COMBINE MATERIALS!') saved_data = Common.SavedData() Common.set_default_stage() - self.generate_combined_tex() Common.switch('OBJECT') - i = 0 - - for index, mesh in enumerate(Common.get_meshes_objects()): + num_combined = 0 - Common.unselect_all() - Common.set_active(mesh) - for file in self.combined_tex: # for each combined mat slot of scene object - combined_textures = self.combined_tex[file] - - # Combining material slots that are similar with only themselves are useless - if len(combined_textures) <= 1: - continue - i += len(combined_textures) + # Hashes of all found materials + mat_hashes = {} + # The first material found for each hash + first_mats_by_hash = {} + for mesh in Common.get_meshes_objects(): + # Generate material hashes and re-assign material slots to the first found material that produces the same + # hash + for mat_name, mat_slot in mesh.material_slots.items(): + mat = mat_slot.material + + # Get the material hash, generating it if needed + if mat_name not in mat_hashes: + mat_hash = self.generate_mat_hash(mat) + mat_hashes[mat_name] = mat_hash + else: + mat_hash = mat_hashes[mat_name] + + # If a material with the same hash has already been found, re-assign the material slot to the previously + # found material, otherwise, add the material to the dictionary of first found materials + if mat_hash in first_mats_by_hash: + replacement_material = first_mats_by_hash[mat_hash] + # The replacement_material material could be the current material if the current material was also + # used on another mesh that was iterated before this mesh. + if mat != replacement_material: + mat_slot.material = replacement_material + num_combined += 1 + else: + first_mats_by_hash[mat_hash] = mat - # print('NEW', file, combined_textures, len(combined_textures)) - Common.switch('EDIT') - bpy.ops.mesh.select_all(action='DESELECT') + # Combine exact duplicate materials within the same mesh + # Get polygon material indices + polygons = mesh.data.polygons + material_indices = np.empty(len(polygons), dtype=np.ushort) + polygons.foreach_get('material_index', material_indices) - # print('UNSELECT ALL') - for mat in mesh.material_slots: # for each scene object material slot - for tex in combined_textures: - if mat.name == tex['mat']: - mesh.active_material_index = tex['index'] - bpy.ops.object.material_slot_select() - # print('SELECT', tex['mat'], tex['index']) + # Find unique sorted material indices and get the inverse array to reconstruct the material indices array + unique_sorted_material_indices, unique_inverse = np.unique(material_indices, return_inverse=True) + # Working with only the unique material indices means we don't need to operate on the entire array + combined_material_indices = self.combine_exact_duplicate_mats(mesh, unique_sorted_material_indices) - bpy.ops.object.material_slot_assign() - # print('ASSIGNED TO SLOT INDEX', bpy.context.object.active_material_index) - bpy.ops.mesh.select_all(action='DESELECT') + # Update the material indices + polygons.foreach_set('material_index', combined_material_indices[unique_inverse]) - Common.unselect_all() - Common.set_active(mesh) - Common.switch('OBJECT') - self.cleanmatslots() + # Remove any unused material slots + self.remove_unused_mat_slots(mesh, combined_material_indices) # Clean material names Common.clean_material_names(mesh) @@ -372,10 +351,10 @@ def execute(self, context): saved_data.load() - if i == 0: + if num_combined == 0: self.report({'INFO'}, t('CombineMaterialsButton.error.noChanges')) else: - self.report({'INFO'}, t('CombineMaterialsButton.success', number=str(i))) + self.report({'INFO'}, t('CombineMaterialsButton.success', number=str(num_combined))) return{'FINISHED'} From 52e83ddea381b66d409a5668f331bbf20e38694b Mon Sep 17 00:00:00 2001 From: Thomas Barlow Date: Wed, 22 Jun 2022 16:49:42 +0100 Subject: [PATCH 2/2] Check for infinities when fixing faulty UV coordinates Optimise fixing faulty UV coordinates using foreach_get/set and numpy Fix resetting fixed_uv_coords count for each iterated mesh --- tools/armature.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/tools/armature.py b/tools/armature.py index 103ebd58..4bd59360 100644 --- a/tools/armature.py +++ b/tools/armature.py @@ -29,7 +29,7 @@ import bpy import copy import math -import platform +import numpy as np from mathutils import Matrix from . import common as Common @@ -391,6 +391,8 @@ def execute(self, context): else: meshes = Common.get_meshes_objects() + # Track how many uv coordinate components we fix + fixed_uv_coords = 0 for mesh in meshes: Common.unselect_all() Common.set_active(mesh) @@ -467,15 +469,25 @@ def execute(self, context): shapekey.name = Translate.fix_jp_chars(shapekey.name) # Fix faulty UV coordinates - fixed_uv_coords = 0 - for uv in mesh.data.uv_layers: - for vert in range(len(uv.data) - 1): - if math.isnan(uv.data[vert].uv.x): - uv.data[vert].uv.x = 0 - fixed_uv_coords += 1 - if math.isnan(uv.data[vert].uv.y): - uv.data[vert].uv.y = 0 - fixed_uv_coords += 1 + uvs = np.empty(len(mesh.data.loops) * 2, dtype=np.single) + uvs_is_non_finite = np.empty(uvs.shape, dtype=bool) + for uv_layer in mesh.data.uv_layers: + uv_layer.data.foreach_get('uv', uvs) + # Get mask of all uvs components that are finite (not Nan and not +/- infinity) and temporarily store + # them in uvs_is_non_finite + np.isfinite(uvs, out=uvs_is_non_finite) + # Invert uvs_is_non_finite so that it is now correctly a mask of all uv components that are non-finite + # (NaN or +/- infinity) + np.invert(uvs_is_non_finite, out=uvs_is_non_finite) + + # Count how many non-finite uv components there are + num_non_finite = np.count_nonzero(uvs_is_non_finite) + if num_non_finite > 0: + fixed_uv_coords += num_non_finite + # Fix the non-finite uv components by setting them to zero + uvs[uvs_is_non_finite] = 0 + # Update the uvs with the fixed values + uv_layer.data.foreach_set('uv', uvs) # Combines same materials # combine_mats runs on all meshes in Common.get_meshes_objects() and gathers material hashes of all the