Skip to content

Commit bbfd1b0

Browse files
eduardodoriaeduardodoria
committed
Mesh instance selection working
1 parent 83a433c commit bbfd1b0

4 files changed

Lines changed: 260 additions & 27 deletions

File tree

‎editor/render/SceneRender.cpp‎

Lines changed: 189 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "util/GraphicUtils.h"
1111

1212
#include <cmath>
13+
#include <cfloat>
1314

1415
using namespace doriax;
1516

@@ -377,6 +378,24 @@ void editor::SceneRender::update(std::vector<Entity> selEntities, std::vector<En
377378
}
378379
}
379380

381+
// Override gizmo for selected instance within instanced mesh
382+
if (selectedInstanceIndex >= 0 && selEntities.size() == 1 && selEntities[0] == selectedInstanceEntity){
383+
InstancedMeshComponent* instmesh = scene->findComponent<InstancedMeshComponent>(selectedInstanceEntity);
384+
Transform* instTransform = scene->findComponent<Transform>(selectedInstanceEntity);
385+
if (instmesh && instTransform && selectedInstanceIndex < (int)instmesh->instances.size()){
386+
OBB instOBB = getInstanceOBB(selectedInstanceEntity, selectedInstanceIndex);
387+
if (!instOBB.isNull()){
388+
const InstanceData& inst = instmesh->instances[selectedInstanceIndex];
389+
Quaternion instWorldRotation = getInstanceWorldRotation(*instTransform, *instmesh, inst);
390+
Vector3 instCenter = instOBB.getCenter();
391+
toolslayer.updateGizmo(camera, instCenter, instWorldRotation, scale, instOBB, mouseRay, mouseClicked, anchorData, anchorArea);
392+
// Replace selection outline with the instance OBB
393+
selBB.clear();
394+
selBB.push_back(instOBB);
395+
}
396+
}
397+
}
398+
380399
if (selBB.size() > 0) {
381400
updateSelLines(selBB);
382401
}
@@ -386,7 +405,7 @@ void editor::SceneRender::update(std::vector<Entity> selEntities, std::vector<En
386405
// Determine selLines visibility: hide for single entity with OBJECT2D gizmo or empty selBB
387406
bool showSelLines = selectionVisibility;
388407
if (!multipleEntitiesSelected && (toolslayer.getGizmoSelected() == GizmoSelected::OBJECT2D || selBB.size() == 0)) {
389-
if (selectedTileIndex < 0) {
408+
if (selectedTileIndex < 0 && selectedInstanceIndex < 0) {
390409
showSelLines = false;
391410
}
392411
}
@@ -402,7 +421,8 @@ void editor::SceneRender::update(std::vector<Entity> selEntities, std::vector<En
402421
if (toolslayer.getGizmoSelected() == GizmoSelected::OBJECT2D && selEntities.size() == 1){
403422
bool isTilemap = scene->getSignature(selEntities[0]).test(scene->getComponentId<TilemapComponent>());
404423
bool isCamera = scene->getSignature(selEntities[0]).test(scene->getComponentId<CameraComponent>());
405-
toolslayer.setShowObject2DRects((!isTilemap || selectedTileIndex >= 0) && !isCamera);
424+
bool isInstancedMesh = scene->getSignature(selEntities[0]).test(scene->getComponentId<InstancedMeshComponent>());
425+
toolslayer.setShowObject2DRects((!isTilemap || selectedTileIndex >= 0) && !isCamera && !isInstancedMesh);
406426
}
407427

408428
uilayer.setViewGizmoImageVisible(true);
@@ -506,6 +526,39 @@ void editor::SceneRender::mouseClickEvent(float x, float y, std::vector<Entity>
506526
}
507527
}
508528
}
529+
530+
// Store instance start state for instance sub-selection drag
531+
if (selectedInstanceIndex >= 0 && selEntities.size() == 1 && selEntities[0] == selectedInstanceEntity){
532+
InstancedMeshComponent* instmesh = scene->findComponent<InstancedMeshComponent>(selectedInstanceEntity);
533+
Transform* instTransform = scene->findComponent<Transform>(selectedInstanceEntity);
534+
if (instmesh && instTransform && selectedInstanceIndex < (int)instmesh->instances.size()){
535+
const InstanceData& inst = instmesh->instances[selectedInstanceIndex];
536+
instanceStartPosition = inst.position;
537+
instanceStartRotation = inst.rotation;
538+
instanceStartScale = inst.scale;
539+
540+
OBB instOBB = getInstanceOBB(selectedInstanceEntity, selectedInstanceIndex);
541+
if (!instOBB.isNull()){
542+
// Gizmo is centered on the instance OBB center (world space); its
543+
// orientation must match getInstanceOBB (see getInstanceWorldRotation).
544+
gizmoStartPosition = instOBB.getCenter();
545+
cursorStartOffset = gizmoStartPosition - rretrun.point;
546+
rotationStartOffset = getInstanceWorldRotation(*instTransform, *instmesh, inst);
547+
548+
// Build the actual instance world matrix (same formula the renderer uses
549+
// in non-billboard mode) so drag math operates on the instance directly.
550+
Matrix4 instanceLocalMatrix = Matrix4::translateMatrix(inst.position)
551+
* inst.rotation.getRotationMatrix()
552+
* Matrix4::scaleMatrix(inst.scale);
553+
Matrix4 instanceWorldMatrix = instTransform->modelMatrix * instanceLocalMatrix;
554+
555+
Matrix4 gizmoM = Matrix4::translateMatrix(gizmoStartPosition)
556+
* rotationStartOffset.getRotationMatrix()
557+
* Matrix4::scaleMatrix(Vector3(1, 1, 1));
558+
objectMatrixOffset[selectedInstanceEntity] = gizmoM.inverse() * instanceWorldMatrix;
559+
}
560+
}
561+
}
509562
}
510563

511564
}
@@ -539,12 +592,50 @@ void editor::SceneRender::mouseDragEvent(float x, float y, float origX, float or
539592

540593
SceneProject* sceneProject = project->getScene(sceneId);
541594

595+
bool isInstanceDrag = selectedInstanceIndex >= 0 && selEntities.size() == 1
596+
&& entity == selectedInstanceEntity
597+
&& scene->findComponent<InstancedMeshComponent>(entity) != nullptr;
598+
599+
auto emitInstanceTransformCmd = [&](const Matrix4& worldMatrix){
600+
Matrix4 instLocalMatrix = transform->modelMatrix.inverse() * worldMatrix;
601+
Vector3 newPos, newScale;
602+
Quaternion newRot;
603+
instLocalMatrix.decomposeStandard(newPos, newScale, newRot);
604+
605+
std::string prefix = "instances[" + std::to_string(selectedInstanceIndex) + "]";
606+
MultiPropertyCmd* multiCmd = new MultiPropertyCmd();
607+
GizmoSelected gz = toolslayer.getGizmoSelected();
608+
if (gz == GizmoSelected::TRANSLATE || gz == GizmoSelected::OBJECT2D){
609+
multiCmd->addPropertyCmd<Vector3>(project, sceneProject->id, entity, ComponentType::InstancedMeshComponent, prefix + ".position", newPos);
610+
}else if (gz == GizmoSelected::ROTATE){
611+
multiCmd->addPropertyCmd<Quaternion>(project, sceneProject->id, entity, ComponentType::InstancedMeshComponent, prefix + ".rotation", newRot);
612+
}else if (gz == GizmoSelected::SCALE){
613+
multiCmd->addPropertyCmd<Vector3>(project, sceneProject->id, entity, ComponentType::InstancedMeshComponent, prefix + ".scale", newScale);
614+
}
615+
lastCommand = multiCmd;
616+
};
617+
542618
if (rretrun){
543619

544620
toolslayer.mouseDrag(rretrun.point);
545621

546622
Transform* transformParent = scene->findComponent<Transform>(transform->parent);
547623

624+
// Emit a transform command for the active drag target (instance or entity).
625+
// worldMatrix is the new object world matrix built by the current gizmo branch.
626+
auto applyObjectTransform = [&](const Matrix4& worldMatrix){
627+
if (toolslayer.getGizmoSideSelected() == GizmoSideSelected::NONE) return;
628+
if (isInstanceDrag){
629+
emitInstanceTransformCmd(worldMatrix);
630+
}else{
631+
Matrix4 m = worldMatrix;
632+
if (transformParent){
633+
m = transformParent->modelMatrix.inverse() * m;
634+
}
635+
lastCommand = new ObjectTransformCmd(project, sceneProject->id, entity, m);
636+
}
637+
};
638+
548639
if (toolslayer.getGizmoSelected() == GizmoSelected::TRANSLATE){
549640
Vector3 deltaPos = gizmoRMatrix.inverse() * ((rretrun.point + cursorStartOffset) - gizmoStartPosition);
550641

@@ -577,13 +668,7 @@ void editor::SceneRender::mouseDragEvent(float x, float y, float origX, float or
577668
Matrix4 gizmoMatrix = Matrix4::translateMatrix(newPos) * gizmoRMatrix * Matrix4::scaleMatrix(Vector3(1,1,1));
578669
Matrix4 objMatrix = gizmoMatrix * objectMatrixOffset[entity];
579670

580-
if (transformParent){
581-
objMatrix = transformParent->modelMatrix.inverse() * objMatrix;
582-
}
583-
584-
if (toolslayer.getGizmoSideSelected() != GizmoSideSelected::NONE){
585-
lastCommand = new ObjectTransformCmd(project, sceneProject->id, entity, objMatrix);
586-
}
671+
applyObjectTransform(objMatrix);
587672
}
588673

589674
if (toolslayer.getGizmoSelected() == GizmoSelected::ROTATE){
@@ -605,13 +690,7 @@ void editor::SceneRender::mouseDragEvent(float x, float y, float origX, float or
605690
Matrix4 gizmoMatrix = Matrix4::translateMatrix(gizmoPosition) * newRot.getRotationMatrix() * Matrix4::scaleMatrix(Vector3(1,1,1));
606691
Matrix4 objMatrix = gizmoMatrix * objectMatrixOffset[entity];
607692

608-
if (transformParent){
609-
objMatrix = transformParent->modelMatrix.inverse() * objMatrix;
610-
}
611-
612-
if (toolslayer.getGizmoSideSelected() != GizmoSideSelected::NONE){
613-
lastCommand = new ObjectTransformCmd(project, sceneProject->id, entity, objMatrix);
614-
}
693+
applyObjectTransform(objMatrix);
615694
}
616695

617696
if (toolslayer.getGizmoSelected() == GizmoSelected::SCALE){
@@ -647,13 +726,7 @@ void editor::SceneRender::mouseDragEvent(float x, float y, float origX, float or
647726
Matrix4 gizmoMatrix = Matrix4::translateMatrix(gizmoPosition) * gizmoRMatrix * Matrix4::scaleMatrix(newScale);
648727
Matrix4 objMatrix = gizmoMatrix * objectMatrixOffset[entity];
649728

650-
if (transformParent){
651-
objMatrix = transformParent->modelMatrix.inverse() * objMatrix;
652-
}
653-
654-
if (toolslayer.getGizmoSideSelected() != GizmoSideSelected::NONE){
655-
lastCommand = new ObjectTransformCmd(project, sceneProject->id, entity, objMatrix);
656-
}
729+
applyObjectTransform(objMatrix);
657730
}
658731

659732
if (toolslayer.getGizmoSelected() == GizmoSelected::OBJECT2D){
@@ -780,6 +853,22 @@ void editor::SceneRender::mouseDragEvent(float x, float y, float origX, float or
780853
lastCommand = multiCmd;
781854
}
782855
}
856+
}else if (isInstanceDrag){
857+
// Instance CENTER drag (position only) for OBJECT2D
858+
if (toolslayer.getGizmo2DSideSelected() == Gizmo2DSideSelected::CENTER){
859+
Vector3 newPos = gizmoRMatrix.inverse() * ((rretrun.point + cursorStartOffset) - gizmoStartPosition);
860+
newPos = gizmoStartPosition + (gizmoRMatrix * newPos);
861+
if (displaySettings.snapToGrid) {
862+
float spacing = displaySettings.gridSpacing2D;
863+
if (spacing > 0.0f) {
864+
newPos.x = std::round(newPos.x / spacing) * spacing;
865+
newPos.y = std::round(newPos.y / spacing) * spacing;
866+
}
867+
}
868+
Matrix4 gizmoMatrix = Matrix4::translateMatrix(newPos) * gizmoRMatrix * Matrix4::scaleMatrix(Vector3(1,1,1));
869+
Matrix4 objMatrix = gizmoMatrix * objectMatrixOffset[entity];
870+
emitInstanceTransformCmd(objMatrix);
871+
}
783872
}else{
784873
// Original entity-level OBJECT2D handling
785874
bool isSprite = scene->getComponentArray<SpriteComponent>()->hasEntity(entity);
@@ -1001,6 +1090,83 @@ OBB editor::SceneRender::getTileOBB(Entity entity, int tileIndex){
10011090
return transform.modelMatrix * tileAABB.getOBB();
10021091
}
10031092

1093+
void editor::SceneRender::selectInstance(Entity entity, int instanceIndex){
1094+
selectedInstanceEntity = entity;
1095+
selectedInstanceIndex = instanceIndex;
1096+
}
1097+
1098+
void editor::SceneRender::clearInstanceSelection(){
1099+
selectedInstanceEntity = 0;
1100+
selectedInstanceIndex = -1;
1101+
}
1102+
1103+
Quaternion editor::SceneRender::getInstanceWorldRotation(const Transform& transform, const InstancedMeshComponent& instmesh, const InstanceData& inst) const{
1104+
// Billboard mode overrides per-instance rotation at render time, so mirror that
1105+
// here by not composing inst.rotation into the picking/gizmo orientation.
1106+
if (instmesh.instancedBillboard){
1107+
return transform.worldRotation;
1108+
}
1109+
return transform.worldRotation * inst.rotation;
1110+
}
1111+
1112+
OBB editor::SceneRender::getInstanceOBB(Entity entity, int instanceIndex){
1113+
if (!scene->getComponentArray<InstancedMeshComponent>()->hasEntity(entity)) return OBB();
1114+
if (!scene->getComponentArray<MeshComponent>()->hasEntity(entity)) return OBB();
1115+
if (!scene->getComponentArray<Transform>()->hasEntity(entity)) return OBB();
1116+
1117+
InstancedMeshComponent& instmesh = scene->getComponent<InstancedMeshComponent>(entity);
1118+
MeshComponent& mesh = scene->getComponent<MeshComponent>(entity);
1119+
Transform& transform = scene->getComponent<Transform>(entity);
1120+
1121+
if (instanceIndex < 0 || instanceIndex >= (int)instmesh.instances.size()) return OBB();
1122+
1123+
const InstanceData& inst = instmesh.instances[instanceIndex];
1124+
// For billboard meshes the per-instance rotation is ignored at render time, so
1125+
// skip it here too to keep the OBB aligned with what is actually drawn.
1126+
Matrix4 rotMatrix = instmesh.instancedBillboard
1127+
? Matrix4()
1128+
: inst.rotation.getRotationMatrix();
1129+
Matrix4 instanceMatrix = Matrix4::translateMatrix(inst.position)
1130+
* rotMatrix
1131+
* Matrix4::scaleMatrix(inst.scale);
1132+
1133+
AABB localAABB = mesh.verticesAABB;
1134+
if (localAABB.isNull() || (localAABB.getMinimum() == Vector3::ZERO && localAABB.getMaximum() == Vector3::ZERO)){
1135+
// Fallback to a small unit cube around origin so the instance is still selectable
1136+
localAABB = AABB(Vector3(-0.5f, -0.5f, -0.5f), Vector3(0.5f, 0.5f, 0.5f));
1137+
}
1138+
1139+
return (transform.modelMatrix * instanceMatrix) * localAABB.getOBB();
1140+
}
1141+
1142+
int editor::SceneRender::hitTestInstance(Entity entity, float x, float y){
1143+
if (!scene->getComponentArray<InstancedMeshComponent>()->hasEntity(entity)) return -1;
1144+
if (!scene->getComponentArray<MeshComponent>()->hasEntity(entity)) return -1;
1145+
if (!scene->getComponentArray<Transform>()->hasEntity(entity)) return -1;
1146+
1147+
InstancedMeshComponent& instmesh = scene->getComponent<InstancedMeshComponent>(entity);
1148+
1149+
Ray ray = camera->screenToRay(x, y);
1150+
1151+
// Only instances within maxInstances are actually rendered, so restrict picking to them.
1152+
int renderedCount = (int)std::min<size_t>(instmesh.instances.size(), instmesh.maxInstances);
1153+
1154+
int bestIndex = -1;
1155+
float bestDistance = FLT_MAX;
1156+
for (int i = 0; i < renderedCount; i++){
1157+
if (!instmesh.instances[i].visible) continue;
1158+
OBB instOBB = getInstanceOBB(entity, i);
1159+
if (instOBB.isNull()) continue;
1160+
RayReturn rr = ray.intersects(instOBB);
1161+
if (rr && rr.distance < bestDistance){
1162+
bestDistance = rr.distance;
1163+
bestIndex = i;
1164+
}
1165+
}
1166+
1167+
return bestIndex;
1168+
}
1169+
10041170
void editor::SceneRender::setupCameraIcon(CameraObjects& co){
10051171
TextureData iconData;
10061172
iconData.loadTextureFromMemory(camera_icon_png, camera_icon_png_len);

‎editor/render/SceneRender.h‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,11 +72,17 @@ namespace doriax::editor{
7272
float tileStartWidth = 0;
7373
float tileStartHeight = 0;
7474

75+
// Instance sub-selection within an instanced mesh entity
76+
Entity selectedInstanceEntity = 0; // NULL_ENTITY
77+
int selectedInstanceIndex = -1;
78+
Vector3 instanceStartPosition;
79+
Quaternion instanceStartRotation;
80+
Vector3 instanceStartScale;
81+
7582
AABB getAABB(Entity entity, bool local);
7683
AABB getFamilyAABB(Entity entity, float offset);
7784

78-
OBB getOBB(Entity entity, bool local);
79-
OBB getFamilyOBB(Entity entity, float offset);
85+
OBB getOBB(Entity entity, bool local); OBB getFamilyOBB(Entity entity, float offset);
8086

8187
protected:
8288
void updateCameraFrustum(CameraObjects& co, const CameraComponent& cameraComponent, bool isMainCamera, bool fixedSizeFrustum = true);
@@ -150,6 +156,15 @@ namespace doriax::editor{
150156
void clearTileSelection();
151157
int hitTestTile(Entity entity, float x, float y);
152158
OBB getTileOBB(Entity entity, int tileIndex);
159+
160+
// Instance sub-selection
161+
int getSelectedInstanceIndex() const { return selectedInstanceIndex; }
162+
Entity getSelectedInstanceEntity() const { return selectedInstanceEntity; }
163+
void selectInstance(Entity entity, int instanceIndex);
164+
void clearInstanceSelection();
165+
int hitTestInstance(Entity entity, float x, float y);
166+
OBB getInstanceOBB(Entity entity, int instanceIndex);
167+
Quaternion getInstanceWorldRotation(const Transform& transform, const InstancedMeshComponent& instmesh, const InstanceData& inst) const;
153168
};
154169

155170
}

‎editor/window/Properties.cpp‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5975,6 +5975,19 @@ void editor::Properties::drawInstancedMeshComponent(ComponentType cpType, SceneP
59755975
std::string instanceGroupStr = "instance_" + std::to_string(i);
59765976
std::string instanceLabel = "[" + std::to_string(i) + "] Instance " + std::to_string(i);
59775977

5978+
bool isInstanceSelected = sceneProject->sceneRender &&
5979+
sceneProject->sceneRender->getSelectedInstanceIndex() == (int)i &&
5980+
sceneProject->sceneRender->getSelectedInstanceEntity() == entities[0];
5981+
5982+
if (isInstanceSelected) {
5983+
ImVec2 cursorPos = ImGui::GetCursorScreenPos();
5984+
float width = ImGui::GetContentRegionAvail().x;
5985+
float height = ImGui::GetFrameHeight();
5986+
ImDrawList* drawList = ImGui::GetWindowDrawList();
5987+
ImVec4 highlightColor = ImGui::GetStyle().Colors[ImGuiCol_HeaderActive];
5988+
highlightColor.w = 0.35f;
5989+
drawList->AddRectFilled(cursorPos, ImVec2(cursorPos.x + width, cursorPos.y + height), ImGui::GetColorU32(highlightColor), 3.0f);
5990+
}
59785991
ImGui::SeparatorText(instanceLabel.c_str());
59795992

59805993
beginTable(cpType, getLabelSize("Position"), instanceGroupStr);

0 commit comments

Comments
 (0)