Enable updatePosition, updateRotation and updateScale of existing Nodes

[wiizarrrd]

Enable updatePosition

[wiizarrrd]

How to use in flutter:

Simple usage:

scale = vector.Vector3(value, value, value);
node.scale.value = scale;

translation = vector.Vector3(value, value, value);
node.position.value = translation;

// be aware, that this is a quaternion
rotation = vector.Vector4(value, value, value, value);
node.rotation.value = rotation

main parts of the implementation:

ArCoreNode node;

ArCoreView in the build method

ArCoreView(
                  key: _key,
                  onArCoreViewCreated: _onArCoreViewCreated,
                  enableTapRecognizer: true,
                  enableUpdateListener: true,
)

_onArCoreViewCreated

void _onArCoreViewCreated(ArCoreController controller) {
    arCoreController = controller;
    arCoreController.onPlaneTap = _onPlaneTap;
}

_onPlaneTap

  Future<void> _onPlaneTap(List<ArCoreHitTestResult> hits) async {
    if (hits.isNotEmpty) {
      final hit = hits.first;
      vector.Vector3 t = hit.pose.translation;
      vector.Vector4 r = hit.pose.rotation;
      vector.Vector3 offSet = vector.Vector3(1, 1, 1);
      if (node != null) {
        //rotateObject to be implemented
        node.rotation.value = rotateObject(r);
        node.position.value = t + offSet;
        node.scale.value = vector.Vector3(0.5, 0.5, 0.5);
      } else {
        final material = ArCoreMaterial(color: Colors.green);
        final sphere = ArCoreSphere(
          materials: [material],
          radius: 0.1,
        );
        node = ArCoreNode(
          name: "uniqueName",
          shape: sphere,
        );
        //no anchor to be able to update the position
        arCoreController.addArCoreNode(node);
      }
    }
  }

[dark42233]

@wiizarrrd can u share full code about rotating 3d model after placing please?

[wiizarrrd]

I can share some important parts. The aim of the code is to rotate an object so that it sticks to a wall like a picture with a frame.

Quarternions file:

---

import 'dart:math';

import 'package:vector_math/vector_math_64.dart';

class QuaternionCalculations {
//Compute the right position for an object. The infoNodeFactor controls the distance between the frame and the infoNode.
static Vector3 getPositionOnCircularOrbit(Vector3 unitVector, Quaternion q) {
final Vector3 normalOfPlane = q.asRotationMatrix() * unitVector;
final offset = Vector3(normalOfPlane.x, 0, normalOfPlane.z);
return offset.normalized();
}

//x and y HAVE TO be switched for some reason
static Vector4 quaternionToVector4Rad(Quaternion q) {
return Vector4(q.y, q.x, q.z, q.w);
}

//Source: https://en.wikipedia.org/wiki/Conversion_between_quaternions_and_Euler_angles
static List toEulerAnglesRad(Quaternion q) {
List angles = List.filled(3, 0);

// roll (x-axis rotation)
final double sinrCosp = 2 * (q.w * q.x + q.y * q.z);
final double cosrCosp = 1 - 2 * (q.x * q.x + q.y * q.y);
angles[0] = atan2(sinrCosp, cosrCosp);

// pitch (y-axis rotation)
final double sinp = 2 * (q.w * q.y - q.z * q.x);
if (sinp.abs() >= 1) {
  angles[1] =
      (pi / 2) * copySign(pi / 2, sinp); // use 90 degrees if out of range
} else {
  angles[1] = asin(sinp);
}

// yaw (z-axis rotation)
final double sinyCosp = 2 * (q.w * q.z + q.x * q.y);
final double cosyCosp = 1 - 2 * (q.y * q.y + q.z * q.z);
angles[2] = atan2(sinyCosp, cosyCosp);

return angles;

}

static double copySign(double value, double sign) {
return value.abs() * (sign < 0 ? -1 : 1);
}

// static List _toEulerAnglesDeg(Quaternion q) {
// List rad = toEulerAnglesRad(q);
// return [degrees(rad[0]), degrees(rad[1]), degrees(rad[2])];
// }
}

---

main file

---

import 'dart:async';
import 'dart:math';

import 'package:arcore_flutter_plugin/arcore_flutter_plugin.dart';
import 'package:flutter/material.dart';
import 'package:flutter/rendering.dart';
import 'package:flutter/services.dart';
import 'package:hswo_mvc_2021_flutter_ar/helperClasses/QuaternionCalculations.dart';
import 'package:hswo_mvc_2021_flutter_ar/helperClasses/SimpleArCoreSphere.dart';
import 'package:hswo_mvc_2021_flutter_ar/helperClasses/SimulatedTochscreenTap.dart';
import 'package:hswo_mvc_2021_flutter_ar/helperClasses/Classifer.dart';
import 'package:uptime/uptime.dart';
import 'package:vector_math/vector_math_64.dart' as vector;
import 'package:vector_math/vector_math.dart' as vMath;

import '../contants.dart';
import '../helperMethods.dart';

class ARCoreScreen extends StatefulWidget {
final bool debug;

ARCoreScreen(this.debug);

@OverRide
_ARCoreScreenState createState() => _ARCoreScreenState();
}

class _ARCoreScreenState extends State {
final _key = GlobalKey();

ArCoreController arCoreController;
ArCoreHitTestResult savedHit;
ArCoreNode frame,
infoNode,
greyCyrcle,
originCyrcle,
cyrcleZ,
cyrcleY,
cyrcleX;
List rotation = List.filled(3, 0);
vector.Vector3 scale = vector.Vector3(1.0, 1.0, 1.0);

Timer timer;
bool activeTimer = true;
static const int interval = 1;

double infoNodeFactor = 1.1;
double sliderDistanceValue = 0.55;
double sliderScaleValue = 0.5;

//4 different options to dis-/enable
bool tapSimulationMode = true;
bool coordinatesMode = false;
bool sliderMode = true;
bool hideFrame = false;

// _ARCoreScreenState(){
// if(!widget.debug){
// tapSimulationMode = false;
// coordinatesMode = false;
// sliderMode=false;
// hideFrame = true;
// }
// }

@OverRide
void initState() {
// TODO: implement initState
super.initState();

if (!widget.debug) {
  tapSimulationMode = false;
  coordinatesMode = false;
  sliderMode = false;
  hideFrame = true;
}

Future.delayed(Duration(seconds: 5), () {
  timer = Timer.periodic(Duration(seconds: interval), (Timer timer) async {
    if (activeTimer) await simulateTouchScreenTapProcess();
  });
});

}

@OverRide
Widget build(BuildContext context) {
return MaterialApp(
home: Scaffold(
appBar: AppBar(
title: const Text('Get more information'),
),
body: SafeArea(
child: Column(
crossAxisAlignment: CrossAxisAlignment.stretch,
children: [
_buildOptionalTouchScreenTap,
_buildOptionalSliders,
Expanded(
child: ArCoreView(
key: _key,
onArCoreViewCreated: _onArCoreViewCreated,
enableTapRecognizer: true,
enableUpdateListener: true,
// debug: true,
),
),
],
),
),
),
);
}

Widget get _buildOptionalTouchScreenTap {
Widget tapSimulation = SizedBox();
if (tapSimulationMode) {
tapSimulation = Padding(
padding: EdgeInsets.symmetric(horizontal: 8.0),
child: ElevatedButton(
onPressed: () {
simulateTouchScreenTap(_key.currentContext.size.width / 2,
_key.currentContext.size.height / 2);
},
child: Text('Simulate a Tap on the Screen'),
),
);
}
return tapSimulation;
}

Widget get _buildOptionalSliders {
Widget editInfoNodeFactor = SizedBox();
if (sliderMode) {
editInfoNodeFactor = Padding(
padding: EdgeInsets.symmetric(horizontal: 8.0),
child: Row(
children: [
Text("Distance"),
Expanded(
child: Slider(
value: sliderDistanceValue,
onChangeEnd: (value) {
sliderDistanceValue = value;
_onInfoNodeFactorChange(sliderDistanceValue);
},
onChanged: (double value) {
setState(() {
sliderDistanceValue = value;
});
},
),
),
Text("Scale"),
Expanded(
child: Slider(
value: sliderScaleValue,
onChangeEnd: (value) async {
sliderScaleValue = value;
await _onScaleChange(sliderScaleValue);
},
onChanged: (double value) {
setState(() {
sliderScaleValue = value;
});
},
),
),
],
),
);
}
return editInfoNodeFactor;
}

_onInfoNodeFactorChange(double newFactor) {
infoNodeFactor = sliderDistanceValue * 2;
rotateObject(savedHit?.pose?.rotation, savedHit?.pose?.translation);
}

_onScaleChange(double newFactor) async {
double value = sliderScaleValue * 2;
scale = vector.Vector3(value, value, value);
infoNode?.scale?.value = scale;
frame?.scale?.value = scale;
rotateObject(savedHit?.pose?.rotation, savedHit?.pose?.translation);
}

@OverRide
void dispose() {
arCoreController?.dispose();
activeTimer = false;
super.dispose();
}

void _onArCoreViewCreated(ArCoreController controller) {
arCoreController = controller;
arCoreController.onPlaneTap = _onPlaneTap;
arCoreController.onNodeTap = _onNodeTap;
arCoreController.screenshotDone = _processImage;
}

Future _onPlaneTap(List hits) async {
if (hits.isNotEmpty) {
final hit = hits.first;
await _addObject(hit);
}
}

Future _onNodeTap(String nodeName) async {
print("Node tapped: $nodeName");
}

Future _addObject(ArCoreHitTestResult hit) async {
savedHit = hit;
vector.Vector3 t = hit.pose.translation;
vector.Vector4 r = hit.pose.rotation;
if (infoNode != null) {
if (coordinatesMode) {
cyrcleZ.position.value = t + zVector;
cyrcleY.position.value = t + upVector;
cyrcleX.position.value = t + xVector;
originCyrcle.position.value = t;
}
if (!hideFrame) {
frame.position.value = t;
}
rotateObject(r, t);
} else {
await addNodes(r, t);
}
}

Future addNodes(vector.Vector4 r, vector.Vector3 t) async {
if (!hideFrame) {
frame = ArCoreReferenceNode(
name: "Frame",
objectUrl: await getImagePathAndroid('Frame_only.glb'),
position: t,
);
}

infoNode = ArCoreReferenceNode(
  name: "Info",
  objectUrl: await getImagePathAndroid('Info_only.glb'),
);

if (coordinatesMode) {
  // +-+-+-+ Helpers +-+-+-+
  greyCyrcle = SimpleArCoreSphere(
          "greyCyrcle", Color.fromARGB(255, 128, 128, 128), null)
      .node;
  originCyrcle = SimpleArCoreSphere(
          "originCyrcle", Color.fromARGB(255, 255, 255, 255), t)
      .node;
  cyrcleZ = SimpleArCoreSphere(
          "CyrcleZ", Color.fromARGB(255, 0, 0, 255), t + zVector)
      .node;
  cyrcleY = SimpleArCoreSphere(
          "CyrcleY", Color.fromARGB(255, 0, 255, 0), t + upVector)
      .node;
  cyrcleX = SimpleArCoreSphere(
          "CyrcleX", Color.fromARGB(255, 255, 0, 0), t + xVector)
      .node;
}
rotateObject(r, t);
if (!hideFrame) {
  arCoreController.addArCoreNode(frame);
}
arCoreController.addArCoreNode(infoNode);
if (coordinatesMode) {
  arCoreController.addArCoreNode(greyCyrcle);
  arCoreController.addArCoreNode(originCyrcle);
  arCoreController.addArCoreNode(cyrcleX);
  arCoreController.addArCoreNode(cyrcleY);
  arCoreController.addArCoreNode(cyrcleZ);
}

}

void rotateObject(vector.Vector4 r, vector.Vector3 t) {
if (r == null || t == null) {
return;
}
vector.Quaternion q = vector.Quaternion(r.x, r.y, r.z, r.w);
// print("Rotation: " + _toEulerAnglesDeg(q).toString());
List euler = QuaternionCalculations.toEulerAnglesRad(q);
vector.Vector3 infoNodePosition;

//Object at the bottom (with some tolerance)
if (euler[0].abs() < 0.001 && euler[2].abs() < 0.001 ||
    euler[0].abs() > pi - 0.001 && euler[2].abs() > pi - 0.001) {
  rotation = [euler[0], euler[1], euler[2]];
  if (coordinatesMode) {
    greyCyrcle.position.value = t + upVector;
  }
  infoNodePosition = t +
      QuaternionCalculations.getPositionOnCircularOrbit(xVector, q) *
          infoNodeFactor *
          scale.x;

  //Object at the roof (with some tolerance)
} else if (euler[0].abs() < 0.001 && euler[2].abs() > pi - 0.001 ||
    euler[0].abs() > pi - 0.001 && euler[2].abs() < 0.001) {
  rotation = [euler[0], euler[1] + pi, euler[2]];
  if (coordinatesMode) {
    greyCyrcle.position.value = t - upVector;
  }
  infoNodePosition = t +
      QuaternionCalculations.getPositionOnCircularOrbit(-xVector, q) *
          infoNodeFactor *
          scale.x;

  //Object at the wall
} else {
  final vector.Vector3 normalOfPlane = q.asRotationMatrix() * upVector;
  double angle1 = atan2(normalOfPlane.x, normalOfPlane.z);
  rotation = [pi / 2, angle1, 0];
  print("Angle: " + vMath.degrees(angle1).toString());

  if (coordinatesMode) {
    greyCyrcle.position.value =
        t + QuaternionCalculations.getPositionOnCircularOrbit(upVector, q);
  }

  angle1 += pi / 2;
  final offset = vector.Vector3(sin(angle1), 0, cos(angle1));
  infoNodePosition = t + offset * infoNodeFactor * scale.x;

  //infoNode sometimes right, sometimes left with this code - depending on the height of the hit
  // infoNode.position.value = t + _getPositionOnCircularOrbit(-xVector, q) * infoNodeFactor;
}
if (!hideFrame) {
  frame.rotation.value = QuaternionCalculations.quaternionToVector4Rad(
      vector.Quaternion.euler(rotation[0], rotation[1], rotation[2]));
}
infoNode.rotation.value = QuaternionCalculations.quaternionToVector4Rad(
    vector.Quaternion.euler(rotation[0], rotation[1], rotation[2]));
infoNode.position.value = infoNodePosition;

}

Future simulateTouchScreenTapProcess() async {
String path = await arCoreController.takeScreenshot();
Map<String, double> tap =
await _processImage(path);
await simulateTouchScreenTap(tap["x"], tap["y"]);
}

Future<Map<String, double>> _processImage(String path) async {
await Classifier.loadModel();
//avoid race condition
// await Future.delayed(Duration(milliseconds: 100));
var recognitions = await Classifier.detectObject(path);

for (int i = 0; i < recognitions.length; i++) {
  if (recognitions[i]["confidenceInClass"] > 0.50 &&
      recognitions[i]["detectedClass"] == "person") {
    print('recognitions: $recognitions');
    // double width = recognitions[i]["rect"]["w"] - recognitions[i]["rect"]["x"];
    // double height = recognitions[i]["rect"]["h"] - recognitions[i]["rect"]["y"];
    //
    // print("width: "+width.toString());
    // print("height: "+height.toString());
    double xPos =
        (recognitions[i]["rect"]["w"] + recognitions[i]["rect"]["x"]) / 2;
    double yPos =
        (recognitions[i]["rect"]["h"] + recognitions[i]["rect"]["y"]) / 2;

    MediaQueryData mqd = MediaQuery.of(context);
    int dpi = (mqd.devicePixelRatio * 160).toInt();
    xPos = ((xPos * dpi).toInt() / dpi) * mqd.size.width;
    yPos = ((yPos * dpi).toInt() / dpi) * mqd.size.height;
    return {"x": xPos, "y": yPos};
  }
}
return {"x": -1, "y": -1};

}

Future simulateTouchScreenTap(double xPos, double yPos) async {
if (xPos == -1 && yPos == -1) return;
int delay = 80;
int uptime = await Uptime.uptime;

SimulatedTouchscreenTap pushDown = SimulatedTouchscreenTap(
  androidViewId: arCoreController.id,
  downTime: uptime,
  eventTime: uptime,
  androidViewControllerAction: AndroidViewController.kActionDown,
  xPos: xPos,
  yPos: yPos,
);

await SystemChannels.platform_views
    .invokeMethod<dynamic>('touch', pushDown.prepareForInvokeMethodCall());

SimulatedTouchscreenTap pushUp = SimulatedTouchscreenTap(
  androidViewId: arCoreController.id,
  downTime: uptime,
  eventTime: uptime + delay,
  androidViewControllerAction: AndroidViewController.kActionUp,
  xPos: xPos,
  yPos: yPos,
);

await SystemChannels.platform_views
    .invokeMethod<dynamic>('touch', pushUp.prepareForInvokeMethodCall());

}
}

---

[naveenbharadwaj19]

@wiizarrrd does your package follows null-safety ?

[wiizarrrd]

@wiizarrrd does your package follows null-safety ?

For the Kotlin part it was trial & error so I can't tell. Probably not.

[naveenbharadwaj19]

@wiizarrrd i tried to add your package . But i'm getting compiler error tried flutter clean still same issue

C:\Users\bhara\AppData\Local\Pub\Cache\git\arcore_flutter_plugin-28f6ad239e3c45bcd5c124c564f050fc81925a62\android\src\main\kotlin\com\difrancescogianmarco\arcore_flutter_plugin\ArCoreView.kt: (281, 38): Object is not abstract and does not implement abstract member public abstract fun onActivityCreated(p0: Activity, p1: Bundle?): Unit defined in android.app.Application.ActivityLifecycleCallbacks
e: C:\Users\bhara\AppData\Local\Pub\Cache\git\arcore_flutter_plugin-28f6ad239e3c45bcd5c124c564f050fc81925a62\android\src\main\kotlin\com\difrancescogianmarco\arcore_flutter_plugin\ArCoreView.kt: (282, 13): 'onActivityCreated' overrides nothing

[ramyak-mehra]

@wiizarrrd any updates on this?