Merge branch 'release_v2.18.0.0' into main

Author: themarpe

CMakeLists.txt

@@ -100,6 +100,7 @@ pybind11_add_module(${TARGET_NAME}
     src/pipeline/AssetManagerBindings.cpp
     src/openvino/OpenVINOBindings.cpp
     src/log/LogBindings.cpp
+    src/VersionBindings.cpp
 
     src/pipeline/node/NodeBindings.cpp
 
@@ -124,6 +125,7 @@ pybind11_add_module(${TARGET_NAME}
     src/pipeline/node/FeatureTrackerBindings.cpp
     src/pipeline/node/AprilTagBindings.cpp
     src/pipeline/node/DetectionParserBindings.cpp
+    src/pipeline/node/WarpBindings.cpp
 
     src/pipeline/datatype/ADatatypeBindings.cpp
     src/pipeline/datatype/AprilTagConfigBindings.cpp

ci/Dockerfile

@@ -1,11 +1,18 @@
 FROM python:3.9-bullseye
 
-RUN apt-get update && apt-get install -y wget build-essential cmake pkg-config libjpeg-dev libtiff5-dev libavcodec-dev libavformat-dev libswscale-dev libv4l-dev libxvidcore-dev libx264-dev libgtk2.0-dev libgtk-3-dev libatlas-base-dev gfortran git
+RUN apt-get update && apt-get install -y wget build-essential cmake pkg-config libjpeg-dev libtiff5-dev libavcodec-dev libavformat-dev libswscale-dev libv4l-dev libxvidcore-dev libx264-dev libgtk2.0-dev libgtk-3-dev libatlas-base-dev gfortran git libopencv-dev
 
 ADD ci/docker_dependencies.sh .
 RUN ./docker_dependencies.sh
 
 RUN pip install -U pip && pip install --extra-index-url https://www.piwheels.org/simple/ --prefer-binary opencv-python
 
+# Copy over the files
 COPY . /depthai-python
+
+# Install C++ library
+RUN cmake -S /depthai-python/depthai-core -B /build -D CMAKE_BUILD_TYPE=Release -D BUILD_SHARED_LIBS=ON -D CMAKE_INSTALL_PREFIX=/usr/local
+RUN cmake --build /build --parallel 4 --config Relase --target install
+
+# Install Python library
 RUN cd /depthai-python && python3 -m pip install .

depthai-core

@@ -32,6 +32,8 @@
 For the 'Select control: ...' options, use these keys to modify the value:
   '-' or '_' to decrease
   '+' or '=' to increase
+
+'/' to toggle showing camera settings: exposure, ISO, lens position, color temperature
 """
 
 import depthai as dai
@@ -117,6 +119,7 @@ def clamp(num, v0, v1):
     luma_denoise = 0
     chroma_denoise = 0
     control = 'none'
+    show = False
 
     awb_mode = cycle([item for name, item in vars(dai.CameraControl.AutoWhiteBalanceMode).items() if name.isupper()])
     anti_banding_mode = cycle([item for name, item in vars(dai.CameraControl.AntiBandingMode).items() if name.isupper()])
@@ -129,6 +132,13 @@ def clamp(num, v0, v1):
 
         ispFrames = ispQueue.tryGetAll()
         for ispFrame in ispFrames:
+            if show:
+                txt = f"[{ispFrame.getSequenceNum()}] "
+                txt += f"Exposure: {ispFrame.getExposureTime().total_seconds()*1000:.3f} ms, "
+                txt += f"ISO: {ispFrame.getSensitivity()}, "
+                txt += f"Lens position: {ispFrame.getLensPosition()}, "
+                txt += f"Color temp: {ispFrame.getColorTemperature()} K"
+                print(txt)
             cv2.imshow('isp', ispFrame.getCvFrame())
 
             # Send new cfg to camera
@@ -150,6 +160,9 @@ def clamp(num, v0, v1):
         key = cv2.waitKey(1)
         if key == ord('q'):
             break
+        elif key == ord('/'):
+            show = not show
+            if not show: print("Printing camera settings: OFF")
         elif key == ord('c'):
             ctrl = dai.CameraControl()
             ctrl.setCaptureStill(True)

examples/ColorCamera/rgb_camera_control.py

@@ -26,6 +26,9 @@
     print('Connected cameras: ', device.getConnectedCameras())
     # Print out usb speed
     print('Usb speed: ', device.getUsbSpeed().name)
+    # Bootloader version
+    if device.getBootloaderVersion() is not None:
+        print('Bootloader version: ', device.getBootloaderVersion())
 
     # Output queue will be used to get the rgb frames from the output defined above
     qRgb = device.getOutputQueue(name="rgb", maxSize=4, blocking=False)

examples/ColorCamera/rgb_preview.py

@@ -0,0 +1,88 @@
+import cv2
+import depthai as dai
+import numpy as np
+
+camRes = dai.ColorCameraProperties.SensorResolution.THE_1080_P
+camSocket = dai.CameraBoardSocket.RGB
+ispScale = (1,2)
+
+def getMesh(calibData, ispSize):
+    M1 = np.array(calibData.getCameraIntrinsics(camSocket, ispSize[0], ispSize[1]))
+    d1 = np.array(calibData.getDistortionCoefficients(camSocket))
+    R1 = np.identity(3)
+    mapX, mapY = cv2.initUndistortRectifyMap(M1, d1, R1, M1, ispSize, cv2.CV_32FC1)
+
+    meshCellSize = 16
+    mesh0 = []
+    # Creates subsampled mesh which will be loaded on to device to undistort the image
+    for y in range(mapX.shape[0] + 1): # iterating over height of the image
+        if y % meshCellSize == 0:
+            rowLeft = []
+            for x in range(mapX.shape[1]): # iterating over width of the image
+                if x % meshCellSize == 0:
+                    if y == mapX.shape[0] and x == mapX.shape[1]:
+                        rowLeft.append(mapX[y - 1, x - 1])
+                        rowLeft.append(mapY[y - 1, x - 1])
+                    elif y == mapX.shape[0]:
+                        rowLeft.append(mapX[y - 1, x])
+                        rowLeft.append(mapY[y - 1, x])
+                    elif x == mapX.shape[1]:
+                        rowLeft.append(mapX[y, x - 1])
+                        rowLeft.append(mapY[y, x - 1])
+                    else:
+                        rowLeft.append(mapX[y, x])
+                        rowLeft.append(mapY[y, x])
+            if (mapX.shape[1] % meshCellSize) % 2 != 0:
+                rowLeft.append(0)
+                rowLeft.append(0)
+
+            mesh0.append(rowLeft)
+
+    mesh0 = np.array(mesh0)
+    meshWidth = mesh0.shape[1] // 2
+    meshHeight = mesh0.shape[0]
+    mesh0.resize(meshWidth * meshHeight, 2)
+
+    mesh = list(map(tuple, mesh0))
+
+    return mesh, meshWidth, meshHeight
+
+def create_pipeline(calibData):
+    pipeline = dai.Pipeline()
+
+    cam = pipeline.create(dai.node.ColorCamera)
+    cam.setIspScale(ispScale)
+    cam.setBoardSocket(camSocket)
+    cam.setResolution(camRes)
+
+    manip = pipeline.create(dai.node.ImageManip)
+    mesh, meshWidth, meshHeight = getMesh(calibData, cam.getIspSize())
+    manip.setWarpMesh(mesh, meshWidth, meshHeight)
+    manip.setMaxOutputFrameSize(cam.getIspWidth() * cam.getIspHeight() * 3 // 2)
+    cam.isp.link(manip.inputImage)
+
+    cam_xout = pipeline.create(dai.node.XLinkOut)
+    cam_xout.setStreamName("Undistorted")
+    manip.out.link(cam_xout.input)
+
+    dist_xout = pipeline.create(dai.node.XLinkOut)
+    dist_xout.setStreamName("Distorted")
+    cam.isp.link(dist_xout.input)
+
+    return pipeline
+
+with dai.Device() as device:
+
+    calibData = device.readCalibration()
+    pipeline = create_pipeline(calibData)
+    device.startPipeline(pipeline)
+
+    queues = [device.getOutputQueue(name, 4, False) for name in ['Undistorted', 'Distorted']]
+
+    while True:
+        for q in queues:
+            frame = q.get().getCvFrame()
+            cv2.imshow(q.getName(), frame)
+
+        if cv2.waitKey(1) == ord('q'):
+            break

examples/ColorCamera/rgb_undistort.py

@@ -55,8 +55,8 @@ def timeDeltaToMilliS(delta) -> float:
             acceleroValues = imuPacket.acceleroMeter
             gyroValues = imuPacket.gyroscope
 
-            acceleroTs = acceleroValues.timestamp.get()
-            gyroTs = gyroValues.timestamp.get()
+            acceleroTs = acceleroValues.getTimestampDevice()
+            gyroTs = gyroValues.getTimestampDevice()
             if baseTs is None:
                 baseTs = acceleroTs if acceleroTs < gyroTs else gyroTs
             acceleroTs = timeDeltaToMilliS(acceleroTs - baseTs)

examples/IMU/imu_firmware_update.py

@@ -46,8 +46,8 @@ def timeDeltaToMilliS(delta) -> float:
             acceleroValues = imuPacket.acceleroMeter
             gyroValues = imuPacket.gyroscope
 
-            acceleroTs = acceleroValues.timestamp.get()
-            gyroTs = gyroValues.timestamp.get()
+            acceleroTs = acceleroValues.getTimestampDevice()
+            gyroTs = gyroValues.getTimestampDevice()
             if baseTs is None:
                 baseTs = acceleroTs if acceleroTs < gyroTs else gyroTs
             acceleroTs = timeDeltaToMilliS(acceleroTs - baseTs)

examples/IMU/imu_gyroscope_accelerometer.py

@@ -43,7 +43,7 @@ def timeDeltaToMilliS(delta) -> float:
         for imuPacket in imuPackets:
             rVvalues = imuPacket.rotationVector
 
-            rvTs = rVvalues.timestamp.get()
+            rvTs = rVvalues.getTimestampDevice()
             if baseTs is None:
                 baseTs = rvTs
             rvTs = rvTs - baseTs

examples/IMU/imu_rotation_vector.py

@@ -12,7 +12,7 @@
 maxFrameSize = camRgb.getPreviewWidth() * camRgb.getPreviewHeight() * 3
 
 # Warp preview frame 1
-manip1 = pipeline.create(dai.node.ImageManip)
+manip1 = pipeline.create(dai.node.Warp)
 # Create a custom warp mesh
 tl = dai.Point2f(20, 20)
 tr = dai.Point2f(460, 20)

examples/ImageManip/image_manip_warp_mesh.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+
+import cv2
+import depthai as dai
+
+if 1:  # PoE config
+    fps = 30
+    res = dai.MonoCameraProperties.SensorResolution.THE_400_P
+    poolSize = 24  # default 3, increased to prevent desync
+else:  # USB
+    fps = 30
+    res = dai.MonoCameraProperties.SensorResolution.THE_720_P
+    poolSize = 8  # default 3, increased to prevent desync
+
+# Create pipeline
+pipeline = dai.Pipeline()
+
+# Define sources and outputs
+monoL = pipeline.create(dai.node.MonoCamera)
+monoR = pipeline.create(dai.node.MonoCamera)
+
+monoL.setBoardSocket(dai.CameraBoardSocket.LEFT)
+monoL.setResolution(res)
+monoL.setFps(fps)
+monoL.setNumFramesPool(poolSize)
+monoR.setBoardSocket(dai.CameraBoardSocket.RIGHT)
+monoR.setResolution(res)
+monoR.setFps(fps)
+monoR.setNumFramesPool(poolSize)
+
+xoutDotL = pipeline.create(dai.node.XLinkOut)
+xoutDotR = pipeline.create(dai.node.XLinkOut)
+xoutFloodL = pipeline.create(dai.node.XLinkOut)
+xoutFloodR = pipeline.create(dai.node.XLinkOut)
+
+xoutDotL.setStreamName('dot-left')
+xoutDotR.setStreamName('dot-right')
+xoutFloodL.setStreamName('flood-left')
+xoutFloodR.setStreamName('flood-right')
+streams = ['dot-left', 'dot-right', 'flood-left', 'flood-right']
+
+# Script node for frame routing and IR dot/flood alternate
+script = pipeline.create(dai.node.Script)
+script.setProcessor(dai.ProcessorType.LEON_CSS)
+script.setScript("""
+    dotBright = 500  # Note: recommended to not exceed 765, for max duty cycle
+    floodBright = 200
+    LOGGING = False  # Set `True` for latency/timings debugging
+
+    node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')
+
+    flagDot = False
+    while True:
+        # Wait first for a frame event, received at MIPI start-of-frame
+        event = node.io['event'].get()
+        if LOGGING: tEvent = Clock.now()
+
+        # Immediately reconfigure the IR driver.
+        # Note the logic is inverted, as it applies for next frame
+        Device.setIrLaserDotProjectorBrightness(0 if flagDot else dotBright)
+        Device.setIrFloodLightBrightness(floodBright if flagDot else 0)
+        if LOGGING: tIrSet = Clock.now()
+
+        # Wait for the actual frames (after MIPI capture and ISP proc is done)
+        frameL = node.io['frameL'].get()
+        if LOGGING: tLeft = Clock.now()
+        frameR = node.io['frameR'].get()
+        if LOGGING: tRight = Clock.now()
+
+        if LOGGING:
+            latIR      = (tIrSet - tEvent               ).total_seconds() * 1000
+            latEv      = (tEvent - event.getTimestamp() ).total_seconds() * 1000
+            latProcL   = (tLeft  - event.getTimestamp() ).total_seconds() * 1000
+            diffRecvRL = (tRight - tLeft                ).total_seconds() * 1000
+            node.warn(f'T[ms] latEv:{latEv:5.3f} latIR:{latIR:5.3f} latProcL:{latProcL:6.3f} '
+                    + f' diffRecvRL:{diffRecvRL:5.3f}')
+
+        # Sync checks
+        diffSeq = frameL.getSequenceNum() - event.getSequenceNum()
+        diffTsEv = (frameL.getTimestamp() - event.getTimestamp()).total_seconds() * 1000
+        diffTsRL = (frameR.getTimestamp() - frameL.getTimestamp()).total_seconds() * 1000
+        if diffSeq or diffTsEv or (abs(diffTsRL) > 0.8):
+            node.error(f'frame/event desync! Fr-Ev: {diffSeq} frames,'
+                    + f' {diffTsEv:.3f} ms; R-L: {diffTsRL:.3f} ms')
+
+        # Route the frames to their respective outputs
+        node.io['dotL' if flagDot else 'floodL'].send(frameL)
+        node.io['dotR' if flagDot else 'floodR'].send(frameR)
+
+        flagDot = not flagDot
+""")
+
+# Linking
+monoL.frameEvent.link(script.inputs['event'])
+monoL.out.link(script.inputs['frameL'])
+monoR.out.link(script.inputs['frameR'])
+
+script.outputs['dotL'].link(xoutDotL.input)
+script.outputs['dotR'].link(xoutDotR.input)
+script.outputs['floodL'].link(xoutFloodL.input)
+script.outputs['floodR'].link(xoutFloodR.input)
+
+# Connect to device and start pipeline
+with dai.Device(pipeline) as device:
+    queues = [device.getOutputQueue(name=s, maxSize=4, blocking=False) for s in streams]
+
+    while True:
+        for q in queues:
+            pkt = q.tryGet()
+            if pkt is not None:
+                name = q.getName()
+                frame = pkt.getCvFrame()
+                cv2.imshow(name, frame)
+
+        if cv2.waitKey(5) == ord('q'):
+            break