How to smooth the jitter from pose landmarker model?

[ly15927086342]

Have I written custom code (as opposed to using a stock example script provided in MediaPipe)

Yes

OS Platform and Distribution

Mac M1

MediaPipe version

0.10.12

Bazel version

No response

Solution

Pose

Programming Language and version

javascript

Describe the actual behavior

pose landmarker model generate too much jitter

Describe the expected behaviour

smooth the jitter, but maintain the key point

Standalone code/steps you may have used to try to get what you need

I found that the example from https://mediapipe-studio.webapps.google.com/studio/demo/pose_landmarker, did not generate the jitter, but when i use the model locally, it works differently. I found there exists some filter method in mediapipe util directory. Is it work in wasm file? How can i get the smooth pose landmarker directly by using js wasm model?
Expecting your reply. Thank you.

Other info / Complete Logs

No response

[ly15927086342]

is there anybody to answer this issue?

[troy-123]

same problem, landmarks jitter all the time in the output video

import cv2
import time

import mediapipe as mp
from mediapipe import solutions
from mediapipe.tasks import python
from mediapipe.tasks.python import vision
from mediapipe.framework.formats import landmark_pb2
import numpy as np


def draw_landmarks_on_image(rgb_image, detection_result):
  face_landmarks_list = detection_result.face_landmarks
  annotated_image = np.copy(rgb_image)

  # Loop through the detected faces to visualize.
  for idx in range(len(face_landmarks_list)):
    face_landmarks = face_landmarks_list[idx]

    # Draw the face landmarks.
    face_landmarks_proto = landmark_pb2.NormalizedLandmarkList()
    face_landmarks_proto.landmark.extend([
      landmark_pb2.NormalizedLandmark(x=landmark.x, y=landmark.y, z=landmark.z) for landmark in face_landmarks
    ])

    solutions.drawing_utils.draw_landmarks(
        image=annotated_image,
        landmark_list=face_landmarks_proto,
        connections=mp.solutions.face_mesh.FACEMESH_TESSELATION,
        landmark_drawing_spec=None,
        connection_drawing_spec=mp.solutions.drawing_styles
        .get_default_face_mesh_tesselation_style())
    solutions.drawing_utils.draw_landmarks(
        image=annotated_image,
        landmark_list=face_landmarks_proto,
        connections=mp.solutions.face_mesh.FACEMESH_CONTOURS,
        landmark_drawing_spec=None,
        connection_drawing_spec=mp.solutions.drawing_styles
        .get_default_face_mesh_contours_style())
    solutions.drawing_utils.draw_landmarks(
        image=annotated_image,
        landmark_list=face_landmarks_proto,
        connections=mp.solutions.face_mesh.FACEMESH_IRISES,
          landmark_drawing_spec=None,
          connection_drawing_spec=mp.solutions.drawing_styles
          .get_default_face_mesh_iris_connections_style())

  return annotated_image


BaseOptions = mp.tasks.BaseOptions
FaceLandmarker = mp.tasks.vision.FaceLandmarker
FaceLandmarkerOptions = mp.tasks.vision.FaceLandmarkerOptions
VisionRunningMode = mp.tasks.vision.RunningMode

options = FaceLandmarkerOptions(
    base_options=BaseOptions(model_asset_path='./face_landmarker.task'),
    running_mode=VisionRunningMode.VIDEO
    )

with FaceLandmarker.create_from_options(options) as landmarker:
    video = cv2.VideoCapture('./input.mp4')
    frame_width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH))
    frame_height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT))
    # output name
    output_filename = 'output.avi'
    fourcc = cv2.VideoWriter_fourcc(*'XVID')
    out = cv2.VideoWriter(output_filename, fourcc, 30.0, (frame_width, frame_height))
    while True:
        success, frame = video.read()
        if not success:
            print("Ignoring empty camera frame.")
            break
        mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=frame)
        face_landmarker_result = landmarker.detect_for_video(mp_image, mp.Timestamp.from_seconds(time.time()).value)
        annotated_image = draw_landmarks_on_image(mp_image.numpy_view(), face_landmarker_result)
        out.write(annotated_image)
        if cv2.waitKey(5) & 0xFF == 27:
            break
    video.release()
    cv2.destroyAllWindows()