hand_tracking getting a landmarklist

[1Yusi1]

Have I written custom code (as opposed to using a stock example script provided in MediaPipe)

Yes

OS Platform and Distribution

Windows 11

Mobile device if the issue happens on mobile device

No response

Browser and version if the issue happens on browser

No response

Programming Language and version

C++

MediaPipe version

No response

Bazel version

6.1.1

Solution

Hand_tracking

Android Studio, NDK, SDK versions (if issue is related to building in Android environment)

No response

Xcode & Tulsi version (if issue is related to building for iOS)

No response

Describe the actual behavior

The application crashes when poller.Next(&packet) is running on

Describe the expected behaviour

It should just skip rather than process the packet.

Standalone code/steps you may have used to try to get what you need

No

Other info / Complete Logs

I changed the code to get the coordinates of the landmark. It works, but as soon as he loses his palm on the frame, the application immediately breaks.

// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// An example of sending OpenCV webcam frames into a MediaPipe graph.
#define _WINSOCK_DEPRECATED_NO_WARNINGS //[+] 
#include <cstdlib>
#include <iostream>  // [+]

#include <winsock2.h>   // [+]
//#include <WS2tcpip.h >  // [+]
#pragma comment(lib, "wsock32.lib")  // [+]

#include "absl/flags/flag.h"
#include "absl/flags/parse.h"
#include "absl/log/absl_log.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/opencv_video_inc.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/util/resource_util.h"

#include "mediapipe/framework/formats/landmark.pb.h" // [+]

constexpr char kInputStream[] = "input_video";
constexpr char kOutputStream[] = "world_landmarks";   // [*] (there was "output_video")
constexpr char kWindowName[] = "MediaPipe";

ABSL_FLAG(std::string, calculator_graph_config_file, "",
          "Name of file containing text format CalculatorGraphConfig proto.");
ABSL_FLAG(std::string, input_video_path, "",
          "Full path of video to load. "
          "If not provided, attempt to use a webcam.");
ABSL_FLAG(std::string, output_video_path, "",
          "Full path of where to save result (.mp4 only). "
          "If not provided, show result in a window.");

absl::Status RunMPPGraph() {

  std::string calculator_graph_config_contents;
  MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
      absl::GetFlag(FLAGS_calculator_graph_config_file),
      &calculator_graph_config_contents));
  ABSL_LOG(INFO) << "Get calculator graph config contents: "
                 << calculator_graph_config_contents;
  mediapipe::CalculatorGraphConfig config =
      mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(
          calculator_graph_config_contents);

  ABSL_LOG(INFO) << "Initialize the calculator graph.";
  mediapipe::CalculatorGraph graph;
  MP_RETURN_IF_ERROR(graph.Initialize(config));

  ABSL_LOG(INFO) << "Initialize the camera or load the video.";
  cv::VideoCapture capture;
  const bool load_video = !absl::GetFlag(FLAGS_input_video_path).empty();
  if (load_video) {
    capture.open(absl::GetFlag(FLAGS_input_video_path));
  } else {
    capture.open(0);
  }
  RET_CHECK(capture.isOpened());

  cv::VideoWriter writer;
  const bool save_video = !absl::GetFlag(FLAGS_output_video_path).empty();
  if (!save_video) {
    cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1);
#if (CV_MAJOR_VERSION >= 3) && (CV_MINOR_VERSION >= 2)
    capture.set(cv::CAP_PROP_FRAME_WIDTH, 640);
    capture.set(cv::CAP_PROP_FRAME_HEIGHT, 480);
    capture.set(cv::CAP_PROP_FPS, 30);
#endif
  }

  ABSL_LOG(INFO) << "Start running the calculator graph.";
  MP_ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller,
                      graph.AddOutputStreamPoller(kOutputStream));
  MP_RETURN_IF_ERROR(graph.StartRun({}));


  //Init socket [+] winsocket
  WSADATA wsaData;
  if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0) {
      fprintf(stderr, "WSAStartup failed.\n");
      exit(1);
  }
  std::string hostname{ "127.0.0.1" };
  uint16_t port = 8080;

  SOCKET sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
  if (sock == INVALID_SOCKET) {
      std::cerr << "Error create socket: " << WSAGetLastError() << std::endl;
      WSACleanup();
  }

  sockaddr_in addr{};
  memset(&addr, 0, sizeof(addr));
  addr.sin_family = AF_INET;
  addr.sin_port = htons(port);
  addr.sin_addr.s_addr = inet_addr(hostname.c_str());
  //End init  [+] winsocket

  ABSL_LOG(INFO) << "Start grabbing and processing frames.";

  bool grab_frames = true;
  while (grab_frames) {
      // Capture opencv camera or video frame.
      cv::Mat camera_frame_raw;
      capture >> camera_frame_raw;
      //cv::flip(camera_frame_raw, camera_frame_raw, /*flipcode=HORIZONTAL*/ 1);
      if (camera_frame_raw.empty()) {
          ABSL_LOG(INFO) << "Empty frame, end of video reached.";
          break;
      }
      cv::imshow(kWindowName, camera_frame_raw);
      cv::Mat camera_frame;
      cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB);
      if (!load_video) {
          cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1);
      }

      // Wrap Mat into an ImageFrame.
      auto input_frame = absl::make_unique<mediapipe::ImageFrame>(
          mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows,
          mediapipe::ImageFrame::kDefaultAlignmentBoundary);
      cv::Mat input_frame_mat = mediapipe::formats::MatView(input_frame.get());
      camera_frame.copyTo(input_frame_mat);

      // Send image packet into the graph.
      size_t frame_timestamp_us =
          (double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6;
      MP_RETURN_IF_ERROR(graph.AddPacketToInputStream(
          kInputStream, mediapipe::Adopt(input_frame.release())
          .At(mediapipe::Timestamp(frame_timestamp_us))));

      // Get the graph result packet, or stop if that fails.
      mediapipe::Packet packet;
      if (!poller.Next(&packet)) {
          ABSL_LOG(INFO) << "No packet";
          continue;
      } 
      auto& landmarks = packet.Get<std::vector<mediapipe::LandmarkList>>();
      if (!landmarks.empty()) {

          for (const auto& list : landmarks) {
              std::vector<float> cords;
              for (int i = 0; i < list.landmark_size(); i++) {
                  //std::cout << list.landmark(i).x() << " " << list.landmark(i).y() << " " << list.landmark(i).z() << std::endl;
                  cords.push_back(list.landmark(i).x());
                  cords.push_back(list.landmark(i).y());
                  cords.push_back(list.landmark(i).z());

              }
              int result = sendto(sock, reinterpret_cast<const char*>(cords.data()), cords.size() * sizeof(float), 0, reinterpret_cast<sockaddr*> (&addr), sizeof(addr));
              if (result == SOCKET_ERROR) {
                  std::cerr << "Error send:  " << WSAGetLastError() << std::endl;
              }
          }
      }
      else {
          ABSL_LOG(INFO) << "Landmarks empty.";
      }

      const int pressed_key = cv::waitKey(5);
      if (pressed_key >= 0 && pressed_key != 255) grab_frames = false;
  }
  
  closesocket(sock);   //[+] winsosocket
  ABSL_LOG(INFO) << "Shutting down.";
  if (writer.isOpened()) writer.release();
  MP_RETURN_IF_ERROR(graph.CloseInputStream(kInputStream));
  return graph.WaitUntilDone();
}

int main(int argc, char** argv) {
  google::InitGoogleLogging(argv[0]);
  absl::ParseCommandLine(argc, argv);
  absl::Status run_status = RunMPPGraph();
  if (!run_status.ok()) {
    ABSL_LOG(ERROR) << "Failed to run the graph: " << run_status.message();
    return EXIT_FAILURE;
  } else {
    ABSL_LOG(INFO) << "Success!";
  }
  return EXIT_SUCCESS;
}

[kuaashish]

Hi @1Yusi1,

Apologies for the delayed response. Could you kindly provide the complete error log you are receiving? It will help us better understand the issue.

Thank you!!

[1Yusi1]

The console does not report any errors. The program just freezes. Is this possible because I changed the execution graph? For the program, it looks like this:

input_stream: "input_video"

output_stream: "world_landmarks"

// Generates side packet cotaining max number of hands to detect/track.
node {
  calculator: "ConstantSidePacketCalculator"
  output_side_packet: "PACKET:num_hands"
  node_options: {
    [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: {
      packet { int_value: 2 }
    }
  }
}

// Detects/tracks hand landmarks.
node {
  calculator: "HandLandmarkTrackingCpu"
  input_stream: "IMAGE:input_video"
  input_side_packet: "NUM_HANDS:num_hands"
  output_stream: "LANDMARKS:landmarks"   
  output_stream: "WORLD_LANDMARKS:world_landmarks"           
  output_stream: "HANDEDNESS:handedness"
  output_stream: "PALM_DETECTIONS:multi_palm_detections"
  output_stream: "HAND_ROIS_FROM_LANDMARKS:multi_hand_rects"
  output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:multi_palm_rects"
}

[1Yusi1]

#1532
Problem solving