HRI code used in Korea

common/foundation_models/lasr_vlm/lasr_vlm/nodes/vlm_service.py

@@ -30,7 +30,7 @@ def __init__(self):
             self.describe_people_callback,
         )
 
-        model_config = ModelConfig(model_name="moondream")
+        model_config = ModelConfig(model_name="gemma3:4b")
         self.vlm = VLMInference(model_config, new_model=False)
         self.get_logger().info("VLM Describe People service started")
 

common/foundation_models/lasr_vlm/lasr_vlm/vlm_inference.py

@@ -137,23 +137,7 @@ def visually_describe_people(input_image, inference: VLMInference) -> dict[str,
     """
     attributes = ["hair_color", "hair_length", "glasses", "hat", "shirt color"]
 
-    user_query = (
-        f"Visually describe the person in the image, using the following attributes:"
-    )
-    for attr in attributes:
-        user_query += f"\n- {attr}"
-    # user_query_example = (
-    #     "\n\n The structure of the response should be a comma separated list of attribute: value pairs. For example, "
-    #     "'hair_color: _, hair_length: _, glasses: _, hat: _, shirt color: _', where the _ is replaced with the model's answer for that attribute. "
-    #     "For true or false attributes, the value should be simply true or false. For example, 'glasses: true' if the model thinks the person is wearing glasses, and 'hat: false' if the model thinks the person is not wearing a hat."
-    # )
-    user_query_example = (
-        "\n\n The structure of the response should be a comma separated list of attribute: value pairs. For example, "
-        "'hair_color: _, hair_length: _, glasses: _, hat: _, shirt color: _', where the _ is replaced with the model's answer for that attribute. "
-        "For glasses and hat, only answer true if they are clearly and visibly present in the image. "
-        "If you are not sure, answer false. For example, 'glasses: false' means the person is definitely not wearing glasses."
-    )
-    user_query += user_query_example
+    user_query = build_prompt(attributes)
 
     print("Running VLM inference query")
     response = inference.query_vision(prompt=user_query, image_path=input_image)
@@ -162,6 +146,27 @@ def visually_describe_people(input_image, inference: VLMInference) -> dict[str,
     return parse_vlm_response(response, attributes)
 
 
+def build_prompt(attributes):
+    template = ", ".join(f"{attr}: _" for attr in attributes)
+
+    boolean_attrs = {"glasses", "hat"}  # extend as needed
+    has_boolean = any(a.lower() in boolean_attrs for a in attributes)
+
+    prompt = (
+        "Describe the person in the image using ONLY this exact format, "
+        "replacing each underscore with your answer. Do not add any other text.\n\n"
+        f"{template}\n"
+    )
+
+    if has_boolean:
+        prompt += (
+            "\nFor glasses and hat: answer true only if clearly visible, "
+            "otherwise answer false.\n"
+        )
+
+    return prompt
+
+
 def parse_vlm_response(response: str, attributes: list[str]) -> dict[str, list]:
     """
     Parse the VLM response string into a dictionary of attribute values.
@@ -211,18 +216,18 @@ def postprocess_value(value: str):
 
 
 def test_vlm_vision_query():
-    model_name = "moondream"  # "gemma3:4b", "qwen2.5vl:3b", "llama3.2"
+    model_name = "gemma3:4b"  # "gemma3:4b", "qwen2.5vl:3b", "llama3.2"
     model_config = ModelConfig(model_name=model_name)
     ensure_model(model_name)
     inference = VLMInference(model_config, new_model=False)
 
-    image_dir = f"{os.getcwd()}/test_images"
-    image_path = f"{image_dir}/person1.jpg"
-
-    response: dict[str, list] = visually_describe_people(
-        input_image=image_path, inference=inference
-    )
-    print(f"Vision response: {response}")
+    image_dir = f"{os.getcwd()}/test_images/more"
+    for image_file in os.listdir(image_dir):
+        image_path = f"{image_dir}/{image_file}"
+        response: dict[str, list] = visually_describe_people(
+            input_image=image_path, inference=inference
+        )
+        print(f"Vision response for {image_file}: {response}")
 
 
 if __name__ == "__main__":

common/foundation_models/lasr_vlm/setup.py

@@ -23,6 +23,7 @@ def run(self):
 
 setup(
     name=package_name,
+    cmdclass={"install": InstallCommand},
     version="0.0.0",
     packages=find_packages(exclude=["test"]),
     data_files=[

common/interfaces/robot_ui/CMakeLists.txt

@@ -0,0 +1,27 @@
+cmake_minimum_required(VERSION 3.8)
+project(robot_ui)
+
+find_package(ament_cmake REQUIRED)
+find_package(rosidl_default_generators REQUIRED)
+
+rosidl_generate_interfaces(${PROJECT_NAME}
+  "msg/Order.msg"
+  "msg/Confirm.msg"
+)
+
+install(PROGRAMS
+  scripts/build
+  scripts/dev
+  scripts/start
+  DESTINATION lib/${PROJECT_NAME}
+)
+
+install(DIRECTORY robot-interface
+  DESTINATION share/${PROJECT_NAME}
+)
+
+install(DIRECTORY launch
+  DESTINATION share/${PROJECT_NAME}
+)
+
+ament_package()

common/interfaces/robot_ui/launch/robot_ui.launch.py

@@ -0,0 +1,65 @@
+import os
+from ament_index_python.packages import get_package_share_directory
+from launch import LaunchDescription
+from launch.actions import DeclareLaunchArgument, ExecuteProcess, TimerAction
+from launch.substitutions import LaunchConfiguration
+from launch_ros.actions import Node
+
+
+def generate_launch_description():
+    ros_ip_arg = DeclareLaunchArgument(
+        "ros_ip",
+        default_value="10.68.0.139",
+        description="IP address rosbridge will bind to (also passed to the Next.js UI as ROS_IP)",
+    )
+
+    port_arg = DeclareLaunchArgument(
+        "port",
+        default_value="9090",
+        description="Port for the rosbridge WebSocket server",
+    )
+
+    ros_ip = LaunchConfiguration("ros_ip")
+    port = LaunchConfiguration("port")
+
+    rosbridge = Node(
+        package="rosbridge_server",
+        executable="rosbridge_websocket",
+        name="rosbridge_websocket",
+        parameters=[
+            {
+                "port": port,
+                "address": ros_ip,
+            }
+        ],
+    )
+
+    ui_dir = os.path.join(
+        get_package_share_directory("robot_ui"),
+        "robot-interface",
+    )
+
+    next_server = TimerAction(
+        period=2.0,
+        actions=[
+            ExecuteProcess(
+                cmd=[
+                    "bash",
+                    "-c",
+                    "npm install && chmod +x node_modules/.bin/next && node_modules/.bin/next build && node_modules/.bin/next start",
+                ],
+                cwd=ui_dir,
+                additional_env={"ROS_IP": ros_ip},
+                output="screen",
+            )
+        ],
+    )
+
+    return LaunchDescription(
+        [
+            ros_ip_arg,
+            port_arg,
+            rosbridge,
+            next_server,
+        ]
+    )

common/interfaces/robot_ui/msg/Confirm.msg

@@ -0,0 +1 @@
+bool value

common/interfaces/robot_ui/msg/Order.msg

@@ -0,0 +1 @@
+string[] products

common/interfaces/robot_ui/package.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<?xml-model href="http://download.ros.org/schema/package_format3.xsd" schematypens="http://www.w3.org/2001/XMLSchema"?>
+<package format="3">
+  <name>robot_ui</name>
+  <version>0.0.0</version>
+  <description>Provides a tablet user interface for ordering and confirming actions within the coffee shop task.</description>
+  <maintainer email="jared.swift@kcl.ac.uk">Jared Swift</maintainer>
+  <license>MIT</license>
+
+  <buildtool_depend>ament_cmake</buildtool_depend>
+  <buildtool_depend>rosidl_default_generators</buildtool_depend>
+
+  <exec_depend>rosidl_default_runtime</exec_depend>
+
+  <member_of_group>rosidl_interface_packages</member_of_group>
+
+  <export>
+    <build_type>ament_cmake</build_type>
+  </export>
+</package>

common/interfaces/robot_ui/robot-interface/.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "next/core-web-vitals"
+}

common/interfaces/robot_ui/robot-interface/app/globals.css

@@ -0,0 +1,3 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;

common/interfaces/robot_ui/robot-interface/app/layout.tsx

@@ -0,0 +1,22 @@
+import "./globals.css";
+import type { Metadata } from "next";
+import { Inter } from "next/font/google";
+
+const inter = Inter({ subsets: ["latin"] });
+
+export const metadata: Metadata = {
+  title: "TIAGo",
+  description: "this is the description of the page",
+};
+
+export default function RootLayout({
+  children,
+}: {
+  children: React.ReactNode;
+}) {
+  return (
+    <html lang="en">
+      <body className={inter.className}>{children}</body>
+    </html>
+  );
+}

common/interfaces/robot_ui/robot-interface/app/page.tsx

@@ -0,0 +1,16 @@
+import dynamic from "next/dynamic";
+
+const App = dynamic(() => import("@/components/App"), {
+  ssr: false,
+  loading: () => <h1 className="text-9xl p-4">Loading...</h1>,
+});
+
+export default function Home() {
+  console.info("Advertising", process.env.ROS_IP, "to clients.");
+
+  return (
+    <main className="select-none">
+      <App rosIp={process.env.ROS_IP as string} />
+    </main>
+  );
+}

common/interfaces/robot_ui/robot-interface/components/App.tsx

@@ -0,0 +1,121 @@
+"use client";
+
+import { useEffect, useRef, useState } from "react";
+import { Message, Ros, Topic } from "roslib";
+
+import { CreateOrder } from "./screens/CreateOrder";
+import { Done } from "./screens/Done";
+import { Ready } from "./screens/Ready";
+import { YesNo } from "./screens/YesNo";
+
+type Screen = "home" | "order" | "done" | "ready" | "yes_no" | "debug";
+type ProductTopic = { products: string[] };
+type ConfirmTopic = { value: boolean };
+
+export default function App({ rosIp }: { rosIp: string }) {
+  const [ros, setROS] = useState<Ros>();
+  const [screen, setScreen] = useState<Screen>("home");
+
+  const doneTopicRef = useRef<Topic>();
+  const readyTopicRef = useRef<Topic>();
+  const orderTopicRef = useRef<Topic<ProductTopic>>();
+  const confirmTopicRef = useRef<Topic<ConfirmTopic>>();
+
+  useEffect(() => {
+    const ros = new Ros({
+      url: rosIp ? `ws://${rosIp}:9090` : "ws://127.0.0.1:9090",
+    });
+
+    ros.on("connection", () => {
+      setROS(ros);
+
+      const screenTopic = new Topic<{ data: Screen }>({
+        ros,
+        name: "/tablet/screen",
+        messageType: "std_msgs/msg/String",
+      });
+      screenTopic.subscribe((message) => setScreen(message.data));
+
+      doneTopicRef.current = new Topic({
+        ros,
+        name: "/tablet/done",
+        messageType: "std_msgs/msg/Empty",
+      });
+      doneTopicRef.current.advertise();
+
+      readyTopicRef.current = new Topic({
+        ros,
+        name: "/tablet/ready",
+        messageType: "std_msgs/msg/Empty",
+      });
+      readyTopicRef.current.advertise();
+
+      orderTopicRef.current = new Topic<ProductTopic>({
+        ros,
+        name: "/tablet/order",
+        messageType: "robot_ui/msg/Order",
+      });
+      orderTopicRef.current.advertise();
+
+      confirmTopicRef.current = new Topic<ConfirmTopic>({
+        ros,
+        name: "/tablet/confirm",
+        messageType: "robot_ui/msg/Confirm",
+      });
+      confirmTopicRef.current.advertise();
+    });
+  }, []);
+
+  if (screen === "order") {
+    return (
+      <CreateOrder
+        finish={(products) => {
+          orderTopicRef.current!.publish(new Message({ products }) as ProductTopic);
+          setScreen("home");
+        }}
+      />
+    );
+  } else if (screen === "done") {
+    return (
+      <Done
+        done={() => {
+          doneTopicRef.current!.publish(new Message({}));
+          setScreen("home");
+        }}
+      />
+    );
+  } else if (screen === "ready") {
+    return (
+      <Ready
+        ready={() => {
+          readyTopicRef.current!.publish(new Message({}));
+          setScreen("home");
+        }}
+      />
+    );
+  } else if (screen === "yes_no") {
+    return (
+      <YesNo
+        yes={() => {
+          confirmTopicRef.current!.publish(new Message({ value: true }) as ConfirmTopic);
+          setScreen("home");
+        }}
+        no={() => {
+          confirmTopicRef.current!.publish(new Message({ value: false }) as ConfirmTopic);
+          setScreen("home");
+        }}
+      />
+    );
+  } else if (screen === "debug") {
+    return <>{ros ? "Connected to ROS =)" : "Connecting to ROS..."}</>;
+  } else {
+    return ros ? (
+      <img
+        src="/tiago.jpg"
+        className="fixed top-0 left-0 w-full h-full object-cover"
+      />
+    ) : (
+      <h1 className="text-9xl p-4">Connecting...</h1>
+    );
+  }
+}