Regarding zone problem #1016
Description
Search before asking
- I have searched the Supervision issues and found no similar feature requests.
Question
Currently, learning two yolov8 model, one for person detection other for object detection : Main problem is for automated selfcheckout based on zone logic : where we check weather person holding object crossing zone from left toward right or right towards left and then prepare recipt accordingly. Need guidance in logic : in this case, should I have to combine detection for person and object or should handle logic alternately ?
Below Code :
#Define empty lists to keep track of labels
original_labels = []
final_labels = []
person_bbox = []
p_items = []
purchased_items = set(p_items)
a_items = []
added_items = set(a_items)
hand_bbox = []
combined_detections = []
#Save result as det_tracking_result
with sv.VideoSink("new_det_tracking_result.mp4", video_info) as sink:
#Iterate through model predictions and tracking results
for index, (result, result1) in enumerate(zip(model.track(source=VID_PATH, show=False, stream=True, verbose=True, persist=True),
model1.track(source=VID_PATH, show=False, stream=True, verbose=True, persist=True))):
#Define variables to store interactions that are refreshed per frame
interactions = []
person_intersection_str = ""
# Obtain predictions from model1
frame1 = result1.orig_img
detections_objects1 = sv.Detections.from_ultralytics(result1)
detections_objects1 = detections_objects1[detections_objects1.class_id == 0]
bboxes1 = result1.boxes
#print(detections_objects1)
#Obtain predictions from yolov8 model
frame = result.orig_img
detections = sv.Detections.from_ultralytics(result)
detections = detections[detections.class_id < 10]
bboxes = result.boxes
# Apply mask over the single Zone
mask1, mask2 = zone.trigger(detections=detections_objects1), zone.trigger(detections=detections)
detections_filtered1, detections_filtered2 = detections_objects1[mask1], detections[mask2]
if detections_objects1 and len(detections_objects1) > 0:
label1 = label_map1[detections_objects1.class_id[0]] # Get the label for the class_id
combined_detections.append((detections_objects1, label1))
for detection, label in combined_detections:
print("Detections:", detection)
print("Label:", label)
if bboxes1.id is not None:
detections_objects1.tracker_id = bboxes1.id.cpu().numpy().astype(int)
labels = [
f'#{tracker_id} {label_map1[class_id]} {confidence:0.2f}'
for _, _, confidence, class_id, tracker_id
in detections_objects1
]
#Print labels for detections from model1
for _, _, confidence, class_id, _ in detections_objects1:
print(f"Label: {label_map1[class_id]} with confidence: {confidence:.2f}")
print(detections)
# Apply mask over the single Zone
mask = zone.trigger(detections=detections)
detections_filtered = detections[mask]
print("mask", mask)
print("Detection", detections_filtered)
if detections and len(detections) > 0:
label = label_map[detections.class_id[0]] # Get the label for the class_id
combined_detections.append((detections, label))
if bboxes.id is not None:
detections.tracker_id = bboxes.id.cpu().numpy().astype(int)
labels = [
f'#{tracker_id} {label_map[class_id]} {confidence:0.2f}'
for _, _, confidence, class_id, tracker_id
in detections
]
frame = box_annotator.annotate(scene=frame, detections=detections_filtered, labels=labels)
frame = zone_annotator.annotate(scene=frame)
objects = [f'#{tracker_id} {label_map[class_id]}' for _, _, confidence, class_id, tracker_id in detections]
# for _, _, confidence, class_id, _ in detections:
# print(f"Label: {label_map[class_id]} with confidence: {confidence:.2f}")
# # Combine detections from both models
# # combined_detections = np.concatenate((detections_objects1, detections))
# print(combined_detections)
# # Extract xyxy attributes from combined detections
# combined_detections_xyxy = [detection[0].xyxy for detection in combined_detections]
# print(combined_detections_xyxy)
# # Check if combined_detections_xyxy is not empty and contains non-empty arrays
# if combined_detections_xyxy and all(arr.size > 0 for arr in combined_detections_xyxy):
# # Concatenate xyxy arrays into a single array
# combined_xyxy_array = np.concatenate(combined_detections_xyxy, axis=0)
# else:
# combined_xyxy_array = np.empty((0, 4)) # Create an empty array
# # Create a Detections object with the concatenated xyxy array
# combined_detections_detections = sv.Detections(xyxy=combined_xyxy_array)
# # Apply mask over the combined detections
# mask = zone.trigger(detections= combined_detections_detections)
# # Filter combined detections based on the mask
# combined_detections_filtered = [combined_detections[i] for i in range(len(combined_detections)) if mask[i]]
# # Print the mask and filtered detections
# #print("Combined Detections mask:", mask)
# #print("Combined Detections filtered:", combined_detections_filtered)
# # Iterate through combined detections to create labels
# combined_labels = []
# for detection in combined_detections_filtered:
# detections, label = detection
# for _, _, confidence, class_id, tracker_id in detections:
# combined_labels.append(f'#{tracker_id} {label_map1[class_id]} {confidence:.2f}')
# # Print labels for combined detections
# for label in combined_labels:
# print("combined_labels", label)
# frame = box_annotator.annotate(scene=frame, detections=combined_detections_filtered, labels=combined_labels)
# frame = zone_annotator.annotate(scene=frame)
# objects = [f'#{tracker_id} {label_map[class_id]}' for _, _, confidence, class_id, tracker_id in combined_detections_filtered]
# print("Combined Objects:", objects)
#If this is the first time we run the application,
#store the objects' labels as they are at the beginning
if index == 0:
original_labels = objects
original_dets = len(detections_filtered)
else:
#To identify if an object has been added or removed
#we'll use the original labels and identify any changes
final_labels = objects
new_dets = len(detections_filtered)
#Identify if an object has been added or removed using Counters
removed_objects = Counter(original_labels) + Counter(final_labels)
added_objects = Counter(final_labels) - Counter(original_labels)
#Create two variables we can increment for drawing text
draw_txt_ir = 1
draw_txt_ia = 1
#Check for objects being added or removed
#if new_dets - original_dets != 0 and len(removed_objects) >= 1:
if new_dets != original_dets or removed_objects:
#An object has been removed
for k,v in removed_objects.items():
#For each of the objects, check the IOU between a designated object
#and a person.
if 'person' not in k:
removed_object_str = f"{v} {k} purchased"
removed_action_str = intersecting_bboxes(bboxes, bboxes1, person_bbox, removed_object_str)
print("Removed Action String:", removed_action_str) # Add this line
if removed_action_str is not None:
log.info(removed_action_str)
#Add the purchased items to a "receipt" of sorts
item = removed_action_str.split()
if len(item) >= 3:
item = f"{item [0]} {item [1]} {item [2]}"
removed_label = item.split(' ')[-1]
if any(removed_label in item for item in purchased_items):
purchased_items = {f"{int(item.split()[0]) + 1} {' '.join(item.split()[1:])}" if removed_label in item else item for item in purchased_items}
else:
purchased_items.add(f"{v} {k}")
p_items.append(f" - {v} {k}")
print("New_Purchased_Items:", purchased_items)
print("Removed_Objects:")
#Draw the result on the screen
draw_text(frame, text=removed_action_str, point=(50, 50 + draw_txt_ir), color=(0, 0, 255))
draw_text(frame, "Receipt: " + str(purchased_items), point=(50, 800), color=(30, 144, 255))
draw_txt_ir += 80
if len(added_objects) >= 1:
#An object has been added
for k,v in added_objects.items():
#For each of the objects, check the IOU between a designated object
#and a person.
if 'person' not in k:
added_object_str = f"{v} {k} returned"
added_action_str = intersecting_bboxes(bboxes, bboxes1, person_bbox, added_object_str)
print("Added Action String:", added_action_str) # Add this line
if added_action_str is not None:
#If we have determined an interaction with a person,
#log the interaction.
log.info(added_action_str)
item = added_object_str.split()
if len(item) >= 3:
item = f"{item [0]} {item [1]} {item [2]}"
item = item.split(' ')[-1]
if any(item in item for item in purchased_items):
purchased_items = {f"{int(item.split()[0]) - 1} {' '.join(item.split()[1:])}" if item in item else item for item in purchased_items}
if any(item.startswith('0 ') for item in purchased_items):
purchased_items = {item for item in purchased_items if not item.startswith('0 ')}
print("Updated_Purchased_Items:", purchased_items)
#p_items.remove(item)
added_items.add(added_object_str)
a_items.append(added_object_str)
print("Added_Objects:")
#Draw the result on the screen
draw_text(frame, text=added_action_str, point=(50, 300 + draw_txt_ia), color=(0, 128, 0))
draw_text(frame, "Receipt: " + str(purchased_items), point=(50, 800), color=(30, 144, 255))
draw_txt_ia += 80
# Clear the combined_detections list
combined_detections.clear()
draw_text(frame, "Receipt: " + str(purchased_items), point=(50, 800), color=(30, 144, 255))
sink.write_frame(frame)
Additional
No response