Processing of same frame multiple times

[revathib01]

Hi,

I'm using Allied vision USB camera as source for my model(trained with YOLOv5x ). As the camera is Gencam it is not supporting Cv2 directly, so I have to access the camera through their library(vimba) and convert the frame into cv2 format for processing. I'm facing an issue, in between the model is processing the same image 2 to 3 times and this is making our model slower. Can you lets know why is the model is processing the same image 2 to 3 times.

Below is the modification from /yolov5/utilis/datasets.py in class LoadStreams
`

def __init__(self, sources='streams.txt', img_size=640, stride=32, auto=True):
    self.mode = 'stream'
    self.img_size = img_size
    self.stride = stride
    
    if os.path.isfile(sources):
        with open(sources) as f:
            sources = [x.strip() for x in f.read().strip().splitlines() if len(x.strip())]
    else:
        sources = [sources]

    n = len(sources)
    self.imgs, self.fps, self.frames, self.threads = [None] * n, [0] * n, [0] * n, [None] * n
    self.sources = [clean_str(x) for x in sources]  # clean source names for later
    self.auto = auto
    for i, s in enumerate(sources):  # index, source
        # Start thread to read frames from video stream
        st = f'{i + 1}/{n}: {s}... '
        if 'youtube.com/' in s or 'youtu.be/' in s:  # if source is YouTube video
            check_requirements(('pafy', 'youtube_dl'))
            import pafy
            s = pafy.new(s).getbest(preftype="mp4").url  # YouTube URL
        if((s=="AV0")or(s=="AV1")):
            #print(s[2])
            with Vimba. get_instance () as vimba: 
                 cams = vimba.get_all_cameras()
                 with cams[int(s[2])] as cam:
                      #cam.DeviceReset.run()
                      #cam.Width.set(1280)
                      #cam.Height.set(720)
                      #cam.AcquisitionFrameRateEnable.set(True)
                      #cam.AcquisitionFrameRate.set(30)
                      cam.set_pixel_format(PixelFormat .Bgr8)
                      w = int(cam.Width.get())
                      h = int(cam.Height.get())
                      fps = cam.AcquisitionFrameRate.get()  
                      self.frames[i] = float('inf')  # infinite stream fallback
                      self.fps[i] = max((fps if math.isfinite(fps) else 0) % 100, 0) or 30  # 30 FPS fallback
                      frame = cam.get_frame()
                      _, self.imgs[i] = 1,frame.as_opencv_image() # guarantee first frame
                      self.threads[i] = Thread(target=self.Avcam, args=([i,s]),daemon=True)
        else:
            s = eval(s) if s.isnumeric() else s  # i.e. s = '0' local webcam
            cap = cv2.VideoCapture(s)
            assert cap.isOpened(), f'{st}Failed to open {s}'
            w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
            h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
            cap.set(cv2.CAP_PROP_FPS, 20)
            fps = cap.get(cv2.CAP_PROP_FPS)  # warning: may return 0 or nan
            self.frames[i] = max(int(cap.get(cv2.CAP_PROP_FRAME_COUNT)), 0) or float('inf')  # infinite stream fallback
            self.fps[i] = max((fps if math.isfinite(fps) else 0) % 100, 0) or 30  # 30 FPS fallback
            _, self.imgs[i] = cap.read()  # guarantee first frame
            self.threads[i] = Thread(target=self.update, args=([i,cap,s]), daemon=True)
        LOGGER.info(f"{st} Success ({self.frames[i]} frames {w}x{h} at {self.fps[i]:.2f} FPS)")
        self.threads[i].start()
    LOGGER.info('')  # newline
    # check for common shapes
    s = np.stack([letterbox(x, self.img_size, stride=self.stride, auto=self.auto)[0].shape for x in self.imgs])
    self.rect = np.unique(s, axis=0).shape[0] == 1  # rect inference if all shapes equal
    if not self.rect:
        LOGGER.warning('WARNING: Stream shapes differ. For optimal performance supply similarly-shaped streams.')
def Frame_Handler(self,cam: Camera, im: Frame):
    global Ivalue
    ID=cam.get_id()
    x=0
    while Ivalue[x] != ID:
          x=x+1
    index=x
    success=1 if im.get_status() == FrameStatus.Complete else 0
    if success==1:
        self.imgs[index] = im.as_numpy_ndarray()
        #success=0
    cam.queue_frame(im)
def I(self,i,ID):
    global Ivalue
    Ivalue[i]=ID
def Avcam(self,i,stream):
    global lock
    k=int(stream[2])      
    # Read stream `i` frames in daemon thread
    n, f, read = 0, self.frames[i], 1  # frame number, frame array, inference every 'read' frame
    with Vimba. get_instance () as vimba: 
         cams = vimba.get_all_cameras()
         with cams[k] as cam:
              ID=cam.get_id()
              self.I(i,ID)
              try:
                 while CameraEvent(2)==CameraEvent.Reachable and n < f:
                       n += 1
                       t1=time.time()
                       cam.start_streaming(handler=self.Frame_Handler,buffer_count=1)
                       t2=time.time()
                       print("time,cam_id",t2-t1,ID)
                       time.sleep(1 / self.fps[i])  # wait time
                       cam.stop_streaming()
              except KeyboardInterrupt:
                 print("Exception occured ")
                 cam.stop_streaming()        
              finally:
                 cam.stop_streaming()
                 print("Done")
def update(self,i, cap, stream):
    # Read stream `i` frames in daemon thread
    n, f, read = 0, self.frames[i], 1  # frame number, frame array, inference every 'read' frame
    while cap.isOpened() and n < f:
        print("n,f:",n,f)  
        n += 1
        #print("n,f:",n,f)
        # _, self.imgs[index] = cap.read()
        t1=time.time()
        cap.grab()
        t2=time.time()
        if n % read == 0:
             
             success, im = cap.retrieve()
             if success:
                
                print("update,delay :",t2-t1)
                self.imgs[i] = im
             else:
                LOGGER.warning('WARNING: Video stream unresponsive, please check your IP camera connection.')
                self.imgs[i] = np.zeros_like(self.imgs[i])
                cap.open(stream)  # re-open stream if signal was lost
        time.sleep(1 / self.fps[i])  # wait time`

Thanks,
Revathi

[revathib01]

Hi @glenn-jocher can you help us with this issue

[yuyang3478]

这是来自QQ邮箱的假期自动回复邮件。您好，我最近正在休假中，无法亲自回复您的邮件。我将在假期结束后，尽快给您回复。

[ghost]

I guess this is not related with yolo. Your library is giving the same image 2 or 3 times. You need to check from your library.
What you can do is check the md5 of the images. If the image you have now has the same md5 with the previous, do not process it.