Low accuracy on lcc-fasd

[stakanich]

Hi. I'm trying to evaluate your model, saved in .onnx format from your drive:
https://drive.google.com/drive/folders/1E1OovqRMEQD_uFIhTDU05efvq3KwwnPE
I use custom face detector for cutting faces from image. It works well.
Here is my preprocessing of face for anti-spuffing:

def preprocess_spoof(imgs, normalize_mean = [0.53875615, 0.45938787, 0.45437699], 
                         normalize_std = [0.28414759, 0.27720253, 0.27737352], 
                         resize_shape = (128, 128), 
                         flipbgr = True):
        normalize_mean = np.array(normalize_mean)
        normalize_std = np.array(normalize_std)
        res = []
        for idx, img in enumerate(imgs):
            #print(img.shape)
            img = cv2.resize(img.astype(np.float32), dsize=resize_shape,
                                     interpolation=cv2.INTER_CUBIC)
            if flipbgr:
                img = cv2.cvtColor(img.astype('float32'),
                                       cv2.COLOR_BGR2RGB)
            img = (img/255 - normalize_mean) / normalize_std
            
            img = np.transpose(img, (2, 0, 1))
            res.append(img)
        return res

I use mean and std counted from train part of dataset.

Then I evaluate model with this code:

TP = 0
FP = 0
TN = 0
FN = 0

total_time = 0
P_probs = []
N_probs = []

detector_errors_P = 0
detector_errors_N = 0

with torch.no_grad():
    for idx,img_name in enumerate(listOfImageNamesUnspoof):
        img = cv2.imread(img_name)

        start = time.time()
        bbox = get_box(img)
        if len(bbox) == 0:
            detector_errors_P += 1
            continue
        face = cutout_bbox(img, bbox)
        prediction = sess.run(None, {'actual_input_1': preprocess_spoof([face])})
        speed = time.time()-start
        #print(prediction[0])
        
        
        #label = np.argmax(prediction[0][0])
        if prediction[0][0][0] > 0.4:
            label = 0
        else:
            label = 1
        value = prediction[0][0][label]

        if label == 0:
            TP += 1
            P_probs.append(value)
        else:
            FN += 1

        total_time += speed

    print("Real images analysis finished")

    P_average_prob = np.mean(P_probs)
    P_min_prob = np.min(P_probs)

    for idx,img_name in enumerate(listOfImageNamesSpoof):
        img = cv2.imread(img_name)

        start = time.time()
        bbox = get_box(img)
        if len(bbox) == 0:
            detector_errors_N += 1
            continue
        face = cutout_bbox(img, bbox)
        prediction = sess.run(None, {'actual_input_1': preprocess_spoof([face])})
        speed = time.time()-start
        #print(prediction[0])
        
        #label = np.argmax(prediction[0][0])
        if prediction[0][0][0] > 0.4:
            label = 0
        else:
            label = 1
        value = prediction[0][0][label]

        if label == 0:
            FP += 1
        else:
            TN += 1
            N_probs.append(value)

        total_time += speed


    N_average_prob = np.mean(N_probs)
    N_min_prob = np.min(N_probs)

    print(f"""
    Total real: {TP + FN}
    TP: {TP} - {TP/(TP + FN)}, FN: {FN} - {FN/(TP + FN)}
    average proba for real: {P_average_prob}, min proba for real: {P_min_prob}
    detector fails on real: {detector_errors_P}
    Total fake: {TN + FP}, TN: {TN} - {TN/(TN + FP)}, FP: {FP} - {FP/(TN + FP)}
    average proba for fake: {N_average_prob}, min proba for fake: {N_min_prob}
    detector fails on fake: {detector_errors_N}

    average_time: {total_time / (TP+TN+FP+FN)}
    """)

And I get:

    Total real: 323
    TP: 253 - 0.7832817337461301, FN: 70 - 0.21671826625386997
    average proba for real: 0.9243021607398987, min proba for real: 0.4475395083427429
    detector fails on real: 0
    Total fake: 7298, TN: 6408 - 0.8780487804878049, FP: 890 - 0.12195121951219512
    average proba for fake: 0.9745228290557861, min proba for fake: 0.6035735607147217
    detector fails on fake: 14

    average_time: 0.012271973548982263

So,it looks like even with low enough threshold real images are counted as fake. Can you give an advise, please: is it a problem with the pretrained model or I do something wrong?
Thanks in advance.