ted_talk_results.py

import os
import json
import glob
import csv
import itertools
import numpy as np
import cPickle as cp
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from list_of_talks import rating_labels

import torch

from TED_data_location import ted_data_path

# ================ Deprecated ===================
# def read_output_log(result_dir = 'SSE_result/',logfile = 'train_logfile.txt'):
#     '''
#     Given a output folder containing the log and model file
#     (generated by the train_model function), this function reads the
#     log and returns the test indices and the model.
#     It also plots the losses as an added convenience.
#     '''
#     inpath = os.path.join(ted_data_path,result_dir)
#     logpath = os.path.join(inpath,logfile)
#     # Open the log file
#     with open(logpath) as fin:
#         alllosses=[]
#         loss_index = -1
#         # Read the log file
#         for aline in fin:
#             if aline.startswith('test_indices'):
#                 # Reading test indices
#                 test_idx = json.loads(aline.strip().split('=')[1])
#             elif aline.startswith('train_indices'):
#                 train_idx = json.loads(aline.strip().split('=')[1])
#             elif aline.startswith('training:'):
#                 # Enlist losses
#                 for items in aline.strip().split(','):
#                     elems = items.split(':')
#                     if elems[0].strip()=='Loss':
#                         alllosses.append(float(elems[1].strip()))
#             elif aline.startswith('model_outfile'):
#                 # Find the model file
#                 modelfile = aline.strip().split('=')[1]
#             else:
#                 print aline.strip()
#         # Save a plot of the loss
#         dest = os.path.join(inpath,'losses.png')
#         # Plot the losses vs. iteration
#         plt.figure(1)
#         plt.clf()
#         plt.semilogy(alllosses)
#         plt.xlabel('Iterations (1 sample per iteration)')
#         plt.ylabel('Loss')
#         plt.savefig(dest)
#         plt.close()
#         # Load the trained model
#         model = torch.load(os.path.join(inpath,modelfile))
#         return test_idx,train_idx,model

def __tonum__(astr):
    try:
        return float(astr)
    except ValueError:
        return astr

def prs(aline):
    retval = {}
    for afield in aline.split(','):
        if ':' in afield:
            k,v = afield.split(':')
            retval.update({k:__tonum__(v)})
    return retval

def __get_rand_id__(filename):
    spltfile = filename.split('.')
    if len(spltfile)>2 and len(spltfile[-2])>=9 and \
        type(__tonum__(spltfile[-2]))==float:
        return int(spltfile[-2])
    else:
        return None

def read_lstm_log(afile,averageonly=True):
    '''
    Reads the contents of LSTM_log files.
    '''
    logdata={}
    with open(afile) as fin:
        logdata['id']=__get_rand_id__(afile)
        for aline in fin:
            aline = aline.strip()
            if '=' in aline and not 'count' in aline:
                k,v = aline.split('=')                    
                logdata[k]=__tonum__(v)
            elif averageonly and aline.startswith('train'):
                traintest = 'train'
                linedic = prs(aline)
                time = linedic['iter_time']                
                itno = int(linedic['train'])
            elif averageonly and aline.startswith('test'):
                traintest = 'test'
                time = linedic['iter_time']
                itno = int(linedic['train'])
            elif averageonly and aline.startswith('Average loss'):
                avgloss = float(aline.split(':')[-1])
                logdata.setdefault(traintest,[]).append([itno,time,avgloss])
            elif averageonly and aline.startswith('Average Train loss'):
                avgloss = float(aline.split(':')[-1])
                logdata.setdefault('train',[]).append([itno,time,avgloss])
            elif averageonly and aline.startswith('Average Test loss'):
                avgloss = float(aline.split(':')[-1])
                logdata.setdefault('test',[]).append([itno,time,avgloss])        
            elif not averageonly and aline.startswith('train'):
                linedat = prs(aline)
                logdata.setdefault('train',[]).append([linedat['train'],\
                    linedat['iter_time'],linedat['Loss']])
            elif not averageonly and aline.startswith('test'):
                linedat = prs(aline)
                logdata.setdefault('test',[]).append([linedat['test'],\
                    linedat['iter_time'],linedat['Loss']])
    return logdata

def tabulate_lstm_results_pkl(prefix='Backup_results/LSTM_results',\
    outfile='deep_learning_results.csv',ignoredfields = \
    {'train','test','train_indices','test_indices','modality','order','dropout'}):
    '''
    Summarize all the LSTM evaluation results (stored in pkl file) with 
    a single csv file.
    '''
    filenames = glob.glob(os.path.join(ted_data_path,'TED_stats/',\
        prefix+'*.pkl'))    
    m = len(filenames)
    outfilename_table = os.path.join(ted_data_path,'TED_stats/',outfile)

    alldata = {}
    average_accuracy={}
    average_auc = {}
    ratings = set(rating_labels)
    new_ratings = set(['average_accuracy','average_auc','average_precision','average_recall','average_fscore'])
    for i,afile in enumerate(filenames):
        print afile
        filepart = os.path.split(afile)[-1]
        dev_test_part = filepart.split('_')[2]
        data = cp.load(open(afile))
        all_accuracy=[]
        all_auc=[]
        all_fscore=[]
        all_precision=[]
        all_recall=[]
        for akey in data:
            if akey.lower() in ignoredfields:
                continue
            if akey in ratings:
                newkeys = [akey+'_'+anorder for anorder in data['order']]
                newvals = data[akey]
                all_precision.append(data[akey][0])
                all_recall.append(data[akey][1])
                all_fscore.append(data[akey][2])
                all_accuracy.append(data[akey][3])
                all_auc.append(data[akey][4])

                for k,v in zip(newkeys,newvals):
                    new_ratings.add(k)
                    alldata.setdefault(k,{}).update({i:v})
            else:
                alldata.setdefault(akey,{}).update({i:data[akey]})
        alldata.setdefault('Dev_or_Test',{}).update({i:dev_test_part})
        alldata.setdefault('average_accuracy',{}).update({i:np.mean(all_accuracy)})
        alldata.setdefault('average_auc',{}).update({i:np.mean(all_auc)})
        alldata.setdefault('average_precision',{}).update({i:np.mean(all_precision)})
        alldata.setdefault('average_recall',{}).update({i:np.mean(all_recall)})        
        alldata.setdefault('average_fscore',{}).update({i:np.mean(all_fscore)})

    log_info = [h for h in alldata.keys() if not h in new_ratings]
    headers = log_info+['Dev_or_Test']+sorted(list(new_ratings))
    with open(outfilename_table,'wb') as fout:
        writer = csv.DictWriter(fout,headers)
        writer.writeheader()
        i=0
        while i < m:
            arow = {}
            for akey in alldata:
                try:
                    arow[akey] = alldata[akey][i]
                except KeyError:
                    print akey,'not found in:', afile
            writer.writerow(arow)
            i+=1
    print 'Result comparison table saved in:'
    print outfilename_table

def summarize_lstm_log(prefix='../TED_models/LSTM_log',averageonly=True,\
    outfile='summary_plot_LSTM_log.pdf',ignoredfields = \
    ['train','test','train_indices','test_indices',\
    'model_outfile','gpunum','modality'],\
    markers = ['x','*','+','|','','.','_','^','v','<','s','>','o'],
    figuresize=(15, 8)):
    '''
    Summarize all the LSTM training logs with a single figure
    '''
    markers = itertools.cycle(markers)
    filenames = glob.glob(os.path.join(ted_data_path,'TED_stats/',\
        prefix+'*'))    
    fpath,fname = os.path.split(outfile)
    fname,fext = fname.split('.')
    outfilename_time = os.path.join(ted_data_path,'TED_stats/',\
        os.path.join(fpath,fname+'_iter_time'+'.'+fext))
    outfilename_iter = os.path.join(ted_data_path,'TED_stats/',\
        os.path.join(fpath,fname+'_iter_no'+'.'+fext))
    alldata={}
    for i,afile in enumerate(filenames):
        filedata = read_lstm_log(afile,averageonly)
        for akey in filedata:
            alldata.setdefault(akey,{}).update({i:filedata[akey]})

    # Isolate the parameters with unique and non-unique values
    unique=[]
    nonunique=[]
    for akey in alldata:
        if not akey in ignoredfields and \
            not 'class' in akey and \
            len(set(alldata[akey].values()))==1:
            unique.append(akey)
        elif not akey in ignoredfields and \
            not 'class' in akey and \
            len(set(alldata[akey].values()))>1:
            nonunique.append(akey)
    
    # Make legends for the nonunique parameters 
    legendtxts=[]
    for i in range(len(filenames)):
        print filenames[i]
        alegend=''
        # make legends for non-unique params
        for j,akey in enumerate(nonunique):
            if akey in alldata and i in alldata[akey]:
                alegend+='{}={}'.format(akey,alldata[akey][i])
            if j<len(nonunique)-1:
                alegend+=','
        legendtxts.append(alegend)

    # Draw iter time vs loss plot
    fig=plt.figure(0,figsize=figuresize)
    plt.clf()
    for i,alegend in zip(range(len(filenames)),legendtxts):
        trainloss = np.array(alldata['train'][i])        
        plt.plot(trainloss[:,1]/3600.,trainloss[:,2],color='blue',\
            marker=markers.next(),label='Train,'+alegend)
        if 'test' in alldata and i in alldata['test']:
            testloss = np.array(alldata['test'][i])
            plt.plot(testloss[:,1]/3600.,testloss[:,2],\
                color='red',marker=markers.next(),label='Test,'+alegend)

    plt.grid('on',which='major')
    plt.grid('on',which='minor')
    plt.xlabel('Iteration time (hour)')
    if averageonly:
        plt.ylabel('Average loss (in an iteration over dataset)')
    else:
        plt.ylabel('Loss per minibatch')
    plt.legend(loc='upper right', bbox_to_anchor=(1, 1))
    plt.tight_layout()
    plt.savefig(outfilename_time)
    plt.close()

    # draw iter vs loss plot
    fig=plt.figure(0,figsize=figuresize)
    plt.clf()
    for i,alegend in zip(range(len(filenames)),legendtxts):
        # draw plots
        m_type = markers.next()
        trainloss = np.array(alldata['train'][i])
        plt.plot(trainloss[:,0],trainloss[:,2],color='blue',\
            marker=m_type,label='Train,'+alegend)
        if 'test' in alldata and i in alldata['test']:
            testloss = np.array(alldata['test'][i])
            plt.plot(testloss[:,0],testloss[:,2],\
                color='red',marker=m_type,label='Dev,'+alegend)
    plt.grid('on',which='major')
    plt.grid('on',which='minor')
    plt.xlabel('Iterations')
    if averageonly:
        plt.ylabel('Average loss (in an iteration over dataset)')
    else:
        plt.ylabel('Loss per minibatch')
    plt.legend(loc='upper right', bbox_to_anchor=(1, 1))
    plt.tight_layout()
    plt.savefig(outfilename_iter)
    plt.close()    
    print 'Loss figure saved in:',outfilename_iter

    # Print the unique parameters
    print 'Other Common parameters:'
    for akey in unique:
        print akey,'=',set(alldata[akey].values()).pop()
    print 'Blue is Train'
    print 'Red is Test'

# ================ Deprecated ===================
# def loss_vs_sense(resultfile='dev_result.pkl',\
#     train_log='train_logfile.txt',folder_prefix='run_',outfile='error_analysis.png'):
#     '''
#     This function reads the model evaluation results (obtained by executing 
#     evaluate_model function) and plots the losses with respect to model
#     parameters (sense_dim).
#     The results are assumed to be located in different folders starting
#     with the same prefix in the ted_data_path.
#     The output plot is saved in outfile. However, if outfile is None, no
#     plot is saved.
#     '''
#     infolders = glob.glob(os.path.join(ted_data_path,folder_prefix)+'*')
#     senselist=[]
#     trainlosslist=[]
#     testlosslist=[]
#     combined = {}
#     for afolder in infolders:
#         logfilename = os.path.join(afolder,train_log)
#         currentresultfile = os.path.join(afolder,resultfile)
#         # Read the training log
#         with open(logfilename) as fin:
#             for aline in fin:
#                 if aline.startswith('sense_dim'):
#                     senselist.append(int(aline.strip().split('=')[1]))
#                 if aline.startswith('Average Loss in last iteration'):
#                     trainlosslist.append(float(aline.strip().split(':')[1]))
#         # Read the current result file
#         results = cp.load(open(currentresultfile))
#         for akey in results:
#             if results[akey] and akey!='order':
#                 if not akey in combined:
#                     combined[akey] = [results[akey]]
#                 else:
#                     combined[akey].append(results[akey])
#     # Sort results
#     idx = np.argsort(senselist)
#     senselist,trainlosslist = zip(*[(senselist[i],trainlosslist[i]) \
#         for i in idx])
#     for akey in combined:
#         combined[akey] = [combined[akey][i] for i in idx]

#     # Plot the numbers
#     if outfile:
#         # Plot the loss
#         name,ext = ''.join(outfile.split('.')[:-1]),'.'+outfile.split('.')[-1]
#         outfilename = os.path.join(ted_data_path,name+'_loss'+ext)
#         plt.figure(1)
#         plt.clf()
#         plt.plot(senselist,trainlosslist,color='blue',label='Train Loss')
#         plt.plot(senselist,combined['average_loss'],\
#             color='red',label='Test Loss')
#         plt.xlabel('Sense Vector Length (Model Complexity)')
#         plt.ylabel('Loss')
#         plt.legend()
#         plt.savefig(outfilename)
#         plt.close()
#         print 'Loss figure saved in:',outfilename

#         for akey in combined:
#             if akey=='average_loss':
#                 continue
#             # Plot other results
#             outfilename = os.path.join(ted_data_path,name+'_'+akey+ext)
#             plt.figure(2)
#             plt.clf()
            
#             plt.plot(senselist,combined[akey])
#             plt.xticks(senselist)
#             plt.grid(True)
#             plt.xlabel('Sense Vector Length (Model Complexity)')
#             plt.ylabel('Metric Value')
#             plt.legend(results['order'])
#             plt.title('Test Result for '+akey)
#             plt.savefig(outfilename)
#             plt.close()
#             print akey+' figure saved in:',outfilename

#     return senselist, trainlosslist, testlosslist,combined

# ================ Deprecated ===================
# def average_results(result_pklfilename='dev_result.pkl',folder_prefix='run_'):
#     '''
#     This function reads the pickle files containing model evaluation results
#     (obtained by executing evaluate_model function) and computes the average.
#     These pickle files are assumed to be located in different folders starting
#     with the same prefix in the ted_data_path. 
#     '''
#     infolders = glob.glob(os.path.join(ted_data_path,folder_prefix)+'*')
#     combined={}
#     for afolder in infolders:
#         results = cp.load(open(os.path.join(afolder,result_pklfilename)))
#         for akey in results:
#             if results[akey] and akey!='order':
#                 if not akey in combined:
#                     combined[akey] = [results[akey]]
#                 else:
#                     combined[akey].append(results[akey])
#     for akey in combined:
#         combined[akey] = np.mean(combined[akey],axis=0).tolist()
#     combined['order']=results['order']
#     return combined

def tabulate_classical_results(outfile='result_classical.csv'):
    '''
    Read and tabulate the results in a csv file for classical experiments
    '''
    rating_names = ['beautiful',
                    'ingenious',
                    'fascinating',
                    'obnoxious',
                    'confusing',
                    'funny',
                    'inspiring',
                    'courageous',
                    'ok',
                    'persuasive',
                    'longwinded',
                    'informative',
                    'jaw-dropping',
                    'unconvincing']
    rating_order = ['Prec','Recall','fscore','Accuracy','AUC']
    result_attributes = ['classifier_type',
                         'c_scale',
                         'modalities_used',
                         'lowerthresh_Y',
                         'upperthresh_Y',
                         'scale_rating']
    summary_results =   ['max_results','avg_results',]
    other =['best_classifier','data_normalizer']
      
    outfilename = os.path.join(ted_data_path,'TED_stats/'+outfile)
    infilenames = glob.glob(os.path.join(ted_data_path,'TED_stats/results_*'))
    with open(outfilename,'wb') as fout:
        header = []
        for i,afile in enumerate(infilenames):
            data = cp.load(open(afile))
            # First time
            if i == 0:
                headers = [att for att in result_attributes]
                headers += [akey for akey in data['avg_results']]
                headers += [akey for akey in data['max_results']]
                headers += [arating+'_'+lab for arating in rating_names \
                    for lab in rating_order]
                writer = csv.DictWriter(fout,headers)
                writer.writeheader()
            # Create rows
            arow = {att:data[att] for att in result_attributes \
                if not att=='modalities_used'}
            arow.update({'modalities_used':'_'.join(data['modalities_used'])})
            arow.update(data['avg_results'])
            arow.update(data['max_results'])
            arow.update({arating+'_'+lab:data[arating][i] for arating in rating_names \
                    for i,lab in enumerate(rating_order)})
            writer.writerow(arow)
    print outfilename