HandleQueuedResults.py

#!/usr/bin/env python
import datetime
import json
import logging
import math
import pickle
import random
import threading
import time
import zlib
from collections import deque

from flask import request
from google.appengine.api import memcache
from google.appengine.ext import db

import structures
from structures import global_dict, load_blob

global_sync = {}
locks = {}


def handle_queued_results():
    out = []
    results = json.loads(request.get_data(as_text=True))

    rumble = results.get("game", "ERROR")

    syncname = str(results.get("melee") == "YES") + "|" + structures.sync
    if syncname not in locks:
        locks[syncname] = threading.Lock()
    sync_lock = locks[syncname]

    global_sync[syncname] = global_sync.get(syncname, {})
    botsync = global_sync[syncname]

    bota = results.get("fname")
    botb = results.get("sname")
    logging.debug("Bots : " + bota + " vs. " + botb)

    bd = [[bota, rumble], [botb, rumble]]

    botHashes = ["|".join(a) for a in bd]

    memget = [rumble]
    memget.extend(botHashes)
    memdict = memcache.get_multi(memget)

    game = memdict.get(rumble, None)
    game_future = None
    if game is None:
        game_future = db.get_async(db.Key.from_path('Rumble', rumble))

    newBot = False

    bots = [memdict.get(h, None) for h in botHashes]

    pairingsarray = [[], []]
    botFutures = [None, None]
    for i in [0, 1]:
        if bots[i] is None or bots[i].PairingsList is None:
            botFutures[i] = db.get_async(db.Key.from_path('BotEntry', botHashes[i]))

    for i in [0, 1]:
        if botFutures[i] is not None:
            modelbot = botFutures[i].get_result()
            if modelbot is not None:
                bots[i] = structures.CachedBotEntry(modelbot)

    for i in [0, 1]:
        if bots[i] is None:
            modelbot = structures.BotEntry(
                key_name=botHashes[i],
                Name=bd[i][0], Battles=0, Pairings=0, APS=0.0,
                Survival=0.0, PL=0, Rumble=rumble, Active=False,
                PairingsList=zlib.compress(pickle.dumps([]), 1))
            bots[i] = structures.CachedBotEntry(modelbot)
            newBot = True
        if isinstance(bots[i], structures.BotEntry):
            bots[i] = structures.CachedBotEntry(bots[i])

        pairingsarray[i] = load_blob(bots[i].PairingsList, [])
        if not isinstance(pairingsarray[i], list):
            pairingsarray[i] = []
        bots[i].PairingsList = None

    if game_future is not None:
        game = game_future.get_result()

    if game is None:
        game = structures.Rumble(
            key_name=rumble,
            Name=rumble, Rounds=int(results["rounds"]),
            Field=results["field"], Melee=results["melee"] == "YES",
            Teams=results["teams"] == "YES", TotalUploads=0,
            MeleeSize=10, ParticipantsScores=db.Blob(zlib.compress(pickle.dumps({}))))
        out.append("CREATED NEW GAME TYPE " + rumble + "\n")
        logging.info("Created new game type: " + rumble)
    else:
        field = game.Field == results["field"]
        rounds = (game.Rounds == int(results["rounds"]))
        teams = game.Teams == (results["teams"] == "YES")
        melee = game.Melee == (results["melee"] == "YES")
        allowed = field and rounds and teams and melee
        if not allowed:
            errstr = "OK. ERROR. Incorrect " + rumble + " config: "
            errorReasons = []
            if not field:
                errorReasons.append("field size ")
            if not rounds:
                errorReasons.append("number of rounds ")
            if not teams:
                errorReasons.append("teams ")
            if not melee:
                errorReasons.append("melee ")
            logging.error(errstr + ", ".join(errorReasons) + "  User: " + results["user"])
            return "".join(out)

    scores = load_blob(game.ParticipantsScores, {})
    game.ParticipantsScores = None
    if not isinstance(scores, dict):
        scores = {}

    for i in [0, 1]:
        if not bots[i].Active or bots[i].Name not in scores:
            bots[i].Active = True
            scores[bots[i].Name] = structures.LiteBot(bots[i])
            newBot = True
            out.append("Added " + bd[i][0] + " to " + rumble + "\n")
            logging.info("added new bot!")

    scorea = float(results["fscore"])
    scoreb = float(results["sscore"])

    if scorea + scoreb > 0:
        APSa = 100 * scorea / (scorea + scoreb)
    else:
        APSa = 50

    survivala = float(results["fsurvival"])
    survivalb = float(results["ssurvival"])

    survivala = 100.0 * survivala / game.Rounds
    survivalb = 100.0 * survivalb / game.Rounds

    battleTimeStr = results.get("time", None)
    if battleTimeStr is not None:
        battleTime = datetime.datetime.utcfromtimestamp(int(battleTimeStr) / 1e3)
        lastUploadStr = battleTime.strftime("%Y-%m-%d %H:%M:%S")
    else:
        lastUploadStr = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    for b, pairings in zip(bots, pairingsarray):
        if len(pairings) > 0:
            removes = []
            for p in pairings:
                try:
                    p.APS = float(p.APS)
                    p.Survival = float(p.Survival)
                    p.Battles = int(p.Battles)
                except (TypeError, ValueError, AttributeError):
                    removes.append(pairings.index(p))
                    continue

            removes.sort(reverse=True)
            for i in removes:
                pairings.pop(i)

    apair = None
    for p in pairingsarray[0]:
        if p.Name == botb:
            apair = p
    if apair is None:
        apair = structures.ScoreSet(name=botb)
        pairingsarray[0].append(apair)

    bpair = None
    for p in pairingsarray[1]:
        if p.Name == bota:
            bpair = p
    if bpair is None:
        bpair = structures.ScoreSet(name=bota)
        pairingsarray[1].append(bpair)

    if (apair.Battles == 1
            and battleTimeStr is not None
            and apair.APS == APSa
            and apair.Survival == survivala
            and apair.LastUpload == lastUploadStr):
        logging.info("Discarding duplicate battle: " + bota + " vs " + botb)
        return "OK. Duplicate battle discarded."

    for b, pairings in zip(bots, pairingsarray):
        i = 0
        while i < len(pairings):
            if pairings[i].Name == b.Name:
                pairings.pop(i)
                continue

            if pairings[i].Name in scores:
                pairings[i].Alive = True
            else:
                pairings[i].Alive = False

            i += 1

    aBattles = apair.Battles

    maxPerPair = 10000 // len(bots)
    if aBattles > maxPerPair:
        aBattles = maxPerPair

    inv_ab = 1.0 / (aBattles + 1.0)

    apaps_old = apair.APS
    apair.APS *= float(aBattles) * inv_ab
    apair.APS += APSa * inv_ab

    delta_a = APSa - apaps_old
    var_a_old = apair.__dict__.get("Var_APS", -1.0)
    if var_a_old is None or var_a_old < 0:
        var_a_old = 0.0
    apair.Var_APS = (1.0 - inv_ab) * (var_a_old + inv_ab * delta_a * delta_a)
    bpair.Var_APS = apair.Var_APS

    apair.__dict__["Min_APS"] = min(APSa, apair.__dict__.get("Min_APS", 100))
    bpair.APS = 100 - apair.APS
    bpair.__dict__["Min_APS"] = min(100 - APSa, bpair.__dict__.get("Min_APS", 100))

    apair.Survival *= float(aBattles) * inv_ab
    apair.Survival += survivala * inv_ab

    bpair.Survival *= float(aBattles) * inv_ab
    bpair.Survival += survivalb * inv_ab

    apair.Battles += 1
    bpair.Battles = apair.Battles

    apair.LastUpload = lastUploadStr
    bpair.LastUpload = apair.LastUpload

    for b, pairings in zip(bots, pairingsarray):
        aps = 0.0
        survival = 0.0
        pl = 0
        battles = 0
        alivePairings = 0
        ci_var_sum = 0.0
        ci_count = 0
        if len(pairings) > 0:
            for p in pairings:
                if p.Alive:
                    aps += p.APS
                    survival += p.Survival
                    if p.APS > 50:
                        pl += 1
                    else:
                        pl -= 1

                    battles += p.Battles
                    alivePairings += 1

                    pvar = p.__dict__.get("Var_APS", -1.0)
                    if pvar is not None and pvar >= 0 and int(p.Battles) >= 2:
                        ci_var_sum += max(0.0, pvar) / (min(int(p.Battles), maxPerPair) - 1)
                        ci_count += 1

            if alivePairings > 0:
                aps /= alivePairings
                survival /= alivePairings
            b.APS = aps
            b.Survival = survival
            b.PL = pl
            b.Battles = battles
            if ci_count > 0 and alivePairings > 0:
                b.APS_CI = 1.96 * math.sqrt(ci_var_sum / (alivePairings * ci_count))
            else:
                b.APS_CI = -1.0

        b.PairingsList = db.Blob(zlib.compress(pickle.dumps(pairings, pickle.HIGHEST_PROTOCOL), 1))
        b.LastUpload = apair.LastUpload
        b.Pairings = alivePairings

    game.TotalUploads += 1

    game.LastUpload = apair.LastUpload
    game.AvgBattles = game.AvgBattles * 0.99 + 0.005 * (bots[0].Battles + bots[1].Battles)
    if game.Uploaders is None:
        uploaders = None
    else:
        uploaders = load_blob(game.Uploaders, {})
    if uploaders is None or not isinstance(uploaders, dict) or len(uploaders) == 0:
        uploaders = {}
    uploaderName = results["user"]

    try:
        uploader = uploaders[uploaderName]
        uploader.latest = apair.LastUpload
        uploader.total += 1
    except KeyError:
        uploader = structures.User(name=uploaderName)
        uploaders[uploaderName] = uploader
    game.Uploaders = zlib.compress(pickle.dumps(uploaders, -1), 1)

    for b in bots:
        try:
            bUploaders = b.Uploaders
            if uploaderName not in bUploaders:
                bUploaders.append(uploaderName)
        except AttributeError:
            b.__dict__["Uploaders"] = [uploaderName]

    wrote = False
    with sync_lock:
        for b in bots:
            if isinstance(b, structures.BotEntry):
                key = b.key().name()
            else:
                key = b.key_name
            botsync[key] = botsync.get(key, 0) + 1

        minSize = min(10, len(scores) // 2)

        logging.debug("botsync: " + str(len(botsync)))

        if len(botsync) > minSize:
            syncset = list(botsync.keys())

            syncbotsDict = memcache.get_multi(syncset)

            syncbots = []
            for sb in syncset:
                b = syncbotsDict.get(sb, None)

                if b is None or b.PairingsList is None:
                    syncbotsDict.pop(sb, 1)
                else:
                    syncbots.append(b)

                botsync.pop(sb, 1)

            try:
                thisput = []
                while len(syncbots) > 0:
                    b = syncbots.pop()

                    if isinstance(b, structures.BotEntry):
                        key = b.key().name()
                    else:
                        key = b.key_name

                    putb = structures.BotEntry(key_name=key)
                    putb.init_from_cache(b)
                    thisput.append(putb)

                db.put(thisput)

                logging.info("wrote " + str(len(thisput)) + " results to database")
                for b in thisput:
                    s = b.key().name()
                    botsync.pop(s, 1)
                    syncbotsDict.pop(s, 1)
                wrote = True

            except Exception as e:
                logging.error('Failed to write data: ' + str(e))

    for b in bots:
        if b.Name in scores:
            lb = scores[b.Name]
            b.ANPP = lb.ANPP
            b.VoteScore = lb.VoteScore
            scores.pop(b.Name, 1)
        scores[b.Name] = structures.LiteBot(b)

    game.ParticipantsScores = db.Blob(zlib.compress(pickle.dumps(scores, pickle.HIGHEST_PROTOCOL), 1))
    game.Participants = []

    if game.BatchScoresAccurate:
        game.BatchScoresAccurate = False
        if not newBot:
            game.put()
            wrote = True

    if newBot or wrote:
        game.put()
        memcache.delete_multi(["home", "home_dark"])

    botsDict = {rumble: game}
    global_dict[rumble] = game
    for b in bots:
        if isinstance(b, structures.BotEntry):
            b = structures.CachedBotEntry(b)

        key = b.key_name
        botsDict[key] = b
    memcache.set_multi(botsDict)

    scoreVals = list(scores.values())
    maxPairs = len(scoreVals) - 1

    if game.PriorityBattles and ((not game.Melee) or
                                 (random.random() < 0.0222
                                  or bots[0].Pairings != maxPairs or bots[1].Pairings != maxPairs
                                  or min([bots[0].Battles, bots[1].Battles]) == min([b.Battles for b in scoreVals])
                                  )):
        priobot = None
        priopairs = None

        missingPairings = [b for b in scoreVals if b.Pairings != maxPairs]

        priobot2 = None

        if len(missingPairings) > 0:
            total = 0
            weighted = [(abs(maxPairs - b.Pairings), b) for b in missingPairings]

            for t in weighted:
                total += t[0]
            running = 0
            point = random.randint(0, max(total - 1, 0))
            for t in weighted:
                running += t[0]
                if running > point:
                    priobot = t[1]
                    break
            if priobot is not None and priobot.Name == bots[0].Name:
                priobot = bots[0]
                priopairs = pairingsarray[0]
            elif priobot is not None and priobot.Name == bots[1].Name:
                priobot = bots[1]
                priopairs = pairingsarray[1]
            elif priobot is not None:
                if priobot.Pairings < maxPairs:
                    bhash = priobot.Name + "|" + rumble
                    fullPrioBot = memcache.get(bhash)
                else:
                    fullPrioBot = None

                if fullPrioBot:
                    priopairs = load_blob(fullPrioBot.PairingsList, [])
                    if not isinstance(priopairs, list):
                        priopairs = []
                    logging.info("memcache lookup shortcut to local search")
                else:
                    priobot2 = random.choice(scoreVals).Name
                    catch = 10
                    while priobot2 == priobot.Name and catch > 0:
                        priobot2 = random.choice(scoreVals).Name
                        catch -= 1
                    if catch == 0:
                        priobot2 = None
                        logging.info("repeatedly found same bot for prio")
                    else:
                        logging.info("global min search successful for non-paired bot")

            if priobot is not None and priobot2 is None and priopairs is not None:
                pairsdict = {}
                for b in priopairs:
                    pairsdict[b.Name] = b

                possPairs = []
                for p in scores:
                    if p not in pairsdict and p != priobot.Name:
                        possPairs.append(p)

                if len(possPairs) > 0:
                    priobot2 = random.choice(possPairs)
                    logging.info("successful local search for new pair")
                else:
                    logging.info("unsuccessful local search for new pair")
        else:
            minBattles = 1.1 * min([b.Battles for b in scoreVals if b.Active])
            possBots = [b for b in scoreVals if b.Battles <= minBattles and b.Active]

            names = [b.Name for b in possBots]
            if bots[1].Name not in names and bots[0].Name not in names:
                priobot = random.choice(possBots)

                bhash = priobot.Name + "|" + rumble
                fullPrioBot = memcache.get(bhash)
                if fullPrioBot:
                    priopairs = load_blob(fullPrioBot.PairingsList, [])
                    if not isinstance(priopairs, list):
                        priopairs = []
                    logging.info("memcache lookup shortcut to global search")
                else:
                    priobot2 = random.choice(scoreVals).Name
                    catch = 10
                    while priobot2 == priobot.Name and catch > 0:
                        priobot2 = random.choice(scoreVals).Name
                        catch -= 1
                    if catch == 0:
                        priobot2 = None
                        logging.info("repeatedly found same bot for prio")
                    else:
                        logging.info("global min search successful for low-battled bot")

            elif bots[0].Battles <= bots[1].Battles:
                priobot = bots[0]
                priopairs = pairingsarray[0]
            else:
                priobot = bots[1]
                priopairs = pairingsarray[1]

        if priobot2 is None and priopairs is not None and priobot is not None:
            alive = [p for p in priopairs
                     if p.Name != priobot.Name and p.Name in scores and scores[p.Name].Active]
            unknown = [p for p in alive if structures.pairing_ci(p) is None]
            if unknown:
                minbat = min(int(p.Battles) for p in unknown)
                poss = [p for p in unknown if int(p.Battles) <= minbat]
                if len(poss) < min(50, 0.5 * len(scores)):
                    poss = [p for p in unknown if int(p.Battles) <= minbat + 1]
                priobot2 = random.choice(poss).Name
                logging.info("successful local search for undefined-CI pair")
            elif alive:
                cap = structures.rolling_battle_cap
                def reduction(p):
                    n = min(int(p.Battles), cap)
                    base = max(0.0, p.Var_APS) / (n * (n + 1.0))
                    ci = structures.pairing_ci(p)
                    if ci is not None and abs(p.APS - 50.0) < ci:
                        base *= 10.0
                    return base
                scored = [(reduction(p), p) for p in alive]
                best = max(r for r, p in scored)
                band = [p for r, p in scored if r >= 0.5 * best]
                if len(band) < min(50, 0.5 * len(scores)):
                    band = [p for r, p in scored if r >= 0.25 * best]
                priobot2 = random.choice(band).Name
                logging.info("successful local search for high-CI pair")
            else:
                logging.info("unsuccessful local search for CI pair")

        if priobot is not None and priobot2 is not None:
            priobots = [priobot.Name, priobot2]
            priobots = [b.replace(' ', '_') for b in priobots]

            prio_string = "[" + ",".join(priobots) + "]\n"
            logging.info("adding priority battle: " + prio_string + ", " + rumble)

            rq_name = rumble + "|queue"
            try:
                rumble_queue = global_dict[rq_name]
                rumble_queue.append(prio_string)
            except KeyError:
                logging.info("No queue for rumble " + rumble + ", adding one!")
                global_dict[rq_name] = deque()
                rumble_queue = global_dict[rq_name]
                rumble_queue.append(prio_string)
        else:
            logging.info("no suitable priority battle found in " + rumble)
            if priobot is None:
                logging.info("priobot is None")
            else:
                logging.info("priobot2 is None")
    else:
        logging.info("no priority battle attempted for " + rumble)

    out.append("\nOK. " + bota + " vs " + botb + " received")
    return "".join(out)