{

if(signal == SIGINT || signal == SIGTERM) {

sigReceived.store(true);

shouldStop.store(true);

}

Board::initHash();

ScoreValue::initTables();

Rand seedRand;

ConfigParser cfg;

string logFile;

string nnPredictorPath;

string sgfOutputDir;

try {

KataGoCommandLine cmd("Play different nets against each other with different search settings in a match or tournament.");

cmd.addConfigFileArg("","match_example.cfg");

TCLAP::ValueArg<string> logFileArg("","log-file","Log file to output to",false,string(),"FILE");

TCLAP::ValueArg<string> sgfOutputDirArg("","sgf-output-dir","Dir to output sgf files",false,string(),"DIR");

cmd.add(logFileArg);

cmd.add(sgfOutputDirArg);

cmd.setShortUsageArgLimit();

cmd.addOverrideConfigArg();

cmd.parseArgs(args);

logFile = logFileArg.getValue();

sgfOutputDir = sgfOutputDirArg.getValue();

cmd.getConfig(cfg);

}

catch (TCLAP::ArgException &e) {

cerr << "Error: " << e.error() << " for argument " << e.argId() << endl;

return 1;

}

Logger logger(&cfg);

logger.addFile(logFile);

logger.write("Match Engine starting...");

logger.write(string("Git revision: ") + Version::getGitRevision());

//Load per-bot search config, first, which also tells us how many bots we're running

vector<SearchParams> paramss = Setup::loadParams(cfg,Setup::SETUP_FOR_MATCH);

assert(paramss.size() > 0);

int numBots = (int)paramss.size();

//Figure out all pairs of bots that will be playing.

std::vector<std::pair<int,int>> matchupsPerRound;

{

//Load a filter on what bots we actually want to run. By default, include everything.

vector<bool> includeBot(numBots);

if(cfg.contains("includeBots")) {

vector<int> includeBotIdxs = cfg.getInts("includeBots",0,Setup::MAX_BOT_PARAMS_FROM_CFG);

for(int i = 0; i<numBots; i++) {

if(contains(includeBotIdxs,i))

includeBot[i] = true;

}

}

else {

for(int i = 0; i<numBots; i++) {

includeBot[i] = true;

}

}

std::vector<int> secondaryBotIdxs;

if(cfg.contains("secondaryBots"))

secondaryBotIdxs = cfg.getInts("secondaryBots",0,Setup::MAX_BOT_PARAMS_FROM_CFG);

std::vector<int> secondaryBot2Idxs;

if(cfg.contains("secondaryBots2"))

secondaryBot2Idxs = cfg.getInts("secondaryBots2",0,Setup::MAX_BOT_PARAMS_FROM_CFG);

for(const auto& botIdxs : {secondaryBotIdxs, secondaryBot2Idxs}) {

for(int i = 0; i<botIdxs.size(); i++)

assert(botIdxs[i] >= 0 && botIdxs[i] < numBots);

}

for(int i = 0; i<numBots; i++) {

if(!includeBot[i])

continue;

for(int j = 0; j<numBots; j++) {

if(!includeBot[j])

continue;

if(i < j

&& !(contains(secondaryBotIdxs,i) && contains(secondaryBotIdxs,j))

&& !(contains(secondaryBot2Idxs,i) && contains(secondaryBot2Idxs,j))

) {

matchupsPerRound.push_back(make_pair(i,j));

matchupsPerRound.push_back(make_pair(j,i));

}

}

}

if(cfg.contains("extraPairs")) {

string pairsStr = cfg.getString("extraPairs");

std::vector<string> pairStrs = Global::split(pairsStr,',');

for(const string& pairStr: pairStrs) {

if(Global::trim(pairStr).size() <= 0)

continue;

std::vector<string> pieces = Global::split(Global::trim(pairStr),'-');

if(pieces.size() != 2) {

throw IOError("Could not parse pair: " + pairStr);

}

bool suc;

int p0;

int p1;

suc = Global::tryStringToInt(pieces[0],p0);

if(!suc)

throw IOError("Could not parse pair: " + pairStr);

suc = Global::tryStringToInt(pieces[1],p1);

if(!suc)

throw IOError("Could not parse pair: " + pairStr);

if(p0 < 0 || p0 >= numBots)

throw IOError("Invalid player index in pair: " + pairStr);

if(p1 < 0 || p1 >= numBots)

throw IOError("Invalid player index in pair: " + pairStr);

if(cfg.contains("extraPairsAreOneSidedBW") && cfg.getBool("extraPairsAreOneSidedBW")) {

matchupsPerRound.push_back(std::make_pair(p0,p1));

}

else {

matchupsPerRound.push_back(std::make_pair(p0,p1));

matchupsPerRound.push_back(std::make_pair(p1,p0));

}

}

}

}

// Maybe load the predictor path

string predictorPath = "";

if (cfg.contains("predictorPath")) {

predictorPath = cfg.getString("predictorPath");

}

//Load the names of the bots and which model each bot is using

vector<string> nnModelFilesByBot(numBots);

vector<string> botNames(numBots);

for(int i = 0; i<numBots; i++) {

string idxStr = Global::intToString(i);

if(cfg.contains("botName"+idxStr))

botNames[i] = cfg.getString("botName"+idxStr);

else if(numBots == 1)

botNames[i] = cfg.getString("botName");

else

throw StringError("If more than one bot, must specify botName0, botName1,... individually");

if(cfg.contains("nnModelFile"+idxStr))

nnModelFilesByBot[i] = cfg.getString("nnModelFile"+idxStr);

else

nnModelFilesByBot[i] = cfg.getString("nnModelFile");

}

vector<bool> botIsUsed(numBots);

for(const std::pair<int,int> pair: matchupsPerRound) {

botIsUsed[pair.first] = true;

botIsUsed[pair.second] = true;

}

//Dedup and load each necessary model exactly once

vector<string> nnModelFiles;

vector<int> whichNNModel(numBots);

for(int i = 0; i<numBots; i++) {

if(!botIsUsed[i])

continue;

const string& desiredFile = nnModelFilesByBot[i];

int alreadyFoundIdx = -1;

for(int j = 0; j<nnModelFiles.size(); j++) {

if(nnModelFiles[j] == desiredFile) {

alreadyFoundIdx = j;

break;

}

}

if(alreadyFoundIdx != -1)

whichNNModel[i] = alreadyFoundIdx;

else {

whichNNModel[i] = (int)nnModelFiles.size();

nnModelFiles.push_back(desiredFile);

}

}

//Load match runner settings

int numGameThreads = cfg.getInt("numGameThreads",1,16384);

const string gameSeedBase = Global::uint64ToHexString(seedRand.nextUInt64());

//Work out an upper bound on how many concurrent nneval requests we could end up making.

int maxConcurrentEvals;

int expectedConcurrentEvals;

{

//Work out the max threads any one bot uses

int maxBotThreads = 0;

for(int i = 0; i<numBots; i++)

if(paramss[i].numThreads > maxBotThreads)

maxBotThreads = paramss[i].numThreads;

//Mutiply by the number of concurrent games we could have

expectedConcurrentEvals = maxBotThreads * numGameThreads;

//Multiply by 2 and add some buffer, just so we have plenty of headroom.

maxConcurrentEvals = expectedConcurrentEvals * 2 + 16;

}

//Initialize object for randomizing game settings and running games

PlaySettings playSettings = PlaySettings::loadForMatch(cfg);

GameRunner* gameRunner = new GameRunner(cfg, playSettings, logger);

const int minBoardXSizeUsed = gameRunner->getGameInitializer()->getMinBoardXSize();

const int minBoardYSizeUsed = gameRunner->getGameInitializer()->getMinBoardYSize();

const int maxBoardXSizeUsed = gameRunner->getGameInitializer()->getMaxBoardXSize();

const int maxBoardYSizeUsed = gameRunner->getGameInitializer()->getMaxBoardYSize();

//Initialize neural net inference engine globals, and load models

Setup::initializeSession(cfg);

const vector<string>& nnModelNames = nnModelFiles;

const int defaultMaxBatchSize = -1;

const bool defaultRequireExactNNLen = minBoardXSizeUsed == maxBoardXSizeUsed && minBoardYSizeUsed == maxBoardYSizeUsed;

const bool disableFP16 = false;

const vector<string> expectedSha256s;

vector<NNEvaluator*> nnEvals = Setup::initializeNNEvaluators(

nnModelNames,nnModelFiles,expectedSha256s,cfg,logger,seedRand,maxConcurrentEvals,expectedConcurrentEvals,

maxBoardXSizeUsed,maxBoardYSizeUsed,defaultMaxBatchSize,defaultRequireExactNNLen,disableFP16,

Setup::SETUP_FOR_MATCH

);

logger.write("Loaded neural net");

NNEvaluator *predictorEval = nullptr;

if (nnPredictorPath != "") {

const string expectedSha256 = "";

predictorEval = Setup::initializeNNEvaluator(

nnPredictorPath, nnPredictorPath, expectedSha256, cfg, logger, seedRand, maxConcurrentEvals, expectedConcurrentEvals,

maxBoardXSizeUsed, maxBoardYSizeUsed, defaultMaxBatchSize, defaultRequireExactNNLen,disableFP16,

Setup::SETUP_FOR_MATCH

);

}

vector<NNEvaluator*> nnEvalsByBot(numBots);

for(int i = 0; i<numBots; i++) {

if(!botIsUsed[i])

continue;

nnEvalsByBot[i] = nnEvals[whichNNModel[i]];

}

std::vector<std::unique_ptr<PatternBonusTable>> patternBonusTables = Setup::loadAvoidSgfPatternBonusTables(cfg,logger);

assert(patternBonusTables.size() == numBots);

//Initialize object for randomly pairing bots

int64_t numGamesTotal = cfg.getInt64("numGamesTotal",1,((int64_t)1) << 62);

MatchPairer* matchPairer = new MatchPairer(cfg,numBots,botNames,nnEvalsByBot,paramss,matchupsPerRound,numGamesTotal);

//Check for unused config keys

cfg.warnUnusedKeys(cerr,&logger);

//Done loading!

//------------------------------------------------------------------------------------

logger.write("Loaded all config stuff, starting matches");

if(!logger.isLoggingToStdout())

cout << "Loaded all config stuff, starting matches" << endl;

if(sgfOutputDir != string())

MakeDir::make(sgfOutputDir);

if(!std::atomic_is_lock_free(&shouldStop))

throw StringError("shouldStop is not lock free, signal-quitting mechanism for terminating matches will NOT work!");

std::signal(SIGINT, signalHandler);

std::signal(SIGTERM, signalHandler);

std::mutex statsMutex;

int64_t gameCount = 0;

std::map<string,double> timeUsedByBotMap;

std::map<string,double> movesByBotMap;

auto runMatchLoop = [

&gameRunner,&matchPairer,&predictorEval,&sgfOutputDir,&logger,&gameSeedBase,&patternBonusTables,

&statsMutex, &gameCount, &timeUsedByBotMap, &movesByBotMap

](

uint64_t threadHash

) {

ofstream* sgfOut = NULL;

if(sgfOutputDir.length() > 0) {

sgfOut = new ofstream();

FileUtils::open(*sgfOut, sgfOutputDir + "/" + Global::uint64ToHexString(threadHash) + ".sgfs");

}

auto shouldStopFunc = []() {

return shouldStop.load();

};

WaitableFlag* shouldPause = nullptr;

Rand thisLoopSeedRand;

while(true) {

if(shouldStop.load())

break;

FinishedGameData* gameData = NULL;

MatchPairer::BotSpec botSpecB;

MatchPairer::BotSpec botSpecW;

if (predictorEval) {

if (botSpecB.botIdx == 1)

botSpecB.predictorNNEval = predictorEval;

else

botSpecW.predictorNNEval = predictorEval;

}

if(matchPairer->getMatchup(botSpecB, botSpecW, logger)) {

string seed = gameSeedBase + ":" + Global::uint64ToHexString(thisLoopSeedRand.nextUInt64());

std::function<void(const MatchPairer::BotSpec&, Search*)> afterInitialization = [&patternBonusTables](const MatchPairer::BotSpec& spec, Search* search) {

assert(spec.botIdx < patternBonusTables.size());

search->setCopyOfExternalPatternBonusTable(patternBonusTables[spec.botIdx]);

};

const string gameDescription = "game " + seed + " between "

+ botSpecB.botName + "-" + botSpecB.baseParams.getSearchAlgoAsStr() + " (B) "

+ botSpecW.botName + "-" + botSpecW.baseParams.getSearchAlgoAsStr() + " (W)";

logger.write("Launching " + gameDescription);

gameData = gameRunner->runGame(

seed, botSpecB, botSpecW, NULL, NULL, logger,

shouldStopFunc, shouldPause, nullptr, afterInitialization, nullptr

);

logger.write("Finished " + gameDescription);

}

bool shouldContinue = gameData != NULL;

if(gameData != NULL) {

if(sgfOut != NULL) {

WriteSgf::writeSgf(*sgfOut,gameData->bName,gameData->wName,gameData->endHist,gameData,false,true);

(*sgfOut) << endl;

}

{

std::lock_guard<std::mutex> lock(statsMutex);

gameCount += 1;

timeUsedByBotMap[gameData->bName] += gameData->bTimeUsed;

timeUsedByBotMap[gameData->wName] += gameData->wTimeUsed;

movesByBotMap[gameData->bName] += (double)gameData->bMoveCount;

movesByBotMap[gameData->wName] += (double)gameData->wMoveCount;

int64_t x = gameCount;

while(x % 2 == 0 && x > 1) x /= 2;

if(x == 1 || x == 3 || x == 5) {

for(auto& pair : timeUsedByBotMap) {

logger.write(

"Avg move time used by " + pair.first + " " +

Global::doubleToString(pair.second / movesByBotMap[pair.first]) + " " +

Global::doubleToString(movesByBotMap[pair.first]) + " moves"

);

}

}

}

delete gameData;

}

if(shouldStop.load())

break;

if(!shouldContinue)

break;

}

if(sgfOut != NULL) {

sgfOut->close();

delete sgfOut;

}

logger.write("Match loop thread terminating");

};

auto runMatchLoopProtected = [&logger,&runMatchLoop](uint64_t threadHash) {

Logger::logThreadUncaught("match loop", &logger, [&](){ runMatchLoop(threadHash); });

};

Rand hashRand;

vector<std::thread> threads;

for(int i = 0; i<numGameThreads; i++) {

threads.push_back(std::thread(runMatchLoopProtected, hashRand.nextUInt64()));

}

for(int i = 0; i<threads.size(); i++)

threads[i].join();

delete matchPairer;

delete gameRunner;

nnEvalsByBot.clear();

for(int i = 0; i<nnEvals.size(); i++) {

if(nnEvals[i] != NULL) {

logger.write(nnEvals[i]->getModelFileName());

logger.write("NN rows: " + Global::int64ToString(nnEvals[i]->numRowsProcessed()));

logger.write("NN batches: " + Global::int64ToString(nnEvals[i]->numBatchesProcessed()));

logger.write("NN avg batch size: " + Global::doubleToString(nnEvals[i]->averageProcessedBatchSize()));

delete nnEvals[i];

}

}

if (predictorEval)

delete predictorEval;

NeuralNet::globalCleanup();

ScoreValue::freeTables();

if(sigReceived.load())

logger.write("Exited cleanly after signal");

logger.write("All cleaned up, quitting");

return 0;