{

std::vector<uint8_t> ahead(len); // the bytes matched

in.read((char*)&ahead.front(), len);

unsigned count = in.gcount();

if (count < len) return false;

unsigned offset=0; // the index mod len which we're in ahead

do

{

bool found=true;

for (unsigned i=0; i < len; i++)

{

if (ahead[(i+offset)%len] != bytes[i])

{

found=false;

break;

}

}

if (found)

return true;

ahead[offset++%len] = in.get();

} while (!in.eof());

return false;

const uint8_t * bytes, const uint64_t len)

{

uint64_t offset = 0; // the index mod len which we're in ahead

do

{

bool found = true;

for (uint64_t i = 0; i < len; i++)

{

if (in[i] != bytes[i])

{

found = false;

break;

}

}

if (found)

return offset;

in++;

offset++;

} while (offset + len< inLen);

return UINT64_MAX;

{

size_t fnum;

string path;

uint64_t filesize;

uint64_t filesizeCumul;

{

public:

size_t fnum_;

size_t pos_;

stopReadingHeaders(size_t fnum, size_t pos) :

fnum_(fnum), pos_(pos)

{}

{

unsigned numBlkFiles=0;

if (blkFiles_.size() > 0)

{

numBlkFiles = blkFiles_.size()-1;

totalBlockchainBytes_ -= blkFiles_.back().filesize;

blkFiles_.pop_back();

}

while(numBlkFiles < UINT16_MAX)

{

string path = BtcUtils::getBlkFilename(blkFileLocation_, numBlkFiles);

uint64_t filesize = BtcUtils::GetFileSize(path);

if(filesize == FILE_DOES_NOT_EXIST)

break;

BlkFile f;

f.fnum = numBlkFiles;

f.path = path;

f.filesize = filesize;

f.filesizeCumul = totalBlockchainBytes_;

blkFiles_.push_back(f);

totalBlockchainBytes_ += filesize;

numBlkFiles++;

}

if(numBlkFiles==UINT16_MAX)

{

throw runtime_error("Error finding blockchain files (blkXXXX.dat)");

}

{

if (fnum==0) return 0;

if (fnum >= blkFiles_.size())

throw std::range_error("block file out of range");

return blkFiles_[fnum].filesizeCumul;

Blockchain &bc

)

{

map<HashString, BlockHeader> &allHeaders = bc.allHeaders();

size_t index=0;

for (; index < blkFiles_.size(); index++)

{

const BinaryData hash = getFirstHash(blkFiles_[index]);

if (allHeaders.find(hash) == allHeaders.end())

{ // not found in this file

if (index == 0)

return { 0, 0 };

break;

}

}

if (index == 0)

return { 0, 0 };

index--;

// ok, now "index" is for the last blkfile that we found a header in

// now lets linearly search this file until we find an unrecognized blk

BlockFilePosition foundAtPosition{ 0, 0 };

bool foundTopBlock = false;

auto topBlockHash = bc.top().getThisHash();

const auto stopIfBlkHeaderRecognized =

[&allHeaders, &foundAtPosition, &foundTopBlock, &topBlockHash] (

const BinaryData &blockheader,

const BlockFilePosition &pos,

uint32_t blksize

)

{

// always set our position so that eventually it's at the end

foundAtPosition = pos;

BlockHeader block;

BinaryRefReader brr(blockheader);

block.unserialize(brr);

const HashString blockhash = block.getThisHash();

auto bhIter = allHeaders.find(blockhash);

if(bhIter == allHeaders.end())

throw StopReading();

if (bhIter->second.getThisHash() == topBlockHash)

foundTopBlock = true;

bhIter->second.setBlockFileNum(pos.first);

bhIter->second.setBlockFileOffset(pos.second);

};

uint64_t returnedOffset = UINT64_MAX;

try

{

returnedOffset = readHeadersFromFile(

blkFiles_[index],

0,

stopIfBlkHeaderRecognized

);

}

catch (StopReading&)

{

// we're fine

}

// but we never find the genesis block, because

// it always appears in Blockchain even if unloaded, and

// we need to load it

if (foundAtPosition.first == 0 && foundAtPosition.second==293)

return { 0, 0 };

if (returnedOffset != UINT64_MAX)

foundAtPosition.second = returnedOffset;

if (!foundTopBlock)

{

LOGWARN << "Couldn't find top block hash in last seen blk file."

" Searching for it further down the chain";

//Couldn't find the top header in the last seen blk file. Since Core

//0.10, this can be an indicator of missing hashes. Let's find the

//the top block header in file.

BlockFilePosition topBlockPos(0, 0);

auto checkBlkHash = [&topBlockHash, &topBlockPos]

(const BinaryData &rawBlock,

const BlockFilePosition &pos,

uint32_t blksize)->void

{

BlockHeader bhUnser(rawBlock);

if (bhUnser.getThisHash() == topBlockHash)

{

topBlockPos = pos;

throw StopReading();

}

};

bool foundTopBlock = false;

int32_t fnum = blkFiles_.size();

if (fnum > 0)

fnum--;

try

{

for (; fnum > -1; fnum--)

readHeadersFromFile(blkFiles_[fnum], 0,

checkBlkHash);

}

catch (StopReading&)

{

foundTopBlock = true;

// we're fine

}

if (!foundTopBlock)

{

//can't find the top header, let's just rescan all headers

LOGERR << "Failed to find last known top block hash in "

"blk files. Rescanning all headers";

return BlockFilePosition(0, 0);

}

//Check this file to see if we are missing any block hashes in there

auto& f = blkFiles_[foundAtPosition.first];

try

{

readHeadersFromFile(f, 0, stopIfBlkHeaderRecognized);

}

catch (StopReading&)

{

//so we are indeed missing some block headers. Let's just scan the

//blocks folder for headers

foundAtPosition.first = 0;

foundAtPosition.second = 0;

LOGWARN << "Inconsistent headers DB, attempting repairs";

}

}

return foundAtPosition;

BlockFilePosition startAt,

const function<void(

const BinaryData &,

const BlockFilePosition &pos,

uint32_t blksize

)> &blockDataCallback

) const

{

if (startAt.first == blkFiles_.size())

return startAt;

if (startAt.first > blkFiles_.size())

throw std::runtime_error("blkFile out of range");

uint64_t finishOffset=startAt.second;

try

{

while (startAt.first < blkFiles_.size())

{

const BlkFile &f = blkFiles_[startAt.first];

finishOffset = readHeadersFromFile(

f, startAt.second, blockDataCallback

);

startAt.second = 0;

startAt.first++;

}

}

catch (stopReadingHeaders& e)

{

startAt.first++;

finishOffset = e.pos_;

}

return { startAt.first -1, finishOffset };

BlockFilePosition startAt,

BlockFilePosition stopAt,

const function<void(

const BinaryData &,

const BlockFilePosition &pos,

uint32_t blksize

)> &blockDataCallback

)

{

if (startAt.first == blkFiles_.size())

return startAt;

if (startAt.first > blkFiles_.size())

throw std::runtime_error("blkFile out of range");

stopAt.first = (std::min)(stopAt.first, blkFiles_.size());

uint64_t finishLocation=stopAt.second;

while (startAt.first <= stopAt.first)

{

const BlkFile &f = blkFiles_[startAt.first];

const uint64_t stopAtOffset

= startAt.first < stopAt.first ? f.filesize : stopAt.second;

finishLocation = readRawBlocksFromFile(

f, startAt.second, stopAtOffset, blockDataCallback

);

startAt.second = 0;

startAt.first++;

}

return { startAt.first-1, finishLocation };

uint32_t fnum,

const function<void(

const BinaryData &,

const BlockFilePosition &pos,

uint32_t blksize

)> &blockDataCallback

)

{

for (int32_t i = fnum; i > -1; i--)

{

const BlkFile &f = blkFiles_[i];

readRawBlocksFromFile(f, 0, f.filesize, blockDataCallback);

}

{

BlockFilePosition filePos = { 0, 0 };

//we dont have the file position for this header, let's find it

class StopReading : public std::exception

{

};

const BinaryData& thisHash = blk.getThisHash();

const auto stopIfBlkHeaderRecognized =

[&thisHash, &filePos](

const BinaryData &blockheader,

const BlockFilePosition &pos,

uint32_t blksize

)

{

filePos = pos;

BlockHeader block;

BinaryRefReader brr(blockheader);

block.unserialize(brr);

const HashString blockhash = block.getThisHash();

if (blockhash == thisHash)

throw StopReading();

};

try

{

//at this point, the last blkFile has been scanned for block, so skip it

for (int32_t i = blkFiles_.size() - 2; i > -1; i--)

{

readHeadersFromFile(

blkFiles_[i],

0,

stopIfBlkHeaderRecognized

);

}

}

catch (StopReading&)

{

// we're fine

}

blk.setBlockFileNum(filePos.first);

blk.setBlockFileOffset(filePos.second);

{

uint8_t* filemap_;

uint64_t size_;

{

MapAndSize mas;

#ifdef WIN32

int fd = _open(path.c_str(), _O_RDONLY | _O_BINARY);

if (fd == -1)

throw std::runtime_error("failed to open file");

HANDLE fdHandle = (HANDLE)_get_osfhandle(fd);

uint32_t sizelo = fileSize & 0xffffffff;

uint32_t sizehi = fileSize >> 16 >> 16;

HANDLE mh = CreateFileMapping(fdHandle, NULL,

PAGE_READONLY | SEC_COMMIT,

sizehi, sizelo, NULL);

if (mh == NULL)

throw std::runtime_error("failed to map file");

mas.filemap_ = (uint8_t*)MapViewOfFile(mh, FILE_MAP_READ,

0, 0, fileSize);

mas.size_ = fileSize;

if(mas.filemap_ == NULL)

throw std::runtime_error("failed to map file");

CloseHandle(mh);

_close(fd);

#else

int fd = open(path.c_str(), O_RDONLY);

if (fd == -1)

throw std::runtime_error("failed to open file");

mas.filemap_ = (uint8_t*)mmap(NULL, fileSize, PROT_READ, MAP_SHARED, fd, 0);

mas.size_ = fileSize;

if(mas.filemap_ == NULL)

throw std::runtime_error("failed to map file");

close(fd);

#endif

return mas;

{

#ifdef WIN32

if (!UnmapViewOfFile(mas.filemap_))

throw std::runtime_error("failed to unmap file");

#else

if(munmap(mas.filemap_, mas.size_))

throw std::runtime_error("failed to unmap file");

#endif

const BlkFile &f, uint64_t blockFileOffset, uint64_t stopBefore,

const function<void(

const BinaryData &,

const BlockFilePosition &pos,

uint32_t blksize

)> &blockDataCallback

)

{

// short circuit

if (blockFileOffset >= stopBefore)

return blockFileOffset;

MapAndSize mas = getMapOfFile(f.path, f.filesize);

BinaryData fileMagic(4);

memcpy(fileMagic.getPtr(), mas.filemap_, 4);

if( fileMagic != magicBytes_ )

{

LOGERR << "Block file '" << f.path << "' is the wrong network! File: "

<< fileMagic.toHexStr()

<< ", expecting " << magicBytes_.toHexStr();

}

// Seek to the supplied offset

uint64_t pos = blockFileOffset;

{

BinaryDataRef magic, szstr, rawBlk;

// read the file, we can't go past what we think is the end,

// because we haven't gone past that in Headers

while(pos < (std::min)(f.filesize, stopBefore))

{

magic = BinaryDataRef(mas.filemap_ + pos, 4);

pos += 4;

if (pos >= f.filesize)

break;

if(magic != magicBytes_)

{

// start scanning for MagicBytes

uint64_t offset = scanFor(mas.filemap_ + pos, f.filesize - pos,

magicBytes_.getPtr(), magicBytes_.getSize());

if (offset == UINT64_MAX)

{

LOGERR << "No more blocks found in file " << f.path;

break;

}

pos += offset +4;

LOGERR << "Next block header found at offset " << pos-4;

}

szstr = BinaryDataRef(mas.filemap_ + pos, 4);

pos += 4;

uint32_t blkSize = READ_UINT32_LE(szstr.getPtr());

if(pos >= f.filesize)

break;

rawBlk = BinaryDataRef(mas.filemap_ +pos, blkSize);

pos += blkSize;

try

{

blockDataCallback(rawBlk, { f.fnum, blockFileOffset }, blkSize);

}

catch (std::exception &e)

{

// this might very well just mean that we tried to load

// blkdata past where we loaded headers. This isn't a problem

LOGERR << e.what() << " (error encountered processing block at byte "

<< blockFileOffset << " file "

<< f.path << ", blocksize " << blkSize << ")";

}

blockFileOffset = pos;

}

}

LOGINFO << "Reading raw blocks finished at file "

<< f.fnum << " offset " << blockFileOffset;

unmapFile(mas);

return blockFileOffset;

const BlkFile &f,

uint64_t blockFileOffset,

const function<void(

const BinaryData &,

const BlockFilePosition &pos,

uint32_t blksize

)> &blockDataCallback

) const

{

ifstream is(f.path, ios::binary);

{

BinaryData fileMagic(4);

is.read(reinterpret_cast<char*>(fileMagic.getPtr()), 4);

if( fileMagic != magicBytes_)

{

std::ostringstream ss;

ss << "Block file '" << f.path << "' is the wrong network! File: "

<< fileMagic.toHexStr()

<< ", expecting " << magicBytes_.toHexStr();

throw runtime_error(ss.str());

}

}

is.seekg(blockFileOffset, ios::beg);

{

const uint32_t HEAD_AND_NTX_SZ = HEADER_SIZE + 10; // enough

BinaryData magic(4), szstr(4), rawHead(HEAD_AND_NTX_SZ);

while(!is.eof())

{

is.read((char*)magic.getPtr(), 4);

if (is.eof())

break;

if(magic != magicBytes_)

{

// I have to start scanning for MagicBytes

if (!scanFor(is, magicBytes_.getPtr(), magicBytes_.getSize()))

{

break;

}

LOGERR << "Next block header found at offset " << uint64_t(is.tellg())-4;

}

is.read(reinterpret_cast<char*>(szstr.getPtr()), 4);

uint32_t nextBlkSize = READ_UINT32_LE(szstr.getPtr());

if(is.eof()) break;

is.read(reinterpret_cast<char*>(rawHead.getPtr()), HEAD_AND_NTX_SZ); // plus #tx var_int

try

{

blockDataCallback(rawHead, { f.fnum, blockFileOffset }, nextBlkSize);

}

catch (debug_replay_blocks&)

{

blockFileOffset += nextBlkSize + 8;

throw stopReadingHeaders(f.fnum, blockFileOffset);

}

blockFileOffset += nextBlkSize+8;

is.seekg(nextBlkSize - HEAD_AND_NTX_SZ, ios::cur);

}

}

return blockFileOffset;

{

ifstream is(f.path, ios::binary);

is.seekg(0, ios::end);

if(is.tellg() < 88)

{

LOGERR << "File: " << f.path << " is less than 88 bytes!";

return {};

}

is.seekg(0, ios::beg);

BinaryData magic(4), szstr(4), rawHead(HEADER_SIZE);

is.read(magic.getCharPtr(), 4);

is.read(szstr.getCharPtr(), 4);

if(magic != magicBytes_)

{

LOGERR << "Magic bytes mismatch. Block file is for another network!";

return {};

}

is.read(rawHead.getCharPtr(), HEADER_SIZE);

BinaryData h(32);

BtcUtils::getHash256(rawHead, h);

return h;

{

armoryDbType = ARMORY_DB_BARE;

pruneType = DB_PRUNE_NONE;

{

if(netname == "Main")

{

genesisBlockHash = READHEX(MAINNET_GENESIS_HASH_HEX);

genesisTxHash = READHEX(MAINNET_GENESIS_TX_HASH_HEX);

magicBytes = READHEX(MAINNET_MAGIC_BYTES);

}

else if(netname == "Test")

{

genesisBlockHash = READHEX(TESTNET_GENESIS_HASH_HEX);

genesisTxHash = READHEX(TESTNET_GENESIS_TX_HASH_HEX);

magicBytes = READHEX(TESTNET_MAGIC_BYTES);

}

{

const uint64_t total_;

time_t then_;

uint64_t lastSample_=0;

double avgSpeed_=0.0;

ProgressMeasurer(uint64_t total)

: total_(total)

{

then_ = time(0);

}

void advance(uint64_t to)

{

static const double smoothingFactor=.75;

if (to == lastSample_) return;

const time_t now = time(0);

if (now == then_) return;

if (now < then_+10) return;

double speed = (to-lastSample_)/double(now-then_);

if (lastSample_ == 0)

avgSpeed_ = speed;

lastSample_ = to;

avgSpeed_ = smoothingFactor*speed + (1-smoothingFactor)*avgSpeed_;

then_ = now;

}

double fractionCompleted() const { return lastSample_/double(total_); }

double unitsPerSecond() const { return avgSpeed_; }

time_t remainingSeconds() const

{

return (total_-lastSample_)/unitsPerSecond();

}

: ScrAddrFilter(bdm->getIFace(), bdm->config().armoryDbType)

, bdm_(bdm)

{

uint32_t startBlock, uint32_t endBlock,

const vector<string>& wltIDs

)

{

class WalletIdProgressReporter : public ProgressReporter

{

const vector<string>& wIDs_;

const function<void(const vector<string>&, double prog,unsigned time)> &cb;

public:

WalletIdProgressReporter(

const vector<string>& wIDs,

const function<void(const vector<string>&, double prog,unsigned time)> &cb

)

: wIDs_(wIDs), cb(cb) {}

virtual void progress(

double progress, unsigned secondsRemaining

)

{

cb(wIDs_, progress, secondsRemaining);

}

};

WalletIdProgressReporter progress(wltIDs, scanThreadProgressCallback_);

//pass to false to skip SDBI top block updates

return bdm_->applyBlockRangeToDB(progress, startBlock, endBlock, *this, false);

: config_(bdmConfig)

, blockchain_(config_.genesisBlockHash)

{

auto isready = [this](void)->bool { return this->isReady(); };

iface_ = new LMDBBlockDatabase(isready);

scrAddrData_ = make_shared<BDM_ScrAddrFilter>(this);

setConfig(bdmConfig);

const BlockDataManagerConfig &bdmConfig)

{

config_ = bdmConfig;

readBlockHeaders_ = make_shared<BitcoinQtBlockFiles>(

config_.blkFileLocation,

config_.magicBytes

);

{

LOGINFO << "blkfile dir: " << config_.blkFileLocation;

LOGINFO << "lmdb dir: " << config_.levelDBLocation;

if (config_.genesisBlockHash.getSize() == 0)

{

throw runtime_error("ERROR: Genesis Block Hash not set!");

}

try

{

iface_->openDatabases(

config_.levelDBLocation,

config_.genesisBlockHash,

config_.genesisTxHash,

config_.magicBytes,

config_.armoryDbType,

config_.pruneType);

}

catch (runtime_error &e)

{

stringstream ss;

ss << "DB failed to open, reporting the following error: " << e.what();

throw runtime_error(ss.str());

}

catch (...)

{

stringstream ss;

ss << "DB failed to open, unknown error";

ss >> criticalError_;

throw runtime_error(ss.str());

}

ProgressReporter &prog,

const BlockFilePosition &fileAndOffset

)

{

readBlockHeaders_->detectAllBlkFiles();

vector<BlockHeader*> blockHeadersAdded;

ProgressFilter progfilter(

&prog,

readBlockHeaders_->totalBlockchainBytes()

);

uint64_t totalOffset=0;

bool suppressOutput = false;

if (fileAndOffset.first == 0 && fileAndOffset.second == 0)

suppressOutput = true;

class StopReading {};

auto blockHeaderCallback

= [&] (const BinaryData &blockdata, const BlockFilePosition &pos, uint32_t blksize)

{

BlockHeader block;

BinaryRefReader brr(blockdata);

block.unserialize(brr);

const HashString blockhash = block.getThisHash();

const uint32_t nTx = brr.get_var_int();

BlockHeader& addedBlock = blockchain().addNewBlock(

blockhash, block, suppressOutput);

blockHeadersAdded.push_back(&addedBlock);

addedBlock.setBlockFileNum(pos.first);

addedBlock.setBlockFileOffset(pos.second);

addedBlock.setNumTx(nTx);

addedBlock.setBlockSize(blksize);