{

ZT_ALWAYS_INLINE RootPeer() : lastSend(0),lastReceive(0),lastEcho(0),lastHello(0),vProto(-1),vMajor(-1),vMinor(-1),vRev(-1) {}

ZT_ALWAYS_INLINE ~RootPeer() { Utils::burn(key,sizeof(key)); }

Identity id; // Identity

uint8_t key[32]; // Shared secret key

InetAddress ip4,ip6; // IPv4 and IPv6 addresses

int64_t lastSend; // Time of last send (any packet)

int64_t lastReceive; // Time of last receive (any packet)

int64_t lastEcho; // Time of last received ECHO

int64_t lastHello; // Time of last received HELLO

int vProto; // Protocol version or -1 if unknown

int vMajor,vMinor,vRev; // Peer version or -1,-1,-1 if unknown

AtomicCounter __refCount;

{

RendezvousKey(const Address &aa,const Address &bb)

{

if (aa > bb) {

a = aa;

b = bb;

} else {

a = bb;

b = aa;

}

}

Address a,b;

ZT_ALWAYS_INLINE bool operator==(const RendezvousKey &k) const { return ((a == k.a)&&(b == k.b)); }

ZT_ALWAYS_INLINE bool operator!=(const RendezvousKey &k) const { return ((a != k.a)||(b != k.b)); }

struct Hasher { ZT_ALWAYS_INLINE std::size_t operator()(const RendezvousKey &k) const { return (std::size_t)(k.a.toInt() ^ k.b.toInt()); } };

};

struct RendezvousStats

{

RendezvousStats() : count(0),ts(0) {}

int64_t count;

int64_t ts;

};

static int64_t s_startTime; // Time service was started

static std::vector<int> s_ports; // Ports to bind for UDP traffic

static int s_relayMaxHops = 0; // Max relay hops

static Identity s_self; // My identity (including secret)

static std::atomic_bool s_run; // Remains true until shutdown is ordered

static json s_config; // JSON config file contents

static std::string s_statsRoot; // Root to write stats, peers, etc.

static std::atomic_bool s_geoInit; // True if geoIP data is initialized

static std::string s_googleMapsAPIKey; // Google maps API key for GeoIP /map feature

static std::map< std::pair< uint32_t,uint32_t >,std::pair< float,float > > s_geoIp4;

static std::map< std::pair< std::array< uint64_t,2 >,std::array< uint64_t,2 > >,std::pair< float,float > > s_geoIp6;

static Meter s_inputRate;

static Meter s_outputRate;

static Meter s_forwardRate;

static Meter s_discardedForwardRate;

static std::string s_planet;

static std::list< SharedPtr<RootPeer> > s_peers;

static std::unordered_map< uint64_t,std::unordered_map< MulticastGroup,std::unordered_map< Address,int64_t,AddressHasher >,MulticastGroupHasher > > s_multicastSubscriptions;

static std::unordered_map< Identity,SharedPtr<RootPeer>,IdentityHasher > s_peersByIdentity;

static std::unordered_map< Address,std::set< SharedPtr<RootPeer> >,AddressHasher > s_peersByVirtAddr;

static std::unordered_map< RendezvousKey,RendezvousStats,RendezvousKey::Hasher > s_rendezvousTracking;

static std::mutex s_planet_l;

static std::mutex s_peers_l;

static std::mutex s_multicastSubscriptions_l;

static std::mutex s_peersByIdentity_l;

static std::mutex s_peersByVirtAddr_l;

static std::mutex s_rendezvousTracking_l;

static ZT_ALWAYS_INLINE uint32_t ip4ToH32(const InetAddress &ip)

{

return Utils::ntoh((uint32_t)(((const struct sockaddr_in *)&ip)->sin_addr.s_addr));

}

static ZT_ALWAYS_INLINE std::array< uint64_t,2 > ip6ToH128(const InetAddress &ip)

{

std::array<uint64_t,2> i128;

memcpy(i128.data(),ip.rawIpData(),16);

i128[0] = Utils::ntoh(i128[0]);

i128[1] = Utils::ntoh(i128[1]);

return i128;

{

char ipstr[128],ipstr2[128],astr[32],astr2[32],tmpstr[256];

const bool fragment = pkt[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_INDICATOR] == ZT_PACKET_FRAGMENT_INDICATOR;

const Address source(pkt.source());

const Address dest(pkt.destination());

const int64_t now = OSUtils::now();

s_inputRate.log(now,pkt.size());

if ((!fragment)&&(pkt.size() < ZT_PROTO_MIN_PACKET_LENGTH))

return;

if ((!fragment)&&(!pkt.fragmented())&&(dest == s_self.address())) {

SharedPtr<RootPeer> peer;

// If this is an un-encrypted HELLO, either learn a new peer or verify

// that this is a peer we already know.

if ((pkt.cipher() == ZT_PROTO_CIPHER_SUITE__POLY1305_NONE)&&(pkt.verb() == Packet::VERB_HELLO)) {

Identity id;

if (id.deserialize(pkt,ZT_PROTO_VERB_HELLO_IDX_IDENTITY)) {

{

std::lock_guard<std::mutex> pbi_l(s_peersByIdentity_l);

auto pById = s_peersByIdentity.find(id);

if (pById != s_peersByIdentity.end()) {

peer = pById->second;

//printf("%s has %s (known (1))" ZT_EOL_S,ip->toString(ipstr),source().toString(astr));

}

}

if (peer) {

if (!pkt.dearmor(peer->key)) {

printf("%s HELLO rejected: packet authentication failed" ZT_EOL_S,ip->toString(ipstr));

return;

}

} else {

peer.set(new RootPeer);

if (!s_self.agree(id,peer->key)) {

printf("%s HELLO rejected: key agreement failed" ZT_EOL_S,ip->toString(ipstr));

return;

}

if (!pkt.dearmor(peer->key)) {

printf("%s HELLO rejected: packet authentication failed" ZT_EOL_S,ip->toString(ipstr));

return;

}

if (!pkt.uncompress()) {

printf("%s HELLO rejected: decompression failed" ZT_EOL_S,ip->toString(ipstr));

return;

}

peer->id = id;

peer->lastReceive = now;

bool added = false;

{

std::lock_guard<std::mutex> pbi_l(s_peersByIdentity_l);

auto existing = s_peersByIdentity.find(id); // make sure another thread didn't do this while we were

if (existing == s_peersByIdentity.end()) {

s_peersByIdentity.emplace(id,peer);

added = true;

} else {

peer = existing->second;

}

}

if (added) {

{

std::lock_guard<std::mutex> pl(s_peers_l);

s_peers.emplace_back(peer);

}

{

std::lock_guard<std::mutex> pbv_l(s_peersByVirtAddr_l);

s_peersByVirtAddr[id.address()].emplace(peer);

}

}

}

}

}

// If it wasn't a HELLO, check to see if any known identities for the sender's

// short ZT address successfully decrypt the packet.

if (!peer) {

std::lock_guard<std::mutex> pbv_l(s_peersByVirtAddr_l);

auto peers = s_peersByVirtAddr.find(source);

if (peers != s_peersByVirtAddr.end()) {

for(auto p=peers->second.begin();p!=peers->second.end();++p) {

if (pkt.dearmor((*p)->key)) {

if (!pkt.uncompress()) {

printf("%s packet rejected: decompression failed" ZT_EOL_S,ip->toString(ipstr));

return;

}

peer = (*p);

break;

}

}

}

}

// If we found the peer, update IP and/or time and handle certain key packet types that the

// root must concern itself with.

if (peer) {

if (ip->isV4())

peer->ip4 = ip;

else if (ip->isV6())

peer->ip6 = ip;

const int64_t now = OSUtils::now();

peer->lastReceive = now;

switch(pkt.verb()) {

case Packet::VERB_HELLO:

try {

if ((now - peer->lastHello) > 500) {

peer->lastHello = now;

peer->vProto = (int)pkt[ZT_PROTO_VERB_HELLO_IDX_PROTOCOL_VERSION];

peer->vMajor = (int)pkt[ZT_PROTO_VERB_HELLO_IDX_MAJOR_VERSION];

peer->vMinor = (int)pkt[ZT_PROTO_VERB_HELLO_IDX_MINOR_VERSION];

peer->vRev = (int)pkt.template at<uint16_t>(ZT_PROTO_VERB_HELLO_IDX_REVISION);

const uint64_t origId = pkt.packetId();

const uint64_t ts = pkt.template at<uint64_t>(ZT_PROTO_VERB_HELLO_IDX_TIMESTAMP);

pkt.reset(source,s_self.address(),Packet::VERB_OK);

pkt.append((uint8_t)Packet::VERB_HELLO);

pkt.append(origId);

pkt.append(ts);

pkt.append((uint8_t)ZT_PROTO_VERSION);

pkt.append((uint8_t)0);

pkt.append((uint8_t)0);

pkt.append((uint16_t)0);

ip->serialize(pkt);

if (peer->vProto < 20) { // send planet file for pre-2.x peers

std::lock_guard<std::mutex> pl(s_planet_l);

if (s_planet.length() > 0) {

pkt.append((uint16_t)s_planet.size());

pkt.append((const uint8_t *)s_planet.data(),s_planet.size());

}

}

pkt.armor(peer->key,true);

sendto(ip->isV4() ? v4s : v6s,pkt.data(),pkt.size(),SENDTO_FLAGS,(const struct sockaddr *)ip,(socklen_t)((ip->ss_family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6)));

s_outputRate.log(now,pkt.size());

peer->lastSend = now;

}

} catch ( ... ) {

printf("* unexpected exception handling HELLO from %s" ZT_EOL_S,ip->toString(ipstr));

}

break;

case Packet::VERB_ECHO:

try {

if ((now - peer->lastEcho) > 500) {

peer->lastEcho = now;

Packet outp(source,s_self.address(),Packet::VERB_OK);

outp.append((uint8_t)Packet::VERB_ECHO);

outp.append(pkt.packetId());

outp.append(((const uint8_t *)pkt.data()) + ZT_PACKET_IDX_PAYLOAD,pkt.size() - ZT_PACKET_IDX_PAYLOAD);

outp.compress();

outp.armor(peer->key,true);

sendto(ip->isV4() ? v4s : v6s,outp.data(),outp.size(),SENDTO_FLAGS,(const struct sockaddr *)ip,(socklen_t)((ip->ss_family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6)));

s_outputRate.log(now,outp.size());

peer->lastSend = now;

}

} catch ( ... ) {

printf("* unexpected exception handling ECHO from %s" ZT_EOL_S,ip->toString(ipstr));

}

case Packet::VERB_WHOIS:

try {

std::vector< SharedPtr<RootPeer> > results;

{

std::lock_guard<std::mutex> l(s_peersByVirtAddr_l);

for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;(ptr+ZT_ADDRESS_LENGTH)<=pkt.size();ptr+=ZT_ADDRESS_LENGTH) {

auto peers = s_peersByVirtAddr.find(Address(pkt.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH));

if (peers != s_peersByVirtAddr.end()) {

for(auto p=peers->second.begin();p!=peers->second.end();++p)

results.push_back(*p);

}

}

}

if (!results.empty()) {

const uint64_t origId = pkt.packetId();

pkt.reset(source,s_self.address(),Packet::VERB_OK);

pkt.append((uint8_t)Packet::VERB_WHOIS);

pkt.append(origId);

for(auto p=results.begin();p!=results.end();++p)

(*p)->id.serialize(pkt,false);

pkt.armor(peer->key,true);

sendto(ip->isV4() ? v4s : v6s,pkt.data(),pkt.size(),SENDTO_FLAGS,(const struct sockaddr *)ip,(socklen_t)((ip->ss_family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6)));

s_outputRate.log(now,pkt.size());

peer->lastSend = now;

}

} catch ( ... ) {

printf("* unexpected exception handling ECHO from %s" ZT_EOL_S,ip->toString(ipstr));

}

case Packet::VERB_MULTICAST_LIKE:

try {

std::lock_guard<std::mutex> l(s_multicastSubscriptions_l);

for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;(ptr+18)<=pkt.size();ptr+=18) {

const uint64_t nwid = pkt.template at<uint64_t>(ptr);

const MulticastGroup mg(MAC(pkt.field(ptr + 8,6),6),pkt.template at<uint32_t>(ptr + 14));

s_multicastSubscriptions[nwid][mg][source] = now;

}

} catch ( ... ) {

printf("* unexpected exception handling MULTICAST_LIKE from %s" ZT_EOL_S,ip->toString(ipstr));

}

break;

case Packet::VERB_MULTICAST_GATHER:

try {

const uint64_t nwid = pkt.template at<uint64_t>(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_NETWORK_ID);

//const unsigned int flags = pkt[ZT_PROTO_VERB_MULTICAST_GATHER_IDX_FLAGS];

const MulticastGroup mg(MAC(pkt.field(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_MAC,6),6),pkt.template at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI));

unsigned int gatherLimit = pkt.template at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT);

if (gatherLimit > 255)

gatherLimit = 255;

const uint64_t origId = pkt.packetId();

pkt.reset(source,s_self.address(),Packet::VERB_OK);

pkt.append((uint8_t)Packet::VERB_MULTICAST_GATHER);

pkt.append(origId);

pkt.append(nwid);

mg.mac().appendTo(pkt);

pkt.append((uint32_t)mg.adi());

{

std::lock_guard<std::mutex> l(s_multicastSubscriptions_l);

auto forNet = s_multicastSubscriptions.find(nwid);

if (forNet != s_multicastSubscriptions.end()) {

auto forGroup = forNet->second.find(mg);

if (forGroup != forNet->second.end()) {

pkt.append((uint32_t)forGroup->second.size());

const unsigned int countAt = pkt.size();

pkt.addSize(2);

unsigned int l = 0;

for(auto g=forGroup->second.begin();((l<gatherLimit)&&(g!=forGroup->second.end()));++g) {

if (g->first != source) {

++l;

g->first.appendTo(pkt);

}

}

if (l > 0) {

pkt.setAt<uint16_t>(countAt,(uint16_t)l);

pkt.armor(peer->key,true);

sendto(ip->isV4() ? v4s : v6s,pkt.data(),pkt.size(),SENDTO_FLAGS,(const struct sockaddr *)ip,(socklen_t)(ip->isV4() ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6)));

s_outputRate.log(now,pkt.size());

peer->lastSend = now;

}

}

}

}

} catch ( ... ) {

printf("* unexpected exception handling MULTICAST_GATHER from %s" ZT_EOL_S,ip->toString(ipstr));

}

break;

default:

break;

}

return;

}

}

// If we made it here, we are forwarding this packet to someone else and also possibly

// sending a RENDEZVOUS message.

int hops = 0;

bool introduce = false;

if (fragment) {

if ((hops = (int)reinterpret_cast<Packet::Fragment *>(&pkt)->incrementHops()) > s_relayMaxHops) {

//printf("%s refused to forward to %s: max hop count exceeded" ZT_EOL_S,ip->toString(ipstr),dest.toString(astr));

s_discardedForwardRate.log(now,pkt.size());

return;

}

} else {

if ((hops = (int)pkt.incrementHops()) > s_relayMaxHops) {

//printf("%s refused to forward to %s: max hop count exceeded" ZT_EOL_S,ip->toString(ipstr),dest.toString(astr));

s_discardedForwardRate.log(now,pkt.size());

return;

}

if (hops == 1) {

RendezvousKey rk(source,dest);

std::lock_guard<std::mutex> l(s_rendezvousTracking_l);

RendezvousStats &lr = s_rendezvousTracking[rk];

if ((now - lr.ts) >= 30000) {

++lr.count;

lr.ts = now;

introduce = true;

}

}

}

std::vector< std::pair< InetAddress *,SharedPtr<RootPeer> > > toAddrs;

{

std::lock_guard<std::mutex> pbv_l(s_peersByVirtAddr_l);

auto peers = s_peersByVirtAddr.find(dest);

if (peers != s_peersByVirtAddr.end()) {

for(auto p=peers->second.begin();p!=peers->second.end();++p) {

if ((*p)->ip4) {

toAddrs.emplace_back(std::pair< InetAddress *,SharedPtr<RootPeer> >(&((*p)->ip4),*p));

} else if ((*p)->ip6) {

toAddrs.emplace_back(std::pair< InetAddress *,SharedPtr<RootPeer> >(&((*p)->ip6),*p));

}

}

}

}

if (toAddrs.empty()) {

s_discardedForwardRate.log(now,pkt.size());

return;

}

if (introduce) {

std::lock_guard<std::mutex> l(s_peersByVirtAddr_l);

auto sources = s_peersByVirtAddr.find(source);

if (sources != s_peersByVirtAddr.end()) {

for(auto a=sources->second.begin();a!=sources->second.end();++a) {

for(auto b=toAddrs.begin();b!=toAddrs.end();++b) {

if (((*a)->ip6)&&(b->second->ip6)) {

//printf("* introducing %s(%s) to %s(%s)" ZT_EOL_S,ip->toString(ipstr),source.toString(astr),b->second->ip6.toString(ipstr2),dest.toString(astr2));

// Introduce source to destination (V6)

Packet outp(source,s_self.address(),Packet::VERB_RENDEZVOUS);

outp.append((uint8_t)0);

dest.appendTo(outp);

outp.append((uint16_t)b->second->ip6.port());

outp.append((uint8_t)16);

outp.append((const uint8_t *)(b->second->ip6.rawIpData()),16);

outp.armor((*a)->key,true);

sendto(v6s,outp.data(),outp.size(),SENDTO_FLAGS,(const struct sockaddr *)&((*a)->ip6),(socklen_t)sizeof(struct sockaddr_in6));

s_outputRate.log(now,outp.size());

(*a)->lastSend = now;

// Introduce destination to source (V6)

outp.reset(dest,s_self.address(),Packet::VERB_RENDEZVOUS);

outp.append((uint8_t)0);

source.appendTo(outp);

outp.append((uint16_t)(*a)->ip6.port());

outp.append((uint8_t)16);

outp.append((const uint8_t *)((*a)->ip6.rawIpData()),16);

outp.armor(b->second->key,true);

sendto(v6s,outp.data(),outp.size(),SENDTO_FLAGS,(const struct sockaddr *)&(b->second->ip6),(socklen_t)sizeof(struct sockaddr_in6));

s_outputRate.log(now,outp.size());

b->second->lastSend = now;

}

if (((*a)->ip4)&&(b->second->ip4)) {

//printf("* introducing %s(%s) to %s(%s)" ZT_EOL_S,ip->toString(ipstr),source.toString(astr),b->second->ip4.toString(ipstr2),dest.toString(astr2));

// Introduce source to destination (V4)

Packet outp(source,s_self.address(),Packet::VERB_RENDEZVOUS);

outp.append((uint8_t)0);

dest.appendTo(outp);

outp.append((uint16_t)b->second->ip4.port());

outp.append((uint8_t)4);

outp.append((const uint8_t *)b->second->ip4.rawIpData(),4);

outp.armor((*a)->key,true);

sendto(v4s,outp.data(),outp.size(),SENDTO_FLAGS,(const struct sockaddr *)&((*a)->ip4),(socklen_t)sizeof(struct sockaddr_in));

s_outputRate.log(now,outp.size());

(*a)->lastSend = now;

// Introduce destination to source (V4)

outp.reset(dest,s_self.address(),Packet::VERB_RENDEZVOUS);

outp.append((uint8_t)0);

source.appendTo(outp);

outp.append((uint16_t)(*a)->ip4.port());

outp.append((uint8_t)4);

outp.append((const uint8_t *)((*a)->ip4.rawIpData()),4);

outp.armor(b->second->key,true);

sendto(v4s,outp.data(),outp.size(),SENDTO_FLAGS,(const struct sockaddr *)&(b->second->ip4),(socklen_t)sizeof(struct sockaddr_in));

s_outputRate.log(now,outp.size());

b->second->lastSend = now;

}

}

}

}

}

for(auto i=toAddrs.begin();i!=toAddrs.end();++i) {

if (sendto(i->first->isV4() ? v4s : v6s,pkt.data(),pkt.size(),SENDTO_FLAGS,(const struct sockaddr *)i->first,(socklen_t)(i->first->isV4() ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6))) > 0) {

s_outputRate.log(now,pkt.size());

s_forwardRate.log(now,pkt.size());

i->second->lastSend = now;

}

}

{

const int s = socket(bindAddr->sa_family,SOCK_DGRAM,0);

if (s < 0) {

close(s);

return -1;

}

int f = 16777216;

while (f > 131072) {

if (setsockopt(s,SOL_SOCKET,SO_RCVBUF,(const char *)&f,sizeof(f)) == 0)

break;

f -= 131072;

}

f = 16777216;

while (f > 131072) {

if (setsockopt(s,SOL_SOCKET,SO_SNDBUF,(const char *)&f,sizeof(f)) == 0)

break;

f -= 131072;

}

if (bindAddr->sa_family == AF_INET6) {

f = 1; setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,(void *)&f,sizeof(f));

#ifdef IPV6_MTU_DISCOVER

f = 0; setsockopt(s,IPPROTO_IPV6,IPV6_MTU_DISCOVER,&f,sizeof(f));

#endif

#ifdef IPV6_DONTFRAG

f = 0; setsockopt(s,IPPROTO_IPV6,IPV6_DONTFRAG,&f,sizeof(f));

#endif

}

#ifdef IP_DONTFRAG

f = 0; setsockopt(s,IPPROTO_IP,IP_DONTFRAG,&f,sizeof(f));

#endif

#ifdef IP_MTU_DISCOVER

f = IP_PMTUDISC_DONT; setsockopt(s,IPPROTO_IP,IP_MTU_DISCOVER,&f,sizeof(f));

#endif

#ifdef SO_REUSEPORT

f = 1; setsockopt(s,SOL_SOCKET,SO_REUSEPORT,(void *)&f,sizeof(f));

#endif

#ifndef __LINUX__ // linux wants just SO_REUSEPORT

f = 1; setsockopt(s,SOL_SOCKET,SO_REUSEADDR,(void *)&f,sizeof(f));

#endif

#ifdef __LINUX__

struct timeval tv;

tv.tv_sec = 1;

tv.tv_usec = 0;

setsockopt(s,SOL_SOCKET,SO_RCVTIMEO,(const void *)&tv,sizeof(tv));

#endif

if (bind(s,bindAddr,(bindAddr->sa_family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6))) {

close(s);

//printf("%s\n",strerror(errno));

return -1;

}

return s;

{

std::vector<std::thread> threads;

std::vector<int> sockets;

int v4Sock = -1,v6Sock = -1;

signal(SIGTERM,shutdownSigHandler);

signal(SIGINT,shutdownSigHandler);

signal(SIGQUIT,shutdownSigHandler);

signal(SIGPIPE,SIG_IGN);

signal(SIGUSR1,SIG_IGN);

signal(SIGUSR2,SIG_IGN);

signal(SIGCHLD,SIG_IGN);

s_startTime = OSUtils::now();

s_geoInit = false;

if (argc < 3) {

printf("Usage: zerotier-root <identity.secret> <config path>" ZT_EOL_S);

return 1;

}

{

std::string myIdStr;

if (!OSUtils::readFile(argv[1],myIdStr)) {

printf("FATAL: cannot read identity.secret at %s" ZT_EOL_S,argv[1]);

return 1;

}

if (!s_self.fromString(myIdStr.c_str())) {

printf("FATAL: cannot read identity.secret at %s (invalid identity)" ZT_EOL_S,argv[1]);

return 1;

}

if (!s_self.hasPrivate()) {

printf("FATAL: cannot read identity.secret at %s (missing secret key)" ZT_EOL_S,argv[1]);

return 1;

}

}

{

std::string configStr;

if (!OSUtils::readFile(argv[2],configStr)) {

printf("FATAL: cannot read config file at %s" ZT_EOL_S,argv[2]);

return 1;

}

try {

s_config = json::parse(configStr);

} catch (std::exception &exc) {

printf("FATAL: config file at %s invalid: %s" ZT_EOL_S,argv[2],exc.what());

return 1;

} catch ( ... ) {

printf("FATAL: config file at %s invalid: unknown exception" ZT_EOL_S,argv[2]);

return 1;

}

if (!s_config.is_object()) {

printf("FATAL: config file at %s invalid: does not contain a JSON object" ZT_EOL_S,argv[2]);

return 1;

}

}

try {

auto jport = s_config["port"];

if (jport.is_array()) {

for(long i=0;i<(long)jport.size();++i) {

int port = jport[i];

if ((port <= 0)||(port > 65535)) {

printf("FATAL: invalid port in config file %d" ZT_EOL_S,port);

return 1;

}

s_ports.push_back(port);

}

} else {

int port = jport;

if ((port <= 0)||(port > 65535)) {

printf("FATAL: invalid port in config file %d" ZT_EOL_S,port);

return 1;

}

s_ports.push_back(port);

}

} catch ( ... ) {}

if (s_ports.empty())

s_ports.push_back(ZT_DEFAULT_PORT);

std::sort(s_ports.begin(),s_ports.end());

int httpPort = ZT_DEFAULT_PORT;

try {

httpPort = s_config["httpPort"];

if ((httpPort <= 0)||(httpPort > 65535)) {

printf("FATAL: invalid HTTP port in config file %d" ZT_EOL_S,httpPort);

return 1;

}

} catch ( ... ) {

httpPort = ZT_DEFAULT_PORT;

}

std::string planetFilePath;

try {

planetFilePath = s_config["planetFile"];

} catch ( ... ) {

planetFilePath = "";

}

try {

s_statsRoot = s_config["statsRoot"];

while ((s_statsRoot.length() > 0)&&(s_statsRoot[s_statsRoot.length()-1] == ZT_PATH_SEPARATOR))

s_statsRoot = s_statsRoot.substr(0,s_statsRoot.length()-1);

if (s_statsRoot.length() > 0)

OSUtils::mkdir(s_statsRoot);

} catch ( ... ) {

s_statsRoot = "";

}

s_relayMaxHops = ZT_RELAY_MAX_HOPS;

try {

s_relayMaxHops = s_config["relayMaxHops"];

if (s_relayMaxHops > ZT_PROTO_MAX_HOPS)

s_relayMaxHops = ZT_PROTO_MAX_HOPS;

else if (s_relayMaxHops < 0)

s_relayMaxHops = 0;

} catch ( ... ) {

s_relayMaxHops = ZT_RELAY_MAX_HOPS;

}

try {

s_googleMapsAPIKey = s_config["googleMapsAPIKey"];

std::string geoIpPath = s_config["geoIp"];

if (geoIpPath.length() > 0) {

FILE *gf = fopen(geoIpPath.c_str(),"rb");

if (gf) {

threads.emplace_back(std::thread([gf]() {

try {

char line[1024];

line[1023] = 0;

while (fgets(line,sizeof(line)-1,gf)) {

InetAddress start,end;

float lat = 0.0F,lon = 0.0F;

int field = 0;

for(char *saveptr=nullptr,*f=Utils::stok(line,",\r\n",&saveptr);(f);f=Utils::stok(nullptr,",\r\n",&saveptr)) {

switch(field++) {

case 0:

start.fromString(f);

break;

case 1:

end.fromString(f);

break;

case 2:

lat = strtof(f,nullptr);

break;

case 3:

lon = strtof(f,nullptr);

break;

}

}

if ((start)&&(end)&&(start.ss_family == end.ss_family)&&(lat >= -90.0F)&&(lat <= 90.0F)&&(lon >= -180.0F)&&(lon <= 180.0F)) {

if (start.ss_family == AF_INET) {

s_geoIp4[std::pair< uint32_t,uint32_t >(ip4ToH32(start),ip4ToH32(end))] = std::pair< float,float >(lat,lon);

} else if (start.ss_family == AF_INET6) {

s_geoIp6[std::pair< std::array< uint64_t,2 >,std::array< uint64_t,2 > >(ip6ToH128(start),ip6ToH128(end))] = std::pair< float,float >(lat,lon);

}

}

}

s_geoInit = true;

} catch ( ... ) {}

fclose(gf);

}));

}

}

} catch ( ... ) {}

unsigned int ncores = std::thread::hardware_concurrency();

if (ncores == 0) ncores = 1;

s_run = true;

for(auto port=s_ports.begin();port!=s_ports.end();++port) {

for(unsigned int tn=0;tn<ncores;++tn) {

struct sockaddr_in6 in6;

memset(&in6,0,sizeof(in6));

in6.sin6_family = AF_INET6;

in6.sin6_port = htons((uint16_t)*port);

const int s6 = bindSocket((struct sockaddr *)&in6);

if (s6 < 0) {

std::cout << "ERROR: unable to bind to port " << *port << ZT_EOL_S;

exit(1);

}

struct sockaddr_in in4;

memset(&in4,0,sizeof(in4));

in4.sin_family = AF_INET;

in4.sin_port = htons((uint16_t)*port);

const int s4 = bindSocket((struct sockaddr *)&in4);

if (s4 < 0) {

std::cout << "ERROR: unable to bind to port " << *port << ZT_EOL_S;

exit(1);

}

sockets.push_back(s6);

sockets.push_back(s4);

if (v4Sock < 0) v4Sock = s4;

if (v6Sock < 0) v6Sock = s6;

threads.push_back(std::thread([s6,s4]() {

struct sockaddr_in6 in6;

Packet *pkt = new Packet();

for(;;) {

memset(&in6,0,sizeof(in6));

socklen_t sl = sizeof(in6);

const int pl = (int)recvfrom(s6,pkt->unsafeData(),pkt->capacity(),RECVFROM_FLAGS,(struct sockaddr *)&in6,&sl);

if (pl > 0) {

if ((pl >= ZT_PROTO_MIN_FRAGMENT_LENGTH)&&(pl <= ZT_PROTO_MAX_PACKET_LENGTH)) {

try {

pkt->setSize((unsigned int)pl);

handlePacket(s4,s6,reinterpret_cast<const InetAddress *>(&in6),*pkt);

} catch (std::exception &exc) {

char ipstr[128];

printf("WARNING: unexpected exception handling packet from %s: %s" ZT_EOL_S,reinterpret_cast<const InetAddress *>(&in6)->toString(ipstr),exc.what());

} catch (int exc) {

char ipstr[128];

printf("WARNING: unexpected exception handling packet from %s: ZT exception code %d" ZT_EOL_S,reinterpret_cast<const InetAddress *>(&in6)->toString(ipstr),exc);

} catch ( ... ) {

char ipstr[128];

printf("WARNING: unexpected exception handling packet from %s: unknown exception" ZT_EOL_S,reinterpret_cast<const InetAddress *>(&in6)->toString(ipstr));

}

}

} else if (!s_run) {

break;

}

}

delete pkt;

}));

threads.push_back(std::thread([s6,s4]() {

struct sockaddr_in in4;

Packet *pkt = new Packet();

for(;;) {

memset(&in4,0,sizeof(in4));

socklen_t sl = sizeof(in4);

const int pl = (int)recvfrom(s4,pkt->unsafeData(),pkt->capacity(),RECVFROM_FLAGS,(struct sockaddr *)&in4,&sl);

if (pl > 0) {

if ((pl >= ZT_PROTO_MIN_FRAGMENT_LENGTH)&&(pl <= ZT_PROTO_MAX_PACKET_LENGTH)) {

try {

pkt->setSize((unsigned int)pl);

handlePacket(s4,s6,reinterpret_cast<const InetAddress *>(&in4),*pkt);

} catch (std::exception &exc) {

char ipstr[128];

printf("WARNING: unexpected exception handling packet from %s: %s" ZT_EOL_S,reinterpret_cast<const InetAddress *>(&in4)->toString(ipstr),exc.what());

} catch (int exc) {

char ipstr[128];

printf("WARNING: unexpected exception handling packet from %s: ZT exception code %d" ZT_EOL_S,reinterpret_cast<const InetAddress *>(&in4)->toString(ipstr),exc);

} catch ( ... ) {

char ipstr[128];

printf("WARNING: unexpected exception handling packet from %s: unknown exception" ZT_EOL_S,reinterpret_cast<const InetAddress *>(&in4)->toString(ipstr));

}

}

} else if (!s_run) {

break;

}

}

delete pkt;

}));

}

}

// A minimal read-only local API for monitoring and status queries

httplib::Server apiServ;

threads.push_back(std::thread([&apiServ,httpPort]() {

// Human readable status page

apiServ.Get("/",[](const httplib::Request &req,httplib::Response &res) {

std::ostringstream o;

o << "ZeroTier Root Server " << ZEROTIER_ONE_VERSION_MAJOR << '.' << ZEROTIER_ONE_VERSION_MINOR << '.' << ZEROTIER_ONE_VERSION_REVISION << ZT_EOL_S;

o << "(c)2019 ZeroTier, Inc." ZT_EOL_S "Licensed under the ZeroTier BSL 1.1" ZT_EOL_S ZT_EOL_S;

s_peersByIdentity_l.lock();

o << "Peers Online: " << s_peersByIdentity.size() << ZT_EOL_S;

s_peersByIdentity_l.unlock();

res.set_content(o.str(),"text/plain");

});

apiServ.Get("/metrics",[](const httplib::Request &req, httplib::Response &res) {

std::ostringstream o;

int64_t now = OSUtils::now();

char buf[11];

const char *root_id = s_self.address().toString(buf);

o << "# HELP root_peers_online Number of active peers online" << ZT_EOL_S;

o << "# TYPE root_peers_online gauge" << ZT_EOL_S;

s_peersByIdentity_l.lock();

o << "root_peers_online{root_id=\"" << root_id << "\"} " << s_peersByIdentity.size() << ZT_EOL_S;

s_peersByIdentity_l.unlock();

o << "# HELP root_input_rate Input rate MiB/s" << ZT_EOL_S;

o << "# TYPE root_input_rate gauge" << ZT_EOL_S;

o << "root_input_rate{root_id=\"" << root_id << "\"} " << std::setprecision(5) << (s_inputRate.perSecond(now)/1048576.0) << ZT_EOL_S;

o << "# HELP root_output_rate Output rate MiB/s" << ZT_EOL_S;

o << "# TYPE root_output_rate gauge" << ZT_EOL_S;

o << "root_output_rate{root_id=\"" << root_id << "\"} " << std::setprecision(5) << (s_outputRate.perSecond(now)/1048576.0) << ZT_EOL_S;

o << "# HELP root_forwarded_rate Forwarded packet rate MiB/s" << ZT_EOL_S;

o << "# TYPE root_forwarded_rate gauge" << ZT_EOL_S;

o << "root_forwarded_rate{root_id=\"" << root_id << "\"} " << std::setprecision(5) << (s_forwardRate.perSecond(now)/1048576.0) << ZT_EOL_S;