Skip to content

Commit 2fa15a0

Browse files
murali-reddyDaanHoogland
murali-reddy
authored and
DaanHoogland
committed
CLOUDSTACK-6609: OVS distributed routing: ensure tunnels are created if
not created already when OvsVpcPhysicalTopologyConfigCommand update is recived Currently if the tunnel creation fails, there is no retry logic. Fix ensures OvsVpcPhysicalTopologyConfigCommand updates as an opputiunity to ensure proper tunnels are established between the hosts.
1 parent 0adf892 commit 2fa15a0

5 files changed

Lines changed: 203 additions & 149 deletions

File tree

‎plugins/network-elements/ovs/src/com/cloud/network/ovs/OvsTunnelManagerImpl.java‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -710,7 +710,7 @@ private void handleVmStateChange(VMInstanceVO vm) {
710710
String bridgeName=generateBridgeNameForVpc(vpcId);
711711

712712
OvsVpcPhysicalTopologyConfigCommand topologyConfigCommand = prepareVpcTopologyUpdate(vpcId);
713-
topologyConfigCommand.setSequenceNumber(getNextSequenceNumber(vpcId));
713+
topologyConfigCommand.setSequenceNumber(getNextTopologyUpdateSequenceNumber(vpcId));
714714

715715
// send topology change update to VPC spanned hosts
716716
for (Long id: vpcSpannedHostIds) {
@@ -820,7 +820,7 @@ public void onPublishMessage(String senderAddress, String subject, Object args)
820820
if (network.getVpcId() != null && isVpcEnabledForDistributedRouter(network.getVpcId())) {
821821
long vpcId = network.getVpcId();
822822
OvsVpcRoutingPolicyConfigCommand cmd = prepareVpcRoutingPolicyUpdate(vpcId);
823-
cmd.setSequenceNumber(getNextSequenceNumber(vpcId));
823+
cmd.setSequenceNumber(getNextRoutingPolicyUpdateSequenceNumber(vpcId));
824824

825825
// get the list of hosts on which VPC spans (i.e hosts that need to be aware of VPC
826826
// network ACL update)
@@ -901,7 +901,7 @@ private boolean sendVpcRoutingPolicyChangeUpdate(OvsVpcRoutingPolicyConfigComman
901901
}
902902
}
903903

904-
private long getNextSequenceNumber(final long vpcId) {
904+
private long getNextTopologyUpdateSequenceNumber(final long vpcId) {
905905

906906
try {
907907
return Transaction.execute(new TransactionCallback<Long>() {
@@ -913,9 +913,31 @@ public Long doInTransaction(TransactionStatus status) {
913913
_vpcDrSeqNoDao.persist(seqVo);
914914
}
915915
seqVo = _vpcDrSeqNoDao.lockRow(seqVo.getId(), true);
916-
seqVo.incrSequenceNo();
916+
seqVo.incrTopologyUpdateSequenceNo();
917917
_vpcDrSeqNoDao.update(seqVo.getId(), seqVo);
918-
return seqVo.getSequenceNo();
918+
return seqVo.getTopologyUpdateSequenceNo();
919+
}
920+
});
921+
} finally {
922+
923+
}
924+
}
925+
926+
private long getNextRoutingPolicyUpdateSequenceNumber(final long vpcId) {
927+
928+
try {
929+
return Transaction.execute(new TransactionCallback<Long>() {
930+
@Override
931+
public Long doInTransaction(TransactionStatus status) {
932+
VpcDistributedRouterSeqNoVO seqVo = _vpcDrSeqNoDao.findByVpcId(vpcId);
933+
if (seqVo == null) {
934+
seqVo = new VpcDistributedRouterSeqNoVO(vpcId);
935+
_vpcDrSeqNoDao.persist(seqVo);
936+
}
937+
seqVo = _vpcDrSeqNoDao.lockRow(seqVo.getId(), true);
938+
seqVo.incrPolicyUpdateSequenceNo();
939+
_vpcDrSeqNoDao.update(seqVo.getId(), seqVo);
940+
return seqVo.getPolicyUpdateSequenceNo();
919941
}
920942
});
921943
} finally {

‎plugins/network-elements/ovs/src/com/cloud/network/ovs/dao/VpcDistributedRouterSeqNoVO.java‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,11 @@ public class VpcDistributedRouterSeqNoVO implements InternalIdentity {
3737
@Column(name = "vpc_id", updatable = false, nullable = false)
3838
private Long vpcId;
3939

40-
@Column(name = "sequence_no")
41-
long sequenceNo = 0;
40+
@Column(name = "topology_update_sequence_no")
41+
long topologyUpdateSequenceNo = 0;
42+
43+
@Column(name = "routing_policy__update_sequence_no")
44+
long policyUpdateSequenceNo = 0;
4245

4346
protected VpcDistributedRouterSeqNoVO() {
4447

@@ -62,11 +65,19 @@ public void setVpcId(Long vpcId) {
6265
this.vpcId = vpcId;
6366
}
6467

65-
public long getSequenceNo() {
66-
return sequenceNo;
68+
public long getTopologyUpdateSequenceNo() {
69+
return topologyUpdateSequenceNo;
70+
}
71+
72+
public void incrTopologyUpdateSequenceNo() {
73+
topologyUpdateSequenceNo++;
74+
}
75+
76+
public long getPolicyUpdateSequenceNo() {
77+
return policyUpdateSequenceNo;
6778
}
6879

69-
public void incrSequenceNo() {
70-
sequenceNo++;
80+
public void incrPolicyUpdateSequenceNo() {
81+
policyUpdateSequenceNo++;
7182
}
7283
}

‎scripts/vm/hypervisor/xenserver/cloudstack_pluginlib.py‎

Lines changed: 146 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -344,8 +344,139 @@ def get_acl(vpcconfig, required_acl_id):
344344
return acl
345345
return None
346346

347-
# Configures the bridge created for a VPC enabled for distributed routing. Management server sends VPC physical topology
348-
# details. Based on the VPC physical topology L2 lookup table and L3 lookup tables are updated by this function.
347+
def check_tunnel_exists(bridge, tunnel_name):
348+
res = do_cmd([VSCTL_PATH, "port-to-br", tunnel_name])
349+
return res == bridge
350+
351+
def create_tunnel(bridge, remote_ip, gre_key, src_host, dst_host, network_uuid):
352+
353+
logging.debug("Creating tunnel from host %s" %src_host + " to host %s" %dst_host + " with GRE key %s" %gre_key)
354+
355+
res = check_switch()
356+
if res != "SUCCESS":
357+
logging.debug("Openvswitch running: NO")
358+
return "FAILURE:%s" % res
359+
360+
# We need to keep the name below 14 characters
361+
# src and target are enough - consider a fixed length hash
362+
name = "t%s-%s-%s" % (gre_key, src_host, dst_host)
363+
364+
# Verify the xapi bridge to be created
365+
# NOTE: Timeout should not be necessary anymore
366+
wait = [VSCTL_PATH, "--timeout=30", "wait-until", "bridge",
367+
bridge, "--", "get", "bridge", bridge, "name"]
368+
res = do_cmd(wait)
369+
if bridge not in res:
370+
logging.debug("WARNING:Can't find bridge %s for creating " +
371+
"tunnel!" % bridge)
372+
return "FAILURE:NO_BRIDGE"
373+
logging.debug("bridge %s for creating tunnel - VERIFIED" % bridge)
374+
tunnel_setup = False
375+
drop_flow_setup = False
376+
try:
377+
# Create a port and configure the tunnel interface for it
378+
add_tunnel = [VSCTL_PATH, "add-port", bridge,
379+
name, "--", "set", "interface",
380+
name, "type=gre", "options:key=%s" % gre_key,
381+
"options:remote_ip=%s" % remote_ip]
382+
do_cmd(add_tunnel)
383+
tunnel_setup = True
384+
# verify port
385+
verify_port = [VSCTL_PATH, "get", "port", name, "interfaces"]
386+
res = do_cmd(verify_port)
387+
# Expecting python-style list as output
388+
iface_list = []
389+
if len(res) > 2:
390+
iface_list = res.strip()[1:-1].split(',')
391+
if len(iface_list) != 1:
392+
logging.debug("WARNING: Unexpected output while verifying " +
393+
"port %s on bridge %s" % (name, bridge))
394+
return "FAILURE:VERIFY_PORT_FAILED"
395+
396+
# verify interface
397+
iface_uuid = iface_list[0]
398+
verify_interface_key = [VSCTL_PATH, "get", "interface",
399+
iface_uuid, "options:key"]
400+
verify_interface_ip = [VSCTL_PATH, "get", "interface",
401+
iface_uuid, "options:remote_ip"]
402+
403+
key_validation = do_cmd(verify_interface_key)
404+
ip_validation = do_cmd(verify_interface_ip)
405+
406+
if not gre_key in key_validation or not remote_ip in ip_validation:
407+
logging.debug("WARNING: Unexpected output while verifying " +
408+
"interface %s on bridge %s" % (name, bridge))
409+
return "FAILURE:VERIFY_INTERFACE_FAILED"
410+
logging.debug("Tunnel interface validated:%s" % verify_interface_ip)
411+
cmd_tun_ofport = [VSCTL_PATH, "get", "interface",
412+
iface_uuid, "ofport"]
413+
tun_ofport = do_cmd(cmd_tun_ofport)
414+
# Ensure no trailing LF
415+
if tun_ofport.endswith('\n'):
416+
tun_ofport = tun_ofport[:-1]
417+
# find xs network for this bridge, verify is used for ovs tunnel network
418+
xs_nw_uuid = do_cmd([XE_PATH, "network-list",
419+
"bridge=%s" % bridge, "--minimal"])
420+
ovs_tunnel_network = False
421+
try:
422+
ovs_tunnel_network = do_cmd([XE_PATH,"network-param-get",
423+
"uuid=%s" % xs_nw_uuid,
424+
"param-name=other-config",
425+
"param-key=is-ovs-tun-network", "--minimal"])
426+
except:
427+
pass
428+
429+
ovs_vpc_distributed_vr_network = False
430+
try:
431+
ovs_vpc_distributed_vr_network = do_cmd([XE_PATH,"network-param-get",
432+
"uuid=%s" % xs_nw_uuid,
433+
"param-name=other-config",
434+
"param-key=is-ovs-vpc-distributed-vr-network", "--minimal"])
435+
except:
436+
pass
437+
438+
if ovs_tunnel_network == 'True':
439+
# add flow entryies for dropping broadcast coming in from gre tunnel
440+
add_flow(bridge, priority=1000, in_port=tun_ofport,
441+
dl_dst='ff:ff:ff:ff:ff:ff', actions='drop')
442+
add_flow(bridge, priority=1000, in_port=tun_ofport,
443+
nw_dst='224.0.0.0/24', actions='drop')
444+
drop_flow_setup = True
445+
logging.debug("Broadcast drop rules added")
446+
447+
if ovs_vpc_distributed_vr_network == 'True':
448+
# add flow rules for dropping broadcast coming in from tunnel ports
449+
add_flow(bridge, priority=1000, in_port=tun_ofport, table=0,
450+
dl_dst='ff:ff:ff:ff:ff:ff', actions='drop')
451+
add_flow(bridge, priority=1000, in_port=tun_ofport, table=0,
452+
nw_dst='224.0.0.0/24', actions='drop')
453+
454+
# add flow rule to send the traffic from tunnel ports to L2 switching table only
455+
add_flow(bridge, priority=1100, in_port=tun_ofport, table=0, actions='resubmit(,1)')
456+
457+
# mark tunnel interface with network id for which this tunnel was created
458+
do_cmd([VSCTL_PATH, "set", "interface", name, "options:cloudstack-network-id=%s" % network_uuid])
459+
update_flooding_rules_on_port_plug_unplug(bridge, name, 'online', network_uuid)
460+
461+
logging.debug("Successfully created tunnel from host %s" %src_host + " to host %s" %dst_host +
462+
" with GRE key %s" %gre_key)
463+
return "SUCCESS:%s creation succeeded" % name
464+
except:
465+
logging.debug("An unexpected error occured. Rolling back")
466+
if tunnel_setup:
467+
logging.debug("Deleting GRE interface")
468+
# Destroy GRE port and interface
469+
del_port(bridge, name)
470+
if drop_flow_setup:
471+
# Delete flows
472+
logging.debug("Deleting flow entries from GRE interface")
473+
del_flows(bridge, in_port=tun_ofport)
474+
# This will not cancel the original exception
475+
raise
476+
477+
# Configures the bridge created for a VPC that is enabled for distributed routing. Management server sends VPC
478+
# physical topology details (which VM from which tier running on which host etc). Based on the VPC physical topology L2
479+
# lookup table and L3 lookup tables are updated by this function.
349480
def configure_vpc_bridge_for_network_topology(bridge, this_host_id, json_config, sequence_no):
350481

351482
vpconfig = jsonLoader(json.loads(json_config)).vpc
@@ -412,8 +543,13 @@ def configure_vpc_bridge_for_network_topology(bridge, this_host_id, json_config,
412543
network = get_network_details(vpconfig, nic.networkuuid)
413544
gre_key = network.grekey
414545

415-
# generate tunnel name as per the tunnel naming convention and get the OF port
546+
# generate tunnel name as per the tunnel naming convention
416547
tunnel_name = "t%s-%s-%s" % (gre_key, this_host_id, host.hostid)
548+
549+
# check if tunnel exists already, if not create a tunnel from this host to remote host
550+
if not check_tunnel_exists(bridge, tunnel_name):
551+
create_tunnel(bridge, host.ipaddress, gre_key, this_host_id, host.hostid, network.networkuuid)
552+
417553
of_port = get_ofport_for_vif(tunnel_name)
418554

419555
# Add flow rule in L2 look up table, if packet's destination mac matches MAC of the VM's nic
@@ -441,10 +577,10 @@ def configure_vpc_bridge_for_network_topology(bridge, this_host_id, json_config,
441577
del_flows(bridge, table=L3_LOOKUP_TABLE)
442578

443579
ofspec.seek(0)
444-
logging.debug("Adding below flows rules L2 & L3 lookup tables:\n" + ofspec.read())
580+
logging.debug("Adding below flows rules in L2 & L3 lookup tables:\n" + ofspec.read())
581+
ofspec.close()
445582

446583
# update bridge with the flow-rules for L2 lookup and L3 lookup in the file in one attempt
447-
ofspec.close()
448584
do_cmd([OFCTL_PATH, 'add-flows', bridge, ofspec_filename])
449585

450586
# now that we updated the bridge with flow rules close and delete the file.
@@ -460,8 +596,9 @@ def configure_vpc_bridge_for_network_topology(bridge, this_host_id, json_config,
460596
os.remove(ofspec_filename)
461597
raise error_message
462598

463-
# Configures the bridge created for a VPC enabled for distributed firewall. Management server sends VPC routing policies
464-
# details. Based on the VPC routing policies ingress ACL table and egress ACL tables are updated by this function.
599+
# Configures the bridge created for a VPC that is enabled for distributed firewall. Management server sends VPC routing
600+
# policy (network ACL applied on the tiers etc) details. Based on the VPC routing policies ingress ACL table and
601+
# egress ACL tables are updated by this function.
465602
def configure_vpc_bridge_for_routing_policies(bridge, json_config, sequence_no):
466603

467604
vpconfig = jsonLoader(json.loads(json_config)).vpc
@@ -564,9 +701,9 @@ def configure_vpc_bridge_for_routing_policies(bridge, json_config, sequence_no):
564701

565702
ofspec.seek(0)
566703
logging.debug("Adding below flows rules Ingress & Egress ACL tables:\n" + ofspec.read())
704+
ofspec.close()
567705

568706
# update bridge with the flow-rules for ingress and egress ACL's added in the file in one attempt
569-
ofspec.close()
570707
do_cmd([OFCTL_PATH, 'add-flows', bridge, ofspec_filename])
571708

572709
# now that we updated the bridge with flow rules delete the file.
@@ -658,9 +795,9 @@ class tier_ports:
658795

659796
ofspec.seek(0)
660797
logging.debug("Adding below flows rules L2 flooding table: \n" + ofspec.read())
798+
ofspec.close()
661799

662800
# update bridge with the flow-rules for broadcast rules added in the file in one attempt
663-
ofspec.close()
664801
do_cmd([OFCTL_PATH, 'add-flows', bridge, ofspec_filename])
665802

666803
# now that we updated the bridge with flow rules delete the file.

0 commit comments

Comments
 (0)