Skip to content

Commit 87562af

Browse files
vdemeestervdemeester
authored
Merge pull request moby#32283 from aboch/clearingress
Daemon to take care of ingress cleanup on cluster leave and graceful shutdown
2 parents 9fe781a + 6f4bb79 commit 87562af

18 files changed

Lines changed: 167 additions & 97 deletions

File tree

‎daemon/cluster/executor/backend.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ type Backend interface {
2727
CreateManagedNetwork(clustertypes.NetworkCreateRequest) error
2828
DeleteManagedNetwork(name string) error
2929
FindNetwork(idName string) (libnetwork.Network, error)
30-
SetupIngress(req clustertypes.NetworkCreateRequest, nodeIP string) error
31-
ReleaseIngress() error
30+
SetupIngress(clustertypes.NetworkCreateRequest, string) (<-chan struct{}, error)
31+
ReleaseIngress() (<-chan struct{}, error)
3232
PullImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
3333
CreateManagedContainer(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error)
3434
ContainerStart(name string, hostConfig *container.HostConfig, checkpoint string, checkpointDir string) error

‎daemon/cluster/executor/container/executor.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,13 +139,15 @@ func (e *executor) Configure(ctx context.Context, node *api.Node) error {
139139
options.IPAM.Config = append(options.IPAM.Config, c)
140140
}
141141

142-
return e.backend.SetupIngress(clustertypes.NetworkCreateRequest{
142+
_, err := e.backend.SetupIngress(clustertypes.NetworkCreateRequest{
143143
ID: na.Network.ID,
144144
NetworkCreateRequest: types.NetworkCreateRequest{
145145
Name: na.Network.Spec.Annotations.Name,
146146
NetworkCreate: options,
147147
},
148148
}, na.Addresses[0])
149+
150+
return err
149151
}
150152

151153
// Controller returns a docker container runner.

‎daemon/daemon.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -445,7 +445,25 @@ func (daemon *Daemon) DaemonLeavesCluster() {
445445
// Daemon is in charge of removing the attachable networks with
446446
// connected containers when the node leaves the swarm
447447
daemon.clearAttachableNetworks()
448+
// We no longer need the cluster provider, stop it now so that
449+
// the network agent will stop listening to cluster events.
448450
daemon.setClusterProvider(nil)
451+
// Wait for the networking cluster agent to stop
452+
daemon.netController.AgentStopWait()
453+
// Daemon is in charge of removing the ingress network when the
454+
// node leaves the swarm. Wait for job to be done or timeout.
455+
// This is called also on graceful daemon shutdown. We need to
456+
// wait, because the ingress release has to happen before the
457+
// network controller is stopped.
458+
if done, err := daemon.ReleaseIngress(); err == nil {
459+
select {
460+
case <-done:
461+
case <-time.After(5 * time.Second):
462+
logrus.Warnf("timeout while waiting for ingress network removal")
463+
}
464+
} else {
465+
logrus.Warnf("failed to initiate ingress network removal: %v", err)
466+
}
449467
}
450468

451469
// setClusterProvider sets a component for querying the current cluster state.
@@ -832,6 +850,12 @@ func (daemon *Daemon) Shutdown() error {
832850
}
833851
}
834852

853+
// If we are part of a cluster, clean up cluster's stuff
854+
if daemon.clusterProvider != nil {
855+
logrus.Debugf("start clean shutdown of cluster resources...")
856+
daemon.DaemonLeavesCluster()
857+
}
858+
835859
// Shutdown plugins after containers and layerstore. Don't change the order.
836860
daemon.pluginShutdown()
837861

‎daemon/network.go‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,9 @@ func (daemon *Daemon) getAllNetworks() []libnetwork.Network {
101101
}
102102

103103
type ingressJob struct {
104-
create *clustertypes.NetworkCreateRequest
105-
ip net.IP
104+
create *clustertypes.NetworkCreateRequest
105+
ip net.IP
106+
jobDone chan struct{}
106107
}
107108

108109
var (
@@ -124,6 +125,7 @@ func (daemon *Daemon) startIngressWorker() {
124125
daemon.releaseIngress(ingressID)
125126
ingressID = ""
126127
}
128+
close(r.jobDone)
127129
}
128130
}
129131
}()
@@ -137,19 +139,23 @@ func (daemon *Daemon) enqueueIngressJob(job *ingressJob) {
137139
}
138140

139141
// SetupIngress setups ingress networking.
140-
func (daemon *Daemon) SetupIngress(create clustertypes.NetworkCreateRequest, nodeIP string) error {
142+
// The function returns a channel which will signal the caller when the programming is completed.
143+
func (daemon *Daemon) SetupIngress(create clustertypes.NetworkCreateRequest, nodeIP string) (<-chan struct{}, error) {
141144
ip, _, err := net.ParseCIDR(nodeIP)
142145
if err != nil {
143-
return err
146+
return nil, err
144147
}
145-
daemon.enqueueIngressJob(&ingressJob{&create, ip})
146-
return nil
148+
done := make(chan struct{})
149+
daemon.enqueueIngressJob(&ingressJob{&create, ip, done})
150+
return done, nil
147151
}
148152

149153
// ReleaseIngress releases the ingress networking.
150-
func (daemon *Daemon) ReleaseIngress() error {
151-
daemon.enqueueIngressJob(&ingressJob{nil, nil})
152-
return nil
154+
// The function returns a channel which will signal the caller when the programming is completed.
155+
func (daemon *Daemon) ReleaseIngress() (<-chan struct{}, error) {
156+
done := make(chan struct{})
157+
daemon.enqueueIngressJob(&ingressJob{nil, nil, done})
158+
return done, nil
153159
}
154160

155161
func (daemon *Daemon) setupIngress(create *clustertypes.NetworkCreateRequest, ip net.IP, staleID string) {

‎vendor.conf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ github.com/RackSec/srslog 456df3a81436d29ba874f3590eeeee25d666f8a5
2424
github.com/imdario/mergo 0.2.1
2525

2626
#get libnetwork packages
27-
github.com/docker/libnetwork b6cb1eee1e7fc27ee05f0eb830d3e60e67a88565
27+
github.com/docker/libnetwork f3c4ca8ce5c128e071bab198c4ed9fd0d08384eb
2828
github.com/docker/go-events 18b43f1bc85d9cdd42c05a6cd2d444c7a200a894
2929
github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
3030
github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec

‎vendor/github.com/docker/libnetwork/agent.go‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/github.com/docker/libnetwork/controller.go‎

Lines changed: 19 additions & 40 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/github.com/docker/libnetwork/drivers/overlay/joinleave.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go‎

Lines changed: 19 additions & 10 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/github.com/docker/libnetwork/drivers/overlay/ov_serf.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)