diff --git a/api/provisioner/provisioner.go b/api/provisioner/provisioner.go index 3d49bb97f90..ffff923dc66 100644 --- a/api/provisioner/provisioner.go +++ b/api/provisioner/provisioner.go @@ -252,26 +252,25 @@ func (st *State) SetHostMachineNetworkConfig(hostMachineID string, netConfig []p return nil } -func (st *State) HostChangesForContainer(containerTag names.MachineTag) ([]network.DeviceToBridge, error) { +func (st *State) HostChangesForContainer(containerTag names.MachineTag) ([]network.DeviceToBridge, int, error) { var result params.HostNetworkChangeResults args := params.Entities{ Entities: []params.Entity{{Tag: containerTag.String()}}, } if err := st.facade.FacadeCall("HostChangesForContainers", args, &result); err != nil { - return nil, err + return nil, 0, err } if len(result.Results) != 1 { - return nil, errors.Errorf("expected 1 result, got %d", len(result.Results)) + return nil, 0, errors.Errorf("expected 1 result, got %d", len(result.Results)) } if err := result.Results[0].Error; err != nil { - return nil, err + return nil, 0, err } newBridges := result.Results[0].NewBridges res := make([]network.DeviceToBridge, len(newBridges)) for i, bridgeInfo := range newBridges { res[i].BridgeName = bridgeInfo.BridgeName res[i].DeviceName = bridgeInfo.HostDeviceName - res[i].NetBondReconfigureDelay = bridgeInfo.NetBondReconfigureDelay } - return res, nil + return res, result.Results[0].ReconfigureDelay, nil } diff --git a/api/provisioner/provisioner_test.go b/api/provisioner/provisioner_test.go index ac4edcd9353..2ad890caa74 100644 --- a/api/provisioner/provisioner_test.go +++ b/api/provisioner/provisioner_test.go @@ -709,7 +709,7 @@ func (s *provisionerSuite) TestHostChangesForContainer(c *gc.C) { container, err := s.State.AddMachineInsideMachine(containerTemplate, s.machine.Id(), instance.LXD) c.Assert(err, jc.ErrorIsNil) - changes, err := s.provisioner.HostChangesForContainer(container.MachineTag()) + changes, reconfigureDelay, err := s.provisioner.HostChangesForContainer(container.MachineTag()) c.Assert(err, gc.ErrorMatches, "dummy provider network config not supported.*") c.Skip("can't test without network support") c.Assert(err, jc.ErrorIsNil) @@ -717,4 +717,5 @@ func (s *provisionerSuite) TestHostChangesForContainer(c *gc.C) { BridgeName: "br-ens3", DeviceName: "ens3", }}) + c.Check(reconfigureDelay, gc.Equals, 0) } diff --git a/apiserver/params/network.go b/apiserver/params/network.go index 8364eef70e8..bb96228dbd7 100644 --- a/apiserver/params/network.go +++ b/apiserver/params/network.go @@ -135,9 +135,8 @@ type NetworkConfig struct { // DeviceBridgeInfo lists the host device and the expected bridge to be // created. type DeviceBridgeInfo struct { - HostDeviceName string `json:"host-device-name"` - BridgeName string `json:"bridge-name"` - NetBondReconfigureDelay int `json:"net-bond-reconfigure-delay"` + HostDeviceName string `json:"host-device-name"` + BridgeName string `json:"bridge-name"` } // ProviderInterfaceInfoResults holds the results of a @@ -471,6 +470,10 @@ type HostNetworkChange struct { // NewBridges lists the bridges that need to be created and what host // device they should be connected to. NewBridges []DeviceBridgeInfo `json:"new-bridges"` + + // ReconfigureDelay is the duration in seconds to sleep before + // raising the bridged interface + ReconfigureDelay int `json:"reconfigure-delay"` } // HostNetworkChangeResults holds the network changes that are necessary for multiple containers to be created. diff --git a/apiserver/provisioner/provisioner.go b/apiserver/provisioner/provisioner.go index 2d891f969eb..c11b98be3d6 100644 --- a/apiserver/provisioner/provisioner.go +++ b/apiserver/provisioner/provisioner.go @@ -797,18 +797,19 @@ type hostChangesContext struct { } func (ctx *hostChangesContext) ProcessOneContainer(netEnv environs.NetworkingEnviron, idx int, host, container *state.Machine) error { - bridges, err := host.FindMissingBridgesForContainer(container) + netBondReconfigureDelay := netEnv.Config().NetBondReconfigureDelay() + bridges, reconfigureDelay, err := host.FindMissingBridgesForContainer(container, netBondReconfigureDelay) if err != nil { return err } - netBondReconfigureDelay := netEnv.Config().NetBondReconfigureDelay() + + ctx.result.Results[idx].ReconfigureDelay = reconfigureDelay for _, bridgeInfo := range bridges { ctx.result.Results[idx].NewBridges = append( ctx.result.Results[idx].NewBridges, params.DeviceBridgeInfo{ - HostDeviceName: bridgeInfo.DeviceName, - BridgeName: bridgeInfo.BridgeName, - NetBondReconfigureDelay: netBondReconfigureDelay, + HostDeviceName: bridgeInfo.DeviceName, + BridgeName: bridgeInfo.BridgeName, }) } return nil diff --git a/network/add-juju-bridge.py b/network/add-juju-bridge.py index 461c3a1166f..46dbd2cd965 100644 --- a/network/add-juju-bridge.py +++ b/network/add-juju-bridge.py @@ -393,7 +393,7 @@ def arg_parser(): parser.add_argument('--interfaces-to-bridge', help="interfaces to bridge; space delimited", type=str, required=True) parser.add_argument('--dry-run', help="dry run, no activation", action='store_true', default=False, required=False) parser.add_argument('--bridge-name', help="bridge name", type=str, required=False) - parser.add_argument('--net-bond-reconfigure-delay', help="delay in seconds before raising bonded interfaces", type=int, required=False, default=30) + parser.add_argument('--reconfigure-delay', help="delay in seconds before raising interfaces", type=int, required=False, default=10) parser.add_argument('filename', help="interfaces(5) based filename") return parser @@ -440,22 +440,8 @@ def main(args): print_stanzas(stanzas, f) f.close() - # On configurations that have bonds in 802.3ad mode there is a - # race condition betweeen an immediate ifdown then ifup. - # - # On the h/w I have a 'sleep 0.1' is sufficient but to accommodate - # other setups we arbitrarily choose something larger. We don't - # want to massively slow bootstrap down but, equally, 0.1 may be - # too small for other configurations. - - for s in stanzas: - if s.is_logical_interface and s.iface.is_bonded: - print("working around https://bugs.launchpad.net/ubuntu/+source/ifenslave/+bug/1269921") - print("working around https://bugs.launchpad.net/juju-core/+bug/1594855") - if args.dry_run and args.dry_run > 0: - shell_cmd("sleep {}".format(args.net_bond_reconfigure_delay, dry_run=args.dry_run)) - break - + if args.reconfigure_delay and args.reconfigure_delay > 0 : + shell_cmd("sleep {}".format(args.reconfigure_delay), dry_run=args.dry_run) shell_cmd("cat {}".format(args.filename), dry_run=args.dry_run) shell_cmd("ifup --exclude=lo --interfaces={} -a".format(args.filename), dry_run=args.dry_run) shell_cmd("ip -d link show", dry_run=args.dry_run) diff --git a/network/bridge.go b/network/bridge.go index 1edd897520a..1502ecb422d 100644 --- a/network/bridge.go +++ b/network/bridge.go @@ -19,7 +19,7 @@ type Bridger interface { // TODO(frobware) - we may want a different type to encompass // and reflect how bridging should be done vis-a-vis what // needs to be bridged. - Bridge(devices []DeviceToBridge) error + Bridge(devices []DeviceToBridge, reconfigureDelay int) error } type etcNetworkInterfacesBridger struct { @@ -64,9 +64,9 @@ func pythonInterpreters() []string { return result } -func (b *etcNetworkInterfacesBridger) Bridge(devices []DeviceToBridge) error { - cmd := bridgeCmd(devices, b.PythonInterpreter, b.BridgePrefix, b.Filename, BridgeScriptPythonContent, b.DryRun) - infoCmd := bridgeCmd(devices, b.PythonInterpreter, b.BridgePrefix, b.Filename, "", b.DryRun) +func (b *etcNetworkInterfacesBridger) Bridge(devices []DeviceToBridge, reconfigureDelay int) error { + cmd := bridgeCmd(devices, b.PythonInterpreter, b.BridgePrefix, b.Filename, BridgeScriptPythonContent, b.DryRun, reconfigureDelay) + infoCmd := bridgeCmd(devices, b.PythonInterpreter, b.BridgePrefix, b.Filename, "", b.DryRun, reconfigureDelay) logger.Infof("bridgescript command=%s", infoCmd) result, err := runCommand(cmd, b.Environ, b.Clock, b.Timeout) if err != nil { @@ -86,7 +86,7 @@ func (b *etcNetworkInterfacesBridger) Bridge(devices []DeviceToBridge) error { return nil } -func bridgeCmd(devices []DeviceToBridge, pythonInterpreter, bridgePrefix, filename, pythonScript string, dryRun bool) string { +func bridgeCmd(devices []DeviceToBridge, pythonInterpreter, bridgePrefix, filename, pythonScript string, dryRun bool, reconfigureDelay int) string { dryRunOption := "" if bridgePrefix != "" { @@ -97,20 +97,16 @@ func bridgeCmd(devices []DeviceToBridge, pythonInterpreter, bridgePrefix, filena dryRunOption = "--dry-run" } - netBondReconfigureDelay := 0 - netBondReconfigureDelayOption := "" + reconfigureDelayOption := "" deviceNames := make([]string, len(devices)) for i, d := range devices { - if d.NetBondReconfigureDelay > netBondReconfigureDelay { - netBondReconfigureDelay = d.NetBondReconfigureDelay - } deviceNames[i] = d.DeviceName } - if netBondReconfigureDelay > 0 { - netBondReconfigureDelayOption = fmt.Sprintf("--net-bond-reconfigure-delay=%v", netBondReconfigureDelay) + if reconfigureDelay >= 0 { + reconfigureDelayOption = fmt.Sprintf("--reconfigure-delay=%v", reconfigureDelay) } return fmt.Sprintf(` @@ -122,7 +118,7 @@ EOF strings.Join(deviceNames, " "), bridgePrefix, dryRunOption, - netBondReconfigureDelayOption, + reconfigureDelayOption, filename, pythonScript) } diff --git a/network/bridge_test.go b/network/bridge_test.go index 724a832d08b..3a756d81f7e 100644 --- a/network/bridge_test.go +++ b/network/bridge_test.go @@ -47,17 +47,17 @@ func assertCmdResult(c *gc.C, cmd, expected string) { c.Assert(string(result.Stderr), gc.Equals, "") } -func assertBridgeCmd(c *gc.C, devices []network.DeviceToBridge, bridgePrefix, filename, script string, dryRun bool, expected string) { +func assertBridgeCmd(c *gc.C, devices []network.DeviceToBridge, bridgePrefix, filename, script string, dryRun bool, reconfigureDelay int, expected string) { for _, python := range network.PythonInterpreters() { - cmd := network.BridgeCmd(devices, python, bridgePrefix, filename, script, dryRun) + cmd := network.BridgeCmd(devices, python, bridgePrefix, filename, script, dryRun, reconfigureDelay) assertCmdResult(c, cmd, expected) } } -func assertENIBridgerError(c *gc.C, devices []network.DeviceToBridge, environ []string, timeout time.Duration, clock clock.Clock, bridgePrefix, filename string, dryRun bool, expected string) { +func assertENIBridgerError(c *gc.C, devices []network.DeviceToBridge, environ []string, timeout time.Duration, clock clock.Clock, bridgePrefix, filename string, dryRun bool, reconfigureDelay int, expected string) { for _, python := range network.PythonInterpreters() { bridger := network.NewEtcNetworkInterfacesBridger(python, environ, clock, timeout, bridgePrefix, filename, dryRun) - err := bridger.Bridge(devices) + err := bridger.Bridge(devices, reconfigureDelay) c.Assert(err, gc.NotNil) c.Assert(err, gc.ErrorMatches, expected) } @@ -76,10 +76,11 @@ func (*BridgeSuite) TestBridgeCmdArgumentsNoBridgePrefixAndDryRun(c *gc.C) { }, } dryRun := true - assertBridgeCmd(c, devices, "", "/etc/network/interfaces", echoArgsScript, dryRun, ` + assertBridgeCmd(c, devices, "", "/etc/network/interfaces", echoArgsScript, dryRun, 0, ` --interfaces-to-bridge=ens3 ens4 bond0 --activate --dry-run +--reconfigure-delay=0 /etc/network/interfaces `[1:]) } @@ -96,10 +97,11 @@ func (*BridgeSuite) TestBridgeCmdArgumentsNoNetBondReconfigureDelay(c *gc.C) { DeviceName: "bond0", }, } - assertBridgeCmd(c, devices, "", "/etc/network/interfaces", echoArgsScript, true, ` + assertBridgeCmd(c, devices, "", "/etc/network/interfaces", echoArgsScript, true, 0, ` --interfaces-to-bridge=ens3 ens4 bond0 --activate --dry-run +--reconfigure-delay=0 /etc/network/interfaces `[1:]) } @@ -116,11 +118,12 @@ func (*BridgeSuite) TestBridgeCmdArgumentsWithBridgePrefixAndDryRun(c *gc.C) { DeviceName: "bond0", }, } - assertBridgeCmd(c, devices, "foo-", "/etc/network/interfaces", echoArgsScript, true, ` + assertBridgeCmd(c, devices, "foo-", "/etc/network/interfaces", echoArgsScript, true, 0, ` --interfaces-to-bridge=ens3 ens4 bond0 --activate --bridge-prefix=foo- --dry-run +--reconfigure-delay=0 /etc/network/interfaces `[1:]) } @@ -137,10 +140,11 @@ func (*BridgeSuite) TestBridgeCmdArgumentsWithBridgePrefixWithoutDryRun(c *gc.C) DeviceName: "bond0", }, } - assertBridgeCmd(c, devices, "foo-", "/etc/network/interfaces", echoArgsScript, false, ` + assertBridgeCmd(c, devices, "foo-", "/etc/network/interfaces", echoArgsScript, false, 0, ` --interfaces-to-bridge=ens3 ens4 bond0 --activate --bridge-prefix=foo- +--reconfigure-delay=0 /etc/network/interfaces `[1:]) } @@ -157,9 +161,10 @@ func (*BridgeSuite) TestBridgeCmdArgumentsWithoutBridgePrefixAndWithoutDryRun(c DeviceName: "bond0", }, } - assertBridgeCmd(c, devices, "", "/etc/network/interfaces", echoArgsScript, false, ` + assertBridgeCmd(c, devices, "", "/etc/network/interfaces", echoArgsScript, false, 0, ` --interfaces-to-bridge=ens3 ens4 bond0 --activate +--reconfigure-delay=0 /etc/network/interfaces `[1:]) } @@ -167,22 +172,19 @@ func (*BridgeSuite) TestBridgeCmdArgumentsWithoutBridgePrefixAndWithoutDryRun(c func (*BridgeSuite) TestBridgeCmdArgumentsWithNetBondReconfigureDelay(c *gc.C) { devices := []network.DeviceToBridge{ network.DeviceToBridge{ - DeviceName: "ens3", - NetBondReconfigureDelay: 0, + DeviceName: "ens3", }, network.DeviceToBridge{ - DeviceName: "ens4", - NetBondReconfigureDelay: 4, + DeviceName: "ens4", }, network.DeviceToBridge{ - DeviceName: "bond0", - NetBondReconfigureDelay: 2, + DeviceName: "bond0", }, } - assertBridgeCmd(c, devices, "", "/etc/network/interfaces", echoArgsScript, false, ` + assertBridgeCmd(c, devices, "", "/etc/network/interfaces", echoArgsScript, false, 4, ` --interfaces-to-bridge=ens3 ens4 bond0 --activate ---net-bond-reconfigure-delay=4 +--reconfigure-delay=4 /etc/network/interfaces `[1:]) } @@ -190,13 +192,13 @@ func (*BridgeSuite) TestBridgeCmdArgumentsWithNetBondReconfigureDelay(c *gc.C) { func (*BridgeSuite) TestENIBridgerWithMissingFilenameArgument(c *gc.C) { devices := []network.DeviceToBridge{} expected := `(?s)bridgescript failed:.*(too few arguments|the following arguments are required: filename)\n` - assertENIBridgerError(c, devices, os.Environ(), 0, clock.WallClock, "br-", "", true, expected) + assertENIBridgerError(c, devices, os.Environ(), 0, clock.WallClock, "br-", "", true, 0, expected) } func (*BridgeSuite) TestENIBridgerWithEmptyDeviceNamesArgument(c *gc.C) { devices := []network.DeviceToBridge{} expected := `(?s)bridgescript failed:.*(too few arguments|no interfaces specified)\n` - assertENIBridgerError(c, devices, os.Environ(), 0, clock.WallClock, "br-", "testdata/non-existent-filename", true, expected) + assertENIBridgerError(c, devices, os.Environ(), 0, clock.WallClock, "br-", "testdata/non-existent-filename", true, 0, expected) } func (*BridgeSuite) TestENIBridgerWithNonExistentFile(c *gc.C) { @@ -206,7 +208,7 @@ func (*BridgeSuite) TestENIBridgerWithNonExistentFile(c *gc.C) { }, } expected := `(?s).*(IOError|FileNotFoundError):.*No such file or directory: 'testdata/non-existent-file'\n` - assertENIBridgerError(c, devices, os.Environ(), 0, clock.WallClock, "br-", "testdata/non-existent-file", true, expected) + assertENIBridgerError(c, devices, os.Environ(), 0, clock.WallClock, "br-", "testdata/non-existent-file", true, 0, expected) } func (*BridgeSuite) TestENIBridgerWithTimeout(c *gc.C) { @@ -217,7 +219,8 @@ func (*BridgeSuite) TestENIBridgerWithTimeout(c *gc.C) { } environ := os.Environ() environ = append(environ, "ADD_JUJU_BRIDGE_SLEEP_PREAMBLE_FOR_TESTING=10") - assertENIBridgerError(c, devices, environ, 500*time.Millisecond, clock.WallClock, "br-", "testdata/non-existent-file", true, "bridgescript timed out after 500ms") + expected := "bridgescript timed out after 500ms" + assertENIBridgerError(c, devices, environ, 500*time.Millisecond, clock.WallClock, "br-", "testdata/non-existent-file", true, 0, expected) } func (*BridgeSuite) TestENIBridgerWithDryRun(c *gc.C) { @@ -228,7 +231,7 @@ func (*BridgeSuite) TestENIBridgerWithDryRun(c *gc.C) { } for _, python := range network.PythonInterpreters() { bridger := network.NewEtcNetworkInterfacesBridger(python, os.Environ(), clock.WallClock, 0, "", "testdata/interfaces", true) - err := bridger.Bridge(devices) + err := bridger.Bridge(devices, 0) c.Assert(err, gc.IsNil) } } diff --git a/network/bridgescript.go b/network/bridgescript.go index ed469912cbb..1aa2b0a9bb8 100644 --- a/network/bridgescript.go +++ b/network/bridgescript.go @@ -397,7 +397,7 @@ def arg_parser(): parser.add_argument('--interfaces-to-bridge', help="interfaces to bridge; space delimited", type=str, required=True) parser.add_argument('--dry-run', help="dry run, no activation", action='store_true', default=False, required=False) parser.add_argument('--bridge-name', help="bridge name", type=str, required=False) - parser.add_argument('--net-bond-reconfigure-delay', help="delay in seconds before raising bonded interfaces", type=int, required=False, default=30) + parser.add_argument('--reconfigure-delay', help="delay in seconds before raising interfaces", type=int, required=False, default=10) parser.add_argument('filename', help="interfaces(5) based filename") return parser @@ -444,22 +444,8 @@ def main(args): print_stanzas(stanzas, f) f.close() - # On configurations that have bonds in 802.3ad mode there is a - # race condition betweeen an immediate ifdown then ifup. - # - # On the h/w I have a 'sleep 0.1' is sufficient but to accommodate - # other setups we arbitrarily choose something larger. We don't - # want to massively slow bootstrap down but, equally, 0.1 may be - # too small for other configurations. - - for s in stanzas: - if s.is_logical_interface and s.iface.is_bonded: - print("working around https://bugs.launchpad.net/ubuntu/+source/ifenslave/+bug/1269921") - print("working around https://bugs.launchpad.net/juju-core/+bug/1594855") - if args.dry_run and args.dry_run > 0: - shell_cmd("sleep {}".format(args.net_bond_reconfigure_delay, dry_run=args.dry_run)) - break - + if args.reconfigure_delay and args.reconfigure_delay > 0 : + shell_cmd("sleep {}".format(args.reconfigure_delay), dry_run=args.dry_run) shell_cmd("cat {}".format(args.filename), dry_run=args.dry_run) shell_cmd("ifup --exclude=lo --interfaces={} -a".format(args.filename), dry_run=args.dry_run) shell_cmd("ip -d link show", dry_run=args.dry_run) diff --git a/network/bridgescript_test.go b/network/bridgescript_test.go index 7cb44a1dcb7..599cee8cec4 100644 --- a/network/bridgescript_test.go +++ b/network/bridgescript_test.go @@ -74,14 +74,14 @@ func (s *bridgeConfigSuite) assertScript(c *gc.C, initialConfig, expectedConfig, } } -func (s *bridgeConfigSuite) assertScriptWithActivationAndDryRun(c *gc.C, isBonded, isAlreadyBridged bool, initialConfig, bridgePrefix string, netBondReconfigureDelay int, interfacesToBridge []string) { - expectedConfig := s.dryRunExpectedOutputHelper(isBonded, isAlreadyBridged, bridgePrefix, netBondReconfigureDelay, interfacesToBridge) +func (s *bridgeConfigSuite) assertScriptWithActivationAndDryRun(c *gc.C, isAlreadyBridged bool, initialConfig, bridgePrefix string, reconfigureDelay int, interfacesToBridge []string) { + expectedConfig := s.dryRunExpectedOutputHelper(isAlreadyBridged, bridgePrefix, reconfigureDelay, interfacesToBridge) for i, python := range s.pythonVersions { c.Logf("test #%v using %s", i, python) err := ioutil.WriteFile(s.testConfigPath, []byte(strings.TrimSuffix(initialConfig, "\n")), 0644) c.Check(err, jc.ErrorIsNil) - output, retcode := s.runScriptWithActivationAndDryRun(c, python, bridgePrefix, netBondReconfigureDelay, interfacesToBridge) + output, retcode := s.runScriptWithActivationAndDryRun(c, python, bridgePrefix, reconfigureDelay, interfacesToBridge) c.Check(retcode, gc.Equals, 0) c.Check(len(output), gc.Equals, len(expectedConfig)) c.Check(output, gc.DeepEquals, expectedConfig) @@ -200,13 +200,13 @@ func (s *bridgeConfigSuite) runScriptWithoutActivation(c *gc.C, pythonBinary, co return stdout, result.Code } -func (s *bridgeConfigSuite) runScriptWithActivationAndDryRun(c *gc.C, pythonBinary, bridgePrefix string, netBondReconfigureDelay int, interfacesToBridge []string) (output []string, exitCode int) { +func (s *bridgeConfigSuite) runScriptWithActivationAndDryRun(c *gc.C, pythonBinary, bridgePrefix string, reconfigureDelay int, interfacesToBridge []string) (output []string, exitCode int) { if bridgePrefix != "" { bridgePrefix = fmt.Sprintf("--bridge-prefix=%q", bridgePrefix) } - script := fmt.Sprintf("%q %q --activate --dry-run %s %s --net-bond-reconfigure-delay=%v --interfaces-to-bridge=%q", - pythonBinary, s.testPythonScript, bridgePrefix, s.testConfigPath, netBondReconfigureDelay, strings.Join(interfacesToBridge, " ")) + script := fmt.Sprintf("%q %q --activate --dry-run %s %s --reconfigure-delay=%v --interfaces-to-bridge=%q", + pythonBinary, s.testPythonScript, bridgePrefix, s.testConfigPath, reconfigureDelay, strings.Join(interfacesToBridge, " ")) c.Log(script) result, err := exec.RunCommands(exec.RunParams{Commands: script}) c.Assert(err, jc.ErrorIsNil, gc.Commentf("script failed unexpectedly")) @@ -228,7 +228,7 @@ func (s *bridgeConfigSuite) runScriptWithActivationAndDryRun(c *gc.C, pythonBina return output, result.Code } -func (s *bridgeConfigSuite) dryRunExpectedOutputHelper(isBonded, isAlreadyBridged bool, bridgePrefix string, netBondReconfigureDelay int, interfacesToBridge []string) []string { +func (s *bridgeConfigSuite) dryRunExpectedOutputHelper(isAlreadyBridged bool, bridgePrefix string, reconfigureDelay int, interfacesToBridge []string) []string { output := make([]string, 0) bridgedNames := make([]string, len(interfacesToBridge)) for i, s := range interfacesToBridge { @@ -244,10 +244,8 @@ func (s *bridgeConfigSuite) dryRunExpectedOutputHelper(isBonded, isAlreadyBridge output = append(output, "brctl show") output = append(output, fmt.Sprintf("ifdown --exclude=lo --interfaces=%[1]s %s", s.testConfigPath, strings.Join(interfacesToBridge, " "))) output = append(output, "**** Activating new configuration") - if isBonded && netBondReconfigureDelay > 0 { - output = append(output, "working around https://bugs.launchpad.net/ubuntu/+source/ifenslave/+bug/1269921") - output = append(output, "working around https://bugs.launchpad.net/juju-core/+bug/1594855") - output = append(output, fmt.Sprintf("sleep %v", netBondReconfigureDelay)) + if reconfigureDelay > 0 { + output = append(output, fmt.Sprintf("sleep %v", reconfigureDelay)) } output = append(output, fmt.Sprintf("cat %s", s.testConfigPath)) output = append(output, fmt.Sprintf("ifup --exclude=lo --interfaces=%[1]s -a", s.testConfigPath)) @@ -261,31 +259,31 @@ func (s *bridgeConfigSuite) dryRunExpectedOutputHelper(isBonded, isAlreadyBridge func (s *bridgeConfigSuite) TestBridgeScriptWithoutBondedInterfaceSingle(c *gc.C) { bridgePrefix := "TestBridgeScriptWithoutBondedInterfaceSingle" interfacesToBridge := []string{"eth0"} - s.assertScriptWithActivationAndDryRun(c, false, false, networkDHCPInitial, bridgePrefix, 0, interfacesToBridge) + s.assertScriptWithActivationAndDryRun(c, false, networkDHCPInitial, bridgePrefix, 0, interfacesToBridge) } func (s *bridgeConfigSuite) TestBridgeScriptWithoutBondedInterfaceMultiple(c *gc.C) { bridgePrefix := "TestBridgeScriptWithoutBondedInterfaceMultiple" interfacesToBridge := []string{"eth0", "eth1"} - s.assertScriptWithActivationAndDryRun(c, false, false, networkDHCPInitial, bridgePrefix, 1, interfacesToBridge) + s.assertScriptWithActivationAndDryRun(c, false, networkDHCPInitial, bridgePrefix, 1, interfacesToBridge) } func (s *bridgeConfigSuite) TestBridgeScriptWithBondedInterfaceSingle(c *gc.C) { bridgePrefix := "TestBridgeScriptWithBondedInterfaceSingle" interfacesToBridge := []string{"bond0"} - s.assertScriptWithActivationAndDryRun(c, true, false, networkDHCPWithBondInitial, bridgePrefix, 2, interfacesToBridge) + s.assertScriptWithActivationAndDryRun(c, false, networkDHCPWithBondInitial, bridgePrefix, 2, interfacesToBridge) } func (s *bridgeConfigSuite) TestBridgeScriptWithBondedInterfaceMultiple(c *gc.C) { bridgePrefix := "TestBridgeScriptWithBondedInterfaceMultiple" interfacesToBridge := []string{"bond0", "bond1"} - s.assertScriptWithActivationAndDryRun(c, true, false, networkDHCPWithBondInitial, bridgePrefix, 4, interfacesToBridge) + s.assertScriptWithActivationAndDryRun(c, false, networkDHCPWithBondInitial, bridgePrefix, 4, interfacesToBridge) } func (s *bridgeConfigSuite) TestBridgeScriptWithBondedInterfaceAlreadyBridged(c *gc.C) { bridgePrefix := "TestBridgeScriptWithBondedInterfaceAlreadyBridged" interfacesToBridge := []string{"br-eth1"} - s.assertScriptWithActivationAndDryRun(c, false, true, networkPartiallyBridgedInitial, bridgePrefix, 8, interfacesToBridge) + s.assertScriptWithActivationAndDryRun(c, true, networkPartiallyBridgedInitial, bridgePrefix, 8, interfacesToBridge) } // The rest of the file contains various forms of network config for diff --git a/network/network.go b/network/network.go index c968d2eb926..bb71ef0db0d 100644 --- a/network/network.go +++ b/network/network.go @@ -329,10 +329,6 @@ type DeviceToBridge struct { // BridgeName is the name of the bridge that we want created. BridgeName string - - // NetBondReconfigureDelay is the duration to sleep (in seconds) before - // bringing the bridged interface up. - NetBondReconfigureDelay int } // LXCNetDefaultConfig is the location of the default network config diff --git a/state/linklayerdevices_test.go b/state/linklayerdevices_test.go index efb509fbdaa..7da0b137d2d 100644 --- a/state/linklayerdevices_test.go +++ b/state/linklayerdevices_test.go @@ -1794,9 +1794,10 @@ func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerNoneMissi Spaces: &[]string{"default"}, }) c.Assert(err, jc.ErrorIsNil) - missing, err := s.machine.FindMissingBridgesForContainer(s.containerMachine) + missing, reconfigureDelay, err := s.machine.FindMissingBridgesForContainer(s.containerMachine, 0) c.Assert(err, jc.ErrorIsNil) c.Check(missing, jc.DeepEquals, []network.DeviceToBridge{}) + c.Check(reconfigureDelay, gc.Equals, 0) } func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerDefaultUnbridged(c *gc.C) { @@ -1807,12 +1808,13 @@ func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerDefaultUn Spaces: &[]string{"default"}, }) c.Assert(err, jc.ErrorIsNil) - missing, err := s.machine.FindMissingBridgesForContainer(s.containerMachine) + missing, reconfigureDelay, err := s.machine.FindMissingBridgesForContainer(s.containerMachine, 0) c.Assert(err, jc.ErrorIsNil) c.Check(missing, gc.DeepEquals, []network.DeviceToBridge{{ DeviceName: "eth0", BridgeName: "br-eth0", }}) + c.Check(reconfigureDelay, gc.Equals, 0) } func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerNoHostDevices(c *gc.C) { @@ -1824,8 +1826,9 @@ func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerNoHostDev Spaces: &[]string{"dmz", "third"}, }) c.Assert(err, jc.ErrorIsNil) - _, err = s.machine.FindMissingBridgesForContainer(s.containerMachine) + _, reconfigureDelay, err := s.machine.FindMissingBridgesForContainer(s.containerMachine, 0) c.Assert(err.Error(), gc.Equals, `host machine "0" has no available device in space(s) "dmz", "third"`) + c.Check(reconfigureDelay, gc.Equals, 0) } func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerTwoSpacesOneMissing(c *gc.C) { @@ -1836,7 +1839,7 @@ func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerTwoSpaces err := s.containerMachine.SetConstraints(constraints.Value{ Spaces: &[]string{"default", "dmz"}, }) - _, err = s.machine.FindMissingBridgesForContainer(s.containerMachine) + _, _, err = s.machine.FindMissingBridgesForContainer(s.containerMachine, 0) // both default and dmz are needed, but default is missing c.Assert(err.Error(), gc.Equals, `host machine "0" has no available device in space(s) "default"`) } @@ -1853,7 +1856,7 @@ func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerNoSpaces( s.addContainerMachine(c) // No defined spaces for the container, no *known* spaces for the host // machine. Triggers the fallback code to have us bridge all devices. - missing, err := s.machine.FindMissingBridgesForContainer(s.containerMachine) + missing, reconfigureDelay, err := s.machine.FindMissingBridgesForContainer(s.containerMachine, 0) c.Assert(err, jc.ErrorIsNil) c.Check(missing, gc.DeepEquals, []network.DeviceToBridge{{ DeviceName: "ens3", @@ -1862,6 +1865,7 @@ func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerNoSpaces( DeviceName: "ens4", BridgeName: "br-ens4", }}) + c.Check(reconfigureDelay, gc.Equals, 0) } func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerUnknownWithConstraint(c *gc.C) { @@ -1877,7 +1881,7 @@ func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerUnknownWi Spaces: &[]string{"default"}, }) c.Assert(err, jc.ErrorIsNil) - _, err = s.machine.FindMissingBridgesForContainer(s.containerMachine) + _, _, err = s.machine.FindMissingBridgesForContainer(s.containerMachine, 0) c.Assert(err.Error(), gc.Equals, `host machine "0" has no available device in space(s) "default"`) } @@ -1893,12 +1897,13 @@ func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerUnknownAn s.createAllDefaultDevices(c, s.machine) s.addContainerMachine(c) // We don't need a container constraint, as the host machine is in a single space. - missing, err := s.machine.FindMissingBridgesForContainer(s.containerMachine) + missing, reconfigureDelay, err := s.machine.FindMissingBridgesForContainer(s.containerMachine, 0) c.Assert(err, jc.ErrorIsNil) c.Check(missing, gc.DeepEquals, []network.DeviceToBridge{{ DeviceName: "ens3", BridgeName: "br-ens3", }}) + c.Check(reconfigureDelay, gc.Equals, 0) } func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerOneOfTwoBridged(c *gc.C) { @@ -1921,13 +1926,14 @@ func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerOneOfTwoB Spaces: &[]string{"default"}, }) c.Assert(err, jc.ErrorIsNil) - missing, err := s.machine.FindMissingBridgesForContainer(s.containerMachine) + missing, reconfigureDelay, err := s.machine.FindMissingBridgesForContainer(s.containerMachine, 0) c.Assert(err, jc.ErrorIsNil) // Only the first device (by sort order) should be selected c.Check(missing, gc.DeepEquals, []network.DeviceToBridge{{ DeviceName: "ens2.1", BridgeName: "br-ens2.1", }}) + c.Check(reconfigureDelay, gc.Equals, 0) } func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerTwoHostDevicesOneBridged(c *gc.C) { @@ -1941,9 +1947,10 @@ func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerTwoHostDe Spaces: &[]string{"default"}, }) c.Assert(err, jc.ErrorIsNil) - missing, err := s.machine.FindMissingBridgesForContainer(s.containerMachine) + missing, reconfigureDelay, err := s.machine.FindMissingBridgesForContainer(s.containerMachine, 0) c.Assert(err, jc.ErrorIsNil) c.Check(missing, jc.DeepEquals, []network.DeviceToBridge{}) + c.Check(reconfigureDelay, gc.Equals, 0) } func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerNoConstraintsDefaultNotSpecial(c *gc.C) { @@ -1956,9 +1963,10 @@ func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerNoConstra // DMZ s.createNICWithIP(c, s.machine, "eth1", "10.10.0.20/24") s.addContainerMachine(c) - missing, err := s.machine.FindMissingBridgesForContainer(s.containerMachine) + missing, reconfigureDelay, err := s.machine.FindMissingBridgesForContainer(s.containerMachine, 0) c.Assert(err, gc.ErrorMatches, "no obvious space for container.*") c.Assert(missing, gc.IsNil) + c.Check(reconfigureDelay, gc.Equals, 0) } func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerTwoSpacesOneBridged(c *gc.C) { @@ -1971,13 +1979,14 @@ func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerTwoSpaces err := s.containerMachine.SetConstraints(constraints.Value{ Spaces: &[]string{"default", "dmz"}, }) - missing, err := s.machine.FindMissingBridgesForContainer(s.containerMachine) + missing, reconfigureDelay, err := s.machine.FindMissingBridgesForContainer(s.containerMachine, 0) c.Assert(err, jc.ErrorIsNil) // both default and dmz are needed, but default needs to be bridged c.Check(missing, jc.DeepEquals, []network.DeviceToBridge{{ DeviceName: "eth0", BridgeName: "br-eth0", }}) + c.Check(reconfigureDelay, gc.Equals, 0) } func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerMultipleSpacesNoneBridged(c *gc.C) { @@ -1993,7 +2002,7 @@ func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerMultipleS err := s.containerMachine.SetConstraints(constraints.Value{ Spaces: &[]string{"default", "dmz", "abba"}, }) - missing, err := s.machine.FindMissingBridgesForContainer(s.containerMachine) + missing, reconfigureDelay, err := s.machine.FindMissingBridgesForContainer(s.containerMachine, 0) c.Assert(err, jc.ErrorIsNil) // both default and dmz are needed, but default needs to be bridged c.Check(missing, jc.DeepEquals, []network.DeviceToBridge{{ @@ -2006,6 +2015,7 @@ func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerMultipleS DeviceName: "eth1", BridgeName: "br-eth1", }}) + c.Check(reconfigureDelay, gc.Equals, 0) } func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerBondedNICs(c *gc.C) { @@ -2062,13 +2072,14 @@ func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerBondedNIC err = s.containerMachine.SetConstraints(constraints.Value{ Spaces: &[]string{"default"}, }) - missing, err := s.machine.FindMissingBridgesForContainer(s.containerMachine) + missing, reconfigureDelay, err := s.machine.FindMissingBridgesForContainer(s.containerMachine, 0) c.Assert(err, jc.ErrorIsNil) // both default and dmz are needed, but default needs to be bridged c.Check(missing, jc.DeepEquals, []network.DeviceToBridge{{ DeviceName: "zbond0", BridgeName: "br-zbond0", }}) + c.Check(reconfigureDelay, gc.Equals, 0) } func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerVLAN(c *gc.C) { @@ -2103,7 +2114,7 @@ func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerVLAN(c *g Spaces: &[]string{"default", "dmz"}, }) c.Assert(err, jc.ErrorIsNil) - missing, err := s.machine.FindMissingBridgesForContainer(s.containerMachine) + missing, reconfigureDelay, err := s.machine.FindMissingBridgesForContainer(s.containerMachine, 0) c.Assert(err, jc.ErrorIsNil) c.Check(missing, jc.DeepEquals, []network.DeviceToBridge{{ DeviceName: "eth0", @@ -2112,6 +2123,7 @@ func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerVLAN(c *g DeviceName: "eth0.100", BridgeName: "br-eth0.100", }}) + c.Check(reconfigureDelay, gc.Equals, 0) } func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerVLANOnBond(c *gc.C) { @@ -2168,7 +2180,7 @@ func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerVLANOnBon Spaces: &[]string{"default", "dmz"}, }) c.Assert(err, jc.ErrorIsNil) - missing, err := s.machine.FindMissingBridgesForContainer(s.containerMachine) + missing, reconfigureDelay, err := s.machine.FindMissingBridgesForContainer(s.containerMachine, 0) c.Assert(err, jc.ErrorIsNil) c.Check(missing, jc.DeepEquals, []network.DeviceToBridge{{ DeviceName: "bond0", @@ -2177,4 +2189,5 @@ func (s *linkLayerDevicesStateSuite) TestFindMissingBridgesForContainerVLANOnBon DeviceName: "bond0.100", BridgeName: "br-bond0.100", }}) + c.Check(reconfigureDelay, gc.Equals, 0) } diff --git a/state/machine_linklayerdevices.go b/state/machine_linklayerdevices.go index 65c394414cc..edf143cf4ae 100644 --- a/state/machine_linklayerdevices.go +++ b/state/machine_linklayerdevices.go @@ -1154,10 +1154,11 @@ func possibleBridgeTarget(dev *LinkLayerDevice) (bool, error) { // This will return an Error if the container wants a space that the host // machine cannot provide. // TODO(jam): 2016-12-28 Move this off of machine -func (m *Machine) FindMissingBridgesForContainer(containerMachine *Machine) ([]network.DeviceToBridge, error) { +func (m *Machine) FindMissingBridgesForContainer(containerMachine *Machine, netBondReconfigureDelay int) ([]network.DeviceToBridge, int, error) { + reconfigureDelay := 0 containerSpaces, devicesPerSpace, err := m.findSpacesAndDevicesForContainer(containerMachine) if err != nil { - return nil, errors.Trace(err) + return nil, 0, errors.Trace(err) } // TODO(jam): 2016-12-28 The formatting for the spaces can look a bit // screwy, because it does [name] instead of quoting it. But if we use @@ -1178,20 +1179,22 @@ func (m *Machine) FindMissingBridgesForContainer(containerMachine *Machine) ([]n notFound := containerSpaces.Difference(spacesFound) if notFound.IsEmpty() { // Nothing to do, just return success - return nil, nil + return nil, 0, nil } hostDeviceNamesToBridge := make([]string, 0) for _, spaceName := range notFound.Values() { hostDeviceNames := make([]string, 0) + hostDeviceByName := make(map[string]*LinkLayerDevice, 0) for _, hostDevice := range devicesPerSpace[spaceName] { possible, err := possibleBridgeTarget(hostDevice) if err != nil { - return nil, err + return nil, 0, err } if !possible { continue } hostDeviceNames = append(hostDeviceNames, hostDevice.Name()) + hostDeviceByName[hostDevice.Name()] = hostDevice spacesFound.Add(spaceName) } if len(hostDeviceNames) > 0 { @@ -1201,6 +1204,11 @@ func (m *Machine) FindMissingBridgesForContainer(containerMachine *Machine) ([]n // don't know what the exact spaces are going to be. for _, deviceName := range hostDeviceNames { hostDeviceNamesToBridge = append(hostDeviceNamesToBridge, deviceName) + if hostDeviceByName[deviceName].Type() == BondDevice { + if reconfigureDelay < netBondReconfigureDelay { + reconfigureDelay = netBondReconfigureDelay + } + } } } else { // This should already be sorted from @@ -1208,6 +1216,11 @@ func (m *Machine) FindMissingBridgesForContainer(containerMachine *Machine) ([]n // pick the host device hostDeviceNames = network.NaturallySortDeviceNames(hostDeviceNames...) hostDeviceNamesToBridge = append(hostDeviceNamesToBridge, hostDeviceNames[0]) + // if hostDeviceByName[deviceName].Type() == BondDevice { + // if reconfigureDelay < netBondReconfigureDelay { + // reconfigureDelay = netBondReconfigureDelay + // } + // } } } } @@ -1222,7 +1235,7 @@ func (m *Machine) FindMissingBridgesForContainer(containerMachine *Machine) ([]n logger.Warningf("container %q wants spaces %s, but host machine %q has %s missing %s", containerMachine.Id(), quoteSpaces(containerSpaces), m.Id(), quoteSpaces(hostSpaces), quoteSpaces(notFound)) - return nil, errors.Errorf("host machine %q has no available device in space(s) %s", + return nil, 0, errors.Errorf("host machine %q has no available device in space(s) %s", m.Id(), quoteSpaces(notFound)) } hostToBridge := make([]network.DeviceToBridge, 0, len(hostDeviceNamesToBridge)) @@ -1233,7 +1246,7 @@ func (m *Machine) FindMissingBridgesForContainer(containerMachine *Machine) ([]n BridgeName: fmt.Sprintf("br-%s", hostName), }) } - return hostToBridge, nil + return hostToBridge, reconfigureDelay, nil } func quoteSpaces(vals set.Strings) string { diff --git a/worker/provisioner/broker.go b/worker/provisioner/broker.go index 2f81dab9767..808b6fd52e3 100644 --- a/worker/provisioner/broker.go +++ b/worker/provisioner/broker.go @@ -24,7 +24,7 @@ type APICalls interface { GetContainerInterfaceInfo(names.MachineTag) ([]network.InterfaceInfo, error) ReleaseContainerAddresses(names.MachineTag) error SetHostMachineNetworkConfig(string, []params.NetworkConfig) error - HostChangesForContainer(containerTag names.MachineTag) ([]network.DeviceToBridge, error) + HostChangesForContainer(containerTag names.MachineTag) ([]network.DeviceToBridge, int, error) } type hostArchToolsFinder struct { @@ -44,7 +44,7 @@ var resolvConf = "/etc/resolv.conf" var getObservedNetworkConfig = common.GetObservedNetworkConfig func prepareHost(bridger network.Bridger, hostMachineID string, containerTag names.MachineTag, api APICalls, log loggo.Logger) error { - devicesToBridge, err := api.HostChangesForContainer(containerTag) + devicesToBridge, reconfigureDelay, err := api.HostChangesForContainer(containerTag) if err != nil { return errors.Annotate(err, "unable to setup network") @@ -55,9 +55,9 @@ func prepareHost(bridger network.Bridger, hostMachineID string, containerTag nam return nil } - log.Tracef("Bridging %+v devices on host %q", devicesToBridge, hostMachineID) + log.Tracef("Bridging %+v devices on host %q with delay=%v", devicesToBridge, hostMachineID, reconfigureDelay) - err = bridger.Bridge(devicesToBridge) + err = bridger.Bridge(devicesToBridge, reconfigureDelay) if err != nil { return errors.Annotate(err, "failed to bridge devices") diff --git a/worker/provisioner/broker_test.go b/worker/provisioner/broker_test.go index a12e125e178..ab61c347d6c 100644 --- a/worker/provisioner/broker_test.go +++ b/worker/provisioner/broker_test.go @@ -127,12 +127,12 @@ func (f *fakeAPI) SetHostMachineNetworkConfig(hostMachineID string, netConfig [] return nil } -func (f *fakeAPI) HostChangesForContainer(machineID names.MachineTag) ([]network.DeviceToBridge, error) { +func (f *fakeAPI) HostChangesForContainer(machineID names.MachineTag) ([]network.DeviceToBridge, int, error) { f.MethodCall(f, "HostChangesForContainer", machineID) if err := f.NextErr(); err != nil { - return nil, err + return nil, 0, err } - return []network.DeviceToBridge{f.fakeDeviceToBridge}, nil + return []network.DeviceToBridge{f.fakeDeviceToBridge}, 0, nil } type patcher interface { @@ -233,7 +233,7 @@ func newFakeBridgerNeverErrors() *fakeBridger { } } -func (f *fakeBridger) Bridge(devices []network.DeviceToBridge) error { +func (f *fakeBridger) Bridge(devices []network.DeviceToBridge, reconfigureDelay int) error { if f.returnError { return errors.New(f.errorMessage) } diff --git a/worker/provisioner/kvm-broker_test.go b/worker/provisioner/kvm-broker_test.go index 2ae29db2651..66155f8442a 100644 --- a/worker/provisioner/kvm-broker_test.go +++ b/worker/provisioner/kvm-broker_test.go @@ -446,7 +446,7 @@ func (s *kvmProvisionerSuite) TestContainerStartedAndStopped(c *gc.C) { // TODO(jam): 2016-12-22 recent changes to check for networking changes // when starting a container cause this test to start failing, because // the Dummy provider does not support Networking configuration. - _, err := s.provisioner.HostChangesForContainer(container.MachineTag()) + _, _, err := s.provisioner.HostChangesForContainer(container.MachineTag()) c.Assert(err, gc.ErrorMatches, "dummy provider network config not supported.*") c.Skip("dummy provider doesn't support network config. https://pad.lv/1651974") instId := s.expectStarted(c, container)