Clean up peas on restart

[#155638517] Signed-off-by: Tom Godkin <[email protected]> Signed-off-by: Julia Nedialkova <[email protected]> Signed-off-by: Georgi Sabev <[email protected]>
hpcloud · Mar 9, 2018 · 20014a8 · 20014a8
1 parent 3555aee
commit 20014a8
Show file tree

Hide file tree

Showing 15 changed files with 818 additions and 28 deletions.
diff --git a/gardener/gardener.go b/gardener/gardener.go
@@ -24,6 +24,7 @@ import (
 //go:generate counterfeiter . Restorer
 //go:generate counterfeiter . Starter
 //go:generate counterfeiter . BulkStarter
+//go:generate counterfeiter . PeaCleaner
 
 const ContainerIPKey = "garden.network.container-ip"
 const BridgeIPKey = "garden.network.host-ip"
@@ -101,6 +102,11 @@ type Restorer interface {
 	Restore(logger lager.Logger, handles []string) []string
 }
 
+type PeaCleaner interface {
+	CleanAll(logger lager.Logger) error
+	Clean(logger lager.Logger, handle string) error
+}
+
 type UidGeneratorFunc func() string
 
 func (fn UidGeneratorFunc) Generate() string {
@@ -142,6 +148,8 @@ type Gardener struct {
 
 	Restorer Restorer
 
+	PeaCleaner PeaCleaner
+
 	AllowPrivilgedContainers bool
 }
 
@@ -457,6 +465,10 @@ func (g *Gardener) Start() error {
 		return fmt.Errorf("bulk starter: %s", err)
 	}
 
+	if err := g.PeaCleaner.CleanAll(log); err != nil {
+		return fmt.Errorf("clean peas: %s", err)
+	}
+
 	handles, err := g.Containerizer.Handles()
 	if err != nil {
 		return err

diff --git a/gardener/gardener_test.go b/gardener/gardener_test.go
@@ -25,6 +25,7 @@ var _ = Describe("Gardener", func() {
 		containerizer   *fakes.FakeContainerizer
 		uidGenerator    *fakes.FakeUidGenerator
 		fakeBulkStarter *fakes.FakeBulkStarter
+		fakePeaCleaner  *fakes.FakePeaCleaner
 		sysinfoProvider *fakes.FakeSysInfoProvider
 		propertyManager *fakes.FakePropertyManager
 		restorer        *fakes.FakeRestorer
@@ -39,6 +40,7 @@ var _ = Describe("Gardener", func() {
 		containerizer = new(fakes.FakeContainerizer)
 		uidGenerator = new(fakes.FakeUidGenerator)
 		fakeBulkStarter = new(fakes.FakeBulkStarter)
+		fakePeaCleaner = new(fakes.FakePeaCleaner)
 		networker = new(fakes.FakeNetworker)
 		volumizer = new(fakes.FakeVolumizer)
 		sysinfoProvider = new(fakes.FakeSysInfoProvider)
@@ -59,6 +61,7 @@ var _ = Describe("Gardener", func() {
 			Logger:                   logger,
 			PropertyManager:          propertyManager,
 			Restorer:                 restorer,
+			PeaCleaner:               fakePeaCleaner,
 			MaxContainers:            0,
 			AllowPrivilgedContainers: false,
 		}
@@ -700,6 +703,21 @@ var _ = Describe("Gardener", func() {
 			containerizer.HandlesReturns([]string{}, errors.New("banana"))
 			Expect(gdnr.Start()).To(MatchError("banana"))
 		})
+
+		It("should cleanup peas", func() {
+			Expect(gdnr.Start()).To(Succeed())
+			Expect(fakePeaCleaner.CleanAllCallCount()).To(Equal(1))
+		})
+
+		Context("when the bulk starter fails", func() {
+			BeforeEach(func() {
+				fakePeaCleaner.CleanAllReturns(errors.New("bam"))
+			})
+
+			It("returns the error", func() {
+				Expect(gdnr.Start()).To(MatchError(ContainSubstring("bam")))
+			})
+		})
 	})
 
 	Describe("listing containers", func() {

diff --git a/gardener/gardenerfakes/fake_pea_cleaner.go b/gardener/gardenerfakes/fake_pea_cleaner.go
diff --git a/gqt/restart_test.go b/gqt/restart_test.go
@@ -35,6 +35,7 @@ var _ = Describe("Surviving Restarts", func() {
 			restartConfig       runner.GdnRunnerConfig
 			gracefulShutdown    bool
 			processImage        garden.ImageRef
+			processID           string
 		)
 
 		BeforeEach(func() {
@@ -58,6 +59,7 @@ var _ = Describe("Surviving Restarts", func() {
 
 			gracefulShutdown = true
 			processImage = garden.ImageRef{}
+			processID = ""
 		})
 
 		JustBeforeEach(func() {
@@ -86,6 +88,7 @@ var _ = Describe("Surviving Restarts", func() {
 			out := gbytes.NewBuffer()
 			existingProc, err = container.Run(
 				garden.ProcessSpec{
+					ID:    processID,
 					Path:  "/bin/sh",
 					Args:  []string{"-c", fmt.Sprintf("while true; do echo %s; sleep 1; done;", container.Handle())},
 					Image: processImage,
@@ -146,6 +149,17 @@ var _ = Describe("Surviving Restarts", func() {
 				Consistently(check, time.Second*2, time.Millisecond*200).Should(Equal("0\n"), "expected user process to stay dead")
 			})
 
+			Context("when running a pea", func() {
+				BeforeEach(func() {
+					processImage = garden.ImageRef{URI: defaultTestRootFS}
+					processID = "unique-potato"
+				})
+
+				It("destroys the pea container", func() {
+					Expect(filepath.Join("/run/runc", processID)).NotTo(BeADirectory())
+				})
+			})
+
 			Context("when the garden server does not shut down gracefully", func() {
 				BeforeEach(func() {
 					gracefulShutdown = false

diff --git a/guardiancmd/command.go b/guardiancmd/command.go
@@ -451,17 +451,19 @@ func (cmd *ServerCommand) Run(signals <-chan os.Signal, ready chan<- struct{}) e
 	}
 
 	var bulkStarter gardener.BulkStarter = gardener.NewBulkStarter(starters)
+	peaCleaner := cmd.wirePeaCleaner(factory, volumizer)
 
 	backend := &gardener.Gardener{
 		UidGenerator:    wireUIDGenerator(),
 		BulkStarter:     bulkStarter,
 		SysInfoProvider: sysinfo.NewResourcesProvider(cmd.Containers.Dir),
 		Networker:       networker,
 		Volumizer:       volumizer,
-		Containerizer:   cmd.wireContainerizer(logger, factory, propManager, volumizer),
+		Containerizer:   cmd.wireContainerizer(logger, factory, propManager, volumizer, peaCleaner),
 		PropertyManager: propManager,
 		MaxContainers:   cmd.Limits.MaxContainers,
 		Restorer:        restorer,
+		PeaCleaner:      peaCleaner,
 
 		// We want to be able to disable privileged containers independently of
 		// whether or not gdn is running as root.
@@ -541,6 +543,15 @@ func (cmd *ServerCommand) Run(signals <-chan os.Signal, ready chan<- struct{}) e
 	return nil
 }
 
+func (cmd *ServerCommand) wirePeaCleaner(factory GardenFactory, volumizer gardener.Volumizer) gardener.PeaCleaner {
+	cmdRunner := factory.CommandRunner()
+	runcLogRunner := runrunc.NewLogRunner(cmdRunner, runrunc.LogDir(os.TempDir()).GenerateLogFile)
+	runcBinary := goci.RuncBinary{Path: cmd.Runtime.Plugin}
+
+	runcDeleter := runrunc.NewDeleter(runcLogRunner, runcBinary)
+	return peas.NewPeaCleaner(runcDeleter, volumizer, cmd.Containers.Dir)
+}
+
 func (cmd *ServerCommand) calculateDefaultMappingLengths(containerRootUID, containerRootGID int) {
 	if cmd.Containers.UIDMapLength == 0 {
 		cmd.Containers.UIDMapLength = uint32(containerRootUID) - cmd.Containers.UIDMapStart
@@ -740,7 +751,7 @@ func (cmd *ServerCommand) wireImagePlugin(commandRunner commandrunner.CommandRun
 }
 
 func (cmd *ServerCommand) wireContainerizer(log lager.Logger, factory GardenFactory,
-	properties gardener.PropertyManager, volumizer peas.Volumizer) *rundmc.Containerizer {
+	properties gardener.PropertyManager, volumizer peas.Volumizer, peaCleaner gardener.PeaCleaner) *rundmc.Containerizer {
 
 	initMount, initPath := initBindMountAndPath(cmd.Bin.Init.Path())
 
@@ -828,12 +839,12 @@ func (cmd *ServerCommand) wireContainerizer(log lager.Logger, factory GardenFact
 	processBuilder := runrunc.NewProcessBuilder(wireEnvFunc(), nonRootMaxCaps)
 
 	cmdRunner := factory.CommandRunner()
-	runcRunner := runrunc.NewLogRunner(cmdRunner, runrunc.LogDir(os.TempDir()).GenerateLogFile)
+	runcLogRunner := runrunc.NewLogRunner(cmdRunner, runrunc.LogDir(os.TempDir()).GenerateLogFile)
 	runcBinary := goci.RuncBinary{Path: cmd.Runtime.Plugin}
 
 	runcrunner := runrunc.New(
 		cmdRunner,
-		runcRunner,
+		runcLogRunner,
 		runcBinary,
 		cmd.Bin.Dadoo.Path(),
 		cmd.Runtime.Plugin,
@@ -860,7 +871,7 @@ func (cmd *ServerCommand) wireContainerizer(log lager.Logger, factory GardenFact
 	pidFileReader := wirePidfileReader()
 	privilegeChecker := &privchecker.PrivilegeChecker{BundleLoader: bndlLoader}
 
-	runcDeleter := runrunc.NewDeleter(runcRunner, runcBinary)
+	runcDeleter := runrunc.NewDeleter(runcLogRunner, runcBinary)
 
 	peaCreator := &peas.PeaCreator{
 		Volumizer:              volumizer,
@@ -872,6 +883,7 @@ func (cmd *ServerCommand) wireContainerizer(log lager.Logger, factory GardenFact
 		BundleSaver:            bundleSaver,
 		ExecRunner:             factory.WireExecRunner("run"),
 		RuncDeleter:            runcDeleter,
+		PeaCleaner:             peaCleaner,
 	}
 
 	nstar := rundmc.NewNstarRunner(cmd.Bin.NSTar.Path(), cmd.Bin.Tar.Path(), cmdRunner)

diff --git a/rundmc/depot/dirdepot.go b/rundmc/depot/dirdepot.go
@@ -105,6 +105,10 @@ func (d *DirectoryDepot) Destroy(log lager.Logger, handle string) error {
 	return os.RemoveAll(d.toDir(handle))
 }
 
+func (d *DirectoryDepot) GetDir() string {
+	return d.dir
+}
+
 //go:generate counterfeiter . BundleLoader
 type BundleLoader interface {
 	Load(bundleDir string) (goci.Bndl, error)

diff --git a/rundmc/depot/dirdepot_test.go b/rundmc/depot/dirdepot_test.go
@@ -202,4 +202,11 @@ var _ = Describe("Depot", func() {
 			})
 		})
 	})
+
+	Describe("GetDir", func() {
+		It("returns the depot dir", func() {
+			dirDepot := depot.New("/path/to/depot", nil, nil, nil)
+			Expect(dirDepot.GetDir()).To(Equal("/path/to/depot"))
+		})
+	})
 })