@@ -2,18 +2,21 @@ package shim
22
33import (
44 "context"
5+ "errors"
56 "fmt"
67 "io"
78 "net"
89 "os"
910 "path/filepath"
11+ "strings"
1012 "time"
1113
1214 "github.com/containerd/containerd/v2/cmd/containerd-shim-runc-v2/process"
1315 runcC "github.com/containerd/go-runc"
1416 "github.com/containerd/log"
1517 "github.com/containerd/ttrpc"
1618 nodev1 "github.com/ctrox/zeropod/api/node/v1"
19+ "github.com/prometheus/procfs"
1720 "google.golang.org/protobuf/types/known/timestamppb"
1821)
1922
@@ -139,6 +142,8 @@ func (c *Container) evac(ctx context.Context) error {
139142 return fmt .Errorf ("failed checkpointing: %w" , err )
140143 case <- lazyStarted :
141144 log .G (ctx ).Info ("successful started lazy checkpointing" )
145+ c .prepareMigrateData (ctx )
146+
142147 log .G (ctx ).Infof ("making evac request with image ID: %s" , c .ID ())
143148 evacReq .MigrationInfo .PausedAt = timestamppb .New (pausedAt )
144149 if _ , err := nodeClient .Evac (ctx , evacReq ); err != nil {
@@ -204,9 +209,89 @@ func (c *Container) evacScaledDown(ctx context.Context) error {
204209 if _ , err := nodeClient .PrepareEvac (ctx , evacReq ); err != nil {
205210 return fmt .Errorf ("requesting evac: %w" , err )
206211 }
212+ c .prepareMigrateData (ctx )
207213
208214 if _ , err := nodeClient .Evac (ctx , evacReq ); err != nil {
209215 return fmt .Errorf ("requesting evac: %w" , err )
210216 }
211217 return nil
212218}
219+
220+ func (c * Container ) prepareMigrateData (ctx context.Context ) {
221+ if c .cfg .DisableMigrateData {
222+ return
223+ }
224+ // for now we allow this to fail and continue with the migration since
225+ // it could be unreliable and is not strictly required to migrate
226+ // (although this depends on the application).
227+ if err := moveUpperDirToImage (c .ID ()); err != nil {
228+ log .G (ctx ).Errorf ("adding container data to image: %s" , err )
229+ }
230+ }
231+
232+ // moveUpperDirToImage finds the overlayfs upper directory of the container and
233+ // moves it to the snapshot upper dir to be transferred by the evac. Only call
234+ // this once the container is either stopped or frozen.
235+ func moveUpperDirToImage (containerID string ) error {
236+ upper , err := findUpperDir (containerID )
237+ if err != nil {
238+ return fmt .Errorf ("finding upper storage dir: %w" , err )
239+ }
240+ to := nodev1 .UpperPath (containerID )
241+ if err := os .MkdirAll (to , 0644 ); err != nil {
242+ return err
243+ }
244+
245+ return renameAllSubDirs (upper , to )
246+ }
247+
248+ // MoveImageToUpperDir does the same as moveUpperDirToImage but in reverse
249+ func MoveImageToUpperDir (containerID , imageDir string ) error {
250+ upper , err := findUpperDir (containerID )
251+ if err != nil {
252+ return fmt .Errorf ("finding upper storage dir: %w" , err )
253+ }
254+
255+ return renameAllSubDirs (filepath .Join (imageDir , nodev1 .UpperSuffix ), upper )
256+ }
257+
258+ func renameAllSubDirs (from , to string ) error {
259+ dirs , err := os .ReadDir (from )
260+ if err != nil {
261+ return err
262+ }
263+
264+ var errs []error
265+ for _ , dir := range dirs {
266+ errs = append (errs , os .Rename (filepath .Join (from , dir .Name ()), filepath .Join (to , dir .Name ())))
267+ }
268+
269+ return errors .Join (errs ... )
270+ }
271+
272+ func findUpperDir (containerID string ) (string , error ) {
273+ p , err := procfs .NewFS ("/proc" )
274+ if err != nil {
275+ return "" , fmt .Errorf ("new procfs: %w" , err )
276+ }
277+
278+ proc , err := p .Self ()
279+ if err != nil {
280+ return "" , fmt .Errorf ("getting process info: %w" , err )
281+ }
282+
283+ mountInfo , err := proc .MountInfo ()
284+ if err != nil {
285+ return "" , fmt .Errorf ("getting mount info: %w" , err )
286+ }
287+
288+ for _ , mount := range mountInfo {
289+ if strings .Contains (mount .MountPoint , containerID ) {
290+ if v , ok := mount .SuperOptions ["upperdir" ]; ok {
291+ return v , nil
292+ }
293+ }
294+ }
295+
296+ return "" , fmt .Errorf ("upper dir not found for container %s" , containerID )
297+ }
0 commit comments