Skip to content

Commit 035abe0

Browse files
elezarelezar
committed
Add create-soname-symlinks hook
This change adds a create-soname-symlinks hook that can be used to ensure that the soname symlinks for injected libraries exist in a container. This is done by calling ldconfig -n -N for the folders containing the injected libraries. This also ensures that libcuda.so is present in the ldcache when the update-ldcache hook is run. Signed-off-by: Evan Lezar <[email protected]>
1 parent ff49044 commit 035abe0

File tree

7 files changed

+237
-25
lines changed

7 files changed

+237
-25
lines changed

‎cmd/nvidia-cdi-hook/commands/commands.go

+2
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"github.com/urfave/cli/v2"
2121

2222
"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/chmod"
23+
soname "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/create-soname-symlinks"
2324
symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/create-symlinks"
2425
"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/cudacompat"
2526
ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/update-ldcache"
@@ -34,5 +35,6 @@ func New(logger logger.Interface) []*cli.Command {
3435
symlinks.NewCommand(logger),
3536
chmod.NewCommand(logger),
3637
cudacompat.NewCommand(logger),
38+
soname.NewCommand(logger),
3739
}
3840
}

‎cmd/nvidia-cdi-hook/create-soname-symlinks/soname-symlinks.go

+131
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
/**
2+
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
**/
16+
17+
package soname
18+
19+
import (
20+
"errors"
21+
"fmt"
22+
"path/filepath"
23+
24+
"github.com/urfave/cli/v2"
25+
26+
"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/utils"
27+
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
28+
"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
29+
)
30+
31+
type command struct {
32+
logger logger.Interface
33+
utils.SafeExecer
34+
}
35+
36+
type options struct {
37+
folders cli.StringSlice
38+
ldconfigPath string
39+
containerSpec string
40+
}
41+
42+
// NewCommand constructs an create-soname-symlinks command with the specified logger
43+
func NewCommand(logger logger.Interface) *cli.Command {
44+
c := command{
45+
logger: logger,
46+
SafeExecer: utils.NewSafeExecer(logger),
47+
}
48+
return c.build()
49+
}
50+
51+
// build the create-soname-symlinks command
52+
func (m command) build() *cli.Command {
53+
cfg := options{}
54+
55+
// Create the 'create-soname-symlinks' command
56+
c := cli.Command{
57+
Name: "create-soname-symlinks",
58+
Usage: "Create soname symlinks for the specified folders using ldconfig -n -N",
59+
Before: func(c *cli.Context) error {
60+
return m.validateFlags(c, &cfg)
61+
},
62+
Action: func(c *cli.Context) error {
63+
return m.run(c, &cfg)
64+
},
65+
}
66+
67+
c.Flags = []cli.Flag{
68+
&cli.StringSliceFlag{
69+
Name: "folder",
70+
Usage: "Specify a folder to search for shared libraries for which soname symlinks need to be created",
71+
Destination: &cfg.folders,
72+
},
73+
&cli.StringFlag{
74+
Name: "ldconfig-path",
75+
Usage: "Specify the path to the ldconfig program",
76+
Destination: &cfg.ldconfigPath,
77+
Value: "/sbin/ldconfig",
78+
},
79+
&cli.StringFlag{
80+
Name: "container-spec",
81+
Usage: "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
82+
Destination: &cfg.containerSpec,
83+
},
84+
}
85+
86+
return &c
87+
}
88+
89+
func (m command) validateFlags(c *cli.Context, cfg *options) error {
90+
if cfg.ldconfigPath == "" {
91+
return errors.New("ldconfig-path must be specified")
92+
}
93+
return nil
94+
}
95+
96+
func (m command) run(c *cli.Context, cfg *options) error {
97+
s, err := oci.LoadContainerState(cfg.containerSpec)
98+
if err != nil {
99+
return fmt.Errorf("failed to load container state: %v", err)
100+
}
101+
102+
containerRoot, err := s.GetContainerRoot()
103+
if err != nil {
104+
return fmt.Errorf("failed to determined container root: %v", err)
105+
}
106+
if containerRoot == "" {
107+
m.logger.Warningf("No container root detected")
108+
return nil
109+
}
110+
111+
dirs := cfg.folders.Value()
112+
if len(dirs) == 0 {
113+
return nil
114+
}
115+
116+
ldconfigPath := utils.ResolveHostLDConfigPath(cfg.ldconfigPath)
117+
args := []string{filepath.Base(ldconfigPath)}
118+
119+
args = append(args,
120+
// Specify the containerRoot to use.
121+
"-r", containerRoot,
122+
// Specify -n to only process the specified folders.
123+
"-n",
124+
// Explicitly disable updating the LDCache.
125+
"-N",
126+
)
127+
// Explicitly specific the directories to add.
128+
args = append(args, dirs...)
129+
130+
return m.Exec(ldconfigPath, args, nil)
131+
}

‎cmd/nvidia-ctk-installer/container/toolkit/toolkit_test.go

+7
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,13 @@ containerEdits:
8686
- --host-driver-version=999.88.77
8787
hookName: createContainer
8888
path: {{ .toolkitRoot }}/nvidia-cdi-hook
89+
- args:
90+
- nvidia-cdi-hook
91+
- create-soname-symlinks
92+
- --folder
93+
- /lib/x86_64-linux-gnu
94+
hookName: createContainer
95+
path: {{ .toolkitRoot }}/nvidia-cdi-hook
8996
- args:
9097
- nvidia-cdi-hook
9198
- update-ldcache

‎cmd/nvidia-ctk/cdi/generate/generate_test.go

+7
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,13 @@ containerEdits:
8585
- --host-driver-version=999.88.77
8686
hookName: createContainer
8787
path: /usr/bin/nvidia-cdi-hook
88+
- args:
89+
- nvidia-cdi-hook
90+
- create-soname-symlinks
91+
- --folder
92+
- /lib/x86_64-linux-gnu
93+
hookName: createContainer
94+
path: /usr/bin/nvidia-cdi-hook
8895
- args:
8996
- nvidia-cdi-hook
9097
- update-ldcache

‎internal/discover/ldconfig.go

+17-10
Original file line numberDiff line numberDiff line change
@@ -50,16 +50,16 @@ func (d ldconfig) Hooks() ([]Hook, error) {
5050
if err != nil {
5151
return nil, fmt.Errorf("failed to discover mounts for ldcache update: %v", err)
5252
}
53-
h := CreateLDCacheUpdateHook(
53+
hooks := CreateLDCacheUpdateHooks(
5454
d.nvidiaCDIHookPath,
5555
d.ldconfigPath,
5656
getLibraryPaths(mounts),
5757
)
58-
return []Hook{h}, nil
58+
return hooks, nil
5959
}
6060

61-
// CreateLDCacheUpdateHook locates the NVIDIA Container Toolkit CLI and creates a hook for updating the LD Cache
62-
func CreateLDCacheUpdateHook(executable string, ldconfig string, libraries []string) Hook {
61+
// CreateLDCacheUpdateHooks locates the NVIDIA Container Toolkit CLI and creates a hook for updating the LD Cache
62+
func CreateLDCacheUpdateHooks(executable string, ldconfig string, libraries []string) []Hook {
6363
var args []string
6464

6565
if ldconfig != "" {
@@ -70,13 +70,20 @@ func CreateLDCacheUpdateHook(executable string, ldconfig string, libraries []str
7070
args = append(args, "--folder", f)
7171
}
7272

73-
hook := CreateNvidiaCDIHook(
74-
executable,
75-
"update-ldcache",
76-
args...,
77-
)
73+
hooks := []Hook{
74+
CreateNvidiaCDIHook(
75+
executable,
76+
"create-soname-symlinks",
77+
args...,
78+
),
79+
CreateNvidiaCDIHook(
80+
executable,
81+
"update-ldcache",
82+
args...,
83+
),
84+
}
7885

79-
return hook
86+
return hooks
8087
}
8188

8289
// getLibraryPaths extracts the library dirs from the specified mounts

‎internal/discover/ldconfig_test.go

+51-15
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,22 @@ func TestLDCacheUpdateHook(t *testing.T) {
3838
mounts []Mount
3939
mountError error
4040
expectedError error
41-
expectedArgs []string
41+
expectedHooks []Hook
4242
}{
4343
{
44-
description: "empty mounts",
45-
expectedArgs: []string{"nvidia-cdi-hook", "update-ldcache"},
44+
description: "empty mounts",
45+
expectedHooks: []Hook{
46+
{
47+
Lifecycle: "createContainer",
48+
Path: testNvidiaCDIHookPath,
49+
Args: []string{"nvidia-cdi-hook", "create-soname-symlinks"},
50+
},
51+
{
52+
Lifecycle: "createContainer",
53+
Path: testNvidiaCDIHookPath,
54+
Args: []string{"nvidia-cdi-hook", "update-ldcache"},
55+
},
56+
},
4657
},
4758
{
4859
description: "mount error",
@@ -65,7 +76,18 @@ func TestLDCacheUpdateHook(t *testing.T) {
6576
Path: "/usr/local/lib/libbar.so",
6677
},
6778
},
68-
expectedArgs: []string{"nvidia-cdi-hook", "update-ldcache", "--folder", "/usr/local/lib", "--folder", "/usr/local/libother"},
79+
expectedHooks: []Hook{
80+
{
81+
Lifecycle: "createContainer",
82+
Path: testNvidiaCDIHookPath,
83+
Args: []string{"nvidia-cdi-hook", "create-soname-symlinks", "--folder", "/usr/local/lib", "--folder", "/usr/local/libother"},
84+
},
85+
{
86+
Lifecycle: "createContainer",
87+
Path: testNvidiaCDIHookPath,
88+
Args: []string{"nvidia-cdi-hook", "update-ldcache", "--folder", "/usr/local/lib", "--folder", "/usr/local/libother"},
89+
},
90+
},
6991
},
7092
{
7193
description: "host paths are ignored",
@@ -75,12 +97,34 @@ func TestLDCacheUpdateHook(t *testing.T) {
7597
Path: "/usr/local/lib/libfoo.so",
7698
},
7799
},
78-
expectedArgs: []string{"nvidia-cdi-hook", "update-ldcache", "--folder", "/usr/local/lib"},
100+
expectedHooks: []Hook{
101+
{
102+
Lifecycle: "createContainer",
103+
Path: testNvidiaCDIHookPath,
104+
Args: []string{"nvidia-cdi-hook", "create-soname-symlinks", "--folder", "/usr/local/lib"},
105+
},
106+
{
107+
Lifecycle: "createContainer",
108+
Path: testNvidiaCDIHookPath,
109+
Args: []string{"nvidia-cdi-hook", "update-ldcache", "--folder", "/usr/local/lib"},
110+
},
111+
},
79112
},
80113
{
81114
description: "explicit ldconfig path is passed",
82115
ldconfigPath: testLdconfigPath,
83-
expectedArgs: []string{"nvidia-cdi-hook", "update-ldcache", "--ldconfig-path", testLdconfigPath},
116+
expectedHooks: []Hook{
117+
{
118+
Lifecycle: "createContainer",
119+
Path: testNvidiaCDIHookPath,
120+
Args: []string{"nvidia-cdi-hook", "create-soname-symlinks", "--ldconfig-path", testLdconfigPath},
121+
},
122+
{
123+
Lifecycle: "createContainer",
124+
Path: testNvidiaCDIHookPath,
125+
Args: []string{"nvidia-cdi-hook", "update-ldcache", "--ldconfig-path", testLdconfigPath},
126+
},
127+
},
84128
},
85129
}
86130

@@ -91,12 +135,6 @@ func TestLDCacheUpdateHook(t *testing.T) {
91135
return tc.mounts, tc.mountError
92136
},
93137
}
94-
expectedHook := Hook{
95-
Path: testNvidiaCDIHookPath,
96-
Args: tc.expectedArgs,
97-
Lifecycle: "createContainer",
98-
}
99-
100138
d, err := NewLDCacheUpdateHook(logger, mountMock, testNvidiaCDIHookPath, tc.ldconfigPath)
101139
require.NoError(t, err)
102140

@@ -110,9 +148,7 @@ func TestLDCacheUpdateHook(t *testing.T) {
110148
}
111149

112150
require.NoError(t, err)
113-
require.Len(t, hooks, 1)
114-
115-
require.EqualValues(t, hooks[0], expectedHook)
151+
require.EqualValues(t, tc.expectedHooks, hooks)
116152

117153
devices, err := d.Devices()
118154
require.NoError(t, err)

‎tests/e2e/nvidia-container-toolkit_test.go

+22
Original file line numberDiff line numberDiff line change
@@ -215,4 +215,26 @@ var _ = Describe("docker", Ordered, ContinueOnFailure, func() {
215215
Expect(ldconfigOut).To(ContainSubstring("/usr/lib64"))
216216
})
217217
})
218+
219+
When("A container is run using CDI", Ordered, func() {
220+
BeforeAll(func(ctx context.Context) {
221+
_, _, err := r.Run("docker pull ubuntu")
222+
Expect(err).ToNot(HaveOccurred())
223+
})
224+
225+
It("should include libcuda.so in the ldcache", func(ctx context.Context) {
226+
ldcacheOutput, _, err := r.Run("docker run --rm -i --runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=runtime.nvidia.com/gpu=all ubuntu bash -c \"ldconfig -p | grep 'libcuda.so'\"")
227+
Expect(err).ToNot(HaveOccurred())
228+
Expect(ldcacheOutput).ToNot(BeEmpty())
229+
230+
ldcacheLines := strings.Split(ldcacheOutput, "\n")
231+
var libs []string
232+
for _, line := range ldcacheLines {
233+
parts := strings.SplitN(line, " (", 2)
234+
libs = append(libs, strings.TrimSpace(parts[0]))
235+
}
236+
237+
Expect(libs).To(ContainElements([]string{"libcuda.so", "libcuda.so.1"}))
238+
})
239+
})
218240
})

0 commit comments

Comments
 (0)