Skip to content

Commit e43da10

Browse files
elezarelezar
committed
Add create-soname-symlinks hook
This change adds a create-soname-symlinks hook that can be used to ensure that the soname symlinks for injected libraries exist in a container. This is done by calling ldconfig -n -N for the folders containing the injected libraries. This also ensures that libcuda.so is present in the ldcache when the update-ldcache hook is run. Signed-off-by: Evan Lezar <[email protected]>
1 parent 04e9bf4 commit e43da10

File tree

6 files changed

+239
-25
lines changed

6 files changed

+239
-25
lines changed

‎cmd/nvidia-cdi-hook/commands/commands.go

+2
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"github.com/urfave/cli/v2"
2121

2222
"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/chmod"
23+
soname "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/create-soname-symlinks"
2324
symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/create-symlinks"
2425
ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/update-ldcache"
2526
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
@@ -32,5 +33,6 @@ func New(logger logger.Interface) []*cli.Command {
3233
ldcache.NewCommand(logger),
3334
symlinks.NewCommand(logger),
3435
chmod.NewCommand(logger),
36+
soname.NewCommand(logger),
3537
}
3638
}

‎cmd/nvidia-cdi-hook/create-soname-symlinks/soname-symlinks.go

+139
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
/**
2+
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
**/
16+
17+
package soname
18+
19+
import (
20+
"errors"
21+
"fmt"
22+
"path/filepath"
23+
"strings"
24+
"syscall"
25+
26+
"github.com/urfave/cli/v2"
27+
28+
"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
29+
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
30+
"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
31+
)
32+
33+
type command struct {
34+
logger logger.Interface
35+
}
36+
37+
type options struct {
38+
folders cli.StringSlice
39+
ldconfigPath string
40+
containerSpec string
41+
}
42+
43+
// NewCommand constructs an create-soname-symlinks command with the specified logger
44+
func NewCommand(logger logger.Interface) *cli.Command {
45+
c := command{
46+
logger: logger,
47+
}
48+
return c.build()
49+
}
50+
51+
// build the create-soname-symlinks command
52+
func (m command) build() *cli.Command {
53+
cfg := options{}
54+
55+
// Create the 'create-soname-symlinks' command
56+
c := cli.Command{
57+
Name: "create-soname-symlinks",
58+
Usage: "Create soname symlinks for the specified folders using ldconfig -n -N",
59+
Before: func(c *cli.Context) error {
60+
return m.validateFlags(c, &cfg)
61+
},
62+
Action: func(c *cli.Context) error {
63+
return m.run(c, &cfg)
64+
},
65+
}
66+
67+
c.Flags = []cli.Flag{
68+
&cli.StringSliceFlag{
69+
Name: "folder",
70+
Usage: "Specify a folder to search for shared libraries for which soname symlinks need to be created",
71+
Destination: &cfg.folders,
72+
},
73+
&cli.StringFlag{
74+
Name: "ldconfig-path",
75+
Usage: "Specify the path to the ldconfig program",
76+
Destination: &cfg.ldconfigPath,
77+
Value: "/sbin/ldconfig",
78+
},
79+
&cli.StringFlag{
80+
Name: "container-spec",
81+
Usage: "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
82+
Destination: &cfg.containerSpec,
83+
},
84+
}
85+
86+
return &c
87+
}
88+
89+
func (m command) validateFlags(c *cli.Context, cfg *options) error {
90+
if cfg.ldconfigPath == "" {
91+
return errors.New("ldconfig-path must be specified")
92+
}
93+
return nil
94+
}
95+
96+
func (m command) run(c *cli.Context, cfg *options) error {
97+
s, err := oci.LoadContainerState(cfg.containerSpec)
98+
if err != nil {
99+
return fmt.Errorf("failed to load container state: %v", err)
100+
}
101+
102+
containerRoot, err := s.GetContainerRoot()
103+
if err != nil {
104+
return fmt.Errorf("failed to determined container root: %v", err)
105+
}
106+
if containerRoot == "" {
107+
m.logger.Warningf("No container root detected")
108+
return nil
109+
}
110+
111+
dirs := cfg.folders.Value()
112+
if len(dirs) == 0 {
113+
return nil
114+
}
115+
116+
ldconfigPath := m.resolveLDConfigPath(cfg.ldconfigPath)
117+
args := []string{filepath.Base(ldconfigPath)}
118+
119+
args = append(args,
120+
// Specify the containerRoot to use.
121+
"-r", containerRoot,
122+
// Specify -n to only process the specified folders.
123+
"-n",
124+
// Explicitly disable updating the LDCache.
125+
"-N",
126+
)
127+
// Explicitly specific the directories to add.
128+
args = append(args, dirs...)
129+
130+
//nolint:gosec // TODO: Can we harden this so that there is less risk of command injection
131+
return syscall.Exec(ldconfigPath, args, nil)
132+
}
133+
134+
// resolveLDConfigPath determines the LDConfig path to use for the system.
135+
// On systems such as Ubuntu where `/sbin/ldconfig` is a wrapper around
136+
// /sbin/ldconfig.real, the latter is returned.
137+
func (m command) resolveLDConfigPath(path string) string {
138+
return strings.TrimPrefix(config.NormalizeLDConfigPath("@"+path), "@")
139+
}

‎cmd/nvidia-ctk-installer/container/toolkit/toolkit_test.go

+7
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,13 @@ containerEdits:
8080
- libcuda.so.1::/lib/x86_64-linux-gnu/libcuda.so
8181
hookName: createContainer
8282
path: {{ .toolkitRoot }}/nvidia-cdi-hook
83+
- args:
84+
- nvidia-cdi-hook
85+
- create-soname-symlinks
86+
- --folder
87+
- /lib/x86_64-linux-gnu
88+
hookName: createContainer
89+
path: {{ .toolkitRoot }}/nvidia-cdi-hook
8390
- args:
8491
- nvidia-cdi-hook
8592
- update-ldcache

‎internal/discover/ldconfig.go

+17-10
Original file line numberDiff line numberDiff line change
@@ -50,16 +50,16 @@ func (d ldconfig) Hooks() ([]Hook, error) {
5050
if err != nil {
5151
return nil, fmt.Errorf("failed to discover mounts for ldcache update: %v", err)
5252
}
53-
h := CreateLDCacheUpdateHook(
53+
hooks := CreateLDCacheUpdateHooks(
5454
d.nvidiaCDIHookPath,
5555
d.ldconfigPath,
5656
getLibraryPaths(mounts),
5757
)
58-
return []Hook{h}, nil
58+
return hooks, nil
5959
}
6060

61-
// CreateLDCacheUpdateHook locates the NVIDIA Container Toolkit CLI and creates a hook for updating the LD Cache
62-
func CreateLDCacheUpdateHook(executable string, ldconfig string, libraries []string) Hook {
61+
// CreateLDCacheUpdateHooks locates the NVIDIA Container Toolkit CLI and creates a hook for updating the LD Cache
62+
func CreateLDCacheUpdateHooks(executable string, ldconfig string, libraries []string) []Hook {
6363
var args []string
6464

6565
if ldconfig != "" {
@@ -70,13 +70,20 @@ func CreateLDCacheUpdateHook(executable string, ldconfig string, libraries []str
7070
args = append(args, "--folder", f)
7171
}
7272

73-
hook := CreateNvidiaCDIHook(
74-
executable,
75-
"update-ldcache",
76-
args...,
77-
)
73+
hooks := []Hook{
74+
CreateNvidiaCDIHook(
75+
executable,
76+
"create-soname-symlinks",
77+
args...,
78+
),
79+
CreateNvidiaCDIHook(
80+
executable,
81+
"update-ldcache",
82+
args...,
83+
),
84+
}
7885

79-
return hook
86+
return hooks
8087
}
8188

8289
// getLibraryPaths extracts the library dirs from the specified mounts

‎internal/discover/ldconfig_test.go

+51-15
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,22 @@ func TestLDCacheUpdateHook(t *testing.T) {
3838
mounts []Mount
3939
mountError error
4040
expectedError error
41-
expectedArgs []string
41+
expectedHooks []Hook
4242
}{
4343
{
44-
description: "empty mounts",
45-
expectedArgs: []string{"nvidia-cdi-hook", "update-ldcache"},
44+
description: "empty mounts",
45+
expectedHooks: []Hook{
46+
{
47+
Lifecycle: "createContainer",
48+
Path: testNvidiaCDIHookPath,
49+
Args: []string{"nvidia-cdi-hook", "create-soname-symlinks"},
50+
},
51+
{
52+
Lifecycle: "createContainer",
53+
Path: testNvidiaCDIHookPath,
54+
Args: []string{"nvidia-cdi-hook", "update-ldcache"},
55+
},
56+
},
4657
},
4758
{
4859
description: "mount error",
@@ -65,7 +76,18 @@ func TestLDCacheUpdateHook(t *testing.T) {
6576
Path: "/usr/local/lib/libbar.so",
6677
},
6778
},
68-
expectedArgs: []string{"nvidia-cdi-hook", "update-ldcache", "--folder", "/usr/local/lib", "--folder", "/usr/local/libother"},
79+
expectedHooks: []Hook{
80+
{
81+
Lifecycle: "createContainer",
82+
Path: testNvidiaCDIHookPath,
83+
Args: []string{"nvidia-cdi-hook", "create-soname-symlinks", "--folder", "/usr/local/lib", "--folder", "/usr/local/libother"},
84+
},
85+
{
86+
Lifecycle: "createContainer",
87+
Path: testNvidiaCDIHookPath,
88+
Args: []string{"nvidia-cdi-hook", "update-ldcache", "--folder", "/usr/local/lib", "--folder", "/usr/local/libother"},
89+
},
90+
},
6991
},
7092
{
7193
description: "host paths are ignored",
@@ -75,12 +97,34 @@ func TestLDCacheUpdateHook(t *testing.T) {
7597
Path: "/usr/local/lib/libfoo.so",
7698
},
7799
},
78-
expectedArgs: []string{"nvidia-cdi-hook", "update-ldcache", "--folder", "/usr/local/lib"},
100+
expectedHooks: []Hook{
101+
{
102+
Lifecycle: "createContainer",
103+
Path: testNvidiaCDIHookPath,
104+
Args: []string{"nvidia-cdi-hook", "create-soname-symlinks", "--folder", "/usr/local/lib"},
105+
},
106+
{
107+
Lifecycle: "createContainer",
108+
Path: testNvidiaCDIHookPath,
109+
Args: []string{"nvidia-cdi-hook", "update-ldcache", "--folder", "/usr/local/lib"},
110+
},
111+
},
79112
},
80113
{
81114
description: "explicit ldconfig path is passed",
82115
ldconfigPath: testLdconfigPath,
83-
expectedArgs: []string{"nvidia-cdi-hook", "update-ldcache", "--ldconfig-path", testLdconfigPath},
116+
expectedHooks: []Hook{
117+
{
118+
Lifecycle: "createContainer",
119+
Path: testNvidiaCDIHookPath,
120+
Args: []string{"nvidia-cdi-hook", "create-soname-symlinks", "--ldconfig-path", testLdconfigPath},
121+
},
122+
{
123+
Lifecycle: "createContainer",
124+
Path: testNvidiaCDIHookPath,
125+
Args: []string{"nvidia-cdi-hook", "update-ldcache", "--ldconfig-path", testLdconfigPath},
126+
},
127+
},
84128
},
85129
}
86130

@@ -91,12 +135,6 @@ func TestLDCacheUpdateHook(t *testing.T) {
91135
return tc.mounts, tc.mountError
92136
},
93137
}
94-
expectedHook := Hook{
95-
Path: testNvidiaCDIHookPath,
96-
Args: tc.expectedArgs,
97-
Lifecycle: "createContainer",
98-
}
99-
100138
d, err := NewLDCacheUpdateHook(logger, mountMock, testNvidiaCDIHookPath, tc.ldconfigPath)
101139
require.NoError(t, err)
102140

@@ -110,9 +148,7 @@ func TestLDCacheUpdateHook(t *testing.T) {
110148
}
111149

112150
require.NoError(t, err)
113-
require.Len(t, hooks, 1)
114-
115-
require.EqualValues(t, hooks[0], expectedHook)
151+
require.EqualValues(t, tc.expectedHooks, hooks)
116152

117153
devices, err := d.Devices()
118154
require.NoError(t, err)

‎tests/e2e/nvidia-container-toolkit_test.go

+23
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package e2e
1818

1919
import (
2020
"context"
21+
"strings"
2122

2223
. "github.com/onsi/ginkgo/v2"
2324
. "github.com/onsi/gomega"
@@ -166,4 +167,26 @@ var _ = Describe("docker", Ordered, func() {
166167
Expect(referenceOutput).To(Equal(out4))
167168
})
168169
})
170+
171+
When("A container is run using CDI", Ordered, func() {
172+
BeforeAll(func(ctx context.Context) {
173+
_, _, err := r.Run("docker pull ubuntu")
174+
Expect(err).ToNot(HaveOccurred())
175+
})
176+
177+
It("should include libcuda.so in the ldcache", func(ctx context.Context) {
178+
ldcacheOutput, _, err := r.Run("docker run --rm -i --runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=runtime.nvidia.com/gpu=all ubuntu bash -c \"ldconfig -p | grep 'libcuda.so'\"")
179+
Expect(err).ToNot(HaveOccurred())
180+
Expect(ldcacheOutput).ToNot(BeEmpty())
181+
182+
ldcacheLines := strings.Split(ldcacheOutput, "\n")
183+
var libs []string
184+
for _, line := range ldcacheLines {
185+
parts := strings.SplitN(line, " (", 2)
186+
libs = append(libs, strings.TrimSpace(parts[0]))
187+
}
188+
189+
Expect(libs).To(ContainElements([]string{"libcuda.so", "libcuda.so.1"}))
190+
})
191+
})
169192
})

0 commit comments

Comments
 (0)