Skip to content

Commit ddf0db2

Browse files
committed
implement thread-per-container namespace isolation for FSNotify (test failing)
1 parent 3ecb250 commit ddf0db2

File tree

6 files changed

+215
-26
lines changed

6 files changed

+215
-26
lines changed

Sources/Containerization/SandboxContext/SandboxContext.pb.swift

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1059,6 +1059,8 @@ public struct Com_Apple_Containerization_Sandbox_V3_NotifyFileSystemEventRequest
10591059

10601060
public var eventType: Com_Apple_Containerization_Sandbox_V3_FileSystemEventType = .create
10611061

1062+
public var containerID: String = String()
1063+
10621064
public var unknownFields = SwiftProtobuf.UnknownStorage()
10631065

10641066
public init() {}
@@ -2992,6 +2994,7 @@ extension Com_Apple_Containerization_Sandbox_V3_NotifyFileSystemEventRequest: Sw
29922994
public static let _protobuf_nameMap: SwiftProtobuf._NameMap = [
29932995
1: .same(proto: "path"),
29942996
2: .standard(proto: "event_type"),
2997+
3: .standard(proto: "container_id"),
29952998
]
29962999

29973000
public mutating func decodeMessage<D: SwiftProtobuf.Decoder>(decoder: inout D) throws {
@@ -3002,6 +3005,7 @@ extension Com_Apple_Containerization_Sandbox_V3_NotifyFileSystemEventRequest: Sw
30023005
switch fieldNumber {
30033006
case 1: try { try decoder.decodeSingularStringField(value: &self.path) }()
30043007
case 2: try { try decoder.decodeSingularEnumField(value: &self.eventType) }()
3008+
case 3: try { try decoder.decodeSingularStringField(value: &self.containerID) }()
30053009
default: break
30063010
}
30073011
}
@@ -3014,12 +3018,16 @@ extension Com_Apple_Containerization_Sandbox_V3_NotifyFileSystemEventRequest: Sw
30143018
if self.eventType != .create {
30153019
try visitor.visitSingularEnumField(value: self.eventType, fieldNumber: 2)
30163020
}
3021+
if !self.containerID.isEmpty {
3022+
try visitor.visitSingularStringField(value: self.containerID, fieldNumber: 3)
3023+
}
30173024
try unknownFields.traverse(visitor: &visitor)
30183025
}
30193026

30203027
public static func ==(lhs: Com_Apple_Containerization_Sandbox_V3_NotifyFileSystemEventRequest, rhs: Com_Apple_Containerization_Sandbox_V3_NotifyFileSystemEventRequest) -> Bool {
30213028
if lhs.path != rhs.path {return false}
30223029
if lhs.eventType != rhs.eventType {return false}
3030+
if lhs.containerID != rhs.containerID {return false}
30233031
if lhs.unknownFields != rhs.unknownFields {return false}
30243032
return true
30253033
}

Sources/Containerization/SandboxContext/SandboxContext.proto

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,7 @@ enum FileSystemEventType {
297297
message NotifyFileSystemEventRequest {
298298
string path = 1;
299299
FileSystemEventType event_type = 2;
300+
string container_id = 3;
300301
}
301302

302303
message NotifyFileSystemEventResponse {

Sources/Containerization/Vminitd.swift

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -369,12 +369,15 @@ extension Vminitd {
369369
}
370370

371371
/// Send a filesystem event notification to the guest.
372-
public func notifyFileSystemEvent(path: String, eventType: Com_Apple_Containerization_Sandbox_V3_FileSystemEventType) async throws
373-
-> Com_Apple_Containerization_Sandbox_V3_NotifyFileSystemEventResponse
374-
{
372+
public func notifyFileSystemEvent(
373+
path: String,
374+
eventType: Com_Apple_Containerization_Sandbox_V3_FileSystemEventType,
375+
containerID: String
376+
) async throws -> Com_Apple_Containerization_Sandbox_V3_NotifyFileSystemEventResponse {
375377
let request = Com_Apple_Containerization_Sandbox_V3_NotifyFileSystemEventRequest.with {
376378
$0.path = path
377379
$0.eventType = eventType
380+
$0.containerID = containerID
378381
}
379382

380383
let requests = AsyncStream<Com_Apple_Containerization_Sandbox_V3_NotifyFileSystemEventRequest> { continuation in

Sources/Integration/VMTests.swift

Lines changed: 27 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -369,21 +369,23 @@ extension IntegrationSuite {
369369
let group = MultiThreadedEventLoopGroup(numberOfThreads: 1)
370370
let agent = Vminitd(connection: connection, group: group)
371371

372-
// Calculate the hashed tag name for the mount source and mount in VM
373-
let mountTag = try hashMountSource(source: directory.path)
374-
let vmMountPath = "/tmp/fsnotify-test"
375-
376-
try await agent.mount(.init(type: "virtiofs", source: mountTag, destination: vmMountPath))
377-
378372
// Test 1: CREATE event on existing file
379-
let createResponse = try await agent.notifyFileSystemEvent(path: "\(vmMountPath)/existing.txt", eventType: .create)
373+
let createResponse = try await agent.notifyFileSystemEvent(
374+
path: "/mnt/existing.txt",
375+
eventType: .create,
376+
containerID: id
377+
)
380378

381379
guard createResponse.success else {
382380
throw IntegrationError.assert(msg: "CREATE event failed: \(createResponse.error)")
383381
}
384382

385383
// Test 2: MODIFY event on existing file
386-
let modifyResponse = try await agent.notifyFileSystemEvent(path: "\(vmMountPath)/existing.txt", eventType: .modify)
384+
let modifyResponse = try await agent.notifyFileSystemEvent(
385+
path: "/mnt/existing.txt",
386+
eventType: .modify,
387+
containerID: id
388+
)
387389
guard modifyResponse.success else {
388390
throw IntegrationError.assert(msg: "MODIFY event failed: \(modifyResponse.error)")
389391
}
@@ -396,34 +398,38 @@ extension IntegrationSuite {
396398
"/bin/sh", "-c",
397399
"""
398400
apk add --no-cache inotify-tools > /dev/null 2>&1 && \
399-
timeout 2 inotifywait -m /mnt -e modify,create,delete --format '%e %f' 2>/dev/null &
400-
INOTIFY_PID=$!
401-
sleep 0.1
402-
# Trigger a modify event that should be detected
403-
touch /mnt/test-inotify.txt
404-
echo "modify test-inotify.txt"
405-
wait $INOTIFY_PID 2>/dev/null || true
401+
timeout 5 inotifywait -m /mnt -e modify,create,delete --format '%e %f' 2>/dev/null || true
406402
""",
407403
]
408404
config.stdout = inotifyBuffer
409405
}
410406

411407
try await inotifyProcess.start()
412408

413-
// While inotify is running, send FSNotify events that should trigger inotify
414-
try await Task.sleep(for: .milliseconds(200))
415-
let _ = try await agent.notifyFileSystemEvent(path: "\(vmMountPath)/test-inotify.txt", eventType: .modify)
409+
// Wait for inotify to start monitoring, then send FSNotify events
410+
try await Task.sleep(for: .milliseconds(500))
411+
412+
// Send ONLY agent-driven events
413+
let _ = try await agent.notifyFileSystemEvent(
414+
path: "/mnt/existing.txt",
415+
eventType: .modify,
416+
containerID: id
417+
)
416418

417419
let _ = try await inotifyProcess.wait()
418420
let inotifyOutput = String(data: inotifyBuffer.data, encoding: .utf8) ?? ""
419421

420422
// Verify that inotify detected the modify event
421-
guard inotifyOutput.contains("modify test-inotify.txt") else {
422-
throw IntegrationError.assert(msg: "inotify did not detect FSNotify-triggered modify event. Output: \(inotifyOutput)")
423+
guard inotifyOutput.contains("MODIFY existing.txt") else {
424+
throw IntegrationError.assert(msg: "inotify did not detect FSNotify agent modify event. Output: '\(inotifyOutput)'")
423425
}
424426

425427
// Test 4: DELETE event on non-existent file
426-
let deleteResponse = try await agent.notifyFileSystemEvent(path: "\(vmMountPath)/nonexistent.txt", eventType: .delete)
428+
let deleteResponse = try await agent.notifyFileSystemEvent(
429+
path: "/mnt/nonexistent.txt",
430+
eventType: .delete,
431+
containerID: id
432+
)
427433
guard deleteResponse.success else {
428434
throw IntegrationError.assert(msg: "DELETE event failed: \(deleteResponse.error)")
429435
}

Sources/cctl/FSNotifyCommand.swift

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,11 @@ extension Application {
9898
let vminitd = Vminitd(client: client)
9999

100100
// Send the notification using the public API
101-
let response = try await vminitd.notifyFileSystemEvent(path: path, eventType: eventType)
101+
let response = try await vminitd.notifyFileSystemEvent(
102+
path: path,
103+
eventType: eventType,
104+
containerID: containerID
105+
)
102106

103107
if !response.success {
104108
let errorMsg = response.hasError ? response.error : "Unknown error"

vminitd/Sources/vminitd/ManagedContainer.swift

Lines changed: 168 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,19 @@
1414
// limitations under the License.
1515
//===----------------------------------------------------------------------===//
1616

17+
import Containerization
1718
import ContainerizationError
1819
import ContainerizationOCI
1920
import ContainerizationOS
2021
import Foundation
2122
import Logging
23+
import Synchronization
24+
25+
#if canImport(Musl)
26+
import Musl
27+
#elseif canImport(Glibc)
28+
import Glibc
29+
#endif
2230

2331
actor ManagedContainer {
2432
let id: String
@@ -28,6 +36,131 @@ actor ManagedContainer {
2836
private let log: Logger
2937
private let bundle: ContainerizationOCI.Bundle
3038
private var execs: [String: ManagedProcess] = [:]
39+
private var namespaceWorker: NamespaceWorker?
40+
41+
/// Worker thread that runs in container's namespace for filesystem operations
42+
private final class NamespaceWorker: @unchecked Sendable {
43+
private let containerID: String
44+
private let containerPID: Int32
45+
private var workerThread: Thread?
46+
private let eventQueue: Mutex<[FileSystemEvent]> = Mutex([])
47+
private let semaphore: DispatchSemaphore = DispatchSemaphore(value: 0)
48+
private let shouldStop: Atomic<Bool> = Atomic(false)
49+
50+
struct FileSystemEvent: Sendable {
51+
let path: String
52+
let eventType: Com_Apple_Containerization_Sandbox_V3_FileSystemEventType
53+
let completion: @Sendable (Result<Void, Error>) -> Void
54+
}
55+
56+
init(containerID: String, containerPID: Int32) {
57+
self.containerID = containerID
58+
self.containerPID = containerPID
59+
}
60+
61+
func start() throws {
62+
guard workerThread == nil else {
63+
throw ContainerizationError(.invalidState, message: "NamespaceWorker already started")
64+
}
65+
66+
let thread = Thread { [weak self] in
67+
self?.runWorkerLoop()
68+
}
69+
thread.name = "namespace-worker-\(containerID)"
70+
workerThread = thread
71+
thread.start()
72+
}
73+
74+
func enqueueEvent(path: String, eventType: Com_Apple_Containerization_Sandbox_V3_FileSystemEventType) async throws {
75+
try await withCheckedThrowingContinuation { continuation in
76+
let event = FileSystemEvent(
77+
path: path,
78+
eventType: eventType,
79+
completion: { @Sendable result in
80+
continuation.resume(with: result)
81+
}
82+
)
83+
84+
eventQueue.withLock { (queue: inout [FileSystemEvent]) in
85+
queue.append(event)
86+
}
87+
semaphore.signal()
88+
}
89+
}
90+
91+
func stop() {
92+
shouldStop.store(true, ordering: .relaxed)
93+
semaphore.signal() // Wake up the worker thread
94+
workerThread?.cancel()
95+
workerThread = nil
96+
}
97+
98+
private func runWorkerLoop() {
99+
// Enter container namespace
100+
do {
101+
try enterContainerNamespace()
102+
} catch {
103+
return
104+
}
105+
106+
// Worker loop
107+
while !shouldStop.load(ordering: .relaxed) {
108+
semaphore.wait()
109+
110+
// Check stop condition again after waking up
111+
if shouldStop.load(ordering: .relaxed) {
112+
break
113+
}
114+
115+
// Process all queued events
116+
let events = eventQueue.withLock { (queue: inout [FileSystemEvent]) -> [FileSystemEvent] in
117+
let currentEvents = Array(queue)
118+
queue.removeAll()
119+
return currentEvents
120+
}
121+
122+
for event in events {
123+
do {
124+
try generateSyntheticInotifyEvent(path: event.path, eventType: event.eventType)
125+
event.completion(.success(()))
126+
} catch {
127+
event.completion(.failure(error))
128+
}
129+
}
130+
}
131+
}
132+
133+
private func enterContainerNamespace() throws {
134+
let nsPath = "/proc/\(containerPID)/ns/mnt"
135+
let fd = open(nsPath, O_RDONLY)
136+
guard fd >= 0 else {
137+
throw ContainerizationError(.internalError, message: "Failed to open namespace file: \(nsPath)")
138+
}
139+
defer { _ = close(fd) }
140+
141+
guard setns(fd, CLONE_NEWNS) == 0 else {
142+
throw ContainerizationError(.internalError, message: "Failed to setns to mount namespace: \(errno)")
143+
}
144+
}
145+
146+
private func generateSyntheticInotifyEvent(
147+
path: String,
148+
eventType: Com_Apple_Containerization_Sandbox_V3_FileSystemEventType
149+
) throws {
150+
if eventType == .delete && !FileManager.default.fileExists(atPath: path) {
151+
return
152+
}
153+
154+
let attributes = try FileManager.default.attributesOfItem(atPath: path)
155+
guard let permissions = attributes[.posixPermissions] as? NSNumber else {
156+
throw ContainerizationError(.internalError, message: "Failed to get file permissions for path: \(path)")
157+
}
158+
try FileManager.default.setAttributes(
159+
[.posixPermissions: permissions],
160+
ofItemAtPath: path
161+
)
162+
}
163+
}
31164

32165
var pid: Int32 {
33166
self.initProcess.pid
@@ -79,6 +212,9 @@ actor ManagedContainer {
79212
self.id = id
80213
self.bundle = bundle
81214
self.log = log
215+
216+
// Initialize namespace worker - will be started after process starts
217+
self.namespaceWorker = nil
82218
} catch {
83219
try? cgManager.delete()
84220
throw error
@@ -96,6 +232,26 @@ extension ManagedContainer {
96232
}
97233
}
98234

235+
/// Start namespace worker thread after container process starts
236+
private func startNamespaceWorker() throws {
237+
let pid = self.initProcess.pid
238+
guard pid > 0 else {
239+
throw ContainerizationError(.invalidState, message: "Container process not started")
240+
}
241+
242+
let worker = NamespaceWorker(containerID: self.id, containerPID: pid)
243+
try worker.start()
244+
self.namespaceWorker = worker
245+
}
246+
247+
/// Execute filesystem event using dedicated namespace thread
248+
func executeFileSystemEvent(path: String, eventType: Com_Apple_Containerization_Sandbox_V3_FileSystemEventType) async throws {
249+
guard let worker = self.namespaceWorker else {
250+
throw ContainerizationError(.invalidState, message: "Namespace worker not started for container \(self.id)")
251+
}
252+
try await worker.enqueueEvent(path: path, eventType: eventType)
253+
}
254+
99255
func createExec(
100256
id: String,
101257
stdio: HostStdio,
@@ -121,7 +277,14 @@ extension ManagedContainer {
121277

122278
func start(execID: String) async throws -> Int32 {
123279
let proc = try self.getExecOrInit(execID: execID)
124-
return try await ProcessSupervisor.default.start(process: proc)
280+
let pid = try await ProcessSupervisor.default.start(process: proc)
281+
282+
// Start namespace worker thread if this is the init process
283+
if execID == self.id {
284+
try self.startNamespaceWorker()
285+
}
286+
287+
return pid
125288
}
126289

127290
func wait(execID: String) async throws -> ManagedProcess.ExitStatus {
@@ -155,6 +318,10 @@ extension ManagedContainer {
155318
}
156319

157320
func delete() throws {
321+
// Stop namespace worker thread
322+
self.namespaceWorker?.stop()
323+
self.namespaceWorker = nil
324+
158325
try self.bundle.delete()
159326
try self.cgroupManager.delete(force: true)
160327
}

0 commit comments

Comments
 (0)