diff --git a/.github/workflows/docc.yml b/.github/workflows/docc.yml index ada030ae..359d3ed6 100644 --- a/.github/workflows/docc.yml +++ b/.github/workflows/docc.yml @@ -8,7 +8,7 @@ env: DEVELOPER_DIR: "/Applications/Xcode_16.0.app/Contents/Developer" jobs: DocC: - runs-on: macos-14 + runs-on: macos-15 steps: - uses: actions/checkout@v4 - name: Build DocC diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index fb10205f..3d19536c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,7 +14,7 @@ jobs: xcode_version: ["16.0"] env: DEVELOPER_DIR: "/Applications/Xcode_${{ matrix.xcode_version }}.app/Contents/Developer" - runs-on: macos-14 + runs-on: macos-15 steps: - uses: actions/checkout@v4 - name: Install SwiftLint diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3b49b94b..431da157 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ env: jobs: release: name: Build and Upload Artifact Bundle - runs-on: macos-14 + runs-on: macos-15 steps: - uses: actions/checkout@v4 - name: Resolve Dependencies diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 36202247..8a4f7577 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ jobs: - "16.0" # 6.0 env: DEVELOPER_DIR: "/Applications/Xcode_${{ matrix.xcode_version }}.app/Contents/Developer" - runs-on: macos-14 + runs-on: macos-15 steps: - name: Get swift version run: swift --version diff --git a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift index 2bb62e1a..27230098 100644 --- a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift +++ b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift @@ -108,7 +108,6 @@ struct CacheSystem: Sendable { static let defaultParalellNumber = 8 private let pinsStore: PinsStore private let outputDirectory: URL - private let storage: (any CacheStorage)? private let fileSystem: any FileSystem struct CacheTarget: Hashable, Sendable { @@ -139,18 +138,23 @@ struct CacheSystem: Sendable { init( pinsStore: PinsStore, outputDirectory: URL, - storage: (any CacheStorage)?, fileSystem: any FileSystem = localFileSystem ) { self.pinsStore = pinsStore self.outputDirectory = outputDirectory - self.storage = storage self.fileSystem = fileSystem } - func cacheFrameworks(_ targets: Set) async { - let chunked = targets.chunks(ofCount: storage?.parallelNumber ?? CacheSystem.defaultParalellNumber) + func cacheFrameworks(_ targets: Set, to storages: [any CacheStorage]) async { + for storage in storages { + await cacheFrameworks(targets, to: storage) + } + } + private func cacheFrameworks(_ targets: Set, to storage: some CacheStorage) async { + let chunked = targets.chunks(ofCount: storage.parallelNumber ?? CacheSystem.defaultParalellNumber) + + let storageName = storage.displayName for chunk in chunked { await withTaskGroup(of: Void.self) { group in for target in chunk { @@ -159,10 +163,10 @@ struct CacheSystem: Sendable { let frameworkPath = outputDirectory.appendingPathComponent(frameworkName) do { logger.info( - "🚀 Cache \(frameworkName) to cache storage", + "🚀 Cache \(frameworkName) to cache storage: \(storageName)", metadata: .color(.green) ) - try await cacheFramework(target, at: frameworkPath) + try await cacheFramework(target, at: frameworkPath, to: storage) } catch { logger.warning("⚠ī¸ Can't create caches for \(frameworkPath.path)") } @@ -173,10 +177,10 @@ struct CacheSystem: Sendable { } } - private func cacheFramework(_ target: CacheTarget, at frameworkPath: URL) async throws { + private func cacheFramework(_ target: CacheTarget, at frameworkPath: URL, to storage: any CacheStorage) async throws { let cacheKey = try await calculateCacheKey(of: target) - try await storage?.cacheFramework(frameworkPath, for: cacheKey) + try await storage.cacheFramework(frameworkPath, for: cacheKey) } func generateVersionFile(for target: CacheTarget) async throws { @@ -210,8 +214,8 @@ struct CacheSystem: Sendable { case failed(LocalizedError?) case noCache } - func restoreCacheIfPossible(target: CacheTarget) async -> RestoreResult { - guard let storage = storage else { return .noCache } + + func restoreCacheIfPossible(target: CacheTarget, storage: some CacheStorage) async -> RestoreResult { do { let cacheKey = try await calculateCacheKey(of: target) if try await storage.existsValidCache(for: cacheKey) { @@ -225,12 +229,6 @@ struct CacheSystem: Sendable { } } - private func fetchArtifacts(target: CacheTarget, to destination: URL) async throws { - guard let storage = storage else { return } - let cacheKey = try await calculateCacheKey(of: target) - try await storage.fetchArtifacts(for: cacheKey, to: destination) - } - func calculateCacheKey(of target: CacheTarget) async throws -> SwiftPMCacheKey { let targetName = target.buildProduct.target.name let pin = try retrievePin(package: target.buildProduct.package) diff --git a/Sources/ScipioKit/Producer/Cache/LocalCacheStorage.swift b/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift similarity index 72% rename from Sources/ScipioKit/Producer/Cache/LocalCacheStorage.swift rename to Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift index d7ebeba7..4a14f40e 100644 --- a/Sources/ScipioKit/Producer/Cache/LocalCacheStorage.swift +++ b/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift @@ -3,37 +3,33 @@ import ScipioStorage import PackageGraph import TSCBasic -public struct LocalCacheStorage: CacheStorage { +struct LocalDiskCacheStorage: CacheStorage { private let fileSystem: any FileSystem - public var parallelNumber: Int? { nil } + var parallelNumber: Int? { nil } enum Error: Swift.Error { case cacheDirectoryIsNotFound } - public enum CacheDirectory: Sendable { - case system - case custom(URL) - } - - private let cacheDirectroy: CacheDirectory + private let baseURL: URL? - public init(cacheDirectory: CacheDirectory = .system, fileSystem: FileSystem = localFileSystem) { - self.cacheDirectroy = cacheDirectory + /// - Parameters: + /// - baseURL: The base url for the local disk cache. When it is nil, the system cache directory (`~/Library/Caches`) will be used. + init(baseURL: URL?, fileSystem: FileSystem = localFileSystem) { + self.baseURL = baseURL self.fileSystem = fileSystem } private func buildBaseDirectoryPath() throws -> URL { let cacheDir: URL - switch cacheDirectroy { - case .system: + if let baseURL { + cacheDir = baseURL + } else { guard let systemCacheDir = fileSystem.cachesDirectory else { throw Error.cacheDirectoryIsNotFound } cacheDir = systemCacheDir.asURL - case .custom(let customPath): - cacheDir = customPath } return cacheDir.appendingPathComponent("Scipio") } @@ -51,7 +47,7 @@ public struct LocalCacheStorage: CacheStorage { .appendingPathComponent(xcFrameworkFileName(for: cacheKey)) } - public func existsValidCache(for cacheKey: some CacheKey) async -> Bool { + func existsValidCache(for cacheKey: some CacheKey) async -> Bool { do { let xcFrameworkPath = try cacheFrameworkPath(for: cacheKey) return fileSystem.exists(xcFrameworkPath.absolutePath) @@ -60,7 +56,7 @@ public struct LocalCacheStorage: CacheStorage { } } - public func cacheFramework(_ frameworkPath: URL, for cacheKey: some CacheKey) async { + func cacheFramework(_ frameworkPath: URL, for cacheKey: some CacheKey) async { do { let destination = try cacheFrameworkPath(for: cacheKey) let directoryPath = destination.deletingLastPathComponent() @@ -72,7 +68,7 @@ public struct LocalCacheStorage: CacheStorage { } } - public func fetchArtifacts(for cacheKey: some CacheKey, to destinationDir: URL) async throws { + func fetchArtifacts(for cacheKey: some CacheKey, to destinationDir: URL) async throws { let source = try cacheFrameworkPath(for: cacheKey) let destination = destinationDir.appendingPathComponent(xcFrameworkFileName(for: cacheKey)) try fileSystem.copy(from: source.absolutePath, to: destination.absolutePath) diff --git a/Sources/ScipioKit/Producer/Cache/ProjectCacheStorage.swift b/Sources/ScipioKit/Producer/Cache/ProjectCacheStorage.swift new file mode 100644 index 00000000..b2179ac6 --- /dev/null +++ b/Sources/ScipioKit/Producer/Cache/ProjectCacheStorage.swift @@ -0,0 +1,10 @@ +import Foundation +import ScipioStorage + +/// The pseudo cache storage for "project cache policy", which treats built frameworks under the project's output directory (e.g. `XCFrameworks`) +/// as valid caches but does not saving / restoring anything. +struct ProjectCacheStorage: CacheStorage { + func existsValidCache(for cacheKey: some ScipioStorage.CacheKey) async throws -> Bool { false } + func fetchArtifacts(for cacheKey: some ScipioStorage.CacheKey, to destinationDir: URL) async throws {} + func cacheFramework(_ frameworkPath: URL, for cacheKey: some ScipioStorage.CacheKey) async throws {} +} diff --git a/Sources/ScipioKit/Producer/CacheStorage+DisplayName.swift b/Sources/ScipioKit/Producer/CacheStorage+DisplayName.swift new file mode 100644 index 00000000..2f520647 --- /dev/null +++ b/Sources/ScipioKit/Producer/CacheStorage+DisplayName.swift @@ -0,0 +1,9 @@ +import ScipioStorage + +extension CacheStorage { + /// The display name of the cache storage used for logging purpose + var displayName: String { + // TODO: Define the property as CacheStorage's requirement in scipio-cache-storage + "\(type(of: self))" + } +} diff --git a/Sources/ScipioKit/Producer/FrameworkProducer.swift b/Sources/ScipioKit/Producer/FrameworkProducer.swift index 33b1c8d4..a4b50ae1 100644 --- a/Sources/ScipioKit/Producer/FrameworkProducer.swift +++ b/Sources/ScipioKit/Producer/FrameworkProducer.swift @@ -10,31 +10,15 @@ struct FrameworkProducer { private let descriptionPackage: DescriptionPackage private let baseBuildOptions: BuildOptions private let buildOptionsMatrix: [String: BuildOptions] - private let cacheMode: Runner.Options.CacheMode + private let cachePolicies: [Runner.Options.CachePolicy] private let overwrite: Bool private let outputDir: URL private let fileSystem: any FileSystem private let toolchainEnvironment: [String: String]? - private var cacheStorage: (any CacheStorage)? { - switch cacheMode { - case .disabled, .project: return nil - case .storage(let storage, _): return storage - } - } - - private var isProducingCacheEnabled: Bool { - switch cacheMode { - case .disabled: return false - case .project: return true - case .storage(_, let actors): - return actors.contains(.producer) - } - } - private var shouldGenerateVersionFile: Bool { - // cacheMode is not disabled - if case .disabled = cacheMode { + // cache is not disabled + guard !cachePolicies.isEmpty else { return false } @@ -49,7 +33,7 @@ struct FrameworkProducer { descriptionPackage: DescriptionPackage, buildOptions: BuildOptions, buildOptionsMatrix: [String: BuildOptions], - cacheMode: Runner.Options.CacheMode, + cachePolicies: [Runner.Options.CachePolicy], overwrite: Bool, outputDir: URL, toolchainEnvironment: [String: String]? = nil, @@ -58,7 +42,7 @@ struct FrameworkProducer { self.descriptionPackage = descriptionPackage self.baseBuildOptions = buildOptions self.buildOptionsMatrix = buildOptionsMatrix - self.cacheMode = cacheMode + self.cachePolicies = cachePolicies self.overwrite = overwrite self.outputDir = outputDir self.toolchainEnvironment = toolchainEnvironment @@ -108,12 +92,14 @@ struct FrameworkProducer { let pinsStore = try descriptionPackage.workspace.pinsStore.load() let cacheSystem = CacheSystem( pinsStore: pinsStore, - outputDirectory: outputDir, - storage: cacheStorage + outputDirectory: outputDir ) let targetsToBuild: OrderedSet - if cacheMode.isConsumingCacheEnabled { + if cachePolicies.isEmpty { + // no-op because cache is disabled + targetsToBuild = allTargets + } else { let targets = Set(allTargets) // Validate the existing frameworks in `outputDir` before restoration @@ -122,16 +108,20 @@ struct FrameworkProducer { cacheSystem: cacheSystem ) - let restored = await restoreAllAvailableCaches( - availableTargets: targets.subtracting(valid), - cacheSystem: cacheSystem - ) - - targetsToBuild = allTargets - .subtracting(valid) - .subtracting(restored) - } else { - targetsToBuild = allTargets + let storagesWithConsumer = cachePolicies.storages(for: .consumer) + if storagesWithConsumer.isEmpty { + // no-op + targetsToBuild = allTargets.subtracting(valid) + } else { + let restored = await restoreAllAvailableCachesIfNeeded( + availableTargets: targets.subtracting(valid), + to: storagesWithConsumer, + cacheSystem: cacheSystem + ) + targetsToBuild = allTargets + .subtracting(valid) + .subtracting(restored) + } } for target in targetsToBuild { @@ -142,9 +132,7 @@ struct FrameworkProducer { ) } - if isProducingCacheEnabled { - await cacheSystem.cacheFrameworks(Set(targetsToBuild)) - } + await cacheFrameworksIfNeeded(Set(targetsToBuild), cacheSystem: cacheSystem) if shouldGenerateVersionFile { // Versionfiles should be generate for all targets @@ -167,13 +155,20 @@ struct FrameworkProducer { group.addTask { [outputDir, fileSystem] in do { let product = target.buildProduct - let outputPath = outputDir.appendingPathComponent(product.frameworkName) + let frameworkName = product.frameworkName + let outputPath = outputDir.appendingPathComponent(frameworkName) let exists = fileSystem.exists(outputPath.absolutePath) guard exists else { return nil } let expectedCacheKey = try await cacheSystem.calculateCacheKey(of: target) let isValidCache = await cacheSystem.existsValidCache(cacheKey: expectedCacheKey) - guard isValidCache else { return nil } + guard isValidCache else { + logger.warning("⚠ī¸ Existing \(frameworkName) is outdated.", metadata: .color(.yellow)) + logger.info("🗑ī¸ Delete \(frameworkName)", metadata: .color(.red)) + try fileSystem.removeFileTree(outputPath.absolutePath) + + return nil + } let expectedCacheKeyHash = try expectedCacheKey.calculateChecksum() logger.info( @@ -194,11 +189,59 @@ struct FrameworkProducer { return validFrameworks } - private func restoreAllAvailableCaches( + private func restoreAllAvailableCachesIfNeeded( availableTargets: Set, + to storages: [any CacheStorage], + cacheSystem: CacheSystem + ) async -> Set { + var remainingTargets = availableTargets + var restored: Set = [] + + for index in storages.indices { + let storage = storages[index] + + let logSuffix = "[\(index)] \(storage.displayName)" + if index == storages.startIndex { + logger.info( + "â–ļī¸ Starting restoration with cache storage: \(logSuffix)", + metadata: .color(.green) + ) + } else { + logger.info( + "⏭ī¸ Falling back to next cache storage: \(logSuffix)", + metadata: .color(.green) + ) + } + + let restoredPerStorage = await restoreCaches( + for: remainingTargets, + from: storage, + cacheSystem: cacheSystem + ) + restored.formUnion(restoredPerStorage) + + logger.info( + "⏸ī¸ Restoration finished with cache storage: \(logSuffix)", + metadata: .color(.green) + ) + + remainingTargets.subtract(restoredPerStorage) + // If all frameworks are successfully restored, we don't need to proceed to next cache storage. + if remainingTargets.isEmpty { + break + } + } + + logger.info("⏚ī¸ Restoration finished", metadata: .color(.green)) + return restored + } + + private func restoreCaches( + for targets: Set, + from cacheStorage: any CacheStorage, cacheSystem: CacheSystem ) async -> Set { - let chunked = availableTargets.chunks(ofCount: cacheStorage?.parallelNumber ?? CacheSystem.defaultParalellNumber) + let chunked = targets.chunks(ofCount: cacheStorage.parallelNumber ?? CacheSystem.defaultParalellNumber) var restored: Set = [] for chunk in chunked { @@ -209,7 +252,8 @@ struct FrameworkProducer { do { let restored = try await restorer.restore( target: target, - cacheSystem: cacheSystem + cacheSystem: cacheSystem, + cacheStorage: cacheStorage ) return restored ? target : nil } catch { @@ -233,23 +277,16 @@ struct FrameworkProducer { // Return true if pre-built artifact is available (already existing or restored from cache) func restore( target: CacheSystem.CacheTarget, - cacheSystem: CacheSystem + cacheSystem: CacheSystem, + cacheStorage: any CacheStorage ) async throws -> Bool { let product = target.buildProduct let frameworkName = product.frameworkName - let outputPath = outputDir.appendingPathComponent(frameworkName) - let exists = fileSystem.exists(outputPath.absolutePath) let expectedCacheKey = try await cacheSystem.calculateCacheKey(of: target) let expectedCacheKeyHash = try expectedCacheKey.calculateChecksum() - if exists { - logger.warning("⚠ī¸ Existing \(frameworkName) is outdated.", metadata: .color(.yellow)) - logger.info("🗑ī¸ Delete \(frameworkName)", metadata: .color(.red)) - try fileSystem.removeFileTree(outputPath.absolutePath) - } - - let restoreResult = await cacheSystem.restoreCacheIfPossible(target: target) + let restoreResult = await cacheSystem.restoreCacheIfPossible(target: target, storage: cacheStorage) switch restoreResult { case .succeeded: logger.info("✅ Restore \(frameworkName) (\(expectedCacheKeyHash)) from cache storage.", metadata: .color(.green)) @@ -261,6 +298,7 @@ struct FrameworkProducer { } return false case .noCache: + logger.info("ℹī¸ Cache not found for \(frameworkName) (\(expectedCacheKeyHash)) from cache storage.", metadata: .color(.green)) return false } } @@ -303,6 +341,13 @@ struct FrameworkProducer { return [] } + private func cacheFrameworksIfNeeded(_ targets: Set, cacheSystem: CacheSystem) async { + let storagesWithProducer = cachePolicies.storages(for: .producer) + if !storagesWithProducer.isEmpty { + await cacheSystem.cacheFrameworks(targets, to: storagesWithProducer) + } + } + private func generateVersionFile(for target: CacheSystem.CacheTarget, using cacheSystem: CacheSystem) async { do { try await cacheSystem.generateVersionFile(for: target) @@ -312,13 +357,12 @@ struct FrameworkProducer { } } -extension Runner.Options.CacheMode { - fileprivate var isConsumingCacheEnabled: Bool { - switch self { - case .disabled: return false - case .project: return true - case .storage(_, let actors): - return actors.contains(.consumer) +extension [Runner.Options.CachePolicy] { + fileprivate func storages(for actor: Runner.Options.CachePolicy.CacheActorKind) -> [any CacheStorage] { + reduce(into: []) { result, cachePolicy in + if cachePolicy.actors.contains(actor) { + result.append(cachePolicy.storage) + } } } } diff --git a/Sources/ScipioKit/Runner.swift b/Sources/ScipioKit/Runner.swift index d3451403..e8636272 100644 --- a/Sources/ScipioKit/Runner.swift +++ b/Sources/ScipioKit/Runner.swift @@ -97,7 +97,7 @@ public struct Runner { descriptionPackage: descriptionPackage, buildOptions: buildOptions, buildOptionsMatrix: buildOptionsMatrix, - cacheMode: options.cacheMode, + cachePolicies: options.cachePolicies, overwrite: options.overwrite, outputDir: outputDir ) @@ -208,7 +208,7 @@ extension Runner { public var buildOptionsMatrix: [String: TargetBuildOptions] } - public enum CacheMode: Sendable { + public struct CachePolicy: Sendable { public enum CacheActorKind: Sendable { // Save built product to cacheStorage case producer @@ -216,9 +216,32 @@ extension Runner { case consumer } - case disabled - case project - case storage(any CacheStorage, Set) + public let storage: any CacheStorage + public let actors: Set + + public init(storage: some CacheStorage, actors: Set) { + self.storage = storage + self.actors = actors + } + + private init(_ storage: LocalDiskCacheStorage) { + self.init(storage: storage, actors: [.producer, .consumer]) + } + + /// The cache policy which treats built frameworks under the project's output directory (e.g. `XCFrameworks`) + /// as valid caches, but does not saving to / restoring from any external locations. + public static let project: Self = Self( + storage: ProjectCacheStorage(), + actors: [.producer] + ) + + /// The cache policy for saving to and restoring from the system cache directory `~/Library/Caches/Scipio`. + public static let localDisk: Self = Self(LocalDiskCacheStorage(baseURL: nil)) + + /// The cache policy for saving to and restoring from the custom cache directory `baseURL.appendingPath("Scipio")`. + public static func localDisk(baseURL: URL) -> Self { + Self(LocalDiskCacheStorage(baseURL: baseURL)) + } } public enum PlatformSpecifier: Equatable { @@ -237,7 +260,7 @@ extension Runner { public var buildOptionsContainer: BuildOptionsContainer public var shouldOnlyUseVersionsFromResolvedFile: Bool - public var cacheMode: CacheMode + public var cachePolicies: [CachePolicy] public var overwrite: Bool public var verbose: Bool public var toolchainEnvironment: [String: String]? @@ -246,7 +269,7 @@ extension Runner { baseBuildOptions: BuildOptions = .init(), buildOptionsMatrix: [String: TargetBuildOptions] = [:], shouldOnlyUseVersionsFromResolvedFile: Bool = false, - cacheMode: CacheMode = .project, + cachePolicies: [CachePolicy] = [.project], overwrite: Bool = false, verbose: Bool = false, toolchainEnvironment: [String: String]? = nil @@ -256,7 +279,7 @@ extension Runner { buildOptionsMatrix: buildOptionsMatrix ) self.shouldOnlyUseVersionsFromResolvedFile = shouldOnlyUseVersionsFromResolvedFile - self.cacheMode = cacheMode + self.cachePolicies = cachePolicies self.overwrite = overwrite self.verbose = verbose self.toolchainEnvironment = toolchainEnvironment @@ -374,3 +397,7 @@ extension Runner.Options.BuildOptionsContainer { } } } + +extension [Runner.Options.CachePolicy] { + public static let disabled: Self = [] +} diff --git a/Sources/scipio/CommandType.swift b/Sources/scipio/CommandType.swift index 3c31d9c2..7d176532 100644 --- a/Sources/scipio/CommandType.swift +++ b/Sources/scipio/CommandType.swift @@ -3,7 +3,7 @@ import ScipioKit enum CommandType { case create(platformSpecifier: Runner.Options.PlatformSpecifier) - case prepare(cacheMode: Runner.Options.CacheMode) + case prepare(cachePolicies: [Runner.Options.CachePolicy]) var mode: Runner.Mode { switch self { @@ -23,12 +23,12 @@ enum CommandType { } } - var cacheMode: Runner.Options.CacheMode { + var cachePolicies: [Runner.Options.CachePolicy] { switch self { case .create: return .disabled - case .prepare(let cacheMode): - return cacheMode + case .prepare(let cachePolicies): + return cachePolicies } } } @@ -51,19 +51,19 @@ extension Runner { let runnerOptions = Runner.Options( baseBuildOptions: baseBuildOptions, shouldOnlyUseVersionsFromResolvedFile: buildOptions.shouldOnlyUseVersionsFromResolvedFile, - cacheMode: Self.cacheMode(from: commandType), + cachePolicies: Self.cachePolicies(from: commandType), overwrite: buildOptions.overwrite, verbose: globalOptions.verbose ) self.init(mode: commandType.mode, options: runnerOptions) } - private static func cacheMode(from commandType: CommandType) -> Runner.Options.CacheMode { + private static func cachePolicies(from commandType: CommandType) -> [Runner.Options.CachePolicy] { switch commandType { case .create: return .disabled - case .prepare(let cacheMode): - return cacheMode + case .prepare(let cachePolicies): + return cachePolicies } } diff --git a/Sources/scipio/PrepareCommands.swift b/Sources/scipio/PrepareCommands.swift index 1f64e848..3f307bf6 100644 --- a/Sources/scipio/PrepareCommands.swift +++ b/Sources/scipio/PrepareCommands.swift @@ -31,18 +31,18 @@ extension Scipio { let logLevel: Logger.Level = globalOptions.verbose ? .trace : .info LoggingSystem.bootstrap(logLevel: logLevel) - let runnerCacheMode: Runner.Options.CacheMode + let runnerCachePolicies: [Runner.Options.CachePolicy] switch cachePolicy { case .disabled: - runnerCacheMode = .disabled + runnerCachePolicies = .disabled case .project: - runnerCacheMode = .project + runnerCachePolicies = [.project] case .local: - runnerCacheMode = .storage(LocalCacheStorage(), [.consumer, .producer]) + runnerCachePolicies = [.localDisk] } let runner = Runner( - commandType: .prepare(cacheMode: runnerCacheMode), + commandType: .prepare(cachePolicies: runnerCachePolicies), buildOptions: buildOptions, globalOptions: globalOptions ) diff --git a/Sources/scipio/scipio.docc/build-pipeline.md b/Sources/scipio/scipio.docc/build-pipeline.md index ce033233..bba9b950 100644 --- a/Sources/scipio/scipio.docc/build-pipeline.md +++ b/Sources/scipio/scipio.docc/build-pipeline.md @@ -153,7 +153,7 @@ let runner = Runner( ], enableLibraryEvolution: false ), - cacheMode: .project, + cachePolicies: [.project], overwrite: true, verbose: true ) @@ -230,7 +230,9 @@ let runner = Runner( buildConfiguration: .release, isSimulatorSupported: true ), - cacheMode: .storage(s3Storage, [.consumer]) + cachePolicies: [ + .init(storage: s3Storage, actors: [.consumer]), + ] ) ) ``` @@ -246,10 +248,36 @@ You can also implement your custom cache storage by implementing `CacheStorage` There are two cache actors `consumer` and `producer`. -You can specify it by a second argument of `.storage` cache mode. +You can specify it by `Runner.Options.CachePolicy.actors`. `consumer` is an actor who can fetch cache from the cache storage. `producer` is an actor who attempt to save cache to the cache storage. When build artifacts are built, then it try to save them. + +### Use multiple cache policies at the same time + +You can also use multiple cache policies, which accepts multiple cache storages with different sets of cache actors: + +```swift +import ScipioS3Storage + +let s3Storage: some CacheStorage = ScipioS3Storage.S3Storage(config: ...) +let runner = Runner( + mode: .prepareDependencies, + options: .init( + baseBuildOptions: .init( + buildConfiguration: .release, + isSimulatorSupported: true + ), + cachePolicies: [ + .init(storage: s3Storage, actors: [.consumer]), + .localDisk, + ] + ) +) +``` + +In the sample above, if some frameworks' caches are not found on `s3Storage`, those are tried to be fetched from the next `.localDisk` cache policie's storage then. The frameworks not found on the storage of `.localDisk` cache policy will be built and cached into it (since the storage is tied to `.producer` actor). + diff --git a/Sources/scipio/scipio.docc/using-s3-storage.md b/Sources/scipio/scipio.docc/using-s3-storage.md index a72266ab..40cd6faa 100644 --- a/Sources/scipio/scipio.docc/using-s3-storage.md +++ b/Sources/scipio/scipio.docc/using-s3-storage.md @@ -39,7 +39,9 @@ let runner = Runner( buildConfiguration: .release, isSimulatorSupported: true ), - cacheMode: .storage(s3Storage, [.consumer, .producer]) + cachePolicies: [ + .init(storage: s3Storage, actors: [.consumer, .producer]), + ] ) ) ``` diff --git a/Tests/ScipioKitTests/CacheSystemTests.swift b/Tests/ScipioKitTests/CacheSystemTests.swift index ace30d9a..e1a0f11c 100644 --- a/Tests/ScipioKitTests/CacheSystemTests.swift +++ b/Tests/ScipioKitTests/CacheSystemTests.swift @@ -95,8 +95,7 @@ final class CacheSystemTests: XCTestCase { ) let cacheSystem = CacheSystem( pinsStore: try descriptionPackage.workspace.pinsStore.load(), - outputDirectory: FileManager.default.temporaryDirectory.appendingPathComponent("XCFrameworks"), - storage: nil + outputDirectory: FileManager.default.temporaryDirectory.appendingPathComponent("XCFrameworks") ) let testingPackage = descriptionPackage .graph diff --git a/Tests/ScipioKitTests/IntegrationTests.swift b/Tests/ScipioKitTests/IntegrationTests.swift index 0be120a0..81dece9d 100644 --- a/Tests/ScipioKitTests/IntegrationTests.swift +++ b/Tests/ScipioKitTests/IntegrationTests.swift @@ -108,7 +108,7 @@ final class IntegrationTests: XCTestCase { ), buildOptionsMatrix: buildOptionsMatrix, shouldOnlyUseVersionsFromResolvedFile: true, - cacheMode: .disabled, + cachePolicies: .disabled, overwrite: true, verbose: false ) diff --git a/Tests/ScipioKitTests/RunnerTests.swift b/Tests/ScipioKitTests/RunnerTests.swift index 6414d573..80cd084e 100644 --- a/Tests/ScipioKitTests/RunnerTests.swift +++ b/Tests/ScipioKitTests/RunnerTests.swift @@ -277,8 +277,7 @@ final class RunnerTests: XCTestCase { let pinsStore = try descriptionPackage.workspace.pinsStore.load() let cacheSystem = CacheSystem( pinsStore: pinsStore, - outputDirectory: frameworkOutputDir, - storage: nil + outputDirectory: frameworkOutputDir ) let packages = descriptionPackage.graph.packages .filter { $0.manifest.displayName != descriptionPackage.manifest.displayName } @@ -307,7 +306,7 @@ final class RunnerTests: XCTestCase { options: .init( baseBuildOptions: .init(enableLibraryEvolution: true), shouldOnlyUseVersionsFromResolvedFile: true, - cacheMode: .project + cachePolicies: [.project] ) ) do { @@ -329,15 +328,17 @@ final class RunnerTests: XCTestCase { } } - func testLocalStorage() async throws { - let storage = LocalCacheStorage(cacheDirectory: .custom(tempDir)) + func testLocalDiskCacheStorage() async throws { + let storage = LocalDiskCacheStorage(baseURL: tempDir) let storageDir = tempDir.appendingPathComponent("Scipio") let runner = Runner( mode: .prepareDependencies, options: .init( shouldOnlyUseVersionsFromResolvedFile: true, - cacheMode: .storage(storage, [.consumer, .producer]) + cachePolicies: [ + .init(storage: storage, actors: [.consumer, .producer]), + ] ) ) do { @@ -347,10 +348,12 @@ final class RunnerTests: XCTestCase { XCTFail("Build should be succeeded. \(error.localizedDescription)") } - XCTAssertTrue(fileManager.fileExists(atPath: storageDir.appendingPathComponent("ScipioTesting").path)) + XCTAssertTrue( + fileManager.fileExists(atPath: storageDir.appendingPathComponent("ScipioTesting").path), + "The framework should be cached to the cache storage" + ) let outputFrameworkPath = frameworkOutputDir.appendingPathComponent("ScipioTesting.xcframework") - try self.fileManager.removeItem(atPath: outputFrameworkPath.path) // Fetch from local storage @@ -361,11 +364,71 @@ final class RunnerTests: XCTestCase { XCTFail("Build should be succeeded.") } - XCTAssertTrue(fileManager.fileExists(atPath: storageDir.appendingPathComponent("ScipioTesting").path)) + XCTAssertTrue( + fileManager.fileExists(atPath: outputFrameworkPath.path), + "The framework should be restored from the cache storage" + ) try fileManager.removeItem(at: storageDir) } + func testMultipleCachePolicies() async throws { + let storage1CacheDir = tempDir.appending(path: "storage1", directoryHint: .isDirectory) + let storage1 = LocalDiskCacheStorage(baseURL: storage1CacheDir) + let storage1Dir = storage1CacheDir.appendingPathComponent("Scipio") + + let storage2CacheDir = tempDir.appending(path: "storage2", directoryHint: .isDirectory) + let storage2 = LocalDiskCacheStorage(baseURL: storage2CacheDir) + let storage2Dir = storage2CacheDir.appendingPathComponent("Scipio") + + let runner = Runner( + mode: .prepareDependencies, + options: .init( + shouldOnlyUseVersionsFromResolvedFile: true, + cachePolicies: [ + .init(storage: storage1, actors: [.consumer, .producer]), + .init(storage: storage2, actors: [.consumer, .producer]), + ] + ) + ) + do { + try await runner.run(packageDirectory: testPackagePath, + frameworkOutputDir: .custom(frameworkOutputDir)) + } catch { + XCTFail("Build should be succeeded. \(error.localizedDescription)") + } + + // The cache are stored into 2 storages + XCTAssertTrue( + fileManager.fileExists(atPath: storage1Dir.appendingPathComponent("ScipioTesting").path), + "The framework should be cached to the 1st cache storage" + ) + XCTAssertTrue( + fileManager.fileExists(atPath: storage2Dir.appendingPathComponent("ScipioTesting").path), + "The framework should be cached to the 2nd cache storage as well" + ) + + let outputFrameworkPath = frameworkOutputDir.appendingPathComponent("ScipioTesting.xcframework") + try self.fileManager.removeItem(atPath: outputFrameworkPath.path) + + // Remove the storage1's cache so storage2's cache should be used instead + do { + try fileManager.removeItem(at: storage1Dir.appendingPathComponent("ScipioTesting")) + try await runner.run(packageDirectory: testPackagePath, + frameworkOutputDir: .custom(frameworkOutputDir)) + } catch { + XCTFail("Build should be succeeded.") + } + + XCTAssertTrue( + fileManager.fileExists(atPath: outputFrameworkPath.path), + "The framework should be restored from the 2nd cache storage" + ) + + try fileManager.removeItem(at: storage1CacheDir) + try fileManager.removeItem(at: storage2CacheDir) + } + func testExtractBinary() async throws { let runner = Runner( mode: .createPackage, @@ -377,7 +440,7 @@ final class RunnerTests: XCTestCase { frameworkType: .dynamic ), shouldOnlyUseVersionsFromResolvedFile: true, - cacheMode: .project, + cachePolicies: [.project], overwrite: false, verbose: false) ) @@ -404,7 +467,7 @@ final class RunnerTests: XCTestCase { frameworkType: .dynamic ), shouldOnlyUseVersionsFromResolvedFile: true, - cacheMode: .project, + cachePolicies: [.project], overwrite: false, verbose: false) ) @@ -436,8 +499,7 @@ final class RunnerTests: XCTestCase { let pinsStore = try descriptionPackage.workspace.pinsStore.load() let cacheSystem = CacheSystem( pinsStore: pinsStore, - outputDirectory: frameworkOutputDir, - storage: nil + outputDirectory: frameworkOutputDir ) let packages = descriptionPackage.graph.packages .filter { $0.manifest.displayName != descriptionPackage.manifest.displayName } @@ -476,7 +538,7 @@ final class RunnerTests: XCTestCase { enableLibraryEvolution: true ), shouldOnlyUseVersionsFromResolvedFile: true, - cacheMode: .project, + cachePolicies: [.project], overwrite: false, verbose: false) ) @@ -512,7 +574,7 @@ final class RunnerTests: XCTestCase { ), ], shouldOnlyUseVersionsFromResolvedFile: true, - cacheMode: .project, + cachePolicies: [.project], overwrite: false, verbose: false) ) @@ -550,7 +612,7 @@ final class RunnerTests: XCTestCase { isSimulatorSupported: true ), shouldOnlyUseVersionsFromResolvedFile: true, - cacheMode: .disabled + cachePolicies: .disabled ) ) @@ -593,7 +655,7 @@ final class RunnerTests: XCTestCase { frameworkType: .mergeable ), shouldOnlyUseVersionsFromResolvedFile: true, - cacheMode: .disabled + cachePolicies: .disabled ) )