Issue #2400 Shared cache for built dependencies

Source/CarthageKit/Constants.swift

@@ -24,7 +24,7 @@ public struct Constants {
 	}()
 
 	/// ~/Library/Caches/org.carthage.CarthageKit/
-	private static let userCachesURL: URL = {
+	internal static let userCachesURL: URL = {
 		let fileManager = FileManager.default
 
 		let urlResult: Result<URL, NSError> = Result(catching: {

Source/CarthageKit/Project.swift

@@ -43,6 +43,9 @@ public enum ProjectEvent {
 	/// Building the project is being skipped because it is cached.
 	case skippedBuildingCached(Dependency)
 
+	/// Copying from shared cache
+	case skippedBuildingCopyFromSharedCache(Dependency)
+
 	/// Rebuilding a cached project because of a version file/framework mismatch.
 	case rebuildingCached(Dependency)
 
@@ -129,8 +132,7 @@ public final class Project { // swiftlint:disable:this type_body_length
 	private var cachedBinaryProjects: CachedBinaryProjects = [:]
 	private let cachedBinaryProjectsQueue = SerialProducerQueue(name: "org.carthage.Constants.Project.cachedBinaryProjectsQueue")
 
-	private lazy var xcodeVersionDirectory: String = XcodeVersion.make()
-		.map { "\($0.version)_\($0.buildVersion)" } ?? "Unknown"
+	private lazy var xcodeVersionDirectory: String = XcodeVersion.makeString()
 
 	/// Attempts to load Cartfile or Cartfile.private from the given directory,
 	/// merging their dependencies.
@@ -1132,6 +1134,25 @@ public final class Project { // swiftlint:disable:this type_body_length
 					return dependenciesIncludingNext
 				}
 			}
+			.flatMap(.concat) { (dependencies: [(Dependency, PinnedVersion)]) -> SignalProducer<[(Dependency, PinnedVersion)], CarthageError> in
+				if options.cacheBuilds {
+					return self.copyBuiltFrameworksFromSharedCacheIfVersionMatches(for: dependencies, options: options)
+						.flatMap(.concat) { dependency, version -> SignalProducer<(Dependency, PinnedVersion), CarthageError> in
+							self._projectEventsObserver.send(value: .skippedBuildingCopyFromSharedCache(dependency))
+							return SignalProducer(value: (dependency, version))
+						}
+						.collect()
+						.map { copiedFromSharedCacheDependencies -> [(Dependency, PinnedVersion)] in
+							// Filters out dependencies that we've copied from shared cache
+							// but preserves the build order
+							return dependencies.filter { dependency -> Bool in
+								!copiedFromSharedCacheDependencies.contains { $0 == dependency }
+							}
+					}
+				} else {
+					return SignalProducer(dependencies).collect()
+				}
+			}
 			.flatMap(.concat) { (dependencies: [(Dependency, PinnedVersion)]) -> SignalProducer<(Dependency, PinnedVersion), CarthageError> in
 				return SignalProducer(dependencies)
 					.flatMap(.concurrent(limit: 4)) { dependency, version -> SignalProducer<(Dependency, PinnedVersion), CarthageError> in
@@ -1207,6 +1228,37 @@ public final class Project { // swiftlint:disable:this type_body_length
 			}
 	}
 
+	private func copyBuiltFrameworksFromSharedCacheIfVersionMatches(
+		for dependencies: [(Dependency, PinnedVersion)],
+		options: BuildOptions
+	) -> SignalProducer<(Dependency, PinnedVersion), CarthageError> {
+		return SignalProducer(dependencies)
+			.flatMap(.concat) { dependency, version -> SignalProducer<(Dependency, PinnedVersion, URL, Bool?), CarthageError> in
+				let sharedCacheURLForDependency = sharedCacheURLFor(toolchain: options.toolchain, dependency: dependency, pinnedVersion: version)
+
+				return SignalProducer.combineLatest(
+					SignalProducer(value: dependency),
+					SignalProducer(value: version),
+					SignalProducer(value: sharedCacheURLForDependency),
+					versionFileMatches(dependency, version: version, platforms: options.platforms,
+									   rootDirectoryURL: sharedCacheURLForDependency, toolchain: options.toolchain))
+			}
+			.flatMap(.concat) { dependency, version, sharedCacheVersioned, sharedCacheMatches -> SignalProducer<(Dependency, PinnedVersion), CarthageError> in
+				if let sharedCacheVersionFileMatches = sharedCacheMatches, sharedCacheVersionFileMatches {
+					return copyFrameworks(in: sharedCacheVersioned, to: self.directoryURL, projectName: dependency.name, pinnedVersion: version, toolchain: options.toolchain)
+						.collect()
+						.flatMap(.merge) { urls -> SignalProducer<(Dependency, PinnedVersion), CarthageError> in
+							if urls.isEmpty {
+								return .empty
+							} else {
+								return SignalProducer(value: (dependency, version))
+							}
+					}
+				}
+				return .empty
+			}
+	}
+
 	private func symlinkBuildPathIfNeeded(for dependency: Dependency, version: PinnedVersion) -> SignalProducer<(), CarthageError> {
 		return dependencySet(for: dependency, version: version)
 			.flatMap(.merge) { dependencies -> SignalProducer<(), CarthageError> in
@@ -1446,6 +1498,103 @@ internal func frameworksInDirectory(_ directoryURL: URL) -> SignalProducer<URL,
 	}
 }
 
+internal func copyFrameworksAndCreateVersionFile(
+	_ frameworkURLs: [URL],
+	from directoryURL: URL,
+	to destinationURL: URL,
+	projectName: String,
+	pinnedVersion: PinnedVersion,
+	toolchain: String?
+) -> SignalProducer<URL, CarthageError> {
+	return SignalProducer(frameworkURLs)
+		.flatMap(.merge) { frameworkURL -> SignalProducer<URL, CarthageError> in
+			copyFramework(frameworkURL, toFolder: destinationURL)
+		}
+		.flatMap(.merge) { frameworkURL -> SignalProducer<URL, CarthageError> in
+			return copyDSYMToBuildFolderForFramework(frameworkURL, fromDirectoryURL: directoryURL)
+				.then(copyBCSymbolMapsToBuildFolderForFramework(frameworkURL, fromDirectoryURL: directoryURL))
+				.then(SignalProducer(value: frameworkURL))
+		}
+		.collect()
+		.flatMap(.concat) { frameworkURLs -> SignalProducer<(), CarthageError> in
+			return createVersionFilesForFrameworks(
+				frameworkURLs,
+				fromDirectoryURL: destinationURL,
+				projectName: projectName,
+				commitish: pinnedVersion.commitish
+			)
+		}
+		.then(SignalProducer<URL, CarthageError>(value: directoryURL))
+}
+
+internal func copyFrameworks(
+	in directoryURL: URL,
+	to destinationURL: URL,
+	projectName: String,
+	pinnedVersion: PinnedVersion,
+	toolchain: String?
+) -> SignalProducer<URL, CarthageError> {
+	return frameworksInDirectory(directoryURL)
+		.flatMap(.merge) { url -> SignalProducer<URL, CarthageError> in
+			return checkFrameworkCompatibility(url, usingToolchain: toolchain)
+				.mapError { error in CarthageError.internalError(description: error.description) }
+		}
+		.collect()
+		.flatMap(.merge, { frameworkURLs -> SignalProducer<URL, CarthageError> in
+			copyFrameworksAndCreateVersionFile(frameworkURLs, from: directoryURL, to: destinationURL,
+											   projectName: projectName, pinnedVersion: pinnedVersion, toolchain: toolchain)
+		})
+}
+
+/// Copies any *.bcsymbolmap files matching the given framework and contained
+/// within the given directory URL to the directory that the framework
+/// resides within.
+///
+/// If no bcsymbolmap files are found for the given framework, completes with
+/// no values.
+///
+/// Sends the URLs of the bcsymbolmap files after copying.
+internal func copyBCSymbolMapsToBuildFolderForFramework(_ frameworkURL: URL, fromDirectoryURL directoryURL: URL) -> SignalProducer<URL, CarthageError> {
+	let destinationDirectoryURL = frameworkURL.deletingLastPathComponent()
+	return BCSymbolMapsForFramework(frameworkURL, inDirectoryURL: directoryURL)
+		.copyFileURLsIntoDirectory(destinationDirectoryURL)
+}
+
+/// Creates a .version file for all of the provided frameworks.
+internal func createVersionFilesForFrameworks(
+	_ frameworkURLs: [URL],
+	fromDirectoryURL directoryURL: URL,
+	projectName: String,
+	commitish: String
+	) -> SignalProducer<(), CarthageError> {
+	return createVersionFileForCommitish(commitish, dependencyName: projectName, buildProducts: frameworkURLs,
+										 rootDirectoryURL: directoryURL)
+}
+
+/// Copies the framework at the given URL into the specified folder.
+///
+/// Sends the URL to the framework after copying.
+internal func copyFramework(_ frameworkURL: URL, toFolder folderURL: URL) -> SignalProducer<URL, CarthageError> {
+	return platformForFramework(frameworkURL)
+		.flatMap(.merge) { platform -> SignalProducer<URL, CarthageError> in
+			let platformFolderURL = folderURL.appendingPathComponent(platform.relativePath, isDirectory: true)
+			return SignalProducer(value: frameworkURL)
+				.copyFileURLsIntoDirectory(platformFolderURL)
+	}
+}
+
+/// Copies the DSYM matching the given framework and contained within the
+/// given directory URL to the directory that the framework resides within.
+///
+/// If no dSYM is found for the given framework, completes with no values.
+///
+/// Sends the URL of the dSYM after copying.
+internal func copyDSYMToBuildFolderForFramework(_ frameworkURL: URL, fromDirectoryURL directoryURL: URL) -> SignalProducer<URL, CarthageError> {
+	let destinationDirectoryURL = frameworkURL.deletingLastPathComponent()
+	return dSYMForFramework(frameworkURL, inDirectoryURL: directoryURL)
+		.copyFileURLsIntoDirectory(destinationDirectoryURL)
+}
+
 /// Sends the URL to each dSYM found in the given directory
 internal func dSYMsInDirectory(_ directoryURL: URL) -> SignalProducer<URL, CarthageError> {
 	return filesInDirectory(directoryURL, "com.apple.xcode.dsym")

Source/CarthageKit/Xcode.swift

@@ -6,6 +6,18 @@ import ReactiveSwift
 import ReactiveTask
 import XCDBLD
 
+/// Gets the shared cache file path URL for dependency at pinned version
+/// ~/Library/Caches/org.carthage.CarthageKit/SharedCache/XcodeVersion/DependencyName/Version, exemple :
+/// ~/Library/Caches/org.carthage.CarthageKit/SharedCache/9.4.1_9F2000/SwiftyUserDefaults/3.0.1
+internal func sharedCacheURLFor(toolchain: String?, dependency: Dependency, pinnedVersion: PinnedVersion) -> URL {
+	let sharedCacheURL = Constants.userCachesURL.appendingPathComponent("SharedCache")
+	let noSpacesToolchain = (toolchain ?? XcodeVersion.makeString()).replacingOccurrences(of: " ", with: "_")
+	let sharedCachePerToolchain = sharedCacheURL.appendingPathComponent(noSpacesToolchain, isDirectory: true)
+	let sharedCachePerDependency = sharedCachePerToolchain.appendingPathComponent(dependency.name, isDirectory: true)
+	let sharedCacheVersioned = sharedCachePerDependency.appendingPathComponent(pinnedVersion.commitish, isDirectory: true)
+	return sharedCacheVersioned
+}
+
 /// Emits the currect Swift version
 internal func swiftVersion(usingToolchain toolchain: String? = nil) -> SignalProducer<String, SwiftVersionError> {
 	return determineSwiftVersion(usingToolchain: toolchain).replayLazily(upTo: 1)
@@ -950,15 +962,23 @@ public func buildInDirectory( // swiftlint:disable:this function_body_length
 						)
 						.flatMapError { _ in .empty }
 				}
-
-				return createVersionFile(
-					for: dependency.dependency,
-					version: dependency.version,
-					platforms: options.platforms,
-					buildProducts: urls,
-					rootDirectoryURL: rootDirectoryURL
-					)
-					.flatMapError { _ in .empty }
+				var copyToSharedCache = SignalProducer<(URL), CarthageError>.empty
+				if options.cacheBuilds {
+					let sharedCacheURLForDependency = sharedCacheURLFor(toolchain: options.toolchain, dependency: dependency.dependency, pinnedVersion: dependency.version)
+					copyToSharedCache = copyFrameworksAndCreateVersionFile(urls, from: rootDirectoryURL, to: sharedCacheURLForDependency,
+																		   projectName: dependency.dependency.name, pinnedVersion: dependency.version, toolchain: options.toolchain)
+							.flatMapError { _ in .empty }
+				}
+				return copyToSharedCache.then(
+					createVersionFile(
+						for: dependency.dependency,
+						version: dependency.version,
+						platforms: options.platforms,
+						buildProducts: urls,
+						rootDirectoryURL: rootDirectoryURL
+						)
+						.flatMapError { _ in SignalProducer<(), CarthageError>.empty }
+				)
 			}
 			// Discard any Success values, since we want to
 			// use our initial value instead of waiting for

Source/XCDBLD/XcodeVersion.swift

@@ -42,4 +42,8 @@ public struct XcodeVersion {
 			}
 			.single()?.value
 	}
+
+	public static func makeString() -> String {
+		return make().map { "\($0.version)_\($0.buildVersion)" } ?? "Unknown"
+	}
 }

Source/carthage/Extensions.swift

@@ -116,6 +116,9 @@ internal struct ProjectEventSink {
 		case let .skippedBuildingCached(dependency):
 			carthage.println(formatting.bullets + "Valid cache found for " + formatting.projectName(dependency.name) + ", skipping build")
 
+		case let .skippedBuildingCopyFromSharedCache(dependency):
+			carthage.println(formatting.bullets + "Copying from shared cache for " + formatting.projectName(dependency.name) + ", skipping build")
+
 		case let .rebuildingCached(dependency):
 			carthage.println(formatting.bullets + "Invalid cache found for " + formatting.projectName(dependency.name)
 				+ ", rebuilding with all downstream dependencies")

Tests/CarthageKitTests/ProjectSpec.swift

@@ -41,8 +41,13 @@ class ProjectSpec: QuickSpec {
 			func buildNoSharedSchemesTest(platforms: Set<Platform> = [], cacheBuilds: Bool = true, dependenciesToBuild: [String]? = nil) -> [String] {
 				return build(directoryURL: noSharedSchemesDirectoryURL, platforms: platforms, cacheBuilds: cacheBuilds, dependenciesToBuild: dependenciesToBuild)
 			}
+
+			func cleanSharedCache() {
+				_ = try? FileManager.default.removeItem(at: Constants.userCachesURL.appendingPathComponent("SharedCache"))
+			}
 
 			beforeEach {
+				cleanSharedCache()
 				_ = try? FileManager.default.removeItem(at: buildDirectoryURL)
 				// Pre-fetch the repos so we have a cache for the given tags
 				let sourceRepoUrl = directoryURL.appendingPathComponent("SourceRepos")
@@ -153,6 +158,7 @@ class ProjectSpec: QuickSpec {
 					let result2 = buildDependencyTest(platforms: [.macOS])
 					expect(result2) == []
 
+					cleanSharedCache()
 					let result3 = buildDependencyTest(platforms: [.macOS], cacheBuilds: false)
 					expect(result3) == expected
 				}
@@ -165,6 +171,7 @@ class ProjectSpec: QuickSpec {
 
 					overwriteFramework("TestFramework3", forPlatformName: "Mac", inDirectory: buildDirectoryURL)
 
+					cleanSharedCache()
 					let result2 = buildDependencyTest(platforms: [.macOS])
 					expect(result2) == expected
 				}
@@ -182,6 +189,7 @@ class ProjectSpec: QuickSpec {
 					let modifiedJson = json.replacingOccurrences(of: "\"commitish\" : \"v1.0\"", with: "\"commitish\" : \"v1.1\"")
 					_ = try! modifiedJson.write(toFile: preludeVersionFilePath, atomically: true, encoding: .utf8)
 
+					cleanSharedCache()
 					let result2 = buildDependencyTest(platforms: [.macOS])
 					expect(result2) == expected
 				}
@@ -202,6 +210,7 @@ class ProjectSpec: QuickSpec {
 						.reduce(true) { acc, next in return  acc && next }
 					expect(allDSymsRemoved) == true
 
+					cleanSharedCache()
 					let result2 = buildDependencyTest(platforms: [.macOS])
 					expect(result2) == expected
 				}
@@ -214,9 +223,24 @@ class ProjectSpec: QuickSpec {
 
 					overwriteFramework("TestFramework2", forPlatformName: "Mac", inDirectory: buildDirectoryURL)
 
+					cleanSharedCache()
 					let result2 = buildDependencyTest(platforms: [.macOS])
 					expect(result2) == ["TestFramework2_Mac", "TestFramework1_Mac"]
 				}
+
+				it("should not rebuild shared cached frameworks unnecessarily") {
+					let expected = ["TestFramework3_Mac", "TestFramework2_Mac", "TestFramework1_Mac"]
+
+					let result1 = buildDependencyTest(platforms: [.macOS])
+					expect(result1) == expected
+
+					overwriteFramework("TestFramework1", forPlatformName: "Mac", inDirectory: buildDirectoryURL)
+					overwriteFramework("TestFramework2", forPlatformName: "Mac", inDirectory: buildDirectoryURL)
+					overwriteFramework("TestFramework3", forPlatformName: "Mac", inDirectory: buildDirectoryURL)
+
+					let result2 = buildDependencyTest(platforms: [.macOS])
+					expect(result2) == []
+				}
 
 				it("should not rebuild cached frameworks (and dependencies) unnecessarily based on dSYMs") {
 					let expected = ["TestFramework3_Mac", "TestFramework2_Mac", "TestFramework1_Mac"]
@@ -245,6 +269,7 @@ class ProjectSpec: QuickSpec {
 
 					overwriteFramework("TestFramework1", forPlatformName: "Mac", inDirectory: buildDirectoryURL)
 
+					cleanSharedCache()
 					let result2 = buildDependencyTest()
 					expect(result2.filter { $0.contains("Mac") }) == ["TestFramework1_Mac"]
 					expect(result2.filter { $0.contains("iOS") }) == ["TestFramework1_iOS"]
@@ -265,6 +290,7 @@ class ProjectSpec: QuickSpec {
 					let modifiedJson = json.replacingOccurrences(of: "\"commitish\" : \"v1.0\"", with: "\"commitish\" : \"v1.1\"")
 					_ = try! modifiedJson.write(toFile: framework2VersionFilePath, atomically: true, encoding: .utf8)
 
+					cleanSharedCache()
 					let result3 = buildNoSharedSchemesTest(platforms: [.iOS])
 					expect(result3) == ["TestFramework1_iOS"]
 				}