From 0e8a927d703905669bf14f05577223d8d2ad1d7a Mon Sep 17 00:00:00 2001 From: Wojciech Charysz Date: Thu, 3 Apr 2025 16:04:09 +0200 Subject: [PATCH 1/2] First version of LFS support. --- Sources/SourceControl/GitRepository.swift | 37 ++++++ .../GitRepositoryTests.swift | 117 ++++++++++++++++++ 2 files changed, 154 insertions(+) diff --git a/Sources/SourceControl/GitRepository.swift b/Sources/SourceControl/GitRepository.swift index 2b6bebb9955..246ce7979a2 100644 --- a/Sources/SourceControl/GitRepository.swift +++ b/Sources/SourceControl/GitRepository.swift @@ -78,6 +78,20 @@ private struct GitShellHelper { throw GitShellError(result: result) } } + + /// Fetches Git LFS objects for the repository at the given path + func fetchLFS(path: AbsolutePath) throws { + // Check if git-lfs is installed + do { + _ = try self.run(["lfs", "version"], environment: .init(Git.environmentBlock)) + } catch { + // Git LFS is not installed, throw a more descriptive error + throw GitInterfaceError.missingGitLFS("Git LFS is required but not installed. Please install Git LFS.") + } + + // Fetch all LFS objects + _ = try self.run(["-C", path.pathString, "lfs", "fetch", "--all"]) + } } // MARK: - GitRepositoryProvider @@ -179,6 +193,18 @@ public struct GitRepositoryProvider: RepositoryProvider, Cancellable { progress: progress ) } + + // Helper method to check for LFS attributes in a bare repository + private func checkForLFSAttributes(in path: AbsolutePath) throws -> Bool { + do { + // Get the HEAD commit and check its tree for .gitattributes + let headOutput = try self.git.run(["-C", path.pathString, "show", "HEAD:.gitattributes"]) + return headOutput.contains("filter=lfs") + } catch { + // .gitattributes might not exist at the root, or HEAD might not exist yet + return false + } + } public func fetch( repository: RepositorySpecifier, @@ -201,6 +227,14 @@ public struct GitRepositoryProvider: RepositoryProvider, Cancellable { ["--mirror"], progress: progressHandler ) + + // For a bare repository, we need to check differently if it uses Git LFS + // Look for .gitattributes in the latest commit + let hasLFSAttributes = try self.checkForLFSAttributes(in: path) + + if hasLFSAttributes { + try self.git.fetchLFS(path: path) + } } public func isValidDirectory(_ directory: Basics.AbsolutePath) throws -> Bool { @@ -1179,6 +1213,9 @@ private enum GitInterfaceError: Swift.Error { /// This indicates that a fatal error was encountered case fatalError + + /// This indicates that Git LFS is required but not installed + case missingGitLFS(String) } public struct GitRepositoryError: Error, CustomStringConvertible, DiagnosticLocationProviding { diff --git a/Tests/SourceControlTests/GitRepositoryTests.swift b/Tests/SourceControlTests/GitRepositoryTests.swift index ac905968bc5..2c466dcd1dd 100644 --- a/Tests/SourceControlTests/GitRepositoryTests.swift +++ b/Tests/SourceControlTests/GitRepositoryTests.swift @@ -21,6 +21,7 @@ import func TSCBasic.makeDirectories import class Basics.AsyncProcess import enum TSCUtility.Git +import SWBUtil class GitRepositoryTests: XCTestCase { @@ -939,4 +940,120 @@ class GitRepositoryTests: XCTestCase { XCTAssertFalse(try repositoryManager.isValidDirectory(packageDir, for: RepositorySpecifier(url: SourceControlURL(packageDir.pathString.appending(".git"))))) } } + + func testGitLFSSupport() throws { + // Skip if git-lfs is not installed + do { + try systemQuietly([Git.tool, "lfs", "version"]) + } catch { + throw XCTSkip("git-lfs not installed, skipping test") + } + + try testWithTemporaryDirectory { path in + // Create a repository with LFS content + let lfsRepoPath = path.appending("lfs-repo") + try makeDirectories(lfsRepoPath) + initGitRepo(lfsRepoPath) + + // Configure Git LFS in the repository + try systemQuietly([Git.tool, "-C", lfsRepoPath.pathString, "lfs", "install"]) + + // Create a .gitattributes file with LFS filters + let attributesContent = "*.bin filter=lfs diff=lfs merge=lfs -text" + try localFileSystem.writeFileContents(lfsRepoPath.appending(".gitattributes"), string: attributesContent) + let lfsRepo = GitRepository(path: lfsRepoPath) + try lfsRepo.stage(file: ".gitattributes") + try lfsRepo.commit(message: "Add LFS configuration") + + // Create a binary file to be tracked by LFS + let binaryData = Data([0xFF, 0x00, 0xFF, 0x00, 0xFF]) // Simple binary content + try localFileSystem.writeFileContents(lfsRepoPath.appending("test.bin"), bytes: ByteString(binaryData)) + + // Track the binary file with LFS and commit it + try systemQuietly([Git.tool, "-C", lfsRepoPath.pathString, "lfs", "track", "*.bin"]) + try lfsRepo.stage(file: "test.bin") + try lfsRepo.commit(message: "Add binary file tracked by LFS") + + // Test the fetch functionality via the provider + let provider = GitRepositoryProvider() + + // Fetch the LFS repository - this will test our LFS detection and fetch code + let lfsClonePath = path.appending("lfs-clone") + let lfsRepoSpec = RepositorySpecifier(path: lfsRepoPath) + try provider.fetch(repository: lfsRepoSpec, to: lfsClonePath) + + // Create a working copy to verify LFS content was correctly fetched + let lfsWorkingPath = path.appending("lfs-working") + _ = try provider.createWorkingCopy(repository: lfsRepoSpec, sourcePath: lfsClonePath, at: lfsWorkingPath, editable: false) + + // Verify the LFS file exists + XCTAssertFileExists(lfsWorkingPath.appending("test.bin")) + + // Read the file content to verify it's not just an LFS pointer + // LFS pointer files are small text files that start with "version https://git-lfs.github.com/spec/" + let fileContent = try localFileSystem.readFileContents(lfsWorkingPath.appending("test.bin")) + let isLFSPointer = fileContent.description.starts(with: "version https://git-lfs.github.com/spec/") + + // If LFS fetching worked correctly, we should have the actual binary content, not the pointer + XCTAssertFalse(isLFSPointer, "File content is an LFS pointer, LFS fetching failed") + + // Check the file size matches our binary data + let fileInfo = try localFileSystem.getFileInfo(lfsWorkingPath.appending("test.bin")) + XCTAssertEqual(fileInfo.size, binaryData.count) + } + } + + func testGitLFSAttributesDetection() throws { + try testWithTemporaryDirectory { path in + // Create two repositories - one with LFS attributes, one without + let lfsRepoPath = path.appending("lfs-repo") + let regularRepoPath = path.appending("regular-repo") + + try makeDirectories(lfsRepoPath) + try makeDirectories(regularRepoPath) + + initGitRepo(lfsRepoPath) + initGitRepo(regularRepoPath) + + // Add .gitattributes with LFS filters to one repo + try localFileSystem.writeFileContents(lfsRepoPath.appending(".gitattributes"), string: "*.png filter=lfs diff=lfs merge=lfs -text") + let lfsRepo = GitRepository(path: lfsRepoPath) + try lfsRepo.stage(file: ".gitattributes") + try lfsRepo.commit(message: "Add LFS attributes") + + // Add a regular file to the other repo + try localFileSystem.writeFileContents(regularRepoPath.appending("test.txt"), string: "Hello, world!") + let regularRepo = GitRepository(path: regularRepoPath) + try regularRepo.stage(file: "test.txt") + try regularRepo.commit(message: "Add regular file") + + // Create a provider to test LFS detection + let provider = GitRepositoryProvider() + + // Use reflection to access the private checkForLFSAttributes method + // This approach is better than creating a full mock + let mirror = Mirror(reflecting: provider) + let checkForLFSAttributesMethods = mirror.children.compactMap { child in + return child.label == "checkForLFSAttributes" ? child.value : nil + } + + // If we can't access the method via reflection, we'll test it indirectly + if checkForLFSAttributesMethods.isEmpty { + // Clone repositories and check properties of the clones + let lfsClonePath = path.appending("lfs-clone") + let regularClonePath = path.appending("regular-clone") + + let lfsRepoSpec = RepositorySpecifier(path: lfsRepoPath) + let regularRepoSpec = RepositorySpecifier(path: regularRepoPath) + + try provider.fetch(repository: lfsRepoSpec, to: lfsClonePath) + try provider.fetch(repository: regularRepoSpec, to: regularClonePath) + + // For now, we just verify the fetch completed successfully + // The actual LFS detection is tested in testGitLFSSupport + XCTAssertDirectoryExists(lfsClonePath) + XCTAssertDirectoryExists(regularClonePath) + } + } + } } From da82b69644c833fd0083139f25edfad1b0e0c849 Mon Sep 17 00:00:00 2001 From: Wojciech Charysz Date: Tue, 28 Oct 2025 12:06:09 +0100 Subject: [PATCH 2/2] Adding LFS support --- Sources/SourceControl/GitRepository.swift | 44 +++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/Sources/SourceControl/GitRepository.swift b/Sources/SourceControl/GitRepository.swift index 238048e23ff..f25c811212f 100644 --- a/Sources/SourceControl/GitRepository.swift +++ b/Sources/SourceControl/GitRepository.swift @@ -92,6 +92,10 @@ private struct GitShellHelper { // Fetch all LFS objects _ = try self.run(["-C", path.pathString, "lfs", "fetch", "--all"]) } + + func pullLFS(path: AbsolutePath) throws -> String { + try self.run(["-C", path.pathString, "lfs", "pull"]) + } } // MARK: - GitRepositoryProvider @@ -448,6 +452,7 @@ public final class GitRepository: Repository, WorkingCheckout { private var cachedBranches = ThreadSafeBox<[String]>() private var cachedIsBareRepo = ThreadSafeBox() private var cachedHasSubmodules = ThreadSafeBox() + private var cachedHasLFS = ThreadSafeBox() public convenience init(path: AbsolutePath, isWorkingRepo: Bool = true, cancellator: Cancellator? = .none) { // used in one-off operations on git repo, as such the terminator is not ver important @@ -716,6 +721,9 @@ public final class GitRepository: Repository, WorkingCheckout { self.cachedHasSubmodules.put(true) try self.updateSubmoduleAndCleanNotOnQueue() } + + // Pull LFS files if the repository uses LFS + try self.pullLFSIfNecessary() } /// Initializes and updates the submodules, if any, and cleans left over the files and directories using git-clean. @@ -792,6 +800,42 @@ public final class GitRepository: Repository, WorkingCheckout { return expected == repositoryPath } + /// Checks if the working repository uses Git LFS by looking for .gitattributes with LFS filters. + private func hasLFS() throws -> Bool { + return try self.cachedHasLFS.memoize { + // Check if .gitattributes exists and contains LFS filters + let gitattributesPath = self.path.appending(".gitattributes") + guard localFileSystem.exists(gitattributesPath) else { + return false + } + + let contents = try localFileSystem.readFileContents(gitattributesPath).cString + return contents.contains("filter=lfs") + } + } + + /// Pulls Git LFS files if the repository uses LFS. + private func pullLFSIfNecessary() throws { + guard try self.hasLFS() else { + return + } + + // Check if git-lfs is installed + do { + _ = try self.callGit("lfs", "version", failureMessage: "") + } catch { + // Git LFS is not installed, throw a more descriptive error + throw GitInterfaceError.missingGitLFS("Git LFS is required but not installed. Please install Git LFS.") + } + + // Pull LFS files + _ = try self.callGit( + "lfs", + "pull", + failureMessage: "Couldn't pull Git LFS files" + ) + } + /// Returns true if the file at `path` is ignored by `git` public func areIgnored(_ paths: [Basics.AbsolutePath]) throws -> [Bool] { try self.lock.withLock {