From e76e7ab26a16a6eff8e25fd43b4008cf91b4764d Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Thu, 13 Nov 2025 14:31:02 +0100 Subject: [PATCH 1/6] C#: Read from dependency directory from extractor option. --- .../DependencyDirectory.cs | 62 +++++++++++++++++++ .../NugetExeWrapper.cs | 6 +- .../NugetPackageRestorer.cs | 12 ++-- .../Semmle.Util/EnvironmentVariables.cs | 10 +++ 4 files changed, 81 insertions(+), 9 deletions(-) create mode 100644 csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs new file mode 100644 index 000000000000..8efcb5d7d787 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs @@ -0,0 +1,62 @@ +using System; +using System.IO; +using Semmle.Util; +using Semmle.Util.Logging; + +namespace Semmle.Extraction.CSharp.DependencyFetching +{ + /// + /// A directory used for storing fetched dependencies. + /// When a specific directory is set via the dependency directory extractor option, + /// we store dependencies in that directory for caching purposes. + /// Otherwise, we create a temporary directory that is deleted upon disposal. + /// + public sealed class DependencyDirectory : IDisposable + { + private readonly string userReportedDirectoryPurpose; + private readonly ILogger logger; + private readonly bool attemptCleanup; + + public DirectoryInfo DirInfo { get; } + + public DependencyDirectory(string subfolderName, string userReportedDirectoryPurpose, ILogger logger) + { + this.logger = logger; + this.userReportedDirectoryPurpose = userReportedDirectoryPurpose; + + string path; + if (EnvironmentVariables.GetBuildlessDependencyDir() is string dir) + { + path = dir; + attemptCleanup = false; + } + else + { + path = FileUtils.GetTemporaryWorkingDirectory(out _); + attemptCleanup = true; + } + DirInfo = new DirectoryInfo(Path.Join(path, subfolderName)); + DirInfo.Create(); + } + + public void Dispose() + { + if (!attemptCleanup) + { + logger.LogInfo($"Keeping {userReportedDirectoryPurpose} directory {DirInfo.FullName} for possible caching purposes."); + return; + } + + try + { + DirInfo.Delete(true); + } + catch (Exception exc) + { + logger.LogInfo($"Couldn't delete {userReportedDirectoryPurpose} directory {exc.Message}"); + } + } + + public override string ToString() => DirInfo.FullName.ToString(); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs index c77daa8899c8..10d89b1e009d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs @@ -25,15 +25,15 @@ internal class NugetExeWrapper : IDisposable /// /// The computed packages directory. - /// This will be in the Temp location + /// This will be in the Cached or Temp location /// so as to not trample the source tree. /// - private readonly TemporaryDirectory packageDirectory; + private readonly DependencyDirectory packageDirectory; /// /// Create the package manager for a specified source tree. /// - public NugetExeWrapper(FileProvider fileProvider, TemporaryDirectory packageDirectory, Semmle.Util.Logging.ILogger logger) + public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger) { this.fileProvider = fileProvider; this.packageDirectory = packageDirectory; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index e0e1bc649fa4..e2e548a46a96 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -24,12 +24,12 @@ internal sealed partial class NugetPackageRestorer : IDisposable private readonly IDotNet dotnet; private readonly DependabotProxy? dependabotProxy; private readonly IDiagnosticsWriter diagnosticsWriter; - private readonly TemporaryDirectory legacyPackageDirectory; - private readonly TemporaryDirectory missingPackageDirectory; + private readonly DependencyDirectory legacyPackageDirectory; + private readonly DependencyDirectory missingPackageDirectory; private readonly ILogger logger; private readonly ICompilationInfoContainer compilationInfoContainer; - public TemporaryDirectory PackageDirectory { get; } + public DependencyDirectory PackageDirectory { get; } public NugetPackageRestorer( FileProvider fileProvider, @@ -48,9 +48,9 @@ public NugetPackageRestorer( this.logger = logger; this.compilationInfoContainer = compilationInfoContainer; - PackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("packages"), "package", logger); - legacyPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("legacypackages"), "legacy package", logger); - missingPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("missingpackages"), "missing package", logger); + PackageDirectory = new DependencyDirectory("packages", "package", logger); + legacyPackageDirectory = new DependencyDirectory("legacypackages", "legacy package", logger); + missingPackageDirectory = new DependencyDirectory("missingpackages", "missing package", logger); } public string? TryRestore(string package) diff --git a/csharp/extractor/Semmle.Util/EnvironmentVariables.cs b/csharp/extractor/Semmle.Util/EnvironmentVariables.cs index edce64a53fe4..9f1519653de8 100644 --- a/csharp/extractor/Semmle.Util/EnvironmentVariables.cs +++ b/csharp/extractor/Semmle.Util/EnvironmentVariables.cs @@ -76,5 +76,15 @@ public static IEnumerable GetURLs(string name) { return Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OVERLAY_BASE_METADATA_OUT"); } + + /// + /// If set, returns the directory where buildless dependencies should be stored. + /// This is needed for caching dependencies. + /// + /// + public static string? GetBuildlessDependencyDir() + { + return Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OPTION_BUILDLESS_DEPENDENCY_DIR"); + } } } From 1256ccf2ebe769bf5b9167a772801590d7249443 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Thu, 13 Nov 2025 14:33:25 +0100 Subject: [PATCH 2/6] C#: Add extractor option for buildless dependency directory. --- csharp/codeql-extractor.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/csharp/codeql-extractor.yml b/csharp/codeql-extractor.yml index da7d665f7a76..8cba8f18e47e 100644 --- a/csharp/codeql-extractor.yml +++ b/csharp/codeql-extractor.yml @@ -74,3 +74,8 @@ options: [EXPERIMENTAL] The value is a path to the MsBuild binary log file that should be extracted. This option only works when `--build-mode none` is also specified. type: array + buildless_dependency_dir: + title: The path where buildless (standalone) extraction should keep dependencies. + description: > + If set, the buildless (standalone) extractor will store dependencies in this directory. + type: string From 2700843a9cb97096674b2e93f6372076afb0899c Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Mon, 17 Nov 2025 12:01:59 +0100 Subject: [PATCH 3/6] C#: Add an integration test for setting the dependency directory in BMN. --- .../Assemblies.expected | 1 + .../standalone_dependency_dir/Assemblies.ql | 7 +++++++ .../standalone_dependency_dir/proj/Program.cs | 6 ++++++ .../standalone_dependency_dir/proj/global.json | 5 +++++ .../proj/standalone.csproj | 16 ++++++++++++++++ .../standalone_dependency_dir/test.py | 8 ++++++++ 6 files changed, 43 insertions(+) create mode 100644 csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/Assemblies.expected create mode 100644 csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/Assemblies.ql create mode 100644 csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/proj/Program.cs create mode 100644 csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/proj/global.json create mode 100644 csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/proj/standalone.csproj create mode 100644 csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/test.py diff --git a/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/Assemblies.expected b/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/Assemblies.expected new file mode 100644 index 000000000000..b98f10a366ca --- /dev/null +++ b/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/Assemblies.expected @@ -0,0 +1 @@ +| dependencies/packages/newtonsoft.json/13.0.1/lib/netstandard2.0/Newtonsoft.Json.dll:0:0:0:0 | Newtonsoft.Json, Version=13.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed | diff --git a/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/Assemblies.ql b/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/Assemblies.ql new file mode 100644 index 000000000000..625fc299761e --- /dev/null +++ b/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/Assemblies.ql @@ -0,0 +1,7 @@ +import csharp + +from Assembly a +where + not a.getCompilation().getOutputAssembly() = a and + a.getName().matches("%Newtonsoft%") +select a diff --git a/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/proj/Program.cs b/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/proj/Program.cs new file mode 100644 index 000000000000..39a9e95bb6e3 --- /dev/null +++ b/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/proj/Program.cs @@ -0,0 +1,6 @@ +class Program +{ + static void Main(string[] args) + { + } +} \ No newline at end of file diff --git a/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/proj/global.json b/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/proj/global.json new file mode 100644 index 000000000000..4c6e2601f69c --- /dev/null +++ b/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/proj/global.json @@ -0,0 +1,5 @@ +{ + "sdk": { + "version": "9.0.304" + } +} diff --git a/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/proj/standalone.csproj b/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/proj/standalone.csproj new file mode 100644 index 000000000000..29604e2cbd87 --- /dev/null +++ b/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/proj/standalone.csproj @@ -0,0 +1,16 @@ + + + + Exe + net9.0 + + + + + + + + + + + diff --git a/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/test.py b/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/test.py new file mode 100644 index 000000000000..6aeb36182dd3 --- /dev/null +++ b/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/test.py @@ -0,0 +1,8 @@ +import os + +def test(codeql, csharp, cwd): + path = os.path.join(cwd, "dependencies") + os.environ["CODEQL_EXTRACTOR_CSHARP_OPTION_BUILDLESS_DEPENDENCY_DIR"] = path + # The Assemblies.ql query shows that the Newtonsoft assembly is found in the + # dependency directory set above. + codeql.database.create(source_root="proj", build_mode="none") From 90dbb7a8eb61876e7c0fcf424cd4093f6874ebbd Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Mon, 17 Nov 2025 12:50:18 +0100 Subject: [PATCH 4/6] C#: Add change note. --- .../ql/lib/change-notes/2025-11-17-dependencies-directory.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 csharp/ql/lib/change-notes/2025-11-17-dependencies-directory.md diff --git a/csharp/ql/lib/change-notes/2025-11-17-dependencies-directory.md b/csharp/ql/lib/change-notes/2025-11-17-dependencies-directory.md new file mode 100644 index 000000000000..ec86dca35b99 --- /dev/null +++ b/csharp/ql/lib/change-notes/2025-11-17-dependencies-directory.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* Added a new extractor option to specify a custom directory for dependency downloads in buildless mode. Use `-O buildless_dependency_dir=` to configure the target directory. From 138441b662a0b884365f15d9aae63a53750dd082 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Mon, 17 Nov 2025 16:03:53 +0100 Subject: [PATCH 5/6] C#: Address review comments. --- .../DependencyDirectory.cs | 2 +- .../NugetExeWrapper.cs | 4 ++-- csharp/extractor/Semmle.Util/EnvironmentVariables.cs | 3 +-- .../all-platforms/standalone_dependency_dir/test.py | 6 ++++++ 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs index 8efcb5d7d787..7c816ed58370 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs @@ -57,6 +57,6 @@ public void Dispose() } } - public override string ToString() => DirInfo.FullName.ToString(); + public override string ToString() => DirInfo.FullName; } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs index 10d89b1e009d..b90b388e865c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs @@ -24,8 +24,8 @@ internal class NugetExeWrapper : IDisposable private readonly FileProvider fileProvider; /// - /// The computed packages directory. - /// This will be in the Cached or Temp location + /// The packages directory. + /// This will be in the user-specified or computed Temp location /// so as to not trample the source tree. /// private readonly DependencyDirectory packageDirectory; diff --git a/csharp/extractor/Semmle.Util/EnvironmentVariables.cs b/csharp/extractor/Semmle.Util/EnvironmentVariables.cs index 9f1519653de8..1af05b9d4ad1 100644 --- a/csharp/extractor/Semmle.Util/EnvironmentVariables.cs +++ b/csharp/extractor/Semmle.Util/EnvironmentVariables.cs @@ -79,9 +79,8 @@ public static IEnumerable GetURLs(string name) /// /// If set, returns the directory where buildless dependencies should be stored. - /// This is needed for caching dependencies. + /// This can be used for caching dependencies. /// - /// public static string? GetBuildlessDependencyDir() { return Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OPTION_BUILDLESS_DEPENDENCY_DIR"); diff --git a/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/test.py b/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/test.py index 6aeb36182dd3..d5574979c631 100644 --- a/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/test.py +++ b/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/test.py @@ -1,4 +1,5 @@ import os +import shutil def test(codeql, csharp, cwd): path = os.path.join(cwd, "dependencies") @@ -6,3 +7,8 @@ def test(codeql, csharp, cwd): # The Assemblies.ql query shows that the Newtonsoft assembly is found in the # dependency directory set above. codeql.database.create(source_root="proj", build_mode="none") + + # Check that the packages directory has been created in the dependecies folder. + packages_dir = os.path.join(path, "packages") + assert os.path.isdir(packages_dir), "The packages directory was not created in the specified dependency directory." + shutil.rmtree(path) From 5c454d23e87cf0c5c25b0ece30ec2f5016ab0090 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Fri, 21 Nov 2025 10:18:41 +0100 Subject: [PATCH 6/6] C#: Fix typo. --- .../all-platforms/standalone_dependency_dir/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/test.py b/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/test.py index d5574979c631..3629693ad29e 100644 --- a/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/test.py +++ b/csharp/ql/integration-tests/all-platforms/standalone_dependency_dir/test.py @@ -8,7 +8,7 @@ def test(codeql, csharp, cwd): # dependency directory set above. codeql.database.create(source_root="proj", build_mode="none") - # Check that the packages directory has been created in the dependecies folder. + # Check that the packages directory has been created in the dependencies folder. packages_dir = os.path.join(path, "packages") assert os.path.isdir(packages_dir), "The packages directory was not created in the specified dependency directory." shutil.rmtree(path)