Skip to content

[Xamarin.Android.Build.Tasks] Path is too long #700

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 27, 2017
Merged

[Xamarin.Android.Build.Tasks] Path is too long #700

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion src/Xamarin.Android.Build.Tasks/Tasks/Aapt.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,14 @@ public class Aapt : AsyncTask
public string ImportsDirectory { get; set; }
public string OutputImportDirectory { get; set; }
public bool UseShortFileNames { get; set; }
public string AssemblyIdentityMapFile { get; set; }

public string ResourceNameCaseMap { get; set; }

public bool ExplicitCrunch { get; set; }

Dictionary<string,string> resource_name_case_map = new Dictionary<string,string> ();
AssemblyIdentityMap assemblyMap = new AssemblyIdentityMap ();

bool ManifestIsUpToDate (string manifestFile)
{
Expand Down Expand Up @@ -197,6 +199,8 @@ public override bool Execute ()
foreach (var arr in ResourceNameCaseMap.Split (';').Select (l => l.Split ('|')).Where (a => a.Length == 2))
resource_name_case_map [arr [1]] = arr [0]; // lowercase -> original

assemblyMap.Load (AssemblyIdentityMapFile);

ThreadingTasks.Parallel.ForEach (ManifestFiles, () => 0, DoExecute, (obj) => { Complete (); });

base.Execute ();
Expand Down Expand Up @@ -315,7 +319,7 @@ string ExpandString (string s)
return s.Substring (0, st + 1) + ExpandString (s.Substring (st + 1));
int ast = st + "${library.imports:".Length;
string aname = s.Substring (ast, ed - ast);
return s.Substring (0, st) + Path.Combine (OutputImportDirectory, UseShortFileNames ? MonoAndroidHelper.GetLibraryImportDirectoryNameForAssembly (aname) : aname, ImportsDirectory) + Path.DirectorySeparatorChar + ExpandString (s.Substring (ed + 1));
return s.Substring (0, st) + Path.Combine (OutputImportDirectory, UseShortFileNames ? assemblyMap.GetLibraryImportDirectoryNameForAssembly (aname) : aname, ImportsDirectory) + Path.DirectorySeparatorChar + ExpandString (s.Substring (ed + 1));
}
else
return s;
Expand Down
2 changes: 1 addition & 1 deletion src/Xamarin.Android.Build.Tasks/Tasks/CreateLibraryResourceArchive.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ public override bool Execute ()

if (Files.ArchiveZip (outpath, f => {
using (var zip = new ZipArchiveEx (f)) {
zip.AddDirectory (OutputDirectory, outDirInfo.Name);
zip.AddDirectory (OutputDirectory, "library_project_imports");
}
})) {
Log.LogDebugMessage ("Saving contents to " + outpath);
Expand Down
2 changes: 1 addition & 1 deletion src/Xamarin.Android.Build.Tasks/Tasks/CreateManagedLibraryResourceArchive.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ public override bool Execute ()

if (Files.ArchiveZip (outpath, f => {
using (var zip = new ZipArchiveEx (f)) {
zip.AddDirectory (OutputDirectory, outDirInfo.Name);
zip.AddDirectory (OutputDirectory, "library_project_imports");
}
})) {
Log.LogDebugMessage ("Saving contents to " + outpath);
Expand Down
2 changes: 1 addition & 1 deletion src/Xamarin.Android.Build.Tasks/Tasks/CreateNativeLibraryArchive.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public override bool Execute ()

if (Files.ArchiveZip (outpath, f => {
using (var zip = new ZipArchiveEx (f)) {
zip.AddDirectory (OutputDirectory, outDirInfo.Name);
zip.AddDirectory (OutputDirectory, "native_library_imports");
}
})) {
Log.LogDebugMessage ("Saving contents to " + outpath);
Expand Down
191 changes: 104 additions & 87 deletions src/Xamarin.Android.Build.Tasks/Tasks/ResolveLibraryProjectImports.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ public class ResolveLibraryProjectImports : Task
[Required]
public bool UseShortFileNames { get; set; }

[Required]
public string AssemblyIdentityMapFile { get; set; }

public string CacheFile { get; set;}

[Output]
Expand All @@ -51,13 +54,14 @@ public class ResolveLibraryProjectImports : Task
[Output]
public ITaskItem [] ResolvedResourceDirectoryStamps { get; set; }

string imports_dir = "library_project_imports";
AssemblyIdentityMap assemblyMap = new AssemblyIdentityMap();

public ResolveLibraryProjectImports ()
{
}

// Extracts library project contents under e.g. obj/Debug/[__library_projects__/*.jar | res/*/*]
// Extracts library project contents under e.g. obj/Debug/[lp/*.jar | res/*/*]
public override bool Execute ()
{
Log.LogDebugMessage ("ResolveLibraryProjectImports Task");
Expand All @@ -72,6 +76,8 @@ public override bool Execute ()
var resolvedAssetDirectories = new List<string> ();
var resolvedEnvironmentFiles = new List<string> ();

assemblyMap.Load (AssemblyIdentityMapFile);

using (var resolver = new DirectoryAssemblyResolver (Log.LogWarning, loadDebugSymbols: false)) {
Extract (resolver, jars, resolvedResourceDirectories, resolvedAssetDirectories, resolvedEnvironmentFiles);
}
Expand Down Expand Up @@ -112,6 +118,8 @@ public override bool Execute ()
document.Save (CacheFile);
}

assemblyMap.Save (AssemblyIdentityMapFile);

Log.LogDebugTaskItems (" Jars: ", Jars.Select (s => new TaskItem (s)).ToArray ());
Log.LogDebugTaskItems (" ResolvedResourceDirectories: ", ResolvedResourceDirectories.Select (s => new TaskItem (s)).ToArray ());
Log.LogDebugTaskItems (" ResolvedAssetDirectories: ", ResolvedAssetDirectories.Select (s => new TaskItem (s)).ToArray ());
Expand All @@ -135,13 +143,20 @@ static string GetTargetAssembly (ITaskItem assemblyName)
}

// Extracts library project contents under e.g. obj/Debug/[__library_projects__/*.jar | res/*/*]
// Extracts library project contents under e.g. obj/Debug/[lp/*.jar | res/*/*]
void Extract (
DirectoryAssemblyResolver res,
ICollection<string> jars,
ICollection<string> resolvedResourceDirectories,
ICollection<string> resolvedAssetDirectories,
ICollection<string> resolvedEnvironments)
{
// lets "upgrade" the old directory.
string oldPath = Path.GetFullPath (Path.Combine (OutputImportDirectory, "..", "__library_projects__"));
if (!OutputImportDirectory.Contains ("__library_projects__") && Directory.Exists (oldPath)) {
MonoAndroidHelper.SetDirectoryWriteable (Path.Combine (oldPath, ".."));
Directory.Delete (oldPath, recursive: true);
}
var outdir = new DirectoryInfo (OutputImportDirectory);
if (!outdir.Exists)
outdir.Create ();
Expand All @@ -154,26 +169,95 @@ void Extract (
.Select (a => GetTargetAssembly (a))
.Where (a => a != null)
.Distinct ()) {
foreach (var imp in new string [] {imports_dir, "library_project_imports"}.Distinct ()) {
string assemblyIdentName = Path.GetFileNameWithoutExtension (assemblyPath);
if (UseShortFileNames) {
assemblyIdentName = Xamarin.Android.Tasks.MonoAndroidHelper.GetLibraryImportDirectoryNameForAssembly (assemblyIdentName);
}
string outDirForDll = Path.Combine (OutputImportDirectory, assemblyIdentName);
string importsDir = Path.Combine (outDirForDll, imp);
string assemblyIdentName = Path.GetFileNameWithoutExtension (assemblyPath);
if (UseShortFileNames) {
assemblyIdentName = assemblyMap.GetLibraryImportDirectoryNameForAssembly (assemblyIdentName);
}
string outDirForDll = Path.Combine (OutputImportDirectory, assemblyIdentName);
string importsDir = Path.Combine (outDirForDll, ImportsDirectory);
#if SEPARATE_CRUNCH
// FIXME: review these binResDir thing and enable this. Eclipse does this.
// Enabling these blindly causes build failure on ActionBarSherlock.
//string binResDir = Path.Combine (importsDir, "bin", "res");
//string binAssemblyDir = Path.Combine (importsDir, "bin", "assets");
#endif
string resDir = Path.Combine (importsDir, "res");
string assemblyDir = Path.Combine (importsDir, "assets");

// Skip already-extracted resources.
var stamp = new FileInfo (Path.Combine (outdir.FullName, assemblyIdentName + ".stamp"));
if (stamp.Exists && stamp.LastWriteTime > new FileInfo (assemblyPath).LastWriteTime) {
Log.LogDebugMessage ("Skipped resource lookup for {0}: extracted files are up to date", assemblyPath);
#if SEPARATE_CRUNCH
// FIXME: review these binResDir thing and enable this. Eclipse does this.
// FIXME: review these binResDir/binAssemblyDir thing and enable this. Eclipse does this.
// Enabling these blindly causes build failure on ActionBarSherlock.
//string binResDir = Path.Combine (importsDir, "bin", "res");
//string binAssemblyDir = Path.Combine (importsDir, "bin", "assets");
if (Directory.Exists (binResDir))
resolvedResourceDirectories.Add (binResDir);
if (Directory.Exists (binAssemblyDir))
resolvedAssetDirectories.Add (binAssemblyDir);
#endif
string resDir = Path.Combine (importsDir, "res");
string assemblyDir = Path.Combine (importsDir, "assets");
if (Directory.Exists (resDir))
resolvedResourceDirectories.Add (resDir);
if (Directory.Exists (assemblyDir))
resolvedAssetDirectories.Add (assemblyDir);
continue;
}

if (Directory.Exists (outDirForDll))
Directory.Delete (outDirForDll, true);
Copy link
Contributor

@jonpryor jonpryor Jul 20, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This presents a theoretical problem -- which presumably won't be all that theoretical? -- in which two assemblies have an md2 which is the same value. (This should plausibly be more common, since we're only taking the first 5 values in the md2 value! See MonoAndroidHelper.GetLibraryImportDirectoryNameForAssembly(). That said, I don't see any issues in my $HOME/.local/share/Xamarin, so maybe conflict isn't that likely?)

If we have two directories which hash to the same value, what should be done, and how can it be supported?

(Can it be supported?)

For example, assume that assemblies A1 and A2 have names which hash to the same value. The Directory.Delete() here will nuke the extraction of one, which will presumably result in a later build break, as e.g. some resource is used which doesn't exist.

Alternatively, instead of the Directory.Delete(), we could instead extract both files into the same directory. The problem with this approach is deleting entries which no longer exist (bleh). Alternatively, there would be a problem if both assemblies provide a resource of the same name, e.g. classes.jar (which iirc is a fairly common name).

Offhand, I'm not immediately sure if this is solvable. At the same time, I'm also not sure if its "fixable", or that it even should be fixed.

What might be a good idea, though, is to check for it: we have all of the Assemblies of interest, and therefore we can compute the abbreviated md2 for all of those assemblies, and check to see if there are any conflicts at all:

    var haveConflicts = new Dictionary<string, int>();
    foreach (var a in Assemblies) {
        var n = Xamarin.Android.Tasks.MonoAndroidHelper.GetLibraryImportDirectoryNameForAssembly (Path.GetFileNameWithoutExtension (a));
        int c;
        if (!haveConflicts.TryGetValue (n, out c)) {
            haveConflicts.Add (n, 1);
        }
        else haveConflicts [n] = c+1;
    }
    if (haveConflicts.Values.Any (v => v > 1))
        Log.LogError ("We have a hash conflict! Please rebuild with `$(UseShortFileNames)`=False.");

That's clearly not a great error message, but the idea is there.

Copy link
Contributor Author

@dellis1972 dellis1972 Jul 21, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The Hash has been replaced by a lookup cache file which will map the assembly to a integer index directory. So you get something like

MyProject=1
Xamarin.Android.Support.v7.AppCompat.21.0.3.0=2

So on the path we only take up "1" or "2" (up to the number of dependencies). Rather than a ton of redundant characters.

Copy link
Contributor

@jonpryor jonpryor Jul 21, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As elaborated on in an earlier comment/down below, I believe that the lookup cache file needs to instead have:

1=MyProject
2=Xamarin.Android.Support.v7.AppCompat.21.0.3.0


var assembly = res.GetAssembly (assemblyPath);

foreach (var mod in assembly.Modules) {
Copy link
Contributor

@jonpryor jonpryor Jul 21, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I half wonder if the diff would be smaller if, before this line, we did:

if (true) {

so that the indentation matched what we had before.

Even if this did result in a smaller diff, I'm not sure that would actually be a good idea. :-/

// android environment files
foreach (var envtxt in mod.Resources
.Where (r => r.Name.StartsWith ("__AndroidEnvironment__", StringComparison.OrdinalIgnoreCase))
.Where (r => r is EmbeddedResource)
.Cast<EmbeddedResource> ()) {
if (!Directory.Exists (outDirForDll))
Directory.CreateDirectory (outDirForDll);
var finfo = new FileInfo (Path.Combine (outDirForDll, envtxt.Name));
using (var fs = finfo.Create ()) {
var data = envtxt.GetResourceData ();
fs.Write (data, 0, data.Length);
}
resolvedEnvironments.Add (finfo.FullName);
}

// embedded jars (EmbeddedJar, EmbeddedReferenceJar)
var resjars = mod.Resources
.Where (r => r.Name.EndsWith (".jar", StringComparison.InvariantCultureIgnoreCase))
.Select (r => (EmbeddedResource) r);
foreach (var resjar in resjars) {
var data = resjar.GetResourceData ();
if (!Directory.Exists (importsDir))
Directory.CreateDirectory (importsDir);
using (var outfs = File.Create (Path.Combine (importsDir, resjar.Name)))
outfs.Write (data, 0, data.Length);
}

// Skip already-extracted resources.
var stamp = new FileInfo (Path.Combine (outdir.FullName, assemblyIdentName + ".stamp"));
if (stamp.Exists && stamp.LastWriteTime > new FileInfo (assemblyPath).LastWriteTime) {
Log.LogDebugMessage ("Skipped resource lookup for {0}: extracted files are up to date", assemblyPath);
// embedded AndroidResourceLibrary archive
var reszip = mod.Resources.FirstOrDefault (r => r.Name == "__AndroidLibraryProjects__.zip") as EmbeddedResource;
if (reszip != null) {
if (!Directory.Exists (outDirForDll))
Directory.CreateDirectory (outDirForDll);
var finfo = new FileInfo (Path.Combine (outDirForDll, reszip.Name));
using (var fs = finfo.Create ()) {
var data = reszip.GetResourceData ();
fs.Write (data, 0, data.Length);
}

// temporarily extracted directory will look like:
// __library_projects__/[dllname]/[library_project_imports | jlibs]/bin
using (var zip = MonoAndroidHelper.ReadZipFile (finfo.FullName)) {
Files.ExtractAll (zip, outDirForDll, modifyCallback: (entryFullName) => {
return entryFullName.Replace ("library_project_imports", ImportsDirectory);
});
}

// We used to *copy* the resources to overwrite other resources,
// which resulted in missing resource issue.
// Here we replaced copy with use of '-S' option and made it to work.
#if SEPARATE_CRUNCH
// FIXME: review these binResDir/binAssemblyDir thing and enable this. Eclipse does this.
// Enabling these blindly causes build failure on ActionBarSherlock.
Expand All @@ -186,80 +270,13 @@ void Extract (
resolvedResourceDirectories.Add (resDir);
if (Directory.Exists (assemblyDir))
resolvedAssetDirectories.Add (assemblyDir);
continue;
}

if (Directory.Exists (outDirForDll))
Directory.Delete (outDirForDll, true);

Directory.CreateDirectory (importsDir);

var assembly = res.GetAssembly (assemblyPath);

foreach (var mod in assembly.Modules) {
// android environment files
foreach (var envtxt in mod.Resources
.Where (r => r.Name.StartsWith ("__AndroidEnvironment__", StringComparison.OrdinalIgnoreCase))
.Where (r => r is EmbeddedResource)
.Cast<EmbeddedResource> ()) {
if (!Directory.Exists (outDirForDll))
Directory.CreateDirectory (outDirForDll);
var finfo = new FileInfo (Path.Combine (outDirForDll, envtxt.Name));
using (var fs = finfo.Create ()) {
var data = envtxt.GetResourceData ();
fs.Write (data, 0, data.Length);
}
resolvedEnvironments.Add (finfo.FullName);
}

// embedded jars (EmbeddedJar, EmbeddedReferenceJar)
var resjars = mod.Resources
.Where (r => r.Name.EndsWith (".jar", StringComparison.InvariantCultureIgnoreCase))
.Select (r => (EmbeddedResource) r);
foreach (var resjar in resjars) {
var data = resjar.GetResourceData ();
using (var outfs = File.Create (Path.Combine (importsDir, resjar.Name)))
outfs.Write (data, 0, data.Length);
}

// embedded AndroidResourceLibrary archive
var reszip = mod.Resources.FirstOrDefault (r => r.Name == "__AndroidLibraryProjects__.zip") as EmbeddedResource;
if (reszip != null) {
if (!Directory.Exists (outDirForDll))
Directory.CreateDirectory (outDirForDll);
var finfo = new FileInfo (Path.Combine (outDirForDll, reszip.Name));
using (var fs = finfo.Create ()) {
var data = reszip.GetResourceData ();
fs.Write (data, 0, data.Length);
}

// temporarily extracted directory will look like:
// __library_projects__/[dllname]/[library_project_imports | jlibs]/bin
using (var zip = MonoAndroidHelper.ReadZipFile (finfo.FullName))
Files.ExtractAll (zip, outDirForDll);

// We used to *copy* the resources to overwrite other resources,
// which resulted in missing resource issue.
// Here we replaced copy with use of '-S' option and made it to work.
#if SEPARATE_CRUNCH
// FIXME: review these binResDir/binAssemblyDir thing and enable this. Eclipse does this.
// Enabling these blindly causes build failure on ActionBarSherlock.
if (Directory.Exists (binResDir))
resolvedResourceDirectories.Add (binResDir);
if (Directory.Exists (binAssemblyDir))
resolvedAssetDirectories.Add (binAssemblyDir);
#endif
if (Directory.Exists (resDir))
resolvedResourceDirectories.Add (resDir);
if (Directory.Exists (assemblyDir))
resolvedAssetDirectories.Add (assemblyDir);

finfo.Delete ();
}
finfo.Delete ();
}
}

if (Directory.Exists (importsDir))
stamp.Create ().Close ();
}
}

foreach (var f in outdir.GetFiles ("*.jar")
Expand Down
23 changes: 20 additions & 3 deletions src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using NUnit.Framework;
using System.IO;
using System.Linq;
using System.Collections.Generic;

namespace Xamarin.Android.Build.Tests
{
Expand Down Expand Up @@ -263,7 +264,7 @@ public void BindngFilterUnsupportedNativeAbiLibraries ()
}

[Test]
public void BindingCheckHiddenFiles ()
public void BindingCheckHiddenFiles ([Values (true, false)] bool useShortFileNames)
{
var binding = new XamarinAndroidBindingProject () {
UseLatestPlatformSdk = true,
Expand All @@ -273,15 +274,31 @@ public void BindingCheckHiddenFiles ()
binding.Jars.Add (new AndroidItem.LibraryProjectZip ("Jars\\mylibrary.aar") {
WebContent = "https://www.dropbox.com/s/astiqp8jo97x91h/mylibrary.aar?dl=1"
});
binding.SetProperty (binding.ActiveConfigurationProperties, "UseShortFileNames", useShortFileNames);
using (var bindingBuilder = CreateDllBuilder (Path.Combine ("temp", "BindingCheckHiddenFiles", "Binding"))) {
bindingBuilder.Verbosity = Microsoft.Build.Framework.LoggerVerbosity.Diagnostic;
Assert.IsTrue (bindingBuilder.Build (binding), "binding build should have succeeded");
var proj = new XamarinAndroidApplicationProject ();
proj.OtherBuildItems.Add (new BuildItem ("ProjectReference", "..\\Binding\\UnnamedProject.csproj"));
proj.SetProperty (proj.ActiveConfigurationProperties, "UseShortFileNames", useShortFileNames);
using (var b = CreateApkBuilder (Path.Combine ("temp", "BindingCheckHiddenFiles", "App"))) {
Assert.IsTrue (b.Build (proj), "Build should have succeeded.");
var dsStorePath = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, "__library_projects__",
"UnnamedProject", "library_project_imports");
var assemblyMap = b.Output.GetIntermediaryPath (Path.Combine ("lp", "map.cache"));
if (useShortFileNames)
Assert.IsTrue (File.Exists (assemblyMap), $"{assemblyMap} should exist.");
else
Assert.IsFalse (File.Exists (assemblyMap), $"{assemblyMap} should not exist.");
var assemblyIdentityMap = new List<string> ();
if (useShortFileNames) {
foreach (var s in File.ReadLines (assemblyMap)) {
assemblyIdentityMap.Add (s);
}
}
var assmeblyIdentity = useShortFileNames ? assemblyIdentityMap.IndexOf ("UnnamedProject").ToString () : "UnnamedProject";
var libaryImportsFolder = useShortFileNames ? "lp" : "__library_projects__";
var jlibs = useShortFileNames ? "jl" : "library_project_imports";
var dsStorePath = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, libaryImportsFolder,
assmeblyIdentity, jlibs);
Assert.IsTrue (Directory.Exists (dsStorePath), "{0} should exist.", dsStorePath);
Assert.IsFalse (File.Exists (Path.Combine (dsStorePath, ".DS_Store")), "{0} should NOT exist.",
Path.Combine (dsStorePath, ".DS_Store"));
Expand Down
Loading