Make compilecache atomic (#36416)

When several Julia processes compile the same package concurrently
(e.g. during a cluster run), they can conflict on the compile cache
file. This change makes a Julia process create a compile cache in a
temporary file and atomically rename it to the final cache file.

Co-authored-by: Takafumi Arakaki <tkf@@users.noreply.github.com>
(cherry picked from commit 3bbb582)

Author: 2 people

base/loading.jl

@@ -1256,9 +1256,9 @@ const MAX_NUM_PRECOMPILE_FILES = 10
 function compilecache(pkg::PkgId, path::String)
     # decide where to put the resulting cache file
     cachefile = compilecache_path(pkg)
+    cachepath = dirname(cachefile)
     # prune the directory with cache files
     if pkg.uuid !== nothing
-        cachepath = dirname(cachefile)
         entrypath, entryfile = cache_file_entry(pkg)
         cachefiles = filter!(x -> startswith(x, entryfile * "_"), readdir(cachepath))
         if length(cachefiles) >= MAX_NUM_PRECOMPILE_FILES
@@ -1276,20 +1276,34 @@ function compilecache(pkg::PkgId, path::String)
     # run the expression and cache the result
     verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug
     @logmsg verbosity "Precompiling $pkg"
-    p = create_expr_cache(path, cachefile, concrete_deps, pkg.uuid)
-    if success(p)
-        # append checksum to the end of the .ji file:
-        open(cachefile, "a+") do f
-            write(f, _crc32c(seekstart(f)))
+
+    # create a temporary file in `cachepath` directory, write the cache in it,
+    # write the checksum, _and then_ atomically move the file to `cachefile`.
+    tmppath, tmpio = mktemp(cachepath)
+    local p
+    try
+        close(tmpio)
+        p = create_expr_cache(path, tmppath, concrete_deps, pkg.uuid)
+        if success(p)
+            # append checksum to the end of the .ji file:
+            open(tmppath, "a+") do f
+                write(f, _crc32c(seekstart(f)))
+            end
+            # inherit permission from the source file
+            chmod(tmppath, filemode(path) & 0o777)
+
+            # this is atomic according to POSIX:
+            rename(tmppath, cachefile)
+            return cachefile
         end
-        # inherit permission from the source file
-        chmod(cachefile, filemode(path) & 0o777)
-    elseif p.exitcode == 125
+    finally
+        rm(tmppath, force=true)
+    end
+    if p.exitcode == 125
         return PrecompilableError()
     else
         error("Failed to precompile $pkg to $cachefile.")
     end
-    return cachefile
 end
 
 module_build_id(m::Module) = ccall(:jl_module_build_id, UInt64, (Any,), m)

doc/src/manual/faq.md

@@ -935,6 +935,22 @@ Julia compiles and uses its own copy of OpenBLAS, with threads currently capped
 
 Modifying OpenBLAS settings or compiling Julia with a different BLAS library, eg [Intel MKL](https://software.intel.com/en-us/mkl), may provide performance improvements. You can use [MKL.jl](https://github.com/JuliaComputing/MKL.jl), a package that makes Julia's linear algebra use Intel MKL BLAS and LAPACK instead of OpenBLAS, or search the discussion forum for suggestions on how to set this up manually. Note that Intel MKL cannot be bundled with Julia, as it is not open source.
 
+## Computing cluster
+
+### How do I manage precompilation caches in distributed file systems?
+
+When using `julia` in high-performance computing (HPC) facilities, invoking
+_n_ `julia` processes simultaneously creates at most _n_ temporary copies of
+precompilation cache files. If this is an issue (slow and/or small distributed
+file system), you may:
+
+1. Use `julia` with `--compiled-modules=no` flag to turn off precompilation.
+2. Configure a private writable depot using `pushfirst!(DEPOT_PATH, private_path)`
+   where `private_path` is a path unique to this `julia` process.  This
+   can also be done by setting environment variable `JULIA_DEPOT_PATH` to
+   `$private_path:$HOME/.julia`.
+3. Create a symlink from `~/.julia/compiled` to a directory in a scratch space.
+
 ## Julia Releases
 
 ### Do I want to use the Stable, LTS, or nightly version of Julia?

src/dump.c

@@ -2837,11 +2837,10 @@ JL_DLLEXPORT jl_value_t *jl_uncompress_argname_n(jl_value_t *syms, size_t i)
 JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist)
 {
     JL_TIMING(SAVE_MODULE);
-    char *tmpfname = strcat(strcpy((char *) alloca(strlen(fname)+8), fname), ".XXXXXX");
     ios_t f;
     jl_array_t *mod_array = NULL, *udeps = NULL;
-    if (ios_mkstemp(&f, tmpfname) == NULL) {
-        jl_printf(JL_STDERR, "Cannot open cache file \"%s\" for writing.\n", tmpfname);
+    if (ios_file(&f, fname, 1, 1, 1, 1) == NULL) {
+        jl_printf(JL_STDERR, "Cannot open cache file \"%s\" for writing.\n", fname);
         return 1;
     }
     JL_GC_PUSH2(&mod_array, &udeps);
@@ -2955,12 +2954,7 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist)
     }
     write_int32(&f, 0); // mark the end of the source text
     ios_close(&f);
-
     JL_GC_POP();
-    if (jl_fs_rename(tmpfname, fname) < 0) {
-        jl_printf(JL_STDERR, "Cannot write cache file \"%s\".\n", fname);
-        return 1;
-    }
 
     return 0;
 }