CLJS-2607: Shared AOT cache: Analyzing cached files

Make transit-clj a direct dep, to ensure we can load analysis quickly.
Make analysis global AOT cache aware. This mostly benefits REPLs where
there is not necessarily a build in dependency order and we have to analyze
as we go.

Author: dnolen

pom.template.xml

@@ -53,10 +53,9 @@
             <version>1.3.0-alpha3</version>
         </dependency>
         <dependency>
-            <groupId>org.clojure</groupId>
-            <artifactId>test.check</artifactId>
-            <version>0.10.0-alpha2</version>
-            <scope>test</scope>
+            <groupId>com.cognitect</groupId>
+            <artifactId>transit-clj</artifactId>
+            <version>0.8.300</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.clojure</groupId>
@@ -65,10 +64,10 @@
             </exclusions>
         </dependency>
         <dependency>
-            <groupId>com.cognitect</groupId>
-            <artifactId>transit-clj</artifactId>
-            <version>0.8.300</version>
-            <scope>provided</scope>
+            <groupId>org.clojure</groupId>
+            <artifactId>test.check</artifactId>
+            <version>0.10.0-alpha2</version>
+            <scope>test</scope>
             <exclusions>
                 <exclusion>
                     <groupId>org.clojure</groupId>

src/main/clojure/cljs/analyzer.cljc

@@ -3833,25 +3833,75 @@
      ([format]
       (if (and (= format :transit) @transit) "json" "edn"))))
 
+#?(:clj
+   (defn build-affecting-options [opts]
+     (select-keys opts
+       [:static-fns :fn-invoke-direct :optimize-constants :elide-asserts :target
+        :cache-key :checked-arrays :language-out])))
+
+#?(:clj
+   (defn build-affecting-options-sha [path opts]
+     (let [m (assoc (build-affecting-options opts) :path path)]
+       (util/content-sha (pr-str m) 7))))
+
+#?(:clj
+   (defn ^File cache-base-path
+     ([path]
+      (cache-base-path path nil))
+     ([path opts]
+      (io/file (System/getProperty "user.home")
+        ".cljs" ".aot_cache" (util/clojurescript-version)
+        (build-affecting-options-sha path opts)))))
+
+#?(:clj
+   (defn cacheable-files
+     ([rsrc ext]
+      (cacheable-files rsrc ext nil))
+     ([rsrc ext opts]
+      (let [{:keys [ns]} (parse-ns rsrc)
+            path (cache-base-path (util/path rsrc) opts)
+            name (util/ns->relpath ns nil File/separatorChar)]
+        (into {}
+          (map
+            (fn [[k v]]
+              [k (io/file path
+                   (if (and (= (str "cljs" File/separatorChar "core$macros") name)
+                         (= :source k))
+                     (str "cljs" File/separatorChar "core.cljc")
+                     (str name v)))]))
+          {:source (str "." ext)
+           :output-file ".js"
+           :source-map ".js.map"
+           :analysis-cache-edn (str "." ext ".cache.edn")
+           :analysis-cache-json (str "." ext ".cache.json")})))))
+
 #?(:clj
    (defn cache-file
      "Given a ClojureScript source file returns the read/write path to the analysis
       cache file. Defaults to the read path which is usually also the write path."
      ([src] (cache-file src "out"))
      ([src output-dir] (cache-file src (parse-ns src) output-dir))
      ([src ns-info output-dir]
-      (cache-file src (parse-ns src) output-dir :read))
+      (cache-file src ns-info output-dir :read nil))
      ([src ns-info output-dir mode]
+      (cache-file src ns-info output-dir mode nil))
+     ([src ns-info output-dir mode opts]
       {:pre [(map? ns-info)]}
       (let [ext (cache-analysis-ext)]
         (if-let [core-cache
                  (and (= mode :read)
                       (= (:ns ns-info) 'cljs.core)
                       (io/resource (str "cljs/core.cljs.cache.aot." ext)))]
           core-cache
-          (let [target-file (util/to-target-file output-dir ns-info
-                              (util/ext (:source-file ns-info)))]
-            (io/file (str target-file ".cache." ext))))))))
+          (let [aot-cache-file
+                (when (util/url? src)
+                  ((keyword (str "analysis-cache-" ext))
+                    (cacheable-files src (util/ext src) opts)))]
+            (if (and aot-cache-file (.exists ^File aot-cache-file))
+              aot-cache-file
+              (let [target-file (util/to-target-file output-dir ns-info
+                                  (util/ext (:source-file ns-info)))]
+                (io/file (str target-file ".cache." ext))))))))))
 
 #?(:clj
    (defn requires-analysis?
@@ -3861,8 +3911,10 @@
      ([src] (requires-analysis? src "out"))
      ([src output-dir]
       (let [cache (cache-file src output-dir)]
-        (requires-analysis? src cache output-dir)))
+        (requires-analysis? src cache output-dir nil)))
      ([src cache output-dir]
+      (requires-analysis? src cache output-dir nil))
+     ([src cache output-dir opts]
       (cond
         (util/url? cache)
         (let [path (.getPath ^URL cache)]
@@ -3876,10 +3928,12 @@
         true
 
         :else
-        (let [out-src (util/to-target-file output-dir (parse-ns src))]
-          (if (not (.exists out-src))
+        (let [out-src   (util/to-target-file output-dir (parse-ns src))
+              cache-src (:output-file (cacheable-files src (util/ext src) opts))]
+          (if (and (not (.exists out-src))
+                   (not (.exists ^File cache-src)))
             true
-            (util/changed? src cache)))))))
+            (or (not cache) (util/changed? src cache))))))))
 
 #?(:clj
    (defn- get-spec-vars
@@ -4049,9 +4103,9 @@
                             (.getPath ^File res)
                             (.getPath ^URL res))
                   cache   (when (:cache-analysis opts)
-                            (cache-file res ns-info output-dir))]
+                            (cache-file res ns-info output-dir :read opts))]
               (when-not (get-in @env/*compiler* [::namespaces (:ns ns-info) :defs])
-                (if (or skip-cache (not cache) (requires-analysis? res output-dir))
+                (if (or skip-cache (not cache) (requires-analysis? res cache output-dir opts))
                   (binding [*cljs-ns* 'cljs.user
                             *cljs-file* path
                             reader/*alias-map* (or reader/*alias-map* {})]

src/main/clojure/cljs/closure.clj

@@ -569,39 +569,6 @@
     (map compiled-file
          (comp/compile-root src-dir out-dir opts))))
 
-(defn build-affecting-options-sha [path opts]
-  (let [m (assoc (comp/build-affecting-options opts) :path path)]
-    (util/content-sha (pr-str m) 7)))
-
-(defn ^File cache-base-path
-  ([path]
-   (cache-base-path path nil))
-  ([path opts]
-   (io/file (System/getProperty "user.home")
-     ".cljs" ".aot_cache" (util/clojurescript-version)
-     (build-affecting-options-sha path opts))))
-
-(defn cacheable-files
-  ([rsrc ext]
-   (cacheable-files rsrc ext nil))
-  ([rsrc ext opts]
-   (let [{:keys [ns]} (ana/parse-ns rsrc)
-         path (cache-base-path (util/path rsrc) opts)
-         name (util/ns->relpath ns nil File/separatorChar)]
-     (into {}
-       (map
-         (fn [[k v]]
-           [k (io/file path
-                (if (and (= (str "cljs" File/separatorChar "core$macros") name)
-                         (= :source k))
-                  (str "cljs" File/separatorChar "core.cljc")
-                  (str name v)))]))
-       {:source (str "." ext)
-        :output-file ".js"
-        :source-map ".js.map"
-        :analysis-cache-edn (str "." ext ".cache.edn")
-        :analysis-cache-json (str "." ext ".cache.json")}))))
-
 (defn ^String path-from-jarfile
   "Given the URL of a file within a jar, return the path of the file
   from the root of the jar."
@@ -632,7 +599,7 @@
   [jar-file {:keys [output-file] :as opts}]
   (let [out-file (when output-file
                    (io/file (util/output-directory opts) output-file))
-        cacheable (cacheable-files jar-file (util/ext jar-file) opts)]
+        cacheable (ana/cacheable-files jar-file (util/ext jar-file) opts)]
     (when (or (nil? out-file)
               (not (.exists ^File out-file))
               (not= (util/compiled-by-version out-file)
@@ -641,7 +608,7 @@
       ;; actually compile from JAR
       (if-not (:aot-cache opts)
         (-compile (jar-file-to-disk jar-file (util/output-directory opts) opts) opts)
-        (let [cache-path (cache-base-path (util/path jar-file) opts)]
+        (let [cache-path (ana/cache-base-path (util/path jar-file) opts)]
           (when-not (.exists (:output-file cacheable))
             (-compile (jar-file-to-disk jar-file cache-path opts)
               (assoc opts :output-dir (util/path cache-path))))

src/main/clojure/cljs/compiler.cljc

@@ -1288,11 +1288,6 @@
    (defn url-path [^File f]
      (.getPath (.toURL (.toURI f)))))
 
-(defn build-affecting-options [opts]
-  (select-keys opts
-    [:static-fns :fn-invoke-direct :optimize-constants :elide-asserts :target
-     :cache-key :checked-arrays :language-out]))
-
 #?(:clj
    (defn compiled-by-string
      ([]
@@ -1303,7 +1298,7 @@
       (str "// Compiled by ClojureScript "
         (util/clojurescript-version)
         (when opts
-          (str " " (pr-str (build-affecting-options opts))))))))
+          (str " " (pr-str (ana/build-affecting-options opts))))))))
 
 #?(:clj
    (defn cached-core [ns ext opts]
@@ -1491,8 +1486,8 @@
                  (and version (not= version version')))
                (and opts
                     (not (and (io/resource "cljs/core.aot.js") (= 'cljs.core ns)))
-                    (not= (build-affecting-options opts)
-                          (build-affecting-options (util/build-options dest))))
+                    (not= (ana/build-affecting-options opts)
+                          (ana/build-affecting-options (util/build-options dest))))
                (and opts (:source-map opts)
                  (if (= (:optimizations opts) :none)
                    (not (.exists (io/file (str (.getPath dest) ".map"))))