WIP: set etags for static resources

Cargo.toml

@@ -102,13 +102,15 @@ fn-error-context = "0.2.0"
 # Templating
 askama = "0.14.0"
 walkdir = "2"
+phf = "0.13.1"
 
 # Date and Time utilities
 chrono = { version = "0.4.11", default-features = false, features = ["clock", "serde"] }
 
 # Transitive dependencies we don't use directly but need to have specific versions of
 thread_local = "1.1.3"
 constant_time_eq = "0.4.2"
+md5 = "0.8.0"
 
 [dev-dependencies]
 criterion = "0.7.0"
@@ -131,8 +133,10 @@ debug = "line-tables-only"
 
 [build-dependencies]
 time = "0.3"
+md5 = "0.8.0"
 gix = { version = "0.74.0", default-features = false }
 string_cache_codegen = "0.6.1"
+phf_codegen = "0.13"
 walkdir = "2"
 anyhow = { version = "1.0.42", features = ["backtrace"] }
 grass = { version = "0.13.1", default-features = false }

NOTES.md

@@ -0,0 +1,4 @@
+# NOTES
+
+* optionalFromRequest for TypedHeader<IfNoneMatch> never retursn "None",
+  and always returns "Some()"

build.rs

@@ -1,5 +1,5 @@
 use anyhow::{Context as _, Error, Result};
-use std::{env, path::Path};
+use std::{env, fs::File, io::Write as _, path::Path};
 
 mod tracked {
     use std::{
@@ -68,13 +68,27 @@ mod tracked {
     }
 }
 
+type ETagMap<'a> = phf_codegen::Map<'a, String>;
+
 fn main() -> Result<()> {
     let out_dir = env::var("OUT_DIR").context("missing OUT_DIR")?;
     let out_dir = Path::new(&out_dir);
     read_git_version()?;
-    compile_sass(out_dir)?;
+
+    let mut etag_map: ETagMap = ETagMap::new();
+
+    compile_sass(out_dir, &mut etag_map)?;
     write_known_targets(out_dir)?;
     compile_syntax(out_dir).context("could not compile syntax files")?;
+    calculate_static_etags(&mut etag_map)?;
+
+    let mut etag_file = File::create(out_dir.join("static_etag_map.rs"))?;
+    writeln!(
+        &mut etag_file,
+        "pub static STATIC_ETAG_MAP: ::phf::Map<&'static str, &'static str> = {};",
+        etag_map.build()
+    )?;
+    etag_file.sync_all()?;
 
     // trigger recompilation when a new migration is added
     println!("cargo:rerun-if-changed=migrations");
@@ -118,6 +132,16 @@ fn get_git_hash() -> Result<Option<String>> {
     }
 }
 
+fn etag_from_path(path: impl AsRef<Path>) -> Result<String> {
+    Ok(etag_from_content(std::fs::read(&path)?))
+}
+
+fn etag_from_content(content: impl AsRef<[u8]>) -> String {
+    let digest = md5::compute(content);
+    let md5_hex = format!("{:x}", digest);
+    format!(r#""\"{md5_hex}\"""#)
+}
+
 fn compile_sass_file(src: &Path, dest: &Path) -> Result<()> {
     let css = grass::from_path(
         src.to_str()
@@ -133,7 +157,7 @@ fn compile_sass_file(src: &Path, dest: &Path) -> Result<()> {
     Ok(())
 }
 
-fn compile_sass(out_dir: &Path) -> Result<()> {
+fn compile_sass(out_dir: &Path, etag_map: &mut ETagMap) -> Result<()> {
     const STYLE_DIR: &str = "templates/style";
 
     for entry in walkdir::WalkDir::new(STYLE_DIR) {
@@ -146,12 +170,13 @@ fn compile_sass(out_dir: &Path) -> Result<()> {
                 .to_str()
                 .context("file name must be a utf-8 string")?;
             if !file_name.starts_with('_') {
-                let dest = out_dir
-                    .join(entry.path().strip_prefix(STYLE_DIR)?)
-                    .with_extension("css");
+                let dest = out_dir.join(file_name).with_extension("css");
                 compile_sass_file(entry.path(), &dest).with_context(|| {
                     format!("compiling {} to {}", entry.path().display(), dest.display())
                 })?;
+
+                let dest_str = dest.file_name().unwrap().to_str().unwrap().to_owned();
+                etag_map.entry(dest_str, etag_from_path(&dest)?);
             }
         }
     }
@@ -160,7 +185,32 @@ fn compile_sass(out_dir: &Path) -> Result<()> {
     let pure = tracked::read_to_string("vendor/pure-css/css/pure-min.css")?;
     let grids = tracked::read_to_string("vendor/pure-css/css/grids-responsive-min.css")?;
     let vendored = pure + &grids;
-    std::fs::write(out_dir.join("vendored").with_extension("css"), vendored)?;
+    std::fs::write(out_dir.join("vendored").with_extension("css"), &vendored)?;
+
+    etag_map.entry(
+        "vendored.css".to_owned(),
+        etag_from_content(vendored.as_bytes()),
+    );
+
+    Ok(())
+}
+
+fn calculate_static_etags(etag_map: &mut ETagMap) -> Result<()> {
+    const STATIC_DIRS: &[&str] = &["static", "vendor"];
+
+    for static_dir in STATIC_DIRS {
+        for entry in walkdir::WalkDir::new(static_dir) {
+            let entry = entry?;
+            let path = entry.path();
+            if !path.is_file() {
+                continue;
+            }
+
+            let partial_path = path.strip_prefix(static_dir).unwrap();
+            let partial_path_str = partial_path.to_string_lossy().to_string();
+            etag_map.entry(partial_path_str, etag_from_path(path)?);
+        }
+    }
 
     Ok(())
 }

src/storage/database.rs

@@ -1,5 +1,5 @@
 use super::{Blob, FileRange, StorageMetrics, StreamingBlob};
-use crate::{InstanceMetrics, db::Pool, error::Result};
+use crate::{InstanceMetrics, db::Pool, error::Result, web::headers::compute_etag};
 use chrono::{DateTime, Utc};
 use futures_util::stream::{Stream, TryStreamExt};
 use sqlx::Acquire;
@@ -123,13 +123,16 @@ impl DatabaseBackend {
         });
         let content = result.content.unwrap_or_default();
         let content_len = content.len();
+
+        let etag = compute_etag(&content);
         Ok(StreamingBlob {
             path: result.path,
             mime: result
                 .mime
                 .parse()
                 .unwrap_or(mime::APPLICATION_OCTET_STREAM),
             date_updated: result.date_updated,
+            etag: Some(etag),
             content: Box::new(io::Cursor::new(content)),
             content_length: content_len,
             compression,