Encode upload loop into S3 migration command

This loop will halt when all files are uploaded to S3, avoiding needing
to write something in bash / externally.

Author: Mark-Simulacrum

src/bin/cratesfyi.rs

@@ -242,7 +242,16 @@ pub fn main() {
             db::update_search_index(&conn).expect("Failed to update search index");
         } else if let Some(_) = matches.subcommand_matches("move-to-s3") {
             let conn = db::connect_db().unwrap();
-            db::file::move_to_s3(&conn, 5_000).expect("Failed to update search index");
+            let mut count = 1;
+            let mut total = 0;
+            while count != 0 {
+                count = db::file::move_to_s3(&conn, 5_000).expect("Failed to upload batch to S3");
+                total += count;
+                eprintln!(
+                    "moved {} rows to s3 in this batch, total moved so far: {}",
+                    count, total
+                );
+            }
         }
     } else if let Some(matches) = matches.subcommand_matches("start-web-server") {
         start_web_server(Some(matches.value_of("SOCKET_ADDR").unwrap_or("0.0.0.0:3000")));

src/db/file.rs

@@ -224,13 +224,14 @@ fn file_list_to_json(file_list: Vec<(String, PathBuf)>) -> Result<Json> {
     Ok(file_list_json.to_json())
 }
 
-pub fn move_to_s3(conn: &Connection, n: usize) -> Result<()> {
+pub fn move_to_s3(conn: &Connection, n: usize) -> Result<usize> {
     let trans = try!(conn.transaction());
     let client = s3_client().expect("configured s3");
 
     let rows = try!(trans.query(
             &format!("SELECT path, mime, content FROM files WHERE content != E'in-s3' LIMIT {}", n),
             &[]));
+    let count = rows.len();
 
     let mut rt = ::tokio::runtime::current_thread::Runtime::new().unwrap();
     let mut futures = Vec::new();
@@ -268,7 +269,7 @@ pub fn move_to_s3(conn: &Connection, n: usize) -> Result<()> {
 
     try!(trans.commit());
 
-    Ok(())
+    Ok(count)
 }
 
 #[cfg(test)]