Pipeline: Dispatch feed indexing to the rayon threadpool.

Cargo.lock

@@ -690,6 +690,7 @@ dependencies = [
  "pretty_assertions 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "rayon 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rayon-futures 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rfc822_sanitizer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "rss 1.4.0 (git+https://github.com/alatiera/rss)",
  "string_cache 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1407,6 +1408,15 @@ dependencies = [
  "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "rayon-futures"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rayon-core 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.1.37"
@@ -2263,6 +2273,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5"
 "checksum rayon 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "80e811e76f1dbf68abf87a759083d34600017fc4e10b6bd5ad84a700f9dba4b1"
 "checksum rayon-core 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9d24ad214285a7729b174ed6d3bcfcb80177807f959d95fafd5bfc5c4f201ac8"
+"checksum rayon-futures 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea70dae42033c388536c7e9bdca084a14d75659c12e25fc105e7552464a87c4a"
 "checksum redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "0d92eecebad22b767915e4d529f89f28ee96dbbf5a4810d2b844373f136417fd"
 "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76"
 "checksum regex 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "aec3f58d903a7d2a9dc2bf0e41a746f4530e0cab6b615494e058f67a3ef947fb"

hammond-data/Cargo.toml

@@ -11,6 +11,7 @@ derive_builder = "0.5.1"
 lazy_static = "1.0.0"
 log = "0.4.1"
 rayon = "1.0.1"
+rayon-futures = "0.1.0"
 rfc822_sanitizer = "0.3.3"
 rss = "1.4.0"
 url = "1.7.0"

hammond-data/src/lib.rs

@@ -51,6 +51,7 @@ extern crate hyper_tls;
 extern crate native_tls;
 extern crate num_cpus;
 extern crate rayon;
+extern crate rayon_futures;
 extern crate rfc822_sanitizer;
 extern crate rss;
 extern crate tokio_core;

hammond-data/src/pipeline.rs

@@ -13,6 +13,8 @@ use hyper_tls::HttpsConnector;
 use tokio_core::reactor::Core;
 
 use num_cpus;
+use rayon;
+use rayon_futures::ScopeFutureExt;
 use rss;
 
 use dbqueries;
@@ -59,7 +61,7 @@ where
 {
     let pipeline = sources
         .and_then(clone!(client => move |s| s.into_feed(client.clone(), ignore_etags)))
-        .and_then(|feed| feed.index())
+        .and_then(|feed| rayon::scope(|s| s.spawn_future(feed.index())))
         // the stream will stop at the first error so
         // we ensure that everything will succeded regardless.
         .map_err(|err| error!("Error: {}", err))

org.gnome.Hammond.json

@@ -29,7 +29,8 @@
         ],
         "env" : {
             "CARGO_HOME" : "/run/build/Hammond/cargo",
-            "RUST_BACKTRACE" : "1"
+            "RUST_BACKTRACE" : "1",
+            "RUSTFLAGS" : "--cfg rayon_unstable"
         }
     },
     "modules" : [

scripts/cargo.sh

@@ -1,5 +1,6 @@
 #!/bin/sh
 
 export CARGO_HOME=$1/target/cargo-home
+export RUSTFLAGS=--cfg rayon_unstable
 
 cargo build --release -p hammond-gtk && cp $1/target/release/hammond-gtk $2