Pipeline: Dispatch feed indexing to the rayon threadpool.

hammond-data/src/lib.rs

@@ -51,6 +51,7 @@ extern crate hyper_tls;
 extern crate native_tls;
 extern crate num_cpus;
 extern crate rayon;
+extern crate rayon_futures;
 extern crate rfc822_sanitizer;
 extern crate rss;
 extern crate tokio_core;

hammond-data/src/pipeline.rs

@@ -13,6 +13,8 @@ use hyper_tls::HttpsConnector;
 use tokio_core::reactor::Core;
 
 use num_cpus;
+use rayon;
+use rayon_futures::ScopeFutureExt;
 use rss;
 
 use dbqueries;
@@ -59,7 +61,7 @@ where
 {
     let pipeline = sources
         .and_then(clone!(client => move |s| s.into_feed(client.clone(), ignore_etags)))
-        .and_then(|feed| feed.index())
+        .and_then(|feed| rayon::scope(|s| s.spawn_future(feed.index())))
         // the stream will stop at the first error so
         // we ensure that everything will succeded regardless.
         .map_err(|err| error!("Error: {}", err))