toxblh/podcasts
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Pipeline: Complete the move to Tokio Runtime

Browse Source
This commit is contained in:
Jordan Petridis
2018-09-29 13:35:45 +03:00
parent 3c4574f2ec
commit 0887789f5e
7 changed files with 26 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
podcasts-data/Cargo.toml
+1 -3
View File
@@ -19,9 +19,7 @@ xml-rs = "0.8.0"
futures = "0.1.23"
hyper = "0.12.11"
http = "0.1.13"
tokio-core = "0.1.17"
tokio-threadpool = "0.1.7"
tokio-executor = "0.1.5"
tokio = "0.1.11"
hyper-tls = "0.3.0"
native-tls = "0.2.1"
num_cpus = "1.8.0"
podcasts-data/src/errors.rs
-9
View File
@@ -5,7 +5,6 @@ use http;
use hyper;
use native_tls;
use rss;
use tokio_executor;
use url;
use xml;
@@ -50,8 +49,6 @@ pub enum DataError {
HyperError(#[cause] hyper::Error),
#[fail(display = "ToStr Error: {}", _0)]
HttpToStr(#[cause] http::header::ToStrError),
#[fail(display = "Tokio Spawn Error: {}", _0)]
SpawnError(#[cause] tokio_executor::SpawnError),
#[fail(display = "Failed to parse a url: {}", _0)]
UrlError(#[cause] url::ParseError),
#[fail(display = "TLS Error: {}", _0)]
@@ -115,12 +112,6 @@ impl From<http::header::ToStrError> for DataError {
}
}
impl From<tokio_executor::SpawnError> for DataError {
fn from(err: tokio_executor::SpawnError) -> Self {
DataError::SpawnError(err)
}
}
impl From<url::ParseError> for DataError {
fn from(err: url::ParseError) -> Self {
DataError::UrlError(err)
podcasts-data/src/feed.rs
+3 -4
View File
@@ -122,7 +122,7 @@ fn batch_insert_episodes(episodes: &[NewEpisode]) {
mod tests {
use failure::Error;
use rss::Channel;
use tokio_core::reactor::Core;
use tokio::{self, prelude::*};
use database::truncate_db;
use dbqueries;
@@ -176,10 +176,9 @@ mod tests {
get_feed(path, s.id())
}).collect();
let mut core = Core::new()?;
// Index the channes
let list: Vec<_> = feeds.into_iter().map(|x| x.index()).collect();
let _foo = core.run(join_all(list));
let stream_ = stream::iter_ok(feeds).for_each(|x| x.index());
tokio::run(stream_.map_err(|_| ()));
// Assert the index rows equal the controlled results
assert_eq!(dbqueries::get_sources()?.len(), 5);
podcasts-data/src/lib.rs
+1 -3
View File
@@ -84,9 +84,7 @@ extern crate num_cpus;
extern crate rayon;
extern crate rfc822_sanitizer;
extern crate rss;
extern crate tokio_core;
extern crate tokio_executor;
extern crate tokio_threadpool;
extern crate tokio;
extern crate url;
extern crate xdg;
extern crate xml;
podcasts-data/src/models/source.rs
+3 -4
View File
@@ -286,8 +286,8 @@ fn response_to_channel(res: Response<Body>) -> impl Future<Item = Channel, Error
mod tests {
use super::*;
use failure::Error;
use tokio_core::reactor::Core;
use num_cpus;
use tokio;
use database::truncate_db;
use utils::get_feed;
@@ -296,7 +296,7 @@ mod tests {
fn test_into_feed() -> Result<(), Error> {
truncate_db()?;
let mut core = Core::new()?;
let mut rt = tokio::runtime::Runtime::new()?;
let https = HttpsConnector::new(num_cpus::get())?;
let client = Client::builder().build::<_, Body>(https);
@@ -304,9 +304,8 @@ mod tests {
com/InterceptedWithJeremyScahill";
let source = Source::from_url(url)?;
let id = source.id();
let feed = source.into_feed(client);
let feed = core.run(feed)?;
let feed = rt.block_on(feed)?;
let expected = get_feed("tests/feeds/2018-01-20-Intercepted.xml", id);
assert_eq!(expected, feed);
podcasts-data/src/pipeline.rs
+9 -12
View File
@@ -1,9 +1,8 @@
// FIXME:
//! Docs.
use futures::{lazy, prelude::*, stream::iter_ok};
use tokio_core::reactor::Core;
use tokio_threadpool::{self, ThreadPool};
use futures::{lazy, prelude::*, future::ok, stream::FuturesOrdered};
use tokio;
use hyper::client::HttpConnector;
use hyper::{Client, Body};
@@ -14,6 +13,8 @@ use num_cpus;
use errors::DataError;
use Source;
use std::iter::FromIterator;
type HttpsClient = Client<HttpsConnector<HttpConnector>>;
/// The pipline to be run for indexing and updating a Podcast feed that originates from
@@ -25,7 +26,6 @@ type HttpsClient = Client<HttpsConnector<HttpConnector>>;
pub fn pipeline<'a, S>(
sources: S,
client: HttpsClient,
pool: tokio_threadpool::Sender,
) -> impl Future<Item = (), Error = ()> + 'a
where
S: Stream<Item = Source, Error = DataError> + 'a,
@@ -41,7 +41,7 @@ where
})
.for_each(move |feed| {
let fut = lazy(|| feed.index().map_err(|err| error!("Error: {}", err)));
pool.spawn(fut).map_err(|_| ())
tokio::spawn(fut)
})
}
@@ -51,17 +51,14 @@ pub fn run<S>(sources: S) -> Result<(), DataError>
where
S: IntoIterator<Item = Source>,
{
let pool = ThreadPool::new();
let sender = pool.sender().clone();
let mut core = Core::new()?;
let https = HttpsConnector::new(num_cpus::get())?;
let client = Client::builder().build::<_, Body>(https);
let stream = iter_ok::<_, DataError>(sources);
let p = pipeline(stream, client, sender);
let _ = core.run(p);
let foo = sources.into_iter().map(ok::<_, _>);
let stream = FuturesOrdered::from_iter(foo);
let p = pipeline(stream, client);
tokio::run(p);
pool.shutdown_on_idle().wait().unwrap();
Ok(())
}