toxblh/podcasts
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add tokio features and remove lazy keyword

Browse Source
This commit is contained in:
Julian Hofer
2020-03-18 01:09:11 +00:00
committed by Jordan Petridis
parent e830589e38
commit 636e2aefde
5 changed files with 50 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
podcasts-data/Cargo.toml
+4 -1
View File
@@ -19,7 +19,6 @@ xml-rs = "0.8.0"
futures = "0.3.4"
hyper = "0.13.2"
http = "0.2.0"
tokio = "0.2.11"
hyper-tls = "0.4.1"
native-tls = "0.2.3"
num_cpus = "1.10.1"
@@ -35,6 +34,10 @@ version = "1.4.3"
features = ["sqlite"]
version = "1.4.0"
[dependencies.tokio]
features = ["rt-core", "rt-threaded"]
version = "0.2.13"
[dev-dependencies]
rand = "0.7.2"
tempdir = "0.3.7"
podcasts-data/src/feed.rs
+20 -20
View File
@@ -45,7 +45,7 @@ pub struct Feed {
impl Feed {
/// Index the contents of the RSS `Feed` into the database.
pub async fn index(self) -> Result<(), DataError>{
pub async fn index(self) -> Result<(), DataError> {
let show = self.parse_podcast().to_podcast()?;
self.index_channel_items(show).await
}
@@ -58,16 +58,14 @@ impl Feed {
let stream = stream::iter(self.channel.into_items());
// Parse the episodes
let episodes = stream.filter_map(move |item| {
let ret = NewEpisodeMinimal::new(&item, pd.id())
let ret = NewEpisodeMinimal::new(&item, pd.id())
.and_then(move |ep| determine_ep_state(ep, &item));
if ret.is_ok() {
future::ready(Some(ret))
}
else {
} else {
future::ready(None)
}
});
}
});
// Filter errors, Index updatable episodes, return insertables.
filter_episodes(episodes)
// Batch index insertable episodes.
@@ -103,23 +101,25 @@ where
S: Stream<Item = Result<IndexState<NewEpisode>, DataError>> + Send + 'a,
{
stream
.try_filter_map(|state| async move {
match state {
IndexState::NotChanged => Ok(None),
// Update individual rows, and filter them
IndexState::Update((ref ep, rowid)) => {
ep.update(rowid)
.map_err(|err| error!("{}", err))
.map_err(|_| error!("Failed to index episode: {:?}.", ep.title()))
.ok();
.try_filter_map(|state| {
async move {
match state {
IndexState::NotChanged => Ok(None),
// Update individual rows, and filter them
IndexState::Update((ref ep, rowid)) => {
ep.update(rowid)
.map_err(|err| error!("{}", err))
.map_err(|_| error!("Failed to index episode: {:?}.", ep.title()))
.ok();
Ok(None)
Ok(None)
}
IndexState::Index(s) => Ok(Some(s)),
}
IndexState::Index(s) => Ok(Some(s)),
}
})
// only Index is left, collect them for batch index
.try_collect()
.try_collect()
}
fn batch_insert_episodes(episodes: &[NewEpisode]) {
@@ -146,9 +146,9 @@ fn batch_insert_episodes(episodes: &[NewEpisode]) {
#[cfg(test)]
mod tests {
use failure::Error;
use futures::executor::block_on;
use rss::Channel;
use tokio;
use futures::executor::block_on;
use crate::database::truncate_db;
use crate::dbqueries;
podcasts-data/src/models/source.rs
+12 -13
View File
@@ -159,7 +159,7 @@ impl Source {
let code = res.status();
if code.is_success() {
// If request is succesful save the etag
// If request is successful save the etag
self = self.update_etag(&res)?
} else {
match code.as_u16() {
@@ -189,7 +189,7 @@ impl Source {
return Err(DataError::FeedRedirect(self));
}
401 => return Err(self.make_err("401: Unauthorized.", code)),
403 => return Err(self.make_err("403: Forbidden.", code)),
403 => return Err(self.make_err("403: Forbidden.", code)),
404 => return Err(self.make_err("404: Not found.", code)),
408 => return Err(self.make_err("408: Request Timeout.", code)),
410 => return Err(self.make_err("410: Feed was deleted..", code)),
@@ -240,23 +240,24 @@ impl Source {
self,
client: Client<HttpsConnector<HttpConnector>>,
) -> Result<Feed, DataError> {
let id = self.id();
let resp = self.get_response(&client).await?;
let chan = response_to_channel(resp).await?;
FeedBuilder::default()
.channel(chan)
.source_id(id)
.build()
.map_err(From::from)
.map_err(From::from)
}
async fn get_response(self, client: &Client<HttpsConnector<HttpConnector>>) -> Result<Response<Body>, DataError> {
async fn get_response(
self,
client: &Client<HttpsConnector<HttpConnector>>,
) -> Result<Response<Body>, DataError> {
let mut source = self;
loop
{
loop {
match source.request_constructor(&client.clone()).await {
Ok(response) => return Ok(response),
Err(err) => match err {
@@ -267,7 +268,7 @@ impl Source {
e => return Err(e),
},
}
};
}
}
async fn request_constructor(
@@ -305,14 +306,12 @@ impl Source {
}
let res = client.request(req).await?;
//.map_err(From::from)
//.map_err(From::from)
self.match_status(res)
}
}
async fn response_to_channel(
res: Response<Body>,
) -> Result<Channel, DataError> {
async fn response_to_channel(res: Response<Body>) -> Result<Channel, DataError> {
let chunk = hyper::body::to_bytes(res.into_body()).await?;
let buf = String::from_utf8_lossy(&chunk).into_owned();
Channel::from_str(&buf).map_err(From::from)
podcasts-data/src/pipeline.rs
+5 -5
View File
@@ -20,7 +20,7 @@
// FIXME:
//! Docs.
use futures::{future::ok, future::lazy, prelude::*, stream::FuturesUnordered};
use futures::{future::ok, prelude::*, stream::FuturesUnordered};
use tokio;
use hyper::client::HttpConnector;
@@ -42,18 +42,18 @@ type HttpsClient = Client<HttpsConnector<HttpConnector>>;
/// Convert `rss::Channel` into `Feed` -> Index Podcast -> Index Episodes.
pub async fn pipeline<'a, S>(mut sources: S, client: HttpsClient)
where
S: Stream<Item = Result<Source, DataError>> + Send + 'a + std::marker::Unpin
S: Stream<Item = Result<Source, DataError>> + Send + 'a + std::marker::Unpin,
{
while let Some(source_result) = sources.next().await {
if let Ok(source) = source_result {
match source.into_feed(client.clone()).await {
Ok(feed) => {
let fut = lazy(|_| feed.index().map_err(|err| error!("Error: {}", err)));
let fut = feed.index().map_err(|err| error!("Error: {}", err));
tokio::spawn(fut);
},
}
// Avoid spamming the stderr when it's not an actual error
Err(DataError::FeedNotModified(_)) => (),
Err(err) => error!("Error: {}", err),
Err(err) => error!("Error: {}", err),
};
}
}