From bfb74c4dba9cc36b35ccc8d4a48d4b93fd728ea8 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Tue, 26 Dec 2017 18:18:48 +0200 Subject: [PATCH] hammond-data: Index_loop does not wait for GET request to finish now. --- hammond-data/benches/bench.rs | 4 +- hammond-data/src/feed.rs | 93 +++++++++------------------- hammond-downloader/src/downloader.rs | 2 +- hammond-gtk/src/utils.rs | 24 ++++--- 4 files changed, 43 insertions(+), 80 deletions(-) diff --git a/hammond-data/benches/bench.rs b/hammond-data/benches/bench.rs index 4a71a8e..d721878 100644 --- a/hammond-data/benches/bench.rs +++ b/hammond-data/benches/bench.rs @@ -13,7 +13,7 @@ use rayon::prelude::*; use test::Bencher; use hammond_data::Source; -use hammond_data::feed::{index, Feed}; +use hammond_data::feed::*; use std::io::BufReader; @@ -43,7 +43,7 @@ fn index_urls() { }) .collect(); - index(feeds); + feeds.par_iter().for_each(|x| index(x)); } #[bench] diff --git a/hammond-data/src/feed.rs b/hammond-data/src/feed.rs index af38f1a..f613f16 100644 --- a/hammond-data/src/feed.rs +++ b/hammond-data/src/feed.rs @@ -99,58 +99,42 @@ impl Feed { .collect(); Ok(episodes) - - // This would return every episode of the feed from the db. - // self.index_channel_items(&pd)?; - // Ok(dbqueries::get_pd_episodes(&pd)?) } } -/// Use's `fetch_all` to retrieve a list of `Feed`s and use index them using `feed::index`. -pub fn index_all() -> Result<()> { - let feeds = fetch_all()?; - - index(feeds); - Ok(()) +/// Handle the indexing of a `Feed` into the Database. +pub fn index(feed: &Feed) { + if let Err(err) = feed.index() { + error!("Error While trying to update the database."); + error!("Error msg: {}", err); + }; } -/// Handle the indexing of a feed `F` into the Database. -/// -/// Consume a `ParallelIterator` and index it. -pub fn index>(feeds: F) { - feeds.into_par_iter().for_each(|f| { - let e = f.index(); - if e.is_err() { - error!("Error While trying to update the database."); - error!("Error msg: {}", e.unwrap_err()); - }; - }); +/// Consume a `Source` and return a `Feed`. +fn fetch(source: Source) -> Result { + let uri = source.uri().to_owned(); + let feed = Feed::from_source(source); + if feed.is_err() { + error!("Error While trying to fetch from source url: {}.", uri); + } + feed +} + +/// Index a "list" of `Source`s. +pub fn index_loop>(sources: S) { + sources + .into_par_iter() + .filter_map(|x| fetch(x).ok()) + .for_each(|x| index(&x)); + info!("Indexing done."); } -/// Retrieve a list of all the `Source` in the database, -/// then use `feed::fetch` to convert them into `Feed`s -/// and return them. -pub fn fetch_all() -> Result> { - let feeds = dbqueries::get_sources()?; - Ok(fetch(feeds)) -} - -/// Consume a `ParallelIterator` and return a list of `Feed`s. -pub fn fetch>(feeds: F) -> Vec { - let results: Vec<_> = feeds - .into_par_iter() - .filter_map(|x| { - let uri = x.uri().to_owned(); - let feed = Feed::from_source(x).ok(); - if feed.is_none() { - error!("Error While trying to fetch from source url: {}.", uri); - } - feed - }) - .collect(); - - results +/// Retrieves all `Sources` from the database and updates/indexes them. +pub fn index_all() -> Result<()> { + let sources = dbqueries::get_sources()?; + index_loop(sources); + Ok(()) } #[cfg(test)] @@ -183,25 +167,6 @@ mod tests { index_all().unwrap(); } - #[test] - /// Insert feeds and update/index them. - fn test_fetch_loop() { - truncate_db().unwrap(); - let inpt = vec![ - "https://request-for-explanation.github.io/podcast/rss.xml", - "https://feeds.feedburner.com/InterceptedWithJeremyScahill", - "http://feeds.propublica.org/propublica/podcast", - "http://feeds.feedburner.com/linuxunplugged", - ]; - - inpt.iter().for_each(|url| { - // Index the urls into the source table. - Source::from_url(url).unwrap(); - }); - - fetch_all().unwrap(); - } - #[test] fn test_complete_index() { // vec of (path, url) tuples. @@ -240,7 +205,7 @@ mod tests { .collect(); // Index the channels - index(feeds); + feeds.par_iter().for_each(|x| index(&x)); // Assert the index rows equal the controlled results assert_eq!(dbqueries::get_sources().unwrap().len(), 4); diff --git a/hammond-downloader/src/downloader.rs b/hammond-downloader/src/downloader.rs index 685986f..9338637 100644 --- a/hammond-downloader/src/downloader.rs +++ b/hammond-downloader/src/downloader.rs @@ -212,7 +212,7 @@ mod tests { // Convert Source it into a Feed and index it let feed = source.into_feed().unwrap(); - index(vec![feed]); + index(&feed); // Get the Podcast let pd = dbqueries::get_podcast_from_source_id(sid).unwrap().into(); diff --git a/hammond-gtk/src/utils.rs b/hammond-gtk/src/utils.rs index b88a6dc..c03c835 100644 --- a/hammond-gtk/src/utils.rs +++ b/hammond-gtk/src/utils.rs @@ -34,20 +34,18 @@ pub fn refresh_feed(content: Rc, source: Option>) { })); thread::spawn(move || { - let feeds = { - if let Some(vec) = source { - Ok(feed::fetch(vec)) - } else { - feed::fetch_all() - } + if let Some(s) = source { + feed::index_loop(s); + } else { + let e = feed::index_all(); + if let Err(err) = e { + error!("Error While trying to update the database."); + error!("Error msg: {}", err); + }; }; - if let Ok(x) = feeds { - feed::index(x); - - sender.send(true).expect("Couldn't send data to channel");; - glib::idle_add(refresh_podcasts_view); - }; + sender.send(true).expect("Couldn't send data to channel");; + glib::idle_add(refresh_podcasts_view); }); } @@ -113,7 +111,7 @@ mod tests { // Convert Source it into a Feed and index it let feed = source.into_feed().unwrap(); - index(vec![feed]); + index(&feed); // Get the Podcast let pd = dbqueries::get_podcast_from_source_id(sid).unwrap();