Even more schema changes.

migrations/2017-09-15-001128_init_schema/down.sql

@@ -1,2 +1,3 @@
-Drop Table Episode;
+Drop Table episode;
-Drop Table Podcast;
+Drop Table podcast;
+Drop Table source;

migrations/2017-09-15-001128_init_schema/up.sql

@@ -1,3 +1,10 @@
+CREATE TABLE `source` (
+	`id`	INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
+	`url`	TEXT NOT NULL UNIQUE,
+	`last_modified`	TEXT,
+	`http_etag`	TEXT
+);
+
 CREATE TABLE `episode` (
 	`id`	INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
 	`title`	TEXT NOT NULL,
@@ -6,7 +13,7 @@ CREATE TABLE `episode` (
 	`description`	TEXT,
 	`published_date` TEXT NOT NULL,
 	`epoch`	INTEGER NOT NULL DEFAULT 0,
-	`length`	INTEGER NOT NULL DEFAULT 0,
+	`length`	INTEGER DEFAULT 0,
 	`guid`	TEXT,
 	`podcast_id`	INTEGER NOT NULL
 );
@@ -14,10 +21,9 @@ CREATE TABLE `episode` (
 CREATE TABLE `podcast` (
 	`id`	INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
 	`title`	TEXT NOT NULL,
-	`uri`	TEXT NOT NULL,
+	`uri`	TEXT UNIQUE NOT NULL,
 	`link`	TEXT,
 	`description`	TEXT,
-	`last_modified`	TEXT,
+	`image_uri`	TEXT,
-	`http_etag`	TEXT,
+	`source_id`	INTEGER NOT NULL
-	`image_uri`	TEXT
 );

src/cli.rs

@@ -1,6 +1,7 @@
 use structopt::StructOpt;
 use loggerv;
 use errors::*;
+use models::NewPodcast;
 
 #[derive(StructOpt, Debug)]
 #[structopt(name = "example", about = "An example of StructOpt usage.")]
@@ -20,6 +21,7 @@ pub fn run() -> Result<()> {
 
     ::init()?;
     // ::parse_feeds::foo();
+    // ::index_feed::foo();
 
     Ok(())
 }

src/dbqueries.rs

@@ -0,0 +1,16 @@
+use diesel::prelude::*;
+use schema::podcast::dsl::*;
+// use schema::episode::dsl::*;
+use models::{Podcast, Episode};
+
+pub fn get_podcasts(con: &SqliteConnection) -> QueryResult<Vec<Podcast>> {
+    let pds = podcast.load::<Podcast>(con);
+    // debug!("Returned Podcasts:\n{:?}", pds);
+    pds
+}
+
+pub fn get_pd_episodes(con: &SqliteConnection, parent: &Podcast) -> QueryResult<Vec<Episode>> {
+    let eps = Episode::belonging_to(parent).load::<Episode>(con);
+    eps
+}
+ 

src/lib.rs

@@ -27,12 +27,16 @@ pub mod cli;
 pub mod schema;
 pub mod models;
 pub mod parse_feeds;
+pub mod index_feed;
+pub mod dbqueries;
 
 pub mod errors {
 
     use reqwest;
     use std::io;
+    use rss;
     use diesel::migrations::RunMigrationsError;
+    use diesel::result;
 
     error_chain! {
         foreign_links {
@@ -40,6 +44,8 @@ pub mod errors {
             IoError(io::Error);
             Log(::log::SetLoggerError);
             MigrationError(RunMigrationsError);
+            RSSError(rss::Error);
+            DieselResultError(result::Error);
         }
     }
 }

src/models.rs

@@ -1,4 +1,8 @@
-use schema::{episode, podcast};
+use reqwest;
+use rss::Channel;
+
+use schema::{episode, podcast, source};
+use errors::*;
 
 #[derive(Queryable, Identifiable)]
 #[derive(Associations)]
@@ -12,14 +16,15 @@ pub struct Episode {
     local_uri: Option<String>,
     description: Option<String>,
     published_date: String,
-    thumbnail: Option<String>,
+    epoch: i32,
     length: Option<i32>,
     guid: Option<String>,
-    epoch: i32,
     podcast_id: i32,
 }
 
 #[derive(Queryable, Identifiable)]
+#[derive(Associations)]
+#[belongs_to(Source, foreign_key = "source_id")]
 #[table_name = "podcast"]
 #[derive(Debug, Clone)]
 pub struct Podcast {
@@ -28,9 +33,18 @@ pub struct Podcast {
     uri: String,
     link: Option<String>,
     description: Option<String>,
+    image_uri: Option<String>,
+    source_id: i32,
+}
+
+#[derive(Queryable, Identifiable)]
+#[table_name = "source"]
+#[derive(Debug, Clone)]
+pub struct Source {
+    id: i32,
+    uri: String,
     last_modified: Option<String>,
     http_etag: Option<String>,
-    image_uri: Option<String>,
 }
 
 
@@ -57,7 +71,49 @@ pub struct NewPodcast<'a> {
     pub uri: &'a str,
     pub link: Option<&'a str>,
     pub description: Option<&'a str>,
-    pub last_modified: Option<&'a str>,
-    pub http_etag: Option<&'a str>,
     pub image_uri: Option<&'a str>,
+}
+
+
+impl<'a> NewPodcast<'a> {
+    // fn from_url(uri: &'a str) -> Result<NewPodcast> {
+    //     let chan = Channel::from_url(uri)?;
+    //     let foo = ::parse_feeds::parse_podcast(&chan, uri)?;
+    //     Ok(foo)
+    // }
+
+    // Ignore this atm
+    // pub fn from_url(uri: &str) -> Result<()> {
+
+    //     use std::io::Read;
+    //     use reqwest::header::*;
+    //     use std::str::FromStr;
+    //     use parse_feeds;
+
+    //     let mut req = reqwest::get(uri)?;
+
+    //     let mut buf = String::new();
+    //     req.read_to_string(&mut buf)?;
+    //     info!("{}", buf);
+
+    //     let headers = req.headers();
+    //     info!("{:#?}", headers);
+
+    //     // for h in headers.iter() {
+    //     //     info!("{}: {}", h.name(), h.value_string());
+    //     // }
+
+    //     // Sometimes dsnt work
+    //     // let etag = headers.get::<ETag>();
+    //     let etag = headers.get_raw("ETag").unwrap();
+    //     let lst_mod = headers.get::<LastModified>().unwrap();
+    //     info!("Etag: {:?}", etag);
+    //     info!("Last mod: {}", lst_mod);
+
+    //     let pd_chan = Channel::from_str(buf.as_str())?;
+    //     // let bar = parse_feeds::parse_podcast(&foo)?;
+    //     // let baz = bar.clone();
+
+    //     Ok(())
+    // }
 }

src/parse_feeds.rs

@@ -1,15 +1,10 @@
 use rss::{Channel, Item};
 use models;
+use errors::*;
 
-pub fn parse_podcast<'a>(pd_chan: &'a Channel, uri: &'a str) -> models::NewPodcast<'a> {
+pub fn parse_podcast<'a>(pd_chan: &'a Channel, uri: &'a str) -> Result<models::NewPodcast<'a>> {
     let title = pd_chan.title();
 
-    // need to get it from reqwest probably
-    // I dont think uri can be consinstantly infered from the Channel
-    // TODO: Add etag support
-    let last_modified = None;
-    let http_etag = None;
-
     let link = Some(pd_chan.link());
     let description = Some(pd_chan.description());
 
@@ -18,18 +13,17 @@ pub fn parse_podcast<'a>(pd_chan: &'a Channel, uri: &'a str) -> models::NewPodca
         None => None,
     };
 
-    models::NewPodcast {
+    let foo = models::NewPodcast {
         title,
         uri,
         link,
         description,
-        last_modified,
-        http_etag,
         image_uri,
-    }
+    };
+    Ok(foo)
 }
 
-pub fn parse_episode<'a>(item: &'a Item, parent_id: i32) -> models::NewEpisode<'a> {
+pub fn parse_episode<'a>(item: &'a Item, parent_id: i32) -> Result<models::NewEpisode<'a>> {
 
     let title = item.title().unwrap();
 
@@ -50,7 +44,7 @@ pub fn parse_episode<'a>(item: &'a Item, parent_id: i32) -> models::NewEpisode<'
 
     let length = Some(item.enclosure().unwrap().length().parse().unwrap());
 
-    models::NewEpisode {
+    let foo = models::NewEpisode {
         title,
         uri,
         local_uri,
@@ -60,7 +54,8 @@ pub fn parse_episode<'a>(item: &'a Item, parent_id: i32) -> models::NewEpisode<'
         epoch,
         guid,
         podcast_id: parent_id,
-    }
+    };
+    Ok(foo)
 }
 
 
@@ -81,7 +76,7 @@ mod tests {
 
         // println!("{:#?}", channel);
         let descr = "The people behind The Intercept’s fearless reporting and incisive commentary—Jeremy Scahill, Glenn Greenwald, Betsy Reed and others—discuss the crucial issues of our time: national security, civil liberties, foreign policy, and criminal justice.  Plus interviews with artists, thinkers, and newsmakers who challenge our preconceptions about the world we live in.";
-        let pd = parse_podcast(&channel, uri);
+        let pd = parse_podcast(&channel, uri).unwrap();
 
         assert_eq!(pd.title, "Intercepted with Jeremy Scahill");
         // assert_eq!(
@@ -90,8 +85,6 @@ mod tests {
         // );
         assert_eq!(pd.link, Some("https://theintercept.com/podcasts"));
         assert_eq!(pd.description, Some(descr));
-        assert_eq!(pd.last_modified, None);
-        assert_eq!(pd.http_etag, None);
         assert_eq!(pd.image_uri, None);
 
 
@@ -102,13 +95,11 @@ mod tests {
 
         // println!("{:#?}", channel);
         let descr = "An open show powered by community LINUX Unplugged takes the best attributes of open collaboration and focuses them into a weekly lifestyle show about Linux.";
-        let pd = parse_podcast(&channel, uri);
+        let pd = parse_podcast(&channel, uri).unwrap();
 
         assert_eq!(pd.title, "LINUX Unplugged Podcast");
         assert_eq!(pd.link, Some("http://www.jupiterbroadcasting.com/"));
         assert_eq!(pd.description, Some(descr));
-        assert_eq!(pd.last_modified, None);
-        assert_eq!(pd.http_etag, None);
         assert_eq!(
             pd.image_uri,
             Some("http://michaeltunnell.com/images/linux-unplugged.jpg")
@@ -122,13 +113,11 @@ mod tests {
 
         // println!("{:#?}", channel);
         let descr = "Latest Articles and Investigations from ProPublica, an independent, non-profit newsroom that produces investigative journalism in the public interest.";
-        let pd = parse_podcast(&channel, uri);
+        let pd = parse_podcast(&channel, uri).unwrap();
 
         assert_eq!(pd.title, "Articles and Investigations - ProPublica");
         assert_eq!(pd.link, Some("https://www.propublica.org/feeds/54Ghome"));
         assert_eq!(pd.description, Some(descr));
-        assert_eq!(pd.last_modified, None);
-        assert_eq!(pd.http_etag, None);
         assert_eq!(
             pd.image_uri,
             Some("https://assets.propublica.org/propublica-rss-logo.png")
@@ -143,7 +132,7 @@ mod tests {
         let descr = "NSA whistleblower Edward Snowden discusses the massive Equifax data breach and allegations of Russian interference in the US election. Commentator Shaun King explains his call for a boycott of the NFL and talks about his campaign to bring violent neo-Nazis to justice. Rapper Open Mike Eagle performs.";
 
         // println!("{:#?}", firstitem);
-        let it = parse_episode(&firstitem, 0);
+        let it = parse_episode(&firstitem, 0).unwrap();
 
         assert_eq!(it.title, "The Super Bowl of Racism");
         assert_eq!(it.uri, "http://traffic.megaphone.fm/PPY6458293736.mp3");
@@ -154,7 +143,7 @@ mod tests {
 
         let second = channel.items().iter().nth(1).unwrap();
         // println!("{:#?}", second);
-        let i2 = parse_episode(&second, 0);
+        let i2 = parse_episode(&second, 0).unwrap();
 
         let descr = "This week on Intercepted: Jeremy gives an update on the aftermath of Blackwater’s 2007 massacre of Iraqi civilians. Intercept reporter Lee Fang lays out how a network of libertarian think tanks called the Atlas Network is insidiously shaping political infrastructure in Latin America. We speak with attorney and former Hugo Chavez adviser Eva Golinger about the Venezuela\'s political turmoil.And we hear Claudia Lizardo of the Caracas-based band, La Pequeña Revancha, talk about her music and hopes for Venezuela.";
         assert_eq!(

src/schema.rs

@@ -7,7 +7,7 @@ table! {
         description -> Nullable<Text>,
         published_date -> Text,
         epoch -> Integer,
-        length -> Integer,
+        length -> Nullable<Integer>,
         guid -> Nullable<Text>,
         podcast_id -> Integer,
     }
@@ -20,8 +20,16 @@ table! {
         uri -> Text,
         link -> Nullable<Text>,
         description -> Nullable<Text>,
-        last_modified -> Nullable<Text>,
-        http_etag -> Nullable<Text>,
         image_uri -> Nullable<Text>,
+        source_id -> Integer,
+    }
+}
+
+table! {
+    source (id) {
+        id -> Integer,
+        url -> Text,
+        last_modified -> Nullable<Text>,
+        http_etag -> Nullable<Text>,
     }
 }