From c204a61ff7d52ef4e7a4bb7f5324ba29dc6fc323 Mon Sep 17 00:00:00 2001 From: Jordan Petridis Date: Mon, 18 Sep 2017 15:44:08 +0300 Subject: [PATCH] Even more schema changes. --- .../2017-09-15-001128_init_schema/down.sql | 5 +- .../2017-09-15-001128_init_schema/up.sql | 16 +++-- src/cli.rs | 2 + src/dbqueries.rs | 16 +++++ src/lib.rs | 6 ++ src/models.rs | 68 +++++++++++++++++-- src/parse_feeds.rs | 39 ++++------- src/schema.rs | 14 +++- 8 files changed, 125 insertions(+), 41 deletions(-) create mode 100644 src/dbqueries.rs diff --git a/migrations/2017-09-15-001128_init_schema/down.sql b/migrations/2017-09-15-001128_init_schema/down.sql index ad06509..835dfe5 100644 --- a/migrations/2017-09-15-001128_init_schema/down.sql +++ b/migrations/2017-09-15-001128_init_schema/down.sql @@ -1,2 +1,3 @@ -Drop Table Episode; -Drop Table Podcast; \ No newline at end of file +Drop Table episode; +Drop Table podcast; +Drop Table source; \ No newline at end of file diff --git a/migrations/2017-09-15-001128_init_schema/up.sql b/migrations/2017-09-15-001128_init_schema/up.sql index c0c95ba..abe3a77 100644 --- a/migrations/2017-09-15-001128_init_schema/up.sql +++ b/migrations/2017-09-15-001128_init_schema/up.sql @@ -1,3 +1,10 @@ +CREATE TABLE `source` ( + `id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, + `url` TEXT NOT NULL UNIQUE, + `last_modified` TEXT, + `http_etag` TEXT +); + CREATE TABLE `episode` ( `id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, `title` TEXT NOT NULL, @@ -6,7 +13,7 @@ CREATE TABLE `episode` ( `description` TEXT, `published_date` TEXT NOT NULL, `epoch` INTEGER NOT NULL DEFAULT 0, - `length` INTEGER NOT NULL DEFAULT 0, + `length` INTEGER DEFAULT 0, `guid` TEXT, `podcast_id` INTEGER NOT NULL ); @@ -14,10 +21,9 @@ CREATE TABLE `episode` ( CREATE TABLE `podcast` ( `id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, `title` TEXT NOT NULL, - `uri` TEXT NOT NULL, + `uri` TEXT UNIQUE NOT NULL, `link` TEXT, `description` TEXT, - `last_modified` TEXT, - `http_etag` TEXT, - `image_uri` TEXT + `image_uri` TEXT, + `source_id` INTEGER NOT NULL ); \ No newline at end of file diff --git a/src/cli.rs b/src/cli.rs index e7ebfb2..24aa948 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,6 +1,7 @@ use structopt::StructOpt; use loggerv; use errors::*; +use models::NewPodcast; #[derive(StructOpt, Debug)] #[structopt(name = "example", about = "An example of StructOpt usage.")] @@ -20,6 +21,7 @@ pub fn run() -> Result<()> { ::init()?; // ::parse_feeds::foo(); + // ::index_feed::foo(); Ok(()) } \ No newline at end of file diff --git a/src/dbqueries.rs b/src/dbqueries.rs new file mode 100644 index 0000000..b998fde --- /dev/null +++ b/src/dbqueries.rs @@ -0,0 +1,16 @@ +use diesel::prelude::*; +use schema::podcast::dsl::*; +// use schema::episode::dsl::*; +use models::{Podcast, Episode}; + +pub fn get_podcasts(con: &SqliteConnection) -> QueryResult> { + let pds = podcast.load::(con); + // debug!("Returned Podcasts:\n{:?}", pds); + pds +} + +pub fn get_pd_episodes(con: &SqliteConnection, parent: &Podcast) -> QueryResult> { + let eps = Episode::belonging_to(parent).load::(con); + eps +} + \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 353bf8a..4811e4b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -27,12 +27,16 @@ pub mod cli; pub mod schema; pub mod models; pub mod parse_feeds; +pub mod index_feed; +pub mod dbqueries; pub mod errors { use reqwest; use std::io; + use rss; use diesel::migrations::RunMigrationsError; + use diesel::result; error_chain! { foreign_links { @@ -40,6 +44,8 @@ pub mod errors { IoError(io::Error); Log(::log::SetLoggerError); MigrationError(RunMigrationsError); + RSSError(rss::Error); + DieselResultError(result::Error); } } } diff --git a/src/models.rs b/src/models.rs index b421821..28d5c27 100644 --- a/src/models.rs +++ b/src/models.rs @@ -1,4 +1,8 @@ -use schema::{episode, podcast}; +use reqwest; +use rss::Channel; + +use schema::{episode, podcast, source}; +use errors::*; #[derive(Queryable, Identifiable)] #[derive(Associations)] @@ -12,14 +16,15 @@ pub struct Episode { local_uri: Option, description: Option, published_date: String, - thumbnail: Option, + epoch: i32, length: Option, guid: Option, - epoch: i32, podcast_id: i32, } #[derive(Queryable, Identifiable)] +#[derive(Associations)] +#[belongs_to(Source, foreign_key = "source_id")] #[table_name = "podcast"] #[derive(Debug, Clone)] pub struct Podcast { @@ -28,9 +33,18 @@ pub struct Podcast { uri: String, link: Option, description: Option, + image_uri: Option, + source_id: i32, +} + +#[derive(Queryable, Identifiable)] +#[table_name = "source"] +#[derive(Debug, Clone)] +pub struct Source { + id: i32, + uri: String, last_modified: Option, http_etag: Option, - image_uri: Option, } @@ -57,7 +71,49 @@ pub struct NewPodcast<'a> { pub uri: &'a str, pub link: Option<&'a str>, pub description: Option<&'a str>, - pub last_modified: Option<&'a str>, - pub http_etag: Option<&'a str>, pub image_uri: Option<&'a str>, +} + + +impl<'a> NewPodcast<'a> { + // fn from_url(uri: &'a str) -> Result { + // let chan = Channel::from_url(uri)?; + // let foo = ::parse_feeds::parse_podcast(&chan, uri)?; + // Ok(foo) + // } + + // Ignore this atm + // pub fn from_url(uri: &str) -> Result<()> { + + // use std::io::Read; + // use reqwest::header::*; + // use std::str::FromStr; + // use parse_feeds; + + // let mut req = reqwest::get(uri)?; + + // let mut buf = String::new(); + // req.read_to_string(&mut buf)?; + // info!("{}", buf); + + // let headers = req.headers(); + // info!("{:#?}", headers); + + // // for h in headers.iter() { + // // info!("{}: {}", h.name(), h.value_string()); + // // } + + // // Sometimes dsnt work + // // let etag = headers.get::(); + // let etag = headers.get_raw("ETag").unwrap(); + // let lst_mod = headers.get::().unwrap(); + // info!("Etag: {:?}", etag); + // info!("Last mod: {}", lst_mod); + + // let pd_chan = Channel::from_str(buf.as_str())?; + // // let bar = parse_feeds::parse_podcast(&foo)?; + // // let baz = bar.clone(); + + // Ok(()) + // } } \ No newline at end of file diff --git a/src/parse_feeds.rs b/src/parse_feeds.rs index 2122308..6e16e23 100644 --- a/src/parse_feeds.rs +++ b/src/parse_feeds.rs @@ -1,15 +1,10 @@ use rss::{Channel, Item}; use models; +use errors::*; -pub fn parse_podcast<'a>(pd_chan: &'a Channel, uri: &'a str) -> models::NewPodcast<'a> { +pub fn parse_podcast<'a>(pd_chan: &'a Channel, uri: &'a str) -> Result> { let title = pd_chan.title(); - // need to get it from reqwest probably - // I dont think uri can be consinstantly infered from the Channel - // TODO: Add etag support - let last_modified = None; - let http_etag = None; - let link = Some(pd_chan.link()); let description = Some(pd_chan.description()); @@ -18,18 +13,17 @@ pub fn parse_podcast<'a>(pd_chan: &'a Channel, uri: &'a str) -> models::NewPodca None => None, }; - models::NewPodcast { + let foo = models::NewPodcast { title, uri, link, description, - last_modified, - http_etag, image_uri, - } + }; + Ok(foo) } -pub fn parse_episode<'a>(item: &'a Item, parent_id: i32) -> models::NewEpisode<'a> { +pub fn parse_episode<'a>(item: &'a Item, parent_id: i32) -> Result> { let title = item.title().unwrap(); @@ -50,7 +44,7 @@ pub fn parse_episode<'a>(item: &'a Item, parent_id: i32) -> models::NewEpisode<' let length = Some(item.enclosure().unwrap().length().parse().unwrap()); - models::NewEpisode { + let foo = models::NewEpisode { title, uri, local_uri, @@ -60,7 +54,8 @@ pub fn parse_episode<'a>(item: &'a Item, parent_id: i32) -> models::NewEpisode<' epoch, guid, podcast_id: parent_id, - } + }; + Ok(foo) } @@ -81,7 +76,7 @@ mod tests { // println!("{:#?}", channel); let descr = "The people behind The Intercept’s fearless reporting and incisive commentary—Jeremy Scahill, Glenn Greenwald, Betsy Reed and others—discuss the crucial issues of our time: national security, civil liberties, foreign policy, and criminal justice. Plus interviews with artists, thinkers, and newsmakers who challenge our preconceptions about the world we live in."; - let pd = parse_podcast(&channel, uri); + let pd = parse_podcast(&channel, uri).unwrap(); assert_eq!(pd.title, "Intercepted with Jeremy Scahill"); // assert_eq!( @@ -90,8 +85,6 @@ mod tests { // ); assert_eq!(pd.link, Some("https://theintercept.com/podcasts")); assert_eq!(pd.description, Some(descr)); - assert_eq!(pd.last_modified, None); - assert_eq!(pd.http_etag, None); assert_eq!(pd.image_uri, None); @@ -102,13 +95,11 @@ mod tests { // println!("{:#?}", channel); let descr = "An open show powered by community LINUX Unplugged takes the best attributes of open collaboration and focuses them into a weekly lifestyle show about Linux."; - let pd = parse_podcast(&channel, uri); + let pd = parse_podcast(&channel, uri).unwrap(); assert_eq!(pd.title, "LINUX Unplugged Podcast"); assert_eq!(pd.link, Some("http://www.jupiterbroadcasting.com/")); assert_eq!(pd.description, Some(descr)); - assert_eq!(pd.last_modified, None); - assert_eq!(pd.http_etag, None); assert_eq!( pd.image_uri, Some("http://michaeltunnell.com/images/linux-unplugged.jpg") @@ -122,13 +113,11 @@ mod tests { // println!("{:#?}", channel); let descr = "Latest Articles and Investigations from ProPublica, an independent, non-profit newsroom that produces investigative journalism in the public interest."; - let pd = parse_podcast(&channel, uri); + let pd = parse_podcast(&channel, uri).unwrap(); assert_eq!(pd.title, "Articles and Investigations - ProPublica"); assert_eq!(pd.link, Some("https://www.propublica.org/feeds/54Ghome")); assert_eq!(pd.description, Some(descr)); - assert_eq!(pd.last_modified, None); - assert_eq!(pd.http_etag, None); assert_eq!( pd.image_uri, Some("https://assets.propublica.org/propublica-rss-logo.png") @@ -143,7 +132,7 @@ mod tests { let descr = "NSA whistleblower Edward Snowden discusses the massive Equifax data breach and allegations of Russian interference in the US election. Commentator Shaun King explains his call for a boycott of the NFL and talks about his campaign to bring violent neo-Nazis to justice. Rapper Open Mike Eagle performs."; // println!("{:#?}", firstitem); - let it = parse_episode(&firstitem, 0); + let it = parse_episode(&firstitem, 0).unwrap(); assert_eq!(it.title, "The Super Bowl of Racism"); assert_eq!(it.uri, "http://traffic.megaphone.fm/PPY6458293736.mp3"); @@ -154,7 +143,7 @@ mod tests { let second = channel.items().iter().nth(1).unwrap(); // println!("{:#?}", second); - let i2 = parse_episode(&second, 0); + let i2 = parse_episode(&second, 0).unwrap(); let descr = "This week on Intercepted: Jeremy gives an update on the aftermath of Blackwater’s 2007 massacre of Iraqi civilians. Intercept reporter Lee Fang lays out how a network of libertarian think tanks called the Atlas Network is insidiously shaping political infrastructure in Latin America. We speak with attorney and former Hugo Chavez adviser Eva Golinger about the Venezuela\'s political turmoil.And we hear Claudia Lizardo of the Caracas-based band, La Pequeña Revancha, talk about her music and hopes for Venezuela."; assert_eq!( diff --git a/src/schema.rs b/src/schema.rs index 97bc52d..a91027e 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -7,7 +7,7 @@ table! { description -> Nullable, published_date -> Text, epoch -> Integer, - length -> Integer, + length -> Nullable, guid -> Nullable, podcast_id -> Integer, } @@ -20,8 +20,16 @@ table! { uri -> Text, link -> Nullable, description -> Nullable, - last_modified -> Nullable, - http_etag -> Nullable, image_uri -> Nullable, + source_id -> Integer, + } +} + +table! { + source (id) { + id -> Integer, + url -> Text, + last_modified -> Nullable, + http_etag -> Nullable, } }