toxblh/podcasts
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

NewEpisode: Move the parsing tests, delete old xml files.

Browse Source
This commit is contained in:
Jordan Petridis
2018-01-21 13:35:35 +02:00
parent 937bd61d5e
commit 843feabf25
8 changed files with 243 additions and 5377 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hammond-data/src/feed.rs
+33 -16
View File
@@ -116,10 +116,14 @@ mod tests {
fn test_index_loop() {
truncate_db().unwrap();
let inpt = vec![
"https://request-for-explanation.github.io/podcast/rss.xml",
"https://feeds.feedburner.com/InterceptedWithJeremyScahill",
"http://feeds.propublica.org/propublica/podcast",
"http://feeds.feedburner.com/linuxunplugged",
"https://web.archive.org/web/20180120083840if_/https://feeds.feedburner.\
com/InterceptedWithJeremyScahill",
"https://web.archive.org/web/20180120110314if_/https://feeds.feedburner.\
com/linuxunplugged",
"https://web.archive.org/web/20180120110727if_/https://rss.acast.com/thetipoff",
"https://web.archive.org/web/20180120104957if_/https://rss.art19.com/steal-the-stars",
"https://web.archive.org/web/20180120104741if_/https://www.greaterthancode.\
com/feed/podcast",
];
inpt.iter().for_each(|url| {
@@ -131,7 +135,12 @@ mod tests {
let sources = dbqueries::get_sources().unwrap();
// Run again to cover Unique constrains erros.
pipeline::pipeline(sources, true).unwrap()
pipeline::pipeline(sources, true).unwrap();
// Assert the index rows equal the controlled results
assert_eq!(dbqueries::get_sources().unwrap().len(), 5);
assert_eq!(dbqueries::get_podcasts().unwrap().len(), 5);
assert_eq!(dbqueries::get_episodes().unwrap().len(), 354);
}
#[test]
@@ -139,20 +148,28 @@ mod tests {
// vec of (path, url) tuples.
let urls = vec![
(
"tests/feeds/Intercepted.xml",
"https://feeds.feedburner.com/InterceptedWithJeremyScahill",
"tests/feeds/2018-01-20-Intercepted.xml",
"https://web.archive.org/web/20180120083840if_/https://feeds.feedburner.\
com/InterceptedWithJeremyScahill",
),
(
"tests/feeds/LinuxUnplugged.xml",
"http://feeds.feedburner.com/linuxunplugged",
"tests/feeds/2018-01-20-LinuxUnplugged.xml",
"https://web.archive.org/web/20180120110314if_/https://feeds.feedburner.\
com/linuxunplugged",
),
(
"tests/feeds/TheBreakthrough.xml",
"http://feeds.propublica.org/propublica/podcast",
"tests/feeds/2018-01-20-TheTipOff.xml",
"https://web.archive.org/web/20180120110727if_/https://rss.acast.com/thetipoff",
),
(
"tests/feeds/R4Explanation.xml",
"https://request-for-explanation.github.io/podcast/rss.xml",
"tests/feeds/2018-01-20-StealTheStars.xml",
"https://web.archive.org/web/20180120104957if_/https://rss.art19.\
com/steal-the-stars",
),
(
"tests/feeds/2018-01-20-GreaterThanCode.xml",
"https://web.archive.org/web/20180120104741if_/https://www.greaterthancode.\
com/feed/podcast",
),
];
@@ -181,8 +198,8 @@ mod tests {
let _foo = core.run(join_all(list));
// Assert the index rows equal the controlled results
assert_eq!(dbqueries::get_sources().unwrap().len(), 4);
assert_eq!(dbqueries::get_podcasts().unwrap().len(), 4);
assert_eq!(dbqueries::get_episodes().unwrap().len(), 274);
assert_eq!(dbqueries::get_sources().unwrap().len(), 5);
assert_eq!(dbqueries::get_podcasts().unwrap().len(), 5);
assert_eq!(dbqueries::get_episodes().unwrap().len(), 354);
}
}
hammond-data/src/models/new_episode.rs
+207 -1
View File
@@ -153,7 +153,7 @@ impl NewEpisode {
#[derive(Insertable, AsChangeset)]
#[table_name = "episode"]
#[derive(Debug, Clone, Default, Builder, PartialEq)]
#[derive(Debug, Clone, Builder, PartialEq)]
#[builder(derive(Debug))]
#[builder(setter(into))]
pub(crate) struct NewEpisodeMinimal {
@@ -262,3 +262,209 @@ impl NewEpisodeMinimal {
self.podcast_id
}
}
#[cfg(test)]
mod tests {
use models::{NewEpisode, NewEpisodeBuilder};
use models::new_episode::{NewEpisodeMinimal, NewEpisodeMinimalBuilder};
use rss::Channel;
use std::fs::File;
use std::io::BufReader;
// TODO: Add tests for other feeds too.
// Especially if you find an *intresting* generated feed.
#[test]
fn test_new_episode_minimal_intercepted() {
let file = File::open("tests/feeds/2018-01-20-Intercepted.xml").unwrap();
let channel = Channel::read_from(BufReader::new(file)).unwrap();
let episode = channel.items().iter().nth(14).unwrap();
let ep = NewEpisodeMinimal::new(&episode, 42).unwrap();
let expected = NewEpisodeMinimalBuilder::default()
.title("The Super Bowl of Racism")
.uri(Some(String::from(
"http://traffic.megaphone.fm/PPY6458293736.mp3",
)))
.guid(Some(String::from("7df4070a-9832-11e7-adac-cb37b05d5e24")))
.epoch(1505296800)
.duration(Some(4171))
.podcast_id(42)
.build()
.unwrap();
assert_eq!(ep, expected);
let episode = channel.items().iter().nth(15).unwrap();
let ep = NewEpisodeMinimal::new(&episode, 42).unwrap();
let expected = NewEpisodeMinimalBuilder::default()
.title("Atlas Golfed — U.S.-Backed Think Tanks Target Latin America")
.uri(Some(String::from(
"http://traffic.megaphone.fm/FL5331443769.mp3",
)))
.guid(Some(String::from("7c207a24-e33f-11e6-9438-eb45dcf36a1d")))
.epoch(1502272800)
.duration(Some(4415))
.podcast_id(42)
.build()
.unwrap();
assert_eq!(ep, expected);
}
#[test]
fn test_new_episode_intercepted() {
let file = File::open("tests/feeds/2018-01-20-Intercepted.xml").unwrap();
let channel = Channel::read_from(BufReader::new(file)).unwrap();
let episode = channel.items().iter().nth(14).unwrap();
let descr = "NSA whistleblower Edward Snowden discusses the massive Equifax data breach \
and allegations of Russian interference in the US election. Commentator \
Shaun King explains his call for a boycott of the NFL and talks about his \
campaign to bring violent neo-Nazis to justice. Rapper Open Mike Eagle \
performs.";
let ep = NewEpisode::new(&episode, 42).unwrap();
let expected = NewEpisodeBuilder::default()
.title("The Super Bowl of Racism")
.uri(Some(String::from(
"http://traffic.megaphone.fm/PPY6458293736.mp3",
)))
.description(Some(String::from(descr)))
.guid(Some(String::from("7df4070a-9832-11e7-adac-cb37b05d5e24")))
.length(Some(66738886))
.epoch(1505296800)
.duration(Some(4171))
.podcast_id(42)
.build()
.unwrap();
assert_eq!(ep, expected);
let episode = channel.items().iter().nth(15).unwrap();
let ep = NewEpisode::new(&episode, 42).unwrap();
let descr = "This week on Intercepted: Jeremy gives an update on the aftermath of \
Blackwater’s 2007 massacre of Iraqi civilians. Intercept reporter Lee Fang \
lays out how a network of libertarian think tanks called the Atlas Network \
is insidiously shaping political infrastructure in Latin America. We speak \
with attorney and former Hugo Chavez adviser Eva Golinger about the \
Venezuela\'s political turmoil.And we hear Claudia Lizardo of the \
Caracas-based band, La Pequeña Revancha, talk about her music and hopes for \
Venezuela.";
let expected = NewEpisodeBuilder::default()
.title("Atlas Golfed — U.S.-Backed Think Tanks Target Latin America")
.uri(Some(String::from(
"http://traffic.megaphone.fm/FL5331443769.mp3",
)))
.description(Some(String::from(descr)))
.guid(Some(String::from("7c207a24-e33f-11e6-9438-eb45dcf36a1d")))
.length(Some(67527575))
.epoch(1502272800)
.duration(Some(4415))
.podcast_id(42)
.build()
.unwrap();
assert_eq!(ep, expected);
}
#[test]
fn test_new_episode_minimal_lup() {
let file = File::open("tests/feeds/2018-01-20-LinuxUnplugged.xml").unwrap();
let channel = Channel::read_from(BufReader::new(file)).unwrap();
let episode = channel.items().iter().nth(18).unwrap();
let ep = NewEpisodeMinimal::new(&episode, 42).unwrap();
let expected = NewEpisodeMinimalBuilder::default()
.title("Hacking Devices with Kali Linux | LUP 214")
.uri(Some(String::from(
"http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/jnite/lup-0214.mp3",
)))
.guid(Some(String::from("78A682B4-73E8-47B8-88C0-1BE62DD4EF9D")))
.epoch(1505280282)
.duration(Some(5733))
.podcast_id(42)
.build()
.unwrap();
assert_eq!(ep, expected);
let episode = channel.items().iter().nth(19).unwrap();
let ep = NewEpisodeMinimal::new(&episode, 42).unwrap();
let expected = NewEpisodeMinimalBuilder::default()
.title("Gnome Does it Again | LUP 213")
.uri(Some(String::from(
"http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/jnite/lup-0213.mp3",
)))
.guid(Some(String::from("1CE57548-B36C-4F14-832A-5D5E0A24E35B")))
.epoch(1504670247)
.duration(Some(4491))
.podcast_id(42)
.build()
.unwrap();
assert_eq!(ep, expected);
}
#[test]
fn test_new_episode_lup() {
let file = File::open("tests/feeds/2018-01-20-LinuxUnplugged.xml").unwrap();
let channel = Channel::read_from(BufReader::new(file)).unwrap();
let episode = channel.items().iter().nth(18).unwrap();
let descr = "Audit your network with a couple of easy commands on Kali Linux. Chris \
decides to blow off a little steam by attacking his IoT devices, Wes has the \
scope on Equifax blaming open source & the Beard just saved the show. \
It’s a really packed episode!";
let ep = NewEpisode::new(&episode, 42).unwrap();
let expected = NewEpisodeBuilder::default()
.title("Hacking Devices with Kali Linux | LUP 214")
.uri(Some(String::from(
"http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/jnite/lup-0214.mp3",
)))
.description(Some(String::from(descr)))
.guid(Some(String::from("78A682B4-73E8-47B8-88C0-1BE62DD4EF9D")))
.length(Some(46479789))
.epoch(1505280282)
.duration(Some(5733))
.podcast_id(42)
.build()
.unwrap();
assert_eq!(ep, expected);
let episode = channel.items().iter().nth(19).unwrap();
let ep = NewEpisode::new(&episode, 42).unwrap();
let descr =
"The Gnome project is about to solve one of our audience's biggest Wayland’s \
concerns. But as the project takes on a new level of relevance, decisions for the \
next version of Gnome have us worried about the future.\nPlus we chat with Wimpy \
about the Ubuntu Rally in NYC, Microsoft’s sneaky move to turn Windows 10 into the \
“ULTIMATE LINUX RUNTIME”, community news & more!";
let expected = NewEpisodeBuilder::default()
.title("Gnome Does it Again | LUP 213")
.uri(Some(String::from(
"http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/jnite/lup-0213.mp3",
)))
.description(Some(String::from(descr)))
.guid(Some(String::from("1CE57548-B36C-4F14-832A-5D5E0A24E35B")))
.length(Some(36544272))
.epoch(1504670247)
.duration(Some(4491))
.podcast_id(42)
.build()
.unwrap();
assert_eq!(ep, expected);
}
}
hammond-data/src/parser.rs
+1 -223
View File
@@ -31,14 +31,9 @@ pub(crate) fn parse_itunes_duration(item: &Item) -> Option<i32> {
#[cfg(test)]
mod tests {
use models::{NewEpisode, NewEpisodeBuilder};
use rss::{Channel, ItemBuilder};
use rss::ItemBuilder;
use rss::extension::itunes::ITunesItemExtensionBuilder;
use std::fs::File;
use std::io::BufReader;
use super::*;
#[test]
@@ -99,221 +94,4 @@ mod tests {
assert_eq!(parse_itunes_duration(&item), Some(6970));
}
#[test]
fn test_new_episode_intercepted() {
let file = File::open("tests/feeds/Intercepted.xml").unwrap();
let channel = Channel::read_from(BufReader::new(file)).unwrap();
let firstitem = channel.items().first().unwrap();
let descr = "NSA whistleblower Edward Snowden discusses the massive Equifax data breach \
and allegations of Russian interference in the US election. Commentator \
Shaun King explains his call for a boycott of the NFL and talks about his \
campaign to bring violent neo-Nazis to justice. Rapper Open Mike Eagle \
performs.";
let ep = NewEpisode::new(&firstitem, 0).unwrap();
let expected = NewEpisodeBuilder::default()
.title("The Super Bowl of Racism")
.uri(Some(String::from(
"http://traffic.megaphone.fm/PPY6458293736.mp3",
)))
.description(Some(String::from(descr)))
.guid(Some(String::from("7df4070a-9832-11e7-adac-cb37b05d5e24")))
.length(Some(66738886))
.epoch(1505296800)
.duration(Some(4171))
.build()
.unwrap();
assert_eq!(ep, expected);
let second = channel.items().iter().nth(1).unwrap();
let ep = NewEpisode::new(&second, 0).unwrap();
let descr = "This week on Intercepted: Jeremy gives an update on the aftermath of \
Blackwater’s 2007 massacre of Iraqi civilians. Intercept reporter Lee Fang \
lays out how a network of libertarian think tanks called the Atlas Network \
is insidiously shaping political infrastructure in Latin America. We speak \
with attorney and former Hugo Chavez adviser Eva Golinger about the \
Venezuela\'s political turmoil.And we hear Claudia Lizardo of the \
Caracas-based band, La Pequeña Revancha, talk about her music and hopes for \
Venezuela.";
let expected = NewEpisodeBuilder::default()
.title("Atlas Golfed — U.S.-Backed Think Tanks Target Latin America")
.uri(Some(String::from(
"http://traffic.megaphone.fm/FL5331443769.mp3",
)))
.description(Some(String::from(descr)))
.guid(Some(String::from("7c207a24-e33f-11e6-9438-eb45dcf36a1d")))
.length(Some(67527575))
.epoch(1502272800)
.duration(Some(4220))
.build()
.unwrap();
assert_eq!(ep, expected);
}
#[test]
fn test_new_episode_breakthrough() {
let file = File::open("tests/feeds/TheBreakthrough.xml").unwrap();
let channel = Channel::read_from(BufReader::new(file)).unwrap();
let firstitem = channel.items().first().unwrap();
let descr =
"A reporter finds that homes meant to replace New York’s troubled psychiatric \
hospitals might be just as bad.";
let ep = NewEpisode::new(&firstitem, 0).unwrap();
let expected = NewEpisodeBuilder::default()
.title("The Breakthrough: Hopelessness and Exploitation Inside Homes for Mentally Ill")
.uri(Some(String::from("http://tracking.feedpress.it/link/10581/6726758/20170908-cliff-levy.mp3")))
.description(Some(String::from(descr)))
.guid(Some(String::from("https://www.propublica.org/podcast/\
the-breakthrough-hopelessness-exploitation-homes-for-mentally-ill#134472")))
.length(Some(33396551))
.epoch(1504872000)
.duration(Some(1670))
.build()
.unwrap();
assert_eq!(ep, expected);
let second = channel.items().iter().nth(1).unwrap();
let ep = NewEpisode::new(&second, 0).unwrap();
let descr =
"Jonathan Allen and Amie Parnes didn’t know their book would be called \
‘Shattered,’ or that their extraordinary access would let them chronicle the \
mounting signs of a doomed campaign.";
let expected =
NewEpisodeBuilder::default()
.title(
"The Breakthrough: Behind the Scenes of Hillary Clinton’s Failed Bid for \
President",
)
.uri(Some(String::from(
"http://tracking.feedpress.it/link/10581/6726759/16_JohnAllen-CRAFT.mp3",
)))
.description(Some(String::from(descr)))
.guid(Some(String::from(
"https://www.propublica.\
org/podcast/the-breakthrough-hillary-clinton-failed-presidential-bid#133721",
)))
.length(Some(17964071))
.epoch(1503662400)
.duration(Some(1125))
.build()
.unwrap();
assert_eq!(ep, expected);
}
#[test]
fn test_new_episode_lup() {
let file = File::open("tests/feeds/LinuxUnplugged.xml").unwrap();
let channel = Channel::read_from(BufReader::new(file)).unwrap();
let firstitem = channel.items().first().unwrap();
let descr = "Audit your network with a couple of easy commands on Kali Linux. Chris \
decides to blow off a little steam by attacking his IoT devices, Wes has the \
scope on Equifax blaming open source &amp; the Beard just saved the show. \
It’s a really packed episode!";
let ep = NewEpisode::new(&firstitem, 0).unwrap();
let expected = NewEpisodeBuilder::default()
.title("Hacking Devices with Kali Linux | LUP 214")
.uri(Some(String::from(
"http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/jnite/lup-0214.mp3",
)))
.description(Some(String::from(descr)))
.guid(Some(String::from("78A682B4-73E8-47B8-88C0-1BE62DD4EF9D")))
.length(Some(46479789))
.epoch(1505280282)
.duration(Some(5733))
.build()
.unwrap();
assert_eq!(ep, expected);
let second = channel.items().iter().nth(1).unwrap();
let ep = NewEpisode::new(&second, 0).unwrap();
let descr =
"The Gnome project is about to solve one of our audience's biggest Wayland’s \
concerns. But as the project takes on a new level of relevance, decisions for the \
next version of Gnome have us worried about the future.\nPlus we chat with Wimpy \
about the Ubuntu Rally in NYC, Microsoft’s sneaky move to turn Windows 10 into the \
“ULTIMATE LINUX RUNTIME”, community news &amp; more!";
let expected = NewEpisodeBuilder::default()
.title("Gnome Does it Again | LUP 213")
.uri(Some(String::from(
"http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/jnite/lup-0213.mp3",
)))
.description(Some(String::from(descr)))
.guid(Some(String::from("1CE57548-B36C-4F14-832A-5D5E0A24E35B")))
.length(Some(36544272))
.epoch(1504670247)
.duration(Some(4491))
.build()
.unwrap();
assert_eq!(ep, expected);
}
#[test]
fn test_new_episode_r4expanation() {
let file = File::open("tests/feeds/R4Explanation.xml").unwrap();
let channel = Channel::read_from(BufReader::new(file)).unwrap();
let firstitem = channel.items().iter().nth(9).unwrap();
let descr = "This week we look at <a href=\"https://github.com/rust-lang/rfcs/pull/2094\" \
rel=\"noopener noreferrer\">RFC 2094</a> \"Non-lexical lifetimes\"";
let ep = NewEpisode::new(&firstitem, 0).unwrap();
let expected = NewEpisodeBuilder::default()
.title("Episode #9 - A Once in a Lifetime RFC")
.uri(Some(String::from(
"http://request-for-explanation.github.\
io/podcast/ep9-a-once-in-a-lifetime-rfc/episode.mp3",
)))
.description(Some(String::from(descr)))
.guid(Some(String::from(
"https://request-for-explanation.github.io/podcast/ep9-a-once-in-a-lifetime-rfc/",
)))
.length(Some(15077388))
.epoch(1503957600)
.duration(Some(2533))
.build()
.unwrap();
assert_eq!(ep, expected);
let second = channel.items().iter().nth(8).unwrap();
let ep = NewEpisode::new(&second, 0).unwrap();
let descr = "This week we look at <a href=\"https://github.com/rust-lang/rfcs/pull/2071\" \
rel=\"noopener noreferrer\">RFC 2071</a> \"Add impl Trait type alias and \
variable declarations\"";
let expected = NewEpisodeBuilder::default()
.title("Episode #8 - An Existential Crisis")
.uri(Some(String::from(
"http://request-for-explanation.github.\
io/podcast/ep8-an-existential-crisis/episode.mp3",
)))
.description(Some(String::from(descr)))
.guid(Some(String::from(
"https://request-for-explanation.github.io/podcast/ep8-an-existential-crisis/",
)))
.length(Some(13713219))
.epoch(1502841600)
.duration(Some(2313))
.build()
.unwrap();
assert_eq!(ep, expected);
}
}