Sort of works atm.

This commit is contained in:
Jordan Petridis 2017-10-03 12:01:01 +03:00
parent fe7ef323c4
commit fc693a569b
No known key found for this signature in database
GPG Key ID: CEABAD9F5683B9A6
7 changed files with 87 additions and 38 deletions

View File

@ -20,8 +20,8 @@ pub fn run() -> Result<()> {
info!("{:?}", foo); info!("{:?}", foo);
::init()?; ::init()?;
downloader::download_to("./foo", "http://traffic.megaphone.fm/FL8700626063.mp3")?; let db = ::establish_connection();
// ::index_feed::foo(); downloader::latest_dl(&db)?;
Ok(()) Ok(())
} }

View File

@ -15,6 +15,13 @@ pub fn get_podcasts(con: &SqliteConnection) -> QueryResult<Vec<Podcast>> {
pds pds
} }
// Maybe later.
// pub fn get_podcasts_ids(con: &SqliteConnection) -> QueryResult<Vec<i32>> {
// use schema::podcast::dsl::*;
// let pds = podcast.select(id).load::<i32>(con);
// pds
// }
pub fn get_episodes(con: &SqliteConnection) -> QueryResult<Vec<Episode>> { pub fn get_episodes(con: &SqliteConnection) -> QueryResult<Vec<Episode>> {
use schema::episode::dsl::*; use schema::episode::dsl::*;

View File

@ -1,9 +1,12 @@
use reqwest; use reqwest;
use hyper::header::*; use hyper::header::*;
use diesel::prelude::*;
use std::fs::File; use std::fs::{File, DirBuilder};
use std::io::{BufWriter, Read, Write}; use std::io::{BufWriter, Read, Write};
use errors::*; use errors::*;
use dbqueries;
// Adapted from https://github.com/mattgathu/rget . // Adapted from https://github.com/mattgathu/rget .
pub fn download_to(target: &str, url: &str) -> Result<()> { pub fn download_to(target: &str, url: &str) -> Result<()> {
@ -19,18 +22,17 @@ pub fn download_to(target: &str, url: &str) -> Result<()> {
info!("Content Type: {:?}", ct_type); info!("Content Type: {:?}", ct_type);
// FIXME // FIXME
let out_file = target.to_owned() + "/bar.mp3"; // let out_file = target.to_owned() + "/bar.mp3";
info!("Save destination: {}", out_file); info!("Save destination: {}", target);
let chunk_size = match ct_len { let chunk_size = match ct_len {
Some(x) => x as usize / 99, Some(x) => x as usize / 99,
None => 1024usize, // default chunk size None => 1024 as usize, // default chunk size
}; };
// let foo_file = let mut writer = BufWriter::new(File::create(target)?);
let mut writer = BufWriter::new(File::create(out_file)?);
// FIXME: not running
loop { loop {
let mut buffer = vec![0; chunk_size]; let mut buffer = vec![0; chunk_size];
let bcount = resp.read(&mut buffer[..]).unwrap(); let bcount = resp.read(&mut buffer[..]).unwrap();
@ -44,3 +46,35 @@ pub fn download_to(target: &str, url: &str) -> Result<()> {
} }
Ok(()) Ok(())
} }
// Initial messy prototype, queries load alot of not needed stuff.
pub fn latest_dl(connection: &SqliteConnection) -> Result<()> {
let pds = dbqueries::get_podcasts(connection)?;
pds.iter()
.map(|x| -> Result<()> {
let eps = dbqueries::get_pd_episodes(connection, &x)?;
// It might be better to make it a hash of the title
let dl_fold = format!("{}/{}", ::DL_DIR.to_str().unwrap(), x.title());
// Create the folder
DirBuilder::new().recursive(true).create(&dl_fold).unwrap();
// Download the episodes
eps.iter()
.map(|y| -> Result<()> {
let ext = y.uri().split(".").last().unwrap();
let dlpath = format!("{}/{}.{}", dl_fold, y.title().unwrap(), ext);
info!("Downloading {:?} into: {}", y.title(), dlpath);
download_to(&dlpath, y.uri())?;
Ok(())
})
.fold((), |(), _| ());
Ok(())
})
.fold((), |(), _| ());
Ok(())
}

View File

@ -6,7 +6,7 @@ use rss;
use reqwest; use reqwest;
use rayon::prelude::*; use rayon::prelude::*;
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use schema; use schema;
use dbqueries; use dbqueries;
use feedparser; use feedparser;
@ -83,11 +83,11 @@ pub fn index_loop(db: SqliteConnection) -> Result<()> {
// f.par_iter_mut().for_each(|&mut (ref mut req, ref source)| { // f.par_iter_mut().for_each(|&mut (ref mut req, ref source)| {
// TODO: Once for_each is stable, uncomment above line and delete collect. // TODO: Once for_each is stable, uncomment above line and delete collect.
let _ : Vec<_> = f.par_iter_mut() let _: Vec<_> = f.par_iter_mut()
.map(|&mut (ref mut req, ref source)| { .map(|&mut (ref mut req, ref source)| {
complete_index_from_source(req, source, m.clone()).unwrap(); complete_index_from_source(req, source, m.clone()).unwrap();
}) })
.collect(); .collect();
Ok(()) Ok(())
} }
@ -257,10 +257,11 @@ mod tests {
"http://feeds.feedburner.com/linuxunplugged", "http://feeds.feedburner.com/linuxunplugged",
]; ];
inpt.iter().map(|feed| { inpt.iter()
index_source(&db, &NewSource::new_with_uri(feed)).unwrap() .map(|feed| {
}) index_source(&db, &NewSource::new_with_uri(feed)).unwrap()
.fold((), |(), _| ()); })
.fold((), |(), _| ());
index_loop(db).unwrap(); index_loop(db).unwrap();
@ -297,21 +298,22 @@ mod tests {
), ),
]; ];
urls.iter().map(|&(path, url)| { urls.iter()
let tempdb = m.lock().unwrap(); .map(|&(path, url)| {
// Create and insert a Source into db let tempdb = m.lock().unwrap();
let s = insert_return_source(&tempdb, url).unwrap(); // Create and insert a Source into db
drop(tempdb); let s = insert_return_source(&tempdb, url).unwrap();
drop(tempdb);
// open the xml file // open the xml file
let feed = fs::File::open(path).unwrap(); let feed = fs::File::open(path).unwrap();
// parse it into a channel // parse it into a channel
let chan = rss::Channel::read_from(BufReader::new(feed)).unwrap(); let chan = rss::Channel::read_from(BufReader::new(feed)).unwrap();
// Index the channel // Index the channel
complete_index(m.clone(), chan, &s).unwrap(); complete_index(m.clone(), chan, &s).unwrap();
}) })
.fold((), |(), _| ()); .fold((), |(), _| ());
// Assert the index rows equal the controlled results // Assert the index rows equal the controlled results
let tempdb = m.lock().unwrap(); let tempdb = m.lock().unwrap();

View File

@ -100,6 +100,11 @@ lazy_static!{
HAMMOND_XDG.place_data_file("hammond.db").unwrap() HAMMOND_XDG.place_data_file("hammond.db").unwrap()
}; };
static ref DL_DIR: PathBuf = {
&HAMMOND_DATA;
HAMMOND_XDG.create_data_directory("Downloads").unwrap()
};
} }
pub fn init() -> Result<()> { pub fn init() -> Result<()> {

View File

@ -14,7 +14,7 @@ use errors::*;
pub struct Episode { pub struct Episode {
id: i32, id: i32,
title: Option<String>, title: Option<String>,
uri: Option<String>, uri: String,
local_uri: Option<String>, local_uri: Option<String>,
description: Option<String>, description: Option<String>,
published_date: Option<String>, published_date: Option<String>,
@ -37,12 +37,13 @@ impl Episode {
self.title = value.map(|x| x.to_string()); self.title = value.map(|x| x.to_string());
} }
pub fn uri(&self) -> Option<&str> { /// uri is guaranted to exist based on the db rules
self.uri.as_ref().map(|s| s.as_str()) pub fn uri(&self) -> &str {
self.uri.as_ref()
} }
pub fn set_uri(&mut self, value: Option<&str>) { pub fn set_uri(&mut self, value: &str) {
self.uri = value.map(|x| x.to_string()); self.uri = value.to_string();
} }
pub fn local_uri(&self) -> Option<&str> { pub fn local_uri(&self) -> Option<&str> {

View File

@ -2,7 +2,7 @@ table! {
episode (id) { episode (id) {
id -> Integer, id -> Integer,
title -> Nullable<Text>, title -> Nullable<Text>,
uri -> Nullable<Text>, uri -> Text,
local_uri -> Nullable<Text>, local_uri -> Nullable<Text>,
description -> Nullable<Text>, description -> Nullable<Text>,
published_date -> Nullable<Text>, published_date -> Nullable<Text>,