Added some comments into the downloader module.

This commit is contained in:
Jordan Petridis 2017-11-13 18:48:37 +02:00
parent 4bf84ed170
commit 1e45adc034
No known key found for this signature in database
GPG Key ID: CEABAD9F5683B9A6

View File

@ -14,68 +14,77 @@ use hammond_data::index_feed::Database;
use hammond_data::models::{Episode, Podcast}; use hammond_data::models::{Episode, Podcast};
use hammond_data::{DL_DIR, HAMMOND_CACHE}; use hammond_data::{DL_DIR, HAMMOND_CACHE};
// TODO: Replace path that are of type &str with std::path.
// TODO: Have a convention/document absolute/relative paths, if they should end with / or not.
// Adapted from https://github.com/mattgathu/rget . // Adapted from https://github.com/mattgathu/rget .
// I never wanted to write a custom downloader. // I never wanted to write a custom downloader.
// Sorry to those who will have to work with that code. // Sorry to those who will have to work with that code.
// Would much rather use a crate, // Would much rather use a crate,
// or bindings for a lib like youtube-dl(python), // or bindings for a lib like youtube-dl(python),
// But cant seem to find one. // But cant seem to find one.
// TODO: Write unit-tests.
fn download_into(dir: &str, file_title: &str, url: &str) -> Result<String> { fn download_into(dir: &str, file_title: &str, url: &str) -> Result<String> {
info!("GET request to: {}", url); info!("GET request to: {}", url);
let client = reqwest::Client::builder().referer(false).build()?; let client = reqwest::Client::builder().referer(false).build()?;
let mut resp = client.get(url).send()?; let mut resp = client.get(url).send()?;
info!("Status Resp: {}", resp.status()); info!("Status Resp: {}", resp.status());
if resp.status().is_success() { if !resp.status().is_success() {
let headers = resp.headers().clone(); // TODO: Return an error instead of panicking.
panic!("Bad request response.");
}
let ct_len = headers.get::<ContentLength>().map(|ct_len| **ct_len); let headers = resp.headers().clone();
let ct_type = headers.get::<ContentType>();
ct_len.map(|x| info!("File Lenght: {}", x));
ct_type.map(|x| info!("Content Type: {}", x));
// This could be prettier. let ct_len = headers.get::<ContentLength>().map(|ct_len| **ct_len);
let ext = if let Some(t) = ct_type { let ct_type = headers.get::<ContentType>();
let mime = mime_guess::get_extensions(t.type_().as_ref(), t.subtype().as_ref()); ct_len.map(|x| info!("File Lenght: {}", x));
if let Some(m) = mime { ct_type.map(|x| info!("Content Type: {}", x));
if m.contains(&t.subtype().as_ref()) {
t.subtype().as_ref().to_string() // This could be prettier.
} else { // Determine the file extension from the http content-type header.
m.first().unwrap().to_string() let ext = if let Some(t) = ct_type {
} let mime = mime_guess::get_extensions(t.type_().as_ref(), t.subtype().as_ref());
if let Some(m) = mime {
if m.contains(&t.subtype().as_ref()) {
t.subtype().as_ref().to_string()
} else { } else {
error!("Unkown mime type. {}", t); m.first().unwrap().to_string()
"unkown".to_string()
} }
} else { } else {
error!("Unkown mime type."); error!("Unkown mime type. {}", t);
"unkown".to_string() "unkown".to_string()
}; }
info!("Extension: {}", ext); } else {
error!("Unkown mime type.");
"unkown".to_string()
};
info!("Extension: {}", ext);
// Construct a temp file to save desired content. // Construct a temp file to save desired content.
let tempdir = TempDir::new_in(dir, "")?; let tempdir = TempDir::new_in(dir, "")?;
let mut rng = rand::thread_rng(); let mut rng = rand::thread_rng();
let out_file = format!( let out_file = format!(
"{}/{}.part", "{}/{}.part",
tempdir.path().to_str().unwrap(), tempdir.path().to_str().unwrap(),
rng.gen::<usize>() rng.gen::<usize>()
); );
save_io(&out_file, &mut resp, ct_len)?; // Save requested content into the file.
save_io(&out_file, &mut resp, ct_len)?;
// Construct the desired path. // Construct the desired path.
let target = format!("{}/{}.{}", dir, file_title, ext); let target = format!("{}/{}.{}", dir, file_title, ext);
// Rename/move the tempfile into a permanent place. // Rename/move the tempfile into a permanent place upon success.
rename(out_file, &target)?; rename(out_file, &target)?;
info!("Downloading of {} completed succesfully.", &target); info!("Downloading of {} completed succesfully.", &target);
return Ok(target); Ok(target)
}
// Ok(String::from(""))
panic!("Bad request response.");
} }
// TODO: Write unit-tests.
/// Handles the I/O of fetching a remote file and saving into a Buffer and A File.
fn save_io(file: &str, resp: &mut reqwest::Response, content_lenght: Option<u64>) -> Result<()> { fn save_io(file: &str, resp: &mut reqwest::Response, content_lenght: Option<u64>) -> Result<()> {
info!("Downloading into: {}", file); info!("Downloading into: {}", file);
let chunk_size = match content_lenght { let chunk_size = match content_lenght {
@ -116,6 +125,7 @@ pub fn get_episode(connection: &Database, ep: &mut Episode, download_folder: &st
return Ok(()); return Ok(());
} }
// If the path is not valid, then set it to None.
ep.set_local_uri(None); ep.set_local_uri(None);
ep.save(connection)?; ep.save(connection)?;
}; };
@ -134,50 +144,51 @@ pub fn get_episode(connection: &Database, ep: &mut Episode, download_folder: &st
} }
pub fn cache_image(pd: &Podcast) -> Option<String> { pub fn cache_image(pd: &Podcast) -> Option<String> {
if pd.image_uri().is_some() { if pd.image_uri().is_none() {
let url = pd.image_uri().unwrap().to_owned(); return None;
if url == "" { }
return None;
} let url = pd.image_uri().unwrap().to_owned();
if url == "" {
let download_fold = format!( return None;
"{}{}", }
HAMMOND_CACHE.to_str().unwrap(),
pd.title().to_owned() let download_fold = format!(
); "{}{}",
HAMMOND_CACHE.to_str().unwrap(),
// Hacky way pd.title().to_owned()
// TODO: make it so it returns the first cover.* file encountered. );
let png = format!("{}/cover.png", download_fold);
let jpg = format!("{}/cover.jpg", download_fold); // Hacky way
let jpe = format!("{}/cover.jpe", download_fold); // TODO: make it so it returns the first cover.* file encountered.
let jpeg = format!("{}/cover.jpeg", download_fold); let png = format!("{}/cover.png", download_fold);
if Path::new(&png).exists() { let jpg = format!("{}/cover.jpg", download_fold);
return Some(png); let jpe = format!("{}/cover.jpe", download_fold);
} else if Path::new(&jpe).exists() { let jpeg = format!("{}/cover.jpeg", download_fold);
return Some(jpe); if Path::new(&png).exists() {
} else if Path::new(&jpg).exists() { return Some(png);
return Some(jpg); } else if Path::new(&jpe).exists() {
} else if Path::new(&jpeg).exists() { return Some(jpe);
return Some(jpeg); } else if Path::new(&jpg).exists() {
}; return Some(jpg);
} else if Path::new(&jpeg).exists() {
DirBuilder::new() return Some(jpeg);
.recursive(true) };
.create(&download_fold)
.unwrap(); DirBuilder::new()
.recursive(true)
let dlpath = download_into(&download_fold, "cover", &url); .create(&download_fold)
if let Ok(path) = dlpath { .unwrap();
info!("Cached img into: {}", &path);
return Some(path); let dlpath = download_into(&download_fold, "cover", &url);
} else { if let Ok(path) = dlpath {
error!("Failed to get feed image."); info!("Cached img into: {}", &path);
error!("Error: {}", dlpath.unwrap_err()); Some(path)
return None; } else {
}; error!("Failed to get feed image.");
error!("Error: {}", dlpath.unwrap_err());
None
} }
None
} }
#[cfg(test)] #[cfg(test)]