From 6f635ac9fab8da657d388b675dc5fc0f9846f282 Mon Sep 17 00:00:00 2001 From: James Wykeham-Martin Date: Sun, 10 Dec 2017 07:58:00 +0000 Subject: [PATCH 1/6] Updated client.get request headers --- hammond-data/src/models/queryables.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/hammond-data/src/models/queryables.rs b/hammond-data/src/models/queryables.rs index f0cab54..ce0d3f6 100644 --- a/hammond-data/src/models/queryables.rs +++ b/hammond-data/src/models/queryables.rs @@ -376,17 +376,21 @@ impl<'a> Source { /// Consumes `self` and Returns the corresponding `Feed` Object. // TODO: Refactor into TryInto once it lands on stable. pub fn into_feed(mut self) -> Result { - use reqwest::header::{ETag, EntityTag, Headers, HttpDate, LastModified}; + use reqwest::header::{EntityTag, Headers, HttpDate, IfNoneMatch, IfModifiedSince}; let mut headers = Headers::new(); if let Some(foo) = self.http_etag() { - headers.set(ETag(EntityTag::new(true, foo.to_owned()))); + headers.set( + IfNoneMatch::Items(vec![ + EntityTag::new(true, foo.to_owned()) + ]) + ); } if let Some(foo) = self.last_modified() { if let Ok(x) = foo.parse::() { - headers.set(LastModified(x)); + headers.set(IfModifiedSince(x)); } } From 82f577c17d236b1ec32cc1a511c910c1c90afbab Mon Sep 17 00:00:00 2001 From: James Wykeham-Martin Date: Fri, 2 Feb 2018 09:38:06 +0000 Subject: [PATCH 2/6] Added custom redirect policy --- hammond-downloader/src/downloader.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/hammond-downloader/src/downloader.rs b/hammond-downloader/src/downloader.rs index dd813b8..5ecdc64 100644 --- a/hammond-downloader/src/downloader.rs +++ b/hammond-downloader/src/downloader.rs @@ -2,6 +2,7 @@ use glob::glob; use hyper::header::*; use mime_guess; use reqwest; +use reqwest::RedirectPolicy; use tempdir::TempDir; use std::fs; @@ -37,7 +38,18 @@ fn download_into( progress: Option>>, ) -> Result { info!("GET request to: {}", url); - let client = reqwest::Client::builder().referer(false).build()?; + // Haven't included the loop check as + // Steal the Stars + let policy = RedirectPolicy::custom(|attempt| { + info!("Redirect Attempt URL: {:?}", attempt.url()); + if attempt.previous().len() > 10 { + attempt.too_many_redirects() + } else { + attempt.follow() + } + }); + + let client = reqwest::Client::builder().redirect(policy).referer(false).build()?; let mut resp = client.get(url).send()?; info!("Status Resp: {}", resp.status()); From d0c5764471aad83da7cc350893550c9a81cf09a0 Mon Sep 17 00:00:00 2001 From: James Wykeham-Martin Date: Fri, 2 Feb 2018 09:40:13 +0000 Subject: [PATCH 3/6] Added custom redirect policy --- hammond-downloader/src/downloader.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hammond-downloader/src/downloader.rs b/hammond-downloader/src/downloader.rs index 5ecdc64..cf614fc 100644 --- a/hammond-downloader/src/downloader.rs +++ b/hammond-downloader/src/downloader.rs @@ -39,7 +39,8 @@ fn download_into( ) -> Result { info!("GET request to: {}", url); // Haven't included the loop check as - // Steal the Stars + // Steal the Stars would tigger it as + // it has a loop back before giving correct url let policy = RedirectPolicy::custom(|attempt| { info!("Redirect Attempt URL: {:?}", attempt.url()); if attempt.previous().len() > 10 { From 5e7d8841afb66c6c4ee9ba4110d8dd61836f95c4 Mon Sep 17 00:00:00 2001 From: James Wykeham-Martin Date: Fri, 2 Feb 2018 09:58:09 +0000 Subject: [PATCH 4/6] Added loopback detection using current last rather than contains --- hammond-downloader/src/downloader.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hammond-downloader/src/downloader.rs b/hammond-downloader/src/downloader.rs index cf614fc..2176922 100644 --- a/hammond-downloader/src/downloader.rs +++ b/hammond-downloader/src/downloader.rs @@ -43,8 +43,10 @@ fn download_into( // it has a loop back before giving correct url let policy = RedirectPolicy::custom(|attempt| { info!("Redirect Attempt URL: {:?}", attempt.url()); - if attempt.previous().len() > 10 { + if attempt.previous().len() > 5 { attempt.too_many_redirects() + } else if attempt.url() == attempt.previous().last().unwrap() { + attempt.loop_detected() } else { attempt.follow() } From 7a0e7708be2657e4e75701b254a0b2235a876c55 Mon Sep 17 00:00:00 2001 From: James Wykeham-Martin Date: Fri, 2 Feb 2018 10:43:20 +0000 Subject: [PATCH 5/6] formatted --- hammond-downloader/src/downloader.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hammond-downloader/src/downloader.rs b/hammond-downloader/src/downloader.rs index 2176922..3a04db6 100644 --- a/hammond-downloader/src/downloader.rs +++ b/hammond-downloader/src/downloader.rs @@ -51,8 +51,11 @@ fn download_into( attempt.follow() } }); - - let client = reqwest::Client::builder().redirect(policy).referer(false).build()?; + + let client = reqwest::Client::builder() + .redirect(policy) + .referer(false) + .build()?; let mut resp = client.get(url).send()?; info!("Status Resp: {}", resp.status()); From a007460029be3abbfb3f9a895e8dcc3dcf70d5c3 Mon Sep 17 00:00:00 2001 From: James Wykeham-Martin Date: Fri, 2 Feb 2018 15:41:00 +0000 Subject: [PATCH 6/6] Added Some to attempt.url() and removed unwrap --- hammond-downloader/src/downloader.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hammond-downloader/src/downloader.rs b/hammond-downloader/src/downloader.rs index 3a04db6..b54573d 100644 --- a/hammond-downloader/src/downloader.rs +++ b/hammond-downloader/src/downloader.rs @@ -45,7 +45,7 @@ fn download_into( info!("Redirect Attempt URL: {:?}", attempt.url()); if attempt.previous().len() > 5 { attempt.too_many_redirects() - } else if attempt.url() == attempt.previous().last().unwrap() { + } else if Some(attempt.url()) == attempt.previous().last() { attempt.loop_detected() } else { attempt.follow()