From 7181c46ed549f34b08cdf07efc58b76cb2c176bc Mon Sep 17 00:00:00 2001 From: Christopher Davis Date: Fri, 28 Sep 2018 21:01:50 -0400 Subject: [PATCH] podcasts-data: Upgrade hyper to 12.X This also allows us to bump hyper_tls and native_tls, bringing support for openssl 1.1. --- Cargo.lock | 202 ++++++++++++++++++++++++++++- podcasts-data/Cargo.toml | 7 +- podcasts-data/src/errors.rs | 9 ++ podcasts-data/src/lib.rs | 1 + podcasts-data/src/models/source.rs | 82 ++++++------ podcasts-data/src/pipeline.rs | 8 +- 6 files changed, 257 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 051018c..a63d089 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -192,6 +192,15 @@ dependencies = [ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "core-foundation" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "core-foundation-sys 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "core-foundation-sys" version = "0.2.3" @@ -200,6 +209,14 @@ dependencies = [ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "core-foundation-sys" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "crc" version = "1.8.1" @@ -828,6 +845,23 @@ dependencies = [ "pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "h2" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", + "indexmap 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "string 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "handy-sys" version = "0.2.0" @@ -873,6 +907,16 @@ dependencies = [ "string_cache_codegen 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "http" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "httparse" version = "1.3.2" @@ -909,6 +953,31 @@ dependencies = [ "want 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "hyper" +version = "0.12.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "h2 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", + "httparse 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-tcp 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-timer 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "want 0.0.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "hyper-tls" version = "0.1.4" @@ -923,6 +992,18 @@ dependencies = [ "tokio-tls 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "hyper-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.12.11 (registry+https://github.com/rust-lang/crates.io-index)", + "native-tls 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ident_case" version = "1.0.0" @@ -938,6 +1019,11 @@ dependencies = [ "unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "indexmap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "iovec" version = "0.1.2" @@ -1252,6 +1338,22 @@ dependencies = [ "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "native-tls" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl 0.10.12 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-sys 0.9.36 (registry+https://github.com/rust-lang/crates.io-index)", + "schannel 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", + "security-framework 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "security-framework-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "net2" version = "0.2.33" @@ -1358,12 +1460,30 @@ dependencies = [ "foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl-sys 0.9.35 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-sys 0.9.36 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "openssl" +version = "0.10.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-sys 0.9.36 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "openssl-probe" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "openssl-sys" -version = "0.9.35" +version = "0.9.36" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1483,12 +1603,13 @@ dependencies = [ "failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "failure_derive 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", - "hyper 0.11.27 (registry+https://github.com/rust-lang/crates.io-index)", - "hyper-tls 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.12.11 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper-tls 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "maplit 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "native-tls 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "native-tls 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_assertions 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1859,6 +1980,17 @@ dependencies = [ "security-framework-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "security-framework" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "core-foundation 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "core-foundation-sys 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "security-framework-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "security-framework-sys" version = "0.1.16" @@ -1868,6 +2000,15 @@ dependencies = [ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "security-framework-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "core-foundation-sys 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "semver" version = "0.9.0" @@ -1955,6 +2096,11 @@ name = "stable_deref_trait" version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "string" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "string_cache" version = "0.5.2" @@ -2073,6 +2219,19 @@ dependencies = [ "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "tempfile" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "tendril" version = "0.4.0" @@ -2322,6 +2481,11 @@ name = "try-lock" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "try-lock" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "ucd-util" version = "0.1.1" @@ -2441,6 +2605,16 @@ dependencies = [ "try-lock 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "want" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "winapi" version = "0.2.8" @@ -2514,7 +2688,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum chrono 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "45912881121cb26fad7c38c17ba7daa18764771836b34fab7d3fbd93ed633878" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum core-foundation 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "25bfd746d203017f7d5cbd31ee5d8e17f94b6521c7af77ece6c9e4b2d4b16c67" +"checksum core-foundation 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "286e0b41c3a20da26536c6000a280585d519fd07b3956b43aed8a79e9edce980" "checksum core-foundation-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "065a5d7ffdcbc8fa145d6f0746f3555025b9097a9e9cda59f7467abae670c78d" +"checksum core-foundation-sys 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "716c271e8613ace48344f723b60b900a93150271e5be206212d052bbc0883efa" "checksum crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" "checksum crossbeam-channel 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a5716fadb87a5633db34c5e83ee6e036e6edc229f8a6bfb7c7c84ed340ba95df" "checksum crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f739f8c5363aca78cfb059edf753d8f0d36908c348f3d8d1503f03d8b75d9cf3" @@ -2576,16 +2752,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum gstreamer-video-sys 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d0e2efb301a0b94fa4af503122faa04247085936dd888fd59fa4e21eab3cbd37" "checksum gtk 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "56a6b30f194f09a17bb7ffa95c3ecdb405abd3b75ff981f831b1f6d18fe115ff" "checksum gtk-sys 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d487d333a4b87072e6bf9f2e55befa0ebef01b9496c2e263c0f4a1ff3d6c04b1" +"checksum h2 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "a27e7ed946e8335bdf9a191bc1b9b14a03ba822d013d2f58437f4fabcbd7fc2c" "checksum handy-sys 0.2.0 (git+https://gitlab.gnome.org/jsparber/libhandy-sys-rs)" = "" "checksum html2text 0.1.8 (git+https://github.com/alatiera/rust-html2text)" = "" "checksum html5ever 0.22.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b04478cf718862650a0bf66acaf8f2f8c906fbc703f35c916c1f4211b069a364" "checksum html5ever-atoms 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8e4a291981feff7291514f8219d5cd2c740d0c042d75cff248a7c00a025f9d40" +"checksum http 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "24f58e8c2d8e886055c3ead7b28793e1455270b5fb39650984c224bc538ba581" "checksum httparse 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7b6288d7db100340ca12873fd4d08ad1b8f206a9457798dfb17c018a33fee540" "checksum humansize 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b6cab2627acfc432780848602f3f558f7e9dd427352224b0d9324025796d2a5e" "checksum hyper 0.11.27 (registry+https://github.com/rust-lang/crates.io-index)" = "34a590ca09d341e94cddf8e5af0bbccde205d5fbc2fa3c09dd67c7f85cea59d7" +"checksum hyper 0.12.11 (registry+https://github.com/rust-lang/crates.io-index)" = "78d50abbd1790e0f4c74cb1d4a2211b439bac661d54107ad5564c55e77906762" "checksum hyper-tls 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ffb1bd5e518d3065840ab315dbbf44e4420e5f7d80e2cb93fa6ffffc50522378" +"checksum hyper-tls 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "caaee4dea92794a9e697038bd401e264307d1f22c883dbcb6f6618ba0d3b3bd3" "checksum ident_case 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3c9826188e666f2ed92071d2dadef6edc430b11b158b5b2b3f4babbcc891eaaa" "checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" +"checksum indexmap 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "08173ba1e906efb6538785a8844dd496f5d34f0a2d88038e95195172fc667220" "checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08" "checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" @@ -2621,6 +2802,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" "checksum muldiv 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "451a9a05d2a32c566c897835e0ea95cf79ed2fdfe957924045a1721a36c9980f" "checksum native-tls 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f74dbadc8b43df7864539cedb7bc91345e532fdd913cfdc23ad94f4d2d40fbc0" +"checksum native-tls 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8b0a7bd714e83db15676d31caf968ad7318e9cc35f93c85a90231c8f22867549" "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" "checksum new_debug_unreachable 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0cdc457076c78ab54d5e0d6fa7c47981757f1e34dc39ff92787f217dede586c4" "checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" @@ -2633,8 +2815,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum objc-foundation 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1add1b659e36c9607c7aab864a76c7a4c2760cd0cd2e120f3fb8b952c7e22bf9" "checksum objc_id 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c92d4ddb4bd7b50d730c215ff871754d0da6b2178849f8a2a2ab69712d0c073b" "checksum open 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eedfa0ca7b54d84d948bfd058b8f82e767d11f362dd78c36866fd1f69c175867" +"checksum openssl 0.10.12 (registry+https://github.com/rust-lang/crates.io-index)" = "5e2e79eede055813a3ac52fb3915caf8e1c9da2dec1587871aec9f6f7b48508d" "checksum openssl 0.9.24 (registry+https://github.com/rust-lang/crates.io-index)" = "a3605c298474a3aa69de92d21139fb5e2a81688d308262359d85cdd0d12a7985" -"checksum openssl-sys 0.9.35 (registry+https://github.com/rust-lang/crates.io-index)" = "912f301a749394e1025d9dcddef6106ddee9252620e6d0a0e5f8d0681de9b129" +"checksum openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" +"checksum openssl-sys 0.9.36 (registry+https://github.com/rust-lang/crates.io-index)" = "409d77eeb492a1aebd6eb322b2ee72ff7c7496b4434d98b3bf8be038755de65e" "checksum owning_ref 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cdf84f41639e037b484f93433aa3897863b561ed65c6e59c7073d7c561710f37" "checksum pango 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4c2cb169402a3eb1ba034a7cc7d95b8b1c106e9be5ba4be79a5a93dc1a2795f4" "checksum pango-sys 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d6eb49268e69dd0c1da5d3001a61aac08e2e9d2bfbe4ae4b19b9963c998f6453" @@ -2681,7 +2865,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum scoped-tls 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "332ffa32bf586782a3efaeb58f127980944bbc8c4d6913a86107ac2a5ab24b28" "checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" "checksum security-framework 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "dfa44ee9c54ce5eecc9de7d5acbad112ee58755239381f687e564004ba4a2332" +"checksum security-framework 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "697d3f3c23a618272ead9e1fb259c1411102b31c6af8b93f1d64cca9c3b0e8e0" "checksum security-framework-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "5421621e836278a0b139268f36eee0dc7e389b784dc3f79d8f11aabadf41bead" +"checksum security-framework-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ab01dfbe5756785b5b4d46e0289e5a18071dfa9a7c2b24213ea00b9ef9b665bf" "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" "checksum serde 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)" = "34b623917345a631dc9608d5194cc206b3fe6c3554cd1c75b937e55e285254af" @@ -2695,6 +2881,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum smallvec 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4c8cbcd6df1e117c2210e13ab5109635ad68a929fcbb8964dc965b76cb5ee013" "checksum smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "153ffa32fd170e9944f7e0838edf824a754ec4c1fc64746fcc9fe1f8fa602e5d" "checksum stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8" +"checksum string 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "00caf261d6f90f588f8450b8e1230fa0d5be49ee6140fdfbcb55335aff350970" "checksum string_cache 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e7c8ba7515dd502b75080d989b819d31fb72686a82320d8006f665003c42ef79" "checksum string_cache 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "25d70109977172b127fe834e5449e5ab1740b9ba49fa18a2020f509174f25423" "checksum string_cache_codegen 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "35293b05cf1494e8ddd042a7df6756bf18d07f42d234f32e71dce8a7aabb0191" @@ -2707,6 +2894,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum synstructure 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "85bb9b7550d063ea184027c9b8c20ac167cd36d3e06b3a40bceb9d746dc1a7b7" "checksum take 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b157868d8ac1f56b64604539990685fa7611d8fa9e5476cf0c02cf34d32917c5" "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +"checksum tempfile 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "55c1195ef8513f3273d55ff59fe5da6940287a0d7a98331254397f464833675b" "checksum tendril 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9de21546595a0873061940d994bbbc5c35f024ae4fd61ec5c5b159115684f508" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" @@ -2728,6 +2916,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum tokio-udp 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "da941144b816d0dcda4db3a1ba87596e4df5e860a72b70783fe435891f80601c" "checksum tokio-uds 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "424c1ed15a0132251813ccea50640b224c809d6ceafb88154c1a8775873a0e89" "checksum try-lock 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee2aa4715743892880f70885373966c83d73ef1b0838a664ef0c76fffd35e7c2" +"checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" "checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" "checksum unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7f4765f83163b74f957c797ad9253caf97f103fb064d3999aea9568d09fc8a33" "checksum unicase 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "284b6d3db520d67fbe88fd778c21510d1b0ba4a551e5d0fbb023d33405f6de8a" @@ -2746,6 +2935,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum version_check 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7716c242968ee87e5542f8021178248f267f295a5c4803beae8b8b7fd9bc6051" "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" "checksum want 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a05d9d966753fa4b5c8db73fcab5eed4549cfe0e1e4e66911e5564a0085c35d1" +"checksum want 0.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "797464475f30ddb8830cc529aaaae648d581f99e2036a928877dfde027ddf6b3" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" "checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" diff --git a/podcasts-data/Cargo.toml b/podcasts-data/Cargo.toml index 6cbcbf9..790206e 100644 --- a/podcasts-data/Cargo.toml +++ b/podcasts-data/Cargo.toml @@ -17,12 +17,13 @@ url = "1.7.1" xdg = "2.1.0" xml-rs = "0.8.0" futures = "0.1.23" -hyper = "0.11.27" +hyper = "0.12.11" +http = "0.1.13" tokio-core = "0.1.17" tokio-threadpool = "0.1.6" tokio-executor = "0.1" -hyper-tls = "0.1.3" -native-tls = "0.1.5" +hyper-tls = "0.3.0" +native-tls = "0.2.1" num_cpus = "1.8.0" failure = "0.1.2" failure_derive = "0.1.2" diff --git a/podcasts-data/src/errors.rs b/podcasts-data/src/errors.rs index a75055f..3947223 100644 --- a/podcasts-data/src/errors.rs +++ b/podcasts-data/src/errors.rs @@ -1,6 +1,7 @@ use diesel; use diesel::r2d2; use diesel_migrations::RunMigrationsError; +use http; use hyper; use native_tls; use rss; @@ -47,6 +48,8 @@ pub enum DataError { R2D2PoolError(#[cause] r2d2::PoolError), #[fail(display = "Hyper Error: {}", _0)] HyperError(#[cause] hyper::Error), + #[fail(display = "ToStr Error: {}", _0)] + HttpToStr(#[cause] http::header::ToStrError), #[fail(display = "Tokio Spawn Error: {}", _0)] SpawnError(#[cause] tokio_executor::SpawnError), #[fail(display = "Failed to parse a url: {}", _0)] @@ -106,6 +109,12 @@ impl From for DataError { } } +impl From for DataError { + fn from(err: http::header::ToStrError) -> Self { + DataError::HttpToStr(err) + } +} + impl From for DataError { fn from(err: tokio_executor::SpawnError) -> Self { DataError::SpawnError(err) diff --git a/podcasts-data/src/lib.rs b/podcasts-data/src/lib.rs index 53fbf70..05fe9b3 100644 --- a/podcasts-data/src/lib.rs +++ b/podcasts-data/src/lib.rs @@ -76,6 +76,7 @@ extern crate log; extern crate ammonia; extern crate chrono; extern crate futures; +extern crate http; extern crate hyper; extern crate hyper_tls; extern crate native_tls; diff --git a/podcasts-data/src/models/source.rs b/podcasts-data/src/models/source.rs index 9842ce4..9652629 100644 --- a/podcasts-data/src/models/source.rs +++ b/podcasts-data/src/models/source.rs @@ -4,12 +4,14 @@ use rss::Channel; use url::Url; use hyper::client::HttpConnector; -use hyper::header::{ - ETag, EntityTag, HttpDate, IfModifiedSince, IfNoneMatch, LastModified, Location, UserAgent, -}; -use hyper::{Client, Method, Request, Response, StatusCode, Uri}; +use hyper::{Client, Body}; use hyper_tls::HttpsConnector; +use http::{Request, Response, StatusCode, Uri}; +use http::header::{ + ETAG, IF_MODIFIED_SINCE, IF_NONE_MATCH, LAST_MODIFIED, LOCATION, USER_AGENT as USER_AGENT_HEADER, + HeaderValue, +}; // use futures::future::ok; use futures::future::{loop_fn, Future, Loop}; use futures::prelude::*; @@ -91,11 +93,11 @@ impl Source { /// Extract Etag and LastModifier from res, and update self and the /// corresponding db row. - fn update_etag(mut self, res: &Response) -> Result { + fn update_etag(mut self, res: &Response) -> Result { let headers = res.headers(); - let etag = headers.get::().map(|x| x.tag()); - let lmod = headers.get::().map(|x| format!("{}", x)); + let etag = headers.get(ETAG).and_then(|h| h.to_str().ok()).map(From::from); + let lmod = headers.get(LAST_MODIFIED).and_then(|h| h.to_str().ok()).map(From::from); if (self.http_etag() != etag) || (self.last_modified != lmod) { self.set_http_etag(etag); @@ -134,51 +136,57 @@ impl Source { // 408: Timeout // 410: Feed deleted // TODO: Rething this api, - fn match_status(mut self, res: Response) -> Result { + fn match_status(mut self, res: Response) -> Result, DataError> { let code = res.status(); - match code { + + if code.is_success() { // If request is succesful save the etag - StatusCode::NotModified | StatusCode::Ok => self = self.update_etag(&res)?, - // Clear the Etag/lmod else - _ => self = self.clear_etags()?, + self = self.update_etag(&res)? + } else { + match code.as_u16() { + // Save etags if it returns NotModified + 304 => self = self.update_etag(&res)?, + // Clear the Etag/lmod else + _ => self = self.clear_etags()?, + }; }; - match code { - StatusCode::NotModified => { + match code.as_u16() { + 304 => { info!("304: Source, (id: {}), is up to date", self.id()); return Err(DataError::FeedNotModified(self)); } - StatusCode::MovedPermanently | StatusCode::Found | StatusCode::PermanentRedirect => { + 301 | 302 | 308 => { warn!("Feed was moved permanently."); self = self.update_url(&res)?; return Err(DataError::FeedRedirect(self)); } - StatusCode::TemporaryRedirect => { + 307 => { warn!("307: Temporary Redirect."); // FIXME: How is it actually handling the redirect? return Err(DataError::FeedRedirect(self)); } - StatusCode::Unauthorized => return Err(self.make_err("401: Unauthorized.", code)), - StatusCode::Forbidden => return Err(self.make_err("403: Forbidden.", code)), - StatusCode::NotFound => return Err(self.make_err("404: Not found.", code)), - StatusCode::RequestTimeout => return Err(self.make_err("408: Request Timeout.", code)), - StatusCode::Gone => return Err(self.make_err("410: Feed was deleted..", code)), + 401 => return Err(self.make_err("401: Unauthorized.", code)), + 403 => return Err(self.make_err("403: Forbidden.", code)), + 404 => return Err(self.make_err("404: Not found.", code)), + 408 => return Err(self.make_err("408: Request Timeout.", code)), + 410 => return Err(self.make_err("410: Feed was deleted..", code)), _ => info!("HTTP StatusCode: {}", code), }; Ok(res) } - fn update_url(mut self, res: &Response) -> Result { + fn update_url(mut self, res: &Response) -> Result { let code = res.status(); let headers = res.headers(); info!("HTTP StatusCode: {}", code); debug!("Headers {:#?}", headers); - if let Some(url) = headers.get::() { + if let Some(url) = headers.get(LOCATION) { debug!("Previous Source: {:#?}", &self); - self.set_uri(url.to_string()); + self.set_uri(url.to_str()?.into()); self = self.clear_etags()?; debug!("Updated Source: {:#?}", &self); @@ -239,23 +247,22 @@ impl Source { fn request_constructor( self, client: &Client>, - ) -> impl Future { + ) -> impl Future, Error = DataError> { // FIXME: remove unwrap somehow let uri = Uri::from_str(self.uri()).unwrap(); - let mut req = Request::new(Method::Get, uri); + let mut req = Request::get(uri) + .body(Body::empty()) + .unwrap(); // Set the UserAgent cause ppl still seem to check it for some reason... - req.headers_mut().set(UserAgent::new(USER_AGENT)); + req.headers_mut().insert(USER_AGENT_HEADER, HeaderValue::from_static(USER_AGENT)); if let Some(etag) = self.http_etag() { - let tag = vec![EntityTag::new(true, etag.to_owned())]; - req.headers_mut().set(IfNoneMatch::Items(tag)); + req.headers_mut().insert(IF_NONE_MATCH, HeaderValue::from_str(etag).unwrap()); } if let Some(lmod) = self.last_modified() { - if let Ok(date) = lmod.parse::() { - req.headers_mut().set(IfModifiedSince(date)); - } + req.headers_mut().insert(IF_MODIFIED_SINCE, HeaderValue::from_str(lmod).unwrap()); } client @@ -265,9 +272,8 @@ impl Source { } } -#[allow(needless_pass_by_value)] -fn response_to_channel(res: Response) -> impl Future + Send { - res.body() +fn response_to_channel(res: Response) -> impl Future + Send { + res.into_body() .concat2() .map(|x| x.into_iter()) .map_err(From::from) @@ -281,6 +287,7 @@ mod tests { use super::*; use failure::Error; use tokio_core::reactor::Core; + use num_cpus; use database::truncate_db; use utils::get_feed; @@ -290,9 +297,8 @@ mod tests { truncate_db()?; let mut core = Core::new()?; - let client = Client::configure() - .connector(HttpsConnector::new(4, &core.handle())?) - .build(&core.handle()); + let https = HttpsConnector::new(num_cpus::get())?; + let client = Client::builder().build::<_, Body>(https); let url = "https://web.archive.org/web/20180120083840if_/https://feeds.feedburner.\ com/InterceptedWithJeremyScahill"; diff --git a/podcasts-data/src/pipeline.rs b/podcasts-data/src/pipeline.rs index 6f2ab34..8355041 100644 --- a/podcasts-data/src/pipeline.rs +++ b/podcasts-data/src/pipeline.rs @@ -6,7 +6,7 @@ use tokio_core::reactor::Core; use tokio_threadpool::{self, ThreadPool}; use hyper::client::HttpConnector; -use hyper::Client; +use hyper::{Client, Body}; use hyper_tls::HttpsConnector; use num_cpus; @@ -57,10 +57,8 @@ where let pool = ThreadPool::new(); let sender = pool.sender().clone(); let mut core = Core::new()?; - let handle = core.handle(); - let client = Client::configure() - .connector(HttpsConnector::new(num_cpus::get(), &handle)?) - .build(&handle); + let https = HttpsConnector::new(num_cpus::get())?; + let client = Client::builder().build::<_, Body>(https); let stream = iter_ok::<_, DataError>(sources); let p = pipeline(stream, client, sender);