From c005e6f2d363cd477e383f0caf4fe9cfc4c6884c Mon Sep 17 00:00:00 2001 From: Mark Lopez Date: Sat, 4 Apr 2026 13:07:41 -0500 Subject: [PATCH] chore: switch gzip to default wreq handling, enabled device emulation, removed unused crates --- Cargo.lock | 183 ++++++-------------------------------------------- Cargo.toml | 8 +-- src/client.rs | 82 ++-------------------- 3 files changed, 31 insertions(+), 242 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b8c17b..e967f08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -170,12 +170,6 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71f4fe417e8cc3bb9b437dfa9290ce92bd2730ba5374719bdfd9147fbc8f17cd" -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -485,22 +479,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - [[package]] name = "core2" version = "0.4.0" @@ -1090,22 +1068,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-rustls" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" -dependencies = [ - "futures-util", - "http 0.2.12", - "hyper", - "log", - "rustls", - "rustls-native-certs", - "tokio", - "tokio-rustls", -] - [[package]] name = "icu_collections" version = "1.5.0" @@ -1488,12 +1450,6 @@ dependencies = [ "syn", ] -[[package]] -name = "openssl-probe" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" - [[package]] name = "parking" version = "2.2.1" @@ -1662,7 +1618,7 @@ dependencies = [ "askama", "async-recursion", "base2048", - "base64 0.22.1", + "base64", "bincode", "brotli 7.0.0", "build_html", @@ -1676,7 +1632,6 @@ dependencies = [ "futures-lite", "htmlescape", "hyper", - "hyper-rustls", "libflate", "lipsum", "log", @@ -1688,7 +1643,6 @@ dependencies = [ "route-recognizer", "rss", "rust-embed", - "rustls", "sealed_test", "serde", "serde_json", @@ -1763,20 +1717,6 @@ dependencies = [ "syn", ] -[[package]] -name = "ring" -version = "0.17.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee" -dependencies = [ - "cc", - "cfg-if", - "getrandom 0.2.15", - "libc", - "untrusted", - "windows-sys 0.52.0", -] - [[package]] name = "rle-decode-fast" version = "1.0.3" @@ -1855,55 +1795,12 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "rustls" -version = "0.21.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" -dependencies = [ - "log", - "ring", - "rustls-webpki", - "sct", -] - -[[package]] -name = "rustls-native-certs" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" -dependencies = [ - "openssl-probe", - "rustls-pemfile", - "schannel", - "security-framework", -] - -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", -] - [[package]] name = "rustls-pki-types" version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" -[[package]] -name = "rustls-webpki" -version = "0.101.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "rustversion" version = "1.0.20" @@ -1937,15 +1834,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "schannel" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" -dependencies = [ - "windows-sys 0.59.0", -] - [[package]] name = "schnellru" version = "0.2.4" @@ -1963,16 +1851,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "sealed_test" version = "1.1.0" @@ -1995,29 +1873,6 @@ dependencies = [ "syn", ] -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "serde" version = "1.0.218" @@ -2213,6 +2068,24 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "syn" version = "2.0.99" @@ -2397,16 +2270,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-rustls" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" -dependencies = [ - "rustls", - "tokio", -] - [[package]] name = "tokio-util" version = "0.7.18" @@ -2570,12 +2433,6 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - [[package]] name = "url" version = "2.5.8" @@ -2925,6 +2782,8 @@ version = "3.0.0-rc.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c6bbe24d28beb9ceb58b514bd6a613c759d3b706f768b9d2950d5d35b543c04" dependencies = [ + "strum", + "strum_macros", "typed-builder", "wreq", ] diff --git a/Cargo.toml b/Cargo.toml index 86e67f4..4f22257 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,18 +51,16 @@ arc-swap = "1.7.1" serde_json_path = "0.7.1" async-recursion = "1.1.1" pulldown-cmark = { version = "0.12.0", features = ["simd", "html"], default-features = false } -hyper-rustls = { version = "0.24.2", features = [ "http2" ] } tegen = "0.1.4" serde_urlencoded = "0.7.1" -chrono = { version = "0.4.39", default-features = false, features = [ "std" ] } +chrono = { version = "0.4.39", default-features = false, features = ["std"] } htmlescape = "0.3.1" bincode = "1.3.3" base2048 = "2.0.2" revision = "0.10.0" fake_user_agent = "0.2.2" -rustls = "0.21.12" -wreq = { version = "6.0.0-rc.28" , features = ["brotli", "gzip", "deflate", "zstd", "json", "stream"] } -wreq-util = "3.0.0-rc.10" +wreq = { version = "6.0.0-rc.28", features = ["brotli", "gzip", "deflate", "zstd", "json", "stream"] } +wreq-util = { version = "3.0.0-rc.10", features = ["emulation-rand"] } [dev-dependencies] lipsum = "0.9.0" diff --git a/src/client.rs b/src/client.rs index 70aec03..b8c94b9 100644 --- a/src/client.rs +++ b/src/client.rs @@ -7,16 +7,15 @@ use cached::proc_macro::cached; use futures_lite::future::block_on; use futures_lite::{future::Boxed, FutureExt}; use hyper::{body::Buf, header, Body, Request, Response}; -use libflate::gzip; use log::{error, trace, warn}; use percent_encoding::{percent_encode, CONTROLS}; use serde_json::Value; +use std::result::Result; use std::sync::atomic::Ordering; use std::sync::atomic::{AtomicBool, AtomicU16}; use std::sync::LazyLock; -use std::{io, result::Result}; -use wreq::header::{HeaderMap, HeaderValue, ACCEPT, ACCEPT_ENCODING, CONNECTION}; use wreq::{Client as WreqClient, Method}; +use wreq_util::Emulation; const REDDIT_URL_BASE: &str = "https://oauth.reddit.com"; const REDDIT_URL_BASE_HOST: &str = "oauth.reddit.com"; @@ -45,12 +44,10 @@ const URL_PAIRS: [(&str, &str); 2] = [ ]; pub fn build_client() -> WreqClient { - let mut headers = HeaderMap::new(); - headers.insert(ACCEPT, HeaderValue::from_static("*/*")); - headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("gzip")); - headers.insert(CONNECTION, HeaderValue::from_static("keep-alive")); - - WreqClient::builder().default_headers(headers).build().unwrap() + WreqClient::builder() + .emulation(Emulation::random()) + .build() + .expect("Should always be able to build a client") } /// Gets the canonical path for a resource on Reddit. This is accomplished by @@ -244,11 +241,8 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo // Build Reddit URL from path. let url = format!("{base_path}{path}"); - // Build request to Reddit. When making a GET, request gzip compression. - // (Reddit doesn't do brotli yet.) let mut headers: Vec<(String, String)> = vec![ ("Host".into(), host.into()), - ("Accept-Encoding".into(), if method == &Method::GET { "gzip".into() } else { "identity".into() }), ( "Cookie".into(), if quarantine { @@ -320,69 +314,7 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo .await; }; - match response.headers().get(wreq::header::CONTENT_ENCODING).and_then(|h| h.to_str().ok()) { - // Content not compressed or encoding invalid. - None | Some("identity") => Ok(to_hyper_response(response)), - - // Content encoded (hopefully with gzip). - Some("gzip") => { - // We get here if the body is gzip-compressed. - - // The body must be something that implements - // std::io::Read, hence the conversion to - // bytes::buf::Buf and then transformation into a - // Reader. - let mut decompressed: Vec; - let mut hyper_res: Response; - { - let status = response.status(); - let version = response.version(); - let mut wreq_headers = HeaderMap::new(); - for (name, value) in response.headers() { - wreq_headers.insert(name.clone(), value.clone()); - } - - let body_bytes = match response.bytes().await { - Ok(b) => b, - Err(e) => return Err(e.to_string()), - }; - - let mut decoder = match gzip::Decoder::new(&body_bytes[..]) { - Ok(decoder) => decoder, - Err(e) => return Err(e.to_string()), - }; - - decompressed = Vec::::new(); - if let Err(e) = io::copy(&mut decoder, &mut decompressed) { - return Err(e.to_string()); - }; - - let mut builder = Response::builder().status(status.as_u16()).version(match version { - wreq::Version::HTTP_09 => hyper::Version::HTTP_09, - wreq::Version::HTTP_10 => hyper::Version::HTTP_10, - wreq::Version::HTTP_11 => hyper::Version::HTTP_11, - wreq::Version::HTTP_2 => hyper::Version::HTTP_2, - wreq::Version::HTTP_3 => hyper::Version::HTTP_3, - _ => hyper::Version::HTTP_11, - }); - - for (name, value) in &wreq_headers { - builder = builder.header( - header::HeaderName::from_bytes(name.as_str().as_bytes()).unwrap(), - header::HeaderValue::from_bytes(value.as_bytes()).unwrap(), - ); - } - hyper_res = builder.body(Body::empty()).unwrap(); - } - - hyper_res.headers_mut().remove(header::CONTENT_ENCODING); - hyper_res.headers_mut().insert(header::CONTENT_LENGTH, decompressed.len().into()); - *(hyper_res.body_mut()) = Body::from(decompressed); - - Ok(hyper_res) - } - Some(_) => Err("Reddit response was encoded with an unsupported compressor".to_string()), - } + Ok(to_hyper_response(response)) } Err(e) => { dbg_msg!("{method} {REDDIT_URL_BASE}{path}: {}", e);