mirror/redlib
1
0
Fork 0
mirror of https://github.com/redlib-org/redlib.git synced 2026-06-11 04:34:27 -04:00
Code Issues Packages Projects Releases Wiki Activity

chore: switch gzip to default wreq handling, enabled device emulation, removed unused crates

Browse Source
This commit is contained in:
Mark Lopez
2026-04-04 13:07:41 -05:00
parent e138fbb1d5
commit c005e6f2d3
3 changed files with 31 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files

183
Cargo.lock generated
View File

@@ -170,12 +170,6 @@ version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71f4fe417e8cc3bb9b437dfa9290ce92bd2730ba5374719bdfd9147fbc8f17cd"
[[package]]
name = "base64"
version = "0.21.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
[[package]]
name = "base64"
version = "0.22.1"
@@ -485,22 +479,6 @@ dependencies = [
"version_check",
]
[[package]]
name = "core-foundation"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "core-foundation-sys"
version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "core2"
version = "0.4.0"
@@ -1090,22 +1068,6 @@ dependencies = [
"want",
]
[[package]]
name = "hyper-rustls"
version = "0.24.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
dependencies = [
"futures-util",
"http 0.2.12",
"hyper",
"log",
"rustls",
"rustls-native-certs",
"tokio",
"tokio-rustls",
]
[[package]]
name = "icu_collections"
version = "1.5.0"
@@ -1488,12 +1450,6 @@ dependencies = [
"syn",
]
[[package]]
name = "openssl-probe"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e"
[[package]]
name = "parking"
version = "2.2.1"
@@ -1662,7 +1618,7 @@ dependencies = [
"askama",
"async-recursion",
"base2048",
"base64 0.22.1",
"base64",
"bincode",
"brotli 7.0.0",
"build_html",
@@ -1676,7 +1632,6 @@ dependencies = [
"futures-lite",
"htmlescape",
"hyper",
"hyper-rustls",
"libflate",
"lipsum",
"log",
@@ -1688,7 +1643,6 @@ dependencies = [
"route-recognizer",
"rss",
"rust-embed",
"rustls",
"sealed_test",
"serde",
"serde_json",
@@ -1763,20 +1717,6 @@ dependencies = [
"syn",
]
[[package]]
name = "ring"
version = "0.17.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee"
dependencies = [
"cc",
"cfg-if",
"getrandom 0.2.15",
"libc",
"untrusted",
"windows-sys 0.52.0",
]
[[package]]
name = "rle-decode-fast"
version = "1.0.3"
@@ -1855,55 +1795,12 @@ dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "rustls"
version = "0.21.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e"
dependencies = [
"log",
"ring",
"rustls-webpki",
"sct",
]
[[package]]
name = "rustls-native-certs"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00"
dependencies = [
"openssl-probe",
"rustls-pemfile",
"schannel",
"security-framework",
]
[[package]]
name = "rustls-pemfile"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
dependencies = [
"base64 0.21.7",
]
[[package]]
name = "rustls-pki-types"
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
[[package]]
name = "rustls-webpki"
version = "0.101.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "rustversion"
version = "1.0.20"
@@ -1937,15 +1834,6 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "schannel"
version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d"
dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "schnellru"
version = "0.2.4"
@@ -1963,16 +1851,6 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "sct"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "sealed_test"
version = "1.1.0"
@@ -1995,29 +1873,6 @@ dependencies = [
"syn",
]
[[package]]
name = "security-framework"
version = "2.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
dependencies = [
"bitflags",
"core-foundation",
"core-foundation-sys",
"libc",
"security-framework-sys",
]
[[package]]
name = "security-framework-sys"
version = "2.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "serde"
version = "1.0.218"
@@ -2213,6 +2068,24 @@ version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "strum"
version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf"
[[package]]
name = "strum_macros"
version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "syn"
version = "2.0.99"
@@ -2397,16 +2270,6 @@ dependencies = [
"syn",
]
[[package]]
name = "tokio-rustls"
version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
dependencies = [
"rustls",
"tokio",
]
[[package]]
name = "tokio-util"
version = "0.7.18"
@@ -2570,12 +2433,6 @@ version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
[[package]]
name = "untrusted"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "url"
version = "2.5.8"
@@ -2925,6 +2782,8 @@ version = "3.0.0-rc.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c6bbe24d28beb9ceb58b514bd6a613c759d3b706f768b9d2950d5d35b543c04"
dependencies = [
"strum",
"strum_macros",
"typed-builder",
"wreq",
]

8
Cargo.toml
View File

@@ -51,18 +51,16 @@ arc-swap = "1.7.1"
serde_json_path = "0.7.1"
async-recursion = "1.1.1"
pulldown-cmark = { version = "0.12.0", features = ["simd", "html"], default-features = false }
hyper-rustls = { version = "0.24.2", features = [ "http2" ] }
tegen = "0.1.4"
serde_urlencoded = "0.7.1"
chrono = { version = "0.4.39", default-features = false, features = [ "std" ] }
chrono = { version = "0.4.39", default-features = false, features = ["std"] }
htmlescape = "0.3.1"
bincode = "1.3.3"
base2048 = "2.0.2"
revision = "0.10.0"
fake_user_agent = "0.2.2"
rustls = "0.21.12"
wreq = { version = "6.0.0-rc.28" , features = ["brotli", "gzip", "deflate", "zstd", "json", "stream"] }
wreq-util = "3.0.0-rc.10"
wreq = { version = "6.0.0-rc.28", features = ["brotli", "gzip", "deflate", "zstd", "json", "stream"] }
wreq-util = { version = "3.0.0-rc.10", features = ["emulation-rand"] }
[dev-dependencies]
lipsum = "0.9.0"

82
src/client.rs
View File

@@ -7,16 +7,15 @@ use cached::proc_macro::cached;
use futures_lite::future::block_on;
use futures_lite::{future::Boxed, FutureExt};
use hyper::{body::Buf, header, Body, Request, Response};
use libflate::gzip;
use log::{error, trace, warn};
use percent_encoding::{percent_encode, CONTROLS};
use serde_json::Value;
use std::result::Result;
use std::sync::atomic::Ordering;
use std::sync::atomic::{AtomicBool, AtomicU16};
use std::sync::LazyLock;
use std::{io, result::Result};
use wreq::header::{HeaderMap, HeaderValue, ACCEPT, ACCEPT_ENCODING, CONNECTION};
use wreq::{Client as WreqClient, Method};
use wreq_util::Emulation;
const REDDIT_URL_BASE: &str = "https://oauth.reddit.com";
const REDDIT_URL_BASE_HOST: &str = "oauth.reddit.com";
@@ -45,12 +44,10 @@ const URL_PAIRS: [(&str, &str); 2] = [
];
pub fn build_client() -> WreqClient {
let mut headers = HeaderMap::new();
headers.insert(ACCEPT, HeaderValue::from_static("*/*"));
headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("gzip"));
headers.insert(CONNECTION, HeaderValue::from_static("keep-alive"));
WreqClient::builder().default_headers(headers).build().unwrap()
WreqClient::builder()
.emulation(Emulation::random())
.build()
.expect("Should always be able to build a client")
}
/// Gets the canonical path for a resource on Reddit. This is accomplished by
@@ -244,11 +241,8 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo
// Build Reddit URL from path.
let url = format!("{base_path}{path}");
// Build request to Reddit. When making a GET, request gzip compression.
// (Reddit doesn't do brotli yet.)
let mut headers: Vec<(String, String)> = vec![
("Host".into(), host.into()),
("Accept-Encoding".into(), if method == &Method::GET { "gzip".into() } else { "identity".into() }),
(
"Cookie".into(),
if quarantine {
@@ -320,69 +314,7 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo
.await;
};
match response.headers().get(wreq::header::CONTENT_ENCODING).and_then(|h| h.to_str().ok()) {
// Content not compressed or encoding invalid.
None | Some("identity") => Ok(to_hyper_response(response)),
// Content encoded (hopefully with gzip).
Some("gzip") => {
// We get here if the body is gzip-compressed.
// The body must be something that implements
// std::io::Read, hence the conversion to
// bytes::buf::Buf and then transformation into a
// Reader.
let mut decompressed: Vec<u8>;
let mut hyper_res: Response<Body>;
{
let status = response.status();
let version = response.version();
let mut wreq_headers = HeaderMap::new();
for (name, value) in response.headers() {
wreq_headers.insert(name.clone(), value.clone());
}
let body_bytes = match response.bytes().await {
Ok(b) => b,
Err(e) => return Err(e.to_string()),
};
let mut decoder = match gzip::Decoder::new(&body_bytes[..]) {
Ok(decoder) => decoder,
Err(e) => return Err(e.to_string()),
};
decompressed = Vec::<u8>::new();
if let Err(e) = io::copy(&mut decoder, &mut decompressed) {
return Err(e.to_string());
};
let mut builder = Response::builder().status(status.as_u16()).version(match version {
wreq::Version::HTTP_09 => hyper::Version::HTTP_09,
wreq::Version::HTTP_10 => hyper::Version::HTTP_10,
wreq::Version::HTTP_11 => hyper::Version::HTTP_11,
wreq::Version::HTTP_2 => hyper::Version::HTTP_2,
wreq::Version::HTTP_3 => hyper::Version::HTTP_3,
_ => hyper::Version::HTTP_11,
});
for (name, value) in &wreq_headers {
builder = builder.header(
header::HeaderName::from_bytes(name.as_str().as_bytes()).unwrap(),
header::HeaderValue::from_bytes(value.as_bytes()).unwrap(),
);
}
hyper_res = builder.body(Body::empty()).unwrap();
}
hyper_res.headers_mut().remove(header::CONTENT_ENCODING);
hyper_res.headers_mut().insert(header::CONTENT_LENGTH, decompressed.len().into());
*(hyper_res.body_mut()) = Body::from(decompressed);
Ok(hyper_res)
}
Some(_) => Err("Reddit response was encoded with an unsupported compressor".to_string()),
}
Ok(to_hyper_response(response))
}
Err(e) => {
dbg_msg!("{method} {REDDIT_URL_BASE}{path}: {}", e);