From 5bfa7d6e8935390fc5da2f79c79609a71bc4b964 Mon Sep 17 00:00:00 2001 From: Mark Lopez Date: Sat, 4 Apr 2026 15:02:18 -0500 Subject: [PATCH] feat: added proxy support --- Cargo.lock | 13 +++++++++++++ Cargo.toml | 2 +- README.md | 26 ++++++++++++++++++++++---- src/client.rs | 11 +++++------ 4 files changed, 41 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e967f08..510c43e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2270,6 +2270,18 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-socks" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d4770b8024672c1101b3f6733eab95b18007dbe0847a8afe341fcf79e06043f" +dependencies = [ + "either", + "futures-util", + "thiserror 1.0.69", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -2767,6 +2779,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-boring2", + "tokio-socks", "tokio-util", "tower", "tower-http", diff --git a/Cargo.toml b/Cargo.toml index 4f22257..edd4f50 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,7 +59,7 @@ bincode = "1.3.3" base2048 = "2.0.2" revision = "0.10.0" fake_user_agent = "0.2.2" -wreq = { version = "6.0.0-rc.28", features = ["brotli", "gzip", "deflate", "zstd", "json", "stream"] } +wreq = { version = "6.0.0-rc.28", features = ["brotli", "gzip", "deflate", "zstd", "json", "stream", "socks"] } wreq-util = { version = "3.0.0-rc.10", features = ["emulation-rand"] } [dev-dependencies] diff --git a/README.md b/README.md index 147fcdd..9446049 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ Last tested on January 12, 2024. Results from Google PageSpeed Insights ([Redlib Report](https://pagespeed.web.dev/report?url=https%3A%2F%2Fredlib.matthew.science%2F), [Reddit Report](https://pagespeed.web.dev/report?url=https://www.reddit.com)). | Performance metric | Redlib | Reddit | -| ------------------- | -------- | --------- | +|---------------------|----------|-----------| | Speed Index | 0.6s | 1.9s | | Performance Score | 100% | 64% | | Time to Interactive | **2.8s** | **12.4s** | @@ -409,14 +409,15 @@ Redlib supports the following command line flags: Assign a default value for each instance-specific setting by passing environment variables to Redlib in the format `REDLIB_{X}`. Replace `{X}` with the setting name (see list below) in capital letters. | Name | Possible values | Default value | Description | -| ------------------------- | --------------- | ---------------- | --------------------------------------------------------------------------------------------------------- | +|---------------------------|-----------------|------------------------|-----------------------------------------------------------------------------------------------------------| | `SFW_ONLY` | `["on", "off"]` | `off` | Enables SFW-only mode for the instance, i.e. all NSFW content is filtered. | | `BANNER` | String | (empty) | Allows the server to set a banner to be displayed. Currently this is displayed on the instance info page. | | `ROBOTS_DISABLE_INDEXING` | `["on", "off"]` | `off` | Disables indexing of the instance by search engines. | | `PUSHSHIFT_FRONTEND` | String | `undelete.pullpush.io` | Allows the server to set the Pushshift frontend to be used with "removed" links. | | `PORT` | Integer 0-65535 | `8080` | The **internal** port Redlib listens on. | | `ENABLE_RSS` | `["on", "off"]` | `off` | Enables RSS feed generation. | -| `FULL_URL` | String | (empty) | Allows for proper URLs (for now, only needed by RSS) +| `FULL_URL` | String | (empty) | Allows for proper URLs (for now, only needed by RSS) | + ## Default user settings Assign a default value for each user-modifiable setting by passing environment variables to Redlib in the format `REDLIB_DEFAULT_{Y}`. Replace `{Y}` with the setting name (see list below) in capital letters. @@ -443,6 +444,23 @@ Assign a default value for each user-modifiable setting by passing environment v | `FIXED_NAVBAR` | `["on", "off"]` | `on` | | `REMOVE_DEFAULT_FEEDS` | `["on", "off"]` | `off` | +## Forward Proxies + +Redlib [supports](https://docs.rs/wreq/latest/wreq/#proxies) proxy usage using the standard `HTTP_PROXY` and +`HTTPS_PROXY` environment variables. Use `ALL_PROXY` to set both at the same time (which you want to do). + +- `http://` is the scheme for http proxy +- `https://` is the scheme for https proxy +- `socks4://` is the scheme for socks4 proxy +- `socks4a://` is the scheme for socks4a proxy +- `socks5://` is the scheme for socks5 proxy +- `socks5h://` is the scheme for socks5h proxy + +## Security + +This project uses [BoringSSL](https://boringssl.googlesource.com/boringssl/), built from source with patches from +the [wreq](https://github.com/0x676e67/wreq) project. Certificates are validated against the OS's trust store. + ## Building Since Redlib uses [`boring-sys2`](https://crates.io/crates/boring-sys2), to build Redlib you will need to build @@ -450,7 +468,7 @@ BoringSSL from source. ### Linux/MacOS -Refer to the [boringssl](https://github.com/google/boringssl/blob/main/BUILDING.md) documentation for instructions. +Refer to the [boringssl](https://github.com/google/boringssl/blob/main/BUILDING.md) documentation for dependencies. ### Windows diff --git a/src/client.rs b/src/client.rs index 470c9d7..a419441 100644 --- a/src/client.rs +++ b/src/client.rs @@ -7,14 +7,14 @@ use cached::proc_macro::cached; use futures_lite::future::block_on; use futures_lite::{future::Boxed, FutureExt}; use hyper::{body::Buf, header, Body, Request as HyperRequest, Response as HyperResponse}; -use log::{error, trace, warn}; +use log::{error, info, trace, warn}; use percent_encoding::{percent_encode, CONTROLS}; use serde_json::Value; use std::result::Result; use std::sync::atomic::Ordering; use std::sync::atomic::{AtomicBool, AtomicU16}; use std::sync::LazyLock; -use wreq::{header as wreq_header, Client as WreqClient, Method, Response as WreqResponse}; +use wreq::{header as wreq_header, Client as WreqClient, EmulationFactory, Method, Response as WreqResponse}; use wreq_util::Emulation; const REDDIT_URL_BASE: &str = "https://oauth.reddit.com"; @@ -44,10 +44,9 @@ const URL_PAIRS: [(&str, &str); 2] = [ ]; pub fn build_client() -> WreqClient { - WreqClient::builder() - .emulation(Emulation::random()) - .build() - .expect("Should always be able to build a client") + let emulation = Emulation::random().emulation(); + info!("Building Wreq client with random emulation {:?}", emulation); + WreqClient::builder().emulation(emulation).build().expect("Should always be able to build a client") } /// Gets the canonical path for a resource on Reddit. This is accomplished by