mirror/redlib
1
0
Fork 0
mirror of https://github.com/redlib-org/redlib.git synced 2026-06-11 20:54:14 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into rss-feed-images

Browse Source
This commit is contained in:
Matthew Esposito
2026-04-24 13:33:14 -04:00
committed by GitHub
parent 726c3433a6 48306ee1fd
commit ba591a4149
18 changed files with 1242 additions and 808 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.devcontainer/devcontainer.json
View File

@@ -1,6 +1,6 @@
{
"name": "Rust",
"image": "mcr.microsoft.com/devcontainers/rust:1.0.9-bookworm",
"image": "mcr.microsoft.com/devcontainers/rust:dev-trixie",
"features": {
"ghcr.io/devcontainers/features/docker-in-docker:2": {}
},
@@ -10,5 +10,6 @@
"onAutoForward": "notify"
}
},
"postCreateCommand": "cargo build"
"postCreateCommand": "sudo apt-get update && sudo apt-get install -y git build-essential cmake libclang-dev",
"postStartCommand": "cargo build"
}

3
.dockerignore Normal file
View File

@@ -0,0 +1,3 @@
target
.github
Dockerfile*

6
.github/workflows/build-artifacts.yaml vendored
View File

@@ -38,17 +38,17 @@ jobs:
- if: matrix.target == 'x86_64-unknown-linux-musl'
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends musl-tools
sudo apt-get install -y --no-install-recommends musl-tools git cmake perl pkg-config libclang-dev
- if: matrix.target == 'armv7-unknown-linux-musleabihf'
run: |
sudo apt update
sudo apt install -y gcc-arm-linux-gnueabihf musl-tools
sudo apt install -y gcc-arm-linux-gnueabihf musl-tools git cmake perl pkg-config libclang-dev
- if: matrix.target == 'aarch64-unknown-linux-musl'
run: |
sudo apt update
sudo apt install -y gcc-aarch64-linux-gnu musl-tools
sudo apt install -y gcc-aarch64-linux-gnu musl-tools git cmake perl pkg-config libclang-dev
- name: Versions
id: version

2
.github/workflows/main-rust.yml vendored
View File

@@ -31,7 +31,7 @@ jobs:
toolchain: stable
- name: Install musl-gcc
run: sudo apt-get install musl-tools
run: sudo apt-get update && sudo apt-get install -y --no-install-recommends musl-tools git cmake perl pkg-config libclang-dev
- name: Install cargo musl target
run: rustup target add x86_64-unknown-linux-musl

948
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

12
Cargo.toml
View File

@@ -17,7 +17,7 @@ askama = { version = "0.14.0", default-features = false, features = [
"std",
"derive",
] }
cached = { version = "0.54.0", features = ["async"] }
cached = { version = "0.59.0", features = ["async"] }
clap = { version = "4.4.11", default-features = false, features = [
"std",
"env",
@@ -50,17 +50,17 @@ rss = "2.0.12"
arc-swap = "1.7.1"
serde_json_path = "0.7.1"
async-recursion = "1.1.1"
pulldown-cmark = { version = "0.12.0", features = ["simd", "html"], default-features = false }
hyper-rustls = { version = "0.24.2", features = [ "http2" ] }
pulldown-cmark = { version = "0.13.3", features = ["simd", "html"], default-features = false }
tegen = "0.1.4"
serde_urlencoded = "0.7.1"
chrono = { version = "0.4.39", default-features = false, features = [ "std" ] }
chrono = { version = "0.4.39", default-features = false, features = ["std"] }
htmlescape = "0.3.1"
bincode = "1.3.3"
base2048 = "2.0.2"
revision = "0.10.0"
revision = "0.17.0"
fake_user_agent = "0.2.2"
wreq = { version = "6.0.0-rc.28", features = ["brotli", "gzip", "deflate", "zstd", "json", "stream", "socks"] }
wreq-util = { version = "3.0.0-rc.10" }
[dev-dependencies]
lipsum = "0.9.0"

17
Dockerfile.alpine
View File

@@ -1,24 +1,29 @@
# supported versions here: https://hub.docker.com/_/rust
ARG ALPINE_VERSION=3.20
ARG RUST_BUILDER_VERSION=slim-bookworm
ARG ALPINE_VERSION=3.23
########################
## builder image
########################
FROM rust:alpine${ALPINE_VERSION} AS builder
FROM ghcr.io/rust-cross/rust-musl-cross:x86_64-musl AS builder
RUN apk add --no-cache musl-dev
RUN apt-get update \
&& apt-get -y install git cmake perl pkg-config libclang-dev \
&& rm -rf /var/lib/apt/lists/*
RUN rustup target add x86_64-unknown-linux-musl
WORKDIR /redlib
# download (most) dependencies in their own layer
COPY Cargo.lock Cargo.toml ./
RUN mkdir src && echo "fn main() { panic!(\"why am i running?\") }" > src/main.rs
RUN cargo build --release --locked --bin redlib
RUN cargo build --release --locked --bin redlib --target x86_64-unknown-linux-musl
RUN rm ./src/main.rs && rmdir ./src
# copy the source and build the redlib binary
COPY . ./
RUN cargo build --release --locked --bin redlib
RUN cargo build --release --locked --bin redlib --target x86_64-unknown-linux-musl
RUN echo "finished building redlib!"
########################
@@ -27,7 +32,7 @@ RUN echo "finished building redlib!"
FROM alpine:${ALPINE_VERSION} AS release
# Import redlib binary from builder
COPY --from=builder /redlib/target/release/redlib /usr/local/bin/redlib
COPY --from=builder /redlib/target/x86_64-unknown-linux-musl/release/redlib /usr/local/bin/redlib
# Add non-root user for running redlib
RUN adduser --home /nonexistent --no-create-home --disabled-password redlib

7
Dockerfile.ubuntu
View File

@@ -7,6 +7,10 @@ ARG UBUNTU_RELEASE_VERSION=noble
########################
FROM rust:${RUST_BUILDER_VERSION} AS builder
RUN apt-get update \
&& apt-get -y install git build-essential cmake libclang-dev \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /redlib
# download (most) dependencies in their own layer
@@ -25,9 +29,6 @@ RUN echo "finished building redlib!"
########################
FROM ubuntu:${UBUNTU_RELEASE_VERSION} AS release
# Install ca-certificates
RUN apt-get update && apt-get install -y ca-certificates
# Import redlib binary from builder
COPY --from=builder /redlib/target/release/redlib /usr/local/bin/redlib

87
README.md
View File

@@ -111,7 +111,7 @@ Last tested on January 12, 2024.
Results from Google PageSpeed Insights ([Redlib Report](https://pagespeed.web.dev/report?url=https%3A%2F%2Fredlib.matthew.science%2F), [Reddit Report](https://pagespeed.web.dev/report?url=https://www.reddit.com)).
| Performance metric | Redlib | Reddit |
| ------------------- | -------- | --------- |
|---------------------|----------|-----------|
| Speed Index | 0.6s | 1.9s |
| Performance Score | 100% | 64% |
| Time to Interactive | **2.8s** | **12.4s** |
@@ -409,36 +409,77 @@ Redlib supports the following command line flags:
Assign a default value for each instance-specific setting by passing environment variables to Redlib in the format `REDLIB_{X}`. Replace `{X}` with the setting name (see list below) in capital letters.
| Name | Possible values | Default value | Description |
| ------------------------- | --------------- | ---------------- | --------------------------------------------------------------------------------------------------------- |
|---------------------------|-----------------|------------------------|-----------------------------------------------------------------------------------------------------------|
| `SFW_ONLY` | `["on", "off"]` | `off` | Enables SFW-only mode for the instance, i.e. all NSFW content is filtered. |
| `BANNER` | String | (empty) | Allows the server to set a banner to be displayed. Currently this is displayed on the instance info page. |
| `ROBOTS_DISABLE_INDEXING` | `["on", "off"]` | `off` | Disables indexing of the instance by search engines. |
| `PUSHSHIFT_FRONTEND` | String | `undelete.pullpush.io` | Allows the server to set the Pushshift frontend to be used with "removed" links. |
| `PORT` | Integer 0-65535 | `8080` | The **internal** port Redlib listens on. |
| `ENABLE_RSS` | `["on", "off"]` | `off` | Enables RSS feed generation. |
| `FULL_URL` | String | (empty) | Allows for proper URLs (for now, only needed by RSS)
| `FULL_URL` | String | (empty) | Allows for proper URLs (for now, only needed by RSS) |
## Default user settings
Assign a default value for each user-modifiable setting by passing environment variables to Redlib in the format `REDLIB_DEFAULT_{Y}`. Replace `{Y}` with the setting name (see list below) in capital letters.
| Name | Possible values | Default value |
| ----------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | ------------- |
| Name | Possible values | Default value |
|-------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|
| `THEME` | `["system", "light", "dark", "black", "dracula", "nord", "laserwave", "violet", "gold", "rosebox", "gruvboxdark", "gruvboxlight", "tokyoNight", "icebergDark", "doomone", "libredditBlack", "libredditDark", "libredditLight"]` | `system` |
| `FRONT_PAGE` | `["default", "popular", "all"]` | `default` |
| `LAYOUT` | `["card", "clean", "compact"]` | `card` |
| `WIDE` | `["on", "off"]` | `off` |
| `POST_SORT` | `["hot", "new", "top", "rising", "controversial"]` | `hot` |
| `COMMENT_SORT` | `["confidence", "top", "new", "controversial", "old"]` | `confidence` |
| `BLUR_SPOILER` | `["on", "off"]` | `off` |
| `SHOW_NSFW` | `["on", "off"]` | `off` |
| `BLUR_NSFW` | `["on", "off"]` | `off` |
| `USE_HLS` | `["on", "off"]` | `off` |
| `HIDE_HLS_NOTIFICATION` | `["on", "off"]` | `off` |
| `AUTOPLAY_VIDEOS` | `["on", "off"]` | `off` |
| `SUBSCRIPTIONS` | `+`-delimited list of subreddits (`sub1+sub2+sub3+...`) | _(none)_ |
| `HIDE_AWARDS` | `["on", "off"]` | `off` |
| `DISABLE_VISIT_REDDIT_CONFIRMATION` | `["on", "off"]` | `off` |
| `HIDE_SCORE` | `["on", "off"]` | `off` |
| `HIDE_SIDEBAR_AND_SUMMARY` | `["on", "off"]` | `off` |
| `FIXED_NAVBAR` | `["on", "off"]` | `on` |
| `REMOVE_DEFAULT_FEEDS` | `["on", "off"]` | `off` |
| `FRONT_PAGE` | `["default", "popular", "all"]` | `default` |
| `LAYOUT` | `["card", "clean", "compact"]` | `card` |
| `WIDE` | `["on", "off"]` | `off` |
| `POST_SORT` | `["hot", "new", "top", "rising", "controversial"]` | `hot` |
| `COMMENT_SORT` | `["confidence", "top", "new", "controversial", "old"]` | `confidence` |
| `BLUR_SPOILER` | `["on", "off"]` | `off` |
| `SHOW_NSFW` | `["on", "off"]` | `off` |
| `BLUR_NSFW` | `["on", "off"]` | `off` |
| `USE_HLS` | `["on", "off"]` | `off` |
| `HIDE_HLS_NOTIFICATION` | `["on", "off"]` | `off` |
| `AUTOPLAY_VIDEOS` | `["on", "off"]` | `off` |
| `SUBSCRIPTIONS` | `+`-delimited list of subreddits (`sub1+sub2+sub3+...`) | _(none)_ |
| `HIDE_AWARDS` | `["on", "off"]` | `off` |
| `DISABLE_VISIT_REDDIT_CONFIRMATION` | `["on", "off"]` | `off` |
| `HIDE_SCORE` | `["on", "off"]` | `off` |
| `HIDE_SIDEBAR_AND_SUMMARY` | `["on", "off"]` | `off` |
| `FIXED_NAVBAR` | `["on", "off"]` | `on` |
| `REMOVE_DEFAULT_FEEDS` | `["on", "off"]` | `off` |
## Forward Proxies
Redlib [supports](https://docs.rs/wreq/latest/wreq/#proxies) proxy usage using the standard `HTTP_PROXY` and
`HTTPS_PROXY` environment variables. Use `ALL_PROXY` to set both at the same time (which you want to do).
- `http://` is the scheme for http proxy
- `https://` is the scheme for https proxy
- `socks4://` is the scheme for socks4 proxy
- `socks4a://` is the scheme for socks4a proxy
- `socks5://` is the scheme for socks5 proxy
- `socks5h://` is the scheme for socks5h proxy
## Security
This project uses [BoringSSL](https://boringssl.googlesource.com/boringssl/), built from source with patches from
the [wreq](https://github.com/0x676e67/wreq) project. Certificates are validated against the embedded trust store from Mozilla.
## Building
Since Redlib uses [`boring-sys2`](https://crates.io/crates/boring-sys2), to build Redlib you will need to build
BoringSSL from source.
### Linux/MacOS
Refer to the [boringssl](https://github.com/google/boringssl/blob/main/BUILDING.md) documentation for dependencies.
### Windows
Install MSVC, which you likely already have for Rust.
```pwsh
# Make sure to update your PATH, some of the installers don't do that by default (hense -i, interactive mode).
winget install -i Kitware.CMake
winget install -i NASM.NASM
winget install -i LLVM.LLVM
# For tests.
winget install GoLang.Go
```

381
src/client.rs
View File

@@ -1,25 +1,22 @@
use arc_swap::ArcSwap;
use cached::proc_macro::cached;
use futures_lite::future::block_on;
use futures_lite::{future::Boxed, FutureExt};
use hyper::client::HttpConnector;
use hyper::header::HeaderValue;
use hyper::{body, body::Buf, header, Body, Client, Method, Request, Response, Uri};
use hyper_rustls::HttpsConnector;
use libflate::gzip;
use log::{error, trace, warn};
use percent_encoding::{percent_encode, CONTROLS};
use serde_json::Value;
use std::sync::atomic::Ordering;
use std::sync::atomic::{AtomicBool, AtomicU16};
use std::sync::LazyLock;
use std::{io, result::Result};
use crate::dbg_msg;
use crate::oauth::{force_refresh_token, token_daemon, Oauth, OauthBackendImpl};
use crate::server::RequestExt;
use crate::utils::{format_url, Post};
use arc_swap::ArcSwap;
use cached::proc_macro::cached;
use futures_lite::future::block_on;
use futures_lite::{future::Boxed, FutureExt};
use hyper::{body::Buf, header, Body, Request as HyperRequest, Response as HyperResponse};
use log::{error, info, trace, warn};
use percent_encoding::{percent_encode, CONTROLS};
use serde_json::Value;
use std::result::Result;
use std::sync::atomic::Ordering;
use std::sync::atomic::{AtomicBool, AtomicU16};
use std::sync::LazyLock;
use wreq::redirect::Policy;
use wreq::{header as wreq_header, Client as WreqClient, EmulationFactory, Method, Response as WreqResponse};
use wreq_util::{Emulation, EmulationOS, EmulationOption};
const REDDIT_URL_BASE: &str = "https://oauth.reddit.com";
const REDDIT_URL_BASE_HOST: &str = "oauth.reddit.com";
@@ -30,10 +27,7 @@ const REDDIT_SHORT_URL_BASE_HOST: &str = "redd.it";
const ALTERNATIVE_REDDIT_URL_BASE: &str = "https://www.reddit.com";
const ALTERNATIVE_REDDIT_URL_BASE_HOST: &str = "www.reddit.com";
pub static HTTPS_CONNECTOR: LazyLock<HttpsConnector<HttpConnector>> =
LazyLock::new(|| hyper_rustls::HttpsConnectorBuilder::new().with_native_roots().https_only().enable_http2().build());
pub static CLIENT: LazyLock<Client<HttpsConnector<HttpConnector>>> = LazyLock::new(|| Client::builder().build::<_, Body>(HTTPS_CONNECTOR.clone()));
pub static CLIENT: LazyLock<WreqClient> = LazyLock::new(build_client);
pub static OAUTH_CLIENT: LazyLock<ArcSwap<Oauth>> = LazyLock::new(|| {
let client = block_on(Oauth::new());
@@ -50,6 +44,28 @@ const URL_PAIRS: [(&str, &str); 2] = [
(REDDIT_SHORT_URL_BASE, REDDIT_SHORT_URL_BASE_HOST),
];
pub fn build_client() -> WreqClient {
// Keeping this list short to aid in privacy.
// The more emulations, the more unique a fingerprint each instance has.
// But some emulations should increase evasiveness.
let emulation = [Emulation::Chrome145, Emulation::Firefox147];
let emulation_os = [EmulationOS::Android, EmulationOS::Windows];
let rand = fastrand::usize(..);
let emulation = EmulationOption::builder()
.emulation(emulation[rand % emulation.len()])
.emulation_os(emulation_os[rand % emulation_os.len()])
.build()
.emulation();
info!("Building Wreq client with random emulation {:?}", emulation);
WreqClient::builder()
.emulation(emulation)
.redirect(Policy::none())
.build()
.expect("Should always be able to build a client")
}
/// Gets the canonical path for a resource on Reddit. This is accomplished by
/// making a `HEAD` request to Reddit at the path given in `path`.
///
@@ -86,14 +102,14 @@ pub async fn canonical_path(path: String, tries: i8) -> Result<Option<String>, S
let res = res.ok_or_else(|| "Unable to make HEAD request to Reddit.".to_string())?;
let status = res.status().as_u16();
let policy_error = res.headers().get(header::RETRY_AFTER).is_some();
let policy_error = res.headers().get(wreq_header::RETRY_AFTER).is_some();
match status {
// If Reddit responds with a 2xx, then the path is already canonical.
200..=299 => Ok(Some(path)),
// If Reddit responds with a 301, then the path is redirected.
301 => match res.headers().get(header::LOCATION) {
301 => match res.headers().get(wreq_header::LOCATION) {
Some(val) => {
let Ok(original) = val.to_str() else {
return Err("Unable to decode Location header.".to_string());
@@ -131,13 +147,13 @@ pub async fn canonical_path(path: String, tries: i8) -> Result<Option<String>, S
_ => Ok(
res
.headers()
.get(header::LOCATION)
.get(wreq_header::LOCATION)
.map(|val| percent_encode(val.as_bytes(), CONTROLS).to_string().trim_start_matches(REDDIT_URL_BASE).to_string()),
),
}
}
pub async fn proxy(req: Request<Body>, format: &str) -> Result<Response<Body>, String> {
pub async fn proxy(req: HyperRequest<Body>, format: &str) -> Result<HyperResponse<Body>, String> {
let mut url = format!("{format}?{}", req.uri().query().unwrap_or_default());
// For each parameter in request
@@ -146,22 +162,15 @@ pub async fn proxy(req: Request<Body>, format: &str) -> Result<Response<Body>, S
url = url.replace(&format!("{{{name}}}"), value);
}
stream(&url, &req).await
}
async fn stream(url: &str, req: &Request<Body>) -> Result<Response<Body>, String> {
// First parameter is target URL (mandatory).
let parsed_uri = url.parse::<Uri>().map_err(|_| "Couldn't parse URL".to_string())?;
let wreq_uri = wreq::Uri::try_from(url).map_err(|_| "Couldn't parse URL".to_string())?;
// Build the hyper client from the HTTPS connector.
let client: &LazyLock<Client<_, Body>> = &CLIENT;
let mut builder = Request::get(parsed_uri);
let mut builder = CLIENT.get(wreq_uri);
// Copy useful headers from original request
for &key in &["Range", "If-Modified-Since", "Cache-Control"] {
if let Some(value) = req.headers().get(key) {
builder = builder.header(key, value);
builder = builder.header(key, value.as_bytes());
}
}
@@ -171,13 +180,16 @@ async fn stream(url: &str, req: &Request<Body>) -> Result<Response<Body>, String
builder = builder.header("User-Agent", client.user_agent());
}
let stream_request = builder.body(Body::empty()).map_err(|_| "Couldn't build empty body in stream".to_string())?;
// This is needed or Reddit will redirect us to a /media landing page that just renders the image.
builder = builder.header(wreq_header::ACCEPT, "*/*");
client
.request(stream_request)
builder
.send()
.await
.map(|mut res| {
let mut rm = |key: &str| res.headers_mut().remove(key);
let headers = res.headers_mut();
let mut rm = |key: &str| headers.remove(key);
rm("access-control-expose-headers");
rm("server");
@@ -192,19 +204,19 @@ async fn stream(url: &str, req: &Request<Body>) -> Result<Response<Body>, String
rm("Nel");
rm("Report-To");
res
res.into_hyper_response()
})
.map_err(|e| e.to_string())
}
/// Makes a GET request to Reddit at `path`. By default, this will honor HTTP
/// 3xx codes Reddit returns and will automatically redirect.
fn reddit_get(path: String, quarantine: bool) -> Boxed<Result<Response<Body>, String>> {
fn reddit_get(path: String, quarantine: bool) -> Boxed<Result<WreqResponse, String>> {
request(&Method::GET, path, true, quarantine, REDDIT_URL_BASE, REDDIT_URL_BASE_HOST)
}
/// Makes a HEAD request to Reddit at `path, using the short URL base. This will not follow redirects.
fn reddit_short_head(path: String, quarantine: bool, base_path: &'static str, host: &'static str) -> Boxed<Result<Response<Body>, String>> {
fn reddit_short_head(path: String, quarantine: bool, base_path: &'static str, host: &'static str) -> Boxed<Result<WreqResponse, String>> {
request(&Method::HEAD, path, false, quarantine, base_path, host)
}
@@ -217,18 +229,12 @@ fn reddit_short_head(path: String, quarantine: bool, base_path: &'static str, ho
/// Makes a request to Reddit. If `redirect` is `true`, `request_with_redirect`
/// will recurse on the URL that Reddit provides in the Location HTTP header
/// in its response.
fn request(method: &'static Method, path: String, redirect: bool, quarantine: bool, base_path: &'static str, host: &'static str) -> Boxed<Result<Response<Body>, String>> {
fn request(method: &'static Method, path: String, redirect: bool, quarantine: bool, base_path: &'static str, host: &'static str) -> Boxed<Result<WreqResponse, String>> {
// Build Reddit URL from path.
let url = format!("{base_path}{path}");
// Construct the hyper client from the HTTPS connector.
let client: &LazyLock<Client<_, Body>> = &CLIENT;
// Build request to Reddit. When making a GET, request gzip compression.
// (Reddit doesn't do brotli yet.)
let mut headers: Vec<(String, String)> = vec![
("Host".into(), host.into()),
("Accept-Encoding".into(), if method == Method::GET { "gzip".into() } else { "identity".into() }),
(
"Cookie".into(),
if quarantine {
@@ -249,114 +255,64 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo
// shuffle headers: https://github.com/redlib-org/redlib/issues/324
fastrand::shuffle(&mut headers);
let mut builder = Request::builder().method(method).uri(&url);
let mut builder = CLIENT.request(method.clone(), &url);
for (key, value) in headers {
builder = builder.header(key, value);
}
let builder = builder.body(Body::empty());
async move {
match builder {
Ok(req) => match client.request(req).await {
Ok(mut response) => {
// Reddit may respond with a 3xx. Decide whether or not to
// redirect based on caller params.
if response.status().is_redirection() {
if !redirect {
return Ok(response);
};
let location_header = response.headers().get(header::LOCATION);
if location_header == Some(&HeaderValue::from_static(ALTERNATIVE_REDDIT_URL_BASE)) {
return Err("Reddit response was invalid".to_string());
}
return request(
method,
location_header
.map(|val| {
// We need to make adjustments to the URI
// we get back from Reddit. Namely, we
// must:
//
// 1. Remove the authority (e.g.
// https://www.reddit.com) that may be
// present, so that we recurse on the
// path (and query parameters) as
// required.
//
// 2. Percent-encode the path.
let new_path = percent_encode(val.as_bytes(), CONTROLS)
.to_string()
.trim_start_matches(REDDIT_URL_BASE)
.trim_start_matches(ALTERNATIVE_REDDIT_URL_BASE)
.to_string();
format!("{new_path}{}raw_json=1", if new_path.contains('?') { "&" } else { "?" })
})
.unwrap_or_default()
.to_string(),
true,
quarantine,
base_path,
host,
)
.await;
match builder.send().await {
Ok(response) => {
// Reddit may respond with a 3xx. Decide whether or not to
// redirect based on caller params.
if response.status().is_redirection() {
if !redirect {
return Ok(response);
};
match response.headers().get(header::CONTENT_ENCODING) {
// Content not compressed.
None => Ok(response),
// Content encoded (hopefully with gzip).
Some(hdr) => {
match hdr.to_str() {
Ok(val) => match val {
"gzip" => {}
"identity" => return Ok(response),
_ => return Err("Reddit response was encoded with an unsupported compressor".to_string()),
},
Err(_) => return Err("Reddit response was invalid".to_string()),
}
// We get here if the body is gzip-compressed.
// The body must be something that implements
// std::io::Read, hence the conversion to
// bytes::buf::Buf and then transformation into a
// Reader.
let mut decompressed: Vec<u8>;
{
let mut aggregated_body = match body::aggregate(response.body_mut()).await {
Ok(b) => b.reader(),
Err(e) => return Err(e.to_string()),
};
let mut decoder = match gzip::Decoder::new(&mut aggregated_body) {
Ok(decoder) => decoder,
Err(e) => return Err(e.to_string()),
};
decompressed = Vec::<u8>::new();
if let Err(e) = io::copy(&mut decoder, &mut decompressed) {
return Err(e.to_string());
};
}
response.headers_mut().remove(header::CONTENT_ENCODING);
response.headers_mut().insert(header::CONTENT_LENGTH, decompressed.len().into());
*(response.body_mut()) = Body::from(decompressed);
Ok(response)
}
let location_header = response.headers().get(wreq::header::LOCATION);
if location_header.and_then(|h| h.to_str().ok()) == Some(ALTERNATIVE_REDDIT_URL_BASE) {
return Err("Reddit response was invalid".to_string());
}
}
Err(e) => {
dbg_msg!("{method} {REDDIT_URL_BASE}{path}: {}", e);
return request(
method,
location_header
.map(|val| {
// We need to make adjustments to the URI
// we get back from Reddit. Namely, we
// must:
//
// 1. Remove the authority (e.g.
// https://www.reddit.com) that may be
// present, so that we recurse on the
// path (and query parameters) as
// required.
//
// 2. Percent-encode the path.
let new_path = percent_encode(val.as_bytes(), CONTROLS)
.to_string()
.trim_start_matches(REDDIT_URL_BASE)
.trim_start_matches(ALTERNATIVE_REDDIT_URL_BASE)
.to_string();
format!("{new_path}{}raw_json=1", if new_path.contains('?') { "&" } else { "?" })
})
.unwrap_or_default()
.to_string(),
true,
quarantine,
base_path,
host,
)
.await;
};
Err(e.to_string())
}
},
Err(_) => Err("Post url contains non-ASCII characters".to_string()),
Ok(response)
}
Err(e) => {
dbg_msg!("{method} {REDDIT_URL_BASE}{path}: {}", e);
Err(e.to_string())
}
}
}
.boxed()
@@ -406,7 +362,7 @@ pub async fn json(path: String, quarantine: bool) -> Result<Value, String> {
};
// asynchronously aggregate the chunks of the body
match hyper::body::aggregate(response).await {
match hyper::body::aggregate(response.into_hyper_response()).await {
Ok(body) => {
let has_remaining = body.has_remaining();
@@ -519,61 +475,92 @@ pub async fn rate_limit_check() -> Result<(), String> {
Ok(())
}
#[cfg(test)]
use {crate::config::get_setting, sealed_test::prelude::*};
#[tokio::test(flavor = "multi_thread")]
async fn test_rate_limit_check() {
rate_limit_check().await.unwrap();
trait IntoHyperResponse {
fn into_hyper_response(self) -> HyperResponse<Body>;
}
#[test]
#[sealed_test(env = [("REDLIB_DEFAULT_SUBSCRIPTIONS", "rust")])]
fn test_default_subscriptions() {
tokio::runtime::Builder::new_multi_thread().enable_all().build().unwrap().block_on(async {
let subscriptions = get_setting("REDLIB_DEFAULT_SUBSCRIPTIONS");
assert!(subscriptions.is_some());
impl IntoHyperResponse for WreqResponse {
fn into_hyper_response(self) -> HyperResponse<Body> {
let status = self.status();
let version = self.version();
// check rate limit
let mut builder = HyperResponse::builder().status(status.as_u16()).version(match version {
wreq::Version::HTTP_09 => hyper::Version::HTTP_09,
wreq::Version::HTTP_10 => hyper::Version::HTTP_10,
wreq::Version::HTTP_11 => hyper::Version::HTTP_11,
wreq::Version::HTTP_2 => hyper::Version::HTTP_2,
wreq::Version::HTTP_3 => hyper::Version::HTTP_3,
_ => hyper::Version::HTTP_11,
});
for (name, value) in self.headers() {
builder = builder.header(
header::HeaderName::from_bytes(name.as_str().as_bytes()).unwrap(),
header::HeaderValue::from_bytes(value.as_bytes()).unwrap(),
);
}
builder.body(Body::wrap_stream(self.bytes_stream())).unwrap()
}
}
#[cfg(test)]
mod tests {
use super::*;
use {crate::config::get_setting, sealed_test::prelude::*};
const POPULAR_URL: &str = "/r/popular/hot.json?&raw_json=1&geo_filter=GLOBAL";
#[tokio::test(flavor = "multi_thread")]
async fn test_rate_limit_check() {
rate_limit_check().await.unwrap();
});
}
}
#[cfg(test)]
const POPULAR_URL: &str = "/r/popular/hot.json?&raw_json=1&geo_filter=GLOBAL";
#[test]
#[sealed_test(env = [("REDLIB_DEFAULT_SUBSCRIPTIONS", "rust")])]
fn test_default_subscriptions() {
tokio::runtime::Builder::new_multi_thread().enable_all().build().unwrap().block_on(async {
let subscriptions = get_setting("REDLIB_DEFAULT_SUBSCRIPTIONS");
assert!(subscriptions.is_some());
#[tokio::test(flavor = "multi_thread")]
async fn test_localization_popular() {
let val = json(POPULAR_URL.to_string(), false).await.unwrap();
assert_eq!("GLOBAL", val["data"]["geo_filter"].as_str().unwrap());
}
// check rate limit
rate_limit_check().await.unwrap();
});
}
#[tokio::test(flavor = "multi_thread")]
async fn test_obfuscated_share_link() {
let share_link = "/r/rust/s/kPgq8WNHRK".into();
// Correct link without share parameters
let canonical_link = "/r/rust/comments/18t5968/why_use_tuple_struct_over_standard_struct/kfbqlbc/".into();
assert_eq!(canonical_path(share_link, 3).await, Ok(Some(canonical_link)));
}
#[tokio::test(flavor = "multi_thread")]
async fn test_localization_popular() {
let val = json(POPULAR_URL.to_string(), false).await.unwrap();
assert_eq!("GLOBAL", val["data"]["geo_filter"].as_str().unwrap());
}
#[tokio::test(flavor = "multi_thread")]
async fn test_private_sub() {
let link = json("/r/suicide/about.json?raw_json=1".into(), true).await;
assert!(link.is_err());
assert_eq!(link, Err("private".into()));
}
#[tokio::test(flavor = "multi_thread")]
async fn test_obfuscated_share_link() {
let share_link = "/r/rust/s/kPgq8WNHRK".into();
// Correct link without share parameters
let canonical_link = "/r/rust/comments/18t5968/why_use_tuple_struct_over_standard_struct/kfbqlbc/".into();
assert_eq!(canonical_path(share_link, 3).await, Ok(Some(canonical_link)));
}
#[tokio::test(flavor = "multi_thread")]
async fn test_banned_sub() {
let link = json("/r/aaa/about.json?raw_json=1".into(), true).await;
assert!(link.is_err());
assert_eq!(link, Err("banned".into()));
}
#[tokio::test(flavor = "multi_thread")]
async fn test_private_sub() {
let link = json("/r/suicide/about.json?raw_json=1".into(), true).await;
assert!(link.is_err());
assert_eq!(link, Err("private".into()));
}
#[tokio::test(flavor = "multi_thread")]
async fn test_gated_sub() {
// quarantine to false to specifically catch when we _don't_ catch it
let link = json("/r/drugs/about.json?raw_json=1".into(), false).await;
assert!(link.is_err());
assert_eq!(link, Err("gated".into()));
#[tokio::test(flavor = "multi_thread")]
async fn test_banned_sub() {
let link = json("/r/aaa/about.json?raw_json=1".into(), true).await;
assert!(link.is_err());
assert_eq!(link, Err("banned".into()));
}
#[tokio::test(flavor = "multi_thread")]
async fn test_gated_sub() {
// quarantine to false to specifically catch when we _don't_ catch it
let link = json("/r/drugs/about.json?raw_json=1".into(), false).await;
assert!(link.is_err());
assert_eq!(link, Err("gated".into()));
}
}

128
src/config.rs
View File

@@ -196,75 +196,79 @@ pub fn get_setting(name: &str) -> Option<String> {
}
#[cfg(test)]
use {sealed_test::prelude::*, std::fs::write};
mod tests {
use super::*;
use {sealed_test::prelude::*, std::fs::write};
#[test]
fn test_deserialize() {
// Must handle empty input
let result = toml::from_str::<Config>("");
assert!(result.is_ok(), "Error: {}", result.unwrap_err());
}
#[test]
fn test_deserialize() {
// Must handle empty input
let result = toml::from_str::<Config>("");
assert!(result.is_ok(), "Error: {}", result.unwrap_err());
}
#[test]
#[sealed_test(env = [("REDLIB_SFW_ONLY", "on")])]
fn test_env_var() {
assert!(crate::utils::sfw_only())
}
#[test]
#[sealed_test(env = [("REDLIB_SFW_ONLY", "on")])]
fn test_env_var() {
assert!(crate::utils::sfw_only())
}
#[test]
#[sealed_test]
fn test_config() {
let config_to_write = r#"REDLIB_DEFAULT_COMMENT_SORT = "best""#;
write("redlib.toml", config_to_write).unwrap();
assert_eq!(get_setting("REDLIB_DEFAULT_COMMENT_SORT"), Some("best".into()));
}
#[test]
#[sealed_test]
fn test_config() {
let config_to_write = r#"REDLIB_DEFAULT_COMMENT_SORT = "best""#;
write("redlib.toml", config_to_write).unwrap();
assert_eq!(get_setting("REDLIB_DEFAULT_COMMENT_SORT"), Some("best".into()));
}
#[test]
#[sealed_test]
fn test_config_legacy() {
let config_to_write = r#"LIBREDDIT_DEFAULT_COMMENT_SORT = "best""#;
write("libreddit.toml", config_to_write).unwrap();
assert_eq!(get_setting("REDLIB_DEFAULT_COMMENT_SORT"), Some("best".into()));
}
#[test]
#[sealed_test]
fn test_config_legacy() {
let config_to_write = r#"LIBREDDIT_DEFAULT_COMMENT_SORT = "best""#;
write("libreddit.toml", config_to_write).unwrap();
assert_eq!(get_setting("REDLIB_DEFAULT_COMMENT_SORT"), Some("best".into()));
}
#[test]
#[sealed_test(env = [("LIBREDDIT_SFW_ONLY", "on")])]
fn test_env_var_legacy() {
assert!(crate::utils::sfw_only())
}
#[test]
#[sealed_test(env = [("LIBREDDIT_SFW_ONLY", "on")])]
fn test_env_var_legacy() {
assert!(crate::utils::sfw_only())
}
#[test]
#[sealed_test(env = [("REDLIB_DEFAULT_COMMENT_SORT", "top")])]
fn test_env_config_precedence() {
let config_to_write = r#"REDLIB_DEFAULT_COMMENT_SORT = "best""#;
write("redlib.toml", config_to_write).unwrap();
assert_eq!(get_setting("REDLIB_DEFAULT_COMMENT_SORT"), Some("top".into()))
}
#[test]
#[sealed_test(env = [("REDLIB_DEFAULT_COMMENT_SORT", "top")])]
fn test_env_config_precedence() {
let config_to_write = r#"REDLIB_DEFAULT_COMMENT_SORT = "best""#;
write("redlib.toml", config_to_write).unwrap();
assert_eq!(get_setting("REDLIB_DEFAULT_COMMENT_SORT"), Some("top".into()))
}
#[test]
#[sealed_test(env = [("REDLIB_DEFAULT_COMMENT_SORT", "top")])]
fn test_alt_env_config_precedence() {
let config_to_write = r#"REDLIB_DEFAULT_COMMENT_SORT = "best""#;
write("redlib.toml", config_to_write).unwrap();
assert_eq!(get_setting("REDLIB_DEFAULT_COMMENT_SORT"), Some("top".into()))
}
#[test]
#[sealed_test(env = [("REDLIB_DEFAULT_SUBSCRIPTIONS", "news+bestof")])]
fn test_default_subscriptions() {
assert_eq!(get_setting("REDLIB_DEFAULT_SUBSCRIPTIONS"), Some("news+bestof".into()));
}
#[test]
#[sealed_test(env = [("REDLIB_DEFAULT_COMMENT_SORT", "top")])]
fn test_alt_env_config_precedence() {
let config_to_write = r#"REDLIB_DEFAULT_COMMENT_SORT = "best""#;
write("redlib.toml", config_to_write).unwrap();
assert_eq!(get_setting("REDLIB_DEFAULT_COMMENT_SORT"), Some("top".into()))
}
#[test]
#[sealed_test(env = [("REDLIB_DEFAULT_FILTERS", "news+bestof")])]
fn test_default_filters() {
assert_eq!(get_setting("REDLIB_DEFAULT_FILTERS"), Some("news+bestof".into()));
}
#[test]
#[sealed_test(env = [("REDLIB_DEFAULT_SUBSCRIPTIONS", "news+bestof")])]
fn test_default_subscriptions() {
assert_eq!(get_setting("REDLIB_DEFAULT_SUBSCRIPTIONS"), Some("news+bestof".into()));
}
#[test]
#[sealed_test]
fn test_pushshift() {
let config_to_write = r#"REDLIB_PUSHSHIFT_FRONTEND = "https://api.pushshift.io""#;
write("redlib.toml", config_to_write).unwrap();
assert!(get_setting("REDLIB_PUSHSHIFT_FRONTEND").is_some());
assert_eq!(get_setting("REDLIB_PUSHSHIFT_FRONTEND"), Some("https://api.pushshift.io".into()));
#[test]
#[sealed_test(env = [("REDLIB_DEFAULT_FILTERS", "news+bestof")])]
fn test_default_filters() {
assert_eq!(get_setting("REDLIB_DEFAULT_FILTERS"), Some("news+bestof".into()));
}
#[test]
#[sealed_test]
fn test_pushshift() {
let config_to_write = r#"REDLIB_PUSHSHIFT_FRONTEND = "https://api.pushshift.io""#;
write("redlib.toml", config_to_write).unwrap();
assert!(get_setting("REDLIB_PUSHSHIFT_FRONTEND").is_some());
assert_eq!(get_setting("REDLIB_PUSHSHIFT_FRONTEND"), Some("https://api.pushshift.io".into()));
}
}

14
src/main.rs
View File

@@ -4,11 +4,9 @@
use cached::proc_macro::cached;
use clap::{Arg, ArgAction, Command};
use std::str::FromStr;
use std::sync::LazyLock;
use futures_lite::FutureExt;
use hyper::Uri;
use hyper::{header::HeaderValue, Body, Request, Response};
use log::{info, warn};
use redlib::client::{canonical_path, proxy, rate_limit_check, CLIENT};
@@ -433,11 +431,9 @@ pub async fn proxy_commit_info() -> Result<Response<Body>, String> {
#[cached(time = 600)]
async fn fetch_commit_info() -> String {
let uri = Uri::from_str("https://github.com/redlib-org/redlib/commits/main.atom").expect("Invalid URI");
let url = "https://github.com/redlib-org/redlib/commits/main.atom";
let resp: Body = CLIENT.get(uri).await.expect("Failed to request GitHub").into_body();
hyper::body::to_bytes(resp).await.expect("Failed to read body").iter().copied().map(|x| x as char).collect()
CLIENT.get(url).send().await.expect("Failed to request GitHub").text().await.expect("Failed to read body")
}
pub async fn proxy_instances() -> Result<Response<Body>, String> {
@@ -452,9 +448,7 @@ pub async fn proxy_instances() -> Result<Response<Body>, String> {
#[cached(time = 600)]
async fn fetch_instances() -> String {
let uri = Uri::from_str("https://raw.githubusercontent.com/redlib-org/redlib-instances/refs/heads/main/instances.json").expect("Invalid URI");
let url = "https://raw.githubusercontent.com/redlib-org/redlib-instances/refs/heads/main/instances.json";
let resp: Body = CLIENT.get(uri).await.expect("Failed to request GitHub").into_body();
hyper::body::to_bytes(resp).await.expect("Failed to read body").iter().copied().map(|x| x as char).collect()
CLIENT.get(url).send().await.expect("Failed to request GitHub").text().await.expect("Failed to read body")
}

161
src/oauth.rs
View File

@@ -1,13 +1,11 @@
use std::{collections::HashMap, sync::atomic::Ordering, time::Duration};
use crate::{
client::{CLIENT, OAUTH_CLIENT, OAUTH_IS_ROLLING_OVER, OAUTH_RATELIMIT_REMAINING},
oauth_resources::ANDROID_APP_VERSION_LIST,
};
use base64::{engine::general_purpose, Engine as _};
use hyper::{client, Body, Method, Request};
use log::{error, info, trace, warn};
use serde_json::json;
use std::{collections::HashMap, sync::atomic::Ordering, time::Duration};
use tegen::tegen::TextGenerator;
use tokio::time::{error::Elapsed, timeout};
@@ -88,7 +86,7 @@ impl Oauth {
error!(
"[⛔] Failed to create OAuth client: {}. Retrying in 5 seconds...",
match e {
AuthError::Hyper(error) => error.to_string(),
AuthError::Wreq(error) => error.to_string(),
AuthError::SerdeDeserialize(error) => error.to_string(),
AuthError::Field((value, error)) => format!("{error}\n{value}"),
}
@@ -142,14 +140,14 @@ impl Oauth {
#[derive(Debug)]
enum AuthError {
Hyper(hyper::Error),
Wreq(wreq::Error),
SerdeDeserialize(serde_json::Error),
Field((serde_json::Value, &'static str)),
}
impl From<hyper::Error> for AuthError {
fn from(err: hyper::Error) -> Self {
AuthError::Hyper(err)
impl From<wreq::Error> for AuthError {
fn from(err: wreq::Error) -> Self {
AuthError::Wreq(err)
}
}
@@ -222,7 +220,7 @@ impl OauthBackend for MobileSpoofAuth {
async fn authenticate(&mut self) -> Result<OauthResponse, AuthError> {
// Construct URL for OAuth token
let url = format!("{AUTH_ENDPOINT}/auth/v2/oauth/access-token/loid");
let mut builder = Request::builder().method(Method::POST).uri(&url);
let mut builder = CLIENT.post(&url);
// Add headers from spoofed client
for (key, value) in &self.device.initial_headers {
@@ -239,16 +237,11 @@ impl OauthBackend for MobileSpoofAuth {
let json = json!({
"scopes": ["*","email", "pii"]
});
let body = Body::from(json.to_string());
// Build request
let request = builder.body(body).unwrap();
trace!("Sending token request...\n\n{request:?}");
trace!("Sending token request to {url}...");
// Send request
let client: &std::sync::LazyLock<client::Client<_, Body>> = &CLIENT;
let resp = client.request(request).await?;
let resp = builder.json(&json).send().await?;
trace!("Received response with status {} and length {:?}", resp.status(), resp.headers().get("content-length"));
trace!("OAuth headers: {:#?}", resp.headers());
@@ -259,19 +252,20 @@ impl OauthBackend for MobileSpoofAuth {
// Not worried about the privacy implications, since this is randomly changed
// and really only as privacy-concerning as the OAuth token itself.
if let Some(header) = resp.headers().get("x-reddit-loid") {
self.additional_headers.insert("x-reddit-loid".to_owned(), header.to_str().unwrap().to_string());
let header_val: &wreq::header::HeaderValue = header;
self.additional_headers.insert("x-reddit-loid".to_owned(), header_val.to_str().unwrap().to_string());
}
// Same with x-reddit-session
if let Some(header) = resp.headers().get("x-reddit-session") {
self.additional_headers.insert("x-reddit-session".to_owned(), header.to_str().unwrap().to_string());
let header_val: &wreq::header::HeaderValue = header;
self.additional_headers.insert("x-reddit-session".to_owned(), header_val.to_str().unwrap().to_string());
}
trace!("Serializing response...");
// Serialize response
let body_bytes = hyper::body::to_bytes(resp.into_body()).await?;
let json: serde_json::Value = serde_json::from_slice(&body_bytes).map_err(AuthError::SerdeDeserialize)?;
let json: serde_json::Value = resp.json().await?;
trace!("Accessing relevant fields...");
@@ -341,7 +335,7 @@ impl OauthBackend for GenericWebAuth {
async fn authenticate(&mut self) -> Result<OauthResponse, AuthError> {
// Construct URL for OAuth token
let url = "https://www.reddit.com/api/v1/access_token";
let mut builder = Request::builder().method(Method::POST).uri(url);
let mut builder = CLIENT.post(url);
// Add minimal headers
builder = builder.header("Host", "www.reddit.com");
@@ -356,16 +350,11 @@ impl OauthBackend for GenericWebAuth {
// Set up form body
let body_str = format!("grant_type=https%3A%2F%2Foauth.reddit.com%2Fgrants%2Finstalled_client&device_id={}", self.device_id);
let body = Body::from(body_str);
// Build request
let request = builder.body(body).unwrap();
trace!("Sending GenericWebAuth token request...\n\n{request:?}");
trace!("Sending GenericWebAuth token request to {url}...");
// Send request
let client: &std::sync::LazyLock<client::Client<_, Body>> = &CLIENT;
let resp = client.request(request).await?;
let resp: wreq::Response = builder.body(body_str).send().await?;
trace!("Received response with status {} and length {:?}", resp.status(), resp.headers().get("content-length"));
trace!("GenericWebAuth headers: {:#?}", resp.headers());
@@ -376,19 +365,20 @@ impl OauthBackend for GenericWebAuth {
// Not worried about the privacy implications, since this is randomly changed
// and really only as privacy-concerning as the OAuth token itself.
if let Some(header) = resp.headers().get("x-reddit-loid") {
self.additional_headers.insert("x-reddit-loid".to_owned(), header.to_str().unwrap().to_string());
let header_val: &wreq::header::HeaderValue = header;
self.additional_headers.insert("x-reddit-loid".to_owned(), header_val.to_str().unwrap().to_string());
}
// Same with x-reddit-session
if let Some(header) = resp.headers().get("x-reddit-session") {
self.additional_headers.insert("x-reddit-session".to_owned(), header.to_str().unwrap().to_string());
let header_val: &wreq::header::HeaderValue = header;
self.additional_headers.insert("x-reddit-session".to_owned(), header_val.to_str().unwrap().to_string());
}
trace!("Serializing GenericWebAuth response...");
// Serialize response
let body_bytes = hyper::body::to_bytes(resp.into_body()).await?;
let json: serde_json::Value = serde_json::from_slice(&body_bytes).map_err(AuthError::SerdeDeserialize)?;
let json: serde_json::Value = resp.json().await?;
trace!("Accessing relevant fields...");
@@ -479,62 +469,67 @@ fn choose<T: Copy>(list: &[T]) -> T {
*fastrand::choose_multiple(list.iter(), 1)[0]
}
#[tokio::test(flavor = "multi_thread")]
async fn test_mobile_spoof_backend() {
// Test MobileSpoofAuth backend specifically
let mut backend = MobileSpoofAuth::new();
let response = backend.authenticate().await;
assert!(response.is_ok());
let response = response.unwrap();
assert!(!response.token.is_empty());
assert!(response.expires_in > 0);
assert!(!backend.user_agent().is_empty());
assert!(!backend.get_headers().is_empty());
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test(flavor = "multi_thread")]
async fn test_generic_web_backend() {
// Test GenericWebAuth backend specifically
let mut backend = GenericWebAuth::new();
let response = backend.authenticate().await;
assert!(response.is_ok());
let response = response.unwrap();
assert!(!response.token.is_empty());
assert!(response.expires_in > 0);
assert!(!backend.user_agent().is_empty());
}
#[tokio::test(flavor = "multi_thread")]
async fn test_mobile_spoof_backend() {
// Test MobileSpoofAuth backend specifically
let mut backend = MobileSpoofAuth::new();
let response = backend.authenticate().await;
assert!(response.is_ok());
let response = response.unwrap();
assert!(!response.token.is_empty());
assert!(response.expires_in > 0);
assert!(!backend.user_agent().is_empty());
assert!(!backend.get_headers().is_empty());
}
#[tokio::test(flavor = "multi_thread")]
async fn test_oauth_client() {
// Integration test - tests the overall Oauth client
assert!(OAUTH_CLIENT.load_full().headers_map.contains_key("Authorization"));
}
#[tokio::test(flavor = "multi_thread")]
async fn test_generic_web_backend() {
// Test GenericWebAuth backend specifically
let mut backend = GenericWebAuth::new();
let response = backend.authenticate().await;
assert!(response.is_ok());
let response = response.unwrap();
assert!(!response.token.is_empty());
assert!(response.expires_in > 0);
assert!(!backend.user_agent().is_empty());
}
#[tokio::test(flavor = "multi_thread")]
async fn test_oauth_client_refresh() {
force_refresh_token().await;
}
#[tokio::test(flavor = "multi_thread")]
async fn test_oauth_client() {
// Integration test - tests the overall Oauth client
assert!(OAUTH_CLIENT.load_full().headers_map.contains_key("Authorization"));
}
#[tokio::test(flavor = "multi_thread")]
async fn test_oauth_token_exists() {
let client = OAUTH_CLIENT.load_full();
let auth_header = client.headers_map.get("Authorization").unwrap();
assert!(auth_header.starts_with("Bearer "));
}
#[tokio::test(flavor = "multi_thread")]
async fn test_oauth_client_refresh() {
force_refresh_token().await;
}
#[tokio::test(flavor = "multi_thread")]
async fn test_oauth_headers_len() {
assert!(OAUTH_CLIENT.load_full().headers_map.len() >= 3);
}
#[tokio::test(flavor = "multi_thread")]
async fn test_oauth_token_exists() {
let client = OAUTH_CLIENT.load_full();
let auth_header = client.headers_map.get("Authorization").unwrap();
assert!(auth_header.starts_with("Bearer "));
}
#[test]
fn test_creating_device() {
Device::new();
}
#[tokio::test(flavor = "multi_thread")]
async fn test_oauth_headers_len() {
assert!(OAUTH_CLIENT.load_full().headers_map.len() >= 3);
}
#[test]
fn test_creating_backends() {
// Test that both backends can be created
MobileSpoofAuth::new();
GenericWebAuth::new();
#[test]
fn test_creating_device() {
Device::new();
}
#[test]
fn test_creating_backends() {
// Test that both backends can be created
MobileSpoofAuth::new();
GenericWebAuth::new();
}
}

5
src/post.rs
View File

@@ -1,6 +1,4 @@
#![allow(clippy::cmp_owned)]
// CRATES
use crate::client::json;
use crate::config::get_setting;
use crate::server::RequestExt;
@@ -8,9 +6,8 @@ use crate::subreddit::{can_access_quarantine, quarantine};
use crate::utils::{
error, format_num, get_filters, nsfw_landing, param, parse_post, rewrite_emotes, setting, template, time, val, Author, Awards, Comment, Flair, FlairPart, Post, Preferences,
};
use hyper::{Body, Request, Response};
use askama::Template;
use hyper::{Body, Request, Response};
use regex::Regex;
use std::collections::{HashMap, HashSet};
use std::sync::LazyLock;

2
src/search.rs
View File

@@ -1,6 +1,4 @@
#![allow(clippy::cmp_owned)]
// CRATES
use crate::utils::{self, catch_random, error, filter_posts, format_num, format_url, get_filters, param, redirect, setting, template, val, Post, Preferences};
use crate::{
client::json,

30
src/subreddit.rs
View File

@@ -1,11 +1,10 @@
#![allow(clippy::cmp_owned)]
use crate::{config, utils};
// CRATES
use crate::utils::{
Post, Preferences, Subreddit, catch_random, error, filter_posts, format_num, format_url, get_filters, info, nsfw_landing, param, redirect, rewrite_urls, setting, template, to_absolute_url, val
};
use crate::{client::json, server::RequestExt, server::ResponseExt};
use crate::{config, utils};
use askama::Template;
use cookie::Cookie;
use htmlescape::decode_html;
@@ -716,16 +715,21 @@ fn get_mime_type(url: &str) -> &'static str {
}
}
#[tokio::test(flavor = "multi_thread")]
async fn test_fetching_subreddit() {
let subreddit = subreddit("rust", false).await;
assert!(subreddit.is_ok());
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test(flavor = "multi_thread")]
async fn test_gated_and_quarantined() {
let quarantined = subreddit("edgy", true).await;
assert!(quarantined.is_ok());
let gated = subreddit("drugs", true).await;
assert!(gated.is_ok());
#[tokio::test(flavor = "multi_thread")]
async fn test_fetching_subreddit() {
let subreddit = subreddit("rust", false).await;
assert!(subreddit.is_ok());
}
#[tokio::test(flavor = "multi_thread")]
async fn test_gated_and_quarantined() {
let quarantined = subreddit("edgy", true).await;
assert!(quarantined.is_ok());
let gated = subreddit("drugs", true).await;
assert!(gated.is_ok());
}
}

19
src/user.rs
View File

@@ -1,6 +1,4 @@
#![allow(clippy::cmp_owned)]
// CRATES
use crate::client::json;
use crate::server::RequestExt;
use crate::utils::{error, filter_posts, format_url, get_filters, nsfw_landing, param, setting, template, Post, Preferences, User};
@@ -58,7 +56,7 @@ pub async fn profile(req: Request<Body>) -> Result<Response<Body>, String> {
// Return landing page if this post if this Reddit deems this user NSFW,
// but we have also disabled the display of NSFW content or if the instance
// is SFW-only.
if user.nsfw && crate::utils::should_be_nsfw_gated(&req, &req_url) {
if user.nsfw && utils::should_be_nsfw_gated(&req, &req_url) {
return Ok(nsfw_landing(req, req_url).await.unwrap_or_default());
}
@@ -185,9 +183,14 @@ pub async fn rss(req: Request<Body>) -> Result<Response<Body>, String> {
Ok(res)
}
#[tokio::test(flavor = "multi_thread")]
async fn test_fetching_user() {
let user = user("spez").await;
assert!(user.is_ok());
assert!(user.unwrap().karma > 100);
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test(flavor = "multi_thread")]
async fn test_fetching_user() {
let user = user("spez").await;
assert!(user.is_ok());
assert!(user.unwrap().karma > 100);
}
}

223
src/utils.rs
View File

@@ -2,9 +2,6 @@
#![allow(clippy::cmp_owned)]
use crate::config::{self, get_setting};
//
// CRATES
//
use crate::{client::json, server::RequestExt};
use askama::Template;
use cookie::Cookie;
@@ -1454,7 +1451,7 @@ pub fn to_absolute_url(relative_path: &str) -> String {
#[cfg(test)]
mod tests {
use super::{format_num, format_url, rewrite_urls, Preferences};
use super::{deflate_compress, deflate_decompress, format_num, format_url, render_bullet_lists, rewrite_emotes, rewrite_urls, url_path_basename, Post, Preferences};
#[test]
fn format_num_works() {
@@ -1551,76 +1548,75 @@ mod tests {
assert_eq!(urlencoded, "theme=laserwave&front_page=default&layout=compact&wide=on&blur_spoiler=on&show_nsfw=off&blur_nsfw=on&hide_hls_notification=off&video_quality=best&hide_sidebar_and_summary=off&use_hls=on&autoplay_videos=on&fixed_navbar=on&disable_visit_reddit_confirmation=on&comment_sort=confidence&post_sort=top&subscriptions=memes%2Bmildlyinteresting&filters=&hide_awards=off&hide_score=off&remove_default_feeds=off");
}
}
#[test]
fn test_rewriting_emoji() {
let input = r#"<div class="md"><p>How can you have such hard feelings towards a license? <img src="https://www.redditstatic.com/marketplace-assets/v1/core/emotes/snoomoji_emotes/free_emotes_pack/shrug.gif" width="20" height="20" style="vertical-align:middle"> Let people use what license they want, and BSD is one of the least restrictive ones AFAIK.</p>"#;
let output = r#"<div class="md"><p>How can you have such hard feelings towards a license? <img src="/static/marketplace-assets/v1/core/emotes/snoomoji_emotes/free_emotes_pack/shrug.gif" width="20" height="20" style="vertical-align:middle"> Let people use what license they want, and BSD is one of the least restrictive ones AFAIK.</p>"#;
assert_eq!(rewrite_urls(input), output);
}
#[tokio::test(flavor = "multi_thread")]
async fn test_fetching_subreddit_quarantined() {
let subreddit = Post::fetch("/r/drugs", true).await;
assert!(subreddit.is_ok());
assert!(!subreddit.unwrap().0.is_empty());
}
#[tokio::test(flavor = "multi_thread")]
async fn test_fetching_nsfw_subreddit() {
// Gonwild is a place for closed, Euclidean Geometric shapes to exchange their nth terms for karma; showing off their edges in a comfortable environment without pressure.
// Find a good sub that is tagged NSFW but that actually isn't in case my future employers are watching (they probably are)
// switched from randnsfw as it is no longer functional.
let subreddit = Post::fetch("/r/gonwild", false).await;
assert!(subreddit.is_ok());
assert!(!subreddit.unwrap().0.is_empty());
}
#[tokio::test(flavor = "multi_thread")]
async fn test_fetching_ws() {
let subreddit = Post::fetch("/r/popular", false).await;
assert!(subreddit.is_ok());
for post in subreddit.unwrap().0 {
assert!(post.ws_url.starts_with("wss://k8s-lb.wss.redditmedia.com/link/"));
#[test]
fn test_rewriting_emoji() {
let input = r#"<div class="md"><p>How can you have such hard feelings towards a license? <img src="https://www.redditstatic.com/marketplace-assets/v1/core/emotes/snoomoji_emotes/free_emotes_pack/shrug.gif" width="20" height="20" style="vertical-align:middle"> Let people use what license they want, and BSD is one of the least restrictive ones AFAIK.</p>"#;
let output = r#"<div class="md"><p>How can you have such hard feelings towards a license? <img src="/static/marketplace-assets/v1/core/emotes/snoomoji_emotes/free_emotes_pack/shrug.gif" width="20" height="20" style="vertical-align:middle"> Let people use what license they want, and BSD is one of the least restrictive ones AFAIK.</p>"#;
assert_eq!(rewrite_urls(input), output);
}
}
#[test]
fn test_rewriting_image_links() {
let input =
r#"<p><a href="https://preview.redd.it/6awags382xo31.png?width=2560&amp;format=png&amp;auto=webp&amp;s=9c563aed4f07a91bdd249b5a3cea43a79710dcfc">caption 1</a></p>"#;
let output = r#"<figure><a href="/preview/pre/6awags382xo31.png?width=2560&amp;format=png&amp;auto=webp&amp;s=9c563aed4f07a91bdd249b5a3cea43a79710dcfc"><img loading="lazy" src="/preview/pre/6awags382xo31.png?width=2560&amp;format=png&amp;auto=webp&amp;s=9c563aed4f07a91bdd249b5a3cea43a79710dcfc"></a><figcaption>caption 1</figcaption></figure>"#;
assert_eq!(rewrite_urls(input), output);
}
#[tokio::test(flavor = "multi_thread")]
async fn test_fetching_subreddit_quarantined() {
let subreddit = Post::fetch("/r/drugs", true).await;
assert!(subreddit.is_ok());
assert!(!subreddit.unwrap().0.is_empty());
}
#[test]
fn test_url_path_basename() {
// without trailing slash
assert_eq!(url_path_basename("/first/last"), "last");
// with trailing slash
assert_eq!(url_path_basename("/first/last/"), "last");
// with query parameters
assert_eq!(url_path_basename("/first/last/?some=query"), "last");
// file path
assert_eq!(url_path_basename("/cdn/image.jpg"), "image.jpg");
// when a full url is passed instead of just a path
assert_eq!(url_path_basename("https://doma.in/first/last"), "last");
// empty path
assert_eq!(url_path_basename("/"), "");
}
#[tokio::test(flavor = "multi_thread")]
async fn test_fetching_nsfw_subreddit() {
// Gonwild is a place for closed, Euclidean Geometric shapes to exchange their nth terms for karma; showing off their edges in a comfortable environment without pressure.
// Find a good sub that is tagged NSFW but that actually isn't in case my future employers are watching (they probably are)
// switched from randnsfw as it is no longer functional.
let subreddit = Post::fetch("/r/gonwild", false).await;
assert!(subreddit.is_ok());
assert!(!subreddit.unwrap().0.is_empty());
}
#[test]
fn test_rewriting_emotes() {
let json_input = serde_json::from_str(r#"{"emote|t5_31hpy|2028":{"e":"Image","id":"emote|t5_31hpy|2028","m":"image/png","s":{"u":"https://reddit-econ-prod-assets-permanent.s3.amazonaws.com/asset-manager/t5_31hpy/PW6WsOaLcd.png","x":60,"y":60},"status":"valid","t":"sticker"}}"#).expect("Valid JSON");
let comment_input = r#"<div class="comment_body "><div class="md"><p>:2028:</p></div></div>"#;
let output = r#"<div class="comment_body "><div class="md"><p><img loading="lazy" src="/emote/t5_31hpy/PW6WsOaLcd.png" width="60" height="60" style="vertical-align:text-bottom"></p></div></div>"#;
assert_eq!(rewrite_emotes(&json_input, comment_input.to_string()), output);
}
#[tokio::test(flavor = "multi_thread")]
async fn test_fetching_ws() {
let subreddit = Post::fetch("/r/popular", false).await;
assert!(subreddit.is_ok());
for post in subreddit.unwrap().0 {
assert!(post.ws_url.starts_with("wss://k8s-lb.wss.redditmedia.com/link/"));
}
}
#[test]
fn test_rewriting_bullet_list() {
let input = r#"<div class="md"><p>Hi, I&#39;ve bought this very same monitor and found no calibration whatsoever. I have an ICC profile that has been set up since I&#39;ve installed its driver from the LG website and it works ok. I also used <a href="http://www.lagom.nl/lcd-test/">http://www.lagom.nl/lcd-test/</a> to calibrate it. After some good tinkering I&#39;ve found the following settings + the color profile from the driver gets me past all the tests perfectly:
#[test]
fn test_rewriting_image_links() {
let input =
r#"<p><a href="https://preview.redd.it/6awags382xo31.png?width=2560&amp;format=png&amp;auto=webp&amp;s=9c563aed4f07a91bdd249b5a3cea43a79710dcfc">caption 1</a></p>"#;
let output = r#"<figure><a href="/preview/pre/6awags382xo31.png?width=2560&amp;format=png&amp;auto=webp&amp;s=9c563aed4f07a91bdd249b5a3cea43a79710dcfc"><img loading="lazy" src="/preview/pre/6awags382xo31.png?width=2560&amp;format=png&amp;auto=webp&amp;s=9c563aed4f07a91bdd249b5a3cea43a79710dcfc"></a><figcaption>caption 1</figcaption></figure>"#;
assert_eq!(rewrite_urls(input), output);
}
#[test]
fn test_url_path_basename() {
// without trailing slash
assert_eq!(url_path_basename("/first/last"), "last");
// with trailing slash
assert_eq!(url_path_basename("/first/last/"), "last");
// with query parameters
assert_eq!(url_path_basename("/first/last/?some=query"), "last");
// file path
assert_eq!(url_path_basename("/cdn/image.jpg"), "image.jpg");
// when a full url is passed instead of just a path
assert_eq!(url_path_basename("https://doma.in/first/last"), "last");
// empty path
assert_eq!(url_path_basename("/"), "");
}
#[test]
fn test_rewriting_emotes() {
let json_input = serde_json::from_str(r#"{"emote|t5_31hpy|2028":{"e":"Image","id":"emote|t5_31hpy|2028","m":"image/png","s":{"u":"https://reddit-econ-prod-assets-permanent.s3.amazonaws.com/asset-manager/t5_31hpy/PW6WsOaLcd.png","x":60,"y":60},"status":"valid","t":"sticker"}}"#).expect("Valid JSON");
let comment_input = r#"<div class="comment_body "><div class="md"><p>:2028:</p></div></div>"#;
let output = r#"<div class="comment_body "><div class="md"><p><img loading="lazy" src="/emote/t5_31hpy/PW6WsOaLcd.png" width="60" height="60" style="vertical-align:text-bottom"></p></div></div>"#;
assert_eq!(rewrite_emotes(&json_input, comment_input.to_string()), output);
}
#[test]
fn test_rewriting_bullet_list() {
let input = r#"<div class="md"><p>Hi, I&#39;ve bought this very same monitor and found no calibration whatsoever. I have an ICC profile that has been set up since I&#39;ve installed its driver from the LG website and it works ok. I also used <a href="http://www.lagom.nl/lcd-test/">http://www.lagom.nl/lcd-test/</a> to calibrate it. After some good tinkering I&#39;ve found the following settings + the color profile from the driver gets me past all the tests perfectly:
- Brightness 50 (still have to settle on this one, it&#39;s personal preference, it controls the backlight, not the colors)
- Contrast 70 (which for me was the default one)
- Picture mode Custom
@@ -1635,59 +1631,60 @@ fn test_rewriting_bullet_list() {
- Color Temp Medium
How`s your monitor by the way? Any IPS bleed whatsoever? I either got lucky or the panel is pretty good, 0 bleed for me, just the usual IPS glow. How about the pixels? I see the pixels even at one meter away, especially on Microsoft Edge&#39;s icon for example, the blue background is just blocky, don&#39;t know why.</p>
</div>"#;
let output = r#"<div class="md"><p>Hi, I&#39;ve bought this very same monitor and found no calibration whatsoever. I have an ICC profile that has been set up since I&#39;ve installed its driver from the LG website and it works ok. I also used <a href="http://www.lagom.nl/lcd-test/">http://www.lagom.nl/lcd-test/</a> to calibrate it. After some good tinkering I&#39;ve found the following settings + the color profile from the driver gets me past all the tests perfectly:
let output = r#"<div class="md"><p>Hi, I&#39;ve bought this very same monitor and found no calibration whatsoever. I have an ICC profile that has been set up since I&#39;ve installed its driver from the LG website and it works ok. I also used <a href="http://www.lagom.nl/lcd-test/">http://www.lagom.nl/lcd-test/</a> to calibrate it. After some good tinkering I&#39;ve found the following settings + the color profile from the driver gets me past all the tests perfectly:
<ul><li>Brightness 50 (still have to settle on this one, it&#39;s personal preference, it controls the backlight, not the colors)</li><li>Contrast 70 (which for me was the default one)</li><li>Picture mode Custom</li><li>Super resolution + Off (it looks horrible anyway)</li><li>Sharpness 50 (default one I think)</li><li>Black level High (low messes up gray colors)</li><li>DFC Off</li><li>Response Time Middle (personal preference, <a href="https://www.blurbusters.com/">https://www.blurbusters.com/</a> show horrible overdrive with it on high)</li><li>Freesync doesn&#39;t matter</li><li>Black stabilizer 50</li><li>Gamma setting on 0</li><li>Color Temp Medium</li></ul>
How`s your monitor by the way? Any IPS bleed whatsoever? I either got lucky or the panel is pretty good, 0 bleed for me, just the usual IPS glow. How about the pixels? I see the pixels even at one meter away, especially on Microsoft Edge&#39;s icon for example, the blue background is just blocky, don&#39;t know why.</p>
</div>"#;
assert_eq!(render_bullet_lists(input), output);
}
#[test]
fn test_default_prefs_serialization_loop_json() {
let prefs = Preferences::default();
let serialized = serde_json::to_string(&prefs).unwrap();
let deserialized: Preferences = serde_json::from_str(&serialized).unwrap();
assert_eq!(prefs, deserialized);
}
#[test]
fn test_default_prefs_serialization_loop_bincode() {
let prefs = Preferences::default();
test_round_trip(&prefs, false);
test_round_trip(&prefs, true);
}
static KNOWN_GOOD_CONFIGS: &[&str] = &[
"ఴӅβØØҞÉဏႢձĬ༧ȒʯऌԔӵ୮༏",
"ਧՊΥÀÃǎƱГ۸ඣമĖฤ႙ʟาúໜϾௐɥঀĜໃહཞઠѫҲɂఙ࿔DzઉƲӟӻĻฅΜδ໖ԜǗဖငƦơ৶Ą௩ԹʛใЛʃශаΏ",
"ਧԩΥÀÃΊ౭൩ඔႠϼҭöҪƸռઇԾॐნɔາǒՍҰच௨ಖມŃЉŐདƦ๙ϩএఠȝഽйʮჯඒϰळՋ௮ສ৵ऎΦѧਹಧଟƙŃ३î༦ŌပղयƟแҜ།",
];
#[test]
fn test_known_good_configs_deserialization() {
for config in KNOWN_GOOD_CONFIGS {
let bytes = base2048::decode(config).unwrap();
let decompressed = deflate_decompress(bytes).unwrap();
assert!(bincode::deserialize::<Preferences>(&decompressed).is_ok());
assert_eq!(render_bullet_lists(input), output);
}
}
#[test]
fn test_known_good_configs_full_round_trip() {
for config in KNOWN_GOOD_CONFIGS {
let bytes = base2048::decode(config).unwrap();
let decompressed = deflate_decompress(bytes).unwrap();
let prefs: Preferences = bincode::deserialize(&decompressed).unwrap();
#[test]
fn test_default_prefs_serialization_loop_json() {
let prefs = Preferences::default();
let serialized = serde_json::to_string(&prefs).unwrap();
let deserialized: Preferences = serde_json::from_str(&serialized).unwrap();
assert_eq!(prefs, deserialized);
}
#[test]
fn test_default_prefs_serialization_loop_bincode() {
let prefs = Preferences::default();
test_round_trip(&prefs, false);
test_round_trip(&prefs, true);
}
}
fn test_round_trip(input: &Preferences, compression: bool) {
let serialized = bincode::serialize(input).unwrap();
let compressed = if compression { deflate_compress(serialized).unwrap() } else { serialized };
let decompressed = if compression { deflate_decompress(compressed).unwrap() } else { compressed };
let deserialized: Preferences = bincode::deserialize(&decompressed).unwrap();
assert_eq!(*input, deserialized);
static KNOWN_GOOD_CONFIGS: &[&str] = &[
"ఴӅβØØҞÉဏႢձĬ༧ȒʯऌԔӵ୮༏",
"ਧՊΥÀÃǎƱГ۸ඣമĖฤ႙ʟาúໜϾௐɥঀĜໃહཞઠѫҲɂఙ࿔DzઉƲӟӻĻฅΜδ໖ԜǗဖငƦơ৶Ą௩ԹʛใЛʃශаΏ",
"ਧԩΥÀÃΊ౭൩ඔႠϼҭöҪƸռઇԾॐნɔາǒՍҰच௨ಖມŃЉŐདƦ๙ϩএఠȝഽйʮჯඒϰळՋ௮ສ৵ऎΦѧਹಧଟƙŃ३î༦ŌပղयƟแҜ།",
];
#[test]
fn test_known_good_configs_deserialization() {
for config in KNOWN_GOOD_CONFIGS {
let bytes = base2048::decode(config).unwrap();
let decompressed = deflate_decompress(bytes).unwrap();
assert!(bincode::deserialize::<Preferences>(&decompressed).is_ok());
}
}
#[test]
fn test_known_good_configs_full_round_trip() {
for config in KNOWN_GOOD_CONFIGS {
let bytes = base2048::decode(config).unwrap();
let decompressed = deflate_decompress(bytes).unwrap();
let prefs: Preferences = bincode::deserialize(&decompressed).unwrap();
test_round_trip(&prefs, false);
test_round_trip(&prefs, true);
}
}
fn test_round_trip(input: &Preferences, compression: bool) {
let serialized = bincode::serialize(input).unwrap();
let compressed = if compression { deflate_compress(serialized).unwrap() } else { serialized };
let decompressed = if compression { deflate_decompress(compressed).unwrap() } else { compressed };
let deserialized: Preferences = bincode::deserialize(&decompressed).unwrap();
assert_eq!(*input, deserialized);
}
}