luckyimnotanigger/redlib
1
0
Fork 0
mirror of https://github.com/redlib-org/redlib.git synced 2025-06-19 07:30:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/pull/819'

Browse source
This commit is contained in:
Matthew Esposito 2023-12-26 15:48:27 -05:00
commit 90d1831352
No known key found for this signature in database
8 changed files with 402 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

37
src/client.rs
View file

@ -1,4 +1,5 @@
use cached::proc_macro::cached;
use futures_lite::future::block_on;
use futures_lite::{future::Boxed, FutureExt};
use hyper::client::HttpConnector;
use hyper::{body, body::Buf, client, header, Body, Client, Method, Request, Response, Uri};
@ -7,19 +8,28 @@ use libflate::gzip;
use once_cell::sync::Lazy;
use percent_encoding::{percent_encode, CONTROLS};
use serde_json::Value;
use std::{io, result::Result, sync::atomic::Ordering::SeqCst};
use crate::instance_info::INSTANCE_INFO;
use std::{io, result::Result};
use tokio::sync::RwLock;
use crate::dbg_msg;
use crate::oauth::{token_daemon, Oauth};
use crate::server::RequestExt;
use crate::{config, dbg_msg};
const REDDIT_URL_BASE: &str = "https://www.reddit.com";
const REDDIT_URL_BASE: &str = "https://oauth.reddit.com";
static CLIENT: Lazy<Client<HttpsConnector<HttpConnector>>> = Lazy::new(|| {
pub(crate) static CLIENT: Lazy<Client<HttpsConnector<HttpConnector>>> = Lazy::new(|| {
let https = hyper_rustls::HttpsConnectorBuilder::new().with_native_roots().https_only().enable_http1().build();
client::Client::builder().build(https)
});
pub(crate) static OAUTH_CLIENT: Lazy<RwLock<Oauth>> = Lazy::new(|| {
let client = block_on(Oauth::new());
tokio::spawn(token_daemon());
RwLock::new(client)
});
/// Gets the canonical path for a resource on Reddit. This is accomplished by
/// making a `HEAD` request to Reddit at the path given in `path`.
///
@ -135,14 +145,27 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo
// Construct the hyper client from the HTTPS connector.
let client: client::Client<_, hyper::Body> = CLIENT.clone();
let (token, vendor_id, device_id, user_agent, loid) = {
let client = block_on(OAUTH_CLIENT.read());
(
client.token.clone(),
client.headers_map.get("Client-Vendor-Id").cloned().unwrap_or_default(),
client.headers_map.get("X-Reddit-Device-Id").cloned().unwrap_or_default(),
client.headers_map.get("User-Agent").cloned().unwrap_or_default(),
client.headers_map.get("x-reddit-loid").cloned().unwrap_or_default(),
)
};
// Build request to Reddit. When making a GET, request gzip compression.
// (Reddit doesn't do brotli yet.)
let builder = Request::builder()
.method(method)
.uri(&url)
.header("User-Agent", concat!("web:libreddit:", env!("CARGO_PKG_VERSION")))
.header("Host", "www.reddit.com")
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
.header("User-Agent", user_agent)
.header("Client-Vendor-Id", vendor_id)
.header("X-Reddit-Device-Id", device_id)
.header("x-reddit-loid", loid)
.header("Host", "oauth.reddit.com")
.header("Authorization", &format!("Bearer {}", token))
.header("Accept-Encoding", if method == Method::GET { "gzip" } else { "identity" })
.header("Accept-Language", "en-US,en;q=0.5")
.header("Connection", "keep-alive")

18
src/main.rs
View file

@ -6,6 +6,7 @@
mod config;
mod duplicates;
mod instance_info;
mod oauth;
mod post;
mod search;
mod settings;
@ -21,10 +22,13 @@ use hyper::{header::HeaderValue, Body, Request, Response};
mod client;
use client::{canonical_path, proxy};
use log::info;
use once_cell::sync::Lazy;
use server::RequestExt;
use utils::{error, redirect, ThemeAssets};
use crate::client::OAUTH_CLIENT;
mod server;
// Create Services
@ -108,6 +112,12 @@ async fn style() -> Result<Response<Body>, String> {
#[tokio::main]
async fn main() {
// Load environment variables
_ = dotenvy::dotenv();
// Initialize logger
pretty_env_logger::init();
let matches = Command::new("Libreddit")
.version(env!("CARGO_PKG_VERSION"))
.about("Private front-end for Reddit written in Rust ")
@ -162,10 +172,16 @@ async fn main() {
// Force evaluation of statics. In instance_info case, we need to evaluate
// the timestamp so deploy date is accurate - in config case, we need to
// evaluate the configuration to avoid paying penalty at first request.
// evaluate the configuration to avoid paying penalty at first request -
// in OAUTH case, we need to retrieve the token to avoid paying penalty
// at first request
info!("Evaluating config.");
Lazy::force(&config::CONFIG);
info!("Evaluating instance info.");
Lazy::force(&instance_info::INSTANCE_INFO);
info!("Creating OAUTH client.");
Lazy::force(&OAUTH_CLIENT);
// Define default headers (added to all responses)
app.default_headers = headers! {

226
src/oauth.rs Normal file
View file

@ -0,0 +1,226 @@
use std::{collections::HashMap, time::Duration};
use crate::client::{CLIENT, OAUTH_CLIENT};
use base64::{engine::general_purpose, Engine as _};
use hyper::{client, Body, Method, Request};
use log::info;
use serde_json::json;
static REDDIT_ANDROID_OAUTH_CLIENT_ID: &str = "ohXpoqrZYub1kg";
static REDDIT_IOS_OAUTH_CLIENT_ID: &str = "LNDo9k1o8UAEUw";
static AUTH_ENDPOINT: &str = "https://accounts.reddit.com";
// Various Android user agents - build numbers from valid APK variants
pub(crate) static ANDROID_USER_AGENT: [&str; 3] = [
"Reddit/Version 2023.21.0/Build 956283/Android 13",
"Reddit/Version 2023.21.0/Build 968223/Android 10",
"Reddit/Version 2023.21.0/Build 946732/Android 12",
];
// Various iOS user agents - iOS versions.
pub(crate) static IOS_USER_AGENT: [&str; 3] = [
"Reddit/Version 2023.22.0/Build 613580/iOS Version 17.0 (Build 21A5248V)",
"Reddit/Version 2023.22.0/Build 613580/iOS Version 16.0 (Build 20A5328h)",
"Reddit/Version 2023.22.0/Build 613580/iOS Version 16.5",
];
// Various iOS device codes. iPhone 11 displays as `iPhone12,1`
// I just changed the number a few times for some plausible values
pub(crate) static IOS_DEVICES: [&str; 5] = ["iPhone8,1", "iPhone11,1", "iPhone12,1", "iPhone13,1", "iPhone14,1"];
#[derive(Debug, Clone, Default)]
pub(crate) struct Oauth {
// Currently unused, may be necessary if we decide to support GQL in the future
pub(crate) headers_map: HashMap<String, String>,
pub(crate) token: String,
expires_in: u64,
device: Device,
}
impl Oauth {
pub(crate) async fn new() -> Self {
let mut oauth = Oauth::default();
oauth.login().await;
oauth
}
pub(crate) fn default() -> Self {
// Generate a random device to spoof
let device = Device::random();
let headers = device.headers.clone();
// For now, just insert headers - no token request
Oauth {
headers_map: headers,
token: String::new(),
expires_in: 0,
device,
}
}
async fn login(&mut self) -> Option<()> {
// Construct URL for OAuth token
let url = format!("{}/api/access_token", AUTH_ENDPOINT);
let mut builder = Request::builder().method(Method::POST).uri(&url);
// Add headers from spoofed client
for (key, value) in self.headers_map.iter() {
// Skip Authorization header - won't be present in `Device` struct
// and will only be there in subsequent token refreshes.
// Sending a bearer auth token when requesting one is a bad idea
// Normally, you'd want to send it along to authenticate a refreshed token,
// but neither Android nor iOS does this - it just requests a new token.
// We try to match behavior as closely as possible.
if key != "Authorization" {
builder = builder.header(key, value);
}
}
// Set up HTTP Basic Auth - basically just the const OAuth ID's with no password,
// Base64-encoded. https://en.wikipedia.org/wiki/Basic_access_authentication
// This could be constant, but I don't think it's worth it. OAuth ID's can change
// over time and we want to be flexible.
let auth = general_purpose::STANDARD.encode(format!("{}:", self.device.oauth_id));
builder = builder.header("Authorization", format!("Basic {auth}"));
// Set JSON body. I couldn't tell you what this means. But that's what the client sends
let json = json!({
"scopes": ["*","email","pii"]
});
let body = Body::from(json.to_string());
// Build request
let request = builder.body(body).unwrap();
// Send request
let client: client::Client<_, hyper::Body> = CLIENT.clone();
let resp = client.request(request).await.ok()?;
// Parse headers - loid header _should_ be saved sent on subsequent token refreshes.
// Technically it's not needed, but it's easy for Reddit API to check for this.
// It's some kind of header that uniquely identifies the device.
if let Some(header) = resp.headers().get("x-reddit-loid") {
self.headers_map.insert("x-reddit-loid".to_owned(), header.to_str().ok()?.to_string());
}
// Serialize response
let body_bytes = hyper::body::to_bytes(resp.into_body()).await.ok()?;
let json: serde_json::Value = serde_json::from_slice(&body_bytes).ok()?;
// Save token and expiry
self.token = json.get("access_token")?.as_str()?.to_string();
self.expires_in = json.get("expires_in")?.as_u64()?;
self.headers_map.insert("Authorization".to_owned(), format!("Bearer {}", self.token));
info!("✅ Success - Retrieved token \"{}...\", expires in {}", &self.token[..32], self.expires_in);
Some(())
}
async fn refresh(&mut self) -> Option<()> {
// Refresh is actually just a subsequent login with the same headers (without the old token
// or anything). This logic is handled in login, so we just call login again.
let refresh = self.login().await;
info!("Refreshing OAuth token... {}", if refresh.is_some() { "success" } else { "failed" });
refresh
}
}
pub(crate) async fn token_daemon() {
// Monitor for refreshing token
loop {
// Get expiry time - be sure to not hold the read lock
let expires_in = { OAUTH_CLIENT.read().await.expires_in };
// sleep for the expiry time minus 2 minutes
let duration = Duration::from_secs(expires_in - 120);
info!("Waiting for {duration:?} seconds before refreshing OAuth token...");
tokio::time::sleep(duration).await;
info!("[{duration:?} ELAPSED] Refreshing OAuth token...");
// Refresh token - in its own scope
{
let mut client = OAUTH_CLIENT.write().await;
client.refresh().await;
}
}
}
#[derive(Debug, Clone, Default)]
struct Device {
oauth_id: String,
headers: HashMap<String, String>,
}
impl Device {
fn android() -> Self {
// Generate uuid
let uuid = uuid::Uuid::new_v4().to_string();
// Select random user agent from ANDROID_USER_AGENT
let android_user_agent = choose(&ANDROID_USER_AGENT).to_string();
// Android device headers
let headers = HashMap::from([
("Client-Vendor-Id".into(), uuid.clone()),
("X-Reddit-Device-Id".into(), uuid.clone()),
("User-Agent".into(), android_user_agent),
]);
info!("Spoofing Android client with headers: {headers:?}, uuid: \"{uuid}\", and OAuth ID \"{REDDIT_ANDROID_OAUTH_CLIENT_ID}\"");
Device {
oauth_id: REDDIT_ANDROID_OAUTH_CLIENT_ID.to_string(),
headers,
}
}
fn ios() -> Self {
// Generate uuid
let uuid = uuid::Uuid::new_v4().to_string();
// Select random user agent from IOS_USER_AGENT
let ios_user_agent = choose(&IOS_USER_AGENT).to_string();
// Select random iOS device from IOS_DEVICES
let ios_device = choose(&IOS_DEVICES).to_string();
// iOS device headers
let headers = HashMap::from([
("X-Reddit-DPR".into(), "2".into()),
("Device-Name".into(), ios_device.clone()),
("X-Reddit-Device-Id".into(), uuid.clone()),
("User-Agent".into(), ios_user_agent),
("Client-Vendor-Id".into(), uuid.clone()),
]);
info!("Spoofing iOS client {ios_device} with headers: {headers:?}, uuid: \"{uuid}\", and OAuth ID \"{REDDIT_IOS_OAUTH_CLIENT_ID}\"");
Device {
oauth_id: REDDIT_IOS_OAUTH_CLIENT_ID.to_string(),
headers,
}
}
// Randomly choose a device
fn random() -> Self {
if fastrand::bool() {
Device::android()
} else {
Device::ios()
}
}
}
// Waiting on fastrand 2.0.0 for the `choose` function
// https://github.com/smol-rs/fastrand/pull/59/
fn choose<T: Copy>(list: &[T]) -> T {
list[fastrand::usize(..list.len())]
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
async fn test_oauth_client() {
assert!(!OAUTH_CLIENT.read().await.token.is_empty());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
async fn test_oauth_client_refresh() {
OAUTH_CLIENT.write().await.refresh().await.unwrap();
}

6
src/subreddit.rs
View file

@ -442,3 +442,9 @@ async fn subreddit(sub: &str, quarantined: bool) -> Result<Subreddit, String> {
nsfw: res["data"]["over18"].as_bool().unwrap_or_default(),
})
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
async fn test_fetching_subreddit() {
let subreddit = subreddit("rust", false).await;
assert!(subreddit.is_ok());
}

7
src/user.rs
View file

@ -129,3 +129,10 @@ async fn user(name: &str) -> Result<User, String> {
}
})
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
async fn test_fetching_user() {
let user = user("spez").await;
assert!(user.is_ok());
assert!(user.unwrap().karma > 100);
}