Improvements to Reddit auth

README.md

@@ -41,10 +41,12 @@ Hacker News sync will be skipped if `KS_HN_AUTH` is not set.
 
 | Variable                 | Required | Description                                      |
 | ------------------------ | -------- | ------------------------------------------------ |
-| `KS_REDDIT_CLIENTID`     | ❌       | Your Reddit app client ID                        |
-| `KS_REDDIT_CLIENTSECRET` | ❌       | Your Reddit app client secret                    |
-| `KS_REDDIT_REFRESHTOKEN` | ❌       | Your Reddit app refresh token                    |
-| `KS_REDDIT_SCHEDULE`     | ❌       | Sync schedule in cron format (default: `@daily`) |
+| `KS_REDDIT_CLIENTID`     | ❌        | Your Reddit app client ID                        |
+| `KS_REDDIT_CLIENTSECRET` | ❌        | Your Reddit app client secret                    |
+| `KS_REDDIT_REFRESHTOKEN` | ❌        | Your Reddit app refresh token                    |
+| `KS_REDDIT_USERNAME`     | ❌        | Your Reddit username (without `u/` prefix)       |
+| `KS_REDDIT_SCHEDULE`     | ❌        | Sync schedule in cron format (default: `@daily`) |
+
 
 To obtain a refresh token, you can follow these steps:
 
@@ -53,11 +55,16 @@ To obtain a refresh token, you can follow these steps:
 3. Make sure to give the app `history` scope access.
 4. Make sure to tick the "permanent" option to get a refresh token.
 
-If you don't want to trust a third party tool, you can also implement the OAuth2 flow yourself using the [Reddit API docs](https://www.reddit.com/dev/api/).
+If you don't want to trust a third party tool, you can also implement the OAuth2 flow yourself using the [Reddit API docs](https://www.reddit.com/dev/api/), or [manually generate a token](/REDDIT_REFRESH_TOKEN.md).
 
 Reddit saves will be synced to a list named `Reddit Saved` in your Karakeep instance.
 
-Reddit sync will be skipped if any of `KS_REDDIT_CLIENTID`, `KS_REDDIT_CLIENTSECRET` or `KS_REDDIT_REFRESHTOKEN` is not set.
+Reddit sync will be skipped if any of the following are not set:
+- `KS_REDDIT_CLIENTID`
+- `KS_REDDIT_CLIENTSECRET`
+- `KS_REDDIT_REFRESHTOKEN`
+- `KS_REDDIT_USERNAME`
+
 
 ### GitHub Stars
 
@@ -105,6 +112,7 @@ services:
       - KS_REDDIT_CLIENTID=<your_reddit_client_id> # optional
       - KS_REDDIT_CLIENTSECRET=<your_reddit_client_secret> # optional
       - KS_REDDIT_REFRESHTOKEN=<your_reddit_refresh_token> # optional
+      - KS_REDDIT_USERNAME=<your_reddit_username> #optional
       - KS_REDDIT_SCHEDULE=@daily # optional Cron format, e.g., "@hourly", "@daily", "0 0 * * *" default is "@daily"
 
       - KS_GITHUB_TOKEN=<your_github_personal_access_token> # optional

REDDIT_REFRESH_TOKEN.md

@@ -0,0 +1,43 @@
+# Manually Generating a Reddit Refresh Token
+
+Obtain your Reddit app's `client ID` and `secret` from the [developed applications](https://www.reddit.com/prefs/apps) section of your account preferences (or from [Old Reddit](https://old.reddit.com/prefs/apps)).
+
+Paste your `client ID` into the following path:
+```plaintext
+https://www.reddit.com/api/v1/authorize?client_id=<CLIENT_ID>&response_type=code&state=debug&redirect_uri=http://localhost&duration=permanent&scope=history%20read%20save
+```
+
+> [!IMPORTANT]
+> The `redirect_uri` must match your Reddit app configuration
+
+Paste the URI into a web browser. It should prompt you to allow access. After granting access, the page will return an error message.
+
+Copy the URI out of the address bar, it will look like this:
+```plaintext
+http://localhost/?state=debug&code=<AUTH_CODE>#_
+```
+
+Copy the `auth code` out of the error URI (remove the `#_` suffix).
+
+Construct the following CURL command:
+```bash
+curl -X POST "https://www.reddit.com/api/v1/access_token" \
+  -u "<CLIENT_ID>:<CLIENT_SECRET>" \
+  -H "User-Agent: karakeep-sync/1.0 by sidoshi" \
+  -d "grant_type=authorization_code" \
+  -d "code=<AUTH_CODE>" \
+  -d "redirect_uri=http://localhost"
+```
+
+Run the CURL command; you should receive a response like this:
+```json
+{
+    "access_token": "######",
+    "token_type": "bearer",
+    "expires_in": 86400,
+    "refresh_token": "######",
+    "scope": "read history"
+}
+```
+
+Extract the `refresh_token`, this is your `KS_REDDIT_REFRESHTOKEN` in Karakeep-sync.

crates/karakeep-client/src/lib.rs

@@ -1,4 +1,4 @@
-use reqwest::{Client, Url};
+use reqwest::{Client, Response, Url};
 
 pub struct KarakeepClient {
     url: String,
@@ -11,14 +11,24 @@ pub struct BookmarkCreate {
     pub created_at: Option<String>,
 }
 
+async fn parse_response(resp: Response) -> anyhow::Result<serde_json::Value> {
+    let status = resp.status();
+    let body = resp.text().await?;
+    if !status.is_success() {
+        return Err(anyhow::anyhow!("Karakeep API returned {}: {}", status, body));
+    }
+    serde_json::from_str(&body)
+        .map_err(|e| anyhow::anyhow!("Failed to parse Karakeep response: {e}\nBody: {body}"))
+}
+
 impl KarakeepClient {
     pub fn new(url: &str, auth_token: &str) -> Self {
         let mut headers = reqwest::header::HeaderMap::new();
         headers.insert(
             reqwest::header::AUTHORIZATION,
             reqwest::header::HeaderValue::from_str(&format!("Bearer {auth_token}")).unwrap(),
         );
-        let client = Client::builder().default_headers(headers).build().unwrap();
+        let client = Client::builder().default_headers(headers).timeout(std::time::Duration::from_secs(30)).build().unwrap();
 
         Self {
             url: url.into(),
@@ -41,14 +51,7 @@ impl KarakeepClient {
             );
         }
 
-        let resp = self
-            .client
-            .post(&api_url)
-            .json(&params)
-            .send()
-            .await?
-            .json::<serde_json::Value>()
-            .await?;
+        let resp = parse_response(self.client.post(&api_url).json(&params).send().await?).await?;
 
         resp.get("id")
             .and_then(|id| id.as_str())
@@ -66,18 +69,18 @@ impl KarakeepClient {
     ) -> anyhow::Result<Option<String>> {
         let url = format!("{}/api/v1/bookmarks/search", self.url);
 
-        let resp = self
-            .client
-            .get(&url)
-            .query(&[
-                ("q", bookmark_url),
-                ("includeContent", "false"),
-                ("limit", "1"),
-            ])
-            .send()
-            .await?
-            .json::<serde_json::Value>()
-            .await?;
+        let resp = parse_response(
+            self.client
+                .get(&url)
+                .query(&[
+                    ("q", bookmark_url),
+                    ("includeContent", "false"),
+                    ("limit", "1"),
+                ])
+                .send()
+                .await?,
+        )
+        .await?;
 
         let bookmarks = resp.get("bookmarks").and_then(|b| b.as_array()).unwrap();
 
@@ -115,14 +118,7 @@ impl KarakeepClient {
     pub async fn ensure_list_exists(&self, list_name: &str) -> anyhow::Result<String> {
         let url = format!("{}/api/v1/lists", self.url);
 
-        // First, check if the list already exists
-        let resp = self
-            .client
-            .get(&url)
-            .send()
-            .await?
-            .json::<serde_json::Value>()
-            .await?;
+        let resp = parse_response(self.client.get(&url).send().await?).await?;
 
         let lists = resp.get("lists").and_then(|l| l.as_array()).unwrap();
 
@@ -141,14 +137,8 @@ impl KarakeepClient {
             "icon": "🚀"
         });
 
-        let resp = self
-            .client
-            .post(&url)
-            .json(&params)
-            .send()
-            .await?
-            .json::<serde_json::Value>()
-            .await?;
+        let resp =
+            parse_response(self.client.post(&url).json(&params).send().await?).await?;
 
         resp.get("id")
             .and_then(|id| id.as_str())

crates/reddit-client/Cargo.toml

@@ -8,3 +8,4 @@ reqwest = { workspace = true }
 anyhow = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
+tracing = { workspace = true }

crates/reddit-client/src/lib.rs

@@ -27,6 +27,7 @@ impl RedditClientRefresher {
         );
         let client = reqwest::Client::builder()
             .default_headers(headers)
+            .timeout(std::time::Duration::from_secs(30))
             .build()
             .unwrap();
 
@@ -38,7 +39,7 @@ impl RedditClientRefresher {
         }
     }
 
-    pub async fn refresh(&self) -> anyhow::Result<RedditClient> {
+    pub async fn refresh(&self, username: String) -> anyhow::Result<RedditClient> {
         let params = [
             ("grant_type", "refresh_token"),
             ("refresh_token", &self.refresh_token),
@@ -58,23 +59,12 @@ impl RedditClientRefresher {
             .and_then(|t| t.as_str())
             .ok_or_else(|| anyhow::anyhow!("Failed to get access token from response: {resp:?}"))?;
 
-        let resp = self
-            .client
-            .get(format!("{APP_URL}/api/v1/me"))
-            .bearer_auth(access_token)
-            .send()
-            .await?
-            .json::<serde_json::Value>()
-            .await?;
-        let username = resp
-            .get("name")
-            .and_then(|n| n.as_str())
-            .ok_or_else(|| anyhow::anyhow!("Failed to get username from response: {resp:?}"))?;
+        tracing::info!("Reddit authenticated as u/{}", username);
 
         Ok(RedditClient {
             access_token: access_token.to_string(),
             client: self.client.clone(),
-            username: username.to_string(),
+            username,
         })
     }
 }
@@ -123,7 +113,14 @@ impl RedditClient {
             req = req.query(&[("after", after)]);
         }
 
-        let resp = req.send().await?.json::<ListingResponse>().await?;
+        let resp = req.send().await?;
+        let status = resp.status();
+        let body = resp.text().await?;
+        if !status.is_success() {
+            return Err(anyhow::anyhow!("Reddit API returned {}: {}", status, body));
+        }
+        let resp = serde_json::from_str::<ListingResponse>(&body)
+            .map_err(|e| anyhow::anyhow!("Failed to parse Reddit response: {e}\nBody: {body}"))?;
 
         let posts = resp
             .data

crates/sync/src/main.rs

@@ -18,7 +18,8 @@ async fn main() -> anyhow::Result<()> {
                 .add_directive("hyper=off".parse().unwrap())
                 .add_directive("reqwest=off".parse().unwrap())
                 .add_directive("karakeep_sync=trace".parse().unwrap())
-                .add_directive("karakeep_client=trace".parse().unwrap()),
+                .add_directive("karakeep_client=trace".parse().unwrap())
+                .add_directive("reddit_client=info".parse().unwrap()),
         )
         .finish();
     tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed");
@@ -45,7 +46,9 @@ async fn main() -> anyhow::Result<()> {
                     tracing::info!("starting immediate sync job for list: {}", list_name);
                     let p = plugin.clone();
                     Box::pin(async move {
-                        let _ = p.sync().await;
+                        if let Err(e) = p.sync().await {
+                            tracing::error!("sync failed for list '{}': {:#}", list_name, e);
+                        }
                     })
                 })?;
             scheduler.add(job).await?;
@@ -59,10 +62,12 @@ async fn main() -> anyhow::Result<()> {
 
         let schedule = plugin.recurring_schedule().to_string();
         let job = Job::new_async(&schedule, move |_uuid, _l| {
-            tracing::info!("starting HN sync daily job");
+            tracing::info!("starting recurring sync job for list: {}", list_name);
             let p = plugin.clone();
             Box::pin(async move {
-                let _ = p.sync().await;
+                if let Err(e) = p.sync().await {
+                    tracing::error!("sync failed for list '{}': {:#}", list_name, e);
+                }
             })
         })?;
         scheduler.add(job).await?;

crates/sync/src/plugin/reddit_saves.rs

@@ -38,9 +38,15 @@ impl super::Plugin for RedditSaves {
             .as_ref()
             .context("Reddit refresh token is not set")?
             .clone();
+        let username = settings
+            .reddit
+            .username
+            .as_ref()
+            .context("Reddit username is not set")?
+            .clone();
 
         let client = RedditClientRefresher::new(client_id, client_secret, refresh_token)
-            .refresh()
+            .refresh(username)
             .await?;
         let client = Arc::new(client);
 
@@ -61,7 +67,13 @@ impl super::Plugin for RedditSaves {
                     return None;
                 }
 
-                let resp = client.list_saved(after.as_deref()).await.ok()?;
+                let resp = match client.list_saved(after.as_deref()).await {
+                    Ok(r) => r,
+                    Err(e) => {
+                        tracing::error!("Reddit list_saved failed: {:#}", e);
+                        return None;
+                    }
+                };
 
                 let items = resp
                     .posts
@@ -74,11 +86,6 @@ impl super::Plugin for RedditSaves {
                     })
                     .collect::<Vec<_>>();
 
-                tracing::debug!(
-                    "fetched {} saved posts from Reddit, after: {:?}",
-                    items.len(),
-                    resp.after
-                );
                 Some((items, StreamState::Next(resp.after)))
             }
         });
@@ -92,6 +99,7 @@ impl super::Plugin for RedditSaves {
         settings.reddit.clientid.is_some()
             && settings.reddit.clientsecret.is_some()
             && settings.reddit.refreshtoken.is_some()
+            && settings.reddit.username.is_some()
     }
 
     fn recurring_schedule(&self) -> String {

crates/sync/src/settings.rs

@@ -27,6 +27,7 @@ pub(crate) struct RedditSettings {
     pub clientid: Option<String>,
     pub clientsecret: Option<String>,
     pub refreshtoken: Option<String>,
+    pub username: Option<String>,
     pub schedule: String,
 }