diff --git a/Cargo.lock b/Cargo.lock index 57c9955..bc64078 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1624,6 +1624,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "tracing", ] [[package]] diff --git a/README.md b/README.md index 4384acb..18d8dd5 100644 --- a/README.md +++ b/README.md @@ -41,10 +41,12 @@ Hacker News sync will be skipped if `KS_HN_AUTH` is not set. | Variable | Required | Description | | ------------------------ | -------- | ------------------------------------------------ | -| `KS_REDDIT_CLIENTID` | ❌ | Your Reddit app client ID | -| `KS_REDDIT_CLIENTSECRET` | ❌ | Your Reddit app client secret | -| `KS_REDDIT_REFRESHTOKEN` | ❌ | Your Reddit app refresh token | -| `KS_REDDIT_SCHEDULE` | ❌ | Sync schedule in cron format (default: `@daily`) | +| `KS_REDDIT_CLIENTID` | ❌ | Your Reddit app client ID | +| `KS_REDDIT_CLIENTSECRET` | ❌ | Your Reddit app client secret | +| `KS_REDDIT_REFRESHTOKEN` | ❌ | Your Reddit app refresh token | +| `KS_REDDIT_USERNAME` | ❌ | Your Reddit username (without `u/` prefix) | +| `KS_REDDIT_SCHEDULE` | ❌ | Sync schedule in cron format (default: `@daily`) | + To obtain a refresh token, you can follow these steps: @@ -53,11 +55,16 @@ To obtain a refresh token, you can follow these steps: 3. Make sure to give the app `history` scope access. 4. Make sure to tick the "permanent" option to get a refresh token. -If you don't want to trust a third party tool, you can also implement the OAuth2 flow yourself using the [Reddit API docs](https://www.reddit.com/dev/api/). +If you don't want to trust a third party tool, you can also implement the OAuth2 flow yourself using the [Reddit API docs](https://www.reddit.com/dev/api/), or [manually generate a token](/REDDIT_REFRESH_TOKEN.md). Reddit saves will be synced to a list named `Reddit Saved` in your Karakeep instance. -Reddit sync will be skipped if any of `KS_REDDIT_CLIENTID`, `KS_REDDIT_CLIENTSECRET` or `KS_REDDIT_REFRESHTOKEN` is not set. +Reddit sync will be skipped if any of the following are not set: +- `KS_REDDIT_CLIENTID` +- `KS_REDDIT_CLIENTSECRET` +- `KS_REDDIT_REFRESHTOKEN` +- `KS_REDDIT_USERNAME` + ### GitHub Stars @@ -105,6 +112,7 @@ services: - KS_REDDIT_CLIENTID= # optional - KS_REDDIT_CLIENTSECRET= # optional - KS_REDDIT_REFRESHTOKEN= # optional + - KS_REDDIT_USERNAME= #optional - KS_REDDIT_SCHEDULE=@daily # optional Cron format, e.g., "@hourly", "@daily", "0 0 * * *" default is "@daily" - KS_GITHUB_TOKEN= # optional diff --git a/REDDIT_REFRESH_TOKEN.md b/REDDIT_REFRESH_TOKEN.md new file mode 100644 index 0000000..b40a9e2 --- /dev/null +++ b/REDDIT_REFRESH_TOKEN.md @@ -0,0 +1,43 @@ +# Manually Generating a Reddit Refresh Token + +Obtain your Reddit app's `client ID` and `secret` from the [developed applications](https://www.reddit.com/prefs/apps) section of your account preferences (or from [Old Reddit](https://old.reddit.com/prefs/apps)). + +Paste your `client ID` into the following path: +```plaintext +https://www.reddit.com/api/v1/authorize?client_id=&response_type=code&state=debug&redirect_uri=http://localhost&duration=permanent&scope=history%20read%20save +``` + +> [!IMPORTANT] +> The `redirect_uri` must match your Reddit app configuration + +Paste the URI into a web browser. It should prompt you to allow access. After granting access, the page will return an error message. + +Copy the URI out of the address bar, it will look like this: +```plaintext +http://localhost/?state=debug&code=#_ +``` + +Copy the `auth code` out of the error URI (remove the `#_` suffix). + +Construct the following CURL command: +```bash +curl -X POST "https://www.reddit.com/api/v1/access_token" \ + -u ":" \ + -H "User-Agent: karakeep-sync/1.0 by sidoshi" \ + -d "grant_type=authorization_code" \ + -d "code=" \ + -d "redirect_uri=http://localhost" +``` + +Run the CURL command; you should receive a response like this: +```json +{ + "access_token": "######", + "token_type": "bearer", + "expires_in": 86400, + "refresh_token": "######", + "scope": "read history" +} +``` + +Extract the `refresh_token`, this is your `KS_REDDIT_REFRESHTOKEN` in Karakeep-sync. diff --git a/crates/karakeep-client/src/lib.rs b/crates/karakeep-client/src/lib.rs index 9961bb3..4bf1d6b 100644 --- a/crates/karakeep-client/src/lib.rs +++ b/crates/karakeep-client/src/lib.rs @@ -1,4 +1,4 @@ -use reqwest::{Client, Url}; +use reqwest::{Client, Response, Url}; pub struct KarakeepClient { url: String, @@ -11,6 +11,20 @@ pub struct BookmarkCreate { pub created_at: Option, } +async fn parse_response(resp: Response) -> anyhow::Result { + let status = resp.status(); + let body = resp.text().await?; + if !status.is_success() { + return Err(anyhow::anyhow!( + "Karakeep API returned {}: {}", + status, + body + )); + } + serde_json::from_str(&body) + .map_err(|e| anyhow::anyhow!("Failed to parse Karakeep response: {e}\nBody: {body}")) +} + impl KarakeepClient { pub fn new(url: &str, auth_token: &str) -> Self { let mut headers = reqwest::header::HeaderMap::new(); @@ -18,7 +32,11 @@ impl KarakeepClient { reqwest::header::AUTHORIZATION, reqwest::header::HeaderValue::from_str(&format!("Bearer {auth_token}")).unwrap(), ); - let client = Client::builder().default_headers(headers).build().unwrap(); + let client = Client::builder() + .default_headers(headers) + .timeout(std::time::Duration::from_secs(30)) + .build() + .unwrap(); Self { url: url.into(), @@ -41,14 +59,7 @@ impl KarakeepClient { ); } - let resp = self - .client - .post(&api_url) - .json(¶ms) - .send() - .await? - .json::() - .await?; + let resp = parse_response(self.client.post(&api_url).json(¶ms).send().await?).await?; resp.get("id") .and_then(|id| id.as_str()) @@ -66,18 +77,18 @@ impl KarakeepClient { ) -> anyhow::Result> { let url = format!("{}/api/v1/bookmarks/search", self.url); - let resp = self - .client - .get(&url) - .query(&[ - ("q", bookmark_url), - ("includeContent", "false"), - ("limit", "1"), - ]) - .send() - .await? - .json::() - .await?; + let resp = parse_response( + self.client + .get(&url) + .query(&[ + ("q", bookmark_url), + ("includeContent", "false"), + ("limit", "1"), + ]) + .send() + .await?, + ) + .await?; let bookmarks = resp.get("bookmarks").and_then(|b| b.as_array()).unwrap(); @@ -115,14 +126,7 @@ impl KarakeepClient { pub async fn ensure_list_exists(&self, list_name: &str) -> anyhow::Result { let url = format!("{}/api/v1/lists", self.url); - // First, check if the list already exists - let resp = self - .client - .get(&url) - .send() - .await? - .json::() - .await?; + let resp = parse_response(self.client.get(&url).send().await?).await?; let lists = resp.get("lists").and_then(|l| l.as_array()).unwrap(); @@ -141,14 +145,7 @@ impl KarakeepClient { "icon": "🚀" }); - let resp = self - .client - .post(&url) - .json(¶ms) - .send() - .await? - .json::() - .await?; + let resp = parse_response(self.client.post(&url).json(¶ms).send().await?).await?; resp.get("id") .and_then(|id| id.as_str()) diff --git a/crates/reddit-client/Cargo.toml b/crates/reddit-client/Cargo.toml index d6790b7..a0f3037 100644 --- a/crates/reddit-client/Cargo.toml +++ b/crates/reddit-client/Cargo.toml @@ -8,3 +8,4 @@ reqwest = { workspace = true } anyhow = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +tracing = { workspace = true } diff --git a/crates/reddit-client/src/lib.rs b/crates/reddit-client/src/lib.rs index 4a32a26..78fe198 100644 --- a/crates/reddit-client/src/lib.rs +++ b/crates/reddit-client/src/lib.rs @@ -27,6 +27,7 @@ impl RedditClientRefresher { ); let client = reqwest::Client::builder() .default_headers(headers) + .timeout(std::time::Duration::from_secs(30)) .build() .unwrap(); @@ -38,7 +39,7 @@ impl RedditClientRefresher { } } - pub async fn refresh(&self) -> anyhow::Result { + pub async fn refresh(&self, username: String) -> anyhow::Result { let params = [ ("grant_type", "refresh_token"), ("refresh_token", &self.refresh_token), @@ -58,23 +59,12 @@ impl RedditClientRefresher { .and_then(|t| t.as_str()) .ok_or_else(|| anyhow::anyhow!("Failed to get access token from response: {resp:?}"))?; - let resp = self - .client - .get(format!("{APP_URL}/api/v1/me")) - .bearer_auth(access_token) - .send() - .await? - .json::() - .await?; - let username = resp - .get("name") - .and_then(|n| n.as_str()) - .ok_or_else(|| anyhow::anyhow!("Failed to get username from response: {resp:?}"))?; + tracing::info!("Reddit authenticated as u/{}", username); Ok(RedditClient { access_token: access_token.to_string(), client: self.client.clone(), - username: username.to_string(), + username, }) } } @@ -123,7 +113,14 @@ impl RedditClient { req = req.query(&[("after", after)]); } - let resp = req.send().await?.json::().await?; + let resp = req.send().await?; + let status = resp.status(); + let body = resp.text().await?; + if !status.is_success() { + return Err(anyhow::anyhow!("Reddit API returned {}: {}", status, body)); + } + let resp = serde_json::from_str::(&body) + .map_err(|e| anyhow::anyhow!("Failed to parse Reddit response: {e}\nBody: {body}"))?; let posts = resp .data diff --git a/crates/sync/src/main.rs b/crates/sync/src/main.rs index 8c30674..fb48c69 100644 --- a/crates/sync/src/main.rs +++ b/crates/sync/src/main.rs @@ -18,7 +18,8 @@ async fn main() -> anyhow::Result<()> { .add_directive("hyper=off".parse().unwrap()) .add_directive("reqwest=off".parse().unwrap()) .add_directive("karakeep_sync=trace".parse().unwrap()) - .add_directive("karakeep_client=trace".parse().unwrap()), + .add_directive("karakeep_client=trace".parse().unwrap()) + .add_directive("reddit_client=info".parse().unwrap()), ) .finish(); tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed"); @@ -45,7 +46,9 @@ async fn main() -> anyhow::Result<()> { tracing::info!("starting immediate sync job for list: {}", list_name); let p = plugin.clone(); Box::pin(async move { - let _ = p.sync().await; + if let Err(e) = p.sync().await { + tracing::error!("sync failed for list '{}': {:#}", list_name, e); + } }) })?; scheduler.add(job).await?; @@ -59,10 +62,12 @@ async fn main() -> anyhow::Result<()> { let schedule = plugin.recurring_schedule().to_string(); let job = Job::new_async(&schedule, move |_uuid, _l| { - tracing::info!("starting HN sync daily job"); + tracing::info!("starting recurring sync job for list: {}", list_name); let p = plugin.clone(); Box::pin(async move { - let _ = p.sync().await; + if let Err(e) = p.sync().await { + tracing::error!("sync failed for list '{}': {:#}", list_name, e); + } }) })?; scheduler.add(job).await?; diff --git a/crates/sync/src/plugin/reddit_saves.rs b/crates/sync/src/plugin/reddit_saves.rs index 347571c..5e24d50 100644 --- a/crates/sync/src/plugin/reddit_saves.rs +++ b/crates/sync/src/plugin/reddit_saves.rs @@ -38,9 +38,15 @@ impl super::Plugin for RedditSaves { .as_ref() .context("Reddit refresh token is not set")? .clone(); + let username = settings + .reddit + .username + .as_ref() + .context("Reddit username is not set")? + .clone(); let client = RedditClientRefresher::new(client_id, client_secret, refresh_token) - .refresh() + .refresh(username) .await?; let client = Arc::new(client); @@ -61,7 +67,13 @@ impl super::Plugin for RedditSaves { return None; } - let resp = client.list_saved(after.as_deref()).await.ok()?; + let resp = match client.list_saved(after.as_deref()).await { + Ok(r) => r, + Err(e) => { + tracing::error!("Reddit list_saved failed: {:#}", e); + return None; + } + }; let items = resp .posts @@ -74,11 +86,6 @@ impl super::Plugin for RedditSaves { }) .collect::>(); - tracing::debug!( - "fetched {} saved posts from Reddit, after: {:?}", - items.len(), - resp.after - ); Some((items, StreamState::Next(resp.after))) } }); @@ -92,6 +99,7 @@ impl super::Plugin for RedditSaves { settings.reddit.clientid.is_some() && settings.reddit.clientsecret.is_some() && settings.reddit.refreshtoken.is_some() + && settings.reddit.username.is_some() } fn recurring_schedule(&self) -> String { diff --git a/crates/sync/src/settings.rs b/crates/sync/src/settings.rs index d388b12..213bdde 100644 --- a/crates/sync/src/settings.rs +++ b/crates/sync/src/settings.rs @@ -27,6 +27,7 @@ pub(crate) struct RedditSettings { pub clientid: Option, pub clientsecret: Option, pub refreshtoken: Option, + pub username: Option, pub schedule: String, }