From 3b5ef06299001e13783a7412a1859b5a31204af1 Mon Sep 17 00:00:00 2001 From: Said Rodrigues Date: Wed, 19 Nov 2025 10:04:37 -0300 Subject: [PATCH 01/10] feat(Config): add new InitialFetchCount option --- FeedCord/src/Common/Config.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/FeedCord/src/Common/Config.cs b/FeedCord/src/Common/Config.cs index f2b8df1..6b7c2b0 100644 --- a/FeedCord/src/Common/Config.cs +++ b/FeedCord/src/Common/Config.cs @@ -39,5 +39,6 @@ public class Config public List? PostFilters { get; set; } public Dictionary? Pings { get; set; } public int ConcurrentRequests { get; set; } = 5; + public int InitialFetchCount { get; set; } = 0; } } From 9080cf44211495019ef3585fb65e84fdf142450f Mon Sep 17 00:00:00 2001 From: Said Rodrigues Date: Wed, 19 Nov 2025 10:59:06 -0300 Subject: [PATCH 02/10] feat(FeedManager): add optional filterByLastPublishedDate param and use a early return on when no post if found --- FeedCord/src/Services/FeedManager.cs | 85 ++++++++++++++-------------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/FeedCord/src/Services/FeedManager.cs b/FeedCord/src/Services/FeedManager.cs index 95325c5..1846990 100644 --- a/FeedCord/src/Services/FeedManager.cs +++ b/FeedCord/src/Services/FeedManager.cs @@ -185,7 +185,7 @@ private async Task TestUrlAsync(string url) return false; } - private async Task CheckSingleFeedAsync(string url, FeedState feedState, ConcurrentBag newPosts, int trim) + private async Task CheckSingleFeedAsync(string url, FeedState feedState, ConcurrentBag newPosts, int trim, bool filterByLastPublishedDate = true) { List posts; @@ -207,28 +207,53 @@ await FetchYoutubeAsync(url) : _instancedConcurrentRequests.Release(); } - var freshlyFetched = posts.Where(p => p?.PublishDate > feedState.LastPublishDate).ToList(); + var fetchedPosts = posts; + if (filterByLastPublishedDate) + { + fetchedPosts = fetchedPosts.Where(p => p?.PublishDate > feedState.LastPublishDate).ToList(); + } - if (freshlyFetched.Any()) + if (fetchedPosts.Count == 0) { - feedState.LastPublishDate = freshlyFetched.Max(p => p!.PublishDate); - feedState.ErrorCount = 0; + _logAggregator.AddLatestUrlPost(url, posts.OrderByDescending(p => p?.PublishDate).FirstOrDefault()); + return; + } + + feedState.LastPublishDate = fetchedPosts.Max(p => p!.PublishDate); + feedState.ErrorCount = 0; - foreach (var post in freshlyFetched) + foreach (var post in fetchedPosts) + { + if (post is null) { - if (post is null) + _logger.LogWarning("Failed to parse a post from {Url}", url); + continue; + } + + //TODO --> Implement Filter checking in to a helper/service & remove from FeedManager + if (_hasFilterEnabled && _config.PostFilters != null) + { + var filter = _config.PostFilters.FirstOrDefault(wf => wf.Url == url); + if (filter != null) { - _logger.LogWarning("Failed to parse a post from {Url}", url); - continue; - } + var filterFound = FilterConfigs.GetFilterSuccess(post, filter.Filters.ToArray()); - //TODO --> Implement Filter checking in to a helper/service & remove from FeedManager - if (_hasFilterEnabled && _config.PostFilters != null) + if (filterFound) + { + newPosts.Add(post); + } + else + { + _logger.LogInformation( + "A new post was omitted because it does not comply to the set filter: {Url}", url); + } + } + else if (_hasAllFilter) { - var filter = _config.PostFilters.FirstOrDefault(wf => wf.Url == url); - if (filter != null) + var allFilter = _config.PostFilters.FirstOrDefault(wf => wf.Url == "all"); + if (allFilter != null) { - var filterFound = FilterConfigs.GetFilterSuccess(post, filter.Filters.ToArray()); + var filterFound = FilterConfigs.GetFilterSuccess(post, allFilter.Filters.ToArray()); if (filterFound) { @@ -240,34 +265,12 @@ await FetchYoutubeAsync(url) : "A new post was omitted because it does not comply to the set filter: {Url}", url); } } - else if (_hasAllFilter) - { - var allFilter = _config.PostFilters.FirstOrDefault(wf => wf.Url == "all"); - if (allFilter != null) - { - var filterFound = FilterConfigs.GetFilterSuccess(post, allFilter.Filters.ToArray()); - - if (filterFound) - { - newPosts.Add(post); - } - else - { - _logger.LogInformation( - "A new post was omitted because it does not comply to the set filter: {Url}", url); - } - } - } - } - else - { - newPosts.Add(post); } } - } - else - { - _logAggregator.AddLatestUrlPost(url, posts.OrderByDescending(p => p?.PublishDate).FirstOrDefault()); + else + { + newPosts.Add(post); + } } } From 241382e294d0922ba1740be914a6a553eaf830b9 Mon Sep 17 00:00:00 2001 From: Said Rodrigues Date: Wed, 19 Nov 2025 10:59:57 -0300 Subject: [PATCH 03/10] feat(IFeedManager): add CheckForLastPostAsync method --- FeedCord/src/Services/Interfaces/IFeedManager.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/FeedCord/src/Services/Interfaces/IFeedManager.cs b/FeedCord/src/Services/Interfaces/IFeedManager.cs index 99ee9fd..f05aba3 100644 --- a/FeedCord/src/Services/Interfaces/IFeedManager.cs +++ b/FeedCord/src/Services/Interfaces/IFeedManager.cs @@ -5,6 +5,7 @@ namespace FeedCord.Services.Interfaces public interface IFeedManager { Task> CheckForNewPostsAsync(); + Task> CheckForLastPostsAsync(int maxPostCount); Task InitializeUrlsAsync(); IReadOnlyDictionary GetAllFeedData(); } From 21f0f5e1e23f05c83078a348accf76bdc86b366b Mon Sep 17 00:00:00 2001 From: Said Rodrigues Date: Wed, 19 Nov 2025 11:00:19 -0300 Subject: [PATCH 04/10] feat(FeedManager): implement CheckForLastPostsAsync method --- FeedCord/src/Services/FeedManager.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/FeedCord/src/Services/FeedManager.cs b/FeedCord/src/Services/FeedManager.cs index 1846990..72da783 100644 --- a/FeedCord/src/Services/FeedManager.cs +++ b/FeedCord/src/Services/FeedManager.cs @@ -60,6 +60,22 @@ public async Task> CheckForNewPostsAsync() return allNewPosts.ToList(); } + + public async Task> CheckForLastPostsAsync(int maxPostCount) + { + ConcurrentBag allNewPosts = new(); + + var tasks = _feedStates.Select(async (feed) => + await CheckSingleFeedAsync(feed.Key, feed.Value, allNewPosts, _config.DescriptionLimit, false)); + + await Task.WhenAll(tasks); + + var filteredPosts = allNewPosts.OrderByDescending(p => p.PublishDate).Take(maxPostCount).ToList(); + + _logAggregator.SetNewPostCount(filteredPosts.Count); + + return filteredPosts.ToList(); + } public async Task InitializeUrlsAsync() { var id = _config.Id; From e832328a5fc144c7c8d8bf32eb41e6c49c7ef8e3 Mon Sep 17 00:00:00 2001 From: Said Rodrigues Date: Wed, 19 Nov 2025 11:00:52 -0300 Subject: [PATCH 05/10] feat(FeedWorker): add initialFetchCount and fetch last posts if initial count is not zero --- FeedCord/src/Infrastructure/Workers/FeedWorker.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/FeedCord/src/Infrastructure/Workers/FeedWorker.cs b/FeedCord/src/Infrastructure/Workers/FeedWorker.cs index e7462bd..3596fda 100644 --- a/FeedCord/src/Infrastructure/Workers/FeedWorker.cs +++ b/FeedCord/src/Infrastructure/Workers/FeedWorker.cs @@ -17,6 +17,7 @@ public class FeedWorker : BackgroundService private readonly bool _persistent; private readonly string _id; private readonly int _delayTime; + private readonly int _initialFetchCount; private bool _isInitialized; @@ -36,6 +37,7 @@ public FeedWorker( _id = config.Id; _isInitialized = false; _persistent = config.PersistenceOnShutdown; + _initialFetchCount = config.InitialFetchCount; _logAggregator = logAggregator; logger.LogInformation("{id} Created with check interval {Interval} minutes", @@ -73,14 +75,18 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) private async Task RunRoutineBackgroundProcessAsync() { + var posts = new List(); + if (!_isInitialized) { _logger.LogInformation("{id}: Initializing Url Checks..", _id); await _feedManager.InitializeUrlsAsync(); _isInitialized = true; + if (_initialFetchCount != 0 ) posts = await _feedManager.CheckForLastPostsAsync(_initialFetchCount); } - var posts = await _feedManager.CheckForNewPostsAsync(); + var newPosts = await _feedManager.CheckForNewPostsAsync(); + posts.AddRange(newPosts); if (posts.Count > 0) { From 7887faaaaeaf07ce0a83091b3271b2fea6506168 Mon Sep 17 00:00:00 2001 From: Said Rodrigues Date: Wed, 19 Nov 2025 11:16:49 -0300 Subject: [PATCH 06/10] feat(Config): add optional OnwardDate --- FeedCord/src/Common/Config.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/FeedCord/src/Common/Config.cs b/FeedCord/src/Common/Config.cs index 6b7c2b0..d2f090c 100644 --- a/FeedCord/src/Common/Config.cs +++ b/FeedCord/src/Common/Config.cs @@ -40,5 +40,6 @@ public class Config public Dictionary? Pings { get; set; } public int ConcurrentRequests { get; set; } = 5; public int InitialFetchCount { get; set; } = 0; + public DateTime? OnwardDate { get; set; } } } From 913d0ed4b72f8b431b19c3fe48bc17469635ffd9 Mon Sep 17 00:00:00 2001 From: Said Rodrigues Date: Wed, 19 Nov 2025 11:17:14 -0300 Subject: [PATCH 07/10] refactor(FeedManager): add filter by OnwardDate --- FeedCord/src/Services/FeedManager.cs | 6 ++++-- FeedCord/src/Services/Interfaces/IFeedManager.cs | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/FeedCord/src/Services/FeedManager.cs b/FeedCord/src/Services/FeedManager.cs index 72da783..9610b71 100644 --- a/FeedCord/src/Services/FeedManager.cs +++ b/FeedCord/src/Services/FeedManager.cs @@ -61,7 +61,7 @@ public async Task> CheckForNewPostsAsync() return allNewPosts.ToList(); } - public async Task> CheckForLastPostsAsync(int maxPostCount) + public async Task> CheckForLastPostsAsync(int maxPostCount, DateTime? onwardDate) { ConcurrentBag allNewPosts = new(); @@ -70,7 +70,9 @@ public async Task> CheckForLastPostsAsync(int maxPostCount) await Task.WhenAll(tasks); - var filteredPosts = allNewPosts.OrderByDescending(p => p.PublishDate).Take(maxPostCount).ToList(); + var filteredPosts = allNewPosts + .Where(p => !onwardDate.HasValue || p.PublishDate > onwardDate.Value) + .OrderByDescending(p => p.PublishDate).Take(maxPostCount).ToList(); _logAggregator.SetNewPostCount(filteredPosts.Count); diff --git a/FeedCord/src/Services/Interfaces/IFeedManager.cs b/FeedCord/src/Services/Interfaces/IFeedManager.cs index f05aba3..bfd381d 100644 --- a/FeedCord/src/Services/Interfaces/IFeedManager.cs +++ b/FeedCord/src/Services/Interfaces/IFeedManager.cs @@ -5,7 +5,7 @@ namespace FeedCord.Services.Interfaces public interface IFeedManager { Task> CheckForNewPostsAsync(); - Task> CheckForLastPostsAsync(int maxPostCount); + Task> CheckForLastPostsAsync(int maxPostCount, DateTime? onwardDate); Task InitializeUrlsAsync(); IReadOnlyDictionary GetAllFeedData(); } From 167e0bbf20d7a2b9d2ca40a0f2484a39d21333b6 Mon Sep 17 00:00:00 2001 From: Said Rodrigues Date: Wed, 19 Nov 2025 11:17:33 -0300 Subject: [PATCH 08/10] refactor(FeedWorker): inject onwardDate for initial fetch --- FeedCord/src/Infrastructure/Workers/FeedWorker.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/FeedCord/src/Infrastructure/Workers/FeedWorker.cs b/FeedCord/src/Infrastructure/Workers/FeedWorker.cs index 3596fda..8fb2be1 100644 --- a/FeedCord/src/Infrastructure/Workers/FeedWorker.cs +++ b/FeedCord/src/Infrastructure/Workers/FeedWorker.cs @@ -1,4 +1,5 @@ -using FeedCord.Common; +using System.Runtime.InteropServices; +using FeedCord.Common; using FeedCord.Core.Interfaces; using FeedCord.Services.Interfaces; using Microsoft.Extensions.Hosting; @@ -18,6 +19,7 @@ public class FeedWorker : BackgroundService private readonly string _id; private readonly int _delayTime; private readonly int _initialFetchCount; + private readonly DateTime? _onwardDate; private bool _isInitialized; @@ -38,6 +40,7 @@ public FeedWorker( _isInitialized = false; _persistent = config.PersistenceOnShutdown; _initialFetchCount = config.InitialFetchCount; + _onwardDate = config.OnwardDate; _logAggregator = logAggregator; logger.LogInformation("{id} Created with check interval {Interval} minutes", @@ -82,7 +85,11 @@ private async Task RunRoutineBackgroundProcessAsync() _logger.LogInformation("{id}: Initializing Url Checks..", _id); await _feedManager.InitializeUrlsAsync(); _isInitialized = true; - if (_initialFetchCount != 0 ) posts = await _feedManager.CheckForLastPostsAsync(_initialFetchCount); + if (_initialFetchCount != 0) + { + _logger.LogInformation("{Id}: Starting Initial Fetch Count {FetchCount}", _id, _initialFetchCount); + posts = await _feedManager.CheckForLastPostsAsync(_initialFetchCount, _onwardDate); + } } var newPosts = await _feedManager.CheckForNewPostsAsync(); From a313517f8861a906b31c8c56cb5254b060227b29 Mon Sep 17 00:00:00 2001 From: Said Rodrigues Date: Wed, 19 Nov 2025 11:37:44 -0300 Subject: [PATCH 09/10] docs(references.md): add Initial Fetch Option example --- FeedCord/docs/reference.md | 66 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/FeedCord/docs/reference.md b/FeedCord/docs/reference.md index 2557e2b..9a4e07c 100644 --- a/FeedCord/docs/reference.md +++ b/FeedCord/docs/reference.md @@ -243,6 +243,70 @@ Luckily you can simply do this to do a filter for all feeds - set `Url` equal to } ``` +### Initial Fetch Configuration + +When setting up a new feed, you might want to backfill recent posts instead of starting from scratch. The `InitialFetchCount` and `OnwardDate` properties allow you to control this behavior. + +**InitialFetchCount**: Specifies how many of the most recent posts to fetch when the feed worker first initializes. By default, this is set to 0, meaning no initial fetch occurs and the feed will only post new items going forward. + +**OnwardDate**: An optional date filter that works in conjunction with `InitialFetchCount`. When specified, only posts published on or after this date will be included in the initial fetch. + +Here's an example of backfilling the last 10 posts: + +``` +{ + "Instances": [ + { + "Id": "Tech News", + "YoutubeUrls": [ + "" + ], + "RssUrls": [ + "https://example.com/feed.rss" + ], + "DiscordWebhookUrl": "https://discord.com/api/webhooks/...", + "RssCheckIntervalMinutes": 15, + "Color": 8411391, + "DescriptionLimit": 500, + "Forum": true, + "MarkdownFormat": false, + "PersistenceOnShutdown": true, + "InitialFetchCount": 10 + } + ], + "ConcurrentRequests": 40 +} +``` + +Here's an example combining both properties to fetch recent posts from a specific date onwards: + +``` +{ + "Instances": [ + { + "Id": "Tech News", + "YoutubeUrls": [ + "" + ], + "RssUrls": [ + "https://example.com/feed.rss" + ], + "DiscordWebhookUrl": "https://discord.com/api/webhooks/...", + "RssCheckIntervalMinutes": 15, + "Color": 8411391, + "DescriptionLimit": 500, + "Forum": true, + "MarkdownFormat": false, + "PersistenceOnShutdown": true, + "InitialFetchCount": 20, + "OnwardDate": "2025-11-01T00:00:00" + } + ], + "ConcurrentRequests": 40 +} +``` + +In this example, FeedCord will fetch up to 20 of the most recent posts, but only include those published on or after November 1st, 2025. --- @@ -274,6 +338,8 @@ Luckily you can simply do this to do a filter for all feeds - set `Url` equal to - **ConcurrentRequests**: How many requests FeedCord can have going at once. - **ConcurrentRequests (Inside Instance)**: How many requests the instance itself can have going at once. - **PostFilters**: A collection of phrases/words that are used to filter out RSS Items (filters the Title & Content) +- **InitialFetchCount**: The number of most recent posts to fetch when the feed worker initializes. Default is 0 (no initial fetch). Useful for backfilling posts when starting a new feed. +- **OnwardDate**: An optional date filter that works with InitialFetchCount. Only posts published on or after this date will be fetched during initial fetch. Format: ISO 8601 date string (e.g., "2025-11-19T00:00:00"). --- From f24a906c8352a20548762ad0397bbc9772a6e813 Mon Sep 17 00:00:00 2001 From: Said Rodrigues Date: Wed, 19 Nov 2025 11:39:19 -0300 Subject: [PATCH 10/10] docs(README): add initial fetch count and onward date docs --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 04ce774..462c448 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,8 @@ Provided below is a quick guide to get up and running. "Color": 8411391, "DescriptionLimit": 250, "MarkdownFormat": false, + "InitialFetchCount": 20, + "OnwardDate": "2025-11-01T00:00:00", "PersistenceOnShutdown": true } ],