diff --git a/lib/Minz b/lib/Minz index 56149ae7..55dd44bc 160000 --- a/lib/Minz +++ b/lib/Minz @@ -1 +1 @@ -Subproject commit 56149ae79404a0b8dfec4247ad6b55a51d0d30ca +Subproject commit 55dd44bcc44bb2d2af479292cdbc9d4b332149dd diff --git a/src/migrations/Migration202104290001CreateFetchLogs.php b/src/migrations/Migration202104290001CreateFetchLogs.php new file mode 100644 index 00000000..14aa8081 --- /dev/null +++ b/src/migrations/Migration202104290001CreateFetchLogs.php @@ -0,0 +1,33 @@ +exec(<<<'SQL' + CREATE TABLE fetch_logs ( + id SERIAL PRIMARY KEY, + created_at TIMESTAMPTZ NOT NULL, + url TEXT NOT NULL, + host TEXT NOT NULL + ); + SQL); + + return true; + } + + public function rollback() + { + $database = \Minz\Database::get(); + + $database->exec(<<<'SQL' + DROP TABLE fetch_logs; + SQL); + + return true; + } +} diff --git a/src/models/FetchLog.php b/src/models/FetchLog.php new file mode 100644 index 00000000..c50b9067 --- /dev/null +++ b/src/models/FetchLog.php @@ -0,0 +1,47 @@ + + * @license http://www.gnu.org/licenses/agpl-3.0.en.html AGPL + */ +class FetchLog extends \Minz\Model +{ + use DaoConnector; + + public const PROPERTIES = [ + 'id' => [ + 'type' => 'integer', + ], + + 'created_at' => [ + 'type' => 'datetime', + ], + + 'url' => [ + 'type' => 'string', + 'required' => true, + ], + + 'host' => [ + 'type' => 'string', + 'required' => true, + ], + ]; + + /** + * Create a log in DB for the given URL. + * + * @param string $url + */ + public static function log($url) + { + $host = \flusio\utils\Belt::host($url); + $fetch_log = new self([ + 'url' => $url, + 'host' => $host, + ]); + $fetch_log->save(); + } +} diff --git a/src/models/dao/FetchLog.php b/src/models/dao/FetchLog.php new file mode 100644 index 00000000..c90e1f06 --- /dev/null +++ b/src/models/dao/FetchLog.php @@ -0,0 +1,19 @@ + + * @license http://www.gnu.org/licenses/agpl-3.0.en.html AGPL + */ +class FetchLog extends \Minz\DatabaseModel +{ + /** + * @throws \Minz\Errors\DatabaseError + */ + public function __construct() + { + $properties = array_keys(\flusio\models\FetchLog::PROPERTIES); + parent::__construct('fetch_logs', 'id', $properties); + } +} diff --git a/src/schema.sql b/src/schema.sql index 038ccb78..febd65f5 100644 --- a/src/schema.sql +++ b/src/schema.sql @@ -75,6 +75,13 @@ CREATE TABLE importations ( user_id TEXT NOT NULL REFERENCES users ON DELETE CASCADE ON UPDATE CASCADE ); +CREATE TABLE fetch_logs ( + id SERIAL PRIMARY KEY, + created_at TIMESTAMPTZ NOT NULL, + url TEXT NOT NULL, + host TEXT NOT NULL +); + CREATE TABLE collections ( id TEXT PRIMARY KEY, created_at TIMESTAMPTZ NOT NULL, diff --git a/src/services/FeedFetcher.php b/src/services/FeedFetcher.php index 62ee2876..b333f608 100644 --- a/src/services/FeedFetcher.php +++ b/src/services/FeedFetcher.php @@ -170,6 +170,7 @@ public function fetchUrl($url) $response = \SpiderBits\Response::fromText($cached_response); } else { // ... or via HTTP + models\FetchLog::log($url); try { $response = $this->http->get($url); } catch (\SpiderBits\HttpError $e) { diff --git a/src/services/Image.php b/src/services/Image.php index db284a84..805c9351 100644 --- a/src/services/Image.php +++ b/src/services/Image.php @@ -57,6 +57,7 @@ public function generatePreviews($image_url) return basename($card_file_exists[0]); } + models\FetchLog::log($image_url); try { $response = $this->http->get($image_url); } catch (\SpiderBits\HttpError $e) { diff --git a/src/services/LinkFetcher.php b/src/services/LinkFetcher.php index 5eeb215e..bffd252f 100644 --- a/src/services/LinkFetcher.php +++ b/src/services/LinkFetcher.php @@ -2,6 +2,7 @@ namespace flusio\services; +use flusio\models; use flusio\utils; /** @@ -102,6 +103,7 @@ public function fetchUrl($url) ]; } + models\FetchLog::log($url); try { $response = $this->http->get($url, [], $options); } catch (\SpiderBits\HttpError $e) { diff --git a/tests/jobs/scheduled/FeedsSyncTest.php b/tests/jobs/scheduled/FeedsSyncTest.php index 75bb03bb..bdfa1d11 100644 --- a/tests/jobs/scheduled/FeedsSyncTest.php +++ b/tests/jobs/scheduled/FeedsSyncTest.php @@ -62,6 +62,27 @@ public function testPerform() $this->assertGreaterThan(0, $links_number); } + public function testPerformLogsFetch() + { + $feed_url = 'https://flus.fr/carnet/feeds/all.atom.xml'; + $collection_id = $this->create('collection', [ + 'type' => 'feed', + 'name' => $this->fake('sentence'), + 'feed_url' => $feed_url, + 'feed_fetched_at' => \Minz\Time::ago(2, 'hours')->format(\Minz\Model::DATETIME_FORMAT), + ]); + $feeds_sync_job = new FeedsSync(); + + $this->assertSame(0, models\FetchLog::count()); + + $feeds_sync_job->perform(); + + $this->assertSame(1, models\FetchLog::count()); + $fetch_log = models\FetchLog::take(); + $this->assertSame($feed_url, $fetch_log->url); + $this->assertSame('flus.fr', $fetch_log->host); + } + public function testPerformSavesResponseInCache() { $feed_url = 'https://flus.fr/carnet/feeds/all.atom.xml'; diff --git a/tests/jobs/scheduled/LinksFetcherTest.php b/tests/jobs/scheduled/LinksFetcherTest.php index 881342fa..d73402a1 100644 --- a/tests/jobs/scheduled/LinksFetcherTest.php +++ b/tests/jobs/scheduled/LinksFetcherTest.php @@ -58,6 +58,24 @@ public function testPerform() $this->assertSame(200, $link->fetched_code); } + public function testPerformLogsFetch() + { + $link_id = $this->create('link', [ + 'url' => 'https://github.com/flusio/flusio', + 'title' => 'https://github.com/flusio/flusio', + ]); + $links_fetcher_job = new LinksFetcher(); + + $this->assertSame(0, models\FetchLog::count()); + + $links_fetcher_job->perform(); + + $this->assertSame(1, models\FetchLog::count()); + $fetch_log = models\FetchLog::take(); + $this->assertSame('https://github.com/flusio/flusio', $fetch_log->url); + $this->assertSame('github.com', $fetch_log->host); + } + public function testPerformSavesResponseInCache() { $url = 'https://github.com/flusio/flusio';