Log fetches to external websites in DB

src/migrations/Migration202104290001CreateFetchLogs.php

@@ -0,0 +1,33 @@
+<?php
+
+namespace flusio\migrations;
+
+class Migration202104290001CreateFetchLogs
+{
+    public function migrate()
+    {
+        $database = \Minz\Database::get();
+
+        $database->exec(<<<'SQL'
+            CREATE TABLE fetch_logs (
+                id SERIAL PRIMARY KEY,
+                created_at TIMESTAMPTZ NOT NULL,
+                url TEXT NOT NULL,
+                host TEXT NOT NULL
+            );
+        SQL);
+
+        return true;
+    }
+
+    public function rollback()
+    {
+        $database = \Minz\Database::get();
+
+        $database->exec(<<<'SQL'
+            DROP TABLE fetch_logs;
+        SQL);
+
+        return true;
+    }
+}

src/models/FetchLog.php

@@ -0,0 +1,47 @@
+<?php
+
+namespace flusio\models;
+
+/**
+ * @author  Marien Fressinaud <[email protected]>
+ * @license http://www.gnu.org/licenses/agpl-3.0.en.html AGPL
+ */
+class FetchLog extends \Minz\Model
+{
+    use DaoConnector;
+
+    public const PROPERTIES = [
+        'id' => [
+            'type' => 'integer',
+        ],
+
+        'created_at' => [
+            'type' => 'datetime',
+        ],
+
+        'url' => [
+            'type' => 'string',
+            'required' => true,
+        ],
+
+        'host' => [
+            'type' => 'string',
+            'required' => true,
+        ],
+    ];
+
+    /**
+     * Create a log in DB for the given URL.
+     *
+     * @param string $url
+     */
+    public static function log($url)
+    {
+        $host = \flusio\utils\Belt::host($url);
+        $fetch_log = new self([
+            'url' => $url,
+            'host' => $host,
+        ]);
+        $fetch_log->save();
+    }
+}

src/models/dao/FetchLog.php

@@ -0,0 +1,19 @@
+<?php
+
+namespace flusio\models\dao;
+
+/**
+ * @author  Marien Fressinaud <[email protected]>
+ * @license http://www.gnu.org/licenses/agpl-3.0.en.html AGPL
+ */
+class FetchLog extends \Minz\DatabaseModel
+{
+    /**
+     * @throws \Minz\Errors\DatabaseError
+     */
+    public function __construct()
+    {
+        $properties = array_keys(\flusio\models\FetchLog::PROPERTIES);
+        parent::__construct('fetch_logs', 'id', $properties);
+    }
+}

src/schema.sql

@@ -75,6 +75,13 @@ CREATE TABLE importations (
     user_id TEXT NOT NULL REFERENCES users ON DELETE CASCADE ON UPDATE CASCADE
 );
 
+CREATE TABLE fetch_logs (
+    id SERIAL PRIMARY KEY,
+    created_at TIMESTAMPTZ NOT NULL,
+    url TEXT NOT NULL,
+    host TEXT NOT NULL
+);
+
 CREATE TABLE collections (
     id TEXT PRIMARY KEY,
     created_at TIMESTAMPTZ NOT NULL,

src/services/FeedFetcher.php

@@ -170,6 +170,7 @@ public function fetchUrl($url)
             $response = \SpiderBits\Response::fromText($cached_response);
         } else {
             // ... or via HTTP
+            models\FetchLog::log($url);
             try {
                 $response = $this->http->get($url);
             } catch (\SpiderBits\HttpError $e) {

src/services/Image.php

@@ -57,6 +57,7 @@ public function generatePreviews($image_url)
             return basename($card_file_exists[0]);
         }
 
+        models\FetchLog::log($image_url);
         try {
             $response = $this->http->get($image_url);
         } catch (\SpiderBits\HttpError $e) {

src/services/LinkFetcher.php

@@ -2,6 +2,7 @@
 
 namespace flusio\services;
 
+use flusio\models;
 use flusio\utils;
 
 /**
@@ -102,6 +103,7 @@ public function fetchUrl($url)
                 ];
             }
 
+            models\FetchLog::log($url);
             try {
                 $response = $this->http->get($url, [], $options);
             } catch (\SpiderBits\HttpError $e) {

tests/jobs/scheduled/FeedsSyncTest.php

@@ -62,6 +62,27 @@ public function testPerform()
         $this->assertGreaterThan(0, $links_number);
     }
 
+    public function testPerformLogsFetch()
+    {
+        $feed_url = 'https://flus.fr/carnet/feeds/all.atom.xml';
+        $collection_id = $this->create('collection', [
+            'type' => 'feed',
+            'name' => $this->fake('sentence'),
+            'feed_url' => $feed_url,
+            'feed_fetched_at' => \Minz\Time::ago(2, 'hours')->format(\Minz\Model::DATETIME_FORMAT),
+        ]);
+        $feeds_sync_job = new FeedsSync();
+
+        $this->assertSame(0, models\FetchLog::count());
+
+        $feeds_sync_job->perform();
+
+        $this->assertSame(1, models\FetchLog::count());
+        $fetch_log = models\FetchLog::take();
+        $this->assertSame($feed_url, $fetch_log->url);
+        $this->assertSame('flus.fr', $fetch_log->host);
+    }
+
     public function testPerformSavesResponseInCache()
     {
         $feed_url = 'https://flus.fr/carnet/feeds/all.atom.xml';

tests/jobs/scheduled/LinksFetcherTest.php

@@ -58,6 +58,24 @@ public function testPerform()
         $this->assertSame(200, $link->fetched_code);
     }
 
+    public function testPerformLogsFetch()
+    {
+        $link_id = $this->create('link', [
+            'url' => 'https://github.com/flusio/flusio',
+            'title' => 'https://github.com/flusio/flusio',
+        ]);
+        $links_fetcher_job = new LinksFetcher();
+
+        $this->assertSame(0, models\FetchLog::count());
+
+        $links_fetcher_job->perform();
+
+        $this->assertSame(1, models\FetchLog::count());
+        $fetch_log = models\FetchLog::take();
+        $this->assertSame('https://github.com/flusio/flusio', $fetch_log->url);
+        $this->assertSame('github.com', $fetch_log->host);
+    }
+
     public function testPerformSavesResponseInCache()
     {
         $url = 'https://github.com/flusio/flusio';