From 05cabe99ae48a187a77e0246dfffc60f2434b5e5 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Wed, 5 Oct 2016 00:39:54 +0200 Subject: [PATCH 1/4] Take better advantage of other users refresh --- app/Controllers/feedController.php | 18 ++++++++++++++++-- app/Models/EntryDAO.php | 7 +++++-- app/Models/Feed.php | 4 ++++ app/Models/FeedDAO.php | 8 ++++---- 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php index c2a22aeb3..15f373833 100755 --- a/app/Controllers/feedController.php +++ b/app/Controllers/feedController.php @@ -242,7 +242,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController { $feeds[] = $feed; } } else { - $feeds = $feedDAO->listFeedsOrderUpdate(FreshRSS_Context::$user_conf->ttl_default); + $feeds = $feedDAO->listFeedsOrderUpdate(-1); } // Calculate date of oldest entries we accept in DB. @@ -266,6 +266,20 @@ class FreshRSS_feed_Controller extends Minz_ActionController { continue; //When PubSubHubbub is used, do not pull refresh so often } + $mtime = 0; + $ttl = $feed->ttl(); + if ($ttl == -1) { + continue; //Feed refresh is disabled + } + if (feed->lastUpdate() < time() - ($ttl == -2 ? FreshRSS_Context::$user_conf->ttl_default : $ttl)) { + //Too early to refresh from source, but check if the feed was already updated by another user + $mtime = feed->cacheModifiedTime(); + if (feed->lastUpdate() >= $mtime + 10) { + continue; //Nothing newer from other users + } + Minz_Log::debug($feed->url() . ' was recently updated by another user'); + } + if (!$feed->lock()) { Minz_Log::notice('Feed already being actualized: ' . $feed->url()); continue; @@ -358,7 +372,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController { $entryDAO->addEntry($entry->toArray()); } } - $entryDAO->updateLastSeen($feed->id(), $oldGuids); + $entryDAO->updateLastSeen($feed->id(), $oldGuids, $mtime); } if ($feed_history >= 0 && rand(0, 30) === 1) { diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php index 63565e73a..43ac21f9e 100644 --- a/app/Models/EntryDAO.php +++ b/app/Models/EntryDAO.php @@ -707,13 +707,16 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { } } - public function updateLastSeen($id_feed, $guids) { + public function updateLastSeen($id_feed, $guids, $mtime = 0) { if (count($guids) < 1) { return 0; } $sql = 'UPDATE `' . $this->prefix . 'entry` SET `lastSeen`=? WHERE id_feed=? AND guid IN (' . str_repeat('?,', count($guids) - 1). '?)'; $stm = $this->bd->prepare($sql); - $values = array(time(), $id_feed); + if ($mtime <= 0) { + $mtime = time(); + } + $values = array($mtime, $id_feed); $values = array_merge($values, $guids); if ($stm && $stm->execute($values)) { return $stm->rowCount(); diff --git a/app/Models/Feed.php b/app/Models/Feed.php index f2f345662..f435620c8 100644 --- a/app/Models/Feed.php +++ b/app/Models/Feed.php @@ -340,6 +340,10 @@ class FreshRSS_Feed extends Minz_Model { $this->entries = $entries; } + function cacheModifiedTime() { + return @filemtime(CACHE_PATH . '/' . md5($this->url) . '.spc'); + } + function lock() { $this->lockPath = TMP_PATH . '/' . $this->hash() . '.freshrss.lock'; if (file_exists($this->lockPath) && ((time() - @filemtime($this->lockPath)) > 3600)) { diff --git a/app/Models/FeedDAO.php b/app/Models/FeedDAO.php index 6e6d8857b..ad3cb0c2e 100644 --- a/app/Models/FeedDAO.php +++ b/app/Models/FeedDAO.php @@ -222,13 +222,13 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return $feedCategoryNames; } + /** + * Use $defaultCacheDuration == -1 to return all feeds, without filtering them by TTL. + */ public function listFeedsOrderUpdate($defaultCacheDuration = 3600) { - if ($defaultCacheDuration < 0) { - $defaultCacheDuration = 2147483647; - } $sql = 'SELECT id, url, name, website, `lastUpdate`, `pathEntries`, `httpAuth`, keep_history, ttl ' . 'FROM `' . $this->prefix . 'feed` ' - . 'WHERE ttl <> -1 AND `lastUpdate` < (' . (time() + 60) . '-(CASE WHEN ttl=-2 THEN ' . intval($defaultCacheDuration) . ' ELSE ttl END)) ' + . ($defaultCacheDuration < 0 ? '' : 'WHERE ttl <> -1 AND `lastUpdate` < (' . (time() + 60) . '-(CASE WHEN ttl=-2 THEN ' . intval($defaultCacheDuration) . ' ELSE ttl END)) ') . 'ORDER BY `lastUpdate`'; $stm = $this->bd->prepare($sql); if (!($stm && $stm->execute())) { From 9e34d7275eb936e1a08b872aca7adbce5a5674c5 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Wed, 5 Oct 2016 00:44:14 +0200 Subject: [PATCH 2/4] Typo --- app/Controllers/feedController.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php index 15f373833..d85a53446 100755 --- a/app/Controllers/feedController.php +++ b/app/Controllers/feedController.php @@ -271,10 +271,10 @@ class FreshRSS_feed_Controller extends Minz_ActionController { if ($ttl == -1) { continue; //Feed refresh is disabled } - if (feed->lastUpdate() < time() - ($ttl == -2 ? FreshRSS_Context::$user_conf->ttl_default : $ttl)) { - //Too early to refresh from source, but check if the feed was already updated by another user - $mtime = feed->cacheModifiedTime(); - if (feed->lastUpdate() >= $mtime + 10) { + if ($feed->lastUpdate() < time() + 60 - ($ttl == -2 ? FreshRSS_Context::$user_conf->ttl_default : $ttl)) { + //Too early to refresh from source, but check whether the feed was updated by another user + $mtime = $feed->cacheModifiedTime(); + if ($feed->lastUpdate() >= $mtime + 10) { continue; //Nothing newer from other users } Minz_Log::debug($feed->url() . ' was recently updated by another user'); From fe65eec5bbdaee37177e3673e31e241b1f1b938d Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Wed, 5 Oct 2016 17:48:24 +0200 Subject: [PATCH 3/4] Better multiuser update --- app/Controllers/feedController.php | 9 +++++---- app/Models/FeedDAO.php | 10 +++++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php index d85a53446..dadc8aa83 100755 --- a/app/Controllers/feedController.php +++ b/app/Controllers/feedController.php @@ -271,13 +271,14 @@ class FreshRSS_feed_Controller extends Minz_ActionController { if ($ttl == -1) { continue; //Feed refresh is disabled } - if ($feed->lastUpdate() < time() + 60 - ($ttl == -2 ? FreshRSS_Context::$user_conf->ttl_default : $ttl)) { + if ($feed->lastUpdate() + 10 >= time() - ($ttl == -2 ? FreshRSS_Context::$user_conf->ttl_default : $ttl)) { //Too early to refresh from source, but check whether the feed was updated by another user $mtime = $feed->cacheModifiedTime(); - if ($feed->lastUpdate() >= $mtime + 10) { + if ($feed->lastUpdate() + 10 >= $mtime) { continue; //Nothing newer from other users } - Minz_Log::debug($feed->url() . ' was recently updated by another user'); + //Minz_Log::debug($feed->url() . ' was updated at ' . date('c', $mtime) . ' by another user'); + //Will take advantage of the newer cache } if (!$feed->lock()) { @@ -391,7 +392,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController { } } - $feedDAO->updateLastUpdate($feed->id(), 0, $entryDAO->inTransaction()); + $feedDAO->updateLastUpdate($feed->id(), false, $entryDAO->inTransaction(), $mtime); if ($entryDAO->inTransaction()) { $entryDAO->commit(); } diff --git a/app/Models/FeedDAO.php b/app/Models/FeedDAO.php index ad3cb0c2e..c680d270c 100644 --- a/app/Models/FeedDAO.php +++ b/app/Models/FeedDAO.php @@ -82,7 +82,7 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable { } } - public function updateLastUpdate($id, $inError = 0, $updateCache = true) { + public function updateLastUpdate($id, $inError = false, $updateCache = true, $mtime = 0) { if ($updateCache) { $sql = 'UPDATE `' . $this->prefix . 'feed` ' //2 sub-requests with FOREIGN KEY(e.id_feed), INDEX(e.is_read) faster than 1 request with GROUP BY or CASE . 'SET `cache_nbEntries`=(SELECT COUNT(e1.id) FROM `' . $this->prefix . 'entry` e1 WHERE e1.id_feed=`' . $this->prefix . 'feed`.id),' @@ -95,9 +95,13 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable { . 'WHERE id=?'; } + if ($mtime <= 0) { + $mtime = time(); + } + $values = array( - time(), - $inError, + $mtime, + $inError ? 1 : 0, $id, ); From 1df76fb6b5bd3dced52bd3b41eda54a518bd399b Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Wed, 5 Oct 2016 21:00:19 +0200 Subject: [PATCH 4/4] Changelog multiuser optimisation --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fcb3410e..823a2e8a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ * Since X hours: `https://freshrss.example/i/?a=rss&hours=3` * Explicit number: `https://freshrss.example/i/?a=rss&nb=10` * Limited by `min_posts_per_rss` and `max_posts_per_rss` in user config + * In a multi-user context, take better advantage of other users’ refreshes [#1280](https://github.com/FreshRSS/FreshRSS/pull/1280) * Support custom ports `localhost:3306` for database servers [#1241](https://github.com/FreshRSS/FreshRSS/issues/1241) * Add date to exported files [#1240](https://github.com/FreshRSS/FreshRSS/issues/1240) * Bug fixing