mirror/FreshRSS
0
0 Fork 0
mirror of https://github.com/FreshRSS/FreshRSS.git synced 2026-10-20 10:14:33 -04:00
Code Issues Packages Projects Releases Wiki Activity

Handle redirects when scraping feed from HTML (#7654)

Browse Source 
* Handle redirects when scraping feed from HTML

* pass codesniffer

* pass PHPStan

* Optimize

* Another approach relying on HTML base
Standard way to save an HTML document with relative references

* Fix case of existing HTML base
which should not be overriden

---------

Co-authored-by: Alexandre Alapetite <alexandre@alapetite.fr>
This commit is contained in:
InverleandAlexandre Alapetite authored and GitHub committed 2025-06-22 00:09:18 +02:00
1 parent a6948218fb
commit 18b5c8ec6d
5 files changed
+55 -12
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/Models/Entry.php
+7 -3
View File
@@ -845,7 +845,7 @@ HTML;
* @param string $url Overridden URL. Will default to the entry URL.
* @throws Minz_Exception
*/
public function getContentByParsing(string $url = '', int $maxRedirs = 3): string {
public function getContentByParsing(string $url = '', int $maxRedirs = 4): string {
$url = $url ?: htmlspecialchars_decode($this->link(), ENT_QUOTES);
$feed = $this->feed();
if ($url === '' || $feed === null || $feed->pathEntries() === '') {
@@ -872,12 +872,16 @@ HTML;
}
$cachePath = $feed->cacheFilename($url . '#' . $feed->pathEntries());
$html = httpGet($url, $cachePath, 'html', $feed->attributes(), $feed->curlOptions());
if (strlen($html) > 0) {
$response = httpGet($url, $cachePath, 'html', $feed->attributes(), $feed->curlOptions());
$html = $response['body'];
if ($html !== '') {
$doc = new DOMDocument();
$doc->loadHTML($html, LIBXML_NONET | LIBXML_NOERROR | LIBXML_NOWARNING);
$xpath = new DOMXPath($doc);
// Account for HTTP redirections
$url = $response['effective_url'] ?: $url;
$maxRedirs -= $response['redirect_count'];
if ($maxRedirs > 0) {
//Follow any HTML redirection
$metas = $xpath->query('//meta[@content]') ?: [];