mirror of
https://github.com/FreshRSS/FreshRSS.git
synced 2026-10-28 14:08:45 -04:00
Contributes to #6139 (export part; the import will follow in another PR) Exporting favourites was collecting the whole file in memory. With many articles, it fails with the default PHP memory limit. ## Changes - New `Minz_View::helperToFile()`, similar to `helperToString()`, writing the output of a helper into a file by chunks of 64 KiB instead of keeping it in memory. - `FreshRSS_Export_Service`: the `generate*()` methods now return the path of a temporary file instead of the content, and `zip()` builds the archive from those files (`ZipArchive::addFile()` instead of `addFromString()`). Temporary files are created in `TMP_PATH` (already required to be writable) and deleted at the end of the script. - The export is sent with `readfile()`: Web UI, `cli/export-zip-for-user.php`, `cli/export-opml-for-user.php` and the OPML export of the Google Reader API. - Note: this changes what `FreshRSS_Export_Service` returns (a path instead of the content). All callers in the repository are updated, and I could not find any public extension using this service. ## Measurements Peak PHP memory (`memory_get_peak_usage(true)`), SQLite database with synthetic starred articles: | Scenario | Before | After | |---|---|---| | Web, export your favourites, 10,000 articles (69 MB JSON) | 136 MB (fails with the default `memory_limit` of 128 MB) | 6 MB | | Web, ZIP (OPML + favourites + labelled articles + 4 feeds), 10,000 articles | 152 MB | 6 MB | | CLI `export-zip-for-user.php`, 10,000 articles | 189 MB (fails with 128 MB) | 8 MB | | Web, export your favourites, 30,000 articles (208 MB JSON) | 404 MB | 10 MB | Co-authored-by: Alejandro Borrego <210245044+abp002@users.noreply.github.com>
200 lines
6.8 KiB
PHP
200 lines
6.8 KiB
PHP
<?php
|
||
declare(strict_types=1);
|
||
|
||
/**
|
||
* Provide useful methods to generate files to export.
|
||
*/
|
||
class FreshRSS_Export_Service {
|
||
|
||
private readonly FreshRSS_CategoryDAO $category_dao;
|
||
|
||
private readonly FreshRSS_FeedDAO $feed_dao;
|
||
|
||
private readonly FreshRSS_EntryDAO $entry_dao;
|
||
|
||
private readonly FreshRSS_TagDAO $tag_dao;
|
||
|
||
final public const FRSS_NAMESPACE = 'https://freshrss.org/opml';
|
||
final public const TYPE_HTML_XPATH = 'HTML+XPath';
|
||
final public const TYPE_XML_XPATH = 'XML+XPath';
|
||
final public const TYPE_RSS_ATOM = 'rss';
|
||
final public const TYPE_JSON_DOTPATH = 'JSON+DotPath'; // Legacy 1.24.0-dev
|
||
final public const TYPE_JSON_DOTNOTATION = 'JSON+DotNotation';
|
||
final public const TYPE_JSONFEED = 'JSONFeed';
|
||
final public const TYPE_HTML_XPATH_JSON_DOTNOTATION = 'HTML+XPath+JSON+DotNotation';
|
||
|
||
final public const PRIORITY_IMPORTANT = 'important';
|
||
final public const PRIORITY_MAIN_STREAM = 'main';
|
||
final public const PRIORITY_CATEGORY = 'category';
|
||
final public const PRIORITY_FEED = 'feed';
|
||
final public const PRIORITY_HIDDEN = 'hidden';
|
||
|
||
/**
|
||
* Initialize the service for the given user.
|
||
*/
|
||
public function __construct(private readonly string $username) {
|
||
$this->category_dao = FreshRSS_Factory::createCategoryDao($this->username);
|
||
$this->feed_dao = FreshRSS_Factory::createFeedDao($this->username);
|
||
$this->entry_dao = FreshRSS_Factory::createEntryDao($this->username);
|
||
$this->tag_dao = FreshRSS_Factory::createTagDao();
|
||
}
|
||
|
||
/**
|
||
* Generate OPML file.
|
||
* @return array{0:string,1:string} First item is the filename, second item is the path of a temporary file with the content
|
||
* @throws Minz_PermissionDeniedException
|
||
*/
|
||
public function generateOpml(): array {
|
||
$view = new FreshRSS_View();
|
||
$day = date('Y-m-d');
|
||
$view->categories = $this->category_dao->listSortedCategories(prePopulateFeeds: true, details: true);
|
||
$view->excludeMutedFeeds = false;
|
||
$view->includeSensitiveCurlParams = true;
|
||
|
||
return [
|
||
"feeds_{$day}.opml.xml",
|
||
self::helperToTempFile($view, 'export/opml'),
|
||
];
|
||
}
|
||
|
||
/**
|
||
* Generate the starred and labelled entries file.
|
||
*
|
||
* Both starred and labelled entries are put into a "starred" file, that’s
|
||
* why there is only one method for both.
|
||
*
|
||
* @phpstan-param 'S'|'T'|'ST' $type
|
||
* @param string $type must be one of:
|
||
* 'S' (starred/favourite),
|
||
* 'T' (taggued/labelled),
|
||
* 'ST' (starred or labelled)
|
||
* @return array{0:string,1:string} First item is the filename, second item is the path of a temporary file with the content
|
||
* @throws Minz_PermissionDeniedException
|
||
*/
|
||
public function generateStarredEntries(string $type): array {
|
||
$view = new FreshRSS_View();
|
||
$view->categories = $this->category_dao->listCategories(prePopulateFeeds: true);
|
||
$day = date('Y-m-d');
|
||
|
||
$view->list_title = _t('sub.import_export.starred_list');
|
||
$view->type = 'starred';
|
||
$entriesId = $this->entry_dao->listIdsWhere($type, 0, FreshRSS_Entry::STATE_ALL, order: 'ASC', limit: -1) ?? [];
|
||
$view->entryIdsTagNames = $this->tag_dao->getEntryIdsTagNames($entriesId);
|
||
// The following is a streamable query, i.e. must be last
|
||
$view->entries = $this->entry_dao->listWhere(
|
||
$type, 0, FreshRSS_Entry::STATE_ALL, order: 'ASC', limit: -1
|
||
);
|
||
|
||
return [
|
||
"starred_{$day}.json",
|
||
self::helperToTempFile($view, 'export/articles'),
|
||
];
|
||
}
|
||
|
||
/**
|
||
* Generate the entries file for the given feed.
|
||
* @return array{0:string,1:string}|null First item is the filename, second item is the path of a temporary file with the content.
|
||
* It also can return null if the feed doesn’t exist.
|
||
* @throws Minz_PermissionDeniedException
|
||
*/
|
||
public function generateFeedEntries(int $feed_id, int $max_number_entries): ?array {
|
||
$view = new FreshRSS_View();
|
||
$view->categories = $this->category_dao->listCategories(prePopulateFeeds: true);
|
||
|
||
$feed = FreshRSS_Category::findFeed($view->categories, $feed_id);
|
||
if ($feed === null) {
|
||
return null;
|
||
}
|
||
$view->feed = $feed;
|
||
|
||
$day = date('Y-m-d');
|
||
$filename = "feed_{$day}_" . $feed->categoryId() . '_' . $feed->id() . '.json';
|
||
|
||
$view->list_title = _t('sub.import_export.feed_list', $feed->name());
|
||
$view->type = 'feed/' . $feed->id();
|
||
$entriesId = $this->entry_dao->listIdsWhere(
|
||
'f', $feed->id(), FreshRSS_Entry::STATE_ALL, order: 'ASC', limit: $max_number_entries
|
||
) ?? [];
|
||
$view->entryIdsTagNames = $this->tag_dao->getEntryIdsTagNames($entriesId);
|
||
// The following is a streamable query, i.e. must be last
|
||
$view->entries = $this->entry_dao->listWhere(
|
||
'f', $feed->id(), FreshRSS_Entry::STATE_ALL, order: 'ASC', limit: $max_number_entries
|
||
);
|
||
|
||
return [
|
||
$filename,
|
||
self::helperToTempFile($view, 'export/articles'),
|
||
];
|
||
}
|
||
|
||
/**
|
||
* Generate the entries files for all the feeds.
|
||
* @return array<string,string> Keys are filenames and values are paths of temporary files with the contents.
|
||
* @throws Minz_PermissionDeniedException
|
||
*/
|
||
public function generateAllFeedEntries(int $max_number_entries): array {
|
||
$feed_ids = $this->feed_dao->listFeedsIds();
|
||
|
||
$exported_files = [];
|
||
foreach ($feed_ids as $feed_id) {
|
||
$result = $this->generateFeedEntries($feed_id, $max_number_entries);
|
||
if ($result === null) {
|
||
continue;
|
||
}
|
||
|
||
[$filename, $path] = $result;
|
||
$exported_files[$filename] = $path;
|
||
}
|
||
|
||
return $exported_files;
|
||
}
|
||
|
||
/**
|
||
* Compress several files in a Zip file.
|
||
* @param array<string,string> $files where the key is the filename, the value is the path of the file
|
||
* @return array{0:string,1:string|false} First item is the zip filename, second item is the path of a temporary zip file, or false in case of error
|
||
* @throws Minz_PermissionDeniedException
|
||
*/
|
||
public function zip(array $files): array {
|
||
$day = date('Y-m-d');
|
||
$zip_filename = 'freshrss_' . $this->username . '_' . $day . '_export.zip';
|
||
|
||
$zip_file = self::tempFile();
|
||
$zip_archive = new ZipArchive();
|
||
$zip_archive->open($zip_file, ZipArchive::OVERWRITE);
|
||
|
||
foreach ($files as $filename => $path) {
|
||
$zip_archive->addFile($path, $filename);
|
||
}
|
||
|
||
return [
|
||
$zip_filename,
|
||
$zip_archive->close() ? $zip_file : false,
|
||
];
|
||
}
|
||
|
||
/**
|
||
* Create a temporary file, which is deleted at the end of the script.
|
||
* @throws Minz_PermissionDeniedException
|
||
*/
|
||
private static function tempFile(): string {
|
||
$filename = tempnam(TMP_PATH, 'export');
|
||
if ($filename === false) {
|
||
throw new Minz_PermissionDeniedException(TMP_PATH);
|
||
}
|
||
register_shutdown_function(static fn() => @unlink($filename));
|
||
return $filename;
|
||
}
|
||
|
||
/**
|
||
* Render a view helper into a temporary file, instead of keeping the whole content in memory.
|
||
* @return string The path of the temporary file
|
||
* @throws Minz_PermissionDeniedException
|
||
*/
|
||
private static function helperToTempFile(FreshRSS_View $view, string $helper): string {
|
||
$filename = self::tempFile();
|
||
$view->helperToFile($helper, $filename);
|
||
return $filename;
|
||
}
|
||
}
|