Files
FreshRSS/app/Services/ExportService.php
T
abp002andAlejandro Borrego 48547b4a6c Reduce memory consumption of export (#9350)
Contributes to #6139 (export part; the import will follow in another PR)

Exporting favourites was collecting the whole file in memory. With many articles, it fails with the default PHP memory limit.

## Changes
- New `Minz_View::helperToFile()`, similar to `helperToString()`, writing the output of a helper into a file by chunks of 64 KiB instead of keeping it in memory.
- `FreshRSS_Export_Service`: the `generate*()` methods now return the path of a temporary file instead of the content, and `zip()` builds the archive from those files (`ZipArchive::addFile()` instead of `addFromString()`). Temporary files are created in `TMP_PATH` (already required to be writable) and deleted at the end of the script.
- The export is sent with `readfile()`: Web UI, `cli/export-zip-for-user.php`, `cli/export-opml-for-user.php` and the OPML export of the Google Reader API.
- Note: this changes what `FreshRSS_Export_Service` returns (a path instead of the content). All callers in the repository are updated, and I could not find any public extension using this service.

## Measurements
Peak PHP memory (`memory_get_peak_usage(true)`), SQLite database with synthetic starred articles:

| Scenario | Before | After |
|---|---|---|
| Web, export your favourites, 10,000 articles (69 MB JSON) | 136 MB (fails with the default `memory_limit` of 128 MB) | 6 MB |
| Web, ZIP (OPML + favourites + labelled articles + 4 feeds), 10,000 articles | 152 MB | 6 MB |
| CLI `export-zip-for-user.php`, 10,000 articles | 189 MB (fails with 128 MB) | 8 MB |
| Web, export your favourites, 30,000 articles (208 MB JSON) | 404 MB | 10 MB |

Co-authored-by: Alejandro Borrego <210245044+abp002@users.noreply.github.com>
2026-10-05 21:47:19 +02:00

200 lines
6.8 KiB
PHP
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<?php
declare(strict_types=1);
/**
* Provide useful methods to generate files to export.
*/
class FreshRSS_Export_Service {
private readonly FreshRSS_CategoryDAO $category_dao;
private readonly FreshRSS_FeedDAO $feed_dao;
private readonly FreshRSS_EntryDAO $entry_dao;
private readonly FreshRSS_TagDAO $tag_dao;
final public const FRSS_NAMESPACE = 'https://freshrss.org/opml';
final public const TYPE_HTML_XPATH = 'HTML+XPath';
final public const TYPE_XML_XPATH = 'XML+XPath';
final public const TYPE_RSS_ATOM = 'rss';
final public const TYPE_JSON_DOTPATH = 'JSON+DotPath'; // Legacy 1.24.0-dev
final public const TYPE_JSON_DOTNOTATION = 'JSON+DotNotation';
final public const TYPE_JSONFEED = 'JSONFeed';
final public const TYPE_HTML_XPATH_JSON_DOTNOTATION = 'HTML+XPath+JSON+DotNotation';
final public const PRIORITY_IMPORTANT = 'important';
final public const PRIORITY_MAIN_STREAM = 'main';
final public const PRIORITY_CATEGORY = 'category';
final public const PRIORITY_FEED = 'feed';
final public const PRIORITY_HIDDEN = 'hidden';
/**
* Initialize the service for the given user.
*/
public function __construct(private readonly string $username) {
$this->category_dao = FreshRSS_Factory::createCategoryDao($this->username);
$this->feed_dao = FreshRSS_Factory::createFeedDao($this->username);
$this->entry_dao = FreshRSS_Factory::createEntryDao($this->username);
$this->tag_dao = FreshRSS_Factory::createTagDao();
}
/**
* Generate OPML file.
* @return array{0:string,1:string} First item is the filename, second item is the path of a temporary file with the content
* @throws Minz_PermissionDeniedException
*/
public function generateOpml(): array {
$view = new FreshRSS_View();
$day = date('Y-m-d');
$view->categories = $this->category_dao->listSortedCategories(prePopulateFeeds: true, details: true);
$view->excludeMutedFeeds = false;
$view->includeSensitiveCurlParams = true;
return [
"feeds_{$day}.opml.xml",
self::helperToTempFile($view, 'export/opml'),
];
}
/**
* Generate the starred and labelled entries file.
*
* Both starred and labelled entries are put into a "starred" file, that’s
* why there is only one method for both.
*
* @phpstan-param 'S'|'T'|'ST' $type
* @param string $type must be one of:
* 'S' (starred/favourite),
* 'T' (taggued/labelled),
* 'ST' (starred or labelled)
* @return array{0:string,1:string} First item is the filename, second item is the path of a temporary file with the content
* @throws Minz_PermissionDeniedException
*/
public function generateStarredEntries(string $type): array {
$view = new FreshRSS_View();
$view->categories = $this->category_dao->listCategories(prePopulateFeeds: true);
$day = date('Y-m-d');
$view->list_title = _t('sub.import_export.starred_list');
$view->type = 'starred';
$entriesId = $this->entry_dao->listIdsWhere($type, 0, FreshRSS_Entry::STATE_ALL, order: 'ASC', limit: -1) ?? [];
$view->entryIdsTagNames = $this->tag_dao->getEntryIdsTagNames($entriesId);
// The following is a streamable query, i.e. must be last
$view->entries = $this->entry_dao->listWhere(
$type, 0, FreshRSS_Entry::STATE_ALL, order: 'ASC', limit: -1
);
return [
"starred_{$day}.json",
self::helperToTempFile($view, 'export/articles'),
];
}
/**
* Generate the entries file for the given feed.
* @return array{0:string,1:string}|null First item is the filename, second item is the path of a temporary file with the content.
* It also can return null if the feed doesn’t exist.
* @throws Minz_PermissionDeniedException
*/
public function generateFeedEntries(int $feed_id, int $max_number_entries): ?array {
$view = new FreshRSS_View();
$view->categories = $this->category_dao->listCategories(prePopulateFeeds: true);
$feed = FreshRSS_Category::findFeed($view->categories, $feed_id);
if ($feed === null) {
return null;
}
$view->feed = $feed;
$day = date('Y-m-d');
$filename = "feed_{$day}_" . $feed->categoryId() . '_' . $feed->id() . '.json';
$view->list_title = _t('sub.import_export.feed_list', $feed->name());
$view->type = 'feed/' . $feed->id();
$entriesId = $this->entry_dao->listIdsWhere(
'f', $feed->id(), FreshRSS_Entry::STATE_ALL, order: 'ASC', limit: $max_number_entries
) ?? [];
$view->entryIdsTagNames = $this->tag_dao->getEntryIdsTagNames($entriesId);
// The following is a streamable query, i.e. must be last
$view->entries = $this->entry_dao->listWhere(
'f', $feed->id(), FreshRSS_Entry::STATE_ALL, order: 'ASC', limit: $max_number_entries
);
return [
$filename,
self::helperToTempFile($view, 'export/articles'),
];
}
/**
* Generate the entries files for all the feeds.
* @return array<string,string> Keys are filenames and values are paths of temporary files with the contents.
* @throws Minz_PermissionDeniedException
*/
public function generateAllFeedEntries(int $max_number_entries): array {
$feed_ids = $this->feed_dao->listFeedsIds();
$exported_files = [];
foreach ($feed_ids as $feed_id) {
$result = $this->generateFeedEntries($feed_id, $max_number_entries);
if ($result === null) {
continue;
}
[$filename, $path] = $result;
$exported_files[$filename] = $path;
}
return $exported_files;
}
/**
* Compress several files in a Zip file.
* @param array<string,string> $files where the key is the filename, the value is the path of the file
* @return array{0:string,1:string|false} First item is the zip filename, second item is the path of a temporary zip file, or false in case of error
* @throws Minz_PermissionDeniedException
*/
public function zip(array $files): array {
$day = date('Y-m-d');
$zip_filename = 'freshrss_' . $this->username . '_' . $day . '_export.zip';
$zip_file = self::tempFile();
$zip_archive = new ZipArchive();
$zip_archive->open($zip_file, ZipArchive::OVERWRITE);
foreach ($files as $filename => $path) {
$zip_archive->addFile($path, $filename);
}
return [
$zip_filename,
$zip_archive->close() ? $zip_file : false,
];
}
/**
* Create a temporary file, which is deleted at the end of the script.
* @throws Minz_PermissionDeniedException
*/
private static function tempFile(): string {
$filename = tempnam(TMP_PATH, 'export');
if ($filename === false) {
throw new Minz_PermissionDeniedException(TMP_PATH);
}
register_shutdown_function(static fn() => @unlink($filename));
return $filename;
}
/**
* Render a view helper into a temporary file, instead of keeping the whole content in memory.
* @return string The path of the temporary file
* @throws Minz_PermissionDeniedException
*/
private static function helperToTempFile(FreshRSS_View $view, string $helper): string {
$filename = self::tempFile();
$view->helperToFile($helper, $filename);
return $filename;
}
}