mirror of
https://github.com/FreshRSS/FreshRSS.git
synced 2026-10-28 14:08:45 -04:00
abp002andAlejandro Borrego
48547b4a6c
Contributes to #6139 (export part; the import will follow in another PR) Exporting favourites was collecting the whole file in memory. With many articles, it fails with the default PHP memory limit. ## Changes - New `Minz_View::helperToFile()`, similar to `helperToString()`, writing the output of a helper into a file by chunks of 64 KiB instead of keeping it in memory. - `FreshRSS_Export_Service`: the `generate*()` methods now return the path of a temporary file instead of the content, and `zip()` builds the archive from those files (`ZipArchive::addFile()` instead of `addFromString()`). Temporary files are created in `TMP_PATH` (already required to be writable) and deleted at the end of the script. - The export is sent with `readfile()`: Web UI, `cli/export-zip-for-user.php`, `cli/export-opml-for-user.php` and the OPML export of the Google Reader API. - Note: this changes what `FreshRSS_Export_Service` returns (a path instead of the content). All callers in the repository are updated, and I could not find any public extension using this service. ## Measurements Peak PHP memory (`memory_get_peak_usage(true)`), SQLite database with synthetic starred articles: | Scenario | Before | After | |---|---|---| | Web, export your favourites, 10,000 articles (69 MB JSON) | 136 MB (fails with the default `memory_limit` of 128 MB) | 6 MB | | Web, ZIP (OPML + favourites + labelled articles + 4 feeds), 10,000 articles | 152 MB | 6 MB | | CLI `export-zip-for-user.php`, 10,000 articles | 189 MB (fails with 128 MB) | 8 MB | | Web, export your favourites, 30,000 articles (208 MB JSON) | 404 MB | 10 MB | Co-authored-by: Alejandro Borrego <210245044+abp002@users.noreply.github.com>
78 lines
2.6 KiB
PHP
78 lines
2.6 KiB
PHP
<?php
|
|
declare(strict_types=1);
|
|
|
|
final class exportArticlesTest extends \PHPUnit\Framework\TestCase {
|
|
private string $filename;
|
|
|
|
#[\Override]
|
|
public static function setUpBeforeClass(): void {
|
|
// `FreshRSS_View` needs a system configuration; the shipped defaults are enough to render an export.
|
|
Minz_Configuration::register('system', FRESHRSS_PATH . '/config.default.php', FRESHRSS_PATH . '/config.default.php');
|
|
}
|
|
|
|
#[\Override]
|
|
protected function setUp(): void {
|
|
$filename = tempnam(sys_get_temp_dir(), 'freshrss_test_');
|
|
self::assertIsString($filename);
|
|
$this->filename = $filename;
|
|
}
|
|
|
|
#[\Override]
|
|
protected function tearDown(): void {
|
|
@unlink($this->filename);
|
|
}
|
|
|
|
/** @return list<FreshRSS_Entry> */
|
|
private static function entries(int $count): array {
|
|
$entries = [];
|
|
for ($i = 1; $i <= $count; $i++) {
|
|
$entry = new FreshRSS_Entry(1, 'guid-' . $i, 'Title ' . $i, '', str_repeat('<p>Contenu exporté ' . $i . '</p>', 200),
|
|
'https://example.net/' . $i, 1700000000 + $i);
|
|
$entry->_id(1700000000000000 + $i);
|
|
$entries[] = $entry;
|
|
}
|
|
return $entries;
|
|
}
|
|
|
|
/** @param iterable<FreshRSS_Entry> $entries */
|
|
private static function view(iterable $entries): FreshRSS_View {
|
|
$view = new FreshRSS_View();
|
|
$view->internal_rendering = true;
|
|
$view->list_title = 'Starred';
|
|
$view->type = 'starred';
|
|
$view->feed = new FreshRSS_Feed('https://example.net/feed.xml', false);
|
|
$view->entries = $entries;
|
|
return $view;
|
|
}
|
|
|
|
public function testHelperToFileWritesTheSameContentAsHelperToString(): void {
|
|
$expected = self::view(self::entries(50))->helperToString('export/articles');
|
|
self::assertGreaterThan(65536, strlen($expected), 'The export must be larger than one chunk of the output buffer');
|
|
|
|
file_put_contents($this->filename, 'Previous content, to be overwritten');
|
|
$this->expectOutputString(''); // Nothing must leak to the output
|
|
self::view(self::entries(50))->helperToFile('export/articles', $this->filename);
|
|
|
|
self::assertSame($expected, file_get_contents($this->filename));
|
|
$json = json_decode($expected, true);
|
|
self::assertIsArray($json);
|
|
self::assertIsArray($json['items'] ?? null);
|
|
self::assertCount(50, $json['items']);
|
|
}
|
|
|
|
public function testHelperToFileClosesTheOutputBufferOnError(): void {
|
|
$entries = (static function (): Generator {
|
|
yield from self::entries(1);
|
|
throw new RuntimeException('Database error');
|
|
})();
|
|
$level = ob_get_level();
|
|
try {
|
|
self::view($entries)->helperToFile('export/articles', $this->filename);
|
|
self::fail('The exception must be propagated');
|
|
} catch (RuntimeException $e) {
|
|
self::assertSame('Database error', $e->getMessage());
|
|
}
|
|
self::assertSame($level, ob_get_level());
|
|
}
|
|
}
|