From a7bd8aec7d39c87aa0463094f32d228eefa98bd6 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 18 Aug 2019 22:18:42 +0200 Subject: [PATCH] CLI to export/import any database to/from SQLite Require PHP 5.5+ https://github.com/FreshRSS/FreshRSS/pull/2495 --- app/Models/CategoryDAO.php | 9 ++ app/Models/DatabaseDAO.php | 163 +++++++++++++++++++++++++++++++++ app/Models/EntryDAO.php | 16 +++- app/Models/FeedDAO.php | 18 +++- app/Models/TagDAO.php | 18 ++++ app/SQL/install.sql.sqlite.php | 2 +- cli/README.md | 25 +++-- cli/export-sqlite-for-user.php | 28 ++++++ cli/import-sqlite-for-user.php | 29 ++++++ lib/Minz/ModelPdo.php | 9 +- 10 files changed, 303 insertions(+), 14 deletions(-) create mode 100644 cli/export-sqlite-for-user.php create mode 100644 cli/import-sqlite-for-user.php diff --git a/app/Models/CategoryDAO.php b/app/Models/CategoryDAO.php index 6535adae7..1e7fb88d0 100644 --- a/app/Models/CategoryDAO.php +++ b/app/Models/CategoryDAO.php @@ -77,6 +77,15 @@ class FreshRSS_CategoryDAO extends Minz_ModelPdo implements FreshRSS_Searchable } } + public function select() { + $sql = 'SELECT id, name FROM `' . $this->prefix . 'category`'; + $stm = $this->bd->prepare($sql); + $stm->execute(); + while ($row = $stm->fetch(PDO::FETCH_ASSOC)) { + yield $row; + } + } + public function searchById($id) { $sql = 'SELECT * FROM `' . $this->prefix . 'category` WHERE id=?'; $stm = $this->bd->prepare($sql); diff --git a/app/Models/DatabaseDAO.php b/app/Models/DatabaseDAO.php index b331eccc3..95a33364c 100644 --- a/app/Models/DatabaseDAO.php +++ b/app/Models/DatabaseDAO.php @@ -164,4 +164,167 @@ class FreshRSS_DatabaseDAO extends Minz_ModelPdo { public function minorDbMaintenance() { $this->ensureCaseInsensitiveGuids(); } + + const SQLITE_EXPORT = 1; + const SQLITE_IMPORT = 2; + + public function dbCopy($filename, $mode) { + $error = ''; + + $catDAO = FreshRSS_Factory::createCategoryDao(); + $feedDAO = FreshRSS_Factory::createFeedDao(); + $entryDAO = FreshRSS_Factory::createEntryDao(); + $tagDAO = FreshRSS_Factory::createTagDao(); + + switch ($mode) { + case self::SQLITE_EXPORT: + if (@filesize($filename) > 0) { + $error = 'Error: SQLite export file already exists: ' . $filename; + } + break; + case self::SQLITE_IMPORT: + if (!is_readable($filename)) { + $error = 'Error: SQLite import file is not readable: ' . $filename; + } else { + $nbEntries = $entryDAO->countUnreadRead(); + if ($nbEntries['all'] > 0) { + $error = 'Error: Destination database already contains some entries!'; + } + } + break; + default: + $error = 'Invalid copy mode!'; + break; + } + if ($error != '') { + goto done; + } + + $sqlite = null; + + try { + $sqlite = new MinzPDOSQLite('sqlite:' . $filename); + $sqlite->exec('PRAGMA foreign_keys = ON;'); + + if ($mode === self::SQLITE_EXPORT) { + require_once(APP_PATH . '/SQL/install.sql.sqlite.php'); + + global $SQL_CREATE_TABLES, $SQL_CREATE_TABLE_ENTRYTMP, $SQL_CREATE_TABLE_TAGS; + if (is_array($SQL_CREATE_TABLES)) { + $instructions = array_merge($SQL_CREATE_TABLES, $SQL_CREATE_TABLE_ENTRYTMP, $SQL_CREATE_TABLE_TAGS, + array('DELETE FROM entrytag', 'DELETE FROM tag', 'DELETE FROM entrytmp', 'DELETE FROM entry', + 'DELETE FROM feed', 'DELETE FROM category')); + foreach ($instructions as $instruction) { + $sql = sprintf($instruction, '', _t('gen.short.default_category')); + $stm = $sqlite->prepare($sql); + $stm->execute(); + } + } + } + } catch (Exception $e) { + $error .= ' Error while initialising SQLite copy: ' . $e->getMessage(); + goto done; + } + + Minz_ModelPdo::clean(); + $categoryDAOSQLite = new FreshRSS_CategoryDAO('', '', $sqlite); + $feedDAOSQLite = new FreshRSS_FeedDAOSQLite('', '', $sqlite); + $entryDAOSQLite = new FreshRSS_EntryDAOSQLite('', '', $sqlite); + $tagDAOSQLite = new FreshRSS_TagDAOSQLite('', '', $sqlite); + + switch ($mode) { + case self::SQLITE_EXPORT: + $catFrom = $catDAO; $catTo = $categoryDAOSQLite; + $feedFrom = $feedDAO; $feedTo = $feedDAOSQLite; + $entryFrom = $entryDAO; $entryTo = $entryDAOSQLite; + $tagFrom = $tagDAO; $tagTo = $tagDAOSQLite; + break; + case self::SQLITE_IMPORT: + $catFrom = $categoryDAOSQLite; $catTo = $catDAO; + $feedFrom = $feedDAOSQLite; $feedTo = $feedDAO; + $entryFrom = $entryDAOSQLite; $entryTo = $entryDAO; + $tagFrom = $tagDAOSQLite; $tagTo = $tagDAO; + break; + } + + $idMaps = []; + + $catTo->beginTransaction(); + foreach ($catFrom->select() as $category) { + $catId = $catTo->addCategory($category); + if ($catId == false) { + $error .= ' Error during SQLite copy of categories!'; + goto done; + } + $idMaps['c' . $category['id']] = $catId; + } + foreach ($feedFrom->select() as $feed) { + $feed['category'] = empty($idMaps['c' . $feed['category']]) ? + FreshRSS_CategoryDAO::DEFAULTCATEGORYID : $idMaps['c' . $feed['category']]; + $feedId = $feedTo->addFeed($feed); + if ($feedId == false) { + $error .= ' Error during SQLite copy of feeds!'; + goto done; + } + $idMaps['f' . $feed['id']] = $feedId; + } + $catTo->commit(); + + $nbEntries = $entryFrom->countUnreadRead(); + $total = $nbEntries['all']; + $n = 0; + $entryTo->beginTransaction(); + foreach ($entryFrom->select() as $entry) { + $n++; + if (!empty($idMaps['f' . $entry['id_feed']])) { + $entry['id_feed'] = $idMaps['f' . $entry['id_feed']]; + if (!$entryTo->addEntry($entry, false)) { + $error .= ' Error during SQLite copy of entries!'; + goto done; + } + } + if ($n % 100 === 1 && defined('STDERR')) { + fwrite(STDERR, "\033[0G" . $n . '/' . $total); + sleep(1); + } + } + if (defined('STDERR')) { + fwrite(STDERR, "\033[0G" . $n . '/' . $total . "\n"); + } + $entryTo->commit(); + $feedTo->updateCachedValues(); + + $idMaps = []; + + $tagTo->beginTransaction(); + foreach ($tagFrom->select() as $tag) { + $tagId = $tagTo->addTag($tag); + if ($tagId == false) { + $error .= ' Error during SQLite copy of tags!'; + goto done; + } + $idMaps['t' . $tag['id']] = $tagId; + } + foreach ($tagFrom->selectEntryTag() as $entryTag) { + if (!empty($idMaps['t' . $entryTag['id_tag']])) { + $entryTag['id_tag'] = $idMaps['t' . $entryTag['id_tag']]; + if (!$tagTo->tagEntry($entryTag['id_tag'], $entryTag['id_entry'])) { + $error .= ' Error during SQLite copy of entry-tags!'; + goto done; + } + } + } + $tagTo->commit(); + + done: + if ($error != '') { + if (defined('STDERR')) { + fwrite(STDERR, $error . "\n"); + } + Minz_Log::error($error); + return false; + } else { + return true; + } + } } diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php index b47cd55ad..4ea185286 100644 --- a/app/Models/EntryDAO.php +++ b/app/Models/EntryDAO.php @@ -152,9 +152,9 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { private $addEntryPrepared = null; - public function addEntry($valuesTmp) { + public function addEntry($valuesTmp, $useTmpTable = true) { if ($this->addEntryPrepared == null) { - $sql = 'INSERT INTO `' . $this->prefix . 'entrytmp` (id, guid, title, author, ' + $sql = 'INSERT INTO `' . $this->prefix . ($useTmpTable ? 'entrytmp' : 'entry') . '` (id, guid, title, author, ' . ($this->isCompressed() ? 'content_bin' : 'content') . ', link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags) ' . 'VALUES(:id, :guid, :title, :author, ' @@ -637,6 +637,18 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { } } + public function select() { + $sql = 'SELECT id, guid, title, author, ' + . ($this->isCompressed() ? 'UNCOMPRESS(content_bin) AS content' : 'content') + . ', link, date, `lastSeen`, ' . $this->sqlHexEncode('hash') . ' AS hash, is_read, is_favorite, id_feed, tags ' + . 'FROM `' . $this->prefix . 'entry`'; + $stm = $this->bd->prepare($sql); + $stm->execute(); + while ($row = $stm->fetch(PDO::FETCH_ASSOC)) { + yield $row; //PHP 5.5+ + } + } + public function searchByGuid($id_feed, $guid) { // un guid est unique pour un flux donné $sql = 'SELECT id, guid, title, author, ' diff --git a/app/Models/FeedDAO.php b/app/Models/FeedDAO.php index c9c9f6301..5a613c488 100644 --- a/app/Models/FeedDAO.php +++ b/app/Models/FeedDAO.php @@ -39,6 +39,7 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable { description, `lastUpdate`, priority, + `pathEntries`, `httpAuth`, error, keep_history, @@ -46,11 +47,14 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable { attributes ) VALUES - (?, ?, ?, ?, ?, ?, 10, ?, 0, ?, ?, ?)'; + (?, ?, ?, ?, ?, ?, 10, ?, ?, 0, ?, ?, ?)'; $stm = $this->bd->prepare($sql); $valuesTmp['url'] = safe_ascii($valuesTmp['url']); $valuesTmp['website'] = safe_ascii($valuesTmp['website']); + if (!isset($valuesTmp['pathEntries'])) { + $valuesTmp['pathEntries'] = ''; + } $values = array( substr($valuesTmp['url'], 0, 511), @@ -59,6 +63,7 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable { substr($valuesTmp['website'], 0, 255), mb_strcut($valuesTmp['description'], 0, 1023, 'UTF-8'), $valuesTmp['lastUpdate'], + mb_strcut($valuesTmp['pathEntries'], 0, 511, 'UTF-8'), base64_encode($valuesTmp['httpAuth']), FreshRSS_Feed::KEEP_HISTORY_DEFAULT, isset($valuesTmp['ttl']) ? intval($valuesTmp['ttl']) : FreshRSS_Feed::TTL_DEFAULT, @@ -238,6 +243,17 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable { } } + public function select() { + $sql = 'SELECT id, url, category, name, website, description, `lastUpdate`, priority, ' + . '`pathEntries`, `httpAuth`, error, keep_history, ttl, attributes ' + . 'FROM `' . $this->prefix . 'feed`'; + $stm = $this->bd->prepare($sql); + $stm->execute(); + while ($row = $stm->fetch(PDO::FETCH_ASSOC)) { + yield $row; + } + } + public function searchById($id) { $sql = 'SELECT * FROM `' . $this->prefix . 'feed` WHERE id=?'; $stm = $this->bd->prepare($sql); diff --git a/app/Models/TagDAO.php b/app/Models/TagDAO.php index 297d24c96..1d6f92923 100644 --- a/app/Models/TagDAO.php +++ b/app/Models/TagDAO.php @@ -139,6 +139,24 @@ class FreshRSS_TagDAO extends Minz_ModelPdo implements FreshRSS_Searchable { } } + public function select() { + $sql = 'SELECT id, name, attributes FROM `' . $this->prefix . 'tag`'; + $stm = $this->bd->prepare($sql); + $stm->execute(); + while ($row = $stm->fetch(PDO::FETCH_ASSOC)) { + yield $row; + } + } + + public function selectEntryTag() { + $sql = 'SELECT id_tag, id_entry FROM `' . $this->prefix . 'entrytag`'; + $stm = $this->bd->prepare($sql); + $stm->execute(); + while ($row = $stm->fetch(PDO::FETCH_ASSOC)) { + yield $row; + } + } + public function searchById($id) { $sql = 'SELECT * FROM `' . $this->prefix . 'tag` WHERE id=?'; $stm = $this->bd->prepare($sql); diff --git a/app/SQL/install.sql.sqlite.php b/app/SQL/install.sql.sqlite.php index 1dd5f2647..1c7f96adc 100644 --- a/app/SQL/install.sql.sqlite.php +++ b/app/SQL/install.sql.sqlite.php @@ -91,7 +91,7 @@ $SQL_CREATE_TABLE_TAGS = array( );', 'CREATE TABLE IF NOT EXISTS `entrytag` ( `id_tag` SMALLINT, - `id_entry` SMALLINT, + `id_entry` BIGINT, PRIMARY KEY (`id_tag`,`id_entry`), FOREIGN KEY (`id_tag`) REFERENCES `tag` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, FOREIGN KEY (`id_entry`) REFERENCES `entry` (`id`) ON DELETE CASCADE ON UPDATE CASCADE diff --git a/cli/README.md b/cli/README.md index e9e336439..75c8970ec 100644 --- a/cli/README.md +++ b/cli/README.md @@ -53,20 +53,14 @@ cd /usr/share/FreshRSS ./cli/update-user.php --user username ( ... ) # Same options as create-user.php, except --no_default_feeds which is only available for create-user.php +./cli/actualize-user.php --user username +# Fetch feeds for the specified user + ./cli/delete-user.php --user username ./cli/list-users.php # Return a list of users, with the default/admin user first -./cli/actualize-user.php --user username - -./cli/import-for-user.php --user username --filename /path/to/file.ext -# The extension of the file { .json, .opml, .xml, .zip } is used to detect the type of import - -./cli/export-opml-for-user.php --user username > /path/to/file.opml.xml - -./cli/export-zip-for-user.php --user username ( --max-feed-entries 100 ) > /path/to/file.zip - ./cli/user-info.php -h --user username # -h is to use a human-readable format # --user can be a username, or '*' to loop on all users @@ -74,6 +68,19 @@ cd /usr/share/FreshRSS # 3) the date/time of last user action, 4) the size occupied, # and the number of: 5) categories, 6) feeds, 7) read articles, 8) unread articles, 9) favourites, and 10) tags +./cli/import-for-user.php --user username --filename /path/to/file.ext +# The extension of the file { .json, .opml, .xml, .zip } is used to detect the type of import + +./cli/export-sqlite-for-user.php --user username --filename /path/to/db.sqlite +# Export the user’s database to a new SQLite file + +./cli/import-sqlite-for-user.php --user username --filename /path/to/db.sqlite +# Import the user’s database from an SQLite file. The user’s database must have 0 entry before starting this command. + +./cli/export-opml-for-user.php --user username > /path/to/file.opml.xml + +./cli/export-zip-for-user.php --user username ( --max-feed-entries 100 ) > /path/to/file.zip + ./cli/db-optimize.php --user username # Optimize database (reduces the size) for a given user (perform `OPTIMIZE TABLE` in MySQL, `VACUUM` in SQLite) ``` diff --git a/cli/export-sqlite-for-user.php b/cli/export-sqlite-for-user.php new file mode 100644 index 000000000..027d13f38 --- /dev/null +++ b/cli/export-sqlite-for-user.php @@ -0,0 +1,28 @@ +#!/usr/bin/php +dbCopy($filename, FreshRSS_DatabaseDAO::SQLITE_EXPORT); + +done($ok); diff --git a/cli/import-sqlite-for-user.php b/cli/import-sqlite-for-user.php new file mode 100644 index 000000000..db2a2ae17 --- /dev/null +++ b/cli/import-sqlite-for-user.php @@ -0,0 +1,29 @@ +#!/usr/bin/php +dbCopy($filename, FreshRSS_DatabaseDAO::SQLITE_IMPORT); +invalidateHttpCache($username); + +done($ok); diff --git a/lib/Minz/ModelPdo.php b/lib/Minz/ModelPdo.php index 733982c14..6ee2168ab 100644 --- a/lib/Minz/ModelPdo.php +++ b/lib/Minz/ModelPdo.php @@ -35,10 +35,17 @@ class Minz_ModelPdo { * Créé la connexion à la base de données à l'aide des variables * HOST, BASE, USER et PASS définies dans le fichier de configuration */ - public function __construct($currentUser = null) { + public function __construct($currentUser = null, $currentPrefix = null, $currentDb = null) { if ($currentUser === null) { $currentUser = Minz_Session::param('currentUser'); } + if ($currentPrefix !== null) { + $this->prefix = $currentPrefix; + } + if ($currentDb != null) { + $this->bd = $currentDb; + return; + } if (self::$useSharedBd && self::$sharedBd != null && ($currentUser == null || $currentUser === self::$sharedCurrentUser)) { $this->bd = self::$sharedBd;