mirror/FreshRSS
0
0
Fork 0
mirror of https://github.com/FreshRSS/FreshRSS.git synced 2026-07-11 07:36:33 -04:00
Code Issues Packages Projects Releases Wiki Activity

CLI to export/import any database to/from SQLite

Browse Source 
Require PHP 5.5+ https://github.com/FreshRSS/FreshRSS/pull/2495
This commit is contained in:
Alexandre Alapetite
2019-08-18 22:18:42 +02:00
parent 38a4b22f7b
commit a7bd8aec7d
10 changed files with 303 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
app/Models/CategoryDAO.php
View File

@@ -77,6 +77,15 @@ class FreshRSS_CategoryDAO extends Minz_ModelPdo implements FreshRSS_Searchable
}
}
public function select() {
$sql = 'SELECT id, name FROM `' . $this->prefix . 'category`';
$stm = $this->bd->prepare($sql);
$stm->execute();
while ($row = $stm->fetch(PDO::FETCH_ASSOC)) {
yield $row;
}
}
public function searchById($id) {
$sql = 'SELECT * FROM `' . $this->prefix . 'category` WHERE id=?';
$stm = $this->bd->prepare($sql);

163
app/Models/DatabaseDAO.php
View File

@@ -164,4 +164,167 @@ class FreshRSS_DatabaseDAO extends Minz_ModelPdo {
public function minorDbMaintenance() {
$this->ensureCaseInsensitiveGuids();
}
const SQLITE_EXPORT = 1;
const SQLITE_IMPORT = 2;
public function dbCopy($filename, $mode) {
$error = '';
$catDAO = FreshRSS_Factory::createCategoryDao();
$feedDAO = FreshRSS_Factory::createFeedDao();
$entryDAO = FreshRSS_Factory::createEntryDao();
$tagDAO = FreshRSS_Factory::createTagDao();
switch ($mode) {
case self::SQLITE_EXPORT:
if (@filesize($filename) > 0) {
$error = 'Error: SQLite export file already exists: ' . $filename;
}
break;
case self::SQLITE_IMPORT:
if (!is_readable($filename)) {
$error = 'Error: SQLite import file is not readable: ' . $filename;
} else {
$nbEntries = $entryDAO->countUnreadRead();
if ($nbEntries['all'] > 0) {
$error = 'Error: Destination database already contains some entries!';
}
}
break;
default:
$error = 'Invalid copy mode!';
break;
}
if ($error != '') {
goto done;
}
$sqlite = null;
try {
$sqlite = new MinzPDOSQLite('sqlite:' . $filename);
$sqlite->exec('PRAGMA foreign_keys = ON;');
if ($mode === self::SQLITE_EXPORT) {
require_once(APP_PATH . '/SQL/install.sql.sqlite.php');
global $SQL_CREATE_TABLES, $SQL_CREATE_TABLE_ENTRYTMP, $SQL_CREATE_TABLE_TAGS;
if (is_array($SQL_CREATE_TABLES)) {
$instructions = array_merge($SQL_CREATE_TABLES, $SQL_CREATE_TABLE_ENTRYTMP, $SQL_CREATE_TABLE_TAGS,
array('DELETE FROM entrytag', 'DELETE FROM tag', 'DELETE FROM entrytmp', 'DELETE FROM entry',
'DELETE FROM feed', 'DELETE FROM category'));
foreach ($instructions as $instruction) {
$sql = sprintf($instruction, '', _t('gen.short.default_category'));
$stm = $sqlite->prepare($sql);
$stm->execute();
}
}
}
} catch (Exception $e) {
$error .= ' Error while initialising SQLite copy: ' . $e->getMessage();
goto done;
}
Minz_ModelPdo::clean();
$categoryDAOSQLite = new FreshRSS_CategoryDAO('', '', $sqlite);
$feedDAOSQLite = new FreshRSS_FeedDAOSQLite('', '', $sqlite);
$entryDAOSQLite = new FreshRSS_EntryDAOSQLite('', '', $sqlite);
$tagDAOSQLite = new FreshRSS_TagDAOSQLite('', '', $sqlite);
switch ($mode) {
case self::SQLITE_EXPORT:
$catFrom = $catDAO; $catTo = $categoryDAOSQLite;
$feedFrom = $feedDAO; $feedTo = $feedDAOSQLite;
$entryFrom = $entryDAO; $entryTo = $entryDAOSQLite;
$tagFrom = $tagDAO; $tagTo = $tagDAOSQLite;
break;
case self::SQLITE_IMPORT:
$catFrom = $categoryDAOSQLite; $catTo = $catDAO;
$feedFrom = $feedDAOSQLite; $feedTo = $feedDAO;
$entryFrom = $entryDAOSQLite; $entryTo = $entryDAO;
$tagFrom = $tagDAOSQLite; $tagTo = $tagDAO;
break;
}
$idMaps = [];
$catTo->beginTransaction();
foreach ($catFrom->select() as $category) {
$catId = $catTo->addCategory($category);
if ($catId == false) {
$error .= ' Error during SQLite copy of categories!';
goto done;
}
$idMaps['c' . $category['id']] = $catId;
}
foreach ($feedFrom->select() as $feed) {
$feed['category'] = empty($idMaps['c' . $feed['category']]) ?
FreshRSS_CategoryDAO::DEFAULTCATEGORYID : $idMaps['c' . $feed['category']];
$feedId = $feedTo->addFeed($feed);
if ($feedId == false) {
$error .= ' Error during SQLite copy of feeds!';
goto done;
}
$idMaps['f' . $feed['id']] = $feedId;
}
$catTo->commit();
$nbEntries = $entryFrom->countUnreadRead();
$total = $nbEntries['all'];
$n = 0;
$entryTo->beginTransaction();
foreach ($entryFrom->select() as $entry) {
$n++;
if (!empty($idMaps['f' . $entry['id_feed']])) {
$entry['id_feed'] = $idMaps['f' . $entry['id_feed']];
if (!$entryTo->addEntry($entry, false)) {
$error .= ' Error during SQLite copy of entries!';
goto done;
}
}
if ($n % 100 === 1 && defined('STDERR')) {
fwrite(STDERR, "\033[0G" . $n . '/' . $total);
sleep(1);
}
}
if (defined('STDERR')) {
fwrite(STDERR, "\033[0G" . $n . '/' . $total . "\n");
}
$entryTo->commit();
$feedTo->updateCachedValues();
$idMaps = [];
$tagTo->beginTransaction();
foreach ($tagFrom->select() as $tag) {
$tagId = $tagTo->addTag($tag);
if ($tagId == false) {
$error .= ' Error during SQLite copy of tags!';
goto done;
}
$idMaps['t' . $tag['id']] = $tagId;
}
foreach ($tagFrom->selectEntryTag() as $entryTag) {
if (!empty($idMaps['t' . $entryTag['id_tag']])) {
$entryTag['id_tag'] = $idMaps['t' . $entryTag['id_tag']];
if (!$tagTo->tagEntry($entryTag['id_tag'], $entryTag['id_entry'])) {
$error .= ' Error during SQLite copy of entry-tags!';
goto done;
}
}
}
$tagTo->commit();
done:
if ($error != '') {
if (defined('STDERR')) {
fwrite(STDERR, $error . "\n");
}
Minz_Log::error($error);
return false;
} else {
return true;
}
}
}

16
app/Models/EntryDAO.php
View File

@@ -152,9 +152,9 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
private $addEntryPrepared = null;
public function addEntry($valuesTmp) {
public function addEntry($valuesTmp, $useTmpTable = true) {
if ($this->addEntryPrepared == null) {
$sql = 'INSERT INTO `' . $this->prefix . 'entrytmp` (id, guid, title, author, '
$sql = 'INSERT INTO `' . $this->prefix . ($useTmpTable ? 'entrytmp' : 'entry') . '` (id, guid, title, author, '
. ($this->isCompressed() ? 'content_bin' : 'content')
. ', link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags) '
. 'VALUES(:id, :guid, :title, :author, '
@@ -637,6 +637,18 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
}
}
public function select() {
$sql = 'SELECT id, guid, title, author, '
. ($this->isCompressed() ? 'UNCOMPRESS(content_bin) AS content' : 'content')
. ', link, date, `lastSeen`, ' . $this->sqlHexEncode('hash') . ' AS hash, is_read, is_favorite, id_feed, tags '
. 'FROM `' . $this->prefix . 'entry`';
$stm = $this->bd->prepare($sql);
$stm->execute();
while ($row = $stm->fetch(PDO::FETCH_ASSOC)) {
yield $row; //PHP 5.5+
}
}
public function searchByGuid($id_feed, $guid) {
// un guid est unique pour un flux donné
$sql = 'SELECT id, guid, title, author, '

18
app/Models/FeedDAO.php
View File

@@ -39,6 +39,7 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
description,
`lastUpdate`,
priority,
`pathEntries`,
`httpAuth`,
error,
keep_history,
@@ -46,11 +47,14 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
attributes
)
VALUES
(?, ?, ?, ?, ?, ?, 10, ?, 0, ?, ?, ?)';
(?, ?, ?, ?, ?, ?, 10, ?, ?, 0, ?, ?, ?)';
$stm = $this->bd->prepare($sql);
$valuesTmp['url'] = safe_ascii($valuesTmp['url']);
$valuesTmp['website'] = safe_ascii($valuesTmp['website']);
if (!isset($valuesTmp['pathEntries'])) {
$valuesTmp['pathEntries'] = '';
}
$values = array(
substr($valuesTmp['url'], 0, 511),
@@ -59,6 +63,7 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
substr($valuesTmp['website'], 0, 255),
mb_strcut($valuesTmp['description'], 0, 1023, 'UTF-8'),
$valuesTmp['lastUpdate'],
mb_strcut($valuesTmp['pathEntries'], 0, 511, 'UTF-8'),
base64_encode($valuesTmp['httpAuth']),
FreshRSS_Feed::KEEP_HISTORY_DEFAULT,
isset($valuesTmp['ttl']) ? intval($valuesTmp['ttl']) : FreshRSS_Feed::TTL_DEFAULT,
@@ -238,6 +243,17 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
}
}
public function select() {
$sql = 'SELECT id, url, category, name, website, description, `lastUpdate`, priority, '
. '`pathEntries`, `httpAuth`, error, keep_history, ttl, attributes '
. 'FROM `' . $this->prefix . 'feed`';
$stm = $this->bd->prepare($sql);
$stm->execute();
while ($row = $stm->fetch(PDO::FETCH_ASSOC)) {
yield $row;
}
}
public function searchById($id) {
$sql = 'SELECT * FROM `' . $this->prefix . 'feed` WHERE id=?';
$stm = $this->bd->prepare($sql);

18
app/Models/TagDAO.php
View File

@@ -139,6 +139,24 @@ class FreshRSS_TagDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
}
}
public function select() {
$sql = 'SELECT id, name, attributes FROM `' . $this->prefix . 'tag`';
$stm = $this->bd->prepare($sql);
$stm->execute();
while ($row = $stm->fetch(PDO::FETCH_ASSOC)) {
yield $row;
}
}
public function selectEntryTag() {
$sql = 'SELECT id_tag, id_entry FROM `' . $this->prefix . 'entrytag`';
$stm = $this->bd->prepare($sql);
$stm->execute();
while ($row = $stm->fetch(PDO::FETCH_ASSOC)) {
yield $row;
}
}
public function searchById($id) {
$sql = 'SELECT * FROM `' . $this->prefix . 'tag` WHERE id=?';
$stm = $this->bd->prepare($sql);

2
app/SQL/install.sql.sqlite.php
View File

@@ -91,7 +91,7 @@ $SQL_CREATE_TABLE_TAGS = array(
);',
'CREATE TABLE IF NOT EXISTS `entrytag` (
`id_tag` SMALLINT,
`id_entry` SMALLINT,
`id_entry` BIGINT,
PRIMARY KEY (`id_tag`,`id_entry`),
FOREIGN KEY (`id_tag`) REFERENCES `tag` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
FOREIGN KEY (`id_entry`) REFERENCES `entry` (`id`) ON DELETE CASCADE ON UPDATE CASCADE

25
cli/README.md
View File

@@ -53,20 +53,14 @@ cd /usr/share/FreshRSS
./cli/update-user.php --user username ( ... )
# Same options as create-user.php, except --no_default_feeds which is only available for create-user.php
./cli/actualize-user.php --user username
# Fetch feeds for the specified user
./cli/delete-user.php --user username
./cli/list-users.php
# Return a list of users, with the default/admin user first
./cli/actualize-user.php --user username
./cli/import-for-user.php --user username --filename /path/to/file.ext
# The extension of the file { .json, .opml, .xml, .zip } is used to detect the type of import
./cli/export-opml-for-user.php --user username > /path/to/file.opml.xml
./cli/export-zip-for-user.php --user username ( --max-feed-entries 100 ) > /path/to/file.zip
./cli/user-info.php -h --user username
# -h is to use a human-readable format
# --user can be a username, or '*' to loop on all users
@@ -74,6 +68,19 @@ cd /usr/share/FreshRSS
# 3) the date/time of last user action, 4) the size occupied,
# and the number of: 5) categories, 6) feeds, 7) read articles, 8) unread articles, 9) favourites, and 10) tags
./cli/import-for-user.php --user username --filename /path/to/file.ext
# The extension of the file { .json, .opml, .xml, .zip } is used to detect the type of import
./cli/export-sqlite-for-user.php --user username --filename /path/to/db.sqlite
# Export the user’s database to a new SQLite file
./cli/import-sqlite-for-user.php --user username --filename /path/to/db.sqlite
# Import the user’s database from an SQLite file. The user’s database must have 0 entry before starting this command.
./cli/export-opml-for-user.php --user username > /path/to/file.opml.xml
./cli/export-zip-for-user.php --user username ( --max-feed-entries 100 ) > /path/to/file.zip
./cli/db-optimize.php --user username
# Optimize database (reduces the size) for a given user (perform `OPTIMIZE TABLE` in MySQL, `VACUUM` in SQLite)
```

28
cli/export-sqlite-for-user.php Normal file
View File

@@ -0,0 +1,28 @@
#!/usr/bin/php
<?php
require(__DIR__ . '/_cli.php');
$params = [
'user:',
'filename:',
];
$options = getopt('', $params);
if (!validateOptions($argv, $params) || empty($options['user']) || empty($options['filename'])) {
fail('Usage: ' . basename(__FILE__) . ' --user username --filename /path/to/db.sqlite');
}
$username = cliInitUser($options['user']);
$filename = $options['filename'];
if (pathinfo($filename, PATHINFO_EXTENSION) !== 'sqlite') {
fail('Only *.sqlite files are supported!');
}
echo 'FreshRSS exporting database to SQLite for user “', $username, "”…\n";
$databaseDAO = FreshRSS_Factory::createDatabaseDAO($username);
$ok = $databaseDAO->dbCopy($filename, FreshRSS_DatabaseDAO::SQLITE_EXPORT);
done($ok);

29
cli/import-sqlite-for-user.php Normal file
View File

@@ -0,0 +1,29 @@
#!/usr/bin/php
<?php
require(__DIR__ . '/_cli.php');
$params = [
'user:',
'filename:',
];
$options = getopt('', $params);
if (!validateOptions($argv, $params) || empty($options['user']) || empty($options['filename'])) {
fail('Usage: ' . basename(__FILE__) . ' --user username --filename /path/to/db.sqlite');
}
$username = cliInitUser($options['user']);
$filename = $options['filename'];
if (pathinfo($filename, PATHINFO_EXTENSION) !== 'sqlite') {
fail('Only *.sqlite files are supported!');
}
echo 'FreshRSS importing database from SQLite for user “', $username, "”…\n";
$databaseDAO = FreshRSS_Factory::createDatabaseDAO($username);
$ok = $databaseDAO->dbCopy($filename, FreshRSS_DatabaseDAO::SQLITE_IMPORT);
invalidateHttpCache($username);
done($ok);

9
lib/Minz/ModelPdo.php
View File

@@ -35,10 +35,17 @@ class Minz_ModelPdo {
* Créé la connexion à la base de données à l'aide des variables
* HOST, BASE, USER et PASS définies dans le fichier de configuration
*/
public function __construct($currentUser = null) {
public function __construct($currentUser = null, $currentPrefix = null, $currentDb = null) {
if ($currentUser === null) {
$currentUser = Minz_Session::param('currentUser');
}
if ($currentPrefix !== null) {
$this->prefix = $currentPrefix;
}
if ($currentDb != null) {
$this->bd = $currentDb;
return;
}
if (self::$useSharedBd && self::$sharedBd != null &&
($currentUser == null || $currentUser === self::$sharedCurrentUser)) {
$this->bd = self::$sharedBd;