diff --git a/.gitignore b/.gitignore index 6f09681f1..8ca0b308b 100644 --- a/.gitignore +++ b/.gitignore @@ -68,10 +68,16 @@ test/test_env image/luts.bin image/assets/fonts/* +scripts/lua/migration/test/databases/* +scripts/lua/migration/test/migrations/* scripts/lua/test/assets* -scripts/lua/test/update/user/temp/* +scripts/lua/test/device/user/temp/recovery_status.json +scripts/lua/test/update/user/* scripts/lua/test/update/target/* scripts/lua/test/update/system/* -scripts/lua/test/update_udm/user/temp/* +scripts/lua/test/update_udm/user/* scripts/lua/test/update_udm/target/* scripts/lua/test/update_udm/system/* +scripts/lua/test/restore/user/* +scripts/lua/test/restore/target/* +scripts/lua/test/restore/system/* diff --git a/scripts/lua/entry.lua b/scripts/lua/entry.lua index 5f6b5a015..063dca643 100644 --- a/scripts/lua/entry.lua +++ b/scripts/lua/entry.lua @@ -37,7 +37,7 @@ local function generate_report_file(boot_reason_str, success, message) local body = string.format( "{\"version\": \"%s\",\"branch\": \"%s\",\"revision\": \"%s\",\"operation\": \"%s\",\"successful\": %s,\"message\": \"%s\"}", rec.version(), rec.branch(), rec.revision(), boot_reason_str, tostring(success), message) - local fd = io.open(file_path, 'w') + local fd = assert(io.open(file_path, 'w')) fd:write(body) fd:close() end diff --git a/scripts/lua/migration/migration.lua b/scripts/lua/migration/migration.lua index 0dbe02f23..3652061ad 100644 --- a/scripts/lua/migration/migration.lua +++ b/scripts/lua/migration/migration.lua @@ -37,10 +37,10 @@ local function build_database_path(path, db_name) end local function db_exec(file, script, version) - local db = assert(sqlite3.open(file), string.format("file: %s", file)) - assert(db:exec(script) == sqlite3.OK, string.format("script:\n%s\n", script)) + local db = assert(sqlite3.open(file)) + assert(db:exec(script) == sqlite3.OK, string.format("Script execution failed:\n%s\n", script)) assert(db:exec(string.format("PRAGMA user_version=%u;", version)) == sqlite3.OK, - string.format("version: %d", version)) + string.format("Setting database version: %d failed", version)) db:close() end @@ -90,26 +90,33 @@ local function db_migrate_down(db_path, scripts, target_version) end local function print_db_set(db_set) - print("database set:") + print("Database set(name,version):") for name, version in pairs(db_set) do - print(string.format("'%s':%d",name,version)) + print(string.format(" '%s':%d", name, version)) end end local function validate_inputs(migration_dir, db_dir) - assert(helpers.exists(migration_dir), "Migrations directory does not exist") + assert(helpers.exists(migration_dir), "Migration directory does not exist") assert(helpers.exists(db_dir), "Databases directory does not exist") return true end local function migrate(db_path, scripts_up, scripts_down, target_version) local db_version = read_db_version(db_path) + if db_version > target_version then + print(string.format("Performing migration-down of '%s' from version %d to %d", db_path, db_version, + target_version)) return migration.down(db_path, scripts_down, target_version) end if db_version < target_version then + print( + string.format("Performing migration-up of '%s' from version %d to %d", db_path, db_version, target_version)) return migration.up(db_path, scripts_up, target_version) end + + print(string.format("Migration not needed, '%s' is already the newest version", db_path)) return migration.retcode.OK end @@ -148,8 +155,8 @@ end -- @param db_set array of {<"database_name"> = } entries -- @return @{retcode} function migration.migrate(db_dir, scripts_dir, db_set) - print(string.format("migrations scripts directory: '%s'", scripts_dir)) - print(string.format("databases directory: '%s'", db_dir)) + print(string.format("Migration scripts directory: '%s'", scripts_dir)) + print(string.format("Databases directory: '%s'", db_dir)) print_db_set(db_set) for name, version in pairs(db_set) do diff --git a/scripts/lua/migration/test/test.lua b/scripts/lua/migration/test/test.lua index 02d54a86f..dd4fa4624 100644 --- a/scripts/lua/migration/test/test.lua +++ b/scripts/lua/migration/test/test.lua @@ -1,6 +1,5 @@ ---Copyright (c) 2017-2022, Mudita Sp. z.o.o. All rights reserved. ---For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md - +-- Copyright (c) 2017-2022, Mudita Sp. z.o.o. All rights reserved. +-- For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md package.path = package.path .. ";../?.lua;../../share/?.lua;../../share/?/?.lua" local lu = require("luaunit") local sqlite = require("lsqlite3complete") @@ -41,35 +40,35 @@ function test_migration_up_success() scripts[5] = "ALTER TABLE test_1 ADD new_column5 TEXT;" --- Migrate from version 0 to 1 - spawn_db("test_1.sql", 0, test_db_schema) - lu.assertEquals(migration.up("test_1.sql", scripts, 1), migration.retcode.OK) - lu.assertEquals(migration.get_db_version("test_1.sql"), 1) + spawn_db("databases/test_1.sql", 0, test_db_schema) + lu.assertEquals(migration.up("databases/test_1.sql", scripts, 1), migration.retcode.OK) + lu.assertEquals(migration.get_db_version("databases/test_1.sql"), 1) -- Migrate from version 1 to 2 - lu.assertEquals(migration.up("test_1.sql", scripts, 2), migration.retcode.OK) - lu.assertEquals(migration.get_db_version("test_1.sql"), 2) + lu.assertEquals(migration.up("databases/test_1.sql", scripts, 2), migration.retcode.OK) + lu.assertEquals(migration.get_db_version("databases/test_1.sql"), 2) --- Migrate from version 0 to 5 - spawn_db("test_1.sql", 0, test_db_schema) - lu.assertEquals(migration.up("test_1.sql", scripts, 5), migration.retcode.OK) - lu.assertEquals(migration.get_db_version("test_1.sql"), 5) + spawn_db("databases/test_1.sql", 0, test_db_schema) + lu.assertEquals(migration.up("databases/test_1.sql", scripts, 5), migration.retcode.OK) + lu.assertEquals(migration.get_db_version("databases/test_1.sql"), 5) end function test_migration_up_db_errors() scripts = {} --- Trigger DB error by trying to operate on empty database - spawn_db("test_1.sql", 1, "") + spawn_db("databases/test_1.sql", 1, "") scripts[1] = "ALTER TABLE test_1 ADD new_column TEXT;" - lu.assertError(migration.up, "test_1.sql", scripts, 2) + lu.assertError(migration.up, "databases/test_1.sql", scripts, 2) --- Target version set to the same value as the current DB version - spawn_db("test_1.sql", 1, test_db_schema) - lu.assertEquals(migration.up("test_1.sql", scripts, 1), migration.retcode.ALREADY_UP_TO_DATE) + spawn_db("databases/test_1.sql", 1, test_db_schema) + lu.assertEquals(migration.up("databases/test_1.sql", scripts, 1), migration.retcode.ALREADY_UP_TO_DATE) --- Target version set to the lower number than the current DB version - spawn_db("test_1.sql", 2, test_db_schema) - lu.assertEquals(migration.up("test_1.sql", scripts, 1), migration.retcode.WRONG_TARGET_VERSION) + spawn_db("databases/test_1.sql", 2, test_db_schema) + lu.assertEquals(migration.up("databases/test_1.sql", scripts, 1), migration.retcode.WRONG_TARGET_VERSION) end function test_migration_down_success() @@ -81,35 +80,35 @@ function test_migration_down_success() scripts[5] = "ALTER TABLE test_1 ADD new_column5 TEXT;" --- Migrate from version 2 to 1 - spawn_db("test_1.sql", 2, test_db_schema) - lu.assertEquals(migration.down("test_1.sql", scripts, 1), migration.retcode.OK) - lu.assertEquals(migration.get_db_version("test_1.sql"), 1) + spawn_db("databases/test_1.sql", 2, test_db_schema) + lu.assertEquals(migration.down("databases/test_1.sql", scripts, 1), migration.retcode.OK) + lu.assertEquals(migration.get_db_version("databases/test_1.sql"), 1) -- Migrate from version 1 to 0 - lu.assertEquals(migration.down("test_1.sql", scripts, 0), migration.retcode.OK) - lu.assertEquals(migration.get_db_version("test_1.sql"), 0) + lu.assertEquals(migration.down("databases/test_1.sql", scripts, 0), migration.retcode.OK) + lu.assertEquals(migration.get_db_version("databases/test_1.sql"), 0) --- Migrate from version 5 to 0 - spawn_db("test_1.sql", 5, test_db_schema) - lu.assertEquals(migration.down("test_1.sql", scripts, 0), migration.retcode.OK) - lu.assertEquals(migration.get_db_version("test_1.sql"), 0) + spawn_db("databases/test_1.sql", 5, test_db_schema) + lu.assertEquals(migration.down("databases/test_1.sql", scripts, 0), migration.retcode.OK) + lu.assertEquals(migration.get_db_version("databases/test_1.sql"), 0) end function test_migration_down_errors() scripts = {} --- Trigger DB error by trying to operate on empty database - spawn_db("test_1.sql", 2, "") + spawn_db("databases/test_1.sql", 2, "") scripts[1] = "ALTER TABLE test_1 ADD new_column TEXT;" - lu.assertError(migration.down, "test_1.sql", scripts, 1) + lu.assertError(migration.down, "databases/test_1.sql", scripts, 1) --- Target version set to the same value as the current DB version - spawn_db("test_1.sql", 1, test_db_schema) - lu.assertEquals(migration.down("test_1.sql", scripts, 1), migration.retcode.ALREADY_UP_TO_DATE) + spawn_db("databases/test_1.sql", 1, test_db_schema) + lu.assertEquals(migration.down("databases/test_1.sql", scripts, 1), migration.retcode.ALREADY_UP_TO_DATE) --- Target version set to the higher number than the current DB version - spawn_db("test_1.sql", 2, test_db_schema) - lu.assertEquals(migration.down("test_1.sql", scripts, 3), migration.retcode.WRONG_TARGET_VERSION) + spawn_db("databases/test_1.sql", 2, test_db_schema) + lu.assertEquals(migration.down("databases/test_1.sql", scripts, 3), migration.retcode.WRONG_TARGET_VERSION) end function test_automatic_migration() @@ -124,6 +123,13 @@ function test_automatic_migration() spawn_script("down.sql", "migrations", "test_2", 1, "delete from test_1 where _id=1;") spawn_script("down.sql", "migrations", "test_2", 2, "delete from test_1 where _id=2;") + migration.migrate("databases", "migrations", { + test_1 = 0, + test_2 = 0 + }) + lu.assertEquals(migration.get_db_version("databases/test_1.db"), 0) + lu.assertEquals(migration.get_db_version("databases/test_2.db"), 0) + migration.migrate("databases", "migrations", { test_1 = 2, test_2 = 2 diff --git a/scripts/lua/restore.lua b/scripts/lua/restore.lua index e8f005450..b3ba44b49 100644 --- a/scripts/lua/restore.lua +++ b/scripts/lua/restore.lua @@ -9,8 +9,6 @@ local migration = require('migration') local restore = {} local unpacked_backup_dir = paths.temp_dir .. "/backup" -local version_file = unpacked_backup_dir .. "/" .. consts.version_file -local legacy_version_file = unpacked_backup_dir .. "/" .. consts.legacy_version_file restore.script_name = "restore.lua" restore.img_in_progress = "assets/gui_image_restore_in_progress.bin" @@ -37,7 +35,7 @@ local function unpack_backup() ltar.unpack(paths.backup_file, unpacked_backup_dir) end -local function build_db_set(file) +local function get_db_array_from_file(file) local contents = helpers.read_whole_file(file) local root = json.decode(contents) local set = {} @@ -47,50 +45,48 @@ local function build_db_set(file) return set end -local function get_legacy_db_set() - local set = { - ["calllog"] = 0, - ["sms"] = 0, - ["events"] = 0, - ["settings_v2"] = 0, - ["notes"] = 0, - ["custom_quotes"] = 0, - ["predefined_quotes"] = 0, - ["contacts"] = 0, - ["alarms"] = 0, - ["notifications"] = 0, - ["multimedia"] = 0 - } +local function get_db_array_from_path(path) + local set = {} + for file in lfs.dir(path) do + local file_path = path .. "/" .. file + if file ~= "." and file ~= ".." then + if lfs.attributes(file_path, "mode") == "file" then + set[helpers.strip_from_extension(file)] = true; + end + end + end + return set +end + +local function build_db_set() + local system_db_set = get_db_array_from_file(paths.version_file) + local backup_db_set = get_db_array_from_path(unpacked_backup_dir) + local set = {} + for name, version in pairs(system_db_set) do + if backup_db_set[name] then + set[name] = tonumber(version) + end + end return set end local function perform_db_migration() print("Performing database migration") - local dbset = {} - if helpers.exists(version_file) then - dbset = build_db_set(version_file) - else - assert(helpers.exists(legacy_version_file)) - print("Legacy backup file, assuming legacy databases set") - dbset = get_legacy_db_set() - end - local result = migration.migrate(unpacked_backup_dir, paths.migration_scripts_dir, dbset) + local result = migration.migrate(unpacked_backup_dir, paths.migration_scripts_dir, build_db_set()) assert(result == migration.retcode.OK, string.format("Database migration process failed with %d", result)) end local function sync_databases() print("Syncing databases:") - print(string.format("Replacing old databases: '%s' with the ones from '%s'", paths.db_dir, unpacked_backup_dir)) - helpers.rm_files_from_dir(paths.db_dir) - helpers.copy_dir(unpacked_backup_dir, paths.db_dir) - - local version_file_path = paths.db_dir .. "/" .. consts.version_file - if not helpers.exists(version_file_path) then - version_file_path = paths.db_dir .. "/" .. consts.legacy_version_file + for name, _ in pairs(build_db_set()) do + local destination = paths.db_dir .. "/" .. name .. ".db" + local source = unpacked_backup_dir .. "/" .. name .. ".db" + print(string.format("Replacing '%s' with '%s'", destination, source)); + assert(os.remove(destination)) + helpers.copy_file(source, destination) end - assert(os.remove(version_file_path)) end local function remove_cache() diff --git a/scripts/lua/share/consts.lua b/scripts/lua/share/consts.lua index 8b97d02a3..2be30dcd4 100644 --- a/scripts/lua/share/consts.lua +++ b/scripts/lua/share/consts.lua @@ -1,7 +1,12 @@ local consts = {} consts.version_file = "version.json" -consts.legacy_version_file = "backup.json" -- Pre-UDM backup package had version.json file named as backup.json consts.indexer_cache_file = ".directory_is_indexed" +local match = {} + +-- Match only files with '.db' extensions and omit such files inside subdirectories +match.only_db_files = '^[^%/]*%.db$' + +consts.match = match return consts diff --git a/scripts/lua/share/helpers.lua b/scripts/lua/share/helpers.lua index 972589869..b5e83da27 100644 --- a/scripts/lua/share/helpers.lua +++ b/scripts/lua/share/helpers.lua @@ -152,7 +152,7 @@ function helpers.move_dir(from, where) if attr.mode == "directory" then assert(lfs.mkdir(build_path(where, name))) else - assert(os.rename(build_path(from, name),build_path(where, name))) + assert(os.rename(build_path(from, name), build_path(where, name))) end end end @@ -235,6 +235,14 @@ function helpers.get_file_extension(file) return file:match("^.+(%..+)$") end +--- Strips file name from its extension +-- @function strip_from_extension +-- @param file file path +-- @return file name without extension +function helpers.strip_from_extension(file) + return file:match("^(.*)%..*$") +end + --- Create directory and all required subdirectories -- @function mkdirp -- @param file file path diff --git a/scripts/lua/test/device/user/temp/recovery_status.json b/scripts/lua/test/device/user/temp/recovery_status.json deleted file mode 100644 index aaaf34713..000000000 --- a/scripts/lua/test/device/user/temp/recovery_status.json +++ /dev/null @@ -1 +0,0 @@ -{"version": "0.0.0","branch": "test","revision": "BABEF00D","operation": "restore","successful": false,"message": "../restore.lua:26: Not enough free space on user disk"} \ No newline at end of file diff --git a/scripts/lua/test/restore/restore1.tar b/scripts/lua/test/restore/restore1.tar new file mode 100644 index 000000000..c163cdf55 Binary files /dev/null and b/scripts/lua/test/restore/restore1.tar differ diff --git a/scripts/lua/test/restore/restore2.tar b/scripts/lua/test/restore/restore2.tar new file mode 100644 index 000000000..04660542f Binary files /dev/null and b/scripts/lua/test/restore/restore2.tar differ diff --git a/scripts/lua/test/test.lua b/scripts/lua/test/test.lua index 809b90b5b..555edb92d 100644 --- a/scripts/lua/test/test.lua +++ b/scripts/lua/test/test.lua @@ -133,12 +133,6 @@ describe("Factory/backup/restore scripts", function() recovery.sys.boot_reason_str.returns("restore") assert.has_no.error(invoke_entry) end) - it("invoke restore script, no free space", function() - recovery.sys.free_space.returns(10) - recovery.sys.boot_reason.returns(recovery.sys.boot_reason_codes.restore) - recovery.sys.boot_reason_str.returns("restore") - assert.has_no.error(invoke_entry) - end) end) local function remove_test_package(path) @@ -151,6 +145,43 @@ local function extract_test_package(path, where) os.execute(string.format("tar xf %s -C %s", path, where)) end +describe("Restore script", function() + recovery.sys.free_space.returns(1024 * 1024 * 1024) + recovery.sys.boot_reason.returns(recovery.sys.boot_reason_codes.restore) + recovery.sys.boot_reason_str.returns("restore") + recovery.sys.source_slot.returns("restore/system") + recovery.sys.target_slot.returns("restore/target") + recovery.sys.user.returns("restore/user") + + package.loaded['paths'] = false + package.loaded['restore'] = false + + it("the same set of databases", function() + -- Prepare test directory and its data + remove_test_package("restore/system") + remove_test_package("restore/user") + extract_test_package("restore/restore1.tar", "restore") + assert.has_no.error(require('restore').execute) + end) + + it("legacy backup", function() + -- Prepare test directory and its data + remove_test_package("restore/system") + remove_test_package("restore/user") + extract_test_package("restore/restore2.tar", "restore") + assert.has_no.error(require('restore').execute) + end) + + it("not enough disk space", function() + recovery.sys.free_space.returns(10) + -- Prepare test directory and its data + remove_test_package("restore/system") + remove_test_package("restore/user") + extract_test_package("restore/restore1.tar", "restore") + assert.has.error(require('restore').execute) + end) +end) + describe("Update script", function() recovery.sys.boot_reason.returns(recovery.sys.boot_reason_codes.update) recovery.sys.boot_reason_str.returns("update") diff --git a/tools/init_databases.py b/tools/init_databases.py index f6e66a492..3f1693560 100755 --- a/tools/init_databases.py +++ b/tools/init_databases.py @@ -10,6 +10,7 @@ import logging import sys import json import shutil +from pathlib import Path log = logging.getLogger(__name__) logging.basicConfig(format='%(asctime)s [%(levelname)s]: %(message)s', level=logging.INFO) @@ -25,6 +26,7 @@ def migrate_database_up(database: str, migration_path: os.path, dst_directory: o db_name_full = f"{database}.db" dst_db_path = os.path.join(dst_directory, db_name_full) + Path(dst_db_path).unlink(missing_ok=True) ret = 0 try: