/* * This file is part of the ZoneMinder Project. See AUTHORS file for Copyright information * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the * Free Software Foundation; either version 2 of the License, or (at your * option) any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along * with this program. If not, see . */ // Drives the real lock-contention retry path against a live server, which is // the only way to reach it: the errors come from InnoDB deciding a deadlock or // timing out a lock wait, and nothing in-process can fake that. // // Tagged [notCI] because it needs a reachable database and credentials in // /etc/zm/zm.conf. It creates and drops one scratch table of its own and never // touches a ZoneMinder table. #include "zm_catch2.h" #include "zm_config.h" #include "zm_db.h" #include #include #include namespace { constexpr const char *kTable = "ZM_ContentionProbe"; // A second session, so there is something to contend with. Raw libmysqlclient // rather than the zmDb* helpers, which share one process-wide connection. MYSQL *OpenAntagonist() { MYSQL *conn = mysql_init(nullptr); REQUIRE(conn != nullptr); MYSQL *ok = mysql_real_connect(conn, staticConfig.DB_HOST.c_str(), staticConfig.DB_USER.c_str(), staticConfig.DB_PASS.c_str(), staticConfig.DB_NAME.c_str(), 0, nullptr, 0); REQUIRE(ok != nullptr); return conn; } void Exec(MYSQL *conn, const std::string &sql) { REQUIRE(mysql_query(conn, sql.c_str()) == 0); } struct Scratch { Scratch() { zmLoadStaticConfig(); REQUIRE(zmDbConnect()); zmDbDo(std::string("DROP TABLE IF EXISTS `") + kTable + "`"); REQUIRE(zmDbDo(std::string("CREATE TABLE `") + kTable + "` (`Id` INT PRIMARY KEY, `V` INT NOT NULL) ENGINE=InnoDB") == 1); std::string rows; for (int id = 1; id <= 200; id++) { if (!rows.empty()) rows += ","; rows += "(" + std::to_string(id) + ",0)"; } REQUIRE(zmDbDo(std::string("INSERT INTO `") + kTable + "` (Id,V) VALUES " + rows) == 1); // Keep the waits short; the default is 50 seconds per attempt. REQUIRE(zmDbDo("SET SESSION innodb_lock_wait_timeout = 1") == 1); } ~Scratch() { zmDbDo(std::string("DROP TABLE IF EXISTS `") + kTable + "`"); } }; int ValueOf(int id) { zmDbRow row; MYSQL_RES *res = row.fetch(std::string("SELECT V FROM `") + kTable + "` WHERE Id=" + std::to_string(id)); REQUIRE(res != nullptr); return atoi(row[0]); } } // namespace TEST_CASE("zmDbDoUpdate rides out a lock held by another session", "[notCI][dblive]") { Scratch scratch; MYSQL *other = OpenAntagonist(); // Hold row 1 locked for long enough to blow the 1s lock wait more than once, // so the update only lands if it is actually being retried. Exec(other, "BEGIN"); Exec(other, std::string("UPDATE `") + kTable + "` SET V=99 WHERE Id=1"); std::thread releaser([other] { std::this_thread::sleep_for(std::chrono::milliseconds(2500)); mysql_query(other, "ROLLBACK"); }); auto started = std::chrono::steady_clock::now(); int affected = zmDbDoUpdate(std::string("UPDATE `") + kTable + "` SET V=7 WHERE Id=1"); auto elapsed = std::chrono::steady_clock::now() - started; releaser.join(); INFO("elapsed ms: " << std::chrono::duration_cast(elapsed).count()); REQUIRE(affected == 1); REQUIRE(ValueOf(1) == 7); // It cannot have succeeded on the first try; the lock was held past the // first timeout. REQUIRE(elapsed > std::chrono::milliseconds(1000)); mysql_close(other); } TEST_CASE("zmDbDoUpdate gives up on a lock that is never released", "[notCI][dblive]") { Scratch scratch; MYSQL *other = OpenAntagonist(); Exec(other, "BEGIN"); Exec(other, std::string("UPDATE `") + kTable + "` SET V=99 WHERE Id=2"); // Before the fix this looped forever. It must now return, and within the // budget: 5 lock waits of 1s plus ~3.1s of backoff. auto started = std::chrono::steady_clock::now(); int rc = zmDbDoUpdate(std::string("UPDATE `") + kTable + "` SET V=7 WHERE Id=2"); auto elapsed = std::chrono::steady_clock::now() - started; INFO("elapsed ms: " << std::chrono::duration_cast(elapsed).count()); REQUIRE(rc < 0); // reported failure REQUIRE(elapsed > std::chrono::seconds(3)); // did retry REQUIRE(elapsed < std::chrono::seconds(20)); // did stop mysql_query(other, "ROLLBACK"); mysql_close(other); } TEST_CASE("zmDbDo retries a real InnoDB deadlock", "[notCI][dblive]") { Scratch scratch; MYSQL *other = OpenAntagonist(); // Both sides reach for a row the other holds, so InnoDB has to break the // cycle by rolling one of them back with ER_LOCK_DEADLOCK. // // It picks the cheaper transaction as the victim, weighted by rows changed, // so the other session deliberately dirties a hundred rows first. Without // that the victim is whichever side InnoDB happens to prefer, and the test // passes without ever reaching the retry. Exec(other, "BEGIN"); Exec(other, std::string("UPDATE `") + kTable + "` SET V=V+1 WHERE Id BETWEEN 101 AND 200"); REQUIRE(zmDbDo("BEGIN") == 1); REQUIRE(zmDbDo(std::string("UPDATE `") + kTable + "` SET V=V+1 WHERE Id=1") == 1); std::thread crosser([other] { // Blocks on row 1, held by the ZoneMinder connection, completing the cycle. mysql_query(other, "UPDATE `ZM_ContentionProbe` SET V=V+1 WHERE Id=1"); mysql_query(other, "COMMIT"); }); std::this_thread::sleep_for(std::chrono::milliseconds(300)); // Reaching for a row the heavy transaction holds. This is the statement that // gets rolled back, and it only reports success if the retry re-ran it. int rc = zmDbDo(std::string("UPDATE `") + kTable + "` SET V=V+1 WHERE Id=150"); zmDbDo("COMMIT"); crosser.join(); REQUIRE(rc == 1); REQUIRE(ValueOf(150) == 2); // the other session's +1, then ours on retry mysql_close(other); }