From 97159d42c757d4489cb69fb3750b260aa3a95512 Mon Sep 17 00:00:00 2001 From: objecttothis Date: Thu, 22 Apr 2021 13:34:22 +0400 Subject: [PATCH] Database Optimizations - Add indexes to tables to improve query times. - Delete orphaned attribute values. - Resolve duplicate attribute values. - Deleted whitespace after migration which was causing Severity: Warning --> Cannot modify header information - headers already sent by --- ...20210103000000_modify_session_datatype.php | 3 +- .../20210422000000_database_optimizations.php | 123 +++++++++++++++++ .../20210422000001_remove_duplicate_links.php | 57 ++++++++ .../3.4.0_database_optimizations.sql | 125 ++++++++++++++++++ application/models/Attribute.php | 21 +++ 5 files changed, 327 insertions(+), 2 deletions(-) create mode 100644 application/migrations/20210422000000_database_optimizations.php create mode 100644 application/migrations/20210422000001_remove_duplicate_links.php create mode 100644 application/migrations/sqlscripts/3.4.0_database_optimizations.sql diff --git a/application/migrations/20210103000000_modify_session_datatype.php b/application/migrations/20210103000000_modify_session_datatype.php index 39866a3f3..b028ea04d 100644 --- a/application/migrations/20210103000000_modify_session_datatype.php +++ b/application/migrations/20210103000000_modify_session_datatype.php @@ -20,5 +20,4 @@ class Migration_modify_session_datatype extends CI_Migration { } } -?> - +?> \ No newline at end of file diff --git a/application/migrations/20210422000000_database_optimizations.php b/application/migrations/20210422000000_database_optimizations.php new file mode 100644 index 000000000..4ec16544f --- /dev/null +++ b/application/migrations/20210422000000_database_optimizations.php @@ -0,0 +1,123 @@ +Attribute->delete_orphaned_values(); + + $this->migrate_duplicate_attribute_values(DECIMAL); + $this->migrate_duplicate_attribute_values(DATE); + + //Select all attributes that have data in more than one column + $this->db->select('attribute_id, attribute_value, attribute_decimal, attribute_date'); + $this->db->group_start(); + $this->db->where('attribute_value IS NOT NULL'); + $this->db->where('attribute_date IS NOT NULL'); + $this->db->group_end(); + $this->db->or_group_start(); + $this->db->where('attribute_value IS NOT NULL'); + $this->db->where('attribute_decimal IS NOT NULL'); + $this->db->group_end(); + $attribute_values = $this->db->get('attribute_values'); + + $this->db->trans_start(); + + //Clean up Attribute values table where there is an attribute value and an attribute_date/attribute_decimal + foreach($attribute_values->result_array() as $attribute_value) + { + $attribute_links = $this->db->query('SELECT links.definition_id, links.item_id, links.attribute_id, defs.definition_type FROM ospos_attribute_links links JOIN ospos_attribute_definitions defs ON defs.definition_id = links.definition_id where attribute_id = '. $attribute_value['attribute_id']); + + $this->db->where('attribute_id', $attribute_value['attribute_id']); + $this->db->delete('attribute_values'); + + foreach($attribute_links->result_array() as $attribute_link) + { + $this->db->where('attribute_id',$attribute_link['attribute_id']); + $this->db->where('item_id',$attribute_link['item_id']); + $this->db->delete('attribute_links'); + + switch($attribute_link['definition_type']) + { + case DECIMAL: + $value = $attribute_value['attribute_decimal']; + break; + case DATE: + $attribute_date = DateTime::createFromFormat('Y-m-d', $attribute_value['attribute_date']); + $value = $attribute_date->format($CI->Appconfig->get('dateformat')); + break; + default: + $value = $attribute_value['attribute_value']; + break; + } + + $CI->Attribute->save_value($value, $attribute_link['definition_id'], $attribute_link['item_id'], FALSE, $attribute_link['definition_type']); + } + } + $this->db->trans_complete(); + + execute_script(APPPATH . 'migrations/sqlscripts/3.4.0_database_optimizations.sql'); + error_log('Migrating database_optimizations completed'); + } + /** + * Given the type of attribute, deletes any duplicates it finds in the attribute_values table and reassigns those + */ + private function migrate_duplicate_attribute_values($attribute_type) + { + //Remove duplicate attribute values needed to make attribute_decimals and attribute_dates unique + $this->db->trans_start(); + + $column = 'attribute_' . strtolower($attribute_type); + + $this->db->select("$column, attribute_id"); + $this->db->group_by($column); + $this->db->having("COUNT($column) > 1"); + $duplicated_values = $this->db->get('attribute_values'); + + foreach($duplicated_values->result_array() as $duplicated_value) + { + $this->db->select('attribute_id'); + $this->db->where($column, $duplicated_value[$column]); + $attribute_ids_to_fix = $this->db->get('attribute_values'); + + $this->reassign_duplicate_attribute_values($attribute_ids_to_fix, $duplicated_value); + } + + $this->db->trans_complete(); + } + + /** + * Updates the attribute_id in all attribute_link rows with duplicated attribute_ids then deletes unneeded rows from attribute_values + * @param attribute_ids_to_fix + * @param decimal + */ + private function reassign_duplicate_attribute_values($attribute_ids_to_fix, $attribute_value) + { + foreach($attribute_ids_to_fix->result_array() as $attribute_id) + { + //Update attribute_link with the attribute_id we are keeping + $this->db->where('attribute_id', $attribute_id['attribute_id']); + $this->db->update('attribute_links', array('attribute_id' => $attribute_value['attribute_id'])); + + //Delete the row from attribute_values if it isn't our keeper + if($attribute_id['attribute_id'] !== $attribute_value['attribute_id']) + { + $this->db->where('attribute_id', $attribute_id['attribute_id']); + $this->db->delete($this->db->dbprefix('attribute_values')); + } + } + } + + public function down() + { + } +} +?> \ No newline at end of file diff --git a/application/migrations/20210422000001_remove_duplicate_links.php b/application/migrations/20210422000001_remove_duplicate_links.php new file mode 100644 index 000000000..0dc7f6bec --- /dev/null +++ b/application/migrations/20210422000001_remove_duplicate_links.php @@ -0,0 +1,57 @@ +migrate_duplicate_attribute_links(); + + error_log('Migrating remove_duplicate_links completed'); + } + /** + * Given the type of attribute, deletes any duplicates it finds in the attribute_values table and reassigns those + */ + private function migrate_duplicate_attribute_links() + { + $CI =& get_instance(); + //Remove duplicate attribute links + $this->db->trans_start(); + + $this->db->where('sale_id'); + $this->db->where('receiving_id'); + $this->db->group_by('item_id'); + $this->db->group_by('definition_id'); + $this->db->group_by('attribute_id'); + $this->db->having('COUNT(item_id) > 1'); + $this->db->having('COUNT(definition_id) > 1'); + $this->db->having('COUNT(attribute_id) > 1'); + $duplicated_links = $this->db->get('attribute_links'); + + foreach($duplicated_links->result_array() as $duplicated_link) + { + $this->db->where('sale_id'); + $this->db->where('receiving_id'); + $this->db->where('item_id', $duplicated_link['item_id']); + $this->db->where('definition_id', $duplicated_link['definition_id']); + $this->db->where('item_id', $duplicated_link['item_id']); + $this->db->delete('attribute_links'); + + $CI->Attribute->save_link($duplicated_link['item_id'], $duplicated_link['definition_id'], $duplicated_link['attribute_id']); + } + + $this->db->trans_complete(); + } + + public function down() + { + } +} +?> \ No newline at end of file diff --git a/application/migrations/sqlscripts/3.4.0_database_optimizations.sql b/application/migrations/sqlscripts/3.4.0_database_optimizations.sql new file mode 100644 index 000000000..0b3f4e2ee --- /dev/null +++ b/application/migrations/sqlscripts/3.4.0_database_optimizations.sql @@ -0,0 +1,125 @@ +#ospos_attribute_values table +ALTER TABLE `ospos_attribute_values` ADD UNIQUE(`attribute_date`); +ALTER TABLE `ospos_attribute_values` ADD UNIQUE(`attribute_decimal`); + +#opsos_attribute_definitions table +ALTER TABLE `ospos_attribute_definitions` MODIFY `definition_flags` tinyint(1) NOT NULL; +ALTER TABLE `ospos_attribute_definitions` ADD INDEX(`definition_name`); +ALTER TABLE `ospos_attribute_definitions` ADD INDEX(`definition_type`); +ALTER TABLE `ospos_attribute_definitions` ADD INDEX(`deleted`); + +#opsos_attribute_links table +ALTER TABLE `ospos_attribute_links` ADD UNIQUE INDEX `attribute_links_uq2` (`item_id`, `receiving_id`, `sale_id`, `definition_id`, `attribute_id`); + +#ospos_cash_up table +ALTER TABLE `ospos_cash_up` MODIFY `deleted` tinyint(1) DEFAULT 0 NOT NULL; + +#ospos_customers table +DROP INDEX `person_id` ON `ospos_customers`; +ALTER TABLE `ospos_customers` MODIFY `taxable` tinyint(1) DEFAULT 1 NOT NULL; +ALTER TABLE `ospos_customers` MODIFY `deleted` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_customers` MODIFY `discount_type` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_customers` ADD PRIMARY KEY(`person_id`); +ALTER TABLE `ospos_customers` ADD INDEX(`deleted`); +ALTER TABLE `ospos_customers` ADD INDEX(`company_name`); + +#ospos_customers_packages table +ALTER TABLE `ospos_customers_packages` MODIFY `deleted` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_customers_packages` ADD INDEX(`deleted`); + +#ospos_dinner_tables table +ALTER TABLE `ospos_dinner_tables` MODIFY `deleted` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_dinner_tables` ADD INDEX(`status`); +ALTER TABLE `ospos_dinner_tables` ADD INDEX(`deleted`); + +#ospos_employees table +DROP INDEX `person_id` ON `ospos_employees`; +ALTER TABLE `ospos_employees` MODIFY `deleted` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_employees` MODIFY `hash_version` tinyint(1) DEFAULT 2 NOT NULL; +ALTER TABLE `ospos_employees` ADD PRIMARY KEY(`person_id`); + +#ospos_expenses table +ALTER TABLE `ospos_expenses` MODIFY `deleted` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_expenses` ADD INDEX(`deleted`); +ALTER TABLE `ospos_expenses` ADD INDEX(`payment_type`); +ALTER TABLE `ospos_expenses` ADD INDEX(`amount`); + +#ospos_expenses_categories table +ALTER TABLE `ospos_expense_categories` MODIFY `deleted` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_expense_categories` ADD INDEX(`category_description`); +ALTER TABLE `ospos_expense_categories` ADD INDEX(`deleted`); + +#ospos_giftcards table +ALTER TABLE `ospos_giftcards` MODIFY `deleted` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_giftcards` ADD INDEX(`deleted`); + +#ospos_items table +ALTER TABLE `ospos_items` MODIFY `deleted` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_items` MODIFY `stock_type` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_items` MODIFY `item_type` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_items` ADD INDEX(`deleted`); +ALTER TABLE `ospos_items` ADD INDEX(`item_type`); +ALTER TABLE `ospos_items` ADD UNIQUE INDEX `items_uq1` (`supplier_id`, `item_id`, `deleted`, `item_type`); + +#ospos_item_kits table +ALTER TABLE `ospos_item_kits` MODIFY `kit_discount_type` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_item_kits` MODIFY `price_option` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_item_kits` MODIFY `print_option` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_item_kits` ADD INDEX(`name`,`description`); + +#ospos_people table +ALTER TABLE `ospos_people` ADD INDEX(`first_name`, `last_name`, `email`, `phone_number`); + +#ospos_receivings_items +ALTER TABLE `ospos_receivings_items` MODIFY `discount_type` tinyint(1) DEFAULT 0 NOT NULL; + +#ospos_sales +ALTER TABLE `ospos_sales` MODIFY `sale_status` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_sales` MODIFY `sale_type` tinyint(1) DEFAULT 0 NOT NULL; + +#ospos_sales_items +ALTER TABLE `ospos_sales_items` MODIFY `discount_type` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_sales_items` MODIFY `print_option` tinyint(1) DEFAULT 0 NOT NULL; + +#ospos_sales_items_taxes +ALTER TABLE `ospos_sales_items_taxes` MODIFY `tax_type` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_sales_items_taxes` MODIFY `rounding_code` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_sales_items_taxes` MODIFY `cascade_sequence` tinyint(1) DEFAULT 0 NOT NULL; + +#ospos_sales_taxes +ALTER TABLE `ospos_sales_taxes` MODIFY `print_sequence` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_sales_taxes` MODIFY `rounding_code` tinyint(1) DEFAULT 0 NOT NULL; + +#ospos_sessions table +ALTER TABLE `ospos_sessions` ADD INDEX(`id`); +ALTER TABLE `ospos_sessions` ADD INDEX(`ip_address`); + +#ospos_stock_locations table +ALTER TABLE `ospos_stock_locations` MODIFY `deleted` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_stock_locations` ADD INDEX(`deleted`); + +#ospos_suppliers table +DROP INDEX `person_id` ON `ospos_suppliers`; +ALTER TABLE `ospos_suppliers` MODIFY `deleted` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_suppliers` MODIFY `category` tinyint(1) NOT NULL; +ALTER TABLE `ospos_suppliers` ADD PRIMARY KEY(`person_id`); +ALTER TABLE `ospos_suppliers` ADD INDEX(`category`); +ALTER TABLE `ospos_suppliers` ADD INDEX(`deleted`); + +#ospos_tax_categories table +ALTER TABLE `ospos_tax_categories` MODIFY `deleted` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_tax_categories` MODIFY `tax_group_sequence` tinyint(1) NOT NULL; +ALTER TABLE `ospos_tax_categories` ADD INDEX(`deleted`); + +#ospos_tax_codes table +ALTER TABLE `ospos_tax_codes` MODIFY `deleted` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_tax_codes` ADD INDEX(`deleted`); + +#ospos_tax_jurisdictions table +ALTER TABLE `ospos_tax_jurisdictions` MODIFY `deleted` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_tax_jurisdictions` MODIFY `tax_group_sequence` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_tax_jurisdictions` MODIFY `cascade_sequence` tinyint(1) DEFAULT 0 NOT NULL; +ALTER TABLE `ospos_tax_jurisdictions` ADD INDEX(`deleted`); + +#ospos_tax_rates table +ALTER TABLE `ospos_tax_rates` MODIFY `tax_rounding_code` tinyint(1) DEFAULT 0 NOT NULL; diff --git a/application/models/Attribute.php b/application/models/Attribute.php index 854de040c..62dc0e987 100644 --- a/application/models/Attribute.php +++ b/application/models/Attribute.php @@ -665,6 +665,27 @@ class Attribute extends CI_Model return $this->db->update('attribute_definitions', array('deleted' => 1)); } + /** + * Deletes any orphaned values that do not have associated links + * @param int $definition_id + * @return boolean TRUE is returned if the delete was successful or FALSE if there were any failures + */ + public function delete_orphaned_values() + { + $this->db->distinct(); + $this->db->select('attribute_id'); + $attribute_ids = $this->db->get_compiled_select('attribute_links'); + + $this->db->trans_start(); + + $this->db->where_not_in('attribute_id', $attribute_ids, FALSE); + $this->db->delete('attribute_values'); + + $this->db->trans_complete(); + + return $this->db->trans_status(); + } + /* Undeletes one attribute definition */