moodle/moodle
3
0
Fork 0
mirror of https://github.com/moodle/moodle.git synced 2025-08-05 08:56:36 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

MDL-36493 improve performance of mysql unsigned/lob upgrade

Browse source
This commit is contained in:
Petr Škoda 2012-11-10 13:23:47 +01:00
parent 7d2c5cd0a0
commit 4df8d1924d
2 changed files with 35 additions and 75 deletions
Download patch file Download diff file Expand all files Collapse all files

13
lib/db/upgrade.php
View file

@ -227,17 +227,10 @@ function xmldb_main_upgrade($oldversion) {
upgrade_main_savepoint(true, 2012030100.01); upgrade_main_savepoint(true, 2012030100.01);
} }
if ($oldversion < 2012030100.02) {
// migrate all numbers to signed - it should be safe to interrupt this and continue later
upgrade_mysql_fix_unsigned_columns();
// Main savepoint reached
upgrade_main_savepoint(true, 2012030100.02);
}
if ($oldversion < 2012030900.01) { if ($oldversion < 2012030900.01) {
// migrate all texts and binaries to big size - it should be safe to interrupt this and continue later // Migrate all numbers to signed & all texts and binaries to big size.
upgrade_mysql_fix_lob_columns(); // It should be safe to interrupt this and continue later.
upgrade_mysql_fix_unsigned_and_lob_columns();
// Main savepoint reached // Main savepoint reached
upgrade_main_savepoint(true, 2012030900.01); upgrade_main_savepoint(true, 2012030900.01);

97
lib/db/upgradelib.php
View file

@ -77,14 +77,15 @@ function upgrade_mysql_get_supported_tables() {
} }
/** /**
* Remove all signed numbers from current database - mysql only. * Remove all signed numbers from current database and change
* text fields to long texts - mysql only.
*/ */
function upgrade_mysql_fix_unsigned_columns() { function upgrade_mysql_fix_unsigned_and_lob_columns() {
// we are not using standard API for changes of column // We are not using standard API for changes of column
// because everything 'signed'-related will be removed soon // because everything 'signed'-related will be removed soon.
// if anybody already has numbers higher than signed limit the execution stops // If anybody already has numbers higher than signed limit the execution stops
// and tables must be fixed manually before continuing upgrade // and tables must be fixed manually before continuing upgrade.
global $DB; global $DB;
@ -92,7 +93,7 @@ function upgrade_mysql_fix_unsigned_columns() {
return; return;
} }
$pbar = new progress_bar('mysqlconvertunsigned', 500, true); $pbar = new progress_bar('mysqlconvertunsignedlobs', 500, true);
$prefix = $DB->get_prefix(); $prefix = $DB->get_prefix();
$tables = upgrade_mysql_get_supported_tables(); $tables = upgrade_mysql_get_supported_tables();
@ -101,11 +102,13 @@ function upgrade_mysql_fix_unsigned_columns() {
$i = 0; $i = 0;
foreach ($tables as $table) { foreach ($tables as $table) {
$i++; $i++;
// set appropriate timeout - 5 minutes per milion of records should be enough, min 60 minutes just in case // Set appropriate timeout - 1 minute per thousand of records should be enough, min 60 minutes just in case.
$count = $DB->count_records($table, array()); $count = $DB->count_records($table, array());
$timeout = ($count/1000000)*5*60; $timeout = ($count/1000)*60;
$timeout = ($timeout < 60*60) ? 60*60 : (int)$timeout; $timeout = ($timeout < 60*60) ? 60*60 : (int)$timeout;
$changes = array();
$sql = "SHOW COLUMNS FROM `{{$table}}`"; $sql = "SHOW COLUMNS FROM `{{$table}}`";
$rs = $DB->get_recordset_sql($sql); $rs = $DB->get_recordset_sql($sql);
foreach ($rs as $column) { foreach ($rs as $column) {
@ -134,67 +137,31 @@ function upgrade_mysql_fix_unsigned_columns() {
$notnull = ($column->null === 'NO') ? 'NOT NULL' : 'NULL'; $notnull = ($column->null === 'NO') ? 'NOT NULL' : 'NULL';
$default = (!is_null($column->default) and $column->default !== '') ? "DEFAULT '$column->default'" : ''; $default = (!is_null($column->default) and $column->default !== '') ? "DEFAULT '$column->default'" : '';
$autoinc = (stripos($column->extra, 'auto_increment') !== false) ? 'AUTO_INCREMENT' : ''; $autoinc = (stripos($column->extra, 'auto_increment') !== false) ? 'AUTO_INCREMENT' : '';
// primary and unique not necessary here, change_database_structure does not add prefix // Primary and unique not necessary here, change_database_structure does not add prefix.
$sql = "ALTER TABLE `{$prefix}$table` MODIFY COLUMN `$column->field` $type $notnull $default $autoinc";
$DB->change_database_structure($sql); $changes[] = "MODIFY COLUMN `$column->field` $type $notnull $default $autoinc";
} else if ($column->type === 'tinytext' or $column->type === 'mediumtext' or $column->type === 'text') {
$notnull = ($column->null === 'NO') ? 'NOT NULL' : 'NULL';
$default = (!is_null($column->default) and $column->default !== '') ? "DEFAULT '$column->default'" : '';
// Primary, unique and inc are not supported for texts.
$changes[] = "MODIFY COLUMN `$column->field` LONGTEXT $notnull $default";
} else if ($column->type === 'tinyblob' or $column->type === 'mediumblob' or $column->type === 'blob') {
$notnull = ($column->null === 'NO') ? 'NOT NULL' : 'NULL';
$default = (!is_null($column->default) and $column->default !== '') ? "DEFAULT '$column->default'" : '';
// Primary, unique and inc are not supported for blobs.
$changes[] = "MODIFY COLUMN `$column->field` LONGBLOB $notnull $default";
} }
} }
$rs->close(); $rs->close();
$pbar->update($i, $tablecount, "Converted unsigned columns in MySQL database - $i/$tablecount."); if ($changes) {
} $sql = "ALTER TABLE `{$prefix}$table` ".implode(', ', $changes);
} $DB->change_database_structure($sql);
/**
* Migrate all text and binary columns to big size - mysql only.
*/
function upgrade_mysql_fix_lob_columns() {
// we are not using standard API for changes of column intentionally
global $DB;
if ($DB->get_dbfamily() !== 'mysql') {
return;
}
$pbar = new progress_bar('mysqlconvertlobs', 500, true);
$prefix = $DB->get_prefix();
$tables = upgrade_mysql_get_supported_tables();
asort($tables);
$tablecount = count($tables);
$i = 0;
foreach ($tables as $table) {
$i++;
// set appropriate timeout - 1 minute per thousand of records should be enough, min 60 minutes just in case
$count = $DB->count_records($table, array());
$timeout = ($count/1000)*60;
$timeout = ($timeout < 60*60) ? 60*60 : (int)$timeout;
$sql = "SHOW COLUMNS FROM `{{$table}}`";
$rs = $DB->get_recordset_sql($sql);
foreach ($rs as $column) {
upgrade_set_timeout($timeout);
$column = (object)array_change_key_case((array)$column, CASE_LOWER);
if ($column->type === 'tinytext' or $column->type === 'mediumtext' or $column->type === 'text') {
$notnull = ($column->null === 'NO') ? 'NOT NULL' : 'NULL';
$default = (!is_null($column->default) and $column->default !== '') ? "DEFAULT '$column->default'" : '';
// primary, unique and inc are not supported for texts
$sql = "ALTER TABLE `{$prefix}$table` MODIFY COLUMN `$column->field` LONGTEXT $notnull $default";
$DB->change_database_structure($sql);
}
if ($column->type === 'tinyblob' or $column->type === 'mediumblob' or $column->type === 'blob') {
$notnull = ($column->null === 'NO') ? 'NOT NULL' : 'NULL';
$default = (!is_null($column->default) and $column->default !== '') ? "DEFAULT '$column->default'" : '';
// primary, unique and inc are not supported for blobs
$sql = "ALTER TABLE `{$prefix}$table` MODIFY COLUMN `$column->field` LONGBLOB $notnull $default";
$DB->change_database_structure($sql);
}
} }
$rs->close();
$pbar->update($i, $tablecount, "Converted LOB columns in MySQL database - $i/$tablecount."); $pbar->update($i, $tablecount, "Converted unsigned/lob columns in MySQL database - $i/$tablecount.");
} }
} }