moodle/moodle
3
0
Fork 0
mirror of https://github.com/moodle/moodle.git synced 2025-08-05 00:46:50 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'wip_MDL-46235_m28_innodbtext' of https://github.com/skodak/moodle

Browse source
This commit is contained in:
Damyon Wiese 2014-08-26 12:28:47 +08:00
commit 112632ed58
4 changed files with 478 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

120
lib/ddl/mysql_sql_generator.php
View file

@ -88,6 +88,9 @@ class mysql_sql_generator extends sql_generator {
/** @var string SQL sentence to rename one key 'TABLENAME', 'OLDKEYNAME' and 'NEWKEYNAME' are dynamically replaced.*/
public $rename_key_sql = null;
/** Maximum size of InnoDB row in Antelope file format */
const ANTELOPE_MAX_ROW_SIZE = 8126;
/**
* Reset a sequence to the id field of a table.
*
@ -108,6 +111,78 @@ class mysql_sql_generator extends sql_generator {
return array("ALTER TABLE $this->prefix$tablename AUTO_INCREMENT = $value");
}
/**
* Calculate proximate row size when using InnoDB
* tables in Antelope row format.
*
* Note: the returned value is a bit higher to compensate for
* errors and changes of column data types.
*
* @param xmldb_field[]|database_column_info[] $columns
* @return int approximate row size in bytes
*/
public function guess_antolope_row_size(array $columns) {
if (empty($columns)) {
return 0;
}
$size = 0;
$first = reset($columns);
if (count($columns) > 1) {
// Do not start with zero because we need to cover changes of field types and
// this calculation is most probably not be accurate.
$size += 1000;
}
if ($first instanceof xmldb_field) {
foreach ($columns as $field) {
switch ($field->getType()) {
case XMLDB_TYPE_TEXT:
$size += 768;
break;
case XMLDB_TYPE_BINARY:
$size += 768;
break;
case XMLDB_TYPE_CHAR:
$bytes = $field->getLength() * 3;
if ($bytes > 768) {
$bytes = 768;
}
$size += $bytes;
break;
default:
// Anything else is usually maximum 8 bytes.
$size += 8;
}
}
} else if ($first instanceof database_column_info) {
foreach ($columns as $column) {
switch ($column->meta_type) {
case 'X':
$size += 768;
break;
case 'B':
$size += 768;
break;
case 'C':
$bytes = $column->max_length * 3;
if ($bytes > 768) {
$bytes = 768;
}
$size += $bytes;
break;
default:
// Anything else is usually maximum 8 bytes.
$size += 8;
}
}
}
return $size;
}
/**
* Given one correct xmldb_table, returns the SQL statements
* to create it (inside one array).
@ -122,6 +197,15 @@ class mysql_sql_generator extends sql_generator {
// Do we know collation?
$collation = $this->mdb->get_dbcollation();
// Do we need to use compressed format for rows?
$rowformat = "";
$size = $this->guess_antolope_row_size($xmldb_table->getFields());
if ($size > self::ANTELOPE_MAX_ROW_SIZE) {
if ($this->mdb->is_compressed_row_format_supported()) {
$rowformat = "\n ROW_FORMAT=Compressed";
}
}
$sqlarr = parent::getCreateTableSQL($xmldb_table);
// This is a very nasty hack that tries to use just one query per created table
@ -144,6 +228,9 @@ class mysql_sql_generator extends sql_generator {
}
$sql .= "\n DEFAULT COLLATE = $collation";
}
if ($rowformat) {
$sql .= $rowformat;
}
$sqls[] = $sql;
continue;
}
@ -186,6 +273,39 @@ class mysql_sql_generator extends sql_generator {
return $sqls;
}
/**
* Given one xmldb_table and one xmldb_field, return the SQL statements needed to add the field to the table.
*
* @param xmldb_table $xmldb_table The table related to $xmldb_field.
* @param xmldb_field $xmldb_field The instance of xmldb_field to create the SQL from.
* @param string $skip_type_clause The type clause on alter columns, NULL by default.
* @param string $skip_default_clause The default clause on alter columns, NULL by default.
* @param string $skip_notnull_clause The null/notnull clause on alter columns, NULL by default.
* @return array The SQL statement for adding a field to the table.
*/
public function getAddFieldSQL($xmldb_table, $xmldb_field, $skip_type_clause = NULL, $skip_default_clause = NULL, $skip_notnull_clause = NULL) {
$sqls = parent::getAddFieldSQL($xmldb_table, $xmldb_field, $skip_type_clause, $skip_default_clause, $skip_notnull_clause);
if ($this->table_exists($xmldb_table)) {
$tablename = $xmldb_table->getName();
$size = $this->guess_antolope_row_size($this->mdb->get_columns($tablename));
$size += $this->guess_antolope_row_size(array($xmldb_field));
if ($size > self::ANTELOPE_MAX_ROW_SIZE) {
if ($this->mdb->is_compressed_row_format_supported()) {
$format = strtolower($this->mdb->get_row_format($tablename));
if ($format === 'compact' or $format === 'redundant') {
// Change the format before conversion so that we do not run out of space.
array_unshift($sqls, "ALTER TABLE {$this->prefix}$tablename ROW_FORMAT=Compressed");
}
}
}
}
return $sqls;
}
/**
* Given one correct xmldb_table, returns the SQL statements
* to create temporary table (inside one array).

87
lib/ddl/tests/ddl_test.php
View file

@ -508,6 +508,93 @@ class core_ddl_testcase extends database_driver_testcase {
}
}
/**
* Test if database supports tables with many TEXT fields,
* InnoDB is known to failed during data insertion instead
* of table creation when text fields contain actual data.
*/
public function test_row_size_limits() {
$DB = $this->tdb; // Do not use global $DB!
$dbman = $this->tdb->get_manager();
$text = str_repeat('š', 1333);
$data = new stdClass();
$data->name = 'test';
$table = new xmldb_table('test_innodb');
$table->add_field('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null);
$table->add_field('name', XMLDB_TYPE_CHAR, '30', null, null, null, null);
$table->add_key('primary', XMLDB_KEY_PRIMARY, array('id'));
for ($i = 0; $i < 20; $i++) {
$table->add_field('text'.$i, XMLDB_TYPE_TEXT, null, null, null, null, null);
$data->{'text'.$i} = $text;
}
$dbman->create_table($table);
try {
$id = $DB->insert_record('test_innodb', $data);
$expected = (array)$data;
$expected['id'] = (string)$id;
$this->assertEquals($expected, (array)$DB->get_record('test_innodb', array('id' => $id)), '', 0, 10, true);
} catch (dml_exception $e) {
// Give some nice error message when known problematic MySQL with InnoDB detected.
if ($DB->get_dbfamily() === 'mysql') {
$engine = strtolower($DB->get_dbengine());
if ($engine === 'innodb' or $engine === 'xtradb') {
if (!$DB->is_compressed_row_format_supported()) {
$this->fail("Row size limit reached in MySQL using InnoDB, configure server to use innodb_file_format=Barracuda and innodb_file_per_table=1");
}
}
}
throw $e;
}
$dbman->drop_table($table);
$data = new stdClass();
$data->name = 'test';
$table = new xmldb_table('test_innodb');
$table->add_field('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null);
$table->add_field('name', XMLDB_TYPE_CHAR, '30', null, null, null, null);
$table->add_key('primary', XMLDB_KEY_PRIMARY, array('id'));
$dbman->create_table($table);
$DB->insert_record('test_innodb', array('name' => 'test'));
for ($i = 0; $i < 20; $i++) {
$field = new xmldb_field('text'.$i, XMLDB_TYPE_TEXT, null, null, null, null, null);
$dbman->add_field($table, $field);
$data->{'text'.$i} = $text;
$id = $DB->insert_record('test_innodb', $data);
$expected = (array)$data;
$expected['id'] = (string)$id;
$this->assertEquals($expected, (array)$DB->get_record('test_innodb', array('id' => $id)), '', 0, 10, true);
}
$dbman->drop_table($table);
// MySQL VARCHAR fields may hit a different 65535 row size limit when creating tables.
$data = new stdClass();
$data->name = 'test';
$table = new xmldb_table('test_innodb');
$table->add_field('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null);
$table->add_field('name', XMLDB_TYPE_CHAR, '30', null, null, null, null);
$table->add_key('primary', XMLDB_KEY_PRIMARY, array('id'));
for ($i = 0; $i < 15; $i++) {
$table->add_field('text'.$i, XMLDB_TYPE_CHAR, '1333', null, null, null, null);
$data->{'text'.$i} = $text;
}
$dbman->create_table($table);
$id = $DB->insert_record('test_innodb', $data);
$expected = (array)$data;
$expected['id'] = (string)$id;
$this->assertEquals($expected, (array)$DB->get_record('test_innodb', array('id' => $id)), '', 0, 10, true);
$dbman->drop_table($table);
}
/**
* Test behaviour of drop_table()
*/

69
lib/dml/mysqli_native_moodle_database.php
View file

@ -39,6 +39,8 @@ class mysqli_native_moodle_database extends moodle_database {
/** @var mysqli $mysqli */
protected $mysqli = null;
/** @var bool is compressed row format supported cache */
protected $compressedrowformatsupported = null;
private $transactions_supported = null;
@ -280,6 +282,73 @@ class mysqli_native_moodle_database extends moodle_database {
return $collation;
}
/**
* Get the row format from the database schema.
*
* @param string $table
* @return string row_format name or null if not known or table does not exist.
*/
public function get_row_format($table) {
$rowformat = null;
$table = $this->mysqli->real_escape_string($table);
$sql = "SELECT row_format
FROM INFORMATION_SCHEMA.TABLES
WHERE table_schema = DATABASE() AND table_name = '{$this->prefix}$table'";
$this->query_start($sql, NULL, SQL_QUERY_AUX);
$result = $this->mysqli->query($sql);
$this->query_end($result);
if ($rec = $result->fetch_assoc()) {
$rowformat = $rec['row_format'];
}
$result->close();
return $rowformat;
}
/**
* Is this database compatible with compressed row format?
* This feature is necessary for support of large number of text
* columns in InnoDB/XtraDB database.
*
* @param bool $cached use cached result
* @return bool true if table can be created or changed to compressed row format.
*/
public function is_compressed_row_format_supported($cached = true) {
if ($cached and isset($this->compressedrowformatsupported)) {
return($this->compressedrowformatsupported);
}
$engine = strtolower($this->get_dbengine());
$info = $this->get_server_info();
if (version_compare($info['version'], '5.5.0') < 0) {
// MySQL 5.1 is not supported here because we cannot read the file format.
$this->compressedrowformatsupported = false;
} else if ($engine !== 'innodb' and $engine !== 'xtradb') {
// Other engines are not supported, most probably not compatible.
$this->compressedrowformatsupported = false;
} else if (!$filepertable = $this->get_record_sql("SHOW VARIABLES LIKE 'innodb_file_per_table'")) {
$this->compressedrowformatsupported = false;
} else if ($filepertable->value !== 'ON') {
$this->compressedrowformatsupported = false;
} else if (!$fileformat = $this->get_record_sql("SHOW VARIABLES LIKE 'innodb_file_format'")) {
$this->compressedrowformatsupported = false;
} else if ($fileformat->value !== 'Barracuda') {
$this->compressedrowformatsupported = false;
} else {
// All the tests passed, we can safely use ROW_FORMAT=Compressed in sql statements.
$this->compressedrowformatsupported = true;
}
return $this->compressedrowformatsupported;
}
/**
* Returns localised database type name
* Note: can be used before connect()