Merge branch 'MDL-66694-master' of git://github.com/junpataleta/moodle

config-dist.php

@@ -643,6 +643,15 @@ $CFG->admin = 'admin';
 //              . 'copy_action_column,preview_action_column,delete_action_column,'
 //              . 'creator_name_column,modifier_name_column';
 //
+// Forum summary report
+//
+// In order for the forum summary report to calculate word count and character count data, those details are now stored
+// for each post in the database when posts are created or updated. For posts that existed prior to a Moodle 3.8 upgrade,
+// these are calculated by the refresh_forum_post_counts ad-hoc task in chunks of 5000 posts per batch by default.
+// That default can be overridden by setting an integer value for $CFG->forumpostcountchunksize.
+//
+//      $CFG->forumpostcountchunksize = 5000;
+//
 //=========================================================================
 // 7. SETTINGS FOR DEVELOPMENT SERVERS - not intended for production use!!!
 //=========================================================================

mod/forum/backup/moodle2/restore_forum_stepslib.php

@@ -116,6 +116,7 @@ class restore_forum_activity_structure_step extends restore_activity_structure_s
             $data->parent = $this->get_mappingid('forum_post', $data->parent);
         }
 
+        \mod_forum\local\entities\post::add_message_counts($data);
         $newitemid = $DB->insert_record('forum_posts', $data);
         $this->set_mapping('forum_post', $oldid, $newitemid, true);
 

mod/forum/classes/local/data_mappers/legacy/post.php

@@ -61,6 +61,8 @@ class post {
                 'mailnow' => $post->should_mail_now(),
                 'deleted' => $post->is_deleted(),
                 'privatereplyto' => $post->get_private_reply_recipient_id(),
+                'wordcount' => $post->get_wordcount(),
+                'charcount' => $post->get_charcount(),
             ];
         }, $posts);
     }

mod/forum/classes/local/entities/post.php

@@ -67,6 +67,10 @@ class post {
     private $deleted;
     /** @var int $privatereplyto The user being privately replied to */
     private $privatereplyto;
+    /** @var int $wordcount Number of words in the message */
+    private $wordcount;
+    /** @var int $charcount Number of chars in the message */
+    private $charcount;
 
     /**
      * Constructor.
@@ -104,7 +108,9 @@ class post {
         int $totalscore,
         bool $mailnow,
         bool $deleted,
-        int $privatereplyto
+        int $privatereplyto,
+        ?int $wordcount,
+        ?int $charcount
     ) {
         $this->id = $id;
         $this->discussionid = $discussionid;
@@ -122,6 +128,8 @@ class post {
         $this->mailnow = $mailnow;
         $this->deleted = $deleted;
         $this->privatereplyto = $privatereplyto;
+        $this->wordcount = $wordcount;
+        $this->charcount = $charcount;
     }
 
     /**
@@ -315,4 +323,35 @@ class post {
     public function is_private_reply_intended_for_user(stdClass $user) : bool {
         return $this->get_private_reply_recipient_id() == $user->id;
     }
+
+    /**
+     * Returns the word count.
+     *
+     * @return int|null
+     */
+    public function get_wordcount() : ?int {
+        return $this->wordcount;
+    }
+
+    /**
+     * Returns the char count.
+     *
+     * @return int|null
+     */
+    public function get_charcount() : ?int {
+        return $this->charcount;
+    }
+
+    /**
+     * This methods adds/updates forum posts' word count and char count attributes based on $data->message.
+     *
+     * @param \stdClass $record A record ready to be inserted / updated in DB.
+     * @return void.
+     */
+    public static function add_message_counts(\stdClass $record) : void {
+        if (!empty($record->message)) {
+            $record->wordcount = count_words($record->message);
+            $record->charcount = count_letters($record->message);
+        }
+    }
 }

mod/forum/classes/local/exporters/post.php

@@ -120,6 +120,12 @@ class post extends exporter {
                 'default' => null,
                 'null' => NULL_ALLOWED
             ],
+            'charcount' => [
+                'type' => PARAM_INT,
+                'optional' => true,
+                'default' => null,
+                'null' => NULL_ALLOWED
+            ],
             'capabilities' => [
                 'type' => [
                     'view' => [
@@ -410,6 +416,15 @@ class post extends exporter {
             $replysubject = "{$strre} {$replysubject}";
         }
 
+        $showwordcount = $forum->should_display_word_count();
+        if ($showwordcount) {
+            $wordcount = $post->get_wordcount() ?? count_words($message);
+            $charcount = $post->get_charcount() ?? count_letters($message);
+        } else {
+            $wordcount = null;
+            $charcount = null;
+        }
+
         return [
             'id' => $post->get_id(),
             'subject' => $subject,
@@ -424,8 +439,9 @@ class post extends exporter {
             'unread' => ($loadcontent && $readreceiptcollection) ? !$readreceiptcollection->has_user_read_post($user, $post) : null,
             'isdeleted' => $isdeleted,
             'isprivatereply' => $isprivatereply,
-            'haswordcount' => $forum->should_display_word_count(),
+            'haswordcount' => $showwordcount,
-            'wordcount' => $forum->should_display_word_count() ? count_words($message) : null,
+            'wordcount' => $wordcount,
+            'charcount' => $charcount,
             'capabilities' => [
                 'view' => $canview,
                 'edit' => $canedit,

mod/forum/classes/local/factories/entity.php

@@ -154,7 +154,9 @@ class entity {
             $record->totalscore,
             $record->mailnow,
             $record->deleted,
-            $record->privatereplyto
+            $record->privatereplyto,
+            $record->wordcount,
+            $record->charcount
         );
     }
 

mod/forum/classes/task/refresh_forum_post_counts.php

@@ -0,0 +1,78 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Adhoc task that updates all of the existing forum_post records with no wordcount or no charcount.
+ *
+ * @package    mod_forum
+ * @copyright  2019 David Monllao
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace mod_forum\task;
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Adhoc task that updates all of the existing forum_post records with no wordcount or no charcount.
+ *
+ * @package     mod_forum
+ * @copyright   2019 David Monllao
+ * @license     http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class refresh_forum_post_counts extends \core\task\adhoc_task {
+
+    /**
+     * Run the task to populate word and character counts on existing forum posts.
+     * If the maximum number of records are updated, the task re-queues itself,
+     * as there may be more records to process.
+     */
+    public function execute() {
+        if ($this->update_null_forum_post_counts()) {
+            \core\task\manager::queue_adhoc_task(new refresh_forum_post_counts());
+        }
+    }
+
+    /**
+     * Updates null forum post counts according to the post message.
+     *
+     * @return bool Whether there may be more rows to process
+     */
+    protected function update_null_forum_post_counts(): bool {
+        global $CFG, $DB;
+
+        // Default to chunks of 5000 records per run, unless overridden in config.php
+        $chunksize = $CFG->forumpostcountchunksize ?? 5000;
+
+        $select = 'wordcount IS NULL OR charcount IS NULL';
+        $recordset = $DB->get_recordset_select('forum_posts', $select, null, 'discussion', 'id, message', 0, $chunksize);
+
+        if (!$recordset->valid()) {
+            $recordset->close();
+            return false;
+        }
+
+        foreach ($recordset as $record) {
+            \mod_forum\local\entities\post::add_message_counts($record);
+            $DB->update_record('forum_posts', $record);
+        }
+
+        $recordscount = count($recordset);
+        $recordset->close();
+
+        return ($recordscount == $chunksize);
+    }
+}

mod/forum/db/install.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<XMLDB PATH="mod/forum/db" VERSION="20190404" COMMENT="XMLDB file for Moodle mod/forum"
+<XMLDB PATH="mod/forum/db" VERSION="20191001" COMMENT="XMLDB file for Moodle mod/forum"
     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
     xsi:noNamespaceSchemaLocation="../../../lib/xmldb/xmldb.xsd"
 >
@@ -85,6 +85,8 @@
         <FIELD NAME="mailnow" TYPE="int" LENGTH="10" NOTNULL="true" DEFAULT="0" SEQUENCE="false"/>
         <FIELD NAME="deleted" TYPE="int" LENGTH="1" NOTNULL="true" DEFAULT="0" SEQUENCE="false"/>
         <FIELD NAME="privatereplyto" TYPE="int" LENGTH="10" NOTNULL="true" DEFAULT="0" SEQUENCE="false"/>
+        <FIELD NAME="wordcount" TYPE="int" LENGTH="20" NOTNULL="false" SEQUENCE="false"/>
+        <FIELD NAME="charcount" TYPE="int" LENGTH="20" NOTNULL="false" SEQUENCE="false"/>
       </FIELDS>
       <KEYS>
         <KEY NAME="primary" TYPE="primary" FIELDS="id"/>
@@ -192,4 +194,4 @@
       </KEYS>
     </TABLE>
   </TABLES>
-</XMLDB>
+</XMLDB>

mod/forum/db/upgrade.php

@@ -157,5 +157,44 @@ function xmldb_forum_upgrade($oldversion) {
     // Automatically generated Moodle v3.7.0 release upgrade line.
     // Put any upgrade step following this.
 
+    if ($oldversion < 2019071901) {
+
+        // Define field wordcount to be added to forum_posts.
+        $table = new xmldb_table('forum_posts');
+        $field = new xmldb_field('wordcount', XMLDB_TYPE_INTEGER, '20', null, null, null, null, 'privatereplyto');
+
+        // Conditionally launch add field wordcount.
+        if (!$dbman->field_exists($table, $field)) {
+            $dbman->add_field($table, $field);
+        }
+
+        // Define field charcount to be added to forum_posts.
+        $table = new xmldb_table('forum_posts');
+        $field = new xmldb_field('charcount', XMLDB_TYPE_INTEGER, '20', null, null, null, null, 'wordcount');
+
+        // Conditionally launch add field charcount.
+        if (!$dbman->field_exists($table, $field)) {
+            $dbman->add_field($table, $field);
+        }
+
+        // Forum savepoint reached.
+        upgrade_mod_savepoint(true, 2019071901, 'forum');
+    }
+
+    if ($oldversion < 2019071902) {
+        // Create adhoc task for upgrading of existing forum_posts.
+        $record = new \stdClass();
+        $record->classname = '\mod_forum\task\refresh_forum_post_counts';
+        $record->component = 'mod_forum';
+
+        // Next run time based from nextruntime computation in \core\task\manager::queue_adhoc_task().
+        $nextruntime = time() - 1;
+        $record->nextruntime = $nextruntime;
+        $DB->insert_record('task_adhoc', $record);
+
+        // Main savepoint reached.
+        upgrade_mod_savepoint(true, 2019071902, 'forum');
+    }
+
     return true;
 }

mod/forum/lib.php

@@ -240,6 +240,7 @@ function forum_update_instance($forum, $mform) {
             $post->message = file_save_draft_area_files($draftid, $modcontext->id, 'mod_forum', 'post', $post->id, $options, $post->message);
         }
 
+        \mod_forum\local\entities\post::add_message_counts($post);
         $DB->update_record('forum_posts', $post);
         $discussion->name = $forum->name;
         $DB->update_record('forum_discussions', $discussion);
@@ -2948,6 +2949,7 @@ function forum_add_new_post($post, $mform, $unused = null) {
         $post->mailnow    = 0;
     }
 
+    \mod_forum\local\entities\post::add_message_counts($post);
     $post->id = $DB->insert_record("forum_posts", $post);
     $post->message = file_save_draft_area_files($post->itemid, $context->id, 'mod_forum', 'post', $post->id,
             mod_forum_post_form::editor_options($context, null), $post->message);
@@ -3018,6 +3020,7 @@ function forum_update_post($newpost, $mform, $unused = null) {
     }
     $post->message = file_save_draft_area_files($newpost->itemid, $context->id, 'mod_forum', 'post', $post->id,
             mod_forum_post_form::editor_options($context, $post->id), $post->message);
+    \mod_forum\local\entities\post::add_message_counts($post);
     $DB->update_record('forum_posts', $post);
     // Note: Discussion modified time/user are intentionally not updated, to enable them to track the latest new post.
     $DB->update_record('forum_discussions', $discussion);
@@ -3080,6 +3083,7 @@ function forum_add_discussion($discussion, $mform=null, $unused=null, $userid=nu
     $post->course        = $forum->course; // speedup
     $post->mailnow       = $discussion->mailnow;
 
+    \mod_forum\local\entities\post::add_message_counts($post);
     $post->id = $DB->insert_record("forum_posts", $post);
 
     // TODO: Fix the calling code so that there always is a $cm when this function is called

mod/forum/report/summary/classes/summary_table.php

@@ -72,6 +72,11 @@ class summary_table extends table_sql {
      */
     protected $context = null;
 
+    /**
+     * @var bool
+     */
+    private $showwordcharcounts = null;
+
     /**
      * Forum report table constructor.
      *
@@ -120,6 +125,11 @@ class summary_table extends table_sql {
             $columnheaders['viewcount'] = get_string('viewcount', 'forumreport_summary');
         }
 
+        if ($this->show_word_char_counts()) {
+            $columnheaders['wordcount'] = get_string('wordcount', 'forumreport_summary');
+            $columnheaders['charcount'] = get_string('charcount', 'forumreport_summary');
+        }
+
         $columnheaders['earliestpost'] = get_string('earliestpost', 'forumreport_summary');
         $columnheaders['latestpost'] = get_string('latestpost', 'forumreport_summary');
 
@@ -440,10 +450,16 @@ class summary_table extends table_sql {
             $this->fill_log_summary_temp_table($this->context->id);
 
             $this->sql->basefields .= ', CASE WHEN tmp.viewcount IS NOT NULL THEN tmp.viewcount ELSE 0 END AS viewcount';
-            $this->sql->basefromjoins .= ' LEFT JOIN {' . self::LOG_SUMMARY_TEMP_TABLE . '} tmp ON tmp.userid = u.id';
+            $this->sql->basefromjoins .= ' LEFT JOIN {' . self::LOG_SUMMARY_TEMP_TABLE . '} tmp ON tmp.userid = u.id ';
             $this->sql->basegroupby .= ', tmp.viewcount';
         }
 
+        if ($this->show_word_char_counts()) {
+            // All p.wordcount values should be NOT NULL, this CASE WHEN is an extra just-in-case.
+            $this->sql->basefields .= ', SUM(CASE WHEN p.wordcount IS NOT NULL THEN p.wordcount ELSE 0 END) AS wordcount';
+            $this->sql->basefields .= ', SUM(CASE WHEN p.charcount IS NOT NULL THEN p.charcount ELSE 0 END) AS charcount';
+        }
+
         $this->sql->params = [
             'component' => 'mod_forum',
             'courseid' => $this->cm->course,
@@ -663,4 +679,31 @@ class summary_table extends table_sql {
         $this->is_downloading($format, $filename);
         $this->out($this->perpage, false);
     }
+
+    /*
+     * Should the word / char counts be displayed?
+     *
+     * We don't want to show word/char columns if there is any null value because this means
+     * that they have not been calculated yet.
+     * @return bool
+     */
+    protected function show_word_char_counts(): bool {
+        global $DB;
+
+        if (is_null($this->showwordcharcounts)) {
+            // This should be really fast.
+            $sql = "SELECT 'x'
+                      FROM {forum_posts} fp
+                      JOIN {forum_discussions} fd ON fd.id = fp.discussion
+                     WHERE fd.forum = :forumid AND (fp.wordcount IS NULL OR fp.charcount IS NULL)";
+
+            if ($DB->record_exists_sql($sql, ['forumid' => $this->cm->instance])) {
+                $this->showwordcharcounts = false;
+            } else {
+                $this->showwordcharcounts = true;
+            }
+        }
+
+        return $this->showwordcharcounts;
+    }
 }

mod/forum/report/summary/lang/en/forumreport_summary.php

@@ -23,6 +23,7 @@
  */
 
 $string['attachmentcount'] = 'Number of attachments';
+$string['charcount'] = 'Character count';
 $string['viewcount'] = 'Number of views';
 $string['earliestpost'] = 'Earliest post';
 $string['filter:groupsbuttonlabel'] = 'Open the groups filter';
@@ -38,4 +39,5 @@ $string['replycount'] = 'Number of replies posted';
 $string['summary:viewall'] = 'Access summary report data for each user within a given forum or forums';
 $string['summary:view'] = 'Access summary report within a given forum or forums';
 $string['summarytitle'] = 'Summary report - {$a}';
-$string['viewsdisclaimer'] = 'Number of views column is not filtered by group';
+$string['viewsdisclaimer'] = 'Number of views column is not filtered by group';
+$string['wordcount'] = 'Word count';

mod/forum/tests/entities_discussion_summary_test.php

@@ -93,7 +93,9 @@ class mod_forum_entities_discussion_summary_testcase extends advanced_testcase {
             0,
             false,
             false,
-            false
+            false,
+            null,
+            null
         );
 
         $discussionsummary = new discussion_summary_entity($discussion, $firstpost, $firstauthor, $lastauthor);

mod/forum/tests/entities_discussion_test.php

@@ -75,7 +75,9 @@ class mod_forum_entities_discussion_testcase extends advanced_testcase {
             0,
             false,
             false,
-            false
+            false,
+            null,
+            null
         );
         $notfirstpost = new post_entity(
             1,
@@ -93,7 +95,9 @@ class mod_forum_entities_discussion_testcase extends advanced_testcase {
             0,
             false,
             false,
-            false
+            false,
+            null,
+            null
         );
 
         $this->assertEquals(1, $discussion->get_id());

mod/forum/tests/entities_post_read_receipt_collection_test.php

@@ -58,7 +58,9 @@ class mod_forum_entities_post_read_receipt_collection_testcase extends advanced_
             0,
             false,
             false,
-            false
+            false,
+            null,
+            null
         );
         $post = new post_entity(
             1,
@@ -76,7 +78,9 @@ class mod_forum_entities_post_read_receipt_collection_testcase extends advanced_
             0,
             false,
             false,
-            false
+            false,
+            null,
+            null
         );
         $collection = new collection_entity([
             (object) [

mod/forum/tests/entities_post_test.php

@@ -59,7 +59,9 @@ class mod_forum_entities_post_testcase extends advanced_testcase {
             0,
             false,
             false,
-            false
+            false,
+            null,
+            null
         );
 
         $this->assertEquals(4, $post->get_id());

mod/forum/tests/externallib_test.php

@@ -582,6 +582,8 @@ class mod_forum_external_testcase extends externallib_advanced_testcase {
         $record = new stdClass();
         $record->course = $course1->id;
         $record->trackingtype = FORUM_TRACKING_OFF;
+        // Display word count. Otherwise, word and char counts will be set to null by the forum post exporter.
+        $record->displaywordcount = true;
         $forum1 = self::getDataGenerator()->create_module('forum', $record);
         $forum1context = context_module::instance($forum1->cmid);
 
@@ -673,6 +675,8 @@ class mod_forum_external_testcase extends externallib_advanced_testcase {
         // User pictures are initially empty, we should get the links once the external function is called.
         $isolatedurl = $urlfactory->get_discussion_view_url_from_discussion_id($discussion1reply2->discussion);
         $isolatedurl->params(['parent' => $discussion1reply2->id]);
+        $message = file_rewrite_pluginfile_urls($discussion1reply2->message, 'pluginfile.php',
+            $forum1context->id, 'mod_forum', 'post', $discussion1reply2->id);
         $expectedposts['posts'][] = array(
             'id' => $discussion1reply2->id,
             'discussionid' => $discussion1reply2->discussion,
@@ -681,14 +685,14 @@ class mod_forum_external_testcase extends externallib_advanced_testcase {
             'timecreated' => $discussion1reply2->created,
             'subject' => $discussion1reply2->subject,
             'replysubject' => get_string('re', 'mod_forum') . " {$discussion1reply2->subject}",
-            'message' => file_rewrite_pluginfile_urls($discussion1reply2->message, 'pluginfile.php',
+            'message' => $message,
-                    $forum1context->id, 'mod_forum', 'post', $discussion1reply2->id),
             'messageformat' => 1,   // This value is usually changed by external_format_text() function.
             'unread' => null,
             'isdeleted' => false,
             'isprivatereply' => false,
-            'haswordcount' => false,
+            'haswordcount' => true,
-            'wordcount' => null,
+            'wordcount' => count_words($message),
+            'charcount' => count_letters($message),
             'author'=> $exporteduser3,
             'attachments' => [],
             'tags' => [],
@@ -728,6 +732,8 @@ class mod_forum_external_testcase extends externallib_advanced_testcase {
 
         $isolatedurl = $urlfactory->get_discussion_view_url_from_discussion_id($discussion1reply1->discussion);
         $isolatedurl->params(['parent' => $discussion1reply1->id]);
+        $message = file_rewrite_pluginfile_urls($discussion1reply1->message, 'pluginfile.php',
+            $forum1context->id, 'mod_forum', 'post', $discussion1reply1->id);
         $expectedposts['posts'][] = array(
             'id' => $discussion1reply1->id,
             'discussionid' => $discussion1reply1->discussion,
@@ -736,14 +742,14 @@ class mod_forum_external_testcase extends externallib_advanced_testcase {
             'timecreated' => $discussion1reply1->created,
             'subject' => $discussion1reply1->subject,
             'replysubject' => get_string('re', 'mod_forum') . " {$discussion1reply1->subject}",
-            'message' => file_rewrite_pluginfile_urls($discussion1reply1->message, 'pluginfile.php',
+            'message' => $message,
-                    $forum1context->id, 'mod_forum', 'post', $discussion1reply1->id),
             'messageformat' => 1,   // This value is usually changed by external_format_text() function.
             'unread' => null,
             'isdeleted' => false,
             'isprivatereply' => false,
-            'haswordcount' => false,
+            'haswordcount' => true,
-            'wordcount' => null,
+            'wordcount' => count_words($message),
+            'charcount' => count_letters($message),
             'author'=> $exporteduser2,
             'attachments' => [],
             'tags' => [],

mod/forum/tests/generator/lib.php

@@ -313,6 +313,7 @@ class mod_forum_generator extends testing_module_generator {
         }
 
         $record = (object) $record;
+        \mod_forum\local\entities\post::add_message_counts($record);
 
         // Add the post.
         $record->id = $DB->insert_record('forum_posts', $record);

mod/forum/upgrade.txt

@@ -5,6 +5,10 @@ information provided here is intended especially for developers.
 
 * The following functions have been finally deprecated and can not be used anymore:
     * forum_scale_used()
+* In order for the forum summary report to calculate word count and character count data, those details are now stored
+    for each post in the database when posts are created or updated. For posts that existed prior to a Moodle 3.8 upgrade, these
+    are calculated by the refresh_forum_post_counts ad-hoc task in chunks of 5000 posts by default. Site admins are able to modify this
+    default, by setting $CFG->forumpostcountchunksize to the required integer value.
 
 === 3.7 ===
   * Changed the forum discussion rendering to use templates rather than print functions.

mod/forum/version.php

@@ -24,6 +24,6 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-$plugin->version   = 2019071900;       // The current module version (Date: YYYYMMDDXX)
+$plugin->version   = 2019071902;       // The current module version (Date: YYYYMMDDXX)
 $plugin->requires  = 2019051100;       // Requires this Moodle version
 $plugin->component = 'mod_forum';      // Full name of the plugin (used for diagnostics)