MDL-69028 repository: Put a rate limit on draft file uploads

lang/en/error.php

@@ -391,6 +391,7 @@ $string['loginasnoenrol'] = 'You cannot use enrol or unenrol when in course "Log
 $string['loginasonecourse'] = 'You cannot enter this course.<br /> You have to terminate the "Login as" session before entering any other course.';
 $string['maxbytesfile'] = 'The file {$a->file} is too large. The maximum size you can upload is {$a->size}.';
 $string['maxareabytes'] = 'The file is larger than the space remaining in this area.';
+$string['maxdraftitemids'] = 'Due to uploading a high volume of files, your file uploads are temporarily limited. Please try again after a few seconds.';
 $string['messageundeliveredbynotificationsettings'] = 'The message could not be sent because personal messages between users (in Notification settings) has been disabled by a site administrator.';
 $string['messagingdisable'] = 'Messaging is disabled on this site';
 $string['mimetexisnotexist'] = 'Your system is not configured to run mimeTeX. You need to obtain the C source from <a href="https://www.forkosh.com/mimetex.zip">https://www.forkosh.com/mimetex.zip</a>, compile it and put the executable into your moodle/filter/tex/ directory.';

lib/filelib.php

@@ -40,6 +40,16 @@ define('IGNORE_FILE_MERGE', -1);
  */
 define('FILE_AREA_MAX_BYTES_UNLIMITED', -1);
 
+/**
+ * Capacity of the draft area bucket when using the leaking bucket technique to limit the draft upload rate.
+ */
+define('DRAFT_AREA_BUCKET_CAPACITY', 50);
+
+/**
+ * Leaking rate of the draft area bucket when using the leaking bucket technique to limit the draft upload rate.
+ */
+define('DRAFT_AREA_BUCKET_LEAK', 0.2);
+
 require_once("$CFG->libdir/filestorage/file_exceptions.php");
 require_once("$CFG->libdir/filestorage/file_storage.php");
 require_once("$CFG->libdir/filestorage/zip_packer.php");
@@ -390,7 +400,7 @@ function file_get_unused_draft_itemid() {
  * @return string|null returns string if $text was passed in, the rewritten $text is returned. Otherwise NULL.
  */
 function file_prepare_draft_area(&$draftitemid, $contextid, $component, $filearea, $itemid, array $options=null, $text=null) {
-    global $CFG, $USER, $CFG;
+    global $CFG, $USER;
 
     $options = (array)$options;
     if (!isset($options['subdirs'])) {
@@ -606,6 +616,55 @@ function file_is_draft_area_limit_reached($draftitemid, $areamaxbytes, $newfiles
     return false;
 }
 
+/**
+ * Returns whether a user has reached their draft area upload rate.
+ *
+ * @param int $userid The user id
+ * @return bool
+ */
+function file_is_draft_areas_limit_reached(int $userid): bool {
+    global $CFG;
+
+    $capacity = $CFG->draft_area_bucket_capacity ?? DRAFT_AREA_BUCKET_CAPACITY;
+    $leak = $CFG->draft_area_bucket_leak ?? DRAFT_AREA_BUCKET_LEAK;
+
+    $since = time() - floor($capacity / $leak); // The items that were in the bucket before this time are already leaked by now.
+                                                // We are going to be a bit generous to the user when using the leaky bucket
+                                                // algorithm below. We are going to assume that the bucket is empty at $since.
+                                                // We have to do an assumption here unless we really want to get ALL user's draft
+                                                // items without any limit and put all of them in the leaking bucket.
+                                                // I decided to favour performance over accuracy here.
+
+    $fs = get_file_storage();
+    $items = $fs->get_user_draft_items($userid, $since);
+    $items = array_reverse($items); // So that the items are sorted based on time in the ascending direction.
+
+    // We only need to store the time that each element in the bucket is going to leak. So $bucket is array of leaking times.
+    $bucket = [];
+    foreach ($items as $item) {
+        $now = $item->timemodified;
+        // First let's see if items can be dropped from the bucket as a result of leakage.
+        while (!empty($bucket) && ($now >= $bucket[0])) {
+            array_shift($bucket);
+        }
+
+        // Calculate the time that the new item we put into the bucket will be leaked from it, and store it into the bucket.
+        if ($bucket) {
+            $bucket[] = max($bucket[count($bucket) - 1], $now) + (1 / $leak);
+        } else {
+            $bucket[] = $now + (1 / $leak);
+        }
+    }
+
+    // Recalculate the bucket's content based on the leakage until now.
+    $now = time();
+    while (!empty($bucket) && ($now >= $bucket[0])) {
+        array_shift($bucket);
+    }
+
+    return count($bucket) >= $capacity;
+}
+
 /**
  * Get used space of files
  * @global moodle_database $DB

lib/filestorage/file_storage.php

@@ -663,6 +663,41 @@ class file_storage {
         return $result;
     }
 
+    /**
+     * Returns the file area item ids and their updatetime for a user's draft uploads, sorted by updatetime DESC.
+     *
+     * @param int $userid user id
+     * @param int $updatedsince only return draft areas updated since this time
+     * @param int $lastnum only return the last specified numbers
+     * @return array
+     */
+    public function get_user_draft_items(int $userid, int $updatedsince = 0, int $lastnum = 0): array {
+        global $DB;
+
+        $params = [
+            'component' => 'user',
+            'filearea' => 'draft',
+            'contextid' => context_user::instance($userid)->id,
+        ];
+
+        $updatedsincesql = '';
+        if ($updatedsince) {
+            $updatedsincesql = 'AND f.timemodified > :time';
+            $params['time'] = $updatedsince;
+        }
+        $sql = "SELECT itemid,
+                       MAX(f.timemodified) AS timemodified
+                  FROM {files} f
+                 WHERE component = :component
+                       AND filearea = :filearea
+                       AND contextid = :contextid
+                       $updatedsincesql
+              GROUP BY itemid
+              ORDER BY MAX(f.timemodified) DESC";
+
+        return $DB->get_records_sql($sql, $params, 0, $lastnum);
+    }
+
     /**
      * Returns array based tree structure of area files
      *

lib/tests/filelib_test.php

@@ -1667,6 +1667,65 @@ EOF;
         $draftfiles = $fs->get_area_files($usercontext->id, 'user', 'draft', $file2->get_itemid(), 'itemid', 0);
         $this->assertCount(1, $draftfiles);
     }
+
+    /**
+     * Test file_is_draft_areas_limit_reached
+     */
+    public function test_file_is_draft_areas_limit_reached() {
+        global $CFG;
+        $this->resetAfterTest(true);
+
+        $capacity = $CFG->draft_area_bucket_capacity = 5;
+        $leak = $CFG->draft_area_bucket_leak = 0.2; // Leaks every 5 seconds.
+
+        $generator = $this->getDataGenerator();
+        $user = $generator->create_user();
+
+        $this->setUser($user);
+
+        $itemids = [];
+        for ($i = 0; $i < $capacity; $i++) {
+            $itemids[$i] = file_get_unused_draft_itemid();
+        }
+
+        // This test highly depends on time. We try to make sure that the test starts at the early moments on the second.
+        // This was not needed if MDL-37327 was implemented.
+        $after = time();
+        while (time() === $after) {
+            usleep(100000);
+        }
+
+        // Burst up to the capacity and make sure that the bucket allows it.
+        for ($i = 0; $i < $capacity; $i++) {
+            if ($i) {
+                sleep(1); // A little delay so we have different timemodified value for files.
+            }
+            $this->assertFalse(file_is_draft_areas_limit_reached($user->id));
+            self::create_draft_file([
+                'filename' => 'file1.png',
+                'itemid' => $itemids[$i],
+            ]);
+        }
+
+        // The bucket should be full after bursting.
+        $this->assertTrue(file_is_draft_areas_limit_reached($user->id));
+
+        // The bucket leaks so it shouldn't be full after a certain time.
+        // Reiterating that this test could have been faster if MDL-37327 was implemented.
+        sleep(ceil(1 / $leak) - ($capacity - 1));
+        $this->assertFalse(file_is_draft_areas_limit_reached($user->id));
+
+        // Only one item was leaked from the bucket. So the bucket should become full again if we add a single item to it.
+        self::create_draft_file([
+            'filename' => 'file2.png',
+            'itemid' => $itemids[0],
+        ]);
+        $this->assertTrue(file_is_draft_areas_limit_reached($user->id));
+
+        // The bucket leaks at a constant rate. It doesn't matter if it is filled as the result of bursting or not.
+        sleep(ceil(1 / $leak));
+        $this->assertFalse(file_is_draft_areas_limit_reached($user->id));
+    }
 }
 
 /**

repository/filepicker.php

@@ -334,6 +334,11 @@ case 'download':
                 unlink($thefile['path']);
                 print_error('maxareabytes');
             }
+            // Ensure the user does not upload too many draft files in a short period.
+            if (file_is_draft_areas_limit_reached($USER->id)) {
+                unlink($thefile['path']);
+                print_error('maxdraftitemids');
+            }
             try {
                 $info = repository::move_to_filepool($thefile['path'], $record);
                 redirect($home_url, get_string('downloadsucc', 'repository'));

repository/repository_ajax.php

@@ -309,6 +309,10 @@ switch ($action) {
                 if (file_is_draft_area_limit_reached($itemid, $areamaxbytes, filesize($downloadedfile['path']))) {
                     throw new file_exception('maxareabytes');
                 }
+                // Ensure the user does not upload too many draft files in a short period.
+                if (file_is_draft_areas_limit_reached($USER->id)) {
+                    throw new file_exception('maxdraftitemids');
+                }
 
                 $info = repository::move_to_filepool($downloadedfile['path'], $record);
                 if (empty($info)) {

repository/upload/lib.php

@@ -199,6 +199,10 @@ class repository_upload extends repository {
         if (file_is_draft_area_limit_reached($record->itemid, $areamaxbytes, filesize($_FILES[$elname]['tmp_name']))) {
             throw new file_exception('maxareabytes');
         }
+        // Ensure the user does not upload too many draft files in a short period.
+        if (file_is_draft_areas_limit_reached($USER->id)) {
+            throw new file_exception('maxdraftitemids');
+        }
 
         $record->contextid = $context->id;
         $record->userid    = $USER->id;