MDL-42016 repository: Implement new sync method in repositories

repository/boxnet/lib.php

@@ -267,34 +267,34 @@ class repository_boxnet extends repository {
         return $source;
     }
 
-    /**
+    public function sync_reference(stored_file $file) {
-     * Returns information about file in this repository by reference
+        if ($file->get_referencelastsync() + DAYSECS > time()) {
-     * {@link repository::get_file_reference()}
+            // Synchronise not more often than once a day.
-     * {@link repository::get_file()}
+            return false;
-     *
-     * Returns null if file not found or is not readable
-     *
-     * @param stdClass $reference file reference db record
-     * @return null|stdClass with attribute 'filepath'
-     */
-    public function get_file_by_reference($reference) {
-        $array = explode('/', $reference->reference);
-        $fileid = array_pop($array);
-        $fileinfo = $this->boxclient->get_file_info($fileid, self::SYNCFILE_TIMEOUT);
-        if ($fileinfo) {
-            $size = (int)$fileinfo->size;
-            if (file_extension_in_typegroup($fileinfo->file_name, 'web_image')) {
-                // this is an image - download it to moodle
-                $path = $this->prepare_file('');
-                $c = new curl;
-                $result = $c->download_one($reference->reference, null, array('filepath' => $path, 'timeout' => self::SYNCIMAGE_TIMEOUT));
-                if ($result === true) {
-                    return (object)array('filepath' => $path);
-                }
-            }
-            return (object)array('filesize' => $size);
         }
-        return null;
+        $c = new curl;
+        if (file_extension_in_typegroup($file->get_filename(), 'web_image')) {
+            $path = $this->prepare_file('');
+            $result = $c->download_one($file->get_reference(), null, array('filepath' => $path, 'timeout' => self::SYNCIMAGE_TIMEOUT));
+            $info = $c->get_info();
+            if ($result === true && isset($info['http_code']) && $info['http_code'] == 200) {
+                $fs = get_file_storage();
+                list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($path);
+                $file->set_synchronized($contenthash, $filesize);
+                return true;
+            }
+        }
+        $c->get($file->get_reference(), null, array('timeout' => self::SYNCIMAGE_TIMEOUT, 'followlocation' => true, 'nobody' => true));
+        $info = $c->get_info();
+        if (isset($info['http_code']) && $info['http_code'] == 200 &&
+                array_key_exists('download_content_length', $info) &&
+                $info['download_content_length'] >= 0) {
+            $filesize = (int)$info['download_content_length'];
+            $file->set_synchronized(null, $filesize);
+            return true;
+        }
+        $file->set_missingsource();
+        return true;
     }
 
     /**
@@ -307,14 +307,8 @@ class repository_boxnet extends repository {
      */
     public function get_reference_details($reference, $filestatus = 0) {
         // Indicate it's from box.net repository + secure URL
-        $array = explode('/', $reference);
-        $fileid = array_pop($array);
-        $fileinfo = $this->boxclient->get_file_info($fileid, self::SYNCFILE_TIMEOUT);
-        if (!empty($fileinfo)) {
-            $reference = (string)$fileinfo->file_name;
-        }
         $details = $this->get_name() . ': ' . $reference;
-        if (!empty($fileinfo)) {
+        if (!$filestatus) {
             return $details;
         } else {
             return get_string('lostsource', 'repository', $details);

repository/dropbox/lib.php

@@ -546,25 +546,18 @@ class repository_dropbox extends repository {
         return serialize($reference);
     }
 
-    /**
+    public function sync_reference(stored_file $file) {
-     * Returns information about file in this repository by reference
+        if ($file->get_referencelastsync() + DAYSECS > time()) {
-     * {@link repository::get_file_reference()}
+            // Synchronise not more often than once a day.
-     * {@link repository::get_file()}
+            return false;
-     *
+        }
-     * Returns null if file not found or is not readable
+        $ref = unserialize($file->get_reference());
-     *
-     * @param stdClass $reference file reference db record
-     * @return null|stdClass that has 'filepath' property
-     */
-    public function get_file_by_reference($reference) {
-        global $USER;
-        $ref = unserialize($reference->reference);
         if (!isset($ref->url)) {
             // this is an old-style reference in DB. We need to fix it
-            $ref = unserialize($this->fix_old_style_reference($reference->reference));
+            $ref = unserialize($this->fix_old_style_reference($file->get_reference()));
         }
         if (!isset($ref->url)) {
-            return null;
+            return false;
         }
         $c = new curl;
         $url = $this->get_file_download_link($ref->url);
@@ -574,7 +567,10 @@ class repository_dropbox extends repository {
                 $result = $c->download_one($url, array(), array('filepath' => $saveas, 'timeout' => self::SYNCIMAGE_TIMEOUT, 'followlocation' => true));
                 $info = $c->get_info();
                 if ($result === true && isset($info['http_code']) && $info['http_code'] == 200) {
-                    return (object)array('filepath' => $saveas);
+                    $fs = get_file_storage();
+                    list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($saveas);
+                    $file->set_synchronized($contenthash, $filesize);
+                    return true;
                 }
             } catch (Exception $e) {}
         }
@@ -583,9 +579,12 @@ class repository_dropbox extends repository {
         if (isset($info['http_code']) && $info['http_code'] == 200 &&
                 array_key_exists('download_content_length', $info) &&
                 $info['download_content_length'] >= 0) {
-            return (object)array('filesize' => (int)$info['download_content_length']);
+            $filesize = (int)$info['download_content_length'];
+            $file->set_synchronized(null, $filesize);
+            return true;
         }
-        return null;
+        $file->set_missingsource();
+        return true;
     }
 
     /**
@@ -648,8 +647,7 @@ class repository_dropbox extends repository {
     /**
      * Returns the maximum size of the Dropbox files to cache in moodle
      *
-     * Note that {@link repository_dropbox::get_file_by_reference()} called by
+     * Note that {@link repository_dropbox::sync_reference()} will try to cache images even
-     * {@link repository::sync_external_file()} will try to cache images even
      * when they are bigger in order to generate thumbnails. However there is
      * a small timeout for downloading images for synchronisation and it will
      * probably fail if the image is too big.

repository/equella/lib.php

@@ -134,7 +134,7 @@ class repository_equella extends repository {
      * If we received the connection timeout more than 3 times in a row, we don't attemt to
      * connect to the server any more during this request.
      *
-     * This function is used by {@link repository_equella::get_file_by_reference()} that
+     * This function is used by {@link repository_equella::sync_reference()} that
      * synchronises the file size of referenced files.
      *
      * @param int $errno omit if we just want to know the return value, the last curl_errno otherwise
@@ -159,18 +159,6 @@ class repository_equella extends repository {
         return ($countfailures[$sess] < 3);
     }
 
-    /**
-     * Decide whether or not the file should be synced
-     *
-     * @param stored_file $storedfile
-     * @return bool
-     */
-    public function sync_individual_file(stored_file $storedfile) {
-        // if we had several unsuccessfull attempts to connect to server - do not try any more
-        return $this->connection_result();
-    }
-
-
     /**
      * Download a file, this function can be overridden by subclass. {@link curl}
      *
@@ -202,34 +190,30 @@ class repository_equella extends repository {
         return array('path'=>$path, 'url'=>$url);
     }
 
-    /**
+    public function sync_reference(stored_file $file) {
-     * Returns information about file in this repository by reference
-     *
-     * If the file is an image we download the contents and save it in our filesystem
-     * so we can generate thumbnails. Otherwise we just request the file size.
-     * Returns null if file not found or can not be accessed
-     *
-     * @param stdClass $reference file reference db record
-     * @return stdClass|null contains one of the following:
-     *   - 'filesize' (for non-image files or files we failed to retrieve fully because of timeout)
-     *   - 'filepath' (for image files that we retrieived and saved)
-     */
-    public function get_file_by_reference($reference) {
         global $USER;
-        $ref = @unserialize(base64_decode($reference->reference));
+        if ($file->get_referencelastsync() + DAYSECS > time() || !$this->connection_result()) {
+            // Synchronise not more often than once a day.
+            // if we had several unsuccessfull attempts to connect to server - do not try any more.
+            return false;
+        }
+        $ref = @unserialize(base64_decode($file->get_reference()));
         if (!isset($ref->url) || !($url = $this->appendtoken($ref->url))) {
             // Occurs when the user isn't known..
-            return null;
+            $file->set_missingsource();
+            return true;
         }
 
-        $return = null;
         $cookiepathname = $this->prepare_file($USER->id. '_'. uniqid('', true). '.cookie');
         $c = new curl(array('cookie' => $cookiepathname));
         if (file_extension_in_typegroup($ref->filename, 'web_image')) {
             $path = $this->prepare_file('');
             $result = $c->download_one($url, null, array('filepath' => $path, 'followlocation' => true, 'timeout' => self::SYNCIMAGE_TIMEOUT));
             if ($result === true) {
-                $return = (object)array('filepath' => $path);
+                $fs = get_file_storage();
+                list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($path);
+                $file->set_synchronized($contenthash, $filesize);
+                return true;
             }
         } else {
             $result = $c->head($url, array('followlocation' => true, 'timeout' => self::SYNCFILE_TIMEOUT));
@@ -241,13 +225,15 @@ class repository_equella extends repository {
 
         $this->connection_result($c->get_errno());
         $curlinfo = $c->get_info();
-        if ($return === null && isset($curlinfo['http_code']) && $curlinfo['http_code'] == 200
+        if (isset($curlinfo['http_code']) && $curlinfo['http_code'] == 200
                 && array_key_exists('download_content_length', $curlinfo)
                 && $curlinfo['download_content_length'] >= 0) {
             // we received a correct header and at least can tell the file size
-            $return = (object)array('filesize' => $curlinfo['download_content_length']);
+            $file->set_synchronized(null, $curlinfo['download_content_length']);
+            return true;
         }
-        return $return;
+        $file->set_missingsource();
+        return true;
     }
 
     /**

repository/filesystem/lib.php

@@ -265,17 +265,6 @@ class repository_filesystem extends repository {
         return FILE_INTERNAL | FILE_REFERENCE;
     }
 
-    /**
-     * Return reference file life time
-     *
-     * @param string $ref
-     * @return int
-     */
-    public function get_reference_file_lifetime($ref) {
-        // Does not cost us much to synchronise within our own filesystem, set to 1 minute
-        return 60;
-    }
-
     /**
      * Return human readable reference information
      *
@@ -292,37 +281,40 @@ class repository_filesystem extends repository {
         }
     }
 
-    /**
+    public function sync_reference(stored_file $file) {
-     * Returns information about file in this repository by reference
+        if ($file->get_referencelastsync() + 60 > time()) {
-     *
+            // Does not cost us much to synchronise within our own filesystem, check every 1 minute.
-     * Returns null if file not found or is not readable
+            return false;
-     *
-     * @param stdClass $reference file reference db record
-     * @return stdClass|null contains one of the following:
-     *   - 'filesize' if file should not be copied to moodle filepool
-     *   - 'filepath' if file should be copied to moodle filepool
-     */
-    public function get_file_by_reference($reference) {
-        $ref = $reference->reference;
-        if ($ref{0} == '/') {
-            $filepath = $this->root_path.substr($ref, 1, strlen($ref)-1);
-        } else {
-            $filepath = $this->root_path.$ref;
         }
+        static $issyncing = false;
+        if ($issyncing) {
+            // Avoid infinite recursion when calling $file->get_filesize() and get_contenthash().
+            return;
+        }
+        $filepath = $this->root_path.ltrim($file->get_reference(), '/');
         if (file_exists($filepath) && is_readable($filepath)) {
+            $fs = get_file_storage();
+            $issyncing = true;
             if (file_extension_in_typegroup($filepath, 'web_image')) {
-                // return path to image files so it will be copied into moodle filepool
+                $contenthash = sha1_file($filepath);
-                // we need the file in filepool to generate an image thumbnail
+                if ($file->get_contenthash() == $contenthash) {
-                return (object)array('filepath' => $filepath);
+                    // File did not change since the last synchronisation.
+                    $filesize = filesize($filepath);
+                } else {
+                    // Copy file into moodle filepool (used to generate an image thumbnail).
+                    list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($filepath);
+                }
             } else {
-                // return just the file size so file will NOT be copied into moodle filepool
+                // Update only file size so file will NOT be copied into moodle filepool.
-                return (object)array(
+                $contenthash = null;
-                    'filesize' => filesize($filepath)
+                $filesize = filesize($filepath);
-                );
             }
+            $issyncing = false;
+            $file->set_synchronized($contenthash, $filesize);
         } else {
-            return null;
+            $file->set_missingsource();
         }
+        return true;
     }
 
     /**