Merge branch 'wip-MDL-33550-master' of git://github.com/marinaglancy/moodle

lang/en/repository.php

@@ -138,6 +138,7 @@ $string['listview'] = 'View as list';
 $string['loading'] = 'Loading...';
 $string['login'] = 'Login';
 $string['logout'] = 'Logout';
+$string['lostsource'] = 'Error. Source is missing. {$a}';
 $string['makefileinternal'] = 'Make a copy of the file';
 $string['makefilelink'] = 'Link to the file directly';
 $string['makefilereference'] = 'Create an alias/shortcut to the file';
@@ -168,6 +169,7 @@ $string['pluginerror'] = 'Errors in repository plugin.';
 $string['popup'] = 'Click "Login" button to login';
 $string['popupblockeddownload'] = 'The downloading window is blocked, please allow the popup window, and try again.';
 $string['preview'] = 'Preview';
+$string['privatefilesof'] = '{$a} Private files';
 $string['readonlyinstance'] = 'You cannot edit/delete a read-only instance';
 $string['referencesexist'] = 'There are {$a} alias/shortcut files that use this file as their source';
 $string['referenceslist'] = 'Aliases/Shortcuts';
@@ -202,6 +204,7 @@ $string['unknownoriginal'] = 'Unknown';
 $string['upload'] = 'Upload this file';
 $string['uploading'] = 'Uploading...';
 $string['uploadsucc'] = 'The file has been uploaded successfully';
+$string['undisclosedsource'] = '(Undisclosed)';
 $string['undisclosedreference'] = '(Undisclosed)';
 $string['uselatestfile'] = 'Use latest file';
 $string['usercontextrepositorydisabled'] = 'You cannot edit this repository in user context';

lib/filebrowser/file_info.php

@@ -227,17 +227,13 @@ abstract class file_info {
     }
 
     /**
-     * Returns the localised human-readable name of the file together with
+     * Returns the localised human-readable name of the file together with virtual path
-     * virtual path
      *
+     * @see file_info_stored::get_readable_fullname()
      * @return string
      */
     public function get_readable_fullname() {
-        $fpath = array();
+        return null;
-        for ($parent = $this; $parent; $parent = $parent->get_parent()) {
-            array_unshift($fpath, $parent->get_visible_name());
-        }
-        return join('/', $fpath);
     }
 
     /**

lib/filebrowser/file_info_stored.php

@@ -113,6 +113,46 @@ class file_info_stored extends file_info {
         }
     }
 
+    /**
+     * Returns the localised human-readable name of the file together with virtual path
+     *
+     * @return string
+     */
+    public function get_readable_fullname() {
+        global $CFG;
+        // retrieve the readable path with all parents (excluding the top most 'System')
+        $fpath = array();
+        for ($parent = $this; $parent && $parent->get_parent(); $parent = $parent->get_parent()) {
+            array_unshift($fpath, $parent->get_visible_name());
+        }
+
+        if ($this->lf->get_component() == 'user' && $this->lf->get_filearea() == 'private') {
+            // use the special syntax for user private files - 'USERNAME Private files: PATH'
+            $username = array_shift($fpath);
+            array_shift($fpath); // get rid of "Private Files/" in the beginning of the path
+            return get_string('privatefilesof', 'repository', $username). ': '. join('/', $fpath);
+        } else {
+            // for all other files (except user private files) return 'Server files: PATH'
+
+            // first, get and cache the name of the repository_local (will be used as prefix for file names):
+            static $replocalname = null;
+            if ($replocalname === null) {
+                require_once($CFG->dirroot . "/repository/lib.php");
+                $instances = repository::get_instances(array('type' => 'local'));
+                if (count($instances)) {
+                    $firstinstance = reset($instances);
+                    $replocalname = $firstinstance->get_name();
+                } else if (get_string_manager()->string_exists('pluginname', 'repository_local')) {
+                    $replocalname = get_string('pluginname', 'repository_local');
+                } else {
+                    $replocalname = get_string('arearoot', 'repository');
+                }
+            }
+
+            return $replocalname. ': '. join('/', $fpath);
+        }
+    }
+
     /**
      * Returns file download url
      *

lib/filelib.php

@@ -596,6 +596,9 @@ function file_get_drafarea_files($draftitemid, $filepath = '/') {
             $item->datemodified = $file->get_timemodified();
             $item->datecreated = $file->get_timecreated();
             $item->isref = $file->is_external_file();
+            if ($item->isref && $file->get_status() == 666) {
+                $item->originalmissing = true;
+            }
             // find the file this draft file was created from and count all references in local
             // system pointing to that file
             $source = unserialize($file->get_source());
@@ -2310,7 +2313,7 @@ function send_stored_file($stored_file, $lifetime=86400 , $filter=0, $forcedownl
     }
 
     // handle external resource
-    if ($stored_file->is_external_file()) {
+    if ($stored_file && $stored_file->is_external_file()) {
         $stored_file->send_file($lifetime, $filter, $forcedownload, $options);
         die;
     }

lib/filestorage/file_storage.php

@@ -1033,6 +1033,7 @@ class file_storage {
         $newrecord->source       = empty($filerecord->source) ? null : $filerecord->source;
         $newrecord->author       = empty($filerecord->author) ? null : $filerecord->author;
         $newrecord->license      = empty($filerecord->license) ? null : $filerecord->license;
+        $newrecord->status       = empty($filerecord->status) ? 0 : $filerecord->status;
         $newrecord->sortorder    = $filerecord->sortorder;
 
         list($newrecord->contenthash, $newrecord->filesize, $newfile) = $this->add_file_to_pool($pathname);
@@ -1149,6 +1150,7 @@ class file_storage {
         $newrecord->source       = empty($filerecord->source) ? null : $filerecord->source;
         $newrecord->author       = empty($filerecord->author) ? null : $filerecord->author;
         $newrecord->license      = empty($filerecord->license) ? null : $filerecord->license;
+        $newrecord->status       = empty($filerecord->status) ? 0 : $filerecord->status;
         $newrecord->sortorder    = $filerecord->sortorder;
 
         list($newrecord->contenthash, $newrecord->filesize, $newfile) = $this->add_string_to_pool($content);
@@ -1223,6 +1225,7 @@ class file_storage {
         $filerecord->source            = empty($filerecord->source) ? null : $filerecord->source;
         $filerecord->author            = empty($filerecord->author) ? null : $filerecord->author;
         $filerecord->license           = empty($filerecord->license) ? null : $filerecord->license;
+        $filerecord->status            = empty($filerecord->status) ? 0 : $filerecord->status;
         $filerecord->filepath          = clean_param($filerecord->filepath, PARAM_PATH);
         if (strpos($filerecord->filepath, '/') !== 0 or strrpos($filerecord->filepath, '/') !== strlen($filerecord->filepath)-1) {
             // Path must start and end with '/'.
@@ -1652,10 +1655,22 @@ class file_storage {
      * Unpack reference field
      *
      * @param string $str
+     * @param bool $cleanparams if set to true, array elements will be passed through {@link clean_param()}
      * @return array
      */
-    public static function unpack_reference($str) {
+    public static function unpack_reference($str, $cleanparams = false) {
-        return unserialize(base64_decode($str));
+        $params = unserialize(base64_decode($str));
+        if (is_array($params) && $cleanparams) {
+            $params = array(
+                'component' => is_null($params['component']) ? ''   : clean_param($params['component'], PARAM_COMPONENT),
+                'filearea'  => is_null($params['filearea'])  ? ''   : clean_param($params['filearea'], PARAM_AREA),
+                'itemid'    => is_null($params['itemid'])    ? 0    : clean_param($params['itemid'], PARAM_INT),
+                'filename'  => is_null($params['filename'])  ? null : clean_param($params['filename'], PARAM_FILE),
+                'filepath'  => is_null($params['filepath'])  ? null : clean_param($params['filepath'], PARAM_PATH),
+                'contextid' => is_null($params['contextid']) ? null : clean_param($params['contextid'], PARAM_INT)
+            );
+        }
+        return $params;
     }
 
     /**

lib/filestorage/stored_file.php

@@ -522,29 +522,16 @@ class stored_file {
     }
 
     /**
-     * Sync external files
+     * Synchronize file if it is a reference and needs synchronizing
      *
-     * @return bool true if file content changed, false if not
+     * Updates contenthash and filesize
      */
     public function sync_external_file() {
-        global $CFG, $DB;
+        global $CFG;
-        if (empty($this->file_record->referencefileid)) {
+        if (!empty($this->file_record->referencefileid)) {
-            return false;
-        }
-        if (empty($this->file_record->referencelastsync) or ($this->file_record->referencelastsync + $this->file_record->referencelifetime < time())) {
             require_once($CFG->dirroot.'/repository/lib.php');
-            if (repository::sync_external_file($this)) {
+            repository::sync_external_file($this);
-                $prevcontent = $this->file_record->contenthash;
-                $sql = "SELECT f.*, r.repositoryid, r.reference
-                          FROM {files} f
-                     LEFT JOIN {files_reference} r
-                               ON f.referencefileid = r.id
-                         WHERE f.id = ?";
-                $this->file_record = $DB->get_record_sql($sql, array($this->file_record->id), MUST_EXIST);
-                return ($prevcontent !== $this->file_record->contenthash);
-            }
         }
-        return false;
     }
 
     /**
@@ -858,7 +845,45 @@ class stored_file {
      * @return string
      */
     public function get_reference_details() {
-        return $this->repository->get_reference_details($this->get_reference());
+        return $this->repository->get_reference_details($this->get_reference(), $this->get_status());
+    }
+
+    /**
+     * Called after reference-file has been synchronized with the repository
+     *
+     * We update contenthash, filesize and status in files table if changed
+     * and we always update lastsync in files_reference table
+     *
+     * @param type $contenthash
+     * @param type $filesize
+     */
+    public function set_synchronized($contenthash, $filesize, $status = 0) {
+        global $DB;
+        if (!$this->is_external_file()) {
+            return;
+        }
+        $now = time();
+        $filerecord = new stdClass();
+        if ($this->get_contenthash() !== $contenthash) {
+            $filerecord->contenthash = $contenthash;
+        }
+        if ($this->get_filesize() != $filesize) {
+            $filerecord->filesize = $filesize;
+        }
+        if ($this->get_status() != $status) {
+            $filerecord->status = $status;
+        }
+        $filerecord->referencelastsync = $now; // TODO MDL-33416 remove this
+        if (!empty($filerecord)) {
+            $this->update($filerecord);
+        }
+
+        $DB->set_field('files_reference', 'lastsync', $now, array('id'=>$this->get_referencefileid()));
+        // $this->file_record->lastsync = $now; // TODO MDL-33416 uncomment or remove
+    }
+
+    public function set_missingsource() {
+        $this->set_synchronized($this->get_contenthash(), 0, 666);
     }
 
     /**

lib/filestorage/tests/file_storage_test.php

@@ -159,9 +159,9 @@ class filestoragelib_testcase extends advanced_testcase {
         // create user
         $generator = $this->getDataGenerator();
         $user = $generator->create_user();
+        $this->setUser($user);
         $usercontext = context_user::instance($user->id);
         $syscontext = context_system::instance();
-        $USER = $DB->get_record('user', array('id'=>$user->id));
 
         $fs = get_file_storage();
 

lib/form/filemanager.js

@@ -527,6 +527,9 @@ M.form_filemanager.init = function(Y, options) {
                     if (node.refcount) {
                         classname = classname + ' fp-hasreferences';
                     }
+                    if (node.originalmissing) {
+                        classname = classname + ' fp-originalmissing';
+                    }
                     if (node.sortorder == 1) { classname = classname + ' fp-mainfile';}
                     return Y.Lang.trim(classname);
                 }

lib/phpunit/classes/util.php

@@ -612,8 +612,11 @@ class phpunit_util {
         get_string_manager()->reset_caches();
         events_get_handlers('reset');
         textlib::reset_caches();
+        if (class_exists('repository')) {
+            repository::reset_caches();
+        }
         $GROUPLIB_CACHE = null;
-        //TODO: add more resets here and probably refactor them to new core function
+        //TODO MDL-25290: add more resets here and probably refactor them to new core function
 
         // purge dataroot directory
         self::reset_dataroot();

mod/resource/locallib.php

@@ -295,23 +295,27 @@ function resource_get_optional_details($resource, $cm) {
         $context = context_module::instance($cm->id);
         $size = '';
         $type = '';
-        if (!empty($options['showsize'])) {
+        $fs = get_file_storage();
-            $size = display_size($DB->get_field_sql(
+        $files = $fs->get_area_files($context->id, 'mod_resource', 'content', 0, 'sortorder DESC, id ASC', false);
-                    'SELECT SUM(filesize) FROM {files} WHERE contextid=?', array($context->id)));
+        if (!empty($options['showsize']) && count($files)) {
+            $sizebytes = 0;
+            foreach ($files as $file) {
+                // this will also synchronize the file size for external files if needed
+                $sizebytes += $file->get_filesize();
+            }
+            if ($sizebytes) {
+                $size = display_size($sizebytes);
+            }
         }
-        if (!empty($options['showtype'])) {
+        if (!empty($options['showtype']) && count($files)) {
             // For a typical file resource, the sortorder is 1 for the main file
             // and 0 for all other files. This sort approach is used just in case
             // there are situations where the file has a different sort order
-            $record = $DB->get_record_sql(
+            $mainfile = reset($files);
-                    'SELECT filename, mimetype FROM {files} WHERE contextid=? ORDER BY sortorder DESC',
+            $type = get_mimetype_description($mainfile);
-                    array($context->id), IGNORE_MULTIPLE);
             // Only show type if it is not unknown
-            if ($record) {
+            if ($type === get_mimetype_description('document/unknown')) {
-                $type = get_mimetype_description($record);
+                $type = '';
-                if ($type === get_mimetype_description('document/unknown')) {
-                    $type = '';
-                }
             }
         }
 

repository/boxnet/lib.php

@@ -267,18 +267,24 @@ class repository_boxnet extends repository {
     }
 
     /**
-     * Get file from external repository by reference
+     * Returns information about file in this repository by reference
      * {@link repository::get_file_reference()}
      * {@link repository::get_file()}
      *
+     * Returns null if file not found or is not readable
+     *
      * @param stdClass $reference file reference db record
-     * @return stdClass|null|false
+     * @return null|stdClass with attribute 'filepath'
      */
     public function get_file_by_reference($reference) {
-        $fileinfo = new stdClass;
         $boxnetfile = $this->get_file($reference->reference);
-        $fileinfo->filepath = $boxnetfile['path'];
+        // Please note that here we will ALWAYS receive a file
-        return $fileinfo;
+        // If source file has been removed from external server, box.com still returns
+        // a plain/text file with content 'no such file' (filesize will be 12 bytes)
+        if (!empty($boxnetfile['path'])) {
+            return (object)array('filepath' => $boxnetfile['path']);
+        }
+        return null;
     }
 
     /**
@@ -286,17 +292,26 @@ class repository_boxnet extends repository {
      * {@link stored_file::get_reference()}
      *
      * @param string $reference
-     * @return string|null
+     * @param int $filestatus status of the file, 0 - ok, 666 - source missing
+     * @return string
      */
-    public function get_reference_details($reference) {
+    public function get_reference_details($reference, $filestatus = 0) {
         // Indicate it's from box.net repository + secure URL
-        return $this->get_name() . ': ' . $reference;
+        $details = $this->get_name() . ': ' . $reference;
+        if (!$filestatus) {
+            return $details;
+        } else {
+            // at the moment for box.net files we never can be sure that source is missing
+            // because box.com never returns 404 error.
+            // So we never change the status and actually this part is unreachable
+            return get_string('lostsource', 'repository', $details);
+        }
     }
 
     /**
-     * Repository method to serve file
+     * Repository method to serve the referenced file
      *
-     * @param stored_file $storedfile
+     * @param stored_file $storedfile the file that contains the reference
      * @param int $lifetime Number of seconds before the file should expire from caches (default 24 hours)
      * @param int $filter 0 (default)=no filtering, 1=all files, 2=html files only
      * @param bool $forcedownload If true (default false), forces download of file rather than view in browser/plugin
@@ -304,7 +319,8 @@ class repository_boxnet extends repository {
      */
     public function send_file($storedfile, $lifetime=86400 , $filter=0, $forcedownload=false, array $options = null) {
         $ref = $storedfile->get_reference();
-        // Let box.net serve the file.
+        // Let box.net serve the file. It will return 'no such file' content if file not found
+        // also if file has the different name than alias, it will be returned with the box.net filename
         header('Location: ' . $ref);
     }
 }

repository/coursefiles/lib.php

@@ -202,83 +202,6 @@ class repository_coursefiles extends repository {
         return true;
     }
 
-    /**
-     * Unpack file info and pack it, mainly for data validation
-     *
-     * @param string $source
-     * @return string file referece
-     */
-    public function get_file_reference($source) {
-        $params = unserialize(base64_decode($source));
-
-        if (!is_array($params)) {
-            throw new repository_exception('invalidparams', 'repository');
-        }
-
-        $filename  = is_null($params['filename'])  ? null : clean_param($params['filename'], PARAM_FILE);
-        $filepath  = is_null($params['filepath'])  ? null : clean_param($params['filepath'], PARAM_PATH);;
-        $contextid = is_null($params['contextid']) ? null : clean_param($params['contextid'], PARAM_INT);
-
-        $reference = array();
-        // hard coded filearea, component and itemid for security
-        $reference['component'] = 'course';
-        $reference['filearea']  = 'legacy';
-        $reference['itemid']    = 0;
-        $reference['contextid'] = $contextid;
-        $reference['filepath']  = $filepath;
-        $reference['filename']  = $filename;
-
-        return file_storage::pack_reference($reference);
-    }
-
-    /**
-     * Get file from external repository by reference
-     * {@link repository::get_file_reference()}
-     * {@link repository::get_file()}
-     *
-     * @param stdClass $reference file reference db record
-     * @return stdClass|null|false
-     */
-    public function get_file_by_reference($reference) {
-        $fs = get_file_storage();
-        $ref = $reference->reference;
-        $params = file_storage::unpack_reference($ref);
-        if (!is_array($params)) {
-            throw new repository_exception('invalidparams', 'repository');
-        }
-        $filename  = is_null($params['filename'])  ? null : clean_param($params['filename'], PARAM_FILE);
-        $filepath  = is_null($params['filepath'])  ? null : clean_param($params['filepath'], PARAM_PATH);;
-        $contextid = is_null($params['contextid']) ? null : clean_param($params['contextid'], PARAM_INT);
-
-        // hard coded filearea, component and itemid for security
-        $storedfile = $fs->get_file($contextid, 'course', 'legacy', 0, $filepath, $filename);
-
-        $fileinfo = new stdClass;
-        $fileinfo->contenthash = $storedfile->get_contenthash();
-        $fileinfo->filesize    = $storedfile->get_filesize();
-        return $fileinfo;
-    }
-
-    /**
-     * Return human readable reference information
-     * {@link stored_file::get_reference()}
-     *
-     * @param string $reference
-     * @return string|null
-     */
-    public function get_reference_details($reference) {
-        $params = file_storage::unpack_reference($reference);
-        list($context, $course, $cm) = get_context_info_array($params['contextid']);
-        $coursename = '';
-        if (!empty($course)) {
-            $coursename = '"' . format_string($course->shortname, true, array('context' => get_course_context($context))) . '" ' . get_string('courselegacyfiles');
-        } else {
-            $coursename = get_string('courselegacyfiles');
-        }
-        // Indicate this is from user private area
-        return $coursename . ': ' . $params['filepath'] . $params['filename'];
-    }
-
     /**
      * Return reference file life time
      *
@@ -289,29 +212,4 @@ class repository_coursefiles extends repository {
         // this should be realtime
         return 0;
     }
-
-    /**
-     * Repository method to serve file
-     *
-     * @param stored_file $storedfile
-     * @param int $lifetime Number of seconds before the file should expire from caches (default 24 hours)
-     * @param int $filter 0 (default)=no filtering, 1=all files, 2=html files only
-     * @param bool $forcedownload If true (default false), forces download of file rather than view in browser/plugin
-     * @param array $options additional options affecting the file serving
-     */
-    public function send_file($storedfile, $lifetime=86400 , $filter=0, $forcedownload=false, array $options = null) {
-        $fs = get_file_storage();
-
-        $reference = $storedfile->get_reference();
-        $params = file_storage::unpack_reference($reference);
-
-        $filename  = is_null($params['filename'])  ? null : clean_param($params['filename'], PARAM_FILE);
-        $filepath  = is_null($params['filepath'])  ? null : clean_param($params['filepath'], PARAM_PATH);;
-        $contextid = is_null($params['contextid']) ? null : clean_param($params['contextid'], PARAM_INT);
-
-        // hard coded file area and component for security
-        $srcfile = $fs->get_file($contextid, 'course', 'legacy', 0, $filepath, $filename);
-
-        send_stored_file($srcfile, $lifetime, $filter, $forcedownload, $options);
-    }
 }

repository/dropbox/lib.php

@@ -362,12 +362,14 @@ class repository_dropbox extends repository {
     }
 
     /**
-     * Get file from external repository by reference
+     * Returns information about file in this repository by reference
      * {@link repository::get_file_reference()}
      * {@link repository::get_file()}
      *
+     * Returns null if file not found or is not readable
+     *
      * @param stdClass $reference file reference db record
-     * @return stdClass|null|false
+     * @return null|stdClass that has 'filepath' property
      */
     public function get_file_by_reference($reference) {
         $reference  = unserialize($reference->reference);
@@ -379,11 +381,11 @@ class repository_dropbox extends repository {
             $path = $this->get_file($reference->path);
             $cachedfilepath = cache_file::create_from_file($reference, $path['path']);
         }
-
+        if ($cachedfilepath && is_readable($cachedfilepath)) {
-        $fileinfo = new stdClass;
+            return (object)array('filepath' => $cachedfilepath);
-        $fileinfo->filepath = $cachedfilepath;
+        } else {
-
+            return null;
-        return $fileinfo;
+        }
     }
 
     /**
@@ -406,36 +408,48 @@ class repository_dropbox extends repository {
      * {@link stored_file::get_reference()}
      *
      * @param string $reference
-     * @return string|null
+     * @param int $filestatus status of the file, 0 - ok, 666 - source missing
+     * @return string
      */
-    public function get_reference_details($reference) {
+    public function get_reference_details($reference, $filestatus = 0) {
         $ref  = unserialize($reference);
-        // Indicate this is from dropbox with path
+        $details = $this->get_name();
-        return $this->get_name() . ': ' . $ref->path;
+        if (isset($ref->path)) {
+            $details .=  ': '. $ref->path;
+        }
+        if (isset($ref->path) && !$filestatus) {
+            // Indicate this is from dropbox with path
+            return $details;
+        } else {
+            return get_string('lostsource', 'repository', $details);
+        }
     }
 
     /**
-     * Repository method to serve file
+     * Repository method to serve the referenced file
      *
-     * @param stored_file $storedfile
+     * This method is ivoked from {@link send_stored_file()}.
+     * Dropbox repository first caches the file by reading it into temporary folder and then
+     * serves from there.
+     *
+     * @param stored_file $storedfile the file that contains the reference
      * @param int $lifetime Number of seconds before the file should expire from caches (default 24 hours)
      * @param int $filter 0 (default)=no filtering, 1=all files, 2=html files only
      * @param bool $forcedownload If true (default false), forces download of file rather than view in browser/plugin
      * @param array $options additional options affecting the file serving
      */
     public function send_file($storedfile, $lifetime=86400 , $filter=0, $forcedownload=false, array $options = null) {
-        $reference = unserialize($storedfile->get_reference());
+        $fileinfo = $this->get_file_by_reference((object)array('reference' => $storedfile->get_reference()));
-
+        if ($fileinfo && !empty($fileinfo->filepath) && is_readable($fileinfo->filepath)) {
-        $cachedfilepath = cache_file::get($reference, array('ttl' => $this->cachedfilettl));
+            $filename = $storedfile->get_filename();
-        if ($cachedfilepath === false) {
+            if ($options && isset($options['filename'])) {
-            // Cache the file.
+                $filename = $options['filename'];
-            $this->set_access_key($reference->access_key);
+            }
-            $this->set_access_secret($reference->access_secret);
+            $dontdie = ($options && isset($options['dontdie']));
-            $path = $this->get_file($reference->path);
+            send_file($fileinfo->filepath, $filename, $lifetime , $filter, false, $forcedownload, '', $dontdie);
-            $cachedfilepath = cache_file::create_from_file($reference, $path['path']);
+        } else {
+            send_file_not_found();
         }
-
-        send_file($cachedfilepath, $storedfile->get_filename(), 'default' , $filter, false, $forcedownload);
     }
 
     public function cron() {

repository/equella/lib.php

@@ -146,12 +146,14 @@ class repository_equella extends repository {
     }
 
     /**
-     * Get file from external repository by reference
+     * Returns information about file in this repository by reference
      * {@link repository::get_file_reference()}
      * {@link repository::get_file()}
      *
+     * Returns null if file not found or can not be accessed
+     *
      * @param stdClass $reference file reference db record
-     * @return stdClass|null|false
+     * @return null|stdClass containing attribute 'filepath'
      */
     public function get_file_by_reference($reference) {
         $ref = base64_decode($reference->reference);
@@ -159,7 +161,7 @@ class repository_equella extends repository {
 
         if (!$url) {
             // Occurs when the user isn't known..
-            return false;
+            return null;
         }
 
         // We use this cache to get the correct file size.
@@ -170,24 +172,29 @@ class repository_equella extends repository {
             $cachedfilepath = cache_file::create_from_file($url, $path['path']);
         }
 
-        $fileinfo = new stdClass;
+        if ($cachedfilepath && is_readable($cachedfilepath)) {
-        $fileinfo->filepath = $cachedfilepath;
+            return (object)array('filepath' => $cachedfilepath);
-
+        }
-        return $fileinfo;
+        return null;
     }
 
     /**
-     * Send equella file to browser
+     * Repository method to serve the referenced file
      *
-     * @param stored_file $stored_file
+     * @param stored_file $storedfile the file that contains the reference
+     * @param int $lifetime Number of seconds before the file should expire from caches (default 24 hours)
+     * @param int $filter 0 (default)=no filtering, 1=all files, 2=html files only
+     * @param bool $forcedownload If true (default false), forces download of file rather than view in browser/plugin
+     * @param array $options additional options affecting the file serving
      */
     public function send_file($stored_file, $lifetime=86400 , $filter=0, $forcedownload=false, array $options = null) {
         $reference = base64_decode($stored_file->get_reference());
         $url = $this->appendtoken($reference);
         if ($url) {
             header('Location: ' . $url);
+        } else {
+            send_file_not_found();
         }
-        die;
     }
 
     /**

repository/filesystem/lib.php

@@ -223,12 +223,18 @@ class repository_filesystem extends repository {
     }
 
     /**
-     * Get file from external repository by reference
+     * Returns information about file in this repository by reference
      * {@link repository::get_file_reference()}
      * {@link repository::get_file()}
      *
+     * Returns null if file not found or is not readable
+     *
      * @param stdClass $reference file reference db record
-     * @return stdClass|null|false
+     * @return stdClass|null contains one of the following:
+     *   - 'contenthash' and 'filesize'
+     *   - 'filepath'
+     *   - 'handle'
+     *   - 'content'
      */
     public function get_file_by_reference($reference) {
         $ref = $reference->reference;
@@ -237,15 +243,19 @@ class repository_filesystem extends repository {
         } else {
             $filepath = $this->root_path.$ref;
         }
-        $fileinfo = new stdClass;
+        if (file_exists($filepath) && is_readable($filepath)) {
-        $fileinfo->filepath = $filepath;
+            return (object)array('filepath' => $filepath);
-        return $fileinfo;
+        } else {
+            return null;
+        }
     }
 
     /**
-     * Repository method to serve file
+     * Repository method to serve the referenced file
      *
-     * @param stored_file $storedfile
+     * @see send_stored_file
+     *
+     * @param stored_file $storedfile the file that contains the reference
      * @param int $lifetime Number of seconds before the file should expire from caches (default 24 hours)
      * @param int $filter 0 (default)=no filtering, 1=all files, 2=html files only
      * @param bool $forcedownload If true (default false), forces download of file rather than view in browser/plugin
@@ -258,6 +268,15 @@ class repository_filesystem extends repository {
         } else {
             $file = $this->root_path.$reference;
         }
-        send_file($file, $storedfile->get_filename(), 'default' , $filter, false, $forcedownload);
+        if (is_readable($file)) {
+            $filename = $storedfile->get_filename();
+            if ($options && isset($options['filename'])) {
+                $filename = $options['filename'];
+            }
+            $dontdie = ($options && isset($options['dontdie']));
+            send_file($file, $filename, $lifetime , $filter, false, $forcedownload, '', $dontdie);
+        } else {
+            send_file_not_found();
+        }
     }
 }

repository/lib.php

@@ -1090,16 +1090,39 @@ abstract class repository {
     }
 
     /**
-     * Repository method to serve file
+     * Repository method to serve the referenced file
      *
-     * @param stored_file $storedfile
+     * @see send_stored_file
+     *
+     * @param stored_file $storedfile the file that contains the reference
      * @param int $lifetime Number of seconds before the file should expire from caches (default 24 hours)
      * @param int $filter 0 (default)=no filtering, 1=all files, 2=html files only
      * @param bool $forcedownload If true (default false), forces download of file rather than view in browser/plugin
      * @param array $options additional options affecting the file serving
      */
     public function send_file($storedfile, $lifetime=86400 , $filter=0, $forcedownload=false, array $options = null) {
-        throw new coding_exception("Repository plugin must implement send_file() method.");
+        if ($this->has_moodle_files()) {
+            $fs = get_file_storage();
+            $params = file_storage::unpack_reference($storedfile->get_reference(), true);
+            $srcfile = null;
+            if (is_array($params)) {
+                $srcfile = $fs->get_file($params['contextid'], $params['component'], $params['filearea'],
+                        $params['itemid'], $params['filepath'], $params['filename']);
+            }
+            if (empty($options)) {
+                $options = array();
+            }
+            if (!isset($options['filename'])) {
+                $options['filename'] = $storedfile->get_filename();
+            }
+            if (!$srcfile) {
+                send_file_not_found();
+            } else {
+                send_stored_file($srcfile, $lifetime, $filter, $forcedownload, $options);
+            }
+        } else {
+            throw new coding_exception("Repository plugin must implement send_file() method.");
+        }
     }
 
     /**
@@ -1128,10 +1151,35 @@ abstract class repository {
      * {@link stored_file::get_reference()}
      *
      * @param string $reference
-     * @return string|null
+     * @param int $filestatus status of the file, 0 - ok, 666 - source missing
+     * @return string
      */
-    public function get_reference_details($reference) {
+    public function get_reference_details($reference, $filestatus = 0) {
-        return null;
+        if ($this->has_moodle_files()) {
+            $fileinfo = null;
+            $params = file_storage::unpack_reference($reference, true);
+            if (is_array($params)) {
+                $context = get_context_instance_by_id($params['contextid']);
+                if ($context) {
+                    $browser = get_file_browser();
+                    $fileinfo = $browser->get_file_info($context, $params['component'], $params['filearea'], $params['itemid'], $params['filepath'], $params['filename']);
+                }
+            }
+            if (empty($fileinfo)) {
+                if ($filestatus == 666) {
+                    if (is_siteadmin() || ($context && has_capability('moodle/course:managefiles', $context))) {
+                        return get_string('lostsource', 'repository',
+                                $params['contextid']. '/'. $params['component']. '/'. $params['filearea']. '/'. $params['itemid']. $params['filepath']. $params['filename']);
+                    } else {
+                        return get_string('lostsource', 'repository', '');
+                    }
+                }
+                return get_string('undisclosedsource', 'repository');
+            } else {
+                return $fileinfo->get_readable_fullname();
+            }
+        }
+        return '';
     }
 
     /**
@@ -1148,14 +1196,33 @@ abstract class repository {
     }
 
     /**
-     * Get file from external repository by reference
+     * Returns information about file in this repository by reference
      * {@link repository::get_file_reference()}
      * {@link repository::get_file()}
      *
+     * Returns null if file not found or is not readable
+     *
      * @param stdClass $reference file reference db record
-     * @return stdClass|null|false
+     * @return stdClass|null contains one of the following:
+     *   - 'contenthash' and 'filesize'
+     *   - 'filepath'
+     *   - 'handle'
+     *   - 'content'
      */
     public function get_file_by_reference($reference) {
+        if ($this->has_moodle_files() && isset($reference->reference)) {
+            $fs = get_file_storage();
+            $params = file_storage::unpack_reference($reference->reference, true);
+            if (!is_array($params) || !($storedfile = $fs->get_file($params['contextid'],
+                    $params['component'], $params['filearea'], $params['itemid'], $params['filepath'],
+                    $params['filename']))) {
+                return null;
+            }
+            return (object)array(
+                'contenthash' => $storedfile->get_contenthash(),
+                'filesize'    => $storedfile->get_filesize()
+            );
+        }
         return null;
     }
 
@@ -1433,6 +1500,13 @@ abstract class repository {
      * @return string file referece
      */
     public function get_file_reference($source) {
+        if ($this->has_moodle_files() && ($this->supported_returntypes() & FILE_REFERENCE)) {
+            $params = file_storage::unpack_reference($source);
+            if (!is_array($params)) {
+                throw new repository_exception('invalidparams', 'repository');
+            }
+            return file_storage::pack_reference($params);
+        }
         return $source;
     }
     /**
@@ -1484,16 +1558,22 @@ abstract class repository {
      *
      * @param string $url the url of file
      * @param string $filename save location
-     * @return string the location of the file
+     * @return array with elements:
+     *   path: internal location of the file
+     *   url: URL to the source (from parameters)
      */
     public function get_file($url, $filename = '') {
         global $CFG;
         $path = $this->prepare_file($filename);
         $fp = fopen($path, 'w');
         $c = new curl;
-        $c->download(array(array('url'=>$url, 'file'=>$fp)));
+        $result = $c->download(array(array('url'=>$url, 'file'=>$fp)));
         // Close file handler.
         fclose($fp);
+        if (empty($result)) {
+            unlink($path);
+            return null;
+        }
         return array('path'=>$path, 'url'=>$url);
     }
 
@@ -2137,25 +2217,47 @@ abstract class repository {
         }
     }
 
-
+    /**
+     * Called from phpunit between tests, resets whatever was cached
+     */
+    public static function reset_caches() {
+        self::sync_external_file(null, true);
+    }
 
     /**
      * Call to request proxy file sync with repository source.
      *
      * @param stored_file $file
+     * @param bool $resetsynchistory whether to reset all history of sync (used by phpunit)
      * @return bool success
      */
-    public static function sync_external_file(stored_file $file) {
+    public static function sync_external_file($file, $resetsynchistory = false) {
         global $DB;
+        // TODO MDL-25290 static should be replaced with MUC code.
+        static $synchronized = array();
+        if ($resetsynchistory) {
+            $synchronized = array();
+        }
 
         $fs = get_file_storage();
 
+        if (!$file || !$file->get_referencefileid()) {
+            return false;
+        }
+        if (array_key_exists($file->get_id(), $synchronized)) {
+            return $synchronized[$file->get_id()];
+        }
+
+        // remember that we already cached in current request to prevent from querying again
+        $synchronized[$file->get_id()] = false;
+
         if (!$reference = $DB->get_record('files_reference', array('id'=>$file->get_referencefileid()))) {
             return false;
         }
 
         if (!empty($reference->lastsync) and ($reference->lastsync + $reference->lifetime > time())) {
-            return false;
+            $synchronized[$file->get_id()] = true;
+            return true;
         }
 
         if (!$repository = self::get_repository_by_id($reference->repositoryid, SYSCONTEXTID)) {
@@ -2169,14 +2271,10 @@ abstract class repository {
         $fileinfo = $repository->get_file_by_reference($reference);
         if ($fileinfo === null) {
             // does not exist any more - set status to missing
-            $sql = "UPDATE {files} SET status = :missing WHERE referencefileid = :referencefileid";
+            $file->set_missingsource();
-            $params = array('referencefileid'=>$reference->id, 'missing'=>666);
-            $DB->execute($sql, $params);
             //TODO: purge content from pool if we set some other content hash and it is no used any more
+            $synchronized[$file->get_id()] = true;
             return true;
-        } else if ($fileinfo === false) {
-            // error
-            return false;
         }
 
         $contenthash = null;
@@ -2205,15 +2303,9 @@ abstract class repository {
             return false;
         }
 
-        $now = time();
         // update files table
-        $sql = "UPDATE {files} SET contenthash = :contenthash, filesize = :filesize, referencelastsync = :now, referencelifetime = :lifetime, timemodified = :now2 WHERE referencefileid = :referencefileid AND contenthash <> :contenthash2";
+        $file->set_synchronized($contenthash, $filesize);
-        $params = array('contenthash'=>$contenthash, 'filesize'=>$filesize, 'now'=>$now, 'lifetime'=>$reference->lifetime,
+        $synchronized[$file->get_id()] = true;
-            'now2'=>$now, 'referencefileid'=>$reference->id, 'contenthash2'=>$contenthash);
-        $DB->execute($sql, $params);
-
-        $DB->set_field('files_reference', 'lastsync', $now, array('id'=>$reference->id));
-
         return true;
     }
 }

repository/user/lib.php

@@ -153,79 +153,6 @@ class repository_user extends repository {
         return FILE_INTERNAL | FILE_REFERENCE;
     }
 
-
-    /**
-     * Prepare file reference information
-     *
-     * @param string $source
-     * @return string file referece
-     */
-    public function get_file_reference($source) {
-        global $USER;
-        $params = unserialize(base64_decode($source));
-        if (is_array($params)) {
-            $filepath = clean_param($params['filepath'], PARAM_PATH);;
-            $filename = clean_param($params['filename'], PARAM_FILE);
-            $contextid = clean_param($params['contextid'], PARAM_INT);
-        }
-        // We store all file parameters, so file api could
-        // find the refernces later.
-        $reference = array();
-        $reference['contextid'] = $contextid;
-        $reference['component'] = 'user';
-        $reference['filearea']  = 'private';
-        $reference['itemid']    = 0;
-        $reference['filepath']  = $filepath;
-        $reference['filename']  = $filename;
-
-        return file_storage::pack_reference($reference);
-    }
-
-    /**
-     * Get file from external repository by reference
-     * {@link repository::get_file_reference()}
-     * {@link repository::get_file()}
-     *
-     * @param stdClass $reference file reference db record
-     * @return stdClass|null|false
-     */
-    public function get_file_by_reference($reference) {
-        $fs = get_file_storage();
-        $ref = $reference->reference;
-        $params = unserialize(base64_decode($ref));
-        if (!is_array($params)) {
-            throw new repository_exception('invalidparams', 'repository');
-        }
-        $filename  = is_null($params['filename'])  ? null : clean_param($params['filename'], PARAM_FILE);
-        $filepath  = is_null($params['filepath'])  ? null : clean_param($params['filepath'], PARAM_PATH);;
-        $contextid = is_null($params['contextid']) ? null : clean_param($params['contextid'], PARAM_INT);
-
-        // hard coded component, filearea and item for security
-        $component = 'user';
-        $filearea  = 'private';
-        $itemid    = 0;
-
-        $storedfile = $fs->get_file($contextid, $component, $filearea, $itemid, $filepath, $filename);
-
-        $fileinfo = new stdClass;
-        $fileinfo->contenthash = $storedfile->get_contenthash();
-        $fileinfo->filesize    = $storedfile->get_filesize();
-        return $fileinfo;
-    }
-
-    /**
-     * Return human readable reference information
-     * {@link stored_file::get_reference()}
-     *
-     * @param string $reference
-     * @return string|null
-     */
-    public function get_reference_details($reference) {
-        $params = file_storage::unpack_reference($reference);
-        // Indicate this is from user private area
-        return $this->get_name() . ': ' . $params['filepath'] . $params['filename'];
-    }
-
     /**
      * Return reference file life time
      *
@@ -236,29 +163,4 @@ class repository_user extends repository {
         // this should be realtime
         return 0;
     }
-
-    /**
-     * Repository method to serve file
-     *
-     * @param stored_file $storedfile
-     * @param int $lifetime Number of seconds before the file should expire from caches (default 24 hours)
-     * @param int $filter 0 (default)=no filtering, 1=all files, 2=html files only
-     * @param bool $forcedownload If true (default false), forces download of file rather than view in browser/plugin
-     * @param array $options additional options affecting the file serving
-     */
-    public function send_file($storedfile, $lifetime=86400 , $filter=0, $forcedownload=false, array $options = null) {
-        $reference = $storedfile->get_reference();
-        $params = file_storage::unpack_reference($reference);
-        $filepath = clean_param($params['filepath'], PARAM_PATH);;
-        $filename = clean_param($params['filename'], PARAM_FILE);
-        $contextid = clean_param($params['contextid'], PARAM_INT);
-        $filearea  = 'private';
-        $component = 'user';
-        $itemid    = 0;
-
-        $fs = get_file_storage();
-        $storedfile = $fs->get_file($contextid, $component, $filearea, $itemid, $filepath, $filename);
-
-        send_stored_file($storedfile, $lifetime, $filter, $forcedownload, $options);
-    }
 }

theme/base/style/filemanager.css

@@ -243,6 +243,9 @@ a.ygtvspacer:hover {color: transparent;text-decoration: none;}
 .filemanager .fp-iconview .fp-file.fp-hasreferences .fp-reficons1 {background: url('[[pix:theme|fp/link]]') no-repeat;background-position:bottom right;}
 .filemanager .fp-iconview .fp-file.fp-isreference .fp-reficons2 {background: url('[[pix:theme|fp/alias]]') no-repeat;background-position:bottom left;}
 
+.filemanager .fp-iconview .fp-file.fp-originalmissing .fp-thumbnail img {display:none;}
+.filemanager .fp-iconview .fp-file.fp-originalmissing .fp-thumbnail {background: url([[pix:s/dead]]) no-repeat;background-position:center center;}
+
 /*
  * Table view (File Manager only)
  */