moodle/moodle
3
0
Fork 0
mirror of https://github.com/moodle/moodle.git synced 2025-08-03 16:13:28 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

MDL-42016 repository: Implement new sync method in repositories

Browse source
This commit is contained in:
Marina Glancy 2013-10-04 23:26:03 +10:00
parent 873555604f
commit 5e5923ea28
4 changed files with 92 additions and 122 deletions
Download patch file Download diff file Expand all files Collapse all files

62
repository/boxnet/lib.php
View file

@ -267,34 +267,34 @@ class repository_boxnet extends repository {
return $source;
}
/**
* Returns information about file in this repository by reference
* {@link repository::get_file_reference()}
* {@link repository::get_file()}
*
* Returns null if file not found or is not readable
*
* @param stdClass $reference file reference db record
* @return null|stdClass with attribute 'filepath'
*/
public function get_file_by_reference($reference) {
$array = explode('/', $reference->reference);
$fileid = array_pop($array);
$fileinfo = $this->boxclient->get_file_info($fileid, self::SYNCFILE_TIMEOUT);
if ($fileinfo) {
$size = (int)$fileinfo->size;
if (file_extension_in_typegroup($fileinfo->file_name, 'web_image')) {
// this is an image - download it to moodle
$path = $this->prepare_file('');
$c = new curl;
$result = $c->download_one($reference->reference, null, array('filepath' => $path, 'timeout' => self::SYNCIMAGE_TIMEOUT));
if ($result === true) {
return (object)array('filepath' => $path);
}
}
return (object)array('filesize' => $size);
public function sync_reference(stored_file $file) {
if ($file->get_referencelastsync() + DAYSECS > time()) {
// Synchronise not more often than once a day.
return false;
}
return null;
$c = new curl;
if (file_extension_in_typegroup($file->get_filename(), 'web_image')) {
$path = $this->prepare_file('');
$result = $c->download_one($file->get_reference(), null, array('filepath' => $path, 'timeout' => self::SYNCIMAGE_TIMEOUT));
$info = $c->get_info();
if ($result === true && isset($info['http_code']) && $info['http_code'] == 200) {
$fs = get_file_storage();
list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($path);
$file->set_synchronized($contenthash, $filesize);
return true;
}
}
$c->get($file->get_reference(), null, array('timeout' => self::SYNCIMAGE_TIMEOUT, 'followlocation' => true, 'nobody' => true));
$info = $c->get_info();
if (isset($info['http_code']) && $info['http_code'] == 200 &&
array_key_exists('download_content_length', $info) &&
$info['download_content_length'] >= 0) {
$filesize = (int)$info['download_content_length'];
$file->set_synchronized(null, $filesize);
return true;
}
$file->set_missingsource();
return true;
}
/**
@ -307,14 +307,8 @@ class repository_boxnet extends repository {
*/
public function get_reference_details($reference, $filestatus = 0) {
// Indicate it's from box.net repository + secure URL
$array = explode('/', $reference);
$fileid = array_pop($array);
$fileinfo = $this->boxclient->get_file_info($fileid, self::SYNCFILE_TIMEOUT);
if (!empty($fileinfo)) {
$reference = (string)$fileinfo->file_name;
}
$details = $this->get_name() . ': ' . $reference;
if (!empty($fileinfo)) {
if (!$filestatus) {
return $details;
} else {
return get_string('lostsource', 'repository', $details);

38
repository/dropbox/lib.php
View file

@ -546,25 +546,18 @@ class repository_dropbox extends repository {
return serialize($reference);
}
/**
* Returns information about file in this repository by reference
* {@link repository::get_file_reference()}
* {@link repository::get_file()}
*
* Returns null if file not found or is not readable
*
* @param stdClass $reference file reference db record
* @return null|stdClass that has 'filepath' property
*/
public function get_file_by_reference($reference) {
global $USER;
$ref = unserialize($reference->reference);
public function sync_reference(stored_file $file) {
if ($file->get_referencelastsync() + DAYSECS > time()) {
// Synchronise not more often than once a day.
return false;
}
$ref = unserialize($file->get_reference());
if (!isset($ref->url)) {
// this is an old-style reference in DB. We need to fix it
$ref = unserialize($this->fix_old_style_reference($reference->reference));
$ref = unserialize($this->fix_old_style_reference($file->get_reference()));
}
if (!isset($ref->url)) {
return null;
return false;
}
$c = new curl;
$url = $this->get_file_download_link($ref->url);
@ -574,7 +567,10 @@ class repository_dropbox extends repository {
$result = $c->download_one($url, array(), array('filepath' => $saveas, 'timeout' => self::SYNCIMAGE_TIMEOUT, 'followlocation' => true));
$info = $c->get_info();
if ($result === true && isset($info['http_code']) && $info['http_code'] == 200) {
return (object)array('filepath' => $saveas);
$fs = get_file_storage();
list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($saveas);
$file->set_synchronized($contenthash, $filesize);
return true;
}
} catch (Exception $e) {}
}
@ -583,9 +579,12 @@ class repository_dropbox extends repository {
if (isset($info['http_code']) && $info['http_code'] == 200 &&
array_key_exists('download_content_length', $info) &&
$info['download_content_length'] >= 0) {
return (object)array('filesize' => (int)$info['download_content_length']);
$filesize = (int)$info['download_content_length'];
$file->set_synchronized(null, $filesize);
return true;
}
return null;
$file->set_missingsource();
return true;
}
/**
@ -648,8 +647,7 @@ class repository_dropbox extends repository {
/**
* Returns the maximum size of the Dropbox files to cache in moodle
*
* Note that {@link repository_dropbox::get_file_by_reference()} called by
* {@link repository::sync_external_file()} will try to cache images even
* Note that {@link repository_dropbox::sync_reference()} will try to cache images even
* when they are bigger in order to generate thumbnails. However there is
* a small timeout for downloading images for synchronisation and it will
* probably fail if the image is too big.

52
repository/equella/lib.php
View file

@ -134,7 +134,7 @@ class repository_equella extends repository {
* If we received the connection timeout more than 3 times in a row, we don't attemt to
* connect to the server any more during this request.
*
* This function is used by {@link repository_equella::get_file_by_reference()} that
* This function is used by {@link repository_equella::sync_reference()} that
* synchronises the file size of referenced files.
*
* @param int $errno omit if we just want to know the return value, the last curl_errno otherwise
@ -159,18 +159,6 @@ class repository_equella extends repository {
return ($countfailures[$sess] < 3);
}
/**
* Decide whether or not the file should be synced
*
* @param stored_file $storedfile
* @return bool
*/
public function sync_individual_file(stored_file $storedfile) {
// if we had several unsuccessfull attempts to connect to server - do not try any more
return $this->connection_result();
}
/**
* Download a file, this function can be overridden by subclass. {@link curl}
*
@ -202,34 +190,30 @@ class repository_equella extends repository {
return array('path'=>$path, 'url'=>$url);
}
/**
* Returns information about file in this repository by reference
*
* If the file is an image we download the contents and save it in our filesystem
* so we can generate thumbnails. Otherwise we just request the file size.
* Returns null if file not found or can not be accessed
*
* @param stdClass $reference file reference db record
* @return stdClass|null contains one of the following:
* - 'filesize' (for non-image files or files we failed to retrieve fully because of timeout)
* - 'filepath' (for image files that we retrieived and saved)
*/
public function get_file_by_reference($reference) {
public function sync_reference(stored_file $file) {
global $USER;
$ref = @unserialize(base64_decode($reference->reference));
if ($file->get_referencelastsync() + DAYSECS > time() || !$this->connection_result()) {
// Synchronise not more often than once a day.
// if we had several unsuccessfull attempts to connect to server - do not try any more.
return false;
}
$ref = @unserialize(base64_decode($file->get_reference()));
if (!isset($ref->url) || !($url = $this->appendtoken($ref->url))) {
// Occurs when the user isn't known..
return null;
$file->set_missingsource();
return true;
}
$return = null;
$cookiepathname = $this->prepare_file($USER->id. '_'. uniqid('', true). '.cookie');
$c = new curl(array('cookie' => $cookiepathname));
if (file_extension_in_typegroup($ref->filename, 'web_image')) {
$path = $this->prepare_file('');
$result = $c->download_one($url, null, array('filepath' => $path, 'followlocation' => true, 'timeout' => self::SYNCIMAGE_TIMEOUT));
if ($result === true) {
$return = (object)array('filepath' => $path);
$fs = get_file_storage();
list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($path);
$file->set_synchronized($contenthash, $filesize);
return true;
}
} else {
$result = $c->head($url, array('followlocation' => true, 'timeout' => self::SYNCFILE_TIMEOUT));
@ -241,13 +225,15 @@ class repository_equella extends repository {
$this->connection_result($c->get_errno());
$curlinfo = $c->get_info();
if ($return === null && isset($curlinfo['http_code']) && $curlinfo['http_code'] == 200
if (isset($curlinfo['http_code']) && $curlinfo['http_code'] == 200
&& array_key_exists('download_content_length', $curlinfo)
&& $curlinfo['download_content_length'] >= 0) {
// we received a correct header and at least can tell the file size
$return = (object)array('filesize' => $curlinfo['download_content_length']);
$file->set_synchronized(null, $curlinfo['download_content_length']);
return true;
}
return $return;
$file->set_missingsource();
return true;
}
/**

62
repository/filesystem/lib.php
View file

@ -265,17 +265,6 @@ class repository_filesystem extends repository {
return FILE_INTERNAL | FILE_REFERENCE;
}
/**
* Return reference file life time
*
* @param string $ref
* @return int
*/
public function get_reference_file_lifetime($ref) {
// Does not cost us much to synchronise within our own filesystem, set to 1 minute
return 60;
}
/**
* Return human readable reference information
*
@ -292,37 +281,40 @@ class repository_filesystem extends repository {
}
}
/**
* Returns information about file in this repository by reference
*
* Returns null if file not found or is not readable
*
* @param stdClass $reference file reference db record
* @return stdClass|null contains one of the following:
* - 'filesize' if file should not be copied to moodle filepool
* - 'filepath' if file should be copied to moodle filepool
*/
public function get_file_by_reference($reference) {
$ref = $reference->reference;
if ($ref{0} == '/') {
$filepath = $this->root_path.substr($ref, 1, strlen($ref)-1);
} else {
$filepath = $this->root_path.$ref;
public function sync_reference(stored_file $file) {
if ($file->get_referencelastsync() + 60 > time()) {
// Does not cost us much to synchronise within our own filesystem, check every 1 minute.
return false;
}
static $issyncing = false;
if ($issyncing) {
// Avoid infinite recursion when calling $file->get_filesize() and get_contenthash().
return;
}
$filepath = $this->root_path.ltrim($file->get_reference(), '/');
if (file_exists($filepath) && is_readable($filepath)) {
$fs = get_file_storage();
$issyncing = true;
if (file_extension_in_typegroup($filepath, 'web_image')) {
// return path to image files so it will be copied into moodle filepool
// we need the file in filepool to generate an image thumbnail
return (object)array('filepath' => $filepath);
$contenthash = sha1_file($filepath);
if ($file->get_contenthash() == $contenthash) {
// File did not change since the last synchronisation.
$filesize = filesize($filepath);
} else {
// Copy file into moodle filepool (used to generate an image thumbnail).
list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($filepath);
}
} else {
// return just the file size so file will NOT be copied into moodle filepool
return (object)array(
'filesize' => filesize($filepath)
);
// Update only file size so file will NOT be copied into moodle filepool.
$contenthash = null;
$filesize = filesize($filepath);
}
$issyncing = false;
$file->set_synchronized($contenthash, $filesize);
} else {
return null;
$file->set_missingsource();
}
return true;
}
/**