MDL-36119: auth_{ldap,cas}: LDAP Sync - implement paged results

Thanks to Jerome Charaoui for the original patch.

auth/cas/auth.php

@@ -3,7 +3,7 @@
 /**
  * @author Martin Dougiamas
  * @author Jerome GUTIERREZ
- * @author I<EFBFBD>aki Arenaza
+ * @author Iñaki Arenaza
  * @license http://www.gnu.org/copyleft/gpl.html GNU Public License
  * @package moodle multiauth
  *
@@ -206,6 +206,10 @@ class auth_plugin_cas extends auth_plugin_ldap {
             }
         }
 
+        if (!ldap_paged_results_supported($this->config->ldap_version)) {
+            echo $OUTPUT->notification(get_string('pagedresultsnotsupp', 'auth_ldap'));
+        }
+
         include($CFG->dirroot.'/auth/cas/config.html');
     }
 
@@ -279,6 +283,9 @@ class auth_plugin_cas extends auth_plugin_ldap {
         if (empty($config->ldapencoding)) {
             $config->ldapencoding = 'utf-8';
         }
+        if (!isset($config->pagesize)) {
+            $config->pagesize = LDAP_DEFAULT_PAGESIZE;
+        }
         if (!isset($config->contexts)) {
             $config->contexts = '';
         }
@@ -339,6 +346,7 @@ class auth_plugin_cas extends auth_plugin_ldap {
         // save LDAP settings
         set_config('host_url', trim($config->host_url), $this->pluginconfig);
         set_config('ldapencoding', trim($config->ldapencoding), $this->pluginconfig);
+        set_config('pagesize', (int)trim($config->pagesize), $this->pluginconfig);
         set_config('contexts', trim($config->contexts), $this->pluginconfig);
         set_config('user_type', textlib::strtolower(trim($config->user_type)), $this->pluginconfig);
         set_config('user_attribute', textlib::strtolower(trim($config->user_attribute)), $this->pluginconfig);

auth/cas/config.html

@@ -44,6 +44,9 @@ if (!isset($config->host_url)) {
 if (empty($config->ldapencoding)) {
     $config->ldapencoding = 'utf-8';
 }
+if (!isset($config->pagesize)) {
+    $config->pagesize = LDAP_DEFAULT_PAGESIZE;
+}
 if (!isset($config->contexts)) {
     $config->contexts = '';
 }
@@ -258,6 +261,25 @@ $yesno = array( get_string('no'), get_string('yes') );
         <?php print_string('auth_ldap_ldap_encoding', 'auth_ldap') ?>
     </td>
 </tr>
+<tr valign="top">
+    <td align="right">
+        <label for="pagesize"><?php print_string('pagesize_key', 'auth_ldap') ?></label>
+    </td>
+    <td>
+        <?php $disabled = (!ldap_paged_results_supported($config->ldap_version)) ? ' disabled="disabled"' : '' ; ?>
+        <input id="pagesize" name="pagesize" type="text" value="<?php echo $config->pagesize ?>" <?php echo $disabled ?>/>
+        <?php
+            if (isset($err['pagesize'])) { echo $OUTPUT->error_text($err['pagesize']); }
+            if ($disabled) {
+                // Don't loose the page size value (disabled fields are not submitted!)
+        ?>
+            <input id="pagesize" name="pagesize" type="hidden" value="<?php echo $config->pagesize ?>" />
+        <?php } ?>
+    </td>
+    <td>
+        <?php print_string('pagesize', 'auth_ldap') ?>
+    </td>
+</tr>
 <tr>
     <td colspan="2">
         <h4><?php print_string('auth_ldap_bind_settings', 'auth_ldap') ?></h4>

auth/cas/version.php

@@ -27,6 +27,6 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-$plugin->version   = 2012061700;        // The current plugin version (Date: YYYYMMDDXX)
+$plugin->version   = 2012110700;        // The current plugin version (Date: YYYYMMDDXX)
 $plugin->requires  = 2012061700;        // Requires this Moodle version
 $plugin->component = 'auth_cas';        // Full name of the plugin (used for diagnostics)

auth/ldap/auth.php

@@ -2,7 +2,7 @@
 
 /**
  * @author Martin Dougiamas
- * @author I<EFBFBD>aki Arenaza
+ * @author Iñaki Arenaza
  * @license http://www.gnu.org/copyleft/gpl.html GNU Public License
  * @package moodle multiauth
  *
@@ -214,6 +214,7 @@ class auth_plugin_ldap extends auth_plugin_base {
 
         $ldapconnection = $this->ldap_connect();
         if(!($user_dn = $this->ldap_find_userdn($ldapconnection, $extusername))) {
+            $this->ldap_close();
             return false;
         }
 
@@ -231,11 +232,13 @@ class auth_plugin_ldap extends auth_plugin_base {
         }
 
         if (!$user_info_result = ldap_read($ldapconnection, $user_dn, '(objectClass=*)', $search_attribs)) {
+            $this->ldap_close();
             return false; // error!
         }
 
         $user_entry = ldap_get_entries_moodle($ldapconnection, $user_info_result);
         if (empty($user_entry)) {
+            $this->ldap_close();
             return false; // entry not found
         }
 
@@ -650,28 +653,31 @@ class auth_plugin_ldap extends auth_plugin_base {
             array_push($contexts, $this->config->create_context);
         }
 
-        $fresult = array();
+        $ldap_pagedresults = ldap_paged_results_supported($this->config->ldap_version);
+        $ldap_cookie = '';
         foreach ($contexts as $context) {
             $context = trim($context);
             if (empty($context)) {
                 continue;
             }
-            if ($this->config->search_sub) {
-                //use ldap_search to find first user from subtree
-                $ldap_result = ldap_search($ldapconnection, $context,
-                                           $filter,
-                                           array($this->config->user_attribute));
-            } else {
-                //search only in this context
-                $ldap_result = ldap_list($ldapconnection, $context,
-                                         $filter,
-                                         array($this->config->user_attribute));
-            }
 
+            do {
+                if ($ldap_pagedresults) {
+                    ldap_control_paged_result($ldapconnection, $this->config->pagesize, true, $ldap_cookie);
+                }
+                if ($this->config->search_sub) {
+                    // Use ldap_search to find first user from subtree.
+                    $ldap_result = ldap_search($ldapconnection, $context, $filter, array($this->config->user_attribute));
+                } else {
+                    // Search only in this context.
+                    $ldap_result = ldap_list($ldapconnection, $context, $filter, array($this->config->user_attribute));
+                }
                 if(!$ldap_result) {
                     continue;
                 }
-
+                if ($ldap_pagedresults) {
+                    ldap_control_paged_result_response($ldapconnection, $ldap_result, $ldap_cookie);
+                }
                 if ($entry = @ldap_first_entry($ldapconnection, $ldap_result)) {
                     do {
                         $value = ldap_get_values_len($ldapconnection, $entry, $this->config->user_attribute);
@@ -679,7 +685,15 @@ class auth_plugin_ldap extends auth_plugin_base {
                         $this->ldap_bulk_insert($value);
                     } while ($entry = ldap_next_entry($ldapconnection, $entry));
                 }
-            unset($ldap_result); // free mem
+                unset($ldap_result); // Free mem.
+            } while ($ldap_pagedresults && !empty($ldap_cookie));
+        }
+
+        // If LDAP paged results were used, the current connection must be completely 
+        // closed and a new one created, to work without paged results from here on.
+        if ($ldap_pagedresults) {
+            $this->ldap_close(true);
+            $ldapconnection = $this->ldap_connect();
         }
 
         /// preserve our user database
@@ -1414,39 +1428,43 @@ class auth_plugin_ldap extends auth_plugin_base {
             array_push($contexts, $this->config->create_context);
         }
 
+        $ldap_pagedresults = ldap_paged_results_supported($this->config->ldap_version);
         foreach ($contexts as $context) {
             $context = trim($context);
             if (empty($context)) {
                 continue;
             }
 
-            if ($this->config->search_sub) {
+            do {
-                // Use ldap_search to find first user from subtree
+                if ($ldap_pagedresults) {
-                $ldap_result = ldap_search($ldapconnection, $context,
+                    ldap_control_paged_result($ldapconnection, $this->config->pagesize, true, $ldap_cookie);
-                                           $filter,
+                }
-                                           array($this->config->user_attribute));
+                if ($this->config->search_sub) {
-            } else {
+                    // Use ldap_search to find first user from subtree.
-                // Search only in this context
+                    $ldap_result = ldap_search($ldapconnection, $context, $filter, array($this->config->user_attribute));
-                $ldap_result = ldap_list($ldapconnection, $context,
+                } else {
-                                         $filter,
+                    // Search only in this context.
-                                         array($this->config->user_attribute));
+                    $ldap_result = ldap_list($ldapconnection, $context, $filter, array($this->config->user_attribute));
                 }
-
                 if(!$ldap_result) {
                     continue;
                 }
-
+                if ($ldap_pagedresults) {
+                    ldap_control_paged_result_response($ldapconnection, $ldap_result, $ldap_cookie);
+                }
                 $users = ldap_get_entries_moodle($ldapconnection, $ldap_result);
-
+                // Add found users to list.
-            // Add found users to list
                 for ($i = 0; $i < count($users); $i++) {
                     $extuser = textlib::convert($users[$i][$this->config->user_attribute][0],
                                                 $this->config->ldapencoding, 'utf-8');
                     array_push($fresult, $extuser);
                 }
+                unset($ldap_result); // Free mem.
+            } while ($ldap_pagedresults && !empty($ldap_cookie));
         }
 
-        $this->ldap_close();
+        // If paged results were used, make sure the current connection is completely closed
+        $this->ldap_close($ldap_pagedresults);
         return $fresult;
     }
 
@@ -1693,6 +1711,10 @@ class auth_plugin_ldap extends auth_plugin_base {
             return;
         }
 
+        if (!ldap_paged_results_supported($this->config->ldap_version)) {
+            echo $OUTPUT->notification(get_string('pagedresultsnotsupp', 'auth_ldap'));
+        }
+
         include($CFG->dirroot.'/auth/ldap/config.html');
     }
 
@@ -1707,6 +1729,9 @@ class auth_plugin_ldap extends auth_plugin_base {
         if (empty($config->ldapencoding)) {
          $config->ldapencoding = 'utf-8';
         }
+        if (!isset($config->pagesize)) {
+            $config->pagesize = LDAP_DEFAULT_PAGESIZE;
+        }
         if (!isset($config->contexts)) {
              $config->contexts = '';
         }
@@ -1807,6 +1832,7 @@ class auth_plugin_ldap extends auth_plugin_base {
         // Save settings
         set_config('host_url', trim($config->host_url), $this->pluginconfig);
         set_config('ldapencoding', trim($config->ldapencoding), $this->pluginconfig);
+        set_config('pagesize', (int)trim($config->pagesize), $this->pluginconfig);
         set_config('contexts', $config->contexts, $this->pluginconfig);
         set_config('user_type', textlib::strtolower(trim($config->user_type)), $this->pluginconfig);
         set_config('user_attribute', textlib::strtolower(trim($config->user_attribute)), $this->pluginconfig);
@@ -2009,10 +2035,14 @@ class auth_plugin_ldap extends auth_plugin_base {
     /**
      * Disconnects from a LDAP server
      *
+     * @param force boolean Forces closing the real connection to the LDAP server, ignoring any
+     *                      cached connections. This is needed when we've used paged results
+     *                      and want to use normal results again.
      */
-    function ldap_close() {
+    function ldap_close($force=false) {
         $this->ldapconns--;
-        if($this->ldapconns == 0) {
+        if (($this->ldapconns == 0) || ($force)) {
+            $this->ldapconns = 0;
             @ldap_close($this->ldapconnection);
             unset($this->ldapconnection);
         }

auth/ldap/config.html

@@ -7,6 +7,9 @@ if (!isset($config->host_url)) {
 if (empty($config->ldapencoding)) {
     $config->ldapencoding = 'utf-8';
 }
+if (!isset($config->pagesize)) {
+    $config->pagesize = LDAP_DEFAULT_PAGESIZE;
+}
 if (!isset($config->contexts)) {
     $config->contexts = '';
 }
@@ -148,6 +151,26 @@ $yesno = array(get_string('no'), get_string('yes'));
         <?php print_string('auth_ldap_ldap_encoding', 'auth_ldap') ?>
     </td>
 </tr>
+<tr valign="top">
+    <td align="right">
+        <label for="pagesize"><?php print_string('pagesize_key', 'auth_ldap') ?></label>
+    </td>
+    <td>
+        <?php $disabled = (!ldap_paged_results_supported($config->ldap_version)) ? ' disabled="disabled"' : '' ; ?>
+        <input id="pagesize" name="pagesize" type="text" value="<?php echo $config->pagesize ?>" <?php echo $disabled ?>/>
+        <?php
+            if (isset($err['pagesize'])) { echo $OUTPUT->error_text($err['pagesize']); }
+            if ($disabled) {
+                // Don't loose the page size value (disabled fields are not submitted!)
+        ?>
+            <input id="pagesize" name="pagesize" type="hidden" value="<?php echo $config->pagesize ?>" />
+        <?php } ?>
+
+    </td>
+    <td>
+        <?php print_string('pagesize', 'auth_ldap') ?>
+    </td>
+</tr>
 <tr>
     <td colspan="2">
         <h4><?php print_string('auth_ldap_bind_settings', 'auth_ldap') ?></h4>

auth/ldap/lang/en/auth_ldap.php

@@ -128,6 +128,9 @@ $string['ntlmsso_attempting'] = 'Attempting Single Sign On via NTLM...';
 $string['ntlmsso_failed'] = 'Auto-login failed, try the normal login page...';
 $string['ntlmsso_isdisabled'] = 'NTLM SSO is disabled.';
 $string['ntlmsso_unknowntype'] = 'Unknown ntlmsso type!';
+$string['pagedresultsnotsupp'] = '<em>LDAP paged results not supported (either your PHP version lacks support or you have configured Moodle to use LDAP protocol version 2)</em>';
+$string['pagesize'] = 'Make sure this value is smaller than your LDAP server result set size limit (the maximum number of entries that can be returned in a single query)';
+$string['pagesize_key'] = 'Page Size';
 $string['pluginname'] = 'LDAP server';
 $string['pluginnotenabled'] = 'Plugin not enabled!';
 $string['renamingnotallowed'] = 'User renaming not allowed in LDAP';

auth/ldap/version.php

@@ -26,6 +26,6 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-$plugin->version   = 2012061700;        // The current plugin version (Date: YYYYMMDDXX)
+$plugin->version   = 2012110700;        // The current plugin version (Date: YYYYMMDDXX)
 $plugin->requires  = 2012061700;        // Requires this Moodle version
 $plugin->component = 'auth_ldap';       // Full name of the plugin (used for diagnostics)

enrol/ldap/lib.php

@@ -304,6 +304,8 @@ class enrol_ldap_plugin extends enrol_plugin {
             return;
         }
 
+        $ldap_pagedresults = ldap_paged_results_supported($this->get_config('ldap_version'));
+
         // we may need a lot of memory here
         @set_time_limit(0);
         raise_memory_limit(MEMORY_HUGE);
@@ -332,12 +334,18 @@ class enrol_ldap_plugin extends enrol_plugin {
             // Define the search pattern
             $ldap_search_pattern = $this->config->objectclass;
 
+            $ldap_cookie = '';
             foreach ($ldap_contexts as $ldap_context) {
                 $ldap_context = trim($ldap_context);
                 if (empty($ldap_context)) {
                     continue; // Next;
                 }
 
+                do {
+                    if ($ldap_pagedresults) {
+                        ldap_control_paged_result($ldapconnection, $this->config->pagesize, true, $ldap_cookie);
+                    }
+
                     if ($this->config->course_search_sub) {
                         // Use ldap_search to find first user from subtree
                         $ldap_result = @ldap_search($ldapconnection,
@@ -355,6 +363,10 @@ class enrol_ldap_plugin extends enrol_plugin {
                         continue; // Next
                     }
 
+                    if ($ldap_pagedresults) {
+                        ldap_control_paged_result_response($ldapconnection, $ldap_result, $ldap_cookie);
+                    }
+
                     // Check and push results
                     $records = ldap_get_entries($ldapconnection, $ldap_result);
 
@@ -365,6 +377,14 @@ class enrol_ldap_plugin extends enrol_plugin {
                     }
                     // Free some mem
                     unset($records);
+                } while ($ldap_pagedresults && !empty($ldap_cookie));
+
+                // If LDAP paged results were used, the current connection must be completely 
+                // closed and a new one created, to work without paged results from here on.
+                if ($ldap_pagedresults) {
+                    $this->ldap_close(true);
+                    $ldapconnection = $this->ldap_connect();
+                }
 
                 if (count($flat_records)) {
                     $ignorehidden = $this->get_config('ignorehiddencourses');
@@ -697,12 +717,19 @@ class enrol_ldap_plugin extends enrol_plugin {
 
         // Get all contexts and look for first matching user
         $ldap_contexts = explode(';', $ldap_contexts);
+        $ldap_pagedresults = ldap_paged_results_supported($this->get_config('ldap_version'));
+        $ldap_cookie = '';
         foreach ($ldap_contexts as $context) {
             $context = trim($context);
             if (empty($context)) {
                 continue;
             }
 
+            do {
+                if ($ldap_pagedresults) {
+                    ldap_control_paged_result($ldapconnection, $this->config->pagesize, true, $ldap_cookie);
+                }
+
                 if ($this->get_config('course_search_sub')) {
                     // Use ldap_search to find first user from subtree
                     $ldap_result = @ldap_search($ldapconnection,
@@ -721,6 +748,10 @@ class enrol_ldap_plugin extends enrol_plugin {
                     continue;
                 }
 
+                if ($ldap_pagedresults) {
+                    ldap_control_paged_result_response($ldapconnection, $ldap_result, $ldap_cookie);
+                }
+
                 // Check and push results. ldap_get_entries() already
                 // lowercases the attribute index, so there's no need to
                 // use array_change_key_case() later.
@@ -731,7 +762,16 @@ class enrol_ldap_plugin extends enrol_plugin {
                 for ($c = 0; $c < $records['count']; $c++) {
                     array_push($flat_records, $records[$c]);
                 }
+                // Free some mem
                 unset($records);
+            } while ($ldap_pagedresults && !empty($ldap_cookie));
+
+            // If LDAP paged results were used, the current connection must be completely 
+            // closed and a new one created, to work without paged results from here on.
+            if ($ldap_pagedresults) {
+                $this->ldap_close(true);
+                $ldapconnection = $this->ldap_connect();
+            }
 
             if (count($flat_records)) {
                 $courses = array_merge($courses, $flat_records);
@@ -788,7 +828,7 @@ class enrol_ldap_plugin extends enrol_plugin {
      *                        groups.
      */
     protected function ldap_find_user_groups_recursively($ldapconnection, $memberdn, &$membergroups) {
-        $result = @ldap_read ($ldapconnection, $memberdn, '(objectClass=*)', array($this->get_config('group_memberofattribute')));
+        $result = @ldap_read($ldapconnection, $memberdn, '(objectClass=*)', array($this->get_config('group_memberofattribute')));
         if (!$result) {
             return;
         }

enrol/ldap/settings.php

@@ -39,6 +39,7 @@ if ($ADMIN->fulltree) {
         require_once($CFG->libdir.'/ldaplib.php');
 
         $yesno = array(get_string('no'), get_string('yes'));
+        $pagedresults = ldap_paged_results_supported(get_config('enrol_ldap', 'ldap_version'));
 
         //--- connection settings ---
         $settings->add(new admin_setting_heading('enrol_ldap_server_settings', get_string('server_settings', 'enrol_ldap'), ''));
@@ -47,6 +48,7 @@ if ($ADMIN->fulltree) {
         $options = array(3=>'3', 2=>'2');
         $settings->add(new admin_setting_configselect('enrol_ldap/ldap_version', get_string('version_key', 'enrol_ldap'), get_string('version', 'enrol_ldap'), 3, $options));
         $settings->add(new admin_setting_configtext_trim_lower('enrol_ldap/ldapencoding', get_string('ldap_encoding_key', 'enrol_ldap'), get_string('ldap_encoding', 'enrol_ldap'), 'utf-8'));
+        $settings->add(new admin_setting_configtext_trim_lower('enrol_ldap/pagesize', get_string('pagesize_key', 'auth_ldap'), get_string('pagesize', 'auth_ldap'), LDAP_DEFAULT_PAGESIZE, true, $pagedresults));
 
         //--- binding settings ---
         $settings->add(new admin_setting_heading('enrol_ldap_bind_settings', get_string('bind_settings', 'enrol_ldap'), ''));

enrol/ldap/settingslib.php

@@ -39,9 +39,11 @@ class admin_setting_configtext_trim_lower extends admin_setting_configtext {
      * @param string $description long localised info
      * @param string $defaultsetting default value for the setting
      * @param boolean $lowercase if true, lowercase the value before writing it to the db.
+     * @param boolean $enabled if true, the input field is enabled, otherwise it's disabled.
      */
-    public function __construct($name, $visiblename, $description, $defaultsetting, $lowercase=false) {
+    public function __construct($name, $visiblename, $description, $defaultsetting, $lowercase=false, $enabled=true) {
         $this->lowercase = $lowercase;
+        $this->enabled = $enabled;
         parent::__construct($name, $visiblename, $description, $defaultsetting);
     }
 
@@ -65,8 +67,24 @@ class admin_setting_configtext_trim_lower extends admin_setting_configtext {
         if ($this->lowercase) {
             $data = textlib::strtolower($data);
         }
+        if (!$this->enabled) {
+            return '';
+        }
         return ($this->config_write($this->name, trim($data)) ? '' : get_string('errorsetting', 'admin'));
     }
+
+    /**
+     * Return an XHTML string for the setting
+     * @return string Returns an XHTML string
+     */
+    public function output_html($data, $query='') {
+        $default = $this->get_defaultsetting();
+        $disabled = $this->enabled ? '': ' disabled="disabled"';
+        return format_admin_setting($this, $this->visiblename,
+        '<div class="form-text defaultsnext"><input type="text" size="'.$this->size.'" id="'.$this->get_id().'" name="'.$this->get_full_name().'" value="'.s($data).'" '.$disabled.' /></div>',
+        $this->description, true, '', $default, $query);
+    }
+
 }
 
 class admin_setting_ldap_rolemapping extends admin_setting {

enrol/ldap/version.php

@@ -26,6 +26,6 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-$plugin->version   = 2012061700;        // The current plugin version (Date: YYYYMMDDXX)
+$plugin->version   = 2012110700;        // The current plugin version (Date: YYYYMMDDXX)
 $plugin->requires  = 2012061700;        // Requires this Moodle version
 $plugin->component = 'enrol_ldap';      // Full name of the plugin (used for diagnostics)

lib/ldaplib.php

@@ -7,11 +7,11 @@
  * data structures, useful for both ldap authentication (or ldap based
  * authentication like CAS) and enrolment plugins.
  *
- * @author     I<EFBFBD>aki Arenaza
+ * @author     Iñaki Arenaza
  * @package    core
  * @subpackage lib
  * @copyright  1999 onwards Martin Dougiamas  http://dougiamas.com
- * @copyright  2010 onwards I<EFBFBD>aki Arenaza
+ * @copyright  2010 onwards Iñaki Arenaza
  * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  */
 
@@ -22,6 +22,11 @@ if (!defined('ROOTDSE')) {
     define ('ROOTDSE', '');
 }
 
+// Default page size when using LDAP paged results
+if (!defined('LDAP_DEFAULT_PAGESIZE')) {
+    define('LDAP_DEFAULT_PAGESIZE', 250);
+}
+
 /**
  * Returns predefined user types
  *
@@ -364,3 +369,24 @@ function ldap_stripslashes($text) {
 
     return $text;
 }
+
+
+/**
+ * Check if PHP supports LDAP paged results and we can use them (we have to use LDAP 
+ * version 3, otherwise the server doesn't use them).
+ *
+ * @param ldapversion integer The LDAP protocol version we use.
+ *
+ * @return boolean true is paged results can be used, false otherwise.
+ */
+function ldap_paged_results_supported($ldapversion) {
+
+    if (((int)$ldapversion === 3) &&
+        function_exists('ldap_control_paged_result') &&
+        function_exists('ldap_control_paged_result_response')) {
+
+        return true;
+    }
+
+    return false;
+}