MDL-83119 search_solr: Implement check on connectivity, space usage

Implements a status check which confirms that the Solr search engine
is available. Optionally, the check can also show a warning if the
index grows beyond a certain size.

As part of this change, a new API was added in search_solr\engine
to allow using http_client (Guzzle) instead of raw Curl; this makes
it easier to create mock tests in PHPunit for the new functionality.

search/engine/solr/classes/check/connection.php

@@ -0,0 +1,126 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace search_solr\check;
+
+use core\check\check;
+use core\check\result;
+use core\output\html_writer;
+
+/**
+ * Check that the connection to Solr works.
+ *
+ * @package search_solr
+ * @copyright 2024 The Open University
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class connection extends check {
+    #[\Override]
+    public function get_name(): string {
+        return get_string('pluginname', 'search_solr');
+    }
+
+    #[\Override]
+    public function get_action_link(): ?\action_link {
+        return new \action_link(
+            new \moodle_url('/admin/settings.php', ['section' => 'searchsolr']),
+            get_string('settings'));
+    }
+
+
+    #[\Override]
+    public function get_result(): result {
+        global $CFG;
+
+        $result = result::OK;
+        $resultstr = '';
+        $resultdetails = '';
+
+        try {
+            // We do not use manager::instance as this will already try to connect to the engine,
+            // we only want to do the specific get_status call below and nothing else. So use
+            // search_engine_instance. We know it will be a Solr instance if we got here.
+            /** @var \search_solr\engine $engine */
+            $engine = \core_search\manager::search_engine_instance();
+
+            // Get engine status.
+            $status = $engine->get_status(5);
+
+            $time = number_format($status['time'], 2) . 's';
+            $resultstr = get_string('check_time', 'search_solr', $time);
+        } catch (\Throwable $t) {
+            $status = [
+                'connected' => false,
+                'foundcore' => false,
+                'error' => 'Exception when creating search manager: ' . $t->getMessage(),
+                'exception' => $t,
+            ];
+        }
+
+        if (!$status['connected']) {
+            // No connection at all.
+            $result = result::ERROR;
+            $resultstr = get_string('check_notconnected', 'search_solr');
+            $resultdetails .= \html_writer::tag('p', s($status['error']));
+
+        } else if (!$status['foundcore']) {
+            // There's a connection, but the core doesn't seem to exist.
+            $result = result::ERROR;
+            $resultstr = get_string('check_nocore', 'search_solr');
+            $resultdetails .= \html_writer::tag('p', s($status['error']));
+
+        } else {
+            // Errors related to finding the core size only show if the size warning is configured.
+            $sizelimit = get_config('search_solr', 'indexsizelimit');
+            if (!array_key_exists('indexsize', $status)) {
+                if ($sizelimit) {
+                    $result = result::ERROR;
+                    $resultstr = get_string('check_nosize', 'search_solr');
+                    $resultdetails .= \html_writer::tag('p', s($status['error']));
+                }
+            } else {
+                // Show the index size in result, even if we aren't checking it.
+                $sizestr = get_string(
+                    'indexsize',
+                    'search_solr',
+                    display_size($status['indexsize']),
+                );
+                $resultdetails .= \html_writer::tag('p', $sizestr);
+                if ($sizelimit) {
+                    // Error at specified index size, warning at 90% of it.
+                    $sizewarning = ($sizelimit * 9) / 10;
+                    if ($status['indexsize'] > $sizewarning) {
+                        if ($status['indexsize'] > $sizelimit) {
+                            $resultstr = get_string('check_indextoobig', 'search_solr');
+                            $result = result::ERROR;
+                        } else {
+                            // We don't say it's too big because it isn't yet, just show the size.
+                            $resultstr = $sizestr;
+                            $result = result::WARNING;
+                        }
+                    }
+                }
+            }
+        }
+
+        $ex = $status['exception'] ?? null;
+        if ($ex) {
+            $resultdetails .= \html_writer::tag('pre', str_replace($CFG->dirroot, '', s($ex->getTraceAsString())));
+        }
+
+        return new result($result, $resultstr, $resultdetails);
+    }
+}

search/engine/solr/classes/engine.php

@@ -1340,6 +1340,102 @@ class engine extends \core_search\engine {
         return function_exists('solr_get_version');
     }
 
+    /** @var int When using the capath option, we generate a bundle containing all the pem files, cached 10 mins. */
+    const CA_PATH_CACHE_TIME = 600;
+
+    /** @var int Expired cache files are deleted after this many seconds. */
+    const CA_PATH_CACHE_DELETE_AFTER = 60;
+
+    /**
+     * Gets status of Solr server.
+     *
+     * The result has the following fields:
+     * - connected - true if we got a valid JSON response from server
+     * - foundcore - true if we found the core defined in config (this could be false if schema not set up)
+     *
+     * It may have these other fields:
+     * - error - text if anything went wrong
+     * - exception - if an exception was thrown
+     * - indexsize - index size in bytes if we found what it is
+     *
+     * @param int $timeout Optional timeout in seconds, otherwise uses config value
+     * @return array Array with information about status
+     * @since Moodle 5.0
+     */
+    public function get_status($timeout = 0): array {
+        $result = ['connected' => false, 'foundcore' => false];
+        try {
+            $options = [];
+            if ($timeout) {
+                $options['connect_timeout'] = $timeout;
+                $options['read_timeout'] = $timeout;
+            }
+            $before = microtime(true);
+            try {
+                $response = $this->raw_get_request('admin/cores', $options);
+            } finally {
+                $result['time'] = microtime(true) - $before;
+            }
+            $status = $response->getStatusCode();
+            if ($status !== 200) {
+                $result['error'] = 'Unsuccessful status code: ' . $status;
+                return $result;
+            }
+            $decoded = json_decode($response->getBody()->getContents());
+            if (!$decoded) {
+                $result['error'] = 'Invalid JSON';
+                return $result;
+            }
+            // Provided we get some valid JSON then probably Solr exists and is responding.
+            // Any following errors we don't count as not connected (ERROR display in the check)
+            // because maybe it happens if Solr changes their JSON format in a future version.
+            $result['connected'] = true;
+            if (!property_exists($decoded, 'status')) {
+                $result['error'] = 'Unexpected JSON: no core status';
+                return $result;
+            }
+            foreach ($decoded->status as $core) {
+                $match = false;
+                if (!property_exists($core, 'name')) {
+                    $result['error'] = 'Unexpected JSON: core has no name';
+                    return $result;
+                }
+                if ($core->name === $this->config->indexname) {
+                    $match = true;
+                }
+                if (!$match && property_exists($core, 'cloud')) {
+                    if (!property_exists($core->cloud, 'collection')) {
+                        $result['error'] = 'Unexpected JSON: core cloud has no name';
+                        return $result;
+                    }
+                    if ($core->cloud->collection === $this->config->indexname) {
+                        $match = true;
+                    }
+                }
+
+                if ($match) {
+                    $result['foundcore'] = true;
+                    if (!property_exists($core, 'index')) {
+                        $result['error'] = 'Unexpected JSON: core has no index';
+                        return $result;
+                    }
+                    if (!property_exists($core->index, 'sizeInBytes')) {
+                        $result['error'] = 'Unexpected JSON: core index has no sizeInBytes';
+                        return $result;
+                    }
+                    $result['indexsize'] = $core->index->sizeInBytes;
+                    return $result;
+                }
+            }
+            $result['error'] = 'Could not find core matching ' . $this->config->indexname;;
+            return $result;
+        } catch (\Throwable $t) {
+            $result['error'] = 'Exception occurred: ' . $t->getMessage();
+            $result['exception'] = $t;
+            return $result;
+        }
+    }
+
     /**
      * Returns the solr client instance.
      *
@@ -1453,23 +1549,128 @@ class engine extends \core_search\engine {
     }
 
     /**
-     * Return a Moodle url object for the server connection.
+     * Return a Moodle url object for the raw server URL (containing all indexes).
      *
      * @param string $path The solr path to append.
      * @return \moodle_url
      */
-    public function get_connection_url($path) {
+    public function get_server_url(string $path): \moodle_url {
         // Must use the proper protocol, or SSL will fail.
         $protocol = !empty($this->config->secure) ? 'https' : 'http';
         $url = $protocol . '://' . rtrim($this->config->server_hostname, '/');
         if (!empty($this->config->server_port)) {
             $url .= ':' . $this->config->server_port;
         }
-        $url .= '/solr/' . $this->config->indexname . '/' . ltrim($path, '/');
+        $url .= '/solr/' . ltrim($path, '/');
-
         return new \moodle_url($url);
     }
 
+    /**
+     * Return a Moodle url object for the server connection including the search index.
+     *
+     * @param string $path The solr path to append.
+     * @return \moodle_url
+     */
+    public function get_connection_url($path) {
+        return $this->get_server_url($this->config->indexname . '/' . ltrim($path, '/'));
+    }
+
+    /**
+     * Calls the Solr engine with a GET request (for things the Solr extension doesn't support).
+     *
+     * This has similar result to get_curl_object but uses the newer (mockable) Guzzle HTTP client.
+     *
+     * @param string $path URL path (after /solr/) e.g. 'admin/cores?action=STATUS&core=frog'
+     * @param array $overrideoptions Optional array of Guzzle options, will override config
+     * @return \Psr\Http\Message\ResponseInterface Response message from Guzzle
+     * @throws \GuzzleHttp\Exception\GuzzleException If any problem connecting
+     * @since Moodle 5.0
+     */
+    public function raw_get_request(
+        string $path,
+        array $overrideoptions = [],
+    ): \Psr\Http\Message\ResponseInterface {
+        $client = \core\di::get(\core\http_client::class);
+        return $client->get(
+            $this->get_server_url($path)->out(false),
+            $this->get_http_client_options($overrideoptions),
+        );
+    }
+
+    /**
+     * Gets the \core\http_client options for a connection.
+     *
+     * @param array $overrideoptions Optional array to override some of the options
+     * @return array Array of http_client options
+     */
+    protected function get_http_client_options(array $overrideoptions = []): array {
+        $options = [
+            'connect_timeout' => !empty($this->config->server_timeout) ? (int)$this->config->server_timeout : 30,
+        ];
+        $options['read_timeout'] = $options['connect_timeout'];
+        if (!empty($this->config->server_username)) {
+            $options['auth'] = [$this->config->server_username, $this->config->server_password];
+        }
+        if (!empty($this->config->ssl_cert)) {
+            $options['cert'] = $this->config->ssl_cert;
+        }
+        if (!empty($this->config->ssl_key)) {
+            if (!empty($this->config->ssl_keypassword)) {
+                $options['ssl_key'] = [$this->config->ssl_key, $this->config->ssl_keypassword];
+            } else {
+                $options['ssl_key'] = $this->config->ssl_key;
+            }
+        }
+        if (!empty($this->config->ssl_cainfo)) {
+            $options['verify'] = $this->config->ssl_cainfo;
+        } else if (!empty($this->config->ssl_capath)) {
+            // Guzzle doesn't support a whole path of CA certs, so we have to make a single file
+            // with all the *.pem files in that directory. It needs to be in filesystem so we can
+            // use it directly, let's put it in local cache for 10 minutes.
+            $cachefolder = make_localcache_directory('search_solr');
+            $prefix = 'capath.' . sha1($this->config->ssl_capath);
+            $now = \core\di::get(\core\clock::class)->time();
+            $got = false;
+            foreach (scandir($cachefolder) as $filename) {
+                // You are not allowed to overwrite files in localcache folders so we use files
+                // with the time in, and delete old files with a 1 minute delay to avoid race
+                // conditions.
+                if (preg_match('~^(.*)\.([0-9]+)$~', $filename, $matches)) {
+                    [1 => $fileprefix, 2 => $time] = $matches;
+                    $pathname = $cachefolder . '/' . $filename;
+                    if ($time > $now - self::CA_PATH_CACHE_TIME && $fileprefix === $prefix) {
+                        $options['verify'] = $pathname;
+                        $got = true;
+                        break;
+                    } else if ($time <= $now - self::CA_PATH_CACHE_TIME - self::CA_PATH_CACHE_DELETE_AFTER) {
+                        unlink($pathname);
+                    }
+                }
+            }
+
+            if (!$got) {
+                // If we don't have it yet, we need to make the cached file.
+                $allpems = '';
+                foreach (scandir($this->config->ssl_capath) as $filename) {
+                    if (preg_match('~\.pem$~', $filename)) {
+                        $pathname = $this->config->ssl_capath . '/' . $filename;
+                        $allpems .= file_get_contents($pathname) . "\n\n";
+                    }
+                }
+                $pathname = $cachefolder . '/' . $prefix . '.' . $now;
+                file_put_contents($pathname, $allpems);
+                $options['verify'] = $pathname;
+            }
+        }
+
+        // Apply other/overridden options.
+        foreach ($overrideoptions as $name => $value) {
+            $options[$name] = $value;
+        }
+
+        return $options;
+    }
+
     /**
      * Solr includes group support in the execute_query function.
      *

search/engine/solr/lang/en/search_solr.php

@@ -22,6 +22,11 @@
  * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  */
 
+$string['check_indextoobig'] = 'Index larger than specified size';
+$string['check_nocore'] = 'Cannot find index on Solr server';
+$string['check_nosize'] = 'Unable to determine index size on Solr server';
+$string['check_notconnected'] = 'Cannot connect to Solr server';
+$string['check_time'] = 'Server responded with status in {$a}';
 $string['connectionerror'] = 'The specified Solr server is not available or the specified index does not exist';
 $string['connectionsettings'] = 'Connection settings';
 $string['errorcreatingschema'] = 'Error creating the Solr schema: {$a}';
@@ -32,6 +37,9 @@ $string['fileindexing'] = 'Enable file indexing';
 $string['fileindexing_help'] = 'If your Solr install supports it, this feature allows Moodle to send files to be indexed.<br/>
 You will need to reindex all site contents after enabling this option for all files to be added.';
 $string['fileindexsettings'] = 'File indexing settings';
+$string['indexsize'] = 'The index is using {$a} on the Solr server.';
+$string['indexsizelimit'] = 'Index size limit';
+$string['indexsizelimit_desc'] = 'Shows an error on the status report page if the search index grows larger than this size (in bytes), and a warning if it exceeds 90%. 0 means no monitoring.';
 $string['maxindexfilekb'] = 'Maximum file size to index (kB)';
 $string['maxindexfilekb_help'] = 'Files larger than this number of kilobytes will not be included in search indexing. If set to zero, files of any size will be indexed.';
 $string['minimumsolr4'] = 'Solr 4.0 is the minimum version required for Moodle';

search/engine/solr/lib.php

@@ -0,0 +1,43 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Moodle API functions.
+ *
+ * @package search_solr
+ * @copyright 2024 The Open University
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+/**
+ * Gets status checks contributed by this plugin.
+ *
+ * If Solr is enabled and indexing is on, returns a check that the connection works.
+ *
+ * @return core\check\check[] Array of status checks
+ */
+function search_solr_status_checks(): array {
+    global $CFG;
+
+    // No checks if search engine is not set to Solr, or is disabled.
+    if (!\core_search\manager::is_indexing_enabled() || $CFG->searchengine !== 'solr') {
+        return [];
+    }
+
+    // Since it's turned on and set to Solr, configuration really should be OK and we ought to
+    // show if it isn't, so turn on the check.
+    return [new \search_solr\check\connection()];
+}

search/engine/solr/settings.php

@@ -49,6 +49,14 @@ if ($ADMIN->fulltree) {
             $settings->add(new admin_setting_configtext('search_solr/ssl_cainfo', new lang_string('solrsslcainfo', 'search_solr'), new lang_string('solrsslcainfo_desc', 'search_solr'), '', PARAM_RAW));
             $settings->add(new admin_setting_configtext('search_solr/ssl_capath', new lang_string('solrsslcapath', 'search_solr'), new lang_string('solrsslcapath_desc', 'search_solr'), '', PARAM_RAW));
 
+            $settings->add(new admin_setting_configtext(
+                'search_solr/indexsizelimit',
+                new lang_string('indexsizelimit', 'search_solr'),
+                new lang_string('indexsizelimit_desc', 'search_solr'),
+                0,
+                PARAM_INT,
+            ));
+
             $settings->add(new admin_setting_heading('search_solr_fileindexing',
                     new lang_string('fileindexsettings', 'search_solr'), ''));
             $settings->add(new admin_setting_configcheckbox('search_solr/fileindexing',

search/engine/solr/tests/engine_test.php

@@ -1471,6 +1471,19 @@ final class engine_test extends \advanced_testcase {
         $this->assertCount(1, $results);
     }
 
+    /**
+     * Tests that the get_status function works OK on the real server (there are more detailed
+     * tests for this function in {@see mock_engine_test}).
+     *
+     * @covers \search_solr\check\connection
+     */
+    public function test_get_status(): void {
+        $status = $this->engine->get_status(5);
+        $this->assertTrue($status['connected']);
+        $this->assertTrue($status['foundcore']);
+        $this->assertGreaterThan(0, $status['indexsize']);
+    }
+
     /**
      * Carries out a raw Solr query using the Solr basic query syntax.
      *

search/engine/solr/tests/mock_engine_test.php

@@ -0,0 +1,634 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace search_solr;
+
+/**
+ * Solr search engine unit tests that can operate using a mock http_client and without creating a
+ * search manager instance.
+ *
+ * These tests can run without the solr PHP extension.
+ *
+ * All 'realistic' tests of searching (e.g. index something then see if it is found by search)
+ * require a real Solr instance for testing and should be placed in {@see engine_test}.
+ * Tests that don't rely heavily on the real search functionality, or where we need to simulate
+ * multiple different ways of configuring the search infrastructure, or unusual failures in
+ * communication, may be better suited for this mock test approach.
+ *
+ * @package search_solr
+ * @category test
+ * @copyright 2024 The Open University
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ * @covers \search_solr\engine
+ */
+final class mock_engine_test extends \advanced_testcase {
+
+    protected function setUp(): void {
+        parent::setUp();
+        $this->resetAfterTest();
+
+        // Minimal configuration.
+        set_config('server_hostname', 'host.invalid', 'search_solr');
+        set_config('indexname', 'myindex', 'search_solr');
+
+        // This is not necessary on my setup but in GitHub Actions, the server_port is set to ''
+        // instead of default 8983.
+        set_config('server_port', '8983', 'search_solr');
+    }
+
+    /**
+     * Tests {@see engine::get_server_url}.
+     */
+    public function test_get_server_url(): void {
+        // Basic URL.
+        $engine = new engine();
+        $this->assertEquals(
+            'http://host.invalid:8983/solr/',
+            $engine->get_server_url('')->out(false),
+        );
+
+        // Same but with specified path.
+        $this->assertEquals(
+            'http://host.invalid:8983/solr/twiddle',
+            $engine->get_server_url('twiddle')->out(false),
+        );
+        // Slash at start of path will be stripped.
+        $this->assertEquals(
+            'http://host.invalid:8983/solr/twiddle',
+            $engine->get_server_url('/twiddle')->out(false),
+        );
+
+        // Turn on https. Due to the way the port setting works, which is bad, this will still have
+        // the default not-secure port (even though the 'default' on the setting page will now be
+        // shown as 8443, hmm). User has to change it manually.
+        set_config('secure', '1', 'search_solr');
+        $engine = new engine();
+        $this->assertEquals(
+            'https://host.invalid:8983/solr/',
+            $engine->get_server_url('')->out(false),
+        );
+
+        // Change port from default. User has to do this manually when enabling secure.
+        set_config('server_port', '8443', 'search_solr');
+        $engine = new engine();
+        $this->assertEquals(
+            'https://host.invalid:8443/solr/',
+            $engine->get_server_url('')->out(false),
+        );
+    }
+
+    /**
+     * Tests {@see engine::get_connection_url}.
+     */
+    public function test_get_connection_url(): void {
+        // Basic URL.
+        $engine = new engine();
+        $this->assertEquals(
+            'http://host.invalid:8983/solr/myindex/',
+            $engine->get_connection_url('')->out(false),
+        );
+    }
+
+    /**
+     * Tests {@see engine::raw_get_request()} with no auth settings.
+     */
+    public function test_raw_get_request_no_auth(): void {
+        $engine = new engine();
+
+        $response = $this->createStub(\Psr\Http\Message\ResponseInterface::class);
+
+        // When there is no auth, there aren't many options, just timeout.
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+        $mockedclient->expects($this->once())->method('get')->with(
+            'http://host.invalid:8983/solr/frog',
+            [
+                'connect_timeout' => 30,
+                'read_timeout' => 30,
+            ],
+        )->willReturn($response);
+        $this->assertEquals($response, $engine->raw_get_request('frog'));
+
+        // Timeout can be changed in config.
+        set_config('server_timeout', '10', 'search_solr');
+        $engine = new engine();
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+        $mockedclient->expects($this->once())->method('get')->with(
+            'http://host.invalid:8983/solr/frog',
+            [
+                'connect_timeout' => 10,
+                'read_timeout' => 10,
+            ],
+        )->willReturn($response);
+        $this->assertEquals($response, $engine->raw_get_request('frog'));
+    }
+
+    /**
+     * Tests {@see engine::raw_get_request()} with basic auth settings.
+     */
+    public function test_raw_get_request_basic_auth(): void {
+        set_config('server_username', 'u', 'search_solr');
+        set_config('server_password', 'p', 'search_solr');
+        $engine = new engine();
+
+        $response = $this->createStub(\Psr\Http\Message\ResponseInterface::class);
+
+        // Basic auth works with an 'auth' option.
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+        $mockedclient->expects($this->once())->method('get')->with(
+            'http://host.invalid:8983/solr/frog',
+            [
+                'connect_timeout' => 30,
+                'read_timeout' => 30,
+                'auth' => ['u', 'p'],
+            ],
+        )->willReturn($response);
+        $this->assertEquals($response, $engine->raw_get_request('frog'));
+    }
+
+    /**
+     * Tests {@see engine::raw_get_request()} with a supplied user certificate.
+     */
+    public function test_raw_get_request_user_cert(): void {
+        set_config('secure', '1', 'search_solr');
+        set_config('ssl_cert', '/tmp/cert.pem', 'search_solr');
+        $engine = new engine();
+
+        $response = $this->createStub(\Psr\Http\Message\ResponseInterface::class);
+
+        // User cert auth uses the 'cert' parameter, with or without a key.
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+        $mockedclient->expects($this->once())->method('get')->with(
+            'https://host.invalid:8983/solr/frog',
+            [
+                'connect_timeout' => 30,
+                'read_timeout' => 30,
+                'cert' => '/tmp/cert.pem',
+            ],
+        )->willReturn($response);
+        $this->assertEquals($response, $engine->raw_get_request('frog'));
+    }
+
+    /**
+     * Tests {@see engine::raw_get_request()} with a user key (with or without password).
+     */
+    public function test_raw_get_request_user_key(): void {
+        set_config('secure', '1', 'search_solr');
+        set_config('ssl_key', '/tmp/key.pem', 'search_solr');
+        $engine = new engine();
+
+        $response = $this->createStub(\Psr\Http\Message\ResponseInterface::class);
+
+        // User cert auth uses the 'cert' parameter, with or without a key.
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+        $mockedclient->expects($this->once())->method('get')->with(
+            'https://host.invalid:8983/solr/frog',
+            [
+                'connect_timeout' => 30,
+                'read_timeout' => 30,
+                'ssl_key' => '/tmp/key.pem',
+            ],
+        )->willReturn($response);
+        $this->assertEquals($response, $engine->raw_get_request('frog'));
+
+        set_config('ssl_keypassword', 'frog', 'search_solr');
+        $engine = new engine();
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+        $mockedclient->expects($this->once())->method('get')->with(
+            'https://host.invalid:8983/solr/frog',
+            [
+                'connect_timeout' => 30,
+                'read_timeout' => 30,
+                'ssl_key' => ['/tmp/key.pem', 'frog'],
+            ],
+        )->willReturn($response);
+        $this->assertEquals($response, $engine->raw_get_request('frog'));
+    }
+
+    /**
+     * Tests {@see engine::raw_get_request()} with a certificate bundle for verifying the server.
+     */
+    public function test_raw_get_request_certificate_bundle(): void {
+        set_config('secure', '1', 'search_solr');
+        set_config('ssl_cainfo', '/tmp/allthecerts.pem', 'search_solr');
+        $engine = new engine();
+
+        $response = $this->createStub(\Psr\Http\Message\ResponseInterface::class);
+
+        // User cert auth uses the 'cert' parameter, with or without a key.
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+        $mockedclient->expects($this->once())->method('get')->with(
+            'https://host.invalid:8983/solr/frog',
+            [
+                'connect_timeout' => 30,
+                'read_timeout' => 30,
+                'verify' => '/tmp/allthecerts.pem',
+            ],
+        )->willReturn($response);
+        $this->assertEquals($response, $engine->raw_get_request('frog'));
+    }
+
+    /**
+     * Tests {@see engine::raw_get_request()} with a certificate folder for verifying the server.
+     * Guzzle doesn't support a certificate folder (curl does) so this code makes a bundle in the
+     * localcache area.
+     */
+    public function test_raw_get_request_certificate_folder(): void {
+        global $CFG;
+
+        // Make a directory full of fake .pem files.
+        $temp = make_request_directory();
+        file_put_contents($temp . '/0.pem', "PEM0\n");
+        file_put_contents($temp . '/1.pem', "PEM1\n");
+        file_put_contents($temp . '/2.txt', "TXT2\n");
+
+        set_config('secure', '1', 'search_solr');
+        set_config('ssl_capath', $temp, 'search_solr');
+
+        $response = $this->createStub(\Psr\Http\Message\ResponseInterface::class);
+
+        // Party like it's 13 February 2009.
+        $time = 1234567890;
+        $this->mock_clock_with_frozen($time);
+
+        // User cert auth uses the 'cert' parameter, with or without a key.
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+
+        // The filename is the hash of the capath setting plus current time.
+        $combinedfile = $CFG->dataroot .
+            '/localcache/search_solr/capath.' .
+            sha1($temp) .
+            '.1234567890';
+
+        $mockedclient->expects($this->once())->method('get')->with(
+            'https://host.invalid:8983/solr/frog',
+            [
+                'connect_timeout' => 30,
+                'read_timeout' => 30,
+                'verify' => $combinedfile,
+            ],
+        )->willReturn($response);
+        $engine = new engine();
+        $this->assertEquals($response, $engine->raw_get_request('frog'));
+
+        // Check the file actually is the .pem files concatenated.
+        $this->assertEquals("PEM0\n\n\nPEM1\n\n\n", file_get_contents($combinedfile));
+
+        // Let's add another .pem file.
+        file_put_contents($temp . '/3.pem', "PEM3\n");
+
+        // 9 minutes 59 seconds later, it will still use the cached version (same file).
+        $time += 599;
+        $this->mock_clock_with_frozen($time);
+
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+        $mockedclient->expects($this->once())->method('get')->with(
+            'https://host.invalid:8983/solr/frog',
+            [
+                'connect_timeout' => 30,
+                'read_timeout' => 30,
+                'verify' => $combinedfile,
+            ],
+        )->willReturn($response);
+        $engine = new engine();
+        $this->assertEquals($response, $engine->raw_get_request('frog'));
+
+        $this->assertEquals("PEM0\n\n\nPEM1\n\n\n", file_get_contents($combinedfile));
+
+        // 10 minutes later, it will make a new cached version.
+        $time += 1;
+        $this->mock_clock_with_frozen($time);
+
+        $combinedfile2 = $CFG->dataroot .
+            '/localcache/search_solr/capath.' .
+            sha1($temp) .
+            '.1234568490';
+
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+        $mockedclient->expects($this->once())->method('get')->with(
+            'https://host.invalid:8983/solr/frog',
+            [
+                'connect_timeout' => 30,
+                'read_timeout' => 30,
+                'verify' => $combinedfile2,
+            ],
+        )->willReturn($response);
+        $engine = new engine();
+        $this->assertEquals($response, $engine->raw_get_request('frog'));
+
+        $this->assertEquals("PEM0\n\n\nPEM1\n\n\nPEM3\n\n\n", file_get_contents($combinedfile2));
+
+        // The old file is still there.
+        $this->assertEquals("PEM0\n\n\nPEM1\n\n\n", file_get_contents($combinedfile));
+
+        // Go another minute. We're still using the same combined file...
+        $time += 60;
+        $this->mock_clock_with_frozen($time);
+
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+        $mockedclient->expects($this->once())->method('get')->with(
+            'https://host.invalid:8983/solr/frog',
+            [
+                'connect_timeout' => 30,
+                'read_timeout' => 30,
+                'verify' => $combinedfile2,
+            ],
+        )->willReturn($response);
+        $engine = new engine();
+        $this->assertEquals($response, $engine->raw_get_request('frog'));
+
+        $this->assertEquals("PEM0\n\n\nPEM1\n\n\nPEM3\n\n\n", file_get_contents($combinedfile2));
+
+        // But now it will delete the old one.
+        $this->assertFalse(file_exists($combinedfile));
+    }
+
+    /**
+     * Tests the {@see engine::get_status()} function when there is an exception connecting.
+     */
+    public function test_get_status_exception_connecting(): void {
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+        $mockedclient->expects($this->once())->method('get')->with(
+            'http://host.invalid:8983/solr/admin/cores',
+            [
+                'connect_timeout' => 30,
+                'read_timeout' => 30,
+            ],
+        )->willThrowException(new \coding_exception('ex'));
+        $engine = new engine();
+        $status = $engine->get_status();
+        $this->assertFalse($status['connected']);
+        $this->assertFalse($status['foundcore']);
+        $this->assertEquals(
+            'Exception occurred: Coding error detected, it must be fixed by a programmer: ex',
+            $status['error'],
+        );
+        $this->assertInstanceOf(\coding_exception::class, $status['exception']);
+    }
+
+    /**
+     * Tests the {@see engine::get_status()} function when the server returns 404.
+     */
+    public function test_get_status_bad_http_status(): void {
+        $response = $this->createStub(\Psr\Http\Message\ResponseInterface::class);
+        $response->method('getStatusCode')->willReturn(404);
+
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+        $mockedclient->expects($this->once())->method('get')->with(
+            'http://host.invalid:8983/solr/admin/cores',
+            [
+                'connect_timeout' => 30,
+                'read_timeout' => 30,
+            ],
+        )->willReturn($response);
+        $engine = new engine();
+        $status = $engine->get_status();
+        $this->assertFalse($status['connected']);
+        $this->assertFalse($status['foundcore']);
+        $this->assertEquals('Unsuccessful status code: 404', $status['error']);
+    }
+
+    /**
+     * Creates a mock ResponseInterface with a body containing the specified string.
+     *
+     * @param string $body Body content
+     * @return \Psr\Http\Message\ResponseInterface Interface
+     */
+    protected function get_fake_response(string $body): \Psr\Http\Message\ResponseInterface {
+        $response = $this->createStub(\Psr\Http\Message\ResponseInterface::class);
+        $response->method('getStatusCode')->willReturn(200);
+        $stream = $this->createStub(\Psr\Http\Message\StreamInterface::class);
+        $response->method('getBody')->willReturn($stream);
+        $stream->method('getContents')->willReturn($body);
+        return $response;
+    }
+
+    /**
+     * Tests the {@see engine::get_status()} function when the server returns invalid JSON.
+     * In real life this would only be likely to happen if the server is down and a load balancer
+     * in front of it for some crazy reason interposes a page with status 200.
+     */
+    public function test_get_status_not_json(): void {
+        $response = $this->get_fake_response('notjson');
+
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+        $mockedclient->expects($this->once())->method('get')->with(
+            'http://host.invalid:8983/solr/admin/cores',
+            [
+                'connect_timeout' => 30,
+                'read_timeout' => 30,
+            ],
+        )->willReturn($response);
+        $engine = new engine();
+        $status = $engine->get_status();
+        $this->assertFalse($status['connected']);
+        $this->assertFalse($status['foundcore']);
+        $this->assertEquals('Invalid JSON', $status['error']);
+    }
+
+    /**
+     * Tests the {@see engine::get_status()} function when the server returns an empty response.
+     *
+     * This could maybe happen if the server has been configured, but not fully initialised.
+     */
+    public function test_get_status_no_cores(): void {
+        $response = $this->get_fake_response('{}');
+
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+        $mockedclient->expects($this->once())->method('get')->with(
+            'http://host.invalid:8983/solr/admin/cores',
+            [
+                'connect_timeout' => 30,
+                'read_timeout' => 30,
+            ],
+        )->willReturn($response);
+        $engine = new engine();
+        $status = $engine->get_status();
+        $this->assertTrue($status['connected']);
+        $this->assertFalse($status['foundcore']);
+        $this->assertEquals('Unexpected JSON: no core status', $status['error']);
+    }
+
+    /**
+     * Tests the {@see engine::get_status()} function when the server returns a core without a name
+     * we can read.
+     *
+     * In real usage this should only happen if the Solr REST interface changes unexpectedly.
+     */
+    public function test_get_status_core_no_name(): void {
+        // A core with no name (in its 'name' field, the 'frog' key is ignored).
+        $response = $this->get_fake_response('{"status":{"frog":{}}}');
+
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+        $mockedclient->expects($this->once())->method('get')->with(
+            'http://host.invalid:8983/solr/admin/cores',
+            [
+                'connect_timeout' => 30,
+                'read_timeout' => 30,
+            ],
+        )->willReturn($response);
+        $engine = new engine();
+        $status = $engine->get_status();
+        $this->assertTrue($status['connected']);
+        $this->assertFalse($status['foundcore']);
+        $this->assertEquals('Unexpected JSON: core has no name', $status['error']);
+    }
+
+    /**
+     * Tests the {@see engine::get_status()} function when the server doesn't return status for a
+     * core that matches the index name in Moodle config.
+     *
+     * In real usage this could happen if the index got wiped from search or something.
+     */
+    public function test_get_status_no_matching_core(): void {
+        // Core is not the one we're looking for.
+        $response = $this->get_fake_response('{"status":{"frog":{"name":"frog"}}}');
+
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+        $mockedclient->expects($this->once())->method('get')->with(
+            'http://host.invalid:8983/solr/admin/cores',
+            [
+                'connect_timeout' => 30,
+                'read_timeout' => 30,
+            ],
+        )->willReturn($response);
+        $engine = new engine();
+        $status = $engine->get_status();
+        $this->assertTrue($status['connected']);
+        $this->assertFalse($status['foundcore']);
+        $this->assertEquals('Could not find core matching myindex', $status['error']);
+    }
+
+    /**
+     * Tests the {@see engine::get_status()} function when the server returns a core without index
+     * information.
+     *
+     * In real usage this should only happen if the Solr REST interface changes unexpectedly. There
+     * is a parameter to not receive index information, but we don't use it.
+     */
+    public function test_get_status_core_no_index(): void {
+        // Core exists but has no index object.
+        $response = $this->get_fake_response('{"status":{"myindex":{"name":"myindex"}}}');
+
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+        $mockedclient->expects($this->once())->method('get')->with(
+            'http://host.invalid:8983/solr/admin/cores',
+            [
+                'connect_timeout' => 30,
+                'read_timeout' => 30,
+            ],
+        )->willReturn($response);
+        $engine = new engine();
+        $status = $engine->get_status();
+        $this->assertTrue($status['connected']);
+        $this->assertTrue($status['foundcore']);
+        $this->assertEquals('Unexpected JSON: core has no index', $status['error']);
+    }
+
+    /**
+     * Tests the {@see engine::get_status()} function when the server returns index information
+     * without size.
+     *
+     * In real usage this should only happen if the Solr REST interface changes unexpectedly.
+     */
+    public function test_get_status_core_index_no_size(): void {
+        // Core index objects doesn't have a size.
+        $response = $this->get_fake_response('{"status":{"myindex":{"name":"myindex","index":{}}}}');
+
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+        $mockedclient->expects($this->once())->method('get')->with(
+            'http://host.invalid:8983/solr/admin/cores',
+            [
+                'connect_timeout' => 30,
+                'read_timeout' => 30,
+            ],
+        )->willReturn($response);
+        $engine = new engine();
+        $status = $engine->get_status();
+        $this->assertTrue($status['connected']);
+        $this->assertTrue($status['foundcore']);
+        $this->assertEquals('Unexpected JSON: core index has no sizeInBytes', $status['error']);
+    }
+
+    /**
+     * Tests the {@see engine::get_status()} function when all desired data is present, using a
+     * single-instance Solr configuration.
+     */
+    public function test_get_status_success_single_server(): void {
+        // Core index complete with size.
+        $response = $this->get_fake_response('{"status":{"myindex":{"name":"myindex",' .
+            '"index":{"sizeInBytes":123}}}}');
+
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+        $mockedclient->expects($this->once())->method('get')->with(
+            'http://host.invalid:8983/solr/admin/cores',
+            [
+                'connect_timeout' => 30,
+                'read_timeout' => 30,
+            ],
+        )->willReturn($response);
+        $engine = new engine();
+        $status = $engine->get_status();
+        $this->assertTrue($status['connected']);
+        $this->assertTrue($status['foundcore']);
+        $this->assertEquals(123, $status['indexsize']);
+    }
+
+    /**
+     * Tests the {@see engine::get_status()} function when all desired data is present, using a
+     * multiple-instance (SolrCloud) configuration.
+     */
+    public function test_get_status_success_solr_cloud(): void {
+        // Index with size, in cloud replica. These have a different name for each node but a
+        // 'collection' field with the original index name.
+        $response = $this->get_fake_response('{"status":{"replica1":{"name":"replica1",' .
+            '"cloud":{"collection":"myindex"},"index":{"sizeInBytes":123}}}}');
+
+        $mockedclient = $this->createMock(\core\http_client::class);
+        \core\di::set(\core\http_client::class, $mockedclient);
+        $mockedclient->expects($this->once())->method('get')->with(
+            'http://host.invalid:8983/solr/admin/cores',
+            [
+                'connect_timeout' => 30,
+                'read_timeout' => 30,
+            ],
+        )->willReturn($response);
+        $engine = new engine();
+        $status = $engine->get_status();
+        $this->assertTrue($status['connected']);
+        $this->assertTrue($status['foundcore']);
+        $this->assertEquals(123, $status['indexsize']);
+    }
+}