moodle/moodle
3
0
Fork 0
mirror of https://github.com/moodle/moodle.git synced 2025-08-09 02:46:40 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

MDL-83154 hub: Add model to AI usage data

Browse source
This commit is contained in:
David Woloszyn 2024-09-26 15:13:05 +10:00
parent dd05af9259
commit e4b39e764d
2 changed files with 27 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

16
lib/classes/hub/registration.php
View file

@ -756,6 +756,7 @@ class registration {
'fail_count' => 0, 'fail_count' => 0,
'times' => [], 'times' => [],
'errors' => [], 'errors' => [],
'modelstemp' => [],
]; ];
} }
@ -769,9 +770,13 @@ class registration {
// Collect errors for determing the predominant one. // Collect errors for determing the predominant one.
$data[$provider][$actionname]['errors'][] = $action->errorcode; $data[$provider][$actionname]['errors'][] = $action->errorcode;
} }
// Collect models used and identify unknown ones.
$model = $action->model ?? 'unknown';
$data[$provider][$actionname]['modelstemp'][] = $model;
} }
// Parse the errors and everage the times, then add them to the data. // Parse the errors, average the times, count the models and then add them to the data.
foreach ($data as $p => $provider) { foreach ($data as $p => $provider) {
foreach ($provider as $a => $actionname) { foreach ($provider as $a => $actionname) {
if (isset($data[$p][$a]['errors'])) { if (isset($data[$p][$a]['errors'])) {
@ -796,6 +801,15 @@ class registration {
} }
} }
unset($data[$p][$a]['times']); unset($data[$p][$a]['times']);
if (isset($data[$p][$a]['modelstemp'])) {
// Create an array with the models counted.
$countedmodels = array_count_values($data[$p][$a]['modelstemp']);
foreach ($countedmodels as $model => $count) {
$data[$p][$a]['models'][$model]['count'] = $count;
}
}
unset($data[$p][$a]['modelstemp']);
} }
} }

14
lib/tests/hub/registration_test.php
View file

@ -94,6 +94,9 @@ class registration_test extends \advanced_testcase {
$this->resetAfterTest(); $this->resetAfterTest();
$clock = $this->mock_clock_with_frozen(); $clock = $this->mock_clock_with_frozen();
$gpt4omodel = 'gpt-4o';
$dalle3model = 'dall-e-3';
// Record some generated text. // Record some generated text.
$record = new \stdClass(); $record = new \stdClass();
$record->provider = 'openai'; $record->provider = 'openai';
@ -104,15 +107,17 @@ class registration_test extends \advanced_testcase {
$record->success = true; $record->success = true;
$record->timecreated = $clock->time() - 5; $record->timecreated = $clock->time() - 5;
$record->timecompleted = $clock->time(); $record->timecompleted = $clock->time();
$record->model = $gpt4omodel;
$DB->insert_record('ai_action_register', $record); $DB->insert_record('ai_action_register', $record);
// Record a generated image. // Record a generated image.
$record->actionname = 'generate_image'; $record->actionname = 'generate_image';
$record->actionid = 2; $record->actionid = 111;
$record->timecreated = $clock->time() - 20; $record->timecreated = $clock->time() - 20;
$record->model = $dalle3model;
$DB->insert_record('ai_action_register', $record); $DB->insert_record('ai_action_register', $record);
// Record another image. // Record another image.
$record->actionid = 3; $record->actionid = 222;
$record->timecreated = $clock->time() - 10; $record->timecreated = $clock->time() - 10;
$DB->insert_record('ai_action_register', $record); $DB->insert_record('ai_action_register', $record);
@ -121,6 +126,7 @@ class registration_test extends \advanced_testcase {
$record->actionid = 4; $record->actionid = 4;
$record->success = false; $record->success = false;
$record->errorcode = 403; $record->errorcode = 403;
$record->model = null;
$DB->insert_record('ai_action_register', $record); $DB->insert_record('ai_action_register', $record);
$record->actionid = 5; $record->actionid = 5;
$record->errorcode = 403; $record->errorcode = 403;
@ -143,5 +149,9 @@ class registration_test extends \advanced_testcase {
// Check time range is set correctly. // Check time range is set correctly.
$this->assertEquals($clock->time() - WEEKSECS, $aisuage->time_range->timefrom); $this->assertEquals($clock->time() - WEEKSECS, $aisuage->time_range->timefrom);
$this->assertEquals($clock->time(), $aisuage->time_range->timeto); $this->assertEquals($clock->time(), $aisuage->time_range->timeto);
// Check model counts.
$this->assertEquals(1, $aisuage->openai->generate_text->models->{$gpt4omodel}->count);
$this->assertEquals(2, $aisuage->openai->generate_image->models->{$dalle3model}->count);
$this->assertEquals(3, $aisuage->openai->generate_image->models->unknown->count);
} }
} }