moodle/moodle
3
0
Fork 0
mirror of https://github.com/moodle/moodle.git synced 2025-08-02 07:39:54 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

MDL-85106 AI: Fix AI usage reporting of explain text tokens

Browse source
This commit is contained in:
David Woloszyn 2025-04-04 14:30:33 +11:00
parent b2b5e32058
commit 378e168d46
3 changed files with 69 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

26
ai/classes/reportbuilder/local/entities/ai_action_register.php
View file

@ -80,6 +80,7 @@ class ai_action_register extends base {
$mainalias = $this->get_table_alias('ai_action_register');
$generatetextalias = 'aagt';
$summarisetextalias = 'aast';
$explaintextalias = 'aaet';
// Action name column.
$columns[] = (new column(
@ -139,7 +140,7 @@ class ai_action_register extends base {
->add_callback([format::class, 'userdate']);
// Prompt tokens column.
// Only available for summarise_text and generate_text actions.
// Only available for summarise_text, generate_text actions and explain_text actions.
$columns[] = (new column(
'prompttokens',
new lang_string('prompttokens', 'core_ai'),
@ -154,8 +155,13 @@ class ai_action_register extends base {
LEFT JOIN {ai_action_summarise_text} {$summarisetextalias}
ON {$mainalias}.actionid = {$summarisetextalias}.id
AND {$mainalias}.actionname = 'summarise_text'")
->add_join("
LEFT JOIN {ai_action_explain_text} {$explaintextalias}
ON {$mainalias}.actionid = {$explaintextalias}.id
AND {$mainalias}.actionname = 'explain_text'")
->set_type(column::TYPE_INTEGER)
->add_field("COALESCE({$generatetextalias}.prompttokens, {$summarisetextalias}.prompttokens)", 'prompttokens')
->add_field("COALESCE({$generatetextalias}.prompttokens, {$summarisetextalias}.prompttokens,
{$explaintextalias}.prompttokens)", 'prompttokens')
->set_is_sortable(true)
->set_help_icon(new help_icon('prompttokens', 'core_ai'))
->add_callback(static function(?int $value): string {
@ -163,7 +169,7 @@ class ai_action_register extends base {
});
// Completion tokens column.
// Only available for summarise_text and generate_text actions.
// Only available for summarise_text, generate_text actions and explain_text actions.
$columns[] = (new column(
'completiontokens',
new lang_string('completiontokens', 'core_ai'),
@ -178,8 +184,13 @@ class ai_action_register extends base {
LEFT JOIN {ai_action_summarise_text} {$summarisetextalias}
ON {$mainalias}.actionid = {$summarisetextalias}.id
AND {$mainalias}.actionname = 'summarise_text'")
->add_join("
LEFT JOIN {ai_action_explain_text} {$explaintextalias}
ON {$mainalias}.actionid = {$explaintextalias}.id
AND {$mainalias}.actionname = 'explain_text'")
->set_type(column::TYPE_INTEGER)
->add_field("COALESCE({$generatetextalias}.completiontoken, {$summarisetextalias}.completiontoken)", 'completiontokens')
->add_field("COALESCE({$generatetextalias}.completiontoken, {$summarisetextalias}.completiontoken,
{$explaintextalias}.completiontoken)", 'completiontokens')
->set_is_sortable(true)
->set_help_icon(new help_icon('completiontokens', 'core_ai'))
->add_callback(static function(?int $value): string {
@ -198,6 +209,7 @@ class ai_action_register extends base {
$mainalias = $this->get_table_alias('ai_action_register');
$generatetextalias = 'aagt';
$summarisetextalias = 'aast';
$explaintextalias = 'aaet';
// Action name filter.
$filters[] = (new filter(
@ -258,7 +270,8 @@ class ai_action_register extends base {
'prompttokens',
new lang_string('prompttokens', 'core_ai'),
$this->get_entity_name(),
"COALESCE({$generatetextalias}.prompttokens, {$summarisetextalias}.prompttokens)",
"COALESCE({$generatetextalias}.prompttokens, {$summarisetextalias}.prompttokens,
{$explaintextalias}.prompttokens)",
))
->add_joins($this->get_joins());
@ -268,7 +281,8 @@ class ai_action_register extends base {
'completiontokens',
new lang_string('completiontokens', 'core_ai'),
$this->get_entity_name(),
"COALESCE({$generatetextalias}.completiontoken, {$summarisetextalias}.completiontoken)",
"COALESCE({$generatetextalias}.completiontoken, {$summarisetextalias}.completiontoken,
{$explaintextalias}.completiontoken)",
))
->add_joins($this->get_joins());

29
ai/tests/behat/ai_usage_report.feature
View file

@ -17,19 +17,21 @@ Feature: AI usage report displays recorded AI data
| provider | name | enabled | apikey | orgid |
| aiprovider_openai | OpenAI API test | 1 | 123 | abc |
And the following "core_ai > ai actions" exist:
| actionname | user | success | provider | contextid |
| generate_text | student1 | 1 | aiprovider_openai | 1 |
| summarise_text | student1 | 0 | aiprovider_openai | 1 |
| generate_image | student2 | 1 | aiprovider_azureai | 1 |
| actionname | user | success | provider | contextid | prompttokens | completiontokens |
| generate_text | student1 | 1 | aiprovider_openai | 1 | 22 | 33 |
| summarise_text | student1 | 0 | aiprovider_openai | 1 | | |
| explain_text | student1 | 1 | aiprovider_openai | 1 | 44 | 55 |
| generate_image | student2 | 1 | aiprovider_azureai | 1 | | |
Scenario: Managers can view the AI usage report
Given I am logged in as "manager1"
When I navigate to "Reports > AI reports > AI usage" in site administration
Then the following should exist in the "AI usage" table:
| Action | First name | Provider | Success |
| Generate text | Student One | OpenAI API provider | Yes |
| Summarise text | Student One | OpenAI API provider | No |
| Generate image | Student Two | Azure AI API provider | Yes |
| Action | First name | Provider | Success | Prompt tokens | Completion tokens |
| Generate text | Student One | OpenAI API provider | Yes | 22 | 33 |
| Summarise text | Student One | OpenAI API provider | No | | |
| Explain text | Student One | OpenAI API provider | Yes | 44 | 55 |
| Generate image | Student Two | Azure AI API provider | Yes | | |
@javascript
Scenario: Managers can filter the AI usage report
@ -39,16 +41,17 @@ Feature: AI usage report displays recorded AI data
And I set the field "Provider value" in the "Provider" "core_reportbuilder > Filter" to "OpenAI API provider"
And I click on "Apply" "button" in the "[data-region='report-filters']" "css_element"
Then the following should exist in the "AI usage" table:
| Action | First name | Provider | Success |
| Generate text | Student One | OpenAI API provider | Yes |
| Summarise text | Student One | OpenAI API provider | No |
| Action | First name | Provider | Success | Prompt tokens | Completion tokens |
| Generate text | Student One | OpenAI API provider | Yes | 22 | 33 |
| Summarise text | Student One | OpenAI API provider | No | | |
| Explain text | Student One | OpenAI API provider | Yes | 44 | 55 |
And I should not see "Azure AI API provider" in the "AI usage" "table"
And I set the following fields in the "Action" "core_reportbuilder > Filter" to these values:
| Action operator | Is equal to |
| Action value | Generate text |
And I click on "Apply" "button" in the "[data-region='report-filters']" "css_element"
And the following should exist in the "AI usage" table:
| Action | First name | Provider | Success |
| Generate text | Student One | OpenAI API provider | Yes |
| Action | First name | Provider | Success | Prompt tokens | Completion tokens |
| Generate text | Student One | OpenAI API provider | Yes | 22 | 33 |
And I should not see "Summarise text" in the "AI usage" "table"
And I should not see "Azure AI API Provider" in the "AI usage" "table"

68
ai/tests/generator/lib.php
View file

@ -47,45 +47,43 @@ class core_ai_generator extends component_generator_base {
throw new Exception('\'ai actions\' requires the field \'provider\' to be specified');
}
$action = new stdClass();
foreach ($data as $key => $value) {
// Add data to parent action record.
$action->$key = $value;
// Create the child action record.
$child = new stdClass();
$child->prompt = 'Prompt text';
// Create the child action record.
$child = new stdClass();
$child->prompt = 'Prompt text';
if ($key === 'actionname') {
// Generate image actions need to be structured differently.
if ($value === 'generate_image') {
$child->numberimages = 1;
$child->quality = 'hd';
$child->aspectratio = 'landscape';
$child->style = 'vivid';
$child->sourceurl = 'http://localhost/yourimage';
$child->revisedprompt = 'Revised prompt';
} else {
// Generate text (and variants).
$child->generatedcontent = 'Your generated content';
$child->prompttokens = 33;
$child->completiontoken = 44;
}
// Simulate an error.
if ($key === 'success' && $value == 0) {
$action->errorcode = 403;
$action->errormessage = 'Forbidden';
}
$childid = $DB->insert_record("ai_action_{$value}", $child);
}
// Generate image actions need to be structured differently.
if ($data['actionname'] === 'generate_image') {
$child->numberimages = 1;
$child->quality = 'hd';
$child->aspectratio = 'landscape';
$child->style = 'vivid';
$child->sourceurl = 'http://localhost/yourimage';
$child->revisedprompt = 'Revised prompt';
} else {
// Generate text (and variants).
$child->generatedcontent = 'Your generated content';
$child->prompttokens = $data['prompttokens'] ?? 111;
$child->completiontoken = $data['completiontokens'] ?? 222;
}
// Simulate an error.
if ($data['success'] == 0) {
$data['errorcode'] = 403;
$data['errormessage'] = 'Forbidden';
// Unset some values that won't be present with an error.
unset($child->generatedcontent);
unset($child->revisedprompt);
unset($child->prompttokens);
unset($child->completiontoken);
}
$childid = $DB->insert_record("ai_action_{$data['actionname']}", $child);
// Finalise some fields before inserting.
$action->actionid = $childid;
$action->timecreated = time();
$action->timecompleted = time() + 1;
$DB->insert_record('ai_action_register', $action);
$data['actionid'] = $childid;
$data['timecreated'] = time();
$data['timecompleted'] = time() + 1;
$DB->insert_record('ai_action_register', $data);
}
/**