【译】轻松评估 AI 应用程序的质量

dotnet add package Microsoft.Extensions.AI.Evaluation
dotnet add package Microsoft.Extensions.AI.Evaluation.Quality
dotnet add package Microsoft.Extensions.AI.Evaluation.Reporting

static ReportingConfiguration GetReportingConfiguration()
{
    // Setup and configure the evaluators you would like to utilize for each AI chat.
    // AnswerScoringEvaluator is an example of a custom evaluator that can be added, while the others
    // are included in the evaluation library.

    // Measures the extent to which the model's generated responses are pertinent and directly related to the given queries.
    IEvaluatorrtcEvaluator=
        newRelevanceTruthAndCompletenessEvaluator(
            newRelevanceTruthAndCompletenessEvaluator.Options(includeReasoning: true));
    // Measures how well the language model can produce output that flows smoothly, reads naturally, and resembles human-like language.
    IEvaluatorcoherenceEvaluator=newCoherenceEvaluator();
    // Measures the grammatical proficiency of a generative AI's predicted answer.
    IEvaluatorfluencyEvaluator=newFluencyEvaluator();
    // Measures how well the model's generated answers align with information from the source data
    IEvaluatorgroundednessEvaluator=newGroundednessEvaluator();
    // Measures the extent to which the model's retrieved documents are pertinent and directly related to the given queries.
    IEvaluatoranswerScoringEvaluator=newAnswerScoringEvaluator();

    varendpoint=newUri(AzureOpenAIEndpoint);
    varoaioptions=newAzureOpenAIClientOptions();
    varazureClient=newAzureOpenAIClient(endpoint, newDefaultAzureCredential(), oaioptions);

    // Setup the chat client that is used to perform the evaluations
    IChatClientchatClient= azureClient.AsChatClient(AzureOpenAIDeploymentName);
    Tokenizertokenizer= TiktokenTokenizer.CreateForModel(AzureOpenAIModelName);

    varchatConfig=newChatConfiguration(chatClient, tokenizer.ToTokenCounter(inputTokenLimit: ));

    // The DiskBasedReportingConfiguration caches LLM responses to reduce costs and
    // increase test run performance.
    return DiskBasedReportingConfiguration.Create(
            storageRootPath: StorageRootPath,
            chatConfiguration: chatConfig,
            evaluators: [
                rtcEvaluator,
                coherenceEvaluator,
                fluencyEvaluator,
                groundednessEvaluator,
                answerScoringEvaluator],
            executionName: ExecutionName);
}

private static async Task EvaluateQuestion(EvalQuestion question, ReportingConfiguration reportingConfiguration, CancellationToken cancellationToken)
{
    // Create a new ScenarioRun to represent each evaluation run.
    awaitusing ScenarioRun scenario = await reportingConfiguration.CreateScenarioRunAsync($"Question_{question.QuestionId}", cancellationToken: cancellationToken);

    // Run the sample through the assistant to generate a response.
    var responseItems = await backend!.AssistantChatAsync(new AssistantChatRequest(
        question.ProductId,
        null,
        null,
        null,
        [new() { IsAssistant = true, Text = question.Question }]),
        cancellationToken);

    var answerBuilder = new StringBuilder();
    awaitforeach (var item in responseItems)
    {
        if (item.Type == AssistantChatReplyItemType.AnswerChunk)
        {
            answerBuilder.Append(item.Text);
        }
    }

    var finalAnswer = answerBuilder.ToString();

    // Invoke the evaluators
    EvaluationResult evalResult = await scenario.EvaluateAsync(
        [new ChatMessage(ChatRole.User, question.Question)],
        new ChatMessage(ChatRole.Assistant, finalAnswer),
        additionalContext: [new AnswerScoringEvaluator.Context(question.Answer)],
        cancellationToken);

    // Assert that the evaluator was able to successfully generate an analysis
    Assert.False(evalResult.Metrics.Values.Any(m => m.Interpretation?.Rating == EvaluationRating.Inconclusive), "Model response was inconclusive");

    // Assert that the evaluators did not report any diagnostic errors
    Assert.False(evalResult.ContainsDiagnostics(d => d.Severity == EvaluationDiagnosticSeverity.Error), "Evaluation had errors.");

}

[Fact]
public async Task EvaluateQuestion_Summit3000TrekkingBackpackStrapAdjustment()
{
    // This is the example question and answer that will be evaluated.
    var question = new EvalQuestion
    {
        QuestionId = ,
        ProductId = ,
        Question = "Hi there, I recently purchased the Summit 3000 Trekking Backpack and I\u0027m having issues with the strap adjustment. Can you provide me with the specified torque value for the strap adjustment bolts?",
        Answer = "15-20 Nm"
    };

    // Construct a reporting configuration to support the evaluation
    var reportingConfiguration = GetReportingConfiguration();

    // Run an evaluation pass and record the results to the cache folder
    await EvaluateQuestion(question, reportingConfiguration, , CancellationToken.None);
}

dotnet tool install Microsoft.Extensions.AI.Evaluation.Console
dotnet aieval report --path <path\to\my\cache\storage> --output report.html