File size: 2,669 Bytes
7db401b 1fc8c4c 7db401b 1fc8c4c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 |
from openai import OpenAI
import anthropic
from together import Together
import os
from atla import Atla
from dotenv import load_dotenv
from .prompts import (
JUDGE_SYSTEM_PROMPT
)
load_dotenv()
# Initialize clients
anthropic_client = anthropic.Anthropic()
openai_client = OpenAI()
together_client = Together()
hf_api_key = os.getenv("HF_API_KEY")
atla_client = Atla()
def get_openai_response(model_name, prompt, system_prompt=JUDGE_SYSTEM_PROMPT, max_tokens=500, temperature=0):
"""Get response from OpenAI API"""
try:
response = openai_client.chat.completions.create(
model=model_name,
messages=[
{"role": "system", "content": system_prompt},
{"role": "user", "content": prompt},
],
max_completion_tokens=max_tokens,
temperature=temperature,
)
return response.choices[0].message.content
except Exception as e:
return f"Error with OpenAI model {model_name}: {str(e)}"
def get_anthropic_response(model_name, prompt, system_prompt=JUDGE_SYSTEM_PROMPT, max_tokens=500, temperature=0):
"""Get response from Anthropic API"""
try:
response = anthropic_client.messages.create(
model=model_name,
max_tokens=max_tokens,
temperature=temperature,
system=system_prompt,
messages=[{"role": "user", "content": [{"type": "text", "text": prompt}]}],
)
return response.content[0].text
except Exception as e:
return f"Error with Anthropic model {model_name}: {str(e)}"
def get_atla_response(model_name, prompt, system_prompt=None, max_tokens=500, temperature=0.01):
"""Get response from Atla API"""
try:
# Extract components from the prompt data
model_input = prompt.get('human_input', '')
model_output = prompt.get('ai_response', '')
expected_output = prompt.get('ground_truth')
evaluation_criteria = prompt.get('eval_criteria', '')
response = atla_client.evaluation.create(
model_id=model_name, # Will be either "atla-selene" or "atla-selene-mini"
model_input=model_input,
model_output=model_output,
expected_model_output=expected_output if expected_output else None,
evaluation_criteria=evaluation_criteria,
)
# Return the score and critique directly
return {
"score": response.result.evaluation.score,
"critique": response.result.evaluation.critique
}
except Exception as e:
return f"Error with Atla model {model_name}: {str(e)}" |