Spaces:
Running
Running
File size: 1,182 Bytes
1c58916 75ddbd7 1c58916 328de20 72647ee 7a02485 72647ee 1c58916 328de20 1c58916 328de20 1c58916 72647ee 1c58916 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 |
import requests
import json
messages = [
{"role": "user", "content": "helo"},
{"role": "assistant", "content": "Hello! How can I assist you today?"},
{"role": "user", "content": "who are you and give me a breif description of who you are"}
]
model = "meta-llama/llama-4-scout-17b-16e-instruct"
url = "http://127.0.0.1:8000/v1/generate"
payload = {
"messages": messages,
"model": model
}
response = requests.post(url, json=payload, stream=True)
if response.status_code == 200:
for line in response.iter_lines():
if line:
print(line)
decoded_line = line.decode('utf-8')
if decoded_line.startswith('data: '):
try:
# Remove 'data: ' prefix and parse JSON
json_data = json.loads(decoded_line[6:])
# Check if there are choices and text
if json_data["choices"] and "text" in json_data["choices"][0]:
print(json_data["choices"][0]["text"], end='')
except json.JSONDecodeError:
continue
else:
print(f"Request failed with status code {response.status_code}")
|