anton_ego_jimmy.py

import streamlit as st
# Model Configuration
#[model]
TYPE = "chat" # so that it uses the chat history
SELECTED_MODEL = "meta-llama/llama-3-1-70b-instruct" # Pick a model_id, you can find them here https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-api-model-ids.html?context=wx&audience=wdp
#SELECTED_MODEL = "meta-llama/llama-3-405b-instruct"
#SELECTED_MODEL = "mistralai/mistral-large"
#SELECTED_MODEL = "mistralai/mixtral-8x7b-instruct-v01"
VERIFY = False

# Prompt Configuration
#[prompt]
SYSTEM_PROMPT = st.secrets["sys_prompt"]
PROMPT_TEMPLATE = "llama3-instruct (llama-3 & 3.1) - system"
#PROMPT_TEMPLATE = "mistral & mixtral v2 tokenizer - system"  # <pick prompt template from model_family_syntax below> For example "llama3-instruct (llama-3 & 3.1) - user" if you don't use a system prompt.
BAKE_IN_PROMPT_SYNTAX = True

# Generation Parameters
DECODING_METHOD = "greedy" # greedy or sample
MAX_NEW_TOKENS = 750
MIN_NEW_TOKENS = 1
REPETITION_PENALTY = 1.0
# LENGTH_PENALTY = {'decay_factor': 1.25, 'start_index': 150}
STOP_SEQUENCES = ["<|end_of_text|>","</s>"] # This one is set up for llama models, if you use mistral </s> is the preferred stop_sequence

# Additional Parameters - Only active if you pick sampling in decoding method
TEMPERATURE = 0.75
TOP_P = 1.0
TOP_K = 50

DISPLAY_CHAT_HISTORY = 1  # 0 to display chat history, 1 to not display chat history

# model_family_syntax = {
#     "llama3-instruct (llama-3 & 3.1) - system": """\n<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n{system_prompt}<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n{prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n""",
#     "llama3-instruct (llama-3 & 3.1) - user": """\n<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n""",
#     "granite-13b-chat & instruct - system": """\n<|system|>\n{system_prompt}\n<|user|>\n{prompt}\n<|assistant|>\n\n""",
#     "granite-13b-chat & instruct - user": """\n<|user|>\n{prompt}\n<|assistant|>\n\n""",
#     "llama2-chat - system": """\n[INST] <<SYS>>\n{system_prompt}\n<</SYS>>\n\n{prompt} [/INST] """,
#     "llama2-chat - user": """\n[INST] {prompt} [/INST] """,
#     "mistral & mixtral v2 tokenizer - system": """\n<s>[INST]System Prompt:[{system_prompt}]\n\n{prompt} [/INST]\n""",
#     "mistral & mixtral v2 tokenizer - system segmented": """\n<s>[INST]System Prompt:{system_prompt}[/INST][INST]{prompt} [/INST]\n""",
#     "mistral & mixtral v2 tokenizer - user": """\n<s>[INST]{prompt} [/INST]\n"""
# }